diff --git a/.github/actions/handle_docker/action.yml b/.github/actions/handle_docker/action.yml index a0aa99c1442d5b..f79bc06afa9f5c 100644 --- a/.github/actions/handle_docker/action.yml +++ b/.github/actions/handle_docker/action.yml @@ -27,17 +27,17 @@ runs: using: 'composite' steps: - name: Checkout head - uses: actions/checkout@v4 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Checkout base - uses: actions/checkout@v4 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: ref: ${{ github.base_ref || github.event.merge_group.base_ref }} sparse-checkout: ${{ inputs.dockerfiles_root_dir }}/docker_tag path: base - name: Install Python dependencies - uses: py-actions/py-dependency-install@v4 + uses: py-actions/py-dependency-install@30aa0023464ed4b5b116bd9fbdab87acf01a484e # v4.1.0 with: path: "${{ github.action_path }}/requirements.txt" update-setuptools: "false" @@ -45,7 +45,10 @@ runs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 + with: + driver-opts: | + image=${{ inputs.registry }}/dockerio/moby/buildkit:buildx-stable-1 - name: Handle docker images id: handle_images @@ -62,7 +65,7 @@ runs: --base_tag_file "base/${{ inputs.dockerfiles_root_dir }}/docker_tag" \ --docker_env_changed "${{ fromJSON(inputs.changed_components).docker_env }}" \ --dockerfiles_changed "${{ fromJSON(inputs.changed_components).dockerfiles }}" \ - --docker_builder "${{ steps.buildx.outputs.name}}" \ + --docker_builder "${{ steps.buildx.outputs.name }}" \ --repo "${{ github.repository }}" \ --ref_name "${{ github.ref_name }}" \ $([[ -n $pr ]] && echo "--pr $pr" || echo '-s ${{ github.sha }}') \ diff --git a/.github/actions/setup_python/action.yml b/.github/actions/setup_python/action.yml index 0d9138bc643d2a..80e22866170535 100644 --- a/.github/actions/setup_python/action.yml +++ b/.github/actions/setup_python/action.yml @@ -56,7 +56,7 @@ runs: - if: ${{ runner.os == 'macOS' || runner.os == 'Windows' || (runner.os == 'Linux' && runner.arch != 'ARM64' && steps.check_python.outputs.installed == 'false' ) }} name: Setup Python ${{ inputs.version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: ${{ inputs.version }} env: diff --git a/.github/actions/smart-ci/action.yml b/.github/actions/smart-ci/action.yml index cd111d617ddc1b..ec65d2f1e9e82c 100644 --- a/.github/actions/smart-ci/action.yml +++ b/.github/actions/smart-ci/action.yml @@ -30,7 +30,6 @@ inputs: components_config_schema: description: "Path to the schema file for components configuration" required: false - default: ".github/actions/smart-ci/components_schema.yml" labeler_config: description: "Path to labeler configuration file" required: false @@ -66,7 +65,7 @@ runs: using: "composite" steps: - name: Wait for labeler to finish - uses: lewagon/wait-on-check-action@v1.3.1 + uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc # v1.3.4 if: ${{ github.event_name == 'pull_request' }} with: ref: ${{ github.event.pull_request.head.sha }} @@ -75,13 +74,13 @@ runs: wait-interval: 10 - name: checkout components file - uses: actions/checkout@v4 + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: sparse-checkout: .github/components.yml sparse-checkout-cone-mode: false - name: Install Python dependencies - uses: py-actions/py-dependency-install@v4 + uses: py-actions/py-dependency-install@30aa0023464ed4b5b116bd9fbdab87acf01a484e # v4.1.0 with: path: "${{ github.action_path }}/requirements.txt" update-setuptools: "false" @@ -101,7 +100,7 @@ runs: -f "${{ inputs.ref_name }}" \ -p "${{ inputs.component_pattern }}" \ -c "${{ inputs.components_config }}" \ - -m "${{ inputs.components_config_schema }}" \ + -m "${{ inputs.components_config_schema || env.DEFAULT_CONFIG_SCHEMA }}" \ -l "${{ inputs.labeler_config }}" \ --enable_for_org "${{ inputs.enable_for_org }}" \ --skip-when-only-listed-labels-set "${{ inputs.skip_when_only_listed_labels_set }}" \ @@ -109,3 +108,4 @@ runs: shell: bash env: GITHUB_TOKEN: ${{ inputs.repo_token }} + DEFAULT_CONFIG_SCHEMA: "${{ github.action_path }}/components_schema.yml" diff --git a/.github/dockerfiles/docker_tag b/.github/dockerfiles/docker_tag index 56faa37d1da67f..5a4f7795ea4a44 100644 --- a/.github/dockerfiles/docker_tag +++ b/.github/dockerfiles/docker_tag @@ -1 +1 @@ -pr-26993 \ No newline at end of file +pr-27384 diff --git a/.github/workflows/android_x64.yml b/.github/workflows/android_x64.yml index a667a07da5bd3e..1cdb2023784979 100644 --- a/.github/workflows/android_x64.yml +++ b/.github/workflows/android_x64.yml @@ -135,6 +135,7 @@ jobs: -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \ -DENABLE_LTO=ON \ -DENABLE_PYTHON=OFF \ + -DENABLE_TESTS=ON \ -DOPENVINO_EXTRA_MODULES=${{ env.OPENVINO_GENAI_REPO }} \ -S ${OPENVINO_REPO} \ -B ${BUILD_DIR} diff --git a/.github/workflows/build_doc.yml b/.github/workflows/build_doc.yml index 6623f5ea182da1..8c78375e61769c 100644 --- a/.github/workflows/build_doc.yml +++ b/.github/workflows/build_doc.yml @@ -29,7 +29,7 @@ jobs: packages: graphviz texlive liblua5.2-0 libclang1-9 libclang-cpp9 version: 3.0 - - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 id: cp310 with: python-version: '3.10' @@ -41,7 +41,7 @@ jobs: - name: Install python dependencies run: | python3 -m pip install -r docs/requirements.txt - (cd docs/openvino_sphinx_theme && python3 setup.py install) + (cd docs/openvino_sphinx_theme && python3 -m pip install .) python3 -m pip install docs/openvino_custom_sphinx_sitemap - name: Download and install doxygen diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index fdb41226f4efb8..db5ba3de1a3c85 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Setup python - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: '3.10.10' architecture: 'x64' diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index d87de4257e0270..9797414cde56c8 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -25,7 +25,56 @@ env: TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }} jobs: + + Smart_CI: + runs-on: ubuntu-latest + outputs: + affected_components: "${{ steps.smart_ci.outputs.affected_components }}" + changed_components: "${{ steps.smart_ci.outputs.changed_components }}" + skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" + steps: + - name: checkout action + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + sparse-checkout: .github/actions/smart-ci + + - name: Get affected components + id: smart_ci + uses: ./.github/actions/smart-ci + with: + repository: ${{ github.repository }} + pr: ${{ github.event.number }} + commit_sha: ${{ github.sha }} + ref_name: ${{ github.ref_name }} + component_pattern: "category: (.*)" + repo_token: ${{ secrets.GITHUB_TOKEN }} + skip_when_only_listed_labels_set: 'docs' + skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg' + + Docker: + needs: Smart_CI + runs-on: aks-linux-4-cores-16gb-docker-build + container: + image: openvinogithubactions.azurecr.io/docker_build:0.2 + volumes: + - /mount:/mount + outputs: + images: "${{ steps.handle_docker.outputs.images }}" + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: ./.github/actions/handle_docker + id: handle_docker + with: + images: | + ov_build/ubuntu_20_04_x64 + registry: 'openvinogithubactions.azurecr.io' + dockerfiles_root_dir: '.github/dockerfiles' + changed_components: ${{ needs.smart_ci.outputs.changed_components }} + Build: + needs: Docker timeout-minutes: 150 defaults: run: @@ -33,7 +82,10 @@ jobs: runs-on: aks-linux-16-cores-32gb if: ${{ github.repository_owner == 'openvinotoolkit' }} container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }} + volumes: + - /mount:/mount + options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING env: DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input CMAKE_BUILD_TYPE: 'Release' @@ -43,13 +95,16 @@ jobs: OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib BUILD_DIR: /__w/openvino/openvino/openvino_build COVERITY_TOOL_DIR: /__w/openvino/openvino/coverity_tool + CMAKE_CXX_COMPILER_LAUNCHER: sccache + CMAKE_C_COMPILER_LAUNCHER: sccache + SCCACHE_IGNORE_SERVER_IO_ERROR: 1 + SCCACHE_SERVER_PORT: 35555 + SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt + SCCACHE_LOG: warn + SCCACHE_AZURE_KEY_PREFIX: coverity_lin + SCCACHE_CACHE_SIZE: 50G steps: - - name: Install git - run: | - apt-get update - apt-get install --assume-yes --no-install-recommends git ca-certificates - - name: Clone OpenVINO uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -86,7 +141,9 @@ jobs: # # Build # - + - name: Clean sccache stats + run: ${SCCACHE_PATH} --zero-stats + - name: CMake configure - OpenVINO run: | cmake \ @@ -108,12 +165,13 @@ jobs: - name: Cmake build - OpenVINO with Coverity run: ${COVERITY_TOOL_DIR}/cov-analysis*/bin/cov-build --dir ${BUILD_DIR}/cov-int cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + + - name: Show sccache stats + run: ${SCCACHE_PATH} --show-stats - name: Pack Artefacts - run: | - pushd ${BUILD_DIR} - tar -cvf - cov-int | pigz > openvino.tgz - popd + run: tar -cvf - cov-int | pigz > openvino.tgz + working-directory: ${{ env.BUILD_DIR }} - name: Submit artefacts run: | diff --git a/.github/workflows/job_gpu_tests.yml b/.github/workflows/job_gpu_tests.yml index b9862eac09cc05..195abbbd5fb0f9 100644 --- a/.github/workflows/job_gpu_tests.yml +++ b/.github/workflows/job_gpu_tests.yml @@ -74,7 +74,7 @@ jobs: DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input TZ: "Europe/London" # to prevent tzdata from waiting user input - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: ${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/job_openvino_js.yml b/.github/workflows/job_openvino_js.yml index 6097d3e6f18bc4..ecb278fdb54ca3 100644 --- a/.github/workflows/job_openvino_js.yml +++ b/.github/workflows/job_openvino_js.yml @@ -52,7 +52,7 @@ jobs: - name: Setup Node ${{ env.NODE_VERSION }} if: runner.os != 'Linux' # Node is already installed in the Docker image - uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4 + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 with: node-version: ${{ env.NODE_VERSION }} diff --git a/.github/workflows/job_python_unit_tests.yml b/.github/workflows/job_python_unit_tests.yml index 1dd6ebbfc204d8..64be9ef4bbcc44 100644 --- a/.github/workflows/job_python_unit_tests.yml +++ b/.github/workflows/job_python_unit_tests.yml @@ -111,10 +111,7 @@ jobs: run: | # To enable pytest parallel features python3 -m pip install pytest-xdist[psutil] - # For torchvision to OpenVINO preprocessing converter - python3 -m pip install -r ${INSTALL_TEST_DIR}/python/preprocess/torchvision/requirements.txt - - # TODO: replace with Python API tests requirements + python3 -m pip install -r ${INSTALL_TEST_DIR}/bindings/python/requirements_test.txt python3 -m pip install -r ${INSTALL_TEST_DIR}/mo/requirements_dev.txt # @@ -158,6 +155,9 @@ jobs: - name: Install Python Layer tests dependencies run: | + # For torchvision to OpenVINO preprocessing converter + python3 -m pip install -r ${INSTALL_TEST_DIR}/python/preprocess/torchvision/requirements.txt + # layer test requirements python3 -m pip install -r ${LAYER_TESTS_INSTALL_DIR}/requirements.txt @@ -267,7 +267,7 @@ jobs: if: ${{ fromJSON(inputs.affected-components).Python_API.test }} run: | python3 -m pip uninstall -y numpy - python3 -m pip install "numpy>=2.0.0,<2.1.0" + python3 -m pip install "numpy>=2.0.0,<2.2.0" python3 -m pip install -r ${INSTALL_TEST_DIR}/bindings/python/requirements_test.txt # for 'template' extension export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}:$LD_LIBRARY_PATH diff --git a/.github/workflows/job_tokenizers.yml b/.github/workflows/job_tokenizers.yml index 14243bda13531a..5c5e59aa3bec97 100644 --- a/.github/workflows/job_tokenizers.yml +++ b/.github/workflows/job_tokenizers.yml @@ -20,12 +20,15 @@ on: description: 'Components that are affected by changes in the commit defined by the Smart CI Action' type: string required: true + python-version: + description: 'Python version to setup. E.g., "3.11"' + type: string + required: true permissions: read-all env: PIP_CACHE_PATH: /mount/caches/pip/linux - PYTHON_VERSION: '3.11' TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }} jobs: @@ -63,10 +66,10 @@ jobs: if: runner.os == 'macOS' run: brew install pigz - - name: Setup Python ${{ env.PYTHON_VERSION }} + - name: Setup Python ${{ inputs.python-version }} uses: ./.github/actions/setup_python with: - version: ${{ env.PYTHON_VERSION }} + version: ${{ inputs.python-version }} pip-cache-path: ${{ runner.os == 'Linux' && env.PIP_CACHE_PATH || '' }} should-setup-pip-paths: ${{ runner.os == 'Linux' }} self-hosted-runner: ${{ runner.os == 'Linux' }} diff --git a/.github/workflows/linux_arm64.yml b/.github/workflows/linux_arm64.yml index 67cd4d0d1a5d84..0af30621a2a7fd 100644 --- a/.github/workflows/linux_arm64.yml +++ b/.github/workflows/linux_arm64.yml @@ -146,6 +146,7 @@ jobs: shell: bash container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_arm64 }}", "volumes": ["/mount:/mount"]}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} + python-version: '3.11' if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS CXX_Unit_Tests: diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 7c47c1c635c2f8..5492ad40aa17b4 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -258,6 +258,7 @@ jobs: runner: 'macos-13' shell: bash affected-components: ${{ needs.smart_ci.outputs.affected_components }} + python-version: '3.11' if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS CXX_Unit_Tests: diff --git a/.github/workflows/mac_arm64.yml b/.github/workflows/mac_arm64.yml index 81cd229d1dd9f6..8100b74734ab17 100644 --- a/.github/workflows/mac_arm64.yml +++ b/.github/workflows/mac_arm64.yml @@ -258,6 +258,7 @@ jobs: runner: 'macos-13-xlarge' shell: bash affected-components: ${{ needs.smart_ci.outputs.affected_components }} + python-version: '3.11' if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS CXX_Unit_Tests: diff --git a/.github/workflows/mo.yml b/.github/workflows/mo.yml index 2405103755b552..f48986d4a0d304 100644 --- a/.github/workflows/mo.yml +++ b/.github/workflows/mo.yml @@ -27,7 +27,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup Python - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: '3.10' diff --git a/.github/workflows/ovc.yml b/.github/workflows/ovc.yml index 7d18643def3ce6..4d69563a741d3a 100644 --- a/.github/workflows/ovc.yml +++ b/.github/workflows/ovc.yml @@ -22,7 +22,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup Python - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: '3.10' diff --git a/.github/workflows/py_checks.yml b/.github/workflows/py_checks.yml index 13ddbaaa1ec41c..caed37eee89056 100644 --- a/.github/workflows/py_checks.yml +++ b/.github/workflows/py_checks.yml @@ -31,7 +31,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup Python - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: '3.9' diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 753708d9b3ba51..92178fce7f5054 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -491,6 +491,7 @@ jobs: shell: bash container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}", "volumes": ["/mount:/mount"]}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} + python-version: '3.11' if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS iGPU: diff --git a/.github/workflows/ubuntu_24.yml b/.github/workflows/ubuntu_24.yml index 9d9aba6739f22f..d874e06a189232 100644 --- a/.github/workflows/ubuntu_24.yml +++ b/.github/workflows/ubuntu_24.yml @@ -144,9 +144,31 @@ jobs: affected-components: ${{ needs.smart_ci.outputs.affected_components }} python-version: '3.12' + TensorFlow_Layer_Tests: + name: TensorFlow Layer Tests + needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] + uses: ./.github/workflows/job_tensorflow_layer_tests.yml + with: + runner: 'aks-linux-4-cores-16gb' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_24_04_x64 }}", "volumes": ["/mount:/mount"]}' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} + python-version: '3.12' + + Openvino_tokenizers: + name: OpenVINO tokenizers extension + needs: [ Build, Smart_CI, Docker ] + uses: ./.github/workflows/job_tokenizers.yml + with: + runner: 'aks-linux-4-cores-16gb' + shell: bash + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_24_04_x64 }}", "volumes": ["/mount:/mount"]}' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} + python-version: '3.12' + if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS + Overall_Status: name: ci/gha_overall_status_ubuntu_24 - needs: [Smart_CI, Build, Debian_Packages, Samples, Python_Unit_Tests] + needs: [Smart_CI, Build, Debian_Packages, Samples, Python_Unit_Tests, Pytorch_Layer_Tests, TensorFlow_Layer_Tests, Openvino_tokenizers] if: ${{ always() }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/windows_vs2019_release.yml b/.github/workflows/windows_vs2019_release.yml index 1a3e8753f15421..b9b8fa76d37c34 100644 --- a/.github/workflows/windows_vs2019_release.yml +++ b/.github/workflows/windows_vs2019_release.yml @@ -189,7 +189,7 @@ jobs: path: ${{ env.OPENVINO_JS_LIBS_DIR }} - name: Setup Node ${{ env.NODE_VERSION }} - uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4 + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 with: node-version: ${{ env.NODE_VERSION }} @@ -235,6 +235,7 @@ jobs: runner: 'aks-win-4-cores-8gb' shell: pwsh affected-components: ${{ needs.smart_ci.outputs.affected_components }} + python-version: '3.11' if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS Python_Unit_Tests: diff --git a/cmake/developer_package/frontends/frontends.cmake b/cmake/developer_package/frontends/frontends.cmake index d2aa0410476245..0815297a11a5eb 100644 --- a/cmake/developer_package/frontends/frontends.cmake +++ b/cmake/developer_package/frontends/frontends.cmake @@ -304,6 +304,9 @@ macro(ov_add_frontend) # then we need to mark it to be CXX ABI free ov_abi_free_target(${TARGET_NAME}) + # public target name + set_target_properties(${TARGET_NAME} PROPERTIES EXPORT_NAME frontend::${OV_FRONTEND_NAME}) + # installation if(NOT OV_FRONTEND_SKIP_INSTALL) @@ -351,9 +354,6 @@ macro(ov_add_frontend) COMPONENT ${dev_component} ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL} FILES_MATCHING PATTERN "*.hpp") - - # public target name - set_target_properties(${TARGET_NAME} PROPERTIES EXPORT_NAME frontend::${OV_FRONTEND_NAME}) endif() else() # skipped frontend has to be installed in static libraries case diff --git a/cmake/developer_package/packaging/common-libraries.cmake b/cmake/developer_package/packaging/common-libraries.cmake index 0ec054da853e2c..4ac0124d3089f0 100644 --- a/cmake/developer_package/packaging/common-libraries.cmake +++ b/cmake/developer_package/packaging/common-libraries.cmake @@ -30,7 +30,7 @@ macro(ov_common_libraries_cpack_set_dirs) ov_get_pyversion(pyversion) if(pyversion) - # should not be used in production; only by setup.py install + # should not be used in production; only by pip install set(OV_CPACK_PYTHONDIR lib/${pyversion}/site-packages) endif() @@ -94,7 +94,7 @@ macro(ov_define_component_include_rules) set(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL EXCLUDE_FROM_ALL) set(OV_CPACK_COMP_BENCHMARK_APP_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) set(OV_CPACK_COMP_OVC_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) - # we don't pack artifacts of setup.py install, because it's called explicitly in conda / brew + # we don't pack artifacts of pip install, because it's called explicitly in conda / brew # or not used at all like in cases with conan / vcpkg set(OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) # we don't need wheels in the distribution packages diff --git a/cmake/developer_package/packaging/debian/debian.cmake b/cmake/developer_package/packaging/debian/debian.cmake index 2b95fcfde5c145..a23d5290044e3d 100644 --- a/cmake/developer_package/packaging/debian/debian.cmake +++ b/cmake/developer_package/packaging/debian/debian.cmake @@ -95,12 +95,12 @@ macro(ov_define_component_include_rules) endif() # python if(ENABLE_PYTHON_PACKAGING) - # pack artifacts of setup.py install + # pack artifacts of pip install unset(OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE_EXCLUDE_ALL) else() set(OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE_EXCLUDE_ALL EXCLUDE_FROM_ALL) endif() - # we don't pack python components itself, we pack artifacts of setup.py install + # we don't pack python components itself, we pack artifacts of pip install set(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL EXCLUDE_FROM_ALL) set(OV_CPACK_COMP_BENCHMARK_APP_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) set(OV_CPACK_COMP_OVC_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) diff --git a/cmake/developer_package/packaging/rpm/rpm.cmake b/cmake/developer_package/packaging/rpm/rpm.cmake index 45d9b0c0ca2121..bb4e7942d7640b 100644 --- a/cmake/developer_package/packaging/rpm/rpm.cmake +++ b/cmake/developer_package/packaging/rpm/rpm.cmake @@ -86,12 +86,12 @@ macro(ov_define_component_include_rules) endif() # python if(ENABLE_PYTHON_PACKAGING) - # pack artifacts of setup.py install + # pack artifacts of pip install unset(OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE_EXCLUDE_ALL) else() set(OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE_EXCLUDE_ALL EXCLUDE_FROM_ALL) endif() - # we don't pack python components itself, we pack artifacts of setup.py install + # we don't pack python components itself, we pack artifacts of pip install set(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL EXCLUDE_FROM_ALL) set(OV_CPACK_COMP_BENCHMARK_APP_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) set(OV_CPACK_COMP_OVC_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) diff --git a/cmake/packaging/debian.cmake b/cmake/packaging/debian.cmake index 0eb020a5584f1c..59b312963c180d 100644 --- a/cmake/packaging/debian.cmake +++ b/cmake/packaging/debian.cmake @@ -98,6 +98,7 @@ macro(ov_cpack_settings) 2024.2.0 2024.3.0 2024.4.0 + 2024.5.0 ) ov_check_conflicts_versions(conflicting_versions) diff --git a/cmake/packaging/rpm.cmake b/cmake/packaging/rpm.cmake index cf915ee852417e..a4a63c35858bf9 100644 --- a/cmake/packaging/rpm.cmake +++ b/cmake/packaging/rpm.cmake @@ -86,6 +86,7 @@ macro(ov_cpack_settings) 2024.2.0 2024.3.0 2024.4.0 + 2024.5.0 ) ov_check_conflicts_versions(conflicting_versions) diff --git a/cmake/toolchains/mt.runtime.win32.toolchain.cmake b/cmake/toolchains/mt.runtime.win32.toolchain.cmake index 7dd4e1e7f96ded..b331d370bfe7bf 100644 --- a/cmake/toolchains/mt.runtime.win32.toolchain.cmake +++ b/cmake/toolchains/mt.runtime.win32.toolchain.cmake @@ -27,6 +27,11 @@ if(use_static_runtime) foreach(build_type "" "_DEBUG" "_MINSIZEREL" "_RELEASE" "_RELWITHDEBINFO") set(flag_var "CMAKE_${lang}_FLAGS${build_type}_INIT") string(REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}") + if (build_type STREQUAL "_DEBUG") + set(${flag_var} "/MTd") + else() + set(${flag_var} "/MT") + endif() endforeach() endforeach() endif() diff --git a/cmake/toolchains/riscv64-071-thead-gnu.toolchain.cmake b/cmake/toolchains/riscv64-071-xuantie-gnu.toolchain.cmake similarity index 95% rename from cmake/toolchains/riscv64-071-thead-gnu.toolchain.cmake rename to cmake/toolchains/riscv64-071-xuantie-gnu.toolchain.cmake index f5e9e68aabedc6..5bc16de8df91e8 100644 --- a/cmake/toolchains/riscv64-071-thead-gnu.toolchain.cmake +++ b/cmake/toolchains/riscv64-071-xuantie-gnu.toolchain.cmake @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # -# NOTE: use T-Head compiler: +# NOTE: use Xuantie compiler: # git clone https://github.com/XUANTIE-RV/xuantie-gnu-toolchain.git # ./configure --prefix=/opt/riscv # make linux @@ -22,10 +22,10 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR riscv64) -set(RISCV64_THEAD ON) +set(RISCV64_XUANTIE ON) set(RISCV64_RVV0p7 ON) -set(RISCV_TOOLCHAIN_ROOT $ENV{RISCV_TOOLCHAIN_ROOT} CACHE PATH "Path to CLANG for RISC-V cross compiler build directory") +set(RISCV_TOOLCHAIN_ROOT $ENV{RISCV_TOOLCHAIN_ROOT} CACHE PATH "Path to GCC for RISC-V cross compiler build directory") set(CMAKE_SYSROOT "${RISCV_TOOLCHAIN_ROOT}/sysroot" CACHE PATH "RISC-V sysroot") set(CMAKE_C_COMPILER ${RISCV_TOOLCHAIN_ROOT}/bin/riscv64-unknown-linux-gnu-gcc) diff --git a/cmake/toolchains/riscv64-100-thead-gnu.toolchain.cmake b/cmake/toolchains/riscv64-100-xuantie-gnu.toolchain.cmake similarity index 95% rename from cmake/toolchains/riscv64-100-thead-gnu.toolchain.cmake rename to cmake/toolchains/riscv64-100-xuantie-gnu.toolchain.cmake index e00e30f975598f..0664b38a9ba68d 100644 --- a/cmake/toolchains/riscv64-100-thead-gnu.toolchain.cmake +++ b/cmake/toolchains/riscv64-100-xuantie-gnu.toolchain.cmake @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # -# NOTE: use T-Head compiler: +# NOTE: use Xuantie compiler: # git clone https://github.com/XUANTIE-RV/xuantie-gnu-toolchain.git # ./configure --prefix=/opt/riscv # make linux @@ -22,10 +22,10 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR riscv64) -set(RISCV64_THEAD ON) +set(RISCV64_XUANTIE ON) set(RISCV64_RVV1p0 ON) -set(RISCV_TOOLCHAIN_ROOT $ENV{RISCV_TOOLCHAIN_ROOT} CACHE PATH "Path to CLANG for RISC-V cross compiler build directory") +set(RISCV_TOOLCHAIN_ROOT $ENV{RISCV_TOOLCHAIN_ROOT} CACHE PATH "Path to GCC for RISC-V cross compiler build directory") set(CMAKE_SYSROOT "${RISCV_TOOLCHAIN_ROOT}/sysroot" CACHE PATH "RISC-V sysroot") set(CMAKE_C_COMPILER ${RISCV_TOOLCHAIN_ROOT}/bin/riscv64-unknown-linux-gnu-gcc) diff --git a/cmake/toolchains/riscv64-gnu.toolchain.cmake b/cmake/toolchains/riscv64-gnu.toolchain.cmake index 994b05f66b52f6..b58dcf169fc2da 100644 --- a/cmake/toolchains/riscv64-gnu.toolchain.cmake +++ b/cmake/toolchains/riscv64-gnu.toolchain.cmake @@ -2,12 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 # -# NOTE: use with the following docker image https://github.com/Incarnation-p-lee/riscv-docker-emulator#llvm-clang-tool-chain +# NOTE: use with the following docker image https://github.com/Incarnation-p-lee/riscv-docker-emulator#gnu-toolchain set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR riscv64) -set(RISCV_TOOLCHAIN_ROOT "/opt/riscv/gnu-toolchain/rv64-linux" CACHE PATH "Path to CLANG for RISC-V cross compiler build directory") +set(RISCV_TOOLCHAIN_ROOT "/opt/riscv/gnu-toolchain/rv64-linux" CACHE PATH "Path to GCC for RISC-V cross compiler build directory") set(CMAKE_SYSROOT "${RISCV_TOOLCHAIN_ROOT}/sysroot" CACHE PATH "RISC-V sysroot") set(CMAKE_C_COMPILER_TARGET riscv64-unknown-linux-gnu) @@ -26,9 +26,6 @@ set(CMAKE_OBJDUMP ${RISCV_TOOLCHAIN_ROOT}/bin/riscv64-unknown-linux-gnu-objdump) set(CMAKE_READELF ${RISCV_TOOLCHAIN_ROOT}/bin/riscv64-unknown-linux-gnu-readelf) set(PKG_CONFIG_EXECUTABLE "NOT-FOUND" CACHE PATH "Path to RISC-V pkg-config") -# Don't run the linker on compiler check -set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) - set(CMAKE_SHARED_LINKER_FLAGS_INIT "-L${CMAKE_SYSROOT}/lib") set(CMAKE_EXE_LINKER_FLAGS_INIT "-L${CMAKE_SYSROOT}/lib") set(CMAKE_MODULE_LINKER_FLAGS_INIT "-L${CMAKE_SYSROOT}/lib") diff --git a/cmake/toolchains/riscv64.linux.toolchain.cmake b/cmake/toolchains/riscv64.linux.toolchain.cmake new file mode 100644 index 00000000000000..cb088f5eca5052 --- /dev/null +++ b/cmake/toolchains/riscv64.linux.toolchain.cmake @@ -0,0 +1,13 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +# Install compiler on debian using: +# apt-get install -y gcc-riscv64-linux-gnu g++-riscv64-linux-gnu binutils-riscv64-linux-gnu + +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR riscv64) + +set(CMAKE_C_COMPILER riscv64-linux-gnu-gcc) +set(CMAKE_CXX_COMPILER riscv64-linux-gnu-g++) +set(CMAKE_STRIP riscv64-linux-gnu-strip) diff --git a/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst b/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst index a49a9d8dba81d1..3f3d0064e8a4c6 100644 --- a/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst +++ b/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst @@ -15,23 +15,25 @@ deep learning models: | For their usage guides, see :doc:`Devices and Modes <../../openvino-workflow/running-inference/inference-devices-and-modes>`. | For a detailed list of devices, see :doc:`System Requirements <../release-notes-openvino/system-requirements>`. + Beside running inference with a specific device, OpenVINO offers the option of running automated inference with the following inference modes: | :doc:`Automatic Device Selection <../../openvino-workflow/running-inference/inference-devices-and-modes/auto-device-selection>`: -| automatically selects the best device available for the given task. It offers many - additional options and optimizations, including inference on multiple devices at the - same time. +| automatically selects the best device available for the given task. It offers many + additional options and optimizations, including inference on multiple devices at the + same time. + | :doc:`Heterogeneous Inference <../../openvino-workflow/running-inference/inference-devices-and-modes/hetero-execution>`: -| enables splitting inference among several devices automatically, for example, if one device - doesn't support certain operations. +| enables splitting inference among several devices automatically, for example, if one device + doesn't support certain operations. | :doc:`Automatic Batching <../../openvino-workflow/running-inference/inference-devices-and-modes/automatic-batching>`: -| automatically groups inference requests to improve device utilization. +| automatically groups inference requests to improve device utilization. | :doc:`(LEGACY) Multi-device Inference <./../../documentation/legacy-features/multi-device>`: -| executes inference on multiple devices. Currently, this mode is considered a legacy - solution. Using Automatic Device Selection instead is advised. +| executes inference on multiple devices. Currently, this mode is considered a legacy + solution. Using Automatic Device Selection instead is advised. Feature Support and API Coverage @@ -74,13 +76,25 @@ Feature Support and API Coverage | HETERO | 61.22 % | 99.24 % | 86.05 % | +-------------------------+-----------+------------------+-------------------+ | || Percentage of API supported by the device, | -| || as of OpenVINO 2023.3, 08 Jan, 2024. | +| || as of OpenVINO 2024.4, 25 Oct, 2024. | +-------------------------+-----------+------------------+-------------------+ For setting up a relevant configuration, refer to the :doc:`Integrate with Customer Application <../../openvino-workflow/running-inference/integrate-openvino-with-your-application>` topic (step 3 "Configure input and output"). +.. dropdown:: Device support across OpenVINO 2024.4 distributions + + =============== ========== ====== =============== ======== ============ ========== ========== ========== + Device Archives PyPI APT/YUM/ZYPPER Conda Homebrew vcpkg Conan npm + =============== ========== ====== =============== ======== ============ ========== ========== ========== + CPU V V V V V V V V + GPU V V V V V V V V + NPU V\* V\* V\ * n/a n/a n/a n/a V\* + =============== ========== ====== =============== ======== ============ ========== ========== ========== + + | \* **Of the Linux systems, versions 22.04 and 24.04 include drivers for NPU.** + | **For Windows, CPU inference on ARM64 is not supported.** .. note:: @@ -89,6 +103,7 @@ topic (step 3 "Configure input and output"). in your solutions, revert to the 2023.3 (LTS) version. With the OpenVINO™ 2023.0 release, support has been cancelled for: + - Intel® Neural Compute Stick 2 powered by the Intel® Movidius™ Myriad™ X - Intel® Vision Accelerator Design with Intel® Movidius™ diff --git a/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst b/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst index 39b27d12c970fd..b8256af650e2f8 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst @@ -3,9 +3,11 @@ Most Efficient Large Language Models for AI PC This page is regularly updated to help you identify the best-performing LLMs on the Intel® Core™ Ultra processor family and AI PCs. +The current data is as of OpenVINO 2024.4, 24 Oct. 2024 -The tables below list key performance indicators for a selection of Large Language Models, -running on an Intel® Core™ Ultra 7-165H based system, on built-in GPUs. +The tables below list the key performance indicators for a selection of Large Language Models, +running on an Intel® Core™ Ultra 7-165H, Intel® Core™ Ultra 7-265V, and Intel® Core™ Ultra +7-288V based system, on built-in GPUs. @@ -15,16 +17,13 @@ running on an Intel® Core™ Ultra 7-165H based system, on built-in GPUs. -.. tab-set:: - - .. tab-item:: OpenVINO - - .. csv-table:: - :class: modeldata stripe - :name: supportedModelsTableOv - :header-rows: 1 - :file: ../../_static/benchmarks_files/llm_models.csv +.. csv-table:: + :class: modeldata stripe + :name: supportedModelsTableOv + :header-rows: 1 + :file: ../../_static/benchmarks_files/llm_models.csv +| .. grid:: 1 1 2 2 :gutter: 4 @@ -34,18 +33,17 @@ running on an Intel® Core™ Ultra 7-165H based system, on built-in GPUs. All models listed here were tested with the following parameters: * Framework: PyTorch - * Model precision: INT4 * Beam: 1 * Batch size: 1 .. grid-item:: - .. button-link:: https://docs.openvino.ai/2024/_static/benchmarks_files/OV-2024.4-platform_list.pdf + .. button-link:: https://docs.openvino.ai/2024/_static/benchmarks_files/llm_models_platform_list_.pdf :color: primary :outline: :expand: - :material-regular:`download;1.5em` Get full system info [PDF] + :material-regular:`download;1.5em` Get system descriptions [PDF] .. button-link:: ../../_static/benchmarks_files/llm_models.csv :color: primary diff --git a/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst b/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst index 50ac3a3350658e..a12cacf8402953 100644 --- a/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst +++ b/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst @@ -114,7 +114,7 @@ Operating systems and developer environment Build environment components: - * Python 3.8-3.12 + * Python 3.9-3.12 * `Intel® HD Graphics Driver `__ required for inference on GPU * GNU Compiler Collection and CMake are needed for building from source: @@ -128,11 +128,16 @@ Operating systems and developer environment .. tab-item:: Windows 10 and 11 + OpenVINO Runtime requires certain C++ libraries to operate. To execute ready-made apps, + the libraries distributed by `Visual Studio redistributable package `__ + are suggested. For development and compilation of OpenVINO-integrated apps, the build + environment components are required instead. + Build environment components: - * `Microsoft Visual Studio 2019 `__ + * `Microsoft Visual Studio 2019 or later `__ * `CMake `__ 3.16 or higher - * `Python `__ 3.8-3.12 + * `Python `__ 3.9-3.12 * `Intel® HD Graphics Driver `__ required for inference on GPU @@ -144,7 +149,7 @@ Operating systems and developer environment * `Xcode `__ 10.3 * `CMake `__ 3.13 or higher - * `Python `__ 3.8-3.12 + * `Python `__ 3.9-3.12 .. tab-item:: DL framework versions: @@ -162,7 +167,193 @@ Operating systems and developer environment OpenVINO Python binaries are built with and redistribute oneTBB libraries. +OpenVINO Distributions +###################### + +Different OpenVINO distributions may support slightly different sets of features. +Read installation guides for particular distributions for more details. +Refer to the :doc:`OpenVINO Release Policy <../../../about-openvino/release-notes-openvino/release-policy>` +to learn more about the release types. + + +.. tab-set:: + + .. tab-item:: Archive + :name: archive-sysreq + + .. tab-set:: + + .. tab-item:: Linux + :name: archive-lnx-sysreq + + * `CMake 3.13 or higher, 64-bit `__ + * `Python 3.9 - 3.12, 64-bit `__ + * GCC: + + .. tab-set:: + + .. tab-item:: Ubuntu + :sync: ubuntu + + * GCC 9.3.0 (for Ubuntu 20.04), GCC 11.3.0 (for Ubuntu 22.04) or GCC 13.2.0 (for Ubuntu 24.04) + + .. tab-item:: RHEL 8 + :sync: rhel-8 + + * GCC 8.4.1 + + .. tab-item:: CentOS 7 + :sync: centos-7 + + * GCC 8.3.1 + + Use the following instructions to install it: + + Install GCC 8.3.1 via devtoolset-8 + + .. code-block:: sh + + sudo yum update -y && sudo yum install -y centos-release-scl epel-release + sudo yum install -y devtoolset-8 + + Enable devtoolset-8 and check current gcc version + + .. code-block:: sh + + source /opt/rh/devtoolset-8/enable + gcc -v + + .. tab-item:: macOS + :name: archive-win-sysreq + + * `CMake 3.13 or higher `__ (choose "macOS 10.13 or later"). Add ``/Applications/CMake.app/Contents/bin`` to path (for default install). + * `Python 3.9 - 3.12 `__ (choose 3.9 - 3.12). Install and add to path. + * Apple Xcode Command Line Tools. In the terminal, run ``xcode-select --install`` from any directory + * (Optional) Apple Xcode IDE (not required for OpenVINO™, but useful for development) + + .. tab-item:: Windows + :name: archive-win-sysreq + + * `C++ libraries (included in Visual Studio redistributable) `__ (a core dependency for OpenVINO Runtime) + * `Microsoft Visual Studio 2019 or later `__ (for development and app compilation with OpenVINO) + * `CMake 3.14 or higher, 64-bit `__ (optional, only required for building sample applications) + * `Python 3.9 - 3.12, 64-bit `__ + + .. note:: + + To install Microsoft Visual Studio, follow the `Microsoft Visual Studio installation guide `__. + You can choose to download the Community version. During installation in the **Workloads** tab, choose **Desktop development with C++**. + + .. note:: + + You can either use `cmake.msi` which is the installation wizard or `cmake.zip` where you have to go into the `bin` folder and then manually add the path to environmental variables. + + .. important:: + + When installing Python, make sure you click the option **Add Python 3.x to PATH** to `add Python `__ to your `PATH` environment variable. + + .. tab-item:: APT + :sync: apt-sysreq + + .. tab-set:: + + .. tab-item:: Linux + :sync: linux + + * `CMake 3.13 or higher, 64-bit `__ + * GCC 9.3.0 (for Ubuntu 20.04), GCC 11.3.0 (for Ubuntu 22.04) or GCC 13.2.0 (for Ubuntu 24.04) + * `Python 3.9 - 3.12, 64-bit `__ + + .. tab-item:: Homebrew + :name: homebrew-sysreq + + .. tab-set:: + + .. tab-item:: Linux + :sync: linux + + * `Homebrew `_ + * `CMake 3.13 or higher, 64-bit `__ + * GCC 9.3.0 (for Ubuntu 20.04), GCC 11.3.0 (for Ubuntu 22.04) or GCC 13.2.0 (for Ubuntu 24.04) + * `Python 3.9 - 3.12, 64-bit `__ + + .. tab-item:: macOS + :sync: macos + + * `Homebrew `_ + * `CMake 3.13 or higher `__ (choose "macOS 10.13 or later"). Add ``/Applications/CMake.app/Contents/bin`` to path (for default installation). + * `Python 3.9 - 3.12 `__ . Install and add it to path. + * Apple Xcode Command Line Tools. In the terminal, run ``xcode-select --install`` from any directory to install it. + * (Optional) Apple Xcode IDE (not required for OpenVINO™, but useful for development) + + .. tab-item:: npm + :name: npm-sysreq + + .. tab-set:: + + .. tab-item:: Linux + :sync: linux + + All x86_64 / arm64 architectures are supported. + + * `Node.js version 21.0.0 and higher `__ + + .. tab-item:: macOS + :sync: macos + + All x86_64 / arm64 architectures are supported, however, only for CPU inference. + + * `Node.js version 21.0.0 and higher `__ + + .. tab-item:: Windows + :sync: Windows + + All x86_64 architectures are supported. Windows ARM is not supported. + + * `Node.js version 21.0.0 and higher `__ + + .. tab-item:: YUM + :name: yum-sysreq + + .. tab-set:: + + .. tab-item:: Linux + :sync: linux + + OpenVINO RPM packages are compatible with and can be run on the following operating systems: + + * RHEL 8.2 and higher + * Amazon Linux 2022 and 2023 + * Rocky Linux 8.7, 8.8 and 9.2-9.3 + * Alma Linux 8.7, 8.8 and 9.2-9.4 + * Oracle Linux 8.7, 8.8 and 9.2-9.4 + * Fedora 29 and higher up to 41 + * OpenEuler 20.03, 22.03, 23.03 and 24.03 + * Anolis OS 8.6 and 8.8 + * CentOS Stream 8 and 9 + + Software: + + * `CMake 3.13 or higher, 64-bit `_ + * GCC 8.4.1 + * `Python 3.9 - 3.12, 64-bit `_ + + .. tab-item:: ZYPPER + :name: zypper-sysreq + + .. tab-set:: + + .. tab-item:: Linux + :sync: linux + + OpenVINO RPM packages are compatible with and can be run on openSUSE Tumbleweed only. + + Software: + + * `CMake 3.13 or higher, 64-bit `_ + * GCC 8.2.0 + * `Python 3.9 - 3.12, 64-bit `_ The claims stated here may not apply to all use cases and setups. See -:doc:`Legal notices and terms of use <../additional-resources/terms-of-use>` for more information. \ No newline at end of file +:doc:`Legal notices and terms of use <../additional-resources/terms-of-use>` for more information. diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset1.rst index 55b29d30fa8502..1ce95152c20c90 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset1.rst @@ -66,7 +66,6 @@ Table of Contents * :doc:`LogicalXor <../operation-specs/logical/logical-xor-1>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-1>` * :doc:`Maximum <../operation-specs/arithmetic/maximum-1>` @@ -122,5 +121,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset10.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset10.rst index 04f151a3e26b84..56397ecd852305 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset10.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset10.rst @@ -107,7 +107,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-8>` @@ -192,4 +192,3 @@ Table of Contents * :doc:`Unique <../operation-specs/movement/unique-10>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset11.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset11.rst index 6ae5882e2e4fd7..8a918bde06ebb1 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset11.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset11.rst @@ -107,7 +107,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-8>` @@ -192,4 +192,3 @@ Table of Contents * :doc:`Unique <../operation-specs/movement/unique-10>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset12.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset12.rst index 6e6ddd61ba1e0e..06cfaa7f8f6a20 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset12.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset12.rst @@ -108,7 +108,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-8>` @@ -193,4 +193,3 @@ Table of Contents * :doc:`Unique <../operation-specs/movement/unique-10>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset13.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset13.rst index 87396dfd542203..6bcea43cdeb103 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset13.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset13.rst @@ -113,7 +113,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-8>` @@ -201,4 +201,3 @@ Table of Contents * :doc:`Unique <../operation-specs/movement/unique-10>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset14.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset14.rst index 05131a907d15b4..eb967a6fc94e11 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset14.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset14.rst @@ -115,7 +115,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-14>` diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset15.rst index b32d7b71580177..12368033a1fa20 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset15.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset15.rst @@ -120,7 +120,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-14>` @@ -198,7 +198,7 @@ Table of Contents * :doc:`Split <../operation-specs/movement/split-1>` * :doc:`Sqrt <../operation-specs/arithmetic/sqrt-1>` * :doc:`SquaredDifference <../operation-specs/arithmetic/squared-difference-1>` -* :doc:`Squeeze <../operation-specs/shape/squeeze-1>` +* :doc:`Squeeze <../operation-specs/shape/squeeze-15>` * :doc:`STFT <../operation-specs/signals/stft-15>` * :doc:`StridedSlice <../operation-specs/movement/strided-slice-1>` * :doc:`StringTensorPack <../operation-specs/type/string-tensor-pack-15>` diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset2.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset2.rst index 5ea1c3bde54018..41170ad2036a25 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset2.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset2.rst @@ -68,7 +68,6 @@ Table of Contents * :doc:`LogicalXor <../operation-specs/logical/logical-xor-1>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-1>` * :doc:`Maximum <../operation-specs/arithmetic/maximum-1>` @@ -128,5 +127,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset3.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset3.rst index fcce876e6944df..9b2992db18cc36 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset3.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset3.rst @@ -76,7 +76,6 @@ Table of Contents * :doc:`LogicalXor <../operation-specs/logical/logical-xor-1>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-1>` * :doc:`Maximum <../operation-specs/arithmetic/maximum-1>` @@ -144,4 +143,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset5.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset5.rst index 0c0ae150b1d21d..fdada28b6bdc06 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset5.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset5.rst @@ -86,7 +86,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-1>` * :doc:`Maximum <../operation-specs/arithmetic/maximum-1>` @@ -162,5 +162,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset6.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset6.rst index 8d626c71be2d89..d587370800aa9e 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset6.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset6.rst @@ -93,7 +93,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-1>` * :doc:`Maximum <../operation-specs/arithmetic/maximum-1>` @@ -168,4 +168,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset7.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset7.rst index 7dd5d8fbba14d7..6de9bccf5921b7 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset7.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset7.rst @@ -96,7 +96,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-1>` * :doc:`Maximum <../operation-specs/arithmetic/maximum-1>` @@ -172,4 +172,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset8.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset8.rst index 0bb5b336f9e2f2..f35299c9f4c97a 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset8.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset8.rst @@ -101,7 +101,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-8>` @@ -183,5 +183,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset9.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset9.rst index ebb5dfdbdc6555..2421e4b9bc53c1 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset9.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/available-opsets/opset9.rst @@ -105,7 +105,7 @@ Table of Contents * :doc:`Loop <../operation-specs/infrastructure/loop-5>` * :doc:`LRN <../operation-specs/normalization/lrn-1>` * :doc:`LSTMCell <../operation-specs/sequence/lstm-cell-1>` -* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-1>` +* :doc:`LSTMSequence <../operation-specs/sequence/lstm-sequence-5>` * :doc:`MatMul <../operation-specs/matrix/matmul-1>` * :doc:`MatrixNMS <../operation-specs/sort/matrix-non-max-suppression-8>` * :doc:`MaxPool <../operation-specs/pooling/max-pool-8>` @@ -189,4 +189,3 @@ Table of Contents * :doc:`Transpose <../operation-specs/movement/transpose-1>` * :doc:`Unsqueeze <../operation-specs/shape/unsqueeze-1>` * :doc:`VariadicSplit <../operation-specs/movement/variadic-split-1>` - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs.rst index 6ecbf2695699f9..9f7badf1a8d06a 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs.rst @@ -215,7 +215,7 @@ Operation Specifications Sin-1 Sinh-1 Slice-8 - SliceScatter + SliceScatter-15 SoftMax-1 SoftMax-8 SoftPlus-4 @@ -226,6 +226,7 @@ Operation Specifications Sqrt-1 SquaredDifference-1 Squeeze-1 + Squeeze-15 STFT-15 StridedSlice-1 StringTensorPack-15 diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-5.rst similarity index 98% rename from docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-1.rst rename to docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-5.rst index dcc21ca5321451..c00b4c819cc66a 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sequence/lstm-sequence-5.rst @@ -3,10 +3,10 @@ LSTMSequence .. meta:: - :description: Learn about LSTMSequence-1 - a sequence processing operation, which + :description: Learn about LSTMSequence-5 - a sequence processing operation, which can be performed on seven required input tensors. -**Versioned name**: *LSTMSequence-1* +**Versioned name**: *LSTMSequence-5* **Category**: *Sequence processing* @@ -145,4 +145,3 @@ A single cell in the sequence is implemented in the same way as in :doc:`LSTM Ce - diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/shape/squeeze-15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/shape/squeeze-15.rst new file mode 100644 index 00000000000000..1e112dce118e26 --- /dev/null +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/shape/squeeze-15.rst @@ -0,0 +1,174 @@ +Squeeze +======= + + +.. meta:: + :description: Learn about Squeeze-15 - a shape manipulation operation, which + can be performed on one required and one optional input tensor. + +**Versioned name**: *Squeeze-15* + +**Category**: *Shape manipulation* + +**Short description**: *Squeeze* removes dimensions equal to 1 from the first input tensor. + +**Detailed description**: *Squeeze* can be used with or without the second input tensor. + +* If only the first input is provided, every dimension that is equal to 1 will be removed from it. +* With the second input provided, each value is an index of a dimension from the first tensor that is to be removed. Specified dimension should be equal to 1, otherwise it will be ignored and copied as is. + Dimension indices can be specified directly, or by negative indices (counting dimensions from the end). + +.. note:: + + - If index of the dimension to squeeze is provided as a constant input and it points to a dynamic dimension that might be `1`, and the *allow_axis_skip* attribute is ``false``, then the dimension is considered as squeezable. Therefore the rank of the output shape will be reduced, but not dynamic. If dynamic rank is expected for such case, *allow_axis_skip* attribute need to be set to ``true``. + - If the input with indices is empty or not provided, dynamic dimension compatible with `1` leads to dynamic rank of the output shape. + + +**Attributes**: + +* *allow_axis_skip* + + * **Description**: If true, shape inference results in a dynamic rank if selected axis has value 1 in its dimension range. + * **Range of values**: ``false`` or ``true`` + * **Type**: ``boolean`` + * **Required**: *no* + * **Default value**: ``false`` + +**Inputs**: + +* **1**: Multidimensional input tensor of type *T*. **Required.** + +* **2**: Scalar or 1D tensor of type *T_INT* with indices of dimensions to squeeze. Values could be negative (have to be from range ``[-R, R-1]``, where ``R`` is the rank of the first input). **Optional.** + +**Outputs**: + +* **1**: Tensor with squeezed values of type *T*. + +**Types** + +* *T*: any numeric type. + +* *T_INT*: any supported integer type. + +**Example** + +*Example 1: squeeze 4D tensor to a 2D tensor* + +.. code-block:: xml + :force: + + + + + + 1 + 3 + 1 + 2 + + + + + 2 + + + + + 3 + 2 + + + + +*Example 2: squeeze 1D tensor with 1 element to a 0D tensor (constant)* + +.. code-block:: xml + :force: + + + + + + 1 + + + + + 1 + + + + + + + + +*Example 3: squeeze 1D tensor with 1 dynamic shape element to a fully dynamic shape* + +.. code-block:: xml + :force: + + + + + + -1 + + + + + 1 + + + + + + + +*Example 4: squeeze 2D tensor with dynamic and static shape elements to a static shape output, according to the opset1 rules* + +.. code-block:: xml + :force: + + + + + + 2 + -1 + + + + + 1 + + + + + 2 + + + + +*Example 5: squeeze 2D tensor with dynamic and static shape elements to a dynamic shape output, according to the opset15 rules* + +.. code-block:: xml + :force: + + + + + + 2 + -1 + + + + + 1 + + + + + + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/signals/stft-15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/signals/stft-15.rst index 4a41df7214317c..bcc420f5db25c9 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/signals/stft-15.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/signals/stft-15.rst @@ -14,14 +14,14 @@ Short Time Fourier Transformation for real-valued input (STFT) **Short description**: *STFT* operation performs Short-Time Fourier Transform (real-to-complex). -**Detailed description**: *STFT* performs Short-Time Fourier Transform of real-valued batched input tensor of shape ``[batch, signal_size]``, and produces complex result represented by separate values for real and imaginary part. +**Detailed description**: *STFT* performs Short-Time Fourier Transform of real-valued input tensor of shape ``[signal_size]`` or ``[batch, signal_size]``, and produces complex result represented by separate values for real and imaginary part. **Attributes**: -* *transform_frames* +* *transpose_frames* - * **Description**: Flag to set output shape layout. If true the ``frames`` dimension is at out_shape[2], otherwise it is at out_shape[1]. + * **Description**: Flag to set output shape layout. If true the ``frames`` dimension is at out_shape[-2], otherwise it is at out_shape[-3]. * **Range of values**: * ``false`` - do not transpose output shape @@ -31,25 +31,25 @@ Short Time Fourier Transformation for real-valued input (STFT) **Inputs** -* **1**: ``signal`` - Tensor of type *T* and 2D shape [batch, signal_size] with signal data for the STFT. **Required.** -* **2**: ``window`` - Tensor of type *T* and 1D shape [window_length], specifying the window values for the signal slice multiplication. **Required.** -* **3**: ``frame_size`` - Scalar tensor of type *T_INT* describing the size of a single frame of the signal to be provided as input to FFT. **Required.** -* **4**: ``frame_step`` - Scalar tensor of type *T_INT* describing The distance (number of samples) between successive frames. **Required.** +* **1**: ``signal`` - Tensor of type *T* and 1D shape [signal_size] or 2D shape [batch, signal_size] with signal data for the STFT. **Required.** +* **2**: ``window`` - Tensor of type *T* and 1D shape [window_length], specifying the window values for the signal slice multiplication. **Required.** +* **3**: ``frame_size`` - Scalar tensor of type *T_INT* describing the size of a single frame of the signal to be provided as input to FFT. **Required.** +* **4**: ``frame_step`` - Scalar tensor of type *T_INT* describing The distance (number of samples) between successive frames. **Required.** **Outputs** -* **1**: The result of STFT operation, tensor of the same type as input ``signal`` tensor and shape: +* **1**: The result of STFT operation, tensor of the same type as input ``signal`` tensor and shape: - + When ``transform_frames == false`` the output shape is ``[batch, frames, fft_results, 2]`` - + When ``transform_frames == true`` the output shape is ``[batch, fft_results, frames, 2]`` + * When ``transpose_frames == false`` the output shape is ``[frames, fft_results, 2]`` for 1D signal input or ``[batch, frames, fft_results, 2]`` for 2D signal input. + * When ``transpose_frames == true`` the output shape is ``[fft_results, frames, 2]`` for 1D signal input or ``[batch, fft_results, frames, 2]`` for 2D signal input. - where: + where: - + ``batch`` is a batch size dimension - + ``frames`` is a number calculated as ``(signal_shape[1] - frame_size) / frame_step) + 1`` - + ``fft_results`` is a number calculated as ``(frame_size / 2) + 1`` - + ``2`` is the last dimension is for complex value real and imaginary part + * ``batch`` is a batch size dimension + * ``frames`` is a number calculated as ``(signal_shape[-1] - frame_size) / frame_step) + 1`` + * ``fft_results`` is a number calculated as ``(frame_size / 2) + 1`` + * ``2`` is the last dimension is for complex value real and imaginary part **Types** @@ -59,27 +59,109 @@ Short Time Fourier Transformation for real-valued input (STFT) * *T_INT*: ``int64`` or ``int32``. -**Example**: +**Examples**: + +*Example 1D signal, transpose_frames=false:* .. code-block:: xml :force: + + 56 + + + 7 + + + + + + 16 + 6 2 - 48 + + + + + +*Example 1D signal, transpose_frames=true:* + +.. code-block:: xml + :force: + + + + + + 56 - 8 + 7 - - + + - + + 6 + 16 2 - 9 - 9 + + + + +*Example 2D signal, transpose_frames=false:* + +.. code-block:: xml + :force: + + + + + + 3 + 56 + + + 7 + + + + + + 3 + 16 + 6 + 2 + + + + + +*Example 2D signal, transpose_frames=true:* + +.. code-block:: xml + :force: + + + + + + 3 + 56 + + + 7 + + + + + + 3 + 6 + 16 2 diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/search-sorted-15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/search-sorted-15.rst index 81c592d3341a35..7a623a1e16739c 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/search-sorted-15.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/search-sorted-15.rst @@ -17,29 +17,32 @@ SearchSorted **Attributes** -* *right* +* *right_mode* - * **Description**: If False, set the first suitable index. If True, return the last suitable index for given value. Default is False. - * **Range of values**: true or false - * **Type**: boolean + * **Description**: flag to control whether output would contain leftmost or rightmost indices for given values. + * **Range of values**: + + * *true* - return the rightmost (last) suitable index for given value. + * *false* - return the leftmost (first) suitable index for given value. + * **Type**: ``boolean`` * **Default value**: false * **Required**: *no* **Inputs**: -* **1**: ``sorted`` - ND input tensor of type *T* - cannot be a scalar, containing monotonically increasing sequence on the innermost dimension. **Required.** +* **1**: ``sorted_sequence`` - ND input tensor of type *T* - cannot be a scalar, containing monotonically increasing sequence on the innermost dimension. **Required.** * **2**: ``values`` - ND input tensor of type *T*, containing the search values. If sorted sequence is 1D, then the values can have any shape, otherwise the rank should be equal to the rank of sorted input. **Required.** **Outputs**: -* **1**: Tensor of type *TOut*, with the same shape as second input tensor, containing the indices. +* **1**: Tensor of type *T_IND*, with the same shape as second input tensor ``values``, containing the indices. **Types** * *T*: any supported floating-point and integer type. -* *TOut*: int64. +* *T_IND*: ``int64``. **Example** @@ -47,7 +50,7 @@ SearchSorted :force: - + 7 @@ -63,7 +66,7 @@ SearchSorted - + 7 256 200 diff --git a/docs/articles_en/get-started/install-openvino.rst b/docs/articles_en/get-started/install-openvino.rst index 6dda915af93b69..7f26ab9ec72c9f 100644 --- a/docs/articles_en/get-started/install-openvino.rst +++ b/docs/articles_en/get-started/install-openvino.rst @@ -30,26 +30,6 @@ All currently supported versions are: * 2023.3 (LTS) * 2022.3 (LTS) -.. dropdown:: Distributions and Device Support - - Different OpenVINO distributions may support slightly different sets of features. - Read installation guides for particular distributions for more details. - Refer to the :doc:`OpenVINO Release Policy <../../../about-openvino/release-notes-openvino/release-policy>` - to learn more about the release types. - - .. dropdown:: Distribution Comparison for OpenVINO 2024.4 - - =============== ========== ====== =============== ======== ============ ========== ========== ========== - Device Archives PyPI APT/YUM/ZYPPER Conda Homebrew vcpkg Conan npm - =============== ========== ====== =============== ======== ============ ========== ========== ========== - CPU V V V V V V V V - GPU V V V V V V V V - NPU V\* V\* V\ * n/a n/a n/a n/a V\* - =============== ========== ====== =============== ======== ============ ========== ========== ========== - - | \* **Of the Linux systems, versions 22.04 and 24.04 include drivers for NPU.** - | **For Windows, CPU inference on ARM64 is not supported.** - .. dropdown:: Effortless GenAI integration with OpenVINO GenAI Flavor A new OpenVINO GenAI Flavor streamlines application development by providing diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst b/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst index 511ecc627d45ad..21a35a14047dca 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst @@ -14,28 +14,8 @@ Install Intel® Distribution of OpenVINO™ Toolkit for Linux Using APT Reposito * is dedicated to Linux users only * additionally includes code samples - -.. tab-set:: - - .. tab-item:: System Requirements - :sync: system-requirements - - | Full requirement listing is available in: - | :doc:`System Requirements Page <../../../about-openvino/release-notes-openvino/system-requirements>` - - .. tab-item:: Processor Notes - :sync: processor-notes - - | To see if your processor includes the integrated graphics technology and supports iGPU inference, refer to: - | `Product Specifications `__ - - .. tab-item:: Software Requirements - :sync: software-requirements - - * `CMake 3.13 or higher, 64-bit `__ - * GCC 7.5.0 (for Ubuntu 18.04), GCC 9.3.0 (for Ubuntu 20.04) or GCC 11.3.0 (for Ubuntu 22.04) - * `Python 3.9 - 3.12, 64-bit `__ - + Before installing OpenVINO, see the + :doc:`System Requirements page <../../../about-openvino/release-notes-openvino/system-requirements>`. Installing OpenVINO Runtime ####################################### diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst index 5b40422115179c..e777c06253a37a 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst @@ -30,58 +30,8 @@ Install OpenVINO™ Runtime on Linux from an Archive File RHEL8 x86_64 V V n/a =================== ===== ===== ===== -.. tab-set:: - - .. tab-item:: System Requirements - :sync: system-requirements - - | Full requirement listing is available in: - | :doc:`System Requirements Page <../../../about-openvino/release-notes-openvino/system-requirements>` - - .. tab-item:: Processor Notes - :sync: processor-notes - - | To see if your processor includes the integrated graphics technology and supports iGPU inference, refer to: - | `Product Specifications `__ - - .. tab-item:: Software - :sync: software - - * `CMake 3.13 or higher, 64-bit `__ - * `Python 3.9 - 3.12, 64-bit `__ - * GCC: - - .. tab-set:: - - .. tab-item:: Ubuntu 20.04 - :sync: ubuntu-20 - - * GCC 9.3.0 - - .. tab-item:: RHEL 8 - :sync: rhel-8 - - * GCC 8.4.1 - - .. tab-item:: CentOS 7 - :sync: centos-7 - - * GCC 8.3.1 - Use the following instructions to install it: - - Install GCC 8.3.1 via devtoolset-8 - - .. code-block:: sh - - sudo yum update -y && sudo yum install -y centos-release-scl epel-release - sudo yum install -y devtoolset-8 - - Enable devtoolset-8 and check current gcc version - - .. code-block:: sh - - source /opt/rh/devtoolset-8/enable - gcc -v + Before installing OpenVINO, see the + :doc:`System Requirements page <../../../about-openvino/release-notes-openvino/system-requirements>`. Installing OpenVINO Runtime diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst index 20f5df9b30a9d1..e9157a99e1c882 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst @@ -16,22 +16,8 @@ Install OpenVINO™ Runtime on macOS from an Archive File * is dedicated to macOS users (archives for other systems are also available) * is only supported for CPU Plugin - -.. tab-set:: - - .. tab-item:: System Requirements - :sync: system-requirements - - | Full requirement listing is available in: - | :doc:`System Requirements Page <../../../about-openvino/release-notes-openvino/system-requirements>` - - .. tab-item:: Software Requirements - :sync: software-requirements - - * `CMake 3.13 or higher `__ (choose "macOS 10.13 or later"). Add ``/Applications/CMake.app/Contents/bin`` to path (for default install). - * `Python 3.9 - 3.12 `__ (choose 3.9 - 3.12). Install and add to path. - * Apple Xcode Command Line Tools. In the terminal, run ``xcode-select --install`` from any directory - * (Optional) Apple Xcode IDE (not required for OpenVINO™, but useful for development) + Before installing OpenVINO, see the + :doc:`System Requirements page <../../../about-openvino/release-notes-openvino/system-requirements>`. Installing OpenVINO Runtime diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst index bcc03f37d74295..8f3efeeb720dc9 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst @@ -15,43 +15,8 @@ Install OpenVINO™ Runtime on Windows from an Archive File * additionally includes code samples * is dedicated to Windows users (archives for other systems are also available) - -System Requirements -#################### - -.. tab-set:: - - .. tab-item:: System Requirements - :sync: system-requirements - - | Full requirement listing is available in: - | :doc:`System Requirements Page <../../../about-openvino/release-notes-openvino/system-requirements>` - - .. tab-item:: Processor Notes - :sync: processor-notes - - | To see if your processor includes the integrated graphics technology and supports iGPU inference, refer to: - | `Product Specifications `__ - - .. tab-item:: Software - :sync: software - - * `Microsoft Visual Studio 2019 with MSBuild `__ or `Microsoft Visual Studio 2022 `__ - * `CMake 3.14 or higher, 64-bit `__ (optional, only required for building sample applications) - * `Python 3.9 - 3.12, 64-bit `__ - - .. note:: - - To install Microsoft Visual Studio 2019, follow the `Microsoft Visual Studio installation guide `__. You can choose to download the Community version. During installation in the **Workloads** tab, choose **Desktop development with C++**. - - .. note:: - - You can either use `cmake.msi` which is the installation wizard or `cmake.zip` where you have to go into the `bin` folder and then manually add the path to environmental variables. - - .. important:: - - When installing Python, make sure you click the option **Add Python 3.x to PATH** to `add Python `__ to your `PATH` environment variable. - + Before installing OpenVINO, see the + :doc:`System Requirements page <../../../about-openvino/release-notes-openvino/system-requirements>`. Installing OpenVINO Runtime diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-brew.rst b/docs/articles_en/get-started/install-openvino/install-openvino-brew.rst index f111c0f18e3fec..b1710f3bb358e8 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-brew.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-brew.rst @@ -14,43 +14,8 @@ Install OpenVINO™ Runtime via Homebrew * does not offer support for NPU inference * is dedicated to macOS (both arm64 and x86_64) and Linux (x86_64 only) users. - -.. tab-set:: - - .. tab-item:: System Requirements - :sync: system-requirements - - | Full requirement listing is available in: - | :doc:`System Requirements Page <../../../about-openvino/release-notes-openvino/system-requirements>` - - .. tab-item:: Processor Notes - :sync: processor-notes - - | To see if your processor includes the integrated graphics technology and supports iGPU inference, refer to: - | `Product Specifications `__ - - .. tab-item:: Software Requirements - :sync: software-requirements - - .. tab-set:: - - .. tab-item:: Linux - :sync: linux - - * `Homebrew `_ - * `CMake 3.13 or higher, 64-bit `__ - * GCC 7.5.0 (for Ubuntu 18.04), GCC 9.3.0 (for Ubuntu 20.04) or GCC 11.3.0 (for Ubuntu 22.04) - * `Python 3.9 - 3.12, 64-bit `__ - - .. tab-item:: macOS - :sync: macos - - * `Homebrew `_ - * `CMake 3.13 or higher `__ (choose "macOS 10.13 or later"). Add ``/Applications/CMake.app/Contents/bin`` to path (for default installation). - * `Python 3.9 - 3.12 `__ . Install and add it to path. - * Apple Xcode Command Line Tools. In the terminal, run ``xcode-select --install`` from any directory to install it. - * (Optional) Apple Xcode IDE (not required for OpenVINO™, but useful for development) - + Before installing OpenVINO, see the + :doc:`System Requirements page <../../../about-openvino/release-notes-openvino/system-requirements>`. Installing OpenVINO Runtime ########################### diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-conan.rst b/docs/articles_en/get-started/install-openvino/install-openvino-conan.rst index 4cf32c7fc474c2..06557003b3cbf6 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-conan.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-conan.rst @@ -15,26 +15,8 @@ Install OpenVINO™ Runtime from Conan Package Manager * is dedicated to users of all major OSes: Windows, Linux, and macOS (all x86_64 / arm64 architectures) - -.. tab-set:: - - .. tab-item:: System Requirements - :sync: system-requirements - - | Full requirement listing is available in: - | :doc:`System Requirements Page <../../../about-openvino/release-notes-openvino/system-requirements>` - - .. tab-item:: Processor Notes - :sync: processor-notes - - | To see if your processor includes the integrated graphics technology and supports iGPU inference, refer to: - | `Product Specifications `__ - - .. tab-item:: Software - :sync: software - - | There are many ways to work with Conan Package Manager. Before you proceed, learn more about it on the - | `Conan distribution page `__ + Before installing OpenVINO, see the + :doc:`System Requirements page <../../../about-openvino/release-notes-openvino/system-requirements>`. Installing OpenVINO Runtime with Conan Package Manager ############################################################ diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-conda.rst b/docs/articles_en/get-started/install-openvino/install-openvino-conda.rst index e551971fd98f92..d1392d3f46a513 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-conda.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-conda.rst @@ -16,28 +16,8 @@ Install OpenVINO™ Runtime from Conda Forge * is dedicated to users of all major OSes: Windows, Linux, and macOS (all x86_64 / arm64 architectures) -.. tab-set:: - - .. tab-item:: System Requirements - :sync: system-requirements - - | Full requirement listing is available in: - | :doc:`System Requirements Page <../../../about-openvino/release-notes-openvino/system-requirements>` - - - .. tab-item:: Processor Notes - :sync: processor-notes - - | To see if your processor includes the integrated graphics technology and supports iGPU inference, refer to: - | `Product Specifications `__ - - - .. tab-item:: Software - :sync: software - - | There are many ways to work with Conda. Before you proceed, learn more about it on the - | `Anaconda distribution page `__ - + Before installing OpenVINO, see the + :doc:`System Requirements page <../../../about-openvino/release-notes-openvino/system-requirements>`. Installing OpenVINO Runtime with Anaconda Package Manager ############################################################ diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-docker-linux.rst b/docs/articles_en/get-started/install-openvino/install-openvino-docker-linux.rst index 38795bd529c082..de7b2ed3ff622e 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-docker-linux.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-docker-linux.rst @@ -6,12 +6,7 @@ Install Intel® Distribution of OpenVINO™ Toolkit From a Docker Image manually to install OpenVINO™ Runtime on Linux and Windows operating systems. This guide presents information on how to use a pre-built Docker image or create a new image -manually, to install OpenVINO™ Runtime. The supported host operating systems for the Docker -base image are: - -- Linux -- Windows (WSL2) -- macOS (CPU exectuion only) +manually, to install OpenVINO™ Runtime. You can get started easily with pre-built and published docker images, which are available at: @@ -34,11 +29,25 @@ The Docker CI repository includes guides on how to `get started with docker images `__ and how to use `OpenVINO™ Toolkit containers with GPU accelerators. `__ -To start using Dockerfiles the following conditions must be met: +To start using Dockerfiles, install Docker Engine or a compatible container +engine on your system: + +.. tab-set:: + + .. tab-item:: Linux + :sync: linux + + * `Docker Desktop `__ + * `Docker Engine `__ + + .. tab-item:: Windows (WSL2) + :sync: win + + OpenVINO can be installed under :ref:`Windows Subsystem for Linux (WSL2) `. + + * `Docker Desktop `__ -- Linux OS or Windows (under :ref:`Windows Subsystem for Linux (WSL2) `) -- Installed docker engine or compatible container engine -- Permissions to run containers (sudo or docker group membership) +Also, verify you have permissions to run containers (sudo or docker group membership). .. note:: diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-npm.rst b/docs/articles_en/get-started/install-openvino/install-openvino-npm.rst index 7d811c69e5991c..5060ccfc654229 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-npm.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-npm.rst @@ -15,19 +15,8 @@ Install Intel® Distribution of OpenVINO™ Toolkit from npm Registry (all x86_64 / arm64 architectures) * macOS offers support only for CPU inference -.. tab-set:: - - .. tab-item:: System Requirements - :sync: system-requirements - - - Windows, Linux, macOS - - x86, ARM (Windows ARM not supported) - - .. tab-item:: Software Requirements - :sync: software-requirements - - `Node.js version 21.0.0 and higher `__ - + Before installing OpenVINO, see the + :doc:`System Requirements page <../../../about-openvino/release-notes-openvino/system-requirements>`. Installing OpenVINO Node.js ########################### diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-pip.rst b/docs/articles_en/get-started/install-openvino/install-openvino-pip.rst index c079f167761ada..6326513fa3cea1 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-pip.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-pip.rst @@ -16,26 +16,8 @@ Install Intel® Distribution of OpenVINO™ Toolkit from PyPI Repository (all x86_64 / arm64 architectures) * macOS offers support only for CPU inference - -.. tab-set:: - - .. tab-item:: System Requirements - :sync: system-requirements - - | Full requirement listing is available in: - | :doc:`System Requirements Page <../../../about-openvino/release-notes-openvino/system-requirements>` - | `PyPI OpenVINO page `__ - - - .. tab-item:: Processor Notes - :sync: processor-notes - - | To see if your processor includes the integrated graphics technology and supports iGPU - inference, refer to: - | `Product Specifications `__ - - - + Before installing OpenVINO, see the + :doc:`System Requirements page <../../../about-openvino/release-notes-openvino/system-requirements>`. Installing OpenVINO Runtime ########################### diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-vcpkg.rst b/docs/articles_en/get-started/install-openvino/install-openvino-vcpkg.rst index 747d4d873057f6..af9fe85528ca5d 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-vcpkg.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-vcpkg.rst @@ -15,26 +15,8 @@ Install OpenVINO™ Runtime via vcpkg * is dedicated to users of all major OSes: Windows, Linux, and macOS (all x86_64 / arm64 architectures) -.. tab-set:: - - .. tab-item:: System Requirements - :sync: system-requirements - - | Full requirement listing is available in: - | :doc:`System Requirements Page <../../../about-openvino/release-notes-openvino/system-requirements>` - - .. tab-item:: Processor Notes - :sync: processor-notes - - | To see if your processor includes the integrated graphics technology and supports iGPU inference, refer to: - | `Product Specifications `__ - - .. tab-item:: Software Requirements - :sync: software-requirements - - * `vcpkg `__ - - + Before installing OpenVINO, see the + :doc:`System Requirements page <../../../about-openvino/release-notes-openvino/system-requirements>`. Installing OpenVINO Runtime ########################### diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-yum.rst b/docs/articles_en/get-started/install-openvino/install-openvino-yum.rst index 6775495fcd2a30..970bb47a095d5b 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-yum.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-yum.rst @@ -15,41 +15,8 @@ Install OpenVINO™ Runtime on Linux From YUM Repository * is dedicated to Linux users only * additionally includes code samples -.. tab-set:: - - .. tab-item:: System Requirements - :sync: system-requirements - - | Full requirement listing is available in: - | :doc:`System Requirements Page <../../../about-openvino/release-notes-openvino/system-requirements>` - - .. note:: - - OpenVINO RPM packages are compatible with and can be run on the following operating systems: - - - RHEL 8.2 and higher - - Amazon Linux 2022 and 2023 - - Rocky Linux 8.7, 8.8 and 9.2-9.3 - - Alma Linux 8.7, 8.8 and 9.2-9.4 - - Oracle Linux 8.7, 8.8 and 9.2-9.4 - - Fedora 29 and higher up to 41 - - OpenEuler 20.03, 22.03, 23.03 and 24.03 - - Anolis OS 8.6 and 8.8 - - CentOS Stream 8 and 9 - - .. tab-item:: Processor Notes - :sync: processor-notes - - | To see if your processor includes the integrated graphics technology and supports iGPU inference, refer to: - | `Product Specifications `__ - - .. tab-item:: Software - :sync: software - - * `CMake 3.13 or higher, 64-bit `_ - * GCC 8.2.0 - * `Python 3.9 - 3.12, 64-bit `_ - + Before installing OpenVINO, see the + :doc:`System Requirements page <../../../about-openvino/release-notes-openvino/system-requirements>`. Install OpenVINO Runtime ######################## diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-zypper.rst b/docs/articles_en/get-started/install-openvino/install-openvino-zypper.rst index 8a1ceff7271187..127b26cac0590f 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-zypper.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-zypper.rst @@ -15,33 +15,8 @@ Install OpenVINO™ Runtime on Linux From ZYPPER Repository * is dedicated to Linux users only * additionally includes code samples -.. tab-set:: - - .. tab-item:: System Requirements - :sync: system-requirements - - | Full requirement listing is available in: - | :doc:`System Requirements Page <../../../about-openvino/release-notes-openvino/system-requirements>` - - .. note:: - - OpenVINO RPM packages are compatible with and can be run on the following operating systems: - - - openSUSE Tumbleweed - - .. tab-item:: Processor Notes - :sync: processor-notes - - | To see if your processor includes the integrated graphics technology and supports iGPU inference, refer to: - | `Product Specifications `__ - - .. tab-item:: Software - :sync: software - - * `CMake 3.13 or higher, 64-bit `_ - * GCC 8.2.0 - * `Python 3.9 - 3.12, 64-bit `_ - + Before installing OpenVINO, see the + :doc:`System Requirements page <../../../about-openvino/release-notes-openvino/system-requirements>`. Install OpenVINO Runtime ######################## diff --git a/docs/dev/build_riscv64.md b/docs/dev/build_riscv64.md index 13ab9e98d56958..19d6e8714b45fc 100644 --- a/docs/dev/build_riscv64.md +++ b/docs/dev/build_riscv64.md @@ -16,15 +16,23 @@ The software was validated on the following devices: - Python 3.10 for OpenVINO Runtime Python API ## How to build +Currently, there are three ways to build OpenVINO Runtime for 64-bit RISC-V platforms: + +1. **Recommended**. The build with vectorized (using RVV instructions) primitives for limited scope of operations from [`SHL`](https://github.com/XUANTIE-RV/csi-nn2) using [`xuantie-gnu-toolchain`](https://github.com/XUANTIE-RV/). This GNU Compiler Toolchain supports RVV 0.7.1, ratified RVV 1.0 and Xuantie-specific instruction sets. The vector intrinsics don't use the common prefix `__riscv_`. This method provides the best performance available at the moment. +2. The build without optimized primitives using [`riscv-gnu-toolchain`](https://github.com/riscv-collab/riscv-gnu-toolchain.git). This GNU Compiler Toolchain supports RVV 0.7.1 and ratified RVV 1.0. The vector intrinsics use the common prefix `__riscv_`. However, as mentioned earlier, this build method doesn't yet provide optimized primitives implemented using the RVV intrinsics. +3. The build without optimized primitives using installed Linux packages. The compilers in these packages don't support RVV intrinsics. + +### Steps + 0. Prerequisite: -- For target with RVV - build `xuantie-gnu-toolchain` and `qemu`: +- For target with vectorized primitives from `SHL` - build `xuantie-gnu-toolchain` and `qemu`: ```sh git clone https://github.com/XUANTIE-RV/xuantie-gnu-toolchain.git cd xuantie-gnu-toolchain ./configure --prefix= make linux build-qemu -j$(nproc) ``` -- For target without RVV - build `riscv-gnu-toolchain`: +- For target without optimized primitives using `riscv-gnu-toolchain`: ```sh git clone https://github.com/riscv-collab/riscv-gnu-toolchain.git cd riscv-gnu-toolchain @@ -32,6 +40,11 @@ The software was validated on the following devices: make linux build-qemu -j$(nproc) ``` > **NOTE**: The `build-qemu` target is optional, as it is used to build the `qemu` simulator. However, it is recommended to build the `qemu` simulator, since it is much more convenient to validate the software on your host than on your devices. More information can be seen [here](https://github.com/riscv-collab/riscv-gnu-toolchain). +- For target without optimized primitives using installed Linux packages: + ```sh + apt-get update + apt-get install -y gcc-riscv64-linux-gnu g++-riscv64-linux-gnu binutils-riscv64-linux-gnu + ``` 1. Clone OpenVINO repository and init submodules: ```sh @@ -50,8 +63,8 @@ The software was validated on the following devices: mkdir build && cd build ``` -4. To cross compile OpenVINO Runtime for RISC-V devices, run `cmake` with specified `CMAKE_TOOLCHAIN_FILE` and `RISCV_TOOLCHAIN_ROOT`. -- For target with RVV: +4. To cross compile OpenVINO Runtime for RISC-V devices, run `cmake` with specified `CMAKE_TOOLCHAIN_FILE` and `RISCV_TOOLCHAIN_ROOT` (the last one is needed only for build using GNU toolchain). +- For target with vectorized primitives from `SHL`: ```sh cmake .. \ -DCMAKE_BUILD_TYPE=Release \ @@ -59,8 +72,8 @@ The software was validated on the following devices: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/ \ -DRISCV_TOOLCHAIN_ROOT= ``` - > **NOTE**: To build OpenVINO Runtime for different versions of RVV, you just need to specify corresponding toolchain files. For exmaple, you can replace `` with `riscv64-071-thead-gnu.toolchain.cmake` for RVV 0.7.1 and `riscv64-100-thead-gnu.toolchain.cmake` for RVV 1.0 respectively. -- For target without RVV: + > **NOTE**: To build OpenVINO Runtime for different versions of RVV, you just need to specify corresponding toolchain files. For example, you can replace `` with `riscv64-071-xuantie-gnu.toolchain.cmake` for RVV 0.7.1 and `riscv64-100-xuantie-gnu.toolchain.cmake` for RVV 1.0 respectively. +- For target without optimized primitives using `riscv-gnu-toolchain`: ```sh cmake .. \ -DCMAKE_BUILD_TYPE=Release \ @@ -69,7 +82,13 @@ The software was validated on the following devices: -DRISCV_TOOLCHAIN_ROOT=/opt/riscv ``` > **NOTE**: The `riscv-gnu-toolchain` is build as there are essential files used for cross compilation under `/opt/riscv/sysroot`. The latest stable versions of Clang or GCC both support compiling source code into RISC-V instructions, so it is acceptable to choose your preferable compilers by specifying `-DCMAKE_C_COMPILER` and `CMAKE_CXX_COMPILER`. But remember to add the key `-DCMAKE_SYSROOT=/opt/riscv/sysroot`, otherwise many fundamental headers and libs could not be found during cross compilation. - +- For target without optimized primitives using installed Linux packages: + ```sh + cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX= \ + -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/riscv64.linux.toolchain.cmake + ``` > **NOTE**: By default OpenVINO is built with OpenMP support on RISC-V devices. Then run `make` to build the project: diff --git a/docs/dev/build_windows.md b/docs/dev/build_windows.md index 10049485202cca..4a9761f5364046 100644 --- a/docs/dev/build_windows.md +++ b/docs/dev/build_windows.md @@ -25,7 +25,7 @@ Supported configurations: ```sh git clone https://github.com/openvinotoolkit/openvino.git cd openvino - git submodule update --init + git submodule update --init --recursive ``` 2. Create build directory: diff --git a/docs/dev/ci/commit_signoff_policy.md b/docs/dev/ci/commit_signoff_policy.md new file mode 100644 index 00000000000000..0328d3c3ec308c --- /dev/null +++ b/docs/dev/ci/commit_signoff_policy.md @@ -0,0 +1,74 @@ +# How to sign-off commits + +We require a sign-off commit message in the following format on each commit in pull request. + +``` +This is a commit message. + +Signed-off-by: Author Name +``` + +## How to sign-off new commits + +In a local Git environment, the sign-off message can be added to a commit either manually (as a text) +or via the **-s** flag used with the “git commit” command, for example: + +`git commit -s -m "My commit message"` + +To avoid manually adding the flag for each commit, we recommend setting up a Git hook with the following steps: + +1. Navigate to the `/.git/hooks` folder. +2. Open the `prepare-commit-msg.sample` file and paste the following content: + +``` +#!/bin/sh + +COMMIT_MSG_FILE=$1 +COMMIT_SOURCE=$2 +SHA1=$3 + +NAME=$(git config user.name) +EMAIL=$(git config user.email) + +if [ -z "$NAME" ]; then + echo "empty git config user.name" + exit 1 +fi + +if [ -z "$EMAIL" ]; then + echo "empty git config user.email" + exit 1 +fi + +git interpret-trailers --if-exists doNothing --trailer \ + "Signed-off-by: $NAME <$EMAIL>" \ + --in-place "$1" +``` + +3. Save the file with the name `prepare-commit-msg` (remove the .sample extension). +4. Make the file executable (on Linux / Git Bash: `chmod +x /.git/hooks/prepare-commit-msg`). + +**Note**: For both sign-off approaches, ensure your user name and email address are configured in Git first: + +``` +git config user.name 'FIRST_NAME LAST_NAME' +git config user.email 'MY_EMAIL@example.com' +``` + +### Sign-off web-based commits + +To enable automatic sign-off of commits made via GitHub web interface, make sure that +[Require contributors to sign off on web-based commits](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/managing-repository-settings/managing-the-commit-signoff-policy-for-your-repository#enabling-or-disabling-compulsory-commit-signoffs-for-your-repository) +setting is selected in the Settings menu of your OpenVINO repository fork. + +## How to sign-off older commits in the history + +If you forget to add the sign-off to your last commit, you can amend it and force-push to GitHub: + +``` +git commit --amend --signoff +``` + +To sign off on even older commits, use an interactive rebase, edit unsigned commits, and execute +`git commit --amend --signoff` for each. However, please note that if others have already started working based on +the commits in this branch, this will rewrite history and may cause issues for collaborators. diff --git a/docs/documentation_build_instructions.md b/docs/documentation_build_instructions.md index d9219454b86a19..a1412cfc3f358d 100644 --- a/docs/documentation_build_instructions.md +++ b/docs/documentation_build_instructions.md @@ -26,7 +26,7 @@ $ source env/bin/activate ``` 5. Install the sphinx theme ``` -(env) $ cd docs/openvino_sphinx_theme && python setup.py install && cd - +(env) $ python -m pip install docs/openvino_sphinx_theme `````` 6. Install the custom sphinx sitemap ``` diff --git a/docs/nbdoc/consts.py b/docs/nbdoc/consts.py index 75c8111bafcc6b..bfad4b042e5359 100644 --- a/docs/nbdoc/consts.py +++ b/docs/nbdoc/consts.py @@ -6,7 +6,7 @@ repo_owner = "openvinotoolkit" repo_name = "openvino_notebooks" repo_branch = "tree/main" -artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20241022220806/dist/rst_files/" +artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20241104220807/dist/rst_files/" blacklisted_extensions = ['.xml', '.bin'] notebooks_repo = "https://github.com/openvinotoolkit/openvino_notebooks/blob/latest/" notebooks_binder = "https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=" diff --git a/docs/notebooks/3D-pose-estimation-with-output.rst b/docs/notebooks/3D-pose-estimation-with-output.rst index e5e8e3813b5173..f39aa93b36851d 100644 --- a/docs/notebooks/3D-pose-estimation-with-output.rst +++ b/docs/notebooks/3D-pose-estimation-with-output.rst @@ -93,7 +93,7 @@ Lab instead.** .. code:: ipython3 - %pip install pythreejs "openvino-dev>=2024.0.0" "opencv-python" "torch" "onnx<1.16.2" --extra-index-url https://download.pytorch.org/whl/cpu + %pip install pythreejs "openvino>=2024.4.0" "opencv-python" "torch" "tqdm" --extra-index-url https://download.pytorch.org/whl/cpu .. parsed-literal:: @@ -101,85 +101,75 @@ Lab instead.** Looking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cpu Collecting pythreejs Using cached pythreejs-2.4.2-py3-none-any.whl.metadata (5.4 kB) - Collecting openvino-dev>=2024.0.0 - Using cached openvino_dev-2024.4.0-16579-py3-none-any.whl.metadata (16 kB) + Collecting openvino>=2024.4.0 + Using cached openvino-2024.4.0-16579-cp38-cp38-manylinux2014_x86_64.whl.metadata (8.3 kB) Collecting opencv-python Using cached opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (20 kB) Collecting torch Using cached https://download.pytorch.org/whl/cpu/torch-2.4.1%2Bcpu-cp38-cp38-linux_x86_64.whl (194.9 MB) - Collecting onnx<1.16.2 - Using cached onnx-1.16.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (16 kB) - Requirement already satisfied: ipywidgets>=7.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (8.1.5) + Collecting tqdm + Using cached tqdm-4.66.6-py3-none-any.whl.metadata (57 kB) + Requirement already satisfied: ipywidgets>=7.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (8.1.5) Collecting ipydatawidgets>=1.1.1 (from pythreejs) Using cached ipydatawidgets-4.3.5-py2.py3-none-any.whl.metadata (1.4 kB) Collecting numpy (from pythreejs) Using cached numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.6 kB) - Requirement already satisfied: traitlets in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (5.14.3) - Requirement already satisfied: defusedxml>=0.7.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (0.7.1) - Collecting networkx<=3.1.0 (from openvino-dev>=2024.0.0) - Using cached networkx-3.1-py3-none-any.whl.metadata (5.3 kB) - Collecting openvino-telemetry>=2023.2.1 (from openvino-dev>=2024.0.0) + Requirement already satisfied: traitlets in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (5.14.3) + Collecting openvino-telemetry>=2023.2.1 (from openvino>=2024.4.0) Using cached openvino_telemetry-2024.1.0-py3-none-any.whl.metadata (2.3 kB) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (24.1) - Requirement already satisfied: pyyaml>=5.4.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (6.0.2) - Requirement already satisfied: requests>=2.25.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (2.32.0) - Collecting openvino==2024.4.0 (from openvino-dev>=2024.0.0) - Using cached openvino-2024.4.0-16579-cp38-cp38-manylinux2014_x86_64.whl.metadata (8.3 kB) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2024.4.0) (24.1) Collecting filelock (from torch) Using cached filelock-3.16.1-py3-none-any.whl.metadata (2.9 kB) - Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (4.12.2) + Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (4.12.2) Collecting sympy (from torch) Using cached sympy-1.13.3-py3-none-any.whl.metadata (12 kB) - Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1.4) + Collecting networkx (from torch) + Using cached https://download.pytorch.org/whl/networkx-3.2.1-py3-none-any.whl (1.6 MB) + Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1.4) Collecting fsspec (from torch) Using cached fsspec-2024.10.0-py3-none-any.whl.metadata (11 kB) - Collecting protobuf>=3.20.2 (from onnx<1.16.2) - Using cached protobuf-5.28.2-cp38-abi3-manylinux2014_x86_64.whl.metadata (592 bytes) Collecting traittypes>=0.2.0 (from ipydatawidgets>=1.1.1->pythreejs) Using cached traittypes-0.2.1-py2.py3-none-any.whl.metadata (1.0 kB) - Requirement already satisfied: comm>=0.1.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (0.2.2) - Requirement already satisfied: ipython>=6.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (8.12.3) - Requirement already satisfied: widgetsnbextension~=4.0.12 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (4.0.13) - Requirement already satisfied: jupyterlab-widgets~=3.0.12 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (3.0.13) - Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (3.4.0) - Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (3.10) - Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (2.2.3) - Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (2024.8.30) - Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch) (2.1.5) + Requirement already satisfied: comm>=0.1.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (0.2.2) + Requirement already satisfied: ipython>=6.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (8.12.3) + Requirement already satisfied: widgetsnbextension~=4.0.12 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (4.0.13) + Requirement already satisfied: jupyterlab-widgets~=3.0.12 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (3.0.13) + Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch) (2.1.5) + INFO: pip is looking at multiple versions of networkx to determine which version is compatible with other requirements. This could take a while. + Collecting networkx (from torch) + Using cached networkx-3.1-py3-none-any.whl.metadata (5.3 kB) Collecting mpmath<1.4,>=1.1.0 (from sympy->torch) Using cached https://download.pytorch.org/whl/mpmath-1.3.0-py3-none-any.whl (536 kB) - Requirement already satisfied: backcall in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.0) - Requirement already satisfied: decorator in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (5.1.1) - Requirement already satisfied: jedi>=0.16 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.19.1) - Requirement already satisfied: matplotlib-inline in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.1.7) - Requirement already satisfied: pickleshare in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.5) - Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (3.0.48) - Requirement already satisfied: pygments>=2.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.18.0) - Requirement already satisfied: stack-data in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.6.3) - Requirement already satisfied: pexpect>4.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (4.9.0) - Requirement already satisfied: parso<0.9.0,>=0.8.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.8.4) - Requirement already satisfied: ptyprocess>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.0) - Requirement already satisfied: wcwidth in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.13) - Requirement already satisfied: executing>=1.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.1.0) - Requirement already satisfied: asttokens>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.4.1) - Requirement already satisfied: pure-eval in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.3) - Requirement already satisfied: six>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (1.16.0) + Requirement already satisfied: backcall in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.0) + Requirement already satisfied: decorator in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (5.1.1) + Requirement already satisfied: jedi>=0.16 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.19.1) + Requirement already satisfied: matplotlib-inline in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.1.7) + Requirement already satisfied: pickleshare in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.5) + Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (3.0.48) + Requirement already satisfied: pygments>=2.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.18.0) + Requirement already satisfied: stack-data in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.6.3) + Requirement already satisfied: pexpect>4.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (4.9.0) + Requirement already satisfied: parso<0.9.0,>=0.8.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.8.4) + Requirement already satisfied: ptyprocess>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.0) + Requirement already satisfied: wcwidth in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.13) + Requirement already satisfied: executing>=1.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.1.0) + Requirement already satisfied: asttokens>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.4.1) + Requirement already satisfied: pure-eval in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.3) + Requirement already satisfied: six>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (1.16.0) Using cached pythreejs-2.4.2-py3-none-any.whl (3.4 MB) - Using cached openvino_dev-2024.4.0-16579-py3-none-any.whl (4.7 MB) Using cached openvino-2024.4.0-16579-cp38-cp38-manylinux2014_x86_64.whl (42.6 MB) Using cached opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (62.5 MB) - Using cached onnx-1.16.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (15.9 MB) + Using cached tqdm-4.66.6-py3-none-any.whl (78 kB) Using cached ipydatawidgets-4.3.5-py2.py3-none-any.whl (271 kB) - Using cached networkx-3.1-py3-none-any.whl (2.1 MB) Using cached numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB) Using cached openvino_telemetry-2024.1.0-py3-none-any.whl (23 kB) - Using cached protobuf-5.28.2-cp38-abi3-manylinux2014_x86_64.whl (316 kB) Using cached filelock-3.16.1-py3-none-any.whl (16 kB) Using cached fsspec-2024.10.0-py3-none-any.whl (179 kB) + Using cached networkx-3.1-py3-none-any.whl (2.1 MB) Using cached sympy-1.13.3-py3-none-any.whl (6.2 MB) Using cached traittypes-0.2.1-py2.py3-none-any.whl (8.6 kB) - Installing collected packages: openvino-telemetry, mpmath, traittypes, sympy, protobuf, numpy, networkx, fsspec, filelock, torch, openvino, opencv-python, onnx, openvino-dev, ipydatawidgets, pythreejs - Successfully installed filelock-3.16.1 fsspec-2024.10.0 ipydatawidgets-4.3.5 mpmath-1.3.0 networkx-3.1 numpy-1.24.4 onnx-1.16.1 opencv-python-4.10.0.84 openvino-2024.4.0 openvino-dev-2024.4.0 openvino-telemetry-2024.1.0 protobuf-5.28.2 pythreejs-2.4.2 sympy-1.13.3 torch-2.4.1+cpu traittypes-0.2.1 + Installing collected packages: openvino-telemetry, mpmath, traittypes, tqdm, sympy, numpy, networkx, fsspec, filelock, torch, openvino, opencv-python, ipydatawidgets, pythreejs + Successfully installed filelock-3.16.1 fsspec-2024.10.0 ipydatawidgets-4.3.5 mpmath-1.3.0 networkx-3.1 numpy-1.24.4 opencv-python-4.10.0.84 openvino-2024.4.0 openvino-telemetry-2024.1.0 pythreejs-2.4.2 sympy-1.13.3 torch-2.4.1+cpu tqdm-4.66.6 traittypes-0.2.1 Note: you may need to restart the kernel to use updated packages. @@ -228,41 +218,31 @@ Download the model -We use ``omz_downloader``, which is a command line tool from the -``openvino-dev`` package. ``omz_downloader`` automatically creates a -directory structure and downloads the selected model. - .. code:: ipython3 - # directory where model will be downloaded - base_model_dir = "model" + from notebook_utils import download_file + import tarfile - # model name as named in Open Model Zoo - model_name = "human-pose-estimation-3d-0001" - # selected precision (FP32, FP16) - precision = "FP32" - BASE_MODEL_NAME = f"{base_model_dir}/public/{model_name}/{model_name}" - model_path = Path(BASE_MODEL_NAME).with_suffix(".pth") - onnx_path = Path(BASE_MODEL_NAME).with_suffix(".onnx") + # directory where model will be downloaded + base_model_dir = Path("model") + + download_file( + "https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2022.1/human-pose-estimation-3d-0001/human-pose-estimation-3d.tar.gz", + directory=base_model_dir, + ) - ir_model_path = Path(f"model/public/{model_name}/{precision}/{model_name}.xml") - model_weights_path = Path(f"model/public/{model_name}/{precision}/{model_name}.bin") + ckpt_file = base_model_dir / "human-pose-estimation-3d-0001.pth" - if not model_path.exists(): - download_command = f"omz_downloader " f"--name {model_name} " f"--output_dir {base_model_dir}" - ! $download_command + if not ckpt_file.exists(): + with tarfile.open(base_model_dir / "human-pose-estimation-3d.tar.gz") as f: + f.extractall(base_model_dir) + .. parsed-literal:: - ################|| Downloading human-pose-estimation-3d-0001 ||################ - - ========== Downloading model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.tar.gz - - - ========== Unpacking model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.tar.gz - + model/human-pose-estimation-3d.tar.gz: 0%| | 0.00/17.6M [00:00 1 or self.sliding_window is not None: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:808: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:861: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! encoder_states = () if output_hidden_states else None - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:813: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:866: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if output_hidden_states: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:836: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:889: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if output_hidden_states: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:839: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:892: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not return_dict: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:935: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:988: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not return_dict: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:1426: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:1486: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not return_dict: @@ -351,13 +352,13 @@ suitable. This function repeats part of ``AmusedPipeline``. .. parsed-literal:: - /tmp/ipykernel_2578393/3779428577.py:34: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /tmp/ipykernel_498025/3779428577.py:34: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! shape=shape.tolist(), - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/autoencoders/vq_model.py:144: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/autoencoders/vq_model.py:144: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not force_not_quantize: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:147: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:147: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if hidden_states.shape[0] >= 64: @@ -490,7 +491,7 @@ And insert wrappers instances in the pipeline: .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -696,7 +697,7 @@ model. .. parsed-literal:: - INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, onnx, openvino + INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, openvino @@ -707,7 +708,7 @@ model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -760,17 +761,17 @@ model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/tensor/tensor.py:100: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) @@ -794,7 +795,7 @@ Demo generation with quantized pipeline .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -878,7 +879,7 @@ a rough estimate of generation quality. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:43: UserWarning: Metric `InceptionScore` will save all extracted features in buffer. For large datasets this may lead to large memory footprint. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:43: UserWarning: Metric `InceptionScore` will save all extracted features in buffer. For large datasets this may lead to large memory footprint. warnings.warn(\*args, \*\*kwargs) # noqa: B028 @@ -890,7 +891,7 @@ a rough estimate of generation quality. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/image/inception.py:175: UserWarning: std(): degrees of freedom is <= 0. Correction should be strictly less than the reduction factor (input numel divided by output numel). (Triggered internally at ../aten/src/ATen/native/ReduceOps.cpp:1808.) + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/image/inception.py:175: UserWarning: std(): degrees of freedom is <= 0. Correction should be strictly less than the reduction factor (input numel divided by output numel). (Triggered internally at ../aten/src/ATen/native/ReduceOps.cpp:1808.) return kl.mean(), kl.std() @@ -908,7 +909,7 @@ a rough estimate of generation quality. .. parsed-literal:: Quantized pipeline Inception Score: 11.0730562210083 - Quantization speed-up: 2.09x + Quantization speed-up: 2.08x Interactive inference diff --git a/docs/notebooks/async-api-with-output.rst b/docs/notebooks/async-api-with-output.rst index e8f5b80d429d81..9f9130a4fe0db2 100644 --- a/docs/notebooks/async-api-with-output.rst +++ b/docs/notebooks/async-api-with-output.rst @@ -61,13 +61,6 @@ Imports %pip install -q "openvino>=2023.1.0" %pip install -q opencv-python "matplotlib>=3.4" - -.. parsed-literal:: - - Note: you may need to restart the kernel to use updated packages. - Note: you may need to restart the kernel to use updated packages. - - .. code:: ipython3 import cv2 @@ -119,14 +112,12 @@ the person in each frame of the video. ################|| Downloading person-detection-0202 ||################ - ========== Downloading model/intel/person-detection-0202/FP16/person-detection-0202.xml - + ========== Retrieving model/intel/person-detection-0202/FP16/person-detection-0202.xml from the cache - ========== Downloading model/intel/person-detection-0202/FP16/person-detection-0202.bin + ========== Retrieving model/intel/person-detection-0202/FP16/person-detection-0202.bin from the cache - Select inference device ~~~~~~~~~~~~~~~~~~~~~~~ @@ -348,8 +339,8 @@ Test performance in Sync Mode .. parsed-literal:: Source ended - average throuput in sync mode: 63.79 fps - + average throuput in sync mode: 55.59 fps + Async Mode ~~~~~~~~~~ @@ -487,8 +478,8 @@ Test the performance in Async Mode .. parsed-literal:: Source ended - average throuput in async mode: 106.83 fps - + average throuput in async mode: 75.17 fps + Compare the performance ~~~~~~~~~~~~~~~~~~~~~~~ @@ -630,5 +621,5 @@ Test the performance with ``AsyncInferQueue`` .. parsed-literal:: - average throughput in async mode with async infer queue: 145.36 fps - + average throughput in async mode with async infer queue: 103.81 fps + diff --git a/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png b/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png index f3492582efc67f..3694e16797b727 100644 --- a/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png +++ b/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:84a9b6aab4a04feb319b3243644da6837b3b894122657a8f6639fa604e3b48dd -size 29468 +oid sha256:fd71c3c8d066b8a16264c215b773c6200e981634ac24d04db99b0288d1ea1cca +size 30409 diff --git a/docs/notebooks/auto-device-with-output.rst b/docs/notebooks/auto-device-with-output.rst index 28d894eb72b22e..2ebcbe7d80deb2 100644 --- a/docs/notebooks/auto-device-with-output.rst +++ b/docs/notebooks/auto-device-with-output.rst @@ -197,16 +197,16 @@ By default, ``compile_model`` API will select **AUTO** as .. parsed-literal:: - [22:37:15.4888]I[plugin.cpp:421][AUTO] device:CPU, config:LOG_LEVEL=LOG_INFO - [22:37:15.4888]I[plugin.cpp:421][AUTO] device:CPU, config:PERFORMANCE_HINT=LATENCY - [22:37:15.4888]I[plugin.cpp:421][AUTO] device:CPU, config:PERFORMANCE_HINT_NUM_REQUESTS=0 - [22:37:15.4888]I[plugin.cpp:421][AUTO] device:CPU, config:PERF_COUNT=NO - [22:37:15.4888]I[plugin.cpp:426][AUTO] device:CPU, priority:0 - [22:37:15.4888]I[schedule.cpp:17][AUTO] scheduler starting - [22:37:15.4888]I[auto_schedule.cpp:181][AUTO] select device:CPU - [22:37:15.5995]I[auto_schedule.cpp:346][AUTO] Device: [CPU]: Compile model took 110.638049 ms - [22:37:15.5996]I[auto_schedule.cpp:112][AUTO] device:CPU compiling model finished - [22:37:15.5997]I[plugin.cpp:454][AUTO] underlying hardware does not support hardware context + [22:41:57.1267]I[plugin.cpp:421][AUTO] device:CPU, config:LOG_LEVEL=LOG_INFO + [22:41:57.1268]I[plugin.cpp:421][AUTO] device:CPU, config:PERFORMANCE_HINT=LATENCY + [22:41:57.1268]I[plugin.cpp:421][AUTO] device:CPU, config:PERFORMANCE_HINT_NUM_REQUESTS=0 + [22:41:57.1268]I[plugin.cpp:421][AUTO] device:CPU, config:PERF_COUNT=NO + [22:41:57.1268]I[plugin.cpp:426][AUTO] device:CPU, priority:0 + [22:41:57.1268]I[schedule.cpp:17][AUTO] scheduler starting + [22:41:57.1269]I[auto_schedule.cpp:181][AUTO] select device:CPU + [22:41:57.2582]I[auto_schedule.cpp:346][AUTO] Device: [CPU]: Compile model took 131.300219 ms + [22:41:57.2583]I[auto_schedule.cpp:112][AUTO] device:CPU compiling model finished + [22:41:57.2584]I[plugin.cpp:454][AUTO] underlying hardware does not support hardware context Successfully compiled model without a device_name. @@ -219,8 +219,8 @@ By default, ``compile_model`` API will select **AUTO** as .. parsed-literal:: - Deleted compiled_model[22:37:15.6060]I[schedule.cpp:308][AUTO] scheduler ending - + Deleted compiled_model + [22:41:57.2639]I[schedule.cpp:308][AUTO] scheduler ending Explicitly pass AUTO as device_name to Core::compile_model API @@ -378,7 +378,7 @@ executed on CPU until GPU is ready. .. parsed-literal:: - Time to load model using AUTO device and get first inference: 0.15 seconds. + Time to load model using AUTO device and get first inference: 0.12 seconds. .. code:: ipython3 @@ -553,12 +553,12 @@ Loop for inference and update the FPS/Latency every Compiling Model for AUTO device with THROUGHPUT hint Start inference, 6 groups of FPS/latency will be measured over 10s intervals - throughput: 184.25fps, latency: 31.12ms, time interval: 10.02s - throughput: 184.19fps, latency: 31.80ms, time interval: 10.00s - throughput: 183.00fps, latency: 32.00ms, time interval: 10.01s - throughput: 183.37fps, latency: 31.91ms, time interval: 10.01s - throughput: 178.30fps, latency: 32.90ms, time interval: 10.01s - throughput: 182.80fps, latency: 32.08ms, time interval: 10.01s + throughput: 179.70fps, latency: 32.12ms, time interval: 10.00s + throughput: 183.61fps, latency: 31.86ms, time interval: 10.01s + throughput: 183.96fps, latency: 31.88ms, time interval: 10.01s + throughput: 183.98fps, latency: 31.91ms, time interval: 10.00s + throughput: 183.26fps, latency: 31.98ms, time interval: 10.01s + throughput: 183.40fps, latency: 32.01ms, time interval: 10.00s Done @@ -604,12 +604,12 @@ Loop for inference and update the FPS/Latency for each Compiling Model for AUTO Device with LATENCY hint Start inference, 6 groups fps/latency will be out with 10s interval - throughput: 139.34fps, latency: 6.64ms, time interval: 10.00s - throughput: 141.45fps, latency: 6.63ms, time interval: 10.00s - throughput: 141.42fps, latency: 6.63ms, time interval: 10.01s - throughput: 141.70fps, latency: 6.62ms, time interval: 10.01s - throughput: 130.57fps, latency: 7.22ms, time interval: 10.00s - throughput: 141.61fps, latency: 6.62ms, time interval: 10.01s + throughput: 130.56fps, latency: 7.18ms, time interval: 10.00s + throughput: 142.51fps, latency: 6.61ms, time interval: 10.01s + throughput: 142.47fps, latency: 6.62ms, time interval: 10.00s + throughput: 142.46fps, latency: 6.61ms, time interval: 10.00s + throughput: 142.63fps, latency: 6.61ms, time interval: 10.00s + throughput: 142.73fps, latency: 6.60ms, time interval: 10.00s Done diff --git a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png index af14e6c0ac24c7..cc037738f18096 100644 --- a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png +++ b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:05bc2541cede3c32f2f9acff3f93d19572aab80a5bb7ac3e6c77bb397817bb54 -size 25899 +oid sha256:1bedd8ff3e65a23fb4af380958a261d0916d2e0134b9426652a2779bdc06d6de +size 26887 diff --git a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png index e12bb8a48eaa0e..21be57ac89d68d 100644 --- a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png +++ b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b1e2e56c28c18b5d31607518b846e78c58e76b2b59a9ee083d02e0d8f8ec2b52 -size 40077 +oid sha256:ed1ab24c30040707a36155169f4aaa91a5bff6cb48a2c5d10401ecbd87ca6f54 +size 40117 diff --git a/docs/notebooks/catvton-with-output.rst b/docs/notebooks/catvton-with-output.rst new file mode 100644 index 00000000000000..a7a9a04359f338 --- /dev/null +++ b/docs/notebooks/catvton-with-output.rst @@ -0,0 +1,360 @@ +Virtual Try-On with CatVTON and OpenVINO +======================================== + +Virtual try-on methods based on diffusion models achieve realistic +try-on effects but replicate the backbone network as a ReferenceNet or +leverage additional image encoders to process condition inputs, +resulting in high training and inference costs. `In this +work `__, authors rethink the necessity +of ReferenceNet and image encoders and innovate the interaction between +garment and person, proposing CatVTON, a simple and efficient virtual +try-on diffusion model. It facilitates the seamless transfer of in-shop +or worn garments of arbitrary categories to target persons by simply +concatenating them in spatial dimensions as inputs. The efficiency of +the model is demonstrated in three aspects: 1. Lightweight network. Only +the original diffusion modules are used, without additional network +modules. The text encoder and cross attentions for text injection in the +backbone are removed, further reducing the parameters by 167.02M. 2. +Parameter-efficient training. We identified the try-on relevant modules +through experiments and achieved high-quality try-on effects by training +only 49.57M parameters (∼5.51% of the backbone network’s parameters). 3. +Simplified inference. CatVTON eliminates all unnecessary conditions and +preprocessing steps, including pose estimation, human parsing, and text +input, requiring only garment reference, target person image, and mask +for the virtual try-on process. Extensive experiments demonstrate that +CatVTON achieves superior qualitative and quantitative results with +fewer prerequisites and trainable parameters than baseline methods. +Furthermore, CatVTON shows good generalization in in-the-wild scenarios +despite using open-source datasets with only 73K samples. + +Teaser image from `CatVTON +GitHub `__ |teaser| + +In this tutorial we consider how to convert and run this model using +OpenVINO. + + +**Table of contents:** + + +- `Prerequisites <#prerequisites>`__ +- `Convert the model to OpenVINO + IR <#convert-the-model-to-openvino-ir>`__ +- `Compiling models <#compiling-models>`__ +- `Interactive demo <#interactive-demo>`__ + +Installation Instructions +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This is a self-contained example that relies solely on its own code. + +We recommend running the notebook in a virtual environment. You only +need a Jupyter server to start. For details, please refer to +`Installation +Guide `__. + +.. |teaser| image:: https://github.com/Zheng-Chong/CatVTON/blob/edited/resource/img/teaser.jpg?raw=true + +Prerequisites +------------- + + + +.. code:: ipython3 + + import platform + + + if platform.system() == "Darwin": + %pip install -q "numpy<2.0.0" + %pip install -q "openvino>=2024.4" + %pip install -q "torch>=2.1" "diffusers>=0.29.1" torchvision opencv_python --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q fvcore "pillow" "tqdm" "gradio>=4.36" "omegaconf==2.4.0.dev3" av pycocotools cloudpickle scipy accelerate "transformers>=4.27.3" + + +.. parsed-literal:: + + Note: you may need to restart the kernel to use updated packages. + Note: you may need to restart the kernel to use updated packages. + Note: you may need to restart the kernel to use updated packages. + + +.. code:: ipython3 + + import requests + + + r = requests.get( + url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", + ) + open("notebook_utils.py", "w").write(r.text) + + r = requests.get( + url="https://raw.githubusercontent.com/aleksandr-mokrov/openvino_notebooks/refs/heads/catvton/utils/cmd_helper.py", + ) + open("cmd_helper.py", "w").write(r.text) + + + + +.. parsed-literal:: + + 741 + + + +.. code:: ipython3 + + from cmd_helper import clone_repo + + + clone_repo("https://github.com/Zheng-Chong/CatVTON.git", "3b795364a4d2f3b5adb365f39cdea376d20bc53c") + + + + +.. parsed-literal:: + + PosixPath('CatVTON') + + + +Convert the model to OpenVINO IR +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +OpenVINO supports PyTorch models via conversion to OpenVINO Intermediate +Representation (IR). `OpenVINO model conversion +API `__ +should be used for these purposes. ``ov.convert_model`` function accepts +original PyTorch model instance and example input for tracing and +returns ``ov.Model`` representing this model in OpenVINO framework. +Converted model can be used for saving on disk using ``ov.save_model`` +function or directly loading on device using ``core.complie_model``. + +``ov_catvton_helper.py`` script contains helper function for models +downloading and models conversion, please check its content if you +interested in conversion details. + +To download checkpoints and load models, just call the helper function +``download_models``. It takes care about it. Functions +``convert_pipeline_models`` and ``convert_automasker_models`` will +convert models from pipeline and ``automasker`` in OpenVINO format. + +The original pipeline contains VAE encoder and decoder and UNET. +|CatVTON-overview| + +The ``automasker`` contains ``DensePose`` with +``detectron2.GeneralizedRCNN`` model and ``SCHP`` (``LIP`` and ``ATR`` +version). + +.. |CatVTON-overview| image:: https://github.com/user-attachments/assets/e35c8dab-1c54-47b1-a73b-2a62e6cdca7c + +.. code:: ipython3 + + from pathlib import Path + + from ov_catvton_helper import download_models, convert_pipeline_models, convert_automasker_models + + + MODEL_DIR = Path("models") + VAE_ENCODER_PATH = MODEL_DIR / "vae_encoder.xml" + VAE_DECODER_PATH = MODEL_DIR / "vae_decoder.xml" + UNET_PATH = MODEL_DIR / "unet.xml" + DENSEPOSE_PROCESSOR_PATH = MODEL_DIR / "densepose_processor.xml" + SCHP_PROCESSOR_ATR = MODEL_DIR / "schp_processor_atr.xml" + SCHP_PROCESSOR_LIP = MODEL_DIR / "schp_processor_lip.xml" + + + pipeline, mask_processor, automasker = download_models(MODEL_DIR) + convert_pipeline_models(pipeline, VAE_ENCODER_PATH, VAE_DECODER_PATH, UNET_PATH) + convert_automasker_models(automasker, DENSEPOSE_PROCESSOR_PATH, SCHP_PROCESSOR_ATR, SCHP_PROCESSOR_LIP) + + +.. parsed-literal:: + + Note: switching to '3b795364a4d2f3b5adb365f39cdea376d20bc53c'. + + You are in 'detached HEAD' state. You can look around, make experimental + changes and commit them, and you can discard any commits you make in this + state without impacting any branches by switching back to a branch. + + If you want to create a new branch to retain commits you create, you may + do so (now or later) by using -c with the switch command. Example: + + git switch -c + + Or undo this operation with: + + git switch - + + Turn off this advice by setting config variable advice.detachedHead to false + + HEAD is now at 3b79536 Update default base model path + + + +.. parsed-literal:: + + Fetching 10 files: 0%| | 0/10 [00:00= 64: + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unets/unet_2d_condition.py:1111: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + if dim % default_overall_up_factor != 0: + + +Compiling models +---------------- + + + +Select device from dropdown list for running inference using OpenVINO. + +.. code:: ipython3 + + import openvino as ov + + from notebook_utils import device_widget + + + core = ov.Core() + + device = device_widget() + + device + + + + +.. parsed-literal:: + + Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') + + + +``get_compiled_pipeline`` and ``get_compiled_automasker`` functions +defined in ``ov_catvton_helper.py`` provides convenient way for getting +the pipeline and the ``automasker`` with compiled ov-models that are +compatible with the original interface. It accepts the original pipeline +and ``automasker``, inference device and directories with converted +models as arguments. Under the hood we create callable wrapper classes +for compiled models to allow interaction with original pipelines. Note +that all of wrapper classes return ``torch.Tensor``\ s instead of +``np.array``\ s. And then insert wrappers instances in the pipeline. + +.. code:: ipython3 + + from ov_catvton_helper import get_compiled_pipeline, get_compiled_automasker + + + pipeline = get_compiled_pipeline(pipeline, core, device, VAE_ENCODER_PATH, VAE_DECODER_PATH, UNET_PATH) + automasker = get_compiled_automasker(automasker, core, device, DENSEPOSE_PROCESSOR_PATH, SCHP_PROCESSOR_ATR, SCHP_PROCESSOR_LIP) + +Interactive inference +--------------------- + + + +Please select below whether you would like to use the quantized models +to launch the interactive demo. + +.. code:: ipython3 + + from gradio_helper import make_demo + + + output_dir = "output" + demo = make_demo(pipeline, mask_processor, automasker, output_dir) + try: + demo.launch(debug=False) + except Exception: + demo.launch(debug=False, share=True) + + +.. parsed-literal:: + + Running on local URL: http://127.0.0.1:7860 + + To create a public link, set `share=True` in `launch()`. + + + + + + + diff --git a/docs/notebooks/clip-zero-shot-classification-with-output.rst b/docs/notebooks/clip-zero-shot-classification-with-output.rst index 3da831e6d9d0dd..fd572a83ffb834 100644 --- a/docs/notebooks/clip-zero-shot-classification-with-output.rst +++ b/docs/notebooks/clip-zero-shot-classification-with-output.rst @@ -729,7 +729,6 @@ up of the dynamic quantized models. Interactive demo ---------------- - Now, it is your turn! You can provide your own image and comma-separated list of labels for zero-shot classification. diff --git a/docs/notebooks/convert-to-openvino-with-output.rst b/docs/notebooks/convert-to-openvino-with-output.rst index 8ec32adef0e04a..2baaf0043e7f04 100644 --- a/docs/notebooks/convert-to-openvino-with-output.rst +++ b/docs/notebooks/convert-to-openvino-with-output.rst @@ -163,7 +163,7 @@ NLP model from Hugging Face and export it in ONNX format: .. code:: ipython3 from transformers import AutoModelForSequenceClassification, AutoTokenizer - from transformers.onnx import export, FeaturesManager + import torch ONNX_NLP_MODEL_PATH = MODEL_DIRECTORY_PATH / "distilbert.onnx" @@ -172,37 +172,27 @@ NLP model from Hugging Face and export it in ONNX format: # initialize tokenizer tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english") - # get model onnx config function for output feature format sequence-classification - model_kind, model_onnx_config = FeaturesManager.check_supported_model_or_raise(hf_model, feature="sequence-classification") - # fill onnx config based on pytorch model config - onnx_config = model_onnx_config(hf_model.config) - - # export to onnx format - export( - preprocessor=tokenizer, - model=hf_model, - config=onnx_config, - opset=onnx_config.default_onnx_opset, - output=ONNX_NLP_MODEL_PATH, - ) + if not ONNX_NLP_MODEL_PATH.exists(): + inputs = tokenizer("Hi, how are you?", return_tensors="pt") + input_names = list(inputs.keys()) + dynamic_axes = {input_name: {0: "batch_size", 1: "seq_length"} for input_name in input_names} + torch.onnx.export( + hf_model, args=dict(inputs), input_names=input_names, output_names=["logits"], dynamic_axes=dynamic_axes, f=ONNX_NLP_MODEL_PATH, opset_version=14 + ) + print(f"ONNX model exported to {ONNX_NLP_MODEL_PATH}") .. parsed-literal:: - 2024-10-22 22:40:21.522113: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-22 22:40:21.555890: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-04 22:48:30.842642: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-04 22:48:30.876775: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-22 22:40:22.084160: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:215: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. - mask, torch.tensor(torch.finfo(scores.dtype).min) - - + 2024-11-04 22:48:31.539454: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: - (['input_ids', 'attention_mask'], ['logits']) - + ONNX model exported to model/distilbert.onnx Fetch @@ -670,7 +660,7 @@ frameworks conversion guides. .. parsed-literal:: - 2024-10-22 22:40:40.138322: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. + 2024-11-04 22:48:47.716205: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. Skipping registering GPU devices... diff --git a/docs/notebooks/convnext-classification-with-output.rst b/docs/notebooks/convnext-classification-with-output.rst index 1204ea2c17f106..6e1c039f7013c6 100644 --- a/docs/notebooks/convnext-classification-with-output.rst +++ b/docs/notebooks/convnext-classification-with-output.rst @@ -192,7 +192,7 @@ And print results Predicted Class: 281 Predicted Label: n02123045 tabby, tabby cat - Predicted Probability: 0.5351971983909607 + Predicted Probability: 0.4661690592765808 Convert the model to OpenVINO Intermediate representation format diff --git a/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst b/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst index 6f003fb71d75fb..30778bafc8e884 100644 --- a/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst +++ b/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst @@ -154,10 +154,10 @@ Imports .. parsed-literal:: - 2024-10-22 22:41:03.560708: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-22 22:41:03.596278: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-04 22:49:10.827255: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-04 22:49:10.861330: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-22 22:41:04.191138: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-04 22:49:11.454332: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -223,7 +223,7 @@ notebook `__. .. parsed-literal:: - /tmp/ipykernel_2583350/1592321960.py:3: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /tmp/ipykernel_503635/1592321960.py:3: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. state_dict = torch.load(state_dict_file, map_location=torch.device("cpu")) @@ -444,7 +444,7 @@ this notebook. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if x_e.shape[-i - 1] != x_0.shape[-i - 1]: @@ -526,18 +526,18 @@ Convert quantized model to OpenVINO IR model and save it. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:340: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:340: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! return self._level_low.item() - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:348: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:348: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! return self._level_high.item() - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if x_e.shape[-i - 1] != x_0.shape[-i - 1]: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: Tensor-likes are not close! - Mismatched elements: 249255 / 262144 (95.1%) - Greatest absolute difference: 3.8265769481658936 at index (0, 0, 126, 353) (up to 1e-05 allowed) - Greatest relative difference: 18364.59337143498 at index (0, 0, 305, 451) (up to 1e-05 allowed) + Mismatched elements: 245783 / 262144 (93.8%) + Greatest absolute difference: 3.1180567741394043 at index (0, 0, 474, 435) (up to 1e-05 allowed) + Greatest relative difference: 16087.83647354372 at index (0, 0, 37, 224) (up to 1e-05 allowed) _check_trace( @@ -663,7 +663,7 @@ be run in the notebook with ``! benchmark_app`` or [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 8.83 ms + [ INFO ] Read model took 8.85 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [1,1,512,512] @@ -677,7 +677,7 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.final_conv/aten::_convolution/Add) : f32 / [...] / [1,1,512,512] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 250.74 ms + [ INFO ] Compile model took 253.47 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -714,17 +714,17 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 51.21 ms + [ INFO ] First inference took 56.51 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 392 iterations - [ INFO ] Duration: 15027.48 ms + [ INFO ] Count: 406 iterations + [ INFO ] Duration: 15019.48 ms [ INFO ] Latency: - [ INFO ] Median: 34.96 ms - [ INFO ] Average: 38.10 ms - [ INFO ] Min: 34.49 ms + [ INFO ] Median: 35.01 ms + [ INFO ] Average: 36.77 ms + [ INFO ] Min: 34.63 ms [ INFO ] Max: 48.05 ms - [ INFO ] Throughput: 26.09 FPS + [ INFO ] Throughput: 27.03 FPS .. code:: ipython3 @@ -750,7 +750,7 @@ be run in the notebook with ``! benchmark_app`` or [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 11.01 ms + [ INFO ] Read model took 10.78 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [1,1,512,512] @@ -764,7 +764,7 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.final_conv/aten::_convolution/Add) : f32 / [...] / [1,1,512,512] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 238.95 ms + [ INFO ] Compile model took 250.08 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model49 @@ -801,17 +801,17 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 30.03 ms + [ INFO ] First inference took 29.09 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 952 iterations - [ INFO ] Duration: 15015.31 ms + [ INFO ] Count: 938 iterations + [ INFO ] Duration: 15008.12 ms [ INFO ] Latency: - [ INFO ] Median: 15.55 ms - [ INFO ] Average: 15.57 ms - [ INFO ] Min: 15.23 ms - [ INFO ] Max: 17.26 ms - [ INFO ] Throughput: 63.40 FPS + [ INFO ] Median: 15.77 ms + [ INFO ] Average: 15.80 ms + [ INFO ] Min: 15.47 ms + [ INFO ] Max: 17.13 ms + [ INFO ] Throughput: 62.50 FPS Visually Compare Inference Results @@ -905,7 +905,7 @@ seed is displayed to enable reproducing specific runs of this cell. .. parsed-literal:: - Visualizing results with seed 1729629747 + Visualizing results with seed 1730757034 @@ -988,8 +988,8 @@ performs inference, and displays the results on the frames loaded in .. parsed-literal:: - Loaded model to AUTO in 0.18 seconds. - Total time for 68 frames: 2.32 seconds, fps:29.79 + Loaded model to AUTO in 0.15 seconds. + Total time for 68 frames: 2.36 seconds, fps:29.25 References diff --git a/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png b/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png index f7c163c1c77604..5aa37909b71cf7 100644 --- a/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png +++ b/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5d329fc21292f69aeee8164a3f805f2e8e61369c42eab565ebafe555cf6d1a1c -size 381131 +oid sha256:894600de56af211d4cc3e64ee092b5a62d1b0158c51048d17accadddea0f046e +size 382725 diff --git a/docs/notebooks/ddcolor-image-colorization-with-output.rst b/docs/notebooks/ddcolor-image-colorization-with-output.rst index b2a76fd1a0ded8..409d2495e2fea6 100644 --- a/docs/notebooks/ddcolor-image-colorization-with-output.rst +++ b/docs/notebooks/ddcolor-image-colorization-with-output.rst @@ -105,12 +105,12 @@ Prerequisites .. parsed-literal:: Cloning into 'DDColor'... - remote: Enumerating objects: 233, done. - remote: Counting objects: 100% (76/76), done. - remote: Compressing objects: 100% (42/42), done. - remote: Total 233 (delta 54), reused 34 (delta 34), pack-reused 157 (from 1) - Receiving objects: 100% (233/233), 13.34 MiB | 641.00 KiB/s, done. - Resolving deltas: 100% (80/80), done. + remote: Enumerating objects: 241, done. + remote: Counting objects: 100% (84/84), done. + remote: Compressing objects: 100% (49/49), done. + remote: Total 241 (delta 57), reused 37 (delta 35), pack-reused 157 (from 1) + Receiving objects: 100% (241/241), 14.10 MiB | 21.95 MiB/s, done. + Resolving deltas: 100% (83/83), done. @@ -131,7 +131,7 @@ Prerequisites .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) @@ -412,10 +412,10 @@ Perform model quantization .. parsed-literal:: - 2024-10-22 22:45:07.339219: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-22 22:45:07.378241: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-04 22:52:53.152561: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-04 22:52:53.191342: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-22 22:45:07.784302: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-04 22:52:53.595160: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -523,7 +523,7 @@ Tool + @@ -307,13 +304,13 @@ loading on device using ``core.complie_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2_layers/patch_embed.py:73: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2_layers/patch_embed.py:73: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}" - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2_layers/patch_embed.py:74: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2_layers/patch_embed.py:74: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}" - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2.py:183: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dinov2.py:183: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if npatch == N and w == h: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dpt.py:147: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/depth_anything_v2/dpt.py:147: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! out = F.interpolate(out, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True) @@ -405,7 +402,7 @@ range. .. parsed-literal:: - + @@ -627,7 +624,7 @@ Run inference on video .. parsed-literal:: - Processed 60 frames in 13.44 seconds. Total FPS (including video processing): 4.46.Inference FPS: 10.42 + Processed 60 frames in 13.24 seconds. Total FPS (including video processing): 4.53.Inference FPS: 10.68 Video saved to 'output/Coco Walking in Berkeley_depth_anything.mp4'. @@ -654,7 +651,7 @@ Run inference on video .. parsed-literal:: Showing video saved at - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/output/Coco Walking in Berkeley_depth_anything.mp4 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/output/Coco Walking in Berkeley_depth_anything.mp4 If you cannot see the video in your browser, please click on the following link to download the video @@ -787,10 +784,10 @@ quantization code below may take some time. .. parsed-literal:: - 2024-10-22 22:53:01.689628: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-22 22:53:01.723459: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-04 23:01:18.047102: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-04 23:01:18.080343: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-22 22:53:02.312695: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-04 23:01:18.654050: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -916,10 +913,10 @@ data. .. parsed-literal:: - Processed 60 frames in 12.93 seconds. Total FPS (including video processing): 4.64.Inference FPS: 12.78 + Processed 60 frames in 12.60 seconds. Total FPS (including video processing): 4.76.Inference FPS: 13.12 Video saved to 'output/Coco Walking in Berkeley_depth_anything_int8.mp4'. Showing video saved at - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/output/Coco Walking in Berkeley_depth_anything.mp4 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything-V2/output/Coco Walking in Berkeley_depth_anything.mp4 If you cannot see the video in your browser, please click on the following link to download the video @@ -999,9 +996,9 @@ Tool =4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (4.12.2) - Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (1.13.3) - Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1) - Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1.4) - Requirement already satisfied: fsspec in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (2024.9.0) - Requirement already satisfied: numpy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchvision) (1.23.5) - Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchvision) (10.4.0) - Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch) (2.1.5) - Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch) (1.3.0) + Requirement already satisfied: torch in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.4.1+cpu) + Requirement already satisfied: torchvision in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (0.19.1+cpu) + Requirement already satisfied: opencv-python in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.10.0.84) + Requirement already satisfied: wheel in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (0.44.0) + Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.16.1) + Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (4.12.2) + Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (1.13.3) + Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1) + Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1.4) + Requirement already satisfied: fsspec in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (2024.9.0) + Requirement already satisfied: numpy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchvision) (1.23.5) + Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchvision) (10.4.0) + Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch) (2.1.5) + Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch) (1.3.0) Looking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cpu Collecting git+https://github.com/facebookresearch/detectron2.git - Cloning https://github.com/facebookresearch/detectron2.git to /tmp/pip-req-build-we1e_5gi + Cloning https://github.com/facebookresearch/detectron2.git to /tmp/pip-req-build-9ds1xx43 .. parsed-literal:: - Running command git clone --filter=blob:none --quiet https://github.com/facebookresearch/detectron2.git /tmp/pip-req-build-we1e_5gi + Running command git clone --filter=blob:none --quiet https://github.com/facebookresearch/detectron2.git /tmp/pip-req-build-9ds1xx43 .. parsed-literal:: @@ -125,88 +125,88 @@ Install required packages for running model Resolved https://github.com/facebookresearch/detectron2.git to commit 8d85329aed8506ea3672e3e208971345973ea761 Preparing metadata (setup.py): started Preparing metadata (setup.py): finished with status 'done' - Requirement already satisfied: Pillow>=7.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (10.4.0) - Requirement already satisfied: black in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (24.3.0) - Collecting cloudpickle (from detectron2==0.6) - Using cached cloudpickle-3.1.0-py3-none-any.whl.metadata (7.0 kB) - Collecting fvcore<0.1.6,>=0.1.5 (from detectron2==0.6) - Using cached fvcore-0.1.5.post20221221-py3-none-any.whl + Requirement already satisfied: Pillow>=7.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (10.4.0) + Requirement already satisfied: black in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (24.3.0) + Requirement already satisfied: cloudpickle in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (3.1.0) + Requirement already satisfied: fvcore<0.1.6,>=0.1.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (0.1.5.post20221221) Collecting hydra-core>=1.1 (from detectron2==0.6) Using cached hydra_core-1.3.2-py3-none-any.whl.metadata (5.5 kB) Collecting iopath<0.1.10,>=0.1.7 (from detectron2==0.6) - Using cached iopath-0.1.9-py3-none-any.whl.metadata (370 bytes) - Requirement already satisfied: matplotlib in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (3.7.5) - Requirement already satisfied: omegaconf<2.4,>=2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (2.3.0) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (24.1) - Collecting pycocotools>=2.0.2 (from detectron2==0.6) - Using cached pycocotools-2.0.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.1 kB) - Requirement already satisfied: tabulate in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (0.9.0) - Requirement already satisfied: tensorboard in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (2.12.3) - Requirement already satisfied: termcolor>=1.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (2.4.0) - Requirement already satisfied: tqdm>4.29.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (4.66.5) - Collecting yacs>=0.1.8 (from detectron2==0.6) - Using cached yacs-0.1.8-py3-none-any.whl.metadata (639 bytes) - Requirement already satisfied: numpy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from fvcore<0.1.6,>=0.1.5->detectron2==0.6) (1.23.5) - Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from fvcore<0.1.6,>=0.1.5->detectron2==0.6) (6.0.2) - Requirement already satisfied: antlr4-python3-runtime==4.9.* in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.1->detectron2==0.6) (4.9.3) - Requirement already satisfied: importlib-resources in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.1->detectron2==0.6) (6.4.5) - Collecting portalocker (from iopath<0.1.10,>=0.1.7->detectron2==0.6) - Using cached portalocker-2.10.1-py3-none-any.whl.metadata (8.5 kB) - Requirement already satisfied: contourpy>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (1.1.1) - Requirement already satisfied: cycler>=0.10 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (0.12.1) - Requirement already satisfied: fonttools>=4.22.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (4.54.1) - Requirement already satisfied: kiwisolver>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (1.4.7) - Requirement already satisfied: pyparsing>=2.3.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (3.1.4) - Requirement already satisfied: python-dateutil>=2.7 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (2.9.0.post0) - Requirement already satisfied: click>=8.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (8.1.7) - Requirement already satisfied: mypy-extensions>=0.4.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (1.0.0) - Requirement already satisfied: pathspec>=0.9.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (0.12.1) - Requirement already satisfied: platformdirs>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (4.3.6) - Requirement already satisfied: tomli>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (2.0.2) - Requirement already satisfied: typing-extensions>=4.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (4.12.2) - Requirement already satisfied: absl-py>=0.4 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (1.4.0) - Requirement already satisfied: grpcio>=1.48.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (1.67.0) - Requirement already satisfied: google-auth<3,>=1.6.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (2.35.0) - Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (1.0.0) - Requirement already satisfied: markdown>=2.6.8 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (3.7) - Requirement already satisfied: protobuf>=3.19.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (3.20.3) - Requirement already satisfied: requests<3,>=2.21.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (2.32.3) - Requirement already satisfied: setuptools>=41.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (44.0.0) - Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (0.7.2) - Requirement already satisfied: werkzeug>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (3.0.4) - Requirement already satisfied: wheel>=0.26 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (0.44.0) - Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (5.5.0) - Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (0.4.1) - Requirement already satisfied: rsa<5,>=3.1.4 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (4.9) - Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard->detectron2==0.6) (2.0.0) - Requirement already satisfied: zipp>=3.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from importlib-resources->hydra-core>=1.1->detectron2==0.6) (3.20.2) - Requirement already satisfied: importlib-metadata>=4.4 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from markdown>=2.6.8->tensorboard->detectron2==0.6) (8.5.0) - Requirement already satisfied: six>=1.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from python-dateutil>=2.7->matplotlib->detectron2==0.6) (1.16.0) - Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (3.4.0) - Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (3.10) - Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (2.2.3) - Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (2024.8.30) - Requirement already satisfied: MarkupSafe>=2.1.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from werkzeug>=1.0.1->tensorboard->detectron2==0.6) (2.1.5) - Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (0.6.1) - Requirement already satisfied: oauthlib>=3.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard->detectron2==0.6) (3.2.2) + Using cached https://download.pytorch.org/whl/iopath-0.1.9-py3-none-any.whl (27 kB) + Requirement already satisfied: matplotlib in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (3.7.5) + Collecting omegaconf<2.4,>=2.1 (from detectron2==0.6) + Using cached omegaconf-2.3.0-py3-none-any.whl.metadata (3.9 kB) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (24.1) + Requirement already satisfied: pycocotools>=2.0.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (2.0.7) + Requirement already satisfied: tabulate in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (0.9.0) + Requirement already satisfied: tensorboard in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (2.12.3) + Requirement already satisfied: termcolor>=1.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (2.4.0) + Requirement already satisfied: tqdm>4.29.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (4.66.6) + Requirement already satisfied: yacs>=0.1.8 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from detectron2==0.6) (0.1.8) + Requirement already satisfied: numpy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from fvcore<0.1.6,>=0.1.5->detectron2==0.6) (1.23.5) + Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from fvcore<0.1.6,>=0.1.5->detectron2==0.6) (6.0.2) + Requirement already satisfied: antlr4-python3-runtime==4.9.* in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.1->detectron2==0.6) (4.9.3) + Requirement already satisfied: importlib-resources in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.1->detectron2==0.6) (6.4.5) + Requirement already satisfied: portalocker in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from iopath<0.1.10,>=0.1.7->detectron2==0.6) (2.10.1) + Requirement already satisfied: contourpy>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (1.1.1) + Requirement already satisfied: cycler>=0.10 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (0.12.1) + Requirement already satisfied: fonttools>=4.22.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (4.54.1) + Requirement already satisfied: kiwisolver>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (1.4.7) + Requirement already satisfied: pyparsing>=2.3.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (3.1.4) + Requirement already satisfied: python-dateutil>=2.7 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from matplotlib->detectron2==0.6) (2.9.0.post0) + Requirement already satisfied: click>=8.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (8.1.7) + Requirement already satisfied: mypy-extensions>=0.4.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (1.0.0) + Requirement already satisfied: pathspec>=0.9.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (0.12.1) + Requirement already satisfied: platformdirs>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (4.3.6) + Requirement already satisfied: tomli>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (2.0.2) + Requirement already satisfied: typing-extensions>=4.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from black->detectron2==0.6) (4.12.2) + Requirement already satisfied: absl-py>=0.4 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (1.4.0) + Requirement already satisfied: grpcio>=1.48.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (1.67.1) + Requirement already satisfied: google-auth<3,>=1.6.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (2.35.0) + Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (1.0.0) + Requirement already satisfied: markdown>=2.6.8 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (3.7) + Requirement already satisfied: protobuf>=3.19.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (3.20.3) + Requirement already satisfied: requests<3,>=2.21.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (2.32.3) + Requirement already satisfied: setuptools>=41.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (44.0.0) + Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (0.7.2) + Requirement already satisfied: werkzeug>=1.0.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (3.0.6) + Requirement already satisfied: wheel>=0.26 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from tensorboard->detectron2==0.6) (0.44.0) + Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (5.5.0) + Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (0.4.1) + Requirement already satisfied: rsa<5,>=3.1.4 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (4.9) + Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard->detectron2==0.6) (2.0.0) + Requirement already satisfied: zipp>=3.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from importlib-resources->hydra-core>=1.1->detectron2==0.6) (3.20.2) + Requirement already satisfied: importlib-metadata>=4.4 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from markdown>=2.6.8->tensorboard->detectron2==0.6) (8.5.0) + Requirement already satisfied: six>=1.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from python-dateutil>=2.7->matplotlib->detectron2==0.6) (1.16.0) + Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (3.4.0) + Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (3.10) + Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (2.2.3) + Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->detectron2==0.6) (2024.8.30) + Requirement already satisfied: MarkupSafe>=2.1.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from werkzeug>=1.0.1->tensorboard->detectron2==0.6) (2.1.5) + Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard->detectron2==0.6) (0.6.1) + Requirement already satisfied: oauthlib>=3.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard->detectron2==0.6) (3.2.2) Using cached hydra_core-1.3.2-py3-none-any.whl (154 kB) - Using cached iopath-0.1.9-py3-none-any.whl (27 kB) - Using cached pycocotools-2.0.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (439 kB) - Using cached yacs-0.1.8-py3-none-any.whl (14 kB) - Using cached cloudpickle-3.1.0-py3-none-any.whl (22 kB) - Using cached portalocker-2.10.1-py3-none-any.whl (18 kB) + Using cached omegaconf-2.3.0-py3-none-any.whl (79 kB) Building wheels for collected packages: detectron2 Building wheel for detectron2 (setup.py): started Building wheel for detectron2 (setup.py): finished with status 'done' - Created wheel for detectron2: filename=detectron2-0.6-cp38-cp38-linux_x86_64.whl size=8313552 sha256=23ceb6e5b734ecc530172b613be139d732deaa2e962d5a8bc940e6b23a85309d - Stored in directory: /tmp/pip-ephem-wheel-cache-65iaghs7/wheels/19/ac/65/e48e5e4ec2702274d927c5a6efb75709b24014371d3bb778f2 + Created wheel for detectron2: filename=detectron2-0.6-cp38-cp38-linux_x86_64.whl size=8313237 sha256=7cd84a15a89de76a7ab5b648f2fb7ebff63b7e43ffc90c7f19a568d16858de8a + Stored in directory: /tmp/pip-ephem-wheel-cache-uvptv5zg/wheels/19/ac/65/e48e5e4ec2702274d927c5a6efb75709b24014371d3bb778f2 Successfully built detectron2 - Installing collected packages: yacs, portalocker, cloudpickle, iopath, hydra-core, pycocotools, fvcore, detectron2 - Successfully installed cloudpickle-3.1.0 detectron2-0.6 fvcore-0.1.5.post20221221 hydra-core-1.3.2 iopath-0.1.9 portalocker-2.10.1 pycocotools-2.0.7 yacs-0.1.8 - Requirement already satisfied: openvino>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2024.4.0) - Requirement already satisfied: numpy<2.1.0,>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (1.23.5) - Requirement already satisfied: openvino-telemetry>=2023.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (2024.1.0) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (24.1) + Installing collected packages: omegaconf, iopath, hydra-core, detectron2 + Attempting uninstall: omegaconf + Found existing installation: omegaconf 2.4.0.dev3 + Uninstalling omegaconf-2.4.0.dev3: + Successfully uninstalled omegaconf-2.4.0.dev3 + Attempting uninstall: iopath + Found existing installation: iopath 0.1.10 + Uninstalling iopath-0.1.10: + Successfully uninstalled iopath-0.1.10 + Successfully installed detectron2-0.6 hydra-core-1.3.2 iopath-0.1.9 omegaconf-2.3.0 + Requirement already satisfied: openvino>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2024.4.0) + Requirement already satisfied: numpy<2.1.0,>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (1.23.5) + Requirement already satisfied: openvino-telemetry>=2023.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (2024.1.0) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (24.1) Define helpers for PyTorch model initialization and conversion diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg index 6160562199f757..f5b1d98eea3213 100644 --- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg +++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b6475f9155bca7152327255fb36bac32dea6f10aa834b6a91d46d4ecc718be0f -size 58279 +oid sha256:0df4e94924f81aab66086702d85a461f463078f0d06f67b1fe5d46ad8480aa91 +size 58652 diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png index 77ac4e5e0c1fd8..f676b44edd1d9a 100644 --- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png +++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b98f4e2e56bb62e6d7c68e536d05695f16b412d4bb8a502f5ced3c466716fe91 -size 508620 +oid sha256:b5a857cd060d740290ccc65aec47252aad9f41c665dc2808195c3185248977e8 +size 509376 diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg index 98087b7c1f10f2..67719cdcbd66b0 100644 --- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg +++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:02cb8226997c872abaee3c70c1ab90c8e6ac078adb81ec6d0721ece6049e7af3 -size 54825 +oid sha256:ddc40900fddf1a115903c4e200899306060114348bf2ca82fbb4d7d92a885b09 +size 53897 diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png index e317a3a3005ef6..af63ef41697b47 100644 --- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png +++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:940552cfe8d62878a7b2fb827529df2ef6ef4fa135f82f578362142650e0d751 -size 457741 +oid sha256:d1276209027e5aac72e4bb6f39f4494d2a807ee4bd85054a1285b0832e4515b9 +size 460797 diff --git a/docs/notebooks/distil-whisper-asr-with-output.rst b/docs/notebooks/distil-whisper-asr-with-output.rst index 2cdecfe17a19d4..53950226a210c7 100644 --- a/docs/notebooks/distil-whisper-asr-with-output.rst +++ b/docs/notebooks/distil-whisper-asr-with-output.rst @@ -133,7 +133,8 @@ using tokenizer. "distil-whisper/distil-small.en", ], "Whisper": [ - "openai/whisper-large-v3-turbo" "openai/whisper-large-v3", + "openai/whisper-large-v3-turbo", + "openai/whisper-large-v3", "openai/whisper-large-v2", "openai/whisper-large", "openai/whisper-medium", diff --git a/docs/notebooks/distilbert-sequence-classification-with-output.rst b/docs/notebooks/distilbert-sequence-classification-with-output.rst index 5f069551357b6d..862079f68aeeb7 100644 --- a/docs/notebooks/distilbert-sequence-classification-with-output.rst +++ b/docs/notebooks/distilbert-sequence-classification-with-output.rst @@ -47,31 +47,31 @@ Imports .. parsed-literal:: Looking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cpu - Requirement already satisfied: openvino>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2024.4.0) - Requirement already satisfied: transformers in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.45.2) - Requirement already satisfied: torch>=2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.4.1+cpu) - Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.66.5) - Requirement already satisfied: numpy<2.1.0,>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (1.23.5) - Requirement already satisfied: openvino-telemetry>=2023.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (2024.1.0) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (24.1) - Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (3.16.1) - Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.26.1) - Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (6.0.2) - Requirement already satisfied: regex!=2019.12.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2024.9.11) - Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2.32.3) - Requirement already satisfied: safetensors>=0.4.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.4.5) - Requirement already satisfied: tokenizers<0.21,>=0.20 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.20.1) - Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (4.12.2) - Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (1.13.3) - Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (3.1) - Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (3.1.4) - Requirement already satisfied: fsspec in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (2024.9.0) - Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch>=2.1) (2.1.5) - Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.4.0) - Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.10) - Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2.2.3) - Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2024.8.30) - Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch>=2.1) (1.3.0) + Requirement already satisfied: openvino>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2024.4.0) + Requirement already satisfied: transformers in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.46.1) + Requirement already satisfied: torch>=2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.4.1+cpu) + Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.66.6) + Requirement already satisfied: numpy<2.1.0,>=1.16.6 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (1.23.5) + Requirement already satisfied: openvino-telemetry>=2023.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (2024.1.0) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino>=2023.1.0) (24.1) + Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (3.16.1) + Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.26.2) + Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (6.0.2) + Requirement already satisfied: regex!=2019.12.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2024.9.11) + Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (2.32.3) + Requirement already satisfied: safetensors>=0.4.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.4.5) + Requirement already satisfied: tokenizers<0.21,>=0.20 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from transformers) (0.20.2) + Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (4.12.2) + Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (1.13.3) + Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (3.1) + Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (3.1.4) + Requirement already satisfied: fsspec in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1) (2024.9.0) + Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch>=2.1) (2.1.5) + Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.4.0) + Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (3.10) + Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2.2.3) + Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests->transformers) (2024.8.30) + Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch>=2.1) (1.3.0) Note: you may need to restart the kernel to use updated packages. @@ -110,6 +110,15 @@ model from Hugging Face. checkpoint = "distilbert-base-uncased-finetuned-sst-2-english" model = AutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path=checkpoint) + +.. parsed-literal:: + + 2024-11-04 23:18:47.102633: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-04 23:18:47.135966: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2024-11-04 23:18:47.793551: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + + Initializing the Tokenizer -------------------------- @@ -166,10 +175,15 @@ optimal execution on end-point target devices. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4779: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. + + +.. parsed-literal:: + + [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:215: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. - mask, torch.tensor(torch.finfo(scores.dtype).min) + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. OpenVINO™ Runtime uses the `Infer diff --git a/docs/notebooks/dolly-2-instruction-following-with-output.rst b/docs/notebooks/dolly-2-instruction-following-with-output.rst index 9f6857b608d962..3b276a1725f0aa 100644 --- a/docs/notebooks/dolly-2-instruction-following-with-output.rst +++ b/docs/notebooks/dolly-2-instruction-following-with-output.rst @@ -136,6 +136,8 @@ documentation `__. .. code:: ipython3 import os + from pathlib import Path + import requests os.environ["GIT_CLONE_PROTECTION_ACTIVE"] = "false" @@ -144,12 +146,17 @@ documentation `__. %pip install -q "diffusers>=0.16.1" "transformers>=4.33.0" "torch>=2.1" "nncf>=2.10.0" "onnx<1.16.2" "gradio>=4.19" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "git+https://github.com/huggingface/optimum-intel.git" - import requests - r = requests.get( - url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", - ) - open("notebook_utils.py", "w").write(r.text) + utility_files = ["notebook_utils.py", "cmd_helper.py"] + + for utility in utility_files: + local_path = Path(utility) + if not local_path.exists(): + r = requests.get( + url=f"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/{local_path.name}", + ) + with local_path.open("w") as f: + f.write(r.text) Convert model using Optimum-CLI tool ------------------------------------ @@ -227,7 +234,7 @@ sacrifice of the model size and inference latency. .. code:: ipython3 - from IPython.display import Markdown, display + from IPython.display import display import ipywidgets as widgets prepare_int4_model = widgets.Checkbox( @@ -272,6 +279,7 @@ sacrifice of the model size and inference latency. .. code:: ipython3 from pathlib import Path + from cmd_helper import optimum_cli model_id = "databricks/dolly-v2-3b" model_path = Path("dolly-v2-3b") @@ -284,36 +292,19 @@ sacrifice of the model size and inference latency. def convert_to_fp16(): if (fp16_model_dir / "openvino_model.xml").exists(): return - fp16_model_dir.mkdir(parents=True, exist_ok=True) - export_command_base = "optimum-cli export openvino --model {} --task text-generation-with-past --weight-format fp16".format(model_id) - export_command = export_command_base + " " + str(fp16_model_dir) - display(Markdown("**Export command:**")) - display(Markdown(f"`{export_command}`")) - ! $export_command + optimum_cli(model_id, fp16_model_dir, additional_args={"weight-format": "fp16"}) def convert_to_int8(): if (int8_model_dir / "openvino_model.xml").exists(): return - int8_model_dir.mkdir(parents=True, exist_ok=True) - export_command_base = "optimum-cli export openvino --model {} --task text-generation-with-past --weight-format int8".format(model_id) - export_command = export_command_base + " " + str(int8_model_dir) - display(Markdown("**Export command:**")) - display(Markdown(f"`{export_command}`")) - ! $export_command + optimum_cli(model_id, int8_model_dir, additional_args={"weight-format": "int8"}) def convert_to_int4(): if (int4_model_dir / "openvino_model.xml").exists(): return - int4_model_dir.mkdir(parents=True, exist_ok=True) - export_command_base = "optimum-cli export openvino --model {} --task text-generation-with-past --weight-format int4 --ratio 1.0 --group-size 128".format( - model_id - ) - export_command = export_command_base + " " + str(int4_model_dir) - display(Markdown("**Export command:**")) - display(Markdown(f"`{export_command}`")) - ! $export_command + optimum_cli(model_id, int4_model_dir, additional_args={"weight-format": "int4"}) if prepare_fp16_model.value: diff --git a/docs/notebooks/dynamicrafter-animating-images-with-output.rst b/docs/notebooks/dynamicrafter-animating-images-with-output.rst index 584c3442c94af2..992c346194e31c 100644 --- a/docs/notebooks/dynamicrafter-animating-images-with-output.rst +++ b/docs/notebooks/dynamicrafter-animating-images-with-output.rst @@ -189,10 +189,10 @@ Prerequisites remote: Counting objects: 100% (153/153), done. remote: Compressing objects: 100% (99/99), done. remote: Total 335 (delta 97), reused 54 (delta 54), pack-reused 182 (from 1) - Receiving objects: 100% (335/335), 72.41 MiB | 22.40 MiB/s, done. + Receiving objects: 100% (335/335), 72.41 MiB | 20.85 MiB/s, done. Resolving deltas: 100% (123/123), done. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images @@ -282,7 +282,7 @@ We will use model for 256x256 resolution as example. Also, models for .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/huggingface_hub/file_download.py:834: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/huggingface_hub/file_download.py:834: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`. For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder. warnings.warn( @@ -300,8 +300,12 @@ We will use model for 256x256 resolution as example. Also, models for .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) + 2024-11-04 23:23:38.980054: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-04 23:23:39.013901: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2024-11-04 23:23:39.616188: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -384,6 +388,17 @@ Convert CLIP text encoder del cond_stage_model gc.collect(); + +.. parsed-literal:: + + WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. + + +.. parsed-literal:: + + WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. + + Convert CLIP image encoder ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -409,43 +424,43 @@ resolutions. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/utils/image.py:226: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/utils/image.py:226: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input.numel() == 0: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:573: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:573: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if size == input_size: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:579: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:579: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! antialias = antialias and (max(factors) > 1) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:581: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:581: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if antialias: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:584: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:584: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! sigmas = (max((factors[0] - 1.0) / 2.0, 0.001), max((factors[1] - 1.0) / 2.0, 0.001)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:589: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:589: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! ks = int(max(2.0 * 2 * sigmas[0], 3)), int(max(2.0 * 2 * sigmas[1], 3)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:589: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/geometry/transform/affwarp.py:589: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! ks = int(max(2.0 * 2 * sigmas[0], 3)), int(max(2.0 * 2 * sigmas[1], 3)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/gaussian.py:55: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/gaussian.py:55: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. sigma = tensor([sigma], device=input.device, dtype=input.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/gaussian.py:55: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/gaussian.py:55: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! sigma = tensor([sigma], device=input.device, dtype=input.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/core/check.py:78: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/core/check.py:78: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if x_shape_to_check[i] != dim: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/kernels.py:92: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/filters/kernels.py:92: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. mean = tensor([[mean]], device=sigma.device, dtype=sigma.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:101: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:101: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if len(mean.shape) == 0 or mean.shape[0] == 1: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if len(std.shape) == 0 or std.shape[0] == 1: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:107: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:107: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if mean.shape and mean.shape[0] != 1: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:108: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:108: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if mean.shape[0] != data.shape[1] and mean.shape[:2] != data.shape[:2]: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:112: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:112: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if std.shape and std.shape[0] != 1: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:113: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:113: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if std.shape[0] != data.shape[1] and std.shape[:2] != data.shape[:2]: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:116: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:116: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. mean = torch.as_tensor(mean, device=data.device, dtype=data.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:117: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/kornia/enhance/normalize.py:117: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. std = torch.as_tensor(std, device=data.device, dtype=data.dtype) @@ -474,7 +489,7 @@ Convert AE encoder .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/ae_modules.py:67: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/ae_modules.py:67: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! w_ = w_ * (int(c)**(-0.5)) @@ -518,15 +533,15 @@ Convert Diffusion U-Net model .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:556: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:556: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if l_context == 77 + t*16: ## !!! HARD CODE here - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:205: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:205: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if batch_size: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:232: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:232: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if self.use_temporal_conv and batch_size: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:76: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:76: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert x.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:99: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/dynamicrafter-animating-images/dynamicrafter/lvdm/modules/networks/openaimodel3d.py:99: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert x.shape[1] == self.channels @@ -913,14 +928,14 @@ Run OpenVINO pipeline inference .. parsed-literal:: Seed set to 234 - /tmp/ipykernel_2590985/2451984876.py:25: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at ../torch/csrc/utils/tensor_numpy.cpp:206.) + /tmp/ipykernel_511478/2451984876.py:25: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at ../torch/csrc/utils/tensor_numpy.cpp:206.) img_tensor = torch.from_numpy(image).permute(2, 0, 1).float().to(model.device) .. parsed-literal:: - start: man fishing in a boat at sunset 2024-10-22 23:16:18 - Saved in man_fishing_in_a_boat_at_sunset.mp4. Time used: 196.25 seconds + start: man fishing in a boat at sunset 2024-11-04 23:26:56 + Saved in man_fishing_in_a_boat_at_sunset.mp4. Time used: 206.55 seconds .. code:: ipython3 @@ -1170,14 +1185,6 @@ quantization time. INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino -.. parsed-literal:: - - 2024-10-22 23:45:51.693284: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-22 23:45:51.735392: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-22 23:45:52.354791: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - - .. parsed-literal:: @@ -1357,8 +1364,8 @@ Let’s run the optimized pipeline .. parsed-literal:: - start: man fishing in a boat at sunset 2024-10-23 00:47:13 - Saved in man_fishing_in_a_boat_at_sunset.mp4. Time used: 98.40 seconds + start: man fishing in a boat at sunset 2024-11-05 00:58:08 + Saved in man_fishing_in_a_boat_at_sunset.mp4. Time used: 97.78 seconds .. code:: ipython3 @@ -1463,9 +1470,9 @@ models, we use median inference time on calibration subset. .. parsed-literal:: - FP32 latency: 195.358 - INT8 latency: 97.265 - Performance speed up: 2.009 + FP32 latency: 201.526 + INT8 latency: 96.036 + Performance speed up: 2.098 Interactive inference diff --git a/docs/notebooks/efficient-sam-with-output.rst b/docs/notebooks/efficient-sam-with-output.rst index 8d725c4594afc6..b50b82341f4af8 100644 --- a/docs/notebooks/efficient-sam-with-output.rst +++ b/docs/notebooks/efficient-sam-with-output.rst @@ -108,9 +108,9 @@ Prerequisites remote: Counting objects: 100% (85/85), done. remote: Compressing objects: 100% (33/33), done. remote: Total 424 (delta 76), reused 52 (delta 52), pack-reused 339 (from 1) - Receiving objects: 100% (424/424), 262.14 MiB | 24.94 MiB/s, done. + Receiving objects: 100% (424/424), 262.14 MiB | 23.37 MiB/s, done. Resolving deltas: 100% (246/246), done. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM .. code:: ipython3 @@ -377,23 +377,23 @@ disk using ``openvino.save_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:220: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:220: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if ( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:241: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:241: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert ( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:163: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:163: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! size = int(math.sqrt(xy_num)) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert size * size == xy_num - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:166: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:166: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if size != h or size != w: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:251: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam_encoder.py:251: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert x.shape[2] == num_patches - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if num_pts > self.decoder_max_num_input_points: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:92: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:92: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! elif num_pts < self.decoder_max_num_input_points: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:126: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam/EfficientSAM/efficient_sam/efficient_sam.py:126: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if output_w > 0 and output_h > 0: @@ -640,10 +640,10 @@ architecture type, we should specify ``transformer`` in ``model_type``. .. parsed-literal:: - 2024-10-23 01:04:24.934254: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:04:24.966235: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 01:15:40.935673: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 01:15:40.968460: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:04:25.612813: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 01:15:41.606156: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -810,7 +810,7 @@ models, we use ``bencmark_app``. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 30.59 ms + [ INFO ] Read model took 30.24 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] batched_images (node: batched_images) : f32 / [...] / [?,?,?,?] @@ -830,7 +830,7 @@ models, we use ``bencmark_app``. [ INFO ] ***NO_NAME*** (node: aten::reshape/Reshape_3) : f32 / [...] / [?,?,?,?,?] [ INFO ] ***NO_NAME*** (node: aten::reshape/Reshape_2) : f32 / [...] / [?,?,?] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 1397.68 ms + [ INFO ] Compile model took 1388.43 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -871,17 +871,17 @@ models, we use ``bencmark_app``. [ INFO ] Fill input 'batched_point_labels' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in full mode (inputs filling are included in measurement loop). - [ INFO ] First inference took 822.58 ms + [ INFO ] First inference took 798.46 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 51 iterations - [ INFO ] Duration: 16272.65 ms + [ INFO ] Count: 49 iterations + [ INFO ] Duration: 16827.30 ms [ INFO ] Latency: - [ INFO ] Median: 1835.79 ms - [ INFO ] Average: 1847.61 ms - [ INFO ] Min: 1271.60 ms - [ INFO ] Max: 2300.40 ms - [ INFO ] Throughput: 3.13 FPS + [ INFO ] Median: 2025.54 ms + [ INFO ] Average: 1991.09 ms + [ INFO ] Min: 816.09 ms + [ INFO ] Max: 2176.67 ms + [ INFO ] Throughput: 2.91 FPS .. code:: ipython3 @@ -907,7 +907,7 @@ models, we use ``bencmark_app``. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 43.92 ms + [ INFO ] Read model took 43.95 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] batched_images (node: batched_images) : f32 / [...] / [?,?,?,?] @@ -927,7 +927,7 @@ models, we use ``bencmark_app``. [ INFO ] ***NO_NAME*** (node: aten::reshape/Reshape_3) : f32 / [...] / [?,?,?,?,?] [ INFO ] ***NO_NAME*** (node: aten::reshape/Reshape_2) : f32 / [...] / [?,?,?] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 1623.93 ms + [ INFO ] Compile model took 1607.96 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -968,17 +968,17 @@ models, we use ``bencmark_app``. [ INFO ] Fill input 'batched_point_labels' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in full mode (inputs filling are included in measurement loop). - [ INFO ] First inference took 578.24 ms + [ INFO ] First inference took 596.94 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] [ INFO ] Count: 55 iterations - [ INFO ] Duration: 15797.98 ms + [ INFO ] Duration: 15959.69 ms [ INFO ] Latency: - [ INFO ] Median: 1695.87 ms - [ INFO ] Average: 1683.49 ms - [ INFO ] Min: 550.40 ms - [ INFO ] Max: 1833.79 ms - [ INFO ] Throughput: 3.48 FPS + [ INFO ] Median: 1701.74 ms + [ INFO ] Average: 1692.86 ms + [ INFO ] Min: 653.76 ms + [ INFO ] Max: 1817.85 ms + [ INFO ] Throughput: 3.45 FPS Interactive segmentation demo @@ -1308,7 +1308,7 @@ Interactive segmentation demo .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/efficient-sam Running on local URL: http://127.0.0.1:7860 To create a public link, set `share=True` in `launch()`. diff --git a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_17_1.png b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_17_1.png index c6c2e7d72e694a..9f65fa9db4554a 100644 --- a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_17_1.png +++ b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_17_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ee7d364aeabe9a2787785e4d0ee5bb8951f530f90a0a625c9f31047bdc157b59 -size 1261144 +oid sha256:9368b1fbd458d1e022a768f24e689af0fd6e5dacc98a920f45d3fc0f63062567 +size 1259373 diff --git a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_25_1.png b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_25_1.png index ff03bff6360a90..7c0716600906a1 100644 --- a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_25_1.png +++ b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_25_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:40a049edc89f5339af256b80e0fc99740d9e3b6d3159e6e9d09e78683d24e0fe -size 1261722 +oid sha256:22f0e5bfd74e7426218d2bd007f9219433556530ddb10f33b9706398eb7cd370 +size 1263404 diff --git a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_36_1.png b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_36_1.png index 766db096877cba..0a717e2c9aa38d 100644 --- a/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_36_1.png +++ b/docs/notebooks/efficient-sam-with-output_files/efficient-sam-with-output_36_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f4fcf4eecf855fa1a9ca72c0dbf5c070e11a4a520873971202b592bc65dd2ccc -size 1261487 +oid sha256:d1863ccc9483f6cbd60768b311d104ee68692c3a7181e06da4bc751b52cf0ca1 +size 1262535 diff --git a/docs/notebooks/encodec-audio-compression-with-output.rst b/docs/notebooks/encodec-audio-compression-with-output.rst index 394c6e5b92e9bf..7f0e153ffa4a55 100644 --- a/docs/notebooks/encodec-audio-compression-with-output.rst +++ b/docs/notebooks/encodec-audio-compression-with-output.rst @@ -142,7 +142,7 @@ bandwidth. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. WeightNorm.apply(module, name, dim) @@ -302,7 +302,7 @@ similar as possible to the original. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. WeightNorm.apply(module, name, dim) @@ -402,13 +402,13 @@ with ``ov.save_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:60: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:60: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! ideal_length = (math.ceil(n_frames) - 1) * stride + (kernel_size - padding_total) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:87: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:87: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! max_pad = max(padding_left, padding_right) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:89: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:89: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if length <= max_pad: @@ -428,11 +428,11 @@ with ``ov.save_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:358: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:358: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. quantized_out = torch.tensor(0.0, device=q_indices.device) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:359: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results). + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/quantization/core_vq.py:359: TracerWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results). for i, indices in enumerate(q_indices): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/encodec/modules/conv.py:103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert (padding_left + padding_right) <= x.shape[-1] diff --git a/docs/notebooks/fast-segment-anything-with-output.rst b/docs/notebooks/fast-segment-anything-with-output.rst index c29a58c33e3490..e0f20e0f79974b 100644 --- a/docs/notebooks/fast-segment-anything-with-output.rst +++ b/docs/notebooks/fast-segment-anything-with-output.rst @@ -158,7 +158,7 @@ model and generate a segmentation map. .. parsed-literal:: - 100%|██████████| 138M/138M [00:02<00:00, 56.5MB/s] + 100%|██████████| 138M/138M [00:02<00:00, 67.7MB/s] @@ -170,8 +170,8 @@ model and generate a segmentation map. .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/fast-segment-anything/coco_bike.jpg: 768x1024 37 objects, 662.7ms - Speed: 3.8ms preprocess, 662.7ms inference, 766.0ms postprocess per image at shape (1, 3, 768, 1024) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/fast-segment-anything/coco_bike.jpg: 768x1024 37 objects, 728.3ms + Speed: 3.1ms preprocess, 728.3ms inference, 768.2ms postprocess per image at shape (1, 3, 768, 1024) The model returns segmentation maps for all the objects on the image. @@ -216,8 +216,8 @@ tracing. The FastSAM model itself is based on YOLOv8 model. OpenVINO: starting export with openvino 2024.4.0-16579-c3152d32c9c-releases/2024/4... OpenVINO: export success ✅ 6.2s, saved as 'FastSAM-x_openvino_model/' (276.1 MB) - Export complete (9.2s) - Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/fast-segment-anything + Export complete (9.1s) + Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/fast-segment-anything Predict: yolo predict task=segment model=FastSAM-x_openvino_model imgsz=1024 Validate: yolo val task=segment model=FastSAM-x_openvino_model imgsz=1024 data=ultralytics/datasets/sa.yaml Visualize: https://netron.app @@ -321,8 +321,8 @@ pipeline. .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/fast-segment-anything/coco_bike.jpg: 1024x1024 42 objects, 558.8ms - Speed: 5.7ms preprocess, 558.8ms inference, 34.6ms postprocess per image at shape (1, 3, 1024, 1024) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/fast-segment-anything/coco_bike.jpg: 1024x1024 42 objects, 504.9ms + Speed: 5.8ms preprocess, 504.9ms inference, 31.6ms postprocess per image at shape (1, 3, 1024, 1024) One can observe the converted model outputs in the next cell, they is @@ -643,9 +643,9 @@ calibration dataset to measure the performance. .. parsed-literal:: - Segmented in 23 seconds - Resulting in 5.57 fps - That is 3.0 times faster! + Segmented in 22 seconds + Resulting in 5.82 fps + That is 3.14 times faster! Try out the converted pipeline diff --git a/docs/notebooks/florence2-with-output.rst b/docs/notebooks/florence2-with-output.rst index 4cca2d85a2bd6c..e929a95fb182c1 100644 --- a/docs/notebooks/florence2-with-output.rst +++ b/docs/notebooks/florence2-with-output.rst @@ -100,10 +100,10 @@ available model. By default, we will use .. parsed-literal:: - 2024-10-23 01:17:38.434215: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:17:38.467940: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 01:28:54.034484: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 01:28:54.069316: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:17:39.118965: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 01:28:54.728430: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -193,49 +193,49 @@ pipeline. .. parsed-literal:: - SECURITY.md: 0%| | 0.00/2.66k [00:00 1 or self.sliding_window is not None: /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/chkpt/modeling_florence2.py:1205: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! is_causal = True if self.is_causal and attention_mask is None and tgt_len > 1 else False diff --git a/docs/notebooks/florence2-with-output_files/florence2-with-output_18_0.png b/docs/notebooks/florence2-with-output_files/florence2-with-output_18_0.png index a1484a3eb4b6d0..37d11a47fd30c9 100644 --- a/docs/notebooks/florence2-with-output_files/florence2-with-output_18_0.png +++ b/docs/notebooks/florence2-with-output_files/florence2-with-output_18_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f5808e660c685e4518f0a83232c2b9c9e88263e05ecd05891f769d263df4c642 -size 259656 +oid sha256:d85b3df68708172ed849a9e182bdec6a94f0174643833bd8cc7184ac0d090fae +size 259636 diff --git a/docs/notebooks/flux.1-image-generation-with-output.rst b/docs/notebooks/flux.1-image-generation-with-output.rst index 62549bd074d4a6..fe0c47899c3601 100644 --- a/docs/notebooks/flux.1-image-generation-with-output.rst +++ b/docs/notebooks/flux.1-image-generation-with-output.rst @@ -26,7 +26,11 @@ using OpenVINO. - `Prerequisites <#prerequisites>`__ - `Select model <#select-model>`__ - `Convert model with OpenVINO <#convert-model-with-openvino>`__ -- `Compress model weights <#compress-model-weights>`__ + + - `Convert model using Optimum + Intel <#convert-model-using-optimum-intel>`__ + - `Compress model weights <#compress-model-weights>`__ + - `Run OpenVINO model inference <#run-openvino-model-inference>`__ - `Interactive demo <#interactive-demo>`__ @@ -47,8 +51,9 @@ Prerequisites .. code:: ipython3 - %pip install -q "gradio>=4.19" "torch>=2.1" "transformers" "nncf>=2.12.0" "diffusers>=0.30.0" "opencv-python" "pillow" "peft>=0.7.0" --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q "gradio>=4.19" "torch>=2.1" "transformers" "nncf>=2.12.0" "diffusers>=0.31.0" "opencv-python" "pillow" "peft>=0.7.0" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "sentencepiece" "protobuf" + %pip install -q "git+https://github.com/huggingface/optimum-intel.git" %pip install -qU "openvino>=2024.4.0" .. code:: ipython3 @@ -56,9 +61,9 @@ Prerequisites import requests from pathlib import Path - if not Path("flux_helper.py").exists(): - r = requests.get(url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/notebooks/flux.1-image-generation/flux_helper.py") - open("flux_helper.py", "w").write(r.text) + if not Path("cmd_helper.py").exists(): + r = requests.get(url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/cmd_helper.py") + open("cmd_helper.py", "w").write(r.text) if not Path("gradio_helper.py").exists(): r = requests.get(url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/notebooks/flux.1-image-generation/gradio_helper.py") @@ -100,25 +105,19 @@ FLUX.1-dev version using widget bellow. .. code:: ipython3 - from flux_helper import get_model_selector + import ipywidgets as widgets - model_selector = get_model_selector() - model_selector - - -.. parsed-literal:: - - INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino + model_ids = ["black-forest-labs/FLUX.1-schnell", "black-forest-labs/FLUX.1-dev"] + + model_selector = widgets.Dropdown( + options=model_ids, + default=model_ids[0], + description="Model:", + ) - -.. parsed-literal:: - - 2024-08-13 17:30:13.543036: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-08-13 17:30:13.544738: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used. - 2024-08-13 17:30:13.579013: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-08-13 17:30:14.449873: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + model_selector + @@ -169,40 +168,44 @@ The pipeline consists of four important parts: - Transformer for step-by-step denoising latent image representation. - Autoencoder (VAE) for decoding latent space to image. -We will use ``convert_flux`` helper function defined in -`flux_helper.py `__ that create original PyTorch model -and convert each part of pipeline using ``ov.convert_model``. - -.. code:: ipython3 +Convert model using Optimum Intel +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - from flux_helper import convert_flux - - # uncomment the line to see model conversion code - # ??convert_flux -.. code:: ipython3 - model_dir = convert_flux(model_selector.value) +For convenience, we will use OpenVINO integration with HuggingFace +Optimum. `Optimum +Intel `__ is the +interface between the Transformers and Diffusers libraries and the +different tools and libraries provided by Intel to accelerate end-to-end +pipelines on Intel architectures. +Among other use cases, Optimum Intel provides a simple interface to +optimize your Transformers and Diffusers models, convert them to the +OpenVINO Intermediate Representation (IR) format and run inference using +OpenVINO Runtime. ``optimum-cli`` provides command line interface for +model conversion and optimization. -.. parsed-literal:: +General command format: - ✅ black-forest-labs/FLUX.1-schnell model already converted and can be found in FLUX.1-schnell - +.. code:: bash -.. code:: ipython3 + optimum-cli export openvino --model --task - from flux_helper import TRANSFORMER_PATH, VAE_DECODER_PATH, TEXT_ENCODER_PATH, TEXT_ENCODER_2_PATH - - model_dict = { - "transformer": model_dir / TRANSFORMER_PATH, - "text_encoder": model_dir / TEXT_ENCODER_PATH, - "text_encoder_2": model_dir / TEXT_ENCODER_2_PATH, - "vae": model_dir / VAE_DECODER_PATH, - } +where task is task to export the model for, if not specified, the task +will be auto-inferred based on the model. You can find a mapping between +tasks and model classes in Optimum TaskManager +`documentation `__. +Additionally, you can specify weights compression using +``--weight-format`` argument with one of following options: ``fp32``, +``fp16``, ``int8`` and ``int4``. Fro int8 and int4 +`nncf `__ will be used for +weight compression. More details about model export provided in `Optimum +Intel +documentation `__. Compress model weights ----------------------- +~~~~~~~~~~~~~~~~~~~~~~ @@ -215,14 +218,16 @@ where the size of weights is relatively larger than the size of activations, for example, Large Language Models (LLM). Compared to INT8 compression, INT4 compression improves performance even more, but introduces a minor drop in prediction quality. We will use -`NNCF `__ for weight -compression. +`NNCF `__ integration to +``optimum-cli`` tool for weight compression. .. code:: ipython3 - from flux_helper import weight_compression_widget - - to_compress = weight_compression_widget() + to_compress = widgets.Checkbox( + value=True, + description="Weight compression", + disabled=False, + ) to_compress @@ -237,112 +242,98 @@ compression. .. code:: ipython3 - import nncf - import openvino as ov - import gc + from pathlib import Path - compression_args = {"mode": nncf.CompressWeightsMode.INT4_SYM, "group_size": 64, "ratio": 1.0} + model_id = model_selector.value - int4_model_dict = {} + model_base_dir = Path(model_id.split("/")[-1]) + additional_args = {} if to_compress.value: - core = ov.Core() - - for model_name, model_path in model_dict.items(): - int4_path = model_path.parent / (model_path.stem + "_int4.xml") - if not int4_path.exists(): - print(f"⌛ {model_path.stem} compression started") - print( - f"Compression parameters:\n\tmode = {compression_args['mode']}\n\tratio = {compression_args['ratio']}\n\tgroup_size = {compression_args['group_size']}" - ) - model = core.read_model(model_path) - compressed_model = nncf.compress_weights(model, **compression_args) - ov.save_model(compressed_model, int4_path) - print(f"✅ {model_path.stem} compression finished") - del compressed_model - del model - gc.collect() - print(f"Compressed {model_path.stem} can be found in {int4_path}") - int4_model_dict[model_name] = int4_path + model_dir = model_base_dir / "INT4" + additional_args.update({"weight-format": "int4", "group-size": "64", "ratio": "1.0"}) + else: + model_dir = model_base_dir / "FP16" + additional_args.update({"weight-format": "fp16"}) +.. code:: ipython3 -.. parsed-literal:: - - Compressed transformer can be found in FLUX.1-schnell/transformer/transformer_int4.xml - Compressed text_encoder can be found in FLUX.1-schnell/text_encoder/text_encoder_int4.xml - Compressed text_encoder_2 can be found in FLUX.1-schnell/text_encoder_2/text_encoder_2_int4.xml - Compressed vae_decoder can be found in FLUX.1-schnell/vae/vae_decoder_int4.xml + from cmd_helper import optimum_cli + if not model_dir.exists(): + optimum_cli(model_id, model_dir, additional_args=additional_args) Run OpenVINO model inference ---------------------------- -``OVFluxPipeline`` class defined in ``flux_helper.py`` provides -convenient way for running model. It accepts directory with converted -model and inference device as arguments. +``OVDiffusionPipeline`` from Optimum Intel provides ready-to-use +interface for running Diffusers models using OpenVINO. It supports +various models including Stable Diffusion, Stable Diffusion XL, LCM, +Stable Diffusion v3 and Flux. Similar to original Diffusers pipeline, +for initialization, we should use ``from_preptrained`` method providing +model id from HuggingFace hub or local directory (both original PyTorch +and OpenVINO models formats supported, in the first case model class +additionally will trigger model conversion). .. code:: ipython3 - from flux_helper import get_pipeline_selection_option + from notebook_utils import device_widget - use_compressed = get_pipeline_selection_option(int4_model_dict) - use_compressed + device = device_widget(default="CPU", exclude=["NPU"]) + device .. parsed-literal:: - Checkbox(value=True, description='Use compressed models') + Dropdown(description='Device:', options=('CPU', 'AUTO'), value='CPU') .. code:: ipython3 - from flux_helper import OVFluxPipeline, init_pipeline # noqa: F401 + import ipywidgets as widgets - # uncomment the line to see model pipeline - # ??OVFluxPipeline - -.. code:: ipython3 - - from notebook_utils import device_widget + model_available = (model_base_dir / "INT4").is_dir() + use_quantized_models = widgets.Checkbox( + value=model_available, + description="Use compressed models", + disabled=not model_available, + ) - device = device_widget(default="CPU", exclude=["NPU"]) - device + use_quantized_models .. parsed-literal:: - Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') + Checkbox(value=True, description='Use compressed models') .. code:: ipython3 - ov_pipe = init_pipeline(model_dir, model_dict if not use_compressed.value else int4_model_dict, device.value) - - -.. parsed-literal:: - - Models compilation - ✅ transformer - Done! - ✅ text_encoder - Done! - ✅ text_encoder_2 - Done! + from optimum.intel.openvino import OVDiffusionPipeline - -.. parsed-literal:: - - You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers + model_dir = model_base_dir / "INT4" if use_quantized_models.value else model_base_dir / "FP16" + ov_pipe = OVDiffusionPipeline.from_pretrained(model_dir, device=device.value) + .. parsed-literal:: - ✅ vae - Done! + 2024-10-28 18:12:30.714636: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-10-28 18:12:30.727116: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:477] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered + WARNING: All log messages before absl::InitializeLog() is called are written to STDERR + E0000 00:00:1730124750.741387 52454 cuda_dnn.cc:8310] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered + E0000 00:00:1730124750.745955 52454 cuda_blas.cc:1418] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered + 2024-10-28 18:12:30.761443: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers .. code:: ipython3 @@ -365,7 +356,7 @@ model and inference device as arguments. -.. image:: flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.png +.. image:: flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.png diff --git a/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.jpg b/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.jpg new file mode 100644 index 00000000000000..6ad8e593ef5115 --- /dev/null +++ b/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5caeca2e3ae8e99146bb786321cbad04584523b693e3cbdba2a5f8c7f0dbb096 +size 13490 diff --git a/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.png b/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.png new file mode 100644 index 00000000000000..7cb1e2903a902f --- /dev/null +++ b/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_16_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0de23e409e068f39c1e0e568b921174c80b0c42e29a1473eeeba54a91b88aaef +size 113356 diff --git a/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.jpg b/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.jpg deleted file mode 100644 index 7223074e225dd8..00000000000000 --- a/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.jpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8923d4a7ce04ed66bb58c28ab4450594a0340b97a15245fc594e669d33b574ef -size 14369 diff --git a/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.png b/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.png deleted file mode 100644 index 83b60f80d75c13..00000000000000 --- a/docs/notebooks/flux.1-image-generation-with-output_files/flux.1-image-generation-with-output_20_1.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:fab1fa5bf3f71d6bb6f505658da6765d37295b7b9618560d5f7c0c344e2a3896 -size 116250 diff --git a/docs/notebooks/freevc-voice-conversion-with-output.rst b/docs/notebooks/freevc-voice-conversion-with-output.rst index a3c914a44b36a7..fe2ac780f5cca6 100644 --- a/docs/notebooks/freevc-voice-conversion-with-output.rst +++ b/docs/notebooks/freevc-voice-conversion-with-output.rst @@ -104,7 +104,7 @@ Check if FreeVC is installed and append its path to ``sys.path`` remote: Counting objects: 100% (74/74), done. remote: Compressing objects: 100% (47/47), done. remote: Total 131 (delta 43), reused 27 (delta 27), pack-reused 57 (from 1) - Receiving objects: 100% (131/131), 15.28 MiB | 3.81 MiB/s, done. + Receiving objects: 100% (131/131), 15.28 MiB | 17.50 MiB/s, done. Resolving deltas: 100% (43/43), done. @@ -134,8 +134,8 @@ Check if FreeVC is installed and append its path to ``sys.path`` Downloading... From: https://drive.google.com/uc?id=12-cB34qCTvByWT-QtOcZaqwwO21FLSqU&confirm=t&uuid=a703c43c-ccce-436c-8799-c11b88e9e7e4 - To: /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/WavLM-Large.pt - 100%|██████████| 1.26G/1.26G [00:26<00:00, 48.4MB/s] + To: /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/WavLM-Large.pt + 100%|██████████| 1.26G/1.26G [00:32<00:00, 38.5MB/s] .. code:: ipython3 @@ -239,7 +239,7 @@ Models initialization .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. warnings.warn("torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.") @@ -288,7 +288,7 @@ Inference .. parsed-literal:: - 2it [00:03, 1.72s/it] + 2it [00:04, 2.03s/it] Result audio files should be available in ‘outputs/freevc’ @@ -360,13 +360,13 @@ Converting to OpenVINO’s IR format. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:495: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:495: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert embed_dim == self.embed_dim - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:496: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:496: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert list(query.size()) == [tgt_len, bsz, embed_dim] - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:500: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:500: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert key_bsz == bsz - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:502: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/freevc-voice-conversion/FreeVC/wavlm/modules.py:502: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert src_len, bsz == value.shape[:2] @@ -581,12 +581,12 @@ function to OpenVINO IR format. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1102: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1102: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: Tensor-likes are not close! - Mismatched elements: 25909 / 25920 (100.0%) - Greatest absolute difference: 0.713585376739502 at index (0, 0, 4783) (up to 1e-05 allowed) - Greatest relative difference: 22806.258007743752 at index (0, 0, 19024) (up to 1e-05 allowed) + Mismatched elements: 25915 / 25920 (100.0%) + Greatest absolute difference: 1.3485908806324005 at index (0, 0, 24258) (up to 1e-05 allowed) + Greatest relative difference: 8204.075456053068 at index (0, 0, 5777) (up to 1e-05 allowed) _check_trace( @@ -645,7 +645,7 @@ And now we can check inference using only IR models. .. parsed-literal:: - 2it [00:01, 1.30it/s] + 2it [00:01, 1.31it/s] Result audio files should be available in ‘outputs/freevc’ and you can @@ -707,7 +707,7 @@ Result audio: diff --git a/docs/notebooks/gpu-device-with-output.rst b/docs/notebooks/gpu-device-with-output.rst index 2b35846b766bd1..732cc297aa9531 100644 --- a/docs/notebooks/gpu-device-with-output.rst +++ b/docs/notebooks/gpu-device-with-output.rst @@ -21,13 +21,7 @@ Working with GPUs in OpenVINO™ - `Compiling a Model on GPU <#compiling-a-model-on-gpu>`__ - - `Download and Convert a Model <#download-and-convert-a-model>`__ - - - `Download and unpack the - Model <#download-and-unpack-the-model>`__ - - `Convert the Model to OpenVINO IR - format <#convert-the-model-to-openvino-ir-format>`__ - + - `Download a Model <#download-a-model>`__ - `Compile with Default Configuration <#compile-with-default-configuration>`__ - `Reduce Compile Time through Model @@ -119,10 +113,7 @@ Install required packages .. code:: ipython3 - %pip install -q "openvino-dev>=2024.0.0" "opencv-python" "tqdm" - %pip install -q "tensorflow-macos>=2.5; sys_platform == 'darwin' and platform_machine == 'arm64' and python_version > '3.8'" # macOS M1 and M2 - %pip install -q "tensorflow>=2.5; sys_platform == 'darwin' and platform_machine != 'arm64' and python_version > '3.8'" # macOS x86 - %pip install -q "tensorflow>=2.5; sys_platform != 'darwin' and python_version > '3.8'" + %pip install -q "openvino>=2024.4.0" "opencv-python" "tqdm" "huggingface_hub" Checking GPUs with Query Device ------------------------------- @@ -305,8 +296,8 @@ properties. We can easily use one for compiling and running models with OpenVINO `GPU plugin `__. -Download and Convert a Model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download a Model +~~~~~~~~~~~~~~~~ @@ -317,19 +308,9 @@ was trained on `Common Objects in Context categories of object. For details, see the `paper `__. -Download and unpack the Model -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - - -Use the ``download_file`` function from the ``notebook_utils`` to -download an archive with the model. It automatically creates a directory -structure and downloads the selected model. This step is skipped if the -package is already downloaded. - .. code:: ipython3 - import tarfile + import huggingface_hub as hf_hub from pathlib import Path # Fetch `notebook_utils` module @@ -340,95 +321,18 @@ package is already downloaded. ) open("notebook_utils.py", "w").write(r.text) - from notebook_utils import download_file # A directory where the model will be downloaded. base_model_dir = Path("./model").expanduser() - model_name = "ssdlite_mobilenet_v2" - archive_name = Path(f"{model_name}_coco_2018_05_09.tar.gz") - - # Download the archive - downloaded_model_path = base_model_dir / archive_name - if not downloaded_model_path.exists(): - model_url = f"http://download.tensorflow.org/models/object_detection/{archive_name}" - download_file(model_url, downloaded_model_path.name, downloaded_model_path.parent) - - # Unpack the model - tf_model_path = base_model_dir / archive_name.with_suffix("").stem / "frozen_inference_graph.pb" - if not tf_model_path.exists(): - with tarfile.open(downloaded_model_path) as file: - file.extractall(base_model_dir) - - - -.. parsed-literal:: - - model/ssdlite_mobilenet_v2_coco_2018_05_09.tar.gz: 0%| | 0.00/48.7M [00:00`__. - -.. code:: ipython3 - - from openvino.tools.mo.front import tf as ov_tf_front - - precision = "FP16" + model_name = "ssdlite_mobilenet_v2_fp16" - # The output path for the conversion. - model_path = base_model_dir / "ir_model" / f"{model_name}_{precision.lower()}.xml" + ov_model_path = base_model_dir / model_name / f"{model_name}.xml" - trans_config_path = Path(ov_tf_front.__file__).parent / "ssd_v2_support.json" - pipeline_config = base_model_dir / archive_name.with_suffix("").stem / "pipeline.config" + if not (ov_model_path).exists(): + hf_hub.snapshot_download("katuni4ka/ssdlite_mobilenet_v2_fp16", local_dir=base_model_dir) - model = None - if not model_path.exists(): - model = ov.tools.mo.convert_model( - input_model=tf_model_path, - input_shape=[1, 300, 300, 3], - layout="NHWC", - transformations_config=trans_config_path, - tensorflow_object_detection_api_pipeline_config=pipeline_config, - reverse_input_channels=True, - ) - ov.save_model(model, model_path, compress_to_fp16=(precision == "FP16")) - print("IR model saved to {}".format(model_path)) - else: - print("Read IR model from {}".format(model_path)) - model = core.read_model(model_path) - - -.. parsed-literal:: - - [ WARNING ] The Preprocessor block has been removed. Only nodes performing mean value subtraction and scaling (if applicable) are kept. - - -.. parsed-literal:: - - IR model saved to model/ir_model/ssdlite_mobilenet_v2_fp16.xml - + model = core.read_model(ov_model_path) Compile with Default Configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -484,7 +388,7 @@ following: core.set_property({props.cache_dir(): cache_folder}) # Compile the model as before - model = core.read_model(model=model_path) + model = core.read_model(ov_model_path) compiled_model = core.compile_model(model, device) print(f"Cache enabled (first time) - compile time: {time.time() - start}s") @@ -502,13 +406,13 @@ compile times with caching enabled and disabled as follows: start = time.time() core = ov.Core() core.set_property({props.cache_dir(): "cache"}) - model = core.read_model(model=model_path) + model = core.read_model(model=ov_model_path) compiled_model = core.compile_model(model, device) print(f"Cache enabled - compile time: {time.time() - start}s") start = time.time() core = ov.Core() - model = core.read_model(model=model_path) + model = core.read_model(ov_model_path) compiled_model = core.compile_model(model, device) print(f"Cache disabled - compile time: {time.time() - start}s") diff --git a/docs/notebooks/grounded-segment-anything-with-output.rst b/docs/notebooks/grounded-segment-anything-with-output.rst index 51522e791e04a4..232629422b14e0 100644 --- a/docs/notebooks/grounded-segment-anything-with-output.rst +++ b/docs/notebooks/grounded-segment-anything-with-output.rst @@ -124,16 +124,16 @@ segmentation you can select vanilla ``SAM``. Cloning into 'GroundingDINO'... remote: Enumerating objects: 379, done. remote: Counting objects: 100% (190/190), done. - remote: Compressing objects: 100% (81/81), done. - remote: Total 379 (delta 135), reused 109 (delta 109), pack-reused 189 (from 1) - Receiving objects: 100% (379/379), 14.03 MiB | 18.28 MiB/s, done. + remote: Compressing objects: 100% (79/79), done. + remote: Total 379 (delta 136), reused 111 (delta 111), pack-reused 189 (from 1) + Receiving objects: 100% (379/379), 14.03 MiB | 20.95 MiB/s, done. Resolving deltas: 100% (194/194), done. Cloning into 'EfficientSAM'... remote: Enumerating objects: 424, done. remote: Counting objects: 100% (85/85), done. remote: Compressing objects: 100% (33/33), done. remote: Total 424 (delta 76), reused 52 (delta 52), pack-reused 339 (from 1) - Receiving objects: 100% (424/424), 262.14 MiB | 30.07 MiB/s, done. + Receiving objects: 100% (424/424), 262.14 MiB | 24.44 MiB/s, done. Resolving deltas: 100% (246/246), done. @@ -222,6 +222,10 @@ GroundingDINO imports .. parsed-literal:: + 2024-11-05 01:34:53.765709: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 01:34:53.988314: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2024-11-05 01:34:54.760718: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers UserWarning: Failed to load custom C++ ops. Running on CPU mode Only! @@ -362,6 +366,30 @@ Convert GroundingDINO to OpenVINO IR format TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + + +.. parsed-literal:: + + output layer_id 0 is nan + num_nan 230400, num_inf 0 + output layer_id 1 is nan + num_nan 230400, num_inf 0 + output layer_id 2 is nan + num_nan 230400, num_inf 0 + output layer_id 3 is nan + num_nan 230400, num_inf 0 + output layer_id 4 is nan + num_nan 230400, num_inf 0 + output layer_id 5 is nan + num_nan 230400, num_inf 0 + WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. + + +.. parsed-literal:: + + [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. Run OpenVINO optimized GroundingDINO @@ -504,15 +532,6 @@ class, but the inference will be done using OpenVINO optimized model. boxes_filt, pred_phrases, logits_filt = get_ov_grounding_output(ov_compiled_grounded_dino, pil_image, classes_prompt, BOX_THRESHOLD, TEXT_THRESHOLD) - -.. parsed-literal:: - - 2024-10-23 01:22:04.926963: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:22:04.966234: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:22:05.582957: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - - Convert predicted boxes to supervision box detections format .. code:: ipython3 @@ -573,11 +592,6 @@ segmentation. First of all let’s convert ``SAM`` model to OpenVINO IR. ov_efficient_sam = core.read_model(ov_efficient_sam_path) -.. parsed-literal:: - - WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. - - .. parsed-literal:: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! diff --git a/docs/notebooks/hello-segmentation-with-output.rst b/docs/notebooks/hello-segmentation-with-output.rst index e490cf48533277..6ddc0e3b0aa78b 100644 --- a/docs/notebooks/hello-segmentation-with-output.rst +++ b/docs/notebooks/hello-segmentation-with-output.rst @@ -188,7 +188,7 @@ is provided. .. parsed-literal:: - + @@ -215,7 +215,7 @@ Do Inference .. parsed-literal:: - + diff --git a/docs/notebooks/hugging-face-hub-with-output.rst b/docs/notebooks/hugging-face-hub-with-output.rst index e17d63a031b445..a92f8cd18fba31 100644 --- a/docs/notebooks/hugging-face-hub-with-output.rst +++ b/docs/notebooks/hugging-face-hub-with-output.rst @@ -132,6 +132,10 @@ tutorials `__. To disable this warning, you can either: - Avoid using `tokenizers` before the fork if possible - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) - 2024-10-23 01:23:29.312809: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:23:29.347879: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:23:29.943155: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Initialize and Convert the Model Automatically using OVModel class @@ -377,11 +384,6 @@ inference run. - This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model). -.. parsed-literal:: - - WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. - - Convert model using Optimum CLI interface ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -434,7 +436,7 @@ Full list of supported arguments available via ``--help`` .. parsed-literal:: - 2024-10-23 01:23:44.450300: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 01:37:12.161579: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT usage: optimum-cli export openvino [-h] -m MODEL [--task TASK] [--framework {pt,tf}] [--trust-remote-code] [--weight-format {fp32,fp16,int8,int4,mxfp4}] @@ -465,20 +467,20 @@ Full list of supported arguments available via ``--help`` --task TASK The task to export the model for. If not specified, the task will be auto-inferred based on the model. Available tasks depend on the model, but are among: - ['audio-xvector', 'text-to-audio', 'fill-mask', - 'feature-extraction', 'text-generation', 'zero-shot- - image-classification', 'text-to-image', 'text2text- - generation', 'zero-shot-object-detection', 'automatic- - speech-recognition', 'text-classification', 'semantic- - segmentation', 'masked-im', 'image-to-text', - 'sentence-similarity', 'object-detection', - 'inpainting', 'audio-frame-classification', 'image-to- - image', 'token-classification', 'question-answering', - 'audio-classification', 'image-segmentation', 'image- - classification', 'mask-generation', 'multiple-choice', - 'depth-estimation']. For decoder models, use `xxx- - with-past` to export the model using past key values - in the decoder. + ['mask-generation', 'image-classification', 'fill- + mask', 'audio-xvector', 'audio-frame-classification', + 'sentence-similarity', 'multiple-choice', 'automatic- + speech-recognition', 'text-to-image', 'token- + classification', 'image-to-text', 'image- + segmentation', 'question-answering', 'depth- + estimation', 'semantic-segmentation', 'feature- + extraction', 'text-generation', 'zero-shot-object- + detection', 'text-to-audio', 'zero-shot-image- + classification', 'object-detection', 'text2text- + generation', 'audio-classification', 'image-to-image', + 'masked-im', 'inpainting', 'text-classification']. For + decoder models, use `xxx-with-past` to export the + model using past key values in the decoder. --framework {pt,tf} The framework to use for the export. If not provided, will attempt to use the local checkpoint's original framework or what is available in the environment. @@ -583,10 +585,11 @@ compression: .. parsed-literal:: - 2024-10-23 01:23:49.987001: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 01:37:17.680673: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Some weights of the model checkpoint at cardiffnlp/twitter-roberta-base-sentiment-latest were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight'] - This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model). - This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model). + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. Tokenizer won't be converted. diff --git a/docs/notebooks/image-classification-quantization-with-output.rst b/docs/notebooks/image-classification-quantization-with-output.rst index b5b33fdfc852e7..7bf7172f720588 100644 --- a/docs/notebooks/image-classification-quantization-with-output.rst +++ b/docs/notebooks/image-classification-quantization-with-output.rst @@ -102,9 +102,9 @@ Model preparation stage has the following steps: Cloning into 'pytorch-cifar-models'... remote: Enumerating objects: 282, done. remote: Counting objects: 100% (281/281), done. - remote: Compressing objects: 100% (96/96), done. - remote: Total 282 (delta 135), reused 269 (delta 128), pack-reused 1 (from 1) - Receiving objects: 100% (282/282), 9.22 MiB | 24.58 MiB/s, done. + remote: Compressing objects: 100% (94/94), done. + remote: Total 282 (delta 135), reused 275 (delta 130), pack-reused 1 (from 1) + Receiving objects: 100% (282/282), 9.22 MiB | 9.58 MiB/s, done. Resolving deltas: 100% (135/135), done. @@ -176,7 +176,7 @@ Preprocessing for model obtained from training .. parsed-literal:: - 100%|██████████| 170498071/170498071 [00:07<00:00, 23490143.19it/s] + 100%|██████████| 170498071/170498071 [00:07<00:00, 22536051.32it/s] .. parsed-literal:: @@ -248,10 +248,10 @@ about supported parameters can be found on this .. parsed-literal:: - 2024-10-23 01:24:29.479834: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:24:29.511386: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 01:37:57.500572: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 01:37:57.532367: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:24:30.071901: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 01:37:58.074631: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -419,7 +419,7 @@ Tool `__ - `Interactive demo <#interactive-demo>`__ + Installation Instructions ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/notebooks/internvl2-with-output.rst b/docs/notebooks/internvl2-with-output.rst index 602bfe84590494..ed67209a0303eb 100644 --- a/docs/notebooks/internvl2-with-output.rst +++ b/docs/notebooks/internvl2-with-output.rst @@ -279,7 +279,7 @@ documentation self.max_seq_len_cached: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/qwen2/modeling_qwen2.py:324: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/qwen2/modeling_qwen2.py:324: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/qwen2/modeling_qwen2.py:339: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/qwen2/modeling_qwen2.py:339: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): @@ -531,7 +531,7 @@ Let’s check model capabilities in answering questions about image: .. parsed-literal:: - The image depicts a red panda, a large feline with a distinctive, reddish-brown coat and white face and chest. It is peeking over what appears to be a wooden platform or platform made for panda viewing in captivity. The background is filled with greenery, indicating that the photo was likely taken in a conservatory or wildlife park where penguins or seabirds are displayed. + The image shows a red panda lying on its side, partially wrapped in a wooden structure, possibly a container or log. The red panda appears to be looking at the camera with large, expressive eyes, displaying an endearing and lively appearance. The background consists of a portion of the red panda's habitat environment, which appears to be a tree and some greenery. Interactive demo ---------------- diff --git a/docs/notebooks/jina-clip-with-output.rst b/docs/notebooks/jina-clip-with-output.rst index 466e3be5d03fd1..1cdb2e1d286245 100644 --- a/docs/notebooks/jina-clip-with-output.rst +++ b/docs/notebooks/jina-clip-with-output.rst @@ -106,11 +106,11 @@ weights, using ``from_pretrained`` method. .. parsed-literal:: - 2024-10-23 01:28:42.177313: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:28:42.211816: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 01:41:58.578137: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 01:41:58.612620: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:28:42.875342: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + 2024-11-05 01:41:59.276782: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) @@ -301,7 +301,7 @@ loading on device using ``core.complie_model``. .. parsed-literal:: WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4713: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4713: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/jinaai/jina-bert-flash-implementation/b78d1595de294f13ffe7b19d6cd63892a6e4e7a4/mha.py:333: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! softmax_scale = self.softmax_scale or 1.0 / math.sqrt(q.shape[-1]) @@ -820,8 +820,8 @@ approximately estimate the speed up of the dynamic quantized models. .. parsed-literal:: - Performance speed up for text model: 1.560 - Performance speed up for vision model: 1.435 + Performance speed up for text model: 1.978 + Performance speed up for vision model: 1.428 Gradio demo diff --git a/docs/notebooks/knowledge-graphs-conve-with-output.rst b/docs/notebooks/knowledge-graphs-conve-with-output.rst index b9e4f56ebb05af..de9115fd9ab4a8 100644 --- a/docs/notebooks/knowledge-graphs-conve-with-output.rst +++ b/docs/notebooks/knowledge-graphs-conve-with-output.rst @@ -233,7 +233,7 @@ Download Model Checkpoint .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/knowledge-graphs-conve/models/conve.pt') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/knowledge-graphs-conve/models/conve.pt') @@ -395,7 +395,7 @@ typical to use metrics such as Mean Reciprocal Rank, Hits@10 etc. .. parsed-literal:: - Average time taken for inference: 0.6894171237945557 ms + Average time taken for inference: 0.7430613040924072 ms Mean accuracy of the model on the test dataset: 0.875 @@ -534,7 +534,7 @@ select device from dropdown list for running inference using OpenVINO .. parsed-literal:: - Average time taken for inference: 0.8764564990997314 ms + Average time taken for inference: 1.0752081871032715 ms Mean accuracy of the model on the test dataset: 0.10416666666666667 @@ -553,7 +553,7 @@ Determine the platform specific speedup obtained through OpenVINO graph optimiza .. parsed-literal:: - Speedup with OpenVINO optimizations: 0.79 X + Speedup with OpenVINO optimizations: 0.69 X Benchmark the converted OpenVINO model using benchmark app @@ -598,7 +598,7 @@ inference can also be obtained by looking at the benchmark app results. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 4.34 ms + [ INFO ] Read model took 4.36 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] e1 (node: e1) : i64 / [...] / [] @@ -614,7 +614,7 @@ inference can also be obtained by looking at the benchmark app results. [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: aten::softmax/Softmax) : f32 / [...] / [1,271] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 72.42 ms + [ INFO ] Compile model took 54.92 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -653,17 +653,17 @@ inference can also be obtained by looking at the benchmark app results. [ INFO ] Fill input 'rel' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 10000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 1.79 ms + [ INFO ] First inference took 1.66 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 94740 iterations - [ INFO ] Duration: 10002.15 ms + [ INFO ] Count: 94596 iterations + [ INFO ] Duration: 10001.69 ms [ INFO ] Latency: - [ INFO ] Median: 1.08 ms - [ INFO ] Average: 1.08 ms - [ INFO ] Min: 0.74 ms - [ INFO ] Max: 9.80 ms - [ INFO ] Throughput: 9471.96 FPS + [ INFO ] Median: 1.07 ms + [ INFO ] Average: 1.09 ms + [ INFO ] Min: 0.79 ms + [ INFO ] Max: 8.57 ms + [ INFO ] Throughput: 9458.00 FPS Conclusions diff --git a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output.rst b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output.rst index c18464f77585b2..b6a7a971fef3be 100644 --- a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output.rst +++ b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output.rst @@ -85,7 +85,7 @@ Install requirements .. parsed-literal:: - Requirement already satisfied: pip in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (24.2) + Requirement already satisfied: pip in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (24.3.1) Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -152,10 +152,10 @@ example `__ .. parsed-literal:: - 2024-10-23 01:31:37.373532: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:31:37.408388: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 01:44:54.753766: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 01:44:54.788691: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:31:37.931462: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 01:44:55.309895: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -374,11 +374,11 @@ Vision model accept ``pixel_values`` and returns ``image_embeds``. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4713: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4713: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:465: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:465: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:505: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:505: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): @@ -406,7 +406,7 @@ Convert Image To Text Projection model .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:165: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at aten/src/ATen/core/TensorBody.h:489.) + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:165: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at aten/src/ATen/core/TensorBody.h:489.) if a.grad is not None: @@ -541,13 +541,13 @@ generated text by ``AutoProcessor``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:804: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:804: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if max_pos > self.weights.size(0): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:1113: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:1113: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input_shape[-1] > 1: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:920: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:920: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attention_mask.size() != (batch_size, 1, seq_length, src_len): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:1206: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/kosmos2/modeling_kosmos2.py:1206: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: @@ -1389,9 +1389,9 @@ pipelines, we use mean inference time on 7 samples. .. parsed-literal:: - FP32 pipeline: 2.696 seconds - Optimized pipeline: 1.137 seconds - Performance speed-up: 2.372 + FP32 pipeline: 2.746 seconds + Optimized pipeline: 1.140 seconds + Performance speed-up: 2.409 Interactive inference diff --git a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.jpg b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.jpg index 9673054cc1b8e0..2310cb001b0c6b 100644 --- a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.jpg +++ b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c45f05bcfa118a3d7e9024cf83963a4bb5504cb97f669c0c8261c469bab6dc75 -size 121985 +oid sha256:9ca596f09c0f6c0dafa4aca0fbe7974941301cfcbc6bcb3a8c4255774c347d0b +size 123320 diff --git a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.png b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.png index a5be9e2155f6cc..91289c35d7c60c 100644 --- a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.png +++ b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_29_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2fc77a26e4c6d30ca9d21fe47ddc25624662d566f6c98ac1c160637fbcb2363b -size 1151151 +oid sha256:56d06f7d654939feda627f67196b813de9b38a718acba9f5daed59a43314829f +size 1150807 diff --git a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_48_1.png b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_48_1.png index 0da4a480d88cc3..d98f56141b1252 100644 --- a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_48_1.png +++ b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_48_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9cfd18faba3aef57349c70b157fabd4dc2827a8bc6074252e717a67a9f9d488a -size 1148752 +oid sha256:0d7f8506e5f1bd369debee273b45c601d05901af4937d8cc976f985cd4a81fed +size 1149292 diff --git a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.jpg b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.jpg index e992bdf66a9da7..b53344f52b7396 100644 --- a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.jpg +++ b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f70d76f01ee01d54713ed161de8d09713816b4f3fd7df99a5ff96c4c5e0b1bf0 -size 124539 +oid sha256:edd5a47baf47ae90532b47bc5ee05e8503b7d1deda59d956a354688ed949c8b5 +size 121605 diff --git a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.png b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.png index ec9e6a6277d4f0..2edc9a038ff8c3 100644 --- a/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.png +++ b/docs/notebooks/kosmos2-multimodal-large-language-model-with-output_files/kosmos2-multimodal-large-language-model-with-output_8_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f4dc2d892ef69df259eff4b46527d5e4382457b9b381a1e3b35b22d1d4312da4 -size 1150974 +oid sha256:aa184084b598dac717e99fe9677f1fe9dd4f6b85ec123c075d4109c75b134841 +size 1150675 diff --git a/docs/notebooks/language-quantize-bert-with-output.rst b/docs/notebooks/language-quantize-bert-with-output.rst index 64f49c2b314811..21ecfe511f1b76 100644 --- a/docs/notebooks/language-quantize-bert-with-output.rst +++ b/docs/notebooks/language-quantize-bert-with-output.rst @@ -101,10 +101,10 @@ Imports .. parsed-literal:: - 2024-10-23 01:38:12.900514: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:38:12.934654: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 01:51:49.197259: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 01:51:49.231710: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:38:13.484434: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 01:51:49.783615: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -211,7 +211,7 @@ PyTorch model formats are supported: .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4713: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4713: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( @@ -247,7 +247,7 @@ tokenizer from HuggingFace. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884 warnings.warn( @@ -505,9 +505,9 @@ Frames Per Second (FPS) for images. .. parsed-literal:: - PyTorch model on CPU: 0.068 seconds per sentence, SPS: 14.67 - IR FP32 model in OpenVINO Runtime/AUTO: 0.020 seconds per sentence, SPS: 49.46 - OpenVINO IR INT8 model in OpenVINO Runtime/AUTO: 0.009 seconds per sentence, SPS: 108.19 + PyTorch model on CPU: 0.068 seconds per sentence, SPS: 14.68 + IR FP32 model in OpenVINO Runtime/AUTO: 0.020 seconds per sentence, SPS: 49.24 + OpenVINO IR INT8 model in OpenVINO Runtime/AUTO: 0.009 seconds per sentence, SPS: 108.47 Finally, measure the inference performance of OpenVINO ``FP32`` and @@ -548,7 +548,7 @@ in OpenVINO. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 18.86 ms + [ INFO ] Read model took 19.11 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,?] @@ -559,7 +559,7 @@ in OpenVINO. [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [ INFO ] Reshaping model: 'input_ids': [1,128], '63': [1,128], 'token_type_ids': [1,128] - [ INFO ] Reshape model took 5.46 ms + [ INFO ] Reshape model took 5.55 ms [Step 6/11] Configuring input of the model [ INFO ] Model inputs: [ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,128] @@ -568,7 +568,7 @@ in OpenVINO. [ INFO ] Model outputs: [ INFO ] logits (node: __module.classifier/aten::linear/Add) : f32 / [...] / [1,2] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 373.39 ms + [ INFO ] Compile model took 344.20 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -609,17 +609,17 @@ in OpenVINO. [ INFO ] Fill input 'token_type_ids' with random values [Step 10/11] Measuring performance (Start inference synchronously, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 24.05 ms + [ INFO ] First inference took 22.90 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 5730 iterations - [ INFO ] Duration: 120000.60 ms + [ INFO ] Count: 6485 iterations + [ INFO ] Duration: 120011.48 ms [ INFO ] Latency: - [ INFO ] Median: 20.64 ms - [ INFO ] Average: 20.84 ms - [ INFO ] Min: 19.84 ms - [ INFO ] Max: 31.70 ms - [ INFO ] Throughput: 47.75 FPS + [ INFO ] Median: 18.09 ms + [ INFO ] Average: 18.41 ms + [ INFO ] Min: 17.32 ms + [ INFO ] Max: 26.49 ms + [ INFO ] Throughput: 54.04 FPS .. code:: ipython3 @@ -646,27 +646,27 @@ in OpenVINO. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 25.60 ms + [ INFO ] Read model took 24.93 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,?] - [ INFO ] 63 , attention_mask (node: attention_mask) : i64 / [...] / [1,?] + [ INFO ] attention_mask , 63 (node: attention_mask) : i64 / [...] / [1,?] [ INFO ] token_type_ids (node: token_type_ids) : i64 / [...] / [1,?] [ INFO ] Model outputs: [ INFO ] logits (node: __module.classifier/aten::linear/Add) : f32 / [...] / [1,2] [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [ INFO ] Reshaping model: 'input_ids': [1,128], '63': [1,128], 'token_type_ids': [1,128] - [ INFO ] Reshape model took 7.46 ms + [ INFO ] Reshape model took 7.14 ms [Step 6/11] Configuring input of the model [ INFO ] Model inputs: [ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,128] - [ INFO ] 63 , attention_mask (node: attention_mask) : i64 / [...] / [1,128] + [ INFO ] attention_mask , 63 (node: attention_mask) : i64 / [...] / [1,128] [ INFO ] token_type_ids (node: token_type_ids) : i64 / [...] / [1,128] [ INFO ] Model outputs: [ INFO ] logits (node: __module.classifier/aten::linear/Add) : f32 / [...] / [1,2] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 1067.96 ms + [ INFO ] Compile model took 1080.21 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -707,15 +707,15 @@ in OpenVINO. [ INFO ] Fill input 'token_type_ids' with random values [Step 10/11] Measuring performance (Start inference synchronously, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 17.45 ms + [ INFO ] First inference took 16.00 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 13316 iterations - [ INFO ] Duration: 120006.19 ms + [ INFO ] Count: 13181 iterations + [ INFO ] Duration: 120003.10 ms [ INFO ] Latency: - [ INFO ] Median: 8.92 ms - [ INFO ] Average: 8.92 ms - [ INFO ] Min: 7.66 ms - [ INFO ] Max: 14.16 ms - [ INFO ] Throughput: 110.96 FPS + [ INFO ] Median: 8.93 ms + [ INFO ] Average: 9.01 ms + [ INFO ] Min: 7.68 ms + [ INFO ] Max: 12.00 ms + [ INFO ] Throughput: 109.84 FPS diff --git a/docs/notebooks/latent-consistency-models-optimum-demo-with-output.rst b/docs/notebooks/latent-consistency-models-optimum-demo-with-output.rst index 3707d3f30d5f09..a0bce9d85c7196 100644 --- a/docs/notebooks/latent-consistency-models-optimum-demo-with-output.rst +++ b/docs/notebooks/latent-consistency-models-optimum-demo-with-output.rst @@ -9,7 +9,7 @@ and time required to generate an image for provided prompt. The notebook can be also used on other Intel hardware with minimal or no modifications. -|image0| +.. image:: https://github.com/openvinotoolkit/openvino_notebooks/assets/10940214/1858dae4-72fd-401e-b055-66d503d82446 Optimum Intel is an interface from Hugging Face between both diffusers and transformers libraries and various tools provided by Intel to @@ -48,8 +48,6 @@ need a Jupyter server to start. For details, please refer to `Installation Guide `__. -.. |image0| image:: https://github.com/openvinotoolkit/openvino_notebooks/assets/10940214/1858dae4-72fd-401e-b055-66d503d82446 - Prerequisites ~~~~~~~~~~~~~ @@ -67,9 +65,7 @@ Install required packages .. parsed-literal:: Note: you may need to restart the kernel to use updated packages. - Note: you may need to restart the kernel to use updated packages. - Note: you may need to restart the kernel to use updated packages. - + .. code:: ipython3 @@ -112,8 +108,12 @@ this .. parsed-literal:: - CPU: Intel(R) Core(TM) i9-10920X CPU @ 3.50GHz - + CPU: Intel(R) Core(TM) Ultra 7 155H + GNA.GNA_SW: GNA_SW + GNA.GNA_HW: GNA_HW + GPU: Intel(R) Arc(TM) Graphics (iGPU) + NPU: Intel(R) AI Boost + Using full precision model in CPU with ``LatentConsistencyModelPipeline`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -132,14 +132,6 @@ https://huggingface.co/docs/diffusers/en/api/pipelines/latent_consistency_models pipeline = LatentConsistencyModelPipeline.from_pretrained("SimianLuo/LCM_Dreamshaper_v7") -.. parsed-literal:: - - 2024-10-23 01:44:02.155955: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:44:02.191160: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:44:02.863862: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - - .. parsed-literal:: @@ -172,6 +164,15 @@ https://huggingface.co/docs/diffusers/en/api/pipelines/latent_consistency_models del pipeline gc.collect(); + + + +.. parsed-literal:: + + 345 + + + Select inference device for text-to-image generation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -190,15 +191,6 @@ Select inference device for text-to-image generation device - - - -.. parsed-literal:: - - Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') - - - Running inference using Optimum Intel ``OVLatentConsistencyModelPipeline`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -228,74 +220,6 @@ and there is no need to do it manually ov_pipeline.reshape(batch_size=1, height=512, width=512, num_images_per_prompt=1) - -.. parsed-literal:: - - Keyword arguments {'subfolder': '', 'trust_remote_code': False} are not expected by LatentConsistencyModelPipeline and will be ignored. - - - -.. parsed-literal:: - - Loading pipeline components...: 0%| | 0/7 [00:00 by passing `safety_checker=None`. Ensure that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered results in services or applications open to the public. Both the diffusers team and Hugging Face strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling it only for use-cases that involve analyzing network behavior or auditing its results. For more information, please have a look at https://github.com/huggingface/diffusers/pull/254 . - - - - -.. parsed-literal:: - - OVLatentConsistencyModelPipeline { - "_class_name": "OVLatentConsistencyModelPipeline", - "_diffusers_version": "0.31.0", - "_name_or_path": "/tmp/tmpz3tx8cvr", - "feature_extractor": [ - "transformers", - "CLIPImageProcessor" - ], - "image_encoder": [ - null, - null - ], - "requires_safety_checker": true, - "safety_checker": [ - null, - null - ], - "scheduler": [ - "diffusers", - "LCMScheduler" - ], - "text_encoder": [ - "optimum.intel.openvino.modeling_diffusion", - "OVModelTextEncoder" - ], - "tokenizer": [ - "transformers", - "CLIPTokenizer" - ], - "unet": [ - "optimum.intel.openvino.modeling_diffusion", - "OVModelUnet" - ], - "vae": [ - "optimum.intel.openvino.modeling_diffusion", - "OVModelVae" - ] - } - - - .. code:: ipython3 ov_pipeline.to(device.value) diff --git a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.jpg b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.jpg index a19eb369148e10..25ee04df1c536d 100644 --- a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.jpg +++ b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8ccbcf6c99b8d75b1c683375852111ce17983fbea50bd0b0bb294d06bbb317fb -size 38768 +oid sha256:7167bdd42f946c71dcff8f7c7f35e48df5434ed2c54b56505da9c61d25a0d740 +size 79839 diff --git a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.png b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.png index d27692124393df..d89126405ceff7 100644 --- a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.png +++ b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_15_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b936a832e23c30affb1988828f17a16e988b42413ff7dfe4381fb723e068ed97 -size 456597 +oid sha256:7f216ac91b5737006142f8426f0716c9ab8e3ce5ddea222987c69e8a4f11ed3e +size 1034680 diff --git a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.jpg b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.jpg index e320b91964889a..f0f1cd042ef9b5 100644 --- a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.jpg +++ b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0ab792ded8330c36072ec13fcfb0e0896ce3014413145167fd7a7c5b570919a4 -size 37604 +oid sha256:900339d0d0c9d4b8e25a5b72a159ca2c966f7e51dc112c30a7ac8636a01b1dd4 +size 81729 diff --git a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.png b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.png index e7a5d16aa88eb2..e083c8da654356 100644 --- a/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.png +++ b/docs/notebooks/latent-consistency-models-optimum-demo-with-output_files/latent-consistency-models-optimum-demo-with-output_8_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f440a9e7cb3d6a9f6609165516be7e5025e26752004ca56baf570f49840af03c -size 470802 +oid sha256:89e0bc6c3f710c16cb1f81011848f41ad350e43ef9dc77961353897aa1d5b6b6 +size 1042433 diff --git a/docs/notebooks/llava-multimodal-chatbot-genai-with-output.rst b/docs/notebooks/llava-multimodal-chatbot-genai-with-output.rst index 435e7eaf62c53b..d035645fb27291 100644 --- a/docs/notebooks/llava-multimodal-chatbot-genai-with-output.rst +++ b/docs/notebooks/llava-multimodal-chatbot-genai-with-output.rst @@ -117,10 +117,25 @@ Install required dependencies .. code:: ipython3 + from pathlib import Path + import requests + %pip install -q "torch>=2.1.0" "torchvision" "torchaudio" --index-url https://download.pytorch.org/whl/cpu %pip install -q "git+https://github.com/eaidova/optimum-intel.git@ea/minicpmv" %pip install -q "nncf>=2.13.0" "sentencepiece" "tokenizers>=0.12.1" "transformers>=4.45.0" "gradio>=4.36" - %pip install -q -U --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly openvino_tokenizers openvino openvino-genai + %pip install -q -U --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly openvino-tokenizers openvino openvino-genai + + + utility_files = ["notebook_utils.py", "cmd_helper.py"] + + for utility in utility_files: + local_path = Path(utility) + if not local_path.exists(): + r = requests.get( + url=f"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/{local_path.name}", + ) + with local_path.open("w") as f: + f.write(r.text) Convert and Optimize Model -------------------------- @@ -172,13 +187,13 @@ documentation =2.1.0" "torchvision" "torchaudio" --index-url https://download.pytorch.org/whl/cpu %pip install -q "git+https://github.com/eaidova/optimum-intel.git@ea/minicpmv" %pip install -q "nncf>=2.13.0" "sentencepiece" "tokenizers>=0.12.1" "transformers>=4.45.0" "gradio>=4.36" - %pip install -q -U --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly openvino_tokenizers openvino openvino-genai + %pip install -q -U --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly openvino-tokenizers openvino openvino-genai + + utility_files = ["notebook_utils.py", "cmd_helper.py"] + + for utility in utility_files: + local_path = Path(utility) + if not local_path.exists(): + r = requests.get( + url=f"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/{local_path.name}", + ) + with local_path.open("w") as f: + f.write(r.text) Convert and Optimize Model -------------------------- @@ -172,13 +186,13 @@ documentation `__, -`blog `__, -`GitHub `__, and -`Documentation `__. +``Qwen2.5`` as LLM in agent pipeline. + +* **qwen2.5-3b-instruct/qwen2.5-7b-instruct/qwen2.5-14b-instruct** - + Qwen2.5 is the latest series of Qwen large language models. Comparing + with Qwen2, Qwen2.5 series brings significant improvements in coding, + mathematics and general knowledge skills. Additionally, it brings + long-context and multiple languages support including Chinese, English, + French, Spanish, Portuguese, German, Italian, Russian, Japanese, Korean, + Vietnamese, Thai, Arabic, and more. For more details, please refer to + `model_card `__, + `blog `__, + `GitHub `__, and + `Documentation `__. .. code:: ipython3 import ipywidgets as widgets - + llm_model_ids = ["Qwen/Qwen2.5-3B-Instruct", "Qwen/Qwen2.5-7B-Instruct", "Qwen/qwen2.5-14b-instruct"] - + llm_model_id = widgets.Dropdown( options=llm_model_ids, value=llm_model_ids[0], description="Model:", disabled=False, ) - + llm_model_id @@ -146,9 +147,9 @@ Vietnamese, Thai, Arabic, and more. For more details, please refer to .. code:: ipython3 from pathlib import Path - + llm_model_path = llm_model_id.value.split("/")[-1] - + if not Path(llm_model_path).exists(): !optimum-cli export openvino --model {llm_model_id.value} --task text-generation-with-past --trust-remote-code --weight-format int4 --group-size 128 --ratio 1.0 --sym {llm_model_path} @@ -160,9 +161,9 @@ Select inference device for LLM .. code:: ipython3 from notebook_utils import device_widget - + llm_device = device_widget("CPU", exclude=["NPU"]) - + llm_device @@ -226,15 +227,15 @@ guide `__ import openvino.properties as props import openvino.properties.hint as hints import openvino.properties.streams as streams - + import json import json5 import torch - + tokenizer = AutoTokenizer.from_pretrained(llm_model_path, trust_remote_code=True) - + ov_config = {hints.performance_mode(): hints.PerformanceMode.LATENCY, streams.num(): "1", props.cache_dir(): ""} - + llm = OVModelForCausalLM.from_pretrained( llm_model_path, device=llm_device.value, @@ -242,7 +243,7 @@ guide `__ config=AutoConfig.from_pretrained(llm_model_path, trust_remote_code=True), trust_remote_code=True, ) - + llm.generation_config.top_k = 1 llm.generation_config.max_length = 2000 @@ -260,31 +261,31 @@ received from tool calling.. class StopSequenceCriteria(StoppingCriteria): """ This class can be used to stop generation whenever a sequence of tokens is encountered. - + Args: stop_sequences (`str` or `List[str]`): The sequence (or list of sequences) on which to stop execution. tokenizer: The tokenizer used to decode the model outputs. """ - + def __init__(self, stop_sequences, tokenizer): if isinstance(stop_sequences, str): stop_sequences = [stop_sequences] self.stop_sequences = stop_sequences self.tokenizer = tokenizer - + def __call__(self, input_ids, scores, **kwargs) -> bool: decoded_output = self.tokenizer.decode(input_ids.tolist()[0]) return any(decoded_output.endswith(stop_sequence) for stop_sequence in self.stop_sequences) - - + + def text_completion(prompt: str, stop_words) -> str: im_end = "<|im_end|>" if im_end not in stop_words: stop_words = stop_words + [im_end] streamer = TextStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True) - + stopping_criteria = StoppingCriteriaList([StopSequenceCriteria(stop_words, tokenizer)]) input_ids = torch.tensor([tokenizer.encode(prompt)]) generate_kwargs = dict( @@ -297,7 +298,7 @@ received from tool calling.. output = tokenizer.decode(output, errors="ignore") assert output.startswith(prompt) output = output[len(prompt) :].replace("<|endoftext|>", "").replace(im_end, "") - + for stop_str in stop_words: idx = output.find(stop_str) if idx != -1: @@ -339,13 +340,13 @@ parameter should be a sequence of messages that contains the .. code:: ipython3 TOOL_DESC = """{name_for_model}: Call this tool to interact with the {name_for_human} API. What is the {name_for_human} API useful for? {description_for_model} Parameters: {parameters}""" - + PROMPT_REACT = """Answer the following questions as best you can. You have access to the following APIs: - + {tools_text} - + Use the following format: - + Question: the input question you must answer Thought: you should always think about what to do Action: the action to take, should be one of [{tools_name_text}] @@ -354,9 +355,9 @@ parameter should be a sequence of messages that contains the ... (this Thought/Action/Action Input/Observation can be repeated zero or more times) Thought: I now know the final answer Final Answer: the final answer to the original input question - + Begin! - + Question: {query}""" Meanwhile we have to create function for consolidate the tools @@ -381,9 +382,9 @@ information and conversation history into the prompt template. raise NotImplementedError tools_text.append(tool) tools_text = "\n\n".join(tools_text) - + tools_name_text = ", ".join([tool_info["name_for_model"] for tool_info in list_of_tool_info]) - + messages = [{"role": "system", "content": "You are a helpful assistant."}] for i, (query, response) in enumerate(chat_history): if list_of_tool_info: @@ -397,9 +398,9 @@ information and conversation history into the prompt template. messages.append({"role": "user", "content": query}) if response: messages.append({"role": "assistant", "content": response}) - + prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False, return_tensors="pt") - + return prompt Create parser @@ -493,7 +494,7 @@ execute them according to the output of LLM. return str(ret) elif tool_name == "image_gen": import urllib.parse - + tool_args = tool_args.replace("(", "").replace(")", "") prompt = json5.loads(tool_args)["prompt"] prompt = urllib.parse.quote(prompt) @@ -503,11 +504,11 @@ execute them according to the output of LLM. ) else: raise NotImplementedError - - + + def llm_with_tool(prompt: str, history, list_of_tool_info=()): chat_history = [(x["user"], x["bot"]) for x in history] + [(prompt, "")] - + planning_prompt = build_input_text(chat_history, list_of_tool_info) text = "" while True: @@ -522,7 +523,7 @@ execute them according to the output of LLM. else: text += output break - + new_history = [] new_history.extend(history) new_history.append({"user": prompt, "bot": text}) @@ -537,7 +538,7 @@ Run agent history = [] query = "get the weather in London, and create a picture of Big Ben based on the weather information" - + response, history = llm_with_tool(prompt=query, history=history, list_of_tool_info=tools) diff --git a/docs/notebooks/meter-reader-with-output.rst b/docs/notebooks/meter-reader-with-output.rst index 7f539abf025ebb..c1317625880917 100644 --- a/docs/notebooks/meter-reader-with-output.rst +++ b/docs/notebooks/meter-reader-with-output.rst @@ -637,7 +637,7 @@ bounds of input batch size. .. parsed-literal:: - + diff --git a/docs/notebooks/minicpm-v-multimodal-chatbot-with-output.rst b/docs/notebooks/minicpm-v-multimodal-chatbot-with-output.rst index f6a22b6f160760..98f1217902a587 100644 --- a/docs/notebooks/minicpm-v-multimodal-chatbot-with-output.rst +++ b/docs/notebooks/minicpm-v-multimodal-chatbot-with-output.rst @@ -61,9 +61,15 @@ Prerequisites .. parsed-literal:: - Note: you may need to restart the kernel to use updated packages. - Note: you may need to restart the kernel to use updated packages. - + WARNING: Error parsing dependencies of torchsde: .* suffix can only be used with `==` or `!=` operators + numpy (>=1.19.*) ; python_version >= "3.7" + ~~~~~~~^ + Note: you may need to restart the kernel to use updated packages. + WARNING: Error parsing dependencies of torchsde: .* suffix can only be used with `==` or `!=` operators + numpy (>=1.19.*) ; python_version >= "3.7" + ~~~~~~~^ + Note: you may need to restart the kernel to use updated packages. + .. code:: ipython3 @@ -107,7 +113,16 @@ details.
+.. raw:: html + + + Click here for more detailed explanation of conversion steps + +.. raw:: html + + + MiniCPM-V2.6 is autoregressive transformer generative model, it means that each next model step depends from model output from previous step. The generation approach is based on the assumption that the probability @@ -205,11 +220,12 @@ Let’s convert each model part. .. parsed-literal:: - 2024-10-23 01:47:25.606377: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:47:25.640217: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-10-07 09:57:53.402018: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-10-07 09:57:53.403877: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used. + 2024-10-07 09:57:53.440490: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:47:26.161344: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - + 2024-10-07 09:57:54.270302: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + .. code:: ipython3 @@ -222,157 +238,7 @@ Let’s convert each model part. ⌛ openbmb/MiniCPM-V-2_6 conversion started. Be patient, it may takes some time. ⌛ Load Original model - - - -.. parsed-literal:: - - Fetching 24 files: 0%| | 0/24 [00:00 self.max_seq_len_cached: - + .. parsed-literal:: - ✅ Language model successfully converted - ⌛ Convert Image embedding model ✅ Image embedding model successfully converted - ⌛ Convert Resamler model - - -.. parsed-literal:: - - /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/ckpt/resampler.py:421: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - assert embed_dim == embed_dim_to_check, \ - /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/ckpt/resampler.py:428: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - assert head_dim * num_heads == embed_dim, f"embed_dim {embed_dim} not divisible by num_heads {num_heads}" - /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/ckpt/resampler.py:434: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - assert key.shape == value.shape, f"key shape {key.shape} does not match value shape {value.shape}" - /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/ckpt/resampler.py:520: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - assert key_padding_mask.shape == (bsz, src_len), \ - /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/ckpt/resampler.py:539: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - q_scaled = q / math.sqrt(E) - - -.. parsed-literal:: - - ✅ Resampler model successfully converted ✅ openbmb/MiniCPM-V-2_6 model sucessfully converted. You can find results in MiniCPM-V-2_6 - + Compress Language Model Weights to 4 bits ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -459,12 +288,21 @@ applied using `NNCF `__.
-Click here for more details about weight compression Weight compression -aims to reduce the memory footprint of a model. It can also lead to -significant performance improvement for large memory-bound models, such -as Large Language Models (LLMs). LLMs and other models, which require -extensive memory to store the weights during inference, can benefit from -weight compression in the following ways: +.. raw:: html + + + +Click here for more details about weight compression + +.. raw:: html + + + +Weight compression aims to reduce the memory footprint of a model. It +can also lead to significant performance improvement for large +memory-bound models, such as Large Language Models (LLMs). LLMs and +other models, which require extensive memory to store the weights during +inference, can benefit from weight compression in the following ways: - enabling the inference of exceptionally large models that cannot be accommodated in the memory of the device; @@ -547,33 +385,14 @@ documentation `__ - `Select model <#select-model>`__ @@ -62,9 +63,9 @@ Prerequisites .. code:: ipython3 from pathlib import Path - + repo_dir = Path("./ml-mobileclip") - + if not repo_dir.exists(): !git clone https://github.com/apple/ml-mobileclip.git @@ -76,15 +77,15 @@ Prerequisites remote: Counting objects: 100% (95/95), done. remote: Compressing objects: 100% (66/66), done. remote: Total 95 (delta 38), reused 85 (delta 28), pack-reused 0 (from 0) - Unpacking objects: 100% (95/95), 469.11 KiB | 3.91 MiB/s, done. + Unpacking objects: 100% (95/95), 469.11 KiB | 3.13 MiB/s, done. .. code:: ipython3 %pip install -q "./ml-mobileclip" --no-deps - + %pip install -q "clip-benchmark>=1.4.0" "datasets>=2.8.0" "open-clip-torch>=2.20.0" "timm>=0.9.5" "torch>=1.13.1" "torchvision>=0.14.1" --extra-index-url https://download.pytorch.org/whl/cpu - + %pip install -q "openvino>=2024.0.0" "gradio>=4.19" "matplotlib" "Pillow" "altair" "pandas" "opencv-python" "tqdm" "matplotlib>=3.4" @@ -141,9 +142,9 @@ comparison purposes, you can select different models among: .. code:: ipython3 import ipywidgets as widgets - + model_dir = Path("checkpoints") - + supported_models = { "MobileCLIP": { "mobileclip_s0": { @@ -184,17 +185,17 @@ comparison purposes, you can select different models among: "image_size": 224, }, "clip-vit-b-16": { - "image_name": "ViT-B-16", + "model_name": "ViT-B-16", "pretrained": "openai", "image_size": 224, }, "clip-vit-l-14": { - "image_name": "ViT-L-14", + "model_name": "ViT-L-14", "pretrained": "datacomp_xl_s13b_b90k", "image_size": 224, }, "clip-vit-h-14": { - "image_name": "ViT-H-14", + "model_name": "ViT-H-14", "pretrained": "laion2b_s32b_b79k", "image_size": 224, }, @@ -212,8 +213,8 @@ comparison purposes, you can select different models among: }, }, } - - + + model_type = widgets.Dropdown(options=supported_models.keys(), default="MobileCLIP", description="Model type:") model_type @@ -229,13 +230,13 @@ comparison purposes, you can select different models among: .. code:: ipython3 available_models = supported_models[model_type.value] - + model_checkpoint = widgets.Dropdown( options=available_models.keys(), default=list(available_models), description="Model:", ) - + model_checkpoint @@ -250,15 +251,15 @@ comparison purposes, you can select different models among: .. code:: ipython3 import requests - + r = requests.get( url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", ) - + open("notebook_utils.py", "w").write(r.text) - + from notebook_utils import download_file, device_widget - + model_config = available_models[model_checkpoint.value] Run model inference @@ -293,8 +294,8 @@ Prepare image gallery import matplotlib.pyplot as plt import numpy as np from PIL import Image - - + + def visualize_result(images: List, query: str = "", selected: List[int] = None): """ Utility function for visualization classification results @@ -322,8 +323,8 @@ Prepare image gallery mask = np.ones_like(np.array(images[idx])) a.imshow(mask, "jet", interpolation="none", alpha=0.75) return fig - - + + images_urls = [ "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/282ce53e-912d-41aa-ab48-2a001c022d74", "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/9bb40168-82b5-4b11-ada6-d8df104c736c", @@ -333,17 +334,17 @@ Prepare image gallery image_names = ["red_panda.png", "cat.png", "raccoon.png", "dog.png"] sample_path = Path("data") sample_path.mkdir(parents=True, exist_ok=True) - + images = [] for image_name, image_url in zip(image_names, images_urls): image_path = sample_path / image_name if not image_path.exists(): download_file(image_url, filename=image_name, directory=sample_path) images.append(Image.open(image_path).convert("RGB").resize((640, 420))) - + input_labels = ["cat"] text_descriptions = [f"This is a photo of a {label}" for label in input_labels] - + visualize_result(images, "image gallery"); @@ -390,7 +391,7 @@ preprocessing utilities from PIL import Image import mobileclip import open_clip - + # instantiate model model_name = model_config["model_name"] pretrained = model_config["pretrained"] @@ -407,7 +408,7 @@ preprocessing utilities .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning) @@ -426,8 +427,8 @@ Perform search image_tensor = torch.stack([preprocess(image) for image in images]) text = tokenizer(text_descriptions) - - + + with torch.no_grad(): # calculate image embeddings image_encoding_start = time.perf_counter() @@ -439,22 +440,22 @@ Perform search text_features = model.encode_text(text) text_encoding_end = time.perf_counter() print(f"Text encoding took {text_encoding_end - text_encoding_start:.3} ms") - + # normalize embeddings image_features /= image_features.norm(dim=-1, keepdim=True) text_features /= text_features.norm(dim=-1, keepdim=True) - + # calcualte similarity score image_probs = (100.0 * text_features @ image_features.T).softmax(dim=-1) selected_image = [torch.argmax(image_probs).item()] - + visualize_result(images, input_labels[0], selected_image); .. parsed-literal:: - Image encoding took 0.136 ms - Text encoding took 0.0123 ms + Image encoding took 0.114 ms + Text encoding took 0.0113 ms @@ -481,8 +482,8 @@ be used separately. Let’s convert each part to OpenVINO. import types import torch.nn.functional as F - - + + def se_block_forward(self, inputs): """Apply forward pass.""" b, c, h, w = inputs.size() @@ -498,12 +499,12 @@ be used separately. Let’s convert each part to OpenVINO. import openvino as ov import gc - + ov_models_dir = Path("ov_models") ov_models_dir.mkdir(exist_ok=True) - + image_encoder_path = ov_models_dir / f"{model_checkpoint.value}_im_encoder.xml" - + if not image_encoder_path.exists(): if "mobileclip_s" in model_name: model.image_encoder.model.conv_exp.se.forward = types.MethodType(se_block_forward, model.image_encoder.model.conv_exp.se) @@ -516,23 +517,23 @@ be used separately. Let’s convert each part to OpenVINO. ov.save_model(ov_image_encoder, image_encoder_path) del ov_image_encoder gc.collect() - + text_encoder_path = ov_models_dir / f"{model_checkpoint.value}_text_encoder.xml" - + if not text_encoder_path.exists(): model.forward = model.encode_text ov_text_encoder = ov.convert_model(model, example_input=text, input=[-1, text.shape[1]]) ov.save_model(ov_text_encoder, text_encoder_path) del ov_text_encoder gc.collect() - + del model gc.collect(); .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/mobileclip/modules/common/transformer.py:125: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/mobileclip/modules/common/transformer.py:125: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if seq_len != self.num_embeddings: @@ -549,9 +550,9 @@ Select device for image encoder .. code:: ipython3 core = ov.Core() - + device = device_widget() - + device @@ -608,17 +609,17 @@ Perform search print(f"Text encoding took {text_encoding_end - text_encoding_start:.3} ms") image_features /= image_features.norm(dim=-1, keepdim=True) text_features /= text_features.norm(dim=-1, keepdim=True) - + image_probs = (100.0 * text_features @ image_features.T).softmax(dim=-1) selected_image = [torch.argmax(image_probs).item()] - + visualize_result(images, input_labels[0], selected_image); .. parsed-literal:: - Image encoding took 0.0297 ms - Text encoding took 0.00513 ms + Image encoding took 0.0294 ms + Text encoding took 0.00498 ms @@ -654,14 +655,14 @@ models can require different optimal threshold for search. ) from open_clip.transform import image_transform from typing import Optional - - + + current_device = device.value current_model = image_encoder_path.name.split("_im_encoder")[0] - + available_converted_models = [model_file.name.split("_im_encoder")[0] for model_file in ov_models_dir.glob("*_im_encoder.xml")] available_devices = list(core.available_devices) + ["AUTO"] - + download_file( "https://storage.openvinotoolkit.org/data/test_data/videos/car-detection.mp4", directory=sample_path, @@ -671,8 +672,8 @@ models can require different optimal threshold for search. directory=sample_path, filename="coco.mp4", ) - - + + def get_preprocess_and_tokenizer(model_name): if "mobileclip" in model_name: resolution = supported_models["MobileCLIP"][model_name]["image_size"] @@ -693,10 +694,10 @@ models can require different optimal threshold for search. resize_size = model_configs[model_name]["image_size"] preprocess = image_transform((resize_size, resize_size), is_train=False, resize_mode="longest") tokenizer = open_clip.get_tokenizer(model_configs[model_name]["model_name"]) - + return preprocess, tokenizer - - + + def run( path: str, text_search: str, @@ -715,7 +716,7 @@ models can require different optimal threshold for search. global tokenizer global ov_compiled_image_encoder global ov_compiled_text_encoder - + if current_model != model_name or device != current_device: ov_compiled_image_encoder = core.compile_model(ov_models_dir / f"{model_name}_im_encoder.xml", device) ov_compiled_text_encoder = core.compile_model(ov_models_dir / f"{model_name}_text_encoder.xml", device) @@ -725,7 +726,7 @@ models can require different optimal threshold for search. # Load video dataset = LoadVideo(path, transforms=preprocess, vid_stride=stride) dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=0) - + # Get image query features if image_search: image = preprocess(image_search).unsqueeze(0) @@ -745,11 +746,11 @@ models can require different optimal threshold for search. for image, orig, frame, timestamp in dataloader: with torch.no_grad(): image_features = torch.from_numpy(ov_compiled_image_encoder(image)[0]) - + image_features /= image_features.norm(dim=-1, keepdim=True) probs = query_features.cpu().numpy() @ image_features.cpu().numpy().T probs = probs[0] - + # Save frame similarity values df = pd.DataFrame( { @@ -759,15 +760,15 @@ models can require different optimal threshold for search. } ) res = pd.concat([res, df]) - + # Check if frame is over threshold for i, p in enumerate(probs): if p > thresh: matches.append(to_pil_image(orig[i])) matches_probs.append(p) - + print(f"Frames: {frame.tolist()} - Probs: {probs}") - + # Create plot of similarity values lines = ( alt.Chart(res) @@ -778,16 +779,16 @@ models can require different optimal threshold for search. ) ).properties(width=600) rule = alt.Chart().mark_rule(strokeDash=[6, 3], size=2).encode(y=alt.datum(thresh)) - + selected_frames = np.argsort(-1 * np.array(matches_probs))[:20] matched_sorted_frames = [matches[idx] for idx in selected_frames] - + return ( lines + rule, matched_sorted_frames, ) # Only return up to 20 images to not crash the UI - - + + class LoadVideo(Dataset): def __init__(self, path, transforms, vid_stride=1): self.transforms = transforms @@ -795,27 +796,27 @@ models can require different optimal threshold for search. self.cur_frame = 0 self.cap = cv2.VideoCapture(path) self.total_frames = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT) / self.vid_stride) - + def __getitem__(self, _): # Read video # Skip over frames for _ in range(self.vid_stride): self.cap.grab() self.cur_frame += 1 - + # Read frame _, img = self.cap.retrieve() timestamp = self.cap.get(cv2.CAP_PROP_POS_MSEC) - + # Convert to PIL img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) img = Image.fromarray(np.uint8(img)) - + # Apply transforms img_t = self.transforms(img) - + return img_t, to_tensor(img), self.cur_frame, timestamp - + def __len__(self): return self.total_frames @@ -837,15 +838,15 @@ models can require different optimal threshold for search. if not Path("gradio_helper.py").exists(): r = requests.get(url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/notebooks/mobileclip-video-search/gradio_helper.py") open("gradio_helper.py", "w").write(r.text) - + from gradio_helper import make_demo, Option - + demo = make_demo( run=run, model_option=Option(choices=available_converted_models, value=model_checkpoint.value), device_option=Option(choices=available_devices, value=device.value), ) - + try: demo.launch(debug=False) except Exception: @@ -858,7 +859,7 @@ models can require different optimal threshold for search. .. parsed-literal:: Running on local URL: http://127.0.0.1:7860 - + To create a public link, set `share=True` in `launch()`. diff --git a/docs/notebooks/mobilevlm-language-assistant-with-output.rst b/docs/notebooks/mobilevlm-language-assistant-with-output.rst index 02efe16d9c0f4a..5902537e3026a5 100644 --- a/docs/notebooks/mobilevlm-language-assistant-with-output.rst +++ b/docs/notebooks/mobilevlm-language-assistant-with-output.rst @@ -119,13 +119,13 @@ Import required packages .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( - 2024-10-23 01:57:03.532418: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:57:03.567584: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 02:02:06.143728: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 02:02:06.177889: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:57:04.078609: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + 2024-11-05 02:02:06.679118: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( @@ -342,15 +342,15 @@ compression instead of INT8 weight compression. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:595: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:595: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input_shape[-1] > 1: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:119: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:119: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if seq_len > self.max_seq_len_cached: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:348: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:348: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:355: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:355: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:365: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py:365: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): @@ -416,7 +416,7 @@ compression instead of INT8 weight compression. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:165: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at aten/src/ATen/core/TensorBody.h:489.) + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:165: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at aten/src/ATen/core/TensorBody.h:489.) if a.grad is not None: diff --git a/docs/notebooks/music-generation-with-output.rst b/docs/notebooks/music-generation-with-output.rst index 566aa1c87a941c..4adc89b9ff79e7 100644 --- a/docs/notebooks/music-generation-with-output.rst +++ b/docs/notebooks/music-generation-with-output.rst @@ -124,13 +124,13 @@ Imports .. parsed-literal:: - 2024-10-23 01:59:20.725760: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 01:59:20.759494: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 02:04:23.419260: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 02:04:23.453089: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 01:59:21.367845: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + 2024-11-05 02:04:24.059462: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( @@ -170,11 +170,11 @@ generate a text-conditioned music sample. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/huggingface_hub/file_download.py:797: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/huggingface_hub/file_download.py:797: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`. warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/utils/generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm. warnings.warn("torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.") @@ -229,7 +229,7 @@ vocabulary. It helps the model understand the context of a sentence. @@ -655,7 +655,7 @@ We can now infer the pipeline backed by OpenVINO models. diff --git a/docs/notebooks/nano-llava-multimodal-chatbot-with-output.rst b/docs/notebooks/nano-llava-multimodal-chatbot-with-output.rst index 082b6613456e28..337458e35bbf0c 100644 --- a/docs/notebooks/nano-llava-multimodal-chatbot-with-output.rst +++ b/docs/notebooks/nano-llava-multimodal-chatbot-with-output.rst @@ -53,7 +53,7 @@ Prerequisites %pip install -q "torch>=2.1" "transformers>=4.40" "accelerate" "pillow" "gradio>=4.26" "tqdm" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "nncf>=2.13" - %pip install -q -U --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly openvino_tokenizers[transformers] "openvino>=2024.4.0" + %pip install -q -U --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly "openvino-tokenizers[transformers]" "openvino>=2024.4.0" %pip install -q "git+https://github.com/eaidova/optimum-intel.git@ea/minicpmv" @@ -64,9 +64,10 @@ Prerequisites Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - openvino-dev 2024.4.0 requires openvino==2024.4.0, but you have openvino 2024.5.0.dev20241014 which is incompatible. openvino-genai 2024.4.0.0 requires openvino_tokenizers~=2024.4.0.0.dev, but you have openvino-tokenizers 2024.5.0.0.dev20241022 which is incompatible. Note: you may need to restart the kernel to use updated packages. + ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. + mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.17.2+cpu which is incompatible. Note: you may need to restart the kernel to use updated packages. @@ -145,13 +146,13 @@ Download PyTorch model .. parsed-literal:: - example_1.png: 0%| | 0.00/200k [00:00 1 or self.sliding_window is not None) and self.is_causal: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/exporters/onnx/model_patcher.py:307: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/exporters/onnx/model_patcher.py:306: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: - /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/qnguyen3/nanoLLaVA/1ea99cffcf50a27c5f06fe5d22a07046aba0bffe/modeling_llava_qwen2.py:939: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/qnguyen3/nanoLLaVA/13d60cec183a86755afed64da495fcc2c382ea80/modeling_llava_qwen2.py:939: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if seq_len > self.max_seq_len_cached: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/cache_utils.py:432: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/cache_utils.py:432: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results. elif len(self.key_cache[layer_idx]) == 0: # fills previously skipped layers; checking for tensor causes errors - /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/qnguyen3/nanoLLaVA/1ea99cffcf50a27c5f06fe5d22a07046aba0bffe/modeling_llava_qwen2.py:1499: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/qnguyen3/nanoLLaVA/13d60cec183a86755afed64da495fcc2c382ea80/modeling_llava_qwen2.py:1499: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): - Unexpectedly found already patched module while applying ModuleExtension during PyTorch model conversion. Result of the conversion maybe broken. Depending on the exact issue it may lead to broken original model. OpenVINO and OpenVINO Tokenizers versions are not binary compatible. OpenVINO version: 2024.5.0-16993 OpenVINO Tokenizers version: 2024.5.0.0 @@ -579,18 +250,18 @@ documentation + File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/bin/optimum-cli", line 10, in sys.exit(main()) - File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/commands/optimum_cli.py", line 208, in main + File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/commands/optimum_cli.py", line 208, in main service.run() - File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/commands/export/openvino.py", line 349, in run + File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/commands/export/openvino.py", line 349, in run main_export( - File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/exporters/openvino/__main__.py", line 416, in main_export + File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/optimum/exporters/openvino/__main__.py", line 416, in main_export core = Core() - File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino_tokenizers/__init__.py", line 53, in new_core_init + File "/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino_tokenizers/__init__.py", line 53, in new_core_init self.add_extension(str(_ext_path)) # Core.add_extension doesn't support Path object RuntimeError: Exception from src/inference/src/cpp/core.cpp:158: - Cannot add extension. Cannot find entry point to the extension library. This error happened: Cannot load library '/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino_tokenizers/lib/libopenvino_tokenizers.so': /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino_tokenizers/lib/libopenvino_tokenizers.so: undefined symbol: _ZNK2ov4Node17can_constant_foldERKSt6vectorINS_6OutputIS0_EESaIS3_EE + Cannot add extension. Cannot find entry point to the extension library. This error happened: Cannot load library '/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino_tokenizers/lib/libopenvino_tokenizers.so': /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino_tokenizers/lib/libopenvino_tokenizers.so: undefined symbol: _ZNK2ov4Node17can_constant_foldERKSt6vectorINS_6OutputIS0_EESaIS3_EE @@ -709,12 +380,12 @@ image encoder model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/quantization/quantize_model.py:432: FutureWarning: `CompressWeightsMode.INT8` is deprecated. Please, use `CompressWeightsMode.INT8_ASYM` as value instead. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/quantization/quantize_model.py:432: FutureWarning: `CompressWeightsMode.INT8` is deprecated. Please, use `CompressWeightsMode.INT8_ASYM` as value instead. warning_deprecated( - 2024-10-23 02:04:33.280788: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 02:04:33.314985: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 02:09:38.791476: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 02:09:38.825207: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 02:04:33.946816: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 02:09:39.427301: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -915,8 +586,8 @@ can use the same tokenizer and image processor that provided with model. Question: Describe this image in detail Answer: - The image portrays a charming and playful scene featuring a white lama. This adorable creature has a playful expression, with its eyes sparkling with joy and its nose in a playful smile. It is adorned with cute, tiny eyes that add a playful touch to its face. The lama's ears are also quite noticeable, with one of them sporting a tiny pink button. The lama's body is covered in fluffy, white fur, and it has its hind legs visible, adding a sense of movement to the image. - The lama is surrounded by a vivid display of fire. The flames are bright and lively, with some areas appearing more intense + The image features a small, adorable white lamb standing amidst a fire. The lamb's fur is fluffy and white, and it is adorned with tiny black eyes that are bright and lively. The lamb's face is cute, with a small black nose and a small mouth. It seems like the lamb is looking straight at the camera, making it appear even more adorable. + The lamb's right ear is visible, and it is white and pink. The lamb's right eye is also black and pink. The lamb's face is quite detailed, with the nose and mouth visible. There are also details like the lamb's right foot, which is white Interactive demo diff --git a/docs/notebooks/object-detection-with-output.rst b/docs/notebooks/object-detection-with-output.rst index b1db0093158982..a34f72f5d8ff1e 100644 --- a/docs/notebooks/object-detection-with-output.rst +++ b/docs/notebooks/object-detection-with-output.rst @@ -2,12 +2,10 @@ Live Object Detection with OpenVINO™ ==================================== This notebook demonstrates live object detection with OpenVINO, using -the `SSDLite -MobileNetV2 `__ -from `Open Model -Zoo `__. Final part -of this notebook shows live inference results from a webcam. -Additionally, you can also upload a video file. +the `Ultralytics +YOLOv8 `__. Final part of +this notebook shows live inference results from a webcam. Additionally, +you can also upload a video file. **NOTE**: To use this notebook with a webcam, you need to run the notebook on a computer with a webcam. If you run the notebook on a @@ -25,13 +23,12 @@ Additionally, you can also upload a video file. - `The Model <#the-model>`__ - - `Download the Model <#download-the-model>`__ - - `Convert the Model <#convert-the-model>`__ + - `Download and convert the + Model <#download-and-convert-the-model>`__ - `Load the Model <#load-the-model>`__ - `Processing <#processing>`__ - - `Process Results <#process-results>`__ - `Main Processing Function <#main-processing-function>`__ - `Run <#run>`__ @@ -62,8 +59,8 @@ Install requirements .. code:: ipython3 - %pip install -q "openvino-dev>=2024.0.0" - %pip install -q tensorflow + %pip install -q "openvino>=2024.4.0" + %pip install -q "ultralytics==8.3.0" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q opencv-python requests tqdm # Fetch `notebook_utils` module @@ -79,10 +76,6 @@ Install requirements .. parsed-literal:: Note: you may need to restart the kernel to use updated packages. - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - magika 0.5.1 requires numpy<2.0,>=1.24; python_version >= "3.8" and python_version < "3.9", but you have numpy 1.23.5 which is incompatible. - mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.17.2+cpu which is incompatible. - supervision 0.24.0 requires numpy<1.23.3,>=1.21.2; python_full_version <= "3.10.0", but you have numpy 1.23.5 which is incompatible. Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -102,17 +95,10 @@ Imports .. code:: ipython3 - import collections - import tarfile - import time from pathlib import Path - - import cv2 - import numpy as np - from IPython import display + import gc import openvino as ov - from openvino.tools.mo.front import tf as ov_tf_front - from openvino.tools import mo + from ultralytics import YOLO import notebook_utils as utils @@ -121,92 +107,53 @@ The Model -Download the Model -~~~~~~~~~~~~~~~~~~ - +Download and convert the Model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Use the ``download_file``, a function from the ``notebook_utils`` file. -It automatically creates a directory structure and downloads the -selected model. This step is skipped if the package is already -downloaded and unpacked. The chosen model comes from the public -directory, which means it must be converted into OpenVINO Intermediate -Representation (OpenVINO IR). - - **NOTE**: Using a model other than ``ssdlite_mobilenet_v2`` may - require different conversion parameters as well as pre- and - post-processing. .. code:: ipython3 # A directory where the model will be downloaded. - base_model_dir = Path("model") - - # The name of the model from Open Model Zoo - model_name = "ssdlite_mobilenet_v2" - archive_name = Path(f"{model_name}_coco_2018_05_09.tar.gz") - model_url = f"https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2022.1/{model_name}/{archive_name}" + # The name of the model + model_name = "yolov8n" - # Download the archive - downloaded_model_path = base_model_dir / archive_name - if not downloaded_model_path.exists(): - utils.download_file(model_url, downloaded_model_path.name, downloaded_model_path.parent) + det_model_path = Path(f"{model_name}_openvino_model/{model_name}.xml") - # Unpack the model - tf_model_path = base_model_dir / archive_name.with_suffix("").stem / "frozen_inference_graph.pb" - if not tf_model_path.exists(): - with tarfile.open(downloaded_model_path) as file: - file.extractall(base_model_dir) - + # export model to OpenVINO format using Ultralytics API + if not det_model_path.exists(): + pt_model = YOLO(f"{model_name}.pt") + pt_model.export(format="openvino", dynamic=True, half=True) + del pt_model + gc.collect() .. parsed-literal:: - model/ssdlite_mobilenet_v2_coco_2018_05_09.tar.gz: 0%| | 0.00/48.7M [00:00`__ -(``mo.convert_model`` function). If the model has been already -converted, this step is skipped. - -.. code:: ipython3 - - precision = "FP16" - # The output path for the conversion. - converted_model_path = Path("model") / f"{model_name}_{precision.lower()}.xml" - - # Convert it to IR if not previously converted - trans_config_path = Path(ov_tf_front.__file__).parent / "ssd_v2_support.json" - if not converted_model_path.exists(): - ov_model = mo.convert_model( - tf_model_path, - compress_to_fp16=(precision == "FP16"), - transformations_config=trans_config_path, - tensorflow_object_detection_api_pipeline_config=tf_model_path.parent / "pipeline.config", - reverse_input_channels=True, - ) - ov.save_model(ov_model, converted_model_path) - del ov_model + Downloading https://github.com/ultralytics/assets/releases/download/v8.3.0/yolov8n.pt to 'yolov8n.pt'... .. parsed-literal:: - [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. - In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. - Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html + 100%|██████████| 6.25M/6.25M [00:00<00:00, 25.9MB/s] .. parsed-literal:: - [ WARNING ] The Preprocessor block has been removed. Only nodes performing mean value subtraction and scaling (if applicable) are kept. + Ultralytics 8.3.0 🚀 Python-3.8.10 torch-2.2.2+cpu CPU (Intel Core(TM) i9-10920X 3.50GHz) + YOLOv8n summary (fused): 168 layers, 3,151,904 parameters, 0 gradients, 8.7 GFLOPs + + PyTorch: starting from 'yolov8n.pt' with input shape (1, 3, 640, 640) BCHW and output shape(s) (1, 84, 8400) (6.2 MB) + + OpenVINO: starting export with openvino 2024.4.0-16579-c3152d32c9c-releases/2024/4... + OpenVINO: export success ✅ 1.3s, saved as 'yolov8n_openvino_model/' (6.4 MB) + + Export complete (1.5s) + Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/object-detection-webcam + Predict: yolo predict task=detect model=yolov8n_openvino_model imgsz=640 half + Validate: yolo val task=detect model=yolov8n_openvino_model imgsz=640 data=coco.yaml half + Visualize: https://netron.app Load the Model @@ -241,208 +188,44 @@ best performance. For that purpose, just use ``AUTO``. .. code:: ipython3 - # Read the network and corresponding weights from a file. - model = core.read_model(model=converted_model_path) - # Compile the model for CPU (you can choose manually CPU, GPU etc.) - # or let the engine choose the best available device (AUTO). - compiled_model = core.compile_model(model=model, device_name=device.value) - - # Get the input and output nodes. - input_layer = compiled_model.input(0) - output_layer = compiled_model.output(0) - - # Get the input size. - height, width = list(input_layer.shape)[1:3] - -Input and output layers have the names of the input node and output node -respectively. In the case of SSDLite MobileNetV2, there is 1 input and 1 -output. - -.. code:: ipython3 - - input_layer.any_name, output_layer.any_name - - - - -.. parsed-literal:: - - ('image_tensor:0', 'detection_boxes:0') - - - -Processing ----------- - - - -Process Results -~~~~~~~~~~~~~~~ - - - -First, list all available classes and create colors for them. Then, in -the post-process stage, transform boxes with normalized coordinates -``[0, 1]`` into boxes with pixel coordinates ``[0, image_size_in_px]``. -Afterward, use `non-maximum -suppression `__ -to reject overlapping detections and those below the probability -threshold (0.5). Finally, draw boxes and labels inside them. - -.. code:: ipython3 - - # https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/ - classes = [ - "background", - "person", - "bicycle", - "car", - "motorcycle", - "airplane", - "bus", - "train", - "truck", - "boat", - "traffic light", - "fire hydrant", - "street sign", - "stop sign", - "parking meter", - "bench", - "bird", - "cat", - "dog", - "horse", - "sheep", - "cow", - "elephant", - "bear", - "zebra", - "giraffe", - "hat", - "backpack", - "umbrella", - "shoe", - "eye glasses", - "handbag", - "tie", - "suitcase", - "frisbee", - "skis", - "snowboard", - "sports ball", - "kite", - "baseball bat", - "baseball glove", - "skateboard", - "surfboard", - "tennis racket", - "bottle", - "plate", - "wine glass", - "cup", - "fork", - "knife", - "spoon", - "bowl", - "banana", - "apple", - "sandwich", - "orange", - "broccoli", - "carrot", - "hot dog", - "pizza", - "donut", - "cake", - "chair", - "couch", - "potted plant", - "bed", - "mirror", - "dining table", - "window", - "desk", - "toilet", - "door", - "tv", - "laptop", - "mouse", - "remote", - "keyboard", - "cell phone", - "microwave", - "oven", - "toaster", - "sink", - "refrigerator", - "blender", - "book", - "clock", - "vase", - "scissors", - "teddy bear", - "hair drier", - "toothbrush", - "hair brush", - ] - - # Colors for the classes above (Rainbow Color Map). - colors = cv2.applyColorMap( - src=np.arange(0, 255, 255 / len(classes), dtype=np.float32).astype(np.uint8), - colormap=cv2.COLORMAP_RAINBOW, - ).squeeze() + core = ov.Core() - def process_results(frame, results, thresh=0.6): - # The size of the original frame. - h, w = frame.shape[:2] - # The 'results' variable is a [1, 1, 100, 7] tensor. - results = results.squeeze() - boxes = [] - labels = [] - scores = [] - for _, label, score, xmin, ymin, xmax, ymax in results: - # Create a box with pixels coordinates from the box with normalized coordinates [0,1]. - boxes.append(tuple(map(int, (xmin * w, ymin * h, (xmax - xmin) * w, (ymax - ymin) * h)))) - labels.append(int(label)) - scores.append(float(score)) + def load_model(det_model_path, device): + compiled_model = compile_model(det_model_path, device) + det_model = YOLO(det_model_path.parent, task="detect") - # Apply non-maximum suppression to get rid of many overlapping entities. - # See https://paperswithcode.com/method/non-maximum-suppression - # This algorithm returns indices of objects to keep. - indices = cv2.dnn.NMSBoxes(bboxes=boxes, scores=scores, score_threshold=thresh, nms_threshold=0.6) + if det_model.predictor is None: + custom = {"conf": 0.25, "batch": 1, "save": False, "mode": "predict"} # method defaults + args = {**det_model.overrides, **custom} + det_model.predictor = det_model._smart_load("predictor")(overrides=args, _callbacks=det_model.callbacks) + det_model.predictor.setup_model(model=det_model.model) - # If there are no boxes. - if len(indices) == 0: - return [] + det_model.predictor.model.ov_compiled_model = compiled_model + return det_model - # Filter detected objects. - return [(labels[idx], scores[idx], boxes[idx]) for idx in indices.flatten()] + def compile_model(det_model_path, device): + det_ov_model = core.read_model(det_model_path) - def draw_boxes(frame, boxes): - for label, score, box in boxes: - # Choose color for the label. - color = tuple(map(int, colors[label])) - # Draw a box. - x2 = box[0] + box[2] - y2 = box[1] + box[3] - cv2.rectangle(img=frame, pt1=box[:2], pt2=(x2, y2), color=color, thickness=3) + ov_config = {} + if device != "CPU": + det_ov_model.reshape({0: [1, 3, 640, 640]}) + if "GPU" in device or ("AUTO" in device and "GPU" in core.available_devices): + ov_config = {"GPU_DISABLE_WINOGRAD_CONVOLUTION": "YES"} + det_compiled_model = core.compile_model(det_ov_model, device, ov_config) + return det_compiled_model - # Draw a label name inside the box. - cv2.putText( - img=frame, - text=f"{classes[label]} {score:.2f}", - org=(box[0] + 10, box[1] + 30), - fontFace=cv2.FONT_HERSHEY_COMPLEX, - fontScale=frame.shape[1] / 1000, - color=color, - thickness=1, - lineType=cv2.LINE_AA, - ) - return frame + det_model = load_model(det_model_path, device.value) + + +.. parsed-literal:: + + Ultralytics 8.3.0 🚀 Python-3.8.10 torch-2.2.2+cpu CPU (Intel Core(TM) i9-10920X 3.50GHz) + Loading yolov8n_openvino_model for OpenVINO inference... + Using OpenVINO LATENCY mode for batch=1 inference... + Main Processing Function ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -454,8 +237,18 @@ file. .. code:: ipython3 + from IPython import display + import cv2 + import numpy as np + + # Main processing function to run object detection. - def run_object_detection(source=0, flip=False, use_popup=False, skip_first_frames=0): + def run_object_detection( + source=0, + flip=False, + use_popup=False, + skip_first_frames=0, + ): player = None try: # Create a video player to play with target fps. @@ -466,7 +259,6 @@ file. title = "Press ESC to Exit" cv2.namedWindow(winname=title, flags=cv2.WINDOW_GUI_NORMAL | cv2.WINDOW_AUTOSIZE) - processing_times = collections.deque() while True: # Grab the frame. frame = player.next() @@ -483,43 +275,10 @@ file. fy=scale, interpolation=cv2.INTER_AREA, ) - - # Resize the image and change dims to fit neural network input. - input_img = cv2.resize(src=frame, dsize=(width, height), interpolation=cv2.INTER_AREA) - # Create a batch of images (size = 1). - input_img = input_img[np.newaxis, ...] - - # Measure processing time. - - start_time = time.time() # Get the results. - results = compiled_model([input_img])[output_layer] - stop_time = time.time() - # Get poses from network results. - boxes = process_results(frame=frame, results=results) - - # Draw boxes on a frame. - frame = draw_boxes(frame=frame, boxes=boxes) - - processing_times.append(stop_time - start_time) - # Use processing times from last 200 frames. - if len(processing_times) > 200: - processing_times.popleft() - - _, f_width = frame.shape[:2] - # Mean processing time [ms]. - processing_time = np.mean(processing_times) * 1000 - fps = 1000 / processing_time - cv2.putText( - img=frame, - text=f"Inference time: {processing_time:.1f}ms ({fps:.1f} FPS)", - org=(20, 40), - fontFace=cv2.FONT_HERSHEY_COMPLEX, - fontScale=f_width / 1000, - color=(0, 0, 255), - thickness=1, - lineType=cv2.LINE_AA, - ) + input_image = np.array(frame) + detections = det_model(input_image, verbose=False) + frame = detections[0].plot() # Use this workaround if there is flickering. if use_popup: @@ -592,22 +351,10 @@ Run the object detection: -.. image:: object-detection-with-output_files/object-detection-with-output_19_0.png +.. image:: object-detection-with-output_files/object-detection-with-output_13_0.png .. parsed-literal:: Source ended - -References ----------- - - - -1. `SSDLite - MobileNetV2 `__ -2. `Open Model - Zoo `__ -3. `Non-Maximum - Suppression `__ diff --git a/docs/notebooks/object-detection-with-output_files/object-detection-with-output_13_0.png b/docs/notebooks/object-detection-with-output_files/object-detection-with-output_13_0.png new file mode 100644 index 00000000000000..1c548a0445636e --- /dev/null +++ b/docs/notebooks/object-detection-with-output_files/object-detection-with-output_13_0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3fca049d156843f0976e9386ece07d1846e92efcacc5c559de140e7733b67b3 +size 171752 diff --git a/docs/notebooks/object-detection-with-output_files/object-detection-with-output_19_0.png b/docs/notebooks/object-detection-with-output_files/object-detection-with-output_19_0.png deleted file mode 100644 index fb2b3b2abdf69f..00000000000000 --- a/docs/notebooks/object-detection-with-output_files/object-detection-with-output_19_0.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1d300cd81ce6f44a5ee2ded6f14b739b0a8ecd8d5d2e487743cd085b92b3533b -size 175076 diff --git a/docs/notebooks/openvino-api-with-output.rst b/docs/notebooks/openvino-api-with-output.rst index a837b88ac6976b..b2b4c8c0f04fdd 100644 --- a/docs/notebooks/openvino-api-with-output.rst +++ b/docs/notebooks/openvino-api-with-output.rst @@ -201,7 +201,7 @@ notebooks. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') @@ -250,7 +250,7 @@ points to the filename of an ONNX model. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/segmentation.onnx') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/segmentation.onnx') @@ -310,7 +310,7 @@ without any conversion step. Pass the filename with extension to .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/inference.pdiparams') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/inference.pdiparams') @@ -354,7 +354,7 @@ TensorFlow models saved in frozen graph format can also be passed to .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.pb') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.pb') @@ -497,7 +497,7 @@ Information about the inputs and outputs of the model are in .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') @@ -703,7 +703,7 @@ produced data as values. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') @@ -892,7 +892,7 @@ input shape. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/segmentation.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/segmentation.bin') @@ -1044,7 +1044,7 @@ the cache. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvino-api/model/classification.bin') diff --git a/docs/notebooks/openvoice-with-output.rst b/docs/notebooks/openvoice-with-output.rst index b73dd8059aa65b..2ee11fcded84dc 100644 --- a/docs/notebooks/openvoice-with-output.rst +++ b/docs/notebooks/openvoice-with-output.rst @@ -100,10 +100,11 @@ Clone repository and install requirements Cloning into 'OpenVoice'... remote: Enumerating objects: 438, done. remote: Total 438 (delta 0), reused 0 (delta 0), pack-reused 438 (from 1) - Receiving objects: 100% (438/438), 3.84 MiB | 19.30 MiB/s, done. + Receiving objects: 100% (438/438), 3.84 MiB | 21.51 MiB/s, done. Resolving deltas: 100% (207/207), done. ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.17.2+cpu which is incompatible. + tensorflow 2.12.0 requires numpy<1.24,>=1.22, but you have numpy 1.24.4 which is incompatible. torchvision 0.17.2+cpu requires torch==2.2.2, but you have torch 2.4.1 which is incompatible. Note: you may need to restart the kernel to use updated packages. @@ -242,9 +243,9 @@ True .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. WeightNorm.apply(module, name, dim) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/api.py:36: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/api.py:36: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. checkpoint_dict = torch.load(ckpt_path, map_location=torch.device(self.device)) @@ -258,9 +259,9 @@ True .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/wavmark/__init__.py:16: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/wavmark/__init__.py:16: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. checkpoint = torch.load(resume_path, map_location=torch.device('cpu')) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/api.py:36: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/api.py:36: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. checkpoint_dict = torch.load(ckpt_path, map_location=torch.device(self.device)) @@ -410,40 +411,40 @@ documentation 0 No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:283: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:283: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert ( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:346: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:346: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! pad_length = max(length - (self.window_size + 1), 0) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:347: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:347: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! slice_start_position = max((self.window_size + 1) - length, 0) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:349: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/attentions.py:349: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if pad_length > 0: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:114: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:114: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if torch.min(inputs) < left or torch.max(inputs) > right: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:119: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:119: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if min_bin_width * num_bins > 1.0: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:121: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:121: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if min_bin_height * num_bins > 1.0: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:171: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/openvoice/OpenVoice/openvoice/transforms.py:171: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert (discriminant >= 0).all() - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes: - %3293 : Float(1, 2, 43, strides=[86, 43, 1], requires_grad=0, device=cpu) = aten::randn(%3288, %3289, %3290, %3291, %3292) # /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:86:0 - %5559 : Float(1, 192, 152, strides=[29184, 1, 192], requires_grad=0, device=cpu) = aten::randn_like(%m_p, %5554, %5555, %5556, %5557, %5558) # /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:86:0 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes: + %3293 : Float(1, 2, 43, strides=[86, 43, 1], requires_grad=0, device=cpu) = aten::randn(%3288, %3289, %3290, %3291, %3292) # /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:86:0 + %5559 : Float(1, 192, 153, strides=[29376, 1, 192], requires_grad=0, device=cpu) = aten::randn_like(%m_p, %5554, %5555, %5556, %5557, %5558) # /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:86:0 This may cause errors in trace checking. To disable trace checking, pass check_trace=False to torch.jit.trace() _check_trace( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: - The values for attribute 'shape' do not match: torch.Size([1, 1, 38656]) != torch.Size([1, 1, 39680]). + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: + The values for attribute 'shape' do not match: torch.Size([1, 1, 39424]) != torch.Size([1, 1, 38656]). _check_trace( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 2. of the traced function does not match the corresponding output of the Python function. Detailed error: - The values for attribute 'shape' do not match: torch.Size([1, 1, 151, 43]) != torch.Size([1, 1, 155, 43]). + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 2. of the traced function does not match the corresponding output of the Python function. Detailed error: + The values for attribute 'shape' do not match: torch.Size([1, 1, 154, 43]) != torch.Size([1, 1, 151, 43]). _check_trace( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 3. of the traced function does not match the corresponding output of the Python function. Detailed error: - The values for attribute 'shape' do not match: torch.Size([1, 1, 151]) != torch.Size([1, 1, 155]). + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 3. of the traced function does not match the corresponding output of the Python function. Detailed error: + The values for attribute 'shape' do not match: torch.Size([1, 1, 154]) != torch.Size([1, 1, 151]). _check_trace( - 2024-10-23 02:08:53.401718: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 02:13:33.268258: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -476,16 +477,16 @@ documentation )`. (Triggered internally at ../aten/src/ATen/native/SpectralOps.cpp:836.) + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/modules/module.py:1562: UserWarning: A window was not provided. A rectangular window will be applied,which is known to cause spectral leakage. Other windows such as torch.hann_window or torch.hamming_window can are recommended to reduce spectral leakage.To suppress this warning and use a rectangular window, explicitly set `window=torch.ones(n_fft, device=)`. (Triggered internally at ../aten/src/ATen/native/SpectralOps.cpp:836.) return forward_call(\*args, \*\*kwargs) @@ -713,7 +714,7 @@ Load speaker embeddings .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/functional.py:666: UserWarning: stft with return_complex=False is deprecated. In a future pytorch release, stft will return complex tensors for all inputs, and return_complex=False will raise an error. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/functional.py:666: UserWarning: stft with return_complex=False is deprecated. In a future pytorch release, stft will return complex tensors for all inputs, and return_complex=False will raise an error. Note: you can still call torch.view_as_real on the complex output to recover the old return format. (Triggered internally at ../aten/src/ATen/native/SpectralOps.cpp:873.) return _VF.stft(input, n_fft, hop_length, win_length, window, # type: ignore[attr-defined] @@ -868,7 +869,7 @@ And finally, run voice tone conversion with OpenVINO optimized model @@ -886,7 +887,7 @@ And finally, run voice tone conversion with OpenVINO optimized model @@ -1075,7 +1076,7 @@ voice tone conversion online. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/gradio/components/dropdown.py:100: UserWarning: The `max_choices` parameter is ignored when `multiselect` is False. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/gradio/components/dropdown.py:100: UserWarning: The `max_choices` parameter is ignored when `multiselect` is False. warnings.warn( diff --git a/docs/notebooks/optical-character-recognition-with-output.rst b/docs/notebooks/optical-character-recognition-with-output.rst index 9efa8e8e7b87f2..7dae2290312e68 100644 --- a/docs/notebooks/optical-character-recognition-with-output.rst +++ b/docs/notebooks/optical-character-recognition-with-output.rst @@ -9,16 +9,9 @@ which shows only text detection. The `horizontal-text-detection-0001 `__ and -`text-recognition-resnet `__ +`text-recognition-0014 `__ models are used together for text detection and then text recognition. -In this tutorial, Open Model Zoo tools including Model Downloader, Model -Converter and Info Dumper are used to download and convert the models -from `Open Model -Zoo `__. For more -information, refer to the -`model-tools `__ tutorial. - **Table of contents:** @@ -26,7 +19,6 @@ information, refer to the - `Imports <#imports>`__ - `Settings <#settings>`__ - `Download Models <#download-models>`__ -- `Convert Models <#convert-models>`__ - `Select inference device <#select-inference-device>`__ - `Object Detection <#object-detection>`__ @@ -61,15 +53,12 @@ Guide =2024.0.0" "onnx<1.16.2" torch torchvision pillow opencv-python "matplotlib>=3.4" --extra-index-url https://download.pytorch.org/whl/cpu + # Install openvino package + %pip install -q "openvino>=2024.4.0" pillow opencv-python "matplotlib>=3.4" .. parsed-literal:: - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.17.2+cpu which is incompatible. - torchaudio 2.4.1+cpu requires torch==2.4.1, but you have torch 2.2.2+cpu which is incompatible. Note: you may need to restart the kernel to use updated packages. @@ -111,7 +100,7 @@ Settings model_dir = Path("model") precision = "FP16" detection_model = "horizontal-text-detection-0001" - recognition_model = "text-recognition-resnet-fc" + recognition_model = "text-recognition-0014" model_dir.mkdir(exist_ok=True) @@ -120,274 +109,35 @@ Download Models -The next cells will run Model Downloader to download the detection and -recognition models. If the models have been downloaded before, they will -not be downloaded again. +The next cells will download the detection and recognition models. If +the models have been downloaded before, they will not be downloaded +again. .. code:: ipython3 - download_command = ( - f"omz_downloader --name {detection_model},{recognition_model} --output_dir {model_dir} --cache_dir {model_dir} --precision {precision} --num_attempts 5" - ) - display(Markdown(f"Download command: `{download_command}`")) - display(Markdown(f"Downloading {detection_model}, {recognition_model}...")) - !$download_command - display(Markdown(f"Finished downloading {detection_model}, {recognition_model}.")) - - detection_model_path = (model_dir / "intel/horizontal-text-detection-0001" / precision / detection_model).with_suffix(".xml") - recognition_model_path = (model_dir / "public/text-recognition-resnet-fc" / precision / recognition_model).with_suffix(".xml") - - - -Download command: -``omz_downloader --name horizontal-text-detection-0001,text-recognition-resnet-fc --output_dir model --cache_dir model --precision FP16 --num_attempts 5`` - - - -Downloading horizontal-text-detection-0001, text-recognition-resnet-fc… - - -.. parsed-literal:: - - ################|| Downloading horizontal-text-detection-0001 ||################ - - ========== Downloading model/intel/horizontal-text-detection-0001/FP16/horizontal-text-detection-0001.xml - - - ========== Downloading model/intel/horizontal-text-detection-0001/FP16/horizontal-text-detection-0001.bin - - - ################|| Downloading text-recognition-resnet-fc ||################ - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/model.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/weight_init.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/heads/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/heads/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/heads/fc_head.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/heads/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/body.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/component.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/sequences/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/sequences/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/sequences/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/bricks/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/bricks/bricks.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/bricks/builder.py - + from notebook_utils import download_ir_model - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/bricks/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/backbones/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/backbones/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/backbones/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/backbones/resnet.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/enhance_modules/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/enhance_modules/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/enhance_modules/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/utils/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/utils/builder.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/utils/conv_module.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/utils/fc_module.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/utils/norm.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/models/utils/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/utils/__init__.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/utils/common.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/utils/registry.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/utils/config.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/configs/resnet_fc.py - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/ckpt/resnet_fc.pth - - - ========== Downloading model/public/text-recognition-resnet-fc/vedastr/addict-2.4.0-py3-none-any.whl + detection_model_url = f"https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/{detection_model}/{precision}/{detection_model}.xml" + recognition_model_url = ( + f"https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/{recognition_model}/{precision}/{recognition_model}.xml" + ) - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/heads/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/sequences/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/component.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/decoders/bricks/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/backbones/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/enhance_modules/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/utils/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/utils/__init__.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/utils/config.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/utils/config.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/utils/config.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/utils/config.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/utils/config.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/backbones/resnet.py - ========== Replacing text in model/public/text-recognition-resnet-fc/vedastr/models/bodies/feature_extractors/encoders/backbones/resnet.py - ========== Unpacking model/public/text-recognition-resnet-fc/vedastr/addict-2.4.0-py3-none-any.whl + detection_model_path = download_ir_model(detection_model_url, model_dir / detection_model / precision) + recognition_model_path = download_ir_model(recognition_model_url, model_dir / recognition_model / precision) -Finished downloading horizontal-text-detection-0001, -text-recognition-resnet-fc. - - -.. code:: ipython3 - - ### The text-recognition-resnet-fc model consists of many files. All filenames are printed in - ### the output of Model Downloader. Uncomment the next two lines to show this output. - - # for line in download_result: - # print(line) - -Convert Models --------------- - - - -The downloaded detection model is an Intel model, which is already in -OpenVINO Intermediate Representation (OpenVINO IR) format. The text -recognition model is a public model which needs to be converted to -OpenVINO IR. Since this model was downloaded from Open Model Zoo, use -Model Converter to convert the model to OpenVINO IR format. - -The output of Model Converter will be displayed. When the conversion is -successful, the last lines of output will include -``[ SUCCESS ] Generated IR version 11 model.`` - -.. code:: ipython3 - - convert_command = f"omz_converter --name {recognition_model} --precisions {precision} --download_dir {model_dir} --output_dir {model_dir}" - display(Markdown(f"Convert command: `{convert_command}`")) - display(Markdown(f"Converting {recognition_model}...")) - ! $convert_command - - - -Convert command: -``omz_converter --name text-recognition-resnet-fc --precisions FP16 --download_dir model --output_dir model`` - +.. parsed-literal:: + model/horizontal-text-detection-0001/FP16/horizontal-text-detection-0001.bin: 0%| | 0.00/3.70M [00:… -Converting text-recognition-resnet-fc… .. parsed-literal:: - ========== Converting text-recognition-resnet-fc to ONNX - Conversion to ONNX command: /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/omz_tools/internal_scripts/pytorch_to_onnx.py --model-path=/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/omz_tools/models/public/text-recognition-resnet-fc --model-path=model/public/text-recognition-resnet-fc --model-name=get_model --import-module=model '--model-param=file_config=r"model/public/text-recognition-resnet-fc/vedastr/configs/resnet_fc.py"' '--model-param=weights=r"model/public/text-recognition-resnet-fc/vedastr/ckpt/resnet_fc.pth"' --input-shape=1,1,32,100 --input-names=input --output-names=output --output-file=model/public/text-recognition-resnet-fc/resnet_fc.onnx - - ONNX check passed successfully. - - ========== Converting text-recognition-resnet-fc to IR (FP16) - Conversion command: /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --output_dir=model/public/text-recognition-resnet-fc/FP16 --model_name=text-recognition-resnet-fc --input=input '--mean_values=input[127.5]' '--scale_values=input[127.5]' --output=output --input_model=model/public/text-recognition-resnet-fc/resnet_fc.onnx '--layout=input(NCHW)' '--input_shape=[1, 1, 32, 100]' --compress_to_fp16=True - - [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. - In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. - Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html - [ INFO ] Generated IR will be compressed to FP16. If you get lower accuracy, please consider disabling compression explicitly by adding argument --compress_to_fp16=False. - Find more information about compression to FP16 at https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_FP16_Compression.html - [ SUCCESS ] Generated IR version 11 model. - [ SUCCESS ] XML file: /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/optical-character-recognition/model/public/text-recognition-resnet-fc/FP16/text-recognition-resnet-fc.xml - [ SUCCESS ] BIN file: /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/optical-character-recognition/model/public/text-recognition-resnet-fc/FP16/text-recognition-resnet-fc.bin - + model/text-recognition-0014/FP16/text-recognition-0014.bin: 0%| | 0.00/17.4M [00:00 + @@ -384,7 +384,7 @@ may be specified is input data .. parsed-literal:: - + @@ -422,7 +422,7 @@ then such conversion will be added explicitly. .. parsed-literal:: - + @@ -636,6 +636,6 @@ Compare performance .. parsed-literal:: - IR model in OpenVINO Runtime/CPU with manual image preprocessing: 0.0154 seconds per image, FPS: 65.10 - IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0142 seconds per image, FPS: 70.53 + IR model in OpenVINO Runtime/CPU with manual image preprocessing: 0.0153 seconds per image, FPS: 65.39 + IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0166 seconds per image, FPS: 60.23 diff --git a/docs/notebooks/paddle-ocr-webcam-with-output.rst b/docs/notebooks/paddle-ocr-webcam-with-output.rst index e5c426d1920f33..9f7510cd5efe96 100644 --- a/docs/notebooks/paddle-ocr-webcam-with-output.rst +++ b/docs/notebooks/paddle-ocr-webcam-with-output.rst @@ -75,6 +75,8 @@ Guide =1.22, but you have numpy 1.24.4 which is incompatible. Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -207,7 +209,7 @@ Download the Model for Text **Detection** .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-no… + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-no… .. parsed-literal:: @@ -253,7 +255,7 @@ Download the Model for Text **Recognition** .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-no… + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-no… .. parsed-literal:: diff --git a/docs/notebooks/paddle-ocr-webcam-with-output_files/paddle-ocr-webcam-with-output_30_0.png b/docs/notebooks/paddle-ocr-webcam-with-output_files/paddle-ocr-webcam-with-output_30_0.png index 43d5bfb39ed4e8..38a0d5d593351b 100644 --- a/docs/notebooks/paddle-ocr-webcam-with-output_files/paddle-ocr-webcam-with-output_30_0.png +++ b/docs/notebooks/paddle-ocr-webcam-with-output_files/paddle-ocr-webcam-with-output_30_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e5f67b60f77e15fb0b7908ada4ff64ab506accc23f579c81193175e5044f8f3b -size 593679 +oid sha256:ac7efd85f2c50b0a189dbf00c0cd2252f362e6469cd014d8f255c53172152c3d +size 591373 diff --git a/docs/notebooks/paddle-to-openvino-classification-with-output.rst b/docs/notebooks/paddle-to-openvino-classification-with-output.rst index 576878a578a323..25feb9293ee93a 100644 --- a/docs/notebooks/paddle-to-openvino-classification-with-output.rst +++ b/docs/notebooks/paddle-to-openvino-classification-with-output.rst @@ -89,11 +89,11 @@ Imports .. parsed-literal:: - --2024-10-23 02:12:15-- http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb + --2024-11-05 02:15:59-- http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb Resolving proxy-dmz.intel.com (proxy-dmz.intel.com)... 10.241.208.166 Connecting to proxy-dmz.intel.com (proxy-dmz.intel.com)|10.241.208.166|:911... connected. Proxy request sent, awaiting response... 404 Not Found - 2024-10-23 02:12:15 ERROR 404: Not Found. + 2024-11-05 02:16:00 ERROR 404: Not Found. dpkg: error: cannot access archive 'libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb': No such file or directory @@ -124,8 +124,8 @@ Imports .. parsed-literal:: - 2024-10-23 02:12:17 INFO: Loading faiss with AVX512 support. - 2024-10-23 02:12:17 INFO: Successfully loaded faiss with AVX512 support. + 2024-11-05 02:16:02 INFO: Loading faiss with AVX512 support. + 2024-11-05 02:16:02 INFO: Successfully loaded faiss with AVX512 support. Settings @@ -209,7 +209,7 @@ inference on that image, and then show the top three prediction results. .. parsed-literal:: - [2024/10/23 02:12:43] ppcls WARNING: The current running environment does not support the use of GPU. CPU has been used instead. + [2024/11/05 02:16:41] ppcls WARNING: The current running environment does not support the use of GPU. CPU has been used instead. Labrador retriever, 0.75138 German short-haired pointer, 0.02373 Great Dane, 0.01848 @@ -275,7 +275,7 @@ clipping values. .. parsed-literal:: - 2024-10-23 02:12:44 WARNING: Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). + 2024-11-05 02:16:42 WARNING: Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). .. parsed-literal:: @@ -287,7 +287,7 @@ clipping values. .. parsed-literal:: - + @@ -462,7 +462,7 @@ Note that many optimizations are possible to improve the performance. .. parsed-literal:: - PaddlePaddle model on CPU: 0.0073 seconds per image, FPS: 137.19 + PaddlePaddle model on CPU: 0.0074 seconds per image, FPS: 134.37 PaddlePaddle result: Labrador retriever, 0.75138 @@ -523,7 +523,7 @@ select device from dropdown list for running inference using OpenVINO .. parsed-literal:: - OpenVINO IR model in OpenVINO Runtime (AUTO): 0.0026 seconds per image, FPS: 382.97 + OpenVINO IR model in OpenVINO Runtime (AUTO): 0.0027 seconds per image, FPS: 373.31 OpenVINO result: Labrador retriever, 0.74909 diff --git a/docs/notebooks/parler-tts-text-to-speech-with-output.rst b/docs/notebooks/parler-tts-text-to-speech-with-output.rst index 3bbb67d2232c99..323959aa17e8ef 100644 --- a/docs/notebooks/parler-tts-text-to-speech-with-output.rst +++ b/docs/notebooks/parler-tts-text-to-speech-with-output.rst @@ -9,7 +9,7 @@ with synthetic annotations `__ by Dan Lyth and Simon King, from Stability AI and Edinburgh University respectively. -|image0| +.. image:: https://images.squarespace-cdn.com/content/v1/657816dfbefe0533e8a69d9a/30c96e25-acc5-4019-acdd-648da6142c4c/architecture_v3.png?format=2500w Text-to-speech models trained on large-scale datasets have demonstrated impressive in-context learning capabilities and naturalness. However, @@ -53,8 +53,6 @@ need a Jupyter server to start. For details, please refer to `Installation Guide `__. -.. |image0| image:: https://images.squarespace-cdn.com/content/v1/657816dfbefe0533e8a69d9a/30c96e25-acc5-4019-acdd-648da6142c4c/architecture_v3.png?format=2500w - Prerequisites ------------- @@ -69,23 +67,6 @@ Prerequisites %pip install -q "openvino>=2024.2.0" %pip install -q git+https://github.com/huggingface/parler-tts.git "gradio>=4.19" transformers "torch>=2.2" --extra-index-url https://download.pytorch.org/whl/cpu - -.. parsed-literal:: - - Note: you may need to restart the kernel to use updated packages. - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - googleapis-common-protos 1.65.0 requires protobuf!=3.20.0,!=3.20.1,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0.dev0,>=3.20.2, but you have protobuf 3.19.6 which is incompatible. - mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.17.2+cpu which is incompatible. - onnx 1.16.1 requires protobuf>=3.20.2, but you have protobuf 3.19.6 which is incompatible. - paddlepaddle 2.6.2 requires protobuf>=3.20.2; platform_system != "Windows", but you have protobuf 3.19.6 which is incompatible. - tensorflow 2.12.0 requires protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3, but you have protobuf 3.19.6 which is incompatible. - tensorflow-datasets 4.9.2 requires protobuf>=3.20, but you have protobuf 3.19.6 which is incompatible. - tensorflow-metadata 1.14.0 requires protobuf<4.21,>=3.20.3, but you have protobuf 3.19.6 which is incompatible. - torchvision 0.17.2+cpu requires torch==2.2.2, but you have torch 2.4.1+cpu which is incompatible. - visualdl 2.5.3 requires protobuf>=3.20.0, but you have protobuf 3.19.6 which is incompatible. - Note: you may need to restart the kernel to use updated packages. - - Load the original model and inference ------------------------------------- @@ -114,19 +95,6 @@ Load the original model and inference audio_arr = generation.cpu().numpy().squeeze() sf.write("parler_tts_out.wav", audio_arr, model.config.sampling_rate) - -.. parsed-literal:: - - 2024-10-23 02:13:22.641328: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 02:13:22.675982: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - Flash attention 2 is not installed - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`. - WeightNorm.apply(module, name, dim) - You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers - The attention mask is not set and cannot be inferred from input because pad token is same as eos token.As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results. - - .. code:: ipython3 import IPython.display as ipd @@ -140,10 +108,10 @@ Load the original model and inference - + @@ -191,13 +159,6 @@ and Decoder (``ParlerTTSDecoder``). Lets convert them one by one. text_encoder_ov_model = convert(model.text_encoder, TEXT_ENCODER_OV_PATH, example_input) - -.. parsed-literal:: - - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4664: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead - warnings.warn( - - The Decoder Model performs in generation pipeline and we can separate it into two stage. In the first stage the model generates ``past_key_values`` into output for the second stage. In the second @@ -232,17 +193,6 @@ stage the model produces tokens during several runs. decoder_1_ov_model = convert(DecoderStage1Wrapper(model.decoder.model.decoder), DECODER_STAGE_1_OV_PATH, example_input) - -.. parsed-literal:: - - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/parler_tts/modeling_parler_tts.py:253: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - if seq_len > self.weights.size(0): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/parler_tts/modeling_parler_tts.py:1599: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - if sequence_length != 1: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/parler_tts/modeling_parler_tts.py:802: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! - if attn_output.size() != (bsz, self.num_heads, tgt_len, self.head_dim): - - .. code:: ipython3 DECODER_STAGE_2_OV_PATH = Path("models/decoder_stage_2_ir.xml") @@ -308,7 +258,7 @@ Select device from dropdown list for running inference using OpenVINO. .. parsed-literal:: - Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') + Dropdown(description='Device:', index=4, options=('CPU', 'GPU.0', 'GPU.1', 'GPU.2', 'AUTO'), value='AUTO') @@ -410,10 +360,10 @@ and run inference. - + @@ -456,27 +406,13 @@ Interactive inference demo = make_demo(fn=infer) try: - demo.queue().launch(debug=False) + demo.queue().launch(debug=True) except Exception: - demo.queue().launch(share=True, debug=False) + demo.queue().launch(share=True, debug=True) # if you are launching remotely, specify server_name and server_port # demo.launch(server_name='your server name', server_port='server port in int') # Read more in the docs: https://gradio.app/docs/ - -.. parsed-literal:: - - Running on local URL: http://127.0.0.1:7860 - - To create a public link, set `share=True` in `launch()`. - - - - - - - - .. code:: ipython3 # please uncomment and run this cell for stopping gradio interface diff --git a/docs/notebooks/person-tracking-with-output.rst b/docs/notebooks/person-tracking-with-output.rst index 2b07305615f09e..653a9b376edf7e 100644 --- a/docs/notebooks/person-tracking-with-output.rst +++ b/docs/notebooks/person-tracking-with-output.rst @@ -128,7 +128,7 @@ Guide =2024.0.0" + %pip install -q "openvino>=2024.0.0" %pip install -q opencv-python requests scipy tqdm "matplotlib>=3.4" @@ -189,16 +189,6 @@ We will use pre-trained models from OpenVINO’s `Open Model Zoo `__ to start the test. -Use ``omz_downloader``, which is a command-line tool from the -``openvino-dev`` package. It automatically creates a directory structure -and downloads the selected model. This step is skipped if the model is -already downloaded. The selected model comes from the public directory, -which means it must be converted into OpenVINO Intermediate -Representation (OpenVINO IR). - - **NOTE**: Using a model outside the list can require different pre- - and post-processing. - In this case, `person detection model `__ is deployed to detect the person in each frame of the video, and @@ -207,60 +197,39 @@ model `__, -``person-reidentification-retail-xxx`` from `Reidentification Models -list `__), -replace the name of the model in the code below. - .. code:: ipython3 + from notebook_utils import download_ir_model + # A directory where the model will be downloaded. base_model_dir = "model" precision = "FP16" # The name of the model from Open Model Zoo detection_model_name = "person-detection-0202" - download_command = ( - f"omz_downloader " f"--name {detection_model_name} " f"--precisions {precision} " f"--output_dir {base_model_dir} " f"--cache_dir {base_model_dir}" - ) - ! $download_command - detection_model_path = f"model/intel/{detection_model_name}/{precision}/{detection_model_name}.xml" + download_det_model_url = ( + f"https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/{detection_model_name}/{precision}/{detection_model_name}.xml" + ) + detection_model_path = download_ir_model(download_det_model_url, Path(base_model_dir) / detection_model_name / precision) reidentification_model_name = "person-reidentification-retail-0287" + download_reid_model_url = f"https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/{reidentification_model_name}/{precision}/{reidentification_model_name}.xml" - download_command = ( - f"omz_downloader " f"--name {reidentification_model_name} " f"--precisions {precision} " f"--output_dir {base_model_dir} " f"--cache_dir {base_model_dir}" - ) - ! $download_command + reidentification_model_path = download_ir_model(download_reid_model_url, Path(base_model_dir) / reidentification_model_name / precision) - reidentification_model_path = f"model/intel/{reidentification_model_name}/{precision}/{reidentification_model_name}.xml" .. parsed-literal:: - ################|| Downloading person-detection-0202 ||################ - - ========== Downloading model/intel/person-detection-0202/FP16/person-detection-0202.xml + model/person-detection-0202/FP16/person-detection-0202.bin: 0%| | 0.00/3.47M [00:00`__ #### -Table of contents: +precision using `NNCF `__ + +**Table of contents:** - `Prerequisites <#prerequisites>`__ - `Select Model <#select-model>`__ @@ -56,11 +57,6 @@ install required packages and setup helper functions. .. parsed-literal:: - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - descript-audiotools 0.7.2 requires protobuf<3.20,>=3.9.2, but you have protobuf 5.28.2 which is incompatible. - open-clip-torch 2.22.0 requires protobuf<4, but you have protobuf 5.28.2 which is incompatible. - tensorflow 2.12.0 requires protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3, but you have protobuf 5.28.2 which is incompatible. - tensorflow-metadata 1.14.0 requires protobuf<4.21,>=3.20.3, but you have protobuf 5.28.2 which is incompatible. Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -264,10 +260,10 @@ documentation 1 or self.sliding_window is not None) and self.is_causal: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb6151705d1ee73c25a80ca/modeling_phi3_v.py:444: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! seq_len = seq_len or torch.max(position_ids) + 1 /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb6151705d1ee73c25a80ca/modeling_phi3_v.py:445: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if seq_len > self.original_max_position_embeddings: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:86: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:86: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. op1 = operator(\*args, \*\*kwargs) + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/cache_utils.py:443: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results. + elif len(self.key_cache[layer_idx]) == 0: # fills previously skipped layers; checking for tensor causes errors /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb6151705d1ee73c25a80ca/modeling_phi3_v.py:683: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb6151705d1ee73c25a80ca/modeling_phi3_v.py:690: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): /opt/home/k8sworker/.cache/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb6151705d1ee73c25a80ca/modeling_phi3_v.py:702: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:165: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at aten/src/ATen/core/TensorBody.h:489.) + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:165: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at aten/src/ATen/core/TensorBody.h:489.) if a.grad is not None: @@ -505,12 +516,6 @@ arguments. For running model we will use ``generate`` method. generate_ids = model.generate(**inputs, eos_token_id=processor.tokenizer.eos_token_id, **generation_args) -.. parsed-literal:: - - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/auto/image_processing_auto.py:513: FutureWarning: The image_processor_class argument is deprecated and will be removed in v4.42. Please use `slow_image_processor_class`, or `fast_image_processor_class` instead - warnings.warn( - - .. parsed-literal:: Answer: diff --git a/docs/notebooks/photo-maker-with-output.rst b/docs/notebooks/photo-maker-with-output.rst index fe49eb9217065e..1685e9c05ea9f1 100644 --- a/docs/notebooks/photo-maker-with-output.rst +++ b/docs/notebooks/photo-maker-with-output.rst @@ -94,13 +94,13 @@ Clone PhotoMaker repository .. parsed-literal:: Cloning into 'PhotoMaker'... - remote: Enumerating objects: 303, done. - remote: Counting objects: 100% (148/148), done. - remote: Compressing objects: 100% (95/95), done. - remote: Total 303 (delta 130), reused 53 (delta 53), pack-reused 155 (from 1) - Receiving objects: 100% (303/303), 10.23 MiB | 29.77 MiB/s, done. - Resolving deltas: 100% (162/162), done. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/photo-maker/PhotoMaker + remote: Enumerating objects: 306, done. + remote: Counting objects: 100% (151/151), done. + remote: Compressing objects: 100% (98/98), done. + remote: Total 306 (delta 132), reused 53 (delta 53), pack-reused 155 (from 1) + Receiving objects: 100% (306/306), 10.24 MiB | 23.03 MiB/s, done. + Resolving deltas: 100% (164/164), done. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/photo-maker/PhotoMaker Note: switching to '1e78aa6514c11a84ef1be27b56c7c72d6c70f8fc'. You are in 'detached HEAD' state. You can look around, make experimental @@ -119,7 +119,7 @@ Clone PhotoMaker repository Turn off this advice by setting config variable advice.detachedHead to false HEAD is now at 1e78aa6 Update README.md - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/photo-maker + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/photo-maker Install required packages @@ -133,11 +133,9 @@ Install required packages .. parsed-literal:: ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - descript-audiotools 0.7.2 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.3 which is incompatible. - mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.17.2+cpu which is incompatible. paddleclas 2.5.2 requires gast==0.3.3, but you have gast 0.4.0 which is incompatible. paddleclas 2.5.2 requires opencv-python==4.6.0.66, but you have opencv-python 4.10.0.84 which is incompatible. - supervision 0.24.0 requires numpy<1.23.3,>=1.21.2; python_full_version <= "3.10.0", but you have numpy 1.24.4 which is incompatible. + parler-tts 0.2.1 requires protobuf>=4.0.0, but you have protobuf 3.20.3 which is incompatible. tensorflow 2.12.0 requires numpy<1.24,>=1.22, but you have numpy 1.24.4 which is incompatible. Note: you may need to restart the kernel to use updated packages. @@ -199,10 +197,10 @@ PhotoMaker to generate the original PhotoMaker pipeline. .. parsed-literal:: - 2024-10-23 02:19:25.748160: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 02:19:25.783265: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 02:22:09.727876: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 02:22:09.761823: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 02:19:26.449413: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 02:22:10.482979: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. code:: ipython3 @@ -391,9 +389,12 @@ output(text embeddings) which will be the input for U-Net model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4664: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/photo-maker/PhotoMaker/photomaker/model.py:84: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:243: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + if not interpolate_pos_encoding and (height != self.image_size or width != self.image_size): + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/photo-maker/PhotoMaker/photomaker/model.py:84: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert class_tokens_mask.sum() == stacked_id_embeds.shape[0], f"{class_tokens_mask.sum()} != {stacked_id_embeds.shape[0]}" @@ -468,9 +469,9 @@ sequence of latent text embeddings. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:86: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:88: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input_shape[-1] > 1 or self.sliding_window is not None: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: @@ -574,15 +575,15 @@ original Stable Diffusion XL model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unets/unet_2d_condition.py:1103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unets/unet_2d_condition.py:1103: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if dim % default_overall_up_factor != 0: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:136: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:136: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:145: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:145: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:146: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:146: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if hidden_states.shape[0] >= 64: diff --git a/docs/notebooks/pixart-with-output.rst b/docs/notebooks/pixart-with-output.rst index ef008d1d82ee52..c1c9a4b4e8ec57 100644 --- a/docs/notebooks/pixart-with-output.rst +++ b/docs/notebooks/pixart-with-output.rst @@ -118,10 +118,10 @@ directly in latent space, achieving super fast inference with few steps. .. parsed-literal:: - 2024-10-23 02:27:23.824587: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 02:27:23.860019: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 02:30:04.644117: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 02:30:04.680089: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 02:27:24.531762: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 02:30:05.360275: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -130,17 +130,17 @@ directly in latent space, achieving super fast inference with few steps. Loading pipeline components...: 0%| | 0/5 [00:00. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 +.. parsed-literal:: + Loading checkpoint shards: 0%| | 0/4 [00:00. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 + Some weights of the model checkpoint were not used when initializing PixArtTransformer2DModel: + ['caption_projection.y_embedding'] @@ -229,8 +229,9 @@ Convert text encoder .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4664: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. Convert transformer @@ -271,11 +272,11 @@ Convert transformer .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/embeddings.py:219: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/embeddings.py:219: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if self.height != height or self.width != width: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/attention_processor.py:682: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/attention_processor.py:682: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if current_length != target_length: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/attention_processor.py:697: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/attention_processor.py:697: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attention_mask.shape[0] < batch_size * head_size: @@ -300,9 +301,9 @@ Convert VAE decoder .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:146: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:146: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if hidden_states.shape[0] >= 64: @@ -448,7 +449,7 @@ And insert wrappers instances in the pipeline: .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'PixArtAlphaPipeline' object attribute is deprecated. Please access '_execution_device' over 'PixArtAlphaPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'PixArtAlphaPipeline' object attribute is deprecated. Please access '_execution_device' over 'PixArtAlphaPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -563,7 +564,7 @@ To collect intermediate model inputs for calibration we should customize .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'PixArtAlphaPipeline' object attribute is deprecated. Please access '_execution_device' over 'PixArtAlphaPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'PixArtAlphaPipeline' object attribute is deprecated. Please access '_execution_device' over 'PixArtAlphaPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -1621,16 +1622,16 @@ pipelines. Loading pipeline components...: 0%| | 0/5 [00:00=4.43.0, but you have transformers 4.45.2 which is incompatible. Note: you may need to restart the kernel to use updated packages. @@ -156,21 +153,21 @@ documentation 0.19. Please downgrade to tokenizers version <= 0.19 to export tokenizers to OpenVINO. + Exporting tokenizers to OpenVINO is not supported for tokenizers version > 0.19 and openvino version <= 2024.4. Please downgrade to tokenizers version <= 0.19 to export tokenizers to OpenVINO. INFO:nncf:Statistics of the bitwidth distribution: ┍━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑ │ Num bits (N) │ % all parameters (layers) │ % ratio-defining parameters (layers) │ @@ -488,7 +485,7 @@ documentation target_length: + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/cache_utils.py:443: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results. elif len(self.key_cache[layer_idx]) == 0: # fills previously skipped layers; checking for tensor causes errors - Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32) .. parsed-literal:: diff --git a/docs/notebooks/qwen2-vl-with-output.rst b/docs/notebooks/qwen2-vl-with-output.rst index 0ff882bb537ef2..a6f49a58ff0f25 100644 --- a/docs/notebooks/qwen2-vl-with-output.rst +++ b/docs/notebooks/qwen2-vl-with-output.rst @@ -139,10 +139,10 @@ using widget bellow: .. parsed-literal:: - 2024-10-23 04:32:38.810057: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 04:32:38.845114: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 04:37:28.225170: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 04:37:28.260034: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 04:32:39.399370: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 04:37:28.814599: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -335,26 +335,44 @@ documentation target_length: + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/cache_utils.py:443: TracerWarning: Using len to get tensor shape might cause the trace to be incorrect. Recommended usage would be tensor.shape[0]. Passing a tensor of different shape might lead to errors or silently give incorrect results. + elif len(self.key_cache[layer_idx]) == 0: # fills previously skipped layers; checking for tensor causes errors .. parsed-literal:: + ✅ Language model successfully converted ⌛ Weights compression with int4_asym mode started INFO:nncf:Statistics of the bitwidth distribution: ┍━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑ │ Num bits (N) │ % all parameters (layers) │ % ratio-defining parameters (layers) │ ┝━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥ - │ 8 │ 1% (1 / 130) │ 0% (0 / 129) │ + │ 8 │ 15% (1 / 197) │ 0% (0 / 196) │ ├────────────────┼─────────────────────────────┼────────────────────────────────────────┤ - │ 4 │ 99% (129 / 130) │ 100% (129 / 129) │ + │ 4 │ 85% (196 / 197) │ 100% (196 / 196) │ ┕━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙ @@ -374,31 +392,15 @@ documentation =3.9.2, but you have protobuf 3.20.3 which is incompatible. mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.19.1+cpu which is incompatible. - parler-tts 0.2 requires transformers<=4.43.3,>=4.43.0, but you have transformers 4.45.2 which is incompatible. + parler-tts 0.2.1 requires protobuf>=4.0.0, but you have protobuf 3.20.3 which is incompatible. Note: you may need to restart the kernel to use updated packages. Collecting iopath>=0.1.10 Using cached iopath-0.1.10-py3-none-any.whl - Requirement already satisfied: pillow>=9.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (10.4.0) - Requirement already satisfied: hydra-core>=1.3.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (1.3.2) - Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from iopath>=0.1.10) (4.66.5) - Requirement already satisfied: typing-extensions in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from iopath>=0.1.10) (4.12.2) - Requirement already satisfied: portalocker in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from iopath>=0.1.10) (2.10.1) - Requirement already satisfied: omegaconf<2.4,>=2.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (2.3.0) - Requirement already satisfied: antlr4-python3-runtime==4.9.* in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (4.9.3) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (24.1) - Requirement already satisfied: importlib-resources in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (6.4.5) - Requirement already satisfied: PyYAML>=5.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from omegaconf<2.4,>=2.2->hydra-core>=1.3.2) (6.0.2) - Requirement already satisfied: zipp>=3.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from importlib-resources->hydra-core>=1.3.2) (3.20.2) + Requirement already satisfied: pillow>=9.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (10.4.0) + Requirement already satisfied: hydra-core>=1.3.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (1.3.2) + Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from iopath>=0.1.10) (4.66.6) + Requirement already satisfied: typing-extensions in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from iopath>=0.1.10) (4.12.2) + Requirement already satisfied: portalocker in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from iopath>=0.1.10) (2.10.1) + Requirement already satisfied: omegaconf<2.4,>=2.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (2.3.0) + Requirement already satisfied: antlr4-python3-runtime==4.9.* in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (4.9.3) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (24.1) + Requirement already satisfied: importlib-resources in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from hydra-core>=1.3.2) (6.4.5) + Requirement already satisfied: PyYAML>=5.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from omegaconf<2.4,>=2.2->hydra-core>=1.3.2) (6.0.2) + Requirement already satisfied: zipp>=3.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from importlib-resources->hydra-core>=1.3.2) (3.20.2) Installing collected packages: iopath Attempting uninstall: iopath Found existing installation: iopath 0.1.9 @@ -204,10 +203,10 @@ Clone and install segment-anything-2 .. parsed-literal:: env: SAM2_BUILD_CUDA=0 - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/sam2-image-segmentation/sam2 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/sam2-image-segmentation/sam2 ERROR: Package 'sam-2' requires a different Python: 3.8.10 not in '>=3.10.0' Note: you may need to restart the kernel to use updated packages. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/sam2-image-segmentation + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/sam2-image-segmentation .. code:: ipython3 diff --git a/docs/notebooks/siglip-zero-shot-image-classification-with-output.rst b/docs/notebooks/siglip-zero-shot-image-classification-with-output.rst index 298a30bf7b6d7a..50aa71af0082b8 100644 --- a/docs/notebooks/siglip-zero-shot-image-classification-with-output.rst +++ b/docs/notebooks/siglip-zero-shot-image-classification-with-output.rst @@ -126,10 +126,10 @@ tokenizer and preparing the images. .. parsed-literal:: - 2024-10-23 04:42:12.224179: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 04:42:12.258186: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 04:45:37.694278: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 04:45:37.728953: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 04:42:12.920282: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 04:45:38.384577: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Run PyTorch model inference @@ -265,8 +265,9 @@ object ready to load on the device and start making predictions. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4779: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. Run OpenVINO model @@ -610,7 +611,7 @@ model are similar to the PyTorch model. .. parsed-literal:: - [{'dog': 0.99}, {'horse': 0.0}, {'cat': 0.0}, {'wolf': 0.0}, {'frog': 0.0}] + [{'dog': 0.99}, {'horse': 0.0}, {'cat': 0.0}, {'wolf': 0.0}, {'tiger': 0.0}] @@ -685,7 +686,7 @@ approximately estimate the speed up of the dynamic quantized models. .. parsed-literal:: - Performance speed up: 2.019 + Performance speed up: 2.491 Interactive inference diff --git a/docs/notebooks/siglip-zero-shot-image-classification-with-output_files/siglip-zero-shot-image-classification-with-output_24_1.png b/docs/notebooks/siglip-zero-shot-image-classification-with-output_files/siglip-zero-shot-image-classification-with-output_24_1.png index 76d6a6178cf64e..90392474870369 100644 --- a/docs/notebooks/siglip-zero-shot-image-classification-with-output_files/siglip-zero-shot-image-classification-with-output_24_1.png +++ b/docs/notebooks/siglip-zero-shot-image-classification-with-output_files/siglip-zero-shot-image-classification-with-output_24_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2f3af06f927b6f5e643d37869bbe32651d9a0f23270d5f3426cebbc2f02765cb -size 580998 +oid sha256:1e64834a97582f6009d2c6f15fd109cba0bdc0d9577d3a7f0f31a4d9f4519d44 +size 581000 diff --git a/docs/notebooks/speculative-sampling-with-output.rst b/docs/notebooks/speculative-sampling-with-output.rst index 127f61ab35085c..a88201eb73453c 100644 --- a/docs/notebooks/speculative-sampling-with-output.rst +++ b/docs/notebooks/speculative-sampling-with-output.rst @@ -1,5 +1,5 @@ -Text Generation via Speculative Sampling, KV Caching, and OpenVINO™ -=================================================================== +Text Generation via Speculative Decoding and OpenVINO™ +====================================================== As model sizes grow, Generative AI implementations require significant inference resources. This not only increases the cost per generation @@ -18,42 +18,49 @@ summary, inference optimizations for text generation are essential to reduce costs and power consumption, while also improving the accuracy and speed of text generation. -Another necessary condition is that the optimizations are compatible -with each other. That is, implementing a certain optimization should not -preclude other optimizations. There are several levels of optimizations -that can provide significant speedup without “bumping into each other” -in a way that will compromise overall efficiency. +Speculative decoding (or +`assisted-generation `__) +is a recent technique, that allows to speed up token generation when an +additional smaller draft model is used alongside with the main model. -For details on this method, please refer to the paper by Chen et al, -http://arxiv.org/abs/2302.01318. Additionally, there’s an interesting -proof of correctness of speculative sampling (showing that the original -distribution is preserved) by Leviathan et al, -http://arxiv.org/abs/2211.17192 +Speculative decoding works the following way. The draft model predicts +the next K tokens one by one in an autoregressive manner, while the main +model validates these predictions and corrects them if necessary. We go +through each predicted token, and if a difference is detected between +the draft and main model, we stop and keep the last token predicted by +the main model. Then the draft model gets the latest main prediction and +again tries to predict the next K tokens, repeating the cycle. -Our blog article describing this implementation with OpenVino is -available at openvino.ai +This approach reduces the need for multiple infer requests to the main +model, enhancing performance. For instance, in more predictable parts of +text generation, the draft model can, in best-case scenarios, generate +the next K tokens that exactly match the target. In that case they are +validated in a single inference request to the main model (which is +bigger, more accurate but slower) instead of running K subsequent +requests. More details can be found in the original +`paper `__. + +|image0| + +In this tutorial we consider how to apply Speculative decoding using +OpenVINO GenAI. **Table of contents:** - `Prerequisites <#prerequisites>`__ +- `Prepare models <#prepare-models>`__ - `Select inference device <#select-inference-device>`__ -- `Create autoregressive and speculative forms of sampling with KV - Cache - support <#create-autoregressive-and-speculative-forms-of-sampling-with-kv-cache-support>`__ - - - `Setup imports <#setup-imports>`__ - - `Prepare autoregressive - sampling <#prepare-autoregressive-sampling>`__ - - `Prepare speculative sampling <#prepare-speculative-sampling>`__ - -- `Main generation function <#main-generation-function>`__ +- `Run target model without speculative + decoding <#run-target-model-without-speculative-decoding>`__ +- `Run Speculative decoding + pipeline <#run-speculative-decoding-pipeline>`__ - - `Download and Convert Model <#download-and-convert-model>`__ ### - Installation Instructions +Installation Instructions +~~~~~~~~~~~~~~~~~~~~~~~~~ This is a self-contained example that relies solely on its own code. @@ -62,29 +69,64 @@ need a Jupyter server to start. For details, please refer to `Installation Guide `__. +.. |image0| image:: https://github.com/user-attachments/assets/eb999dea-d98b-42bb-835e-28d3054e1a84 + Prerequisites ------------- -First, we should install the `Hugging Face -Optimum `__ library -accelerated by OpenVINO integration. The Hugging Face Optimum Intel API -is a high-level API that enables us to convert and quantize models from -the Hugging Face Transformers library to the OpenVINO™ IR format. For -more details, refer to the `Hugging Face Optimum Intel -documentation `__. +First, we should install the `OpenVINO +GenAI `__ for running +model inference. + +|image01| + +OpenVINO™ GenAI is a library of the most popular Generative AI model +pipelines, optimized execution methods, and samples that run on top of +highly performant `OpenVINO +Runtime `__. + +This library is friendly to PC and laptop execution, and optimized for +resource consumption. It requires no external dependencies to run +generative models as it already includes all the core functionality +(e.g. tokenization via openvino-tokenizers). -We will also need to install transformers (HuggingFace) and some other -useful modules. +.. |image01| image:: https://media.githubusercontent.com/media/openvinotoolkit/openvino.genai/refs/heads/master/src/docs/openvino_genai.svg .. code:: ipython3 - %pip install -Uq pip - %pip uninstall -q -y optimum optimum-intel - %pip install --pre -Uq "openvino>=2024.2.0" openvino-tokenizers[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly - %pip install -q --upgrade transformers "torch>=2.1" "torchvision" "gradio>=4.19" accelerate "onnx<1.16.2" ipywidgets --extra-index-url https://download.pytorch.org/whl/cpu - %pip install -q "git+https://github.com/huggingface/optimum-intel.git" + %pip install --pre -Uq "openvino>=2024.4.0" openvino-tokenizers openvino-genai huggingface_hub --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + +Prepare models +-------------- + + + +As example, we will use already converted LLMs from `OpenVINO +collection `__, +but in case, if you want run own models, you should convert them using +`Hugging Face +Optimum `__ +library accelerated by OpenVINO integration. More details about model +preparation can be found in `OpenVINO LLM inference +guide `__ + +.. code:: ipython3 + + from pathlib import Path + import huggingface_hub as hf_hub + + draft_model_id = "OpenVINO/dolly-v2-3b-int4-ov" + target_model_id = "OpenVINO/dolly-v2-7b-int8-ov" + + draft_model_path = Path(draft_model_id.split("/")[-1]) + target_model_path = Path(target_model_id.split("/")[-1]) + + if not draft_model_path.exists(): + hf_hub.snapshot_download(draft_model_id, local_dir=draft_model_path) + if not target_model_path.exists(): + hf_hub.snapshot_download(target_model_id, local_dir=target_model_path) Select inference device ~~~~~~~~~~~~~~~~~~~~~~~ @@ -97,336 +139,184 @@ OpenVINO. .. code:: ipython3 import requests - + r = requests.get( url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", ) open("notebook_utils.py", "w").write(r.text) - + from notebook_utils import device_widget - - device = device_widget() - - device + device = device_widget(default="CPU", exclude=["NPU", "AUTO"]) + device -.. parsed-literal:: - Dropdown(description='Device:', options=('CPU', 'GPU.0', 'GPU.1', 'AUTO'), value='CPU') +.. parsed-literal:: + Dropdown(description='Device:', options=('CPU',), value='CPU') -Create autoregressive and speculative forms of sampling with KV Cache support ------------------------------------------------------------------------------ +Run target model without speculative decoding +--------------------------------------------- -Text generation is often done in an autoregressive fashion. We will all -support a KV cache (aka Past Value Cache) in the code. Note that we are -using greedy sampling. We do not adjust other text generation parameters -(e.g. temperature) so keep this illustration of speculative sampling as -simple and understandable as possible. -Setup imports -~~~~~~~~~~~~~ +OpenVINO GenAI provides easy-to-use API for running text generation. +Firstly we will create pipeline with ``LLMPipeline``. ``LLMPipeline`` is +the main object used for decoding. You can construct it straight away +from the folder with the converted model. It will automatically load the +``main model``, ``tokenizer``, ``detokenizer`` and default +``generation configuration``. After that we will configure parameters +for decoding. Then we just run ``generate`` method and get the output in +text format. We do not need to encode input prompt according to model +expected template or write post-processing code for logits decoder, it +will be done easily with LLMPipeline. +To obtain intermediate generation results without waiting until when +generation is finished, we will write streamer function. .. code:: ipython3 + import openvino_genai import time - import numpy as np - import openvino as ov -Prepare autoregressive sampling -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + pipe = openvino_genai.LLMPipeline(target_model_path, device.value) + config = openvino_genai.GenerationConfig() + config.max_new_tokens = 100 -.. code:: ipython3 + def streamer(subword): + print(subword, end="", flush=True) + # Return flag corresponds whether generation should be stopped. + # False means continue generation. + return False - def autoregressive_sampling_with_pkv(input, model, N=30): - input_ids, attention_mask = input.input_ids, input.attention_mask - seq_len = input_ids.shape[-1] - position_ids = np.arange(0, seq_len, dtype=np.int64).reshape([-1, seq_len]) - - # in all subsequent inferences we feed tokens one by one, - # but for the first one we feed the whole encoded prompt - request = model.create_infer_request() - request.infer((input_ids, attention_mask, position_ids, np.array([0]))) - next_token = np.argmax(request.results["logits"][:, -1]).reshape([1]) - - all_tokens = [] - all_tokens.extend(input_ids[0]) - all_tokens.append(next_token[0]) - - while seq_len < N: - input_ids = next_token.reshape([1, 1]) - attention_mask = np.concatenate((attention_mask, np.array([1]).reshape([1, 1])), axis=1) - position_ids = np.array([attention_mask.shape[1]]).reshape([1, 1]) - - request.infer((input_ids, attention_mask, position_ids, np.array([0]))) - next_token = np.argmax(request.results["logits"][:, -1]) - all_tokens.append(next_token) - seq_len += 1 - - return all_tokens - -Prepare speculative sampling -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - -- Step 1: With speculative sampling, we first generate K samples from - the draft model (in an autoregressive manner). -- Step 2: These are now candidates to examine using the main model - (step 2) using a batch size of K. -- Step 3: We go through each K predicted tokens, and if tokens differ, - we stop and keep the last token predicted by the main model. -- Step 4: We update KV-cache dropping keys & values for differing - tokens and repeat Step 1. -.. code:: ipython3 + start_time = time.perf_counter() + pipe.generate(["Sun is yellow because"], config, streamer=streamer) + end_time = time.perf_counter() - def update_state(request, seq_len): - for state in request.query_state(): - old_seq_len = state.state.shape[2] - if seq_len >= old_seq_len: - continue - # After the inference request, key/values have shape [BATCH_SIZE, seq_len + K, vocab_size]. - # Increment the sequence length by the number of matched tokens, and - # trim the KV cache to match the new sequence length. - state.state = ov.Tensor(state.state.data[:, :, :seq_len]) - - - def speculative_sampling_with_pkv(input, draft_model, main_model, K, N=30, **kwargs): - input_ids, attention_mask = input.input_ids, input.attention_mask - # seq_len number of key/values or number of already processed input tokens - seq_len = input_ids.shape[-1] - position_ids = np.arange(0, seq_len, dtype=np.int64).reshape([-1, seq_len]) - - draft_request = draft_model.create_infer_request() - draft_request.infer((input_ids, attention_mask, position_ids, np.array([0]))) - - main_request = main_model.create_infer_request() - main_request.infer((input_ids, attention_mask, position_ids, np.array([0]))) - first_token = np.argmax(main_request.results["logits"][:, -1]).reshape([1]) - - all_tokens = [] - all_tokens.extend(input_ids[0]) - all_tokens.append(first_token[0]) - - accum_draft_tokens = [] - while seq_len < N: - next_token = first_token - for i in range(K): - input_ids = next_token.reshape([1, 1]) - attention_mask = np.concatenate((attention_mask, np.array([1]).reshape([1, 1])), axis=1) - position_ids = np.array([attention_mask.shape[1]]).reshape([1, 1]) - - draft_request.infer((input_ids, attention_mask, position_ids, np.array([0]))) - next_token = np.argmax(draft_request.results["logits"][:, -1]) - accum_draft_tokens.append(next_token) - - # main model will give also K out tokens - # feed the same first token to the main model and do not give the last token generated by the draft - input_ids = np.concatenate((first_token.reshape([1]), accum_draft_tokens[:-1])).reshape([1, -1]) - attention_mask = np.ones((1, seq_len + K)) - position_ids = np.arange(seq_len, seq_len + K, dtype=np.int64).reshape([1, -1]) - - main_request.infer((input_ids, attention_mask, position_ids, np.array([0]))) - next_tokens = np.argmax(main_request.results["logits"], axis=-1)[0] - - # if disagrees from the very beggining then context will be expanded only for one element - # all elements match then context will be expanded to K elements - for disagree_idx, (t1, t2) in enumerate(zip(accum_draft_tokens, next_tokens)): - if t1 != t2: - break - - first_token = next_tokens[disagree_idx] - all_tokens.extend(next_tokens[: disagree_idx + 1]) - seq_len += disagree_idx + 1 - - # cut key/values depending on the position where disagreement starts - update_state(draft_request, seq_len) - update_state(main_request, seq_len) - - attention_mask = np.ones((1, seq_len)) - accum_draft_tokens = [] - all_tokens.extend(accum_draft_tokens) - return all_tokens - -Main generation function ------------------------- - - - -Download and Convert Model -~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - -Optimum Intel can be used to load optimized models from the `Hugging -Face Hub `__ and -create pipelines to run an inference with OpenVINO Runtime using Hugging -Face APIs. For speculative decoding we need to manually update states, -therefore we will use directly openvino inference api, and optimum only -for model conversion. >To download Llama-2-7b-chat-hf, you will need to -accept license agreement. You must be a registered user in Hugging -Face Hub. Please visit HuggingFace model -`card `__, -carefully read terms of usage and click accept button. You will need to -use an access token for the code below to run. For more information on -access tokens, refer to this section of the documentation. -.. code:: ipython3 +.. parsed-literal:: - from pathlib import Path - - main_model_id = "meta-llama/Llama-2-7b-chat-hf" - main_model_path = Path("Llama-2-7b-chat-hf") - draft_model_id = "TinyLlama/TinyLlama-1.1B-Chat-v1.0" - draft_model_path = Path("TinyLlama-1.1B-Chat-v1.0") - - from transformers import AutoTokenizer - - main_tokenizer = AutoTokenizer.from_pretrained(main_model_id) - draft_tokenizer = AutoTokenizer.from_pretrained(draft_model_id) + it is made of gas. The gas is heated to a high temperature and then cooled. The gas is yellow because it has a band of light called the "Bondeson Pendulum Effect." The Bondeson Pendulum Effect is caused by the light waves bouncing off of the gas molecules. The light waves bounce off of the gas molecules in different ways, some of the light waves get scattered, and some of the light waves get reflected. The light waves that get scattered and reflected combine to .. code:: ipython3 - # In order for speculative sampling to work, both main and draft tokenizers should be the same. - token_test_txt = "text to ensure tokenizers work the same, as of 2024" - tokens_1 = draft_tokenizer(token_test_txt, return_tensors="pt").input_ids - tokens_2 = main_tokenizer(token_test_txt, return_tensors="pt").input_ids - - assert all((tokens_1 - tokens_2)[0] == 0) + import gc -.. code:: ipython3 + print(f"Generation time: {end_time - start_time:.2f}s") + del pipe + gc.collect(); - if not main_model_path.exists(): - !optimum-cli export openvino --model $main_model_id --weight-format fp16 {main_model_path} - if not draft_model_path.exists(): - !optimum-cli export openvino --model $draft_model_id --weight-format fp16 {draft_model_path} -Infer directly using OpenVINO Inference Pipeline +.. parsed-literal:: -.. code:: ipython3 + Generation time: 18.44s - core = ov.Core() - draft_ov_model = core.read_model(draft_model_path / "openvino_model.xml") - draft_model = core.compile_model(draft_ov_model, device_name=device.value) - - main_ov_model = core.read_model(main_model_path / "openvino_model.xml") - main_model = core.compile_model(main_ov_model, device_name=device.value) -.. code:: ipython3 +Run Speculative decoding pipeline +--------------------------------- - def main( - prompt: str, - n_tokens_to_generate: int = 75, - K: int = 5, - seed: int = 5555, - ): - # seed numpy rng - np.random.seed(seed) - tokenized = main_tokenizer(prompt, return_tensors="pt") - - def run_autoregressive_sampling_fn(decode_fn, tokenized, **kwargs): - start = time.perf_counter() - output_ids = decode_fn(tokenized, **kwargs) - text = main_tokenizer.decode(output_ids, skip_special_tokens=True) - elapsed_time = time.perf_counter() - start - return text, elapsed_time - - def run_speculative_sampling_fn(decode_fn, input_ids, **kwargs): - start = time.perf_counter() - output_ids = decode_fn(input_ids, **kwargs) - text = main_tokenizer.decode(output_ids, skip_special_tokens=True) - elapsed_time = time.perf_counter() - start - return text, elapsed_time - - autoregressive_text, autoregressive_time = run_autoregressive_sampling_fn( - autoregressive_sampling_with_pkv, - tokenized, - model=main_model, - N=n_tokens_to_generate, - ) - - speculative_text, speculative_time = run_speculative_sampling_fn( - speculative_sampling_with_pkv, - tokenized, - main_model=main_model, - draft_model=draft_model, - N=n_tokens_to_generate, - K=K, - ) - - # Format results for output in gradio - out = "\n" + "Autoregressive Decode" + "\n" + "---------------------" + "\n" - out = out + f"Time = {autoregressive_time:.2f}s" + "\n" + f"Text = {autoregressive_text}" + "\n" - out = out + "\n" + "Speculative Decode" + "\n" + "------------------" + "\n" - out = out + f"Time = {speculative_time:.2f}s" + "\n" + f"Text = {speculative_text}" - return out + + +To enable Speculative decoding in ``LLMPipeline,`` we should +additionally provide the ``draft_model`` structure and +``SchedulerConfig`` for resource management. + +|image011| + +As shown in the figure above, speculative decoding works by splitting +the generative process into two stages. In the first stage, a fast, but +less accurate draft model (AKA assistant) autoregressively generates a +sequence of tokens. In the second stage, a large, but more accurate +target model conducts parallelized verification over the generated draft +tokens. This process allows the target model to produce multiple tokens +in a single forward pass and thus accelerate autoregressive decoding. +The success of speculative decoding largely hinges on the speculation +lookahead (SL), i.e. the number of tokens produced by the draft model in +each iteration. The straightforward method, based on `Leviathan et +al. `__, uses a static value of the +speculation lookahead and involves generating a constant number of +candidate tokens at each speculative iteration. You can adjust the +number of candidates using ``num_assistant_tokens`` parameter in +generation config. If the assistant model’s confidence in its prediction +for the current token is lower than this threshold, the assistant model +stops the current token generation iteration is not yet reached. + +.. |image011| image:: https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/dynamic_speculation_lookahead/spec_dec_diagram.png .. code:: ipython3 - res = main("Alan Turing was a", n_tokens_to_generate=100) - print(res) + scheduler_config = openvino_genai.SchedulerConfig() + # cache params + scheduler_config.cache_size = 2 + scheduler_config.block_size = 16 if "GPU" in device.value else 32 + + draft_model = openvino_genai.draft_model(draft_model_path, device.value) + + pipe = openvino_genai.LLMPipeline(target_model_path, device.value, draft_model=draft_model, scheduler_config=scheduler_config) + + config = openvino_genai.GenerationConfig() + config.max_new_tokens = 100 + config.num_assistant_tokens = 5 + start_time = time.perf_counter() + result = pipe.generate(["Sun is yellow because"], config, streamer=streamer) + end_time = time.perf_counter() .. parsed-literal:: - 2024-04-17 10:21:41.642283: I tensorflow/core/util/port.cc:111] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-04-17 10:21:41.644834: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used. - 2024-04-17 10:21:41.677055: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered - 2024-04-17 10:21:41.677093: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered - 2024-04-17 10:21:41.677119: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered - 2024-04-17 10:21:41.683198: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used. - 2024-04-17 10:21:41.683977: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-04-17 10:21:42.477656: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + it is made of gas. The gas is heated to a high temperature and then cooled. The gas changes from a hot gas to a cold gas and then from a cold gas to a hot gas. The gas is very hot when it changes from a hot gas to a cold gas and very cold when it changes from a cold gas to a hot gas. When the gas changes from a hot gas to a cold gas it becomes yellow. When the gas changes from a cold gas to a hot gas it + +.. code:: ipython3 + + print(f"Generation time: {end_time - start_time:.2f}s") .. parsed-literal:: - - Autoregressive Decode - --------------------- - Time = 44.39s - Text = Alan Turing was a British mathematician, computer scientist, and codebreaker who played a pivotal role in cracking the German Enigma code during World War II. He was also a pioneer in the field of artificial intelligence and made significant contributions to the development of computer science. - - Turing was born on June 23, 1912, in London, England. He was educated at Cambridge University, where he earned a degree in mathematics in - - Speculative Decode - ------------------ - Time = 22.96s - Text = Alan Turing was a British mathematician, computer scientist, and codebreaker who played a pivotal role in cracking the German Enigma code during World War II. He was also a pioneer in the field of artificial intelligence and made significant contributions to the development of computer science. - - Turing was born on June 23, 1912, in London, England. He was educated at Cambridge University, where he earned a degree in mathematics in 1 + Generation time: 15.62s +Alternative approach, Dynamic Speculative Decoding, described in the +`paper `__ is based on heuristics and +adjusts the number of candidate tokens for the next iteration based on +the acceptance rate of the current iteration. If all speculative tokens +are correct, the number of candidate tokens increases; otherwise, it +decreases. For adjusting number of tokens +``assistant_confidence_threshold`` parameters should be used. If the +assistant model’s confidence in its prediction for the current token is +lower than this threshold, the assistant model stops the current token +generation iteration, even if the number of ``num_assistant_tokens`` is +not yet reached. You can find more details in this `blog +post `__. + .. code:: ipython3 - if not Path("gradio_helper.py").exists(): - r = requests.get(url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/notebooks/speculative-sampling/gradio_helper.py") - open("gradio_helper.py", "w").write(r.text) - - from gradio_helper import make_demo - - demo = make_demo(fn=main) - - try: - demo.launch(debug=False) - except Exception: - demo.launch(share=True, debug=False) - # If you are launching remotely, specify server_name and server_port - # EXAMPLE: `demo.launch(server_name='your server name', server_port='server port in int')` - # To learn more please refer to the Gradio docs: https://gradio.app/docs/ + config = openvino_genai.GenerationConfig() + config.max_new_tokens = 100 + config.assistant_confidence_threshold = 0.05 + start_time = time.perf_counter() + result = pipe.generate(["Sun is yellow because"], config, streamer) + end_time = time.perf_counter() + + +.. parsed-literal:: + + it is made of gas. The gas is heated to a high temperature and then cooled. The gas changes from a hot gas to a cold gas and then from a cold gas to a hot gas. The gas is very hot when it changes from a hot gas to a cold gas and very cold when it changes from a cold gas to a hot gas. The gas is very light and can float in the air. When the gas cools it becomes a liquid. The Sun is a huge sphere of .. code:: ipython3 - # please uncomment and run this cell for stopping gradio interface - # demo.close() + print(f"Generation time: {end_time - start_time:.2f}s") + + +.. parsed-literal:: + + Generation time: 17.97s + diff --git a/docs/notebooks/speech-recognition-quantization-wav2vec2-with-output.rst b/docs/notebooks/speech-recognition-quantization-wav2vec2-with-output.rst index be35e6d00f9293..b4a9cf92289306 100644 --- a/docs/notebooks/speech-recognition-quantization-wav2vec2-with-output.rst +++ b/docs/notebooks/speech-recognition-quantization-wav2vec2-with-output.rst @@ -57,47 +57,47 @@ Guide =0.11.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (1.5.0) - Requirement already satisfied: torch>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.4.1+cpu) - Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (3.16.1) - Requirement already satisfied: numpy>=1.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (1.24.4) - Requirement already satisfied: pyarrow>=15.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (17.0.0) - Requirement already satisfied: dill<0.3.9,>=0.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (0.3.8) - Requirement already satisfied: pandas in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (2.0.3) - Requirement already satisfied: requests>=2.32.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (2.32.3) - Requirement already satisfied: tqdm>=4.66.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (4.66.5) - Requirement already satisfied: xxhash in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (3.5.0) - Requirement already satisfied: multiprocess<0.70.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (0.70.16) - Requirement already satisfied: fsspec<=2024.9.0,>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets) (2024.9.0) - Requirement already satisfied: aiohttp in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (3.10.10) - Requirement already satisfied: huggingface-hub>=0.23.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (0.26.1) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (24.1) - Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (6.0.2) - Requirement already satisfied: lightning-utilities>=0.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchmetrics>=0.11.0) (0.11.8) - Requirement already satisfied: typing-extensions in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchmetrics>=0.11.0) (4.12.2) - Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1.0) (1.13.3) - Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1.0) (3.1) - Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1.0) (3.1.4) - Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (2.4.3) - Requirement already satisfied: aiosignal>=1.1.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (1.3.1) - Requirement already satisfied: attrs>=17.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (24.2.0) - Requirement already satisfied: frozenlist>=1.1.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (1.4.1) - Requirement already satisfied: multidict<7.0,>=4.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (6.1.0) - Requirement already satisfied: yarl<2.0,>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (1.15.2) - Requirement already satisfied: async-timeout<5.0,>=4.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (4.0.3) - Requirement already satisfied: setuptools in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from lightning-utilities>=0.8.0->torchmetrics>=0.11.0) (75.2.0) - Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (3.4.0) - Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (3.10) - Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (2.2.3) - Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (2024.8.30) - Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch>=2.1.0) (2.1.5) - Requirement already satisfied: python-dateutil>=2.8.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pandas->datasets) (2.9.0.post0) - Requirement already satisfied: pytz>=2020.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pandas->datasets) (2024.2) - Requirement already satisfied: tzdata>=2022.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pandas->datasets) (2024.2) - Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch>=2.1.0) (1.3.0) - Requirement already satisfied: six>=1.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0) - Requirement already satisfied: propcache>=0.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from yarl<2.0,>=1.12.0->aiohttp->datasets) (0.2.0) + Requirement already satisfied: datasets in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (3.1.0) + Requirement already satisfied: torchmetrics>=0.11.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (1.5.1) + Requirement already satisfied: torch>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.4.1+cpu) + Requirement already satisfied: filelock in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (3.16.1) + Requirement already satisfied: numpy>=1.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (1.24.4) + Requirement already satisfied: pyarrow>=15.0.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (17.0.0) + Requirement already satisfied: dill<0.3.9,>=0.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (0.3.8) + Requirement already satisfied: pandas in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (2.0.3) + Requirement already satisfied: requests>=2.32.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (2.32.3) + Requirement already satisfied: tqdm>=4.66.3 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (4.66.6) + Requirement already satisfied: xxhash in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (3.5.0) + Requirement already satisfied: multiprocess<0.70.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (0.70.16) + Requirement already satisfied: fsspec<=2024.9.0,>=2023.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets) (2024.9.0) + Requirement already satisfied: aiohttp in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (3.10.10) + Requirement already satisfied: huggingface-hub>=0.23.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (0.26.2) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (24.1) + Requirement already satisfied: pyyaml>=5.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from datasets) (6.0.2) + Requirement already satisfied: lightning-utilities>=0.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchmetrics>=0.11.0) (0.11.8) + Requirement already satisfied: typing-extensions in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torchmetrics>=0.11.0) (4.12.2) + Requirement already satisfied: sympy in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1.0) (1.13.3) + Requirement already satisfied: networkx in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1.0) (3.1) + Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch>=2.1.0) (3.1.4) + Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (2.4.3) + Requirement already satisfied: aiosignal>=1.1.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (1.3.1) + Requirement already satisfied: attrs>=17.3.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (24.2.0) + Requirement already satisfied: frozenlist>=1.1.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (1.5.0) + Requirement already satisfied: multidict<7.0,>=4.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (6.1.0) + Requirement already satisfied: yarl<2.0,>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (1.15.2) + Requirement already satisfied: async-timeout<5.0,>=4.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from aiohttp->datasets) (4.0.3) + Requirement already satisfied: setuptools in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from lightning-utilities>=0.8.0->torchmetrics>=0.11.0) (75.3.0) + Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (3.4.0) + Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (3.10) + Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (2.2.3) + Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.32.2->datasets) (2024.8.30) + Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch>=2.1.0) (2.1.5) + Requirement already satisfied: python-dateutil>=2.8.2 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pandas->datasets) (2.9.0.post0) + Requirement already satisfied: pytz>=2020.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pandas->datasets) (2024.2) + Requirement already satisfied: tzdata>=2022.1 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pandas->datasets) (2024.2) + Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from sympy->torch>=2.1.0) (1.3.0) + Requirement already satisfied: six>=1.5 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0) + Requirement already satisfied: propcache>=0.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from yarl<2.0,>=1.12.0->aiohttp->datasets) (0.2.0) Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -116,6 +116,15 @@ Imports from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor + +.. parsed-literal:: + + 2024-11-05 04:52:12.108210: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 04:52:12.142263: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2024-11-05 04:52:12.803683: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + + Settings -------- @@ -163,9 +172,16 @@ IR). .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4779: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11. + + +.. parsed-literal:: + + [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py:871: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/wav2vec2/modeling_wav2vec2.py:872: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_output.size() != (bsz, self.num_heads, tgt_len, self.head_dim): @@ -272,14 +288,6 @@ steps: INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino -.. parsed-literal:: - - 2024-10-23 04:48:23.521146: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 04:48:23.553631: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. - To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 04:48:24.182318: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - - .. parsed-literal:: @@ -500,7 +508,7 @@ quantized model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:62: FutureWarning: Importing `WordErrorRate` from `torchmetrics` was deprecated and will be removed in 2.0. Import `WordErrorRate` from `torchmetrics.text` instead. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:62: FutureWarning: Importing `WordErrorRate` from `torchmetrics` was deprecated and will be removed in 2.0. Import `WordErrorRate` from `torchmetrics.text` instead. _future_warning( @@ -570,23 +578,23 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 17.90 ms + [ INFO ] Read model took 18.07 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: - [ INFO ] input_values , 45 (node: input_values) : f32 / [...] / [?,?] + [ INFO ] 45 , input_values (node: input_values) : f32 / [...] / [?,?] [ INFO ] Model outputs: [ INFO ] logits (node: __module.lm_head/aten::linear/Add) : f32 / [...] / [?,1..,32] [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [ INFO ] Reshaping model: '45': [1,30480] - [ INFO ] Reshape model took 4.29 ms + [ INFO ] Reshape model took 4.26 ms [Step 6/11] Configuring input of the model [ INFO ] Model inputs: - [ INFO ] input_values , 45 (node: input_values) : f32 / [...] / [1,30480] + [ INFO ] 45 , input_values (node: input_values) : f32 / [...] / [1,30480] [ INFO ] Model outputs: [ INFO ] logits (node: __module.lm_head/aten::linear/Add) : f32 / [...] / [1,95,32] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 493.63 ms + [ INFO ] Compile model took 495.05 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -623,17 +631,17 @@ models. [ INFO ] Fill input '45' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 69.32 ms + [ INFO ] First inference took 69.60 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 5400 iterations - [ INFO ] Duration: 120141.91 ms + [ INFO ] Count: 5406 iterations + [ INFO ] Duration: 120228.72 ms [ INFO ] Latency: - [ INFO ] Median: 131.56 ms - [ INFO ] Average: 133.32 ms - [ INFO ] Min: 88.10 ms - [ INFO ] Max: 314.58 ms - [ INFO ] Throughput: 44.95 FPS + [ INFO ] Median: 131.74 ms + [ INFO ] Average: 133.26 ms + [ INFO ] Min: 107.29 ms + [ INFO ] Max: 340.52 ms + [ INFO ] Throughput: 44.96 FPS .. code:: ipython3 @@ -660,7 +668,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 24.51 ms + [ INFO ] Read model took 23.99 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] 45 , input_values (node: input_values) : f32 / [...] / [?,?] @@ -669,14 +677,14 @@ models. [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [ INFO ] Reshaping model: '45': [1,30480] - [ INFO ] Reshape model took 5.99 ms + [ INFO ] Reshape model took 6.04 ms [Step 6/11] Configuring input of the model [ INFO ] Model inputs: [ INFO ] 45 , input_values (node: input_values) : f32 / [...] / [1,30480] [ INFO ] Model outputs: [ INFO ] logits (node: __module.lm_head/aten::linear/Add) : f32 / [...] / [1,95,32] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 1234.19 ms + [ INFO ] Compile model took 1203.24 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -713,15 +721,15 @@ models. [ INFO ] Fill input '45' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 56.91 ms + [ INFO ] First inference took 53.37 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 7974 iterations - [ INFO ] Duration: 120125.59 ms + [ INFO ] Count: 8004 iterations + [ INFO ] Duration: 120086.07 ms [ INFO ] Latency: - [ INFO ] Median: 88.71 ms - [ INFO ] Average: 90.24 ms - [ INFO ] Min: 68.79 ms - [ INFO ] Max: 212.19 ms - [ INFO ] Throughput: 66.38 FPS + [ INFO ] Median: 88.68 ms + [ INFO ] Average: 89.86 ms + [ INFO ] Min: 68.86 ms + [ INFO ] Max: 235.63 ms + [ INFO ] Throughput: 66.65 FPS diff --git a/docs/notebooks/speechbrain-emotion-recognition-with-output.rst b/docs/notebooks/speechbrain-emotion-recognition-with-output.rst index 55e3b31765b191..42783b7d598b01 100644 --- a/docs/notebooks/speechbrain-emotion-recognition-with-output.rst +++ b/docs/notebooks/speechbrain-emotion-recognition-with-output.rst @@ -53,25 +53,6 @@ Installations %pip install -q "transformers>=4.30.0" "huggingface_hub>=0.8.0" "SoundFile" %pip install -q "openvino>=2024.1.0" - -.. parsed-literal:: - - Note: you may need to restart the kernel to use updated packages. - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - altair 5.4.1 requires typing-extensions>=4.10.0; python_version < "3.13", but you have typing-extensions 4.9.0 which is incompatible. - descript-audiotools 0.7.2 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.3 which is incompatible. - detectron2 0.6 requires iopath<0.1.10,>=0.1.7, but you have iopath 0.1.10 which is incompatible. - mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.19.1+cpu which is incompatible. - modelscope-studio 0.5.0 requires gradio<5.0,>=4.0, but you have gradio 3.43.1 which is incompatible. - openvino-dev 2024.4.0 requires openvino==2024.4.0, but you have openvino 2024.5.0.dev20241014 which is incompatible. - parler-tts 0.2 requires transformers<=4.43.3,>=4.43.0, but you have transformers 4.45.2 which is incompatible. - tensorflow 2.12.0 requires numpy<1.24,>=1.22, but you have numpy 1.24.4 which is incompatible. - typeguard 4.3.0 requires typing-extensions>=4.10.0, but you have typing-extensions 4.9.0 which is incompatible. - Note: you may need to restart the kernel to use updated packages. - Note: you may need to restart the kernel to use updated packages. - Note: you may need to restart the kernel to use updated packages. - - Imports ~~~~~~~ @@ -85,6 +66,12 @@ Imports import openvino as ov + +.. parsed-literal:: + + torchvision is not available - cannot save figures + + Prepare base model ~~~~~~~~~~~~~~~~~~ @@ -116,42 +103,6 @@ SpeechBrain codebase. source="speechbrain/emotion-recognition-wav2vec2-IEMOCAP", pymodule_file="custom_interface.py", classname="CustomEncoderWav2vec2Classifier" ) - - -.. parsed-literal:: - - config.json: 0%| | 0.00/1.84k [00:00=2.2" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q "stable-audio-tools" "nncf>=2.12.0" --extra-index-url https://download.pytorch.org/whl/cpu - %pip install -q "openvino>=2024.3.0" + if platform.system() == "Darwin": + %pip install -q "numpy>=1.26,<2.0.0" "pandas>2.0.2" + else: + %pip install -q "numpy>=1.26" "pandas>2.0.2" + %pip install -q "openvino>=2024.4.0" Load the original model and inference ------------------------------------- @@ -108,15 +114,28 @@ Load the original model and inference model, model_config = get_pretrained_model("stabilityai/stable-audio-open-1.0") + +.. parsed-literal:: + + model_config.json: 0%| | 0.00/4.17k [00:00=t0 but got ta=0.29999998211860657 and t0=0.3. + /home/ea/work/py311/lib/python3.11/site-packages/torchsde/_brownian/brownian_interval.py:599: UserWarning: Should have ta>=t0 but got ta=0.29999998211860657 and t0=0.3. warnings.warn(f"Should have ta>=t0 but got ta={ta} and t0={self._start}.") @@ -210,7 +235,7 @@ Load the original model and inference @@ -287,7 +312,7 @@ documentation =t0 but got ta=0.29999998211860657 and t0=0.3. + /home/ea/work/py311/lib/python3.11/site-packages/torchsde/_brownian/brownian_interval.py:599: UserWarning: Should have ta>=t0 but got ta=0.29999998211860657 and t0=0.3. warnings.warn(f"Should have ta>=t0 but got ta={ta} and t0={self._start}.") @@ -679,7 +685,7 @@ and run inference. @@ -736,8 +742,3 @@ Interactive inference # If you are launching remotely, specify server_name and server_port # EXAMPLE: `demo.launch(server_name='your server name', server_port='server port in int')` # To learn more please refer to the Gradio docs: https://gradio.app/docs/ - -.. code:: ipython3 - - # please uncomment and run this cell for stopping gradio interface - # demo.close() diff --git a/docs/notebooks/stable-cascade-image-generation-with-output.rst b/docs/notebooks/stable-cascade-image-generation-with-output.rst index 969ed4e4dbe336..8bd358958e47b6 100644 --- a/docs/notebooks/stable-cascade-image-generation-with-output.rst +++ b/docs/notebooks/stable-cascade-image-generation-with-output.rst @@ -56,13 +56,8 @@ Prerequisites .. parsed-literal:: ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. - descript-audiotools 0.7.2 requires protobuf<3.20,>=3.9.2, but you have protobuf 5.28.3 which is incompatible. mobileclip 0.1.0 requires torchvision==0.14.1, but you have torchvision 0.19.1+cpu which is incompatible. - open-clip-torch 2.22.0 requires protobuf<4, but you have protobuf 5.28.3 which is incompatible. s3fs 2024.10.0 requires fsspec==2024.10.0.*, but you have fsspec 2024.9.0 which is incompatible. - tensorflow 2.12.0 requires protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3, but you have protobuf 5.28.3 which is incompatible. - tensorflow-metadata 1.14.0 requires protobuf<4.21,>=3.20.3, but you have protobuf 5.28.3 which is incompatible. - wandb 0.15.4 requires protobuf!=4.21.0,<5,>=3.12.0; python_version < "3.9" and sys_platform == "linux", but you have protobuf 5.28.3 which is incompatible. Note: you may need to restart the kernel to use updated packages. @@ -85,10 +80,10 @@ Load and run the original pipeline .. parsed-literal:: - 2024-10-23 04:56:18.757348: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 04:56:18.791430: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 05:00:13.157963: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 05:00:13.192172: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 04:56:19.346484: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 05:00:13.880563: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -257,11 +252,12 @@ here, we always use fixed shapes in conversion by using an .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4779: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:88: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:88: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input_shape[-1] > 1 or self.sliding_window is not None: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: @@ -310,7 +306,7 @@ here, we always use fixed shapes in conversion by using an .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unets/unet_stable_cascade.py:548: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unets/unet_stable_cascade.py:548: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if skip is not None and (x.size(-1) != skip.size(-1) or x.size(-2) != skip.size(-2)): diff --git a/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.jpg b/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.jpg index ff7c5a51fa8623..240c88c559da36 100644 --- a/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.jpg +++ b/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.jpg @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8ea7494f5ec5eee5a399e31abce1aafc5b2df1e446414a4c8ece593b465608e2 -size 81605 +oid sha256:8f0845e937cadada4b26550b58858292fd42eebd18367b454b1401143e263338 +size 85254 diff --git a/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.png b/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.png index 81e228f915393f..349c8fa79af81f 100644 --- a/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.png +++ b/docs/notebooks/stable-cascade-image-generation-with-output_files/stable-cascade-image-generation-with-output_29_2.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:33518ccf125cee029c73380add209aae63208485ac340655ac3123d0e19c0426 -size 1629985 +oid sha256:d44cddd44f25458a49a6bf4a70431bbd39eecdc67d015c2c88aafa644f7c8ff9 +size 1644717 diff --git a/docs/notebooks/stable-diffusion-ip-adapter-with-output.rst b/docs/notebooks/stable-diffusion-ip-adapter-with-output.rst index 9f84e3178267ff..15c490942998d6 100644 --- a/docs/notebooks/stable-diffusion-ip-adapter-with-output.rst +++ b/docs/notebooks/stable-diffusion-ip-adapter-with-output.rst @@ -75,8 +75,6 @@ Prerequisites ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. botocore 1.35.36 requires urllib3<1.27,>=1.25.4; python_version < "3.10", but you have urllib3 2.2.3 which is incompatible. - tensorflow 2.12.0 requires protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3, but you have protobuf 5.28.3 which is incompatible. - wandb 0.15.4 requires protobuf!=4.21.0,<5,>=3.12.0; python_version < "3.9" and sys_platform == "linux", but you have protobuf 5.28.3 which is incompatible. Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -197,10 +195,10 @@ Additionally, LCM requires using LCMScheduler for efficient generation. .. parsed-literal:: - 2024-10-23 05:07:50.575524: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 05:07:50.610025: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 05:11:37.931642: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 05:11:37.965522: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 05:07:51.160950: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 05:11:38.628517: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -211,7 +209,7 @@ Additionally, LCM requires using LCMScheduler for efficient generation. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/feature_extraction_clip.py:28: FutureWarning: The class CLIPFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please use CLIPImageProcessor instead. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/feature_extraction_clip.py:28: FutureWarning: The class CLIPFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please use CLIPImageProcessor instead. warnings.warn( @@ -293,8 +291,11 @@ extractor as input and returns image embeddings. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4779: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:243: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + if not interpolate_pos_encoding and (height != self.image_size or width != self.image_size): U-net @@ -355,17 +356,17 @@ Model predicts the ``sample`` state for the next step. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unets/unet_2d_condition.py:1111: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/unets/unet_2d_condition.py:1111: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if dim % default_overall_up_factor != 0: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/embeddings.py:1801: FutureWarning: You have passed a tensor as `image_embeds`.This is deprecated and will be removed in a future release. Please make sure to update your script to pass `image_embeds` as a list of tensors to suppress this warning. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/embeddings.py:1801: FutureWarning: You have passed a tensor as `image_embeds`.This is deprecated and will be removed in a future release. Please make sure to update your script to pass `image_embeds` as a list of tensors to suppress this warning. deprecate("image_embeds not a list", "1.0.0", deprecation_message, standard_warn=False) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:136: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:136: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:145: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/downsampling.py:145: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:147: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:147: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if hidden_states.shape[0] >= 64: @@ -443,16 +444,16 @@ image in pipeline, we can discuss it in inference examples. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes: - %2506 : Float(1, 4, 64, 64, strides=[16384, 4096, 64, 1], requires_grad=0, device=cpu) = aten::randn(%2500, %2501, %2502, %2503, %2504, %2505) # /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/torch_utils.py:81:0 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes: + %2506 : Float(1, 4, 64, 64, strides=[16384, 4096, 64, 1], requires_grad=0, device=cpu) = aten::randn(%2500, %2501, %2502, %2503, %2504, %2505) # /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/torch_utils.py:81:0 This may cause errors in trace checking. To disable trace checking, pass check_trace=False to torch.jit.trace() _check_trace( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1303: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: Tensor-likes are not close! - Mismatched elements: 10395 / 16384 (63.4%) - Greatest absolute difference: 0.0018606185913085938 at index (0, 1, 63, 63) (up to 1e-05 allowed) - Greatest relative difference: 0.005050937208143127 at index (0, 3, 63, 59) (up to 1e-05 allowed) + Mismatched elements: 10398 / 16384 (63.5%) + Greatest absolute difference: 0.0014954805374145508 at index (0, 1, 0, 63) (up to 1e-05 allowed) + Greatest relative difference: 0.004146566421115057 at index (0, 3, 63, 59) (up to 1e-05 allowed) _check_trace( @@ -498,9 +499,9 @@ hidden states. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:88: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:88: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input_shape[-1] > 1 or self.sliding_window is not None: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:164: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: diff --git a/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_22_1.png b/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_22_1.png index 9e41d1a02b25ee..b202db52895198 100644 --- a/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_22_1.png +++ b/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_22_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0d5fa26a0d01a01557d2baf61229065adafcf06e97d796f49cad83cfeef8c42e -size 964520 +oid sha256:f9454f039912f132f1c278b4d58fddc6b737c8a6d56ddbc5358142ad1e9f987c +size 980237 diff --git a/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_25_0.png b/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_25_0.png index 45f0dc6e8e9e10..299ec38e04f457 100644 --- a/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_25_0.png +++ b/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_25_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3b0f7041fd5c84ecde36f1036e957a5f64eb12507270cd300392042c8a703f6d -size 961384 +oid sha256:8e9b89ad55b5255e86b4a7e7d40984fa3e8ca7d64dc5cff8ca1367dcc0de5a72 +size 927436 diff --git a/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_28_0.png b/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_28_0.png index 8b9cea0390f80a..9f1e4c4d220755 100644 --- a/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_28_0.png +++ b/docs/notebooks/stable-diffusion-ip-adapter-with-output_files/stable-diffusion-ip-adapter-with-output_28_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e434bcc6a4403efc3f9272a49882d42ea354dae2de6231c7d71458ea79fd6de9 -size 593289 +oid sha256:46120a84ef8334f29241e07bd43c8313977cd2318ce36a951f1f8ed3c626ba84 +size 598424 diff --git a/docs/notebooks/stable-diffusion-torchdynamo-backend-with-output.rst b/docs/notebooks/stable-diffusion-torchdynamo-backend-with-output.rst index e03a4ab614c769..151a5a5c30588d 100644 --- a/docs/notebooks/stable-diffusion-torchdynamo-backend-with-output.rst +++ b/docs/notebooks/stable-diffusion-torchdynamo-backend-with-output.rst @@ -117,8 +117,7 @@ torch.compile it goes through the following steps: 1. Graph acquisition - the model is rewritten as blocks of subgraphs that are either: - compiled by TorchDynamo and “flattened”, - - falling back to the eager-mode, due to unsupported Python constructs (like control-flow - code). + - falling back to the eager-mode, due to unsupported Python constructs (like control-flow code). 2. Graph lowering - all PyTorch operations are decomposed into their constituent kernels specific to the chosen backend. diff --git a/docs/notebooks/style-transfer-with-output.rst b/docs/notebooks/style-transfer-with-output.rst index dbbe6395574657..9b902aeb79f7a7 100644 --- a/docs/notebooks/style-transfer-with-output.rst +++ b/docs/notebooks/style-transfer-with-output.rst @@ -193,7 +193,7 @@ OpenVINO Intermediate Representation (IR) with ``FP16`` precision. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/style-transfer-webcam/model/mosaic-9.onnx') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/style-transfer-webcam/model/mosaic-9.onnx') diff --git a/docs/notebooks/style-transfer-with-output_files/style-transfer-with-output_25_0.png b/docs/notebooks/style-transfer-with-output_files/style-transfer-with-output_25_0.png index d9cfe3e6360a7a..69ebcfdd60f395 100644 --- a/docs/notebooks/style-transfer-with-output_files/style-transfer-with-output_25_0.png +++ b/docs/notebooks/style-transfer-with-output_files/style-transfer-with-output_25_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0565650551a7561fb5450ec915c747ffad759c8cba258eec1d1cd2d30f85c812 -size 81718 +oid sha256:0aa8c4fc4dd201212c4b63e6436c2a5e9cd6f7994271244bfabe91a5cef7e675 +size 81862 diff --git a/docs/notebooks/table-question-answering-with-output.rst b/docs/notebooks/table-question-answering-with-output.rst index 88edfa22333b86..e1cad24a567f4b 100644 --- a/docs/notebooks/table-question-answering-with-output.rst +++ b/docs/notebooks/table-question-answering-with-output.rst @@ -80,10 +80,10 @@ Prerequisites .. parsed-literal:: - 2024-10-23 05:11:15.997414: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 05:11:16.031769: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 05:15:04.432298: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 05:15:04.467903: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 05:11:16.587915: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 05:15:05.158075: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Use ``TapasForQuestionAnswering.from_pretrained`` to download a @@ -219,57 +219,58 @@ function to serialize the result of conversion. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4779: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:5006: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1571: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1571: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. self.indices = torch.as_tensor(indices) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1572: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1572: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. self.num_segments = torch.as_tensor(num_segments, device=indices.device) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1674: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1674: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. batch_size = torch.prod(torch.tensor(list(index.batch_shape()))) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1750: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1750: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. [torch.as_tensor([-1], dtype=torch.long), torch.as_tensor(vector_shape, dtype=torch.long)], dim=0 - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1753: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1753: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! flat_values = values.reshape(flattened_shape.tolist()) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1755: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1755: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! out = torch.zeros(int(flat_index.num_segments), dtype=torch.float, device=flat_values.device) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1763: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1763: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor(index.batch_shape(), dtype=torch.long), - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1764: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1764: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor([index.num_segments], dtype=torch.long), - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1765: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1765: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor(vector_shape, dtype=torch.long), - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1770: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1770: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! output_values = segment_means.clone().view(new_shape.tolist()).to(values.dtype) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1701: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1701: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. batch_shape = torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1705: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1705: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. num_segments = torch.as_tensor(num_segments) # create a rank 0 tensor (scalar) containing num_segments (e.g. 64) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1716: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1716: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! new_shape = [int(x) for x in new_tensor.tolist()] - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1719: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1719: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. multiples = torch.cat([batch_shape, torch.as_tensor([1])], dim=0) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1720: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1720: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! indices = indices.repeat(multiples.tolist()) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:282: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:282: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor(self.config.max_position_embeddings - 1, device=device), position - first_position - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1231: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1231: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. indices=torch.min(row_ids, torch.as_tensor(self.config.max_num_rows - 1, device=row_ids.device)), - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1236: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1236: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. indices=torch.min(column_ids, torch.as_tensor(self.config.max_num_columns - 1, device=column_ids.device)), - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1928: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1928: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. column_logits += CLOSE_ENOUGH_TO_LOG_ZERO * torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1933: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1933: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. column_logits += CLOSE_ENOUGH_TO_LOG_ZERO * torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1969: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1969: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. labels_per_column, _ = reduce_sum(torch.as_tensor(labels, dtype=torch.float32, device=labels.device), col_index) - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1992: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1992: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. torch.as_tensor(labels, dtype=torch.long, device=labels.device), cell_index - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1999: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:1999: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. column_mask = torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2024: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2024: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. selected_column_id = torch.as_tensor( - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2029: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/tapas/modeling_tapas.py:2029: TracerWarning: torch.as_tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. selected_column_mask = torch.as_tensor( diff --git a/docs/notebooks/tensorflow-classification-to-openvino-with-output.rst b/docs/notebooks/tensorflow-classification-to-openvino-with-output.rst index b2efda5f0aa31a..45eb4a96573be1 100644 --- a/docs/notebooks/tensorflow-classification-to-openvino-with-output.rst +++ b/docs/notebooks/tensorflow-classification-to-openvino-with-output.rst @@ -338,5 +338,5 @@ performance. .. parsed-literal:: - IR model in OpenVINO Runtime/CPU: 0.0010 seconds per image, FPS: 995.15 + IR model in OpenVINO Runtime/CPU: 0.0010 seconds per image, FPS: 959.64 diff --git a/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output.rst b/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output.rst index 0c0357fa707f9c..2410d1657c9b1c 100644 --- a/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output.rst +++ b/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output.rst @@ -391,7 +391,7 @@ Read the image, resize and convert it to the input shape of the network: .. parsed-literal:: - + diff --git a/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output_files/tensorflow-instance-segmentation-to-openvino-with-output_39_0.png b/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output_files/tensorflow-instance-segmentation-to-openvino-with-output_39_0.png index d91e735a1d6467..4940e220329fbf 100644 --- a/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output_files/tensorflow-instance-segmentation-to-openvino-with-output_39_0.png +++ b/docs/notebooks/tensorflow-instance-segmentation-to-openvino-with-output_files/tensorflow-instance-segmentation-to-openvino-with-output_39_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8492fcb14f954229b35321f809f600bc9c9fb445cf44cf1cab1dbae5473e2d78 -size 391367 +oid sha256:69ea6c54eefea173c120ff5666caa829b5a008727ebd3a83f7e10d1986de07bd +size 394182 diff --git a/docs/notebooks/tensorflow-object-detection-to-openvino-with-output.rst b/docs/notebooks/tensorflow-object-detection-to-openvino-with-output.rst index 970ff5dd5278ae..11dc1675dcfeb4 100644 --- a/docs/notebooks/tensorflow-object-detection-to-openvino-with-output.rst +++ b/docs/notebooks/tensorflow-object-detection-to-openvino-with-output.rst @@ -189,7 +189,7 @@ from TensorFlow Hub: .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-object-detection-to-openvino/od-model/tf/faster_rcnn_resnet50_v1_640x640.tar.gz') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-object-detection-to-openvino/od-model/tf/faster_rcnn_resnet50_v1_640x640.tar.gz') @@ -368,7 +368,7 @@ Load and save an image: .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-object-detection-to-openvino/data/coco_bike.jpg') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-object-detection-to-openvino/data/coco_bike.jpg') @@ -396,7 +396,7 @@ Read the image, resize and convert it to the input shape of the network: .. parsed-literal:: - + @@ -697,7 +697,7 @@ Zoo `__: .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-object-detection-to-openvino/data/coco_91cl.txt') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-object-detection-to-openvino/data/coco_91cl.txt') diff --git a/docs/notebooks/tensorflow-object-detection-to-openvino-with-output_files/tensorflow-object-detection-to-openvino-with-output_38_0.png b/docs/notebooks/tensorflow-object-detection-to-openvino-with-output_files/tensorflow-object-detection-to-openvino-with-output_38_0.png index e3868b416ee4e5..41c96a6eeb7d0e 100644 --- a/docs/notebooks/tensorflow-object-detection-to-openvino-with-output_files/tensorflow-object-detection-to-openvino-with-output_38_0.png +++ b/docs/notebooks/tensorflow-object-detection-to-openvino-with-output_files/tensorflow-object-detection-to-openvino-with-output_38_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dfdccf9cf18ab90165dbca452027d6afdd51d2f355b8cdc45d7431f9e88c5782 -size 392207 +oid sha256:63da618f6999e82c3f9e461cb912450186fbb00bc247ccd311a0dfd0bbc825b3 +size 390893 diff --git a/docs/notebooks/tensorflow-quantization-aware-training-with-output.rst b/docs/notebooks/tensorflow-quantization-aware-training-with-output.rst index 8fe9423b3dcc53..4a4285ca97cc9a 100644 --- a/docs/notebooks/tensorflow-quantization-aware-training-with-output.rst +++ b/docs/notebooks/tensorflow-quantization-aware-training-with-output.rst @@ -127,10 +127,10 @@ models will be stored. .. parsed-literal:: - 2024-10-23 05:14:10.399928: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 05:14:10.434579: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 05:17:59.724629: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 05:17:59.759583: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 05:14:11.041876: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 05:18:00.367471: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -140,7 +140,7 @@ models will be stored. Downloading data from https://storage.openvinotoolkit.org/repositories/nncf/openvino_notebook_ckpts/305_resnet18_imagenette_fp32_v1.h5 134604992/134604992 [==============================] - 2s 0us/step Absolute path where the model weights are saved: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-quantization-aware-training/model/ResNet-18_fp32.h5 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/tensorflow-quantization-aware-training/model/ResNet-18_fp32.h5 Dataset Preprocessing @@ -175,13 +175,13 @@ Download and prepare Imagenette 160px dataset. .. parsed-literal:: - 2024-10-23 05:14:16.300917: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. + 2024-11-05 05:18:05.378970: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform. Skipping registering GPU devices... - 2024-10-23 05:14:16.433616: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_2' with dtype string and shape [1] - [[{{node Placeholder/_2}}]] - 2024-10-23 05:14:16.434383: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_0' with dtype string and shape [1] - [[{{node Placeholder/_0}}]] - 2024-10-23 05:14:16.493317: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. + 2024-11-05 05:18:05.496251: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_3' with dtype int64 and shape [1] + [[{{node Placeholder/_3}}]] + 2024-11-05 05:18:05.496608: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_1' with dtype string and shape [1] + [[{{node Placeholder/_1}}]] + 2024-11-05 05:18:05.549189: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. @@ -325,15 +325,15 @@ model and a training pipeline. .. parsed-literal:: - 2024-10-23 05:14:17.636634: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_2' with dtype string and shape [1] - [[{{node Placeholder/_2}}]] - 2024-10-23 05:14:17.637423: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_3' with dtype int64 and shape [1] - [[{{node Placeholder/_3}}]] + 2024-11-05 05:18:06.706690: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_4' with dtype int64 and shape [1] + [[{{node Placeholder/_4}}]] + 2024-11-05 05:18:06.707393: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_0' with dtype string and shape [1] + [[{{node Placeholder/_0}}]] .. parsed-literal:: - 4/4 [==============================] - 1s 250ms/sample - loss: 0.9807 - acc@1: 0.8220 + 4/4 [==============================] - 1s 241ms/sample - loss: 0.9807 - acc@1: 0.8220 Accuracy of FP32 model: 0.822 @@ -379,13 +379,13 @@ scenario and requires only 3 modifications. .. parsed-literal:: - 2024-10-23 05:14:20.291889: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_2' with dtype string and shape [1] + 2024-11-05 05:18:09.325779: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_2' with dtype string and shape [1] [[{{node Placeholder/_2}}]] - 2024-10-23 05:14:20.292274: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_1' with dtype string and shape [1] - [[{{node Placeholder/_1}}]] - 2024-10-23 05:14:21.242391: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. - 2024-10-23 05:14:21.895899: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. - 2024-10-23 05:14:30.320766: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. + 2024-11-05 05:18:09.326170: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_0' with dtype string and shape [1] + [[{{node Placeholder/_0}}]] + 2024-11-05 05:18:10.310403: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. + 2024-11-05 05:18:10.931815: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. + 2024-11-05 05:18:19.063128: W tensorflow/core/kernels/data/cache_dataset_ops.cc:856] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead. Evaluate the new model on the validation set after initialization of @@ -411,7 +411,7 @@ demonstrated here. .. parsed-literal:: - 4/4 [==============================] - 1s 306ms/sample - loss: 0.9766 - acc@1: 0.8120 + 4/4 [==============================] - 1s 305ms/sample - loss: 0.9766 - acc@1: 0.8120 Fine-tune the Compressed Model @@ -446,10 +446,10 @@ training pipeline are required. Here is a simple example. Accuracy of INT8 model after initialization: 0.812 Epoch 1/2 - 101/101 [==============================] - 49s 418ms/step - loss: 0.7134 - acc@1: 0.9299 + 101/101 [==============================] - 49s 416ms/step - loss: 0.7134 - acc@1: 0.9299 Epoch 2/2 - 101/101 [==============================] - 42s 415ms/step - loss: 0.6807 - acc@1: 0.9489 - 4/4 [==============================] - 1s 144ms/sample - loss: 0.9760 - acc@1: 0.8160 + 101/101 [==============================] - 42s 414ms/step - loss: 0.6807 - acc@1: 0.9489 + 4/4 [==============================] - 1s 141ms/sample - loss: 0.9760 - acc@1: 0.8160 Accuracy of INT8 model after fine-tuning: 0.816 @@ -559,10 +559,10 @@ Please select a benchmarking device using the dropdown list: .. parsed-literal:: Benchmark FP32 model (IR) - [ INFO ] Throughput: 2786.08 FPS + [ INFO ] Throughput: 2758.24 FPS Benchmark INT8 model (IR) - [ INFO ] Throughput: 10924.61 FPS + [ INFO ] Throughput: 11317.45 FPS Show Device Information for reference. diff --git a/docs/notebooks/tflite-selfie-segmentation-with-output.rst b/docs/notebooks/tflite-selfie-segmentation-with-output.rst index 9acc434aa435f6..cc8a203e4f15a9 100644 --- a/docs/notebooks/tflite-selfie-segmentation-with-output.rst +++ b/docs/notebooks/tflite-selfie-segmentation-with-output.rst @@ -143,7 +143,7 @@ Download pretrained model and test image .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/tflite-selfie-segmentation/selfie_multiclass_256x256.tflite') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/tflite-selfie-segmentation/selfie_multiclass_256x256.tflite') diff --git a/docs/notebooks/tflite-selfie-segmentation-with-output_files/tflite-selfie-segmentation-with-output_33_0.png b/docs/notebooks/tflite-selfie-segmentation-with-output_files/tflite-selfie-segmentation-with-output_33_0.png index d0b2b849bf51c2..882a9d712dd7b0 100644 --- a/docs/notebooks/tflite-selfie-segmentation-with-output_files/tflite-selfie-segmentation-with-output_33_0.png +++ b/docs/notebooks/tflite-selfie-segmentation-with-output_files/tflite-selfie-segmentation-with-output_33_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e2963c9b4c7dacdca6b1bbe9ea3ef832a19ba962c0a9980363722083e66b7358 -size 14233 +oid sha256:2cda2d6f5f72e6e7c24898ead882b6b26b696d1d1c61ff589d08fd55d98357ef +size 14403 diff --git a/docs/notebooks/tflite-to-openvino-with-output.rst b/docs/notebooks/tflite-to-openvino-with-output.rst index dcd7a98897f334..a70e10853c0dc7 100644 --- a/docs/notebooks/tflite-to-openvino-with-output.rst +++ b/docs/notebooks/tflite-to-openvino-with-output.rst @@ -274,7 +274,7 @@ GPU. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 9.48 ms + [ INFO ] Read model took 10.03 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] images (node: images) : f32 / [...] / [1,224,224,3] @@ -288,7 +288,7 @@ GPU. [ INFO ] Model outputs: [ INFO ] Softmax (node: 61) : f32 / [...] / [1,1000] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 157.24 ms + [ INFO ] Compile model took 165.61 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: TensorFlow_Lite_Frontend_IR @@ -325,15 +325,15 @@ GPU. [ INFO ] Fill input 'images' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 7.14 ms + [ INFO ] First inference took 6.93 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 17436 iterations - [ INFO ] Duration: 15008.41 ms + [ INFO ] Count: 14982 iterations + [ INFO ] Duration: 15006.90 ms [ INFO ] Latency: - [ INFO ] Median: 4.99 ms - [ INFO ] Average: 5.03 ms - [ INFO ] Min: 2.84 ms - [ INFO ] Max: 24.60 ms - [ INFO ] Throughput: 1161.75 FPS + [ INFO ] Median: 5.47 ms + [ INFO ] Average: 5.91 ms + [ INFO ] Min: 3.02 ms + [ INFO ] Max: 34.80 ms + [ INFO ] Throughput: 998.34 FPS diff --git a/docs/notebooks/typo-detector-with-output.rst b/docs/notebooks/typo-detector-with-output.rst index 5c03dc870f8a69..94431f153ae72d 100644 --- a/docs/notebooks/typo-detector-with-output.rst +++ b/docs/notebooks/typo-detector-with-output.rst @@ -108,10 +108,10 @@ Imports .. parsed-literal:: - 2024-10-23 05:18:31.129817: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 05:18:31.163652: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 05:23:42.228206: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 05:23:42.262729: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 05:18:31.823748: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 05:23:42.928779: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT Methods @@ -255,8 +255,7 @@ your model. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/dynamic_graph/wrappers.py:86: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. - op1 = operator(\*args, \*\*kwargs) + `loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`. Load the tokenizer @@ -372,7 +371,7 @@ Let’s run a demo using the Hugging Face Optimum API. [Input]: I have been stuying for my math exam all week, but I'm stil not very confidet that I will pass it, because there are so many formuals to remeber. [Detected]: I have been stuying for my math exam all week, but I'm stil not very confidet that I will pass it, because there are so many formuals to remeber. ---------------------------------------------------------------------------------------------------------------------------------- - Time elapsed: 0.14796948432922363 + Time elapsed: 0.14881014823913574 2. Converting the model to OpenVINO IR @@ -618,5 +617,5 @@ Let’s run a demo using the converted OpenVINO IR model. [Input]: I have been stuying for my math exam all week, but I'm stil not very confidet that I will pass it, because there are so many formuals to remeber. [Detected]: I have been stuying for my math exam all week, but I'm stil not very confidet that I will pass it, because there are so many formuals to remeber. ---------------------------------------------------------------------------------------------------------------------------------- - Time elapsed: 0.10040116310119629 + Time elapsed: 0.10274600982666016 diff --git a/docs/notebooks/vision-background-removal-with-output.rst b/docs/notebooks/vision-background-removal-with-output.rst index 63e02d9c787aaf..4e79038a213347 100644 --- a/docs/notebooks/vision-background-removal-with-output.rst +++ b/docs/notebooks/vision-background-removal-with-output.rst @@ -195,7 +195,7 @@ next cell loads the model and the pre-trained weights. Downloading... From: https://drive.google.com/uc?id=1W8E4FHIlTVstfRkYmNOjbr0VDXTZm0jD To: <_io.BufferedWriter name='model/u2net_lite/u2net_lite.pth'> - 100%|██████████| 4.68M/4.68M [00:00<00:00, 34.0MB/s] + 100%|██████████| 4.68M/4.68M [00:00<00:00, 33.7MB/s] .. parsed-literal:: @@ -225,7 +225,7 @@ next cell loads the model and the pre-trained weights. .. parsed-literal:: - /tmp/ipykernel_2664614/1036642300.py:7: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /tmp/ipykernel_586189/1036642300.py:7: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. net.load_state_dict(state_dict=torch.load(model_path, map_location="cpu")) @@ -252,7 +252,7 @@ OpenVINO IR format. Executing the following command may take a while. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/vision-background-removal/model/u2net.py:23: UserWarning: `nn.functional.upsample` is deprecated. Use `nn.functional.interpolate` instead. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/vision-background-removal/model/u2net.py:23: UserWarning: `nn.functional.upsample` is deprecated. Use `nn.functional.interpolate` instead. src = F.upsample(src,size=tar.shape[2:],mode='bilinear') @@ -341,7 +341,7 @@ Load the OpenVINO IR model to OpenVINO Runtime and do inference. .. parsed-literal:: - Inference finished. Inference time: 0.107 seconds, FPS: 9.31. + Inference finished. Inference time: 0.106 seconds, FPS: 9.42. Visualize Results diff --git a/docs/notebooks/vision-monodepth-with-output.rst b/docs/notebooks/vision-monodepth-with-output.rst index ea81314fe49df7..dfb20643e991e7 100644 --- a/docs/notebooks/vision-monodepth-with-output.rst +++ b/docs/notebooks/vision-monodepth-with-output.rst @@ -309,7 +309,7 @@ original image shape. .. parsed-literal:: - /tmp/ipykernel_2664839/2076527990.py:15: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead. + /tmp/ipykernel_586416/2076527990.py:15: MatplotlibDeprecationWarning: The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead. cmap = matplotlib.cm.get_cmap(colormap) @@ -508,7 +508,7 @@ Do Inference on a Video and Create Monodepth Video .. parsed-literal:: - Processed 60 frames in 24.30 seconds. Total FPS (including video processing): 2.47.Inference FPS: 46.34 + Processed 60 frames in 25.55 seconds. Total FPS (including video processing): 2.35.Inference FPS: 48.61 Monodepth Video saved to 'output/Coco%20Walking%20in%20Berkeley_monodepth.mp4'. @@ -535,7 +535,7 @@ Display Monodepth Video .. parsed-literal:: Showing monodepth video saved at - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/vision-monodepth/output/Coco%20Walking%20in%20Berkeley_monodepth.mp4 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/vision-monodepth/output/Coco%20Walking%20in%20Berkeley_monodepth.mp4 If you cannot see the video in your browser, please click on the following link to download the video diff --git a/docs/notebooks/wav2lip-with-output.rst b/docs/notebooks/wav2lip-with-output.rst index 3f6b510cf00374..c70015d2614d8d 100644 --- a/docs/notebooks/wav2lip-with-output.rst +++ b/docs/notebooks/wav2lip-with-output.rst @@ -140,7 +140,7 @@ Download example files. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/data_video_sun_5s.mp4') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/data_video_sun_5s.mp4') @@ -177,7 +177,7 @@ and will convert both model in OpenVINO format. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/ov_wav2lip_helper.py:43: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/ov_wav2lip_helper.py:43: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. model_weights = torch.load(path_to_detector) @@ -200,7 +200,7 @@ and will convert both model in OpenVINO format. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/ov_wav2lip_helper.py:16: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/ov_wav2lip_helper.py:16: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage) @@ -263,7 +263,7 @@ python API and converted OpenVINO models. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/Wav2Lip/audio.py:100: FutureWarning: Pass sr=16000, n_fft=800 as keyword args. From version 0.10 passing these as positional arguments will result in an error + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/wav2lip/Wav2Lip/audio.py:100: FutureWarning: Pass sr=16000, n_fft=800 as keyword args. From version 0.10 passing these as positional arguments will result in an error return librosa.filters.mel(hp.sample_rate, hp.n_fft, n_mels=hp.num_mels, @@ -286,14 +286,14 @@ python API and converted OpenVINO models. 0%| | 0/8 [00:00 #0:0 (mpeg4 (native) -> h264 (libx264)) Stream #0:0 -> #0:1 (pcm_s16le (native) -> aac (native)) Press [q] to stop, [?] for help - [libx264 @ 0x56537dc66840] -qscale is ignored, -crf is recommended. - [libx264 @ 0x56537dc66840] using SAR=1/1 - [libx264 @ 0x56537dc66840] using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2 AVX FMA3 BMI2 AVX2 AVX512 - [libx264 @ 0x56537dc66840] profile High, level 3.1 - [libx264 @ 0x56537dc66840] 264 - core 155 r2917 0a84d98 - H.264/MPEG-4 AVC codec - Copyleft 2003-2018 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=24 lookahead_threads=4 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=0 weightp=2 keyint=250 keyint_min=25 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=23.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00 + [libx264 @ 0x55edc3fbd840] -qscale is ignored, -crf is recommended. + [libx264 @ 0x55edc3fbd840] using SAR=1/1 + [libx264 @ 0x55edc3fbd840] using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2 AVX FMA3 BMI2 AVX2 AVX512 + [libx264 @ 0x55edc3fbd840] profile High, level 3.1 + [libx264 @ 0x55edc3fbd840] 264 - core 155 r2917 0a84d98 - H.264/MPEG-4 AVC codec - Copyleft 2003-2018 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=24 lookahead_threads=4 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=0 weightp=2 keyint=250 keyint_min=25 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=23.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00 Output #0, mp4, to 'results/result_voice.mp4': Metadata: encoder : Lavf58.29.100 @@ -349,25 +349,25 @@ python API and converted OpenVINO models. encoder : Lavc58.54.100 aac frame= 123 fps=0.0 q=-1.0 Lsize= 621kB time=00:00:05.06 bitrate=1005.8kbits/s speed=10.8x video:573kB audio:43kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.827166% - [libx264 @ 0x56537dc66840] frame I:1 Avg QP:22.24 size: 31028 - [libx264 @ 0x56537dc66840] frame P:75 Avg QP:22.01 size: 6954 - [libx264 @ 0x56537dc66840] frame B:47 Avg QP:25.58 size: 718 - [libx264 @ 0x56537dc66840] consecutive B-frames: 38.2% 27.6% 14.6% 19.5% - [libx264 @ 0x56537dc66840] mb I I16..4: 14.0% 83.9% 2.1% - [libx264 @ 0x56537dc66840] mb P I16..4: 1.3% 3.3% 0.1% P16..4: 37.8% 8.2% 6.4% 0.0% 0.0% skip:43.0% - [libx264 @ 0x56537dc66840] mb B I16..4: 0.2% 0.7% 0.0% B16..8: 27.9% 0.4% 0.1% direct: 0.2% skip:70.6% L0:43.9% L1:54.2% BI: 1.9% - [libx264 @ 0x56537dc66840] 8x8 transform intra:73.3% inter:77.1% - [libx264 @ 0x56537dc66840] coded y,uvDC,uvAC intra: 56.9% 72.4% 8.1% inter: 11.4% 13.0% 0.2% - [libx264 @ 0x56537dc66840] i16 v,h,dc,p: 20% 23% 9% 48% - [libx264 @ 0x56537dc66840] i8 v,h,dc,ddl,ddr,vr,hd,vl,hu: 25% 23% 36% 3% 3% 2% 2% 3% 3% - [libx264 @ 0x56537dc66840] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu: 39% 14% 14% 4% 6% 7% 4% 9% 3% - [libx264 @ 0x56537dc66840] i8c dc,h,v,p: 42% 25% 29% 4% - [libx264 @ 0x56537dc66840] Weighted P-Frames: Y:0.0% UV:0.0% - [libx264 @ 0x56537dc66840] ref P L0: 74.2% 10.4% 11.1% 4.3% - [libx264 @ 0x56537dc66840] ref B L0: 86.1% 11.2% 2.8% - [libx264 @ 0x56537dc66840] ref B L1: 98.3% 1.7% - [libx264 @ 0x56537dc66840] kb/s:953.36 - [aac @ 0x56537dc68140] Qavg: 121.673 + [libx264 @ 0x55edc3fbd840] frame I:1 Avg QP:22.24 size: 31028 + [libx264 @ 0x55edc3fbd840] frame P:75 Avg QP:22.01 size: 6954 + [libx264 @ 0x55edc3fbd840] frame B:47 Avg QP:25.58 size: 718 + [libx264 @ 0x55edc3fbd840] consecutive B-frames: 38.2% 27.6% 14.6% 19.5% + [libx264 @ 0x55edc3fbd840] mb I I16..4: 14.0% 83.9% 2.1% + [libx264 @ 0x55edc3fbd840] mb P I16..4: 1.3% 3.3% 0.1% P16..4: 37.8% 8.2% 6.4% 0.0% 0.0% skip:43.0% + [libx264 @ 0x55edc3fbd840] mb B I16..4: 0.2% 0.7% 0.0% B16..8: 27.9% 0.4% 0.1% direct: 0.2% skip:70.6% L0:43.9% L1:54.2% BI: 1.9% + [libx264 @ 0x55edc3fbd840] 8x8 transform intra:73.3% inter:77.1% + [libx264 @ 0x55edc3fbd840] coded y,uvDC,uvAC intra: 56.9% 72.4% 8.1% inter: 11.4% 13.0% 0.2% + [libx264 @ 0x55edc3fbd840] i16 v,h,dc,p: 20% 23% 9% 48% + [libx264 @ 0x55edc3fbd840] i8 v,h,dc,ddl,ddr,vr,hd,vl,hu: 25% 23% 36% 3% 3% 2% 2% 3% 3% + [libx264 @ 0x55edc3fbd840] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu: 39% 14% 14% 4% 6% 7% 4% 9% 3% + [libx264 @ 0x55edc3fbd840] i8c dc,h,v,p: 42% 25% 29% 4% + [libx264 @ 0x55edc3fbd840] Weighted P-Frames: Y:0.0% UV:0.0% + [libx264 @ 0x55edc3fbd840] ref P L0: 74.2% 10.4% 11.1% 4.3% + [libx264 @ 0x55edc3fbd840] ref B L0: 86.1% 11.2% 2.8% + [libx264 @ 0x55edc3fbd840] ref B L1: 98.3% 1.7% + [libx264 @ 0x55edc3fbd840] kb/s:953.36 + [aac @ 0x55edc3fbf140] Qavg: 121.673 diff --git a/docs/notebooks/whisper-asr-genai-with-output.rst b/docs/notebooks/whisper-asr-genai-with-output.rst index 45cc8291b8acc2..3f4339e01b3acc 100644 --- a/docs/notebooks/whisper-asr-genai-with-output.rst +++ b/docs/notebooks/whisper-asr-genai-with-output.rst @@ -75,8 +75,8 @@ Prerequisites .. code:: ipython3 - %pip install -q "transformers>=4.35" "torch>=2.3" "torchvision>=0.18.1" "onnx>=1.16.1" --extra-index-url https://download.pytorch.org/whl/cpu - %pip install -q "git+https://github.com/huggingface/optimum-intel.git" + %pip install -q "torch>=2.3" "torchvision>=0.18.1" --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q "transformers>=4.45" "git+https://github.com/huggingface/optimum-intel.git" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q --pre -U "openvino" "openvino-tokenizers" "openvino-genai" --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly %pip install -q datasets "gradio>=4.0" "soundfile>=0.12" "librosa" "python-ffmpeg<=1.0.16" %pip install -q "nncf>=2.13.0" "jiwer" @@ -91,6 +91,12 @@ Prerequisites url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", ) open("notebook_utils.py", "w").write(r.text) + + if not Path("cmd_helper.py").exists(): + r = requests.get( + url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/cmd_helper.py", + ) + open("cmd_helper.py", "w").write(r.text) Load PyTorch model ------------------ @@ -266,7 +272,7 @@ Let’s check how to work the ``transcribe`` task. .. parsed-literal:: - /home/labuser/work/notebook/genai_whisper/lib/python3.10/site-packages/transformers/models/whisper/generation_whisper.py:496: FutureWarning: The input name `inputs` is deprecated. Please make sure to use `input_features` instead. + /home/labuser/work/notebook/whisper_new/lib/python3.10/site-packages/transformers/models/whisper/generation_whisper.py:496: FutureWarning: The input name `inputs` is deprecated. Please make sure to use `input_features` instead. warnings.warn( @@ -349,8 +355,8 @@ be found in the `paper `__. Result: The blog is our tool that is prefilled to encourage collaboration and develop the learning of the students and to attract a normal school class. -Download and convert model to OpenVINO IR via Optimum Intel CLI ---------------------------------------------------------------- +Convert model to OpenVINO IR via Optimum Intel CLI +-------------------------------------------------- @@ -381,20 +387,13 @@ documentation `__. import logging import nncf - import os - from IPython.display import display, Markdown + from cmd_helper import optimum_cli nncf.set_log_level(logging.ERROR) model_path = Path(model_id.value.split("/")[1]) - export_command = f"optimum-cli export openvino --model {model_id.value} --library transformers --task automatic-speech-recognition-with-past --framework pt {str(model_path)}" - - display(Markdown("**Export command:**")) - display(Markdown(f"`{export_command}`")) - - exit_code = os.system(export_command) - if exit_code != 0: - raise Exception("Failed to load and convert model!") + optimum_cli(model_id.value, model_path) + print(f"✅ {model_id.value} model converted and can be found in {model_path}") Run inference OpenVINO model with WhisperPipeline ------------------------------------------------- @@ -604,9 +603,9 @@ Compare performance PyTorch vs OpenVINO .. parsed-literal:: - Mean torch openai/whisper-tiny generation time: 0.624s - Mean openvino openai/whisper-tiny generation time: 0.344s - Performance openai/whisper-tiny openvino speedup: 1.814 + Mean torch openai/whisper-tiny generation time: 0.564s + Mean openvino openai/whisper-tiny generation time: 0.311s + Performance openai/whisper-tiny openvino speedup: 1.815 Quantization @@ -934,7 +933,7 @@ for Word Error Rate. .. parsed-literal:: - Whole pipeline performance speedup: 1.499 + Whole pipeline performance speedup: 1.350 Whisper transcription word accuracy. Original model: 82.88%. Quantized model: 84.13%. Accuracy drop: -1.25%. @@ -960,7 +959,7 @@ upload button) or record using your microphone. pipe = ov_quantized_pipe if to_quantize.value else ov_pipe - gr_pipeline = GradioPipeline(pipe, multilingual=(not model_id.value.endswith(".en")), quantized=to_quantize.value) + gr_pipeline = GradioPipeline(pipe, model_id.value, quantized=to_quantize.value) demo = make_demo(gr_pipeline) diff --git a/docs/notebooks/whisper-subtitles-generation-with-output.rst b/docs/notebooks/whisper-subtitles-generation-with-output.rst index 8bfbfa8a6e86a3..b1e16ffa40c626 100644 --- a/docs/notebooks/whisper-subtitles-generation-with-output.rst +++ b/docs/notebooks/whisper-subtitles-generation-with-output.rst @@ -80,7 +80,7 @@ Install dependencies. %pip install -q "nncf>=2.13.0" %pip install -q --pre -U "openvino" "openvino-tokenizers" "openvino-genai" --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly - %pip install -q "python-ffmpeg<=1.0.16" "ffmpeg" "moviepy" "onnx!=1.16.2" "git+https://github.com/huggingface/optimum-intel.git" "torch>=2.1" --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q "python-ffmpeg<=1.0.16" "ffmpeg" "moviepy" "transformers>=4.45" "git+https://github.com/huggingface/optimum-intel.git" "torch>=2.1" --extra-index-url https://download.pytorch.org/whl/cpu %pip install -q -U "yt_dlp>=2024.8.6" soundfile librosa jiwer %pip install -q "gradio>=4.19" @@ -94,6 +94,12 @@ Install dependencies. url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", ) open("notebook_utils.py", "w").write(r.text) + + if not Path("cmd_helper.py").exists(): + r = requests.get( + url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/cmd_helper.py", + ) + open("cmd_helper.py", "w").write(r.text) Instantiate model ----------------- @@ -184,10 +190,12 @@ documentation `__. .. code:: ipython3 + from cmd_helper import optimum_cli + model_dir = model_id.value.split("/")[-1] if not Path(model_dir).exists(): - !optimum-cli export openvino -m {model_id.value} {model_dir} --weight-format fp16 + optimum_cli(model_id.value, model_dir) Prepare inference pipeline -------------------------- @@ -218,13 +226,6 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 - import requests - - r = requests.get( - url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", - ) - open("notebook_utils.py", "w").write(r.text) - from notebook_utils import device_widget device = device_widget(default="CPU", exclude=["NPU"]) diff --git a/docs/notebooks/wuerstchen-image-generation-with-output.rst b/docs/notebooks/wuerstchen-image-generation-with-output.rst index 342ddbafdc7edb..237bfb8b4bd943 100644 --- a/docs/notebooks/wuerstchen-image-generation-with-output.rst +++ b/docs/notebooks/wuerstchen-image-generation-with-output.rst @@ -229,7 +229,8 @@ parameter to generate a less memory-demanding model. Text encoder model has 2 inputs: -- ``input_ids``: vector of tokenized input sentence. Default tokenizer vector length is 77. +- ``input_ids``: vector of tokenized + input sentence. Default tokenizer vector length is 77. - ``attention_mask``: vector of same length as ``input_ids`` describing the attention mask. @@ -285,6 +286,7 @@ Decoder pipeline Decoder pipeline consists of 3 parts: decoder, text encoder and VQGAN. Decoder model is the WuerstchenDiffNeXt UNet decoder. Inputs are: + - ``x``: sample - ``r``: timestep - ``effnet``: interpolation block diff --git a/docs/notebooks/yolov11-instance-segmentation-with-output.rst b/docs/notebooks/yolov11-instance-segmentation-with-output.rst index 9d9b5e6bd15c0e..9cbca9fe4f1c37 100644 --- a/docs/notebooks/yolov11-instance-segmentation-with-output.rst +++ b/docs/notebooks/yolov11-instance-segmentation-with-output.rst @@ -143,7 +143,7 @@ Import required utility functions. The lower cell will download the .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg') @@ -216,14 +216,14 @@ Let us consider the examples: .. parsed-literal:: - 100%|██████████| 5.90M/5.90M [00:00<00:00, 25.1MB/s] + 100%|██████████| 5.90M/5.90M [00:00<00:00, 25.2MB/s] .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 480x640 3 bicycles, 2 cars, 1 motorcycle, 1 dog, 66.4ms - Speed: 1.8ms preprocess, 66.4ms inference, 2.8ms postprocess per image at shape (1, 3, 480, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 480x640 3 bicycles, 2 cars, 1 motorcycle, 1 dog, 69.6ms + Speed: 2.0ms preprocess, 69.6ms inference, 4.8ms postprocess per image at shape (1, 3, 480, 640) @@ -260,7 +260,7 @@ preserve dynamic shapes in the model. OpenVINO: export success ✅ 2.0s, saved as 'yolo11n-seg_openvino_model/' (6.0 MB) Export complete (2.2s) - Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization + Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization Predict: yolo predict task=segment model=yolo11n-seg_openvino_model imgsz=640 half Validate: yolo val task=segment model=yolo11n-seg_openvino_model imgsz=640 data=/ultralytics/ultralytics/cfg/datasets/coco.yaml half Visualize: https://netron.app @@ -345,8 +345,8 @@ Test on single image .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 3 bicycles, 2 cars, 1 dog, 23.2ms - Speed: 3.6ms preprocess, 23.2ms inference, 3.8ms postprocess per image at shape (1, 3, 640, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 3 bicycles, 2 cars, 1 dog, 23.2ms + Speed: 1.8ms preprocess, 23.2ms inference, 3.8ms postprocess per image at shape (1, 3, 640, 640) @@ -668,8 +668,8 @@ on the image. .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 2 bicycles, 2 cars, 1 dog, 17.7ms - Speed: 2.1ms preprocess, 17.7ms inference, 3.8ms postprocess per image at shape (1, 3, 640, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 2 bicycles, 2 cars, 1 dog, 11.2ms + Speed: 1.9ms preprocess, 11.2ms inference, 3.4ms postprocess per image at shape (1, 3, 640, 640) @@ -728,7 +728,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 20.05 ms + [ INFO ] Read model took 19.61 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [?,3,?,?] @@ -738,7 +738,7 @@ models. [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [ INFO ] Reshaping model: 'x': [1,3,640,640] - [ INFO ] Reshape model took 8.82 ms + [ INFO ] Reshape model took 8.69 ms [Step 6/11] Configuring input of the model [ INFO ] Model inputs: [ INFO ] x (node: x) : u8 / [N,C,H,W] / [1,3,640,640] @@ -746,7 +746,7 @@ models. [ INFO ] ***NO_NAME*** (node: __module.model.23/aten::cat/Concat_8) : f32 / [...] / [1,116,8400] [ INFO ] input.255 (node: __module.model.23.cv4.2.1.act/aten::silu_/Swish_46) : f32 / [...] / [1,32,160,160] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 387.38 ms + [ INFO ] Compile model took 390.55 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -783,17 +783,17 @@ models. [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 36.42 ms + [ INFO ] First inference took 36.99 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 1788 iterations - [ INFO ] Duration: 15050.22 ms + [ INFO ] Count: 1806 iterations + [ INFO ] Duration: 15074.73 ms [ INFO ] Latency: - [ INFO ] Median: 49.80 ms - [ INFO ] Average: 50.35 ms - [ INFO ] Min: 33.27 ms - [ INFO ] Max: 104.15 ms - [ INFO ] Throughput: 118.80 FPS + [ INFO ] Median: 49.77 ms + [ INFO ] Average: 49.90 ms + [ INFO ] Min: 24.69 ms + [ INFO ] Max: 66.18 ms + [ INFO ] Throughput: 119.80 FPS .. code:: ipython3 @@ -819,7 +819,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 29.13 ms + [ INFO ] Read model took 29.59 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [1,3,640,640] @@ -837,7 +837,7 @@ models. [ INFO ] ***NO_NAME*** (node: __module.model.23/aten::cat/Concat_8) : f32 / [...] / [1,116,8400] [ INFO ] input.255 (node: __module.model.23.cv4.2.1.act/aten::silu_/Swish_46) : f32 / [...] / [1,32,160,160] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 594.13 ms + [ INFO ] Compile model took 592.55 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -874,17 +874,17 @@ models. [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 27.63 ms + [ INFO ] First inference took 24.82 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 3714 iterations - [ INFO ] Duration: 15026.92 ms + [ INFO ] Count: 3702 iterations + [ INFO ] Duration: 15040.97 ms [ INFO ] Latency: - [ INFO ] Median: 23.95 ms - [ INFO ] Average: 24.14 ms - [ INFO ] Min: 17.70 ms - [ INFO ] Max: 39.05 ms - [ INFO ] Throughput: 247.16 FPS + [ INFO ] Median: 24.15 ms + [ INFO ] Average: 24.25 ms + [ INFO ] Min: 20.10 ms + [ INFO ] Max: 43.05 ms + [ INFO ] Throughput: 246.13 FPS Other ways to optimize model diff --git a/docs/notebooks/yolov11-instance-segmentation-with-output_files/yolov11-instance-segmentation-with-output_46_0.png b/docs/notebooks/yolov11-instance-segmentation-with-output_files/yolov11-instance-segmentation-with-output_46_0.png index 011e13859e8a3e..737d0af981d026 100644 --- a/docs/notebooks/yolov11-instance-segmentation-with-output_files/yolov11-instance-segmentation-with-output_46_0.png +++ b/docs/notebooks/yolov11-instance-segmentation-with-output_files/yolov11-instance-segmentation-with-output_46_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e393b8edd8a20af1aaab6b69932bf0e715bf4f9c35814dd766dfa74ae27f2ae4 -size 493271 +oid sha256:f7d6739facd4204cf94a8bb5d471ea56a2ce3eab4dcd1e3807ba40c8e43baa44 +size 502582 diff --git a/docs/notebooks/yolov11-keypoint-detection-with-output.rst b/docs/notebooks/yolov11-keypoint-detection-with-output.rst index 97c549e6751dd0..cf94af9ab1e794 100644 --- a/docs/notebooks/yolov11-keypoint-detection-with-output.rst +++ b/docs/notebooks/yolov11-keypoint-detection-with-output.rst @@ -143,7 +143,7 @@ Import required utility functions. The lower cell will download the .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg') @@ -214,14 +214,14 @@ Let us consider the examples: .. parsed-literal:: - 100%|██████████| 5.97M/5.97M [00:00<00:00, 25.3MB/s] + 100%|██████████| 5.97M/5.97M [00:00<00:00, 21.1MB/s] .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg: 480x640 1 person, 59.6ms - Speed: 2.1ms preprocess, 59.6ms inference, 0.9ms postprocess per image at shape (1, 3, 480, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg: 480x640 1 person, 62.6ms + Speed: 1.8ms preprocess, 62.6ms inference, 0.9ms postprocess per image at shape (1, 3, 480, 640) @@ -258,7 +258,7 @@ preserve dynamic shapes in the model. OpenVINO: export success ✅ 2.0s, saved as 'yolo11n-pose_openvino_model/' (6.0 MB) Export complete (2.1s) - Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization + Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization Predict: yolo predict task=pose model=yolo11n-pose_openvino_model imgsz=640 half Validate: yolo val task=pose model=yolo11n-pose_openvino_model imgsz=640 data=/ultralytics/ultralytics/cfg/datasets/coco-pose.yaml half Visualize: https://netron.app @@ -338,8 +338,8 @@ ready to check model prediction. Loading yolo11n-pose_openvino_model for OpenVINO inference... Using OpenVINO LATENCY mode for batch=1 inference... - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg: 640x640 1 person, 19.9ms - Speed: 2.3ms preprocess, 19.9ms inference, 0.9ms postprocess per image at shape (1, 3, 640, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg: 640x640 1 person, 20.1ms + Speed: 2.8ms preprocess, 20.1ms inference, 1.0ms postprocess per image at shape (1, 3, 640, 640) @@ -686,8 +686,8 @@ on the image. Loading yolo11n-pose_openvino_model for OpenVINO inference... Using OpenVINO LATENCY mode for batch=1 inference... - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg: 640x640 1 person, 28.0ms - Speed: 2.0ms preprocess, 28.0ms inference, 1.1ms postprocess per image at shape (1, 3, 640, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/intel_rnb.jpg: 640x640 1 person, 31.4ms + Speed: 2.1ms preprocess, 31.4ms inference, 1.0ms postprocess per image at shape (1, 3, 640, 640) @@ -748,7 +748,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 19.55 ms + [ INFO ] Read model took 19.20 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [?,3,?,?] @@ -764,7 +764,7 @@ models. [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.model.23/aten::cat/Concat_9) : f32 / [...] / [1,56,8400] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 329.35 ms + [ INFO ] Compile model took 338.50 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -801,17 +801,17 @@ models. [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 33.67 ms + [ INFO ] First inference took 33.80 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 17280 iterations - [ INFO ] Duration: 120049.75 ms + [ INFO ] Count: 17082 iterations + [ INFO ] Duration: 120040.01 ms [ INFO ] Latency: - [ INFO ] Median: 40.74 ms - [ INFO ] Average: 41.55 ms - [ INFO ] Min: 24.20 ms - [ INFO ] Max: 98.48 ms - [ INFO ] Throughput: 143.94 FPS + [ INFO ] Median: 40.53 ms + [ INFO ] Average: 42.03 ms + [ INFO ] Min: 30.12 ms + [ INFO ] Max: 171.39 ms + [ INFO ] Throughput: 142.30 FPS .. code:: ipython3 @@ -838,7 +838,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 28.39 ms + [ INFO ] Read model took 28.43 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [1,3,640,640] @@ -854,7 +854,7 @@ models. [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.model.23/aten::cat/Concat_9) : f32 / [...] / [1,56,8400] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 556.59 ms + [ INFO ] Compile model took 573.24 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -891,17 +891,17 @@ models. [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 30.44 ms + [ INFO ] First inference took 25.90 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 5160 iterations - [ INFO ] Duration: 15031.11 ms + [ INFO ] Count: 5124 iterations + [ INFO ] Duration: 15036.37 ms [ INFO ] Latency: - [ INFO ] Median: 34.62 ms - [ INFO ] Average: 34.76 ms - [ INFO ] Min: 25.46 ms - [ INFO ] Max: 51.89 ms - [ INFO ] Throughput: 343.29 FPS + [ INFO ] Median: 34.72 ms + [ INFO ] Average: 35.02 ms + [ INFO ] Min: 25.74 ms + [ INFO ] Max: 52.00 ms + [ INFO ] Throughput: 340.77 FPS Compare accuracy of the Original and Quantized Models diff --git a/docs/notebooks/yolov11-keypoint-detection-with-output_files/yolov11-keypoint-detection-with-output_43_0.png b/docs/notebooks/yolov11-keypoint-detection-with-output_files/yolov11-keypoint-detection-with-output_43_0.png index f4e1de3947dc95..24dd29e6b7751f 100644 --- a/docs/notebooks/yolov11-keypoint-detection-with-output_files/yolov11-keypoint-detection-with-output_43_0.png +++ b/docs/notebooks/yolov11-keypoint-detection-with-output_files/yolov11-keypoint-detection-with-output_43_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:44dd0441baec6d73d3cc7552c26d2878c806514edacb3da332031733686c654b -size 507432 +oid sha256:498642bae277603810b461dcd81b4e67eb3cce1e0f1099f346b8d832dde4720f +size 504317 diff --git a/docs/notebooks/yolov11-object-detection-with-output.rst b/docs/notebooks/yolov11-object-detection-with-output.rst index d987f4148e7265..b446d6d3c1d41e 100644 --- a/docs/notebooks/yolov11-object-detection-with-output.rst +++ b/docs/notebooks/yolov11-object-detection-with-output.rst @@ -141,7 +141,7 @@ Import required utility functions. The lower cell will download the .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg') @@ -207,14 +207,14 @@ Let us consider the examples: .. parsed-literal:: - 100%|██████████| 5.35M/5.35M [00:00<00:00, 23.2MB/s] + 100%|██████████| 5.35M/5.35M [00:00<00:00, 24.0MB/s] .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 480x640 2 bicycles, 2 cars, 2 dogs, 78.0ms - Speed: 2.3ms preprocess, 78.0ms inference, 0.9ms postprocess per image at shape (1, 3, 480, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 480x640 2 bicycles, 2 cars, 2 dogs, 80.3ms + Speed: 2.2ms preprocess, 80.3ms inference, 1.0ms postprocess per image at shape (1, 3, 480, 640) @@ -250,8 +250,8 @@ preserve dynamic shapes in the model. OpenVINO: starting export with openvino 2024.5.0-16993-9c432a3641a... OpenVINO: export success ✅ 1.8s, saved as 'yolo11n_openvino_model/' (5.4 MB) - Export complete (1.9s) - Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization + Export complete (2.0s) + Results saved to /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization Predict: yolo predict task=detect model=yolo11n_openvino_model imgsz=640 half Validate: yolo val task=detect model=yolo11n_openvino_model imgsz=640 data=/usr/src/ultralytics/ultralytics/cfg/datasets/coco.yaml half Visualize: https://netron.app @@ -330,8 +330,8 @@ ready to check model prediction for object detection. Loading yolo11n_openvino_model for OpenVINO inference... Using OpenVINO LATENCY mode for batch=1 inference... - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 1 bicycle, 2 cars, 1 dog, 18.7ms - Speed: 2.0ms preprocess, 18.7ms inference, 1.0ms postprocess per image at shape (1, 3, 640, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 1 bicycle, 2 cars, 1 dog, 19.9ms + Speed: 1.8ms preprocess, 19.9ms inference, 0.9ms postprocess per image at shape (1, 3, 640, 640) @@ -645,8 +645,8 @@ on the image. .. parsed-literal:: - image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 1 bicycle, 2 cars, 1 dog, 19.7ms - Speed: 1.8ms preprocess, 19.7ms inference, 0.9ms postprocess per image at shape (1, 3, 640, 640) + image 1/1 /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov11-optimization/data/coco_bike.jpg: 640x640 1 bicycle, 2 cars, 1 dog, 18.1ms + Speed: 1.8ms preprocess, 18.1ms inference, 1.0ms postprocess per image at shape (1, 3, 640, 640) @@ -708,7 +708,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 18.73 ms + [ INFO ] Read model took 18.52 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [?,3,?,?] @@ -724,7 +724,7 @@ models. [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.model.23/aten::cat/Concat_7) : f32 / [...] / [1,84,8400] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 316.96 ms + [ INFO ] Compile model took 317.51 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -761,17 +761,17 @@ models. [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 120000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 30.27 ms + [ INFO ] First inference took 30.05 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 18480 iterations - [ INFO ] Duration: 120044.94 ms + [ INFO ] Count: 18396 iterations + [ INFO ] Duration: 120062.14 ms [ INFO ] Latency: - [ INFO ] Median: 38.09 ms - [ INFO ] Average: 38.84 ms - [ INFO ] Min: 20.45 ms - [ INFO ] Max: 97.13 ms - [ INFO ] Throughput: 153.94 FPS + [ INFO ] Median: 38.20 ms + [ INFO ] Average: 39.02 ms + [ INFO ] Min: 22.85 ms + [ INFO ] Max: 99.30 ms + [ INFO ] Throughput: 153.22 FPS .. code:: ipython3 @@ -798,7 +798,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 26.48 ms + [ INFO ] Read model took 26.57 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [1,3,640,640] @@ -814,7 +814,7 @@ models. [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.model.23/aten::cat/Concat_7) : f32 / [...] / [1,84,8400] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 556.87 ms + [ INFO ] Compile model took 533.02 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -851,17 +851,17 @@ models. [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 31.34 ms + [ INFO ] First inference took 29.56 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 5208 iterations - [ INFO ] Duration: 15031.88 ms + [ INFO ] Count: 5148 iterations + [ INFO ] Duration: 15046.93 ms [ INFO ] Latency: - [ INFO ] Median: 34.21 ms - [ INFO ] Average: 34.44 ms - [ INFO ] Min: 18.94 ms - [ INFO ] Max: 52.48 ms - [ INFO ] Throughput: 346.46 FPS + [ INFO ] Median: 34.51 ms + [ INFO ] Average: 34.87 ms + [ INFO ] Min: 24.28 ms + [ INFO ] Max: 69.01 ms + [ INFO ] Throughput: 342.13 FPS Next steps diff --git a/docs/notebooks/yolov11-object-detection-with-output_files/yolov11-object-detection-with-output_43_0.png b/docs/notebooks/yolov11-object-detection-with-output_files/yolov11-object-detection-with-output_43_0.png index 02e6e38bb12a7b..d5c2464840ad65 100644 --- a/docs/notebooks/yolov11-object-detection-with-output_files/yolov11-object-detection-with-output_43_0.png +++ b/docs/notebooks/yolov11-object-detection-with-output_files/yolov11-object-detection-with-output_43_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0904f4ceeae66d4c516656b23389b16d93bc601214a4c9b8d1ee30f7b5366526 -size 572206 +oid sha256:4e54004e6e28473d61aacba64acb407291de3581ae9a0b792a3d8f96bd394f08 +size 572509 diff --git a/docs/notebooks/yolov8-object-detection-with-output.rst b/docs/notebooks/yolov8-object-detection-with-output.rst index 6b7dacd839a587..16648e4559f618 100644 --- a/docs/notebooks/yolov8-object-detection-with-output.rst +++ b/docs/notebooks/yolov8-object-detection-with-output.rst @@ -121,7 +121,7 @@ Install necessary packages. .. code:: ipython3 %pip install -q "openvino>=2024.0.0" "nncf>=2.9.0" - %pip install -q "torch>=2.1" "torchvision>=0.16" "ultralytics==8.3.0" onnx tqdm opencv-python --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -q "torch>=2.1" "torchvision>=0.16" "ultralytics==8.2.24" onnx tqdm opencv-python --extra-index-url https://download.pytorch.org/whl/cpu Import required utility functions. The lower cell will download the ``notebook_utils`` Python module from GitHub. @@ -161,7 +161,7 @@ Import required utility functions. The lower cell will download the .. parsed-literal:: - PosixPath('/home/ea/work/openvino_notebooks_new_clone/openvino_notebooks/notebooks/yolov8-optimization/data/coco_bike.jpg') + PosixPath('/home/akash/intel/openvino_notebooks/notebooks/yolov8-optimization/data/coco_bike.jpg') @@ -194,10 +194,9 @@ Let us consider the examples: from PIL import Image from ultralytics import YOLO - DET_MODEL_NAME = "yolo11n" + DET_MODEL_NAME = "yolov8n" - det_model = YOLO(f"{DET_MODEL_NAME}.pt") - det_model.to("cpu") + det_model = YOLO(models_dir / f"{DET_MODEL_NAME}.pt") label_map = det_model.model.names res = det_model(IMAGE_PATH) @@ -206,14 +205,24 @@ Let us consider the examples: .. parsed-literal:: + Downloading https://github.com/ultralytics/assets/releases/download/v8.2.0/yolov8n.pt to 'models/yolov8n.pt'... - image 1/1 /home/ea/work/openvino_notebooks_new_clone/openvino_notebooks/notebooks/yolov8-optimization/data/coco_bike.jpg: 480x640 2 bicycles, 2 cars, 2 dogs, 101.6ms - Speed: 3.1ms preprocess, 101.6ms inference, 1.4ms postprocess per image at shape (1, 3, 480, 640) +.. parsed-literal:: + + 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 6.25M/6.25M [00:02<00:00, 2.29MB/s] + + +.. parsed-literal:: + image 1/1 /home/akash/intel/openvino_notebooks/notebooks/yolov8-optimization/data/coco_bike.jpg: 480x640 2 bicycles, 2 cars, 1 dog, 82.9ms + Speed: 2.4ms preprocess, 82.9ms inference, 475.6ms postprocess per image at shape (1, 3, 480, 640) -.. image:: yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.png + + + +.. image:: yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_3.png @@ -230,10 +239,27 @@ preserve dynamic shapes in the model. .. code:: ipython3 # object detection model - det_model_path = Path(f"{DET_MODEL_NAME}_openvino_model/{DET_MODEL_NAME}.xml") + det_model_path = models_dir / f"{DET_MODEL_NAME}_openvino_model/{DET_MODEL_NAME}.xml" if not det_model_path.exists(): det_model.export(format="openvino", dynamic=True, half=True) + +.. parsed-literal:: + + Ultralytics YOLOv8.2.24 🚀 Python-3.8.10 torch-2.1.0+cu121 CPU (Intel Core(TM) i9-10980XE 3.00GHz) + + PyTorch: starting from 'models/yolov8n.pt' with input shape (1, 3, 640, 640) BCHW and output shape(s) (1, 84, 8400) (6.2 MB) + + OpenVINO: starting export with openvino 2024.3.0-16041-1e3b88e4e3f-releases/2024/3... + OpenVINO: export success ✅ 1.7s, saved as 'models/yolov8n_openvino_model/' (6.4 MB) + + Export complete (3.1s) + Results saved to /home/akash/intel/openvino_notebooks/notebooks/yolov8-optimization/models + Predict: yolo predict task=detect model=models/yolov8n_openvino_model imgsz=640 half + Validate: yolo val task=detect model=models/yolov8n_openvino_model imgsz=640 data=coco.yaml half + Visualize: https://netron.app + + Verify model inference ~~~~~~~~~~~~~~~~~~~~~~ @@ -297,7 +323,6 @@ ready to check model prediction for object detection. det_model.predictor.inference = infer det_model.predictor.model.pt = False - det_model res = det_model(IMAGE_PATH) Image.fromarray(res[0].plot()[:, :, ::-1]) @@ -306,8 +331,8 @@ ready to check model prediction for object detection. .. parsed-literal:: - image 1/1 /home/ea/work/openvino_notebooks_new_clone/openvino_notebooks/notebooks/yolov8-optimization/data/coco_bike.jpg: 640x640 1 bicycle, 2 cars, 1 dog, 16.9ms - Speed: 3.7ms preprocess, 16.9ms inference, 1.7ms postprocess per image at shape (1, 3, 640, 640) + image 1/1 /home/akash/intel/openvino_notebooks/notebooks/yolov8-optimization/data/coco_bike.jpg: 640x640 2 bicycles, 2 cars, 1 dog, 16.1ms + Speed: 3.4ms preprocess, 16.1ms inference, 1.8ms postprocess per image at shape (1, 3, 640, 640) @@ -368,16 +393,245 @@ evaluation function. zip_ref.extractall(OUT_DIR / "coco/images") + +.. parsed-literal:: + + /home/akash/intel/NNCF/nncf/examples/post_training_quantization/openvino/yolov8/datasets/val2017.zip: 0%| … + + +.. parsed-literal:: + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + IOPub message rate exceeded. + The Jupyter server will temporarily stop sending output + to the client in order to avoid crashing it. + To change this limit, set the config variable + `--ServerApp.iopub_msg_rate_limit`. + + Current values: + ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec) + ServerApp.rate_limit_window=3.0 (secs) + + + .. parsed-literal:: - '/home/ea/work/openvino_notebooks_new_clone/openvino_notebooks/notebooks/yolov8-optimization/datasets/val2017.zip' already exists. - '/home/ea/work/openvino_notebooks_new_clone/openvino_notebooks/notebooks/yolov8-optimization/datasets/coco2017labels-segments.zip' already exists. + '/home/akash/intel/NNCF/nncf/examples/post_training_quantization/openvino/yolov8/datasets/coco2017labels-segments.zip' already exists. .. parsed-literal:: - /home/ea/work/openvino_notebooks_new_clone/openvino_notebooks/notebooks/yolov8-optimization/datasets/coco.yaml… + /home/akash/intel/NNCF/nncf/examples/post_training_quantization/openvino/yolov8/datasets/coco.yaml: 0%| … Define validation function @@ -520,7 +774,7 @@ validator class instance. .. parsed-literal:: - val: Scanning /home/ea/work/openvino_notebooks_new_clone/openvino_notebooks/notebooks/yolov8-optimization/datasets/coco/labels/val2017.cache... 4952 images, 48 backgrounds, 0 corrupt: 100%|██████████| 5000/5000 [00:00\ **Note**: Model evaluation is time consuming +process and can take several minutes, depending on the hardware. For +reducing calculation time, we define ``num_samples`` parameter with +evaluation subset size, but in this case, accuracy can be noncomparable +with originally reported by the authors of the model, due to validation +subset difference. *To validate the models on the full dataset set +``NUM_TEST_SAMPLES = None``.* + +.. code:: ipython3 + + NUM_TEST_SAMPLES = 300 + +.. code:: ipython3 + + fp_det_stats = test(det_ov_model, core, det_data_loader, det_validator, num_samples=NUM_TEST_SAMPLES) + + + +.. parsed-literal:: + + 0%| | 0/300 [00:00 ([1,640,640,3], [N,H,W,C], f32) convert layout [N,C,H,W]: ([1,640,640,3], [N,H,W,C], f32) -> ([1,3,640,640], [N,C,H,W], f32) @@ -1348,7 +1704,7 @@ Now, we can skip these preprocessing steps in detect function: -.. image:: yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.png +.. image:: yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_70_0.png @@ -1527,7 +1883,7 @@ Run the object detection: -.. image:: yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_67_0.png +.. image:: yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_76_0.png .. parsed-literal:: diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.jpg b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.jpg deleted file mode 100644 index 7a742941b54c17..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.jpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:394895a09cc413a6343edb2a6f6242258701b6911c09d8405eacc7c27f6fac62 -size 110819 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.png index 6d4bfe012b6187..703c1375a98784 100644 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.png +++ b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_16_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7b18cc2e60a802c059cd5cc05598a06c4affd959216e03674eeb68e822295843 -size 913173 +oid sha256:e85f0434876997adc7cf2152fa667b4e271817a443a75ca350d284aee48f9145 +size 908034 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_38_1.jpg b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_38_1.jpg deleted file mode 100644 index 40ac3a8da0dc2a..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_38_1.jpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:90d9079b877726f30f154885745b36d8bc281f4970ce10f86529b9739a56ac72 -size 112567 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_38_1.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_38_1.png deleted file mode 100644 index e7c4fb31035d98..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_38_1.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6fad172495a797bc753d19d54bd4afaf8f08ed36dfd1b9761df7bfa948673aab -size 911538 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_43_1.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_43_1.png new file mode 100644 index 00000000000000..44e7623ec662d1 --- /dev/null +++ b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_43_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6778bdd31a39722d8ce3fb9ddea6383e451ab1b32cf23fad23b556bd89c99707 +size 907722 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.jpg b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.jpg deleted file mode 100644 index 202af866d221dd..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.jpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:02d585dcd341e2580baf166b6f35337d2da17561a251fb5475041e2a20d6e558 -size 110078 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.png deleted file mode 100644 index d0d2028049c4f3..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_61_0.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:44c1e80aa859828b9aae3240fb544b483df8dbd8b7c7bca29479a2a0714612c2 -size 929124 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_67_0.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_67_0.png deleted file mode 100644 index cdae11c722ca65..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_67_0.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2c6c2609efd786cba1d9baac015d06040939ff8d1e7644f67d10d6df5c171a62 -size 569438 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_70_0.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_70_0.png new file mode 100644 index 00000000000000..3b025ea9323f25 --- /dev/null +++ b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_70_0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b44f35cf238dcd208fcebe73bb388947f5528a167b511c8098d040b05d9bb819 +size 931491 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_76_0.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_76_0.png new file mode 100644 index 00000000000000..6d628536486d93 --- /dev/null +++ b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_76_0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc0d406144e3b07ae6bef629cd7b4973424847958b9865e7133bd0f0c7deeb5c +size 534154 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.jpg b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.jpg deleted file mode 100644 index bcf00f3d50dcb5..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.jpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cc502c95e9f5526befe662fcc79a6a91d4f5ff0cfce09165436e91f4ef8224ad -size 113613 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.png deleted file mode 100644 index e886ccd29b47f9..00000000000000 --- a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_1.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2c7bb27d4264dbc22e0006e42fb2f1619992aeac7220232119578e42a0d8a083 -size 904352 diff --git a/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_3.png b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_3.png new file mode 100644 index 00000000000000..5073fa9f3eca09 --- /dev/null +++ b/docs/notebooks/yolov8-object-detection-with-output_files/yolov8-object-detection-with-output_9_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46d38bd078c6eeda706640d0e35907a9a504bf78f9dd8dbf2961de865c294802 +size 907783 diff --git a/docs/notebooks/yolov9-optimization-with-output.rst b/docs/notebooks/yolov9-optimization-with-output.rst index 39690ea292721d..b132215f390216 100644 --- a/docs/notebooks/yolov9-optimization-with-output.rst +++ b/docs/notebooks/yolov9-optimization-with-output.rst @@ -21,7 +21,6 @@ The tutorial consists of the following steps: - Compare performance of the FP32 and quantized models. - Run optimized model inference on video - **Table of contents:** - `Prerequisites <#prerequisites>`__ @@ -58,7 +57,6 @@ Guide =2023.3.0" "nncf>=2.8.1" "opencv-python" "matplotlib>=3.4" "seaborn" "pandas" "scikit-learn" "torch" "torchvision" "tqdm" --extra-index-url https://download.pytorch.org/whl/cpu @@ -93,9 +91,9 @@ Prerequisites Cloning into 'yolov9'... remote: Enumerating objects: 781, done. remote: Total 781 (delta 0), reused 0 (delta 0), pack-reused 781 (from 1) - Receiving objects: 100% (781/781), 3.27 MiB | 19.93 MiB/s, done. + Receiving objects: 100% (781/781), 3.27 MiB | 16.41 MiB/s, done. Resolving deltas: 100% (331/331), done. - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9 + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9 Get PyTorch model @@ -133,7 +131,7 @@ applicable for other models from YOLO V9 family. .. parsed-literal:: - PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9/model/gelan-c.pt') + PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9/model/gelan-c.pt') @@ -188,11 +186,11 @@ using ``ov.save_model``. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9/models/experimental.py:243: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9/models/experimental.py:243: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. ckpt = torch.load(attempt_download(w), map_location='cpu') # load Fusing layers... Model summary: 387 layers, 25288768 parameters, 0 gradients, 102.1 GFLOPs - /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/801/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9/models/yolo.py:108: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/jobs/ov-notebook/jobs/OVNotebookOps/builds/810/archive/.workspace/scm/ov-notebook/notebooks/yolov9-optimization/yolov9/models/yolo.py:108: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! elif self.dynamic or self.shape != shape: @@ -573,10 +571,10 @@ asymmetric quantization of activations. .. parsed-literal:: - 2024-10-23 05:32:14.632848: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-10-23 05:32:14.668324: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-11-05 05:37:29.322072: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-11-05 05:37:29.357249: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-10-23 05:32:15.276616: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-11-05 05:37:29.968269: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT @@ -675,7 +673,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 26.16 ms + [ INFO ] Read model took 26.54 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] images (node: x) : f32 / [...] / [?,3,?,?] @@ -687,7 +685,7 @@ models. [Step 5/11] Resizing model to match image sizes and given batch [ INFO ] Model batch size: 1 [ INFO ] Reshaping model: 'images': [1,3,640,640] - [ INFO ] Reshape model took 7.97 ms + [ INFO ] Reshape model took 8.17 ms [Step 6/11] Configuring input of the model [ INFO ] Model inputs: [ INFO ] images (node: x) : u8 / [N,C,H,W] / [1,3,640,640] @@ -697,7 +695,7 @@ models. [ INFO ] xi.3 (node: __module.model.22/aten::cat/Concat_1) : f32 / [...] / [1,144,40,40] [ INFO ] xi (node: __module.model.22/aten::cat/Concat) : f32 / [...] / [1,144,20,20] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 475.37 ms + [ INFO ] Compile model took 499.10 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -734,17 +732,17 @@ models. [ INFO ] Fill input 'images' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 182.45 ms + [ INFO ] First inference took 170.42 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 228 iterations - [ INFO ] Duration: 15670.39 ms + [ INFO ] Count: 222 iterations + [ INFO ] Duration: 15614.22 ms [ INFO ] Latency: - [ INFO ] Median: 412.82 ms - [ INFO ] Average: 410.86 ms - [ INFO ] Min: 309.65 ms - [ INFO ] Max: 431.51 ms - [ INFO ] Throughput: 14.55 FPS + [ INFO ] Median: 412.60 ms + [ INFO ] Average: 419.41 ms + [ INFO ] Min: 210.55 ms + [ INFO ] Max: 885.97 ms + [ INFO ] Throughput: 14.22 FPS .. code:: ipython3 @@ -769,7 +767,7 @@ models. [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 41.07 ms + [ INFO ] Read model took 41.88 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] images (node: x) : f32 / [...] / [1,3,640,640] @@ -791,7 +789,7 @@ models. [ INFO ] xi.3 (node: __module.model.22/aten::cat/Concat_1) : f32 / [...] / [1,144,40,40] [ INFO ] xi (node: __module.model.22/aten::cat/Concat) : f32 / [...] / [1,144,20,20] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 943.07 ms + [ INFO ] Compile model took 946.19 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -828,17 +826,17 @@ models. [ INFO ] Fill input 'images' with random values [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 64.84 ms + [ INFO ] First inference took 68.41 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 720 iterations - [ INFO ] Duration: 15158.98 ms + [ INFO ] Count: 726 iterations + [ INFO ] Duration: 15191.13 ms [ INFO ] Latency: - [ INFO ] Median: 119.92 ms - [ INFO ] Average: 125.90 ms - [ INFO ] Min: 80.89 ms - [ INFO ] Max: 277.84 ms - [ INFO ] Throughput: 47.50 FPS + [ INFO ] Median: 121.43 ms + [ INFO ] Average: 125.10 ms + [ INFO ] Min: 56.89 ms + [ INFO ] Max: 305.84 ms + [ INFO ] Throughput: 47.79 FPS Run Live Object Detection diff --git a/docs/notebooks/yolov9-optimization-with-output_files/yolov9-optimization-with-output_36_0.png b/docs/notebooks/yolov9-optimization-with-output_files/yolov9-optimization-with-output_36_0.png index d6b48379756722..ccb15ab8382b86 100644 --- a/docs/notebooks/yolov9-optimization-with-output_files/yolov9-optimization-with-output_36_0.png +++ b/docs/notebooks/yolov9-optimization-with-output_files/yolov9-optimization-with-output_36_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:014be25aa9fb5ebd46a7f69afca67c33d8059694ca54bc2e32c0abfa460cb825 -size 496427 +oid sha256:13788e6c93a735cf601513bf18c5e6cae29819bdff2bd506fa90a4d34ae4bcc5 +size 496199 diff --git a/docs/openvino_sphinx_theme/README.md b/docs/openvino_sphinx_theme/README.md index 7931c481c308aa..2e82fa06e8c185 100644 --- a/docs/openvino_sphinx_theme/README.md +++ b/docs/openvino_sphinx_theme/README.md @@ -4,7 +4,8 @@ 1. Install the `openvino_sphinx_theme` using `python`: ``` -python setup.py install --user +cd openvino/docs/openvino_sphinx_theme +python -m pip install --user . ``` 2. Update the `html_theme` variable in your `conf.py`: diff --git a/docs/openvino_sphinx_theme/openvino_sphinx_theme/templates/layout.html b/docs/openvino_sphinx_theme/openvino_sphinx_theme/templates/layout.html index 25acb3c1e5cbda..a2ab53c6a57a83 100644 --- a/docs/openvino_sphinx_theme/openvino_sphinx_theme/templates/layout.html +++ b/docs/openvino_sphinx_theme/openvino_sphinx_theme/templates/layout.html @@ -28,7 +28,7 @@ {# The data-cfasync attribute disables CloudFlare's Rocket loader so that #} {# mode/theme are correctly set before the browser renders the page. #} {# https://github.com/pydata/pydata-sphinx-theme/pull/1045 #} - + - - - - + + - - - - {% endblock %} - {% block docs_navbar %} {{ super() }} {% include 'baner.html' %} diff --git a/docs/sphinx_setup/_templates/search.html b/docs/sphinx_setup/_templates/search.html index 3519f6e7e02f19..5430f24f74aa8c 100644 --- a/docs/sphinx_setup/_templates/search.html +++ b/docs/sphinx_setup/_templates/search.html @@ -2,133 +2,100 @@ {% set title = _('Search') %} {%- block content %} - {% block docs_navbar %} - {{ super() }} - {% include 'baner.html' %} - {% endblock %} +{% block docs_navbar %} +{{ super() }} +{% include 'baner.html' %} +{% endblock %} - {% block body %} - - - - - - - - - - - -
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {% endblock %} - - {%- block scripts_end %} - {{ _webpack.body_post() }} - {%- endblock %} +{% block body %} + + + + + + + + +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +{% endblock %} +{%- block scripts_end %} +{{ _webpack.body_post() }} {%- endblock %} + +{%- endblock %} \ No newline at end of file diff --git a/docs/sphinx_setup/conf.py b/docs/sphinx_setup/conf.py index def41af5943b3c..01c74de0175bcf 100644 --- a/docs/sphinx_setup/conf.py +++ b/docs/sphinx_setup/conf.py @@ -193,7 +193,6 @@ 'css/textfield.css', 'css/tabs.css', 'css/coveo_custom.css', - 'https://static.cloud.coveo.com/atomic/v2/themes/coveo.css', 'https://cdn.jsdelivr.net/npm/@splidejs/splide@4.1.4/dist/css/splide.min.css', ] diff --git a/docs/sphinx_setup/index.rst b/docs/sphinx_setup/index.rst index a65b5b5892c560..47dc76bf16cf29 100644 --- a/docs/sphinx_setup/index.rst +++ b/docs/sphinx_setup/index.rst @@ -124,11 +124,11 @@ Places to Begin Cloud-ready deployments for microservice applications. - .. button-link:: openvino-workflow/running-inference.html + .. button-link:: openvino-workflow/openvino-workflow/model-server/ovms_what_is_openvino_model_server.html :color: primary :outline: - Try it out + Check out Model Server .. grid-item-card:: Model Compression :img-top: ./assets/images/home_begin_tile_06.png diff --git a/samples/cpp/common/utils/src/args_helper.cpp b/samples/cpp/common/utils/src/args_helper.cpp index f4a3d10ceb0b5b..ba58f98e498e90 100644 --- a/samples/cpp/common/utils/src/args_helper.cpp +++ b/samples/cpp/common/utils/src/args_helper.cpp @@ -29,8 +29,7 @@ void readInputFilesArguments(std::vector& files, const std::string& arg) { struct stat sb; if (stat(arg.c_str(), &sb) != 0) { - slog::warn << "File " << arg << " cannot be opened!" << slog::endl; - return; + throw std::invalid_argument(arg + " file or directory not found."); } if (S_ISDIR(sb.st_mode)) { struct CloseDir { @@ -43,17 +42,20 @@ void readInputFilesArguments(std::vector& files, const std::string& using Dir = std::unique_ptr; Dir dp(opendir(arg.c_str())); if (dp == nullptr) { - slog::warn << "Directory " << arg << " cannot be opened!" << slog::endl; - return; + throw std::invalid_argument(arg + " directory cannot be opened!"); } struct dirent* ep; + size_t files_size = files.size(); while (nullptr != (ep = readdir(dp.get()))) { std::string fileName = ep->d_name; if (fileName == "." || fileName == "..") continue; files.push_back(arg + "/" + ep->d_name); } + if (files.size() == files_size) { + throw std::invalid_argument("No files were found in directory " + arg); + } } else { files.push_back(arg); } diff --git a/samples/js/node/.gitignore b/samples/js/node/.gitignore index 2ab910508d5529..c836a5862da256 100644 --- a/samples/js/node/.gitignore +++ b/samples/js/node/.gitignore @@ -1,2 +1,3 @@ node_modules -hello_reshape_ssd/out.jpg +out*.jpg +output/ diff --git a/samples/js/node/README.md b/samples/js/node/README.md index 9e69769778d8a8..2fd4dbe76e5900 100644 --- a/samples/js/node/README.md +++ b/samples/js/node/README.md @@ -7,13 +7,15 @@ To run samples, install dependencies first. In current directory run: npm install ``` -Note: Perform these steps also before running notebooks. +Note: Perform this step also before running notebooks. ## Samples - hello_classification - hello_reshape_ssd - classification_sample_async + - optical_character_recognition + - vision_background_removal ## Notebooks diff --git a/samples/js/node/classification_sample_async/README.md b/samples/js/node/classification_sample_async/README.md index 0b19e908587505..6a0dcba6d16e02 100644 --- a/samples/js/node/classification_sample_async/README.md +++ b/samples/js/node/classification_sample_async/README.md @@ -4,7 +4,12 @@ Models with only 1 input and output are supported. Run: ```bash -node classification_sample_async.js -m *path_to_model_file* -i *path_to_img1* -i *path_to_img2* -d AUTO +node classification_sample_async.js -m ../../assets/models/v3-small_224_1.0_float.xml -i ../../assets/images/coco.jpg -i ../../assets/images/coco_hollywood.jpg -d AUTO +``` + +Where +```bash +node classification_sample_async.js -m *path_to_model_file* -i *path_to_img1* -i *path_to_img2* -d *device* ``` Other details see in [../../../python/classification_sample_async/README.md](../../../python/classification_sample_async/README.md) diff --git a/samples/js/node/classification_sample_async/classification_sample_async.js b/samples/js/node/classification_sample_async/classification_sample_async.js index 94f4d7828c95c2..96724be80d3b0b 100644 --- a/samples/js/node/classification_sample_async/classification_sample_async.js +++ b/samples/js/node/classification_sample_async/classification_sample_async.js @@ -1,39 +1,30 @@ const { addon: ov } = require('openvino-node'); const args = require('args'); -const { cv } = require('opencv-wasm'); -const { getImageData } = require('../helpers.js'); - -args.options([{ - name: 'img', - defaultValue: [], -}, { - name: 'model', -}, { - name: 'device', -}]); -const { model: modelPath, device: deviceName, img: images } = - args.parse(process.argv); - -main(modelPath, images, deviceName); - -function completionCallback(result, imagePath) { - const predictions = Array.from(result.data) - .map((prediction, classId) => ({ prediction, classId })) - .sort(({ prediction: predictionA }, { prediction: predictionB }) => - predictionA === predictionB ? 0 : predictionA > predictionB ? -1 : 1); - - console.log(`Image path: ${imagePath}`); - console.log('Top 10 results:'); - console.log('class_id probability'); - console.log('--------------------'); - predictions.slice(0, 10).forEach(({ classId, prediction }) => - console.log(`${classId}\t ${prediction.toFixed(7)}`), - ); - console.log(); -} - -async function main(modelPath, images, deviceName) { +const Image = require('../image.js'); +const imagenetClassesMap = require('../../assets/datasets/imagenet_class_index.json'); + +args.options([ + { + name: 'img', + defaultValue: [], + }, + { + name: 'model', + }, + { + name: 'device', + }, +]); +const { + model: modelPath, + device: deviceName, + img: imgPaths +} = args.parse(process.argv); + +main(modelPath, imgPaths, deviceName); + +async function main(modelPath, imgPaths, deviceName) { //----------- Step 1. Initialize OpenVINO Runtime Core ----------------------- console.log('Creating OpenVINO Runtime Core'); const core = new ov.Core(); @@ -42,8 +33,6 @@ async function main(modelPath, images, deviceName) { console.log(`Reading the model: ${modelPath}`); // (.xml and .bin files) or (.onnx file) const model = await core.readModel(modelPath); - const [h, w] = model.inputs[0].shape.slice(-2); - const tensorShape = [1, h, w, 3]; if (model.inputs.length !== 1) throw new Error('Sample supports only single input topologies'); @@ -52,56 +41,65 @@ async function main(modelPath, images, deviceName) { throw new Error('Sample supports only single output topologies'); //----------- Step 3. Set up input ------------------------------------------- - // Read input image - const imagesData = []; - - for (const imagePath of images) - imagesData.push(await getImageData(imagePath)); + const inputImages = []; + const [, inputHeight, inputWidth] = model.inputs[0].getShape(); - const preprocessedImages = imagesData.map((imgData) => { - // Use opencv-wasm to preprocess image. - const originalImage = cv.matFromImageData(imgData); - const image = new cv.Mat(); - // The MobileNet model expects images in RGB format. - cv.cvtColor(originalImage, image, cv.COLOR_RGBA2RGB); - cv.resize(image, image, new cv.Size(w, h)); + // Read input image, resize it to the model's input size and convert it to a tensor. + for (const path of imgPaths) { + const img = await Image.load(path); + const resized = img.resize(inputWidth, inputHeight); - return new Uint8Array(image.data); - }); + inputImages.push(resized); + } //----------- Step 4. Apply preprocessing ------------------------------------ const _ppp = new ov.preprocess.PrePostProcessor(model); _ppp.input().tensor().setLayout('NHWC').setElementType(ov.element.u8); - _ppp.input().model().setLayout('NCHW'); + _ppp.input().model().setLayout('NHWC'); _ppp.output().tensor().setElementType(ov.element.f32); _ppp.build(); - //----------------- Step 5. Loading model to the device ---------------------- + //----------- Step 5. Loading model to the device ---------------------------- console.log('Loading the model to the plugin'); const compiledModel = await core.compileModel(model, deviceName); const outputName = compiledModel.output(0).toString(); - //----------- Step 6. Collecting promises to react when they resolve --------- - console.log('Starting inference in asynchronous mode'); + //----------- Step 6. Do inference ------------------------------------------- + console.log('Starting inference\n'); // Create infer request const inferRequest = compiledModel.createInferRequest(); - - const promises = preprocessedImages.map((tensorData, i) => { - const inferPromise = inferRequest.inferAsync([ - new ov.Tensor(ov.element.u8, tensorShape, tensorData) - ]); + const promises = inputImages.map((img, i) => { + const inferPromise = inferRequest.inferAsync([img.toTensor()]); inferPromise.then(result => - completionCallback(result[outputName], images[i])); + completionCallback(result[outputName], imgPaths[i])); return inferPromise; }); - //----------- Step 7. Do inference ------------------------------------------- + //----------- Step 7. Wait till all inferences execute ----------------------- await Promise.all(promises); console.log('All inferences executed'); console.log('\nThis sample is an API example, for any performance ' + 'measurements please use the dedicated benchmark_app tool'); } + +function completionCallback(result, imagePath) { + const predictions = Array.from(result.data) + .map((prediction, classId) => ({ prediction, classId })) + .sort(({ prediction: predictionA }, { prediction: predictionB }) => + predictionA === predictionB ? 0 : predictionA > predictionB ? -1 : 1); + + const imagenetClasses = ['background', ...Object.values(imagenetClassesMap)]; + + console.log(`Image path: ${imagePath}`); + console.log('Top 5 results:\n'); + console.log('id\tprobability\tlabel'); + console.log('---------------------------------'); + predictions.slice(0, 5).forEach(({ classId, prediction }) => + console.log(`${classId}\t${prediction.toFixed(7)}\t${imagenetClasses[classId][1]}`), + ); + console.log(); +} diff --git a/samples/js/node/fetch-samples-assets.js b/samples/js/node/fetch-samples-assets.js new file mode 100644 index 00000000000000..33dd509a922f85 --- /dev/null +++ b/samples/js/node/fetch-samples-assets.js @@ -0,0 +1,83 @@ +const { downloadFile } = require('./helpers.js'); + +const host = 'https://storage.openvinotoolkit.org'; + +const models = [ + // hello classification + '/repositories/openvino_notebooks/models/mobelinet-v3-tf/FP32/v3-small_224_1.0_float.xml', + '/repositories/openvino_notebooks/models/mobelinet-v3-tf/FP32/v3-small_224_1.0_float.bin', + + // hello reshape ssd + '/repositories/open_model_zoo/2022.3/models_bin/1/road-segmentation-adas-0001/FP32/road-segmentation-adas-0001.xml', + '/repositories/open_model_zoo/2022.3/models_bin/1/road-segmentation-adas-0001/FP32/road-segmentation-adas-0001.bin', + + // hello detection, optical character recognition + '/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.xml', + '/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.bin', + + '/repositories/open_model_zoo/public/text-recognition-resnet-fc/text-recognition-resnet-fc.xml', + '/repositories/open_model_zoo/public/text-recognition-resnet-fc/text-recognition-resnet-fc.bin', + + // vision background removal + '/repositories/open_model_zoo/public/vision-background-removal/unet_ir_model.xml', + '/repositories/open_model_zoo/public/vision-background-removal/unet_ir_model.bin', + + // pose estimation + '/repositories/open_model_zoo/2022.1/models_bin/3/human-pose-estimation-0001/FP16-INT8/human-pose-estimation-0001.xml', + '/repositories/open_model_zoo/2022.1/models_bin/3/human-pose-estimation-0001/FP16-INT8/human-pose-estimation-0001.bin', + + // question answering + '/repositories/open_model_zoo/2022.3/models_bin/1/bert-small-uncased-whole-word-masking-squad-0001/FP16/bert-small-uncased-whole-word-masking-squad-0001.xml', +]; +const modelsDir = __dirname + '/../assets/models'; + + +const images = [ + // hello classification + '/repositories/openvino_notebooks/data/data/image/coco.jpg', + + // hello reshape ssd + '/repositories/openvino_notebooks/data/data/image/empty_road_mapillary.jpg', + + // hello detection, optical character recognition, pose estimation + '/repositories/openvino_notebooks/data/data/image/intel_rnb.jpg', + + // vision background removal + '/repositories/openvino_notebooks/data/data/image/coco_hollywood.jpg', + '/repositories/openvino_notebooks/data/data/image/wall.jpg', +]; +const imagesDir = __dirname + '/../assets/images'; + +const datasets = [ + // hello classification + '/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_class_index.json', +]; +const datasetsDir = __dirname + '/../assets/datasets'; + +const vocab = [ + '/repositories/openvino_notebooks/data/data/text/bert-uncased/vocab.txt', +]; +const vocabDir = __dirname + '/../assets/vocab'; + +try { + main(); +} catch(error) { + console.error('Error Occurred', error); +} + +async function main() { + await downloadAssets(models, modelsDir); + await downloadAssets(images, imagesDir); + await downloadAssets(datasets, datasetsDir); + await downloadAssets(vocab, vocabDir); +} + +async function downloadAssets(links, destinationDir) { + for (const link of links) { + const url = host + link; + const filename = link.split('/').pop(); + + await downloadFile(url, filename, destinationDir); + console.log(`Downloaded: ${filename} \n`); + } +} diff --git a/samples/js/node/hello_classification/README.md b/samples/js/node/hello_classification/README.md index 2de983af58334b..1d20bba8ac63e6 100644 --- a/samples/js/node/hello_classification/README.md +++ b/samples/js/node/hello_classification/README.md @@ -2,9 +2,13 @@ Models with only 1 input and output are supported. -Run: +Run sample: ```bash -node hello_classification.js *path_to_model_file* *path_to_img* AUTO +node hello_classification.js ../../assets/models/v3-small_224_1.0_float.xml ../../assets/images/coco.jpg AUTO +``` +Where +```bash +node hello_classification.js *path_to_model_file* *path_to_img* *device* ``` Other details see in [../../../python/hello_classification/README.md](../../../python/hello_classification/README.md) diff --git a/samples/js/node/hello_classification/hello_classification.js b/samples/js/node/hello_classification/hello_classification.js index 00ba868e40f3a0..9edbc998d006bd 100644 --- a/samples/js/node/hello_classification/hello_classification.js +++ b/samples/js/node/hello_classification/hello_classification.js @@ -1,7 +1,7 @@ const { addon: ov } = require('openvino-node'); -const { cv } = require('opencv-wasm'); -const { getImageData } = require('../helpers.js'); +const Image = require('../image.js'); +const imagenetClassesMap = require('../../assets/datasets/imagenet_class_index.json'); // Parsing and validation of input arguments if (process.argv.length !== 5) @@ -31,21 +31,12 @@ async function main(modelPath, imagePath, deviceName) { //----------------- Step 3. Set up input ------------------------------------- // Read input image - const imgData = await getImageData(imagePath); - - // Use opencv-wasm to preprocess image. - const originalImage = cv.matFromImageData(imgData); - const image = new cv.Mat(); - // The MobileNet model expects images in RGB format. - cv.cvtColor(originalImage, image, cv.COLOR_RGBA2RGB); - - const tensorData = new Float32Array(image.data); - const shape = [1, image.rows, image.cols, 3]; - const inputTensor = new ov.Tensor(ov.element.f32, shape, tensorData); + const img = await Image.load(imagePath); + const inputTensor = img.toTensor(); //----------------- Step 4. Apply preprocessing ------------------------------ const _ppp = new ov.preprocess.PrePostProcessor(model); - _ppp.input().tensor().setShape(shape).setLayout('NHWC'); + _ppp.input().tensor().setElementType(ov.element.u8).setShape(inputTensor.getShape()).setLayout('NHWC'); _ppp.input().preprocess().resize(ov.preprocess.resizeAlgorithm.RESIZE_LINEAR); _ppp.input().model().setLayout('NHWC'); _ppp.output().tensor().setElementType(ov.element.f32); @@ -69,12 +60,14 @@ async function main(modelPath, imagePath, deviceName) { .sort(({ prediction: predictionA }, { prediction: predictionB }) => predictionA === predictionB ? 0 : predictionA > predictionB ? -1 : 1); + const imagenetClasses = ['background', ...Object.values(imagenetClassesMap)]; + console.log(`Image path: ${imagePath}`); - console.log('Top 10 results:'); - console.log('class_id probability'); - console.log('--------------------'); + console.log('Top 10 results:\n'); + console.log('id\tprobability\tlabel'); + console.log('---------------------------------'); predictions.slice(0, 10).forEach(({ classId, prediction }) => - console.log(`${classId}\t ${prediction.toFixed(7)}`), + console.log(`${classId}\t${prediction.toFixed(7)}\t${imagenetClasses[classId][1]}`), ); console.log('\nThis sample is an API example, for any performance ' diff --git a/samples/js/node/hello_reshape_ssd/README.md b/samples/js/node/hello_reshape_ssd/README.md index 21d8be8ec4b50d..547cf989478a87 100644 --- a/samples/js/node/hello_reshape_ssd/README.md +++ b/samples/js/node/hello_reshape_ssd/README.md @@ -2,9 +2,13 @@ Models with only 1 input and output are supported. -Run: +Run sample: ```bash -node hello_reshape_ssd.js *path_to_model_file* *path_to_img* AUTO +node hello_reshape_ssd.js ../../assets/models/road-segmentation-adas-0001.xml ../../assets/images/empty_road_mapillary.jpg AUTO +``` +Where +```bash +node hello_reshape_ssd.js *path_to_model_file* *path_to_img* *device* ``` Other details see in [../../../python/hello_reshape_ssd/README.md](../../../python/hello_reshape_ssd/README.md) diff --git a/samples/js/node/hello_reshape_ssd/hello_reshape_ssd.js b/samples/js/node/hello_reshape_ssd/hello_reshape_ssd.js index 416e8bc8180668..ebcc8e83223853 100644 --- a/samples/js/node/hello_reshape_ssd/hello_reshape_ssd.js +++ b/samples/js/node/hello_reshape_ssd/hello_reshape_ssd.js @@ -1,13 +1,5 @@ const { addon: ov } = require('openvino-node'); - -const fs = require('node:fs/promises'); -const { cv } = require('opencv-wasm'); -const { - setShape, - getImageData, - getImageBuffer, - arrayToImageData, -} = require('../helpers.js'); +const Image = require('../image.js'); // Parsing and validation of input arguments if (process.argv.length !== 5) @@ -38,24 +30,15 @@ async function main(modelPath, imagePath, deviceName) { //----------------- Step 3. Set up input ------------------------------------- // Read input image - const imgData = await getImageData(imagePath); - - // Use opencv-wasm to preprocess image. - const originalImage = cv.matFromImageData(imgData); - const image = new cv.Mat(); - // The MobileNet model expects images in RGB format. - cv.cvtColor(originalImage, image, cv.COLOR_RGBA2RGB); - - const tensorData = new Uint8Array(image.data); - const shape = [1, image.rows, image.cols, 3]; - const inputTensor = new ov.Tensor(ov.element.u8, shape, tensorData); + const img = await Image.load(imagePath); + const inputTensor = img.toTensor(); //----------------- Step 4. Apply preprocessing ------------------------------ const _ppp = new ov.preprocess.PrePostProcessor(model); _ppp.input().preprocess().resize(ov.preprocess.resizeAlgorithm.RESIZE_LINEAR); _ppp.input().tensor() - .setShape(shape) + .setShape(inputTensor.getShape()) .setElementType(ov.element.u8) .setLayout('NHWC'); @@ -70,48 +53,51 @@ async function main(modelPath, imagePath, deviceName) { //---------------- Step 6. Create infer request and do inference synchronously console.log('Starting inference in synchronous mode'); const inferRequest = compiledModel.createInferRequest(); - inferRequest.setInputTensor(inputTensor); - inferRequest.infer(); + const outputs = inferRequest.infer([inputTensor]); //----------------- Step 7. Process output ----------------------------------- const outputLayer = compiledModel.outputs[0]; - const resultInfer = inferRequest.getTensor(outputLayer); - const predictions = Array.from(resultInfer.data); - const [height, width] = [originalImage.rows, originalImage.cols]; - - const detections = setShape(predictions, [100, 7]); - const color = [255, 0, 0, 255]; - const THROUGHPUT = 0.9; - - detections.forEach(detection => { - const [classId, confidence, xmin, ymin, xmax, ymax] = detection.slice(1); - - if (confidence < THROUGHPUT) return; - - console.log(`Found: classId = ${classId}, ` - + `confidence = ${confidence.toFixed(2)}, ` - + `coords = (${xmin}, ${ymin}), (${xmax}, ${ymax})`, - ); - - // Draw a bounding box on a output image - cv.rectangle(originalImage, - new cv.Point(xmin*width, ymin*height), - new cv.Point(xmax*width, ymax*height), - color, - 2, - ); - }); - - const resultImgData = arrayToImageData(originalImage.data, width, height); + const output = outputs[outputLayer]; + const outputData = output.data; + const resultLayer = []; + const colormap = [ + [68, 1, 84, 255], + [48, 103, 141, 255], + [53, 183, 120, 255], + [199, 216, 52, 255], + ]; + const size = outputData.length/4; + + for (let i = 0; i < size; i++) { + const valueAt = (i, number) => outputData[i + number*size]; + const currentValues = { + bg: valueAt(i, 0), + c: valueAt(i, 1), + h: valueAt(i, 2), + w: valueAt(i, 3), + }; + const values = Object.values(currentValues); + const maxIndex = values.indexOf(Math.max(...values)); + + resultLayer.push(maxIndex); + } + + const pixels = []; + resultLayer.forEach(i => pixels.push(...colormap[i])); + + const alpha = 0.6; const filename = 'out.jpg'; + const [, , H, W] = output.getShape(); - await fs.writeFile(`./${filename}`, getImageBuffer(resultImgData)); + const segmentsImg = Image.fromArray(pixels, W, H); + const resizedSegments = segmentsImg.resize(img.width, img.height); + const mergedImg = Image.overlay(img, resizedSegments, alpha); try { - await fs.readFile(filename); - console.log('Image out.jpg was created!'); + await mergedImg.save(filename); + console.log(`Image '${filename}' was created.`); } catch(err) { - console.log(`Image ${filename} was not created. Check your permissions.`); + console.log(`Image '${filename}' was not created. Check your permissions.`); } console.log('\nThis sample is an API example, for any performance ' diff --git a/samples/js/node/helpers.js b/samples/js/node/helpers.js index 5cbd6650495070..5d56d4c60a7139 100644 --- a/samples/js/node/helpers.js +++ b/samples/js/node/helpers.js @@ -1,17 +1,8 @@ const path = require('node:path'); -const { cv } = require('opencv-wasm'); const { createWriteStream } = require('node:fs'); const { mkdir, stat } = require('node:fs/promises'); const { HttpsProxyAgent } = require('https-proxy-agent'); -const { - Image, - ImageData, - loadImage, - createCanvas, - createImageData, -} = require('canvas'); - module.exports = { exp, sum, @@ -23,96 +14,29 @@ module.exports = { setShape, transform, downloadFile, - displayImage, - getImageData, extractValues, - getImageBuffer, - arrayToImageData, - displayArrayAsImage, matrixMultiplication, }; -function arrayToImageData(array, width, height) { - return createImageData(new Uint8ClampedArray(array), width, height); -} - -function getImageBuffer(imageOrImageData) { - const canvas = createCanvas(imageOrImageData.width, imageOrImageData.height); - const ctx = canvas.getContext('2d'); - - if (imageOrImageData instanceof Image) - ctx.drawImage(imageOrImageData, 0, 0); - else if (imageOrImageData instanceof ImageData) - ctx.putImageData(imageOrImageData, 0, 0); - else - throw Error(`Passed parameters has type '${typeof imageOrImageData}'. ` - + 'It is\'t supported.'); - - return canvas.toBuffer('image/jpeg'); -} - -function displayImage(imageOrImageData, display) { - const buffer = getImageBuffer(imageOrImageData); - - display.image(buffer); -} - -function displayArrayAsImage(arr, width, height, display) { - const alpha = 255; - const componentsPerPixel = arr.length / (width*height); - - try { - switch (componentsPerPixel) { - case 1: - arr = arr.reduce((acc, val) => { - acc.push(val, val, val, alpha); - - return acc; - }, []); - break; - - case 3: - arr = arr.reduce((acc, val, index) => { - if (index && index%3 === 0) acc.push(alpha); - - acc.push(val); - - return acc; - }, []); - break; - } - } catch(e) { - console.log(e); - } - - const imageData = arrayToImageData(arr, width, height); - - displayImage(imageData, display); -} - -async function getImageData(path) { - const image = await loadImage(path); - const { width, height } = image; - - const canvas = await createCanvas(width, height); - const ctx = canvas.getContext('2d'); - - ctx.drawImage(image, 0, 0); - - return ctx.getImageData(0, 0, width, height); -} - function transform(arr, { width, height }, order) { - const img = new cv.Mat(height, width, cv.CV_8UC3); - - img.data.set(arr, 0, arr.length); + // Calculate the number of pixels and the size of each channel + const numPixels = width * height; + const channels = [[], [], []]; + + // Separate RGB channels + for (let i = 0; i < numPixels; i++) { + channels[0].push(arr[i * 3]); // Red channel + channels[1].push(arr[i * 3 + 1]); // Green channel + channels[2].push(arr[i * 3 + 2]); // Blue channel + } - const channels = new cv.MatVector(); - cv.split(img, channels); + // Reorder channels based on the 'order' array + const reorderedChannels = order.map(num => channels[num]); - const val = order.map(num => [...channels.get(num).data]); + // Flatten reordered channels into a single array + const result = reorderedChannels.flat(); - return [].concat(...val); + return result; } async function downloadFile(url, filename, destination) { diff --git a/samples/js/node/image.js b/samples/js/node/image.js new file mode 100644 index 00000000000000..30ac35032d7ed7 --- /dev/null +++ b/samples/js/node/image.js @@ -0,0 +1,275 @@ +const { + ImageData, + loadImage, + createCanvas, +} = require('@napi-rs/canvas'); +const path = require('node:path'); +const fs = require('node:fs/promises'); +const { addon: ov } = require('openvino-node'); + +const codeENOENT = 'ENOENT'; + +class OvImage { + constructor(imageData) { + this.imageData = imageData; + this.channels = imageData.data.length / (this.width*this.height); + } + + get width() { + return this.imageData.width; + } + + get height() { + return this.imageData.height; + } + + get rgb() { + return this.imageData.data.filter((_, index) => index % 4 !== 3); + } + + get rgba() { + return this.imageData.data; + } + + get grayscale() { + const grayData = new Uint8ClampedArray(this.width * this.height); + + for (let i = 0; i < this.imageData.data.length; i += 4) { + const [r, g, b] = this.imageData.data.slice(i, i + 3); + const gray = 0.299 * r + 0.587 * g + 0.114 * b; + + grayData[i / 4] = gray; + } + + return grayData; + } + + get canvasCtx() { + const canvas = createCanvas(this.width, this.height); + const ctx = canvas.getContext('2d'); + + ctx.putImageData(this.imageData, 0, 0); + + return ctx; + } + + get buffer() { + return this.canvasCtx.canvas.toBuffer('image/jpeg'); + } + + drawRect(x, y, width, height, properties) { + const ctx = this.canvasCtx; + + ctx.strokeStyle = properties.color || 'red'; + ctx.lineWidth = properties.width || 1; + ctx.strokeRect(x, y, width, height); + + const imageData = ctx.getImageData(0, 0, this.width, this.height); + + return new OvImage(imageData); + } + + drawText(text, x, y, properties) { + const ctx = this.canvasCtx; + + ctx.font = properties.font || '30px Arial'; + ctx.fillStyle = properties.color || 'red'; + ctx.fillText(text, x, y); + + const imageData = ctx.getImageData(0, 0, this.width, this.height); + + return new OvImage(imageData); + } + + drawCircle(x, y, radius, properties) { + const ctx = this.canvasCtx; + + ctx.strokeStyle = properties.color || 'red'; + ctx.lineWidth = properties.width || 1; + ctx.beginPath(); + ctx.arc(x, y, radius, 0, 2 * Math.PI); + ctx.stroke(); + + const imageData = ctx.getImageData(0, 0, this.width, this.height); + + return new OvImage(imageData); + } + + drawLine(x1, y1, x2, y2, properties) { + const ctx = this.canvasCtx; + + ctx.strokeStyle = properties.color || 'red'; + ctx.lineWidth = properties.width || 1; + ctx.beginPath(); + ctx.moveTo(x1, y1); + ctx.lineTo(x2, y2); + ctx.stroke(); + + const imageData = ctx.getImageData(0, 0, this.width, this.height); + + return new OvImage(imageData); + } + + toTensor() { + return new ov.Tensor( + ov.element.u8, + [1, this.height, this.width, 3], + new Uint8ClampedArray(this.rgb), + ); + } + + resize(newWidth, newHeight) { + const ctx = this.canvasCtx; + + const canvas2 = createCanvas(newWidth, newHeight); + const ctx2 = canvas2.getContext('2d'); + ctx2.drawImage(ctx.canvas, 0, 0, newWidth, newHeight); + + const imageData = ctx2.getImageData(0, 0, newWidth, newHeight); + + return new OvImage(imageData); + } + + invert() { + const invertedData = this.rgba.map((value, index) => { + if (index % 4 === 3) + return 255; + + return 255 - value; + }); + + return OvImage.fromArray(invertedData, this.width, this.height); + } + + crop(x, y, width, height) { + const canvas2 = createCanvas(width, height); + const ctx2 = canvas2.getContext('2d'); + + ctx2.drawImage(this.canvasCtx.canvas, x, y, width, height, 0, 0, width, height); + + const imageData = ctx2.getImageData(0, 0, width, height); + + return new OvImage(imageData); + } + + async save(filepath) { + const destination = path.dirname(filepath); + + try { + await fs.access(destination); + } catch(error) { + if (error.code !== codeENOENT) throw error; + + await fs.mkdir(destination, { recursive: true }); + } + + return await fs.writeFile(filepath, this.buffer); + } + + // Display the image using the node notebook display object + display(display) { + display.image(this.buffer); + } + + static async load(path) { + const image = await loadImage(path); + const { width, height } = image; + + const canvas = await createCanvas(width, height); + const ctx = canvas.getContext('2d'); + + ctx.drawImage(image, 0, 0); + + return new OvImage(ctx.getImageData(0, 0, width, height)); + } + + static fromArray(arr, width, height) { + const canvas = createCanvas(width, height); + const ctx = canvas.getContext('2d'); + + const imageData = new ImageData( + new Uint8ClampedArray(arr), + width, + height, + ); + + ctx.putImageData(imageData, 0, 0); + + return new OvImage(ctx.getImageData(0, 0, width, height)); + } + + static merge(img1, img2) { + if (img1.width !== img2.width || img1.height !== img2.height) + throw new Error('Images should have the same size'); + + const canvas = createCanvas(img1.width, img1.height); + const ctx = canvas.getContext('2d'); + + const img1Data = img1.imageData.data; + const img2Data = img2.imageData.data; + + const mergedData = img1Data.map((_, index) => { + if (index % 4 === 3) + return 255; + + return (img1Data[index] + img2Data[index]); + }); + + const imageData = new ImageData( + new Uint8ClampedArray(mergedData), + img1.width, + img1.height, + ); + + ctx.putImageData(imageData, 0, 0); + + return new OvImage(ctx.getImageData(0, 0, img1.width, img1.height)); + } + + static mask(img1, img2) { + if (img1.width !== img2.width || img1.height !== img2.height) + throw new Error('Images should have the same size'); + + const canvas = createCanvas(img1.width, img1.height); + const ctx = canvas.getContext('2d'); + + const img1Data = img1.imageData.data; + const img2Data = img2.imageData.data; + + const subtractedData = img1Data.map((_, index) => { + if (index % 4 === 3) + return 255; + + return img1Data[index] * (img2Data[index] / 255); + }); + + const imageData = new ImageData( + new Uint8ClampedArray(subtractedData), + img1.width, + img1.height, + ); + + ctx.putImageData(imageData, 0, 0); + + return new OvImage(ctx.getImageData(0, 0, img1.width, img1.height)); + } + + static overlay(img1, img2, alpha) { + if (img1.width !== img2.width || img1.height !== img2.height) + throw new Error('Images should have the same size'); + + const img1Data = img1.imageData.data; + const img2Data = img2.imageData.data; + + const overlayedData = img1Data.map((_, index) => { + if (index % 4 === 3) + return 255; + + return img1Data[index] * (1 - alpha) + img2Data[index] * alpha; + }); + + return OvImage.fromArray(overlayedData, img1.width, img1.height); + } +} + +module.exports = OvImage; diff --git a/samples/js/node/notebooks/hello-detection.nnb b/samples/js/node/notebooks/hello-detection.nnb index f6139ec7ec8b98..60640b3bd042ea 100644 --- a/samples/js/node/notebooks/hello-detection.nnb +++ b/samples/js/node/notebooks/hello-detection.nnb @@ -17,57 +17,42 @@ { "language": "typescript", "source": [ - "const { cv } = require('opencv-wasm');\nconst { display } = require('node-kernel');\nconst { transform, getImageData, displayArrayAsImage, downloadFile } = require('../helpers.js');\n\nconst { addon: ov } = require('openvino-node'); \n" + "const { addon: ov } = require('openvino-node');\nconst { display } = require('node-kernel');\n\nconst Image = require('../image');\nconst { transform, argMax, setShape } = require('../helpers.js');\n" ], "outputs": [] }, { "language": "markdown", "source": [ - "## Download the Model" + "## Load the Model" ], "outputs": [] }, { "language": "typescript", "source": [ - "const baseArtifactsDir = '../../assets/models';\n\nconst modelName = 'horizontal-text-detection-0001';\nconst modelXMLName = `${modelName}.xml`;\nconst modelBINName = `${modelName}.bin`;\n\nconst modelXMLPath = baseArtifactsDir + '/' + modelXMLName;\n\nconst baseURL = 'https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/';\n\nawait downloadFile(baseURL + modelXMLName, modelXMLName, baseArtifactsDir);\nawait downloadFile(baseURL + modelBINName, modelBINName, baseArtifactsDir);\n" + "const modelXMLPath = '../../assets/models/horizontal-text-detection-0001.xml';\n\n// Initialize OpenVINO core and load the detection model\nconst core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'AUTO');\nconst inputLayer = compiledModel.input(0);\nconst outputLayer = compiledModel.output('boxes');\n" ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/models/horizontal-text-detection-0001.bin'", - "" - ] - } - ] - } - ] + "outputs": [] }, { "language": "markdown", "source": [ - "## Download an Image" + "## Load an Image" ], "outputs": [] }, { "language": "typescript", "source": [ - "const baseImagesDir = '../../assets/images';\nconst imgUrl = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/intel_rnb.jpg';\n\nawait downloadFile(imgUrl, 'intel_rnb.jpg', baseImagesDir);\n" + "const imagePath = '../../assets/images/intel_rnb.jpg';\nconst img = await Image.load(imagePath);\nimg.display(display);\n\n// Resize the image to meet network input size\nconst [inputHeight, inputWidth] = inputLayer.shape.slice(2);\nconst resizedImg = img.resize(inputWidth, inputHeight);\n\n// Prepare input tensor\nconst inputImageTransformedData = transform(\n resizedImg.rgb,\n { width: inputWidth, height: inputHeight },\n [0, 1, 2],\n);\nconst tensorData = new Float32Array(inputImageTransformedData);\nconst tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, tensorData);\n" ], "outputs": [ { "items": [ { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/images/intel_rnb.jpg'", - "" - ] + "mime": "image/jpeg", + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAIFArIDASIAAhEBAxEB/8QAHgAAAAYDAQEAAAAAAAAAAAAAAwQFBgcIAAECCQr/xABgEAABAwEGAwQHBQUEBwMHBg8BAgMRBAAFBhIhMQdBURMiYXEIFDKBkaHwI7HB0eEJFUJS8RYzYnIXJEOCkqLCJVOyGDRUY5PD0iZEc4OEo7PT1OIZdIWVJzY3V5S08v/EABwBAAEFAQEBAAAAAAAAAAAAAAMAAQIEBQYHCP/EAD4RAAEDAgQDBQcDAwMEAgMAAAEAAgMEEQUSITETQVEGFCJhkRUyQlJxgaEjsfAzwdE0U+EWJENicvGSouL/2gAMAwEAAhEDEQA/ALv1/D+mrU9hULp3EdolwdxxB7qgobOdQOVq9Yn/AGbfA7E9TW1jqr/pXq91b7qqW98vfUvOSA605Go22tbZ8fa/71u0J0gbRatBg9DSOJp4g2/QWR3YjUztAleTbqqJXn+yo4ePsli6cd4tpARlBdepaka8o7NuzTvr9lJei6ZdNc3GCqCDEesXAgnQ9U1U/LmbejyQRHxt2ArQCTY5pY0MTOXlje37Lvi/TBBu3HmHqkI5VFNWs5tD/I0sdBvtZr1/7PH0hrvcccYbw9X5hH+r3mWjpz/1hDfKLevIHMmeeot2LDNG081LjnmF4w1HoU+kvSNvLcwEl5ZJhLF9Xc6c07kJqCdieXOyRW+jP6Ql2LbNTwjxgtKVStbF0vPp2M95oLHIaz8be2a6dh0/asNrHigGxddyXO4cy7spSevZAG0O522KlxwvDauwBxAuB5X77wriKhGgJrbreZynX+dI8Nv0smJNQjMhyo74ChOpMg7wfjrb3dTcV2IENU6mo/7txSPuIsRvLA+Gb3SUXndrdWgjVNQO1HwXItE0juqQnC8PB61CApRTzJjWYnYeVjSVVIcbUgqUG0zEcx8iBPj8rex15ejpwbvYZavh1hpfPMq5KNR+JamzarfQ44D1qys4CudJUCO4w40Pgy4gWAaN6Mydo3XjxQvOhKmlPqH2rqTmBIKsx8OvP4dbKdO9UKZpEthUBgd6J3Agx0j7xb1Ir/2fnAV9KvUcKIpVKWVZmLzrkak6+264B8LNmu/ZwcL3UoRR11/UobASgNXw0pIgR/HSE7f4rAjpJI3XIVh9RG9tgV5udqsOPKIWpIIQQB7vdv8AdYBx57KEuKzSDlVlIMbaj4cuYt6BXn+zQusqUq68fXyyCdEuUlPUwPMONTy5WaN6/s0sZNoSi6cfMPZABNVdKmwQDMHs3XPHkbH4bhyVbMCqYKWorQS4ggGCDodjrHn15j3W6Qp1SlpDyoEDmQRIJnny+/xtaW8f2d/GmkWVU984ZqEjXKF1zZ6mJpoHxNmxePoMekJdqHPVcM3RWZ5ILN80zafCA8pB5DcWYsd0TgjdQGaioTT/AN6oEa97XKTy135fW4yHKtDqU9qojX2hsI/W0oXn6KvH+7cod4V3s+E/xUZaqxoJ2YUvn4WbtZwd4uXQpa704Y4wpUITGd25KpCTvuSn65WgWlTDrpqIWt9SgQmASCcp26+Gg+Xw7DrgCASMrgkBJkjToDvIsM/dd6UKEt3jR1FK4pRSUvMls94xpI+o8LcoacU4nOAvLuDrroOn1p5WGQUQboVLjhJSAqBqoAAxHkd/CzUvWqqWcX0QQSlRQkE7QIVoT7hpvoN7OtCVlRcSgGO6CI1Hx8febM/EBLd+UbqUhQUJCgrY5zEfAbdfG1eUXCLGdU76GtU7TD7QLhwpTyOsdP0sIagpAVmKhM6HfQxv7+nzsVutRZoxUgEqzwnUCNI+P11sKtZSlDAQo9p0AJ8dvH52hCLNsiyHXVbVUj2QpIA1OUkHnEe7x5Wxx1aU5t5IGYAHXff3ePnbSVIUtXdByawQY168v0Ng1KZLZW6gHPsCI1nQa+GvWxiNEJCLcfLhUVnQDcATvt8frmH2rmaXHezMlUeB6Dy1+HhbboST2crlROYgkDmdJ32J/rboM5kkmco7oGbTLO53Ez91hndSC4Ln93nUpCiQUjmPEbmJJ+duczql5lgpgbKIgDpHXzsL2S8wSCpStyRtt+Y0Fu22HUqJSQRlAiABtM6npPhaBcNlJYhS0biY1OxA5HXccrBPqU9TrSgkSkwCoxrET7wDYyEOBScyUgqMZYVqCdRPlpvbtxhwtrDgKwpBVAJ9xEjXn87BcbKYFwmRhmpeYXWoOYlRbKgBEjvEggeeh56eVnj6rTOhL4ZCAqVkEbp5nx0J+PnZn3W6tqvrmEtnrAXEEHrM7E+Pv0L+u3Mq7mDkOVxAmQdT16fXvsB3hddFaLtsk792MuZAplQJ5SdTpymenwtyu629crzomE6KEmfPnP1FnClpKEhRCEyARrAAjp7j9+lulst5R2bWYkTMkCfGPhZuImyJtm5/9iioUdBmBG35/jrYA3E4SSstqBHdzp021j3b+XhZ1GmaUSkgjpG5HXTy+t7Y3Slaxl1MTM7b6jp+ZNomXKpCO6a6MMKeAWpinWoQDqUyOp002+jbHsNLywijBhRMJX4bn4geVnuiiAjKkaSOkHmNbcLYKgpMAAxPOImfn+FhGZEEQUduXBUHRFI6r/ECJUJGkn+n4FH8PPpUFON1TQCtcyCIMQBr9RNpNXTlIJUnIUGRmA0P56H5+FgX6JzsFOBCjCTA10+6efztESlIQgqJUB5p6AM0EpIgkaGZjzO3gOeyi1QF9lClhtKuSp31289j8bavRnsalcpJBWYIPMkGfPT5ednphNmnqrkbS6w2qFrbUCnMd5A+cCzufsUzW8kx6rDLbbC3qm76VwJlUJbSSY8D7/iNptHWMbiLVKqqprqSyEqEFpvLpr0EQZHj7rT7f9LRFCqRphISDKykECZ9nT3/AI+EcYrogLnqUtp7qFhRO/ekfn5+VtKhlc1wN1TqWCxsoQLTzZlKFiOgItsVdYiQH3RJzEZjv1+6zkcpyT7a/tCdVDnJIOn1sdrFfVgtBQO9oSJTIgRJ193wt0YlB3WQWW5pFNbWaKLyjlgCdYtoVbs94A89RZTVSIyBwoSZB0CRvrp+nlbj1RKkSpjKNydhvtH1r7rSD2qNj1SaalZGoEnnbr1o5MhEgCBr9ePxsZVSN5ZUlSYE6a6bfX0LAPU6ENZkZt9ZtIFpTeJY447CXF5u8IBJ5W49ad73e0VuOth3vtKSmCie6lSB5hRP/ULFQyogmRoJ52cWKRuuxUlJzBInnbv11ZEZQJnYxYAtrHLxtrKomAk2ewTZijLVS2lQK0d2RpvYyq8myns20qy7ZiYMT4WTi24BqhW3S3MHpZsoKWYo76y2pARzHUaTJ3FtB5OoJEyNZixPWLZryFllCV0NUHNCpGvSwUzFtWy0homJQtNUO01S1UtLKVtLStJG4IMg2+kvhvik4u4d4XxU2sKTfFzUVfI59qwlf42+a8biLe+foIYmViz0ReGt5Lc7RynupV3rPT1Z5xgD/hbFuZ7Sx3jjk6Ej1/8ApaOHWLiCp+VUuRvYI1Du4NuTvbStbcmCtgNb0Wdu4T7Vs7dzUz8LcxztqCbGCnlC6YfcS+4gq0ISr7x+FjHbOz7RsTSSmoQdsyFD4EWHMzboaM5ogq8oAKG7Z06lRtoPu/zW4mRFsMnna0AhErrt3Brmtnbu8lbWD52zbnaQF026F9YdOmY2T7xecD9K8VnRZQdeRFjnw91iV6pWaUrbIlCkrnyNk4aJ26Fddu9Ht25NQ8P44i2gZTIG9uFD32YjRPsuxUukRPnYdNS6UTPIcrFI0MjWwrAlAHutXc2yI3VDGpeiCr52QcV3hetHh+9Lwbvc0RpaV2oStFPnKAhJVqDObbkBOtld5fZpkIKlKMJSOZ/AflZBxLhlvEVyV913tVPuIrmVtAIOVDBI0KUiMxBAPenUctrBcwyeEKZIaLqvtb6TdwIum777vC5Hq6rrgy+lhbDCV0ockF3OHVBJA1LaghY2UQdLN3iZifBF4JrUVGI78Xfq3E0jtGkLcNGcx7nYJhttSvaKHVoUEmTJ3auIeH1z4lwxfuE2r0dp8QYKfco/WnezUpTKUgrkhQBCyc0nUEZddU2rWzipVLSUGHnr6YdvO6K3KyENoU0wlDS1ZFFaSFpSvUEqKQqImYtnVmHiWBzSbaH1We6tkY4Ai4KngYnwGkBLlPifOBCovCmQJ5wlNIQPIEgdTvbLNxn0g6ltlDdVSXut5KQHFJSkgqjUgpWARPQAeA2tlvODhtbf3f8A9v8AhaPFh6r00fH2unU27bAUItw+T20RPW3aDpv/AFt9IO3XIsFghkjrHvt3AAtwklO9u0kWHZSXYEH9bbAjbblbkEKI193S3YPKySWwBO3vt0NtjbkRztsGySXUbWwTG58ranwi29Js1k4XQJne2wSeY00twPf1t2Dv94tEqS3MA6W34C3MmSAf6WyYmySXQI2Nt8teduZERMe+2TO+1kkujAOmltnvD5W4Jts+NmskuF01K5o5TtL/AMyQbBG67s3/AHfTiBulsA/EWMSAOnKbZmAHusrBPcom7ct2vI7J2nJR/KVqjfpNm9ePCXhve6iu88FXJVlW6qi72XifetBs7c2kZjbJjnItEsb0ThxHNRbeXoxcEbykO8NsNImT9nc1K375Q2k8zz52Zt9egj6N99uoqKrh7RpcbgJWzVVjMdNEPgTqeVrBzr52wq/mG1hGCM7hSEzxsVVyo/Z58Du+LuavigB1Cae815QY3+1S59RZtXr+zdwS8srujGl/0hCSEh19l8A6awGm522kWuQCImbZPLoLR7tH0UuO/mVQyv8A2a14pacRRcTXVqXr3rmQE+UipnkOVm1eX7OXiYktmgxdcjyWdU+sMVLc6RPcQ5yn5W9GQZETbJBM/jaBo4ypCpeF5fXr6BXHSgQ+umbw9XrI0DFc42VdD9s22PmLNS8PQ/473UhPaYBYdZbJUssXpRO6Cd0h2ddeXK3rbMG3C0NrTDiUqG3eFhOoWnmiNq3cwvHeo4BcaKdpVUvhPil1vdJprsee0Ame4kiCbIN4cPMa3PkbvjCV/UJBlRqbufZHiAVpHhrb2ZZu27lMJSugpiAIjsk/lbf7luoiRRpSTuUKUj7iLAOGki4KK2tA3C8T1MBLzheXBb2BV4anXXythpnCySh0qcXIToNR8NOWtvaerwtcVcCippVug6Qt1agfcSbNy8OCXC+9ZN4YMuV8k7vXVSOke9bRPzsB2GSciisrWdF4gFhdPfTzI1UtCspJ3Mjl9fKz/wAPhblz0yVKISjMQdUlRnbeNvhb1QvP0O/R5vZwvVfDO4e0IIzt0QYMHfVkoj3WRX/Qf4Glo093YeRRNme6xVVYVr0Kn1D5WBJhsxGiKysi5rziSD2WctaAADuwCeRjz+UeFhEUPahLJKte8VHyEyNxuOdr+13oD8MnVZ6C879Yyg5Ui8GsvwVTqPzshu+gPd1Mpxd3Yxr0FQyp7Snaeypjb2m7U34fUDYKwyrg6qlibto1OEqQgAAgefMkWGZu1lAAUkdo4ZlKpPu6jz62tzXegrfzYQKDGTbwSc3293hsTB17jy+fhys3rx9DTiRSl1ynrrlfWYCAE1Qnz+wI687U5KOpbqWlWWTwHZyrQaNtx09nmSGkgFSj7W5kGOXOwaqF0iO1IUuSMyY8tJHytPFb6J3F2myhvD1HUJTqpabwYQSfJakn5ToLIFVwB4qUJccdwBey1AQgUyA+B1nsyoHrvy91q5gnbu0+iIJIzsQohcu5RWpKMhABUpWUgDwHWRJ8osXdpXjTEEN96RouCdxG3OD8bSZX8LcdUK0t1WCsR0qUpCnFO3c+kdYkpj4aWalfc9TQKAqmalkpEJQtGVRPM6jx3iwjnG4RBl5KCsU0zjL6kqQjNmClFKpgRqT0P6WXMG3itm5nmwStaXjl0gQQNfHbeweM6dxpNSktDtnRmUQQTAI06RH32T8GEvUNQ2ZjMkgEzGhnfxA3tdY0uaqrjlfolioUVOFZEqV/MfGd/KRp1izQxRTtuUNal0SEtzmJ9525ab9Dpzh4LQsoKpGYmZjeOnWLNy/mu0aqUdmcq2VAa5pGXz1/XcWu0/hIsq0wuFEjzZglCSspBPtZp13j4+MneycQpQLWQ5VCSfEffEHz+FlRSsoSlJcBI5QoGefhtBkdegsmqKHESAdIkHmBz+ult9iyHIJ3u5ieQGpn5e4fpYN1KRlKwgKGh72/1pHlYQrTKlNrBV7KtfaHMjw0+VgFKXKc6CEq2GhMSeXLf52KAolaXkICey0He8hyHx+uhaqALTgTEggkk8/A/XOwy8wSCEqO8iZ25zYJzvIcCsw011nX6ixQo25INKpups5NGqhXe/zJGn/LbuG15UEBWYHUHWeW/kPnbinSV3bVoynuLbXoNvaH4i3KMywJB3g66H38rSKbktltmII3MEnQzt8jbMqUEJVEjoYiLYVkmVCJ8YERE/dbkgqOgGo73Ub/AJWSS2UBU8xpGuvPfrbSmkpJTJmYBI++3RK8xQpskHc89LczEFWhO08/ysklzlAJ0BIEeztblTWsKA12k2EzbnLmmCAVaQPvtpUT3ANehj6FnumsgFoSkab+dg/Owy092dbBAHUeFpAqBC0N7ezH7JXFKr79F6puNx3MvD2I6ylSn+VtxDTw/wCZxdvGcbi3p1+xnxOSxxNwa47oFXdebSJ/+mbWY/8AZ/K2PjrM9GT0IP8Ab+6uUDrTW6r0vVI3NucwG1tr2sGfPa3DhdC0XCE32tzEWxB2EW7AB2scC6bZBqGVbSjAhUfERYY6WAqP7on+UhQ9xsY20623KA/p2VeYc1mlt7crYJ520QCdvCbXwECywxFtDU23HIi2jPM2e1kitknb77BVCA4y4iRCkkRYUSdANbaUJGnjaYbcJr2SfTnMygkz3bbVJMxYKldQvtmEKSVMuqQoT7PMA+4/dYYyNOtoqV7Lka6WEY/iB1OhsGNz0t0FFJOQCSDEmAT0+hYMlgLlSjJvZDpbW4sISmVHQWLvVV1IafQ/XtJ7MFR74JSoDXQaz4WVKUXWy16zeVTTgKSSO1cASB5ExaLMR8TbgRjd3AhvKop3HmBVIbYIRnbGiw4QFOAAHNkSmf5ykA2zpcQFI3OSAeV/79FIjiuyhVX9Jhy8rt4vf21wJX0lO09dvqt4O1TDlI2t0lTZaW64lKZKM0EHMIIjQC1bsPcI8l5tY/xxjS6MOXLeLy00LLjwb/eBKiVFIAKyiDBUhJTmj2gDF4+L/EK5au42rmvPD9XeVI+UP01AxdqqenpRPcWXXWuxUtAhUZwiQUybUjxDwexbjYGvxNiynpO3fW8mvvJwUgLCRAAClAOEJ0OQEaEawTbDqMUdM0tdIGA7m2v23Qn07WPzBubyVgaK7vRhp6NhhuopXUNtJQlxdM6VLAAAJIqkiT/lHkNrZaGqTCuGaWlZpX8YXG44y2lta+wq+8oCCf7sfdbLckaB1/8AUv8A59la4v8A6BesLwPaanmbdpBjnpbToh3mJnlbtAAPzt9DO3XJN2WFRG9kyqqC2Xni66lLQJISqNAmdLKS5Inpysi1yS41VNE5QtKkExtKY/GxIxcoUhsod4b+mXwX4nX7Q4auDEN60V53ioN0tPeV3qaDqyJCM4zICjsASNdN9LTqxeFROV4gnlpaA8L+i/hC471uy+lOsVDt3PtVTbqWsqittQUk7nmkc7PX/SE3XY+VhGhpK5H7upi9VvVFK4whS1qGRLecDOISs5kynaCYMFyBw1CgHkHRSo29mAUIE2AfvLsl5Gmgop9okwBYGncIZnnvaOeOdJWXtwnxDdFDUOMu3my1RFbaoV2bzyEOAEaiUKUPI2GIxdTMhUgUWJ6SvQXaN6lqmkqyqXTPpcCVdCUzr4TZUbrG3Eg6687VI9HDgtWcKMcJvG66t1q77zpXaerpQtRaUQkrQqD/ABBSd/8AERzNrOitS1W+qkjVAWPfI/CzuiHJJshS84+00guLMJTvpNghedISCXonqg/lYlVuk0wE7qSPmLVy9KjGHGrCN4YUc4SX4xRodarnLwpnqFp9NUUFjsgStJKQJcHdInNrtYYiuph5vorQt1LDolp5CttjtYTMI018rMLCF+V1dhS48QX2wilra+hpn6poApDbriE5kgHUd5URZ2MP5kkg6EWgYraJxLdKMwRI+NsCjOhNmBjjE72FMI3/AItprqdvN66aKprk0TTnZrqS0lSg2lUGCcsTB32tBuG/TKevpxtlzg1jNhxUJKadYeKSTH8qbLhW3Kdry7YK2RV/NbM0gybJtHXLqWkO95IKQSFb2J1V6PNrqXe3LbVPMwmdAkKJ2nnZhGTsnMgG6Xs3Tbfe2FWu8C0K4B9Kbg5xLvZi48FcRqO8LwqZLNK7RP0y3YGYhPaoQCYBOUGYB0tLVNeSnQC4iDzA2sxjISEgKP5gedsJjnZEvfEdJdTjDTym0mozZO0dCAYAkCdzrYNGJ2nAFJplKB5pcCh7utmylSzBL+bqedsmNd5slUl+MVb3YJYdSYJlWWPvsoocSsSk2jZSQgV1tk8h77czHKwL9XT0ye0qX22kA6rcWEiTyk2iQkjClSf1tmfxsC3UMvJztuIWnqhQIPwt3mkSNpsyS6zdbbCup+NuJm2oJ5+dknWNHukTzI+dhM/ObF2v40nXvq++wgII3+NmCRQoV1A1tokg6TbknTytiTqQI31s6YLqdiZ+Ftheg59LBn5zbcgbCzFTBuhQrnGlszGelgkkbm25J062FayVkIFDcae+2ZzOlg+ZtgPjZJITNHP320tDbgyuISoHcETYMHnboK162YgIjSgTdt2LIUbvpyRseyEj5W5dui7Xk5F05KTyC1AfAGxgbaC2wR77QyN6I2Z3VNq9OFPDe+ypV8YLuWtKxCjVUDL+YdD2iTNmnU+ir6PtXnP+ivDjJXGb1e7WWCQNtWkpOk6WlIdbbBg2gYIzyT8R45qB709Bv0dryzKTgv1VZEZqevrEn4B6PlZl33+zb4HXqVOU15YkoSrkxeKIG+3aMuHmedrXZjpJkC2wrTQ+c2h3WLkFLjPtYlUDvj9kXw9fzC5OJeJqQFMJ9YFNVR5gNtE/HnZgX3+x5v1KZuHjN2ipJAqbiCEjXTVFUo6T/LyFvT1KhvFu0rGotPhW2KhfyXkBe/7I3j7TFa7rxvhCrSmcgWuvaWR7qZYB9/vswb7/AGY/pX3WhaafCtyXlGYJVT32w0I/+0Fsj4W9vgsH8rdJUN+VlZw5peE8l4AXz6DPpV3OV+t8F75qCJA9QfpqyYG8MOLmzQvf0afSFuQKVe/AzH1OlOpWvDdYUAdcwbKY9/nb6LVtsuj7VtCv8wBsXF13UTP7upgY5MpFkHvG6bhtK+Z57DGJLnTWMXzcN5XeoNHu1VI40ZChocwEc7I4SoNzty1t9Or1xXVUJKHqcqSd0B1YSfcDFmve/BLhNfxUb64fYfr83teuXWxUT/7RCrPxnDkkYgea+bMocEZtNNdCNOU20pRM9yATtNvoWvj0KvRdvxJFbwTwinNrNPc9PTn4tISfnZi3v+zK9EO9ypX+i+npFKMzTXjXNQfIP5flZd5tu1R4PmvCSCD3UxG2ttlLihCUmJ0jW3tTe/7JH0Y64f8AZzWI7ujb1a9zt/8AXNu2Y1+fscOGTqnF3BxDxTSAju+sLpaqD4jsmpG/OzGraNwUuCSvI9CDlzDQzrIjT6/C2dkiJKhGmo13FvTS+P2Nl8IBVcnF9xwnlVXA2B4SUVZJ/wCGzDvz9kVx2oCs3TjHCNcn+FLyK5pR/wCBhxM++yFXGU/AeqCqSj+ImT42BKSDFrhX/wDsvvSwuwRRYcuC8ttaa+m2gT/9o7I/LnaPr59Aj0uLiUF1fBi8qhJUB/qNbSVh+DLqiPeLFbPGfiCiYX9FXvUWu/8AsjcRqur0k7zuJTuVu+8M1TQST7S2nWXR8krtXC+/Rg9I3D6lG9eBOPWG0/7T+z9UtEf50oKfnaT/AEDW8TcOfS84fVN/3Fed1oq6167HPW6RxmfWGHGkg5gP4lJsGvyy0rwDyT07XMmabc17kKMggW4522VamLaHjbz4DVdKFnusIgzIFuBr4W2BrY7Eiu3EZ21IP8QIttpWdtCjzSLYk5hB3txTK+yUjTuLUny1kD4EW16A2JCrS6BCj3W3FtCDvpbYmNLaoCr3Wc5m2hlgcrbJ03tke+LOEyyCBIM25URtbek/ibYZ2FihRJ1SahIRW1CEpAzwvQb8rdq31GptzVAiubWD7aCkj527VKhMzaFksy5G4IG3Kx+6ruFW+X1tpyta7bq5CxFsdo+3TI1ccMJTv7/IWdvYeo0JbpU5lJRKdPaV1PvtXka06nkmfIWaDcpt3xQ3Ve7qziR5hm52VJbDbjvZpfeUYhZkSASEhM6qJkGBaMuLXD3hxdlwV9fT05w2mphp28aeudoHHgogqSMiVuPKMJ7mQ5tdbTS8GKTItbWcNkFAy5lZoiR46n4myPifCVFjK7Td+KqRD9A6tLhogshKinUBwpIzCYOX2dNZti1VFHIx0b2hznc7bfflbknZKWuBB0VSmlPXtThq7r0x5f13UjfaJXWMuPtKUlKyQy0ptxSFQ2YByLy90e1Brzih3CWIcbU9wVeNsVu1F/VyqZ2gqaPs1sNkpzuZE5glAyz3koACZCXIMXz4m33U1l10l1YBoFIYoX6dKKimYCmlpCi2tlkJICYSCCpRSjZIVMlNQ+N+M2g1fdVgm+Lpuy+H7qDFe3dRbdqWVtocYCHKhBkAdqhTeTKFDtNJFuIqaLu02Vri4deQPodlrts+LMdPJMuvYwNS11RTU/ol8SFNNOrQgqdr21FIJAlCachJjkCQNgTbLRX+4OMqu9dtVfIpDrTh15IWG/4cwKhBiJ0FssT2fBzkP/5uWfxz8v4C9jnxD0a26A0j522+gdqOptiRpJ3t7s46rmW7LgxyMWbGKgpdwXy2DGekqBv/AOrNnUpEg6TOwtCXEk8a2MUVFDhqruN3Dt4ZW+zqGFB9lCkhLneG5nMR0kTtYjHNb7xshPY5xsAq+cD+BGL+GuMbgv65MX3kKBuoZbq6NVQS08wSErSobHunSZiBGokW6vZtg3lQurbBXlcSkxqB3dPKxdjDNXR3C45QpbVXNMqUwlWiS4BKQT0kCzDwXjTifizEjd1Yu4d010U9Iha/XGq0LBWIGUIJkg7zOmWxjIzS7kMRv6KaaUxTb7J52b+M2/WMLwnvAu0yjPTtE62ctIwpLGRQ5Rp5WirEfFO6+Hle9hziFdte1dzhUaS82KdTzCmiZyOBPeSpO2gMgA6WUbgTome0gJ23Mwhp2mWlIgSNtu4q3dVUZ8TlCYhFK3OvPMo/daO3fSS4HXVS9uzi5yscAhqnp6J8uL/wjMgCfMjzs4OGtdeuLkvYtr7vdoU3m6XWaZ722mAAlsKn+LKkE+JNnkeL6pRxuAUiPkilbJH+0Ry/xC0I+kT6QaeCF44cp3eHL2Jqa+G6px19moCFUYZU1HdLas2btDzEZec6TNiC8KK5LndvS9KxqlpKYoU686oIQ2nMO8onQDW0Z4pc4WcRaqkqazG9wPCjbcbaSmtYcBzkT/Fp7ItAG+ykR1SrTvUHFDD+G+INzXveTFGtlN409GVBDTwUg6PJAkqTJjvQCNuYf11uq7FIVMxr1tGT+NcDYNuq6sB4cvOlrat9SKOkpqV9DikIJlbiwk9xITmOu5gDwki6AtVOhShEj32Y7papGxVV3Fd+FL7vDFLymbnpaSqevB0pUrLTJSpTioSCowgEwATpoLRThfiz6MFU8y3hvHFD2zykoaQ4y+gqUSAAM7Y1JI0m0t4hu5u/MN3xcamw569TVNNkVEKzhSYM6c+doywl6PuE7ir2rxew5QpeaUlSVdkhRBBBBkeNhSa7i6LFpzspqoT9mAgDLG4snOjMa9tWynCCfNtIsp0iUNtBAVoBFk1SAXbwTPtKPL/1abTZtZQfuoMwn6MWGsLYoubFDQp2qu6ahFShxlBCllMmNRsZg+FpaubGFLe991l3ULNUhN3rDLqqimWxmcgE5UrAJSAU96IM6EjW1UeF3B/HvDbGlwX1dGN73dudNY23VUD1UtbTjKyEqSRMHQzrMeYBtbspaNawsphZSoBUcpGnuk2ThlCduuyReLfCy4OKNysUN/UYfVSKU5TrzlJbUoQSIjcCLIfCzh/SYGwBW4QpFLS01UVgSSokgODNuTM942lhgZmQCSdLIaUpS/eLYG7xPxbT+VmGoNynvYgWULej5wovPAN6uXnUYmvq8EuU3YIarKtTjbYKgSQk8+6NfE2shTkBEEAWbGGqdtLDZSkeyPOzoQiBpp0Nq1iNCbqxodghiqzH4qKUq4Utg93t0KIGs2e5O8aWZnEtAXcZJ1hxJtNouDdRcU0sC4j/AHe47drildk/30CfZXGvxH3WTcMX1UDGia1T7qkLqnioZjBnMNttzaLqzGFdc3GsYMqZRS12H2Lyu1w/+kNPvJqEeeQsq9yrPHBztQqsadqW1NuevP6Hp2ysp96YjwsPZgBUhq64ViaR8OpmbGQZ0H62SLpUpTQk8hZVBgWThYpLlr2nJ5L5+QsLMaxYFsgLcGs5gY9wt2TPKbMBdJNvFGMjhypYYaoRUrdSVLlzJlHLkd9bD0+LqN65Dfbram0NtqU6gKzFJTumY1/UWaXEgH96056tHkep/OzIrMTMKwviJuiqgsUzL7TyQofZvNo1BHIxB8QRYLn8OpEbvdIVlsXEp87dwVM2HcYUWIu0SxSvsqaSlRz5dZ8jZaU8IBSPKbRZgt9VHVBLayQ418dRZ04wvx66cNVVayftQjKggxClGAfnNnpnGVlzuo1EbYn2CP1mOML0FSqkqr4aDqDlWlCVOZT0JSCAfDeynd18XZerPb3bXM1LYgEtqnKfEcvI2qDxIuviBfd2sNYAv83XUtqW86sNBRd07qJMwNyY128QXDwIxbiZVPc1fiIKbvJThorwSABmIWUEkDT+VWkCRpYTXvc3iHa9kQxNByDdWnW6lKcyiAneegttDza0haCCCORs0cd3dU4iwZel0UTpbeqadSWyFFPeGoBjkSAD1BItDvCfGLl3Vie3loOj1SrbUfZcSdCRyIPyUbWnxZWB/K9j5Ks05nFvNWQDiSqBbqQBJMWgyqxbed94pQunvWrYo+1S0hNO8pGZsHU6Eb6md9RaYUV6V06n8xCInvdLV43CVxDeSNJGYwCeaUA6Npt2FAiRHhaBeIPGupuC76zELL9Yi66SAhFFTh59+SACB4zO4AAk7TZV4QcaWsd0FBeLNWqpu+8wQy462EPNOAlOVYTpuCOfIgmwjKL2tpe10UQuAvfXeymYK26WwLAgmJsj35frNzXPVXq+kqRStlzKCAVRy9+gsyl8YKKmDZvCkYpw4cqM9aE5zEwJSJ0tOVzYQC/mmia6a4aNlJwUPL3W2FTZsYWxtQYnU8zSsqQplAWSXEqBBMbizjSqedmY8PF2qTmFhsUMF6aRbebxsDm00Pvsi1OMLhoqpyiqr4pWnmzC0rXGUxt0tImwuVEAk2CcIX4267TlrZJu2/bqvRRbobxpahYEkNPJUQPEAzY+XAEk9LR0cLhIgjdGQsEaG3Qc87FG3Q4mYOmltioSpWSdRaNk4dbdGguNfjbfaCJBsB2gSnMT520lwKEpMi0HAqV77IwHDoOVug6ORsWKwNSdLbCtgPvtGymLI12pnU6W32s7WLZo0m2ZoG5i0UrAoyVk24JB3sBnJnWTbO0nS0SErWQqjPT4WBcS05IU2lU9QDbCsnYC3Kj4izFo5qQNkWcu+hUcyqNgnr2YsA5dNCuCWimNgHFCPcDY6penP324Kj1sMxNI2RWvI5og5dFNlIbdfR4h0n75sCbqXMt3jUf7wQR/4bKSvPa3A9oePUWAaOF3wowleOaTV0Vcn+7rWj0zMT9yhbnsrzQdfVV+9SPwNlJY03iwWnvtDuER5KXHciYVXo1NG2SP5XvzAsQumpvdx2rVUXe00pTuYN9tJAiNwCD7PzstGedirY7OsWJjOJjyj9bJtI2IgtT8YuFiF0l2qjvUh8crgP5W2agpmaV8R0SD9xsPsJ5G28xHQWthh6oOZFjWsDRQdT5tK/K3IvGhKoVVIBA2On32NnXU21AIgj42bK4c09wghU07glD7avJYNuiZ2I+NuHKWncELZbV4lANgF3XdylKPqbQKtyEx91peJRJARe93006WH1BUJeSNBsCYtwKk1Ci3TQcuil7pT+ZsSv8Aw829ddSihedYdyKKFlxa0pMb5CqDz0sfo0JRTNJbEJyiABA26WXiLiEjlAuE4rkugUKDVOJJed9oq9qPwstKcSlkvKCiEAkhKSokeAGpPhZGpr9ZRTIRUZlOAQTHTY2Zt78U8WYXDlTeHDh28KMvFDT913gh4pRBILqHEtqSdDojPrAEkgWo1dXFRsMkxIA52JQhDJO6zRqn8+yoH1+oKm+5AQVd1tPjynqfdytEfFbGOJr/AHEYLwNSMlKSl2vqKxakN1TWaFU7aEguLSZBWpMDKCnMCrRuY746X1fFUjCzLFNcbrikuFovpXVvNgahKTATBIBJSqI2HJpXxj2uuK4n0U123I1U3k7kqF1iy6p4oSIU44rMpzQgDNISAQJ2tw2JY/NUPbHQg8M2u62p672stinwtzGZp7X6XQOJGL1RcdXduLcCVfaPBZq7wonGbxbJgAKNPmQy3DaUpSghRSAhIB7pNauINw8J8Mm7bwwtW3qugvFxLLpW0W3UvaOJcebXlSlBKoSlKEiFE6zaY73x/iRFEf3HeWHlVT6FNoRTXc0ptDkEpUVhsJyJMGSoa6QSYMNX9iKiqr27bGr9Vie+EEIStkqpbsIlJAKEJ7RWaNYDbhKT3VFOmTJO6STQG3RWZImxstcEqJ6ziHxOoKt+hpMNYmqGKdxTTTq+yWpxCTAUVZBJIEzAm2WnRdXg5S1KYwuA2SSjs7szoy8sqlZlKHQlRJ5k72y0Pa0X+x+/+VT4D+q9HK1ID5002AFuECdbD14+28zysEkaAje3v+65FYEnmLAP0TbypUmTY0lMxIt3Gm1mukiaaVKUZQnfSxZFzU7b5fQ0ArmQNbKuWPdbeWNxZiE40QCGsqYsmXzh6ivhos1bCHEq0IUkEGy0RbYRPKzGx0KVyNlGzPCTCtPV+ss3PSoWFTmSykH4gWeN3XU1RNpbbSEpSNNLK5QneLZk8LQbG1puAiF7njVJd93LRX5dNRdN40zVRTVKOzcacSFJUnoQbQzeXoxcPKl0rGH6VJJJ7qMv3WnuNY2i2lNpVrFk5ufmQosdl5KHsD8DsI4LrfXrpudhp+IDmXMuJ/mOvO0pU9Lkby5YseDCQZCBbYbA0jeyY3JzTvOc3UC8SeFPEC9cTVl9YW4o3/c9PVZVCkZdJabUEgHKkECDE+ZNmuzw89ICjVDXGa8lJ2lyhQ5/4jr8rWfXTocPeTYH1Fon2bJ3EJ0d+yk0sA1aku4qapRQMN1z6nn0NpStwpAK1AQVQOptHHEpji9SYlQvAlfc6bqqEIS81WUylLS5JClBSdxGXTqPG0wIYCNrcO0jb0ZgNPCxRI5vuoRY07hR5WYWq0YYqP3e22quaYK6cKkJLidUgnlqBZtcO8R8TMR3w5SYzwpd91sUqCpLzFSVKWskDKEEmExOpPIeNpoFIkJKQNCLF27saadLiUJBO8c7J0rzZIMYNVqnZKWwmZ05WijHPFCiwNit6467DWIKxValt1p+hou0aMjLlKiod7T5i0xJbCRBFk+8bjpq91LrzQUpBBEizGQtGgT5Gk6pMw2y6mmaUtBHdG/lZxIiwNPSpa7g0FjQSRzm0C6+6f6LShzizU4hKabuJx2ocQ022tJUtaglKdeZNnYYsh4uwzdmLLhqrhvilRUUlWkJcbWNDBBHzANptcBuolt9lWbi5hb+0NPRYtw0609emGyirplMrCs6Uk52iRyUhR06xZwYUvBurTQXi1HZPlpaOcTH17rSZgvhXhzBF3O3Xc13IZZdcU4oanMowJJOp0AHus0mvR2w5RYzTimmerkZaj1lNOH1BoLmQMo5Axp4RZnyMcwA30U2scHXBCk5m86e7KL1qqUoIGVOgkkkgAD42WKG8qeta7ZkrAnUKGvWzVxlharv7C1RdV2Xi9d1UooU1UsgFbakqB0kEco99g+GtwX/AHBcvqOI7+qb2qi4VqffQlBgwAkBIiBHnqbQc5ugCQaSLp7oIDqyD0sIYiB52DRHan/KPxsJrGny1sgbJlHvEVsGtoTqJSsH5G1abvYewpxvxVcN6OKduPiA0iupQZCUVbTCGX2v99AQeWxtYrjeMVU1zUt4YRuFF71bbxQthTgQA2RqZO50A352j29eHWIMb4GpLxvG7W7vxBQuJvCmbQ4Fll5JJCMw01T3Ty15xaMkQkmu46EW8wUeKUxxWA1BunnhOWXaRBWV5GuzzEbxGtl3iInPg2tVPsBCjH+dJtGnC/FGJ7/xCbvvDAd5XS3ToUXqipWAnMDAATAJJmegFpixDc7t8YavC7ElIXUU60IJ2C47vzi0aBhibleeaeueJHBzeijLBFO09UPIcAPdGh982bTVM3d2Jq9lgZQ3eJWlIIG+VRPz+fwRcP8AEy7bivF27b+fTdF70Us1FPW9xKj1B25Tv5SLK1y3xdGKMTFdBeVLUv1b4fcbpnA5lACQToTGiRv1tGcmGEwuGt1KJvElEoOllYW7yXmQlQ9oSbQZxCwyvC2PEV9Ex/qF/hanUpGiKhIkn/eE+ZJtOt0sqSykqSQSOlmbxkp2UYfp612Einq0qUTskEEHXkLXYiHtLDsQqjyWuDhyKaWA7sFXWpeCISyAkCI1/paVL3aW3h+tSgwoUzkeeU2YnCd5itpqlTS0ryPDVJB3SPyNpNqmQ9RONEGHEFJ99qFEwM1Kt1j87rKBLnuakv6nprtq2kONPISopIme7NiuA+Bt54Txg0u7sVVjNwU9Waxq7W2UBMk5spWQTGbpGgjxsewheLVI+2HVAu3e+umfSN0KBKdRuOtlurxXiF6/akXbWFijZDaWShCVdpIOcmZnUx7rQe0QGQOOt9kZhdNky7WT/wAdtF7Bd7J3PqjioPgmfwtA2LMCYf4i3PRUOI6RL7NO4mpRqQUqyFMgggjRRtYS+0+v3DXMAhXa0riSOspNq+X1hZeOcIUlzovK8LtWvsHA/RP9k6CncBQ5HW0q8foMcDazv3CjQn9V7SL6KRuBeB8H4Boai78M0iWn31BypXlVncAJCZUSdBJ08T1tMCFAjS0O8DOHDuBaasdqsQXxe79UUp7S8axT3ZoHJA2Enc76C0wtwIEWFFfLqbqUpu7QWXSzCSqBpaueL2K5yoxExQVPY1ZeqksO5QoocIORUHTQwddNLWLcEII8LQPithTOLb6Z2BdSsH/MgGxpm3pJD0sfyoU7rVTPuPwom9HzF/EWsYvJGPHQu+LjvINtVTdOGe0QUgiQmE/orW1wq6rd/ddS7TLyudgtSD0VlMWp1wu4mYsv7Gl64Oxfh2koK26EJq2HqZa+zeRngaL6778iItbS5a395XMzVEAdu0CoDkrmPKZtVpvDOQ4WvY25KxU6wgg3tcXUY8KL6ql3rRVT9W8760y424VrUqVQTJnnmTZOv3H2JWOIl7tJvaraZu6raShlt0oQWsoMFMwZk6n8NC/D4qo65uiUqDRXk9TGdP4yPuNknHbAo+Kd8NgkGtpaapGnROT/AKbWclu9M6EkeoKCTmdA7qNfQqx98Xv6hctbeaEhz1enceCSfaKUkx8rMThdjy/MQVqWr2qmltP0xeQlLSUwvunQjlBO/wCFnJQqF7YRZKiT63QpB/3m9fvtEXB+rLdddCFqMhC6dXmAoR8hYUv9Bsg+YflThH6pYeh/ClniTjheDropaxqkFS9VVKaZsKVlSkkE5jGpGnzFj2BsVuYquU3g/SIp3W3lMqQhZUNAkgz/AL1mNx+aWvBbFakEmjvFh0lPQkp+9Qsa4NvqFFeFMDol5Dm38wj/AKbKpbkbG4c7g/ZRhGYvB5WUp5tJJgWaD/EG6mrxqaBAqnV0zhbWUNjLmAkgSRMbWdWYlo+VoHraZpWI8QXW6ohDlQ8hcEpIDgmQeWhsKoJjpXys3bb97KcQDpmxu2P+FLl14zuutqE0peW24vRIdbKJPSdvnZwFYgkjYWptga5MUcOuKNXhC8MSXneV1V1CirovXX1OlohRBAKjpsqYiRl6Ta113166+5G3s5Lq2ilRj+ICCfiLAhdJxBFLbUAg+RRZGsDC5nI2QtBim4rxvB26qW+7verWZz0zdShTqY3lAMj4WUKioSykLV15m1P+ENcbs4q3U84YUt9ynVP+NC0ffFpW9J+sqk4GpW6d1xLTtXkdCVQFdxRTPUSLaXdCap1MDsqgmHAE3VTQKlCmi6NR52xp1LqSocjztSHBHF298J4GxHhlu8HGDUsioonkKOZp0KSFhJPs5kbbQU6b2kL0TsXXtX35f103tfFdWBVMh5hNRVLdSgIcIOXMSBIWnbpau6LLEJfOyKJLyZLcrqzi3ENpzKMCwYfaV3QseVos488W3uGuH2lXZSt1F5VoV2Ad/u2wClJWoSCYKhA0nrpatN1+krxfob7pUXlfgfRWoDjDdRdzaWXo3CSlIJGm4V4WC4ZCARujtOa5HJXqJ3iwcmY1+Nmxw1x3ScQsI0mJKdsNOOZmqhnNPZPJ9pM8xsR4KFnOZJ0982YG4upWI0K0TpYBaQKhtXXSw5sDUEwhZ/hVPu+hZiEhoULyg7W2OpFsgeVtDQWkm3Wz+FtGPxtsxbRjysklzy099sAG8aWw+0BbfLeLOAoE3QT6M6FIOx08bJd2LUujQD7TZLavAgxZYKQQfvsjXelbVZXU5QUgPB1JPRQn8LORqCkDcI3zsKxTJrWKilISSUZ0Trqk6WDI2jnbpisp6BZq6peRlCFZ1QTpHIDU68hqdhNq9ZE2SFzX7WSY4tddqhniRhbD143k7dV9XVR16ajI96u8hKikAZS7CtgkzqNdY52jOvwLetY0wrCGOr0u5EuEu1bKa9socVmJbS6QpCU6BOVaZAG8ZrTPWXRWXled84irxUUZqXQxT077Ce0ZaQhJQTvBJVmyzAJJUCRAZ2Kqegw/hWqqL5YcNG42pIVUBZ9ZUUmEFQEkqOm0axoLcuGxTh0j2hreS0XPdlGtyojvPC4xxfT12VN90JumgQH2001OA7VMKAGd5/MokLV2gGUoTAWQkQCqrvEa5r6fxRe9z4rqTddxpqVUFJV1N6rWt2hSv7HMlJU6lgZEqg5UGCRBM2t7dFNdd0XCqrvGqup29rzfFdXBx1CWqZRAgBH+AAJSk8xPIm1M/S4wPi7EGPX2MMvu3hddV6rXNGnqszBdUgNrWsiQpcok7kZwAABFhU9JTMcA0ht9z0VeWRwZnOqkWixXh2moqemp+M16sNNNIQhpm63ltoSAAEpUTKkgaA8xbLVcpMAY6apWWximlZyNpT2YVIRA9nQxptpbLVjglJf/AFP7f4Qu/SfKvoEvBMPTE62AQkAzEWjJXpNcMa5wFKr2ZkxDlIlUHxyLNjjHH7hY4QF37UNHnnu6o0+CDb03vtOfjCyjSTjXKVIoEamTboJ5RttZmMcZeGFSQG8YUqSqQA4063/4kiyizxI4ev8AsY2uQHoqtbSfmbSFVC7Z49VAwSjdp9E4x1tmnI2TKbFmE6whNJie6HyZIDda0o/JVlFqppXxLNS05PNCwfuNiCVjtio8N43C7CRz2tuBprbYBPl4W3GmvOz3B2UcpXIE6xbUbQPK3cC2FMiPDeyulYriATG1thM7i3Uaz1thERAsk65jprp8bZljXrboCOdsg8gPGySXOU7wbaIFhBPO2ojU2SS4CegtoDwsJGvnbUDysklxEn5b21APK3ZTrpbRGnKySXOWbajmBpbsiNbaAOmlkdklxHO248LbAJNsIFo3SWoi2iARFtkAWwxOhs4KSDLSCdtRbSmUkju2FjlbCOUWYpIEspIjLpbQp0o1Fh9+Vtc7MkDZAAQ9BH8H42F/G2iPtkx/Kfwt0Brz1sk5QD1Ml9JDmo8rF03e22kpSjQ6WUIEW1AsrJrpLRc9Oy8XktpCj0FjYYATE2NRO3lbRAA2swFtlIuJ3TJxXw0wrishd93FR1ixsp5lKiPIkfU2Awvwxw3hVR/c1z0tGknUMtJRPwGtn5CToQTbMoBmPjaDow7VFZI5osi7LJQkAJ2sk4uwtdeL7iqLivmlD1LUxnTJGxkajxFl4pk6aWzLB8D4WkPCLKGl7lMvAnD+5MCXebruKhRTsFwuqAklSjzJOpMAD3CztLYiCPfYfKBy3thSOQ3tBtwk52bkoP4j8EbwvG/nsW4HxG9cl5VA/wBZR2QdYfPUoMa9T74kkkrgnh5julvIO4sxGzWttnut09IGQT1UZJPkIFp4LSTopMjxtx6q0NQkWhM0zm7iiwzCEWCTW6E+qKZP8Scutq13hcXHG4apygu+5sP1dEyvKypanc5bB0JMgT8tedrUdkBpFi713svd5SATztKQufHw9LeaeKRrHZlBPCwcTqjFTCsS4fu+76BlKluONVClLWYISkJ23M67AdbT62DA3NizV2MsrzIQPdY4hJToLAiZwxZFkfxDdbIlO/haAOMd9U2EcWLrKu7rxdRXMIUFUtKp0ApEEqI0Gw8bWAKZG1k687npLzATUMpXG0ixi48NzLXuLIbG2kD77Kq1xYhwhe2I0Vt23fXqvh1j1QRQvJPZ580KMZRrzVEdetlcM0S6O56elM/Zog+Z1PzNuqfCN3MO9o3SoB8BZdYpEtJygaARanDE5kmd3TrdXJpmvZlCrJiK/KfAfEm+Lovl31RqufF40LzhhtzOdRPLvSP90+EgYgrWMQ4rZxGipQmKFNGptKgvOrOVAgj3iPH4z9jbhxhnHFMinxBdNPWdlPZqWnvInfKrcfGzQw5wAwPhi8mryu25kIfa9hS1rXk8RmJg2PVylxc6O4LhY7WKHTNaAA+xynTqnhhKlfpcM0FLUAhxthIUFbp8D5bWhfBxF14qdpFLCfVL8eaAHJPbEfibWJYpQ0yG4G0Wjp7gxci8cOYvcerlFx71j1YVKksdrM5igb6666WiXAU3Cd5fhJo/X4g8/wAo1xjpPXeHF8ICdWmkvjwyLSr8LNThhf1Ldb7q6qpbbbqWUKCnCEpkHqT4m0r3/cjV93FW3O7KU1lOthRGpGZJE2gB7gpxMoV9lQcQAWkJhKHLuQrbaDM/0s1TK3gsF9QSefP6JU8ZMjjyIVh7svmgvRtSaSupnlJElLbqVEDqQDaHcVA0nEq8Wtk1TTL4HIwkJJ+Is4OEeC8T4bdqa7FGIE3g+62GW0N0qWUITIJUYJJJgdAIOmuiRxiw1jI4go8R4Ru6krf9V9XeQ+8pEQokEQDO/wArMyVslNI19hcH/jdIxFszC3kU3cfVDC8f4IrGWzndbrGXCOY7JJGvSc1piwmvNc6G9SEkgSPGbQncrHE6tvBinvXB9EyAYNQasKDaecAJJMwNLTzcF3+pULbIT7KdfE8zbPjnfPUMcRYNbbe/O6tuiEUThe9zdVKeAw1xXVJyJor/AFK6AJD2Ye6Dad+P93/vDhpWPBoLVRvMVAA6Zwk/JRtFPF/B9/ucTqtu57irqpV4OtvsraYJaBKACSvYQpOsnTe0746uypvXh1fNGtuX3bucISnX7QIkAddRbojOzv7ZQd8t/QXWS2NwpTGR1Xn3ftYbuvWgoH0j1a8TUUi9IhyMyNtpCVj3i0w+i/WLu7iNR0jjgJqaR+mJ2JypChPn2YPvs06Lh5WcSlXpS3RSrefoGFVlK8EnK3UoUCElWwKgFp95PKx3g5UVl3cSsOOMsrU6qtbbcZSk50BRKVynlAKiegtWqWBocGciiwuNwXcwpY9LiiccauioB7qqd9M/ylK21fdaIb1fwtfPDvCF2VV41VLfNKmpcDrCEHIgPKRBKtJICYAk6eU2K9Jy53KzA9NeyEHLQVMOq/lbcTln/iy/G1QLzuK4b/w4u471oSa2kqUPUbwBKS2VpLjZ5xAJ6mTtZ5WtLYpfIi/Q/wAKlGSDIzzB+quT6NNRhZjCNTc2HReRVTPJeq3K0pzuuOJgKGQwBCIgAbe+0xe61a/RfvamVf183fTkBLtKhYAEAdm4Ux7s5FrJjQcrZcTrt1V6UAHRYSOf32AqhmZUNOXwsMTrbhYlBSIkgxYiDm1WNrK2kK6pBsII8D42BpyFMAARqR89LCwroRZwE91s21Bjb326A0g20Tl2tKyiSuYBVMW3sNLb3E21ykDezgXTFcxqYFkp0djfAg919nXzSfyNlcmdhZKvRLiKqifQUhIeKHJmcqknb3gWk4aKLSbowRAg2T2Ev1N8+tLATSUKkhkA+29rnUfBIISPHP0BsokHlbjKRyG8/Ow5Yw8C+ydr7LvGdEhTYVplLfTSR9C0J4wu2mW5QU79Q3UJ7cut0TErqKp5CcyG0iQIEZlE9wAakCbSvjS9MVLpWmLlwgK5KmQfWX69DDYVqCnQKV/CDtztFmOafElx3JeN6Ul3q9ZNGhVT2KF1JLpABbQoABDQWUnUAGFLOaABxWNYkyJggidlcfLofNaFMwubZ2qiS/KGpvrGBoMc381d131LZ9ToWVkBLTaAVl6pAISDmzFKMkCMyyBZIviguvEpv/1LBN03w4puoU9elJkcTd4SUpbCUalS1DM4ZDclQSP5rOPGF13jeDbNNft5LoLu7MdpSMinFStRITlSjOpEbqzJUSmQIJEGHMYUzhvN5q7OJCaO4qVCHEUdNUFpa3FJKjCSMzhSkI1JWSqAkTojh6yaorMzJTa3Mixv9v8A6V4MjA8IToprs4bMUzTD90XitxtCULUq87vSVKAgkjszHlJ8zbLNym9H+jrKZqrreJ1bTVD6EuOslxxRbWRKkyNDBJE+Fstl54v98+hUO7u+T8owmsfZcICjERqrbX7j+FlmivntIC3O8AJJ+ug+tbN2pCu1zoVKo1PLTwsNTKSAEkqSQDsTI57+/wCFvYHAOUmnVPRitQ6MyVGJ1P19bWOtvQkGTJgmPrzs0GKp1pRQZgSnQa6T+dlenr3EaFRKSPamD8vo2rubbZGCWezpVqIVTNEHmUja3aAw2oFDLaVcoSBr9fdYo1VJJAIzDaNtrDpey+zz3ObUifnYWoU7ApTp71raYZqevqmRMy08pPwIP1NlBjGmLKYhLGKb4T4CudEdf4rIGaUgKBMb89fzsIFKju5Y9qSdrSEjxsVHhsO4Top+I+Pqf2MXXvOvt1a1j5k2PU/GDiSyMreK6k9M7TS+n8yTNmSFEg6nfeZ/p+tuiSQCFGDMg8yenW0xUzN2cfVRMER3aPRSIzxz4jte1fLLw09uja/6Ug2UGfSEx63o43dLpEHv0yh80rFouJVIg65hEb23mUCM3MQCN4ixBXVLdnlCNHA74QphpvSMxUEkVFx3S5z7gcQfms2U2PSKvAEJqcJUyidJRXKTr0jIbQe0qFZjoJ5jb6+uVj4JTAKdY11+tbOcVq2bPSGG0zvhU3s+kRSE/wCsYTeRoD3K0K+9A+hY416QmG5+1uC9kx/KWlf9YtA6c2TMI0gAwNfgbdQRHIRr4fXSzjG6tu7vwonCKY8lYFnj3gtyO0ob4bkfxMNn7nDY6zxs4fOiV3jVs/56Nw/+EG1cBJiDOoI0ts94QR4b72KMeqh0QzgtPyurMt8W+HjsD+0aEE8nKd5B+aLHWuImBX9G8V3dP+J4I/8AFFqsBR1BXA315nmLYFZQf8R57WK3tDPzaEM4HEdnFWzaxdhN7RjE90r/AMta2fxsbavK7KkA095Uzs80PJI+RtUMx3cyhodlDy+Vs7MAQAhR2ggRYg7RP+JigcCbyerigpUJBB56W3lPIfG1O0uraMoWtvrCog2Ms3zfFPBp73rG/wCXK8tMHyBsQdohzZ+UP2EeT/wrdFJki2ssbDQ2qk3jHFzRQGsTXwIjatcjb/N9eVj7XEnHTMZMVV2n86grT/eB1sQdoYubCoHA5OTgrP5TMHW2ik7jytWxri9xBZ0OIe0H+KmZJ/8ABY61xux417VRQPf/AElKNf8AhIixW4/THcFCOCVA2IVh8p6WwDTa0Cs8fMYJGZ277ncA/wDUup/95Y4x6Qd8pI7fDNEsEEAoqFIn4hVjNxqkPP8ACGcIqhyU0uJPbNmZkED5W7KVbHpaF1ekM+p1n/5HNqgkri8iI0j/ALrrZRZ9IO6yQKvDFWgk69lUoWB8QmxBi9GfjQ3YZVN3apXiNbaA1tGzHH3CTpAdui+W5P8A3bKh0/7yx9njfgJ4AuVFcwSf46RR/wDDNitxKlds8IZoKkfAU+oi2Qf6WaDXGDh04Qn+0BQf8dG+n55LHGOJWAqgAIxVQCdB2iij/wAQFiCsp3bPHqhmlnG7D6JxgTuLYNwPwsks4ywe+qGcV3QszsK5v/4rHWr4uiogsXvROf5KhB+42IJozs4JcJ7dwUZG2otgTvbG1trEtuJUOqVA27CSBt8rIuB2KgWlcbxOtsIn87dZfD5W3AJ20s11GxQcTbevW3URrbeXpZJWXBAO41trKI1sIQR/W2o1328bLVTa0LkjlbI6i3RTbcGLRRRouI8LbCSNNyLdAco87YBzsk60ExuOW1ugBGmlsANuh1skwN1wUz5m2ZU9Ld76RbNedklsuABO9uQgTO9hIjS2tzZrJ1yUAjTnYJdM2r2kjXwsYgRobaMTaJATglAIpkI2AFtPU6HRC0CPHWxkjwtnKw8o2RLlJwuxgKzBsAjnFjDbSUaAaWHO1syjff8AG0crW7J85O6I1F3sPvJWpsFSdASNrCGnT2ZbIBB0jlYwBOgGk20oa6WiGAG6mHl26Rqa4KOjUv1ZhCAskmBGtiNLg+56C83Lzortp2ah0krWhsBSid5Is5YHObai0DE3opiRyTbxuqlvOgeu+tpkP076FNutuJCkqSRqCDuLQ7W+jNgYVxfpKWoaZJJ7BNQvswT0EyI5a6WnIwExAsGUCdQLJ7Mzct0mOyG9lG/DHg/hbhoHBcN3lC1jJ2jjilqCJkgEnQE6mNzvaQ9hFhCgDby2topgaHSzMjyCyd7y/dcGYtuARoLbgbC2xIG+tiAIaLUWhdb/AJXDHlt94NjPu+VizJy1rySPaAUPL+pNjOvOyGyc7rUSN9bZOu9tnTW2jyjz2s6ZYZ5W142yYNsnraTQouWaEWIXskqpFrTuhQcHuM2PconnYB9Hasrb2zpKfiLEy3CgTlIQRhUFPMSLckachNuKRZXSNFUzlAPmLdkEmAJ8rROyfmjlb2rmE6j1YgPtZktEicqtwdxtPUWiDGtNVVl206KW8lO0SaxmrdbZ7M+uBvcBS1ACeomTAkWlW7EOVtFUi8EAQA81TTIQobFRjVW2mw8YksDGLLLVU4cpzNkq0EyI+W/ytyNfhTquodODYGw213utCmmbG0sOvNRVju6E4iv25PW7iZ9RKFO1hL6G3oSUltpboClIaJCtG5Ku6CUpJUWTxFevNi6K697n4YYPRW3fSuKo6x55Lwpg02rs1hosCYCQQiYmJjUhzXzidiVUtwUK7zq0lSnmqNSUttLMqV2jqiEJ10iSrX2d7IqaavfadqcSvFKCoBNFQjMFZoHZqKhLkwRIyCCZGmYCdgVPw+NOcz7eVgkKt2bIwWCqSjj5d7aEt1fFHE5fSAHSze7oQVjfLodJ21Olss8qnhE05UurquO2D6J5S1FymRR3clLCp1QAgZAEnSE93TTS2W5rg4d/D/8Ayj3l6/z1T1eYUHFlTQKQqdQZ1t2w0AlJicumgH0bHq5tindAfdS0snKAo5ZPQdTbmmCFpBDyPNKwR7vr426dsheLhaXDDTqtttZRGUq015TznlH6+NhkII1CZI5kdBtHy+tRUNACEKhQgDKI5bfG3SWURKCBz5QAOXyn32fMeafRdsOrZhCgNwe7Ikz4an63scp34y50E6bxr9b2KJCSQG1BR035ba8uVttZoUEkpn2goROvl9fG0NSkldqqMyTOUaAySN+XusOHErSQVKGUCTvr9ac7IyXVg5pAB1E6jx2saS64mJJMwZj5f152ikErar7ySDpmnUx+e/L+uABI7yyRzER5WJofaJhJ19oGdrGEOEApHI+IPy91olSFkKNUnTNrI393lbsJGbvEnXQE9frnbhKkFWRSpJOutuoA2KSN9Oenj5fKzXS3XTcEhKVHTl9fX3WUmhrMnXcT9fQslpISuQIOhk9LKbIUUAAEc42/rYL3IzAhQOoAkb7gR02+vkISmII02idd+mxtwAAkETqRoPv+uttaD2UgEgASJ90WDdFXahGgIAAiR5W4zDUZRJB0Enl9fO2ylSBBOp11525WmJjKQkHTcz+VkHJrBYDmJAOg5qGk+62JSQhSAkE+HMG2kJJASVQDvpvbsZMw0zTuBaWZINCxKlRpA7pIO/1+ttgaKATMQBB05/X9bbjLJUdY0OWOmnytzMgkCN/hZsxSsF1KJzK5aGfmR9b25I/mA1122t0QZAlRJ130H1FtEmAZJBMz9ctrK6VliU6aiYMEA8o/rbYBUNADpzHL6i29IgQDy8vr7ra0UkhEQI25fX10srpZQszCQQZ06WxWaRm1zawZtmWMpKtQZJtikiTCfa05QLK6kGrFFM9IOk24KzBPdGmwkC3ZUCsAkQBsNR/T63sErVYTOm6tBr9aWldRLViElR0SASdQTbshMQSdNBPPS2IkLBBG0CTrbQSQMo8xH3zZXUcq6BSqZTuAIOh62wqCSTuROv3fn8bB5SATmk767eHjbM0kHLJ3GtlcqWVCBYJg6lPe+fP6/TEqnULO0z9fW9uQRIO8aax9dfoW5zEJ0ImAd+XSzJWCFzARlnXTnv8AU22EoUkhQBM7lPI2CzJTosHUaEk7aT9edsUojed40/P6/N7lItBQg7JKcyEAEHSBHXWxpm87wYEsXjUsj/1by0/cd7EEHXukzNu4TlKTpG3P6/WziRzdio8Np3CV28WYnp1DscS3sgdPX3o/8VjzPETHNPOTFd4nnCnQsj/imzbGpzATvtvr/S21AJGUp8ztaYqpW7OPqoOponaloTxZ4t8QWpBxGpR6Lp2DH/J+NjzXGnHjUZq6je1g9pSJE/8ADHjZhBZ1GoI3gW1miSspPU+E2IK+pGzz6oZoqc7sHopKa4642RAcobncJ5erOA//AITwPKxxnj7fyT9vcd3L/wAinEfeTaKUqjcmddYH1426zqQNTpzmJ/rYgxOrbs8qJw6mPwKYUekC+NHsKtkTEpriP/dmx6n4+3WrR/DtUjWPs30r+8C0I51d2ZHIjeT0torOsK0iTp93xtMYxVj4vwoeyqV3wqfWuOuFF6O3bezXU9m2Y/57HmeM+B3P7yorGZ/npyf/AAk2rulzU6AjlpGvS3ZdSpBHOSJ6+FiDHKpu9j9lE4NTHqrJM8WsAPaC/cp2IVTPDX/hix5riDgl7RGJaIHfvrKP/EBar3aHMCCDOxjf7rCIdIGYaggxztMdoKgbtCH7Cg5OKtM3jDCTh7mKbqM7f643+djbN83NUa097Ubv+R9B/G1UA8UjMeQ38Ldl4HURAnf8eXWxG9opObB6qBwFnJ5VtkKbWJQtKh/hM23lIExapCH0NglshMExACfr9bGmr7vOmEMXpVtRqMj6xHwNijtGOcf5UDgJ5P8AwrXR0FtAR0tWCmxtihkkJxHeXd1M1Kz+NlFniRjJtP2eIakEie+Qr/xA2mO0UR3YUM4FKNnBWOgzbIJ1mbV+p+LGNkGF34XP8zDX35bHW+MuLmwM7lG4T/M0PjpGlpjH6Y7gqJwaoGxCnQjS2iNBFoXa424hTo7Q3ev/AHFj7lfU2OtccK0D7e42FRuUOFI+c2K3HKR3NDOE1I5KWcvUW0RPMzytGLPG5lyO0w/5xUz/ANFjbXGi5lf3t0VYMx3FJV98WkMXoz8f7oZwyqb8KkAjTS2iJMmzHTxiworRxmvRPVpJ/wCqxpHFXBbioNe63/nYV+E2MMQpXbPCh3OoZuwp2Eac4to/UWbieI2C3YKL7QOWrLgj/lsOjHWEHTKcQ0Y6Z15fvi0hVwHZ49U3d5Ru0+iWSP6W0Rynaya3i3C1QYZxHdiyeSaxske6bG27yu5/RivpXOmV5J+42IJY3bEKBjeNwhjAi2trYClWqSFeRm2bH7jYgIPNRsRuizgy1rawfbSUx8TYyI5WKVi+zXTrzR9oEz9/3WNTNnGqYmyyR10trz52w2yZ52mBZQJutTrNs2E2yTrblci0gNVG9ltREwNR5W4JGwBtsmRpztogDcx52Khk3RCmkds0f4HVD3HX8bCmBztwRkrXdNFpSofdYRPZpUVrbKpjYkbG1eZz2x5ohmPS9lMAX1QlGspfE7KkHlvaMsV1uH8U1T9Ca9qoDCRT113heVaVAyc5BCikiIiARvIOkjBlpb6qjOpDqiCDnJJ126QLQnxi9G+7uJd5pxPhrFVRcN9pUFmraSpbShBSMpQpJQqYJjNMDQb283xTtTWUVSGVlOWMO/O/Qgj8rQZC0tzMNykl+gum7gKO7aJuhokLKGmGx3GUcthsAdTHxtEF+4ixtdV5uX41w0U9eDLgoEqTe4qqdpAJU6rsktoUnlmc7ydU94JylU23fw2xPgO54vjEVbjV5tta0gppqNU5MoRncKirvEwYHtTrlSmyFT3PfF9X+HXcMLud2jQ2F0nZ0ricsAlTjqUaoEQMqk90pISCqLZtT2tbi8nAo3ZWMF9Tv9lYipQwZ37lV3q8OYK9ae9evhwVPaK7bMaGc896YqDznnbLT07gThE46td5YdxG7VqUS+5+5anvuT3joobmeQ8hbLcycQk+f8q3wWdFBH7SN91PCzC6W82ZV+hRCZB0pXQNj1Vt42p9wmerH3qY+sPFJeQrL2qj/tIMz59LW3/aUOBOA8I00qld7Oq0J2SwP/i+dqqcHWypqgfcKSXX946rBgdfZ1529c7INDsNjzdT+6r9o3EVr7dB+ynBg1aSnsqmoSsHQJdIkTqLGU1F7wkKvOrmeTy4A6an8bD0rUkhPeBkgAagdT01H1yUUspCMypkDOQdQI5z9bW7gwxn4QuWbM/kUVRfGJGtr8rjJghSyoRr8o+8WHRiLFaQQL3dzDTvoSdtuVh/VW0d0IG+mms+X199hAwEk6SToBMAkeJ+tB7hmlhPwD0UxUzN2cVprFWKVGDXIWoEe20mT8o5WNN40xQ0hUrpVmQYLZ0PlPXz+63Ip4ICkknXUq1P0frnbXqiFRAjkIAMD+htHuFM7dgS79UDZxRxviJf7aIXS0SiNBAUP+rpOsdLG2eJt7ApR+6WN5hLhEn4GyP6ohw5VpB2khMdN45W36kjQGNDsdvl7vjZjhVI7dgUhiVSNnJyI4p1MlK7jBlO4fiOp1TYdPFunQftLkqJTz7RJgz7vocrNY0pSk6k6+Ex9bTYF6hbAkDluFbRH6WF7Eo3fCpDF6ofElq8/SMwlctYilvS5b6+0QHQppppxIExBlYM/XPR30PGLC9VStP+r3kgOoCk52U7ETOivK1UOLCRT4hpwkZZpUlJA55lDnqOlpXw/SB26KLMASqmbJnSZSOlq3/T1G9x0PqrIxyqY0bKZE8W8JjQOVgJM606uv4mLGWeKODXJi8XUkclUzgPxy+fw6Wif92JUAlCCDMzGp167WEF2AiASdYT3Znf9LRPZijOxPqnHaGp5gKXEcQ8HPwEX8wgnUhSSnUyOYsYRjfCjxTkxFQ6TMvJAHxOp3+jaGTdiUHIlESNp5bW4/dzcglIAUIOn42Cey0HJ5RG9o5ubQpyZxJh9xOZu+6JUDRQqEH7vxsbReF3KJyV7BJI2dST4c/D5WgIXQlZClogA8+Rg87JOIbjbN2jMhKe8IkjTQ76eVgP7LMtdsn4VhnaJ5OrPyrNJcbcTAWlQB3GutukwSkAwU6mOnl7rUTNzOoZxC0y842S52iileUwUDXQyPMW4oziK5GbnqF4ivZoq7R10Jr3QJUM2ozbDkDtFqR7OO5P/CsjHm82flXw7oKVE5Y5D7/G2ZSVnQCNByg/DwtQ+6cRcSW27xWrHGJEKfp/WWki9nzkBUqAJXA0jbw6WNU2OeLlHhWorE8Rb+VVracfC3atThQYVCQFzoJGnnaH/Tkx2cFMY9FzaVeiDIVl0gbGdPxtrJJGbXzHyNvPZv0kuN93qpEDiHUu/ZCUuUtOrOqSCTLcjXoeVnfeXpLcX2K1qhuzEKFpQ60h5bt3sqzZtx3UDl+O9gu7P1IOhCMMcp7agq7ZICYUox8PKLZlG6c0zp+Fqtscf+KTKJeXdLmm7lEtPjqErHSyW16VnFf94rof7PYaezVCkolh9spaBMrVDx5+7Q2G/AatvT1RI8bpT1Vt/AaxAIJIPX8rBN5ZnUk6zHW1c6v0mcXtU3af2XutYQMy4fcTmEbc4nTXW3OHfSxvW9aJVVWcPGachzIUN3sVcgebQ8R7rQOC1bfh/KmMXpHa5vwrH5SU7wIg68uluAFE66j2o52gCg9Lahr7zcov9H9elticzzde2sSOQlKZ+Njd1elphCurXrvqMI4hZcpwVKUkMLQqI2PaAn4WGcLqx8Cn7SpfnU7KkQE6dJGptoyQAJTsY5D32iVr0mMCLJbduq/mjMwaVpQ+Tp2/Gwa/Sp4QMPFirvC82SlWUlV2uKAOuncmfjaD6CpZuwqYrqZ2zwpdMqJSCRzkdYtpS1yZ0E6D7/rxtFdB6T/BO8nwyxix4Oq1SF3XVjlrqW4sof8AlBcGkDI9xCu5pQMw8lxv3d5I6iwzSVAFyw+iIKqA/GPVSHDgUQkanSPrfzt0ToCUkwdI62ZtLxk4UVlMKtniJh4spJClqr0JSI3kqgfXusoUfETh9XJD1Fjq4Hkq0Cm7yZXPl3rDMMo3afREEsR2cE4QtciVAd4zrEfWmvhbtBgyrQRvGvw87JTeJMOVaiilv67XiDJSirbUR8D9a2PNVDDolp9Czr7KgZ1/pYZjf0Uw5pO6MIUJjkByj5/E/O2AkaDSJ5zHOwYWFHMoqgHkeWn18LbzqSlRHvnX3wdrRylEuF2VBSgRruRH10toqC+6IgxB2A/KwfaFQEjz0PhOn1vbAtJMKEx3QFcvD5xZWUbC6FCiU59QowRPKLdQCSEg/pv9edggrMnvE67SLaUsAZSozEAcjvaJKmuwYICY1OoHwttJEzqIgSRM2DU5IOswdTER7vM2wuc8uhEifrxs6VkMleVZIUQRsR99ulOKyxlkRAnXnzjyFixWCISFaxuRB6W0FAR3Vd4bbkT9CyToyFKkwYMx1k6c+dug6QmZGus7/W1ipdSUwhwCOXIfUW2pxRJJ0O4gaxy++zWCSNKcIUcsmddtBytovpKjBjoQTOuv4WJl5RkgzsRJ6iwangYO2mYg/n7trSDVEmyP9uToTqfZ12130sEqpHsyRrIJHlNiaqlJ0JBCvjFgXXEpEkiZJiY08PfZw3VRLilNqpBVMgbETNjKKrUCeUab9bICKgSJJ3MTsff8LG0VEphIJJ1B8Pj7rMWJw5KwqjKjm55o5/hbaakwVZ9dBqoiPGyUupKUyVagxG2vT8LcqqFgSBqNJmPo/XiW4afNZLPrJjRZnWTGtsTVd3KIEDUn+vnZFFWUJ3X+m5+vDlbBWLygKJEHy1suGlnS6mqBmDHgJ+H11sIKyEgKOaTAmZ/TnZvpvA6lWyTuDpYRNcEggEkHQTp9fpaPCTh10uOVatSTqd4Jg2Lmr725lMbaT9RZLXWAolZ1Gm02LqrkhRVMfAg6aizhlki5ORutGxWMp8dvz/pZTo6sFtcFJnfys0KatzDTNodROtlGgqVnOkq0KfcRaBbZLdLpqydDG068tpsEp5gnVttUn+UHz+6yaag6EqV4gjQ2CXXJQM4OaRKoO4+hZ81gllBSsmoabOdpCUET7Jgz7udtP33VtKDSbyvELUe41TVTqVK2/lULI1JU1d6KPqaQloEzUESmP8P8x+X3Wcd03PTU+wzuq0W6rVa4Okn8LMJJHGzSmLY2i7gnDgdnEVbfTNfeV/3mplkLy0i61x1pRUIClhRMkSYjTztNaVZkg9QCLRth1k07YUBEQR10tIVAsqpW9ZKQU/AxPyt22FROjhAcbkrk8ReJJbhGDrbRVp525nTe2iQdd7a4aLLOJXRX15W0VTrtbmfG2ieVnAsok3XW1smIOmh2tzOsb+VsJ5mzlocLFMmRV3RjK5L2L1zXh69d7qlKbpnSntGZM5Ez7SZmNRAPQAErXY9vy43xTX1hvZIKnUdqgBJMaDIoEyU/xRvrMAviukobcSfYWknlodDbeYwRy3g7W4ebsxiFJK6TC6xzAfhd4mj6X2V1tVG4ASsv57KOF8fMI0rymaihqUAJIU4mHFZp0AToIy7nltrvZZf4/wDCu80UbbN8qYeeb7qC0SJy5ssolJVoRCSTNnU7hnDl407tRfVyUFQh320rpkFThG0kjxPxNilbwywFW3EXWMMUzKHKz1tbbRyDtu+kKJTEwHFGOsdLYsrMcjjk45ZIG3Bdb8bIzXQZ25QQmO5xF4fioXSoxdS17j7SnBSsKCh2YgKK1L7oyg7CI3M72MUmELqxAh69KW6WmrwdUGVoDhabyIMJBMFKgmVQcp1UYiZshYj4HcN27s7Wsqa+7RSNoWqqbqsqWw3uvvEhOkydI5QNLMa+cLYyuXDVy1XC2/7/AE0ja6pp1uoS409eXbrCWsichyjKlULzIIkuSmc1vKMQpe8zNe1uRx0NtB+FsZ3NYU4rxuTCtDeFVRHihTUBp3ltGkRWUiE0+VRHZhJZJSExEE6RbLEWeFHEBTKFJuTBNOCkEMvUNM843p7KnOz76hsVczrbLZ/cJvmP4S4jlVP9pNUpbuHA7BOUqrq1QBjk0z+dq28I6eWrt7MSDVg6bATBB9yp8vdawH7S99xDGAWCrRTt5LO2kJpgNDrz5HwtBPB1A7G68xzpU6lUE+z3x9Tb6R7JtLcNi+/7rP7Q61sn0H7KeKenSIUsSSo6DkdOtjrbYJC9DmICoET0tlM2pwDMgEkHNInT85mxxLYKc5B08OfXT+nhbuFygRfIrMogaROkR9fl8BEtNgxlJA32P1z+Fhy1kMqGmggE727CTIzJlKo5x0G5sk65CJMGTCgY3M68vra3RbkBAUZIzHymJ+vnbaUFaQsDQp1HP3WFSMqJ3BkwTt5zZDRIhFyjvQgRBInqT08bYWwO7Mge4T+n52M5MyjOm5nr9aWEbQQsyCDMGRyHM2kHFRyhFcpSc0kkecxufrzsAtrKMxIRzKuWn32PKbCiEqSCmZ328PDnYFxAyBKhA9k676nw8voaECE4KC+MrahiOiISdaQZknYnOrY7c/6c5cw22n9y0BKSqaVklQH+FM/1tFfGhJRiChJghVJ/ESNlqJJjfl8JtLeFU57luxYUrL6ozrm1IyD42gz3ipP9wJZbaISUlsTvH3/WlhlNKIyJza/ETH142xtGcAHQ/wAR2n4+7ysaaQYOg8OY+trFQwEWLQ7xJMnWIGn4HT77dFgwEwkbR4fp+djfZj2tI156T93LrztvsUrHsEDw+ulo3U7IoWUpJ7gzAyZEST9fdZMxAwTdxICNFpmY67n4ffZdDZXCkchtoeX52T8QJX+7nNUg5kED3jW0XHRTbuoZviA/eFIyAFVLzWYdEpRJPjsB7+Vhb2ok1d03YyhP2joQhJUdgpOp+HzNlKqpEO3teb24ap8gB5lSdfkPn4WTW+1rKq7lQezplMsmP5imTGmp0A+NqGbkrFlzeLTF33g72iE5fUuzSOW8DTz/AB5WLUNMheGkpyaBtQIPvH5Gyte9I1WX4xTqjKhBcWAYzZRIHjqQSPKyMmqcpbhbpaZMvvFSUgEEAZjJ6xFptfYprKBb3pHmK5Tysqe3ddcKdw0M+VKd5OieelpRormYo7rulpuCpTiHXFlMFaiJ931Fo6xE4WqhtDolP2kAkDXOSmeca9eZ87Sqwkqu26Vbn7E9f4Z002P4fCBsnA0TuqA1R0q6l0oyNpiTzPT5WRsN3emorK2ucT31wkEkbSZge762so1ua97yRdjTn2KCHH1Ae0BGnXpbq4WQ1X3gltEDtIUEnQROnhaRdchK1gha272QytISkqAMHmRGv3fGzWomE09xrZbQO1edUG4TrsNdeY39/us/H2dCHEgI1EnrtZGue7kVdT61ADFNDbKDBM/zH6+60HHVSaNLopdmHG7vo0oDaO1UQtxQT/F+m30DZKu66kJxLVKDY7wV3ADA202s+XWgDnI9rXaIjn91kFhnJiCoU6YzJUdTpGUbfP6FmcQLJgCUUvWjFJTKAZCVnRIVy8fAAeFmfUXE072knvkZySYzHX8/nZ53i4qodW842CkHKmfd18vqLFmbue9Vqatxak5Gzl19o/kPxsF7s5RWtyjVJFwYWpmKUVicgdfkkjUAdBOu/WzB4k0IbecEGcqenvEDXl89xaZLsSk3WxBJ0UN4jvfrtaMOKzYS444dEhgEmRroQJ58jItZAGSyCb3RTBN3ovLBbqlnMkLdSYSCSNT+nu52cd03MmiuVplLIzrnKnTNr0/LnvYjwkdQrDdWHmwsIqlSkEHTKkhPnr8xZ+XFTprX1VruVQbgITyST7unjy8bDIBUwSBdIl34KapmnHXG0l92C4qNhyA+P1AsYOHuzWClJSdwdR3hOvh8vyea2kkynNlnQxEdD+FuCyAdUcpiDKfeLTDG9FDO7qo9xFdl4Ju9Cma+rbhWYFD6kyNddCAN7GrmcxUzd1N2OJb3QpDeXMK51K9CRMhXn9buW/KUOXevKjREKk/l749/nYG6adJu5KSTCSrKOe8zPx+tLC4UZdqFMSvtoSiTN/cQWyezxtf4O4AvB7QjpKo6fKxhOPeKlMO7ja9lqTIBcqCsnror+ulj6aNKBJQoZRzgz0H3WTbwCQU09OkqcWSICeXTyGnwtF9PCfhHopiomGzj6oGt40ca2F+rXdjmpLuaO/TML0I3lTZ0++OQsoYp4u+krg/hg/xFXfrT1C1WNUYfqbqZCHXFSSlMIGgA1JO+mpmF7h1g7C1Vei3MXX01Q0zKO1VmHffXE5EkAgDqTy+QvpT4oocTcNqy4KKrpX6dllK2GaRRDFOlsQlOXROaNNtLUZKWHihjYx56K0yqmDC4vPqouub00+Nj1DSVFQbgdLxhal3eofxRPdcH3cjpaa7m4/cQ6ulpqiqoLmUt5tLpCWXEjUeKzHS1G7lX2V00iyUhbLyiNJKe94ef3WtzhRpp65aFUJI7IAGOmnP6/EU9BT5tGBWoK6e2ripCTx3xZlh24LscgScq3ESOe5Nkq/PSkvy4KhltfD9ipQ6gqzpvFTeUjSILZ/PXaxJujbKMyZlQjMNT8fePl7mvjS52nfVnSgfxJChGpOU/He1Y4bTk+6rPf57XDk6GPTGXH+s8M6hJ3IbvZKtCd9WR9T0saR6ZGHf/AJ7gS/mlRPdeZWNh1I18LRA7c7JQSGQMoBjw2Aj3jpvYBVxMKSW0tp7o1JOoPL36j4D3L2VTnl+VH2nUDmpuR6YeAlJHbYaxM1lmfsGD16PWNtel5woWtCapF9UxIMldATH/AAqMjpavb2HaYBLgZAVmgDLAEdAOcWKvYcplgpI12MEg+f3mzeyYPNP7UnHRWcR6VXBgpleJ6tnQBIcuupHjuEH4Hpbs+lBwOdjtOINKwSY+2pn2yNBvmQLVQqML07g6zCenWJJnx+jZo42wkGsNV1a0oyhtLglKYhKhMabxvEa++0m4LATuUxxidvIK9NL6QnBeoyqa4mXCkL0GasSjXc+15iy7S8YeGFWlKafiFhxcjSLzYk/80g/lbyjCQbt9sgtPkAQJOYfH+HytJdzYeYqqBiqzD7RlKylQkAESZ16CedmmwOJguHFPHjUh3aF6UMYzwvWpilxJdT0iU5K1pU9I71jib2onATT1jbo1EoWDO/IG3m8jB7Trax2bcKVuoazHL5WLVGEG0oIZSEagHQjWfDw/rztV9jt+b8I/tk/KvS1FQpau6k/zbG3K6gpCcxUDyMxH19b28z3LrvmnQUUF41dO4ASns6laSDsBIIj+lsums4puVCGbpxriRnZIKLzqEADzzac/CzDBbj3/AMKQxgbZPyvS4ViidVTqBBM/1sIKs5j3zoNzbznqsY8WsPIqUq4o4peebpi73r3dUEHKCAApWuubWOVvQv8AZ3XhT8WeCV4V3EBIvy97txA/Rqq6wZnuzNOw4hJOhgKWuPfajX0JoYeNe4urVPiLZn5Mtij5qEwUp0OsgdOmti5qiFHKdxsTv5bWss5wqwC6sFWHGgRp3XXBpHgqxKp4LcPHUn/sVxGsyiqdH/VbD76w7hXuICq/U1WlR9rSPM/W1lGmvBKVxmmQR3Ty52l1XAjA+cqbXejRIiE1UwPDMk2S724DXOKCoVdOIr4YqA2sslamVoC4MSOzBI20BGnjrYT62MorHC6jV683C6mnZK3HVaIQg5iT4R/TX4H6G4nqpSKm9FglGqaZJ+zH+b+Y+ER52Gum62LraCAkrd0Lji9VrPOSPHlZXZlXskkA9d7FAL9kYkN3RylaSAlISBAGg5eHh+ll27GSVhMjTWI2sl0jXf118J+tP0s4rsaOYCPr8OVtehpLuuVm1U9gnJdyQ20kZrPK63SqkE8o+YBsz6YykQIB3s5bidzNKTGwAHun8xbsKZuUWXLzuuSUqGB8banlOm9sVrvbU8totcVRbm2TG/W3JM79LamCRFkkuiRFtkwIm3OYDf3W0SOR2sRouoOKDqgVU6wD/DP425bV2gSQfaiwiwFAid50sVo1zTo1hSZBM7EGzSNuNEmmyUq+oKECjbVlSkArG0+FlC4FdvdlVTkAlBzDnuP0sgLJJJJ1O5ssYUdiqeZ3K2iY6kEfgbZbaFlPSmLfmfPqiGQl11H/ABGuqoqVJqQhD9Oy1nFKoEIUvcqXyUAIhJEAyTJCSmMLzvXE17KqLwF6OpZpHwXVKJyoQQo5UnRIVmLZyiSRrGk2mrHN2P19E7TuVK6OnAUlRa0dOh0kapBMajXe0d3Fd923vclbhp1TdGhNZTpT2RgqK1wkRtEiNAJ6zJt5RiBMMksnC8LXN3Gtr62C6mN+aJrk3W71xU42lxqVoUkKSrt3u8Dsf7s/fbLOV65eKSnVqoqinYpyollputShLaJ7qQk6pAECDtbLQ7/S/K3+fZQzu6/lUG/aVvKVeWBGEgZkNXkSOYzGmA+vC0QcIkZKe5pEAuiIGhAWB9eZtJH7Ri8k3jfOAXAsEO3bVVGUnRWdTUjwMDf87R/wUSU0tymA4hIlBI0iQdB7z7yT429I7Li2HQ38/wByqOPm9ZL/ADkp2aASNikpAO+qdY+jZQQlBg77FSYnTnYGmaTGpEATO342NMtpSoOJiRqe7rH1Nu0K5Zi67M5YkmAYB2nztgbymQM3Lp5/hYSFAaKEbQR7/fbbYUUTOidyTr4/cfrdlPZBoQgEbmTHiNNP6WES0kiQiSrQfXw/QW6y94AnbUbaH6Bt0EEJIBA0gac+k/W1m5p7LhSE54ClGFSRzjrtboJU4EqKozAiQNhpr8bCBoklKN9Y6+/4W6S0QrMCRGsxtvPO0m7qB8kGpvPBUIM6dB5xBPLf52LuAgArPdXvpPWx0tiCDlOgOmgGvv8ACxd9sEEpOkjnqfHxP1pYgKE7RQZxqQE3/QqOX/zQxzg51a+7r4Wl7CaALguskpBNIyCUjXRCefx+t4e45PKYxBQKUTrSGNZjvqncdPqdbTNg1Pa4ZupW00DJI13yDT7t9bQabuKd3uBLSQZKROWZAj4a/X42MBJI3WDI5azvrP1zsEhpSITm217p5yPzsZQjuyRmKRoDpE8vjH5WndIN6LBmTmkjWNfztsypMkH4cuk8v6+YFyiFKBSrcpgfKOv1pbZaK0yjRJEz9dPqLQzKeRcFtt2cqBr743EWTL/Qk3O4Z1BTBPPUH8N7KxCj9mEazESfLymCbJ+Ims91FRQUwtChBAHtb/daDz4bqQteyi90ttVV7uOQlKEImTzKNuvhYqpluguq7HFFKQt8PLOpjuk8t401t1eqKh6tr6OTlWQ+4QmYQhEhPxiw95ssv0V00IVlU/2YIgk5con4gx77Z4N1bLbJKo0OPXkqrqMx9aZcdyie6g+yPKBYvddChFG7eJbkraWhnT2UDUnwPP8ALmtXm63R3q4pxSAlqiCQBuCTAjnrG1ilKhCMLqTkDf2CiSeW/X6199pNOqidlX/GjRpn6YQFFRWVACSQSCPHcWk27Uo/svdlQ4AUtNNEZjrITt933+UZY9palm9Gal4q7J5aQwkz/dhIOY6fxEmz6w2upvDD7JGZNPQ07bZgxLmg5dPw8bO86aKLVI+H6FVPSl99RNRU/aKJBA6xB56j5W1c6CL5vBuSZykz0/rZQuxSnKKnehXebSYI8Jj77J5qEXbfNY45KR2IVrpJ7unhaDn2AUgCSu72ddqH27sphC1n7VSU6BM8/G3GG0LS1UtGe67IEb6Gxy46NSkLvCpb+2qSdDyG+lgsPqIfrG0ggBYJnzMHX6087QzG4KJawsj5aK+6ogAgaQTGo+o/OzdvbsKeqdQ2SpxyM5iMqQBp8RPw5WcNfVikpy4QS4rQCdvrrZqOrK3Vd2XHPdJ6WhLJyTxs5oJqkdrKhLSUwZIJCdBG+mll6spG6W7FtgiEtkeY+P177D3TdvqlPnWgdusBR5EeH3W6vQoNE8sEphByab9NPrn7nYLC5TO1KR7obP7oQlaZIKok+PT4j32jjiqwFtKKk589OSJ5mTy56H52kq6gP3bK1kgFYM777fO0XcWq0tVtBRM95VRmSokR3BqZ8ADr9G1kP8KCW3KT+EKXqluupW5Qy26HVkARqiABvqcvwFpQwgo9k/3CVSkGTBMg7cuQ26jbW0bcEXgp++KYO6lDK99J74jbTUm0l4UIVVVCTmAgEEkDYxH3fG0Bo4JyLhONLZCipSSeY1326/nbakhw6DWADzsOlGkAgAwIgQr3e/5W0sFMqCASZGu4672IXWQ7JLvBkqpnwRMpzCI0ixK4GvsXACkd8qM6GCBH3fA2W6hsKYWI1UCkTrB26a7WS6JtV2UxU6lQccOYJHtJ6DTn9eAA99ijMbosq3Q0kpaBUpZhISZPhGus6+VuWKBDRDrywt5emugG8gfP42NUtEsE1dSk9sU6g6ZQY+vo2HWhKjmKQogzod9R+vnr75NN9SmcCNAiLzZ3TGkakAacvo2bmM6P1nDN604JhdK4BG3sGB152dikQ2nMZCdo0PXp52SL3Z9YoqhkNkpW2pJ25jUx7+tjB10NVRuBIXcyykat1RV7MwCgfltztbjhs8iqwrdz8JKS0CAozlJE/cd9/K1RbkzBm8KZ0GW3EmOcDMAZPP3aT7rWo4K1SarBVGAUnswEiCemgM9NeVqE+hV+nN1IiEgJAUAQQSTOpEneOW39bI2LqcGibWmCpL0qy7TB2+htZaSTpBMTATMc+envslYoYcXdayNezWlUGZJ1n8dPO1cHW6snayZi2gV51AFIgrIIPL5crBqaSVShISrLMTPTY9NbGiVQViNeih8Pr8bAkKIEqBUvUEyDp4jz+fvtO6FlKKvNlXdciBAJ2B3nfwn6mxdynASI72ZRMxzjWden3WOrHIQpKRIMz8jz0+psC4DEEAzqSZUfv6zr158rSDkxBCSnWoKgITExlEyfDf7vush4hoVVWH70pkmC9RvBIAgHuqgT5x9HV0Og6JKSJEGBoT4AddLEFNNrXkUkZXDliNwRB/H84sVj0J4uquIAVd9W2D7K23B0AkiZ/wB4b/jaXsBn1jDdA8pBSpLYbHkFZZHwn32iembLSLwpCAn7I7nmlQPTwNpO4XuqXcLbaiAWX3G8pXBjQxHLVXzsSY+FDjGqejdMFNwptMxBAJkDbltPv0FsqaVJHdgGQoZun5ef42PUTC6hQQhuSNxMjnpZy0FwBoBdWiVghRBGsxvp5WomSytNjzJrXfhior3gqpQUN6QojU6A6/1jfay8i6mKJnsmGkJ3IjnJ308ffvZwBlKUZUaRHs7j6j3WK1DBSCoAFMSVAamPMeXwtHPcooZl2UW44owmrJKSQ/TwT0MFPlOgtbv9ktfpcuLiJhtxzv09Vdt4oSVa99DzazHL+7b+ItVbHNPlbpnUp0Wl1AMTB7p085+tbTV+yuvlV38cMV4acXlbvO4XyhEmFLYqWiOgJyrXrvali8fFw+QfdEpnZKhq9SlKzBKxztysbmNvlblBOTIRJSbdnYE9Lec5rhdBayLySY8LcOd5JB59bYruqPibcrJ6687VXuRwogxPdqqS+ahoAgFXaJI5pVr8NxYmw0SrmCBtFnrji7ypdNXBP8zSvvT+NmuhgJUCD7RCtetulwz9WJpSnfZHKNGylAizgu1CUictkiiaBIlMjTUdOdl6jQEpA20Bi3WUkVlg1Ut0rMCNJ8tbLlyOFLhRAgg/E6/gbIbO4kTtZTu1woqkkiNR8zH3G2ywWWU83Fk4s3L5xbQOk7W523tkzyG/S1iyAuzANtE6c7clXKPfbWbSQR02tMC6i46LeYz0i250gA624J10+RtudvqLEAtshreaNedijJyrebVoQskacjrY2ZA+VipBTVL6KQFaWdzdFEHVbVI11ix+4ngzezCp9olHxEWIK2gHytth0svNu6koWFAdYM2DK27SpDeyUsWs9omqYBGYgKA8x+dq135dj1ViE063VMtzmcWnVSUDfKB7SuSRzMWtDilphoLvR1xLbXq5zqJgBI1k/G1c78pqyrvh6vraL1elzzTU7iJW8mBC3kqEATJDfP8AiiChXA4g8xVRZzcNP2XTULg+nt0QPaYjGlFganXT/wCxUuhU6pSP4SVz3zEd7nvbLAOofcdW4uuqFKUokkrJJNstR9nu6D1KNlcqD/tFgaTGmEaBpay0xdT/AHQnUA1CgnUchlHysyuF1+MXTd1yPLoHnQzTNlWWO8ZPMxvHys6v2j7yFcVsOsqmUXGDIMb1Tw3jwP1oWRgVtCLgu7MpK1+rwVAbwpQ0A8vD3W3sHlMGFwuG9v7oGKxiXEJQVNdPxNuwIzLuq8UR3vYQfOO9ZQb4lXCtKZo7wEawGQdd/wCbxtGjJCtU6a5QdPrp8LH29dycpSFEnXz92/uNtH2vUeSoDDYTsFJKeIuG1EZk1if4ZNOdtddPrawox7hnNnFY+haeZZXEzt4/raNVEpkZuUkbe4/XW3TTn8JXqTMRI/TSyGMT9AkcNiHVSYMcYXWkAXmQo7Zm3E6c+Xn1sO3jDDIhRvZCTMGUKTp7x5fnaMkEBGZAhQ21Ik/ntpbBlKjISSNdYgjp7xtaYxiboE3s2MDcqV04pw2vX990yIOmZUfH65zYdWJMPLISL8oDP8PbDX3ToNrRGMqRGSQARp05aDe3C0oKspQEzoNDprO48/lYgxqQbtCGcNZyKmBV93MQUi9qEmJCvWExoeUmwLl6XcpJQmvpVyP++STHM/faHVBKgVkp03ISPGTp4jrZOqG0IGbKmBpqkSBsPf8ApY7cZdzahPwxp+JEvSEvClZxJdrjdQFAUiwSjvJJznQxv+k8rTrgOoZVhO53UPNyqgp9AQdChO9qecTKhulvWlhOUlgmAIJ1M/Ln/W0o4JQ1+4LtUlrL/qrZPjmSNz1384sV+KGJoky7oLaASeAHZWYbamCkkjeM31P62MpaWRGYmIJ3133tBbSyJKHFpIJEhagN9R9a2OsuViFQK6rBA0yvLEbfgRYftxp3Z+UZuFH5lNqUrACgjUGSSd9bcKQFhKUCCVd46yPH8LQ81WXoFAi96+I0ioWIO/M+djLd832lRIviviT7T50MzpPOyGNM5tKRwx4+JS6G3CAFEnmfHY2IYhZIu1yRGVSDB0nW0dtYhv8A3F9VszrLkgfEfXxt07ft9PMFt+931pJAhYSIM7xG82TsZiItlKQwuS97hE32A5UX5ULTK0NpQCBqAETHjr93xTqZtdVeFBXLJDbbiWGkjolBzGOWv11UC5VqU8fXCF1Byud0d6NNfjbhHrKAwhupBTTd5AKEwPqdz8LVe/tKOaN3VFr4o01l8upPeRTsdq4J10BgQOROvusRvBwUWEgy7MvU6kpOYaq1P6+6ymk1HbPVPaoLtQkocJQIUnb3crJN6Ui1sIYWpPZtJLaO54Dx6c7TbXNuhmjcBZQ7xUZ7J66lJJByiFCTpkG3U6c4G3lZ8YTpWlYNYDaEguskqjfNJnz208rR/wAYKqpbvOkaecZ7JDSShITBBGk76iJ+Nnnw9rKp7DVIhCmw0UrQAUmR3j+mnjaw+qaGhyC2nObKpDw04hV0UjhgwkT3toJ1+XzsDWpRVXy1Qg5QofaKJjNl5H652Bul6qoKRNIylooaGxmYOw315/EWEpaNVPVtVqVBS0KKzmJAVJ5fOwXVjDZGbSuGqcJROVKR3tk8tIgCOnvsjXQtDV716VqgDveIEz79Dv8AfY2bwqITmpW+8BJC4/CxBbalOVD7bZSuoA2cnTy8TFovrGHZJtM7mid61yqlztcwDY0SNNxGo+uXhYK7Uti8qclMyCSOR9qPmLDLux1xwLUgZZ7wB1jw6GxhxsCuZq2WMqGk5coAJPvJsFtQ1zrlT4JAsEuISMwgFI72ggz+ViN5qc9SqAncIUSE68j+thHL0bylAZeMa6QZ8NPrX4l3q5laFtFh0ZioSE/PQ9LWzUsI3QOA66Q2K4UlzmQEuFZSgH+Lx6xIsyMaXS4xd5r6pJdfcQ5EicgIGg+vus+W26cVIqKptamW0yygJkCOZ+uYs1uJV9XexdiVOdoCHMmrSiCMp0+AP67WnBOwmxKhJC7kEyOBr039eTedULpkkjlIWPd/EOfIWlXC5AveqYSpUAGQobd4T56RaFODla1R44qEuGErYdRojUnMlXlyO3M6bWmi5nWmL3cqDPZrCyFAEASZGmp5b+FiPka0i6i2IkJ6AKTA3MyUawR/S3OYgBOYka6kxA931HSwCK+kVC0uaSQRkIjwPxH0bcmto0JEOglIjQHX6+uVkZ29VEQO6IV5wNBRLkEnKJ1InfSyPdrvrN4rU+SVISSBG2oBP11saaNPVPqfqqlKUtGEJGhmN94PKxK7ghm+M7iClolSc61aGNdLAMoJRhGQFIGFsHVV/IXWVDTjVC2YK0IJ7VendSQfif0s9m8GUraRU1NMM5gtsKplAFPXYwPrlobuevw6zRU1QusoA22kIYYC2+8rbvQsGPPebNfGePGKtLt3XJUoGfR6oByk9W24J7um/uEWpcaSaTKDYKwI2sbcpv4weuamqF3fdTVO6AYdqENwkGdkaTp1+iyKtoOtKacSoiNiY57+X15KroQ6jKciSDoREDr9eNiL7SYIXCzqAAY+vO2vEWsFrrPeCTeyqOKcUWIL+oVpMocUlJCfZhyNdOh/K1h+AdSp3C6k9pmLTxgTMyVaef6+doHxcw1Q8R79SogBa3VeJkJVtvyPnAtL/o5VYcoK6mCtW3NRMwAE7aaaqI91gT66hWafQ2U5D2+6ZI7x05EffFid+JWq6X4Ue6kKBnQAEH362HaUEbKEAQOQI5gDblbi8kLVdz7eTMOzUkBPPQ6j77Vb9VbITEhQbUSCCkb69I5+/TpFtKme9OVEgac9dfo9LYMygM57vTbXrqZ5fK2nFQAoAZTIJjYzB9/62WYqGqDcSogbxJ0nlpt7jy8bF3CDIShSkxp4zr7/AOlhnDIClRMZRIJEDX387BrAGYrVPj7IJ16+X1ye9kxF0TdSCjNmMk/xaxA/IfebE1kpJVm0kyEmI8vl030sfWghBRlM6Du6H3aa+Fk5yCoqbJQREa6yNxy+Hl0ixGOUCxV3vOhFLi+9qKAn7eobTmJ5lUHbyP6WenBqgeq6SupUKUOzdSsmTpmTEf8AKLEca4ddYxtedcqEtpcbqEJIjtApCSr7zPzsucEHQxe16URUkKcaQ4AddEkif+bz2sSaUFhsoRM8YBUyXVd7NIj7BAznUqGm2u1lNORCIUQBvmgGfw6WLMBCTomQNBM+8DX6+VjTeZKgT7QEkAwAOURtr+NssvJWk0Zdl0AcoBSJBABA12/SxWraQpBJWJMhWpy7H8Z5/jY9nbyltXsjYlW+3vsA+vPLeaYEacvrTztNjkzhcKPcdtH1BLqMwyu97wKkq1G33WX/AEEL3GGvTAw4gryt3l6zRr5SHaRzLPh2iUgDqNhZPxe12l0PpTIIAUBmgykgTp4HkDZpcHr5GGPSM4dX2p1LSW7+uxTi9oR6ylKx55VQYsSZvFppGdQVXHhkafNe4TgCHjoIULdQcsg2yrEBKk/w6W0FZkZh1t5aTYkLpBq0FFnCM2k/G3Czp4m3bx72YHysC4tKUZlKypAkk8rVHHWysNSbf9Ga266hCRKmwHE+adfumzHbbBMQI3iNPdZ7XBii4cS1l4UNz1yav1HIHlISS33s2iVbK9kzEizYqqI0VY7Rx/dqKRP8o2+VuowAktLCg1ZLRYrVIyAEpywPvssUoAjY/hYlTtkiefjI8bH6cQZP6/X5W7inaucqDqj7UaAHaxymVkdBHIEj4WItgnU6x00mxxpQDiB0InS2gAVSKc8yJtgOu/jYGnWTTtkmTlAPiRp+Fu82k2sNF1XJXebbS3M+XwtqdJNskSdTFiAWUVu250gW0DztknXfSxGhQcV1uQZOti9QYfZPJWZJ+Gn42HzSIsWrzlbS5rCHEkkeOn42d2yiN12uY1ANuFE8rdr2iLBKj9LCIuFO9tU73w3WXPQ1C0BRQkakTBAyk/faNMQXWw9fKEO0rD6nVJQgPOFIBJ3ge1y+7ykW5HQ9cBaKSeycKYB6kH8bMfHtMUStsHTQkC3DdooA5wsbHa638IdmuwrhWL8IUyjTJuWgcS0cgWEQFAaT7FstH6KetCQBSlQjQyrX52y3M+xm/wC8fwr/AHZnX915kftFqlL3GW62CdG7gZASdv8AzioM/Dy3s1cFXrdVPcN3sKr6VCk0yM4U8kETrJE6fW+tnB+0Bq546tN51IyXDToVlOuVTjxI+B1tXJ1a26t4pUClCUyrUmAnkfceUdbdnhkAlw2Fu3hCq4i/JXSu81ZmnvK6VhJXelICdY9YTry/Hlzso095XcoS3XsKTtAcSY+B0P5m1VkrKSG8w6FJPOZPOBt91hA+4qftCkp2TG48+vLToLGND5quKu3JWtS6wCAFIlQiMwP9B+Ws2HadbKoBTpBOuny89otU3198O5mXVKyaAJJA322E2GF6VvtIraprN/K8oCYj4k7fpaPcD1TmrvrZWyQNsyMwG06/ERHW3RPfSSNCd55yBPv026/GqP7+vplGRF61k7oKX1yDrpM/P6I6MUX822rs74vAJjNHrKwJB6A9fx87IUThzS72DyVqM6T3lD3jofD32CXqBsVLmYHPQ7D3fpar6cZ4padStGILyTAlIFWoyfj5+8RGuopx7ixtMJxHeLgGhJfXI03k8pAP0ZXc39U3em9FZJxzQ6aCT3dJ2+WliNUoqbKW0knYJjbXl0HjaAP7fYxC0n+0NYRCRose6Ad9RHvNsc4i4vQslV/VCiNJUlJM890zGlptpXBRNQ0pV4tKP72ozMqDJ7qpIkL5+Pn+dpbwEQrDl2GFKmlakHQg5ROkfXlauV939XX4tp29Krt6hsZCopAAEzEAR46/dZy3XxKxPdFAxS0NSjsWWwltKmkqyoA6kbDTqeVizwOdG1oQ4pAxxKspTlJJBkEbkax+Gwjl7rH2YAKMvdOqRmmdPHztW9jjTjRvMf3hSLypEE0yRqRodNxzsKjjtjZmXlqoCEyO+wY8van4dbVO6SK02pYFZCAU9nlk6yYB2+/66WFUIJUmDodFJjLGsz00PxtXRHHvGTYSHmLqMjUqYWNOYHf1HUe7wsMj0hsWNjK7d90rIAEZXNRMT7fj9/jEe6SqXeWFWGQoIUkJSSTyKQJHTw+trDFwABMZgfCPfavzfpE4jcAcNz3Yon2gAsSf+Lp9dR//ACiL2ySrDtIoxMIeXrv4Gef0bD7tLfZOKlnVTwtaUAIKjvlgj6PU25LgUhJJVI7xM6fX6++DT6Rtdp2mGKdQkQU1SvgJR+PvsKn0iCSpbmGGxlISIqzqd49nw++bS7tKOSTp4zzU1F0qAIBUB3tOn46mxCqWSCEpmTEknnvvtytErfpF0+QH+y6wZ/8AS5IOw/gncx8bdOcfLpfht/D1WnOJBDiVQQR4Df3WkIJRyUOMzqkTjU+pm8KR5YzAtgmDrIKtvH56bWcnCt/PhOiJzjVYnb+NXnaM+ImN6PFq2amjo3EFpIQouK3Gs7acvv8AGy1gLiDd9yXQihrW3lLbUv2EAiM07kz03takY4wgc0Bj28UlTtROpQgtgTBmTv5aWPBQ9uD5QAOcjz8uvjaMabizhtJyPmqBO/2cx8DzIjrr5WUmOLeGFjv1bySZlKmTI09/3/pSLHdFazN6p/FacxCEKEkg+J6/L522H5Ep0iRI016WYSOLGEnUpzV7iQqRJaVpyMaaWEPFPCbkFN5Ebe02rpodvgfK0cjuibO3qnwp5OmZSSkaEE7W47RMEpIyzJiCT08LMxvifhRRE3qnfUqbWAPCfj99hDxFwrCc96sogTJKgTr8+vvssruie7eqd4cQSc2VI6zIUZ/rbha5OUyTsYO31+fLSzU/0hYVUlSf3ywMp1BBTMfXL9baXj/Cao/7bphEfxHmOfnHhys4DkszU4ahYKJkgREDfeeVmJxHR21wLhE5HUK05jXT4kedld3G+F1lShfVGTOxdA100jlpv/WGxjLEN01dz1TTF407q0EHKlxMjWNp87FizByE+xCjnho+lnHjICHAHUutAjYHISRt4WsPQuNyhJ7vOVbG1Z8D1JYx5QvO1CQ2XVQpaoE5CPhvHj77WJu+taUQA43ooQAsER4662tVRIcLdEGntlTnQ5pnWlWkEAnU6dfKdfEW6UmQc/dJiCInf+vysTafBR3VJWFbkqzGJ5df1sIV5+4UjMZISSZBOn5fCwMxR7BDZgDAkynT8o+vGwZCEg9xKsuswCNt/C3AczxIkAgAdNiR4crcrKoHdkxJEzI8Pz+hDOQU2UEoZSmZns0idJiBp9f1sCtDajqjUTpz32+o+duA8kyqQka6a7f0n4W5S4BJIEjqPqN/rS0muI1CcgELhxumBV9mdpVz005GxN9phMjMJkp7itD5/n42NuqXOdRmI7pJjQ/pPv52IVbpyaKHe89dNo+vdpYrZHIRYFAvE+mpqHGdcptpSHFobX3SRqpAGo+O362WOCdbUZLybYefaKVtkKbcKT3grQ/8E+4WKcYk9niFhax3XaZuROspWsae4ixDgpU9nfd5UqlFGdhCyADulUGf+I2vbw3VVotIp6o6+8sySL0q8hTmOZxXhr7/ABsfTeF4uoLZvN1SVe1KpnbSLItIoQCTBSnlMA7n4afDnNlFCiuCFpz5tTB+P3DyFqRerll03TpSezLiipJgz/KfuGse+3PqKW5Qe6YiZ6aDyibCtAwSQF5J1Pj1NuswBCUjPrzO+34j5WbiJZEVVQqcTCXzvr4Dw0+osEqiW0dHVBI1GbWPd462UFFQCAoBQG53I8dLcrAzEEwNpjNrHT42QkT5Aks0CiAAqFaScs/p1+hYSiuhIXKpUQSYB/A+Nji1JTKVAyQc2pAPv9/3W7pipOfKoZo06aTO3O0XSkJ2x3UUcWrvLF8B9KYL9CCCOUFSIHwHL9Gpwoq+yxmQCQmqYeSfEApVIkbaRp0jS0i8XGULN11KkZkkOskjYgKSf+oxaJ8CVDlNjG7H1riV5B3twoEdOZM8+vjY8RLoygvblkCsU25lgpUSAdkgwBrGsfUGx1oyBCDA2mfqbJTDxy6CEn4zHQDXysptuAqIWEjMZ0Op08/qLU72VyyHSsIkpEzqoETuOvvNizroCfZ2gGFcvKPD6i3bjyCsHRJ29qY/WxV5R7M5iND3jO/kQNOVlnsoloSHiFfb0NWyQe/TrMRJ2MfP66Qnfd4uUd5XPeVPUErpnCtKpGhBSUx4fmLTrXMocPYrEpV3DG4B029/1raAMUsqaoqdb+YervBKiZiYIJJnUfiogWuwOz+EqrM3Lqvfa6bzYvy4aG96ZQLV4UrVU2oHQpcQFA/A2Fp1ygojY2h3gLxRwvSei7w/xviu/qS7qJGHaKmefqHQAXWWw0tI5qVmbVoJOlo0xn6aGE6x9+7sK3i/RUmoNUunV2rgmJTyQN+qv8pt5lJSzOmc1jb2JC6OnAkaATb6qwuMcfXFhVBRUOmqrYJTSMEFw+KuSR4n3TaDca8TL0xJmarqkNUavYomSez8M5/2nvhO3d52hiv44YXqFKDV5KlxRUouBRUtU6lROpJgySZsg1fFi4qkdp+8RBIzTOg8J5/rYseGy3uWrZiNLAPeBKs56OmIlVeNrzutTuZNTd/bQf5m3EjT3LNpdxXR9neDdSgAJfRrA/iTpPwi1SvRp4i3NU8ZLku+nrkqcvFFRSxrJ+xUoD4pHytdHE1H6xdpdCZUwoLnwOhts4cw08wDgsnEXtkcS03TYp0QBEa766WNITA2JjeOtgGUmNSJiT52NIBMb6c58bdrBouXm1KGaPeGg+FjKCIjWbFQggwRIImw6IjqALXwqbjZOChcLjGm4Vrp11/GxgEga72IXWsFlSd/oj7osZqKlikZXUVTyGWkAqWtZypSOck6Wss91Vzuh5B1P3WyZMfGLErvvi6r0Cv3deDFTl3DTgJA8rCvV1JTKSioq2WlL9kLcCSfKbSBvqmNxojExtbYURubcTIzA6WxDrbkltSVwdcqpixAoG5QgIkfLzsBeCM1E9lE90ke7WwwMH8BbagFIKCB3tNbORfRMEXaUlxhC07KSDYNRBGwmwdAT6mlB1UglJ8wbdq02Ox1sMpwbpzYNd7RNZTKOikpUPmD+Fid90jK2jUVaQpFOMykKHtnkPrlNucIvdje4bUdHm1J167/AIWO3+3Vllxlqkbc1UVAu5Y6cukW5PtM0CAuIP2/4Wjh7iJALqO3cQ1/aryVlalOYwEvhKQJ5DLoPC2WFVQXlmMPJQJ9kPmE+A7tst5pxaXq78rq+DB0Xjt6d9QH/SBqEK2Zuuja1PIpKoj/AHrV+dRlrXg5kSZE6c51PUnfr5WnX04HQv0hL4gpJbo6FEEbf6sknbz++0Hpl111KgVZjJCANSCem34Tb1PCfDQQ/wDxCwcTN6yT6lY2tSVGRqARoRvyMcvHr87YoDdSlyRMcvH8fnYXskqUgnUgyNRrHIc+h8rbKBKUKRJUQcytCvkDa6SFVsuA6sGMqCcpMEkRJ+vEn3W5UVAlQIAXMwZBiD4jSwrjQQe1KgnNtJAn6gW6yntcx9pRIPlGs/D4RZgQmIRZK1JAJCgRJIUcsn3am2FAIhKVAHQEzJkcjtPOxwUxVBaSNJ0KYHXrrp9x6W0lghZMZyYEHkZ0jofxBsswTWRXMSU5Mqc2wI1PMj4R8Bbckp7NOYgA6H4R9c43saaZWoKzLKkKUE5MsyYO/Xx+hblDE5QpKpIAUCDKdNIB8h46WWYJWRRTnfUUglc666kzofv38ettFwk50JS2F65Rpm941A0ixkMFAyuwdu6pMwYgac4/XW3Cmm0ELDiRlB7u5MbR8+gtIOGyiWlFHu73ipPdUUyFeyRsB1Gm/vscbV9i2DrkTlPe08ZjfSefPxiwlz3BeeJ74u/D1wUIqryvCpTSUtOlQGd5ZASmSQBMxJIGnhaZ2vQz9IjKnPw+Da0gZpvOk1MAf97E/lYc9RDCAJHAfVEhgkk1YCVCZdKEw2DJE5ojXxP5xtYJeZa5LZMch8/Ly8bTw16FfpFPNKcVgVjKTAUL1o9CeUB3Tn9C2K9CL0j31AHAzKUmCVqvajEjkf7zpytXGIUvzj1R+51HyH0UFZ0hQWcqVJ001BPjHhpG3lrPJUOQzJUCU6mTrsfqNPC0/U/oH+ktUoUn+xdGlcSAb0pBqDpsvb8OlhFegj6S60JT/YmnGg0/elMZMc/tPDbUfOz+0aQacQeqXc6n5D6KAmaw0y1KU224E6QtJEgxt8v6WDcqFOKLuiQBO2kc48Plv1tOFV6FfpFUL6mXcEdotRnKiqaUEncR3o35gnWwafQ09It5SGG8BOdoo+0urZCBvBJKoH15WcVtKdQ8eqj3WcHVpUKh0lRCsoBGpSZ+Hl566+NtKqEg/wCGYWRroSOuh28vO01VXodekQy6pl7AL5KVAkoqmFpmBsQvpE6mxdfoh+kM0Y/0d1kbaPsxsdiF+PLws4q6b5x6pGnmHwlQ2y6ACjNJ1lXnMCOWg38B4WxFSkKSsoJCgQsq00k9fDTTx8LS4PRH9INtGdXDWrkzmBfYA9raAqwb/om8f0hShw3rRmyxDjPTX+PX+mtpiqpyffHqocCb5SogdcWlAEmJJgCMw6zy016fiPROuBpS0LJSdiDBkGYPjqLSo76JvpBPR2PDSvOg2daGwP8Ai+tfcNR+iP6RmY9vw0vGEnTK80ND1Gc9BaRq6fL749Uwpps3un0UXCoCRnCjEZVp0J0HhqNjH522mrWlsgFxOXuQFEx5dPaHnpaW/wDyTfSBUkp/0Y3jomP71oGQIjRX4fqGr0S+PpKh/o1vIlSZSSpoAe4K6Tt1sAVdOd3j1RO7zfKVE4qFElTqwCSMpKTsPCdLdIrFtNnvEKKdRO45e+NvM+FpSd9E3j+tAC+G145yNs7Yj/m5e/Qne2j6J/pC8uGV4qndRW0PCNFSZEbfPW0u80x+MeqjwJvlPoosNeIRKVBQTlCdZj3/AFM+Fuk16m2woAJUTqdZ023kdP62lRHol+kOMyTw4vDUnTtWRAMae34fRttHoj+kIUmeHFaNIhTrCuZnXPvy1/Oy7xTfOPVPwJj8JUTGqcKiVASdfHXnr110tyqsXlBUUnN3lZdSfr3be60wUvogekVVKLTfDerQcsqDlVTtgbjYrg6RYm96JvH8OhocOLyMzIDrcbbHvRrPXwsu9Uw0Lx6p+7T75T6KKfXnHVZjJ7+k90kdfDfn4205XJgJzglGgI0Khz193ytKi/RN9IVPcPDa8VFCoCwtBmdCR3o5DX3+YZ9E30hFZQeGl5gL9mVogR5K25bjraQq6T5x6hR7tP8AIfRQ4zULS+k5u9m2B56Tv7rCuVa/ZClCRqdI3IPly8rSqPRK9IcjKOGl4pymZzNiJJ/xfGwg9EX0hgkn/RtWmei0TtzGaTt99imtpL/1G+oUBTVA2YfRRSm8atB7RNStBBjRZSANfHwt3+/b3ZWEC8aoZdSA6RqBvvvtp4Wk1fokekIBH+jWvSMsHvI1A0/m3841t0fRJ9IVauzHDi8MwJUR2jYnx9rQ+z87OKuk+dvqEjT1Hyn0UcNYwxElv7K/7yGbUgVKwAdgfa6fR3sMnHWLUggYmvXKnQJFY5I16yR7uptITvolekOCVL4b14OnsutydDyzaa+Hwtx/5IvpEZj/APw2rIgwS41+KvGyFTRn42+oTd3qflPomEjH2L2niTiW98+bVKqxZkcxv52Ms8ScYoBR/aW8CdMoNQodQdeY5e/wmzzHokekQVZDw4rgNVBXaNxA2nveduv/ACRPSFSSP9H1WQZEesNCSdt1/Plz5WRqKP52+oTiCp+UpoDihjYDL/aWs7xMZoObXy89vH3iK4oY0nK7iKqVlGgUEq320KY6z1s8mPRA9I50Ky8OagQMxipagzznPrrv00sI/wCh36RlK6KdfDupJSAoFupZUCd9FBZE78/wtDj0fzt9Qn4FT8pUV37im/b7Uly+bwU+42ChtRbSCU9O6BOvUT8LB4exHeuGrxXX3a8EPOJUlX2YWMogzB05TItKb3oe+kS240g8PKllLyghKlVDYTqNicx+XjY8z6EnpFrcCP7FJlYE/wCvMpHQDVUWJ3ykAsXj1CYUlUTfIfRM+k40YtpUABVA6EjvBTG/iYVoNrH0ceMUJASaK7FEGP7lwaTy73h9a2XXvQr9I9Mf/IJE8z+8Kcgjr7evX6NiS/Q/9IlogKwCQnQAiupxv17/AI2HxqJ3xt9VPg1YPuH0QSOPt/JEOXNdpIJGUlyZjU+1Y4xx+rwuVYdpSEzJD6tDJ3H4+Gx2toeh76RmbshgY5hyTeFNA0HMOdB93U2T8XejJxxwPh+sxfibBC6O6buQlVQ/67TuhCSpKQYQszqRyMTaOajeQ1rhf6qWSpaLuafRLjHH15JLT1wU6lA65amB5+yfryEjDjrTrRK8NHltVDUdfZ+f32gpDqScoUZCdJJk6/1sMisWCA0U5iIJHj79dvusQ0jELvDlOjPHK7FhAcuWoTmTrlfCo59OQ/ppZSuzi5cr1WW03XVoJBKSFJ18teX1pE1+RUulXf1EglJBPj4eP4WV7jrlivbICiVEtgde78/zJsCWlAaSEWOc31Ur4/xjd9+0NMzTU1ShxpalhKgADKRpIO8gfQ0iukrBSYhYq+yJSxWdovL3tA5Ow8Of3aWXax5LjYhJWU+zmIBEadekWZ9epxFdUKIgFSSAkyIgRty0+Vo0guCE8ztQVYCnxtc0aLeSdDBR8Bryj4ab2UqfHdwZUpD7vd6smY2AiNbQ9TPj1ftE5hm/igxG8dI/DnBix/tlBKFgpIA0gkpBJ+ufI62pvaQUZshKls41uDIhSqogZjqELmdNzFuFYtuIqCPXBKgIJQrmPL9NLROmpKWgSVJMAeKvx5+Fki9cXXfdMtBYdeEdxBTod9Ty3PWyjifKbNCd8wbq5THV4iuQs9qLyQhIGq4IHI7kRv8Al1tA+PMTXS9X1t33YoPoeqSS4kygJzT3dBOs8ufibN2+cVXnfK1B15LTSlZuzbACfCepFkJwkmSZ1mbbNLRcLV+6zqiqzCzVKmAr341cQGKfB2GsSvP0eH6ZRpKOodSWqdtTpWoNpUCBK1knTnvtZ4VHDb0j2UFa3KF0azFOwZ08W+kfK0HXNiG+MM3i5W3FeD1G8oFGdpUEpkGPEaDeztp+PHFSmbDIxdUOJ59q22v/AMSfK0Kimnz3pwy3mNUemqKcMtOXX8inJX3Fx/pCfWKJokDVSael0Hnl0skOPcYKYAVFA2SZGrLIM6jlEGNNNbJr/GviHUp/1i/AuTJ/1dvU/DysQe4nYrqEdm7XpI6hpI6fDblppZmQVHxsYmfNBfwOcpd9HfG3ELD/AB84eXne1MGqRGJLvaqF5QPsnH0tq2P8qjb3KdYTUMuMLHdcSUH32+di7uI+IaO8qO8PW1BdLUIqElOhBSoEa+75+VvoiuO9Ke/blu++6Q5mbwpWqto8ihxAUD8CLY+LQujex5aB9FcoZQ4OaCT9UzUMqbWppYhTZKTpzBiw7YHjoI99j1/U3YXipYHddAXMc9j8wbEgNhyGnutqUzszQ5VphZ1l2lMkdPusKkaagadDYNPtDTT8bCjUTpBtoBU3C4SldLgJKBznX4R+NkXibct8X9hh2guJCXKntW1ltS8gWkKBIn3A+6ynd6gH0jxB+8fjY7etYLuu2prigrFOytzKNzlEx8rHZq2yrk2ddRTd11qwfii7qekrHl1D6GV1dOp3tQytRhQSdwkj+HbSQBNlHEmGb7vS+r2qat6poKdDanmK5JbWjRIypyqkx10G2h52TOGlddt5V1RinEd5sisdcK0pWSYJG/kBoB0FnLxFvdbyKO6KJ0BqqAdcczQnL/DJ6c/daLcoYTyRHE5gk2lvq8nsBVQLq0inqGm8xOobUpMp6xr8LJlzvV+Hb4u+9hea3ae8hPZFKQG8qsq0SB3gZB11BnXYB4C6rsRgqtua7K2nqX/V1uKLa0qUpwCZIHjHysxql1NXh673WlhS2Ktad9Q24jMD5EpsW1gChHc2Srik35e2ILypKGuS1UU4KqZK0lSIS2F5YBETr3ht42e3D2+n73uQesvKdcYUE5lGVKQQCCTz52adfUBnGN3Vyu4iupmHFA6DWUq+Ay2P8MnF0lfeF1qOjcoTP/q1lA+UWINDdQOye7cIdqGxpC8w94ti5UdNracBTeCyVaONgx5G21DXSyKYI1dLwprzpn9gl1M+RMGz4vCmCnV6e0i0diAd4jUG0lKWKmmYqQNHGwr4ibZWKQiSFwPRGhcWu0TGdoQHVjXRR5+Nss5F0iSonJubZbzI056Lb72V4K+mj3/SIxElBVPZUKYH/wCqtdbQ8UqcqHgW1KHakZ52n3dTG2utpu9L6hqLz9ILEdXdzJqqZfqWR5glaTlpmgdU+ItHQTdVI5UpqaRaX+1IjMBy13Ply1t3VC7JRxNt8I/YIdb4qqQ+ZTfNOvIVdmXIMEARoRoD4mCN+Vum0KUMqgYmSkIgnu6Hw3s42b+uWmSWkUhb7Q6gqSABy5zpztpNfhDR00jp00MmUmdNSoSefS1i7uir3HVN5NOtQSCFDIJJjbTczz11526TTKSQrs1qOgEE7xp5f1svJvDCi/sadp3vryJGZQiPMzz84B6ahorMNLUnKFoSvu7geMb7fITZru6Jhbe6RhTK7RBZUQANYTOu8nx25fp2lsOBbYBQghSVExB1PMzO55zrvZ3XFdFFftO7U0YORtZQVqWB3okiOehn8rKQwa0lKgnKIEAFcmQJjlGmthGUg2RRGXahR+pgEkZDm9jQDfw6a+7QG3JowBlghSN4BHI9OUH7+VpEZwSwk5XklKcxSR2qZBBI3jflz2EeBhGBLrUQlZcAk5T2qNTrtO58+gtHj2UxASoxUylWiGxKUEFMTJH6fHnIsG9TOskZ21QRA1ISTtM/dytK5wTcwSsOVDwC+jiMpJEztzn5ixd7h/cDo1vB5IMk6tkgbRtqNPhNk2qF05pnEaJF9Hp3Jx74al0pAViiggbjvPo01/W3uUmloHgla6RgnL7RbTP3W8VOGVzU9w+kPw2pGHnHUDEt2rK1lJOY1CBuPK3tMmqCW0x/CItQxSPjuY63JXMPcY2uaeqEN33aglQoKYf/AFQ/K3aqK7VaepsE7atpsA7ejKmxThiFzq5OkeVg1VC4y5pGkmd7ZxogFeFTfdDGiu4HSiY8fsxbZobuUO9Rsmf/AFY+elihvFFKStxCnJ0ATv524RXlYzGAN4mRZdyFr2T95J0uu13bQF7tU0rIUTybEnzNulUdINSw2I/wjewZqlHvAfG2i/Jjl1tNlKBpZMZ/NGW6Ojyx6ugQP5RvtYZNFRAEdg1GsDKLE26pI3V7tjYUVOmpGg2iyNN5JhPfmhzR0mqfV24Oh7oNgxd9Eoz2KPegfjbkVOYwUjrbn1sTEj4z8LIU9uSRm80KigokqJ7BsGI9gdbCt0tIP9mkTzyixUVAnp5CbdesmRoI5gmzmm8kuN5o0mnpEpILKRp/LblynpZksok7EpFi/rYgiQD+NuVVRmUmTt52j3XySE3mhPVmEq1QjQxqgeFuls0uiuxQSR/L87F1VSo9tXh4W4NSdif8Vpik8k3H80OaSjgD1VuR/hEWDVS0YBHq6IGh7osCKobAHa3HrRGvLfe0hS+SXeD1QppqQDL6syY/wWLVdNS9lpTteyf4BrbFVBgzJG2n19aWL1dQgNgmUwCJ8bQlo/Dspsqdd0LTtUxcWoMtzl0gC3KmKUttJLLRk7FINidNUIzokiVIn3zbhNUcyNZCSNZ6k/laiaMjkrPeB1Rw0lGWCo07R3klI5Hy1sXNBSBgqFM3Ko2SNPlYJVcAxCtyo87dPVI9VmRMA6ajey7nfkl3i3NGU0VFH/mrIEfyixdykpVLX/q7crIGqR8bYa0BvMCCI5mwCKxLj8rJGUEjl9fpYvc9ALKPeB1Q1VSUbaW/sG+kBA1t2qgoVJj1VnQDdsWLVtUFJSSNlHnrNuzWJUEqAExOlpso7HZQdU+aKu0NEapI9VbgJiEpAsVqqCkfUGkU7Mjc5IPKTYeqqwHisBMJTPgd9LF2XyUhawoKWrrMaHT5TaQpNU3eQEUqbsoWlJSmnYCYBICQDv5R9TzsiY3xHhPBlzLr75yUyHCllpTdOFLJUYlIjkDvECLK1+1rV3Ublc8VFDTS3lhOphIJ08Yn5+NoQ4ecV3Mb4pp75vAyUsetBouy2wlZAbaGsGJk76gnSNLMVAXnyUHVYbZPpGI7wxS8tGCOGN64gdY0RV1zq6eka/wJ7RRSo6knK2Umedg8S1XGbDF3euO8G2qtE5nDSVDTwbSQMwCMoV58trWCw1elB6m0xSMNiESCjQKOu/Unfzsevq8l07IU6ltKlwMugPv1m1luHQBuZyn3mRzg0BQNhDGOHcbUqwzdrdLXspHrNI8hCXEGNSUnvRPgR42PVV0UTy8qaJkRqqGx3dfdzstYmuS4W3nsUMXOzS1obzKrWBCnAnXKobEECPf8C13w9Tt1KSIdAXIiACJ395+hanHTtc4taE9RI6IAlEGLtpUVHZqp2vaBAygwLNT0grgoL34K43u8UjJAw9XLSnsk6KSwpaY6EFIPmB0s9KmvS24qmTJgZdAkDNmCs8xmmAU7xB2nWxHGVMi9sKXxdgSVeu0FSwZ6LaUn8TazFSBjwQFTkqszCF4h5UZoygAmDHx+vK3aCkyFLylJCZ2/pbpdMtsqVJkHv5tANSBPvHhbvsXHXClCBCk7JTqdfrbqbdXcLm7ELaELKu6tJCRvBhX1B+Njl1lTVYyvXuuJGZI0Eke75TNgGGlzmLYJBE5hOkgayfr7jbbaG4dQhKoVmkxAjYa76RMfKwJDcEIjBrdOqoUkJCFAZld4BSOh6jWdBGvPxs2r6UoV6Va+wI7s5tT79tyfnZ1VKE9iEtmUq0UCmZI0HPw38OtmriYgJaWlMjKqToZ1EHfTUqtSpCC+yszaNunDdLinKJhxMx2aO8lUkRp74GnusHeV60tCz6xVuZDHcj2lJ5QOc/gOmjTRiZdDQM0lOwntG5+0WJiSSBHPc7/CyLUvu1DxdfcLq1ays62tMosziXbKu6osLDdLd8YxrK9JZoppmtZP8Z8yPKzdKpOqjPjbR00Bi2jMa9NBa+yNsYs0Ko97n6uWxIgfdbSgZ1523Cuu56227BAM76xFiKFlysagpESBbREDS3RBCUK1gyAffbakZR9aWSVkGCelsOltwd7ZB2skloEzMW+gL0Q8TDGPoxcM7+zlS1YcpKRxUnVynR2C/wDmaNvn9O1vaj9lpiY396Jd13ctQKsPXzeN276wVioH/wDs/K2PjTM0LXdCtDDXWkLfJWgxLThdO3UBMFCsp8jZux7vfZ6V1P6zRusACVI08+VmbBB1EG1fDn3ZlVuqbZ110AZkEeXIGwiVEka6+f1424GU9QDbsCQRlPiTbXas8lDMOdm6FE6CSfdr+Flh9hqqYcp30BbTqShaeoIg2RRBOVZ7p0PlsbK9EtTlGwtftKaRm841+djRlVnjmmgxwgwXSyKRiuYBnRuvfA+SrKd74Dui+GqVpb1Wx6m32LamX1JOTSAeu1nGCYInztuYMDexg1vRQLnHmm5h3A1Bh2rcraetrH1LQW8r7gUACZ6eXwslvcKbr9eNTQ3pV01MpZcNIkIU2kkycpUkqSPAGBsIFnwd+VtiNgdtpNpBjbWso5nXum5iXA9DiGipWEVTtFU0IinfagkJIAKVA6KSYEjwBEHW3eEsHqw4XXaiu9bfcGUrDeQAeAk/fZwiAJmBbtKjvpaQaN02Y7IpWJy1NO71KkE+Yn8LdLSDuAD0i27wgNIcP8DiVfOD+NtrEAfGzEalK6BIg66Wf2H3RU3DTmZLctn3EgfKLMJXx/GzwwW6V3fUsGfs3Mw94/S1WqbmYQptKUSjU6Wyw5RJJtluU7i1Wsy+aKnqV0TLbTbriTJBKVQo6kgT79rYupf7ZSHUOJWhSgoFKe71GniTp5c7ErrRUPM06e0lSjHdIEHYDz8PlZw3dwt4kX7eTrd10HaPIQp05qpsbECSSdTJHnNulbAXOsAhGXKLkpKRXLJyw6cw7sjUHp9DWNfAVq8AiPsXE94SFIBJOkeOhP1vZyDgFxjQO9cyEiZM1rO//F4bmxpjgDxgUpSTddOTlJE1jJ13B38D9RYvc3n4UPvLPmTTTeBUlJLJjUElrUnQgGPKI8Oe1uvXErhCKdefY/ZSYIjpB+Pws5TwF4xpQS3crJCz/BVMTIn/ABCdLcI4KcY2kR+5BABTpUMczP8AN9TaJo5OicVLOqRmbzq20GlpnapCDKy220QFH+JRKfHLyHnYRF+X0FZU1l46gpGVTkDXbnHLygC0l4G4H41qKGpcxBhvtKhD47OKhsSMp7oKFa6xqfGzne4G30hSVUmFVNgCClNQCSZTrqvTTWdxpas+FzXWyq0x4cLhyg9q/b+efSUVd4EqTlBGeQR+oHT4myg1V4zWgGnTeeoBlLSwVancAHx8J1i0v0/BLETTqVKw3UNAKGoqEk6kyPa8APjY+nh3i6iWtNLh10BJlMZVctt/r52rSCQe7H+FYiyH3nqFm2+JTqClqhvtSBBMsLUZjQ6J93Tffe3NQviTQNhdTdd9ozGVLVSLlX/LpqPhNp2RceNqVK0OYXW5BmSyAdSdsvMQPjYk7duNy6Gqm461IcGfSnUlIEDaDpyA8rAa6e+seit8OntpIbqPeB9ZflRxzwCm8EVSHEYju1MvtFsgesNnmBvJ+t/Z9tw5QZiYOlvIzCN3V9Dx64fP3hQu0q1Ygu4gKbKCQKtI5nb9PG3rg2QUJGWFRO9lUsz5TayhTuyZgDddKUCRJJIHvm2jWtoEKGw+vvsE4HMhKUmBz6e/4WJL7RRJSox1ja0WQjZNJNbZGi4y4vOSNIk/XnbsOM5co166/dZPQh51RShJUqJIAtyXSFBK5B5z52L3cHRCFQQlULY3UOe9tlxoSQSn652S0vAjQg/hbC8JIzeR/GyFMEjVEpUFSxvqPCNhbBV0/JXh42SC6IJB5RpbkvnNGaROhn8LTFK0qPeiEtqqmSYMkjTe2GpZkEEn+tkUPySAr/NNsD+XvEyN43+jZd1AS70Ut+tMzEE7RbYqmTM6TvrZE7eUwTPPlpp1t0KgACSQPPlZxTCyXenJX9Zp5g8vvtpVVTzqPjZIL8GMxHnbXrIUrUieXl9Cy7q1LvTkrmqYzSNJtz6wxzSoRpZK7bTQK0H420qo2AmDG1n7sEu9OSsaljcoG3X87cmop5jKdZ5zZJU/zUrfXfS2lPnWDM89vCzimCgak8kql6mH8A8JNi1dUU5bIybggQbEvWFaAq0+Q+tLF66oCkJSN/Pwsn04spNqTdGG6uk7Rr7OPso+FuG6qm7ErCIlwDfTSyR61kQy6NB2SjbTbw9QJB2XJ1Gus2rmlaiiqclM1jJEZdZJ1OlhV1dMaUSn+GRrHQbWRS9nDywQYlI/H7rb7UmlCBBCkxE77WburVLvTilV+8KRLIT2YTmHUzpbVLW0xdCuzVKknUzIshPvqUqAe6BAVI0+NhkuparEQdkZddtv0FnbTC6Y1Tglm8axgNCG9AQLCJq2FITDWkdddtrIdc7LG+oMkGd55fXjYQ1IQ2FKVASnlB1+vv8AGxO7tCH3lyNVVZTrUlkszm8TPl8/qbDpepk5R2QypIhP4x9bWbztSMyVpJAKp9qSQDrz8fD32URUAnTWDBEnXr+Bs4p2pjUush72cpahCW1sIhYLahJ2Pj7vvtT2huS+Lr4o3VgG6rlvK7WzXKU89UUpYUulQohK050gqCjICkyIOh1M20q35Q2pKEnKdEqEz4b6frZCvbC9Bd9WxjCiTSsXVdjaW007Lcu1DrypfccUTOZICVCAPZJObQgUwMTSQFeobTu8RTpvaivdFzihuZNTkd7jy2qs0y8kAR22vZDQkkSoxAHSHeGuDb5XxCqnDTXZR05ePrDlHfFRebi2ZIAU8/MKjWABExrEmaqrE1Jh+6n61x1LyKZrN2TRAL6tMonb3k87RvVcT3aamqKujxzhK770Lkpu11lLzCWx/CEoU2oqMR2hJEzAgRbOa9rhYro44XXzNSDju7cV0HEKsu0O3k83TKytqpL1LJSwvvJdUye46M3dggkQdRNpWo1Jprup6NISSwy23sCAQmNuXW0TMcSr0xPeP72xHRXA4+hSWQihV2mQQIUFEmCJnyI90stMLcp01B9rKmRsdt4O36/CWG5JJH25Klj7XQsjzbn/AISReCyp9WQhPUAnWf6WFgrSG3CQFd0687Gl3VUVIeqaVJU2ynv94AzBMAc9EkwOlgygBgJOsJBEiNB+v1tbS4Wq59slwvNm8sCcHLvvu8LuqnrhFRTVbtO62qrSlSVIWpJBHaCCIiPC3bWC+CRICTcS0nvDLeBAPwc00+trNLjhwLvV7jDjN+kqHEt1F91j7Tfq61aOOFYggbDONf62YrnAnEAWA1UAhR0+yXMe4dCLVXQQk6zkFXmzSgf0AVOKeG3BVxtB7O6w257ATeSiFDYH2/EWOU/CfgzUFSUpoxlBKibzc/Az0087V/HAvEaRPraZ2MMOEBU7SUiPx194DvBbFDUjtmySDA7NzUjlMb/XOzGljdtUH+fdEFU9uppwrOK4c8ICnskNNqXlCQG7wcUmYEjzAFqc3iLxrb2qaAqW8tpbjaEN6gZZmAPBO/vsv1PCLFTCe0DjWTkTnEnp7NmZX0dVdN4u3fVAh6ncKFiTuD8bXsNpGQF2WTOT+Fn4jUumDbxZP7p9XDhe4a7ClYbxuS90X96wU072YCnDRA9pJ70iF+FuVYBu+FIKqxKgdsyT06J6fh0s38J4VxBiv1v9zPpHqSUrc7R0p0VIEfCysOGWMgExWsAiYAfVpzI2tZkeGOIMllTaC9oIZdSdgXhJwbxRW0lxPO4pTXvMkrcK2UMlaEyvLCVabxNpHc9D/hS072buIL7ZOmgrGCU+ctfWlq50XD3iOw+HrvvMtOgZQtutUg+QOnX52U2cJ8Z2wkM4orW8qcoAvZxOUCDG+m3ytmzRyudeOqsFq080TW2kprlT+PQ14WOphGKr/SIkS9TDX/2c/jpZgcePRkwhwy4fuYrw3iO8q2pYqmWXGKhbSkhC5E9xIMg5fjysyxh3j8ggJxjevcOYRfbndPX2tLB3lhHjtfV2que9MUV1fd7qg4unfvhTjRXMyUqVBPPa0YW1EUjXPqgWg6jTVFnkppYnMZSkOI0KiTIC0DmEhUWuxhD0DcD4rwtc2J0cTrxT+9rvp67sk0TXd7RtKymc+sTFqzI4GcRilSfUGACrUGpRqRtz8fnZ9XXdPpQXNd1Pdd1YwvVijpEBqnp270ORCEiAlImAkACBt87HxOZ07AKSoawjfbVVMMiZA5xq4C8HZTLWfs7bkRpR8RbyWraP3ahUq5ahyyY7+zrrQFOIx7VoQJGZdyk69P7wC0c1N+el/hygqr2/ttfzbFNTlx51F4gqS0kEqVvOgmSNbM930rPSHfc7V/ipfLqoABcUhcR0lOm3Le2dDT4zKLxVTXD6f8LQmqMIjNpaZzf59VNDn7PG/hAZx4XAoEibnWNBv/tOXO16/wBndwlvvgjgfFeBb2vYV7b96tXqwoUy2cpcaDaxCiZ/uU7W88sE8YPTexrdn77wfiW9K+iDymu2TRUhSFpAJT3m9BBGm2vna5/7PnGHpPXlxFv+6ePDla/dT1zFygcfp6ZsJqUPokS0kKJKVq3kd21eU4iHcOpqGOA3A979lMCgczPTQPaep2/dX0Eyd5s0L0p/V651uNCrMNeRs8Dpz+FkLEVPq1UAanuq/C1+hdlfbqqFS3M26RQkK84iI5WETA184jS2RsInn4fWttyDzIjebbjVllbAIOpI5WVaNwqZjmFKj3mR8jZK1EwD0Nj9CsdkpAPMHT4fhY8e6rP2R0RuAdrbBIEGfjbgGBoSIttKidBz6WsoS7EDczbAflbkdZ8NLbmdY162ko2QgO8iBbtJ1iRYESnQi3YOgPwtNMua5BcpXgPayGNOY1FuGFh6nadJMqSDYdRERBI6WKUcJpuyM/ZqUjfofytB26fkuliBFnFgd/JWVFPycbzecH9TZvOa9d+dlPCjpavlkkwFSk+MiwpRdqdqeZQqT52yxssgmbZbG7mUXiBfMtcjA7aibK0/3qClMcyofD63tZ3hc36xf1YhCkIHqyl94EQntE89eg+XlaugumruW/W7uvOkeo66iqEMvsutdm42sLAIUCMyTrz6aWsrwoYR+8a59TqJSwlGTKRoVSeWu3n+G3R6vVaq0jKkZdMppAl5CkKITlSqR4ae+PrUJIKHQsEpBVm2nQHr77HcjRkKfSJEknpPM9BbYZbGZfboBGkZuu8/XxtsgmyyQUSWlWYo7Mnp3oEfX0RbR70KKRBIkg6n3fXLxsdfYYOV3t0HQagmVEe7cQDrYFbSfZDiQIHsmAfL57/rZkjqlbDRSadyASUOpMxABj4fHxstKUlStOZlI57DSLJeHWULYeQFBQDiBGsDfpZXXTLSUkJOmo5SNJ8BztTl95aEHuIE6HOAZBJH/N0sGoLQoOIKCokknfpv99jCkZm0tBG0xGmup28/usGsJSoZkERqI122/wDDHxsFGGqKqzlPaTOxPPSDpHvB+teHRnhACQlMDKPMc/d91jAZUpvvpXqoJlU7aTYvV1LdHTO1VUshptBWoxpAE/jt99mNgNVMAnRQ/jNeT0hOE7cJzJvqjUop3A9aa+Wvvjwt6ctLSpIn3z99vLS9Gaq/fSD4bLW6Kd6uvyhT2rmoZ/1lsIEc430G8nc29L2KO+0o717g5tQex/W2FWuzPuFp0oLGkFONVe+ulVQJy9i5qrTWdOfwsWU1CVJMGBpZKFDfK+7+945/3c6WGFBewSqL0UDEQECwmEnkneAOaNMVzt1ZnWMvaKGUZkzAsjl9alKWtUqUdfPf8LFqu7b47WXL3cyFRgBAsSeu+8CIN8PgjnA87W27WsqjjqlX1kb5zvEDTlbPWcycyVE6TG9oM46Y9xDw0cuGoo8Q1KGq9x9t1HZoOfKEkHVM/wAR2NowX6SWLgsZb2UtMgqJCfdACeevuFkXBptZSa0kXureqqhmJzGAfo20qoE6Ekcp3+dqjtekdiEkhV8PAjT2Uyd9+7p52EHpE4i0y31UwqIhKJ+Q0mzZ/JPa+xVtPWhB36RI0+Fs9aSRJWZiNDGlqmH0hMQd0G/a47pgBMzoZOnnbn/T5f5Ol+VyRMESkH7tbNxQOSYMPVW2FWEyc2069NLb9bkwCmRpEWqV/p7xCdVX1XA8wVjTly8j8LCU/G3EdQSVX9XQTJUXfZ+G9mNQ0clIROPNWx9ZHXXTUW5VVECSRHhoZtU2q46XyVkN35eMCRIfIMjn4f1sGjjXfjhzfvu8SAQO8+rXXz+tLN3hvRLhHqrb+tDLpJBmYtwqqMiAdOfutWq6uKtdVFJevW8jO/8ArTiQJGmk2cCMcuVCErTe9fBEE+trnx5/XvtXfXtbyRmUbnbFTv6yVaJnoLaVUkCASOW0i0L1eKKlqlacN61ozpgkVi5knzsVRiJ51p1z98Xr3RoPXV9dt7C9qMteyMMNeeam81JzBOXUE6g2BrXz2QUOZ1kfXhaCXb8rKkBDWJLwaVkIbKqt1SSepAUNJj5CwN137eBeyvX9XVCk6k+sOhOgOuUqJGvj91huxmP5URuFP+ZTCuqUthoQZlSTO0TzsIalQpHgCqA5oY+utonZv+rKCo17pzLSgFVUvTnP6+Fi19YpfpXVoReD8KUSMtQuCB0B/HwsAYzETbKjHCpAL3UwNVLqg6QFEFRiIkfXW22X3vVUgNLmDMDSfD5Wgr+2zyVgmtq1CCTNQojXbn5/CxhnHDq25Qt5Qj2u1VqfKbSOLxj4VEYa/qpmaW84+ErScqQCox8vhNhHalxNa2pKFFJESR4nnaB7yxw42goXUOqJ3KXj+BiNvhZBXjZ4KPaVTpB1Ci+slPXn9aWePFmO+FDfQObzVl66rdLEgKI0Pd1+vr3l3K1x0NUqEmcokiQR8uUm1bnMc1IkGodAgCDUbTz38/DXzsUVjp4pKy+pUaH7YzHjrpzHw6WsiuaeSGaM9VZyrqlN9inUEGIAGpnffz+djhqnNglUyf4ZJPKBGvPlap7mOHgtPioEgLk85ETry+/yKHH4OZJU0O9qC7KcwjQ5fMaeHS021YOwUDSG2pVsbxqldmlHZqJBJyxOn19TaMPSJxTjnCnCisxZgJNcm8qNTLTqW1fYKpXe64X2j3FoTIVKhoY5G0LnHynm0eqqpShLkKcCgrNtpvE7QCJJOk2ScF8QF3txiw3ct+UZXdNRUVNNLi1BLlSqle7NfLKUODunNIORW4GW9Rs71K1hGhKE+8Au11lIXDT0m7qpqumwJxDvLtkhxNHdt91tKmkReCkJHdfaStaWnpAiFqSsQZSoFFrC0103FV3c/eVxpuVmsqCpbtSwsUzq1QdStEFS45yeWsW8++M/CKsww1+/ENVd/wCFryf1p6koTVXZUFagtKqpQygGdFOA5jEqBIKmRUcU77unA1ZePD7HS8PpuqsFEblvO+k1NWpvROdpl1lSijaQl6E5TCY1tOrwaKbNNSOA6tO/mtCmx2SjLYatpPQhXK4yYwwVwJuavx/WUl2VuIq1Xq9Gl51T9RVviCltS1KKuzBBJAMQdNYtYrBF93hiXBNz37fN0JuevvS72KqpoFOZjSuLQFFsqOuk8xO0i3kRgjF1XefECi4gcW37yxxdeG6+ncqEKr1BC2i5CcgdTAbDmUqTlSFARIkm3pNgb0sfR+xYlqkfvV3DTroPZi+qZLDKjvo8gqaSnoVqTNqFPQdzbrqTuVHEcUOIuBAs0bKV62rqKcpaoah5AWkoX2f8YJ1mNPr32KB1wU4TEkAjfb61+XSbGHsPUV50qX7rZpqhh5vtkLbKS2pEe0FTlKYjWY1FkdrD9K2pxtVGhKkKKFJUgAhWs7+/4GxXAhUWmyrnxXQzR8Rb57NKFhxTa0EkZUKU2gkx5n60s2A26j7ZbCcqlykFyCRzI0mPHaNLSHxpuVFJjlKadDbKXrvac/hSBlzJ5+CR8epsyGcqJyMl0+ylS/ZSNTt5C3F1pLJ3DzXdULeJTsd5LhgMoYzZVJcKYy66HqTv10+6LH2mXngKiqaUltBCUKU53SY/hAEmZ5bEWDZoXHSArNCtBmPdHn8fnZUTdL7pSy2pdQ5tDae6B4E6nWOUb2znSW5rTZH5ImKqgQpt31N1SGgdVuAIB0hWUCOnhFqN+kXTFvi/fj4IWKksv5+SiplEkeGabeiF13DX3dFYinpW3G5WHKhtDoSRyAUCkk77T5Wpb6bVHUjinQXvUKzLr7paK3A2lAUtDriSYAAGgTbb7N1F6wsvuFido6Y9zD+hC59C6n/e2OcSYVDvZt3lcrskqEDK4gAmd4Cztr94tUOHVUzeVHedNXUyl07yXyhLQyZwyhtSRprIC9uZGuhtUT0QHew9IC7LvIKhedNVsiDz7Eug7TugfI29J7rwq5UOJHYFSU+0IgRoTJ2+Nqnauc01YDfdoP8AZT7N0sdVR+MbEqPMJ4QbovWHb3u2jNZVViqjtGmwvs0FCU5RmAgyJmOehGlnpdeAsPVdO56xdVOiUkLqHmEFI1kBMe0dzz6xvL2p8POOSw02hLSo7jXsqM8zur39DtZdpsOUdMhJqFKWoaBpG4HjytxMuIk6grq2U8UIDbKPKbhjherMIuelcQFQ48ujbAGkTJM/j4WUqbhfhCmeCV4fuqoWUHKF3Y2RmJmAI1Gp5c9rSQzdzi8qWW1JCR3SqCoeXIfCyxQ4ddbkgZVnfWVK8z+dqJrppNiVCSaCIXcAo7oOFeCSpT14YQuxxajISqiaBPQkAaWXLv4V4LcgNYNuQKX7S13YxI8u7oPnaUrlwJWVagpTEN75z3RY9el64AwIAL3vBuqrNk0rHeUVchE2uwYdW1DeLK7IzqTb05n7LBqcZhzcOFuZ3QBRbV+jvhHFNxXncacHXW4LwpHaYlNE2kDOhQnNGhE7i3hzxG4dXxw2xpeeEb/aQ1V3dVO0TyErCgl1CilQmI3B+FvoO/fHE/HFP2OFbmbw3diyUioqgUuFP8yUDX/w+dvC70trkvXCPpA4uw7ftb65W3ffNQhx9ZBLoLkpcMndSSFak7629D7JRGlc6OMuc0jciwNul9Vy+LSvqG5pi3MOQNyPryV2P2ceMMVcTOCN68JcNcMrgcbuS80et3sWUoUtD4zhTqjmKnAUKAMeyEiNJN2uG/o7v8P6lzFl8YkeqrxCFgUzCA1SthW4Cd1eZPLa1Gf2K+NW7vvfijgV1SZfYoL1Z8m1utL/APwjdvTm8757VtTM6KFliUNFS1r5JSS7Sw5BCgr6p0Igi0bz80kDbaDYnerBfoXAAJRCx7rHJ1525MKGU7EG1+E5SCjPFwQmeOsgz42wTMjQGwlS0WnnGlE91ZE++wc5uZ/AW6JhuLrHcLGy6TqOZPjv9a2OUCu8U9Qfl/8A9WKSeW/LnYakUA8kAjptGkH9LGYbFVnahKUmYjY22FQJiPEWDzE6je28wBiLWggEXQmaZTvytuTGhmbBBRmY526Ss9ZtJOhAZ8bdpUOWgsCCTBB1Put2lR+OlphQQoXsZINiVIspq6xhX8LiVjyUkfkbG83LSxMyi95HsvMdOaVf/nWi8bFONkO7oJPxFh7pdLVc06P4FhW9gHNBofhbu7z/AKwD1NmcLhIbqVBqJtlk+nvBHq7WYiciZ+FstTUrFfNrWV1bXJRet51rtTVKQl156ocK1qypAEqJJMJGk9By0tYngjUprKSovBsaPU7CoIgiQTEe+LV2qGgLlSpBRm9WJUCYkZee/M+Um00ejVWqqcIOu5iXG1JpieYIJI6clC1ujFpECqN2KbC+5AheUwJmdD0+utuVPgAwkQFTM5RP19b2Khas2VCu8dBrB1+vvtwpRUCrkkhOXTveevWOlthZeVKDT2dpSQJKVBQGXSI18d48rAhwySnQq0EeHMfpYohamnFdwEK7p1Oug+YHw0tijlcJJUrNqeYIImNfIfnaOymnThSsy1D1KUAlwBaZPdGU6/DMT7rONbyCQgiVITAVO8FJ1+MWjm7KxNBWsPAEhtUkZdxz05GNvGz8SISEoXmSQIV/MO7B+61OYWKu05u2yMMvISsApygQRIB5R+I362AcLagcqSd1eB0PTTmbdylTaSBtuNoJIGg22tpRJUZUqFnb4Db37bWDcKyAuUaryqOhJJ18T87I2Lgs3E4gIGjyFwQNO9p8I+7nZYUnLlJXKTBGg20/GfzslYlyKud7MYSSg9Ce8PlqbQkPgKLH7wUMKqOx4/8AC6pOzV/UMgE/+lta+PP6iPUlpsxMEwCZHW3lZeYnjhw1hZEX5Qmcsf8Azps/n8fHT1bZQopAJFsaUXsrrTqUWUS0A2imzLJnPPs7frboBWVRB1Ow8rGVNDmN/C2nE9xUdOfSzxhQkKJ0ztE27F6MKdaA0SkwZshrQnOpSUQCo5QRqB4+NlRxBzKmd+n142LKbjQER4WtgWVRzrlVf9M656u9qHCTNO6tuKypzZACSOzSNbVsRgG+VFIcrKzSNCEkaeY6/WptfLiPhFjFt7YeoqmQGXKhxJH+VOnvANuKDgbd7jyHiwSknNISAfAQBZxTmQ3Tcfhiyo23w/vNcj1qtBOvdKNP+WfH32PIwJXJSCl+tOoCZCZ93d+tLXsXwHuju5GVSmAJTP8ATexlrgVdqDPqvxH4Wn3MqPegqIJwHeEBWesIBPtZdd9u6PxG9hhgO8gJCKzQiCVDUdfZ00+6192uB9zpVnTTQeo0sMOCVyDUUZgHTS0DQuUxVjmqG0GAK591IcTXgaD24jbbTw+tLLF58Na273TTUVNWrbCBqp32tj003td5rgvcqSFeqGec62Wn+EFyuHtnKRJlA3+dq76B+ZWI6tll51u4FvZHdXTVZnQ/aRI3108rHKHh7XLeSk0tVvAhwxH1+Fr5VPCO5ipR9QB3GwtjPCK6GzIowPCNjafs1xG6H31oOyq1hLg9QVYS7WM3hHMetKSZ6bfPe0iUvBPCqyls3ZW6gZj+8nzO8fxxawdz8OqKmPcpwnTTay1/YulzFXYjWIITFsqfCZS7RyvR4lGOSr89wiwfTtMIF2POyCoFV41BI8j2mnWwdTwswozROrauasStEJCfX6jIZ1JIK9d/qLTveuCahwJDLJjKU5kAfH6628z8cYZ/aNqxbfbWG2sYrulNc8KVSDShJRmUE5QUhRERB3587Uzg8p0LrfVWhiUe4F1ZxXDK6xUKdapX2kJbjWteMGN9VGyUOHFAp1akNqKgDqmqdkADX+L7vO1Y2cAftBK51v8AeNJj5wsHtRD9MgDSNI2JlQ05ac7KdRgn0z3r8uupYuDH10MNPI9Y9YvoOodGcSVZj7O8gRvYLsFfb+oFaZirR8CsJT4Fu9jLLJlpfdT27vd8fasUxJgpht7KGCtJzEkvLk6jUiYn65Wl+mulS6VXb06J9kefhYLENztIC1LYQgI0BI121PytmjDZA+4Kv99YWbKBmsFpddCVUIWCDI7VZg+czZbpuHt1qYBN1hWWIJWvlp18LSUzcbZeUUJQVpakwNvjZbp7nb9VQUtCAAkmOfjaE9DKdiox1cfRV8vrANMg5WrvSlQ1lWvyP32a39imC6lpd3MnOqAC0DAM/wBfoWshfVyoVVju6BJkH5mzZaw8isvgLQM6Ge9qoHcHl9a2NTUEttSgyVMbnbKMk8MbqLKC9dFKpSid2hr05bRbP9F91ggG6qYKTHeDYkyfHn9SbTe1cgdrkJUgJSlJy6aTyI87N3irXnA+B77xGyUdtRUi+zOSD2yhkbPuUR8PO15lJITa6Z08YbchV6xJWcN8LVC6KoqbtNY2ohbDbaFKCphXe29qAdZ5RaO784jYYbzO3ZdFIcpCSpNKlZSMxmAfE6SOXIG0H4mxA8isdqHHnFKSSnvKJKgrnr4qmPDaDYOnqRW0pccKpA0bIKshH8swTtMctDzgdLT4VGwAuJJXPT4i95s0WT3vjiPeb1S0G10rjTaloW2uUKAncKghIGndMb++xa4MT1lNja4L6CkMro71pS2oHulKXRmBnWSFDX/EnlNmiw/V0jpDThAI0UZJBkAgSNyY2nkecEdtpVMhT2dSHCgrStIyKROqCCBMSEEHQg8tBGxTMbBI1zRss6SV0gsSvQXGV4YGwhfd4UeKcU3BS0t5MN1FXd1dVM9sTBAWlgErIUIBASQYEHS1cr89HrhXiJmuxzhelCrsCnOzbu+uz0bikGFZA2SQSdShKkgA6AaS9MK8McO35h6nqWKSodVWzUVL1QhtLri1AZlOEqKnDMklYk69CLC02AcRYAVUN4Dfao1X19nWNICGGe6f78NQWe0RoElKEiFwSqBbpMS7LT1UxrovCHWJAR6XE2CIQSjNba6hG4LiwtgehvnDeJcO3y7WYkpy09R3fRJcRQtLOVDinCrVSRC8iUkqJEETJhVyuvW6w/dbNaXQw640VFpSFwkwVDMkLEmTCo6RuLXbu3hu5dzjiryzrUFFTpceU4tSlbrUtRJJOWdSdtY1mpPGTDLmGMf3zSIS6Gapw1rXbd4kLGYyfNIPKfG1TFcKdTUzZCNAbIImDnWCVeF/pBcUOE1b22DMT1NHTklL1F/eUTxO4cp3B2apEwSnN4ybXO4E+nhhziJiSgwnj+7kXdfl8VQYp6ykcz0j76zoFhfeaKlaAyqSRsLeatSvIwiFSRBUDtyEA78o91nFwh7ao4r4PaTmWVX7QAJG5PrCBvbl5IwRcozHkFeqnHG6qV7EN1V9StMKo3Gy3qCtSVmBMae2qzAYuXOlvtXWWGPZTlkz7iJVEa9J91pm4yXS9UouqrbpwsIddbzLXCAk5TmPX2fDW0dUNzrdWG0I7VU6JRJA10HMW82xl4jqnBem4I3iUbD/ADdFKG52QAmhbSt7LmlaSpXIa6QN+c+dly77odox6s4GHFE5g4pailCQNyI3205dOqvQ3HUttQ5UlKj3C21OWOijME/GzmufDi6hQWtpa0aEhLkfPbf62tzU9UG810LIRa6bzOHWnVhKn0vKMI7V3MAAdZSkCE/MxtaoX7RLB5uqowNiBqVtVbNdRlzIBJbU0rQHl3yfPpb0EpMOF9SEppwgJ2yDl4nfkLVj/aWYNSrhJhzEKlrcXd9+dgpaRo2l1hZiY6tJ26c7W+zeIEYpGL6G4/Cyu0LGPoHs57qkPox1f7u9JDh/UOr7tVejDEzH96C0Nv8ANb2WoriSWkNtAhAOYlSjAJ3EbTbxT4cOruXilgvEJSoC7r3oqlRAMFLT6Vmemk299Lrwo7VFMoXl5SJ0tr9uYJKmoh4QuSCPz/yuZwCvFFTva421v+E0KW4EKgoWrJoO5oTPLUaD32WrvwsXIbbbSE8zMk2kCnwlS0jIfvB5LLaBqVKAgW0nEFAyr1XCt1OXk+NO1AhtJ/zH8Lc9B2dcyzqx2QdN3H6Aaok+PSS3EAv58h90lXZgNcdpUIS02N1rMe+Bbt2+sK3C56ldFK5fdeNmqdOYA+JGgsqqwrfmISHMUXqtDMz6pTHIg/5jubL13XRc9xMdhd9G20kcwNTbp6PAxEA6JgjHzP1d9m7D7rEnxDP/AFnl56DQfc7n7JlvXBxDxn3b2vRFwXar/wCb0ozPKT4q2Hzsr4a4bYMwco1NBdqHKxft1T57R1XXvHbyECy9U3k2nTOJ8LI1Ze5JMKseWWgoTn/qSdXa+g2CCJamobw2+FvQaevM/dLNReLbSSlOg5AW8I/2m9ym5/S5xZUpSQ3eTdFeCARvnpmwT/xJV8Le1VZeiiCSvS3kn+1suZTfGTDWJUIHZ3jh9DKlcytl92f+VaLTwTGnVmJiNx3BspS0Jhp3PTX/AGW+Lk4a9Kr92lwJRiG4a6iAmASkIqBHX+5NvXqrvYTova3hB6IWJ3MJ+k1w4vhCgkOXyxQrI/kqJYJ+DlvbWprZTJWetsjt259PWxvbs5v7Fa3Z6AVMRvuCpDYcDrLbwiFpChbsn3WTsPVAqbmpXSZOTLPkY/CygRPS3RUcnFgY/qAgzNyPLTySDflNlqQ9ycHPwsnbQmbOC+WO1owoAyhU6dLIBkHTT3W6KlddiyahtnrBpqEgT1sIyrIsKUNiPv1+63Bk6A7cptitQcupIIHvFrgVN6UgsEQBbYPLadCbAoWlaEuCNRmAt2DPTzm1oFVl2DJ/W3QJjbQ2DzQJJ+dsMbREfK0wSUkNm1mBJ0tsKFgcwkazHhYRK4OvLnraQ3UShQfmOtita4GqmjcUDBcLStP5knf3gWHzToCd7Er5JTQKfjVhaHh/uqBPymzu2SG6PLAGsW6oD9tGYb/KwayFAKSeVuqEy/EjU9bPa6ZOHtgnTMNNLZbjtY0z/Xwtlo5U914CjDF51TzeGKVDS611RpQkudwrHdOp3E2lLgZgHEmBbtvS7sQNMtpfqUuMBt4OSQkydNv4dN7NCgvRq78XU9/PtvFCK1VQ4EEFWqiYGw/DW0nJ4tXD2edVBXBJ1MhBAGvRX1tZ4A1pzFBmzOGUBO/snDyVCvPrO0W0hAzQsaSJ2BNmsnilcqT9jQ16hoTojQ+Pe8LCOcVbneWEro64qjeGzz12V4/0td4zeqqmF/ROMIQE6iBB0A8NPLa2EAqKSJkgghQ1Ph+fhZqjiZcSgJu6vnLzCJI5D2vz2nlbj/Sdc0hSaCs7xgwhHPkdfL61tHjNPNNwXdE7MkEpAkgaCJ5bfXWz3uO8F1d1MBTiitr7NUSNdPwj52hb/SfcsEpu6uGwGUJKZ6b2WLl4zXNd6HBUXZeC0rWFApCNwddyLCme0jRWYGFh1UyNr7LveGmnlr9dLBqKe66VLOXnA8/wGnjaNFcecMoUpSLnvUhMn2UaaHX2uv10Bc4+YcUSpVyXnodT9mJG++bxtUuVcu1SgVlYKu6MiYSIAGk6eW3wslYsUHLldAaUkKLaRCZjvjU+8g+7pZhf6esOp7v7kvFUrAA7kHb/ABaR8fvsh4g9IXDNTQrpE3DeoUohRV9mUgAjcZtNh9A2hJctKnG5oIuka/n0N8aeHC0AJSq/aIZVT3pqmjqI93xPjb1gYTz8Zt450WPbvxxxmwC5QUdXTt0t/wBDKX8uZU1KehOkJj429jWMuUqtnSggC6sxkOJIXZA8bBuCUkT4jlFhZzDU8uti61GNRoREE2ePdQek9xJKjGvj+FgHEgjbflYw7lmBpvpG1gHCNSdevS1oKod1GnFvifcHCJVx4mxGwHqR6peowQ6hspUpoqBlZA2R5xNm9d/p4cHKJIDwS5IEA3ixz13BM9bS1eXD/BfEPs7sxvhW6r+oqUmpap7ypEVDSHQICwlYICsqlCY2JsJSejXwFa77fBnBDcq0/wCwabf3otaiY8i7TZAe9l7EKKnf2hHBptJKEUupgZr0QNJ30QbBp/aLcHogJu/XSDeit9dNGPC05U/o/cEWPY4QYKTGki4aX/4NbKlPwW4PsgdjwqwinyuKlH/RYhEt9XfhQvH0VdHP2j/CBoED90CP571dHkdKY2B//SQ8K1hRaRcRSBJJvZ+I6/8AmvutaFrhZwyb/uuHWGE9SLopxP8AyWOtcP8AAbOreCrhTH8t2sD/AKbMRKfi/CcGMclVBX7RnhslIXkw+kHc/vWo8Y19VjlZx8P/AE12eLl+jCPDzCt13xeop11RYavF5IDSN1Fa2EpGp2mTayzOD8JtK+zwxdKdI7tE0PuFkXEeEMOUuJMOX22qkudVK5VNKfZbaazBbJGUlSYiQNOoFo8OXcuUhKwaBqYycccd6lvtafgzd6klZSCb6b5Eg7gcxbpvGHH6deEN1thO83u2f+sTaQ7rvDBeH7vYuX+2NI8GQopdrL0bW8tSlFRKlFQkyr7rcv4vwBTuFD+MLjTG+e82Rr/x2nY295Qv/wCqZNLi7jyCFL4Z3WhM8qtC9f8A2wsrJxVxrfAIwPdSOpKkKAHT/wA4squ4/wCF7cJc4hYZQNfavdiYHjn8R8bFHuKHBptaVK4m4QTBMhV8Uxn/AJ/CwXRB27z+ERshHwoL+0fGwoQBhO6Ukci2g8v/ANatir341LaKv3DcbawJGdsR8qk2Bd4y8EEgpc4p4MSRv/2tTkj/AJvOxNfHLgCwpOfivg9J03vRgg/81h93b859Qp8U/KEkIxfxgqcT3lhdVHh9ipu6npqtwuAJQ42+XQnIe3JMFlc6CNPcBV0PF2tWS/V4XKCIID8Ry6mLN9rjv6PlBxlvm/HuI+GvU67Dd3MesNPBxtTjVTVnICkEZgHZj9LOR70p/RoZlw8T7k5SEtOqPyR9TYJpWOHiefVHFQ9uzB6JNGF+JACQqvwwCFSAKwiAP902AvLBXEqvIKL2ww0Jkk1qlSTz/uvL42UXfS79Ghru/wCkuhmD7FHUE6eTdi6vTF9GlOUJ4hNuFX/d3dVEdf8Au7B9mU97lx9UX2hPawaPRIqOH3FJDhJv7DQVGXuvk6dP7n8LKFFw84oBsoOJLhaB1MOFQA6asW096Z/o4NwP7Z1C9/YuqpP/AEWJu+mx6OiFjs8SXm4BAhN1Pa+RI8LI4bSu3J9UhX1LdgPRavXhpxCTUN0q8d3Il6pQtbaEoClLyiVR9lrAPum1frrvX0o3kmsu/gdip1twdol5D93ZXkkSFJzjY6EaDe063h6a3o3VgT67UXnWBsktpcucrCVbGMx00tN+EKmkvPCl13tQAopaykZqGUqTkKW1oCkSBtoQI5WYYZBfwE+qXf5j7w/CpWh/0vVLC08E8VMKCplVdc45eI5bWjT0hr69ISgwCpjibgW87humvq0NIfq667nQt5ILiW4pxnH92TOg7oHO3pe42lWuZSo130FqQftPb1FLhPBOH2n1j1mqraxeXcdmhtAO/wD61VpsoY43Ai6d1bI9pBXmpfleaipfo9VJBykEEco0iQdvOeY3OsPVqFt9lUIQS1o53u8Y7uvSZHWNTFi150jz7/qxfl9CU5QQSV6+0k7GdIjUiPKyVdC3Wb5WioQtCghasq1FMq3106Hn8RbSGipHVPNAh5KwpQJVBSQO0KVc/foROghR3GhxwsOMJNMlJSheckJBJVAjLGkbkxGknUmyQ68yKY5ohTeVKUxmiQZBJ5jLygag84VKdCHaft22kkvZVgxCpVqTqBMd1MAnaLEB1QyFdHgTiFN58Lborbrap6q8KakQy6pwCUqQMpkJPtd0HfprAssVd4XpUX3RLvOnYC3g8zlQgBJGRRA0HRJjrA15Wjf0XKxdZgN1ph5Lz1FeD9P2mYg5VK7RAkRMhwGfPraWb/rGbsoGUXheDC6x19DFKvLnzLPdEjQ5UiTPMa29ao5jNRRvJ5BUMoa8oq4xTUzLeSVvvqLhStSkjLmkb8tU8to6gCsnpY3O36/dWJkNKyPB6lXmlIJIC0ySY0g+Gm2lrTUIrXaVs1iaUuinCXXmimFLCf8AKmBuRoNOdoU9KO6BX4BcrUTnu91urQtAJIAVCh19lZ6T100p4vTiegkHlf01RGutILKlt6UjrKAtQCm80CE90adR7vL42mb0HMDPY39JPCVMFJDV1PKvh8qVEtsJKoAG8rKNLRXedCqppKgtKW6UDNoNIA0M89Ofh01tLvoD37T3H6UGFRVultNciqo0cgVuMLge8gfK3kktw0rVi94XXqxxKuxuvuimzoSQ3VJV3gVASkjbnrGm3zs1rnwglynQtCg3BBJVoOegSNTsLLPGPiLc/Dfh3e+PL9u+trbvuZLT77VIhJeKVOpQCkKUBoVgmSNAedq5UP7SXgLTJSo4Oxok5o7tJTHl/wDTjW3mfaHC6+qqeJSsJBC9EwTFKWlpOHM4B11ZGkwlffrlMzTqpRd0y6VpVnnllSNDz18rPShuFCkpabBCQZUY0nymLVPb/ah8BmVBDuEMbDkQKSmk+f2/SylS/tU/RrZCUqwjjxShqR6lSwP/AL+3MO7NYxKfFEbK3P2ggI8LwrfUVwNlSQlRjSTl3tDfp38Oq3EvozX8q76QvPXZUUlekIElKUOhK1dAAhaiT0BmwPDX9pF6K+KWKqorqnEFx+rIzobvG70y+RulPYrXB/zZQZ3tHPHz9qBwsvrAmI8D4Nw/frKr1u6oom6ldKypJzpKTmlzuggkTBInrbWwrs9NSyNlkvnB0AFz/gLAqcRkqTl+HmSV504YF13Nf91VF+NOLpaesCqhpqQSkqCjlUecAxuRmt9AuEMXf2uw3dN9YOut0UN50TFWzVVQy/ZuICkmJ1MEeFvnIvbFlPeLv+rsraV2gXJAiNY0HmfibeqPBz9rN6PODOEGDMJYnuTGH74uS4qK7a40l3MrZU8wyltakKLwJSSmRIB12t31TRyyObISR1sLn7dFgPkYW5Wi/wB/5dX/AP7Kt1boqL+rXK5Q1CFaNg+CRpZVQKGgaDdMyhtI0ASItRB79sX6MhScmH8en/8AZtP/APlFkiq/bDejg7OXDOPv/wB30v8A+UWpOZJS37pTku6kXPqULK6SwkeLdL6K/dVe6EJMKsjVd9Eg97a1DH/2u3o7uJ7uGMee+gpf/wAosnP/ALWn0fXdE4Yxz/8A4NL/APj7c/Ww41UknhOWhTxUrPeeFeuqveQTnsk1V7knRR91qNu/tXeASz//ACvjffnR03/4+xZz9qlwEUSRhbG2o50lN/8Aj7c/NgWNy/8Ahd+FsQzUDN5ArtVNepYMkxzt59ftWbjN4YfwPiBtAJpna+kUoH+YMrSP+RZjz6Wcq/2pvAY7YXxqf/sdNp/9/aEPSt9Nbg7x34fUuGrgw9iZmvpa9NU25WU7CEZOzWhQlLqjMqTy5G17s/geLUWJRVEsJDRufqFOuq6GWkfGyQXsqdYPvtzDmLbhxEyqHbqvGnq0naC26lY+63vO24qqbQtkKUHkhSMvMHa3gAXGoOUrCs8pECI8+u1vQ+6v2leC6PB9y3HWXfidytprspqas9VpmaZgvIaSlfeDxdcTmCjJWienK3U9ssFqMWEJp23Ivf72Wd2cxGKi4jZSBe269LsFFaLnNM8QHGXVJKc3eE96D0OuxsvH52qt6CXpE4W47XfjCmw7RVVG5cz9I+8y/TIa/vw4AUlK1Ff9zqVGflFqCTy57WVBTSUlOyGUWcBZFqpGTSufGbgoN9CXmltH+IEWajoKVqQT7Oh5WdiiYjptZuXs0WaxRSNDChHW27RHUhZlUNMyLFcmT8YtmbY6H3/hYMqG+Xa2T1M21AFmk3R2mUexSAT3ZR7gYsLPONTpFidOsHOkKJAVtzgpB++bGAox7VrLdQq50KFJO2sx0tgUDr99uArlI91thQnY2INEyECpgHXw526QSICTt99gSSYAMaW6SoDY677WkFBDBXXQddrc1CQ/TuMqAIWgp+ItyFDUdRtboq52kkuaR3taNlRkkoE+ca2M0J+3G8DlZPu+Ay5TwPsnVpA8Jn8bH6IgO6ybME3NOEAwNVjT+U2ywE/4494/O2WbVOvBLFj1RdVzVFVSOBL7CUqCikESVBJEDQTJ8JnY7POl4L8R32EKTjO6klSAsRSHTNy9n7vHaTZp4+addw1WNpRJWWyB/vp28fz8bXP4bcLsVcRlmlwpc79c5TMhx1LSUhKExuSSAJ1jXrFngYx9y9AqHvjtkVa0cEeJAUD/AGzurLvpSaAx5dOf52OM8B+IjmZs4yuwqA/9CO/w+ptYiuw7X3Jejl1XrTqZqadwtPIdTlUlQ3kdfMWn7g36OTeP7mXfTleKdpIyJzJzZiRsI5DT8bHfBCwZjsqzaiVxsvPRXAviUpIYRjCgccyapRQKJnTQaz8bBVfBLiRSns6nGFG0txEoCrtIPmJjQxHu+Fz+LnDF/h9fT13qqJLZA7RM7bxpr0+Nogryt1Z7RS3FqTlGYyoDkDrP372dtPG7UJd4kGhUCK4M4/UEhOPqGMuXS7xrvO/ntbocGcdOlLTnEOhSUKETdiSNNf5tt/qRaaydZKQJEaazpy+NgVCEgISZUNRMeOtp91jS7y++qjZPoy8THvs1cTbvPabkXYJI+Omn4eNu2vRj4gKzLd4p0kHci6ZEkbyVdJ1+dp/uCo9dulp7tSVtQ0oHckAR8RZWaaQlQCgCI70E6kae/e1F7A02WrGA5oKrafRZxmpUq4uU4MkkJuYaf8402+hZNvv0VMTUtGqre4rNumUpUn9yhJgq5faTufna0ak5kqDigNpBG8yNpsk4uJNxvvrKRlW1ISQZ+0SDHjpvYLxZpIKO1rSbWVJcH4aq8J8drjuCpvX15V34guuHUNBorCnG3CCkT1A3O3lb21p9WwSCR57728a6tzJ6TNG2lQM4hulRKdI1ZG46EgW9ladOVkJ13/G2fMSbFTjADiAuiBrCjO1i7o38NrGzH8p157WLPEkRPO0Y908mqT3x1PnYosD6HKx51IVqoSZsUcAzGSdTztdaqTt0g4jx+3w6p6e9nrpdrvWnxShDbiUFJKFLzEnl3PnZF/8AKhaCspwZUHKo6evJHOP5LJXHdSRhq7FBWSL0SJ56sPWhcrCFGUgjoTP1z+hbSpxdiz6hxDtFYJPpVZDCcBu6fzXkAZ9zdulelo4lHcwGD4KvOBtyhq1eC6ADCoyiD1n89futvPKiCZGx0kR8I6fCxiwKvxXBWEHpaXidRgGnTO03kr7+ysEr0tr7UTkwPd4CTBJvFZj/AO7tABUZKDBIMAADUe/3eNtFzLmKNdAJkb/X32YRhPxXFT9/5WeJCPssGXbmJ29ccP8A062jPjrxRxDx4uK6uHF43XQ3VRXheIW7UUzq1vJ7NpawAFaaqTB8LM5DqjCgkASIkCAI3tpt4C/bjWFjuVsQQebTg/H5+6zujBFikJHA3UeXn6N9yXS+lly/rwd7T2VBLYA5aSnoBYJv0fsMEBJve80iDoFNa6bexvaasbAB+mKz7aTlPUTIizadS6QlNMop111gTpYLqeIGwCK2olIvdR2jgBg1IPaXhei0lUd59AJJj/Dp/X3n6b0fcAuJUVPXooySo+tAAAeQ2+t7OpXrCNFE9yCYnnH6WU7sbqVqLocJZSTIBJ5dP0snQRgbJ2zyE7pns+j1w+ToBeUE6j1uOcRtrzEWFPo/cOwCVUlYRuQatR5eFn22+JHdAB0nbUTYwHhJSo5tYPXw/pYfCj6Iwlk6qKxwZwJS4zcpPUaxdIbiRU9gurWUl0VK2yrcH2SNJjawGI+F+AqGgNTRXIUlCtvWXTmHT2uWnjzs/wCpj+2NOAlQL2HqpGpOvZ1jB06aOfMWJYgbS/dlSkojKmdfDX3bfdZNiYWnRM6V9xqq+1F2YdocYpoauhSiiNKt3Kp1YCSJ55pnQWR10NNX095111uf6vQZ1JVmOZahEASdEgCyvxBCXLzu+7GVILl41CGSJM9mkyr7wIsnXShLFNjChIc+zUslO0pyHaNfD4+FgcME2Rw4gXuuL8TRMYQu69qenCXVuNhxRJMg7zr0+NlDF120jF001TdlKlBD7bi8p3RHekg7ADX39RZv19QHeEFI5lGdCWiJVlg9oNZ5acvEWGxxfjQwlS0/antH2mllKFDNlSBOnL9dLOWNASDyUvUNDSXpX0l5OUCWqBVShDKQTDqTzKehOvPa0gMekTxsurHtfguk4k3wxdF306WqSmYUhKWUpIQAFBIMACIn+GzPerW2rloVtRp2KgpIGUdwxrz57a/IWI0V309+8YMXFXaJdoH1BsNuCSlSlSTrBGgPu85rl7Y9URoLzYJ8Yy9IHjhQ1tyCk4uYkZZqKrsn0prSMwzJBG08ztraNePHEHFONa6hpMSYnvO+F3e0tTfrtWt/ss6xmCcxIAMD4DrZ/V+DLjvFFNU1dS9FKsKaIcgJjcQNP5flrFoa4x0dLcWJaxlpwllNGzUOBagpSlqJAA1E6GI6Zd7RiqGSPsCpuhfGLuCjW9WbmZYFZeTrocHeATEue1oJ13nXpBnqhXU+aquXUKDjbKGyEJMkJBVpvuAfu99hU3bX30+XnnA0ypQCUZ5AJPw2AJ8culuKBz1e/wBxpa0rIZCTlIOWNNCdPERtp0i1y6CAlStIQlTq3G0ZoIycuZ567mSTqCZ9rRQuFdL6g+yttE0jhCyCnKUqAKRJMGCFCdD4kwLJNQ/nRmU4MpOUzOkAAakwZjYfqQ7nr2qW9w0FJyvJLalZAOzcnTWJGo5awrrrZwUiLqyfow34xTLvi7XnlNKqa1DwUteYZVMp0JJ72bLI13OgAm0xYii8b1ZbpQrLRvNLSEoJBOcRkERAI3HkIM2gX0cK+hpb0v5h2mbfdaYZdZW23KQsqVMa92cqToNhB1Gtg2qijuG5U194IJqKlxBKFwIWVARMyImTOwSelvRsCcZ6JrCdAqsoyOzJTfevlbgRTILbTOeQFQVyUgqnfaN4jMN50j/jXR1tVgesoaynQl2qpXVEqCiJGx25k6CBAjbe0rUbl7UIFJeCkhltxZ7VB9mZEQBI/jnU6kbSRaD+OlVijGSqy67mqV0VPTJUkrRIUBGoEQBJJlWsDpbWrv06Z25JFrIAsXKoFI6s04qFZlFsqCgsghMAaEnlJnnv1Ih8ejXcldXekRgahuZKg+zf9NUJUknvMIV2ijOmyEqnaRaPaEKoXaijWQHWVqQogagpkE6eZ36aeE2ehzUXm16TmCn7vofW+0edZfIJORvsF5l+BSmTr0iZt49OMoK1Y9SAvUXiNw6Z4q4Fvvh69WmiRf8ASGj9ZDQd7ElQIWEEgKggGCeloJw3+yDuh8pdf4qVrrR1KlXO2lIEzElw/KbWww/VmjvelebTT5kOCO3MNEzAzdBrvy3i0tLwtf8AiBXaYsxKlFJ/6Dd0ttqH+JZ7yvkLcfiT6lsrRC426CwH1JO32W1E9jW+Kw8zcn7BULxx+zN4F0uGV4duHHF/Xli3OlLRoqZlxAE94KQfZ0nXtBqduVkPCX7GFm8UCsxbxdrLvS5B9XYuxtbgA2lRXA05Cbell20WHsOUwpboomKdCf5EiT4k87Y9fiQD3xapHX9yJdUTX/8AVu3qdf2UZJDMMsTPud/QaD8qgzv7GvhmyiaTjVilpyAMyqGnUNOUaWbOI/2OWDKS6KyspuOF+u1DFO4tptV1MhK1JSSkGF9dPfb0Nqb8PJXlZHr73C21IUuQQQdbZdX2pyG8Rsiw0U0lg7ZfOJiPD7mH3+xW4Xc5WknLlhQ/rPvsgkk762lHjldT12Y2vu63GC3+7r2raWTpOVwj/pn3j3RcbeiUcpmha87kLLqWCOQtCwEjrrbWpMzZwYXwBjbG63GsGYUva/nWBmeau2jcqXGkyAFLS2CUgkwCRBIMbWc7Po38f6gEs8E8dKCQVKIw/VQANyTk0HjYrp42GznAH6hDbC94uGkhRvHj8rZH1FnyngbxmVd5vUcKMX+pAhPrH7kqezzHQDNkixu7PR44730Cq6eDuMqpI1zN3JUER55LMamFupePUKXdpT8B9Co7n6m2ptKn/kq+kh//AGOxt/8AuV8f9NtK9Fj0jU78EMaj/wDYz/8A8Nm77Tf7jfUJu6VB+A+hUWTbJNpTT6LPpGKH/wDRTGI0nW6Xh96bIONOC/FLh1SUtdjvA173DT1y1t0zldTKaDq0pzFKZ3MWdlXTvdla8E/UJnUs7G5nMIH0KZxKOzSQCVGQokaDyta70W/Qww16QnDeoxxX8Qa66KilvV67V0jNCh1PcaaWFZisHUO7RytU4CTA10tfn9mji1LeHcb4UdcSOwq6O8Gkk6nOhaFn/wC7b+Ns7Hpp6aidLTmzhb91pYFDDU1jYpxcG6tD6Efow3T6N+NcQuXRjisvlrEF3NtOMP0iWkpW05mSoFKjJhaxEc7XEUAB0tAvDe+P/l3d7ZX3XS40T5tq08dQLT0sgm3N4ZVzVkZknN3XXQYlSRUcoZCLNsuTtp8bJd/U6i2h5P8ACcp8rKZE6T8bB1iQ9SLQRrEi21TuyuBWRKMzSE0gQRr8OQt0NQAk67W6W3lXlI3+G+otwQRzJ87bQN1jrthSU1CtNFtg/An/AOKxoEakjfexFJipaXOqgpGviJ/6bGgokyNxroLWI9UF267WsBClRMAxA18rETfjYb7Ruiqz3gkJyd6T4T4fUi3dQ3WLUg0tSlCdMySkEqPmdtJ1twwxegMvXmFgH+FoCR7/AH2sC3NQWv36Nhd1aZ00b8/H4+Nh03oolQRRvrCUhSTEBYMbTz12MWBTS3j2SgbyPakkpUGwQE6QI5nTfxtgpbxJSTeikpgZgECVEAbdOvTXytMZVDUJQpKpVQz2qqdbJkpyrGog7/jYxm55vOyQijvI6qvlRBER2YAmInrM62U0kaAnUDXxNmPknC4YHZ1dSkyO0SlwT7wfuFlCintZI21M2TVKCK9gn/aJWjptB/OynQwHdQNdpsyY7pYJcn+P/itlu8q/5j8D+Vsskrrwix0Ut3O64AmVvN5UgR3ivn035G12+DvGzFfCRblbhlVKU1bKG6hmoaztuAbEwQQRPI8/G1Hcfqi7MpTJNS2kykbZtvLzB5xvBtvgekbrr8u26n31N01ZUM0dQsESEKWATBHjt1Fo09g1xdsq1W7IA7pdODEWNr1xhieuxNfjra66ue7V7KnKgdIA5AD5a2lrhP6Rl84Au9y6aZNPUU7hBh0TCo3GotCOO2XbrxZet0VFUKhd1Vb1AhQBnIy4UjyGnzskCsW0vVR7u+5nra+A2WMdFRa/4gpE4k8R67Hl5PXneVSlx15ROZOgHwtHVSorzKKwrQ5v4swnX46726FQt0FOYBRkmDqBtH3/ADtyumdySUgxA8+WotJoDdAkdURX9mVBa0lJ1JmZ1+fPzsC4gJC+8mSDmnw5mfvsO8ypkKGcApOgkgzGu+362Kr7EgGPaVlgiYAPTfaetpqVku4OrS1eDlE4MyKgFQMmM41HxmPfZ8inKgtbYPdCiIEgaSPPW0UtPLYqEvtuFK0rCx5gjr5fK0rXZUoraFmub9h1tKiPZAJ3Hxtm1rcpzBalC7MC1a7BRILghMQCDHlHwsj4kZUbmqe0AUlKm1GdiM4PPx+FnI6krSY7oTJ1O2ug+unKyLiUpbuKtKkKOVvMAP8AMJ+Gvvtnvf4CtNjRmCp3eXZf+UpRuoSQRiC7EjSRopnaNDrPXTpb2UZ9kSdZt41XshbHpJ0a3m0JWu/LqJykAkHsTOn/ABafdNvZlhsqG8wdPC1N+wKdos931WKA3IA8bF3AJOXTrpY4ptIAJEE9dJsXdEbAd6zxqMlkQfT/ABGxNY1knfbWyg8kEa8uosVdGkHny+vfa4xUnbqLePCJwrRKRui8myD/APVOjl52gtSVESpwiecTawPGulNRhBsSO7XNK1H+FY5+doL9VOQ5IMGNOk/QtpU7hlWfUC7kSKHE+06SQZIHPQ6WyEhJK1bEqBj5WNCiKl5QrlIMbDmQfjp/Wwhu45gsZYVHTQ/Gxy8KuGEpOWnUHMesgb23lURmUo68pkedj3qLijlCisAGdJOnh7xZSGCsUuaowxehCgYUKNzaPLptZuIBumLcoSAAAoEOCR46HUaRHXS28sXpdBKlaXg0NY/ilMfOzhbwXiYmFYfvBASI7zKkjbTePGwNXh2su+puk19C5Tl28KbIF6EgOoCo672eQPa3MRomicx7soOqU8ZsNvOUzmZRAQuIMDkdf6WQH6SiYbJqHy2FzMqglQ5fX52kLHFFSU6KPMNcq9zue7zmdvrWzDqqKjrlBpzVKDMAxrEWBmzOurIZlbZI+e7O1ITUI7pglKzGpPTcfjZZux66lMFtypl9xXdbBIBP1Fk9Vx0CVplWm8lXQWV7pui6FINVCs7ZzIlU9I+vO0pDcJmA3Q/7uaUkuEKB5iSLdpu2kV3SkyQCRmyjx0sYbcaQgAqA0mI57zba3QFE5yEgHQnW1Y5lYBaEitXbTPcRcLUj8hiqprxpV5SQcpcpVEAnnAJtYo+jVw7rad6neqb5hYKFZ3kAHkdkTavS6lDGOcIVadOzq69uVA86dK//AHYteqnKF5VpU7KpC+6DPlP4WdhNyEQNadVX4ehbwGr69ivqrvvSrrKdyWXFVzqQ31PdTvFq6q4G4Iub01bw4Su3atzDF80dNUdh6wvOsKp3CZXIVPaI5Hp1t6FpLKKxFO264CpQMEgc9rU44vu09z/tEcEValhBvO5aJkAfxK7V9ESOcdfw0T9BdStqpFV6Gno5U92i42MDldCjRTLl41K0HWYADg5iy/Reh36NtRTobf4X3W8lXdX27j7q4GwSSuQOqZg6dLSUimSlbgNP2ozGSpBUd+tlKjWhitQ00ltDaVbjKB99nyhTK8oeJ1z3VcWLsSYeudopaui86mjaZjRLbT6kITEyISNz1+LYu6vbuPi/iu8jop+haqCDPeKmWzv/ALwPPU2lD0iOxTx/xndAZCnBej7hWhspzFZC9B5L3/O0NY8qv3VxJdfhpLddc1ACc2i4ZbAB96OfXxtgTvzZmq3C0MIcny/idx9r1dNMUzBUEg+O4O+p+pNq/cfMRF/ESqenq/tlsMioCTp3QrKPCM34crTSw6EpyoUSkkSY358vM8udq/Y7rbiuvEF4VlawuovJ+ocWULSFKPeMKHJIMCNJgDTe0MNitIXKxWyXYAmrQ3UpTX7wvmsU3StpDmVLkK1nQRt9QDNidxEVt6POMjszULCUJ1UQkqGnWdtfzsSve+7yvJXZ1CuzanMGhonpPjt+UWPYWYZX21TVEpYbGVRyk77bAzoDobbw1KyksVyUOiG15iqVKAVJEqVB3BIjXWJnxEAUNMyh1Km1r7SJXASoBO55kcyI59NbdVVQqtdW3TH7NRhKQgqBSNJA3946z5K1z3a52SHnyvtFpzNk5VH+Hr1kE6z5SDaQTXspU9G5+maxnedOtHZmooGnWyXEqSpSXCConY6q0HtR1nSxNUmkvq9WaN9C/VKZKgooBObQTsfPQ6E8zAip3CtxN3Yu7OnXToNTSko7QCCCpBBTH8UAxA6nu2tfgHLWAO1YDdMwTnzQSogGQJOqTJEa7jfe3oPZQ54cnmqdUbJ//vKlvC7qZ6vQhvM22pwkTBiCoHLI5eMAdNI1xBfdK8XqOnpEvOOLWArKYHidNDHUTIsttXbfl7FdTTlTFDXIb7FlDZSppABMgiFEkRoTpJ8ij46qbg4Y4bcvK8B2lW4nsaWnBSVrXlA7oMRoB3jEAa7ablTNI65Is0c1ARty76qkXEe628McQ7zoG3dA8KhfeMIUtIWU+4k/LeLWo/Z3cP8ACuLOId441r/WfX8IUjTt3ttuFDa1v9ohTi41VlSmAj2RnJM2pvi696+98U3hX3opHrD7pK8iRlkcgee0Ttpppa2H7NTGDN3cY7ywvU1CUpvu43ENIOmZ5haXAB45S6fcfK3kdeQ57y3a5WlT6EXXpWtJgoCimRGvI/XXp4Wk67cXisuqjqUOn7ZhDmvikG0aEZACJBkEHx008YsTwzfS0Xf6kt3WmqamnA27qHlpT8ki3l/bR0kMEckZ52XW4HStrHuY7lqpaN+5v9p75sWdvmBAXy62Z7F5LIkqgdbdprHnJKQVC3mMlZKRuumGFsZunE5eqlbqsUerVLzEqs3b+xJcOFKMV+K78orppyCQupeCCqOSU+0o+ABtENf6XOA6y+hhfh9QvX3eTiHC29VTTU2ZCFKjYrJISYBCZsSCkq6zVjTbqmLIotBuvO/0xbo/dHF3G9IJHZ349VJGXbtjnGu+zgtWVY70Hfr1tYr0l8Y3lj7G154xvqjpad+/GmansadGVIypCAIMqBhB5mTm8LV4fTleWidlHc297wMPbRMa/cALgcUt3hxCsl6CnGK9uEfEe+nLsulq8F3zcy6YIWCcqkOoWFRmE6BQ1nfa3qbw8xXjjiVgq578rXbvpGq9txLqC0HFMOtOFCoTAQc2ihKSRmiTpbxm9HV15PFy5KZh7s11Rep80SO80vlz1j329XPR4r7wuS7X7lqL0dqKRCzUNIWkZW1rAC8kCQDkTpMaeJtzHa+kbm47R4tLroOzt5Yw0C9id1NzOELnbfTW3s4/elSmMrlY52gR/lT7KfcLKq6plhIbYQlCQIASIFkdy9JTmK9vlYhUXmn2s5PlztwJc9xXVMpC7V6Vn7wzSUka87Enq/2hm/h6WQai+EJIBJ73dPn9ffYkq+0FwIBVOaDyOtpNieVbZAxqWHq9UnUkjpao37Raj/e3B65rwSlS1XdiGnKo0hDjTyDt4lNrF1N9CNUE+AE/G0Fel4j9/cBcTsgEuUSaeuRB0T2L7az5HLmtt4KHQVsT/MflVMWjbJRSNHReVbYAeSlQnvAGbWa9AG/1XZxbvS6VOEIvS5XUZR/EtDrah8s9q13m32N4VDfLtFERGxMjbTaLSn6Kt9m4+PeG6kKypqHX6YiND2jK0pHxKbep4lHxqORvUFeX4ZIYayN3mvU7DV6+oYruisU5AbrmCoyNElwA7+B/C1tVSBsR77UOcv1xpaVoPfTCkqTvprP0Putei6a5N7XPR17eqKumafTpyUkKHyNuLwhpYHNK7PGCHFrghdTO8W5UNNvObCEd3aOlgtSQdd9bbrVhOTerGexfcbKdAZFi0QSN4iLKl+tlLiHAJBTFkjOIiSZM22YTmYCsiZuVxWLARkUDP2iADPUx9xsaUDvrMWJvLzU7gTuEkiORAkfOw5UFd4bK1Enra0xVXLuNASB8LYJ3g624JCjIMQOm1sJOsWM0qKEBAOs+E23MjYx42DmCbZKgO8SOh6WkN0yHCjPSLdJOkA2AnQk26BA269LTCig6x0t9g+f9m8nNtoDofvstUSSHQqeetkWvldI6ge1lMeBFlW7XA8GnU7LAV5yLNzTFL47KBK1T/mtlg5XyHyNss6ivCPFLK3vUaZMKDl40yQo7jM4np5n7rWkpHWbrvWmeYqFuCkfQ7mCQmQkyCPhasV8oS5e9yNgI+1vqkB5SC6OWx5b9bWQCmO0zdoolMqmJkD7/AK5WalAc1wKDWNzANKdPEi9rtxBj2/7+w+4t2hvCsXVNKKCk98Z1Ag6yFFfwJs28xQjNBCZmD5fnYamepmVDMlBSVDvBOpj6H32AcUA+oMCRIKDoIT019/x+NyMcNoYOSoNjytsE48OUVG/XNCrcSlskZsx0I+/lt+trD8e+GXCfCGBbnvDCV8Nv3lUhJ7FL6Vl1BQSXYHswQB5ExsbVYZrw2orQrKoEd7WRtE8/CxuovesqEIQpwkbAE+/rpP4fFnMLnA32RGkAEEIjWFJWpCI906ifAfU2JLSqZIMHY7DQ/rYy48V5SlKes9Tty+uVgXYUpRgERsRqOUfW/usZMizhXn7qo5TI08fGz+wFePbUDtA6onsHMyEgwQFdNev32YqilQOaITsAr6+jZWwreKKC+W1OrCWXQWXCZ0Ctj7iB8LAqWcRhVmlfkkF1JqkrcGYwAdIB0kfW9kLFb7X7gr1OtlUMrJCUnSNP1s6G2iV95EmdQNgQfu+/TrYK9sM3Zet016qu6aisKWVuOZKtbAUjLqkwRp4jW3PyyBjTddLHCXOFlSPFLbbPpH0IzAkXvdJVuI1YOx+7wNvZil1bBIHnO9vIDiu7d49KKjF03O3dTIvO6AKZLzrwBlrMSpxRUVQPLUC3sDSp+yA89d7AzZmNPkhEFsjh5rtSYOUn5WLPIMd2QT1NjkaRqYsA8kkSYE87Ti3Q5EmPNFIzAeJsSVnBnUjf7rKTqVEEQdPr8rFFt9NLXWbKk7dR5xjH/wAjXiEgqTUNKAJ09qNPjaCVuKAJBBUI5ESRayWPcPV+Jbgdue7snbvLQW86so7qgTry0Fo5a4E44en7S7kpOgBfUYHlHnY8MgAsUJ1PJJq0XUasOvjtIy90DulBn2t7Yc6gFrg5f8IP1+VpSY9HzGc96tuxJI/71Z2/3bGEejpi05gu97tQCRMKcM/8utj8RvVN3Sb5VEzailSVJUBE94JEabfA2sXT0eK7wxQmloKaaJdIX4FIsoIyjvF+SkOZz7EezrZpN+jbf5JLt/0KZ1gNKMfd4WsZdzS6G76KkcKlqapm2yUqIEpSATYT421Dw6+yHJSvDbSCyhG88L40D5aXTVCW5knsZkEnQGCD8edmHxSpKukVh128KJ1hz1/RLqVAkB6n6jbU2ts25n1S0sjrnItG/HjBtDiSiuy8Kt6oaVdSXX2wgiCoKbVCpnmi2xUYiZqbu5b91m0eCiCp4zXX8lCPFNKWW6RYTnjPIgkgkTHytGaqvUpSiNRO43+ulrJYz4fXZfCKcVdTUoCe8cigJ0Om3ibNNHA7CHcdcvipaKpUEu1KUkbiYyzyNs2PIxt5CB9VozRPbqoZcUlSQ4CZGuuomRA+duEVTqUBJMAazGv1r99poqeDWGkLZWzXqcZz/bldQoymI0hO+1jl18FcM3nUep0dK6+s94Ht3ECJ11IAInz1NtBlI6RnEHu9VTEjS7IHC6gw1D/eUVJJnURofDW2hWPgZHFjLrBJiB5/C1jD6NtGqR+6WkAwBmr1H3xNhmvRwu1CMpuakWZ1mtWY133tAQs2LgjcF/VVmq6gi9sLOqUJavladejlDUjUcvZte+gqr0fZS7TM0wBQgpK1K1BSDMR4/K1YOIeCLjw3f+E6amoWmSjE9M2+lKyQZbebO/8Anj32ny7ajiDUJo13bQ3KqgU0gJU++4FhvL7WVOhOg0tQkZlkIBV1sZDBdK9Q9i92qC0+opQDKSS6dJ8B52qH6WNS7cXpl8EL7co0B+sRQUrrgkpyprlZo/49Z8OtrW1h4mIW23SLw602lJWo9g+dJ10I1PhOtof464WTfeLsIYnxvdty195Xa7/2LWMsvNmnUHUKJKFHecp1nnYEjAxhJd6osERc6w1U8VDvZVL7eUQVbEAzz5+djV2sMuTVVORKAuAcolR9ws0X7g4pOFC0YvuRKzq4Dd4USdNRKhpY5S0PEhthVLW44u5aVtlttLF1IC23FbLEOQYO4Omp02tF1Q5p0YT6f5RTCLe8FQf0uaNNL6TGJ6pFC6lD6adztFJAzBVOiQJ5Ei0C8YqepN8XDUUbai0bmZc7yohSHHUTm6kCBH9PQDjbw/YViCorsRUNFet6di0V1y6BtLiwG+6CNdojfpau67ipK647tcfu6kqE9g4wv1hsFZyvOGAVCYlWn6254y3neD1Wm2jORpB3ChO+7wqLqw3WXnRFPbtMFTMapCyAAfGJnTf5Wr7UC/l1r15XvW0pUtwrdccZQtZHMlRg+HI8tNBa7XGPDLTfCu8ks0NPTKFOw8XmmwVNthaFKOgkd2R4a9Dan99XBU1VQxSuU7nq7RzEBv8AvFbgRrp0G2scxOthzW5SRuqVe0scAUxqhVBeFS67TXYopAzFZWQCBuY9x08LH7gaQ5QrQpglSASQATBBiSD4yfjZRvlygu8pYapQl52EIbgIKiecEd3UDUkHWOWiHh6sSHgl2YKlSE6kSQTruToPhz2toWsVn8k46ah9Zd7R4BDaJAR3gNNdQdJ18PjFllYCiVaZQkKG6ZBTAPUgQRoTPhsUxiscK0DtUtkgFYCoVqRBkba7CBMKEnax5ASUoZMpaIGQa5SeWWBvCTtoQANgLEFlE3Tj4bXDdN74urn78qkMMMMNraUl/IrOVqBHMnRMayII91orgxhhGjSKKmPrbBCQvMSG1I1J1JkySN94IE2pLf8AeS7vvOhCbwq6dLzayXKdIcdCTAgBZGhKZ3j2tySbG2sSVq1oScT4gdDeV5X2jTGTvR/ATpr/ADJjXXXTdw3tAcKi4bGXPVDdTCY3cVf268dUCqJ1ugYbW+whxLVOkjtFd4htvr4DfztCmOMJXjeDzmLeKOMriw+04fsmaoOuFtG+QNgCVQrwPnpEc3Vf1PiKixJdt3XTd1319Fh929LrrKWtvNb6qhp1swntap5pRyKXqUJ1GhBAmE7zosUXrXrqb9avCsL4LKH3lFX2hQVIhexkiY5ifO0n9shW3Y+Pbz0Vp+H8FoLTundxMw7w9eZvC9sMX9eVbXUykEKcok01O4galSUFSlxEwSBOmmtkfgXxE/0XcV8MY3KVFu6rzYdfWCqfVpKHwE7HM2tW4nQeMtKvw5iK52FGpYcbQYzpQrMOe5GnXn18bJLSwk6xbBqp21bs4aB9FBoLF79NOs1KE1VK8HG30hxpaVEhaSNCPAg/pZAuy7qoXleqgclO1XKV2ijlQApCHDJOg1WdzaKPQk4kq4k+jth52rqA9XYezXBWKzEmWMvZE+bC2Z6kG0t1uBncS3sqprL6qGqBBT/qzaiO+EiTJ2+HO3n/AGsp2y0dnm1iux7NzGOpJbzCGvDHWEMPhLT9W5Xvo0DNImRMfzK0jxE2j7ilxJ4pHB1ZfeE7qcw9RUICnHw3mcWhSgkd9WoEqB7oB0tMFxYNwnh8A0V1sl0a9o4nOqfM7W3i3933pctXdNdStP01S0W3GVpCkqBG0Hxt5zEKencMjL/Vde4OmNivOq+nr7xa469fN6Vt51rpzFbq1KJ18dfjPjbrDGCr2ui86e+0kNGiUFgCRCVaHnyBtL153BRXPeT1PTUiGktuFIKURsdPh0/SyVXU6hdlU0CEq9XcAEazlJ0/rbtoA3J4RosmRmV2qq56TOC/7H3tdtI26p5NZdhdbKuudyR7ifnatlXlNQsp2nTvZvna5/paU6Lxw5ga/SkrU7SusuKE8uzIBPLVRtTSubKXSIjSD5jT8Lddgz81OLrkMZYGVBsnNwgvL908UsKV+bKG73pUqIEylTgSoR5E29acA1HqSllIIKgM0bbj9fhbxvu2tdoLxpaxpZSumeQ6gjkUqBHzt694Yr23WmH6cqyuN5wJ3BAItm9pY87W+a2Oy0li8dLKWk34exCo0MJiYsBUP1q2PWgw6pkAEqIMfHazOq6q+XrtfcuHsXK5KCtpt0ZkLIkhJE8wPKfhZHvG+8cUF4uKpahqqo/9vV1BZaUhsLSp7sk54WjsfZJH+1BPsxbhe6i9gu54lgnfVXkEyQRIEExuPfYj+9MiwpB1Cs0A8/H9bRD/AGl4h1VO4jtmmiw6puoqHHqYN07YeMqcIkocCMkgiIUo5QQAUh7FPENbfb017XZ2IabU4XKilKQpWUqCCFbDK4BOsLmTGlttJYIJnCma8azK+4BlgOKA8ACev1rZj8UaYX7w+xPci0hXrt0VTKQT/GplUfMj6i2sP3lfVTRLqL3eLy1vqNO8S2S4wQkpV9kcn8REjkATFh36kFDjTpSUqBzK3O3nqLGhZw3gjkUCd/EjLeoXk5e6Yqg5/wB62hcxEmNfPWdbLnDm9E4d4h4Xvxz+7pbypKhU6d1LozfcbEsXXau6b0eu10ELon36VUjctuqFkrO4E07wJGQlIM7EGf8Aqt6hYSR26heUaxzX6FeodTVBCnElSd9YInrqOXONbXi4L3qb34W4arcwWRQJYJmZ7Ils68/Zt543HiD99Yeu2+P4a2iYqTrM50BXLnJ+Pwtd30Ub6/evChmmKyTdt41NL7iQ6P8A8Lbh6QFkxaV29a7iQtcpiW4CdvjYIFIVoPjbpYUFEczYMq33HytsNWM5Fb3bD1MrmUCRFm2oiDuZnQ2dSxmSUnYgzZsPo7J9SI1m2lSvuMqz6pvxLgZTuBE7T7rapyoUzQVqoJynxI0M/C2gSnROkfX4Wxmci0ad1Zj394ffbQYs526GS57/AB52zMVAmdOnO3BBSMusb72TcU3hX3Xhu9rzupppyspKF9+mS6CUKdQhSkpUAQYJA5ix2tubBQJtqlfVXI6HnrbCZ1B28bUmwN6aPE3FeMrhw5XUOG6SmvW8qWjfebo3iptDrqUKUnM7EjNz6a6WDxn6ZnFPDuJ72uW66W43aOlq3WqV6pul9h15lKyErUguSkkDY9DttbSGFz5suipe0IcuZXeQogga/dbvMTqCNrRd6PXE29OK3DWlxXfTdI3eHrVRS1ApkFLYUhfdgEmO4pM6nWfK0mBeuadrVJGGJxY7cKyx4kaHN2KGIBBBOn4WM4bJVTNJIMtEtxG2Ux+FiYUNhPMGbD4c7tXVsSDldDg/3kj8QbD5pynWA3HtK9yv1tlszDnr4xbLOorwsvFXaYiw4JVlcvqkBT4Z59/9Da1eH+GeNMUYavbF9z4eU9c1xNlyvrFvtNtsjLmiVqGdUa5UAq1SI1AtVQFbuMMMJkEfvmnJzbEAq2/S1hRfF4GiN2+vVKaFbgeXTJeIZWsCErKNirlJE+O9npAcpshVRFxdGG6tpKx2uciYGVUEH6ixiorbsLY7FnQiZWSZMb/H8bJBUDrmUYMHkYOlhmUodKc1QEQogSnSIGvQ/pa6QqjTyQodbOZKlDQSlMcz9ffbSVlRMwDEJ6+7x+utjNRQ3Y3SreYvVLjgUkBIGqgQTIPh+J8rECiMoBCu0GwIJ30sg66T25U9qDhfjG98BVXEq7Ll9ZuKgdW1W1DLzalNQBOdrNnSnvA5ogAnYWZjry1AlZlJOxMg8pH1ysZo6pxumVSevPsUz5HbJbXKVBJkZkgwqInXaycpyFrzIBEEnWJ059LM24vdLQrvMZKiSDMmDvHn9b2CQ4CvNlBKZkjcnkfrxti1Zs06lRGXu6H8Im2kZcojvCD9T0tI6pWI1U7YWdF63LS3oO92rSUrlMnMNFfnqbLdddtSm5quW1rIYUCk7ER9eVmTwOvBNQqtuN4pzpAqWcx1UNlgf8p+Np2XcD711vuCQosKUJgBRAPTf9LcPis/dXuYV3+Exd5ia8Lzc4xsPp9J2k9Z7y03hdRKYnL/AHR1+Xu8pt7CUgHZJHKLeS3pH0iaf0r2xCoVW3OpRUQYJDUxA1Ezp4n3+tdKMzYG+nysSF+eCNw6LOqWZKiQeaEUCRziJ05WLupBkEbWNGCIIsXdAPKPfazGqkiIPJ3PP52LLBEEifPSx14abb8osUcQCSP62uA2Cpu3WqFkPXiw2oCFKj5Wcn7qSowgRz3slYdYadvqmS8FLBKoiAZymLPpFIxoMhKyYiLU5qjgvstWitw9U3EXTMCdRrvYYXdklWcCBFnA9dyWnMpSZiSI2tyaFKjog2Ea031VwFh1Cb4u9PenIqeZBsa7JP2aQFdxMaRt7zZV9RVICU2i3HDmK6fGdBddy3BUVdHVOhFRUpLiUsiEzqCEpjU67xG9iQV0mfLG3MT9lQxBrOHmUiIQkKBUpKI11KPwmzc4lJaqLgcCVJWpLLswoGO74WjnE+FMcLSpV0IrHFKgR2io35DMTtY1hK5MS0VNe7GIbqqGW/UwGnXUnvq70jUkE7W6uaiAoDVF4Bt7vNcxRYm+SubTGIgX35Jy32xmpaZZbnOAdvCwD+FL4vegut+6KV5Y7TKtbamk5AVnvqLg7yUxqkanNZdqGE1F20ZWAqUIOomO5ZzYZShOHWApWgcUkHwm2A3JVgRuO2q36uLLHqmHiHhxiFxeeidWoBKgA27oOYOWQfvtrBWGcTXDe5Vel2OIpFMhIqCtJ7+cQDBm0qJS0AVhC1eMH8LcXqibtT2QKSl1tUbfxCd7dGzEnx03dgBlXNtwqJtQJwTdNdLyytQXUBWXUd//APOtwy8DW5jVNhtAkjONPnYk/RVJrHUU93Z0CcpzkSN9Zt0xRrbqCp27UJAjve1+lqBcStUBQJxre7XFLIU2kCkxRdbydBpmqW0b/wD1nztO+GHH03Ddz6GkkCmSglRG4/py6WhDjDTmpxZeVIgKzt1t01MRoIraUz4/xWmvC6S7cdHTrCoZcdSvWIKHFJiPMfK2XT1YlqXRX1H9lp1VPkp2v+iVKt91xoF8tMgHKftNz8LRP6QLbBurDFc2FOdjeRBcSdEkgaT4wfhaXLxS0WW0qfcSdVQk/kLRpx1b7DAFNVFC3BTXmyvMqZEyB7rPi1+6PI5IeG/6hqkCrQczb6l/3qQQYnkN9LdsKQYCnFKVoBCDH3WApq0qoKN9DpPa0zSwnswqJSNjY41WVK4IQqOoSRra1GbsBVaQWcQo64y3RmvVpxPsv0yEqCzOska/PS1asH4fFdd6qAuNU4ZvKuaUsyUpAcJkgRrCveSbXJxvdibzTSvrpVnIjLtJkEHW0A4PwA5TX7iK565ioOS9XahpaUz3FpQobGNYjXnbzzEq4U9fM08l2VDGJaeJM1zCtXelz1VCqkYdaVRuBTTmVKXUZCChQXpqNIPWLecDSnqYrWoO9jT/ANy0lKlCOUQPADx13Jt6/wBVgKnWFNrYqlkpnRAmPEzuJt5U4twjWYZ4g35gdmlqnHqC9ai722A0pTi1B0pQEpA1JkQAJMjfSdns3iUdW57QdRZZ+P0xjDHBQ/iq76h8evu9x5HflAgpGndOs6dfPobN26socCSlI27xBEe8edrFYp9F3j7U3Sa1zgziwobaLsLup1Dkc4bKc51MwBPnFq6M07rFQ7T1QdYdaWULbWjIpKhoQRyMyCI/TqQ9rj4SuaLHN94WT4u9aUNJ7sh0FYSVZpSrQAjcmDoSNvdB1Va1TsIeqVko0Q7vGpKYgQdgDzOh9yJd9W6+wGkLLpXlAQO8YgbRvOnjqNZ1srU+R2mV2ri9ICpSUqjmJIHh4HveBBDtdDG64p7sujE9Yy/U09RVpbbDQyrCCoiSZ6SVSdASZM7gvuk4dYFBaqV3cEIBSvItySUyDrpsdevSRANm1hGlpW1ApaDebMstApgJEHUGe7toeg5mLSM1RKr6dxLVQ4ytSCEpSJlcER3dCJJ1/ITUe7VW42i109KzE2JmmBclTibEVRRJRl9UevqrdpMsD/YrcKCNAfZjwEgFrXlU1tSlVMzTodfXKVlxCSkEADSdxPvAMgawVpVCxUZnF0q3FK75Qaggie9B1mfZBET3T71+5bhYu/8A199hDLmUFAbKiEpTvqdZjXcaEbmZotj8eiuF4DdVD2O8Fk0tPdD9a7U11e+X3VqRo00kZin3lSZB5DTYi0PY7wevDlSHUCW1mCMuUDoRrqCQrXwtYzEGIk/vOqbuWlF43s6kNgAgopmwdSSJynfnpGuoi0eYk4XX256xeN7XoXHX0BbrSUjKnSDB9+mm0baC16N1tFTkZfVS3+zJ4mKuLiVfvDKsf/1XFFB61SIUY/1ymlUD/Myp2T/gTb0raqjTKypJAIzDSBPP8LeJXAPFiuHHHjB2JV1Abbu2/adFQudOwU4G3dehbUu3tXXNPdokMtrXlJCoExEW57tRFnonn6futrs3IG1jQfP9koKvEx3TJJnfn1s3sQ3oQ0pKjoNQZ36/jYRp0rdNNIzA+yDKp8vzs3sZpfoshcQtPaoIlSdd/wBRbzKCIOcAvSiWsGii3FzQF5uupJBc75HIA8/hZoOvMZ3GVuIyuJI1Me77/efGzb9K67r6vjC12uXGxW1LzVaha26VtayUlpUkhPSB4CfG1YF4Oxo+sJXhm+1KCVJCjROkHXxB3/LS3dUFG2SEOc6y5DEa10MxY1l1MfH5lq8eCOHa1PfN3XkunJSSYQQsa66juga8xalt7oW3UqDhJUFKSSdzBi1wqu67wHowXndl4UjtPU3Re2cNuJUlYBcTuk7e2dAOdqj4iaCax4hMDMFeyREjYj3D4m23g/gzMHIrAxe73NkPMBI6SAqSSdot6ncIbyF48PMKV/a5lVFz0ZUSdz2KZ+dvNDCmBMWYzZvFzC9zLrxdbBqKvItCS23B70KIJ2O08rX89GK/EVvBHCri1KUpll1hRV/D2by0/wDhSkRvFmx5odCCORVjs68smcDzCn27a5lBKHKgMBSZQ9knKQDv4QT8rM++8B4Vqyum/eVTUsuOPLU0lKUtJLq21OJAKSSCltLcaAIJAG0HVXgrtFIGXdQ3BB8/nrZNfrXHElOeRMdJ3Ma7D9PC3HiOxuF2xm0subnuOhw8p5TNU+92zaWk9uUkJSHHFxIGveeWdZ0IHIyjHCNzUzjLtNU1iTTsllAzpMBTDLJJBSQZQy3vIke6x9yqQYSVbQM0gn4/CyXV4iuikKzVXhTgMhIWC4BBUYEjoZgeJsUMdyQXStCUqNTV03bS3bTKcW1SMJZQXInIJAkJAGmmwiwb1YtQJCQOsjvEzHWbM7EnEu6bkHq6E1dbUtPFhTbKJIWUBUiY0gk2Srg4iKvi/XLlqLvcpklbgacK8wUps94EEaEDX32kIHWzWQHVLb5bqnPHy60XZxNxMw2Dl/ertQDyyvpDoH/MfjaPQqaMI17jkjwzD/8ANFpw9LC6fU+IVTWhGtfQ0lWVCd0lbROvLuJ+ItBrWYtupGoACjr4x+Nu+on56djvILzyubkqXjzV3uCN7qvLhRhx8rzKbpjTqE/904tsD4JFrw+hXfHbXXiq5Fr1YqaarSJ/nQpJI/8AZpt58+i7V+vcMCwXCDQXi80BMQClC9+Q75tdj0Oa/wBU4h3rdpOlfdK1gciW3UEH3BSrcvM0RVjh5rqYncSiafIK4qyTpHhtzsEogEgHTxFu1HQBR8NrckSJO4EG18bLOcgzptsLIV8NZX88ABQmy6RH8PlpZNvhouMByYKDNrVM7K9Vp25mpECtRtvbQJQ+sH+LKr8PwtsEjWI0+VuXTleSqVd5J1+H621mFZDguys+f3e62nEIdbU0sBSHElKgeYPKwZJMkE/CwiTm0J1262O02N1A6rzUwPdbuFcb41u67aNmoxJcNHWt3K280HIqWKlKXFNoUIU6KdL6k7kESNQmyxflK/x0uG9eIH7pormvqirspdqL4cNPeCFJW6622at5QQtnVYQghORRTlBCQXPxXr8H4L9ILElJia4Kena9YReVHeVKy+qr7Z7s3ZkPJSIKnYKQNUpEzJsTxFxi4ZY07BzFar5vI0zRSy3U0IeDaiolRBW6dFdySEySCSTpbr2vc7LIByC51zGtu0lSv6Bl9B7COJsOqOYUdezVtidMrreX4S1a0yTrBJtTf0T8c3JUcXr2w9h+7qWkuuvuou05TRpYeccbLZIcykgxmcjwtcVKhHdMDlH9bYWJDLUE23WrRG8IA5IYHmBFh7nWpN+Fue69TgjzSqP+qxUHcETpuR0t3TrLd60D2aE9oppXhmSY+aRairaeBWQSA2rToB+dst0Qf5h/w2y0rKC8LaOf7X4WbSqc15tqkmYTlJk++0+BtfZgI72YyVCPO0PcNbmpsR8Y8EXDVkhmrvQ5+zVBIS2smDvy15ee1rGY1wZeGCq5FIuoRV0r5Utl1RyqIEaKE77Dx08rBpaiNruETqVOopZHs4oGgTWKVJM5NJPLUHlI+vnbpCVKJQUkgaSOY15/UW6R2qYJG41/xfOetty9yCSD3Z11Gv4eNtJZpaQu10lY0pptTYCn2wtAS4DAOxMHQ848+tt/u+rU4hkkZlQQUid9uWp187cBb0ZgogDUgcunl+PjbpT78hXaGABqTJj6/K0EuS6coWmnVBSyttGsoSSR+E/lZWoMLU79PU1rd+XZFGwip9WeccbdfSokFDfdIK08xI0IidrJQeeyZS7qDBE6R0+t5t3mcy5+0He00HXz8J++zXKkwDdF/VXVuHK0pMKn2dNbdtXe+tU9ivvK8pPhPn/Wwmd0H+8gDunXYdT8/iLGqdx4K7ripI1jSInXp9G0XPsEVrASnVwvoL6oMY3VeNNd7zqUVCUuhJAzNqGVXxBP32vrS8Onl0hCqcErTAhMR7rU14QXY/VXwy4/fLlOG3EqVlGigCDB/L6HpTSdmqmaUgCCgEfC3I4pQjFanIXWsF0lLVy4bTjh/EV40emnhR7DvpetqRTqYQ47crgVObOe6JI5bK/4bemtOn7MJ6DkOVqM/tK6mjR6UGFWmwA8KC7M8AAk+tOkeegA+HS16qdMNAxvZ4IhBEIfl09Ez5DMeI7c6rsjTQzqOdgHQrmNzYyQAnSxd3RUgz52txoEmyJOpkculiqgORmeVjjwJBg/CxZWnKR0iLWhsqTt0tYLol1F/wBMAkkJzKPlFpPZuSmYcD7aPtB7JKiQPjZgcOnUt3+G51caWn7j+FpSsPu0cri526mZXMGVpSebtKlZlLBJ5xbf7sT/ADfKx+2rIUEPRR4z+qTlXUM2YLIs279o3Ke8EoB0WJnw+hZ7WZ+Mkreq0MIcLZKU99O41sanpY4X5mpnSueLORFDD5khf/MbA3ywwq7FNVL6EBXdClGBJ0H32GZSkaheb/Ny+VkDiI8lrC7z6icrbrStAN8w62JWXdA8A8kSlIErT5o3WUNPR0bNK64gdkhKVGf8Nhbpve6Ke620KveiZCXCSFupGnvsycUYuSjDf7wIkuNsn2huUjb42g/+1rt4VaqTs1JbWVQQqduX3Wy8MjeHZyVcr5gW5Fbqgvu5bzfNPQXtRVTqBnUht1KiBMSYGmpFkS/+JXD+kZqKB/F9zN1dK6hLzKqtCVIUlwBQIMajWRvpas3Cbig7c2Lr2W/RNNs09K4yt3OZUe0bUInwGv4c6t8bcVJr+JuLqZKHMwvm8WlgBJzD1hcRrpztYkxBzah0FtAL3VdlLeISL0lVxV4fpa7Y49w6lDmYoJrW06DnqbJTnHfhC2ShXFLDSldG61C5iZjL5W8h70oO2qCUERIJ0MfDmYmJ/OythdbtE620llspdJylwDTSO6T9a2aTEntFwAnZTNcbFXdvPHN1439IfEgufECLzuP+zrVVTqaI7EOsOsLVGkk906/mLTvcuOsPXbelfhyovkN1jV93gyGyy4oQp9a0gkJyjRQ3NqEcHa5268WXqUENn+z95pMd4KHq6laEHXUD4/G+PDSuFSziq8W1gqexC66DodFsskffbJoXSOry9thcFadU1vdAHa2Wrx48cOqSodbVfF7PFtICBTXHVuSfAhuOdo19IHjRhS9uF1Vdd33fipdY7UUz6FPXHUU7YTnT7S1pCRII3PO0+BbQcDvaLDmqvaB1syvSEu5F48DcTLb7RTrbTb4UqVDMh9CtZMAaeQt0c0b5InNcdLdFjxSCOQOaiOHeIt4MYdutit4XY6VUUtCyy8TdraE50oAJlTg5iy9SY+vSpQlTPC7FEFYB7RVG3kB0ky/pZ0sKdqKSnUFEoUyhXdJ1lIOuhsKGCtktFgqSSJCl7+6LRax4AAck6RpNyE78EIF53A3W3ldDtG84pYVT1Km1rbAMQShSkmQJ0POxS57suR3Gd+/6o0XXAw97MT9mEz4+zYvS38q7WPVpykoCgmR3dI/C0f4axg7RcTK9TrxKKi72gEk6ZkqXr12Atw+L4pTRVrYZGg5TdxIGv8utakop3xPkYbAjT9/7KblXFdJkmjb11Olq1VnAnAtwek7fnFZFxMuXrX09EaVSzKadwtOIdebSRotYbSkkawDEZlTNasdojQjpvaO+KOMW6FNPialby1LaFUxezJypBIUnMD7R0WEgc1e4zjxrDpZWx0zcpPMC1/JHoqGpZJeYXH81RPF9+12IcMX7T4NBqb8oO2p2WWnG0qTUR9mkl0pROqdyE672rB6TfoH3xxa4S1fEWg4WJu/izRULVVUuUlZToF7LRBeQ622tTa3CnNlUIUVBIKiNLWTu2+mMRXawzclTVqr1OGrZKHWwipZXBWwhSicqgqVpkEb6idJZurEdcu5KVF40LgrOySHEOuo1UBqQQTPLlbooXxRXcT/OibEGyEBoHlY/uvnBpVusVZaqEdmtpeVSH0/aJgxGXeQJERzMxNluirHXstE2mFKXBQlsJgkbzy3jXkRpOzn9KG/KW9/SQ4kXrTerIaqcT168tMhSWsweMmFEyoqCieWYqIgaBpYZZcrq1LbKApLac5BB1IGgE9Qk69NdptsB92By57JZ+VSNhu72WWUCqqEBoASlC/4oJUdCRqJ6ezI5w+aaqoqWlXVKUlDQQlkpJgulQBGg31JjSN/ez7mp1LyjtF5MwS6lRSEpRsN+U6g9DoNdT6a5V44gTdbCVrYokdstG/fBImY2Gux5DqZq3VkaJ60RqEIaebIbU0hpOQAJDZA7wmNgSfAkaaRY3iC8W0Xe4q8L2NDRDWurguFBKQO63J5qOmg0g6zIbLTl7NNLF2XJVXgtxa0MttMyAoFW6v4B8EiSQoEgWaF5Grvy/vVb+CMS3q2rPS4du5al01MJ9upcSCDB/gSSesa2ZrFJzwnBQXneeImVuYFo0YYw+2vK9fVa3nqahRkANiJBIPtEaZtxtZJxkL7ujDt40GErqQrtEOF28at1NQ/Ut6lSmiNBok+UGNdbO5N14+o0NM3veV9uZEBKLvuW7mkU7ImQgrdCW4BjmeZI11HdpXFJdvC9MP0tDQKKUupafVVuyBqXEspU2mYiSAJG82TrstZJnjuCqdds72/rGcl3Nnzc80zNvoF9GrE124qwzh7El8sMPJvzDlFeJ7RIWA6tlpaonbVareCuOruo7sxTX0920L1JRF0rp23TJCD0I3EzG/nb2B9FS/3mvRu4d3o24uWrgZYzeCMzceXd+QtVxYnujnsFyBdSoI884iJtc2VxqLDfDe7b1dvunuunTVOkklSipCSd8qCcqfcBaLPShq7lXh+53LuYp0OCrcQpTbYBCS2dJHkLN1eO3yDDx32n6+hZj8TcSO3xczTTjshmpC5JkbEfXu6284irp5rQuY0A9Au3iwcwSCd0hJHUqOLweDtO2tw7oIMiDueVkJ9BSolSArLoPET+Bsoh0u0pQQqW3VxI2mCJ95P1pYg+2VLStWqdoT49PrrbYiuBZHlN9UwMbUSarBPEW6gkSKduuAg6kIzT/wAlqE4jbUlxRJV3kpWJA6662v5i2+7iuWtv27r/AK/1YX7h91llJbWtTq05kFICQdgofRi1Br/RmJABzBGp111Hu6/K3QYOTc3XMY1Y5bKTvRKoLwvviJVYZu+vRRuXnQFGdwdxUOtkpV4QT9+1p79EaocHDy8rkWYVdN+1NLBABywhX3k6edqbYDxcrBOIBfIo/WkFhxhbXaZCQsRIVBggwoHqLWk9DDEH71XjlKW3GkuV7NehrNmy9r2siefsJExrFrmKxXgc76Krg8uWoa36qyt6VxobvfvAI7QMNqcKQTJhOYjnp+u/OKXOJd9OtuuKu5NI8232hBM7OJnfcFtYM7zNpWvSnNTdLjaQIfZUiPAgjTwPnZm4IuHB9+UqU30lvt3rpqiFB7sz2zdOS2TtOrZGXmRzNuQLmxNL3C9l0WJVZpI+JyCiitrcUX1dr1det+VgYYWioyJGQBC3uzUiQP4QJ/rZPvynXSVF6pbecW6aippnVrWZUlCEutz0Pd6fKz4xoxddC9fdz4feD121NO460M8hDiUhaoVzGk/U2Z98OGpvBxQImsdoX0idFdswW1QB8NuVr8JD2gqlHPxWB/VEm8Sl28HL+qAVBupoqpzTUgNlCwI0mT/WzjuasZXjynDTSm1OVq6pHad0hLrCTlMayT9GbNld1N3bc9JebLi1mtpFVOUgHIptbRKfHUKg2cD14C/LyexDc9PVlCU01QkFogqUhZQoaTOhB8uXKzyDTRGYTfVMT0xKMKvK461tvL6xRVVOpQBglpxLg/8AGr42rCx7SkxMoVa3vpZU9PV4auKtCwt1mvWyRm72R1pU6c9UJ18LVEpShNUguju5oPltbpMJdelaFzmMNy1RKsn6IV4lV34nukqJKHaaobRJ/iC0qP8AyptdT0ar2N28ZriClZUVSX6RwEiTmZVlHxSn428/vRUvVdDji8rvERW3avQjdSFoI18s1rfcL8UqpMZYWxEmnfpUNXpS1DjT7fZuNo7ZIUlQkxpM6++LY2KMLKvN9FtYY8Pow36r0tMjTTfabcBY2j3W7WkSRYMyZ10PysduyrOC0TCtJ1PhYCob7VpaOosKfvMGbcq10jcdbGjNihOF01lpyLj4zYs+uFsqJH95G/UH9LKN4tdnUGdcxneIsmV8ppysT3CFbHSDP4W2YzcArHlGUoYKiJM687Zmk/kbB5wdj4bW2FgiQDvqbHCCmhizg5wwxzexvzFuDaG8rwS2lnt3s2YoTsNCBpP1pZPb9Hrgg0YHDO4p379Pm+8mz/Jnmeu3O3WYEyCD0FjCaQCwcbKBjYTchNfDvCvhvhK8G73w1gi57srWkqSh+npkocSkiFAEa6jSzuziQTuTYFMyQDttboK5GY/CycXP1cUgwN2Q2aFQRqTuPrW3T6+xpjUwf9XUh4HwSoE/IGwSTr15AED8rDsNCqbdpiow4hSYHORFlyST2EKAISNddxbLItDfVKaKnLriCvskZiTBmBMi2WV1FeP3o+U5rfSMwIghRSmorHEk6exSPHTx0taf0gm+zVdcqUkqW7qkxO3S1VvR0ve67h9InCV7X0+lmnaF4ZlDUgKpVpCo6SqSbWo9Ie9aSrq7laoFesUzrLzwqmyC1qQIkHQwJ8iPK2Ixru/sNtLLZDmjD3i+t/8ACh9IX7KBBBkQN56/CbYEjMQolREA6fX38xbk+0CNUwTrt+mltLcWlTaEtjKYKl5hpA2321j426XMubcCdl2kFRJJUQCImYEa/f8AW8uzAnDDG/EapepMHXI5Xu0wSp1KXENlCTOpzqE7HbrZqNZipJJM5ZJI5gHTl0+tbTd6NvGC5eFWIKt+/qGofo6+mU248wpOdtQ9nuKgKBk89PGbVauaSOPNELn+XVeXOwXaE38V+jxxUwTci8QYkwuqiu9jV1xb7RjXbKlZOp+4b2jpTSm5JM6wdYMGfr+lrVcffSdwhxGwa5hu4aGuQ8taVhbyk5dCJkAnqfhaqzzrmVAbLamypWckwTtET7+fusGlqHygl4t0UoHOkLtNOS5SFkglSeUEjc/h9eNjdIUthJMAzIEjX8T52IHvJJkkRp9c9PvNt9u6xXU7Xao7BSV9oT7QgCPASJ38LWHG6vRgjdSPh2/F3U2FNqGZSTEbyIA+voej2E8Ts1OGrrqFrkuUbKyZ6oBt5QNYlpkP9mVkIKAoKC9zMbe75i19MKX5e6LhuarReTIu5FzNIcpewlxb2VGVwOZtAAFDLGs725DH5pKLLLEbHVdFh1MMQaYzyVF/2it9euemRdqUOEIapLmRy3DqlDfzOvU+63o9SkltMjcW8o/TfvFd4elqw+pKVDPdCM6lRslBj5/G3q5Sf3SDMyLFpZDNTxyO3IuhSx8GV0XQ2Q2w06WLuA6zNjPKEjaxdyZ9m12NAk2RR3TcTrr42Jr31A8NfxsddAiSqDYksiTvta23ZU3JawY/2GJKI5oCllOvOUkWl31jxtC2HXezvyi1g9smdfr6NpUD6dy58LU6ioMD7DmiRx8QXSqalI0zC2jUp/mFod4nUnGKuvZj/R/f130VDl+17UALnnMpM+H3WZ7GEvSVfWDU8S7uQJByoTBIkaSEdJ+Nsp2MyNcQGo/dNLqyJqQP4vnYnXO08turbbUpSwmSkEx52gqgwDx4NWp68OKbC2IP2Scw321y8rJ94cMOLTRbdreMlQ6jMEdmAsDMrTNoRsT9bgT8Ync33LJ20tzaysGp+kCNGWx/ui1esb3zUVN43pQu1jpaTUOBLRUcoAUY02sewvw8xtc1+sXve3E6trmGgUqpghWVWo17yjvEGQdzpZj40XWVN9XyKOhqHFduuDkVBM7gjlE2nSV76guEumnVWG0xis63NJGM8d3XdmGGbuvEuNBLTQUsxlToNTroNN7RZX3t6k8aynQVBIKgoRCkkcldI8Om9lnHmEl4muenpai6Hq1qqaTTVSFtqIXPdIOm28zZiX/g/GFEf3BdKaBmmQ203ShVSWWqdpDYRkInMDmIAM6xynS1R4jG0lhOqsS4aZrOCRWeIzVff980t1OJhmkfFWAUqOdCM0KAJymNYOugtE/F+voU8XsTUorKf1movSufQyVjP2ZeUVKjoPrY2dTmBcTNC9ayguBhip7Baq9+nvN1IcX2fe0CeaREK8OmpXHfCjGlTebNY5hJNTXN1Sah+obr6p5VQg6qTnU0BCiUkmd0iwH1Eb5y4m5VxtCWRBpNgowva96O7KZuoqKpLLfaNoJGuZSjAAjXf6HJfuPDF8OtuXulS0UqT2pKlFZypBKsqUyTok/E6WdNBwlxzfDjd3XZwifqKhDgdW6ahXsJBJnMkBIBiNOnnaTcK8D7+wjR3fjC8JpHqNDjjt2B5cBOZDYDpMZ9V5oAH5wmnDv049XdFXdTxxt4j3CybnDbD1et2nxGwAq77xuuqLKnEFDvZu0zgBKTqJkCCJtdXgOt1zB941TicwqaqmqTlOvfpGPDwtV66Kq9K69mVKcR6utS0qASJgpMDcmPrrazPoy1J/sS4AqAaK7HU5o1JpUjSP8AL+vQ2EFxnBeLbqpVEOgIBUpd1CR2jZEiRLh69ALN3jBUso4K4yLKkhtm6qh5ztZAyhMkydtAfhZx0bb9W8lIp3d9ZO3ysVxzcYvrhrii7aqnU2mruypSFJElEIMESNwdY2t1jj4T9FjN94XRPDb7l5YWuKspCpSaq66V3OlWhlsHcedlH93VrkypWvUkwfhZD4Xv0qcBYbz3klambqp21lSgCopQEydRExZ4IvO7SNK+nHPV5P3Taq2aMAXcEZ0br6BNLiJet8XPdrCsO3WzeN6mnSG6JyqDAcQlxCVrzkGMqVFW28DnaKa/EqKbF7TgPeQ0UKUDGmkDz1+dpVx6blvBNBfNLU0tY/RF+mbcacS4UBYSVpBB0koTPlaA8ZLCL0F4NABam1d5StQruAQBJjQyfC3lmPRsdizxvf8AwF22G3FCw9P8p3X5xOFz3O9ejjNQ+Gl99NOM69xsPeNN9LMy8uM9fetK/QU2EMRr7oIdFLKAdwcwmCIB948rIVRej5lSX8iErSciUkJEjaRrEjSIn4WI196vrYRDiUpUowhBgTpqdNSQBqZPwtnQ4a9sgkYbWVqWWQ+6bBP/AIVt4Hp8dsv3piJdy1NSs1BQ6RTMFxuFSFPDupWNwkpMjSRNrQU173RfDLFXdN8UlbTLK0h2lcDqCUyCMyZToQQddDahSHM9Q0673ZUMxUdIOwMnvbxETrazvo15XeG1OQSsUlbVNKTJAGYpXtsdFbW7ihmMjTG8LGr25iJb+S8P+MlSLz4vY2vFht4JqcR3m82lxICgFVLh7wGx1E9OtjfD9hosu16mo74SnUzGkQYjkd4k9NbC8W8L3oji7jOiapHVJaxHebUlIBEVbggHbX8bLWCqIXVRIS+UtFtQUCW90iJ170qjp0IjQgdTmGQALmADxCU5nKtu5qA1i1oKhK0hW4TuDrHXz1HhZLuCuF33YuuqWWy/WLcWolBQGxMgaRpAMT1POLI1/Xk9eTzVI04QiW1QYIMqiZ3/AJZnXqOvOIKpmkw+HJcSW2cspaAQTExmT4wec5pOthjdFJsjVxXXiHGFGq9FV1TS0Lsslqgbc7SobKl7qBykScvQDcjUWfOGsM44p2zdFwleGKN9QU52PedcIhOqUEob0P8AEpapPeVBtF3CK7qq91paN7Xsw00FJCKGrU2reYhOsakmNYCjytNN1YNuWqWad++Knt2SA4t++lFYVr1WFEzpGXqOoFgNQb3F067l4dXPSpZrL8qK283kAK7WtqFOpTGoGXNlT10EactBY7WjD1K+t4XK1U1a++z2DailsFOhzT4H/hjlYK7cBNNJC6LF1/IS6hRJavNahJJmBJEQUz4gzsAVZ24q+haQzTX/AFqnXXgS442hSkplXdlKR4GdfCdZUjfDspROs7UqsvpKYcxLT1d04kvRRXRvoXTN6x2apKwIJnUGOnct6N+jJTu0Pop4IQolJFwh+SJgFxbg+R+7laPuCeE8HYprsQ8PMZXK3iG7r/u9Dj6bxlwywsmUKJzNkdoohSSCCmZtZJjD90Ydwi3hfD9Cmhu27qD1KkpkEkNNJbISkE6mBzMzuddbUZf1IDGehVuL9OobIOoUcV5FdVUNa5WVDaqF1biUsvqS28S2pGV1I0WkSSEnZQB6WJ3zVLdplo2SCCNJjXx99gy8pRgDJOwUfl4bn5eFilYttynWrPIKQqCdQN/cfC3BMjs5eiufmam5cN3U92NV1LRh1XrFWurczuqclbk5ozGUiRokQByG9t3rdlFeKGU3gylxLL6KlsKUTldQQUKBG8HXXSw7S/tnQ3ErbSYAjYwJ5zqfrYvWr7QJypSYMjMdI3+vO15tyVTdYCyhL0kqd4XnhapabcUpw1LJCEZiYSDA66n4pFqkYnpFNVFQFtLQ4HlpWlehmOc68t/O3oNWZmMU4XrArKyiscZWSsZYWggEmY3199qZekLdyrp4l4qomwMiL0deBREK7TvSDrp350+Ok22cMltJw1zmLQWbxfNQwQQTr4Wsj6EN4djje/7tj+/utL2WNy26kR5982reqQpQGmpEWmn0RLyNDxmoqQEj94UVTTwEzJydoB/93barm5qZ48ljYe7JVMPmr7gzTJlQUFSI36Tr01tAV53NetNeFdRqq6amVSlLdGpx5I7YKccURG40Vt4Wn1umfUyUIacgHUpQTP1HX8rQ5ijBtU5eOJquku6tXeDDjVTTZW1ysBKHCECIVrmGnXe3HRWJ1XaVbBI0JpNV1x0d7Fb9/qqUIu9a1stNnvqLMKIJ5RqNvvsQbvzDz1Zh5dVdNcs0zCEJfdUlJCUuZUqWmNQDHhrrrbt7B1+rpKWMPXgHHqKkUVCkc0UUuNuJJywNCkkfG3N8XXiG8rquxScNXkzUN0gu97tadSQlaFtrzmYgGFAWttDQswNLdEPR33fd5oFNdd30TNNS3a/UBPZ51NkhaSEzP8Y6bGPCxOuvi8r6eZoWK55lirco1oDaoSnM2oECDp309eZspXVcWIbrxAhtCaVqmdeqKV9x99tKewU6F5gCoanOY+ob13XDfFA8k1C6EhpTCm0qr2IGVwnKZXvClHymzjLdTOYJv8QbscvDCd9XpeTT6bwu9ugrGw4TJBUlKiR1gn5+6ulYkMVzyUH2HVZSPA2tnxRauW9275ra+8qWlbFzvtt/620S4+kKyCAomDIjx6a2qhehBrFL0OcJVIMzoJ+c23cKdeMhYOKttICn7wJrU3fxbuwAqDb6n6cwRspteXz1g+61wUOuN/aBWVxEKkHUKGs7bjTx362ozg6vF044uC880JZrKVxRB5BSZ/G14iF5oiJ9lRI7wnr9fK1LGG2ka7yV/BX3ic3zXqDcF6ovq4LsvlBBRX0bFUkg7hxsK/Gxs7TGvKzD4DXsm+ODWEqsKBLd2opSQZ1ZJZ/93Z9FZid9dLQjN2hPJ7xWirkB+ttEpP8AEPIna2iZ1OnMGLYFCd9POxmoZSXe7RypcQNt/KyHUI7ZlTYGqgRt1s669pCqRUAT1FmusBILZMC2nTO8Nlm1Tdboow52tO2sgkrQJA1A0sMFAHVRsUpcyWygiezcWmOneMfhYcrAM5pOutrrdlSQoMjOTynW3Sl9IB6czYHNpJk/hbclIAkmBB+FnCS7B05TMwPhbpKh1FggqNQQB523PdM6DxM2I1MUYC5ESNNxY5d6z2gOokjlZNCifOx275L0pAI8TytJRRGrumtNU8WsmTtFZdtp0tlnWA0RJCjPOBbLNZJeKXDlSneKVzZs8JYrTqoCT2UT56/P32nK8alxRSlagQ37ObWNNuZ+vC0DcNcx4l3a+3KUilrIJGvsAQefX4fGbKxxa1ZlQM2kCdz49JJNo0oFlCdxCUbocbW/2D6Z7p15HedRvZeRTXWACppaxMQTJH1rZsXOHF1YQ0palAH2Rsn4672cKaWqKc2Rck90DWDp9fCzzuIcpU4BajDbF0iUpZQVQCN9Dv05eFszXXJUmmSBm012259NB8rFvVHSCANANSUyIHS2IoahSlZkEGTrEz89duVg5r7oxaOiMofoE94sCEkyZB26/UWDF5Ui2VBqnbQAZCQkRIjadOY91nPw9qcEXNelS9xBwnVX7d66VaGWKarVTqQ+fZVKSJGhB8510FmTTXah12pqKBisQhyoUpKHnw4ltkgZEDuzI70kmCI0EahbN+oW2+6TmWFwuqq9EtIU4G0IA0OgTpt+P6dYy4p4gdpzS9nWqpkPFTyHmnRmWpP8I0IiCNbSBfNCG7vc7ZKXEqRGVUqkbTHgT87RDjanaq7qVQ366oC7XnTTrGi0oDZKZ3OsHTnHhawZARYKGU2vZIlPi+svFppP+sLbbyOLcRGfss0cup8tjyNvTTB9/qewJdh76YuqmkH2ky0nffWD87eSeDayrXiO76Tt2mmmkn2gopeCQVJmJ0M/Lnz9OcJ1dS7hC7HA9lDlCx2qAmQfsk6AgCNfwty3aOzomA9Sun7NN/Uf9lSr0s60VHpMtOFXdVUXUIIn+FAPloPn5W9faNQDSAYPd59beO/pSOJHpCtuBSVRU3aIIlMZUa+MGLewbDhS0kJj2Qd+djU1m00Y8lUqjeqk+qPKUACSIsVUoKnvflbl2oUUkcuosVU64hJSlX6fUWssdqqj23QrokRPkBYi4VHz3t07WLSIUJI8N7AdoHBIBHWetrrDdUpBYo1dbmS9KRcah5G3mLSGKg8zaNKNeSupiFbPJOnnZUxNibGF2XimluDBjl5sqQFesB9KU5uYiZHvtlYi1zpGhvRaNDlEZJ6p31NR34nlbhl+VgZjaL7yxjxgCwKbhk2QU7qrET8M02BoMWcaXatpL/D1inaPtrcXlyn3qja2MaSYuzC3qtATRhtv7KW1vKSgjMdrJd5N1lRRGoaaUtDLralFMaALHKyRdF642fPY33h2iaaXr27dUJQOgSCcx+Fur8eWLvKEOEfbM6BW/wBom1acljsrlZgYXatR1L/aNgg9fvscuOhadSkqCSV5j46k2ZmIr3vq4LqevS7bpfvt5TjbbdE0Uo7KTClExmO45H3CbR1ScZuOd3VlWbo4G195sFYFPnrm2UpjQgZgCAT1J/C2fVtdkDgRa/UBGlBylrd1M9JhiidoqlKkJOSvqIPT7ZR/G1R6UJxyi+LwvwtJdTeFRQLQ0S2laW1pUhQ0VBnnaSLw4t+kPTUNS9h7hNSvu1L63KhirvFseqOk95sd9OYDQyOtmpwSwTjaiu2+Knifhpq7a9293Kmnp2KhDiFNuJSVElCl/wAQiCfjY+AU0T3yx3HEdbLqDte/4WHiz62FjXsJyjf+ybV716cM3XV0VPhxTlDWp7OoccUteQZcpgpSOXnZXwlg5ji5iW7rxxfdXq13hjsU+rvOtrWy2hRCozkgk8yNYtN9Lhi4b/q6S7r4uplyncWESUKGWTyOX6m0l3LgbC911blZd90MNONMt06kjNqEgK16+2belwQUdHTENjHFIPiXGvfiFVKOJMTGOSjvAvDC7LrpK5nBd1tXeisoEJbfqAtQP2hUoKUVHMYTHvsncVrsu5d5XDTVWHmKn1uqWXA62SFpU2yYGWZjoAdhadaFxhhqqSgobQ1XKQEto0CTlgADzsiXqhjtcN1ZS2VLUCFFrMrZA8+VqmGwtpJONKMzzzVqoa+WPhtcQFRu8KamuvFQu+gZaZY7dCuzSmISoAyOca7+NpF4EMUVVcVMa+9Kllmjw/RukMVq2QopcW1Kwg6gBOlov47XXfFNx4qnbuvtFIyxVsDsywnuNKS2VISqJAJBkeI6WlH0eGkm5qpxx1BULoeZzhOwavB1IHun5m1OZpFTdwtcrepzelsw3NlLdAnCa3pXeFWWzquK6pMjp7VsqsJ4Jvmgfu+gfrnELS6G2lV1VmUSgka9pBGmx0+JsiuM1AZTVrvdwoUY+ybI98AiBZy4R9TecDZvd5a+8hbTiT35SQIO552stc29i5voqnDqr3ymyZ/B7DGHqrBV33kq67uVVPNFFQ4872alFDigAdCDGo8Pufirmu6jpgtu7bndCEglAW2VHrr2ZJ5+O1mR6PwuHA3C6pbrqv1G7rqrK5bz9WvRH26lElR5SvTziyhx04iJuLAVNRYRq01V84yWm7rmNMrMVh2Ap5JHIBQg9VJsMRUkcHFkaCbeV1pRR1NVMI2E2JtfkEqXnc12oubtqO66WjWmpTHZJyyhQVI9lPOOXS0EcSmgy9TOSnIEqT2Z3Ptk68hoOn5Ta3hSswDwiu3Dtbez94Vl3UjIqap90uKcd7QZ0pnZAKiEjkkC0E8TH1OUaXjU5yXwUIygIGm3iZUfjbhu0UAjxKNzW5QWjRdLhjv+zezNms46pq1Abo6ZxReTAAcgrByJ13PT39LM+pxzhtKJReSld/8AkV3oHKQJ2NnQZU1Udq23k7spSJ128REffZvvXbdjIIRdtICSYKW0DWZ1kaj68LEpwAPEpSlxHhKkP0Z8aYGvPiarDeIjdtRQ31d7rSG7yZQWVvIKVoA7TQEgLgbmfK1tb3r8CcOblcdTW3Hhu6kK7VRSqnpGCoiMxUcqZgASSNhrbzsxM3R/uCuUzTtpCWs6YQBoFAx87QHiFls11KU0zchG4CR/EfEa6D4626XD8r4i0D7rn63M2UOJRHiS7RXzxExVe9LUest1d919Sl4rDhdS5ULWFlR3BzAyJ331tH2IA5R1ZpojOgGFwAQrMIJ1/lTHlZ+VFGpL3rDaCVAAajSRHyMRPgJsxMWKDeJFBKsy+waGgOgCBpGpI3119nkINtVrcoWc83KQ3hmrUhJAynRKZSDqCZBiOo6wOYAEmcEsM3Vj7jThHCl73QKyjcql1VbS1DYW06wyw46ttwCNFqSBtOuh1i0bOrX2wSXCpuMqe8SARodz3tQABuJMCZixPoG3Eu9uK1/YpcSSxclxinQQggJeqXkhJKjucjLo1AjYxFpDqhkqNOIfB9PAvi1WYXfp3H7hrgKy6aioTnTUUilSUHkpxCvs1bDuhRhKxMnYduvDdattHqt2LdVlQGQhHd8Ak7QD117u0Ws16RXCFzi5w+Xdt104cxFdLiq65CBlUt2BnpwojZ1ICde7nS2SSE2qbgLEVzVN20iL/KaepQEsIqHWygKAMJzEpEK12keybGifcoZGllI1BcFxUziBTXM0C0kEFSYSJA56k6jU7fCy5UNrQ0W6d0hIRIZUSYUNU6HTdI8N4sWo23Ujtm6gLQBIUkhUDKDA3nn120O9jVTUFhptyoqEBBIPeBOYyIAE9AOfwItYIuLJgbFGuHV/jD/EK5q5Cc3a1CaRSQJlD0NkSOYnN45d4tbtaJQUFO+ijprrHvO3hHzpVdwqKK9LvqlNKaFPVtrU84AhKQlwDSTzhI/C10pSrK4FTm1HMRH1tbNtqQrpOxUHvH1dbrZckpKkEHwO23hP42LuqzIUqICh7WojWx+/Gyi+bxaSAmKpzKZ27x26/lZJzAwcxCBokAkgc9umv4W4tzcryF38bs0Yd5JN7TI4QnMpGQjKdY035eO/3WAcCVNpIIABy5QfP6mwzhJeCciSO8IVqeev18drArCt15RrIA2+o8t7GZsgP3TZxhRIXc7LroMNVjS8wAB9qJ84J+t6n+krdzlHxCvRJVm7SnpnEZtCQltICtNDOVQ5bWuDitvPhyuAcEpbzgHXVJB1+Hjas/pYUA/tPQVoZJ9fuhKSQNMyVLI194+XS16gdaoBWVijc0BKrA8haXFhQykEgjpZ0cKn65jiJh/93VRp6l6uap23c5Rl7Q5NxqBCrNmqSUOEqEBRnT68LHsMXiu58R3Ve6NDR1rD6f8AcWFfhbqZBmjI8lycRyyA+avkcB8Q6snt8UU5CY0NQ4SSY3gfUWZuL8PX9c9Q5R3liZCFt0prEkqUO2E+wk81d0j3/GxNO+XEqaKWgEIUUrTOZwGDKuUg6CI5WinjI1R09XdlWu6KetUAsuFaiAlpMZo1EnvE8+fW3ER3z2XeSxNEeYKJ7nuK8r8vdm6Hr3dZcdbVJgqAytpVMTrmC/dHjYhTXPUP3iu7S7XOPs1RpnsrZKYlYSoKE6d0ctJHSzyp72vpgNXwtFDTtVDz9Aw602C4lSW1JTrvoUiNdbI9cnENQzVXjUX0+Qq7kV6G0EpEFYChAI1GuvOfO15oWcWhJDWFMTVdGPVqGrLy+zhLikpGUpJVuNO8I6ajxs4qXhHeDqUqqahxjvupV9olRDYSOzMZdTPtD87A1dEq4MS0dQ1XVDgbbo3UrWuc7anChcj3p15R7zNYbaW2IJgpmeu33a/GwZXFmyNDC2S91Cj/AAHZvWgTS3teS0rWpZcSyCAQYygTz05zyEaWqdfVO7T1CW3QQpsFozO6FFJ38vlb0cW20qcrpnmSnbnprpr91qFcX7pTc+Ob+ogILV61KgIjuLVnRp5K+uerg0znFzCsnGqdsbGuamcVlAp3UqBKR8CFE/ja91y3gm9LloL2GiKykaqCJ5KQlX4/1tQ/NNMkfyrPPqB+Vrn8IKxV5cNMP1K8qoouwOo/2alNj5JHxsfGGjhtd5oGCu8bmeS9DfRAvb94cIvUSpM3XelTTAA7JVld+9xXwtNizm1J6aC1YvQkvQKu/FdylWjb9LVgbRnStKjr/kTazZOoAn3b2oQG7Ar1QMrytEzpl+Vucxnn4k2w842+RtpUA6g7xa0Aqy2tRUjIdNLNytbLTziQefvNnCCOfLl0skXw2EL7QAz4Wt07rFVagXCbyDkqahE/xBfuIj/pNhe9src62De7l4FRgZ2jpt7J/wDzrCwQTvpOvW2iHLOy3WwVEHlB87YSCdNuttgADMIjz2tzIgGBtsTad1Aiy2CeRjwt1JKpBO+45W51gb9NLbEAkDlpvNpApIT+HnET9fXjY9duj0iRBkEediAgDNqNNTY5dqwXhJ36WnmTWS7mV/Mfj+tst3I5pPwtlnumsF4lcM3GnOI1AEADLSvwQNRKIIPXb63tM1aoq3A0VEjbNy/HS0BYbvNygxbTViS6FBtQBb3BJM7bDTUe60x/vt112nKlJyrTkUlZglXKD8eth0rxsh1DDZOfDlQxT1K3luQ2hKlFJ1JJgQP6bWcn9pLuTKU1QIkQAkjl9R5WZLVSHEKKHZSlI7wMneeX18LJTeMcNVN7G4W79o11qF5Q12kd4bpBmJB5WsSRNebkoUUjoxYKSP7RUQ0TUKhMykJJnpy+pFsOJbuQmEuvTvAT9x56j7rNBSiVZwuQDIG8Drpy1+psVfuulcS46C/nkkZHVAnw3sMwNGimKh19Urf6TLsDjyGk1LnZqUpUDKpIHICZMnN8+thrqxfdbLKnXlLQH5W2Csax3dNfC0SVdO/dle8s9tSvqqELJUSpKUKVBJB3EqEgHn8ClbVBq8Cp99aA3TqWWUA5RrmciNSIEg6e0Olq/CaDqrHFcVI9+cSsP1lSm53rurCqsR2aXM6YCjtPT2YHuHMWhXFN+Xo/iSouxlgU7iGFtLJIlYAGWY9rSfOT42K1d7NPYhReCnUClbzLKe1OZOwAk6pMgR4WPY0Zum/slZdVak1rTRcqsylZy2ACAnrHlyEkRYZYBqFIPJ0KalKw/QVNPedA0X00rAWVIQYCh3TMbDfUyBMyOfpdgqto3ME3MglfaKuumME6BRaSTz21+63nx6P9TUNcW7ho3nHTS1NS5TOtrkocQtKklJTEFJEA78t+d9cHMCmw1dTCNENUjKE/5UoEcvK3KdoJvdjXYdmodHSfZU59KJeXj8l/vKCa27svP/Zo938Q93vI9h6Mj1ZvqEj7rePHpPKqf9OzqUIzJS5dyiJ0ACEanl0+WtvYOhUPVWyI1QmBPha7F/po/osqoFqqT6owqCNI5TYBzQSdLDKJywJIsAvYySDy0sZirP3RV6NdhpYNLiCkSoA+zbp8xuY91iClALmSddLXojZUZBdDVThQyXBopIkR5WkGlrqenp6aodcCkuInmSqEyYtGN6OuGieS1mJKFZQnUzl5Dn5WUrqvCleoqO7b7rn6KspgShtxXZ9qlQIB130V7iLVa7KSLq9h4JBCetXilpK2k0jK051tpKS3lOVcwofCxV/FS+yWqldOcQpBUBCkhQB5/fZLNy3Q8ptZvioKkJaSgodEjs5g6DnOtuFque68110lPX1y3+8GUyoIBMzOkCeptS4cLtbG6vWc3pZLS67tLyqKdtKSkNJW5GgCyTG3MiyXfAJu1eckQttQI1iFgiyw1TU1KghDYT2hK3CTJKj1Nku/XqRu6aoh5uQmYKwCSCNBbnKt5mkuwaBalPZgsUE046WF5idHCNbaoHHHkqbaOy1b+dq6ekTjnGuEsROUVwYnq6KmdpmnOzZUEhMyM207gfU2jfhBxVx5U49aui9Ma3pUtLu+ocUlT6sodbQVBR2nbn5WjNgUtfC14cAE02IRwEtI1VnHa5aai+kZjCa9fLaW0H8bHw6Vsu6eyQR8bVh4c4mxFet63O/XX268X62pRUGqrFJ7Q5SBofbIED4WmzGWP2MAppPWLmrL3F6LUy2i7yFKS4mIB/zToBqSDarhOGvwzHImucDcHXbkoYlUNq8LeWN2t+4Up4VuBy96lhLzeRtSlKLhQdcsEhO31FpPpafsHKxEDVaDMa6oQPwtB/CXi5UXziClwreWCr0uSpZafqmU1jgzLbKADAA01AtJV74urrvvRh6iuVb7DzR7ZfaQltSVN5QdDuCr/hNvR3V0LpMmYE/j1XDilkYzOWlLlI6n1uqoUkh5yoDwCUCco7MHfnvZJdK313Ow2taV9g6EZkwEqCiAfA6WaFXxBrk3tX3bfLN33feqb1Fz0zqqpa2lLfp+3pyVISImEJMDcmDZKxNj/GN38PcI1tNclCbzevZFLUuJCl07QK1pJEkK9tI3smVcLn5A7UJjBIG5rKBOPl01DfFaqfBX2vZ06lkIkAgQSSdNYs4PRtdSfXaRxorKWb5bhaZHdvAqAjn7f9LJHFy9q+9sW3m5ejjIrWH105DTZSlQSs8iTr7/ANUbhhxBc4fCuvVq6m62L2vOlUyp3IPtC05MgHr03NsuWq48pudAVt08PBiFt1Z2vo6ZVO0aamoygCQU06QT/hM2N4ZaYbvNpZYaSudFBpIAPmNffaD1elqmlZWUYApiWXJ71URGXxyjfX4WLNembWF5akYBuxsurCmwqrUVDXfQa/Kyjpmh4kz7eSM6pcWFmXdSNw1obnvvCt/YexBdLV40D173jTPsKSFpWkOoIMcuoO4InQxZL4f8ArpwVxBRiqnvq8Lxuq7qdxq4rorSV/u1bipcKVkmRE5dAe8SSTraO8I8Yb0uKhbeorroS3f2IKhxxDgWexDic3dII00G/nZ0V3HrFN1ur9Ru67dSUgFtShrof4rAOO4bGQyYEub5KzHS4g1juA6zXDVTJj+rTWYPvNtLTKw2xmUW3kqUPtG9RHTXTwtV3G7VOKGDq32yMhUkFOoCfjP1vZ6o40Ymv27Ki7ayhutLVUytl0tMZVpCo2M7iB77R3j+oUigW8SsuEIKSkQE9/Ukxofuje2LjNdDilSyWC+gtqtHDaWWjp3Mk6psNqSSs5gc7KTqTqdDttH5/ErUJEE6CTII57zy3tujcZbqHRmbh5kqSQqTy5H38traeCQe8ZSUn2j4fp7rTjZZFDrjVIl9tKfuWuQoEDsHASE6zB6RHL4jxtAN7sk1lKpSYys65e8SokztqPx08IsVWILjDjOY6oIgqI7xEDz/AFtXu+kLTWMJUVo7iifDU7CR0jWN/cN3DOYWLiVrghFF0oUkNpSFBwkJEAzO2/h8hvrrGGOlrRi11lBStammSsAaEwdgdBvpHOPG0uZUrpgrPokkgJRG52mNdvkZtEeMW3TjKqWQ4AlDSQQDlCUpA5aJ2I35DlvsHTdZCbF6OJbZP2hZ7pAA1JG5AkxzOg8N51uv+z1uYMYAxbfq2Qh68L3p2ElUSW2WVZST/meVptNqZ1rZbU2pohpKt0hJJIG5VlkgQBp7zta/foe0hu3hLd7CgEKqGXH3FzqpZq3N9NwhTY66x0tAu0skWG11YBhYQ82SCiXEmDqNDr4zytVP0m+GrPDnisMT3ZSsvXPj0uVZp1mEtV6dahAIB9sqDyYTut0RATFplvB0LUTMmY3iB7Xuny36WaHpe4VdxNwWcvmmRnfwxVs3qlJQTDAOR/XfRCswjmkWJEOqC7dVZuE07bk3De9RdFSVpK6Z9P2RJg+yTkM/4eR5aS70VN5U7S3qxhLiwgFdTRZS6jaTlV5qO6p00A0sz7rW3e1GyatpLanUZgsIzNqVoIJ5HefETrsFemdvhl9NMilp3A0rKAvPlCZEEb6RBHPWPK6L2TaIKofoK+p9bpnK6pLRCg6XC4G1CAUuNASkTuop5yJMWurhu/rvxLcNFfV2OqVTPsjeApKhoUqHUEHTbmJFqlm82GElLFBSmqpEBZiRl05KOux5eNpn9HS+xel1X62spZeZrEKXThITlC0e2RuMxBH+5ram9mQ36q0w5m/Raxi0lnE94IU4QVrCogfxJBnw36c7IOYI0SsnUCCJ8tfrcdDZ0Y/T2eJVrGYdsy25A5wCPPl4ffZrrhJgKJ2ACTuDH9f6W4upGWdw813lG7NTMPkEnPRnJUnTNOVKtpMeHM/OwtfeNZe9WurvB4O1BSEKWlpLcwMo0SANhyHjYNxSFLKwQZTmnp9R7rBlCgtYUpMpJ1B0mdND+Vostuk7dEr2R211VzOUqK6dY3jXKYgfC1efSipE1eHcF3vlzJVTO05JBB0Q2R8cp67eOtkHEJdZW1MhcpPkRt06f0tAfHen9Y4MXHWnRVJeRZzAHMBDqY98C1qlOWZpVGubmgcFTKqSe0IPSBGs2DGmUjQ2N3o2WqtwHkpWk8pNiidd5gba87dkDcLiSLOXqZgWrRfGEbjvfKkmvu2nqNua2kn7z9a2bPEu72Ha253axoLaNQ4w4CO7C21AyI/w216O16m9uDGEKtRzKboU02sa9ktTfXX2fnZc4jXLXXtcLoutoO1lMtNQwhX8aknUE8pBNuGk/TnI6FehsPFp2kdAoJvairX6G9WqRbBu2mvhDlIECVAuGTqNCDnTGvw3tl1sCsuq6sySBVXVX0Cu9MlJKk6HWeeutnJR4CxtT0i7uacpBT1LbDruZZzJdayxAAOgPjysuU2AHmHmClxrsqavfqEADdl5JGSeRBPwB95zK0c1REDyb2UUXjdrzV23BeCax56aNbyw4rRYQtKy3PQSYHhadqZTb9Oy8mSy4hKwI5ESNNtrNC7+EjlIQivvp99torS22pEJQlaVJUnUnfMD7hZ63ZQKum7Ka7+0W6KZoNJW5BJCREnxgTpYUjw61kaCJzL3QBGubNmOYRJMe/Weg62pv6TV1+pcQ7xcQCG6xmlqxrz7Psjp5tm1zykOKJbGh1EmTzm1WvS4uxTWIrqvHISmsux1kDmC05m19zlr2Evy1FuoVHGGZqa/RVyT/drT4hX4fja23o4VvrXC1hs6+p1j7A00EqC//eH5dLVJb2WkH2k/jP4Wsv6KVcHsNX5dsyaatbfAnX7Rsgx/7O2zizb0xPQhYWDutUgdQVez0NLwNJxEva7VKlFbdSymNAVNuoI89FKNrhkCJBBnUfUWoj6Nt5fuvjVcUqKUVZfpVyrQ5mVgefeCffa9qtR1nmLYtG7MxbNa2z7rkkbTr4W1I/mHnbCevKbaKp1BB8elr4OqolYVayOXU2J3ijOxm0kGZsaOg0V+dg3EhbZTrtGtjxmzkKQZhZM68YRVUbs6Z1tzPVJP3pFhFKMT1nn8rc4hBapu1CR9g82uToAAoT8ptyXCdFKI1tpN1WY4WQwJ1BPWdbcZxOaee2/ysGFzuoaeFsSoxIjQT7rFQzqhCr/FAI+hboKGmo1sAVA6lQgn5W2les8wNT7rSCijCVCQTpr52ULuMvCSNfhFkpKjoYBGkb2PXYoh4lOkmzpJw9465FGedst0kqCQAQABoNfztlldNZeFt2uLcv5pTK4IbKjpmA7x5/Xy0khd4ssNttpWQ9HfVvsDoDytGVHUvM1ofadSCtoIiSTJUoZSI015b9edlOmeNRWINZV5G0rH2Sf4txlHnPS1OGTK+yPK3My6k2jr6JbdZW1dc3SMss9h2qnIT7OpknWPM2gfDt3qrMa0d00zrK4rUI7VAELSkznBAJ1ACtfnZ2OYgvHEeMF3XcDKKmmo6MIcpnmwWgSoZu6d+WsEjXTezfxdTVt3Xi5idvLRVhWCWko7MNKCP4QOQI5TuN7FkqA6QN6ITIMrMysxIQSpzTNuCPzg7ee1kqura3M9QUTjQeKc6Co65dR8otFGCOMVbfj7d24jdS28mSXUJAD7eypT/MBrp/LtZyXziWhorwarrqWqpWzqsZSQtOkjSZ1+467i1wzNIuqYiN9UnY14gLTVfutSG+1bb7+QEnLl5yJBnTTl1myNc+J6Jy7mBVQHIepS4rQhtfsE7SAREnXWzRxFUqvW+qq9A2XjUAx2h76UnyjbbrqLJ2d0NBzsysICSlREKzcwTtMz7o2tTdISbq01uiXroQxUVgNQouq7ZGUwDnSdNZ6aGenTeyxildwUNW0y48ilqEoUlKgmSdu8Fb7TodIJHMCzc7WjprnZrWQj1kLLayFGEEiN9xpr9GyFez66iriofZdVKUpWlRJAzTEDwjaRaObRSy6p8cEK5DPGHCyG4UF3glskCI7x2Hjrpp0629C8PN+rXbSU8AFDSW+6Omn15284eFFS5d/FDDAcKghV6UuZIVIyZ09OQIn3jxt6R3emClClAlKtYG5+63H9ov6jfou47MaxP+qpb6Uqn18a6ktsqWhpy7yswDlBSjWOWsDx8dLev1BPqTJ/9Wn36C3kd6TAW5xiv1ZnI0i7TIOhkIAnp3iPhy1t643WqbvpjG7SP/CLaUJ/7aP6LFqv9VJ9UOqSNyD0+NgFkDfWw6+u3vsA5CTMmJibHYqkm6KPE5SUzZOeJCjGkazPKyg+dCT9aWTniUqOb3C11ipO3QTrhQ2FlQ0Oxs8WWrvvIJRX0zTyBqlLiAoA2YV6LyUDqkaZUEaEb6/Gzeuzite1Cy1TrYpXltISC8oGVmBJIBAE+Atk4pFM97HRcls4U6MMeHqX6m57ru9SF0FC0yXQSrIInpZUuWpaep1oSrvIWUK5QbQ5WcWq2uhJp6VAAgKAOpmeulgUcWL8pyX6ZFKjMgIgNzoCYOusyTapNFPJGBzV8GMNsFNd7UBrLqrKbOUl5hxEjlKSJtVo4Pbu91utVedQsMLaecBEJCkKBI15aa2fo4uYweQWl1NN39IQ0n62szcTXhU1V2V3eUkOU7mZIG/dPTzsfDWvpY3sfzQJWZyHHku+MfCrEHFPFLd/4QraStusUYpHlIcQoB1ClHQ5t+8LRlhDgBi1jHlW3TPH1lmiqaVZW0W2/tG8hKVmUqIzgwPG0o4dxFeN00C7suyscpqUqkNNGE6gAkH3WUrprKt6822zVOlS1GVZzJlPz1tWjNVAzhtItyVl8EM13OCiGj4PYhw9fFLSP3i0XbgvIvVAiQtKwlcJHx+tLSPj55arvuZwCezvajVtzKo29/zshKvK9r6vm9GRTOl9KkpcyZipRSVJ133AHw0ssXtSX7eNzXfdzGH7yW+iop1rcLUJCUlKlGTB2PTpbncbZK6Rk0h1Gmi08P4LGOYxSBSqZp/SDuRmjBLdRddQgyddW0HSd9bSiH01dPWoomchd7GEDmQp1JP/ACWjVWH78XxlwriJikKqGmpnGHl9ohORSmgdZI0EGfMdbPbD96rp3GEKbLiKpx5sOZkgI7N11UHXmFC0KGbJAL9VmV8YfJp0UY49uTE7l7GtorsqKhTeKMN141T/AHaGcjitSPZjz87P3EN1XnW4Oorv9RUmopL6ZqchWB3PWnVTIO2UTYHHF93mwm7627rsadp6h9g1LinsuTIe7yPU2PVWOLpfVWpq6htAWhtKcn2moU500mFTaD8V4TiCfomiw5zg1zRf+BVx4533Q4fxpfDroLiKmudWgtgEkZp357je0S1eJXLpoH0N0CqhFdf6yAV5ezC6NhyTod491nfx8bqryvtxNyrccZbeKiVgCRA3H1vaPL5o36i5+wFOVuN3tQqUBEpzUWU6+bZtvYdM6ogEjhuq1VEIZDGOSb2JMYV1NT1BoWksuFKnEhcKElI2MdR05+GonBzEFXjd+8aS9lIS5QXS7XNOtpCe0eSpMSD/AAwdh4WCvnC9XeDislFVS02EfZvAJnXT2T9352C9HfCl8IxGhbjnqtPX0FQyVDvLKCYIhQiZSRPX4W0q0uZRPew2ICp0wDqprXbEqer6u6kuzBWE6ykzD1i82ah5RUT9opMEidBbeI+6pS0uyQZjYgdfhY7fl3preGN3sqcdSbuelDiCJCmlgDl+H6tzEN21KlLcXfdcsETBWmD8BbzaneZnEvOtyuykZw22aEo3BWOKd7k6J0kTOnz5fP3gY1vJ1y73W3E50ZQQG0gKQoTr1O/yFkXD93L9YQpdVXENnMiX1CDOugjoPrSx7GNSF0aUqUo5G1gBP8QJGubwk26CkiBKz55Dw7JisXvVqvMMGtdKEFSUjMkQlIICDpqJgzpYd696p7MguZTBk+6IB843/OxBwA3yQpaQkIJT706g/P424chuUjdMqlQ0jntt/X39XHC0hYgmdshHLxqYUpVQEwJJSqCTp7+XjytFWNKb1atafZbgLp0wmdSozJ2+fv01tJRgKPfMHRI3k6D6/rEf8QW15aEpJgJUkhM5dFQdSANo6bnxtfpmhrrBVKhxc3VJFMEIpmm21LkAAaxBPOY036xaK8XpbcxZWGVZVJbhSVFMSAd4EwI1E6AgRNpbaC2mmUqHdWghJExsNTtJ1+YtE+MwhGKKhIyolCJWI0ievgTvpqCTBtek2VJu6Qn2XO2Cex1EKgxMhOwlJkRHlJ9qNL9ei204zw+u9haCjJSONgT/ACs0qyI5d5S9DO3utQ9m6a687xZu2ha7SqqXfV2G0albqyUoGpGpKkADf8PSXhthFOEq2tw22nOi7r2q6EGCkqSGqhE/BlPht4Wz5KhrJWxk6lW2RF8bnDknu3lKMxKVSnbnO2u/L8h4vJFFSX7hp26a5rNT19Kad4RuFIjU78z8BrZnobAIyNJI1SZERp/QbeXQvK5HAq7KcK5JKSJ5Ax/UxbUh1WZICF58XfddZgvEd+YOqmlF26K0sFDumZqe4oAaGUxr5+NlleJ7rZayVzTiUBQlcCEyOfIa8vxOsm+lTge7bvxvdWMX2Cilv9tV3VjidCl5JKm1GOpJET+NoXqLrFCtNLfa1qQAEtPgZUkRPeG0iB8NbHFwmFk6ruqKGtYUKJxCg4ggrI74zHfSSdRy8LLvo6X5U3TxONzuvlabxYfoHpOuZCS6hRnUx2agPPxtFfqNTh50Xjd7yXqZRUVBObKRm/VXxEQLPHhverN3cWcP3wiE09bUtZikwJWOz+ROvny516gE2JVmAgXarFcT0lm86N5wlPa06kAdcqv/AM6zKeKs2ZRggRBO0HlI8BrtaQ+KDJW1dlQQDlU6gnXnlP4ffaPygwcq16HloCY+Ox+QtxuI+Gpcu3wo56RqKPaHRJzBWsA68p21On324CJ1UFRyjUkmNLGHUACCqVbkHefjr5/1tgbCSE95JMAg/Dn4H6Nq7XK4Qi6EQUiI1yzzPhHx+fvhvizSJc4JYkpkpK/3fexUkDU/+cJB+Sj9TaaQjvJ7qQSZM8vrX62jfG91GrwTxIuzRSktmqhWsS0F/enz0saJ9pAfNVqhmaMjyKoFiFtSK1xSgqVHMcw28PrwslAHefdZbxI1DyFlSdUaRzg/rZEgpIJ687dvEbsC4CQWcVfn0Nr0/eHBZimMKN3XlU04E/5Xf/ebeVp3q0BTi0lByqhRPnOuvl9C1YPQJvAv4YxXc+4pa6mqYB1HatqT/wC6tad5pQUkgBQKEkwNenwi3GYlaOqePP8Add9hX6lIw+SbrlMhJWooJSOZB1Ov19RYq6CO8Rv7RA2ssPMdmrKpKjBM6AHQn7+nn7yzjKkJyrJGx16dfL8LVA7RXS3qk1TRSSQYiBOUnx/GwLrZKQRqUiFJn3Rt1/Cx5bSoKAICTGUHw/LW3DrZUFDKESQAcsmf6n+vJZlAtSZ6uS4A5Ble400PMfXwtAfpa3IFYcuS8YUr1avcppSkzDzRny/ugPo2sT2aQoK1HMHcHmD4Wi30m7oVePC28qwA9pd9TTVaIO0OBJP/AArVa1QyhtSw+ao18OelePJUKYJQ6I3UCn4iOdp19EqtCcR35dhVHb0Tb4SZhRbXH3OG0G1QDNc6EiAh0keAnS0qejHW+pcWaej/APTaaop+uyc//Rbrq4ZqZ/0XF4e7JVM+qupgG8U3JjvDt7Z4FLedOtROpyh1M+Wk/fb0YXvChzNvNLsnEEFsAKBlPWdNQenj+tvR65q9N63LQXmmCmspWXxB3zpCp+duZoHXuF1GINtYo3JAIBI8baOUGSZg21oNTz5Dba2lanTxnztqBZRWiTEJi2oAgSdd7Yo6ae6bcEgezzHWxGqJSBiSnz0z6EjVbagPONLJLbwdbQ6lI7wCvlNnJejeZrMNdLNC7yPU22809kS2PJJI/C2lAbtWbM2zkcSsRqQR4W2VJTy36CwQWJ31md7aU4RBkE8tbWEA6IUuGdBrPPY24SogjLsI0sGXB/Fy6flbErJOunIeHnaQUCjGYSJ08bKF2Ll0EyD5WSg5qZ8rH7tX9qFE6zBnpFnSTlzeHzNssW7cnULV8Ytln0SXhtToPbdo0ohfZgRP8MqmPl7/ACsl3lWu094KDLigVEhIGggbxJ5DQfGbHlBDVQU5kqcKUicxJ/i0A8Bz+YmyHenapqyW2ySrUInTUbn63nebZLTd6su0YnJwCcbViy8nFKzFdJI7uYzmHx3g+fnZxcfblo7vu677yacf7esqXAtKl5kju6x02Gg+divo7Ubjd/X04R9iKZCSojmpUxrry+6yl6StTFDcCASmXXlKABmQE6jlztQe93tENG3/AAtBjWmhLioSoVLaq2HE9popBUYgDw8f00s9VXkWVpbWVpSQFpAOUK0Gh5kD8POzFpXXHXAylYUCCkyJA5j693msNorKgpyrygnUExJ8Ou/LXX3HZKyLXR+sfL6hUR33BqUkpTI2GvmfrWxVWoBb7yIzJkwBAPL4a/1sI25Rs0q2nHnO2S3lQotiJJ0Opk7CxFSlpUpS5kJCYBmTED89OQHuhdTDbIwe0RdrhS4stqdBykQNtY8do6jSxZtinS0CzVla0lIHcMHUGROxBnwiLH2altFw3i0pJDgdZyJHQEgxZz8EsM3NizEVbTX3SLep26PtUtpcUgZgtI1ymdj1IO9k52VtypRszuDQieCKRf8ApGwwUZVBN6UygpLk/wC0TOu4OpMHfXwt6W0yCH8gP8RBA5T4/X4WqtdfDnCNx1Ld4XRc/ZPMLS62ovLUc41ESo7ETaSKbHOKFVBm9VkKXJUEpGvPlHn525vFqV1c5pYbWXWYNUNoGua8XuoQ9KdxLPF++EFZyuNXaQiIChCQTOnWfdy5+uFzqP7tpTzLKJ5/wi3jF6Qt511dxMTU1tV6w4pdKlRJ1UAgQPPp529mLkcz3XRqECadskDxQLXWxmOBjTyCx5ZBLUSPHMo+rWRpr8rAr7o8Ok2FnSAJ8LAOGRqN9NbFjQZEUqFGCOY0+vjZLqVD8IspVE5SdxEWSqhQzd4kRpubXY9AqTt0n3oo+pPpAPsH7rQ+h0lITPLMNeVpRxHXpoqBS1ICpWlEGBuQLRNKgfsxGhO3142r1R1Cv4fsUdcccQcxMyIkjcHmPrewjLzimgAsnQRB0J5D7/hYFplxSA6dErEAzpp0svO3Cmjuakvc3jd7oqVFHqzTpXUNQN1pIgDfYncWqFwG61W2SahzKiCvNzBzbfptYO9Q47dtZlMDsHDrJjuWPCmQSomdR0Bjp+Nua1DblBVtoVlHYrBPuPT+thlymRoi1yqMLjVICVEk66izkuZIN7U6kdwJUXCCZ0G4n42bFwLbDUZs32bevIyD8LLDLrjJS+0ohSZgzqDtYLnIrBcI6w83QY4vp4QhJbaVPQZ3Nfus6arENHT3ezX07qFMpppWlImDlSPwFoxvVxSr6cedJUpymRmV4gkGfj99lJKj/ZJ4TCvVleMQJi3M4/8A0g4dVdw+IGQgqSq/Hl20artRU0j77hUpEpUEpyuMqSQZ15C0ccPsc4gdua86FVcrs6a9qppogSoJBB3InmfrSwmIFqULodTAipZKj7lD8vhZr8PVZP7RU6yqU3y8QCd5Sg/jbnWyvNM4XV10DGSDRPl2vqak/wCsvrUJmFK9+g5W7p3YYdIQQMyDodjrZNaWVAhASTJGxgHSxqnUBS1Ikx3DBGg70e/e2TLdw1WhDYHRRlxGUkXy6twEQpsoG4Pd+vnZl3kHm1VhfbLZ9aup5zTUSXUSnl0+PlLo4mlRvolIASWk7nQQd/62ZF9EG4Kp6A4ct3LKkqGsVKwN+ma3pmDEChYuPxJpNU8pRqmb+XeVQi5KtppbnZlYcUmDmCwD0Jifo2S+Da6qjxLQU1Y4e0acq2AAQYTnXtHKR87aIq26xaxTqWCykqlGmUKMHpsT+Z5FeF9YXMZMrS1l/wC0H2zrIzaz7vztp1MjX0kjPI/ss6KJzahjz1CnKq+0wFXtR3W6ipBG8aqj7jv0s27/AINMlSUpgpnMTvpsdPLpz87L6HO1wnfzAUD2dY7KSIiUn8x8fKzUvSpSu72FwlwqaQRJ2gDX4fC3m1K20h+q7Oc+FEbrdWiobT2mx25dDYXFwzXUah0pAMJUBoSJ2Hwsm0a0+uSkZioDbYGeYO4so40dyYfnLmUmSMoiBy8utupoG+KxWLMTkJTDcP8A25ToQhQJQ0mI2OQb/rbhxwZkyAqQBMxtrz3ttlxJxBdjgXJeYAhQkEAKTy/yi23EJQokmDPMyd+cW6dgsAsflqgCnXUgFWpJJjfrz3/CzSx6wHbqpQgFSjVlJA0kEKOvTX7x1s7VIVCdhAnVJn68bI2JKJFVRphCcqKyVDf2kxv4kp+FrcRs4IEgBCZ1e83S19LTtrjK2FEb6kdeoET5+M2iDHhdbxG8nLoUoVmgzlEpGgMjY+EdNZka/albmK1IAStLRyyQYAAG3vsweJNODfs/xFhCglOYKhRXrmAjkZAP6XpPdVNu6kf0PLsw1ePpD4OVid1pNBRmqvL/AFhxLaEVFOw48xmnUQ602YmDAE6xa/l2ss/2pvp5t9p5k4gbWlbawUkO5Z1iP/nJET5m3nH6P9MmuxZRvnuOUdHULCVGMwy5BvvPbTIHIWu1gXFfbqvChXlQqnqKKqSUp9kfYI2O393qPo85W0TjVNqw7YWt91r0k7QwxEbqTbwp+wyrbT3wqVFSiRPhOx3+t1jDzyl3cO8cyVLCiQBrM7ddRv42L37SuGhU4gpTACp9/LXx8PlYLDDx7BxsqIUl5Wk9QNduv1pNt6hkzBZlfHwym96QeEjjDhdezLDWaru1k19MOZLfeIka6p5c4i1U7sq3LzuxqqCkuO5EoWh0DvEAjyG3LbytetS21NkKKS2qJS4AQQRJBH1obUkxPhRzh7ja+MLk5EUb3a0bhRJXTOd9oH+Y5YB/xJMaW0gs4bJBXedBTuOU9XdymAFlLiuzKg31GmoH5xZHqaG8KWtpKy45cu9tZqWXkLBLK5BiCZyyJjw2s6y/SV+ZvtGfWhqhsHNJG8E7gz9aWal54aVRPKqWS5ShRAV2JhPgY8RpEWhKzOEWN2Q3Kt7f15t4jwLc1+tKT/rIaeBCtDmQZ+JFmaGFJSUhtWih4ieZ6cvv6W64VXw3fXB5u7lJKn7oqUsuyqIGeQRHgo/WljrrCCdjlgSkjUc/ONNrcLjX6dT9l32A+Ol06pPcazJyJGgjUmJ8umszt+NuAyUnWZMmeUjTfx91lJVIe/CSZOoA85+j08LYmmIJbVCcwJEk7z05fpbObItd0d0nFlQMhsAgbETzP17turQr7s9bvrFVzutApvG5wQAdT3FNny5fpZ+dkTIypzg6nWZ5e/WzbrGjT8QqMkAJrbrdaOb/AALBgctlT+liCSxugyRaLzRxQyWi0SFqWkrQSUwCfqfP42biSM2qrP7ijdq7sv29bvISBTXk+wU8wErUkbT05mzAJ/pbv6V2aIFebVTckpCtb6Ad4BOLMVXRrFTd7FT1gtu5Zj/62121s52wQAZBHKN9flbyg4f8QL44eV9Re1xXneNDVvs9gXaKoDKi2SCUklKtJSk7crPWp9JfiJUQHcT4nUiDAN+upJk/4Ep+VsXEcIlq6gysOi6DC8bio6YRPGoXozU0y85WUwqdQr66/Wtij7PY/arypB3JGUEzprbzWqOOWNKpZcevK83TO7t8Viv/AHosj1vEi+a8RVUtO9rIL7jz0f8AG4q1duAS83Ky/tLDyYvSupvS46VIU/fNDT6QS7VISB8TptZKq8dYBowUVeNbhYiUkLvJlO2n83X66eddXje713GxTUeH2Wb0SuXqpSGFtqTroGy1I5bqOtks40xBMtv07es9ykZT/wBNit7Pnm5Af2kA2YvRB/i3wqZyqd4hXDmA7wFe2qPckzZjcVuKXC2/sAX/AHBQ40oqmqrbsebp2WULWVu5CUJEJ3mN9Nd7UlGMsUBGRF9VCADPcVl193nYJeKcSL1Vf1frp/5wv87WIsDbG8Pzbfzoq0vaF0jCzLui16ILdYsFKhMKhQ12s8ODFci7+LOGanQJcr2mjJ/7zuHb/NZjvPPVDhdfdW4tW6lqJJ95spXFeK7ov26r3QrvUdQy+NIgoXP4W2pWZoizyWDC/LM1/mF6Mu02ZClJOsRJGgPu+t7Xg4LXsm+OFmHajtSotUYpD1lolvX/AIbU3NEh4Z0rCklIIUn+U6g/Py1taT0aaxT/AA4VRL3orwfbAGwCsrn3rVbiMPk/WLV3mJRfohyleQZVOswbaXA5HT7raKwDPQ8utuToOfibbwNlzxCwkc7cz3tVfrbZidJMTYMkphWliNKgUHUp7RoyIB982YiT2NZXU8Zcr+YCYEKAP3zZ/GeXdmzFvlBpcQvI2D7CFjT+UqBPzFrtOdbKpUt0uuSsT5bRbCsjUfxHTf6+jYuVQoiO9PP47212hKj3TrFr40CziTzRgLkxPObbCpkGR52Lhz2iTIB/O2docsCNefIWYJIyl3vROx5iOdjt3KQFhUgzpFkrPBJ02JMn8bHbvcJcmfHW0kk4gWo1Qj/hNssT7Y81GfP9LZZJLxJbShx1JGSEpRIPNZGumnnJ62bV89yoAUtJlSiDlMefnqZjpZ63E80KlxL1zIqiWkBvNKshCNB3dRvGvSzNvgMP3wrs8jas8DtDlAJM8xtrEx91spnvq04eBFKC+7wupLwu+scbK0pzAKgSkzm035n3CxW+sSXxfxR+9LzfqAyT2aHHVKCQd45chtZT/c4p7tdr3XkunIowkBSMoHL5a/1snYeo2K6vU3VoWUpSe6kgT4a6Ry958rWLNJzWQbuAy3QF3FCXD2iQVESkpgR5+74QdrLKVJUITUNpCEqJka+cxrZMokssXv2N4JBaT2mZJOhACiNt+Xy1sUefcIIWo5hy5DroP05b2kRdII6muLhLSFEpTKYGw3126A+dh/3felQhVSxTOLGhzhJJ0HMgfXU2IZEC71OhgjMYTz1nWOQEiB1087LeD6p9mrSz6yOzfaX3PaPdBIMHy+doW6KXJCUV2VIuS9XKqlXTuoDQQtwHWTqOh2nnZ+ejEioVxbui6C0H2b4cFFUtkqnIVicsHQyBy/KyDVsXtXUblFKEKbbShtZhISU9IE/HWBp4yZ6Il1Vdx8fsJ1FYtotpvBlSggkkAKkjY7wPz52pYo8x0cjgbWB1VzDh/wB1HpfUadVde8uBV2Xa+Gaq7rwoXCgKLTxKCUk6HUAxvZEPBmnrsU4cw3czvqb181RpA68VLShWWQTp4G1sMfuUeMMRm87vzqZDKG0lSCNt9PfaJeJiavA1fhTFtIgKdu29kvpaOgchCiUz4gEc97eN0ONVLsVFO2YvZcgG+hXqMppZ8NzGJrJSOmxXnp6aXDi8+FnHWqwdeNUxWv0hoXg4ylRQtLjKVCJ1G5BB3+/1pw+ofuahIE/6s1zn+EW8p/T54hL4m8fa3Fa7tTd6Q1RsBie1VlQ0BJJAAJ18reqWG1RcN3yFf+as79OzG9vXic0TSvMg1zXuDt+aVnVg6D5dLAqI8QbcrcMxynrYFaySRMWkxM8armpV3CkdJ1HSyPUqIVAEFPysovLTBnTUeBsl1RAzEDW12MKi/dIWIKf1qiIBAhxCpUdNCDaL2EoVGcQCN8vlr9fO0jYnrPV6VALaVBbyEAHkCRr9eFo9pUutOB0t5pGh3kf1tUrCGkXWhhoLmmyO05GVLKQQc4GYnXX6FjzFM+KlFKkoK3FJy/agATy6AfL3CxJgrWorICNgByj32N0TgaWQHgYJ0kc/K1FzxbdarWuulO/7jvbDV4uXZeHZOOJShYUy4HEFKkhSSCPP60NkqobfqmXUobVJbUI8YNjoq6hxtTDDKio7QkmPrWwoum9HmVLZpKrMpBBKWVHceAsDjR8yEcMdbVIuHrveQw0lTZbBZbk+Q/X7rOu5MP0d4vPU9ffjF2IbZK0uOtlYcVIhIA2Pn02Nke4MN32/TU7q6avP2CBm9XVqYH62Muqu273HEVF50bK2yUrS9WNoUDMEEKMgzPlYZka7QFTa1wbrokS9FNt3m2M8EsqAI0kBX6iw7UO4aqG1GT2LonmNCPOyRfuILjor5ZqHb+ursGad1LrjdcysI1QQDlUTJ1IToTBgGCbF/wDSRwvaw5WrqOI9yM1YaeU3SKdzLUrXKCAeZjlztj4xC+WAZATr0VuilZHIc7gE4b+dKrsuh4JWPtKUn/iGtm3ggOKvTE1OJBF55xoRllCfft+W9nVfxQrDVx1ly1CaxippqR9DoQUyhQBB1PL8bM6kvG9rtvm+zdxp1OOVKFLKyAIyR+FsKmpDJC5p/mqvzSDOCnmlt5AJyKKIkzuP6zY5RUL1RSVzil5S2ylwJGv+0SP+q0VY5xRxDwi6husfYQH0B1tLSWlHL0IOu0dbN65OOV+XI5XVmI311tK5SqaDLKmEqzykgwFTysX/AKenkjzssVU9rRRPyO0Tox9QuOXxTuLGqm0gSNTr/SzPxV/2XdV55cpKKKncCjpIRVoP/V8DtZKvrj1dN53k3U1WHqzsEo7PIKltCyudN0kRqevL3iO4ho8Y3BU11NRO0jNRd1ewGy6l1WZtxhU5gACNZ26a26mhopKenayQWssipqop5XFhusxTj5Vx3+miSFLF4MJpYBEQuCN+UT9CyTw3PZX88862EuG/DkSnUpSQkqMzp7XwmzjS3dKHGEFvtMzJV9otIM93WSFfdZIwq7SUWIrzcdbQyo16VIWupSmJbQZkogjx2kRrbS4DeE4W5KjxnGQa6XUv09Uhu7MRMOOIQfWUKTmIT7SBtOh/OzXdbcq7oYTTqbUvsEJKc6R3oEjfl+JstVV94GpmLxpqi9roSy+8pxIqLwRJzNhJgjLPPYWR0cScAXXddLRqxLcNMQgIQtNaSox0lcHQTNuJjw+UPJa0rqHVERaLuHqkekoa5FeCtATPtEyd99uUdJs4cS3S7V3ItlS0EKSOoPz23087N+o4rYHdWFnGVM62ZXmStS0wn2tUjoJ05DlYzenGG4Kijauiiv5au0CUJT2KkhR0PIagzGvjbbpaeZrgcpWbNLAGkZgmo7Shq65UELcQkoS4ESUp7RWk7jy8bBO01S8ciKOoXuYQhR5jw/H4WLXxiapu67luNVtUwh4uaNrUJggnQEfzaT4+Nm9XY0rEUxfCbxqkhJKW0Fbi1wNkJzySSdhbfaHbBZBcy2qdf7pvBaNLnqxlAMinVp4gjxn5b6WLVV2VyKZ9x+kXlYyrmBKOU685HMTr42IXthvjLd1DT3m/wYxE5TV1V6oyS/Rgl3IpUKHbkoASg95WUap/mTmJ0VxcXu3eeq+FrdBQMEKq6moxHdmZluJKg0l4rWe8YSkEkiBJ0tZjhmvchV3TRWsCojrHSvE1SkupPeIKdJAzRvv9bbWbeOkF2/G1KdCCKJnN0iVZTBBG86/fFlanqQb4vV4rlOdScs65dTE6gCBPXfbQFJxasm/EuIbIQKdhBOUd2UEjYHSfLzOlrsujFUjN3Jwej9CceN0RqG6cKp6oKU7IiAT3oB1OXpufOLM4arqejxHWsovFkrdoElBSXMqsgdVEkCCCkHWPZ+NXOGVQaXHdO8w+EF1l8pJCRP2e8mRsTtpoDrraacM3ktvH11tOOhYrqd5gAgjvKSoDWeYUI8uU2qGPO25RhLwnWVvb4xmVti6UUqcgbSguEnMRlTrH4xy+AmEapDi6hSQApCkkmIGoifr7osxTVIeTS1AM9pTMrnMBMoTp57n6NnJgysQF1TYRlJSDMHKTP6nbla/BE2P3VSnqHzHxJ9oeMZzIO0gwBvzPT8BavfpS3Kmnrrgxi1lQHELu2pUE8hLjRJHIZnRr/N8Z29ZSJCF6lWsj3/nFo74+3Z++eGd5paRLlEpFagp1goOoieh+uVpAG6qXe12sVawqmcVS1EaKRPfPP7t+eo0sni/sZYZa/wC2WG75u7ZS0GKhtG0xML2894mLOGhbpqgpadbHaglSeRI6ePnvYW87vfVTeriidfSvvJypzECOfMxMafqXHkpJ/cCcS3bUVN4XZRVIXR3xSds0UmSlxCk6EHYkQAD06zaXBdy1DOSRpmlQM7ePmP1tTHB9deuD8WC8qVxTK6KozOsQoJWhR1PwJ1O3vtKt9en7w8w/V1F2PYIxC7VsSHUqXStJzc4JcUfKRNuMx+iqaqdroG30Xa9nq+mpYHMndbVT2LuIGQJggxO493uHy5WaGO8QvYYqaZDFMFl1srClAAGCNPEwLQdV/tJbtaKv3bwukcu3vkJM+SGFzv1sy8W+n1eOIXm3f9H2HZaSpKS7UPumCRpJaTGw5WyYsFxIm5Zp9QtiXHcOAsJPwVNq+KN6dt2bNG0dAQDMDfwgmwNNi69r5xPc9QxR0xqEIfbbLtSmnRK49pxyEjQc+ZjzrbX+mtf9SkAYCwuSkHLnpnF5SZndQ+jZPc9M7GGdK2cF4RTCgdbsVt0/vBa2MFr3aFoH3VF2O0Pzk/ZNHjk4qoxziVwpQhx283nXG0LzpSpa5gHmNdCNNBaKVgpMEROsRZz44x/W49xBWYivWipqd6tKCtmjbDTKcqQnup1iconXcmzYdWVrJTMTp5W7KkidDC1j9wFxNZK2WZz2bXXEkn9LZJtsDwNukMur9htR8hNrVwFVAJ2QfKJtllWmwtiWuKfUsPXlUZtuypXFz8BZbpeDvFiuT2lHw0xQ8nqi6XyPjltEysG5CmIZHbNKZ5tlpFZ9HfjY8ATw3vlmf/SGex/8ZFlam9FTjfUkZsMUNOOtRflA1HuU9PysJ1XTt3ePUIjaKodsw+iiS2WnOl9Dviq8tCau8cKUebm7fbS4/wDZZj+NlFj0NMVqhVTxIwShEEns36x1WngKeD7jYRxKkG8gRhhlW7/xlV7sICezSeijr8LTze3oj3zRNJ/d2Pbnr3ioBSRSVaEBJHtZuzJJ20y9elku+fRxxDT09IKK9E1jwSUPpRRuIDcE5SFKjMDsIEiNQLMMSpXbPSOHVTN2KYLk9MbBVFhu7aGrwzez1UxSMsvLFQwkKcQhIUr251IJEj7rW89Abj1c/F9vGl03fdT1Cq6FUNRDr4cLgdDqSRAER2aRudxbzYZ9GnGjzq0LrqBhAPcLpczKHIwlJA5c+trZ+gDhtXAPiPfVXizFd1/u2/rp9WVkDoKX0OpU3Eo1MFYjxtkmnw+JxfEfEtdtTiEoEco8K9LiBBBMdPK3BEkFU2baOJODHBrfraY3lpwf9NuzxAwhl1vtpMSPZX/8NiAhQIKXzJGWNPG2lHUkiROulm8riHg3VJxBTyN9FA/MW0OIGDlwRf8AS6kcjr8rFaQoWS+ZOhI8bMzGgTT3vdtSRHbIdYn3JVP/ACmx88RcFqOUYho5iIkzt5WaHEDiTgZd30dQ3iu7ZZrW0yXgkAElB1O0E78rHhdZ4QJm3Yhy6mZzdd9ibaU8MsgQAJ56+P42Z6+JeAp7mMbnVzPZ1aF/cdbAL4r4GbGX9+hYEqPY0zzmkxplSZtqBwssstN09i6rKYIk6aibYHV7/GZtH7vGLCKE50G9HQRmHZ3dUQQPNAEe+xN/jXcwSVMYcxJVka5WqJIJBPLMsfdZZ29UshUmIdklWY66QB+djdC6rPvpO4M2h5fGrKErYwRfaUknV9bDWUzz7552CTxxr0OpbYw1StKkgiovZtKiY6JSo/XnZcRvVNlKnr1hX/dzbLQOfSEqp790UKVcx+8FaH/2Vss/EYnyFecWG73prsrnFVlQ23IahM75Up1kcgNbN7Eibscqn30XYhROZye2V3lFR1A6GR8eU2WLpuJi+u2XUuVCXG+zA7MBUS2CZ8dPmd7GXMNUhfX2y1uBKYcJA1010Gn142zmtANwjuJLbJnUrwVhZ1fZqbydqhISnSYOniYPwnyshXRU1LDjjlMklcphQBOn3/XlaWUYXu9V2Jo2WVntVLXlRmTMgCfl7p8LN7DGH6Zq9HkdittxKCCHASJCtToYO/hpaecBR4ZTFaQ/WV+iFF1ZIG4JVOnOehtst1DAW28nIuSlQOneG8+P66WkGluSiOK1JNK2odqVEqOitAoQM3htvvFiFdRUbVa4h6kZLnalJzJBJUegJ8fy52YypxGmW29V9kA6oRlkTrrPv5nfw+JzD5q/3nSpacASlckjURGun4+VnOlttlKUM01PCQCSGkASNYk6/lB62cFz3u9dylKPqq0ue0SClQMKlOngB578tYcbyRBEDuUSYvphwlo0z5LitSlgke6PADkd7SRwVxjdGC+JOG8X3o1XPUdNUmoeSzSlTmUaEAaA7zG/SyVdlfc9Y6C664lxWhBWU5vgZ5bG1g+B/EC+LsxDdVw4forhpS66hAqnLoZcdKSdu0UCrmee4tkYxUPFK9uW4IN9baLYwqlD52kOsbi31XpFwsuW7cb4RuvGFFUrapb1pkVLLbzOVxKFCQFCdD4WH4o8BGOIdzUd2ovlFCKWo9YU4aftMwCCIAzDmRzsNw2xOGblZVUpr62qKYcdRTHIs9QE90e6zyfxfUdiQ1h6vfCkkEJyII03OdQtw3Z+HAhA2SRtpOW5RMQqMQiq3AO2Pl/deGXppXF/ZnizUUyHy+lYp1BxKN9FAbmeU/Dbe3q5ht1IuG7gND6mx/8Ag028qfTwvBD/ABuqAzSGlSGaXKha0lSAlKhymNj7/O3qVh11P7gutYWNaGn08ezTzt3NEXyUUT37kKvW5WVcgG10vF47lQ0PKwC3SRvy26aWA7aJUVkE2CU+lIJBMza7GFSedLrbqlEEkiB87JtUuZB10iQRYR6qOoB+f5+6xB+qSoKhRB56xra9GqLymvjZ1SKBASVZy+0BB/xg/ODaPF8VqClSpn1/DzawezCTQUxUFD/MfM2fmL1F6gRlT7DzaxHgr5crVSqOG2Nqm9qx5i87oZZdqXFN9rWEHIVHTKlJ8PeLUq+kiqbcVXcOqZIL5AprPG1i7kKzYioGAhPtNUlKiI/yoNkCp9J6qF1m+6CqxjXUJLQRUUdxVaULLpR2QSoU6Ukq7VvKATOYRpZiNcF8Q1zeStxPcDJMSQ68RvB/2ImdrOO8eH/Em9sDVeA7x4yXPRUlaw005UXbcznbgoXmzJUXUgEnOSAAJcVAACMtCLDMPB8S0ZMQrLeEIWo9I3HdS4UU/D3iw8YnM5QuU6cpkSC4tGmh6bWTneNnG69FqbujhHfThADYFfiqjp1KUrTVCnySIIOk72I/6Dr/AHFKcqvSYxMhayVqNDc1PTHVOUmUKBEjf42Nsej5h+sR/wBuceOJ1eUjLJvdTaQOaQnNtoDHha2KLDI/hVc1mIO529Es4Yf9IrEWFf7QXXceC6dph5d3KTU4hdrSXm1BJlTDZb1Xoe/IyqBE5AuvLt53tfF4V15VdV2K7yfNU4yw+0tCFqkqCVQoECYkHUczM2sfdvArhlctyi56XHeN3qRTrtQpkXi0ErdchTi1/ZHOpRSkkmSYBNspeBnBK72g21SXupAGVKXbwCRHMJyISP6Wm2Wji/pMKG9lVN/VcoHo3nkUF5tpfcUvNTOZyUGVd8SMqfjp7usfXyy47er9Q4+lt4p7NYLhB5HknQ6+O3SbXd4f4C9HGmxXX0+MrvdaucXch1PaVdW4VPpdEA9monYmOXnZ+P3B6ANM+updwg1WOjvKKmrwcnTotUbco6+NrcTGyNz3A+qozF7HZSL2Rz0VsE4f4pej7cddf97pp74uxs0NQGHy802lMKZ33IbW3JmCQYkEG0k1Ho18PXX1O0+LLypnnoLq2nWyVmImFJIHlZiYT40eiZwsoHbt4e4Mq7ppKlztHWKG7ktocWB7RzOCTrv4mx2s9MLhAynPR4LvZ/SQVtsJnX/MTbn58BvMXRvAB+q0o8TmyAHN+Fxf3oRYIxRXiqvji1iJSEApaQ2ikSUg6xmLZnWTtZEqf2efCzs0pZ4jYodMqBUamjSQDMQBTGYnn0HvMv8ApuYNbVF3cO6lQCZBXWIT9yDZPe9OELBFBgBhvkC7WqUPkkWsMwt7G5eNb6BV3Tve7NlufOyUL4/Z58Kv3c0qhxZfoq0tgqcU/TkFQj+ENA6gkb2QMbei3dWDr+wnw+wPU19VR37QXww7X1ie1RTvrabKc/ZpASIb209k22z6auI6usRSruC5qJtaoLi0vOhOmg9tOvnG9ka8fSsxpfF+Plhu6kO4eD9dTON0xhZFO4lUhSiDovad7F7iGnMZnH7JMllB90LVL+zvxIxTNPI4o3SF5NUroXhlkaj2zrtrZMT6C18XZTXhdruH8N33WvU7zTd5OZp7ZeYh2VpCgUk7D+YiYSkWGvj0weJC8L0153XjGjF4uPllylTdjAI/xQUkxGszGsaGJZ1T6VfHerckY1daSdYboqdvMfcjwI+tbrIQzd5P2CBd7twPyjCPQh4vmmYo0YCwAk05KkOu1LRQoqHflJplHU6iSY1EmbLTHoX8cHDTn93cLaQMoS3lyv8AdSkJiOzZTGoJIECzIqeP/HSs7Qrx1iASIhtwoAPhlA6GyU9xT4xXiSKjGOJXAUzlNe8fCdDtp87HbwhyJQ8khO4CnK5fQ84yUlyLuivxtg9IXVLqlONUdQ4U5lBQSmcsgQB3sxICQfZTAFD6CD13qZqL14pUqDTlJATdsAkebvMeW9q+V+L8bvq/7QxJeCgBr214Lg6c8xjxsRbxrRUz03ziqgayJkJevRoHNGk5lTGg8dbJ72W9z8qTIjzcrcJ9E7A1WpuivLiQFoQFANs09OSAQJ0UVxsOX5W096IvCSmpyKziDWtrElFQxT07LqCdlBUGFDkY5C1dMN4lbTTIvK78V3C626jOgU19Urjzic0CGkuFyJ11T+dlml4juVbZQKhS+zJHNMwddSQCPLfflbn6yjq5ps8L8jei1qd8LWZZDc/ZTnjPg9w6xRdlNc2IuOuO6iloK1q8aZNFX0dO43UoCsqwpqmzpIzEyFTt0Fo+vTgRwHp2H6g4p4j3vUsIW4x69iuqca7USpMp7oV3tSCCDPjZpnHt3sNntjVJcAGvZiNJ9qFGPnYgriLQh3su1aWTplW8WyN+qDGmpHKLPFSVgILpnFT4lK0WDAql0CEsXrXNF9anUuHPpJUqdgBt77JOJqpFTiB2idVDblM2oK3IITOnTc+8bRBsp36WrmxtX05ZkOuqjMqcxMgCPu8OUmS2ceN1z96M1FG4e19RQ+dcoSQpYieum2sb8tOjlbdtlixkBylT0e37voeJFEbyu+mq30M1QaRVMJdQrMytSjkUCDEaeY2MWsXe98pTWUj7F03XTt09ShRRTU7bJJ1BEhO0aeHTSLU24ZYrql4mu2qpUst1zVQGlB/MpuSFAylIkjX5HQTFp/r7/wAT1jYp2a65UkfaZfVHEBX8MFUEjUg7fwaWqCIk3VgygCym648UhN30zDyuzNPTpQDnUUqygDSE9Bz+Vlq68e0NPVlDy3HEpbOYyNDIMTIJO2w+Vq502M8ZUSECruRFWB3CaN5KsoiR7WTSYnbysoXVjqrZce/fFxX22jIChKaBbpJBB2RmhUHrHnaXElZsVFscL9wrIP8AFK4kZkoS8SBKRkIjTp08fumyVf3Eq77wu6tu8Xc86h5pbZ+02RGnLUaRaETjWpeSr1bBOKXs47uWhQiZ0ABWtMeI5aDTkC7i3EeSWOGuJVAOEZnDRpCvH+/5yOWnSy7xKVMU8I2TPrHKuiUUUDSu1Z+ybCU5u0ImAQOcT+HMWHouIVdSU/7vvu7F0r7gKEPAKUiSJBmSY2jXTxto1lW3eKk19zuXa+9Ck0ry0qVB0IBQSD066crKrDRq6IU1Td7eUjKlK/aIOhkeGkxy99tJhLm6Kg8ZXWUbsLqaXE7zlfVmqdqGg4XE6COURuJHmOe4szMacGb9xziFD2GXKFtfZTUduvsUp2gzBnf9BaR8cfuG4nKOloe0L6AXEyvNkQDvqO7OnhoI3gH8DVT4vlhDbjSfWQCQqQkRIymNtz59elGokdELs3V2BjJSGv2UOU3olY+djtL9uBtRMZfWXFEDXUwjQaWU6X0PMSuEJrMW3a2SrKQ00tyI33jUWt61hWqWjtV1rKQoZpCSoRpB5HXTXXnPiZ/stTlcKeUdZlLStOeknr5fxG2OcTqid/wtgYZS72VTqf0OWQlKqvHbpGpPZXcNvCXPqD0soseiFhgNgOYivl1ZJkpYaRtyAk2tMjDTCE51POkhQUQEpkkxA5/QHSLcJuRCAQpDnfkxMFOwA2jX63tA19UfiUvZ9KNmqrTfonYZaWVGsvd5IPsqLYkf7v52UKT0fsNU/ZB3DNO4GNQVuLOYwPaP8Xlt8bWabuWlChNKoKToSXOe/wCP1rY03ctAkqBaHtaEpKpg6+fh+dl3uodu4pxRU7dgFCV24FwywUzgy4E6ADLc9OpQjlJaM/ebPq6KG5acBpDZpRySw0hoDyCEAnb5+cPj9z0rawOzAGWYCYKj+f1rYdm62ATm7MA84AMeX1vFqspkk3JVmJjGbJvsNXIdDWXs5rpFYuNOWg6fcfeaXdOHHgC/SV7p3lSlL000knXpy99loUVMSFtvZeUjlGkdPrW3WRhAKu1Qe9BzGAB4/XO1R0DirrZmgJDTh3CiYb/ckhQkjsEpgaiNQLGG7luZuOxuhttJgE5ANPLz+7lZQ7ahbQSKxlMmVBOUFXnHh0sGuqusArVeCE8zlTqIB/KfjZhSOdqUu8sCLqo6RlIZao0pTqlMNhURp1Ej68LBOUza4LTB/l9kR8fcLGFXrdyYArFLSToAFd0jTSdP6+8l3b9oUd3I8SgmZIBjkZOoOtjNondEJ1a3qiz91B0LUhpRUnYaajruR+FijtwuqKciQDvGYjofhpt1sOrEqKdXaN0rqwoc3SBttsQPMfGwasXvhGVqna1AgBRMGfCJ6WMKF3IIBrWdUUXhN5z2Xm2wZASSrcz+njbTWAXg+ioTWUh7NQISEqMkeM9bCOYsr4zAhsrICYRMyPnufxsFUYlvFxMh9Y1IEo0O3h4/fY7aF26A+sbZPz+2XElCswvO7VJTpoyohI02Mxy38ffbSuI/ElCEhN53eVgzm7AamBpPLytHSr/vInKu8HSknNCVQPl1/CwDl4vKTPrfaSRCQdxvvP18rXmQOG5VIzMOwUkDipxJaUkE3OEEz9oyTpM8+nu3sIrjfi9kBVUxdUHMZCwnnOkp2AP1Fovdq86lAJBInQmdvMzsBrrysTdUUq0akpGwMR0+psdsPVAdKOQUqVnH7EQaCWKOgDhAClSFCdNu7tr42a198Y8V3pdtVRFqk7VxIKc9OlbIWFDQpyDoedmWsvAKQGgkCJ70fHT62sEITmQAFDMNAZG+nysdkQaq75CUZVxEx/kaDDd0UyhGjdAgADfnr0/WxR7H3EhSxlvlhhajrkom4g+YsCoA6KUo5pEZjIJkfhYFRSk6uKWknuxp7v08PjYGgVY3ujBxVjpwfaYsrBmH+ybQmOswNP112sVqLyxHUFPbYsv/ACohWRqrLZUImO75/dbgF0FKQoBMmdNvGfP77chLqHFxmk7FKfa06c5185s4N0yDU7W5i65e18vKByku3g6sHTmCojb7zrbVJQUS3FOuUBVJzSZPIHmeo/pbS1uFEIKswMa7gzBiLd0rr6HA4VqMEbkHNy/Dpz99nUeaV1UtMklPqdNoY9q2WD7cjTt6Yf5iqffbLJSUBYTpKpPrLaLpqHytwFPZtKXICEyrSZEzp8jY1U3NiJ50qZuGvymFaUjpk7b5dedkJ2/b0Cm6egvyoQ0RAp6eoUWwEgbBKvLl46WDapMR3stTbVLetToR3ad5yYBGmWQTsCJ8bUuIAbkhHLdLWKcbVz4mYJdXd1akDSFslsak/wA3ujTbrNkeow/edO6qtp2qRhwuKWQutaSVAmCIJBBj5iJ6KFBws4g3o+kXbw9xJWKCVK+zut5YzR1y7jbSZjWdrOSl9HfjdVDNS8JcUw1lzZ7rcbnyzAT0/O0DUxDd49U4he4eFpTHqrhfdf8AW275oGnyElYVXMkGCCYyqJGw018bdVtzUtc0n1jEN1IegpW4ha1FQ3juIIO206T8ZFovRR9Iy80hFPwiv1CETlLyW2gnXYZ1pMf16Q57k9Cv0laqkFK5w9do0O95zt71pkpXERKQudvHkdp1G6tpm7yD1Ck2mmcbZCoKqcL3aw32T2LKNZjOvs23jIIBGnZwIE6cjHmCbtLctOMovqocy91RbpjuFTOsRz+trOL/AGenpC3mpta13KylOUlNTeZB/wAQSpCFxJJ8tOYss0n7M7ixUECuxphqkbUDIbW8sg+A7ODvvpqBpaHtKiaNZAiCiqTsxVIbqKBltCk11cQkDKCyk89/7wGZidPnZw3NxavfCt60943TVVyamnWFsudi1lCkkwqFZs3vEDpa2NN+zGvpsJVePE670LUZhmjWoZv94iRJPQ2TMRfs6blwxQqr774yKPfhDFPcwCl66JTmeUDtzSRYL8Uw+UZHOuDysjxUVcw5mCx+qhwenhx9pmSzR4wvVthIlXZerIyzAGgpxGp/PpZv4g9Mfjpf6CzeGNb7fQhSsgNeWpO0DsUo8eukbb2ma4PRQ4SXSnNfbF9X+oxnTU1XYtKUZIIS0kHmNCSB42Wbl9GTgvdJXWN4McfQ4uQ3V1LjsRuEdBrB5bedqjThEJvHCPQKz3fEX6ufv5qnbH9suOOM7ow9QUia6/L4rGaKkU7UOEurWopQlS3VGEjMdZ2mOQt7SXJhvFdLc9BQruJ4uU1Kyy4ovNgFSEAGJUOYtVvhlwr4WYQ4gXBjFOEbpuZu6rxYrF1qlrZRTpQoHtCoqCQkAH2hFrbv+klwIYPe4s4ZJBBhu8EOH4JJNtyidBXMuPCAsqsjmpH2dqSgxh3E5VBuohX/ANK3+CrYrCOKlIn1Sn22L+vyBFir/pR8CpAZx9T1OYSn1ajqXfeMjZ6WAX6T3CpwD1SsvurB1HY3DWxy1lTQHS1zgUzd3flUjJO7QNR5WAsVPCJoWxvq+o/9FsHCy/XR9re1I3m6NqXr8R4WSXPSZwiof6phXF7+4GW6w3P/ALRxNgV+k3duUKpuG+LFZpyqd9TQFQfF+0g+lZ8Q9UwhqH/CUp3hwSq7xZDNViZsSQruUZ3Go3c6gG1ZscYGvfBmNrzw/S+sXimmWhaakpS2FlaAsjeABmjmbT0/6S95rSk0XC6sUVSP9Zvhhr/wBdqocbuH2NuK/EO9eJLV3Ul0M3t6u09Qt3gt5SFMtJQlRWEICswQTsIg87VqmWleAA7VWaWGpiNy0p1Iu+/SEj/U2l7BK69lB3B5q8/rY41h29alAXUX7czM6FK7yQqDz9mfoWiZjhBjmmQlKr+eGQZUZaxeg/Lb4fFRa4L4mqaJyrfvlLrlOB2jfrC1fZye/udtARGkg9Tanan+ZXCKgn3VI5wuttwCpxVc6Ecsjrq9IP8AKj8bCfuW4WiXKjiHd7Ujf1d8GB5pHL62tF44NPPAl6tQhURISSofEHx+Fke/eEF7oeU5ddTX1lSEStptZSlwRqUDYK0iOc/GYfT7XUeHPvZTU2xw4aSpD/FBpa1CMrVMCJ8QpwWBdvnhPSJUl/GV5vKQdS1TMEGOUdrM9P6C1Z14dShYBaqMqVBCwXFdomdIIInQmOs6RYw1hdCnVKepMhnvZ1SUj+EHqJ6RGp8bSAi6KNpFLl+474aM1Ieu3EN4IQtvs3C/SElW2VGRtKlTuc22h11Fo9vPixciVrTSUd81SUHKFhumbS4BzAceSYPjrrrEGxWnuO7kKk0KAf4wo5ZPgTM6E6EcuekdruG7lhJpmG8hjMkwVpM8pGh5QdN9RFpBzNgFAxPKTjxKp3iS1hy9FOJBVDtVSIEDxQ6uN+li7vEyscTKML0YiCBUXupOUxpORhX1O1l5i56NLKltsU4UCQlWbKtI0kKH5aanewrVJRNlTdRT0JSD3ihsJIHOQN9ht8LPnaOSXCd1TUVxBxSs5qTDtxIJMgfvCpe93/m6RzB/G3CMe46qSDSsXAjLCu5d9S8oeEKdQeXSz0TQXeQFt9kpCBJOSU+Gk6aA+/3ACqpGgpCXVNO51EZ0yABGsGJn4fCz8RvRLhO6pkvYp4iVSM6bzo2FEQFMXFoUxvKqhfjuOVg6W9OKDJNSxiOrQp1tSF9nQ0SISqApMqbXuCefLY8n0UNtj7GrKm82Ud4kaK3zcvCZ326YtN3kOIDLqEoSJUEqUknfcanSN+k72WcHknEVuaYwRjqpSttWJL33yryro2gCf8SKcR7jYBVyYp0m9b/cBUZUq93U9Ng2pJPhFpCUllaA2h1LkwSYKUg7zIEQPvBsUUulpVQXSwUju65dI1PTmT8bLOeQUTE0c0yH8PXu+Slyvr+zMEB29q5QCifF73e7xsWTw+bMO1VPRvISSYqC4/mMHUdotUc9OelpAVeF3lkOvusLGXRSklJjTYyZkT5/G3CL8uLtBMoTsC4nuxGpJEjl79dtLPndyS4bBzTGRw1p1EH1G5BK9k3dT5iNdNUmDPjpPlYdrBaaAksIp2ERKizRtInXllSIj3naN7PE3vQq7zb5dBUDBToNDrJAkydPZHzkmu+Wm1JUgVaFH7Pvagxz3gQAddN+tnDnkJZWJCbw29TAJVfFfKoltuoPenkIIM6kRH52DXhRJKfWLwvFQX9p9pUq+ZnqOnMbQBZWXftRHbJSHz7MklKzroBMgnbXYR11sD++CSta6aoZW0Ncq8xSRz5wIB5nppzcZioEsCSncF0SnEreS4pYO+dR7x8NYI0Mc9ekW03hWhbC0JpAk94lJSSCkEEnczrzAIGtlJV5vOIWpCG3mykthOvtGQDI56fLwsXVV1jTZbZLK1ZSpKVNnQ6A7nYGOg231tINclmaNkwOJjBpryYrGGkDOhIzCEiBy+MmAdPLZIvqvURdFWhxWRbCmVKnTRYnfWe/8jPi7cYNO3wzSBCe0U2tSHFIQUwcoM7bkGI+in/6Nq+/rvy0VQhlxh9KgCNNRBHloOg02BsYtzNVfPlcmngfs7j4n3RWuUyX6f1hD5S4S2VpTm6+HUcx1tauj4i4RQM9XdTlKVgFJ9XQ4gzOwSoq2B3Gs6QbVWw4067fl3VTS1NVFAp1KylWVUFtXQydQIA5E+QerlVeLjy00ynV5CV5e3JWEiACQRO+kDSdRIJIrmLiI7ZuHsrP3XjnB1WhIZxDQU7RkZXkGmgiJMupHjqNJ18bOWlFDeQDtBX09Qkpkpb7N0AwRMp3G+418TtTF5+pUwXKt1LaHgMgeQEAhOg7pTBAHxB5wMum6ZapyJCnHSrs3GYCgYTqNATHdOnTnpAnUAJuHIza+27VdZN1A5JS2VA5VQlSZ1/wjl5jSdNLdKu9AOZypUkKVJCHACAVcp8lD3adLVAo7w4gtIU5dl9XwOzSM6E132aNDIyFW+3IiAIibLNPxB4m3WpNO9WULiFOe1VNZQrQj/ZpH825Mkk66pFoGiI5ogrmnkpl4jYeYqlU9VSvpFSiYzKSpQJgaeACR+PKzQa9YRlSpalPnuqQ2iQ3G+o03G8czvtZCo+KVbeakM112UruZc9uzXtJKQVCVFrMTHMQZ5cps5GKhupWVs5Ql6BKBqecCNOY/W2jTtyMylZ87g5+ZMa97lqXA5eNW12inHSvMUZlZUmBqRy2033625u6pXQv01UhnMqnUkkkGMydSNtOfWzrqKO8bzZe+0QaMKI7Rbm4EzEjUb2Zj6YfCCS4lKsgJOveJSojoTAn42BIy2hVmN3MKzWGsVXVV3QyFU7ilsogxkGh0CpmdgDz0B3spqxLSnNlpnFAnUKKYHnA00jYfHnCXDm+lCmNCXRkTISknvTPdnfl93xfPrpglTSBzCijuidDJ/p4TvaiaRm6vCsdbROtzFCASn93NgEc3BMAHUmBpJ/LbQsb/qHTnTTsJjXQkySNx4STy5WbqUuqzkJdgHVSRB93XlqND4cukKXklp+QCdMugnkep8Y5edmFKwJd5eUtuX/WlSiWkIIhIOhjXfU7mLF1XxeRVC6ooAJMBrSJ69dB8DZNQt9KZcQ2oKI2Okaajl9c97bUakgpLaTr3VJXJHl0P62mKdvRRNQ5GV3nWLzJU+ohRIgaKHhNgV1tSvKsOVZyynQnT7rBdmsKynMhJH8SRA6n77dlIH8KlJ2griZBmz8FvROJndUF6w+pJU6p9SojvqO/uNgCtbye1kFJAEFU6axzscLLKzk7IjWAUrIA6nU6b25XTEkykzoQT3h5SOf1rZxEByUeKSk5ZUqAqphZ/lVIJidtpP1vbaXHm0jsy4Qf5kyBt5QR5crGlsqSofaEzsVHRXn1+/SwC6dwgKSnTf2hOmm8aj8vO0xGFAyFABbgXmD6xlEzlj39I+X4cds85JCxrqDMeEEcxHzNjKYalYpgVezJ+/Qn657W5cUy6AFNrzSNMoB+Ea6eNlw+ibiFF3HCpJJQc0xOhM9frrblVW7JSpGaAU8oynwG3152NpbpJBzpSVCQI8ySdbcpaYSQpsz3dRl08NuW/n4WcMUeIgBeb6Up7enATvO8x4/X3W5XWBIhQVmJGkj468vlvYwpTYWEHs5BOp6chE+dglFOfIgIKpOZWWQT4wNPdaQYomRFV1UHMArKkDWNYnQx9eU2364gtlS+UaEgEmBrv9c7C5UhQDYQAAYypgADbQ+7624cQoFKkOazrJnpr+n5m0gwBRL7aoNT7LiSnKjTukBM6fU/py5X6stJiIImQ3O/u0+tbcvJcXK0rSUE5gc0Dr1j68rAO0j4UVpKErB0IPMg69Zm08qhxEIUsqCkpIBnSFEGdvyGnhzsEtDLqVBM6+ySomJ8J+XxtnY1bZWtVQkwCQpEj3dOR+fnYMpfy5VOI5zrpz1Gn1NnSJutBppMErSYEaD37b/1+IZQUAwsnKAMp03nTrYYiqOZBQ0FIESdpiOW/KwKwpCftWExMKTlMn468vlZhYJkCtxzMYdCiPEA7RMxvv4WCJyHs8+p0235zM9eU/GLGF1DSCEPUwCRoDsPOAI+EWCS8wrLKXVyiAUpMRHXbpp/Wz5gFEtKLOOEBZIWoZYIDJBPjvG3KwrK+8QUpgakwZ6g6672MpaSoAMB0IgElaD3p5z4WOU91rWnK0jInYBQiCT5T9RaOcAJ+ESi4LahmcYbzHVWnO2WWE3UnKPs6c6b5x+dss3ECfhFemDOCcMU7aW6a4ruZQkylKKRAAPkBY81c9G0kJS02ANgGwBbLZbysDmugzuWlUbZVAJSPC2l3a1spaiCLZbLFawHdFD3DYoBVOw0fYKvM20420kBRRO9stlpZQEZrid0A8hLeoSkgQdRbRcPZ5ghG8bWy2WgNCjt1tdJN8Xg5QUD1chtKi2kkJ2+doGxD65iWt9dvWvcWZKUoQkJSnyHutlssamF7kqyNkjN4epFvoQpxw5yATpznWwbt0UalZ1hZCQAlIV7I+vnNstlroUCuU3PQ1YVTPtl1CiEqDpzhW2457/Kwjty3RQVLtNRXZStBhZaBDKZOWNdt5M2y2WsROIGhQXtB3C6ZaOU9kUtgJBgJ01/rYaoazuoZmOzbSJA3mSfvtlssYOJ5oeUDkglICVBKiVSqN42BV/02FS029QNqKT3XlpTJkjQSfPS2WyxWqFkCmlRCtRokKHdH80fh87HKKjS6p1hSzl7FatBr3UlQ/8ADHvNstln5qKKu3ZTJUVkEgTI66n3cultUdOyxWJep0BK2pOsKCtNiOYiR77ZbLTCbmlG88NUdM+2thWRt9tLyUBPsBU92eYkf15kDhmjfLrSllJVELQkBSTMZgeotlss5UVGfE3BV1G4l42bKkVbDxp6pEAipCHuzCp/hXporXTQ5tCImcbp00yCadJLqdY00k7dNRMjrbLZbSpdWC6zanR+iIu3hTtU7MUeZNQ6lEKUk5SQVBRlOvsj36+FiSMQPs0jaktznqOxlRGYabzGuw0+62Wy10AKmXFGW78qX3mWlNt5ylHfKZ1UAdvqbEay/Lwp+wUFpyvgaJSEkBRPPYxlHK2WyxA0KDnFAu3/AHmahspqD2iVwFlKTptG22vPpYVmtvAVZoqmvdeStsOjXKBJiIHv+XSbZbLEDRZBLjddPdrTutvO1Dj32yGgCYgKSNZGvP5DxkOqU+h5omqdJdkI7x7iYBg9efTe2Wy0g0KNyiqAXqVpxMI7RAMCYB113125/pbpqrWhtmkqJcU64W86CG4iNSANfKbZbLPYJXKMIYaU6WinKSgjM3CfZ0H4W5ao1J0D6ozaAgHYkc9eXW2Wy0VFxKEVdrfbJSh1beUTKAAdCTE+87zvYuFR30rcIUsAZlSRKtRIAEeEWy2Weya5XCXKZDgacpyo1BDeYKAgDroZ38PnYneF4ponlqQ0tbhVBUtwGYT3dI1iOetstlpBMiDd8orVIUKFKFwO8VzG+2gj2jtZzXfdabyaCw4lkAxCWknqJncnTckna2WyzlRRm9sIquui/eD9e3UpXWIQEBgoIKp1nMQdxpHLygRm7BTkoafUC04jKYE7kfh87ZbLEbsou3UOfudqjxIwpDplFUlKyEiVy5BMGRyO4I185sXRcNcLVlR6oll5ChMkqTljLmgBKUn4kxPutlstUYd1ZIFlqqwFhy66pLCGKlRTupNa+gK15pC4P9etu2cLYdcZW45dTToBHdfJemT/AIyefSLZbLEuVEALlFw4SaUhJwtQBSnEolDLYGugJlBnfr12k2VH8PUN00yapimpBmyyEUyU848Rsek2y2WiN1MgLlla3m0qKjKBIkyJOm2x2G/TSLNN9S1XmrKoJ7VQTokaAkEbDkeke62WyxYt0OXqmnjTiJeVx3mMK0FIh1SkOPGoq3VuDRShAQkpj2eR2jpaPl8RXXXqti8rnadcpCUFdO6WkuAAH2VBcbxv162y2WHL7yND7icHDTiWbyv391MXMacBttXaGpzEkzEgIExB8dRqItY0pacAUtKoypV3VQZO/KPlbLZYbt1Jmy6FGlaA+lxaYbC45klAO/kQPdbWVxGqXlSO6CQDz8bZbLMnXSW1HtBn0B7o1EaeBH0Lbapg+o5XXEwspMkKBgnr5Wy2WSa5utuULjOVfrMhPRMHrvNiK3nUuDOsuELCDm1BM7m2Wy0gE91yupcStCFZVaADSBHl1sJSvl1bVOJCnAO+YMEgcotlsskgjd50TtNFQakKOYAAIIiVKHU8k/OyahxaiSFxBSmMoIPyn52y2WdJcuZ1uKSVDuiZj422ptJBJkkqMTEaAfr8bZbLJILHLuUlDjhfBAAhOTSJ23sVLUuKSCkajZO5m2WyzFMgkMdsla82UJ7sRJJBiZNtmmGVBSqAZ0InYn8vnbLZZX0SQakONFQS6QE6R74sNTtuuICy/MxlCkggDeOhtlstFPYI4i7FK/vXkrEbdnHQ8j42E/c1PGVxxZjQ5DEyJ5z0i2WyzOJT2C6YuGjqELWlx9OUjRTmaZ+Fjhw3SUrgQFlWROhjx5yTNstlhucUVrQtJudhTgADQUteSS0DBjewT9MhlGdpKE8kiDoQTrvbLZYYcbqQaEWSiUoeSopzlWmmgG2vXxsNT0frLvZLWkCSkdwHafytlss9ynaAumaCmeLjamkDTMTlmZIH5fCwzV20iSFKSqCf4SAdQDrIPW2WywwSpWF0IugpMxlmTO5O9stlstJPZf/Z" } ] } @@ -76,51 +61,28 @@ { "language": "markdown", "source": [ - "## Load the Model" + "## Do Inference" ], "outputs": [] }, { "language": "typescript", "source": [ - "const core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'CPU');\n\nconst inputLayer = compiledModel.input(0);\nconst outputLayer = compiledModel.output('boxes');\n" - ], - "outputs": [] - }, - { - "language": "markdown", - "source": [ - "## Load an Image" + "const inferRequest = compiledModel.createInferRequest();\nconst result = await inferRequest.inferAsync([tensor]);\n" ], "outputs": [] }, - { - "language": "typescript", - "source": [ - "const imgData = await getImageData('../../assets/images/intel_rnb.jpg');\nconst originalImage = cv.matFromImageData(imgData);\nconst { cols: originalWidth, rows: originalHeight } = originalImage;\n\nconst image = new cv.Mat();\nconst resizedImage = new cv.Mat();\ncv.cvtColor(originalImage, image, cv.COLOR_RGBA2RGB);\ncv.cvtColor(image, image, cv.COLOR_BGR2RGB);\n\nconst [B, C, H, W] = inputLayer.shape;\n\ncv.resize(image, resizedImage, new cv.Size(W, H));\n\nconst inputImage = transform(resizedImage.data, { width: W, height: H }, [0, 1, 2]); // NHWC to NCHW\n\ndisplayArrayAsImage(originalImage.data, originalWidth, originalHeight, display);\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAIFArIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1SXSkkG1ihGQehHQ59a429+D3h+9klkY3SNKxZjHcY5Jz/Epr0RvvfjSgVEcPSg/djYp1pyWrPJpvgbpjKVh1G/j/AN5kf+grPufgXMYysGtvj/btR/R69rFLVciFzM8Cn+CetpjytRtHx/fSRc/kpqhL8JfE0RYqLSX/AHJtv/oQFfRtLS9mh8x8yP8ADjxXGrE6cGPolzE38nqtJ4M8TQld+i6gRnkrAzD81zX1IUVuqg/hTDbQHrEn/fNL2Ycx8py6VqVqx+0Wd3F/11gZcfmKgG4ZBbnmvrMWsI6KV/3SRUU2mWlwCJYhIPR/m/nS9mw5j5W+fAzxTwW3AjJAFfS03hHQpx8+l2Z9zbRn/wBlqjJ8PfDshz/Z1uPopX/0Eip9mylJHzZEzYI3H7x6/Wp0ZiseM/d6173L8KfDjA+XZhDnOUmkH82NUZfg/pDABJLpAOBtuAR+sdSqbTLc00eJbjuY8kdOKYWbHPPocV7DN8Gockw6lcL7NGj/ANVrOn+Dd8ABDqStj/npBj+TGqsyLnmJPI5FKCSSNxrvpvhJr0Zys9k49MyD/wBkqhN8MvE0IbZaW8mf7twgH/jxFFmByO9gn3j+PanAuGAyfxrem8DeJIcZ0edsd49sn/oJNUpPD2tW5Jm0nUEAHVrZwP5UrDuZ4JYnp+VLuPHofSnNBNEAJUdGzjDLjrQFJYZ5xUlCgnOOfes+d3XU4scHA/rWiAc5x7Vm3fF3EQM57596mQ0aMUm6PrnnAp2/AznNRwHbFv754pxOAFweaUdimBftkY9qCxAz+tAIJPtTSV25I6+tUSOJbOc03cc8nHelbHTn3o25Ht069qkYmfu5yD2oyScnj60u05xyTTgpB49KVxgCRTWO5Dj09afg5HAye1KVODnnIzUsZl2TsplHPbI/OtLYjYbbgHms2Bis0q4/WteHJgTjgipe41sQ+SpxlTSGAf3m9OtXAoAzwKUqMcLmi4WKX2f+EMffNM+ynPOCO2RWhsB4oCZPFLmHYoCyLAEqhP5UNZnHCd+gNagj9BSFc5qeYfKYxtW7Ix9/Wo2tGByQ6jPcVulMDnjHrTGjOwnB6Ucwcpzo3K/r2qdYtyAnAPrSTrtkPHetSwVXtACoPJB4zQ2CRlPZAIWaJD34ArF1C12xl0hC8/wjFdfdJHgoFGO+KxL6P/RZMdAc/jW1KTTM5o5bawPANHmOP4m/OrxT3PPrUezII6/hXZzHPYq+Y/8AePFHmHvU5jGM4H5Unlgj7uPei6EQbzS7+MU8xjHIIpjIAuRmnoGopJ4JzzTd555609uY09gR+v8A9eo9poAXfil8w+lNwaTB9KYXJFcA8jinmYYwM49ahwfQ0lKwXJd4IxRuFRUUWAc/NNoopiHI5SRXBwQcg19uaPffb9EsLwHInt45fzUGviGvrv4X3pv/AIa6HKTkrAYj/wAAYr/SuLGLRM2o7nXlzSbzSUhrhOiyF3GjcaSkqh2FViHYZ9DT9x9ajHDj3Bp1ddPWJEh24+tG4+tJRViF3H1o3NTaKYh28+tQzMd8bZ74qWopwfLyOxzQxoXc3rSb29aTtSGkAu808OcVHTl6VLQ0O3tVS+lmjsriUT+XsQsCEzjAz+NWGOB0yT0FVLyyF3aTQzOxEileOAv0Hf8AGpavoPY46TxpbC2huJLdpZJNrBSqgpn+LO44+nB9ap6ze6fKJQ11cm7JCNGMny/baOAT6MQcVn3ekwXlhd2YmZLzT2MfmNgkqBz379f0riFvtkUNs06tPBJhcAFVABODkc4PvjNY1KV4tGTqNM6z7bpwHK3ue+JUH6eXRVJfFjBQHS4LAckAdfzorx/Y1OxtzRPdG+9Sjmkb71KK+wZ56HClpBxSikMWlpOtLQAtFFFABS0lLQAUtJS0hhS0lFAC0UlFAC0daSloAaUQ9VU/UU3yIv8Anmn4CpKM0ARNbRMMFePTJqnNoOlXBzLYW0me7xK38xWhmilZDuYM3gvQJs7tKsx9LdB/ICsy5+GHha5YM+mRhh0KvIv8mrsaM1PKg5mcE/wm8P8APlC4i9kmOP8Ax4GqM/we09jmC/uo+ONzK3/sor0uijkQ+ZnkkvwclCsI9WYk+tuP/i6ozfCHVhjy722YL03q4/kDXtNLS9mh87PBp/hb4hiDlBaSn/ZlIz/30BWfL8P/ABFABnTVZRyds8bfpur6JpCARyAfrUukhqoz5sfwrryKXOjXzDtshZv5Cqcuk39vgT2V1F674WX+Yr6aWGIoAY0/75FH2aH+4B9Dip9iUqh8ulcM249PejYdpwck9K+n5LG3l4ZC31YmqUvhvSJ/9bYWzH1aBG/mtQ6LGqiPlfaUu2XuRxWxaZNqgPQZ9s179N8PvDM7bn0m13eqx7f/AEHFVW+Gnh/btithGvoryZ/VjUujIpVInig+7nH6U4RbgF5+tevy/CzSWOY5bpcdB5q4/wDQDVRvhZEhJivpRngZQNj9RWbpTLVSJ5gIULHIGKcsKjr1Poa9Gl+GNyMeXfBsf34sfyY1Sm+HeqpuKyWzHtw//wATWbpz7FqUe5w/lgtxkbR1PemmI468n1FdbJ4E1tMYto3A6kSqP5kVTfwrrEWSdNnJ7BBu/lmo5Zdh3RzhhOSBj3OKYyN5fbn3rcl0TUYiA9heIAOS0LD+lUJbd4j86upHQEYNLUehyd8hVyCBnOTg1a06YrasOpDcUmooVDjHzNyah075oXHuK0SuiHuWXOWz3PrWbeoDDKD0A61pEHGe9Urpcq4xwVPvWkCJHOsPTnHvmoOTxjg1YJwABn+dQHBFdaOdjW4zSMBxnGfrS56kHnofemknjI4NMBDjpj3pj/dbFOOcd6aeQetUIQH/AEccfdfr9R/9al4OB1zSIMwSD0INIMkUwDC0YwcGjNHX+tABjNBUDijnOCOKPrQAmP8AOKQrTs+2fxpD7UCGkYptOI4pKYhK+mfgNffafh89uWybW8kQD0BAb+ZNfM1e7fs73vya7Yk94plH/fQP9K58Ur0zSk/ePcTRmg0leYdaFopBS1QCHgqfenU1/u/TmnV1UtiJC0UUVqSFJS0lAC01xlSPUU6kNOwEKcoPpSmmowO5RjKsQfanUgEpy96bS5xnHWpY0OAJOAOaazwhXDSKMe/Q1Onkqu+V05H8R4rAvNathqzad5roWXeFTjI756n3wB9cVlKryK493Y4DxmZYfE32/TpI0VodkrSKYwTyMFiAOmfeuJtNAxOupahf29naysfKUtjzeecd8e4HX1r1XxBq0ElosEttJNG2GSJISiJ6HLLtyOvXFeWXfh+91L95d3iR7mLCWc+X8o+p549K5Z13JWbsS4JPa52McPhJIkUMjAKACUbJ/wDHxRXMx2Nokao19bFlABO1+f0org9k/wCdl83kfQjfepRQ33qUV9azgQZqB3xubLAD0NTmqsoysi9MgimhM5rR/iJoOtXkVrbXM8c8pwiTxFdx9M8jNdYsrfxVyFl4Ksra4gnJV2iZXDBcHI5Fav8Aawl1k2UaSjyk3SM6FQSTwBnr0PI4qrCub4bIpjTbTgDOOtNQ/JWL4mjefw5eQRsVaZRHkHnDMAf0JpWHc2I71JRlGR1BwSjA4/KpxICK878H+HH0LVhLC7LDMhSSPPB4yD9cj9a7rzNsuz2zQ4gmXCwUZPSm+cn979Kikb93+IrivHGoa9YT6edFuFjDLIZUaJWD4246jjv09aXKO53gdW6MDTs1kafcyS6da3NwoSWSJGdRxgkDI/M1oq2RS5Q5iajNY+p3rWOm3l6kLTNBG8giU4L7QTgH8K5Sz+IbXLBToWoqx4wh3Y/QUcoJnoeaM1BHKXUHkcd6jedgXO7Cr7e2aVh3LeaXNctpXjfQ9ZuUt7DVI5Zn+7G0bIW78bgK6JJtw5FFguTZoqrcXiQFAxA3ZxlsU0XoI+4SPY5osFy5miq8dysjbQrD61MDmkMdmikprSKgy7BR6k4pAPJozTQ6sMggj1Bpc0ALmjNJRQAi9KdmmL3HvTu1AC5opKBQAtGaSigY7NGaaKKkB2aM0lFAC5oIB4IBptLmgaG+TEefLT/vmka3iYYK8fU0+ilZFXZRn0LS7nPn2FtJnr5kStn8xWe/gbw1Jn/iT2a567IVX/0ECt+ilyod2clP8M/DE2SLDYfVJZB/7NWZc/B7w/OSVlvIif7kw/qpr0HNGaXJEOZnkNx8A9MbP2fVr2Pjjfsf+i1j3P7PtwB/o+u5P+3a4H6Oa93BpQafKI+b7j4CeJEyYdQ0+QDpkyqf/QDWRc/BfxlADts7ab0KXKr/AOhYr6ozQDRqGh8gXHwy8Y2+d+g3D/8AXFkk/wDQSazbjwb4mtgTN4e1VAO5s5MfyxX2iQrdQD9aZ5EP/PJP++RRdhZHw01ldW4lWe2miO3pJGV7+9VsHFfdjWsLjDLkem44qjceG9Gus+fplpLnr5kCv/MGjmYcp8RYPegn2r7IuPhx4RuQfM0CwGf7luif+ggVk3HwY8E3GT/ZCRk/3JpV/wDZqOfyFynybRgnpX1BcfAXwlL/AKoXkP8AuXH/AMUGrJuf2etJYk22p30fpvKP/wCyrR7RBynzqBxRtHrXulx+zzOMm31sn/rpaj+kn9KyLn4B+Ios+TfWEo7BhIp/RSKPaIfKzyEgd6bjmvSbr4KeMof9Xa2s3+5chf8A0PFY9x8LfGtqcvoUzjP/ACykjk/9BY01JdxcrONr1T4CXnkeO57fOBcWTrj1IZW/oa4m58FeKLQnzvDuqKB3+yOR+YFb3wuF3pHxL0d7m3mgDyNCfMjK/eUqOvuRU1bODCCakj6rNJRmivKO0KUUlFUgHEZBHrQpyoPtQOaRD8pHoSK6KREh1FFFbkBRRRQAUhpaKoRABiVwB15pTSP/AK5T6jFKeaQXCpYIfMfcQMCohy6oOp6CtHb5UOEGSBx7moaQN2KVxFDcMftTKtspA2lsBmPr/LFYWvaTpcNnNIq/Yw/ytMkrRFs9RwCWPTjHNdO22PBIzjpxk5qte2EeoweVeIGhYg+Vng49cdfp0rmnTTTTV2wUtTztS06Yim1S6hQZBkUspwD91SCQePY4471xt61ld6sls9/fM9zKUaJ48FRxk4GcDj0H0Net61cvJbxw6bGQkbKA6LkEZwVXHt3OB2z6ec+JdRG26ewnghuWg2yrb4Z1IBXBYf7wIxjPNebOnyysnc6FrG5lyrp6Sui+C9YKqxAy0o4+gTiiuf8AsuunmJ7jy/4NzDOO2eaKfso9/wAWY83kfSzfeopWHzUCvqGcaEqhfc2dyPWNv5VoEVy2sf2+uoPHavbNZS4GHU7lBGDz+dNNLclps47w14XvdG1Ozube9m8oOokjL8MvQg/hXos4Xz4iRzg4/SmLZPHZsYwDKFyoPTPasnTtR1a/vhDe6WluiAnzFkzz6Y/z0qrruKzOoT/V/hVLURv0/jn5kP6ir0akJiufvNch0mZrXU4pVhOfLnRCylfQ45BFCYmjRt1CtGf89DSu+dQwOyD+ZrGbxj4fgj3LemRv4USNsn25FXNGlmvw17JE0QmbcqN1Vegz74FDYJM2W/1Y+o/nXLeLvFn/AAjU9kraW16k4csyvgx7dvbBznP6V013LHbWrTTOqRpgszHAArDvjo+ryRtJqFq3lghQJFPX8fakMsI0et2VlqcE0yxFRKkecBsj+IeorXgY7BmsJtS0/Tra3061lSSRiI40jYEgdycdBjNbdvkoCaAK19JbxaddS3jFbZI3aVsE4QZJ6c9K56y17wk7KLXUItzEBQVYZP4iuiu4Rc2NzbkZ8xHTB75yKw7DwpZWsyytaxBgQQdoNSyonURdOOlQNz5wPc/0FTxgBcZqEj5ph6n+lNCZydh4LtbHULa7GxZIHDgqOTiuit9QS4u5Io1cCI7WLoVyfbPbpzXn2ieH9S0fVbO4h1C4a28wB4mclSp4r0bA81DjnHWhgirr2h22t2qR3Me4oSUOcYJqroelJpmjS2SZCq8mOfXn+tdCvK1UAw8w/wBr+goA5jwnoUulXDStd3MoKbQskhIH+cV2ydKoWaAIvHar4FQWPzWVrhJswO24GtSszWRm0/EU0Jmdpl55TNESdrcj2NQWVy39qiQsxBds8/WsGTUJLfxX9gfiOS0WaFv9oMwYfltP51paezGVWcEHzW/9COP0pdB9TtI23Cn1XgOVqehgC9T9aWmjqfrS0gKV7qP2N0UR7yRk84xTkv0a0+0EEAAlhnOMVn6x/wAfCf7tZUl6p0+9Eb5CKwYZ6MBUt2nZ7FqN4XR01pqEd3uCoykAHnFWi3FYGnMY5MA9Vq/qFy0FhJIv3sYH1NODuhTSTJpNTtInKPOu4dQATj8qnhuIp03RSK49j0rzjWINSuYEGm3PkOCWY7c7vQVc8L392UtpLrInJ8uUevOP8DUptq5XKtjvy2BntQGBGR0rO1SF7vS54I2wzoQOcc1zWhagYZRu+XPySA9iP8/rVuNlchb2O2zzS1yj38tzqAKzSLHuAARsZFdIJQULZ496lO70KasTbqXNcjq3iRrWGS5VpBAnQRJuZqseH/Ea6pDDKrl4ZvulhhlPTBxU8w+VnT5ozVa5ult7aSZhkIM49ayz4gjTHmoqZ4GZMZ/SnJqO4RTlsbuaM1RsdSjvSyopBUZ6g1dBoTuNqw7NGabmqz6hbRyGN50Vh1BPSgRczS5qtDdQznEcqOfRWBqbPFG4EmaM1GGyKN4JxSC5JmjNNzgUA5HFJjH5o3UzNGaQyTdS7qjzRmgB+aSm5ozSAcaYQD1AP1ozSGiwxhijPPlr+VNMEZ7Y+hNSE0manlQ0yI2644LD/gVM8g9pX/HH+FTmk71Ps4voVzMgMcg6SL+K/wD16TbKP7h/MVOabS9lEfMyPMg/gH4NUMDzlpC0SqS2cbv8+lWqjHEp96FTS2DmuKGfun5Gjfj+Bvyp9Ga0sTcZ5i/7Q/4CaTzo8/fFSUUrMY3ep6MD+NLSFFPVQfwphgiJPyLz7U9RDLhtgVjnhhSb95wn4nsKiurQNbuI2ZWxwckgfhmpowBGoHTFGtw0Lttb+UN5+8euetWSQFLHPHoM1WS6UIA2S1Zlxrl7Zbml0tpY92FaCUNgf7QIBH4ZrKpUjBXkTyuT0Ndl/wCWjZHHTPSud13ULu6YWGnouBgyvISA690AHJHqR24zzVLVPE89xILNVS2YkHaWzIw9h2/I1nXGqSWtm4SK2V5mw5lO4tgdSTkn8a8ytipSaVPbudEKDSvIbeLMLSSK906TLZ8yaIrMPrsyFHAAA5xwPSuH1a10ay8iWzknMMpCtlcMG6gsDxjnjAHWuluNVuhF/o8toZGBACQqQD2OcYwPrXM3V3G9zuv2e9uRwCuUh/IDJz+B46HFYOTbLcUkc9Jq2rRSNGlpeuiEqrHaSQO+cUV1hksSTts/l7bYcj8Cc5/Oil7eP8pnys9qkHz0gp0v36QV9UcAYpjRhjzUgFLQBEEAGKYLdQ5YDmrGKMUANC4FV7i0juFKuoIPqKtUuKQGGug2aSb1gQHPUKK04YBGoAGBVjFGKSSRV2yC5to7m2eGVFdHGCpGQa5ibwXpjtn7Mg+gxXX0hANDVxJ2Ob0zwzZadL5kMCq/rjJ/Ot9EwMYqXaPSjFCVgepyWsaFqU9/LPZ6vdW6Pg+WrfKDjsKoLpPiSM8a7MR/tRA/zruygPUU3yl9KHfuNW7Fe1RxCokYs4ABbHX3rF1ldbjvwdOktxbuBuWRCSD3P8q6QLikaMN1qrvoTZGNJYuNPfywDKFygPTI6VQ0i81a8uil9ZxQIgyGV8kn0x6V1HljGKYIVVsgChyYWQiLha5/U9bj0zUWt5LS7kMgDK8UeV9MZz1rpAMVDNbJKwZhkii9h2RBZqwjUkdquimogXin4pXADWfqxAsyzEKARkk4ArQqrf2UV/ZyW06B43HINNMVjhtfsftaRXtqytcWmHQqc5x1H4g1csZRIIZV+62CK3NO0O102BoYIgqkkn3NZy+EbWPVReI0ow+8IGO3P0pNqw0nc3VmWGLe+ccCrUUyyLuXP41n6jYvdae8MMrQycFXXqCKbo1rc2tr5d1cvPJnJdgB+HFJtBY1R9406mj734UtAGPq4/exfQ1w0StY+LNQtpiWtNTUSJ6B1UKw/EYrtPEv2xLWOWythcSBsFCccVjT6Rc6lpEcssQivIyJUUHO1h2z9OKHG8tSoytE07D5WjGc4XGTVvVxnS5T6YP61h6Je3d1e+VLp00CoDudzxn6V0t3bm4sJouMshA+tKkrKzHVd3dGFpqKzsD6VRVBDfzKvAE2QPyNVbTWYrWdorlhb3MfyukvANWba4hvb/McqO7tuIQ5x0/wpS92PKwiry5jsYvmUA965XVrI2OsCSNf3NzksB2YV1cCkIM1m+IUUWSSHjbICfatY6qxD0dzO0uHzJQ2OF4rfuFIspcddh/lWToTLJHIVIOG7Vuuu6Jh6jFZU1Yuo7s5G3t0ukSJwCrDpUel+GZrDU1MV5ItmsnmLCqjHrjP1qXT5hG4z96JijD0PSrcl9cteP5T7YlwFwAc+tJrlvcpXlaxr6ou7Srgf9Mya5K/0u21e1ijukDKpDj2OMf1rsbkebZyr13If5Vx1zYnU9MjgEssJO074m2tx70VfhTCl8TRteGNMsdKieK0QK7HLnByfSukBrm/DOjtpiSM9zcXDvgZmkLYHtXSClHYJbinpmuK1BZC96sb7ZNzhWxnB7Gu0PSuSvlK6ldL/tA/mKqS/dsUH76Oe8J6hqkiTjUTm5tpsK4TbkY9v8816TLIfs7lDhtpI/KvNdE1m8utVuLG9tY4pYAHVkJwwz716HbSedaq/wDeXms4aS1Ln8JhaFcubiJ2dm3qQcnPP/6xUN1qt0ut3KiaRVikUBQ2BjHpTNJzHMsZ6xzMn61W1RfL8Q3I/wCeiI/6Y/pV2+NE9Ys7a4uPKtZZQM7ULY9eKydE1S4u5QJnUqybgAoHPFXoj5+mrn+OIfqK5zw/Jia2B9Cp/WlL4Uxx+Kx0OsamdPto3Cb2dwgBOAPepdMvjfWnmMgRg20gHPpWV4qUnSkkH/LOZTx+X9ak8PN+6mT0YH8//wBVE1ZJijrc381mtq0KzvGN7FDg4HGa0M/LXJSIDfXkJPBdge3WpnpByXQcdZJM6KDUYZHCbiGPQMMVdzXmemW13pHiCSxluppreSIPH5rFtv5/jXoMUpltA2fmK4J96mLd7SKaVrodFfW80zQpcRNKvVFcFh+FTO4UZrzfQJfJ8RW7HqWKnPuCK6HxrI40iMKSFaTDYPXg1t7P33Az5vd5jpt4K5oVgwryzTdfnsNIvbQSld67o2B5VsjOPqP5Vs+BL+aW7u4Zp5ZMoGUO5YDB7Z+oqHH3blX1sd2SAOabuB4zWB4o15tGslMSB55M7d3Qdsn864aDxlrcV3GJbjcJBlQ8ICt9MD+tS9ClqesUlUNG1RNW0yO6UbScq65+6w6j/PrV+gYlNI+cU6mv2PoaQDqKKKYBRRRQAlFHeloENYZBFV4STEPUcGrOKrRArLKuMfNkfjTAkpyp5iOnHTIzTaVZFiJdzhQDk1FSKcWmCdmczrFjbTTtDPDHKGw2xgDjtu59KwpdMmkVPsWozwjn5pFEowTnjdyB6YI/rXTyW7zT3N1Jvj3ttRHUZUADH884/Os2+SO006Rp1PlkEZfPzn0z71xWjK7asjVt2Ocmsv7Tu2ie4i+zxDcAifM6nuzZPU59B149eC1e3uG1C4gvH8i1DmOOR5yS0YPy5AywXgH0r0e3SG3sy8rwNcTN5kuWACH6e3QCvM/HumXt3rDraM0sD7JF2PldxGCT6nj9amFOCemhMm7XNmO+tUiRF12dVVQAqwMQB6A9xRXBR6VqKxqPtiLgAbc9Paio+rU/5yfavsfYUo+emAVhHxnpMp6zr9Y8/wAjUq+KtHPW4dfrC/8AhXte0h3MPZy7G1S4rLXxFpD4xfIM/wB5SP5ip11jTG6X9t+MgFHPHuLll2LtFQJf2Un3Lu3b6SKf61Mro33XU/Q1XMhWYuKWil7UBYSiloxQA2lxRS0AJRilooATFFLRQA3FFLRQAlFLiigBMUlLRQAlFLRSASiiimAm0UhUU6ikA3aKTYBT6KAGfx/hTqP4h9KWgBjIGHNMEQAxipqKAK4t1ViwAzUm3ipKTtQO5l32jWd9zcW8ch9WUGm2Wi2tif3ECRj/AGVArWoxSauNNoYq4FV7+xh1Cze3nTdG/UVbxRigRl6XpVvpcBht4wiZyfc1o44p+KMUkDZyuseGpZrxr3T7pradvvjbuVvwqPTdJ1FJ83t0sijoEj211u0dxSbF9KUlzblRlykAi/dlfbFcRLa+ILWRo4oLSSJThSSc4rv9tMaJW6im7tWCLSdzk9D/ALWfUUN1bRRQqCSVckn2xXXCmLCqnIFSgYqYqxTdw7Vx/iG5Sw1Eu8UzCRQcxoW6etdjioJrdJhhlB+tVfRoSWtzgLW7sp74SRRSm5Zdn+qYcZzz2/Ou4sozHaonoKEsIlbIQZq2se0YrOMWndmkpJqxwt3crpeu3ME58tZG82Jj0OaZdyLd6it0GA/deWQDnJzmuw1LR7TU4wtzCkmOhI5H0NZtn4V0+ynWWKAB16EknH51VSV7tdSYJdehpWCMlhEjfeCjIPauY0/9xqLJnGy5Yfhurs1TauKxm8O251Y3xaU5bfs3kLn1xSv7nKwXxXH+IY/M0O5Hoob8iDWfot0kDsXcBXUHJ4FdDdWwubOWA8CRCp/EVx7eG9WiOI9S+UDgGEGiclyoILVnZw3Ec4ISRGI7K2a5q+/d69MvZ1Vv0xVzQNOu7NpJLy5ErsNoCoFAFV/ENnffbY7qyijk+TYwdiO9CknB3DltJWKWqup1nSnUcsJFP5D/AOvXSWBzagelcvbLq0kyLNYxqP7/AJmcfpXW2sXlwqvoKxUnKafZGjjaLPO2/wBD8RnsI7on8N2a63xVF5ugyNjJjZW/XH9a5/xBp9ydfkEFvK5lYMpVeOnr9RXW6nC8+h3MZHzmE8D1xXZzL2ql6GCT5LHjt1J5NxDG33JdyH69R/I10ngqQw65GhP342T9M/0rPj0l9ZNwkKFmjXzEbHAcHpn35FS+HneHXbIqpLGQAqByM8H+tRNb2HF9zofHsZK2zdijD6YINc3O1ncaJpsTyulygc7kA4G4jv8AhXaeNLcyaSkwHEb/ADH0BGP54rzia1t7qxNvNH+9Rw0bdsZGRRJK0WNdUel+DXs1014LXzsowaRpcZYkdePpXS1w/gqdTeXMa9CgOPocf1rt6wi9DWQU1+VNOpD0IqibgDlQfalpqcoKdzQAUdqWjpTEJ3ooooASq7fLcj0Zf5VZqvOCJImGMbsH6GmxIfUKhnut54jjI2+7dz+HT86mpMVMlcaY7UYwVHpiuW1CFC0KswcbtwiTlnYDIA/me1dDqM94Y1W3sfNBX77ShRn07ntWBqaXVtaTzJEd/lgvtBfn0B7LnH6nmvOxNZJcsXZmsFpqc5cxNc6n5eoXKwwuP3cSnooHO5+3XOBj3NV7iKG8N55dhBcEhi08eCIsYAwO5PJ7dce9XdQgllVUuJTFBjmNdu8npgDJHvkHiua1BD9oZYdVEdogBEaPtJJGenU4GPXnp7eZUlOd1I1suhfSHSlRVaCYsAASZohn9KKpJ4USRFeTVpEdgCy5JwfSisLx/mFyPsPEjK3X9asx3Gep5qk+d2R1pyEe4r6B6gjUWQN0NShuKzVcqcfhVlJSO/HrUtFFnCE8ov5Uo2g8KAfpUav+NPDY6fzqRk6TyJysjr/usRUy6jeJ928uB/21b/GqeeKdk9setO7CyL6axqSdL24/GQmpU8QaqowLx/xVT/MVl5pe1Pnl3Fyx7GyvibVF6zq31jX/AAqZfFmojqIG+qH+hrB/xoyaftZ9yfZx7HSJ4vvMfNbwH6ZH9asL4ul/jskP0lI/pXKKec9qm6Ue3qLqHsYPodUvi1P4rJh9JM/0qVfFlr/FbTj6bT/WuRGcZpaf1mp3D6vA7FfFNgesdwPqo/xqRfEmmN1lkX6xn+lcVR1p/Wpk/VoHdDXtMb/l6A+qMP6VIur6c3S8h/FsVwOfejOKpYuXYX1aPc9DW/sm+7dwH6SCpFmif7sqN9GFecemT+dGPoaf1t9hfVV3PSuDS4rzUMVPBI/GpFuJ0+7PIPTDEVX1vyF9V8z0XFGK8+GoXq4xd3H/AH8P+NTLrGor0vJfxOf50/rcewvqz7nd4oxXELr+pr/y85+qL/hUq+JdRXq0TfVP8KpYqBP1aZ2WKK5JfFN8OsVuR/usP61IviycfetIz9HI/wAapYmmL6vM6cj5lpcVzB8WsWX/AEEH1/ff/Y1OviyH+O0kH+64P+FP6xT7kujPsdBRWIviqyY8wXA/BT/Wpl8S6cw5aVfrGf6VSrQ7i9lPsatFZy+INLJ/4+cfWNh/SpF1nTX6XkX4nH86ftIdxckuxdoquuo2LH5by3P/AG1H+NSrcQP92aM/RxT5l3DlY+jFAIPQg/Q0uKLisJRS4ooEJRS0YoASjFLRQNISilxRSKEoxTqSgAxS0UUAGKTFOooAbRilooATFNKA9afRSAYEAoZA3UU+ilYoh8lc5xTwuKdRilZBcieJWYEjkUuwbcVJSGiw7lZLVIydqgZ9KiTT4IpzNHEiu3Ugcmr1JS5UO7IJoEmhaKRQyMMMpGQRXNyeDNP84siOq/3A5xXVdqTFDV1YE7GJovh+z0YH7NHgnjJJJxWzTsUmKSVgbuJRS0VQhkf8Q9DT6YvErD1p9ABRRRQAUlFFNCYtQzjMRI7HNS0xhuUj1FOwhOvSkpIzmNfpS0hkkm46c+z74zt9q5zUkeSBAkpaISK7KuP3mPcmt+EGSJ/NH+0qen196yNQVVkPHI5rgq0HObkawkkrHP6pbi7vLXfbr5WC0mWAbjGATyQOvTr7DmsvV2mW2lng0jTxLEhMcjMG2bQcHbt9ulXri9XlLaMzSDJZYiAFPU5J4H8/aqoSRlZ7psDPEUXOc9jnr+lS8LC3NLVh7R3sjzseKYgAH1e93j7224bGfaitN9ABdi/iLT42JOUEcQCn044/KiuPlo/1/wAMV7xqMvzHjjNKq8D2qWUKjfMQp9+KRMEfeH4Gu1O5tawBcds08D2pwX0PPtQFHai4xVYrwf0qVG6ZFRjHY5oXOD29c0gLKv8ApT8gjqeKqhj1qQMRQBY68j60dOpqMMP608HjFIB3b1pcc00EZxml/KgBR1xmp1quODU65xUtlIdTuKTtSfQVJQppM+1LgikI+nFFwDr/APXoA4IxQB+VLxn1ouFgB/lR68UdOv8AKii4Bx1o+tL+dJQAAUdaWk6jigLC5pD796XHSkI9utAWA4oz9KM80h64/OmFgAzS0DrSY4oFYXg5ozikx15ozQOwuaAfek70Z4pALn0pcAj/AOtTc460E0wsO4AyB+VPWaVfuyuv0Yioh7UvGKLsVkWBf3aH5bqcD/rq3+NTLq+oJ0vJvxbP86o+9B47U+eXcThF9DTXXtTX/l6J+qL/AIVKviPUV6yRt9Yx/SsjNGfXFP2s+4vZw7G4vie/HWO3P/AD/jUi+Krkfet4T9CRXPg0ucVXtqncXsYdjpB4rb+KzH4S/wD1qlTxVCfvWrj6MDXLZPFGfen9Yqdxewg+h1y+J7M9Yp1/Af41MviLTz1aRfqn+FcYDS7gQaf1mYvq8Dt117TW/wCXjH1Rv8KlXVrBul3H+JxXB55pQ1P63PsL6rHud8NQsj0vIP8Av4KkW4gf7s0bfRhXn27HNLuprFvsL6qu56ICD0IP0pcV50GA6cfpUi3MqfdmkX6Mar655C+q+Z6DRXBpqV2vS6m4/wBs1MusXwHF0/480/rcewvqsu521Fccmu3463Gfqq/4VKPEV6OpjP1Wn9agL6vM6uiuYXxJcjrHEfwP+NSr4lk/it1P0OKpYmmL2EzocUVhL4lU9bb8n/8ArVIviOA9YJPwINH1in3J9jPsbFJWWPENmeqyj/gI/wAaeNcsCf8AWMPqpqvawfUXs5roaNJVIavYN0uB/wB8n/CnjU7JulzH+JxT9pHuHJLsWaKgF/Zv926hP0kFSCaJvuyIfowp8yFZjqKOvTmiqEMPEoPqMU+o5DgoffFSUAFJRRTEFFFIaYhTTaWkpiIk/iHoxp1J0lb3ANKMA5IzUybSutRixnD/AFrDvpLa+kePzFfaNssOcEH39q2toLlskMfeuW8Q+D4tZnF3a3j2t0DnzFBKntxgjB/OvHr46pTnapCyNVFWuiu0UMPyRRiKIHCoOiiucurvUILhrgaUWmU+UCLjeqjqTjAI9z06c9M9RFo93pdr+/upNSYAkcJGemMZOc8/z9gKqpbz3N7uNobdowMx4QjHqWA6fQjt61jPHqs+Wm7JFxhbVnFyWdh5jeZOd+Tuz5fXv/HRXXNpeiFiZbW8aQn5j9mfk9+9FcXtX3NOVHJ/GFiPD1hjOTdZwP8AcP8AjXm+gs7MnzNjcDjcfWvRPjGcaPpyc8zk/kv/ANevP/DwysLHHLf1r38vV6KuRjH+8Z1S7xjDuD7NTw8+B+9k/wC+jT0Xr3qYKMZP1r0uVdjhUmMFxdL/AMvEv4nNOF3eD/lu2fcCn7AOMUu3Bpckew+eXcFvrw/8tAT7qKkGo3ag8ofwpNn+c0nlg0vZQ7B7WfclGr3IHKRn8/8AGpF1qbIHkr+Bqt5YPB/lR5Yo9hT7D9tPuXhrj9Db9uzf/Wp415R1t3yPcVn7MCmtEP8AJqfq1PsP6xPuWpvF9lbShJoLnkZyqqR/OtGLxDaPGrbZhkZGVH+Nefa6Nl6mOPkGPzNdBaR7raLPdB/Ko+qU2y/rM0jphr1l6yf98U9dbsT/AMtWHsUP+Fc95IPAFO8n/wCtxR9Sph9bmdENWsW6XCj6jFPGpWbEYuovxYVzPkgcAUnkj86n6jHuNYyXY6tby2I4uIz77xUgliPSRf8AvoVyH2cE5Iqvd2w8jkAc1DwK7lrFvsdyCCOoNKO3tXk/2chbwKxHOTg47Ukf2q2W2Y3U65yWxK3fn1rP6m+5f1pdj1rjjtRjmvJYLvVQsxOoXgLJvUee3HJ96emp61HpzuNUujIVLZaQnHtzS+py7j+tR7Hq9GPWvGx4x8QRGMf2m7fL0KIcn8q0pvGWtrKscVyCAwDFolOfyFS8JMr6zA9S7c0Y+ua4JfFWrqPmMB9zGR/I1XXxzrPnmP7NZtlyB8rDA9T81J4WoNYmB6LTRXFSeM71Uz9kgOOT8xGaLTx3NPEXfTFTnGBPn/2Wl9Wqdh/WKb6na44pOc1yEXj2OW4aP+zZQq9WWUH+gqSDx5ZSytE1ldqy8kjaQf1qfYVOw/bQ7nWGk/SueXxnpxODDdL/AMAU/wDs1IfHGiK215JlwcZMJP8AKk6U10H7WD6nRdTQSeawIvGugTPtW9bcemYH/wAKl/4SvQhw2pwqf9oEfzFL2c+xXPHubXOeKO1ZqeIdGkj3rqdptHUmUAVLHq+myjdHqNqwPcTKf60uWXYfNHuXcn9aUVXF5ayHCXELewkBqVXVujA/Q1NmO6Hg0Umc8nNGcA0WGLnJozmm5zS5Hf8AWgBc8Z70tNzkf40E9qQxaBSE0ZoAcDg0pJxTM/Wkz7GgCTJyaN3FR7uODSk0ASE88Um7mo91NLf407CJt3/1qaXqIuKaxAosFydX5qQPVMPUgfiiwXLO/rQH75/Wq5fApC5pWC5a3+9AfjFVfMwO9HmHHNFguWw9L5nHPNUxL604S4o5R3LRkpnmc+9QGTjmmGUZosFy6JPfip45PlPSs5JM+tTROefTFKwFvzKaWX0B/CoN/uaQygDPX1ouFiwHA5AA+lI1y6naJZsnoqOwJ/I1Wjd5z8gwv989Pw9auwW6J7serHqaLt7CsluXdMW6ku1kluZiq5xGZCynPrmuoByBWHaLsGa2YjmNfbivSoRajqcNZ3kPpM0UldFjEXNGaSigQUtJRRa4GVJb31tcloJfNhYnCNjK+w9RUcuqXFs4S4te3LDI/ofbvWrL0U+hpc15ssFVhJujUa8nqjRTT3RiHxTZIxVo3Axyw5OfpVlvFWjzCILOVZhwNvtnHHetBrK1mRmnt4nDdQUGTUcmi6dJZ7ltEUGTzCF4+bkZ4+prmksSk+azt1KTjdWMo6vpu8oL1JWZSfLQ5475J44p8enw3YaZIVWZjtIB2jA6fXHPbvVS88M6WINzvLDsAJkD4Ax356VlXFjfW1havpFzdCMF1YOCrTbjhccccA85Hr714VWHNJNKx0XaRdmtrOKaSP8AtdItrFfLEiAJg9MbeMUVCuhakVGLfTU4+60SMR7E45PvRWXspdwuzz/4xOBZ6UvTMsn/AKCtcRoCfLBj/np/n+ddh8ZWITSFz/FMf/QK5Pw8Plt88gsD9Oa+wwC/cxMsX/EZ1qIOp65qUDnPr1oQZ6j61IBxmvTOEZg5NOCj0p23HWnY9RwaAEx/OjHGM89aUDIz7U4cCgBmOeKMdqfjJNOA5NFxWI8YphXA9PepSMnpxTSOOfpVCZyniIf6dFx/yz5H4muisx/osPGfkXn8BWB4jGL2L3j7/U10ViM2kByceWvf2pLcHsWQvGMc08qenNIBke/c1IopiGbevrS7e3FSY70bQe1AyPaB25qC7X9wenUVbxnkVDdg+QenUUmNHM3H35kXq7DP4ClnjEltAo6nAGfcVO8Ya5nbsqY/SoBmSSH0QqvHrWVywmVYpmyBjy8Co4kBsAMdjVi4jEl2i9gMn3xVYOUs1RR87ZA/OmmI5G4jZZSxwNzE4/u84A/St+O3WO3t1HUkEnHU1i3Z2uoPTn+fFdAozBbn/d/lSGaL4jjLnGAKq2cQeWWQjk1PJm4nEIPyjlj60tqu2aYAd+cU7gLJEu08DNUI1CWhUD5mY44rXZeOelVbeESPv/gThRSYIjhsxFEBgbjyTjvVeGAC/kOOueK1mXv61UVcXrk9x/SkwIp4/LjPy4PbNZr2obPr1+tacxLsWI46CmLC3lu5JGBx71Ldykita2KLGH43NWRrEWGb6CumhA+zr+P86wddGCT221fQkj02ITaU2eRlu1XYLcR2qrt5PQVFoLA2Mm4Zw54/AVrWqCRzIcEDoPSkMqxaaqKSQN7dTT/smDxxWoVHbOKbt9v/AK1OyFdmPdwyiEFZJBznhiKfbm8WBNt1cAgYz5rA/wA6vXKZgPHTmmwIPIA9M4qeVXHzMiW61IdL+6+nmt/jTxqmsIOL+cn3fP8AOpRGB2PFQy9lUZJ9qHCPYfPLuNk8R6+p2Rag+7PdFP8AMVNfa/4r0/w++qG4VohII9zwLhiew4q3pGn2b3BN7cLEijcc9WPoKd44vY73QpLaN0ZFUFVjPyoB2x0rJwjzWSLU5WvcwLf4ja+0UbN9lbd1JiP9DXUW/irU5I0Z47clgDwpH9a8qtjttozxlW/LmvRbFQ1pEePu1MqUL7Fxqy7myPFF5jm2hP0JFV7nxvcWrqDpquGGciYjH6VEIxjP61Q1G3DbDj1Gaj2MOxftZdy+vxCOPn0lwfacH/2WpB8Q7X/lpp10p9mU1zjW64+70phtVIIAHFHsIB7aZ1I+IOmkfNaXq4/2V/8AiqkXx/oxIDi5T6xf4GuOa0Tg7ec+lRtZoaPYRD28juh450HHN3IvoDA/+FO/4TXw+2M6mi/7yMP5ivPXslP8qztSsNthLID0APQdjTWGiH1iR6unizQXwRq1qM+sgH86tp4g0mQALqdmf+2y/wCNfPmP3HX7rfz/AP1VuW9orwo/qoODSlhorqCxL7HuC6jaSD5LqBvTEin+tSCeM/dcMPUGvExp6sp4HJ6mo308AfKMVH1ddyvrHke4h8nj60hfGM5rw4wToMRyyIe2HIogk1guFhv7xfpMw/rR9W8x/WPI9x8w+tL5nPWvFn1DWrQPnV75mCZ5uGOPzP1r2T4SSrr3hSaTUh9puYrtozJLy2NqkD9TWVWl7OPMXCspO1ibeOgpm/nj9a7g6HprHm1X8GP+NRP4c0xh/wAe5H0dv8a5vaI1ucekgPepklwf8K6M+F9PySDOv0f/AOtUE/heDyX8m6uFfB25KkZ/KpdRFJmG0x3BVyWPQDmporVnIaY9OiDp+PrToIFgXGMt3J6mrK81W5WxIijgYq3CvNV415q7CvIrelT1MZyLsIworTgbMVZydKu2rZUivQgrHDJliikNFaEBRRSUALRRmkqkJiPyh+lIDnHvSnkGmRn5B6ik0CJpXwAgOB3qe1O63kXuOapn9as2LfvGXuV6ViqSjCw76mPq8DOQ+AyKufLPQn1Pr9P/AK2MGae7nLy+cwVG+YnoBzx9c4rqNThaWJlLGNOQSvX/AOtWNaxRXFpLaNiMCRANvfJ4rwqvuuT5dE0dyfuopCe8KgjkEcHc3P6UVea21gsTGyKmflUSAADsMUUvaw7IV2eQ/GRiZ9JUdQsv/slc5oAwlt6bv61t/F2bzrrSDn70Lvj1yVrG8NjEdr3A6V7GC/gxMsV/EkdavHtj9KmAHH6imooxT1GDn+leicKFxxRjHvTufWgZx9KBjQBmnBR6daXHNGOKQxCBnqetLjOD607b2HWgLzTQhCM0w9s9DUuPpTGHpTJZyniMYvIjx/q/6mujsB/ocHT/AFa9PpXNeJm23sJ/6Z/1NdPpw3WFuf8Apkv8qS3B7Foenan4+uaaFI4z+VPA49cUwsHIzR1FOx1PB9KNuRx0pXHYTAbtUN0B9mP4VY9sfhUN2ubcnGOQf1pPYZgthZLkngAD+VRlRFbwE4GW3GlnDNLKnY/MeOwHSnTKrRW8fQtj8sVkaWK8YLTl2z86k49B2pkEQETS46qQvsKtTMI7hiSMLHio0AGnnjHymmhHH6iux075zmt2ED+z4GPQKOv0rC1RHW4V2ztYjaPbHX8a1rMvLZL1CRoB+NDEjatIike5vvvyaLcf6VMKmhJMSNzyBUO8Q3UhPHy5/lSbGLOxdxCnU/eIHakswQsg9GqS2jJBlcfM9JaH55R70rjJdueDVOfakjAck9fYVclk8tCf4jWexyx9TSkxpDVjMjgD+VW5IwkBHoKfBD5aZI+c0s+PKY9OOKEJlW3H+jDI9axdcXKnjOUrbg/1HJ9awdek2ywxryWyD9Ku+hNiHQAziVBwobcfy/8ArVvaeflbjnisTw02XuUz2U/zrcseZH60uoy6BznFKRk04Dj2/nQeOcc1VySvKuY2+maitV+U9OuatOMofU1BGDDHlgcnnHpUNlJBI20YHJPTFIsQU5Y5Y1IkZ5dx82Pyp5AJzjNNAyFhVLUY99hcL2KH+VaJHyjPSq1wu+J1xwQRVXEefWozanHUPnp7V6Lo7B9OhbjG3v2rzm2zsmRuxH9a7/w44fSo+nHFZSNYG0Bxz0qrfp+6BHUNzirQ/wAiq96pNufYg1BZmFecnp3ppUZ44OKk56/1pvOOvJpk2I2GevSmFOPXJqU/mBTT/nvTuBXZevb6VVu4t9lOg/ijbH5Veb6VEVBOMcHiqTJZwQ5gkHuD/n866PS/nsYWxggY/XFc8g2iZOny/wAq3dEbNmAeqsRjNOWwkagTI6Ch09PrzUsalzgDmr0Vrt5cc9ayuWkUIrJpW+YYX1q2IFjXCgCrm0AYH6VG64BPb1ouVaxganHiXp95P/rV6N8B7rNnrVqTyrxSgZ9QwP8AIVwGpphYyB1DD+VdR8D7gxeLdQtScCa1bA9Srr/QmssQr0mODtNHvZOcGg0g6Y9KWvIudYzvSHkGg8GkNQ2Uc3ewmO6ce+R+NRqvNampxZKSf8BNUAuCPfmuyjrFBJkkYq7CMCq0a1bjGBXfTics5Fhat2xw2KqLU8JxIK6EYMu5opKKskWikzRmmJhmlpKKYgzUa8Fh71JUfSQ+4oaEKaltW23Ke/FRGhW2uG9DmpktBk9+uRIvfrXD3MLPe7SSo7kdQPb1Nd7fKqgzEgLs5J9K4q5R5LlpJI9kefkQjlh6sD/L8/Q+XVdp27nZSd4Dc3X/ACz09Cn8JMRYkdue/wBaKYwYsSZHJJ9aKy9kyrM8i+Lv7vVdOjBO1YG4x/tHH8qy9EuVggtWMbMFQZx3rQ+MDA+IrJT2tv8A2dqy9MAFnBnBOzr+Jrqw8uWhFk11erI6hNahxkwzDv0H+NTDWbYgfJL/AN8//XrDXnp9KlH6YzW31iZl7GJuDV7U9RIP+AUv9qWmc72BH+yaxDx3pVPbNH1iQexibn9p2ZH+t59wRTxqFpx++A/AisIdMjrR37U/rEg9ijoBfWp/5eEH1NPN5anj7RF9N1c7wB0ppAzjGKf1l9ifYo6Q3MH/AD2i+u8U0zxEYEiH/gQrmjg88flULgDnj8qpYh9hOiu5F4slRb6Ahs/uzyOe9dZpbqdNtiGGTEv8hXm2suEuY+3y1vaaF+xwHGPkH8qp17K9iVSvodyF9P51IFPrXJqfQkfialUuD/rJPwY0vrK7FKh5nUgHrjn3pCMgY9ea5pZJs/6+X/vs1ILi4B/18v4tR9ZXYPYvudFg96iu1/cH2IrGW7uf+fiT86Gurhkw07EehxQ8RHsHsGRsuXu2I5AA/SoEBeaGQ/dBCqPoKny5LfPy3B4600bxtAbhORxWftUV7NjLiPzLph1Crk1FKfL00Kf4kIHPepxu3M+RuYYJx1qtPGSgU4wowOKaqoXs2c3ri7Xt8enX8K1bBAdLTAHK8/WsfxA7i4jDFdoUYGK09JkdrCMDG3BGMVbmrEqGtjZsyDbRn29abIA90sfQdz602BniiCKFwvrSpGUkWQHJBzz3qXURSgy5joB17VWtyFuZgT71J5rcZQc+9REEs7AYLe9J1ECgyKeUu2c8dqbCB56cf55pxhJOccd6eR+9V1XAUYxUqauPlLYHI7daimJ8p8eh6UpnGMbWpjSqQRtbnPatOdEcrKqy+Xa+jZwPesrUYCsPmONzkH8K1QF8zc4JUD5RiqGs3MSwAnPXHK+1OMkJxZleGW/0ycZPKf1/+vXQWX/HzIoJ/H61y/h6QR6s+ehUjp7g109uwW5LfwnPNNtIFE1OR/hSZ4xmmCVDyD+GKPMQDr09KOZC5WKx2g889KrwtvnO7qBT12u5Z3AC9BUUOFuskYXkZJqeYqxsWOnvdAuwKxDuB94+grVGnIBvZOeylD0p9vLarFGxki2gYVcjn681R1HVFkDRW7jn7zjj8B7VnzOTsXZJFPUGgRzHCEb+84HH4VlSLuUgg1YbDDsKhZeuea3jZGLPO9nl3t3GR0JA46c12XhZy2nnnO1v8a5K/URa7dj1JP8AI10nhCTMMq56H/D/ABqZFwOr71Fcgm2fnoM09TikmBMDjGeCOKg0MnnB9qQ9fYUdevSgn8qLiGkGmHvwSKcf/rUh75P9KAImHH+NRnjnP5VMRxjH5VAeuRxTTFY4yaLZqdxH0+ZwM1qeHYmkjlQZ4YH86i1K0K6rPIeFBDAeuQM1a8NNtubiPIyVB59v/wBdVKWgorU6aCJYx8o59anGAP61GuAakHB96wubLQXtUcigjrz3qXIxg9PrTGOeKaYmY+qL+5BGeG5/KrfwvuPsfxNshnCzb4z/AMCQ/wBQKh1Bc2zjv161n+H7j7F440W4yFAuoST7bwD+lOSvBonZo+qDw31pe1EnYijORXiHZ0Iz1pD0pW600nAznArNlIhuo/Mt3HcciskCtW1vba8lmjgkEnl43EDjnPQ9+lUHj8uVk9DXdhdrE1NBI16cVaSokFSpXpQRyTJl7VKhw1QipVPIrYyL1FNQ/IPpS5qkSLmikoqhC0UUU0Ji1G/31980/NMl4UH0IoYhTTTTj0pppDNFsSWsTEZwP/rVh3cCtdAFFYsQBuOK2rZt1lj+6cVlaomORXmYuN2deHfQQ6hZIdggiIXjOOv6UVjhJMD5M/nRXF9XX8xryI8L+LrhvFEC9harx/wJ6z9NnhSzhUyICEGcsKufFaT/AIq9RkjFqgOPQlq4piRI2DwAOa9GjG9GK8jOs7VZHcpNCQMzR/8AfYqZJoiOJFI+tcADjjP4U7cT3xjtVeyJ9oegBl45H509WGe1ee+a27KknHpTvPk6iR1z6MaXsg9oehD6Zpe4rz77VOowJpPYhjTxe3IU4nlx1++aPZsPaHf5Hemnt6muDGo3isCLmYen7w07+1L0Di6mI/3jR7Nhzo7Yn/IqJzlcAfhXH/2rfZH+lSdu9KdXvQeblz9QP8Kagxc6J9eP+kxeu3ofrXRaXzYwdT8g/lXF3N1JclWmfc44yRV2DWbu3hRI3G1RgAqDgVUotpIUXZncJUy+mOO3NcSviO/Gf3sZ4/uCnDxPqC/MTF+K/wD16z9my1NHa9sU4+o/UdK4seKb4Y3LB+Kn/GnjxbejgxQH2wf8aPZyHzo7IHBHH6U7P41x48XXRwfIhPr1/wAad/wls2ObWMn2Y1PJIOdHWkgDGaM5A6+tcp/wl8ne0Q/Rz/hTh4t6k2g44/1n/wBanySByR0+7I9aic+lc6PFy4/49D/38/8ArUp8Uwtw1tIM+4NPlkLmRV8RsVnjY88Vd0Ns6dF17/zNYer6kl+VaNCMDByas6Xq0VtbCOQMSCeg96tp8pKa5jrY2AGKlz3rCTXrUcNv/Kpl160PV2HsVrOzLujYyM8A0bvSsga7ZMB+8Iz6qaX+3LM/8tfzBpWYXRqlunIxRkdulZY1qzJ/1w/EGnf2vZ8ZmUfnRZhoaWRznA/rTSe1Z/8Aa1mQR568fhQdVszj/SE/OjULouOeKydYG6yPHRgastqVoc/v4/8AvqqGo3cMltIFlRiOwIqo3uSzF0ZtusLweQw/SuziI4HT61w2mPt1iJmYbdx5J9q7OKQHHI6+tXPcmGxfB7nNBHHPFRq3HBBz75pc547+lTcsdn+VJwM8A0mc0h6UrisOJX0FNIB7Um4delIDTTGIQmTx9ajZVHf24NSMT1/SoZG46002TY5HWkSLVJSAQSAePcVZ8NyNicKzLgjkHHr/AIVH4hGL1CehQfzNReG3xdzoTjKg/kf/AK9a/ZIW510csuR++kxjPJqUSysMeaxB681VjP8AKpwc9xnNZ3NACAcZORR5QHHSnL+eKM/jSuFiMxZH3qQxlT94496mOcDPNIev+TRcLFfyuPf6U6O3GeeakJA6/jTkyM880OQJHPa9Ftutw6tF/iKz9Ck26p7OrD+Rra19QTA5GRyv8v8AGue0tymqQMT3x19aqOqJa1OzBx0P5VKv0/Oq6tx7VODzzjmszQeDjp+NMZuOn60pYZHb8ajY8HP40XCxVuzuikX1Q1y1zMY57aVW5Q5B/KuslUE7T0PFcffLtiQtn5Wwf8/561pF30M5I+vYJlubOKdPuyorg+xGach4IrmvC2t2kfw+0fULy5jhiFpGjO7fxKNpHucg1h6j8RbKR3is5Wjj/wCehQ7j9PT+f0rxnCXM0kdkNUdlqGq29iMMd8vaNOv4+lcpqWszXmRI22M9Il6fj6/yrmJfEto5OJTycknOSfeqkmu278+bx3pqjLsdEeSPU7rwjd+Zq08JOQ8W78iP8a6O+jxMrjow/UV534N1eB/FFrEsgJlDJ/46T/SvT72PfBu7qc100Vyy1MKzTehQQdKkApiipBXoxOKQ5etSCo8U4VqZsuRHKfjUlQwH5CKe7rGpZ2CqOSTwBVrYgfR3qKK4hnz5Uiv/ALppzSohAZ1Unpk4pgPozSe/agMD0IP0NMQ6mSjMTfSnUHkY9aYhqnKAjuKaabF/qgO44pxqRl7TmyJE9QDUVzGpXc/IXkg96LBttyAf4gRUt0H2soQH1+bFcONXu3NaT1MVruTccPIBns2P6UU4xS5+8B7Bun6UV43NDzO/lifNnxQfd40cdlgjH9f61xzDErZwK6z4ltnxrc9OI4x/44K5UfMzA859K9uhpSj6HLW/iMASDQfqadtBIpcdBjr3PetTMbuPt0pDnOfWnlcc5xmjHzZ7mkAwHHr+PFGOOAak2Z+7/KgLz60XAjz0xgZ9aXtjnFPVSc85BOMYpAvTIPv7UXAjJ5OOtGecgAZ7etP24GD+RFIVA5yOO1O4rEbcc5HXHXpUoPyj2GOtLb2st7dQ2ttHvnlcIiA9WPQV06/DvxPgZ0zBHX98n/xVKU4x3Y4xb2OW3YHH5005J6V1q/DjxQyknTlx6+fH/wDFUH4a+KWP/IPUD1M8f+NR7WHcv2c+xyeec8AijP4jtXXp8L/Fbgj7BGD/ANd0/wAaU/DDxWQB9gT/AL/p/jR7an3D2c+xyKybCSQD9RTS5Jz0rqn+HHieJyp0/JPYOpx+tIPh34oYhRpp3HuZFx/On7SHcXJLscvu57fhQXH4d66d/h74nViraa3B7Op/rTD4A8TL/wAwyT/vpf8AGj2kO4ckuxzSt2z+NAcZBx165rov+EC8TAZOlSe/zL6/WkbwJ4kGf+JVJzjoV/xp88O4uWXY5tiQPb+dPjY7SQeK328CeJW+7pUv/fS/40+PwF4oyd2lTcejL/jT9pC24ckr7GBvxznjuKBIQO4xx1/z610f/CCeJSD/AMSmbp/eH+NNPgPxJk/8Sqbkccr/AI1HtIdx8kuxz289z9OKUSFV684rfbwJ4kI50qbP1H+NJ/wgnib/AKBMx98j/Gnzw7i5ZdjA83pwc4xilEpC56Gt8eA/E/I/suX/AL6X/GlHgLxNg/8AErk/FlP9aOeHcfLLsc9vOeaaZDjnHPJxXSp8P/E7kgaVIOOcuo/rUTeBPEm7H9lzf99D/Gjnh3Dkl2Of80sfx+lIZRwM8jv610B8CeJhx/ZUxweuR/jSf8IJ4mOP+JVNz05H+NP2lPuLkl2OZVyHHrmnGQ9Mmug/4QPxP0/sqYY9x/jTv+EB8TY/5BUn5j/Gn7Sn3FyT7HPCZxyHI/HFO+1TqQPNfj/ardPgLxN/0CpRx6j/ABo/4QLxMTj+y5c9eo/xp+0p9w5J9jFXULoLxczc+jn/ABpw1O9A/wCPufA7eYa2G8B+J++lS/gw/wAaT/hAfE+f+QVJ+Y/xo56fdByT7GQNVvVb/j7uM56GQ1IusXwGPtcvt85rT/4QLxPnH9ly/XI/xpf+EB8TD/mGSf8Afa/40c9Pug5Z9jN/tvUOn2uT8e9OOt3/AEN05x64P9K0l+H/AIpbONLf/vtf8ac3w+8UI206Y+R6Op/XNLmp90Pln2MC6vri5IM8hZhwCQOn4U20vJrOcyRNhiCDxnit9vh94nBUf2Y6hjgEuMfzqVfht4oLAfYBz/01Uf1p+0p9xezn2M2PxHeoP+WTY65X/wCvUw8UXg48uEn/AHT/AI1bb4ceKR/zDh9fNT/Goj8P/E69dN4/66p/jS5qb6j5anYaPFVwOsEP05/xqRfFUmebVOP9o0f8K+8UZx/Z/PtKn+NQ3/gvxBpllJe3enmO3iALv5iNgZA7Glem+o7TXQtL4qYfK1shPs//ANan/wDCTqRzafk//wBauTDds9qcJD2xmn7NE87OsXxNEcZgcZHZs1NDr8DSY8mQenIrjw5zz+VWbaU+cOvPH6VMoaFKR0Gq6hFdQoqo4YHOD9KwI5PLvUfHCyZOOe9W5GyvTPpms2UkSv8AUdKVMJM7BNSgx1YfhUyapbYA3N/3zXNo3yZGee9TbuAeMVm0UmdF/aVtgHf39DSG/t848zr7GueD4XuD/Oq09/FBxnc3oMUKLew3Kx0sl3b7c+aAPWuS1S9haaWKI7gz/eHTr2qlcX01wTlgqk52jpVU9a6YU7bmM532N/S7jXtVRLG0u2aK1Q+XE7DagLZOAfc1pPo/ilQSTE3/AABf8K5W3u57KdpLeRo2PGVPatFPFGsIu37a5H+0Af6Upwlf3bFQnC3vXLstr4kjPzRr9Qif4VXLa2n3ox+KrUDeJNTcfNcZ/wCACoW1q8cYMg/75FJRn1SByj0bOj8Jalqdp4x0aWZNsYvIlc4/hLAH9DX1Yyh1KnoRivjCHWLmOeOXeco4YY9jX2bbTrdWsNwnKyorr9CM1z14tNOxpSle6MwLglT1FPFS3SbZyex5qKt4O6uRLcUCnCkHWlrUzZPAe1Vtatp7rT2jtwDJuBwTjIzU8R+cVLPJ5MDyYztUnH0q1sR1OehgOn6hCqOxdgpkQtu2k9ce1T3llcT3Vwzs8SAFllGCOnAwar6NLFNK95dSr5hOQDV/V7gsI4I2+V/mLZ4x2pK1inuQJcyto8nzEbXUZ9jjiq9u0lpdQzeaWSX+HA4wcEe9afkRDSpYIZEdthJ2kEk1kuwksoSDkrIR+BGf6VRJYvvtE97OkcgV05QEZHAzitTSblri0+dizKep7is+V9uqQSdBIik/yP8ASptFJjmmhPbgfgcUxGoOGce+aDyaDxMfcUGgCSBtlwjdgwrVlTLH3FYtbhO9Eb1GawrxvFlRdmZTRfMfrRV0xjJ4orxeQ6faHyP8RufG97jP3Yxj/gC1zeCXbgn5utdT4/iabxpevEu9D5eGXkcIvpWLiGMvvQh93rXqUnanFeRNTWbKew4zjPtSgZ4P5Y9qurdQINoTGfcUCWy67G/w/Wq1JKYQnHXigIQQcE1cEtmflUNycDk00SWhIxkA8UaiK2w5G0/pSgZyOg71o2tvHdIzp0BxknvU/wDZygHp+JqeYqxjleTxz0pPL7dxW0umqD8wwM4+8KeNLhPBzjt8wo5h8pglQeg6DpSMhXqD/Sug/s2DByzc+4xTG0q2Yf61h+VCmHIyt4TbHjHQ845vY/8A0IV9WBI2wSi5x1xXy9otutr420NFYsPtkRycdd4r6eD4Ue1ZV1zNM0pOyY7yogf9Wn/fNKY4j/Av5CmtOpG3bz/eppc1j7M15x3lxZ/1a/lS+VEesa/lUfnBOSCfpTRLnmj2Yc4phj37gi5+lKY0/uj8qTfSbqagHMSCOPH3R+VOEcf91fyqIOPWnb6OQOYf5af3R+VN8qM/wj8qN+aTzKOQOYcIowfuj8qcET0H5VHvpd9HIHMPCIB90flQUT+6PypnmUhejkDmHbFB6D8qCqf3R+VMLn1NJvp+zFzD/Lj/ALg/KmlE/uj8qbvpN9HIHOO2J/dX8qjkRNv3V6elBemSOMelKVPQamORU3H5R0pCqbVG1fyqNHGR7ikD8is/Zl85J5abD8q/lTPKTZnYMn2pDL8nPrQz/u/ej2Yc5II48fcX8qYY0JPyjJ9qTzPlzTRJl+e1P2Yc4540AHyj8qUxRkfcX8qZI+QPrS+YCAaapicyNoo/MHyDp2FMeJGOAq/lTnk+bPHApitxk5yTT9mLnI3hjUgBVx9Kq6leWenWpknwgPyqQmT+FWLqQQxGRs4VSxx7Vy2k66dS1BJ5OoXft3cKD0FXGlcTqGsLyW+YjT9Jnu2XpJKxRF9hk8/gMU28fXbKDedDWQdT5bq2B9MZrsbOePylVFHTt3qW5mKJyACe1WqMbD523Y5HT9QtdSjO2IJMo+eNgAR+FTPbxscCNffjpVm9trcM12sCpLjJlTqcdjTIvmQOP4uazUE3Yc20QrCgfBVevpWf4rtY7jwrqsWxf+PSQgbR1Ckj+Vajy4YqPp265zn19qi1FBPp1zFjPmROv5qRVxp2Zm53R8sYGf8AClGO5xjilKEZ/XNLtLNwO3YV3HGABzwRx+tSwfLKp9xyKaqnrj86eAByADzmpY0aDkYwep55FUbknzgfb061oOBtGOh68Vn3vRSPT/P9azp7ly2LsBJiQj0HINNmnSJNznHp6ms8XpihVFUZHc1VdmdtzHcferVO71Icy1cag8o2x/Iv61Sz70UVqklsQ22FIaWlamIQ0lL2BoIxQAlFFFABX2D4Avf7Q8AaFcZyTZohPug2n9Vr4+r6f+CN79q+G8ER62txLD+u/wD9nrnxKvFM1ovU7y8TKBvQ4qlWpKm+Jl9RWZ3qaL0NJrUWlBpKWt0YjlOGzVllDoVYZVhgiq3selWIzmJCepUZqkQzOXw/YJ9xZV/3ZW/xqe40uC4WNS0i+WNoKsRxV2iqshXZStNMjtJDIskjEjGGOagbQofOLxzSIhOfLGCB9MjIrUpadkF2UrzTI7uKNQ7RvH9x19PQ+opbDT/se5mk8xzxnGKuUoNOwrkcgxIh+opSKJfug+hBpTSAbWvaNvs09uKyDWlpzZgdfQ5qJq6GibFFPxRXD7JGlz4cRzGqgMR9DQXbcQwIIJzwOKigDMiDPPtV2HRNUup2EMeWAzy4rsUbsnmsVxKf9rnpTllx/CRz3FXf+EV1wD/UAD/rov8AjUi+FdbJI8lOn/PRar2b7C513M4S5A+X/wAdpfMzwFOf92r3/CLa4Bxbrz6Ov+NIPDeuKP8Aj34/31/xpezYc6KqzOBsUuB1wq9fU8fhThcz5wHm9OCa3NM8NX7wubm1y4bj5x6exq+3hmcEFLMj2D/T3qHFp7Fp36nKrdXLOMPLyMd6mWS+I+UTfgp5rpE8N3SsCbV15/vj/GpRpF7GSEtW46dDUO/RFxt1ZzAGqsOI7kj/AHSf6UjnVYhl4bkZ6kxnn9K6wW1+gINoT/wH/Comh1DdhreQZ5+4QKlOXY0tDuY/hqS4fxdpAlDgi8iHzLj+MV9Ng8V86WEMkXjHR2ljZCbuLqMfxivoodBxzRNXsKDtcUmk8wCmnOOBUZyTSURORJlSc0uVxUADMeBk0bucGq5Bc5PlaXK/SoA1Juo5A5yxvWjzF9arbutG7mnyIXOWi60b1qru/wDr0bsfSjkDnLW9aN61V3cUu+jkDnZZ3rSF1qvu96Tfk0ciDnZZ3rmk3r6GoN31pC9HIHOyxvX0pN61XLe9BanyC5yxuX0qOV129Kj3n1qOV8gUOA1MkEibl4/hpodNucd6r78BW/2TQG/cn61HIh87J/MX0pTInl9O1Vt2dx/Cjd+7x6ijkQ+dlhpUCYxjNIkibuh5FVWYk+3rTgwWUfTFCgHOyzNIu3pThIpA44qpK3yU7fhc+gp8iJ52SPIpIXb1pwZBjjgdqps/II6Z9am30+RBzsfOUcAFRg8GvNoraeDxBb6bDBNCPNJZnTaSgPBGRzmvRJG4BAHHY1Unso4pEvowi28IwEUcuzH5iT6jg/hUy0RrS956l+eOYWojg34bhismw4/3u3865rRtOnOtSHZDGm75zHcPMSvuzd66h71LS3eQsGCLnav8R7ViPrRSN3j1CwinzxAyhlA+gIOfesU0dai9ypqkN5FrckWZmVDgGOfbhTyCV6HniugjISBEGPlUCueXWZr2fzrqO1LghcRHOPfNdCqkoG74FOjZtmeKTikV5Tlzjil6jB78VIYGfc6cqo55/wA+lJj5PwrblORM8Qm0vQ4ruaJ2td6SMrAyAEEEj1pV07QO32UjrxL/APXrP8S+GJm8T6myMQr3MjKNhPU5/rWSfDFznhuv+yahxj/MaqUv5Tqho+glRxBtPTE3X9alTQtCckAR8df3xrj/APhGLofxj/vk/wCFMbw5dr/EPyNLkX84+d/yHdnSNExgAE4xxKTXmk3myXLx5LFSQAPb/wDVVx9AvFGcrj8f8KzJY3gnaJ/vKcGtaNNRvZ3Mq03K3u2NW1sraXTpPNt7gXe/CNn5NvuOvrQdKjwRmQH6j/CqdhY3N95nkMP3YyctirH9i33H7xf++jVt2e5mtVsbumaDod7LHbsb4TMvJyoXIHOODW2fh/oythrm5U/9dF4/8drio9J1RX3RS7W9RIRU62GujG28kHGP9eRisZKTekzeEopawOw/4V5o7Di8uh/wJP8ACsjxR4LsdF0Y3lrdTSOrqpVypGD9B9Ky/sniQdL6fj/p5PH60k1h4iuYDBNdyywsclGuCRn6E0oqaabnoVJwcWlDU5zHy/jXqOn/AAu0++0+2uxq0w8+JZNojXjIBx1rhh4Z1TBHlrjP98VrQQeLreBIYb6dY0G1EE/AHoParrScl7k0jOjFRb543Omk+EluPuapMT/1xB/rUDfCOTkjUZAPU23/ANesR7nxvZwyTfb7oIiZZhN0A61mt458TM2W1i4Y+5B/pWMYYh/DNM1lPDreDR05+Etx/DqOc+tuf8a9Y+Emg3HhrSdQ0+abzVadZlOwrjK4PX/dFeNab4g8f6lb/aLG6nli3FdwjjxkduRXp3wo1DxbNrl5D4iMjW7W+YiyIMOGH90ehP5VEvbXtOaY/wB01eEWj1ys2dNkzDtmtKql2n3W/A1rSdmZTWhVxmnCiiupGItTxnKfiagqWI/KRVIhktFFANWSLRSUUxDqUU3pS0wElGY29cUincit6inmo4+I9voSKTADV3TGxK6+ozVM1PYttul9+KmWw0aeDmipNtFc/sx3Phi2X5ohkfeGB+Nd3og33koGANmefqK4vyHtrxYpUaOWNwrKy4IOe/pXcaEo8+ViRwoGMe9dNPciextFCo+8CDxgGmjhs9Oc1Lgc5YfjRtHJ3D866DAiIOcY/WjrjipWVeDuH+NMKj1H4UAWbPGw+zCrJIJ/lUFooKMM55FWShGOKzluax2Gd8/570hyDkYzTyMgDFNOAen+f8ipKIznGaRueOMDtT9pK8g9cc0yRxHGzuflAyaQzm9ROPGvh4cZ+0xk4/3xXuykEV4HOr3XjTQyTsaS5jG4/wAPzjFe4rHcAf67r/s1zVHdm8NEXTKxjMYxtPWmFeCKr+VOf+W36U/ypsH98fyqUDHrK0GSuNx45FVtxJJJ5NMkhn3czHGfSomik/57tWiM2WN/vRvyODXKeJ9UutGa0ZLlwshYMMDnGPb3rBPjG9z/AK4kd+n+FFxpHo5fk0F686Xxfc952z9B/hS/8Jddf893x9BSuB6Jvo3jHWvO/wDhLLnj/SJfTjFH/CU3P/PxKKOYLHonmYo8z6V53/wlN13uJfzpyeJLp/8Al5l+u7pRzofKz0LfSF689fxPPk4uJv8AvqmjxJcHn7RN+LGlzoOU9F38UheuHg1yR8bppv8Avsirg1MuARPL/wB/DUuqkUqbZ1u/PSkL1zEl6yxqfOk5H/PQ1GLtmVj58/H/AE0NT7dFexZ1W/nGKbI3y5rk2uXfhbqVTjjMjEfzpkF1Luw1zK5H+22PyzSeIXYaoPudKXJRfypd58tuvWueW6fGfMbk4yXNMub5kYgSt17Oaj6xHsV7BnSK5O7r1oVm8scGuT/tJgf9ZIf+BmnrqZK8Fj77jT+sLsL2LOnUsX5HA60rORKCAcVyU2pkDBZj9GqodSbPLt9Sx4oVddhOkzuJZDs7/hTTIW2oB27VxB1N/wC83/fdRnU2xncT/wACq/aoXszupHI2ipd59D+VeenU2yPr2NR/2r1GV6/3u/4U1UF7M9Cmf5QMH6VheLr7ULHw5Je6d5onjKqwU/KUbg7l6EDrzXMf2qWA2FMA8nOc1W07VTP4nsoLiPNuzunzE4LlGwfbB/ofprTXPJIl+71NnRvGkKSpp2py7hkRw3UqCMS4HRlBIVvxIPseK7JILeSB5bcWyyNks6HYxP1HU1474i0B7JftAWS6sJm+4+A8LZOcuePxPX19ct9cuINIll0zUTaCGTy/ss9yHcjpkKVPH0bjHSnUw8ZXlB/I1hinC0Zo9M8Q6hYeGLWbUnSGS9kO2MMxZpG7AknOK7PTbmS80m2uJ4BbyzRK7xE52EjOM1856bfvNrUWpa002pwWkqlwZTgjPGMjpnGRxXt2meO/DV+FRpms2PQXSBVP/AhkY+pFZQpciCtX9q/I6CSRkwsbMMjBx3qPcdmKe1pHNGGhVHRhuBXGCPXPTFVltEBIKAEHBBFUzJHF66Fj1y5wAckEexwKoYI+YqME8ZNbXiO2EerjaAoaJT2HqP6VlLgdF3dgT0FebU0kz06SvBCLtCdCDjpUqqzfM4IA4BJ/lSLEW9efXpU4gZsKCXPsOKybN0iPfGCp2MQvqePrivKfFyY8TXTdd+1s+vyivZoLWSH5wqBhzucBsfnxXmHxJjf/AISGGduTJAuTjGSCR/hXTg5/vLHNjIfu7h8OU8/V76zzgS2x7+hH+Nd//ZDrPHKkiEqwbAXjO0Aj+dec/D9tvjOCPr5yOv8A47u/pXt8FiXYfLkDrWeOly1B4OCnT1Maw08R72nij8x5C25RnAwBjmtODS7aRG3QoOOXZRj8PWtRLQn5VACnsvQ/41bSzRAN2Sf7orzZVjuUIx0MZNFtJDxAhGeWMYqdNEskbBtoHOOMwittYScBQRjpnrVmK0I9j+prL2smJyitzGi0OwyWksoST2Ma1ai0OwPSxtsnqTCv+Fb9tpbyHO3j16VLPPpul/6+UPL2jXk1rGjUkrt2RyzxEb2irswJPCVlfWc9v9hgPmxsnEYHUHvXyrq+kT6Pqs9lcgLJE5jYA5wQcGvsX7Rq2ppts4BZwHjfJwfwH/6q+UfHttNYeM9RtbiTzJYrhgXP8XPB/EV62AjyNpXaOKvJyV5Wueo/CDULzWfCdxotppNqRbzDzLjaASG5+Y85PB/SvUtH8JNpUjXs90zzYPyINqDPt3ryj9nLUhFc69p5I+ZYpl/Asp/mK92muNylaK0acKjb3JjVny8sditUc67om9uakpOtaxKZm0tK67XIPY03rXWjnYo6VLEecVHToz8wqkQyejNJmjNaEi5pabmlBoAWlBptKDTEOzUUZxJKvuD+YqTNR9Ln2Zf5H/69DAc1PgbbMp9DmmGli+/SYHQUVCko2Ln0orMZ8RySySgTSyM8hAZmc5Jx7/Suz8NOJInlH8SKa4x1/wBFGMZ2c/lXT+DZN+mMc8ghP8/nWlPcmex1O4+uKQt7cZ+lR5OcDrSE5GfTjHrXQYWJlbKkehz0pueuO9RglSeOvFB4buaQy/YyYdkx15H4f/rq6WHTuB1rFhk8qVG9D6VrjpgHI9fXpWcjSGw9WAPTAphwc4HvS8FR/Kg9ep5qSxB15qtf5+yEY/iB/WrBGMc8VXvMG1b04/nSexS3OZL7fGegt6XUf/oa174or5/m/wCRt0Pn/l5j7f7Yr6CUHFc8jRDD8uAFyfX0pecGnlaCODQhMiRow375Sy+gqqRycDjPFTkcmmEVoQ2cH8Rbd54tOVWI/ePnH0FcQNKnOMvJ+letaxYLf3NnG3RSxH5Cki8MxFg23j2FHJcXNY8qGlSn+OT8Mf4VKNLkA+9J7f5xXrJ8Lw8YU8e1PXwxEP4Kfsw5zyUaXJ1zJ+OP8Kd/ZcvpJ+f/ANavXV8NQA5CU7/hG7f+5S9kx+0PJYtKkZhnzfzqzNo0kTFY1kK47t1r1NfDkA/gq03h+3PzFB0qXSdylUR4u2mTDqkn51LFpMhYfI/5162+gQZP7uhdAhH8FP2LF7RHAWHh+OQAusv/AH2RW0nhuzOB5Unv++b/ABrsbfSI06LirX9nJn7v6VhKhK+5qqyOPbQLFFQeUzd+Zn/xpr6HZrExEEgI4x5r4/nXWz6axA2r2xkV4bqdl8UTqV0tquoG3ErbCNmMZOMcZrP6vLuX7ZHdHRYd5KoyjH/PRv8AGq/9jxliQDn2dv8AGuFXSviVKy+amqnb833kH+e9TvpvjtruB1ttVt0Vhv33O4Nz3z2qXhn3LVddjsU0yJcfL0PA3Hj9ajvNNUNjbkf7xrpEgJjO5RnpTbu3AzlQAKx9i7mvtFY5JdODNjy8/wDAjVpNJhKD9znHua3FthuOAMhatJbjyxx7UpUpAqiOOudKUcCIA+9Z/wDZq7gDEvJ/u121zbAyfhVFbQSXWRyF561UKUiXNXMIaLDtG6FCT/s0f2JD/wA8UyO+K6pbbdMMjAA4qlrkv9maRdXS43RxnBx/EeB+taKmxOSscbeSaXYuY2eHzAeUABOe/NY1zq9oMmKGPjjIQHFcpe3bCVmLEkccnrn/APXSI/mRknP064+ldsKCW5ySrNmrcaxKzrgoVBIIPB/PtTLW9dNWs58hTHOmD2wG5/n+orNVnjbg8etPC7AWyQ2Mgjgj0/pXRBKLTRk5NnsWoy6fp93Ml5eWqRyqGkhldd31C9efpXFXPhPR7tZdQtEzBk4WGXMZI64x/LIrTsdFtbmyR1R2MnzOzgBiT1zzk/jTk0q60ouNOZYzccSKMKvH8W37uR2wB1712VsDKcvaLS5UKy5eV6nLWtrZ6ZFc2t3a3DSXabWjhjBESngEnPUdcAc1y5lmgDwrJu2sVztIPHfkZ/OvUodHMJJlyTnLbmyST3JP0rzvxDZGy1m5QBtrnzF3c9ef6VnXoOEEyea7LGieK9X0GXdY3bxp0aLrG31Q8fpXpvhj4oWur30VlqUQhu532pLGcozH1zyuT9a8Pc4Qc/Wrugbn8R6aBkk3UXA/3hXE0UmfQHiaBGvbeRyOYyMepB/+vWOttkDJVU6DH+ea6bxDAzi3cLnDMMk8dv8ACsWK3LHAG4+grxsQ7TZ7WGV6aI4rdekYBbHcZNW4rcx/IdpPXJJwBViK2YLy+D02r0/Gr9vZlzkgkexrjlOx1qJSW0DHG4Mem5s/oK84+Len+Q+lXI5V1kjzj0IP9a9ijs9xAC4x6Vwnxk04Hw1ZXOSTFc7SR2DKf8BWmDq/v0YYtJ0mjyzwXJ5PjrR2Y8POq/8AfXy/1r6YjtRtAHTrya+X9HY23iHS7nB/dXEbn6Bga+u4LFnxwcVvmcXKceU4sLV9nB3M5LUHoTj2qzFY5wABitlLBI13SsFUeppBdxqdlnCZn/vdh+NckcI1rUdhyxTfwleHSzjLAKvqac1zZ2reXAhuZv7qDNTmxuLvm7mIX/nmnA/GrcNvBaptiQKK7aeGtrFW83v9xzSq3+J3MxrXU9R4mmFrB/cj5Y/jVmz0ew08744gZT1kflj+NW3mA71VkuPerlKlT13fmTzTkrLRFp5go4r5M+NFt9n+JWouB8sojlH4oP6g19QST+9fO/x5t8eKLG6A+WW0Ck+6s39CKeGxLnWsxypcsGyh8Er8WfxE8rOBdWskf8n/APZa+jZJ+etfJngC9Nh4+0SccZuFjP0b5f619SPJx1rnzRuNRNdUb4SPNE2VO5Vb1GaWoLR99rGfbFTV103eKZMlZ2KlymH3etQ+1XLhd0X0NU67IPQwmtQpynBzSUH2rQzZNmlpgOQDS1ZA6ikzRTAdnmlzTM04GmIdUch2vEffB/KnZqO44hLd1Ib8jQwJTSxfe60080sX36ALm7Heik3e9FFgPkH7FK7i0QKZSdmM8Z6Vv+GdKutMguIrkKAzgrhs9qzopxFqSXDBsCTcQOvWt0a9bbc+XLj8P8aI2WpMrmltPvSAc89KoDW4B92OU/l/jSnXIGPKS5/D/GtOZEcrLuBijvj+Rqh/bNuf+WUvT2/xpv8AbUGeI5OfYUcyDlZo4rVtpTJbpknK8GuX/tqD/nlL+GKs23iKCINuhlIJzxipk0XFWOmB280044OTWGfFFoCcQT4HsP8AGmnxTanrbzfp/jWZpob2c88cDiq9+c2rcEZwOB71kf8ACU2o4+zzHn2/xqrd+LLR4Sgt58nvxj+dJ7DTRVumA8VaIRwPtUfB7/OtfQiivmuPVI9T8U6QY45ECXUfDYyfnH+FfSq4xWUi0LTT0p3WmE0ITISOaaRT2xTDVmZh69rVtoBtbu6XdGztH94DBK57/SqUXxQ0OMfNg/8AbZa6KbSrDVsRahZwXUSfOqTRhwD64PfmnR+DvDi8jQtNHP8Az6p/hWkU+hLaOfb4saEBwE/Gcf4U0fF3Q8dIv+/5/wDia6tPCnh9emiaaP8At1T/AAqwnhzQ1+7o9gP+3VP8KfvdxaHFH4waIv8Az7/jO3/xFN/4XBo5ztFrj/ru3/xFd6uh6SOml2Q/7d0/wqRdK05elhaj6Qr/AIUe93DQ8+Pxe0oDOLQD/ru//wARV3SviQuv3n2LTLOC4uNhfYszD5R3yVAruF0+zU8WkA+ka1WvNPtUv7K4Hl25QuCyhVzlenIpWl3HzLsZI1PxE43LoURGcf8AHyKUah4k/wCgJAMf9PA/xrZglsLSFIPt0bBc4aScFjk55OfekbUNNQkNfWw+sy/40/mL5GWl/wCI+p0mAD/roD/7NVgX2vt/zD4B+IP/ALPVhtV0gcHU7ID3uF/xqNtb0IEE6tp4+twn+NS4+Y0/Ib9s1/A/0KAfgP8A4ug3GvFf+Pa2B9x/9nTW8ReHxkHWNOH/AG3T/Goj4m8NqRnWdPH/AG3X/GlyLuPm8iuNQ1t7+e02WivEiOd3AIbdjHzf7JpkkWtyH5nssf71U18UeGYvFFzcNqln5clnEu9WyCVeTjjv81Xm8ceFF5/te2/BWP8ASp5E92VztdCD7FqvGZLLr/z0/wDrUybTdVlxiayX/toT/wCy1M3j7wov/MVi/CN/8KafiF4UGP8AiZgn2hf/AAqfYw7le1l2Ko0nVw3/AB82eenDf/Y1NHpOr7cfarVfxz/7LSN8RfCw/wCX9z9IH/wqNviR4XB4upj9IGo9jBh7WYT6NqYcIdRtgzgkADJOOv8ADXHQT+L2BeLw/fMp5DBosMPUZrrJfiP4WkA8xppMdAbfOPzrqtPdJtOt5o+I5I1ZQRjAIyKXsY9A9rLqeXhvGxORoF8pz3lt6w/Ftz4mi0Yrq2nTWtvJIFDySwtlhyB8vPb9K9yIB7k15X8ap9mnaXbBj87ySHHsAP8A2Y01SSYOo2jw+5l3yPH1HSktJAVwwGV688+lMmjZn2bvnA49/pVe3LLdEMCDgnBOOa2MzTH3gcnr074/z/WpTtKDaBgHPA71WZl2HPpgAdf89KsIAybgB82D78/5AqhHp3hi787w9bSQqjzJGFYnsRx/SrMksz3cRlVctuXAHsf8KxPBMhk0dgrBmjlZM598j+ddDdSLDCoklUyMwVDjOT0/Svepy5qaZlazGFURRjl2OcEkcZrhfHduPOt7sA4bchzx7iu+i8xo13hN2zDMuOTj6CuX8bW/m6M0g6xMHBH6/oazxEOakxp6nmE8ZUZ6rn0rpvhnpjal4705cjbCxuGyeyjP88VgTRb43xlsc10fwsultviBp+84EgeMfUqa8CWxvHc+gdZhEtsmQOHB557Vn2+ngoCDj6/4Va8Q6vBo+iXGo3EUkkMAVmWMDdgsBxk+9cVF8YvDiAH7DqI57Rp/8VXjYuhVnO8Eevhq8IU7SZ20dhcebGqlPJ/iyDn8BWpFaggAdO9eeD42eHVOGstS/wC/af8AxVTp8cfCqgf6FqhP/XNP/iq4ng8Q94lyxcejPR47UZGD+lc18UNIkvPAV2Yk3NC6SjHYBsE/kTTdG+MPg++SRpGu7bYMgTRD5vYbSf1xWL4p+Nej3Oj3un2NtdKZoWjDlFI5GOea3oYSUGpPc5Z1nLToeLWXk295btcAmNZMuq+mc8H/AD1r7C0+/wDt9jbz2MLeVNGrrJJxwRkV8Vz36zN8qlTnNe/eHvjv4Z07wzplld2+ofabe1jhl8uFSpZVAJB3dOK9WdOTaZytq1j2D7CJG3XMhlPoen5VOPLiXCqAPavJm/aE8J44ttU/78p/8XVZ/wBoLws3/Lpqv/fpP/i6zacPghqTZvdnrz3AA61WkufevJG+Pvhgji01T/v0n/xdQN8ePDTdLTU/+/Sf/FVyVI4mX2WawjBbs9Ze496rPcV5U3x08Nn/AJc9S/79p/8AFUw/HHw4f+XPUv8Av2n/AMVXJLC4l/ZZ0RlSXU9ReUmvHfjnbebZaVcgfcaVCfrtI/kavH43+HP+fTUv+/af/FVy3jr4j6H4o0WO0tra9WZJd4MiKBjBHZj6itcJhq9OtGUo6Dq1KTptJnmun3Js9StLpfvQzK4/Ag19bAlwCuSGGRivkDIxxnOeK9lg+MlhHplrbvHemVIUSTy0VF3BQD/FuIznuK7sxw063LyIywdaMLpnuOm5FqUbqrEYzz61crgPhh4us/E8OpJaxvGbdkZlZAv3s9ME5+73rvqKUHCCjLcc2pSbQjDcpHrWe3BI9K0TVGddspxXTTMZjM0ZpuaWtzEkQ/KPbinVGh6j3p+ferRAtLmm5pc0wFzSim0A0xD80jjchU9xikzS5pgJG26JT7VJF9+oYvulfRiKlj+9SAu9u/5UUz8aKAPka/ZoLV3Q4dQDnHvitNPDmqMgP2+AZGf9X/8AWrP1VS1hKMdcfzFem6Pol5q52WcDSsi5YKBwKIpPcmba2OIHhrVcj/T4Mf8AXOpF8L6mcj7dDn/rnXZy2kltcGGZSrqdrBhgg12Hh7wgNVtTOZNijgZGc1bjFakKcjxo+GNVI2i+iLY6CI0knhvVU4e+jUkcZhr03X9FbSbpot3TuK5uXLHkkkjHPWhQTDnZyR8O6kcY1KLp/wA8qP8AhHdRbCnU4hg/88R/jXUU09OBzT5EHOzEHgzVm4/taLn/AKY0q+C9SOSdYjx/1w/+vXX2r+ZbK2eR8pqyqgHn8ayasbrVHEf8IRfk860n4W//ANeoLnwNdpEXbWA3QEfZsd/rXekZByar3/8Ax6OxxwV6fUVL2KSR5dp9m9h4vtLZ5vNMV3D8wXbnJB6V9Rp92vmiQ48exAH/AJe4Dx/wGvpdBhcVlIa3FpjVJ+FMakgZC1RmpWGajPWtUZsqXmqjSESZoWl3tswpAxwTn9Kq/wDCbDOPsD8H/nqP8Kr+KD/oEBzj9+P/AEFq5jODW0NjKb1OxHjnH/MOb8Zv/rUp8eEDjTvzm/8Asa4zd+lGeaqxHMzsv+E8l/6BqD/tsf8A4mmnx7cHpp8X/f4/4VyGe1JnGcUrBzM7D/hPLr+Gwhz/ANdD/hWF4n1u68UWlvpcsMUEUs2WdGJYYBPf3FZoY9cUgb/TLQ56Sf0NNoLsxpvB9vA4U3MrZ6HA/wAKaPClp/z2mH4r/hXUal99M9xxVFgeNpxUuESlORjDwrYjrLORnuw/wqVPCmmkHLTn1+etA7h17VPCHJzn5RQ4oFJmavhPTR/z2/7+U7/hFNM7pJ/38Nawb2p+78anlRXMzA/4R3Tk1Rk8uQx/ZQ+wyHGd5GaZeaJpsUO+O3xg/wB88/rWu/8AyFE6/NaOPykX/GorsBrdxjoKFFWByZx7w2sWqCN4wIthbBY8frVYxLKk8kJ+SPJBzyT/AIVY1bBnhiUjdK4X8B1qGABU1KPngnj2xU2KuJciNdMhmVcMSMmpr+FFtkaFAPmBOPTvVOV93hmM45AHfHenancj+zY1z8zKDgHnFFkFy3FEk80cpjCwlwFA/iH0rYXxdr8GsS2CarcLbRIBGikAKBx1x/nFZrSBbWIj/ZOR06VFHEt14n1HqGiY42ntk1N0ilqauo+K/EEUtrs1q8VWfDASdeRWH4o1a81KaJLq7muDEpI82QttyecZrYl063mCO7t8hyvPT/PFcz4hjS1v5FU/KI1Y5OSSf8/ypRmmxuLRhzrAqb5S2euB361UgbfKWOQoHA9OacIZLl9zHapPAz/n/OKSI7L1lJB+XHHatCSxJwCSQM+n+f8AP41LalPJdSB8h56dD0/rVd2yMk8dKS3lCXOMjDDBOOh/z/OgDt/BVyqG5iZipeQNknPVf1ziulu/31woTOI2BGB79q5HwfLGlxdqyhmCqykDjOT+XQV2KulraiSUfO5HB9c/5/KvYwr5qaRnLRk7NOWwowq579enP+fWsfxJHI+kyRuoDOhPOa6CMzRfJLjaCfmHb/PNcr4ne71EywwOY0QHkf5/Wt6ukGT1POI2OwNycZzntWr4OtpJfG2lRwA71ukcEd1Byf0BrGi/dM8Z+8pIP4V1Hw9eVfH2ltHH5mWKt7Dacn8BXz8tDdHver6QuuaRdaY0nli5j8vft3beeuO9clZ/ACBsFtYkZfU24A/nXoVpJ5dzGwCZB/j+7+NdEbG5uzm9uwI/+eUPAP1PU159Zz5lys6YtJankep/Bnw8lgbW21C6m1HIC+Uike+R2/Oqth+zqsw33utyRA/wJCCfzzXuEMdrZpsgjVAPQUrXI9azVX2espfJCb5tEjyFv2eNJUfJr18repiQ1RvP2e7GO2ldPEF0zqhKgwLgkDjvXsr3PvVaW4yCCawqY63wlRpye58UXlobR9pO7OR0xzVSt/xNA0OrXUJXHlTyJ+v/ANasCvXpy5opmE1Z2CirllpV/qRYWFnPdMvLLBGXKj1IHSr6+D/Ej/d0DUzjr/oj/wCFU5JbsSi30MSitUeGddMPnf2NqHldN/2Z8fnipIfCXiK5GYdD1Fx6rbP/AIUuePcOSXYxaK6D/hBvFP8A0L+pf+Azf4Uh8D+KB/zL+pf+A7f4Ue0h3Qezn2MCit8eCPFB/wCYBqH/AH4b/Cqmo+HNX0iOOTUdPuLVJCQhlQruIGcChVIN2TBwkldozeNo9e9eg+CPh1aeLNCfUJNSlt3SdoTGsQYcBTnOfevPa9d+DV/ix1WzJHyyRyqPqCD/ACFZYqUo03KG5thYxlUSkd58NvBUPg/VbwwahJcLdQgFXjCgFTkHr7mvSjXI6Pcf8TiEZ4OR+hrrjXHRqSmry3OqtTjCVo7CVXuk+UMO3FWKbIN0RFdEHZmEtUZ1L2pSMHFJXSc4qnDn3H+f51JUI/1in1yP8/lUmatEsUngn+VRfaRtyI5OuMY5pXDkjY4A7jHWkVZs/NLn6LVCE+1f9MpPyp4n64RjgZHvTQku0/vfm7HHajZLx++IHfA609BE0b71ztK9sGn596rCOX/nufyqcUAIvErj1ANTR/eqAnEy+4IqxF96gCzz7/nRS4NFAHyZqeBbMeOWGB75r1Lw94kvNBLSWhTDqA6uuQa8q1U/uOn8YHI969F0yMS3cELMQkjrG5HoTShs7kVHbUuXepTahqEt3cEGWRtzYGBXRaF4vn0qAwoEdD/e9a5XVFMGpXEDPvMMjRA+ynFV/MKn6Vro0ZJ9TZ1jWJNUnaWVgWY9RWM5zk5z6980by3GeaQocU0BEeM5IxTSMZ5HvT2XbnnkVGdv5mmMt6fJtmMZ5DfzrV2ZyR2rn1Yq4YHBBzXQQuJIVkHRgD6VjUXU2pO+gbT36VWvFP2V88gEH9ausMj0AqreYFnLweBn9axb0N0tTzabb/wncRH/AD9w/wA1r6YXpXzPOCvjuIsACbmHp/wH/wDXX0yozWbBbsDTD7VIRTGpoTImFRGpmFRtWiM2YPigf8S6IjtMP/QWrkyD3Ndh4kTfpg9pQf51ymzjitoPQynuRYI6n8KO3J96k8vJxTvJ5zxg1VyLEBHvS4PepfKJOOtTf2beHpaT89/LNFwtYqdxzRj/AEi35P8ArV/Xiro067720oHupFMktHikt/MjKFpUxn/eFDulcUWm7FjUVDMhyeh6VTaONR8zYz7962NTjRBH9D/Ssl40lOD0FTe7LtZFbMW77w/A1ZhaEpgv85PAqE20YI5/WrMFvARv5yORzQwQ/wAlSM80ohQ8f1xTwQBQW5PPFRqXoVVhRtcsEb7jpKhx9UNdp/whumSIys9xzwcsP8K40uF1fTnHaSQZP+4D/SvWEwcEbsnrxTQ7I4//AIVz4clmSR455JVPysZWGPyFcWfDOn2/xVm0VoibG4jRtm85OUPfr1FeyDaJQoY8mvNfEDLb/G3SnJx51tGv1O5hQxmyfh34XSD7Oun5iHVTM5H86uR/D7wq6ANpEDA9d5Zj+HP6VuBACfl3c9xmp4yFlCjAUH2osM+e9at4bXUr22gGFgmeNV9ArED9Kowyi28TahL3aJX+uVH+Nbvi3aPGepwbct57HIGM55/rXNao/ka6zfKBJbx9+vyj/CuWTvdFxVtTVa9LLt2Y9cVx/im73XxVX+Yqu/H44/nXTq2BweK4/VJLeC9meRS87OTgjJ69falRjrcuo9DPigJXzZ5CIwM4BqK1/eXDFeCxwB171FcXMsxwxwvXaOlTWKqdzPwo4PFdRgWZQG6HOevP1pkSKGBBOe+MHildzIxCdD0GM8VYt4TtDNncRkd/SgDoPB7IuqTqRjdErDkEEg//AF67NwlzcKjA+Wg5x3rz3Qz5OpYUoN6cZ+o6e/8AnivQdKxJy/CL1z3/APrV62B1jYzmbHnJLAjSADIBP+PSsO7uUbdGqBmJPOKtLDcT5dcrFIBtUDBUfzqtqb22i2Blk5kI2onGScdv8a6Zyb9BWVjyzWIBZa3PGD/FuPPQnnFegfCTSrO/1ue/k3+dYxholU4BLZBJ9cAdOnNeaX9xJcahNJMRvZucDivQfg3qCw+KJ7RmAFxbEKPVlIP8s14NXVuxtA9xI7ZxW7DqHmW8bg/eUH9Kw+lRWVyRB5ZP3HdPwDED+VeJmLcYpo78NBTbTOi+1Z70xrj3rNWY+tKJGPTmvFdSR2ewSLpnJ71E0mc81Tury2sYvMvLiOBPV2xn6DvXNy+PtOkuhaaZG1zOQcM/yJwCfr29qcadSeyC0UeM/EK3+z+JdVT0uWcDH97n+tcMetdp4y1CXVdWnvp40R7hVfagwOBj+n864xhhyPevqcNdU0meVX+Nnb/DHxDNoGu3RhhWU3FuUwexDA56/WvfNJvtQ1nSra4kMUayghgVyVZTg8dOevTvXzN4RZh4ltVVsF9yZ+qmvoLwlLLbQPA0zPGDuUEcAnrj8hXFmFNX5ludeE1Vjql0+AOJJi07jo0hzj6DoKnLqowoAHtVcz8ZzULzD1ry7tncqd9yw0uaiaXrz2qo9wB+NRG5G4DnrQos0UUiy0tedfF2Pz/DFtKASYrtc49CrD/Cu0e59q5Px+PtXg6/X+KMLIPbawP8s104a8akWZ10nTaPn8ffGfWu6+FV0YfEtxCTxNbMMepBB/xriJhtnce9b/ga5+zeMbF84DMyfmpFe5WXNTaPEovlqJnvtnP5Wo20hPSVc/TNeiGvJTdFSCOo5BFerwSie1jkHR0DD8RmvOw6tdHo4jWzFoNL2ptdKOZlORdrkYplT3QwwPtVfPFdMdjnkrMDxg+4p5/Wo2OUOPSn5z+NWiGLRSdTRVIQtLTaKYD80Cm0UxCSNjY3owzVmMfNVaXmJh3xViE7tpHfmkBb49T+dFJzRTEfJl8pbykHOZkGfxFd9GywXCMrE7GDZxjpXCXAzc2o4+a5T+ddtld3U8c0oapk1Fcv6xPFd6zeXNsS0UshdTjHXk/rmqWcD2pyMintjPXFNJ+c7fwNaLRWMkrIu2ccbTLvIA75rsvFOi6Np+kW0tlOGnfHyhgdwx972rgVlwcg4PrUj3DuACaGrsaIpMZIFREGnls44FNbkmqAYc561r6XNuhaM/wnIH1rJPPXt71YsZhFdKScKflP41M1dFwdmbpBIzVS+YfYpSRn5T0FXwvPSmz2UU9tMXheQhSTiQrkY6VySdkdijdnlt8AvjmLn/l4hz/47X0yn3a+b9daL/hYMfkwCBfOg+QMzf3c8k19IIPlqb3SFs2OIqNhUtMYU4ksgZcc1Ec1Ow4qMitEZsxvEP8AyC245Dqa5Inj3rt9UtJLyzaCLG9iMZOO9Yq+GNQbvCB/vGqiyXBvYw1ZuenHbHvRyeT29q318J33eSEf8CP+FPHhG95zPCPxP+FXdB7OXY50HBHP44rtUjvJdQCRp+6Me7/VnHTru9c9qzh4Oue9zEP+Amu0hUxQxIckhAMg+gqWlJicHbU5aayv9+0o4X/drJ1uN4zZGSNlbzejA/3lr0QHPQH86xPFGnR3kUEjs6mEMwx+B5/Kuidbmhy2MaeG5Z8yZy2uDasZxnrWEZOeBXb6jpMVwE3u4Hsazh4ZsuCZ3XPOGcD+lYqyWprKLOZJyM0gcgYrp38O2gKlZCVz82XPT8qkg8OWkz7ERmPX7xFaqm2r9DO6va5ym9uTxR5jdCeK7T/hDkP/ACxUfWU05fB8QGPIjP8A20NLlXcrlZw0j/6Tp5J6XBH5xv8A4V61E8zKCqpjAxkn0rg9W023s7zTlSNVIvUDAH2Yf1rr4X1NxEYo7YwlRguxzjHoKya1NUtCw7XrSZHlAdvvV5z47c2vxR8K3BQb3ESMe3EnP869Bk/tYEBDaKoGT8rVzfiexFzqWm3eoRW0s8Tf6NKqsNhyD0P4VLVluOMbs6122yOMd6fCqn53wBn061nNa6uSCL22B7/us/1qVItUCFJNQhIIwAsAyCe/Wk5tdCuXzPIvH0YTx7fuI2AYKckdcoK5HxCjfarRowdv2dTyfQsOteweJNKU3ryXUcc9xtGZTEATxxXGG1SW0gLRRuNpU7xz94+v1rk5veZsqeiOXuZWgsZZo8b1XK+ma45/tJlaWeRMk5ZioJ/OvUfENkB4dnCxoh2K25RyBkE15xc2rO6IVOxeenU10UUrGdVWZkuY5XZkhOOuc1NaqDCQV5HaprgxxYVUwzcAdM/4VVtJBu56Z7VqZF1It7ZbhR25qyeT7Yz6VAshyOQPXnmpR0C9F7Dt+H5UxF3R7WG41KVrhwqKoKkNg5ya7211CyjHlp+8XjOehFeW3UxiuIsSyIGB+ZBk4/H6fzqRbyQkf6XdtjDHkLjn2/xFdVHF+xjZIThzbnr8GpxmJhGoLqCFQdTzwK5fU7CWVmvdXvrW0U/dWTJwPTHrzWLBdLdxX0UUMMU0do00EkckxYsrDpudlPBPYVy80d3PMWuBLJu+UOxzzjI5pvMfaaNFulyrQ0dZtNMZZprS5mklQjlowikfTJNVvDGr/wBieI7DUMHbDMrMefudG4+hNZ0tndW6HepA7gHNVlODXLOSm72EtD6+VlcB0bKsMqQeoqpDC/n3HZBLnJ4HIB/rXPfDbWDrHgiyZ23S2ubWQ5/u4x/46VropNMN5cl3ndYRj5FPevJx8E6ep6GDlaeg6XU7K0wGcyuP4Yx/WsfW9Y1f+y5bizhNpFHyWxkkZx1P17V0lrp1laf6uFd394jJpb/yp7WSGRFZHXBUjINeRHki9Eeg7yPF7lri/ZmnmkmlbnLEmiy02a3uEuB8vlnP510k1rHbzsqoFAPYVXlT/R5B0Ow/yr0o2toYNanA+M9O/s+5gQEsHhyM/U1xEn3zjp9c16d48QTWOlXGMlkKk/l/jXmcow1d+Hd4Hn4hWmXvD83keIdPk6YuEB+hODX0RpT+WT79a+aYZDFPHIDgowYfga+jrKUMqMucEZrHGK6R0YF7nQi5+X9Ka7SFN+1ivrWZI87QObbaZQMqG5B9qrzXOoRTsUZXi/ikfapAyC23nkben19q8zkPTuaLzYqLz8HI9c1zn2zU3QjcqlWIdyyYQbup9DjH5niqzX2pkbkmh27QTudMZ4zjn2P51oqZPMdNNJh26YyaytbT7Vot9bkZ8y3dR9SpotJp3iLTtuJY7G45XjH3eO9PZ+CDjB6mqirMmTurHzzcD95n+8Aat6RP9k1uwuD91JkY/QNzUV/CYLhoj1jZkP4Mar5OEb04/wA/nXtbo8LaR7y74LDivVfDk/2jw9YyZz+6C/8AfPH9K8atrv7TZQT9pI1f8xmvUvAtz5/hxUz/AKqZ0/8AZv8A2avNp6SselUd4pnSk03vSnrSZroRzsjuBujPtVI1fPIxVFhtcitoPoYzQn8qRP8AVrnrjFHShehHof8A69aoyY4GjOaTpUF9LJBY3EsIUypEzIG6EgEjNUkIs9aK8t0z4jatfapaWskdnHHNMkbMI2yAzAEj5vek1H4iaxaahcQQpbNEkhCM8DKWXPBIzxW3sJXsZe1ja56mDS5rB8J61Nrugx3k4jE29kcIMDIPH6EVuZqGrOxad1dD+1Ps+UUf3ePyqLNPs+JJF9Gz+YqRmhx6n86KM0UxHyhMc31l1wblOPxr0C00a/vbC4vYLYtbWwzLIWUBeM9zyfYe1ef8tqdgP+nhev412X2iXyvK8xxETuKBvlJ9cetFPYmY8SAHnP4GnvJFj5V/E1Xz7mnKA2Mtjn0rQhC7hzk/QUA5J/SnvFCIyyzAtkcetRY6d80XBqxqRaJfXGjyarFb77SJisjqwJX6rnIHPWsxmJ69PQ1JG5EZTzGVG+8AeDj1HeoCeTxQgFz3poPOaCc59+nFA6etMDrLFvPtY5uuVGeO/erUsLi1k4JO08Vl+GZQ5lt2xn7657+tdYbVmt3PfaTz3rzK8uRtHq0I88UzxLxCrDx/Hv5Ilh49Pu19JR/dFfO/jCMJ8Rl68yQE5/4DX0QnKinF3imYzVpscaYwqSmNVozZCwqM1K1Rkc1oZsIl3ToPervkA9KgtFVrpA2T1/lWqI19OazlPlZvT2KQgp3k45zV1oQrYxSeUCelL2hpoU/KHPQ1Jt6DngVP5R7CsHUzeJqsMMFs8kTth3GQF6fgKcarvZK5lVStc2QBnkgflVLWQHs25BIU9D7Vi3tjqBB8gSEn3P8AjUlhbXccdwtzC6jy/lZh1613Sp/uue5xU6zdVQ5S7crmNDjrTWsZ7iG3aBGPOCQQMc9TnqB6Vbdd8EWeeB1+lXrIAWKfUiuVWnozrqRsjIu9HuicxkkYPRv6UabZXdrckzRERlcbye+a6AAYzgmknH7gY4wwP612Ks1DkONUI8/MZ4Y5OWz+P/16RW/e/fGB71E0b+awWLI7c0qxkPzEB79ayubnI+JG3aivA+S9hYfi4H9a6yyLCzhYKPuAc1y3iBN+pToM5EkD/wDkRP8A69dRZDdaRKc/KWB/AkVhCpebRtOFoJliRiV+bavbrXPeKwv2ewkGTtm6jtXRTAbVG4g9eKw/E426Mj4J2TKcmiv/AA2Kj8aNiQcq2eopVI7kk/SmpJmGJgfvIDjGe1SLIxxwfyrRbEPcxfEVvm5UjoyDOa4fT7TzYTHlU2zSKT2HNemalCJhGxQ8DFchp+lFLy9gkV+J2ZSB2IBryK1XlqyR6FJXhEzDYvPayRlFZTGcq2AGGOhzXialkyTu2r91QM19HvpanIKuePSvn+/sHstau9PVHLRTvEF2kkndgYFdODrKbaMsVC1mc3fRM37w8MOeO1U4MbgMD612d94J8SPbGQ6Ff4C7uYGB/LGa4tUKuyvuVgcEEYIrtumcdmtzViICjjrzjOetSmQIgZjx0b+VVYpCyYB3Zxx1qwmGjOSffjH+f/11RIiQw3sqs6vIANvBxn/Oa1o9I07KsYsDrgntVGwRAeBjOTtGOn+Fbax+ahAYqccAev4Vm2aJGnJe3ap9ne7u3ixjy2uXaP8A75Jx+lUJnkcFFUM54OQKsmJXyShJPOC/4/4flVy2tVi/eMoU44xngCsktTS+hzeqad+7SFpGeWRtzEj7oHOP1Fc1qmnmzfI+6fbH+e9drd3f+kSCBPOuGGPZB/Ssa80S4O+WabLMMsoHArVMzaOi+DGsm11270l2/d3kW+MH/nonP6qW/IV7gr7Dj8a+W/C1/wD2P4w027LYWG6UOf8AZJw36E19QSq24bQTjrXJjY3ps6cG7VETGaqd3P8AKRSq2W2d/Qdap6iGjxkEZHcV4sY3Z7GiMC/X/SGI781nMy5KkjBFUfHUM9xp8Bt1kdhICQgJ4wa4M6ffsebS5JwRnyzXqUqacbtnBWqOMrJHTeKlE3hOykHPlTFePTn/AArzC4BDnPXJFekyQS/8K/nhkRkeCfO1gQeo/wAa85u1xK3Hf0rpw+l0cmI1aZXHWvfdAm87RNPk3ZLW8eT/AMBFeG2Ol3morMbSAy+Su+TBAwPXmvX/AAXciTwnp5JJKqVOe2GI/kKWKV4l4R2kzr4ZVBwW25HDY6Vm3Ol2cmU813UliVAAUbiCR09Bj6VIZTuI471C0hIxnivPselzCW9tHaFirs24BRuxwMk/zY1W+wQIylXkBVdo5HdVX09FFSlx0z+NQSXcEed8qfLjPPrVWZLkieMiCCOJMlUUKM+n4U1pCe1Zt5rMNsNoEkjq20qo74zVe11c3F4YGjKDJ2nOckdafKyXNbHmnimAQ6/fKOnnlvwb5v61j5/dY9D/AJ/lXVeO4PL1p5Mf62JHz9Mr/QVyi52sPxr1KbvBHkVVabPU/DVwZvDlk2ckJtP4Ej+leq/Di43W+oW5PKujj8QR/QV454Jk83w+Vz/q5mX+R/rXqPw9l8vW7iL/AJ6QE/kR/ia4pK1RndF3po9KNNNKfejtWpixtVLhcPn1q3UNwuUz6VcHqRJXRUz0o6OffBoobhh15FdCMGLmkIDKQeQRg0lKOapCPD9MgNjq2qRRIr3ttHILYMufnVwCQD32BiKs3KN4ns7jUvJjt7qOXG57g7JQcswHmMcFeuBxg9OlX9dlsdO8aXqXdsiLvEsc0asZNzYb+8B3PT0FRXfiHSdR2G8NxMUXCh4t2Dnnq3fjtXoJt2ZxtLY6H4XXO7Tb616+XKsgH+8Mf+y13wrzTwJqdu/iW4traJI7eWDKYjCsSMdcfU16UDXLW+M3p/COp9uSLsjsyfyP/wBeo6VDi4hbPGSD+NZGhpZ9jRS/j+lFMR8oR/8AITsAD1mBrr8HbxznvXN6NbpeeKNKt3+68/OD7Gu01LTpdNmCFhJG2SrHg1MJpPlHODa5jPwR2pQCeMUoyP8APWj5vb0rYxsOMbqVBHLDIwaPKcsF7n0pMt1zxSlm4OfzpABiCscnIHoKsxWKskkguIf3ah9jEhmz2HHUVX3NjGaXJxnNA0M2EnoRz6UqxMT909aXLevtUiFs9TSbGkaGiRTxanbyrEzAOA2O4PBr1xNIYxcr1HYV5n4fhZ7pS05TBBOO9e4R4MakelcFel7adrnZCpKlDTqfM3xGsWtPiYuFKgtbnPXPT/P4V7og+XFeUfGR0HxA08D73lQ5/wC+2r1hB8tEY8seXsDfNqxaa1P7UxutaIlkTCmGpGphqzJlnToy94nHTJreW2RTuA+btzWRpDYvMeqmt6lyJu7HzNEPk5Oc0eSKmoo9lEOZkJg5zmqN1GUmA9a1KzdRBaQKDjgciqhBRd0JybIgrev6024VTAQ7AZ4yacoHrn61T1dtunsx6BgePrTqaxZUPiRJJEscSoxHygZ/KnQXEKW4BnjUA92FZd7fgWPm+oXv7Vyv29pZSmCFOe9YUU73NKstLHo0VzBM+2OaN2HJCsDVW61nTY1eNr23EiMAymQAgg81wug62bfUrgtGoVUKls9eQa4LxJfCXX9RTBz9olU9OfmNW6rU3EhQ9257adc00Lu/tG0APTMgFQHxRog4Or2ZPtID/KvnKeLc/FWLItGwUKMN0z/Sh1mgUEepzanDqXja+8i5E1p9kV0K/d3Kyk/yrrLbU7aG4ltWnxILmVcFSerEjtivI/D0pg1K4xx/okw9c/KTXrWjS711CUHlrtm/NVrnpNurdG80vZ6hN4o0uN2BnnbA42Wzn+lYfivxHZz+H5IYor4yl0YFrZ0HUdSRiuvyM5yd31rL8WQibwjfEZLBQ2TzyGBrskm4tM54uzuiK01eVbKBZNI1MukSq37kAZA9zVuPVZnAK6Re4z3MYx/49V9SXjQ9ioPFO25XG3I9zSSfcG0aWmjzrISSwNGxJyjkEj8iRUdvDbtql38i7jtbp7YqNLryU2dOM49KyLPUDHr0xZuHiXj6E15uIrwVRRa23N6dOTi2jqTaw8/IK4eTwxp1r4/u9ZFupuJUj2E9EO0hmA9TgD/9ZrqDqgrG1vUBEEu0GJANm7IwO4z+v501iaLklBWKp0pp+8RahdSXen3a2Pz3Ue5VVSAQ3YfNgentXB+NPhfPr3huTVI9HEXiGOIO5jkUeeR94MASCcZweucc13ENyt3Ai27yGYneuGGHU9VBPQ55FdDBeSG0jEsZ8zaMhmHWuuLitQqp7HxQhKyYYYIOCGHI/CrUchbEYHJPQDFXvG1ylx461uZNgV72U4QEDO7375z+OazrJTLKAoyAM/5/Kui+lzktrY27OJVUb2G30B79/wDPtWqjxpGXOAuAuP71ZtuhOOTjOGBxgCphKZr0QqCUjG4j3qCzUj3AKw4KhRjpipLuYCE+bN5UX/LWXPOB2H41RUzKp8q3eUkkKqr3579v881mzb7m82XOLy4BylnCSUT3cj+QpJDbLkU0t2hOnILKzBw11KMu59v8feq+o/aLexnjsoQcg7ppGDs477fyrREGpRhVmluTgYENrCoVfbLYFPZCQ0k1skUJxuCsZD+IUECh6AtTzXcd+7PzZzmvsHwbexX1hZ3U6qwuLOOXkZ+YqpP8zXyRqcKQ6jMsUbRxbsorelfSHga6ZfAuizAn5bVVz9Mj+lRX/htoKSvLlPSo7PSoblrhYUEh9TkD6DoKwPG0kBsrYxKgPmEEgdsVTOqN/erJ1q8NxaqCfuvmvHjVlL3bI9OOH5XzNmLK25AT6VUYYPIzU27dHjnhjUTDJBPT2roiVIx9Sj36VrUOP4BJ+mf6V5HeAhj15ANevX9zb20t3FcybPtNoyqME7iMjHH1ryK6GfriuvDnFiehu+A4pbnW5LSKQRtNFjJ6HkcGuu8Auf7Entz1gunT+RrzTS786be+fs3jaVK5x196734dXfnnVQAVBlWULnON2f8AAVpXj7rZGHl76R3M8vlQvLjO0E4H0zXPHWZ2ViYhGwGf1H9DXQTpvtmA/iUis3TbWxuYwJwNzQPyGx8wXj+XSvPukrs661TkVznpJLu5gaSa4k2KQ2BxwWwR+FQ3KGN7gBiW3ujEnqAAwrV1FYYmuoLZt0DoWUZ6EDNZtwd8xP8Az0aNh77lwa1jqjNSurkQvN0xuW5w8bn8sGrtvIp1hMAgmQuM8cMo4qiYBDbRzKSfMj349CCv/wBerjS/aZ2uYFkwAjD5epBwabKRk/EGPM9pIBjdG6k/Qg/1NcIvU/SvRvHaLJYWkmcsspXrzhlP+ArzlMCQZ6ZrsoP3EceIXvnb+AJswX8OejIwH5g/yFeoeDZ/J8U2mTgOGQ/ipx/IV4/4GnMWrTxdpIT+YI/+vXo+iXxj1SwugrIFnRirDBA3DINc1dWqXOmi707HuNJmlIptWiGHemONykU6g1SJZnkYNMY8r9ammXDn3qCXhM+nNdCOeQ7NGeabmlzVEmdf+HtJ1O5+0XtjFNMAF3tnOKgHhPw+p/5BNr+KZrYpc1XM+4rIoWmh6VYTLPaafbwyqCA6IAQO9aOaaKM0O7Cw/PNDHam7+6Q35GkFOVd6smeoIoA1Ovaiq0VynlJkjO0ZooEfN3hNPM8caSOcb5CPwRq7/wAVjBg6jJPSuA8I3ENr4306adgqDzcn/gBFd/4tnSSS1EZ3oys3mD7tcyT9qjpuvZM5oZ6ClxzzzSd/agkggAcdzmu25xsUc+uK0NL0XUNXkZLG3MrJywBAx+ZrPXOR9K6nwd4gg0O9ka5jdo5EILKRkHtwetZ1JNK6IldbFO+8Jaxptobm6tDHCv3mLL/LNYpUivQPFXjSx1fS2tbaOUMSDliMVwDMcDGCM81MJt7ji27iDPtUkfGKh6il3FZkXI2kHPrVs0RtWlyYFyDyRXtlheq9hAxPWNT+lfPa3iB8Z4xnOa9csbmcWds4lXyRbgGPbyWwMHOfrxXn4qThaUTsow9orHlHxcufM+KEAB4EduP1Jr2tPuivnz4lTGX4kI3B5gGSfpX0HH90VUHzQTZMlyyaHUw0+mmtESyJqjNSt9aiNaIzZa05tt9F9cfpXR765e0bF3F/vCug3D1rOc+VjSuT76N49a5rWo9cluU/s24ijix827r/ACrNWw8Vsfn1WEewH/1qweId9ivZnb7/AHqKVlyCQCScZxXJxaV4i8wtJrClP7oz/hUMui60u1pNcdhnGBnqal4iVthqB2JZMfdH5Vx2pXDPPcRs7bQ5wueOtS2Wk6hb3iTzatJKi9UwcH8zWVqJd7q68uNydx7GnTquV7lKFitqOqQw2CxS5XCrk9hWBLP5beYozjnPrVnVLA3tqiNA0iuoRwQee1ZN1p99Gfs0PlKgACZfaqADGPXrWlOsti5UebUrLq6y3lykJHyxt5nQ8gZ59K57xBLGPE18nmJ5jTyMFzzjcc1fOmXa/aJI7ZVfaTK6TMMnHPb09aZqmhX73CubIPKHDs6yuxcdxkr34/Koc05XNFStGxgz3CQxhmcKMgfUmrdtZTsrT5IjHPJz09MVoRaDqFwwii0RncHcW3noPqOK3bHw1c2EUN9L+7aMEtBuPTIHzevXNKUr6LchwSV2ylo9pIWS6XmGWB9pIw2GQ9q9Q8LljpkzkZ3uj8e6LXBW7zS3KkkbCTnj2ruvBj/8So+nlwkZ/wBwf4VWHvzakT+E3uABkfrVLxA6jwrqe0jCwMx3elXow0jAbWqPU7b7ToN/E6kB4XGR24rvexzLcjs2M2n2kiZIeBGyD7VN5Mh9aqaIyDRrLMoJWBQcnrgYrSE0X/PRP++hUKS7lNMztXnnt4ENrCs1xsGImfbkZAJz9DmuflvAmpqe4XBNdBqnkSiKdHSR49yBlIOM4z/IVyOonFwJB1weSe/FeHiknXZ6VH+EjRuda+z2zTFXbaeQnJrMm8RSTxvGtleHj72zj86qPO3UNgAjgDiopZ2KDkAE9BWMaLTujSUn0NfQxp6awrTXRtnc7sN8i5HPVux9sV3iXEFwqPDPHIhyN0Z3D8xxXkgOXUtxzzn/ADzXdeDsNoSd9kjqR9cH+tenSldWZzVVfU+V/EL+d4m1SVQ2HvJmAI5++etSaUo2tJjvgU7XrKYeJdTjCMQt5MvT/bNWtNj8iIBsLg56dv15ruvocXUvGQW8JckZ6jPpUFrL5VuZHUbnJJ4xiqt1M0zqinjg0XbrHZZ5GFxwvH5j/PNSUPtYLrUIjMZHSI/LtiBy4ye/T2rWs7LUEBgts2UTHnbyx7dBwPxJPqawNAhe4IHnTqozxFIQf0/z1rqINOt3O1p33L1LXJzn8/6VdiTQttIgQK9w0kzDndK5YD8M4FSyfZkct5CvIeV2A4HH+fypsOlhRmO9ugCD92Yn/ParDWskShUuZCxb7zAEgc8cChrQcXqcN4xs7tJLe6m5jYFB7Hr/AJ+le0+C0MXw60odP9F3fqTWP4bsLG+mvdMvoBdw3MQLCbn7p7HqOp5HpXbraQ2mmC0toxFBFF5caD+EAcVlLWNi46TTMSX97JFIZHBjYkBWIDcEYI7j2qO4cshHam7ifb61HIQUPPbNeWlqeu3oUrWFYVlRNx3SFzlieT169PpRPDHMFEqghWDjJ6EdDT1PzNjuOlMkOQMY/GtUZs5bxijfaLB1BJO9eBmvO72Mq75BDbiCDXsUmV1CwfOFEhU8+orzLxZCYNe1CMdBOW4755/rXRRlrY5K8dLnMV2/w1l26tdxf3oN2Pow/wAa4k9TXT+AZvK8UxJ/z1jdOntn+ldNVXgzmpO00eu/wDnINchNbzJNKhdEKcRlmHzZJP8AWuvCMVwAfwFcze6c5nvnSKQzKQ6YB56Hj9a86J6VRXRnrLbx3OWuS4ERJVR1O3moRc2rS2ReGUlFADsQOM4BIpW0+4Mcf+iy5aNDnyz6EEdPpSXEF1NbwH7JMriPym3IRggg5/nWisYWHx3NxMNsMcaokLNjGSOo4/GopbiW5dY1kZUcoRg9Mg/1FTwWtzBegAIqMzoxdgPlLZz196pw2s8TDcYuCpAMq9j9fc0aD1KerQmXTbqaUMJohHIN31wa4uQbZmA7McV6Hra29wtzJJKiL9nYD94vLDOO9eez/wCtJ9cGuqg9DlrrU1/DEnleJIOuGLL+YNekBiOc4I/nXlOny+Rq1pLnhZEJ/MV6rzn+R9azxC1TNcM/daPeLWcXNnBOOksauPxGalrI8Lz/AGjwvpz56QhP++fl/pWtmkthvcTNHFFGapEle4XgEVVcblI9a0JVBjNUDxxW0HoYzRGp3Ip9RTs1GmQuPQkfrTs1ojMd2zSk03PFHSmAtKDTc0vamgH5qSI81BmpYvvUxEUkEnmNjGMnFFaHHvRSA+XtIJbxDbZzwsnf2rq5nJwCenTNcjo2f7egYcDZJ/KupkJJ570QFJk1uQX2sO1WwkPcE1Rt8mTAJJ9quBHx0PtRJ6jhsPCw9AozS5h6hB1qPyz/AJFAiYk5FTcqw8NGOdvApPOQqcKB7Yq/pL6fb3EjanZPdQlCFVJChDdjxWUkIZnaNZAC5IDNkBew6fWpUtbA0K84UE4A/SsLXLsp5eJCgb5gytySO1bFxFiFtwBGOh5rnNSQSW5juCf3THYe4GOKu4rFVNQkmVR85UYJYddte56fdbtIg6j9wn4cCvnfTpHN9Cm5VVR3zhscivdbCR20yA7sZiXcMewrhxmsUduDWrPL/Hkm/wAeqc8F4f5Cvo+M/KPpXzZ43I/4TUHIPzw/ToK+kVOFGPSqhpBGc/jZKTxTCc0jOcVGWIHBq0zNoc1RHNK0hA5pmc1ojNkkBxcRn/aFbO+sOM4lT/eFT3t7fQzhLaxMykZ37gBmsKybaNqVrGk780itz1rBm1DW8/JpI6d5B/jTYr/XmkUNpiovck4x+tc/s5XNeZWOiLECq8wd4twBIVhnH1qtbz37fLcWsaqf4w/T8O9LcsfJwD/Evf3FRLR2Lir7EobIqW2iDAZxzWXd3E9rbtNFC1yxIAiXjb6n1rGj8ReIYZZPJ8PSzJn5MyhQPzrKona5UtrI6aOyjaJxgcSt/wChGvOkxqYuZbjaGErRELwCAQQe9bcuveJ0idrXRUZnYl1kmH7s9x1Gaz/Dem38cFy+rWiwzNcF0RHBBBAz0J71WFhFuS6vY5q7qRSa2KNxKLK3kjW1JikGHLEnHGOwqzYaeuv38Mt7Dsh27RsZgSoB5611KWVtdSRxTwqUJx0PH6Vu22mWkEjPFAqsqhSOe3P9a9mMacIaL3u55zdWctZaGLpmixQRyrYQrEJIgAz5PfJyc89Kh12GIz2iPbK+9zncOuQtdXEVVZAMALKRgDtxVWcLusXwMk9duT2rOjFQfM9WXNNqyZ5VKiQaj5UaqqbgdoHY1teF1jezj8yZ1WO0jOEkK5wSvOPpWD4ngnTxhIYrgRqsi/LtHAIGQDW94SUfZZCSM/Z2XOPSVhWcl7+p1Q+DQ6OIWRbmWTHf96/+NI9hYXELxRtKQQ2FMr5PH1qsVbaHM5wf7oq7YbGOPPYnkFSOvFWmu6ItPsZ3h+ytn0qGUwwmRlw5ZsE4JrWNvFHHkRW7YHTIz/Ksrwp9m0zw84kfyoYZJCzSHp82ev41P4n1cWujpHZOHur8iK32HOd3Vh+f6ily01G7RtFTnKyJ5reIWu5IUjIcfdGOOfYVyWsrtZDxjBGPzrqRYvpXhqC1kmaWSKNd8jtnLZ5x7c8e1cnrLZiDbs/NwMcf55rzMXG1ZNKx2UX+7avcz3xGjHcPXr0FZr6nagcSk8+h5q/1V8gY44FU2hiXpFH+AFOHmOV+hs+DNR0+bXza3XkvFcRMAsyjaWGCOv416JcS6dpFoxEltZ24OTgrGv58V4xehPscu1QAFyOPeuRu1HnR4QdPau2lZxsclS/Nch1ho7jW9QnRt4e5kcNnO4Fyc5/Gse7zHJs9R398/wCFa7x4bcBzWRfnF+cHJ2j+X+fyrdIxZVbmUY7dhxW54bsodV8VadZzQeZGXLyRuMqyqpYg/UisRid3XI6Dmuz+F1qZ/Ed3eEfJb220cdGduP0VqBGHq3h8eGPEktmylrST95A7jIdCenuR0P0z0NbtpBayEDZCW4G3A4/D/Pau68XaAdf0UxQrm9gPmW3Yk91z/tDj0yB6V55pd3A8EYucI4wodhjPpnjr/hVRYjbitbdCNkC5UdxVpwQuFPGPun17fypkYIG4Nkeo57VI77VBZhj371Yh+kXX2TWraQc5cIR7Nx/n6V6ORxjH1ry+HdHcQvgrtkBLHgDB/wD1V6dwcHPWsTU5VvkLDPTIpjHIJ/WprkYupgOPnOPzqvn8u1ec1Znqp3VyDOG7kY6Uw4Kj06Ypx+9jA79aac98VSIZR1CMG1UnosinNee+MoTHrVwM5yiEZ+g5/Q16RfDNjLz0GcVw3juL/T4pNv8ArLcD8QTWlJ++YV1eJwjAhj2q/obSLrdn5T7HaVVDZxjPH9aouMNzUtlMbe+t5x/yzkVh+BzXc9UcMdGet/2Xqcn3rtMf75rM1C0ubd2jluwCE8wcn5vYe/FdmjZBGF4HBHf61z/iFUSSBzAkh5zk9FHX+deatz1JRVrnPW9rLc3KwNOylgffoAf1zUKW7NOYt0pdX2Ngcd+c/hWmk9woWciJFdmjVlHIwCB/Kq0ouXWSVp25hEoUcd60RjYrLY3ckXyRybjjhiBxjn9aupoErAFmK8sDyDx27fnTZI/st/Gwkc4EbAk9QTg/0rqMAr+FTJ2KjFM5dvC6zwhJpSCSchf0rzy5Qo4DdR8v5cV7WQD0P6V5H4ggFvq93H6TufwPI/nW+Hk3dGGJgkk0ZucbDnp/jXrNtKJ7WGbtJGrfmAa8lz+7Hsa9O8PyGbQbNjg/u9v5Ej+lXiFomRhnq0ey/D+fzfDXl8fuZ3T88N/U11B5rhPhtPmHUIM9GR/zBB/kK7qso7Gs9wpM0UGrIAnIxVKQbWIq7Va4GDmtIMiZSHEjj3zTqRuJvqv8v/106tbmNgo70UUxBS96SjvTAd2qWH71Q1LCfmp3At596KX8KKBHy3oxB1yHHZG/lXTSHP51yNnMYtSRxuzg9K6X7SWZMkYIwQfWlBimi9ZuqSFieAOlXvtkQyA/6VlK+QcHjHWoBqFo9z9nFxGZQcbc9/SrcUyYto2/tcfZj9MUfbIgOC35Vmk85zUbQIQx+bPsxqeVD52WP7Zi3MAHOCSccEU+DUIVUliRu5Az+Fc7IjQzMfmR94PPIAz/APXqKR9sxLMQAhO0fmf8+9TyovmZt3Os20jiBopMyDAbI6/5FcvfXMzXzxKuwhSpz39KjknDXolJHljJxu5H+FTaisN1h4ZB5oXL5Jzj2qbDuZ6K0UiSxjcEXOQO/SvcdNkQ6TbDncYE/PArx3wo7L4ktI2LeW7lGB6EEEYx6V63p67LC3UdFjUD6Yrhxctkehg47s818bHHjINycSRY/If419JR/wCrX6V82+NS3/CXsAMgGI/oK+kYj+7X6CtI/AjCfxseaYaeelMNUiGRtSAjFK1Qk81rEyY5zhc9xWwkqoiMxyCPz4rCnY+UwGc4OMfSp4JUaKKK4kaOROgJxuz/APrqKtjWkakl8AVCKRkgYxjg1G18dpKHnqM9+ag+zQMQfPfIAAw3pSEwQZhRZZS38I5xWVos01RZMuZ3UAY2gn61Bcf6g59Qf1qyqIgOBjPJ+tQXTILaT5h09a5Kj5nobw0GKTsOfWiIlgQPU1xfi7U7+wvmjtruSNCgO1TjFYnh/XNRfWFgmv53UxMSCxxkDOaUsLKpFO4SqqOh3TSkPdDPSU/yFTbsq1cJpF7dT3Fs0lwzbpHDmSQjP+NdRqOqrpQj3QSXHnHaBDyQf/r1FCi6WJimxVpqdB2Rv2NqbiRAwwCSckenpW8ibTKPcfyFcroOvtcXsdnLYT2zqrOolPJGK3Li/kiuEaOAsjL8xzwDkY/r+Veu6sW7XPM5Glexbjb95JH/ABF93A+lVmyxtlBIO1sZHQ5rOk1aQXM0U6xRXAn8hGLkqSybk6D6Cq97qt9FounSJbxee04R2GSq8kfXqKFUi3a4crtc5HxTAw8RSNzuwpPFW/Bzf61CucLOOR6S5/rVfX55J9SnMxXzVYr8oxnBqrourHSRLMIRJ+/mQqWx12msZT5pHTCPLE7uWNSi7Ejx6hB+VS2SqLhTtUH12iuVPjzYpxpqfKe7+n4VGvxEk3kjTYRk8ZkOaFBXvcpzdrWNvRooLnTru2uYVmha4lRkIyCMiq+leFYdN1oXizyzW8SFbW3l58knrg/y+tYth4gmtYg0cMWLm7YkNn5cjPFX5fFN5Ax8uKH05BP9aj61RWkt0WoVUnyvRnTarJ5mmTgBThcnDAnqK4LUlXyvbcMZHHpWoPEd3dQPE8cAV1KttXBGaxtVciEtzu4xjtzXPiKsas04mtGEoQaZQBHPuoqNxxSxlQ7dPmXI5pG/THemkVcq3KlrWUf7J7VyFwv72MkdF7c812cgyjLnt61x1yCJVByODXTROasRlMjAGc9KwdTJGpMowSVXNdHjMec9PQVzmoA/2pIeeAPpwK6DAozkBeu3ivUPhNb7dG1G4K4aW4VRn0VTj9WNeZyDBGPlB7Y/wr174fR+T4bhU8F1LE+p3n+hFK4WOwU4Ze3Irz/xpoy6R4i+1worW2pZfYeiyj7w/HO7p3b2rvS27JrO8f2JvfCrTqMvZyLOAR/D0b9Dn8KqJLOBtdob/Rp3t3zyjjj8un5Voh5UUtIoJxy8WNw/A/jWbARPEu8AEjOcZBNWUadXCBEIU45zjFaCGu0csm9DK+3ndnOPqO31r1Czuo7yziniYlGX8QfQ1535yqPljTzEGfp+NdP4RufPtrsHCssgymMYyOv4/wBKzasaLVBqC7dQmGepz+lU846Gr+qjF+Tz8yg8VQPB/wAK86ekmepTd4Ihbqc9M9AadLNJcSF5W3PjBIUD27U0kE59s0mDk5I4pIGRTjdbyr1JQ/yrjfG0YksdLnxkFGXn6D/Cu2IDKR68VyPihN/ha1k7pNtz3/iFXDSSMqq91nmTj5qb6VJONsjfU1GK9E83qe+aZILjTbSfAzLCr/moqjrMStLbFxld5U56cg/4UeEZ/P8AC2nOeSIgn/fJI/pVrV7aSezbyRulQhlB7kV5j0kestYI5OeORobhUK+QlwDHjrz/APrFEC+Zb2/+3BJEefTmrsel36RmIGPY4VmyeQwxVtNKZWTBXakrOP8AdYdKrmRnys56aFlgs5A7N+7LHJ64IOK61CGRWH3SAazYtBMfEk7MFyACOACCCP1rUhiMFukeS2xcAt7Um7lRi0N/HPNeZ+M4PL1uYjo6o/6bf6V6djJ4/WuB8ewlb63lxxJCy/8AfJz/AFrSg7TMsQvcOKH3SK9F8Hyb/DyD/nnIy/1/rXnQ7j2rufAsu6wu4v7kgbH1H/1q6a69w5cO/fPWPh3L5et3ER6SQHH1DD/E16VXkvg6byPFVpzgPuQ8+qn+uK9ZNc1N6HTUWolFFJmtTIXNRTDKZqSmkZFUtyWZs3EkTe5H6UpNF38secfdYH9aTPrWyMWOpM0maAaoQufelzTM+9ANMRIDUsP36gBqWE/NxQBc59DRSjOKKAPlCEk3ilTzj6962zMqqBn5u5rDjdllDBhyuP1NTo2+Ub3woP3R39qzi7MqS0N2OWMrJI8gjVV27ieOlcjaRGTVYoUKn94BuHcDv/WtE3ct5qZhtlDokeCjD5ffiqd+kkM5uxiOXP3QMbTjtVOd3YlRsrnc9OTVeWSTLRxld2MjPpXP6b4hkuXEV0QGH8QH3h9PWrtxeRxzLJCS5Xrx1FacyM+Uh1LVSJPJIG4DnH0qrb3sZgTf1+ZNx9D0rOu3M91JNjcW/vdQKhy23OM46E9azbNEi3bhXlG47juGPerN8baKRVLBHAwCB+uapbkS2WRcb84PPSqk7F5PmZWPQEGlcLGr4alC+J7ADnMoH617JaDZBGvouOK8U0JzF4gsM5wZ0yM9sivbYh0HvXn4vdHp4L4WeYeNyx8VyYUkAxZ9uBX0fF/ql+lfOvjPJ8T3Z7KIv6f1r6KgP7lP90fyraPwI5p/Gx5php5pp4qkQyJuhqFutTNULcGtUZMaxwua0lWObAkRWHYEZrInOIWI7CqcOuzRKqlUYgD5j3rnrxk2nE6aDVnc6R7eGIgxxqpPXFT2zqyEA8g4Nc1Jr0kv8KD3po124T5kCDjHSolGTRrpY6ieLzLeRM43KR+lcD/Z4iYSGVztIY+mQa1/7fvmBBdOfRRWZeys9vLyRlDkfhVUU4JpkSV9RfEOh3OuagLmxkjkt/L2MQQfmBPv71haf4VvV1iQI37xY3Q5XA5GOD0PWt+0u5YITFE5SPP3V6VNBI7XCje2Seuahc8VZFuMZas5yPw/dWlzGjSrutptz+4ODxW3qrEw2p9J4z+tVDNNc3U67W3ggHGcnHFWZ47ia1hiW2mLh1Jbbxxg1yYlSbUmb0uVJpGwhVPGlqqdGgYfoK3twkSURrjdt4H1I/pWGbS4PinT7pUzEiFWOQMHbWraTlCgIyHLDORxhmP9aVKVomFVXZg6pbXZufMjhdyL20k7dAuCfwrXu4JZNMii8sh0uVfGe29jTdTuZVEMkUSsjMu8lsYx0qV9ThYyh2AyABjnuf8AGk69mEaLdmjivE1zHaarck8h5WIK/WudkvDBC4EZcSXR74xmNTWl4pDzXZFuSVDZ5rGuI2e227csJ4yfbMeP6V1UZOUbkTjyuxSvNQkRH8tQpxkZ57U7w9dvqTzJMQGigMisBjLAj9KbcWTyk4jfIGOG4/lTfCVjOL4EnYkkTLnqcfj9K2qXVNtGcPjSZ11zCkOlac6Z+aZWY57mi84JOaluYhJoEK5YeU3BHqDVK7hckk3Ep/EV40HzPU9FqyJrWQluPSm6lMxhYHkY7Doaq2kJ3jLy4HT5jUuoPmIZzwD07/jXXTiYyehkrcObjb5jYGcc+nantcO2RnBqE/8AH1yRjHFIeOPT1ruUUc3MxxmbnLY+hrn9RTZKrKOqdPetzv1+lY+rA4i+hHHTrWsFZkTd0VkwI1AJrn9QAOpS9cEDkHFdGuVVc9COK57UcDUH6Dgc1qzJFRlO77vP/wBb6f5969e8EKV0WFSMYjI/8dQ/zzXkqwSTTrFGMyO21QO7HgfzFe3aPYCwlltQMiKd4/TIww/9lFZOaUkjRRumzUGMdjxWmI0urBoZFykibW/EVmgegHpWnbHNun0xW0TFnjkUEmnX13YuDugk24buvY/lVk3sKriQEDPX0rd8caZFFq1vfMuI7keVIR2Ycg1zDweUQlwSR0Vug/GrEaELxyIfLIOR1781b8I3Lwa/5DNkSq0TfUfMD+hrn/Ke0bzYmDRnrjOOv/6609HnWHxJZzjhJHXOPfiomXE7PWhtuImPGUx+R/8Ar1mN1z3rZ1tcrA3oSP5Vj444Jrzq2k2enQ1pojb6c5puPXNPYcdeaMY9R9alM0GAdPyzXNa9GD4Tvkxnyp8gf8CH+NdPjkcCsTUoPM0nW4epA38/TP8ASqi9SJrQ8guxiVuvPPNQVavF+YHjpVXpXpR2PKe5678PJ/N8KonXypnX+Tf1rrZBliMcGuD+Fsu7T9Qg/uSo/wCYI/8AZa79lOR34rzq2k2epQ1popFAM8cVG3r+dWWXBxzTCpA5rO5rYgK4/wD1U1hwP1FSle3p2pGGQeMfhRcViDZ83Pr2rkPHltmxtZeTslKZA/vD/wCxrs8DOawfGlv53h6d/wCKJ0cfnj+RNaUpWmjKrG8GeRrw314rrPAcmL67iz96MNj1wf8A69co/wAszY7NXQeC5PL8SIn/AD0Rl/TP9K76usGedSdpo9P0qYW2sWU2fuTKT9MivaTXh20jp17V7Xbyie0hlHSRFb8xmuKkztqolpO9FBrcwEooNJTQineJmNx6g1XDblBx15q7OMrms6L/AFSjP3ePyraOxjJakgNLnFNzSE1ZI7NID6U3PrSg0xD81NCfmFV81LCfmoAvZopm73NFAHymg+bI64/TmoJpCkx2k89KlOFfsTgd/rVWfIk4HXtWC3Lexd8KkHUpz1zH6Z71d8VWyRQQygtvkc5BOR0qPwjGReXR/h2AZ/Gp/GT/ALqzHT5mJ/Ssm37axqkvZXOWiJWRSM9RmtQzbSAcgdR2zWSjFjtByOlWQHfGDgV0GBNI247u57jioz7cjtSgosZBY7gOCRURJBJPpikOw/kQNgnBbpTAqhflfJGO1Sq4FnMD13LgCr/huygv72VLhCyCPcACRzkUN2QJXZHpsZ/tyxxg4nQ5B9xXuKD58e9efwaRZW0gkgg2spDA7iefzrbTU7wv/rjyeuBXHXg6jVjvw81TTTOV8cHb4muRngrFx69K+i7c/uI/90V8yeLJpJdeDyPvJKA+/FfTNsc28R/2B/KtUrRSOeTvNslNNPAp1MNNEsjc1A5qZ+lV3PNaozZDOf3Tj2rmw3AFb95KI4ScZ5Arne/FRM1pEpJHNOViV601VJAPY1ba1EdrHP5sTbzjYrZdfqKzubogBwOufxpJ8tBL/un+VS7Bk0kgBhkA4+U0rjGWx6+nBq7bj/SUI4wc1RtSNvXPAqyrFcMDyO9S2NEisItWum6DCn9TWg92iQrIrAqE5A+lYU5Juix5JQZNTA/8S1vXYa4sV8NzWlHU3JdUijMAdGZskcHHVSKxNJ1O5a1njMhwk7hfXFOuzn7MR/fXP61Q0k4+2qc8XDf0rkUnyM0cUmazSs/32JHuaVG+RuO4qBTkHGKkQ/u3/D+dYSNYmFq5H2pifUY/KsubcDLuGDvhY/8Ajw4q/rWftXHTaKyrn/jzkbrxEcg/7Z/xr2cP/CR59Ze+yd1uDO4t3VScZyR71X8PF47+FHPKl1/DJo+cSk7SRtGeKj0STOqKQMfvWH41tNp02jKMXzpnVvzo8w7B3/rVK6/1YPHTrVsHdpt2uekh4/Cs+d8wIeDlRXjwWp6MtiKBiHXmnX/NvuOPeoIz+945qbUWxZdMkeld1JanNLYyT/x9oAD0A/SkJ5HehSPtsBz95e/4igjBrtRzjcfmaztUXdbR45PmY/nWiR0/wqreRh4hwMCTkfUf/qq47kszZWCTRqD0HSub1TIvmHsDn26Vt3Tk6iRwQvFY+sJ/pnvtBwM55z3rV7GaNv4fQ2k3jbTTdlRDHvl+cgAMillz+KivX4VX+0bpgysv2sEEHj5sf/F14p4UTzdSibo0cbHB78Y/9mr1LTL7cZozgFXjcYHT7o/pXHUpvnU7m9OStym5Km3BA5zzk1atGzAOeQTmmXSHySRgd6bZN8jDuGNdVJ3MaqsU/Flh/aHh+4VVzJCvmp+HP8q8+hkM1ur5BbGCG716wSCOcYPY15be2J0nVbm06CNt0bY6oeR9eP1FbGJUM0aEq8RXnk4ziqzxSpLG9vzCDvVgfun/AArQ3JLkZXzOw61nTWZjYuuUz129KUlcpOx6PdTC80i2uBj58N19RWbtIGMHrS6HcC58MLF1eB9rc+9SMoz7eleZidJnq4XWBCVyMCk24qcx9aAnbpmsUzdoh2nPSs2WHzLrUICOJbf+hFa+36ZqjIuzWovSSArz7H/69VclxPDr1du3qSMjkVRHWtjW4TDeXEfHyTMuPoSKx69WDvE8eatI9B+FcuNS1CD+/Er/AJHH/s1epFcqK+fNK1WfSZnmt5ZopGXbuifacenQ+grTfxlqb9bu9I/6+mH8gK5q2HlOd0ddDExhDlZ7U6HOcc+9Rsu3k4H6V4e/ia/c5aWZj/tXEh/9mqtJrE8v30Rv94s38zULCy7lvGR7HuLz26AFp4k/3nAqvJqemxjD39qvbmZR/WvF5NSiNoiJbKtwD80hCkH8Nv8AWoP7RuezIPpGo/pVLCeZDxnkezNr2jrjOp2ue/70H+VZOu63pF1o15bR38bySQsEVQTlscdq8u/tG7xgTuPocU0310etxL/32auOGSd7kSxbatYjnGJTwfXmtLw7KIvEli/Yyhfz4/rWUzM5yxJJ7k1PazG3u7ecHlHVvyNdEleNjli7STPamTINep+HJxceHrJs5xHs/wC+eP6V5p5YYZzkY6j0rvvBshbQzGf+WcrDH15/qa82k/esepWj7tzoPeg0ZpK6jkCk70tN6c00IRxlayR8ssq9MNn861/0rJuBsvWH95QfyzWsCJoTNGf1pmefejPPStTEfmjNMz1ozxSAkDc1LCRnNV881JEfmpgXePQflRUe73ooA+WgAWHTgD86o3HD8kdT2rUtWHmHdAH+UYzzjis242tdHGAc9+KwW5b2I4rmSAN5bkZAzz6VHc3k90R50rPt6AsTip/s+yBpGYNx25GKhtI1lmIcHAHQVem5OuwyHAPI57YqyDn+IDANQRhVudsn3RnI/Oo2Y9+tAEolzwDwOKf5UzgsqEj1xUOB5JO3r0q1p7ssgXdwynjr0pDFjhf7JcF0KMMYJrX8Fhj4ltoMblnPluvPTPaqkizSxNHwCAAD0xitzwDA9t4y05nK7RKucfWs67tTbNKPxo9Rm8MRQvteKWI4+63HFVf+EdWXULK1gPltcPs3NkgHFehaqY9QvvNiyV2gDIrndZD6ZNp96gy0U4YL68HivnqWJn7fkUro9uXJKj8KUjxv4jaPNofi+SxldZHTy2yo4IKg19EWh/0WL/cX+VfPvxT1Y614ylvDEIhtjXZ944Ar3+zP+hw9fuL/ACr3/so8WzTdywxpp/WkJppNNAxHPFVnPNTsarv3rRGTKl2m+L8QeawVAPX+Vbd7JsjHAOWArGQFTuxms6mhtR2JU6BR69alVG3hOMkjHzVEuSc9KkjO0/erNs3SZPdW01nOYpdpYAHKnI5GagcM6sADnBqTzGKlVU59hTvImZSVR847KajmRVmVrSJgigjHyitC2tI5nZZLhYQFyGYZz7VWtbO4aNCUl+6Odhp7GKJiGljUjghpAD+tK6Y0nYqzkCdef4Tz+NPX5rBwf7pqtdXdvHdKxuINqoQxEqnHT0P6Uz+2NIWxlLapbLJtYiMtkk9q58RFuOhpTkk9WXLps29s3PVP51S03JuL5f8Aptn6cCr91g2Fo8DCRHRGDYxwazY5pobq68rYSXBOfpXLCneLRrJ6mmAw7HFSxxM8cpzjC5x+Irn9TvdTsGAdlG4bgFCmqdt4muLZpXumMkZQrtUqDnj3qvqkmroz9vFOzL+qxE3SE9wKzL79xbz4xxGpz9HH+NQXPimGacM9tJtxjG8A5/Kla7TULN5EjaNWhlXG4NyCprupU3GCTOec4yk7Bfaqba8EfJ81Qn51X0f5bxmIw32ngDsOM/zq6BCCgxnK55I9vrVexZI72ckBT5oIJcDsPatuVcrM+Z3OjRwLe9UkD5wRnjqKoMDJbIFIJ2jjI61ae509EmRprcKzZG+Udxj2qsNY02C3jQ3VqnGARJz/ADrzVSlfRHa5xtuVo4pBNyMepq5eQGS0KkjkVTfXdPY5+3Iy9cgkinz+ILZ4lgjuCc4AG0jNdMIST2MZSjbcz2Tbb9iwGAQOnJpGR24COfoDUdxevDASsjqGz0J/z3qnLqLhN375xjgDJJ+gzXUrnPdGh5EpH+ok4/2DUbwyCNyyHC4P0qKez12GJJW0K7KSPsX5o/vYJ5+bjgdTjt6jMcdrre9mfSBFCv8ArHe8hyo9dobJ69BVqMiXKJzkjZv3GR16fjVHUxuuwScfu1z+tWUf/Sbhs8Zxj2qtfn/SwQONijp04rSWxmty54U41gR7gnyPkt/X8q7qzlWO+lUSrlohjGcHG4/0/SuC0V9msIytjKtj8q6eymI1mAE58xGXn1IP+NRa6K5rM9HuNRyvkhBjAGe/QUtg4JcjqCKyd4YRt6opzn2FXtOkGZBjHH4VrGKWxnKbluawbvXHeN7YJLZ3wwMgwucfiv8ANvzrrN/XB71i+KoftGgz4HMZEgI9qsg88nhWQjaSj+o71D9qvrJf36i4g7kffA/rVyIK+AR83UU6aJjHt8tmB5GBmgZr+GLyJ5Joo2zHPHuXHYjFdH5JIz+PNeZafLNp+oiVCVMb5ZOcEGugufipplpI8TafdtIv3gSijP5mvOxVOc5JxR6WEqwhFqTOu8ntis3VLtrKRAqZ3DOTXKyfGKJc+VpH/fdxj+Smsu/+Kct26n+zLT5QQNzs3/sorCOGrdjoliqPc6k63NuwqLTUv5rjULZljTeAwG5wg59SeK4iX4j3Lgf8S6y46ZQnH61CfiLfZBWwsBz/AM8T/jWn1aq+hk8VS7md4mO/V744AJmYkA5wSa588Ve1PVZNUvZbqaNEaTGVjG1RgY4H4VRY5NehTi4xSZ5tSScm0NopaUKT0BrQgbRVhLG7lx5dtM+f7qE1aTw9rMozHpV6w9RA3+FLmQ+V9jNoraXwlr7f8wq4X/fXb/OrCeBvED9bSJP9+5iX+bVLqQXUapzfQ52iurT4fawxAeWxjz/euVP8s1Ovw7vDy+qaaB/stI3/ALJS9tT7lexqdjjad2H1rrZ/AU8ajytRt5WzyPLcDH1xVe48IXSJH5cwkbGGAjIx6cnrS9tB9Q9jNdDpbb4hWEdjBG9pO0ixqrHeoyQBnvXo3ws8UweIF1OGKFojAY2wzZzncP6CvEF8G37MQZIlHbdnJ/SvQ/hVZnwtrl097eQeRcwbDjdwwYEdvrWHJSTvHc6FOq9JbHuVJ9aojWLA/wDLwB/wE/4Uv9q2OP8Aj4Ufgf8ACmIuUh/SqZ1ax5H2lP1o/tWxP/LylUhFuszUcJcwN/eDL/I1N/a9gePtUeazdV1jTzDEwvIcrIBy2Paqi9SJLQfuoLcVmnWdN7X1ufpIDTDrunjj7Rnv8qMf5Ct7mNjU3cUbjWO3iGyAyPPb6Qt/hUbeJIMEra3knssY/qaLoLG4G75qSJua5s+I8YK6fcgf7ZVcfrTR4mkDALaIp/25wD+gNF0Kx128+lFcl/wlj94Igf8Arqf/AImii6HY8Us7hIZT5jAfd4/CqV4ITI7CIHqfvHk1ZgtVudxcuCMdOe1SGzTedxJwOaySKexmo2dPY4IxuAwKqW7urMUHPrXQiyiNuEVT8xJwMiqdlaItww2kMB3+tO4rGSoaSbodxowy5DDB6Ee9bKW0f9okbAfmzz37+tQyxxrKwZF3bscjvS5h2MsM+3k9qltN/wBojweAavgBQAqJ/wB8irdvcNCT9wg9ex78UuYfKRrcqTja2Sey1t+HNQg07XbK+mErRI+5gqZOKrwywSN8zEE++M12PhrVZ4b23traO1TcwHmG3Utj6nmsMRN8jVjooQvJantuh20WpaZBfRuVjmQOqsuCAfWna34WXVrWOITiLY+4tsznj60/R73baqXEskmOWCcH8uK0m1Btvy2srcdsD+ZrzMJHC8qbWo6s6qqPU+U/iNa/YvEbqG3A7TkD619BWbD7HD/1zX+Qr5/+KEobxY+1Ng2pgEjIxmvfLRh9jgOf+WS/yFenTu6cWyallUZb3e9MLUzd70hYCtEZMGPWoHNOZ6hZ855rVGbKGpNiEYzncv8AOsY67GgK+ZaA9MeUmc/jWtqB3QjA6MD+tefvo9+9zKyy26qzkjdJ2z6AVlVpxluaUZuOx1H/AAkiwg5uolwOqxoP6VTfxq/2f7RG+oSxfLh47VwDnG3B2Ac5GPrWSvhy5lGJLu1X/gTf/E1dm0rVZ9Ik06XXbeOORQpeG3O7g5yDuHv+Z9sZRo0ups6tToOfxfqLthdM11vcxFB+pFQt4k1+ckQaLcnt+9vo0JJ9i1Rf8Izckkv4rvQTyfKt0T27GpF8J20g/wBI8Ra1Ljv9oIH061p7OiuhHtKrLNk3ie7077TDb6ciqxiIe7MnzA46qMdffsfbPGNNNcTSyu+0ytvKqykAnrg8120PhjSba18hNR1JoyxYr5y4JPJJ+XknAoTwzoES4CXBHQBpcflgCmpU18KE1OW7OSjZhDOAxJyhyce/oKx7hS1w7FgGxg5P/wBavU9K0vwsmozLfRstt5II3SOcsD7GtZrX4bI5drJZG6nKyn+ZrSKTVzKV07EvgfTbbW/BdpJczBLmEeW+xtwAHK/oRW2/g7TGclbyZGb7xVhz+YrJsPEfgzQ4Wi0yxkgjc5ZYoQAT68mpZPiDoijKWE7fUKP61ySwvvXTNlWlYS6+Gun3swefWrsgcKAEGPxxVV/hNo+0BdUvW998Y/8AZKe3xJsQf3WluRjvIB/SoW+Jef8AV6ao/wB6TP8ASrVBpW5iHJt3sTXHwm0fyFMd5deYBySy9fptqnqXgiHT7zT9M095XiuYp1aWQbgjFRjOBx0/SlX4j3UkoQ21vGCfvEM2P1FVZvHN/cXbbRAGtd0iEJ1+Ug9T70/ZW+0wUpdgT4SXSxqw1eDOOhibj9agHwxnhjmiNtZ3MrIwE5z949+Rn/8AX7CnXHxA1U6eksN9H5xbaYxCv59KzX8c+IpD/wAf7KP9mNRn9K1UbdSdR4+Gut+WkY03Sht5DM4wc9eNhqyvw68QHZ+60SPaAMYbjH0UVlv4q8QyZzqN3+Bx/Kq7a5rk33r68PHTzWqlyiszq7b4fa5HaGCS/wBPGXLkrG5xk5x2/XPb0FMi+F7RFWm1dBtx0h/+yrjpdQ1Bj+9upT/vSn+tRDUo0b9/eRLgdGnHX8TQ2uwKPmeijwJp8hEcuq5AzwqL/XNI3gHRUQ79TkB7OiIpHvn1ri7O8AjEsV5asGGRsuULEf7oOf0q0msGQY3E4/CuSpTqSleLsjog4pWZ1eo+H9LvbdILrxDqjxxSLKgiljQhxnByEyOvrWNP4X8Oojt9s1i4kUEr5t85Geo44zzWf/akSr828N9P/r1CdXj3Y3KfYtj+lONOp1kx3h2PPIgFuJRuJbPPuar3rh71o2+6UBz+FT3WLfVpl29WPU9aoaoJGuFZD83lBvTHWuuS0OZbnQeE2ii12LzYkkcK+0OoYH5SehrtLi4xLGywwIFccIgX+leZ6LfOb+B02iUPtO/JH5CuwluruRdqyWwPXHlkZ7df89KjlL5jqra+xDGrHG1AOvBx+FWYNUjSQhiSAOa4tNRvowN9uJO37tgcfnipoNTdWbz7e5AxwBEW/lnmi8kJKLO2bW7cZwG9uKr3WsxywSxeUzBlI69q5b+0mYHZYXzZ6YiA/mRTWv7rHy6VeHnqfLGf/HqOeQ+SJmSF4ziMHcvAwM5p8erSRp5VxEUc8BuSKPMcTnzIDCzciNiCf0qdV8yLY8Yx0APWtlqjJ7mIpdNQYyPvZ1zkVmaj4duNTvQ1oYgdvz7ztArb1P7NatGked45HOcD+lS6Y7fakAKjf69PpWc21saRSejOaTwHqTdbi1B9N5P9KsJ8PrsnD3sIOcfKpNejLYuRkyKM88DNSfYVzyx/BTXP7aZ0exgeeJ8PVwC+ot/wGH/69Tr4AtNozdXDH2VRXfCzUDJZuuegpBbAZ4PP6Uvaz7j9lDscCPAlopzvuGHocf0qaPwpaJtzaIdvqTz9fWu6FsmfuHI96kFtGM/L37jNHtJ9w9nA5aHTLRcf6Baj6W6E/wDoNa1vFAgwBs9kUL/IVq/Z0B6dvTrTlgX2rOV2XFJFNVt/785/7aGpDBasPmSVvqSateWmchsUYUdx171DizVSRVFpZjj7P/46BTxbQD7sAA+lTbowPvqPXGOaaXh6+aB9BS9mw50NMaKNqoAOgwM0woDjavt0qQzxD+MkfjxUbXUY4w3FUqbJdRDGg3ZIBzUbWpOMD9aebwIchGP/AAKmnUGxgKv51XsmT7REZsGPRgPahdKbeGDx8Hpg0pv5OvTPTimveSkfeP5VapMh1Ea/9o6qDnzYSB/snig6xqoAxLFn121im6l6GVsexphmYj7+fatFFmfMjb/tzVVI/wCPfH+0tKfEt8uN6wd++P6VgtJkn1qNjg9OlWokuRvyeKrraNqRZ7nr/SqFz4hvJ4JI8R7iOMoCuc+mKzDu5GMfjTemR1GapRIbHnV9Swu0W6H0EQqNtV1UnidVJ9IxTT75596YcDuSO1WQSfbtRPW9k/4CAKjea6fG69usDnCyYz+VJzwO1Jghj1z7DrQA0tJnJmuGPT5pWP8AWiOKMkkx5oJOOM5pUZgc5P8AjQIsFEBx5afnRSbv9pPxzRQM5CwjcbwIXbJ4wpPYc1I9vdMxK28uOvEZ/wAKqNdTZCx3Dhf7iOccfQ01Y7qckBJ3+iMf5VncqxdW3u1OTFIPquP51We0lRjIojU5JwZFGani0PU53HlaZeSHB+7Ax/pV1PCWvv8Ac0W++XGcwEfzpc8e4creyMp7VmfeJ4lfjOZV/oaWS3SVRuuoA3cgk5/IVsx+BfFEwwuiXQA6bgFx+ZFX7b4ceK3j2HTGjB5O6dAD+GaTqQ7jUJdjk3solGGvYz3OA3+FRskCcee57HCe9d0fhN4mmIJNsoGOHm/PoDVmP4M6y+PMv7OMexY/0pe2prqP2c+x52HjVRiSXjp8o/xq5b69PY3CSwvKHU5U7RjPrznNehp8F5xgy6tED/sxk/zqC7+EVvZQmS4109eES35Ptyx/lUuvSejZcadVao5n/haHiRF2pfThO+3YMf8AjlU7v4heIbobZL+5YAnH73b/AOggV09r4F0WAZuFubo9w77VP4KKs23gvQYMuLAsCfuyOW/Ks19XjtEvkrPdnmy/bvE2qW1tGglu55FjjLOfmJ4AJY9Oa+nbazvEtoYzbtuRFUncOoH1rgtF0PR9P1qzvhZQW4hmWQyklQgB69cY+teiN4x8Or11my/4DKD/ACrppuNRdjCopQeon2S7z/qef94f40GwvCPuJ+LVG3jbw9/DqSPnpsjdv5CmHxro5+5Jcyf7trJ/Va05YLqZXk+hKdLvG/55D/gR/wAKX+w7hus0Y/Amqx8Z2RHyWeoN9IMfzIpp8aRYymlX5z0LeWM/+PU7wXUOWb6E8vht5l2vdj14j/8Ar1w2p6ZPp2rT2yb5ghBD4AzkZ/rXXN4ymIHl6RIc/wB+4Vf5Zrz3xLpN/rut3GqiKO3WfarRCUtgqAAc4Gc4qJyg+pcIzXQ0BFccf6tT6GVR/WpFtJnGWuLdfYzA/wAq55fD+oIADctxwMSGpl8OXbxF2nDFeo3k8etZ+53NPf7G39iIPzXluB7Mx/kKd9mtl5bU4l/4A3+FYH/COs33pAD9OarXXh+YMTC8skmOVU4B+nvTvAVpHUBdLUENq6knsqf4mmtcaNGDuvpmI/uov/xVcMbTB6PgHByTkU9bIFiWTHrk9PSn7oanRXWqaUsm6K5lAIwd8fX2wBWPNrtuCQkdw4HGcIM/mwqNLaIH/Vj3zxSm1iONqjHcdxRdC5WQ/wBsq33bWcsP7zxj+TGmNrLkcWkf/A7gjH5KatrboFJCpnsc4I+tOWOMZDLFjvgYp3QcrM86teHlLW1Ht5rt/wCyCmjVNRfGxbUY54idj+rCtQRRdRtIHtTjGMgHa2T1FF0HKzLa+1NxkTRqfVbXt+LmmpPq6/Mt1ICQQcRRjg9RyDWtgAfK+RnHWgiL5htYADrjIouHKZONRcEG6uPQ4Ma/qEpptrz/AJ7XR56m4YfyIrYIUgAMD+gphKIeu3HTtRcXKjKa0nbgyS7f9qeQ8/8AfVRjSR951jYD+/ls/mTWyZY9uWKnjqRimi5t93oP9ocUXYWRlDRlP/LO269oVz/Knrpwi+6EUd9saj+QrS+0RnkNuGfT/P8ASozcAEEeYD05p3YWRUFmyDHny89g/Wmmx6bpJjnnlzVg3Tfexu/Q037R1JV1I9DnFGotCu2nRkgtkn6nrQLGMZATHXjHapzMxBIAZcYxUZkdVwu0nHAIp2YXRkayuydZFA5A5HFVrmU/6O4JwVKk/j/9etHUFNwseBkgkEgY7VD/AGPJdQ4jYKVYGqtoTfUz9MxbeILaQoGTeGw3GQK9Aj1eyHLwlM9PkBH6HNcBZqzXcLglXjJBIOD0NaZeUsdhY45xu5xUctylKx3kGp2MgG25iRfRhs/9CFXk8ubmORHGOi4avMWZimXIAbpuGOntj/P8gIT0GS2cFevb/wCtUul5lKr5HqAg6cDPQ8Y/lSmId3Iyex9//wBdebxy6moJinuOByBLwPwzVpNW1aAhWeIjPWRcZ/Ie/wDnil7MftUdNq9orlHRhvHqQTWau4YGSW6EAdKqx63JMQskSNk/fWVRjnrtzV1XDklcYb0raCsrGUnd3Mq4tnO6V1yS2c4ycCkhcxOjhclSPzrQeOWZG5Hl56k1mMPnA6gHHPvwalotM7qzvoZLZcqSVHt+dTm8TnCk/XFctpFyfL8vPHYd/atXzPUD644rL2aNfaM0Dejp5Q/OmfambkKoqkATnhseopQTjhv0o5EHOy0bqTJ4A96YbiXPL4/4DUALAcgHNKd3TA9iDT5ELnY8zOcgseaaZHODmT0puDnHIHuKXHsSPrRyoOZjd7EHO4n3NNyWGe31qTap4x+RpCnt/WjlDmITk9X59jQCwAxn8RUhUg9fz70woeuP1p2Fcbk5zuP5Um5jnmnj5ednPSkJVuoOfpRYVxhOR05pDIfTPb8KlAT1AzSBVHT09KdguM85gBuXikMmOuc1ISMgcZ9aYcZwMZ7nFFhXGF/rgUeYNuTTsDPGPwFIR0INOwriFlI6D8qQ7CP/AK1IwJ5yMdetMaNs5GAadhXHYU5H9aaQrA4/DJo2uCSWH1FNw2OSKADaB3FJjHfp2p3z8jC5FNPA5Ue4xQAwk565pvTjNPLqOGXikDKccMeOoFFxWIyevU8f3acp56CpAoP3d2PUjrUiQEjAGB70XHyjODyVGe9FWRAMdE/OilcOU9yXTbRFAS3hUDoBGKlW3RRwB+VFFeGdd2BjGfSgwjuTRRVJDuxhRV7ZpCAOcUUU7FJjWAHYUmflzgUUUi0V7iUxQtIADgdK5K78y8l8yaQn0AGAKKKqBZVFohcAk80jW6E5OeOgz0oorQQgt45Mqw3A8HdzmnNbQRSMkcSLtOPujtRRVxZLQqrxxgcelOdcsF9AKKKq5NhpGDjrz/8AX/pTgoaFT6MQKKKaEMCDn6Z6VLHGGLKTxtJ/IZ/pRRTAjaFQc0RoqyhlGCPXnNFFMRNNZxo6leFYBgMdM9qi+xIxYE49wORRRQBha1psP2M34yJFba4/v4bGfY/55rniFCD5R8wooraGxjPciaVVRf3eQzY5PTvnp7VELtljBA6vt560UVoZ3Hi5dmUYGcDnHrUUlzImw5GG9Bj/AD0ooqrCbGNdS7xhvmB6kCnLJL5nlvIzAjd6UUU7E3FbKMGLFvmA/OmuWDr87fN056CiinYQwfNGp6ZFKshAVG5LHGRxRRQA4KC23GDjqOOlCx4/iPWiikJimEbhgke4pme4J69zRRTAQFA20rndxnNRyzeWxIBJz1JoooAhFwJCD5YB9c/59avxQCZc5C/RRRRTEPnsDBF5rSBwZAMbcdfxpVh2cBjwRg0UU0JnNfZxHfIQejgHjrzXaR6NZyPsCsD9RjpnsBRRWaLB9LtYJAoVyR3EjDP4ZpVsbUqSYVb2f5v50UVQCC1slIH2OLJOOFH+FWGtI4Iw6pHz6JiiikMRSWA56VnsSZzjjJx0ooqokyM7UdXltrj7HGgYkFt8jE9z2GPSsc6uWeRZYFYpxlG25/A5ooqZblR2Lmjaz5155KwbOAc78/0rtMAgZz0B4NFFJjQvlgjcCRxn9KMEdGP40UUAKAfm5+lCpuPBI5+tFFAAYiuDu/SoixDcnPOOaKKAELkEA4NORtxVe570UUASTRlPm3559Pr/AIVACSTz6DpRRTADksRmggfrRRQAGEgE7vwxUe35j0/KiikA0LuBOcYpdnA5oooAaQVzg8CnICRnd9MiiigCUQ56sD+FL9nXuT+FFFJjFW1RwSCwx6nNS/Y0Q4znAoopNjSEFupb+HJOM7aayBRkYHpRRSuOwwDgMDjOacke9sEj06UUUAhViVsjA9elOWFByQfwoopDHGJMn5aKKKAP/9k=" - } - ] - } - ] - }, { "language": "markdown", "source": [ - "## Do Inference" + "## Initialize helper functions\n" ], "outputs": [] }, { "language": "typescript", "source": [ - "const tensorData = new Float32Array(inputImage);\nconst tensor = new ov.Tensor(ov.element.f32, Int32Array.from(inputLayer.shape), tensorData);\n\nconst inferRequest = compiledModel.createInferRequest();\ninferRequest.setInputTensor(tensor);\ninferRequest.infer();\n\nconst output = inferRequest.getTensor(outputLayer);\nconst { data: boxes } = output;\n" + "// Function to extract bounding boxes from the model output\nfunction extractBoundingBoxes(output) {\n const { data: boxes } = output;\n const foldingCoefficient = 5;\n const numberOfBoxes = boxes.length / foldingCoefficient;\n\n return setShape(boxes, [numberOfBoxes, foldingCoefficient]);\n}\n\n// Function to adjust bounding box coordinates by a given ratio\nfunction multiplyByRatio(ratioX, ratioY, box) {\n const scaleShape = (shape, idx) => {\n const position = idx % 2\n ? Math.max(shape * ratioY, 10)\n : shape * ratioX;\n\n return Math.floor(position);\n }\n\n return box.map(scaleShape);\n}" ], "outputs": [] }, @@ -134,14 +96,14 @@ { "language": "typescript", "source": [ - "// For each detection, the description is in the [x_min, y_min, x_max, y_max, conf] format:\n// The image passed here is in BGR format with changed width and height. To display it in colors expected by matplotlib, use cvtColor function\nfunction convertResultToImage(bgrImage, resizedImage, boxes, componentsCount, options) {\n\tconst defaultOptions = { threshold: 0.3, confLabels: true };\n\tconst { threshold, confLabels } = Object.assign(defaultOptions, options);\n\n\t// Define colors for boxes and descriptions.\n\tconst colors = { red: [255, 0, 0, 255], green: [0, 255, 0, 255] };\n\n\t// Fetch the image shapes to calculate a ratio.\n\tconst [realY, realX] = [bgrImage.rows, bgrImage.cols];\n\tconst [resizedY, resizedX] = [resizedImage.rows, resizedImage.cols];\n\n\tconst [ratioX, ratioY] = [realX / resizedX, realY / resizedY];\n\n\t// Convert the base image from BGR to RGB format.\n\tconst rgbImage = new cv.Mat();\n\tcv.cvtColor(bgrImage, rgbImage, cv.COLOR_BGR2RGB);\n\n\t// Iterate through non-zero boxes.\n\tfor (let i = 0; i < boxes.length; i += componentsCount) {\n\t\tconst box = boxes.slice(i, i + componentsCount);\n\n\t\t// Pick a confidence factor from the last place in an array.\n\t\tconst conf = box[box.length - 1];\n\n\t\tif (conf < threshold) continue;\n\n\t\t// Convert float to int and multiply corner position of each box by x and y ratio.\n\t\t// If the bounding box is found at the top of the image, \n\t\t// position the upper box bar little lower to make it visible on the image. \n\t\tconst [xMin, yMin, xMax, yMax] = box.slice(0, -1).map((cornerPosition, idx) =>\n\t\t\tidx % 2\n\t\t\t\t? parseInt(Math.max(cornerPosition * ratioY, 10))\n\t\t\t\t: parseInt(cornerPosition * ratioX)\n\t\t);\n\t\t// Draw a box based on the position, parameters in rectangle function are: image, start_point, end_point, color, thickness.\n\t\tcv.rectangle(rgbImage, new cv.Point(xMin, yMin), new cv.Point(xMax, yMax), colors.green, 2);\n\n\t\t// Add text to the image based on position and confidence.\n\t\t// Parameters in text function are: image, text, bottom-left_corner_textfield, font, font_scale, color, thickness, line_type.\n\t\tif (confLabels)\n\t\t\tcv.putText(\n\t\t\t\trgbImage,\n\t\t\t\t`${conf.toFixed(2)}`,\n\t\t\t\tnew cv.Point(xMin, yMin - 10),\n\t\t\t\tcv.FONT_HERSHEY_SIMPLEX,\n\t\t\t\t0.8,\n\t\t\t\tcolors.red,\n\t\t\t\t1,\n\t\t\t\tcv.LINE_AA,\n\t\t\t);\n\t}\n\n\treturn rgbImage;\n}\n\nconst [ __, componentsCount] = output.getShape();\nconst img = convertResultToImage(image, resizedImage, boxes, componentsCount, { confLabels: false });\n\ndisplayArrayAsImage(img.data, originalWidth, originalHeight, display);\n" + "// Calculate ratios\nconst [ratioX, ratioY] = [img.width / inputWidth, img.height / inputHeight];\nconst boundingBoxesArray = extractBoundingBoxes(result[outputLayer]);\n// Resize bounding boxes to the original image size\nconst boundingBoxesOriginalSizeArray = boundingBoxesArray.map(box =>\n [...multiplyByRatio(ratioX, ratioY, box), box[4]]);\n\n// Takes original image and bounding boxes\n// and returns the image with bounding boxes drawn on it\nasync function putBoundingBoxesOnImage(img, boxes, threshold = 0.3) {\n let finalImage = img;\n\n for (const box of boxes) {\n const conf = box[box.length - 1];\n\n if (conf < threshold) continue;\n\n const [xMin, yMin, xMax, yMax] = box;\n\n finalImage = finalImage.drawRect(\n xMin, yMin,\n xMax - xMin, yMax - yMin,\n { color: 'red', width: 3 },\n );\n }\n\n return finalImage;\n}\n\n\nconst resultImg = await putBoundingBoxesOnImage(\n img,\n boundingBoxesOriginalSizeArray,\n);\nresultImg.display(display);\n" ], "outputs": [ { "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAIFArIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1SXSkkG1ihGQehHQ59a429+D3h+9klkY3SNKxZjHcY5Jz/Epr0RvvfjSgVEcPSg/djYp1pyWrPJpvgbpjKVh1G/j/AN5kf+grPufgXMYysGtvj/btR/R69rFLVciFzM8Cn+CetpjytRtHx/fSRc/kpqhL8JfE0RYqLSX/AHJtv/oQFfRtLS9mh8x8yP8ADjxXGrE6cGPolzE38nqtJ4M8TQld+i6gRnkrAzD81zX1IUVuqg/hTDbQHrEn/fNL2Ycx8py6VqVqx+0Wd3F/11gZcfmKgG4ZBbnmvrMWsI6KV/3SRUU2mWlwCJYhIPR/m/nS9mw5j5W+fAzxTwW3AjJAFfS03hHQpx8+l2Z9zbRn/wBlqjJ8PfDshz/Z1uPopX/0Eip9mylJHzZEzYI3H7x6/Wp0ZiseM/d6173L8KfDjA+XZhDnOUmkH82NUZfg/pDABJLpAOBtuAR+sdSqbTLc00eJbjuY8kdOKYWbHPPocV7DN8Gockw6lcL7NGj/ANVrOn+Dd8ABDqStj/npBj+TGqsyLnmJPI5FKCSSNxrvpvhJr0Zys9k49MyD/wBkqhN8MvE0IbZaW8mf7twgH/jxFFmByO9gn3j+PanAuGAyfxrem8DeJIcZ0edsd49sn/oJNUpPD2tW5Jm0nUEAHVrZwP5UrDuZ4JYnp+VLuPHofSnNBNEAJUdGzjDLjrQFJYZ5xUlCgnOOfes+d3XU4scHA/rWiAc5x7Vm3fF3EQM57596mQ0aMUm6PrnnAp2/AznNRwHbFv754pxOAFweaUdimBftkY9qCxAz+tAIJPtTSV25I6+tUSOJbOc03cc8nHelbHTn3o25Ht069qkYmfu5yD2oyScnj60u05xyTTgpB49KVxgCRTWO5Dj09afg5HAye1KVODnnIzUsZl2TsplHPbI/OtLYjYbbgHms2Bis0q4/WteHJgTjgipe41sQ+SpxlTSGAf3m9OtXAoAzwKUqMcLmi4WKX2f+EMffNM+ynPOCO2RWhsB4oCZPFLmHYoCyLAEqhP5UNZnHCd+gNagj9BSFc5qeYfKYxtW7Ix9/Wo2tGByQ6jPcVulMDnjHrTGjOwnB6Ucwcpzo3K/r2qdYtyAnAPrSTrtkPHetSwVXtACoPJB4zQ2CRlPZAIWaJD34ArF1C12xl0hC8/wjFdfdJHgoFGO+KxL6P/RZMdAc/jW1KTTM5o5bawPANHmOP4m/OrxT3PPrUezII6/hXZzHPYq+Y/8AePFHmHvU5jGM4H5Unlgj7uPei6EQbzS7+MU8xjHIIpjIAuRmnoGopJ4JzzTd555609uY09gR+v8A9eo9poAXfil8w+lNwaTB9KYXJFcA8jinmYYwM49ahwfQ0lKwXJd4IxRuFRUUWAc/NNoopiHI5SRXBwQcg19uaPffb9EsLwHInt45fzUGviGvrv4X3pv/AIa6HKTkrAYj/wAAYr/SuLGLRM2o7nXlzSbzSUhrhOiyF3GjcaSkqh2FViHYZ9DT9x9ajHDj3Bp1ddPWJEh24+tG4+tJRViF3H1o3NTaKYh28+tQzMd8bZ74qWopwfLyOxzQxoXc3rSb29aTtSGkAu808OcVHTl6VLQ0O3tVS+lmjsriUT+XsQsCEzjAz+NWGOB0yT0FVLyyF3aTQzOxEileOAv0Hf8AGpavoPY46TxpbC2huJLdpZJNrBSqgpn+LO44+nB9ap6ze6fKJQ11cm7JCNGMny/baOAT6MQcVn3ekwXlhd2YmZLzT2MfmNgkqBz379f0riFvtkUNs06tPBJhcAFVABODkc4PvjNY1KV4tGTqNM6z7bpwHK3ue+JUH6eXRVJfFjBQHS4LAckAdfzorx/Y1OxtzRPdG+9Sjmkb71KK+wZ56HClpBxSikMWlpOtLQAtFFFABS0lLQAUtJS0hhS0lFAC0UlFAC0daSloAaUQ9VU/UU3yIv8Anmn4CpKM0ARNbRMMFePTJqnNoOlXBzLYW0me7xK38xWhmilZDuYM3gvQJs7tKsx9LdB/ICsy5+GHha5YM+mRhh0KvIv8mrsaM1PKg5mcE/wm8P8APlC4i9kmOP8Ax4GqM/we09jmC/uo+ONzK3/sor0uijkQ+ZnkkvwclCsI9WYk+tuP/i6ozfCHVhjy722YL03q4/kDXtNLS9mh87PBp/hb4hiDlBaSn/ZlIz/30BWfL8P/ABFABnTVZRyds8bfpur6JpCARyAfrUukhqoz5sfwrryKXOjXzDtshZv5Cqcuk39vgT2V1F674WX+Yr6aWGIoAY0/75FH2aH+4B9Dip9iUqh8ulcM249PejYdpwck9K+n5LG3l4ZC31YmqUvhvSJ/9bYWzH1aBG/mtQ6LGqiPlfaUu2XuRxWxaZNqgPQZ9s179N8PvDM7bn0m13eqx7f/AEHFVW+Gnh/btithGvoryZ/VjUujIpVInig+7nH6U4RbgF5+tevy/CzSWOY5bpcdB5q4/wDQDVRvhZEhJivpRngZQNj9RWbpTLVSJ5gIULHIGKcsKjr1Poa9Gl+GNyMeXfBsf34sfyY1Sm+HeqpuKyWzHtw//wATWbpz7FqUe5w/lgtxkbR1PemmI468n1FdbJ4E1tMYto3A6kSqP5kVTfwrrEWSdNnJ7BBu/lmo5Zdh3RzhhOSBj3OKYyN5fbn3rcl0TUYiA9heIAOS0LD+lUJbd4j86upHQEYNLUehyd8hVyCBnOTg1a06YrasOpDcUmooVDjHzNyah075oXHuK0SuiHuWXOWz3PrWbeoDDKD0A61pEHGe9Urpcq4xwVPvWkCJHOsPTnHvmoOTxjg1YJwABn+dQHBFdaOdjW4zSMBxnGfrS56kHnofemknjI4NMBDjpj3pj/dbFOOcd6aeQetUIQH/AEccfdfr9R/9al4OB1zSIMwSD0INIMkUwDC0YwcGjNHX+tABjNBUDijnOCOKPrQAmP8AOKQrTs+2fxpD7UCGkYptOI4pKYhK+mfgNffafh89uWybW8kQD0BAb+ZNfM1e7fs73vya7Yk94plH/fQP9K58Ur0zSk/ePcTRmg0leYdaFopBS1QCHgqfenU1/u/TmnV1UtiJC0UUVqSFJS0lAC01xlSPUU6kNOwEKcoPpSmmowO5RjKsQfanUgEpy96bS5xnHWpY0OAJOAOaazwhXDSKMe/Q1Onkqu+V05H8R4rAvNathqzad5roWXeFTjI756n3wB9cVlKryK493Y4DxmZYfE32/TpI0VodkrSKYwTyMFiAOmfeuJtNAxOupahf29naysfKUtjzeecd8e4HX1r1XxBq0ElosEttJNG2GSJISiJ6HLLtyOvXFeWXfh+91L95d3iR7mLCWc+X8o+p549K5Z13JWbsS4JPa52McPhJIkUMjAKACUbJ/wDHxRXMx2Nokao19bFlABO1+f0org9k/wCdl83kfQjfepRQ33qUV9azgRzviZPPu9LtmdhHLKVYKfdRn68n86a3hawUMfNucL/tL/hUviD/AJCujf8AXf8A9mStWUZWRemQRXlU8JRr4qs6sb2a/wDSUeHHB0MRi67rRTs4/wDpKPPNI1fwjrV3FaW17fRzynCJMgXJ9MgEZrql8KWHeW5/76X/AAqhZeCrK2uIJyVdomVwwXByORWr/awl1k2UaSjyk3SM6FQSTwBnr0PI4rp/svB/8+0dH9k4L/n2ho8I6eR/rrn/AL6X/wCJpjeFNPBwJbk+vzr/APE1vofkrF8TRvP4cvII2KtMojyDzhmAP6E0v7Lwf/PtD/snA/8APtFaPw3pkoJS4ndQcEpKpx+QqceEtNI/111/30v/AMTXOeD/AA4+hasJYXZYZkKSR54PGQfrkfrXdeZtl2e2abyrB/8APtAspwP/AD7RlHwhpqjJnusf7y//ABNN/wCEU0v/AJ+Lr/vof/E1uSN+7/EVxXjjUNesJ9POi3CxhlkMqNErB8bcdRx36etT/ZeE/wCfaH/ZOB/59I2x4Q0xulxcn/ga/wDxNL/wh2nf89rr/vpf/iat6fcyS6da3NwoSWSJGdRxgkDI/M1oq2RR/ZeE/wCfaD+ycD/z6Rh/8Idp3/Pa6/76X/4mj/hDtO/57XX/AH0v/wATVrU71rHTby9SFpmgjeQRKcF9oJwD+FcpZ/ENrlgp0LUVY8YQ7sfoKP7Lwn8iH/ZOB/59I6L/AIQ3Tv8Antdf99L/APE0f8Ibp3/Pa6/76X/4mtqOUuoPI471G87Aud2FX29s0v7Lwn/PtB/ZOB/59Iyf+EO07/ntdf8Afa//ABNH/CG6d/z2uv8Avtf/AImoNK8b6HrNylvYapHLM/3Y2jZC3fjcBXRJNuHIo/svCf8APtB/ZOB/59Iw/wDhDtO/57XX/fS//E0f8Idp3/Pa6/76X/4mti4vEgKBiBuzjLYpovQR9wkexzR/ZmE/59of9kYH/n0jCvfClhbWNxOk1yWjiZ1BZcZAJ9K1PDbs/h+1LMWOGGSc8BiAPyp+oXKyaXeqFYH7PJ1/3TUXhk/8U/a/8D/9DNYUsPSoY1KmrJxf5owo4Wjh8wUaMbJwf/pSNfNFJWPqHiSy067a2lWZ5FALbFGBnnHJHau6tWp0Y81R2R6dfEUsPHnqysjZJozXO/8ACY6f/wA8br/vlf8A4qj/AITHT/8Anjdf98r/APFVzf2lhP8An4jj/tfA/wDP1HRZozWFa+KrC6uooFSdGkYKpdRjJ6dCa3K6KOIpV1enK514fFUcQnKjJNIRelOzTF7j3p3atjcXNFJQKAFozSUUDHZozTRRUgOzRmkooAXNBAPBANNpc0DQ3yYjz5af980jW8TDBXj6mn0UrIq7KM+haXc58+wtpM9fMiVs/mKz38DeGpM/8SezXPXZCq/+ggVv0UuVDuzkp/hn4YmyRYbD6pLIP/ZqzLn4PeH5ySst5ET/AHJh/VTXoOaM0uSIczPIbj4B6Y2fs+rXsfHG/Y/9FrHuf2fbgD/R9dyf9u1wP0c17uDSg0+UR833HwE8SJkw6hp8gHTJlU/+gGsi5+C/jKAHbZ203oUuVX/0LFfVGaAaNQ0PkC4+GXjG3zv0G4f/AK4skn/oJNZtx4N8TWwJm8PaqgHc2cmP5Yr7RIVuoB+tM8iH/nkn/fIouwsj4aayurcSrPbTRHb0kjK9/eq2DivuxrWFxhlyPTccVRuPDejXWfP0y0lz18yBX/mDRzMOU+IsHvQT7V9kXHw48I3IPmaBYDP9y3RP/QQKybj4MeCbjJ/shIyf7k0q/wDs1HP5C5T5NowT0r6guPgL4Sl/1QvIf9y4/wDig1ZNz+z1pLEm21O+j9N5R/8A2VaPaIOU+dQOKNo9a90uP2eZxk2+tk/9dLUf0k/pWRc/APxFFnyb6wlHYMJFP6KRR7RD5WeQkDvTcc16TdfBTxlD/q7W1m/3LkL/AOh4rHuPhb41tTl9CmcZ/wCWUkcn/oLGmpLuLlZxteqfAS88jx3Pb5wLiydcepDK39DXE3PgrxRaE+d4d1RQO/2RyPzAre+Fwu9I+Jejvc280AeRoT5kZX7ylR19yKmrZwYQTUkfVZpKM0V5R2hSikoqkA4jII9aFOVB9qBzSIflI9CRXRSIkOooorcgKKKKACkNLRVCIAMSuAOvNKaR/wDXKfUYpTzSC4VLBD5j7iBgVEOXVB1PQVo7fKhwgyQOPc1DSBuxSuIobhj9qZVtlIG0tgMx9f5YrC17SdLhs5pFX7GH+VpklaItnqOASx6cY5rp22x4JGcdOMnNVr2wj1GDyrxA0LEHys8HHrjr9Olc06aaaau2ClqedqWnTEU2qXUKDIMillOAfuqQSDx7HHHeuNvWsrvVktnv75nuZSjRPHgqOMnAzgceg+hr1vWrl5LeOHTYyEjZQHRcgjOCq49u5wO2fTznxLqI23T2E8ENy0G2VbfDOpAK4LD/AHgRjGea82dPllZO50LWNzLlXT0ldF8F6wVViBlpRx9AnFFc/wDZddPMT3Hl/wAG5hnHbPNFP2Ue/wCLMebyPpZvvUUrD5qBX1DONHP6/wD8hbRv+u//ALMlaF9zZ3I9Y2/lWf4gH/E20b/rv/7MlVtY/t9dQeO1e2aylwMOp3KCMHn864sK0sTXv3j/AOko8zCpvF4j1j/6SjjvDXhe90bU7O5t72byg6iSMvwy9CD+FeizhfPiJHODj9KYtk8dmxjAMoXKg9M9qydO1HVr++EN7paW6ICfMWTPPpj/AD0r0LruehZnUJ/q/wAKpaiN+n8c/Mh/UVejUhMVz95rkOkzNa6nFKsJz5c6IWUr6HHIIoTE0aNuoVoz/noaV3zqGB2QfzNYzeMfD8Ee5b0yN/CiRtk+3Iq5o0s1+GvZImiEzblRuqr0GffAobBJmy3+rH1H865bxd4s/wCEanslbS2vUnDlmV8GPbt7YOc5/Sumu5Y7a1aaZ1SNMFmY4AFYd8dH1eSNpNQtW8sEKBIp6/j7UhlhGj1uystTgmmWIqJUjzgNkfxD1Fa8DHYM1hNqWn6dbW+nWsqSSMRHGkbAkDuTjoMZrbt8lATQBWvpLeLTrqW8YrbJG7StgnCDJPTnpXPWWveEnZRa6hFuYgKCrDJ/EV0V3CLmxubcjPmI6YPfORWHYeFLK1mWVrWIMCCDtBqWVE6iLpx0qBufOB7n+gqeMALjNQkfNMPU/wBKaEzk7DwXa2OoW12NiyQOHBUcnFdFb6glxdyRRq4ER2sXQrk+2e3TmvPtE8P6lo+q2dxDqFw1t5gDxM5KlTxXo2B5qHHOOtDBFXXtDttbtUjuY9xQkoc4wTVXQ9KTTNGlskyFV5Mc+vP9a6FeVqoBh5h/tf0FAHHaLoUulWd7K93cyg2rqFkkJA4/+tXV+Gj/AMSC2/4F/wChGo51A0e7OP8Al3f/ANBNSeGh/wASC1/4H/6Ga85/78v8D/NHBL/kYx/wP/0qJsZri9R58T3/AP1zX+SV2dcdfDPie/8A+uS/ySjG/FS/xr8mGY/HQ/xr8pF3TLzymaIk7W5Hsagsrlv7VEhZiC7Z5+tYMmoSW/iv7A/Eclos0Lf7QZgw/LafzrS09mMqs4IPmt/6EcfpXd0PS6mnrrbtV0X/AK7/APsyV0lcvrBzqei/9d//AGZK6euHD/7zW9V/6SjzcJ/veI9Y/wDpKBep+tQX8rwadcyxna6QsynGcEAkVMOp+tVtU/5BN7/1wf8A9BNdNV2pya7M7a7apSa7P8jkYk1CWyguBq10DKGJXe3y4JHXPtV1NOvWtPtDa7dAAEsPmOMf8CqKz/5BFl/uv/6G1El6p0+9Eb5CKwYZ6MBXmYbBUJ06cpp6pP4pbtep5mCyvDVcJTqSTcmk370u3qS2mnXt3u265eKQAec//F1a/sDUP+g9df8Aj3/xVO05jHJgHqtX9QuWgsJJF+9jA+prohl2Ha2f/gUv8zpnlGET2f8A4FL/ADMSSxlico/iW4DDqAHOPyap4dGu503ReI7hx7buP/Hq5nWINSuYEGm3PkOCWY7c7vQVc8L392UtpLrInJ8uUevOP8DUrL6Fr2f/AIFL/Mf9j4Paz/8AApf5m8dA1ADJ1+6x/wAC/wDiqBoGoEZGv3X/AI9/8VWhqkL3elzwRthnQgc45rmtC1Awyjd8ufkkB7Ef5/WtHlmHSvZ/+BS/zIWU4S9rP/wKX+Zq/wBgahn/AJD91/49/wDFUv8Awj2of9B+6/8AHv8A4qqr38tzqAKzSLHuAARsZFdIJQULZ496hZfhm9E//Apf5lPJ8Kuj/wDApf5mJ/YGof8AQfuv/Hv/AIql/wCEe1D/AKD91/49/wDFVS1bxI1rDJcq0ggToIk3M1WPD/iNdUhhlVy8M33Swwynpg4pfUMP2f8A4FL/ADH/AGNhez/8Cl/mS/8ACPaj/wBB+6/8e/8AiqP+Ef1H/oP3X/j3/wAVWzc3S29tJMwyEGcetZZ8QRpjzUVM8DMmM/pTll+Gjun/AOBS/wAwjk+FltF/+BS/zIv+Ee1D/oP3X5N/8VU/hW7nutKdriV5WWYqGc5OMA9e/U1dsdSjvSyopBUZ6g1l+Dv+QRL/ANdz/wCgrWMaMKOJpqnezUr6t7W7s5/qtPC46lGldcylfVva1t2zo80ZrO1q7lsdIuLiAgSKAFJGcZIGf1rAjutckiR/7YtF3qG2sACARnn5K6K2LVOp7NRcna+lv80ddfHqlW9jGEpStfRLa9urXY7HNLmuViXxFOcRavZOfRcH/wBkqb7J4ox/yErX/vkf/EVH1xv/AJdS+5f5kf2jP/nxP7l/8kdLmjNcyLXxQR/yErX/AL5H/wARS/ZvFBOP7Stf++R/8RR9cl/z6l9y/wAw/tKf/Pif3L/5I6XNGa5v7J4pA/5CVr/3yP8A4igWvik9NStf++R/8RS+uS/59S+5f5h/aUv+fE/uX/yR0uaN1c19k8U/9BK1/wC+R/8AEUfZPFP/AEErX/vkf/EUvrkv+fUvuX+Y/wC0Zf8APif3L/5I6bdS7q5j7J4q/wCgla/98j/4ij7J4q/6CVr/AN8j/wCIo+uS/wCfUvuX+Yf2jL/nxP7o/wDyR02aSua+yeKv+gla/wDfI/8AiKPsnin/AKCVr/3yP/iKX1uX/PqX3L/MP7Rl/wA+J/cv/kjpTTCAeoB+tc79k8U/9BK1/wC+R/8AEUfZPFP/AEErX/vkf/EUfW5f8+pfcv8AMf8AaU/+fE/uj/8AJG8Yozz5a/lTTBGe2PoTWF9k8Uf9BK1/75H/AMRSfZPFH/QStf8Avkf/ABFL60/+fUvuX+Y/7Tn/AM+Kn3R/+SNw2644LD/gVM8g9pX/ABx/hWN9k8T/APQRtf8Avkf/ABFJ9k8T/wDQRtv++R/8RU/WL/8ALqX3L/Mr+1Kn/Pip90f/AJI2THIOki/iv/16TbKP7h/MVzmoXHiDSYUuLi9gkQyBdqoOep5+UccetdRV0KkKzlHlaatv5/Nm+FzBYiUocri42uml1vbZvsR5kH8A/BqhgectIWiVSWzjd/n0q1UY4lPvXQqaWx281xQz90/I0b8fwN+VPozWlibjPMX/AGh/wE0nnR5++KkopWYxu9T0YH8aWkKKeqg/hTDBESfkXn2p6iGXDbArHPDCk37zhPxPYVFdWga3cRsytjg5JA/DNTRgCNQOmKNbhoXba38obz949c9askgKWOePQZqsl0oQBslqzLjXL2y3NLpbSx7sK0EobA/2gQCPwzWVSpGCvInlcnoa7L/y0bI46Z6Vzuu6hd3TCw09FwMGV5CQHXugA5I9SO3Geapap4nnuJBZqqWzEg7S2ZGHsO35Gs641SS1s3CRWyvM2HMp3FsDqSck/jXmVsVKTSp7dzohQaV5DbxZhaSRXunSZbPmTRFZh9dmQo4AAHOOB6Vw+rWujWXkS2ck5hlIVsrhg3UFgeMc8YA610txqt0Iv9HltDIwIASFSAexzjGB9a5m6u43ud1+z3tyOAVykP5AZOfwPHQ4rBybZbikjnpNW1aKRo0tL10QlVY7SSB3ziiusMliSdtn8vbbDkfgTnP50Uvbx/lM+VntUg+ekFOl+/SCvqjgOe8Qf8hfRf8Arv8A+zJW80YY81ma3pVzqD2k1pLHHNbuWG/p2Oeh6EelVvsfij/oI2v/AHyP/iK8xVJ0MRUbpyala1rdku6PHVSph8VVk6UpKTTTVukUurRuBABimC3UOWA5rG+x+KP+gja/98j/AOIo+x+KP+gja/8AfI/+IrT67L/nzP7l/mb/ANoT/wCfE/uX/wAkb4XAqvcWkdwpV1BB9RWR9j8Uf9BG1/75H/xFH2LxR/0EbX/vkf8AxFH12X/Pqf3L/MP7Qn/z4n9y/wDkiZdBs0k3rAgOeoUVpwwCNQAMCsb7F4n/AOgja/8AfI/+Io+x+KP+gja/98j/AOIpLGNf8uZ/cv8AMf8AaM3/AMuJ/cv/AJI2rm2jubZ4ZUV0cYKkZBrmJvBemO2fsyD6DFXfsfij/oJWv/fI/wDiKPsXif8A6CNr/wB8j/4ih4yT/wCXU/uX+YlmMl/y4n9y/wDkhmmeGbLTpfMhgVX9cZP51vomBjFYf2HxP/0EbX/vkf8AxFH2LxR/0EbX/vkf/EULGSX/AC6n9y/zB5jN/wDLif3L/wCSKesaFqU9/LPZ6vdW6Pg+WrfKDjsKoLpPiSM8a7MR/tRA/wA62jY+Jz11G1/75H/xFJ/Z/iX/AKCFr/3yP/iKHjZ/8+p/cv8AMazF/wDPif3L/wCSNi1RxCokYs4ABbHX3rF1ldbjvwdOktxbuBuWRCSD3P8AKniy8TjpqNr/AN8j/wCIoNh4mbrqFr/3yP8A4iq+vT6Up/cv8xfX5f8APif3L/5InksXGnv5YBlC5QHpkdKoaReateXRS+s4oEQZDK+ST6Y9Kn+w+JsY/tC1/wC+R/8AEUg07xIpyL+1z/uj/wCIoeOn/wA+pfcv8w+vy/58T+5f/JG8i4Wuf1PW49M1FreS0u5DIAyvFHlfTGc9ak+xeJx/zEbX/vkf/EVHJpniKQgvfWrEf7I/+Io+vS/58y+5f5h9fl/z4n9y/wDki7cKw0W7JH/Lu/8A6Cad4a/5F+1/4H/6EaoPpniKWF4Xv7Uo6lWGMZB/4BWzpVk2n6ZDau4dkByQOMkk/wBaypSnVxSqODilFrW2915szoSqVsaqrpyjFRa1tu2n0bLhrkLsqPFGoFiFAhXJJwBwldfXF6u9tB4hv1v4JHgnhVQAvX7vPJHGVP5VWYTjT9lObslJfkx5rUjT9jObslNXfykUNfsftaRXtqytcWmHQqc5x1H4g1csZRIIZV+62CKbp2o6NpkDRQWs6qSSflHJ/wC+qoKujR6mLxFvBh94Tjbn6ZqnmmDa+NF/2vgb/wAVG9rLBdQ0dj0E2f1WulimWRdy5/GuK1G9j1q5soLJZo5FYjcVGRnHIwT0wTXR6Na3Nra+XdXLzyZyXYAfhxWeDqwq1qs4O6bX/pKIy6rCtXr1abvFuOv/AG6jVH3jVbVP+QRe/wDXB/8A0E1ZH3vwqtqn/IIvf+uD/wDoJrtrfwpejO/EfwZ+j/I5yyH/ABJbD6P/AOhGubiVrHxZqFtMS1pqaiRPQOqhWH4jFb/+mJ4VspbK2FxIGcFCccbjUc+kXOpaRHLLEIryMiVFBztYds/TiscHG9Cnf+VfkRlsrYKl/hj+SNOw+VoxnOFxk1b1cZ0uU+mD+tYeiXt3dXvlS6dNAqA7nc8Z+ldLd25uLCaLjLIQPrXTSVlZnZVd3dGFpqKzsD6VRVBDfzKvAE2QPyNVbTWYrWdorlhb3MfyukvANWba4hvb/McqO7tuIQ5x0/wpS92PKwiry5jsYvmUA965XVrI2OsCSNf3NzksB2YV1cCkIM1m+IUUWSSHjbICfatY6qxD0dzO0uHzJQ2OF4rfuFIspcddh/lWToTLJHIVIOG7Vuuu6Jh6jFZU1Yuo7s5G3t0ukSJwCrDpUel+GZrDU1MV5ItmsnmLCqjHrjP1qXT5hG4z96JijD0PSrcl9cteP5T7YlwFwAc+tJrlvcpXlaxr6ou7Srgf9Mya5K/0u21e1ijukDKpDj2OMf1rsbkebZyr13If5Vx1zYnU9MjgEssJO074m2tx70VfhTCl8TRteGNMsdKieK0QK7HLnByfSpPB3/IJl/67n/0Faj8M6O2mJIz3NxcO+BmaQtge1SeDv+QTL/13P/oK1wVP96pekv0PNxX+/wBD0n/7aXPE3/Iv3X/AP/QxXLKsjacVjbbJ5XytjODt4NdR4l/5F+6/4B/6GK52JSsar/0yjP5oKbX+1T/wL/0pipv/AIVX/gX/AKVIyPCeoapIk41E5ubabCuE25GPb/PNekyyH7O5Q4baSPyrzXRNZvLrVbixvbWOKWAB1ZCcMM+9eh20nnWqv/eXmuuGktT1Z/CYWhXLm4idnZt6kHJzz/8ArFQ3Wq3S63cqJpFWKRQFDYGMelM0nMcyxnrHMyfrVbVF8vxDcj/noiP+mP6VdvjRPWLO2uLjyrWWUDO1C2PXisnRNUuLuUCZ1Ksm4AKBzxV6I+fpq5/jiH6iuc8PyYmtgfQqf1pS+FMcfisdDrGpnT7aNwm9ncIATgD3qXTL431p5jIEYNtIBz6VleKlJ0pJB/yzmU8fl/WpPDzfupk9GB/P/wDVRNWSYo63N/NZratCs7xjexQ4OBxmtDPy1yUiA315CTwXYHt1qZ6Qcl0HHWSTOig1GGRwm4hj0DDFXc15npltd6R4gksZbqaa3kiDx+axbb+f416DFKZbQNn5iuCfepi3e0imla6HRX1vNM0KXETSr1RXBYfhUzuFGa830CXyfEVux6lipz7giuh8ayONIjCkhWkw2D14Nbez99wM+b3eY6beCuaFYMK8s03X57DSL20EpXeu6NgeVbIzj6j+VbPgS/mlu7uGaeWTKBlDuWAwe2fqKhx925V9bHdkgDmm7geM1geKNebRrJTEgeeTO3d0HbJ/OuGg8Za3FdxiW43CQZUPCArfTA/rUvQpanceMP8AkExf9dx/6C1b9chq2ppq3he2ulG0mcB1/usFbI/z6119cVH/AHqr6R/U8zDf7/X9If8AtwlNI+cU6mv2Poa7D1B1FFFMAooooASijvS0CGsMgiq8JJiHqODVnFVogVllXGPmyPxpgSU5U8xHTjpkZptKsixEu5woByaipFOLTBOzOZ1ixtpp2hnhjlDYbYwBx23c+lYUumTSKn2LUZ4Rz80iiUYJzxu5A9MEf1rp5Ld5p7m6k3x722ojqMqABj+ecfnWbfJHaadI06nyyCMvn5z6Z964rRldtWRq27HOTWX9p3bRPcRfZ4huARPmdT3Zsnqc+g68evBavb3DahcQXj+RahzHHI85JaMH5cgZYLwD6V6PbpDb2ZeV4GuJm8yXLABD9PboBXmfj3TL271h1tGaWB9ki7Hyu4jBJ9Tx+tTCnBPTQmTdrmzHfWqRIi67OqqoAVYGIA9Ae4orgo9K1FY1H2xFwANuentRUfVqf85PtX2PsKUfPTAKwj4z0mU9Z1+sef5GpV8VaOetw6/WF/8ACva9pDuYezl2NqlxWWviLSHxi+QZ/vKR/MVOusaY3S/tvxkAo549xcsuxdoqBL+yk+5d27fSRT/WpldG+66n6Gq5kKzFxS0UvagLCUUtGKAG0uKKWgBKMUtFACYopaKAG4opaKAEopcUUAJikpaKAEopaKQCUUUUwE2ikKinUUgG7RSbAKfRQAz+P8Krap/yCL3/AK4P/wCgmrf8Q+lIyK6MjqGVhggjII9DUVI80HHuRVjzwce6MDQbyyj0K3jmu4Ecbsq8gBHzHtWgL/TgMfbrX/v8v+NRf8IzpH/Pp/5Ef/Gj/hGdI/59P/Ij/wCNcFGGNp0400o6JLd9PkeXQhmNGlGklD3UlvLordh4vNNVtwvbXP8A11X/ABqX+0tPx/x/2v8A3+X/ABqv/wAIzpH/AD6f+RH/AMaP+Ea0j/n0/wDIj/41pfHdofe/8jbmzL+WH3y/yK19Fod/zPNYyH1aRTTLKHRbE/uJ7KMf7Miirn/CNaR/z6f+RH/xo/4RrSP+fT/yI/8AjSf119I/e/8AIanmS+zD75f/ACJOupaeAP8ATrX/AL/L/jVe/m0nULR7ee8tWjfqPOX/ABpf+EZ0j/n0/wDIj/40n/CM6R/z6f8AkR/8aL47tH73/kLmzL+WH3y/+RK+lrpGlweTb3VqiZyf3y8n860f7S0/H/H9a/8Af5f8ar/8IzpH/Pp/5Ef/ABo/4RnSP+fT/wAiP/jQnju0fvf+QOWZfyw++X/yJgavptpPeNe6drEFtO/3x5isrfhmmabE8c+691a1kUdAhVf1zXQ/8IzpH/Pp/wCRH/xo/wCEZ0j/AJ9P/Ij/AONKTx0t1H73/kVGpmUdow++X+Q8X9h5ZX7dbdMf61f8a4qSbVLaRo4ZNNkiU4U+bzj867L/AIRnSP8An0/8iP8A40h8M6T/AM+v/kRv8abljmrWh97/AMhKeZJ35YffL/5E57Rb+6OoIbxrKCFQSWE4yfbrWv4P/wCQTL/13P8A6CtWv+EZ0rP/AB6/+RG/xrQtLOCxg8m2jEceScDnJ9ST1rOnRxEq0alXlSins31t3XkEKOLqYmFavypRT2be9u6XYo+Jf+Rfuv8AgH/oYrmr65SwjtJHimYSW0ZzGhboo610vib/AJF66/4B/wChipLG3SbSLLcoP7hOv+6KtP8A2yS/uL82OH/Izb/uL/0qRxdrd2U98JIopTcsuz/VMOM557fnXcWUZjtUT0FCWEStkIM1bWPaMV1Ri07s9eUk1Y4W7uV0vXbmCc+WsjebEx6HNMu5Fu9RW6DAfuvLIBzk5zXYalo9pqcYW5hSTHQkcj6Gs2z8K6fZTrLFAA69CSTj86qpK92upMEuvQ0rBGSwiRvvBRkHtXMaf+41FkzjZcsPw3V2aptXFYzeHbc6sb4tKctv2byFz64pX9zlYL4rj/EMfmaHcj0UN+RBrP0W6SB2LuArqDk8CuhurYXNnLAeBIhU/iK49vDerRHEepfKBwDCDROS5UEFqzs4biOcEJIjEdlbNc1ffu9emXs6q36Yq5oGnXdm0kl5ciV2G0BUCgCq/iGzvvtsd1ZRRyfJsYOxHehSTg7hy2krFLVXU6zpTqOWEin8h/8AXrpLA5tQPSuXtl1aSZFmsY1H9/zM4/SuttYvLhVfQVipOU0+yNHG0Wedt/ofiM9hHdE/huzXW+KovN0GRsZMbK364/rXP+INPuTr8ggt5XMrBlKrx09fqK63U4Xn0O5jI+cwngeuK7OZe1UvQwSfJY8dupPJuIY2+5LuQ/XqP5Guk8FSGHXI0J+/Gyfpn+lZ8ekvrJuEhQs0a+YjY4Dg9M+/IqXw87w67ZFVJYyAFQORng/1qJrew4vudD49jJW2bsUYfTBBrm52s7jRNNieV0uUDncgHA3Ed/wrtPGluZNJSYDiN/mPoCMfzxXnE1rb3VibeaP96jho27YyMiiSVosa6o6yNrRfC4gtfOylyrSNLjLEq3PH0r0SvKbadTbSxr0Lqfy3D+teq15lD/eavpH9Tgof8jCv6Q/9uCmvypp1IehFdx6VwByoPtS01OUFO5oAKO1LR0piE70UUUAJVdvluR6Mv8qs1XnBEkTDGN2D9DTYkPqFQz3W88Rxkbfdu5/Dp+dTUmKmSuNMdqMYKj0xXLahChaFWYON24RJyzsBkAfzPauh1Ge8Marb2Pmgr99pQoz6dz2rA1NLq2tJ5kiO/wAsF9oL8+gPZc4/U8152JrJLli7M1gtNTnLmJrnU/L1C5WGFx+7iU9FA53P265wMe5qvcRQ3hvPLsILgkMWnjwRFjAGB3J5Pbrj3q7qEEsqqlxKYoMcxrt3k9MAZI98g8VzWoIftDLDqojtEAIjR9pJIz06nAx689PbzKkpzupGtl0L6Q6UqKrQTFgACTNEM/pRVJPCiSIryatIjsAWXJOD6UVheP8AMLkfYeJGVuv61ZjuM9TzVJ87sjrTkI9xX0D1BGosgboalDcVmq5U4/CrKSkd+PWpaKLOEJ5RfypRtB4UA/So1f8AGnhsdP51IydJ5E5WR1/3WIqZdRvE+7eXA/7at/jVPPFOye2PWndhZF9NY1JOl7cfjITUqeINVUYF4/4qp/mKy80vanzy7i5Y9jZXxNqi9Z1b6xr/AIVMvizUR1EDfVD/AENYP+NGTT9rPuT7OPY6RPF95j5reA/TI/rVhfF0v8dkh+kpH9K5RTzntU3Sj29RdQ9jB9Dql8Wp/FZMPpJn+lSr4stf4racfTaf61yIzjNLT+s1O4fV4HYr4psD1juB9VH+NSL4k0xussi/WM/0riqOtP61Mn6tA7oa9pjf8vQH1Rh/SpF1fTm6XkP4tiuBz70ZxVLFy7C+rR7noa39k33buA/SQVIs0T/dlRvowrzj0yfzox9DT+tvsL6qu56VwaXFeahip4JH41ItxOn3Z5B6YYiq+t+Qvqvmei4oxXnw1C9XGLu4/wC/h/xqZdY1Fel5L+Jz/On9bj2F9Wfc7vFGK4hdf1Nf+XnP1Rf8KlXxLqK9Wib6p/hVLFQJ+rTOyxRXJL4pvh1ityP91h/WpF8WTj71pGfo5H+NUsTTF9XmdOR8y0uK5g+LWLL/AKCD6/vv/sanXxZD/HaSD/dcH/Cn9Yp9yXRn2OgorEXxVZMeYLgfgp/rUy+JdOYctKv1jP8ASqVaHcXsp9jVorOXxBpZP/Hzj6xsP6VIus6a/S8i/E4/nT9pDuLkl2LtFV11GxY/LeW5/wC2o/xqVbiB/uzRn6OKfMu4crH0YoBB6EH6GlxRcVhKKXFFAhKKWjFACUYpaKBpCUUuKKRRS1WybUNMntUYIzgYJHGQQf6VkRad4lgiSKPULUIihVGM4AGB/BXS0lctbCQqz9o207W0bRw4jAU61T2rlJStbRtab9Dnvsfij/oI2v8A3yP/AIij7H4o/wCgla/98j/4iuioqPqMf55f+BMy/syH/Pyf/gbOd+x+KP8AoJWv/fI/+Io+xeKP+gla/wDfI/8AiK6Oij6jH+eX/gTD+zIf8/Z/+Bs5z7H4o/6CVr/3yP8A4ik+xeKP+gla/wDfI/8AiK6Oij6jH+eX/gTD+zI/8/Z/+Bs537H4o/6CNr/3yP8A4immx8THrqNr/wB8j/4iukopfUY/zy/8CY/7Mj/z9n/4GzmxY+Jx01G1/wC+R/8AEUGx8THrqFqf+Aj/AOIrpKKX1GP88v8AwJj/ALMj/wA/Z/8Agcjmf7O8Sf8AP/a/98j/AOIpwsvE46aja/8AfI/+Iro6MUvqMf55f+BMP7Mj/wA/Z/8Agcjmm0/xIzAm/tSR32j/AOIpfsPibGP7Qtcf7o/+IrpKQ0fUY/zy/wDAmP8AsyP/AD9n/wCByOYTS/EMedl7aDPoo/8AiKjTRdcimMqXNmsjdWCDJ/8AHK6ukpfUI/zy/wDAmH9mR/5+1P8AwORzUuneIpomjkvbR42GGUoCCP8AvisU+CboHrb/AIO2P5V33akxT+pL+eX/AIEw/suH/P2f/gcjhrXwXcwKIke3ji3DOCzEfTI9/Xmu3p2KTFaUMNGi2022+7vsdGGwVPDuUottytdttvTbf1EopaK6TrGR/wAQ9DT6YvErD1p9ABRRRQAUlFFNCYtQzjMRI7HNS0xhuUj1FOwhOvSkpIzmNfpS0hkkm46c+z74zt9q5zUkeSBAkpaISK7KuP3mPcmt+EGSJ/NH+0qen196yNQVVkPHI5rgq0HObkawkkrHP6pbi7vLXfbr5WC0mWAbjGATyQOvTr7DmsvV2mW2lng0jTxLEhMcjMG2bQcHbt9ulXri9XlLaMzSDJZYiAFPU5J4H8/aqoSRlZ7psDPEUXOc9jnr+lS8LC3NLVh7R3sjzseKYgAH1e93j7224bGfaitN9ABdi/iLT42JOUEcQCn044/KiuPlo/1/wxXvGoy/MeOM0qrwPapZQqN8xCn34pEwR94fga7U7m1rAFx2zTwPanBfQ8+1AUdqLjFVivB/SpUbpkVGMdjmhc4Pb1zSAsq/6U/II6niqoY9akDEUAWOvI+tHTqajDD+tPB4xSAd29aXHNNBGcZpfyoAUdcZqdarjg1OucVLZSHU7ik7Un0FSUKaTPtS4IpCPpxRcA6//XoA4IxQB+VLxn1ouFgB/lR68UdOv8qKLgHHWj60v50lAABR1paTqOKAsLmkPv3pcdKQj260BYDijP0ozzSHrj86YWADNLQOtJjigVheDmjOKTHXmjNA7C5oB96TvRnikAufSlwCP/rU3OOtBNMLDuAMgflT1mlX7srr9GIqIe1Lxii7FZFgX92h+W6nA/66t/jUy6vqCdLyb8Wz/OqPvQeO1Pnl3E4RfQ0117U1/wCXon6ov+FSr4j1FeskbfWMf0rIzRn1xT9rPuL2cOxuL4nvx1jtz/wA/wCNSL4quR963hP0JFc+DS5xVe2qdxexh2OkHitv4rMfhL/9apU8VQn71q4+jA1y2TxRn3p/WKncXsIPodcviezPWKdfwH+NTL4i089WkX6p/hXGA0u4EGn9ZmL6vA7dde01v+XjH1Rv8KlXVrBul3H+JxXB55pQ1P63PsL6rHud8NQsj0vIP+/gqRbiB/uzRt9GFefbsc0u6msW+wvqq7nogIPQg/SlxXnQYDpx+lSLcyp92aRfoxqvrnkL6r5noNFcGmpXa9Lqbj/bNTLrF8BxdP8AjzT+tx7C+qy7nbUVxya7fjrcZ+qr/hUo8RXo6mM/Vaf1qAvq8zq6K5hfElyOscR/A/41KviWT+K3U/Q4qliaYvYTOhxRWEviVT1tvyf/AOtUi+I4D1gk/Ag0fWKfcn2M+xsUlZY8Q2Z6rKP+Aj/GnjXLAn/WMPqpqvawfUXs5roaNJVIavYN0uB/3yf8KeNTsm6XMf4nFP2ke4ckuxZoqAX9m/3bqE/SQVIJom+7Ih+jCnzIVmOoo69OaKoQw8Sg+oxT6jkOCh98VJQAUlFFMQUUUhpiFNNpaSmIiT+IejGnUnSVvcA0owDkjNTJtK61GLGcP9aw76S2vpHj8xX2jbLDnBB9/atraC5bJDH3rlvEPg+LWZxd2t49rdA58xQSp7cYIwfzrx6+OqU52qQsjVRVrortFDD8kUYiiBwqDoornLq71CC4a4GlFplPlAi43qo6k4wCPc9OnPTPURaPd6Xa/v7qTUmAJHCRnpjGTnPP8/YCqqW89ze7jaG3aMDMeEIx6lgOn0I7etYzx6rPlpuyRcYW1ZxclnYeY3mTnfk7s+X17/x0V1zaXohYmW1vGkJ+Y/Zn5PfvRXF7V9zTlRyfxhYjw9YYzk3WcD/cP+Neb6CzsyfM2NwONx9a9E+MZxo+nJzzOT+S/wD168/8PDKwscct/Wvfy9Xoq5GMf7xnVLvGMO4Ps1PDz4H72T/vo09F696mCjGT9a9LlXY4VJjBcXS/8vEv4nNOF3eD/lu2fcCn7AOMUu3Bpckew+eXcFvrw/8ALQE+6ipBqN2oPKH8KTZ/nNJ5YNL2UOwe1n3JRq9yBykZ/P8AxqRdamyB5K/gareWDwf5UeWKPYU+w/bT7l4a4/Q2/bs3/wBanjXlHW3fI9xWfswKa0Q/yan6tT7D+sT7lqbxfZW0oSaC55GcqqkfzrRi8Q2jxq22YZGRlR/jXn2ujZepjj5Bj8zXQWke62iz3QfyqPqlNsv6zNI6Ya9Zesn/AHxT11uxP/LVh7FD/hXPeSDwBTvJ/wDrcUfUqYfW5nRDVrFulwo+oxTxqVmxGLqL8WFcz5IHAFJ5I/Op+ox7jWMl2OrW8tiOLiM++8VIJYj0kX/voVyH2cE5Iqvd2w8jkAc1DwK7lrFvsdyCCOoNKO3tXk/2chbwKxHOTg47Ukf2q2W2Y3U65yWxK3fn1rP6m+5f1pdj1rjjtRjmvJYLvVQsxOoXgLJvUee3HJ96emp61HpzuNUujIVLZaQnHtzS+py7j+tR7Hq9GPWvGx4x8QRGMf2m7fL0KIcn8q0pvGWtrKscVyCAwDFolOfyFS8JMr6zA9S7c0Y+ua4JfFWrqPmMB9zGR/I1XXxzrPnmP7NZtlyB8rDA9T81J4WoNYmB6LTRXFSeM71Uz9kgOOT8xGaLTx3NPEXfTFTnGBPn/wBlpfVqnYf1im+p2uOKTnNchF49jluGj/s2UKvVllB/oKkg8eWUsrRNZXasvJI2kH9an2FTsP20O51hpP0rnl8Z6cTgw3S/8AU/+zUh8caIrbXkmXBxkwk/ypOlNdB+1g+p0XU0EnmsCLxroEz7VvW3HpmB/wDCpf8AhK9CHDanCp/2gR/MUvZz7Fc8e5tc54o7Vmp4h0aSPeup2m0dSZQBUser6bKN0eo2rA9xMp/rS5Zdh80e5dyf1pRVcXlrIcJcQt7CQGpVdW6MD9DU2Y7oeDRSZzyc0ZwDRYYucmjOabnNLkd/1oAXPGe9LTc5H+NBPakMWgUhNGaAHA4NKScUzP1pM+xoAkycmjdxUe7jg0pNAEhPPFJu5qPdTS3+NOwibd/9aml6iLimsQKLBcnV+akD1TD1IH4osFyzv60B++f1quXwKQuaVguWt/vQH4xVXzMDvR5hxzRYLlsPS+ZxzzVMS+tOEuKOUdy0ZKZ5nPvUBk45phlGaLBcuiT34qeOT5T0rOSTPrU0Tnn0xSsBb8ymll9AfwqDf7mkMoAz19aLhYsBwOQAPpSNcup2iWbJ6KjsCfyNVo3ec/IML/fPT8PWrsFuie7Hqx6mi7ewrJbl3TFupLtZJbmYqucRmQspz65rqAcgVh2i7BmtmI5jX24r0qEWo6nDWd5D6TNFJXRYxFzRmkooEFLSUUWuBlSW99bXJaCXzYWJwjYyvsPUVHLqlxbOEuLXtywyP6H271qy9FPoaXNebLBVYSbo1GvJ6o0U090Yh8U2SMVaNwMcsOTn6VZbxVo8wiCzlWYcDb7Zxx3rQaytZkZp7eJw3UFBk1HJounSWe5bRFBk8whePm5GePqa5pLEpPms7dSk43VjKOr6bvKC9SVmUny0OeO+SeOKfHp8N2GmSFVmY7SAdowOn1xz271UvPDOliDc7yw7ACZA+AMd+elZVxY31tYWr6Rc3QjBdWDgq0244XHHHAPOR6+9eFVhzSTSsdF2kXZrazimkj/tdItrFfLEiAJg9MbeMUVCuhakVGLfTU4+60SMR7E45PvRWXspdwuzz/4xOBZ6UvTMsn/oK1xGgJ8sGP8Anp/n+ddh8ZWITSFz/FMf/QK5Pw8Plt88gsD9Oa+wwC/cxMsX/EZ1qIOp65qUDnPr1oQZ6j61IBxmvTOEZg5NOCj0p23HWnY9RwaAEx/OjHGM89aUDIz7U4cCgBmOeKMdqfjJNOA5NFxWI8YphXA9PepSMnpxTSOOfpVCZyniIf6dFx/yz5H4muisx/osPGfkXn8BWB4jGL2L3j7/AFNdFYjNpAcnHlr39qS3B7FkLxjHNPKnpzSAZHv3NSKKYhm3r60u3txUmO9G0HtQMj2gduagu1/cHp1FW8Z5FQ3YPkHp1FJjRzNx9+ZF6uwz+ApZ4xJbQKOpwBn3FTvGGuZ27KmP0qAZkkh9EKrx61lcsJlWKZsgY8vAqOJAbADHY1YuIxJdovYDJ98VWDlLNUUfO2QPzppiORuI2WUscDcxOP7vOAP0rfjt1jt7dR1JBJx1NYt2drqD05/nxXQKMwW5/wB3+VIZoviOMucYAqrZxB5ZZCOTU8mbicQg/KOWPrS2q7ZpgB35xTuAskS7TwM1QjUJaFQPmZjjitdl456VVt4RI+/+BOFFJgiOGzEUQGBuPJOO9V4YAL+Q4654rWZe/rVRVxeuT3H9KTAinj8uM/Lg9s1mvahs+vX61pzEuxYjjoKYsLeW7kkYHHvUt3KSK1rYosYfjc1ZGsRYZvoK6aED7Ov4/wA6wddGCT221fQkj02ITaU2eRlu1XYLcR2qrt5PQVFoLA2Mm4Zw54/AVrWqCRzIcEDoPSkMqxaaqKSQN7dTT/smDxxWoVHbOKbt9v8A61OyFdmPdwyiEFZJBznhiKfbm8WBNt1cAgYz5rA/zq9cpmA8dOabAg8gD0zip5VcfMyJbrUh0v7r6ea3+NPGqawg4v5yfd8/zqURgdjxUMvZVGSfahwj2Hzy7jZPEevqdkWoPuz3RT/MVNfa/wCK9P8AD76obhWiEgj3PAuGJ7DirekafZvcE3twsSKNxz1Y+gp3ji9jvdCkto3RkVQVWM/KgHbHSsnCPNZItTla9zAt/iNr7RRs32Vt3UmI/wBDXUW/irU5I0Z47clgDwpH9a8qtjttozxlW/LmvRbFQ1pEePu1MqUL7Fxqy7myPFF5jm2hP0JFV7nxvcWrqDpquGGciYjH6VEIxjP61Q1G3DbDj1Gaj2MOxftZdy+vxCOPn0lwfacH/wBlqQfEO1/5aaddKfZlNc41uuPu9KYbVSCABxR7CAe2mdSPiDppHzWl6uP9lf8A4qpF8f6MSA4uU+sX+BrjmtE4O3nPpUbWaGj2EQ9vI7oeOdBxzdyL6AwP/hTv+E18PtjOpov+8jD+Yrz17JT/ACrO1Kw22EsgPQA9B2NNYaIfWJHq6eLNBfBGrWoz6yAfzq2niDSZAAup2Z/7bL/jXz5j9x1+638//wBVblvaK8KP6qDg0pYaK6gsS+x7guo2kg+S6gb0xIp/rUgnjP3XDD1BrxMaerKeByepqN9PAHyjFR9XXcr6x5HuIfJ4+tIXxjOa8OME6DEcsiHthyKIJNYLhYb+8X6TMP60fVvMf1jyPcfMPrS+Zz1rxZ9Q1q0D51e+Zgmebhjj8z9a9k+Ekq694Umk1IfabmK7aMyS8tjapA/U1lVpezjzFwrKTtYm3joKZv54/Wu4Oh6ax5tV/Bj/AI1E/hzTGH/HuR9Hb/Gub2iNbnHpID3qZJcH/CujPhfT8kgzr9H/APrVBP4Xg8l/JurhXwduSpGfyqXURSZhtMdwVclj0A5qaK1ZyGmPTog6fj606CBYFxjLdyepqyvNVuVsSIo4GKtwrzVeNeauwryK3pU9TGci7CMKK04GzFWcnSrtq2VIr0IKxwyZYopDRWhAUUUlAC0UZpKpCYj8ofpSA5x70p5BpkZ+QeopNAiaV8AIDgd6ntTut5F7jmqZ/WrNi37xl7lelYqkowsO+pj6vAzkPgMirnyz0J9T6/T/AOtjBmnu5y8vnMFRvmJ6Ac8fXOK6jU4WliZSxjTkEr1/+tWNaxRXFpLaNiMCRANvfJ4rwqvuuT5dE0dyfuopCe8KgjkEcHc3P6UVea21gsTGyKmflUSAADsMUUvaw7IV2eQ/GRiZ9JUdQsv/ALJXOaAMJbem7+tbfxdm8660g5+9C749claxvDYxHa9wOlexgv4MTLFfxJHWrx7Y/SpgBx+opqKMU9Rg5/pXonChccUYx707n1oGcfSgY0AZpwUenWlxzRjikMQgZ6nrS4zg+tO29h1oC800IQjNMPbPQ1Lj6Uxh6UyWcp4jGLyI8f6v+pro7Af6HB0/1a9PpXNeJm23sJ/6Z/1NdPpw3WFuf+mS/wAqS3B7Foenan4+uaaFI4z+VPA49cUwsHIzR1FOx1PB9KNuRx0pXHYTAbtUN0B9mP4VY9sfhUN2ubcnGOQf1pPYZgthZLkngAD+VRlRFbwE4GW3GlnDNLKnY/MeOwHSnTKrRW8fQtj8sVkaWK8YLTl2z86k49B2pkEQETS46qQvsKtTMI7hiSMLHio0AGnnjHymmhHH6iux075zmt2ED+z4GPQKOv0rC1RHW4V2ztYjaPbHX8a1rMvLZL1CRoB+NDEjatIike5vvvyaLcf6VMKmhJMSNzyBUO8Q3UhPHy5/lSbGLOxdxCnU/eIHakswQsg9GqS2jJBlcfM9JaH55R70rjJdueDVOfakjAck9fYVclk8tCf4jWexyx9TSkxpDVjMjgD+VW5IwkBHoKfBD5aZI+c0s+PKY9OOKEJlW3H+jDI9axdcXKnjOUrbg/1HJ9awdek2ywxryWyD9Ku+hNiHQAziVBwobcfy/wDrVvaeflbjnisTw02XuUz2U/zrcseZH60uoy6BznFKRk04Dj2/nQeOcc1VySvKuY2+maitV+U9OuatOMofU1BGDDHlgcnnHpUNlJBI20YHJPTFIsQU5Y5Y1IkZ5dx82Pyp5AJzjNNAyFhVLUY99hcL2KH+VaJHyjPSq1wu+J1xwQRVXEefWozanHUPnp7V6Lo7B9OhbjG3v2rzm2zsmRuxH9a7/wAOOH0qPpxxWUjWBtAcc9Kq36fugR1Dc4q0P8iq96pNufYg1BZmFecnp3ppUZ44OKk56/1pvOOvJpk2I2GevSmFOPXJqU/mBTT/AJ707gV2Xr2+lVbuLfZToP4o2x+VXm+lRFQTjHB4qkyWcEOYJB7g/wCfzro9L+exhbGCBj9cVzyDaJk6fL/Kt3RGzZgHqrEYzTlsJGoEyOgodPT681LGpc4A5q9Fa7eXHPWsrlpFCKyaVvmGF9atiBY1woAq5tAGB+lRuuAT29aLlWsYGpx4l6feT/61ejfAe6zZ61ak8q8UoGfUMD/IVwGpphYyB1DD+VdR8D7gxeLdQtScCa1bA9Srr/QmssQr0mODtNHvZOcGg0g6Y9KWvIudYzvSHkGg8GkNQ2Uc3ewmO6ce+R+NRqvNampxZKSf8BNUAuCPfmuyjrFBJkkYq7CMCq0a1bjGBXfTics5Fhat2xw2KqLU8JxIK6EYMu5opKKskWikzRmmJhmlpKKYgzUa8Fh71JUfSQ+4oaEKaltW23Ke/FRGhW2uG9DmpktBk9+uRIvfrXD3MLPe7SSo7kdQPb1Nd7fKqgzEgLs5J9K4q5R5LlpJI9kefkQjlh6sD/L8/Q+XVdp27nZSd4Dc3X/LPT0KfwkxFiR257/WimMGLEmRySfWisvZMqzPIvi7+71XTowTtWBuMf7Rx/KsvRLlYILVjGzBUGcd60PjAwPiKyU9rb/2dqy9MAFnBnBOzr+Jrqw8uWhFk11erI6hNahxkwzDv0H+NTDWbYgfJL/3z/8AXrDXnp9KlH6YzW31iZl7GJuDV7U9RIP+AUv9qWmc72BH+yaxDx3pVPbNH1iQexibn9p2ZH+t59wRTxqFpx++A/AisIdMjrR37U/rEg9ijoBfWp/5eEH1NPN5anj7RF9N1c7wB0ppAzjGKf1l9ifYo6Q3MH/PaL67xTTPERgSIf8AgQrmjg88flULgDnj8qpYh9hOiu5F4slRb6Ahs/uzyOe9dZpbqdNtiGGTEv8AIV5trLhLmPt8tb2mhfscBxj5B/KqdeyvYlUr6HchfT+dSBT61yan0JH4mpVLg/6yT8GNL6yuxSoeZ1IB64596QjIGPXmuaWSbP8Ar5f++zUguLgH/Xy/i1H1ldg9i+50WD3qK7X9wfYisZbu5/5+JPzoa6uGTDTsR6HFDxEewewZGy5e7YjkAD9KgQF5oZD90EKo+gqfLkt8/LcHjrTRvG0BuE5HFZ+1RXs2MuI/MumHUKuTUUp8vTQp/iQgc96nG7cz5G5hgnHWq08ZKBTjCjA4pqqhezZzeuLte3x6dfwrVsEB0tMAcrz9ax/EDuLiMMV2hRgYrT0mR2sIwMbcEYxVuasSoa2NmzINtGfb1psgD3Sx9B3PrTYGeKIIoXC+tKkZSRZAckHPPepdRFKDLmOgHXtVa3IW5mBPvUnmtxlBz71EQSzsBgt70nUQKDIp5S7Zzx2psIHnpx/nmnGEk5xx3p5H71XVcBRjFSpq4+Utgcjt1qKYnynx6HpSmcYxtamNKpBG1uc9q050RysqrL5dr6NnA96ytRgKw+Y43OQfwrVAXzNzglQPlGKoazcxLACc9ccr7U4yQnFmV4Zb/TJxk8p/X/69dBZf8fMign8frXL+HpBHqz56FSOnuDXT27Bbkt/Cc8020gUTU5H+FJnjGaYJUPIP4Yo8xAOvT0o5kLlYrHaDzz0qvC2+c7uoFPXa7lncAL0FRQ4W6yRheRkmp5irGxY6e90C7ArEO4H3j6CtUacgG9k57KUPSn28tqsUbGSLaBhVyOfrzVHUdUWQNFbuOfvOOPwHtWfM5OxdkkU9QaBHMcIRv7zgcfhWVIu5SCDVhsMOwqFl655reNkYs872eXe3cZHQkDjpzXZeFnLaeec7W/xrkr9RFrt2PUk/yNdJ4QkzDKueh/w/xqZFwOr71Fcgm2fnoM09TikmBMDjGeCOKg0MnnB9qQ9fYUdevSgn8qLiGkGmHvwSKcf/AK1Ie+T/AEoAiYcf41GeOc/lUxHGMflUB65HFNMVjjJotmp3EfT5nAzWp4diaSOVBnhgfzqLUrQrqs8h4UEMB65AzVrw0225uI8jJUHn2/8A11UpaCitTpoIljHyjn1qcYA/rUa4BqQcH3rC5stBe1RyKCOvPepcjGD0+tMY54ppiZj6ov7kEZ4bn8qt/C+4+x/E2yGcLNvjP/AkP9QKh1Bc2zjv161n+H7j7F440W4yFAuoST7bwD+lOSvBonZo+qDw31pe1EnYijORXiHZ0Iz1pD0pW600nAznArNlIhuo/Mt3HcciskCtW1vba8lmjgkEnl43EDjnPQ9+lUHj8uVk9DXdhdrE1NBI16cVaSokFSpXpQRyTJl7VKhw1QipVPIrYyL1FNQ/IPpS5qkSLmikoqhC0UUU0Ji1G/31980/NMl4UH0IoYhTTTTj0pppDNFsSWsTEZwP/rVh3cCtdAFFYsQBuOK2rZt1lj+6cVlaomORXmYuN2deHfQQ6hZIdggiIXjOOv6UVjhJMD5M/nRXF9XX8xryI8L+LrhvFEC9harx/wACes/TZ4Us4VMiAhBnLCrnxWk/4q9RkjFqgOPQlq4piRI2DwAOa9GjG9GK8jOs7VZHcpNCQMzR/wDfYqZJoiOJFI+tcADjjP4U7cT3xjtVeyJ9oegBl45H509WGe1ee+a27KknHpTvPk6iR1z6MaXsg9oehD6Zpe4rz77VOowJpPYhjTxe3IU4nlx1++aPZsPaHf5Hemnt6muDGo3isCLmYen7w07+1L0Di6mI/wB40ezYc6O2J/yKic5XAH4Vx/8Aat9kf6VJ270p1e9B5uXP1A/wpqDFzon14/6TF67eh+tdFpfNjB1PyD+VcXc3UlyVaZ9zjjJFXYNZu7eFEjcbVGACoOBVSi2khRdmdwlTL6Y47c1xK+I78Z/exnj+4KcPE+oL8xMX4r/9es/ZstTR2vbFOPqP1HSuLHim+GNywfip/wAaePFt6ODFAfbB/wAaPZyHzo7IHBHH6U7P41x48XXRwfIhPr1/xp3/AAls2ObWMn2Y1PJIOdHWkgDGaM5A6+tcp/wl8ne0Q/Rz/hTh4t6k2g44/wBZ/wDWp8kgckdPuyPWonPpXOjxcuP+PQ/9/P8A61KfFMLcNbSDPuDT5ZC5kVfEbFZ42PPFXdDbOnRde/8AM1h6vqSX5Vo0IwMHJqzperRW1sI5AxIJ6D3q2nykprmOtjYAYqXPesJNetRw2/8AKpl160PV2HsVrOzLujYyM8A0bvSsga7ZMB+8Iz6qaX+3LM/8tfzBpWYXRqlunIxRkdulZY1qzJ/1w/EGnf2vZ8ZmUfnRZhoaWRznA/rTSe1Z/wDa1mQR568fhQdVszj/AEhPzo1C6LjnisnWBusjx0YGrLalaHP7+P8A76qhqN3DJbSBZUYjsCKqN7ksxdGbbrC8HkMP0rs4iOB0+tcNpj7dYiZmG3ceSfauzikBxyOvrVz3JhsXwe5zQRxzxUatxwQc++aXOeO/pU3LHZ/lScDPANJnNIelK4rDiV9BTSAe1JuHXpSA00xiEJk8fWo2VR39uDUjE9f0qGRuOtNNk2OR1pEi1SUgEEgHj3FWfDcjYnCsy4I5Bx6/4VH4hGL1CehQfzNReG3xdzoTjKg/kf8A69a/ZIW510csuR++kxjPJqUSysMeaxB681VjP8qnBz3Gc1nc0AIBxk5FHlAcdKcv54oz+NK4WIzFkfepDGVP3jj3qY5wM80h6/5NFwsV/K49/pTo7cZ55qQkDr+NOTIzzzQ5Akc9r0W263Dq0X+IrP0KTbqns6sP5GtrX1BMDkZHK/y/xrntLcpqkDE98dfWqjqiWtTswcdD+VSr9Pzqurce1Tg8845rM0Hg46fjTGbjp+tKWGR2/Go2PBz+NFwsVbs7opF9UNctczGOe2lVuUOQfyrrJVBO09DxXH3y7YkLZ+VsH/P+etaRd9DOSPr2CZbmzinT7sqK4PsRmnIeCK5rwtrdpH8PtH1C8uY4YhaRozu38SjaR7nINYeo/EWykd4rOVo4/wDnoUO4/T0/n9K8ZwlzNJHZDVHZahqtvYjDHfL2jTr+PpXKalrM15kSNtjPSJen4+v8q5iXxLaOTiU8nJJzkn3qpJrtu/Pm8d6aoy7HRHkj1O68I3fmatPCTkPFu/Ij/Gujvo8TK46MP1Fed+DdXgfxRaxLICZQyf8AjpP9K9PvY98G7upzXTRXLLUwrNN6FBB0qQCmKKkFejE4pDl61IKjxThWpmy5Ecp+NSVDAfkIrP1TWv7OuUgW2aZ2TecNjA59j6Gpq1oUYc9R2Ry4jEU8PDnquy+b39DWo71z8fiO5lz5ekyv/usT/wCy0reILtCA2kTKT0yxH/stc/8AaWG7v/wGX+Ryf2vhO7/8Bl/kb9Gawv7cvv8AoC3H5t/8TQNevT00W4P0J/8Aiaf9pYbu/wDwGX+Qv7Wwnd/+Ay/yN6mSjMTfSsX+3b7/AKAlz/49/wDE0sevym6ht7nTZYBM2xWYnqeOhAzyRR/aOGel390v8gWbYS6XM/8AwGS/Q2VOUBHcU002L/VAdxxTjXaekXtObIkT1ANRXMaldz8heSD3osG23IB/iBFS3QfayhAfX5sVw41e7c1pPUxWu5Nxw8gGezY/pRTjFLn7wHsG6fpRXjc0PM7+WJ82fFB93jRx2WCMf1/rXHMMStnArrPiW2fGtz04jjH/AI4K5UfMzA859K9uhpSj6HLW/iMASDQfqadtBIpcdBjr3PetTMbuPt0pDnOfWnlcc5xmjHzZ7mkAwHHr+PFGOOAak2Z+7/KgLz60XAjz0xgZ9aXtjnFPVSc85BOMYpAvTIPv7UXAjJ5OOtGecgAZ7etP24GD+RFIVA5yOO1O4rEbcc5HXHXpUoPyj2GOtLb2st7dQ2ttHvnlcIiA9WPQV06/DvxPgZ0zBHX98n/xVKU4x3Y4xb2OW3YHH5005J6V1q/DjxQyknTlx6+fH/8AFUH4a+KWP/IPUD1M8f8AjUe1h3L9nPscnnnPAIoz+I7V16fC/wAVuCPsEYP/AF3T/GlPww8VkAfYE/7/AKf40e2p9w9nPscismwkkA/UU0uSc9K6p/hx4nicqdPyT2DqcfrSD4d+KGIUaadx7mRcfzp+0h3FyS7HL7ue34UFx+Heunf4e+J1Yq2mtwezqf60w+APEy/8wyT/AL6X/Gj2kO4ckuxzSt2z+NAcZBx165rov+EC8TAZOlSe/wAy+v1pG8CeJBn/AIlUnOOhX/Gnzw7i5Zdjm2JA9v50+NjtJB4rfbwJ4lb7ulS/99L/AI0+PwF4oyd2lTcejL/jT9pC24ckr7GBvxznjuKBIQO4xx1/z610f/CCeJSD/wASmbp/eH+NNPgPxJk/8Sqbkccr/jUe0h3HyS7HPbz3P04pRIVXrzit9vAniQjnSps/Uf40n/CCeJv+gTMffI/xp88O4uWXYwPN6cHOMYpRKQuehrfHgPxPyP7Ll/76X/GlHgLxNg/8SuT8WU/1o54dx8suxz2855ppkOOcc8nFdKnw/wDE7kgaVIOOcuo/rUTeBPEm7H9lzf8AfQ/xo54dw5Jdjn/NLH8fpSGUcDPI7+tdAfAniYcf2VMcHrkf40n/AAgniY4/4lU3PTkf40/aU+4uSXY5lXIceuacZD0ya6D/AIQPxP0/sqYY9x/jTv8AhAfE2P8AkFSfmP8AGn7Sn3FyT7HPCZxyHI/HFO+1TqQPNfj/AGq3T4C8Tf8AQKlHHqP8aP8AhAvExOP7Llz16j/Gn7Sn3Dkn2MVdQugvFzNz6Of8acNTvQP+PufA7eYa2G8B+J++lS/gw/xpP+EB8T5/5BUn5j/Gjnp90HJPsZA1W9Vv+Pu4znoZDUi6xfAY+1y+3zmtP/hAvE+cf2XL9cj/ABpf+EB8TD/mGSf99r/jRz0+6Dln2M3+29Q6fa5Px70463f9DdOceuD/AErSX4f+KWzjS3/77X/GnN8PvFCNtOmPkejqf1zS5qfdD5Z9jAur64uSDPIWYcAkDp+FNtLyaznMkTYYgg8Z4rfb4feJwVH9mOoY4BLjH86lX4beKCwH2Ac/9NVH9aftKfcXs59jNj8R3qD/AJZNjrlf/r1MPFF4OPLhJ/3T/jVtvhx4pH/MOH181P8AGoj8P/E69dN4/wCuqf40uam+o+Wp2GjxVcDrBD9Of8akXxVJnm1Tj/aNH/CvvFGcf2fz7Sp/jUN/4L8QaZZSXt3p5jt4gC7+YjYGQOxpXpvqO010LS+KmHytbIT7P/8AWp//AAk6kc2n5P8A/Wrkw3bPanCQ9sZp+zRPOzrF8TRHGYHGR2bNTQ6/A0mPJkHpyK48Oc8/lVm2lPnDrzx+lTKGhSkdBquoRXUKKqOGBzg/SsCOTy71HxwsmTjnvVuRsr0z6ZrNlJEr/UdKVMJM7BNSgx1YfhUyapbYA3N/3zXNo3yZGee9TbuAeMVm0UmdF/aVtgHf39DSG/t848zr7GueD4XuD/Oq09/FBxnc3oMUKLew3Kx0sl3b7c+aAPWuS1S9haaWKI7gz/eHTr2qlcX01wTlgqk52jpVU9a6YU7bmM532N/S7jXtVRLG0u2aK1Q+XE7DagLZOAfc1pPo/ilQSTE3/AF/wrlbe7nsp2kt5GjY8ZU9q0U8Uawi7ftrkf7QB/pSnCV/dsVCcLe9cuy2viSM/NGv1CJ/hVctrafejH4qtQN4k1Nx81xn/gAqFtavHGDIP++RSUZ9Ugco9Gzo/CWpanaeMdGlmTbGLyJXOP4SwB/Q19WModSp6EYr4wh1i5jnjl3nKOGGPY19m2063VrDcJysqK6/QjNc9eLTTsaUpXujMC4JU9RTxUt0m2cnseaireDurkS3FApwpB1pa1M2TwHtXO+IYJbnW4IoFDSGEYBOM4LGugiPzisjUJPK8T20mM7bdjj8Hrkx3ww/xR/M8rM/hp/44fmRQwHT9QhVHYuwUyIW3bSeuPap7yyuJ7q4Z2eJACyyjBHTgYNV9Glimle8upV8wnIBq/q9wWEcEbfK/wAxbPGO1dqtY9R7kCXMraPJ8xG11GfY44qvbtJaXUM3mlkl/hwOMHBHvWn5EQ0qWCGRHbYSdpBJNZLsJLKEg5KyEfgRn+lUSWL77RPezpHIFdOUBGRwM4qvNctcXOj72LMs+MnuMpVyV9uqQSdBIik/yP8ASssgpqlgn924Cj8GA/pXFmX+7v1j/wClI8zNv90frH/0pHXDhnHvmg8mg8TH3FBruPSJIG2XCN2DCtWVMsfcVi1uE70RvUZrCvG8WVF2ZlNF8x+tFXTGMniivF5Dp9ofI/xG58b3uM/djGP+ALXN4JduCfm611Pj+JpvGl68S70Pl4ZeRwi+lYuIYy+9CH3etepSdqcV5E1NZsp7DjOM+1KBng/lj2q6t1Ag2hMZ9xQJbLrsb/D9arUkphCcdeKAhBBwTVwS2Z+VQ3JwOTTRJaEjGQDxRqIrbDkbT+lKBnI6DvWja28d0jOnQHGSe9T/ANnKAen4mp5irGOV5PHPSk8vt3FbS6aoPzDAzj7wp40uE8HOO3zCjmHymCVB6DoOlIyFeoP9K6D+zYMHLNz7jFMbSrZh/rWH5UKYcjK3hNseMdDzjm9j/wDQhX1YEjbBKLnHXFfL2i262vjbQ0Viw+2RHJx13ivp4PhR7VlXXM0zSk7JjvKiB/1af980pjiP8C/kKa06kbdvP96mlzWPszXnHeXFn/Vr+VL5UR6xr+VR+cE5IJ+lNEueaPZhzimGPfuCLn6UpjT+6PypN9JupqAcxII48fdH5U4Rx/3V/Kog49advo5A5h/lp/dH5U3yoz/CPyo35pPMo5A5hwijB+6PypwRPQflUe+l30cgcw8IgH3R+VBRP7o/KmeZSF6OQOYdsUHoPyoKp/dH5UwufU0m+n7MXMP8uP8AuD8qaUT+6Pypu+k30cgc47Yn91fyqORE2/dXp6UF6ZI4x6UpU9BqY5FTcflHSkKptUbV/Ko0cZHuKQPyKz9mXzknlpsPyr+VM8pNmdgyfakMvyc+tDP+796PZhzkgjjx9xfyphjQk/KMn2pPM+XNNEmX57U/ZhzjnjQAfKPypTFGR9xfypkj5A+tL5gIBpqmJzI2ij8wfIOnYUx4kY4Cr+VOeT5s8cCmK3GTnJNP2YucjeGNSAFXH0qrqV5Z6damSfCA/KpCZP4VYupBDEZGzhVLHHtXLaTrp1LUEnk6hd+3dwoPQVcaVxOoawvJb5iNP0me7ZekkrFEX2GTz+AxTbx9dsoN50NZB1PlurYH0xmuxs54/KVUUdO3epbmYonIAJ7VaoxsPnbdjkdP1C11KM7Ygkyj542ABH4VM9vGxwI19+OlWb22twzXawKkuMmVOpx2NMi+ZA4/i5rNQTdhzbRCsKB8FV6+lZ/iu1juPCuqxbF/49JCBtHUKSP5VqPLhio+nbrnOfX2qLUUE+nXMWM+ZE6/mpFXGnZmbndHyxgZ/wAKUY7nGOKUoRn9c0u0s3A7dhXccYAHPBHH61LB8sqn3HIpqqeuPzp4AHIAPOaljRoORjB6nnkVRuSfOB9vTrWg4G0Y6HrxWfe9FI9P8/1rOnuXLYuwEmJCPQcg02adIk3OcenqazxemKFUVRkdzVV2Z23Mdx96tU7vUhzLVxqDyjbH8i/rVLPvRRWqSWxDbYUhpaVqYhDSUvYGgjFACUUUUAFfYPgC9/tDwBoVxnJNmiE+6Daf1Wvj6vp/4I3v2r4bwRHra3EsP67/AP2eufEq8UzWi9TvLxMoG9DiqVakqb4mX1FZnepovQ0mtRaUGkpa3RiOU4bNZ2rWV6+pw3lpCkwWIoVZsevXkdmrR9j0qxGcxIT1KjNZ1qEa8OVtrVPTyOTF4WOJhySbVmndb3X3nJrpOop9zTkX/dnI/wDZ6nntdZuFjDWUa+Wu0FZMcf8AfVdRRWP1F/8AP2X3r/I5P7Pn/wA/5/ev/kTl7W21u0kMiWiMSMfPICP/AEKof7N1jzCyWiIpOfLDrtH0yc/rXXUtP6i/+fs/vX+Qf2fP/n/P71/8icxdW+t3SRq1lEhj+6yOAcenLdKbbaTqj6hayTwKiRSBy28dAQexPpXU0oNJ5epaTqSa7XXTXsTLLOdWnVm1o7Nq2jv2I5BiRD9RSkUS/dB9CDSmu89MbWvaNvs09uKyDWlpzZgdfQ5qJq6GibFFPxRXD7JGlz4cRzGqgMR9DQXbcQwIIJzwOKigDMiDPPtV2HRNUup2EMeWAzy4rsUbsnmsVxKf9rnpTllx/CRz3FXf+EV1wD/UAD/rov8AjUi+FdbJI8lOn/PRar2b7C513M4S5A+X/wAdpfMzwFOf92r3/CLa4Bxbrz6Ov+NIPDeuKP8Aj34/31/xpezYc6KqzOBsUuB1wq9fU8fhThcz5wHm9OCa3NM8NX7wubm1y4bj5x6exq+3hmcEFLMj2D/T3qHFp7Fp36nKrdXLOMPLyMd6mWS+I+UTfgp5rpE8N3SsCbV15/vj/GpRpF7GSEtW46dDUO/RFxt1ZzAGqsOI7kj/AHSf6UjnVYhl4bkZ6kxnn9K6wW1+gINoT/wH/Comh1DdhreQZ5+4QKlOXY0tDuY/hqS4fxdpAlDgi8iHzLj+MV9Ng8V86WEMkXjHR2ljZCbuLqMfxivoodBxzRNXsKDtcUmk8wCmnOOBUZyTSURORJlSc0uVxUADMeBk0bucGq5Bc5PlaXK/SoA1Juo5A5yxvWjzF9arbutG7mnyIXOWi60b1qru/wDr0bsfSjkDnLW9aN61V3cUu+jkDnZZ3rSF1qvu96Tfk0ciDnZZ3rmk3r6GoN31pC9HIHOyxvX0pN61XLe9BanyC5yxuX0qOV129Kj3n1qOV8gUOA1MkEibl4/hpodNucd6r78BW/2TQG/cn61HIh87J/MX0pTInl9O1Vt2dx/Cjd+7x6ijkQ+dlhpUCYxjNIkibuh5FVWYk+3rTgwWUfTFCgHOyzNIu3pThIpA44qpK3yU7fhc+gp8iJ52SPIpIXb1pwZBjjgdqps/II6Z9am30+RBzsfOUcAFRg8GvNoraeDxBb6bDBNCPNJZnTaSgPBGRzmvRJG4BAHHY1Unso4pEvowi28IwEUcuzH5iT6jg/hUy0RrS956l+eOYWojg34bhismw4/3u3865rRtOnOtSHZDGm75zHcPMSvuzd66h71LS3eQsGCLnav8R7ViPrRSN3j1CwinzxAyhlA+gIOfesU0dai9ypqkN5FrckWZmVDgGOfbhTyCV6HniugjISBEGPlUCueXWZr2fzrqO1LghcRHOPfNdCqkoG74FOjZtmeKTikV5Tlzjil6jB78VIYGfc6cqo55/wA+lJj5PwrblORM8Qm0vQ4ruaJ2td6SMrAyAEEEj1pV07QO32UjrxL/APXrP8S+GJm8T6myMQr3MjKNhPU5/rWSfDFznhuv+yahxj/MaqUv5Tqho+glRxBtPTE3X9alTQtCckAR8df3xrj/APhGLofxj/vk/wCFMbw5dr/EPyNLkX84+d/yHdnSNExgAE4xxKTXmk3myXLx5LFSQAPb/wDVVx9AvFGcrj8f8KzJY3gnaJ/vKcGtaNNRvZ3Mq03K3u2NW1sraXTpPNt7gXe/CNn5NvuOvrQdKjwRmQH6j/CqdhY3N95nkMP3YyctirH9i33H7xf++jVt2e5mtVsbumaDod7LHbsb4TMvJyoXIHOODW2fh/oythrm5U/9dF4/8drio9J1RX3RS7W9RIRU62GujG28kHGP9eRisZKTekzeEopawOw/4V5o7Di8uh/wJP8ACsjxR4LsdF0Y3lrdTSOrqpVypGD9B9Ky/sniQdL6fj/p5PH60k1h4iuYDBNdyywsclGuCRn6E0oqaabnoVJwcWlDU5zHy/jXqOn/AAu0++0+2uxq0w8+JZNojXjIBx1rhh4Z1TBHlrjP98VrQQeLreBIYb6dY0G1EE/AHoParrScl7k0jOjFRb543Omk+EluPuapMT/1xB/rUDfCOTkjUZAPU23/ANesR7nxvZwyTfb7oIiZZhN0A61mt458TM2W1i4Y+5B/pWMYYh/DNM1lPDreDR05+Etx/DqOc+tuf8a9Y+Emg3HhrSdQ0+abzVadZlOwrjK4PX/dFeNab4g8f6lb/aLG6nli3FdwjjxkduRXp3wo1DxbNrl5D4iMjW7W+YiyIMOGH90ehP5VEvbXtOaY/wB01eEWj1ys2dNkzDtmtKql2n3W/A1rSdmZTWhVxmnCiiupGItTxnKfiagqWI/KRVIhktFFANWSLRSUUxDqUU3pS0wElGY29cUincit6inmo4+I9voSKTADV3TGxK6+ozVM1PYttul9+KmWw0aeDmipNtFc/sx3Phi2X5ohkfeGB+Nd3og33koGANmefqK4vyHtrxYpUaOWNwrKy4IOe/pXcaEo8+ViRwoGMe9dNPciextFCo+8CDxgGmjhs9Oc1Lgc5YfjRtHJ3D866DAiIOcY/WjrjipWVeDuH+NMKj1H4UAWbPGw+zCrJIJ/lUFooKMM55FWShGOKzluax2Gd8/570hyDkYzTyMgDFNOAen+f8ipKIznGaRueOMDtT9pK8g9cc0yRxHGzuflAyaQzm9ROPGvh4cZ+0xk4/3xXuykEV4HOr3XjTQyTsaS5jG4/wAPzjFe4rHcAf67r/s1zVHdm8NEXTKxjMYxtPWmFeCKr+VOf+W36U/ypsH98fyqUDHrK0GSuNx45FVtxJJJ5NMkhn3czHGfSomik/57tWiM2WN/vRvyODXKeJ9UutGa0ZLlwshYMMDnGPb3rBPjG9z/AK4kd+n+FFxpHo5fk0F686Xxfc952z9B/hS/8Jddf893x9BSuB6Jvo3jHWvO/wDhLLnj/SJfTjFH/CU3P/PxKKOYLHonmYo8z6V53/wlN13uJfzpyeJLp/8Al5l+u7pRzofKz0LfSF689fxPPk4uJv8AvqmjxJcHn7RN+LGlzoOU9F38UheuHg1yR8bppv8Avsirg1MuARPL/wB/DUuqkUqbZ1u/PSkL1zEl6yxqfOk5H/PQ1GLtmVj58/H/AE0NT7dFexZ1W/nGKbI3y5rk2uXfhbqVTjjMjEfzpkF1Luw1zK5H+22PyzSeIXYaoPudKXJRfypd58tuvWueW6fGfMbk4yXNMub5kYgSt17Oaj6xHsV7BnSK5O7r1oVm8scGuT/tJgf9ZIf+BmnrqZK8Fj77jT+sLsL2LOnUsX5HA60rORKCAcVyU2pkDBZj9GqodSbPLt9Sx4oVddhOkzuJZDs7/hTTIW2oB27VxB1N/wC83/fdRnU2xncT/wACq/aoXszupHI2ipd59D+VeenU2yPr2NR/2r1GV6/3u/4U1UF7M9Cmf5QMH6VheLr7ULHw5Je6d5onjKqwU/KUbg7l6EDrzXMf2qWA2FMA8nOc1W07VTP4nsoLiPNuzunzE4LlGwfbB/ofprTXPJIl+71NnRvGkKSpp2py7hkRw3UqCMS4HRlBIVvxIPseK7JILeSB5bcWyyNks6HYxP1HU1474i0B7JftAWS6sJm+4+A8LZOcuePxPX19ct9cuINIll0zUTaCGTy/ss9yHcjpkKVPH0bjHSnUw8ZXlB/I1hinC0Zo9M8Q6hYeGLWbUnSGS9kO2MMxZpG7AknOK7PTbmS80m2uJ4BbyzRK7xE52EjOM1856bfvNrUWpa002pwWkqlwZTgjPGMjpnGRxXt2meO/DV+FRpms2PQXSBVP/AhkY+pFZQpciCtX9q/I6CSRkwsbMMjBx3qPcdmKe1pHNGGhVHRhuBXGCPXPTFVltEBIKAEHBBFUzJHF66Fj1y5wAckEexwKoYI+YqME8ZNbXiO2EerjaAoaJT2HqP6VlLgdF3dgT0FebU0kz06SvBCLtCdCDjpUqqzfM4IA4BJ/lSLEW9efXpU4gZsKCXPsOKybN0iPfGCp2MQvqePrivKfFyY8TXTdd+1s+vyivZoLWSH5wqBhzucBsfnxXmHxJjf/AISGGduTJAuTjGSCR/hXTg5/vLHNjIfu7h8OU8/V76zzgS2x7+hH+Nd//ZDrPHKkiEqwbAXjO0Aj+dec/D9tvjOCPr5yOv8A47u/pXt8FiXYfLkDrWeOly1B4OCnT1Maw08R72nij8x5C25RnAwBjmtODS7aRG3QoOOXZRj8PWtRLQn5VACnsvQ/41bSzRAN2Sf7orzZVjuUIx0MZNFtJDxAhGeWMYqdNEskbBtoHOOMwittYScBQRjpnrVmK0I9j+prL2smJyitzGi0OwyWksoST2Ma1ai0OwPSxtsnqTCv+Fb9tpbyHO3j16VLPPpul/6+UPL2jXk1rGjUkrt2RyzxEb2irswJPCVlfWc9v9hgPmxsnEYHUHvXyrq+kT6Pqs9lcgLJE5jYA5wQcGvsX7Rq2ppts4BZwHjfJwfwH/6q+UfHttNYeM9RtbiTzJYrhgXP8XPB/EV62AjyNpXaOKvJyV5Wueo/CDULzWfCdxotppNqRbzDzLjaASG5+Y85PB/SvUtH8JNpUjXs90zzYPyINqDPt3ryj9nLUhFc69p5I+ZYpl/Asp/mK92muNylaK0acKjb3JjVny8sditUc67om9uakpOtaxKZm0tK67XIPY03rXWjnYo6VLEecVHToz8wqkQyejNJmjNaEi5pabmlBoAWlBptKDTEOzUUZxJKvuD+YqTNR9Ln2Zf5H/69DAc1PgbbMp9DmmGli+/SYHQUVCko2Ln0orMZ8RySySgTSyM8hAZmc5Jx7/Suz8NOJInlH8SKa4x1/wBFGMZ2c/lXT+DZN+mMc8ghP8/nWlPcmex1O4+uKQt7cZ+lR5OcDrSE5GfTjHrXQYWJlbKkehz0pueuO9RglSeOvFB4buaQy/YyYdkx15H4f/rq6WHTuB1rFhk8qVG9D6VrjpgHI9fXpWcjSGw9WAPTAphwc4HvS8FR/Kg9ep5qSxB15qtf5+yEY/iB/WrBGMc8VXvMG1b04/nSexS3OZL7fGegt6XUf/oa174or5/m/wCRt0Pn/l5j7f7Yr6CUHFc8jRDD8uAFyfX0pecGnlaCODQhMiRow375Sy+gqqRycDjPFTkcmmEVoQ2cH8Rbd54tOVWI/ePnH0FcQNKnOMvJ+letaxYLf3NnG3RSxH5Cki8MxFg23j2FHJcXNY8qGlSn+OT8Mf4VKNLkA+9J7f5xXrJ8Lw8YU8e1PXwxEP4Kfsw5zyUaXJ1zJ+OP8Kd/ZcvpJ+f/ANavXV8NQA5CU7/hG7f+5S9kx+0PJYtKkZhnzfzqzNo0kTFY1kK47t1r1NfDkA/gq03h+3PzFB0qXSdylUR4u2mTDqkn51LFpMhYfI/5162+gQZP7uhdAhH8FP2LF7RHAWHh+OQAusv/AH2RW0nhuzOB5Unv++b/ABrsbfSI06LirX9nJn7v6VhKhK+5qqyOPbQLFFQeUzd+Zn/xpr6HZrExEEgI4x5r4/nXWz6axA2r2xkV4bqdl8UTqV0tquoG3ErbCNmMZOMcZrP6vLuX7ZHdHRYd5KoyjH/PRv8AGq/9jxliQDn2dv8AGuFXSviVKy+amqnb833kH+e9TvpvjtruB1ttVt0Vhv33O4Nz3z2qXhn3LVddjsU0yJcfL0PA3Hj9ajvNNUNjbkf7xrpEgJjO5RnpTbu3AzlQAKx9i7mvtFY5JdODNjy8/wDAjVpNJhKD9znHua3FthuOAMhatJbjyxx7UpUpAqiOOudKUcCIA+9Z/wDZq7gDEvJ/u121zbAyfhVFbQSXWRyF561UKUiXNXMIaLDtG6FCT/s0f2JD/wA8UyO+K6pbbdMMjAA4qlrkv9maRdXS43RxnBx/EeB+taKmxOSscbeSaXYuY2eHzAeUABOe/NY1zq9oMmKGPjjIQHFcpe3bCVmLEkccnrn/APXSI/mRknP064+ldsKCW5ySrNmrcaxKzrgoVBIIPB/PtTLW9dNWs58hTHOmD2wG5/n+orNVnjbg8etPC7AWyQ2Mgjgj0/pXRBKLTRk5NnsWoy6fp93Ml5eWqRyqGkhldd31C9efpXFXPhPR7tZdQtEzBk4WGXMZI64x/LIrTsdFtbmyR1R2MnzOzgBiT1zzk/jTk0q60ouNOZYzccSKMKvH8W37uR2wB1712VsDKcvaLS5UKy5eV6nLWtrZ6ZFc2t3a3DSXabWjhjBESngEnPUdcAc1y5lmgDwrJu2sVztIPHfkZ/OvUodHMJJlyTnLbmyST3JP0rzvxDZGy1m5QBtrnzF3c9ef6VnXoOEEyea7LGieK9X0GXdY3bxp0aLrG31Q8fpXpvhj4oWur30VlqUQhu532pLGcozH1zyuT9a8Pc4Qc/Wrugbn8R6aBkk3UXA/3hXE0UmfQHiaBGvbeRyOYyMepB/+vWOttkDJVU6DH+ea6bxDAzi3cLnDMMk8dv8ACsWK3LHAG4+grxsQ7TZ7WGV6aI4rdekYBbHcZNW4rcx/IdpPXJJwBViK2YLy+D02r0/Gr9vZlzkgkexrjlOx1qJSW0DHG4Mem5s/oK84+Len+Q+lXI5V1kjzj0IP9a9ijs9xAC4x6Vwnxk04Hw1ZXOSTFc7SR2DKf8BWmDq/v0YYtJ0mjyzwXJ5PjrR2Y8POq/8AfXy/1r6YjtRtAHTrya+X9HY23iHS7nB/dXEbn6Bga+u4LFnxwcVvmcXKceU4sLV9nB3M5LUHoTj2qzFY5wABitlLBI13SsFUeppBdxqdlnCZn/vdh+NckcI1rUdhyxTfwleHSzjLAKvqac1zZ2reXAhuZv7qDNTmxuLvm7mIX/nmnA/GrcNvBaptiQKK7aeGtrFW83v9xzSq3+J3MxrXU9R4mmFrB/cj5Y/jVmz0ew08744gZT1kflj+NW3mA71VkuPerlKlT13fmTzTkrLRFp5go4r5M+NFt9n+JWouB8sojlH4oP6g19QST+9fO/x5t8eKLG6A+WW0Ck+6s39CKeGxLnWsxypcsGyh8Er8WfxE8rOBdWskf8n/APZa+jZJ+etfJngC9Nh4+0SccZuFjP0b5f619SPJx1rnzRuNRNdUb4SPNE2VO5Vb1GaWoLR99rGfbFTV103eKZMlZ2KlymH3etQ+1XLhd0X0NU67IPQwmtQpynBzSUH2rQzZNmlpgOQDS1ZA6ikzRTAdnmlzTM04GmIdUch2vEffB/KnZqO44hLd1Ib8jQwJTSxfe60080sX36ALm7Heik3e9FFgPkH7FK7i0QKZSdmM8Z6Vv+GdKutMguIrkKAzgrhs9qzopxFqSXDBsCTcQOvWt0a9bbc+XLj8P8aI2WpMrmltPvSAc89KoDW4B92OU/l/jSnXIGPKS5/D/GtOZEcrLuBijvj+Rqh/bNuf+WUvT2/xpv8AbUGeI5OfYUcyDlZo4rVtpTJbpknK8GuX/tqD/nlL+GKs23iKCINuhlIJzxipk0XFWOmB280044OTWGfFFoCcQT4HsP8AGmnxTanrbzfp/jWZpob2c88cDiq9+c2rcEZwOB71kf8ACU2o4+zzHn2/xqrd+LLR4Sgt58nvxj+dJ7DTRVumA8VaIRwPtUfB7/OtfQiivmuPVI9T8U6QY45ECXUfDYyfnH+FfSq4xWUi0LTT0p3WmE0ITISOaaRT2xTDVmZh69rVtoBtbu6XdGztH94DBK57/SqUXxQ0OMfNg/8AbZa6KbSrDVsRahZwXUSfOqTRhwD64PfmnR+DvDi8jQtNHP8Az6p/hWkU+hLaOfb4saEBwE/Gcf4U0fF3Q8dIv+/5/wDia6tPCnh9emiaaP8At1T/AAqwnhzQ1+7o9gP+3VP8KfvdxaHFH4waIv8Az7/jO3/xFN/4XBo5ztFrj/ru3/xFd6uh6SOml2Q/7d0/wqRdK05elhaj6Qr/AIUe93DQ8+Pxe0oDOLQD/ru//wARV3SviQuv3n2LTLOC4uNhfYszD5R3yVAruF0+zU8WkA+ka1WvNPtUv7K4Hl25QuCyhVzlenIpWl3HzLsZI1PxE43LoURGcf8AHyKUah4k/wCgJAMf9PA/xrZglsLSFIPt0bBc4aScFjk55OfekbUNNQkNfWw+sy/40/mL5GWl/wCI+p0mAD/roD/7NVgX2vt/zD4B+IP/ALPVhtV0gcHU7ID3uF/xqNtb0IEE6tp4+twn+NS4+Y0/Ib9s1/A/0KAfgP8A4ug3GvFf+Pa2B9x/9nTW8ReHxkHWNOH/AG3T/Goj4m8NqRnWdPH/AG3X/GlyLuPm8iuNQ1t7+e02WivEiOd3AIbdjHzf7JpkkWtyH5nssf71U18UeGYvFFzcNqln5clnEu9WyCVeTjjv81Xm8ceFF5/te2/BWP8ASp5E92VztdCD7FqvGZLLr/z0/wDrUybTdVlxiayX/toT/wCy1M3j7wov/MVi/CN/8KafiF4UGP8AiZgn2hf/AAqfYw7le1l2Ko0nVw3/AB82eenDf/Y1NHpOr7cfarVfxz/7LSN8RfCw/wCX9z9IH/wqNviR4XB4upj9IGo9jBh7WYT6NqYcIdRtgzgkADJOOv8ADXHQT+L2BeLw/fMp5DBosMPUZrrJfiP4WkA8xppMdAbfOPzrqtPdJtOt5o+I5I1ZQRjAIyKXsY9A9rLqeXhvGxORoF8pz3lt6w/Ftz4mi0Yrq2nTWtvJIFDySwtlhyB8vPb9K9yIB7k15X8ap9mnaXbBj87ySHHsAP8A2Y01SSYOo2jw+5l3yPH1HSktJAVwwGV688+lMmjZn2bvnA49/pVe3LLdEMCDgnBOOa2MzTH3gcnr074/z/WpTtKDaBgHPA71WZl2HPpgAdf89KsIAybgB82D78/5AqhHp3hi787w9bSQqjzJGFYnsRx/SrMksz3cRlVctuXAHsf8KxPBMhk0dgrBmjlZM598j+ddDdSLDCoklUyMwVDjOT0/Svepy5qaZlazGFURRjl2OcEkcZrhfHduPOt7sA4bchzx7iu+i8xo13hN2zDMuOTj6CuX8bW/m6M0g6xMHBH6/oazxEOakxp6nmE8ZUZ6rn0rpvhnpjal4705cjbCxuGyeyjP88VgTRb43xlsc10fwsultviBp+84EgeMfUqa8CWxvHc+gdZhEtsmQOHB557Vn2+ngoCDj6/4Va8Q6vBo+iXGo3EUkkMAVmWMDdgsBxk+9cVF8YvDiAH7DqI57Rp/8VXjYuhVnO8Eevhq8IU7SZ20dhcebGqlPJ/iyDn8BWpFaggAdO9eeD42eHVOGstS/wC/af8AxVTp8cfCqgf6FqhP/XNP/iq4ng8Q94lyxcejPR47UZGD+lc18UNIkvPAV2Yk3NC6SjHYBsE/kTTdG+MPg++SRpGu7bYMgTRD5vYbSf1xWL4p+Nej3Oj3un2NtdKZoWjDlFI5GOea3oYSUGpPc5Z1nLToeLWXk295btcAmNZMuq+mc8H/AD1r7C0+/wDt9jbz2MLeVNGrrJJxwRkV8Vz36zN8qlTnNe/eHvjv4Z07wzplld2+ofabe1jhl8uFSpZVAJB3dOK9WdOTaZytq1j2D7CJG3XMhlPoen5VOPLiXCqAPavJm/aE8J44ttU/78p/8XVZ/wBoLws3/Lpqv/fpP/i6zacPghqTZvdnrz3AA61WkufevJG+Pvhgji01T/v0n/xdQN8ePDTdLTU/+/Sf/FVyVI4mX2WawjBbs9Ze496rPcV5U3x08Nn/AJc9S/79p/8AFUw/HHw4f+XPUv8Av2n/AMVXJLC4l/ZZ0RlSXU9ReUmvHfjnbebZaVcgfcaVCfrtI/kavH43+HP+fTUv+/af/FVy3jr4j6H4o0WO0tra9WZJd4MiKBjBHZj6itcJhq9OtGUo6Dq1KTptJnmun3Js9StLpfvQzK4/Ag19bAlwCuSGGRivkDIxxnOeK9lg+MlhHplrbvHemVIUSTy0VF3BQD/FuIznuK7sxw063LyIywdaMLpnuOm5FqUbqrEYzz61crgPhh4us/E8OpJaxvGbdkZlZAv3s9ME5+73rvqKUHCCjLcc2pSbQjDcpHrWe3BI9K0TVGddspxXTTMZjM0ZpuaWtzEkQ/KPbinVGh6j3p+ferRAtLmm5pc0wFzSim0A0xD80jjchU9xikzS5pgJG26JT7VJF9+oYvulfRiKlj+9SAu9u/5UUz8aKAPka/ZoLV3Q4dQDnHvitNPDmqMgP2+AZGf9X/8AWrP1VS1hKMdcfzFem6Pol5q52WcDSsi5YKBwKIpPcmba2OIHhrVcj/T4Mf8AXOpF8L6mcj7dDn/rnXZy2kltcGGZSrqdrBhgg12Hh7wgNVtTOZNijgZGc1bjFakKcjxo+GNVI2i+iLY6CI0knhvVU4e+jUkcZhr03X9FbSbpot3TuK5uXLHkkkjHPWhQTDnZyR8O6kcY1KLp/wA8qP8AhHdRbCnU4hg/88R/jXUU09OBzT5EHOzEHgzVm4/taLn/AKY0q+C9SOSdYjx/1w/+vXX2r+ZbK2eR8pqyqgHn8ayasbrVHEf8IRfk860n4W//ANeoLnwNdpEXbWA3QEfZsd/rXekZByar3/8Ax6OxxwV6fUVL2KSR5dp9m9h4vtLZ5vNMV3D8wXbnJB6V9Rp92vmiQ48exAH/AJe4Dx/wGvpdBhcVlIa3FpjVJ+FMakgZC1RmpWGajPWtUZsqXmqjSESZoWl3tswpAxwTn9Kq/wDCbDOPsD8H/nqP8Kr+KD/oEBzj9+P/AEFq5jODW0NjKb1OxHjnH/MOb8Zv/rUp8eEDjTvzm/8Asa4zd+lGeaqxHMzsv+E8l/6BqD/tsf8A4mmnx7cHpp8X/f4/4VyGe1JnGcUrBzM7D/hPLr+Gwhz/ANdD/hWF4n1u68UWlvpcsMUEUs2WdGJYYBPf3FZoY9cUgb/TLQ56Sf0NNoLsxpvB9vA4U3MrZ6HA/wAKaPClp/z2mH4r/hXUal99M9xxVFgeNpxUuESlORjDwrYjrLORnuw/wqVPCmmkHLTn1+etA7h17VPCHJzn5RQ4oFJmavhPTR/z2/7+U7/hFNM7pJ/38Nawb2p+78anlRXMzA/4R3Tk1Rk8uQx/ZQ+wyHGd5GaZeaJpsUO+O3xg/wB88/rWu/8AyFE6/NaOPykX/GorsBrdxjoKFFWByZx7w2sWqCN4wIthbBY8frVYxLKk8kJ+SPJBzyT/AIVY1bBnhiUjdK4X8B1qGABU1KPngnj2xU2KuJciNdMhmVcMSMmpr+FFtkaFAPmBOPTvVOV93hmM45AHfHenancj+zY1z8zKDgHnFFkFy3FEk80cpjCwlwFA/iH0rYXxdr8GsS2CarcLbRIBGikAKBx1x/nFZrSBbWIj/ZOR06VFHEt14n1HqGiY42ntk1N0ilqauo+K/EEUtrs1q8VWfDASdeRWH4o1a81KaJLq7muDEpI82QttyecZrYl063mCO7t8hyvPT/PFcz4hjS1v5FU/KI1Y5OSSf8/ypRmmxuLRhzrAqb5S2euB361UgbfKWOQoHA9OacIZLl9zHapPAz/n/OKSI7L1lJB+XHHatCSxJwCSQM+n+f8AP41LalPJdSB8h56dD0/rVd2yMk8dKS3lCXOMjDDBOOh/z/OgDt/BVyqG5iZipeQNknPVf1ziulu/31woTOI2BGB79q5HwfLGlxdqyhmCqykDjOT+XQV2KulraiSUfO5HB9c/5/KvYwr5qaRnLRk7NOWwowq579enP+fWsfxJHI+kyRuoDOhPOa6CMzRfJLjaCfmHb/PNcr4ne71EywwOY0QHkf5/Wt6ukGT1POI2OwNycZzntWr4OtpJfG2lRwA71ukcEd1Byf0BrGi/dM8Z+8pIP4V1Hw9eVfH2ltHH5mWKt7Dacn8BXz8tDdHver6QuuaRdaY0nli5j8vft3beeuO9clZ/ACBsFtYkZfU24A/nXoVpJ5dzGwCZB/j+7+NdEbG5uzm9uwI/+eUPAP1PU159Zz5lys6YtJankep/Bnw8lgbW21C6m1HIC+Uike+R2/Oqth+zqsw33utyRA/wJCCfzzXuEMdrZpsgjVAPQUrXI9azVX2espfJCb5tEjyFv2eNJUfJr18repiQ1RvP2e7GO2ldPEF0zqhKgwLgkDjvXsr3PvVaW4yCCawqY63wlRpye58UXlobR9pO7OR0xzVSt/xNA0OrXUJXHlTyJ+v/ANasCvXpy5opmE1Z2CirllpV/qRYWFnPdMvLLBGXKj1IHSr6+D/Ej/d0DUzjr/oj/wCFU5JbsSi30MSitUeGddMPnf2NqHldN/2Z8fnipIfCXiK5GYdD1Fx6rbP/AIUuePcOSXYxaK6D/hBvFP8A0L+pf+Azf4Uh8D+KB/zL+pf+A7f4Ue0h3Qezn2MCit8eCPFB/wCYBqH/AH4b/Cqmo+HNX0iOOTUdPuLVJCQhlQruIGcChVIN2TBwkldozeNo9e9eg+CPh1aeLNCfUJNSlt3SdoTGsQYcBTnOfevPa9d+DV/ix1WzJHyyRyqPqCD/ACFZYqUo03KG5thYxlUSkd58NvBUPg/VbwwahJcLdQgFXjCgFTkHr7mvSjXI6Pcf8TiEZ4OR+hrrjXHRqSmry3OqtTjCVo7CVXuk+UMO3FWKbIN0RFdEHZmEtUZ1L2pSMHFJXSc4qnDn3H+f51JUI/1in1yP8/lUmatEsUngn+VRfaRtyI5OuMY5pXDkjY4A7jHWkVZs/NLn6LVCE+1f9MpPyp4n64RjgZHvTQku0/vfm7HHajZLx++IHfA609BE0b71ztK9sGn596rCOX/nufyqcUAIvErj1ANTR/eqAnEy+4IqxF96gCzz7/nRS4NFAHyZqeBbMeOWGB75r1Lw94kvNBLSWhTDqA6uuQa8q1U/uOn8YHI969F0yMS3cELMQkjrG5HoTShs7kVHbUuXepTahqEt3cEGWRtzYGBXRaF4vn0qAwoEdD/e9a5XVFMGpXEDPvMMjRA+ynFV/MKn6Vro0ZJ9TZ1jWJNUnaWVgWY9RWM5zk5z6980by3GeaQocU0BEeM5IxTSMZ5HvT2XbnnkVGdv5mmMt6fJtmMZ5DfzrV2ZyR2rn1Yq4YHBBzXQQuJIVkHRgD6VjUXU2pO+gbT36VWvFP2V88gEH9ausMj0AqreYFnLweBn9axb0N0tTzabb/wncRH/AD9w/wA1r6YXpXzPOCvjuIsACbmHp/wH/wDXX0yozWbBbsDTD7VIRTGpoTImFRGpmFRtWiM2YPigf8S6IjtMP/QWrkyD3Ndh4kTfpg9pQf51ymzjitoPQynuRYI6n8KO3J96k8vJxTvJ5zxg1VyLEBHvS4PepfKJOOtTf2beHpaT89/LNFwtYqdxzRj/AEi35P8ArV/Xiro067720oHupFMktHikt/MjKFpUxn/eFDulcUWm7FjUVDMhyeh6VTaONR8zYz7962NTjRBH9D/Ssl40lOD0FTe7LtZFbMW77w/A1ZhaEpgv85PAqE20YI5/WrMFvARv5yORzQwQ/wAlSM80ohQ8f1xTwQBQW5PPFRqXoVVhRtcsEb7jpKhx9UNdp/whumSIys9xzwcsP8K40uF1fTnHaSQZP+4D/SvWEwcEbsnrxTQ7I4//AIVz4clmSR455JVPysZWGPyFcWfDOn2/xVm0VoibG4jRtm85OUPfr1FeyDaJQoY8mvNfEDLb/G3SnJx51tGv1O5hQxmyfh34XSD7Oun5iHVTM5H86uR/D7wq6ANpEDA9d5Zj+HP6VuBACfl3c9xmp4yFlCjAUH2osM+e9at4bXUr22gGFgmeNV9ArED9Kowyi28TahL3aJX+uVH+Nbvi3aPGepwbct57HIGM55/rXNao/ka6zfKBJbx9+vyj/CuWTvdFxVtTVa9LLt2Y9cVx/im73XxVX+Yqu/H44/nXTq2BweK4/VJLeC9meRS87OTgjJ69falRjrcuo9DPigJXzZ5CIwM4BqK1/eXDFeCxwB171FcXMsxwxwvXaOlTWKqdzPwo4PFdRgWZQG6HOevP1pkSKGBBOe+MHildzIxCdD0GM8VYt4TtDNncRkd/SgDoPB7IuqTqRjdErDkEEg//AF67NwlzcKjA+Wg5x3rz3Qz5OpYUoN6cZ+o6e/8AnivQdKxJy/CL1z3/APrV62B1jYzmbHnJLAjSADIBP+PSsO7uUbdGqBmJPOKtLDcT5dcrFIBtUDBUfzqtqb22i2Blk5kI2onGScdv8a6Zyb9BWVjyzWIBZa3PGD/FuPPQnnFegfCTSrO/1ue/k3+dYxholU4BLZBJ9cAdOnNeaX9xJcahNJMRvZucDivQfg3qCw+KJ7RmAFxbEKPVlIP8s14NXVuxtA9xI7ZxW7DqHmW8bg/eUH9Kw+lRWVyRB5ZP3HdPwDED+VeJmLcYpo78NBTbTOi+1Z70xrj3rNWY+tKJGPTmvFdSR2ewSLpnJ71E0mc81Tury2sYvMvLiOBPV2xn6DvXNy+PtOkuhaaZG1zOQcM/yJwCfr29qcadSeyC0UeM/EK3+z+JdVT0uWcDH97n+tcMetdp4y1CXVdWnvp40R7hVfagwOBj+n864xhhyPevqcNdU0meVX+Nnb/DHxDNoGu3RhhWU3FuUwexDA56/WvfNJvtQ1nSra4kMUayghgVyVZTg8dOevTvXzN4RZh4ltVVsF9yZ+qmvoLwlLLbQPA0zPGDuUEcAnrj8hXFmFNX5ludeE1Vjql0+AOJJi07jo0hzj6DoKnLqowoAHtVcz8ZzULzD1ry7tncqd9yw0uaiaXrz2qo9wB+NRG5G4DnrQos0UUiy0tedfF2Pz/DFtKASYrtc49CrD/Cu0e59q5Px+PtXg6/X+KMLIPbawP8s104a8akWZ10nTaPn8ffGfWu6+FV0YfEtxCTxNbMMepBB/xriJhtnce9b/ga5+zeMbF84DMyfmpFe5WXNTaPEovlqJnvtnP5Wo20hPSVc/TNeiGvJTdFSCOo5BFerwSie1jkHR0DD8RmvOw6tdHo4jWzFoNL2ptdKOZlORdrkYplT3QwwPtVfPFdMdjnkrMDxg+4p5/Wo2OUOPSn5z+NWiGLRSdTRVIQtLTaKYD80Cm0UxCSNjY3owzVmMfNVaXmJh3xViE7tpHfmkBb49T+dFJzRTEfJl8pbykHOZkGfxFd9GywXCMrE7GDZxjpXCXAzc2o4+a5T+ddtld3U8c0oapk1Fcv6xPFd6zeXNsS0UshdTjHXk/rmqWcD2pyMintjPXFNJ+c7fwNaLRWMkrIu2ccbTLvIA75rsvFOi6Np+kW0tlOGnfHyhgdwx972rgVlwcg4PrUj3DuACaGrsaIpMZIFREGnls44FNbkmqAYc561r6XNuhaM/wnIH1rJPPXt71YsZhFdKScKflP41M1dFwdmbpBIzVS+YfYpSRn5T0FXwvPSmz2UU9tMXheQhSTiQrkY6VySdkdijdnlt8AvjmLn/l4hz/47X0yn3a+b9daL/hYMfkwCBfOg+QMzf3c8k19IIPlqb3SFs2OIqNhUtMYU4ksgZcc1Ec1Ow4qMitEZsxvEP8AyC245Dqa5Inj3rt9UtJLyzaCLG9iMZOO9Yq+GNQbvCB/vGqiyXBvYw1ZuenHbHvRyeT29q318J33eSEf8CP+FPHhG95zPCPxP+FXdB7OXY50HBHP44rtUjvJdQCRp+6Me7/VnHTru9c9qzh4Oue9zEP+Amu0hUxQxIckhAMg+gqWlJicHbU5aayv9+0o4X/drJ1uN4zZGSNlbzejA/3lr0QHPQH86xPFGnR3kUEjs6mEMwx+B5/Kuidbmhy2MaeG5Z8yZy2uDasZxnrWEZOeBXb6jpMVwE3u4Hsazh4ZsuCZ3XPOGcD+lYqyWprKLOZJyM0gcgYrp38O2gKlZCVz82XPT8qkg8OWkz7ERmPX7xFaqm2r9DO6va5ym9uTxR5jdCeK7T/hDkP/ACxUfWU05fB8QGPIjP8A20NLlXcrlZw0j/6Tp5J6XBH5xv8A4V61E8zKCqpjAxkn0rg9W023s7zTlSNVIvUDAH2Yf1rr4X1NxEYo7YwlRguxzjHoKya1NUtCw7XrSZHlAdvvV5z47c2vxR8K3BQb3ESMe3EnP869Bk/tYEBDaKoGT8rVzfiexFzqWm3eoRW0s8Tf6NKqsNhyD0P4VLVluOMbs6122yOMd6fCqn53wBn061nNa6uSCL22B7/us/1qVItUCFJNQhIIwAsAyCe/Wk5tdCuXzPIvH0YTx7fuI2AYKckdcoK5HxCjfarRowdv2dTyfQsOteweJNKU3ryXUcc9xtGZTEATxxXGG1SW0gLRRuNpU7xz94+v1rk5veZsqeiOXuZWgsZZo8b1XK+ma45/tJlaWeRMk5ZioJ/OvUfENkB4dnCxoh2K25RyBkE15xc2rO6IVOxeenU10UUrGdVWZkuY5XZkhOOuc1NaqDCQV5HaprgxxYVUwzcAdM/4VVtJBu56Z7VqZF1It7ZbhR25qyeT7Yz6VAshyOQPXnmpR0C9F7Dt+H5UxF3R7WG41KVrhwqKoKkNg5ya7211CyjHlp+8XjOehFeW3UxiuIsSyIGB+ZBk4/H6fzqRbyQkf6XdtjDHkLjn2/xFdVHF+xjZIThzbnr8GpxmJhGoLqCFQdTzwK5fU7CWVmvdXvrW0U/dWTJwPTHrzWLBdLdxX0UUMMU0do00EkckxYsrDpudlPBPYVy80d3PMWuBLJu+UOxzzjI5pvMfaaNFulyrQ0dZtNMZZprS5mklQjlowikfTJNVvDGr/wBieI7DUMHbDMrMefudG4+hNZ0tndW6HepA7gHNVlODXLOSm72EtD6+VlcB0bKsMqQeoqpDC/n3HZBLnJ4HIB/rXPfDbWDrHgiyZ23S2ubWQ5/u4x/46VropNMN5cl3ndYRj5FPevJx8E6ep6GDlaeg6XU7K0wGcyuP4Yx/WsfW9Y1f+y5bizhNpFHyWxkkZx1P17V0lrp1laf6uFd394jJpb/yp7WSGRFZHXBUjINeRHki9Eeg7yPF7lri/ZmnmkmlbnLEmiy02a3uEuB8vlnP510k1rHbzsqoFAPYVXlT/R5B0Ow/yr0o2toYNanA+M9O/s+5gQEsHhyM/U1xEn3zjp9c16d48QTWOlXGMlkKk/l/jXmcow1d+Hd4Hn4hWmXvD83keIdPk6YuEB+hODX0RpT+WT79a+aYZDFPHIDgowYfga+jrKUMqMucEZrHGK6R0YF7nQi5+X9Ka7SFN+1ivrWZI87QObbaZQMqG5B9qrzXOoRTsUZXi/ikfapAyC23nkben19q8zkPTuaLzYqLz8HI9c1zn2zU3QjcqlWIdyyYQbup9DjH5niqzX2pkbkmh27QTudMZ4zjn2P51oqZPMdNNJh26YyaytbT7Vot9bkZ8y3dR9SpotJp3iLTtuJY7G45XjH3eO9PZ+CDjB6mqirMmTurHzzcD95n+8Aat6RP9k1uwuD91JkY/QNzUV/CYLhoj1jZkP4Mar5OEb04/wA/nXtbo8LaR7y74LDivVfDk/2jw9YyZz+6C/8AfPH9K8atrv7TZQT9pI1f8xmvUvAtz5/hxUz/AKqZ0/8AZv8A2avNp6SselUd4pnSk03vSnrSZroRzsjuBujPtVI1fPIxVFhtcitoPoYzQn8qRP8AVrnrjFHShehHof8A69aoyY4GjOaTpUF9LJBY3EsIUypEzIG6EgEjNUkIs9aK8t0z4jatfapaWskdnHHNMkbMI2yAzAEj5vek1H4iaxaahcQQpbNEkhCM8DKWXPBIzxW3sJXsZe1ja56mDS5rB8J61Nrugx3k4jE29kcIMDIPH6EVuZqGrOxad1dD+1Ps+UUf3ePyqLNPs+JJF9Gz+YqRmhx6n86KM0UxHyhMc31l1wblOPxr0C00a/vbC4vYLYtbWwzLIWUBeM9zyfYe1ef8tqdgP+nhev412X2iXyvK8xxETuKBvlJ9cetFPYmY8SAHnP4GnvJFj5V/E1Xz7mnKA2Mtjn0rQhC7hzk/QUA5J/SnvFCIyyzAtkcetRY6d80XBqxqRaJfXGjyarFb77SJisjqwJX6rnIHPWsxmJ69PQ1JG5EZTzGVG+8AeDj1HeoCeTxQgFz3poPOaCc59+nFA6etMDrLFvPtY5uuVGeO/erUsLi1k4JO08Vl+GZQ5lt2xn7657+tdYbVmt3PfaTz3rzK8uRtHq0I88UzxLxCrDx/Hv5Ilh49Pu19JR/dFfO/jCMJ8Rl68yQE5/4DX0QnKinF3imYzVpscaYwqSmNVozZCwqM1K1Rkc1oZsIl3ToPervkA9KgtFVrpA2T1/lWqI19OazlPlZvT2KQgp3k45zV1oQrYxSeUCelL2hpoU/KHPQ1Jt6DngVP5R7CsHUzeJqsMMFs8kTth3GQF6fgKcarvZK5lVStc2QBnkgflVLWQHs25BIU9D7Vi3tjqBB8gSEn3P8AjUlhbXccdwtzC6jy/lZh1613Sp/uue5xU6zdVQ5S7crmNDjrTWsZ7iG3aBGPOCQQMc9TnqB6Vbdd8EWeeB1+lXrIAWKfUiuVWnozrqRsjIu9HuicxkkYPRv6UabZXdrckzRERlcbye+a6AAYzgmknH7gY4wwP612Ks1DkONUI8/MZ4Y5OWz+P/16RW/e/fGB71E0b+awWLI7c0qxkPzEB79ayubnI+JG3aivA+S9hYfi4H9a6yyLCzhYKPuAc1y3iBN+pToM5EkD/wDkRP8A69dRZDdaRKc/KWB/AkVhCpebRtOFoJliRiV+bavbrXPeKwv2ewkGTtm6jtXRTAbVG4g9eKw/E426Mj4J2TKcmiv/AA2Kj8aNiQcq2eopVI7kk/SmpJmGJgfvIDjGe1SLIxxwfyrRbEPcxfEVvm5UjoyDOa4fT7TzYTHlU2zSKT2HNemalCJhGxQ8DFchp+lFLy9gkV+J2ZSB2IBryK1XlqyR6FJXhEzDYvPayRlFZTGcq2AGGOhzXialkyTu2r91QM19HvpanIKuePSvn+/sHstau9PVHLRTvEF2kkndgYFdODrKbaMsVC1mc3fRM37w8MOeO1U4MbgMD612d94J8SPbGQ6Ff4C7uYGB/LGa4tUKuyvuVgcEEYIrtumcdmtzViICjjrzjOetSmQIgZjx0b+VVYpCyYB3Zxx1qwmGjOSffjH+f/11RIiQw3sqs6vIANvBxn/Oa1o9I07KsYsDrgntVGwRAeBjOTtGOn+Fbax+ahAYqccAev4Vm2aJGnJe3ap9ne7u3ixjy2uXaP8A75Jx+lUJnkcFFUM54OQKsmJXyShJPOC/4/4flVy2tVi/eMoU44xngCsktTS+hzeqad+7SFpGeWRtzEj7oHOP1Fc1qmnmzfI+6fbH+e9drd3f+kSCBPOuGGPZB/Ssa80S4O+WabLMMsoHArVMzaOi+DGsm11270l2/d3kW+MH/nonP6qW/IV7gr7Dj8a+W/C1/wD2P4w027LYWG6UOf8AZJw36E19QSq24bQTjrXJjY3ps6cG7VETGaqd3P8AKRSq2W2d/Qdap6iGjxkEZHcV4sY3Z7GiMC/X/SGI781nMy5KkjBFUfHUM9xp8Bt1kdhICQgJ4wa4M6ffsebS5JwRnyzXqUqacbtnBWqOMrJHTeKlE3hOykHPlTFePTn/AArzC4BDnPXJFekyQS/8K/nhkRkeCfO1gQeo/wAa85u1xK3Hf0rpw+l0cmI1aZXHWvfdAm87RNPk3ZLW8eT/AMBFeG2Ol3morMbSAy+Su+TBAwPXmvX/AAXciTwnp5JJKqVOe2GI/kKWKV4l4R2kzr4ZVBwW25HDY6Vm3Ol2cmU813UliVAAUbiCR09Bj6VIZTuI471C0hIxnivPselzCW9tHaFirs24BRuxwMk/zY1W+wQIylXkBVdo5HdVX09FFSlx0z+NQSXcEed8qfLjPPrVWZLkieMiCCOJMlUUKM+n4U1pCe1Zt5rMNsNoEkjq20qo74zVe11c3F4YGjKDJ2nOckdafKyXNbHmnimAQ6/fKOnnlvwb5v61j5/dY9D/AJ/lXVeO4PL1p5Mf62JHz9Mr/QVyi52sPxr1KbvBHkVVabPU/DVwZvDlk2ckJtP4Ej+leq/Di43W+oW5PKujj8QR/QV454Jk83w+Vz/q5mX+R/rXqPw9l8vW7iL/AJ6QE/kR/ia4pK1RndF3po9KNNNKfejtWpixtVLhcPn1q3UNwuUz6VcHqRJXRUz0o6OffBoobhh15FdCMGLmkIDKQeQRg0lKOapCPD9MgNjq2qRRIr3ttHILYMufnVwCQD32BiKs3KN4ns7jUvJjt7qOXG57g7JQcswHmMcFeuBxg9OlX9dlsdO8aXqXdsiLvEsc0asZNzYb+8B3PT0FRXfiHSdR2G8NxMUXCh4t2Dnnq3fjtXoJt2ZxtLY6H4XXO7Tb616+XKsgH+8Mf+y13wrzTwJqdu/iW4traJI7eWDKYjCsSMdcfU16UDXLW+M3p/COp9uSLsjsyfyP/wBeo6VDi4hbPGSD+NZGhpZ9jRS/j+lFMR8oR/8AITsAD1mBrr8HbxznvXN6NbpeeKNKt3+68/OD7Gu01LTpdNmCFhJG2SrHg1MJpPlHODa5jPwR2pQCeMUoyP8APWj5vb0rYxsOMbqVBHLDIwaPKcsF7n0pMt1zxSlm4OfzpABiCscnIHoKsxWKskkguIf3ah9jEhmz2HHUVX3NjGaXJxnNA0M2EnoRz6UqxMT909aXLevtUiFs9TSbGkaGiRTxanbyrEzAOA2O4PBr1xNIYxcr1HYV5n4fhZ7pS05TBBOO9e4R4MakelcFel7adrnZCpKlDTqfM3xGsWtPiYuFKgtbnPXPT/P4V7og+XFeUfGR0HxA08D73lQ5/wC+2r1hB8tEY8seXsDfNqxaa1P7UxutaIlkTCmGpGphqzJlnToy94nHTJreW2RTuA+btzWRpDYvMeqmt6lyJu7HzNEPk5Oc0eSKmoo9lEOZkJg5zmqN1GUmA9a1KzdRBaQKDjgciqhBRd0JybIgrev6024VTAQ7AZ4yacoHrn61T1dtunsx6BgePrTqaxZUPiRJJEscSoxHygZ/KnQXEKW4BnjUA92FZd7fgWPm+oXv7Vyv29pZSmCFOe9YUU73NKstLHo0VzBM+2OaN2HJCsDVW61nTY1eNr23EiMAymQAgg81wug62bfUrgtGoVUKls9eQa4LxJfCXX9RTBz9olU9OfmNW6rU3EhQ9257adc00Lu/tG0APTMgFQHxRog4Or2ZPtID/KvnKeLc/FWLItGwUKMN0z/Sh1mgUEepzanDqXja+8i5E1p9kV0K/d3Kyk/yrrLbU7aG4ltWnxILmVcFSerEjtivI/D0pg1K4xx/okw9c/KTXrWjS711CUHlrtm/NVrnpNurdG80vZ6hN4o0uN2BnnbA42Wzn+lYfivxHZz+H5IYor4yl0YFrZ0HUdSRiuvyM5yd31rL8WQibwjfEZLBQ2TzyGBrskm4tM54uzuiK01eVbKBZNI1MukSq37kAZA9zVuPVZnAK6Re4z3MYx/49V9SXjQ9ioPFO25XG3I9zSSfcG0aWmjzrISSwNGxJyjkEj8iRUdvDbtql38i7jtbp7YqNLryU2dOM49KyLPUDHr0xZuHiXj6E15uIrwVRRa23N6dOTi2jqTaw8/IK4eTwxp1r4/u9ZFupuJUj2E9EO0hmA9TgD/9ZrqDqgrG1vUBEEu0GJANm7IwO4z+v501iaLklBWKp0pp+8RahdSXen3a2Pz3Ue5VVSAQ3YfNgentXB+NPhfPr3huTVI9HEXiGOIO5jkUeeR94MASCcZweucc13ENyt3Ai27yGYneuGGHU9VBPQ55FdDBeSG0jEsZ8zaMhmHWuuLitQqp7HxQhKyYYYIOCGHI/CrUchbEYHJPQDFXvG1ylx461uZNgV72U4QEDO7375z+OazrJTLKAoyAM/5/Kui+lzktrY27OJVUb2G30B79/wDPtWqjxpGXOAuAuP71ZtuhOOTjOGBxgCphKZr0QqCUjG4j3qCzUj3AKw4KhRjpipLuYCE+bN5UX/LWXPOB2H41RUzKp8q3eUkkKqr3579v881mzb7m82XOLy4BylnCSUT3cj+QpJDbLkU0t2hOnILKzBw11KMu59v8feq+o/aLexnjsoQcg7ppGDs477fyrREGpRhVmluTgYENrCoVfbLYFPZCQ0k1skUJxuCsZD+IUECh6AtTzXcd+7PzZzmvsHwbexX1hZ3U6qwuLOOXkZ+YqpP8zXyRqcKQ6jMsUbRxbsorelfSHga6ZfAuizAn5bVVz9Mj+lRX/htoKSvLlPSo7PSoblrhYUEh9TkD6DoKwPG0kBsrYxKgPmEEgdsVTOqN/erJ1q8NxaqCfuvmvHjVlL3bI9OOH5XzNmLK25AT6VUYYPIzU27dHjnhjUTDJBPT2roiVIx9Sj36VrUOP4BJ+mf6V5HeAhj15ANevX9zb20t3FcybPtNoyqME7iMjHH1ryK6GfriuvDnFiehu+A4pbnW5LSKQRtNFjJ6HkcGuu8Auf7Entz1gunT+RrzTS786be+fs3jaVK5x196734dXfnnVQAVBlWULnON2f8AAVpXj7rZGHl76R3M8vlQvLjO0E4H0zXPHWZ2ViYhGwGf1H9DXQTpvtmA/iUis3TbWxuYwJwNzQPyGx8wXj+XSvPukrs661TkVznpJLu5gaSa4k2KQ2BxwWwR+FQ3KGN7gBiW3ujEnqAAwrV1FYYmuoLZt0DoWUZ6EDNZtwd8xP8Az0aNh77lwa1jqjNSurkQvN0xuW5w8bn8sGrtvIp1hMAgmQuM8cMo4qiYBDbRzKSfMj349CCv/wBerjS/aZ2uYFkwAjD5epBwabKRk/EGPM9pIBjdG6k/Qg/1NcIvU/SvRvHaLJYWkmcsspXrzhlP+ArzlMCQZ6ZrsoP3EceIXvnb+AJswX8OejIwH5g/yFeoeDZ/J8U2mTgOGQ/ipx/IV4/4GnMWrTxdpIT+YI/+vXo+iXxj1SwugrIFnRirDBA3DINc1dWqXOmi707HuNJmlIptWiGHemONykU6g1SJZnkYNMY8r9ammXDn3qCXhM+nNdCOeQ7NGeabmlzVEmdf+HtJ1O5+0XtjFNMAF3tnOKgHhPw+p/5BNr+KZrYpc1XM+4rIoWmh6VYTLPaafbwyqCA6IAQO9aOaaKM0O7Cw/PNDHam7+6Q35GkFOVd6smeoIoA1Ovaiq0VynlJkjO0ZooEfN3hNPM8caSOcb5CPwRq7/wAVjBg6jJPSuA8I3ENr4306adgqDzcn/gBFd/4tnSSS1EZ3oys3mD7tcyT9qjpuvZM5oZ6ClxzzzSd/agkggAcdzmu25xsUc+uK0NL0XUNXkZLG3MrJywBAx+ZrPXOR9K6nwd4gg0O9ka5jdo5EILKRkHtwetZ1JNK6IldbFO+8Jaxptobm6tDHCv3mLL/LNYpUivQPFXjSx1fS2tbaOUMSDliMVwDMcDGCM81MJt7ji27iDPtUkfGKh6il3FZkXI2kHPrVs0RtWlyYFyDyRXtlheq9hAxPWNT+lfPa3iB8Z4xnOa9csbmcWds4lXyRbgGPbyWwMHOfrxXn4qThaUTsow9orHlHxcufM+KEAB4EduP1Jr2tPuivnz4lTGX4kI3B5gGSfpX0HH90VUHzQTZMlyyaHUw0+mmtESyJqjNSt9aiNaIzZa05tt9F9cfpXR765e0bF3F/vCug3D1rOc+VjSuT76N49a5rWo9cluU/s24ijix827r/ACrNWw8Vsfn1WEewH/1qweId9ivZnb7/AHqKVlyCQCScZxXJxaV4i8wtJrClP7oz/hUMui60u1pNcdhnGBnqal4iVthqB2JZMfdH5Vx2pXDPPcRs7bQ5wueOtS2Wk6hb3iTzatJKi9UwcH8zWVqJd7q68uNydx7GnTquV7lKFitqOqQw2CxS5XCrk9hWBLP5beYozjnPrVnVLA3tqiNA0iuoRwQee1ZN1p99Gfs0PlKgACZfaqADGPXrWlOsti5UebUrLq6y3lykJHyxt5nQ8gZ59K57xBLGPE18nmJ5jTyMFzzjcc1fOmXa/aJI7ZVfaTK6TMMnHPb09aZqmhX73CubIPKHDs6yuxcdxkr34/Koc05XNFStGxgz3CQxhmcKMgfUmrdtZTsrT5IjHPJz09MVoRaDqFwwii0RncHcW3noPqOK3bHw1c2EUN9L+7aMEtBuPTIHzevXNKUr6LchwSV2ylo9pIWS6XmGWB9pIw2GQ9q9Q8LljpkzkZ3uj8e6LXBW7zS3KkkbCTnj2ruvBj/8So+nlwkZ/wBwf4VWHvzakT+E3uABkfrVLxA6jwrqe0jCwMx3elXow0jAbWqPU7b7ToN/E6kB4XGR24rvexzLcjs2M2n2kiZIeBGyD7VN5Mh9aqaIyDRrLMoJWBQcnrgYrSE0X/PRP++hUKS7lNMztXnnt4ENrCs1xsGImfbkZAJz9DmuflvAmpqe4XBNdBqnkSiKdHSR49yBlIOM4z/IVyOonFwJB1weSe/FeHiknXZ6VH+EjRuda+z2zTFXbaeQnJrMm8RSTxvGtleHj72zj86qPO3UNgAjgDiopZ2KDkAE9BWMaLTujSUn0NfQxp6awrTXRtnc7sN8i5HPVux9sV3iXEFwqPDPHIhyN0Z3D8xxXkgOXUtxzzn/ADzXdeDsNoSd9kjqR9cH+tenSldWZzVVfU+V/EL+d4m1SVQ2HvJmAI5++etSaUo2tJjvgU7XrKYeJdTjCMQt5MvT/bNWtNj8iIBsLg56dv15ruvocXUvGQW8JckZ6jPpUFrL5VuZHUbnJJ4xiqt1M0zqinjg0XbrHZZ5GFxwvH5j/PNSUPtYLrUIjMZHSI/LtiBy4ye/T2rWs7LUEBgts2UTHnbyx7dBwPxJPqawNAhe4IHnTqozxFIQf0/z1rqINOt3O1p33L1LXJzn8/6VdiTQttIgQK9w0kzDndK5YD8M4FSyfZkct5CvIeV2A4HH+fypsOlhRmO9ugCD92Yn/ParDWskShUuZCxb7zAEgc8cChrQcXqcN4xs7tJLe6m5jYFB7Hr/AJ+le0+C0MXw60odP9F3fqTWP4bsLG+mvdMvoBdw3MQLCbn7p7HqOp5HpXbraQ2mmC0toxFBFF5caD+EAcVlLWNi46TTMSX97JFIZHBjYkBWIDcEYI7j2qO4cshHam7ifb61HIQUPPbNeWlqeu3oUrWFYVlRNx3SFzlieT169PpRPDHMFEqghWDjJ6EdDT1PzNjuOlMkOQMY/GtUZs5bxijfaLB1BJO9eBmvO72Mq75BDbiCDXsUmV1CwfOFEhU8+orzLxZCYNe1CMdBOW4755/rXRRlrY5K8dLnMV2/w1l26tdxf3oN2Pow/wAa4k9TXT+AZvK8UxJ/z1jdOntn+ldNVXgzmpO00eu/wDnINchNbzJNKhdEKcRlmHzZJP8AWuvCMVwAfwFcze6c5nvnSKQzKQ6YB56Hj9a86J6VRXRnrLbx3OWuS4ERJVR1O3moRc2rS2ReGUlFADsQOM4BIpW0+4Mcf+iy5aNDnyz6EEdPpSXEF1NbwH7JMriPym3IRggg5/nWisYWHx3NxMNsMcaokLNjGSOo4/GopbiW5dY1kZUcoRg9Mg/1FTwWtzBegAIqMzoxdgPlLZz196pw2s8TDcYuCpAMq9j9fc0aD1KerQmXTbqaUMJohHIN31wa4uQbZmA7McV6Hra29wtzJJKiL9nYD94vLDOO9eez/wCtJ9cGuqg9DlrrU1/DEnleJIOuGLL+YNekBiOc4I/nXlOny+Rq1pLnhZEJ/MV6rzn+R9azxC1TNcM/daPeLWcXNnBOOksauPxGalrI8Lz/AGjwvpz56QhP++fl/pWtmkthvcTNHFFGapEle4XgEVVcblI9a0JVBjNUDxxW0HoYzRGp3Ip9RTs1GmQuPQkfrTs1ojMd2zSk03PFHSmAtKDTc0vamgH5qSI81BmpYvvUxEUkEnmNjGMnFFaHHvRSA+XtIJbxDbZzwsnf2rq5nJwCenTNcjo2f7egYcDZJ/KupkJJ570QFJk1uQX2sO1WwkPcE1Rt8mTAJJ9quBHx0PtRJ6jhsPCw9AozS5h6hB1qPyz/AJFAiYk5FTcqw8NGOdvApPOQqcKB7Yq/pL6fb3EjanZPdQlCFVJChDdjxWUkIZnaNZAC5IDNkBew6fWpUtbA0K84UE4A/SsLXLsp5eJCgb5gytySO1bFxFiFtwBGOh5rnNSQSW5juCf3THYe4GOKu4rFVNQkmVR85UYJYddte56fdbtIg6j9wn4cCvnfTpHN9Cm5VVR3zhscivdbCR20yA7sZiXcMewrhxmsUduDWrPL/Hkm/wAeqc8F4f5Cvo+M/KPpXzZ43I/4TUHIPzw/ToK+kVOFGPSqhpBGc/jZKTxTCc0jOcVGWIHBq0zNoc1RHNK0hA5pmc1ojNkkBxcRn/aFbO+sOM4lT/eFT3t7fQzhLaxMykZ37gBmsKybaNqVrGk780itz1rBm1DW8/JpI6d5B/jTYr/XmkUNpiovck4x+tc/s5XNeZWOiLECq8wd4twBIVhnH1qtbz37fLcWsaqf4w/T8O9LcsfJwD/Evf3FRLR2Lir7EobIqW2iDAZxzWXd3E9rbtNFC1yxIAiXjb6n1rGj8ReIYZZPJ8PSzJn5MyhQPzrKona5UtrI6aOyjaJxgcSt/wChGvOkxqYuZbjaGErRELwCAQQe9bcuveJ0idrXRUZnYl1kmH7s9x1Gaz/Dem38cFy+rWiwzNcF0RHBBBAz0J71WFhFuS6vY5q7qRSa2KNxKLK3kjW1JikGHLEnHGOwqzYaeuv38Mt7Dsh27RsZgSoB5611KWVtdSRxTwqUJx0PH6Vu22mWkEjPFAqsqhSOe3P9a9mMacIaL3u55zdWctZaGLpmixQRyrYQrEJIgAz5PfJyc89Kh12GIz2iPbK+9zncOuQtdXEVVZAMALKRgDtxVWcLusXwMk9duT2rOjFQfM9WXNNqyZ5VKiQaj5UaqqbgdoHY1teF1jezj8yZ1WO0jOEkK5wSvOPpWD4ngnTxhIYrgRqsi/LtHAIGQDW94SUfZZCSM/Z2XOPSVhWcl7+p1Q+DQ6OIWRbmWTHf96/+NI9hYXELxRtKQQ2FMr5PH1qsVbaHM5wf7oq7YbGOPPYnkFSOvFWmu6ItPsZ3h+ytn0qGUwwmRlw5ZsE4JrWNvFHHkRW7YHTIz/Ksrwp9m0zw84kfyoYZJCzSHp82ev41P4n1cWujpHZOHur8iK32HOd3Vh+f6ily01G7RtFTnKyJ5reIWu5IUjIcfdGOOfYVyWsrtZDxjBGPzrqRYvpXhqC1kmaWSKNd8jtnLZ5x7c8e1cnrLZiDbs/NwMcf55rzMXG1ZNKx2UX+7avcz3xGjHcPXr0FZr6nagcSk8+h5q/1V8gY44FU2hiXpFH+AFOHmOV+hs+DNR0+bXza3XkvFcRMAsyjaWGCOv416JcS6dpFoxEltZ24OTgrGv58V4xehPscu1QAFyOPeuRu1HnR4QdPau2lZxsclS/Nch1ho7jW9QnRt4e5kcNnO4Fyc5/Gse7zHJs9R398/wCFa7x4bcBzWRfnF+cHJ2j+X+fyrdIxZVbmUY7dhxW54bsodV8VadZzQeZGXLyRuMqyqpYg/UisRid3XI6Dmuz+F1qZ/Ed3eEfJb220cdGduP0VqBGHq3h8eGPEktmylrST95A7jIdCenuR0P0z0NbtpBayEDZCW4G3A4/D/Pau68XaAdf0UxQrm9gPmW3Yk91z/tDj0yB6V55pd3A8EYucI4wodhjPpnjr/hVRYjbitbdCNkC5UdxVpwQuFPGPun17fypkYIG4Nkeo57VI77VBZhj371Yh+kXX2TWraQc5cIR7Nx/n6V6ORxjH1ry+HdHcQvgrtkBLHgDB/wD1V6dwcHPWsTU5VvkLDPTIpjHIJ/WprkYupgOPnOPzqvn8u1ec1Znqp3VyDOG7kY6Uw4Kj06Ypx+9jA79aac98VSIZR1CMG1UnosinNee+MoTHrVwM5yiEZ+g5/Q16RfDNjLz0GcVw3juL/T4pNv8ArLcD8QTWlJ++YV1eJwjAhj2q/obSLrdn5T7HaVVDZxjPH9aouMNzUtlMbe+t5x/yzkVh+BzXc9UcMdGet/2Xqcn3rtMf75qlc6RqRLBWlkCpvyiMQ3tkd661GyCMLwOCO/1qvq99cadpUM1tjPmgNwPu4JP8q8WtVnT5VBXbdtTTMsRUw8KaopOU5KOraWzfS/Y4+HRdTnmWOSK6jBByxibAwB/PP6VCujagZdjWl7kPtJ8lsEc4IP5fnWwNf1fYJXuEEbuyLhF64OO3rUTa7rQjeT7dx5IlUeSnrjHSnfF9o/e/8jkvmv8ALT/8Cl/8iZY0TVXi+WyutxxwyEdueo9asf8ACOzqR52+MnPDdcdj0/OtCbXdVgukX7aWQhG/1acgnB7fSuj1AmS3tWY5LJk+/AqfbV4VIwqJWd9m+1+wqeJxkMTTpYiMbTuvdb0sm+qXY4hvC6zwhJpSCSchf0rzy5Qo4DdR8v5cV7WQD0P6V5H4ggFvq93H6TufwPI/nXpYeTd0ejiYJJNGbnGw56f416zbSie1hm7SRq35gGvJc/ux7GvTvD8hm0GzY4P7vb+RI/pV4haJkYZ6tHsvw/n83w15fH7md0/PDf1NdQea4T4bT5h1CDPRkf8AMEH+Qruqyjsaz3CkzRQasgCcjFUpBtYirtVrgYOa0gyJlIcSOPfNOpG4m+q/y/8A106tbmNgo70UUxBS96SjvTAd2qWH71Q1LCfmp3At596KX8KKBHy3oxB1yHHZG/lXTSHP51yNnMYtSRxuzg9K6X7SWZMkYIwQfWlBimi9ZuqSFieAOlXvtkQyA/6VlK+QcHjHWoBqFo9z9nFxGZQcbc9/SrcUyYto2/tcfZj9MUfbIgOC35Vmk85zUbQIQx+bPsxqeVD52WP7Zi3MAHOCSccEU+DUIVUliRu5Az+Fc7IjQzMfmR94PPIAz/8AXqKR9sxLMQAhO0fmf8+9TyovmZt3Os20jiBopMyDAbI6/wCRXL31zM188SrsIUqc9/So5Jw16JSR5YycbuR/hU2orDdYeGQeaFy+Sc49qmw7meitFIksY3BFzkDv0r3HTZEOk2w53GBPzwK8d8KOy+JLSNi3lu5RgehBBGMelet6euywt1HRY1A+mK4cXLZHoYOO7PNfGxx4yDcnEkWPyH+NfSUf+rX6V82+NS3/AAl7ADIBiP6CvpGI/u1+grSPwIwn8bHmmGnnpTDVIhkbUgIxStUJPNaxMmOc4XPcVsJKqIjMcgj8+Kwp2PlMBnODjH0qeCVGiiiuJGjkToCcbs//AK6irY1pGpJfAFQikZIGMY4NRtfHaSh56jPfmoPs0DEHz3yAAMN6UhMEGYUWWUt/COcVlaLNNUWTLmd1AGNoJ+tQXH+oOfUH9asqiIDgYzyfrUF0yC2k+YdPWuSo+Z6G8NBik7Dn1oiJYED1NcX4u1O/sL5o7a7kjQoDtU4xWJ4f1zUX1hYJr+d1MTEgscZAzmlLCyqRTuEqqjod00pD3Qz0lP8AIVNuyrVwmkXt1PcWzSXDNukcOZJCM/411Go6qulCPdBJcecdoEPJB/8Ar1FCi6WJimxVpqdB2Rv2NqbiRAwwCSckenpW8ibTKPcfyFcroOvtcXsdnLYT2zqrOolPJGK3Li/kiuEaOAsjL8xzwDkY/r+Veu6sW7XPM5Glexbjb95JH/EX3cD6VWbLG2UEg7WxkdDms6TVpBczRTrFFcCfyEYuSpLJuToPoKr3uq30Wi6dIlvF57ThHYZKryR9eooVSLdrhyu1zkfFMDDxFI3O7Ck8Vb8HN/rUK5ws45HpLn+tV9fnkn1KczFfNVivyjGcGqui6sdJEswhEn7+ZCpbHXaaxlPmkdMI8sTu5Y1KLsSPHqEH5VLZKouFO1QfXaK5U+PNinGmp8p7v6fhUa/ESTeSNNhGTxmQ5oUFe9ynN2tY29GigudOu7a5hWaFriVGQjIIyKr6V4Vh03WheLPLNbxIVtbeXnySeuD/AC+tYth4gmtYg0cMWLm7YkNn5cjPFX5fFN5Ax8uKH05BP9aj61RWkt0WoVUnyvRnTarJ5mmTgBThcnDAnqK4LUlXyvbcMZHHpWoPEd3dQPE8cAV1KttXBGaxtVciEtzu4xjtzXPiKsas04mtGEoQaZQBHPuoqNxxSxlQ7dPmXI5pG/THemkVcq3KlrWUf7J7VyFwv72MkdF7c812cgyjLnt61x1yCJVByODXTROasRlMjAGc9KwdTJGpMowSVXNdHjMec9PQVzmoA/2pIeeAPpwK6DAozkBeu3ivUPhNb7dG1G4K4aW4VRn0VTj9WNeZyDBGPlB7Y/wr174fR+T4bhU8F1LE+p3n+hFK4WOwU4Ze3Irz/wAaaMukeIvtcKK1tqWX2Hoso+8Pxzu6d29q70tuyazvH9ib3wq06jL2cizgEfw9G/Q5/CqiSzgbXaG/0ad7d88o44/Lp+VaIeVFLSKCccvFjcPwP41mwETxLvABIznGQTVlGnVwgRCFOOc4xWghrtHLJvQyvt53Zzj6jt9a9Qs7qO8s4p4mJRl/EH0Ned+cqj5Y08xBn6fjXT+Ebnz7a7BwrLIMpjGMjr+P9KzasaLVBqC7dQmGepz+lU846Gr+qjF+Tz8yg8VQPB/wrzp6SZ6lN3giFupz0z0Bp0s0lxIXlbc+MEhQPbtTSQTn2zSYOTkjikgZFON1vKvUlD/KuN8bRiSx0ufGQUZefoP8K7YgMpHrxXI+KE3+FrWTuk23Pf8AiFXDSSMqq91nmTj5qb6VJONsjfU1GK9E83qe+aZILjTbSfAzLCr/AJqKbrSqbC1WQAqZ8EH/AHWqn4Rn8/wtpznkiIJ/3ySP6VpataTXelhLdQ0qOHUE4yRmvBxjUJwcnZKX6MjOKkIRw85tJKotXovhkcXPHI0NwqFfIS4Bjx15/wD1iiBfMt7f/bgkiPPpzWpH4f1ZIzEFj2OFLZccMKtp4fuUZPlXakrOBkfdYdK3+t0P5196Nf7TwP8Az+h/4Ev8zl5oWWCzkDs37sscnrgg4ruroh7OyYfdMYP6CseHwrcJgSSM6jO0ZGACCCOtbVxEbeys4TyY49mT7ACuarWp1K1Pkknq9n5M4quKw9fHYZUqkZO8tmn9h9ih+Oea8z8ZweXrcxHR1R/02/0r07GTx+tcD49hK31vLjiSFl/75Of616VB2me1iF7hxQ+6RXovg+Tf4eQf885GX+v9a86Hce1dz4Fl3WF3F/ckDY+o/wDrV0117hy4d++esfDuXy9buIj0kgOPqGH+Jr0qvJfB03keKrTnAfch59VP9cV6ya5qb0OmotRKKKTNamQuaimGUzUlNIyKpbkszZuJIm9yP0pSaLv5Y84+6wP60mfWtkYsdSZpM0A1Qhc+9LmmZ96AaYiQGpYfv1ADUsJ+bigC5z6GilGcUUAfKEJJvFKnnH171tmZVUDPzdzWHG7LKGDDlcfqanRt8o3vhQfujv7VnF2ZUlobscsZWSR5BGqrt3E8dK5G0iMmqxQoVP7wDcO4Hf8ArWibuW81Mw2yh0SPBRh8vvxVO/SSGc3YxHLn7oGNpx2qnO7sSo2VzuenJqvLJJlo4yu7GRn0rn9N8QyXLiK6IDD+ID7w+nrV24vI45lkhJcr146itOZGfKQ6lqpEnkkDcBzj6VVt72MwJv6/Mm4+h6VnXbme6kmxuLf3uoFQ5bbnGcdCetZtmiRbtwryjcdx3DHvVm+NtFIqlgjgYBA/XNUtyJbLIuN+cHnpVSdi8nzMrHoCDSuFjV8NShfE9gBzmUD9a9ktBsgjX0XHFeKaE5i8QWGc4M6ZGe2RXtsQ6D3rz8Xuj08F8LPMPG5Y+K5MKSAYs+3Ar6Pi/wBUv0r518Z5Pie7PZRF/T+tfRUB/cp/uj+VbR+BHNP42PNMNPNNPFUiGRN0NQt1qZqhbg1qjJjWOFzWkqxzYEiKw7AjNZE5xCxHYVTh12aJVUqjEAfMe9c9eMm04nTQas7nSPbwxEGONVJ64qe2dWQgHkHBrmpNekl/hQe9NGu3CfMgQcY6VEoyaNdLHUTxeZbyJnG5SP0rgf7PETCQyudpDH0yDWv/AG/fMCC6c+iisy9lZ7eXkjKHI/CqopwTTIkr6i+IdDudc1AXNjJHJb+XsYgg/MCff3rC0/wrerrEgRv3ixuhyuByMcHoetb9pdywQmKJykefur0qaCR2uFG9sk9c1C54qyLcYy1Zzkfh+6tLmNGlXdbTbn9wcHitvVWJhtT6Txn9aqGaa5up12tvBAOM5OOKszx3E1rDEttMXDqS23jjBrkxKk2pM3pcqTSNhCqeNLVU6NAw/QVvbhIkojXG7bwPqR/SsM2lwfFOn3SpmJEKscgYO2tW0nKFARkOWGcjjDMf60qUrRMKquzB1S2uzc+ZHC7kXtpJ26BcE/hWvdwSyaZFF5ZDpcq+M9t7Gm6ncyqIZIolZGZd5LYxjpUr6nCxlDsBkADHPc/40nXswjRbs0cV4muY7TVbknkPKxBX61zsl4YIXAjLiS6PfGMxqa0vFIea7ItySobPNY1xGz223blhPGT7Zjx/SuqjJyjciceV2KV5qEiI/lqFOMjPPaneHrt9SeZJiA0UBkVgMZYEfpTbiyeUnEb5Axw3H8qb4SsZxfAk7EkiZc9Tj8fpW1S6ptozh8aTOuuYUh0rTnTPzTKzHPc0XnBJzUtzEJNAhXLDym4I9QapXcLkkm4lP4ivGg+Z6notWRNayEtx6U3UpmMLA8jHYdDVW0hO8ZeXA6fMal1B8xDOeAenf8a66cTGT0MlbhzcbfMbAzjn07U9rh2yM4NQn/j65IxjikPHHp613KKObmY4zNzlsfQ1z+opslVlHVOnvW536/SsfVgcRfQjjp1rWCsyJu6KyYEagE1z+oAHUpeuCByDiujXKquehHFc9qOBqD9BwOa1Zkioynd93n/630/z71694IUrosKkYxGR/wCOof55ryVYJJp1ijGZHbaoHdjwP5ivbtHsBYSy2oGRFO8fpkYYf+yisnNKSRoo3TZqDGOx4rTEaXVg0Mi5SRNrfiKzQPQD0rTtjm3T6YraJizxyKCTTr67sXB3QSbcN3Xsfyqyb2FVxICBnr6Vu+ONMii1a3vmXEdyPKkI7MOQa5h4PKIS4JI6K3QfjViNCF45EPlkHI69+at+Ebl4Nf8AIZsiVWib6j5gf0Nc/wCU9o3mxMGjPXGcdf8A9daejzrD4ks5xwkjrnHvxUTLidnrQ23ETHjKY/I//XrMbrnvWzra5WBvQkfyrHxxwTXnVtJs9OhrTRG305zTceuaew4680Yx6j61KZoMA6flmua16MHwnfJjPlT5A/4EP8a6fHI4FYmpQeZpOtw9SBv5+mf6VUXqRNaHkF2MSt1555qCrV4vzA8dKq9K9KOx5T3PXfh5P5vhVE6+VM6/yb+tdcxIYgEgHng1wXwtl3afqEH9yVH/ADBH/std+ynI78V5teK52melThGpSSmroql5Bn53x9TUZll/56P+BNTMuDjmmFSBzWPs4dkL6pQ/kX3IjMsoz+9f8zUblnwWYsQMHJqQr29O1IwyDxj8KahGLukVDD0oO8YpPySINnzc+vauQ8eW2bG1l5OyUpkD+8P/ALGuzwM5rB8aW/neHp3/AIonRx+eP5E1tSlaaCrG8GeRrw314rrPAcmL67iz96MNj1wf/r1yj/LM2OzV0HguTy/EiJ/z0Rl/TP8ASu+rrBnnUnaaPT9KmFtrFlNn7kyk/TIr2k14dtI6de1e128ontIZR0kRW/MZripM7aqJaTvRQa3MBKKDSU0Ip3iZjceoNVw25QcdeauzjK5rOi/1SjP3ePyraOxjJakgNLnFNzSE1ZI7NID6U3PrSg0xD81NCfmFV81LCfmoAvZopm73NFAHymg+bI64/TmoJpCkx2k89KlOFfsTgd/rVWfIk4HXtWC3Lexd8KkHUpz1zH6Z71d8VWyRQQygtvkc5BOR0qPwjGReXR/h2AZ/Gp/GT/urMdPmYn9KybftrGqS9lc5aIlZFIz1Ga1DNtIByB1HbNZKMWO0HI6VZAd8YOBXQYE0jbju7nuOKjPtyO1KCixkFjuA4JFREkEk+mKQ7D+RA2CcFulMCqF+V8kY7VKrgWcwPXcuAKv+G7KC/vZUuELII9wAJHORQ3ZAldkemxn+3LHGDidDkH3Fe4oPnx715/BpFlbSCSCDaykMDuJ5/OttNTvC/wDrjyeuBXHXg6jVjvw81TTTOV8cHb4muRngrFx69K+i7c/uI/8AdFfMniyaSXXg8j7ySgPvxX0zbHNvEf8AYH8q1StFI55O82yU008CnUw00SyNzUDmpn6VXc81qjNkM5/dOPaubDcAVv3kojhJxnkCud78VEzWkSkkc05WJXrTVUkA9jVtrUR2sc/mxNvONitl1+orO5uiAHA65/Gkny0Ev+6f5VLsGTSSAGGQDj5TSuMZbHr6cGrtuP8ASUI4wc1RtSNvXPAqyrFcMDyO9S2NEisItWum6DCn9TWg92iQrIrAqE5A+lYU5Juix5JQZNTA/wDEtb12GuLFfDc1pR1NyXVIozAHRmbJHBx1UisTSdTuWtZ4zIcJO4X1xTrs5+zEf31z+tUNJOPtqnPFw39K5FJ8jNHFJms0rP8AfYke5pUb5G47ioFOQcYqRD+7f8P51hI1iYWrkfamJ9Rj8qy5twMu4YO+Fj/48OKv61n7Vx02isq5/wCPORuvERyD/tn/ABr2cP8AwkefWXvsndbgzuLd1UnGcke9V/DxeO/hRzypdfwyaPnEpO0kbRnio9EkzqikDH71h+NbTadNoyjF86Z1b86PMOwd/wCtUrr/AFYPHTrVsHdpt2uekh4/Cs+d8wIeDlRXjwWp6MtiKBiHXmnX/NvuOPeoIz+945qbUWxZdMkeld1JanNLYyT/AMfaAA9AP0pCeR3oUj7bAc/eXv8AiKCMGu1HONx+ZrO1Rd1tHjk+Zj+daJHT/Cqt5GHiHAwJOR9R/wDqq47kszZWCTRqD0HSub1TIvmHsDn26Vt3Tk6iRwQvFY+sJ/pnvtBwM55z3rV7GaNv4fQ2k3jbTTdlRDHvl+cgAMillz+KivX4VX+0bpgysv2sEEHj5sf/ABdeKeFE83Uom6NHGxwe/GP/AGavUtMvtxmjOAVeNxgdPuj+lcdSm+dTub05K3KbkqbcEDnPOTVq0bMA55BOaZdIfJJGB3ptk3yMO4Y11UncxqqxT8WWH9oeH7hVXMkK+an4c/yrz6GQzW6vkFsYIbvXrBII5xg9jXlt7YnSdVubToI23Rtjqh5H14/UVsYlQzRoSrxFeeTjOKrPFKksb2/MIO9WB+6f8K0NyS5GV8zsOtZ01mY2LrlM9dvSlJXKTsej3UwvNItrgY+fDdfUVm7SBjB60uh3AufDCxdXgfa3PvUjKM+3pXmYnSZ6uF1gQlcjApNuKnMfWgJ26ZrFM3aIdpz0rNlh8y61CAjiW3/oRWvt+maoyLs1qL0kgK8+x/8Ar1VyXE8OvV27epIyORVEda2NbhMN5cR8fJMy4+hIrHr1YO8Tx5q0j0H4Vy41LUIP78Sv+Rx/7NXqRXKivnzStVn0mZ5reWaKRl27on2nHp0PoK038Zam/W7vSP8Ar6YfyArmrYeU53R10MTGEOVntToc5xz71Gy7eTgfpXh7+Jr9zlpZmP8AtXEh/wDZqrSaxPL99Eb/AHizfzNQsLLuW8ZHse4vPboAWniT/ecCq8mp6bGMPf2q9uZlH9a8Xk1KI2iIlsq3APzSEKQfw2/1qD+0bnsyD6RqP6VSwnmQ8Z5Hsza9o64zqdrnv+9B/lWTrut6RdaNeW0d/G8kkLBFUE5bHHavLv7Ru8YE7j6HFNN9dHrcS/8AfZq44ZJ3uRLFtq1iOcYlPB9ea0vDsoi8SWL9jKF/Pj+tZTMznLEknuTU9rMbe7t5weUdW/I10SV42OWLtJM9qZMg16n4cnFx4esmznEez/vnj+leaeWGGc5GOo9K77wbIW0Mxn/lnKwx9ef6mvNpP3rHqVo+7c6D3oNGaSuo5ApO9LTenNNCEcZWskfLLKvTDZ/Otf8ASsm4Gy9Yf3lB/LNawImhM0Z/WmZ596M89K1MR+aM0zPWjPFICQNzUsJGc1XzzUkR+amBd49B+VFR7veigD5aABYdOAPzqjccPyR1PatS1YeYd0Af5RjPOOKzbja10cYBz34rBblvYjiuZIA3luRkDPPpUdzeT3RHnSs+3oCxOKn+z7IGkZg3HbkYqG0jWWYhwcAdBV6bk67DIcA8jntirIOf4gMA1BGFW52yfdGcj86jZj360ASiXPAPA4p/lTOCyoSPXFQ4Hkk7evSrWnuyyBd3DKeOvSkMWOF/slwXQowxgmtfwWGPiW2gxuWc+W689M9qqSLNLE0fAIAAPTGK3PAMD23jLTmcrtEq5x9azru1Ns0o/Gj1GbwxFC+14pYjj7rccVV/4R1ZdQsrWA+W1w+zc2SAcV6Fqpj1C+82LJXaAMiud1kPpk2n3qDLRThgvrweK+epYmft+RSuj25ckqPwpSPG/iNo82h+L5LGV1kdPLbKjggqDX0RaH/RYv8AcX+VfPvxT1Y614ylvDEIhtjXZ944Ar3+zP8AocPX7i/yr3/so8WzTdywxpp/WkJppNNAxHPFVnPNTsarv3rRGTKl2m+L8QeawVAPX+Vbd7JsjHAOWArGQFTuxms6mhtR2JU6BR69alVG3hOMkjHzVEuSc9KkjO0/erNs3SZPdW01nOYpdpYAHKnI5GagcM6sADnBqTzGKlVU59hTvImZSVR847KajmRVmVrSJgigjHyitC2tI5nZZLhYQFyGYZz7VWtbO4aNCUl+6Odhp7GKJiGljUjghpAD+tK6Y0nYqzkCdef4Tz+NPX5rBwf7pqtdXdvHdKxuINqoQxEqnHT0P6Uz+2NIWxlLapbLJtYiMtkk9q58RFuOhpTkk9WXLps29s3PVP51S03JuL5f+m2fpwKv3WDYWjwMJEdEYNjHBrNjmmhurrythJcE5+lcsKd4tGsnqaYDDscVLHEzxynOMLnH4iuf1O91OwYB2UbhuAUKap23ia4tmle6YyRlCu1SoOePeq+qSaujP28U7Mv6rETdIT3ArMvv3FvPjHEanP0cf41Bc+KYZpwz20m3GMbwDn8qVrtNQs3kSNo1aGVcbg3IKmu6lTcYJM55zjKTsF9qptrwR8nzVCfnVfR/lvGYjDfaeAOw4z/OroEIKDGcrnkj2+tV7FkjvZyQFPmgglwOw9q25Vysz5nc6NHAt71SQPnBGeOoqgwMlsgUgnaOMjrVp7nT0SZGmtwrNkb5R3GPaqw1jTYLeNDdWqcYBEnP8681UpX0R2ucbblaOKQTcjHqauXkBktCpI5FU313T2OftyMvXIJIp8/iC2eJYI7gnOABtIzXTCEk9jGUo23M9k22/YsBgEDpyaRkduAjn6A1HcXrwwErI6hs9Cf896py6i4Td++cY4AySfoM11K5z3RoeRKR/qJOP9g1G8MgjcshwuD9Kins9dhiSVtCuykj7F+aP72Cefm44HU47eozHHa63vZn0gRQr/rHe8hyo9dobJ69BVqMiXKJzkjZv3GR16fjVHUxuuwScfu1z+tWUf8A0m4bPGcY9qrX5/0sEDjYo6dOK0lsZrcueFONYEe4J8j5Lf1/Ku6s5VjvpVEq5aIYxnBxuP8AT9K4LRX2awjK2Mq2Pyrp7KYjWYATnzEZefUg/wCNRa6K5rM9HuNRyvkhBjAGe/QUtg4JcjqCKyd4YRt6opzn2FXtOkGZBjHH4VrGKWxnKbluawbvXHeN7YJLZ3wwMgwucfiv82/Ous39cHvWL4qh+0aDPgcxkSAj2qyDzyeFZCNpKP6jvUP2q+sl/fqLiDuR98D+tXIgr4BHzdRTpomMe3y2YHkYGaBmv4YvInkmijbMc8e5cdiMV0fkkjP4815lp8s2n6iJUJUxvlk5wQa6C5+KmmWkjxNp920i/eBKKM/ma87FU5zknFHpYSrCEWpM67ye2KzdUu2spECpncM5NcrJ8Yolz5Wkf993GP5Kay7/AOKct26n+zLT5QQNzs3/ALKKwjhq3Y6JYqj3OpOtzbsKi01L+a41C2ZY03gMBucIOfUniuIl+I9y4H/EusuOmUJx+tQn4i32QVsLAc/88T/jWn1aq+hk8VS7md4mO/V744AJmYkA5wSa588Ve1PVZNUvZbqaNEaTGVjG1RgY4H4VRY5NehTi4xSZ5tSScm0NopaUKT0BrQgbRVhLG7lx5dtM+f7qE1aTw9rMozHpV6w9RA3+FLmQ+V9jNoraXwlr7f8AMKuF/wB9dv8AOrCeBvED9bSJP9+5iX+bVLqQXUapzfQ52iurT4fawxAeWxjz/euVP8s1Ovw7vDy+qaaB/stI3/slL21PuV7Gp2ONp3YfWutn8BTxqPK1G3lbPI8twMfXFV7jwhdIkflzCRsYYCMjHpyetL20H1D2M10OltviFYR2MEb2k7SLGqsd6jJAGe9ejfCzxTB4gXU4YoWiMBjbDNnOdw/oK8QXwbfsxBkiUdt2cn9K9D+FVmfC2uXT3t5B5FzBsON3DBgR2+tYclJO8dzoU6r0lse5Un1qiNYsD/y8Af8AAT/hS/2rY4/4+FH4H/CmIuUh/SqZ1ax5H2lP1o/tWxP/AC8pVIRbrM1HCXMDf3gy/wAjU39r2B4+1R5rN1XWNPMMTC8hysgHLY9qqL1IktB+6gtxWadZ03tfW5+kgNMOu6eOPtGe/wAqMf5Ct7mNjU3cUbjWO3iGyAyPPb6Qt/hUbeJIMEra3knssY/qaLoLG4G75qSJua5s+I8YK6fcgf7ZVcfrTR4mkDALaIp/25wD+gNF0Kx128+lFcl/wlj94Igf+up/+Joouh2PFLO4SGU+YwH3ePwqleCEyOwiB6n7x5NWYLVbncXLgjHTntUhs03ncScDmskinsZqNnT2OCMbgMCqlu7qzFBz610IsojbhFU/MScDIqnZWiLcMNpDAd/rTuKxkqGkm6HcaMMuQwwehHvWyltH/aJGwH5s89+/rUMscaysGRd27HI70uYdjLDPt5PapbTf9ojweAavgBQAqJ/3yKt29w0JP3CD17HvxS5h8pGtypONrZJ7LW34c1CDTtdsr6YStEj7mCpk4qvDLBI3zMQT74zXY+GtVnhvbe2to7VNzAeYbdS2PqeawxE3yNWOihC8lqe26HbRalpkF9G5WOZA6qy4IB9adrfhZdWtY4hOItj7i2zOePrT9HvdtqpcSySY5YJwfy4rSbUG2/Laytx2wP5mvMwkcLyptajqzqqo9T5T+I1r9i8RuobcDtOQPrX0FZsPscP/AFzX+Qr5/wDihKG8WPtTYNqYBIyMZr3y0YfY4Dn/AJZL/IV6dO7pxbJqWVRlvd70wtTN3vSFgK0RkwY9agc05nqFnznmtUZsoak2IRjOdy/zrGOuxoCvmWgPTHlJnP41ragd0IwOjA/rXn76Pfvcysstuqs5I3Sds+gFZVacZbmlGbjsdR/wkiwg5uolwOqxoP6VTfxq/wBn+0RvqEsXy4eO1cA5xtwdgHORj61kr4cuZRiS7tV/4E3/AMTV2bStVn0iTTpddt445FCl4bc7uDnIO4e/5n2xlGjS6mzq1Og5/F+ou2F0zXW9zEUH6kVC3iTX5yRBotye372+jQkn2LVF/wAIzckkv4rvQTyfKt0T27GpF8J20g/0jxFrUuO/2ggfTrWns6K6Ee0qss2TeJ7vTvtMNvpyKrGIh7syfMDjqox19+x9s8Y001xNLK77TK28qrKQCeuDzXbQ+GNJtrXyE1HUmjLFivnLgk8kn5eScChPDOgRLgJcEdAGlx+WAKalTXwoTU5bs5KNmEM4DEnKHJx7+grHuFLXDsWAbGDk/wD1q9T0rS/CyajMt9Gy23kgjdI5ywPsa1mtfhsjl2slkbqcrKf5mtIpNXMpXTsS+B9Nttb8F2klzMEuYR5b7G3AAcr+hFbb+DtMZyVvJkZvvFWHP5ismw8R+DNDhaLTLGSCNzllihABPryalk+IOiKMpYTt9Qo/rXJLC+9dM2VaVhLr4a6fezB59auyBwoAQY/HFVX+E2j7QF1S9b33xj/2Snt8SbEH91pbkY7yAf0qFviXn/V6ao/3pM/0q1QaVuYhybd7E1x8JtH8hTHeXXmAcksvX6bap6l4Ih0+80/TNPeV4rmKdWlkG4IxUYzgcdP0pV+I91JKENtbxgn7xDNj9RVWbxzf3F220QBrXdIhCdflIPU+9P2VvtMFKXYE+El0sasNXgzjoYm4/WoB8MZ4Y5ojbWdzKyMBOc/ePfkZ/wD1+wp1x8QNVOnpLDfR+cW2mMQr+fSs1/HPiKQ/8f7KP9mNRn9K1UbdSdR4+Gut+WkY03Sht5DM4wc9eNhqyvw68QHZ+60SPaAMYbjH0UVlv4q8QyZzqN3+Bx/Kq7a5rk33r68PHTzWqlyiszq7b4fa5HaGCS/08ZcuSsbnGTnHb9c9vQUyL4XtEVabV0G3HSH/AOyrjpdQ1Bj+9upT/vSn+tRDUo0b9/eRLgdGnHX8TQ2uwKPmeijwJp8hEcuq5AzwqL/XNI3gHRUQ79TkB7OiIpHvn1ri7O8AjEsV5asGGRsuULEf7oOf0q0msGQY3E4/CuSpTqSleLsjog4pWZ1eo+H9LvbdILrxDqjxxSLKgiljQhxnByEyOvrWNP4X8Oojt9s1i4kUEr5t85Geo44zzWf/AGpEq/NvDfT/AOvUJ1ePdjcp9i2P6U406nWTHeHY88iAW4lG4ls8+5qveuHvWjb7pQHP4VPdYt9WmXb1Y9T1qhqgka4VkPzeUG9Mda65LQ5ludB4TaKLXYvNiSRwr7Q6hgflJ6Gu0uLjEsbLDAgVxwiBf6V5not85v4HTaJQ+078kfkK7CW6u5F2rJbA9ceWRnt1/wA9KjlL5jqra+xDGrHG1AOvBx+FWYNUjSQhiSAOa4tNRvowN9uJO37tgcfnipoNTdWbz7e5AxwBEW/lnmi8kJKLO2bW7cZwG9uKr3WsxywSxeUzBlI69q5b+0mYHZYXzZ6YiA/mRTWv7rHy6VeHnqfLGf8Ax6jnkPkiZkheM4jB3LwMDOafHq0kaeVcRFHPAbkijzHE58yAws3IjYgn9KnVfMi2PGMdAD1rZaoye5iKXTUGMj72dc5FZmo+HbjU70NaGIHb8+87QK29T+zWrRpHneORznA/pUumO32pACo3+vT6VnNtbGkUnozmk8B6k3W4tQfTeT/SrCfD67Jw97CDnHyqTXoy2LkZMijPPAzUn2Fc8sfwU1z+2mdHsYHnifD1cAvqLf8AAYf/AK9Tr4AtNozdXDH2VRXfCzUDJZuuegpBbAZ4PP6Uvaz7j9lDscCPAlopzvuGHocf0qaPwpaJtzaIdvqTz9fWu6FsmfuHI96kFtGM/L37jNHtJ9w9nA5aHTLRcf6Baj6W6E/+g1rW8UCDAGz2RQv8hWr9nQHp29OtOWBfas5XZcUkU1W3/vzn/toakMFqw+ZJW+pJq15aZyGxRhR3HXvUOLNVJFUWlmOPs/8A46BTxbQD7sAA+lTbowPvqPXGOaaXh6+aB9BS9mw50NMaKNqoAOgwM0woDjavt0qQzxD+MkfjxUbXUY4w3FUqbJdRDGg3ZIBzUbWpOMD9aebwIchGP/AqadQbGAq/nVeyZPtERmwY9GA9qF0pt4YPHwemDSm/k69M9OKa95KR94/lVqkyHURr/wBo6qDnzYSB/snig6xqoAxLFn121im6l6GVsexphmYj7+fatFFmfMjb/tzVVI/498f7S0p8S3y43rB374/pWC0mSfWo2OD06VaiS5G/J4quto2pFnuev9KoXPiG8ngkjxHuI4ygK5z6YrMO7kYx+NN6ZHUZqlEhsedX1LC7RbofQRCo21XVSeJ1Un0jFNPvnn3phwO5I7VZBJ9u1E9b2T/gIAqN5rp8br26wOcLJjP5UnPA7UmCGPXPsOtADS0mcma4Y9PmlY/1ojijJJMeaCTjjOaVGYHOT/jQIsFEBx5afnRSbv8AaT8c0UDOQsI3G8CF2yeMKT2HNSPb3TMStvLjrxGf8KqNdTZCx3Dhf7iOccfQ01Y7qckBJ3+iMf5VncqxdW3u1OTFIPquP51We0lRjIojU5JwZFGani0PU53HlaZeSHB+7Ax/pV1PCWvv9zRb75cZzAR/Olzx7hyt7IyntWZ94niV+M5lX+hpZLdJVG66gDdyCTn8hWzH4F8UTDC6JdADpuAXH5kVftvhx4rePYdMaMHk7p0AP4ZpOpDuNQl2OTeyiUYa9jPc4Df4VGyQJx57nscJ713R+E3iaYgk2ygY4eb8+gNWY/gzrL48y/s4x7Fj/Sl7amuo/Zz7HnYeNVGJJeOnyj/Grlvr09jcJLC8odTlTtGM+vOc16GnwXnGDLq0QP8Asxk/zqC7+EVvZQmS4109eES35Ptyx/lUuvSejZcadVao5n/haHiRF2pfThO+3YMf+OVTu/iF4huhtkv7lgCcfvdv/oIFdPa+BdFgGbhbm6PcO+1T+CirNt4L0GDLiwLAn7sjlvyrNfV47RL5Kz3Z5sv27xNqltbRoJbueRY4yzn5ieACWPTmvp22s7xLaGM27bkRVJ3DqB9a4LRdD0fT9as74WUFuIZlkMpJUIAevXGPrXojeMfDq9dZsv8AgMoP8q6abjUXYwqKUHqJ9ku8/wCp5/3h/jQbC8I+4n4tUbeNvD38OpI+emyN2/kKYfGujn7klzJ/u2sn9VrTlgupleT6Ep0u8b/nkP8AgR/wpf7DuG6zRj8CarHxnZEfJZ6g30gx/MimnxpFjKaVfnPQt5Yz/wCPU7wXUOWb6E8vht5l2vdj14j/APr1w2p6ZPp2rT2yb5ghBD4AzkZ/rXXN4ymIHl6RIc/37hV/lmvPfEuk3+u63caqIo7dZ9qtEJS2CoABzgZzionKD6lwjNdDQEVxx/q1PoZVH9akW0mcZa4t19jMD/KueXw/qCAA3LccDEhqZfDl28RdpwxXqN5PHrWfudzT3+xt/YiD815bgezMf5CnfZrZeW1OJf8AgDf4Vgf8I6zfekAP05qtdeH5gxMLyySY5VTgH6e9O8BWkdQF0tQQ2rqSeyp/iaa1xo0YO6+mYj+6i/8AxVcMbTB6PgHByTkU9bIFiWTHrk9PSn7oanRXWqaUsm6K5lAIwd8fX2wBWPNrtuCQkdw4HGcIM/mwqNLaIH/Vj3zxSm1iONqjHcdxRdC5WQ/2yrfdtZyw/vPGP5MaY2suRxaR/wDA7gjH5KatrboFJCpnsc4I+tOWOMZDLFjvgYp3QcrM86teHlLW1Ht5rt/7IKaNU1F8bFtRjniJ2P6sK1BFF1G0ge1OMYyAdrZPUUXQcrMtr7U3GRNGp9Vte34uaak+rr8y3UgJBBxFGOD1HINa2AB8r5GcdaCIvmG1gAOuMii4cpk41FwQbq49Dgxr+oSmm2vP+e10eepuGH8iK2CFIADA/oKYSiHrtx07UXFyoymtJ24Mku3/AGp5Dz/31UY0kfedY2A/v5bP5k1smWPblip46kYpoubfd6D/AGhxRdhZGUNGU/8ALO269oVz/Knrpwi+6EUd9saj+QrS+0RnkNuGfT/P9KjNwAQR5gPTmndhZFQWbIMefLz2D9aabHpukmOeeXNWDdN97G79DTftHUlXUj0OcUai0K7adGSC2SfqetAsYxkBMdeMdqnMzEEgBlxjFRmR1XC7SccAinZhdGRrK7J1kUDkDkcVWuZT/o7gnBUqT+P/ANetHUFNwseBkgkEgY7VD/Y8l1DiNgpVgaq2hN9TP0zFt4gtpCgZN4bDcZAr0CPV7IcvCUz0+QEfoc1wFmrNdwuCVeMkEg4PQ1pl5Sx2FjjnG7nFRy3KUrHeQanYyAbbmJF9GGz/ANCFXk8ubmORHGOi4avMWZimXIAbpuGOntj/AD/ICE9BktnBXr2/+tUul5lKr5HqAg6cDPQ8Y/lSmId3Iyex9/8A9debxy6moJinuOByBLwPwzVpNW1aAhWeIjPWRcZ/Ie/+eKXsx+1R02r2iuUdGG8epBNZq7hgZJboQB0qrHrckxCyRI2T99ZVGOeu3NXVcOSVxhvStoKysZSd3cyri2c7pXXJLZzjJwKSFzE6OFyVI/OtB45ZkbkeXnqTWYw+cDqAcc+/BqWi0zurO+hktlypJUe351ObxOcKT9cVy2kXJ8vy88dh39q1fM9QPrjisvZo19ozQN6OnlD86Z9qZuQqiqQBOeGx6ilBOOG/SjkQc7LRupMngD3phuJc8vj/AIDUALAcgHNKd3TA9iDT5ELnY8zOcgseaaZHODmT0puDnHIHuKXHsSPrRyoOZjd7EHO4n3NNyWGe31qTap4x+RpCnt/WjlDmITk9X59jQCwAxn8RUhUg9fz70woeuP1p2Fcbk5zuP5Um5jnmnj5ednPSkJVuoOfpRYVxhOR05pDIfTPb8KlAT1AzSBVHT09KdguM85gBuXikMmOuc1ISMgcZ9aYcZwMZ7nFFhXGF/rgUeYNuTTsDPGPwFIR0INOwriFlI6D8qQ7CP/rUjAnnIx160xo2zkYBp2FcdhTkf1ppCsDj8Mmja4JJYfUU3DY5IoANoHcUmMd+nanfPyMLkU08DlR7jFADCTnrmm9OM08uo4ZeKQMpxwx46gUXFYjJ69Tx/dpynnoKkCg/d3Y9SOtSJASMAYHvRcfKM4PJUZ70VZEAx0T86KVw5T3JdNtEUBLeFQOgEYqVbdFHAH5UUV4Z13YGMZ9KDCO5NFFUkO7GFFXtmkIA5xRRTsUmNYAdhSZ+XOBRRSLRXuJTFC0gAOB0rkrvzLyXzJpCfQAYAooqoFlUWiFwCTzSNboTk546DPSiitBCC3jkyrDcDwd3Oac1tBFIyRxIu04+6O1FFXFktCqvHGBx6U51ywX0Aooqrk2GkYOOvP8A9f8ApTgoaFT6MQKKKaEMCDn6Z6VLHGGLKTxtJ/IZ/pRRTAjaFQc0RoqyhlGCPXnNFFMRNNZxo6leFYBgMdM9qi+xIxYE49wORRRQBha1psP2M34yJFba4/v4bGfY/wCea54hQg+UfMKKK2hsYz3ImlVUX93kM2OT0756e1RC7ZYwQOr7eetFFaGdx4uXZlGBnA5x61FJcyJsORhvQY/z0ooqrCbGNdS7xhvmB6kCnLJL5nlvIzAjd6UUU7E3FbKMGLFvmA/OmuWDr87fN056CiinYQwfNGp6ZFKshAVG5LHGRxRRQA4KC23GDjqOOlCx4/iPWiikJimEbhgke4pme4J69zRRTAQFA20rndxnNRyzeWxIBJz1JoooAhFwJCD5YB9c/wCfWr8UAmXOQv0UUUUxD57AwRea0gcGQDG3HX8aVYdnAY8EYNFFNCZzX2cR3yEHo4B46812kejWcj7ArA/UY6Z7AUUVmiwfS7WCQKFckdxIwz+GaVbG1KkmFW9n+b+dFFUAgtbJSB9jiyTjhR/hVhrSOCMOqR8+iYoopDEUlgOelZ7Emc44ycdKKKqJMjO1HV5ba4+xxoGJBbfIxPc9hj0rHOrlnkWWBWKcZRtufwOaKKmW5Udi5o2s+deeSsGzgHO/P9K7TAIGc9AeDRRSY0L5YI3AkcZ/SjBHRj+NFFACgH5ufpQqbjwSOfrRRQAGIrg7v0qIsQ3JzzjmiigBC5BAODTkbcVXue9FFAEk0ZT5t+efT6/4VACSTz6DpRRTADksRmggfrRRQAGEgE7vwxUe35j0/KiikA0LuBOcYpdnA5oooAaQVzg8CnICRnd9MiiigCUQ56sD+FL9nXuT+FFFJjFW1RwSCwx6nNS/Y0Q4znAoopNjSEFupb+HJOM7aayBRkYHpRRSuOwwDgMDjOacke9sEj06UUUAhViVsjA9elOWFByQfwoopDHGJMn5aKKKAP/Z" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAIFArIDASIAAhEBAxEB/8QAHgAAAAYDAQEAAAAAAAAAAAAAAwQFBgcIAAECCQr/xABvEAABAwEGAgYGBQUKBwgOBQ0BAgMRBAAFBhIhMQdBCBMiUWFxFDKBkaHwI7HB0eEJFUJS8RYXGCQzYnKTwtJDgpKVorLiJSY0U2Nzg6MnNTY3R1RVVldlZ4WzwzhERVh1hNPUGShGSHRmdpS08v/EABwBAAEFAQEBAAAAAAAAAAAAAAMAAQIEBQYHCP/EAFYRAAEDAgQDBAUGBwsKBAcAAAEAAgMEEQUSITETQVEGFGGRIkJxodEVMlKBkrEHI1OiwdLwFhczNENFVJSjpNMkNVViZHJ0srPhRGOT8SUmc4KEwuL/2gAMAwEAAhEDEQA/ALv1/D+mrU9RULp3EdYlwdhxB7KgobOd4HK1esT/AJNvgdieprax1V/0r1e6t91VLe+XtqXnJAdacjUbbWts+Ppf8a3aE6QNotWgwehpHE08QbfoLI7sRqZ2gSvJt1VErz/JUcPH2SxdOO8W0gIyguvUtSNeUdW3Zp31+SkvRdMumubjBVBBiPSLgQToe9NVPw5m3o8kER77dgK0Ak2OaWNDEzl5Y3t+S74v0wQbtx5h6pCOVRTVrObQ/qNLHcN9rNev/J49Ia73HHGG8PV+YR/F7zLR05/xhDfKLevIHMmeeot2LDNG081LjnmF4w1HQp6S9I28tzASXlkmEsX1dzpzTuQmoJ2J5c7JFb0Z+kJdi2zU8I8YLSlUrWxdLz6djPaaCxyGs++3tmunYdP0rDax4oBsXXclzuHMu7KUnv6oA2h3O2xUuOF4bV2AOIFwPK/PeFcRUI0BNbdbzOU6/rpHht+FkxJqEZkOVHbAUJ1JkHeD79be7qbiuxAhqnU1H/FuKR9RFiN5YHwze6Si87tbq0EapqB1o9y5Fomkd1SE4Xh4PSoQFKKeZMazE7DysaSqpDjakFSg2mYjmPgQJ8fhb2OvLo6cG72GWr4dYaXzzKuSjUfeWps2q3occB61ZWcBXOkqBHYYcaHuZcQLANG9GZO0brx4oXnQlTSn1D6V1JzAkFWY+Hfz93fZTp3qhTNIlsKgMDtRO4EGO6PrFvUiv/J+cBX0q9BwoilUpZVmYvOuRqTr67rgHus2a78nBwvdShFHXX9ShsBKA1fDSkiBH6dITt/OsCOkkjdchWH1Eb22BXm51qw48ohakghBAHs9m/1WAceeyhLis0g5VZSDG2o93LmLegV5/k0LrKlKuvH18sgnRLlJT1MDzDjU8uVmjev5NLGTaEounHzD2QATVXSpsEAzB6t1zx5Gx+G4clWzAqmClqK0EuIIBgg6HY6x59/Mey3SFOqUtIeVAgcyCJBM8+X1+NrS3j+Tv400iyqnvnDNQka5QuubPeYmmge82bF49BjpCXahz0XDN0VmeSCzfNM2nwgPKQeQ3FmLHdE4I3UBmoqE0/8AKqBGva1yk8td+XzuMhyrQ6lPWqI19YbCPxtKF59FXj/duUO8K72fCf0qMtVY0E7MKXz8LN2s4O8XLoUtd6cMcYUqEJjO7clUhJ33JT88rQLSph101ELW+pQITAJBOU7d/hoPh7uw64AgEjK4JASZI07gd5Fhn7rvShQlu8aOopXFKKSl5ktntGNJHzHhblDTinE5wF5dwdddB3fOnlYZBRBuhUuOElICoGqgADEeR38LNS9aqpZxfRBBKVFCQTtAhWhPsGm+g3s60JWVFxKAY7IIjUe/x9psz8QEt35RupSFBQkKCtjnMR7ht3+Nq8ouEWM6p30Nap2mH0gXDhSnkdY7vwsIagpAVmKhM6HfQxv7e742K3Wos0YqQCVZ4TqBGke/577CrWUpQwEKPWdwBPjt4/G0IRZtkWQ66raqkeqFJAGpykg84j2ePK2OOrSnNvJAzAA677+zx87aSpClq7IOTWCDGvfy/A2DUpktlbqAc+wIjWdBr4a99jEaISEW4+XCorOgG4Anfb3/ADzD61zNLjvVmSqPA9w8tfd4W26Ek9XK5UTmIJA5nSd9if226DOZJJnKOyBm0yzudxM/VYZ3UguC5/J51KQokFI5jxG5iSfjbnM6peZYKYGyiIA7o7/OwvVLzBIKlK3JG233jQW7bYdSolJBGUCIAG0zqe6fC0C4bKSxClo3ExqdiByOu45WCfUp6nWlBIlJgFRjWIn2gGxkIcCk5kpBUYywrUE6ifLTe3bjDhbWHAVhSCqAT7CJGvP42C42UwLhMjDNS8wutQcxKi2VACJHaJBA89Dz08rPH0WmdCXwyEBUrII3TzPjoT7/ADsz7rdW1X1zCWz3wFxBB75nYnx9uhf125lXcwchyuIEyDqe/u+fbYDvRddFaLtsk782MuZAplQJ5SdTpyme73W5XdbeuV50TCdFCTPnzn5izhS0lCQohCZAI1gAR3ew/XpbpbLeUdW1mJEzJAnxj3WbiJsibZuf/AoqFHQZgRt9/wButgDcThJKy2oEdnOnTbWPZv5eFnUaZpRKSCO6NyO/Ty+d7Y3Slaxl1MTM7b6ju+8m0TLlUhHdNdGGFPALUxTrUIB1KZHedNNvk2x7DS8sIowYUTCV+G594HlZ7oogIypGkjug8xrbhbBUFJgAGJ5xEz8fssIzIgiCjty4Kg6IpHVfzgRKhI0k/s+wo/h59KgpxuqaAVrmQRBiANfmJtJq6cpBKk5CgyMwGh+/Q/HwsC/ROdQpwIUYSYGun1Tz+NoiUpCEFRKgPNPQBmglJEEjQzMeZ28Bz2UWqAvsoUsNpVyVO+u3nsffbV6M9TUrlJIKzBB5kgz56fDzs9MJs09VcjaXWG1QtbagU5jvIHxgWdz9ima3kmPVYZbbYW9U3fSuBMqhLaSTHgfb7xtNo6xjcRapVVVNdSWQlQgtN5dNe4RBkePstPt/0tEUKpGmEhIMrKQQJn1dPb9vhHGK6IC56lLaeyhYUTv2pH3+flbSoZXNcDdU6lgsbKEC082ZShYjuBFtirrESA+6JOYjMd+/6rORynJPrr+kJ1UOckg6fOx2sV9GC0FA7WhIlMiBEnX2e63RiUHdZBZbmkU1tZoovKOWAJ1i2hVuz2gDz1FlNVIjIHChJkHQJG+un4eVuPREqRKmMo3J2G+0fOvstIPao2PVJpqVkagSeduvSjkyESAIGvz4++xlVI3llSVJgTprpt8/IsA9ToQ1mRm31m0gWlN6Sxxx2EuLzdoQCTytx6U72u1orcd9h3vpKSmCieylSB5hRP8AaFioZUQTI0E87OLFI3XYqSk5gkTzt36asiMoEzsYsAW1jl421lUTASbPYJsxRlqpbSoFaOzI03sZVeTZT1baVZdsxMGJ8LJxbcA1QrbutzB7rNlBSzFHfSW1ICOY7xpMncW0Hk6gkTI1mLE9YtmvIWWUJXQ1Qc0Kka91gpmLatlpDRMShaaodpqlqpaWUraWlaSNwQZBt9JfDfFJxdw7wviptYUm+Lmoq+Rz61hK/tt8143EW98+ghiZWLOiLw1vJbnWOU91Ku9Z7vRnnGAP8lsW5ntLHeOOToSPP/2Wjh1i4gqflVLkb2CNQ7uDbk720rW3JgrYDW9FnXuE+tbOvc1M+63Mc7agmxgp5QumH3EvuIKtCEq+sfZYx1zs+sbE0kpqEHbMhQ9xFhzM26GjOaIKvKAChuudOpUbaD7v61uJkRbDJ52tAIRK669wa5rZ17vJW1g+ds252kBdNuhfSHTpmNk+8XnA/SvFZ0WUHXkRY57vZYleqVmlK2yJQpK58jZOGiduhXXXvR69uTUPD9OItoGUyBvbhQ9tmI0T7LsVLpET52HTUulEzyHKxSNDI1sKwJQB7LV3NsiN1QxqXogq+NkHFd4XrR4fvS8G73NEaWldqErRT5ygISVagzm25ATrZXeX1aZCCpSjCUjmfsH3WQcS4ZbxFclfdd7VT7iK5lbQCDlQwSNClIjMQQD2p1HLawXMMnohTJDRdV9rek3cCLpu++7wuR6uq64MvpYWwwldKHJBdzh1QSQNS2oIWNlEHSzd4mYnwReCa1FRiO/F36txNI7RpC3DRnMex1CYbbUr1ih1aFBJkyd2riHh9c+JcMX7hNq9HafEGCn3KP0p3q1KUylIK5IUAQsnNJ1BGXXVNq1s4qVS0lBh56+mHbzuityshDaFNMJQ0tWRRWkhaUr1BKikKiJmLZ1Zh4lgc0m2h81nurZGOAIuCp4GJ8BpAS5T4nzgQqLwpkCecJTSEDyBIHed7ZZuM9IOpbZQ3VUl7reSkBxSUpIKo1IKVgET3ADwG1st5wcNrb/N/O/7LR4sPVemj4+l07zbtsBQi3D5PXRE99u0HTf9tvpB265FgsEMkd8e23cACxCvvm6blbQ7e96UlChwkIVUvpbCiNwCoibE044wV/533J/nBr+9YBexpsSrUdJUStzRsJHUAlLwEH8bbAjbblZDGOMFEj/fhcg8Pzg1p/pW6TjnBW37r7k/zgz/AHrNxY/pDzU+4VX5J32T8EuACdvbbobbGyEMcYK54wuT/ODX962xjnBX/nhcn+cGf71lxWfSHml3Cq/JO+yfgl2NrYJjc+VkP93WCv8AzvuQf+8Gv71t/u5wST/3X3J/nBr+9ZcSP6Q804oKr8k77J+CXQTO9tgk8xppZBGOcFDU4wuTv/7YM/3rdfu6wTr/AL8Lj8xeDX960eLH9Ieal3Cq/JO+yfgl2YB0tvwFkH93OCtf9+Nyf5wZ/vW3+7rBIn/fjcn+cGv71m4sf0h5pdwqvyTvsn4JdBGxtvlrzshfu6wTt+7G5P8AOLP962v3dYKO+Mbjj/8AEGv71lxY/pDzS7hV/knfZPwS8YB00ts9ofCyCcdYJ/8APC4/84M/3rb/AHdYJ54xuT/ODX96zcWP6QS7hV/knfZPwSwumpXNHKdpf9JINgjdd2b/AJvpxA3S2AfeLJn7usER/wB2FyDlP5xZ/vWz93eCQP8AuxuPb/ygz/esuJH1CXcav8m77J+CPu3LdryOqdpyUfqlao37ps3rx4S8N73UV3ngq5Ksq3VUXey8T7VoNlH93WCYj92Vyf5wZ/vWz93OCh/++NyEf/iDP960S6HqE4o6wfybvI/BMm8ujFwRvKQ7w2w0iZP0dzUrftlDaTzPPnZm310EejffbqKiq4e0aXG4CVs1VYzHdoh8CdTytNH7usEg/wDdhcmv/rBn+9ZSob0u69acVd211PWMZsodp3UuIkbjMCRNh8OF5sLFM+OqgGaRpA8QQqy1H5PPgd2xdzV8UAOoTT3mvKDG/wBKlz5izavX8m7gl5ZXdGNL/pCEkJDr7L4B01gNNzttItcgERM2yeXcLP3aPohcd/Mqhlf+TWvFLTiKLia6tS9e1cyAnykVM8hys2ry/Jy8TEls0GLrkeSzqn0hipbnSJ7CHOU/C3oyDIibZIJn7bQNHGVIVLwvL69egVx0oEPrpm8PV6yNAxXONlXcfpm2x8RZqXh0P+O91IT1mAWHWWyVLLF6UTugndIdnXXlyt62zBtwtDa0w4lKht2hYTqFp5ojat3MLx3qOAXGinaVVL4T4pdb3Saa7HntAJnsJIgmyDeHDzGtz5G74wlf1CQZUam7n2R4gFaR4a29mWbtu5TCUroKYgCI6pP3W3+ZbqIkUaUk7lClI+oiwDhpIuCitrQNwvE9TAS84XlwW9gVeGp118rYaZwskodKnFyE6DUe7Tlrb2nq8LXFXAoqaVboOkLdWoH2EmzcvDglwvvWTeGDLlfJO711UjpHtW0T8bAdhknIorK1nReIBYXT308yNVLQrKSdzI5fPws/8Phblz0yVKISjMQdUlRnbeNvdb1QvPod9Hm9nC9V8M7h6wgjO3RBgwd9WSiPZZFf6D/A0tGnu7DyKJsz2WKqrCte4qfUPhYEmGzEaIrKyLmvOJIPVZy1oAAOzAJ5GPP4R4WERQ9aEskq17RUfITI3G452v7XdAfhk6rPQXnfrGUHKkXg1l9yqdR+NkN3oD3dTKcXd2Ma9BUMqesp2nsqY29Zu1N+H1A2CsMq4OqpYm7aNThKkIAAIHnzJFhmbtZQAFJHWOGZSqT7O8effa3Nd0Fb+bCBQYybeCTm+nu8NiYOvYeXz8OVm9ePQ04kUpdcp665X1mAgBNUJ8/oCO/nanJR1LdS0qyyeA7OVaDRtuOnq8yQ0kAqUfW3Mgxy52DVQukR1pClyRmTHlpI+Fp4reidxdpsobw9R1CU6qWm8GEEnyWpJ+E6CyBVcAeKlCXHHcAXstQEIFMgPgd89WVA9+/L2WrmCdu7T5IgkjOxCiFy7lFakoyEAFSlZSAPAd8iT5RYu7SvGmIIb7UjRcE7iNucH32kyv4W46oVpbqsFYjpUpSFOKdu59I74kpj3aWalfc9TQKAqmalkpEJQtGVRPM6jx3iwjnG4RBl5KCsU0zjL6kqQjNmClFKpgRqT3H8LLmDbxWzczzYJWtLxy6QIIGvjtvYPGdO40mpSWh1zozKIIJgEad0R9dk/BhL1DUNmYzJIBMxoZ38QN7XWNLmqq45X6JYqFFThWRKlfrHxnfykad8WaGKKdtyhrUuiQlucxPtO3LTfuOnOHgtCygqkZiZmN47u+LNy/musaqUdWcq2VAa5pGXz1/HcWu0/okWVaYXCiR5swShJWUgn1s067x7/GTvZOIUoFrIcqhJPiPriD5+6yopWUJSkuAkcoUDPPw2gyO/uFk1RQ4iQDpEg8wOfz3W32LIcgnezmJ5Aamfh7B+Fg3UpGUrCAoaHtb/ADpHlYQrTKlNrBV6qtfWHMjw0+FgFKXKc6CEq2GhMSeXLf42KAolaXkICeq0Ha8hyHv+e4tVAFpwJiQQSSefgfnnYZeYJBCVHeRM7c5sE52kOBWYaa6zr8xYoUbckGlU3U2cmjVQrtf0kjT/AEbdw2vKggKzA6g6zy38h8bcU6Su7atGU9hba9Bt6w+0W5RmWBIO8HXQ+3laRTclstsxBG5gk6GdvgbZlSghKokdxiIthWSZUInxgRET9VuSCo6Aajtd43+6ySWygKnmNI115799tKaSklMmZgEj67dErzFCmyQdzz0tzMQVaE7Tz+6ySXOUAnQEgR6u1uVNawoDXaTYTNucuaYIBVpA+u2lRPYA17jHyLPdNZALQlI0387B+dhlp7M62CAOo8LSBUCFob29mPySuKVX30Xqm43Hcy8PYjrKVKf1W3ENPD/ScXbxnG4t6dfkZ8TkscTcGuO6BV3Xm0if+ebWY/q/hbHx1mejJ6EH9H6VcoHWmt1XpeqRubc5gNrbXtYM+e1uHC6FouEJvtbmItiDsIt2ADtY4F02yDUMq2lGBCo94iwx0sBUfyRP6pCh7DYxtp323KA/i7KvMOazS29uVsE87aIBO3hNr4CBZYYi2hqbbjkRbRnmbPayRWyTt9dgqhAcZcRIhSSIsKJOgGttKEjTxtMNuE17JPpzmZQSZ7NtqkmYsFSuoX1zCFJKmXVIUJ9XmAfYfqsMZGnfaKley5GulhGP0gdTobBjc91ugopJyASQYkwCe75FgyWAuVKMm9kOltbiwhKZUdBYu9VXUhp9D9e0nqwVHtglKgNdBrPhZUpRdbLXpN5VNOApJI61wBIHkTFosxHxNuBGN3cCG8qinceYFUhtghGdsaLDhAU4AAc2RKZ/XKQDbOlxAUjc5IB5X/T0UiOK7KFVfpMOXldvF792uBK+kp2nrt9FvB2qYcpG1ukqbLS3XEpTJRmgg5hBEaAWrdh7hHkvNrH+OMaXRhy5bxeWmhZceDf5wJUSopABWUQYKkJKc0esAYvHxf4hXLV3G1c154fq7ypHyh+moGLtVT09KJ7Cy6611KloEKjOESCmTakeIeD2LcbA1+JsWU9J1763k195OCkBYSIAAUoBwhOhyAjQjWCbYdRijpmlrpAwHc21+rdCfTtY/MG5vBWBoru6MNPRsMN1FK6htpKEuLpnSpYAABJFUkSf6I8htbLQ1SYVwzS0rNK/jC43HGW0trX1FX2lAQT/ACY+q2W5I0Dr/wAZf+31K1xf9QL1heB6zU8zbtIMc9LadEO8xM8rdoAB+NvoZ265Juyq107KGnvS7+Hl2VgUWKvEPUO5TByLSlJg8tCbKdR0JeArIecNBfYQ0CTF4q2AnmLFOm6THDM//wBTo/s2sFXJLjVU0TlC0qQTG0pj7bYNPQUtbiNSaiMOtktcX9Ves13ajGez3ZDB24VVPhD+8Zsji29pBa9t7XKo5w5uXoNcUL8osNXDV4rorzvFQbpae8i6yHFkSEZ05kBR2AJGum+lpxZ6EXAcnK7dt8zyi8lfdYzhfov4QuO9bsvpTrFQ7dz7VU26lrKorbUFJO55pHOz1/fCbrsfKwjQ0lcj83Uxeq3qilcYQpa1DIlvOBnEJWcyZTtBMGNP5Cw0j+Ab5Bcp++X2vv8A5ym/9R3xTNR0GOAC0g/m6+tf/WavusXf6EXABtWRu6r6WR6xN6KAHwtYGncIZnnvaOeOdJWXtwnxDdFDUOMu3my1RFbaoV1bzyEOAEaiUKUPI2gMCw6/8C3yCkfwl9r/APSU323fFRxRdDfo616S7R0l41TSVZVLpr66wJPccoOvhNlRvoN9HtYB/N1+Cf8A1mr7rM/o4cFqzhRjhN43XVutXfedK7T1dKFqLSiElaFQf0gpO/8AOI5m1nRWparfRSRqgLHtkfZZ3YDhvKFvkE7fwmdr/wDSU323fFQs50Fuj20guLoL7CRv/uks/ZYEdCHo5mCaS+wD33g5/dtPdW6TTATupI+ItXLpUYw41YRvDCjnCS/GKNDrVc5eFM9QtPpqigsdUCVpJSBLg7JE5tdrQGA4cf5FvkFP98vtffTEZv8A1HfFLDfQX6PDolukvlXleitPhYX+Af0fuV332f8A3mr7rSXhC/K6uwpceIL7YRS1tfQ0z9U0AUht1xCcyQDqO0qIs7GH8ySQdCLQOBYeP5FvkEh+Eztcf5xm+274qCP4B/R+/wDJ1+f5zV91sHQP6P5P/a6/P85K+60k44xO9hTCN/4tprqdvN66aKprk0TTnVrqS0lSg2lUGCcsTB32tBuG+mU9fTjbLnBrGbDioSU06w8UkmP1U2XyFh43ib5BSb+Erte7bEZvtu+KdP8AAO6Po3u6/P8AOavutg6B3R+j/tfff+c1fdad6OuXUtId7SQUgkK3sTqr0ebXUu9eW2qeZhM6BIUTtPO0RgdAdoW+QSP4S+1o3xGb7bvioU/gH9H3/wAnX3/nNX3Ww9A7o+/+T77H/vNX3WX8A9Kbg5xLvZi48FcRqO8LwqZLNK7RP0y3YGYhPWoQCYBOUGYB0tLVNeSnQC4iDzA2sjgdAP5FvkE4/CX2tP8AOU323fFQN/AP6Pv/AJPvv/Oavuto9A/o/f8Ak6+/85q+603XviOkupxhp5TaTUZsnWOhAMASBO51sGjE7TgCk0ylA80uBQ9nfZvkSg/It8gn/fJ7W/6Sm+274qA8T9B7gNdWG72vSkoL5D9HQ1D7RVeKiM6G1KEiNdQLKfQOMdH+k0/+1K3/AFhaWcW34xWYQv8AYSw6k/mqrMqyx/JK8bRJ0DyP3gKQf+tKz/WFqDaOnpMTjEDA27HXsLc2rpZu0WK9oOxFY7FKh8xZUQZc7i612S3tfrYKxQV32yeQ9tuZjlat2OOmldGDcdX3gZrhpiC96i4qgsPvUa0qSrQdqIJA1521Kurgomh07rA6Df8AQuDwLs3inaWV8GFxcRzBmOrRYXAvdxA3ICsmpUn8bZn8bVZT05kLGZPArGhHeG5/s23/AA4+7gTjaP8Amv8AZtS+W6D8p7nfBdH+9d2s/on9pF+urSZu+2wrvPvtW7BfTOuzFuO7jwHVcMcQ3LVX5UBhl2uUlCU6HtQQCRpytY2Cefna5S1kFa0ugdcDQ7/pXOY52bxTs1KyHFIuG54zDVpuLkXu0kbghY0eyRPMj42Ez85sXa/TSde2r67CAgjf32sBYhQoV3ga20SQdJtyTp5WxJ1IEb62dMF1OxM+62wvQc+6wZ+M23IGwsxUwboUK5xpbMxnusEkjc23JOnfYVrJWQgUNxp7bZnM6WD5m2A+NkkhM0c/bbS0NuDK4hKgdwRNgwedugrXvsxARGlAm7bsWQo3fTkjY9UJHwty7dF2vJyLpyUnkFqA9wNjA20Ftgj22hkb0RszuqbV6cKeG99lSr4wXctaViFGqoGX8w7j1iTNmnU9FXo+1ec/vV4cZK4zej3aywSBtq0lJ0nS0pDvtsGDaBgjPJPxHjmoHvToN9Ha8syk4L9FWRGanr6xJ9wej4WZd9/k2+B16lTlNeWJKEq5MXiiBvt1jLh5nna12Y6SZAtsK00PnNod1i5BS4z7WJVA74/JF8PX8wuTiXiakBTCfSBTVUeYDbRPv52YF9/keb9Smbh4zdYqSQKm4ghI101RVKOk/q8hb09SobxbtKxqLT4VtioX8F5AXv8AkjePtMVruvG+EKtKZyBa69pZHsplgH2+2zBvv8mP0r7rQtNPhW5LyjMEqp77YaEf/lBbI91vb4LB+63SVDflZWcOaXonkvAC+egz0q7nK/S+C981BEgegP01ZMDeGHFzZoXv0aekLcgUq9+BmPqdKdSteG6woA78wbKY9vnb6LVtsuj6VtCv6QBsXF13UTP5upgY5MpFkHvG6bhtK+Z57DGJLnTWMXzcN5XeoNHs1VI40ZChocwEc7I4SoNzty1t9Or1xXVUJKHqcqSd0B1YSfYDFmve/BLhNfxUb64fYfr83remXWxUT/WIVZ+M4ckjEDzXzZlDgjNpproRpym2lKJnsQCdpt9C18dCrou34kit4J4RTm1mnuenpz72kJPxsxb3/JldEO9ypX719PSKUZmmvGuag+Qfy/Cy7zbdqjwfFeEkEHspiNtbbKXFCEpMTpGtvam9/wAkj0Y64f7nNYju6NvRr3O3/TNu2Y1+fkcOGTqnF3BxDxTSAjs+kLpaqD4jqmpG/OzGraNwUuCSvI9CDlzDQzrIjT5+y2dUiJKhGmo13FvTS+PyNl8IBVcnF9xwnlVXA2B4SUVZJ/ybMO/PyRXHagKzdOMcI1yf0UvIrmlH/IYcTPtshVxlPwHqgqko/SJk+NgSkgxa4V//AJL7pYXYIosOXBeW2tNfTbQJ/wDyjqj8Odo+vnoEdLi4lBdXwYvKoSVAfxGtpKw+5l1RHtFitnjPrBRML+ir3qLXf/JG4jVdXSTvO4lO5W77wzVNBJPrLadZdHwSu1cL76MHSNw+pRvXgTj1htP+E/c/VLRH9NKCn42k/oGt4m4c9Lzh9U3/AHFed1oq6167HPS6RxmfSGHGkg5gP0lJsGvyy0rwDyT07XMmabc17kKMggW4522VamLaHjbz4DVdKFnssIgzIFuBr4W2BrY7Eiu3EZ21IP6QIttpWdtCjzSLYk5hB3txTK+iUjTsLUny1kD3EW16A2JCrS6BCj2W3FtCDvpbYmNLaoCr3Wc5m2hlgcrbJ03tke2LOEyyCBIM25URtbek/abYZ2FihRJ1SahIRW1CEpAzwvQb8rdq31GptzVAiubWD66Ckj427VKhMzaFksy5G4IG3Kx+6ruFW+X1tpyta7bq5CxFsdY+3TI1ccMJTv7fIWdvUeg0JbpU5lJRKdPWV3n22ryNadTyTPkLNBuU274obqvd1ZxI8wzc7Kkthtx3q0vvKMQsyJAJCQmdVEyDAtGXFrh7w4uy4K+vp6c4bTUw07eNPXO0DjwUQVJGRK3HlGE9jIc2utppeDFJkWtrOGyCgZcys0RI8dT7zZHxPhKixldpu/FVIh+gdWlw0QWQlRTqA4UkZhMHL6ums2xaqijkY6N7Q5zudtvr5W5J2SlrgQdFUppT17U4au69MeX9d1I31iV1jLj7SlJSskMtKbcUhUNmAci8vZHrQa84odwliHG1PcFXjbFbtRf1cqmdoKmj6tbDZKc7mROYJQMs9pKAAmQlyDF8+Jt91NZddJdWAaBSGKF+nSiopmAppaQotrZZCSAmEggqUUo2SFTJTUPjfjNoNX3VYJvi6bsvh+6gxXt3UW3allbaHGAhyoQZAHWoU3kyhQ6zSRbiKmi7tNla4uHXkD5HZa7bPizHTwTLr2MDUtdUU1P0S+JCmmnVoQVO17aikEgShNOQkxyBIGwJtlor/MHGVXau2qvkUh1pw68kLDf6OYFQgxE6C2WJ8nwc5D9tyz+Ofo+4L2OfEPRrboDSPjbb6B1o7zbEjSTvb3Zx1XMt2VYOm5H/AGMv/wC50f2bTrioKXcF8tgxnpKgb/8AJm0GdN5P/ey5zihH9izw4knjWxiioocNVdxu4dvDK31dQwoPsoUkJc7Q3M5iO6RO1szD3NbiFUXG3zP+Vd52lY53ZLBAB/Sf+qFXzgfwIxfw1xjcF/XJi+8hQN1DLdXRqqCWnmCQlaVDY9k6TMQI1Ei3V7NsG8qF1bYK8riUmNQOzp5WLsYZq6O4XHKFLaq5plSmEq0SXAJSCe6QLMPBeNOJ+LMSN3Vi7h3TXRT0iFr9MarQsFYgZQgmSDvM6ZbbpkZpdy86Eb+immlMU2+yedm/jNv0jC8J7QLtMoz3dYnWzlpGFJYyKHKNPK0VYj4p3Xw8r3sOcQrtr2rucKjSXmxTqeYU0TORwJ7SVJ20BkAHSyjcCdEz2kBO25mENO0y0pECRtt2FW7qqjPicoTEIpW5155lH6rR270kuB11UvXs4ucrHAIap6eifLi/5ozIAnzI87ODhrXXri5L2La+73aFN5ul1mme9dpgAJbCp/SypBPiTZ5Hi+qUcbgFIj5IpWyR/hEcv5wtCPSJ6QaeCF44cp3eHL2Jqa+G6px19moCFUYZU1HZLas2brDzEZec6TNiC8KK5LndvS9KxqlpKYoU686oIQ2nMO0onQDW0Z4pc4WcRaqkqazG9wPCjbcbaSmtYcBzkT+lp6otAG+ykR1SrTvUHFDD+G+INzXveTFGtlN409GVBDTwUg6PJAkqTJjtQCNuYf11uq6lIVMxr32jJ/GuBsG3VdWA8OXnS1tW+pFHSU1K+hxSEEytxYSewkJzHXcwB4SRdAWqnQpQiR7bMd0tUjYqq7iu/Cl93hil5TNz0tJVPXg6UqVlpkpUpxUJBUYQCYAJ00FopwvxZ6MFU8y3hvHFD1zykoaQ4y+gqUSAAM7Y1JI0m0t4hu5u/MN3xcamw56dTVNNkVEKzhSYM6c+doywl0fcJ3FXtXi9hyhS80pKkq6pCiCCCDI8bCk13F0WLTnZTVQn6MBAGWNxZOdGY17atlOEE+baRZTpEobaCArQCLJqkAu3gmfWUeX/ACabTZtZQfuoMwn0YsNYWxRc2KGhTtVd01CKlDjKCFLKZMajYzB8LS1c2MKW977rLuoWapCbvWGXVVFMtjM5AJypWASkAp7UQZ0JGtqo8LuD+PeG2NLgvq6Mb3u7c6axtuqoHqpa2nGVkJUkiYOhnWY8wDa3ZS0a1hZTCylQCo5SNPZJsnDKE7ddki8W+FlwcUblYob+ow+qkUpynXnKS2pQgkRG4EWQ+FnD+kwNgCtwhSKWlpqorAklRJAcGbcmZ7RtLDAzMgEk6WQ0pSl+8WwN3ife2n7rMNQblPexAsq+8MOE954BuLEV5VOJr6vBLlx1TCGqyrU42gFskkJPPsjXuJs5+giQOAVKDH/bSs/1haQbxYbTgW+ilI/7VVPn/IqtHvQSTPAKkO3+6lZr/jC2BMCMTiub+g/72r0jDrHsPX2/pFP/AMkysUVWpVhqVdI/jIAdPSackDn2xpa6hO8aWphhJOfpIcZydYeZOn9MWfEBeopv98/8jlHsef8A4RjX/Dt/68Sm/AuI/wA3uO3a4pXVP9tAn1Vxr7x9Vk3DF9VAxomtU+6pC6p4qGYwZzDbbc2i6sxhXXNxrGDKmUUtdh9i8rtcP/jDT7yahHnkLKvYqzxwc7UKrGnaltTbnpz+h7uuVlPtTEeFtTZgBXAjV1wkjje913Sv4MqknVz/AFza1IM6D8bVM4xKKulRwZkz/Kf65tbEGBbJoP4zU/74/wCVq9A7Y/5owX/h3f8AXlXLXrOTyXz8hZKxo4tnBt+utKKFouyqUlSTBBDSoINlRsgLcGs5gY9gskY5P+8vEGn/ANl1X/wlW0ZPmO9hXFUOtVFf6TfvCoVwj4T3hxDwBRYvvDiji2nqaqpqGVsMVqsiEtkBJkkkk6+60o0/RMo3rkN9u8ccbtobbUp1AfzFBTumc2v4ixDovA/vLXWeRrq0c+Sx99pMrMTMKwviJuiqgsUzL7TyQofRvNo1BHIxB8QRbmcNw2jkjhMzL5mgk3O5HtXs/bbt72jw7HK6loqjKyOV7WjJHo0GwGrCdkw8O9E2ixF1gY42Y8ZU0lKiHHk6z5LstHoRpiRx2xt/W/7VpPwW+qjqgltZIca9+os6cYX49dOGqqtZP0oRlQQYhSjAPxm2hTYLQysuY9fa74rlKj8J3aqJ9hV/2cf6irbWdFbC9BUqpKrpHYwDqDlWlC1uZT3EpJAPhvZTu7oa3ZezPpF29ITGFQ2NCW6gHKe4jNp5GyFxIuviBfd2sNYAv83XUtqW86sNBRd07KJMwNyY128QXDwIxbiZVPc1fiIKbvJThorwSABmIWUEkDT9VWkCRpYTcHo3N4hj0vbd3xUz+EztSDkFXrb8nF+ohV9CJKE5lcd8ahO89d/tWxHQiQtAWjjzjUg9z0/2rTrju7qnEWDL0uiidLb1TTqS2Qop7Q1AMciQAe8Ei0O8J8YuXdWJ6+Wg6PRKttR9VxJ0JHIg/BRtad2foWsD8ml7HV2nvVZv4Uu1bnFvetf/AKcX6iTR0IQTA4742/rf9q3X8B0ASeO+Nh/0v+1Z31WLbzvvFKF0961bFH1qWkJp3lIzNg6nQjfUzvqLTCivSunU/mIRE9rutXjwfD5XENZt4n4o0n4TO1sYBNXv/wCXH+oq2joQj/08Y2/rf9q3Q6DoIn9/jG39b/tWdPEHjXU3Bd9ZiFl+sRddJAQiipw8+/JABA8ZncAASdpsq8IONLWO6CgvFmrVU3feYIZcdbCHmnASnKsJ03BHPkQTYfyVQXtw9L2vc/FFH4Su1oF+9+Nskf6iYQ6Dv/t4xt/W/wC1bX8B4af9njG39d/tWsZfl+s3Nc9Ver6SpFK2XMoIBVHL26CzKXxgoqYNm8KRinDhyoz1oTnMTAlInS05cHoIQC9m/i74qMX4Te1sxIbVbf6kX6ihq/8AoXLue4byvdvjpjVxdDRvVKUKegKKEFQBObws9Og5ed43rwHpaq9K+prHzedYku1DqnFkBQgZlEmz+r8bUGKMG4lYpWVIUzdFUuStKgQWljlaNugef+wBSD/1pWf6wtVhp4Kevj7uLAtdfU8i3qtyt7QYn2h7G1cmKyZ3MnhDTlaLAtlv80DewVjQvTSLbzeNoM6YeNMUYE4L1OIMH33U3VeCLxpWk1NOQFhClHMNRztE1BgbpIVlFS1T3SuYpl1NO1UFpxC5bzoCgkmIkA2u1GImGYwsjLiACbW53tuR0XN4T2Ojr8MZitXXRU7HvcxoeJCSWBpPzGO09IK5oX4266zlrao13cIOlBeiii7+lbTPqAkhoFRA8QNbHjwH6WwSVfwndB/yC/usP5QmeLind5t/WVg9j8JG+M0/2Zv8JWtCwRobdBzztVBvgR0tnEkjpOnT/kF/dbY4FdLVSyj+E+ZH/IL+6zd+n/o7vNv6ycdksJG+M0/2Zv8ACVrguNffbfWCJBtVI8BulukFR6UGn/ML+62J4DdLdQlPSfkf8wv7rQNbP/R3fm/FP+5HCT/PNP8AZm/wla0OHQcrdB0cjaqJ4C9Lga/wn/8AqF/dbBwD6XJiOk//ANQv7rR77P8A0d3m34qQ7I4R/pmD7M3+ErYdaZ1Olt9bO1qofvCdLkafwoP+oX91tfvCdLkDXpQGP+YX91m79P8AkHebfil+5DCD/PNP9mb/AAlbArJtwSDvaqX7wvS6P/8AFASe7qF/dbP3helyf/4oP+oX91m77P8AkHfm/FL9yGEj+eYPszf4StWoz3e6wLiWnJCm0qnvANqs/vCdLn/7z/8A1C/uto8BOlvz6T//AFK/utE1k3Ond+b8VIdksJH88wfZm/wlZxy76FRzKo2Ce/qxYBy6aFcEtFMbAOKEewG1aDwG6Ww36Tp/qF/dbR4DdLbn0nP+pX91oGpkO9O7834og7K4WP55g+zN/hKybl0U2Uht19HiHSfrmwJupcy3eNR/jBBH+rauB4C9LQb9Jv8A6hf3W5HAbpZkgfwmd+9hf3WATm/8M7834oo7L4YP55g+zN/hKxy6KuT/ACda0e7MxP1KFueqvNB19FX7VI+w2ptxOpeknwUvPCVViLjrVXzS35fLVCplhJRAzJKs2Yaggxa62nttKmjiqXPYYy0ttcG3P2ErP7QYC7AoKeqjqWTxz58rmBwHoEA3zNadz0RMKr0amjbJH6r33gWIXTU3u47VqqLvaaUp3MG+ukgRG4BB9X42WjPOxVsdXWLExnEx5R+NrjaRsRBauY4xcLELpLtVHapD45XAfuts1BTM0r4juSD9RsPsJ5G28xHcLWww9UHMixrWBooOp82lfdbkXjQlUKqkAgbHT67GzrqbagEQR77NlcOae4QQqadwSh9tXksG3RM7Ee+3DlLTuCFstq8SgGwC7ru5SlH0NoFW5CY+q0vSUSQEXvd9NOlh9QVCXkjQbAmLcCpNQot00HLope6U/ebEr/w829ddSihedYdyKKFlxa0pMb5CqDz0sfo0JRTNJbEJyiABA27rL0i4hI5QLhOK5LoFCg1TiSXnfWKvWj7LLSnEpZLygohAJISkqJHgBqT4WRqa/WUUyEVGZTgEEx3bGzNvfinizC4cqbw4cO3hRl4oafuu8EPFKIJBdQ4ltSTodEZ9YAkkC1Grq4qNhkmJAHOxKEIZJ3WaNU/n2VA+n1BU32ICCrstp8eU959nK0R8VsY4mv8AcRgvA1IyUpKXa+orFqQ3VNZoVTtoSC4tJkFakwMoKcwKtG5jvjpfV8VSMLMsU1xuuKS4Wi+ldW82BqEpMBMEgElKojYcmlfGPa64rifRTXbcjVTeTuSoXWLLqnihIhTjisynNCAM0hIBAna3DYlj81Q9sdCDwza7ranrvay2KfC3MZmntfpdA4kYvVFx1d24twJV9Y8FmrvCicZvFsmAAo0+ZDLcNpSlKCFFICEgHsk1q4g3DwnwybtvDC1beq6C8XEsulbRbdS9o4lx5teVKUEqhKUoSIUTrNpjvfH+JEUR/Md5YeVVPoU2hFNdzSm0OQSlRWGwnIkwZKhrpBJgw1f2IqKqvbrsav1WJ74QQhK2SqluwiUkAoQnrFZo1gNuEpPZUU6ZMk7pJNAbdFZkibGy1wSonrOIfE6gq36Gkw1iaoYp3FNNOr6panEJMBRVkEkgTMCbZadF1eDlLUpjC4DZJKOruzOjLyyqVmUodxKiTzJ3tlofK0X5D7/iqfAf1Xo5WpAfOmmwAtwgTrYevH03meVgkjQEb29/3XIqr/TfB/7GMj/96Ef2LWZfom3lSpMm0c8duA12cdrrui7LyxDXXP8AmiqVVtuUraVKUpSYjtbRvpaMP4Dv/t2xt/W/7VsIispqyWWKLO1+X1gNhbmvUI5OzuN9naChrq/u8tPxbjhPkvnfmGrbDYeO6ssmlSlGUJ30sWRc1O2+X0NAK5kDW1c/4Df/ALd8bf1v+1bP4Dh58d8bf1v+1Ypq6/8Ao357VQHZ3smP56/u0vxVm0NZUxZMvnD1FfDRZq2EOJVoQpIINq7/AMBz/wBu+Nv63/atv+A0T/4d8bf1v+1ZjWV50NN+e1IdnuyY2xr+7S/FSyzwkwrT1fpLNz0qFhU5kspB94Fnjd11NUTaW20hKUjTS1dD0G//AG742/rf9q2fwG//AG7Y2/rf9q0G1Fa03FL+e1TOAdlXjXGv7tL8VY6+7lor8umoum8aZqopqlHVuNOJCkqT3EG0M3l0YuHlS6VjD9Kkkk9lGX6rNgdBvWP398bf1v8AtW1/AaB/8O+Nv63/AGrJ9VXv/wDDn7YTM7P9k2/zz/dpPipBwPwOwjgut9Oum52Gn4gOZcy4n9Y687SlT0uRvLli1bP4DPdx1xr/AFv+1bP4DR/9O2Nv63/asmVVezan/PCd/Z7sm83+Wf7tJ8U6OJPCniBeuJqy+sLcUb/uenqsqhSMuktNqCQDlSCBBifMmzXZ4edICjVDXGa8lJ2lyhQ5/rHX4W0egzmOvHXGp/6T/atz/AZTP/f0xp/Wf7Vk6sxMnSD89vwTt7PdkANcYv8A/jy/FWIuKmqUUDDdc+p59DaUrcKQCtQEFUDvNo44lMcXqTEqF4Er7nTdVQhCXmqymUpaXJIUoKTuIy6d48bMAdBuP/DtjUeTv+1bR6DKVb8dcaHzc/2rFGIYk35tP+e1DPZvsid8Z/u0vxUtVmFqtGGKj83ttqrmmCunCpCS4nVIJ5agWbXDvEfEzEd8OUmM8KXfdbFKgqS8xUlSlrJAyhBJhMTqTyHjZlfwGuQ46Y0j/nP9qwY6DASokcccZCeYc/2rJ2IYk616b88JDs12RH88/wB3k+KsxTslLYTM6crRRjnihRYGxW9cddhrEFYqtS260/Q0XWNGRlylRUO1p8RZhDoORvx2xqPJ3/asC90FkrIUrjdjFZHNSwf7VkK/EG/+G/PakezfZIm/y1/dpPipovNl5OAr6UtBH+5VVv8A8yq0b9BGP3gaTv8AzpWf6wsgfwGUOoUy7xwxkUKBSpJckEHcEZrTZwW4TXfwXwQ3gi7L2qLyYaqXakP1DaUKJcIJEJ00sCLvVRXMnmiyNa1w+cDuR09ivYhNgGFdmJ8Kw6u7xJLLE/8Agnx2DGyA6uuD84J9KHOLU2wSppvpI8aXah1DTbbjClLWrKlP0g3JtcoxahOIsT/uC478WDiThlf+I7qxI8imSKSmVl7CkrCsxEESBEcxZ8Vmjp5KeWT5oeb8/Vco/g/w+oxaixehpBmlkgAaLgXImiO5IGwJ3Ug8XMLfuhp6LFuGnWnr0w2UVdMplYVnSknO0SOSkKOnfFnBhS8G6tNBeLUdU+Wlo5xMfPstGeDekFhnA93OXZcvAHGDTLq1OKHUklSjAkkgzoAPZZrU/EnCtJi9OLKbg7xAQU1HpKaUFQaC5kDKE7TGnhYrsfw5zQ3OdPA/BVB+CztaHX7r/aR/rqTONVW1R9Jzg9VPZsjYcJgSfXIta6hvKnrWuuZKwJ1Chr32obivF9+ceeM+A6m48FYkwuLvK6U1VTTEltSiVBYOXKIjna4nDW4L/uC5fQcR39U3tVFwrU++hKDBgBICRECPPU2r4XPHPLPJGbgv03+i1XvwgYdUYXRYTRVYyyRwEOFwbEzSHkSNiDunuggOrIPdZJxzH7ir/AP/ANl1Z/6pVlZEdaf6I+2yTjj/ALisQf8A4XV7f8yq2tIbMd7F57QfxqL/AHm/eFVTottg8D7mVtN5V4PsUk2TLvYewpxvxVcN6OKduPiA0iupQZCUVbTCGX2v8dAQeWxsNwCGKqfo0XVeOEbhRe9W3fNahbCnAiGyRKpO+wHttIV68OsQY3wNSXjeN2t3fiChcTeFM2hwLLLySSEZhpqnsnlrzi2dhUQfSQFx0LAPYV2X4Q5jH2pxIAa8eQ+9PPCcsu0iCsryNdXmI3iNbLvEROfBtaqfUCFGP6aTaNOF+KMT3/iE3feGA7yulunQovVFSsBOYGAAmASTM9wFpixDc7t8YavC7ElIXUU60IJ2C47Pxi2jQMMTcrzzXFVzxI4Ob0UZYIp2nqh5DgB7I0Ptmzaapm7uxNXssDKG7xK0pBA3yqJ+Px9yLh/iZdtxXi7dt/Ppui96KWainrewlR7wduU7+Uiytct8XRijExXQXlS1L9W+H3G6ZwOZQAkE6Exokb99ozkwwmFw1upRN4kolB0srC3eS8yEqHrCTaDOIWGV4Wx4ivomP4hf4Wp1KRoioSJJ/wAYT5km063SypLKSpJBI7rM3jJTsow/T1rsJFPVpUonZIIIOvIWuxEPaWHYhVHktcHDkU0sB3YKutS8EQlkBIERr+y0qXu0tvD9alBhQpnI88psxOE7zFbTVKmlpXkeGqSDukfcbSbVMh6icaIMOIKT7bUKJgZqVbrH53WUCXPc1Jf1PTXbVtIcaeQlRSRM9mbFcB8DbzwnjBpd3YqrGbgp6s1jV2tsoCZJzZSsgmM3dGgjxsewheLVI+2HVAu3e+umfSN0KBKdRuO+y3V4rxC9ftSLtrCxRshtLJQhKuskHOTMzqY9loPaIDIHHW+yMwumyZdrJ/47aL2C72TufRHFQfBM/ZaBsWYEw/xFueiocR0iX2adxNSjUgpVkKZBBBGijawl9p9PuGuYBCutpXEkd8pNq+X1hZeOcIUlzovK8LtWvqHA/RP9U6CncBQ5HW0q8fiGOBtZ33hRoT+Ne0i+idmAsD4PwDw+xPd2GaRLT9RdtS5UryqzuANLCQSSdBJ08T32KdBEj94GkH/rSs/1hYzgLhw7gXAeKnqrEF83u/VXZUJ6y8axT3VoDStEDYSdzvoLFugh/wB4Kk3/AO2lZ/rC2Nr8oRXN/Rd97V39GQexdfYW/wAog/5Jlvp368AKv/8AFKL/AFjZvtM1zuFXGKCp6irNFlYdyhWRwsgoVB00MHXTSy/07hHACrH/AK0o/wDWNiVIyplhpnYGho1g/wBKnQbELb1FSejGH84pql1uyOHf8RUf9OJMDo+Yv4i1jF5Ix46F3xcd5Btqqbpwz1iCkESEwn8Fa2uFXVbv5rqXaZeVzqFqQe5WUxanXC7iZiy/saXrg7F+HaSgrboQmrYeplr6t5GeBovv335ERa2ly1v5yuZmqIA69oFQHJXMeUza5TejOQ4WvY25LiKnWEEG9ri6jHhRfVUu9aKqfq3nfSmXG3CtalSqCZM88ybJ1+4+xKxxEvdpN7VbTN3VbSUMtulCC1lBgpmDMnU/ZoX4fFVHXN0SlQaK8nqYzp+mR9Rsk47YFHxTvhsEg1tLTVI07k5P7NrOS3emdCSPMFBJzOgd1GvkVY++L39AuWtvNCQ56PTuPBJPrFKSY+FmJwux5fmIK1LV7VTS2n6YvISlpKYX2ToRygnf7LOShUL2wiyVEn0uhSD/AIzev12iLg/VluuuhC1GQhdOrzAUI+AsKX+AbIPpD3qcI/Glh6H3KWeJOOF4OuilrGqQVL1VUppmwpWVKSQTmMakafEWPYGxW5iq5TeD9IindbeUypCFlQ0CSDP+NZjcfmlrwWxWpBJo7xYdJT3ElP1qFjXBt9QorwpgdEvIc2/WEf2bKpbkbG4c7g/UowjMXg8rKU82kkwLNB/iDdTV41NAgVTq6ZwtrKGxlzASQJImNrOrMS0fK0D1tM0rEeILrdUQhyoeQuCUkBwTIPLQ2FUEx0r5Wbtt99lOIB0zY3bH4KXLrxnddbUJpS8ttxeiQ62USe6dvjZwFYgkjYWptga5MUcOuKNXhC8MSXneV1V1CirovTX1OlohRBAKjpsqYiRl7pta6769dfcjb2cl1bRSox+kBBPvFgQuk4giltqAQfAosjWBhczkbIWgxTcV43g7dVLfd3vVrM56ZupQp1MbygGR7rKFRUJZSFq7+ZtT/hDXG7OKt1POGFLfcp1T/PQtH1xaVuk/WVScDUrdO64lp2ryOhKoCuwopnvEi2l3QmqdTA7KoJhwBN1U0CpQpoujUedsadS6kqHI87UhwRxdvfCeBsR4ZbvBxg1LIqKJ5CjmadCkhYST6uZG20FOm9pC6J2Lr2r78v66b2viurAqmQ8wmoqlupQEOEHLmJAkLTt3WruiyxCXxsiiS8mS3K6s4txDacyjAsGH2ldkLHlaLOPPFt7hrh9pV2UrdReVaFdQHf5NsApSVqEgmCoQNJ79LVpuvpK8X6G+6VF5X4H0VqA4w3UXc2ll6NwkpSCRpuFeFguGQgEbo7TmuRyUl9OD/wAGUf8AnQj+xazcmY199qddJPHVLxDwhwnxJTNhpbmKEtVDMz1TycmZM8xsR4KFrimSdPbNsylN6uc/7v3LvO0AI7L4OD/tH/UC0TpYBaQKhtXfpYc2BqCYQs/oqn2fItokLhRoULyg7W2O8i2QPK2hoLSTbrZ+y2jH222Ytox5WSS55ae22ADeNLYfWAtvlvFnAUCboJ9GdCkHY6eNku7FqXRoB9ZsltXgQYssFIIP12RrvStqsrqcoKQHg6knuUJ+yzkagpA3CN87CsUya1iopSEklGdE66pOlgyNo526YrKegWauqXkZQhWdUE6RyA1OvIanYTavWRNkhc1+1kmOLXXaoZ4kYWw9eN5O3VfV1UdemoyPejvISopAGUuwrYJM6jXWOdozr8C3rWNMKwhjq9LuRLhLtWymvbKHFZiW0ukKQlOgTlWmQBvGa0z1l0Vl5XnfOIq8VFGal0MU9O+wnrGWkISUE7wSVZsswCSVAkQGdiqnoMP4Vqqi+WHDRuNqSFVAWfSVFJhBUBJKjptGsaC3LhsU4dI9oa3ktFz3ZRrcqI7zwuMcX09dlTfdCbpoEB9tNNTgO1TCgBnefzKJC1dYBlKEwFkJEAqq7xGua+n8UXvc+K6k3XcaalVBSVdTeq1rdoUr+hzJSVOpYGRKoOVBgkQTNre3RTXXdFwqq7xqrqdva83xXVwcdQlqmUQIAR/MACUpPMTyJtTPpcYHxdiDHr7GGX3bwuuq9FrmjT1WZguqQG1rWRIUuUSdyM4AAAiwqekpmOAaQ2+56KvLI4MznVSLRYrw7TUVPTU/Ga9WGmmkIQ0zdby20JAACUqJlSQNAeYtlquUmAMdNUrLYxTSs5G0p6sKkIgeroY020tlqxwSkv8Axn7vghd+k+ivoEvBMPTE62AQkAzEWjJXSa4Y1zgKVXszJiHKRKoPjkWbHGOP3CxwgLv2oaPPPd1Rp7kG3pvfac+uFlGknGuUqRQI1Mm3QTyjbazMY4y8MKkgN4wpUlUgBxp1v/WSLKLPEjh6/wCpja5Ae5Va2k/E2kKqF2zx5qBglG7T5JxjvtmnI2TKbFmE6whNJie6HyZIDda0o/BVlFqppXxLNS05PNCwfqNiCVjtio8N43C7CRz2tuBprbYBPl4W3GmvOz3B2UcpXIE6xbUbQPK3cC2FMiPDeyulYriATG1thM7i3Uaz32wiIgWSdcx3a6e+2ZY177dARztkHkB42SS5yneDbRAsIJ521EamySXAT3C2gPCwka+dtQPKySXESfhvbUA8rdlOultEacrJJc5ZtqOYGluyI1toA6aWR2SXEc7bjwtsAk2wgWjdJaiLaIBEW2QBbDE6GzgpIMtIJ21FtKZSSOzYWOVsI5RZikgSykiMultCnSjUWH35W1zsyQNkABD0EfofbZJxx/3FX/43XV//AAlWWiPpkx+qfstxVUlPXUz1FWNB1ipbU04hWy0KBBB8CCbRe3M0t6o9PKIZmSu2BB8iqgdEXjLwlwXwXp8PYyxzdl13gi8Kp001QohQQpQynQc7TKnpH9HxCSlPFG5IPLrFf3bH/wCC90f/AP0V3J/Vq/vWz+C/0f8A/wBFlyf1av71sSlp8TpYWwNMZDRb1uS9Ox3F+w2PYlPiczatrpXF5A4NgSb2F7myS0dIfo8NPF5HE+4go8wtX3WNjpMcAAmP31Lk/rFf3bGf4L3AA7cK7k0/5NX962j0X+j+B/3q7l/yFf3rGAxUbcP85ZZd2CO/fP7FMvFXFLor4sIXfWNsMVixsp5OYjyJTYDDHE7ovYVUfzNjrDVGkmSGQUT7k62ff8F/o/7fvV3Kf+jV/etn8F/gB/6K7l/yFf3rRdHibjc8P85FZJ2DaLDvn9ii7PSY4AtpAHFK5BH/ACiv7tknFvHXo24wuOouG+OJNxv01TGdHWrGxkagd9l49F7gATpwsuUf9Gr+9bP4L/AGdeFdyx/zav71pD5VaLfi/wA5Q/8AkG9/8s/sUzMCcW+jRgO7jddx8RLhp2FOFxQDiyVKPMkiSYAHsFnZ/Ca4AxH76Vyf1iv7th/4L/ADnwsuTf8A4tf962fwX+AHLhXcn+Qr+9aA+VR+T/OSc7sC7+mf2KhLiJivgret/vYswJx0uq5byqR/GUQXWHz3lBA18fbEkkgYL4nYGorxD+LuOGH6xpBlLdOx1IJ71HUnyEC05/wXuAP/AKLLl/q1f3ra/gu8AQP+9bcv+Qr+9aMwxWf55j/ORIpewUIs3vn9iktvpG8BBSqYVxSuXtJy6uK+61dqrjU5dFQ5RXVjHAtbQsrysKVUO5y2DoSdNfZGvO1nf4L3AEf+C25f6tX96wa+i3wDVtwvuYeSFffabzij2cM8Mj/7k0cnYFjsw75/YqveH+Pt0VabxYxDiTDFBSqu6rSFM1q1uOuKZWlCAmI1Kuf12lLoIg/vBUkA/wDbSs/1hZ4p6LXAVKteGFzEf0Ff3rP3CGDMMYDudOH8IXLT3Xd6VqdTTsAhAWr1jB5nSwKajqhUtnny2AI9G/O3X2KzimO9no8BlwnB2zF0kkbyZeHYBgeLDJ1zdOShPp3j/wDZ/qzO16UY/wBI2S8WX1S4Rp7jrau7rxdRXXJRKC6WlU6AUspBKiNBsPGyr07xHR/q9P8A7Uo/9Y2lrDVz0l54Jw8mpZSuLrpdx/ySbFjce/Tste7Gj3uQa4W7GYe6+1ROfzIVXG4sQ4QvbEaK27bvr1Xw6x6IIoXknq8+aFGMo15qiO/vsrhmiXR3PT0pn6NEHzOp+Jt1T4Ru5h3rG6VAPgLLrFIlpOUDQCLWIYnMkzu6dbrjZpmvZlCrJiK/KfAfEm+Lovl30RqufF40LzhhtzOdRPLtSP8AFPhIGIK1jEOK2cRoqUJihTRqbSoLzqzlQII9ojx98/Y24cYZxxTIp8QXTT1nVT1alp7SJ3yq3Hvs0MOcAMD4YvJq8rtuZCH2vUUta15PEZiYNj1cpcXOjuC4WO1ih0zWgAPscp06p4YSpX6XDNBS1AIcbYSFBW6fA+W1oXwcRdeKnaRSwn0S/HmgByT1xH2m1iWKUNMhuBtFo6e4MXIvHDmL3Hq5Rce9I9GFSpLHWzOYoG+uuulolwFNwneHuSaPx/EHj70a4x0npvDi+EBOrTSXx4ZFpV9lmpwwv6lut91dVUttt1LKFBThCUyD3k+JtK9/3I1fdxVtzuylNZTrYURqRmSRNoAe4KcTKFfVUHEAFpCYShy7kK22gzP7LNUyt4LBfUEnnz9iVPGTI48iFYe7L5oL0bUmkrqZ5SRJS26lRA7yAbQ7ioGk4lXi1smqaZfA5GEhJPvFnBwjwXifDbtTXYoxAm8H3Wwy2hulSyhCZBKjBJJMDuAg6a6JHGLDWMjiCjxHhG7qSt/ivo7yH3lIiFEgiAZ3+FmZK2Smka+wuD/23SMRbMwt5FN3H1QwvH+CKxls53W6xlwjmOqSRr3TmtMWE15rnQ3qQkkCR4zaE7lY4nVt4MU964PomQDBqDVhQbTzgBJJmBpaebgu/wBCoW2Qn1U6+J5m2fHO+eoY4iwa229+d1bdEIonC97m6qU8BhriuqTkTRX+pXcAkPZh7INp34/3f+cOGlY8GgtVG8xUADuzhJ+CjaKeL+D7/c4nVbdz3FXVSrwdbfZW0wS0CUAElewhSdZOm9p3x1dlTevDq+aNbcvu3c4QlOv0gRIA79RbojOzv7ZQd8t/IXWS2NwpTGR1Xn3ftYbuvWgoH0j0a8TUUi9IhyMyNtpCVj2i0w9F+sXd3EajpHHATU0j9MTsTlSFCfPqwfbZp0XDys4lKvSluilW8/QMKrKV4JOVupQoEJKtgVALT7SeVjvByorLu4lYccZZWp1Va224ylJzoCiUrlPKAVE9wtWqWBocGciiwuNwXcwpY6XFE441dFQD2VU76Z/VKVtq+q0Q3q/ha+eHeELsqrxqqW+aVNS4HWEIORAeUiCVaSQEwBJ08psV0nLncrMD017IQctBUw6r9VtxOWf8rL77VAvO4rhv/Di7jvWhJraSpQ9RvAEpLZWkuNnnEAnvMnazytaWxS+BF+h/YqUZIMjPEH2p2cRXsLs4Q4eXNhwXiV0uLWXqtyuKc7rjiUgKGU5QIREADb22v/7LebWNK6ndrsKMU0BP7oqQwBAGVREeyYt6SDQcrc1Qkmqnv/q/cV6R2jAHZfB7f7R/1AsJHP67AVQzMqGnL3WGJ1twsSgpESQYtqrz/NqsbWVtIV3pBsII8D42BpyFMAARqR8dLCwruIs4Ce62bagxt7bdAaQbaJy7WlZRJXMAqmLb2Glt7iba5SBvZwLpiuY1MCyU6OpvgQey+zr5pP3GyuTOwslXolxFVRPoKQkPFDkzOVSTt7QLScNFFpN0YIgQbJ7CX6m+fSlgJpKFSQyAfXe1zqPgkEJHjn7gbKJB5W4ykchvPxsOWMPAvsna+y7xnRIU2FaZS33aSPkWhPGF20y3KCnfqG6hPXl1uiYldRVPITmQ2kSBAjMonsADUgTaV8aXpipdK0xcuEBXJUyD6S/XoYbCtQU6BSv0QdudosxzT4kuO5LxvSku9XpJo0KqepQupJdIALaFAAIaCyk6gAwpZzQAOKxrEmRMEETsrj4dD4rQpmFzbO1USX5Q1N9YwNBjm/mruu+pbPodCysgJabQCsvVIBCQc2YpRkgRmWQLJF8UF14lN/8AoWCbpvhxTdQp69KTI4m7wkpS2Eo1KlqGZwyG5KgkfrWceMLrvG8G2aa/byXQXd1Y6ykZFOKlaiQnKlGdSI3VmSolMgQSIMOYwpnDebzV2cSE0dxUqEOIo6aoLS1uKSVGEkZnClIRqSslUBInRHD1k1RWZmSm1uZFjf6v/ZXgyMD0QnRTXZw2YpmmH7ovFbjaEoWpV53ekqUBBJHVmPKT5m2WblN0f6Ospmqut4nVtNUPoS46yXHFFtZEqTI0MEkT4Wy2Xni/LnyKh3d30PejCax9lwgKMRGqttfqP2WWaK+esgLc7QAkn57h862btSFdbnQqVRqeWnhYamUkAJJUkgHYmRz39vut7A4Byk06p6MVqHRmSoxOp+fnax1t6EgyZMEx8+dmgxVOtKKDMCU6DXSfvsr09e4jQqJSR60wfh8m1dzbbIwSz1dKtRCqZog8ykbW7QGG1AoZbSrlCQNfn6rFGqpJIBGYbRttYdL2X1ee5zakT8bC1CnYFKdPetbTDNT19UyJmWnlJ9xB+ZsoMY0xZTEJYxTfCfAVzojv/SsgZpSAoExvz1++wgUqOzlj1pJ2tISPGxUeGw7hOin4j4+p/Uxde86+vVrWPiTY9T8YOJLIyt4rqT3Z2ml936yTNmSFEg6nfeZ/Z+NuiSQCFGDMg8ye7vtMVMzdnHzUTBEd2jyUiM8c+I7XrXyy8NPXo2v7KQbKDPSEx63o43dLpEHt0yh8UrFouJVIg65hEb23mUCM3MQCN4ixBXVLdnlCNHA71QphpukZioJIqLjulzn2A4g/FZspsdIq8AQmpwlTKJ0lFcpOvdGQ2g9pUKzHQTzG3z88rHwSmAU6xrr862c4rVs2ekMNpneqpvZ6RFIT/GMJvI0B7FaFfWgfIsca6QmG5+luC9kx+qWlf2xaB05smYRpABga+426giOQjXw+e6zjG6tu7vconCKY8lYFnj3gtyOsob4bkfpMNn6nDY6zxs4fOiV3jVs/06Nw/wCqDauAkxBnUEaW2e0II8N97FGPVQ6IZwWn5XVmW+LfDx2B+6NCCeTlO8g/FFjrXETAr+jeK7un+c8Ef60WqwFHUFcDfXmeYtgVlB/nHntYre0M/NoQzgcR2cVbNrF2E3tGMT3Sv+jWtn7bG2ryuypANPeVM7PNDySPgbVDMdnMoaHZQ8vhbOrAEAIUdoIEWIO0T/WYoHAm8nq4oKVCQQeelt5TyHvtTtLq2jKFrb74VEGxlm+b4p4NPe9Y3+rleWmD5A2IO0Q5s96H8hHk/wByt0UmSLayxsNDaqTeMcXNFAaxNfAiNq1yNv6Xz5WPtcScdMxkxVXafrqCtP8AGB1sQdoYubCoHA5OTgrP5TMHW2ik7jytWxri9xBZ0OIesH86mZJ/1LHWuN2PGvWqKB7/AJylGv8AkkRYrcfpjuChHBKgbEKw+U91sA02tArPHzGCRmdu+53AP+RdT/8AMscY6Qd8pI6/DNEsEEAoqFIn3hVjNxqkPP3IZwiqHJTS4k9c2ZmQQPhbspVse60Lq6Qz6nWf95zaoJK4vIiNI/4rvsos9IO6yQKvDFWgk69VUoWB7wmxBi9GfXQ3YZVN3apXiNbaA1tGzHH3CTpAdui+W5P/ABbKh3f8ZY+zxvwE8AXKiuYJP6dIo/6s2K3EqV2zwhmgqR6hT6iLZB/ZZoNcYOHThCf3QFB/n0b6fjkscY4lYCqAAjFVAJ0HWKKP9YCxBWU7tnjzQzSzjdh8k4wJ3FsG4H2WSWcZYPfVDOK7oWZ2Fc3/AHrHWr4uiogsXvROf0KhB+o2IJozs4JcJ7dwUZG2otgTvbG1trEtuJUO9KgbdhJA2+FkXA7FQLSuN4nW2ET99usvh8LbgE7aWa6jYoOJtvXvt1Ea23l7rJKy4IB3GttZRGthCCP221Gu+3jZaqbWhckcrZHeLdFNtwYtFFGijDpD8Jrx40cN3sD3XetNdr7tYxUh+oQpaAGySRCddZtElJ0eOldQ0zNDSdJZLTFO2lppCWFwhCRAA02Ai1qgOUedsA52oT4bBUScV1w61tHEbewrr8J7b4pgtEMPgEbog4uAfFHJYuABIL2m17Dboqt/vB9LaP8A6TfL/iF/dbY4BdLYbdJ3/qF/da0gBt0O+wvkiD6T/tu+KvD8I+Ln+Sp/6tD+oqs/vBdLcn/6Tn/UL+62v3gulr/95wf1C/utaffSLZrzsvkiD6Tvtu+Kf98bFvyVP/VoP1FVkcA+ltP/ANJ3/qF/dbX7wXS1O3ScH9Qv7rWmiNLa3Nl8kU/0nfbd8Uv3x8X/ACVP/Vof1FVr94Lpa/8A3nND/wAgv7rcK6P3SxV63SYSZ76df3WtTAjQ20Ym0Tg9P1d9t3xTj8I+L/kqf+rQ/qKq6ej90sk+r0mUj/oF/dbS+j90sliHOkulQ8adf3WtUR4WzlaHyRT7Xd9p3xU/3xsX/JQf1eH9RVRHR36Vc5v4SSAe/wBHX91hU8AelknRPSZA8mF/da052tmUb7/baJwinGxd9p3xT/vj4ud4oP6vD+oqpu9HnpVuOhbnSSQtY2Uadcj4W7PR/wClhlKD0l0lJ0I6hcfVa04E6AaTbShrpZhhEHV32nfFSH4RsWO8UH9Xh/UVTmOjh0oqUqFP0imW8xk5aZYk+6xSm6MXSToa1dfR8fKNmpdJK3m6NSVqJ3khOtrdQOc21FonCIOrvtO+Kl++Ni/5KD+rw/qKqVb0d+lLeFG7RV/SLaqaZ5JbdZdplKQtJ3BBEEWZTnQm4wsqyo4sXdlmYTTuAA+AiBa8BgJiBYMoE6gWRwqEiwc77TvikPwi4qDfhU/9Xh/UVJbl6DnEZi8bpdvDiPdT1Jddc1VpY9Gc/RWFKy9xOvtM2ursIsIUAbeW1tFMDQ6WNS0MdJm4d9bXuSdvasbH+1OIdo2xMrMobFmyhjGsAzEE6NAGpC4MxbcAjQW3A2FtiQN9bXQFzaLUWhdb/VcMeW31g2M+z4WLMnLWvJI9YBQ8v2k2M687IbJzutRI31tk6722dNbaPKPPazplhnlbXjbJg2ye+0mhRcs0IsQvZJVSLWndCg4PYZse5RPOwD6OtZW3tnSU+8WJluFAnKQgjCoKeYkW5I05CbcUiyukaKpnKAfMW7IJMAT5Widk/NHK3rXMJ1HoxAfazJaJE5Vbg7jae8WiDGtNVVl206KW8lO0SaxmrdbZ6s+mBvcBS1ACe8TJgSLSrdiHK2iqReCAIAeappkIUNioxqrbTYeMSWBjFllqqcOU5myVaCZEfDf4W5Gvwp1XUOnBsDYba73WhTTNjaWHXmoqx3dCcRX7cnpdxM+glCnawl9Db0JKS20t0BSkNEhWjclXZBKUkqLJ4ivXmxdFde9z8MMHorbvpXFUdY88l4UwabV1aw0WBMBIIRMTExqQ5r5xOxKqW4KFd51aSpTzVGpKW2lmVK6x1RCE66RJVr6u9kVNNXvtO1OJXilBUAmioRmCs0Dq1FQlyYIkZBBMjTMBOwKn4fGnOZ9vCwSFW7NkYLBVJRx8u9tCW6vijicvpADpZvd0IKxvl0Ok7anS2WeVTwiacqXV1XHbB9E8pai5TIo7uSlhU6oAQMgCTpCezpppbLc1wcO/Y/8A8o95ev7eaerzCg4sqaBSFTqDOtu2GgEpMTl00A+TY9XNsU7oD7qWlk5QFHLJ7h3m3NMELSCHkeaVgj2fPvt07ZC8XC0uGGnVbbayiMpVprynnPKPx8bDIQRqEyRzI7htHw+dRUNACEKhQgDKI5be+3SWURKCBz5QAOXwn22fMeafRdsOrZhCgNwezIkz4an53scp34y50E6bxr872KJCSQG1BR035ba8uVttZoUEkpn1goROvl8++0NSkldqqMyTOUaAySN+XssOHErSQVKGUCTvr86c7IyXVg5pAB1E6jx2saS64mJJMwZj4ft52ikErar7SSDpmnUx9+/L9uABI7SyRzER5WJofaJhJ19YGdrGEOEApHI+IPw9lolSFkKNUnTNrI39nlbsJGbtEnXQE9/zztwlSCrIpUknXW3UAbFJG+nPTx8vhZrpbrpuCQlKjpy+fn6rKTQ1mTruJ+fkWS0kJXIEHQye6ymyFFAABHONv22C9yMwIUDvAEjfcCO7b5+AhKYgjTaJ137tjbgABIInUjQfX899taD1UgEgASJ9kWDdFXahGgIAAiR5W4zDUZRJB0Enl8/G2ylSBBOp11525WmJjKQkHTcz91kHJrBYDmJAOg5qGk+y2JSQhSAkE+HMG2kJJASVQDvpvbsZMw0zTuBaWZINCxKlRpA7JIO/z+NtgaKATMQBB05/P7bbjLJUdY0OWO7T4W5mQSBG/us2YpWC6lE5lctDPxI+d7ckfrAa67bW6IMgSok676D5i2iTAMkgmZ+eW1ldKyxKdNRMGCAeUfttsAqGgB05jl8xbekQIB5eXz9VtaKSQiIEbcvn57rK6WULMwkEGdO62KzSM2ubWDNsyxlJVqDJNsUkSYT62nKBZXUg1Yopnug6TbgrME9kabCQLdlQKwCRAGw1H7PnewStVhM6bq0GvzpaV1EtWISVHRIBJ1BNuyExBJ00E89LYiQsEEbQJOttBJAyjzEfXNldRyroFKplO4Ag6HvthUEkncidfq+/32DykAnNJ3128PG2ZpIOWTuNbK5UsqECwTB1Ke18efz+GJVOoWdpn5+d7cgiQd401j57/kW5zEJ0ImAd+XdZkrBC5gIyzrpz3+ZtsJQpJCgCZ3KeRsFmSnRYOo0JJ20n587YpRG87xp9/wA/e9ykWgoQdUlOZCACDpAjv1saZvO8GBLF41LI/wCTeWn6jvYgg69kmZt3CcpSdI25/P42cSObsVHhtO4Su3izE9OodTiW9kDu9Pej/WseZ4iY5p5yYrvE84U6Fkf5U2bY1OYCd9t9f2W2oBIylPmdrTFVK3Zx81B1NE7UtCeLPFviC1IOI1KPcunYMf6H22PNcaceNRmrqN7WD1lIkT/kx42YQWdRqCN4FtZokrKT3nwmxBX1I2efNDNFTndg8lJTXHXGyIDlDc7hPL0ZwH/4ngeVjjPH2/kn6e47uX/QU4j6ybRSlUbkzrrA+fG3WdSBqdOcxP7bEGJ1bdnlROHUx9RTCjpAvjR7CrZExKa4j/5Zsep+Pt1q0fw7VI1j6N9K/rAtCOdXZmRyI3k91tFZ1hWkSdPq99pjGKset7lD5KpXeqp9a464UXo7dt7Nd56tsx/p2PM8Z8DufylRWMz+vTk/6pNq7pc1OgI5aRr3W7LqVII5yRPf4WIMcqm72P1KJwamPVWSZ4tYAe0F+5TsQqmeGv8AkxY81xBwS9ojEtEDv21lH+sBar3WHMCCDOxjf6rCIdIGYaggxztMdoKgbtCH8hQcnFWmbxhhJw9jFN1Gdv44399jbN83NUa097Ubv9B9B+21UA8UjMeQ38Ldl4HURAnf7eXfYje0UnNg81A4Czk8q2yFNrEoWlQ/mmbbykCYtUhD6GwS2QmCYgBPz+NjTV93nTCGL0q2o1GR9Yj3GxR2jHOP3qBwE8n+5WujuFtAR3WrBTY2xQySE4jvLs6malZ+2yizxIxk2n6PENSCRPbIV/rA2mO0UR3YUM4FKNnBWOgzbIJ1mbV+p+LGNkGF34XP6TDX15bHW+MuLmwM7lG4T+s0PfpGlpjH6Y7gqJwaoGxCnQjS2iNBFoXa424hTo7Q3ev/ABFj6lfM2OtccK0D6e42FRuUOFI+M2K3HKR3NDOE1I5KWcveLaInmZ5WjFnjcy5HWYf84qZ/sWNtcaLmV/K3RVgzHYUlX1xaQxejPr/ehnDKpvqqQCNNLaIkybMdPGLCitHGa9E97ST/AGrGkcVcFuKg17rf9NhX2TYwxClds8KHc6hm7CnYRpzi2j8xZuJ4jYLdgovtA5asuCP9Gw6MdYQdMpxDRjuzry/XFpCrgOzx5pu7yjdp8kskfstojlO1k1vFuFqgwziO7Fk8k1jZI9k2Nt3ldz+jFfSud2V5J+o2IJY3bEKBjeNwhjAi2trYClWqSFeRm2bH6jYgIPNRsRuizgy1rawfXSUx7zYyI5WKVi+rXTrzR9IEz9f1WNTNnGqYmyyR36W1587YbZM87TAsoE3Wp1m2bCbZJ1tyuRaQGqjey2oiYGo8rcEjYA22TI0520QBuY87FQybohTSOuaP6Dqh7Dr9thTA524IyVrumi0pUPqsInq0qK1tlUxsSNjavM57Y80QzHpeymAL6oSjWUvidlSDy3tGWK63D+Kap+hNe1UBhIp667wvKtKgZOcghRSRERAI3kHSRgy0t9VRnUh1RBBzkk67d0C0J8Yujfd3Eu804nw1iqouG+0qCzVtJUtpQgpGUoUkoVMExmmBoN7eb4p2prKKpDKynLGHfnfoQR71oMhaW5mG5SS/QXTdwFHdtE3Q0SFlDTDY7DKOWw2AOpj32iC/cRY2uq83L8a4aKevBlwUCVJvcVVO0gEqdV1SW0KTyzOdpOqe0E5Sqbbv4bYnwHc8XxiKtxq822taQU01GqcmUIzuFRV2iYMD1p1ypTZCp7nvi+r/AA67hhdzu0aGwuk6ulcTlgEqcdSjVAiBlUnslJCQVRbNqe1rcXk4FG7Kxgvqd/qViKlDBnfuVXerw5gr0p706+HBU9YrrsxoZzz2pioPOedstPTuBOETjq13lh3EbtWpRL7n5lqe25PaOihuZ5DyFstzJxCT6fvVvgs6KCPykb7qeFmF0t5syr9CiEyDpSugbHvVt42p9wmerH3qY+kPFJeQrL1qj/hIMz591rb/AJShwJwHhGmlUrvZ1WhOyWB/e+NqqcHWypqgfcKSXX9471gwO/1dedvXOyDQ7DY83U/eq/aNxFa+3QfcpwYNWkp6qpqErB0CXSJE6ixlNRe8JCrzq5nk8uAO7U/bYelakkJ7QMkADUDvPdqPnkopZSEZlTIGcg6gRzn52t3BhjPqhcs2Z/Ioqi+MSNbX5XGTBCllQjX4R9YsOjEWK0ggXu7mGnbQk7bcrD+ito7IQN9NNZ8vn67CBgJJ0knQCYBI8T86D2DNLCfUHkpipmbs4rTWKsUqMGuQtQI9dpMn4RysabxpihpCpXSrMgwWzofKe/z+q3Ip4ICkknXUq1PyfnnbXoiFRAjkIAMD9htHuFM7dgS79UDZxRxviJf7aIXS0SiNBAUP7XdOsd1jbPE29gUo/NLG8wlwiT7jZH9EQ4cq0g7SQmO7eOVt+hI0BjQ7Hb4ez32Y4VSO3YFIYlUjZyciOKdTJSu4wZTuH4jvOqbDp4t06D9JclRKefWJMGfZ8jlZrGlKUnUnXwmPnabAvULYEgctwraI/CwvkSjd6qkMXqh6yWrz6RmErlrEUt6XLfX0iA6FNNNOJAmIMrBn556O+h4xYXqqVp/0e8kB1AUnOynYiZ0V5WqhxYSKfENOEjLNKkpIHPMoc9R3WlfD9IHbooswBKqZsmdJlI7rVv3PUb3HQ+asjHKpjRspkTxbwmNA5WAkzrTq7/tMWMs8UcGuTF4upI5KpnAffl8/d3Wif82JUAlCCDMzGp179rCC7ARAJOsJ7Mzv+FonsxRnYnzTjtDU8wFLiOIeDn4CL+YQTqQpJTqZHMWMIxvhR4pyYiodJmXkgD3nU7/JtDJuxKDkSiJG08trcfm5uQSkAKEHT7bBPZaDk8oje0c3NoU5M4kw+4nM3fdEqBooVCD9X22NovC7lE5K9gkkbOpJ8Ofh8LQELoSshS0QAefIwedknENxtm7RmQlPaESRpod9PKwH9lmWu2T3KwztE8nVnvVmkuNuJgLSoA7jXW3SYJSAYKdTHd5ey1EzczqGcQtMvONkudYopXlMFA10MjzFuKM4iuRm56heIr2aKusddCa90CVDNqM2w5A7RakezjuT/crIx5vNnvV8OyClROWOQ+vxtmUlZ0AjQcoPu8LUPunEXEltu8VqxxiRCn6f0lpIvZ85AVKgCVwNI28O6xqmxzxco8K1FYniLfyqtbTj4W7VqcKDCoSAudBI087Q/c5MdnBTGPRc2lXogyFZdIGxnT7baySRm18x8Dbz2b6SXG+71UiBxDqXfohKXKWnVnVJBJluRr3HlZ33l0luL7Fa1Q3ZiFC0odaQ8t272VZs247KBy+3ewXdn6kHQhGGOU9tQVdskBMKUY93lFsyjdOaZ0+y1W2OP/FJlEvLulzTdyiWnx1CVjusltdKziv+cV0P7nsNPZqhSUSw+2UtAmVqh48/ZobDfgNW3p5okeN0p6q2/gNYgEEkHv8AusE3lmdSTrMd9q51fSZxe1TdZ+5e61hAzLh9xOYRtzidNdbc4d6WN63rRKqqzh4zTkOZChu9irkDzaHiPZaBwWrb6vvUxi9I7XN7lY/KSneBEHXl3W4AUTrqPWjnaAKDpbUNfeblF+9/XpbYnM83XtrEjkJSmffY3dXS0whXVr131GEcQsuU4KlKSGFoVEbHrAT7rDOF1Y9RT+UqX6anZUiAnTukam2jJAAlOxjkPbaJWukxgRZLbt1X80ZmDStKHwdO322DX0qeEDDxYq7wvNkpVlJVdrigDrp2Jn32g+gqWbsKmK6mds8KXTKiUgkc5HfFtKWuTOgnQfX8+NoroOk/wTvJ8MsYseDqtUhd11Y5a6luLKH8ILg0gZHuIV3NKBmHkuN+ztJHeLDNJUAXLD5IgqoD6481IcOBRCRqdI+d/O3ROgJSTB0jvszaXjJworKYVbPETDxZSSFLVXoSkRvJVA+fZZQo+InD6uSHqLHVwPJVoFN3kyufLtWGYZRu0+SIJYjs4Jwha5EqA7RnWI+dNfC3aDBlWgjeNfd52Sm8SYcq1FFLf12vEGSlFW2oj3H51seaqGHRLT6FnX1VAzr+ywzG/ophzSd0YQoTHIDlHx95+NsBI0GkTzmOdgwsKOZRVAPI8tPn3W3nUlKiPbOvtg7WjlKJcLsqClAjXciPnutoqC+yIgxB2A+6wfWFQEjz0PhOnzvbAtJMKEx2QFcvD4xZWUbC6FCiU59QowRPKLdQCSEg/hv8+dggrMntE67SLaUsAZSozEAcjvaJKmuwYICY1OoHuttJEzqIgSRM2DU5IOswdTER7PM2wuc8uhEifnxs6VkMleVZIUQRsR9dulOKyxlkRAnXnzjyFixWCISFaxuRB7raCgI7Ku0NtyJ+RZJ0ZClSYMGY75OnPnboOkJmRrrO/wA7WKl1JTCHAI5ch8xbanFEknQ7iBrHL67NYJI0pwhRyyZ120HK2i+kqMGO4gmddfssTLyjJBnYiT3iwangYO2mYg/f7NrSDVEmyP8AXk6E6n1ddtd9LBKqR6skayCR5TYmqpSdCQQr3xYF1xKRJImSYmNPD22cN1US4pTaqQVTIGxEzYyiq1AnlGm/fZARUCRJO5idj7fdY2iolMJBJOoPh7/ZZixOHJWFUZUc3PNHP7LbTUmCrProNVER42Sl1JSmSrUGI217vstyqoWBIGo0mY+T8+Jbhp81ks+kmNFmdZMa2xNV2cogQNSf2+dkUVZQndf4bn58OVsFYvKAokQfLWy4aWdLqaoGYMeAn3fPfYQVkJAUc0mBMz+HOzfTeB1KtkncHSwia4JBAJIOgnT5/C0eEnDrpccq1aknU7wTBsXNX2tzKY20n5iyWusBRKzqNNpsXVXJCiqY9xB01FnDLJFycjdaNisZT47ff+yynR1YLa4KTO/lZoU1bmGmbQ6idbKNBUrOdJVoU+wi0C2yW6XTVk6GNp15bTYJTzBOrbapP6oPn9Vk01B0JUrxBGhsEuuSgZwc0iVQdx8iz5rBLKClZNQ02c7SEoIn1TBn2c7afvuraUGk3leIWo9hqmqnUqVt+qoWRqSpq70UfQ0hLQJmoIlMfzf1j8Pqs47puemp9hndVot1Wq1wdJP2WYSSONmlMWxtF3BOHA7OIq2+ma+8r/vNTLIXlpF1rjrSioQFLCiZIkxGnnaa0qzJB7wCLRth1k07YUBEQR36WkKgWVUreslIKfcYn4W7bConRwgONyVyeIvEktwjB1toq087czpvbRIOu9tcNFlnEror7+VtFU67W5nxtonlZwLKJN11tbJiDpodrczrG/lbCeZs5aHCxTJkVd0YyuS9i9c14enXe6pSm6Z0p6xmTORM+smZjUQD3AAla7Ht+XG+Ka+sN7JBU6jrUAJJjQZFAmSn9KN9ZgF8V0lDbiT6i0k8tDobbzGCOW8Ha3DzdmMQpJXSYXWOYD6rvSaPZfZXW1UbgBKy/jso4Xx8wjSvKZqKGpQAkhTiYcVmnQBOgjLueW2u9ll/j/wrvNFG2zfKmHnm+ygtEicubLKJSVaEQkkzZ1O4Zw5eNO7UX1clBUId9dK6ZBU4RtJI8T7zYpW8MsBVtxF1jDFMyhys9LW20cg67tpCiUxMBxRjvjutiysxyOOTjlkgbcF1vdsjNdBnblBCY7nEXh+KhdKjF1LXuPtKcFKwoKHViAorUvsjKDsIjczvYxSYQurECHr0pbpaavB1QZWgOFpvIgwkEwUqCZVBynVRiJmyFiPgdw3buzraypr7tFI2haqpuqypbDe6+0SE6TJ0jlA0sxr5wtjK5cNXLVcLb/v9NI2uqadbqEuNPXl16wlrInIcoypVC8yCJLkpnNbyjEKXvMzXtbkcdDbQe5bGdzWFOK8bkwrQ3hVUR4oU1Aad5bRpEVlIhNPlUR1YSWSUhMRBOkWyxFnhRxAUyhSbkwTTgpBDL1DTPON6eqpzq+2obFXM62y2f3Cb6R9yXEcqp/lJqlLdw4HYJylVdWqAMcmmfvtW3hHTy1dvViQasHTYCYIPsVPl7LWA/KXvuIYwCwVaKdvJZ20hNMBodefI+FoJ4OoHU3XmOdKnUqgn1e2PmbfSPZNpbhsX1/es/tDrWyewfcp4p6dIhSxJKjoOR077HW2wSF6HMQFQInutlM2pwDMgEkHNInT75mxxLYKc5B08Offp+zwt3C5QIvkVmUQNInSI+fu9wiWmwYykgb7H55+6w5ayGVDTQQCd7dhJkZkylUc47hubJOuQiTBkwoGNzOvL52t0W5AQFGSMx8pifn422lBWkLA0KdRz9lhUjKidwZME7ec2Q0SIRco7UIEQSJ7ye7xthbA7MyB7BP4ffYzkzKM6bme/50sI2ghZkEGYMjkOZtIOKjlCK5Sk5pJI85jc/PnYBbWUZiQjmVctPrseU2FEJUkFMzvt4eHOwLiBkCVCB6p131Ph5fI0IEJwUF8ZW1DEdEQk60gzJOxOdWx25/s5y5httP5loCUlU0rJKgP5qZ/baK+NCSjEFCTBCqT9IkbLUSTG/L3TaW8Kpz3LdiwpWX0RnXNqRkHvtBnzipP+YEsttEJKS2J3j6/nSwymlEZE5tfeJj58bY2jOADof0jtPv8AZ5WNNIMHQeHMfO1ioYCLFodokmTrEDT7Dp9duiwYCYSNo8Pw++xvqx62ka89J+rl387b6lKx6hA8PnutG6nZFCylJPYGYGTIiSfn6rJmIGCbuJARotMzHfufd9dl0NlcKRyG2h5ffZPxAlf5uc1SDmQQPaNbRcdFNu6hm+ID94UjIAVUvNZh3JSiSfHYD28rC3tRJq7puxlCfpHQhCSo7BSdT7vibKVVSIdva83tw1T5ADzKk6/AfHwsmt9bWVV3Kg9XTKZZMfrFMmNNToB77UM3JWLLm8WmLvvB3rEJy+hdWkct4Gnn9vKxahpkLw0lOTQNqBB9o+42Vr3pGqy/GKdUZUILiwDGbKJA8dSCR5WRk1TlLcLdLTJl94qSkAggDMZPfEWm19imsoFvekeYrlPKyp6911wp3DQz5Up3k6J56WlGiuZijuu6Wm4KlOIdcWUwVqIn2fMWjrEThaqG0OiU/SQCQNc5KZ5xr38z52lVhJVdt0q3P0J7/wBGdNNj9nugbJwNE7qgNUdKupdKMjaYk8z3fCyNhu701FZW1zie2uEgkjaTMD2fO1lGtzXveSLsac+hQQ4+oD1gI07+63VwshqvvBLaIHWQoJOgidPC0i65CVrBC1t3shlaQlJUAYPMiNfq99mtRMJp7jWy2gda86oNwnXYa68xv7fZZ+Ps6EOJARqJPftZGue7kVdT6VADFNDbKDBM/rH5+q0HHVSaNLopdmHG7vo0oDaOtUQtxQT+l+G3yDZKu66kJxLVKDY7QV2ADA202s+XWgDnI9bXaIjn9VkFhnJiCoU6YzJUdTpGUbfH5FmcQLJgCUUvWjFJTKAZCVnRIVy8fAAeFmfUXE071kntkZySYzHX7/jZ53i4qodW842CkHKmfZ3+XzFizN3Pei1NW4tScjZy6+sfuH22C92corW5Rqki4MLUzFKKxOQOvySRqAO4Trv32YPEmhDbzggzlT3e0QNeXx3FpkuxKTdbEEnRQ3iO1+O1ow4rNhLjjh0SGASZGuhAnnyMi1kAZLIJvdFME3ei8sFuqWcyQt1JhIJI1P4eznZx3TcyaK5WmUsjOucqdM2vd93PexHhI6hWG6sPNhYRVKlIIOmVJCfPX4iz8uKnTWvqrXcqg3AQnkkn2d3jy8bDIBUwSBdIl34KapmnHXG0l92C4qNhyA9/zAsYOHurWClJSdwdR2hOvh8Puea2kkynNlnQxEdx+y3BZAOqOUxBlPtFphjeihnd1Ue4iuy8E3ehTNfVtwrMCh9SZGuuhAG9jVzOYqZu6m6nEt7oUhvLmFc6lehImQrz+d3LflKHLvXlRoiFSfu9se3zsDdNOk3clJJhJVlHPeZn3/OlhcKMu1CmJX20JRJm/uILZPV42v8AB3AF4PaEd0qju+FjCce8VKYdnG17LUmQC5UFZPfor9ulj6aNKBJQoZRzgz3D6rJt4BIKaenSVOLJEBPLu8hp7rRfTwn1R5KYqJhs4+aBreNHGthfo13Y5qS7mjt0zC9CN5U2dPrjkLKGKeLvSVwfwwf4irv1p6harGqMP1N1MhDripJSmEDQAaknfTUzC9w6wdhaqvRbmLr6aoaZlHWqzDtvriciSAQB3k8vgL0p8UUOJuG1ZcFFV0r9OyylbDNIohinS2ISnLonNGm2lqMlLDxQxsY8dFaZVTBhcXnzUXXN00+Nj1DSVFQbgdLxhal3eofpRPZcH1cjpaa7m4/cQ6ulpqiqoLmUt5tLpCWXEjUeKzHdajdyr6q6aRZKQtl5RGklPa8PP6rW5wo009ctCqEkdUADHdpz+ftFPQU+bRgVqCuntq4qQk8d8WZYduC7HIEnKtxEjnuTZKvzpSX5cFQy2vh+xUodQVZ03ipvKRpEFs/frtYk3RtlGZMyoRmGp9/tHw9jXxpc7TvozpQP0khQjUnKffvascNpyfmqz3+e1w5OhjpjLj+M8M6hJ3IbvZKtCd9WR8z3WNI6ZGHf/ruBL+aVE9l5lY2HeRr4WiB252SgkMgZQDHhsBHtHdvYBVxMKSW0tp7I1JOoPL26j3D2L5Kpzy96j8p1A5qbkdMPASkjrsNYmayzP0DB7+56xtrpecKFrQmqRfVMSDJXQEx/kqMjutXt7DtMAlwMgKzQBlgCO4DnFir2HKZYKSNdjBIPn9Zs3yTB4p/lScdFZxHSq4MFMrxPVs6AJDl11I8dwg+491uz0oOBzsdZxBpWCTH01M+2RoN8yBaqFRhencHfMJ7u+JJnx+TZo42wkGsNV1a0oyhtLglKYhKhMabxvEa+20m4LATuUxxidvIK9NL0hOC9RlU1xMuFIXoM1YlGu59bzFl2l4w8MKtKU0/ELDi5GkXmxJ/0pB+63lGEg3b65BafIAgScw9/6PlaS7mw8xVUDFVmH0jKVlKhIAIkzr3CedmmwOJguHFPHjUh3aF6UMYzwvWpilxJdT0iU5K1pU90dqxxN7UTgJp6xt0aiULBnfkDbzeRg9p1tY6tuFK3UNZjl8LFqjCDaUEMpCNQDoRrPh4ft52q/I7fpe5H+WT9FelqKhS1dlJ/W2NuV1BSE5ioHkZiPn53t5nuXXfNOgooLxq6dwAlPV1K0kHYCQRH7LZdNZxTcqEM3TjXEjOyQUXnUIAHnm05+FmGC3Hz/cpDGBtk969LhWKJ1VOoEEz+2wgqzmPbOg3NvOeqxjxaw8ipSrijil55umLvavd1QQcoIACla65tY5W9C/yd14U/FngleFdxASL8ve7cQP0aqusGZ7qzTsOISToYClrj22o19CaGHjXuLq1T4i2Z+TLYo+ahMFKdDrIHd3a2LmqIUcp3GxO/ltayznCrALqwVYcaBGnZdcGkeCrEqngtw8dSf9xXEazKKp0f2rYffWHcK9xAVX6mq0qPraR5n52so014JSuM0yCOyeXO0uq4EYHzlTa70aJEQmqmB4ZkmyXe3Aa5xQVCrpxFfDFQG1lkrUytAXBiR1YJG2gI08dbCfWxlFY4XUavXm4XU07JW46rRCEHMSfCP2a+4/Q3E9VKRU3osEo1TTJP0Y/pfrHwiPOw103WxdbQQElbuhccXqtZ5yR48rK7Mq9UkgHv3sUAv2RiQ3dHKVpICUhIEAaDl4eH4WXbsZJWEyNNYjayXSNdvXXwn50/Cziuxo5gI+fs5W16Gku65WbVT2Ccl3JDbSRms8rrdKqQTyj4gGzPpjKRAgHezluJ3M0pMbAAeyfvFuwpm5RZcvO65JSoYHvtqeU6b2xWu9tTy2i1xVFubZMb99uSZ37ramCRFkkuiRFtkwIm3OYDf2W0SOR2sRouoOKDqgVU6wD+jP225bV1gSQfWiwiwFAid50sVo1zTo1hSZBM7EGzSNuNEmmyUq+oKECjbVlSkArG0+FlC4FdfdlVTkAlBzDnuPwsgLJJJJ1O5ssYUdiqeZ3K2iY7yCPsNsttCynpTFvzPj1RDIS66j/iNdVRUqTUhCH6dlrOKVQIQpe5UvkoARCSIBkmSElMYXneuJr2VUXgL0dSzSPguqUTlQghRypOiQrMWzlEkjWNJtNWObsfr6J2ncqV0dOApKi1o6dDpI1SCY1Gu9o7uK77tve5K3DTqm6NCaynSnqjBUVrhIjaJEaAT3zJt5RiBMMksnC9Frm7jW19bBdTG/NE1ybrd64qcbS41K0KSFJV173aB2P8mfrtlnK9cvFJTq1UVRTsU5USy03WpQltE9lISdUgCBB2tlod/pfot/b6lDO7r71Qb8pW8pV5YEYSBmQ1eRI5jMaYD58LRBwiRkp7mkQC6IgaEBYHz5m0kflGLyTeN84BcCwQ7dtVUZSdFZ1NSPAwN/vtH/BRJTS3KYDiEiUEjSJB0HtPtJPjb0jsuLYdDfx+8qjj5vWS/tyU7NAJGxSUgHfVOsfJsoISgwd9ipMTpzsDTNJjUiAJnb7bGmW0pUHExI1PZ1j5m3aFcsxddWcsSTAMA7T52wN5TIGbl3ef2WEhQGihG0Ee32222FFEzonck6+P1H53ZT2QaEIBG5kx4jTT9lhEtJIkIkq0Hz7vwFusvaAJ21G2h+QbdBBCSAQNIGnPun52s3NPZcKQnPAUowqSOcd+1uglTgSoqjMCJA2GmvvsIGiSUo31jv8Ab7rdJaIVmBIjWY23nnaTd1A+CDU3ngqEGdO4ecQTy3+Ni7gIAKz2V76T32OlsQQcp0B00A19vhYu+2CCUnSRz1Pj4n50sQFCdooM41ICb/oVHL/wQxzg51a+zv8AC0vYTQBcF1klIJpGQSka6ITz9/zvD3HJ5TGIKBSidaQxrMdtU7ju+Z1tM2DU9bhm6lbTQMkjXfINPq31tBpu4p3fMCWkgyUicsyBHu1+ftsYCSRusGRy1nfWfnnYJDSkQnNtr2TzkffYyhHZkjMUjQHSJ5e+PutO6Qb0WDMnNJGsa/fbZlSZIPu5d08v2+YFyiFKBSrcpgfCO/50tstFaZRokiZ+e75i0MynkXBbbdnKga+2NxFky/0JNzuGdQUwTz1B+zeysQo/RhGsxEny8pgmyfiJrPdRUUFMLQoQQB62/wBVoPPo3Uha9lF7pbaqr3cchKUIRMnmUbd/hYqpluguq7HFFKQt8PLOpjsk8t401t1eqKh6tr6OTlWQ+4QmYQhEhPviw95ssv0V00IVlU/1YIgk5con3gx7bZ4N1bLbJKo0OPXkqrqMx9KZcdyieyg+qPKBYvddChFG7eJbkraWhnT1UDUnwPP7ua1ebrdHerinFICWqIJAG4JMCOesbWKUqEIwupOQN/QKJJ5b9/zr7bSadVE7Kv8AjRo0z9MICiorKgBJIJBHjuLSbdqUfuXuyocAKWmmiMx1kJ2+r6/KMse0tSzejNS8VdU8tIYSZ/kwkHMdP0iTZ9YbXU3hh9kjMmnoadtswYlzQcu77PGzvOmii1SPh+hVT0pffUTUVP0iiQQO+IPPUfC2rnQRfN4NyTOUme79tlC7FKcoqd6FdptJgjwmPrsnmoRdt81jjkpHUhWukns6eFoOfYBSAJK7vZ12ofbuymELWfpVJToEzz8bcYbQtLVS0Z7LsgRvobHLjo1KQu8Klv6apJ0PIb6WCw+oh+sbSCAFgmfMwdfnTztDMbgolrCyPlor7KiACBpBMaj5j77N29uop6p1DZKnHIzmIypAGnvE+7lZw19WKSnLhBLitAJ2+e+zUdWVuq7MuOeyT3WhLJyTxs5oJqkdrKhLSUwZIJCdBG+mll6spG6W7FtgiEtkeY9/z7bD3TdvolPnWgdesBR5EeH1W6vQoNE8sEphByab92nzz9jsFhcpnalI90Nn80IStMkFUSfHu949to44qsBbSipOfPTkieZk8ueh+NpKuoD82ytZIBWDO++3xtF3FqtLVbQUTPaVUZkqJEdgamfAA6/JtZD/AEUEtuUn8IUvVLddStyhlt0OrIAjVEADfU5fcLShhBR6p/sEqlIMmCZB25cht3jbW0bcEXgp++KYO6lDK99J7YjbTUm0l4UIVVVCTmAgEEkDYxH1e+0Bo4JyLhONLZCipSSeY1327/vttSQ4dBrAB52HSjSAQAYEQIV7Pb8LaWCmVBAJMjXcd+9iF1kOyS7wZKqZ8ETKcwiNIsSuBr6FwApHbKjOhggR9XuNluobCmFiNVApE6wdu7XayXRNquymKnUqDjhzBI9ZPcNOfz4AD32KMxuiyrdDSSloFSlmEhJk+Ea6zr5W5YoENEOvLC3l6a6AbyB8ffY1S0SwTV1KT1xTqDplBj5+TYdaEqOYpCiDOh31H4+evtk031KZwI0CIvNndMaRqQBpy+TZuYzo/ScM3rTgmF0rgEbeoYHfzs7FIhtOYyE7Roe/u87JF7s+kUVQyGyUrbUk7cxqY9vfYwddDVUbgSF3MspGrdUVerMAoH3bc7W44bPIqsK3c/CSktAgKM5SRP1HffytUW5MwZvCmdBltxJjnAzAGTz9mk+y1qOCtUmqwVRgFJ6sBIgnu0Bnu15WoT6FX6c3UiISAkBQBBBJM6kSd45bftsjYupwaJtaYKkvSrLtMHb5G1lpJOkExMBMxz56e2yVihhxd1rI16taVQZknWft087VwdbqydrJmLaBXnUAUiCsgg8vhysGppJVKEhKssxM92x7tbGiVQViNe5Q93z9tgSFECVAqXqCZB08R5/H22ndCylFXmyrsuRAgE7A7zv4T8zYu5TgJEdrMomY5xrOvd9VjqxyEKSkSDM/A89PmbAuAxBAM6kmVH6++de/nytIOTEEJKdagqAhMTGUTJ8N/q+qyHiGhVVYfvSmSYL1G8EgCAeyqBPnHydXQ6DokpIkQYGhPgB36WIKabWvIpIyuHLEbgiD9v3xYrHoTxdVcQAq76tsH1VtuDuAkiZ/xhv9tpewGfSMN0DykFKkthseQVlke6fbaJ6ZstIvCkICfojueaVA93gbSdwvdUu4W21EAsvuN5SuDGhiOWqvjYkx9FDjGqejdMFNwptMxBAJkDbltPt0FsqaVJHZgGQoZu77vP7bHqJhdQoIQ3JG4mRz0s5aC4A0AurRKwQogjWY308rUTJZWmx5k1rvwxUV7wVUoKG9IURqdAdf2xvtZeRdTFEz1TDSE7kRzk76ePt3s4AylKMqNIj1dx8x7LFahgpBUACmJKgNTHmPL3WjnuUUMy7KLccUYTVklJIfp4J7jBT5ToLW7/JLX6XLi4iYbcc7dPVXbeKElWvbQ82sxy/k2/eLVWxzT5W6Z1KdFpdQDEweydPOfnW01fkrr5Vd/HDFeGnF5W7zuF8oRJhS2KlojuBOVa9d7UsXj4uHyD60SmdkqGr1KUrMErHO3KxuY2+FuUE5MhElJt2dgT3W85zXC6C1kXkkx4W4c7SSDz77YrsqPibcrJ79edqr3I4UQYnu1VJfNQ0AQCrrEkc0q1924sTYaJVzBA2iz1xxd5Uumrgn9ZpX1p+2zXQwEqBB9YhWvfbpcM/GxNKU77I5Ro2UoEWcF2oSkTlskUTQJEpkaaju52XqNASkDbQGLdZSRWWDVS3SswI0ny1suXI4UuFECCD7zr9hshs7iRO1lO7XCiqSSI1HxMfUbbLBZZTzcWTizcvjFtA6Ttbnbe2TPIb91rFkBdmAbaJ0525KuUe22s2kgju2tMC6i46LeYz3Rbc6QAdbcE66fA23O3zFiAW2Q1vNGvOxRk5VvNq0IWSNOR1sbMgfCxUgpql9ykBWlnc3RRB1W1SNdYsfuJ4M3swqfWJR7xFiCtoB8rbYdLLzbupKFhQHfBmwZW3aVIb2Sli1nrE1TAIzEBQHmPvtWu/LseqsQmnW6pluczi06qSgb5QPWVySOZi1ocUtMNBd6OuJba9HOdRMAJGsn32rnflNWVd8PV9bRej0ueaancRK3kwIW8lQgCZIb5/pRBQrgcQeYqos5uGn3LpqFwfT26IHrMRjSiwNTrp/8CpdCp1SkfokrntmI7XPe2WAdQ+46txddUKUpRJJWSSbZaj8nu6DzKNlcqD/AJRYGkxphGgaWstMXU/2QnUA1CgnUchlHwsyuF1+MXTd1yPLoHnQzTNlWWO0ZPMxvHws6vyj7yFcVsOsqmUXGDIMb1Tw3jwPzoWRgVtCLgu7MpK1+jwVAbwpQ0A8vD2W3sHlMGFwuG9v0oGKxiXEJQVNdPxNuwIzLuq8UR2vUQfOO1ZQb4lXCtKZo7wEawGQdd/1vG0aMkK1TprlB0+e73WPt67k5SkKJOvn7N/YbaPyvUeCoDDYTsFJKeIuG1EZk1if0ZNOdtddPnawox7hnNnFY+haeZZXEzt4/jaNVEpkZuUkbew/Pfbppz9Er1JmIkfhpZDGJ+gSOGxDqpMGOMLrSALzIUdszbidOfLz77Dt4wwyIUb2QkzBlCk6e0eX32jJBARmQIUNtSJP37aWwZSoyEkjXWII7vaNrTGMTdAm+TYwNypXTinDa9fz3TIg6ZlR7/nnNh1Ykw8shIvygM/o9cNfZOg2tEYypEZJABGndy0G9uFpQVZSgJnQaHTWdx5/CxBjUg3aEM4azkVMCr7uYgpF7UJMSFekJjQ8pNgXL0u5SShNfSrkf8ckmOZ+u0OqCVArJTpuQkeMnTxHfZOqG0IGbKmBpqkSBsPb+Fjtxl3NqE/DGn1kS6Ql4UrOJLtcbqAoCkWCUdpJOc6GN/wnladcB1DKsJ3O6h5uVUFPoCDoUJ3tTziZUN0t60sJyksEwBBOpn4c/wBtpRwShr8wXapLWX+Ktk+OZI3Pfv5xYr8UMTRJl3QW0Ak9AHZWYbamCkkjeM3zP42MpaWRGYmIJ3133tBbSyJKHFpIJEhagN9R862OsuViFQK6rBA0yvLEbfYRYfy407s96M3Cj9JTalKwAoI1BkknfW3CkBYSlAglXaOsjx+y0PNVl6BQIveviNIqFiDvzPnYy3fN9pUSL4r4k+s+dDM6TzshjTObSkcMePWUuhtwgBRJ5nx2NiGIWSLtckRlUgwdJ1tHbWIb/wBxfVbM6y5IHvHz77dO37fTzBbfvd9aSQIWEiDO8RvNk7GYiLZSkMLkve4RN9gOVF+VC0ytDaUAgagBEx46/V706mbXVXhQVyyQ224lhpI7koOYxy1+e9QLlWpTx9MIXUHK52R2o0199uEekoDCG6kFNN2kAoTA+Z3PutV7+0o5o3dUWvijTWXy6k9pFOx1rgnXQGBA5E6+yxG8HBRYSDLsy9TqSk5hqrU/j7LKaTUdc9U9agu1CShwlAhSdvZysk3pSLWwhhak9W0kto7HgPHu52m2ubdDNG4Cyh3ioz1T11KSSDlEKEnTINu86c4G3lZ8YTpWlYNYDaEguskqjfNJnz208rR/xgqqlu86Rp5xnqkNJKEhMEEaTvqIn32efD2sqnsNUiEKbDRStABSZHaP4aeNrD6poaHILac5sqkPDTiFXRSOGDCRPa2gnX4fGwNalFVfLVCDlCh9IomM2XkfnnYG6XqqgpE0jKWihobGZg7DfXn7xYSlo1U9W1WpUFLQorOYkBUnl8bBdWMNkZtK4apwlE5UpHa2Ty0iAI7vbZGuhaGr3r0rVAHa8QJn26Hf67GzeFRCc1K32gJIXH2WILbUpyofbbKV1AGzk6eXiYtF9Yw7JNpnc0TvWuVUudbmAbGiRpuI1Hzy8LBXalsXlTkpmQSRyPrR8RYZd2OuOBakDLPaAOseHcbGHGwK5mrZYyoaTlygAk+0mwW1DXOuVPgkCwS4hIzCAUjtaCDP3WI3mpz0KoCdwhRITryP42EcvRvKUBl4xrpBnw0+dfeXermVoW0WHRmKhIT8dD3WtmpYRugcB10hsVwpLnMgJcKylAP6Xj3xIsyMaXS4xd5r6pJdfcQ5EicgIGg+fqs+W26cVIqKptamW0yygJkCOZ+eYs1uJV9XexdiVOdYCHMmrSiCMp09wP47WnBOwmxKhJC7kEyOBr039eTedULpkkjlIWPZ+kOfIWlXC5AveqYSpUAGQobdoT56RaFODla1R44qEuGErYdRojUnMlXlyO3M6bWmi5nWmL3cqDPVrCyFAEASZGmp5b+FiPka0i6i2IkJ6AKTA3MyUawR+y3OYgBOYka6kxA9nzHdYBFfSKhaXNJIIyER4H3j5NuTW0aEiHQSkRoDr8/PKyM7eqiIHdEK84GgolyCTlE6kTvpZHu130m8VqfJKkJJAjbUAn577GmjT1T6n6qpSlLRhCRoZjfeDysSu4IZvjO4gpaJUnOtWhjXSwDKCUYRkBSBhbB1VfyF1lQ041QtmCtCCetXp2UkH3n8LPZvBlK2kVNTTDOYLbCqZQBT37GB88tDdz1+HWaKmqF1lAG20hDDAW32lbdqFgx57zZr4zx4xVpdu65KlAz6PVAOUnvbbgns6b+wRalxpJpMoNgrAjaxtym/jB65qaoXd91NU7oBh2oQ3CQZ2RpOnf8AJZFW0HWlNOJURGxMc9/L58lV0IdRlORJB0IiB3/PjYi+0mCFws6gAGPnztrxFrBa6z3gk3sqjinFFiC/qFaTKHFJSQn1YcjXTuP3WsPwDqVO4XUnrMxaeMCZmSrTz/HztA+LmGqHiPfqVEALW6rxMhKtt+R84Fpf6OVWHKCupgrVtzUTMABO2mmqiPZYE+uoVmn0NlOQ9fsmSO0dORH1xYnfiVqul+FHspCgZ0ABB9uth2lBGyhAEDkCOYA25W4vJC1Xc+3kzDq1JATz0Oo+u1W/VWyExIUG1EggpG+vdHP26d0W0qZ7U5USBpz11+T3WwZlAZz2e7bXv1M8vhbTioAUAMpkExsZg+38bLMVDVBuJUQN4k6Ty029h5eNi7hBkJQpSY08Z19v7LDOGQFKiYyiQSIGvt52DWAMxWqfH1QTr3+Xzye9kxF0TdSCjNmMk/paxA+4fWbE1kpJVm0kyEmI8vh3b6WPrQQgoymdB2dD7NNfCyc5BUVNkoIiNdZG45e7y7osRjlAsVd7zoRS4vvaigJ+nqG05ieZVB28j+FnpwaoHqukrqVClDq3UrJk6ZkxH+iLEca4ddYxtedcqEtpcbqEJIjrApCSr6zPxsucEHQxe16URUkKcaQ4AddEkif9Lz2sSaUFhsoRM9MAqZLqu9mkR9AgZzqVDTbXaymnIhEKIA3zQDP2d1izAQk6JkDQTPtA1+fhY03mSoE+sBJAMADlEba/bbLLyVpNGXZdAHKAUiQQAQNdvwsVq2kKQSViTIVqcux+2ef22PZ28pbV6o2JVvt7bAPrzy3mmBGnL5087TY5M4XCj3HbR9AS6jMMrva8CpKtRt9Vl/oIXuMNdMDDiCvK3eXpNGvlIdpHMs+HWJSAO8bCyfi9rrLofSmQQAoDNBlJAnTwPIGzS4PXyMMdIzh1fanUtJbv67FOL2hHpKUrHnlVBixJm8WmkZ1BVcejI0+K9wnAEPHQQoW6g5ZBtlWICVJ/R0toKzIzDvt5aTYkLpBq0FFnCM2k++3Czp4m3bx7WYHysC4tKUZlKypAkk8rVHHWysNSbf8ARmtuuoQkSpsBxPmnX6psx22wTECN4jT2We1wYouHEtZeFDc9cmr9ByB5SEkt9rNolWyvVMxIs2KqiNFWO0cfyaikT+qNvhbqMAJLSwoNWS0WK1SMgBKcsD67LFKAI2P2WJU7ZInn4yPGx+nEGT+Pz91u4p2rnKg6o+1GgB2scplZHQRyBI91iLYJ1Osd2k2ONKAcQO4idLaABVIpzzIm2A67+NgadZNO2SZOUA+JGn2W7zaTaw0XVcld5ttLcz5e62p0k2yRJ1MWIBZRW7bnSBbQPO2Sdd9LEaFBxXW5Bk62L1Bh9k8lZkn3afbYfNIixavOVtLmsIcSSR46fbZ3bKI3Xa5jUA24UTyt2vaIsEqPwsIi4U721TvfDdZc9DULQFFCRqRMEDKT9do0xBdbD18oQ7SsPqdUlCA84UgEneB63L6vKRbkdD1wFopJ6pwpgHvIP22Y+PaYolbYOmhIFuG7RQBzhY2O11v4Q7NdhXCsX4QplGmTctA4lo5AsIgKA0n1LZaP0U9aEgClKhGhlWvxtluZ+Rm/lj7lf7szr968yPyi1Sl7jLdbBOjdwMgJO3/CKgz7vLezVwVet1U9w3ewqvpUKTTIzhTyQROskTp8762cH5QGrnjq03nUjJcNOhWU65VOPEj3HW1cnVrbq3ilQKUJTKtSYCeR9h5R327PDIBLhsLdvRCq4i/JXSu8VZmnvK6VhJXelICdY9ITry+3lzso095XcoS3XsKTtAcSY9x0P3m1VkrKSG8w7iknnMnnA2+qwgfcVP0hSU7Jjceffy07hYxofFVxV25K1qXWAQApEqERmB/YPu1mw7TrZVAKdIJ10+HntFqm+nvh3My6pWTQBJIG+2wmwwvSt9ZFbVNZv1XlATEe8nb8LR7geqc1d9bK2SBtmRmA2nX3iI77dE9tJI0J3nnIE+3Tbv8AfVH8/X0yjIi9ayd0FL65B10mfj8kdGKL+bbV1d8XgExmj0lYEg9wPf8Ab52QonDml3sHkrUZ0ntKHtHcfD22CXqBsVLmYHPQ7D2fhar6cZ4padStGILyTAlIFWoyff5+0RGuopx7ixtMJxHeLgGhJfXI03k8pAPyZXc39U3em9FZJxzQ6aCT2dJ2+GliNUoqbKW0knYJjbXl3DxtAH7vsYhaT+6GsIhI0WPZAO+oj2m2OcRcXoWSq/qhRGkqSkmee6ZjS020rgomoaUq8WlH87UZmVBk9lUkSF8/Hz++0t4CIVhy7DClTStSDoQconSPnytXK+7+rr8W07elV19Q2MhUUgACZiAI8dfqs5br4lYnuigYpaGpR1LLYS2lTSVZUAd5Gw07zysWeBzo2tCHFIGOJVlKcpJIMgjcjWPs2EcvZY+zABRl7J1SM0zp4+dq3scacaN5j+cKReVIgmmSNSNDpuOdhUcdsbMy8tVAQmR22DHl60+7vtU7pIrTalgVkIBT1eWTrJgHb6/nusKoQSpMHQ6KTGWNZnu0PvtXRHHvGTYSHmLqMjUqYWNOYHb1HePZ4WGR0hsWNjK7d90rIAEZXNRMT6/j9fjEe6SqXeWFWGQoIUkJSSTyKQJHd4fO1hi4AAmMwPhHttX5vpE4jcAcNz3Yon1gAsSf8ru+e8f+ERe2SVYdpFGJhDy9d/Azz+TYfdpb7JxUs6qeFrSgBBUd8sEfJ7zbkuBSEklUjtEzp8/j7YNPSNrtOswxTqEiCmqV7hKPt9thU9IgkqW5hhsZSEiKs6nePV8Prm0u7Sjkk6eM81NRdKgCAVAdrTu+3U2IVSyQQlMyYkk8999uVolb6RdPkB/cusGf/G5IOw/Qncx77dOcfLpfht/D1WnOJBDiVQQR4Df2WkIJRyUOMzqkTjU+pm8KR5YzAtgmDrIKtvH46bWcnCt/PhOiJzjVYnb9NXnaM+ImN6PFq2amjo3EFpIQouK3Gs7acvr8bLWAuIN33JdCKGtbeUttS/UQCIzTuTPdva1IxxhA5oDHt4pKnaidShBbAmDMnfy0seCh68HygAc5Hn5d/jaMabizhtJyPmqBO/0cx7jzIjv18rKTHFvDCx26t5JMylTJkae36/wpFjuitZm9U/itOYhCFCSQfE9/w+Nth+RKdIkSNNe6zCRxYwk6lOavcSFSJLStORjTSwh4p4TcgpvIjb1m1d2h29x8rRyO6Js7eqfCnk6ZlJKRoQTtbjrEwSkjLMmIJPd4WZjfE/CiiJvVO+pU2sAeE+/67CHiLhWE571ZRAmSVAnX49/tssruie7eqd4cQSc2VI75kKM/ttwtcnKZJ2MHb5+/lpZqfvhYVUlSfzywMp1BBTMfPL8baXj/AAmqP926YRH6R5jn5x4crOA5LM1OGoWCiZIERA33nlZicR0ddcC4ROR1CtOY1095HnZXdxvhdZUoX1RkzsXQNdNI5ab/ALYbGMsQ3TV3PVNMXjTurQQcqXEyNY2nzsWLMHIT7EKOeGj6WceMgIcAdS60CNgchJG3haw9C43KEns85VsbVnwPUljHlC87UJDZdVClqgTkI928ePttYm761pRADjeihACwRHjrra1VEhwt0Qae2VOdDmmdaVaQQCdTp3+U6+It0pMg5+yTEERO/wC34WJtPgo7KkrCtyVZjE8u/wDGwhXn7BSMxkhJJkE6fd7rAzFHsENmAMCTKdPuj58bBkISD2Eqy6zAI238LcBzPEiQCAB3bEjw5W5WVQOzJiSJmR4ff8iGcgpsoJQylMzPVpE6TEDT5/bYFaG1HVGonTnvt8x8bcB5JlUhI1012/ZPutylwCSQJHePmN/nS0muI1CcgELhxumBV9GdpVz005GxN9phMjMJkp7CtD5/f42NuqXOdRmI7JJjQ/hPt52IVbpyaKHa89dNo+fZpYrZHIRYFAvE+mpqHGdcptpSHFobX2SRqpAGo9+342WOCdbUZLybYefaKVtkKbcKT2grQ/5E+wWKcYk9XiFhax2XaZuROspWsaewixDgpU9Xfd5UqlFGdhCyADulUGf8o2vbw3VVotIp6o6+8sySL0q8hTmOZxXhr7fGx9N4Xi6gtm83VJV60qmdtIsi0ihAJMFKeUwDufdp7uc2UUKK4IWnPm1MH3/UPIWpF6uWXTdOlJ6suKKkmDP6p+oax7bc+gpblB7JiJnu0HlE2FaBgkgLyTqfHvNuswBCUjPrzO+32j4WbiJZEVVQqcTCXzvr4Dw0+YsEqiW0dHVBI1GbWPZ462UFFQCAoBQG53I8dLcrAzEEwNpjNrHd77ISJ8gSWaBRAAVCtJOWfw7/AJFhKK6EhcqlRBJgH7D42OLUlMpUDJBzakA+32/VbumKk58qhmjTu0mdudoulITtjuoo4tXeWL4D6UwX6EEEcoKkQPcOX4NThRV9VjMgEhNUw8k+IBSqRI20jTujS0i8XGULN11KkZkkOskjYgKSf7Ri0T4EqHKbGN2PrXEryDtbhQI7uZM8+/xseIl0ZQXtyyBWKbcywUqJAOyQYA1jWPmDY60ZAhBgbTPzNkph45dBCT75juA18rKbbgKiFhIzGdDqdPP5i1O9lcsh0rCJKRM6qBE7jv8AabFnXQE+rtAMK5eUeHzFu3HkFYOiTt60x+Niryj1ZzEaHtGd/IgacrLPZRLQkPEK+voatkg9unWYiTsY+Pz3Qnfd4uUd5XPeVPUErpnCtKpGhBSUx4feLTrXMocPUrEpV2DG4B029vzraAMUsqaoqdb+YejvBKiZiYIJJnUfaogWuwOz+iVVmbl1XvtdN5sX5cNDe9MoFq8KVqqbUDoUuICgfcbC065QURsbQ7wF4o4XpOi7w/xviu/qS7qJGHaKmefqHQAXWWw0tI5qVmbVoJOlo0xn00MJ1j793YVvF+ipNQapdOrrXBMSnkgb96v6Jt5lJSzOmc1jb2JC6OnAkaATb2qwuMcfXFhVBRUOmqrYJTSMEFw+KuSR4n2TaDca8TL0xJmarqkNUavUomSer8M5/wAJ7YTt2edoYr+OGF6hSg1eSpcUVKLgUVLVOpUTqSYMkmbINXxYuKpHWfnEQSM0zoPCef42LHhst7lq2YjSwD5wJVnOjpiJVXja87rU7mTU3f10H9ZtxI09izaXcV0fV3g3UoACX0awP0k6T7otUro08RbmqeMlyXfT1yVOXiiopY1k/QqUB70j4Wujiaj9Iu0uhMqYUFz4HQ22cOYaeYBwWTiL2yOJabpsU6IAiNd9dLGkJgbExvHfYBlJjUiYk+djSATG+nOfG3awaLl5tShmj2hoPdYygiI1mxUIIMESCJsOiI7wBa+FTcbJwULhcY03CtdO/X7bGASBrvYhdawWVJ3+SPqi0DcWukbj/BfFJ/hvgvAFHfzjVA3XFS31IXlKZWTsIHfZqisio4hJLexNtATr9S08B7PV3aWpdS0Abma0vOZzWANFrkucQBa45qxMg6n6rZMmPfFqvUPSH6Sd5Zhd/ACnqcupDVWpUD2KsI/x+6TdMpKKjo9stKX6oXVKST5Sq1UYzT75X/Yd8F0f72+L/laf+swfrqzsxtbYURubVn/fy6U0Zv4OOn/PL++2kcdulG5Jb6OaVQdctQs/bYgxqmHqv+w74KB/BtjB/laf+swfrqzQIkfDzsBeCM1E9lE9kkezW1bxxy6VAOnRv/65f32zD/ST4uL4lYb4ecQOE1Lh390b3VocceWXOr1lSRsdRFkcapSQ0hwuQNWOAudBqQmP4NcbyPkjdC/I1ziGzxOdlaC5xAa8k2AJ0Cse0pLjCFp2UkGwaiCNhNg6An0NKDqpBKT5g27VpsdjrbTK4EG6c2DXesTWUyjopKVD4g/ZYnfdIyto1FWkKRTjMpCh655D55TbnCL3U3uG1HR5tSde/f7LHb/bqyy4y1SNuaqKgXcsd3Lui3J9pmgQFxB+r/stHD3ESAXUdu4hr+tXkrK1KcxgJfCUgTyGXQeFssKqgvLMYeSgT6ofMJ8B2bZbzTi0vV3vXV8GDovHbp31Af6QNQhWzN10bWp5FJVEf41q/Ooy1rwcyJMidOc6nvJ37/K069OB0L6Ql8QUkt0dCiCNv4sknbz+u0Hpl111KgVZjJCANSCe7b7Jt6nhPo0EP+6Fg4mb1kntKxtakqMjUAjQjfkY5ePf8bYoDdSlyRMcvH7fjYXqkqUgnUgyNRrHIc+4+VtlAlKFIkqIOZWhXyBtdJCq2XAdWDGVBOUmCSIk/PiT7LcqKgSoEALmYMgxB8RpYVxoIPWlQTm2kgT8wLdZT1uY+sokHyjWfd7oswITEIslakgEhQIkkKOWT7NTbCgEQlKgDoCZkyOR2nnY4KYqgtJGk6FMDv79dPqPdbSWCFkxnJgQeRnSO4/aDZZgmsiuYkpyZU5tgRqeZHuj3C25JT1acxAB0Puj55xvY00ytQVmWVIUoJyZZkwd+/x+RblDE5QpKpIAUCDKdNIB8h46WWYJWRRTnbUUglc666kzofr38e+2i4Sc6EpbC9co0ze0agaRYyGCgZXYO3ZUmYMQNOcfjrbhTTaCFhxIyg9ncmNo+PcLSDhsolpRR7s9oqT2VFMhXqkbAd4039tjjavoWwdcicp7WnjMb6Tz5+MWEue4LzxPfF34euChFVeV4VKaSlp0qAzvLICUySAJmJJA08LTO10M+kRlTn4fBtaQM03nSamAP+NifusOeohhAEjgPaiQwSSasBKhMulCYbBkic0Rr4n742sEvMtclsmOQ+Pl5eNp4a6FfSKeaU4rArGUmAoXrR6E8oDunP5FsV0Iuke+oA4GZSkwStV7UYkcj/Kd3K1cYhS/THmj9zqPoHyUFZ0hQWcqVJ001BPjHhpG3lrPJUOQzJUCU6mTrsfmNPC0/U/QP6S1ShSf3F0aVxIBvSkGoOmy9vs7rCK6CPSXWhKf3E040Gn50pjJjn9J4baj42f5RpBpxB5pdzqfoHyUBM1hplqUpttwJ0haSJBjb4fssG5UKcUXdEgCdtI5x4fDfvtOFV0K+kVQvqZdwR1i1GcqKppQSdxHajfmCdbBp6GnSLeUhhvATnWKPrLq2QgbwSSqB8+VnFbSnUPHmo91nB1aVCodJUQrKARqUmfd5eeuvjbSqhIP82YWRroSO/Q7eXnaaqroddIhl1TL2AXyUqBJRVMLTMDYhfdE6mxdfRD6QzRj97usjbR9mNjsQvx5eFnFXTfTHmkaeYeqVDbLoAKM0nWVecwI5aDfwHhbEVKQpKygkKBCyrTST3+Gmnj4WlwdEfpBtozq4a1cmcwL7AHrbQFWDf6JvH9IUocN60ZssQ4z3a/p6/s1tMVVOT88eahwJvolRA64tKAJMSTAEZh3zy017vtHonXA0paFkpOxBgyDMHx1FpUd6JvSCejqeGledBs60Ngf53zr7BqPoj9IzMev4aXjCTpleaGh7xnPcLSNXT5fnjzTCmmzfNPkouFQEjOFGIyrToToPDUbGPvttNWtLZALicvYgKJjy7vWHnpaW/4JvSBUkp/exvHRMfyrQMgRGivs/ENXRL4+kqH72t5EqTKSVNAD2BXdO3fYAq6c7vHmid3m+iVE4qFElTqwCSMpKTsPCdLdIrFtNntEKKdRO45e2NvM+FpSd6JvH9aAF8NrxzkbZ2xH+ly9uhO9tHon9IXlwyvFU7qK2h4RoqTIjb462l3mmPrjzUeBN9E+Siw14hEpUFBOUJ1mPb8zPhbpNeptsKACVE6nWdNt5Hd+20qI6JfSHGZJ4cXhqTp1rIgGNPX8Pk22joj9IQpM8OK0aRCnWFczOufflr99l3im+mPNPwJj6pUTGqcKiVASdfHXnr366W5VWLygqKTm7SsupPz7NvZaYKXogdIqqUWm+G9Wg5ZUHKqnbA3GxXB0ixN7om8fw6Ghw4vIzMgOtxtse1Gs9/hZd6phoXjzT92n3ynyUU+nOOqzGT29J7JI7/Dfn4205XJgJzglGgI0Khz19nwtKi+ib0hU9g8NrxUUKgLC0GZ0JHajkNfb5hnom9IRWUHhpeYC/VlaIEeStuW477SFXSfTHmFHu0/0D5KHGahaX0nN2s2wPPSd/ZYVyrX6oUoSNTpG5B8uXlaVR0SukORlHDS8U5TM5mxEk/zvfYQdEXpDBJP721aZ7lonbmM0nb67FNbSX/hG+YUBTVA2YfJRSm8atB6xNStBBjRZSANfHwt3+fb3ZWEC8aoZdSA6RqBvvvtp4Wk1fRI6QgEfva16Rlg9pGoGn62/nGtuj0SekKtXVjhxeGYEqI6xsT4+tofV+NnFXSfTb5hI09R9E+SjhrGGIkt/RX/eQzakCpWADsD63d8newycdYtSCBia9cqdAkVjkjXvkj2d5tITvRK6Q4JUvhvXg6eq63J0PLNpr4e63H8EXpEZj/2NqyIMEuNfarxshU0Z9dvmE3d6n6J8kwkY+xe08ScS3vnzapVWLMjmN/OxlniTjFAKP3S3gTplBqFDvB15jl7fCbPMdEjpEFWQ8OK4DVQV1jcQNp7Xnbr+CJ0hUkj976rIMiPSGhJO26/jy58rI1FH9NvmE4gqfolNAcUMbAZf3S1naJjNBza+Xnt4+0RXFDGk5XcRVSso0CglW+2hTHfPfZ5MdEDpHOhWXhzUCBmMVLUGec59dd+7Swj/AEO+kZSuinXw7qSUgKBbqWVAnfRQWRO/P7LQ49H9NvmE/AqfolRXfuKb9vtSXL5vBT7jYKG1FtIJT3dkCde8T7rB4exHeuGrxXX3a8EPOJUlX0YWMogzB05TItKb3Q96RLbjSDw8qWUvKCEqVUNhOo2JzH4eNjzPQk6Ra3Aj9xSZWBP8eZSO4DVUWJ3ykAsXjzCYUlUTfIfJM+k40YtpUABVA6EjtBTG/iYVoNrH0ceMUJASaK7FEGP5FwaTy7Xh862XXuhX0j0x/vCRPM/nCnII7/X17/k2JL6H/SJaICsAkJ0AIrqcb9/b8bD41E712+anwasH5h8kEjj7fyRDlzXaSCRlJcmY1PrWOMcfq8LlWHaUhMyQ+rQydx9vhsdraHQ96RmbqhgY5hyTeFNA0HMOdw+rvNk/F3Rk444Hw/WYvxNghdHdN3ISqof9Np3QhJUlIMIWZ1I5GJtHNRvIa1wv7VLJUtF3NPklxjj68klp64KdSgdctTA8/VPz5CRhx1p1oleGjy2qhqO/1fj9doKQ6knKFGQnSSZOv7bDIrFggNFOYiCR4+3Xb6rENIxC7w5TozxyuxYQHLlqE5k65XwqOfdyH7NLKV2cXLleqy2m66tBIJSQpOvlry+dImvyKl0q7eokEpIJ8fDx+yyvcdcsV7ZAUSolsDv7Px+8mwJaUBpIRY5zfVSvj/GN337Q0zNNTVKHGlqWEqAAMpGkg7yB8jSK6SsFJiFir6olLFZ1i8va0Dk7Dw5/VpZdrHkuNiElZT6uYgERp390WZ9epxFdUKIgFSSAkyIgRty0+Fo0guCE8ztQVYCnxtc0aLeSdDBR7hryj3ab2UqfHdwZUpD7vZ72TMbARGtoepnx6P1icwzfpQYjeO6Ps5wYsf65QShYKSANIJKQSfnnyOtqb2kFGbISpbONbgyIUqqIGY6hC5nTcxbhWLbiKgj0wSoCCUK5jy/DS0TpqSloElSTAHir7efhZIvXF133TLQWHXhHYQU6HfU8tz32UcT5TZoTvmDdXKY6vEVyFnrReSEJA1XBA5HciN/u77QPjzE10vV9bd92KD6HqkkuJMoCc09nQTrPLn4mzdvnFV53ytQdeS00pWbq2wAnwnvIshOEkyTOszbZpaLhav3WdUVWYWapUwFe/GriAxT4Ow1iV5+jw/TKNJR1DqS1TtqdK1BtKgQJWsk6c99rPCo4bdI9lBWtyhdGsxTsGdPFvuj4Wg65sQ3xhm8XK24rweo3lAoztKglMgx4jQb2dtPx44qUzYZGLqhxPPrW21/6yfK0Kimnz3pwy3iNUemqKcMtOXX8CnJX3Fx/pCfSKJokDVSael0Hnl0skOPcYKYAVFA2SZGrLIM6jlEGNNNbJr/GviHUp/jF+BcmT/F29T7vKxB7idiuoR1btekjvDSR3e7blppZmQVHrsYmfNBf0HOUu9HfG3ELD/Hzh5ed7UwapEYku9qoXlA+icfS2rY/qqNvcp1hNQy4wsdlxJQfbb52Lu4j4ho7yo7w9LUF0tQioSU6EFKgRr7Pj5W+iK470p79uW777pDmZvClaq2jyKHEBQPuItj4tC6N7HloHsVyhlDg5oJPtTNQyptamliFNkpOnMGLDtgeOgj22PX9TdReKlgdl0Bcxz2PxBsSA2HIaey2pTOzNDlWmFnWXaUyR3fVYVI01A07jYNPrDTT7bCjUTpBtoBU3C4SldLgJKBznX3R9tqrcSbova/Ol5VXdcraXapeHWlJQpeQKCQlRE+QtaK71APpHiD9Y+21eb0rBd/TXqq4oKxT4VW4Ujc5WiY+FqOJC7YB/wCY39K7bsHo7Ez/ALHN/wDqn7d11qwfii7qekrHl1D6GV1dOp3rQytRhQSdwkj9HbSQBNlHEmGb7vS+r2qat6poKdDanmK5JbWjRIypyqkx36DbQ87JnDSuu28q6oxTiO82RWOuFaUrJMEjfyA0A7hZy8Rb3W8ijuiidAaqgHXHM0Jy/oye7n7LabcoYTyXCuJzBJtLfV5PYCqgXVpFPUNN5idQ2pSZT3xr7rJlzvV+Hb4u+9hea3ae8hPVFKQG8qsq0SB2gZB11BnXYB4C6rsRgqtua7K2nqX/AEdbii2tKlKcAmSB4x8LMapdTV4eu91pYUtirWnfUNuIzA+RKbFtYAoR3Nkq4pN+XtiC8qShrktVFOCqmStJUiEtheWARE69obeNogxhe9Re/SJ4OrqnVOrZW4gLUZUpMyJPPe0019QGcY3dXK7CK6mYcUDoNZSr3DLaCMSNrpuk1wwpCf8Ag9U8yJ7krUkfAC2ZjNuA3/fZ/wAwXefg3/ztL/w9T/0Hq3rcIdqGxpC8w9oti5UdNracBTeCyVaONgx5G21DXS2uV5+Eaul4U150z+wS6mfImDZ8XhTBTq9PWRaOxAO8RqDaSlLFTTMVIGjjYV7xNsrFIRJC4HojQuLXaJjO0IDqxroo8/G2Wci6RJUTk3Nst5kac9Ft97K8Femj2+kRiJKCqeqoUwP/AOVa77Q8UqcqHgW1KHWkZ52n2d5jbXW03dL6hqLz6QWI6u7mTVUy/QsjzBK0nLTNA6p8RaOgm6qRypTU0i0v9aRGYDlrufLlrbuqF2Sjibb1R9wQ630qqQ+JTfNOvIVdWXIMEARoRoD4mCN+Vum0KUMqgYmSkIgns6Hw3s42b+uWmSWkUhb6w6gqSABy5zpztpNfhDR00jp00MmUmdNSoSefdaxd3RV7jqm8mnWoJBChkEkxtpuZ56687dJplJIV1a1HQCCd408v22Xk3hhRf0NO07215EjMoRHmZ5+cA92oaKzDS1JyhaEr7O4HjG+3wE2a7uiYW3ukYUyusQWVEADWEzrvJ8duX4dpbDgW2AUIIUlRMQdTzMzuec672d1xXRRX7Tu1NGDkbWUFalgdqJIjnoZ+6ykMGtJSoJyiBABXJkCY5RprYRlINkURl2oUfqYBJGQ5vU0A38O7X2aA25NGAMsEKRvAI5Hu5Qfr5WkRnBLCTleSUpzFJHWpkEEjeN+XPYR4GEYEutRCVlwCTlPWo1Ou07nz7haPHspiAlRiplKtENiUoIKYmSPw9/ORYN6mdZIztqgiBqQknaZ+rlaVzgm5glYcqHgF9ziMpJEztzn4ixd7h/cDo1vB5IMk6tkgbRtqNPdNk2qF05pnEaJF6PTuTj3w1LpSArFFBA3HafRpr+NvcpNLQPBK10jBOX1i2mfqt4qcMrmp7h6Q/DakYecdQMS3asrWUk5jUIG48re0yaoJbTH6Ii1DFI+O5jrclcw9xja5p6oQ3fdqCVCgph/0Q+63aqK7VaehsE7atpsA7ejKmxThiFzq5OkeVg1VC4y5pGkmd7ZxogFeFTfdDGiu4HSiY8foxbZobuUO1Rsmf+THx0sUN4opSVuIU5OgCd/O3CK8rGYwBvEyLLuQteyfvJOl12u7aAvdamlZCieTYk+Zt0qjpBqWGxH80b2DNUo9oD320X5Mcu+02UoGlkxn8UZbo6PLHo6BA/VG+1hk0VEAR1DUawMosTbqkjdXs2NhRU6akaDaLI03gmE9+aHNHSap9Hbg6Hsg2DF30SjPUo9qB9tuRU5jBSO+3PpYmJHvn3WQp7ckjN4oVFBRJUT1DYMR6g77Ct0tIP8ABpE88osVFQJ7vITbr0kyNBHME2c03glxvFGk09IlJBZSNP1bcuU9LMllEnYlIsX9LEESAfttyqqMykydvO0e6+CQm8UJ6MwlWqEaGNUDwt0tml0V1KCSP1fjYuqqVHrq8PC3BqTsT/OtMUngm4/ihzSUcAeityP5oiwaqWjAI9HRA0PZFgRVDYA7W49KI15b72kKXwS7weqFNNSAZfRmTH8yxarpqXqtKdr1T+gNbYqoMGZI20+fnSxerqEBsEymARPjaEtH6Oymyp13QtO1TFxagy3OXSALcqYpS20kstGTsUg2J01QjOiSJUifbNuE1RzI1kJI1nvJ+61E0ZHJWe8DqjhpKMsFRp2jvJKRyPlrYuaCkDBUKZuVRskafCwSq4BiFblR526eqR6LMiYB01G9l3O/JLvFuaMpoqKP+CsgR+qLF3KSlUtf8XblZA1SPfbDWgN5gQRHM2ARWJcflZIygkcvn8LF7noBZR7wOqGqqSjbS39A33QEDW3aqChUmPRWdAN2xYtW1QUlJI2Uees27NYlQSoATE6WmyjsdlB1T4oq7Q0Rqkj0VuAmISkCxWqoKR9QaRTsyNzkg8pNh6qrAeKwEwlM+B30sXZfJSFrCgpau+Y0Onwm0hSapu8gIpU3ZQtKSlNOwEwCQEgHfyj5nnZExviPCeDLmXX3zkpkOFLLSm6cKWSoxKRHIHeIEWVr9rWruo3K54qKGmlvLCdTCQTp4xPx8bQhw84ruY3xTT3zeBkpY9KDRdlthKyA20NYMTJ31BOkaWYqAvPgoOqw2yfSMR3hil5aMEcMb1xA6xoirrnV09I1/MT1iilR1JOVspM87B4lquM2GLu9Md4NtVaJzOGkqGng2kgZgEZQrz5bWsFhq9KD0NpikYbEIkFGgUdd+8nfzsevq8l07IU6ltKlwMugPt1m1luHQBuZyn3mRzg0BQNhDGOHcbUqwzdrdLXspHpNI8hCXEGNSUntRPgR42PVV0UTy8qaJkRqqGx2dfZzstYmuS4W3nsUMXOzS1obzKrWBCnAnXKobEECPb7i13w9Tt1KSIdAXIiACJ39p+RanHTtc4taE9RI6IAlEGLtpUVHVqp2vWBAygwLNTpBXBQXvwVxvd4pGSBh6uWlPVJ0UlhS0x3EFIPmB3WelTXpbcVTJkwMugSBmzBWeYzTAKd4g7TrYjjKmRe2FL4uwJKvTaCpYM9y2lJ+02sxUgY8EBU5KrMwheIeVGaMoAJgx7/nyt2gpMhS8pSQmdv2W6XTLbKlSZB7ebQDUgT7R4W76lx1wpQgQpOyU6nX527zbq7hc3YhbQhZV2VpISN4MK+YPvscusqarGV69lxIzJGgkj2fCZsAw0ucxbBIInMJ0kDWT8/UbbbQ3DqEJVCs0mIEbDXfSJj4WBIbghEYNbp1VCkhIQoDMrtAKR3HvGs6CNefjZtX0pQr0q19QR2Zzan27bk/GzqqUJ6kJbMpVooFMyRoOfhv4d9mriYgJaWlMjKqToZ1EHfTUqtSpCC+yszaNunDdLinKJhxMx1aO0lUkRp7YGnssHeV60tCz6RVuZDHYj1lJ5QOc/YO7RpoxMuhoGaSnYT1jc/SLExJJAjnud/dZFqX3ah4uvuF1atZWdbWmUWZxLtlXdUWFhulu+MY1leks0U0zWsn9M+ZHlZulUnVRnxto6aAxbRmNe7QWvsjbGLNCqPe5+rlsSIH1W0oGdedtwrv3PfbbsEAzvrEWIoWXKxqCkRIFtEQNLdEEJQrWDIB9ttqRlHzpZJWQYJ7rYdLbg72yDtZJLQJmYt9AXRDxMMY9GLhnf2cqWrDlJSOKk6uU6OoX/pNG3z+na3tR+S0xMb+6Jd13ctQKsPXzeN276wVioH/APs/C2PjTM0LXdCtDDXWkLfBWgxLThdO3UBMFCsp8jZux7PbZ6V1P6TRusACVI08+VmbBB1EG1fDn3ZlVuqbZ110AZkEeXIGwiVEka6+fz424GU94Bt2BIIynxJtrtWeShmHOrdCidBJPs1+y1cuLNycWrg6RT/EfBPDSoxLRLuZFBE5Wl5kZViQZkfbaxYgnKs9k6Hy2NleiWpyjYWv1lNIzeca/GwamkFawMzFpBDgRbce262uznaF3ZqplnELZWyRujc1+axa61/mlp5dVT5i8ON1LIpOjVXsA8m72qUj4Lsp3tjPpF3u1StO9HesY9Da6ltTNc4k5NIB112tbUEwRPnbcwYG9hDDJv6S/wAm/qrZ/dlhR/man+1P/iqpOH8b9JLDtWutpej9XPqWgtw/VqUkAmTpp4e6ycu++kWK1VRScAq+mp1LLnoiHwppJJkhOYFSQTyBgbCBa5B35W2I2B22k2l8lz2t3l/k39VR/dnhN7/ItP8Aan/xVUrEOOukliClpGFdHh+keoRDD7D6swTABSQZCkmBII5AiDrZFwphXjjirjZgfFeL+F1ddFHctUQ++IKEoMkqOumv12ugIAmYFu0qO+lk7B3ykcWdzgCDb0eRuNgrFP2/paESOoMKgie9j2ZgZSQHtLXWzSEbE7hFKxOWpp3e8qQT5ifst0tIO4APdFt3hAaQ4f0HEq+MH7bbWIA99tgjUrze6BIg66Wf2H3RU3DTmZLctn2EgfCLMJXv+2zwwW6V3fUsGfo3Mw9o/C1WqbmYQptKUSjU6Wyw5RJJtluU7i1Wsy+aKnqV0TLbTbriTJBKVQo6kgT7drYupf65SHUOJWhSgoFKez3jTxJ08udiV1oqHmadPWSpRjskCDsB5+Hws4bu4W8SL9vJ1u66DrHkIU6c1U2NiBJJOpkjzm3StgLnWAQjLlFyUlIrlk5YdOYdmRqD3fI1jXwFavAIj6FxPaEhSASTpHjoT872cg4BcY0DtXMhImTNazv/AJXhubGmOAPGBSlJN105OUkTWMnXcHfwPzFi9zefVQ+8s+kmmm8CpKSWTGoJLWpOhAMeUR4c9rdemJXCEU68+x+ikwRHdB9/us5TwF4xpQS3crJCz+hVMTIn+cJ0twjgpxjaRH5kEAFOlQxzM/rfM2iaOTonFSzqkZm86ttBpaZ2qQgystttEBR/SUSnxy8h52ERfl9BWVNZeOoKRlU5A125xy8oAtJeBuB+NaihqXMQYb6yoQ+OriobEjKeyChWusanxs53uBt9IUlVJhVTYAgpTUAkmU66r001ncaWrPhc11sqtMeHC4coPav2/nn0lFXeBKk5QRnkEfiB3e82UGqvGa0A06bz1AMpaWCrU7gA+PhOsWl+n4JYiadSpWG6hoBQ1FQknUmR63gB77H08O8XUS1ppcOugJMpjKrltv8APxtWkEg+bH7lYiyH5z1CzbfEp1BS1Q32pAgmWFqMxodE+zu333tzUL4k0DYXU3XfaMxlS1Ui5V/o6aj3TadkXHjalStDmF1uQZksgHUnbLzED32JO3bjcuhqpuOtSHBn0p1JSBA2g6cgPKwGunvrHorfDp7aSG6j3gfWX5Ucc8ApvBFUhxGI7tTL7RbIHpDZ5gbyfnf2fbcOUGYmDpbyMwjd1fQ8euHz94ULtKtWILuICmygkCrSOZ2/Dxt64NkFCRlhUTvZVLM+U2soU7smYA3XSlAkSSSB7Zto1raBChsPn67BOBzISlJgc+72+6xJfWKJKVGO+NrRZCNk0k1tkaLjLi85I0iT8+duw4zlyjXv1+qyehDzqilCSpUSQBbkukKCVyDznzsXu4OiEKghKoWxuoc97bLjQkglPzzslpeBGhB+y2F4SRm8j9tkKYJGqJSoKljfUeEbC2Crp+SvDxskF0QSDyjS3JfOaM0idDP2WmKVpUe9EJbVVMkwZJGm9sNSzIIJP7bIofkkBX9KbYH8vaJkbxv8my7qAl3opb9KZmIJ2i2xVMmZ0nfWyJ18pgmefLTTvt0KgACSQPPlZxTCyXenJX9Jp5g8vrtpVVTzqPfZIL8GMxHnbXpIUrUieXl8iy7q1LvTkrmqYzSNJtz6QxzSoRpZK67TQK0H220qo2AmDG1n7sEu9OSsaljcoG3f99uTUU8xlOs85skqf5qVvrvpbSnzrBmee3hZxTBQNSeSVS9TD9AeEmxauqKctkZNwQINiXpCtAVafAfOli9dUBSEpG/n4WT6cWUm1JujDdXSdY19HH0Ue63DdVTdSVhES4BvppZI9KyIZdGg6pRtpt4egEg7Lk6jXWbVzStRRVOSmaxkiMusk6nSwq6umNKJT+jI1juG1kUvZw8sEGJSPt+q2+tJpQgQQpMRO+1m7q1S704pVfvCkSyE9WE5h3mdLapa2mLoV1apUk6mZFkJ99SlQD2QICpGnvsMl1LVYiDsjLrtt+As7aYXTGqcEs3jWMBoQ3oCBYRNWwpCYa0jv122sh1zssb6gyQZ3nl8+NhDUhDYUpUBKeUHX5+vxsTu7Qh95cjVVWU61JZLM5vEz5fH5mw6XqZOUdUMqSIT9sfO1m87UjMlaSQCqfWkkA68/Hw9tlEVAJ01gwRJ17/sNnFO1Mal1kPezlLUIS2thELBbUJOx8fZ9dqe0NyXxdfFG6sA3Vct5Xa2a5SnnqilLCl0qFEJWnOkFQUZAUmRB0Opm2lW/KG1JQk5TolQmfDfT8bIV7YXoLvq2MYUSaVi6rsbS2mnZbl2odeVL7jiiZzJASoQB6pJzaECmBiaSAr1Dad3pFOm9qK90XOKG5k1OR3sPLaqzTLyQBHXa9UNCSRKjEAd0O8NcG3yviFVOGmuyjpy8fSHKO+Ki83FsyQAp5+YVGsACJjWJM1VWJqTD91P1rjqXkUzWbqmiAX1aZRO3tJ52jeq4nu01NUVdHjnCV33oXJTdrrKXmEtj9EJQptRUYjrCSJmBAi2c17XCxXRxwuvmakHHd24roOIVZdodvJ5umVlbVSXqWSlhfaS6pk9h0ZuzBBIg6ibStRqTTXdT0aQklhltvYEAhMbcu+0TMcSr0xPeP52xHRXA4+hSWQihV1mQQIUFEmCJnyI9kstMLcp01B9bKmRsdt4O34+6WG5JJH25Klj7XQsjzbn/ski8FlT6shCe8AnWf2WFgrSG3CQFdk687Gl3VUVIeqaVJU2ynt9oAzBMAc9EkwO6wZQAwEnWEgiRGg/H52tpcLVc+2S4XmzeWBODl333eF3VT1wiopqt2ndbVVpSpKkLUkgjrBBERHhbtrBfBIkBJuJaT2hlvAgH3OaafO1mlxw4F3q9xhxm/SVDiW6i+6x9pv0datHHCsQQNhnGv7bMVzgTiALAaqAQo6fRLmPYO4i1V0EJOs5BV5s0oH8ACpxTw24KuNoPV3WG3PUCbyUQobA+v4ixyn4T8GagqSlNGMoJUTebn2Ge7TztX8cC8RpE+lpnYww4QFTtJSI+3X2gO8FsUNSOubJIMDq3NSOUxv887MaWN21Qf2+tEFU9uppwrOK4c8ICnqkNNqXlCQG7wcUmYEjzAFqc3iLxrb2qaAqW8tpbjaEN6gZZmAPBO/tsv1PCLFTCesDjWTkTnEnu9WzMr6Oqum8XbvqgQ9TuFCxJ3B99r2G0jIC7LJnJ9yz8RqXTBt4sn6U+rhwvcNdhSsN43Je6L+9IKad7MBThogesk9qRC/C3KsA3fCkFVYlQO2ZJ7u5Pd9ndZv4TwriDFfpf5mfSPQkpW51jpToqQI91lYcMsZAJitYBEwA+rTmRtazI8McQZLKm0F7QQy6k7AvCTg3iitpLiedxSmveZJW4VsoZK0JleWEq03ibSO50P8AhS071buIL7ZOmgrGCU+ctfOlq50XD3iOw+HrvvMtOgZQtutUg+QOnf8AGymzhPjO2EhnFFa3lTlAF7OJygQY302+Fs2aOVzrx1VgtWnmia20lNcqfx0NeFjqYRiq/wBIiRL1MNf6uft0swOPHRkwhwy4fuYrw3iO8q2pYqmWXGKhbSkhC5E9hIMg5ffysyxh3j8ggJxjevYOYRfbnZPf62lg7ywjx2vq7VXPemKK6vu91QcXTv3wpxormZKVKgnntaMLaiKRrn1QLQdRpqizyU0sTmMpSHEaFRJkBaBzCQqLXYwh0DcD4rwtc2J0cTrxT+drvp67qk0TXZ6xtKymc+sTFqzI4GcRilSfQGACrUGpRqRtz8fjZ9XXdPSgua7qe67qxherFHSIDVPTt3ociEJEBKRMBIAEDb42Piczp2AUlQ1hG+2qqYZEyBzjVwF4OymWs/J23IjSj4i3ktW0fm1CpVy1Dlkx38nXWgKcRj2rQgSMy7lJ17v5QC0c1N+dL/DlBVXt+7a/m2KanLjzqLxBUlpIJUredBMka2Z7vSs6Q77nWv8AFS+XVQAC4pC4julOm3Le2dDT4zKLxVTXD2f9loTVGERm0tM5v7e1TQ5+Txv4QGceFwKBIm51jQb/AOE5c7Xr/J3cJb74I4HxXgW9r2Fe2/erV6sKFMtnKXGg2sQomf5FO1vPLBPGDpvY1uz894PxLelfRB5TXXJoqQpC0gEp7TeggjTbXztc/wDJ84w6T15cRb/unjw5Wv3U9cxcoHH6embCalD6JEtJCiSlat5HZtXlOIh3DqahjgNwPnfcpgUDmZ6aB7T1O33q+gmTvNmhelP6PXOtxoVZhryNngdOfushYip9WqgDU9lX2Wv0Lsr7dVQqW5m3SKEhXnERHKwiYGvnEaWyNhE8/D51tuQeZEbzbcassrYBB1JHKyrRuFTMcwpUe0yPgbJWomAe42P0Kx1SkA8wdPd9ljx7qs/ZHRG4B2tsEgQZ99uAYGhIi20qJ0HPutZQl2IG5m2A/C3I758NLbmdY177SUbIQHeRAt2k6xIsCJToRbsHQH3WmmXNcguUrwHrZDGnMai3DCw9TtOkmVJBsOoiIgkd1ilHCabqjP0alI37j91oO3T8l0sQIs4sDv5Kyop+Tjebzg/ibN5zXv352U8KOlq+WSTAVKT4yLClF2p2p5lCpPnbLGyyCZtlsbuZReIF8y1yMDrqJsrT/KoKUxzKh7vne1neFzfpF/ViEKQgejKX2gRCesTz17h8PK1dBdNXct+t3dedI9R11FUIZfZda6txtYWAQoEZknXn3aWsrwoYR+ca59TqJSwlGTKRoVSeWu3n9m3R6vVaq0jKkZdMppAl5CkKITlSqR4ae2PnUJIKHQsEpBVm2nQHv9tjuRoyFPpEiST3TzPcLbDLYzL69AI0jN37z8++2yCbLJBRJaVZijqye7tQI+fki2j2oUUiCRJB1Ps+eXjY6+wwcrvXoOg1BMqI9m4gHWwK2k+qHEgQPVMA+Xx3/GzJHVK2Gik07kAkodSZiADHu9/jZaUpKlaczKRz2GkWS8OsoWw8gKCgHECNYG/dZXXTLSUkJOmo5SNJ8BztTl+ctCD5iBOhzgGQSR/pd1g1BaFBxBQVEkk792/12MKRmbS0EbTEaa6nbz+qwawlKhmQRGojXbb/AFY99gow1RVWcp6yZ2J56QdI9oPzrw6M8IASEpgZR5jn7PqsYDKlN9tK9VBMqnbSbF6upbo6Z2qqlkNNoK1GNIAn7dvrsxsBqpgE6KH8ZrydIThO3Ccyb6o1KKdwPSmvhr7Y8LenLS0qSJ9s/Xby0vRmqv3pB8NlrdFO9XX5Qp61zUM/xlsIEc430G8nc29L2KO+0o7V7g5tQep/G2FWuzPuFp0oLGkFONVe+ulVQJy9S5qrTWdOfusWU1CVJMGBpZKFDfK+z+d45/yc6WGFBewSqL0UDEQECwmEnkneAOaNMVzt1ZnWMvWKGUZkzAsjl9alKWtUqUdfPf7LFqu7b462XL3cyFRgBAsSeu+8CIN8PgjnA87W27WsqjjqlX0kb5zvEDTlbPScycyVE6TG9oM46Y9xDw0cuGoo8Q1KGq9x9t1HVoOfKEkHVM/pHY2jBfSSxcFjLeylpkFRIT7IATz19gsi4NNrKTWki91b1VUMxOYwD8m2lVAnQkjlO/xtUdrpHYhJIVfDwI09VMnffs6edhB0icRaZb6qYVEQlE/AaTZs/gntfYq2npQg790SNPdbPSkkSVmYjQxpaph6QmIOyDftcd0wAmZ0MnTztz+/5f5Ol+VyRMESkH6tbNxQOSYMPVW2FWEyc20692lt+lyYBTI0iLVK/f7xCdVX1XA8wVjTly8j7rCU/G3EdQSVX9XQTJUXfV929mNQ0clIROPNWx9JHfrpqLcqqiBJIjw0M2qbVcdL5KyG78vGBIkPkGRz8P22DRxrvxw5vz3eJAIHafVrr5/Olm7w3olwj1Vt/Shl0kgzMW4VVGRAOnP2WrVdXFWuqikvXreRnf8AjTiQJGmk2cCMcuVCErTe9fBEE+lrnx5/PttXfXtbyRmUbnbFTv6SVaJnuFtKqSBAJHLaRaF6vFFS1StOG9a0Z0wSKxcyT52KoxE86065+eL17I0Hpq+/bewvlRlr2RhhrzzU3mpOYJy6gnUGwNa+eqChzOsj58LQS7flZUgIaxJeDSshDZVVuqST3kBQ0mPgLA3Xft4F7K9f1dUKTqT6Q6E6A65Soka+P1WG7GY/oojcKf8ASUwrqlLYaEGZUkztE87CGpUKR4AqgOaGPnvtE7N/1ZQVGvdOZaUAqql6c5/HwsWvrFL9K6tCLwfhSiRlqFwQO4H7fCwBjMRNsqMcKkAvdTA1UuqDpAUQVGIiR899tsvveipAaXMGYGk+HwtBX7tnkrBNbVqEEmahRGu3Pz91jDOOHVtyhbyhHrdarU+U2kcXjHqqIw1/VTM0t5x8JWk5UgFRj4e6bCO1Lia1tSUKKSIkjxPO0D3ljhxtBQuodUTuUvH7DEbe6yCvGzwUesqnSDqFF9ZKe/n86WePFmO9VDfQObzVl66rdLEgKI0PZ1+fn2l3K1x0NUqEmcokiQR8OUm1bnMc1IkGodAgCDUbTz38/DXzsUVjp4pKy+pUaH6YzHjrpzHu7rWRXNPJDNGeqs5V1Sm+pTqCDEADUzvv5/Gxw1TmwSqZP6MknlAjXnytU9zHDwWnxUCQFyeciJ15fX5FDj8HMkqaHa1BdlOYRocvmNPDutNtWDsFA0htqVbG8apXVpR1aiQScsTp8/M2jDpE4pxzhThRWYswEmuTeVGplp1LavoFUrvZcL7R7C0JkKlQ0McjaFzj5TzaPRVUpQlyFOBQVm203idoBEknSbJOC+IC724xYbuW/KMrumoqKmmlxaglypVSvdWvllKHB2TmkHIrcDLeo2d6lawjQlCfeAXa6ykLhp0m7qpqumwJxDvLrkhxNHdt91tKmkReCkJHZfaStaWnpAiFqSsQZSoFFrC0103FV3c/eVxpuVmsqCpbtSwsUzq1QdStEFS45yeWsW8++M/CKsww1+fENVd/4WvJ/WnqShNVdlQVqC0qqlDKAZ0U4DmMSoEgqZFRxTvu6cDVl48PsdLw+m6qwURuW876TU1am9E52mXWVKKNpCXoTlMJjW06vBops01I4Dq07+K0KbHZKMthq2k9CFcrjJjDBXAm5q/H9ZSXZW4irVej0aXnVP1FW+IKW1LUoq6sEEkAxB01i1isEX3eGJcE3Pft83Qm56+9LvYqqmgU5mNK4tAUWyo66TzE7SLeRGCMXVd58QKLiBxbfvLHF14br6dyoQqvUELaLkJyB1MBsOZSpOVIUBEiSbek2Bulj0fsWJapH71dw066D1YvqmSwyo76PIKmkp7itSZtQp6DubddSdyo4jihxFwIFmjZSvW1dRTlLVDUPIC0lC+r/TBOsxp8+2xQOuCnCYkgEb7fOvw7psYew9RXnSpfutmmqGHm+uQtspLakR6wVOUpiNZjUWR2sP0ranG1UaEqQooUlSACFazv7fcbFcCFRabKufFdDNHxFvnq0oWHFNrQSRlQpTaCTHmfnSzYDbqPplsJyqXKQXIJHMjSY8do0tIfGm5UUmOUpp0Nspeu9pz9FIGXMnn4JHv7zZkM5UTkZLp9VKl+qkanbyFuLrSWTuHiu6oW8SnY7wXDAZQxmyqS4Uxl10PeTv36fVFj7TLzwFRVNKS2ghKFKc7JMfogCTM8tiLBs0LjpAVmhWgzHsjz9/xsqJul90pZbUuoc2htPZA8CdTrHKN7ZzpLc1psj8ETFVQIU276G6pDQOq3AEA6QrKBHd4RajfSLpi3xfvx8ELFSWX8/JRUyiSPDNNvRC67hr7uisRT0rbjcrDlQ2h0JI5AKBSSd9p8rUt6bVHUjinQXvUKzLr7paK3A2lAUtDriSYAAGgTbb7N1F6wsvuFido6Y9zD+hC56F1P+dsc4kwqHerbvK5XZJUIGVxABM7wFnbX6xaocOqpm8qO86auplLp3kvlCWhkzhlDakjTWQF7cyNdDaonRAd6jpAXZd5BULzpqtkQefUl0Had0D4G3pPdeFXKhxI6gqSn1hECNCZO3vtU7VzmmrAb7tB/Qp9m6WOqo/TGxKjzCeEG6L0h297tozWVVYqo6xpsL6tBQlOUZgIMiZjnoRpZ6XXgLD1XTuekXVTolJC6h5hBSNZATHrHc8++N5e1Ph5xyWGm0JaVHYa9VRnmd1e3uO1l2mw5R0yEmoUpahoGkbgePK3Ey4iTqCurZTxQgNso8puGOF6swi56VxAVDjy6NsAaRMkz9vhZSpuF+EKZ4JXh+6qhZQcoXdjZGYmYAjUanlz2tJDN3OLypZbUkJHZKoKh5ch7rLFDh11uSBlWd9ZUrzP32omumk2JUJJoIhdwCjug4V4JKlPXhhC7HFqMhKqJoE9xIA0suXfwrwW5AawbcgUv1lruxiR5dnQfG0pXLgSsq1BSmIb3znsix69L1wBgQAXveDdVWbJpWO0oq5CJtdgw6tqG8WV2RnUm3lzP1LBqcZhzcOFuZ3QBRbV9HfCOKbivO404OutwXhSO0xKaJtIGdChOaNCJ3FvDniNw6vjhtjS88I3+0hqru6qdonkJWFBLqFFKhMRuD7rfQd+eOJ+OKfqcK3M3hu7FkpFRVApcKf1koGv+r528Lultcl64R6QOLsO37W+mVt33zUIcfWQS6C5KXDJ3UkhWpO+tvQ+yURpXOjjLnNI3IsDbpfVcvi0r6huaYtzDkDcj28ldj8nHjDFXEzgjevCXDXDK4HG7kvNHpd7FlKFLQ+M4U6o5ipwFCgDHqhIjSTdrhv0d3+H9S5iy+MSPVV4hCwKZhAapWwrcBO6vMnltajP5FfGrd33vxRwK6pMvsUF6s+Ta3Wl//Ebt6c3nfPWtqZnRQssShoqWtfJKSXaWHIIUFfVOhEEWjefikgbbQbE71YL9C4ABKIWPZY5OvO3JhQynYg2vwnKQUZ4uCEzx3yDPjbBMyNAbCVLRaecaUT2VkT7bBzm5n7BbomG4usdwsbLpOo5k+O/zrY5QK7RT3g/D/wD6sUk8t+XOw1IoB5IBHdtGkH8LGYbFVnahKUmYjY22FQJiPEWDzE6je28wBiLWggEXQmaZTvytuTGhmbBBRmY526Ss982knQgM+Nu0qHLQWBBJgg6n2W7So+/S0woIUL2MkGxKkWU1dYwr9FxKx5KSPuNjeblpYmZRe8j1XmO7mlX+1aLxsU42Q7ugk+8WHul0tVzTo/QWFb2Ac0Gh91u7vP8AGAe82ZwuEhupUGom2WT6e8EejtZiJyJn3Wy1NSsV82tZXVtclF63nWu1NUpCXXnqhwrWrKkASokkwkaT3DlpaxPBGpTWUlReDY0ep2FQRBEgmI9sWrtUNAXKlSCjN6MSoExIy89+Z8pNpo6NVaqpwg67mJcbUmmJ5ggkju5KFrdGLSIFUbsU2F9yBC8pgTM6Hu+e+3KnwAYSICpmcon5+d7FQtWbKhXaOg1g6/P124UoqBVySQnLp2vPXvjutsLLypQaeztKSBJSoKAy6RGvjvHlYEOGSU6FWgjw5j8LFELU04rsAhXZOp10HxA92lsUcrhJKlZtTzBBExr5D77R2U06cKVmWoepSgEuALTJ7IynX3ZifZZxreQSEESpCYCp3gpOvvi0c3ZWJoK1h4AkNqkjLuOenIxt42fiRCQlC8ySBCv1h2YP1WpzCxV2nN22Rhl5CVgFOUCCJAPKPtG/fYBwtqBypJ3V4HQ92nM27lKm0kDbcbQSQNBttbSiSoypULO3uG3t22sG4VkBco1XlUdCSTr4n42RsXBZuJxAQNHkLggadrT3R9XOywpOXKSuUmCNBtp9s/fZKxLkVc72YwklB7ie0PhqbQkPoFFj+cFDCqjqeP8AwuqTs1f1DIBP/jbWvjz+Yj1JabMTBMAmR328rLzE8cOGsLIi/KEzlj/602fv9/jp6tsoUUgEi2NKL2V1p1KLKJaAbRTZlkznn1dvxt0ArKog6nYeVjKmhzG/hbTiewqO7n3WeMKEhROmdom3YvRhTrQGiUmDNkNaE51KSiAVHKCNQPHxsqOIOZUzv3fPjYspuNARHha2BZVHOuVV/pnXPV3tQ4SZp3VtxWVObIASR1aRratiMA3yopDlZWaRoQkjTzHf86m18uI+EWMW3th6iqZAZcqHEkf0U6e0A24oOBt3uPIeLBKSc0hIB8BAFnFOZDdNx+GLKjbfD+81yPSq0E69ko0/0Z8fbY8jAlclIKX606gJkJn2dn50texfAe6OzkZVKYAlM/s3sZa4FXagz6L7x9lp9zKj3oKiCcB3hAVnrCAT62XXfbsj7RvYYYDvICQis0IglQ1Hf6umn1Wvu1wPudKs6aaD3jSww4JXINRRmAdNLQNC5TFWOaobQYArn3UhxNeBoPXiNttPD50ssXnw1rbvdNNRU1atsIGqnfW2Pdpva7zXBe5UkK9EM851stP8ILlcPXOUiTKBv8bV30D8ysR1bLLzrdwLeyOyumqzOh+kiRvrp5WOUPD2uW8lJpareBDhiPn7LXyqeEdzFSj6ADuNhbGeEV0NmRRgeEbG0/k1xG6H31oOyq1hLg9QVYS7WM3hHMelKSZ7tvjvaRKXgnhVZS2bsrdQMx/OT5neP04tYO5+HVFTHsU4TpptZa/cXS5irqRrEEJi2VPhMpdo5Xo8SjHJV+e4RYPp2mEC7HnZBUCq8agkeR6zTvsHU8LMKM0Tq2rmrErRCQn0+oyGdSSCvXf5i073rgmocCQyyYylOZAHv+e+3mfjjDP5RtWLb7aw21jFd0prnhSqQaUJKMygnKCkKIiIO/PnamcHlOhdb2q0MSj3AurOK4ZXWKhTrVK+0hLca1rxgxvqo2Shw4oFOrUhtRUAdU1TsgAa/pfV52rGzgD8oJXOt/nGkx84WD1oh+mQBpGkbEyoactOdlOowT0z3r8uupYuDH10MNPI9I9IvoOodGcSVZj6u8gRvYLsFfb+ECtMxVo9RWEp8C3exllky0vsp693s+PrWKYkwUw29lDBWk5iSXlydRqRMT88rS/TXSpdKrr6dE+qPPwsFiG52kBalsIQEaAka7an4WzRhsgfcFX++sLNlAzWC0uuhKqELBBkdaswfOZst03D261MAm6wrLEErXy07/C0lM3G2XlFCUFaWpMDb32W6e52/RUFLQgAJJjn42hPQynYqMdXH0VfL6wDTIOVq70pUNZVr8D9dmt+4pgupaXdzJzqgAtAwDP7fkWshfVyoVVjs6BJkH4mzZaw8isvgLQM6Ge1qoHcHl862NTUEttSgyVMbnbKMk8MbqLKC9dFKpSid2hr3ctotn7191ggG6qYKTHaDYkyfHn8ybTe1cgdrkJUgJSlJy6aTyI87N3irXnA+B77xGyUddRUi+rOSD1yhkbPsUR7vO15lJITa6Z08YbchV6xJWcN8LVC6KoqbtNY2ohbDbaFKCphXa29aAdZ5RaO784jYYbzO3ZdFIcpCSpNKlZSMxmAfE6SOXIG0H4mxA8isdqHHnFKSSntKJKgrnr4qmPDaDYOnqRW0pccKpA0bIKshH6swTtMctDzgdLT4VGwAuJJXPT4i95s0WT3vjiPeb1S0G10rjTaloW2uUKAncKghIGnZMb+2xa4MT1lNja4L6CkMro71pS2oHslKXRmBnWSFDX+cnlNmiw/V0jpDThAI0UZJBkAgSNyY2nkecEdtpVMhT2dSHCgrStIyKROqCCBMSEEHQg8tBGxTMbBI1zRss6SV0gsSvQXGV4YGwhfd4UeKcU3BS0t5MN1FXd1dVM9cTBAWlgErIUIBASQYEHS1cr86PXCvETNdjnC9KFXYFOdW3d9dno3FIMKyBskgk6lCVJAB0A0l6YV4Y4dvzD1PUsUlQ6qtmoqXqhDaXXFqAzKcJUVOGZJKxJ17iLC02AcRYAVUN4Dfao1X19HWNICGGeyf5cNQWesRoElKEiFwSqBbpMS7LT1UxrovRDrEgI9LibBEIJRmttdQjcFxYWwPQ3zhvEuHb5drMSU5aeo7vokuIoWlnKhxThVqpIheRKSVEiCJkwq5XXrdYfutmtLoYdcaKi0pC4SYKhmSFiTJhUd0bi127t4buXc44q8s61BRU6XHlOLUpW61LUSSTlnUnbWNZqTxkwy5hjH980iEuhmqcNa113aJCxmMnzSDynxtUxXCnU1M2QjQGyCJg51glXhf0guKHCat67BmJ6mjpySl6i/lKJ4ncOU7g6tUiYJTm8ZNrncCenhhziJiSgwnj+7kXdfl8VQYp6ykcz0j76zoFhfaaKlaAyqSRsLeatSvIwiFSRBUDtyEA78o9lnFwh66o4r4PaTmWVX7QAJG5PpCBvbl5IwRcozHkFeqnHG6qV7EN1V9StMKo3Gy3qCtSVmBMaeuqzAYuXOlvrXWWGPVTlkz7CJVEa90+y0zcZLpeqUXVVt04WEOut5lrhAScpzHv8AV8NbR1Q3Ot1YbQjrVTolEkDXQcxbzbGXiOqcF6bgjeJRsP7bopQ3OyAE0LaVvZc0rSVK5DXSBvznzsuXfdDtGPRnAw4onMHFLUUoSBuRG+2nLu71ehuOpbahypKVHsFtqcsdyjME++zmufDi6hQWtpa0aEhLkfHbf52tzU9UG810LIRa6bzOHWnVhKn0vKMI613MAAdZSkCE/ExtaoX5RLB5uqowNiBqVtVbNdRlzIBJbU0rQHl2yfPut6CUmHC+pCU04QE7ZBy8TvyFqx/lLMGpVwkw5iFS1uLu+/OoUtI0bS6wsxMd7Sdu7na32bxAjFIxfQ3HuWV2hYx9A9nPdUh6MdX+bukhw/qHV9mqvRhiZj+VBaG39K3stRXEktIbaBCAcxKlGATuI2m3inw4dXcvFLBeISlQF3XvRVKiAYKWn0rM92k299Lrwo7VFMoXl5SJ0tr9uYJKmoh4QuSCPf8A91zOAV4oqd7XG2t/cmhS3AhUFC1ZNB2NCZ5ajQe2y1d+Fi5DbbaQnmZkm0gU+EqWkZD94PJZbQNSpQEC2k4goGVei4Vupy8nxp1oENpP9I/ZbnoOzrmWdWOyDpu4+wDVEnx6SW4gF/HkPrSVdmA1x1lQhLTY3Wsx7YFu3b6wrcLnoV0Url9142ap05gD4kaCyqrCt+YhIcxReq0MzPolMciD/SO5svXddFz3Ex1F30bbSRzA1Nuno8DEQDomCMfSfq76m7D61iT4hn/hnl56DQfWdz9SZb1wcQ8Z9m9r0RcF2q/+r0ozPKT4q2Hxsr4a4bYMwco1NBdqHKxfr1T56x1Xf2jt5CBZeqbybTpnE+FkasvckmFWPLLQUJz/AMJJ1dr5DYIIlqahvDb6Leg08+Z+tLNReLbSSlOg5AW8I/ym9ym5+lziypSkhu8m6K8EAjfPTNgn/KSr3W9qqy9FEElelvJP8rZcym+MmGsSoQOrvHD6GVK5lbL7s/6K0WngmNOrMTEbjuDZSloTDTuemv8Akt8XJw10qvzaXAlGIbhrqICYBKQioEd/8ibevVXewnRe1vCDohYncwn0muHF8IUEhy+WKFZH6lRLBPuct7a1NbKZKz32yO3bn09bG9uzm/cVrdnoBUxG+4KkNhwOstvCIWkKFuyfZZOw9UCpualdJk5Ms+Rj7LKBE91uio5OLAx/UBBmbkeWnkkG/KbLUh7k4OfhZO2hM2cF8sdbRhQBlCp07rIBkHTT2W6KlddiyahtnrBpqEgT32EZVkWFKGxH16/VbgydAduU2xWoOXUkED2i1wKm9KQWCIAtsHltOhNgULStCXBGozAW7Bnu85taBVZdgyfxt0CY20Ng80CSfjbDG0RHwtMElJDZtZgSdLbChYHMJGsx4WESuDry562kN1EoUH4jvsVrXA1U0bigYLhaVp+sk7+0Cw+adATvYlfJKaBT8asLQ8P8VQJ+E2d2yQ3R5YA1i3VAfpozDf4WDWQoBSTyt1QmX4kanvs9rpk4euCdMw00tluOtjTP8+62WjlT3XgKMMXnVPN4YpUNLrXVGlCS52Csdk6ncTaUuBmAcSYFu29LuxA0y2l+pS4wG3g5JCTJ02/R03s0KC9GrvxdT38+28UIrVVDgQQVaqJgbD7NbScni1cPV51UFcEnUyEEAa9yvnazwBrTmKDNmcMoCd/VOHkqFeffO0W0hAzQsaSJ2BNmsnilcqT9DQ16hoTojQ+Pa8LCOcVbneWEro64qjeGzz12V4/std4zeqqmF/ROMIQE6iBB0A8NPLa2EAqKSJkgghQ1Ph9/hZqjiZcSgJu6vnLzCJI5D1vv2nlbj9865pCk0FZ2jBhCOfI6+XzraPGaeabgu6J2ZIJSBJA0ETy2+e+z3uO8F1d1MBTiitr6NUSNdPsj42hb98+5YJTd1cNgMoSUz3b2WLl4zXNd6HBUXZeC0rWFApCNwddyLCme0jRWYGFh1UyNr6rteGmnlr891g1FPZdKlnLzgef2DTxtGiuPOGUKUpFz3qQmT6qNNDr63f8APcC5x8w4olSrkvPQ6n6MSN983japcq5dqlArKwVdkZEwkQANJ08tvdZKxYoOXK6A0pIUW0iEzHbGp9pB9ndZhfv9YdT2fzJeKpWAB2IO387SPf8AXZDxB0hcM1NCukTcN6hSiFFX0ZSACNxm02HyDaEly0qcbmgi6Rr+fQ3xp4cLQAlKr9ohlVPamqaOoj2e8+NvWBhPPxm3jnRY9u/HHGbALlBR1dO3S3/Qyl/LmVNSnuJ0hMe+3saxlylVs6UEAXVmMhxJC7IHjYNwSkifEcosLOYanl32LrUY1GhEQTZ491B6T3EkqMa+P2WAcSCNt+VjDuWYGm+kbWAcI1J17+61oKod1GnFvifcHCJVx4mxGwHqR6peowQ6hspUpoqBlZA2R5xNm9d/Tw4OUSQHglyQIBvFjnruCZ77S1eXD/BfEPq7sxvhW6r+oqUmpap7ypEVDSHQICwlYICsqlCY2JsJSdGvgK122+DOCG5Vp/uDTb+1FrUTHkXabID3svYhRU7+UI4NNpJQil1MDNeiBpO+iDYNP5Rbg9EBN366Qb0Vvrpox4WnKn6P3BFj1OEGCkxpIuGl/ua2VKfgtwfZA6nhVhFPlcVKP7FiES31d7lC8fRV0c/KP8IGgQPzQI/XvV0eR0pjYH/9ZDwrWFFpFxFIEkm9n4jv/wCC+y1oWuFnDJv+S4dYYT3kXRTif9Cx1rh/gNnVvBVwpj9W7WB/ZsxEp9b3JwYxyVUFflGeGyUheTD6Qdz+dajxjX0WOVnHw/6a7PFy/RhHh5hW674vUU66osNXi8kBpG6itbCUjU7TJtZZnB+E2lfR4YulOkdmiaH1CyLiPCGHKXEmHL7bVSXOqlcqmlPsttNZgtkjKSpMRIGneBaPDl3LlISsGgamMnHHHepb62n4M3epJWUgm+m+RIO4HMW6bxhx+nXhDdbYTvN7tn+2JtId13hgvD93sXL+7GkeDIUUu1l6NreWpSiolSioSZV9VuX8X4Ap3Ch/GFxpjfPebI1/y7TsbfOUL/6qZNLi7jyCFL4Z3WhM8qtC9f64WVk4q41vgEYHupHeSpCgB3f8Isqu4/4XtwlziFhlA19a92JgeOfxHvsUe4ocGm1pUribhBMEyFXxTGf9PwsF0Qdu8+5EbIR6qC/dHxsKEAYTulJHItoPL/8AmrYq9+NS2ir8w3G2sCRnbEfCpNgXeMvBBIKXOKeDEkb/AO61OSP9LzsTXxy4AsKTn4r4PSdN70YIP+lYfd2/TPmFPin6ISQjF/GCpxPeWF1UeH2Km7qemq3C4AlDjb5dCch68kwWVzoI09gFXQ8Xa1ZL9XhcoIggPxHLvMWb7XHfo+UHGW+b8e4j4a9DrsN3cx6Q08HG1ONVNWcgKQRmAdmPws5HulP0aGZcPE+5OUhLTqj8EfM2CaVjh6Tz5o4qHt2YPJJowvxIASFV+GAQqQBWEQB/imwF5YK4lV5BRe2GGhMkmtUqSef8l5e+yi70u+jQ12f3y6GYPqUdQTp5N2Lq6YvRpTlCeITbhV/xd3VRHf8A8XYPyZT3uXHzRflCe1g0eSRUcPuKSHCTf2GgqMvZfJ07v5H7LKFFw84oBsoOJLhaB1MOFQA7tWLae6Z/RwbgfuzqF7+pdVSf7FibvTY6OiFjq8SXm4BAhN1Pa+RI8LI4bSu3J80hX1LdgPJavXhpxCTUN0q8d3Il6pQtbaEoClLyiVR9FrAPsm1frrvXpRvJNZd/A7FTrbg6xLyH7uyvJIkKTnGx0I0G9p1vDprdG6sCfTai86wNkltLlzlYSrYxmOmlpvwhU0l54Uuu9qAFFLWUjNQylSchS2tAUiQNtCBHKzDDIL+gT5pd/mPzh7lStD/S9UsLTwTxUwoKmVV1zjl4jltaNOkNfXSEoMAqY4m4FvO4bpr6tDSH6uuu50LeSC4luKcZx/JkzoOyBzt6XuNpVrmUqNd9BakH5T29RS4TwTh9p9Y9Jqq2sXl3HVobQDv/AMqq02UMcbgRdO6tke0grzUvyvNRUv0eqkg5SCCOUaRIO3nPMbnWHq1C2+qqEIJa0c7XaMdnXumR3xqYsWvOkeff9GL8voSnKCCSvX1knYzpEakR5WSroW6zfK0VCFoUELVlWoplW+uncefvFtIaKkdU80CHkrClAlUFJA6wpVz9uhE6CFHcaHHCw4wk0yUlKF5yQkElUCMsaRuTEaSdSbJDrzIpjmiFN5UpTGaJBkEnmMvKBqDzhUp0Idp+vbaSS9lWDEKlWpOoEx2UwCdosQHVDIV0eBOIU3nwtuitutqnqrwpqRDLqnAJSpAymQk+t2Qd+7WBZYq7wvSovuiXedOwFvB5nKhACSMiiBoO5Jjvga8rRv0XKxdZgN1ph5Lz1FeD9P1mYg5VK6xAkRMhwGfPvtLN/wBYzdlAyi8LwYXWOvoYpV5c+ZZ7IkaHKkSZ5jW3rVHMZqKN5PIKhlDXlFXGKamZbySt99RcKVqUkZc0jflqnltHeAKydLG52/T7qxMhpWR4PUq80pBJAWmSTGkHw020taahFa7StmsTSl0U4S680UwpYT/RTA3I0GnO0KdKO6BX4BcrUTnu91urQtAJIAVCh3+qs909+mlPF6cT0Eg8L+WqI11pBZUtvSkdZQFqAU3mgQnsjTvHs8vfaZug5gZ7G/STwlTBSQ1dTyr4fKlRLbCSqABvKyjS0V3nQqqaSoLSlulAzaDSANDPPTn4d2tpd6A9+09x9KDCoq3S2muRVUaOQK3GFwPaQPhbySW4aVqxfOF16scSrsbr7ops6EkN1SVdoFQEpI256xpt8bNa58IJcp0LQoNwQSVaDnoEjU7Cyzxj4i3Pw34d3vjy/bvra277mS0++1SISXilTqUApClAaFYJkjQHnauVD+Ul4C0yUqODsaJOaOzSUx5f8+NbeZ9ocLr6qp4lKwkEL0TBMUpaWk4czgHXVkaTCV9+mUzNOqlF3TLpWlWeeWVI0PPXys9KG4UKSlpsEJBlRjSfKYtU9v8AKh8BmVBDuEMbDkQKSmk+f0/dZSpfyqfRrZCUqwjjxShqR6FSwP8Ar7cw7s1jEp9KI2VuftBAR6LwrfUVwNlSQlRjSTl3tDfTv4dVuJejNfyrvpC89dlRSV6QgSUpQ6ErV3ABC1EnuBmwPDX8pF0V8UsVVRXVOILj9GRnQ3eN3pl8jdKepWuD/SygzvaOePn5UDhZfWBMR4Hwbh+/WVXrd1RRN1K6VlSTnSUnNLnZBBImCRPfbWwrs9NSyNlkvnB0AFz8AsCpxGSpOX1eZJXnThgXXc1/3VUX404ulp6wKqGmpBKSoKOVR5wDG5Ga30C4Qxd+67Dd031g663RQ3nRMVbNVVDL9G4gKSYnUwR4W+ci9sWU94u/xdlbSusC5IERrGg8z7zb1R4OflZujzgzhBgzCWJ7kxh+eLkuKiu2uNJdzK2VPMMpbWpCi8CUkpkSAddrd9U0csjmyEkdbC5+rosB8jC3K0X+v9rq/wD+5VurdFRf1a5XKGoQrRsHwSNLKqBQ0DQbpmUNpGgCRFqIPfli+jIUnJh/Hp/920//AOkWSKr8sN0cHZy4Zx9/m+l//SLUnMkpb90pyXdSLnzKFldJYSPFul9FfuqvdCEmFWRqu+iQe1tahj/5Xbo7uJ7OGMee2gpf/wBIsnP/AJWno+u6Jwxjn/8AwaX/APP25+thxqpJPCctCnipWfOeFeuqveQTnsk1V7knRR9lqNu/lXeASz/3L43350dN/wDn7FnPyqXARRJGFsbajnSU3/5+3PzYFjcv8i73LYhmoGbyBXaqa9SwZJjnbz6/Ks3Gbww/gfEDaATTO19IpQP6wZWkf6CzHn3Wcq/ypvAY7YXxqf8A8jptP+vtCHSt6a3B3jvw+pcNXBh7EzNfS16aptysp2EIydWtChKXVGZUnlyNr3Z/A8WosSiqJYSGjc+0KddV0MtI+NkgvZU6wffbmHMW3DiJlUO3VeNPVpO0Ft1Kx9VvedtxVU2hbIUoPJCkZeYO1vAAuNQcpWFZ5SIER59+1vQ+6vyleC6PB9y3HWXfidytprspqas9FpmaZgvIaSlfaDxdcTmCjJWie7lbqe2WC1GLCE07bkXv9dlndnMRiouI2UgXtuvS7BRWi5zTPEBxl1SSnN2hPag9x12Nl4/G1Vugl0icLcdrvxhTYdoqqjcuZ+kfeZfpkNfy4cAKSlaiv+R1KjPwi1BJ5c9rKgppKSnZDKLOAsi1UjJpXPjNwUG+hLzS2j+kCLNR0FK1IJ9XQ8rOxRMR3bWbl7NFmsUUjQwoR323aI6kLMqhpmRYrkyffFszbHQ+37LBlQ3y7Wye8zbUAWaTdHaZR6lIBPZlHsBiws841OkWJ06wc6QokBW3OCkH65sYCjHrWst1CrnQoUk7azHdbAoHX67cBXKR7LbChOxsQaJkIFTAOvhzt0gkQEnb67AkkwAY0t0lQGx132tIKCGCu/Qd+1uahIfp3GVAELQU+8W5ChqO8bW6KudpJLmkd62jZUZJKBPnGtjNCfpxvA5WT7vgMuU8D6J1aQPCZ+2x+iIDusmzBNzThAMDVY0/VNssBP8APj2j77ZZtU68EsWPVF1XNUVVI4EvsJSoKKQRJUEkQNBMnwmdjs86XgvxHfYQpOM7qSVICxFIdM3L1fq8dpNmnj5p13DVY2lElZbIH+Onbx+/xtc/htwuxVxGWaXClzv1zlMyHHUtJSEoTG5JIAnWNe+LPAxj7l6BUPfHbIq1o4I8SAoH92d1Zd9KTQGPLu5/fY4zwH4iOZmzjK7CoD/xI7+75m1iK7Dtfcl6OXVetOpmpp3C08h1OVSVDeR3+YtP3Bvo5N4/uZd9OV4p2kjInMnNmJGwjkNPtsd8ELBmOyrNqJXGy89FcC+JSkhhGMKBxzJqlFAomdNBrPvsFV8EuJFKerqcYUbS3ESgKu0g+YmNDEez3XP4ucMX+H19PXeqoktkDrEztvGmvd77RBXlbqz1iluLUnKMxlQHIHWfr3s7aeN2oS7xINCoEVwZx+oJCcfUMZcul3jXed/Pa3Q4M46dKWnOIdCkoUIm7Ekaa/rbb/Mi01k6yUgSI01nTl77AqEJAQkyoaiY8dbT7rGl3l99VGyejLxMe+jVxNu89ZuRdgkj36afZ427a6MfEBWZbvFOkg7kXTIkjeSrunX42n+4Kj026WnutJW1DSgdyQBHvFlZppCVAKAIjtQTqRp7d7UXsDTZasYDmgqtp6LOM1KlXFynBkkhNzDT/TGm3yLJt99FTE1LRqq3uKzbplKVJ/MoSYKuX0k7n42tGpOZKg4oDaQRvMjabJOLiTcb76ykZVtSEkGfpEgx46b2C8WaSCjta0m1lSXB+GqvCfHa47gqb19OVd+ILrh1DQaKwpxtwgpE94G528re2tPq2CQSPPfe3jXVuZOkzRtpUDOIbpUSnSNWRuO4kC3srTpyshOu/wBts+Yk2KnGAHEBdEDWFGdrF3Rv4bWNmP1Trz2sWeJIiedox7p5NUnvjvPnYosD5HKx51IVqoSZsUcAzGSdTztdaqTt0g4jx+3w6p6e9nrpdrvSnxShDbiUFJKFLzEnl2PjZF/hQtBWU4MqDlUdPTkjnH6lkrjupIw1digrJF6JE89WHrQuVhCjKQR3Ez88/kW0qcXYs+ocQ7RWCT0qshhOA3dP1ryAM+xu3Sulo4lHYwGD4KvOBtyhq1eC6ADCoyiD3z9+v1W3nlRBMjY6SI90d3usYsCr8VwVhB0tLxOowDTpnabyV9fVWCV0tr7UTkwPd4CTBJvFZj/q7QAVGSgwSDAAA1Ht9njbRcy5ijXQCZG/z9dmEYT8VxU/fws8SEfRYMu3MTt6Y4f7Otoz468UcQ8eLiurhxeN10N1UV4XiFu1FM6tbyeraWsABWmqkwfCzOQ6owoJAEiJAgCN7abeAv241hY7FbEEHm04Pt+Pss7owRYpCRwN1Hl59G+5LpfSy5f14O9Z6qglsActJT3AWCb6P2GCAk3veaRB0CmtdNvU3tNWNgA/TFZ9dJynvEyIs2nUukJTTKKdddYE6WC6niBsAitqJSL3Udo4AYNSD1l4XotJVHafQCSY/m6ft9p+m6PuAXEqKnr0UZJUfSgAAPIbfO9nUr0hGiiexBMTzj8LKd2N1K1F0OEspJkAk8u78LJ0EYGyds8hO6Z7PR64fJ0AvKCdR6XHOI215iLCno/cOwCVUlYRuQatR5eFn22+JHZAB0nbUTYwHhJSo5tYPf4fssPhR9EYSydVFY4M4EpcZuUnoNYukNxIqeoXVrKS6KlbZVuD6pGkxtYDEfC/AVDQGporkKShW3pLpzDu9blp487P+pj92NOAlQL2HqpGpOvV1jB07tHPiLEsQNpfuypSURlTOvhr7NvqsmxMLTomdK+41Vfai7MO0OMU0NXQpRRGlW7lU6sBJE880zoLI66Gmr6e86663P4vQZ1JVmOZahEASdEgCyvxBCXLzu+7GVILl41CGSJM9Wkyr6wIsnXShLFNjChIc+jUslO0pyHaNfD3+FgcME2Rw4gXuuL8TRMYQu69qenCXVuNhxRJMg7zr3e+yhi67aRi6aapuylSgh9txeU7ojtSQdgBr7e8Wb9fUB3hBSOZRnQloiVZYPWDWeWnLxFhscX40MJUtP1p6x9ppZShQzZUgTpy/HSzljQEg8lL1DQ0l6V9JeTlAlqgVUoQykEw6k8ynuJ157WkBjpE8bLqx7X4LpOJN8MXRd9OlqkpmFISllKSEABQSDAAiJ/Rsz3q1tq5aFbUadSoKSBlHYMa8+e2vwFiNFd9PfvGDFxV1iXaB9QbDbgkpUpUk6wRoD7POa5e2PVEaC82CfGMukDxwoa25BScXMSMs1FV1T6U1pGYZkgjaeZ21tGvHjiDinGtdQ0mJMT3nfC7vaWpv02rW/1WdYzBOYkAGB7h32f1fgy47xRTVNXUvRSrCmiHICY3EDT9X4axaGuMdHS3FiWsZacJZTRs1DgWoKUpaiQANROhiO7LvaMVQyR9gVN0L4xdwUa3qzczLArLyddDg7QCYlz1tBOu8690Ge9Cup81VcuoUHG2UNkISZISCrTfcA/V7bCpu2vvp8vPOBplSgEozyASfdsAT45dLcUDno9/uNLWlZDIScpByxpoTp4iNtO6LXLoICVK0hCVOrcbRmgjJy5nnruZJOoJn1tFC4V0voD7K20TSOELIKcpSoApEkwYIUJ0PiTAsk1D+dGZTgyk5TM6QABqTBmNh+JDuevapb3DQUnK8ktqVkA6tydNYkajlrCu/WzgpEXVk+jDfjFMu+LteeU0qprUPBS15hlUynQkntZssjXc6ACbTFiKLxvVlulCstG80tISgkE5xGQREAjceQgzaBejhX0NLel/MO0zb7rTDLrK225SFlSpjXszlSdBsIOo1sG1UUdw3KmvvBBNRUuIJQuBCyoCJmREyZ2CT3W9GwJxnomsJ0CqyjI7MlN96+VuBFMgttM55AVBXJSCqd9o3iMw3nSP+NdHW1WB6yhrKdCXaqldUSoKIkbHbmToIECNt7StRuXtQgUl4KSGW3FnrUH1ZkRAEj9OdTqRtJFoP46VWKMZKrLruapXRU9MlSStEhQEagRAEkmVawO62tXfi6Z25JFrIAsXKoFI6s04qFZlFsqCgsghMAaEnlJnnv3kQ+OjXcldXdIjA1DcyVB9m/6aoSpJPaYQrrFGdNkJVO0i0e0IVQu1FGsgOsrUhRA1BTIJ08zv3aeE2dDmovNrpOYKfu+h9L6x51l8gk5G+oXmX4FKZOvdEzbx6cZQVqx6kBeovEbh0zxVwLffD16tNEi/6Q0fpIaDvUkqBCwgkBUEAwT3WgnDf5IO6Hyl1/ipWutHUqVc7aUgTMSXD8JtbDD9WaO96V5tNPmQ4I68w0TMDN3DXflvFpaXha/8QK6zFmJUopP/ABG7pbbUP5yz2lfAW4/En1LZWiFxt0FgPaSdvqW1E9jW+lYeJuT9QVC8cfkzeBdLhleHbhxxf15YtzpS0aKmZcQBPaCkH1dJ16wanblZDwl+RhZvFArMW8Xay70uQfR2LsbW4ANpUVwNOQm3pZdtFh7DlMKW6KJinQn9RIk+JPO2PX4kA9sWqR1/ciXVE1/9Vu3mdfuUZJDMMsTPrO/kNB71QZ38jXwzZRNJxqxS05AGZVDTqGnKNLNnEf5HLBlJdFZWU3HC/XahincW02q6mQlakpJSDC+/T229Dam/DyV5WR6+9wttSFLkEEHW2XV9qchvEbIsNFNJYO2XziYjw+5h9/qVuF3OVpJy5YUP2z7bIJJO+tpR45XU9dmNr7utxgt/m69q2lk6TlcI/sz7R7IuNvRKOUzQtedyFl1LBHIWhYCR3621qTM2cGF8AY2xutxrBmFL2v51gZnmrto3KlxpMgBS0tglIJMAkQSDG1nOz0b+P9QCWeCeOlBIKlEYfqoAG5JyaDxsV08bDZzgD7Qhthe8XDSQo3jx+Fsj5iz5TwN4zKu83qOFGL/QgQn0j8yVPV5joBmyRY3dnR44730Cq6eDuMqpI1zN3JUER55LMamFupePMKXdpT6h8io7n5m2ptKn8FXpIf8AoOxt/mV8f2baV0WOkanfghjUf+5n/wC7Zu+035RvmE3dKg+ofIqLJtkm0pp6LPSMUP8AvKYxGk63S8PrTZBxpwX4pcOqSlrsd4Gve4aeuWtumcrqZTQdWlOYpTO5izsq6d7srXgn2hM6lnY3M5hA9hTOJR1aSASoyFEjQeVrXdFvoYYa6QnDeoxxX8Qa66KilvV67V0jNCh1PYaaWFZisHUO7RytU4CTA10tfn8mji1LeHcb4UdcSOoq6O8Gkk6nOhaFn/q2/fbOx6aemonS05s4W+9aWBQw1NY2KcXBurQ9CPow3T0b8a4hcujHFZfLWILubacYfpEtJStpzMlQKVGTC1iI52uIoADutAvDe+P9/d3tlfZdLjRPm2rTx1AtPSyCbc3hlXNWRmSc3dddBiVJFRyhkIs2y5O2nvsl39TqLaHk/onKfKymROk++wdYkPUi0EaxIttU7srgVkSjM0hNIEEa+7kLdDUAJOu1ult5V5SN/dvqLcEEcyfO20DdY67YUlNQrTRbYPuJ/vWNAjUkb72IpMVLS51UFI18RP8AZsaCiTI3GugtYj1QXbrtawEKVEwDEDXysRN+NhvrG6KrPaCQnJ2pPhPh8yLd1DdYtSDS1KUJ0zJKQSo+Z20nW3DDF6Ay9eYWAf0WgJHt9trAtzUFr8+jYXdWmdNG/Px9/jYdN6KJUEUb6wlIUkxAWDG089djFgU0t49UoG8j1pJKVBsEBOkCOZ038bYKW8SUk3opKYGYBAlRAG3d392vlaYyqGoShSVSqhnrVU62TJTlWNRB3+2xjNzzedkhFHeR1VfKiCIjqwBMRPfM62U0kaAnUDXxNmPgnC4YHV1dSkyOsSlwT7QfqFlCinrZI21M2TVKCK9gn/CJWju2g/fZToYDuoGu02ZMd0sEuT+n/lWy3eVf6x9x+62WSV14RY6KW7ndcATK3m8qQI7RXz7t+Rtdvg7xsxXwkW5W4ZVSlNWyhuoZqGs7bgGxMEEETyPPxtR3H6ouzKUyTUtpMpG2bby8wecbwbb4HpG66/Ltup99TdNWVDNHULBEhClgEwR47d4tGnsGuLtlWq3ZAHdLpwYixteuMMT12Jr8dbXXVz3WvZU5UDugDkAPhraWuE/SMvnAF3uXTTJp6incIMOiYVG41FoRx2y7deLL1uioqhULuqreoEKAM5GXCkeQ0+NkgVi2l6qPZ33M99r4DZYx0VFr/WCkTiTxHrseXk9ed5VKXHXlE5k6Ae60dVKivMorCtDm/SzCdffrvboVC3QU5gFGSYOoG0fX8bcrpncklIMQPPlqLSaA3QJHVEV/RlQWtJSdSZmdfjz87AuICQvtJkg5p8OZn67DvMqZChnAKToJIMxrvt+Niq+pIBj1lZYImAD3b7T32mpWS7g6tLV4OUTgzIqAVAyYzjUe+Y9tnyKcqC1tg9kKIgSBpI89bRS08tioS+24UrSsLHmCO/y+FpWuypRW0LNc36jraVEeqATuPfbNrW5TmC1KF2YFq11CiQXBCYgEGPKPdZHxIyo3NU9YApKVNqM7EZwefj7rOR1JWkx2QmTqdtdB893KyLiUpbuKtKkKOVvMAP6Qn3a+22e9/oFabGjMFTu8uq/hKUbqEkEYguxI0kaKZ2jQ6z36d1vZRn1RJ1m3jVeyFsdJOjW82hK135dROUgEg9SZ0/ytPqm3syw2VDeYOnham/YFO0We72rFAbkAeNi7gEnLp36WOKbSACRBPfpNi7ojYDtWeNRksiD6f0jYmsayTvtrZQeSCNeXeLFXRpB58vn22uMVJ26i3jwicK0SkbovJsg/9E6OXnaC1JURKnCJ5xNrA8a6U1GEGxI7Nc0rUfzVjn52gv0U5DkgwY07p+RbSp3DKs+oF3IkUOJ9Z0kgyQOeh0tkJCSVq2JUDHwsaFEVLyhXKQY2HMg+/T9thDdxzBYywqO7Q++xy8KuGEpOWnUHMe+QN7byqIzKUdeUyPOx70FxRyhRWADOknTw9ospDBWKXNUYYvQhQMKFG5tHl3bWbiAbpi3KEgAAKBDgkeOh1GkR36W3li9LoJUrS8GhrH6Upj42cLeC8TEwrD94ICRHaZUkbabx42Bq8O1l31N0mvoXKcu3hTZAvQkB1AVHfvZ5A9rcxGiaJzHuyg6pTxmw285TOZlEBC4gwOR1/ZZAfpKJhsmofLYXMyqCVDl8/faQscUVJToo8w1yr3O57POZ2+dbMOqoqOuUGnNUoMwDGsRYGbM66shmVtkj57s60hNQjsmCUrMak924+2yzdj11KYLblTL7iuy2CQCfmLJ6rjoErTKtN5Ku4WV7pui6FINVCs7ZzIlU90fPnaUhuEzAbof83NKSXCFA8xJFu03bSK7JSZIBIzZR46WMNuNIQAVAaTEc95ttboCic5CQDoTrascysAtCRWrtpnuIuFqR+QxVU140q8pIOUuUqiATzgE2sUejVw7rad6neqb5hYKFZ3kAHkdkTavS6lDGOcIVadOrq69uVA86dK//AJYteqnKF5VpU7KpC+yDPlP2WdhNyEQNadVX4dC3gNX17FfVXfelXWU7ksuKrnUhvvPZTvFq6q4G4Iubpq3hwldu1bmGL5o6ao6j0hedYVTuEyuQqesRyPd329C0llFYinbdcBUoGCQOe1qccX3ae5/yiOCKtSwg3nctEyAP0lda+iJHOO/7NE/QXUraqRVdDTo5U92i42MDldCjRTLl41K0HWYADg5iy/RdDvo21FOht/hfdbyVdlfXuPurgbBJK5A70zB07rSUimSlbgNP1ozGSpBUd++ylRrQxWoaaS2htKtxlA+uz5QpleUPE657quLF2JMPXO0UtXRedTRtMxoltp9SEJiZEJG57/e2Lur27j4v4rvI6KfoWqggz2ipls7/AOMDz1NpQ6RHUp4/4zugMhTgvR9wrQ2U5isheg8l7/faGseVX5q4kuvw0luuuagBObRcMtgA+1HPv8bYE782ZqtwtDCHJ8v4ncfa9HTTFMwVBIPjuDvqfmTav3HzERfxEqnp6v6ZbDIqAk6dkKyjwjN9nK00sOhKcqFEpJEmN+fLzPLnav2O624rrxBeFZWsLqLyfqHFlC0hSj2jChySDAjSYA03tDDYrSFysVsl2AJq0N1KU1+cL5rFN0raQ5lS5CtZ0EbfMAzYncRFbejzjI6s1CwlCdVEJKhp3ztr99iV733eV5K6uoV1bU5g0NE90+O33RY9hZhlfXVNUSlhsZVHKTvtsDOgOhtvDUrKSxXJQ6IbXmKpUoBUkSpUHcEiNdYmfEQBQ0zKHUqbWvrIlcBKgE7nmRzIjn3a26qqhVa6tumP0ajCUhBUCkaSBv7R3z5K1z3a51SHnyvrFpzNk5VH9Hv75BOs+Ug2kE17KVOjc/TNYzvOnWjqzUUDTrZLiVJUpLhBUTsdVaD1o750sTVJpL6vVmjfQv0SmSoKKATm0E7Hz0OhPMwIqdwrcTd2Lurp106DU0pKOsAggqQQUx+lAMQO89m1r8A5awB2rAbpmCc+aCVEAyBJ1SZIjXcb729B7KHPDk8VTqjZP/8AOVLeF3Uz1ehDeZttThImDEFQOWRy8YA7tI1xBfdK8XqOnpEvOOLWArKYHidNDHeJkWW2rtvy9iuppypihrkN9SyhspU0gAmQRCiSI0J0k+RR8dVNwcMcNuXleA6yrcT1NLTgpK1rygdkGI0A7RiANdtNypmkdckWaOagI25d9VSLiPdbeGOId50DbugeFQvtGEKWkLKfYSfhvFrUfk7uH+FcWcQ7xxrX+k+n4QpGnbvbbcKG1rf6xCnFxqrKlMBHqjOSZtTfF171974pvCvvRSPSH3SV5EjLI5A89onbTTS1sPyamMGbu4x3lhepqEpTfdxuIaQdMzzC0uADxyl0+w+VvI68hz3lu1ytKn0IuvStaTBQFFMiNeR+e/u8LSdduLxWXVR1KHT9MwhzXxSDaNCMgBEgyCD46aeMWJ4ZvpaLv9CW7rTVNTTgbdlDy0p+CRby/to6SGCOSM87LrcDpW1j3Mdy1UtG/c3+E9s2LO3zAgL5d9mexeSyJKoHfbtNY85JSCoW8xkrJSN10wwtjN04nL1UrdVij1apeYlVm7f2JLhwpRivxXflFdNOQSF1LwQVRySn1lHwANohr+lzgOsvoYX4fUL193k4hwtvVU01NmQhSo2KySEmAQmbEgpKus1Y026piyKLQbrzv6Yt0fmji7jekEjq78eqkjLt1xzjXfZwWrKsdqDv399rFdJfGN5Y+xteeMb6o6Wnfvxpmp6mnRlSMqQgCDKgYQeZk5vC1eH05XlonZR3Nve8DD20TGv3AC4HFLd4cQrJdBTjFe3CPiPfTl2XS1eC75uZdMELBOVSHULCozCdAoazvtb1N4eYrxxxKwVc9+Vrt30jVe24l1BaDimHWnChUJgIObRQlJIzRJ0t4zdHV15PFy5KZh7q11Rep80SO00vlz1j229XOjxX3hcl2v3LUXo7UUiFmoaQtIytrWAF5IEgHInSY08TbmO19I3Nx2j0tLroOzt5Yw0C9id1NzOELnbfTW3s4/elSmMrlY51gR/RT6qfYLKq6plhIbYQlCQIASIFkdy9JTmK9vhYhUXmn1s5PlztwJc9xXVMpC7V6Vn7wzSUka87Enq/1hm/R7rINRfCEkAk9rsnz+frsSVfaC4EAqnNB5HW0mxPKtsgY1LD1eqTqSR3WqN+UWo/ztweua8EpUtV3YhpyqNIQ408g7eJTaxdTfQjVBPgBPvtBXS8R+fuAuJ2QCXKJNPXIg6J6l9tZ8jlzW28FDoK2J/iPeqmLRtkopGjovKtsAPJSoT2gDNrNdAG/wBV2cW70ulThCL0uV1GUfpLQ62ofDPatd5t9TeFQ3y6xRERsTI202i0p9FW+zcfHvDdSFZU1Dr9MRGh6xlaUj3lNvU8Sj41HI3qCvL8MkMNZG7xXqdhq9fQMV3RWKcgN1zBUZGiS4Ad/A/Za2qpA2I9tqHOX640tK0HtphSVJ301n5H1WvRdNcm9rno69vVFXTNPp05KSFD4G3F4Q0sDmldnjBDi1wQupneLcqGm3nNhCOztHdYLUkHXfW261YTk3qxnqX3GynQGRYtEEjeIiypfrZS4hwCQUxZIziIkmTNtmE5mArImblcViwEZFAz9IgAz3mPqNjSgd9ZixN5eancCdwkkRyIEj42HKgrtDZWok99rTFVcu40BIHutgneDrbgkKMgxA7trYSdYsZpUUICAdZ8JtuZGxjxsHME2yVAdokdx7rSG6ZDhRnui3STpANgJ0JNugQNu/utMKKDrHS31D5/wbyc22gOh+uy1RJIdCp562Ra+V0jqB62Ux4EWVbtcDwadTssBXnIs3NMUvjqoErVP9K2WDlfIfA2yzqK8I8Usre9BpkwoOXjTJCjuMzie7zP1WtJSOs3XetM8xULcFI+h3MEhMhJkEe61Yr5Qly97kbAR9LfVIDykF0ctjy377WQCmOszdYolMqmJkD6/nlZqUBzXAoNY3MA0p08SL2u3EGPb/v7D7i3aG8KxdU0ooKT2xnUCDrIUV+4mzbzFCM0EJmYPl99hqZ6mZUMyUFJUO0E6mPkfXYBxQD6gwJEgoOghPdr7ff77kY4bQwclQbHlbYJx4coqN+uaFW4lLZIzZjoR9fLb8bWH498MuE+EMC3PeGEr4bfvKpCT1KX0rLqCgkuwPVggDyJjY2qwzXhtRWhWVQI7WsjaJ5+FjdRe9ZUIQhThI2AJ9vfpP2e9nMLnA32RGkAEEIjWFJWpCI9k6ifAfM2JLSqZIMHY7DQ/jYy48V5SlKe+e87cvnlYF2FKUYBEbEajlHzv7LGTIs4V5+yqOUyNPHxs/sBXj11A7QOqJ6hzMhIMEBXdr3/AF2YqilQOaITsAr5+TZWwreKKC+W1OrCWXQWXCZ0Ctj7CB7rAqWcRhVmlfkkF1JqkrcGYwAdIB0kfO9kLFb7X5gr1OtlUMrJCUnSNPxs6G2iV9pEmdQNgQfq+vTvsFe2GbsvW6a9VXdNRWFLK3HMlWtgKRl1SYI08Rrbn5ZAxpuuljhLnCypHilttnpH0IzAkXvdJVuI1YOx+rwNvZil1bBIHnO9vIDiu7d46UVGLpudu6mRed0AUyXnXgDLWYlTiioqgeWoFvYGlT9EB5672BmzMafBCILZHDxXakwcpPwsWeQY7MgnvNjkaRqYsA8kkSYE87Ti3Q5EmPNFIzAeJsSVnBnUjf6rKTqVEEQdPn7rFFt92lrrNlSduo84xj/ea8QkFSahpQBOnrRp77QStxQBIIKhHIiSLWSx7h6vxLcDtz3dk695aC3nVlHZUCdeWgtHLXAnHD0/SXclJ0AL6jA8o87HhkAFihOp5JNWi6jVh18dZGXsgdkoM+tvbDnUAtcHL/NB+futKTHR8xnParbsSSP+NWdv8WxhHR0xacwXe92oBImFOGf9HWx+I3qm7pN9FRM2opUlSVARPaCRGm3uNrF09Hiu8MUJpaCmmiXSF+BSLKCMo7RfkpDmc+pHq62aTfRtv8kl2/6FM6wGlGPq8LWMu5pdDd9FSOFS1NUzbZKVECUpAJsJ8bah4dfZDkpXhtpBZQjeeF8aB8tLpqhLcyT1MyCToDBB9/OzD4pUlXSKw67eFE6w56fol1KgSA9T9421NrbNuZ9UtLI785Fo348YNocSUV2XhVvVDSrqS6+2EEQVBTaoVM80W2KjETNTd3LfrWbR4KIKnjNdfwUI8U0pZbpFhOeM8iCSCRMfC0Zqq9SlKI1E7jf57rWSxnw+uy+EU4q6mpQE9o5FATodNvE2aaOB2EOw65fFS0VSoJdqUpI3ExlnkbZseRjbyED2rRmie3VQy4pKkhwEyNddRMiB8bcIqnUoCSYA1mNfnX67TRU8GsNIWytmvU4zn+nK6hRlMRpCd9rHLr4K4ZvOo9Do6V19Z7QPXuIETrqQARPnqbaDKR0jOIPm9VTEjS7IHC6gw1D/AGlFSSZ1EaHw1toVj4GRxYy6wSYgefutYw9G2jVI/NLSAYAzV6j7YmwzXRwu1CMpuakWZ1mtWY133tAQs2LgjcF/VVmq6gi9sLOqUJavlade5yhqRqOXq2vfQVV6PspdpmaYAoQUlalagpBmI8fhasHEPBFx4bv/AAnTU1C0yUYnpm30pWSDLbzZ3/px7bT5dtRxBqE0a7toblVQKaQEqffcCw3l9bKnQnQaWoSMyyEAq62Mhguleoexe7VBafQUoBlJJdOk+A87VD6WNS7cXTL4IX25RoD9YigpXXBJTlTXKzR/l6z4d9rW1h4mIW23SLw602lJWo9Q+dJ10I1PhOtof464WTfeLsIYnxvdty195Xa7/uLWMsvNmnUHUKJKFHecp1nnYEjAxhJd5osERc6w1U8VDvVVL7eUQVbEAzz5+djV2sMuTVVORKAuAcolR9gs0X7g4pOFC0YvuRKzq4Dd4USdNRKhpY5S0PEhthVLW44u5aVtlttLF1IC23FbLEOQYO4Omp02tF1Q5p0YT5fFFMIt84Kg/S5o00vSYxPVIoXUofTTudYpIGYKp0SBPIkWgXjFT1Jvi4aijbUWjczLnaVEKQ46ic3eQIEfs9AONvD9hWIKiuxFQ0V63p1LRXXLoG0uLAb7II12iN+61d13FSV1x3a4/d1JUJ6hxhfpDYKzlecMAqExKtPxtzxlvO8HqtNtGcjSDuFCd93hUXVhusvOiKevaYKmY1SFkAA+MTOm/wALV9qBfy6168r3raUqW4VuuOMoWsjmSowfDkeWmgtdrjHhlpvhXeSWaGnplCnYeLzTYKm2wtClHQSOzI8Ne42p/fVwVNVUMUrlO56O0cxAb/lFbgRrp3DbWOYnWw5rcpI3VKvaWOAKY1QqgvCpddprsUUgZisrIBA3Mew6eFj9wNIcoVoUwSpAJIAJggxJB8ZPvso3y5QXeUsNUoS87CENwEFRPOCOzqBqSDrHLRDw9WJDwS7MFSpCdSJIJ13J0Hu57W0LWKz+ScdNQ+ku9Y8AhtEgI7QGmuoOk6+HviyysBRKtMoSFDdMgpgHvIEEaEz4bFMYrHCtA61LZIBWAqFakQZG2uwgTChJ2seQElKGTKWiBkGuUnllgbwk7aEADYCxBZRN04+G1w3Te+Lq5+/KpDDDDDa2lJfyKzlagRzJ0TGsiCPZaK4MYYRo0iipj6WwQkLzEhtSNSdSZMkjfeCBNqS3/eS7vvOhCbwq6dLzayXKdIcdCTAgBZGhKZ3j1tySbG2sSVq1oScT4gdDeV5X0jTGTtR+gTpr+smNdddN3De0BwqLhsZc9UN1MJjdxV/brx1QKonW6Bhtb7CHEtU6SOsV2iG2+/wG/naFMcYSvG8HnMW8UcZXFh9pw/RM1QdcLaN8gbAEqhXgfPSI5uq/qfEVFiS7buum7rvr6LD7t6XXWUtbea31VDTrZhPW1TzSjkUvUoTqNCCBMJ3nRYovWvXU361eFYXwWUPvKKvpCgqRC9jJExzE+dpP7ZCtux8e3jorT8P4LQWndO7iZh3h68zeF7YYv68q2uplIIU5RJpqdxA1KkoKlLiJgkCdNNbI/AviJ+9dxXwxjcpUW7qvNh19YKp9GkofATscza1bidB4y0q/DmIrnYUalhxtBjOlCsw57kad/Pv8bJLSwk6xbBqp21bs4aB7FBoLF79NOs1KE1VK8HG30hxpaVEhaSNCPAg/hZAuy7qoXleqgclO1XKV1ijlQApCHDJOg1WdzaKOhJxJVxJ6O2HnauoD1dh7NcFYrMSZYy9UT5sLZnvINpbrcDO4lvZVTWX1UNUCCn+LNqI7YSJMnb3c7ef9rKdstHZ5tYrsezcxjqSW8whrwx1hDD4S0/VuV76NAzSJkTH6ytI8RNo+4pcSeKRwdWX3hO6nMPUVCApx8N5nFoUoJHbVqBKgeyAdLTBcWDcJ4fANFdbJdGvWOJzqnzO1t4t/N96XLV3TXUrT9NUtFtxlaQpKgRtB8becxCnp3DIy/tXXuDpjYrzqvp6+8WuOvXzelbeda6cxW6tSidfHX3z426wxgq9rovOnvtJDRolBYAkQlWh58gbS9edwUVz3k9T01IhpLbhSClEbHT3d34WSq6nULsqmgQlXo7gAjWcpOn7bdtAG5PRGiyZGZXaqrnSZwX+4+9rtpG3VPJrLsLrZV353JHsJ+Nq2VeU1CynadO1m+Nrn9LSnReOHMDX6UlanaV1lxQnl1ZAJ5aqNqaVzZS6REaQfMafZbrsGfmpxdchjLAyoNk5uEF5fmnilhSvzZQ3e9KlRAmUqcCVCPIm3rTgGo9CUspBBUBmjbcfj7reN921rtBeNLWNLKV0zyHUEcilQI+NvXvDFe260w/TlWVxvOBO4IBFs3tLHna3xWx2WksXjpZS0m/D1IVGhhMTFgKh+tWx6UGHVMgAlRBj37WZ1XVXy9dr7lw9S5XJQVtNujMhZEkJInmB5T7rI9433jigvFxVLUNVVH/h6uoLLSkNhaVPdUnPC0dT6pI/woJ9WLcL3UXsF3PEsE76q8gmSCJAgmNx7bEfzpkWFIOoVmgHn4/jaIf3S8Q6qncR1zTRYdU3UVDj1MG6dsPGVOESUOBGSQREKUcoIAKQ9iniGtvr6a9rs6kNNqcLlRSlIUrKVBBCthlcAnWFzJjS22ksEEzhTNeNZlfcAywHFAeABPf8AOtmPxRphfvD7E9yLSFem3RVMpBP6amVR8SPmLaw/eV9VNEuovd4vLW+o07xLZLjBCSlX0RyfpESOQBMWHfqQUONOlJSoHMrc7eeosaFnDeCORQJ38SMt6heTl7piqDn/ABraFzESY189Z1sucOb0Th3iHhe/HP5OlvKkqFTp2UujN9RsSxddq7pvR67XQQuiffpVSNy26oWSs7gTTvAkZCUgzsQZ/tW9QsJI7dQvKNY5r9CvUOpqghTiSpO+sET36jlzjW14uC96m9+FuGq3MFkUCWCZmeqJbOvP1beeNx4g/PWHrtvj9GtomKk6zOdAVy5yff7rXd6KN9fnXhQzTFZJu28aml9hIdH/AMW3D0gLJi0rt613Eha5TEtwE7e+wQKQrQe+3SwoKI5mwZVvuPhbYasZyK3u2HqZXMoEiLNtREHczOhs6ljMkpOxBmzYfR1T6kRrNtKlfcZVn1TfWXAyncCJ2n2W1TlQpmgrVQTlPiRoZ91tAlOidI+fstjM5Fo07KzHt7Q+u2gxZzt0Mlz2+PO2ZioEzp3c7cEFIy6xvvZNxTeFfdeG72vO6mmnKykoX36ZLoJQp1CFKSlQBBgkDmLHa25sFAm2qV9VcjoeetsJnUHbxtSbA3TR4m4rxlcOHK6hw3SU163lS0b7zdG8VNoddShSk5nYkZufdrpYPGfTM4p4dxPe1y3XS3G7R0tW61SvVN0vsOvMpWQlakFyUkgbHuO21tIYXPmy6Kl8oQ5cyu8hRBA1+q3eYnUEbWi7o9cTb04rcNaXFd9N0jd4elVFLUCmQUthSF9mASY7CkzqdZ8rSYF65p2tUkYYnFjtwrLHiRoc3YoYgEEE6fZYzhslVM0kgy0S3EbZTH2WJhQ2E8wZsPhzs1dWxIOV0OD/ABkj7QbD5pynWA3HrK9ivxtlszDnr4xbLOorwsvFXWYiw4JVlcvqkBT4Z59v7Da1eH+GeNMUYavbF9z4eU9c1xNlyvrFvtNtsjLmiVqGdUa5UAq1SI1AtVQFbuMMMJkEfnmnJzbEAq2/C1hRfF4GiN2+nVKaFbgeXTJeIZWsCErKNirlJE+O9npAcpshVRFxdGG6tpKx1uciYGVUEH5ixiorbsLY6lnQiZWSZMb+/wC2yQVA65lGDB5GDpYZlKHSnNUBEKIEp0iBr3H8LXSFUaeSFDrZzJUoaCUpjmfn67aSsqJmAYhPf7PH577GaihuxulW8xeqXHApICQNVAgmQfD7T5WIFEZQCFdYNgQTvpZB10ntyp7UHC/GN74CquJV2XL6TcVA6tqtqGXm1KagCc7WbOlPaBzRABOwszHXlqBKzKSdiZB5SPnlYzR1TjdMqk9OfYpnyOuS2uUqCTIzJBhUROu1k5TkLXmQCIJOsTpz7rM24vdLQrvMZKiSDMmDvHn872CQ4CvNlBKZkjcnkfnxti1Zs06lRGXs6H7Im2kZcojtCD8z3WkdUrEaqdsLOi9blpb0Ha61pKVymTmGivv1Nluuu2pTc1XLa1kMKBSdiI+fKzJ4HXgmoVW3G8U50gVLOY6qGywP9E++07LuB96633BIUWFKEwAogHu3/C3D4rP3V7mFd/hMXeYmvC83OMbD6ek7Sek9pabwuolMTl/kjr8PZ5Tb2EpAOqSOUW8lukfSJp+le2IVCq251KKiDBIamIGomdPE+31rpRmbA30+FiQvzwRuHRZ1SzJUSDxQigSOcROnKxd1IMgjaxowRBFi7oB5R7bWY1UkRB5O55/GxZYIgkT56WOvDTbflFijiASR+21wGwVN261Qsh68WG1AQpUfCzk/NSVGECOe9krDrDTt9UyXgpYJVEQDOUxZ9IpGNBkJWTERanNUcF9lq0VuHqm4i6ZgTqNd7DC7skqzgQIs4HruS05lKTMSRG1uTQpUdEGwjWm+quAsOoTfF3p7U5FTzINjXVJ+jSArsJjSNvabKvoKpASm0W44cxXT4zoLruW4Kiro6p0IqKlJcSlkQmdQQlManXeI3sSCukz5Y25ifqVDEGs4eZSIhCQoFSkojXUo+ybNziUlqouBwJUlaksuzCgY7PhaOcT4UxwtKlXQiscUqBHWKjfkMxO1jWErkxLRU17sYhuqoZb9DAaddSe2rtSNSQTtbq5qICgNUXgG3zea5iixN8lc2mMRAvvyTlvtjNS0yy3OcA7eFgH8KXxe9Bdb90Uryx1mVa21NJyArPbUXB2kpjVI1Oay7UMJqLtoysBUoQdRMdizmwylCcOsBStA4pIPhNsBuSrAjcdtVv1cWWPVMPEPDjELi89E6tQCVABt3Qcwcsg/XbWCsM4muG9yq9LscRSKZCRUFaT284gGDNpUSloArCFq8YP2W4vVE3anqgUlLrao2/SE726NmJPjpu7ADKubbhUTagTgm6a6XllaguoCsuo7f+1bhl4GtzGqbDaBJGcafGxJ+iqTWOop7uzoE5TnIkb6zbpijW3UFTt2oSBHa9b8LUC4laoCgTjW91uKWQptIFJii63k6DTNUto3/wCk+Np3ww4+m4bufQ0kgUyUEqI3H7OXdaEOMNOanFl5UiArO3W3TUxGgitpTPj+laa8LpLtx0dOsKhlx1K9YgocUmI8x8LZdPViWpdFfUfoWnVU+Sna/wBiVKt91xoF8tMgHKfpNz7rRP0gW2DdWGK5sKc6m8iC4k6JJA0nxg+60uXilostpU+4k6qhJ+4WjTjq31GAKaqKFuCmvNleZUyJkD2WfFr90eRyQ8N/jDVIFWg5m31L/lUggxPIb6W7YUgwFOKUrQCEGPqsBTVpVQUb6HSetpmlhPVhUSkbGxxqsqVwQhUd4SRra1GbsBVaQWcQo64y3RmvVpxPqv0yEqCzOska/HS1asH4fFdd6qAuNU4ZvKuaUsyUpAcJkgRrCvaSbXJxvdibzTSvrpVnIjLtJkEHW0A4PwA5TX7iK565ioOS9XahpaUz2FpQobGNYjXnbzzEq4U9fM08l2VDGJaeJM1zCtXelz1VCqkYdaVRuBTTmVKXUZCChQXpqNIPfFvOBpT1MVrUHepp/wCRaSlShHKIHgB467k29f6rAVOsKbWxVLJTOiBMeJncTbypxbhGswzxBvzA7NLVOPUF61F3tsBpSnFqDpSgJSBqTIgASZG+k7PZvEo6tz2g6iyz8fpjGGOCh/FV31D49Pd7DyO3KBBSNOydZ07/AD7jZu3VlDgSUpG3aIIj2jztYrFPRd4+1N0mtc4M4sKG2i7C7qdQ5HOGynOdTMAT5xaujNO6xUO09UHWHWllC21oyKSoaEEcjMgiPw6kPa4+iVzRY5vzhZPi71pQ0nsyHQVhJVmlKtACNyYOhI29kHVVrVOwh6pWSjRDu8akpiBB2APM6H2Il31br7AaQsuleUBA7RiBtG86eOo1nWytT5HaZXWuL0gKlJSqOYkgeHge14EEO10Mbrinuy6MT1jL9TT1FWltsNDKsIKiJJnukqk6AkyZ3BfdJw6wKC1Uru4IQCleRbkkpkHXTY69/dIgGzawjS0ragUtBvNmWWgUwEiDqDPZ20PcOZi0jNUSq+ncS1UOMrUghKUiZXBEdnQiSdfuE1Hu1VuNotdPSsxNiZpgXJU4mxFUUSUZfRHr6q3aTLA/wK3CgjQH1Y8BIBa15VNbUpVTM06HX1ylZcQkpBAA0ncT7QDIGsFaVQsVGZxdKtxSu2UGoIIntQdZn1QRE9k+1fuW4WLv/j77CGXMoKA2VEJSnfU6zGu40I3MzRbH6eiuF4DdVD2O8Fk0tPdD9a7U11e+X3VqRo00kZin2lSZB5DTYi0PY7wevDlSHUCW1mCMuUDuI11BIVr4WsZiDESfznVN3LSi8b2dSGwAQUUzYOpJE5Tvz0jXURaPMScLr7c9IvG9r0Ljr6At1pKRlTpBg+3TTaNtBa9G62ipyMvqpb/Jk8TFXFxKv3hlWP8A8VxRQelUiFGP45TSqB/SZU7J/mJt6VtVRplZUkgEZhpAnn9lvErgHixXDjjxg7Eq6gNt3bftOioXOnUKcDbuvcW1Lt7V1zT3WJDLa15SQqBMRFue7URZ6J59n3ra7NyBtY0Hx+5KCrxMdkySZ3599m9iG9CGlJUdBqDO/f8AbYRp0rdNNIzA+qDKp8vvs3sZpfoshcQtPWoIlSdd/wARbzKCIOcAvSiWsGii3FzQF5uupJBc7ZHIA8/dZoOvMZ3GVuIyuJI1Mez6/afGzb6V13X1fGFrtcuNitqXmq1C1t0ra1kpLSpJCe6B4CfG1YF4Oxo+sJXhm+1KCVJCjROkHXxB3+7S3dUFG2SEOc6y5DEa10MxY1l1MfH5lq8eCOHa1PbN3XkunJSSYQQsa66jsga8xalt7oW3UqDhJUFKSSdzBi1wqu67wHRgvO7LwpHaepui9s4bcSpKwC4ndJ29c6Ac7VHxE0E1jxCYGYK9UiJGxHsHvNtvB/QzMHIrAxe73NkPMBI6SAqSSdot6ncIbyF48PMKV/W5lVFz0ZUSdz1KZ+NvNDCmBMWYzZvFzC9zLrxdbBqKvItCS23B7UKIJ2O08rX86MV+IreCOFXFqUpTLLrCir9Hq3lp/wBVKRG8WbHmh0II5FWOzryyZwPMKfbtrmUEocqAwFJlD2ScpAO/hBPwsz77wHhWrK6b85VNSy448tTSUpS0kurbU4kApJIKW0txoAgkAbQdVeCusUgZd1DcEHz+Otk1+tccSU55Ex3TuY12H4eFuPEdjcLtjNpZc3PcdDh5Tymap97rm0tJ68pISkOOLiQNe08s6zoQORlGOEbmpnGXaaprEmnZLKBnSYCmGWSSCkgyhlveRI9lj7lUgwkq2gZpBPv91kurxFdFIVmqvCnAZCQsFwCCowJHcZgeJsUMdyQXStCUqNTV03bS3bTKcW1SMJZQXInIJAkJAGmmwiwb1YtQJCQO+R2iZjvmzOxJxLum5B6OhNXW1LTxYU2yiSFlAVImNIJNkq4OIir4v1y5ai73KZJW4GnCvMFKbPaBBGhA19tpCB1s1kB1S2+W6pzx8utF2cTcTMNg5fzq7UA8sr6Q6B/pH32j0KmjCNew5I8Mw/2RacOlhdPofEKprQjWvoaSrKhO6StonXl2E+8Wg1rMW3UjUABR18Y+23fUT89Ox3gF55XNyVLx4q73BG91Xlwow4+V5lN0xp1Cf+KcW2B7ki14ehXfHXXXiq5Fr1YqaarSJ/XQpJI/q028+ei7V+ncMCwXCDQXi80BMQClC9+Q7Ztdjoc1/onEO9btJ0r7pWsDkS26gg+wKVbl5miKscPFdTE7iUTT4BXFWSdI8NudglEAkA6eIt2o6AKPhtbkiRJ3Ag2vjZZzkGdNthZCvhrK/ngAKE2XSI/R8tLJt8NFxgOTBQZtapnZXqtO3M1IgVqNt7aBKH1g/pZVfZ9ltgkaxGnwty6cryVSrtJOvu/G2swrIcF2Vnz+r2W04hDramlgKQ4kpUDzB5WDJJkgn3WESc2hOu3fY7TY3UDqvNTA91u4VxvjW7rto2ajElw0da3crbzQcipYqUpcU2hQhTop0vqTuQRI1CbLF+Ur/HS4b14gfmmiua+qKuyl2ovhw094IUlbrrbZq3lBC2dVhCCE5FFOUEJBc/FevwfgvpBYkpMTXBT07XpCLyo7ypWX1VfXPdW7Mh5KRBU7BSBqlImZNieIuMXDLGnUOYrVfN5GmaKWW6mhDwbUVEqIK3TorsSQmSQSSdLde17nZZAOQXOuY1t2kqV+gZfQewjibDqjmFHXs1bYnTK63l90tWtMk6wSbU36J+ObkqOL17Yew/d1LSXXX3UXacpo0sPOONlskOZSQYzOR4WuKlQjsmByj9tsLEhlqCbbrVojeEAckMDzAiw9zrUm/C3PZepwR5pVH9qxUHcETpuR3W7p1lu9aB7NCesU0rwzJMfFItRVtPArIJAbVp3AffbLdEH9Yf5NstKygvC2jn91+Fm0qnNebapJmE5SZPttPgbX1YCO1mMlQjztD3DW5qbEfGPBFw1ZIZq70Ofq1QSEtrJg78teXntaxmNcGXhgquRSLqEVdK+VLZdUcqiBGihO+w8dPKwaWoja7hE6lTqKWR7OKBoE1ilSTOTSTy1B5SPn426QlSiUFJIGkjmNefzFukdamCRuNf53xnvtuXuQSQezOuo1+zxtpLNLSF2ukrGlNNqbAU+2FoCXAYB2Jg6HnHn323+b6tTiGSRmVBBSJ325anXztwFvRmCiANSBy7vL7fG3Sn35CusMADUmTHz91oJcl05QtNOqCllbaNZQkkj7J+6ytQYWp36eprW78uyKNhFT6M84426+lRIKG+yQVp5iRoRE7WSg89kyl3UGCJ0ju+d5t3mcy5+sHa00Hf5+E/XZrlSYBui/orq3DlaUmFT6umtu2rvfWqepX2leUnwnz/bYTO6D/KQB2TrsO8/H3ixqnceCuy4qSNY0iJ17vk2i59gitYCU6uF9BfVBjG6rxprvedSioSl0JIGZtQyq94J+u19aXh08ukIVTglaYEJiPZamvCC7H6q+GXH75cpw24lSso0UAQYP3fI9KaTq1UzSkAQUAj3W5HFKEYrU5C61gukpauXDaccP1ivGjpp4Uew70vW1Ip1MIcduVwKnNnPZEkctlf5NvTWnT9GE9w5DlajP5Supo0dKDCrTYAeFBdmeAASfSnSPPQAe7uteqnTDQMb2eCIQRCH6OnkmfIZjxHbnVdkaaGdRzsA6FcxubGSAE6WLu6KkGfO1uNAk2RJ1Mjl3WKqA5GZ5WOPAkGD7rFlacpHdEWtDZUnbpawXRLqL/pgEkhOZR8otJ7NyUzDgfbR9IPVJUSB77MDh06lu/wANzq40tP1H7LSlYfdo5XFzt1MyuYMrSk83aVKzKWCTzi2/zYn9b4WP21ZCgh6KPGf1Scq6hmzBZFm3ftG5T3glAOixM+HyLPazPxklb1WhhDhbJSntp3GtjU9LHC/M1M6VzxZyIoYfMkL/ANI2BvlhhV2KaqX0ICuyFKMCToPrsMylI1C839Ll8LIHER5LWF3n1E5W3WlaAb5h32JWXdA8A8kSlIErT4o3WUNPR0bNK64gdUhKVGf5thbpve6Ke620KveiZCXCSFupGntsycUYuSjDf5wIkuNsn1huUjb32g/91rt4VaqTq1JbWVQQqduX1Wy8MjeHZyVcr5gW5Fbqgvu5bzfNPQXtRVTqBnUht1KiBMSYGmpFkS/+JXD+kZqKB/F9zN1dK6hLzKqtCVIUlwBQIMajWRvpas3Cbig7c2Lr2W/RNNs09K4yt3OZUesbUInwGv2c6t8bcVJr+JuLqZKHMwvm8WlgBJzD0hcRrpztYkxBzah0FtAL3VdlLeISL0lVxV4fpa6449w6lDmYoJrW06DnqbJTnHfhC2ShXFLDSldzdahcxMxl8reQ96UHXVBKCIkE6GPdzMTE/fZWwut2idbaSy2Uuk5S4BppHZJ+dbNJiT2i4ATspmuNiru3njm68b9IfEgufECLzuP9zrVVTqaI6kOsOsLVGkk9k6/eLTvcuOsPXbelfhyovkN1jV93gyGyy4oQp9a0gkJyjRQ3NqEcHa5268WXqUENn9z95pMdoKHo6laEHXUD3+++PDSuFSziq8W1gqexC66DodFsskfXbJoXSOry9thcFadU1vdAHa2Wrx48cOqSodbVfF7PFtICBTXHVuSfAhuOdo16QPGjCl7cLqq67vu/FS6x2opn0KeuOop2wnOn1lrSEiQRuedp8C2g4HesWHNVesDrZldIS7kXjwNxMtvrFOttNvhSpUMyH0K1kwBp5C3RzRvkic1x0t0WPFII5A5qI4d4i3gxh262K3hdjpVRS0LLLxN2toTnSgAmVODmLL1Jj69KlCVM8LsUQVgHrFUbeQHSTL+lnSwp2opKdQUShTKFdknWUg66GwoYK2S0WCpJIkKXv7ItFrHgAByTpGk3ITvwQgXncDdbeV0O0bzilhVPUqbWtsAxBKFKSZAnQ87FLnuy5HcZ37/FGi64GHvVifowmfH1bF6W/lXax6NOUlAUEyOzpH2Wj/DWMHaLiZXqdeJRUXe0AknTMlS9e/YC3D4vilNFWthkaDlN3Ega/tda1JRTvifIw2BGn3/oU3KuK6TJNG3rqdLVqrOBOBbg6Tt+cVkXEy5etfT0RpVLMpp3C04h15tJGi1htKSRrAMRmVM1qx2iNCO7e0d8UcYt0KafE1K3lqW0Kpi9mTlSCQpOYH1josJA5q9hnHjWHSytjpm5SeYFr+CPRUNSyS8wuP21RPF9+12IcMX7T4NBqb8oOup2WWnG0qTUR9Gkl0pROqdyE672rB0m+gffHFrhLV8RaDhYm7+LNFQtVVS5SVlOgXstEF5Drba1NrcKc2VQhRUEgqI0tZO7b6YxFdrDNyVNWqvU4atkodbCKllcFbCFKJyqCpWmQRvqJ0lm6sR1y7kpUXjQuCs6pIcQ66jVQGpBBM8uVuihfFFdxP7dE2INkIDQPCx+9fODSrdYqy1UI6tbS8qkPp+kTBiMu8gSIjmZibLdFWOvZaJtMKUuChLYTBI3nlvGvIjSdnP0ob8pb36SHEi9ab0ZDVTievXlpkKS1mDxkwomVFQUTyzFREDQNLDLLldWpbZQFJbTnIIOpA0AnvCTr3a7TbYD7sDlz2Sz8qkbDd3sssoFVUIDQAlKF/pQSo6EjUT3erI5w+aaqoqWlXVKUlDQQlkpJgulQBGg31JjSN/az7mp1LyjrF5MwS6lRSEpRsN+U6g9x0Gup9Ncq8cQJuthK1sUSOuWjftgkTMbDXY8h3mat1ZGietEahCGnmyG1NIaTkACQ2QO0JjYEnwJGmkWN4gvFtF3uKvC9jQ0Q1rq4LhQSkDstyeajpoNIOsyGy05ezTSxdlyVV4LcWtDLbTMgKBVur9Ae5IkkKBIFmheRq78v70W/gjEt6tqz0uHbuWpdNTCfXqXEggwf0EknvjWzNYpOeE4KC87zxEytzAtGjDGH215Xr6rW89TUKMgBsRIJB9YjTNuNrJOMhfd0YdvGgwldSFdYhwu3jVupqH6lvUqU0RoNEnygxrrZ3JuvH1Ghpm97yvtzIgJRd9y3c0inZEyEFboS3AMczzJGuo7tK4pLt4Xph+loaBRSl1LT6qt2QNS4llKm0zESQBI3myddlrJM9O4Kp11zvX+kZyXc2fNzzTM2+gXo1Ymu3FWGcPYkvlhh5N+YcorxPWJCwHVstLVE7arVbwVx1d1HdmKa+nu2hepKIuldO26ZIQe4jcTMb+dvYHoqX+810buHd6NuLlq4GWM3gjM3Hl2fgLVcWJ7o57BcgXUqCPPOIibXNlcaiw3w3u29Xb7p7rp01TpJJUoqQknfKgnKn2AWizpQ1dyrw/c7l3MU6HBVuIUptsAhJbOkjyFm6vHb5Bh477T8/Isx+JuJHb4uZppx2QzUhckyNiPn2d9vOIq6ea0LmNAPQLt4sHMEgndISR1Kji8Hg7TtrcO6CDIg7nlZCfQUqJUgKy6DxE/YbKIdLtKUEKlt1cSNpgifaT86WIPtlS0rVqnaE+Pd899tiK4FkeU31TAxtRJqsE8RbqCRIp264CDqQjNP+hahOI21JcUSVdpKViQO/XW1/MW33cVy1t+3df9f6ML9w+6yyktrWp1acyCkBIOwUPkxag1/ozEgA5gjU666j2d/wALdBg5NzdcxjVjlspO6JVBeF98RKrDN316KNy86AozuDsKh1slKvCCfr2tPfRGqHBw8vK5FmFXTftTSwQAcsIV9ZOnnam2A8XKwTiAXyKP0pBYcYW11mQkLESFQYIMKB7xa0nQwxB+dV45SltxpLlezXoazZsvW9bInn6iRMaxa5isV4HO9iq4PLlqGt9qsrelcaG737wCOsDDanCkEyYTmI56fjvzilziXfTrbriruTSPNt9YQTOziZ33BbWDO8zaVr0pzU3S42kCH2VIjwII08D52ZuCLhwfflKlN9Jb6966aohQe6s9c3Tktk7Tq2Rl5kczbkC5sTS9wvZdFiVWaSPicgoora3FF9Xa9XXrflYGGFoqMiRkAQt7q1IkD9ECf22T78p10lReqW3nFumoqaZ1a1mVJQhLrc9x7Pd8LPjGjF10L193Ph94PXbU07jrQzyEOJSFqhXMaT8zZn3w4am8HFAiax2hfSJ0V1zBbVAHu25WvwkPaCqUc/FYH9USbxKXbwcv6oBUG6miqnNNSA2ULAjSZP7bOO5qxlePKcNNKbU5WrqkdZ2SEusJOUxrJPyZs2V3U3dtz0l5suLWa2kVU5SAcim1tEp8dQqDZwPXgL8vJ7ENz09WUJTTVCQWiCpSFlChpM6EHy5crPINNEZhN9UxOmJRhV5XHWtt5fSKKqp1KAMEtOJcH+ur32rCx6ykxMoVa3vSyp6erw1cVaFhbrNetkjN2sjrSp056oTr4WqJSlCapBdHZzQfLa3SYS69K0LnMYblqiVZPohXiVXfie6SokodpqhtEn9ILSo/6KbXU6NV7G7eM1xBSsqKpL9I4CRJzMqyj3pT77ef3RUvVdDji8rvERW3avQjdSFoI18s1rfcL8UqpMZYWxEmnfpUNXpS1DjT7fVuNo65IUlQkxpM6+2LY2KMLKvN7FtYY8Pow32r0tMjTTfabcBY2j2W7WkSRYMyZ10PwsduyrOC0TCtJ1PhYCob61paO8WFP1mDNuVa6RuO+xozYoThdNZaci4982LPrhbKiR/KRv3g/hZRvFrq6gzrmM7xFkyvlNOViewQrY6QZ+y2zGbgFY8oylDBURJnXnbM0n7jYPODsfDa2wsESAd9TY4QU0MWcHOGGOb2N+YtwbQ3leCW0s9e9mzFCdhoQNJ+dLJ7fR64INGBwzuKd+3T5vrJs/yZ5nv2526zAmQQe4WMJpALBxsoGNhNyE18O8K+G+ErwbvfDWCLnuytaSpKH6emShxKSIUARrqNLO7OJBO5NgUzJAO21ugrkZj7LJxc/VxSDA3ZDZoVBGpO4+dbdPr6mmNTB/i6kPA+CVAn4A2CSde/kAQPusOw0Kpt2mKjDiFJgc5EWXJJPYQoAhI113Fssi0N9UpoqcuuIK+qRmJMGYEyLZZXUV4/dHynNb0jMCIIUUpqKxxJOnqUjx08dLWn6QTfVquuVKSVLd1SYnbutVbo6Xvddw9InCV7X0+lmnaF4ZlDUgKpVpCo7pVJNrUdIe9aSrq7laoFekUzrLzwqmyC1qQIkHQwJ8iPK2Ixru/sNtLLZDmjD3i+t/gofSF+qgQQZEDee/3TbAkZiFEqIgHT5+vmLcn1gRqmCddvw0tpbi0qbQlsZTBUvMNIG2+2se+3S5lzbgTsu0gqJJKiAREzAjX6/neXZgThhjfiNUvUmDrkcr3aYJU6lLiGyhJnU51Cdjt32ajWYqSSTOWSSOYB05d3zrabujbxguXhViCrfv6hqH6OvplNuPMKTnbUPV7CoCgZPPTxm1WrmkjjzRC5/a6ry52C7Qm/ivo8cVME3IvEGJMLqorvY1dcW+0Y12ypWTqfqG9o6U0puSTOsHWDBn5/Za1XH3pO4Q4jYNcw3cNDXIeWtKwt5ScuhEyAT3n3Wqs865lQGy2psqVnJME7RE+3n7LBpah8oJeLdFKBzpC7TTkuUhZIJUnlBI3P2fPjY3SFLYSTAMyBI1+0+diB7SSZJEafPPT6zbfXusV1O11qOoUlfWE+sIAjwEid/C1hxur0YI3Uj4dvxd1NhTahmUkxG8iAPn5Ho9hPE7NThq66ha5LlGysme9ANvKBrEtMh/qyshBQFBQXuZjb2fEWvphS/L3RcNzVaLyZF3IuZpDlL1EuLeyoyuBzNoAAoZY1ne3IY/NJRZZYjY6rosOphiDTGeSov+UVvr0zpkXalDhCGqS5kctw6pQ38zr3n2W9HqUktpkbi3lH037xXeHS1YfUlKhnuhGdSo2Sgx8ffb1cpP5JBmZFi0shmp45HbkXQpY+DK6LobIbYad1i7gOszYzyhI2sXcmfVtdjQJNkUd03E66+Nia99QPDX7bHXQIkqg2JLIk77Wtt2VNyWsGP9RiSiOaApZTrzlJFpd9I8bQth13q78otYPXJnX5+TaVA+ncue61OoqDA+w5okcfEF0qmpSNMwto1Kf1haHeJ1Jxirr2Y/e/v676Khy/S9aAFzzmUmfD6rM9jCXSVfWDU8S7uQJByoTBIkaSEd0++2U7GZGuIDUfuml1ZE1IH6XxsTrnaeW3VttqUpYTJSCY87QVQYB48GrU9eHFNhbEH6JOYb7a5eVk+8OGHFpotu1vGSodRmCOrAWBmVpm0I2J+dwJ+MTub8yydtLc2srBqfpAjRlsf4otXrG981FTeN6ULtY6Wk1DgS0VHKAFGNNrHsL8PMbXNfrF73txOra5hoFKqYIVlVqNe0o7xBkHc6WY+NF1lTfV8ijoahxXXrg5FQTO4I5RNp0le+oLhLpp1VhtMYrOtzSRjPHd13Zhhm7rxLjQS00FLMZU6DU66DTe0WV97ehPGsp0FQSCoKEQpJHJXdHh3b2WceYSXia56elqLoerWqppNNVIW2ohc9kg6bbzNmJf+D8YUR/MF0poGaZDbTdKFVJZap2kNhGQicwOYgAzrHKdLVHiMbSWE6qxLhpms4JFZ4jNV9/3zS3U4mGaR8VYBSo50IzQoAnKY1g66C0T8X6+hTxexNSisp/Sai9K59DJWM/Vl5RUqO4fOxs6nMC4maF61lBcDDFT1C1V79PebqQ4vq+1oE80iIV4d2pXHfCjGlTebNY5hJNTXN1Sah+obr6p5VQg6qTnU0BCiUkmd0iwH1Eb5y4m5VxtCWRBpNgowva96O7KZuoqKpLLfWNoJGuZSjAAjXf5HJfuPDF8OtuXulS0UqT1pKlFZypBKsqUyTok+86WdNBwlxzfDjd3XZwifqKhDgdW6ahXqJBJnMkBIBiNO7ztJuFeB9/YRo7vxheE0j1Ghxx27A8uAnMhsB0mM+q80AD74TTh34uPV3RV3U8cbeI9wsm5w2w9Xrdp8RsAKu+8brqiypxBQ71btM4ASk6iZAgibXV4DrdcwfeNU4nMKmqpqk5Tr26Rjw8LVeuiqvSuvZlSnEejrUtKgEiYKTA3Jj577WZ6MtSf3EuAKgGiux1OaNSaVI0j+j+PcbCC4zgvFt1UqiHQEAqUuyhI6xsiRIlw9/cBZu8YKllHBXGRZUkNs3VUPOdbIGUJkmTtoD7rOOjbfq3kpFO7vrJ2+FiuObjF9cNcUXbVU6m01d2VKQpIkohBgiRuDrG1uscfRPsWM35wuieG33LywtcVZSFSk1V10rudKtDLYO487KP5urXJlSte8kwfdZD4Xv0qcBYbz3klambqp21lSgCopQEydRExZ4IvO7SNK+nHPV5P1Taq2aMAXcEZ0br6BNLiJet8XPdrCsO3WzeN6mnSG6JyqDAcQlxCVrzkGMqVFW28DnaKa/EqKbF7TgPaQ0UKUDGmkDz1+NpVx6blvBNBfNLU0tY/RF+mbcacS4UBYSVpBB0koTPlaA8ZLCL0F4NABam1dpStQrsAQBJjQyfC3lmPRsdizxvf4BdthtxQsPT4p3X5xOFz3O9ejjNQ+Gl9tNOM69xsPaNN9LMy8uM9fetK/QU2EMRr7IIdFLKAdwcwmCIB9o8rIVRej5lSX8iErSciUkJEjaRrEjSIn3WI196vrYRDiUpUowhBgTpqdNSQBqZPutnQ4a9sgkYbWVqWWQ/NNgn/AMK28D0+O2X70xEu5ampWagodIpmC43CpCnh2UrG4SUmRpIm1oKa97ovhlirum+KStpllaQ7SuB1BKZBGZMp0IIOuhtQpDmeoadd7MqGYqOkHYGT2t4iJ1tZ3o15XeG1OQSsUlbVNKTJAGYpXtsdFbW7ihmMjTG8LGr25iJb+C8P+MlSLz4vY2vFht4JqcR3m82lxICgFVLh7QGx1E93fY3w/YaLLtepqO2Ep1MxpEGI5HeJPdrYXi3he9EcXcZ0TVI6pLWI7zakpAIircEA7a/bZawVRC6qJCXylotqCgS3ukROvalUd3cRGhA6nMMgAXMAHiEpzOVbdzUBrFrQVCVpCtwncHWO/wA9R4WS7grhd92Lrqllsv1i3FqJQUBsTIGkaQDE955xZGv68nryeapGnCES2qDBBlUTO/6szr3jv5xBVM0mHw5LiS2zllLQCCYmMyfGDznNJ1sMbopNkauK68Q4wo1XoquqaWhdlktUDbnWVDZUvdQOUiTl7gNyNRZ84awzjinbN0XCV4Yo31BTnU9p1wiE6pQShvQ/pKWqT2lQbRdwiu6qvdaWje17MNNBSQihq1Nq3mITrGpJjWAo8rTTdWDblqlmnfvip69kgOLfvpRWFa96womdIy947wLAag3uLp13Lw6uelSzWX5UVt5vIAV1tbUKdSmNQMubKnv0EactBY7WjD1K+t4XK1U1a+2z1DailsFOhzT4H/JjlYK7cBNNJC6LF1/IS6hRJavNahJJmBJEQUz4gzsAVZ24q+haQzTX/WqddeBLjjaFKSmVdmUpHgZ18J1lSN9HZSidZ2pVZekphzEtPV3TiS9FFdG+hdM3rHVqkrAgmdQY7uxb0b6MlO7Q9FPBCFEpIuEPyRMAuLcHwP1crR9wTwng7FNdiHh5jK5W8Q3df93ocfTeMuGWFkyhROZsjrFEKSQQUzNrJMYfujDuEW8L4foU0N23dQehUlMgkhppLZCUgnUwOZmdzrrajL+MgMZ6FW4vxdQ2QdQo4ryK6qoa1ysqG1ULq3EpZfUlt4ltSMrqRotIkkJOygD3WJ3zVLdplo2SCCNJjXx9tgy8pRgDJOwUfh4bn4eFilYttynWrPIKQqCdQN/YfC3BMjs5eiufmam5cN3U92NV1LRh1XpFWurczuqclbk5ozGUiRokQByG9t3rdlFeKGU3gylxLL6KlsKUTldQQUKBG8HXXSw7S/pnQ3ErbSYAjYwJ5zqfnYvWr6wJypSYMjMdI3+fO15tyVTdYCyhLpJU7wvPC1S024pThqWSEIzEwkGB36n3pFqkYnpFNVFQFtLQ4HlpWlehmOc68t/O3oNWZmMU4XrArKyiscZWSsZYWggEmY319tqZdIW7lXTxLxVRNgZEXo68CiIV1nakHXTtzp79Jts4ZLaThrnMWgs3i+KhgggnXwtZHoQ3h1ON7/u2P5e60vZY3LbqRHn2zat6pClAaakRaaeiJeRoeM1FSAkfnCiqaeAmZOTrAP8Aq7bVc3NTPHgsbD3ZKph8VfcGaZMqCgqRG/dOvdraArzua9aa8K6jVV01MqlKW6NTjyR1wU44oiNxorbwtPrdM+pkoQ05AOpSgmfmO/7rQ5ijBtU5eOJquku6tXeDDjVTTZW1ysBKHCECIVrmGnfvbjorE6rtKtgkaE0mq646O9it+/1VKEXeta2Wmz21FmFEE8o1G312IN35h56sw8uqumuWaZhCEvuqSkhKXMqVLTGoBjw111t29g6/V0lLGHrwDj1FSKKhSOaKKXG3Ek5YGhSSPfbm+LrxDeV1XYpOGryZqG6QXe91tOpIStC215zMQDCgLW2hoWYGluiHo77vu80Cmuu76JmmpbtfqAnq86myQtJCZn9Md2xjwsTrr4vK+nmaFiueZYq3KNaA2qEpzNqBAg6dtPfzNlK6rixDdeIENoTStUzr1RSvuPvtpT1CnQvMAVDU5zHzDeu64b4oHkmoXQkNKYU2lVexAyuE5TK94Uo+U2cZbqZzBN/iDdjl4YTvq9LyafTeF3t0FY2HCZIKkpUSO+Cfj7K6ViQxXPJQfUdVlI8Da2fFFq5b3bvmtr7ypaVsXO+23/G2iXH0hWQQFEwZEePdraqF6EGsUvQ5wlUgzOgn4zbdwp14yFg4q20gKfvAmtTd/Fu7ACoNvqfpzBGym15fPWD7LXBQ6439IFZXEQqQdQoaztuNPHfvtRnB1eLpxxcF55oSzWUriiDyCkz9trxELzRET6qiR2hPf8/C1LGG2ka7wV/BX3ic3xXqDcF6ovq4LsvlBBRX0bFUkg7hxsK+2xs7TGvKzD4DXsm+ODWEqsKBLd2opSQZ1ZJZ/wDl2fRWYnfXS0IzdoTyfOK0VcgPxtolJ/SHkTtbRM6nTmDFsChO+nnYzUMpLvdo5UuIG2/lZDqEdcypsDVQI277OuvaQqkVAE94s11gJBbJgW06Z3o2WbVN1uijDnW07ayCStAkDUDSwwUAdVGxSlzJbKCJ6txaY7u0Y+yw5WAZzSddbXW7KkhQZGcnlOtulL7oB7uZsDm0kyfstuSkASTAg+6zhJdg6cpmYHut0lQ7xYIKjUEAedtz2TOg8TNiNTFGAuREjTcWOXes9YDqJI5WTQonzsdu+S9KQCPE8rSUURq7prTVPFrJk6xWXbadLZZ1gNESQozzgWyzWSXilw5Up3ilc2bPCWK06qAk9VE+evx9tpyvGpcUUpWoEN+rm1jTbmfnwtA3DXMeJd2vtylIpayCRr6gEHn3+73zZWOLWrMqBm0gTufHukk2jSgWUJ3EJRuhxtb/AFD6Z7J15HedRvZeRTXWACppaxMQTJHzrZsXOHF1YQ0palAH1Rsn3672cKaWqKc2Rck9kDWDp8+6zzuIcpU4BajDbF0iUpZQVQCN9Dv3cvC2ZrrkqTTJAzaa7bc+7QfCxb0R0ggDQDUlMiB3WxFDUKUrMggydYmfjrtysHNfdGLR0RlD9AntFgQkmTIO3f8AMWDF5Ui2VBqnbQAZCQkRIjadOY9lnPw9qcEXNelS9xBwnVX7d66VaGWKarVTqQ+fVVKSJGhB8510FmTTXah12pqKBisQhyoUpKHnw4ltkgZEDszI7UkmCI0EahbN+MLbfWk5lhcLqqvRLSFOBtCANDoE6bfb+HfGXFPEDtOaXq61VMh4qeQ806My1J/RGhEQRraQL5oQ3d7nXJS4lSIyqlUjaY8CfjaIcbU7VXdSqG/XVAXa86adY0WlAbJTO51g6c48LWDICLBQym17JEp8X1l4tNJ/jC228ji3ERn6rNHLvPlseRt6aYPv9T2BLsPbTF1U0g+smWk776wfjbyTwbWVa8R3fSde0000k+sFFLwSCpMxOhn4c+fpzhOrqXcIXY4HsocoWOtQEyD9EnQEARr9luW7R2dEwHqV0/Zpv4x/1KlXSzrRUdJlpwq7Kqi6hBE/ooB8tB8fK3r7RqAaQDB7PPvt479KRxI6QrbgUlUVN2iCJTGVGvjBi3sGw4UtJCY9UHfnY1NZtNGPBVKo3qpPajylAAkiLFVKCp7X3W5dqFFJHLvFiqnXEJKUq/D5i1ljtVUe26FdEiJ8gLEXCo+e9unaxaRChJHhvYDrA4JAI7577XWG6pSCxRq63Ml6Ui41DyNvMWkMVB5m0aUa8ldTEK2eSdPOypibE2MLsvFNLcGDHLzZUgK9ID6UpzcxEyPbbKxFrnSNDei0aHKIyT1TvqajtxPK3DL8rAzG0X3ljHjAFgU3DJsgp3VWIn3ZpsDQYs40u1bSX+HrFO0fXW4vLlPtVG1sY0kxdmFvNaAmjDbfoUtreUlBGY7WS7ybrKiiNQ00paGXW1KKY0AWOVki6L1xs+epvvDtE00vXr26oSgdwSCcx91ur8eWLvKEOEfTM6BW/wBIm1acljsrlZgYXatR1L/WNgg9/wBdjlx0LTqUlQSSvMfHUmzMxFe99XBdT16XbdL99vKcbbbomilHVSYUomMx3HI+wTaOqTjNxzu6sqzdHA2vvNgrAp89c2ylMaEDMAQCe8n7LZ9W12QOBFr9QEaUHKWt3Uz0mGKJ2iqUqQk5K+og930yj9tqj0oTjlF8XhfhaS6m8KigWholtK0trSpChoqDPO0kXhxb6Q9NQ1L2HuE1K+7UvrcqGKu8Wx6I6T2mx205gNDI77NTglgnG1Fdt8VPE/DTV217t7uVNPTsVCHEKbcSkqJKFL/SEQT77HwCmie+WO44jrZdQdr39yw8WfWwsa9hOUb/AKE2r3r04ZuuroqfDinKGtT1dQ44pa8gy5TBSkcvOyvhLBzHFzEt3Xji+6vRrvDHUp9HedbWtltCiFRnJBJ5kaxab6XDFw3/AFdJd18XUy5TuLCJKFDLJ5HL8zaS7lwNhe66tysu+6GGnGmW6dSRm1CQFa9/rm3pcEFHR0xDYxxSD6S4178QqpRxJiYxyUd4F4YXZddJXM4Lutq70VlAhLb9QFqB+kKlBSio5jCY9tk7itdl3LvK4aaqw8xU+l1Sy4HWyQtKm2TAyzMdwB2Fp1oXGGGqpKChtDVcpAS2jQJOWAAPOyJeqGOtw3VlLZUtQIUWsytkDz5WqYbC2kk40ozPPNWqhr5Y+G1xAVG7wpqa68VC76Blpljr0K6tKYhKgDI5xrv42kXgQxRVVxUxr70qWWaPD9G6QxWrZCilxbUrCDqAE6Wi/jtdd8U3Hiqdu6+0UjLFWwOrLCew0pLZUhKokAkGR4jutKPR4aSbmqnHHUFQuh5nOE7Bq8HUgeyfibU5mkVN3C1yt6nN6WzDc2Ut0CcJreld4VZbOq4rqkyO71rZVYTwTfNA/d9A/XOIWl0NtKrqrMolBI16yCNNjp7zZFcZqAymrXe7hQox9E2R7YBECzlwj6G84Gze7y19pC2nEntykgQdzztZa5t7FzfJVOHVXvlNkz+D2GMPVWCrvvJV13cqqeaKKhx53q1KKHFAA6EGNR4fU/FXNd1HTBbd23O6EJBKAtsqPfr1ZJ5+O1mR0fhcOBuF1S3XVfoN3XVWVy3n6teiPp1KJKjylennFlDjpxETcWAqaiwjVpqr5xktN3XMaZWYrDsBTySOQChB71JsMRUkcHFkaCbeF1pRR1NVMI2E2JtfkEqXnc12oubrqO66WjWmpTHVJyyhQVI9VPOOXdaCOJTQZepnJTkCVJ6s7n1ydeQ0Hd902t4UrMA8Irtw7W3s/eFZd1IyKmqfdLinHesGdKZ2QCohI5JAtBPEx9TlGl41Ocl8FCMoCBpt4mVH324btFAI8Sjc1uUFo0XS4Y7/ACN7M2azjqmrUBujpnFF5MAByCsHInXc93t7rM+pxzhtKJReSldv9RXagcpAnY2dBlTVR1rbeTsylInXbxER9dm+9dt2MghF20gJJgpbQNZnWRqPnwsSnAA9JSlLiPRKkPoz40wNefE1WG8RG7aihvq73WkN3kygsreQUrQB1mgJAXA3M+Vra3vX4E4c3K46mtuPDd1IV1qilVPSMFREZio5UzAAkkbDW3nZiZuj/MFcpmnbSEtZ0wgDQKBj42gPELLZrqUppm5CNwEj9I+I10Hv1t0uH5XxFoH1rn63M2UOJRHiS7RXzxExVe9LUekt1d919Sl4rDhdS5ULWFlR3BzAyJ331tH2IA5R1ZpojOgGFwAQrMIJ1/VTHlZ+VFGpL3pDaCVAAajSRHwMRPgJsxMWKDeJFBKsy+oaGgOgCBpGpI3119XkINtVrcoWc83KQ3hmrUhJAynRKZSDqCZBiO8d8DmABJnBLDN1Y+404Rwpe90Cso3KpdVW0tQ2FtOsMsOOrbcAjRakgbTrodYtGzq19cElwqbjKntEgEaHc9rUAAbiTAmYsT0DbiXe3Fa/sUuJJYuS4xToIQQEvVLyQklR3ORl0agRsYi0h1QyVGnEPg+ngXxarMLv07j9w1wFZdNRUJzpqKRSpKDyU4hX0ath2QowlYmTsO3XhutW2j0W7FuqyoDIQjs+ASdoB79eztFrNdIrhC5xc4fLu266cOYiulxVdchAyqW7Az04URs6kBOvZzpbJJCbVNwFiK5qm7aRF/lNPUoCWEVDrZQFAGE5iUiFa7SPVNjRPuUMjSykaguC4qZxAprmaBaSCCpMJEgc9SdRqdvdZcqG1oaLdO6QkIkMqJMKGqdDpukeG8WLUbbqR1zdQFoAkKSQqBlBgbzz79tDvY1U1BYabcqKhAQSD2gTmMiABPcBz9xFrBFxZMDYo1w6v8Yf4hXNXITm62oTSKSBMoehsiRzE5vHLvFrdrRKCgp30UdNdY9p28I+NKruFRRXpd9UppTQp6ttannAEJSEuAaSecJH2WulKVZXAqc2o5iI+drZttSFdJ2Kg94+jrdbLklJUgg+B228J+2xd1WZClRAUPW1Ea2P342UXzeLSQExVOZTO3aO3f8AdZJzAwcxCBokAkgc9u7X7LcW5uV5C7+N2aMO8Em9ZkcITmUjIRlOsab8vHf6rAOBKm0kEAA5coPn8zYZwkvBORJHaEK1PPX59+1gVhW68o1kAbfMeW9jM2QH7ps4wokLudl10GGqxpeYAA+tE+cE/O9T+krdzlHxCvRJVm6ynpnEZtCQltICtNDOVQ5bWuDitvPhyuAcEpbzgHXVJB193jas/SwoB+6egrQyT6fdCUkgaZkqWRr7R8O616gdaoBWVijc0BKrA8haXFhQykEgjus6OFT9cxxEw/8Am6qNPUvVzVO27nKMvWHJuNQIVZs1SShwlQgKM6fPhY9hi8V3PiO6r3RoaOtYfT/iLCvst1MgzRkeC5OI5ZAfFXyOA+IdWT1+KKchMaGocJJMbwPmLMe/7rvVDz9KcVtvKp2i7mQVHrFA/wAmD+t2fjNrH075cSpopaAQhRStM5nAYMq5SDoIjlavN7PIpHHqgXc3VEVzxcKnFJytJjNEESe1Pst5++SVkrY47XN9/Be0YFgmFVVBVYhijpMkPD0jy3Odxb62mia1Bcd4XhWt01ffLlI2pCgp1UqSnKkQd/0sx8svjYm1clSqoDL79XIfLS1wAgo7QStJk6aJ5aZhvFnQ2/eHUN3hU09C3SVVRUUbJQmSFhCgkmZ/SAjvmyXUUl7mnqLwVeBCfzcivbbS2EgAqAUnvka62tAVn+r71XLexX+1f2SRfzFfDlH9El3rT1YyreQABlJVuNO0I9o8bOK6+GLt5o61F4EBK3ULKXUqyoyjqzATqZ9Yffbu8KVd0X3TN+mvPMZKN7txDiFOFC5Ec5Ttt8TIeCGmy1eLYlKU1BAA2jlYLp6iKVrJLWN9r8lbdgnZyvwmqr8LdMHwcPSTJY53W9XXRR0/wHZvWgTS3teS0rWpZcSyCAQYygTz05zyEaWqdfVO7T1CW3QQpsFozO6FFJ38vhb0cW20qcrpnmSnbnprpr9VqFcX7pTc+Ob+ogILV61KgIjsLVnRp5K+efRYNM5xcwry/GqdsbGuamcVlAp3UqBKR7iFE/ba91y3gm9LloL2GiKykaqCJ5KQlX2/ttQ/NNMkfqrPPvA+61z+EFYq8uGmH6leVUUXUHUf4NSmx8Ej32PjDRw2u8UDBXem5ngvQ3ogXt+cOEXoJUmbrvSppgAdkqyu/W4r3WmxZzak92gtWLoSXoFXfiu5SrRt+lqwNozpWlR1/oJtZsnUAT7N7UIDdgV6oGV5WiZ0y/C3OYzz8SbYecbfA20qAdQd4taAVZbWoqRkOmlm5Wtlp5xIPP2mzhBHPly7rJF8NhC+sAM+FrdO6xVWoFwm8g5KmoRP6QX7CI/smwva2VudbBvdi8CowM7R029U/wC1YWCCd9J177aIcs7LdbBUQeUHzthIJ0277bAAGYRHntbmRAMDbYm07qBFlsE8jHhbqSVSCd9xytzrA37tLbEAkDlpvNpApIT9HnET8/PjY9duj0iRBkEediAgDNqNNTY5dqwXhJ37rTzJrJdzK/WPv/G2W7kc0n3Wyz3TWC8SuGbjTnEagCABlpX4IGolEEHv2+d7TNWqKtwNFRI2zcvt0tAWG7zcoMW01YkuhQbUAW9wSTO2w01HstMf57dddpypScq05FJWYJVyg+/vsOleNkOoYbJz4cqGKepW8tyG0JUopOpJMCB+zazk/dJdyZSmqBEiAEkcvmPKzJaqQ4hRQ7KUpHaBk7zy+fdZKbxjhqpvY3C3ftGutQvKGusjtDdIMxIPK1iSJrzclCikdGLBSR+6KiGiahUJmUhJM93L5kWw4lu5CYS69O8BP1HnqPqs0FKJVnC5AMgbwO/Tlr8zYq/ddK4lx0F/PJIyOqBPhvYZgaNFMVDr6pW/fMuwOPIaTUudWpSlQMqkgcgJkyc3x77DXVi+62WVOvKWgPytsFY1js6a+Fokq6d+7K95Z66lfVUIWSolSUoUqCSDuJUJAPP3FK2qDV4FT760BunUssoByjXM5EakQJB09Yd1q/CaDqrHFcVI9+cSsP1lSm53rurCqsR1aXM6YCjtPd6sD2DmLQrim/L0fxJUXYywKdxDC2lkkSsADLMetpPnJ8bFau9mnsQovBTqBSt5llPWnMnYASdUmQI8LHsaM3Tf2SsuqtSa1pouVWZSs5bABAT3x5chJEWGWAahSDydCmpSsP0FTT3nQNF9NKwFlSEGAodkzGw31MgTMjn6XYKraNzBNzIJX1irrpjBOgUWkk89tfqt58dH+pqGuLdw0bzjppampcpnW1yUOIWlSSkpiCkiAd+W/O+uDmBTYauphGiGqRlCf6KUCOXlblO0E3zY12HZqHR0n1KnPSiXl4/Jf7Sgmtu7Lz/waPZ+kPZ7SPYejI9Gb7wkfVbx46Tyqn9/Z1KEZkpcu5RE6ABCNTy7vhrb2DoVD0VsiNUJgT4WuxfxaP2LKqBaqk9qMKgjSOU2Ac0EnSwyicsCSLAL2Mkg8tLGYqz90VejXYaWDS4gpEqAPq26fMbmPZYgpQC5knXS16I2VGQXQ1U4UMlwaKSJEeVpBpa6np6emqHXApLiJ5kqhMmLRjejrhonktZiShWUJ1M5eQ5+VlK6rwpXqKju2+65+irKYEobcV1fWpUCAdd9Fewi1Wuyki6vYeCQQnrV4paStpNIytOdbaSkt5TlXMKHusVfxUvqlqpXTnEKQVAQpIUAef12Szct0PKbWb4qCpCWkoKHRI6uYOg5zrbharnuvNddJT19ct/tBlMqCATMzpAnvNqXDhdrY3V6zm9LJaXXdZeVRTtpSUhpK3I0AWSY25kWS74BN2rzkiFtqBGsQsEWWGqampUEIbCesJW4SZJUe82S79epG7pqiHm5CZgrAJII0Fucq3maS7BoFqU9mCxQTTjpYXmJ0cI1tqgcceSpto7LVv52rp0icc41wliJyiuDE9XRUztM051bKgkJmRm2ncD5m0b8IOKuPKnHrV0XpjW9Klpd31DikqfVlDraCoKO07c/K0ZsClr4WvDgAmmxCOAlpGqs47XLTUX0jMYTXr5bS2g/bY+HStl3T1SCPfasPDnE2Ir1vW5366+3Xi/W1KKg1VYpPWHKQND65Age602Yyx+xgFNJ6Rc1Ze4vRamW0XeQpSXExAP9KdANSQbVcJw1+GY5E1zgbg67clDEqhtXhbyxu1vvClPCtwOXvUsJebyNqUpRcKDrlgkJ2+YtJ9LT9Q5WIgarQZjXVCB9loP4S8XKi+cQUuFbywVelyVLLT9UymscGZbZQAYAGmoFpKvfF1dd96MPUVyrfYeaPXL6yEtqSpvKDodwVf5Jt6O6uhdJkzAn3ea4cUsjGZy0pcpHU+l1VCkkPOVAeASgTlHVg7897JLpW+u52G1rSvqHQjMmAlQUQD4HSzQq+INcm9q+7b5Zu+771TeouemdVVLW0pb9P19OSpCREwhJgbkwbJWJsf4xu/h7hGtprkoTeb17IpalxIUunaBWtJIkhXrpG9kyrhc/IHahMYJA3NZQJx8umob4rVT4K+t6unUshEgECCSTprFnB0bXUn02kcaKylm+W4WmR2bwKgI5+v8AsskcXL2r72xbebl6OMitYfXTkNNlKVBKzyJOvt/FG4YcQXOHwrr1auputi9rzpVMqdyD6QtOTIB7+7c2y5arjym50BW3Tw8GIW3Vna+jplU7RpqajKAJBTTpBP8ANM2N4ZaYbvNpZYaSudFBpIAPmNfbaD1dLVNKysowBTEsuT2qoiMvjlG+vusWa6ZtYXlqRgG7Gy6sKbCqtRUNd9Br8LKOmaHiTPt4IzqlxYWZd1I3DWhue+8K39h7EF0tXjQPXveNM+wpIWlaQ6ggxy7wdwROhiyXw/4BXTgriCjFVPfV4XjdV3U7jVxXRWkr/Nq3FS4UrJMiJy6A9okknW0d4R4w3pcVC29RXXQlu/sQVDjiHAs9SHE5uyQRpoN/OzoruPWKbrdX6Dd126kpALalDXQ/pWAcdw2MhkwJc3wVmOlxBrHcB1muGqmTH9WmswfebaWmVhtjMotvJUofSN6iO7XTwtV3G7VOKGDq31yMhUkFOoCffPzvZ6o40Ymv27Ki7ayhutLVUytl0tMZVpCo2M7iB7bR3j+oUigW8SsuEIKSkQE9vUkxofqje2LjNdDilSyWC+gtqtHDaWWjp3Mk6psNqSSs5gc7KTqTqdDttH3+8rUJEE6CTII57zy3tujcZbqHRmbh5kqSQqTy5H28traeCQe0ZSUn1j4fh7LTjZZFDrjVIl9tKfuWuQoEDqHASE6zB7ojl7x42gG92SaylUpMZWdcvaJUSZ21H26eEWKrEFxhxnMdUEQVEdoiB5/javd9IWmsYSorR2FE+Gp2EjujWN/YN3DOYWLiVrghFF0oUkNpSFBwkJEAzO2/h8BvrrGGOlrRi11lBStammSsAaEwdgdBvpHOPG0uZUrpgrPokkgJRG52mNdvgZtEeMW3TjKqWQ4AlDSQQDlCUpA5aJ2I35DlvsHTdZCbF6OJbZP0hZ7JAA1JG5AkxzOg8N51uv8Ak9bmDGAMW36tkIevC96dhJVEltllWUk/0nlabTamda2W1NqaIaSrdISSSBuVZZIEAae07Wv30PaQ3bwlu9hQCFVDLj7i51Us1bm+m4Qpsd+sd1oF2lkiw2urAMLCHmyQUS4kwdRodfGeVqp9Jvhqzw54rDE92UrL1z49LlWadZhLVenWoQCAfXKg8mE7rdEQExaZbwdC1EzJmN4get7J8t+6zQ6XuFXcTcFnL5pkZ38MVbN6pSUEwwDkf130QrMI5pFiRDqgu3VWbhNO25Nw3vUXRUlaSumfT9ESYPqk5DP83keWku9FTeVO0t6sYS4sIBXU0WUuo2k5VeajuqdNANLM+61t3tRsmraS2p1GYLCMzalaCCeR3nxE67BXpnb4ZfTTIpadwNKygLz5QmRBG+kQRz1jyui9k2iCqH6CvqfS6ZyuqS0QoOlwuBtQgFLjQEpE7qKeciTFrq4bv678S3DRX1djqlUz7I3gKSoaFKh3gg6bcxItUs3mwwkpYoKU1VIgLMSMunJR12PLxtM/R0vsXpdV+trKWXmaxCl04SE5QtHrkbjMQR/ia2pvZkN+qtMOZvsWsYtJZxPeCFOEFawqIH6SQZ8N+7nZBzBGiVk6gQRPlr87juNnRj9PV4lWsZh1zLbkDnAI8+Xh9dmuuEmAonYAJO4Mft/Zbi6kZZ3DxXeUbs1Mw+ASc9GclSdM05Uq2kx4cz8bC1941l71a6u8Hg7UFIQpaWktzAyjRIA2HIeNg3FIUsrBBlOae75j2WDKFBawpSZSTqDpM6aH7rRZbdJ26JXsjrrqrmcpUV06xvGuUxA91q89KKkTV4dwXe+XMlVM7TkkEHRDZHvynv28dbIOIS6ytqZC5SfIjbu7v2WgPjvT+kcGLjrToqkvIs5gDmAh1Me2Ba1SnLM0qjXNzQOCplVJPWEHugRrNgxplI0Njd6NlqrcB5KVpPKTYonXeYG2vO3ZA3C4kizl6mYFq0XxhG473ypJr7tp6jbmtpJ+s/OtohZoqd+pU5WNhbP53ep3AodnKttcyPZZ79Ha9Te3BjCFWo5lN0KabWNeqWpvv19X42Y1TedFT3XfVCXIrxeXpNMFIJQopJBBI2kE287q3NgrGl5sAXL37srQ1WL9nMRp6KMySOEBAaLk2eSdPBN69qKtfob1apFsG7aa+EOUgQJUC4ZOo0IOdMa+7e2XWwKy6rqzJIFVdVfQK7UyUkqTodZ5662yldrqWkcu5q8WPR6hDDjpUlUh1uCI0207+Vlqmew/TvMEViurpq+oqEJDBksupIKPAgn6/baNfTj1x5rnx2B7TE/xGX7BTOvG7XmrtuC8E1jz00a3lhxWiwhaVlue4SYHhaUcB5HW7wXBLaqmYjlB5WZ13XThymyprcQ1tS23nDTZp4ShK0qSpIknQ5gfYLPPhsEN0NcGiVoS8AlSgJICd/OLVJqmKeePhuva/wBy6Wg7O4rgfZvE3YjTuiDuDbMCL2eb2v7U4iNc2bMcwiSY9us9w77U36TV1+hcQ7xcQCG6xmlqxrz6vqjp5tm1zykOKJbGh1EmTzm1WulxdimsRXVeOQlNZdjrIHMFpzNr7HLbmEvy1FuoXlOMMzU1+irkn+TWnxCvs+21tujhW+lcLWGzr6HWPsDTQSoL/wDmH4d1qkt7LSD6yftn7LWX6KVcHsNX5dsyaatbfAnX6Rsgx/V22cWbemJ6ELCwd1qkDqCr2dDS8DScRL2u1SpRW3UspjQFTbqCPPRSja4ZAiQQZ1HzFqI9G28vzXxquKVFKKsv0q5VoczKwPPtBPtte1Wo755i2LRuzMWzWts+65JG06+FtSP1h52wnv5TbRVOoIPj3Wvg6qiVhVrI5d5sTvFGdjNpIMzY0dBor77BuJC2ynXaNbHjNnIUgzCyZ14wiqo3Z0zrbme9JP1pFhFKMT3zz+FucQgtU3WhI+gebXJ0AAUJ+E25LhOilEa20m6rMcLIYE6gnvnW3GcTmnntv8LBhc7qGnhbEqMSI0E+yxUM6oQq/nQCPkW6ChpqNbAFQOpUIJ+FtpXrPMDU+y0goowlQkE6a+dlC7jLwkjX3RZKSo6GARpG9j12KIeJTpJs6ScPaOuRRnnbLdJKgkAEAAaDX77ZZXTWXhbdri3L+aUyuCGyo6ZgO0efz8NJIXeLLDbbaVkPR21b7A6A8rRlR1LzNaH2nUgraCIkkyVKGUiNNeW/fzsp0zxqKxBrKvI2lY+iT+luMo857rU4ZMr7I8rczLqTaOvolt1lbV1zdIyyz1HWqchPq6mSdY8zaB8O3eqsxrR3TTOsritQjrUAQtKTOcEAnUAK1+NnY5iC8cR4wXddwMoqaajowhymebBaBKhm7J35awSNdN7N/F1NW3deLmJ28tFWFYJaSjqw0oI/RA5AjlO43sWSoDpA3ohMgyszKzEhBKnNM24I++Dt57WSq6trcz1BRONB4pzoKjrl1Hwi0UYI4xVt+Pt3biN1LbyZJdQkAPt7KlP6wGun6u1nJfOJaGivBquupaqlbOqxlJC06SNJnX6jruLXDM0i6piI31SdjXiAtNV+a1Ib61tvt5AScuXnIkGdNOXfNka58T0Tl3MCqgOQ9SlxWhDa/UJ2kAiJOutmjiKpVet9VV6BsvGoBjrD20pPlG23fqLJ2d0NBzqysICSlREKzcwTtMz7I2tTdISbq01uiXroQxUVgNQouq65GUwDnSdNZ7tDPd3b2WMUruChq2mXHkUtQlCkpUEyTt2grfadDpBI5gWbnW0dNc7NayEekhZbWQowgkRvuNNfk2Qr2fXUVcVD7LqpSlK0qJIGaYgeEbSLRzaKWXVPjghXIZ4w4WQ3Cgu8EtkgRHaOw8ddNO7vt6F4eb9Gu2kp4AKGkt9kd2nz5284eFFS5d/FDDAcKghV6UuZIVIyZ093IET7R429I7vTBShSgSlWsDc/Vbj+0X8I32LuOzGsT/aqW9KVT6+NdSW2VLQ05d5WYBygpRrHLWB4+OlvX6gn0Jk/8mn26C3kd0mAtzjFfqzORpF2mQdDIQBPd2iPdy1t643WqbvpjG7SP9UW0oT/AJNH7Fi1X8ak9qHVJG5B7vfYBZA31sOvv29tgHISZkxMTY7FUk3RR4nKSmbJzxIUY0jWZ5WUHzoSfnSyc8SlRzewWusVJ26CdcKGwsqGh2Nniy1d95BKK+maeQNUpcQFAGzCvReSgdUjTKgjQjfX32b12cVr2oWWqdbFK8tpCQXlAyswJJAIAnwFsnFIpnvY6LktnCnRhjw9S/U3Pdd3qQugoWmS6CVZBE91lS5alp6nWhKu0hZQrlBtDlZxara6EmnpUACAoA6mZ79LAo4sX5Tkv0yKVGZARAbnQEwddZkm1SaKeSMDmr4MYbYKa72oDWXVWU2cpLzDiJHKUkTarRwe3d7rdaq86hYYW084CISFIUCRry01s/Rxcxg8gtLqabt6QhpPztZm4mvCpqrsru0pIcp3MyQN+ye7zsfDWvpY3sfzQJWZyHHku+MfCrEHFPFLd/4QraStusUYpHlIcQoB1ClHQ5t+0LRlhDgBi1jHlW3TPH0lmiqaVZW0W2/pG8hKVmUqIzgwPG0o4dxFeN00C7suyscpqUqkNNGE6gAkH2WUrprKt6822zVOlS1GVZzJlPx1tWjNVAzhtItyVl8EM13OCiGj4PYhw9fFLSP3i0XbgvIvVAiQtKwlcJHv+dLSPj55arvuZwCervajVtzKo29vxshKvK9r6vm9GRTOl9KkpcyZipRSVJ133AHu0ssXtSX7eNzXfdzGH7yW+iop1rcLUJCUlKlGTB2Pd3W53G2SukZNIdRpotPD+CxjmMUgUqmafpB3IzRgluouuoQZOuraDpO+tpRD6aunrUUTOQu9TCBzIU6kn/QtGqsP34vjLhXETFIVUNNTOMPL6xCcilNA6yRoIM+Y77PbD96rp3GEKbLiKpx5sOZkgI6t11UHXmFC0KGbJAL9VmV8YfJp0UY49uTE7l7GtorsqKhTeKMN141T/JoZyOK1I9WPPzs/cQ3Vedbg6iu/0FSaikvpmpyFYHY9KdVMg7ZRNgccX3ebCbvrbuuxp2nqH2DUuKey5Mh7PI95seqscXS+qtTV1DaAtDaU5PpNQpzu0mFTaD8V4TiCfYmiw5zg1zRf9gq48c77ocP40vh10FxFTXOrQWwCSM0789xvaJavErl00D6G6BVQiuv9ZAK8vVhdGw5J0O8eyzv4+N1V5X24m5VuOMtvFRKwBIgbj53tHl80b9Rc/UCnK3G72oVKAiU5qLKdfNs23sOmdUQCRw3VaqiEMhjHJN7EmMK6mp6g0LSWXClTiQuFCSkbGO8d3Pw1E4OYgq8bv3jSXspCXKC6Xa5p1tIT1jyVJiQf0YOw8LBXzhervBxWSiqpabCPo3gEzrp6p+r77BdHfCl8IxGhbjnotPX0FQyVDtLKCYIhQiZSRPf7raVaXMonvYbEBU6YB1U1rtiVPV9XdSXZgrCdZSZh6RebNQ8oqJ+kUmCROgtvEfZUpaXZIMxsQO/3WO35d6a3hjd7KnHUm7npQ4giQppYA5fZ+LcxDdtSpS3F33XLBEwVpg+4W82p3mZxLzrcrspGcNtmhKNwVjinexOidJEzp8eXx9oGNbydcu91txOdGUEBtICkKE6953+Asi4fu5fpCFLqq4hs5kS+oQZ10Edw+dLHsY1IXRpSpSjkbWAE/pAka5vCTboKSIErPnkPDsmKxe9Wq8wwa10oQVJSMyRCUggIOmomDOlh3r3qnsyC5lMGT7IgHzjf77EHADfJClpCQglPtTqD8ffbhyG5SN0yqVDSOe237fb1ccLSFiCZ2yEcvGphSlVATAklKoJOnt5ePK0VY0pvRq1p9luAunTCZ1KjMnb4+3TW0lGAo9swdEjeToPn9sR/xBbXloSkmAlSSEzl0VB1IA2ju3Pja/TNDXWCqVDi5uqSKYIRTNNtqXIAA1iCecxpv3xaK8XpbcxZWGVZVJbhSVFMSAd4EwI1E6AgRNpbaC2mmUqHZWghJExsNTtJ1+ItE+MwhGKKhIyolCJWI0ie/wACd9NQSYNr0mypN3SE+y51wT1OohUGJkJ2EpMiI8pPrRpfrottOM8PrvYWgoyUjjYE/qs0qyI5dpS9DO3stQ9m6a687xZu2ha6yqqXfR2G0albqyUoGpGpKkADf7PSXhthFOEq2tw22nOi7r2q6EGCkqSGqhE+5lPht4Wz5KhrJWxk6lW2RF8bnDknu3lKMxKVSnbnO2u/L7h4vJFFSX7hp26a5rNT19Kad4RuFIjU78z7hrZnobAIyNJI1SZERp+wbeXcXlcjgVdlOFcklJE8gY/aYtqQ6rMkBC8+LvuuswXiO/MHVTSi7dFaWCh3TM1PYUANDKY18/GyyvE91stZK5pxKAoSuBCZHPkNeX2nWTelTge7bvxvdWMX2Cilv9tV3VjidCl5JKm1GO8kiJ+20L1F1ihWmlvta1IACWnwMqSIntDaRA92tji4TCydV3VFDWsKFE4hQcQQVkdsZjvpJOo5eFl3o6X5U3TxONzuvlabxYfoHpOuZCS6hRnUx1agPPxtFfoNTh50Xjd7yXqZRUVBObKRm/FXvEQLPHhverN3cWcP3wiE09bUtZikwJWOr+BOvny516gE2JVmAgXarFcT0lm86N5wlPW06kAd+VX+1ZlPFWbMowQIgnaDykeA12tIfFBkrauyoIByqdQTrzyn7PrtH5QYOVa9Dy0BMe/Y/AW43EfRqXLt8KOekaij2h0ScwVrAOvKdtTp9duAidVBUco1JJjSxh1AAgqlW5B3n36+f7bYGwkhPaSTAIPu5+B+Tau1yuEIuhEFIiNcs8z4R7/j7Yb4s0iXOCWJKZKSv833sVJA1P8AwhIPwUfmbTSEdpPZSCTJnl86/O0b43uo1eCeJF2aKUls1UK1iWgv60+eljRPtID4qtUMzRkeBVAsQtqRWuKUFSo5jmG3h8+FkoA7z7LLeJGoeQsqTqjSOcH8bIkFJBPfzt28RuwLgJBZxV+ehten5w4LMUxhRu68qmnAn+i7/wDM28rTdWXRda3XJuynIX2iepTOsmZIm1aegTeBfwxiu59xS11NUwDqOtbUn/5VrTvNKCkkAKBQkmBr3e6LcXiQDKp4PX716Fg80gpGOjcRpbQ22TVXct2IKz+bqfKOZZGp18Pn4WKuXZQAZjQU0H1oaGnws4HmOrVlUlRgmdADoT9fd5+0s4ypCcqyRsde7v8AL7LUxlWiamo5vd5lIqrqoEkkUVOI0/kQbc+iMso+gZQjTtpQAAdI1gd/2WUltKgoAgJMZQfD7tbcOtlQUMoRJAByyZ/af28n9EbIT5ppG5XvJHtKTPRyXAHIMr3Gmh5j591oD6WtyBWHLkvGFK9Gr3KaUpMw80Z8v5ID5NrE9WkKCtRzB3B5g+Fot6Td0KvHhbeVYAesu+ppqtEHaHAkn/JWq1uhlDalh8Vl18OelePBUKYJQ6I3UCn3iOdp16JVaE4jvy7CqOvom3wkzCi2uPqcNoNqgGa50JEBDpI8BOlpU6Mdb6FxZp6P/wAdpqin79k5/wCxbrq4ZqZ/sXF4e7JVM9qupgG8U3JjvDt7Z4FLedOtROpyh1M+Wk/Xb0YXvChzNvNLqnEEFsAKBlPfOmoPd4/jb0euavTety0F5pgprKVl8Qd86QqfjbmaB17hdRiDbWKNyQCASPG2jlBkmYNtaDU8+Q22tpWp08Z87agWUVokxCYtqAIEnXe2KOmnsm3BIHq8x32I1RKQMSU+emfQkarbUB5xpZJbeDraHUpHaAV8Js5L0bzNZhrpZoXeR6G23mnqiWx5JJH2W0oDdqzZm2cjiViNSCPC2ypKeW/cLBBYnfWZ3tpThEGQTy1tYQDohS4Z0Gs89jbhKiCMuwjSwZcH6XLu+62JWSddOQ8PO0goFGMwkTp42ULsXLoJkHyslBzUz5WP3av6UKJ1mDPdFnSTlzeHxNssW68nULV74tln0SXhtToPXdY0ohfVgRP6MqmPh7fKyXeVa7T3goMuKBUSEgaCBvEnkNB75seUENVBTmSpwpSJzEn9LQDwHP4ibId6damrJbbJKtQidNRufned5tktN3qy7RicnAJxtWLLycUrMV0kjs5jOYe/eD5+dnFx9uWju+7rvvJpx/r6ypcC0qXmSOzrHdsNB8bFejtRuN39fThH0IpkJKiOalTGuvL6rKXSVqYobgQCUy68pQAMyAnUcudqD3u+UQ0bf9loMa00JcVCVCpbVWw4nrNFIKjEAeHj+Glnqq8iytLaytKSAtIByhWg0PMgfZ52YtK6464GUrCgQUmRIHMfPs81htFZUFOVeUE6gmJPh378tdfYdkrItdH6x8vqFRHbcGpSSlMjYa+Z+dbFVagFvtIjMmTAEA8vdr+2wjblGzSracec65LeVCi2IknQ6mTsLEVKWlSlLmQkJgGZMQPv05AeyF1MNsjB6xF2uFLiy2p0HKRA21jx2jvGlizbFOloFmrK1pKQOwYOoMidiDPhEWPs1LaLhvFpSSHA6zkSO4EgxZz8EsM3NizEVbTX3SLep26PrUtpcUgZgtI1ymdj3kHeyc7K25Uo2Z3BoRPBFIv98bDBRlUE3pTKCkuT/hEzruDqTB318LeltMgh/ID+kQQOU+Pz9lqrXXw5wjcdS3eF0XP1TzC0utqLy1HONREqOxE2kimxzihVQZvVZClyVBKRrz5R5+dubxaldXOaWG1l1mDVDaBrmvF7qEOlO4lni/fCCs5XGrtIREBQhIJnTvn2cufrhc6j+baU8yyief6It4xdIW866u4mJqa2q9IcUulSok6qAQIHn3edvZi5HM910ahAmnbJA8UC11sZjgY08gseWQS1EjxzKPq1kaa/CwK+yPDumws6QBPhYBwyNRvprYsaDIilQowRzGnz77JdSofZFlKonKTuIiyVUKGbtEiNNza7HoFSduk+9FH0J9IB9Q/VaH0OkpCZ5ZhrytKOI69NFQKWpAVK0ogwNyBaJpUD9GI0J2+fG1eqOoV/D9ijrjjiDmJmREkbg8x872EZecU0AFk6CIOhPIfX7rAtMuKQHTolYgGdNO6y87cKaO5qS9zeN3uipUUejNOldQ1A3WkiAN9idxaoXAbrVbZJqHMqIK83MHNt+G1g71Djt21mUwOocOsmOxY8KZBKiZ1HcDHd9tua1DblBVtoVlHUrBPsPd+2wy5TI0Ra5VGFxqkBKiSddRZyXMkG9qdSOwEqLhBM6DcT77Ni4FthqM2b6NvXkZB91lhl1xkpfaUQpMwZ1B2sFzkVguEdYeboMcX08IQkttKnuGdzX6rOmqxDR093s19O6hTKaaVpSJg5Uj7BaMb1cUq+nHnSVKcpkZleIJBn3/XZSSo/uSeEwr0ZXjECYtzOP/wQcOqu4fEDIQVJVfjy7aNV2oqaR99wqUiUqCU5XGVJIM68haOOH2OcQO3NedCquV1dNe1U00QJUEgg7kTzPzpYTEC1KF0OpgRUslR9ih93us1+Hqsn7oqdZVKb5eIBO8pQfttzrZXmmcLq66BjJBony7X1NSf4y+tQmYUr26Dlbundhh0hBAzIOh2Otk1pZUCEBJMkbGAdLGqdQFLUiTHYMEaDtR7d7ZMt3DVaENgdFGXEZSRfLq3ARCmygbg9n5+NmXeQebVWF9stn0q6nnNNRJdRKeXd7/KXRxNKjfRKQAktJ3Ogg7/tsyL6INwVT0Bw5buWVJUNYqVgb92a3pmDEChYuPxJpNU8pRqmb+XeVQi5KtppbnVlYcUmDmCwD3ExPybJfBtdVR4loKascPWNOVbAAIMJzr2jlI+NtEVbdYtYp1LBZSVSjTKFGD3bE/eeRXhfWFzGTK0tZf8AdB9s6yM2s+z77adTI19JIzwP3LOiic2oY89QpyqvpMBV7UdluoqQRvGqo+o791m3f8GmSpKUwUzmJ302Onl3c/Oy+hzrcJ38wFA9XWOykiIlJ+8e/wArNS9KlK7vYXCXCppBEnaANfd7rebUrbSH2rs5z6KI3W6tFQ2nrNjty7jYXFwzXUah0pAMJUBoSJ2Husm0a0+mSkZioDbYGeYO4so40dyYfnLmUmSMoiBy8u+3U0DfSsVizE5CUw3D/u5ToQhQJQ0mI2OQb/jbhxwZkyAqQBMxtrz3ttlxJxBdjgXJeYAhQkEAKTy/oi23EJQokmDPMyd+cW6dgsAsflqgCnXUgFWpJJjfv57/AGWaWPWA7dVKEAqUaspIGkghR17tfrHfZ2qQqE7CBOqTPz42RsSUSKqjTCE5UVkqG/rJjfxJT7rW4jZwQJACEzq95ulr6WnbXGVsKI31I7+8CJ8/GbRBjwut4jeTl0KUKzQZyiUjQGRsfCO7WZGv2pW5itSAErS0cskGAABt7bMHiTTg37P6RYQoJTmCoUV65gI5GQD+F6T5qpt3Uj9Dy7MNXj0h8HKxO60mgozVXl/GHEtoRUU7DjzGadRDrTZiYMATrFr+Xayz+6m+nm32nmTiBtaVtrBSQ7lnWI/+skRPmbecfR/pk12LKN89hyjo6hYSoxmGXIN9566ZA5C12sC4r69V4UK8qFU9RRVSSlPqj6BGx2/k9R8nnK2icaptWHbC1vrWvSTtDDERupNvCn6jKttPbCpUVKJE+E7Hf53WMPPKXdw7RzJUsKJAGszt36jfxsXv2lcNCpxBSmAFT7eWvj4fCwWGHj1DjZUQpLytJ7wNdu/50m29QyZgsyvj4ZTe6QeEjjDhdezLDWaru1k19MOZLfaIka6p5c4i1U7sq3LzuxqqCkuO5EoWh0DtEAjyG3LbytetS21NkKKS2qJS4AQQRJBHzobUkxPhRzh7ja+MLk5EUb3W0bhRJXTOdtoH9Y5YB/nJMaW0gs4bJBXedBTuOU9XdymAFlLiurKg33jTUD74sj1NDeFLW0lZccuXe2s1LLyFgllcgxBM5ZEx4bWdZfpK/M31jPpQ1Q2DmkjeCdwZ+dLNS88NKonlVLJcpQogK6kwnwMeI0iLQlZnCLG7IblW9v6828R4Fua/WlJ/jIaeBCtDmQZ95FmaGFJSUhtWih4ieZ7uX191uuFV8N31webu5SSp+6KlLLsqiBnkER4KPzpY66wgnY5YEpI1HPzjTa3C41+LqfqXfYD6dLp1Se41mTkSNBGpMT5d2szt9tuAyUnWZMmeUjTfx9llJVIe3CSZOoA85+T3eFsTTEEtqhOYEiSd57uX4WzmyLXdHdJxZUDIbAIGxE8z8+zbvaFfdnpd9Yqud1oFN43OCADqewps+XL8LPzqiZGVOcHU6zPL262bdY0afiFRkgBNbdbrRzfzFgwOWyp/CxBJY3QZItF5o4oZLRaJC1LSVoJKYBPzPn77NxJGbVVn9xRu1d2X7et3kJApryfYKeYCVqSNp7uZswCf2W7+ldmiBXm1U3JKQrW9AO8AnFmKro1ipu9ip74LbuWY/wCltdtbOdsEAGQRyjfX4W8oOH/EC+OHlfUXtcV53jQ1b7PUF2iqAyotkglJJSrSUpO3Kz1qekvxEqIDuJ8TqRBgG/XUkyf5iU/C2LiOES1dQZWHRdBheNxUdMInjUL0ZqaZecrKYVOoV89/zrYo+z1P0q8qQdyRlBM6a281qjjljSqWXHryvN0zu7fFYr/5osj1vEi+a8RVUtO9rIL7jz0f5birV24BLzcrL+0sPJi9K6m9LjpUhT980NPpBLtUhIHvOm1kqrx1gGjBRV41uFiJSQu8mU7afrd/z3eddXje713GxTUeH2Wb0SuXqpSGFtqTroGy1I5bqOtks40xBMtv07es9ikZT/ZsVvZ883ID+0gGzF6IP8W+FTOVTvEK4cwHaAr21R7EmbMbitxS4W39gC/7gocaUVTVVt2PN07LKFrK3chKEiE7zG+mu9qSjGWKAjIi+qhABnsKy6+zzsEvFOJF6qv6v10/4Qv77WIsDbG8Pzbft0VaXtC6RhZl3Ra9EFusWClQmFQoa7WeHBiuRd/FnDNToEuV7TRk/wDGdg7f0rMd556ocLr7q3Fq3UtRJPtNlK4rxXdF+3Ve6FdqjqGXxpEFC5+y21KzNEWeCwYX5Zmv8QvRl2mzIUpJ1iJI0B9nzva8HBa9k3xwsw7UdaVFqjFIe+WiW9f8m1NzRIeGdKwpJSCFJ/VOoPx8tbWk6NNYp/hwqiXvRXg+2ANgFZXPrWq3EYfJ+OLV3mJRfiQ5SvIMqnWYNtLgcjp9VtFYBnuPLvtydBz8TbeBsueIWEjnbme1qr8bbMTpJibBklMK0sRpUCg6lPWNGRAPtmzESeprK6njLlfzATAhQB+ubP4zy7M2Yt8oNLiF5GwfYQsafqlQJ+ItdpzrZVKlul1yVifLaLYVkaj9I6b/AD8mxcqhREdqefv3trrCVHsnWLXxoFnEnmjAXJiec22FTIMjzsXDnrEmQD99s6w5YEa8+QswSRlLvaidjzEc7HbuUgLCpBnSLJWeCTpsSZP22O3e4S5M+OtpJJxAtRqhH+SbZYn1x5qM+f4WyySXiS2lDjqSMkJSiQeayNdNPOT32bV89ioAUtJlSiDlMefnqZjus9bieaFS4l65kVRLSA3mlWQhGg7Oo3jXuszb4DD98K6vI2rPA6w5QCTPMbaxMfVbKZ89WnD0EUoL7vC6kvC76xxsrSnMAqBKTObTfmfYLFb6xJfF/FH50vN+oDJPVocdUoJB3jlyG1lP8zinu12vdeS6cijCQFIygcvhr+2ydh6jYrq9TdWhZSlJ7KSBPhrpHL2nytYs0nNZBu4DLdAXcUJcPWJBURKSmBHn7PdB2sspUlQhNQ2kISomRr5zGtkyiSyxe/U3gkFpPWZkk6EAKI235fDWxR59wghajmHLkO/Qfhy3tIi6QR1NcXCWkKJSmUwNhvrt3A+dh/zfelQhVSxTOLGhzhJJ0HMgfPebEMiBd6nQwRmMJ56zrHICRA79POy3g+qfZq0s+kjq32l9j1j2QSDB8vjaFuilyQlFdlSLkvVyqpV07qA0ELcB1k6juO087PzoxIqFcW7ougtB9m+HBRVLZKpyFYnLB0Mgcvusg1bF7V1G5RShCm20obWYSElPdAn36wNPGTOiJdVXcfH7CdRWLaLabwZUoIJJACpI2O8D7+dqWKPMdHI4G1gdVcw4f5VHpfUadVde8uBV2Xa+Gaq7rwoXCgKLTxKCUk6HUAxvZEPBmnrsU4cw3czvob181RpA68VLShWWQTp4G1sMfuUeMMRm87vzqZDKG0lSCNt9PbaJeJiavA1fhTFtIgKdu29kvpaOgchCiUz4gEc97eN0ONVLsVFO2YvZcgG+hXqMppZ8NzGJrJSOmxXnp00uHF58LOOtVg68apitfpDQvBxlKihaXGUqETqNyCDv9frTh9Q/M1CQJ/izXOf0Rbyn6fPEJfE3j7W4rXdqbvSGqNgMT1qsqGgJJIABOvlb1Sw2qLhu+Qr/AIKzv3dWN7evE5omleZBrmvcHb80rOrB0Hw7rAqI8QbcrcMxynvsCtZJImLSYmeNVzUq7BSO6dR3WR6lRCoAgp+FlF5aYM6ajwNkuqIGYga2uxhUX7pCxBT+lURAIEOIVKjpoQbRewlCoziARvl8tfn42kbE9Z6PSoBbSoLeQgA8gSNfnwtHtKl1pwOlvNI0O8j9tqlYQ0i60MNBc02R2nIypZSCDnAzE66/IseYpnxUopUlBW4pOX6UACeXcB8PYLEmCtaisgI2AHKPbY3ROBpZAeBgnSRz8rUXPFt1qta66U7/ALjvbDV4uXZeHVOOJShYUy4HEFKkhSSCPP50NkqobfqmXUobVJbUI8YNjoq6hxtTDDKio7QkmPnWwoum9HmVLZpKrMpBBKWVHceAsDjR8yEcMdbVIuHrveQw0lTZbBZbk+Q/H6rOu5MP0d4vPU9ffjF2IbZK0uOtlYcVIhIA2Pn3bGyPcGG77fpqd1dNXn6BAzejq1MD8bGXVXbd7jiKi86NlbZKVperG0KBmCCFGQZnysMyNdoCpta4N10SJeim27zbGeCWVAEaSAr8RYdqHcNVDajJ6l0TzGhHnZIv3EFx0V8s1Dt/XV1DNO6l1xuuZWEaoIByqJk6kJ0JgwDBNi/75HC9rDlauo4j3IzVhp5TdIp3MtStcoIB5mOXO2PjEL5YBkBOvRW6KVkchzuAThv50quy6HglY+kpSf8AKGtm3ggOKvTE1OJBF55xoRllCfbt929nVfxQrDVx1ly1CaxippqR9DoQUyhQBB1PL7bM6kvG9rtvm+zdxp1OOVKFLKyAIyR9lsKmpDJC5p/bVX5pBnBTzS28gE5FFESZ3H7ZscoqF6opK5xS8pbZS4EjX/CJH9q0VY5xRxDwi6husfYQH0B1tLSWlHL3EHXaO+zeuTjlflyOV1ZiN9dbSuUqmgyyphKs8pIMBU8rF/c9PJHnZYqp8rRRPyO0Tox9QuOXxTuLGqm0gSNTr+yzPxV/uXdV55cpKKKncCjpIRVoP9r3HayVfXHq6bzvJupqsPVnUJR1eQVLaFlc6bpIjU9/L2iO4ho8Y3BU11NRO0jNRd1ewGy6l1WZtxhU5gACNZ27tbdTQ0UlPTtZILWWRU1UU8riw3WYpx8q47/TRJCli8GE0sAiIXBG/KJ+RZJ4bnqr+eedbCXDfhyJTqUpISVGZ09b3TZxpbulDjCC31mZkq+kWkGezrJCvqskYVdpKLEV5uOtoZUa9KkLXUpTEtoMyUQR47SI1tpcBvCcLclR4zjINdLqX6eqQ3dmImHHEIPpKFJzEJ9ZA2nQ/fZrutuVd0MJp1NqX1CElOdI7UCRvy+02Wqq+8DUzF401Re10JZfeU4kVF4Ik5mwkwRlnnsLI6OJOALruulo1YluGmIQEIWmtJUY7pXB0EzbiY8PlDyWtK6h1REWi7h5pHpKGuRXgrQEz6xMnffblHdNnDiW6Xau5FsqWghSR3g/HbfTzs36jitgd1YWcZUzrZleZK1LTCfW1SO4TpyHKxm9OMNwVFG1dFFfy1dYEoSnqVJCjoeQ1BmNfG23S08zXA5Ss2aWANIzBNR2lDV1yoIW4hJQlwIkpT1itJ3Hl42Cdpql45EUdQvcwhCjzHh9vusWvjE1Td13LcaraphDxc0bWoTBBOgI/W0nx8bN6uxpWIpi+E3jVJCSUtoK3FrgbITnkkk7C2+0O2CyC5ltU6/zTeC0aXPVjKAZFOrTxBHjPw30sWqrsrkUz7j9IvKxlXMCUcp15yOYnXxsQvbDfGW7qGnvN/gxiJymrqr0Rkl+jBLuRSoUOvJQAlB7Sso1T+snMTori4vde89V8LW6CgYIVV1NRiO7MzLcSVBpLxWs9owlIJJECTpazHDNe5CrumitYFRHWOleJqlJdSe0QU6SBmjff522s28dILt+NqU6EEUTObuiVZTBBG86/XFlanqQb4vV4rlOdScs65dTE6gCBPfvtoCk4tWTfiXENkIFOwgnKOzKCRsDpPl5nS12XRiqRm7k4Oj9CceN0RqG6cKp6oKU7IiAT2oB1OXu3PnFmcNV1PR4jrWUXiyVu0CSgpLmVWQOqiSBBBSDrHq++rnDKoNLjuneYfCC6y+UkhIn6PeTI2J200B11tNOGbyW3j662nHQsV1O8wAQR2lJUBrPMKEeXKbVDHnbcowl4TrK3t8YzK2xdKKVOQNpQXCTmIyp1j7Y5e4TCNUhxdQpIAUhSSTEDURPz9UWYpqkPJpagGespmVzmAmUJ089z8mzkwZWIC6psIykpBmDlJn8TtytfgibH81Up6h8x9JPtDxjOZB2kGAN+Z7vsFq99KW5U09dcGMWsqA4hd21KgnkJcaJI5DM6Nf1vfO3pKRIQvUq1ke374tHfH27PzzwzvNLSJcolIrUFOsFB1ET3H55WkAbqpd7XaxVrCqZxVLURopE9s8/q356jSyeL+xlhlr/AHZYbvm7tlLQYqG0bTEwvbz3iYs4aFumqClp1sdaCVJ5Eju8fPewt53e+qm9HFE6+lfaTlTmIEc+ZiY0/EuPBST+4E4lu2oqbwuyiqQujvik65opMlLiFJ0IOxIgAHu75tLgu5ahnJI0zSoGdvHzH42pjg+uvXB+LBeVK4pldFUZnWIUErQo6n3E6nb22lW+un7w8w/V1F2PYIxC7VsSHUqXStJzc4JcUfKRNuMx+iqaqdroG30Xa9nq+mpYHMndbVT2LuIGQJggxO49nsHw5WaGO8QvYYqaZDFMFl1srClAAGCNPEwLQdV/lJbtaKvzbwukcuvvkJM+SGFzv32ZeLen1eOIXm3f3vsOy0lSUl2ofdMEjSS0mNhytkxYLiRNyzT2hbEuO4cBYSe4qbV8Ub067q2aNo6AgGYG/hBNgabF17Xzie56hijpjUIQ+22XalNOiVx6zjkJGg58zHnW2v6a1/1KQBgLC5KQcuemcXlJmd1D5Nk9zpnYwzpWzgvCKYUDrditu7+UFrYwWvdoWgfWqLsdofpk/UmjxycVUY5xK4UoQ47ebzrjaF50pUtcwDzGuhGmgtFKwUmCInWIs58cY/rce4grMRXrRU1O9WlBWzRthplOVIT2U6xOUTruTZsOrK1kpmJ08rdlSROhhax+4C4mslbLM57NrriST+Fsk22B4G3SGXV+o2o+Qm1q4CqgE7IPlE2yyrTYWxLXFPoWHryqM23VUri59wst0vB3ixXJ6yj4aYoeT3oul8j35bRMrBuQpiGR2zSmebZaRWejvxseAJ4b3yzP/jDPU/65Flam6KnG+pIzYYoacd9RflA1HsU9PwsJ1XTt3ePMIjaKodsw+SiS2WnOl6HfFV5aE1d44Uo83N2+2lx/VZj9tlFjoaYrVCqniRglCIJPVv1jqtPAU8H2GwjiVIN5AjDDKt38mVXuwgJ6tJ7lHX3Wnm9uiPfNE0n83Y9ueveKgFJFJVoQEketm6sknbTL391ku+ejjiGnp6QUV6JrHgkofSijcQG4JykKVGYHYQJEagWYYlSu2ekcOqmbsUwXJ0xsFUWG7toavDN7PVTFIyy8sVDCQpxCEhSvXnUgkSPqtbzoDcern4vt40um77qeoVXQqhqIdfDhcDodSSIAiOrSNzuLebDPRpxo86tC66gYQD2C6XMyhyMJSQOXPvtbPoA4bVwD4j31V4sxXdf5tv66fRlZA6Cl9DqVNxKNTBWI8bZJp8PicXxH0lrtqcQlAjlHor0uIEEEx3eVuCJIKps20cScGODW/W0xvLTg/s27PEDCGXW+2kxI9Vf92xAQoEFL5kjLGnjbSjqSRInXSzeVxDwbqk4gp5G+igfiLaHEDBy4Iv8ApdSOR1+FitIULJfMnQkeNmZjQJp73u2pIjrkOsT7Eqn/AETY+eIuC1HKMQ0cxESZ28rNDiBxJwMu76OobxXdss1raZLwSACSg6naCd+Vjwus8IEzbsQ5dTM5u/fYm2lPDLIEACeevj9tmeviXgKexjG51cz1dWhf1HWwC+K+Bmxl/PoWBKj1NM85pMaZUmbagcLLLLTdPYuqymCJOmom2B1e/vmbR+7xiwihOdBvR0EZh1d3VEEDzQBHtsTf413MElTGHMSVZGuVqiSCQTyzLH1WWdvVLIVJiHZJVmOukAffY3Quqz76TuDNoeXxqyhK2MEX2lJJ1fWw1lM8+2edgk8ca9DqW2MNUrSpIIqL2bSomO5KVH587LiN6pspU9ekK/4ubZaBz0hKqe3dFClXMfnBWh/qrZZ+IxPkK84sN3vTXZXOKrKhtuQ1CZ3ypTrI5Aa2b2JE3Y5VPvouxCiczk9crtKKjqB3GR7+U2WLpuJi+uuXUuVCXG+rA6sBUS2CZ8dPid7GXMNUhfX1y1uBKYcJA1010Gnz42zmtANwjuJLbJnUrwVhZ1fVqbydahISnSYOniYPunyshXRU1LDjjlMklcphQBOn1/PlaWUYXu9V2Jo2WVnrVLXlRmTMgCfh7J8LN7DGH6Zq9HkdSttxKCCHASJCtToYO/hpaecBR4ZTFaQ/WV+iFF1ZIG4JVOnOe422W6hgLbeTkXJSoHTtDefH8dLSDS3JRHFakmlbUOtKiVHRWgUIGbw233ixCuoqNqtcQ9SMlzrSk5kgkqPcCfH7udmMqcRpltvVfVAOqEZZE66z7eZ38Pecw+av850qWnAEpXJI1ERrp9vlZzpbbZSlDNNTwkAkhpAEjWJOv3Qe+zgue93ruUpR9FWlz1iQUqBhUp08APPflrDjeCIIgdyiTF9MOEtGmfJcVqUsEj2R4AcjvaSOCuMbowXxJw3i+9Gq56jpqk1DyWaUqcyjQgDQHeY37rJV2V9z1joLrriXFaEFZTm9xnlsbWD4H8QL4uzEN1XDh+iuGlLrqECqcuhlx0pJ26xQKuZ57i2RjFQ8Ur25bgg31totjCqUPnaQ6xuLe1ekXCy5btxvhG68YUVStqlvWmRUstvM5XEoUJAUJ0PhYfijwEY4h3NR3ai+UUIpaj0hThp+szAIIgDMOZHOw3DbE4ZuVlVSmvraophx1FMciz3gJ7I9lnk/i+o6khrD1e+FJIITkQRpuc6hbhuz8OBCBskjbSctyiYhUYhFVuAdsfD9K8MumlcX7meLNRTIfL6VinUHEo30UBuZ5T7tt7ermG3Ui4buA0PobH/w028qenheCH+N1QGaQ0qQzS5ULWkqQEpUOUxsfb529SsOup/MF1rCxrQ0+nj1aedu5oi+Siie/chV63KyrkA2ul4vHcqGh5WAW6SN+W3dpYDrolRWQTYJT6UgkEzNrsYVJ50utuqUQSSIHxsm1S5kHXSJBFhHqo6gH4/f7LEH6pKgqFEHnrGtr0aovKa+NnVIoEBJVnL7QEH+eD8YNo8XxWoKVKmfT8PNrB6sJNBTFQUP6R8zZ+YvUXqBGVPqPNrEeCvhytVKo4bY2qb2rHmLzuhll2pcU31tYQchUdMqUnw9otSr6SKptxVdw6pkgvkCms8bWLuQrNiKgYCE+s1SUqIj+ig2QKnpPVQus33QVWMa6hJaCKijuKrShZdKOqCVCnSklXWt5QCZzCNLMRrgviGubyVuJ7gZJiSHXiN4P+BEztZx3jw/4k3tgarwHePGS56KkrWGmnKi7bmc68FC82ZKi6kAk5yQABLioAARloRYZh4PpLRkxCst6IQtR0jcd1LhRT8PeLDxiczlC5TpymRILi0aaHu2snO8bON16LU3dHCO+nCAGwK/FVHTqUpWmqFPkkQQdJ3sR/eOv9xSnKrpMYmQtZK1GhuanpjqnKTKFAiRv77G2Oj5h+sR/u5x44nV5SMsm91NpA5pCc22gMeFrYosMj9VVzWYg7nbySzhh/pFYiwr+6C67jwXTtMPLu5SanELtaS82oJMqYbLeq9D25GVQInIF15dvO9r4vCuvKrqupXeT5qnGWH2loQtUlQSqFAgTEg6jmZm1j7t4FcMrluUXPS47xu9SKddqFMi8WglbrkKcWv6I51KKUkkyTAJtlLwM4JXe0G2qS91IAypS7eASI5hORCR+y02y0cX8Ewob2VU38K5QPRvPIoLzbS+4peamczkoMq7YkZU+/T2d8fXyy47er9Q4+lt4p6tYLhB5HknQ6+O3dNru8P8BdHGmxXX0+MrvdaucXch1PWVdW4VPpdEA9WonYmOXnZ+P3B0AaZ9dS7hBqsdHaUVNXg5OnctUbco7/G1uJjZG57ge1UZi9jspF7I50VsE4f4pdH2466/73TT3xdjZoagMPl5ptKYUzvuQ2tuTMEgxIINpJqOjXw9dfU7T4svKmeeguradbJWYiYUkgeVmJhPjR0TOFlA7dvD3BlXdNJUudY6xQ3cltDiwPWOZwSdd/E2O1nTC4QMpz0eC72f0kFbbCZ1/pE25+fAbzF0bwAfatKPE5sgBze5cX90IsEYorxVXxxaxEpCAUtIbRSJKQdYzFszrJ2siVP5PPhZ1aUs8RsUOmVAqNTRpIBmIApjMTz7h7TL/Tcwa2qLu4d1KgEyCusQn6kGye904QsEUGAGG+QLtapQ+CRawzC3sbl41vYFXdO97s2W58bJQvj8nnwq/NzSqHFl+irS2CpxT9OQVCP0Q0DqCRvZAxt0W7qwdf2E+H2B6mvqqO/aC+GHa+sT1qKd9bTZTn6tICRDe2nqm22emriOrrEUq7guaibWqC4tLzoTpoPXTr5xvZGvHpWY0vi/Hyw3dSHcPB+upnG6YwsincSqQpRB0XtO9i9xDTmMzj9STJZQfmhapfyd+JGKZp5HFG6QvJqldC8MsjUeuddtbJiegtfF2U14Xa7h/Dd91r1O803eTmaeuXmIdlaQoFJOw/WImEpFhr46YPEheF6a87rxjRi8XHyy5SpuxgEfzoKSYjWZjWNDEs6p6VfHerckY1daSdYboqdvMfYjwI+dbrIQzd5P1BAu924HvRhHQh4vmmYo0YCwAk05KkOu1LRQoqHblJplHU6iSY1EmbLTHQv44OGnP5u4W0gZQlvLlf7KUhMR1bKY1BJAgWZFTx/46VnWFeOsQCRENuFAB8MoHcbJT3FPjFeJIqMY4lcBTOU17x8J0O2nxsdvCHIlDySE7gKcrl6HnGSkuRd0V+NsHpC6pdUpxqjqHCnMoKCUzlkCAO1mJASD6qYAoegg9d6mai9eKVKg05SQE3bAJHm7zHlvavlfi/G76v8AdDEl4KAGvXXguDpzzGPGxFvGtFTPTfOKqBrImQl69Ggc0aTmVMaDx1snvZb5nvUmRHm5W4T0TsDVam6K8uJAWhAUA2zT05IBAnRRXGw5fdbT3RF4SU1ORWcQa1tYkoqGKenZdQTsoKgwocjHIWrphvEraaZF5Xfiu4XW3UZ0CmvqlcecTmgQ0lwuROuqfvss0vEdyrbKBUKX1ZI5pmDrqSAR5b78rc/WUdXNNnhfkb0WtTvhazLIbn6lOeM+D3DrFF2U1zYi4647qKWgrWrxpk0VfR07jdSgKyrCmqbOkjMTIVO3cLR9enAjgPTsP1BxTxHvepYQtxj07FdU411olSZT2QrtakEEGfGzTOPbvYbPXGqS4ANerEaT60KMfGxBXEWhDvVda0snTKt4tkb96DGmpHKLPFSVgILpnFT4lK0WDAql0CEsXrXNF9anUuHPpJUqdgBt7bJOJqpFTiB2idVDblM2oK3IITOndufaNog2U79LVzY2r6csyHXVRmVOYmQBH1eHKTJbOPG65+9GaijcPW+gofOuUJIUsRPfptrG/LTo5W3bZYsZAcpU6Pb930PEiiN5XfTVb6GaoNIqmEuoVmZWpRyKBBiNPMbGLWLve+UprKR9i6brp26epQoopqdtkk6giQnaNPDu0i1NuGWK6peJrtqqVLLdc1UBpQfzKbkhQMpSJI1+B0Exaf6+/wDE9Y2KdmuuVJH0mX0RxAV+jBVBI1IO36GlqgiJN1YMoAspuuPFITd9Mw8rqzT06UA51FKsoA0hPcOfwstXXj2hp6soeW44lLZzGRoZBiZBJ22HwtXOmxnjKiQgVdyIqwOwTRvJVlESPWyaTE7eVlC6sdVbLj354uK+20ZAUJTQLdJIIOyM0Kg98edpcSVmxUWxwv3Csg/xSuJGZKEvEgSkZCI07u7x+qbJV/cSrvvC7q27xdzzqHmltn6TZEactRpFoRONal5KvRsE4pezjs5aFCJnQAFa0x4jloNOQLuLcR5JY4a4lUA4RmcNGkK8f5fnI5ad1l3iUqYp4RsmfWOVdEoooGldaz9E2EpzdYRMAgc4n7OYsPRcQq6kp/zffd2LpX3AUIeAUpEkSDMkxtGunjbRrKtu8VJr7ncu196FJpXlpUqDoQCgkHu79OVlVho1dEKapu9vKRlSlfrEHQyPDSY5e22kwlzdFQeMrrKN2F1NLid5yvqzVO1DQcLidBHKI3EjzHPcWZmNODN+45xCh7DLlC2vqpqOvX1KU7QZgzv+AtI+OPzDcTlHS0PWF9ALiZXmyIB31HZnTw0EbwD+Bqp8XywhtxpPpIBIVISIkZTG258+/uo1Ejohdm6uwMZKQ1+yhym6JWPnY6y/bgbUTGX0lxRA11MI0GllOl6HmJXCE1mLbtbJVlIaaW5Eb7xqLW9awrVLR1q61lIUM0hJUI0g8jrprrznxM/uWpyuFPKOsylpWnPST3+X6RtjnE6onf3LYGGUu9lU6n6HLISlVXjt0jUnqruG3hLnzB7rKLHRCwwGwHMRXy6skyUsNI25ASbWmRhphCc6nnSQoKICUySYgc/kDui3CbkQgEKQ525MTBTsANo1+d7QNfVH1lL5PpRs1Vab6J2GWllRrL3eSD6qi2JH+L99lCk6P2Gqfqg7hmncDGoK3FnMYHrH9Ly299rNN3LShQmlUFJ0JLnPf7fnWxpu5aBJUC0PW0JSVTB18/D77LvdQ7dxTiip27AKErtwLhlgpnBlwJ0AGW56dShHKS0Z+s2fV0UNy04DSGzSjklhpDQHkEIBO3x84fH5npW1gdWAMswEwVH7/nWw7N1sAnN1YB5wAY8vneLVZTJJuSrMTGM2TfYauQ6GsvZzXSKxcactB3fUfaaXdOHHgC/SV7p3lSlL000knXu5e2y0KKmJC23svKRyjSO751t1kYQCrrUHtQcxgAePzztUdA4q62ZoCQ04dwomG/zJIUJI6hKYGojUCxhu5bmbjqbobbSYBOQDTy8/q5WUOuoW0EisZTJlQTlBV5x4d1g11V1gFarwQnmcqdRAP3T77MKRztSl3lgRdVHSMpDLVGlKdUphsKiNO8SPnwsE5TNrgtMH9X1RHv8AYLGFXrdyYArFLSToAFdkjTSdP2+0l3b9oUdnI8SgmZIBjkZOoOtjNondEJ1a3qiz91B0LUhpRUnYaajv3I+yxR24XVFORIB3jMR3H3abd9h1YlRTq6xuldWFDm6QNttiB5j32DVi98IytU7WoEAKJgz4RPdYwoXcggGtZ1RReE3nPVebbBkBJKtzP4eNtNYBeD6KhNZSHq1AhISoyR4z32EcxZXxmBDZWQEwiZkfHc/bYKoxLeLiZD6xqQJRodvDx+ux20Lt0B9Y2yfn7suJKFZhed2qSnTRlRCRpsZjlv4+22lcR+JKEJCbzu8rBnN1A1MDSeXlaOlX/eROVd4OlJOaEqgfDv8AssA5eLykz6X1kkQkHcb7z8/C15kDhuVSMzDsFJA4qcSWlJBNzhBM/SMk6TPPu9m9hFcb8XsgKqmLqg5jIWE850lOwB+YtF7tXnUoBIJE6Ezt5mdgNdeVibqilWjUlI2BiO75mx2w9UB0o5BSpWcfsRBoJYo6AOEAKVIUJ027O2vjZrX3xjxXel21VEWqTrXEgpz06VshYUNCnIO487MtZeAUgNBIET2o9+nztYIQnMgAKGYaAyN9PhY7Ig1V3yEoyriJj/I0GG7oplCNG6BAAG/PXu/GxR7H3EhSxlvlhhajrkom4g+YsCoA6KUo5pEZjIJkfZYFRSk6uKWknsxp7Pw8PfYGgVY3ujBxVjpwfSYsrBmH+CbQmO+YGn467WK1F5YjqCnrsWX/AJUQrI1VlsqETHZ8/qtwC6ClIUAmTOm3jPn9duQl1Di4zSdilPrad3OdfObODdMg1O1uYuuXtfLygcpLt4OrB05gqI2+s621SUFEtxTrlAVSc0mTyB5nvH7LaWtwohBVmBjXcGYMRbuldfQ4HCtRgjcg5uX2d3P22dR5pXVS0ySU+h02hj1rZYPryNOvph/SKp9tsslJQFhOkqk+ktoumofK3AU9W0pcgITKtJkTOnwNjVTc2InnSpm4a/KYVpSOmTtvl152Qnb9vQKbp6C/KhDRECnp6hRbASBsEq8uXjpYNqkxHey1NtUt61OhHZp3nJgEaZZBOwInxtS4gBuSEct0tYpxtXPiZgl1d3VqQNIWyWxqT+t7I0275sj1GH7zp3VVtO1SMOFxSyF1rSSoEwRBIIMfERPcoUHCziDej6RdvD3ElYoJUr6O63ljNHfl3G2kzGs7WclL0d+N1UM1LwlxTDWXNnutxufLMBPd99oGpiG7x5pxC9w9FpTHqrhfdf8AS275oGnyElYVXMkGCCYyqJGw018bdVtzUtc0n0jEN1IegpW4ha1FQ3jsIIO206T75Fouij0jLzSEU/CK/UIROUvJbaCddhnWkx+3uhz3J0K+krVUgpXOHrtGh3tOdfetMlK4iJSFzt48jtOo3VtM3eQeYUm00zjbIVBVThe7WG+qexZRrMZ19W28ZBAI06uBAnTkY8wTdpblpxlF9VDmXsqLdMdwqZ1iOfztZxf5PTpC3mpta13KylOUlNTeZB/nBKkIXEkny05iyzSfkzuLFQQK7GmGqRtQMhtbyyD4Dq4O++moGlofKVE0ayBEFFUnZiqQ3UUDLaFJrq4hIGUFlJ57/wAoDMxOnxs4bm4tXvhW9ae8bpqq5NTTrC2XOpayhSSYVCs2b2iB3WtjTfkxr6bCVXjxOu9C1GYZo1qGb/GIkST3GyZiL8nTcuGKFVfffGRR7cIYp7mAUvXRKczygduaSLBfimHyjI51weVkeKirmHMwWPtUODp4cfaZks0eML1bYSJV1XoyMswBoKcRqfv7rN/EHTH46X+gs3hjW+30IUrIDXlqTtA6lKPHv0jbe0zXB0UOEl0pzX2xfV/qMZ01NV1LSlGSCEtJB5jQkgeNlm5ejJwXukrrG8GOPocXIbq6lx2I3CO4aweW3nao04RCbxwjyCs93xF+rn7+Kp2x+7LjjjO6MPUFImuvy+KxmipFO1DhLq1qKUJUt1RhIzHWdpjkLe0lyYbxXS3PQUK7ieLlNSssuKLzYBUhABiVDmLVb4ZcK+FmEOIFwYxThG6bmbuq8WKxdapa2UU6UKB6wqKgkJAB9YRa27/SS4EMHtcWcMkggw3eCHD7kkm25ROgrmXHogLKrI5qR9nakoMYdxOVQbqIV/zrf2KtisI4qUifRKfbYv6/AEWKv9KPgVIDOPqepzCU+jUdS77RkbPdYBfSe4VOAeiVl91YOo6m4a2OWsqaA7rXOBTN3d71SMk7tA1HlYCxU8ImhbG+r6j/AGLYOFl+uj6W9qRvN3NqXr7x4WSXOkzhFQ/imFcXv7gZbrDc/wBY4mwK+k3duUKpuG+LFZpyqd9DQFQfF+0g+lZ6w80whqH+qUp3hwSq7xZDNViZsSQrsUZ3Go3c7wDas2OMDXvgzG154fpfSLxTTLQtNSUpbCytAWRvAAzRzNp6f6S95rSk0XC6sUVSP4zfDDX+oF2qhxu4fY24r8Q714ktXdSXQze3o7T1C3eC3lIUy0lCVFYQgKzBBOwiDztWqZaV4ADtVZpYamI3LSnUi779ISP4m0vYJXXsoO4PNXn87HGsO3rUoC6i/bmZnQpXeSFQefqz8i0TMcIMc0yEpVfzwyDKjLWL0H3be73qLXBfE1TROVb98pdcpwOsb9IWr6OT29ztoCI0kHvNqdqf6SuEVBPzVI5wuttwCpxVc6Ecsjrq9IP6qPtsJ+ZbhaJcqOId3tSN/R3wYHmkcvna0Xjg088CXq1CFREhJKh7wfH3WR794QXuh5Tl11NfWVIRK2m1lKXBGpQNgrSI5z75h9PtdR4c+9lNTbHDhpKkP8UGlrUIytUwInxCnBYF2+eE9IlSX8ZXm8pB1LVMwQY5R1sz3fsFqzrw6lCwC1UZUqCFguK6xM6QQROhMd86RYw1hdCnVKepMhntZ1SUj9EHvE90RqfG0gIuijaRS5fuO+GjNSHrtxDeCELb6twv0hJVtlRkbSpU7nNtoddRaPbz4sXIla00lHfNUlByhYbpm0uAcwHHkmD4666xBsVp7ju5CpNCgH9MKOWT4EzOhOhHLnpHa7hu5YSaZhvIYzJMFaTPKRoeUHTfURaQczYBQMTyk48Sqd4ktYcvRTiQVQ7VUiBA8UOrjfusXd4mVjiZRhejEQQKi91JymNJyMK+Z2svMXPRpZUttinCgSEqzZVpGkhQ+7TU72FapKJsqbqKehKQe0UNhJA5yBvsNvdZ87RyS4TuqaiuIOKVnNSYduJBJkD84VL3s/4OkcwfttwjHuOqkg0rFwIywrsXfUvKHhCnUHl3WeiaC7yAtvqlIQJJySnw0nTQH2+wAVVI0FIS6pp3OojOmQAI1gxM+73WfiN6JcJ3VMl7FPESqRnTedGwoiApi4tCmN5VUL8dxysHS3pxQZJqWMR1aFOtqQvq6GiRCVQFJlTa9wTz5bHk+ihtsfQ1ZU3myjtEjRW+bl4TO+3di03eQ4gMuoShIlQSpSSd9xqdI37p3ss4PJOIrc0xgjHVSlbasSXvvlXlXRtAE/zkU4j2GwCrkxTpN63+4CoypV7up7tg2pJPhFpCUllaA2h1LkwSYKUg7zIEQPrBsUUulpVQXSwUjs65dI1PdzJ99lnPIKJiaOaZD+Hr3fJS5X1/VmCA7e1coBRPi97PZ42LJ4fNmHaqno3kJJMVBcfzGDqOsWqOenPS0gKvC7yyHX3WFjLopSSkxpsZMyJ8/fbhF+XF1gmUJ2BcT2YjUkiRy9uu2lnzu5JcNg5pjI4a06iD6Dcgleybup8xGumqTBnx0nysO1gtNASWEU7CIlRZo2kTryypER7TtG9nib3oVdpt8ugqBgp0Gh1kgSZOnqj4yTXfLTakqQKtCj9H2tQY57wIAOum/fZw55CWViQm8NvUwCVXxXyqJbbqD2p5CCDOpER99g14USSn0i8LxUF/SfSVKviZ7x3cxtAFlZd+1EdclIfPqySUrOugEyCdtdhHfrYH88Ela101Qytoa5V5ikjnzgQDzPdpzcZioEsCSncF0SnEreS4pYO+dR7R8NYI0Mc9e6LabwrQthaE0gSe0SkpJBSCCTuZ15gEDWykq83nELUhDbzZSWwnX1jIBkc9Ph4WLqq6xpstsllaspUlKmzodAdzsDHcNt9bSDXJZmjZMDiYwaa8mKxhpAzoSMwhIgcvfJgHTy2SL6r1EXRVocVkWwplSp00WJ31nt/Az4u3GDTt8M0gQnrFNrUhxSEFMHKDO25BiPkp/wC9tX39d+WiqEMuMPpUARpqII8tB3DTYGxi3M1V8+VyaeB+ruPifdFa5TJfp/SEPlLhLZWlObv8O8cx32tXR8RcIoGerupylKwCk+jocQZnYJUVbA7jWdINqrYcaddvy7qppamqigU6lZSrKqC2ruMnUCAORPkHq5VXi48tNMp1eQleXryVhIgAkETvpA0nUSCSK5i4iO2bh7Kz9145wdVoSGcQ0FO0ZGV5BpoIiTLqR46jSdfGzlpRQ3kA7QV9PUJKZKW+rdAMETKdxvuNfE7UxefqVMFyrdS2h4DIHkBAIToOyUwQB7wecDLpumWqciQpx0q6txmAoGE6jQEx2Tp3c9IE6gBNw5GbX23arrJuoHJKWyoHKqEqTOv80cvMaTppbpV3oBzOVKkhSpIQ4AQCrlPkoezTutUCjvDiC0hTl2X1fA6tIzoTXfRo0MjIVb7ciIAiJss0/EHibdak071ZQuIU561U1lCtCP8ABpH625Mkk66pFoGiI5ogrmnkpl4jYeYqlU9VSvpFSiYzKSpQJgaeACR9vKzQa9IRlSpalPnsqQ2iQ3G+o03G8czvtZCo+KVbeakM112UruZc9ezXtJKQVCVFrMTHMQZ5cps5GKhupWVs5Ql6BKBqecCNOY/G2jTtyMylZ87g5+ZMa97lqXA5eNW11inHSvMUZlZUmBqRy203377c3dUroX6aqQzmVTqSSSDGZOpG2nPvs66ijvG82XvpEGjCiOsW5uBMxI1G9mY+mHwgkuJSrICTr2iUqI7iYE++wJGW0Ksxu5hWaw1iq6qu6GQqncUtlEGMg0OgVMzsAeegO9lNWJaU5stM4oE6hRTA84GmkbD384S4c30oUxoS6MiZCUk9qZ7M78vq9759NMEqaQOYUUdkToZP7PCd7UTSM3V4VjraJ1uYoQCU/m5sAjm4JgA6kwNJP3baFjf9Q6c6adhMa6EmSRuPCSeXKzdSl1WchLsA6qSIPs7+Wo0Phy6QpeSWn5AJ0y6CeR7z4xy87MKVgS7y8pbcv+tKlEtIQRCQdDGu+p3MWLqvi8iqF1RQASYDWkT39+g9xsmoW+lMuIbUFEbHSNNRy+ee9tqNSQUltJ17Kkrkjy7j+Npinb0UTUORld51i8yVPqIUSIGih4TYFdbUryrDlWcsp0J0+qwXVrCspzISR+kkQO8/XbspA/RUpO0FcTIM2fgt6JxM7qgvSH1JKnVPqVEdtR39hsAVreT1sgpIAgqnTWOdjhZZWcnVEawClZAHedTpvbldMSTKTOhBPaHlI5/OtnEQHJR4pKTllSoCqmFn9VUgmJ22k/O9tpcebSOrLhB/WTIG3lBHlysaWypKh9ITOxUdFeff9elgF07hAUlOm/rCdNN41H3edpiMKBkKAC3AvMH1jKJnLHt7o+H2cdc85JCxrqDMeEEcxHxNjKYalYpgVerJ+vQn557W5cUy6AFNrzSNMoB90a6eNlw+ibiFF3HCpJJQc0xOhM9/z325VVuyUqRmgFPKMp8Bt8+djaW6SQc6UlQkCPMknW3KWmEkKbM9nUZdPDblv5+FnDFHiIAXm+lKevpwE7zvMePz9VuV1gSIUFZiRpI9+vL4b2MKU2FhB6uQTqe7kInzsEopz5EBBVJzKyyCfGBp7LSDFEyIquqg5gFZUgaxrE6GPnym2/TEFsqXyjQkAkwNd/nnYXKkKAbCAADGVMAAbaH2fO3DiFApUhzWdZM92v4febSDAFEvtqg1PsuJKcqNOyQEzp8z+HLlfoy0mIgiZDc7+zT51ty8lxcrStJQTmBzQO/vj58rAO0j4UVpKErB0IPMg698zaeVQ4iEKWVBSUkAzpCiDO33DTw52CWhl1KgmdfVJUTE+E/D32zqatsrWqoSYBIUiR7O7kfj52DKX8uVTiOc66c9Rp8zZ0ibrQaaTBK0mBGg9u2/7feGUFAMLJygDKdN5077DEVRzIKGgpAiTtMRy35WBWFIT9KwmJhScpk+/Xl8LMLBMgVuOZjDoUR4gHaJmN9/CwROQ9Xn1Om2/OZnv5T74sYXUNIIQ9TAJGgOw84Aj3RYJLzCsspdXKIBSkxEd+3dp+2z5gFEtKLOOEBZIWoZYIDJBPjvG3KwrK+0QUpgakwZ7wddd7GUtJUAGA6EQCStB7U858LHKe61rTlaRkTsAoRBJ8p+YtHOAE/CJRcFtQzOMN5jqrTnbLLCbqTlH0dOdN84++2WbiBPwivTBnBOGKdtLdNcV3MoSZSlFIgAHyAseauejaSEpabAGwDYAtlst5WBzXQZ3LSqNsqgEpHhbS7ta2UtRBFstlitYDuih7hsUAqnYaPqFXmbacbaSAoone2Wy0soCM1xO6AeQlvUJSQIOotouHq8wQjeNrZbLQGhR262ukm+LwcoKB6uQ2lRbSSE7fG0DYh9MxLW+m3rXuLMlKUISEpT5D2Wy2WNTC9yVZGyRm8PUi30IU44c5AJ05zrYN26KNSs6wshIASkK9UfPxm2Wy10KBXKbnoasKpn2y6hRCVB05wrbcc9/hYR25booKl2morspWgwstAhlMnLGu28mbZbLWInEDQoL2g7hdMtHKeqKWwEgwE6a/tsNUNZ3UMzHVtpEgbzJP12y2WMHE80PKByQSkBKglRKpVG8bAq/s2FS029QNqKT2XlpTJkjQSfPS2WyxWqFkCmlRCtRokKHZH60fZ8bHKKjS6p1hSzl6latBr2UlQ/1Y9ptlss/NRRV27KZKisgkCZHfqfZy7rao6dlisS9ToCVtSdYUFabEcxEj22y2WmE3NKN54ao6Z9tbCsjb7aXkoCfUCp7M8xI/bzIHDNG+XWlLKSqIWhICkmYzA94tlss5UVGfE3BV1G4l42bKkVbDxp6pEAipCHurCp/RXporXTQ5tCImcbp00yCadJLqdY00k7d2omR32y2W0qXVgus2p0foiLt4U7VOzFHmTUOpRClJOUkFQUZTr6o9uvhYkjED7NI2pLc56jqZURmGm8xrsNPqtlstdACplxRlu/Kl95lpTbecpR2ymdVAHb5mxGsvy8KfqFBacr4GiUhJAUTz2MZRytlssQNCg5xQLt/wB5mobKag9YlcBZSk6bRttrz7rCs1t4CrNFU17ryVth0a5QJMRA9vw7ptlssQNFkEuN1091tO6287UOPfTIaAJiApI1ka8/gPGQ6pT6Hmiap0l2QjtHsJgGD38+7e2Wy0g0KNyiqAXqVpxMI6xAMCYB113125/hbpqrWhtmkqJcU64W86CG4iNSANfKbZbLPYJXKMIYaU6WinKSgjM3CfV0H2W5ao1J0D6ozaAgHYkc9eXfbLZaKi4lCKu1vrkpQ6tvKJlAAOhJifad53sXCo7aVuEKWAMypIlWokACPCLZbLPZNcrhLlMhwNOU5UaghvMFAQB36Gd/D42J3heKaJ5akNLW4VQVLcBmE9nSNYjnrbLZaQTIg3fKK1SFChShcDtFcxvtoI9Y7Wc133Wm8mgsOJZAMQlpJ7xM7k6bkk7Wy2WcqKM3thFV10X5wfr26lK6xCAgMFBBVOs5iDuNI5eUCM3YKclDT6gWnEZTAncj7PjbLZYjdlF26hz8ztUeJGFIdMoqkpWQkSuXIJgyOR3BGvnNi6LhrhasqPREsvIUJklScsZc0AJSk+8mJ9lstlqjDurJAstVWAsOXXVJYQxUqKd1JrX0BWvNIXB/b327Zwth1xlbjl1NOgEdl8l6ZP8APJ590Wy2WJcqIAXKLhwk0pCThagClOJRKGWwNdATKDO/f37SbKj+HqG6aZNUxTUgzZZCKZKeceI2PdNstlojdTIC5ZWt5tKioygSJMiTptsdhv3aRZpvqWq81ZVBPWqCdEjQEgjYcj3R7LZbLFi3Q5eqaeNOIl5XHeYwrQUiHVKQ48airdW4NFKEBCSmPV5HaO60fL4iuuvVbF5XO065SEoK6d0tJcAAPqqC43jfv77ZbLDl+cjQ/MTg4acSzeV+/mpi5jTgNtq6w1OYkmYkBAmIPjqNRFrGlLTgClpVGVKuyqDJ35R8LZbLDdupM2XQo0rQH0uLTDYXHMkoB38iB7LayuI1S8qR2QSAefjbLZZk66S2o9YM+gPZGojTwI+RbbVMH1HK64mFlJkhQME9/lbLZZJrm625QuM5V+kyE9yYPfvNiK3nUuDOsuELCDm1BM7m2Wy0gE91yupcStCFZVaADSBHl32EpXy6tqnEhTgHbMGCQOUWy2WSQRu86J2mioNSFHMAAEERKlDvPJPxsmocWokhcQUpjKCD8J+NstlnSXLmdbiklQ7ImY99tqbSQSZJKjExGgH4++2WyySCxy7lJQ44XwQAITk0idt7FS1LikgpGo2TuZtlssxTIJDHXJWvNlCezESSQYmTbZphlQUqgGdCJ2J+742y2WV9EkGpDjRUEukBOke2LDU7briAsvzMZQpIIA3juNstlop7BHEXYpX8q8lYjbq47jyPjYT8zU8ZXHFmNDkMTInnPdFstlmcSnsF0xcNHUIWtLj6cpGinM0z7rHDhukpXAgLKsidDHjzkmbZbLDc4orWhaTc7CnAAGgpa8kloGDG9gn6ZDKM7SUJ5JEHQgnXe2Wyww43Ug0IslEpQ8lRTnKtNNANte/xsNT0fpLvVLWkCSkdgHafutlss9ynaAumaCmeLjamkDTMTlmZIH3e6wzV20iSFKSqCf0SAdQDrIPfbLZYYJUrC6EXQUmYyzJncne2Wy2Wknsv/9k=" } ] } diff --git a/samples/js/node/notebooks/hello-segmentation.nnb b/samples/js/node/notebooks/hello-segmentation.nnb index 8df4e6227701e9..a7da34a2799edf 100644 --- a/samples/js/node/notebooks/hello-segmentation.nnb +++ b/samples/js/node/notebooks/hello-segmentation.nnb @@ -17,62 +17,10 @@ { "language": "typescript", "source": [ - "const {\n getImageData, \n displayArrayAsImage, \n arrayToImageData,\n transform,\n downloadFile,\n} = require('../helpers');\n\nconst { cv } = require('opencv-wasm');\nconst { display } = require('node-kernel');\n\nconst { addon: ov } = require('openvino-node'); \n" + "const { display } = require('node-kernel');\nconst { addon: ov } = require('openvino-node');\n\nconst Image = require('../image.js');\nconst {\n transform,\n} = require('../helpers');\n" ], "outputs": [] }, - { - "language": "markdown", - "source": [ - "## Download the Model" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "const baseArtifactsDir = '../../assets/models';\n\nconst modelName = 'road-segmentation-adas-0001';\nconst modelXMLName = `${modelName}.xml`;\nconst modelBINName = `${modelName}.bin`;\n\nconst modelXMLPath = baseArtifactsDir + '/' + modelXMLName;\n\nconst baseURL = 'https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/road-segmentation-adas-0001/FP32/';\n\n\nawait downloadFile(baseURL + modelXMLName, modelXMLName, baseArtifactsDir);\nawait downloadFile(baseURL + modelBINName, modelBINName, baseArtifactsDir);\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/models/road-segmentation-adas-0001.bin'", - "" - ] - } - ] - } - ] - }, - { - "language": "markdown", - "source": [ - "## Download an Image" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "const baseImagesDir = '../../assets/images';\nconst imgUrl = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/empty_road_mapillary.jpg';\n\nawait downloadFile(imgUrl, 'empty_road_mapillary.jpg', baseImagesDir);\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/images/empty_road_mapillary.jpg'", - "" - ] - } - ] - } - ] - }, { "language": "markdown", "source": [ @@ -83,7 +31,7 @@ { "language": "typescript", "source": [ - "const core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'CPU');\n\nconst inputLayer = compiledModel.input(0);\nconst outputLayer = compiledModel.output(0);\n" + "const modelXMLPath = '../../assets/models/road-segmentation-adas-0001.xml';\n\nconst core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'AUTO');\n\nconst inputLayer = compiledModel.input(0);\nconst outputLayer = compiledModel.output(0);\n" ], "outputs": [] }, @@ -97,14 +45,14 @@ { "language": "typescript", "source": [ - "const imgData = await getImageData('../../assets/images/empty_road_mapillary.jpg');\n\nconst originalImage = cv.matFromImageData(imgData);\nconst { cols: originalWidth, rows: originalHeight } = originalImage;\n\nconst image = new cv.Mat();\ncv.cvtColor(originalImage, image, cv.COLOR_RGBA2RGB);\ncv.cvtColor(image, image, cv.COLOR_BGR2RGB); \n\nconst [B, C, H, W] = inputLayer.shape;\n\ncv.resize(image, image, new cv.Size(W, H));\n\nconst inputImage = transform(image.data, { width: W, height: H }, [0, 1, 2]); // NHWC to NCHW\n\ndisplayArrayAsImage(originalImage.data, originalWidth, originalHeight, display);\n" + "const img = await Image.load('../../assets/images/empty_road_mapillary.jpg');\n\nimg.display(display);\n" ], "outputs": [ { "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAQ4B4ADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwCx4XdIbgwXClJAcYIrssYrk7gJZ+KIGkjYK5xzxjIrr3FWQQPGG7ZqFowBjaPyqyaaVzTQNlR0J6ZqIAhquFcVG0eaBDUPIqUVDjaanXkCgLCgU4UCnAUrgIWVfvHFV2v7NThp8f8AAaleNXPzVk6nczD/AESygLOxwXI4FFwNpfLlj3RuGU+lQpGyN8qVz2690NVVEMspGWx0ak0zxZLdTzm52xRIpwBzzRcDo7xZZLVo5ogUPUt0FcT4vjV9PWFLcu0S7g6t/KsbxJr2q6gpQSyRRdgpxuFclcSXaQhjdyEnqhJ/nmi4G9Bayf2YrXTt5XYHnFZwt5NPaSeODzbSTPNN0ttW1GSKxQHa2TyOgrptT0+e0SLTtwbzyACO5+lRYozPC9oxnNxO7pDnMTk8KfSrFxZXWq6pKhkwinGR0NdPB4NuLbSlsTiVT8xYtjr7Vj654a1DTLAy2l68iLnMbEZA9qLknNWdvZt4lSw1KZIIkfBd+h46e1ekapofhiTQ3itbW1lfqrQvl8151pmjWuuRyy3F7I16Af3ZPJPqKs2do2j3W3UHe3f+CRTw1FigMNzfmSzZ5kQnGSMgVh3qQ6TcSQ3OLtgMLvY/J9K37+5s4Uku7TUZ/O4+UKSK4i+mF5cmYNu3ck+posMbyy5dyccDPpSp5j4iRiwP8PrUGT0NSo2wbwSGHQikMle3a32MwwG4/GlMMu9VVo0ZuhPSo5JnuJA8jZIHFNl+fBzjHqaBFiOwmhunlmRdiqfmVsirlnq8aOLecHyuzjksazYZbiAMUnYbhg81FEI96+aCU74GaAOma1vb2LN2c24BMRPQCs/Sb6XS9ZaWGTauChfHBFR34WFU+zXkkkJGdp7UyK4SCycpjcccEdaLAaCfYW1Bri8uRKkmfM2HnNFtaaPJrvkmV2tHX5SeADW54d8IxX1/YT3UsctpcIxIBwVPvXP6xp8en+K5rGJiUSQKM0cpPKa3/CRW+n6VLpRtvNWOQiNtwI/+vWdLfvfeSWhSIrxGV/i9jXUaLf6A1u0VxpSy6tACrlE4ceuOlcfr1zDf6k0ljbm2hH/LMrgq1FxGxYx6yiPPbysYozzbl8jHrius0m9uNZ0aPSxGz+Ud7luOnY15vFdS28f7pyr92Hc1JDq2pBJFjvJVLnJx3NFxntH9px6NYCEaatoshCCRGGCxqSx8L2L6L5M9tDNMxyW7ZryDS9TggvVN804PVlYlgT6kVr2Wt6k2rvdWM9xb2rr0JOwn27VdijW8b29jolsmmWMT2zyMDMuMBh65/Ouv0bSINK8Mw29pLuE+1nkHcnrivK9V1m4u/EkE085uUgkDZf8Ah9f6V6ZpemQ2iS6hdX4e0UiaGPOAhP40WA0k8PabBA8lvbtHK45dj39TWHeeEtav1mT+04obV1+ZVGQaS+8QLrFuWsdThtXRuR1Dntn9ay9P1XXre4KX0m23kGHA6E+tFiTAh0QQKy3lsjOGIBVvm46GmXGjrcN5xuQFbpvo1a9QXUiJvkmMvX0FUr7T9QjszNJBMVHYdBWViiSFvNu5LCaczRNgI4GcCsnVNOW0umjVwy+3arNnLHEUaMlJR1BHarchnlu4DLHHsY43D19/SncDnLNoor+NHOYycH2qvdwtDdyow6NXQz26jxEzMmE/vVXks5Lu6ZhA7qRkOBwaLgR6ZHBtUuiuCuWU9V9/erjRL+/gT94sS8HvikvdMXTRbXdrI6lxhl7gikjl8jJKhzMnKH1+tFwHQ6v/AMSiO2AbzYTgEc1dmvFvL+NGUujR4BHYjvWdYW7Rid3+VA3LdhWlp1qE1Rd1wrfLuGB0NK4FaF5NPuLmGX5cpjHqOxre0v7Hb6QA82/7QNzfWoNatfNtBJcxhplG1JE6fQ1oeH9NR7ZnVU2SQlJS3QN2wKmKDmNvSrP/AIp8zykmV1O0k+nT+taOmbjp0LFshhkVV06QvpK2TnEkP7s++O9aMKCKFIl+6owK6LEjwMUhHel6UlUAUtJRSELSGlooAjNJin4ppoAb1NGOaWjGaAuJilHSlxS4pjGkUAU4iigBrNtGaWCZWIPSo5BmliibOcVRJphQYwRUDrzxUkZITFBFSUVXU1CRVt1qBl5pCINtIVqbbSbaQyIJShOamC8UFaAI9vtTSg9KnxSFaLAVWX1phWrTJUbJzQBX2imkVY2cUxkxQBARTcVMVppWkMiIpMVIVoxQBHikIqTFBFAERFIRUpFNxQAzFJipMUm30oAjxzSYqTaaTFIYykxT8CkxQA3FGOKdikxzQAgFLilA5pcUgGd6KdRigBvain49aO1AxopRSUCgBaU0YooAaKUdaKKQh1FAooGLS0lLTAWgdaKKQDhSj0pBSimA6l70lLSAWnCm04UALSikFLTEKKUUgpRSAfRmkpRQMdSikpRQAvanCmU4UCHUUlLQA+lpKWgYopRSUooAUGnCminCmIeDzTxTBThSGPHWng1Fmng0APBoFNoFAiUVMv3agWph0oAUmkNBpKAEpCaWm0APBxSim9KXNAx+RSg0zNKDSAcTTTRTSaYAaUU3NLmi4D80ZpuaXNABmjNNzRmgQpNMNBNNNACGkNBpppgFGaSigBDSUppBQAAUYp1KBzQAgWlp+KaRzQAyilNJSAaaZTiaQ0wGUGlxSGkIbSUGkpjFzRSYopCH5opKTNMB1SJUQp6NQBLTCafnimHpQAgozSZNJmgB1Lmo80uaAHGgU3NGaAHUuabmjNAgJpDRSUDFzSZppooASkzRmkpiFopKKQwozSUUxDs0A03tRmgB1JSUZoAU0UlAoAdnijim5pM0wHUmaQmkzSAXJpM0lJQAE0E0lFMBKKKKBB1pwoHWloAKKKWgBaKQGlzQAE0AU0mgE1Qx/ia1kuIopYx8yMK20bdBGe+2msM0oGBikAE80maQ9aKQBmkxS0U7iGlAetAUCloyO5ouAopsk2w4qvcysq/KSPpUU95ALcCOVTIecZ5pDLJkZeoqNRmXfVa9kjn0yNjJskDcnNUDfMjRxGVX+Yc96AsP1CaSbW4rfdiM/LVG80NbHUzAo220y88dfWrerTLDqNpcNxGxA39hXUtBFcQDzE3rj5TnoaAsczqGj262tpBFEpdCCGPJNYfjTwvazy2ssXlwHA80HgYrtzEJLtfnK7RjjvXGfEJ8S2IkYNC77WweooAoWULadeIli32tmXop5X61ng3sni22jnhZpRKpZAeMV3ek6NZ6LaSvZfMzru8xhyP85rm9FtpI9evNTuYXZ8kxbRn5j2oGelTRL5eWcooABIrzbxTqWr30jWOnW6JGmMOx5biup0576SxuVudy5O5cjt6Vz3inRr+Robz7UEVzsKL2HHSk0Jnlsj3dpeFixjnQkFh2q/NLe6pCn2m/WTBz89a1/wCHGMFzLHuRIunncF/oa4pNjPtkBU+hFSBuHX5tMtzp6rDOACPMx61z24MxIqw624+6OajcFsYA49BSuNMiAJODSsjk7U5HpTtjbWYDODTo2KkOOtMpsiKNFIokBXNPniMMm3IIxwR3pJjNPLjBfaOwqOLDyojuUVmALDt70CJ7SI3kiwPyn8X0q3sTTrs7YhLEOhI5qG6tDp8u1ZVk3DKuBzVcMXU+Y7b/AO8KANq70uC4i+1wXg2D70eMlRWG6Ir7d4fHpSwxyvJsVm2n72PSpPJj7ZNAEsF5c2+PJuHRR/Dnj8qfF5moapCs0nMjgM+KhW3kc4UZ/GpRZMMGSXyx65qrgdVIkPhy6uooh5wKhlmJywqbw3Z6bqtrqOoawrM0Z+8DjA/yRXItEyY/0hnHuc1NC1ykbi3LjeMNt9KzbFY0tbsLCO8B0S5ae0cZYHsao2NqJrp4kcJMqlgD3rtPDvhY33hWV5SYZXYlC3Q81VtNK0vULI292Gt9TgdQ7g4DjOD/ADoTHY5B7We8lFtj/SmbaF969Wsw0Hw3vbG8txDcWsZQqR36g1uQfDjQ9KC6jGZBKg35JzzXnfjHX5NavfsWnRzNgkPGuCHx0z6d62iSbHww8OR6tb6gdQsXZJ4sQzMvGO+P0qfUNH0fQ9IuLTVrt/tIBCRq/JA6ECs/wleLp0NtbX+sXVtE5YmINgIew9s1razp2g6eZNVhvzdTOp8uOV965ApNDueZxRWE1lJLJPsmHRTwSKQX1xdxJHJfSiMdOegq5a6rpNxqlxPqOnCNJU+VV6BvWsmeFc5ThGOF+lSM6S4hgj0+3u7SdDdRjJb1FT2WvavqEXkrFGPLIOQOvtXOz3c4t47d9vygEMBjNaaSXVppUd1FcQFCPLcoecepFJgJqN47tMTbwiVjkup/pWcs7hIw0jDnPFFzAWlneGbeEGcp0NU4ZPNUHeW+tZtAbdgWvXleaYqQcZA61WmnNtdvHHPIoQ44OM0unXQhlSN1Gxm5IqvfQT/206hGdexAoiBbjPyr5rn94Sdp7elMMeV8wdiRRczrKY9oxs4NOVHZGQNt4yoPQn0zTAmjvV+zzWMUf+uA3M45I9KjiDWFwYmOG2n8qWyTzpVhn2pyRuA5FNu57ckvHLvkVwMkUAbbySXOnwsZ0KsACPf0rU0yaW30bZN8m2XbnqcVyEcm+3EWPuHdmui82aeSG4ZCYpQAQvQNVw+ImR1kDoAjDGSOo71dSbJxmucspyYsng9Mela1tKG613OJCkaeaQGo1cY60oOelZcrHcfmnA1Fup6sMcmpaZQ+iiikAlNIqQDmkK4NIRGaKUjmkxTGKKXrQKKAFxShM0lOBoABECeelTKoUYFR5pd1MRLmg00HilNIY01E45qU0wikBHijbUmzjNAHNIBAuBRipQopCvFMCHHNJUhFIRQBGRTSvtUu2kxSAgK4qNhVlhURWgCArTCKnK0wrQMhxSYqTbSYpDI8UmKkxSYpCGEUmKeRQRTAZjikxT8UhFIBhBppFSEcU3FAxmKTFSYpMUAMxSYqTFJigBmKXFOxQBSAbijFSEUmKBkVJin4pMUCEIo74ooxQMKXvTcU7FIApO9LRTAAKUCgCloAKO1BpRSAKWkFLTAXjvSikpRQAtOBpopwpAFOFNpwpiHUtJS0hiU8U0UtAD6UUynUAOpwplOFIB9FApaAFpabS96YDxSimU6gB2aKBSimAopRSClpAOBpwpgNKKGIeDSim04Ux2Hg04UwDmpApxQIep5qYdKhVcVMp4oGFLilxRSAaajqQ0ygAzS5plFICQUtMFOpgITxTc0Gm0CHA0uaYKcKBjqXNNooAXIppNBNNJoEL2pKXNGKAGUhNOIpMUANoHWlxRimAGkxS4oA5pAGKXp0pTzSgHFACbjTSTmnlaYRQA0mmE04000AIaaaU02mAUUuKWkIYRTaeabmmMDTc0pNJmgQZoptFADqcDimClzQBIDmnGowcUu6gBc0mabmjNACilzTRRmgBTSZpM0lAD80uaZmnZoAM0maM0lABSZoJpKADNJRmkzTAKWm0UCFzRmkzRmgYUZpM0UCFzRmkooAdmjNJmkJoAXNJmkNIaYC5NJmkzSZoAXPNGaKKQBQaBRTEJTgKMU4UAJ0pTS0goAKWkooAKM0lHagA60nSkzR1oA3s0ZpDTaoaHUZFNzRnBqQH0ZpM5pelFxh1qOUqibm/CpDwR79KjnGUAPTNAitDeQyXL20g2OBnLnANc5r32TyjLHIySKc8fxVvtpYu76OUPt2ds4zXE+LJ3GtmyilEODkE9CeOKAIv7QIMQkd1EnTf0zVPWdUm01kWFtsp/EGub1rUp7iQW7kfuzkMoway1nuridYvNZjnI3VBR1U3izVVtTaXMUbqwBViOQfauk0HxdcrZRxSyomBgF+Qa8+vp58RJcMhMfTbVR76UqUjfA+lAHuEWom4w8DxyEclYzyBWd46ijvfCq3jIUeE5XNcj4YEU2mNIJWinT+INiut1KfT9U0A6W16FeQ43Hkj8apMRkaFqUmpaQkUeqLDcJ8vlSfxVYvPFdzo+2w1aBUIYNFcwcqxH/66msfh9p2l6fPNdTvOVjJVwcAEfSvNZINQ1i7kWFHn8rOFzkgUAevDxjaXNk8UKPJK6ZV8gDP9a57UtQj1KysnF5KyRnbMiA5HvmuBt7s6PqAE1o3AIdJOK7Lw3rM0LyQW+nKtvI28SSdQe9MkveKrDTZtOs47HXBI7ZykkmR+I7V57rrKZULlPPVdp2dDWl4xsvtPiJ/s0AVushXow9a5i5RlcBuo6mk2BN5DN5e1gzOcYFWLjSdTs4vMuLKSNOmT61QlBWINHK5Y/w5rYOr6hPpq20tzJKrjZ5Mg4+uagaKEKs6McEEDJ47VC7fIxHBBpHkuIcxuCo96YxOMZ60xk+lTyxXhdWwwU444qO9kWe53KBkj5towM1BU9k8cV0DMpKEEHHagYyIKrZatDR4LW6v5o7hx5Yj3AHjms+fZ5xMb7k7cYqa1iMu8KV3KM4J5P0oAsWkoivZhHllAK8DkirV5cafLbokStG6dQ3VvxqSCx+0Oy2ifZ7tVLP5vAI71Jb+H5nc3GpKUhxnzAeCKAMqMSErJAwUdiTSzHIKTMrdwR2NKVSF/KEgeAdHHU0hi08sWEzrntUgFo1usmZ4y4IIBXqD61q/Y760ZRbEN55wjjrk+tZZWzjQskjMR612tlNDfeF97bEuYkLA+vPBoYE19r2seE4rW2do5QV6EVm6p4l0W+/0hIZldkw2I8AHvV6Oa21fw/BfNGftturLluhPeuFklUrtzz3HYVSJN5/E+srYfY7fXLmS2kXHkvjKj0zV3w7qFjo2m3LTiSK+IIBdMnPtXH52pvH3gRitSPWJJJoGu2WWDd86kYq+YD0U6bDqvw2W7MZkmdi0rr97OcV55eaWttYxPJNNCXBK7hxnv9K6zTfGT6Zqk9laQeZplxggHjySeCR7UzxNaafbaebu3n+2sjB2Jbhs8fdNK1xnn8KbgCSa0ES1WEpI7GYnII6UXdxa3DoLW38kbeR0ANJZadcXLkww+b5XzNhsYFS0M0bTSILu2umlmMckI3D3WqukXWy3ngeON4pj86sO9TfbLm8na3jZI/MOwZ42nvmnrpL6ewildX3thWXpmoAg0NhG1+m9VHl4XPqM4qpbad5sLzB/KkDcHPy/lUF3byWV5LBKMODnHsa1YFil0J1iyHYbmUnIDCgCgFuoZyqxszoQVK9Ca0rq8ubJ4rlo/JuGXJGOJBVOwmlbVLSPqjjbIPVe9aOrQSXYEdm5a2gXkHsanYClcQyfZorqNWfzgWIA5U+lLcgJDFIxwSAKtw30YsYotrieI8LjrVm2sotVgYbtrLnYjDv9a0Ap6XA9xDPI4BUKdgB5yPWrmnabFfWSPOmzaSCR1P1rMXzLCa6gD4ypXOOtX7KZk066jL/N5fHHWqsSNtIEdJFDh9rkfL1rpEglXSIrdT8ynJqrofh1fIhuLkOzyDLK3Ax2xW/5AiQIu7A6ZNa0YJO7IcilEnlxn3JNaUNtdeWrKnDDOarrA0jhQK6izQeSoI6DFdcnYzMRxcQHDg/hRHekcFq6pY1YcqD9ayrnQ7eR8qNtZcxfKZn2z/aFSR3g7tVG+02exYHIaM9MVXhLyNhRmteWJJ0Mc6sOtTqc1hWskhulRvxrozCVVcdxWM4lp3EQZanMoFKmFPNK5BrMpkBFMNSN1qNw3YUwEFP7VHHuP3hipQKGgEozT8UmBU2GNNAPNBGaBheTVCJQMU4moDcqXwtPL8UgHUlJmjNIY4dKUUwHmnA80AyVVpXXilUjFDHIpDITSYpxHNJQAhFNIqTFLtFAEOKYy1YK0m0UgKbLzUZFWXXGagYU2IiIptSYppFIoZTcU+kxSENopaKQDKKdSGmAyjFOxSEUhjcUmKfikxSAbijFOo4psBlAzTj1oxSASkxT8UmKAG00ipMU3HNAyMg0Yp5FIRQA2jFLij2oAbS0YpcUAFKBRigCgBMUU7FAFABS96MUYoQBSikpaAHDpS02loAUUvam04UCHZpRTKcDSGOFLSClFABTxTePxpR70AKKeKaKUUIQ/NOFMpwNAx1IDRQKAHA0opoNLmkFx9LSClFMBacKbSimAuaUUlLjikIeDTlHNNAqdFHFMY5F9amCjFNpy9KQCYp4FKBk07GBQAgoNFFAhpFJTj0ptAxtNp9JSAQUuaBS0ANPWmU8ikxTYDadmlxSGkAtGaSk60wCko6UdaBBThTacKAExSAc07FLigBMUhFOphPNACGgdaQ0ooAeBzT6bQTxQAE1GxpSajJoAa1NJoY00mgBCaBSUUwJBRmm5pCeKBCE02lzTCaQCmkpKSmAGlFJRQAtLmm0tAC5ozSZpM0AOzRmm5ozQAuaCaQmkoAdRmkzSE0AKDzS5plLTAfTc0tJSAKSikoAUmkzQTTaADNGaQmkpgLmjNJRTAdmjtTc0UCHA0pNNopAKTSZpCaTNAC5pCaSg0xBRRRQAtGKUClxSGIKUUUtACilpopaYC4pKTNFAhSeKQmkzSZoAdmkzRmkzTsMSlpKWkI26SlpKoEJS0lJSGOBpJJNiFicAUmaGUOuCKTQGdqBl1CydYi8c0ZDp9aTS9X/ALQi+zzgpeR8OpqDxBqU2kQRfZlG5upP4Vz6+KYtPZ7i8gjW5Iwqj+L9KAOwvbuXTIlnWDzVzgiuE8UyW/iHV7aL7OttLjJJbJPTpUN94i1HX9LnlgvYIkT/AJdj94j61xEt3vwXeTeOjA9KBFjxJpQ0jUvKRmZSM5brVfUNMl023gujICsgyCB0pZ9VkvihvXM2wYBIwcUXeoJd2ot8kRqcgEdKzZSZn3DiRtwlaQ9yajiZkkq1Z2U105SNVJHqcVPqOlT6fbiWXbgnHBpjIobwRZXzWBHQirn7yezWZZJGLdcdqzRZSPAsyjIY4q5G0lnErRTHPdTyKkC6uv6ylqdP+0O0J7MxOKowPeWVys8c3lOOhzW3BdWckSsoiaU9Vk61jX1lHDcA+d5wkxkf3TTuBrXfiFrmF0nWK5kfBUhfmBFYr6zrEsJgWSQJ/sqAarRMlrdqxGFHWrb38xlWW1+7jB561SYmjQsvEr2dkLe4sxLKAQJJOuK56Us7sWPU5rQRftVzsn3xzHoTwKoXAaOYxsMEUMdhbS2luJsRAMQM4JxRLdbX2OcMKYkkkTbo22t610cGoaUumk3GjeZuGHlWQFs+tILHNyFpny7k/WnFAF6g1CWR5WeMFY8/KD6Uu85wKkYMvPFSLGpHLhahaQr1NPEc8lu04jZo0PzEDpTAa2BUiW8pg+1RFRt6g9RUgs2NkLgSIc/wZ5FRwTyW8qOG+TcNw7EUxGql0Lu18i7il+1OAUf7uR7VQMtxCpt3mn2d0LHFd5/ZMEeqwOYHktJVzHJn/Vk/0qxomm3D3Gpx/ZftMMQZopP73tVE8x5z8w5ZSE9TTjt7GpNQnkluZoXQIgcgRj+EelVTWZRdVbSZzmTao6pj+tWppGiC267hHjIIGdtZSY3gMdoPetBDsQDezD1NAD7DUJbS3ktJpnjRjlXUdD6VCsP9p6nHbxMi7hgNjrikuJUaQjPyeleo/DfwhZDR5NV1aOMtI+YnJ5QD/OKYjzTV9AutIWF5WR4puEZfX/JrOC4YpIMr6V6t4u0S91S+EVvLBLawneixnkHiuL1uGC/vAtjDzEoR+MZNMDKi1vUICYoptsQAGMc4Hv8AjTb28iurjzYg67hllY8VZTS1isJp5xsl28DrzVeeGzSxtp4ZXFw3yzDPQ0AQxqz9BWhZedbRTbWYGZdoAPeqAklg/eRK+R/s1r7Y4dO81GLZAJBPepGNlaK7liSLCzKmCQO4qG4g36c1z9okkYNtIY8ZqOYgSq8bbXU88VOLK5e3N0UIhb0bgmkBtaBeWs+lXh1IRNIqbVaVckj0rmLSbyt8YOFLHFWhMUhjVejEhh6inT6NcQ2sN5BCzxP6daAJrO2UE3OdskJ+aMDt6itPSoUs7S8l+0RzfaIyQEPIHuK59bieJpX+ZX+7gjgirCWzQOHgnQ+opMAsZVj1IOqbVxgDPSui0Nbn+zmuBCZIWlI3g8rXPRR7pQo/1jdDXS6Tr1nZ6HbWV1avL5UjFijcgH2xTTAteLPD8EQtrm2Oxm+VzjPBxiqreF5bTS3mk+e5hx50ZOMA9xXZaRfaZrdnNFGBIn3fmHIHarj6TLNKocK1uVCuM8MAO9apCsZPhjWpr9vsd3aqnlxBo36ZHTpWxd2CuS6rVGwsvsfiaS0QfuY4cxjuAe2fwroimOKuLsJxMSC0WNjJjDdKv2hcrh1A9KsGFc5xShAOlVKQuUmTG3igrmmqcVIOlTcLFW7t45bdg65rKttPijdtq9TW3cD9y1Z9s+XxVxYWGxaUY5xMy/KKtSSDOKuZ3IKrPAvLNQ3cSVipJcRI2M0pNPEFkvzyLSJJA8m1Y8D19aQCAUJIN5UD8xV1lWIfMOKpyXdtJJiJGz60wQr7Qh45qu2c5xUsgO3IpY3Qrz2FMZGCWxilYFRkimo2GyBVyMbozu6UgKIcEZBqZMbeeaSWNd3AxTQSDigCH7MiSFxnJNPqU0ygABpSab3pe1IYgNPDcVCzgGoBc470+UVzQV8VMCG6VnJNuPWrkT4GaHEdybjGDUZFIXOaTdzSAXOKN1JkUh4qQH5pCabnim5oAZJmoGqzJjH4VAwoAhIoxTyOaTFIbIyKQinkUmKQMZim4qTFNIoAZQRQRRQMSilxSUgE70lLSUgCilo7UDEFFApaAENBp1FADaMClpcUAMIppFS7aaVoAjxSYFSYpuKAG0U7FGKAG0UtLigBvalFLigCgAxRilxS0gG0oFLijFMBMUAUtGKBDhRQKKQwp3Wm0tADhSikFAoAdS0lL3oAf2opBThTAdSim5paQDs0uabiloAdRSUtMB4pRTRS0AOpwpopwpAPUcU4Ug6U9VNACoKmXimovFSbTTEL1qRBxTB1pynFK4ycAYoIpgNOzQAmKKXNJSAQ9KaacaaaAEpKWg0AJRRRQAUlLRQAw0lKetJQId0FNpT0pKYwzSUppKAAU4UgpwpAKKDS01jxTAQ000hPNITQAtANNooESZpC1MzSE0ABaoyaCaaaAAmkoNNpgLmkzSUUXAXNIaKbQAuaaTS0lAhM0lBpuaBi0ZozSUCHZpaZS5oAXNFJmjNAC5ozSUUALSUUUAFFOxRigBKKWloASko70lAC00mgmm0AJmjNNNFMBc0ZpM0ZpiFoptFIB1LmkzSZoAXNGabmkzTAdmkzSZpM0gHiikFKKYC4pQKSnCgBaDRmmk0ALmjNNJoBpCHZozTSaKbGLmjNNBoJoELmkpBS5oAXNJRmimMKdmm0oFIRt5oyKZminYB1JmjtTc0xjhSimgilHJ4oAz9ZsheW6Ejc6dvWsfxLoFnN4fScw7pUAIOeldJe6jZadEHvJVj9M96zP7ZXUID5NoJLdv4wf6UmBwj+GxNpss9tZTBgvBQ5P5VzVn4dnvNLub1Z1VrfOYs8mvatOjFtavJMwRWzyxwK5q5XRYLO6totRiDTnOQcVAjzUeHb5tDbVQqiEdieajsNMku7ZsA7wMjFejW9pBB4TuLBbqKXAJUjtmp/CunWU3hxohHh1B78g0ho8ohaQymKPfv77etTiPzZfLv5pNq9MnoauWxg03xG7XERby3IwvWt3UNF0290yW/iYxvjIx1/GgZhx2K29qrXOJrQch4z90+prJYoLgpGSU7ZNdBY6RqtxYNcadBKyAZburD0rHgjVLwFkMUiEZVuxoJuTzSW1ntU2+9iMhwc4PtWeLhmnyxzXcatpum63osUiwiyu4+SR91geh/Suc1bQ4tNtElikZnzglhwaRSZhXB3MTUkMxCCNWVW7M3Sr39pmHTTbLEnFZiW+9Nz8L6imijSubgxxoyujzBhVG7mmmlMkwXcfQVEqRrxg/iasr5DwMjLhuzZ6UAUvMbsaRI3nnCKwBPcnFPdFAO081Hg96AJp7V7Vgrspz/dOaWGJZoHlVvunGMVJp91BZ3XmXFsJ0xjB6j6Ve/dJYy3MRhPmEkwgcpSAzYXIcqqK7MMAGuhs7iGw0xw7I0zL8yN0IrK8P2ljfautvqE5iiP3SDj5u1WdW09beeaC3geTyn+aTdnAoAzbOD7VOIt5VmJIA6flUlxpssUDzBkZUODzWhpmnxRag8F9G4lKZiZD3qP+z9XuvNYQzJaBstJKMBRTJNzwtqX2F1s7+K5V5PlSRn/AHeO1dn4f1a7tr6404SIbGMMQyDkHHr3rg7bxLc2+mHTJIre4VVIVioPFOtPEI/sgafFbmK4b5fMJwp9zxVXJsYOojy9UuxI+5jK2GI5IzVF2PGPXFdvP4V02z8PnVLm7M9yAdyA9+xri2jqWjRGvrHhrUNGjtZ5ykkMvzK0Zzj1zUepXNvJNG1q+VK/MuPumt3w54kgWyk0vWn32ZTajvzsHpXL3UUEN9PFayiaIOfLkA+8KVgZbtYnluonS3kuERg0iRrklR1r3+wj0qbQIb3SbVGikx1OAp/2q89+FmnPp/jGBr2YQvLC5jjz9446Guq8X+HJrTV4zppa2t704ZYzhN49RmgRgeLbi50W4P2DTIlhlwPtUB6juMDpXL2baXFcCVLoq7oVkEnPzetdp4xt5ND8Mw21pazSS43vcdlPfj8a8vuLSRI0L3CbJF3Bl5xTAlvtOvoHUzyZ85vlXOciqstnF9jcK6xyxnOD3pwvLmTCXExmSI5Qj+GqmoMJnE688YagC+NVFzpUnnTZuAuAMdKrWUrSJ5TnKms9ZMoV9afGWX7pxSaA1d32h5G2DIOCRU9pqDoBbNyndapK6hGUNywweO9QoGjlDK2D60gNdPJmuIikLfu23EE44rf1LxBFaPFJZo4Q/K8XbNcdJNJbzRS7w/zfdHWrd4r3CGTcrZ7g0AP1I/aLgzJEELAFkXoBTNPgjnlmLuVUrhSO5qxpM0ah/OZDMUKqW6AVBplzFbXTxXCg2z/I2eo9xSYFo2EiRAb0HIG8ngVs2dlBZ65pxK7UkyrknKnIrLhG3UDaPLuiP3GY/eHatSexnmtbazjO0xvuE3cD0qQLcMR8Oa3PayQtLa3vzw+WTlcGvQNFu5bvTonnVg5zknvXG6BJd6nqNklzFm600MWLH5ZkPp7100/2mzvLm1tmKrgTxAHqO61rEDa8lPtX2gj95t2A+1PNCSLPbpPGPlcZx6UVoA0ijFOoApCY0LUqrxSAVIooAQx7lwRVA2qwElfWtQVHKgZaaYimk6gYqG7uVEYAPWkliZM1VmAaPBANaRJIWnh6F8n0pw3Kcjj09qoXCsmWXqKsxPPJpe4/6zOK1sSXBICv7ycM/fJqK3bN0FxWLHDLJtl6N3961dNldpWz24osBqTKwHyj61kTSMLjZnFbec8GoJdKD3CzZxjtWaKILUFiOKujgEU5YljGAMU1iFPNIaIJRgk1VL/NUtxKBmquc1QiYSetKGB6VUZyOlEMvzEGjlC5boJ4puaRjxSsMrS/eJqizFWP1q5LyaqyrkVoiGTQvggg1cW4x1NZCMQQKsh6bQkzUEoYUoeqCykEAmrCtkVm0WmWlOadVdXx3p4fPeosUPpKQmmk0ANdqbuzSO3aow2G5osBJSU6kpNANxSbakoxzSGiIimEVYKioyKQyEikxUpWm7aQEdGKdijFICOkpxpKAEoNJS0gEPWijvS0DF7UUnalHSgBaWkFLQAYpMU6lAoAjIPpSbalpMEmgCLFGOKl2UbaAIcUuKk2ik2igBm00bakxxRtoEM2mjFOxS7c0DI8UoFP2UFDQIbikxT8cUYoGMxRin44pNtACUUUUALS0lLSAcKUUlLTAdRTTS0gHilFNFOFADs0uabS4osAtOFNAp1MBwpRTRThSAUCnrTVHNSoKYhwFSpTAKeg6UgLKgYqULxTEGcVN0oGQFcGlApxFJSAcDxRmmZpQaAH5ozTc0uaAENJmlppoAKDSUpoAbS0lGaYBmim5ozQAtNzSZozSAUmkJpCwppagBc0mabmlpiJBTxUa0/NAx+aY5oJpjUXAQmmZoNJmgBc0Z9KbmjNAhc0wmgmmmgAJ5oNJRmmAUlFJ3pABoFJRQAuaYTSmkzQAlBNJSUwFJpmaDSYpAxxozSYpaYgooooAKKKKAFzRRiloAQU8CmgU8dKADFBFLQelADKKDTTQAE0maSmmgBSaSkNJmgAzSUZpM0wDvS5pM0ZoAM0ZpKKYhc0ZopKQBRRmjNMApaSigB4paYDS5oAUmlzTaKBDs0ZptFABml6UlBoAXNJmg0lMYtJRS0WAKKMUuKQriUtLgUYpoaEIpQKXFLigDVopM0ZosAuaaTSmmmhCDOKcjfOM1ETQDTA4f4p4lSzRTlt3T8Kh8JaVqc8G17r7MijIKnORXReI9Mj1N7dpGwQ3XGa1dPtobC28uJcDigDi/F+nXsCRouozXGfm2g4ParelW+m/wBmWqXttG8zfKVIrWuEV72csoPy9xUWl6fHJckuwbYMjHrUtAVdI8Pae+r3mIv3Y5QZ6GtK1uYrA3EZiEaryu08MKu6Vp72s80sjZMlZ/iK2Fxc2UDlgkhw204OKVhnL20FtrfiL+0LgRRrE5BjAwWrofFup2drp32aG0jkecYGFxjpXO6hpDaR4iSC0BdG+7k81A8d1a+LoU1BykQIKeZ0xilYCjo3im/0ATWVvBmJiSVYfd+lRaVaRa5q8v218Sv82CMAj2qe905r/wAVyxWtyYYmGQw74HpVq1uf7J1sJqUJeEfcnZRgH8KLBEp6lpc1pMIEvPMttwUAHlPrXWJ4VgvtGl0yaR5WRN8bN1VvauF1aBv7YuLmKQi2kbdG69GroH8XMpVrRpLe82hSzIWDAetIpGBH4dl1UGx8qC0uIPvSSdSawdUsbnSJvsk00b5GcxnIrubPTZWuXvNSeKVGy24N0NcHq5R9VnMTs6buM9qBFZMbT6ioizOePypd2OKfHE0mdgzgZNIaREoZR81PiiluZhFCm9z0GcUkSGW9jiJ4Y4Naclj/AGfqKNb3SnjPPagdipdadeWUUclxDsV+nOajt4pZZMRRl27heuKsXj3dxeJbXFyrBnGGJ4FXbTT5NO121El7FDE52mcN8oB9aAK+kWVnf6p9nvnZCwyCOu7sK3Lu01Dw9JIikMsqbQwXIYdhVzxj4Vs7DSYNTstRilcEAojDL+4IrMuNP1jS7C3urudZIWHCK2SBQSO8GTrD4usJJrjYoY/f5B46Gu91No9R1i4077YYk3YMI6MK4e3g0/VNRcORHJ5YZGBwQRRpsd9qOqz3Tali7g+VZCMk4poCpq+n29rqsiadvCxkqWYcZ9Kht2mnSW2l2gj5gyjoatDV7u+vpLS7YDLZ85R0PrTdNtby8u7uKwaNmVfnV+Cw9qbGZXmPv/ezuyng/NSzW+GXYwKMMqfWoZonimeKRcbTyKd5gG1XBK57dqgaI5EMbFHGD6GrFjpbX/m+VOInQAj3pkv2d2z8wqaykht5WMwPkupUlTQBbGp358i5N0RcW3yqyjBGK9W0TXdU8Z+GQt00fnW0gImAwZCPX0NeTNb26W6zwSb2zgx5zxVjT7m50/Zd2d80UpOGQNximB65ZaXHdlpJrm/mZchvOb5eO1eVXOnQubxIbj93DIw2sK6DTvH2tabatcCf7REWGYp06fQjpXLavrd1q095La2/lW8sm9to5Ge36UxEdpLEuhziP93cZ+Zm6MPQVlywf6OZfMTjtmmb/wB1sqHFAAjFXBxmrUfMqr+Jqsq81OMqQvr1oGWreNpZgijJNJdwS2+0uMBjgVeYCKSNogF+Xmq19MZFZW+9xjioAoPzKFq7bTrHI1ueAwwPaqODn3q4bKZDHKg8wMM5WgCxsW0uDCnzqwDfNzg1P/ZNzKkUsIWT7QchQeQfQ1CrzuglkhwVJGD/ABCuj8F3ktxLcW7HhAGU9Tj0pXAoXcEcEkcUoeTEeCF5MR/woj1S7tnjtJzmLOBL/eFTXUzjXX81Uik3FSc8OvaqUitcGS3hzJhs4A6CkFjQh1e60bULa4lYM0T7iAMEr3rt7rXbK8mtNVjudloyeW23l4z2JrjQJL4gvCTDGFSVtuSCfT/PaoodK+xC5sby2c+WciReQPTP+e9aRZEtD1axtJLa44vnnSQZVPY1pEYryzTvE99K+nyESOli4V5NvSPOMGvSl1SyuPnWdF9mOK0QE9OFGMgMCCp6EHNANMB4FSAUxafmkxjqYTTs0hpIZC6hhgis2W3IkJHStVhULKO4qyTLNvu6jmnJFsXbjirjRgdqaVFO4it5a4I2jBqvFGsMjD15q+VqtMu0lh1rRMTRNFyc1Y87bxmsGbUJkYqoA+lVhe3BkJ39afIK503mhulNkGVrDGoyRr1zTo795AcnFHs2HMWZly1QE7aFmLc5zTuG607WC4EBl4pgjxz6U8LtGKTnNAEu4kU0mkHAprGpSC4x8Gq0inmpmbFMLbgc1oiWVigB4NKNw+lKacOQBVskdF8y57irSZAqrFkOR2qyrcVDRoiTJxSCQikLjFRswz1qLDJ0cnvmn7sdaqxMPNxU8hyuaVgGSsGHHWm56GmbgTzRkFwFOKqwFhW4FPFRgYFLuxUNDRIBRiqxnKnGakjmDEjPelyMZKeaaRTxyKTFTYCPbRtqULShRSGQFfSo2U5q2UzUbJipAqlTTcVYZcjFR7DQMixRipStM280gGmkFPIpuKQwxRiiloABSikooAUGlzSCloAXPNKKQCnCkAuKQilpaYDcUmKfikIoAZilxTsUYpCGYpwFLThTGGKaRzUmKQ0AM20m2pKSkBHijFP20baAI9vtTSBmp8cVGy0AR0ooIooAWlpopRQA6lptLTAcDThTB1pwpAOBpRTadQA4UoptOFMBwFPWmAU9etAEqCnikSpAKQChc1KiHjiiJQasqo9KBD1WnGm9qU0hjSaYTStTTQAlIKDSUAPBpaaKXNABmkopKYC0GkoNIBKSlNNNACE0hNBNNJoACabu5pCaTNMB2aTNNooAcDThTRTgaQDs0u6m5puaAHk8U3NNyaTNNCH5pjGkJNITTQxM0bqSigQuaaTS5ppNABRSUUgA0lITSZpgLRTc0ZpABNJmlNNoELSUpNJTATFBFONIaAEpMUuKbQAtFJS0AFFFFADqKbTgaAHA0oNNzRmgB4NGabmjNACE03NBNNzQAGkJoJpKAEpKKQmqEBpKU02gAozS5ozSAKKSlFMAoozRmgBKKKKACilzSYoELilpBS07jCl60lFIAzS0YpcU7CEpcGnbaMYpAN2mjFOopjG4oxTsUYpgAFKRRSjrSsK4mKXFLS0x3ExRincUUAXc0oNJikoAdmimZpc0AwxTcU7NJmgViNwGGGGRS9sDpQRk0qrmgLFd7SGV9zLkmnAWmmo1w5CKBySaim1S1tLjyZm2tWBrd/C84mhYSqRswD/TvSHYvz+OtKt3wWk2/wB4LmqNx490y5nU2kTzsmMLt6/4Vx82oSWcstpd2gIlGVI9Kx5mbTN/lkDJyMHNJyA6vV/G0OoOmy1Nu8TbgxYZqjrXiL+3bGAfZjvhOdwIrndEeOfUybhVZAN3I6Gn6tqcUV4yWcQRf4hjAzS5gOy8J6ha3+6FbJRdxgkOxJLVDq17pmpz+TeTTwvG3GE4U1y+k6pqelXIu1QKjAg5HUVp6le6ffO0qxuGcAtLjgGle42dhbaJHHah7m7S+ij/ANWpXAX8qw9eVrW5W+sG/clf3i7ThcfzqHT/ABLFFBbxwW5knDeW4PT2NdRr/iGDT9CWC7WEXEwxtUfIKTBM8wvvEU15e7rfbECMHaOCfWq8UEcLyPdcGTlfpXSldPstShmuXtns5l646MazpL2xg1RxaWy3Ck5APagDN/scyxJKC2xhkmLkAVtSaNpcu2FS8UoTO9TwxqB9UtI5HmGba4xgIBxWNdXsV1FhpWVl6BOhpDRDenHym3KshwZMfeqluDAZ7VKJpHQRySEp71G8YRwA4YGgCRSjMjSsQgYZweo9K67WNK0CwXT7rT7w3O/BlhYkjHvXIArtVQgGDnjvWnLPpv2Vf9HPmhcZHFAy3fX9jLOquuxR90IcACtOa/i1W0tbPULqNbdMJGyH5s+9Ya2fkaKl46YffkDsRVa2jill3zP5aHqFFAmXNY0ubQL+3mjfzImOVYnnPp+tMg1aKO8N1PB8zDDbe9DLGQ/myzSwIpKDP3TVcXcE2mi3aMZH3ZB1FAGnZ3NvJdCXTbJ3cAl43HB9609FiRZBqtk228jkPmW55DL9Kj8KasqRHT3UCVF3RH1HvVTQtRWHULzgrcylgqNwpNMCtqiRXGqXEsM4ZXzIQRgqfSsyQYbFa93HpkIlivo3F8rHOw8EHpWJuzUjHLG0twsYUlT1wMnFXbiOLYsNqGeUHA44/GptB1OPStThuZELhjswvUZ71sXOm6fqPicwi8+ypP8AMzIcgHFAHPS6fLaOFu2MLEZUDnIqCO5MfBUN9asa/bfYdXltxem8VOBLnIxWarZpgdJca9FfRpb+SttkYeUdxVRG/s95LeznR0lGS23JFZKtukCjr6VsaZexafKZBDHP5ilSC33TTAypHYN1jOe8fQ1EDk0+XaZXdeNzE7fSmrlWyKQDsYGTVuxiFxdRAcrzu+lQhoyPnz+BxV60kgjmkW33FCvVjk0APmLPfm3thnFTaqmWiXHzKvNM0HR2mUX99KUswxU7Dzx6+lEKKHuHZZJYQ/3lGQR60mBmMrRsGIq7aSLIN8lw0W3qFHAqa9SHyllt4C1u4yGYVTtnCJKMffosBeaOS0nDSMHBBwwPDCr1hdtBPBfQwbVQETbe4rHjmd7aOJ2yI84/GrunyuunzbBlUYnOetS0BpSanaec8d7HmNmLRTqOpNVdCvfs2oyy/wAD5XcegoYLL5EUBGxiQwI6VYs7K409JkjTzI2IOFPBHPWkBs6fcztcNJaSeagKl4gOXArY13WZIL1rm2Tdb30QO3HKnvn8a5ywLWupC/05l3RctATjI7ivSdJ0TT9Rso7ma2TcSTg9Qev9asDnfCHkJcaqXBbT5EQSIBwCfWt250O2XS4I7VDMsbb4snnaT936VPp2nNpuqyT2/wDqpAUdO7Y6fl/WtYQeTKyRr+5JyP8AZ9qtIlooWcP2T5IAVhkb5oif9WavYpdjBjg4U9R60u2rFYAaeDUfOadigB+c0hbBpuaRhnmhDHk5qNhTlOBiggGgCEjioyKmI5ppFAWK5FMIqZhTCKpCMe7twGJ2GqDoEXcvWujkQMuDVE2yd1raEjOSMMyBupqxaEHcvfqKtzWUZBIAH0qgV8qTbg5rZSIJoWKy7TV5eKzU67quxuSozUtFJlncDS4FVhIM4zUqHP0rOw7jm4FQs1PlJ28GoA3PNCQwfmMkdarsxqyePpULxZBxxWiJIQ9SrzUf2WUruDAiiItu2kHIpisSop3HAqcKdtLGvyc0FhnFQxjGPFQhjvwalc1XkYK9NAywq5cMKuf8sj9KpwSDaM1ZZv3eal7jRmTymKTB4q0iPkOVIBFNG15V3gE571p7QVA7DpQxFYH1qOSQdBTrnMaHbVTfkZoSKuRSN8/WhJCG61DK53VGshDVrykXNqGYFeTVgOpGayYHJz7mraSEGspQLUi4rZbFTBM1TSVdwNXohuQN2NYyRaY5EGaV4MjIqRVwalA4qGMy3jIbpTClaMi54xUDJzSAq+X7UwxcdKurHk9KkNuDQBlFD6UzbWjLb7RkCqzRmkMr7aQipihppShgRY5pcU/FJikMbS0uKMUAJThSYpaEA4UtNzxTqQBR3opaAE70UUUxCU6kpc0DFzSnpSClpANopaO9AAKeBTQKcKQCEcVGw4qU0xhxTAgIpKcRSUAJS0lFMBRS0gpaQDhSiminCgBwpRTRSigB4pRTRThTAeKkTrUYqZKAJVqVajUVOq8UAPiyKtL90VBGuKnzSACaQ0U00gEamZpTTaAEop1GBQAUUUlABRRRQAUUUUAIelNNONMPWgBpNMJ4pT1ppNMBCaSg0lAC0ZpM0ZoAcDTs1GDSg0wHZozTc0ZqQFozTc0lUFx2aYTRmkpCFzSUZozQAUlFJTAKQ0tIaQCZ5pCaWm4psQUmafim0DCm0tJigBaWm0tABmm96WkxSEBoNLSGmAlFBoNABmjNJRQAtOBpopaAFNGKKAaAFzSUUnSgBpNJSk02mIKSlNIaYwxSAUoNBpANNFLSUCEooopgGaM0YpKQC0maMUtMBKXrRilxQAYpcUuKXFACYopcUYoEIRSgUuKdTsMTFAFONBphYUYpDSA0E0CuFFFFABRR1paBhijvSijFAhaKKWgGFLSYNOAoAulSKQg1YK00rSQysRScVOycdKiK4ouISkxQeKTNFxCUqnDA0ZpuaBs5jxTaM0ss+3K7civPWsZ3vUVrV9hYZKNkV6/qMK3NqUas86PHF5U8BIZcfKeR+VAzzzUdNM17DDItwYiAFyegpniHRI9DEIjnMqSZ+Vuo6f413HiPTxewW8SSmGYNuUgZrnZ/CNy1m8t3dvIVGRWbQHIQwqWGB8zdB61G0KGRg4eN179q6xdQ0vTrIIwEjr2K8n8az4dY0+aKa2WzeZpux61NgMGa9uJV8mWXeB04oiebyfLDfJ3FSPam3ucTxtGx6A+lQtceRMR5bMPbpVItmhp2ovpUzOsaOjjDhuvtiotSulvrbdLKz4+6D2FUpHy2aZsZxx0piKoUZBOTg9zVja5RpVzhRTGUKcVGZHU4B4qQLMMYmhLy8p79RUUojQDY4K+wqNnbbwcVeWCNdLEkgzI3NFgRQ25Ge1MJUEe1XRDC+nmTzdsqHhMdaZDcwbAJrZZB78ZoAljji3fabaGaWMff46UqWM90oljZFB980kWpzWqyx2Z8qGTqnXj60mnXsdozLdRCaJs5GaBlnUp71Z4g53qi8svIqiJgshYqD9RXRX3iazl0oWMFkVQ9W6muchgMqSPscoh5bHagC1b3UCOCxwjAhqjNvDMkstu4VI+gPelsDp4lc3RIXadpx3p9pdy2TyG2CyRv1VlyCKANSy05tTurd9Bjk86EBpg/AXPetjWdFg0ee0upZke+ZstEgOGXiub0XWrvQ9aWWJWHmOFkjIxuB7frXW/EjTJ5rm2v7VWaJogSAOR9aCDjNZvI77UWnjRkyACD7VQBxmlZNnD/AHqEV3P7tN2OvOMCgpG5D4fkFpaX8pD20hBYj+EV0MXgbU721mOkSWMtorblQy5d+M4BrJ0a8gtrUWt3IeJQ8anlJATyCfyr0N9a0HwjrttcCAl5Y/mhg5H1xQSeT60lsWUQwPBLHlZo36gjH/16ziAqgAV3/iiwtfEl/d6zp+oQRLIATBIm1yQP5mvPdxYc0FJl7SZLKC/338XmW5GG4yRUN1JB9tlNkrJCxyM1AOlKOKYwCk9OtLtJOB1qdHt2hKNGfNPR6c2VK5A49KQFUttOCKt2OxLeeTuDjFVpvnl3IpHHINWLVXYGNR8vfNAGkgj/ALOmjLMjN82xTwapW0z2/wA0ErAMMFW5GKmEMrTqSyF2IVctxk+tdY3w8+z2Ukt1eiOXP/LNtyAn19PrWfMByU9x53lxIuNowADxnvSoIIdNkMi/vy3TOCtPNn5Su6u0oDHcUX7oqFoWuIBMsZZASCR1H1q1IAWHzIjKXC45JPp2rVsUjvbG1hWb9652FemT2quAskVvKVG2MBPKAwSveoHEunXlvNbuHHmCWNBwQR2NSwOki01LC+a3uIGS4THzjn9Kfpdt5WpJby3GFncBJF6A+9Nv9fkvpW1FLfDPGFaNj0I/CorKSSezilt3AzIBKDwUIPWpAdc2F3FqV8UgKtCuXCDIb3zXS+F7jUtL0QX0jM1vdMV/ek5iI5GfY1P4ctpda1u8T7QUSWLbMpOSwPUiu2+wR6lpsun29xEiFNmGPIIxTAlii+0WNndRLt3Lv/E1aYHbzVuzhUaTaKo+5EFP1FRyxcVtcRT2im7an8ujYaoRBsNJg1PtpNlAWK5pD0p7rg0wigENzijNNJwaZ5q5xmqAeetAppcetN3jsaCR7DioWFS5yKYw4pDIjUTKM1KaYRVJgR+UrDBpjW0YGdoJHrU6HBxSSCtEyTnbqJ4pSR92pYLgGPaal1JcA1nRnArdamRM8n73OasRSHH3hVFjk03JBp8oXNcnMdRAVUjuGVAGJNSrcjIOalxHcs7SaaVNPjlRhnNSqVakMrIGDAHpVny4uuOaUqOwpsiZTqQRSKFOAKgYDdxUkaO4wTTGRo3O7pSCwx0Zl4qu6HHIrQjIZeKr3UZWqTE0VkDAYFXs/uRUUcZZOKsLE3lgUmCKUiknI61etpMxBWPzDrTPIO7GKl8gqMik2UOljEqEZxWaYjG2GFaQbA5qncENyKaFIoXCA/MBzUKId3IqzMdq1HH8+cDpWqZmSwZVvap2YhqZEQpAPrViTZsJqGNDIyqnLnArXtnBiGOlYSsXU1p2PyLjJIJqJrQtM0wakB4quGqZW4rnaNEP2gioXX5uKnB4prLzxU2AYFFOphPNAPNAx7oGSqbxc1aJprAEUAZ7pzxUTCrjpjNVnU5oAhxRTsUbaQDKKXbS4pDI8GlxT8UbaAGilp20UmKACiiigBaDQBSmkAlJRS0AKKKKKACnCm04UAGaXNJS5oASmk8U6mE0ARnrTacabQAnelpKUUAFLQKKAFFOFIKUUALTh0ptOFADh1p4plOWhASCpEqMVKlAEydRVhelQouAKmQHNAEy9qeTTQOKQmgB2aaTSZppNIAJoFNJpwoAdQaKSgBKDRSE0wA0ZpuaBSAeTSUlKaAEJqM9acaYaAGmm08jmkI4pgMPWkpaSgBpoooNAh2aM0yloGLmgmkpDTACaM0hpKBC5pc02loGGaTNFFKwBmikpcUCExR2paMUwG0GnEUhoASkp1GKQDcUmKfijFMBlJTiOaTFACUlOooENxSYp1IaAGmiig0CGmkpTSUDHUtJS4oAWilFBoAQ0ynUygANJRRTADQKWkxQAHijrRR0oATFGKWigQ0ikpxopgNxRS0uKEAlGKcBS0hCYpQKXFLTAKSnYNGKoYmKXFFGaQBRRmjNMAozSZooAKKSloEApRSAU4CgAApwFKFp4WgQgWl2ipAhpdhoGRbaXaKkCGnbKAIggp22pfLpdntQIuYpMUmaMioKGt0qF+tTHpUbcmmBAfSmVKwqJhiqExCabmg00mhCQMaSkJpuaCirdW4m1C3ZvuqeTWX4nmlZTFCxDBQDg9RW7kZyaq3NpDKzTSbfck0gODl0iwt4oDKUDuckSnkVSGt6XpksgtIAWIx5ijIrR1XS5dc1MW9m+63jGWkboKwb7SrKCeVYJXLoM/41kBlzXU2p3wkmbGTgnritlols7JbdwskTHHmEYNQJYT6VapfyJsLHFZl1qc964R9oTsoFMdxl6IUujHAxKjqTRARvVdwC45qq/Ei/SlBQocplgeooGSOyYZeozkGqpODTyjBA/Y1CSSaQEjFSuFzTjLM8XllyVHTNRgU8Ng4FNDEJJGDQkUjfdXIpySRRyDzELL3xUzOu7MIOw9AaQiJrd0GTj8KfFGisskmGQ8EelaEEUbQ7izDjlTVbaiu2F70DILoRpJiJgR6DtSw38sFvLbrtKSdQR0NXJpA1v5awKW7OBzUEHkJBJHPbbnY8P3WgCiK0lnjjsYSs0e5W3Mo4OBVRkD8RLz6VPBpklxC7PF904xnn8qANvxLq+j6zp9rd2gaO+iIEgK4yOx6+1dfH4wsJ/AObmeOW98swtCD8ynpzXmj29rGyE5IJwRmrF7o7WKx3G4G2mXKMOx9KBNGdM/mTE1qeHNNbU76WH7QYIypBl2hgD2yKypBg5Favh1VnuJ4ftklrIy5Up0P1oA6e3E2gyyabeRxzxJyssicE+lQa1ZSaclveuy/Zh8yheqk9ADVLTNQlu71tG1STzrdsqrMPmB9jVK8ZrR5bWaczxQvgQschh2zQIovqQud8dw+9GOVbuD9arSCMfcH196mu5bOdB5NsYWHfOc1H9qPleXsQqOnFAIQAbfemFcqexHUU1iScDOT0ApMhmBwQ3QjvTKHr96rStvTABDDkMap7vmxgj61atU86YIxIGM0AN8x5JAXOSBiniR0zsOM9aVEQ+akgPXAINTQ2kr2zNEM7Oo9akDa8G+G08SXN7HLPLCIY9wdOmT0BrovB7yx3Or6PIr3DhGWPJyG29qrfDK11CSbUrq3VGt1QJcRnqeD0/Wnm2k0+/t7zTY9l3bnMm48SCjlJJbizv/C+hbzHCzXsuI9y/MhPY/0qi3huC3u7qxeSYTSQCaMbu/8AEPzruZ44/Fv2e5vYvs0dkd7Jn7xPOc1594rur661mS/J2RfciZe6imo3GmNtdFtnWGNtRG6QfdxgqfTrxUMmj29yxVJzFMQfLJbIyOx9K9O8N6Jp1zZaHqptQryNloyPlJxjNbqeENNiu9QgkgEkMo82FSACh7gGk4DPHo9EurTS1umKT27DO9D0PcEUWtqbGW2BTzYWxOzIeQv0r0nw/YadP4ebT5jmG4m8tyP+WbhuAT2yMVsHwhpyzvDDEVVI8Kw6qPY0uUDmPA2izKk2t20D3OHKxlGxhfRhWXd+KNd0DWJESzeLEpkMbruOD29q66HQ5/C32LUdKMhSR9l1E7ZU56H25rcuNPOoXTmWICWXazMONgHvTsSLoGqrqfkTRKUS4jIdT1Rx1H61syw84xVfSdHi069uSh+SQiQJjhWxgn8a0pACxNUMypI8HpUe2r8qA5qqUIancCIx+1RFTV0JkU0xDuKEBnScVWZ8VpyQKT0qu9krgjkfSqRJntIGGKryAbsirTacYmLKzH2NRyxbEBPerQik82w4zSLMW6GoLj/WEd6jRyta2JNaOTIwTTzWYkrAg5qzHPk4NTKJSZKw5phGKkZgRUbHipsFwAyalkX93kdRVdWw4zVndlKYNGDqPmlxtGaqyWpTbtXGRk/WtyaIMSaoTxv2FawkQ0ZPRqXAp7xtvORimEEcEYrdNGdhrCmYNTqm4470v2aQsML1NF0FhihlOau28hzk0jQFV+YU1RtFQ2UkX0OcGpOCOaqJKABnpVsEGHeKhlIFHz8dKc6bhVdZzuxV1SGUYqbDKgQKeBTgu/hhTypBxinIvNK4xqw46CpVQ45qUDAoFQ5DsRheeacVyMdqdSilcdjOmicKQKotFN8xLDHpW66Bhg1WNuN3IrSMiJIwtrO+0cmra2rQxdOW5NaQtY1bcFAp0iBom46Cr5ybGV5TN0FLsOMGnpJhyMVIACvHWquFiEKFXjtU9rKKZ5TEU+GAp0FKWw0aoHFSKcCoYSdoz1qYGudmiJAaUtTAaQmoY0NJJNIDQetJ3oGPzSE02l6ikIjY1CwzVgrTSgoGVCtJirBj56U3y/akBBilxT8YoxQMjApccU/bS7eKQEeKTFS7TRs9qAIiKAKl2e1BT2oAjxxSVLtpNnFAEWKSnkUwigAopKWgApwptOFIBc0lJmjNABTGPNPpGUEUgIqQipNtIaYDKKd1oNMBKKKKEAtLRilApAOApaBSgUAKKeopBTlHFMB61MgyajVanjXGKQE6rwKmRajXpU6UgFxTDUmajc80ANJphNKaaaYC5py00UoNIB5NNzSE0maAFJpDSGkJoAUdKKSlFADhQaSimA003FPIpMUANA4pDTu1JigCIikxUhFJimIZim4qQimkUAMop9GKBkdFOxRigQ2inYoxSAZil70EUGgBppKWkpgLS0lKKQC4paKKYCYpDTs0lIQmKKXNNpjFpCaKQ0gA02gmkJpgKTTSaCaYTQIdmikozQAtJS0YoAbSYp+KTFADR1pwpBTh1pgPFIRS0GkBH3pCKcabTAb3o570tHSgBBQaUUGmFxKSikpALRRRTEBoApTQKADikNLRQAYpRSYpw602hCgUoFKBTgKQCYpCKfikxTGRmm1MVFMK0wGUU7bRigBtLilwaXBNAhoFOC0oWnhaBDApqQKaeENSKlAEYU1MicU9UqRE5poBoj9qf5ee1TBPapkgBGTQBS8r2pdmD0rQ8pQOBUTxjnikMqYFGBU3l0vl+1AEWKaadRioGMJpuakKik2imIjbpULdasFMimFPammMqtTCDVkx+1MZPamIgppFSlOelMIoGiExoeufzqG6gjaxnUA5KetWSKbQBn6FBHb2HyDG7Oa5LSNM87W79bhcGNsYrt40W3DBeFJzj0qNYlNw0wUAsOcd6hoDlNfthdR2unHj5s4rEuPBN9b3u6CAvEvUk9BXoFzZo9xHMiYlQ53VpktIdwHWlYZ4/caQk87M8Z+VeAg549qp3NgtvpLTLGd27aB3H1r1TWo9OtY2uGlhiuFBOc/ePHFefy3U+uTra2tqNq5ZincUDORduMVHXWxRaJCztJGZJkOAFzmsfUYn1W/muLSFgnfdwcikBmxxl/u9qnaGOHbuYMWGQR2ptjHNNM0UKb2KnvitfStMFrFcTXyIygY2ZyaBmOwANWZ7UxWqTxTI+Rkp0IPpVeZ0klLxcIegpm7FAD2mc4G847igSdqiIIGaAe9AF1ZJY1+RsD0pxnDxkMeTVeOQnhvSo24OKAJFJU5Q4bsalEkrzxKLhi5ODgYxVZXwa0vD1zbWuuRzXbbYSCCcd+1AEU1tGsp89hKfUUt5c+darbRs6qDny25Fb99o8UdxNe+Wj2cxJWTPKH2rlJmHnkocr2NADSh2gGtbQLm2sZpHlUiUr8rjoKy92RTd9AE8UrSag9xdP8xYnOM81YuZLKb9+FYyqRuB6OPSqI560hXNAixqJtGlR7ThWGWT+6aWO8tIoWQ2oZmGC1UnGGxSoOKAJbWTyLhJcZ2nOKbcS+ddSSYxuOcelSw2skkDzoNyIeT7VBJsDZXvQAKSalgcpOrDrnFQocVJECX3dlNAyyGBlxn5mbJHpW94cs5Ly9uIIrlIpQu5VYffxzgVzhbEjkDnOQa7OytbK58GwX0UW3Vre4+WUHG5c8g/pSA6bwadQ0fxAZpIH+zXS+XcdgD2b+fNdXJ4Z/tS/j3rGkELHBBySM5pfD1w17qMSS2ptZRD+8QkEEY4Irq4ojEOlAGcugQG5uUlwbadQpUewryrVtAjddVlgkdbe0YJGDyCc9K9f1E3LWhS2bazcFu4FU7bRrSPSJdOdMrJ98nqTWiAoeEy0vhi0tpkCzWDCP5enfBH4V0t1dra2c14yhvLGcd6wPDUEtrDfQTMSUnKjPoK2LiAXUaqRlQckHoaSA8p1nRb7TtIg1m2aZYppC06qcBWDfKcd817DbMtxZWtyvJlhRifcgGsvW9P+0+HJbFG2pw2cZ6HNatm3+gW/wD1zX+VKTAlKq8bRyKGRhgqe9EaiKJY1ztUYXPXFJmnLzUibHrwc01pDnFOHApjcmgY5uRULLmngEil28UwGKuBSMo20Hg0A5FAivIDzTY8g81M65qPaadxCzKrp0rNuLIyN1xj2rUCk0FKLjOYn0a4kkLqykfWqlxps8Kg4zXWtGRTQiupDCteck5mKwm8oEpUE6NCMkYrqJIljHHSsi5hWZ2U1XPcVjLjuiTgmrKtk4qtJaFJOBT9+zGetWItPGAMigHjFLHIrKKftHaoY2RkZphQelT7RRtGKVwsVXt4pBgqPrVc2EQyu0kdqv4AowKrmYWMp7JY2IUYPrULNIsmxjkY71suoYVVmhUgcVfOyeUriNpUy3WmPEEGDxVpcgYwRSMAwwRRzCsUFj3HFTKCqlD0qYRgHildQTnFHMFissZLZxVqIlSBTVHOKlC0XGTZDClCAHOOtQb8dKnjfI5qGUh+KSnZBpKhjG0o4pKWkMWmlcmnUlMBMU4KMYoxThS5mIpXFuokyqYz71GsYU1oOu4Uzyl9K05ybFJ1IYcVPGoKipdi9xTgoHShyuOwqjCjFPpBS1BSDNBNBNNNSCFpveloNAxCaM000maQD80oFNzTloANtLtGKdSYoAgeLk8VFsq9sJppi9qTArLEfSl8v2q6kXHSlaA9cUgKWz2o2e1WTCfSlEftQMqFDRtq0ye1M8v2oArleaYwqyUNNMftQBVK0wirZjFRMvNICAijFSlOOlN20ARig8U7GKa1ACZpKKUCgAp1GKWgBh6VGalbpUZFACUdqXFGKYCY4pcUuKXFACUoFO204LSAaBTwKULUgT2oAaq81Kq8dKckfPSp0j9qYEaJz0qwq4FKseO1PC0gBRUq9KjA6VIKAA0w040w0AMNJQaSkA6jOKTNBpgGaM03NGaQDs0lJmjNADgKeBxSCnimA0igClNJmgBKaadSGgQ2ilNJjmgYlGKdilxTAZto2Cn4oxQBCVx2pMVPtzQUpAQYpNpqbZSbaBEWKSn03HNADSKbUlNIpgMxSYqTbSbaQDDQKkCUFPamA2ijBoFACUU7FGKAG0007FIRQA3NNJp5FNIoENppNPxTCKBjaMU7FGKYhlLS4NJQA8U/bxTVHFSAcUARkUmKeRTSKQDO9LQRQKYD8UdqO1FIBp6U2nUhpiG4pKWkoGJRS4oxTENop1NoAKXFGKKAF7UYpQOaXFADcUuKdtNOC+1MBgFPC04JTwtCAbtpQpqTYaeF4oSAh2Gl24qUjFNIzTAjIphFSGmEc0CGbaNpqQClCEmgRGFNOC+tSiM04RmgCIJUix89KkEftUipTGRiPnOKkCU/bUiJmkIjCn0qWNGJ6YqVVAp4FADhgCnbhTKQigCTdTTzTc0oPNA7hto2in7TQVIoC5RpajR1K43AkdRTxUALijbTlUnoM0/YR1IH40DIttOEdSBO/anBaAKrw96hMVaZTIqF4sUXAzmj5qFo+avsozTDED2pp2AzytMZcKT6Cr5gB7UwwKAcjIx0p3A5m9nhtrZrq9kKoOir1Y1m6Z4qm1C4McNji2U4MkhxgetHiexudRjijkkVFeT5ETrj1rJ8RQroekw6bb/62QZkPc5pMDVvfH+lWl00At5ZtvVkPGayYtS13VrpJLS9a3t5jhRkcDvxVOTwjEnh+C5uQ4mlIHyjPB710lnpv2G+06Jx+7VR83elzAYWpafptlevHema5uGXq7ZBP07VteAdHihgm1AjLS5RfYc1DqlgLzXbm4VN3kgZT1Faek6nZ6dBJDAxeH/WdMCMntSuM5q/0628OeLPPnd/s0vzxnGcE9v1rnr2JZdWu7mCXy7bJIbNaPiPVpPEWpoJV+zxRr8m8cEj3rknldWIDnHYigY+1ke1u3ljbHGAadcvcTP5slyXD+hqG3iuLlysa7sdaZ5Uu9lC8jrQMdsYIScACmc9qbkg4bt2p4IxSARQTQeDinxttcGmOAHJAwKQwyamiHmKc8kVEoBOCce9OGVfgjmnYQSLt5FSRGJDHNIokCnJjPek3KD83TvUTLtOB93t9KYG7f8AiJ9Rs1tRbrDEuMbTyBWRsVmwWx71CpYnykQtI3QCtG80p7G1imklXdJ/yzJ+YUAZ5BDEdqTNW7S2a5dh0UDJNLb2P2m78kHHBOaQEMBi2SeYG3fwEVMY18pXU8N29KZNGIpCg6Dio1bDUgJpLAmIzhs4/hxzUccAa383oO9WoL0rjG047MOKZfvPE6QOiJEwyFUYxTEH21v7P+zIxDdyR1FUSKsTiFHHlS7wV9MYqLG8jCgHFADMHtU0JzkVHyvGCDU6gxnPqKBjiOB9cV6Z4Dh2G70K7EM1vLF50Kt1JxzgfgK81iRptwGMKMkE4q3p+q31hqVpfxSyM9qcjb129xQB6l4R8Rzwa9Z6fqIKbHaLzX6KDnANeqyDaxFeHafd2vinx2si28ps7o7SWGCHx1/MV65oum6lYW5jvNQeccgK4ycdjmgC8emKjPFTEU3bTJKZRVkaVRtZvvY71IspAp0keRVN2dDjGecVSYF3zA4ZCeGGDU8R2xqg6KMCqUMeDuByPeraUpK40ywlSAkVGpxUwwakBc03+On4xTT1pDJBQaYGpS3FAEUg5poFK5yD601Qcc0APApKUUuKAGUhqTApjCgLDDg1A3ysCKlLBWxTW2sDVCIphujNZEo8uU5rVZuMVj6nFI5V0bGDk1pEkinG5dycn0rPlb5ueDV6NWCdaq3cYX5yK0TEwhPQ5q4HB6GspJl34BqzHMM9atom5fHNLiolkG3NOEnvWViriEUhGBSk0jHigBuajbrRuIOaj3Zfk1SQEvlgjmo2iwanQZUU4rmk2BUMRJ9qQxHHNW9oo2gilzAUApB5pwarLIO9MaAYyKtAVx97mpl6U0fN8jLyOhpu0o554pkkyyYbBqQHNQHbjP8AFQjkNg1FikywBQBSA06paKuIaQUppKQC5pc02ikMeDRimg07NMTQ0ilopBSuFh1LTaWgBaXFJSikDDAo204UhoGREYNN2nNObrSiqJG4pwpCcU0MM0WGTjpS4OaRTxTh1pAOUVIFzQq8VIg5qWMdHH8w4qfy/aljXJxVtIMqO9SBReDKHiqrJtOK3zCFjOR2rMlh+Y8UAUioxUbDmrLJtyKj8slqYyDYabtq0Y/amGM0gKrrioiuTV0x5HNQvFg0AVWGKjNWHQ4NQlaAImphGalK03ZQBFinAVIEp2ygCMCjFSbTTSpoAjI5o2in4oxxQBHtpQtOApwWmAzZS7BUu00BTmkBHtpwWpAlOCe1AEYU1YijyRSInPSrUSYIoAVIsHpUyoBS4FOoAQqKaRUhpppANApc0lJQApqNqkNMbpTAjooxS4oAbRmnkCmGkAhppNOIpppgGacvWmYp6jmgCVacaRaDQA0mkoNNNAh2aCaSlAJoGNxTgpp6oMU/aaYiPaaXFP2mjYaBkWKMU/FJigQAc0/FNA5qTHFAyMjioyKkam0CI9ntSEVMBTWWgCAigCpCtIFoATbRsqYJThH7UgIdtBWp9ntSFPamIqlKZsOelXPLppQUDIAntQU9qsbaaVoAg8umlQKnIppAoAg25FNK4qYjFRkUAREU0ipcUhFMRFilxT9tG2gCMijFSbaNtACqOKdiilxQAwimEVMRTSKBEOKMVIRTcUxjaXFO20uKBEZFNIqQimEUhiUmKdjml20wuNxSEU/aaaRQIYaSn4pQooAbil21IFFOC5qgIwpp4QmphGKcEoAhCU4Kal2UoFKwEYWnBcU8CnBSaaQDcUYqTYaXZQIiI4puKlK4ppFAEW2jZUmKXFAEYWnhacBSgUxAFpwXJpAKkUd6LAKsftUgQUm6lDGgB2wAUoxTc0ooAkGO9OyO1RZNAzngGgZYRM1J5PuKiQsCKnGTSAjZMUzgHmrQQHr0rPurhY5NqJuHcnilcC6pBFJKNqbqypdesrKD94skj/3VFc3qHjC7nlVYITDGM/ITzRzIBkOrQpgQywSXEjdPMGR9K6eCKfygXWvn4zZ5EbxHsQeRXRaB4l8RwTCGyujPH/cnbNSM9MvLzUbu8NrYTJGo6jGCDUkHhj5995fTyy98SYrkE8XWOoXHk6pBLZ3wJBmt84P1rWsvE8FtcC0uZbh8/cdm5xQB0jWd7YIWsZRMneKU8/gam0/VbbUXMOTDcqcNDIMEH696q2+rtKT9mQz47L1pmoQ2+pBHltLi3uEOVmQc596QG8QF+9QF3CuS1vX9S0vTQtxYfakY485R0rqdIuodS0q3ubdw+5fmwc4PpQBFPCQcgVWBxW20KuuCKz7izKnKCgCkWUU1iCCBUptn9KiMLjqKYFKOzt438zywX9TXP32kf2hePPJxk4AxnArqWUioDHTAx9Us/wDiUJHGOIyD+VNkha8ihdBwACDWncRNJbyIB1GKjjRobZARggYpNAcxqd0LDTbudjt3NtzXKXGo2kssdvBNMEK/N5Y7+9dxqmj/ANoW/kH7pbJrEns7TRr4rFD+9Mfyog5JpWGYXixI7iGw+zgHII4GM9K5W5tPJyJGCMOx71r3eqS3N4zaiTD5TYQAYGaytWu/tty0i9D0oGV7VpgziB9pI596ZlhySQ3rmrhS2a3jaAlJx97PeqssbxPtcYNSMj60opSOOlGDTAUcEH0pZPmJI6GozTh0x2oAaaUA1Mtu+8BkIU+tPK5bYrAKD065pgQybcLjr0IoIKnB9M1dtVso7kPeRl48YwO1Q3n2drpvspbyuwagBdMe2h1OKa6LhEOQV7Gr+v6iuqXquhRkT7rKuCfrWR5Mp5C5FABXrQBo28jRWkhXuMUljM1tfQv6nB+lSWsayaBPcbstG208darsvnW0Dhgr9ee+KkCzNHGLx2kXcmckVmTmNrhjECE7AjmrxlE9sYkObo9F7n6VR6/e+/3B60ARvkRGt3Vv7Lk8PadLbT7rwgLIvUr6isUrkU6L92Q20H6imAxF6E1Zh/18eR8u7mkd4i3AI49KsWCGRvNCnyh1Y8YoALmNjNkLkdzUOK1P3Ul0AkiuuQGA9PWpriGygW5VoYyCQY2AyQakCva2dpLYPI29JVxwOBk0SW97ozYlTY00eUbqDVzQ1uZbiIWaec7Ha6EcMPSr/iRf7QuBEqvG9sgDxuOVNNMDr/h34Wm07xBY6hcSo/mxMWRRwjHocV66/UfSuJ8G2tzppgtGPnIbZJYnccqO61163AkAPTimArCo8VKRTSKAIWAqJ4g1TNTO1NCuVA/lsecCpUuOjZ4pJYgQTUP3PvdKoRpxyBhkVOHxWPEzJ8ynrU6zuTgmpaGawcGkY5qoknFThs0hi96Q9aTNIDSEOpwqMml3UxklLUW7BpWlCLknvzSAfQRxRkBdxPFKrBhwc0AVpY8DIqu2avuuRVVk+c07gQbciqsyKyEEd60PLb0qGaA+WeKpMTRjzDauR2qhNMrKVkFa7Q7hjFV5dN8xcYx71cZEtHOyJsbcp4NJHJJvHHHrV5tOnLgPFwPerIgVR80YBro50Z2KrzkRjmkjufm5NWruyKw5RQRnsahex+QMvFK6GTJKp71JnNU/KZIdxOKZHOVGCaXKFy057VWkzvBBxUykP061HIO/enYLlyI4QVJmqUTnI5q0G5rOSGP60YpyAGnlRUlFWUkDNMWYMoFTTL8hrMlDKSVrWImXHG35sjPtTHIMZI6jmqJuSODT4rg59VxzVcpPMENwJGIPY1b25OR0rNeIxyiRCChGeKuW8v7sBqOUVy0p4p27mot3HFJu5rNopE+aKjDGnA1DRSY7FJS0uKmxQlApcUYoELRRRQMSlpKWgBRSg0ClA4pCFFFFLTAibrSKcU9hUTNtPNNIQr0xFJbpT+DirESccVQEWccYqZBmn+Rk5qZIQAOKljAI2OlPVean2gpQEGahlE0QGRV+HG0VRjq5CeBUgTSD5KqyRZXOKtlgVphHrSAzXt8nOKjMO3tWmQtV5Co4xTAqGPjpUZjx2qyKY44oApMuDUbDip3PNV3PNAETIKgdOasE0wigCoUNJsNWCtNxzTAhKmjaam20hFAEWDQRT8GgrxQBFikxUhU0gU0ANCinhfSnhKcENADNtKFNShDShaQDApp4SnquadtwaAFSPjpUoXbSKMU+i4BmlBpKTOKQCk00mgmmk0wFNANNJoFMB/U0EUoFL0oAjxSZxTiaYRzQAhNNNLSYpAJTT1p2KCKYMYOtOHWilFAMlHSg0yloAU0ynmm0AIKlUcVGOtSr60hEqrxUqx5FMRhUoboBTAPK9qjZOat8bahfrQMgKe1NKVMTTSaBEe3FFL1oxQBEw5puKlxRtoAYOlLjNO2mpEjzQBAUPpQEFXGh4zimeX7UgGInPSnlaljTnkU9k46UAVitNKipylBjpiK2KTbVny/ak8v2oGVipppU1aMftTCgoAqlTTCKsMKiZTQBCajxUxFNIoEQkUmKftNGKYDcUYp+KMUAR4pcU7FLimIaBTsUYoIoGNpDTjTTSENIoAp1FMY3FLinYpcUCGEDFRkVKRTdtADQKXFOCnFOC+tUBGRxTdtT7RSbPakBBs9qcEqbZ7UoT2piIghqRUOKmVOOlKEoGNC8U8JTgppwFAhvl+1NMfPSp8UYoAhEfNSbMdqdilJxQBGaQA1IoDGpNijtQBWZTTMVaYDPSo9g9KAIcUYqUqBTDxTEG0UnANIXqFmO6rSFcnyM07eKreZxSbzTsK5b3Uu6q6yEClMlKwXJw9OD881V8wDoacrljRYLmhEYywyathIz3A+tZBcoM4pst+yxsTwAKljTNYy2yybfMyw6gDOKtxxo4+U156/iRopNscZIHYitG38QTy8Bin1FZcxdztntwsZcsAB1ya4rxJrNvZyGONt2ep6elU9Q8QXFwzQJcmTpkIKwL1YWG+UbpfSspSC5FNq8szlg1Rm9g6yDL9+ay3mZpDkADsAKcqq/JHNZgYWmRf2rcGCdzsAyuOxp2mWKtqskJldYo+jocGtPVdB/saW3awk3zMPmTqSa17PQUOmJI1vtuW5kZmIH1rcox/D+kx6hq8z3BMkKD7zdQfWsvUrW8u9blt7VprrYcJtPQV266adPiNvYSo7XAyQx6Gtbw14eh0K2lurrC3BGSRzTTJOWsJdX8KrDcXSuIpCAeeRz6Cu5uvG1tbaFJe2p82VVOEcYxXET67ZXmtzXGpuy26nbGSOD7Yps+uaJdP5NrAHPbnbmmAad8RriLzxqkDXJc/Kq4wo9MVXTxYbaaS70u+lsJnOTCqfK1UNfvNOjMTWhVZkHIA6Gsq2v7eS9WfV4Hmth1K0gPY/DXiHxFqOmx3TXmnXCN/z0wp/Gu4imWSOP7VGIncZG0/Ln0r54u5dEuGB0dbuHvIi5IIrrPBfjewWI6Zc3BWVeI2u3O0j2znFAHrk1vFGu+QBV9zWXJqGknpPu/wB0ZrktV1+K0QTXsjJbv/zyk8yNx+HStXSPG/hcpDG0lqok/i6lSfUUAaTm3mGYGLL64IqtJF82K6l7VLuBJLa4jCsMq8ZBBFc3qcGs2lrLM9xZLFGMmRlP8qYHLa5PJb3nl6bdf6SeXjPK44rPbxhFYzrDrEKqW/5bQkEZ965rW/E1/OJYZRbhgceZD0Nc5eabqAgjvJVzbt/HnpQB6w/ibw+VDC+XJ6YUk1yus+IoLG9M4CXTyL8pU9P0rkHuGgt0hATkZ3L1qiTukVR1Y4FSMTUtQ/tG9e5aPy2fqoHGaqRpucA1rvD9jUx3cCu0nzK4ORiqf2dY5F2tnPvTTGM8tSpI6qabcSNOF+Ucd/WpySIihZRjmqwL+X14zUjBQNpHekUryGpy7RyVbjqandY5FURL0Hr1pgVPLZj8ozRJDJCAXGM9K0dLsWuZC75CJnIB6026sC0iTJL+6kYKN38NADLf7HJab7iaQSA4Cg9aqvsVsoTg9DVnU9POmyKplD7hnIGKpAF15PFMCaYgiInjK1ERhqdK2YlX+7UQJqQJOvG4j6UjEnOeTTgueR1oZdpywxTuBNaT3CWE9mqqYn+Y556Va0yKfVHgtYiqbSTnbwB71a0nSZ71okgnVPPU5DLx+degW39mabpcdvbNHJfwZXhcHPvQySj4ahsNEiupr6GCa6hbdG8aZcg+1cV4oNpLqX221WVBcZd45Bjac9q9R/sez0/RHury1W5lfLMcYIz2FcRqHhHWdSsH1BpImjhVtsIOSVHpSGmc7YxRwSq2pQOlvKv7t/U1RJUOQp3KDwcdR61oanq/2/TrO2MWxoOp9wMVBYmFL23knGYg3PNMZZ0hGubh4yhZAvdeB+NWVt73UrP7Na4laNuY4xyB6mtiDUbW0e4u4oxOxOHz8oRfbioLXUf7N1f7dASLS74ORgAf5NAFXTNLlttMTU3RHieQwFf40anz2Evkyh1Bmf5kweCK6KwmhfQpY8KVF8JRkZwOai1W1sotZN1Zxy3KyEFYYkJRieuDQB1GheCoNK8Gtq8UpkvriEMg7ZJ4A54rL0wzDWb+71a3QJMn2aZMZKtjrxWl4Z1C71bRTpMsEz6dBMu+UcGFepU/Su/0zwrpmnzlrWWWVJBkh8EZoAsafp0VtLDdp/z7LGMDjoORVTUIBbXscyn91K3QcBT6frVDQvECS6neaZINnlSsIeMZA6itrUIPt9iFjbADhgcdxQBbDiRFdejDNRsaWPmFe3FNIpAMY03NK3WmZpoTA1BJEHGM4qbNBppjI0Ajjwe1V0m3XGB0qa5Vmt3Ve4rLsoZoMrIeetUiTZWXawBNWFk4rJd2PGcH1q3A58sZOTSaGXg9KDUANPDetSBNmimUuaQyOV8rtH3j0rFGrSxXcVkyCS5dd208VttHvwQcEdGHUVx2q6HardPf311IJYoy0cm7B45BB/GgDodW1a0sURrncemABk4NSWepx6hpM1xa70TBC71wa87XxRbX/kPqLTmWNdqzRjqvbPrWU2qTyXoRJz8r7FlDkBl7bhU8xfKezWt5BJpqXTTqImGQTxmrIjyqt6jNcHDrkd7oLabb2RHkoB5mPlI9RXVabrP2mC3DxOmUHLdCe+P8OKCDYWNcdKR4lIxilhlWZSVzgU9SHGVIIqgKRtOelILT2q8VoC0AZEtp1FZtxaYyMV0Eq8k1QmXLGtEBh3HmRQ7UGfaooGLx89fStOaEMCcVmopWZgRVpkWI7mFjEcDJrOkiZVJxW2wytVnQEkYrSMrEtGTHMQcGraYcZqvf20kS+ci5Udaggu/lArTcjY0FXBPGKnTmq0c2QOcirCHAyDxUtFJlhWCjmgy+hqs0lQyTBRwanlKuW5JFKHmqMveoTdGo5bgstXGNiXIjnCYJzzUMbshO09RikZi3Wm4rYzJ45W27c8VbhK7CD+FZ6tircU6pxgE+9S0NMuRuD8tDHDGqyNtcsemamL7s1m0UmTKc4qQGqIdlfGeKtRtuXNS4lJkmalXpUJzT1PFQ0NElFApcVBSDFFFITSsMKUU0U7vTJHUtAoNKxQopD1oHWg80yRrE1E8Zk6damI4oTGaoRGkTBhkVdiXFRgU8OFIFAFpFyamCDFVI5SD0zVtJAwz0rNlDsYFGKaWpw6ZpFD1OKsRuABVWnqcVLQF0OKC4xVUMaUtxSAc8o6ZqF33GmP1qPcRQBOKQ4IqLf70B+aYEUqc8VXdParrYNQOtICtspu0VORioyKYFcrzSbamIphFIBgXnmkKCpMUUwIStJtNT7RSbaAINtLtqbZRtoAjAp4FAFPAoATFJin0lACjinCmiigB4p26o80hNAEmaTNMBNKTQApNJmmk0ZpgLjNSKMAUxQSRUwFAC4pKdjimmgBhppzTsZpcUARYpDUmKaRTENpuKfikpAJikA5p1GKAFzRRgmgCkMMUuKcFFKRTAjFPBphpwHFAiQMacGNRAU8DmgZOrMT1qYJkUyKM8GpwcUCIzFTCtWGPy1HigCuRg001NIB2qLFADdppQtPApcUAIFqxAoyKYqnFTRjFSMtCMY6VA8XzE1YRuOaJGAHAoArKntTvLzQp+arKLnFMRW8ql8mrvle1NMXtTAomP2ppjq20ZHaoGHNICuy4qJwassKgcUwK5FRtUzDmmMOKYiuwqMipiDTSvFAEJFJUhBpuKAEApQKUCngUARFaUCpdgpCtMRFSGnkc03FAxpFNqTFNIoENxRilooAKWkpaADFAFKBTgM0AGB6UBRTsUoFMBoSnCOpFWn7DQBAV9KVYyTU4izU6QcChsCrsxThHV0W/tS+RjtSuBT2GlCe1WTEc9KPL9qLgVwgp4j9qlEZz0p4j9qLhYg8r0FRNG2eRWgqACmvHntTTAogBacp3HA5qU27N2qWO22dqdwIDHx0phTHWrLkJVdnB5poTInGBVZzg1ZdwRgDmoxGDyRzVCK4JIpNmas7F9KQgDoBVIlorGPFJwKkeoT61RI7cKaWphNNJp2C5JuFTJOFAGaqZyKjYEjGT+FFgubCSRN1pbyGM2jcVhLJOp+Uk1natd6kYmVtzR9gBWMykZ06MbwlWR/arD3M8cZ3PH+ArCVAr7nLZ+tK0kUreWZCoPUk1zTkapBPcy+YxWQj6VVe8fOGbmp3twJBHExdj0zVyLSohbvLdQ5dRnGayYWK2k2C6lcMJX2gDP1rVvtLtbSDdCGD9iWzWVA6GUGMbMdNvFRXt66/IXaqGZcYedwYJ5Fm/ilkPAFJpq3F3rUdvJPdXVsM+Z5bEZrKv9Yn1O/XAjgiAxkda6Cw8SR6Jppt4hHLJziUe/861JO6MWgaFax3k0BgxnAJJIrkNd+JSXIaCytlMZ6M3OfyrjdT16/wBVfNzLu/CqSbOrpuNCKOgPi+/uYGtBa2qxOMHK81Xt7OO5lCb0SY93OBWK7AcgUksVwIhJ5L7OxxTA39X046UiNJJFMrf3eoqhYMCSySEJ3RuhFZB8xvvGQexJq5bTeUhXYG+tAjafULKytgbF3+1E4KYyBWIC19cDcUQkdcU1g0sm1Rlj0FT3GlyW1sk8jlWY42dxQBbsZINPvCupRTTW/BVVcgA/TNOa60iXUt7rJFbsQMqMkVjAgjHzH9ansoVub2KFpBHk5DN0oA7221o6Lp4uvDOtXECr96GZshvwNM/4THW/FbTw3dzdSLsJKWyZz+VY1zbSag7W7OjzIuEZF4PtW74F0vWtJlkvreSFWRjHc2cow7j1FAcxx2oCa2cxSQvGv8O8c11FnZXOq+GEP9qW0MQXlSORz9ateNYR4muvM06OWB4hmVZk25bjgc1zFkDPqFjp97CIMyBZGJ4YelMVymLbKuscRklXPzL0NRRW80ieYF+6Qa9L8RR6Jo9lLaaS0LSsm4oGyfrXntvFdXcDwG4WFVxlWODSaATVr+C8WBEhcSoPyqn5RWLzJQQvfmrqWdvZS75ZyJR91wcirSQx3en3NxOx5Hy7eAR60DMyaS1aIJGPn9apspPerkdkXaMIy4cZyeKhaKWO/EC8SAgA9hSGIkjrF5ZohG5k/E1f1PS5LERsXV2fqq9RWazPBJ0KOvqKBks8jrPtikZVPpQ0ku4RvIHx0IqJnMnzNy3rUmkWovNReEsysRkYoAbcxkweaXLEdjT/AOz5BpyXiuHQjLqP4fxqSFPMYrMr+TkhttMe2YXhtbCV2X+EdM0wEaO3aFGgJJbsTmqpGDWk2nmC0aW5SaF1JGWPWqAXLUmBprpwisILovkSkAjH3afNbxK+2GdJPQ9M1Lp9qt1KIpneKJFJG88ZquhthqMwusKittGB1PapArS3NzaSNEsrR4/hU8VradrQSCKOS2Rk3qZpN3zMM1lXRgLGSMBx2yKn0WWye6ZbzZGhQ4PQZq7iaPaNVuom8ONNvC/ugyZ6k9RXBTa5cal5ttbqLGJhkuCc5+lXND1BLbw1LIFjuprSQ7Fk5ASus0vUdD1rQ59QFnGrRqRJHs5B9OnNJgeFzrExJj3bskMSc5PrTS2IgprQu9MuRcvcGAx28shZSx5ANOmV5IBDCUKJgts4INAyBbxo7VYlkwh6jHapm81NJjdbrMW7iJhyKzzGQOe1Ee0uAxwvrQBouz3HlbPNTjBJPH511tp4n+w6XGbSUCWM+W1uRkMK5awvZbBHjjXzIZDhkY9Ce4qfSY4pri4iktna3xub++vv71IHpWheJYI9StJ4JRbzXAxPGw/dyj3PZv8AGvTdK1KyvIpZbaaIeUp3xA4MZ+lfPd/bwabqdva3StNaPGHViOVHrW74bs7Oeyub2B5g+4xlVlP7wDoOaaYHb6dpLBHv5Yv30swmV8/7Wf8AGuwTUbRNUNkxJeRfMUAdPavN9P1d18MyTS3FxIiZVrdCN0Q/rWxoTWkOq2yQzPdzSQ+bblZBnHcGmB28roikk9KrpMsmcVgyX2p6pOUs7MRKrf6wy9fwrdPABP3zy31oAQ0ynUhoAbRRRTEO7VDKPlqZWXdhjinsgKHjNAFF9gj+tNh2r0NElvIzfeII6YpFgdcA9fWrEXEcN0p78p0ye31pkUYQUTMm0o+SGGMDqahjGyXJiDh5MFRnCmljm3KGBcbum7vXD65rUlvcESbZrSNtsiBsEntz3qiniq+aygkgkEbyMY2jkH7sD1qGM9IvLwWVi85K5jxlScZzWNq15p+teFbiW9hKQ7vLBJ5Uk4yK5c6xbpZafD5wnmiDBwmSMg+/NRz6yl3OulyL5VnNiUKQVGfyoAyNOMFneSeXNJc2ifJ5ZTcMHuOeajSy0y61Se08yVHIHkcbct6H0p+p2LWUqzWLyRnnDRjIJxWVpcty1xJeSRedMAU/eDbg9+KyZaZ08F1FYQXP2u58mZothj3gnPbin2FjZS/Zp31F4txUohlPzOOua5a00mfU7qWZSipboXaQtkMe2PyrqrfS4n0V9Sv3maRWCmOOPPXuQORSTEzYg8Q29jcQ79YWBZHJVMbxn3IrstI8R2OpTi1imSSYgsPJGQR6+1eO3T6fdadPNGnlTW+0xEHIJ9vSvR/A1yt1pEWoSRxx3ZUq4RcF1HcitFIVjtCeaTpSK4dQ3qKTOTVCI5BnNZ10Vj5Y49Pc1qunFYmo2/2qDYxZHbJQjsQatMdiGWVI0O84zVZ1Awcg/SsLU9bfSr+CwuEV1YbyxI3AD0q1DcrDcksji1k+aMqMjHv6VVxWL+MVG4UDOealyJVYxYcBsEg1UnLA4wRVollqNVki2kZU1RudBWUFrchD3B71es2wig1oY4p8wjlP7Iv4HI3KUHTmmCR42KsMMK6SVs9ayb21WRWZF+ftWqnczaKnnbuDTWXKnmq4kCna6kGpQ2ThelXYlld+DimVaaDf3AqL7M49KoRBtpNhxUnKtg07INUIgwRQCQcinMOaTGKYEgkLHk1JHK26qxpVbFHKBePPSpI5dgxUMJDAAmpGjPWs3EtMtCVT3qRWzVKPhhV1F4zWckUiQHnFPFMA5p9ZWLTGk4FRtIM0spwhxVZiQMmmkK5ZDDFK0gHeqyPx1pGLHp0quUXMWhLxml80HvVEK/JHanKzA80cgcxdD570vmVU+0BeDQ04K8Ucgcxc3ZpVOKz1uXB4NTx3BPWjkDmLgek3jPNQvMuz5etReaaXKHMaSNwKnRuKzY5egzVqJ/lqHEpMvqfUZp+7A4qukmRUm7NQWO38in7qhHWn5qWBJk0heo9xppakBIzA1GTTN1JmgB+aUGo91G6kBJmmmkDUpNAxjCmYqY9KYRSEQkUwqalYUwiqAZijBp1FACYpQKKU0AJimmlNGKYDMU4UUUgDNGaTFFMApabS0AFJmikxQAoNBNJijFABTgOabUsY70APQDjipMCmqMU6gQppuKdS4oGR4oxUm2jbVCIsU0ipcU0jmgCLFJinmkqQExRilpwWgBoFOAFO20uKAuNApdlPVealCCgCqU9qULirPlimFRnFAEaqDUqR0m3FTxKcUAWIYgU6gU2RcHFRO5XOKZ5zGgB+aCeOKjD5NKTigBT0pmBTqADQABeOlO2+lPA4pVFIBVU4qQDFSBRigr6UhiKaeRkc01RzUhHFAEaL81XI06VAiZIq8i4AoAUKMUFVx0p1MY4FMCtKtVXXmp5ZcE1VaUE07CEZahdalLZqJmzRYRAUJprJgVPjio36UwKxFMIqRutMNMQzbTdtPNJQMbinAUUopAFNNOphpgMammnnrUZoEGaSlIyOaQKB0GKAENJTyKQigBKWlApwFAgApyigCpEWgYBOOaNoqXFKEoAYKlQZpRFkVIseO1ADlUelWIxgVCBg1YTpQBKq96GHFAPFJmpGR7cmniDd3xSHigsaYhWjCDApmKcSTRQMTGKcBmkpRQA/CgdKikfHSn9agk60xFWZiSaqybjVuRc1A0ZIq1uSytkhqkVmPABpRGA3PWpw20dBVMCszYODTSQRzSyHL5PemEECmiSKU9qgqZ+vNPSLcOlWtCSsR6UhjbuK0Y7MFhkVYNqoH3aXMg5TGCYp+ytB7ZQCTVW4eOGJnYhQOSTS5kPlIQqA8nFZOpXayRlIWyPX1rNvvEKtK0cKGRT/ABMMVlXOoTGI4AX6VlORoiRrbLZcgfjVOZrWN9qRrI3vVEyyS/fYsPerUCxpksOa55OxSHIkhk81nKnsBSzzzPkNIxHoTSTTJkEMaiH7wcVne5QkEnlycUkyyXVyqLGB6t2/GlTaj5YZqb7ZGvTimkBwsybZXHlsnOdh6igxsRhIHUd91eh/EXTYrKW2uY1VXxg4HXFYEQ1HxDbuE8tRCobJOM1sZnKyRsn3hiowHZwiDJPQV09isN/dx2OoQGTB2qIxzn1qzr+lHw7cRPDJuhPSN1GV6UFJnMahYy2UabyCXGQB1FbdnY6Yujhry7mWRgCADxVJPM1rUlJjXp82PStaTQzq0otdGLl0XLrIev0oGYKTK8RjXcT/ALVRwI00/lKPm7Uskc+lagbeePa6vtk3L0FamoaMotVvLFi6kZJDcYpgYs8M1lKGIw3at620PU9d8NyaoLpNsLf6tl7Y55/+tWI1heytkQyv+GcV3PhDVbCLRZvD2pgwvPnErfdbP8ulAjzyC5azuop8FShzg9xWvcwwNOt5G25WGcYxiu/tdDtms50uRFcW8YwCUGcfWuGur6xSWS3htX+zKx2nPI/GgBunSahbytdaexhwcMmeTWtY+Jri61x5FkOCMsoPcD1rmhPNDdB4ZJEUfwNSzTtLP5wGx/VTQB3Fi2t+Mbqazgu4IJYgfvj5sen6U19GthM9jrMBhvoRxJvwG9wa5zSNeXSreRTbiSZv+Wwba341lXd9dajcmW6neUjhdxzgUwsTEtY6nLtKOY2K7hyDVa5nL3TzDh2xzUbS7SQtRlixyagY+e4luCDKQcDoBgUeYfJCBiOeg7U1QucGmuNsm0UDNO6uv9Bht3Xc68rIDWVM5eTeXO71pSOKjwScAZNUIfBNKt5C/mklTkeYeBWjqCtcN9pLxcgDahzj8ayfKdgXQZx2q7Lco6psg8pgMMo6UhjYUaa5hth/y1cIfpXZavow8LarZT2kZm81CHBAHXFcjZBpL2HylJmB3Jj1Fdhb2eva6UmupNy28hAVlA/AH8KoDlryGezu/OkG3zyXC9h7frVcyObjzkl8r1YHkVpata3lzezsSH8ptmOhH4ViTRyRNskGD6UgOhuruZ4YxqCfa4yMpIeCKxSPSp5Hu3sIS4/0dejHqaSNV3KzjKKcsB1xQBZfU7iWJY/MWKJRggDk1BHAJmwpUD3NbeqqLu0jubezITb98dfxrFQDoOtS0BXmURyFB2roNLnFtpwS7tUdVO6PcMYBPWpdDSW7jFvHYNcsHyxPXFN8RXwGp/ZzE4WJQpHrTQHVaXpdnb+KJ7VAFtru24UnoTUHhKH7H4g1XSpHG0qRg8biDVXQ1uYFTUb63ea2i+aBt3zqAegFaP2mH+059dfTUltpXBSSZtrA+woAm1DQJDpt3DcR7IUG8S9ST2FeYQOwuEWJCd33sc8V2/jnxSJ1FhaTEQlB5iKMZPcVjtpkWnRRXdrdgb4uQRkHPWgDOj02W+vfscBG8jILcCqk9ibNDFK3+ko5R0x09629Engj1Ca7upQrohKZHU1kXN7FdTyyun71m5bNAETlgF2nGCDWlpogN4kt9dSW6SDKyIcc+9RXFhPaafHNdJ5YlGY+c7hTZLWOfw6l15i7427/AFoA3Ndmu9QuLVp0AnVSgx3HY1kx3H2DVIo53ZIlcGQKSARVyytru90u1vEuVQI21vM6qanW3W4zY3VsTMZOLtBkZ7UAXdaazt9ek1fQJWkhTa0oPIbI6e9aGkX0ek+LdM19RmzJxIiH7m4YPFQzLb6l4ZSxso47a9iOySFDkORj7v1rBurCfTdLt5pmZVdtrIvHHtUgfQWo2SyxG+sgOAHSSM8E96tN91T6jNea/DPxJa29lPZ3LzgI3yuzEjYfUdsV6Y21kV0ZXQ/dZehFNMCE03NPYYptMAqKRygz+VTAUMqsMMKLgUog9ywU5GD1xWoBtWoY9quKtZ+Q0wIW29TURwTkVKR61EQBTAUUFQTkLlsYFJmnBsEEVIHkN9awSy3Nud3yTEOQcYIPp71WsbhljdLy3+02wU+WVABz712niLT9NSe4uLddkwO5n6jGfmH9a4zxDpr6BdRSRX322KePIK9I2PQH68/lUyKLlrbS6bdxy2bpBMVOTKcjGKzdQ1nU9Yt2e5+zR2y55XAB6dsVn2MyXUjT6mjXTqp+QfK+f69qr3toqWZeISRqPuxE/wCcUkiGaVp4kuYNGMCSBpFZfIJH3R3/AKVYsr+9kluNPuLbfLNjcSeFOOp9Kn8JiCDWtOOpWHlwvE0cbyLwSehBrfbSLTTfE3mWVwCtxEzSB3GMdgCKOUVzlpI5bbULa0s7jaHOXg+8qkHv6iu3ZdQ026huHvLS2iKBruyCnkZwSPwrlbV47a9s7LxFJi0mZjHLAfnDZ6EjtzU93B5uuSNqFy0tipWKFnPIi6ZB74NCKRWsJtNnOt28d0gtiN8CkYLDdwR9BivSfDel3MMOmPPCimKMKyg8gY+8a8uuo7SC/TTYS7XCSGJZUA2FDzjNd94MaHSNRUG4d7e9JWFi3AIzx+OaLDPQWKxoWY4Udax31+2t9W+wyfuww+R3PU+larupdlHJHUVxfjiKa5giksrPzHRtyTK2NoUZOT3poDpE1iBrm4hhmSYx4LnGcEkDGap6tIIL2CcuquuYzyM4PT615fb+Lb8+Qm2DZK4VzGPmcZzitDX76XV9Ugu03eSUACsMMSDnGPxAqVIoy/HmoreT6aoDS3NupFwwHUZ9aoR3t5FaWyC6kKFg32djz9PpzXRrbtc+FdRup4YYbRI28kqcMXHr+P8AKudEdtcLb7pAk6hSYX4Mij0P402waPQ9J1Bb2UTqBHsGyVMYycDkVdY+e5VvwrF0LTt4lmtb5PtDModHfcFXPHTrxW4be4W4OImGGxnsR61pGRDQ6KNoyMCtBTlBSLGOARzTtlVe4rFScYzWVFMXJ3cEdq1rqFpIWUDk1nLatArvIOgzWkSWZ97b7naRBVRI5Ew54qaW7J6HirsUayWYYjJraLM2ij5hpVyaHUoSSOBSebu6YFWiBH2k89aQKCMYpuwlv61Iq7f4iaYFcxMCSelIwHpVluRUDIaaYrEJWmgVMUb04phFO4WHLmrkbnaMt+dVFODUgO6pYy3jLbh61bQjaKqQkEjNW02dBWMjRE3akNKBxUix5GazKKzgmq8vTFaYiXviqVzEiMSrA5q0SUg2DipUOaZsyc4qRI2HB6VoySUKNuMUKnPIqVVAOOtTFPlFRcspTWyyDOSDVfyH7GtExse1N8huwp3FylWOE5yRT2jwOmKtJEwqYwKw96XMMzlGBinhCSMDj1qw1tg9KmEIMQU0cwFZYmDDGCParsUbFaYkITgVdgT5elZyZSK+1lNTIGqw0WFNRpgcGoKJEVs9KcykCpEIApJORUjKxNMJqRhUTelSAmaTNNooAN1JuptJSGP30ok96gJoBoAs+b70eYMVBmkoETFgTSHmo80oPNMBTRS9qbg0gFFBpKKAClxQBTgKYDNtG01LilxTQiHFBFSEU3FMRHikIqQimkUANxRinYp2KBke2lC1KFpdlMCEJzUoXinBRTwtIBAKcKXbSgc0AAXPapNnNOUVIV9KBkBWm7alCk0hXFMRCRTCKkNJxQIjxTcVJSYoGMxT1FG2nAc0hBilwKWigAUc1KBUanBqUMKBjcGmsQOtSnGKryAk0AIJBViOYAetUSCDUyA4FAE7NuHXJphX0pQM1KicdKQiAKafsJqbYKz9X1eDRbdHlXzJZPuRKeSPU+lAy3sNOCkVz6eKZ5EDrpyhT0JlFQzeOreyWNZ9Nd3YZ/dyrx9aAOoGakCmuf0zxpp2oXUVu1tNbvIcDfyM11f2d/7tAEeKKeY3/umk8qT+4aQxAKeKQRuP4T+VOCMD90/lQBJHxU+8AdahAI7GlKsRwKAJPPT1qOSYMMCqEgaPqCPrUYuPeqJJJTkmqzttqRplxzULyIaBgHJ70m6oTkj5WApBkdWzVCLAemOc0zcaaSaBDWPNNpSDSUgGmgUuM0m00DENIacaYaBATSZoxQKAGmmYqQjmkxQA3FGKWloATFG2pNtG2gBgXmnhRTgtOC0AIFqRV46UBeamVaAGhPapFTnpUqpUgUelIYwJ7U4J7U/igUAII/anAGlyKARQA8CjFJvGOtIXXHWhCA03IpN+aAAaLAL1pwFGMU0uM9adgHHim7wKid2IIBqnJJIh5NOwjQMoA61C0gNUVnLdDUyPgZJqrASNk9KQIe9RtdhM4GaQXBcZoETBFx71E6HsKliBPLVYCpjNAGcFIP3aGUt/DV5ioBwKgZxnmmBAtsGGSKd5Wztika5VTjNN+0Bu9Ai0nalYgCq4LHvT8nuaQxrruBFZGsQwLZnzDv68dq2CRjqKyNRUMr7wCvpmolIDz1kf+HitKK706GEROPMl7jbRfRFfmEZXPrVERbqwcy0MmgglmLoNgPQdaT7MAOM1LJZTRR7lT361WFxcr6VPMMqTRFHIBzRGwAxnmnuXkfk8nrSpBuGV59zUXKI2QHJzUQQFsk8VM4C9T+VRFgvA6VSYHUfEC1kvr+C3T7g6+3FWtL8PR6To97bg7nljDK2a37qwS8vWaTv0q+1pG8IXHQYrpMjj/BPhq2tYZL+U+ZcEkdfu1k+ILJtTnma7WTaDiJkGR7c11mmQ/wBjSzpLMDCcsTjgVzl/43jt7xreyKLChI3vyPwpDRq+H/C9ppVkZ0GZZY8MGUHBqx4N05bKO6mPLu5AOK5Kz8YJqN1KLu9eOKPqv3Q34Vv+GvFNlOLhYwRbJlhMTwaoZT+Jmix3mnjUI4wJoyASByRx1rP8P2FgfC0ck6hkT75Jxya5zXfFcl5rEkc1xJNZAkADjvWFd6gHQw20sghP8J6UAbmi6tFY6/cpb4NsSSoLY/nUHiJ7CVEW1tjFMHBbLZz6VzafKc1YEmRUgb1x4pvDYra2sf2VMYcg5Lfn0rBS7mjBC7Tk55GaUtkU1E3PtBGT0BPWpAtS3gvgondI3QYVcdarFWGTj5exqOWNlYhhginC8k8oRYUgdMimmAxn4yKar8Zp7xXTsW8g89xUG10bDowP0oKJJJAT9KXDBA5jfYf4scGtTw3pLahq8QZ1TPRX6MfSu81PwVfizU+ZEsZYZQjAH4YoA8tXMkiqqOSTgDFbs/hy8tYkle0nyRnaV7V61pvgLTJI4o7jaNmD5a4AB9aZ44uNNsNFaztpz9rRcKgbJUYoA8QkUM5Krgemc1HHI1vOHTqOldRpWl3nim1uYoPJje3UtkjBb8fwqvpaXPh29h1C5tWZM4Kt0IoAx13XN2FVBmUgBF4BNdDa+EibqSKRZkdl4+XI/OpbuwbxK0uo6dEYXjOdowMCt3TJpV0xJYbmSW7C7Sc8qfcUAYmm2mpabbyl/JDQt8sLJyQe4Oa39PkittMvL0TI91J/yxU8Jn2rPsm1xpJJb7ZMhU5V8Ka5qaG5vL6T7Mzqin5mU44pXAq6hPdwanct5sodmzk8VnMZJCXlYs3vWnctL5yNLN5+wYDdyKoy/vJSwUjPY0wHiaU2wtz9ypodgkj837hcBvpmofKkXG8YqaO2+2KbfeEYgkE+1MDpdaij0XfYWrM6XCB0zwUB9KwbGPdeIsitn0UZNad5qzXWlWtreBkv7U4WRUzvXtV3StPZrFdZiu9lzGSGV1zx+dSAzSb6XT9QuUV3gVhkbk6itextE1qURQQq0xBlMkqE49qzNZdtQSGZUMbr8uW6Z9K9L0G0XTdGtrcjDhAT6nPNBJytxOUM1vNbSrMEKDaMqPerDwR3Hw/8yYY8lCIT1JcdOK6aO2SCedgoYynqR0BqWxtbeysTAUBhBzg9qY0zwe8DXHLJN9o6zFh3pYLaK2uoWD+ap6qDzXpeq+HoZNH1O7uRm4c5j2twF9MVzj2cF5YLc29sirCoUluOaBmFN9ogfzIl2j+6y81BBDa3LJE0D+bnLbG5I+lX3lt1ulheVbrccMwORinXltFHODBbvs/2eTUgbaz6NrVp9jvIJY/sce2A5wW7YFN1zwpZ29po9vIohvZZFEiIeSp9fesfTbh11SGZWAuoGDxlzw2P4TXe6HJBr2qtrOoQtvE3lhM5EYxwD/ntVcxJ55rEN74euZ9PSQi0cbgp9DWxounNe6Q1jJK9rdtiRPM4DDnpXS+L7C2ub+2tFjRg0it9oB5UZxtb1FJ8S7S3sptPdI1O2z8sA+i4AotcbZy4huY5UNuqwXNvna8LYb86v3vhnULvTtOiLu19czZVC2QFPOW9P/r1veA3ifypGWNnuYs4KfMpXsPzroNb0iW80M6lDI8VzaSGWLaeAnGRRyk2ORl8Py+DlW6nnWdrkGGaBMZHoRXT6uk2k+FLW4jbEq7drDgjNcx4m1KKeHTbeIs8vDOzHJJJHWt240u5s7u6tipntri18yDzD91sDKj6VNykdtZXcc9haNLIBNJCrEHucc1YxXE6W22x0N5U3xqjRtx0PSuxhLruST+E4U9yKdxk46UhHFIDTqYEZU0x2mTlTkVP3oZdwwBmmBWF3iTbJwvc1YlwvU4rKur63tZSkgy31rF+3352R6fdqsQznzlHA9s9qAOoMgKsVPQ4oWX5GPVgMgetZ1jKsqB2E8m770xG1SfYVY388UgOf1jXBdC50u6lSzDoMosZc89jgVxmoL9j0i9Z08y3DqyDOMsM44PXrXYappV7c6g93DOPOEJCoOrHHFc8+najf6vbafrGxZZoZBC6j5VOBzjrmnLUaOMSLUNR/wCPWJpXjAc47Cq/2vUWlN5cyu652rkcAjsa6w6JN4bUi7tZgjfK17C+UYe47VRWwtvsNzbQ30b75GcInUis2rAyCXWbqG0ttPuLrb5J3oJF4T2HoKZpmoQRXNs9xJHJC26JlRzx74qve3FltUBwzBQrbx8ysPf8ai8Nm3bxDYy3K/6MJPmKr7U+YVjr7PRZB4o04NuuLe6iZ4yU5Xg/Lg/Su/l0CO90KCG+tgLiCAqhB+5gdffNZmlI194oubs3GDZwiFS3IJbuKqeLBqGitZ/YLwSvcsyMr9sAfpzRcaRdtLHTNEt45LxYQkiEIkSbmcHqfXPTpXPpqljaactrBcW0qxyNJ86sjoCeME9SKXSvEcQtZrzUDMLxAyhVO9EAPRfTPFZ6WsHi5LmSwcROtwrtBJF+8we4bPT2xRIfKdfb+KNTl8m0S3WX93vkuQPn8rnr71xV/rOp/wBnJGlzNPpMMu0pIdpb1BI7VrfYL7wrquxnD6fcxlNpHJGOQKxPEt7EIba3EH2K3lX5tr7lAz3A70kyb2KVz9mkdLi0tRaurAhEb8xTpr29mumkjjfEnyLuBO0+gx3qGMTrKZYbqCXyvuRr0bPGD9KktYr6HVxPeTwrFu3lUl4B9QMUmhpmrqF9PaaJa6VfW7+S0heQMvO8e/41jSunkmWSFxDGw8knqF961dQuINQ0+4lMr3AjYOd+QTzjGazLbbBqoLr5tsSCYgS+0kccd6B3Or0vVk0Fbaa2gSaGRiZfLXJVT3B7fSuji8SRamXNrdXNs8eQolQKJD6DNeTXEZttSe4tHkhtnfaEJ5HqKlgubp7m3nmO+3tZtyA/eB44pRdhM9YsNWu5Ih9vdVEm14Jtv3hjkH8a6MR15zp+tQNdWt3cSgQxt5fllcgA969D0++h1C1WeE8N71opCHNHxVeWH5Tgdav4zTWjzVCscZPolwGcwrlc5x6VZsLa7igxLC4UEjOK6FhtfFSDpWyqE8hgzaerRNwc1jz2rwjcFrsGTccCqF1pklxnH3T1FONQnlOXDkdaVZc1pz6MI4mIJ3Yzk1ijK8HrXSZFxHGc1OZFIwQDVBXOMZp+WXrTsMn+T0qN4MnK0Kd1WEOMGlcCk8ZQ89KaGKmtSSJZI8is94z5mMUILEkTbmHNa0MKEDcTVGG0G3eWAx2qb7SEYL1x3FZyHE1FRFXgU4ELms5btiQAeDVhYJ5gQrAA+tZNFhK4IIzWdJHliRWmumTbiXYMD+GKsR6UD1Ip81gMqFTs4FOww6giugTS4gQQvNS/2etT7RDsc8nDDirAZT3rUbTVJ5GKQaTD1wd3rS50OxQABpQtTPbeWcAU0Jii4Efl05UNPAqRR60XAZ5XtR5R7CrIC4pwA9Km47FXyz6VNHhRg1NsBHSoGHPFF7gicnK4qk0u2baanBYdajZAz5NAydDxUuwsuahXgdamWTC4qWMiaM+lQsuKt/eNEkYxmkBnlTTdpq0yimFakCqRTSMVYK0wrmgZAaKeyYNMxg0CEowaM0maAsOFSKm4ZqIVMhwKAHbaaRT8000AR4pwUmnYqaNMigCJV5qUJ7VL5I7Cl2VQEZFMIqYrTCKYERFNIqQimkUxEdJinEUAZosITbRip0jzStHigERoOak20IvepMGgCPaKULUmz2pRH7UAJt4pFXmpvL4pyx98Uhkaip1AIo2Ck5HSgBCoBqOTFOYk1C/WhCInNRZqVhUTdKsA3U4c1COtSBsUBceKUUwMKcGpWAdSGgUN0pWGNJo3mmE0gPNOwE+80hkpO1NIpWFcdjJzUiDimqOKdnBxRYLkyp3qdRxUKPxUuaBjwua8m8R38tzrt1PNG6KrbF3DACjpXrCvikk2SDLxRsfdc0gPEfOEp4yx9hmrlnpl5fy+XBZzyP7IQPzr16O2tVcOtpAreojGahvrqSFSEwv0o5Q5jltI8DSgeZqM5gK8pHC+Sp9Sa1ptGkRsDXL/AP77pft8x6saaZnk/iNXyEXBNKuOi6/fj8QamTRr7+DxFdj6gUW+8yDAzV9HbPK4qXEpMqjQ9S7eIrjPuBS/2HrI6eIJfxStqEZUGpx0qCjAOka7/wBDEw+qUxtP8Qr9zxAn4xiugeqcpwaAMKe28SJ/zG4m/wC2Qqqw8Rr/AMxC2b6x1vSfNUXkk9RxWhJi+b4i6Cezb6pS7/En/TkfwrUlUR9KqzTMIzg1ZJRM/iRfvQ2Tf8CpPt3iLHOn2rfQ1bSVmHJzTs7qTQ7lEajr466VD/wF8U7+09d/6A6/991dTANWFbIpWAyn1fWAOdDH4OTUP9uasD/yAX/WtzdQZGHQ4pWAw/8AhIdRH3tEkH50z/hJLkfe0WYfia3DNJ60z7TJ60WAxP8AhJWHL6Zcqvc1JF4o012CyJNE3+2vFbAmZuoB+opslra3KYnt43z6qKVgI4NRsLoYhuUZj2qbbzWVdeGLKb/j3LQOfTkVnsda0KQbj9og/wA/40AdJto21TstWtr5PlbY/wDdarwBoAbspQnPNOpRVCEC07aKM4oDAmiwwIxTd4FObpTUAZqdguSR7mI44qyq+tCAYpWcKMmk0FyQcUpaqZu0U8mlFyG70coFrdmjfiqjXGOhoWbd1p8oXLBlI6UCXPU1VeUBSarNNnqxFHKTc02lAHBqIyHGc5qgu89CTUclyyHGarkDmNJZznk0rX0cY6n8KxvtDk/eNOyW71XKHMajaiCuAppYrjzATyPrWatWoRwBS5RXLT3SRDPLfSqz3qOMGOnuiBSWFVCFB6UkguSG4JGFUU+OGSQZbgVECqnIFNe5kIwDxTsFy4LeP05p4RV6VmpcSq2d1WknyMk0rBcnMhFNM5HQ1E0i461CSSaLBcna6PrULSF+/FMIUdaaXUdKYXDZjrTlKg1GZSevSnRtuOABVWJLayHtUuciqocL1NO8/cMZxUtFIo3kzBmANZl3dyDMaROw/vGtG+Rd37tSzN1IFYesStFAVAYMOoIrlkWU7uQyqSWH41mFttaFpB5sPmyj5jVO9JMpG0DHoK52aIjmu7h1Cq3ygdKqF2HDDmphxxzn0ojWEyn7QQB6Gs7g2V8YOT0pjvvX9309au3AsNw8tsn0zmoz5XlnAxTApLbu2Wd1AqBwqOQGBHrVqXGMCmArwu0H3NWgPXlAzuPQVlXWrPP/AKPp6GWU/wAQ4Aq5cZliMecZpbWFLWEIi12GRzV6kunaNffbWPnSLgKTnmuf0jwXv0WbUbuN2cqWRC2MdeTXR+KNOluZoJ1PVgu3tXQrL9l0VViGGMeFHuaQz5yuQy3UoYYbd0r0HS/DTWHhKW9a5dZp14j6AHPvVO78EanJr8ZkRCs7bs7uorvPECMNItbNdrygKMA9cVIHjlvoN1d6x9gTbvYkkjnaKktPD7trx0y7kxgEh06ZxXd3Ea6Hq0F09tvfGCIutcrrWqFdYN3BC8EvbeORVJFmNqmmR6dMY1n8wjr7VQ3YXFTXtzNdXLTTPudupqtnvQ0IVWIrQ06KCVGZ5drx5xxmmpbz3tqp8ny0H8Q61f0nSZ7uZktCVdRksw4/OsxFCVzIxdjnPehLowrhY0+uK0NS0O400p58yMWONqDJFULqylto1Mq7dwyvuKAHR3MsjDJrYdrNl3A4YdM1jRROsanFKHUybHoGXpbqWG6hLKBAG+Z14yPavRbXUVtWWA36LYTplHuH3YPoK83SSN7JrUDOTlfap2vJRZravtKj1HIp3Gex6bAdHsJLlrkSqwJ3HuPrXncOm/8ACRXckoBRpHJDZ4X6+tZK6nf3EC27XErxrkbR6Va06GUytjzkVRklaLgdLeHT/CGnXCLKpubiLBjUc/X9a5M6xczWUUQtw8KHO6Qc1DqF7Pq14Bdu7sgwpfqBT9ItLa5vTayuyZHy47n0ouB0vhbWrG0tXimjxcSNuCoOCKqJf3+nX9zFb2sapMxPmIuWUewrFaWbSNWIV8PGcHjqK3k8ZQ2tzHdSaZMXQYDoRii4FW51SzgbbZPNJMwIbzuCTS2mi3cvlzRz26CYYeOA/MB61QvtattW1qTU47FowygOpGR9a25rj+zNLjuraNJJf4XxjBo5QMvTNMsbfxXJp2pZkhHyox4+bHHFSeNdCj01obuOdDEDs8rHzKMfrVS7l82N7uVkmlmwzHuPp6VY0W4Sa8ju9VuxcDbtiDEttP0pXA5m5uYrhlMZ5AxjFNhmjjlTzRlCcHFaXiOKz+1iWzVUzncijAXpj+tP8K3FjHe3MWoeWIZISN0gyAaaYDpBbmSa6F1GfLX5e4I967bTdUsL7QGupY4oYogAyhhknHWvLIUMN350XGxyRnoa3bLy9S1JA1m8nmZ3QxNgOfU0AdJZ6/a6ri4WEK9nMG8ofxJ0zXokjC5gju06MMgfWvL75dN07VNPWzja1O8xTxS9QDjqK9QgikitVt4lxEgwh9qoTRFeajZ2EkaXEhXeM525A+tSv+9QrF827oTwKhuNIiuZ455vmZBjB6GrkXyxqD97uKAOX8azS6do9tFFOUMrmOTH90iuEvJZU0qexguzJZpJn5F+/wDU16rqukW2qlnu4g4VCIyeoNYPh/wu8FisNwuEEpcMQCSOwpAeYXDwokLW8ZjaMgEt1P1rRuL1kkQ28m7cM7McVqeONBuF8RTy28SGNgDtTtWd/ZV7ps1v9rtWh3dMnqKhoZlkySz/AGnA/duOgxivSDBPaW9he6favNb3kYea3jPOQOoriEsGudX/ALNhkSMSt1Y8V6z4YsrqNIFulcRRwiKLd37E/jSsBxVhrr6TE7i1Nypcsku4Bl9VYHrU0Gk3fxE1v7UyvbaYAM5OfYgYrFuLW/udVu5Utw/zMWUnhRmvVdNh/smztxbQuVmiUiNBxvI/TNWmJHH6fBb+FvHLRWyYsYV2MrHO0kcmvRbl47vR7iC1bf5ibV4x1rz7WvDt9oMd9rl7eb5JzuSCMFiCT0J/rVbQv7dtLOC/uLwxrcSBbaGRcs4Jxn2/rRcdjQ+IOnfZZNFuIISXDLE2OckYrsddtGuI7RkYK8Rzg91I5q5qGlfbZrN5fuQESEYz89XZIRJPG+3dtHT6/wD6qQHK29g0dlpXl/Mv2jDL2zk108g/eGqVrDsgETqdokZlyMYINXiM1SAjpwpCMU0sA2CeaQEwFNmjlliKRA5Y8kHGBUbuwxtBPPOPSp4SLqFZ4ipAPr0pgc61/bNrrWSbDPFnfHIvJ9MVLq9zZiwMUw23BBCoq8DPWuV1yc2HjCbU3VIsBXAlJAcYxiuevPFkVzcS3LRN5z42OnVABjHPWpA7tdQ1rSraBorSK6siNqs8m0r6ZrRU3c+nSXX2eOK8Iz5aNkVwdnr0Fv8AudYu7uRMApHGNyMCMjOOlauieI7m58Q/apknh0hk2Qv5R2A+7AUrgU21y4S/uhLfotycIFC8Ag9P0rA1WeeS/gvLbVLppXbZJ5gCmE+laHiVLvRL4JPGlysgea3uoxktk55FZb2sosreWNGljuD5yhRkqw6hqpOw0U73Vr6Xzo55rmW3GEnBbAbFZ39pwvOGRmhBXaJQeV/CodR1KOZZdrmKWZgJY1XAwO9VGgkuYnuYrdUhjGGZe4Hem1cCUXjMjwtiVpTkkjkmobO7eG5+ThsEFD1qaGBr+6i+ykoxGARV2AZ0u5ae2HnwyDErDlue9Q3YDZ0HV9Qsx9tsLhHuIjl4HXhx7itC/wDFj6nZyv8AY2ilmY5O7cUBP8NctLpU+maoVaRrncgdpIV+Ubhnk1oWkkcdxCHcIi+pqVIIkOoXLTz20MbxxCXCts4wf9quni8RRaMkVi1pbWQaPY88PJJOeQfrWJqN3p7MPJth9pjOUdBwx96xrxLyO4864SErKcEIcgYocrjNp768ttVYSX5u4URkWWVshd3II9D/AI1mF3uZoIL6VipjaKMqvJOe9Fva2szSiSbYiR8QjgginwSxNqdulgAEVCczDhTg0iDY1g22m6RpOUjLrFsIQY3HjJJqrqktidPtZY4MwkkTMvDIwx1q6kkFzpvmXjhFtCRuOCjn6VyvnWjSyJtwjnIbOQT2plmpNJZQac0dtJPFKrh40kOVlGafo90lorzvceVdq3BfkEfSs9SIVjhuIF2jJXnJHoRTLmBfMjDc7xmgDopke/jM9zdwsCxcIgwATjP8qzJLKKOKK8WR2RwTgnBDZqk8UsNvKLVsREbgnY1saYiTWUNu6TSvAd8iE4DjOcVDEILlvmCsjRcthu/Ar0/whJ/Z3hz94MgSDYw6bGAI/ma4DUdQsNQ1JDDpnkxRkbVUAHPocceldf4RuLi7uY7aeIxwzbnWM/w4/wAiriwO7Eg708sMVn3CvA5wxxU0cgZK1sIZISZM7TirEYyOaYTTg4UUwHbAHyKaxANL5gNRPy2aaBoyNcnKBdvAI5rmJeZC3rXT6xbtPbuF6gVzTwSKeldtLYwkRqxVsintIz9aYAfQ0v3TyDWjMyxF96rarlRVGNsYNWY5c4BOKloZZU8VFLGSdyipYpEHBNK0WW3L0qCio0zgFexGKbHyakkjweRSKBngUxE0BZJQwrctpBwaxYlO8VpRNgVlIuJtAgqMVIi+1UYJOmTV4SrisGWWF6U/Iqr5voaUSH1qbFFgjJppFIr5p+RRYZC8SsOlVpIgOAKuEjFQPyadwsVhCMdKa0eDxVgnHFRO1USxqjBFS9BmogwzT9/GKVgDf2puMtmm0ZoGOKk0wjBqTfgVETk0ALnilBzTKcKQEwOBTC5zik3HFMPWgBxNNJFIc0hFTYYw80uAaQ05aAGsnHSo/Lqc0KOaQFVo/aoypq8yAioTHz0oArAHNTKDiniP2pwQUDG0U/bSbaBDR1qZDTAlTomKAJlPFIRT1T5aa44pgRGoz1p5phpiIz1oxTiKXbTAaEBp4i9BTkXJxVpEHemBHHGB2qfylI5FO2gUcUhWIDEo4ApVjXrU2xTzShAKQEXlg9qcIvapAKeKBkBUilXpU7INvSouAaLgNphFSEVC2c4FMBr9KhcipymeTUTxnNMRCeRULDtUzKRUZB61SEREYGaaXHrUj/cqk7Yb61SVwHNMQeKmWYYqNUGM0hAWnYm5ZWTNPMmapGXHemG4x3o5A5i1JJgZqEzgdTVWS5J71WaZj3q1ATkbcNwjgAZqRnCjNYUdy0Z4NWhdGROtHIHMXxc80vnFm4qnGwC81atSrkkVDRVy3CxI5qwDUQAXGe9Tx4xk1DKuOCtTwvrTTIo4zTTNjvU6gWAoqrdW6SAjHNPE2e9IZB3OaNQMiSxYnGOKlitVXGVNaO7PYU6NR3FO4rENrbnzPu4q+YAozinRKFAIqU81N7lIhUH8KcTS0hqQFJzUUkXmCn04GgZRa1f2FVZyyKRmtdl3DFUJ7IKSS5NWiTCmnffyahaQsOa0ZbIE5NVpbL5flIFbxZmVRJt5o+0elI0LKcVEY2zxVEkm9mfrVqCQqcCoI4jjOKswoM81LKRZByM0EUClNZljdopPLWndKOlIQ3bjpRT6Q4oGN3HNPDBhtYAg+tM70CgDG1Pw7vzPp7eVN1254JqPStbeOX7BfqUdehNdBvxWZq2mx38BcKFmT5lIFAjTZecjkU3pWBoururGxu/9YpwCTzW2zZqhj2dfWhBuPFVH/vUizMvQ1ViCa6Z1AA4qukrbxk0jMzHLHNRFtr1VhXN6KVPLGTUdxPHtI3YrI+0N2NRM5dssc0cgXJZZN8pIPFTIHCDFV4lVmHFasFtvXBOKHZAiqC3epg2BSzwtECazZZW3daFZiLctyvIBqm8zbs5qEtSE1aiK5YW6kAwDSFi3WoOlL5maYXJRwatQFWIWqO4mnIx7UmO5qF44vlAWnQv82WPFVoLRm+ZgSBV5LcsoUAjPesmMbLNGRjNU2fPStJNIVvvyMfxp39hxZ++351PMBkbyaUc1fubGKBcrkmqTbQKvmAjY4FRossz4VwB71YVY2GZeE71iX9+ba5K2/wBz1zU3A6BIwvDGp/K/d7h0rl/7ZfgPcpn0rUh1Sd7ZUtgHPfNRzl8pNdSrCuWOKqLdwN/y0A+tVtcuJI7HfPG6SDpxWPZM00fmsm1T3YVPOHKdIJ4WHDg1BHe+ZceWn3f71YzTeZujRS69wK2dPtESJZHYR46g1cZ3E4mskZIBIqGe5ggPzHPtVSXWooMgMJMf3a5mW7uJ5TIxx7ZqJVUirHTXWuRWqBkt9zHvnpWJc6pDcEl4Hye9ZMmoucplfqTVVrxjwGB+lc8qvMOx0MMaum5RisTULl4rgrtLfQUQ6hdqmwPtUdKpSyyvdGSQ59KzepSYm+aRsrC/5UNp95Kd/wBnkCepFX4tcuLdNsccSj/dqOfxDezKUZlC+gFPkHcp+T5Ay6srD+9TGl3GmPK83LHmo844NOwXJsg9Tikwp75qA7Ryc5oEmOlOwHr5FKKUoTSrFxXWZCEBxhgCPQ0jRq6hT0FPKEdqVUYnoaTHciS2iDBpCAB/eNYc15aTajNLEs1zsGdyDIU1Pr0dxc31rZ7mWJuuKZcz6TpFk8UV3BbPjAJqbAeaz6iJdZneW9YbXYAH0zVJ30eeZnvbqTeexJOKr3EkNxrskS3KPEXb5x0PNLqml2sc4+yMyMBg44BpMpFMWlpLq6pHva0P97inR6SkupNFG2IR0DHrUTG4jG12qxaf6MrTvIWdhgZ7VAzZk1LTrJFgnWR/ZKk/4SUMFh062FtH/eIzXLSSGaQu55qe2kVgQv3gMgVNhGnEv2i9M0spkwSDntVDVr1ry5KMcovanaQW3ybxjcCcVmzrJHcuzoVz60RQF/z0NugJ5BqpK5ExdRkGoonDKUb7/wDCadJI6Ku4cetVYBpuPLl3ZxUq3F1j7QIt8R74qxLbrbNFcCMSRtwwPHFXbnUYhbx2unLlSMkMOlADIYri3lTzF2bhke4qyNRubdiInxnrxSXfiRbm3S3urBInjx86Dk1TiuorsskMLscZBqRnXfZrSaztrm4by3mH8I6n0rB1SA6XqsTox2qwYZ7VPaX5OkRRyn95FJ8v0FM8Q/6TfR89UBqQLevwpeXEV+RvikTntWGbk7fs8I3J2RjmtDT5m8hrK45QY2n0FZt3pz2t6os/mDDI7VVwLCwXEdo/lRrHuUgkelO0e/a7njsr68WK1PUuOKZrOoSR2q26Jsl2/MMVnWgXfHI6BkRgzBhwQKpAdBa2EE2sSWQnM0KZ2SA9cVPJBo8kMiXt0kEsRIxEDlvypDcade3DLozJaoib2aQbTn0Brd0nwlYap4aVr3Vba1cs2XXBJ+tNoDg528yxZoo8Qg4DE5P41nocVNcq2nahc2treCaJXIEi/wAQ9fao0XjmlYC3AqSIT6DNaPhSUrrsUvV4wSi9ifSs+ICK1Z6lhig/s6S5jk8u+iOdoPX6UgNDWLS/vdbkae2zLJyCnQiuz0LxfKfD4sZrhYL2DK+ZJwCAeP0rCk0LV4tPg1uK6TUGZA7xA5OD261m+GZUvLm/s5lBD/Mm7oCKoD0jw7qGpatDcGa6jDQvs5j+979a3tpDc1zXh+LShby+VM8Tn5XROmfWt6MtBcGGJo7pMAlV++op3EWcU5TUS3EckhjAMbgfck4NS7COtMBgighcz+SHlPViMk1jav4fi1LSJ7vUDIZo1LphsYNbtPxvheM/dYYNS0M800Cxi1LS5oJ4guoW7rPE54JXNerQMHtIH/vIDXK6JZ4tZyflmhfYjDqore095Ps/lSDBjOKdgMu10Jop9Skjjz9qYKOwAySea6lY1jEap0VQB+VV14qdX6UWBEzxRzAeYocD+FhkH8KzNQ0tb+4hMkAKx8oBwFNaStmpEpWGRRw+SHEe5mY5wTVNyJLV44U3vnYS0mCvrin399LDKscUCPg7mbocfWsDxHf2OnPZ3dunPm4m2yYIBHU5osIvWc9xpPlpcXAvoXbapLHKH19D1rSsLv7d50wx5IchZAMDFcxpk1hqmmxWLXcNrcMxeDbMrt+OD1rrNLRtN06zs8LgkiRiOAaEAy4CRThnfbngAjIJqkJ4H3v9pRgpw23sa2NT01byDyWmUbuynk/SublS007UNO0aE/M7EuT9O9IC491JHAyywSrCwI86NgcA9DWRPcahZeHLmR3SztImDE7jvce1ZesatcQ3UELTRWz2975flZ4CgZ6dwQar+Mtdt9W0/wCy2tyQ5YBU2EK7Hjqen/1qYFC3ure+t7vUry+jvXMbJGJBl1Hpiua0S50iDUopdSjZ7UghlZeKrS3lzot0sURimmUZbI4X6GoNQktjNbzWtu5iIJZeev1qJAdjqkejWMsEtpayCSdS0M3nhge/IPbmqNx4x1ZIIljH2eRV2Fo5BsZfQp0rL1WyntdN0+W6lTbJGTEvoOOM1nyxTQbRNGwVxkA9xWYHokl6bfT0kSOyZTEJFlcklXPUAdBXP63rP2/T7O4ilMM2Cs3lnBVvX8cVzgngVVUxv5anJz61buNNiaCKW3uv3Z+6rDpmjmLMXU7a4IjeZtxGcepHFZnnPCD5bEbhtI7EV1MVtLfaW0Tyneh+YYyyD1FZc+iNZeVdrOJYDnDbdpDAelXGYmhbfTLuGOOW2uAmMN5g6AV0EthJcLHcx36KkxwydA3r+Nc7BqE89m8G5E2vnPTNadyYLd7OK0vvtFuxBaFxgo5/DkUNXEaCajNb6dqVrp7xqhAMpYZyPY1gNLHcyR5XLKc5HTFRsjG8mDllUkjIPertndxpolxp0karjLxuOuPSlYaHzqy2Es0QB2HBBHaozewSabFC0Rkl53kdAP61WjVnuCS7JFs27R0b1zT45TDatbzIRCxycc1I2MSZod237pGAKTG6KK5KOqD7jZxk1JfC0gR4w7xzKQQjDKkexqC3mikinUziIA5CEcH6elUIt3Y+0W6Ao+/h3hQ/KPeqU4WJleNcW7YZfUH0NWUupDbiX7QVk27BgdB/Wst1k2lRIWQ+tUBotc/a3EtxCFjUbeD+VWLe3hOnT3hWcmAqAhbghj19ulZKtuwHG5cAYrWEiPoFzHFqHlyHG62boVByADQwIobiIyhJGck/dbrtPrXRWMMTS3MbvtleMNHsGARWJbWdtqMUimVoNgUF9uQPWluhHbXCR291JLsUfvDwPw9qyaA27TSppdQRbYmaGILLcSdAoz6+9ey2zaLY/ZVWOX7QELpH5ZL5bkk9ufrXksWrbfMkt5lLSWwjuYCudwA7H171peENU1aHxHb20dzIIp4idrHIX3564qo+6I9SuI1lJz37dxUAg2cCn6ZcLqVobsjbKXZJFzxlTjI+tXDFW6lcCkUao5Cyg4GcVeZOaidKdyTMFyw65B+lPFw1TyQKwxiqwhKsRzVXAa8xcFSMe1UJrYYLYrQaHHzU6PAbkD8auMrCepzLFVbLCniMXSlYo2Zh6V0F1pdrecumG9RToNPgtkAQHPrWntTKxyTI0bFWGCO1IGzW7rllvY3KfeP3vc1gDg1tGVyJE6ufWrtvcgHY3foazecZpQapoEzSuI2D5wSpFQ7MGiC7woR2xxjNT5BqAEjbDAVb34qBVUHNPkGRUMtMtxTgDGanS4weTWYqMBmpUWUsAATWbRSZsLMMA5p6zCqYUqnNKrkdazsWmX1nAOQacbjPeqYIZeKjaTa2KLBc0fOyOtN3A1VRyalU5pWGSNUTjipG6UwkAc0ARdKXdigslGVPGaoQm8etNaQZ60wriq5yGxRYZa8wetAcVWz71IuDjNNoknB5qQHNRHikDYqCixgYqM8Gm7zSjJPNIBaQ04Cn7BikMrkc0A0rjmmikA+lUc02pFxikAhNLtzTCfmqQdKAEC4HSmsmO1SqQaDgigZGRxTVHNSHpTV60CJQozUiqM0xTzUoNAEwA21HIAFpwao5HpgVW602nHrTaYhQOaftpgFSDrTAciEc1Mpx1qME4pwoAkaUBelIpyM0CnCgCRRxRQDSikMMUopaAaAHZ4qJ1JOakFDD5aBEXUYpojOckU8DBqTqKAITH7UeWKmxTTQBVeEelVJo2A6VpEVSuSVPFaIRnurkYNU5lJ6Vdkfg4qo4Oc1pFkMi8yRFwDUZmY9TUjdKruBnitCWxWkJ70wk00nmgGqsTcCaZinmgDimFyLvU8bAYFREc0A4NDFcuoN+PmAq5bBA20OPfmscnIoVtoqXEpM6YuvZgRUclzsU4rIjujjGeKeZyR1qOUdy19uYnGDUqsXwxJqjF8z1pKo2DFJopD4mYN1NSHLkZJGD2qqWKnipInJOTUWHcuKwyAKsqvFUlPOaso5qGUi0p7VLVPzOetP8w+tQUTEikLACq5kJPWk3E0WEStJihJMmoTmo2YqeKdgLrSccGq0svJJqIzH1qJnJosAjnNROuRzTy1RySqoOTVpEMrvErDFReQi8jOac0wzim781oiWhDxTVPNKTmm9DQBaU8CnVCjHFSA1DRSAnBpNxoPWkbpQMN9G+oieaN1FmK5JmkzTQwxQTRYGyTORSjrUe6mvJinYRla7pmXF7APnU5Iq/YXS3dlG4HzAYNOkfzIXQ/wAQxWRo032a8mgzhCcAVVgNeX0qPYzD5RmrW1SeamjiU0yWjOMMg6ionRu4raMS46VXktwT0o5g5TMCGmlD2FXzbnPApRAFHI5q+YOUWyhBIPHHrWzHsVBiseImM5qY3vO0ce9RJXGtCxfkNCcHmudc5c1pPMWfBbNMZEbGVGaI6CepnU3ca2fske0YWj7JGGyVAxVc6J5TKEUzchCRUken3Ug3GPYvua2olCt8tWFiZ2pOp2K5Dnhp13gt5J2g9c1Lb2cxlAMZJ7106KduKd5K1HtGPkKMETRjDDBqVnxUzxgDiq7Vk2VYesrZq0vzCs7fzU8crFcZpMaQ65MYQl6wrq9tYCS/AHer1/cQbzGUfPdq5nW7a38lp4i7HuuM4pcw7EWpXct6u20UqnqawZnlDeVPKPcCrNnLO6BY2wT09KoXtqpuygmdnP4VlzBYtFLOOERiQFj6Vr6PezWvFtH5nOdzcYrnksJIJfnTP41NFa30kgWNiiHpihSGd9f3kU2mybsSOBkKBnFefG/nkVlLYYHBFb8kN/YabhTnd1JHNYMIgQMZR8zHOaq4F/Rr9LaUx78O5zirmsSztCSjDnqAc1z73FtBIGDVo2+pWLJkt+FFwKMbyhcNkfWn3UckEBYuCT6VZlubO4bcjgD3rMvLvzN+FIXsT3rNq4GbNndyc0qKoPHJ9KjY7jmrNl5Rk2Sg/hUjJQsnl7wuMd6heVmPXNaNwbeBCACfbNZbzLuJCAUgsMdzgjBFQruz2x9adNOzZUH5ar7ip4qwLiuRwBk0xww5IxUAd8j5iKeVkkICsXPeqAN9Kpz9Ka0ZQ4brQD2pAenXWpa6Iibeytx/tZzTLXU/EH3riyjlz0ReP1zSNq2rRRMZ9LAX/pg+4fyrLi8bxm4kim8yGOP725a7DM15r7xLNn7PpcMPpufOKxtWn8V2VqLq4ukhXO0qiggGsfX/AIlzMrWukRsg/wCe0g5/LFcVJquq3bBrq9nmwc4LcVLYHoF1L4rl0smS5s8H+NgQy/Q15xJdape3LWrqbuUHG7+9WtN4h1G7tWtmu/KjC8j1xWFY6jc6dKZLaTaxGDkZBqLjL2j6a51BjcRbfKzxnkVflvLd7hwBnHes6HVrmR5N3LSdSo6VSkkaOTC1m9S0aMhiuHLNggHiqt24wEBJWkMyqVJABxyO1RMN8u9TlO4NSUyNgdtNjZ4ZN1TLi4uo7eMZ3HBq5q1jDYQxpFw561aJsZpmJbMZO71AzU0xMigOfn71tiGODSI2jVQ2M881Sms18pZnBwf40OQalBYyHVozkcVaaNrmybA5Q11uhxWsllcghMp91nHY1kQxQtFKse3czfMVOQaYFeJkvNJIeVUaMgEGoDb2kdpFKl4TPuAYAdqksb6a1WS2eKI25b5mZelRuqLqUayRskW4cH0oEdVrOg6VdWcEmkSie8xkqCTuHpWDPqKiWCI2RheM4YAY5rXNxPpFwb6yKQLCv4NXNXOpXN/fNdzMN7PuOBiqaGS37sl0VQ4GM4rSklF1FbSD7yjaaxHMl1fxr96RyFA9TXS6ho2tadpTO1guE/jRwcfUVm4gVzH5uAiBZu+KRXmtJc3OSo+5gdDUJvZLa1ikeNGZ+CytyD/Wolu5rl1WJQxBzgilYBby2nawnuJFIfOQSKZH5jWKeYE3Y/h7irepa5M/kLtTg4ZcVZu7S3sJ1v40zFIPuZwM0WAzGttLe9gWSZoYpPvlRnB9a6ObTfDen2X+o85P7/mHJrk5I1klZwWTcc4U8Cr2oYTToUVZBzjLjrVJgQ2ekDU7meW1TyLNTne/QVSnVYJ2iWQPtxyK09PTVNOsxd28itbzkGSLuee9GvXf290dbRIWQc7T96ncCvcw7IIPnBBGCBUa2j71OwOu4ZVW5IqZIjNYw49STWi9otxpkM0blZYyRgd6m4Euy/0iRvsUV0bQoC+1/u57giqUN5c298LnTISm5dpYx5De5zW34V1d4Fms7sRrCx3eZM2FA5yDx3qnrMenxNu0/UCIs48gdvp7VdwKkWtaxpN5Jc7xH5/38KMH6V3fg66slgN7YN597J/rFuJRvxXlt0xMh/evInbecmo7e6hV1wJ4pR0kRitAHucviGy1C6ayRZBLC2JT5eQp9ARV2RpbdXNikk4HUHDAivOvhxrNnpmqXpu9RijSZV+WXkucnPNemulnewG60i9BVRlvJbIA/p3pgV11VWfZNaTWznp5vAP41aEkn9z9ap2mtabqUf2SW7hldflYNwc/SrOoTQaRbi5mlVLdThgwyT9KAHpuTzCIvmkOWOepqOUaj5R+xmKJ++/ncKyYvG2kTviGK+lX+9Hblh/OtODxDpcjhfMnVj0DwEUwIrXXZ4s/b0i8rO3z4Wyqkdm9K0l1Ldb/AGxDFJaKuWZTk4+lUHtIrXXUeNB9mvlIeMjI3Dv+tN0W2XTtYvrAFjbOokjB7DmgCxL4lW3WK4Fsbixfhri358s/7Q7V0EciTRLJE25HAKn1FckNPj0XXoWtmK2OoOY3i7JJ2I9utdbGJVXbIFGPSoYzM1XS9QntJnsEgkuGwqrJJtAHc15f4gtZdNvRDrMUKSsD/qZ/N8wentXrGoaw2nx5tmgaUdRKcKB7mvMvEut6NqN7M3kQyXrY/wBIVt0akemeakDkLb7WL2WSysfKjkAGxjtK/T3r0jS/Hlhp6wwxuXhZMSwzEkxOPQ9T1rz5biSFrkT3ibzhlI5DD2NabQJrjfZ0SENGAWliiJ59CRQmB19942juZraYNb3CRHcRCxR0HpipbHWdO1jXDrMsbRLEc+ZKQADjFcha31vY3V1C+nR3LSKFViudnHX/AOvVfxLZ2dhFBJFFtecZK54YfSncBPEeuWmo6+95feVdOH2r9kO1WVcYJNZWo6ut6w8q1SG3Ufuoi2Qvv061DFDp7rujiMfy7TngA1nyWt1G0W+3mRJGARyhAceoz1pJgNTbJdbp3b/ezV6y1aW1EjfeC/LHzjim6zYW9rdxxRvsDAZLHOTVZoIYr02zSOoHQquaTEWb6e6vbK2kmOURsBO45p5uWvLZpAso2Y8sZzxVW2ulR5laX9yBjkcGlYFMfYrjCHqE6ZqWgGrPIGHAbnHNTyTRRssfJTOcCqqIyy8ttQ/eNEkm+UJESzA/KwHalYZox3LW7BbaV0dyAdpxxT/3Jl2XDyN/dI9apzSxXSxtuUSgYfAwM05IkNvIHYGX7ynPGPf0pJWGMmit7JZEnhMksnTBA4qtEHmfOflj+YCpJfs32VpDFM5xhecbT61VjY7Sw4HatEBZmjumcsUy7EkDPJH0oe1u47dLh0X7OwzvDdKmgnW0tBdgI1wsnyP1Kj0+lUzcymzniaJSk8nmZDEc0Ej45nGUz9z5vc1NJdRz2jDOHP8ADTLNriDUoJWZdzYUhTklTxitKeKK8uZ1ZPlUAptOM+tJoq5j3jSXEsRPJ2AE1RZWDAkfKTwa17l7GG5Yi1EiL90E9aSPTYNU2tbuLfP8OCy/nVcwFSUYjBB6kACopE2MUJ+cdqvXehX9rh8efCBjzI+cfUdqoxRLKTl/Lb+HeCAT6Vd7gT282SkH3WY4DEcE1eggxHNA6gv1wex7iqtzF9laG2lX51xISDyPapPtMlzetKWA3/LtC4A9/rUMC3aRK7ktnYv3gDirsys0Ko0SJEufLcjOB71ngFGZkXPODzjmppZZ0SMMpAlGVPY/SsuYDYs7KNrSJPILyy9MHrW/4Zgdb9pI08+WwX93tfBBPUn1+lcvp+oBY4lec2+ThHKbsepxWpBpF7aXSpFeo8VycrPCxG4fTtU3A76Hx5bWE9taiwESTTBbmVzyGPcCui1jWUsCo8xY/Nx5SsQCw7n2FeR6vfRXlxLHJA+4IAr92IqpcXlzqLxPeuZYVj8uFicFAPStVIR7RaahusnnuNq7VLYDZJH0qe1nW7RiBgqcMM9D1ryKyuDaQRhLiW7eSM+VE3ymPP3u/wBa7DwbffbVnmt2kZgFSWIjJb0IP4datSCx2JUHtUTxjfmrOMjkEHuD2qNl5qybFdkBHSqzRkOcVobaYyZNUmMzyue5FDysqgVLNA/VTVGXerYOcetWmSywAJ4NrjIrCvrJUfdGvfmtqNjj2qtLG0kpGMr3rSLJM9oEaA4Xn1qmtrI77QOTW48CjAA470kUYE2FTpVc5PKZn9muBy6k+gqHypkche1dELVZYzgdakh09UXDCjnDlMaBJXRTgsCeCK2I7OJ8FhnFWFt41GAOB2pQArZHSs3K5aQhs43fuB6VJHZRqeM0nmYpVulJwDUXKsOkhUDGKz52Vcrnmr28u3Jqhe2ZaQGIkEnmmiWVluHjOM1FNdPvzmpZYhDIN4qC5iWVMw8n0rfQQsN8Q3Jq6t+n96ufbKuQeoo34p8iI52dQl8rnHIp7uHXiubjuJAMZq5DfMpAJqHTKUi46/MTT0YIOtICJF460zY8bZI4qSiZ3cruUZqsJt7kEEU37QT8tJtHWiwh0jAEc05ZxgAmq7EFTTA1VYVzSWXKg0oeqiOdmOKcrMDzUNDuXARUoIqrHIG71KGqGiyQv89Sgsar/wAWatxgFRUDIHFR1ZdOagZTnpQAlPjpgU08cCgBrdaA+KVuaYVpWAlVhUgOargYp6mlYCQ00DmnA04UAAODTw1MxRyKAJN1MZqQmmGmAh602nAUuKYCCnrSAU4DBqgJF6U6mg08CkAm6jfSEACoi+DTEWPM96XzPequ80oYmiwFkNk07JqKPgjNWCBtyKQCrJgjmn7gRVMsQ3NKrMTwaLAWgRT81XBNPyRRYCQ0hqMv70m8GkA481n3Z5NXc1BPGrdqpAYzDJoxxV37Mo6imPGoyMVqmQ0Zco5NV2XJrRaIbuRxUTRJk8c1opENFLyuM5H0qNhg4qeVcHioCM1ZImaUMM0wmkB5piJiQelQlSDUsUgUnIzRIwYjjFFgIhRmlNNzTAMmpUfjrUWaM0AW4pCG4NXkugFGTWQCe1PDt61DQ0zV85WPBqzCQRkVjRyEGrtvcEsFqGikzUUnNSB8DFQK4I4NG8ismjS5aVsmng1TSYg81J5wHepsO5ZyKVSKp/aVHelaYsuFNFguWJZ40HJqv54foeKgEAL7myTUyxZGQMUxDWJzSE8VOIsDkVC8ZJ9KAK0zMVwvWqjJPtyQMfWr5tyx+9inCBQMEk1RJjguzYGTVu3tmcbnbH+zitDyUA4QCqs0vlkqo59apO4DZIdo+UVBtJPANPFw560rSNQIYOOtOyPWojk0mGPSnYLk3mKvU1G1yvY1GYz3qNlAquULk3mlqaZBUJY1LDCzsDihoVyRMnoDUpVscirkVvtTgUr2xYYyBWbZRQORUTZNX2snwSGBqqEOcVSYmRqp71iXaNa6rFIBhd3NdEIuazfEEDfZI5V+8rVXMI0VYEAjvVmI1BpUa3GnRSAhmxzWgsIUdKzciiMnioz1qw6gCq7UhijGMCoZMg0/pTH+amIrSSD7oqMVOIV35bmmSIAeBVokVIxnNSxrl6gjtpJTwxC+tXbe0MPRifrUsETBQFqMqWbAGatLFnrUyRBcGs2y7FeG2I+Yqc1cXgU7cApzUDSDtU3KJvM9KPMqvv5oMnFK4EryZPtVdyM5pN2aOooAiOKQyiNSScCpRHmiSCPYfOJC+1IDnLzUbQStJ5v4YpD4jtLi1e3t7N5XYY+78tUtZGloxit1ZnPf0rmZJZIQYY3KqPSobGTx3r2108Z2QqrHNU7i5U3RlV95/vYxVGdiXbv7nrUIbC1LA2rTUyLhfNi88Doh6ZrVfxeyyhBZIPfbUOn28Fhp4uZVAk755rFurq3kdposCQ+gpoD0mwnkmRBMikN2IrL8VadZNCZUURuDklRxU/h65tEslmuLsFz2JH+NR61ew6iphh5B9+tMDz2KFry7EUJLL/eJrW/siS2ba4L+4FbEdhJaWplEEceO5NY6a5dJcMQ4de4IrKQDZi0bYWB+O9ULmWR48OpGOgIrRuNZuZnDeUigfjTXlS9AeeRFx0GMVKYHPksp9KlgZy2RViawQyHbOGA7gVEI/JO0Pu98VV0UEsr9CaiLbhT5lcjJqHIHWhMkcVG0cjrTcqo4ANMJUnpSdelNIof5Rk5BpMmP5QxH0pMN6kUv1pgJuJ60E+nWjP5Um5c9aYEt54p110ZZJfs8bdkNY1pqF1b3W62bc8nHz8jPrVFJWkcLNISncmoZZTG58luB901vzEGnfR38dw0ksSfN1IGAPwqC01VbTzC8PmO4wD6VOwuJLFWkkJJ9axpY9rkE0uYCSW4EpJAxk9KjyMVHnBxTiCFBpNDHxSFH3DPHtTg+6TOCWpkdw0asoPymptPEr3WYwuR2NIbI5Zct0IYUkEjBZBnkjii5803MhlChs/w1EOKLAmaWiIHuGkI+ZelRXt3Jd3jhzwpxTbC4NrLvAyCMYpLjy2keVBjcc4pWGajt58Eew5CdRU/9opNL9nMe22P90VkW0rR4YHrUsa75Dg9aLAHnRoZIVYvHnAOcUyIsVWJAB8/BHY1XuEaKbDEH3FWYY3heN3IUdeelKxJdup5rHTntPKV95zvByKk1q4iuNM094pA5WMA47Gs69vJbi4ZnZSvYrTVtJzbNOEzGPfk0AbV9q9m/hW2swd84HznuBmufjceWtRMQfpQDjFO4yxK0kMgkX5XU5U+hr2fQJr7U/DUczahDMxBEjFMlPb3xXiksrScn0wK04JZrKDzLLVGhWVfnRWwDQwJ/EP2a2uvs1rP9oQMT5o4B59Khs51t7YSq/wA574rPAi3ZlZvdt1Pk8sgx27FyPWiwD9r3WponXfIOK6nxEIreGytlbKAYz3PSsDT4GF/aM5ZcyDJBq/r6L/agRXkfoxL9M+1JoCnImxiB2qKS4neMNIWkUdianiZXvYzIcRscGodQt/sV2YwXKEAqd1KwE1pqEkuIDlYR0QNzn61fnia6tV8td0oyMnqR6VR0vTUuw8ol8tkIH1pA3neYgPC0gNOzs7a1t4v7TkePJ+4h5A96n1NYLOaKaB820seU+bP+etZ9nZSoyzQwNMewbkVPd232u3EghEMh/wCWfTFQBBBewRCWKWDzFmXbk9BWjc+Hl8iHdqMMSSAAL5RBz9axZhLZPCZoPlySFbo1aM9ld69YTanFdwKlu3Fsxwy/SrSAxGsWiuGSY/IpILqciqK/NMY4wzH2FbLQS2dxb/aJAVlGSRTbTyYmYWjgXEj4DNwCKtAUbNYpLlYpuEbqR1FdhomNOi1C606aWN7UBuWyJAezLWBHoly9/CSUVGbazoMhc9zXQWEul+HfEt5Z3lz5kMsQCTtyAcH/ABpgSXXizR9UuITc6WEmchZChwSfUEVp+HLnT4b2ew1LUZ7Uxv8AuEufmjI9cHvXAWtmt1rDRqvmqJDjtkZrX128t73ZE1sRNGu0yDqR2o5gOo1XW9W0PUpTbXSxoMeV5SjZMvr7132na7Y6ppqfaZEgd1w6txz6+9eVX2lRR+F7LW4DKNpWKW2fkA+ora8LXq39sbDUEjEgbzICx+8CeQSO+KnmA6m+e/sbWGS0dJFhmGSw3AA9CP0qbSvENpdap9mvYvsmoRqVZWJwf8Pxrl7vTLtPEc8WiyNEioJPLLEo+Odtalp4k0q91y1eeIF7lPIulZMhXByOfxq1qB272yybRIgYKwYfX1qyAaSGD7Ogj37gOh9qnUA9akRy3imzuFsLiWHU5LcSjb5UaLhm6DJPOOlcDa6DD4US1vNWkiYTZWRHTOz0217NPawzxESRo+Adu5c4PrXn3ibwxDplpHqMoeZIw5kDvuVjjgYPSlYaZ5tdJHqbX1zZ2qR2qN8u4nJH49Kt6Brb6VoNxBaTGJ2ky7qu5seme1QC5g/sp2SMLFcOwZTyE+lJ4d1K10syfabead5iCgiUEnH1pWGdRpbSQRWc88UHnX0TKqy5Akwehx0rnfEtpPp2pul45Xy8N5DNk4PXHHNbniTxXYXdgGfS7iK7iGLZ5GACn19+1KRrXxJ0Gwcw2yi2kKyTcLk96RMTCtreO3VdZubeJrcNn7MzlQ+Dwa0dX8QL4g2W7WHkO23yQ82VQf7OBxVnU4LydjDpsQurOJAqrGm8Fx2wenOa5O1V9K121utUikg8tjujZDlfw7UWKHXb+fF5U1rhnbaszEkcHtU1m39nSm7gmSbykwVddwq0y2WrJdXdpbSF4MyKbhuGHsOwrGlvHmSRQApcYwo4oAfe3DXl5JctDFG0mMiP7vA7VFDIoZ1fsMjFMjikTS2m2g+WcYPetDwxajXr8wSgqlvGZcouSSOgI70rAZghebLB9yv91VYAk+9RxiW1mMiD97ETXq8vgTTEeDVdZktopbpfmgQlRn/ZPavNbuJNP1K4aKNhAkjINzkk+9FgGXM1tPKl0gMbucSIegNNW6gEThisLEYEmMgfWonNrLKiRBlD4xkdDVa7tJrOXyZXUyeg7VVgLUE0cF6Yppd9swwxxjNT3L20kUCW1rvijbl2Pc/zqlbxafc6ikNzK8URGAyno3uagb9xf7Ipd6Ry5BzkMOKqxJeurWaKyF01qyW8rFYyO5FRNsnELp50iAEGNOxrZPiWd57V5TH9ps598cO0FJAeufwH61a1XX9JurCa6t4BZ6u1wshiVSox9BxSSGYcLrY6ilysbMFXlX96WZt7OVJ2SfMATnHtTXuLi9tRJO+5+2RVS6AjlCrwpGQKTQ0XZxBsEVvGA4GNzck0smq3MkcjKY4tmAY1GOB3p1rPE+mzPKw82JgASv8ADjrUJheS3E5Rdp5LD09aVgNDTLmS7aQ25ZZFA4Y4Eg9Krays0UqLczOwkQOEdQCh6dqZYxyx3y+W4R0+ZSehH1qza3dpL4ksJ9bkLaeJP3ynOduO36U0A2x0S51XSjdQLuMcxi3O3AAGcfkD+VMihhKhkIH1Ndh4hjtvCsYsdDuvO0zVoPtAMp+52yD7g9K4gIq3KrA7zRZwrKOST6UMkmuJiibRt59DmtS4Vbnw9ZRRyJ5kRLj5s5B61S1ErIyQrbSJJDlZNwBJPFX4ks7Xwc0olje7mnUMEbJRc/xDt0/WpcSkVF+3qgVQzgdMgHFTJfSpkiB0GMSM3BUVSW8bS2jmtpEkeQfOp5CE9x6VPPdxXlsbZjumfo/QFvWlyjNCx1CO2t7mHH2lCV+8eQe496pm/VDJGYnSCaTzIc/wjjg/nSSafJHKnl3G10iDNhMnPpS2WlS6pYSi1lhaWNtotmcK7d+M0uUDoNNezniXzJ2m8q4AXyz86r14rsNKv2l1nzrONPLtX+bYMGVG4+77f0rkfB2nwafHHfXkht7iC4EexUyyMfVe/tjrXqdro+/U1vInC2/2cLuVQCzFs8j6VSiLmNGzkaRGLZBznB7VNt5qRYVj3bf4mLGgrWiEmIFBpjR45p+CKRiSOaYEW0d6rXNuHGOgPpVs5pNu7rVXAxzE0akYJqMpL/Aprb+ze1I0OO1VzEtHOyPIpw4PXtVuGM7eRzV11+bFIEAo5hWFiAjTjvTt2acIc1KsIAxSuMqbWL5B4pWVguauCJQOlRtHkEdqLjMe4nlVgqLnJ59qW3Ry4L8VLPEBKD0xVaW7EUm0HHvWgi+eKkVg1Y7X7FsKcmpYL0seTzRysVyfUbdpImZeorPWF4YmVuhGM1qGfPGaUorjkZppiZy6wyGXaeferqQ2rKFb/WjqK1xaRA5C0gtIQxJXOavnJ5TNNsrfdFKuls3IdxW7b20QjHFWooox/DU85XKZ1jYmIZOT9aumJCCCoIq0VA6Cq8jhM1ncqxhXVn5cvyDAqPeqKBn5qt3NyrT4Jqo6KzZrSJJWLHdgnApqyYfBqZ4TjNV2TDcVpdEMtLIARk1KHQg1TRc4U1KqEHaQaTQ0yeBVXhT3q2rVUhRhzirCoxfNZNFpkynmp42wcVXCkVKg5BqGUXFAK801kGelIGwKjMvzYzzU2AcVFN2ik35p/G2lYZEQKaRTZJCr4FKHyKdhC0opp6ZqMvRYZYBpwNVg/vUgf3pNAWVwTTmxioFfinF8ilYBCabmkJpuadgHr1qTFRKRmpVIOKLCFC04LzS0m4UwHhRmkc7acHBpj880xsiLnNN6018hqeo45qiQGM4qZE9uPWosDNWAQE60mAuwipFOE5qAzHBGaYsme9IBZGG6nI+BTNoY5NSIgzQMjad84BIxUiznHPWnGJcdKb5QFAChsml5pAuKeCMUAR5YGgHNDMM0mRQIR1yOKgZD3FWM80FcimmBnyDAqnIPmNarQlj0qu9t8x4q4smxkyDOarOMVrzWmFLVlyr81bRkQ0QsMioqn2MegzUbLg9KskBSmm4pKZIUlLRQDEpaUDmgg5pAA4pw5oVCxAFWBAE6nNFxjFBxT1cocig4FSxqrLkipbGie1nZnBIIHvVlpgTVCRig+TgVNAysQTUNXLTLJLHkDikcMFyaeZVHFJ5gapsMrb2zU0Uh4FPEUZJOKUIoPFDQFqJsgVMpqtEQOKsxglulZsokAyKhkHJqzjC81EwyalAylLIY+lQ+exp91VEuRWyJZaa52jrVdm81smogf7xpVkUUySZUWpPKDCoY23GrCrSZQsVpGRliT+NOeCNB8owKerAd6RyGpAZ7nkioWUk8VoGFSc4o8kZ4Wq5hFOKBmcACtWG1aJAWFT2kaxKGA+YVNK5frUN3HYhBC0yRi3ApzcCowwDYxmpKFTOMGozbfPuz+FWQBTgBTuBV8gVBqFj9psJF/ujNaW0Uvl7kZD0YYNK4GB4PP+hzQH7yHJ/Wt01y2jO1jr89uxxk4rrClTe4yvIpZeKjEJxz1q5tFNbAFVcTRUMBPUVFIoTOKtMx7VVcMXOelNMVivGkkkw5wtW2tg3BHFEaEMMLirR6U2wsRBQo2gYAqSNMkU09aejY6VDZViwYcDOaYzYXFO875cYqFiSam4DWkPeoy9K4+XJquzHNIRNuBppbFQ7zTS5pjJfMwaer571VzmnruzQIuK1VNRL+S8wf7gztxwalDYqle6xZWIxdNgfzNIZx114l1DlDbxRj1KZqpcOBbK5gWQt1wOa0da1KxvIN0NvkDvnFQaddSJGGa3QIOh9agDm5ZBI/l+R5QqJEidwol/Ejity/a4uLppAkYB6AiqUlpNcDBgxik0MgLQv+6ad5F9O1QvHEDhAAtNLBZdm0KPWonZt2AflosUSrgHKZ3dsGpoby9hkBjkdSPbOKq4dU3BSMdyKfFN8/zk59qhgarXt3Km+5l8w44yaz1wCxHUmnSujkGNMD60xRliMj8agQ1zk8mmFMnO4D60SnDdR+FM34p2GSn5l2qwFMGEJGdx9qjD5oLtjinyoQpYucHpULqAadtyC2T+FCIucvuI9M1XKgI1jLnjFDrsOKlKLn5cgfWk8te+aYEWTSn7tOdVUcE/SoySRQMQ03OKWmnkUxEmh+GdSvWuI47SMpnaZJDjB56Vd1D4eXlnYNJblXZeo3ZOK6S/uv7C8RM9zv+yAkoy8YNcvqnje6+33BslfypPwA/StCSLw5e2l3aS6ZfLGjYwjHjn+lcvqVn9iujGZfMbv7VXlLNK7ldu45x6UzFIAAXvxSY3SKuTtJxUtsqG7QSkBO+asiGNtUAyFhVuvrTQxLvTFtPLzIW3+2MVC8BhUskyg/WtS/K6m8dtao0jj0FUHiezaSGe3OSMcjpVkkAhnliM2wuo6kVJbWiSIzXKSL/dwK1bVYjoki26szsMMoHStXw/4bu3QTXF28MWMjod1HKHMcc+1HKqcqKeVT7OZPMIYdFHetDW3hmumhtQj+W33olwGrJdT5ZB4I6ik4jUjSk+xrZQeRITKfvAnODVWcyJIFyRkdcUEiONf3e3cMhu1QtM8nDtkVLKLDYk+Y9afNcGTCsMYGOKgR8NtPSnsB61IWI2QcbCfxqYY+zBC7D1GeKaBilMbEHfG6kdiKYWKxG0bab0qQjjNR0AG7nNSKqtyRzUVTjCoMdaQCsT0xk9hWwu97Xa6Ism3sMVmqkaqJHPzLzil+0SXcpz1PQUMDR0+ZoFt5s52N37V0OqpFeaKLuJRvRhuwOxrkS0iYRBnHJ9q3EjebS1nhnDW54KkYOaiwEujJaXcJKxf6Qgy0bDgmkv7e4vlK+SSg+6V6g1lw6o1nfxSBQqqcNjuDV7xFd3FtLGbWQpBOu5WH8RpAZgP2ZmB5k9au6YIXeYzPgKc4xy1QX1jKjwAuu9ky+PWoVVk3FRkr3pgdRp2ri4uhZWpCRgEneOfwqWefF4beRsuO1cuJjCDcRMUlHcGrV5cfZ7qGYSeY/wB5ju61m0Br6hO0bQmRQ0anOD2Nc5DdtHLdRyoxW4GBjsa1vEl15j2O18QyJvIx34rKjuIyrgxlmY5DDqtXFgWraexltbn7WrPMylRuGcYqnbxN5AB4yQKBCjM0aD94RlakR5QgWMorKfmD9c+lWmAyW8voy1sjzIQcMqjG4VDDFEbiMXsskaZ+YnkqPWpDe/aLsSyLh8YJUdainlae8bb8zdhTuBv3Vjc+Hoo9TsLi2ntJsKpxkgevNZ1/a3Nr5ck6uFnG9N3pUR1ae600abNbRsVbcjHqpq1d6mbnTbWzuQS1vkK3U4qQOrh1SzufCkmkmZN7Y8nJ6H3NVLixe1+wPJF5TxpvYMfvkf8A66zpp7a5lsJbuxmitosRzMnUntXQSalY/aLawaZJ4onBE7npH/dP+NMD0fQLeEw/awm15JPMAzyARxXMajZQ2PiG6sbdATdSxyIrDhTnJOa2JbnUbC0thYPDGhYhYpRkMOoAbtVvSNQttXmeeW3FtqkRCSQP1B9qpMDoHYZAB6DFORjmq287vm609HHJJwAMk0AXQx6VneILK01HSZILy7+zQn7zeo+lVv7ftVuI1WTzonYJ5kfOD6Edq05LWB2M8kQd1GQTSA8U8S2dil5FFpWopeW85YSQxxhAuB1AHr/SuLWzSN90MsiBeVIbvW7foy67dz6m8yvJIwRkAAAz6Csa5nSO5ljWBHjcYUtngUhl3Xbyzu4bOG1u2uHjBMofgEmn6F4k1Tw2twunusaXGN+4bhkcDHpTNPeGW3jhktopLgDEW043D3qpOlzYyPbXMQaEtuyvOfoe1KwjSsnu7+3kdFvIruPc/mREgS5OTwP881l67dST3qMTdbFQI32k5bI963fCmosNXjgaSeK3ZWjhDHAUkdc96TxNDAmsmW5cTy8KXjOVl9+OjCqJMCC982J0f5MRlQVJGTU9m91HZBoYldx95m6gU+00GXUbDUb6zSRIbJdzI46+oBqpo01pPeiC9Wd9+FjEPXNJopM6/R4LCd1stTvS1vLbGVZCfL8pvY/p7Va0yHTvDlzaaxZaVcNZTwtBcpKxYhePnDcd6yde1aOwvbY24QvZrs8qVchx9KINeudaigt9WmaKzbO2KBQuT2BI6/lTsM3/ABdaWVg1tcxS3N3p90AyQGTeYxjllPbFcrc67by2U1hJZmewI+WYgLID6k1QudXOl3ckGl3szJwFJOBj0IrOF3O4eKTbslYsQB1NFgEuxHFcIvmiaFG6jg49DUYZ2uDsD7Tzlu61ObltwiOwY77cmqqusbuZPmym0dsH1ouAx1XzCFbcPXHWp7e0Z7iBAf8AWnaVRvmxVi50pIIt8d3HPgZYI2CBVf7ZItmLdRwrbkkz8wNVcQk6xxXrwRoYwpxtPUU3Zul29W7nuae6PNaNqDTGScybGBA9Mg00mERwSK8nn7v3h6DHtSA047O9lsEeOHESr9/cDmoRYXWqQS3EQUvAo8yMnDfgKidg8BbOWxk1o6PPHp95vDSEyRkSEdhUMDKhgZwFY7c9CemalW1uYj5ZfLZAXYeCaa0ckkcpRNyKck+grQidrvwy8m4I9tIFUnuKBhZTzWWuQwXhEsTHy3QjjBrW8RalZ3EsVrBEq/ZX8sORyRWWZo1+yy3JBS5wGbHKEcZBpLZRJfqkkiorSiOV2Gcf7Rpkk2s6i2pXqygFY0QJHGRgKB6D9apeYVRVxtI68dTUupxNFcShSG2kpx+FR3gieyiuIZP3n3fKY/MvvSYFaGaaOVpBK+WOSc1YWRpWO4bEb7208HHTIplsYZIJYTMkcxOV3nApkhEeUjZJvLIJbtRYsn8kXEqxWY8xj/D0oYLaSxNuzvXcp6VY0JpJ9TYxeRHL5bbRIdoJ9j61FfrssrWLBMoY4wPek0BYg1P7ExkgicSsMb1b+hpsWnGaD7Y5zLLJgkHnPqaI9O85Y0EiM+NwUngkdRV2ylkEpFnHGEHVSuSppAdP4Y165htXS9sbW4tIHSKdwmZz2RvevStCku45JrO4l2sB5kLYJEie4PQjvXIeF9Fhn0S9v7gOftRVGiUY2lD1/OvTBbmSZbhRnEYUfQ8mqQmiSVGSIuBkjtTS6lFKnqM1M86xQyO/3UUsfoK5yPU5mvYUhtneGRTIFyAf19KYkjbNNPWs2fVXGoW1pHGVMwOWJ446getaQUmmMTFPVcmlCU4DBoESBQBTJFwtOzTZCSuKAM+YYJNRLyatSKCOai8vHQVSYWFDEDilWY5waAjEdKPKYsMLQInXkZFNKMzcU+OMqMdaUh1PApDK7aeJjlxVebRbTqyEn3atJRIaDC5680XCxz0+jgqRBwfWqBsLiGYJtyO5zXZpbf7P60rWgP8ADWnOTymBaWIzuck46CrjRhf4SaveQqngVHLhUOKXMO1jPZcVSe4O4gdjV2Ukg1lTAxkk9KtIllpLkqOTU8d4q4JbFY5dQc7ieM0tvchiVPPNVyILnSRXCydDxUVz0JqtbSBcZq0+JFwDWYzBeKS5v3G3btxTzDJG+04OK1im3kVmXEjiU5FXFkNEqru7U7yYf4oxmoIWk3/7PeraruGTTuOwwW8OAAKmSFMY2g0oj9BT1GKm5VhpgTGdooCD0qQk4poNAWEIHpSgUUo60hjSTUEqv1WrWzNOEXHWkBno8gfDGp/O2jBqVol9Kha33Kc07iGsyOchqkjiyvWqcsTLGSOtVUupkIXNUkK5ruuFqmTyakSdmQZNVZGbceaSiFydWycVIM1QRzmrySbgM03EEx+cU0vg1MqBgM0ptVY5FTYdyHzl9acJFPeqlxaTRykp90/pQFcLzyfWnYVy2XHagSD1qi0jKeaYJWNVyi5jVE/bNDSVQiZt3JqaaTYQME59KVhoso5zzUhcY61nCZt+3BqwrGpsO5Z3CgGoMsB2pn2jBwaLCLlNLmq4nz3qRSH6GnYYFjmlRxu5NL5ZNN8s55pCJw4HepFcHoaq+Rk1MqYUCgZLvanbiaiGe9I5wvWkMeZAKb5tVjJk0qtQK5OWzQKjXJ61ICBQBKq8U8CovN9KA59aVhjj1qNzg0rMcVWkc5qkK46QBlxWXc23zbgODV7fUUnzDFXFkSRmiMqMYqCVea0GUg1TmHzmtYyIaK22mHOeATVkHAxip45kAxtFU2KxSaGRApZcA0iqScAVrK0cqbW6VXaMK2MYFHMOxWEDYB4NDQsBkirca5YVd8lCuSKnmCxRtwiDLLmms4LnPAqw0JHaonssqWycmlzXHYquwPSnxMdtL9kYd81ft4UjjwT+lNsmxUEcki7lQkUj+dCM+S/1xWqrgDFSefhCueDU8xdjDFzJuwav7SsIekNtA8u8sV9hSXcmyEKhp7iBJN54NKzsDiqUUpQ1YEuTk0coF2DezjIrWhGABWTBcqQOMVdS4xg5rJmiLrJmo3j2jNMW6zUyzKRWYGfcwFgTist4WDYxXSMARVKWBS2cVUWJoxWhbGMU+K2OckVfaH/ZojjIPSr5uwrES22DnFBWrmKYY1PapuVYpEkGnocnFWGgB6CiO3I5Ip3JsKseamWIAdKVFPTFWETjpUtlESqQKXbVjZ7UbBUgVXjyKj8vaau7BUbIDRcCtmlXLU9YiDzT1jwaLjFValBC1H0oJzQI5HW0ay8RpcqMIxBrrUbeit6jNc94wj/0aGYdVP8AhWppM6z6ZG47DFIouk1C55qQ0winckiIPSnLEvU0/FNJqrgDFV6VC8nNK7CoW5oaEKZM1LGeKrDip43GBUMZYzQDSBhSMwApDGyKSKrFDmpmk4qIuBQFyMqc0hXilMgzim7hQIciiphtVarhgKUSc0wCQk8VDeaPbXtrmSPL+uan3AmpQ2VxQM8y1O2W3mkitmkO3sK1NG08vaGWa7yyjPl54FdjFYWaPvMIY+pqaSCB4iixhQfQVLQHmd1eE3jHy2bb8oCiqsup3EWMgrnpkYr0YRWOmxSFYFbAydxyTXneqQT3l292+I1f7qk9qloZlyOjuXbqe1ITGVG0kGnNFtPQGoyvNAEoaSTCs5cD1pCmx8kUwErQWzjJqGhkhnI4XAFNjlIkDZyfU00qpGakQIBgiiwDZsNl2PzdqgzkVM4G7gZqNiqn0NADQRT8FulK0wKbQB+VEbBWGSRQA5UIHINNlhkVdzLgVPu3OqQkux7VbeG9hg3S2jFf92pAxgzCnLIW6IDUsswY7fI2+5pnmcYAH5VaAiYnJ4puPSnPkjNLGo6mnYCMqaAhqeNFeXCvyP0qd0BGF5NIZr6tqOrT2CX2t6G0iscbyxAH/AfyrntU1jTZNGMFlbGOU/eIXGD/AFrpPE/iC21aBd2sW/2cdY06t9a87uTF8whlMi5+8RitCCzpjacu6S82+accN0NUL7yZJmeBdqdhnNMC5pCjMMIMnsKAIOB1o8wYwDircWl6hJ8y25wOfm70+4lUkCVELAYOF6UyjpdAjutN0g3cMNnMzHPMuGrKk195bqRzAivIMeXngVh740cYLBB/CDxVy7ubW7VNluEZRgEGgk0bPXrjSndXs4WjZt2M4xVrUNc1O8tA4fyrQ/wrXK7iwILE/Wrq307WItSm9B7ZwKvmA6Lwpc6dp/n/ANozxq8uNu4cY5rK1qXTle4jt2E29spIvQD0rLg8iW48q5k8pPVhUl3FYIMWkpcDqSKrmGU3keRArHIAwKjXlsVIqhuCcVJbwgyfNgj0rIYwHHJp0dwyPwAfqKWQDzCAOAajYDOQKGDZaWTZGXzip7WWW73LIxYkcZqjI+YduOlLDKY8gZ5HrSAJ1McjJkHHpUBNObrTOtABuOacCaaKXNAE8su9R7DFS6Uga8MjfdUZNVM9qniuPJiKoME9TSsBakl/fSN03VCs7RZEMpAJzt7VW3szEsaF+dsRgse+KdgNBp1njwwAb1Fa+lo995NrO0ZihXgHnIrGSylMuxBv9xU0Uxs5ArLkjtnFSBpeIrkJeRQwArtTaAR1FUjvhUK4w3cVJealbX0kMsrhWT+DqaJCk2Juq9qTArTrt5ByrAGiwtvt4ljEgjK9M96tXVkxRZYRlCM47Cizs/McSKrEL96NWwaEwK0skjDZPKH8v5VI9KZbSMsw2nHuelWvJ0796FWUB+gJ5WoYYI4Mu0hfHQGncCeaxlRSzFTyeFOTTJrWJZW82STzjyMela+lQ20ixXc0oDc/uvSrtzBFdSm6tpF3xfLtPf8AGs+YDkonWCY7qvacUkvyYPLVxg/vBkYrXvdDkjij+1QoXkGU2n58DtWH5UdrdusZeN9pzu5qlIo2NOl01b2U/Zx9p3ffkHyqPpVy71mxlt2t4beMbujkck+1crC0wbjDMepIq3ZyPJdqEg+0TD+HoKLEM3EnH9mz2Nwj5lbeDjjFVdLt1aZXEImSFvniPVhVq83PFHEYQZlwwRD/AFplkY9Kk+2XKOijpGxySfY1HMM7KzvIdQdrNr6WBC/7mKVehHYelT+Jkt3trW9W5RJraVY5pEbaxB781i290uuvbpHdIkwAMQcckfXirx0zVBqwedLWeL+LaME1pGQHX6bdSR7R9qN9ZygeXP1cEDkH9K01KscFm29wDiuY8K2sunxXVtKBGjSmSJGPI9R/KugVqsDlvEVhdopkt9QNnbImIo0GS/PfHfmrWk3N7plvY3c7XLSXf7to55dxXPoff0rdeOJ3QyhDsOQG6ZrM1KGTUdV8oSOlvboHXYec9qBGB8UYbYra3SgtcfcijRcZJ6k+uK8sl85ZWWdHXacb3QqP1r0e4ih+0/aJLi8ju4zlFkXvj9elXdbW51zw3aX8N1DtwBLbyxgq5z60h3OF8Oy2KT3a3U0ClVAiEw4J571vfZ7l4rqMxRwxXAVFUOHJ56gfia5fUNPGo6y1raWtuBEAJJLZflUeprb0STUvCqPA1gJ1MgkFwnzgqOoBpEmlDokz6nbeF/OgEPn+YjuD8x/u5/h+tat14U0/TPEHmWIitbqMAG2c+dEQe+eCPqKoT+KY7m1Go6VayWl9A5KswDB1bgg5rqrmxY2kWpa5eSS3EsfliOIGNskZ2gj7woL5Ti/GXilLWyvNEjETSzhQ0kT42nOfxqPQ00zW7dHnsoLXU7Qo0RB2+YQOD9Ola17YXC2qf2dpttHBz8104d+3UntXKXF5FbeXcyMkl8kmA0eREy91HtTDlsVtWkubHV7h7sQu8zFmhVwxTp6Vj3Nz5oRlVkdD8uDwB6Yq7qM1q1qRJb4vZJd5lTuPSqcsLxbUELIVOVLdx2pCNKKHTtRmS4uHgtxtIdHbDE+uKwbomO7MQAxE3y4GMitn7bPNHc7reI72UDIy6kHrUev6zBq+sxHy9lpGojyqYIPf9aaYzMLpvWRlLMei+lM8syyhSQoJ+8xwBT0heecxW6GQg4UIvzEVHJHIsIfnaTjnsaSAddWZt5G+eJlBGCjdarsQuMGnM28qGGHx8pxgGmiN3yeAAecnpTsBJ5jS5YkcDoBxTi5eJM4LJwpx0FG6KBn2Eyr/AA5pY2DyopAXf39KLAWbNI7m4VJ5fKTbt69TWrodk13JclXw1rHnJUFWrNsUV7ww/ZfPd1KoMgYNXptQFvbGHT4vsz7Qk0i8Fz3BH+etDEVWWVEaXySFdiQD3HcV0GjroVzpF/YyLOlxdvvt0KZELKOu7pyeK55dQv0t/Ie4d4cECNuQM+lamjTf2jZz6Y0UbOI/9HYqCQ2c4z+FSMzNSju9NlS3vbWPj5kKsGBHsRUl/qMN5dLCxEKgYLg8UrXEL3QhuVKqf3ciLlm/D8q0bbTtPh86OytLq5YgAwzxbTtzyRTEZMt+jyJBFkhFChiMbgKlhmh+x3UEiqJpSrpMwzt29QfrWumgRhReao0VnCse2COQ/vZD2z/nvVKLS4pNMllkMhuC+1FH8Hsfr/SpAz4IxOkjuICzHIyM4Pal2HeyMmx3H3QMCtCG2uGWW2CPH+7JQkc59qz7fUJYrjzZAHJXY6HpRcZMkDbimM4+8vfFQOfkVXcDaMDPpT/tdyqMIlVg3GT1A9KnEkEbjMCSwlfmVh0PsaYxYZY7Se0MsEi+WoByOoPetd4okSUWUvmGZ+GTPy+gbNUtUPnPGwmd7eMCMCRcMPqe9LFNIIVTzhHk5yBgcVDA3dI8U3mg2Jtt4dM5KqxBz7V1/h7XJtb1VxfXP2VFQbd03c9CM155LPHLatDPFGhLbldFyT7GgXQOySW3kJxgY+63pS5gPavEuvy6Vodx5CeZcNHtWTjbjPXFcfqOvXOm6NZ2lucXGxVkkZeQp9B+FcbFqcMurQPArMAvlpDJnCvjAP611Gh2Op6g93NcQB7dMwShmzIp7gD8KvmAvaTLLd+KbWG/mZIYt5iZ25L8ce3t9K9JWQDrXkvg9V1CwZpk/ewXKt5jkjIOc13tlNIiAyzF/TNWkQdEJVp2c1mxTCTODyDV+NvlFOwySkYjFKWGKiLUhiYBqN1wcink4ppOaaAEHy0vmEcYFKDgUlAgErIc8GrEcquMnaDVVqZQBqAj2p2RWdHIUarPmZ70gLAYCkdhioN/vUTykcZoAlLg1TndeRmneZg5zVG4mUSNzVxQmNkdMYzVSVInjO9sVXmukSU5Gaj+1xS/JnFaqJBaS3gccIDSNYquWjRQabFOqYwatCZnjJUAgHvRcZRe1miO/eSMdB2p8F1OJQhU898VZium3EMmOasNublcA+opNjBpBjB4NRGIPziqlyk0bPKGLDOantbpWh3Nx9aEhEotgelWI7cAYNZj6iVchT+NSDVk/vYp8rA1dqBapSuVY4qv9v8AlzuppulmzvxgjHFCiwJftGe9Hm+hqt+5wADwKGkjUbVPNVyhctLOM4NTLMtZsci4yx59KDcD+GjkDmNQTR+tO85OxrK+0qOxNL9oB71PKHMaZlXBFM3VQ+0EdDTvtBx1o5Q5i243DFVJ7dG7Un2hvWkM2e9CQrkXl7RwTSbCetSMeCai8xuxq0JgVwaUNijBIzikCnOe1AFhLhh0qeO6YHNUcYbGQafnFKwXNA3IdcGq8innbVUuw708Tt0J4osO4jx7xz1qJFMTHcKn3g9KcMN1qkIh388VfVQQAaznTa/FL5zKoANS0CNFkjJyaQKg6VlPNKWzuJoF1IOrUco7mvwKayI4wQKzvtjetSrdZ70+ULk/k46U5UKnNRrP707zh60mFy0rnvT81T+0e9SLMp71Nh3LQIp2arecB3pDORSsxkruaj3E9ab5qt0prPiizEP2Z5oxg0xZfekaUdqLCJPMAo8wHq1U3lYGoWdiapRC5o+eo6HNH2nFZXmOGzmn+cT1NVyCuaBus0gkVutZ5k9DSibHejkC5bY800NzzUQlz1oJzzSsFyWTBWqcqFjnFWAwJwaeQMAAVURGYUIODSBVHTrVmaM/eFRrHlqu5NhiOAwBNWEdScEAj3pkkCgdMUnlELkHNIZcTYfQVOjAVkpKVbk1cjnBGahxY7l1gCOKjcfKcdaFkpWYHrUoplNpGDYxUinIFNlZQ3ApvmD1qyS2qE9aVgAKijuF28tTZJgRjNLlYXK874bg1CWLjBol5Y0iD5atECqozzVlYd6ALxVY9auwEeUMUmNAtrJCNzkY9qUSEcCpY0bbhiSKR4RyRUMsfFJz1q0j81nZKMKsqxAFQ0CZoLL60rMGGBVNJcnBNSq4pNF3JDHTdgFOEmeKCalAkNxTtgIpKdmi4hvlnd0qZY+KRWp+8CgYBAO1PAAFMEgJ608EGkAu4DvRuFRPntUYYg9aALBIpppgbmlzQAuKaeKXNMPNACE8UgNLtpDTAo65AtxpcqnsMis3wnOz2bwnqpzW3PH51vJHnqprk9AmNprEkGcAkikB2HWjFJnk0vWmIQ9KiY81ITxUTtxTAYRUbj5aeXFMY5FMRXJo3Gg02k0MnSUjvUhkBHWqmcUF+KTQXJnI9agZ8HrTCxqMtQkIkL+9N34PWoi1JmnYZMZM96A+O9RCnAVIiUSVIkvOKrgVInWgZaBJNV7rVLLT/wDj6m8v8M1OprH8QaKuowB1JDrnkfhUMZzmsa9Dd3ey1kwnrWTLbtK24TSyua3rPwctxGXZn3AE1qaf4aWE4ZXU+u7NAzkU0O7liaQqVVRk7zWQW2uR6V6kdDurj92bjZCeq4/+vWJ4j8LwwRD7PLmRjj5hioA4U7pJRtGc097cxrlq6Kz8MsqhpHz/ALtUtQ0S9gcuctF2zVDMQnFPjUMMs5X6VPLaSRrl0A+tNgtZrh9sQGRzyaLgLAqq25hkCn7C8vmR2yke/FSh5QRbtsAPdetTNJHbARyLuX0zjNFgKctvJIhkxGvsDUtpNbwxYkwWPYiuss9R0/7EEtREmf4W4OazJPDc0s/nTPFGPYA0rAY0bxrciZOCDn2ret/ExCiIoHPqSKy760jtCV+R8Vks4J4AH0oaA19b8+/kE4iAAGBisfYU6inec/8AeJ+tMaRj1qbARS7j9PSogMnpUjE0wNiqAs2+3eFY4rYhs7ZNskk0mzuAKwAeasgzsMfaCF9CaQzk0urZ3AaBIh6rRNNbhMKc571BFZm4zhguPWrttZCAjOJCOxHBqiCLT4Jb5njiddiDJBqfbHZuBnEq9DWwLqaO3Ij07Yv8TIKt6NqmiR/JfWbPMTklo8mmmBni9tr63WKW7SKX+8TwKztQ0Kayia4muoXQ8qUOQRXf2K+E73cLi2O7ccRmHZiqOveINCTTJtL07S12sMbmQ4FO5SPOIhAZAZiQvrT28rOI33D6VdSSK225jVvY1BdXCytuSFIyP7tMCJ4olhLCT5uynrUmnxl5wHlMSd3HUVteGDZC3uZbrazjsRWJLwzH1Y0CZY1PTWt7iJ2ufNt36tjmqZtyx/cj5KlBMp3OckkCp7i0ubHy9zAeZ0xzSuBnywNC4Dd6XbtFTfPLdBWRpP8AZXqa2J9PmgurRY7L946/6sDr9KYHPck04jaK29fsmtJ0MlrJb/wnemAaxX56UAR8bcnOf0pgJp3zYxTcUFAeaQDFWIIVcMWkVSOxoVRnOAcGgRAyMuMjFIAT0FWiRJIM9qjc/Nx0pDItmakEDFdxIAp0ZUZ3CrEMZkhOFBx0XpSAhtkxKrgBtpzgjINac130NvZRwnoSOciqreXbwsOknYVWNxIe9Ai7FLIwlGcMBkGlsit1aTJKf9Izlcjk1S89scHrSROyPuBoAaV2yFT1HWr6iVrSOOEbmByR3xVVleQ5UZzW3pTm3t2UjMxU4pMCqL640++VoG2owDMuOM/SrlzLa6gyS2+Y5DgSKowCPWpo5Re6NL56BpEbcc9qzpbeQXUS2isfMXcqioA1dcWxtfIW1tEjeSMM7qev4VhmIvH5qD5B3zVy7uGaAfalb7RD8vlnrj2qsxcRBgzIhbdsB5+lUA6JszmIj5GU8+tQafcSwagsauSrPsbPcf5NRy3DiUODgjpVbO6Vj6nNHKB6DrGoQyaeYt+Li1w0b9xXMardQahHFdRB1uAQHHTJrI6HqasW8oaQQyLuQ9R0o5QJllR4tyqRIP4WHBrSnhl+z219YP5COu2UoPun0rHmt/JlaQSM0fbBrb0F1fTL63lOYWTcq+hpWAqWc8NtNMVleU9Q7Hkmtb7VBqNhsmgllkP3XzgAexrnooGRVLDqePpWrDMIrK5h2sRIAFx26/41LQCWW2SYtCWV4huRScbsds11Vj4q+1T2yxwGC0uH2sXwSjeoNYttpMmo6JpkEWyOZjIQ2MlhUb2U8fhJisZbyJizkdAM0bAetWN5b6nH9nkZfOi4PPOexFN1PVzpa7IbG5vLjssS8fiTXk/nLNYW9zNvSHfsL5II/Kuk07Xb+31G1tn1B7i0uBtV3Odue2e9axYF6/uvEWrbba9tUS3DbvJiILA+57Vz9zZtbCWVr+SG5Z9sMEdwS2fQ+ldxf2V3pUAntJWaVmAYyDcMk4zivPvEUF7qOt3apbQy/ZQHeWziwc9gT+FO4NFWfxJf2Uslre3EV40WAAw3FPYN2plzrOqSaHDZpbRw6cW78kn1NY8lhMJl8+KSN5RldyHLH6dq1LvTr+BFlvWPzpiNRwPpQA/Sr+zh1mOWeSKCGNcbIySJT6E11MUUer30Fs8McWnW8ZnlSAnaWzjGe9c1oUGnvamxu7UFmJcu3GBU1r4gutPjltbeJHiBKKW5BWochHXzT6AcmYiVn+QbAQEHbA7VU1TxFfaZqNrczu901lwsPRWDAjdn3H8qxI7qPUIJfPsoY2RRteJjnPOKjnee8urOS6tvMWAAMFP3wPap57FNieIPFt5r7C3a0WyikOeByeO5rElkRPJjkXzrSH0OOfb1rS8RXWn6nfxpYW8lm38ZkbAP4VkWf9nxlhdW01xc7j8ucJVp3C5HPqKy3rXVvGIs/dA6ClF6bm1WG45YHIfvVue+zI7LaJCjnOwDis6WRXnLLGFA9O9AGiTBf7/P8yPavy7DnJ9/esubSrmBwFXzMjPy9qsx6hcJhECD5txO2r5vba5lAnQWl2i/u5E5RvrU3Ah0b7BZ63aTwSTsq5S4JH3SR1qTVLTSbS3mc6o13dl/miUY+pFZInuxNvjby2JJ+Wq/+rlkM43M/Ofc1aA1tTiS2021hiVAhxINxy2fasy5gmiSJpbcw7l4B6kUSzvJFFGzkpEMID2qOS4nnIE0zyAdNxzimBFjIqRH2lGA+XjDdjWr4d0SXWdSWNY3eKF1M6rwdufX3rtfFXg6yt9Htb/T7SXzjMIpYQpO9fUjs3FAHEaG7DVncbDKkZZA/APrz2p8t7FeXkk1zGi4z8qDAbFLcqmmiaaO1kj+0DbH5gycfxD+VayaJa3EVuzEWURQANMCXkPU4HWkwIZ7G0tNOsLggrLN8xUr8hGM9aNCu7O31KOOCJTLJu3Oo6ccY9s1n6nNYedFZ27keQAvm5JDH0welS6Tc/Zbk5uBE0ikGVV3Z9qViRFjbR75b+WJXKuxjDEja2eM/nT/AO1dS1ecTzyOzISA8Y27B74q9rVq0un29ugjWMZbz5D8zHism2Weybydw2t97HegCSYXF3L5kl1LujBIMxyfoKn0+S9+0STI3nIFzIXbGfc1JIbGFH2SyRtLhdzHIFQWV5FZecp3OkibRgdTWTYzV01GnuYE+1Ip35JkOAPoa57V7Say167gldHJfflDkYPP9alk2zQYYNBMv/PM4B+tRLAoGGkLHuzck00A4fuwgUqpBwwY85NaV2q2lwluB97/AI+GHRifT0xVExoGjcPDGwYYyfmNak04v7uO1tgJbidgpYjA3UDC8me0lCFVlljH3ugwemKpy3HnQqv2dMj9KnvYoltUuI5wWjYxywE5bcPSqMMxM3mKCowRtPQ0DHl1jcuXwzcrsbhR6Vr3T7Vs03bPlB2joD61mB027ZQehBGMHnuK1ryK5l0uK7+0rJFHH8itjdipaAv6fc6hPrVpZ2sVk6SSAlkiGRg8mtrXNbvfDHiOaGxXZ5z+a/G5HRgMHB6Hg1ztrM4k063lu0gbZv8APUcoff2q94iaa8naS6uLSWWX7rwcjAGOPSmpAXYNbWGDyGto4EddolQEIfTtXT2tzNBHbQTSIWYdUORj1rjpZ5tbs7PS3jWF1IDXWcKQB64GK0YL6507VGIngLW2Ii3l71YY/ixW0ZiaPSYMIcgYrQjk+Wsq0lu2tg12sAGAyvDnBB+tWI51ZwqmqEXy2abRGpdcilKEc1IxpPNNzTmHNMxTuAu4Uu8VCTTN1AE5cUgbPeoiaASBQInyKd5mKr7yKQyqTjvTsBZEoJxmhmGKqGQKetOEytwTRZgEjEqayLrzt7MsZJHrWqzL601ih75rSOhLOWuo70/vHjITOKbAjFgSMk9hXUMsbDBAI9DTUihixtiXitFMmxlwWMhw549qvpG0aYIqV5cdABVe4M0qYicJUvUY7fHv2k/NUySrjGahaBERNo5A5NRt8o4pWAsTr5sRUHrWPLHPE2wc47gVZjuWF0qO3yHrVyXYw+U00BgyKUXd/FVbaTzV27RssAOBVUB/Qj6it0ZsmjmwuCacZm7VAInPagZFUSOaR8/eNJvb1OaaTzzSZGOKBXZIs7g8mpllyaqgjNODgGjlC5oJtYYJxmlKBTwc1UWYYAwKnSXPU1FikPOVGaVZQQBjBp/GKYAByRQUSgjb1pMGkytKGAqbASJ05pSFHYUwNnpS545NKwDsikyKbuGcGlFAxNo7UmDTtwFI0gzQIaRSYpwZWFLgUwGjIFPDEUZHej6UhjshuGHFRtCRyvSlyaeoPegCEIc4IpGgGOBVnaKQinzCsUGjK03kVdaoSgz0qriIt5HenCX35pCnFQuNpoETmY+tOWYjvVbnGaTntTsO5ofaD60faPU1R3mjeaLBcvi4A6Upm3CqakmnHIpcqC5YEvvSGUhs1ACc0EkmlZBcsGRXppGKg3EGpVlBXBp2ARgMZH5VETUrdciomzQITdSg01UZugzT1Q0AKGOakVjjFM8pvSnqMHmgaHYJ5FTJkjnrUWCOlORiGqbDJSvFMKKpzjmp+CKjccVIyrK56UmSI8CnsoJ5o21SYikwPXHFKpINWjFxxUbx7BnFXcklgYqOSTU0jgrwaqrmn0mikxjvwarHOasOpzwKidSKEiWNEh4xTt7EcmmYxTu1UIdkE1IYSQNpxUSDBBqwrjcM0rgNeEoMmnwS7G61NOoePhhmqLIV/iBpDNJblScVJ5qnvWZCcP0/OrBbAzUtFJls7etJu9KppI2evFWARilygSg1Kj81WUnPWpRUtDuWA4pfMxUHOOtNJNTYq5Z833pRL71U3EUu+iwXLXmY70ok96gBJFLRYLlnf709ZCKp7iKeJTSsK5aMmaTNV95pQxNLlGThh60oYVCDTgaVgJd1JmmZozRYCTNMagHNBosAwNhq4y+VrHxIHBzuOenrXZY+cVzHi6Py5IbhT82cD26UDOmDBkVx0Io3+9Z2l3Hn6bGc9BVndVEj2lGaYXyKjbrQCCadgHVEXy2M1IThOOtVmODQIc3Wm08Uh4pDGGmEmnmmkUAyIk00iptvrS7PSlYNyvtNAU1Z2AUmwUrjIQKeOlOIpppAOoU8U3NLQBKGpd2RgjI9KjpwFICZHCjCgAe1PDmoQKeDRYBLia4QboRk96wr3Wo1xHI6yTHoR0H0rT1CCe5j2QYyeuTisGbRpY5RJMkEa+paoAyrye+VmaK5CL1wKoR3t1cOFnlLj3qTWrmFJ/Kgbdj7xHSsfzpf4aBpmreZddpEaj1BqpFcRWpGOfWqJeTq3XvTCNxyaQ7mvDdRSSjCov4VX1AebOgj5z1qpBxKK3P7Ido/MS6h+maYy5pDtbRZNvbLj+PvXb6e1oLfz9odwM815hNp8hiZ3uANvoetT2GpRaZFw8snuvSgCHxVJJc61MY1KqccnvWQtvKF6VoXl01/OZFB3HoKntX8tCskYLeh7VIGOue9Ka0LkxNnKBW9FqK1sLm8ZhDESo7nigCg3WmEjtVq7sZ7SbZKADjPBqvsx1oAQGpUlLcOeB2pECk4JxUioM0xswFt1HKHDdjV6JPLWLzOCxxmsx5gqErJiQdsVd+0+bbRyFCGAxz3qANeTUZ7HDRMpTsCOtYcuqXDXhYpEoY5+VeabNdGVNpNVcAODQhNG3ca4Z4sFdk2APMVsfpisee+QRGMglj1cHrSPgJuqsyhuapMBRcM0QQx/wDAj1oiVC2ZeFqHcqmpZDtUYqwNOwESxXPl9xk1myvulNMguWglEg5x27GkvLz7RKX8lF9sZqgJG+WLcOoIIrV1WeK5sbW4Eqeb3QHioXsGvZLWzS3MVw38RPFdRe6TpFrpCWtxbbrlVypUYOfrSEcz4aW8fXoZbQRPKhB2OcZHQ16H4wh8h7O7jGL7cPKXtmuIsrK2gQSCOZbgdHB5FbWnzG7vlm1OSeWGEAxknoR0qeYCz4ob7VpttDfu4uwQfIxgmvPr+3nt59oi2AnAyc13XjB4rl7a9VPNhwU647CuGkKf3yfrTTAXUNNudPiSSUqwb0HSqyRSOu8Rvs9cV08Fnbah4PkuppJDKuSCWJ5BrEtbq4trQxoFIJzz2p3GU9hUZNWbKPz5ADjb3qtulZj+7bPoBmrKXDLalFADeopAFysKSHyh9SOlQbd3QVp2cEL2Sh0yx75p8unNEm5EzQMyGT5cAc1OkqwQjHEvrTHyWz3qZYPt0ZKlQycbR1NAFFnZzljk0o+5n3pVt52LBYzwe5xT4rdhGzSMq89M9aAIRyakTcWAHT1qyoRYZFdACOQarpE8hwvHvQBdj3xlUKnnuBkVaImjnW5Rcqv3hnnFWdPm861aBxloxkc9a0EtoLwLCrgO/wB9Txx7VnzAZ8Mge6byEf7PMmCSvQ+tT6ZBHFI0l9I8flZ8tkODintE9mpt95CjoRXPaikwuztZ3BGc5xSuB2S6la3e2GGzWSE9ZZ15rl75m+2MpVFXtsORVy31BrfRUt5ZAc5yBVWPZMpdgcZ4piKZj3OOAcc81A6sbklUAH6VozQxm386DIY8EH2qnbh2HzdxzVIoaylCRIpGenvTonbzEEf388D1q/CuY90zAqM43DJqvaTKmofaCB8v8IGAaAL90ltMjxOojfbuUj9abp1yun2rxSRFi4wGB6VUmuWnmZzxk5x6VftFja2E8w3srfKV7mgQ2VoraD7NKga5LcMQRtqeaGePTHYESLGy73QZAzVyCZZta+3XFsnBCurncFHrWnouoWWna7cxRTotm3WJlyGPsKSEXvDrQ6F4T/tCdpHedisa/wBzJ6j0q1pNzFJoeu2ESibMYdN3Rs9fyrGk1KfU9MvTPdJ5Mcu5ImAGBnjBq/4M8TafZWS6feqkeGJWYjK89if607Eh4i0+Kx+HtrEse1iVJ56Hqa5rSoJ7jTIYxIkWJDtLHBU10/i/UbXVJYktbuCaGLP7vfxn6965yTFtOltcKpSIb/kHek5FI6688X31/FFpbxpC2Qk0qyZMgHp6U/wnBaXF9eG3YhUK7iz/AH8Hv7GuBW0uLp5ZIkfykJO3ox/Gphpl/pd3ZzXBaCGVh88b84/CpUgPQPE5jgvbi7ureSOFiqKynkgDGfpXHazrlhdyxbZJ1ZOM4yMeopfFV032uOFZZnQjH71srWHPMr2qwuOhyCOopsDUs7hrCWSJyJomGfOJ5XNQzWl5c2b3Vnp0i2kZwZ2GA1ZBeaeaKNTjJCZPTn1rq40vmP8AZOo6pJJbIq+VGv3SaOUDEgheNcy8Rt94etWPtc0myZJiPKbAAHOPQnvTFsLu41G4iSRJRbg7nzhAKzXmucMkcimMnAAqWB0NnPpX2S/udUiLzSP+7IGcVirZFZVuDLGA2GXJ6n0qqrTGQ28smzcQHQnB9quaTINPuPLnQyW4Y5Y87aLgirdPObk+bGF9VFQSqomBXhHGQK2FNousea4LwyE+YsnBUeoqhcRRfayqSCRQxww61SkMIokFn5zDo2CagnMflg4DsvXI4rSgtVjhZriZFj/iTqTVKe2fc8uUaJxhOfvDtQwGW+wqrtU4aN1aDyP3mcgbM5qN7ZoRboF/eFeUzVu3ultZCzbZVZCNuOc+1AFC4toT8yjAPVR0zUCW6Rv+97dVq6YZJ0ChTGzdN/GRWaykgZ+93q0wOk8PavfaZczNZXEcELkGXzBkED9fyrY13xLcPPbSefO+DuV1JWJh3GK57S9O/tSK8DM0Tw2vmxrGBmQjsfaqzvF9gVLhmDxDCEHHJp3As6pd22oPZJ9oEa72Z2c52Akda6S30n7AttqJke8u2nH2a4jkLIwPTKnt1rj9Nv4NOvluZ4fPUIV8v1/Suhj8XxLqFmFt4lsLSXfsiYgvnPb2NFwMnxMs9tq9zZ3EUYdXMvmKuNwPPFZxx9khlH3snFaWv3Vhqc0MsE0kkzSnzDJEUOGOf0rKmEH2zybY5ViAv1pCsOWZpbhWmldjnPzEmtSWETKskJG9zgEmsq4tZLWVfMwHHTBzSzTmeGBGAHldCvBqSrGnLbxfYwz7C4IDAtgg1ILOCHTGup2Z3D7DGnTNYskrOjhiWZiDnvxW3Jd2mowLDF50VzPh5Ax+QMBUNCsSzXT3FqskMEOwAKWK/OMdiaW5vItT+eeIRAAAFRgqfXPpWTLA8eXRiOcYzwasRXKgDcD8oJwBkH60rBYsNa29qp/tCzNyspzC8bkEAew+tJokzaZq1vcxPjy2DsHG4EemKk0m6kN+shuY/ljaOJp+FGR9KsweHZbm3vvJmhkkgcYmwduepGe1NDKF1ffaNUeZ0ieFmYgKu0DPoB2qllllbK9eQR0IrZ/4R+906L7fdfZ0SPHyRSBiynGTgfWqcsKy3ZiQmLkbRMdvBqwGW7E36FTnIABccCmuZFZowzFVY8KcfpUUnmQu9uxICNggdM1KhaOZViPRsh/ek0BYtcrB5mHKnKgkHr3Fbugtb3+pRWbw3ckhBZI7bG8v6jNaeg3CX+l3WmX1hHeRBmuVVmKCJ8YPzCvTPCXh3T7a1sruWxiXUIIgpkAIfBB6D8aiwHneoJ4hsZrXTEsDKt1bOkcOQS3Ocn/aGelYNveNo16vmqRcICssM8ZwQexBr6GntYZniaRNxicOhbqpFcv4o8L21/r2k6u0YPlSLFOoX7yk4B98VfLYDG8NXMc8sdil3OolBP2UxkbR/slh0rr4tAjhTAaQn+9xmrNnamBIBM5kkt8qrtgkr2yauSSnNaRbtZisQRxiNNoJPuacRmlBzTgM0AV3TAqAir7oCtVJFANAEDiowtWCmahyN2KYhcU0inZoJyaAIz0qrNuXkVcxUbAA5NWhGYstwxOCSBTxc4BUqQ3fNaDokaEqMVhXk+y5cZxWsfeJLDXe75TlT71Ud3HRmquxZR5ivu9RTWmZlzWvKiOYv21y4cBmJHvVw3BPI5FYJuCVxjHvSxXUkR4OV9DR7NC5+5tSyh4sg5PpUIYKwYsRjtVEXRPTinTsZY1K8YHNTyD5kXmvGZSo4z3qqbpt2GPFZgmcd6PNb1q+RE86NVdkhyakUFcZbNZcVwVPJq0s+4daOQalcuMcjPekV4lHzRr9agWbjBqVSGBHY1NhkVy6sCIzj6VQJw2K0Tb5+7ULWLbtwNUmQ0UmOTSYqaW2dOcZ+lRqOeaoQw02pyg7VGUNMBm4inpKQeaaV4puKdhXLgum9acLr+8aoinClYLmh52RlTR5pxjNUgxB4NPEnY0uUq5aWVvWpVfK5z0qnvUjGaXcRSsO5cDhiOaXzgrbc1T3sOhpu85zRYLl5nBpu4GoEkLHBOKnigeRgAeD3qWO43JFPDUxl2uVznBxTcmgRNuPY0CZk9PxpgcU9VD0FXHi7JONgpwuCWxgVGYeeKkSIA5NSBIHqVVDik8tcA1JHtWoKuQMmD0pmwZq6yoetCohPSi4WKPk0fZs9RWgIkPan+WoHAo5wsZbWvy4xVR4XRsYraaM56VG0SnqKtTE0YuxvSnKjE4xWn9mT0py2yA5A5o5yLFKGFjLip2gJcALnmpkj2SE+9XI8EZxS5yrGNMojkKelRZ9K3JreOYfMvNUHsAhwCaalcGiiaZkg1NIhU4NR7c1dyR6yAjGacMHrVc5Xkdacjtn5qdgJzuA+U8U3JpRIuMA5pQRipAerk1KFOM1Bj0qVJQBhqTGS8GmFDUqSRGpl8tveoKIEYg81Iy7hSvF6CmqGFIZA0TrnIpvSr+0N1FNa3U9BRzBYpZoI3jFTyW5AyBTApp3EQbNppfapWU1HghulVcmwqjPFMZOSMVZUZFLt5o5h2M9o8dqYVxWhJECOBVdkAHSq5kJogGfSkLEGpHLYAx7VAc55pkkwkJGDTgwHJAP1qAGnb6LAKWx0pfMJHOKYTkU00WAlDYNTLJVTNOD4NDQXLgk561OkueKoB+KcsmDSsVc0g/FHFVEmqRZPes3Gw7kxFNoDjFN3elOw7kytxTtwqsHweaGlz0pWAs7h60u4CqwY08NkcmlYCbevrTt4HQ1UZsGnqcigCx5gz1pd9VyaUGpsBZElLvzUBahW5osMshqdnIqAGpFNKw7it0rI8SRNLpLYGcEGtduRUV3CJrKVG7rSsBg+Gpt1s0BHzKOa2SMVy3h+48jUWQnG44rrHFUBWc80wEg098ZqP8AjqiSYHIqJ1weKevWnkAikBEDTGPNSFT6VG/HWnYGyNnxUPnHPNJIeTURPNXGKsTdlpWB5qZDmqimrEXapkhpkpFNxTj0pmazLGkU3FOIpMHtU2AbilFOwaUKaVgEzTgaTaaULRYQ4NSlqQLS7RSGIG5rnvE9lNfqqwjJBz/Kui2ik2c5xSaA89Xwjq78iAbfUtWZe2M1hcGGUYavVXE2P3YzWHdeHlv7v7RdqcexpWGjgBA7cnAHqTULJg8V3Oo+FLRIzJE/TqN1cbdJHbTNF0x2osMrOjLjkfhU1pv83Cnn09arlwx4pysR90kH2pjO0uTbLZJHKoV2UYAp0Xh6K4tQWXFYFvpk5iE8dwHYdm6VOdaujD5ZkKsPQ0AX5NAWPmJOfWkXT7RFPmyBX71gvrN6jkLJ/Oqd3qd3MpMkgIPYLUtEmqmm41BTvWRWOBiusNnbQW++GTawGflOa4rRxLeFw7tgY4HHrXT21grRcyvxTsNM5WdZ5r+VpSWJPU1VukVHwrbq27+3nO4+S4A6tjgVkNbNKeCM+9JoZUGByOtOEjnoPyFOkt5YOZFwOxqIOB0NKwHLuSTmnrM6Js521DTg2D0BpDLKhGj3jPFRv7CkaUrCdtW7CNbu3kUjEq9KEBQZ227R0qBmKjirUg8uRkbhgapv9400IcE3x7sgUbyV2+lI5j4CFs980i1oBLFtV8yIXX+6Kkj+ySXy/uyIz/CTjBqszEHFNzmgDQluZI7hAkjBlOEbPIFakVteTMJrm6mz65rm9zblKjcQc4z1rqrqdksoipxuWspFFyySUXCtK4MXvxRq1pG26TfKASBhWwPyrGtL2cTq2/8ADvXSrp2o3cX7y2Yoe54NZXsJo5bVLGWweJfOkaOQE8n/AD61lFXJyFynrXY3ekJO6p9pkd0GAr/NgVzs1leKxXyZgo6Dy62jMViKyef7PPAk/lofmI9aijhd2Cg5J6Up8naFIbeOpIq5pNvLdaisUJ59D0qrgQL5tvL5Wdu7hh60ycwK4WHnHWtoWzaVczNdKjowxgnIqrdXVuAXs4ljx2FAFOTP2ZGjX5lOetdHZXcN5MkbHIaPnjoaxIkluLN58jK5yMU/SY3nl82GQCWL5tp70gIZ4PIv5FC4UNwKlaFBIJY2IYelSXIivLx1aTZjvmobdnhuniSXzYh70JjKl3eyTy+VD+7fuF71HE7wxyQyxh9xyCe1SXESxXzSR/gahclnzTAthRJFtPeogfKPlkcVLG23bx061Nfxq8qvCuVcZzQBNobQNNdNMwUKvygmrWkTS3Ky3O4BonIUAdqwLeNmuQh/jBGPU1YhvJ9GE9uYv9Z/e7UpRA6SS9jntmlZQrocMB0rm31CZWI+U+5FST3klqV8tQySryGHBNUo13RjPWpjEB4PmyiSSePA/hUY/StSx+85BXYOBkgVSt4VktXJlRSDgc5qK4jeOL5xgE4FNoDXgiU3QjlZAuCR83Wor2wzv2hkq5d6Rbabo9leRXifb2xmAcgg9KhJbz3d3LFsZzUSQFayhTfslPyjqKiv0tYZyLYfJWjC8UkhQ9u9Ztwizb89RTTASO3J8p8YDjIrVt5Bax+XtZizfKo6ZNZjs7wxJ5hVVGCoqaNmaaJ2OfLIP5VVxnQXWl/2fB5d55kdw6Buf4hVrwbodzHr9vqV1bx/Y3DBXc5+Y9MVpalcQeL7C5uIUaCGxj3I7cFnx0PtxWBJFeWek2V3udIbsgNHu4zU2Ea2s6DZ6Ybm7vWCvNN+4RckKn0Fc9Fqlna6XfWYi3XEvAypXI9a9B8PobXSbue4s/PvhKAU27mYY4HNcXqunXU2oXuo6jbrbNs3RpngelUxNGLZX0mloyNbQzCXDEHtWtYanF/aUt3LDL5LL8rqMjNUdNitbm4uZJ7YSJ5WFBODn2Ndv4ajudVlljsjA8MCBWhmOBz2yKSiQ2ZFhdx6ikkk08UX8KxtwTUUMVlf+ZZ3U+2aNCYZnfaAfpWfr+kXfhvVneZAqu5McLMHCqfcfWta6a2gktimkxXJkUSZU4GKllnOXkzWsEdo19DdKepU7thz61mSO4fB7VpavMmqSPcR2q2vlkAxCs+NGund3faiDLNjOKpAPiKtGpU5cnHtWnpqNLqHl3K5KKWX2Iwax4biJZYVlH7oNuz64rVN4k+oZhOwFtoG3qO9ICae51DUNE8gTnYjY8pQAeO2e9Z1vZ3EEsQKZiHzEentWhdXGzxB5SFIvlGT/ePrWfc3bpcOqvn5jSQF/UHiCpNLEH2HKkLwR6H0qlPKjMGgOVYZog1Wa1YyxsGIHzKw4I9KnmMF75t3YyRLK4w9seG3eopWAyt8kk5DyEhjyR2qRhKjpJDgMh4IFJHDLc3Ki1iaU4BcD+En1ol/dYEjbiDjHT9aqwGpcTRyoG8vGAR19apWq2Uce66E8m042RtjAqsZJZGwnCIQTUxlL/IQoHYAUDLDu00/m2KysU7M24oKSwBMUl0z/vonzscYDVnw3b2lw7JKY1YYJFakNxbu20jcG++Om4+v1osBvTpFrcdnPI/k2zhkRMY2yYx+NYN/osttdPa3caQXOwvC55Lim27S2Gq26lpHiWYOIpOh9s1oeJpZdUuI795GP3sxnGI1zwAa0QGZoojs73/TLme2EiYjkj52k9c+3SpNa08qtokDJMsjELIh4cg1muMGN1JdM5dSecVPcXkMl1EYon+yRPuWB3/qKHIAjtQis08bK7ZVM8DjrVFxscqeo611y+JdPsNLuF06ynivJv42wy/hntXMTPLf3QkuZY42bCjIAAFK4DYriX7SJWbOF2jitC2toI7TdIo81yW3HnYPXFQ39lcxOJTEjQ4x5kJylTapmGe3+zykwmIMqnHy5oAqXEq3B3lst2PrVU8VIx5I43e1RZy2KYxykHOQDx3qzCTG28Z3EYyB0quBhiPQ1etoWlXC8HYeDSaJLLN59uzzRvKgTAKHGD6n1p9pbST6WcWzzM74DL1z7+1U4jd7JI7YNIWXdsUc7ajjumjCqsjxMDkMpwMj1qLDNGaw1GzcWkloUdjtjGNwY+gNbmlXWlrpdpFcM6TwFhc243BZhn+I5welc/Nc3SfZr2K7druPO9pGzhgccV3PhbWtLktZLe8sIop7o+ZIHTesnbKZ5GfSqsBzfiXUNP1FrWOxsorWWIFZDFj5h1HA/nUmnx6bq+m3l5qLv9tt4GKskeRx90tVnxHHeR3V1bafpAtorZvOWUrhjGRjv1Gcn2rjv3in5ZJEVlwyjIDfWgRPH5lzgYLStwAB1NbGs6fBBpdk6rsuEhWOeAcN5gY5LD6YrLtbtrZHAVSrFTu7gqciut1hYrW5ur27XDXtmsqeoZxyP1pMofpulS6rpGpavbXbRWlqU86HYQxBXqK3tI1zxCb3+zot8WqyKQtxeSnLIOQAOmcc1ieGNa+z6dqGls2LTUbclnflopFHyj6HGK6S9s1k8K6PrF7LJJLJdKfOjYgxwlT6dNpFTEDprLxPeWeqW2k63eRyTXDBFmjtivzH+HP97kV2TorptcZGQfxByK8RvLjUwlnqP9nPcQ2NwsgugxYuQ2QWUdM13eneM7e5szfSweTM43GAncHPqv8AdqwOvfrUT5rGsfFUF3qSWF3azWU0y77cycrMPY+vtW0cU0A1WOamVxiosc0nSmImaQAcmqkrZJx0pWY5pB1oART2NVHil84nGKvBc04Jk800BmEuhwaer561buIcgnbWdJDIilh0FUSWQc0pQMMYqrBLgjdV1WDAYpsCB4/3TCuYvrdlkZgwb6V1p5rm7yMR+aQMDcc1rS+IiRmKTilicICGAP1pmSDSEV0mI92Rj8oApVjQkb5QvsOpqCg1QkStHtP3gR6irJkRoQgPzYqgDinBj2pWC4jqVcg9qZmnNknJptMQoNOBPrTc0ZosJFhZ2Hepln9TVLdg804NU2KuaaXPvipBcH+8ayg5HSpUkJ70rF3NVZkbgjrTZLWOXnofUVRV29amW5KjBNTYLiSWzJ71E8RxkCryShxyaVolP3SCKLhYyTUbCtGa2OCQKqNGQOapMhorA06lZabirEOB5oJyaSigBwzmpA1RA04GkO5JupSwA5plW9Pto7u6WKQ4GDnikxodDpt/NCJo4f3bchs9qs2/m2wCzKQwrpra1RLaNAOFXAqte6a8nzovSuWVQ2UTnZFZnLAdaj74NbKxiBcTRlR6mqk1rGWLIMZqo1LicSoEQ96dtb+GkCGOQozA89RV6YIIVKAA47Vo5E2Ky76kLkdqVPtAUMIGZfpQ8uTlo2T/AIDgVncqw9ZMjoR9alU5NItpKIjIy4XGRUaOMZwRSuBaVc1IEAquH9iKuG1nCg7DgjrUlDQKcGA61HJDLEAScgmoWcikBdwmOtQOnoKg81vWpEnHemBEc56U5WGMHrUrGJvun5vSkWME5pisRF8GnLcIOtJJC204FVijDqKdwuXhOvrQWRqqLzxipAKQBJCjiqMkLKcYrSApGWM/e5qkwsY5jJ7VG8bDtWzIsQX5VGarFQetaJkNGaoOcVa8plABZT9KkMA6gUBAvagVgEZpDESalBpxIqWxkIjI61Yh+U00c1KoqZFFyPDDmneUpqspIqVZWFQUSiIClAxSpICcGpvLU1IFcxZFMECntzV0IAKXFFxlH7KR2qB7fDHArWxmmNGp7UXAyhHTvJJ7VeEC5qVYlHai4jNMBx0qBrY7s1tFU9KiaNfSnzBYxpLXjOKzpVxIRXSSRrjGKz5bVSTkVpGRDiZIBPSgirEkPlnFQsK0TIaGigjFITindaoBhpKeQaTBoEICaeDSiHjJYCniNOzZoAVM5FTDIqIMqninmQGpZQ4uaercVBvX1pRIOxFICUmkzTN+akA4obAd2zSZNJnnFPC5FIENGTUmCBUZ4PFPDnFSMXmpEyaaBkZqaEgHFIYm1vSmnINXvlxUDqpOaVxkKvzzUyyDFQsuBkUwPg80WC5eRxUykE49azhJVqKUcE1Nh3OHvUaw15Sowu7OK7Nf3sKSY4YZrmvGCBbuC4UYzWxo119p0hOeVGKYEkg61ATU8hyaiYVaJY1H55qyjAiqxWnxtiiwrlg1BKuc1IXAppOVoKKEwIzVc5q/IoNVXQA8CrizNjEODVqNs4warKvPNWIwo6UNDRMCSOaOlMaZVFRNPurPkKuWMj1pdyLzkVUDD1oDA9TVcguYuCRfWpFwaoggdKcJ2Wk4BzMulRTflB5IFVTdMaZuaQ0uQOZlwyRj+IUgkT1qqIjnpViO3OMnIo5EPUnRqeB7U2OM5watLEMCokikQhCegoeHK8itBIARwKmFqO4zWbZRx2oaTbz7ma6lVz/AOgqqnh3ToUE1y6P7vya63UdMgkQsgw46Vhrb/Z5cXbqEHrSuBw+sWmkv5klmSu04KgcCsO1t2nc7VzjrXX63Z/2ldNb6ZEqqOWbtk+v5VzUmk3mnT7ZGjB/3utAzenNomklY2WJgOVzya5JtzOzA8E1auBPsHmDK+1QBuMYpCGgDOTWhFZpIg2Sx7j2NZ+N7hfWtKz0jL5+1AfSkBatf+JcrHerOf4RU8PiFbfIdRk9s9P0rOuES1ueJfMwOeauXN/ps1qcWaLIem3nFAyK915rxDGGCoewFUYpraNg7Jucd81l85qzA0qtlIDKfQCi4yzd3S3i7UQhvQ1myQSRY3rjPSr0aXM90v7jyj3z0rXisbedgk80av2HWgDzFSCuabuzTVdfLzkUwzEjAAFCGTnlMUW87290siH7vb1qJWJXk0h9aQGhqYil23MQ+8Pm+tZhOTSiR1UqDwe1JTQBwRmk7UAc0rI2MgZqkwGGpFgcx7wRt+tQHPdWH1FPV2xjJxTAt6fp8uoF1hlRZF6K3VvpVu9uZg0NrNG0bxcHPeqOnz/ZdQhuM48s5q/qF7/a+qrKNqKMYpWA19IeLTEa8lKSMRhV7U3+3NTnut5mPsq8ACk/tGCNvKSzhOO/SpJNQsrdN8Kn7R7DpWDQx1jezrOf3hJ39+1WbnVtWs7hmR0eMdQ65ArMtW2WxnchXY5GTTdQuWlsSwelYRVkNnNcNLdTOWJyVReBViyv4YLgRWdo5Rvvsg+dhWaLa48rzWXKeuanglC267UP2gfddeorVCOstNOsdUvvMvUa3hjXkSttzWJqyWMeqlbeVZbX+IRjPFVWkvJomWa2mJP8AG7GtDSLYWlvJeTQK7KQI1J4zVASyXumwZS2spo2cAhvLIBqHTz5M7yyeWpfqQeAPpVnWtSvmjt5Z3ibp+6VelZMMCXJLMuSegpOQFfUbQRXRuInDxHuKpqzLKWHGa6VbWN7GW2AwEAIrmm3mYIOWpXGPlZw21+napLVAC0x+6gyajnYTyAycMoxgVpWkCf2PcAjrmkBnMSHYqcKTkCtK1eNtOEWcN2OOlVLSLzQUcYwM59aI0e3uW2xO6cZXNK4CtDJazJKVyUbcD2NVblZ9QuAzhSzfwgcmtKWaSSJgkRZRyVHUCqcUU5kS4tnwR90ueR+FO4FaV5E/czlgU6KR0pqMAMFCynsK2INNe/vidReQysm9Qq4B/GsfdtEiAEKpwAw5/GtEwJvNSKMYAL+h7UyaRrkBMYA7ZqDeGp8LENkGhgSNGYym4EtjjJq9DKdpLelVCckE84HFRkvn5ice1TJAXmEkVwoj5LDNSRwxTrKdzCZOw71HAd4Vv7tP2hG3/wAXrWdgJNHVZdVEbjK7TULlo7qVFPCnGa0NNiia6zFNskAz93is+X5bqXHzbnwp7E00xmt/aMB0YW4hCywnIlDcnPYgU++1RhNaQanIRaJEHijjORn+lZ8WlL9qSOadUeTGRn7n1rW1fS/IRBcSxzJDHjeEwQPzqiCxomZfEkErz3kNtIwR5FchwDwAfaun8e6FYWulrFbpJLeP8/myuWYKP6e1Yeg86S01wSyyfMWJ64qPUvE+qalZfYSwiLjaGHLsnoTSegznlitpbTy4pdjNyecZNN8PpKNTaNJ7iFxkrJE/II//AF1VZXglXP8AD932qXTp5I5ppUbbhuTioA623ub+4md9Rma542ATfMCPb0rA1Cdy5/0mQOjmNEHAVc9BVvTLktI1zne3mDk9CPpXR22laNdX/wBjl1C1eaZPNXeQQrego3KPOJpmjWSEdGOST1qGBNzhCzBWIU4PUV2R8DrcXV3bR3KSyoflkiPyKPSsvRfD2q3N3I1nbZa2yHM3Az6fWrSsDRk3EDW8pjI+7SvI8kYkZ/LwMKV7+tWL2QzSmGe38i7QlZuetZ0w2Nt7UMVi3YSRGZZL53dR04zXQyadYajZRTWoxLAQZB1cj6VywICYFbFssjWzXEUvlEKNrdyfSkhDPEUUUOpRSQoghmQHap7+47Vm3NnNZvFPLausRb5GbgP9K7rwbYWl7qVnqdzdW4liZklgm4LcYBHrWx480W6ur+xu3MbWkKEFShKqOuT61rygeX2d1LDdSvDEw80YcA5wvc1HdtCbpiHLR5yDjv6VZvEdNSZrSDKhyRsHyOPf0/8ArVVui8spnEPlo5yF3Z/WkwJ1uvnkTyNpcYLN3qRbC7aRdlvIX5ZMDggdaghmZJhcCNXEfUHpiugMDalDp32Im3lY+SzAllQnpn61IHMzXPmOCkSJxgjqSfWnwMygyrGx29TjpXSavos9ikNneWcaXkeWikhxtkQdSeazm2zzgQzFIWQlvNG0A+i+1MYkwubuyjusf6p8qc8qR7Vo6nJbSfZHRg8ci75EB+64rPtWjbSp0D/PuyvHOPWrNp9lns0jltcqh2A525qQIJLAus86IEXy9wwOoqpDEttkzoJbd/lJVsEfSuj1VEtojFbp8jR7ck4Cg+prBMwtYYrc2tvKq/MXzkn8aAEu7OSx3iJ98IYYJGQCe1QPBLGFNwmCwyPcVdu9Rk1C2htS4Qp2C/ePrUI1RvsYt3h3SqSN/XA+lMBLOc21rcwPE7RSjPy9A1Qxme9v0QyOqhckqcHApJpdybUZh9KW1kaMpddITmF2/uk96YFKSRGuG2ZHPfr+NN6YbvnNWL+4a5uxny9sQ8tSiBdwHc1AMMxDHHBNMA3ck+tW4ZTs3RyFJV5UmqAOTUigk4pgXoLyaB5DE23zF2kjqPoe1aOi3S6fPO8ixlJY9rF03Fe+QPX2rJZVRUVshuuCKl8wlfmqWgJGiW7vb2WO4ZogxdGUYDMfbtWjb6rdwPYz2ECfa7N87dpY57ZHvzVe1t2it5HZWRGxyD1HPFRJIsd2LiFy0vmYwy4GPQ0AdPe+NNf1mKSyEdrOWhKt5ERUgdxyT0rnb0xnT4bfzo5Jk+Y4P3Qf4frmqBMwuT5SuZdxOFNX7/TblHgNxc2xdyqKi5BAPfkDPuaQywltPDasttbQyOoMdyJCMK7dACfxpbq7mm0yF7tnnOxYt/TYi9F4qmJ59Lurq02pIGwkyZ5IHYf41saLYxX1pcrEkVxFH0jdisg7nb6/ShkFeOJf7PSQFVcoy/IeCox19a7m01iTT9HttMtkkhe6jVSroGCkkAkE9BXE6hZS2eoCK0XERfy18zCljjJ/D3plvfy2t5Dcm8J8pgeeeP7v0qL2Kuelpd3p1qPQ7K83yzA+dI8fzqikFQQPx5q5DoUn/CexQo4W3Cid4sdGI5A/LNYWj6rBZ+OhftLIqS7FSQnMYRsD5sdB6fhXcWkouPGmpTL/AMsIVQn0z/8AqpwZJpa5osOuWXkNIYbiM+ZbzqOY3HQ/Sq+ianNewS2+oARana/JPH03AdHHsa0/M5rG16ynklg1bTlzqNoMeWOlxH3Q+vtVlJm6KTvVTT9Qg1SwivLZso+QVPVGHVT7ip1PNMAkGKiD4NTtyKrunPFAXJlkHrUyOD3rPGVNSxuRTsBeOGFQSquw8Uzzsd6Y8oZcUXAzZxtkOOlLDPggZpZvv0wR9xVkl8HNY15LHFcurDd6gVoiTb1qndQRyuZMfMetVT+ITMC5Ub969D2qNelWZEIGSKrMGDYIrsic8hhGDSdakyMc0mznimIZtpACDUwQnHpSMhWi4DQueoqIqVODVhetPCqWORwaLhYp1PBbtK2WBC4z060ohj6Mcj6VqR3qLGqjpjGMdqUmCRFEka4QoD9RT7zTgyrLbx7c/eGamIgaUSKNvtiry3ESwbcE1BpYwfsoCFWPznoKrYI4roZI4pYvMVcMKzLm36si00xWKAJzTw571dSyRgCCfenNpYkYlZtvou2jmRNmVFk9DU6XDIapFZIndSpBU809WJ60xmiJgw5NBSNxVIH0NSJIV6miwDpbQHlATVNkKtgjmtAT46GkKpJ8xGTQgsZ2KTFWGix0pjJgdKq5NiKlp200u002IBVvTxIl4kqYyh5BPUVWUYq3bHy2DgZ9qiWxaO2tmLRgtxVon5cVjWU4Krk1qxvmuGSOhFW8thcRFGGRVWOwhVwzoGYDAJrXIGKj2L6UrjOduNNRZHcKRzmm2ltJPMkajKg5NdBJGp6iorO3W1kYozEE5APaq5wLIiI++oz9KUwowwVH5U7cTSs3FZ3Y7AsKYwQD+FVptLildW5+U5x61YR+al8wUXYWK7WcJXHlL+VR+WVJBPHarbSA9KiPNF2FitJb+apX9ayptMvFkO3aU7EGt4CjaDVKVibHKuk0TEOhFRb5CcKrE+gFdZLaRTIUZcg0kGn29uuETFUp3FyHO+S6jJbLemKsW1rPNnkLj1rofs8eOVBoEMScqoBocx8pnmwZ/wDVnb/vVXuLKWJNzLuXuVHStrijaKXOHKc2kLSNhUOfcVOunSNneSv4VssADwBScU+cOUxJrFkTIzVFsqcV1IVD1Ga5zUIXju3AQlScgDtVRkFiqWzTaHBHJGKYWOK2Rkx+6kNQ+ZzmpFdSeaYhGD4yBxSKx71OrqeO1OEaHpSegEauRUsbknmjyfQU5IiGzUt3LLQI2c0wsKlVAVwRTxGoHSoYEKE5xVpXOMZqrKCH+UUgdvWpGXg1O4NU0lPQmpQ57GgZPikLdqYGPejOTUjH5p6gn2pgxmpFYUgF2fWmlKsqm4cU7yT6UrjKDwtjOKFtQwyc1oiL1FGwCi4jCvdMkwWRc+1Zo0u7cnEWP944rr9uaaUGKuNSRPKckNEu3cDCL77qS40mS16uH+grqdnPFQtbbz8yg1XtpE8pyjFe46UzI7Vry6ROC7EoR1wKonT5hypBrZTRLgyozHpQqSf3TWxDZIoBcZapTbJnKij2iQcjMMxSO4UI+T04rorPQrRYw07u79xnFJbw5mG7tWko2nFZSqovkYiWGnxAhbdOfXmmnSdPkXHkBfcGpDGWORTjlQAKjmYzNk8O24/1csg/HNVZdJnhTK4f6GtsSZNSqqMOaOZgcp9jud2TFgfWpkglA5X9a6CURg1Uk2/w1XtGKxkvbSE5xUZhdRyKvyvjNVGct1NWtRFYlgeppwcjuaZIRupgNXYm5aEzcDdUnm571TyacHHelYLlncSKgZuaeJFx1pvlNK3y9KVhiB6kVyKfHpzHlnAqU2GOkmfxougsZWvQfaNLdsZKEGqXhi4wjw+ma3bu0aSyljHcZ/KuS0abyNR25xzipY0dRLkNTGbgYqxcLk5FVW4qkJjC5phlI70jHmom5rRIgsLLu708OapAkGniU0coXLROahamGRvWm+ZRYLimm7jSl6bmqEISTRRkZpuaaQCk05RmmZ5qaNlGKLAPCMegpwhY9qsRsuM1MGFZtjsV1tQRk1NHEqdBUmaKi5Vhdi5zjmn54pmTSioKJBUqnioRUydKljJ4ZGDiryTg9TWaCBTt5qGho0WeNvvVkahpFleyrJLPICBjCmpt5pMqeopWGc9qbyQr9ms7Pcg6vjmodR0S0j0wSXSZuCMk+ldSoUnpUptYblNkq5FAzxq7nso22oJMe/NZaBXuASMoxxivUvFHhjS5rItGjC5U5GD1rzq60yOwI86Rw3pQBqw6dpXmqYbcu/YA81JqqG2sy0dnLG/Zmqlp+vQac2/7OX+pqrq+vyaoQELogOduaQGWbiRARgZPJyM1H5znv+lSra3M6l44i6gdajMMmzHlMCOvGaLgRkiux8LWtvFB50t8iORgjpn9a4sKTTo/LWQA8Uhmv4hi23zNHIZF9Q4qLTJnjcAKD7kZNKtzYqgBRvqKmtlN5MEsoyfrxSA8uLc04c03FKDirGSsSgyaQyGmvKWjCntTM0mgLGwSKNg+bvSrEw+8MUyFwgOatD5kGaLANjiBHA5pJA8fLIdvrU0UJl+RX2n1q4qIlsIpZFfHpSuBmtcyOFCDCqMZquyljnA/CtMywj5FgUj1NPihjlkCbQCfSi4GPip4FKEOByOlaCWyw3giniU56ELTLqBrabYy4zyPcUXAZDJK10HaDzV9KszP5z7vI8r2qvDcNC4K8+1XdrTDzBEUz2NQxlSSQKNh70x0aKIGVSob7ppLgZcZIAHWtG8mL6dGMLsXp70JAZ0SSc7mJj7Amp40cMGi6jpVYSEirtjJsk+6CT0J7UwNTWr6QabEof53A38d6yTqVz9mFq05ER6gd62Bd2D24S7hZj3HSqcD6bbXXmQ2yOD93d2pCM9xI7Ftxb6mremM0dwxk4+XitGTWkjaQW1oiM4wS3PPrTJ/sj2Pmv8A8fZ/g7VLAt6dL5aSMQjl859qvalDpsFtHHNbIN4JVlXGD9RXOR3DRx4WtC11lpI/IusFB0YjPFSBiNaBJCE5q7L/AMgooPvEhR9avyTWoby8KkZ6OBzmo5LSC3tSbicFA27gfpVJgVYtHvkiRVRImU53M/WtGODym3PIDJ3CNxVE+RcnZE7qPVjireLWNhHCvTq2eamTAbcxwxIrhOWbDHPUGpjpVteRRvYv5UuMbc8GqlzG8uNpJA7E9KqyzX9tEBA2FzjFTcDTTVrnTbkSzp56ouxlxjB+tc5OP7QvZZB+7L/MFatO3vZpYXgu5WkLHOWANJbQvPOzR26NajKyHoRW8GBz00UkD7ZFwe3vTVJDCtu7W2kuGtYAZIM/Kx/hb61kMhjcq6kEHHNWBOVO0leSBmlKqLdJAxLHqPSpLWETzoGHy96VpCGlQABAegpAOtpVihkLfxDArZWwaOzivJo3a0b+NDg1z5+4T/D2rbmiaPQVRJpOcMELcVLQFlLFVR5oB5qOMA9Cf8apG0V7y3VWCBfmwT+lJb6lJHpYDH5kk+U966GLSoL6JLxoyhChlw2cioGQ+GLiI+LJ7O+hR1uU2DdztPXirc9gktpcWtzeCC0W4J5bcSB05/xrmpAIrwXWWEqNkEelVLxZJ2a4bkMdz/41opiaLTak4Y2On3LyxebhWbsM+lSahcub1clBKuFLxcFgax7VPvThgB2ycZq2YhNdCZZAzyOCQO1MVi9daWxv44oX3OyhsMeualstGkkvnsSGtw8e4kYIB9TUqTefNeXanEtsMR+q4qbR9Tl0y7maZPMhuYypYdUqGgE+yLHokyW8SCWBx5jIeW9xWbJo8sNq1zcSeRKzfu4zjJzXQ3evWf8AZE0NrbOQoCgv1ODXL3d/PqJ+1TEGUHKKowFx6UrCidTba3aeEdDfTfInbUJQzEg/KSe4P9KxJvGOuXFukEUwt4toVxGvL++TWTd31zeL511IzzEYJPoOlQwz+VLFJjOw5wehrVFCTJIr73lDZ6tuyaakfmQyMzcoQKv6lJ5l4lw9uke9lbYjZGKbqNxp81xI1jC4SUAsB0Uim0MzmJ2cda0ZL7zdOt7aNSEQZbPc1nMV6jmnFnU4BwuM4qWhE1vMtvdLMQG2nODXW23ifUIbB72aYzwk4+ztJ8qg/r61xQUs2MgfWr0NwY4Vili3Rc+YE6uKpMhnU6hqMNxcwwS3kVmnkkh4fucjvWcdN0S00+4tW1KK5u2GY5hnZH9TWdp2nx3sN1Db20hYjfC+cBVHXNQalbSWBjhKKElXdxzn8e9CGRtPFHCUgAJcFXY8gj2rp9Gu719GmurG0AW1TE1yOMnjt3xXHHhSfTtXqenX9s/hK+0i2sTaiK0MpzJuyW59O9JsZV8RXMR8H2F3e2kd7eyIUE0xOVyckgCuC+17kRXRW2jAzWpf6rEltb2+RNnEjBxwvHQenX9KxrpT9p3BNsbAEc0rjLceozW52whFB6jb1FW7OZJLmM7XiReODwD61kS8XI9AorWtUWC3E43bJuNx/hI7ilMCzfX7yQSQlsXULAsH4GP61kT3bzsm8qX5zhcVOVWe+cXDeY3Z15P50Jp7m/ijjAznPJ61KAqOyKuekg70ssymJMbHLjLN1OaluLJzeOvllRjdk9CPUVScqGIU7l9cVSAco2ozEggcmohLI0Rt8/ui+8r71J5MRR2y6nPAVc4HqaiIliypVlyM4I61QAeKTrWnb6Jc3lna3FsRL57FSo4KH3qreWdzY30tlcQss0TFXC8gH60AVR1qSNl3rubaCcE+lbHhbSoNZ1tbK43BXjbaRx8/YZ7Uy80xtA1uWyu0+eLG484ORnimBYexW102LUh/pVvMGT50IKtnA61nqrtH52390jYLZ710OjztqqJoM9zBb6SsgluJXbBx6DPcmk1bR7bTh5tldJJbTTssUeeQOOrdCeagOYxvJiW1mHnSxzFhtUHhgc5zVV5cxBas3yXEc5Qjntggg1WmKSH5I/LGACN2eaYGharcWdtFcQK0twQT5ZTqR3962oL7UrnS4W1KxRIFcTQXTjYQRyeO/Ss2xnimjlmvf3awxpFuTqeetS65q0mpWlrY2l3vtIUACEYLHPf6UEkeo3tndXP2mBmmuZJt7Ns2hV9PpioJLqaPMsDeTKHBEkfAUjuDWbbss0oQsQ56Y7mryS4tTAVypOSc96ANnybzWNKudWuNQV57ZBIqBQSwzjkjGP8A61Z/lwNZQvcSbWlQleMpkmrWmeXFBLZ3HEOoR4D91KnI+van2+h3l3by2qPGfKbcpDcZ9/QVDY0V9OaRLkW0svlRkjdzyoz1x3r13wS0kl1q7zSpLKdis6HINWPDei6fJE+pNaxGWdU3xsgPlSKMNj8aPC0KQazr6xIETz+FXoOTVRiB1FKDtPFRb6N1UIxrkN4f1STU4U/4l12wF4qj/VOeBIPr3+groE5XcOQehHQioG2SRPDLGskTjDI3Qj/Jrm7XX7PwvfvoWqXRS3XDWkzjJCN0Vj7H9KBnWGkK1HHNHNGssMscsT8q8bZBH1p+8UAMkXAqqXwetW3IIxVOVBVIBfN460xnyc5qMhgcUHIqrCEc7hT4Mltp6U0c1JGNpzTYCXMTBdyjjvVJpccNWwWV4qyLwBXYgcU4PUTKzeXuNMeCOQZHFMLE0qybeDXSjKxCbNR1zVXoxABwPWtffG6461QmQiShEtEa471IqqxApm30pN5jfnpVCEaHDHFOVflqQOCM0hTJpAVHUg4FIjEN1qcqcnNNEQLZxVAWPMbZhetEd0Rw5yKSMAOQfSmGPax4qRmhDdqRt7U150EpA6Y61mZdTwaXLbST1pWC5oQSfv8AGcg81bU4dfrWLHKQ2atfaWH3WOKlxKUjbaNZonjb7rDFcxcwtbSmNuorXhuJWxzUF1byXTECP5wQQc0JjZl5pwkNWF02ct85VF9c5p50m6wGQBwfStLoysyBXqZWqNrK7jGXhZR703Dp94EfWgZZAzSFM8UxHPSp0dCeeKQyuYj6U0ow7VohFPQ1G1uewo5gsVUCNhWH41KqmNvVexo8ghwxHGeavyW6hQIyCDyKTY0h9pNg8Gtqzn+UZPNc/DG6N0rWtwwTmuee5rE1hLnqalXmsgTN5mwdauQzSdCOPWsmWWZEyc1GFwaV5uMVGJKkCfOKjZiTTA57ninKQTSGPU07NMFOyKQxSaWmFgKA4FADzQDTc7uaSgRMGxSh8modxpu/FMCcyEGmtIO1RF8imF8UASebil86qrsRzURmx3pgW2lqMzVX80HvUgCsM0wLEcmSKmdVbBIB471UUBTTzJt70gCS3jb+BfyrPuLKEyEkHJ9OKsvMx70zdnqaq5NjLOk7myrlR9KYdLdDxlh61rA1ItX7Ri5Ec7JDJGxUL0qSxt3ln/eEqg65FbL28bMXI+am4C9Kr2jJ5EJJbjZ8g5rPYyRuFcFTWosnrTZUEoAqFJl8qKqynFWY2Vhk0wW4XjFP8sKMgU2xJEgEZpkkCYyo5pmcGpl5FRcdiqY8VKg45qYoPSmkDsKdwsNIpRSYpC2DSuA7vTlODUWeacDSAvRS4AxUizgvyazw5HSl3E9aBmnJOuzg81W+0H1qruozQBcWfNK06gZNUs4pc5FAExuFzUMs+W4NRmoyKYhxmk/vGmhzyD3pNtJii4D1ANSBFNRpxU24FuBRcY9I8c1IMiowx71IHxSAXLY4pV3N1pocZpS56DikIXYB0oLMo4pmWHemSyELyaYiJ3ANRFsmms+TSpjdQMpXzFc1nNKRXUF0aLy2RT9aWO3tUO9YEDDvitYVLEtHJ+bnsfyp6vW/qEME0ZJQBh3ArnlQ5wPWt1JMzasSk0dqetoSM7jUckUkZx2oETWzqJQGAI961VMQ6KBWLF98VeV2Hes5GiLZk544FJ5vqarGTjrUZkPrWQ7mgjK24eoxXn+oQ/Y9cZenzg12KzMvSuV8Sri8E3dl5oTEdWP3sCP6ioHSo9El8/SY/myV4qxIOa0TE0U3Q81GE9qt4yeRTwijtWnMS0UTD9aYUq+4XGMVGIlxVcwrFFuBTKtSQHBquykHBp3IaEzRmmkUoqgG5pc0uKMUCDNAPNGKMUxk6yEVOk/qKpj36VqWFgbnBcfu+5rOSKTESdScVJvB6Vcn022jTEa4I71nbGRsZ6VnuWlYnzmlFNU8U5TzUsaJUGTUoU5ojQ1MEqGVYZtNKFNShaXFQFhm3ilCn0p4WnBaQxQgxUijFMBxTw3FDAbMpKHy1Bb1NeeeJ9DnuZgDHsGcs+c16MGom2vburKpyMcikB4vd6XpVpaktO7z/WsDFej3PhvR/tsjXcoznJVTgVyOuQWFtdiPT2Zk5yT+FAyfw00lzc/ZDIqxEc561t6rZW9lB56lmlY4AY5Bri7W6a0m8xHZT7VtXWvQ3NogZ3aVTn5qQFN7fzUaQrj224qmPsyDE0IZvUmpJtaupl25Cj2FUJZGkfcaBkrW4kYmMYQ9BW14ajls9RSU8KtOtLS2ltleFJH9GxxUiXJtCUZd+PfFAHjuD2BNJmppQqEqDn3prmIoAnL96oYygDmlAp2KYApwasqxCjcRVXmjFD1AuJKQcqamVw4IGPwqopCx7QKfCxVWI71LQD3YLnb96prR3RkmZvlzyMdKrA5NEe/DKQSpNKxJuancLDEJIQCzY69qoS6pHdQos5H2hOjAY4qpc3s5h+z7/kyDiqgOT/8AWosMswyKkpdjV03huMBnbavQCs9F3DoR9RVyOMpBuIpNDJBALxX2clRnFIzY0tBnvir2jxsttK7DAOazHRnJhH3QaQxYFVyERS7e1axtpLOz8xrZI3H3j1oshBa2bC1AMrfeZhyTUohZ9Il2ks8hzyaYGbNE32Z3Lhs+nam6dcpEpR49+ehxnFSra3Ag2uflPUVLYRLFOP3LyDuFpAO1GGO2dR56NuGfSqT3HmsOc4GKfBpg1PV2t4kaFM/MzD7g9TTL+0Sw1D7Pbu9wg/j24o5QJbRZLpmjjGWovYzaPtb0BquPPtbkY3xOan1CV7q3RJ1HnKc7xx+lSwEy09kGXkZrTe9MuixCZx5g7E8ms2zdEt0R+VBzinXsiSsREuE7CkhAjGdv3RCJ71KZHhXIOayjLJAphClPqKs6ZMDOYp34kGBx3o5QNG31RrSQSOm9P4hjtVp7m0mUtBIkobqrDBQf1rHvV2Ryx+gpmlsC+0/3TU8oFq6RRH5sMqDac5z1qeK78qze3j2EN82V6j61SjtZ1RkmZFjJz1ziktozB9oyOB0rVAIvzRSxj7x5qIEXdqUkcC6zgFuhq7bWqtbGcyhWcEKpHU1QdIt5VWTj+6KpMAglNjIySHbNjBpGDPdlQOozV6406V7aLUvIL4Hz45JHrVRSVP2kqyoeAcYGfagCJ0kDFW+6Olb4WCTSX+2PGJVj/dKGyc1lW6PqCFY8Bs45Na2h2US69DLdCF7a3GJecDJ//VSaAzkXdCq+iir+nTJEzGRSQozkVe1z+xoZ/MsXdzI3zKAcA+1QW0VlslXJ89kyATWciR32iGd1MqiOFzgDvVXUtPmhulkVR9kY4HOQPrVa6uF3rCv3YzmrNvcNtlWK432jx9JOcfSkijJ1CIQTFA6v7r0qfTtsVzCGOFdT83cGqsrqYDbLtYh87sfyqaCJ9yLJKiqDhcnvVtgWnSaznmaORHSVdr7T0FP06USXYjmOVIJFbC2ESWEsl4kbKkXybG71ze9pY0GVXZnYQOQfrUgXjAwsrmdUxb79oYngH0qhaOPLVRglDnPY1paVf/2XOYJ3MqTH50foTU99oSwTy3lvIEt5RmNFG7J9B6VSEYV7dNdXDBgAw6Y71VGdxU9V6iulj0yS00mIpAXupQwICEsR7isySCfS9v2y1eKaZMsD1IrRCIlRpo7RGbA37Nx6KPerl1uWXyrZQcHPmLwKpSTxuuIojGvoTnmrFtJAbVkmkMbxqMEUmWVpI5JLlmZfmxk1DJIZZEBx6VpCW1vbhkRJIwyY2qcnj3rOuYntFSXOHJPHpQBNNbus4hV1dsA5HAA96kS2e1kT7QoKOu5cHqKzvtFw4cq3LjDH1FW/tDXNrZx7gDEjLk8d6aJaLdprM2i3Lf2eYyk4y6umVQ/n71otq638lvbXECfZhy6p8x+uccCuenVVlZVbcoGQcYzUqSLIrSrKUyu3b3qkKxd1LSEe3m1OzkhFv5wiCbycdecnirmiatqVtK0sgha2mHlTx4GSgFUrjTZI9CjvLWeaePdmSL+FT6mtLUvCgTSINUtrnzJZ1DLaqctyM9c0mhmXews7z/ZED2aNw56qKzhK4i2biyZ71YgkjktzCXwSSMnoKgRVMRBdgR0A70WAWPdM+AKtLI8kS27S7kHQKecVJatG0So4SEjOGPf60ySH50aB4gVGMjvUyQxsM1xatIkB2lznd3FWS013BCzOZbheCQvzg1WVC0gVpk3nsKspdy2zGFWcqoHK9fxqbgWkk8wIjuwwMArwao3NnLanMaMyfxllzzVy1e3ZS9xGdjDAY8Yqae6EVvGLaZnVTyzD7w9Ki5RkSfvbclHUYGSO5pZXa4sTc3LGR14VlPCH0YVbvStzeQG2gwznJVOmavjSJ3sri1jiNy00weXyuREPQ1cZEM0Ph++pWOtrKthO+nhf3pCZ2Z/i9/pXZjw5o+ma4t7qGrLbxmR5I4E6tux1PPFbHhSTS4QtjBdEXkSbHilIV2AHGfXiuW1e+Om+Mru4u7mM2BuFWSNow5+7xwegrSwxv/CNwx6pqdnpOJrKVkeG480M6uOSQe4yefrVLxZp8+v+JLezSWF5rSzVL6cLtRSCevPp/Kuf8Q6wNXbyo8rDHMzxEfKQp6DH4frWl4T1eSwtbrTYJ4YLu6mWSOaYZXHQgmlck7HQvCvh+Kxl+zQRXzrh98h3PjHUen/1qXxCdNm0g2lzC8sUgDGWPBEXI5Poea57UX1rwhe3F09xBdNewvDHMhAAAwcgD0z+tYy6itz4ft9I02G6llkJ891UsSCOh/LinEDI1iyl0bWjCZY5JFxIrBuHB56VlpIGlO/5VYknHQVr6vcyXV9BeXu3aF8ry8YcBR3FZsP2U/aGm3L+7PlKD1btzQ0NCqY/LZN3Ymuq1PwhG+jWniDTJzd2DruuTGAGiYdf1rkbdhuYEZ3KVrpPDNzfWTxafb3b21penypyyBg+c9F9+g+tRYGZt40q6hA4sSnlwAhQMF0PR8e9RW7QR2lq7EMzs7MueRg8A12niy1ubxo9btbSa3so1jtoGliK7lGRvGf4eg9az7IE6XqUV3bQz/ZogyBkxgnP86AM6w0q41iFp7a0FxMuW8uFzlceo7A/0qfTrk2OoMrtmKdmWWJuCh6NkfnR4R1K50HUo7mEMI5SEljP8Y5x+VdzLoFh4mtzeC2mt7u4Pm+Z12ngEdfapSAl8C6oFaWIzSyuDsjR+jpng59RXQ+HhjVtcb1uMfzrn9D0X+w7q5KJtyiqjMckHHJHp2q9od19jsm7tKclu5IJraMHa4HVE4NKGrLttQWeUx5O7GRU7XAVsZxTcbEpmgrZrg/HGh3jalFqtpbvcJJEIpVAyYyOnHpyfyrskmz0qdHpWHc4XwFdXcOoPY+TOtvICwQj5VIzz7V35NQxRxQKUhiSNScnaMZNS5zSsMaWNRsc04im4oAYaY3NPIpm4A4PWrQhM4NTrggGqpcU5ZiBxTAtdqrS8nnrVgEEVXnwpzSW4jOnTaScVV3jPIzWm6rIMVTltMfMorpTM2iruIbjpSltw560jKU6ioWkOeOBVohj2JHSmtJvHzDmm+Z60vyNVkDMkdDTlmNIV96aRSHcmDqwznBozg1ByKeFYR7iOKBllZB+NPAV/wAapZNSK5BpATSQgISqk4qFoyqBj0qypPdx9KnazuPs4kkhYKTwD1qblWMnGKmhyTjFPkt+Mr0qSBki5ddxHamIsQearA7Onc1NLNMMkKDVI3E8j/KoApsnng4JqLDuTx3DNKN+BW1azKsQJrBtrO4uXIOFHrmtKHT0hxl3JHvUyHENR1GOORQhzkZNZF1eLOACelbmyD+JAfrUqRW+MiFPyojKwNHLhmHOCBT/ADkHU10rWkUrgyICAeB2qZLG1JJMKYPtVe0QcpzCzqvRqv2qXVxjy4iVPc8V0CWFrE4aOFAfpU7Nt6YH4VlKpcuxgyWc0Shpo9oPGc0ioQMDpWrdgSRYrOBKsQRQncGiaAAkZFXEWoINmRV5Au0VlLcpEQQZzjmplIApCM9KRgQKgsR3BNIDTcUoFADtxBoDmkKmm0ATiQ0m8k1EWpPMApAWHkAFRFyajL5NIGFAE6y4HWnrJk9apl+aFk560AXS4qNn4zVZpsHrTHm+Q80WAs+aPWgyZHWs7zSD1pRce9MRcdztqEnmoTcccmjzVoAUyAHg09LgjgVWYgnikU4NMZpCVjSlyetVopAeDUpYHpQApNNzil5pGHFIBN/NSI/NQHrSjNAFotlagc5bijJxTMHNABk5qRelRd6etMCYHjrSl1xjNQk1EzYai4mSOQDmpYmBANVWbIp0cm047UAXetMYgdaaJKZK4IODRcB5ZT0qPq1VxLtbFTI5ZhQA/bSgVIBxRjFIBmDS0ucUmaAEpM07FJtoAWjNNopgDHiod5p56VXduaQEu/3pC1Q7xSGT3pgSiTBqaJsmqe4Gp4WwRQIvkcA0zFKGyvWkFK4yVUB59KCQDSFtq1XaTJoAsFxVaV91HmYFR5zQIjxSg4NPwKQimAoYmpPNZVwKhzihn4oAVlMgINZ72zBztFXA57UGtFImxVjBXg9akKluvNSYHpSU7isQ429hTGNStUbChsCIsTTQeak20hWpGIDk1keJYc2COOqtWwo5qLU4PO09/lzigDM8KzErLETk44FdA4rjPD8/laiq7sbjXZsabYiFlwc00MKlYZpgjAqkwaEbBFNFPKimkEVXMJoaTxUckasOlSN0qLcQapEtFf7O2eFpRav6VaVqkDU3IVii1vIoyVqLYR1rU3Z61E6I3ajmDlM8iipnjA96Z5fHStLk2EijeSQKBkGuotwILZFxhu4rG06ZYZACK02m8xs1lULRcGJOD3qldWJjYuGB9qswPgjNNu5iRxxWFyzN2N6VNFGM5NCgmpkRvSruFi1GOBTmGBTEOFoLnpWTLHAU8YqEMaeDmpAkpCwpVGTTjHmgCIk05TxS7acqCgYoNBRZE2v0PWnBRUoUYoEcjrXgW2vlklt7l4nPbmvOrjw5qmnz/vExF/fxnNe6YFUNTs1vbYwMcKfbNAHi50gyW7Tbndh1qG18P3V5C84wiJnqDzXqdt4WtY02yl3X64q22g2n2WS3UuqOMdaAPDHRkcg00hzwoyfSu+1nwvp+jj7QJGkbsGORXNG0hmclJSh9ADQM6Hw61zbaUolQIMZyOv5VjX+paW87hlMjn04rTtdQe1hEcoMq+wxWbqtrb3OJIIEh5yw3ZyaQHlLLjlvzqSbyTGDHEUPSn3EKtIWUd6kmuWkQIVXPbAoGUQtOp5Qr1GDUtr9nEp+0jKY/WqTArClHJpZtnnHy/udqu2cFhLGTNLMs393bmmBSPFKGZRxT5hEm7KsAPWtiKyGpacJoVSGONfc5oAx4v9YCelaOl28F9LJE179nI+4P7xrP8tpGSFPvMcV6K/hTS/DtrZ6izt5u0fe53E+1AHC3Wj6iJiqWczqP4wh5re0RrSXSLnNn5Tov31HLGtqLxjbG0ngzJJO5IXK8KPasAWl3BpM0tpHKUDfOsfQGkxGC0s9zIqyDitqDff2P7mEnygQ2FxzVHTtIv9VUyQx/ugcFzwM+1WY7O6jnnh0/zQI8eZk9fWpaGRQzSQlgpwlV9kgl8xVyp6VcslgYmKVJJJT91V5yavLFrFrOtoLCZEJxtdORUWAybQXCzCGKMPLK3yqe5q6DNDL5UiyxSlvmVei5rf1rwtLpQtHnd284AmWNdoTNei3vhq21fw9ZwQRL5rJgT4xtHck0xnkOrtDABHFK3mBRwKtaTZTRISlwiXky/KhGSBW3rPgy58MRNKZbe9icAEsMsPoO9aXgzQoLGGXVbpRJcMp8qED5l9OKfKBy/httPs9cmOrMyxNkb+2fcUa3NJqspt7C3jisoT5kRA2l/wAfWmzSE6xK7Rqo8zcY25IqxrGuWsssRlsljUfLujYZouQY2o70haTZliQBuOcVW03T5tRkLySpHCv3i3+farmqIq2QdJC6uMjNZNrM1uxK9xiplqU2bd3Y6XBbqtvMzXBOAQazZraeBiHSow0jhbiNThDncBV+O7a4sJPNIG0ZXAxzU2Ay/kc/P2qKzVp9Uj2j5VOc1ZuUh2eZEMbuo96u+HNKW/juJEmfzkXhP/r1ohlPUpY57kRouHxgnPWm21mIWzKwU+hNTXkQjjRWjj8xs/OnBPNQR2nys92sqwjoS33jTaAtzujqhhXHXOWzUSSeck0YHzbevrTJRbG3Q24dNow3OM1JbWkttaLPKu1JDhT2xSsIrQXkxt4oifkjbOKZJZrFP5qn/WfNgDgVZWFftCvjEJ+83YVYUQyRFCcuPuntip5gNmDWCnlWUMWESPgS8E1m6tcSX2nRQBUihRtzDPWmmKZnS+uZUC/dXnqK0JbOS1aOWNoZ/VM+tSBmWUW/YunwSOif62THQe9aFwI7OyiOfMknfADDkj1q1Lrw0zw/qdrJHFFeXcnCKvGCKk8IeG7u7EeqTiI2+3CEn5gfUVqkSZTpHBAWeH516BuCKq6Y6vrCbzjchx9alksWuNXvT9ok+V9qiXqw96zryP7JfNGHVinR1/pS5QCdf3hLff3bWpsUqxq1vcA7M5Vl5IrdsdH3Wbvc8h+Tkc59TWXdWclooZzGwP8Ad5qUMoIkf2jkEJ/eqRmcxbF4HrT/ALFc7EmIUo4zwelOAZYX2rnIwT6GqYy4slnOMTPwq9MdaphEdJHUbUU4WoyuIFXuTg1OzRIphiOVHtU2AJLkSKAVXdjBJ5qzDfXUUSxRSHYnzAHqPYVTitpbi4EUKBnPQFsVZkt2t51SdflXBYUWA27nVJ7O4D2tzc2zzx7mMmH3H2rn72a8vJXlu/Me5f16/lV68m+0a1bssWYwo28/nV69hh1QvOsz2rocF8jaDQmLkIrzwg3/AAjkOo6bePcKVBngkADq3fFc/K0aFkjDsSoU9+c811tjpbiJd2pibzeYsHJYd6ym0+JYhJHI+8tt2g8mjnKSMGSOW2ZHOVYjIIrQ0iVpVuElCy5KkLMOOtTTTi3uoneHzI4xjaeWzUmoQx6gz3cErtJt5jVTk49fSmpBYr6hHFa3A2W5Ecg4jVs4Y9xWZKuwKzxglDnY1TWUTS3G1SIpAMgE5qa5ilWKSaYKQpxsYZz9apMGV5b5ZvNAtEiL42Z6qPY1qabpdzcrFc2EAEkQ5Zz8rn+7WPdSx3CB0c7h/wAs2OStPsLi8jKwW87xxhxIxzwuOc1aJOn1KTUIZ5LKR7VXkiy0UR4UHt9aq2sml31k1olq9rcRpnzhIC5b0BqtrF1YKM21559y7Zl44PuD2rLeNThw2d4yR6VXMTykh0uePlhCF74kyf5U5NPuC4UNGo9C3J+nrUCxyGMlFyq9T6Vc1Kbz5YJI1YlEwSO1ZlEV4sSuqoAhH3gT1NLb54hQKCOBk43e1Uyd0hbuetOSRkuQ5G4AcfWhoZdEq2l+jhf3gOCM9KZI4tr9fNOcnL/j0qsi+Y5c9T1NX/ItpwFcjexALseTUtAXZYzK22RNqN/q8HjFVmPkWbWrDEobco9RUhDeU0u7PlgKpx6d/wBaqR20rNhUJkboSMEipaKuaErWP2JLmHzPN58xVPCn1FLpEuq2t093BO9uj4dg/IZOe3fH9agjt3VXt5MK/dSaBDNEzTRF3i27Cso5HrxQhM6Xwnq628GoRajIradJlluQuGEhPGPr/Su0vvBkmtaHZ213cQQXgYyvclcu64OCfwNeOCTbayW6OxV2DbT0GK6i28f6unhw6H5QlYxmPziSW2nqMVpcVir4e8N3OvarPD5/l2sDbXnCZDDnp9cV3N3pPha0046a628U0iDaZyS/HcE55+lefWWq3FtYSafC0iea4OFOHU9wfatQWbeIrhpXnlju4IstLOfkIHT36GlcVirdaTfzXogjLTWasVEx+VRke9dJbeG5dDtrfUhqN1+7iMz7IQwRh6+oqhczXpt1068uDIIMTwsQDG4AyOT1rQ0nxcNd0y/0u9ke3a4i2RM/C7jncobt2wKcWFji9bnW7nlup18ya6k8yOYjGQABgDtWWW8qKNiAUY4yDXR+LLC/t7mzgvWYpDAAjrCwTd6A9/c1jwlRAvyqzds0XBFvSo9MurWSG7ZA8WfJc8ZHPJ9+BUVjcDS9Uie9hnEW0kKpGRnoRUtteRbdk9tGSAAJAvP4joa1bKAXJXUG8lrdF8sq3JwPb+lRzDPRfCE8Hi7w9/wimquZGtk87aSVO3PyDjrjIP4CvOb3R9Si1fV7C0V3t7YM/mSqVDRp6Ej3rX8K69/wjXjJ77VQ7JdwmCK4QZCdMHb36Yr1612+JfDezUoikxZ4Jtny7sHBx9Rj86LiPn63uJDbOvHGPvjBP09q9w8K2KyeHtPvIoPJ863XdHjqe5/HrVzU/C+nXtra2EFlHGguEcuByqjqM+/FdBFbw2tukECbIoxhVz0FNMLGHfaS9yrOFwwUk+9cZpgMliihSWXrgdOa9RHO4f7J/lXAeFLOS5sJp45AjJKV5HUGt6cyZRK9vMsF4pPYVPcSq8hdTwa27jSVnXDKoYfdYDoa5y6t5rSXy5l2nt7itou5nsWodQCLtIPFWYtR3uAKxM1IkgWrdNE8zOqinSTGDVgVzlnchXyWrchuUcDJrmnTaZpGRYKZFMK4qRZEPQ0rYPSs7FkJFU70lYgQOc9avGq1zH5sRWqQGYku49asxyADBqu9sRMr8jaMHjr70OQsRcHpWtrkmgJQOlUrm4O4kmqv2sgZzUDymTOauNPuS5FgXPfNSpdAnFZhzQrkdK05URzGpJGsnSqM9synIFOjuyBgnNWFnDjPB+tArmUwKnBptaUsMcgzt59aqPbkHiqTJIQzdKfRGMSDdwKfIpeX5cYxT2AZwTV+0UuoRTg04WdsOjMT71LDC8UmVOAOgqJMsz7qEQykLwPSok3u21Rkmta5sxcMrBtrDrxnNWdK0xFuN0g3elTzq12HIM0fSQGW4mQl+6k5FdQYFniw3U09IowM4p64HTpXNKo27s3sc7qWgzxRmWBw3IGwD1PrWbcaVeWspWSMED+JTkV3WQUxQFU8EZpxq2J5ThYhHG/z1OfIbmrHiHSZraf7VDH+4fG7aclT71hec46mtovmM2rGk6HGYZQp9CM01ry4jQZUEetZvnyZ4YilM0jDBYkVdhcxdFy0j44Gat2zyFtmM471jhiDmtnSbgNNhgR71EkOLuXgGA5FN80KfmYD6mtHy1btVK60y3mmLlee/oayZqgW9hUY8wE+1K15G3eq50i3RC8e+Nv9k8VVis7k3IjYZTuc1NxmorCXgVHPZnZuU59RirVta+XkgdauLahh8w4+tTcGc+jEMMVdjdimM1pnToOvlgGq89hMuGhTcvpmm5XASBsLUjciqsYkQ4YbTUm445qCiQIDSMoHSnKKlWP1FAEaMjJtP3qRrY4Jp7RhG3Cmic7yCeKAKTggkUw1qpBFIct1qhdqisQlAFcuKb5lMam4oAUyc0hkqN926jnHSgQrSn1qNpDTTmmHOaYD85owTSDin7wKQDMEU7NIXzTetAD92aelNiUZ5q4kS7c4pgQqcGrCHK1GY9rdKkQDFICQUpGRQOKeMUhkXlH0pRHjtUoqVFXPIp3AqkH0ptWpEGeKjMftSAr7TSZKmpiuKjcd6YCk5qGUGpkXIz2pJEwmaBFIsRR5hpJCM1HuFMCx52O9IZgQeark0wnBoAmDZarULYqipqRJMGgDS83FJ5ue9VDJx1pBJSAubyRQGwOarCXikM3qaALYk96XzKomcUoloAuZzSGoVlHQmnh80ABPFQPyamY8VAwPWmA0R0jRkHgU4bgakVvWkIgANSqcU5lHWm0wJkkORk8VOJAR1qjnBqQNSAsSScYquX96a7Go80AS7qlTpVcVMjYoAkI4qMmnluKiNADSeaaTzQxppNMYqtzipN3FQDrUinimIXrTW4NSA1E/3qEwsMNNNLSU7iEqTycjnikUDcDUu4UXENSIA5q0sKvEyEdQagz6U9HbOM0DPPHjNpq5wMBZMAV3CfPCj/3hmuQ8RRNbauSe/IrptHnNxp0bN2GKpiLNISKUjmm4qQFJFJkUhpMU0wGOOtVnODVlulQsATnFbQIYK1SBqgyAcU8MfQ02hE2aaQT0pUBapkTHbmo2KKjIR2NRE84HWtyGHdgN0NWlsIP7tHtBcpkWVmXO9hV/yNtWHRYuFGBUfJNQ5tlWHxqNtOe2MqkClQYAqzGcCs7lWKKWpj7VIFxVxlyc0xox2ouFiuRgVGwNWGTioHBHFAyHfg809JhmoZFOCagDkN1oEayuKlDg1mLPx1p3nnsaANQhaaRVRLg9zUvn+9AybJqQMarCT3p4f3pCJsmmkZNNL+9G6gBrYAqvISykCppOQag5B4pMDzvxPdyGeS3+zE5/iPNcrb6hJYuQhAceozXsF9pNpqCEToc/3gcEV5h4h0iG21Ax2z8DruoGZF3qN3cOGeZeOmEqo8s0j7mfcx71Osy27FSoP1FRSyJI+7IH0FMZyS4PXpVm4aJI1eNNwxg1UT7uaJHOMUhkMhyxNRNU6IH5bOKVodynauB600AltYS3SkxDO3rV7SgsErvKcY+UDHWs9ZJI1ESOU3tgkeldvqFtbWXh6ARfddeWPVjTA5+6tYJLuISSKyy/w9x9a6WOx/s6CKwKAW8xySeorjY45Gk+1AF0QgnA7V0t74otL+3iWKORLhPulhwDTAtW3he1tvF1kks6/ZC24eZwAR71tfEK/iuxBYWk0TsnLYcHaMVxF/qVzqWwXt6PlxhVHArPmhgMpMcjlfXNAF7SNNeTU1AYOifeI75rttG1aCwXUNLRXnnuG/dxqOG+tcBFI9qrG3ldGYbSc54os7y7sLj7RbzlZsEbjzUAdVPo2r+HbQytIDaTEl0iOChPYVEum32j6TPeyw7ElGCHky2D+FZ8niG/1KBLe8mM0m8EOTj9OldZ4vaQ6XaoW+Vo+R+VBJa+FmkWdxbyancRLI6PhGPb3rt9WtLbVLxd7MvljCOnBOeufyrA+HskVv4X2SQsu5yVIHBHrXS+ZannJH1FNoaZZmsoLnTjaSmOVCACW61DZeZYWy24UMicLg9BVSW5t41Ll9wH93nFRprEA/vn/gNKwrktxEt7qqTTBmSLlcDIzRf6ZFfv5ltGYbjs4GB+NSWtykqtKJUKMc5z0rn/ABX4lnt7Jo7QFFPBk/iYe3pWg0zjPFNxC+pRWsdky3S5EkinhiPSuK1FJrWfZNGyOT0Pp610the/aLiS4v7hIXj5UvyT71k30y6jdmcyBSjcE96TiSZaSvIwVySp7ZqZo1VcrV23tbjVbhjGo8sHG8D+lVXjkhuXgkQqVOOazcSxoDJGf3v3uq54NOeVfJ+Q/e4296BCzSgDbjvmm+WDcHaflU/nU2AbbSeTdROwyitlhntXXaRLLJrypaIgWYYAzgOK5EwvPdLDGMsxxXYaN5Vv4g0oiRNqPgnPHStEBNrHhN59W8vT4d0kWGmYHgE+lYGtw3Ftqv2OYgRLhmwvWu8tIv7U1jWWtbh4p7d9yENgE89a5TXAdQKzTGSO+Y7NgHDD19hUknOvAs8ohhYBT/ETW7qPmfYYbJOUVAT+FZus6O2iQQB73zJnzujTov8AWoNKu9j3KzSMVCDbn8c1LAv2yLcxRafGpETMAxPcVSvLYWd5NAjMwzkZro/DsCXml3cqOI2tct83RvrXP3syyXckvlqkhY5A6CoZSZo31pnw5bRJcIX4zH3Y1NpK2dxYEXcZ+0QDCg9ag0GyuNRvt0SJtjzulc8LTZ4bmCaa5JBPmbQQetIDP1RpNZ1MrbwsHC/dbrXReH9U1zT/AA3N9n0uWWEZ/e5ACj6d6eixWmlCaKCP7W5I3lcnn3rUudGubvw7Dp1rfwxIilzhvmYnqK1jITOKtLd7qWCGS4Q/aJN7J3HrU+p6P5WpTXfku1khxuQ8ZqHz7a0ijaMMLuM/NJjGDnoKlZNY1FLaOLMkUxIjQnkHiquBnzanNdzbWLCEdgetOjae6nkUB5CF4HXApk2k3VpqH2OQKJcgDByCTWxpllPaagN5CMQck/xD0rNlFCK6hOnyru2vFwq4zxUJm8vTREw5lk3n6dqn1W2S2uY54h+5mJYfWs2WeS4uIxIfk3c00IkSNmcb/lX1q5Ywxrqrh0EiIpbBqvcSiK1iVupc4+lLFcMs8kq9DGRSAs20qSSOyRGIE58zPI+lPuVMoJUlmb1PWsmO6YjYvGa0rYzGWJkRikRzIyjIFMC3pcayTXbMOIo8KPepYY7e4tLuKPps3P8A7wqGCZUSdoSMBizY61nw3zgmWPfGWIO76VPKO5blX7NEI4jtkjXI74zUFhOxDo5z1YH3pTPC0ktxIfnYY+lStbbEWVJC5Iy2ewosMiCIVLX0ivGVOcdj2qjBdy2yutu/lK+Mg8nirUR+0xTK/I6KfeqZsZtxyMYIxLjg+1NAROzQkSL94VZvbSTcknqoP50+ZY52BJ8p8fNu6E+tW72z8nREuku3d04xnII9MdqpMgxWtZFUvjiprSz+0+cFuPLZFO4Y61LNFczWq3MkqIuMLEDzj1pYkZtOlVdvDdScH86tMCoUby8MQxjHYY4p6IrKkYfY5/iY8H2ppYRkqRkilQq7bpBkUXA0Tpz27xo8yPuUNII2zsHvVmXSLk+TFbq8cx4c/wADjHXNVIEiZttvACxGSAcZHpThqlxjA37B91PMOFoApC1n3spTDK+wj3q02lXVvud0Hyds/e+lQefcDZ9njYbDk8Z4q6dQmv7gmSU7TztHABqRjYYMyPFtSONcYHrmrKRAL5QSNCjZGR94etZ8zyz/AHSWdm28nrW1YaM/2w2dx94bSXJyFB6ZpARz3WEljSOJMnJJ7g9qigvvI80OFdTgLIOQoFW/FF+yXU+jxxBLeABUKDnPrWhJpdpe+DWuzbmKaKNWiYHlh/ePr9OtPlAzo7G2F6J3lX5vmJc4IB6Yq5daU9xaTXcc8bJEvAXktisV5Eukk8tJH8pQFY+lTWtlKkWxriWMyAuoU/KwrMZkBCj5fCn0JqWKSKNbiRo3LldsUitgJ7n1qS+i+130EVovmSsvzc96mk0a7gtpJXMRSMZYo+QBWjArKLmGVZnDq4GUkHJJ96tXGtST28VtLKQgbcSo5d85wSKz1jzavcKzMEYKwPXmtC60qWxjtprhP3VwodD0I+ooA0ri+j1Fz/ae+1KKEWNDxgVlwT4dIrqWf+zN/Jhx1HQ89ep/OpkiudVtbzUF+dbZQ0xHUZOBU7WMtn4Vs7ucxzJdFvKizgxjP3vfp+lBBst4ps59DudLVrkW8g8oJIS/y/3wSeD7VD4V8Lwa9o+pzyXLxzWa7oRHyTgE8j8K5/R4ftt8lkkJkmnwkZGflPc4HX/61d/L4K1bwsovNJuXuBIPIuUUZPltwTj8aYzlNB0mHUNAu9cu2ka2tJAjQxjDMCBz9OagsB5XmzwlgjSbI1kPDivXbTwnZ6frDmxhVbC+tUEibcANHjBx6t3rHn8OabYaQVu8PHFcyzLA7hWIJxtGOT9KloZytzo1+bWxuA8E0NwS6neB5R6c5Ndh4M8cJZNc2erXEbt5vyXC7sNx0wAcn8qxfF9olxFbxWdja23kKssiwL+8QHjkVzNtK+nsphmFxGH3sCoG3HSoA9rfx5ph1SHT7QpdzTkCMxSrtHrvJ+7+tdKlx5ibt6N/uNkfnXg1vPF4q1P7FdeXvMZeK5ggEbA9TuHp0r1Hw5eiGwh07y8pbpsWaIZjbH8jVoDqftCqjsT0U1yngd8aTdj/AKbf0rXuJM2lyc/8sX/9BNYXg1tujTH+84P6VonYHqdUSMVzfiSJcQyrxj5cetbqvupLiFZYGRgCD2IpxnYiUThB0zR3rfOlwLGY1XAHSsi8tWtpRzlXGQcV2wncwlEZG2OhqVZmjIwaqgkUu4mrsTc2LO83Sjea2UuIyPvVyKSMhyDVqC6YNkt0rGVM1hM6N5lAzmmiQMtZUd6XYKeQalS4w5UnvWTg0ac1y6+MZrGkPlQzJkkNzz2rUcs8Xynk1j3hMeVc8sOKqnuKexUMnFNEwHBNbGhaEmsQySzyvGittATqfxrp7Pw1pVp/y7+aT1aVtxrWVWMXYzUThOtIBXTa34ZaJ/P0+JmjOd0YAwuBXOKaqM0yOVkeaeGqc2zSKrIhOR0PGKrMCjlT1FMLFpJFKAA0peqitimafcG6tXkcYKOU+uKAsWWjBHHWowCrc08kAbj0HOacrK6Y3r0z1oFYtGaOSBFZQHXo460NLIG3Fsr9KpbqlSUshVjxSsO5oxESxl1f5umBSLcXNrjjcB3NZqylGyKkNy7rgA/nU8hXMdjbahFPCHU+xHoanWTdXHWd89qeeRXSWtyk0IdW61z1IG0ZmgHJPJqZGA71SV+alEuBWFirmgxEsZVuhrjtb0Ly5YhYwsxYkMB+GP6106T8UMwY5qoTcWEonnMsckErRSoVdeoNNDc13l3p1reTieeESOBjnpiq50uziB2WyD6Cuj2pl7M5S3jLSjiuq06zgFsuV5qK1sAJxuFaoiVBhRis5VLjhCw5EjXgCl2Bs4FNAINKKxua2IZIi3anw2yD5tvNSGgNQFh4XFSCoRJTw9AE3WkIpAwxSkg0AQyRo3UVXeJQeKmkJ3VVkchqAJBgdKkVxiqe80nmn1pAWmcE8monZMcDn1qAyE96buJ70XAl85x0NVpCWOTUtNKrTArFTUZ4NWmUdKrMuDk07gJgE5prHikZwBiomegQN1ppFJvFGaAAikNKTTCeKAG5wacp5pmaAwoAtRsBirCz4GM1niTBpwloAuNNnvQsuD1qp5me9OD+9IZeWXI61Mr1nrJ71YWQYFAFtSKlBqosgFSK/fNAFoUYqETcUokyaQA6e1V3jbPSrJOaKdwK4RlFTBQ8eDQwyKaZCgwKdwM66ixJwKqnir07Ekk1SkOTTEM3UhYYzmoyaZk0CJd1G/3qEk03caALPmZGM0okqtk+tLkigCyZuKaZPeq+TShjmmBPvJpwcioA1ODGgCyslTRyc9apbjTlkIpAaG/NBftVVZcjrTw/vSGTZpCcVHv96a8gxTAUzgdTSCfNU2fJNN8zHQ0AXjL3pyzA9azfMPrSh29aBGmXBppYVR85vWl880WAvb+akWSs7zz609Zs96Bmh5tMZiRVcSc9aeH5oEKWNFBIozQAuKUGkzS4oGOByaUjINMxinAmgCNlpuMVKRSbMmmSNAOKdzT1UDtQwGOKBjA2KXdmmnrSDrTEc34uhZnhmH3VG0/pU/hObdA0MjfNjP8AOrfiKLztKfAyVINYHhy58q+Vc43giqQzr2NNqRlwcGmAVIhuKXFSYoI4oAhZDioimRVzbmkMQNXGRLRVjtwW5qbyEHapAgHSjHvTuFhgT0qRVpKevBpXCxajIABqcSj1qlvp4aoGSSuWNNBIpeKQkUhihznFTxsRUK9aeGpAWw/FBbNVfMwetOEo70DLAGetNlVFTmmCYDvUU84KEZoAqzSKMgGqTEmpXBpFjLUxEOTTskVaFv6077MmOlFwKwmb1qQSt60ptgKaYyKQFlJc96mWTI61TUYqZOtFwLaetSVFH0qUVIDTURBz0qcrSYoAztSjv2tT9h2b++6vLdbimQZurjdL3C9q9YvvMNuVQkZ9DXH61pNjFoz3F0MzertzQM8zlXe+RTxZ4GZjsX1rX0Kz+1Su6W6Ns61JOq3N8sM8SiIHLY9KBnm6JyFps0YRsbgfoa7e80eLVtLtfsssKOi1zVzZw6d5kNyA0w+6yng+tXykmMHC08SAip44N9sXNnJIvd1rRgtEu7Ly9PtTIw6uw6fWjlGY0VvNczK0UZZEIZm7AV03ibU4zb21jGcRKm6tTSbFrbw5cWa7WmK5wDXC3UM6XDB0fj2o5REv2uVITCkrCM/wioVJFRqPWpO1Iq4uNxFTELHwTilsLdri/hiUjJbvWlqml3ba19kEBUZwCFpk3M1RI0q+VG0megArVfRrgWH2kIwbvG3BFdkfC0FnpUElpI6soDMSBlj9avt4bsZrEXEpuZJXALHfkH8KOUOc88sNNeSAXZlRWjbmM9a7XxBdw3+mWiwqz4QFmHRfxrSg8L2Fra3hW2Vw6Egls81e8O6Zptzo4Wa2R2zhhilyivc2tBeKTQrVYSpAXoDnH+cVoeU/pWCmmaPaXHlrM0THojS7am1BLDSrKS7udWuoIkGT8+SfpxQyka/l89B+IzVa5iEh2bAfoMVw3h3xTq2u63LaW2olLVOjyxgnHbPH1rtlXWI8qYLKf/a37c/hSBor3eh2l1GI8PG5/wCWitjFecava6jq+utp8VwTbw/Kkm3t716JdalqlorPcaPCYUGXaObkCvPk169/0p9J0qbyZGOJT82w++KpMlGXbWcdtqn2FiLm7UkO8x+VR6iqfimyggnXyZkd3+8i9FrGuEP2x2kdy+47hnvWhYJpiSM1wshG3ghulNsBYdauLKxS3t0WN1/5ajrVVJ5JXZ5T5jscln5NLfiOOcqlVYnxIBUspIts77xs6UrPt74JFORAyOxbaFqtGpmmbPOOn0rIo39Btrdm+0uX8wNtC7uDVrXLSO1uneGHGQGYKOprJAWCJNwKA55Nb8WqyxWqwTKpXGMtyWFSUW/AV0INXlSZSIbtdqluASO1X/HGuW8FwkFlbGa7hH3xyIxxx71mafbgmOOK4MELvlJCeV9cVs6t4ciTTbpLSUszxF5ZXOd5HPFUmZtHlVw73UjSyOzuxySavaRpUt3Y3VxG4Uw9j3q1pHh5rvTrye58xGi+WMDjJrc8L6fLaaVfW9+ixI2cuW5HGK1DmI/A2s2lha301+ybXI+XqxP0rM1m+OsapJeJaeWjABNo6gevrWaJo1uXg2hbcHBI6sPrW/JcxOltaxTJBEMfPt5H41nIEjHj86JGCvNFu67WK5qWxjUXcUbeY6nPQ10Nzb+e8cc9yj26jAdT9+q/h/T/ADvFCQuxjQKxVx/Dxwc1I7kRF3darDpyShN/+r3dMe9dl4fgkkuLnS9QVFlRDgx9fqDXJ6ZZNB4wuBOzuLRWYuRnBA4JrYjdBp8WptMyXEt15cjgdQatRAz9Q8P217qF3LO+yytk2RunAlepvC99bxWMMIg8+4gkOPn27Ae/vW54lso5dT0rR7dN1uD5jLnk/WsK/wDDhsNSktRIYYt+63kC8EehPY1pYDmNSjuf7fvmVS2JGfCjgCtOz1JLcIZog6MMh8cin+KoxputSrany0mhUv3ye9Z95qkN1aW9ukRjihGMk/eNZSiNMua1a4WSeNg9sV3mMrwp9RXKQnfcJn7oOTWmmqzmCe1Z+nCHtgf/AK6zvKcMAB1pICXUQ0syMGBTooHanzjbtGccYqFF8uQB+KYoDIxdjvHXvVAMBWOTAOasPeu0TQp8qNjKrxmlX7NaWKXaXW+6zgJjhfqKqksZDIxySSc07ALKOC6kr64Na1sVPhdvNP8Ay2yv0zWUZMxmMjg9T3qx5rLYLCeU4bFFgEfaItobce5rViKjRY4pZf30zHaMc4+tY8Yzs9zin/MJY33f6s5UelQB1Vmmnal5cTIkNyo2MFON2KoS3RjEVnI2EBySeuewrKzI9yHj4JbOfSkvDuut/Tv1NTyjTL90IDG5kjLr1JA6VjbCcKuduc9aJHbLhC2G601GULtOc9uauMQbJtkzcmMyAduwFI7cfKgx6NTGuJI4nhVsb+pHpTDKzY3c8VaQgbBbL9fY0kjAEBemKdGwXLBRuzmh8NkkfhQVYtQvLFam7iONjbW9jU1uqC3lmlZY0yMZPJJ7VSkk8u1aBHLCX5mz61NMMRIpOSRkmgVieXzFZRE7KjKcYqnapIsvmbcoO9XbSaNFDZYsAVGeQBST4ERcEH6DFQIfpTNBqEMoj8xFYsUxkkV2eh3ulm01C31W9ewuLh1KTsnLIB0+gNYM+gNouiW+rXV24lZx+5jxkAjP6d6517+dnLoxKnu53VQG9qlq4t9lpNHPDES5nBwXJqpHqGoR6MbN5m+z7cqh6D6VDPKtyijzBEAo3kcbjUSx3K8rHM8QIJfbletIDb8P2yPqot/tCMrwsSqgnPt9ac08FrPK01rO9ovyxbJcFD78V1unX9/aQ291Hpck0MkRcgRIChx/e7DivNNQvHvL+4u5DtM0jNtHQUWJHLcI13cXCyyRSD542bk596lkkAtW3S7vMHzcEZOapwOiuZH3cchgM4NdFH4h06506KG+3FwoEnmRZUkHrximVcwjMi2qSQqQed2futW7rd7rFro1lpmqmOZZFWdHYDeo7Ln/AD1rL0yKO/ee3aVbZJgzxqBx9PxqtKtzdLFI5d1Rdm9myAB2HsKRJ2/gW7tJ9L1Pw+oSK81O3ZTcMxwSAdo249z3rH1zT9Rsm0vRbuaN4I48JJGcgjOTxwRjIzmqWmXAsdx8tHYj5SRyp9RWyWbUbmw1G6vxDMHd90gBUtwMY7DgUwN74deH7OYRasXiF5DcFYgHClkAIORnJOTXpjEj3rzDw1eaf4fu7q6u4If7Tnk/0RlbcqqTycdq9Ss5jqGnwXTwGGSQZZCMYNUMrS3EkZEavGksgIQOfvH6d64rxH5ttqkDSNJfa5OuLZVTYkf+0D+tdnJpYbxV9vmG5I7MJFnkK+ecfnVTxBZRx6S+qSPtubBGeGQn+JsDkd6APOxoLT2VwbrUTFrwYmVZW+Ur2HvnjB965qawg+0QwpLMh8kF/kJxIM5rY1u/PiLV1mt3S3cKlvIkjcnnGQfTmrd5bapoNqljcmzSZVLxyiQSMQTwB6Hj0qZCkZijT/J08mS5W9KZZ0UkEen/AOquk8LeKtS0vVzbajGW064mGGWPaFduF7cDiuWvr26MNvHfztEWUtEcKSoPXnqfzqxplza3s7adcXr+TK0ZSZ25Ow9P1rMGzu9R8W/YprvTpophdsWKGQ7YgpHQHvgfzrX8Kny9EYf7Q/8AQRXJ62serPAL2CRG09SrR55ZTjDg9+g/Ous8MRJFoIEa7V3dM1rEZtLNzVgSgjk1QVlqwImYZHSnYBspGTiqksYkXaw4q48TAZqFl9jVxdiWrnPXVv5UpOeD0qGJfMlWMfeY4FdQ+mx3cfTn1qSz0S3t8NtJf1JroVYycDm20+483y442dvVRxUzaLqCRCTyQf8AZDc/lXaxQJxxVj7OhHAqPaj9mefrDPbsGmiZB7083CM2Qa6vUtJa7TbvVPc81zt9oMtpFvibzfUDrVRqRY3GxPYzCQbc9KvSaXbXLI8y7tmcA9OfWsGy862ZZ5oXWL1xXSi4UopHcZrOoVEtaeI7RNoCjJycKBk/5xWlHOsjYFYPmc5q9ayYw1Y7l2NtXAGGPy1zet6bbLLbyWypFkkMFXrmtb7Qrd6qX22SMZGcHIpxYWM6K1QHOKn+ywkgmIE+uKFNO3+pp8zCxn6sEtY4fItwWdvmbHCqOTXk82q3BS5lilCo07IBHwME9a9TuPElnBqQsW/eEjPAyK8nl8rUoJoIIEidHeZAnBOG+6foDx9KPaMLHU32ofZdFYFs3HlD2AJHHNcZFqXlruZ5TKn9w8YrVuILy4tS8p+d1w6+y9Bj1rFnlhgl2QqEwAGx3Na+0ZlyHZQa7ZG1jwXOBg8Y5rTtJlurdZYuVYZrz2a3mkKJAckwhwg6sScVv+HdRk0pJbK5i8ttpYsW4FaRqXJcDqGVgM4pqsQar2uoLd4C8DGSDVnbitUQKDmrlpdm3kUE/LnmqS9aerbXBx0NKUUCZ1VvdrIoc8e1XI5A44rl479EkCpF8uepPNbMUgAyTXJOm0zaMjUBqQZqhHdx5wGyatpJuXNZNGu5YHNIVxTFfBoZ8ikBEWw9SiQGqrk7s0K/vQMt7hSbgKh8zjrUElxtNFgLbSiozKM1mS3pVuTTft4H3gwHrV8rJuaolFOWYetZy3AYAg08SYqHEdzSEo9aesq+tZgnHrT1nHrRYZouciqE7BWNP+0Z71XnbdmpAaZKM5qvmnq+KYiZuBUW7FBfdUDk560AWPNpPNqqXNNL+9MCy03NQPLmomf3qF2oAV3Oc1GZPemMxqMtTES7+aXf71BuNKGoAm3GlzUYNPWgAPSmk1I2KiagBN1AemE5ooAk304PUINKKALKv71MklVFqRSaALok96sxsCKzQxqaOQgjmgZe6HJpwYZquJvWlDbjxSAshqeGFRKCRxT9hxzSAnGCtRTKCvXFCkioZpOvNMCnccCqL9atTNuNVyMmmIj20eWM06jvTAbsAFNKjPAqUjimcUAJtphWpQwFR3brFYzS7grKp2k+vagBoXNIRiuUh0nXrjEskpi39ckA47cV1ccflQqm4ttGMt1NMBM04Gmd6UHikIfmlzUeaC4XmgCUMRSiXHU1XM3tUbSZY0wLLXHoajaYnvVYsaTcTQBPv96NwqAtSbzTQixvFJvqHdRuoAl3n1o3VFuo3UwJg5p6vz1qvupwalYC2r89anQkdTVFH561YSXGKQy1uJp2ajByKcGApASrUyoarrIM1YWQYoAGjpAhBp5cUzeKAH4o4qIyVGZOetICx3xQQKiV/epQeKAGsvPFM21M2MVETTQXI7iJZ7OaM8krxXn9k32fUoz/AHHxXogPX3GK4DU4fI1OYYxzkVSYHebvMjWTOdwzTaradN5unRHP3RirG6gRKKUYqMMacDSAdikpwNGBTRI2kNKQaYQetWAuKXOBTC5FMLmqsTcmyaTcaj30bhRysLkwlPc07zR61X3r3NNLilyj5i6j7jVheaz42IxVyJs4qHEpMlKmmYxU4I21GwBqSiMsKgdueanZBmmGMUCK+RT1cCgwjPSnC3J6UhDxIKXzM0nkbR8xA/3jilKqoJLoAOp3ClcYuc00iqTa5pCS+U1/GHPar0LLcrvhYOnYg0XAbinoKhlu7W3fbPcRRn0ZsGmHWdKjHzX0A/4GKdwLynFTK1Ya+KdFMvl/bVHuRxWrBc290m+CeN19jSAsbxSFxUJbaeaa0o9aQE7Ed6pXlha30XlXEW9PTOKl8zNG+gZSttCsLRSttBtU9Rurm4tJV9fP+iBU/iyK7HzMVHJO8ETyRDc6jIGetAHz1pqwwfOqSTcYwHIxXU2XhnTb+1jnlhJ3nBVjk/jWLZeF9VjZnTeuBlSozu/Wuy8IafeqxuLt9yD5duK2RBZ0/wAJW8LeVMrLaD/llnIrfh06xS3Ntb2scUZGPkUZFWgPNYL/AA96HuYocR28Rmk9AcUcw0zyx7eXwz4kNvckmKUnBPQj2rM8ctbz3VtFaANJ0wg6nt/Wum+KFhcPBFqGcsAQEHRe9ec3l/HcRRSopE0fUntVCH3fhy802zW5vNse7OE6nP8Ak1mHiunuLptZ0JLi8uEjManbEvQH1NN8L6A2sM9xEwkeAglH4yOanlKuR6Zos4tbbWbdhKsbbmjHBGOK761sItauoNRiAKR/eJOOapWln/ZtzcWzxPFFPwqsBjPt+dauhQzWc7W+8NA5yuO1a8pnzG24AIwMYHSopTgCr32ORz2FNbSJHYEscDtmpshlaN9waM/dZcGuK1K91fS9RbStFG7zPmDbclf8816C1jsHYfSq0WnpDK0sYCuxyTmlZFHN6Z4Yt7HGp6xcvd3TfNh84U1V1Twpd+Ibxrm6uvLt0X92q44rR8QeJNO0xkjkYzSq33E6fjUcfiKXVNLd7WGO2h2n95I3A/Sp5RIyfhnpstrqups3KbTGG9TmuovPE13beZY2lm7zR5BlkO1FHqa880nxfqWjySww+VJE0xZ5HXJb6Vb8ZeILzUYbWGEiNJ1+ZEGMjjgnvSZVynda5qU2nXaTXZdSx4UnAz6V1GkatPofg2Gzt7CRZZSSZWwUAPc15kLaQ8CVN4/gzzWmfEOsCPyncMAu1ecYFTcZHqNlbRJPLFqC3EztuIK4yax4LiWGYOjYYdKsuGMO5xhz1qnjDZqblEskjSvvc/MaZECboY5FMLblqe1UsjOuN69M0CCWQ9AaW1maOVyFDHHQ1XI5JyOTVy0t5JFBj/1rnaF7mpaGSea91eJI5woGNo6VelumumBKsu0Y5qtJaPYsyz4V16jNWLQh4TJghWGBkVLQ0xI7zyWVUfh2C/SuvsNVXTYzp2ppI1u7Y82M8n2rzqKTyJX+UNjK81Ol1NNNFCXYhnACk5AqkhM96fQ9NXTI40GIfvbjySK8m8T3F1e+IntLaVpoYjlFToB7+tekaubkeGjaWo3TLEq8dceorir+zj0DTGnt5Y2aZCCzjLlv6VpYzsVfDAia9uZb6G3Ns/7uQPwVIrsNc0jSTo/kw2iRb8MkyIPlH1rzW2tj9k8yThmJP1Fej+FLm9t7Nbi6vDdWLr/qxyYzzzUWLOd8Q2l/aaTagwJ9hQAeejdT2rS8MhpdDu57hQrTgJHjg8Dj6Va8VQXPiNGSE+Tp1mvmGUDh2+lczoumIkmmym6cTPNgKx4x2pqIHV+GrGOPTdQiuGVLgybZ3PXaK42OO41HXhbxsTbqzPEo4AA9q9I09lmS6tLq3VXkJQTIOv1qrbaQsfitTGuYre0wx9yP/rU7CLHh7SlkthqF67tfNwWzggDOBV/VbRbuyYqc+V8xB6Yqxah1RfMRUIGCq9Ky9YSewnvNVUeZbOoDqrYbPriqQzkfHulTyR291DbZiiQAbevOM59cYrnbbRri40CRZkEYOXjONzE9uO1dn51rc/Zf7ThvIEnJWH96cHpxTNfguZtIvpFmNvb26bIlzywHUmq5bknAR6HdXan7IgZAAC7Ngk96zxJNFKYpDnyiRj0NaNpqkljpghQu7yAkkHGz6Vjo+N29skkkk9SahxsNMsvKZo23KmR0I7U10liZVK/eFQI5TOO9XXuVuSHkQA4xgGs2MpSx+Wx3KAaI3G/OMr6VPcKSFIOMDFVMYbGaLjJsYGaI5FDYcBhjGDSptYEeimoiPkU96YEsA2iQ993FOJNT6bpt/qu82kDGNDhpD0zWjc+Hr2xj8+RopYR954zjafcGkwsUEuGhUADqM/jVeW6WaOM+XtcDDMDkNU1lG9xKqdR2PrVO5t5LeZopRh1OCKVgE8zqq5JIxUZDKuGIz7UYIp6qCMmrGR5LAH8KX5h1qW3gkuLlbeFd0jn5RnH60tzA9u7RSrtkXqM5xQIbjbknqRwKaXYn5jk96ZvYkEnkUHJOc/jQBPFG0z4BwAMkntU8ixrbOyjLA43etV4JjFv9SMUpyYhg8UDH28kaxYL/AD5zjFaFvJBIixyRgELt3A8sfesYDa1aGmrFJcqtxP5MRyS/pik0B10d5aa1pCadPHmazGVfd8gH985wd31rB1G20w3ANlcvPCDiSUptAatWDQbK80W4u/30MQUsigYaQ925+8p9BUGmavFawrbR6bDLbbAZTIMFnHUZNAGGLF53ZoChQDP3hwK2tH1t7WZHmR5EjcCSJACrLjB49an1K30638IWmpwxwm5urpmLIclE/uEfhUI0trhpb+UtDuUOIo+Dj1qQGap4iuNTla0th9j05XJEK5QsOPvf4VreFtL0nUrC6hkjSLULZiymU7sqegx3/wDr1g32m28Ulx5VxuCkFSox1qPw/ftpWri9SZBJEpIRzgS+1XEDo7nwhcx5lu/ItrWUlpPJAwpHfFYemafpEuty6W8skgmXFvMTtBk6qCB0z0zUGp6/qGq6lPeSebB5q7FjjPCpWeYP3Ec7YKS52kH0NNsg9b1L4e3t6lxeyS2cVwkBWKC3jyGx6n+orypoLuytIhPbyRQu/ExGVbB5xXdfD/xWulR3VvqU9wbI4WJ9pYK3Pv3rodUt9Pvvh9PaQgzSRKxt1mG0l93UDNAHmTR+RdxLnO4E/him6Yq3F35So8m87MIMmtfS9D+1WU9tduINRQqyEjc4XHAx3/CtbwD4NubjUpZrmd4bYFlwmVeQj0Pap5QOw8EWWm6pod9b31pb/aYCIGcxbSBjgj3rs2eO00+VxNCwgRgq+YMEgZ/pXmvht7uyuNUtrG5g/sxbjDC4IYsR94g9azri5WyvL61068uriJFAgSTqgbO4Y+nFA0d9Y63bas0E/wDaFvFtVS8QbGGPaud+JN8+oxxadZ3UQtIXEs7oS7FvbHp9fSuI1AS6dJNp9vKtwSivJIUClR6UybUdWit7WKNt55aMDDFhnnIqLlmfp1o1/dmGI4UBmdyOUA7mnXNz5VzGlrCs72rtukZdwbsDnvir9rqELXEs9zKpS7t2hunji2eSegqHSLy0tHFpIH3XKECRjhdvPIH4U73IsZdzfXM6o1zl/LUKu4cY9jU9hYm51Bg8boscTMxxnaccUya1hhhuoJLg+dGcx5PDA81ZV54nWd5EV1VeCchwR0IqQZ63a6FaSeAmmuvLuJo7XzUlH30IAO3PoOOK1fCUSt4WiZuWMjfzrybQNeu7D+0hIsn2e6hMciovyZPQgfQYr1Dwzq2nwaLa2z3aJJJI3liQFN59BmriwNg24EmTkVcQYWo5DzinxSKQFzzWlwQpGalihjZcMvXvURHzZp5YgZFIGhwhW3bg5zTxMvrUJfctVmfa2KVxWNSOQVbibIrGinO4ZNaEU3OM0IZcZA4qhdWspRvKcqx4BHarX2kVMkiuoNIVjl7q1vIk3SsZT3OMVW3kda7BljccjNYOrWixFXjIOc5B4q+a4JFVTnFWI3KdDVNMjGamRjmkxl1JXHQ02WRmbLEk0xTTmGRmkMYDSuMRs+RhRk03O081M9stzbFS2UYYOPSi4HA6xd29tcQyxpHMyzGQ4b7o6fnXMajq0Q11Zo7PDv8AMy98Y4z7/wCFeny+D9HlSRRZjc44y3APrXnPirSHtbiKa3t449i7ZjEQQxyMEfgaAMi+1f7QYljOQd5J784zkf0rNu9hljKEFmXnBrophpxsjJ5EEDKrAoo+Zg3QjvxjpXKW0f8ApEUR4OcH2qbgbWj2E01nfXMcrrdwxhgo5ynOce9a0tr9iWEXsEvnyqJDG/GB25NLoV/Y6ZlJy6yuxEm1d20A8A/jXo/i6xhfQY9XkUOY1Vhkc7Wxmtac1ezIkjgYLz+zmkQRb5ZZBgH5doPHFbY8z7fIR80QxgjoD/WuLuzJe381vY27ZhBLkMCO35V1un7bK1is2cvKBlgOTk9a64yMHEugc0v8dSQ21xPKscUEhdugxW7F4UaQZluJI29Nmf61XMLlOfB54qdLqVFAVql1HS7nTJj5q/uCQFl7NVQYNLca0NGynDzAynn1rXW+RX2dvWucgjnlcC3hlk75VcjFTeeF4c4as3BFxkdRFcoxxnmpvMBrlILh/OGGrbjlbb1rCdOxopFw80xlIHSlRsmpdwxis7Flfdg81SvH4wp5q3ODuGKqvHvzmrQjHmeUSAMGIPoKPLjZCXD5PQk9K0ZDsG3GSfWoGUNGSRzW0WZ2K9tcTRHazBh29q00mDjJNYTuQ/GRViC6K/ePSidNPUUZdzZJpN5FVkvI3x81TbgelYOJqmSeae5pDJ71Ec5oqBkw5pGOKjDYoL570AO3YqJ5OvNML+9RM1MBWf3pC9MJFITxQIN1MZuaXFRlCTxQAE5puBT9hHagIaAE20m3FTbCRR5Z7igCLNKDUhT2o2UAN3UwmpfLoMdAEGKeqGpBHzUqoAKAIAhpREc1ZEYp4QUAVhGfSnhMVPt9qYRg0ARgVIKYaATQBMOlT24DMM1Ap4xU0DBTkmgZqosSIKhklUHGahaUMMCo8ZpALJP1AqrI5PWrHljGTUMiUAVXPNRE1O6nNM2CmIjFGKfgU00ANY4qLdT36VFI6QxtJIwVF6mmAskqQxNJI20Cqkcb3kq3F0uEHKQ54HuaYkb3c4uLhcIvMSH+Zq2XxQA9my2aazcVEz56Um84piAvg0b6YaaeKAHlyBTDITTS3HNMJFAEm/3pCwqLdRuoAeWFIW9Kj3Um7mmBJupd1RZo3UCJd1G6ot1GaYibcKN1RA0oNICUGlBqMGnA0xkoPNSKTUa1KopMCZXYDGaeHJ6mohSg0gLCvg9alWXFVA1SBqQyyZuKTzs1XZvSm5OaYiyXBpu7mo1JqVIy54oAerEVKrHFNEDVL5DgdKQ7Cbjimk8U4Ie9BXigRFuNct4kjxcpIOMjFdUVwawvEsObUSg8qQKaAd4alL2ckbHO05rXrmfDdwI7xkbo9dQ20Hk1YB0pwNRbwelNLmnyEXLO6nBhVUMfWjzdvenyBdFsEUpAIqmJ/elMxI60ctg5kNkO0nmojJSuSTUZX0rWK0M2xxl4603zD61HtapY0XvTsAwyn1pUZi3JqwY49vAqAIS+FFLQRbjPyjk1J5m0ZzUAjlA4FRmOUnkVLRaZcF2f71OF361TETelKIZPSo5UPmZd+1D1qpda9p1kP9IuUH+6d38qp6k0ltYSSYbjsBXmNxdvNcSOw+bOKlxGmei3fjnT7f8A1EL3H4YrFu/iBfScQQxwj0Iz/WuNa4du9RNIR1rNlWNK81zUbyXfcXsj+wJH9arjULnOBdTAem81UGW6Y/E4ppXB6j8DUWKLBMf/AC0kOani1Se1Tbb3cyKewas45PU0Bc9wKVh2LUty08heSR3Y92bNAYeg/GqjDZxkH6U3eV6GiwFwgN2qSK8ntn/dzSp/unFUFuCDk9KsCK5uE3xQu49QKQHa6b46khiWG5i8w/3y3Nadx4xzbb7UKH9CBXmUcjByp+8OtWRI2ODQB1j+N9SKkBUVuxA6fpVVvGutj/l4T/vgVzUjOqFiat2sPnQBiM0Ad5oHi03yrHelRKf4h/hXUZH8TAfWvH0dLOXzPs7Er0waXUNbnvoRGGeNB2zVJiPSDb9lUD6UkksdnayZYK4GfWuUsvGF1qenz3FvtBjU5BXoar/D3UX1HV7tLp/MmYk5bnFdPMQbeh6rca6Z4AyRLGcMe5rpoLdLWElB8w/i7muMvo5PB2uzX/kPLYSgn5OuT+dWdM159S/065u0S3zhI94AxUjRzvjm18R3Nq8t9e2qWSHcsULcn3Oe9cVaaR5dgbi+JiiP3COc1ueI9Tlk157aW4EloSBvHSjUdSt9QubGxgYtDGACWGFb/OKAMC00qXUH2WsqHJwFJwa7/wAF+A9X0/UBfXU6QRDAMa5Yv14NV/B+nyaf4wUTwqIZc7SeQeK9YJoAwb7R4Lq+XePuHI4qzHYxwvuVelWQM3TNVjAIoAZEeeasqQRVU9eKzdV8U6fo21GD3E5GfLj7fU0kxWNqSJWU7uF9a5y7ivp7hobYbo/4j2I+tWdK1e58RRiZbM29tnGC2d1bC25jGN5VfarTsM8e8S6VBN4gitl/drgeaeuTRrcE0r2mj2wCQuQMAZrudO0PzdZvdQuYlfnERPtV9LS2W+N3Km5+1Xzknl2s+Fv7MgSOB98rHBXjk+vtU17bxppNhqEyhZISEYAcn3r0G6ijur5XC7EAxlqhv/D1o2lypbQfvAMhmOcfhU3A8fvrq2m1SWdCFDYwAvWq8StLeYXnIr0K58K6Ze+H7mcQmGaPkyDqxrnJdOuptOt4ba2MKR/fum4XP1rnkjRMwLoGG58hhwazZgyOVNXb0blDmcSyAYLq2RVLY8nJOcUihitiNl9e9OgZkkABwvek2+lKiMWCjjJpiEnjCyEqDXV6Boy6rcEyyiFoIg6GM85rJisEubqOITphxk+1dVZ6K9hoN1NKcyyv5MTYwCD3+lDGYkmoAzzoQlxMW8vey8fUVYvruOUWtkYzEyLypOc1i39hJpdwIZGDbhkMo4NPBe7n87cAQAFJqbElKRCJX4/iNWdMg/4mdpNKMW/mjc1b+laUmoSyWk7OhC7mZDwRWz4a0G3s7u5WU+Yn8KsDwRW0Ig5HcFt0sdwP9UUGD6isTUbCG71lZ2gRrdBwCOM+taMYaOMBWJ+pps5leB1QrvIwCR0ro9mYcxwniuIPq7RI6eYdiiNB611n9j/2Vpy/YSxcQ4kRujD1qumkWVtf2807eZcOQWZjjmuieSBnUFsknAFHs4hzHE6n4zF9BBpxs5LSxZgJJnGMgdsYqiNTsdP8VKyWkl3bomESJclD6gfhWn4vLXniXTNNWHdDuDsq98nmrVxLa2Pj6xZQEDxiMlRhQecVMo2Li7nQQXbSS+asbxpIAcOMEfhSyXHlatHKi8FMM2e9aFxBCZjuO0/pWHeXsZnUQ7FtFPz3UhwD7L607Em4LwH+KnG6UDhqzdO1HStUleKzuVeROqkYJHqKnmXa2BT5IhzFbXLeLUrNBnEsUgdDj8xXD+J9Q1u/1SPT7dPItpVyoIHIHUmu0nRpNuP4TkfWsDUrO8vdRDL8jRxkK+ATz1quRE8x5nKk1nPLEz7sZXcBwfpVUdcCun163uVtvsyLlImzI56k1gx2MpAkIG09K42bogBwMHrVm1KF8SKSp9O1RXKBZyFXC9qdaxM064YKo5Yn0qGiyxexbPmjB2+hOcVS5Jxjmt6aWCcphAFAx0xmqF1CiAyIMGouBTZWQ7WBBIpGbCfQU+ddrqMkllzVeQ/LtPQnFWgPQNJlmsfDlikMpVWBl+XrzWzc6fdTWXlXtl9pt54w2+I4YD0/Cql5YyWllbR+U4hSFRv25H+ea6W11azv7RVjlWN7dRnecZHoKwfxHoxhF00eeXuitp9s1/pdw80cP34WABjHrXN3t3cX0glncsQMDjoK7W+kWHRtTlPDGMqv1NcOmDAB3rWLOWtBQdkVtwHbn3qUn0xTJFAIxSZIBGPzFUYmno0s7apDFA0KNz88o4qHW3D6k+1E+X5S6EkOfWs9gamxmBRTAFgcpuVc1GCO/WnxoZZVQttBPJJ4qWC0aW4KZUqOc560AQd66Dw1pmn6nd/Z7+fy9+BGu7Bc+mar21oqCS7CjEbbuTiu78P2NxFp4a5trYwmQTK2wEx88/lQBwXiPSF0LxBc2MZcwIcxM5ySp/yaSygijiBu7Nn8whoWP8ft7Ve8eyRSeLbryJxKi7QCrbgOM4/Wi8vLrS5rCC6s0k8gLKgdsHkZoII9b1PVJfI0y/mRorM/ukUfcBHT8qyzgoEB+QfwjpVnU71dRu5bp4ESWQ5yv+eapBhtAoKTHFEC5UVpxeIr2JI4EKeTjY29dxA71nRDcG9hnA61D05pNDOi1G8in8x7crsYFCoXBJ4xxWEVy7D0OKeDmLdSDHX1pLQBZZJJVBkO4KNuPb0rr9Rg0RfDP2+GMyRmRDHBJnCNxuHv9a5Ly8itJdQkfRotNZWKRzB0YdF9jTE0aWmLbXGnz6XCk1q91OjxNKMqiDrk+361Lr2mXuls2kO8t5bwN9oglORlGBzxVe71eXV5rppYyDbLmEQDlR3z7dK7p/FvhW60exj1++S7vFjXLWcDEgDBwwOPx7VaZmc5pGi3Rs767Rne509Y5LcOuHBzllwf4a9Qup5bSwJtVRXfktt6ZHJ4qlBqvh7V9Ya+0rU4GMkIguLWTKu687Tg9cc8ChtYnvL2W0sNJlkSNtrTzny0/DOTQho8guZotP1Sbyo51USiSJmJB2dc++a17q8tNa1N1ubl7NmfdHdfeCcdG6ce9ega34dOqQXmI43d41jhUgADnk+x96841/S77Ti10VitootsRjik3hic9+9DiMrXmi3tlFcXCx+fbIMfak+7IucA5pr6zbIltHp8Ds6j5pZFCuo9MjrUOn+JdWsbYW0Mka2+CHVogwbP16VWgtWvpLrylBCqZMg7fwrFxKTLcmsQRotu8aHKssjKnOD61dm1SBtPiji0iEW8arGtzMDuPGMmsKG3e6SUpEJXZec9T/jRDcPFm2kQ7O0bn7v0+tK47jbiJSxjJEhbG11PDE1akE8NuzTRkMAoUgg+1DpbyJlIPKeMhj6Eeo9akiaSfTkhaHduYssnXHPPFAnqbukNfXGj3t6RFLBbFVO8HJzgYGPrXQeF4IPEOiTwXUjs0c2J0U4MB/hkH8j9K5qzW7021e2+0wm0mKsQrZ59a9I8KaRb6dGb+J/3k8YVwpwCQetUoiNexN9Hb/Z9QZZZo/uzoPllTsfY+tWgxDZpplUcZFV5LgKDitVG4i+kxJ5NWFbIrCS/UShSfxrUWRSQM9RT5bCuSSE7vl6VE3rU3BqKUjtUlDNxHSpkuGAx7VXzk1IozQBPEJG+51rVjO0YrMWUR425BqwlwSvWkA+e+ljlKpGQF6lqxr26edyGOfStGb5kI9apPAg5IoQFNWJHNToaYy46Uitg1Qi6rU/fxWXe6lDYwqzn5mOFXpk/XtXO33ia/aa3NkiJDuIlZ2BH09u9JoZseIdQjgEdtPM9qJeBLjKn2NZt5q2s/bCNM3oiRopLKCoGOpqhq97ba0LZWtrsSFwDIwO1R0yB0rRW70/TozZPfyCJgARIC5YHjOfyqBnWkPdWLm4leNkXDbOp9T71w3ifVLJrO1/s5oJZWbzjvGAQnAyPU9K662VNM877PIZPPZRGZGwQB/CM9e9eVt4dkln1GynuUWaCTcrkZXDc9c8GgBljrdtP9se4sPP88grsUExt9Tz/APqrnpInl1SWMLtYtwuelV54TbPsPJOcUtuWluQxlCOf4m6UMCzAjSTyHy3dI/vbTivSbnxxYaj4aW0uUeNTHtCKMhAqkevfNeby/aJXKouPMOX8vufetzTbK2EGlySoLlHnBkjA+8oPIoQmTeFrK9a4aGAyK1029WIySB3r1by7Lw1BaxvaCSVxzIFyxPck1m2N94W1HVrS50/MXlMYSACqdM9PzqDXPEtpJrAls5IrpCPs0QDYCseuSfyq+Ymx6DAIniSROVcZU4xkVaGMVj6XfW11B5VscrbhYz7HHStJWqbsuwt1aw3cLRTIHRgQQaw7bwpYQKyzK02TkZJGB6da6HPFRM+DVc4WFtoIrWBILdBHEvRRUNxp9pMjo0KlXGDgDNWI3DU/ip5wscmvhbyJQ0E+7H8LD+uaSVGtnwynHqBXVBATmo5LWORSrjINVzhY52N8jNSZyakv9MaztJJbZiwUZKd8e1Y0V7I38L/lVEmuw3LUQTmkimLgYqUK3pUjGNErLjArPuLeVDlVyta0fJwaHGKpSAwpl3QkbRuA4rMZDv2YIb0NbN1bjeWQkH2qgtrvcs2SQe9dEZaGMkQNC0WGcAirkNym0Dp7UyRS6sCp4qpgpQ7SQXsbCyBhwaCapRMQwxVpQSOawkrGkZCs4FM308xk0gt2z1rOxRCxpuTVgwc9aT7PQBWwaTB9KvC35pxhHpQBngGnhasmAZ6UGNR2oAjEeetL5R9KcCKlRh3oAgERHanrH2xU4Kmg4FAyPyh3phiHapGfmmeZzQITyqayYqbeKinR3TKMB9aoCMYzS7hUbQyAZWRT60gBHU5p2Fcl80DrR9oXsahPNIAB2quVCuywJloMqE9arE0wtgVLQ0XGkTb1qFpR0BqsWNMLHNILl0SGpFlwaz1lYCl86kO5qLce9TC4X1rF86gXQHWgDZ+0L60nnKe9ZH2ketN+1j1oA1JJEAyDVdpgOprPe5J703zi3WiwFkzHNN82q5bNId2M0wLJkBrLydQut7D/AESBtqqeQx9anmmEVvI+5QwU4yapWF5aW1hCslwAxGSAM80Aahc0m7NZ76taDp5jfRcVE2rqf9XbuR6scUgNQmgGsd9Xmz8sKAehOagbVLw9HRD6qlAG8TTC1ZFnqNxLP5c0m8HoSORWgZBTEKznNM3U1n5pmaYiTcKM0zNGaBjs0U3NLmgQUU7jPSk70wEFLTgKMUAIKcBQBTgKLgKKkApgPNPBpASqKeBTVbtTwwpAOpc0gOTUirntQAzmpUGRUiqB2qQYoAj8omjy+elXI0BFK0fbFAFRY/arsEeADinxQDhttThQO1ACDAp5ddvXmoyaiZsUgHtyaRRzUfmU9GB70wHNGCOaxdatZJbGTYucDNbm6o3iWeN426MpzTQHnNnK0N0jDjnFdk8UmxGx94ZrjLtPJv5VHRXyBXf2Ukd1pcUn+zWhJmhyKeJM06aAqxI6UxUrVNWMx+7NMbrTlHNTLEvUildAitg08ZHWrO1R2FNZFPQUrhZkB60maUqc0hGKtPQkUc1R1PVI9KiLyY3DsRVwsV6VwPiF5JtVk3uGx0x2qSomwPGmBzbqfx/+tUY8c3YbCQQqvpjNcsLdpDhWA+tP+wHGfPSsHORpynR3vjC5uQNpSL/dFZra9eP1v5/oprFaM8/Op/GoSj1HPIfKbb65fupBu3+tNi1i8ibd9sY1iEkGmFttHPIDqX8W6k0DQmdXUjGWXmueYvvY7upz0quHYnipEdncIqszH0Gam8ihw461asorGaUi7lKgdAB1qa20wzyrvjfj2NbMmn2ttbFvLdXX/ZzQBSSPQ0cERyNj/ZNSy3EM2be00+IKejMOaopcs55Cf98irdrbxzzbpLlYVHcnrQBSudFktlzJINx/hAyapvpl4uGFvKUPcIa7ix03S2uhPcajFMV6AyYrTn8UeH7aUxyXalh3VSRQB5g9nInAjkP/AAA05tLu1i8ySFlX3r0WTX9AunzFeoW9NhrK1i7+2gJa26yL3YDGKAOFFtNIflicj6VvWWrx6bbeTIhPsD0rSt7Am03PI0bD+FelY1xo15e3Rjtk3Z75oAsadJo+o3phuowu853Cp/EPhyHTLcXNrJJ3+Un6Vd8OeDZrG/W41KMEDsDwK6jXLnTvsLW74kB6ANim0B5XayTTzCNEDn3rpoNPvmjVVtuT/d6VU0+G2025Mxj8xM5yzV2cPibRjEXLpE393NDQHKDwvfLIHltpwo77qdDolrc3HlbHz3O+tW+8e28bGK3tTIp/iLf/AFqwLa8u7uV7i2WRMnPHaiwHnUMsgXMcjrnrtOM1o6bd3mk3CX1q22Yd6pMr7yNoGPSmySScDPSgo6DWPGeta1b+RcT/ALrGCuBzVfw3LFDJMk8W9H6nHTrWazsLXc0JC/38cVFE7bMqxXPpQQXtUWEXbGEOF9G7VZWzuNRs4p7eMYi+8w7etVrIOXiuZ4pp7cnAwOCfrXQzlreeGVrO7trckF49vykduKtMD0iy09LvRtLkdmDxRqQR3rfRtwrnLDxZoU1tHGt2sQUAYkBBrQfddqVS5/0c/wDPJuT/AIU7gKdWhgllhjje5mDcrEMgfjVO58UWFpdLb3yi3lYZ2tKCcfhSapPH4d8O3E9shMzDapHUsT1rzjU/CVyvhptcvWeS7nbdgPkgHFSB31/488PWVpMy3e+YL8qKMkmuQ0rxNe6vYXsnkwsyZzlfmrjrDw3q+pSrHb6fKN38TcAfU109lq1v4altLKW1MUyHZMCv8JpoR2Pw41NpNEuEGwyiXB3Hr3rori4vpnZN8YT0FYdvpj2erJd6eQtvcqN6jpXQLGqgAChgOgSXywHfOPWho8HHGPYU8NgUFqAIBFGvRcn3pbiZYI9oBLyDC0E80k2JYfKK53cdelAFax08mEWlyvygliPWuL+KerRW1lFodkEBYAyIp6DPH9a0fGWvXeiW4+zybZQuCxGSRXktzdTX8zXN1I0krdSaTGirEp3JE3APWp7i2NrMFVtyuODUDIM9/wA6cZmMi7mJ2fd9qmxTYwwuhwFOKfAhLAkECpJbwspXHJ71JBLuj+cZ56UgTJbj9wkMsX38gBexr1e+mk17TtM01yYdTCCRVAwvTue1eX2UqDWrOaQZijdTs9ea7XV/EsOkfECG9nikWDyc7COTxximMpeO9Ok03RLG2uTE9wJcl43yOnTHauMtpnj2fuEO05x2NPvb06nqU88m8mSQupY9Aa3vCmlrqGovEbgQvCvmAFc7vakJm0Ir/TFt9Qu7CKNLhQvynDYNdi+i3kltDqEQR4mj/vYxXH+I9Yu9YeCC3gYWduVV2A7+ldzceLLHwzHbaTHCHUIpOWxtB/nWsKnKZuNyulrciMF4CF9Qc1z2oaoFuzFCchep966yH4haZN51q1u64X768iuItdPXVdWuAswW2UklicA/jV+3YvZFW5ea5ljlf+A5U/zrRt7uaWJJQp2DufWtOzs9LgYLcQtvHCleQBS38FjdafPNBvX7P8zHpxU88g5DktS1ho57i+vZCLjYYoAo5A9QOwqrZ2jXHhE3TgPcCQvHIDz1rJ1eebV79SqPwdkaY5JrrrONdJ8MR2z7YrloyTETyTVxYnGxqx6i1xpESz588IAwHNcp4rvWZLWJ2ZYAMYB4yMVvaRe2t1Zu8cTJNEo8zeeQeetcrPDHftPd3u4/L+7jU4K0pTsKJ0ug6zoWmpCLvy/tBGVukQ4x7+9dkXjmiWWNw6N0Zehrx9HFlEiqzbHOCDzxXcaDaw2v7/TJzLCwxJb5yEPqKcalwaOgdkRSzMAo6k1jeI9cttDtYppN8hk+6kR5I9a1Lyz+0oqxqN4ORuYgV514x07UpJTf6iYtiDCrHkhRWrnYmMTGvtYl1GZmKCOMtuC55rPuLyUoIkOFHSkjjYAORgVFPgzNsGF7VyM6kBdWKgFmPfinHDHYTgH3qFeDTlBLAAZPpUgXVmBU7f4eBS3eZLAyAYNVsFfu96dNKPskUKnJBLN9amwwuP8AXwntsoiiS4vraE9HkANQb2YgselavhqNbjxLZxv90Et+IFMD1mO+hdhELnM27b5b9MVJc6TpWpzui+Wl1nBdCcgj2rkZv3lxI/qxqS1vrmwnWaBgHXpuGRXM5LqeqsPLlVmV/GVl/ZWhvB56TeZMvKgg/jXn4znFd5471T7founB0AmaQl2HcDoa4VB82a2p2scNf4gfggU7cCcGkIYnmnxSi2di0W8suBz0rQwJbe0jmVmkkCLztz3qSeKFdvkIwjP3WbqarrKwURMAYu64qxcXrTmL5AqRrtVB0FBBAixgNuI3n7pz0qbakfyiUM46kDgVVbLNntUturTy+Sz4UqSD0xikVc2bfdNp6WW3E0zqikHhwTXpWrTxaX4ZnuYn2SiARhe4bGOh/GvL9GiurzUbSGJ2YQN5pxwVx1qbXZxeate3E9y7s0m1G7NjtQBm6XpkuqatZ2UA/ezyBR+dej+LvCWjQLcarf30ksyW+ERSAjMOAuOv61xPhnUF0jxBbag8YkjgyxUnBPHb3rc1e9f4ga5bQ2cE9rYQRNyVyM9Tkj37UyTjYLCa5iM8fOWChf60y5tp7S4NvKmJgQCufWtXUdKv9DtbaXztrXIZlhXhwg6MaxphMdksyyfPhlZx94e3rQNFy2vDp+qWtx5BTymGdo5I71a8SavaavfI1naCBEBy20AuT64rOedZ0Ik+8B8p96hRN7YzQMkjDMuwCjkVYsrV7i48pH2fKWJYdhXeaV8PVn0+w1Kac+VNbl5IzGSMke3tQM4ZZS0CKei5x+NTRTwqA7ybW/u4yPxq34gNhOLKSyRkm8srMqKcFh/+qsDdzk9KQjo47i9fw/cW2mviNW3zMgw5Unp/Oom8TahPo0Vj5qEx/unBjBLDtk/lVXRLmS31FNtyYIz1bGRnsCK7nTtT0S68GX0t1Fp/9oQy4zGgRzz8rr79cVSQmjlE0XVLNbfULS1mhu0PEewl8/3se9dTZQeI/wDhGj4l0/Ur1nikJmt5WG1kA5IHfByK67SfFHh/xHNY7tRMeo27kIsmV8wAcZ7Hr0z1raEmmW9o620aTrbBtzFsRKc5Pf3pCseTt4q1+DV7n7Dd3MkUwzHFMpkIBHJGRxio726vnihe6g3RRuIxGx5ckc4HTHNel6XpUl3E/iHUmjjupogsaqmxYoweB+PX8a4/WrS2Gsot1fTGV0eXYiBVhQfTrWlyTnriPTx5saW0MUxj3bZMgD2HrWRbtLukjjTesmN6ZwCB61avL+KeJYonVtyjPGfyNZy5Mm1WKt6iueRaLsZgW/YCN47dm3YBxg+1RXvlzfvXYh14BB7ZogiuLlZBEd7r1DHk/SmXISOYW8q4JG7IORipGLBeXMSoltLnccYYZrQsYNVWKS6ji8oW0gDNIMDLHAGO4rF2QbPkYs5B+XoAexr0jS9V/tTQ7mWYxxyNB5Ko79XQdRzx2q0BBqPhvyZReFPLguEiEkI5+bJ3EH9a7DS0e30K2SMuypuUd8AHArjb/wAUJeaclpBE6quxg7HG1lGGGPr+VdF4X8RQyW0FlNkzPO6AgjAGMjNbQcYyINTz5M9aUzs4wTXMXHiQ2doBNCwZdySODkoegyMd6rJr91AixhopipyzAYJXvxXR7WJlaR1ZGWrXs5RJHweRxXPaVfrq9zIlvG2yNQzMeOtb0Vm0RyoolOMgipGirkjmgmqkLy7mV1x6VZUE8muZo1TCnK2DS7KXy6RQuQe9KHx3qIgg0nIpAWhISOaQgN1quHxUqPmgBxgDDAqjLGY5MVqRc1XvIt7bh1FFwKoCEYKq31GawJNdgmmn0640zfEflKqmT9apX15rPnXNuYGBLho2UYxjnINWTd6e2lTXU8/kfKUJ/iDegFFwIbrTL+TTRbxLIkJ++ZB86qOhxWdNaQS2cbJIxaAbCo7+h9TUnhrxBPdFLO+vNkL5SNiAGJ9Ca3mg0m1mZZHVjx846n8aTA53X9V1C/srW3kxarBMrN8xBAxgYNclq2vyXd3cywMUEzDfkdSBXoHiO1syjT3rq1vOmzzMZIHvXl95ZJbuzJLujJIU9c4osMqwTB7tDdPlckk49adbsfLUY53k/lURT509CwrRsEVprYEZ3T7fwJpAXvDn+lXkmnFH2zkEyKcBAByT6DGa6q9FvYWE8+mzRTxQlYxj/lmu0k/U8HmsW/1DTNIiZ9GtRDM58l5DnAB69aJNTtB4buIkhVl3KHUt94kYz7df0oEzFsruRFngSZ4InG8spx0//XW9peiLBaW02ozk28hMgihfBJ9Sfyrkw8mwRxqC0nynPpXQ2VjFNEk8ySF4pEzGGJDAk54/Dt60Aei+HPEzR6o2mxWSQQs3yKseDjHUnvXeLdqe9cfbfYrSzinkeMysp2tn7rHqAO3aprbU2LxxysFkkG5Exzj1oGdd9rAHWqr3O5sg1l/acjrSLcYfrQBsJOcjmrCTHOc1kiXgEGrEc/yjmk0M2EkUjOadvFZazEd6lSYlhzQBbZPMQqRwRg1gy6Wi3G7ng1uPNtXIqjIxOSapSAiSFVQAAU4RZpm8igy46GlcRI8WFyKzJ5Jo5f8AVkj2rQWUt3qTaGHNWmBzsl0krbUOWNJweP4hwRUuoWMSPuUYz/OqUccglBx8h71tHYiROQADSxwbxkAY96jkba4HY1prGFRcelIRVFuc8ipVj4xipMGlArNspIj8njNMZSKtgcVG6rU3KKhOKVXXvUhQE0hgAwR3oAcDTgRQqDbQE3NtB5pCIZG+bionOV561LIpVyCKjb51AxyKYFckijzKcQMVGV5oAkE/FBnPrVc5zTCTQBMZTnrSCWqxY0Bj60wLnnYFNM/vVYE+tOAJpiJvP45NJ5lRBDml2mi4Dy4pN1Mbiot5BzmquInJ4qJqaZqZ5hJpDHGnAZFREk0obAoEOI4qOrCrujzTxZu0Rb0pDKRJqM9KldSp5pmMigCMsabup7LTdhoATdSq3IpfKYnNRtwcGgRn3l9cGd40kKBTj5aqM8jr+8kdvq1SXY/0t8DrzUWcipKE48vhaAuQDijoaMkAkUXHYVtoFCkYoCuRu2HHrimHihhYcab2pO9O7UrgLbHZco2cDNaprGyRIMetbG7IzVIQmaTcaTNJVEj9xpc8UylpgPzTgeaaBTwKQDgOKAOacqk1IsdACBeKXbUoQ0BDQBERSYqcx0nl89KAIqUVOICR0pfINAEQNSqrHtUi2/tU6RBcZFHKIhjjO4Z6VaCHFPVkHHFSgoR1puLC6GJGzHGKsLbZ74p8QUDOacTjvUjESIL3zUqqCai3UoYipGXyV24FQseKrmYgdajMxJ60wuSO+M1ASWNPPIpBgUCEK8UgBBzTyRQKYBvx1oEwHemMOKgbANUkK5x+vReVqbEHhwDW54ZuybBoW5x3+lZniaFhLHMOmMGjw7Li5ZPYmtLCOrDo3WmOqgZAquZAneqkssxORMAvpTJLeeacZsdTWVvfzAWkOO9OkJPzAlh7GqJL5uAejUqzGs+Mnp3q2isB0oFcle4EaFnPyise68VaVBlS7u4/hVaPEIuP7MYwEDHXJxXn8PliQeZ1qZFROym8VW09q3lI8b9t1ctLOLm5LBs5/SqtxKvmEINq9hVvSZlil5tI5MetZ8xfKPktJBjg8+lWItBublNwutg9GrSk1pIk/c2oD91rNvfEk7qYooWTPUgUwKd5prWUoRpUkz3U1LFbFo9yxFvpVQyzPy/zMe5605Zp0+7x9DWcmNMcLe1YkTrkZ5Wt7Q9L0a4QsbQcHHzHNc2zN1cYoF08WfLbr1qecZ1DPoK3pils0RE6nB5rXh1rw1ZRCOGNF91SvO3u5H++c1D53NTzhc9TXWdKuWAhnjD9soKZeatZvbtbSLGzEYUgivNFnFDTIe9HOFzSvbGSS4LB40UdAtVncRp5bHpVI3WOj1E1yG6kk0cwXNO1ginhYsYxz361QnhhVyqSqfoM1XMretAlbuaq4yWBDFKGEmDWil/c253pcSZ9qxmck5NTCQtHgGgDobXxF9pl8q6uSiep4FPvHslUPY6iwlHfdXNx2U0y70A2n1NROPs0uJOGFAHRWviXVLcFZLsSRnuRms3VLmW8u/MWXI7bTUUEiyLlcGopJCk+dpP0oA6O2t91mqS53H8cVBdafGkBbcPx4qmusThQMJ+VXW1a3eDY0e5/XHWkBnW8duZRmp2vrizlxbsY1HYiq5l3SK20DBzxXQPcWk1ltfDemR0pJgecWUEt1qUdsJCrFtpI7Vt3vha9j1VbWzbz9wzvf5a512ezuXMch3E5yalTWtTVwwu33DpVAdjJ4Y1qBEjc20sRONiHn+VY/i3SY9OuY0t0jRWAzhu/rUUPjHXBAYWuhtAzu2jJ/GsG7vbrULgy3Lsx9znFBR694F0+V9Al0u8QCWNvNR1YMcnkciu8s5Q0CpO6rKnykE4r540PVrzR7hZbedlTuua3D4qfzWuJ3MhPYsaTIPXrrQtIuP8AXWloxPdcZrmbjwlapltL1mW0/wBiOUEGvLL7xDf3EztbXk0MLfwI5xW74U16XR9m+2WdJDgknkfjVJgTL4ivLK6Ol6iJLqCCTJZRhjjvnv1r0W38R6Brlqtr9oMDZyI5lINZmr+H11g22p2DCK5j+YcDD+xzUptbPUYha6tpaRXBGPNReP0pgReIfFx0hki02xN3hf8AXgnH4V5Vqs8ur3pu0imwxAYkZCmvRJtC1LR98unzx3EKKf3cg5A9s1S8MzaZcWF3pd4RDcTuScjHP1oA6vww93YaTa2mpbRIU3IQcjFdB9Kqada/ZLCO2lKSiNcI2OcVaDgUAHNGaazDFN3HsadgFAp6yLGcseMVAXkL4GMetDRRRgedIMscDPepA8o8UxXepazLHEmYppQu7NY+raC+i6nHZySl43AO7bgCvZrqCEKrxxoUU7nOOgrhfEslp4j1qz061uRujDZdRuGT27UAcla6ImpX5tLUtvAzknisKe2mtbuS3mXa8bFSM+lenaHNF4egvrG5hEVxEpZZ2HD+1eaXNw93dz3Epy7OSTQMIvKU7pBk9q0Le1aWJZtgKg5KE4NVbXTprzJXCIP426Vfto5LSAq53AdwOKiwI2Li2uNbsvMsrFbeGzXzGk3AE47CtDxsYNV8KaXqXmBZigj2Hqcd67DwX4cs28PNcSNI32tSGUtworjrrw/bTatc2NxEsHkDcgDH56Y7nn65WrsVzPbmO4t5TFODgSDnH4VJaaLd3msnT412tkncRwoHrUNwjQO9s33kOM0Aeh69Zy6d4GsRBPva6cPM45LFuePypmh+Gbq1c69rzlLRV2p9p5Y+mab4Uu/7RTSlvZI5IoJgnkMTuxzz7V0fia+tte8UW+kgO9pDy3lr39D+XWnIk4bQrBtY8XnTraXZaTSF3XoGQcnFetMLezunsrPS4WS2TIUHBP6da4i8vtM8NePrN4k8tPKKMFHc9P61q6lqEunubiymcTT9d7Bm20riubGsXT3tlY3dlCV3sUkjxyvsa56znZL+6snifa/8PqareE9auoteNvJK7W8hZ2UDI3etdFpV3dTz3xhjhDI+cuMmi4zmrDw266/JezqsKQMNsLHnnvmueab7b4kvNQu3JitB8oXjt2q5qHiGU6ldsTl3OMjgAfSsTULqSS1cR4jSTClQOoqbgMW/ngmadXxDecuvYGnOdy4zWTNNIFit3UL5XTFTxy+eVAPB61Fw5Se9TbZO3p0q7o8n2G7ISR1hkjyeepqo0uV8qRdy0rS+WAUwp9Sadw5To7vWZLLTzJDI3nTPjLHgViyTrcRE3EryICCwJyT9KpSTGQ7pmzjoB0FV45llzs+72Nae0kHKJcSo8rKi4X0rPlhbJK4PsKuXOWmARTnGOKY6m3UscGQjpnpSvcu1jPTipUfyzuX72MZqDBXr071IjfKKYDWkA4/i70hwVBFNxmWrJt0MSkSKMjODQBUJJOB1rpfBNqz6leXDD/VQEg+hrADCNgyr8w6HtXX+Ecpot5cEf66UAfhUz+E0p6zSNTZ83GMUhQ1JkUhPT3OK4ep76MHxRE9zqNlaQrlzBhF9Tmoda8P/ANmQWskRdzICJdx+6a6LXtKnsNctNTOxoYlUE45U9jVHU3u9UnW0iGZJcMo6AD+lehSh7p4WJmpVG0cpJGTgRr8wGMeta1/4baOwtJrJGd1i8yZyeCe39aYNNnh1JrKQbZgcA+9dHJqYttKNpK4DAg7Qcn86t6HMmcGzgorNw+SCO4qd4SLLzP4z90etXdatUDieWQfaHxvRemaikeI28Rb+ACsxlELgc1Pa+VHL5kp+6CVHcmrEdk0yglSvz9Dwa6vwfoNtdSXH9o2/mW5bA9EPTJP40DZx2nXk9rJMbYKZ7gFB3IB9BTJZJJWjhkb5YsfIeoNbVzcwaZ4pvXsbZVgg3IgHIz6571ioslxdtITueQlmPvTBI1dD0OXVnlkMhgtof9ZLgdfau7t9LvtH8MWNjpl5HNPcXDbriIfcB549a46xnaLSGs7mf91NN5cCq+3Y2fvN6ivUvA9hFYeHI7Q3EF3LHKzO8RyoJ7A04ikcnrPg1xq1gtxqU15LdQu03nNsGRjHrhcnpW9qnhm2v9QtdYFqDDZWpt2tXXOzYpIIPfqKi1fQdQ1LxDFcNd/urljE0Y42wjrj06CugtLsaJp0en3cr3E0YZIowMvIueCf8apoEzzHT/A+7SX1DVn+zxPC08KRph3I6ZOfSqC+Erp42klsniP2NrmN0PEgGMn9a9n06xVh9q1CNAVUhIs5SFfSp9mn3UEkkAjOYmiSROgB9P8APNKxVzw/wnpv9oapaC7hV7EviVz1TPTI64J4Ne0xQyWFgtnCqhYywQqMADBxx+NYOszaZ4XttPuImU/Z2Fu4jQbyrdSQOvSq8PxM0afcq2dyJScQq2MSfU/w/jSJLNho0WmadpUqWhbasvnK6jIdyOT+VeVeJPDzaBqctobwStKBLGNh6E/pXpKeOJ4dTe21UQR25UvEYEyCcfd3d6zYdTsn1BvFOryxiK9heCGEJnaMdPf6+9MDzJ4WtrwwB45XH/PNsjp61Jie13yH5A/ysO+KS4FhFMDYRyou0bi/BDc5x7UxpfM+VmLDuDSQzW0L7IZZomhaaWZQkSgZJbPT2+teg6bY6ntg+2aXaw29rzDFNcKFz3LYyW7cZrgtLkt5rJ4JLNt0IKrOi45PQV6jb6la6VpFm2p6P5F3MVjKQruDnHUeh5p8oGL4+vdVaytIzfTNbzhlCW8YVCw6e5GM/lXGzGVf9Kt0NvbuohlcfxHoV/Gu41O6S51u3vLSwnW3gRldpxggnoQPauMhmnlKwMiS2qTvI24YDE+tQ5WE0c9EibnIPlc4XdVuJYIA3mOk0rj5dp+771ftJoE3STqhRMqsH3gD6jitHTdVtNGiuLiTTxcJdxlIflHyE8c56df0qG7jRlacV+0gmVYmAJUsOCfSjV45UvRIkYO0gqQmQR7irF14a1izvbC0ltzF9qkCwztyG9c/St/W9C0q2m3XuoMLhYT5sUKEB2x8mPbNUM42OASS/vYiHYgYAKitK3m+x28qoSohkUq4GSCeD/Kugj8Gatp/h5NVvJE89lIWCQHIz90A9NxqnqESWmnywXVu1kzXJZlc5LMqAEL+P86AMCaFluNhl8z5dwfPf6VqRyyRXUDxXW3YA2F6B+5pgtlaCOK6OzeBLETyVBOMGoI7j7NdHKRuka/Pt7j2rMbRda3vb0XaxyPdyzSKzhFJbC/T/wCtT2SdpTFdRSBIZlUyMuAgYcA/lXd+AdDtrKCTU7S5klFygEqsApjfPQ4rd1LRbPU7eeKeIZlO4uOobGAa05SGjlPCUttBripHO0bFCk8TjAPIwQfTNd+jg9a5+08PpZakZoRvieNFkJGCSFx1/AGtvBHY1ohrQtFEPTB+lAXFJbJvIDEL7mrXlNG5UqD79aVwK+2nqOOalMXtSBMUDIXh7iq7A5IrRAGeRUEyDdwKAKgGTT1BBpQMGp1C4oELE2KS4+aJ/cUqj5sVk2F7O+sXNrdfKiOdm7jIFICvqAaCzmupZEhcAhQ54JHrXk99dz6k5WQ/J5m4IOgNd54t1prG8aAXyBg24pwwCnoMEe1cFPPJf3q3MSbQWw79NxPT+RpgRRXTW+qW6QNjayuT2B711Vtr8YilRbNXfzi75O4fliuRtreFkY+cqXIyNjDnI7Vdt7aUWyT2x2PMp+0dwBnpQB1NrrCahPc2OoOiW6jfCenFcl4hNv8AbibbySvBDQ/dZfWt7TdOhvWO+4SLyYSpKjjHbNcmbMz388Ma/LEzAewzQMgu4vnjSIcsMgVqeGzbxzQtKCGVmdiBk8DGTWRcuIZ1iBz5eQfUZ4p9tKsEqeexVCp+6eTn0pAaerNFPqXlWUhe2dwdzdS/eqD/APHuqke59u1b+mWKXeg25lmMEsDyOWaPPBzjJzWM0N3ePHBCgZpmIRRxnHf6cUAVEPzgoeldDozSvdW9rJL5KZDu7qCeDwPepbnQ7fSb1JBOj4wBGoOQ3Bzz2q4Ln+0obx1jCvDtZ26BgD+lIRv39t9jsrTCu0Uc5wd+CQR6Ve08O2qefJxLsxjsB6Vj6dbz6ra2l8rEpHId0TPkADoAPWti007Uhf8A2gTf6Ozbirjlv8KANzcRRvIqOe4hgaNZZAhkOEB7mjeKBk4lf1qVJWyDmqgkFL5o7UAai3Oe9Wop/lHNYiynPFWY5G9aBm35xPel61Rjm6A1aWUEUIB2xTxUEsLAk8GpiwHemu4I4oArqwXrUqSgnGaqSH5zSqm4ZzVIC0+1s55FZlxC+8lDkelWSXXqaVV3nmncVjLX5p1ypGK0OtSvER9Krs216fNcSRKRmlCE0sbA9asoARU3GiEKRTZYQU3KefSrRjGKpyE5IyaQyvn1o3dPamtnNJ3xQIf5gApBJ82abtFBXA4pASswY5NQyOqg471EzkVDKxNMBGYZpu5aictzULFjmmImaRQaiZwelRncaYc0APJpAeaQU9UOQaYDgpJ4qaOJsZoQYNTq2KYCCJvSmSIQcVoRNHt5p7RRtzwaAMR6iK5rWa2QyYC9aX+zUboxouBilTSBTW4dKB+6cmq82lzxc4yD6Gi4GcBipkiDjNP+zSjqjVPFEVHNK4WGwxhetWQRjFOjhVsZzVtbZMcClcDJntUZSw6iqDRY6CuoFkrjGOtUb2x8g8gUXAwwhPal8sjtVoxY6U1kYDpTEVWBAqFxk1YZW9KhZTTAxNTIS857qKpeZ6Gr+sKBNE2OSpBrMfAHFSUPLju1M80qGJ9KYOtDglTSGT6LBJfyzrJI5QKWDbjx+Gajt3Zohuq94Y+S5nX1X/GquNrOvoxoYCYOaf2zRkAUUgGkfNmtGI5hU+orPY4Ga17G3M9qjKOBVJiZGKcBVwWLZ6VKLFv7tWiGigFp22r62Lf3alSx9VouCM5Y29KlWFz0FbCWMaj5iKmS3gVgNwzRcDKS3kx92pBbyelbgt0H3cH6U8wbRkgUgMYWkhHSkFq46itxYARnFOECelAGJ9nJ7VOltgdOa11tVPRaDb4/hpXAzPs/tTVg5xtNanlj0pGT0FNMDNeAqMgVAwNbcaBjgjiq0+lybtysuw+taRkTKJlHIpQ2TWg2moRgSYb0pBo04O7cNvrWnMjPlY2P7lP5p4gaMY604JisZGoIB3oYjPBzShadsFSMgNMXhqnK5PFQsp3UALKJAuQKYglc4C1o27BlAekdRGcLjHtTuBTZGVcmo1kHQ1ffDoRiqS2o807i2PrTEIXquUdmrQFvF2JNNMSqcU1IRg67as+mOwHSua02Vo7+Lb3ODXoNxCk9hPGf4l615nC7R3GFOGVuDT5gO1ezlZtzqcU24uorWAK1uD7gVPDNc3NqpVwM+tO8slCJwrN3o5gsYcl7HNC/lxOr9qS0nlWP94OPWtWe2h8pgq4NUIDlHSRQhXPSp5gsNGq29tJ+8zx7VTvtcuLhSIpUhiP8WOa5e+vzPcSKpxg4rOc4Y5pc8gsdHcSFB5ktz52fesJ9hLFUbk9aEfYnDDFNVjI+1Rk1DnIuw+Jgj7jHv9jWlBO6JlbUc1jvJIjEZxT11S9iUIk52+hFIqx0FpHZO7S3MwhbshNU7+GI3GbWVZUIzx2rPgvnaQPNtYjuRXS2EGkatCyy3LJcZxsU4FHMTY5h5BGSp+8KdBuddxFbd54ais7sRSF2DHCuTgVffwpdJEzieN8dtuKNxWOTvpgyKqqAR3FZxkYdG5rQuFhZ23OwIOMAVScIBxQUMG89TSEn1Ao3Vftkk2HZZtKCOcjmgDPDc4yPwprblGTV+PTpZi8kUPl452niqU7nOw9RQBDuJ69KUEDoDSrEWUsO3arth9lWTFySvoKdgGiAyw4jSXJ9eapmOVWxx+dbc8jrGz2shEQHAJ5rCeRmOT1osBOYW2g7GP0qIqVPXaKjEsgPEjAegNOVd5AY4HrTAngneF9yyMfxroIrmyurcK0SO/qy1giFP4XFPUmMYU4qbgbLWEcaFolg/A1iR3TwXDM0at6ZprvNn/WfpTWbnnrRcBLm6M0xcRqmey0kdy0ZyMj3xQdrdBTQVJwelTcDRk1FJYeUXd6iqiShjjzKjIDDCjFOW2fbuyv50IDmzIsiB2PzU1W5zUKnIq8ET7IT3FWBVaQ7xjvTySmVIHNR7WK7gOKmSMyIMKMCkMapbAAomyFHvUqwv2BNLPCWwApGPWkAlvATAVcfStCxuzDEISqsF6Z7VBHI2wK3amqoWckd+tAHa23ii8tbVHF8rbOPJAxx7V3mi6rHrGnidPvZwR3FePWtwsDhniEijqK2dO8Qz6dO8tkBGHOXQ8g1SYmj1Qgbtp6dxWdqegafqsWGiEMo+7InUGszRPE7amwiuokjl/vqeDXQl9vStDM5lNT1TwrPHb6in2jTycLKg5A/ya6i3vYL2BJreQOjDORUc5juoGguIxIjdQa4++t7nwtdLdWUsjWcp5RhkDpxTA7ncaN1UbG/i1O0FxbPweoPUGpmLr96mBbEiorOxwFGaoWaNe3R1CdWwpxGhGAB9KeLmEcSNkf3R1NOW5eSMpBGSMY3NwKhoDm/ElxepokhRyRdTbB6KPauM0OwmsfF9tBMMFW4PqOK9E1uyI0y2Gc7J9zewqG7tIo9Qn1UoGaOH5SexFKwzkfiRf8AmXUdj5KfKd/md+nSuDIwvHer+r39xqepy3Nw25ycZ9hUMLpEfMeISY/hPSoLJoZ76S1SASHYDhUHvXQSWN7YC3ivnjkSdeEHJX61nG7glWGRVVGU5KgdK377UYdXubbFtI0VvGNxHIJ/Ci4HQaX4xTQ9GXTZ7eS5uACIEh549CccVmateXNxHazSpHa6nJnao6lT61zc+oTaXqMeoW0KlcEESCobXVLvVPEtldXZyHk2LjoM9qYHX+GbJxNNqshwyDyvLiGS3vmuP1W1Mev3Xmxsnzk4bqAeldpNqMXhzUr2Nw21gZIlGeT/AJNcTeT3N6bi+eJz5jbmOOntUsCKKZkuA68YrsfD/iCw0VmnaKRnYfMUGcGuOtoPMXO4/jVm2kZRuhkKvkjio5ii14ijjufN1mVpIprmfMSN020t1ezGSJy4ZygHB61I2j6hrzQ5mi8mP7zMeQPp+FXTp9lpfiCz+wwtMiLtbPIJ9aSZLR1nw/jt30iaaZAZvNIJPUcVOl1LaRazcQkB8ZGRkZqxZaXcwLmBPkl+YDsv0rQs9MileZHlMiBsMrDgmtUhWPGHs9QlkYtbuvmHO9hgGi8uIXSIQj5Y+o969H8YIJbY3KOALc7RGfvEYrzy8so1sYpoVI3jJBOcVEijCMpZ2cnccd6ZHOykKjEZ9KmuLeRUbZFtwMsSeKzlPFAGysm9clgBjGaz5o1S4YsWZAcZJ5pkZxnjj9KY2S/NNAXojE7pFGcljjOelTAeXI0O0Ap6d6zcqjDBwaHmkjlDq5yfWhoDQaRlYlSQfaq7h5yQ0jAsetVXuXeTdu69RT1uBnDPhfpTQFnUEs4VQW/zsFIdjwWPHNUXHyxj0WpXQsgcjCt933FRkUwHHaZAQPlxStEsmNzbcdKSA4lB64GaV4pZmMjtye1ADXxnHau40iEW3hnT4l5Mw8z864sRBY1LsOfSu5065t77R7Py7lIJYk2YkGBkelRU+E6MPJRldkpV4ceajpnpuUjP51JCvnypGvUkVrQ6vqyJsuLeG+t+hLLu49MiqupX0E0k15aWf2VIYCCMYBb1rnhH3j0frCtdF5E/trUprPUERFVVxsOBIKfc2UcHiK3W2jCGG0O0jtzxXER6nOLmDUo3Cyx9STjcKS68WXk+oSXjSCGUxiKPbyFFepGrG1jxJx95hdaqsEhuXTzblXbJxyzVm2lzLNJJJOiF5OpxVNpcHGOp3EnqTQlw6SjFc8pDRpzSQs5by0diu07hmsmfcZHO0+SjfMQOB9asQ3DtdK4AyX2HIzkmpb1JdOvruwOyRXxvI6e1RcBovZsZV8OfbrVqO4vp4po4p5lRl/eKr4DfWs5lZgrIPlzkVNNcrDFCkfEy/M5/lUlIdHb5tQFk8soSCpOD1oigK2ckxjBSM4YmmO8Eqs2MSuevtR+8aJ4hvKPjcF70XKuVZNrY3LuXPIz1Fd3oWuXulaDCujXKsGc5tGjLksffqK5CCAmRIVjIz0BrvPAV9YwxXdnvELo+4hzjJ55rSLM5am/p+vazq+Y30s6XtO2eZ+So9FHrWvAtppS70M0szdX+9I5rGfxHayXT3EVyGso4z5h8skZ9fpUttqUl5au9nq8EgJyFSLLKPTrWthJHL+KtY1fWdVbT7e3utLiWFpJ0mfa8qjnkDjH+NY9jpt3bWlrfSX8kN7zNHE8hyEHQ+1dzNeaGjxvJcYvbYFGe5b5sdxnoRXKeJdTsUu5LmyuIrmWVQGcfMFGAAAPzrKQ0S3Hi59QntBc2URmUAq8aAmSToP50usaZe6bZare3FnZhioMkwiI3EkDC5PPWue0Ozk1q7gEMot5LZSxmccZ6jH+e1Gv3N5M7W1xq73kPU7WyAe1TyjKNlrDJPZqfOaK3ckbudoIwand4L69CRGWK0+ZlkkXlf8KyCRFzH09a6PwXY2Gt60dO1DzD5qHy2D4Ckdfr2q0I5q4INzJ5bbos/IT1Ip8iMlpATGi7g3zKcljx1rW1XTdS0iWGG+06byoy0aSMnXmo9PaGG4FwWiYJuZYpI9wx9O1AHY2epQQX9pqBsLm4s2s42ZdhCkoAM+4Bqj4r8aXWuNBb/ZWtoovn2sMFm7H6Cp9A8eeVo8mkz6VFMgR1BWbYcN2AxUurWEGu6HpeoCRIrqdZS4ccbU4+8OmMfrQByP8AaM/3jdXIlHR/NPH4UttNNtE4mbaJFVi7ZBJ74qukSo8m4q6beGHSrNjama0Cwou9mOHHBqZIC1r0kcF/FbwbMKFLbEwAT296u+LPCy6Rr0lpZXkkqG3WeTfx8xzwPyrDubeaSaQT+b9qJxtIyTj0rUuLa71C4Sa8vyb4oFMU6/eHpnp0xUWA6XR/F1pqGgWWl6pC6y2UkckE8RLGQgnPH44rqfAmjwpd6vPJI9zCNkESzDIKfe7/AFrjvCNg1h4hglvCLZYgQZU+ZSTjAxj/ADivWFlxzG6lfUCtEBZMCi1+zMA0I4CMOAM8CvLfF/g3WdU1g3kMbG3lnPlxDLsgbGWOOAM816X50nrSo7MeaBnmmreHrnTbnTLq/KMShiWNTwCozn9elZHiPwzc2OmQ3aSFpbmXZ5cS8MCM5BH1ruPHLZfTl/us5/QV0VvDG2n2RdFJjjUqT1B2jpSsBneFdMWw8MaeWgMFxJAPOXux9W9TW0FBNL1FApiLlttQAVcjso5Wyf5VmLKoI5q9bXYBwTQMujToP7tKbZewpwuRjrSG4TGM0gIXjXmqkhw1WpJOOKpSH5qAEJprc0hNR7jmhAI0ZzTSCtTghh70xh2poBm71Nc/cI0uoKly0fkwZYln5IPT+tbF46RWzMz7T0XjvXHeIrW9kgjkEb7shi+OV9/pQI5rWtOsEvEvra6knifc0y7ssD16VBqF7HdwwxpIqeWgJLdTiujv9Kjt7HTr+W0m3BNtx8pwykf5P41zWqWkEkQNq6K0gKptHKjjgipAp3yosVpKEdJlVlkLHPTpUEbmSJYxMUiLbZQvp2wKkhtNQMTeaCN48pdxJXPue1UntjBHbrN8jM5Jbrx2qkwOkubKaKC4lsHKLEFfZ1aQk9D6Vg6fPcRar9qliJEZY4PAz1rZ0S4ksNNvJNhihyB57ckjHTHfr+tYNvOxhnKv8rc4x0JNMZXuW3fvZUAeRmYsOhBPFW9vk6coaMF5PmQt1FZs0cvyk8qX2BuxNaM9pL/aIgQM02cLk5GKkDTi1CeTTPsUcYjRf9blcs2eQcVm2N+kNx5zTOnlxlUKjOCa2Wtbm8tfs7ReVcsUiBU8ZY461d15U1Fv7J0xY0t40BBSPBZk4YepyaAJodTW50g3DoWRE/eyFdpkI6Aevekj1+0n077JHaFEXBmZv41GeMfXFZE1rf6ddWE97E8dpOwQRSHg46Aj8c1ratpUKyX00MyyvKqrgD7p4yfp1+lMDHk1VroBIUa2XdjEchwT613/AISvXe3ktmP7qAnfIz5yT2rz+G2jhsEniJkeCYF1CnDFun8jWnpV040a8jQurXUvGwjK/wCcUCN+Nv7Q1iSa6+a2tOY9/AJPGcVvw3kN0xEL7sdTg4qLRvCpt9FV/MZrmdAS0h4B7VsWujwW0WxODkk/WkMqBCelPETelXhagdKcIiKYFRYWqVVde1XFXPBqQQ8UgIEBHWp1JHQ0jRMKawcdKBjmkbPWnLKAvNV9zd6QlvSmAruN2c09JMVXOacDTEWHcGkWXBqLtTfwpAWWlLDGarydc0uDSMARzQIEJGDmp0lI71U6UvmgDBoGaAmyODUchyKqpOMUpnHrQMawHWmgc0Fx60gYetIRIBS7SaYGqRX5oAZ9nY9qPsRParSv61aVkMeKAMySyi2gAZqs1gpPyuAPStzyEPSmPaqBkUwOfawOe5qaPSVZcs1aLJg9KsRRrt5FAGWNHUdDTTpwXtW2AmKQmM0CMM2RB6GlFma2gqE0vkoe1MDKFqB3NPEWOmfxq5JCV7VGEJ7UrgV/LOM4pQxWpmUioyopgSRSfNzVvIZeKzc4NTQzEcE0gJpoQ0RwKoNCB2rWU7kqrLGuSMUiimq4qZZCtRt8pxUZYk02IurcAL15qheytPJjk49KcCacqgGkIo+RKxwFI+tBt5R1WtAnFIWzTAzHt2IwRioxZl+4rRdSx4pgQqeRQByviS0a3it5CRyxXH+fpXPlcrXbeKIVfQzKfvRuCP5Vxg+7QMibhenIpq889akYDNRgbXNIZPocjJrOwH5WU5qS6XZezr/tk/nVOxOzWoR/eOK0tUXbqkvuAaYFRvv4xTu+KRlzhv4hQT0470gYj/dNdb4YhFzpD8Z2SYrkyNwxXZeBvnsL2P8AuuD+lNCNmKziUfdqYWsZ6rSSrNG2V5WkFy3c1VybDvsUX92n/ZIim3Zj6VF9pPrT0nJ70AN+wIOgP50w2EZbJjBPvVzzAe9HmgUBcRE24G2pAwB5FM81aTcGORSAn8wY2jpSKoLZxSIo6k1JwozSAcSFTgVCxZh92kaZfWmGcetMCNwVOaaXyKe0qt1qM4J4poCWI4cVdco0O3FUAcVOrgpgmkMYIwHqwHbbtz2qFRhuuamHShsLEZRfSoXjBOcVaGM80/bEetO4NGWwwcUoNaEkceMAVUa2PJWncViMgN0qMx81Ljb1qFn+bipAcFIpwRs5xSo4qUMDRcBY1QD5qSYJ/AetLjNN2Z6UXAiWFQ27JokVT1ocspxTdwPWi4FWdp1LJCcJ615zqcMlrqM6OMNu6V6cGibjfXCeLrYQamjoeHGKpMVjotBuFuNKRl/g61oy7JI/l6iud8Gzs0M9uT8wOcfnXRMoDMFUgjuRSGZl2PkxWcke59pZQD1BODU+qefcJLEoCOB8pArmtNvJHkYyMWbOMmgCtqXh24k1dm06DfG3PXGTSz6FfBAs9oA3+w2cV1tqpjjWSZsLS3WoadAwleYhfagqxxFy13Y2pV4ITFgjlOawkYivRNbttO1TSjPA28KcnjtXBrBG8myJ1HoDU2FYIRlwHj/FxxWqdLa4hxE9ruPo3NZt1PK+Y5jkVHveCAGFSvrmnYZBKktrMySDDA0z7TJG4dG2sO4FRM8kzlnkJPuajY4OKloTLr6peSkbryU/jU51bUhB5f2yXZ6ZrNjC960bN4GJWQ4poRT3mmnmrk1urynyuR60yXTp4ojKQdo65oGUQ0sbb1HI6V0uj+LGtFKXltuXjaydR1rATaR8xq1HNHGmPLVvrQB0lzqdtrEbw2rrCzdd/BzXK3entZOA0m/PQ5qORkDltnWopJGlfJNAGjZRs0eUXdzUF1CySncMHuKS2meE5V8e1NmuHec5PFMCJpj0BIH0qIru/jFW45IYkJ80g9xtqDzYi24gH8KQDGiCqG3CkEn+0BTHkDHjpTM00wJW3f3z+Bo8zgDJP1qPeB1qYw/JuFKwCJMScGnGRc84FRBeaGA7iiwFlAvZxTTGC2c1XDYp3msKVgJG+XpQN2OauaXbLeuVc496s3lmltuQMhx3zRYDiEikDj5afO+MpmtBYfItfOnOyU9F71QmWNl3gHPc1QDYW+Qx+tW4oZ4cL5TEHuKrRQF4y69q6ZY/OtY2j+ZgMYoAyN/l98UGff3zW3HaWyIGvUjTPpzT9WiWOyj8i2jSHIAdxyfpUDME527sKKiYjOalfYGwSD9KaYlY5FOwAhHXNTI+7oRj1FRi3YDt+dWobfC5JAqGAz96P9W7j8a9I8KalNeafsumG9M4wMcVwaCIEd/pWhp87CYRRyMnoB3pxkQ0emEd6R4Yp4TFNGro3Y1zm64scSSh8ejGujhuY/sol4KepFdKZJx2p2N34TvUu7AlrN8bkByD9a6vTJYNXtRMlxKw/iRTgA1Vn1ZZ5TZ21qLtn+U7uFBrjLnTtQ0C/T7UZIbaY/OYH4x6VQHfzXOnWRPmTxhv7oO5vyqIa9ZjHkRXM6+qRHAqaxtdNFqk1nErBxne3Jq4jY7L+QpAUotY0vUc2rOwZuscilSa4nx7rjwzNo9ocRDBZ+5HHFdD4x1LTbOy8u+sppHYZimjAAVuO9eSyEzzGRmJJ7scmpZYwrgZPelaPEW41IyllQL2OasraveX8FpH/G3NZtlmz8PtGj1nXJYp0zbrH857jPSutvfDWo+FhIbLbfWcjbzG/BX2FRaOmmeGfFcccTpCjW48wluM+/511fiR5r6xe3t2CoULtL6Ac8UMhnj+szSajeSO4WFPuiMH0qxY2rx6WJFHzxShlNTafbQy6Fe3T/PLbEnHY1JFexqtvcsM27nZIo/pWaLOzaSzvrqB1aJrhoRsJIyp9a4vUQ9re3FobgTZbcxVQAD6UvisvaajDc28hjlKgLt647c1oTeGZX06KU3JM1xg8DJY/WqM7HHzShXKICFHoalsn+cZHyjvW/qul21p9j03cPNkb94w5JrN1Cyj029ktYyzIvUt1NS0Wi/aXCtqK2YYJ5i/Kc85/rXVz6X+5gjg3B1+VgD1FYrWn2vw1FPaWyRuOrv97Ga2rXX7SLSolln23iLtJxmqiSy3LqN1pAhC7m8vqrEk4Hr+dA8S2yWtyqSF5p2LBUGdnsa5oa3eS2ckhjLs2VMpHAqo11JpmkC4XYZpG+XvxVMCXWdVkuLwHd8wQDYf61j3F1K8axsFVAeABVaa5aVvM3ZY9aY9xuVRjpWMtS0x128clsYXOGNYclv5S7AOnerV1JmU89KriVWz5m4+m2rQMjL7EGAcZpTLmLaDwetMZl3bT92oiwVcDrViFY5NCsd4Y8mmfWpBjaR60gGk9cDk09AUZXX7w55puBu4oMmDjtTAl2SMxlYg+uKUYJx2pYlLxNz0qMYC5JoAuwobaeORScYw2B2pkjKZ2PUGlilgeEksVYDC+lQbsLjqfWgC7DZtcTRRIu6RW3BOxrstFskmUebEowSCoHANcbp9wi3BWQuu4YBTrmvQNEXbBH7oWP1osIxPFN62mX8cdlM0DFNzbCRnnvWPP4j1C8s2hmuB9Qgy1M8Uy/aPEVwMY2gL19BWNHlXpWK5yyHkSCNX7HIpiyJvBkGR6UTCbO6RXCkfKW70+2s57iVVjj3OQSFz1xU2JbuLC0ckxVk3k/d5xg1aVIt7g8YU1Y022tLS1u7u8i37G2iMjJ5qmrIUkl81OQTtJ5FUCAxxhWVD945prKsXLOC56nvSrJaywSkqyysf3YPoO9RzwuhXeuM9PcUrADPvj2Ken3famT/ujlsFiOx600siH7jbuzAcU+cq0Qj3BmLdR2oaKLVwi2lvCoUGaQZ57U+SWNrW2ZRiYnDJ3GOlVbkb5Qck7Rjk5qS1mQKUZE3AZUY6ipaAsujXMoaPK3C/cGeMnHeqqWl35syeR++Qk53cE1uqqXaxJDsgjYYf1Bq1JabbidntxEnlBdpP+sI5JxTQkUJrq5j002NzOhZyFyvLIvdT9f6VK+r6lp6+QtykeACoSIBiD7/hSG4hh0kTyQxzyvMU2McFR+NZVxcNdX8MqxuqJ1DMDj1qrlWOo063ttRgN5cJvRBvdZTwGB/Wuf1+109tZee0mSGFwCETpu7gDtVqzt7m7kWBUkjimJIkIwGA579ay7ySE/ZIw+5owxZ8ck56UIzsP8+dbJoIpS0CjLrH2Hqat6VbWF/ZXSvG63CxYQBiAXJwDV/wxZadqIvIpruOHzGTeC2AI85bB96ytbt7LTb/AGafeRXEe5iWhY4AzwDUjMh3lt53hlQb0OGB7Guh8K2theakTdagdPmiOYJo15zznvWJHZm7gvbySQ7omXjHLZ969G+FukRo1zeXD2sizxbI4ywL4z83B/CqQytqNjqXiTSry+utclkXTWACMuQUPVuO/HP0rhRHhyGSTb/AQMA/4V7Zpmix6f4d1u3uk2xTSOWTvswcAEeorzKXQ9Vey1KRQzWVhhHL8bRgED6jIpiM9/D+rLoP9tiz/wBBBK+cpHBBwePTPeug8HeKntmh0q78hreQNEr5+YFs/pmvVdCtodU+H8On6hbrClxAY5EAwfTP1rwnxDodtoWv31hZTGaG1xuduoPvQBu6jZQW3iG20XyBsSAjIyN7Nzk5+lZVxpE1lqHk28UjqqDEgOVOetU21q7upImuZnLKnl+aB85X0JqeW8nTTYg90fMjIVNhxx6VDAvaegTUYbuaTcOVVs5wxGBWW/2sXrQXAD4kIMpHGfWp7OVRPHFDyZ2RTu5AJPUV0NzY22lvfNNbzGa2hwfNbhpG/iH+e9KwGXAkkELBL+RlkYM6A8ZByK9Y0nxRpupiOAPtvsASR7eSa8lWTztMkESbpTMZMKuDsUDk/nWh4PTU28S2qQW7sS3nHzeBs6Fv1oQz2ISbhxn6Ecinhqc208gVHkZrURyfjZs32nL6RFv1rrIDizg/65J/IVxvjFs63ZL6QH+Zrr4m/wBGhHpGo/SnYZY34FRtMBTMkimFM0iSQTZNSJPjviqRUilBYd6B3NmK7yo5qYThuprEWQrUy3B9alodzY84Y5NV5JATwaqeeMdab5maVhloNk07Zmq6NzU4k4pgIQVakLHNOJ3Con+UFj0AyaAHFQ3Wsu7gu2E9zKQPLbdHh+CPQ0y38Q2EzOBcDC4wTxmq+q6jb6lDJpQlYGZMpNFzsYcigkoXuoandX8EKRqIFYK6K+4MfX8MVPq2h6XJM0klttuVJH7kbSR3J9a5XwjqMukXGbyYsGnSLazDKEE7v5jmvSZrWC6bfndn+NDSA4vw9Z2clpqNjc3gmE4wqkEEEZwf94VyV9bXaxyRPHsv4uHjc5BGMArXrUenWtvP5aWyGNh8ysMlj65rhvH8H9k6ho93aSOzoxGx+gUYbb755poDl57ny9Bt7VS7ecR5g6lQvBqa+0GWxtUuSu2KQqREjZDLwT+lWJ9b03UNXLJvSGRSTvXGD6fp1qa81ma6tbU3LJGscLAbF+b0zjNMZyl9cL9ghtF6QTFlbuc9K6iDSf7P8I3msT3IknkVUVTwRhh096wfIjl1rS4WHzO6Bz6jPA/Kt7WZ2sJxbvM4ihbbuK5RQx7/AIZqQNTw9o8txerrV5IkcBiFyIjztHIUE/QE+2RUZaG/1+W40W5KW+S95cuvyxg88E9c81Tg0q9vrWNbB5lSfIkUnIRM4Dfj/SqtlYNYXd5bmXzbSOISTSqcA+nHc+1AGvLJ/bTmHSkMuxC01xNJk4H8XtVHz2tZwoSfcsh3SOuAT7Vv+FdZsxol9ZwRnzBEN80yhRk8ZyK1tO0/RpITffbo7y4gzlgeM/Q0wMSG9t9OspVNowyx5JyzmucvFla9P2l4rd5TuRI1JyD0b3r0S4tZb26soI0RbmNmeRx0jGO/5/pUkHhDT/t8d881zNcxMctK3DcY6dqoDlm0/wAQLtZdYmNozKGYAqWGB2/TmvSIJd1rET94ryaq3Wo2liTbb0jYddw4H1NcoLvW5dTuLqzLXVoQAJZPlQDPYD/PSpA7ZWzTwvNUI7+0N0bVLhJJlGSinJp9tPP+884Y+b5fpVAXwMGrCDIqiJiW61YWQgcVIFjaDUUgGcGhX55pXwTnNIZWZQKOD0NPdSarsCtMQOBk1EcU4tkc1GetAiRTVqHBaqVSJLsoAuOM5FV2ibGMc0LcAnmnGcetAFF43BIIqIgjrVqWZSarO+aYyJwT3xSAsv8AETSk03NAh4kPc1Ikg7Gq5YU3cM8GgC/uyOKEkAPJqmsmD1NP35PWkMvrKuOtTJNzWemeKtRUrAX45qkabiqoIAo3j1oGDsCc5oNwqjrUEki4xmq7HNMRb+2gVGbvnrVIutA55pgXVuiDnNTJd/N1rPFPFSBq+duXk03zUHeqSu+ODTHEjA00BNPcKV4NNjlDrxVPyJD1qaNTGMGmBKw5oXINML5pN9IRbSYgU2SUk5zVbzMUhlzQA5z3pgamls03k0ATqy5pxIOSKqFivBp0cuDSAVmYGm+Y1TNhxgVBIhUZoAcJsU/zVYc1SYnrTN7UAGv4fw9dAfwgH9a4UdK7S/YyaTdxnvGa4teUBpgNcqg3PwKapWQb16ds1MQG4I4qPAUEKAPpSGVVbZqFvJ/tgVta4MalE3qmP1rEuP3YWTGdjBsVs6tIssNpcDnIIz69KBlfgj8KbkdKZ5h71HvJc80FExOFrrfhxJ/p1/AR1TdmuNkfC10PgS6+z+JShPyyxkGgR6PJECCCOKgCxR8CMD3qWScFjzULuGHFNEWM/U4XjYSwoWB647VRjumHDHBroIZ9q4c1k6jYZ3TwL8vp3rWJEiA3TdmqaCVpOpqK1SK4g2MMy1fisxGmBVSJGmk+YdKnMJFCp61lc0GRtITgVLIJQnTinhQDwKeVLLgVIGaZGzipII3mbHI96cLYrOCw+WtRAu0cUwK39mMQD5w/GkazaMHJB+lWSzKcUNJ8pGKQGVLIUbFLFcA9TTLsHPFUS5U1QGykqmplcHpWQk4HQ1OtyRSaGXnbioQ+DycVXa5PrULXGSaANAnAz5g/OoTdbP4qotMT3qNizUwLU1wHBKmoBIT3qszEdKb5jDvQBfSQ+tWI3Y1l+aI1LscAdawLvx6ltMY7K2Eu3gu5IGfakB3Q3VIhOOSD9K8j1LxrrV+uxJY4EPaLg/nWAk13GwZbyUMP+mhoA93nG5cVTc7eDXltj441zT5QZp2uYh/BIev4132geIbTxAjbI2SZRkqTn/PSgC3sAm3DNYfjC2MlrHcr1QgV0bJg1la/E0+kTAfwjNAHPeDrpLfWlEp+VxivQ5RF3FeP2Nx5E8EpOArc16lFcJdW8cqOpVl7GtJCKWooqwvKihSvoK8slut/iAMnGZOg6V6xfxlrOXH3cYJ9K80NhDY6ks1yyvC7dStQBt6rr97a2I22kac43AmuSF3JdTfvDlj3q34ivkklVLSffbkA+tZ+m7TeI7khR6DNSO51E+oxWmitDCuW2kbWFcbJM7uWCBc+ldPcT2JbdLIdvtxVGeXTI0Z4WcsfbIoYHPyPIPmIJ+lMa4kxjca17q1Ecfml0ZD2SsiVELZTcAexouBFvJ4pOh5FKRjr0preXjCls980xEnmY6VKL6URiMBQB3xzVPdzinUgJRPIrbldgx9DT/PncYaRiPQmqwJzTg3qaCiTdgdeaeJBgHp61CXUJxyaYH45pCZZLqe4NMJIP3h+NQ+YKejBzg80CHrJk4yKc5x75qFogOQTTdxHc0IokxuNI8ZUZOKZuI6U9Q8vTH4mqER4J6VKIZCucU9YNjrvIwT2rVTyljADHPsKhuwGRCG8wFACw9a0pJJJ4dpjRfcCns4HRVH4UK2Vwaz9oBmrE27BwKZNEynON30rVmaMxYRRuqqzeVEZMBhVc4Gdkjqp/Kl3ex/KrSamx+4q/iKruwLFj1Jqk7gEc0kRyjFTSvO7Zy2SaiIJ6DIpyxuf4DVgPSAalpkl7cFy4GVVOi1Bp1nczW8kqbNq/wB4da9Ej1DSptOkFvECu37gjx+tcjBbTwaZcyKNqnOF7iq5QOeRmyw44P8AD0pQztwZHA9AcVNb2rtGX2PgnsKgZdsjDawPuKkDR0kGOVpmYPsGQshzk1vyays/7trZUB7uOK5LcwUAHFW0mEo/e3D5oAtXmouJ2xDAy9iB1qpBEZBmrSzJCu0Yb6ioxMAeBUSGPJC/IwH4UpKnGDjiot4Y0juFGTWNgJlIBq1YTLBfRzP91c5rKa5A4AYsemBTfNnddpXaKLAdxe65DIn7y481m+7Gi1o+FdKutRQXF7fGGDOFtVPJ+v6VwWnTpY3azbNz+rHNdppRVLoS7mG9uCDiriyD0aKzt4I9sEKKv0qnf6NbapaPbXKcN0YdQakje6yqFhReWt3cxeUZ5IlPUxnB/OuiExNHA2uqjwZqM+l6rI62+N0Rxnj6e9QX/wAS7WKULaWLzp/ekO3P4VreJfAts2mveWz3El3H8zNJJuLD6VzGoXek3/hiQzbE1CP5RH0Oc1pe5Fin4q8Z/wDCQWcNrDA8MQ+ZwzZ544/SuXD4FRtwcUmcVDkapFlZSFGDgk4rttO0f+xrdtUnm8yXZnI6f55rhIlaZgiKWPoBV06hfPAtrPcM9sDzHnGR6ZqFqNl28kufsc2oXMThrh+GxwD6V22n+Ilm8AyyO3kugMTHJbINcLr+trqEEFrBAYbeFQoBbOa3JdF1C28EbXkiEbsrGMDDH607EHLQy3TRz2VrPI0UhKkA4DD3qxYedcTWemORHG0vOByK7Lwxof8AZAkmliSTcnLKckZHTH+elW7HRY73X0SGFYhIMu5TLqKGh3DxjYQR6AkoT543VVbPIFXNP1CQ6LbeXCJWhi+8TwDWxqvhuK90KXToZWbyWMgeU5OR6n8azNAsIF8MtFZ8z/cK9857VLQJHL/2fLO8mp3YEk33gAcDNJZacI5TfXsygk58vGTXZr4RnOmPJd3ZjCjO1RjmuOudOuDFIzOjQqcbs8tSbLRSbUJyCvmypb5J8s9KpyTu8bRk/u26sOtXJUt44/JMfzj3rPuFESsEH7sYwffvSTE0ael3cIhlsWDMGBJZj/SsuC6YXaMwd4o8qV6ioIZJFkeSN9rBetaMOgXAs2urmXyYjliO/wCNU3cdijI0ZkZowVjJ4HpUUjKAcdQKkW4iWNcRhQByeuarSSIxZhyCeKzsNGcxOSecnrmmZNWhGGbtknjNOa28tmDD6VaEUWDEDjknFMyVbBHIrZtooo0YkfPjIPpVCaFdgk4yfQUwIQmYvMPTNNB5poHOKUYDcjNMYh60hBpxwTx0pOKBEiSMnAPB6ikbk5p8ahyFNEq7VH1xQAkR+Vh7UqjEfJ+b0pVjfZwjfUip4osRs8jquOgJ60Aa0Usc1zHFGcrs+7XdaYv7qM+gxXAaPbyNexyhfk6ZrvseVps3tG38qCTzXUZmm1e4nb+OQ4PqKihXNwjH7qnJFE5kKQb33IFOFx0yali8tNPZ8YZn6+tAFk3nn6bLaTc7HLq2e/0qBJXVQ+4gIuDt6mqDSsCcHrWhBIbaMrIoPnpgE9R9KhooqSOj42ZGO+eTSwK/npujEgBzsPRvai1tJLu68iIgHrljgAVtadpMB0q51G4uUQws0aowwAR0waaArX96Lu5B8rylUYC96jJiEkfmttRTkVSlJeyhkwwlcklz0IHpT5Y5EjhkmOFkX5fXjrTEXJDEsatHEYixI2DgOPWoPNDSErtJIwPlxg0rrK6EL5jFRnc3aqih1DEHH0oYF0fubeQTFS7nIwfu1AqzqwmZR83QjjNPQLcQCAj970V+5NWJVZzHA0PKgIStJgjW0eGI6dcXc8Ym8oglEPU++T2qjc3dzZaobiaFfN7BjkFaebCONTCJ33/3e1XoQ32K1lfAfzTEXJG4k9OKgEYt3cQyStMo27gCR71F56RSRt9/jLr04rp7XTbO60nVjNCFFvuYzdwew/H0rkIY/OuI4s43ttz6VSZVz0/whdXOpeGLi0KlngcxxNNxlSMjntXm15A1rdzW7/66Jyrema29K1G+k8TQ20WoRqTKBGWbbExA4Bq5qvhTWmv72+vLVWJcyMInG1lxzg/h6VoyGcrBNJFk5wex963YdbWOAqLeFvNi8qXcuS/v7VPrPg++0zTXvPLD26xrKXzym7+E+v1rKg2tKscm6BW27eOQc96lgjsdG8H/AGrSNSNzJiSUr5axnlSADg+uc11+neFNHtFglWApcR/NvjcqQ3f1q9aozRBACSiqpcj73HWp7i4ttPspbu8mWKGIZZjQIo+JPIubKQtd+WgBMcJbAlcHr79elVZ9OWDSdQ0q7IW31JRIsqcMhCgtn6YHNef+KPEUOtasH0sSCFYjEhkGDnqSBXZ+NJr7VfCJbS5YxYxQrJcTBs7wBjYPx5NMpMg8K+N4rLwzdLqt0Xmt2eO3YJkOAOBn1ry2V54vOSbO+ZtzljksK0BFeR+HGtbmyKW7TrIs46qxA/mK1vBejQa/rkcsjwr9nch1kfBlbGRigZx+cYX2rY0JoopZpLiKOWHaAwdyuPcevf8AOuu8ZeCotP0271TzHNwZNzshxFEDgBQO+ePpXFaTpcms3ttp8cgjad9gcjIB+lAHpvhrwpaS3cGriWCeFPnjSHpv7cdsVv8AiSwg1HTmkuUB8hWZRj77HAGaf4dimj0K3W6hEd2g8qUhQN+3gHj2rVChsbucEGgDnPBXg2zi1G/mu7ZQYoI7cxdQ7MoZmPvnjiuxTRbK3uvtNvCsTrGsSbP4UHYUts3lNuXgt973q55vepaAzp1IY5qDaScYrRcxSj5jzUTpEo4qyTgvGMfl+JLRM5xag/q1djGmIY/9wfyrj/F7b/FMHtaqP1au1JCpGPRB/KmMjCMO1IwKjkVMrjvTLuD7TGApwwORzQiSuWpoqsxmjOJEZR6npUiSlgADWnKTzE+OKToaMkCmFuayaLJdxp6sSagzTS7KeCaQy8pxViPmuY1bxPZ6J5Qui7vL91Ixk4965G7+I+pNJcrBDEluw2xKwIZB65HWkUekXWv6NYy+VcalBHLnBUsOKJtV06YtBFf27uwwAHHP0rwWa4E8rSyHLscsfU1EZY6nmA9K1K10vSpP3w2u+WXfyg7DPpXPXfiCSC6CC4MgjYMHjPDGuYe+kfAlYuvuc1Gzeac9KOYC/c6j9qmaSQjLMWOB6mtDS/FOpabLEIL2YQofufe4rmpQ2QVFIrsGAYYzUgemW/ie41/U2tVkkSIjdGQcMCOuara1cGCW2eaZ7iaA4IlHyuT6e9cRbvIJFMchjdTlWHUGuvj8RQWFhDFd2sd1eEbpJW5AHY0JiMmUrZajJNJEvnTHcmF4jb0wetN1O+N+PNltlSUYBaMfmKsaprkWqq4NvGjFMblzkEYxj8qZp0kF9ZfY5wxmjBKybsZJzwadwMi5c29zAwbLAbwfQ1NfX8t1YRxMcozM3mHq7epqqbZ31GOOSQ7QQu484xTAZLq6UD7ofaAOgHtVXA7jSrqzGj3ELyvLLw7EkgkBcD8OvFYz3F5b6beRwwHZPKI95HO0dqsDTrm3vYI4o9i3AVGTOdx/pVjxLayaREI2l5MXmrznHOMfrQBz4unt4DDbg7JMNICMZIziuz0XUYf7InDKkCAcYGGkY84X6VxNlE1/exIxG0HoT1PpXS+F9HnumhvZoUjt4Zcpk8Ejqf5imB2mgrLZQJqd2Zmvp2G6LpjGcD9ae2uiKzeONZPtM0hYuTkKM/5FQ3er2Ny6j7QYZSMRseD+H1rIntJEdJYTGzNneg5IFUSay3drqN9PLLIoPzYU9FOMf0NEdtff2cYbW4thbqxLF4yTiqFzaamYh9jto3QqCc44J7Vu6VFLbWgimYlh/eOTSsO5StdHig1Lz7eRo5UIMhz/AKw9wB2H+NdFnNQgDOacH7UxkwapVlI71ULYNG8nvSsFzRSYdM04zcday97etPD8DmlYReM/vUTzx925qAyZ71FIEdST1+tVYB5nG7FO85fWqZAHem7/AHp2Fcv+au3rTN4NVA5qRDkVIyznBpWb5Tioc+9HPrxSGRySHNRGSnyKTVdgc1VhCmT3phkx3qJs54phYigRN5nPWl34HWqwOWpzbgfagZZEme9OWXb1NUhKRTllJFAGnHOuM7qniu0J2hqxgxxxQGZTkHmgR0X2lcdagmuflO081li5bGCaa0x9aLAWhPLySeBUjTkwnaefSs4zHaRmohOR0NFgNOFi6kn71Tb1QYNYrXDjo1NNw56saLBc1ZbsAgKael3wATWKZTnrT1l96LDudFHcqRnNP+0CsFLrAxmpluc9TSsNM2fOU00tms4XAz1pxu1HVqLCLbGomcjoaqSXq4wDVf7X6miwF5piOppv2j3qg9zkEg1F5/vVAaon561IJvesgT+9SLP70Aanm7uMComyG4qukualD5FTYCzFKQeTVjcGHNUA3NSiSkBI0anmoWiLdBTt59aUSEUARmzeSORem5GH6VwA6Y/z1r0u3kJfGa83laNLqaMt8yuQR+NA0HamkU84ppZfWkMo3gxC1a19+80izcdgBWZdlXiIBqzbNNJoOZZQ6o2FA7CmBHtPfFIF5zTBMv8AEak3gj5Y5CPUKaAH8VZ8PztB4mtGX+Jtv51TKXDrhLaZge4Q1b0nSdRk1OGX7PJCsbBi7rgAVViT09gQzZ9ajJxUZuSzZzSGUY5pWAkXrirMQGMVSjlXeOasrKOxp2AnMUSnKIAT1NJ0pizLnk1KrKRUgIBup4h9aaSFGRSpcr/FUgP8selSiPApvnIV3A0guFpjHFV7imc/wmkklUrxVV59p4OKaYiyzFRl+lQvOoGRVV7wlSCarNPnoaAJZn3g1mzZBNWxJu61FOu4EimgKYlweTUonI6Gq5Qk0szQWcBmuZljUD+LvVElrzWbrSjmuC1DxpeGVkto4o09R1osvG+oJxMkUgHTK4J/GpKPQkgZ8YFWBaPjkVwEXxJvIQVl0yGQ9irEUjfFO/RxnSoRH6ZOfzoA71rZT2qCaCKFfMmdI0HdmxWZF4vtdQ0eW5tT5UyjJRzyD+VeY6rq1/qshku7l3A+6vQKKQHV+IfFEUyNp+nEPv4aU9B7VyraZfpy4VV9Qc1mRyGP7takWt35cKQr5/vLRcC9N4WvYrMXIu4mXGSADxWQ5mgfaxrYnvb6FR+9yhGdgGBUd3dWF4kaPJsdegcYH50gKgmvIhuKMEP8QWrPhzUTp2vwyCTYkh2uGGMit5dKgvNBhfIR16M3TNZfiNbiLS4I7iG3bp+8jNNMD0+2vbXUVZradJCOoU5xTbqBmtpUZcqV5FeOaZfT6bJ9os7kxP7DNOk1rVZbkytqNxk/3XIFNAXJkaO6miIwFbivRdFjF3pqSq6o+ME4xmvN4/mUOeWPU9zXT+HtSnSJrVW/2lrSQjq8vEjRufl/vK1eW+JfOttUaGRy6dgfwrsLi4n8/d5rB16YrE8QzLq1ssMkYSaPo7cfnUAYVlHILYyGAMACcE1ow+KII7f5rZGcfw7MVSitdSstPlZ1WS3C/wALc1zwJJJ981Iy/fX/ANuumm2hc/wr0FVt038MbbfXFQHcpyCKnN7P5e3dQA1pXU4PB96cs2U2kCq7SFj8zZNOWNn+7zQAvyuDkgfWoihB4pxUqcMORSc0wFRYjzJnPbFNK8+1PVdxxTZB5ZxuNABntTTU/kw+WG8/5iM7fSq5XgmkNB1pppNxB9qkVMqDnrTYiLODzTgTnilkVVPXn0IpNpxkUASBjmneYB/Dmo1UEcn8qM7Txk/QUWAkJUn7grTTTo2t1lDk57AYxWYqTSdI2P4Vp2OpzSRfZxGhx0oAk+yW23cQ2R/tVCYiufLDPT5y6cvgE9hVVpZANymudoCZmkKHcjKR61EZ8Lt6Un2qdlIdN3pioShfmqUe4FuCNZSA3INaf2WFQBs596x7R2jlAJ4rbyWGTV8qAi+yWw6Qp+VL5MQ6RIPwp9FC0AaFQDAjT8qTC/3V/KnhCeQKRlKnBFMZQ/4SeUR7Le1hVvWqdzqN7dIwfAVuoXgVkI+xs083MhUgHrTEbVjAWstwnMQzgALnJqGXTZlbfPvBPI3DqK3PDUktxppBRV8s7cjvTNYZ4lw+CWOFIPSgDEgid8rBD5rDqKvW5tEVYr61RZR70yyvFsAHY4U9fWrV1e6dOhkVCWP8dMB91FZm0MibC46KvWsK58sSYckH2q35MbXAMD72PbPSrg0666mOMfVs1LJMJ1RE/d4Vj6nqaEtmMe+STJ9Knn/fly4G9TgYqNXZUCntUMpjAhz6e9IA6995p7ycYA/OomyOST+FSBIA5P3a3bTWZkRY5EKgfxDmsKF+e9WfOK9KEI7jw9rkK6kkRumbzCBmQ4C4r1KSJpOUGV9a8E0sJfXaW8kYCnuvWu0juL7Tdp0nU3l9YpTuBFaRZVj0IWz/AMYwnevAfE1vDF4nvo4CPLDfKAfau61X4gs+my219azWt267QiA7SPXP415hGWaZnZtzE5yetaqQlEh+zGS7zjiorwRxnZt5q/KNqFqh+ySC2XUZI1MKN/FyDUoqxal0a7tLWG6tWDs/3lHUCtPw1p1u63EWowbpufkYYI+hpNA1SCa/8u8YIpP7rJwFPpXpi2rNboFkgWVhkEIORTQmjzOHw+Z9Xj8m3kMSMGcHnAFdhr8K3NgAshCxsF2jp+NR2zppEszSz7mJIeNTkZ9RVGC9hupRCpmJ3biqjNMyaN/w+sUNn+5yZdwUknqMVet7iOO8uJppY1eRdqleDmsy3E0soit2WMHogXJqze2S2dvHNHH5l9I2EDHjPrQXc07iWHTfDd2s0ybip7471i+Cr2OPR72SZwAoypY8t61w/iKO7jv5Irm/WeYnLRp0X61nCSSSLy0eYsOwPFJsEz0y38SXOq6Y9u+nS/Z3ypkjOciuc1vU5rAi2gjMSsu4qx5BqXQtRnbQfsqbY2iYbmUckE/pUXjm0jtru1lRx88YBVm5HSs2K5yslxtBaRvmq6hiuPD4AGCxA+lZd3CrneauWSTwIm9A0ZOAjdKEirk0tvFHaRAsBt7nqahvLzbEAZpGB7Fqlvt15epDAudiZI9PWs7UAXa3gX7vC59M0x3IXJZOKXchQDZjAxnNX9S086Xd28czqY3UdB0qG8W1FkXicl2IAB6ik0MoM2yX1HbI61Iz55PeomkYhQSNq9iKejRsuW69qaAsCRDbsSApB4UHk1UlV59rllxjAUdqUOCm4sM1GZMUgIHhZBkiq5GTVmaYtFwe9ViaoBegpO9KFLDFBG08mmA5H2tmnM5JpbaCS8uo7eIZdzgVtjwlfDrd2w/E1M5xgrydgsYiyP68U9nLIRW4fBt2OW1C3UepBpy+DrntqMDKf4gpxWP1ml/MVySLHh1mdkXqq9PxFdHr0/2Xw3dODhioUfiazdG0saUNpuEnYHJ2jGKu6vD/AGpYi13iJc5LHnNH1qj/ADB7OR56+WiWQfdxgUNMptliXoprpU8LbOl+o+kYpjeElc5N+PwQCj61R/mK9jI5c81qx6j51kLZ40VUGFIHOao3EHkXUsAO7Y23OMZqxpbeTeAPCjh1IBYfdPqK6DE2PDmhXGrPJJbTGOaN1RSF3D5s8kfhWj4l0W78PSQW121rdIxZkMfy5bj7wPSul8E+IdI0TwzdhIt2po+TEF+eYk4XHtnt2rmdX0rU4XvLrVIQ9xMhn83fnaM9B+dICjpl9aD7PFfxpClu25dse4MT6+lF95V54fu74KklwL7G7usYztAHbmmm6gishbzWavLKoYP/AHh7im/ZLCPRxH5vk301xgg9BH6mgkhmjlNpHJJDcJNdLtjcjCMfQVQhQpK9s4xIuQR710+otqkuiWVoEt72K2PnRy2xyUH8Wa5p8S3b3JODIc/jQM0NGQW2qSExsz7fkx0zSQT3YvpPLKK7FgWb0q34a02XUrySJb2OHsQw+dx6DmtDXdOtrS6sII2DQwjDttwc/nUMZzEaiW9DTiaTa3/LHgn6V195pmm20drfLGyzSLvVWbcCfQ/41S0y2jhv5JlUjyThVYYIHeun1OAanpttcwQY8i42kPyADSsOJzMI0OGK4l1e2lc+YqGCNyFJ5ycfl61WTw/bXvhPUNYt5fLhsZTGjBMvNlhjOewBFekaf4TtBf2jXdqsq2sbKXKAI5P3eO+K0F8J6Zb6HqWmQB/KvNzurHIDdsVUAZ5T4f8ADEmqaI2qpJtmhvESONk4ckjpz7/pXQeJvFd3Zanc2dnDtt4nC3AlXiQ9x9K6v4aFE0a5sJYsva3GcMPu5HGPyre1DQNPv7W9gkhVftZBkcDkkHIqyTO1WzGueH7a3QqIbkxtKO3l9x9OKxrrwXY3uvTXc6b4XUMgRsAEDAFdZHYrawpDBxEihVXsPpUkcR53cVQEWVVQijCgAAfhVHUbKG9g2yQpKy8orjIB9cVqyQdxVcgg9KBHkeoabBpXiRbi6wlsUkkj3HiQjgce5JNangOObVfBusacowktxkv/AHQRz/Ku+vtKsr5IpbqFZBArYDcgDHcVyXw7tJo7TV9PdXtJbgLJEW4O0gjIqgOc8Saw0Ol6b4f063HkyBXlik5Lnd8vPv1qWXwXrGlWttLBGzX0c5laOJQRERyDnv0qPxK0Vj47ghW23+Q9upJXlgvUqPfNew3Mf7wN/eGaQHPXBk8V+ErJL2MRJOyPdqTgFVOW6Yxnb+tcP4ZXSbfxHBNbKtvaJfOY5pWwGUL0ya9H1Oymn0W6tbSNWlkjKIhbaDnqM9q8p1O1ufD98kjvbebbzbzag70jJ9+9Joq57J7jlT0PqKBWR4c8QL4i0lLlkEdwnyzIOx9fxrU8zmlYktoTiry4KD6VnxOCtTm42x4FJopMJFQEkMc1XZiSKTzgc5o3qe9CA4nxH+88Wn/ZhQfrXYtJ0+g/kK43VX8zxhdD+4qD+Vdc1UBIsnvVhJsCs3cQakWXFIDUwsiYZQQexqnd23KvEAMcbelC3GFxmneeW71SYmBUFKgeNuwqTdgVleLdf/4R7w89ymDNKfLi9QT3oYrlG/8AFen2Et5Du8yW3UEKCPnPcD6Vz8HxAa5uMXMbW8R6svzGuA+0MRwMA9fc0xpmU8VkM1Ndvxql8biJHzk5ZmySD0/IVkPJOvG0n6Cnm9YDk0gu2zycCpKTGQxvI/IyCKsi2UdH6HHWq8t1leDj6CmIWMfP3e1OwyyVjBBYg47A1IssbKF6HHWqTPn7ufwFPgJQuWx8wwOKVgLPQhSCeOtMdd0ZXnI+7TvOTuaDOBwcAH071IDAJkbpxin7yBlsc9sUqzI3T/69PWVD/vetCAiWbnGCKdnLZDGpC0e7tk+tQXAZAskeAc8jNMCcbs5yc1s6VBbzMvmOuY/m+bgMfQ1gC4fjNXLO4mWTZFII/M+Uk9MVSYHcLcrBaHV2G6GBmSIKcc/7Irmr37T4hvEkZsNIrbF7oo/x5rtbTSWj0+JJp45SU4YHK7eeB/jVUrpukRAzNCPNIO5kAKIhwatEHIpZNDcRQL+5uxztPJX3/AVvx6lbWtnNE94jPbxAITJy49MYrD1m8aG//tC2s/LS5Vlh3DACjAB+vr9azEjsjaFpEmE395R39aGUmamnahFceIYJLmeSC3ZuHyWZV/LjNeoW/wDZ95bXNvYNC6Io8wR9iemfevI/DtnDfXM8TLM82z91HF95znkCum0nVP8AhDTqltq4aHUvJX7MgG4EYJHP1P8AOnEk7qEyIqoybQoAAzmpSxrC8Jane6/bXGp3sKxxkhIQB1A6n866HYMVqSRhzmniXHemPE2flFRbJPSoY7lgy570CUDvVY7x1qHzSZWjz8wGcd6QXLzzDjnqcUzzh61k6pJtsGkPmbVZcke5qS3uRc24mQfIxIU/SgLl/wA49qkSXPeqIY57/lU8ecUwJH+9T0hkY9OPWm9TzVqO4VU244q76WJtrcheJo+uPwpFcjpV+B4i25ucCrz2NoYxKkaksMk1DLSMQTc9akMqbeTTry2jiYFB2qiwzUMokabBIzTC4NQOGDUgNNEkjc81EwzS5p6IDQBGEpSrYxVxYOMimmI96BlAxGgIw7VbEWDTvLx2oArxq23oakij84HAIPYdzWhaERtntWgu0jIUUgME2c/aM1WcFWKnqK6Zzxjiqc0CMCQi7sdaLgYDNioWars8BTtVN15qxDNxpC5pD0phNAh3mGk8w0yk6UxEnmn1pRMR0NQ4oFAyz9ob1o89/WoKXNFhk5lOOtBlJquDThRYCXeTSjJpgqVaQgAbNSorGhRzUyigBVBFTKTjihF3VYiiXIpNjIh5lSqJe4q6LdO1TrbAJkkVIygqt3p22rvkqO1Na1DHIoAggOJRWVf+EdNvb17lZpLdn5dYwME+vPSto25U8UeWaAMWHwfpqZDT3EufV8Yqu/hzTY/E1vaeXI1vLCWKM+fm7VvlthrI1Gdk8UaWwON0ZFFhmgvhrQxx/Z0J+uT/AFri9PC2c89uUEkcNzgK3TFd4s4Vsk1wkp26zqQ9ZSwq7EHbCWzjzss7cD/rmP8ACmteY/1caIPRRiqiEvCjnuoorVQRm5snF3JvyTkelNaYk5DH86hpKrlRHOydJiDT2mzVYGng1Lgiucnh8x3+VS2OuK0FibbmqVuOM5x+NTFmHes5KxpFk+QKeJVHes95znGaaJ8d6nlZVzSaf0qIyZqotyBwTUqTRk5zU2C5aiZ2GBn8akZSBzUP2tB3prXin+Kiw7jnkKmoHkLHmmvcITkmoWuEB4NOwh7DNRNxTTcJj71QSThujUWAmEuDTjKHXANUd+T1rF13xG2iMsSQeY7jKnPT9PeiwHQXUsVpayXEz7UQZJryjULqS+vJZXldxuIUM2QBU1/4g1DVQVupyUP8A4AqnFGzghADUgSrYx/ZvM+0ASgfdNQ7HHTb+dK+9GKuMEdaZupDFkjmUbtw/CojPIvfP1p0rmNCeT7GmW0VzfuVtoDIw6gdqSYDWnkPt9K1tMXRpbcm83mf+43esu60/UbJQ1xbsinoarjJ5p3A7qx0fS9TTMNq8Dg4GRgUyWxW1ujbjy0kHQvwK5a013UdP4t7j5P7rDIrTl8VSagIhcWULSoCA+f6UIB+rwyafdR+e6PE4zlDmnHTdK1GH7Q10toB1B5rHu5Hnl3vwPSk+zzSY2puT60ASyPJpz+Ta3gli7FTkUxtUu2ga3OySM/wsKaq/Z5PLeJEB6d802eJdrMpIz6GmgIfIk8rfjikifb1q3aXzQR7GAYdwakn+z3OJI4gkh646U0BatZN0Iq3DdSWk0c0Z5BxWdaf3atTKfJNVe4HWvFcSESMnyMM5rF1Xw7c6hKTFNtx13tgVf8ADesm+hFnKhV4hgZ7it25eMwklArdiKQHN6Zp0/kyWFxcRtlf4TkGsHWPDklnI6wICF/hU5NbrXBhvPNH3xVTX9SMphnWORHzjcDxVNAYVv4evXhBa3cn1xUX9iyh2EoZFHcCvSrLWYYtBW5Y78fw9K4jV/FYv5CILcRr6E5qWgObubURSDY+9PWlt3aCTcuCfQ1LLN5pJYAA+lVGYZ4pWAllDu+4oVJ9aXy3I4UY/wB6o/t0+3Z5h2+lLFdRom1oA5z13Yp2GB3xHpiomfefm5q9Htvm8mCA+YegLdarXdlcWThbiIqT0xzUiI/MULgAZ9aQPkYzULcnikANUBKy85pgcgj0pQexNIcBulIDTjv1+ylHhjfjAJHIrOw2OCPpSrgU5cE470wNPRFjmmEUsSMp9q6Q6da2bq626BicVzmjh47wHb8vr6VvSytNcgNJuwcjiqA6FfsBHlyWuwn/AGc1yWsaUthqKSQoxSToAK7VGiuLFULANjAzXI3FvdR3ircSiRNw5z0pNEmRO0mQxjI+tQNcoV2ugH0rs9YtIn0l5BGuR0IrjYrzy02mBG/SspDTFSS2C/MHzUZmO4+Whx2zUDyh35UD6U3cc8HFIZKpcSg1vQEGBcVzyt843Ekelb1qf9HAppASHIqN+RTzTG+6adhmdKW801agc7RmopRk5pYDwKpAcsGU9TinAhuFq5b6Y0untdE/d/hxyarJbXCL5wjIUdaYjqfD0TWtv5kkkjDOViXqa17y0SRBevGVQ9Fb+tcxY60IFRp1yUzhUFbc11qWrwrIsaW1uBgbj96kBn3P2RkZpYsegUcCsxVjl46r3FXLgyb2imnQoOpq7atFaoQU3++MmgBLCOeNQ9vbpGo6vJyKqXU9w96T9rBx/c4H4VeGrPGSHt8QntnBqi4gmk3xpgfWoYht4u5ldEUDGOO9RxQvcJuRcgVJMwWIgmiO8ltrTZCqkH1rNlGdKdjlWQ5FRtIp/h/Wppyz8t1NA0+QjeWAFAEKZDZHSn5YnNRsjrxUizBMK4waYi/aLubaCwJ7g813nhiJ7iIKYUhVDgtnk15/a3MccgbPSuoi8WR2Wlsy+QJ8YVRxVIZleNL2S51s2bOjRW/3dvXPvWDEmXpqF7ud52++7EtUm/yjwpZvQVdxoluRuhNKlz9qsoNM24Qt8zZzn8KkTy/LUXRdUP3lXvVe/j0+IB7Kdy3909qEwKl5ZiznaMjJHIPqK6ePxfNNosdvONrIoQFc5xXJuzOcsxP1p27bCB60rkluS53OSGkX6sav6drk2mFvIKOGHIdc8+tYGTUiHHYn6CouBty67evdm6E2xh0VBgCtmLxLNdJmbUb0j+6owK4oyjODV1bxY7JlU43DBI6ii4GomqxXaXEcVuoQDBlI+Y1HpN8unwTOuGnY4GR0rKt5jZWkgb/lpyPpQky4ye9UmFjaTWJYkk8uBAWOWk7g1kvJcXN150sjSD+8TSq53gMML61Irw52k9akdiJ9+zcx+WpPtZkVcZBFQsxlVkyAF6VDExZCQOhpgbcN41vbyPCqLMwxnFYeya5dV8wKynJz60GaZW25pwyPmPU9TQBsxzzFfIl2GWQbfOfqB6Vj3URgnaIuH2/xDoadNcGULuPQYqDcDTuMdBbmQbiyqPRqb5ajk9KaWoDgnOaYBK0UcZ6k9h71Tf7qnuRV1YRKxBYLxnmo0h3OCQCo6c0AUtpP0owBWpPbGZThQB/dFUWt5EGTGwFAEltbyXU0cMQy7nAFWxZzWb3azQKfKXlnQ4/OjTJJLa8gnig854ySEPQ1ua/rj6hp/wBnktTbXBOXJIwy9xTJM3w1BB/a1rcBwx2lpAOiirF632oXd3sLoZQschblRU3hG1CG5JiUKwyGK8kc1d/s3QvN3mRB/slziuGvNKWp00tiJ0S91GC0uHfyUtw23PUjvVW11A6dDcKjPsaQrHxnArVubfRLtw008ZIGBh8cVYjm06ODyIprfy8Ebc1ze0ja1mb8qM7w0yy29xJuLOZPmJNQnxJdhmC6WWUHAJ3c/pW1YwwW9skVugWMeg6n1Jp017b2spjmuEicdQ5wazco87vG47o5nUPEl81m8Z09rbd/y05GK1dA1K5vbWR7mEps/wCWmMB6tTX+lXChJ7uCQA5AZuM1Hd6rp62U5W8iPyEKoPJOK0fLKPLGAnfe5ymp2/lXbyb1bzGLYHaopbvzFtwibWi+82MbjVQMx6nNSxAM23ua9iO1jhZ634X0/TtU8OjxFdQqbhSyqqjCxBeOAO/vXE69r7+IzaIjFGhUozOwXJJ6/pUuneKxpXhxtJEDCbDhWXOCWNXfC3gy01TRbjUb6RnfBWOFG2gMM8nn6cUgOp0Sztrvw1p729tA93gQySKgZlA7k56f415zqsfn+JL+Xb+7WUge1bulWFz4YnS5jngW8A2vbb8hwe5rBa7a7a/ldAMzMSV6ZJ4FBKZt+Er2zsNRvZr6UR2ht2XH97PYCuUbl3QKVCsduR2zx+lTmVTAUT72N4b0qSaVLu0UTrtkAxvU4H41IG94U0+SJBqsqSJIrn7KzDEUvr83tXW/8IsmqXrTrJG0cfzLCOTuxk5H1rnvDnj6+0nR4tBms7a4sOfLaUE7R14x71qWY1Cz0G91+389XaMsjAh0A6YOOc0AWU8O3EUU001nKjgEu2/I9hgdq6PSYbW501UCNhgAWVsDj2rC0jxtGdLj1G/tHKw4huJY39TgErjkda7OwttPbQVms5vtVpJ80cqcnNUNBH8sm2rPlMxyBWbe6immRg/2fqVz/wBe8ZauCk8cxap4404QRzLZKTH5cuUZX9SO+MdKpqwMu+LNbvPCfjA3FjChhuYELpIMq5Hp6V0H/CZWbWVtNIBDNMnmGBjhlFYfxD05v7KuNXuUeaaRI4bUJyEUcs3H4VxNtb3et2wmuU2ywzxRHK4JDDvUCse1212lzCskbHkZIIwRVhDnrVfy47cBE6AAZ9cDFPSQZ61aJLO6oyo3kkUgkHrTuGFAxrKrAg9DwRTPs8IkjkCL5kalUYcED0qQqRTCwHWmBxvjjws2sSDUIZ/JmtYMqckZxyK62xhuBpGni45mFsm8984qUosseGGQaUzS9jmpADGy9q8n8a6Ws8+pXsULtML3Ydp4ChR2r1wXTeVsdR+dUja2shcvbxsXOSSueaAOH8E6Fd6bCL2YyQm5QnyscEZ+XPP1/OuxJPetKVVlAJ6iqkkXPFMCJJSo60puM96X7I7dCKie3dTg4zTsApmJPFCyNuHNVikoOKcocMMgnmiwjlbpt/jDUW/6aD+ldnK3zYriAS/inUf+u+P1rsZX/emiwwyaUVEXGacp560CJc4p6viqGoX8Wm2T3U4cxp94qM4+tcdB46uG1IuI0a0c8R4GUHpmlcD0NG3ECvO/iHrdhqsVnZ2jrKbd2dpQeF6DFXdO8a2YSRb9WV5y7DnhAeg+led3cqKZRGMLk4HoM9KdwK7MHk8tTjNMbEUmTIGP16VXJIyQQKAkki4Rc496ixZcVwwydh+tVZnTdgdT+VKAUBVgcjggUI4TqAR70kgGAMRxQM9jVjzVeBjsxt6Pjoe1PmeKQQy7XLJHtO7vimBDFLsORz2pzSkjIz+AqByWbNICQcZAHv0osMs+aAwyc/Wno6JkjbknqKrr5alhwfftTCoDZFJoCzICG8wHimLICBng9xTRJuQqxppTHfinYC0kyBwxAYj1q0HSQbSBz6VlZ2mrKSAqAM59qloC35UeVyM4NGY43IIJB5GTVdS/XcCO4zUm4OoycEHpSsB1Frr1u+gpYySTRywISj7+Caq6jrE0/wBllliXZb8QlxncfU+uOOKwNyjpV+21bUrBI7+18krCwBLxhlye2DVoTQ7U9Yl1WZXkmMiJkoD2Jx/hVG2ilZ2USFs5bmn3l++o3TXMscMbt1Eabf0qv5jKMqcGgRoaPd/YdQhv9m7yjkDdjntyKv69rEvijxCl0bRxtRY/LibcQoPr+NYi/fdBymflb1xWjpzzWSLfRSA3BZoY4Eb5mJ9R+XFO4M9f0ONI/DVotrBNFCEwqTDDjnvVxQR1plrosugeHvtNxe/eRZJEnl+4cZIGf5e1XItOuZ7VLiNo5EcArhuxrYyYiEbBT+MVJDo145HmBUX2bJqydJeIfKWb61DKiZU8TSwSKo52kjHXNctbanE+q2ksyGORldTn+IY4NdsYXDlRG+7sMV5Re6skdzDJBF++ilcGKTqp3ZANIpo2fFt1Ot3BYQgpD5ZnlZh296ueELuO80JI14eFirj3POfx5rB8c6hOuqRBQod7MJIp5C7jzitjwPIk2n3McMHlRxsqjPUnGTmgVjownpRjFTeQVGS35VG8bZzjimMjLAUm80FD6U3FAh281Zj1CaOAwpjaTnJ6iqlSLtoAkluHZSW61WaVsVYKqwxSi1DCk0NFPzM9aaWq6LRN3zKfzpzWihCygcdqVwsZ241IkuDQ0ak9OfSo/JPPIFO4F5LsAYzT/PRu9ZRBXvSiUjvQBpBhmp49rYBrMWX3qzHLtOc1Iy9tCnAqZZgqgZrPaf0NMM5HeiwGgZhnrQZV9azPtBz1zS+f70WAtTRpJVOSyyp5FOFx708TZ70AUDZsSRiq81u0X3hgVscZomjjuLWaORcqY2/DincDnyuKbitDw5eR6t4fWWUK0kbGNsDHIP8AhirbWlu38FNMTRi44oxWz/Z0TdCV+lDaEChaK4y3oy//AF6dxWMejFXJtNuYeSm4f7PNV/s83/PJ/wAqdxjMUuKeIpB1jb8qNj/3G/KkIMYqReKZtfP3G/KpEhuJDhIJGPsKQD1YDrTw1NW1nP3lx7Gn/ZpB2pgSo/NWY3Hc1VFvLn7tSrDKOq/rUgaMc3H3qtxygjBOayo45epXH41YhlCsATz3oGaeO9G4CoPtSBcZFMNyvalysd0WWkXFQmUZqq1xnpTfMz1quVk8yJJF3tmsDXY2i1LSZG4G/BNbwkT1rD8TN8ljJ2SYZq4oTNd0bftFcnqyGHxJOrDHmRhv0rrJJcSkiuZ8SH/ic2snd4yD+dakG3Zgvp8DjpsFPIqtpbE6Yik/dO2rOaaJYYpKOaTL9lqrkWFApc1BKb1GBjSLZnBDVMomY4KKPxpMZZiJK9aeSoXrzUUasByRS7T3rF6s0WxE3WkqdoxilSBNu45NVdWFqQbEbqMmm+UoOeasmPA4AH403yWIyOazZSZAxI71EZG9asSwuF5FVWQ0ihpk54NN3mk2NnpSlcDmiwDd5o3U0mmlhTESBsGuQ8Txrea/BCSjqRgLuwR/nFdVurzDWTcweILiXzhvVsKQM4FAXNzWPD1vptkbmKV2IIGGFc6s0nmLsBHrmtGHxNcBcXapOw+6xGCBS3Gux3lu0clsgY/xr1AqWjRMpTzCRTvc7veqiTSRPuU9KYMA8uT9aSR1C1LBE7XZfl8Z9hW1Y+IbWCEwrasu4YLKec+tcv5ooEgzxRYZv6prEl5ttvPlMC45bqazJFCplHB/CqfmFjyaemXcAsFXuSaTQgJ9akifYwNaj6FBHb+e84lX1D4rOhtXuLhEt0zg5xmnYCw0m4ZwR9asJLiwGEfcOjY4pt6k0alZofKb0osL6SG0mt9oIYZBPaswK2Ayhnk3NSEN/C5FVDNlyRxmpUclhg0ATCeER7WVAw74qAT7WyueB61C67XOeuachT+MZFUkBs2EiuqydzWkDvRh2xWFDPE6qikoQemetbVsckA9CKpAZrSyWl35sUjRuOhFbdp4suGHl3Ijmz/EVwfzqJrKC5+R1YE9+lYl1DFbXJjV8474pgdCl1HdTFmXaD0Gc0zyRcwTW3vuWobCCWS1LwfPj7wHWn2xm+0Bo0OBwxPGKdwNXQtMSexeKaR9yghkzwKx7jwVcyXDNazII+wY11WiBY5LjJ++M1t29vCttk/hQSeaf8IhexyiOZoiD6NWLfaZcWty0QgfA6cV6zHZfaZTvRh6ZrKm0BZ7jZeO8an7m0/epDTPL5rWeBdzpjNQDPfrXouv6Tc2afJp+63H/LRORXETJGZTwU+opjuRW07W8okXqK0rrX5bu18iYIc9TjrVdNKnlg82LDr7VQCqeopMBhGKcI3KbgMinlARVq0W1e3ZX8xZB6GgCgwIGSCKTNTeXu+85x6EU1kCnigBIyMNk9elKBk4zg+tNKkDNLGmX+bp9aAN/SmiZ/LkcqT0IrX+wK06st6g9mrnbSza4mTbOkef73etO5sXtUVt6SZOPlPSqTJNmctAFCTq2fTtWHqM8iTA7wSfTtUqwSdO3fmrN/oyrYNcIzMV7EUCG3erLJpPlRvhv4h61zYWrCuhgwT83pSwxwyHEuR7g1ElcaKZVRIQ2cfSrQt7Dy97SOrVDdxxRyfunLD3queakZo29pZXCkrORjsTWjbhBiKM5IrAtRMbpBEu4+ldNZQSK6mVAjegpjGtGwOCKa0bntWr9iDfMGNO+y4pDMCWJgp4qqpKSc10E9tknisi9h2DIHSqRJnf2w8XEdvGidgOg/Gq9jZalrks0Vk8KhTkoGwKgS/f7P8AZhbqS38Ratfw9FNoHiK3e7KIkqE7i3GKsZk3mnyWrYWGTzomw56jPtXUul9qWkQSldqqMsqntWjqM+nvqUs0d5AUccqvQn1qk99Atq0cdym0e9JwM+Y5y78lpFRTj1NdTp15bW9kkRRXx3bqa425liaY+WelSW1/LbEkANn1rPmNCzqhb7Y2w4X0qCDeBlulNM/2iUySHaPzxV37Zbta+SkbN/tUm7gQvIpC8BjnvTZSSclQntULMuRsGMGmzTKhDux9qgYTOAvJqaGUPGQWrJvLgsyYyFB4o87J3L27dqLAaBAVjk59DSMqYyQKe20AEYKkcc00lADuXIpAQGJncbMYPcmmEPuwcH6UNICdq/c7Cn25BlCnjPerQFmGPalTLHFEpmblh+tOuInW2IQgNWc+YlwxJoGT3N6JR/q9uPSs5m3OWFLLJlSRUf3YR6mmgJADjnpU0nlu0exvlUcjFQIxaMD3qVYpB5m5CBjvTBoa3zTNgfJ2NTWkwjudj42MMHimxI3RwQPUipjHCr56+9S2K5QuVRXco/IPANJDvVQ4ByKmuVicgx8gdarMxBwDgUxEjXB5JUMx/iNNVmYEYJB7igR/LliBTgFDbV60yi5Jtfbh+g6GgKp6t+VMTG4Bgc0S/K5FKxJPGELY3ED1FSsIrRhtAOPXvWdlh0NKW3ABsk0WAtPGHYscIT3phVli+Y809yTEOAR70wyZHzciixaISR2pM4z0prSZY4AFMLFiRjp3p2EAyenJqRYznpT448KHx1qYbcdg3vQBCArSfPnGMdKNxQYFTrGhQljg4OOe9VSrNjoCBQBOk0ijOee1RzzMJW2n92cUyMTOWDEgL60jhtmfegBokYNuVip7YOKfI0ty5aR2diMc1Goqx5gyCibQBjrmgDYs9dns7EW/2RGKpsD7sYH0qjNdNdWqQGKNQhyGC8/nUW/jDH6iouT0qQK7QrHco7jcgYZFb8wtJ4YntuZWONo4GKx0O99prTtB5l3F7cVVgudtpcKfZEXbyK4nUzaXur35uJGVlJKkdyOldzav5FpLLj7iE5rgL64iuUSfjzDncQuM80WJuymLZJYFkVcc42g8CoZB5CMRC21iVBYVNE6eWAG59PapdX1FLp44rbcII1xyOp70cpV2ZSmp4Y2mlEafePSoYoZJ5hHGMk1uwSW+nxC3i2GX+OZ1yVPtVNWIM57J4nIlxv74OaInlXaTdSKoOTErYNSNMZJgx6scE0hgaO5Ew5WoKTHPOLm8LnczdyzEmp7FbRmcyTGN0fITs1ZcMhDSsO7VLAzLJIyIGdhgZ7U0xEtxcSvdSyOVwRj5V4A9qktZVdg03+rIIqN1zaxxt94HJrQsYonWC2ePOVLHmhgbGrFYvD2lCO3kRbVBvlYYDFs8Cu0+G9oy295a3UrO0gSUWwIKFP7w449/WqEvhu08Q+HLO+W4mtYlx5xY7htXrgcZPTFc94j8VpFqNn/wjiXWn/Y4PI82QYaRfp6cUooD0zxNpq6no5svJW3STMjSKgBGznAx61xotvEafDlbDTbWRo5LjzUltpQSE5JU915qxpPjKa9ijs7Sa3ndjtBv5NrljyeldTov9pWWszQ3cNulndDfEkT70DAcqDjqf6VfKNSPKNG8S+I9It7zSLd7gXdzLnbJyQSOcZ6HpWtonhew1Lwdf3rRzwavZyNueQ7cHqOPp/Ouj8b+Hra3tLrVS8k2p3Nyi2zKNpQk8AY69Pauj8P3dlrnh5muYBHcP+6vYsYPmDg5pNBe5oaZM2seELSK4BDSwKJAw5yMUg8M2xkvJGAP2q5S4cDplBgAVctlgtYEhiLbEUKu45OBVtZ0I60rCK0lpHIecg/WmCxQdHIq5kHpRgd6q4imbCT+BwTUiaZeYyDGPqasrIFqT7UR0NFwKUlleouTEGX1DVWMMrH5kINahvMpgnmqrz5PWi4Fb7M4HEmPrUbQzryCrD2NStMajM7DoaoCPEnemEuDyKlEgJ5pxXeMg0AMSRs5NWAqsu4VXGBTC+G64osBoK2xRikEUch+YVRE7H+KnCdx0NAFtrSLdxyKPsKll24HNVFuZAetWre7zMoY0AecQLu8U6j/ANfA/nXd+RA85D8VwmnSCXXbxh3uv/Zq7aV8TUAXEtYY2+VAfrUws7ZhloVJqmk7ZBJqcXJIpAMvPD9pqVq0AdomyCDu4zz1ryHWYZtF02awnsGiuHmJM7DaGAPBU/SvZYp2Dg1518VI/MubKTG3dFgN2JB6fqakRw4/s+KTDXEsm0Z5PBNZcs0l7cs6QpGp/hXoK0NTtrWyjjjilM08sauWIx5ZP8NZkT+UMUkyh7RQhNrqN3emRNt+UZA9qkeTzFxnJAqqJduE2jApDLrxxS8j72OoqM2qEckk+tQeftGRUpZgOT1FAFy3QyRS26D/AFmMDICgjOSazAzhmSQYdDhh6GrUbgSRuT9xg1F3Osks5UDZKBnAxzTAo7yDxSq2H3MM801gc0maoB3AYnsaCeKaMkjPSg9aADOcfWpCWKAe9MAxVuAoWUFdzHtQMr7TjJoUkdDirMkahWZVO3P5VW70rgWI5OME1I52lWHQiq6Rs4+UjNWWjlWMq4+mDSAdHiTrWlpk8tt5zLatcW4wsyjtz1rFhdkfkN+NakF/PCjqjYWQYYdiKEBd1WXw9JpFk1jG8OoMxaTDFj075rCD5P7w1oQw2ziWRyFlB/d5PaqNzbvBdBJFK71DLkYO09KBEkS75AIgWbGMAVfktrjSbyAzxqJlCyqA2QPrU2l6tDp1lJaG2SUysWMjnkHjH5YqLUp5b64a8kZSCAN3Qn6CmBra34hm1a3tY5CWhhT5oz0Zuea9f+HMinwpbXEK7VfIKMpwCOOPrwa8g8Kroqi8l1MwvKsRMEUvALAE8HueOldnaeP7keFLTTtF057i9Fp8zQJk25z345PFArHq8jnO4cUxp/LXJOK8/wBA17XvFJtLmQNZ6dagpMSfnuZABwemAOPz9q63n+8fzpXBIuXV2kdrJJIcKoySOuK8ql0m1uvET31rKBukJUSHhiB1/SvRrh0NrP5v3FjZj9BXl1vqNtNbXlzp++Z0mkW2AJyRtzwPaqTA5nXr6+vNRFvcTed9nOzzAuFIDdc11/gOJLa/1AlyWeJXBbpyTz+PauR1Asui2sDnZMVMrd8n0/WtXRNZEc9va3EsdvBAweRgPmkwOATSuB6ad38VHFVbe6a7sYpmQoZAWAI7dv0p2TVoRISM1H5YJzRk04GgBklueqYx6GoXVosbsc1aDNUdyDLGAOo6UxEQkxVhJyBgVTMbg0qllqRovGQsasxA7eazFkcNwanS5dTyakosS6fFJEzLlZMZ471mNBIOMVtw3EewEmmlI5JN+c0Ac7LBP/dP4VXaOZeSjflXSymMHgY+lQlh2p3JOe8yVesb/lTvtD9w35VtK4Vs0j+WTkqPyp3GZC3JPrS/aG9DV51jPG0Gm+TCRyopiKfnE04OxHerHkIDwKURL6UAQhjTw5FXIrPzeEGTVhdJZu6j68VIzMEhqT7UsdvOW7RMf0q8dFPdx+FR3WhhdMvJ9xwkEhI6/wAJoA4T4eaiDZalCTwJRIPoeP6V1wu0z1riPhOltc61e6fcbszwhkIOB8p5/mK9jg0WwgTb5Af3c5NJAc5FdRnHNXUcsOK1ZNHsC24QY9geKY1nFHwi4FUmIpKrk5zj6VYbDYLAVKIBSSKqrtxz60rgEkIRQQikEdxUIVHO0gKfQCnrIx+U5/CoZDtfP8VFwLEFtGJQSoP1rRRVHBVR9KxPte0/eqaPUR0JoAuT6ZHcSFw+Ce2Kpz6W0C7yvy+tW1ulKg7qes6twzZX0ouBiMVPSm5rSvYEuMBMIRzlRWU+Y3KnrVrUl6FhHC4zTpUinG4AIfUVUVmY4GSatQBCnzuQwNDVgTIBay7jsG4AZ4NRBq1BakxeYsg+lSQuVkBCID3IHWqU7CcTJB/D60bvcV0M1vbTwb5Ixv7HFUjZQD/lmPyp+0FyGXnnqKyvEg3aSG/uyqa6j7FB/dP51meJ7KJPDV06g7lKkZ+tHPEOQjXmGFvVAaw/FC7HsJz91WKn8cV1OnrBNpNpL5ZfdGOR9KwfGrRf2LCodN6Sg7c5Jq+eIcg/RiWt5V9Gq+VUAbnRf95sVzK6tPppzBEjiVATuPTgVmyTy6le+behX9FXgAVm6sUX7Jyeh182o2FowW4u40J9Dn+VVJfE1lESYVeYD+LG0fnXKC0VHdlAyxzgjNOEuw4dGX/d6VnPE9jaGG7mhd67qF3MGhfyUByAvf61Yj8Q6lEgE0MU/wDwHbWYs0HRHyR2IwakWYt0asPbyNfq8Dcj8VwBgtxY3CepUhhV+DxBpVwuVuTH/wBdUKj865bfIO9H3/vtR7eQfV6fY646xpuOL2Jsf3GzWNqOpROryWmoy7/4YwMCsVrSJzxx9KabKIH77/g1V7ZdSfq0S9Y2sWoxynUtTeKXI2hj29azJHu9J1ArbXjSohBVlbIP4VK8LSIUZlcfw5XkfjVZbWWI/LESPY5o9rEPq5uT+LNRRI5DDbFSMlc4qXRvFH9qagLW4txEz/cKn8/6ViFoTxPbKjHowQ0RxQmVTBcIkg6FTyKPaEewO/eLy2IqF4Gk+6K5qLxNqdo+2W6t7kf9NF5/OtK28ajYftFnEcfxQNz+RrX2lzOULFtrK4P8FR/2dd/88/1p6eMNIkcKzyxN/tpx+daUOr6dcYEN7AxPbeM0cxPKZX9nXvJVMEDOc15HfmZ9SuWmXa5fkZr305zwK868WeCTAtxq1jIzZO6SHb0+hp8wmjz8xs3QUKNtTQxyvgyN5MbfdZh1FT3mm+QqPHcJKGxzjFTcpFB/YjNSI8QT513t6VEAB97rSllXoOaQ0LvjD7vJH41AwAyfenmQnrUbMKYx6gMvGARRjPBxUBJHANALYwTQxEyqzjZkn2JqUNcWNwhBKN1HNVw+Oh5qSNLm8cBUZinf0pjNW/vJLoJJJ/CMCqccyAHnqMUk8qqvlM+WHtUKKDWQiEgh+lWkI2qcYpZIkKhwsme/yGgqAi4J/KgodIqOQ5kUA9RVeRUDYibcKm8mV1xGCaj8l0bkYYdeauBIqQNgMwrobFv3C1iSGRQB5bj3xxWxYxSpADIMelWBq27yCUCMBmPQGmS6RPJckX9juTsVOMUtu2y4ib/arpXG4ZHXsaqxJnWmnQWG9oAyhhgqTmnmGW1QzKqtFL8pBFSzsywkuuB3PatiwiF1YhU5IpWHc49NRaDUDGTgHpXXacskluGfOO1YXiXS44rMXOESWMgEYxkV0Og3H2jQopM5pEl1RiqupELEs2MsvSrfSqV6/nstsgySaBoWGRr+JTPyndOxpt1oel3UJR7OP2I6itC2hjht1XHzd6HxQM5fUdHFlaKtuA8XOV28j6GuMu7TTCjM0UtvcHoWHFescZzVW+0yw1FCtzbI2e/Q1JNzxprSXYXV0dB1OcVBCzJLhfvN0HrXU+INHj0qfYJB5T4wAPu1ziosVxuRt208NjFMssRadPcsC+Y89yvFLNp/2YkGVX+lWr/VPPtlEcmHxgrispp5MdaACS3ZBkDj0qEDBoaeT1pgfPPU0gL0TFlC1at41FygmkdIj1PWqFvcKsqhq3baexknRZ1GzvnmmmJo1FbTY1CpJI7dq6S2tTJYeWy5VhisMwW7YNnAjnscV0tk7fZ1VkZSPWncVjgZNFnbUGh8xY17ZGaiGjvHO0Ukyqw7iul1hfI1JX/zzVcJnWI2/vDNUgMqHSbcSqJ7nDdgwqlqukC2USxyrIregxXe39jZTW/78eX/ALaDmvM9UJS5aKGaWaIevNTyjJ9MmaxmaRrQSgjHzDpVy41pmulO0Jj0qha3zC3Cu/I7YqrI4eQtUMZ6DbjdbI3qM04iqeiTifT1w+7FX9lIZVmX5SAKxNRj+Rs10piJ6DNYmswSJEz7etNEnARt5dxGCpYKckAda2LvVYNSv7dJLXbGuFzjn8qzA8E8kkj7osnICnis5ptsxZGYEdMVQzsJde0jTW8m303evf8AyaxNTv7a/bfb2wgbnOO9ZLzNIctzTo2oDlNTSliZSsy7mPepry0kG94oX2DpxWXHNJC4KGnSX11K2XkIX0FSAgkwM5HNWUuFRcM2DVdJFGW2A4qDzHkLFgAo7ClYC21wD901WkLO2WYH0x2qIN2FSJE7ngcUcpRID5/3gBt4FRuUi+6pL96WQFRsHB71ER5ZBPemBZR2faWxwe1X5pTJBkIRxjiqEEe5snNILqRZGi3kDNKwD0BHUEVPFFulXJIANRLkuMkmr0Kg80gNAEMmKy9VdQwVRjHWtFfljZj91Rk1jOpuS75xzSiBWCliAOpNMm++F7LxViJGjl3PyFGeKqt8zE+prQksWmTOR6DNaUc3+ilihPPYVnWzeU7MBliMCnXF9LBIUjO31pNDuTtqcLfLKjA+y0vnWh/jNUmZJIjL1fuajR1DBjUWAtFbZpBy6oDnI71Xk2tKSmdueN3Wn7zJxgUxhtOKoBxbf17VatIrc5kkYhiOAO1VklRVI25cjvVm3VREDjmkIUwbpwfM2g9zSSLsfaSG9xTS5zgkcelIxRVDAkn3pFAQAabNjhU545pskzEDoKajFmppCRYtW4KsTjHHemSnA20mGTkZHvTQQ7cgZ74p2Aj6imbiGIFPYkdfwpFUF8mmMeJnSPaMUvmtKQzdRQYwxwDigWkuco3H0qQLBBdR71HIu1x6gVMVmSEAKSfUrjFDRoRyTuPWmBCZGJAyMe9IWXOM0xRGZMESEg+nFS+WBN842pjI96AAQkR7iKu/ZENsPL+ZvSql07LBEqnq1WZWiWCIsckNyKAKfmYZgPvdDQWKpx1pZI3L7hGMMeMUEYJWRSrDtSsAkQyCR1rY0SNpLvDHO3npWTF0Nb/h2PcfM7nJNMg6ed449EuDI+xChBbGccV5pA6xuY5433k5z7V3+uyeToe3H35FH4VxmsSxPfRNEfuoAaAKl1C8bB40PlN9wjuKoknJzXQNfwnTDApUHGAoHIrBcZXn73eqTKTLFhM8M+UAJI71qwsi3UbXDspxzu5xWMuUwQetWBxzkk0mxMvQRwNeqM4jYnB9KW5QI8xgk8xEGemOfSqXmhOtXTG0CYkBCSjjaM5PpSYIzpGaUqrKiAc4BogDCTI4wetQbgGx2q1AVHJ5BHShDJZWWSQ7a29AijkuS1yT5UUWSRx61hxR5NakPyQMp+6wwaAO90PXvtFktrbW/wC4tiRHCxwGfOdzHH3R6V57qt9c3uoaiJ5BLcTS4IgUFMDsPQVpWt8LS1lAEgkddo29QfrVrQbS2s7wJNKm+SM5YL29zTUgOZ0C3srnXrODUGdLSV9jshwQT05+tev2mnnRbQwee80VvN56SO2SB6V5fJaQWd9BcwRTSWIuEdpHGCxzyK9nmt4wVOPvDNaLUTKkuv29ykRNhPcneGiAToR0b2q9FcwGRxDGEcnMojXgP3ye5rmr/SZbOLUrjT3LTTQkJGf4foc1zdr4m1fwtZRabLY7XTJIlGQfenYg9OMzjvUGp6xHo+lvfTvwvCr3Y+lZOhatf38DjU7VLadcbMDaJM+lUPHup2sfh99PEoN2zrhB1UAjP6VLKR0+ieJYtZsFnhH7wffjzkitJ9S44Rh9a8U0jUG0LXLW5J3hVHmKnXaa9Y/tSzfTm1BZVe0VC7MPSpuMtnVYkmjilnjSWQZVC3JFP+3oerE/hXies6g+qeJJr+Bn8rfmJ36gfSu+8Gao+s6ddjUAjywOqhgOMEdP0qhHXi8Q9HFDXUeOtZ/kWvYEfQ0hgg7An6mnYksPcJ/eFQNcc/eFVmgUdGP50ww+kjVVgLYufcVMt1hcZrM8puz/AJ04RSd3BosBfNzz1pDKGrKneSBlZ22QjJZsZ/Kp7a5W4hEqBwpOBuGM0WAsz3P2eIv5byY7IMmm2N3cTRA3FuIySeQ2ePSq96ZGt2MTEOoyMHrVK11O0jnNp5ubggFhjq3pWbQzos0sZw4PpXDQ3s6XX2i0LiVJGRklfIHPTFdGmpmC3d7p18xVO8oOKBnLaDzezN63X9c12krlnzXDaDLtge5xnEpOK7BptyK2MZHTINNEkwlYd6cLkqck1SM3vTfNFWBsrexEctXMeMtTNpZw3RtUuYR8hDjIUk1bF1Ebr7Nn96F3ke1Z2rwW2rrJYyXXleV8zLkbXH9cf1pcoHmGpz/ar6a92CNJDnavQfSqDua3pore4tdiKsagkFiewPFc/JwayNAWQg5ppOZdzfiBTQGY/KOnWn7SKYFnMDY3bgBTv3WCAzHPrVTJzUm07cik0A/ftXH1qNmbJyeD7UhJUg570jMGOAOKdgA8008UuMVp6HZ2d7fNFdRmRxGzRIGK7iOoyPagDMz6UV1cOnafcKWi0GZ1BwSsznmmP4ftOraXqq/7r8fyo5i/ZM5jmjcy/dOD610Z0XTR96LU4vqAf6Ui6Bps0qRx3t+JXOFDxDGanmD2UjBVyy4YmnCPJzuFJPCba7mgLrJ5blNy9Dg0YGAfUU0Qx8RMbHB6irKT5OCeKpk4pCzD7tDQGhlX3Ln5hzinNwKro2xiygc8kVKXBAPqKgYqsM/MAR6GknE0my4kyyudquzZJA7fSkUFmwKkFvK0akY8v+EZ6VQDrWf7NdJPsD7c/K3Q8YpI0nu5VihjeWVjhURc1PaWuZQsqk4baQBnA9au6VdXPhrxAJlbchOdx5DD+lFwJPD/AIZu9du3t3lFqVUsfMjOQBjtx610ui+BNWsNaKvdkWJmVJTExQyL15BHTt+dOHjpbmYz+Xbm4U7IItpBGeMk1p2/jLULe1nbUdPWSWCPzFeE7U+hFFyDvjMvTCqB0UDAAoWRW4zXmeieLvEmrStOIIZbZDteBExnIzwT0+tegQ3AaCOaRfI3DJSRhkUDDW9T/snSJrxIlm8sZMZPUd68u8P3VpYXt3fqHWWSSQpAq7kVcHj9f0rrvGer6cNFMZuJVlkU7PLwQw9/biuMkaLT9Minmm84TD/ll0TA/wD1U0Aq2q3wnjYKx4eVy23aSRgc9MAVHodo93cXVjb28bCNzJLOrA7h2UfjWLaST314losuyFyZHXPBxXRW3iEaciRpZbGV9paBQokPOAR16mgDttO1OGVPsrzbriIAOAOBx2rQXa65BzXIaRpVzoWhX2pXse65nUOPn5CnufTrW34allu9J+2yqA0zsRjpgcf0NJMDUxzTtnFIxxXNeL/Fd74a+wrbWKStMWDNICRkY4GO/NUmB0JMi9qeo3KDWPZ+K9Ln0iK+vJFtppBlrccsD9MVSfxpa3Eoh0623ue8xwMew71QjqNuF6VCLZ27VzS+Kb8S5aKBo/7mCP1zU6+LZ/8AnyjH/AzQCOgW2ZGBNS+Wp6iuUfxRqTyqyJCiDqhXOfxpW8ciyuES/wBOyjjIeA8j8DUtDudG4KNjtVe8vBYWkl1I2Io8bj6AnH9RS2HiXw5qZCrerHKf+Wc3yH9a4Txn4zLNqeiWtiiwhvLFx5pJYA5JA6dqVhnfifeisG3BgCCOhBpDJWTo87SaDpzt977Og/SrvmZqrEEjSUwyH1p6IrEZOB61OtvE4BxkEZBpDKe9vWgO1aAsIfQinrYxBvWi4zJM7C9Ntj5hGHJzUyljViYWFv4itYnZTc3EJWNc84HP9DV9oIj1Wi4FOOR0AIODUq3EhbJbJq0LaED7n6077NAf4P1qbgRrdsTgmpLm6ih0jUJZGwi2zkn/AICakjtLQfeR1HqpqPxDHaWvhDVnJ+UWr5P4cfrimgPI/hXI0fj60C/xxSKfpjP9K94LKDya+ePh/cPb+O9FdDjfIY29wVNe+3TlXJBoZRYlkjVfvVVkbcCarm5IGTyarvcu5znHtQkJsvwHMy7jxmp7pI8/KKzYZvLYHOcGrBm82gkIyu8Z4FT3FpFLAPLx5h5yaqgVb81TCBwCB2pFGNcWUsL4k249Qc1Abdv4au3iMzBkYn1zVdHZHBPY1SJEitbpj8q4H+0cVYEF+iZaPCjvkVM13uTPSnRX77NmAR3oAqxTb2wzcVb+y20g3MvJrIEga4YqAoz0FXEdtnWm0BZSOG1l3Qrg+tZs6zQF5CuEZic1fjbcctU1za+fbjbU3FYx0upPWrME7mUc04ac3bA+tR3if2Xp895IUbYvyj3p3BGFrvi3ULfUDa6dIirGBvXaCSe/WqcfifWyocXEMgP/AEyWuLnnN1etOxKuXJBU4Iqy+oS7So2pnrjuazkdEUdefF2tJ1a2/wCBRiquoeJtUvrCW2meDZIMHYmDiuQOobTy4P0Oaf8A2wzMiHcUJxjaKkq0ex0MWo6pLYJbJfMkSjAC8ED61nXdmIrdnb5n/vbsmqN9etasqLnDDPBxVNdSkd9hJKnrk5ovILROk3b7S3Y/88wKiWSOGVS7Yz0qKSX/AIkYk/uDFc+l9cOobykH4miYoSOzCKw3DGD05FMOFGQFP4iuROoXA/ghA+lM+33B6eSf+AVHKa851hIbqFqB4S2PKm2AfwluPyrnf7TvP+mH/fFIdSvWP3ofxXNHKLnOj826j6yRN/uvz/KnLdK7bZJQPqa5wXd6R1g/79//AF6a+oTxKS0cLH/dxRyj5zq90WPvg/Q0oC4+XNcnBqVzNuKW8I29fmIq9De3jrlFi/4DIaXKLnN8Bv4UY/hR++7DFc7/AG1eRuUaLDD/AGzTv7euB95ZB9HzU8o+c3d8r/L1o+xM/LLWB/wkb/3n/wC+v/rUf8JG/wDef/vr/wCtV8ouc3Dp6DsV+ho+xlfuSnPqRnFYX/CSP/ef/vulHiKU/d3n/gf/ANajlYc5rTWcjqcxhj6jiq/keT9+FxVT+3rgdY3p0euzO4XEoz70cxJpwTmMMYLiaBvXeRmrcmpaqtg6x6k0yMMMrjdkVntJLMuGOc+tCRSIPu5+lHMP3exl6NKpEtvLGsmDkA0680+6uIfLQbFzkcZq7a6Ssdy1wqsu7Py4wBWgIyDyQPqav2gezicZLpN4GwNjfjj+dVpNPvoiPNtpAp7gbv5V3wAA/wCWZ+ppDGh7Rj6MRR7Qn2cThHsJFgMynKjqMYIqi2K9DNnGc5k+U9Ru4P6VQuPD9lPnDQr+lUp3IcDiNwpcgnFdUfC9r1eQhR3Rwf51Pp3hq0W6eaO4MhVfuOMA1XNclxORHy12OgtbGyzAjLI2N4Pf0rJ1BIhqJKWSJ5ZIOf4qW2neC8WSPag7oW4x3qkxWN64t7dAZVtYQw/2aWyuEdSvkRLjuBTxLHdwMY2DDGeDmqFi5W7KeoqWhWNS5uRBbMxVMfSuOum3yNJ61ua9dLDEISPvDOa5x5VZcCsxjlbitrwzpy6rduskgG3opHWsaykRLyEyjKBuRXttrb28ljFNb28a7lHIXBrWBJjQ6KkShRErfWq2vaTLFp/mrGFC9cV0/kv6VDd2RnsZlI7ZqyTzlX5R/Q5r0GyhgmtY5MZyorzhfvsvocV3fhoyTaWO+04ptiNmFYYz/q1x7imNCIJTPYAROTll/hal2sO1IWIpCOd8YxrqGl+a48u5h+8q9GHv+VSeCpo5NGMBbDKSa09RsI9SsZImZkkxwy9a5LwpMdNvri2n4Pr+dAzp9Q1A2jEGnWKrKFuSOWBwa5PU3lv9dZCcxKQRW5b/AG2FljX7npQM3C/NMLj1pfJl25ZSKYYmz0oEG8UocGmGNx2oCMOop6CMbxLoEWrWjOmVlT5gQeteY+TLbuVeM5BxgV7UAelcL4ltRp1280g+SRsA+tJlJnKtcI6FWQD6VIbCKazMokYY7CteKx0y4sjcqQWAywPHPpWJ5bOpRCwX0z0oGZu3LYFXpNNmS388bGQf3etPi0+UPuAzWoDI9q0W1U9xSGYEEiRPkoH+tXYLiNp0byEG05rMlTypimc4qxaPH56qzYzQI73TvEWnJsgaAxt6jmukininXfC25a4a0+x20qyOqkdya7iyuLa5tg1v5Yx1CmmBi+Jx5axSfSqbHM8Eo6EYrc1y0+16c6gZYdK4Wz1SQ3UdvKej4NUmB6VHHHLafvUDL3BrkNWuNKh8yO2gMchGDgV0k2oQWlmDI4KnsDXn9xBLcXsjQuojc8ZqQMGQASMVNMXcDzV7UNPuLPaxUMGOCQelQCEk88D1qWhli1vp7M7oOtWf+EjvipUqRnuDVEGJP4wfpUkLwBs96zkBYfU9S6rcuKjN/eTrtnunI9KfmE9WwKbLHDszG4Yjrip5hnM8xKDuBLDoO1VmHOadmnD5gRWoxi471IjooxmoiAjcmmZyaBFvenY0x5PlPBP0qIZyKnRRwT1oARVOOabg98AU6RivQHHrUWdx5pgSIoLbQc+9X4ise3OQPUVQJEScDmpoFkeL75C96Qx87+bN8o+Ud/Wq0rbOgJPapchWKjtUxKtCFAAoAfalVhDycH0qpGIpJnYDLA9akkV5QkQbAzilS3Fudh6igCaNdzitKFBjAFUoQOMVqxRqFA9amQEN1j7MVJIBrIAEO4RcjoRmtS8IMgUHgDpVBsH5aSAhEjRIPlJJPSmTFH2grt4zU8sYwp3qAMYBpSqqu/5XancZXt2Q3CRqKbd2ztdN0A96twQh7tWKgY9KmvoT5uT8v+9TuIp2+loy7nuFHsKlubdLWEp5YPuab/y6kgSY7FehqsiOy7vPk+hOaQDxAdgYMOecVE2QcGg3I6O2RUghiYhske1NiFt4y7OzdAOKVZMdDUgIUHb6VAFO7AoCwucZoL5XbTjtK9PmoigeQkBcilcB+yCSP5CRIOT71Gh28d6dt8s5VsH2NRk8k96Yi45U22A65HaoECrn5s5qHJNPgkESuWG4se9AxnR8kU7rQX3Y4FShVZlxwDwaLjYKuUOOvb6VM0kqWoljchgcfhQ0apjaSATwKn2ItpsOMk8ZNAFWK4acEytIQOV3U8AMQevFI8EjdCAvdqsxOsKMCQ+RgAjFJgOEjSRGPYflH3gKqyBWjw3agzzIpHIUjoKFYeXk0kBCwEpUA4CjHNPVUbKszEgjJJqLePMBKk81IZBIGG0cmgCeaTDIqkYx0HaoJpfMky3JxjNR4wc0xV3SqpbaCeuOlUA/iut8PxeXbL69K5iS32bfmzk44612Wkw7YYxQJlPxpIy2dlCp++5bHrgVyUcSSA7nCze5rb8az7tVgh3Z8qLOPTNc0hPmBqCS1bpE3EilmQ5YjsKHWLDndgjnpxQSVUkMVJwDjvTECj5T0oKGpE8rYQZI5qdEL4A65xikidY5dynBwRTI8+YTuYNjgg4waALktm5ii3ffAw3FS3l3F9iSFWzgZFS2t1PtilU5baQwA5NY1zsBTZ7/AM6BEA+YnmrSjaq47iquxgxXHIGTVpDlVx24NUMvW4zirMkg3KEcNtGCPSmW6jZn2zUEg/fs0bZL5OcVmUaca7oQ/RqTT77abpJohKu4BmJ7Z6VSNzKluoYDnuafp0sKvFHIg27gZHbpntQhMtaxd3N4vmxwPHbIMJt6A+pNeq+F3u9S8PWk88mJSgDo559q8uOoJpd4NP1C2eWOKQShUfG8dQfTFepWvibT7+KOfTXtWkcZe2eXy5M+gz+Na8xmy3cxSRPGV7nB+lU5vD9hqcwe7i3zKpRHz90H2rQu5g72atG0DSuRiQ85x7UjXCWl1MZM7UTfgDJPtVIDPu7xdMs5n1GBNlsoCEH7x7YrzorDr19cahqV35LKu4DGQRUXiGHVLHUUg1XzBNcKZwGkJKgk9vwrGlkXaUJ+U1DYIlmcXEzyJJsXGfmHWpH1a+fShpqTs1vu3eWOAT71l5Xfgip0ZcfLSuMuzrJbxW4EispxgDtXY6Nf2Ok3c8huQEkRd6xnIJ57Vw6SukflOVCEAgEZojlUTADlT1HqaaYHscN5bXLokNxG7upYKrZJFTc15FbXhstRt7hDgq4P4ZrvPFviuK2sre30cxu10u5pefkXjoex61VyGdBzRzUHg2WTU/DFtLOvzIWTfnlsHqa2mskPaquMywrk8GhknUZxxWoNMfGUIoNjMP7p/GlcLGHdAyWsiSjIIPFV7a5it7A7XaTYCTn19K6MafP3hQ/U1lXmnTWX2idolWIjKgEYJ9DRcLFUu9/YSEB4Ny/KwIJx61ykF2olWctEZUOA3fqP8K6z7Db3Vg1zHcOJVXdlW4U4zj6VxU9h5BhlSYmHIBmI4YjuBWNUaNC3vbaPWBdT8RFt7IgyM/Sr+oXtldRmf+0GcqCyosRBBx3rnpm87VT5kirGwGHbpip/NaFZRbSRyDaQWPQjB4rnjORZY0bcmgyEHGWNXLKeK0hWbzJX8yXYrydMfSqVn+78Lyn1JH61Pa3kKGCWeLzijZVC2B7n8OK1lPlA6m6itrWKBt5XzCXdj1+gqaBbV8SKs0bDtKP6Vyurao2q31ssS7Io25DNg4retNRW/eXCbUhAG7OQ1a06ib0JaOclW/0i9Mk0AmcuS8qnKtG3Qe3OK5DUtQnilntWgSFwSrFPQ+9d3qXiHy7aOaxWOdQ375HByB7VwuqSwX2oSTxQskLtu2seaqUhWMgBplbOdq9agkPBI6GnyyGJyqniqzHcPfNQixY2wpyaQvzkio80E1YFhSuQTUpbA+XpVYEYo3H1pNAPeRW5P5UxWx0pjHPNJRYCUuafBcSW1xFcRHEkTblNQgU/acUAejeH7+OHUlKt/ol8vmIvcP3z+Namno97KdSu3cuzOiRA/IE5HT3FcDoczSWU1jvxLGfPgOOQR1ArtdNk1DU4EvbK7ixIu17eZflRhwee1ZJGrd1cnsb5NI1C/spJJPskSLPH5hJKZ4Kj9K5rV9WktopNSfi8vMpBGeCid2/pVy/n827eO6uPNhgPmXUwXGcfwAdhXF6leyahevcyDbnhEB4VR0Ap2KcnGOnUqgnHr9alLDAx6VD3pc8Y71oYCliaekjgYU0zqKkjQgcGgCzGDsGetOLYqRYzwQQaeI0YHI+YdqzGNScE7cBWAwD6in+Y6MCGzTHCsMMvI9sUIMgCmwNKOfcp+X8qW6fz7eWCQK6kZjkfqv0qoQyKAvc4NJNIyhELZAOQRWbiSbCQWEtuJo0MRVfLAVuMjofrWcZ2VBDI/wA5JDjHp0pjMsNvgkj6GqylXUEHLUoxA0rG9GmMWimmViMMI22g+lKviHVBB9nW6cxZJwxycnryaymfmo91apAdNdXa3eiKIw5AYlwx/iANQX99AljZtaohTJ3o4yRjvWMs/lwMAzDJyQDwTUbupbaq49eeppoC4tz5V0Z7c7Gzkd8U+C6vLifmYb94kDnggjpVANipYX/er9c0wOluvE+pXsJs7q8VjIQrllxtUV2+ma1peiaNDYQNNeGLOG2hQc89a8vEQYtMQC3fmrn9vajn/jwh/wC+jQB2994x1SVWWysbaHPR3bcR/T9Kxft+oz+fcapOJdjbkz0UD0HbtWF/b1+R/wAg+E/8CNMm1i/mt5ojp0SKyEFhJ0HrQB0qvG6K+xCGGQcdqQtGP+WSflXPRa3fxwIgsY5QqhQxcjgVF/wkV753ljTEd/7qSbjVNgdDcXsNpA00wkEa9SFzVQeKdFx99wf+uZFVbia6uLNkuxb2cT8E8sx9sVTVbK1Zzb2guZm/5aTcDP07VNx2Nf8A4SjSCflmc/8AAKyoioYxwSNOznKqH3YH9KRZLQy+Y9u9pcHrJEu5Mf7tTmK8PMJiuUP8UOM/lTuFhJbURpuv5rW2QdQSHkP4Vk6jc2L24W0jckH/AFztyR6Yq0EhkJGE3DqGxkVR1OJYokICjcccCgDZ03xAtlDEiTz2rYxlTvRyO5Hb/wCvXXQeKXMTObOO8x/FaSjJ+qnpXn+mRpJZyK38LelXYYkt28yH5H9VGKaYmjrh48sfmAsLkMFO5WIBBrtNPG+ws5h914EYflXkEgMzFnbLHvgCrVpqup2CiO11GdI16Rlsr+VArHsq4Pen7OMjg+teaW3jzUYUP2m0glf+8uV/SjTvHd/Fe3DzKkqy/cjY4CY9DUSAd4xuJ7H4gaRIJ0WSONcM68LliOfbFdub4WXlLqE9qGl/1ZjjbLfhzXkniDULzWdUM90UNwi7FCdAPT9a7TWfE7TPp/8AZXllDgFpFDbWGOD6fWlzgd+U/dE1GTtTNeXyeLdT0q981DFJ5oBZQ5ZR9PSu30vW4tc0wXcSGNicOrHJzRF3GavnCsvxdfLb+C9SfaJA8exlboQTipjKRXPeNrkDwZfqT97YB9d1WB5l4Xu00/xRpNzLxFFcqWPoOle+3U8hbcDwelfN393/AHhXvguFNvDl8/u0/wDQRSYFgzkdTSLMD3qo7qw61HuI6GqEaQm9TVuCUMuM81i7nqxBIy0Aa275utPxx98Vnee1Hn+tKwF49cVXmT5lPbPNNSU5pZJOMnNCQCzKiRAo2eaW15kxTPKaROKdHbOvzhxx2pgVryT/AE5wABj0FSW8hcYNWnjtI7SSSSDfMTkCsS+vf7M0/wC0MCHb5UU8ZNDJLF/4i03SbgQ3DSs2Mny0zt+tJH490ZUA23Z/7Y//AF681udTZb13UqyMckMM7j71Jb39jcYKOsEw6xMcAj0FZyN4xO9v/H8WwiwsXdx/FLwM/QVyeq61q+rROtxK3ldRGq4ANUBrFqXVBA+WOB8wpsurQIzK9uVYHBBcVnzGvLAy3SVSeMVWc5kw4I9DurQm1GCQE+Xx7EVneakkgAPJPeqQ3oV85iZ8Y2kdqtshFzFgAg4OaZgCKeM9avRjMMZ/2aNibj9RiWVInPOFra021tINMWUwR7gCSxXNZbrutue1bFiPN0t09VP8jVGZGmpQrOIpFRFb7jAZDVX1e2hCpMsaq+cEisrTHltRb3F6Ue35jQn/AJZntmt7VF3acx9CDVBE5z7IRIWJJU849KnFnGVyUb8qm6pVyNcxJznisUXM5+7twipsHcg1UCbZFJz19a27+MbCAwO1s5rLkUYBBBwask1EQcEgCs7U4wt0AD94ZxWrFyik9xVfUlB8tj0H51mn7xqZ2nvFFLKssmzcvHGa07S8s4WJluUb02riqwtx/dFSx2656Cq5ieUfc+XPeGSDmJgOaDboy4IqcKBSniouOxmnSy/QEVDJpDp0YD610luoZc0+WFW+8uK3MzjjaSKfm21La2+JsMfyrRkiAlIPODUIASYECgUXqSy27CPMZJapIo8RAMOasgDFLisJM2IJnuYovMilIAODkZxUC3t+/wDy8/8AjtaRhEtnIPQ1Tit1jBGc5pXEND6k4yJxj3IpNuon/lsuceopZo1yDiiyRRdg9iKdiLjMaj/z1X/voUjtfpjMw/Aitc26E/cFVbqEKc7QPpVWCMtTNM15/wA96b9ou+09WjGrAgiozCiJwKSKaN3StKF5DFLNHIkvO4yNhTVS61C9iuGtdLtgxzgEfNn6cV6FoC3F/oFsdiBFXbuY1pQaWqOGVI93qAKuKM2zyuXRPE97GPtFjj2xtNbUXw4gmiRpNSkicj5kZBlT6V6NNYSSxE7lXHrWRIWRthlTI7bhWyMmcve6XBotuyPaYX+GWMcE+9cyG26juH3W6V6eEcqVkQOp7Hoa8s8UWsunarNCvyoQCo9qXKLmMnX7j7RqBCtlEGBWYhI6mhyWfJpyxqZwD0qXEpSuOJ4r1P4b6wLjTXsZn5jOU9h/kV529tAsIdSD9DV3whrH9j66jMP3UuFbnpz/APXqY6F8p7S0iA8VJGwYFQevFV2VWwykFSMg0+LaDzVkHmWsQtba9Ojdzmug8GzBbi4iY/e6frVLxnbrFqqXI6yAA1B4auPL1uAbsBzitESd86nOMVBICDzWw+nk5YSqRVZ7VE++QfpUqQuUoRferhvGNqbHWo5o/lWbkn34r0MQLu+QVyPxDspBZW92eittx/n6U7k2KrWiwvbTA5Z+W4rpVAEq1jw27XNhZOOwArb8hywOKLlMtm4QrtzShQRkVQdVgkDzSJGv+0cUxvE2iWgKy6jCGHbJoGaRXFRPgVkS+OfD20lbp3x2WMmoV8b6DJ8heVc/xMhAoGbG4ZrnPGln9r0guCA6HIyevStGPXtHmYLHqERJ9cioNcgjv9MdYpoXx0+ekSeY2s3loQyjacg1VaV1kYhiCDUk37tig42nFUnZtx5/OmWjRXVblEx5hqFtVuD0O36VWhjaZ9vP4GtBNF39GX8ZBUFGZI4Z8mkC85Fa0vh+RU3LIgx/eas54WgfazI3+6c0AAXP8T/nW3o+qyaWBg7x2DHpWIrYpxkzQB6zomsQanAMArIOoNcLr1ktrrjbMBGORg5rCFxMPuyOv+6cVNBJLO6mWV3b1Zs0JiaNeYOIwVYsvcE1UM9uv3Jykv8AdJpJEuIkLJOhX0rLl5myQoz6Ci4rFme/kkXazlvrVVrl+mavjTkkh3+YF+tY8hxIRRcLE29Ty3WnCZVORUA5OCDipPJXsMUrAWDcF1wB+NEc5j6DNVxlaNx71NkBkAHGTSqTTmK9BmmgZ6VYxSAG5xmlHJ4AqPGZMHpUowGwKAEdSWBHappDiLdTDwaSVgyhR0oASGUsNrkkHsR1odRGT0HoBUMeC3HaiRixyaYyWI7nAYAipmVwOOlVUyORU6TuuNzZxSAdHvZTlCKlij3qw3AY55oS6jOVKnkYqWO3EpGHC5pMCNU2/NkEj0pyjc2fWnttiV02lvcUsKkRK20nJ7UAT2qnzRxWkG2jcegqGGFVQNjmppDstmOASexqWwMqV98ruucE96jBQoWLcjtUYuWdmG0AqajxtyfU0IAZVaTcwB+tW4xH5YLVSzkZwaQ75FKgEqO1AGjAji4SRV43DaasXky3V15bHcBSWk0HlrHHE4Zf43PFULRootQ/0g/uWb5qkZainNo4z8sJONg6Cpb7Took823YPERkYpdWtIIZ1NvLvRhkDPSs/Dj7px7dqAIGh28+WBSD5amlkcqFbH4U2OJpQ5GPl9au4ERlwMYp8TJwT1piorOQ+fbFTRRxoxJBJPSm2IYcFtwBA96ersFbDEcdqdKnyLtHJPNRxxq8wR/u96kBhiyAY2JPoakit5Zt21D8vBzU5UJkIhx2NSgSQkjjJGTmgCr9imPZR9TTJbWSIAkowPXac4rQSUCIvPjFVjNskKox29qaAohSDUybh7CrvmKU2kZFQvCyAYywIouAD5XjkX5iO1TyN54IYDB7CqaTBBtYHPpUsUrk7hwvvU3AXc7JjkL6U5FwRyefWkdiOmfm9aN7KQQB+NMklXKE4AOR3qqffGM9qlLFuajGC23IB96aGQvg/dB/Gnxs6QnGAWbGSKkkyFGMHHpUZboDQMVRG5YNnAHanMsZwycE9c0zOOwpKALNvl7qJPVhXc6bFjZXF6bE0t/HjtXdWAwit6DNWQcF4iZrrxHdSbSyjCDHYAVVgSOBSCrH/ep8snnahcSYxvkJxUEsnOB1pFjGyWYk5HUCk3bue9RMXYk0R5JoAnTG4Z+73qYqpG9OMjAqueMe9TQyFlIwSF54HagRdUmGNGU4x2psjC8kUBQoUYAArOEjGfJPFWkL/wDLOTn0xUgVXEgDOeXckcVJboRjPWr0KmLeGZFkI6HvUEK/MaaYFxTttWf0FQRzvMFiyi8H5u9Pv90OmHafvMBVK2OCrmpKNGUK1kFUgsgwMjrV7RtLiv7uzs5PlMs672xkgDmmWpt5ohtGXfruGBx6VsadHDbOt79p2TQt8saNhj680IhmprwuLXX7q8NtawwxRLFA8w3EgdAB2zXBXMtwpaOZdjiQsRtAIb617QdHXUNOtxPJ5rrKs5fqCB2P51w/j/QpBdm9tIv9HhjCv3JbJJ/DmtLEDF1+51Hwi91d3aNqFrcRC0X+JAOM++RXT+H/ABfa6rqLPeoIbtIOYgchgvOentXkavLb3SRNuicjJBXnHapluWtZJpIZmV5F8tmHXFN6AolvXdbu/EOuT6jOwZAdse0YCqOlUrcZkzsD47GoLYqhCMcIeppzyGNiIzg+tSyrD94jMrIoBfg+1Nz825QSPeo0ORVpGVYPm6nikUW4oZLq1aQwOFCEE9xVFDtbNS/bJVi8tJWCnrjvUSsBTQi0WUoAepHHvTUkdfmQkdiDVdJGDjnmpChGWLZyT07UhG7pGuX9pbvaRXnlW0pG5W6Lz1/WvUv7bsdF0i0F3dNIxj+VgM7h9a8ZhcxBHU4KnNbENpd3toLuOIfZn6Zf+XtQI6y9GtXkt1f2d7dIjMGiVTgEEV13hmSeTQYBeyf6YCQQ5+dh6kV5rFqeof2cbXz5ZMjCkDgD611WiWy6Hp39rX8zrM6gFpvvHPamB0qf2lDEsbGLdGxYymQ5wTnpXG6pqFjI8Vu+qSXMKEmRME7m46frVweMre8tr+C4ZQh+WKQLtyDnqK4ZhDB+7hd5GH8Q6VnKZZ0jaybWwmtdLUIX6sy4bFWNK0ePXtHlt0umW6j2kyOCyg+gFcdFLEl0J5lMhAIwWrei8RW8VrF9jhmt5QMFVkG0+uRWXOKxn6haS22oyWtxIv8Ao2Vcq3DelSXX2dbd0tkdSEyWY5zmojf2YeUrbSiWU5Ybg2aZNqcCxPbwWcivIpUyyngD2FKEfeNDY06H7ToIh3qm4E7m6DFUrJ4m8xJlbrglOQuPSrdqsY0KIS/cZgp/E0tzpd3YyymyaPyWjClnAGR7VpVIIPtcENwZmaaUjg54yKk17VEj8NSRwJszMoiVThiuM5PrUdrZgmU30flbioD7Tj/PSnappMd7dRvEvmw7cLIjYGKiDsJnHtHKrsfMIz947sc05pFcYyBxTLqSBZJEQsY1JAJPXFRgSqAfKKhhkbhjIq9wIbiKIguW5+lUWABrQmOICpUZPU1msea0QDWA65plONNzzWiKJBRSCnGmIj706lAyaCKAFTFPzjpUa+lPXBoGT2s8lrcxXEZw8bbh/hXXw6hc2LzLZcx36hoiOMMc5wfbmuLxW7pd7LB4dvZFwXtj+5ZhkqW6/wA6iaHCfKQ67dqoXSoG3RRNumfs8n/1qxz/AEo/wz+NHaiw3LmEpaSjNUQOqSJsNnGaiFSxqCp5xSAuRykAe1TrMrDIABz19azw/wAuT1p8BZmPPAFQ0Mtvlhzn6imxMU+Vuvv2prSFEJYE+wqJGBUyABST0z0oA0EfLYz2/Wg4fdvzuXpx1qi0+2Xe1TCZJVPJwR260WII7qZn2gklB3xVYSFTkdabLK0mSc7exxTgkYHDMT/tVSQxyS4kBIGMdaczfLn3qNlC4yQ3PY05nVhhRimA1pDjGeKA3NRHPanj7uaY0To579qlQ/NuHUVEMFQe/ek3EdKVgLXnP604SyHvUllb+eRxWounRgVBRnK9wnCzKF9Nn/16SWWfynzMNpUggLjIrU+wJVe+s0jspnH3ghIp3FYp2jmTyvNDvDg5jXjP41pQPM+5baGO2jHZF5P1NR6LGsmnhiM/MR/KtSziVfNwMfN/QVNzSxnmxlL7s5Pq4zTsNCxRpFyOoWPNacv7uNpOyjJpsb2UW1FuoxuG4k9SaVyWzN2mdwqyoCezx4NQtYSR3WYyEb1XitW5aymykt3C20ZVs4INNjPmCGQ9WXmmK5l3F1NuAvrdLoDo+NrgfUVkX0hcKMts3/Kr9VHpXVT26yEZFY2u2axWiMRjLU0xlCymaOJtrhcseq5zVn7VL/z2X/v2aj0lI3lZH/u8fWtb7AvZadwsZhvplP8Arov+/ZoF1O3zCSMj/cNa39nqB0FINOU9VouHKZZnuCpHmRf98mmiWT1/KtVtPjx0APuay7iIwOQSpX1BqG7ksDLg+tbmlwrPbO3m+UkpEYyehJ5Jrm1f5gatR3LCJY25QdqhEnWX2j/Z9GkjW2KGBiyXHmDBB9a6bwbb6XNo6xwztFcqf3vmNgsT3xXEWGoW8ekvaxmQSmQcEZzzmtK01W61K5FteSxrbxEu0sKAEEdDkdau9gPSH0pAP9c5rlviDbSQeDXMOWBnTfk9Bk/1xWp4PuZbuK7CtPLbK/7qabj8Ki+JX2mPwd9ntow/2uZYWz6cn+lapgeJyNtgY96+gtK0NLzRtPunuwPNto24H+yPevn0rvZEzjOFJPTNex+BtZkTwzZ6c1yLiVbp4lwfuxjB/Lmk9QOvj8OWoGTdSt+AqCfSoIRlWY47Gr4vVSIruxVOS8RsgtmkBUESDnFNOAflp5dG+XdjPeo3hwMrJuHsatEiHzB2pymVhnYTTVR2OFJJqxslVRlCufWgQ0Pt608XQAAIBwc1XeORjkYH40KHX7wU0DLIvQp5p63invVN1V+wH0pBAQflbNAjRM4FZer2FtrpiW8mmjWPp5RwT/nFSNbXKqWzn2qlHqFvLu82RIipxhmzmgDm/EXhDT7LTxcW13cv821lduMH/wDVWNbeFbGbwpc6s0kv2hHIA3cYzjFdpq17aXOlyW8T+Y56ccVzWlysPD+tWbAFFUPgjoc//WoNbmPNZI9rbzeqhdw61UbTImvmhZnOMYYmt2MK2gxsRnawx+dZkkmdQ83HYDHY1kaJmVdWSQSlOuO9V4Fzeog9c1qX/wA1xK4/3qzVPk3KTBQ3OCD6VJRqRbSLtWx8wyCe1SQ4WJQDkDvUcG1ppCxAXbwSakjPyckH6UWJLTnbaE+4FaWiyb7Z09Dis0spsn3MFx61b0CaIyTKrhv4uPxqySNb0SrPaw6W8sKvhxvGCav3yhrGRFGBtBArGzbWuuXXn3c8Klt6KOA3r/StvfHPaNJE25GU4PrVEoxgMpVmIv5a7ADx3FVlPyURalBakpJvBz/drFGsiaSKQvnYn0xxUMkDbGYWsJwM/dpzaxZs2dzj/gNNbVLN1ZfMkGRjhaokjgbcgOAD6Cnz7DEfMjLjtUMDR4YI5cdiRU8u0wEFioPcVh9o1Il5jFOjUtIEXqelMUqEAVgw9adG225iOf4qoC2Ldh96rIitFQCSJy3cq1LKdqk+gzXNXOpzSSMy7kQ/welUZcxqW+si13qbcsu44Y1Bc6v5l1uj/wBV6Vhm5kZsFjg04SbRWiZnc6u1h0/UVMnkzB+480DNJe2+m2EavJbytk4AEormrPU5LaUMMHHYinXGqTTTFyFIPbrTuB0UUmlSrlYJvp5nNWTBYBA3lXHPbzBXLLcNlXVh9AMVq2upvMEhZC2OjCsmrmkZGoi2Owr9nm57+ZTGgs0wTDcYPQ+aOailZo1ytWnkzYo+0HoORRyl8xXaPTz1iuP+/oqSK101cSeROfrJSzwRCDcFwTRD81s6/SrsZMvFbJsMEmwR2amTw2D2zB0mIA9qI8GJQOwxTnUGFsnHFMRmra6exx/pP/fQpslpp6tj/SfzFOtzm4x60XIxKamxfMXdN1WfS18qzvLsITkRnBXNXZfF2oq21551P+zGuaybcjaM+tTT7t25eM1aM2TzahLfrma/1M57BsCoE06w+8J7tX9d+KTLG3JJzTYslQatMktgtEMR6rfKo7eYKztVc33krc3ksyp8o3def/1U+b75qlOOCfSncLCSeHdEHPm3gz6YNUdZ0uxttNS4tQ7MjfecYOK2Zxm3ik9VqvfRGbRJlUZbYcCkwRzsAimjykmW/u461SQol6hl+4rDIpNJfZeIh7gilnRTcy7h/FUl3PdtLdJ9Nga3GU2jAq/DC7SAbGH4V4rY6tewWwEOpTRD+6G4q2vifWU4XVZ6OYlxO98dWG3TY7rqYzj6dK4uwufIuoJycBW5qvNreqXq+Vd3800Z6o7ZFLAA0e2qUhPQ9kstUsr20VobmNmxjANJIMt1rxaHUbvSbgvaS+W3rjNaMPxC1+CYM80U6/3ZF4qQuesx7UOXYKo6k9K5vxvqGlXfh6e3F9D9oT5oxnOTiuA1nxhf60yq0YgXGCI2ODWIEtD96X973DUgOn0rxkbDSo4PsLSPH91l6Gq9z481ybiJY7dfZcn865PrNsrWi08BeHJz61fMVylSe9u7knz7p3z1BkOKr7Yh1Kj8avT24gXc0CuvcgdKiR7Z/ulB7Hg0cxfKRLJGp+9mpkuIz90/pTzsQZARqgeVSfuhR6Ck5ATiZM5ZAx96abqQf6vao9AarNIMHFMD+tZ8zJsObcSSeSTULqhb5lz+NaVlFFK2371TPYDeRGCPYij2jHYzbUwI6hoGk+hq+VsjyLYo3qWzTvs6wf6xAAfTinFbY9CQfrU8zKsR/Znm6RoV9fMNP/sqzA/euUPoKerogwufzpGkTqzgfU0czCxVk0+3cYgZ8+pqBrCVOpFaYjUjKyL+BqFzg4JJo5hNC2dnbH/Wqxb2NT3MVuIw0aEEd6ijnKcr+IqQ3eewFHMykl1IonLryCMU2SyhlGUVg/tQ0mDxgUqzbe9PmJsRR6fKox87fXpUM1hNyQq4+tXDdPnIkNHnFh1pXCxnosaALJw1StaTSDdFHlfrUy2/myD5c5rbtFWGMKOKqwrHJyRsh2sMN6VGsbMwAFa15ptz9q80kNH6ilESqOUAqiDll2tnAJPtSAEcEEGpopRbkhMdPSkL7zlhzVDGpH825h8tNkwH46VKXQJjbz65qu/Xg0ADkEcHmmqrE9qEVnOAM09V2sQc5oAljtVZW5xx61UP7tzGOQO9WknMMgI5pyxid94QKT1xQMrRpNK4VImb6U54njOHGD6Vd+aO3fbIVA7DvVHeXIzSAfGDG6uQCM1oxm3kz5jEMBkY7VVmUJGuOpGaSPjnuaAJXG4Yq5bbggAqomCauwMq4ywH1qQL0TYGD60XDGQZXoOtOWPcRt6GqtxKsZI61LAqQ2kZaQjAB5qqybTjk1LK5AyvXtT7dJJIWfjgd6aAdYxGSdYVKb3OMZ6VpWeiDEySTss2SRGowP51iWjzR6nDKOqtxW1qWqyxXCSRKA46k85xWcmBTkeazcwyxgxj1qlMUdiyjANb1wYNUsftC4Wf+IE1Us7OWKVJwisE+8D0xTTGZ0cilcE0FwKt60R9ojaJFCFeMCssvWoEjkHmnQyBW4OM9aru5IxTVbnFFiS5IiBz8wP0pqKXcAetR54pVbFKwy2savJtz171EYQJztOab5p7U9ZAOuc0rATrL5Yxmh38193U4xVdpAwxT0YgcUgIJ3IwrDAJqMYBVhzzkj2q+E3IzelVfNTA4IcHlu1WgAyyc7UwD7VYhDvHhmGPSiKSOOAGRCzjo46VahgR7feGwSc9M0gKciqg+cVUllVUOw1fmRZVKlsN2GOtZktuyMVP3aAL6w3UsKysm1ccbqhZ8Hax5o88iNQHb5fU077THI+5sGT6UCE2vtYhePWot3IPoKkd8g4OKgz6mqAkL54HWmhWbOBTQwXrUsUckpO3bjGeTiiwDBnvTldlbgDHvTmxjGDu7+1ChWVvmwR2pga+hgNMCvOOtdaZjaabLOP4EJzXM+HoSkZ3Dqxx9K2fEMy2/hx0/ilIUfTvVEnCRghlYHvmnbs7/k3Meh9KndkbAVNoxTIwGYMD8uahlIBZRmLd5h5GelUEXYxXIODjIrbESBGAkXJ756VkeUyTuG4APWmMk24UL608TvHH5SYGPSnkBGEbsVUjOTUKKoJ+bd+NIdiJjlySAD7VJbSiO4jZycKc8VKkaF89Sac1oQyMMHcDwD0piLV20MsInRCCT3qC39aJUaO2Vcnlqs2yDKgDrUjsN1MM1qkSjL9cVlwOEmwxwoOM1palJsmZcZO3APpVGOwlmjZ128DOCetMDTtmaRN0ALJ2IFbVpZE28VwzrIpzkjjArF02Ka3skxMF3dcHqK2kvlfYm2bYFKthuPyqGS0bV8bix0xTHqbskg+4pyc9hVC58SX81qtvCojiWMo4k+bePSsmeWXyGSPHytuDk/NVe3leR9jnINLnkRY2vFd0kv8AZV5FFErvGpLFCHJHY+3pWHqjWk9p5VrEjSbtzTHh2Pf6CotY1W71Fle6kB8ptqoowFFVIpsValcaKpDDIFMVjuwR1q8+2SN0Uc43fjVby933eGqhj40Bb6VKNrMY/lwfU/yogiYZOOQOtRcvAZCVBB6dKQzTvb83lvBDKkKpCMKIxjJ9TWcEdvuiq6vjuKtxzAIBihAMceUF3HJJ5xT0YBRnP4UzmSQAjGe9PQFTuyNo7mgSRYziIYrd0LU90lrZRqSp+XDN8nc8jFV9KjtjaCSa28yaU5ww4A/pVxrcysipFHGFPGw8/Sp5jWMBILjE0lmY3ChyCYhwPxp97NeXUiJcXMtxFGcoJDwKcsfkoFCqCPSoXDBTk/PgkP0ArKUivZBNCqxKVGGbuTmo/mUlWTK/zqqt557hQxJ7ZNOubvygQSFx2rMLCuIWUtEhQKOhOTio1lCZxwD0qJJJHAkjHBGc+opIyJZFVmAyeSf1osQSSOqPGyhw+zO7dT0G6D7R7kbqfdQlrpDCv7pTsznpT5C6Bo9ykeqiqh8RRtXSt/wi8AQZYupx+NNttaKWCwXsZnaJwEKkAqO2KnuWVPDsO49NuPrisq3faVJUMx6k1rN2JSOh03UdFi3m4ubqfcc+Q6DFFpYwXeri7RsW6fehEZAJPoPYVkwmNbsSuCzeg5qPUNefSb+BbfzCVG+RB8oOegP61EXcU42K/i/RNPsr21OnyulvNuDx/wBw/wCTWDfXxnljDDasaBFGc8AU281A3lw0zjbIzZ65AqIFGGW6+taJEDXlQgqU3ZHBziqMoKu2RwKtyFWKNGpC4zzUEvJJPfmqQFcDceuKXGKB1oHJrQoDQBQRSCmIcKQmlBxmmmkAmaUGkxRTGSA1sQny/CF2+P8AWXCr+AIrF7VtuNnguH/ppcn+tMRj54pDkLS4wKaTSAM0oFNpwFIYuafG5A57UwDNKxwABQA44aQhfu5q3AVSFnY8dBVW3G786mZlaPBJJByOaALKyCW8giUna52H8TUDwRpLIryhQrEA/So4jieEk4xIufzpbri+uFPUSHI/GpsSRNnPJzz19achbov41Gw75H0FPUqn3gDkUWAkfpsYrkelJHGskgQSBSeeaiI/elRwN1LgqcBhkd6YD5EKSFcEr2bHWkUntUhctEAJC3PIqPYSrZIBzge9MBB0p4jO0Ed6YgGOTg9/pVp2hWByAx2jC96BoiLYVV7gc0qH5qiqaJT5RfBIDYNDA19PuobeM+YyDP8AeroF2uispyCM5rnNIhS4uoxIuVznFdO5VMYQ/hWBRGF5qlrY2aY7jt8pHsQRWgGz/A/5VW1YK+iXe5Twm4ZHehMoq+G9p011PVZD/IVpwcST/wC8P5VjeFjm1uR3yD/Stq34lm/3v6UgHTD/AEWb/dp8kMf775Yfu+tDDehX1qI2aYy8hQepQYFNCY5Iowlu22HlfWoIF3Rw/wDAv5mqlqY57cMjE4JBwue5psyRybVkuisQGPLjOD1/SrFY0Lie0gYCW5RfUdTXPa/fw3MMMELbsSBi3Srw8iI/6PEvP8bDLfnWNrchM8fmDc45DcDA9KYBoUqx6pvc4UqRXWBMjcv3fWuK04BrnLHGBkfWt+3e4jVTG/A/hPSkM19lGMAngEDPNVkvyijzoSBj7ynOanS7tpRjcDkYweDSsNswLq4nluvMDgYXdhDxVKRJZAMRkeufWr99bRWHzLKGVwQFA+6KpIVZNwNDRL1Icba0dGja51KG1UKfOPlnIzgGsp3Acgc1PZmRryERffZgqnsDkVLRNjo7/QG0q4FpNdwmZySi9MjjFM0q+fSWmQ24mZ3CnDcqPyqCK+2axLcawj3EiAgNu4B7VAdSWe4lZLdESRgQM5CgdqBHtOnotxpsE8MwiEqbiisCAa5rxzqVqvhq4sl1GOS7V0kRFbccrnoe3BrhbUXsyMUnCx5xtWXYKSXTpRbTYEf+rPSQGtYsdjnbidZ7rzhxyDj3xius8G6/baRDcm4tJJndsK8Z5A7iuIU4NdFokMr2ZZEyCx5yKq4jvP8AhOrLPFjd/iwrQtPFOi3mxXne2kb+GZcAfiK4cWtwf+Wf/jwpDYzN1jH4uKljseoJa20sHnx3e+M/xK2RTjpyy2jGO42MfuyA8A15YLW6gyY8op6lJB/jVzT9b1DTNwiuAyMclJsMKSkKx3dibjTftK3U32gAb0bPJ9f6VeXV3l09HS2Rt4yAZOcd8cV5hca1JdXgnDPBMTgujnpWmmoX0Fg0hvpSpypyN2DS5risd9DOs0SSDuMkd1PoakV/Nzs52nB+tchoE0Elu1vHrLiZzuKbBnPtk082OoSrKw1KXz8ZQhsKT71SZVjrPK5+ZG/KoZ4cphHZG9RXMY8TQQJs1CU88qHzgetE58RJdJbyXfyOMicDIBxnFMOQ3/KkXT5YpbiSXK8knFccIgetWb+fxNp9q1xNqKFFbadoBP8AKqNvquty3iQ74J2YZCSIvIqbl8hagXEwTsa5vVEK3Z/X6V0yQ37XSG4gWGUnICnIOPSsDXVeG43GJm+lJSBxsaWnnzfDrDH3TisQghiK0tBvYW026id1iw+NrnB6Vkz3phlKrGrjsw5zQNA/3GB7jFUjxWpZWWo6xBJPaWm9I+vIUk+gz1rLlbaQf9rbjuDUtDuWhAjhSw5xU6IIxheBWedQeLCCNDjvmrqzK6humagq5eXm0lHqpqbQUVL4gDG5TVW2bfFIvqpqtoF5K2qxozcHrWpmb93eXK6p9jjNrHlN4aYdfarkCyGIeds3nrs+7WdrFm0mpwXP2M3cKphkGAQe1X7O4aaMZtJLfYANsmOaokz7jS57G2EztGyM3GG5/KqHy5+ZVP1Fb/iGaV9NhVok2rJnd+Fck19ElyN7fJ3rCaNou5cM1sjYd41PuKimaNwHhkQkHtWJcTo1yxDcMc1NHNbsgjNwI2PfGcVFmLmNAXGP7o+lSrdsRjIrHnIjKmK488H+6vSiIzBgQj/98mp5WHMaou4FOJXKn/dzVuyjW/uUhtbgeaTkCTgH8a5+Zt8mSpH1FW9NmaG8jlXqtVZhzGvq99e6fcNa3MaK5XcpHRhXOyXBkbcUDKeqnoa9F1u1g1nRJjcRoXWPzIpUPKnGa8z02BrnUreynlaIFvmcdq1IHzSK7blRUPotRZ9a3vEPhqHRfKuLfUhdpKxXaRyvv71jLazz/cic/hTRLQxQnc4p6svY5qVdMue8ci/SmT2M8MZfy5G+ophYXcSMAAj3q5a3n2WIKkZLeoNZi+eOsMo+qGpYy7cFHH/ATUBY6m21CCS1/fkrIfUUovysflrMrJ6GI1kWUTD5ZI2HpkYq+FA6Uyky4NWXZtdtw9ozToL6F38sfIG6llxVQKRTioYYIBqR2N5GiSPb9ohP/AqVZov+e0f/AH1XPNap2GPpW9bWNvNpQJjAY9SKLyCxVNsfN3xyQD/gVK1tO5yfs5+j1TOm8kBwMeopgscttLD86i8hmilrKv3kT22vmqN7e3MEwhNqjoPfFMk0/apZZOnbNQpA00LMG6HHNPnmHKXIrqVmCiMDPvVkRz/880/77/8ArViGB4AS04b2AqWK3Eq7hMw9qrnkHKaphmbrGp/4H/8AWqKW2kKEeR/49We8QgIzKxz6GrEdg1zHuimz6rv5/KjnkHKSQtO0PkCFZEXsX21IIZ2jZPJijBBH+trIKMs3lt94VtLp0Nzpjg5EjKdrA8g0c8iOU4WWzm03U9k2NxJIKnIxTr/5Jw38LDJNU2V4rgiXzS6HByOtaOoOrafDIvPAzWrdwaC2KlBgo361I0ixv80PA7rWdZt++Hyv+VbcYtWQGSXy29GU1mAxZ4nICP8AN6MMVpWhG4bjVL7NZP8A8tkzVqEBdvtWsSWQaou1iR6VRgtJJSOOtbc20vuYZpI5IiwXaB+FEgiRwaXIVwAg99wqvceFZZZjKJ1QHritcRRk5IqXe0fyq3yelIsxIvDQiYN9pzj3rTS0CIFJzVorQF/2k/OgCnItvF/rm2L69aqTRaRJy8iNj8Ks6ku2HOVP0NYUjUx3J7k6SsJMIYsO4NYsky7uvFTXBypqgQc0EEpm546Upm+ToTVcrT495wqEBu2aXKBq6W8CShvOfd3WtkzFv9TIA/vWJaaddpMsjxYVeuDmtwSx9fJSs5mpWmE7NidgR6iovs0ef9f+lWJZY5TyAB6Co2aCPjbms0Ji+XFjAIak+xxOu8SqG/umq74OWAxiomkAq0QXnkSJcKgUjuKpvLv5qtJKSMbzj0qETOp+U8fSnyiNGK48v70Yce9R3E4kOY0Kj0zUXmhoxxzUJdlNXYdxS7joSaF+0E7iPl9aZvoWf+65/CiwXLRkUffGR6VIs6kZAI+tUi241KpAX5ulIdy8t/IoxuU/hThfTP8Ax5rKd/3mVPFSxyfN1oFc00eT+Jjj60ryQMmHU8e+Kp/acev4VG0wY55FK4jnicHnr608hwu4DinzWwj3OWAycgUx3KWzAHGRitUIdkMuTULEBqRJlWHBzupn3qYD9xAyvWnJK2csSTjvSqq7SDUZGGxQA/ILcjj1qwrusR8vgEYprwBbIt/F2qAXDlQh+6KAJ0LbME8HrTGCq2B2NPRwBRKnmMCDx396TQydFMrKx5XGKe0Ch9w6Ac0wP5cQHP4U/cWySeoxUAQ2xY3QA5FahtvMXco5FZ9oh3n2NbET4IGCfpQA6NmtbItIfmXpWM9w0jlia0NYdlt1VTy5wKxckUATM5arKzrYMF4aNxzk1nljUqMssJjYEuOVxTsBeWPbP5iDuDUOoyk3PPGQDSLJLDHyenrVS5naVwx7DFTYC9FdBNuxAAB+dWvt/wDo7ZYKxPSsETP607zW9afKBcuLnzFCdh0NVCc0m8nrQCKuwgNHegHmlzQFx+40bqTFJg0CJVkxTt4Y1WLc0FqLDLG7AzTknIPQVWDZ6nFOBGetIC8shPPSi4Ectrtc4Oeoqurc4p6nPWpGXre4jFkYwuPLHy5Gamt7lI4FjduTzVFGwCB3FKuCwA6ilcCSRh5rYqpdyYYR4+YjNW5OccCojHLK2PKU46N3ouBTSQRjDruWpi1tIoKLsIpJodvBbn0FNhiBJBZR9TTAY5OMgj8arlstTpgyTFD+FLCqmTDelaIaFWMucdu5qQfuuBk/jQRt6VNEFwSwzmk2KxJZDeJHJACimlovLJTgqcU0kQglFwfqaRkDqh7lulIGdZoyERJxR4wk22llBjl2JH6Vc02PaqAVgeMpmbV7dAcmKPJHpk1aIMG4m2zfIAAB0pYroRw7SgOT29KpuCXNSxwSSA/I2F4pNF3LBljKsqsSO2Rimlt4+Zjn1qJbecSLmMhc9acxVnIXpnilYLi3B89wXOdqgCnwiIJ5ZyM9fSmR280kgTbhj61NNA0aiN2y6tjAHSiwXHrF5cyhWDJ2xU0TqHXdkDJ6jrUKgrMtaUtwtzGqzNnA+UkcA1IyjdMr3EZUEc9KvWSZdaz2AN0MEEAAAitayXBz6A0mBh30jNfzZOQGwKmtjKYxsYhicAeorOmYtcOR3Yn9aejOqh1PKmqJOjiVTGisnJHLZ6VVcMk7EswHYVWjmMkJkIAcdFNWornzFxPHuxgAg1m9BDjLBjhCGHQls05LdpIGmQEgds8GrROk4AW2Mjk4znaAPWmyNbK+2JXVR2zmkBgXBZmkdwQxPIpYlJZOOM8/Srt9FD95mI3Y2iqbS7eF6+taIoSZzFcHB+WkM4XletMmIcMx61X61Yi69wSq7NykDk+9QqGYFVBLegpqgscDrU8e2M5/ipWsBEEI6jmlGRjHanyPlyTSCT0oAAWMm5jkmr6KrqInJGSM4qivLZ7VppdLFFhIk80jJcjoKlga2iSKyyNIxbcQEBPYGtNZMOy5AK9c1zRvArApx64GMmoFvWjmMjHJPWsWaQqHTXEnlwedtLjsF6ms67nS7iSLe6bc7grYzms8ag0iOAeCc02OVWPJ5qLFSqETXCW8r+WoUjgn2qJpzL8rAtuHQ8Zp7wIzEuA/oRxn61KfmxgDIGBWlzNSEjZmwpbbtGCM9BUzJH5e6P5xnHSqob58ZA9/WpHnATG7AUdqVgvcs20JgdZZRhv4PpUzF3cl9u09lqsk6zRJuY/Kc9KmWfzZWXHQZzRH4hnR6h/yALYerr/6CazRLhgoTbnvnir2tSLb6Habj/Gv/oNYDanFJlFVuO5qqg4GstwsR/eqyk9DjiodTuoGh86PcbhBgKBwwqi+ouBgEBfQjNV3mEkbyL95Bj8//wBVZrQmTuVHdmGGRB9Vx+VRbD2p7LjGeQeaXJUkDg1siBmXYFMLwM/L1NViSy5KkexqyJVJXao4GN3eoidxx3POapAVC3sBSIcvirDRRgHP3u1VlBWStBjnHNApwAZuacVVc4PQ0xkZ4puae3P+NMx8wFIQop2KQDDH2p1ADa2L19vhjS4cfedn/LP+NY7CtrWzssNHjPVYST+lU2MyCTjmoz1qUHnFMcYapENFPFNozzimMkHHNJ1Yn1o7UDrSEPyV4AyT+VKCWCoAM5/Omk7jg+vFIGKNlSc+tIC2iAFWkHAYZ/Oor841W7/66GiSc+WR7g07Vv8AkKze4U/+OigCqDg9cZpSrEfSm4p2Tg5yeKYiSOQcDrg5we9LICXLMMA1AM4p47CgCZGxwOlOY4IyBnvUS9aV8qdpOSO9IBWI60+MBg2f7jVD1p+SF69sUxk0kIRIto6oGNOhbGfQ9RSLKSAGOcDAp1suXXGOueakDUsobk4ktx83aukt5HcEOhVh1BqtY7zJGWhhbCMRhMYPFW/tMn/PtbfqKyKJT0rN1+dYdCmJGTLiMD6960PtR/59IP8Avs1jeJZ/N0oJ5CR/vFbKnrikiir4Tz/pv+6v8zWu17a2jzGaVQS33Qcmuc0d1SCTzLl4UduVTq/pV6KYo+bK3Rc9Jn+ZjTA0GvruaLzLe2+zR/8APW4OP0rKu76w3M1xNPqDnqqnan5ZqK5SBpyt9e5lXtJ2pix6Wv8Ay9Q/nVAJb3lgYo99pNbuuf3sLnPX0zVpFkKhbS5ivMdR91/yPFQbNM/5+of++qYYNNPKXUYbsQ2CKYrFsXCoxjmUwupBIkGB19aztVZpGjDH5SDx7g1oGe7t0KSN9oiH8MvNZd61s0cbQQGFgTvTdkfh6UAJYrmRR/tE1tAsq4X1z/KsWz3LHvVdxWtnTln1Jp0ihw8SgqGYfOc9KAHl23j5jgAGkKq/JGWznPepDY6gp/49k6Y/1o/wo+xaiVbbZbv92QVVySvJIY0YuwYYwoYZxWK31rYfStQkUj7FKf8AgYqs+hakOloR9XFSwKSPEMI0WffdT0IicNGMY6U+70u8sIEnuYwiM23hsnNVPMx0NS0BpvLFLZyAriVsYbPvTYLZZLYMx5yQapxPlhzV1JGAVEOF6/jSEa+j2FvJa34lxu8sbQTjPWksLGFNEu3mKbsHGHGSMVnGaUDCKGb0xVeaW9EblrJwNpyfLJqkyzHrc09Ul0llP3kfINYQbIrV05wkByM5ORzVXFY6uLT4N9qzL8nAbnsaoxaLcah4ka3tIS6pON7Doq55qkDIyhhGpB9TzXUfaW8P+F1S3Ba8uVDMRx/ng1jOfKtDSKuSz+D9Kiv2+069HDvOfLij3EfiT/SqdxpHhGGUxt4kudw/6dga5oyT3LbrnAf3NV5fvFazh7S12F49DrYNB0G8jD23iGfnoJLYD8+abPpF1ZTxb2FxaO3+shORx6+nWuPXMbB0Yqw7itvSNWuBKlpJM5D/AMRP86U1UirxHFxbszr9L061a6e72ESxHAweOR6Vqgup+SqGkkhbv2Zf5VoxGuvD39mubcUhweYnB6Ub51GBj8RTwwpC2exrcgbIvnxNHOqSK3UMvBqlf6fHcQeXCFgYH5XVeVH51eprdaRRx2qWs2mtH5148rMCVIJ4/OqWtaeGeK6N7KfNA6jPPr1rY8VSQxy2/mgnKnGBn0qK8ktf7KgllwyLjBIrgraSL+KLOVk08bc/aA3+8Mf1pkNpcxMXSEMmMbkNbX2qyPSD/wAdqS2uYf3gRNuV/u4qzMi8OnUvtEy2l41opb5gVBBPPrVKTSp1uJRJJk7ySV7mtbSJ1kvVVQR9RUOqTzQ6lOithQ3TFUmSU105V6W7uf8Aa5qzDBKowLYAe4qA38wXjaP+A1FLqFwINwfIPTikxpmhboI52X+LGCKoQG0tNatmjO5i+PpSaFPvM/mDc+4Yb061mPGYdR83bjbKDn8RWhB22vzzWjWzRzSRRtIVdo+Tio9JvHmvWia5luLcDIeRcFSKvapaPfwQiMruRxJz3FXD8xqgRgamqi/k3O/OOM8VBFAvUxr9cVb1qKD7UryL8zL1zWSWlBwk7KvpmspmiLNxbO0nRP8AgQqv/ZQk5ZYfyNSGK4Az9sz7ZqP973lf86gB8NlLHxGYVUdgtSta3DnPnKKqtwMmRseoNSGGLaCbhieuBQIebSXvLGw91zUBi2N0X8BikkQYUxsxHQ5amqNp6mgZajjQAEbx9GNNmsoJiGLMOehpm8gfKajMh27XbNMLmhJosMlgssWVKnJzzUKQMOBLtP1ogu5IonjLZRhjFV5JYehFAXL3lyAAecahUPbyuBIWDjOTVSJhGwKuQueV7GrU9xFLtZUCkDoDRcLkombuaPMbsaoNdqp5yB61YBNBRP5rHqaTeQahBNNMqq6gsAc0DsTGfb1oWUmmatIuIpFVQxABxWd9ocfxUEGr5rHvTJJpwv7s4P1NZclzg/xfhR9pJTcTuPpmgVy8J77+I59yRVlZZP4ic1iNfsfujDe9Kt7M3UjNFgubLs7/AMRqOGd/LZSehqgt25G0mlWbb16UBcnkZmU4ojFwsYYMv0qPKschzin+YqjG4/jRYaY6O/8ANHQcHkdasLemECRFJIqk1uIo/NBBVueKRZEf5TRYdy19o+0z+cfvHrViOVwc72H0NU1h+zhcNuUniplamUWCQecKT64pd3GMLj6VCHpQ2eKBEm4D+FP++aRpM8MFP1FMJweaaSCeKQ7Dwy/3F/Ko2++CBSK3OKf5Zc8VUHqTNaEkmCoaojwMirsVm067N4X3NQz2ptW2lw/uKuRECAXDKoycCobi43DKNn0wagvXfaEVSc+lVYFkWQdcehrG5qWhqLjCyKQfrUonRl4xzVSaBndW6Y/Wp4oQoBJz7VPMSLIxK1XaEsM4q+EV1wBTPLMJBqvaiMySHaDkVnS4z0rqVa3kxvXJp7WGnyAZiA+lHtwOLbFX7bTYriIOblFz2zW5c6HZTrhMp9Krx6Hswm9T71XtQjEZCkkKbBLvUe+aa4kJJEb49cVcS1FrhQSQOmTUckkyOGj6DrU8xdjM81v9qk3yucAkfUVotfRTYDRKr/3qbI0T4wVyOwqjJlIiWNeHH4VE7sRz171ektmIJUj86z3cK2D1qrAJgH+IfjUogKnLDioS47VIs7tGUz8tUAsg28cVCzYWntvNQlWzjFSFg4ce9ARt3TilQfN8wqXA7UwsQmMk8mnhSBjP505qQtnr1FK4hOccYpRkHNM3ntRuz1oAl8yjeM4Y1EdrDmjaM8dadgMdmZjliSfekMjFdueKUjJxThbsQCcCrEQgHrUq1ajtAqckU1bVmY5wF9RRcBhUgjBBpxBUZNTrEqP6j3qGY5fb2oAbuL/L3NRSRmNgO9OZD1BxTywKgHtTAIySAD1qboKigO6TIU8VPLtbpUjFUFhz2p6qxORSJKi2+wod3rT4WQtgZP1pWAsQbV5fgetTmedo8QRlVH/LQ/4UDyY4t8v4Cqd1dS3DcuVQdEHSixRXMTPMWaRnYDOT2qADLH2NWg+yF+eTVIMQwAGc0ySURliMdO59Ktww7YzJHEzgHG6q8fJxkD61tWCFtNlAxwT3pAZDS+YW4wT2qrMmOTUxBRmYgcGmzqHt/MxyOlAFbjGc0AimkUg4qhEhNNJ5oppoBDwRikzTRQaYDt1G73pmaBQBICM0uRUdFAD8g04cGoxTgaLAyYHnNLv96g3GnBvaiwFsS561NAyluoqmp5p+4DvUOIy+zgDOaVZVMRGfmNZ7OWGAacgUc5w3rSsBaitU8wltxJ96f5CeaykKuB8p96YsmRlXBNMkkYHcDmkBVuUlWX94Bg/dIqDzGU4AH1xVm6YllJPUVXaJsA45PIq0BJC4ZsH0p4cKwNVlba1SgZXdQIsyNk5qSxPm3kSZ4zmqgJbqa09EhR7/ACR0BqhM7XTY8sgPpXE+Irj7Tr124OVDbR9BXe2QCRtIeiITXl086yXM0i9GckUxRBUJfAA3H1NaNy+YwEJJPDEVjNIW61dilLEKeuKRQs87xR7FYeb0PsKhiVQhDd6WZ1lkAQZbufWnrbOWUu6gDt3NAGjFBi38wn5guc1VLAuWzVrzt0flKcJ6VQlRYpSckA9+tQwL6ujR8qA386bJuJAXoKhRxsQMTkntU4BKMcnGODQMiQb7itWN/Js5H6YBrPt1zICfxq1eSeTpsh45YDmkxmBEuZjn1NPZQqlffNa01vaQwJ58JDyDcrD0qiY4v4QTTTEMiYKOSAe2alV3UkkHbjrilQRr7E9cjNSrtc4XuP0qW7isSo8IGDNJt7ELSwyIbhhuO0Drio/LAG1egpxVIbc4+83OaVhC3ES3UgV5Qqg/Lmszy2JOBXqekabplxp1us9nFJKF+ZmHNcRraxQ6xPHBEkaDGFTpWkRmGImdSAPrTTCVj3Zye4qzlFBAYKT1pCp8kEdzincQkto1vBbzFgVmzjHbFQSHB9q1tfQxw2EQ/gQkj0rKdGeFFUZZjii4wK7lBpu3Feow2Vrc6hGnlRi3s4/mQIMFyOufavO7+dbjVLmQoEy2AFHGBQBWjzhuetWojGSQwBBHeqgBA4PJPSnAletJgWuO9SjTbq5tWuo4HNupxvA4zVJmPlNXRzSSSvo+mxyZtoljZlB4JJ5JrOSBGOlpLbookjfLDIO3qKYJIx/FXoD6nbadLON6u7fcQDIBNcU9kkkru5G92LHAxjNJjehVDg/xZpuJy5wQQRkE1dXTk2gKQP72erU17B2UDzNoHBINO5JRjLOCXGR1GKm2hsDYCScDNLM0cREMDbgowABRbuyzxtgsVbIx0zTKTJYrC/R9zWM4j9NtSWTNJKcxlMHHPet6fxBcSQvGoC7hjIFZcAzKKVibml4rRn0exCuFYSA4Pf5a5SCK9lLeVaPcBThvLQkA/hXWeKXZLKzA9/5VLqeqT6TY6dpenTPbpHAsszIcGR2H8XrSqSsWjj5XDSB/LAxxgcHPvU9vAz2D3QkzhtsqY5+taOsIBqkme6qx+pHNUI2MfmgH5ZF2kUCJY9H1G5hikt7R5IivDdM1Ql3JOyMMMOorpLbxKbGxgtEhBSJdpyev6Vzs7pNO0hPzN1rRCIM4NNZgRinugyCGHPHNCQBtxbORwPSqArZpDU7wOoztLDOBt5qNlZBl0YD3FUAsNnPd7/JUHYMtk4wKSWCSCMB8c9CDkGtjw5EsrXe77pVQR+dUtbXyL9IQcqEBH40xlBevShlKsCopA+09M0F8gY4pCH5/OiogTuFSkelADSK2fEa4nso/7luKxsFp44/7zgV0HiNVOpspGdtuoH5mhgc9nnNKxz35qMGpERpMBUYnOOlADab/AB1KYJ0G54JAB3xTCpxkgge4p3GOqaCEzswBA2jNQ5FXNOHzyn/ZFIRXI2nH60Crk1s8r5SMnjJ9KYLO5GP3LHPTFICnLnYatapzqBbs0aEflUUkcrfeRh26GpdQUiSAnvbp/KgCoDzmpGKMn+0P1qP270BHwW2HA60wL1jpzXqOyvtCnHTNXF0SRc/vkOCR0q94Zi8zTZWPeTFXwMhv941MgMaHR5Z5REiO8h6BFz+fNSan4Xk0m3NxeX1snpGDlmPsO9bthd3WnXPm27/Iw2yRno4+vbFQeILe01vE/wBne3ulwPML7gR6EUkyjmrbSry+jeaytpZ4kGdyrgEfj1+lRQW8t3OsEQQSnqHcLg/jXUadruoaBYfYwsNxHGP3R6Ffr61zUsovtXa91VWn8wkyLENhPpjFUmKxY1HQ9S0qNJby3CRMcb1YMB9cdKjsk8ydFHGTiupvvFjzafFaWVr9nV4+DLhsLjAwKzNIsP8ASBLt4HaokI3rO2a3iBMzyHaQN2OM04WLzsAty8P+6Ac1ZVMQx/hT4uDmsOY0RVGk3O/aNVccE9BUV74clu4PJm1R3XII+QHBrUU/vR/umntJzRzFHP8A/CEwsgSTUdw9PLq5DoUVjFFbxy+YMMQcfStMP8wNDnN1F9G/pVcwGLd+DLW9u3uHvJY2bHCoMcDFRjwLZj/mJTf9+1rpmOEJqPfl1HqKOYDm38D2S9b6dv8AgIpV8Dae2f8ATLngZ+4K3b65FrAXB57Vky6nJLn9/tz2Q8VPtJIC3/YFt/FJJ+BxUH/CF6GwyftJb/rqaZbagYPN/elgACN5+WrEfiGNc7kR/oCKn2kgIP8AhHdOsb62s4lkMV2CH3vkjHpVn+ytO0W7jWykl3SOFcSNnApk+pxXd1ZXcWALYkspOCc9hVG9vDf3xnK7WKhSAeOKrmkB0ElhA7EyXXln+7ikFrbr8320cDGAMZrno8QZLLMdwwW35A+gNVWtrTcQDK7+rSdfyq+YOU6Uad8/mecTnsRUkNiyElZcZ/2aqjXEZAnklHXuW4H6VLbapuf94EC+qtmp5yeUr6hYRXjzQXK+YkcfmDsc44qnp3hfSr3RraWaFjLIu5nVyDTr7VT9pkeEcSReX1/Wm2euNaW0FqkAKxrt3buP5Uc4cpIfBmnRR5i847QSdz5zWbJZWTJK8kfyJ0j3c5FXLnU55rgXCXBiWNf9X2NE2pmeFmkmQZ/hA61HOHKVLHdFPAbaFjLL1QHOPxrrDCCpVnkdXXkM5xg1y8eqSGLZC8aOoAJ28/hT01i6VWSWVnB/ujn8KLgaw8O6LnnTYPyNLJpOgwZX7DCuP4cGstdYlFs0auRu6l1+bFVjNHJlSeT1Pc0c5RFOIUuJEiGFzmtrVLuKxGnTywmaJo2QBTg5rnpZbcbkVX3Huxra1Rra50WyVjkq3PHTinOWw6ZRn1yOey8h9OjIDblcdRzkVmapqralGiPHGjqch0XBP1q9E8KbkSMAKcdc5qlqVmkzC7tkCnH7xF7+4pxmS4mYC2MHrT7d2iukkYZOcVCZUDblJ+hpYz5jAhh1rYizvodzp2rrpMsizI8kcrLubP3eK6q2vLS7iEtpKJI++OoPoawNEiguHvIbhUZJI0OGPt2rU0zS7bS1kS3kLhznBYHH+c1pF2NpqxpCSMAEtjNOLLt4cD60LGT2/rSmMr2GPTFbXMxo2d3XP1qK8mFvZyzogmeMZVAetSFYz1RfyoVVU5VQPpSCxwl/e317cIb2COFgMIrKQcVDYzrNG1u0mI1OOmaveNj/AMTq2/65VgxcQuR97ccCuSpH3jWnrc27iKx2oGdmYZ+ZVHP1qube1gkDi4Xb37VQT7QFww/Wklj3phhWfKZFiyvIrO7WYsG29getLf3cV/qMkyZVXxwfpVAWkvZNw9uaaQUKj1qySyQuwKTz607bbLGoKljjJ54qAW0zcF1ZWHBBpJI7lOkTUALHIltJiCPaGPOKfJOrRkgA465qnieUhUT58+tMVZ/OZHUjjmqFY9ItpPMsbZvWOooElQt5jhsnIwOlU7AtN4b5ZtyxELjg5HvVjTZPN023ctuJXk+9aCM3xKhxbOPUisII3c/ka6LxChNgrgZ2ODXLNMyyhGQgH1qJotMuhNg5bI+tI8wHQ1QklWNwpWRSRwD0pjPIOGY7e4FTYReW5H8RXFMkulHOB9FrOLYY7g2O2aYVkdx5cbEeop8ojSFxuXIyPY1E1xIpyenrVPcyOVYkinuzbevHrRyhYtJeNnBI/Gkkuo3bDOQR6VkOTFKHzk55x3q2jC5OUAXHBwaOUC15zx4O4kds0n2oFsnrUUezcUdxgHB3UskcMTDGCD3BosMn+14HtSpN5nKMPxqD90i79ykf3T1oW4syM5I+hosBeISRMGrEUg2BQwOPSsuO7SIngPn1rRs1mvHVba3DO38IIH86TRSJnfYhasiaQvJuJrdudC1pk2rp8pz6EGsyTQdbj66XcN9AKdipMpyTyPgMc4oViasDR9Yb/mGTA/7WBQuh60eunuP+BUWJsVijjlmB+lM+Vm2lsVd/4R3W/wDn2/8AHhTx4Y1p+sEY/wCBUWDlZngAVIGAq5/wi2sesX/fR/wpy+FdWPVo1+pNFg5GZzRzO+UFSBZ0HzAVrReGdVVcCSP8Mmpx4YvnQrIQ30BouHs5GKsme9IZcMT5n4Yrdj8KXasCBkem00knguR33MZUz2HAqOYPZyKztu0wfQVlxszSriupi8MXkMBhBZlIwM9qjHgzUByhiH+82DU8xfIygS32VQVxtNMFwg4JrYfwzqi2rIXhPTnfWfa+Hri+Rit1bqVOGVm5BppiaIfOTH+sX86fHMnrUn/CE3AOTqFov1arUXhSZUC/2na8U7klKSZPWoGuFPGavz+H5IG2tfwN/ujP9ahGhxnl7t93qq8UWAoiRw4OasxXL+tSHSMfdvWP1jFWZLeGO3+VPnH8WauKJJbG9MbYYZp15dpOSnkFcdyazUznipkbaQTWjIjoRspJ4qRbQMm/gN71YeeDZ/x7gN6isu41ArkIuB6E1xyvfQ3uTl4ScOM46VBIVDfLwKxpb0+YSGpn21m6vQqbJua3mH+9+VGCwyWNZ8dzngsB9aJLtgu0Gh02iS6SynINWYZywA3D8awpLxiDhqdb3RLBSankYI6CeaQJgA/hSRSlsZB/GoYmkYY81cdietPbzgNrEEdiKaRpewl4JfL3pjHvWZ9pY8bjT7piSRuPFUicVrGJLZJKd3NNBOOaI5CHHAI96neJXGVUBj3rWwrFZ2cZIPHtVZic81IWdSQ/5VG3NMQ5csPlIz6GnJKEJU9+9Q4IPTimltxpiRoxTbiARlamIDDBHHasuOR0HUH8KtJP8uWY1DiMmMahuBTGwOlNMwLdTijzF9TUpMGRM5H40zcfWpXIPNREj0qxCZNLuJ4pvFGKoBcntTlbnG0GmCpFXjORSAzVGHFTxkM+DTAuenWpYwEwxT5veqET7WIzg7fWm7ZHYKhwp70ACYhnc4HYVbjnigQhQCT2NSAyexRLYuk3PcNWcUrUsbkzX/lMFKn1q5qdsog3LGOPTikmBzwyBzTCNzYqwF3nBBFQybUfAqwHoDHnnGakjQyH29aqjJbJPFXYMtEQh7daQxsgCvgEmiN2jkzioishOWFLbsWm2HmgDR3JPAGfgf3aY1tK6+YwC+xNSQx+WcuuQO1R3k/yncfoKBXIpiqKFUKT371SCBGLepqSEGaUKtWTaqRg/e7UDKqSfvBmtTQZ2a4miY/Ix6dhWTNE0b4IqxaSraJvXhjSAkvMLcMD90nmtPVY7caZFJEign0rIuG83L+tXHk83RoePu1IGRKgHIqLvzWlfJH9milwQSMkLWaORmtEIeNoyaYcE5p67e5pDjdkUANHBpGNP+83J60SIE6MD9KYEYHrRmkzSYNAx+aCaaDTvwoABTthC5pYyuSCBUu4jpTEQU4GkYfNQKQEmc0q0IoK89alWEkZDCkAvBXANAOKjKsrY2n60rEEUNASq/0FSgAjKnIqslXYo+AMioYytPFJgvjin7wUVHAyB6ValTcR6DtUMsPmfOxANNMGV5YlYhlX5abgAY6CpBCcZTJJ7A0sUZYfMpzTuIgH3iK3fD6Zlkb3x/Os0W/PIroNCgVISR1LGmmJmxfT/ZtCunzhimB+NeaCJinA716L4gG3RhEBzKwUVx/2UD7opcwIyvLI6ihlYVrfZvlJK8jtUBtmNHMWVYwdhIHIqW2kKlkx97knvUwgaNsH8qlhRVmDkCm2JorGVvNJwetK02clvu/3RUzWwZmYNj2xUZtGPDqQfSpugGRS+e/zMBt4q2xZAYtwIp0NpFHEoZeQeTSSY3cUcyAmtlzUt3LHHFGkgyrnFNg+UU3UYfMjt22s209qTGZ7XMk332LY6ZPQUitWhbWqqmWXrViK0gbdlOhxS5gMQOWNXIV/db+Q47EVbuLKLyyYl+anyRRsV4I+UdDRzAQA5wrHg+lRSMgUZA6j34rS+zp9lIC5Y981lzWsi7ifXpSjJMVjrYPGNhbhmEUxITC8Y5rk7u+F1dyXBXbvOduc4qt5Eh6Co9uDg1aYEomB6Kp+oqe3bfIkW0AM47VWACjOKtaf8+owDtkn8hVCsT+IpN16i/3UArKWQrsYdUOR9a1dVjSTVZQ/QBR+lVPKtEYrsBPqRk0cwHZ6frllZaI8csyNdSAu5zjcxGP6Vw7kg5FTvCdpaKJsj2pFjU8MPm7ijmArmV/UD8BQJDnqD+FWJLGVzmMAqaVbLyEJuMBhRcCzb28cigsoz9K6HSdDl1S1LW8yB0OGVxgA/X/61c5YvI95Gq/6rOSa1RuHSR1/3WxWTDmK99E9mWbzkkOSuUzgEVBHMMBmPNSalu8uGALwFMh56/5xVIRsSSeBSsJO5c+0OSQrbVx371DNcFUPzA5qkzguVU/Xg0kEIZvMnYhM4Cjqaqw7DY1eSUbATV2KKWInLAD0x0pVZY8LFGwHsOlCTGQ/LG31Y8Gi4yyJF6E89qsWvzS1ntJJnAQHAycCrVkL1pM2ywHBGTIaFuSafis4Fiv1P8qztZuFk1NiP4URP0qzrbXzX2nRX32fnGBF7sBzVDUl3eIp4P4PMVf5VNQtEmrTGfVJnC4xtX8gKpEE9wPrRrU7Ra5eInCiT+gqj9sOfmNWIsOSB0U1FkHrSpNuHKhfpUUrKrdetUiQY05Jwp5APFRbs00jBxTGattJGbCU7/nQ79uOo5rT8QWinRY+f9XEjrx0ycGuZR2ViqnG4YP0rpJtWjmhWGWIOnlLEwLDqKVxjtAiVNNMiFSr4wQcnj1rC8QnOssPSNR+lamnT22nW7xIW2sc8npUEsdpc3Ty3JDPIcjaeAMcUc4chz5oArcFtpsfLKjZOPmbOPoKDaabJwAqMfusCaOdDsYwX5lPvVl8biR3NXBb2UKKXWRmz2PBqQw2L4I8xc9ietHOgsZ1um++gH/TRf51qa3Or6vchecAR/TFTWdnaPdW7xp+9Ei8f5+lLex2cuoXTMCHMrZwaOdBYybazwQ7fgKugFeFOKfsSKYLn5ccUqS2oXbJvdx12nFTzXCxNHcARlWwSOpNOE8eMOpIznkcCq8YifLnCnkY/wDr06EF9ykBWUZC+1K47CTWNhNltrq57g8flT7axji3eS+cjncaPLWRQzylPoOtNMfyFQ8h9xzRcLFpIMHmUZHUA0sgeNMjBA61WgYp8rxsRg5JByaeLqZZgPLlaIdFRDzRcTQonyetPuoLebyPOTdlTjnpzSmJ2LNDaTb2ORujwBRNHd+RGI7WcsUIyIyccmi4rDPssNkg8pNnpzTfPk9j9RTkt794QHsbn2/dmlSz1hGylnOIvRkwTRcLCRX0luhjXaiE5IA6mpY7t25wMVXn0+/cbk0+4OOvy1LBZ6iU2nTrgH0x1oCxo28u9PpTLmVlxjgdzTILTUkyBYTDPqKnl0/UZISPspDdgXApJjMly5znmqrWrSPuArUTSdX3gyW8bL6eaBU0fh7VJs+Xbru/urMMmmmMr+SHjh45EYArX0sjYF7qcGr1p4Lv5cefeQwPGADEg8wj+VaaeE1iXD31x/wGPbUt3HYpt91QOuFwPzoi/wBWp+v86vr4Zsxy11csfXOKsroViqBd85x/tVDQWMhuGRvXNOzuUGtGTRIDG3klw4Hy734JrI/s7xPF8qWttKo/iDDBpWNFEnBxQG3XMRP3RnJqm0PiZfv6XEfpTf8AioE+9pGf+An/ABosXyR7mi8vVawvEGrfYLZY432zP0PpV3dr+f8AkBg/gahnj1OVt1x4b8w+u0mi4ezj0OWl1ItZHzZZJGbuXAxVNLtmVXXjJznPWuseCUqSfC44/wCmZ/wpiwShgw8LgEdD5ZyP0qrh7M5+O+lDFSwKkYAHamwXrz3CxFVDN1Yda6UW8rf8yx/44f8ACg2LN/zKzfhGaXKg9mc/FcOtxsZg57YNSG5VJZSHwShAPvW6LJ1+74YmH/ADSfYZif8AkW5PxQ1VkT7LzOeS+Jz95sd8cCo47jfPgHrXVLpGoH7nhf8ANTT/AOxtTPJ8KjPqENFkHsvM5aWfKqHP3RgfSoBc4jKI78nOdxrr10DUz/zLKfr/AI1J/Ymsfw+G4R/wBf8AGi0Q9l5nHx3DhNuc+560ya7dl8rcRjuCcmux/sPWwefDkP8A37X+hp48Pa43K6DaD/eAB/nRyoPZeZxkd4fkBkPyjHzHNSebH1yPwBrsI9A8QOf+QXaL7NsFXI/D3icDCW1ig9Mj+lFg9l5nDxMrP5i7y3ptpZY7mVtyJIvoTGa7ptA8Ugf6uyH0kA/pTV0HxSTxLbp9GFFivZx7nEpbai/LwSsfUKasRWGoOcixuSR0CjrXajw14pYf8f0A+jUDwr4mPXVYR+J/wqLB7OHc406JqssZcWMqv23YA/nWh/Z13d2CW0OA8a/MWPGcYron8K+IEQtLq0IUDJ69PyqnY+Fby9g81NUROcEYJosXGEI9TBj8IawckzQofrmr8Hhu8jTbNeW4buQpya3x4FldNz6xMD/sRk/1qNfBdp/y01m4c+uyjkJ9wzV8NWgYPNeJu9kXB/OrLaNpK274S3Zwp+bCirq+C9J/5aX9w49OBUy+C9AP/LxOPrIP8KNSuamYdrZQY33ZATbgZPf1qyqaMnWRPzNXtJsLS/kmhvF3pHyB75xWi2jaEnW1B+uTUxkXKUDnzcaUv3biYf7pP+NI+pqFK213d7+3zH+tdEum6Oh/dWS590JrQhtY0HyWsQ99grUw54GAdM12REaPW8ZGSGjBpy6b4hUf8hmA/W3H+FdRHZvJwML744FWF0mVv+WyfgCarnkZcx57qHg3U9WuBNdatbllGB+5KgD8Kxh4av1vX0/z4PMBwGAIBr13+yUB+e5x9FrlLqNbXxqEd1ZCQSR9KmbLhI5U+BNeUEwz2sv+wsnJ/Ss250PXbLi402dPfgj869XutU0az5NxGG7ENmsifxnGqukMf2hCMBXHFBnzHncdpq6qV+yuCfQ4pGsNTbO+0b8SK2jJJI7uF2hmJ2g8ClVnHUmqsM586XqhPMP/AI8KsJpmpmLYYl/FwK2jI3qacsjY6miwGD/ZOog8RRj/ALaCpRpmpNzKsJPqGArbyT1NB/GqsK5Lo8U0Omvbzbdy5GVOQetVLVNQ06L7NHd2TRgk4brWlYHhge+TWHBbRnUbqGWxM+Jj+8B+6PpVIlo1tVi+16bLEGwzYwa5waBNtUG6UkdCQa6qQZjNUytJgYTaBM/37xB/wA1Iugusexr0t6EL0/OtgJTinFKwzCbw+GUq107Ke2wUL4fA6Xkw/AVuBaQrTFYxRoEH/LSVyfXgUHw/bMeZJcegbFbOyjZTKsYp8M2Dct5x/wCB1JF4fsIgdkcoz6yZrXC04Jmgmxlro1ijZ+zhv94k08adaqcpAqn26VolATSCIGpGUE0+3Rg4hi3DnJWp1t7MuWlsbd29duKs+SKPKHoKAITDppGPsCr/ALmKQWtmrAxTywY7BBU/lj0o8ugLiCEjBTWdh7F1YCp1h1bYWt9ZtZfTEhGahEeKNlAXHtL4uRNylZMf3WBFM+3+IwgaS1HPViRTkMkTbo3Kt6irkWqXqOC83mL3V1BFAXM+aXVrpQHIiPch8ZpiwaiP+X3/AMfraRNJv7rEiSwyt+A/OkuNBuouYTHMo77lBoC5m/6ftwb1V+rCk8u4ZcHVCD6ZyKCCpIYYI4PFFFg5mLHbsg51Ft3qCcU8KwPzajIw9AKi3CpBFIwG2NyfZaA5mSecYuUvJx+FUri6upsh7qVge26rE1ndGIlbeT/vmuana7sJ5BOm1WOQc96ymjaDNPzLxVCiZto7Z6U155QpPmysR1G8ispdXkBwFFSXE01xGJIUwf7uf61jYu5uWcltPCGdbjdnk7zUdtHAt5MHV9u44DVlaPd3a58yMjnAzxV15tk5fFboykrmrIbGJd32ct9GNVJry1XLLGEX+6eaS2ufObbuRT2Dd6stbxvxN5TD0rmnN3sjTkK1tf287bdiH8KtkI/RQB7Vk3FgIZ82o2ofxrRt7lokw1v5n41cJGbgSeSmOlNeFDE/HanyXczECK3jXHrzSC8vBhSkA99uK3U0Z8kjHXC8HqDUjcrkVFMjCUjByecCpI0kKdK2U00S0TSbEtdzHArCnt2uctEMk9q2THIRgoW9l5qWCxvrh9ltbS/QrismNI4qazukkO6B6h8mUH/Vtn6V6SvhjV5sLJHEqn+JjnFW4vBs2SJLmIe4SmpFWPMRDOPvROPwpspK8EEfUV65H4Qt+kl1Kw7gAVaTwppKkF4Wlx2cg0c4WPEWapIHUNk5H1Fe3r4Z0Uf8uMdPXQNEU86fAMeoo5gseORyRNyTk1J5k44jLhfTYTXsy2mjRcx2tqjDuEFO36ep4W1H4CpYHiTrM3GyQsemUIpF0+9k+5bua9nvp9IltylzJb47YAzXJfatK0xnd98yE5DZxQpCscjH4e1d1BFlIue7cCr0fg3W5EBVYhnsXxW5H440dX8lLaZQO5xiql541bBS1giQdmDc1XNYDm7rQdSs5fLuLfafUNkGoPsDKf30nlD1pL/XNQ1CbM07bR2BqltJ+8zH6mrAvzWlkkZZb7cR2qgwQH5CSPU1Iir6DNDAEYoEVw5DYOKnEgxioTEwbPUU4j0ptXAcXFAcY68UgQkUoQdKVgF3g96DimlQp5Jpu8DoSaLAOzzS5phbPak3HHQfWiwiTdQG96iyaMmnYCLcQeKUuxHJqRokY55FNMaqcdaBCh9sdKrZ5bim5ToevpTioxz0pAJG5juBKh5FbkGqwXUTwT/KM4z1zWETEPu0qBQ2SM1NgLE7RoGWIAjsaqBl/iUk1O2CcjgVE4/u1QDOvFWYZNkAiRST61XEEsjcA/hU8aSLkqDxQMfn92T3pdOh/wBK8x1zULDEm41o6Ynnl9sgXYMnPegCyx86XpgdxWFeSmW6cAfKDge9b8rrbwySHqq9K5wjfIc9CSaIiNCyiMEfnH7x6Us7YXPemRysWBb/AFKjGKaZ1dyduV7UrDH6jxBDJjkjBrPXdJKkaDJY4xV+6kWexVAfnU5xUemCOCZ7iY4aMZX3pgSajELVI4Qf3p6r6VPC3kaeIpOM1m7pL3UTN2PQegq7crJJIUHIHQUgJXtDPCoMoEI9DSRzx248vYnl+45q49rLbWoQoTkZyKxr4FcD1rMB160EnzRLiqFOYllqPBFbxACabTihwT6UymIerc0pINMHJp4jbGcUAM6U7NKyHtUsFtJMSF7UDIe9O3MKs/Z0WLcDkiotuaLgNBp6kZ6U3b6UKrHGFJB744qbiJQ1Ow+3cEbHr2pNvFTKXkKIXbb0pAVwrn5z0qQID1qWSPY20twBx70wHFFxksKqv3jT3fDYB4quWycehoLc1LAtrJnqeaHcscVADge9ORhu5pJASqWVsYIqwFwAagDk9afv4qRj2ccCug0tCIE461y7EmWID+9XZ6fEBGo9KuBLKniZ8tbwnoBk1z+8KK0vE10H1YovRI1GKwGmINEgLXm/LjcTURkA6+tVDO3c0gmz1NKwFl5FJzmmiXFVTL1FN8w+tOwF4Tkc+napRKCdxP0rNWTkVKH5pWAvh8nrmmsT5owKjjbPPenqCz7s8UWKLka5OBViSRjNs+UBcdarwn5xViWaJPvf/rqZAIdgyT1phugvy7sj6VFJNHtJ3fQVnNcfOcVnYDU+0fN/s4pI5gx+bHtis4Tqfvcj0pRNjvRysDW8/I+UYA4xRlXb5uhrPWQsMknFWo5QACAD9aLWGWDDERhI2J9qpSWE5OUtJ2PqgrWtZmTDxNtPqBWil3dsMLfNGPQiriwsci+n3f8ADazH/gBqawsdQjv4ZPsMxUHB49a6djO33tQJ+i0xoy339QnH+4MVrcLHPXun6rLdyyfYZSrMcY54qaC01RExFpLn1LJzW39mhA/4+bs/8CpyxRbdvm3P/fR/xqZDsZQi1jaVewl59qT+zdQmA3WUmR0J4xWuptI1wJr38HFV55rYLlJL1z6EkVJViidN1eLCJbFfcSAGnnRdQmRTIg3ekko4qYXaocKLoD0FW49joGZ5hn+8aomxk/2Xe2JEkzw+XnGEbJFOJNXGhjuHYi9ihAONshyaBpzHn7XbMvrv60MxkW18H61qqQahai3jhMYCNLLgkDvjFTJ8O9caPzHu7P6Ak/0qKNp1iSJJ5tiDCgSYAFVbqbVEmT7KJpMDL/vDwDSKiZAa6YERWjuQSMpHkVElpqCptXTrg/VcV0sUkvlKGbY3cZxTLlpjAQrEnqc1JRgi01VuljKv1FRyadqhXBspT9BW7E0wTdIwPpirEUzDJYEgegplWMOz0zULiXbMn2ZO7y9K0rPTzp8piNylxuOcoCAKsvdQH71kh/4Cf8abDNam5DKnlL6KlUmS0WrqwivtStbiWdomhxtUDIYA55PaoLnRbI3zXi3ErMXD+UwGKsTXtmSvlzRNxg70PBqLz7c/8trT/vk0pRuUivL4W0vUZpLua9mjaRixVAMA1WbwvoUTEG9vWx6KK3rPTp7/AD9mexfAzgSc4+mKbdeHteY7IrK2GOp8xabdimzCbQtAXpJft+KioxouhZ+5fN9WFdHFoetRKA9jbMf9lgae2ma2nSwj/BQaXMK5z66LoAHMNyB/tP8A/XqUaN4fP3LVn+sta32TWv8Anw/8hCql3LrFowVtNYsf7sIIo5guVxpOkjppyn/toKlWx0teP7PiB9M1k3F9eQ7RMk6EjOGiIqkdQZGdlkZXY5JKkVHPIq506QaOHCNZR89QCRSSNo4n8q10+NsfeZ+Rn2rj21AFiJZA59RVmO9ga22hm2/3RxU++Xc6W3utLliZ47GBipxtCVaXUdLQZl0Ozx6kZrlIJSG2xNsPb5sVcOoGF9jss3oCOT+NZcs0Fzo4vEKwODb6bZiLsCmcVJrN9Ya0IDJZLH5O7G1OucemPSsH7RFPAQu5W27gQucGpkvFMS4Vh9eKcJe8ZyZI1rp6PG8NmqyhwNxzQ8OnM5Y6ZG7HqWU5Jpnm5lt+GOXzwPQUn2qMuFB+Y9K6bmbZYEdon3dOtwP92lzbHrp9sx9TGKYkhfO0g44NO3MKpWAlE6AYWztwPaMUv2gZz9ng/wC+BUG80ZzQBN9qYdIogfQIKPt8w6JEP+AVFijbTAl+33H+x/3wKb9tuuzgD02io9tO20AO+2XX/PdvwAo+1XJ6zv8AnTdtGKAHedP/AM9X/OkMsxP+sf8A76NHSlyKLAJvk7yP0/vGjfJ08yT/AL6NLRRYBp3H7zMfqxoKk9Tke9P/AANH4GgCPaewFNVJ4pUmglMUyHIdamz7H8qUfjU2A0rLxLqtnnzEhuPQsmCK1Y/G3/PfTGx6hs1zVGKbQrnWDxlp0iZksLhT6BAaRfFWhMf3kF0n1irk8UoFFh3OwHifw6R924/GOj/hJvDvpP8A9+65LYDR5YosVc6tvFHh9ekdwfpHSDxfoidILz/v3/8AXrlfLP8ACKXYe9FgudWPF+jH/lld/jH/APXo/wCEw0sfdhuv++P/AK9cr5YNJtAosFzqP+EytQf+PW5x/uinjxrZf8+Vz+lcrgUYzRYLnVf8JnY97O5H4Cj/AIS6z/55Xg/AVyu0UbRRYLnTnxhbdoLw/lUbeMlz8lheP/wPFc5kJR50frRYDof+EzP/AEDLj8XFL/wmR/6Bs3/fY/wrnGljbvW1B4fsJ0X/AInsKO38Owjn65pWAmPi6X+HT2/GSo28WXf8OnxH/el/+tUkng+7gBYSvKg/iiUMKpNpcSHEl1MjejQ1Ng5SwfFt6P8AmG2//f3/AOtTT4vv/wCGyhH/AAI1ENKtSPmvpD9I6cNLsh/y9zH/ALZii5XJIQeKNR80yeRb8jGATUn/AAluqf8APCGmf2ZZZ/4+pv8Av2KeNNsiP9fcH6IKOaI/ZSE/4SrVD/yxgI91o/4SXUSc+RCPoKDp1hnH2i4B/wB0Uf2XZ9r2f/vkUc0Q9lIP+Ek1PusQ9ttA8R6l/wA84vyo/sy1PW9kP/ABR/Zdn/z+zf8AfIo5oh7KQ1/EWryRvFmNVcYOF7fnTIdQvLFBDCkewdNygmkvLS3sYo5o7p5WZtuxhipvsNtOqyPeyozDO3y84/WsuYvklawv/CQ6sOB5GP8AcFH/AAkOpjlUhDeuwUf2XajpfOP+AUf2Zb/8/wC//fsVrzxI5JCHxBqjdY7b/v0KcPEOqkf8sR/2zFJ/Zlv/AM/7/wDfsUv9mwf8/wCB9Yz/AI0c8Q5JFS0nlhSaaJgsjkhvlyOv/wBelGpaiDkXGPfyxSW8SM80HmYw5G7FWF0zP/L6g/7ZmuelNal1IS0Gf2xqgGPtTfgoFRtqWpP969m/OrP9lf8AT/EPqhFIdJm6R3ls59N2K6OeJlySKp1DUV6X9wPo2Ka19qEg+e/uT/wM1b/sLV25W3SRfVJAc1Wks7u3XM9tNHjruQ4p3BxsVz5rHLTzsfdzSwx/vjksxI6k5oEvYMM06IgXIz3FZV/guVR+Ii+zxjonI70vlgdqm6Mc9ATSVpF6Ih7kewelGz1FSUvBqxEXlgik8sVNijFAiIJTigH+FP2mkwfWmA+24c/Ss2+S3gvpGGqPbPIAzpjvWnB8rkn0rI1Rbj/hJIzbwwzeZCBslHBpjNO2cSWwxcfaP+mmOtNIqaMbYgDGkbEZZU6A00rQwIsUYqTbRipAZtpccU7FGKAG7RSYp+OKbigQmKPpTsUmKAEoBxS4o2mi4CbsGjPp0pdtGB0oAQnmjdS4FBAFFxibsUbqCKI4ZLiZIYhl3PFAhrOBSCZT0Ut9BXS23hu3jbdcO0jjtnitiK3toU2pbxL7haAOZ06ztHdJZJbiOTsvlbgP1rektZTjyYbaQerrt/rSXOp21o6+ZgKTjPpV+aD7Xp5NpKh8xflaoA5efTLS23NfXqJk5KJ0FVX1DQrfCRW7XLDuAT+tZc9lew3jQ3oKN/efgGomhCHqp+ho5i7Go3iIoP8AR9PRR2J//VVWTXNRuYvlniiz/dXmqE1v567d5Ue1IljKgItp9vswzmp5gsR3V3fwxmQ38hUddpqh9l/tGESmfJb++ea0CJSzJdCI46ikSKyYlYZGDDqgPSlzAZR0OVAWE6H2rRtCkUIic4cdsVYCjGFOao3LxI/Mbbx/GORU8wcxdfC44PPc1DPgwtkgcZp0TJLEpSQE46E0zdbykpKqkj0rPmDmKdpPH5X7yUK3oaS4kBJkS+UY7E1NdWti0R2sPxqqtvbSw+WkRJHqcVRfMaen37TQ5L5weauHUYk43YrnbO0urffsHyscgZ6VcCBhmRMSD3rNxKU7GhLqKhC0cgBHrWPHfNJdiSU5arSeXIwQck9qzZbOWGclSKcUPmN6W9ghtxNs3N9aLTV7e7ibEDAj0rDktbqSHduAx2JptoZrdTtcq3qpq7GfMdFZXTNeqhZVdTnB711smtwW9qGkfLD+HpXmSSyed5kr7vU1em1Hzk2sQx7FhzRck7mx1pbubYPlHrmt2NcivIrbVbizuxswR6EV29h4njNuHuIJEz3j6VakBvXN/b2kZd26dqoDxXpu4KyuM965LWL2S5kaSN8A1z7z3Qfy9ynPcUXA9hguYLlN8ByKSWEP94EiuX8K3jRRpFKOG/iz0rsOtFwMuXTI3fcoxXL+JdJ1AxBooA0fOWj+92ru8ClqkB4rcRta5wsyuP7ymqU9xM67Wc817jdfYRbs1zHGT2ygOa831C505r2TEUDxHoAtXcDjhChOSKf5UYHSup07Q9M1e6CgvCjfxockVrTfDJutrqZb/Zlj/rmruB528YHIFNx7g/Su5bwBqkDYbyZf91qnttDubBSZtJWRfUjIo5gscNBBPKSY4JHUdwKs/YZHHzQsp9669i9ux+z2jQoedi9M1UuJpblwXTGKy5wsc8unYHz5zUMtmA/AroyF25YcVTaEE5xS9qHKYRtz2FMMB7itxrb/AGaZ9nP9yq9qgsYnkEnpTvsZb7oya2jbo6/cANM+xsOeceo60/aoVjI+xOOqmopIGQ42k1vpEFGD831ppt4+yml7VBYwkh3DO7HtTjB61qmzXOdpHvTvsqj1p+0iFjBWKQpuVcilSNM5lc5HYUm5+zGl2nqaszGvHHksueB1NV97HqassCFqtjBoAkQxg42sT7VNlNmOQewqNHYDGBinjkUXAiLMTxUyxlQC5Az2FCKoOcU2TrmgC1DKYpFAOFqy0yuSQy7jWQHZmFSFkDh3JGBjiiwE0q5JzUcJkglVowc5p28zKXToKlhkVJQZORSHYtaixa3UH75GWrJjXK5Nac86ygqqrjsRVTaBwKEFiJnfbsXhe4xSYIWpGGDjvSFSaodiMqxXJ7U3HympGRh2puxtpBBGaRIWrFHDrWxaTxqzvIoIP6VmRJsHTipATk4pMs6C01Nd3lqcRmotV0OC+HmW8nlP6dqwgWHerBu7kj/Xv+dZiM54JIHMcikMpx9absFTOxZssxJ9TSbfStFIRXZcnbimGMA4xVlUx9aXySxz3qrgVUjYN0q6gBQA0ogIAOOtSC3PBzz6UXAjNtkHHQ1NaKokKDI465qVBg7QRk9qcAFfgAfSochEXkKkxHLIfWnPHGIzgKuOmRU/B60jQqxzWfOwKS2vAOCMHn3p7oDk4wo7elaKRF8RoMg1C8IViuRke9VzsdigxIAG0ACmq7qdyrz2NTmGQucLkU6RT5e0EDHaq5gKpLOcE5Y9B3NMdZEYh0ZfrU63k1qp+zoFlJyZDyajSS5u5wJm3Me7NTQEIyGzTyc0jL94fxL19Kh8wDvVCZYDYNKrYIPWoQwIpQxpAWt5PTinA1XVieKlAPUk0WGXbFd9/D7HNdrYR/KK5HSI914GPau1t9qQSzE/dQkCqEcBq05m1a6fORvwPwrPZqcXeV3k2n5mJqF84pDI2bIpqtzyads3d8ULC74CAMT2U0xNDWam5zVhdPvGDE27AAd6UWMxVSi7s8EZ6UAQLycVYRccHqDTns5Inx1I7VYO1mXERUheRjvSsUCBsdKsRn5aqq8zMFEO0Z5LHpVtYHY4SUZ9QKVgJ4cmVR6mmTOgnYMeRTVtbuKTzBMrFOQp70q2r7y8+5Q45K0WKKcsmVxz9RVWSQDgVsC1tY0wQ7/VqtQNbxWm1rdT9OKLAc15mTUyEnFdJFcWCoMWIB/2j/8AWqwNRgQfu7EZ9cf/AFqLgYMaSKmNpPuKv2ts8kedpH1qzJIt0oIYIx6gngVageCNNs92JCOyLnFZyQEdnG8cmx1xjv2NXiifaAAuT2FVzd20fCB2X++xxj8Kel3aO2RKGb0FJKwEpjXcfMU+3NCmOTPlrtx3xSSXtqCNwfn8aVpYvL3qxTBz61QxgfEZO35enIpwWQqXDk7RwhHFBuLYNwWaRhxkcCkN5Aq4E+GUcqoNADlWSSMsUXJ/hUU7YVXk4qNXVo1O4FiM/Kxp0DW4zHvLMxyd3SgBpVkALS5BOPlpfsMcm5rhmVQO3XNT/ufM2JIOOgppkU/flVhu5PagCJ7eF327T8o9qFs7Jo2G0xyAZ3EdaV/LBZvlXd6Hg/hQryyDiISADAIVjQAQRsJcqqspBAzU3l7f+WcY+nNOh07WJl3RWblfUnFTw+H9XeUtKixZ6kyY/TFK4FYgeiH/AICRSNtHAHJrX/4Ru4/jv4vqFOani8MWSsXnuJ5WxjrgfyouBz+4D72PxHFRytCRtWQ5Poa6waDpC9baRvq1WYrDS4lATT4QR/E4BNAHCSkR7BbZkYsNwKk8VoC1urtgttYyFB1IXGD+NdwhEYxCsar6KAKfvY/fYfnTA4xPC2ouu4iGP2eTBq5beD1Dq11e/L3SJev/AAKulO0/xD86TIHRx+dFwINO0yx0mJksbdIy2N8h5Zvx/GrRZgOtMyP7w/OmFSf+Wjf980mA8v6ml+hT8TUDRn/nqfyqJkftIB+FIZa3OOjL/wB9UGeQf3PzqmbGdzkSsD6MuKaNOvHbaJUHv0oFYtvK7jBRGHpkf1qpI6p0skP4L/hTW0S9LhgwlYfw7wBSSWWqW0butkhZRwN24E+nWpuVYqzWVpeZ8/TLZs9SyDNczrug6A0SfZN8M+ckRnKgDHWukk1LWvI23fh2B0Hcf/Wqkb21PM3hmAN6ncP6UKY1E46fQQ2z7LcMrZyRICcn+lRfZJrW7/0hlKx/MdpyDXSX4F3cb4YBZpjGxM4FUjp8cok+0XTPvOSU+UihyuKSM933A+U5+chRjjIrQlMENnunkQP/ALIqrMEt7f7LaQM5/vE5P50yFZb238q8TysNlGxxkVDRm0WbOWOTIjcsWO3BpktsyMzpujQEKCVzg96F26flIikgP8VPjg1K7R98U8Vu7BlkboopCsW3O3TRKiZ2AsRu6+p4pmlXDG8R1iSQMWOHGQePSi3tLjTMJLJbspb7w6kH1rRtfDN1c3Qms7ORNpBDlgox9DTsVYma/wBKkcpf6ebYj/lpbNjH1FbE/heJrVJLCZ2dxkCV+CPwFT2nhuysnMt/eCeUYynAHtn1rW+05I8lvkwAK15gscZLoupQNhrYlR/EjZFVtiQki4W5RuwVQP510ls1/HrNyklyghXLrH3Of/1Vp7mfiTDr6EcVXMOxxO+17NdH8VpS0DY8tJc9y5H9K6i50KxuUYpF5Mp/jT+o71yqrjP1IppiaFpVC7vnDY/2etLSjpVEi/6P/wA85v8AvsUYtu8Ux/4GP8KSii4CbfQYpMGn4pcUhj0NvjEtvNIfVZMCjNl/z5Tf9/6bijFMQ7zLMdLKQH1MxxUbnc5IXavYZzilopAJj1p8flhsyqzL6KcGmUooGT7rL/n1n/7/AFG6x/59Z/8Av7/9aosUUASFrTHyW8yt6mTIqKlxSigBtSRNGsgMsZkUfwhtufxpMUYpAWftdh/0Bk/7/NSfarE9dK/K4aq2KWmBMbix7aUwP/Xw1QHqTjA9PSnZFIaQxuKkUCHDTWwlVhwCxH8qQClx70gLEGp6dDKDNocbr6q2T+RrobHUPDku77OsVpMRjEkeDiuW2imGFD2oA3tR0XU0Tz9Ps9OvUPTYNp/nXPteXdqg/tTSXt/R9p25qWJ7i2bNtcSxem1jj8ula0fiO9CbLqCG6XuHGM0rgZ1tr/l4e2v3hPoH4/Wt2PxRqJjG5LWdf7zryaUweFtTgSa7srWDcM5VypH5AVkal4e0LmbR9YFtKPuo8hZf1qSjZGs6E+Te6bJb46unK/pU/wBi0i4iE9pq8QjPQPzmuHn/ALYt0dTdWl4jDHyNgimxtctp32c2sSse5b/61TYr2rO1NhaBsf2pbH8DQdPs+2p2wrkTf35dVNtEoxjIfP8ASrY1FokAcRnPoKLB7eR0i6ZaN/zE4f8Avn/69ZsNvbXGqzW6X0YSIAbiOCe9ZSarMz4S3iRv7zNx/KqViLnTbu5kuSknmndtRvun1xRYv2rOz/smxX/mI2//AAFaYdMsm+7qcH4rn+tY0upRxIGUrLxnCtzTW1O3MUbbSWYZZdw+U+lFg9qzQuvDdrc7f+J3bx7c8KhxV1dMthEqHVrY7RjOCKwU1S3xnygPq9PTVLd1z5Q/B6LB7WRtDTrMddVhH/ATSNp+nfxavB+CGsCXV7fz1VUO3+IbhQ2r2f8Azxl/AUWD2sje/s/T/wCHV4T/ANszSrY2K/8AMVh/74Nc/wD2va9o5R9QP8aP7Yt/7sn5D/GlYPayLtrbRx63LaPdKyZyJVXj8s1rNp9qOmrQD6qa523uUmu5Z19M4pTrNoe7/wDfNZ01ZtFyqyOgGn2n8esW/wD3yaeLCy7axD/37Ncx/a1oT0c/hTxq9oO7/wDfNa2I9pI6UafbDldYhU+u0irsUzwJtXXY2UdmGRXIHWbHH3X/ACqIazZE8RuaLEc8jtGbSrg/6Yunzt6rGYz+lYfiG10m0ntJtNZB5nyyqjEjPFY02qhwn2YbMHLbzwR6Uk19HdyRhIim1snnOamr8DCn8R0t1Bpdlbx3UcEN3uxld5zk9yKz2utGuZMy6dNbp3aF84/Cqkt4ozEqtx3IxVf7SOhUD6VdP4UTJanV2vhfStUgLafqUgb3wSPwrH1LQdQ0mUi4jZ4+0qDIxWasyIwaN2Rh3UkGtuy8XX9rlZXF1F/cc4P51oSYYOadXWWsnhXWP9ZGbOc98hOe/I4qpeeE9imSy1GG4XsGcA/nQI5+kww6VJLDJbSeXOjI3o1IB6N+lUmA0dagv5ba3kiurhZi6ghWjXOB71aA75ob54nT1U00wK9rqNvqKsbcudoz8ykcVNVXRBjSI1/usy/rVxhimwG4oxRg0vzZ5qQG4ox7U7J6dqTJ9PxoATB9KNpB6U73xmlBPpgUIRHg+lGPan7jkccUbh3U49qQyM4yB0NGPc1J5ieh/KgAHoMUAR7SO5pNuT9/9KlwO9G0UwIsGjBqTAJ+9z6UbRnigCPB6UivLBNHPC22SM5FS7aQrRcDp7TW7S7OJD5M3cN0/OtIRs4ymGX1BrhduDT4pri2bfBPJGfY1NgNjX9Dnu4cxDkHOK5m1v8AUdHlAWaW3cfwE5FbsPiPUosBtkwH98c/nVlvEGnXmEv9PJ9SuCBRYCsni15ovLvrOOdfbAqNz4YvsYWaxc9ecip00Tw/fMDZ38ls390t/jVW78IXsXNrcRXC9yTg0mhpjx4XSfJ0/VYZU9H+U1n3Gga1ATi08xR0aNgRVW40rUbXPnWcoH95RkUW+sX9rjyr+4THYvkUuUZXu7GdJf8AS4ZInPqcZqvBaxWjl4lwSMGtWbxPqzrsmninQ9VZOtVhqtq7ATaRatnrtbaTRyAUZJJYmMsTHcOzHIrOS9vJGYTK2CMY2108d54dOfP02eBfWOQkVaiHg6ZsebMmf7xIo9mFzhY4ryN9yxhx71YjLSTfNalCfSu2k0TSZG3Weqhc9AQD/Wk/4RW7KFob2OX0+XGfyqeUDjpba2izPIxOP4D0qa0nsp5Ni25ic/xg4Fbj+FNRB/eW6OO4D/8A1qrtpN3b8DT5MemN1HKTYoTXCwKYmfa30qissxb5XBGfSn3+lXE1wXmWWEn0U4NJb6fDFgG7fcOxGKOULGmYtsCyxqnnAggrUEyNI4nLbXxhkxxViKQgDJ4qjqixBPPEmHH901HKFiNzLO7IpwPWqy2l02dg3VB9pl7GrlvczQJ5rYPsDV2EZoeVHw/Bz0qWa5CxHNTyvbX8qmaB4/8AbQ81B9kW3Ygs0ydsU7Aa1lq9tcQCKeFXbs4XkGmT6pJATHHLtX6VkR3McE3ETj8Kln2XILKg3HqRVcgE5v1uDtmlOT/EKRjaRESfapCR0wtZ32d1cNjpV2GCOVMuKOQDVtdQjMY8qQsvrWlFrl1GoVbuZQO2a5tRAnCTkfUVPGwb+MN9KbgUmdQviu7j6zq3+8lTR+NpBIDJDG6915Fc0skRQq0hXPpUMiJwYyWH0pcozqtV8TxX9k0S2m1j75rhbkkzlqv7mI+VSfpUTQs55hf8qrlFci06+u7O5EkD4x2r0fTvElxdQrmNN5rz5bMjkRS/gKuQQyFAuHXH4UpIdz0Jr+9Yc2y/nVG61C6WPDrsz75rlo1vouUu5R7ZqUXOoE7XnLL6GsJMDTa+mC5yD7EVXlvGkHzRp+VU5JrhEyAp+tVGu5m++APpWTYEtyylcgAc1W8zB6VFLcfKec1Ua4Y9OKLAaW/K5waYDk1TS4fIBqzHPCzY3gN6GpswLBCkfdxSbARgGlDKw45/GjhjkdaV2MhZMGmnAGTUj8ZzUDMpBz0qkIUgnnGaiYntU0UitwSB9aZIgXJGPwp6gc6sLgcilcYOO1OVyU3dqaXzXaZEZ4pNgbtQx5ppcDoaaARkKnihSx60u4nrSeao+XBzTGO+YUjDPWjzcDHf1phY0JACjDA9xUtpCJrvDfcUZNRgc1akZobL5OHY9KYinI5+0MqKBH2FSlAfuHI9abNbvbxRs+dzDPNRo7Kox3pDZa3jAC8Y4pTLHEpLcufTtVXdgEiiJfPlwz7QOfrQItGNhGJnI55wKVTuOOBSswMioMkdABSvCEUtj5vTvSHcQ4jO4jPtTC24kkgVGS2CT0FQGQMMCiwFkOD0IxTgwxxVIHLAVaQEJnFFhXHMfSk38YqMyqOpp0K+e5VAScdqVh2FeBhtc4C03cvrSzQyo+1lcr9M0xl2LkqQPeiwWHDBOR0p4bFRxHdVkRKOCPmouFhFbBBBJ9j2qwqJ1IOfY1CIdhzkn6ipAHZchsfhSY7CMNpJ4pUeq0jlTgmovNfPBqeULF7zd3AGTT4tzN84wvc1HZlC7M5wFGc00XLyTY4CnoAKOULF1TgMQxCjpik5dfr370ZIjOG259qI8hfXPrU2HYVYQF+Y1EYF3E78fWpWWXduEqBR1yOlREw/8tLsFu5IppAV5Iy0m3cuPWq8kTKcKefarJFsvzLcgn1FIiCQ5U7h61pF2FYzx50T7lXPsan8kXHzLGQ/92rb2u/omaR4fs20INrEZNVcLGeYmV9vFSiBE4c7j6rWizQLGqypvfONwB4qNwiOFR9qEdcZNFxXIXJlx8gGB2FMRdrMck5PSrB5Uqki47n1pYbWVWLsvDd6Bo1dGjypcdCxxW/qLtb+H7l14JXaD9azNIj2xqtaetHOiCDs7DNUScHDu8kAHHJp3kI33jWwNJtwxIuRt9D2pxsLVf8Al5BPstQWjH+zccIT9KaqMjhkTBHQ1t/Y4AfuOfrU8VjbSMQyMABkkNQMxI5LjdukbgUwozH5fu+tdAthang+YM9Ae4p402IcBOPrRzAYSpIvcN796FSSR8YzW41gqthFFB0+Pjeoz+tPmEZYSRV2JDuI/iPIFHlz8MY2J/2BWoLbaD8m4DoKlhjeU7WJ2dl6UrjMMLMzjarfjVhYLl4wHQ7R0CmtdowyhcbcHOAacYEK4Vwh7k07gYwjdDjyCwJwc064RkdUEDgYz61rbI4iFEobPUCowttI7MxYDod9IDHEEzHAicfUU82d2py2GUdhWyDAWZEfGRglulQqqLnemWP3dh60XAzDp1zIodYTs9Saeto6HAwCavSLcNgW6yr/ALO3NX4or2ZREunT7vaMipeoGd5DxkpkBh19qjFvJknIY+o7V0A8N6tIf+PeBfaSTBq1D4JuGBa4vlhc/wAMPzAfjTA5YwMqkyOT7BaZ5kK5Dxu4POD613Vn4P062yZZ7iZiMZLYq1/wiuhnmS0Mrertk0AeeLMgcEwDI7Mc1OwN2GKw/vD0MaGvRYNL0y0x5FjAmOmEq0SB/q1C/QYoA4G20e6ukUDTHVu7kgCrcPha+bBeSCA/7Z3EflXYbG65J+pprRM3ap5gObbwpv4k1FWXvtjIP86vR6Bp0C4Fusv/AF0YkVq+SaPJai4ynHY2kRylhbK3qFqwA54Xj2AFSNGy8YOfSmYcdVNTcBnlFj8xY/jTvJI6BqcN2PunNO8yZeimpuAwoR/C35UpRh/A/wCVL58pUkg5HalE0pH+rP500AzH1/KlIQ/eanFyoywOKha/gWNnknREXrkincB+IP8Anqo+ppViib/lon/fVUZNf02NSwLSkdvLqm/imCRSsGmbz3YnAWncDd+zuQfJG/HXBzTksrp0ztI+tcVPcz3k+VkEKd0hOBSb7yP/AFF7NGfXcTU3A7P7HKD8z/lUosnI+9I1cPv1dlL/AG+R8d9+Ks2994hgg3Q3Lsn+0c0XA63yWi6xk/UUee44EY/75rmE8SeIYWz+5c+jinQ+Ltblfa9jAy+4IouB0xuXP3mqMzD+8KxD4luFVZZ9Pt2jzhgn3hV2LXbC4bH2K4X64ppjLn2gZ5wfrUgukxjYfwFU31LTl5aOdB67QaZ/a+lDg3Myn0MRpiNBbiEfdLLn604zKf4zVEanpmcHUIzntg1KLuyYZW9gx2+agCfe7fxxv/suKi8vzOtpA5/AU5XtmGRcQH6SCnAKxwpVv93mgCDyY0PzadB+CimSWGnu29tMt2Y9flq4I2PG1h9Riop57W1GJrhVb+71NS0VYrPY6XJG0R02IBuvv+lMfRdJWFfMiYMgwAZSOKsxm4uyVtrfCf35l2irSaRb+ZvuWaST+6vCj/GnYVijpui2CTC4NqDt5QSNu/HFaN/LNHBiHhmIBYdVHqKnZ44kLNKox2zyaqtczSOWVQV9TRYLFC0s1CmSTcGP3nkPWrRmjHETbm9R0oeAztmTLj+4TxUc97Y2S7JpERx/AvJosJodBCsW45Znc5ZieSafNNBaqHnmWNewPU/hWDN4iuJBi2hWEdmYZasl1eV98jF29WOTTGbF9rhuImhtVdEPV2xkj29KyT1oAbGKAD3pohijpgUYNKKWqEJilxRRTAKUUlLxQAnNL0OKSigoKBQelN70EjulJuxRRigY4UuPWmg44zS0AOoFFJSAdS02lpgFIaWge9ABRRSUgQ7vRnmm80tAC0optLQA/ilzmmc5opAG1Seg/Kl2KP4RQDzgUZ5pWGGAO1NIUngU6o2ByAGA3HHNIAZRnPaoyBV2XSrqKJneSI7RnHmDOKSw0u71OIyWqLIo6/Ngj60h2KHejFb/APwiOqH/AJ4A/wC//wDWqnqPh/VtNtTcyQpMgOCInyR+GKBl/SLjSIrVIr7R4ZwPvSjO4/UZ5rqbPTPCt9xBaWu/+43yn9TXl9nr6xyLHdQtH7jtW7b3UN3EJreQMvZh1FAHW3vhiKJibfSLFx2U5BNYMwtbV9lzokEL+hBq1Z+JtTskCbxOg7SdfzrobPxJpuqIIrhUhkP8E4yPzxUFxkcn5lif+YTaH/gNM2Wh+7o1sfpxXU3nha0ncy28rW7joq/cNYN1p9/p5JnhLoP+WkfINSbx5GUQtuvP9i234uaUNCORo1p9O9K0u8/KwLehOKQ+h+93o5jX2USs25L8XMVkkIxgxgcH1q0s8J+/o9r9ccUFCOtPVOOorLYvkiSCaBcMuk2ePUjNDXK/w6VYj6LTAPUZHpnmk2E9CPxq+ZkeziPNyvX+y7IH125oM6OOdLsm99gppj55JHsRkfnQU4xj8RzRzMPZxHrJABxptkfqmKqXyC7VFhtoLZ1OcxjBNTgYNLRurMPZxEjniECR3NhbuyjBbB5qTzdP/wCgZCT+Ypm1TxkZ96Ni+lTrsh+zj2JVaxGG/su2/UVKJ7Mf8wq2P1FVQoFKCB1GavmYvZQ7E7SWDHnSrcfQkUqS2CHI0uP/AL7NVjtJzgijI9aOZk+yh2NWK40GXK3Wlsuf4t+f0p58OaJernTtRkgb+4/I/KsfINAI7UczJ9hEtXXg7VoRugWOeP8AvK4B/I1kSWV3avtuLeWP1LKcfnWnBd31q+63uGT2JyK028RXMtlNDdxRyqyEHHBNbQmc7oSRxtjCttE0YfcCxYcdM1YfBNUtJ+zyapbx3m77O7bWKnBFdpP4Qsbhs6fqOxh/C53D+daSlczcbHL96Q9a15/Cerw5KKk4H9xgD+VZU0E9q+y5heJ/RxikmTYYaTJpA6E4zS5HY07hYATS7jig0lAh2aNxwKb2ooGP3ds8Umabml4oELRmjNJQMXiim5pc07CHDFIetIDSk0AHHpzRx6UmRS5GKQwwvccUhSM/wCl7UZwKEAwwQn/lmKepkjOY5ZFI9GpD7UA+tMC1HqV5H/y1z9RmmXMsN7/x82sb+44NQZFGRQBTn0TT5STG9zEfUNmsbUPDd5I2be5STHYjbXTZoyKAPP5NG12B8JHMp/vI4IpWbXrSIedC8i+pXca7/NGaCbnnA8QTqxWW1iPuDirkevQjnfdR/wDXNiBXZXGn2N0CZrWJj67BWbJ4V0l8YikT/cNAXKMHiydGBi1F93ZZV4NbFv411IJtJtpPwxWW3guxd8pezxj0KCoW8DEcJqsZbsHQj+tAXOmTxnk/vbBwPVGBFSrrXh+/k2Xdqw3d2TA/SuWt/CGsW0oeKe1mjPVVkwT+lXH0rUICPOspV9do3CjlC52kPhPQ7yITW8IZOxjlrEvfBywS4SRkU/8APUcfpWdZz3eny77XzofVShxXbab4os74CG48uKY8eW/8RqJIpM5U+AJXYO1xHsP8Sg4pX+HqOAJNQwP+uddNqN69g5MFrOQeqrhlp1trtjdW+67gmhbuXU4FSFznYPA8MS7DqrMndfLqdfBOmfx3Mx+mBXTRrbT5NvKjAe+P51J9kPdc0Bc5mPwVoEbb3jlkb/aepD4Y0BT8tn/48a3jHEpwRTdkWeBTAyRoOk9Bp6fnUcvhnT3bdFbrF64Gc1ucAdKTznHSgLnPt4YiX7sYP/ARTB4fiT71sn5V0XnPimgs33qq4HNvpEef9WKa2jJ2lRfwrpTCh7Uvkxj+Gi4zmf7NCH/Wqfoop66aX6/N+FdLiI9Uj/KnD7OP7oPtRcDm/wCzAByjf8BphsYu4l49q6cywoOZVA+tVpb2yXrKhP1ouBzRhtx95JaidbbPAcfWugkuLZlIVEOfasy8t43O5IgD6ispRuBiXRRIyFIIrGuZAw4rYurdmztUg1zl00sEpWRSvoexrP2bFcilfjIaoRK2CV59qqSu2/5cD1xVf7RIrjBxW6psOY1ku4ycMpU1N5ke3cMMKyBOx+8aekxBzmj2bA2UkUjKHmplmI6ms23kWTIPBAzkVNu/2ufQ1m6bKLjzg9TVZpgHODUQRyeXFMmjKru3Co9mybllWV1DKRmmGRwcORiqUZYsMH9ae7sRhiDV8jC5AikqM9KjkRCcKSPxq9thVPmlAFR77ArgByfWugRRkEaDlwDSCGF1DbyT7Vd82zB+W0Dn+8xpyzRY+SKNB9KaGUcxJ98Nt9qfJLbyRbIYGLepHNSu7S5TG1fXFRqfKOEOT60xFYWlzIflherEOk30jAKir/vGpRPKv8bfnQbiRhy1AWLEGiyrKDNcQqPrU7WSxSrILiJlGc5FZgfuU3N65qKWZn4b7vpUiLNxb2kspeXUBz/CvQUxJtJtWwGacj1FZkoUvlRgmpBZyCLzZU2L9adgLt3eQ3Me2OIRj2GM1ViXY4Y9Kh68CpYxvIi9aBF+BVUiUN06A0h2/aVZtxZuuRSrHIkojXGAPWpYonurpFUAbetSxotLp0M12qOSEPYVXuLK0hlKraNIv99W4rQubeQZkXGR0warpI+QHk2n6Ui7EH2O02ho4nRj/s5p6W0atzlvqMVovKyRlmZGA61XUrOm4Jg9zVXJaK7RAfdjTPrimA7AWX5XHUgVoW4AfBt5pMdQBjFEgQyYSB1H+1U3KsQw3Em3cT8x61MHFwQJUQgdiODUq2i4+cN+FRPHFG+Fjl/Ki5VivKBK4j+zxRYPUCnLEo+UEFh1xV2IZTDIcds0pZIvl8iIH2NTcVirFaRySFW+Ve5qO5tFRf3MybO1WWx9pGfuH71TCO0J4lYD0CZ/rTuJoyU0pZmAeEuT1bOKU6VbBiqo4A/u1rGWzX5Vlcj0ZcUo8nqrMPpRcdjGGmhEdRujVhgs44FTxaOsSb/ORyOvYn8K0G+Yheq96XYkak5A9s80XCxTe1t1tkknimJzxsHWoPNjaXcsMqjspWtmPbt3P+FG9WPylB+hoCxmbJpOVtZtvrtp0VvC65eINn1rVb0JlP0NQm1cvmORVH91utAWKS6fAjg+WG/2D0NWSidrOJR7VP5EucsYz9KkNuREH3qc9hTbBoqpb28jBTbAsey0sml2oBLQYPp3qdUlicOpKkdwaWZhJIHz83Vs9zU3FYpm0tY1DLAXcf3iaPslvL8wt13dlNW3jJYMSoUDtQ1uJIlbzCCx5xVXHYh+znJU20YHqAKXy4kQBIFyD1xV14lVVEaYboSz5J+tQs7R7tp7c45pXAbZrmTPqavaiEeJIiRkdh1qCwHzAUzUZCLp9oPGBkVozJfEQJbQBhvi+metMlgtAT5YcsOopssrAAMrEt071Mph27fvS4yeKzNBpbKhVRRxycc0slrCQWEuSOi9KGMUcnluSysAQ68YpB5BkAPmeWx+Ygc4pcxROY1DwSByV2cqOM0svzNkEqOwFSSwPO8a6fb3EigYOU6GrK+HNVaRd0exe5dhTuJozVDZ5p7AAfKOa2I/DFxuJl1CKH1ATfVmPw1Z7t0uoSyH/YjAFFxWOay6+1SJIAcs+fYda7BdK0telmG/3uanSC3iTbFbQIB22incZxPlSSyKFgmJY4A2Grsel6hLIUitCV/vPxXX+Y/qn6UbyRyw/OlcDl4/DF5LIpneCBR3B3E1aHhGy3gvdtgdkXr+tboOGzkfnSlgT1H507gZY8L6UvaVvq1WYdH0qDGLCFsd3GTVveP7ppwkA6oT+VACxNFEAsSRRgdNq4qXzn/56GojNAR90imF4GGPm/KpAm5Y5aQn8aXH+3+ZqvttR3x9aTFt/wA9do7470AWc00saixZr9yc59zT/NgH/LVPzp3AUv603zVHU0hmjyeFI9dwqPzlJ+4T9OaGBYFzGeM804TL6Z/GqckqqufKLAdgtVXvrcYEqvFnpuHWkBqh89KTe3YZrKOpafj/AI+x9F5NV31+2hIEaSyE9mGKANnz2B4604T3LdGrnZvEdwIWaOxjX0YuCfyxWb/bupXUeZHNso7ggCpA7GSWeNd0syIv+2RVd9WtY/8AW3cUntGMmuNcJMu/zHkI/iLk5p8ks0sQjLRZHt0pAdDceI4Ism3tpZMf3vlBqofEN5PgqLdAekfes9BmJAeGAwSvGaFVfKKkDdnKsT8wNNAPnuZ5dwnncBuNqtx+NVdij/WKSD6nBqZlVsKQRk4yRSxzbFZpAJB90H1FMCNihAY7vl5+YUoYmLlBgd+9DPuAGDgtwOooVoljlDtuYDHA4FDKFErYwoAApRMwBZsNxgDsKVY1MTFiQfQUiCIZ3Nj8OKVhhHO0fDI3PftUxurqHgJ5KH26ipVmsiGjZEduMZNSW8ksxVHtkaIHO7IA/GiwrFMzf3ywJ6EjFTrLO0QLWrSpglGX1HrV26uZYZcXnliJj8qqdxxUC6t9miU28ZCp93ccc+pFArlaRvm3PYuM/wB/Ix+VPDWTdRMg7FAef0q0PFU/U28LMOhDZNadvrGt3qYi0lAp/icYH8qA5jnH8tHIErEe5qJ5o1/jX8TXY/2ZczfvL9LAN22x7iKtrbadCgH2O3JHRmjBNMVzjbeylu8+TZyykdwODWnD4RaXc96Vg9FVgx/Guhku+NsWFXsFGBVYyTscbjt+tAEVpoGjacocRmV17uxIP4VdF6o/1MRAPcLioRBuHzZ/OpR5UKhSxHsOSaAG75SeeKTy1WQXDQxyOvcikMzyfLFHtz0eQY/Sk/s6J3LytI7HsHIH5UMdxz6msisRC/HWoGmuZCNmY19COatm2htl3yFIYx3dsCsm68SWduwW0j+0vz8w+4Px70gLqQknPlruPVjyarXes2lgxjZ/NmH8CD+vauavNR1C/mBnkAiHSJBtUCoRGg7Uybl+61y9vEKArAh7Rjn86ohfx9z1pQFHanqM1Y7htPoaXaaf2oxQIZg96XinGm0wDvRQaDQK4UdaOtHWgdgFHWg0UEhSZpc0mKACjtR2o/lQMBSE/MRS0nQ5pgKuCaXOKZgd8+2KcCMdz9aAHZp2aZilFIBwo9aTNGaYC0Y96SigBe1FFFIQbsCjPGaMeuDRQMM04HikwM9KMDtQAufrRmgACjigBO9KSKcgg8sl5WV+yhev40wZIyaBi5IprZPBUH8KdS0gICmOifpTojOhMkLtGy/xI2KfSEDORmpA39M8WyRqsOoLv9ZxgN+NdZBcRXMIlhlWRD3Vs15ky0+Gee1bfbzPE/qpoGjt9a8P6XryZuF2TjO2WPgg+/rXA3fhHXdCZpLR/MiwTuQccD+Idq6zT/FKysEvUVH7TKOM+/pXQLeZQMJAQejDkUhnlVp4oVvkuomV/UVuxSQ3Sb15FbereGNK1OdLr7NH54PzBTtD1yWtaHeaS3m2enTJF2kibcPypNCR0Flq19pr4hmLRf3JPmH611Vh4ssbo7LhDby9+6V5Ja+Kf4buL/ga/wCFb1veW1wB5cyHPTmlYdz0a90DT74B40VG/wCekXQ1zl54fvLRTJFieIfxK3P5VmWmpXenyA28zBe6HkV1Gn+J7W4CpeKYZT/EB8pqbGsakjlxIo4fKt6NwakHzDIJ/A12l1p9lqsW5wkgb7sinBrDuPC1zAo+xzCb1Q8EVFjohUMlVJUkMp9u9NwegY7vRqGZlYJcQmN14yBgilzuXdu8wZxz1pGqlcCZF56H8xSB2PPB9xShgrDYxVvRulBcg44U/wCz0pDFVg2QCpPuMGm4YfeKr6Z5FGQ6/OVcHp6inIcHHmBc/wB7pTYCYY84BA7qcijcR0AP0NStFxnp/tJ0qNlfbtDK/wBeopAAbnlR9G4pxbDD5P61GCw4yFP+0OKQ7ge4PqvSmBKVznC5+hpg5BAKA/7RwaQ5cA5DH34NKPl5cgf7wyDQIZyO/wCGKkwzAFV/I1GTzxx/u9KUZY5+TPoeCKYiX5iMjH580oxg5HUYwRUWWDfMcL9M09Tg5U5Hop4oQHLwHbcD0En9a6p1wd6nDH+JTiuZlUx3koPZs/rXRo+I06fdFayMIRV2mW7fVtQtsBLhiB0V+RWlF4n6Le2iOv8AeAH9aws80v1qCvZo2mtPDOqY3f6LMem35arT+CS/zWOo719JRkn8RWdjI4ojkmt33wzMjf7DUc5n7FEN3oGq2XLWjSD1jIb9OtZm/acSBkP91hg11dv4hv4cCR1lA7MMH86ttr+n3a7b2wz6kKGH51rzmXsWcZml611x0Lw/qHNtN5LnsrYx+FUZ/Bd4i7rW7Sb2YY/xqlIiUbHPYpatXOlanZfNcWUip/eGCP0qnvp3IsFLmkBowDTAXOcDvSUds0lAgpaSimApPNHTvRSGi4xc0A0lFIB2RRmmYo5piH5FLx61FzRk0gJuKSotxp240Ah9FM3H8aUPQMcR3xRR5nGD0pMj8KYgNGM0vGKTdQITafofapFnnThJ5VHoGpmaM0h3LA1C7UYMgcf7Sg01rtZDmayhk9eMVDk0vFAXNK11iK3XaLeaNfQSEj+VXF120bgsR/vrWDgUbQKAudDv0y4+bNsW9A201OlrFjMEjr7LJnFcnsT+6KXYBypKn2NKwXOvCSrx9oJ/3lpP9J7G3f6iuWWe5T7t1KP+BVKNT1BOk27/AHhmixR0m+4H37ND/uS5p3noPvW8i/hmsCPXLsfLJCjqe2cVONcA+9HIn0NJoRrNcW56vt+oNCvC5+SWM/8AAqyptVEqAR3pj/30zWVewX17zHfWLE98FTRYDrjER1ZfzqpPe2kLFZbhFPpmuNbQ9e/gdJU9EmHNUnivbVytxbS8d8ZpjOqkv9Myf9Ik/AU0S6TMoP8AaDLnsQa5MXcYPzBl+oqxHNDKPvqfxoEdUul2Mv3b1G/4FikfSI4m+WEzL2KuCfyrmPLjPTn6U5TIn3JJV+hoA333xNhdOuOPQVSl1hoXKtp9wPciqseo38P3LuT8Tmpl8Q6ovWZHH+0maQypP4p8lsfZCfwrE1PXI75SGtAnua6v+3lkH+labaTf8AxVd5vD86kT6F83qj0WEeayMHYkVF5L5zjivRP7N8IXBbdDc2zfXNI3hnw1MmItUmj/AN6tLisefCMntViO3buK7H/hC7Rj/outwufRsCh/BWqRqWhmt51/2WFNMDlo7Z0cNj5R1qX6Vrv4d1SN9rWbY9Qc1C+nvbuRPA6D1xUsZnb3BqCeQv8AeNXrgwLEdgO73rMmdC3DikkIRJdhyRkd6V7lGGBG2PXNREZ7j86QRjuwFVZAQKQWGKmCs3QVabTRFIQZt0g9BUBSWOXaeKOZARbJN2DxTgxQj5c0+RWQjG4+uRVv/UQh/LJJHpSckBT3Ss4zGQPfinO7b87Pyp/nycloSB64qNLgs2CjfgpqUxibpGP3MUwpzk1djVyCTCxyO4qH7NOx/wBU4+op3JIUPPy05rWSRQAQufWr0Fi8cRaRetP2H+FGb6KaLgZ0dhGjbi+8juOK0fIeeyZY4PNx14+6KeI3EDlbaUswxgKataY93b2nlvbruJJ+7U3LMG3tTJKAkbH68VIunSx3oyOtb8VjMZ9zBkz6VcFhFuDSAsw7k0rgc7c2M9uRLIuFptvBIbhBj7xxW7cWkEgIdpGB7Co7WxCAbAeKdxWHPazqSke9z6dRVDUojBaGQrtkBANaYNw77VuAn1FJLYm4XFz+99yaVzSxBp6R3GkRmRC2c5xUE9yLf91HGVUetWjbIvyi5aH/AGIzUMlhHKMbpn9ycmkyWia3vHuDsWVQMc5qwsibdrSBselVP7Lt40yvmbvQnFPitTDtkdMKffNNABuVRwqBtxPVhSEu2S0wYk9qvSKrIuY4sEcFzT4xGkIRfJDd9vemBQRFJw53L6U7yYWnIjiP4c1prpls2d0/zBdxCdazkScqBGj7f7zHikBYRTGgDx5HfC5NPDxrykeD7riqqm6dwqz49c8AUyO88u5EbyKzDqxWqAkKoAwKd+pFRkL6VcuPLliZdwyemKqJH5YG7JUdz3pATrJFIT5cewgdTUTxhwWkUE+o4NTNalVMjISCONpoJWIDcrY9aQyCOH5cbRGynv3q2QCUJEY57UojsC2bmYrxkKB3pojZjvT7lAieC4MMzEorJjG41A5s2ckl8+i9KYJkDEE/KOtQ70eX5BjPegCUugACRvwerGhZGdtijyx/eJoXyPLw0rk1A06RkY3P9KBl5rTjJulGfWoxZqykSTA+hTmiAfaBlbSct6BCaujStRuAgtrZos/eMq7APxoEVVihhXDzz7O+BQxt34VJDH2xxWtb+E7k/vLu+jGOyknFXE0DTgoWW5kkPf5sZoA5V41Eu1S6x/3i1E0cMSgrcl2JxgV2o0zTO1rC3+8Mk1neK/Jjhs4Io0QbtzBVx6VQjIsxhwAOTwB6mrb+HNW81n2xjcc4Lcio7GLzbmFRxlxXXtBIR/x8JVzM1ucv/wAI1cD/AF92iH0Xmprfw5aJlpLiV37MgAxW8beQ/wDLRG/GlEDrzgVBoZ9tplhacxQb2HeXmrecfdiiX6JU3kue1HkP/dqQI97kYLYHoOKZt9SxqQxPjkU3bJggK34UwGbOeV/Onqo9Kbv2n5g2fpTS/PRh+FICY9MZoAWq/nqD0yfpR54J7gegoAsjB70o+U9apifB5HPpQZSeQCPY0AWS1N34qp5rVG1yVOCDTGXvN+v40hkYjmqH2iQnG0/hQBcyNhVOPc0hFwykdQn51G0+Dj5fwpY9OvG+bIx3zTTbWsB33t3Gme24UAI0rt05pgLN/wAsyfpTW1zRLaUC3Jnl7Kq4P61HceJ7mNQsNosGehl5oGXU0+4kYDylGe5apW0mKP8A4+btIx3GcVzU2qeIbyQx293CmeoiABUetUrmeZ7ryLhrZ2UfvGZt280AdS2peHrMiMy+c/ohLZofxPao2y1sTjsH+U/yrnlESZkS0UsvPyDHHrTFmeWZWLASDrg5xQBq3Wu6rdAhEVU/uqMVlOs93N/pbNgcrnnBqUq0cqyK53ZxjqaY7TJOXZ3hx0Z48DNAEMNqFlLebIo7KOP/ANdTFyyorBGkGF+71oW4V3BD78dRjvRcPLudV8s5GMgcipLHSxokJf0p/lLNECBuJGSagJCXWWBeFBkA9yakjuJyWWNCiuME9sUAMEXkqW242kjb3P4U0TrvwIGGfRasNF3LAN3Jq9DbTShZiyqijG8nkUCaMtZP3hbY/AzjFPe5j9ZqsiBpd8nlM7ZwB2b61OyGNkcxrCQMAyAkGgVihFFLdXEUVsHffj5m4AqxPYxx3bWqXE0kyjLKIwBx15rQhhuZ5dy3UO4AOBH1PpUUmkTsxfMiShtwJ6rQBnXEEkE3kpayvLt3ffAOPpQlhfuoY2cij0Ygf1q4sJV3eZxv5XcX5Ue1TWIBdUG6XPJkJIGPxoGZBLJ95SO3NSWN9ZtNLBOrmUdI2XOce/at+9v7CFi0kPneX9OvfrUFnrP2j/kG6C3md5MdPxpjJtIt4WyTYpGwGQzDdn8avXkMEcTo6L7EDAApUstZlX/SLxbNB1WIZb8CKsW+kaXBhn8yeT+/MxOfw6VIHNiyubt1W2tzckDG9nwAe2T+dacHheeVvM1G7BfpsiHb0Jrae/iRikZUgdNgqD7TO544pkti2mnaZpiDybSNXH8brk/rTpdU+Yqnz49KiuC20M7EjODmo2SKNDI5CoOrMcAUCEaWeVicKM+lIYnY/PVd9YsYJ44VV59wzuhG4Cp1u7qfiC1KejSdfypgSiJUXcxAHqTiojfQLJsjDSN22jj86abR5mDXDNKR2Y8CrC27KMKgUUgKxN3OuHk8pD2Tr+dPZYrKESLG8jA465PNQ3Wq2NiAGl8x/wC5GMmsC+1m5v1Kxb7ZPVThqaA3bvxHplrqS2bXCl8ZYj+E+lVLvxQwYrZQNx0lfj9K5lLaNMsqfOeS5OST71KN1UMdPJcXj+ZdTvK2c4PT8qVFUDGAAKBTgtCQg4oxmn7ccEUuBjmmK4wbak7jAwKABRxQIOKWm0UxgaKKKBBRRR3oABR2oopgFHaiikFxKWkoAoAKKKKAEpKWkxmmA0sB1pwNMI5pykA+tAEgPrR346Unf2oBwKQDqUEfjTc0ZoGOPA5pe/FNzzjtS5xQIXvR3puTmlPJoAU9KFpAe1GeaAHd8UUlHQ80AHQ5pabnk/WlyKAF60ntRkdqM0AhaWm0vFAwOaXtgdaQdKUUgE5ORTNhzTz1zSmgQ2WDytvzq24Z+U9KnsNQn05swNlP4o35Uj6VARSbaBnXabrdnflUB+z3B6xSHAz7GtxRjg9e9eaGNTyRWrp/iS7slEdwv2qDurfeH0NS0CNzWPB2k6uGk8kW8zdXjUYz9K891Dw1q3hu5E7wGe3XP7yEZH5dutep2Gp2uqQeZZzFmHWJ+GFW1fGQVDA9QRxSuM8gsvEqOdlwjBx3Arft7uC5gDRyKwPpW/qvgnSdUZnjgWCdudynALf0rg77QtY8OTZMPmAnAkiO4EetDA6mHVLnSV86GdljHVTyK6jSfFtleov2lfKc/wAfY15emr7rSSG/i8pzj5j0H1rR0q7tpbFNswZwSOnWoaHc9YkjsNTt9rJFMn94DkfjWJeeFQWMllJjP8Df41y1vdT20oktpSjenY/Wti28bz27+Xf2QdO8kfGPzpWK9oUpIrmybbdQ7Qf71RK8bNkMo+tdtBqWn6naxyRyRskmcLIOaz7zw1BNve0l8huy9V/Wk4nRGtfc50LDjJUB/UU3ac8bWq3No+pQHbJCZfdDkVVeIw8OksT/AEqWjaMkxv3OFk2H07UpXcdzoG/2lNNLP1YBl9e9IGTaSs2w5+6TSHclRpAMpIuP7jmnAHq5ZPQr0qAPD/y1QsPUGmmVSuILhSF6qxoAt5foAso9QeaiLyAnb8ns/OKi+02qg+bLsb1DVHJqUHQTiYfTmgZYUdWdfqyUnD8BRIfQ1VGr2g4/eR/7y5o/tW2z/f8A90YpgXcFOMMi+hPFIIi3IVGHqKp/2vajjy5h+GaUahGfu2szfQUAUL9f9PZR6D862IyfJjOwjKjgc1h3pnnvC8cTLgcg962bKQTWwZkeBh8pXPBPrVGK+JkmGYEYU/Q80oUL1LfjS/I/UqxHtg0YxwGYD/apGg9enyg/UUZJ6lMe/BqMKp5yM+qnmjg/elJ/3xSG0P47swHcDkUYXsobPvg0zaOoBwe6ml5yfnH0IwaYhzKRxs2+5/xqxDqF7bDENw+3+7nNQDIXqwHscikGOowR70EuJs2vii4hwbiItj+LbV0X2g6n/wAfEESk95F5/OuaJboc0zy0b7xRvxquYiVGJv3HgvT7hQ1rcywZ7FgwrHufCGpWozE8dyvqDtNRKZIuYJpE9PmzWlb+INQgwHKTgf3utXzGHsWc3Pb3FsStxbzRe7oQKriRG+6wNd/F4lgkzHd2rKp9twphsPDWqsQgiV++xvLP5VSkZyhY4XqKQV1Nx4IdP+PO+BX+5IP65rJvfD+q2CGSW1LoOrRHdVXIsZtLTQwJIOQw6g9RSnFMAzRSUUALzRRmkoAKKDSUABoPXigUuadhCUUZoosAmecUucDFGKTFADt3PNJmjHc0UALmjdTc0ZoEO3Uu6mZooAk3UbhTM8UdBQFh+RQDTRQOKAuPpOabk07cfWkO4ZpKKKdgQYB4IyKaI0/uinZzS5H40AIBt6Mw+hqRbm5ThJmx6Hmo6KAJXupXGHitn92iFV9lufv2NuT6qMU9qbikO5HJa2p+7BJEP9ls1EbKH+C7df8AeWrNGKLAVTpdwf8AV3Vu3/AsUjaXqSrkRJIP9lhVnavoKNo7ZH40AZzwXked9pLx1wKjE237yun1WtYtL/z8SAfWnrPKBguHH+0uaYjFM0L9Tz7ik/dt0C1tOLZ+ZLGJj6jioHtLBzn7K6H/AGWpAZbRRH7uR+NMEbocxzPn/erS/s21bpNMnsTmoX01x/q5lb8MU0BXW71FPu3Un509dR1FfvT7vqKf/Z94vRQ/0YVBLBdp962k/AZpXGP+1xP/AK/ToZfXAxSiTw+5/f6Q6j1U1UZ9v3kkX6rQrwP95qExF46d4TuAcJNDn601fC3hub7upyL/ALxxUSpCP/10/bD6frSuBXit7OFMK8ef7x605Y4d2RsI9cVnylBkbAKIk39CQPapsUamLf8Ai2flSgW+cjafwqg6SRru2EgdqsW0LuhaQBB6dKRRZY2zcmNCfpTHlgjGTEmPZarlIEz505/3RUPmJMCEDBQe4oAti8tiOY1/BaQ3Vt2iz+FUzEQAFU596VRsPzoPyqriLRv7ZfvRUHV4+ixYpiiDaGaNQO9KsdoJd6RFvrUjaHLqu77sRP0FP/tCRvuwH8RSNfO2FjAVfQLSG1eZTKGH0oAf5903/Luw/CkkmkiwJcKSOB1pqwykfNM+fTNV5YpPN+7vYdMmgCYXTngKp98VEzTMTtDAn0qYRt5a4A3HsKRI3TJljIPvQIgaFsZaTD/WmQRtM+yWdwtXUhFwCd6J/vGojARKUjuE96Bph9jgUfu5mLe4pYrG4d932xowO2etNEThwyyvn604RXLcNISvoDQD1JZFAbbLIGHtUUEaq25mO31PNHkeX99CM981dksmjsA+8qOwFNAVjEv39+5fp0pnyK+FH5Co/wB+qhXUsCcdKlDTRMDgYHY0xWLkBIt5GVgrtxn2qAcLtEbbvc0kkuYdxKhs8gUkd4g6ExkjHzHNSUNEboNzoM+lOjmfb8sSAj1XNJhpHIjYyMf7oqZLDUpTthtnOe5GKEJoi+dv3j9Keis3zMuYh3q/b+HNXkwk3lxwj+83I/CtCPwxbpxPqDH/AGVAAp3FY58srjmdRj0NQiVVPzsG+hzXXppOjQqB5LP7sxqaNNNg5gs0Dep5pjOKdZruRfJgldR3CGteHw5fypuSCVP944FdGL2bGEVEX0UUhkmk+9K350hGGnhWdv8Aj4lgiHcg5NXI/C2lLt868eQL/CuADWhtJ60eWndaAI1sdHiOItPVmHc1Ok0UYxFaRR/RBTdi9kz+NOGUGNqg0CFN3cn+LH04pDLMfvSP+dO3jAFKNtAEITIyST9TRtGOgqXJyaC+BSAjCkHiud8STtcarFERhYkx+fNdLH87gZrkdRk+0a1MwORu4NXEDQ0lALyE5Hy5PNdE8hY5zWHpUa/aHlboi/L9TWqX4pz3IRKHI6mkLE87jUBlI7Cm+f2wB9KgsseYw+6xFAml/wCejCqhmPY1GZ29aYjQ+0SH+P8AOk+2yqOJUx6EVnq5c4B596Rm7bgSfTmgDS/tCTqNho/tNz1EdZvlzN9xC30oFrO33lx/OgZfOpAdY1P0pDqUK9Y1P0qsulAnLs4/3nqCeXTrPh5N7DsmGNIDQGoQv1tGP4UqtFO2BastYi+ImjZhDYs6/wAJZsY/CmS+ItTkh+Xy4B6Bcn86Bm9Jpxk+6pX6ms+5GnWKgzXytIeqqdxFc9Pc3t9GUM0zHu3mHAFQ2lnNDcN5UMUiEYLy9qANt/EllAu2G2eZx0I6U0azqEmHCpAhGeBkj8aYiW9vB5a2cTSZJL9h9PaqYhcOW891Y9ADwKnmDlJLm8vHl/eXk5X0VsCs9rRTJ5iyPu7BvmFXHOCrFzID/Ge9W4ZZEjkhjgiZZRhiV5/CjmKMnyGSULE+5yMlsVJIwkdDNIzlBwD2NTJHtl3J8q5AxTzIASFAOPUU0xWIPNikTDsEdvvYHUVM8NmANsXmsf4dvNEchVsKdrUpfbL5gBLDptGTRcoZvDN83AYcAcYqQCGJVjC/MpycVH5+brdPFMijq7L0q1stwpby5VyMhmPJpCZAZnEokU4A6GpZbxrhMO25vX3prtGbVkIBbu2OQagV4V6wq2fWncROLcbmfChQOWB60RRR7cqDgnBZj1qoVjQkrGyAjjBzip03GMIZCCem4YouO4/JThSv0IzT53ISIebIzhfmx2FQmNhKuPnPcLyafHGqh1+6WOSH4IoJHMxWJRGDIx5LdSBQqbzjJz/tNgA01pURysUYUZwWB61Mlu8sGVGVznJamO5aik2AxhlkK5YY45qxE15cwqdiv5i/MOw+lQ2dqGs2E6tlTgNnBNXbW7it9ywK0zLjCJyTUjIYorpMwwQBM8s6nqasWyXMTkysX4wMt0qGLw5ql3K0sjC2DHOHPzY+la9v4fsoVIleSdz1YtgflQIxXmuZLoR20Idj3jUNV2DSdSusC/k8tR0EZHX8DzW6PKgTagVFHZRgVXa9/uHmgBlvomn2yt5kXnuxyWm5/SrL3kUH7tAgUdkFUHeaUndIRn0oWLHXmgCw97JIcInB9ahaN5QN5JqaOLLDApJp4IJBG7Euf4VGcfWgBiQgYGKn2rENzsFX1Jqqt3JKWWOIxrjhm5JpnkKz75maVvVjQJElxOLiEpbI7k9HxgCqdzpk98Nt5flV/wCecajFXtiY4UVHLLHbrvnkSNexY4osOwtvaw2kQihIUf7KipiqqpaSaYKOpLACsG48URL8tpFv/wCmjcCsW6u7i+k33Epb/ZHC/lTQWOiu/EVjbsVtg1w477ztrAudUvb04ebYn92MkCq6qB2pcD0FVYBgVUxxTsj1H50/FIIxnJAosSIFNPCGngU4HA460wGhSKcBSU6gQg65paT2pe+KAENFLSCgA7Zo/HvSUUxi0UUlAgooz6UZpgHbNApDRQAoPPNHHamk0Z4oAWjNJ3oxQIdSUfWkpDF4z70Dmm980ZxQAc9jSc5pRQevFAC5JxmndqYBg53Z9qcQwJHQUAPpM4oHDc4OKME84pDQEgHqOtL3pMDvj8aCenGaYhffvRmkJ+U0ZoAXHQ+tGaKMUALjPFJ3xRSYoAOtLziij8aYAMjrSjIFJyTRSAWjJpKKBi8npS5496bmlzQIXAwTQRzwc0g5Bo6CgBe1O+lN/Gg9eKBi8YphAz0/On/d6ikyDke1KwEWZI3DxSPG46MhwRXQWHi6eLbFqCNOn/PRAA4Pv61gYo21LQ0ejWV/bXsYltZklTP8J5H1FWXEcylJUDIeoNec2FoZJTIt8ttJFjBB55//AFV1dpqbxxhbi7iuivePAP41BTM7WPh/aalvaym+zyt1VySprhLvRdV0C8Z2ikjQdGI3LXsMF5DcIJIWyvQ+oNEl1bupjba/qpGRQSzyKy8QMObl14/iAxW3HeR31uGjbcDWvqvg7Tb1Ge0DQTnptOQfwri9Q8Pa3phYxRSBB0ZMYqgNpN1tL+6dkzydpwCauRX88bBknk+hY4rkxrmoQfJe2zZHVimDVyDW7SQgZZc+oppCOpHjc2fy3tu/+y8TnBFadp4j0vVkCi78tm6CXOa4qSSCchWZefeq9ubK4JTOGBwOOaLBdnoreG9Nuz5qaiEkP91uKifwWXUiHUI2J/vKCa4qJLy0XzLC/I3fejb5g1Tpd6xI7vJeCJ8ZBjJAP4ZqeU09pI6WbwfqMK5FxG/sBiq0nhy6jTfLZhvVkasuHxRrcahZ5xIg7lRmnSeMdQWMtBPtcdARkGp5S/byLX9nmNg0drux1DcUwiAOFmtlif2OagtviFq1rxPDDcRHqGGP5Vr23xA0e5/4+dL8seu0Glyle3XUpKBzsRJEPXcuaaxs1ODAYn9QvFdFHr3hS4QFowM/7FTifwhIOZYh9SRRYv28Tn1uJiMhUmX61G0sTPl2aFvQHit4W/hLkpfqmfRzVeWXwtE5VtSjK993NFg9vEzAT/y0ZZY6aZoF4jQoPQ1oxWXhy8fda6uqKe2/AH51pjwZ5vK6jvX1IBNOxXtYnOrI753RoyjoynBp6t2WdkH91m4rf/4QeYD93fD/AL5/wpv/AAhV32uoG+qmlYv2sTC/dg5dAw7MnWlVgfunH/XSts+Db1PuXMQ+maY3hbUV6vA31JosHtYmWA45ZVHuvShm39Xz9TWkvhjUEyRIin1V6F8PamOvkP8AVqLB7WJlgAdAfwp4duhC/wDAq0/+Ed1Ej5PKQ+ofNNHhbVSf9dDU2DniZmCOT0o+QnkBvrWsPCWoH71xGv0Jx/KpU8JXPR7mJh3G0mqsTzxMXIA+XI+pzRv55IP0GK6D/hDyORdkfgalXwrCP9ZO7n1xinYXtYnN717s3500yJ/dB9xwa7CHw7ZQkHyy5H945q1/ZlqP+WEQ/wB6lYzdSJz+kXd6zYLMUHZxXRRyv603FrH1dBj0NMa6sk/5bLn61SZzT1GX2kWGqp/pUQSTtKnytXC65pE2jXojd/MhcZR+/uDXYXGqqFKxAMT61TFpBqwMN2TjqpB6GrTJscUBkZFBBrqJvBToubS+U/7Mg5rCu9J1Sxwbi0bb/eT5gfyp3CxUpajEqnrx7HinB1I61SEKRQTTsimbvmyKQwooPWl4xTuA2lpCPSii4gzzQaKKQxM04ZppoAqriFopCKKAFpDSZNLmgA7UnSlxxQaBBmjPtSd6KAFzS5zTaTkUDHZNKDTeaQk9qQMkzRmo91G7tTESZoBxTM80u7mgLD92aM1HkilzQA+l4xTAwoBFAx1FJuo3CgQEUmKdkUnFACZzRRj0oxQAUYo70mcnGKAuH51KJpMY3nFR0ZpDTHmVsEHaQezDNRlLd/vWkDH6YoxRigZGbOwPWzZf9xqT7Bp56S3MRH41LRikBzYVS+SuR6Vcit2IykSRj1J5qKJ0dQBnPpinmJ8EncPYnFSWEsEvVrlQvoKh3QdJTK7eg4pyidGARI8e/NWjcNsIOzPpSAqrDHjOPzp2xUGaXaXHDAH0oFux6kn6UABRp0wquPfFIlosC5ZmLe9SRRSofmlYL6CqsjKsmzc559aAFPkscS8p3p8kgGPK+524qeGJIn3FQfrTp7clt6fdPSgCmuC2SDmrsZAQVTYzh8hwR6VZAYrncB+FAD2PykiqqMHlyBuNWI/MDYdwyk+lDTRq+3IUeuKQDPLmeQbVIx0qSSKbAV+3ekLM3KPke1Bn2Z8xsgelMBHgjXBCZPc5oWJdwbFCObhtscMzN7JV1NG1WeH91aug/vONooAhjgaRsBAB2YtTDEwBWSZV+hq/b+ENQLrLcXqxgds5rS/4Re0cf6TfFx/dTAFAHNeWnmoDKHUHJBarF1J9sdfs7gbRjYnNdFFpGi2z7kt2Zh/eJNaEclsoxFAiL7CgnmOMGlaiw4hlYdi3FTx+GNUmAdjHGv8AtNyK7D7QP/1ikMwb6e1MLnPr4YgVf9JvVL/7C5/rV630nR4MfuBK3cvV5hC33gTSeXb/APPPP1oKuAuIIRtgtIkHsoFNN5Ke4FT+RC38P60fZoP7p/OgRSYs+cuefembF9M1fa2gPdh9KYLWIjKl8VIrlNQAeaeqrnNWvsqe/wCNJ9mXsaYXIdq+tKNvapfsp7Sp+Jo+zN/C6E+xoAZx603eu7buGfSkayk3ZKEn2NI1vMBkQc+vegdyUgDvig8mq5juCeUY/hTWEq8FCPwoETZo3e9VWZx3x9aYZ9v3mFMC7v8AUikZxjGaomc+2KQTH2oGX0YbGb0BNcdbjzJmk9WNbt3dNFZTMO6lfzrF08fKK1iRI3LJgkT5/i4qcygd6qQJKwxGMH1qcWNy7ZIZ/UKKznuOOw5rlR0bNRtNlic1K1tDbpuuAsX++1RNd6THEzyTLKB/DH/jUjsIJvenpG8zYAJ+gzVFvEtlDj7FpReX1k6CqE/iDVZ8iOUxD+7EuKsDpDZogzM0aj/bbFVX1TSrTcDMCw/55rn9a56e3Oxi0kjP/ebimmFXiEbbyPRagDabxKglC20P/A5TgUXl9rELqNwQEZLIMVnFpN6EwoNoxU1vfN5nltvkVP4O1BQ1nt5WEt7dTyN3UdvxotrxxEBbsqyD+6uWqJsSlw7dTmkto2Q+ZB97lTmpGNmaSaQM7bJAfmA704qRkKo2jtTzG6FmeNSzdTmpYtiwSTTE+gA6mhDsEV8YMbIIwR1wOtRwt5vONjkk7m4FA2uofbj1NPE2wcIp/CgLAYZ3dflIQnIkPQ02UcsqnLYxuHSrED7o8EEkHgGo/tNnHJ82COhAFSOwtxEqqFLIzH+6c4FMgVoXjmx3wpqOS4+VY/LfcckNtyMVJauxjAuLeeTH3cRnFABHIzXDu2CeTTvMEgByEc/xY4pLoPjiJLfPGwdce9JP5UaIAMnufWqsK44Qoqby6vyASvUZoDpFJ8hfj+9TISEzsmEQ9CCQaJFLqZA6uo4ZjxzSC4875BncWHvQjyo24AFR1yKWC2muBsiIUH+InirSaNdJyt3EreppgJbWpvFK+Ym49s8gVC9jFDLhpssO23/69TT6ZdYXddoxJxlTTodEvIZVJvIgrHB+bJx9KSY2jOvrZ7uVVhLsVO5WUdvWofIuo3G+UuO4NdHJpqw4jku5GUdAgAFVJdLR33WiTSOeznIpoVinGk3yrEoOASSBipooBcXWJnyXxkKemBWlYaDqjr++K2yEk7jya1Lfw/aWxBeSSb0y2BTE2c81taeaI4bVpWP3cMRmtKHRr252/uFtEAxhiSa6GPyLZNsUaRr/ALIApr3qD+LmlcVynF4dtF2/aGe4wMYY4H5VeiFtZRiKGJIkHZRiqpubhz8pApuyR+WIJpgWJL75gFUnNVnmneTAOFxxxUgjApyx7mAA5oEVljd2wxNOWLBwKlklt7VczzKrf3Ryfyqk+pGRcW0LD/bcf0oAveWI/mkYKvqarHUYSxS2UzsOuOFqsUaV98xd2PqMirIDN90HPuKBkbfarg5lYRxnrGnSnRwpH90VL9nuT/Dn8aqXeoW9h8ssitL/AHEO40wLheTHOKhmuoLUbrmdE9u/5VhXWvXU3/HvGsK9mZcmsgxBnLvlmPdjk0AbV34kZ1KWabQf+WjdfyrId5Lh99xI0j+pNIF9BT1XPaqJGhF9KUAU/HOKAtADQKAKk2jFIhDAkZ49RTAMYNL1FKRgUwMxJ+X5aAHg8UtIQT93g+ppee5yaAEJPSijFFMBM04UnalBwOmaAFNHfFJ6UhyOR1pCAijoaM560UxgTxSZoNJTELRmkozxzQAZozSZozQIXNFFFAxKXJxmik9RQA7rSdKKKQBnNBpMGj0zTAATjmkzjqcUEjdjiloAUc8ilB9abuGcHNCgkkE0XAeTzSg0nejrSBCke9Ao7UA9u1AC7s8YpB70g60tAAO1KOpppyGH1p386AADBANHINHPrmgHLYoAOc89KPel65puOaAFBo470UEcUAHFJRR3oAM0tJjFLyTQAtGeaTOQPpRQApxjp1pOaXv9KOtACc7qCOaXgGjNAIbyBTTkg46048saaAcGgLkTwCQYbP4VGbJQMiR1q1g0pGakLlWJJ7d98F3Kje1X4tY1JJVFxdM8XfAw351BjmhkzQWdDaT294pRNSk3t91ZflOfzqC40rxVAnnQXa3EQ6KOp/CsB0rpdD1O1t7dFnbUS/qqb1I/Kk/dEYsl6JG8nV7Ro275Q1HL4U0u6iMljdRx/wB1Sa743Wl6oBbI8nmN0WeDGf8ACqCeE1tJvPs4QX/uMcihMR51deFr6AEqgdB/FE2RWYdPlHKuVb2r2R9Vis18q90jyv8Aai5FVJrLw/q77gU8xvUbWFO4Hk0V7fWb4Kbse2atJ4hlRh5iYPptr0D/AIQiVmJg1CNYz0DLmq0nw8jmuCLy6R4+zRIQaXMNROJ/t6BpGDR4z/dqGbUouqByD6riu8k+GGk+U32e6nSX+EsARn3rlb7wndaZOqyyIM5wc5B/zmjmHylCK6je3Y/xdhipbcxyZ46U2bTLnaVV0P0GKrLZ3cHO2U+680MUkdHaQRmMfLVk28YHSuUW9vVOFlUnsp4NPTV71GyyZ/GiwHUCBOm2mT2Y8o/u1P1rnD4huHOI1Kyehp512+T70ZP1FNoTLqaTHJcAsmAeoB4rYsXns18mG6nVeyhzgVy58RzqeYcH/dqxb+IyG3SQu5+uKqwHYLqWooeL6f8A77NWV1fVFHF27fWuS/4SqEH/AI9m/E08eLYB/wAuz/8AfYqLAdX/AGzq3/Pwf1/xpRrWqDrckfhXLjxXa/8APGT8OalXxPYN97zF+q0WA6Ua9qY4+1H8qX/hINW/5/D/AN81yr67Zs24S7V9cZqzDqlo6hkvYTns4waLAdCPEGqE83ZP4VOniDUgP+PjP1WsJbxXHyvA/wDumgXsKnDuoPoDmiwG6dd1M/8ALwP++KT+3NT/AOfo/wDfNZIvbUDmZfwo+22g6zoKLAah1nUj/wAvb/gMUz+078/8vclZn9p6f/z8pTTq2nqf+PlfwGaLAaZvdQPW5lK/XFKsk0q5M0mf96sl9fshhYS87noiLzTF8TWqDDW8qN79PzpNAbiqxPzMxqUADufzrDg1kpiR4zIp6BetSPq1xcKfslqf95jSuBuKwz71VvbpflSC9SG5ByvNczd3WouwMyPH6YHFQxqDknbu9WouB1dp4t1COUpfW8b7TgsCFNdJY+ItNuuI7oRv/dk4NeZiVl4zS5V/vCnYD1W50fT78ZntI2PZwMH86w7rwNbnLWd08Tf3ZBuFcZHJPFkRzyIvoHNXINY1a14gvpcejNkfrTuBbuvDGr2+WWASoP4kOaz7SxurycQRRkSn+GQbTV5fFeuKeZYm+sYp7eKr+XHnQW0hHRvLwQaLgTN4L8QjpZq3sJBmsm5tbqwkMN3A8Uo7MMfrV5vFOuPwt8Y19Auabc+IdUvbU213c+fEezL0PqKaYjNzQOaax284z7Cm/aIgPvbfZhTESmkpFZXAZTlT3FISwOAePQ0DHGjPFGaSgBRz1o6GjI96OO1MBM0pxSYooAKM0YoxzQAZ5pM0vfFJimK4UUUUAGaO1JRg0CDFLnjFFJnFAATR1ozxRmgBORS5NGc9aTrQA7tmkzR2pKAFzS5pDjtSUAOyaXJpuaMjPWgQ7J7GgtTcjPHSgnLcdKBjgaXNMzzSmgB4IoO3Ge9MzQaAHDrSmmZoBOc0AO4opM8UA8YoAxkVIH3SR7vQZxU014bmVdsCDBzVcQnv0qZEVWBWsDUurPEzhRbwrnu3SpM2w/10kIT/AKZjmsyYqTtIzTY2CqFzhR0ppgXbi4t2XFvG4/2mqFd3UMR9KrSTgNtUMcegp0dvqE3+qgnK+oXFMC2CAPmUtTWlUdI1HvirNvoGrsu7ymBPTeelaMXhSfaDdXaKO42mgDFaSKRAAQrdyTUDRkfduc/jXTx+GNKjcPNO8je5xV6PTtGhbKwK/wBSaA5jhVzn5lY+y8k1citb2dgIdPnK+tdj58UPEVvCo7cUx9QuG4XaPpQTcwU0HU2IBtlVT1LN0qyvhGJv+Pm/A/2UUVoGe6f+LFR+W5OTikO5HD4e0O2wS88hHq/H8qtrBpEP+qtAT2JJqExv7U5Im9BQK5ZS+aIYhhWMe1RvfSsfnYAelNC+tPUKv8IP1FAXITJI44Jx65pVjduuan7ZCr+VP3ewpjRAIcdqAnOCDU/HQ00kA8UhCBF96X5e2aTJpd3GKAEwD1p/ygetRbuaPMQdW5piJSQexo3HtmmecAcBT+NRtM2eDQMn3sB6U0yv/fOKgMhPXrTWfigCczEfxUC4buRVYMCM0bwKALBupP4VTHrihbps5KJn1xVXzBTDIKALzXZPUD8DR9vCjBjJ/GqO5SM7hURmY9BQI0xqA/uGg30J6kissb3OBViOzlk6igZd/tC2x98/lTTdWr9UV/8AgNRjTooEZ5njQAZyTWTHr9iuR5Jc5wAtMDWkEB6QJ+VMS0WXO21P4CsufX7ony7C0QH/AG8Gsu71fWC53yCNu6qBikM1datJbTT3LhFDEALuyazrRNqg4qgRPM6yzy72Pv0rTRcWjn/ZrWJlInl8R/ZIV+z6ewwMbn5z+lSxatcXdg00xaMnlEGAuaxmikliCPIXjBzk1fMS+QMl2C9FPNZT3No7FJHy4kvgszD+PdkfhSTG0klIELA9gBwauw200FwkjWoKLn903U/hVqe823Hnx2IjlP8Az0xx+FSMzT+7OMZGOlPd3YKseFzzgCkGNrPIWdmOeeKEJGSy4I980xWGqcn95Im31zTpxHGD5KNIR0IxQNogZwibD1bHNRylYwuGC5OOam5VhQdyAuuJD2zUkBCbyq/eGDUtoFjuk8ySMp3zTH8u3lfFxGUY5GDVJhYdcNHIVk8sIQMDFVWl+cKh+Y9qfLEsyApOGZTkKRgGmY2xhZNiSf3qAsTCWB4pZJl2tH8q7DyT60LEkUTSPJkscncajYtJtzKHKjAx2FOZnOAPnf07mgZYmEJKrBwpGWU9c1A8Myyj5cAUsfnSfJ5ADno5PI+lXLXTb6fDFx5fd3bAFSSyu0qRQbiD5voBUSSFjlM49SMU3VGSxdUF1FM5xxHyB+NMiuN2EjXePXOKLCuWzLdKP3DOD/s08RXpgWWa5cknAXcSRSQzLAC9xaTSY+6FOK04pJbvKLp4t8d5GG40guZ0lpK0Bd3y56ZUkn8alXRrqRVJQcDu2K1YJCFMRYgp1x700llmDNcuwJxt9KpBconS5Y8SSQQlR94A8mmMZJHVI4VDYxjHWtb7aqy+WwrnInafUrgoXkk8zCLEMjFAzZhhNpE2+VHcnJ7VBcbnufNjhZl/jYetX49JvruJFeFYcHJaRsE/hWnBo1nGmJV81v8Aa6flQBza3ieeqWdrLJKpz8vTPpWilhql+F8yOKzRTkc5NdCvlwptjjRFH91QKia5Re9KwrlOLRbdX3TTPKR+FXl8q2yIvkU9s1VlnZlwvFVzvbqc00K5cmvWjw28EdMVXmu5X+6AATUUkGbdjjpVkxKGzjtmgZD5bucuSadHEAelTLGzHCikaS2t1DTS4J6KvJoEP2AdKftKx7nIRfVjiq32u5lGLWBY1/56S9aYLPzGDXEzTMPU8UDJmvYkfbChnb1HC/nTC11NuEjCNDj5UqZY1VQFGKeFpiKqWNsjhxGS/wDeY5NTeWmfuipJGhgXzJ5UjQdSxrDvvFVuoaHT4/MftK3ApgbJbb1AAHU9hWXd+IrW3cpCpnZfQ4XNc5cX17e8XExZeyjgflUAQj0oAu3Wtane/K0wji7onGfxqh5ajpT9poCnPaqELSAEmn46/KBSjNOwCDcBgDmnLnvSgUvvQAEcZpBS546U3NAD85PTpSdaO1GccUCCkpaBQMM0uabxnn0o5pgAPzUppO9FAC/XpRkYzQaSgBc0maOKOKAuGaQmj3oNAMU0lJilzTYgzRSUYpAHWijtSUwFoHXOKKAOaAF4AzSZyKPr3pKAFpR0pKUde1AByTTT1+tL2pD9w469qAA8dqAc0bQcE9cUdMUgF74NAUl/lfCikzmjIH40APpc8mkzxR2oBC89e1FJnjiigYtA680nIPHpS9RQIXvR3pOKTNADxnvRTaX2oAQ9OKXPTPWjFIetMBaDnOTTaXJpALmj+EdKBgnrQeO9ABikxzTwyhGBTcWGA3pTM0AKBignke9A60Z9eaAFOcUDpntSFs8UDigBcZBPajikzR3oBC0nvS55NJnigBM0vWmYIOe1OBzQAY5peAMUnel45pDGruR1YY4OcEZBrorLxMLewWGa3k3j+KPgEVz4NB61EocysBtXniIXEQEMUiuDkO5GVPtT7LxrqVtxeRJcxfk351gY5oojDlGeg2viXSNSi2u3kS/3Jv6HvS3Gi2V9llVc9nTH9K87Mat1FT2l7e2Eu+1unj/2c5H5UWEdj/ZGoWXNpP5ijojHn86fHrssO2O8t5Is92XIrLtPGkqHF7abz/fiIH6V0FvrWk6imzz4z/sycGlYLkJax1aLy5DuHbY+Cp/CsXUPB9wU32l205HRJjg/nXQT+HLSV98WYHHRk61Se01nT3LROLtO4b71AXOJvNOvrA/6VbPGp6NjI/OoYzxXeJ4iVXEV9atAT2K5H50DTfD2uZaKPa6khmi+XBo5gucG21j8yg/Wo3tbaT70CGusuPBEyK0lpeCYdlfArAvNNv7Dm5tXRP7/AFH51aYFFbC1jdZEhQOp6nmrBbd99cj/AGRzTFbNONK4D2gtiMrdYP8A00ixUQhH8KLjvwKdjNKCR0ouBBJZwyHmNR9BTTptqwx5Qz696sGmbyKLAJ9mUpt2rtx0Ipi2UBj2PChx0OKm3saFYk80wuVG021/54gfSkGmWvlsoTGe9XeM804LkdKAuZ6afAgAwxx3zjNOOnwk5+b86vbRTSMUCKwt1XoT+NWLaxguGImmSP03d6KQjNABPplvbSqFeKVGz86HJH1pqWsB3b2CYGRkcU8DFOoGR2+be5jntmRJYzlSwzWqdYvJVKy21nNnr+7FZjLH3FNSYI2VbAFSUTNJulLeWqZP3R0FCmVSxR3QsMHacZqNrtvP8kBW7jjk1ZwR1GDRyoOYYZbh08uSV2X0JqJoskHGcVYwO4pCAOlOyEQlRjgGlA9akz60nHemSOBFG8Zxmk2qDxTtoP1FAC5A75poODkdKCh68Ug3D6UwHZH40ucdCR70zPNKGHNACOGZhlzx6U5sMMOAw9xTVOe3NO4PWmAKNi4UYFO3bhzzRg49qYQOCc8elADs+oxSnFMJGODSA46nigCSijNAI70gG5wcUo5pMg8Y59aM80wFJwaTPOaXANGMdaAEzk0tAFJht/WgTFoo5BoLbRk0wCkPXincYptAgNHJFLRQAnakNLSHFABmijFGM0ABOKKTFLQAUCkFL1NAgpM80tJQNC9KM0lHY0xAaAT0oFB9R1pDFzikzRyRSdaAHBhS7hUeKKAJMigEVHyOtKORmgCqLa7l/wBVaTPnp8uKtw+H9TmTd5Ii9nPNdO2p4+5Gn5VDJfzv0IX6CsyzGTwfdyrm4vVjXuEzWna+GdIs+ZJHmb/bbj8qQzSkcyMfxppLN945pDuaAGl25/d28WfZae2pqvESKPwrNApwWgRae+mY53Y+lV3kkc5MjGjBPalCkHJoAjC89KXYak6GnAUCIthNKE4zUhBP3eKQDtSAQCncUUYpjCjJ7UuKMDvSAM468UuQRjGaRtuOKaTimIeMqMDpS5G2ot/FJuOM0DHk+9G4VCzUcjrQBKZMjFJuNQljmgscdaQmSk5pucHmo9zDuKiL5f5ulMRbDp61GzoDgGqjTooPPSqn2wKfmkX86ZSNVpBTC4PcVnreiTIBYD+9t4ps9wLeNWfeSxwARUgX9xA5Oahe5CHBNWYbPNv5tzOIfYHNZ32zR7m6NvDJJPJ7cA0wJvPBGVBP0FFuXmPz7Vz2Y81VfWGtZXgtLRAU/ifBNZV1fXfzSMfLJ/u96kDonSJJMNIg/GrhvdLtIQGJd/7oHJrn724aGzt2VdwdMjjNVEimmzJJLjPRehqeYDYn15n2m1sRCFOcs2SfwxVa61y+uBhZPJT+7GMZ+tVfI7eY5/GmsIkXylWRj6kVopFEHn3C3Cy+e7Y7NyKnEcM92JV2hv7q9KQQMeAOKWK1aORjFDl+7YzS5gJXVWbKtz7GnwjaxD4k3DHI6VHFHcM+Y7iHcP4dwzUkUrMx+1Pu2nG3GKLgUPlN4Qv3a1NhW0IPcYrOTZJeuyLtUt8qjsK0pIpJURfMESj7zMeBW32TF/EVkBWMDHSpY5HQ7XBAPTIqOa/soE8n7V50n/TNM/rUlxrt1qSwwtAI4IxgHGCfrXPLc6I7DGM08zOzbiafNGYi0WSXHc9qSF4lImYk7f4QeTT7m5jZRKD8zHAU9aENlZ1l3YO38TQftkdq0hQRLu2nHOaljuoVbdOuR65oYQsXdhleNo7UybFazguYn86K42oQQfNbr+FLgSSt5sqyOOpXpUoYr0Ix7UqqCcnA+gpDLVumlSACWfEx6dqxrkrFfIAgnAzkCr8s33E2gbTnI71GboQjlSc9x1oArNgzpI9rKoX+A96uXEakh2gZ1bpkVMfOwkcpDMeQfQU2bzHdmd02g4GTgfhVDKUbxCdApHPpVoMivkmobcx3AkZYBAU6u4xmnwGGRsbCx7sOlEhFhWtixXe5btsX+tSxadJdRNuJj4yN8nFVpfKD/uhtU/mKj8uI/efP41IFGazgd0Dqrbjjg9KtR6UlpKj+ay8ZAqeNbZYpQoBYr0NRLObtwoikdh0wOgqhNF+O+woWYb1HQZxUj6ozoxMJJcYG3p+dW7PQdQmQCWCOFO/mcH9K0bfw1a25xNPLMP7p4H5Ck0KxzX264m3pbW7PLxnYOv1q3a6FrF7iSVVtEPXcen4V1sUdtaJstokjU9doxQbkg/eosBmWnhO0gYPcTSXDDsTtH5Vrww29ou2CFIl77Rg1A123Y5qJp5G70wLzzKFO5qrSXHGUyRUITKg8596eq0ANZndOSaaY8qOOlThcDFPWI4zSAjSPK5pRFzx1pJLu0twBJcIrelZr3kV1LMRG4f7sTYyMeooA0rgeXayPIdqAcn0qC1lFwhnEi7D8q7uDxWLb6NqD3TSXtyWjc5Kl8n8q0F0u1glLru/2V7D6VIBcNqMszJFMBbg8EEDNOt4jAMeUGb1zmnBZM8AYqxErFvm4+lUkFxd1yf8AlmKBFdFgNoGfaq93q1pZH95Nub+7GdxrGuvEl5OhW1Qwxt3YgtTsK5v3U6afFvurlE9gMmsG48TzHiyTK/33H9KxHBlcvMS8h6s3WnKoxwKsBZXnupDJczNK3bJ4FAAXpTgKMUJCE6dqX607bml20wG89qUA4pcUvH94A+9AhFBUcUoOOlLkmkxxTGLyFo70E5NIaQC0DAOcUZwKDyelAhMknilKnHXIo6dOlGR0pjCkHIooFABS4yKNoJooAKO1GeaKAFNICO9GAR1xSH1oFYKD1oooAKKSjryaAQUUe1FAAKO9Heg0wCiiikAUopuKWmAE84oFJkUCgBaQUvag+3FACduetFLRQA3achs49qUj1puMEjtmlFAC0opoOTSg9vSgBx6dKXtQDzSbsAjt1pAKBxzQAAOBwKQHNA4HPNAxe5NJnmlpcDvSASjkUNkgY9aXBIBpiuICAxBBpaQnJGaWgYc0cYo7AUEDNMQlKKQ8AmjPA5oC47jtTaXntR9aQBngUdzRRnigBMf4UucGikzQArZJzRQKKACjIooA5oBC5HSkPoKBjJoPIxigBOc4J+lKDil6kHuKQigBc0HIOccH9KQDPHejd60AgJpRQfQml4A4pDF/hyBSYyKQUufxosIQijFPwDRgYzigCLbmmsqkYxyPzqYgGmbQTilYZbs9Y1KwYGG5fb3V/mBFb1n40jB23dk0Y/vxcj8jXMbfek2AjHNFgPTIb7TdVjIjeG4Ufwt1/Ks648M2kkxmtme2l7FTwPwrgCjI4dHdSOhU4Na9l4o1Sy2q8guo1/hk6/mKloDdeDxBpikxSC6iH8JxmnweJ7fd5d/byRHuGXI/Oiz8Z2E/Fwj27Dpu+YH8a2v9D1S1z+5mRv4lOaVwM19P0DVi0pRN56tH8p/SsXUPBskSCSyuPOGcbX6/nWxP4Vh3b7OdreU9NvSqcsXiDTSpZRdxDqQOaLlHM3Wl39jzcWkip/fAyKqK6t0rurbxZbNIIbuNoSezjI/Op203QNV+YLEGPR42waLiPPmqLvXXXfgaRTus74Op7OP61g3eh6lYufOtXKL1dBuFUmBU4xilAXafWmBsnHTFSEfLheOOtMQgXnBFHQ4BpR5i8MQaAvJNAhe1KFzyaTvRuyKBjiOtIF9KDkigdBg0AN2kmlC804MKWkFyFwyt90kUhIkGBj8qsgjPzDI9KYQpPAxTGmUJLdlfzRzjsOtWIr2SX78bKR3PerBUGgQIR0oAcAXAcHINFV3hkjfdG2SOQDUzLI0Kykbc9U/wpiFxnjFDRgGmBTj/AFjKPTFOU475oEIF29OlKN3UUodR25ppwxyMj2pAGTTgT3pMUAUwEwd3zDjNK4GeKCCBTdxBouA3oPanDJ6UNx170BgOhoAN7A8H8KcHLA5603KkcdaQnB4oAfSjp0qLdUm849aAHdRQVDd+lIHU9BijrQAuKToKXOKTrTAFNLmkBpQcnigBwJAyKQdKOhpN2KQC0oPXIpuc0AGmAuAO9L8vrTO9OGCMg02KwnelwcZxRnmjeR0pDAgdjSAUHrS4oEJnmkJIHApaWgY2jFL+FJQAUZxRij60yQzSUtGKAEOaQ5yMUvTnPFL9KAEpDQaMd6AFzRkUhyaMCgBaTvRk+tJQAvFHSiigDaCgtycU4gA4XJp5iAOaVeDwKyGM24FKFzTsEml2kc0WKsLt46U3BHUU/Py00kGgQoIIpCcds0gxnmlLUhiAnPSnFm9MVHvwevNIZDmgRLuApDIKjLZFMLUwJvNUUhkBqHeO5oMkQH3j+VAyTzuaUygd6pyX1uh/iP0FQNqUWeEagRo+ZmkZx/eFZb6soPCgexqu+pyMflCqPamM2t49aTzCDisP+0JO7U37bOx+Tdn2FAG6WzRvwKyI31CU8Kx/CtGDTrtgHljkb6cUAK83pUJecngVorHaWy7pvLGOuTnFLJrekRJnztx9FGaQMoLBduelXo9HuZEBJ6+1Vh4xiH/HnZbm7MWx/Oqd74g1KcYhlTP8ewfdP1oEao0COOPzbxo1PfLU3OixKFCpK3YKDmuc3X8y7rhnmz2c8VNFeXltjO2OMegpjNLVb+VYVW1i8kd8isGR7i8AWU5PrV64niuo93mF29DVaRkt0ychvSs+UoiGm2y8XFxIzH+FO9RyaTZHHkMYcd881PHcoy5Yc1HLIoI560coEllp32di0cwduxc8CrN3pV3exbS9uD6hqoeYhH3mH0qxHerbwZ8uVj0B60uUkh8y/SIW7kMinYOOMVPDbdFkmjjA/iJqNGkcbmGM84pfMK9KpRKJp4Y1AC3hkJ7DpUllJDBu8yVucdeazpbmeNseXu/3RVi2luXXcqKnsy81VgJRIeSoIXccEjrRa3n2F3Pl+ar9QxqvdX94QYwUP/ADU6otxpXmQ5+0H7uTSaDmHSXHl8jT44SfulU5zUYs7ueF7qSOWNFGSDgCnWGszwRLbz6VNNOPvNxjH1qS4u9avYniJ8m2bHyvjNOwXuVNOikaZW29at3wxLt9hn61lWlz5XmxvdjzUbauDjimWWsRf2u6zM8yoMgds1XMLlNOKLy0xIB6jYOophGGz823sCavrq9hdIxEbKx6ADJqnLdwofMQK2P4GH9KkZCZY4TvMQfHY1dMu+HMFmsEnrIMn8qgXW7oyeV5EUSHsI8ZqVL1FfMkTH2FAXIle5kXLm3/AAGKWO0SbPn3KovsatXckFvMn7nqM/LVKZ7lUKpCN7ElcntQFxsaxRHy0+6Kmwh+4c02PTmSPzHMjOQCQv3RUgiAwEkAf+6aTQ0ypJnAZipU9R6VJbzn5xA0e/GcuucVJFcJ52JYwX7/ACmmXUjCQ7I3jVuMBcCi4rkUF3cecDcoBx95vlrRtp7MZE7x+bnIJ7D0rNDSmVRsEjdgBuNW4/D2uXZDpaRxL/ekO0j8KB3J57k3ULoW3KTxgYAHpWdsdSVTls4K10UHg+4ZD9s1dx6JGlbVto+l2UW0Q+a/9+U7jScSTkbSwlu+lvNIp7oOPzrXtPCcB+e63qvZQ/P410RmReAcL6VG9ynrVWApQ+HdFtyCbfzD/tsSK0xJHHkRLGi+ioKovcbulR7nfqaB3Lslxs6t+lV2vskbVJBHWoxEMYOTTlQAYouIYWlduelKsZzg5p6r+8B7VJjBNAEWwCpEXBzipUjLdRUUt5Z2xw0pd/7iDJpCJVXd0FO2hPmchV9Saom+u3XFtAIx2YjNNFqzsZLgvK56lm4oKJ31KNeLeNp39QvAqpMNTvBly0SH+FOPzq4vycKiqPYU7zG9BQBRi01YzxGCfUjJq4ICq8gCnDe1Mnmitl3zzxxKf75oABGCRnH405o0UZaRVX61hT+JoxlbaFpP9pxgflWNc3NzqEu+4mZl7J0AFAHRXviOxtWKW6tNKvXHAH41z9zq+oX6gPIIoz1SMYFVvJUdABT1QrVEjEjVTlRye/epAp60YwadmnYBm1j1IpQKfjj0pQPlzimAzHNPAz1OKQmgNnkUwFFApOaKAFwM85FFJS4wM0AAPrS/jSA0UgFzRmkpcUCAjBwacOlN5oyaYwPWjHOaKXt1oAO/tSHrRQMZJoAXPHFHJo6UZoASjvQaO1AB3pDSd6U9aBC9KTNHajrQAUmaOaSgBc80HrSUZoAWlzSUUwFFB9qbzmg0AOopPel6igBoAUcUtFIflPUUAKT7Ud6AcZooAMjNIrqTwc0HrxSH1oAMc9aO9APNKKQB0NKPvU1EC4APFKD8+Og9aYCuGIwG20+NimCMZ96aaBQAUtN+tLmkA7p0oGMUnNHbHegbHZpBSDGaU+lIA4pabSimIX/Cg0nWgfr60AHelpGwWJHftS42tz2p2ADRnjFIelA6YpAFHajtRQA4daKbyKB0OaQAKXvRQPQ0wEyDyKXNJ9KUjsetAgHOfaig4xQOeKBgCeQaUDtSUZoAWm4zS0d6AEBxxilPPSkK560elAxQaXoetNOD2/GjHy7vfFADg3PNPPTOfwqHtmnA+hoAk2kjg4pAppm4g9zSljk0hC85oxz1pMn0p386AGnnIpu38qkGO9H8qAItgbjFSQyS2zboJZIm9VbFIOGBpxwxzgD6VIzatvF2o2yhJNk4HqMH866Ox8WaZd5WWT7PL/ck7/Q1wO0UhQGk0M9LurHT9ShzLDG6t0ZaxpvCUUeZNPlkhfsN1clbXV3Zf8e9zLH7Bjj8q3LTxdeQoBcwpOF6so2n9KVhloTa/prBdizj9auW3i6EzeXeW72/u3SrNh4k02/UYkWJ/wC5N8p/Cr0tjZ30QJjVh2ZCCKaEULjTdC1h/MKxhm/jj+U1j3nguWMFrG4EqdlfGfzq9L4RVWL2U7xnsQelV2l17SuX/foP4t3NMDmrzTL6wdRPbSBT/EBkGq4KnpXbW/i2EYivIjE/oyVO9noGsL5gRd//AD0jyppiOEEYNHl4rqLjwa33rK+Df7Mg4/MVjXekapZDM1ozIOrpyP0oAz9vek2c9aXzAflJwe4pSAOjA5oAQLR0NLk4oHIzQAtNFDnFXJNO22/nwXMM6D721sFfwpgioT3o3flSc7TigY9xSEKr5OKqTTS2dwzsfMtycgf3atZKjKgfjTXZZEKyRAg0hjftdtKm9XZR6OKerpIvyMCagCNCNqAPETwjdRUqCNRlF2568UwJNpHpRSdqMHFMQho69T2pOtAwO5pALjHc0UuRjgg03NMB6sFzlQ3pmnyPbSID5BjlHGVPBqMCl2A9aAI8AHIFKSCKVsLxQMUAN60opM09TtGMA5oAbgZ9qN2OlBYAkFDn1BpucHmmA8H1oPqPzoXaetBznGeKQC87eKTf7GlHApQcUARpcIx2kMG9CKceTTiVJ+4M03qc8UAOAGeaCfm4zR+NAwTzTAWginYqN3MZAKk5GcigBwoFME8R6sUPutPUq5+V1I+tIBaTkfSg5B5GKORTAXJopM+xpCQKAF60fSgEA89KccdulADee9JnNOoIGc0EsCcmkpSeehppoEFLQOlAA70DDFIKcw4+Ug0360wCjFHBNL9KQCUmKX6UfUCmSIAaSn03vQM6QwTf3aTyp1/gFWRHIelz+lDJcDHlzj3zUGhW2T4/1f6Uwrcj+A1c82+QABY2oE93/FAh/GkBQ2yj7ykUFigyQTVwvPnLW/603zcf6yA4+lIkpecD060b2NWfOtz1ib8BRm1P/LF6CipkDksKaJFJ61ZMdn3R/wAqQwWQXOG+lAFRpADjNMaT0NXPsto3McYHu1Me1siR5k5B9qoCkzmojluprUFjYHpcMx9AacNNgP3JCo/2jiixJiNBvP3gPrSrYO3atxrewgx5k8Qb03VR/t/S4JOAzAegp2GQLoDSdZk59qtweEkVcGVj/n61Um8ZquPsVplvVhmqFx4i1K5iLvcFc/woMAUmxnTw6DptrjzGG4di1NuNQ0XTmyX3Efwrya4qKWS7Y7pJHPc7qBGGl2CPPuam4HUTeM7flLKyLN2ZlxWFf+KtQuF8tpjE7fwoMChCbOU+Vbh090xTJI4pn3NGq/Si4GfIl1LEC0hd3OOtWINKvU+V3UL3Oas8RoBjgUsZmkiMsRKKP4iKLgVfsqxRmPJIPX3qSCBgmYB8h6Ukglkky87tnr2q7aRSOBHG4UDuetFwKZjmJG7zBg96leENHkySsx7Z4p8zAblaZjjuTUQMysoWJ5FP8QFOxQwW1zEqhSDk9zV/7BcvblpImYnvUjwuke+RGUDuelQx36SRGBWkBbupyaTQFddHmP8Ayz/8eobS5YlLyxFVHcmmyu6oVjknDnoSc0W2+RSs7TNGoyRnFKwDM2wH3xn61JbLFNMqBcyE4DHgVMJdDBxIsm70JJqyL7Swv7qSRPdI+fzoEWLm1S2YJIyu3fB6UiWFrMocxgA9CWqLyjMjTxQszes5wT+FQQ2d1db1up0VEGREg4zQBrxiG0TerptHXPNZ4vobq8d0PygZzVaBLQowlt2O04Gehoa2tLl8qfJiUgsAeo9KYyee7jXC7vMJ/gQc1EGSZA214V+vNWbPWLWyHkJYpH/db7xJqxcXccy4dUJ+lAmzDmWZnZROzLxjDdKgeS7VlW2lWTbndvbI5/ya1obeW7RmDxpg4wRii0SW1vsiJHYKcjpQK5jjSFukaS7uY4VHUInP51PFodony2rHzezkZNdDNo8srO7YgDcnnIzTba0S1IYXkOehf2+lAzIFpJ5uz7RJ9SgFWYDHDdqpt2Zycb8dPpXRSzafIF3zq205GBTfMsBL5Z2gnhQxwSaOYClJd/asWtvbNv8A77cgVA+m3RxiSL8OKt3V+tpKY4LeIsOmHFZsWo3ZiYCWEBzkrjmi4FqWxt5QpLMzqMb88ZrOmhnkuNoljD9Fy3b8quR2+p3UYRLb5CclugNaMXh47f3t4Y2PZAM/nQBjrJ9lPlz33K/ePYfjU0Gpvcz7LbTPPbtMRwa3otG02HDGBZnH8UnNXlZVGFREX0UUAZFva6pM5E1pbQD+8ev4VbHh+y377mSSbnoTgVcLmq0l6qsVz0pAWoksrQYt7eKM+oUZ/Onmfdyck+tZL3btnC/lTFmklHLHFMRfmuApJLflVdrkt0z9TVYqc9SSak2fKv1oAeXYjFNCfNnNSGPmnBKBjQoJGBTwp7U9U9OT7U4qV5kZUHqTSACOKbUMt5F5TNA3myKcBOgNVVTVblSTL9mU9BGOfzpAaLSwW5BnmRc9B61C+qRGQLbQGT/bNU00NQ4eSZ5WH96ra6ei9qaAQtLOwZ5QFPbpinIYLc4V48+uM077DF705bNFOAo/GkANfIP+Wo/KmG8DnCvn6AVJL9ktFDzywovvWTdeJ7OMbLOAzk/x/dA/xoA1lZm6g/lVS61extOJJiz9lQZNczc6je3fyyzEIeqJwKqbFB4UVQGteeIr64OLRRbx+/LVlMjSsZJnLuepJqQcDpSZ9qqwhAMdqXFOpKAEycU7kjmm0deKoBc0vOQQSPpSY5xTtuODSAOT3oA5zTu9GMUAMVvVTTs05QpOGbHpSxmNMlwSewpgNxS7eOKXqc46ntR1OKQhOlGc0h4JpoYkfdI+tAC4pQMd6KXrQMQCnY5wKQU4ZNACUdsUd6TNMAOelJR9aWgAPNHSiigGL1NJwBSdKOooAO+KOKTmgfeAz1oAWg9eKMDPWkzQIU9KTpS0UANzzRQRzQSAKAFpKPTjrRQA7NN7UZpaYCd6M0ZooELnjFHakzRQMXPNJ2zR1PFIT0+tAC0UmKcBu4oATueaBRjjNHakAgABPPXtRyOopCfmGBnNO4JGDTACcUDmhwwBxw3rSZoAcCMd+KAwbkAj60ijqQc5HSnKcAj1pDDmkHXFKaUDPtQAZ4pQMmkpccZoACMcAUlBODilFACdvelHY+1IRQOmeOKAFpKUMCMjpSgZBx1oEGfUdqTNICcYJzSc5oEOJpE9TQPSgUximgZpDRQA7b34xSdeBSE80o9utAAODQeaO+KKADvRSZpTQAdqTk8GndqTkc1IATQPSjtRTAXtRk0nalFAC5yOaSjdnNJmgEJ3FKcg4PSg4wKO1AC4ycdqaF5waUkg5FLuX8aQCcfjS5pMZOe1GcGgBSaAxzSZJoAGcGgCTOe/NGQDjPNR/JuIz0pDxQBITxzSbjwO1MBp6470DFXLNjgD1pp3D0pCcc4OO+KQ4bgcGgQ8E45oIXrTS2VHrikBOPegLh8h6gVZttQvbFs2l1In+yTkVX47gUEjsBUhc6az8cSxL5d7abv9uM4rpdO1uy1VcW0qu3eNvvV5n35ApxaMfdBDeooGen3GnW0y4aFQfpWFc+E0TMlnJ5TjoAeK5ux1/UdPf5ZRKn9yUZFb1n42tm+W7t2hYdWXlf5UhkcTa9pz7WVbhfTPJrRtfFSI2y9ikiP0PFatnqFpqMXm2syzJ3x1pl1plvd5EiZB7GlcRXlttF1hhut4pB2dcA/pWZe+B0bDafOUx/BKSf1pt34TaFhLp00sLj+FTxUSXuv6YoEgEyr3zzRcDJu/D+q2A3TWrFP70fzCs/pwQQfQiu2t/GloSouI5ImPZ6uXEejaogdlgb0YAAiqTA8+27hTViI612Nz4HjwXs7sq3ZHJNYV14f1i1+ZoBIn96M5FO4GbsbtTQrEZIp4Zw211KsOoIxUg56UAQsnHJ6HihQeualZSeDTNuDgdKAEZeM0YpxBwTjilJ+UYpiIgOcUHd0B4pwHqKcBxSGM24po61J0JzS7VIyRTJIf90AUgznBqbZ6CmlCDmgYzdg4PWl30rKSc01lIoAXqMmgccdqbgijNAD+DSd6QEetKOmRQAn1pSMjpmgZxjPFA4HNMAA9aQjJ4zTj6UnINIBhJA5pynIyDTt2OoFN3r6UwEJING40uV70nBoAUAn5uKXPvzRtFJg9qAHqxxQWNIFYDpQy7h1xQA7eW6gflUTQQE5aMZ9RxThuXr0pc0AR/ZkH3JpVPualye9IWpu4d6AJc0nemliBn+dAfHWgB1JjmkMi+nNJuzQA/PFIDkUgNOH0oAA3NJuTdgnB9KCOc0mB3A+tArC8dqAMnrQVXbwcUAUBYUntSYpe1LjjIoFYZjFLmjGKWmAhzSZpxxnBpMCgAJGaU4pMUmMmgRoeVKO5/OjEq/8ALRhTzIx71Hvf2qCgWa4U5WRz+NJ9svR0kNAZh9KmS3mk6RHB70AQ/br7/noaPtt7/ExNX4tLlPzOQF71bTT7cd2f8KAMgahd/wB1anjur1uiCtlbAKPkgzUcptrZd1xcRRY6gnmgCkqXbfeaIfhVkWxYZxmqtx4h022/49oXuH9+lZ1z4j1KVgbdYbcf3cA5pDN1rN0TdIVRfc1n3N5p1o2ZJkZh/DjJrkb271O5l8y7upNg7LwKbaGJpczKTH6nvTGbsvimJX/0aCIY/iY5/Ssu7vb3VGLNOEX+6vAqwxjMoW0s4z/thf6VMbeUfNcmNM+h5/KpGZltp6mYNcy7of72c1dudPtWhzAdq+rd6deTI21YHWIKMdOtVo4Tt82SbzT6dqLisVrd5LUn9x5ielAuUkuAZLdvdBwBVxbkgYHAqKSZCeULfSi5Vi7HcWzrsKhMjqBWbPdPAS0Rj4PBJzmp9kZ7D86EggEgPlqW7E9qTE0Ed/PdRfvM56dMCmSCYMQkLsPXFThNnQD8RmklvrgDYrChBYpj7WT/AKkj61Mjy8K/B9Kme6U25j/jPU1Z0qy+3hwZfLhBwxHUiguxnS+YzkxD5e1XIIw1tliQw64qzqU2nafsS2iDOpyxJ4/CsybUJ9RUQoiRRA5+UdfrQS0OMEaqzr8xHducVNbyXaKp3fuR2oaYmPygFCegFIJWxhVDH3pisTNdvLhZYnaL+6zdag2NA5nS3KL2YmnxLKxLyuoB6Z4xU81jPcRo73H7lRgDNAyut7IzhtqjHpT/AD2dizNgH2oSGCMbWmB91p0sRh2SRuMLjbigB8l3EMFY1GBjlaZ5Rn/eDYP0pl1LdX0XzrvC9lXHWmR2El1tEzPGgHC460AOdTIcGVvzpsJWxibaznccnvVt7KyijyJnRx75qWynszYmAwBmIwXfqT61IFFZfOXcolPrzirtnaQXCGMWbqw6NnIqyljKIAsZVQfWrsJure3EZAVe7gYFNAZR0Ihy1wSo9u1Wo9GsOMSSfiadNdeZKbeO4BftnofpUqNstwWUlx1wKYmW4rBUwIUytWH0iyc7pn2y/wCy+MVgzagskwje4eNB2XOTVdrC81C7EtqJ3i7MVxmpGi9qX2iHckbSTL/ebmqa3cUdpsaFBMO9bWm6VdWyYupAV7IB0/GtA2Fk3LWyGmmDkchDcXEiqBAjynoU5/Sr7aDqtzKGmSFfUseR+FdMqRx8Rxoi+iilaVU+8aYrmJF4ZjVw0+ZD7NV6K3tLFvk08rj+NRk1O14i96iN9u+4CPrSC5N9qWT7u4ezdaYzoPvNWXqU8gv4fLbG9eaUB2+82aALTXka8Dmqz3Ts4AyM0CH2p4iHpQIQBmHJNJ5YJOQasKAOaUnIwKYFcw5XIp0SADFP+bGKdBA+ws3AJ6tSGNMYPSpQp2jimtd2dvhZH3ueip1qF9QupF220IjT1c0CLvkkJudgo96ga8t4jgMZG/uqKrC3STDXNw8jemeKsJJbQLtjjApjIJJdUuZQYAtpF6kZanLp2WD3Fw87D+9U/wBpBHAppkfsuaQE0cUMX3YxmpC+TmoEErDpiori7trXP2i6RSOwOT+VFgLe45p+Ce4H1rnLnxIp+W1iz/tv/hWNPeXV0+Zp3K/3QcCgDq7vWrG0480yP/dVc1kXniK8uUK2qeQh7n71YoUL0FO3Gr5QEffKczSNIe+6lAUDhRTgKOBRYQ0HPY8d6djmlL4OQDmmqf735gUwFoySuSAKbmlXDDPamAo55FJg0u3ninjOMYpCEA9aDjNOzg0nAHTNA0OFNpKcOmaAEJxS5yMHvSYz1oJGaAFH3h0oPSgH2pMnFMBQ5wcccYozTetKAc0ALRjJ9BRty3JFGBSAOO1Oxg8ikA9KN3NMA6UuCByKQ9vTFAyTgUAHU0UlHNAC9aUZVqB6UnegBD60d6D19qM0ABNJSj34ox+VACHJoFLSUCDFHejHGc/hRQAppKQ9aWgANJjNBNKBx9KYgDHABPAo4pO9GKBgetFHtQRQIPxFHfNGKOKQwK8c0H7vWikPA5pgHXrSDBJx2OKXtRjFADmbcc45pAT3pATn2oyc9KAHD0oUFhkemaTIHJ6UZ54oATNGeaTnNL0BoAMkmjpSDOM0FgvLsoXtmlcBVOOKdmmggqWAzxxjvSR52nKMncZ70DJV+ZeePrSYG0etJjIyTSdzSEOByKNwyFyMjpRtwAexoIBOSOaYAetKM0YpPxxQMXrSd6DnNFAhccUDPWjNG4/hQAYpSvy57Unv29KTOQCBg0AKQFIwQeKUHNN6nNLnIzQAmecUop28LGylQc98cimdqYC5weBRSU6gBO9Hej3oFIBORS5GMe9DEk00DJoAcKXikHSgcigBTSUHrRQAtGaQ9KBg/WgBRgjOKUYpBwMUZyaAA85oo70H0oACe1IMZ5pSDmjHtzSAX1+tJikIpaBCdBTST+NPU4YE84ppxvyKYwJHVh1HalOD0pGOcUAcUDF6oMUq/wC1+dNPtS845oAcp25561ETsIOcZOKcc00nK4IpCAjB605ZCuRng1HnnpTtgK5pjFyTz2pucUpBBx2pAN3FIQpOVyDzSBhinLHnvijZg9KBiA5NKVHekxjtS5bBzyKQCRtJA2+CRon7MvWte28XazbOvmSC4QdRIOT+NZBOaT86loDsrXx2jMRc2bqOxU5/pXT281nqFuJomWRG7g15NToJ7u1lElrPJE3fY2M0rAel3ehaddqyuuCehxyKwrrwaC+6zuGiHbDZqja+MNSiRRc28dwo6sow1dDZeJtLvEAafyJe6SjGPxpgYS33ivRMoYRdRDo4XP8ASrNr49dcJe2AV/UHFdZlQeWB91ORWdd6Hpt6zPInzHvUgJb3+i+IIWWaCISHHyyEA/gc1Ru/BcDKTZXTxt6OQwqhd+CYWy9vO6P2Dc1Elx4l0BdzkXduPXkj6cU0wKdx4d1i0yWtzMg/ij5FZ6zKnDghvQiukj8foP8AX2Uiv7Gr3mad4lgUtPGrN/CRh/8APFVcDjvPT1pcg11s/gi0dALe4kif/a+YGsO68LarZnOwTp6x9fyp3AoBgFwaYWGeKjLYykilGB5BFPVVI4NAD8BsdqCNvBx+FRNGPX86PKAbimIlypBx2puaZjBzSfuyfmJBpAKWoVwDzSAJ3YHNIVHamAMUbIxim5A4NACk4JNKRg+1ILjWwozSD5lyDilbaRgjNCgEcEZ9KAEDEZG4Gn5yOtMIOaaQf7xFMCUcGnE9jUQyR1z9aP3vYj8aAHcg4JyKa0ZzkNx6UoZ+6g/SjzBjG0imAgz3pDycZ5qRWQnBOD707y0IyRzQBGGOcd6fuwBnvQ0ZYcGmeS2M5z7UATD60h60i+lBzmgBwz3GRR0PSmljn2pNxoAeQM00qM0nOacKAGkenSlIG0DFIcjvS5oAbsBNIUHvUg9cUdecUAMGc07JHelIB6UmPWgBCxHUUD9KXbS/dyPWgBuPQmnDPrTRTu1AC896M+lNye5zRzmgB2fWjr0pOfwoyaADBPNHSlpvNAhecUUc0UCsba6fL/EMVMth/s1WfxNb7ysEDz+hA4NQTa/qci7YrURL71AzXSyZOQEA9WpZZra2XdcXcY9l5Ncde31/LxcXzID/AAK3FVg6n70maQHS3WvB/l0+Hcw6yPwBVI3+qMP3mrIrf3UHArLT7Oeqsx9qViycxxBPciqKJ5/7UvSQNccAf3hiq39gwO2+61YvL3waJrUyRA72yevNU12xThTUhY03ij8rZvDehFNgj249qpyb92IIyz9sVZWKQQ7piVb+6DRcdidjJN/q4xheSWquwEuN6gY7imLcNjy4oG2j+ImlWXzH2KhzSuBftFkjDGGbyVPU96hmjxKSWZie5pEkni/hGPeopJJJHzigLiApFyVU/UVKLxim1VGPYVWkiMhBIYAd6tRpEYgjzBADnCjmgYxLWeaPeIxtz0zilMSbZPlVdvXB61WljkJGyYqc4AzxirDxSABCcnGSRzSGQQW6CIyyI4I/iY9KJJF27YHJc9KkDAqIXQsp64pH8uGUeTC5I70ANWGVIS06zE+uetPhjimhDtHIPYmpEnd7gFtx+gqzcx2xPloZPN7gnFMRSjuYEOx7YMB0IptxIxHmRRbIvpV2O2j24I+YUDAl8sAE+4oGmYk9xPdZggTbGepx1qxYW9zbxyGVPlC8N2rSubeV0aOOHa399eopoNxHF5UsjH1BoApxSqAQzE80O7GXy0Vtx7+lXkt0ZdwlRfXI6U5LYuzeVIGI6luAaQXGWdqrSfPLg/7VT7tpKA8Ui6Rd3XLOqj+6BVqPQbwH90IgPVjkmmhGc9rlS2XH+6KiEc0TiKFPn6jJrVvtM1GGICaZdrfwxjk1Rjt2thvJbePXtTAT7Pq4Gf3ca92yOKS4haFQ0l6XcjJwOlNuna8VUubk7Ac4A602VIobfbGA3rkZNAGhaaJJcIJ5XTyO7A8n6VNItrabTaWiyqh+ZnPJPpUEK6vcQJbWcEiwqMEvlRVq38IynD3F4IyP4UNAFeXWJrnZ5kXlhDnKt/8AWqC4mudQXyojPNjoiLmuptdFsLUAhDM4/ikOa0F2xj93HHGvoigCgk5Oz0DUWi2tAtsD/G/JH4VpwaEY12S3jSJ3VeM1stNt6kCoXnUfxCgBIrS1t02xW6L6nqTU2847VSkvlTvVWS9ZvusR9KANJpwO4qpPqUcT7cgt6CqRLsOtReTukBbrQBda+ZxhQRVUmV2yWJqcoEUkdQKiiYEZPU0gHlT5ZNTRp8opoUmpgMUAU7xHGpWzbcgIRVqNRjNQXL5vYST2IxVhUcjptHvQMXgU3OTxSSXdhb7VmuV3nqq8mq66i7y4tLchP+er9PyoAtmJ29B9TVa5v7KzQmSdSy9VXk02WKe4bM8rOvpnFOhs4o87IBz3PJoASXVWnQR6dbvHKQCXlXAFIIbuT/j5ujJ7LwKtbGzjAFSLGPUUwKotVUDCKD608QMPSraxg/xL+dOYRwrvmkVEHViaAKfkkmni1B5NVJ/Eel25xG5uG9E6fnWJeeIr25QpCPIU+mCfzqgOof7NbJumkRF9WNZk/iGyibbbq05/2eBXMEmXmVmZvUkmlVVXoKAL91rWoXKsikxRnqE6/nVAR+pY+7HJqReaDxQSN+XFLxjNIcZoJHGDTsMdgbc0ygA4z60cAe9MLjsgUZU8Ekn0FIpzzil2qGLUAGPSlUEjHalcfIdtKJMqMkg470gGlDg4/wD1UqKVHLFvrSgnpS9qBBRS4pKAFpKXtSdaBh3pdwpMUYoAMg0fKMZpfakBDHigQvak+nWncYwOtIOByOaBhS9Dmk3Yxx3ozmmAtJSE4/OlHPSgBSeeKaRz+FLx6c0Ac0AKTnoMCkUbQSCaO9L1oAKTJFLjPGcUYwPegA+tBNFA6GgBKTHNLSHpwcUAL0JFFJk0Y5oEFOYdx0NN6AUZpgGcdBR1o+ooosAh60UCgCgAOcZP0o/GjJHFA5oASlpD1HtS0ABpR1pppe3XpQAp6cUlApRQAmSfT2ppGeozS9xR1NABz1ozSnABwKQc4oAXIx0oFHbNANACZo696CKTGetACHnKgjNJGSE+cYIJFLtQknbznGaQ8ZDYxQAEkEkEY9KVkjdQXAOeBTFRQSAOppfKG4EE8HOM8UgRIuAoVRgDtSlssowee/pSDGKNu3oSfrQArOqABjilzjt1oAzSk4NAwJwcUA800n5tx/GnAgqDng0CsLz2pD6UAjvSEYbPagYpNHbNHHal7UAJjNG0kUA4BPejJ9aBXEEnO3aSTTzjA45B602lzQFhCeB7Uveg0dqBChwoIKg5HekO/AKrnJwKQ89hTg2OnX1pAIwZDhhhhwRRmhgzfMecnk0pwPujApjEpM0UvGR70wGk5OaUUvY0gGelIBwoI+XjpSUGkAhJzk0ZpeaCKoBO1KOtHO32FIKQDu4PakpB6UuaAFB5FLnk8U3NGaAAnmlzRnIxSCgAJo980lAHzYPTGaAFJpKXtSAleaAFJBPXPvQCDxTT7UmCW44xQCJMBe9Jlew5oBBpMEGgB3GD6ijAIB9e1IvPHendBQAY4xjikGMlRml7UUgG/WlxjpRRg4pgNbKjil3kYyKXGRjNIVyKQATk4poTH8eR6GkIxUwX9365oAQDPHGPUUeVycUKoHHQU7PPFAEZUCj5iOBUjDcc0gbAxnmgBg3gYPNNKhj8wzUnWgjJxRYCa01G9smDQ3MgA/gJyp/Cuhs/GQZwt/b7c9ZIhwP+A1zAXFDIDUNDPS7a/tb6IPa3Ecvsp5qyFDcOMr3FeUpGY3DRko3qpwa17LxHqVkojkxcxf3ZDz+dFgOtu/Dun3rM0kI3H0GK5u+8JtA5a2d+Og9K1LHxLp1xKgkMlq/o33R+Nb0UizruilSVfVTmhAcNb6lrmikKfMeBf4ZPmFbVn40t3+W6geNh1IGa6EWyEcis698NWV18xj8t/wC8tIC3/wASrVoAzrBMD3ZRmsi68GWsnz2cphI6L1FZV14UvIWaWym8zb0Xdg/zpLbxFqukMEuoPMUdd4Of50ARXfhrUbZGIj85R2XrWST5bFJAyMOzDBrubDxfpl4+2Uvbt/t4x+da8ltp+qJukignX+91P51SYHl+VoUKOorsL3wPbyS77O5eAn+FhuFc7qHh/VtO3NJbtLEvV48EUXApNDF5W+KQZ7qx5qHNRNIofa+Vb0bin4zTAcCc5AJ+gpSynoGBHXIqMTzWsvmQvtb6daklvJbkKZQpYdwOaBCFwByQKQEYqKR1x8wJFCOuODxTAlJppPNN86I8Z57+1KOgI6UAKGFLkCkPApN2aLASDPpxS5qIHGcGlU5PXFADwNx96fvIGCaiY7WHelJBPFAEoOaCHPQ1WMTM2VlKn60YuE/5ag0wLHPpSZqHdcFc4XP1qRNx4JB9xQA8n8qOMcDFMyehGKcrYzxmgBxAAzkUgNIeaO1ADuvelwCME1H3p3XpQAuCvBoppJoBNADue/FN/GlLE9ab3oAeCM4pDnuKQdaWgBf4KQZo6fSgGgBaOlJ16UUAFAoXGcHpQcA8UALSUZJ7U3kmgkcDikzuzxikAxwBTv50DM8/aLZSYpxu9QKtWltqN4gZtQRM9sVCqszBQM5qd4ZY03AYUVkUacenQQofMEcsn94nFOUacgCTIhx/Coyaxkdn4EjD6UyadoX2mVz+NBVjZkvILdv3Fmo9GYVnTkTPvYfMagZyRw7N9ahMrDqxFArF9Y8wEl1x2XvVQG1MmyTG/wB6hF1Jv2oOfUippFh2oSpMh5OKBl63uI7aNwI9zMMAYqAW8ssnAxmo1IDAmr11qT3aiGNBCmMHHX86YDBo9/jIkiVfUmnw2kVpIHuNQtlx1Gaz5khziaSQJ3+Y0ljZaPdXaxvJLg9zkCpJNO6+zX7B7e4DBBg7e9NtYGtmE02xU7lqu/8ACP2UCE2cpLn0NZYtUhdhLLvfvzxTAbcXCyXO1eYzSFIgymKIs/bFPi0pWAlaZ8/pU8NrcHd5MLOR3HagZD5MoHzLEp9OtGSsewHBNOkWaB9s1tLn2GRUtrBDeyhGjlXHJPYCgY1dO2ReZNc4z0VV5NKAsqgLEcDue9bQsLVHVSXKnqSOlJNYadu2+a350Bc5maVhlYvkbON3Wq8tpIU8yRpnI/iz0rrIrCwV9oy2fU1ej0jqBGNh/hzxQBxVjYLcPzduPRVOc1dOnXiE+Su5f4SetdcmixJyY9n+6KkFoUIKgDFBJykdvq/l7BH175pw0a/kUNLNEH7g11DIzHlRUf2GPzd7TY9s0wML+x9qAzshPopzitOLYqKIYUBAxnFW7iCMwnazsewFU7S0u/LxvVF9H60gLMQIYEjFRTakkeRGjSMP7vT86kfT0mwJXYj0FTw2cFsoEcYz/ePJoAx4r7VbmUIIVHuRUc+i6rezFpWVUbox7V0ZcgUhk460xmXaeGbG3YPKRIw6kcZrRjs7GF90dtHu9WGTUUlyqd6iF4G6UCNDeT0NRtIAeazPtkrSlc4FRO77z8xoFc03uo04ByfSq73khGAuBVOMZbJqXGaQxryyPnLGlVDswSTQwxTlPyj1pgRSKAMk00DPQHHvUk2BGSw4FNjZDGTnoKBiopxTdp81T2FAmUUNIVwcAZ9akRNIRsOPSoIgCQpqpNrNkj7PMMj+ka5H50yeG/vJV+zOsS9mxzQUbIUr1IA9zUM95FAuS6t7Kc1nxeHyJVe8vJrgj+HdgVdi022jcFYgKYjOuGuNRaNraJk2n7wOM1LHo146ZuNQceq5rbC8UbaAMmPRFjbInU+4TJ/OrcdhsXb58p+pFXVXmn4oAp/2du/5eJvzpRpsQYFpJmx/t0lzrNhZDEk4d+ypyaxrnxZJIo+x2rD1ZyKYHQrawr6/8CeqN1qulWiNvlLuP4EySa5O5vr6+l33MjbeyrwBUQjX+7+dUSat5rouVAhtQi+rE5/KsmRDcLtld2X0LcU/aB1p44oAiRFRQqjAFPBIIBB/AUrYAyelIHPbP41QXHZOPuEH1NI2ARjk4oG44oxnnHIoC4H260rEhelNVWYndkelPC9yc0AR/hz2pzA8Y6+nrT8DGKXAznFICMcNjv6U4pnk49gKdijORTHcAABS0bSRmlApCEDDnPWkIB5oAHp3pRTGHNLkYpaTvSELjOKCccUZP4UdqAEBzwKMEHBpOnTrSls9KYwPXFFJ1OaBQAtKFGCwABx0pMfnQOTz0oEBpQc0vHWmnrQNDuDScZyDmjoTS4FIQYOOAMZpOhpWPHFIOlMYmaUZzntQORRQIUjjNFJkdKcMcZoAaTR2oJ5ooAO1FJmjnOO9AwozxRjmlxx7UAIeRR3o7GigBKKQnmjNMVhaQ+1BoxxQIM84pc4pRTTxQxi54x2paQjHWgUAB9KKKKAEIGMkcUoozgUgPPSgQAkngdKR2ZQMLnJp5yPrTSx3BcEk96BhzSKMZ+uadyRSAc0AHIOfejPWkJ5H60vegBdxIC9h0qN0EnysDjNOJwR9cU8jBx1pAMH3eetHelGOMdqDxTYATxUUeWZyxzzx7VJkGo0GHfnJJzQA8nAORgg4oJC5PUCkDqwPsacSm3B7igAV1ZsBgTT6hUIvzbQD60M/lIWRsjvz2pAT5puSaYkgkxtPBFOWNllz5xZf7vamMd9elL9Kjy+4ggbR0NOqQF4o6mgsAce2aAcUwFPApOaDz34zSnI+tAgxSDrRnNHegYtGaOtJ1NBI7p1HNCtjPGaCeMelNANIAHPegjhSDz3FGBS57YpjDnAzSg8UmeMUlADuwNIOtIBzmlz2pCAmgHFIetB61QxelJnJo7H1pI3Vs7o2Vh60AP3dPakySaX5cUmOaAEBIJ4pT7UGgj5cikAlKOaSigBaTNHelBxQAZpQeKTjPWkPWgBSaOaSgGgBc0FgBknFIe5peGG1hkGgBN2KQPgYApTgNwKKAAY5OOTTsHGaQc0uODQAA7T0pS1NzkYpwIKjFIBQR070vYDvTMHOaXmgBOefal7ZpfrSfTkUwEOMcGlDAYB70e56UpAJyBxQApz0FN+YcA0vNHekADk80p60wqT7UozgAnOKAHqSOwpo+Y8jBoBFGeaAEOc0oNGRSsQozjigBvPalye9WIY4J8hbhFYHoaZNbtDKELqwI4ZTxQMjBxRnJoC4wTSjFAg+UnO0fjToLm5tJlmtZWikXoV6U3b6UinNAHTWPjadcpqFv5iH+OMYIrdtNZsLzAguhvPRJPlP6152y8GowmTUtDPViMffUn6U2WKKVNm0EHqDXn1nreq2ChI5y6d1kJIrds/GFtKwjvbcwN/fTJWlYCxd+E7GbJRWhJ7jpWK+h6rpsvmWM7OB/wA8mwa7aGYTpvt5klj9VOacwjPMibW7lTQByVt4x1SyIj1G3Z8fxMCprfs/FemXrhfO8pz2ccfnVmWxtrpNksazIex4P51gXvgu12ZtJTA/91+n50wN6+06O9gZhZ2s7HoSMH865n/hC7l2Iidbc/3ZDuH51SltNd0N/wB3JIydnj+YVbsvG95EqpexJcD+8oCn/wCvTAybzQNRs5cXFu7R/wDPRFJBqnd2EloqPlXicZDjivRrTxJpd/wk4Rx1STip7zS7C/i8ue2BHqnH60AeVleKNqeldldeB0PNndFB/ckBP61z15oOqWDnz7Rig6vGdwpiM4RoDnaKXIUYxjFKZVIxwD6Uvlhu+aAGbweDS8UoRaXYO1AhqgDtwKXjPAp23BoIHagBhoBFOOMU3aDQMM4Oe1JkHpTudu0nIpmzac0APDEGl3DPHFNo/nQBISDSEjtTAecUoY5pgPHOaM5GOM03IoHNAC0oNNPHJpQcchjQAu4e/wCNHB6UFs+9HvQAYNLk9KMjvRkUAKOtBNICRSjmgBOKSlOKTtQAUpAx96k70EAjBoAWlpgjAztYg+mOKPnHBxQA7digEEcdaNvfP4UY5oEBY+goUnFBIz05pQaAE37ULRwvuFQC4upZtsiPtPbFPIvi5dF/WnXGoTi38uT7/fmsih/2XzOjhPrVZbKGSUvMwbacACsuS4Jb5pCPpVuK8tY4wu/FSO5pLHEF4TCj3qDbI3MaBh71gzyXjXZe2Dunp1rc0uXU5IgptGUDqxTigZcltY4LRfkLTMM8dqzgHjHzod3oK37fTZbkmSeQJ6kDJ/nUd0ILJwsMxeX3ppiMYx3G0O0flqf7x5qPBDgk5rtUtLeSFDLHufGTzVPULCWUCO1iREHfPWmBziSuh+WDfWlBZXF1EJTaYT121o6Z5GlIwvJbYuxyNxqxL4itS/kwgMT/AM8+lILlT7BdSwkeWyg+tPh0iFIwXGZO9XUvH2h3icL9KbPJd3ERS2tpAp/jI5oFcrrp9y7CNZYoof7gGTVuOxljXED8dzjrUdvG2nxCSfeW755pLnXIoFBTePbbTGWFSfOHfmmta7J93mOf8Kij1Ca4UPCqnPtTZEv7jOwqrnpk8CmBOzo7eQAS57AVOEtI4drxKfc9qpx6ZfLKskt2jqP4V4zVz7KH+/SJKDeQz77ceWwP3jVhL+aLGZmI9qtLawoOEFPCKvRV/KgLjPt1zNFuR3z24pI5bt12yEge461LvI6mjzB3YUxJiBAFxQIoxzt5pj3MaHG6qst+AcKCKZVjQ+6MimhlPUisv7bI+cZpn2mQnkn8qQrGq88SDJYVCbxe3NZ+ATk07y+OKAJXu5Wz2qISO3UnNIVNCrxQNCOCRTU+XnGacwOeKcvApACgbycdaQ/epM8k00uAfmIFMLDwcU4NVZJlaTaM/WmzTxopJkANIC0zA96ckoHQZrJN8hbZEHkb2qaGHUJMHYIoz1yeaALGpz/8S+VRjcexOK56G/mMxiG7H+yMg10I0qBpQ85d/UZ61eiigt0CwQquPagtM59bXVbsbUTyE/vsefyq1FoEQbN5eTTN/dzgVtFnY80eXnmixLK8NvZ24AtrdFx3281ONxPSpAntTtoH3jgepNOwXGbTTxkDGKqPq2nROVe8QEdqy7rxXCPltITKf7x4qrBc6DB6YqvNfW1sD50yKw7VyFxqer3nD3TJH/djGBVUQA8uzM/ruxSaJOgufFMYJjtUZz9Mf0rFutQv77ieZgv91eBUYTHSl2mnYBiRrGQQQPc9aeiqOgpCtPUYFUAfKAeOtJ1peO9BIHagBuCe1HLdxxTj60uQBg0ANNG0ZzmlxQRj5cfjQNigcUHpSgGkHJpCEzyR6UvTkmlpM9jzTGKT6UHPYZpMY+lLjigQmT0IpQfzpR1PvR3pALg03PNOyQRRgE0AN6U7IzQOWpQQrEADr3pgGcUZz2o4zSc5pAB4FAPA+XHtSc5xThnHLZoAbmkwM5p+Kb3pjACl96UrikZsDk0ALwOT1oyeePc0mPm5paBCA55HNL36UmcHFLnmgEHOeDilHT3pOppOh4oAcRikoyduCc4o7DFA0FLuwucZPpScY7k0nSgBATxkUvPel3HPFJk5yaBXAcgGjnNAPNIxxQAfSigUDrQAvU0UGkGTQADrRR60cY9+9ACHqPailppoAU5YgDjFGaMUCgAoPTFBwOaOe9MAAoo7UtAARRk0gozQIQlvX9KUHBpO9LgigB2786byVK+veijHtQMRV2DGSfc0uaKTdg0AB5OTSgUClJwMUAIRQOlNyTjJzjpSg889KAFPtQTkYpFOVz0pMFjncQKQCgYprRo+4HPAzxTjjsc1HszITkikAqxpHnAJJHPNAVXjKYI7UoHyHk8HvQp4pgKMrEVYgr3JFQCaFiUKipsiRSCD171H9nhJBKHjtmmBKdkSgAYHSnk9MHtTSuVx0pFTb0zikMeCcYOPwoJ4xQO9IetADhgigcYNHbj0pqsCcHr6UIBcgHBpwORimt0wAPrQKCRTQT7UpFNP0zQNjh655pBRRnmqEKCM80vG3Ipgxnml6VIxVbac0pIYqQuPWkAB6mgg9AaAGksHx/DmndDTFMnR8celLmgBwZlyo6HrQCByaZhlfO7KkdCOlODYYH0oAUc9u9ITRk5GKKAE70pYnrR1JpAeaYBjNO6UmaOe1AB9RQaM0UgCjpQelA5FACGjtRS8ZJHftQAEZFHbBpKP4uaAFoxzmjOBnvRkYoAd8oXg03HzdKQ4xg0ooAQYHQYFO60zGAadzt4oACaM5oxxQKAE707vSEUYyuDQA4HHI70uc8VHSc5FICUABeuaRSQ+F6ntTQc89Kd0INMAGSTuGKdn5cCkJzkmgdM0gDPFKpHpTDQtMES8E0wdSCOe3NLxijAHOOaAG9DmnggkmkGR15FNBOelIBxHNA9KUHJ5700ZHUfnQA1ogTuX5W9RSK0qD5h5n0qUnFIKYxFkEg3DP0IpQeMUHHOKAQeDUiFycnmmZIIp3c0hUbck4pgAJOc0vajB25FIAwPJ4oAfznJprIhPvQTQOuaAFikmt23QSvE3qp610Nr4vuUQR3sQuF7uOGrns000mgPQLHWNM1FQ0Fx5bf3JTtP61pEyf7EiV5cEVhzViy1W/wBNk/0e5fb3V+RUWEelhofZTVK60i0ulHm2kL+jqcGsWy8YW9wwTUIPJfvIg+U10lu8N3EJbKdJI/VapFI5S88GBnL2c4wf+Wb8Y/HNZ7XHiDQ8R/vvKXpuJYGu+YMp/epx6qKRtjYUSAj+5J0pAcvZeOI2UC7gKn1U5/pXR2msWl/blrS5R89VP9RWbfeHtPvGLtZ+U5/ih6Vz1x4PvYH82zkEy/3d+1hTTA6ttKttRcreWNuVP8cYwR+VYd74JdOdPuDJn/llJwR+NZFrrup6RMEkR9v92ROv411Fp400+bictC3oxGKdwOPvdMv9Ox9otZAp6MoyKpJMh7163bXUN3FvhkWRD1AOR+NUb7w9peoriS2VG/vRgA0XFY81MinoaBzXT3XgO4jUyWVysi9lkIBrn7myu9Pl8u7t3j/2sZH5imBAwGMYpuMGl3Kfutk+9My+/jB+tAhTnHFJyRmnDPINAFAxh6ZFKD7ZoxTdvzZBIoAXNKSQAcE/SkzQCRxnigB1JuxSGg9KAFHJ5o6GkzS96AFzgZNKr+lAyORgj0pv4UwHluabn0py4z83ShgM/L0pCE3UoJpuKTJ7GmBIT60ox2qMZPWnA4OKBjuBwaCufYetIeRnIpCxxgUAKVKng5pdwx8x5puTj3ox370AB60oNJSCgVx5NFJSMOKBiObln3QzoqGqk9vcGQ7h5hPUrWzY2NkYQJnf/dUVet7m3tNyW9qSR1Y1kM56HQnvIgsdiwz1d+lPfQI7J/JYBnHp0Na99rUzAJCdgHXbStHLdWke9XDN1OKVhkMMy26BWkgiP+yvNTTXfnKFgnBz3PGahl0iOJCE+aZupYdKltdMRYf3jOJPXNVYB08F20B/exj6U23sLhYgcRn6mnGyVD8vmN9DVy3njjjEciPuHtSsBC1+tpHhxvf+6prJvNQNxKWaZ4x2Ve1bf9nQ3NyZJEyvpVoaXaiXesSD8M0WJOWtrC3unMwNxMw65yK3LCxS1USLAsePug8mtJrUP8u7avooxTo7dIwEGSo6ZNFgIFSR3Lkn2Hapi908ZjHU9G9KlCAdOlOVQKoEZY0yfzd8s+72xVpdLsj800Rlc9S1WWOOtRNdRJ1agZIIYU/1Ue2nHcOtUH1KMHjdUbajIxytFyTS3EU1plU4LAVktczP1qM7j94n86QzWa7j7MDULXvJANUQMkAVMIx1JJNIBXuZC3Wo2d2OdxpCnzZzTgKAQgQg5pSvOcVJjPGcUbccZoGRkHGOlNUODjAx61K3TAIpAwIPPSgBAjZ56VMcCq3nL/eFRtcSO2yJC5747U7gWS600yIO9V/s14fmIC57Eiq5tHL4lu4VY9F3c0gLck6DoaryXB/hNQtbXKviCPe3Yk1PHo11P81zMsef4V/xquUCv9okJwrZJ7DqagmuLndsis2kl7bq37XSILUhsl3/ALxq4FCdKGwObj0vV7pg0xjt1Po3NX4tAiRt8kssrf7TcVr7jRupWAqJZpFwgCe4FWBGO5JqULu7VWub6ztEJmnRT6Z5oAk8taXywO1YM3iy1DYt7d5T2J4rNuPEGqzqyowiVvQDIoA7F3iiTdJIiKO7Nisq68SWMHEO64b/AKZjiuVVJJeZp2kb1ZhUixBehH51RJqS+Jrt02wW4j+ozWZNdahePuuJ2I9AMClxjuPzozVDIRboO1PCgUEEnrRt9zSAeCPSkJAJpccUm3PBoATPpS5+XqM0dOOKUDPWmIT3FGcDmlIJPoKb/Fjt60DGhsjIRvx4pxzSbyCAQSD6U/AzxQCEJPehj8v3enelYANhT+dHOduc0AxAR6UUpGKMigA3YHNFJj5qXFAw5Dc9KXHekI4pR0x2oAX2o2+2aTBzyMUuT0oAUISTgdBk0xjg89PWnB8AnOeaMZGDyOtAhAQRkc0oOAcUmB2GBSgAcUDFzntimkHtQD6UuSTzQFg/DFL25pO9KaBXDinfd7dab0GaMksCelADs4FLuyMfjTcHPSkJHSkApdjxRgHqM0lAHegAo57daOjZ60Eg9Bg0wE69evelHUfWk6GlxQCFP3iaKKKBiA8/hR15oOQKO+KBADg0ZJOaMDFJ0FAC96TNGecUpHFAXG9aU4xSDNLQFhKO/WjrRigLgD+dA44/nQR0NJnAzQMWjtQORmjIyfp0oEwK89aMZak3Z7YoBIYGgBWznFFBPX60L1yelMAPI9cUhJ60EgdKU0CEB65FOHNJ1pOlAC4ooPc01Tk0AL/HSqPmPHWmj7xGfelJwPegBaTJzjHbNLk44pMgDODQMU+o6U0n5l6UpJzjtSYbPAzz+NAC55wO1Jz6UYO0464pFBAJz1oAcAT9KDxSJuC4PagkdDnjvQAihig3H5snNJGTsOeMk5p+c0MeM0gEAAGQBg+lNAfzAW4UfrTl6VFNlsbH2kUASuuflyRVYSOspVVBVh0ParIO4A96ryRv5vmjqBigCZScZIwfSkjWTzCzMMdgKI5N6qWByR3FScCmApPNCgLnGeT3o+XBLdAKheWRWCAAZPHFAExUk43DPrRs3D5uc0DJHXHFL1HXFIYINuQT9KCAee/TNHWjcqkhiPxoAXHFGTjIH0pD+H4UHigBc8U3ABznFKOlBUFaAFzUTsDlScVIDg01grE4HWmKwwPt461IDuwD0z1puwZ6UoBzx0pBYkJjP+rzgHHNJmmjCnp3zS980ALmmgEdaflR97OM9qbjmgBiq4diWyp6D0qQAbeT+FLnHApp60wFoPSk6UZoAXHQ0cUhJ4AoNIANPX7gP4U1jwBnihaYCAc4o5pDyaUHK/hzSACcigEjkUdqXtQAAUHFHWjB70AITxmgg8H1ooI6HNACYJIFKR6Ud8UGgAPK4NA9aKOgyaAD60YozRkZoAcSS+G6UYFD7S2VHFJ2oAM0c/WgYHUUEBjwMewoGBHSgA4yBSd8UHOeKAQu7ijPGOaTFKDxQIByad0pnOaXNAC55oB5o42+9FAIOQSDS5pCeaAMgn0oAUH5jTsAj5evemqE/i496CwB4PNIQEFWpSxIGaTdk0HrQMUmgdOetAPHTj1pKBoVqBQBk4oxg4oEHfNKDkDNHA60nPagBVHzcetGeaXLjgjtwaac9T3oADRyTgDJpO1BHGelIA57jB9KCSKccEZ7+tN780wAEgdcUHrmlooEJwetKjywvvgleNvVDigU7igDXs/FGo2wxK4nX0cc/nXQWPiLTb/asziGc9Qw4/OuHamipaGeqBJlGYpN6+oORTNygnzYiH/vKcV5xYapqGlt/odyyJ3QjIretPGgLBdQtQc9ZUOMfhRYR0zwJcKVkiiuIj2brWFf+ErG6fdAz20h7HkVtW13YX6h7K6VmP4EVORMpw6hx70DucFLoWvaK/2izYnHVoWGfyp9v451S3YpcxLKw67lKtXcq8R+6zxOPaq9zp1pfHN5ZJIP+egGCakVyvpviyxvwFbMEh7SMMfnW0WWVCroGXuCMg1xl14NibJsbrbjpHKCf1rPK+I9EjEY8xY+2BvFUmB2F74a0m+UgW/kuerIf6Vzl74JvYD/AKFcJOv90nB/Wn6b40cbY7+Ld6yqQP0rpbfWNOul3QXcTH+6x2mncDzm7tbqxl2XcDRe7dKhDq3Rga9ZYxyJskjRx/tLmsa68J6TdsXCSwue8b4H5YppgcBtJORQQ3tW9f8Ag3UbRS1rKLhB2AOawZIrm1bbcwSx+7LgUDGlcU0g46VIpVhkNmkzg/KcGgBn0pDUpGRk9ajINMkZzS7qeAO9IVFBQgzmkJxS4ppHagBVkb1BFO3Z6VHgjpS5weKCSUOwoB55AphbLUoxSKH7uflHHpSHbgbc571GMg0v86CR2Peik6nIoyc0ygJNKpJpM0ZoAftb0yKToeabub1o69aAH5wc44pC2TSAkU3dhufyoFYvWekajL+9muzHCOpAwalv7eGziAWd5N3949K3TbSOcO5x7Uo0y3bmRS7epNZFHOaakMZV5JA7jqBXRi6BGED/APfNTJawR/diUfhUm0L93imSUFSYyB5FBPvVhV3dUH5VMRUZcL1pjF8te6gUvlx91B+tRNcoozmozep2NANlrgdqUMKoG9DdKY1y5GBxSuK5otIq9TUTXMank1nkk9TTTRcC81+o+6CaryXkjjC8VDtJpQmKLiQxpJCMsxqLbk7j39atMgx0piqM4AoGRiPJqZI8DGKco2nIpvmHfjNAyQR800/extzR5maaW75oAkTjqKdz2qESLsJLioGvACVU8+lAFtk3CmBQuKotczE5BxUDzOT80rZ9FFIZrlwOpqpNeANtXmqFxLJaxeYyv1A2seTVKW8vpoisNoUz0fPNAGs80rqQpqBmnX77Vnw2mqyJ5Mh2Ke56n8atxaG7nEzsR6BqAGm4jVTkiR/4UB70kN5r8jBI4ti/7Kitq0022tRlIAGHcnNX4+OgA+lAGJH4flucyX1xMzt1Ga07bRrC2x5cB3D+JmyauZNOUFqAE2BfujFJuPpT9h9cfWqF5rWnWGA8wlkP8KcmqTAvKxNPCMetczP4sdkxa2wXP8T1jz6hf3ZPn3LbT2XgUMk7C51ewtOJJxu/uryay7jxTFylpDJIezMMVzgiT0qSMbelUhpk8+qapdZEkhVT2UYFVTAHwX3lvUmrINNLYpiuIERHUgcAUvHYUw7s5XHPXNOzgdM0WAOO1K3KgYzSdadj1oAQBgBmk5zzS5APFOoGNpQcDikNAHFADhzSg8mmjdj5Rk0itxz170AKeaT5s9sUhOeOfrSjoB2oAdk0vSkppcnI2n60hXHLzRSAUvSmMSlxg0ikY54NKeR1oAU8j3pAuRSquBjrS4wKAYnQUvLc96QY70hPp0oGOPTtTaaSSOMZpVyVBIxmgBxxjGeaQfWjFLQAU5BuOMgY65prccY5o7fWgLi9s0mSaP5Uuc9qAEVdop2cKRjk0ZJIHajg80AJg96XtS8E5x+FJ3OKBBxgg96B92gjoaM8UDF5GCc4pKD7kmkoAWg8imsaFGBxQK4vNA5xQd2B6HtRQA7vkjNHI4PfmmjPegYBznpQCF5z8pwaXNJ3NBx6UAHJFJ2pePWkoACaUYxSYGDkfSkAx0oACPzpRkDBOcd6ByKKBhmkoxQMZ5oEHSjqaTnvS0AGaAeeR9RSUADPvQA4n2A9hSU080uMimAcE8iig0daAAmlVuemaTqP5UE4ORQAdRig0D/9VBwetAhDTh0GeaQKNxJJ56Ck3ccDJ9KYADnIpB1o5PHQ0uMHBpDF7Eim04EAEUkaHaxyOtAC8UhOBR1FIVDABhkDpQAHkg+lJ85b5R+NKCQTnGO1LkigBW+7gHk0Hnp0pueaeo70AC8UjdM0Z5xjg0gByTnOaAFA7e1IV4xk9Ka2Sw9jQBgnk80AAK5IyT9BTbjCxZBCnPU05cgcnJ9aVgGXkA/WkAKGSNVYgkDkig5PSjhQSewoSRJFyhyM0wHdQCetGM1A1yol8va2fXtUqh/YigABAG3k+oIqQbGKl1yoPOKT6mms3yttwcUgYp4pRUcMjuCXA9ql/hzQMM1G8SSHBzk9807ILYFATDZyaABV2ALknFBIB706mCZHLBCCVPNADwKWoXLZDMnyeoNSKMqPegCSJYmf94xUY6ioWHzZFP2AUADBzQA0ZzzUh27B8vzD+L1pACOSfpSE8UCFKts3Y49aQZFRuDIvlh2A6kVIOB9KAHc4qNXYyFSvHY07NBcAgCgBRSHrU9u9uN/nxs2fu4OMVCwoAQ9KOgpT1pX24AHNMBgpeO/4UvoKSgAxxnFHtR/BQPWgA6HFJzQ2Q2fWgUAAp2eKao3HinDHekAgP5UE5NKBtc9qCBQA2jr0NFGKAFzS8ZHemc5xS5APvQAp+8fSg8igcrk0fSgBM07AxkUykLqvVgpPQGmBJx2/GjtTc/LkN+FHOKAHUbT1pAaNx6UhCH71KKMUnTtQA8jmkzg47UhNAGaQwXdtyRQfbrQGZeAxx3BoK7lPtTAUHtSdOM0mTk5FG7cM9D6UAOOcZoI700Z79KCfQ0AOGSaUhc4NNRg2fUU4hT1JB9aAAEBcUcHmmtnPFKMDGT0pAOAppBz8pH1pWPOVOKTOBkjrTBMVQD83IpMHkZzS9aKQIU84pR6UDGOaCcOuOlAXDnpTgQVpjHD+1Lt7560ABGDzSkjacmkOfXik5GCBQAiuAxDBsEcHtS9TRuPpikLUAOKNjNIDzTSxzkmnDkZFACjBYDNBzjiggfjSYNACg+tKPUU2gHBxigAJIpQytwRzQf1poHzUBYcCyOGjYow7jrWrZ+J9Ys0CNKJ0HQSDn86yqOaAO5sPE2m3zbLl/s8/oxyD+NbYhcqGjIZT0IORXlDLmrdhqd9pjD7LcFU/uNyD+dS0Kx6Yc9ZYwT6jihWjXiOU+yN0rk7fxw/C39vuTu8Yxj8K6KzvNN1Nd9pOj/7Pf8qVguNvNF069+a6swr/AN6M7TXN3/gyVVMljcK4/uP8p/nXWlXXo5P1pMqeSGD/AN4GmK5wa6nr/h1xHNHIYl/hmBZfzrZ0/wAc20yBbmN4X9Q25T/hXTsjOhV1jnjbqrc5rFvfC2kXW75HtHP9wcCpGbVpqEN1EJLe4jkHfa3NSylLhNk0aSIeocV57d+DtWsP31rIsqjo0Zwf50yDxTq+m4iu42l/67DB/OncZ1V94N0y8bfD5lsx/uHIrBvvA+p2q77d0uVH+1g1oWHjWyuXC3SNan13blro4rh54RLaXCSoehU8VSYjyuQTQNsuI3jb0YYoDjbXqDyPKCt9ZJLGepKg1g3nhHTLslrWd7Vz2YjFK4HFkqTwaXFa2oeEdWsF3rEJ09Y25/Ksgu0TbHUqw7MOaaYDu2aZjmpN6kc9aaOTimAmzcMrSYPIA+lPBKcdqQnJpiIsnIUrj3p3SnUm3JpDEJz3oOacEFIePemMQHBpc85xRwelIcgYoAlDAj0pjBgeRxTQTS7iCKAD+dJz3px55702gBaAce9GRjmjjHvQJnctKq9WAqJruNf4xWFLIxOGY4pFyTjJNQBsNqManHX6VGdSPZaoFMYp4WkBI97MSTnFVHuJnkxvOKfKcHFRqOQaYFhQT3NPA4xREOMmpDgUhjdvHSl207pRkEcCgBMUoAzSZxShlBznNADgKQjBoadEGc1Ve7yeDmgRY3dqMqOtZ6vNIxwCPrUbwS7sySYX60FF97mNTjcBVY3kYYkmqUklvEMb9xpouEK7YrZmPuM0AW3uml+4eKafNZPmlCL71HGLhgTJthTH0NJEse4s9vLP6EZAzQKxNBZvcv8AuFeRe7DoKZObi0lKCAFv7xNaVpd6jJwLdYwfujbgAVba0muvnu5V3+irQBzgubs/64oqfSon1K6jl2W0akno2zJNdAuhWKOGLTNjsX4q7DFbQJsiiRR9KBnKiz1i/cM8ezPVn6Cr9poUsDlprjznPb0reLbhxTdrHpRygVfsz+1SxQsvWlllaLGIndj0VR1qnfajqllbG4XSpFjH8cnAosBpGOonu7O1TdcXUKf8CzXE3Wr6lqLkyXDIp/hTgVVW1AIZ2LfWq5CTrZfFFijFYhJKR6Lis248T6hK2beKONPpmsrbg5FPXgUcokwlu9RuSfOu32/3QaiWEL2/GpCM0AHtVDQoQBc4oNPGelIRSFYBgjA60gJzjaRSgcBhS89aY0gFAzknFGcnGDSgc0DDaaUICM5pN4DYLfhSnHXqKBCNzwAKXNNJGeOfalzzQMKA1JnNKFoAGOSOaQnApQB6UZFADcEkHcRjsKf1NBxjikFACnAOBSY7mg80YwCeTQAc9qUU4qdm7IAzSUANG7dywxTlUuTjkCggYowMcE5J6UAGB3oY4PSlbIOBSUBccCeoprMSNwyTQScYyMd6auen6UBYASfajAOc8Cl5BxinY4oHcTAHAFFLijFACE5GBS0Ypf50AJgk57U4DnNIr7c8ZpoJx0xQIcRjr0pVAPemnexy2PwpSdo5H5UAKRz1ptLgggEYzzQcZoAXFHekwc470ZoAWjtTScClB4zQMM9BQARnNBHQgZxSg/LnnnsaBCAhqXHpSKBzge9Lng0AKSePWm4oJOeDSkc9c0AGabnnFO43YpMjOO9AAowO/wCNA+6KMcZJooAPpS9elJxnHakII/GgBxBK5684pD8v4UgYkYIxg0UwDJxx1oGc80mefenUCGnI6UoyF560ZHQ9aTNIYo9aQ8mjmjgdaACjNIGJxxRQAtJzyKKUUwDIwc0Dk4FIQCRkZFBxnigB2aQ0gGOKWgAP9KjSPH3myalx6HmmZwM8mgB3G4Z4zSKef6UnX0z70ucHI78CgBSctkUH7xY8ZpApABI4paAEPII7UABcKBxQOTx+NLnHGKAE24OcmhuMe/SlJ4pjgcYOCeOaAFAwtAyBz1o/hx+NA5xk5BoACAcUm1c7icGlwQ3tSggnBoAcMZwRmm/xEA9KUYDZJwKOQ3p70AKOM55pjj5eCRmjcQwzzzTPLO4lSTnpQBIDjAPXsfWmnAOc0R528+tK7Kqkngd6AEA3oc96FVYEdieM5pytlQykFWHag8+mKAIjJHIVCyDLdqeAYxwaYsaI2FVVzT0bfK8eCFXg5oAUvvyF5x1pc/TNNFvGkokXOQeeetLjmkDHKPlx3o3ZGAaBSMo2kKcHsaAYhBzkHFSdaQEq6tkZHX3qQsHJJAB9qQxmccUzy1Ds4UbjTx1NIW20wGu4VTn06etLGcxg9M0EZ5p2ByeMe1ACg5pDyfagUh60gFJ+brxQcHpTR7jtR1XGKYhwCg7v4qQ9aCTszjmjntQAqthgcZx2odlyzuMd+O1NYkY4oxkcjNACqQRkEEU5D83QfjUUaLEMKuM1LHjPPFAA+FOOc9803G3jueacHDAj0NNbcWUgZ29fpQApyR83SlBGM9aQngnt1/CmscLlVzxTAcTxjtR/KmB9wzSg8UAOP6dqbnmlPSkxnNAC7vmB6UE80h9DSdhigB3YeppATk9aYAVJ+YkUucYzSAkpe9MA5zmlb5SO+fSkAUcE80hPGaRfmUMO9MBx9jxRkAc0Z3HgY7UHnimAgPcdDTWQPjcM4p3sKQ5xxSAANvA4p3SkB46UZ55pgKDx70pPSmA8nnignPSkIk4KjHUUHoQe9NHWnM2ADyTTAQgYxQM9qCc80gyDSGhTjOeSO9IAGYFCcYoODkZ5oA4piG8nIoVSFKk554NPC45NIzAHOOD+lIYL93B5OMU3AD8dfSnkoMEtgg8cUH+9QAdO3NBOBk0A5pRycZX8TQAmQacyIVySfwobg7SORSGgBCOMYBp6q2OCB9aaQR1pN3rQICSH2kfjTgCT1pu/5sH86XvnNAwB5p5OOgpmVzg8UH71IB4ywPtRhdvXmm55yDShtvUZzQAppAcnA60MRnIGBSAjqKADJyciilzTc4oAMDvSjjocU4MM4YUfLng0AANNctt+Wg8DgihW9aAGgllznNKHbvzTuKCoHK5/GgQvzMA2BtPQ0FR1HFIBtGBwCc4ppNA2xQcml5PSmA80tArj1GTQVweaZyRwaduOOTQADHbBo27SHjZkcd14oyQcigHJoA17LxTqlogjaVZo/wC7IMn866Sw8X6bfNsnzaP6SEEfnXC7QKjK5NIVj1XZHKoeCQOvqjcUZbo2CPQivLoJ7q0bda3UsXsp4ro7HxpcxKI9Rg85P+eigAilYZ1nyJzFLJG/oRSzxx3kJS7tknQ9TtqtY6tpOpr+4mCyf3JCFNXWR0H7o4pAc1deCtPuGY2M7W7H+EjcP1rFm0DXdEy1uszqOrwtx+QruxkHMqH/AHgKeC44ilJHZW4poDidP8aXtuBHdxLIg/76/lW9aeI9Iv3OXaGT+7IMD86u3WnadeKwu7FRIerAc1z174G835tOukb/AGJMA0AdjHc46MCvZgQQaiuLSzv8/aII5PU9683Zdd8PTDcZ4seg3LWtYeNjG3+k2hb/AG4sA0IDSv8AwPaybnsJzbsf4GyRXM3XhrWbFS7W7SoP4o1yK7ux1qy1FQ0Nym7+452sPwrSWbZ/FiquM8g84dGUg9ww5p6lGGcCvULvStM1NT9ptI3c/wAYGDXPXXgUdbG6Cj+5Ln9DTEcdyDTgxxVy/wBE1LT3xNAxX+8vIrPV8UgJeTTcGn7lI4pCGUcigBOKaynPHTrTgTnNISc0xjBwM0FieaGNLsIUN2NArDcnNLSZoGKBAfmFKBxjOacH7FRimFSrZU8UDNd1UOWIqygDYIx0phVScGp0wFHIAqLgNPDYpHO04pDNGznBzio5G3NkUrlDWXJp5HIPoKh8wg4bg1HIzu+AaLgXDOqrnPSlS4DDJrPVJOd1SJhTlnwKBF43QFNN2WGFxn6VTe5t1z1P41CdQUf6uLIoAu7pj3zSKjNy8mPqaqL/AGlcNtjgZffGKtR+H72Vd1xJtz6GgBxmtRw06/nUMt3bq+23Pmt7dquQeHrON98qmVv9o8Vfis7WBspCigdqAMQf2hOwWJDGPWpk0m5dczzfN6CtsyJ2FQtfWqfflQN/dByaBlWDS4YiNy5x61dEcS8BRSrIkyhkzg9M0MUXqyfnRzDsKYUPO0VKDgfw/jWbOuqXORZgbPUdazprS9tx5s24yr6nNTzlWOjyxHygGoJJSn3niT6tWLDBq95gJHLk9MHArVj8OXMMDTSsksn+03Sp5g5SzaS25Q+bOjy9gh4p01/Z227zETcOwGayX0zUriYCBYoF/X86tWfh2WScC6uJJc91Py/jVEcon/CROTi2ti47AJV+C+1CbBuLWO2hAJdnGCBSOllpdwN0iqg6szEflWNq3i4NOx0sb/l2mRhjB+lWBoeZe61PugmTT7JSAJ5flZselM1zxFp2naNJpVpcSajcvwZG5A964e8lv9UdPt95JIinIQcClihjhTbEu1fTNUSOhXah3D5jTxgjkUnSnctjHAqgG9BijbnvSqpGd+PanDAOKQxABtpV4GaUZP4UDpQAmGo2mlL8Yppbj1oAcx6YFKSB19KaEJ6mmiNSSSTQBIZfl2hQKbkjrTcc8UuPzoEKCSelHfk0o4FISO5oGGRTlwATjOaYcDk9KDyev4UAOLZozim4Bp3UUALz+FNK89aUsc8LmkGSMmgBRQTS4pCAetAC7cDOKUjtRkkYJ+lJnnHegBRkZ4pQQB6mk7GkBBGQaBig54xSEc5NIuxQdoJ9aa0jMxQcCgCQuAp/vUhz1xz6ULx1xTiM80hDFDYzjrT8YI9qCWJz7UA0BcOtKKN2KaGzyODTGONJjJpAxLHIxTiQBmgQEqqDjBoBOB70nBAPBB6UuD3oGNDnsuacTkUEYAz3pB1oABx60uacjFCSO4xTaBWFwTy3ekK4NLjABz+FIeelAWCjHpRinA7ecZ4oCxGrAn1pc84xxSvIuBuGPTinAAigdxtKelBO45JpKBCr0x3pDR0o6nigAyPWgjoaagOMGnds9qAE5pc96KTntQA7gnuKTHGc0DnOfzpMcZoABkHnFGaQfexRQApo+tLwfwpvU4pgG0Zz3oFBIBxRjNACk+1JnGPegnigYIoAKQ9KB94jpQeMjrQAnHJz2oweKCGBwCMUopALxmgkLzSZA+tKefwpgIjcZH5UHk0vJJzQMdulACqaTp0oIxSepoAXtTSTxjikYsV4/OlGCOO1AhCCTwKdwcp3x2pOnPpSkd6QCYxjJJNKCKBnA/vGgg54x+NOwxAR8xFGcjNHfPrQc4OBk+lACAUDjIGCeuKF65PBoyN27nIoQCMPl4UUKwyFwfWgFifqacM5waAF79aADzznmgHqCMc0Y25oAa2Bzg49qUydBtPJxmlPA5yKYc544oAVhyDQyl1KinNg4wMcCoJJkjONxz6CgCVECAgZP1pxAZTmmxMJISfemgyiTjBTuDQA3zYraIBmwo9qmVlZAwPBGRTFRHjCkBh708jKgDoO1ABxTS2JMdqeFJIHrUalJMkZOD6UASHr7CgdSevpTJYxMgG9l57UQx+Uu3cW56mgCSlI4zjimsSMYGSaCTjFSAuKVjg49qarYIJ6ClYA8jpQADrmjGWFC8H2oJpjQMTG+Me1J3B7UoHqKczbh+lAhCwYAjH4UgwcjOD2z3oUKBjH0pCPnVvSgY7GBTSSBnGaXOacVIAz6ZoEM7UeanYj86Oe1NMSM+SvPrQBIcEAjpQKQEKvsKNwNACig9KaxYY2jPPPNOx+dMBBx2HNOVyhyKb3P6UYpAK7bivbimhhu2g9qUdaRgRyBmmAYFWblLYQxNbk5I+cHsarUCgBQQCMmlPBpMkdsjvRkEZFAARnmgHqKQE4oOccdR3pAIQD1FJkjtnNOwSAc54pvNMBDkH29acOOaQZzilIxQA5Su7jmm/hgUBQDkdTQchsYOKQC4xSd6XOD60HnoMUwFFGKaoJ4NByDgdaAFHWmvu6L1p3QUFsnnFFwGcgYIwaB8uD2p5AxSAAEjt2oEBKkcE0oGR1xSbCVLAZA70LkNzQA6lGO9AIPFIaABuGoOcDFISS3NOB2nJ6d6BgCSuQOKTGeDTuACFJxTaQCZ6qRSqOpz+FLtz8w7UjJkA+hoAUZ9KDtbr+VCnPUdRTSvagBcdOc0uM9aaOMUuc0APPNNODwRSKdxwpBPpS4Pfg0AJtpfwoyMU3JoAdJgbQe4pfM3ADHTvTevWlAFAClkLAqMHvTSSaNozmlwKAHqpKkkjjtQFQclj9Kbg0oGTycUgHbkPAzmmsAM4PWlKgYPWkJHagBSSfvUm0A5oJNGaAEJwaByMigkAc0CgCQ4YdcEUwe9DEYHY+opoJNAh5NNxxSqBQ2RQIQDA56UEE8g0m5sYPNOU8GgY3cQMUoOetIeTSFiOtFhj+lNGQc05GJ+7we2aUl8kHFAAzg0m7KgelJtGeRS7SOe1AgPFOI/Kmk80n7wdfu9qLADqMZH3h0NaOn+INT07asVy7RD+B+RWdkmlIxRYDtLHxtayny7+Brc/31IK10MRiuIhLbyrIh7qa8p5qS3nuLJi9tPJG3YBuPypWGenssy8EcVG1tE2HX91L/eWuSsPGt5A+NRi86H1QfNXV2Or6bqi/6PMof+452n9amwEhlu0+WRFuIh3HWs240LQ9SPmNC0Mh7rlTWzs7CopVUEebDkf3l6imgOJ1LwRfW37y0lSeP8d351Rt9X1jR5QplmRh1SUcV6GjbGHlSn2VjTp1ju4jFeQJIh9RmpuBzNn42tGYR3Nu9u/99Tlf5V01tdR3UQlt5hOh7p2rnL3wPZXLE2Nwbdj/AAtyP51z1xoWuaFMXjDbD1khbj/PNUmI9NEoxhgQD1U9DWfeaFpGoKRNZxhj/EnBrjLHxne274u40uU9WO0j/GumtvEemXKrsvVRz/BIeRSuBg6j4Iu7eXfp8izQnoGOCP1rCu7W/sMC4t5Yvdl4/OvTknYfN99P7y8irPmR3CFJFEiHqrdKaYzyASHuaeMkV6Be+D9Ku8eTEbZvWM8flXO3ngvU7NC1vIs6eig5qkwOedeM0KSB14PUUSpcW7FLiGSNv9oYoR0K4NMAwccd6O2CKcFTorH6Ggqe1ADGDEcUgJHB6VIPemN60Aa/mVG8znjOKKKzELDZzSrvQcetLNC1um6a4CL7UUUhi2tzpQiy85c+pFUptQs1mJtx+OaKKYEC6hJcPsgUu3sK0o9HvZlBnby1PbPWiiqGXItAtwwZmc4960Y7S0gbIjB+tFFICbzf7oAHtSF2PeiikIa2TTdpNFFAxwj3KQWCDux6CsmXV9G0+6ZY1DOPvSYzk0UUDRI/inR5PvTMP+AkUxfEOgZyzO3/AAE0UVmbl5fF2gW6bY5Jseioahk8ZaOPuQ3Eh9lxRRVmdyu/jexH+qsJ3PpuxTB47BGBpcg+r0UVVhXIl8cTeYGOmoR6ZNQ33irUb9PLiH2WI9Qmc4+tFFOxJi+UCcsWc+rsTUnQUUVRAnJ7GgB8DK4oopiHY4NKCQtFFACZBNB65oopDFVwM8A0ZJPYL60UUCAEYoIA5oooKDJxgHijPrRRQAvO04FIMsue1FFACHIpe2SeaKKADPvQvDE9aKKAHYzS4oooATkkelLiiigBQvvRxRRQAEd6TAI70UUAHHAP400/eoooBBtJUgUqjAoooGOwO4zTs4BHfsaKKQgJO2mqpycmiigQ4JgdfzoGMZoooGgIyKaFY8dqKKYDuAAB2oB5NFFABjJyaXGSfSiigA4FJ+NFFAwo4FFFAgJxRxRRQAYHB9KO9FFACFRgkE0YyoPrRRQAvQUcEEUUUAIBgdaFGBgdKKKAHZU4zxTAxYZAoooAAeAR0NAI6UUUALnGcd6O1FFACZAFHUUUUwDoAKM8UUUAJjNL3xRRQAdDSZ4/GiigBqknJPSloooAT5SRn7w6UoI5oooAUHJFL3oooEBGRTSONuOtFFAwPtTQuOe/rRRQCFzkdKVScHPaiikApGRnvmkwvOT2oooEAxS7vSiimMZuIYY7U1HkkckpgdR70UUATjABzimZOc9qKKQBgn880Nyce3WiimArBu/4UDO0ZIoooAQ1C1urOWJPNFFADwPLRTtP4U/vRRQBHDIjRnyiSqnFPL/KcDP86KKAK5mdmCYPPWp0BVACOlFFADsU15Y4+HbBPT3oooAcSAQQDRkmiikAuD2FNSEKNzsd3pRRQBIOlBUqwz9aKKAF6n2pDxRRQAdaX86KKARGFw+4cmnghductnPBoooBji27nGKYTgZzg0UUAIF+XBJIo6CiigBR5mBuxjtS8460UUAKKMMScdhk0UUADIykbgRkdxS7sCiigCPJL4xQMjdu9eKKKAF3AnGeaMjOO9FFAB3oXPc5oopgJzxjoablskY+YUUUAKTx70m/OARn+lFFAAD8wG7tgE0uemfzoooAceDRnqRRRSAcM7STTM8+9FFNgLk9BS4yvIwaKKQCE0hyPxoopgIcg9/rRnjk80UUAKQcZHSkU84JxRRSAUygDG3JoDhwCOlFFAAc9qdyaKKAGkE8AkH2pwJzRRTAOe1KOQTRRSAaQxPHPpSYZfvAg+lFFAC7R1HB9qeSCBuHzevrRRQAhUZyDxScZx3oooAGYZ54peAM0UUAL1FJ0I54oooAGOPunNBZvSiikAAk0ooooAd1FNoooANisMMcGkG1eM5oopgO3R96aWG7C0UUgHbTgHNMZNxzkg0UUAPAyMd+9JjBoooAO9KRkYNFFADcYOBTue9FFAATTTnFFFADlwfvDFKWA4zxRRQAiuAfWkzRRQIAaeAaKKBgemKi8v5t6/Kw5DDjFFFJiNix8VarYqEeX7Qg6CTr+ddNYeMtNu22XMTQOe7EYoopDNryYZ13wsjj1Vs0hilTtxRRSYDCR3609ZpBwAjr6UUUgKV5pGmaimLi1RG/vKcEVzN74IuFUyWVysy/3SoB/nRRQBkx3Gt6BKQTcRKOoYfIa3bXxtG/F3aBT/fjPP8AKiimgOisdYsr5R9nu97n/lm3WtET44ZWX60UUwCeO2ulxNHFLnqSMmue1HwVYXbb7aQ2svbacj8qKKoRz194R1WzXegFyP8Apn1rDfzYW2TRPGf9pcUUUAHmIe/NAwaKKYH/2Q==" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAQ4B4ADASIAAhEBAxEB/8QAHQAAAQUBAQEBAAAAAAAAAAAAAgADBAUGBwgBCf/EAGEQAAEDAwIEAgcFAwgECgcAEwIAAwQFBhITIgcUMkIjUggVM0NhYnIWJFOCkjRjohclRFFUc7LCNdLi8AERJkFFZIOEk5QYITZGVVZ08ic3o0dXZXWhswkxSGaVpIWRw//EABsBAQEBAQEBAQEAAAAAAAAAAAACAwQBBQYH/8QALhEBAQEAAgMAAQIGAQQDAQEAAAISAxMBBBQRIjIFFSEjM1IxQUJRYiQ0Q1Nh/9oADAMBAAIRAxEAPwC14DyYlNqh0euMHGlgeGBgvQ2GC4TWAiW3xkpj86C8DUo8N+zDMF36SC6HOrJMNt73ear3oYAGGgH6FcGmjZzWkPLtQyYxn7PNQgBwHVoDZwUR6HmvUmox7wU0FX4aJqzZ3gCPMDAE6CQJ0AWe3oTeYZ9ueCqXrttVk9N+s4f9ip0mGxJPx1hL5rdVD/kzaNHN6Q+eByDDYCbHRWeRnxdeDKB5o/Iq+NDfjO+BEXJ9a7uFbTTESKc+cYZvYdDyGxvSEqNeqNTOv6MCFFZPAA3702Ou3IzUplGdg1Wmgcc+s3egF5v9I2GxJtdqlRKGcl6nM64SGnv8C55xs4tcSbwaOI3UZkCF2A0eGsC4PWJlzxoAPuXNMMz645mf+PNNjqFKoM77HtP3HLe5LsA9+CyQUedZ7suswaPz9Cm5701Yb3E285UKzojZ6L+Z7w6AXZb5s+sW9HhWHrg963MAAw7z+hc2GzGcB7efOpHXKzKkxqfnnAkGexk/IrasWxcd/wB5Tork7COweGYdBrtFK9GmuUSy2rOcwntH45um9h1/Iuf8VOCF8WNbJ1K2bukzI7GecR0wzAPkTbFxy26Par3FyPZV/wBVjUqFFk4HIkdB7Oj5F7AvzhX6O0zhzIp1uW5QZ8jrZep8nOVmvI9jcNLc4qRZtSrl3THrjAD+6Ge8z84K4tu3n+G9Z0L4lyaPI/o0tk9jyYbEdNr92lLtV+VUo0czwzMMwBc0ueNSuHtUl0q4MLheAMGeYeP7t9C6ndtbtWmx5dz2xfdV9YbPBBkzBebLqqTdyVYqq2/ra+8z85phRrxHmtSXLM8NgZ+RHG56ThTYj5vAfufOqzNz2bimxnuWHmwcMHQ6DBQtOk0d+j8u++GAP7PzojptS5hphh+HGde6DPoUWZUpdYlBKnP5mAbEzP8AvOB54Yec0QtodpVWm1mRUqrFZ5dhk/GZezBaC2+IsGNICh1kD5LskBvN41j6bPrlKB1yLWXg1wwPeoMAIPMNesmzOP34Bmg7M9QbuueFnc550oAM4Bn0ACynD26qjYd/u1GlTtFrA4pyMNhgol2gxTWo/qC6pkynmGekfYo8CsRKVb8g4mGqeGww60wNXG+xz10O1y6q+E+PNz5zlz35pUS3uFcziR6qOoyXqFKZ8Ez2ABrpfBn0daddVzWvWLjqUOfQq3GfMwA8DZP51yniRZ8Gz+NlQs6nPmceLMBgM06WPS3f8s1Ds+y5vDVy3+fahTDCG9rAYf7ayU+7ZV1erzfpUaAbGyGbPvvkNdq4Y3bwTepb1OrnDVmffdIA2ZJx42ySHnw6F5/4tVulXbdrs6zqGdEp4f0Q2cDZfTSXQLVh8WIzEis0OpPHChHvpJycww8+C7tw9ueucS7BicOQgvSfVx81JN3Z0dhrx/Ar1Ro8X+bZRsyO94O81KpvEK/wYlsQbqnsnKPM8O802t+i325g8NLZClN2AzbzU0wihLjvBgb5qVavAezpPD71VWaBTalUHzzN7szXgKw75o1KuFpy8X6qB9bzLpm8Bn5zBb22OKN/vX1IuOzqxWKPRJTPQZnypn8nYunDVu/Slo9ncLqRH4eWdTZNEkTXgOpNYYA8Hnz/AFrvvDTh1RrA4QU+h2zUtYKvoPyZYd5n14Lw/f8AxLrlw8WqZVazWTrcekTAfzke58/+Rey7DsalW8xNvi471CTQmDCqU2JngEYz/OmBsI3BywKVTZE6h0N6HNlBvkPH3+c1zS5PR44t3a1UIv8AKHAptElM+M00GYGhurjAxxIpZv2dxDptBkRXt4dYST7M/wCNYmz7/wCNNHqhxLxnaNKmhhJAOgz86YZOW03hc3SmnWLqt+M9IB4wA2XvG2dBqPWOG7FYd9auV8Aaf6OYS4hXPECsy4sTmZlQOf1+QFnLqs++IdBOqzqNUjaDsDoBcGGqXTXufrcuyqrWTqUJ/AI0gAzwBYS/LMYt6suwWJQPNfJ2K4tufBgHHfguHGmh1gYdivZh1ifW6Y5UYMPl3zw1g8/z+RVscitt6nQLmiRJbmcQzwP5FU3DTX6bW5sR9voeXWarR2A4quvvxMI/4yqZltzrhrLr4UeTJaMMwkAGw17sRbGh0bSaOXEZkg4zm80fWz8/zrQPQGP5zo8T741Tmdh9+CC57GYskaPc9uTpLJygwea7wMEEOf6qzNxgJJ1SNvjn5/rTYepvEX/kLEt9tt7naWeAGG9aKpXIxclzRIj7ByY78PADDsMO9ZK0qO/DCpy5fgxwe3vdgLYWZQQjXk1r1xl7wdYMA6DUbFPTZM6z6pWKVUvBzjYYecOw11Cw/srR7GBuVVeZ9dhrvfWqzidQefoYTrggg9UGA0Y0uP0fQa1fB+yYkmkOymGI3LzaacWeb3QD/ZgCx4Y/qdzpFgW3/wDavOsVJwzmymT0TM/J0f51rbG1ztSnvm/mD4ZgqWzJhybIatGW5hLpf3I/nw71rabGbgQI9OY9kwGAL7HiGSQAYoDD3iPoQLcJGgSUIGgNGkgiGhw+KkYfFMmga6zSw3o0sM0ebBh8UYdCLD4o8PiraGjBIATpgkgaee0R1EVKqTDxg50KJMDNHAgP5Z6a1c/hswZbOKBgqySzu8NS4ZmDGmkYLJ0eFJJZNV5gr6Syqx5ncoQrNFCbKsNFBorJaEEZEEbcrAGdiRsoIuh8iZOMHkVnh8UBsr3ApXmfxFHNlXT0ZRXo29eCq0W/60yYK15bao70bBBWmCZw+KsDZTRsqFoRggw+Klmylh8UETD4oDBS8PikYIIRggMFNME1h8UEfD4oMPipeHxQaP4aCLhvQYfFStFz+tDh8VKkdBh8VIxBDh8UDOHxSw2J7D4oMN6AABHh8UYBvRYfFQI/eknksPigZ7ElIw/ES7EWaBGCBIEBojSw+KSBoEQda+pKEHUkgSRY0SFErBpB1pJKA6CMPw/+JACMFYdR96BGoBp0E0nQQGiBCCJWgYIwQAjBQJCWfwQIwRZ1GCBGCAuxPAo6dBEHUkKJBIRoEaLGCIEKIEBgadBNAnQVoSAPepAKOCeBQs+HWpIGoOfwUgDQSANIE0kCITQVgz7JVrKsA6EBGaE0jQIAQmaJNIJAHijBM9CPP4ItIyBGBqPn8EQGpDxmmTSTRmqCNGCZz+CPP4JsSM/gln8E1n8EWfwQLP4JZ/BNZ/BLP4IgZmo5pGaZNAJoDSNNGrCSz+CBJABoURoQQIASw+KdRgG9AAMo1Iw+KZMN6COkjNAoDRqOnTNCasR0jRYfFCahJlCkaBWoefwSQYfFJQlIz+CSBDn8FYdUuL/zqCCkRnkE5RzNSM9ijn0IABLP4IMjQ5/BA6jz+CiZ/BHn8EDppAms/gln8EDqPP4JnP4JZ/BEEZoTSQIsefwQZ/BNGkgBBn8Es/ghVoEkhSULJLP4IElaDufwSA012JZ/BA6hQJZ/BAZpIEgQO57Etiaz+CDP4Kg8gz+CAzQ5/BSCyNDn8ECBAjNIzQpKgCSSSILrToJB1o0CSSRoCSQgaLP4L3wEZpACaM0gM1utI45UGdWIcKpQQ8WK8C6RGe1qdEc79FNPBmiAMA01kEZ70OfwQH1pKQs/ggw+KNJXtBo4zZ+0XwGWwRpZh7xxNgwTUypcsemqmtz32WvAMw+hQqrclGClg3BqTJyz34Z71mtcHMfZ621EZDObzSp7nmQaraER853LSwe3nmsud1PxnYlOOpMyfGDf3o8wkXhUp1S4hwaHr4RD8BZq5OFbFq3gdHYDRo9UZ37Ovzq94hVJim3XQa4/siPmAcx2Au3PUqm1imhz0XmWsPBPPoNDDjN4cN6GzRqFR6bTmTkRTAwePeZrmvpOcB7cqs2iVKncnSjwDnwPYGC9JHACZXGvvRs6AYbO9ed/TGk4TLZCc+D1PlSdB7A+sEesvbFNfsyvR4lnP/aF19noZPez9ayoHd0zjhR4NZpTz00J7BvRwPZgvTvD3hpavDGhzZdpeM7KZ1+beDeH++a4/wAMaJOh8S7g4h1+lSXpGZnA0Qz8c+xFvYtSgMcrm/KOM0AABmC8e8e724oXVKes6xKHGjRIuGEh4972xdtsyTeMy3KwxX9ZnM9dnMOzyLk/Hvhpesx2n3V9owjNSj5U47PYGzoU3DO3iSZJue3q8b7j5w6nFMwN4OxaipT7uvyBH+0F6szMDz+8Ld3bwWfOm1ipQdaNHp3R6w2HJ+g15yjco8/oTgNk/IYLJ66UfF2q2NSzsdhim1UAAw5vDzrk+s286bgK2ks0MPYN7lEkgb2GDYbPICz2qLQgBwzwNE9GlmehE3h5E9yz+k6+23ngadhvOMkEoOtW2u0I4z8CU03ObNnNSKrAOmytDMDHDYYd6CpHVarM09M5OgHYCiQNOTMjxZco4zT7wAbwdnzohZ27BcuOUzR5W5j330K95aJZlbPQpwT4QdBmG9V9et5yz5mgxUmZmuGbMgA3qqB45LR89Ke5j8UEHRbhsOj1iF9p6PdQcuHtomGZsguayY0RmRoc2EnDyIqbDqMmVyjD72kftsPIpfq2D7vM0E2lXJX6Ph6qrkmM0Hus9n6FIgc9eF5U9iqzt82SAPSMFXs0edJPBgM/zqcFsPhgc6pcmHnzW2x3CZGpXBas1unU5v1kBsg8zUTPN4FYcE7bsC/6Ndt8cVGHnnYR+2A8MA/3MFwR6A/Gw/nx6SHznmrCmvXBGiyG6GckOaDB7S8i47tGGw4o2lZUOvAfCG4HqrQpQZvAfYazVq0EKlWZFNiygjVBhk3wA+9eiuDPARy6uC02XUjOmzZTxnGN7oPeqW3rA4c3hbx0O5m3qPedIksBJkAeASQzwP8AxpFrz4cBk0Gs3JNC38P57fe0AZ+de5rbB+leiXcdnXVQwptVt6GcU2jDv6wNdKpXoVcHbABq+4JzAmxQ5rMzz3ryV6SfF2dxOuH7I2JBqT2BmEmIzgYScOjPyd6+lwsnQPQS4LQeIVLuty+LOkvR6vAwptQeZ2Yd+H8Cs7w4b8K+FdjVW2eJtzSfXAAYRojUneYB0GALKejxcjFmQKPQL14qV6iQpRvmcEHsAjH2B8ma3fEuzOC1nnL4lUq9Tr1QlMnycSdJ5lnMAUXCtvGkCBZVSt6XUp1Z5aoB0MnsMwQBdVcuGFHgzrxnhEDo39ALQUG/+GVYvKq1m+7DCHHqMbwWmugH/OsLVaaxlqRNkd88GfoWK3YKxTaNDtelXNbFYjHW4QZm95wVlbHFrijeEP1UxTYYckYHmAdfyLktVuGshS4tDl6PgABg6AYZrZRplx29ZUS46bXKUccw5KScc9+HnMFFgbzuSXJdqBuUOmhNfPM5DR/5FkWarKCPEbfnPBvz2JVulG9NqculVXmQihnnH6DWfpszn2gPmze+tcdwOkWkb9zyJsqq1U2SA8MwDrVPUqqdErciDBrMxkIp4bDwzR2ZXm6bMjwZbAcu+9vMFVXVSqx/KDIYbiPSWuwwBOEXsM/Ca9ZSz++mZ6J9nkUc4ebXPN9hmCVbqrE8omg3hyuw06zGlvMOxG39HZmyB9Bn5M1oLCHc7HquoWdTYP8ApQA1nZAbzDyKLAB+0qodNfPB3RP9CO2I3rKa1Sqzoxt5hrAG8EzcNVoZkcuDUuZlsSQDMwQdIkzJ1btenvnWYxtPgAGHz+RbWxqlUqPYPK1X7toT9HPrPBcChzOZpYU7T/ZT1s11vn6rVZVPrj8QzhVEAAwZ6AfXR63+Zjyu60qTFAY77eGZh1h3rRxqlmWnqLkNsVUzhajmw+jDyLd0SeD3tF+pvh/o5o5mzz+CADURmSGHWjA8+hcPTa9pGfwToGoWspDLwYeI4sbi235SEkklL0CaMFLAN6A2cDUIRDSRmG9Bh8Vaxgj60gSQFh8UYRs0CeA0CCABn4nQrBlltkNMFEz+CPWVoTc/gkaaA9iI1CzRqFJDeppqOYKBEw+KWipfLbNRIA3rIADOAJYfFTgZbQGzsWor8N6BSzBAYIIpgmTZ+RTtFBh8VkKw2cFFeBXDwKCbKCtNlRzBWRsqObKLV+HxQ4fFStFDh8VCkTD4oMPipeHxQYfFSlHMEOHxUgwSMFQj4bEGHxUjD4oDBSI5gaaMFLMNiZw+KKR8Pihw+KlYfFDh8UEfD4oMPipeHxQYfFBHw+KPD4p3D4pACgNYfFLD4qUYIcPii0FDh8VIw+KDD4ogBgl3aaSWHxRZI+9NYfFO4fFQEg70aSsIARgCQAiQJLsSNGCgJGgBGrBeH7xGCBGCA06BpoE6CgJOgmk6CtB5EhRKFhUgEyCNBIRgo6dQOp4FHToKRISSBGgJGmkXeqEgEYKOnUD2fwSSBECsGCMEAI1AdA06CjgaMEpCQBowTKeBX4XhIA06CjgG9SwZcxRCQye9WAdCr2WcFYMnsRZIsPijw+KSgNGoilmo6BZ/BFn8FHSUiWCNRwTqoAZ7E1n8EjTSIOgaPP4KOCdBFnUefwTSSAsgTRmkZpozRAuxCiz+CWHxQR0JmnTBBh8UDSQdaPD4pYfFWEaDD4o8PikAb1AWHxRdHs0Z70YAeCANZz+pNGZ5qQbKjmCBkzUczTxpk0AmmTRmmlYSSLD4o1CUcwTSkGmc/grURprP4IzNBn8ESWfwSTSSB1OgeKjgjz+CCWB5J01EA8UesgPP4IM/gms/gln8EBAjz+CaBLP4ICNDn8EGfwQIJGfwR5/BR8/gnc/ggWfwQ5/BLP4IECQZ/BIzQoFn8ECWfwQZ/BWEjTSSIFn8Es/ghz+CWfwRZJZ/BBn8EkQPP4JZ/BAkgdz+CWfwQ5/BAZoDz+CDP4IDQmqBZGhz+CDP4IM/ggPPeln8EklISRpAkqQBPACWHxToIA6EZo0AIEjQJIEln8ECXYgXWh6EGfwS616OoZ/BLP4IDTS3VB1LIE1n8Es8CXO9SEs/ggzzR9C92sutRJ5sRo+u/wDkUs9hh8/QolVDNgGz6M14hT025KVMq0igTg5aQAZ5yDwA1yLi19meSOpQZz0aWweez3y6m9Ybdw3HEqTcvR5XszwzXm/0hKrLDiIdpU2ohTcDzAz6DPZsQQftg4BQgnS5LITujmOjNZ/iXflVsl2OxSn9Gaf5wNcf4nXtWaxKChy3A+5HmDzQYGsUzVbjrFRap3rF5488w1lztXcKl6QnElmjHbNwU2HJafADZdMN4H8i7Bwl9Iq4GbfiU6pVKNGwDADkbwNeUrqqtYxhRK4/GMoXRoqhk3VUjaOJBl4B9CD9K4F5nWMJVGlQ5hhvNmIe8AWS9K6BBufgs1dT8Q40ilnmzmuCcCgptSs92c3UnoFTi++B7Bd4vaq2PfnDI+HL92gzImnhqnvMPzraLSwXCi9p17WLHp0HiMzTarF8DkZfvla3J6QNf4b6VlcTaOzGNt4H4FZpm9l4w/8As1YWr6Hlh2Ha9Tqtx1mTVTYhmbMgDwADD6F46mUq+OJFblsUqJJqvq7PBnPMwBB76D0k7ZrdvSKdSokmZNlRs2ZGYAGf+dcova8IN7W/bksLqnvR4R6FSjxwPMPnzXlmj3CfDe6ACq2y9sAwkx5exeheCfEuq02RLo1DsNlmlTXuaCXL6wPvVMWm4/WlYFStS34NncYgmSH8848uZmH5w7F5P4rvMHMjuSnI3rNhnQPl+g1sPSTtj13xVkeoKMDLvXMNnoeDzri9bjPsyAB/rDrNTdiw9VPvcpoPg87KPDAFa1jh7xCtuHz1ctGZDj9GZ+dZeeDjMIH4NSkm6fuc10A+It8VW0mrfqVwTJ7UoOV9XSw2fXmudUMvTWX5LDrmmYGAZns7FXyXvu7pt7DA0EmZXKbnBlgbIfOo7xuYaep1qmizsCq1KBXjlMP4OgyeGzYolzzGKrVtdhsMzDxtEMAzVarK2JMGBWQOqsGccwMDw7EUjwAYZd1H1q+G9KtyvXNUINclhygQ9cAPZvWUqvK8+bkGXrR+zZgrCgwDn8wDDjOqwGeBnvP6EFrb09uBcNQCDm80AGxsDeYK7uSsWPPpceLTmHociL1g91vfnUulWr64kOsWzF9T11hk3pPPbAMO9S6Pwfqsl865f7BxqfhnzYHsMEGHhhOMmp1HfBkOwzNFUjzA4lVfZe7wMOw0Zsxaa/6tCcEmmB0SA6zQHAsc3TfCqyWc+xZBW89Q2ZWpWYJyQMDADZ6wPzrcfZu8bedZbt9wHvW54RpAdeZ+dYo2bVhxzfiTnnjDzr0bbFSpV1cG+bf5aNWKdGN8D8+/Ya8sWF1cWuKno9w6Jb8t+HPA2egwWQvzjfwkur+fItKqTMh+Ng9hDwAD71pYdSoHEXhfTLxfgn9o6Iy+zm90GfevMkyewbWhqb+8OwFtDJ1CTx14rs219laHxircyjzWcPV0jDNkPJmtHwZvCzuGlpVh+sNzIFymBgByI2Z5/IvP+ejH5oPagYYLaw+JE6ZPpj9zvsz6ZrfeWTDBdPcPXB2TSr/9Ehq5jgnMqEp4358hn22eeC8mXJYbFEtyFLnVWpU05QGbOsGzPv8AoXdbJ9JaXY15VO0bYo/OWZW8DAD2erTPYZh8iY45W9Y9Etc7modY+07sV4JTxm9sez2exNRna3lSmxtYQcNw1q40a3GYBxJ0p46gZ5gYdCVw1i3Kw/HC3KH6tDR3h0ABoLYsyuVt8zpVK5/1d472D2GALG4U1tvcOqNcNIrb9Rqpw5dLDWD52FS8Oq9y1LqdGlQYcmFVD+8svB3qw+0lfuSou0OC/Gh86fKhns0T781IZ4eyrPdCnVGUzJ5p7Bl1rozXOKzhW83DduiLzbLIcngzn5wzwVFRLM5+BIqrcvkJYPbDz8H9CrLho862K9No9SbwkAeeHyGtxSmabP4bSGKbmEh8NZ5ozzAHwQZcGbjptRNhiC89IimBsmz0Ga2FeuSv2w/BuB+D6tqz7OZhhsmAs/aVSqT15UKD1x5QaEwPOx3rW8QqVOuEAg2rLN6j0hneB9hrH/GM7WKbO9UQrjgsPSfWgG+YAG9k/IirYNxoEKc+5gZgAK+pt1QQtyFTtCSFTpx7GcOtXFEtinX/AE11vX0XmM+WjvB3/WuvwM/YdKl1iBU50oANoGT5YAPfmHnWgsyyabdVvR5dZi8toGYGYdZ/WsYzz1pT63Rm5eGbJsZ4da1FsVJ+NalbgnL8Xk9mzrW2GRq3qVFkx5bDcoJOhJMPC612CNSqkzYsKhsH4rB5mqThXwZY9XU+uXA3JekTQzeZe2Bh2YLqXqpuBHCKxrYB0Zmu/wDhvrRF7ty3zM7AjcnFP5zM1sKbRLj5Vp9iJtfDPNVTNKfmSAYbBdqtuM3yDQON9AYL9BzX4hyObyQrlKPTlAf5EodzuB4ZvruDMNh4PEYA/rWHrfCuhzJGbDeiuHudHSxn2k/4PxwUuHcge8fWauqyazaroOagPRD6MFVU05Ux3TYbzXd1cPll/V1iHVWHh61ZMnmuZUGZOOstRH/zrrp01xlprT7wXzvZ4fHh0RewxgzdTrzLYI42myW9FJMDXI2tWmCjmpb3WokkH+wFT0AKR2KJD1z9u3gpwAlwASz+CkYfFBiCxws0aQHvSMMkgwZ3uLZCaAYp4zVYdbYN/BhSDk7VIdQoM/gln8FCzwdCIFHA96dA96PLTmWUUlnYjZMMUnjzFQvwrzQYfFPGG9Aj0BgmjBS8Pij0W/60Fdh8VHeZVsbKDRb/AK1l4GfeZ3KIYK4ks4ZqteBaUhBME0peHxTJgs2qOmsPipCDD4qEmkkaSkR0k6gNUI6WHxT2HxQGClRrD4oMPipGHxQYfFR4DWHxSw+KdS2K6EdINROn1r7h8VABBh8VIw+KHD4oGU0YKXh8UzhvRaKYGlh8VIMEBggaSw+KLD4pfu0DSNLD4osPigSMASw+KQAgDD4pJ7D4pACBIu9LD4pYfFICRggRoHg6ESaRIDBH2JlPAiDufwRgo6dA1CzoI0AIwQJSATWz86IP3iAwUgE0CIEhCRn8E8Cjp0DRZ1CBpJAgdA0YJkDR5/BSbSEaAEYKgSeBMowVg8/giBCiw2KEJAGnmQ3pkAVnGZDYrWdjM/iKwBkMU0nWehQBw+KkACIAzJPYYCgAEjSSRBowQJ4+hMos0mVIQqQAI8/gkCJA0fWo6kGCDD4qrDSdz+CPD4oDUg0s/ggQ9aoJAl0JdaIJOgmk6CAcPigAN6dw+KPD4oAw+KEwTqjme9ABpB1oTRAgkAG9SE0kZ7ECM1EeNGZqIZoGnk0ZpPGmTNAJmkCBJWJYJZ/BM5/BCZ7ESAzTSLP4KOZqAZoEKBWEaMECSA0WfwTSNAWfwSz+CHP4IM/ggdz+CWfwTWfwSz+CA8/gkZoDNAgeSz+CDP4IDNAYHvR5/BR0SsSE1n8EaBQEgSQoCM0OfwSM0ygWfwSz+CEzQKwefwSz+CBJUHc/gl2JrP4JIg6BozNNJKQRmhz+CAzQZ/BAefwQGaFI1SCSSSQElh8UYAjw+KlYARgkiQGCNMgjVA8PigQZ/BJEDM9iAzQ5/BBn8EDufwQ5/BLP4IM/gtMLAjQI1mh0hAjQLoeQFEhQLJodA0MyZy0c33HMABBn8EnmQktaZgpuBkrwOpXhb0hinOSYdQhGEqN9aCw+Iv2wh+o6y2ca4IWyS0arOMF7VXh1ToX2fYDVf6zP8i5Uzx7ptnuyK5dVGhs1gwwZaD338C8HoC57hqVjQmqwxRufazwMF5i49zKHxjvmj031GzRJuGZmb2Zn0dCr7q4zX3xds2p1Kj3dSoEeL/0MftjD615sn3DzOBypUzmA6HgPoRC142WA3w6u31dEfeeaMM83utVV4WNUrJpdMuNycBtTQzAwDoR1W/511HHcu6UdS5UMAMwwPBK4bwiXDRgoeZhEYPMAMOhcltYtlKxJbmO64VJ6YfeZqJAefjSld23bFVr0g4sFhkzDzngrO87ArNn0sKlUdHAzw2Gr8LQabcjcDNj1k8Bh0GC0H36q0FqqsTpjxv8AXh2LHhbE6TTWqqw3mD54LQQ3p1twmn6bVTz72T3gsho2eLvFiNRjsf15Jep59jrxngs1SpN1WxVmqxBqvISA6DzXSKVXrVmQmn2AgPTT62pfWueXVbEGm1QHPWvrIJuGYfgmq2N3cPGN+twJEWssQK3LlYGyYM+MBgucyeJfFSfAOjsTpgR/3LIAap4D0Sg1xp9wMGg61fSbtqpzWqlbnssMD39a2i0XDVWxxvlW3bwUOuWqE+aAGAS5fXguTzzfkvum+fWea1cZn19V+UrHMw6gfQZ7AWXrAPw55wXwwMEteB29RKlWJ+nTmweMAzwM8Ep9e0X+Ulng6Cjxpk6A7rwX9F3zrrtKvDhszaRuVzhPzmuGEmczMA3s/OoMOPzDfqUjUlSzP606cYAa6wNV5vRZM12XBA2YmfggfkR8y5npgslk8zv2KWzDYMPElAyq96YbPW4pAQ6zMpbtZbgvPRIp+MYB0Kwy9pgpcaj1E6b9o6a4yGh1gfWClhbb528FcbnRjz/o+e8FEpVVnUebHlNv/d9YNYOwwVIbiNXm7ho3qa56bP8AXcoAONI9jmHyLLnPrlNaOhy6rVeX745vHgvUH8ntGh3pTZblGkzKFUWc4cvP9jM/8iteF1k1yTVLzg/Zz13T6cD78CX+N8i2Y9ryL44eI+wYR/OadPQ924pd4VWdPq9QpUqKEaOEkwCIHuQ8ipDXI2aJlm2ak+epO0Wg64+H+dXdSmPwAaobGsETDMDAM9FYeNhzAA+eiB961cY+WYBvm3ng85oJFpXhUbepcu2arVZMOO+ebMhkOg/Iq9mm/bm8IlDpz8ZnXDAHsOvBBWJ8V6UYan3fyL2x6E/o5WkFhy+JXE2DDN2bJzgSDPfGAP8AfBWl434i8Irj4dNU+XUX40mFVNkZ1r8T/c1kgZwdOJODNryL3L6RXC67b8uMKbQ6lSp9EpZ81GZiHvA9i868UabRrtrwMWdSt9OZCLJ2YZmqGHgcUb4pRnTqbVdGEAAGGG/APn/OmbnuSm16qesac3JZ1wzeaePYriNYbEC2qhWawHLTdHYHXvVVVabasa3KPWaVUZIVV/wKkGfQaCvhsvyfZtrV2x61okKoaD7wHVGdEAA+9ZcJlSpX36nMScw/cre6MGm2r6xiPm9mAGYGfeslGp71OuGZCiU3BmoMRsDMA7wVfWKVzNqu3B68mTHQe0DB49maiVIwCa1Lgv6Mhg9+xWYWxcEmlncZxDCnv+R7YZqB0XhFcluVWyrgcv8AbgPS2I2gy9OZzMw8i4vb1S5DmIIOYNG8eCuwqRxoERhjofMweDzgnarw0rlNo1PuqjUp6TCleTrQWFt0RgDO4NTRl0s/GiAHZ5wWzsCmxLbolwVL15DqXruGZgEc94B84LlLNYrEB2bK8ZmR7HAw2GCto1EfpUgZVHrEY/OCzsK1Z7EO7QlMRNFrDAAz6F1rhWzcH2UdrgUo5lPfnmHMAe9lcngQ9aYDDf7W/wBBrsfD3i1att8OaPaNx25Jn+rpj5vHHe3gB/Jgrixd+kJwfo0AaPcFAc5Z5/wJJ4Z7DwwVI9wHqVvWbIqs77zWKXh6yiGeGAH3gvQvDq6uHnFGg1CmwQCZH9j4wbwDsWgk8PalUprTcsGXqUbIMyQz2PAAd674hGGF4FcTqrdrv2Vua3GY3JQAfhyOjMOjoXQLhtJiSZy2GFmbStj7N8X5dsxW/wCb4VNzhh3gB9mf5F1s42GxdPDfWi+FzelW8xDdOdp4PdC1FvHKNrTlMAHkVqdNYyz00YRmw9mtubmR0rCNp6WxI2c00yeClh0LHZhSXDR4M+luhLYzWHoln06G+7oMdZrpFYD7g6spRJOb+munh5DBqBYBw6iFVfY8IFdTJjeWmtBnrRwVPJpTG999LvsREYUUysU2M7p6iMzUgKVaTP3ucwgjTKNJlaDEHAPP51LwAAlGmBrmw23+sFo3mWIA+OGxZ+ZcNvzJWnTYj2fnVvYHJ0Ajn4e9VL2eeppqdMBzSzBFDkxTa39gKmiKBuPYaaJ4HGR1HG01GewdzbbWghhrRT1+hQM0Elsx1ANWEbDS370E+Gxq7AwTQGYHggr/AFJFjSjlN55makKaajoEBozNM96PsUrABqQD2xV70lsDVaFbw94q6We2qZk4KwAwe9mslGqWsfWtBAk4DqJfCvaw2YabiiGCE5J5oNbepejzwS1kGQID2LESM/ghM0znsTWfwQR5moqx5XEzTw/9fkVa8CCvMEsPipBhvQYfFZKtEMEJgpBggw+Kl5aPh8U1h8VKw+KaMF4pHSMEjBJFASR4fFApA96BGgUBJI0uxFgBJIEaADSNOpIGksQRosPigjmCaMFN0U0bKCJh8UOIKVh8U1h8UDSSdw+KWHxQNJIkeHxQNdiIEeHxSAECw+KWHxR4fFEoDSMARYfFLD4qwOHxSAESWHxRB4EkgSULJO9aaRIHgRAhBIEDqNAj70EjsSQAnQVh5ECaz+CNQHc/gjz+CZw+KNA6khRKxIBGCaBEgeToJoE6CgSGQ2J0EIdCkMsmgKMCsGdiajM7FK0XP61aB9alxg2KOHWnWTwUbWswAMEjBRwNO5/BAOHxSRZ/BCpAH0Jo06aaNACBGkaAEkkkCQokkEc0KI+tCiDvQCaRn0IFSyz+CBGaBAgToIQToKQYJGiTTx7FQA00aAz3oTNASQGmkkQl5/BAbyj5/BAZoEbyimaRmmTQIzQpGmVYPP4IM/ghSTYLP4ITSTKA8/gmjNEhRIM/ggSNNZ/BFDSz+CWfwQIk7n8Eajo8/ggLP4JIc/gln8EBZ/BLP4IUkBIUkkCSTuHxSw+KAEkaNACBLvQoCTRmkZppAGfwSz+CaNJWCz+CWfwQ5/BLP4KkCSTSSkOo8/ggz+CHP4ICz+CWfwTWfwQZ/BUHs/ggz+CDP4Ic/gpEgEkAIwVAsPijAECeBASRpZ/BNGaA8/gln8EyZpAanwg9n8Es/gmTNJXax5/BLP4JoDSM14gWfwQoAR5/BAWfwQpZ/BJWsk7n8E0iAFCHSM/glkCj5/BJaZDyDP4Jdiaz+C0WdBGCZAwRhvPw0GV4l2wFyUuO44GtIi9nnXPuN/CK1alwvj1lyla02KAGB59C7Bc952jZkIJd1VJmH5M+9Y3+Uti8Kcfqq2QmUp/+kAf+RZ2PMMngm3UrSm1m37SqQOgzsOOeZ/oXHbb4M1m5LNrF3MVhll2iZ5wc95r9GrMht0SjSJ1VfCM0/nvePAFx2ts8JKVQa3b9OvyADtXPPMDwXMh45DgzeL3Dl3iU2wyFPDsM96iWlY064aQ7gB8wAZhgvXdHt6j0rgjVbKYuOBPwAzZMOzNWfAGzLSqXCh2mhBwkMAffvA1CoeF6a9OOYdOg8zzHfpdaswh8/N5G9arM0mOjM+g1oKIdHsnivIfrlON7kpJhg11rpt4cMbAuez5t60184cjDMMOv86NHNIdqsUejNP1/CpUIN4SIh+xPzmsK8cQKocSC4Zx+zM11W1eHXEqsWy7XLDo896OAZvd7LweRc/pUNiNXgcfiHAlxTDNl7sNGG1nUplv23pMHQ+ZdMMwkAeeB/IsoFYfeqWo+5mvSvEKybA4o8PoU5ilBbFdhbzMPYvAfQf8AAuRcQuFdOsmiR6lTpzz0jPAzeDYaltFuZVg9Z03FKptScCOEFh9ll7sdd6Fpvty5TbSOgMU2NsWMjUfmY+vK2NecFUNmwrdYchxY77EqNJqAPAs1cNSqtSmnOqrbOqfkBQWY0Fnw8D/OauGfU0mmuxX2MHex7PoQZ3nH+w0EaHLqtRCIw+AGfeZ4KRJjMAB6Dm9RMHPeILCq0GXQXQYlPsnn+EeaOmwGKlTZFSYf9geGGCl2fXqNbdZ56uUAKrHwwwPrD6Fpf5ujW5Nr9Ncpp86ZmdOAN8ZSMfTZLgSDYYiMyXXwwADXWLbrFKtK0JAS34z1QfZ8aO70GCw/B+3rOuq+mqHfFZOBCP2JgeHj9iuOIVnsUepVCjUOjSZnq6T40vWzwBBj7bpXr6ohTuaNl18zMADo/QpdYsmpQKbIqoPxnmop4HvWqsaz6dAuiRRrxgyQmnGzgPRz71F+x/FGvc6+3SqlGoQPZvS5wYAyCpk6VwEvb7KyGrVvSm1tmRN8CNLdk/csOxeiOD/EK56JcdVsMJ0Y7ahA+YPRw3geHn715fonG+4KPaB8PJ1No9YaYZMGXjZA9iet7jGH2GCx6bQzgVV/wObM8GT+c9i22ww5deYcneVdCdK1nTnv4PGG8wzWakvObMPPgvSdV4A2BbfC8+I1wXMdVrAAetHA+/sNedHoaxuHZDfcSOCF8cNItErNYcjTKfUfHZeiHnh581Evat0OZPiP25KzaNnxmcPYmum8FuNlGZt6Xw54ty+Zt842hGkSN/LB5FxWvQKNTbjqdNtyohUoQST5OWAe2BRgte0GBLn1mDKi0OZWI8V4HpkeIzmZsB1r9U7Sh8Nqlwxp928MrcjPQpuHWeAMn++Xk/0BrMl2fx7pj921UKbIqNNlHDiZ+2PDoNdw9I3gtVbevmIdgG9RKVdZ4PMxDwjc0HnDNeocs9IesXBwxqh/Yrh5AZp9RwD13TD6w7wwDoXFLbe4cwKoFSiXGbMiVGNmYEvf4/nXov0k6PO4V8IKfb9s25Upk3DmpNW7GT79n514qrFvTo0WOcquRuXms64PM78FQm3VZl40qQydZnZ+tHvBZzzzBUk+26b6hkNsS2Yc2EeeB96eC5LgmYRK5VTqUenHnGMPcqhvB5upPhWWN+zB5BqAv9ut2VL9a1XOqts4AGHQqe2J78xj1bLPNo1lWZmcc2POpEM32fYOYKLgbjW9cPy3+VDMDwMwVlb14S4whb7++P3srOsyWAYdYB/e+GB7O9V8YH4c0H2H8D86kb2N6qqVUhORaU99ye1jAzw2Lqd7cYKbb0iFOtWJJCOfgSYPZmvPkypTqPPhVLmwk+N7EOtXtyMy6wwc7XZez7wNBJvY/XFUOqxKcEY3wA3o7XQAKNZ9Kg1WbUHJcs2WjZwZMO81bcPalBZCR61fjHUDjGwyb3QAKssat06iVmRTq4wB0eV91ez6w+cFnYuztKdGhA3zcYN4BzBnsBdCtu2KNbfEa0jcY0Y83NmSZnmyeYLFU0NG6DtiXUtaEf7M88ftg7FtaratYqVGo9qwXNE4UnXCo94B5Fj4F9TYDnBbiJU7cnUp6fRbr+9U3lDPNnA16q4Y3DUbhtSFKrLDwSDzzM+9eeuEUy575uq3IlwU7Ot2GD5vG6fg1KKfk+ddlqv2gtuvVi3KA+bLWAVeAAH1h3srv4B0b1bE9c+vDD73o8qB/IpBpRpjFVpcesQW/ClBnh5El1hkwSw+KeSAFCLMgypzLO1AAKWyCPQHD1mtNwFlzoLFKI3GPOtqCiz4zbzS0i0M/GqrABpqvuGtsBFBts+tBPgPxs1SVIG3oum42Brs4XOr3qrSuhyVmfkTwa7J5t7PJ8iy9YZfjZvsdYK4gSaxMs3XP9rzwXdjwyaAJjZtffqyD0jvzNQaO9nWQYwXOodNqUzSqPQ73/OtxZM+W9Nd1OzYmBtaky+AeAH1rBVKY+FU5TPBdIzz8M1WT7ACTVGqrnhp9i5Iaq2gg48YeGtEGwDBPMwGIY6YBgmnjbZPxFDSFZPDAjNUhyfFU2sTwDNUuea2ZrAJnnRA8B+zVE9JMPZpU2f4pgadL3a9SM9iaz+CB49ijDRTz/am4s088bLp/WtBP3mqSezmK7IctrCmycDBxtxaBmsYdZrBRnjAxbVwElXcIi21Ce28KMJKy7M9wDBtw1bMvZiuO4dMWumTzTqqmZOHvFICTn7xc2G6QgQmaZM16GpLya1s0El7s/4VEB7B3evMCWgTyBRcBrD4oNFS0sN6lUIJgo5grY2W1EMFktXmCDD4qcbKZ0VAipYfFO4fFLD4qREQp00CAUjQI1AA+tJLvRosXYkh7EYdCAkaAEaBYfFBh8U6jAEEQwc/DQ6KmoMHDNBCw+KWGxTeV+KWigr8Pijw+Kl6Lf8AWg0W/wCtBH0XP60tFS8NiWiiEbRc/rX3D4p3D4o9HNFomHxRgCkcr8UjjOIgzh8UOHxUjDYlh8UWj4fFLD4qRhsQaKAEkkkBo0CNQHQRggRqw6kmjRKBIBGCaBOggdz+CPP4JpFh8UwDToJoATqsPAiBNAnQUAwBSGU0yG9TYwK0HQBTov8AzqOAKRGDo/41AuGQDFTQZ2KPGDPBWHQi1YbOBowBOmCFSHQPYln8FHz+CMDQSM/gln8Ezn8EefwQAaHP4Ik0aBJGgRmgaRIUs/gqCz+CSZz+CWfwQGms/ggz+CWfwUgzNAZoDebTRvIDz+CDP4JrP4IlSEsFIBRGVIz+CLSM/go8k0jNR3k2BM1Hz+CRoM/ggPP4JZ/hpnP4JZ/BEDz+CjmaRmmjQIz3pGgSz+CsJAkh71ARpAgSQHn8FHM0Roc/ggBIzQIVYIzUfP4JGhw+Kgs6aWfwQYfFGrSSSSSBJJJIDz+CSWHxRIBBSABMgCkh0IPmHxSMEaR9CCOkkaaNAjNBn8ECaNAZmgQGhz+CBZ/BAln8EGfwVhd6PP4IM/gln8ECz+CWfwQpKkCz+CWfwSQqQkks/gln8FQSJCkgkAiUcDR5/BegzNFn8E0kvEHc/gln8E0kgWfwR9CBI0B5/BBn8EjQK1jQJI17gJJLD4o8Pis2ewI0WIJYfFaQuAmCIAR4fFHh8V49bhJBn8Es/gqwDz+CZM0Zpo15CCzwTsZ77wGooRmkBrQeafT5058e34rB5u63R+RV/o8WBxDqtO0JVx+pI7AZgbJ55gut8abGg3zIpT85/A23uvDNbiz6JSrSpHI05jANiDzp6RtmXbSmIkRi/KlWM/H0QPA+xX1gUewPshRIl22/Dk1B/wAA2TBbqsRmJNwVNx9gD8HvBQrDs+DMqxuS3we5UMww86wuBScOuDljyb6uDTpv3QN8YM+g1sKDW6baR1WCdNCGyxvZ0j2PAtHYFny6DUqhUpz+ZzVlOM1ECsVe3KNKN4I808HtE8DwUYW4pRKVb/FHir9uK4ECG1TpJgcQAwN5dZ9Ie+bVoNq+oKVbEOZIq4YBgzhh0Lkl4cOn+HXFWPRrZA5Md/2OZ71WSYdx0HjnT4t8SzjQgMDjc30YYKMDNcNOPd6cIgqFo0Oj5wnzMzZeD2P0KFYFvU7ipfU77Wy8Jsrx8DDADD5FZXPZj928a51OtyvnTYT4Zg8HfgHkV3Qa3/J7xDCLf9KOTTw/Zqm8yGAH+RMPOJn72sOq29PCjxbq5yj6wMgAHvjfWu6xuANGuqwJvD2qzpM92LG5qG891sv/ACLzLxCpT/28qtwU2cYUea9rw5DPQ8uqyfSKfZNp+2X5lHuDRBk3noxvA8AedQ2hyyHwZqN/gdneraVb1Vo/tpcvrM1y+/LVuDh1P+zFVqsOTmGecQ8wXpi27JqT1XkXVf8ALgT47+b+qD3Qa8v8RTiSb1qZ02U9Jj62zPsRKnjaekfnBQjefkn4f6Eeth4akQ4D8zPlAzwDM1KohBZB9kPHUiBAqNbnhTaVE5mQfQGeCGBGcn3DEprjmx88DWymWr9j7qjv0O42T2Z7+xF4UVesy6rYhRJ1cpXLNSujfmolHgVGfK06dBOS73g114K2uSTc9Yr0egVyvsvA/JDB0z2AtFb1nzrM4kUQJ12wKbClHoHUwe8EAPzoKrh1bFq3beXqO8Zb0Y3wzAw69fsBdLuG3r44OSpcRhwHmajG0QdBnMHg7AWg9JPgDatpWRTOIdpX5AnyAMAOPHeDOT84GCxlYs/ipYdtUq47mrLMynvhsjtPZmAIyO+jNVWKbxztedVa5yzQPH+0bwPZ0GvUd8vQbzvyq2J9qjgR9bA6cHQ8C81UelWPfl1SAlmEOXyYPRnQPAwMErJh3jed6VO437/wrtI8BmWYZmeCqBQ8RbPodBvSXEsPmQahGbJvPBsz8ir6O9VarHnW/UtEDDxweZDoNXYcRbnuq45dsXM+AZvZ+sGg6D86asmg3VclbrtNst+G86yz95ZkbDeD5FdqYfnJfMfzjWZLzR7D8ZFUqPg61yj4HHfDNk/Oq+pQJcCfIp05jDQPeCe5xsNJiWBm1n2di51wiTIzkN04ksMD8hq1tWw37t531bWAgSIoAYfOo8/1HJd1PGBWFsTKVR5jrlVbP1fKZNgzaNBfBfN6H6sr53GYVWg+Ay60GBhgvcvC7ivxG9JnhADFxvw/WFBmAYVEAwOYYefyGvCT1HocaltVmjTuZdzwOJnnsVrZ9buCz+Xue1bxegTTPB6OD2zBWPe1sWHBuEnZ1Vr921J1jMHvWL3g7OxeHK3ZlKklcESlVz7pS5j4aTwLqtmel1xbsmjO1xuseuIRvBnBqcbo+gw6FxHiLxRuPiFUrgqVuUPkKVUZnNPaIbwz7P4FSEW3p9NZ4c1NuD9zqufjPPdDweQFip9K/ms6lz0bZ2ZqPzP/AAclyir8PigUZ5xmQDmnmrqHvmNMfnNU7LO5WYajJgx5+tFrujw3588IrDeZmguGlVKj6TkpvAHzwBaV4AgSoj9OAGfB3qnuqpHMadYf9rsw2LnGXk75oMLR0SqsQ5TtDc2A+GAfIs1g5l860B2xVYxRKlEb5wHwzzaQWvLMW9VDpUT7yy+AP+LvwNWf8ntwT2IVRpQMzPXZ5g0B7wPyGq9mTWJLAVGdSsDYMwwP3wLrvox3JUaxNqtDfPZFAHmT6zw8iz2MvcNKg0qVEp1RbkzMIeBg1vOAf+olDvy5qI/EtisuZws8AnfjArCvVKUHEiR6xYjQJesbBnnsksdizkxl+sHLodKzmYPZ4AHQCzMNXTeItx8NLoo9cqL4PPU6TrmABgZsd69J17ivaNyT6FxKg3By1Cfjck9pb5UM+wzXnkAnXUQHKpRnT4QMRZ72jmYGfk/37FCptgfZgKxZ11UCSfJHmEtreAeTP/fvXbw25uX9D3LatvTqJVPDvGTVY80M2Y/yGtgYYrxFZnHW8Z79qTnG5kmPaEkGJMvR6IWeGBr2KzflpVj72xWIzPyPHguyHqzToJYZgD7bgG0fQYHmkBqhIAFLAFHZUjP4KbWdUczT2fwQGohSvksg8GDjax8+juBKNwOhbh4FXvMh3trqZMUdH1utvenY0DlmtDDYtA9DbD3aaNltX2IU/JsYG3oBgaqYENimynW/PvWoNlU9SZ0TN9vrXXFs7hYQN55q29ZaOzNcvqV4VWM6bDAAH0KnC564coz5vrV/OjbsvPtvezTUwM2lzQLznQ2uvNPQ7tlzAPUPBPko7lvUmc3VWmeikzUje8TUzTux72ivGDZGAPNbFHCHh4nkUgGdEdNBvyXjxO1nDFMmaENgpl41lEPdo8nTNU8xlzerB57BRze1gPNdnGxtTnGAD8NxGGuH0IzTobwAF02yOwPGa1O8FdxtQBVJA1AkGHYrhl7aua4dfGl5uYoAmOAhOSGCiPPBn7Rc2FLONJM+/NSNbD2ipIDzfO6as5h5taijAjz3m3g8PrTOfQaj6wGfiJZgcgG2DwW2BbMvbQUgFEAMBR62C5rhpCUAJYfFU51Vxk8NRS4dSbeMw1O9R81tE096aMFIDeKDD4rHAi6KWipoMowZbUrVhs/hqI8y5kr442aiPRsFzikNlxNYfFWrzOY6aicu4ikLD4pYfFTjZUfR3qQyaEFIMEzh8VCyw+KWHxSRoECMECSAwNFn8EIIkB570YIABOgoB4fFAYI0asM4fFDh8VIw+KAwQR8Pijw+Kdw+KWHxUIR8PinQBGnQVrLD4pow3qXh8UBoI+ig0VLQKBFw+KWHxUjRS0UETQ+RNGAZqyw2KK8ygiIwSMEkBI00CIEDyJNI1YdA06Cjh1p4FAdA0YJlPIHQRgmU8CsOgCkMqOAKQz1oJsYFIBDF/wCdSgBQDBnNTY8ZzZ4a+wGQNW7LIeREJDLKeNM9iM1KzRmo5mieTRoAQgkaBBIA0aZBHn8ECz+CBJCqBJGhSNSAQIzTRoAM0BmkZpozQIzTWtvQGaDP4Kg9n8EGfwTSSB0DToJoE6BqQ9n8EWsms/gms/ggkGexM5/BNZGgz+CqEJGfwUd40JmaAzVwsGfwS1kKSIFn8E0Zo8/gmTNAkkKSgI0CAzQ5/BWCSTWfwSz+CgIzQZ/BGaaRIkKIzQqwGHxSME6aE0AIMPijw+KaQEkhRIEkkkgeSTKeA0DoGjA0zn8Es/ggkAaWfwTWfwSz+CADNNZ/BIzTWfwQI0BmkZoEAoEkBmtUEaFEaaQJLP4I8/gln8FISSBGCoJJLP4JZ/BACSSSBJI8/ggw+KIHh8USEES02skfWgSWYWfwRpYfFHh8VphAEeJp3RSwwUBnRc/rX3D4p1JUs1h8UsPincPilh8VqEAIzBJEHWs8M9hw+KLD4o0a0XsGHxSw+Kd2JI9aPP4IwNBh8UCB7P4JKPn8EWfwR5ZYfFNYfFO5/BDn8ERhEkg28GD4Zgj7NNvoSMMyRMs5oYVUm3qVPka77GZmnQC2LJYdrktwIzQBvMzUKpX5blvVT1VVX9F1cs4o3bSpNRCq0p8J7RhyuAH/AJO9ZLw1FV9K7hrR39Nx+ZpfjAzms1WPS34eVupNOWzTpNVdi4YNaPX/AKi8/wBSvCdbcybbNz2yBhUAzZMPIuf1J5+xuY5FwAzPMMDzU3zPXc+IvpRUq8H4/KW4dHkU57WB43gzWZ4ncZv5V7cpjf2fPmKWeeqBguS8LpMGq3eblcYZejgGvvDoNSOIV802BXnYtq00IzXvgwwDNR3+B6F9Hu8Lcu3VpTFpMhXYQGYSHjMzeVfxCufh5fNS9VXVVarTZEJ7ZhG2MmuKcPb84h2BVwuZiKDMd4DA8w6wWzva57HuqQ7UWIMkHZQAb07DYBqN9iregKJwugw6MEq4LmjXVChfsbJs4Az+hc04tMv0GrM3jZb/APN5s/fGtE8GcP8AGq+z+N9NgU6lQaNQzmVMHuSkgfR8hrtfF3jHR7P4bM0a52KaFVqgYaLIfdgUWRbxZdXGaq3JcOvQ9GABhgeiGwz86qoFKg01+XKuPYc3ez9C7GbNj2xdtPqtwSqJJt+qM9eHQ+ayMy57OpV5SG7YoDNYaM8wA+xHrH/ybuT4ceog49y74ZmcHeAAujTOGnDmfpUphyTAmhGz5ho9jxqsk35bEOVIqoZ0Sq4YBHANi55Xrnp1eh4P1F5l1joCP0GpVCvuc8PAOhmy7FPA5eHtlndZt4A1OxTQqU6THCDOnGcf51EkwwjSAbblg8BoJbJxXnY79SfMI4PBngfWHkXe+JFgcErSatS47Huo63zWBz6e8ZmGHzrgIGxpMsNxADTPPZ3rZz6rYHqZr+Yz50GcMw2Ite3VdtnT6k0xLY5ZoPYhHPAABbOpXbTr/olEtW+Ljhs0qLhFhuxz8bP51zRm2/VXD6PdUuJhI5nMA7DBU9Eh06fM5qqy+Tjn1gyCItoOJFh1XhFc1KqsGXzkJ882XTPfn5P41HpXEKnQ68dx1ij+K+GD2j3pPMwTCR6yqNSn0yKyZxgz9iaqguGjVK0gob8EMw9jLDrBBs7brdDmVkKlYFpSZMgAM5MSQGw/nWy4YwIrMoOJNov6NwQph85ST3g8x9Ci+j9xCYjQjseWwATYrOtAPzh86oeFF5sU26Lg8M2axUTfBmO9sZM1Qp78jU6sXlValSqwDzUrOaYGGBsn5FjZgYO4Le3DD4e00JtNvGDJC5WHjz5c9hgfQub62ayWdZhvz6o1BBgzaPrwDM8Fo6xDp2g1SrcbekzQPANmz86sOEt8wbAu+n3BOinJB4+Vwa6wz710Ct2TY958YjpTd1eoY9X8Z56OeYAeCDk8+z6jb0gGLnfOmuuBmyAb8wVZDrZw/DcYB761bcXaJ9lb6m0Nu7TuRqLsCdnmGCx7L2asdgrHFqnXVFj0P1UzRMwwkzg7wVDGe+x78uh2rWY0mPUQzN7RzMFhWXtaUDAdfkXQLGuem2fNOcFKh1XnWTZMDe9iaoYeZJfB32kM8++J0GoIHmSkT9A5siUxs13jPS8iaZ1GXdRtSHcMBzcV9asBusVmEAb2t+t9CrwegmH3rP8AIeC0tvTKNDny2KHrHHNnrdPM0EipG/JuY6Hb4Z4Kwv8AjZuwmNPxWGd6j8JeG79SaC9LxqRxrfB42T5c9+zz+RKmxmAfqst9iZPp4SfbNBmBh51nYxjzL8N0H3AWjt6YxMHmp1cegaHWDQbAVhc8alck1UqHRzepUoMwddBZ+iSQjMTQ0/2pMDSvQ51vVEH5z4SQcA8HgPY8C01pXC/SqlTLxpVG0WooGFS0u8Fz+HUpUmkxKbKfzCDnh+daKz58tm16hygZtRXjPPPrWNwNhMvm2efkQbtg5xH3jfgVNkOszVLwouf1Jdc2pf0eVmzqn0Ak8DE/1ZTaMYcu+Zg8Bh0K1tu2K5Z7FQiQYvORHzA8GT2GG/rUeB0Oz63WXqo7Otidz8cDYOTBAN8kAXQOK/EudSrgduC34utSrvgAelhvZPvz/OuQ2kb9Bu0L1sN9nVp296mGeGYd4L2Hw94XWPedvRLgqtvxtUzM8D6wPr/zrq8Dkno5ep41Uvg5YG9ak2NFCZHANgGfnXTq3wroDNm0yDbkU6k1Ce5qBme/QM/YfQrKzLMfsm9ZdZof7FNA4smP3vYdH6P863YUr1bNdiwWP5vcPMP3PyLoiGNwy9t037PfdKOBs0+a940Ez/YzWlw+KPlnwdPBzBo+sPOj0V0IwQGpAGom/JO4fFHqRnmhN7Akzn8ELwZ717C0gzyUR4E6yeAaaRg2a8FeYbFFMFYGG9MmCJwqjBRzBWDwKOYLeHjn9w0dsHTc5U1l5MYIzWux1rrsyMDzWDizR0SL3sL6Xrczi5oc0OYD3tHFbW8YHqsd/WCvalbEExNwGwD6FlzZ5CVoaZ5r6UcznWFNecZm6Di0rOxY+N1660cOS4bQaixuGsWuNYDR4gqcJjeWnmpsY8/oXHhezr2wVXvPKRPNzS8NxVoPb96RCik74pm31qpeeNXB7P8An2KvkwMxPT2LshkrwkqczvUT1DUTa1wfAwSgG/q6DjZ5ArRhNjMnqngCswZPSRw2fu/iJG83nprmtaO8exV4PHzGBqbJNVUx5tmQrgtass5vg+C0H/D+xn9Cz9KmN6QaiuHnvuuayv8AeuGMqs9yBKwc2K7jRpWQynGDADBNBoSZrXNNgZ5962Wi2bQN9gdC8tKnA/xFFmTA9mCdrepDjnoKi5nMNRIhrtBmPfePaJRphg77RV8+UeqorMwwdXf0ubbotNqTZtbzVqElgwzWFpUkzz+c1expjgGuDm9Z0xzNAy9m7pqwCNms/GnsaoGtLADWjg/2Gvm80OmLPRowZIpNKzDUbUtlnA1NANi5raMVJhmDvs1HOMtbMZz8PBVr0besxScp+7Uc4Gz2a0bMPMvZqUdHA0GHOMfkUfRWun0fRDUbbVO9DcUqVOihMFYHGcTJxksQsN6PD4qRh8UGHxULNIkeHxSw+KAE8CDD4o0gOgjTOexPKAku9JEgHvSSSVoAnUKLP4IsefwRn0IARqAykjS70CBSABNACdBSAMNiivBsU01HeDYqFYYIU6YIEAokKSsECNACNQHQRgmgToIHQRgmgRAgkAjBNAnQViQClx+tRAVhF/50E1lTmVEZBWbLO1BIgZgrpn2QKths4Kyz+CgIzQmkmTUgXlHz+CI00gBJPJYggSSSBAkkkkCSSSQCfQmTTpqOfWgaM1HM9iI+tNGaoCZoEjQIDSz+CDP4JZ/BA8Bp3P4KIBowNWHc/gln8E1n8Es/gsgaWfwTOfwQrU2dz+CjmaWfwQKEDz+CBLP4JZ/BAkKSBWEgNGgNT4A570BmjTOHxVWkkOfwUjD4plFEmkaDD4oDRJpGgWfwTPejQYfFQkjSNGgNWBSSNI0Cz+CWfwQJIDToGmgRIDNLD4pJAaA8/ggSQdCBozQIzNNK0EgRmgNUssPigAEYGkakNGkiQogCSSSoLP4JZ/BLD4oFINBn8EsPijVAEfWlh8UeHxQLD4osPijw+KPD4r0Bh8UkeHxSw+K8QAwRgCLD4p5aYWDD4pACdNI1o8wMNNAaEDSM0Rskkkkekkl1okWWHxS70YJYfFEDSSRI8skaDE08AI9aM2XAQGBq1NlNGys4WpzBBsVm9G2ezUE2cE2gKDD4pHsQZ/Be7ZhRMng6BpZ/BNZ/BSunF+PdvPvTZtZ0M2tHMF5PetWsSbgjsP25J5c3gzOO9mC9+3nTWK3RjivrKnw3gwOSrNGMwdYw8I94foRo8mXnZLlSuGn0qcxWDhGAAzmfQCj8Y+F0HhWFPCDWTnx5ufhPdYdH+uvSvGmzwuenUqnRKidNqAPazJgGa5LVfR1uB6gyKlc1zyZhsBmC5LgcBptNYN0NNvxX+gPOor1NinKdblBJhyGO/sXdWbw4c2ZbwRHwCZIY7DZ3n+dZSm8SLHqUKoUBi1ZNSdqnYfWscHhy+pXPXJ7Xqqo1LmQDo2JQJNV5DkQf+794KXJoJ0eradZgvQ3j6APyKverHqqebfIvPB8nQtodFtXZl5yrAnuymIMaTHlBhJB7r+TBQb2rzF1UnXqVRek4exA+wFnZknN3NR+Wfkh4fQrQpAZDMHHMzwPvNWujKNh2osZ4MAo7zLbJ4KKcyWyem25sWQuKbDbqUA5dS3x/n6wUGeEGMAcpKA2vkBRHpL+lscwWmZpUFmzQnTm85b+9eYIZfRzHPsUczYAw+RaMKbSpNrnO9ZaM2KeyPh1qPTa3R9AAqtAZmB8+zNeibDh03V+0Fv0qpT4gftOzoRxrVrNeZCowX4zIH8+aCBfNVoLU2Dap8hT5vXH6/D+tBZlzwbeddYuOmhUoT2eYZotcXtVbuZqUJuU5zLUVne8zvBZoKkDMo3zYA/rBdburjlas+yws6jWibMc+t3rNciptKcnx5crlZJx4p73sOxBdUevUeM+Dj54R3wMHlEOj0qpR5tSocoGY8LoA+9HaR2OEyQ5cZmDWieieHepFvXDUrYkSzt9tmZEldbLzOYGCDa2xZj981mlS+C0GZ6wpYA9UgkbAZz710DiXwxo/DepUK46lVY0m5X3s3oMcDweY2Lj/AAx4nXNwr4gtVGnMPBzskGJkQww1gPs/jXePTYsas1Kr0e9LcYeehPwAMwAN4fWjned+JdyQbqup2swYj0bMAAwP5FlwPDNG9G5bw5XtUozMuSX3GJrYde/DAEaw6XTeD84KHQr1qTgSaPNMDeMPcgusQPRU4hXPRqg5wvnWrPoTD2szHOfnKk7M8ANYThpclGolGC3LmnHsnhKhsnvjTAM94Gf6F6zk8TuC3o68SaPXG6OZyKjD8an0zeH14IyeFOJ0a3zdaClUeTSpsHNipRJHWBhh/trJGAMtA2Da9UceLStzjZctd4r2PfFKgNTQAzpkuNoyjMA/xmvJ+s48HiI1i2l4ezLRpVzc1elN5ylGGD2zMwVfXplG+0M07RYejU988wzVaHQiDYqWQMuH7PrRaJmWDfWrONJob0A4j8E+dPokJ57Nk2tRsNnkUikN7RLTcbV7avKRqXU53eB4YKnqX3mbrxGDDZvA1bUFmW8BwWG/C780GwjBB+ytQgm+9Gdf8bl2T2Gs5RKlLo/j0apPAD4YGy7vDBWAU2pPVFozfjHIfMGGc3tmZ+dd1e9Db1Pb0uo3HdwQ5uf9Ee1ooGfn8n1ri7Rweq1j1lylNiMYaAYAAHsz70cYKPTbSlnOY/nM3ujPA2VIO2+QakS2Jb08AePWOOz7EFXvU1+sU4KqxBN6OBmBmHWH1rpjmCZpvOQzqRywZw3mZ+TsW4tWNBue3KJSmKr99lHyptdGZ9iqgBiZCpVSNgNKEAReRAMDNjvVZJCo2ZXqVVaHLCSHOBUIccNhgYdhrGx2CBZMW0rjdodco70aqxcPvAb/AOBSLDonIXbHodRrmDVXkgEaWz0Afzpq7eLs66pjt9xKHg9Khgy9EdPoMPyKDbEydVaDCqNDlAGcwAngew4xgfWsQ9W7SueBdtyuRaMbLtLZzkhHDMHvnzXYuA9Yv+w+HgXjOfeepVwvGx9+M84BhvDP5DVnwWolR4ncRLgi+vDjR6jA0KkyZ5m8B9ZgvSH2Sg3taU2x6HXIEaOcblcHj3gYYK/AmwIHri3LfuOmsaOuzzX5zV28Dml4ivbbprAWRQmGG/2WADJ/WCiz4GxfS8Whn9Fv+tM6Ks+TS5dxapVnLuIcTVlooeV+KPMKk0J9CkSWcDUcwXvggzngln8E0Z4Eo/PsZYai38PUg+tIE0ckPOmeZb924jFIeDYq94FOzzFR3g2LJaCahPMhmppqOYLeLEXkGHh03FHeokEBz0AMw86sox4HpoZgLsi3O5JXoEuBMI2/ZKbSqw2cXQNTr2ZwE1kYZ4Avqx/ccSwkzPvupqK2gTHMfbgsy8eZJrNwDV9LzbemecVQgBUUOsPsxwB9wzU1mtt5g5msb4V7XGiZpo2XFIhz4jw55qaybDylanjA+DoNudCuOTpvXhvRmy37ttMzI2cfrMDBZNRnpgCrHgb1fDUuHGlSR03HFHejPw5B6/QvHuEeTGfea8NVUmM5jvbWqhmDzXhqqr0M2VrFouFPGB8AwBaXP7gChQ4ZvMbFbMwH+VFtRZDOTGTM9QOtaWiTM4YMPn4odaj+qnNXT01N9VGyGo2ou2p6fDbnxzDUwWPOA5Dd03wWwB7AN6z9YMHvEBXCeZl6xGA/HBveq+NGPV3gripHotKJD+854B0Lvi3Im0rNl35FZvPOA6o8A22SBs/OrWZynLm4ua1QjwzYZLUlOYAt9RJIHDDT6FzFl45LRrZ2r92a09QzAzXN7MfodMW2YGpQHsVUDysGXtq+PcOyEjRbMVXyWfF8NWYHsTLzO/YscPUcGW08o5nvSA968UkSYzbzCz8mBvV0Zpp4GzFBlJMbd4ahPAtBJjYZqnksuZIK/D4pJ3D4paKkR0keijw+KhaJiaPD4qRh8UtFAyCNPaLf9aDD4oEkkkgJI0gBGagAhSRIDBJJJAk6CaToIFn8EefwQI8/ggBNGexOqOZoIh9aaTxplAPeiQogQJGkCSAwToIARggNOh0JpOggdDrUgFHTzKQJQKXF/wCdRAU6L/zoLCN1irVnoVfGZwEFYRgczQWDXapBmmgDYgM0DufwTRmgz+CaM1IRmkCaM06CB1I0kKAEjSQGaoI0s/gms/gkCkSDNAhRGgEzUQ+tOmo5oGjTSkGG9AYbFQjn1oEaBA0aSSRog7n8Es/go6NFjz+CRmgQGrCM0s/ghNAiB5/BFn8E0jRZZ/BBn8EklGAs/gkgR4fFEAw+KXYjSw+KsNJGnTBAaAUCdSw+KgNYfFBh8VIw+KWHxViOhTphvQYfFAKBOpIk1h8UGHxTyA0DRpJJGiDJoURoUWdRoEeHxQEkjBI0AGo6eUdAjQJJKwjSBGgw+KBHsS60kuhAGHxSw+KNJEGjBAnTSVBrD4pI0WHxXsAUsPinQBGoQDD4owBFh8UavwEgT2JpYfFbrBh8UeHxSSz+CyCSSz+CWfwWoSWfwQZ/BJAkkCNEECMEAAnQBAgBPACIGVIBlGQQZR6Lf9alBGcRcu4jVC0Uei3/AFqWEZxO8r8UEIIzad0VN5NHy3yIyaDD4oMPihz+CWQLldBl7oUCR1qwPoUR7eSvwKw/w/8AiUdTXgUJ4MFuzsBmms/gkaaM15CIJ40CAzTWfwXrdSV6jhUropT7/smD3msVx1qVSeaOnUp8wdBkAPA+sF03NvLUcVLW7epU912qztH5zM1kPL0/h1ZVHh0xypORgkSjzMJx7wWdDijw5sabLC2aOBumGHNtBmC1t/2HUeKl4BQ7Vla1KhBm9Ld6AXL7qsC0aVUprFGqUk5EUM/9dcQxVSr1Vvm4wnVV/DM8DPrwXQ3oDFt281Q5bbMyE+eHNmGBqsjWlWbAo0e9J0TljfPBYyvXzWLnfCJK0Qj9jIAqVtHucKVGrJwaM+ZtB1maVKNvmGmNcAaw3qlk7JTX0IgOKcc9SJm6B9YItLkvRcXWOsM8wNUhngSkHGfBgJXYarzMzNSJbxsG1psZp059Vkw+RclGbQdGaigCkA9gWm2qhYDNwx0zSjQJz3sGMwTsaZTocoOeiG8134KweksaupSmz5c+gDUoQnqPKjBm5h+RSIEOIy61OnYPRz2GHkWqpUCC9A13H3g2b2TVRoxGZDuDHei1ZXggxpWFNfAw8gdiOm3bUqVS5tDY0TjzesDDoNaCpTAepfIsUdk3eyQAb1WUr1NGpsuDWbf1pD57JHeygzQLYM1WDDtynmxVYeqw9rvMhsPAFQvRgk7Kaxv8is6VY06sQJD8qnewPDDPf+hB0jjfxF4WcS7Xolz2y29DuWnmDcwTZwzDsPr+Rd+iekfZlU9GTO4avEnXHyZ0x6nCfjMn0b142k0e3Ibsdw8zAzwMM1bXPw3ftVqJXNcDo9UZzjPB2H5ERcMjUpPOTzNbbgtZL983HNpXrw6VENkwOdog8AH2ZgsPMDA9QFt+DLLFVqlTpX2qmUGW+zmycfoP60HaaOFV4SzZdgXVBh1WFF3szpcbYZ+RVnE62J1mMUq7pb7PqcPGZBrrZM+gANZyxrwqVw3C7wo4jTvWVKfzYZeeDxgP5DWduR5+3pE23KrWTqsKlycApzx5g8HZmiWZk3s3W+Yg1yVzMd882Xu8D+tU8wIIfsjf1/OrC4Z9q1WOHqqgHTXQ78881F9fOcnyPKRjaDo2I8gAA3pfOo5s5tH2GHWCaeMzLTbzzPoAEGYPOg5pmDvQYd6pskM+1V2y9zLGm22YOhvB01n9bxdPTMPrV1QY3rKeER8zAMM0DXOSpkoXJTmZgGCkBMlxs+Vcwz60caNFPnYs4D68AMDVhTbeqUmkvP00M+V6w86yHRfRp4JxONlXuODUazPpoUuHrBIj9GZ9AGut+jfJqUOr37wrnMSaxICM+xDzPMHtHsVP6DFBviZPvC46GxGepTEYItYiH1nsPo/jUg6JOs+5aVdVgQeWrtEPOZrHsmAvOlkm1i2704D8NubODTXnrrn4Q9Znxoxn2H/kWae4J0ej1yt2dLnVIKhNpQVSGGt3++D9a9MVWHB9If1VcF3U71JEtQ+aej5+2M9+ea8o+kDXrxr1/wAu9XD5aF+ywHme9gFccO1RZqg8MaBJap8F+/A1ZoexwwNk/J17FXzOG9DrbpsRawcCoGB8mZvZhmHYfkXtDgnwusOt2/w04knbgMyJr2b0Qw8EzwwzXTY3o5WBArl10edRwmU+oh6wprRgAHGPvADUX6y35+w+F1x29ZrNxvuRqrSnwz5iOfQfeBglQaC5as2jtuROfp7+FXeejnvBj6F7C4P2lYlV4WO2PVXM6fW6lyUkw/ocoH9gGfZmGC6Afo5WGzUZFKpVONlqLDwZdDrZD5DUdI4v6KnDGqssVDi9QKPJreEk2IZx3sMGPI8CxVw8eOMvCK/JcSLasmBpzzmnEkM6x4H2fIu903hXWeAn2cvzhs5MOPNk8rXoMh7Nk8+g/k3rpVYs9y8KzIOo04Am1HQeedDZywB86vDIfCK/2L59WVWnMHGj1uGYSWj640oOsP410OfTd2GCqeHvDenWZcNYOIf3eaYTQj4bGX8MDP8AOthMADdNxbKYeZDwP2ai6K1E+M2eapDjGDqvYhHD/dqCbLi0YRsxTRwA942kDIzNip3pOC2cylMGfs1VSbYYkgbe8PoW0MmUemNvBgqqYDermCunrMcgOm+w+8fyGos+By0cHD7108aGck1Llj080DNSce6HFW1j9qMO9RI0k2V34YN3DmZhpm4pBrGRp74GDmoriHVcywNY8vC2i014N6jmGKlvPAYqI8exY4NkAZkp0xn7rmHWCqWXsHwzVxrZsKy4cvvP1kcgNAM1SzKC5G0tBjDMMz+tdKqUBt4zNZeqw5XY2u/1uZzXDC9DqLEFIkw3+YPNvBRzBwPDcbwX1YuHHg08Cj4mrNmNrHh3ovUk43Q02Os03BhHZB9k81o6PMPPUcQvUpxlrx20yyGiC5rttENRGPPBxS9hh4ioY09sBDU6FfAbZwObBc1toJkPH2dCdkxtYFVM1VzV01omTbeaDTWWFKIIwMn4badBnmdj7akGy4B6eCejM71ntZlmm4dAKczGPDepoBgKQLmvmaYRAZ3708bOY6fYnUYKNrwyNSgSgaNsFmnoFV8U3Hww8i6bJjA8Om4qc6OGrvbXZw8zm5ocy0X5MjRDeavWaC/TYXs97+81sAoMFl3XBgAT0yMD0J3Z0Aunx7LHDD8g+97NtFyx4abikRpmEgwwUsAbNrZ1rbZhXgyDLWzsVnQZ4KPyD5gpFNpTkboBRzftXDcAGxSmTwFV9NM9INTrVgBr49uyEsDRm8o4GgM1zWuDRmZmgA0j60HevFpGfwQmaZR9YKEojxqveDNWxspk4zaKURsoMPirY4e/oTPKfu1IrcPiiw+KkYYJYfFFooAiw2KRoo9HYoETD4ocPipui5/Wly3yIIRgkAKby3yJHG+RBEw2IVN0UHLbUELD4oFIMFHMECSQokCTwJlPAoBZ/BCgz+CWfwQJR3j3qQgeZbMFIhIDBS9FAaoR0k91pGrAJJJJANElh8UYAoDoAjSBGAICBSGQQgnWQ2KxIZVhGDMlEZZVnDZwwUCzZZ2grCMyojPQrOL/AM6kFh8VHNS8/gokk96BozUczRmmTVA8/gnWU0CMDUiQZpnP4ITNBn8EBmaA0BoTNAYdC+IUQIHgSNAkqDRpnD4qQYIcPigZANiE072IMPighGCDD4qWYIMPirQj4fFNYfFSzBMmCCOkpCWHxRaKkncPilh8UQaSTuHxSw+KgR8Pij70jBI0DJoUSFWCRoEYKAeHxRJJKwOHxQGnc/ghUJBh8UkefwTKtQ0BmkhNQEaZSM0JmrBGaaM0jNRzNEnc/gkhSz+CAkKJLD4oGkGHxUjD4ocPigZDrTwIATodasSAQmCJI1Ai96AwTpppWGu9LxPeIkuhAIJGiBI1TzYUCSBS9GkkkqQRpACM0gQLYgNGkvQsPiiBDh8U6HWruEDAEQAjAE6ALMDh8UBgpGHxQYfFWtFNMqwNltRzZWojpJ7RSw+KBlHh8UeJosHDRBoAToMowZUgGUZI4MuKWDLikBGcUpmMgigy4rCNG2KQzGUqNG3qoDQQ/wB2pHJ5+7VgEb5FYRqUBhma9Gc5D5EfLYF7NavkGAHwwUGTDb3+GoaqLEEsQVhyaPlP3aCDh8U0adSw+K5Vo5mms/gpZstoNFv+tUhEe6FAe61anGzBRzjfItItakeUcwNXBw/3ajvRv3a0QrE0YKacbd0KOYIqFecOKftM/wBar69SoL1uVNhts8zjedXBgmkespwopUGj2190DDXzzXB+HVjesuId0MVxjA4T2GC9Jw4zFHF1tjY04eeHkURmAwdUdqoMABvhvw71y3A4bxdogV6LRLEPZ42eC5vWPRdvGj3Dr0ajnJhMdZmfQC9VVu24smqRKrFiYTYp56y2Zm/MPXAOtRhb8/6xw6iVWouvy4J+AzsCOG/Z8iz9btJij2Q7VWIJ6utogHeH1r3BxOh2JQYr1cfqVNgVVgDPPP2x7Ni8qT69WeKlSaty3LcDSYzeeOP3ginBZL23TURd4gQOEFNdkPzoJzKhFPAAZzzXP7zgS7/uaoVy2KU8Efv1dh5goGPhw3JPsOxWb1Ng03S13weN8MwMOxM2rDqtSnu02lROZdNk+/Bb6wLGboMKq1W8YkZ5kAw5fPM0W5+8DYGriq0FyBRo9ZptVjSdQMzj9BgfkVTUpMSZNOVTdkc+gFH1sEEh6pSjwDmzw7wSCZ7v/hUIwcANRID94g0bMyow2vuj+AeROnVW5MU23z3mqqHMcPY/5FEe2HpoJbJmyWcU8Hew1NCZUZNShMBXHjkOHgeAYYKnZk4Ethwcrdv0HiNEqtzP6NPMDAzw7+xBCqVEgszD9cvhPPzgiuSt+sqM1b8F+Sy0B58o7vBdTurhvTYdUqF3cjGk2/VDM2Zee+MfyLhlSeD1kZxXM2uw0DRxnNIG3Fu+EVbt+1Z8uXUWDCabPgyA6AWJ1sxTXNfBBZwJ78y6JFcuOX4pvGeeGe9WtbmWjUv55bYeOawYawH0SQ8izQb+tAbOaIWt5nbL02PKtnY0+Gb0f8E0cO5LZgQHYjluA86+GBvLOSQwdwRxg2IJtBmeqqpHqOnnoHngmqxP9ZVmXO08Nc88PIptNoM6ZTZFYit60eKe8/kVbM5QHc2O9AmTcNTaVJcjVJp8OvPBV8Y8FLgAZyNfsYNFrgHmzm6ep4r72Zh5F1Dgtbc65LgqtGptfjQJoM6zLTwftOG/AFyE3sJUhwG9+eYGvRFsUG0q3wCpl406naN9USseDOA8NZjPeB/wKR2b0aTvjhvxQOqzqNJ9T3CzyVY7AA+x/wDx713KZwN+3lzROaYhxqZS3jwMDzMwzzR8HKw/c91wotRtw6DNCm/fI5mBgYYbDBdzgQHIA+zTwMizwiox1esRajgdHq7IMGyHyAvDvELhFBktXxUqNOks0q2ngiwwPeBnn0L37eZ3A9Qzi0B/Rdf2G93gCz9E4aWxDsabYkuLm1N/aTPrM118Yy/o9m/P4O0KgVWIDNQs14IXg9HfgYfkXY69cLFBoNQup9gHuSDPDvXLOCFKqNBgXLRqq+ZnFqpshn5AXQKxSm69FaYMM2gPMwPoNRA8NcS+GN42ZYtM4r2+/UmYVUmG/VWmjwBl8H/BPDvzX6BUR5isW/RLgY3nUabFeM/nMANYrijZ/rvhPNs6K/ox9j2eGfQea3Ftvf8AJqlf/Rsf4FHNYmmyxJivQZzAPR3wwNo+9KGy3AhtQWM9JkMGc+vBDn8E6zvWSLtIZ2HqJp6YeemnQ2Co728kWde3iq95nNSAAzBHo7FQjss4CgeZb0kj2EkB5iiFVMA96ah6gH4isJLOaiaLn9avaR1JliTH6Fj6xbDkx3rw0/kW1Blw0jjJtTi9V4aVyZKOWw+yYfWqKsWTWaa0DmGa7w9DcBMhGYktG2+2u/x7LJxqBaVV5IDOKqyqxn6aGoYYLtcyAxDHZ0LBVumsVKQ6w4tvp7GeGJh14zLTM1cMvZlpqnmW85GlbG1I5nlsNTrXQzXcmG2A5gkB7dNHDmMPNApGi37tc9tKRDDJRzjB5FZ6Lf8AWlohgo2YUkmj06YGBsB9aqjtKnBmxoGYdi1GDYJYgtvHNZhh5NsMQ3TBhvA/Oq956czK5R9zMMO9dCksg8CpalTWDAPDXR9NselUhDfnx9R/rUeTAbjBg5sV2zqAOnpmCB4G3gwMF73Iwy7MPWPTVgyDjLRxHOhWAQ2wPw0cllszz007jCnZhuG7qYK7gGbJg2mWQ36anAym1LDNt4EQRmwPPT61W8zh0Kyhycx8Rc1t4SMPihTubZoFzW0NIw2IEalYk0bOZJ1CvQGHxToMt46aWHxTwKe60M5WKOwErNiLhn86iMwwZNauSzrAo/IR/Iuzx7LDDOSWXAdDw1Zw2QNoFN5ZjvbTwMgHs0vm2vAmQwaDTUhCCJc7aCz+CRmkZpo1zkDTPejSNFgM0s/gmjQZ/BSJGfwRgCaz+CdZXgWij0QwTqDD4oK2TA3H4ag8r8VpuWcNMnA+RTYp2YDn4aPlP3a0caBt9miepTnXgoGd5b5EuW+RXB01zyIgh/u0WojjOJaKu3o37tR+U/doKk2d6jvArg4ziaOH+7QUhsqOYK+OG2oLzO5QK0wSw+KmnG29Ca0UEQEj2J3DBNPIAz+CFJEAIEnksPiiQMH0KIamvdCiGCAUuxFh8UsPirA4bEWHxR4fFHh8UAIwBO6KdBlQGgBSABGDKlBG+RA0yzuU1lnZ7NPRoe/2aso0P5FYixo2/oVqyzgKNmHh7tSAZUBMgprPQogB0KWCBGo5p01HNBHNCkaBSHUs8EOfwSNUFn8Es/gmc/gln8FIez+CBBn8Es/ggeAFIANiAFIBUGjBIARGhz+CAE0adQGiDSSM0GG9FgSw+Kew+KLD4qxH0V95dtP4fFLD4oK82cPdocPirLRzSOMoFbh8UGi5/WrDlfig0UQhYfFApCaw3oGjBMqWmjBWI+HxQYfFS9FBoqBHNIFLCMkcb5FYZSSxNIEApJ3D4pYfFA0mTT2HxQGCBrP4JkzUgwTRgiTSZM1Iw+KjmCKNJYfFO4fFLD4q0o6NHiaBBIBSNHYmmQ2KUAbEEUwQYfFSDBNGCgR+9EkYJArEjD4pdiXYkoDR9CaTqA1aDWHxQI0CLCkiw+KWHxVINJJ1NIEjw+KWHxSXoLsSw+KMA3o8PigZw+KPD4p3Rc/rTwM/IqEcAUgGU6EZSAZXsBrRRAy4pfLuKQDOxIgV/LuI9HBTTDFNGGS0EQwUcwUs1HMN6IR9FfNFz+tSwBGEZwzRkiAy4ngZ/EU0IbidCG4ghBGUtmHv6FKCH+7UtmMraogQ9+ppqWEZSNFS40bNQyRAZP8ADU2HGfM/Z4Kcyy2CkACB0NMAT2s3/UoyEwQStZNHvTWfwRAe9Gmy0V90W/60/ouf1r6bLgIbZlGokaSwbWGuBm31gpALlejw+KWinWWTc6AzUjljDrMA/Oi0LRToQ1LCN39idBlBSSab3qvOAtmcbMFXyYGCbGSeh71XvQ961DzIZqOcBs/dq4vAyhsqO8zg0bnkBag6U2fu1HOlMAB6jeYYdCvY41c9VpVEpDtx3dONmOHQ0z1vGsfY3H6q3hVDg0qzsKOweBy5Z4YB50uOtq3BecWFBnTmYzUqZ92jx+vDzrCcZqaxwrsin2BQ/wBtmhnMPvPNRY3Fz+l7w2t6su0cKHPqWh1vRz2ZrCQL24y8QqzHnWxdr1HpVUPBkMw2B37Fn5no606NwvplwXAEkKhUTAPBDPYfeuwW3ZP2VuO0qbLb+6MMh4veo7hzK9rPsC2LhkQbucqVbqr7PXIezAz+jsXRfRF4b02m06oXw43m7Uc4rPyBvVfflpBcnEisVxiLreqwDOP5wWz4e3zatmU2XSqO+cmn/tvRgEMz7FG1uOXbZlA4LcbvXFZlyfU9R+9QzwzwM+z+NcnueAxPveu3BRqlydHzMwezWu408Qp3Ga744VJj1PChM/duYDYZh864PJny2XTBuWeHYYIpIoMyXQa3IqUF/DZgBp2tya5Un/WM64DkhK8hqvo8CuVuQbEFjWw61H5Co8w6w2xvDrRZ3lnwjm45gAAo/ie7TWZgeD/u+xSAMMFABkDNI9h6akQ3tF8HFHkg2D5uNt4ApWWRqwgBzjR6m8wUJkGzLBw8PnToajL+ww3qsIKYzo+IClwDpsYolVnMBMBg8ziH3oNZgD8fo71CeZ0T02/Zdn0Kx027eM0u86C1bgUNmmwmMMNI94AsFyzDzuDj+HzqvZN8y9WxYhvS3+gAWtuSwJVq0aFVZ1SZ1Zv9EM/GBBlTBwHTDsQZ/BXtvUR+tyHg6GgDMzR0e1fXdb9VAeGwzzUCvpR07Ql8829q/wBGMFYHDY5JqWw5sf7PIo9ShhAlHFb6A2KIy9g6pFhMtJw4R1gH88Pc4b1Fh0pt6l+sugO9XVKudxnDDRPDseDYo92yazAkR6PKiRo0J8MwaZDDBUgvtO/9l/s/EfMHe8zDrBZowVrVQpUZ8PVtR5kDZ8mGChYcyYabAAeCCPg57tWFNPPMFE8Rnw9MwNWbIOQz1POCLOmGwPrwXsv0S6byx13g1c7dNqVKqMD1pTWneszw34B+QF46gRn6lqgGGDAZmBngr2z7/vG0rtoV606pTHpFvHmGj16HeCD236OvGmsUriXb9j322cblZL9P56R0MgeeAGvcUwNF021+aVn3DbnHv0lGpzFDnnb9wnomboYGErDr/WC968MbJv8AtKlnBuq+JNVDeANSAzPDsPNBpj6dNRD2KwME1oq2TPnGYZlO1FhvRdf9th3qUzPMAT0yHmKz8l6VGPT089+C2ixo+cbkg7Fcc2PhgasoB6MVqK30MBgCzlNh4Hrg5mHzq+i/86jmjaotaxf+dSwNwFEZPBWAaZrEHn8Ez75SMME0fWpWlgkajg8iN7YghTA3poARSTzA/OmmQPHegkACBGCPD4oI6A1LxBR3gQwjnpmqx7wXQcBTTeBl3TTT2g8BrVCFUg1oprAzw5Oaea3Dz23Bc/vmBOkk1Kiv4YHma7OFkhVUNZrXi7z8iyk97xd+w1pobL4Me0VJcMNtn7242uyLZ2VNPoPNaAJIH0OLDxqkxzGm24riHUm8utdNwx21Ab0eHxUFmY3pZp4Jn7xcGG2wGCAwwFGZoHj2L16az+Civda+azgHqKLrZyN7i1iBN5MDHxFFegYErKMGbQJ02c1F2KI4BmfyIDgHh4ivtFv+tLRbMVHcMuDJge9Og8rh6MHeo71KbwzBdMCpD2viKwZ6E0HjfdX2N4dBprRcjPnv2KmCwZmYO6bilgeSrD0MdT3qUaS4DuDi58NotagCQAhA06sbhtsBoQRGhUvRZ/BHn8E0kslJAGlh8UyBp7P4LVFwZMEaSAFnsweRJpGvHokeHxQIwUPLLEEtFOghNe+F+EEwwJNaJ5p57rRAt2RrD4p0EBnimgebzXmFrMOhFg5khZPYnQ61mHWQUsGc0mWdqlxg3rK1HYcPxQ8NWfKfu0UNnM8FfRqVm0HesRmZNKzjn4apXo2iWmupHTQZinqN9ixs+m+KfhoM4bIYqI8G9XD0bRzbUXk3DdVrVnLuJrRV0cP92o5w3FApZLOCgmzmS0Zw8x8RV8mBgaCkeDBRTVrJjOYmq82UEJ5RzDJTTZTXK/FBBw+KeAFKCMnuV+KCIAJYfFStFz+tNGy4gimG9LRb/rUjD4pYbEETRRgynQBOgyrEflfij5dtTdFz+tIGTyUCLop0GVLCMnQjfIgiAy4raBDzMEEaNv6FdwI2BggKNAwL2asGYzYI8QTqATZbTJgpZpo1IZAEefwQIUBGorylGo73QqEVJLD4osPigaSz+CkGAKOakAaaM06YJo1QWfwTzPWo2HxUhkN6CcynTQMpGgaM0CRpo0Qdz+CRmhRADhos1h8U6DLikMxm8VI0XP61aETRc/rR4fFSNFz+tLl3EWhYfFLD4qRh8UGHxRBAG9SMPimgDepeGxFohhsUQwUt5NIhE5b5EBgrAATTzKCsMEgBSzZQAygHRS5X4qwCMnQh/u1Ar9FI2VZ8t8iA43yK0qQ4yj8s5n7NaDk00cZtFKwI3yJHG+RW2imTZQVvJpk2QBWZgmjAEFZo5imjZwVgYYqIYIIRgmTBTsPigMFaULD4osPipGilooIpglh8VK0UtFAbIbE7h8UkeHxQRzBRzBWBgmjBEK/D4pYfFSjBNYfFUs0iw+Ke0UWHxRCKYJkwUswUcwUrAhw+Kdw3o9FW82Zw+KEwT+i5/WmjBeJRzQqRh8UQMtr0NYfFHoqUDLaeBnNbiIDLikBGM1YBDbToRkFeEZPAy4pvK/FGALPAiAynQZwUgATwMuGriAzh8UsPipfLuI+V+K9QgmGxNYfFTTZwTRgghaKXK/FSsPijw+KCIDKkAyngBEAK2RAyngZzNAAKWyHvF5gEzD/59NSwjNoNZGDzi9DvLAAow001n8EQIJYafenc2/dqFkaQZ5+G2aNVtGjZqV6t/eAoUY3wMFZBqGoEV6Ngo/hgfiK6CMB+06Fla9WGIcrQiRNYO8z2LPY0TJtmKGeGjH11h5/Fq0rYpv35iZMkfgtAuP3h6SFz1Wa0xR6UdNiBn93M96d8CPTeIVKjYBSqlSplWmvdHOBmH0LtFKgVnkwclsL8pzqWfiNwZMA+wwPeC61wi438e6VPClWlcZ1WJ/Z6m9msVvZdyXJflw147csqqxobQdYYYGBqXSuBXj83dV41WfN78JmC4DG9IqzrwqnqriNRp9t3KBmB1Ck54H9a3dscdaNRKoFs1+pViTn+zSHnt+CDsD1t3baUc37OqIVKP3wZx7/yGrCz7/t+85B0rUOm1hg8HqdLDAwP6+9UlH4ivzyP1BEOq4djPWo94U2h3sEeXUbZrFHqsU82ahHDfn86gdQMAZ9ukDOsC4PxR4u3/YdpAxXLK9fR3zw9YNB0Lt3DqvUq9rKpVwUOWEnXZ8bA88D8iCDVaaYHqA2qcDxXSHqaxJa0zBZSsW24yWpFbQZ03mATTxtmJtgpp0SV5FCOmyg621YzkO26HDkc9yIHI85rlV1cOvthXpFZnbMzwAMM8AXbXmXAVYcNUMBflt/8ho8GC3shGB/oTMymv3JCp8qIGwAAwNbOsQH5lLlxGw6wwUSHGfptJjtuN4GAYKbgcXvmvBaVpV2svuaOu9oZrhlYvO2J82JQ6PVakEc2fG5QO/516Wvzhv8AbCl+pz9kb2Zrm9Vtu2OGlxmxTqV99OH4MeOG8zUYW5j6QkaDWIFr+o2wPMDDYGGfQuH1u3vVubc58Izodh9631w35Uq3XnX78cOm+r3sIwAGAZrD8Qrh+09XdnMdB9CKVNBeqoOyG6NL0TMN/wA6j5vhvcMwd8+a0BxrfepcR+juHGqYe2z71Sz4cqBI0JTeBrJaJ1owRGG3oSxNUDDYYOeRFM8Yzcb6DUU06HTp9iBo0YAasGaPK5gAfimDR+dSDZzd5Rh8AZA+jrzVivmaGLWHX0GCRg4yWm55M1oqCzaUOrhLuqCcmJhhgHYq+5PUb1Zd+zhvcl2A8gOxpNApt4QqrcZyQjxTzA2ew1qOLt5sX5cLUuKcZ6PF9i8yzgZ/WsF6tqR7wYzBIAcZ9og1tHmPwKHLNjvDBBatSfolx0+X5zwP6FLoMNiZwxqdc183YT2gezrVS8z6ypNMlNvgzI69/fgshc1KHBCvSH5zGtHzzMFjKqcF6qPOU1swj9gGG9aU57dVpB02IedbPoa7z+hZrr9v+0d4H1oIknMIRrp3EL+TmZwttGpW/WNa4DAGJjPWbPnBc5NnME9A+5mD+gB/WCoR4zPQbit6b/pKJqN+Frb0MmTTTd8MDDZ5Fa2lGcmO+sQYPkg63j2YIFW4b5z8wYzDvNV+HxW2/m2ZWQbiTmZLWYA8AeTzqwrFNtKlNVhh+lQzAzA4boBmYGshU0G27Zn2zInP8zGmsYbA2BmaUyj3dw0d06jE5Z2qQ84z3WBrQcK2a/PqkJu1YnrKQ+ejJjmGx4PItRxsZ+2FUCnMMSYcigxgCTEkBvZNVFjv3oZ8BKrZnFC2r4rlSjSfWMB83o7QbIz59B4L3vJ6w+hecPRpoNwWSdMtl9z1lHOiRahAkSA3sh3sLvrNYbmADnRsVg3gUTD4qaYJowQV7wNqFJgNvKweUfsVwjaiCTybp+JgCmxqx0P6mxBPgAYmar/2b2/QtUtnDmA8Go2rIJOC5/AefjeOwfWrNmqyzLTccWVwpugkgaF481RRpmxWQPZqFj70B9aHP4IANSg6ngUQzR6ypaUjULWwJE9PbjNZm5371IkJGGxLMAa1zc2I2XgeHwzzQU8+HgGoCqntRaiSzmKpHo3jmq2KzRzFUtSjMPRzbcb71quUe/DVfUqU5yp7FtFouHP6kGi1mHYsvUqkw80bE4FvnqbrDp4Kqn2TzjWnp4fOunh5mNw5JMjcs7rsHsNBDmTuYDZs860r1mVg5ANy6bsD51cBSmGQ8eCAGvsfTDjx5UsmqmEUPEQQ634u81d3DbDjMDOKwBhn2Gq+TavgA+xsUbhSwjT2D94pWeaz/IPxoGu45go8OqmyGmbijpe7Xck+xU8zU1wcbcwVgyYSfZ9aiTA953q8G2ggHhHBS8/gs5AknkG9XYPb1x80K8JHWlh8U7GADUg2W1k6PClnmYBqKOzUm3mgBWFSZ8A1jJ4PsmZsLu4WdtBJDR8fMM/kUeSbZxTcb6w3rNHWzDYakQKwefnZw3rbpY9xU2sBMdNs+w1e6OZ5h0LHyYBw5oTohgccwz2LQUef91AH06UbXbJ7E7rb1C1tnhoNbeuO4bQs8/gkogPOJ0DXNcNos9h8UCNFh8VjhsFIEWHxSw+K8SNJJJFARoEaAwRAaQIwDYoSMEkkSsQnutAyeCkPAoTz2ifiLSIQOV/zKPGZM3ehSPDPBWsCNs8NbiFnhs01YRgzUj1VmeorCNTQAA2LC1kEZ/H2akMs7lZaIGwkEZvNc1tVhABvMFqKbhpAs1DWgpp7QWIsJgeAqSZAza1MFfG82bSjmH4ikY+TR8z1NNRDpuj2LZmDCqZhsB4emqFEcPZ0KKcPD3atwUeSGxBnXmcCUR4Nis5J71UyT3oIT0YFWSY29Wxmo5ggoTjOIOXcVsbKaw3qxXmy4vmi5/WrHRQGCCFiaRgpGJpGzsQQcPihw+KlGy4hBlxAyDLakAz+GpARk8EZxBH0UQMuKcEZxEDKgRwZcUgIykMs5p7RwJAUaHs9mpwM6KBkMFITYWfwRgaBBngpBmaaM0jNMmaoGaQGmjNIFYkdZpGCIAR9CCJh8UOeCdM1HMN6ADNNGjQYfFQATR9adw+KRgrLRw606HWvqIELTg6EjUdEgM1HUg00gAFNZDYogdanM+dQhNZZ2qczDzBR4zwKaD3Q2CsLkPkUR6NvV94ekq+R1otWHG+RNHGVgZpkzRCLo4JIutLD4oITwb01h8VNw+KWigjh0I8M07ouf1qXGh5oKw4x+RIIza0D1N254KPyn7tQI8aNv6FINlTYcbf4gKQ9G2ezQU5spo2W1ZnGSOGrSp8Pig0Vccp+7Qcp+7RSnNlxNGy4rs4f7tRzjNoKQ2XFHMFavAoTzJoK81Ew+KsDBNGCJV5ggw+KkaLn9aWHxVhrD4pYfFSMPilh8UETD4o8PincPijw+KpBoATuHxSw+KRgizSA06aaNSg0YJACdSVLNYfFFh8U9h8UWHxRCOYBgopgppgmtFehkAR4fFOgy5ingZ/EW4iGGxNaKs9Fv+tBy3yLIVnLfInQjKw5b5EYRvkWqEIIzilMxjxVgzG2ezRhGRZoGdikBGToMuJ0ARBrlP3aaOHv6FZYfFLD4oK8Ie5S+Ww92ncPijM8UEQ0IAalsg28alcswHu0FO8yaj4fFXbwBn7NROWDyIK/D4pYfFTTZAFHPYrZPui3/Wg8MDQHJVe885qroiE7WebeSd5htU/ObUHMOK8I2vtZFrKqZmOACM5ijBtZBJTwSd/iKk5wA9madZkuPGmDbVwDgm6Go4r0I0E/eAH1rBHJcjBqaaan3a+zFdM9gACxtcW3Rz7fZlaHPZuh1gAZ4K+hw4skPAcXk+TxsfgStCDBMwb7DBa2j8YKxP8ADbfON9YLg7nTt6Qk0dtmKctx8AAOvM15y42cS6Hbco4MF/Wz6z6PIs/eHGCuVh12jxK+czozCOC5Zc7NKeDmqi3rTfIuDm5jaFUuItSqUg3231EO56N1zgzkd+9YqTUn3pR6jYAHYAAnWWWJO9xveuN65lY0D7f1Q6NWZZ8uAZs4dhp2xrVYevSXSnKjJZhQuiRHPA1s7/4S/wAmk2lP2XO5moPh40frMzW9tvhLEOz485+h6NYf3zHnXjAPrX1Wrn/B/h7BvC+qhLrhnMp8UPbPdYH51ir2oN1XDxDm0O3H6lXuVPCNon0AvSTNknZ8M6HZVRjSXa2GZg8fQa3fBDg5SuFFIm3HceDNVMMzMN6uLYOJWlP4o8AWqfXLjYkhCmmAHv3hv8gL0xXvSioFE4by7ttw+fmsMnhHkBhgvNlV4r2jcnEOoVziHLeZpLB6EMzDYfyYJqq8VOENekeqrcowST7N+jmrUVmemlXIHrNviNRnq2co/BaawwZDyYKpjekIdEny7n4c3jPtKoSjzOnsxvBeWX4u3JYcM4T9smyzUIobwAOg1h6JdtDmXA1WeKFHk1Kjh1myoH6EcEOMfHe87SiXG/dVmViO/wD2vBk/zr0tAqTEyLE+0cEIEiUGYaR+Dn5F+S9wz+ENYdA+FbFw03vmR2czAwXdfRj9KWy2YR8PbgrhszWNkN2vST0TD5M88EHvWpUenQ2ubnADLXzmsVMvDhkfs6zrf3QZrg9/8XabbzA1W7Zz0alSv7DM5yFJD8nQtxw69KX0czj0+C/OoLITffdZsmfnBBsJJ0OpBnRnzea8+BgqeZA8XBdtk0GJcNOjzrfrkMGnwzZkRDAwMFyC+aVxYt6jTarKrlsMwoQZnLeZP/ArHEeKlVnUevcjYFx/zwe+TEPezhsWUe9JCnWrUmqVxUpTLJv/APSFOMDDP51x3ijxyvWqhNpVSCjg6B4c3T+g1yG5LJvgKbEuqpMZ0p/+kZ9CD3XJ45cDzaB9u8Wcz6MGTM1w/iXxmo9q3CdZAI1ekTWfBNk+j+BcCk1h+lUuPSgbjbwz1WetZkz1pTTAdb54Aslhva8PtncMi4H4PJuyutkA2ZqhhxtaQDbi30mm/Zto4NzUZmS7O8dmQB5hgs/6nYhymtB/PP51UWpH5Ng2jcb62DTNYmP1UWfADZ3+dWZm4EM4jj7IYb1TgcvlevZmslkyAaRt96Bk2N7b6eZ0A3uMPbOs1ZSWYMxppumsdAefrVCi5N94vAbzSmU2dTQByW3hn0LW2Har9blHLl5hHi55gB9aar1pOPSo9Vi1H7lOeBgNX3KCPR/srMofN1yqzAlgeAMgfWqSTyrLucRw8D6DVxfNnuWTKaYcqQSdcM8wDBZ0AckteIexWLCpG2YwjPZmyoJhg6np72cNpj8BQQM1kJfXs1zD6EDxmeee808DOfiB1pPM6Jaj7eCrYn29Va5Gtqp2qwwycKV4557+hXdjQKxfkimW5TXGY2gZnno7AD51d8PeHtYud2FEo1ZZjet2TzB1nZ+teq6J/J5ZNmxKHb78OZdFIzY2M4Hn86WyZnghTbK4XQ63VbxpVKqVbpb2vDkRI2ckwP5F5y48HbE+7ftbbjE+MFbzlSYksMNE8+xe2P5N7Vs/h5IuO6rcZrc2Vm+8eGBhn2AvNl4ejrxXva2ZF8PzoD0Slsv6NPA8zNgPIpVFuS2rAg0qY0/f9HkxqVUWfucjzms0ZsBINthzWaA9h4dYedaq+eIv2ttW37fcpvLPUjrP5wDBVlqnSo1w0qdWW84QPb96pa44dRn63VJEFyKb0cGe9nYH51cs0e7r2oP2ftzCe7Ce3xIgbwDzmt/Srzty3n6rc1OghVXTPCTn4IRmPk2KtoN5/Ym+ftjRnDChXLsPMMAAP9zQUljWHUaJaEfiFLixpMKVMOkGz/Soz6kVW0qjyE1uUwB1CV40bA9hgut2lUqVJ4bTYOmybQXUFQDMM8A3qDf9BtGBfx3HasGfW2ppgbNPgxjOK8Z9eBoO18KPRgo9gcAneKFOqRzLlrdNB+MHZmZ7ADfsWKsY6qF/3Rc/E2hxgj1SN6iqUfDM2X8OvYthwNvC5+IXD4+GVRo9Sk2lSKkxzM4Nh01jrNg/oXqixuAPDyz6kb9uVKfPjzQzMJGBhmgtbPsym0SbT7ni/wDwRiEGAbOgN4KivClBRLhiVVhz7lUXugNgMn5P41l+FHGCLPvC4OHs5vlvV098KbswzAOsF0W8KV9rbcBiC/gASQeA8O8EF6EkJkdqWx0PBmorxooe+A12bE0YKRHeNNZ/BG91qNn8FUIojVbMgBJDDPBWGfwSNXFr8IkYAhxdNzsVVGqWtVNMOhWFbZfepchhjvBYm2KbVaVmxOPf1raGTobM/RdBs3FaszNiwsmS+fh54H51e0qS5yoZuZmouFNKElGBqsA1IB78RYiwz+CSjo8/gpWiT5ObWg37U+hc6DiFUYFchWi/ECZWJTOvpHsXSHofM4ONngYdDodYLz5f/Cu3GazIvS8bjmBNp0M34cvWwPZvAwP86DrHELiFbNqsR37g1j6MAAMzwNS7bvmDeFkVCuW5zMaPgYM8wzga8lM8eKBdvqyVfb9VObCZ0WajEDrY7M/OsO9flYmXAEWJWD8CTyrM4JJgDzHZrAsO509L9EKDclHmWlHuN+sMhBfDMDPZmrgIebTT/nDNeX6bxUg3Pw0esCh2iYeq4wBzeHgmHnBdwsniX67p1Kbl02TGzjBve6DPvw/1Ni9czoDMNjDoQSYDBhp4I6bPYqTRmxniCkMmEkdRhwDBbDOHb27oQBb37taY2UgZQYGfb3WH/AsfWLewzbwXVp7O43Fl6kzm6a6+Mc0rHPQIGhFDP5FCpTzkmL4nX5FsqlTQeA3NNY+MybM91twF0xbmwi1umvnDPTbzNZGZAfZaM8F0h4M2lTyYzZkTemuzh5sMbhhYdScA8HFexsJIZqqu2iToDXrWKxm0HWqylXD4QAuz/I5v2NUyzgR7MFZxt6qIdSzEN+YK1jHgGYObFjcNotasvNsjvSOf+G4qd6Yq+ZUm2R8NxY9Lba9mTGDjnvWan9yrzrxqJPrDjzS6eHhwxvmRaqETA3M96r4cl+MZ6B9YYIHnnHvaJrD4r6LkWUOe/paGpsV9TTY0Dbc/Isoy9gr2BVWI2zTAz+dZXCotoYcls/AXx48HTVPGe0ZBvn0ZqwOTrZrjuG0WsGTzwUsDWaCS+zI09TYrqG9rNaixvhbRaXn8FNZ6FXnqKQyexc1wuEtJIEeHxXO2gsPikkgM1GFkjBMgnu9WyOo0gSNZ4aiBAfWkHWvp71oyMvG4oUmG5M9n1qwMNiUbDNbIRI0B8HR1G1o4DOCiACkBJbZIG0F3GZzNWARgwVDDnmB9Gavo0wHhz6FxW2O4YClh8U0bydDp1FDVIZPBW0OS2AgqRSGTwWNwNGEltI5IYKkB5xGb2xSHZM8PZ6ir5MnWNR5HWomsYILMEB6ZgoXM/OkEncqEKfG37FUyY37taN7TNVkllSKflfimtFv+tWZhiohgqFUbO9BoqwMFHMFIjgzv3oDjNqXh8UlQrzZQaLn9as9Fv+tBooK3RRaKsOV+KWigigCkACQApAAgDD4oMPipCFAQbE8CZBJBIBPayiZ/BAZoJefwQZ/BRwM0ZmgMzQZ/BNGaWfwVgsM1LZDAAUdkDMgVgAIDw+KBO4bE0aCOaaPUTuGaPD4oIOHxQmpWHxTRgrQaTOHxUjD4oVADD4oADenksPigLP4JJYOGkAKFlh8UeHxToMtozBWIgKQBqOaeANiISgecTwPOKCAKQAb0Ws2XnzP2isAjZio8CG5scVmB4ohEOAo5sq1ePwlFw+KCpMMCTRqwmA37tQsPigZ0XP60YMqQAI8PigAGVa0plvMFHZZPBWEMMFktdhDbw9mqyTA8UzVtGe2+IlMebAfDBBTsxv3ad5PNJk/FVxGZzwVIU/Io/VvxWi5D5E0cD5FYzRw/3aZOGr56GYe7VY8G9QKp5nBQZIOK4eBVskFYqTBRXlYPBvUd4NitKpeBRTBWBgaZNnYgrzBCpRgaaw+KAABGAIwBSABBBNlGAKdy7aA2VSEFCakGG9M4fFFmjBNKVh8U0YIg1h8UsPiiSQJGgRoFh8UgBGAJ0AyQLBvyJAy2ncPijAFYaCMnQhqWyyn+XcQVps/ho2YbhmrIIGas41K2AvbsUnLYJ4Ia0QUf5EfqrD3ajYz/AC7iMI3yK4OA5n7NLlP3abFSEZtSAh/u1OCG5n7NSAh/u028wrOQ/DbUJ6G/n4gLVMxmwBNSYef/AAezVxb1mgAGU6yesWmG9TTo773Yp0Oict7tXsVhw9nQo5xsPaK4km3GVU9JA96uGdoMlvAFTyXMDVxJktmGm23vUUIbZ+I4G9bM1SBuGKHls1ccsz5EBgAdDYLaGVwpzh4INgKXK/5lXn+J/wAa1YntZv8AqTJvKOZpozWmDaXrN/1KfGqrbIg3qKizzFRHgcMcMz/ImDboEaZTnvaI7kpsE6I74a5kzMrDJ+AZmsjxCuG/zhOsP6z0TsAAXzfYbQyVVjPnXjNh+NJ+RWsmt1iHFPXlw/yAuZMxm2ZGvLN7P60b0ymz3eROcbIH1mZr4nscrviCqtbqPNOmxOMPoVJJuSXlg+/vVnJo4BKCDTnzkun0ZrQQLApoUuRUrjpWchgM8M1wWYU/D20mL2qjoVGVog2Gf1rcXVYduW9TdelA8EjsM3s1h6VJinNByC3y2HRpbFBue55TP3U5by18LYqGEuqvgdGrMxmoe+nSz2ACCyWa5cPEGJQ51Yr1eo4Z85yjxhmsPdvEis3zczOAQ6VCAMMw611W0uNkHhdaR0OnBDny9+E4Pn/xr6DJ6bOBwT4UUaJdVVo50rDPADMzMFwLiv6b0Stg7RrSoDJxD6Hnt+f6F56vni1et/yNS4KjrfkWcjcp1y4msaQ1dVP0jb1rdNdtkLcoLMGUGB5s71VUe24NbmhF5uNGqB98g8AXOpLzYeIDaCfArgQgneqpPL9h4Kx1PiLZh2BHjvzp0CpNP/hdYLL2k82ZG/EnGEfvju9BgsCfPPe3cmB8hma0FEqXIRzY5QHvrRLo0m8LRtikA5Z0uT67M8Dj4ZgC5uBv3VVA13I0YzDrwTLwPz5Wgw3m6fQCs6xYc6iUmPWZ0s2XXzw5fvBBe2rMo1n142L/AKbUqlSthsssyTAAP6M089XuF8+7eblMTIFKfMAzZDMwXPAMDDDxj/jVnbFNYrdwQqU/OCHmeYPPdCD1FROJznDG1wuP0euLdYpTTHtqdUXswe/Iaj/+knxf9IF2p0q57gr0xrljM49GjZ5/oXPa3RJ14Pu0N+XGk1CKzhGejs7D+RdN9FGw+LfD2bLvGhzqay7FeOFW7enBhKkh5wR52vPl4BVaI+dOnUqTDa9zzAb12u27YuC/+Dsdz+UaiU2EDO9kw3hv+tXfpP01vjlWeesSDPpUinBnPZqEbRzf2bA3ri9sAdVui2rHu2lBSs5gMTHjPY8HkVo2z4UTNqQxBpxzJrGfjM9BqDAo9VmR+eBj2Bga9kcZofCDhvb022OGT9NemvxtY44PZn9a8n0eBcdw02RRzrjNNaYwzZdPA1Fw9DxCu2jXI1TIsSlSQmxQ/Qs/yBsw+eqQGDXfvWijW3Q7Ymc3UqwYTQ9jIA8wV3GpsG4bXrFcrD57w8HR2AYedFsZUpluPQgiQW/vHnWfeZcPvWgh2w5JdiBEfZwlBnmexV70CpQ7mCjMbJbZgAH2AoUGNMlMw+RNKmhrOx/zmtRfNhzrVCI+cpmS7K62WusFj3npdKlezONIY84ItNqsyUzUdCmznmWj8iT0yo6oQZU4JOHQYKE9JOZ47+93zqVw6oIXJdUilOPvMumGYYIGq3DM6d6yclG8YdhqR9j5wWpHupiWEmOYZyWg9z+dS6bG5x02KqxJ9X5mD2ko8miPhXjtyyqlJea9yHRmrAPQ6G9Ajv0YzM3+wzzVIYYEtg9Z50qiO1K4IlSpshgzDN4+tZcGc3VFjZs2Y3AtqmXGcrMKiYAYYexUipUemsv6FKrMaZ5D6M1Ns+gsV6aFOqsqTAhRWTMOYPZmqqMdAC66gFx4Mx2HtAMA6z7FgKefW6/b0p2nMVJ6Hh7po9i3dmcTgjU2FBnW/Gej8ywdSl63jPBmsPXjoxunOggEkOzMFZcMZ9oyay6xdXLQ45xjwPoDNdW0XD9Fr/r1Ne4Tu1XmgZ+4A9Gz6zPrBeXKlxUrl7c7b9DYC1YT4ZnIAzzz+haDhXeESicI5s4GIdeqFtTD5ZqXvAIq7rYd58HeJ3Dmp3w3asNl6EyYTInLbwPydG9RY/Mqqs014jODrauZg8ZnnmfnTRvYQwYNaq4bGr4VeRXHKOcOlVGYbzJvHvADT1SZlTKcFKpTkY48XA3uX2GBotWM3I/DozVNYnYRz6ww7FYPesY1kRJbFx5wtbZBdDeCyhw3ADxOxKHoHIAH3MGfOg10l6XWOS5Tn42zAzM9n613i3uOv2Vs2I5bFRAJsI+SepJhmDwLiNpXPUrSjyIMFjnKfNPB6O6fQZ94Kz4ew6bUqpVabOoEl6lYa739qZ+f51kPYvCjjfRod20KsUapBR6hWwwqsR0PuU8PnPse/wBdezrAva0rkhTajb9VgB6uZPmYIHgcM/oX5P3bR6NZN4Uq3LjYeqVClQwlMvGG9kPOum8E7btWq2/WLuo0qpBI1jhGyzPP74AdAb1UWPSdmcPXwjyL1qVN/nCo1IKmzIz/AH+f+uvQEa87YjXkdpPmZyJrPOsgAdHyLx/Z/EWWzwgl1WpVysTI8XNh6kxzDWgB/nXQOFD1sU29KPEpVVk3DUJtN9YUc2ZgZ4d4GrHpKfJiRmjcNzoVVGqTEzPTXL5l1cQ78qRxLVtUIDTD37Wc/r/IumnsEHHP2g9731oBNR06gNA0kkkrQd7FAnh4SnsvMauD7mCkPRgNg/DzTwM1J5UIv1pqm6DPszSmUec877cwMOjBAzSpbODZ9fnXQloI0lt72akSd8f2eZ9n1qPAhtxgSqT0XSOLLzMHwwwDrNc9qNTK25AGQ3KnYGwGeDJoodS1mgfA5Ia/Rrd681cVOJ06j1Q252jUqFCe0JkcHsDM+zf3rNRuP14vW9TJ1GnBDkTXjhPRJYfcgDzrmtb2BclyN2xbkisOOM5wsM2TPDPNc84hXJY/E7gtValdtKONT9bkgMz3smZ4ZguKHxIoca37UpXrUKrUKcD4SQj5mGYH8+9RarxLiXDUWeHM5jkLfqmFQBkwNkM/0L0YGzDo1t16XyNVmVuhRfuvKHG1gwPvDfvUWNbHDyvXlU7Z56fGkGAeqtmjm/5D8ikXzar9sTGqrZ0qZDPfg7EDMDPBYew59feqku6p1O9ZVAAOL97DRwPv2L59umLdopVep1pU2sfaev8Aq2oPwOVOJzIGefZsUi0rVtKf6nrMq+5MDXNg40c55+NKDrzXEbe4e1m+azNqrBxmY9EjHKelm9mDx9mH6F3Cj2HTpPD6Rf8AesupPS2HgZOJEh59feYBvBTF+UW6BSuMdDtWqU/m+KjNKamyTNmPhzIZ/OYL0Nw6402de1SC3KdVY0yoGBvh6vDMDDz/ACL8+a9JsevWpU6rBichUKJoHAMDzAz+TyL196KlbYr1jQr4nQYcOumybEkI7OByWA7zBdccyMPRRnvQ9CBmSEloH/OCDPMluhFmBnmsjXjYh+I+eHk+c1uJMbYub3nR/X1N5V83o0h/M4xh2GBroi14V8+fFhxz5tzDNU8llsMDzA/oXMr54oy7AuamWVXIjMlp8OaN4zDWAA8iu6bW2KbVjcfiSQok3x4ZshmGHz+RbbRhqMMFEkgwAame9TcwntOuU7CSAPYGYGqKqm+BYYGC6YY2u4bLEyHoON5smszW+ErE8TfoZhGPvA+9aa23sI7TZrVYbFfclw3+Tq9aU+Ya7Jxw6N6jhMlw3TYfDB0F2Ce9n7RYS56CxMaefisfeOxd8ex2OK4UXrLW8M0y8zm0e9VQTAZLQlMGBqaD2Z6bHQunDG1VJ2Hgo6u3qVzPvABQfUkoPItkq3RQcseCl72XdNxO5tmtUKzBwF9A3APUbTrwb0GGCsSwmG8XiOKVDnv6qpzRsvYL3pGlPf0KXDn8sOmq+mmDwgDjilvQ3Otcd8Loi10E9g/eKWy9ms5D2OgtHGZ26i4+aG0JYHu01IBRwDepC4MOmLMmeAqK9MDNFPPBg8FTvGYBqGriEbXAPBijemAHvFTxpOzrQPG+fs+hbdKO5dhP2aiPn2z94syDMrebfYnmXnwPxE+Y7miCTn7xHziovXDbPhmk9VWza8NPlO5oNbNEyeCyrNblAWxxWUOsOH7RPmO5oAkoeZby3qvk1JjQ8DrUHnjUdJ3NhGe2ArOM9sWPhz+hv/gcV3Ak+Eua+FtFtQyf4jeaka2A7FUxpmYKXrZrldJ3mdwKRrKvDrUjP4LKxLyNAclRNZz+pNG8oEt55s1EM1H1kOfwQSM/giA1F1ktZSJWfwTRoAeRmaLR3gUfD4qwPoUcwUIV5go5suKa8CjmC1EfD4pYmnkkAYfFGAJIjQDh8UyaM0sPirEfD4p0EklAWfwSz+CHD4pKwkSaRoEgz+CSDD4oDA0jNBh8UsPigSeAN6ZU6GHegkRgDZ4alYgmmQwTqIGaaw+KdR4fFFomHxSw+Kl6KWitUIOHxTRgpuHxTRhvQQsPigw+KkGgWQHD4pYfFEngZQMgCeAATuijw+KGzIAj5X4qQyzuU0IzaClON8iIGcFccmH9ajmy3lggisstmpsaGg0cFZwGTwQWtNgNnH6wBNTGcDwUKTJNnPBR/WT5oJGfwSM9nhqIEnMkZnigM+hRsQTyQAaBAzt9mndH8NSADYjZBQCZZPBSwDFSwZDFI2fw1KwMmpBhmG9NMhvUow2IIsZnxVoIcboVZGjZmC0sZnAQQGDIYpGyxh7NOqO8eAqhTz2VSyWd6sp8/AzVK9PbMleEAeZVfJZU03s1CeezTCVYcZw009GwBWeGxRJHQrFOYKOYKW91qOapCPopnRUg0KLM4fFPACSIFISaNOqOaoR3k0akH1qIaILP4IURhmG9CDIB0N4IANApBghMEAI0QAngBEEAJ1kEgBS4zKLII2zxEtFv+tTcPijCMgjgpsYM0YQMwUpmHh7tA8yyHkVrDDAFXgGBK1j9CCcyz7xJ4NiQHsQZ/BZLRNHMlICla3vMEJ7EjecVoE9DbjBptqPh8U8ZuGkiwYYJ0AyQogQSMGAD2ahTJOHQpHWqyZ1q/CFLUnnDM1STNc1ezGc1WvQzMV08f72FqfNwHVKZefPYAGjCG2Du/rVmD2iHQC2t4p3nsCwNMmbZh4iOYeb+Z96jmBgCuGSFPP3arFYSerxFIjQNYOhdMecMlOYfhoDhv+8Ba6HbYG6Go2rU6CwAewUd0POlzwI2Ckcr8Vq5NEYATcNUlYkwabCdlPmDIN7zM1HdC+lXgzFA/EcwWFva4WJkU4lKfzDz+dY+6uMbD0x2DSohzGj968GCw9bvCqnCPTbBn6FwezzOyPCU9RM3dSU4AfnWfqT1uQ5GhEgszHfnWaOfOn/tT5vB86u6UzBjZm+3vXx+a8NoOxo045XrJ+WbJ9gAjqtVqsnMH5zxh5DNBUqlEzA23zUIP+DnB8Nce+xsClTOTlbEFSZnV6rtRGIIB53uz86ONoRn83281YfaSCz0bFcQPMlSjaMyQ3yL0bfny59YJHDfMcIlGksh36y9Z+mfZNNtiZR6/BYZZkYYHgHXguWQAvvjHS5DcXk2QpbIPZmeGa+k5HD5kN+N7cMFFAJb0gIsQMzPoBdotVmlXbXIlnXxRjmYHoMhEDfn51ccXbAPgzVIUulTtann0RJDIZs9CNYtxe8LVqNsRY/NmBnKDMADrBdItu1eHjNhg/dVz1JmW+AGAAexZyNz3E67WjOCz0eNh5Fu5nCs+IUwLc4TnJORFZzktSz6/oRbl8apMSYZwWNYz/fKJSoz9SqPq1hvxexHMh1iwLoOh1mDoyGJOhM1WegFtbw4aMBRmrqs585LRhmZg9swVjnVVptVtiYD7jeDvYuoUXhXxC4r8JpfEZu442jS3v2R5n3eG/f/ALC5u9aV2z3dRulT5P5M8F6Y9HK/7KgcPqhwM4hAdNkVfPCc97F7P/B0Il5MpVbftuswaxpmycU88D7wW9rdNo71SauqC/rMvhnhhhgvVFB4V2+9QanEr7cCsUqCGAGcYM8PrXmivXVZ0abLodKtyT6nYePRPPeH50DNmTL4o81247HfOm4Hg9Hz3mt3avHKuV7iNInMTjwMM3mQPvAPOuOBVarTayEulTpkZoP6O8jqVVfn1L1q2HLSPOyaPHpW1XuL/pJ1moWrR7mpVKm04D/aA8bDyfwJqTw0oAT5FncV6MdNuWlhsl8zgD3zga5Dw64tMWBS5bDlDCZUH/8ApEHtF786w9w3Vcd51Y6lcdYkzzDYzqnngCswsDN+1bwm6DkaScJ42NUN4Gqet1VyTWZFVb2SH8N6ivT9EzbYUQ3nHizNc60iq1io1gwcqJgeAdABgCXOHyARW3zDf0B2JpkGM9M0zJDRlaDaLbOvV7/k5T6HKY1pDG9mWBrD1KS5Jlc25KPV86Mw2KJg4ZabYZmtUJFKqVSZr1Pl+sjM2DzDmz2AtdeDL9Yd+0DkqBvAA0Y554fnWD5CU8Jy4jeeHYtHPrcWS1H5Sjcg6AYPNB0KFmqbGfqVWp9AD/pGSEU/oXoXiLw0b4CXpblZtmCdS9YxjCSBgAdeC4LbAPzLhp/q1gzqAHrRsPOC9AUe2+NPFc49VuOdrM0SYYA08yAfkA/yLYcRuSm1m26361nN6PrgzlAz2B8n8aqjmSjqnrWLUuQ87wHvBbDiFQbqrdwVN9xwJPq57lcOgw/Iub1KHOgO8pObwPyKB1ivXDVZMCI3fET7QxHAzjSz2GC50YfhqymSbmk21T3Jbf8ANTHQ8fWaGGyxqtPy2847B5vAHXgguJN81yfCag88zAhMBgYAG81WQ6U3UndNg2QD5zXSL/ZbuGhxLgodqmEfR/aA6/zrnUYA6G+tY3Aqaky3DlHEDsXVrDqoUS1AiXPbkaS0wevD1gwwAz61N4VxqlcMUKHBsp6tuhJzeM+vBNcZrqALw9RuU6SDVOZBkw86uB3Cw7DtWj8ZKnbkUAZo9y0TY0Z9Bmqz0eKb9m+KF8cNZ0sNI2TDA9msYGqThWzX6U1Hvy8aHJqVHp3j0p7W+8sgB9AAtd67pX2wqfGaXYEafR6jJA40uovaLwH8gILC8OEU47SrtKrkHlqfFDmgndZmfYC8WUqS+FUjsU6KZ6/tsN+xek/Sr49hVWgsq2KqYU9yMHOR2QwzPvBc/esam2ZChXPblzgHNQN4GGYHn1oMjDsmpXVcP2Vo7gcwYZgb2wFRVW1Tttg6dUn/AOeIsk4smPh0fOukcLqrR4d0VC57jqIMyIsYzjZh1msFW7np1eqM2oy4n3197e7mggyTfAWtA8MDA1sLJCjHXo9RvG45lHjzgzZlxzw3/OoVYtKs29a8Sq3HE5MKiGcPfnrAmplBg1XhVHuPnmeYhPd/1oOlcV6lc14VSiP1mKAVNhk4oYd4dhrCw6x9kryhQaxKejQmJIHMBozADBaC2KJc9z2bRLqiXAzGCK9oPc31smrJmjsVjOzrjoBnUDmbK9HDMM+xBo+Jz1q0fiXL4ocEqk9Mp8XQfnge8Hsw6PnWr4dXVB4e8cLM42MN52+Z4TI8c/2bWDA9ir6kzQ724QR7OtGDDolx04+VmU6OeYSTDD2P1rl9etKsWTZtKqtVfeZalPaD0dnZs+RYD9WrzthifCO8bSbDYASo0uIewz71dveyaPzhmvHXoN8bLco9vVO1a/KqoBFe8GQ68Zhyp+cOzBeyntB6O1KiPsyY5+xea6DBaRYrzTWfwUh4ME0rCUGZJOMGp+hWAAk8yw8GD7abGdgBLrboMHmGB9eC2oBotKvh6DMgFd5/dz/41Yr3tD2jigngZ5gpxh+IoJg2CsGCRstmeo2xm7hgCDP4J4HsDFxtYjwJdVBo8+bWKG5rfdakYSTA8MDA/J86p7VrD7MWREuqh+u6ODJ8mbQAB5/OvRfGaz7AjVKq1yhsctUAPWekdYYZ+MH+ded+MdkyuEVZhTqdeP2nhVeHmBs9EN8+gD+vf+hY8zVoKDRKlZNbiVK1ZcalVA2TzOceYYYLH3hxL4hcSKW7LuD1JDo7Oe9rAAPo7MFlLVqUSvSnaxxDiPV6Qwyf3cPBlZ/5+xVVz28xGoJy6c3MhtB7GCZ/74KIhz22FvcbLgpVgHRok4HpbDzHqozD2Id/+RWtsXbd0ybVbHrlA5mbVMNYzPYyeHWfkVn6PYUalcQbScv+y+Tp8qA/ChyJbOwzPoMDXUnuHVs2Txf560a4BtVuA+9MCRJDDDsADBOlG3EpkOo0S6KPbNq1zRCUecmme2ZZMD7/ADgvSbzN8WTWafXJd1W9RIRxgfuG2wZPeGeBmH5Fw+gyYNEuG37S47zsKFVHnzhzqYf3oH8+gzDs3qyuGlc/xGlv3xcD0+2mDYp9NekHvCndGYH34GkeG0Ki0qlYFVPiRQ4Nxxgo5hzVKZMMDeDX2GH0BgvYXBOw6/TYFmSqxSozJ06GDLzIHvAMPbmvE9eh2zSrmj2BSjkvVWLMOnszo4ByxxT34Zr1L6Mz1K4dXW025XJMmlXWZs0103tgGGez8+aYW9XPGxDYN988Gg61z+TxdoFHvf7HTvuYPh92kSD6z8i3EmSwch1hveYdYLzp6VECq1unQp1o2rzkiK9rxqi09hogyGZ5n3q4HYI3EijvVaq0qlVWNUjhYHJPDPAzMAwzWf4hTG6VcFMrDktlmQxnCPeGeB9H1rxVR/SHvU/VkXQpXL1GSDMk4geNJDPPBari7dVS4i3lTLmia3q84wADTwYPGYHnhh+cAWMczVivS0vNi5KjZ7DYPT6xRGTCsPAHWGfnWXh3PdUCh0eKFxzDjm8D/ql49/0fRvXXWaO/W+C123HWaVTabQosN/1abJ4PHKDz/n/wLkoQ7frDVK15wRqmwDBnT5Gw5jAeQ/zq7t7cPWXD28GLnmjWWACHyocrPj4YZngG8Fo3j9avmw/+Rc64UWZzITarbl4xvWr7zASY8iTrAyxns6OvYulHR64zVDwprwYPYZ9hh512cPM5rg9AhvwyDBtatk844IGYbextxveneV+K23tGFFVQwzWHgVI5JHr7DDsW6r1NfmQHWADeayTNBfpTUiXOb6AzXZwsbZS56PrPuzoraoo0OdGwlnsVhPuFw/ZubVo4ENiZQQfMMzX0uG3HcMzzh/1I2dQ0pLLkYzNxvYCHn9b2eALphzgk6Bn4nWgBlsxwwTXLOG7/AJ1KZZ0ffmasVRwHwIzPoQPAH4auHt4qtejGri2eFebKZAFYHGf/AA9ijmCvZg8zqLQQ5LmkGb/61QsngSlgessbaeF7hm7rt+dX0Yw0gVDTTbMgzV9G5T2ba+bzOvjWHYgNGAbFKZh5jmuRup5IGaqp/TprZhAY95gs5W4ESM6bjD4HmuiP+XOzgPYHpqdGPNR+WzPUwUqNDfDY50LstknAyGlp6aTMbf4gKayy2B6fWrA43hAubboZypURiYGeoYGqr1VL92a1pw3z92mvVT/u21e09Kkh008tQwUh6HgHRgruNAfBWB0ph4PnUdyvwyTIYDpqQEYzMNMNvnVq9RMD6FYBTWzhAwadwp2YD4OhhgYfItHAhvm0o8amtxvDbWipUbwvZrj5rbwqtF9k1YRgfVq9AwaNRY2mHhmuVslRmX8vZp55lwAUqMYACGZvBZLU5mo5mpTwKE95FiAz+CHP4JlJAtZDrJlCpWkc18EQTP3irTNIDQXHP/OlzjeCrM/ggRCwN4DJCe9RM/gjA96oEaSPsTWJqQQJGhSQJFh8UgBPACoR9FfNFz+tTsPijw+KuEK/D4pGClGCaw+KtKJh8UJgpRgmjBA1h8UsPincPincPiikXRRAypwMouV+KoV4RtynAzsToMtqQDKkAAJ4EWijAN6BAzn7tS+W3J1kFLNn8NFqw2U1oqaDLhoDZwVIV5go5gpZoNi9Si4fFM4fFS0GHxXikfD4qQyCWingDepSWHxRYgjSQJkN6mgCisngSmg82ijWJpl4wDrU48MVUzAMzQCExtWsOpAAedZowMDVhGA8AQWTz2sHtMzUc2fw0YBkpsaNs6FCVaDLikcs4asOXbWU4i8RaNwxpceVUWOcmzf2aC0e8w85+RFL3l3E8DJguUxuPdYmMBLYsNkGj6DOeCr6l6V1DthqIxWbAkyZD4Z/dJ7Oz60Haw1FLBlxcpsb0nLDvCswqG/QKlR5E08A5jeGa7n6nl/gIImHxSUg4cv8A0PITf7IahYABSAQBDlh7g/0J4Iz4F7A/wBCCVD2Kz5kAD2irwBwOw0ZsvmHhtoJfrWJ+KokypA8ODay8wH4ftGzD61ECsfOtmSXPPMzVPJe0VKeqTGG9V8mZENFEElw/eIdZV56hj4D4AgDUD2j+a1StQkqPJPNR9Zz+pNGZog08e9MozA0CgNGkCPDNBouf1osBoTTpqOaIIzQZ/BLD4pAgaNR8PipZhvQYfFA1h8UsPiiRr0Bh8UtFS9FLRXgjgzvUgGW06DKeBlAAMqWyzs9mkDO9WDLKBkI3yKWzG3+zU1mMpYMh+GpWjhG+ROhG+RSNiQIBCH+7ToAaPIEgMEEgASw+KDmW8PaIDksY+0SEEaayBBzOaQA2aY8g+tOgCWGCaOSGftFeA6exNcy2ChSZL5gbbbiz8yZOjHvcV4Q1Rz2wH2ir3pgGs0zVTe9masI0nANQ3FtgS3tQ/ZoAjOe8UR64W42eDeaEKw5JHUXrJYBGYx+dQZMY/dtqdAAz3vq1BmJhmgyQMuAXsEnmXHvcLSvGwAnpgqx6S3l4isVjNEB4NRwE9yHLdmCB6tsMnp5pr1w297xELuP2InjbAVVAb5+8UjM/eOKFmpLOsBgsDxIptGZoJ88fM9ezsXQDMMOsFgrzZB5qRzbYG15M1y83MPJ70aX7jYthAuGw6bACnSw5yb3hopXVAcZ8duCbOfnWZCBrL5V+y6oMVKlUafPOXEb5YD6A60HqQADw81NmWxVYEXXYifP1qnCsV9nyLHv8LUVSgORpBttuZpQ3mwDTz3qRJOXMkeIe8+tHGpWsGoxv+c1zbbIj0YDzczUIIzZu6hnsVhJAGfaOfoUE3m2fDDoW0WO1+mBQZ11XNTKHE/Zw6/k2K7sPg5B4e2HcdDbc1pFRhg+y7mupV60otyXA6/O7+hah63oMmADGn0BgvtuF5/9F3ghb9BgS71qR85VTMw6/YrC8YLYfvmpVB+5mJmkB4QHo4Zh8m9d1sam/wAmk2pxajVQOnnm+Z4bAXIrt9KWDR687Q7RcjM0+KZhzEjeH5FCobjg/wAB7YsC3jrEUM5tRh4PA6yB4GrX0abMYtiLW6q5vkSpJgB4Lg9t+khEvOszgua7ZMOFC62vYg9+RdT4Ice7Sqo1ViC2YUeLm8FQM9hrVoz/AKcnDGDclrhfEGCAVCEYAZgG8w2daynB+0rLPgzEnVhgHo8X9pMzw3muRcV/SBnXJfkuDVa5MqVuAZgABs71zG4bwbksHSrfqUwKefuT6EHS+GPEKnWrxOrESh4HRzMzZA3sP8arOM0my58eOxblAOBUG5IG9m9nn5Fx+N4JaitgmZish1Csce7qO3GrctyD6hj4YSTA8ze/X0Ll8a4arDE22NE8zz3hmiN7MU1Gja0jQBwMz6AM+tZC7n3IF1A0FYlxociKGDLOHWqc2XwzPT8LsNRJ8N9l0wfDAwToXJO5IKdgyYB0ZgqixHek7c200zJ2aikSYFxyXTf9THv7wVZoyozunLiPAf0I1S5kxsy+hHg+DASnIMnlz99hsNbXgnw9fvC+oTb8tmNn0NSOh4/IvUF8+jBegUFo+egMxHHgzjmGAB+TBB4kZ1JkpphiLJMzPAAwXTqrwWuqgwo9RlWzVczDPRNnsXvCyfRI4ezIsKDXNEOVwPlGsAAD86j+lRWLAtLh87atv1g/XsVnBmOD2Zshgg/NeYy29IM2GMA8meaiQ5j9HqQSonWHQu12BYd1ce6NWKdR/VsORRGTfzMMDe/P+RVNhxrg4M3BT74uC3Hno+eBtPdBgg5+zr1uuAwxEDOomABHZ2AZrrFB9Hhw6zLp05ipRpD7Ozwcw/Wp1w2k/wAb3Zt92JTjpsiEeekGAYAum2NUqkzZ8epUqvzJ9dBnRM897J/OCDm9k29f9k0ua5K9Wg7S3vBp70beYH3gea6nZ8ym0S0Lgu1uqxpNbm/9HMnsjZ/IspbD3GJ6VLqN48tUo5snmzIwZNccqVNuC5Ljl/Z9+SzHYPxnmTw2KNikvCq3PSrwrD/rKeEh97PM9iyTxzphnLqL5vO/OtnW3qlz8d+pVX1ryoYA93mCzU/75NN9tgwz7DViQFSqR0kKGf7OrCm8oEqJ6y/ZzkgD30Zqu5CczhzYYKxh0T7SNHQ+aCM6YGYGfyKx2LidAg8MeYsq3H3pMetxglRs9hxgPyLmFqw9avR2JzD2fkaDM1srk4hP16yqJbl1A9Gui3jwZltRs+ZY7Fo7As9963GuK9OuflqxCMweZkM57P1rIR+Ht1VKz7orERiVJpTT4ZhrRusFvbVt6JxOmhTaPSmXqgYHUDlzoxnh8ixnEuS/eDFPqrEU4chjwM3ejPyL2Rwlt5iybBo9DcbwkNxgM/Oee9GTh9YqrkY6hQ6rQJ7NQCMcUNIM2Q+dW0mlQax6L/PVUMPVcYwpp9ZnKDo2LssOiRKVUam+DAPHUT6zDoA1NtWg0O2LcOjORQOngeeB9ipUW/L+5AfrG9+JUvWvXUjeDvR0qiU6iVmnvhK59o+toD3r2Rf/AAcpUyw7zua4Azqso84ek9sBjyYLkUm26NclstXBQ6BGZapbIMmb2zei3Mal68pT/PU5jRD8F1neq2lU23K27Hpz9Gk87nm9y728w+haiTPobNZZpUupM17XPB54DzDBO3JRKbDqQHRqHJ5f91vNZDpDNV4UcTqH9lbqo8+H9l4ejSjzwN7swBNcVPR+tWj0Ph/Q5zAU246jMYCZHjnvNg/P865/ZNYlM3lT6qw+AVukPBKhnIPY9h7g16i4VzKNxavR3ivfFKe5gKlyQR88whhhsA/9+xa9zJ5N4kU27uDlXqdjxJxhQpQa4Mn5DW/4Y2Y/c9jO2dOqUmg15/CbG5vYDwb+hdj9I20rfrdzUe2GIMZ4H5jD3rYD3shnhoPecEPpw29Q7Yn2pKiwWT0Lb5IAPyM4ACY2q7cUCm1+HNjuUNhmlViiZ6MinPYPfrWoufgbfFw2raVNclyXrlr1SzZjm9mAMHvze8n+2uoeiXJp0nkpz7EN6RXoGeBxvGZNnsD9a6rxR4dVK5OHJ3/Sp0mBWLamHUIGiewIuzME6WOHBZ/B+pejY0zcdYrLNVdrwHS6lTI+GYeQwXaOIsaq8PeClFrkF/CaxoaLwbDDNcX45XtTqrAs+h003pM3Y/JdePMzMzDrXTaxYdftuuVu33GDqtHrdB52lc2fsX8AzZD6FjttD0hbFwwarbNCfqM4AqE2msPGB954b1a4fFecLDe0bc4ayqlF5mIxGfhPbOg+hegqacpnViTvcHgyfeYK9rWYdCAw2IANPKxENlxR5L1VjeIweYKz718eZ1hwAM1YqAuHCVoTtjXeatZ+mz1uYLD166qHQZhxZzebv1rnX2tvQ+Xg2Pc7LMIM8/WDIbA+TPsQdqOY2bTrjDnQeCTM/wC7uudboBmAedZK1Z7E9gJT4VWZr+2qBhosmfyArXmd/hqRyriRxUCvDWOHNx1KNbYSowZx2YZyT39h4AvO94M/ZuxrjflxOcpQSWHowZ4ZvhnhsPr616AvywLurd0SLnpVZD1gFNMGY4dbx4bFyeTZl+XbfVHsfipyzM2qU2aFNkMh4LJ4Bvw681XL/cXDzvGgXxef/s5Tnp8iEASjw7AVV9ob8emndVwVGTJaz0Gcw2AYdhrup8LqrwTaNu57cqQR3/AeuSnSc4rwfOHYsyzaVv8A2crFv0q8Ycnmpj8oI8frMFx3HWWrZ/Eu46bQ6PY9cuPR9VnzUYJbOyN8geQFHsa8KNAq9HlVydDmU9/Xp7zUeSez58FVXPWLS0mm25YPOgyDD3MB4zL4fP8AnUHgmdDe4pW1Ua+x/M4TPGNln5FfajDv1t8MZwcZLSbf1qxSrigPyoZnG3s7D8HA/oXqifwig3Pw3plKvGgAFVpFKNmMYH+zYB1/PmsZYEZ+6uMlYudyuYHa9NCmMm7vAze7wVF6QgXxwxdt/wCxd1BPkV55+K8zI7MAD+Dem1xDRW9avD3hdS4k66mKaEebGMI0eDG1npIH1n58+joXKo1+Wdb1qNW5Rq5RJ7MKY/N+8MvRpUYDPZgZ9ZgjsDjTTQo1Quq+HKkFwRQfZBlo+ZjRgA+hnyZ7FlI1Bo3pFMVidZcsIEhisMSnqZLgffcD7wez6PkwTlX0u+0fjxxCn+r7Yi0Nmf8Ac+amVkA+8+rt/X8685XbxL4hfZWPBiV+pVWxKXP0Tjyz0Te84GYdi3f2SvHgDenKPywk2pXoZxdIw3mGG8AXN+N9z04IFHoYUb7MUqos+NoydZkAz7wDvURbHeGcrf2fmSI9cti3AoMhh4DCPHe/WCdqVz3bUqy7OgwZOE37qzrAZ6J+QMO9V8MKwzNOo0q46VP9Xfs0Rnoez2YH9Cl0GBeNNvoKzdVZprMLW5o2Y8/YB+cAwUXC4tuLwuqs29w7onDW8aHJ9XvzDlTAeZ380Hz/AJ1zyfJi8gdRnUqSFPhPB6tM+sGPnW4vCsUe8LXqtSOpSawEJ4JR8xmBnvwwzWNomhSr1ByWxz9HMwM4IGcnRMw2bO9eL27lYfEKLwlao9Vt+jxqlT5rxnP5RnM2WD7wPs+hdegcbKdfJyHLcuOt0SRCzBkJ0YGQmH5AzXhGsQ3KJdsiuWzKmU2jypOgEcz3h5wU6lVu45NXpVZqrnM0q3qlrRgP2wHs2KeG+tFvdVpcQrnmQw+2stlkJuhKpVR0fbBhvA/zrrwQ15Es/idRnqzRLmrlSAKfCe5LlDZzAAPvXrOz7qpV4UZqsUo9j/zrsjmQdeh7FVT6b4R6Yda1GGaZeh5rb8ow871XhdXAdkOUpjNrPPDyK4tKiXNAp2FRpUkGgMwzwXWHg0X8FLDoX0o9xj8zl9Ss9h6G74Z5rn9VoMqmhrtsL0A9G1j021l69Y06sZ4exPrBacPuMelxQJJh7RGzPzWzqvDQIcN023D1cM8zXOQ1GfDc619rx/VwtBGkt5ZqzOYwYabjYGsuzJcxwzUjN9nrV4Us/wDji/hqJJpWZ6jCTJ6ytYx4YOKNjOyYZxj39CaB5xk1tZkBiZFzbWUkwz5rDTSDCXAe1nQ8Rbqm02IYhruGs1Tbeb0ubN8Aw7FYeu24zoMdeHeC4+ZfC2zMaIy14badA22c1kWbhfMwbbPYatmaVWKkJgw+AAfnXBcOgp8lswNvUWRmQ83TcbWzZsaq6pOS3weA/wAmCtYdgAfWYK+7rGHprJ6Hhtp7B8OtswXVY1h04DA22N6m/ZCOsfslphyeNsdDw1bA8wfetq9ZLBlvbwQBw9pXXpnq+dR9MLwy4A2aMGVYSaJyZYACaCNgm3qJyadZjGpAApbIfiJsR+Q+RLkHPdgrgAYwToA3+GstqwpeTPyKwh6bIaZqw5YDD2arHg3eGvN7IWZnm1prOPT9GfoGrMDfD2iiPRm3n83F4tZxj2KbyzjzWoq9nYPtFYMzMGsFla0J6G5+Gq95nBXvtiSmQ28c1AyhsuJrRc/rV28yCjmyshSGCaMMVamyo5s5otWGkpD0bAlHwwJEBSxNLP4IM/ghg6ClsxtYNRQgVhGcwBA7opowUjP4Jk0EXD4p0GXDTuHxVhDjZgghMs7lNCN8inerW+xtFyvxWoimCjmCsDZUcwViCYJowUswTJgqQioMPinTBIAyXuEh0UsPirKNDzRvQ8F48hEjBvUvRSjM9/8AwKXiaPUTRb/rRgypXLfIjCH+7QBo7ELLO5WHJ7U6zD79NSpEZBWbIAYpcu2g8QOhAJsgBKJM0068ZmoEjrSEoMk1Cz+CmvAoL3QuofdZOhvVeHWpYPYI82kAiBRwebTwPLPD06gNIEnuhRhRozS5hxRzNAB71eBZ8w4gOYh7EyYKcI2ewzLUUqMGxNMhsT2eBaa8wbT2Y3vFZshsVfGk7FNz+C8WkAzmvCXGm7alW+JNbrNVgyYzTD3Ks6wYADAdC91sycEEzlJg5y6dDeP52c1I/Nr1kE8tmbx/IGa0Ft2NdV2zeRo1q1WZI+SMYB+te+odEtxl8JTFsUpl3zhDDNV91V6dTWjbiYM/QnSdziXDr0VKiA89flZOlGxvjRKfJzNk/OZrd1LhpOjO6bfGK7f/ADKL7W1U+t800dSlTPfmun53Nso1gVz2bHGy7Q/OBqwjcNLx/onHe4Q+sASo/NHKDTbzWojSX897GCxvhbRakDhXf/u+O9Yz+cAR/wAlfFgPZ8cJ/wCeMui00M2gcVmHQuZs5YfDrjN/+Pd4PrjKO9Z/HNn9l44RvzwwXVpX/Ms/PPA174HMarROP0b/APC/Ae/7gCpXg49s/wDvxRHvrhrqEzxlB9WuH1t7F1eGTnXP8d+hus2299cZHzPH7/8Alg/yLaz2Qh9CpKlUnwing4urx4Ys0dV4/M+3pVsPf9sg+1XHfDfY9Be+g1fRp77weI5mnc9ZRcK2zQXnxtD2nDWm/wDYycE79ueM3/4rGf8AzK0cbTAlasvZiow9YeTxF4qAHicHQ/JJM1X/AMqnE0D/APuLSf410vWSOY+HQeCjA5p/LHfge34QTA/Wo/8ALZcAe34SVIPzmulnUp34ij+vJ3nTA5v/AC3vh4krh7W2Wu81KgceLAkugxOiVKA7/wBYZ2LoAVJ972jYH9YJqZQbcrcfTrFDhyc/OyCjAiUq87Lrw4Uq4Izzp9isNHesPXuBVo1L/Qbj1KkH5N4LKvHxb4USg13PXFM/3/10HYNFLRWftjiFb91MeA/y0j8F5aUANA1yvxRBG3+InkQLdAQZTui3/WlngkDzZmvMLIwxTXMtgnXuhNRgB51Xh5tKh67xB4exXDLP4iUYG8Eb0kGRzNRcG0sNiI3lnzuGIye9xGFbbe94nS9XetmlzOConqxh0GvjNS1utX0vNrY57gdCQT8/aOKkkzwBozVO9Us+t8wTpY7bN6eAB4ZqCcxzHUzzWYZ5o/ZuGaiTK2/GPDNbePVO5sGaqefiOInrqgww6z/Iue+uJRl7c07m497xbdB3Nq9ebZtYAwaOBWP+CYBubw+tY9lXdNDYAKOlG11Jr0SAGpve+hU8m54skNM4KkSY0QGjN9tURgwBezURBtLOsOGGmwwCkQ6bOmBm/sBQgNhk9QG0zJrc4xwbc2K8G2gCjwfJvUgIzDPs1j41YqLLueuruNVcx1HHFGDayOYYJo6qYezNQnpjGHWq8zMzTBtZPV5z8RV70w5PfsUcwYD2iaOSwHQrNly2HWnWTYAlFOeZ9fQnYb2sem22C2wxXrMw/dqdnmKpAkgz7RxO+tdYdPUwWNw2hmrkqT4OuttmsZcNwzgzgxabJeD8Y1rrqjMav3Fg3nX+swBc04kT34FNNgG3geDrAwXweZ1M/cMw57RuOPh+dYw3tFau3qVz8D1lUm/FNZ+5zcOYbegAYeQF8e3XCJUrhrkloGGH/CAOhURyXw8N8N6sA2eHvz8iUNmlHMP144AB5DXHsu1VhgebnQo8mTzLX3Ho860VYCy9UORfzPyZ5qKfq3lTwbwVDOM0eW9m/KlsgCrJIMRpBttvgYedXc/DDTBRwNjYxoAfzmuqB77ZAM9dzoBYevcQpVV/mOx4hz5p+9DYALQVjUnwjg6mGaKg02LQYARIjC/SOFx2541SsywLl+1z5+sJrOAMmee9cp4dejHzPD6oX3c8GS9INk340c3sMOvea69x4syo1ufTKywfW8DGl2LrDM/1Dw+aYpreDpw8GQ+c1C35D1sH2azNB9vB3W6F6tsPgg/aXA6bdr9wSWahV2dkToADz+dZ+4fRa4hzOJsRydFjG1WHtbPW6wXqDjBGfCxaJarGjJmgDAYAfXgsjw/Pej8JbjuG/PsVE0eYfMzMw36IKVb3B+W9xLPh5c87DADMJEfozwXp6sQ2OFd70y45Vv8AMyMMDCD1rh/E6/HGb8O5qPSpNKm9nMBvBaxDo8OeX5Y0GzJ5wWKxzhh1/IsvrYNaasLnrdVr1XdqtVla0h/rNU+fvEuEDZecBauzIFHnx3X5dR0ZELPDZmmY1HrNz0Zo/VXJxw98HWtRw94e1m4Z7sW2HDZkMBmbzwbP1rkSy8+S5MdOW+eefelGrzlNa02IMb68Fq724V1yyTj+uarGeN88NGOGZgsvXrYqNEitHUmNHXDNn5wQOw63UZjoZuLoEl61XmtdtzB0OjNc9gQJTMVpzBEElg5XKy1Kmmn16o02s0832ACmA940hrZmHyL1xQbzYoLrVHO9YzNr1eNnGkVaTrYH5AXj+NMgybedtxsM9Q82fkVk9clSCgtW5L0TaDzhvBabW/QiyaUfDe2pdwP3AE9p8DPVPvD615Kptk/yzVyXUmwOM7NkmYO57Gfr86wjN83rWKc1Q365PkxGMw0Q8iu7MptSOa7p+sozTAZmbSbHY7kOx/RytSqxGKkydYrcDA4jIb/r/jXCT4kXBUrfhU1uhhJp8U89WWG9V94XPWeIVeBu55UmS7FDBk5HWAKRw6t6gVu4DtypSno2oHg4d5+RebHY+AnE6zreo0im1WDhVZr2uDUcNhgqKNdt6WZc1Yp1DtyGzHqjxnzcdnN5kPkBc5en1Xh1e5tsSsJEI8D2dYLqEb0lqVQavEuOdw8qRyIoYBIjmGC92KWt35atKd0LSl1KZUHwMHvWGwzNFb3DG5p/KVWDWKPGCqBhJiUw/GAPOsvdXE6gcQuIMviFBs56GD7IBJaMMw+tdIqVY+w1mxLjt+DGmTfcyMMMDTqGKsaxrOo/GuXYd/5zKeHgxnT2ePhs2KX6T/CiDZLtPueDWYxwgPleRw8ZkMP41Q3DP5+LIuepPxqlNqmDzx94fR5FbcMaxEqVeiXNxJucKwGjowAdM3tE/oUbHGa3W6bWHWjgnvAMMME1TalBhzY/rJvOO4eB4LYcaYFq+vAqNqsMxs89aOyGAM9GH+dSOANYs6HcFYpt8cmFPm00w1ZYZgBq4sOzAoZyqhcbdxwz5Jnwe8DD516Qsm/LKurhi7cdSgwKbCpwADzIPBmZ4da8RU2M5Ta561puzlZJmGfQa6bbHI3tdsdt+1ZMznc9anQXsAkn5zXo7BbfF23L/wAK4xSgZkWvUge5EPfRejNetpjwVunRLmi9D4ZgH1rxTdTNgWZeVqMWrBeoJ8ydPqsGd1gB4dYL2tSoE6BRmqHTWMIUUMIx/ItUXCDcl52raUqJErk42eaDPPRzAPrU6T9/YNineNr9BnsBV9Y4dU2t1KLWar4zsUMMD6DWggeDFaA/a94IOKek/UqjZlh0em06sHGOoyThTMPwDBeYrkn1GNZdTs6jXMcy34szP7uz+0/Wa9xX/wAOrfv83ZdzU4JIMRjCGZ9YGuX8H+A8qlW41Sq4xhHCecoHTADMw7AUDxbWJNKjMU9+hwDhuwjADN3rP61raxc78aVHcoc7W1wz5fDYtr6VHCWuM8VanUaHTYxxHwA9GP2LJfYC7bJn0r7T249TdfozPrBc1wtijOdPqXr/AEw+5SQ6AwwXsA6VWbepdr3dY9uSalSrohhKqVJiHvzAOsF5sjWk/W75+wFKnRoYVF7rePYvdvAq2Ljhx6YxcbEkIUKmhT4Gt39hn+dTgecrS4ry+HsORLC3DrbJyTejTtYAeZ87DwH1qwpXD25vTM4ifaN9iTRLMAAzzPP5DAMFzqsUG9K3etdqMWhhJ8Z83mjPYyGa9xWTTf5PaDSm6BSpJtVSAwYRI4bOaMP4M10RaIef7PpVD4Cekc7TbfiYW1S2eVeZePPRMw3mvXFbkwbhsOq0a3H+Z52NoM7MOteUuJ3Bm8eEsW5eMV23VzMurnrxqZEA3jAzPoM/86p+FH8stvUGmXrXLqOG1WpgMUSny2c3pIGeGfyf502vDV+mHZnqGVw6rlHpRnIB5invYb8zDBeguK9vP1iLQn4r4MyKceeB97BhvWgvCwPtPPt+XUv2ekGE0wwz+9LRzKa3MqUSXoa2gHR9f/2CkcPo9pPw7esfkfGa9cYPM9meZrtEwP8Ah5o1nKDTeWpoU6Wwek3MfeZzDDAwNaYwyWsCInQQmGKZN4AdwNzepFgAJqpQ6lPhHFpoHm+e8wPDAFEkyXww0AM9+/DyKzpphXoDVZprjJg2fn6FQ5I9dtvvcSXbRicsdTp2fMxJbO8/Jgp3EWt2qFsnTqq3o1UwMGY7LOwM+tcP4qVU7S48VDiHKYjQMAYlAE4zAJIYYYLk9yekJTa3VJtffpr3rCVhy0iP1xgAMMN/Wsh6dZvDi3YFJpj9OtiBXrcMNBl6RM0TZ8ma1rJ3NVbVl3H6jhwLgMM+UjvZgvL9t8WqNR/5q4qXNcMyPgBxokQNaK8BhmGeHQtxwu401+t8U/tHVYtVpthPxuVpsjkD5YD+d4AWexn3uKlcjXLWwqV6xmaweEUGQZ2AYH0fwLlt/wBVrMy5qZdVv8Rq89NlPcrM5sAZOmn5FquN8a5uF1xhFrMGNW2poSqpR65EDM3szz3gsVJoNSC3qVUYMV6fErZ+s2QaDM2Xw6weW0XhcM/c/EK8Z/rCDWarW59KDCLVQN7AHsFkvtzSpNRB+I+9TQNnRCcB72fyKvvO9oNSam6Es4E2qPAE+IyzgGAd6oXqVOrcORcFOobManwgwedZ7wDvV3HYJwXI+9HkUp/Ce7UTzMzDeZqvtu4ZdNq/3TY7gYHHPrVhTaU/dtZhfZwzjOmGAGC0VKDOzaw/WaAHrOlzAwnPBve3965rvA6Hwl4i3xbY/a2y65Gk1WnHnJpkhnZJD5wWru30hJV80GdK+yr0CbVHjzPW1jjAZ+5XEZ9gVixryNh+c9W9eMEp6XTmfBDWDPea1dvTIMOqU8JUsI0djzmsY5jhV94Vt+q1Kj0qDLhwAqGDD3L7MD/fLtEDjNTeGjEKzn7YolsA/D5WTU6dvMzPPeB/Wub3ncNjvOh6qt8PXEI840iOGx4/nXPbkjXVDqnrWuRKabVRPAwjnmAYJfNtbosm6rqol6utzr0O4afFjPxWZ057MGdbeBh5D/11jDky63PplGvGpPGycN+nwzZZ3mefelR6DblSdmtzqry0eLD2U4NhgYKRSp9NevClRLKbAI7EYzzqIbGTwNQ53QOJB2/ZNjWJqRIZyGIHKmEcMNY9mZmapL8n2cdr0SpQaNnTzMwqTzWx6M+GHWtHGmUat2lz11SwjNW0ZhrHgcWSf0Lh/rK2HpsuJoYR5R5g9nmBn2Km/htalMtKlWo7Bt+dVYE1iSEqHHlnmzPDNSOG9eiW81IrEuuchXWHthyN4GH0LKsm3TWolKrlGZ0gzNnfmYeQwUet0pjmooP7+aDNB1qpRpV2xTrFfuamvAbxygjxwwADPDP/AALGzLYpsOHCupidJejygM8DPAwfzWckwKjTaXNbtx/CEYa4R+w10Cxo0WpW9T6HKiVKfIpB81MjmeASQzzwXPaQBW3/ABQYfjPQt72D3fsBe0/RymfYzhR9+DMAmByzwdHKvABh/jNeV7zvCyrwu2O5SuHnq2FCMNFlkAA8/IeGzyLvvo61iuXDVolArNNOHT6prymYh+5w/wBwXTwWPTwTG/eKQbwYrKVhmVSpB6b54KwhzAejrvyhHmG4crPQPBWsMMw3qOZp4JLbIqw7ywBI1ATLxtgaPnGzUKTvd1Fcf8lwwXFSquRga0NgGG9cXn75RP8AnXaOJFHfqtLkNsdYAuOSaVOZP2a/T+h5/S+VzeERl42Xc21IemPyetRwA/IaP2JeIBrstyLWn+1V8yzm0CzMN7DBxXEOfngBuYLG4WuGT2KFPhuGWuw2psCZEDw3DRvQM3ddjoXO1UT1SlABsdhhgmYe81LmQ8C8RtAyDeXhtq0LClG/GmA+2ul0SY3sNc6gMua4LYQHsBXz+Z08LooG2bQYKXGZ/drM0qZ0ZuLTBPYwXy7dS1Z6FIyBUnP/AIbiMJh+dYYarUwzJNGCBmTmpGQJhavkwGHh9mqeZAbDwwbWgMwwVbJ/4MyV6MKcKa3j0Jp6HgexWpnh4agyXlqxs0y3gYKb0BmoQPN5qRzOzTU4elzPu01hm7qJlLP4LxR42TNRzDAlK5nAVCM8yQHnsRAeSjp4FIsAPAVHOSeeCDWPBRz60DpmmjMEB6iEwWOFo570WDZoTTrKBp6Ns9monJqzNJkN6kUj0P8A59NRDZcWmejNmCrzh7+hBTgB5qwZBzBSAh/u06EZtFmklI0UGiiDIdasYxqOEZWUaNggsGT2ITBSGY3hJmSGxUIRqIfWpBqOatCIfWvuHxTpgj0VYZCM2akBA8jaejM5ngrqNGb71Qiw4YB7tWXIMGPiAntEAS2KUYVhwGA2Nto2YbHtFYcswfiIgjACh6hcm2fu06ED5FLAFIBFK02XARM9Cs3ozel0KD4YGmw0o5gpZgq97Uz021YZkdCr5JtqzONn4jihSYbmapKvPeKr3g92rB5lwFEMD9ot4QhGGAaiZOSHnUuV+zrOSXsHfrW0RsPPVIwLYrBmpBiojMZvDP8A4EBg2yrww2uGZmakHMzWdOfh3qOdYw94nzHcupkzAc1XnVWw9o4qWZWzP3ip3qk+fvF0x6yL5nSKbWIskAbDNSnpLbIai5lDrb8M/DNXYV5yYx1p8x3NQFb3ovWRvO+Gs/DebBrxFd0E2JJGba5rhtte014zDerYDUIAbZw1O9WUPDHNxc1ttnQZfUgGfxE0cxgPD1E0dSw94sP1i1BltUleo8SYJhp71ICpZ+8QnMb73M17+t6wMy1XzPDT2KdAoLDOGbBrW62fYCdhst97avbPCvoNHPmvYYLUHSm2Rz009AZbZEHG1NPesd7bQr2QP8idM0aA1iDM8lCmQOcBSE6BotmnqDK+QFSVU34zRt5revM6w6ay9VthtkzNyUZrphk5lUqrL5jxDVe9MceDxFrZ9sNmeoap59seF4DgAvqcNuJShM0fES9cfhoHqa+yWmoRw389i2YpXMvvSPaK7pUxxk8G1WQ4DmGppq4psZvPxFlbaFwB5hqJGCQIzXI6jWi3/Wg5NlO9CXQoQa0cPZpKQgPTRZrWPNSAebeDQfbAwPzqP3pAg55fPBnmc6zY7/IVDr0s9hmotgcUZUOb9ir0YONIY6DNdV5nBYziFZMG7aacttgGahF8ZkwBENm8zv1G94JroXLOGPEWUy6dnXN+1sHgBme9dIeezWy0h6Sx+IlGDWLw1RSfx0DNSfZ6HFrhyLCvPSmQBtvYqqNPf5gNRxA88+8eo+5moRvaL62wnbqECfE5UM3FErFVg6Rt6+CwXrh/3ZqC9JOS7m+eafMbTp8zmZhuNubVYRglAwGCqYDLDzobFuKVROZawNzBLxxkKUDf94rAHsBRVWmvwBNxY+fPf1faJGORC+n1tjeDbiz8mpP6upqKvN5AZrpjhRtas16cAabZoTece9oqzoR85mtDaaGwld0o2HjBhZnWM09GeP3azte21OTBgeA2DKdpsnxdR9zYqelW8+9474GYAtNGo5vNAw2Bhn3rgtZmfUoJjp5rPvSc+hbCNw6Ye/a5zx/nT38ldNy/anv1rHuGB5lw0Yb1qK3atOpTWbGZms49oAC6e7wIrx4CokZmpVJ/TYlAAfOrVlmC8GdR2R+9c3u27XKJVzYof7P581jvwOrRoYM+G+asuQ+667fQuK/yly9gS6/Gz8i2tNvysyaQ1Et9sJJ9+a5vodPSsK9PYprWo+5gqJm4aO9/TgD61T8VKxOh25zdZgyY0sOjYuf2w8/UovrJ+LotH3vAsfpOl2AKrSnh2SwNVkO5+cqnIxPZfirnj1S5zVgxWDktd4Auh2fb0WNCanS3wh4dYGunh9naL4W7jQzMQcNtV9VrdGpR+O5n8ioZ/E6nUrNtt8JmH4K41PuGuVWac588PkzXNze/HG2w7LXuKlOoMcH4lD1nT78+hc3rd+UqsGbkqjScz71hZl5yjziZs/WZqkeuR89gPgf0L4/N73evDrFNhsSY+uw3gub3hW5UCqGxoG99AJU28LnZj8o3K0Wg6FnJ8+oyayc6c5n5Fx3/AHG0WHmarMd1GKVJ/Qk9Z91Tz5r1HMCP5zBaiBxUrlHY0IMGAyH9yolV4x3bUmjiPvsg15ABX8y9s/6t9VDqS2HmXQ/GUd6frEo8mfKqXiPub1Ezw8M1eDawzbPrPBDgwfvM1WHoB4jmeaQTMOhXge/TBGCM4zhomYG1fonCEwbkhg+2Bh5DQPQ2JLQMOdAKQcYw92jZjPmfszWdq2gxqJTgdB+c4AAH4xrmtSuS2aldVQqVNYqVb5UM9WOGYMmrLi1DrlbuOiWrrvMwn+vBR63VeGXDq3pFOptz0qiSMMAM1jh68dVW8259/VOXUrteDQkvgAH5M1nJMnhXVZ7sq7rjmcwfYZmeCqqxMpVY4kS6axcEaTCOS/8AeA6D3o78sO3IdSD7MPvRnQDA8NgGotrDPhb1sT76aiweZeoJ/i7E7D4exZ92u02C/hTw6AePrUJ465DDQlvq1t7+ZGnaxLnG9IfDAM+xc/5W6HMvaxLYjtUassTJPyR1K/lvbeFqlWHQAokT8YwzXEpkxypSjlSj3qzokxh4DbY9qAZgCxwhsoDPri4DqtSqJzMDMDz7Fl+IVzv3JVziPnnHY7E9w6N/mJfNBhrgZ4LH1VmdDq0h+VENnPzrzhgaj1rEOlx2zc3gaoZ8lwJ5y2AzA1CgSQeaOI/+0e5NPTJkqM01rhs862wGTrHJzdfUwU5msXHj68bp3MwnO/BWs+jsUR2FXAghMiP7HgPZsWird500KXFtyw2M2jDMwdDoXoj02BXKPNj88xy2uGYfOCuAvOv0d026bLwz69iG4eNjFbpceh3HZcaBIhYfeI4bzWfgV6m3CbsSlUqS8eGYGudbvXqS2KlQaPX64/yciqB7kOs/IuX35SjsO9IUuI+ekw8DwZ9is7eu0zsaDBqLn3unTPB+gFH4x/z3ccTxOuMBrIXvF2mxbkqkK9TDmYU2Nv7FzU62el6jpTetH7I7x5rVWfUn/Vrto1zfHDDRPyAsfcNmS6DcDQWr4wPhmHYtti2ZpVch0SR6ugsw9dkwMw8id4b3a/cNSiWjeN1MwKIfWcgNij8S7wnQ6M1Q4sXlpuj4wYLJW8DHMRJ0qID0eK8D7wPBsMAW0DqtBtKjVK/JdpN1g6lT4ufLSwPrwVnMpXCuZAlxbuuONSptOMwwggeb36EB1ixLnqjrHCh+NQY8WNzTz0sNE8/IBrpvD30eLKvzhE0/dvEqiUGQbz+chnAzP61dwPL9Ve5y3HX6dBwp4HgDpnmf51lYx4KwrbL9mXRWLcty6gqUJiSYBLa98Hn+RRIzO3xFGBe0pmLMjm55AzWt9H6ebPEmFUeuRCAzjNdhn5FlYABAozspTabAo32Ul3BBncnctOPPRA+v6FI1fEi3r1ufiJLfrNv5zZu8Dj9BgvRHCj0jakfC8LOqtcZpVxUjNjm5ewDAD2fwLmUzhRxQgWvTOL9NuONeDr0YJUmCB5ngfZ1rH8DZ8W5KvdFq1VgDCV48bV6AMFsPYHBm8L/4hQKqdWuOGDtLk8rvh+2+frXUNEwd3rjnB+Bw2ClzfVtVkwJB+BJjx+jPzrqEM36VVDpVNfh16PgBmyz+0sgq2hcYfFOsmoTNYgzJRwWwOHIAP2eXsNTeWcD2isRwgUemvnWfVQSZp9bphmZrnvEXg/Tb2sap3PfDkw6hCZOVGwewwNdNUjDmYEiCfsnwwNY3C3jfhFatOvazahR6xTgZuuiSWKvAkHsM2M17lpTzcmh02V+PGA1w/hdbeFGqZn4NQpcnlYzwdbILqFnyZ3qv1bODA4R4K8DFUHhQ/AqV4ToMHP7QvAyHYABmZnvXbWYbEMYjEToYZAA/QqpnYrNmT0JghYSYEGpAHPMBJAPdPBmB/kWMvCw2LtqlPcnUYDag74wBsBk1sGXs1Li/86jC0GHTfVoyG4Os86+eeBms/JMJlGkQaVE5mRnypm7MwNnz4KRdt1VGmzWoNOo8aTges870Hh9a5bxpu2zrMkW/c9Di7+fwqWlMwMAMOs815hDS23Va5w95SJXK4F1U+U9oMmbx5xj8/kPrWwtK4ftV6wqreHq8JJgzLAMAwXF7GqVl35aUKzn7nptBqr7xyqVo1JmS9+fA+td1sOM/ZNqW/aumzgZmEx4w2Aa9gR6wESBUQflStHPYAGGYGazoVWjSeYlev4zwMHg9pdhroF82SxclO9VP1VkNfsaPef0Lj8+NbNmXRaXCelH4sp4zkmf0d6kaCTXp0OnOsVGjT2ae+Bh6xiPAeAH0GsDVaxfFscKKxOlyo1t0KnPA+Z6x81JD5FiuJHEKuU2s0ylP1WBRJFEufkuRz2AwAZ9HeBgaqvSW4r0PiFa/2ctyvmEg3gBmPyxgzJfPZ1n0f7CsZej16h3VS67f91XjDueQcN+LDCWGclkPJguOcLq3wupV2wqlf8F6TRDAweadZ2Knn3JX+GNZaptOcgVKoMBm9mGxn6DVZeEy3zn0qq25Q5JwjAzea39f1rl5h6DvyHwntWZTalbFuTAl1dk36bUPWoPAffvA+zes1WPST4mxqdCYgh6nlsM8qb0SYHKvMeQ43QsTf9sVig2lalRuOpRtKbDM4DPkDZszWVnwKrStJuqwXgalBmAH3guQ8PWsy53KPa8edFg2w80cAJrM6QZmbMo+sADoBcq4o8S/tba9v1ym1I6bUMDYqXKHgbL/AJ/z4LkQVWjMtNMOQZPKMHmefnV7WLJpz1OhVKh3H90P2LLodGa873R4c5vmiVwxiSqq/rGGeHnMNixnrKVTRPkXzDXDQMOwwXboFEqV1Wa7TZdSPmIp+MGGb0YPOCxVV4XP2xyVzsVkJ9MPPB3R0TB8A8i6eH2U3AqPY1z02LEqVv1wI2GD/Nh0AC6tPtKdWGolfg3rGZj1Q8Ho/QD3n/OuSUq8KzVaDIo2vGjaEnPPozWzrZ0ajyLfptsXj64pT5gb1PkBgcaUf5N4JcdiGqjXnVaPal4W5Y8qGzHMAOeboZ5h8hrlr0+DW5UTUYzdYPPMOjBRHoz516oBLceZZMzDMD71o7buGDG4eVWxJ0FlnDOVDkB14eRRhcJFVZfZtqbVac2B8qeBgYdiiHc9HmWlCpT9OOZN38yYdAB/nVPDZfk1QzclPRoXLaGiHQ9581Ihz3KbRnaHVYphT3zzPDesV2jxqk/TdXQ9kYYACDDWhQq+5Fksxw/Zns8MzUu6gtmlR5EEJUmHUGDAwjuhmyYfIarKPUqbMhVNg6yEAAPMI5hsP6PItma+uEPXFLjtnFk8xslSafHPwQ+dZyqgxAdalwWMKU9g8z5wPyGriNXpx0sKj68NmXo8qGAdAf51ipLM7SNhucb0c/OtRrXq39oZA1KuUoGYjAaOw/0K1o9HpR2rU7qcYqpnSDYAI5vbDB4+v5OhYVl7Wwblt6zWABgt0EyJJ4ZViDTr45OW5hrUZ7oNgDzAANe2IVNrFNOaESc/JMz9i716J+ddbtWm016bWIMqVozZUMH4fLhgBgub0S27fvOHLYcqL1K5UGAORo5gHnRV4INEqkeDQ7jmT+VZD72ewPyfIuC4HSLesCqz7ojsUBw6lT6cDFQrEvoBkM/P86/Q6iPcJLV9SMMQZ/rUIxyo0TkzOVm9vMz7N/1rwfA4haPNzqHVWTdm0QIVbphs56wAHYfn71sPRyvzibTeK9KoEG4JgQqvAM9J48wZ+ff14Lbh/spe26xDYnkeff2d4KsClct4bakWNWGL2oZ3O4GjNOS/FmM57M2TwzD61oDgL6sc2xnDjPqJMN9kD0288Fpno29QZMZXtixoVt8PaZgf0KQFYfVlMpTDwaemqcKa4y6be9bbDUmpHJA2HAw+RZepUQMDf01q3qbh46dh4A74jYfnXTw83Wi/G3GnjYZd1HwUgIYV5o2KdBeedDyLqtesO3Lk3youD3nBO0qz6NRGAbiAefnXb97gw4O9Gfhumw+GBh2IQezXTeKlscy6dwRPan7b5zXLA2Evo8PN2OblWbMk/OtHR622Bcq/39BrH+JjqIgNbXBFthWIb4SNTTM2jBV/LYElSrhwaCJKfw2YZqzzbNc70EN7B0G1fczgqxllgC1FImBmC5bdEWvYFVABwzVnGrGBbzWMZjPgGop0ZmpG6IA2ZrjuG0W6AzUm8APUUhmpNrPAybLG9GzJcD2i48OmLahmqgB5tuJ46xn7xZ8Dbea8NRHpmi7pr3BtrfWWYe0TWsBqkjSTNTmTzWeFpTyhSQ2KU90JgzAB3oIPQj1sEjeipZsH4eot0g5kPOmnpgZ+0Uc2cFVHqA7gvMKXXOB50gktqnz+dS2dM8NRXcMlmB71KA8lCPYgB7BcrVbYN4qIewkzzDiMMzPxFINAaeAFI5YMFC1SYb0gNHJDemgUCQiZDemlLZ08FIAzR6OajmfiqWHQgAGcB6Ey9Gw92pzJgaR6Zgi0Qw2JpkN6ln0JlnrRCcDLealMshmo7J71NA0FgAN6SizAbBpOg8okySqFK91plOn1ppWgYBvUjRUcAUoOtWHo0ZwPEVgyeHtFEAzwTwIJb09sGvZoGTzHUSBOgglshsSSA0QKFlh8UYI0gNA7nsUKSy4Z5qUCTweEiELrDBNBDczzcbUgAwJS+sEFccP92lyYf1qww+KaNPApJNNb/DVFUob4B7NbAwWcrZmyWxdkJZWSzLMNM1n6kyZ+zWimSdp6aopIHnqLs4bctoXOTozWmBqIdSfP2hqU90KqkgGfhrtY3Y3phn7xRzM00Z70gNa4Y7IzUfD4qQaQBsVm0HvVnDebDAFCMN6QHgSUjbRRg5nDxwBaGiBEB3QCUHz71z4zzFJl7RBY3wtot2Y5LHY+Bgokyt8s0eCwUOvOY6epsUg6q4Yda5ule119qnzPDTNTmXnJOL7jhrMwPGkLYMst6Aaai4bQkQHnwd9oalnqSTDNwwwPsVIbzjRbFLgSTMtQ1zYXtoGXgzBsFcMs7VnGT36iuI0k1zW2hdsn7tTVn+c3+0UjnD865mywMwQm82AqpOY4Z+0Q6xmmEJr0zBKNMzJV56iiPPOMlsV4GjembfDNU8+fvNxxQjqTnnUF6S4aYAyTzUGSzmHiKQbyizJ7DIlqOLpiHNapkwGHg01C9VRWd4Z5p16pN54Jrmc12QxuAHsTTJ70Znkmeg0eLtk9oJ1V8Z48VLA1zXDeCM8CQazn9SR9aB7oXrR95r4Jc18FBM96WsvMWz2l5/BBn8E0DwYpGaYLtLzzFEHWousmZMzBXhmw/Fexs3wu6jt/eGTzMFqLSrzFw29ElNt+KAYGnpknnIEiKfvwwWB4aVL1JXqhR88I7h4AC2wN9P8Aw/8AjUTln3h8AM1d6LBl4isIcBg1bK4ZE6bODrbUKTGf7210U4DGHs1VTKOBn0J3HSxgRnE0cZz3ba1B0dzLY2jClNshvb3rp7jpFbFNAyBzZs866HD5VlgNNc/gG5DPUVgdz79BvZ865uaNrj9C1u023oB4Ob1yWSeb5rYSak49IwN/NR3o0R7DUYDNOH+2i/1simtZz+pdD+z0HSDBhL7PQQdzNgAwW/0wx6WHCBVXvECIZgpcOz7jmDrnB5Zr5zXRYDLbLvgK1ZgPyXVnfuf6NvHrOThZlzYm/wCqj0gPrzU2j23VTmA25BMz712iMy5pYJ31awub7LX8zNUqA/DHTfbwNTnpOCsJMNsA8NVLy4LtthIZnv5K7Z8YFkuZ3Kyhz3za081FLiD1bOCDBuSlzGvXPblKIzl7ADvWmu2sUbmDgnEk5d7y4zxRolD5B2s045Lx97OGeCjuVhCva4alc7WhbDBsx/Oa5fUpNSB31bWakHzgCuLbn1mTHBiC/gZ9HkWXuegsHWziBVZL0g/yLh88z3C7ONasOAEFucBun5FveG9z1Wg+HQIPOb89V7ZguTxrSnUqZ97i5/nVhAoN4zJQMQXzjRz6MF5HMt6lu25KbUrSl6+EyQAZgyAZ4Lykd21iY06w4/g6B4GC6nMpt6WlaWDB56/WZhvXL6aFHjC65Um/FfPPNbbGo4aXbEok04PNYSJR54LQcSJ9YegGcR8N/WAHmuUyaxQKVKB8H1rqPe1nPMZuP/kTsGZhyaiDWm/mH1qRXoc6lU033JQGZ+RXE+t2rWHdeJKAA+dY25Lh5zmMGDBrsM+9cdzsY+pamr4jmaKMywB+HvPyKK8esWori2PVpyuVqTZ/kWK00GZ3K82DGGn3qvkz33j9pmtbWDodKYMAbM/kzWKk1JjVNwIgApMI8mSeBt4GCr2dfLsw+tO1KqvvZsAfhKq1nGT2LfwNAzJMNjbeZqPJB8PEcDBVgSZeYeOYKQbM6YYAw+ck+9bBc18EbJ5/QmnobkY9N/rSA/dqB7Qr17cZQhGdDtGjh+9zzUeg3zxw9vXLShz8+iO1s/jzQPcQuJsCG6dY4cgDX/5Mk6wf4FioHpSwTqkum1XnKbEhe21WV+mcje1K6vSCqWfqPhzTab5NaTnguecQqr6S1sUYLjrlxxqa1nomzHZAwA1z/i76cNVeadtzhfBejB/8Rlhv/RgvOUy/+JNwug/cd21WpYHngb2xYXY9V16f6S8+zTOdcFt4n/SHgMHmfoNeQZle4jXPV3bclMHcM0Dw1fxlu6lxjvy4aM9b79z8hEBneHnwXMrVvO4LMmHOt+doumGB5hmBrm2tpeG9kyjuh065TtH1dnsz3gtRPuShyapIBsM8O9ZKm8QrgmSJevvdm9Zsh0LOTJj8OVpsLjv9bohrZh02sPm+/gYAexUtwyQxCK2Zm0gOpMMm0ZtgB4bw7FCeDmZnNMOZx+8DWDa0R4D0k1Del02VrqwZwrFZiUOC3nrnga0HEK1aVaUCJEp2yQfWumGOGPOpGbupBcPV84BmrCpG5MaBuWf3jvXSApsGlWLEfgsMg7hnv3rOVK22OSaqsps8D/pEc8wNZQYYKSy/DLUDYrp6G/W7ed0298U13jhXAtyZb1YbcCNnF9i7IDsNYKmwKU9DmsQdHVfe8Y2TzA1oKqA9FuSyDblVJmM7CMAMDVYdHtmHRIVSiXUZ1PWAHgAOxS7Vuqq0FqXQJVNgHSje8Z51noUSSzEZu2IxOgvRoWsGw/IiHcOJfCXhrXqDTJ3C6pBVbgwzNoDM9YPIuX1W82AmUymnaJ02RCPB4ADDet6dYrPDqqHeNpORqU1S2fyPLjtbvav3bcbtz1V8OYfk654Bgtbhabdsl+NWTYiOYBhngthMnhXodHnB7VgNA1zeSc6vXNEY9tLlGDIB5zXY7w4acW7Mst2W/ZTOEX+kR5IHh9YLjvhFScPn8G4kQGah34IWZNVt6bqXBmbIfs2AdBqvO551Eo0KdKgxnnZWw3mXt4H/AJ1BZuGq1t9pinMA8YHngYKMA7kolYetmp1ycwYSM8wMwUeHzz1uR+ebjauHuu8Fe3txUqsn1YxoRvDPB5nBXFw29Q7SqTV6wYmcKaH7PngGa9wMa9ROHMm4aYxOqr1NhTf2k2gzwPzrrtSsngDZ9vf6H9ZR/wC0c4eZrhUyGxMmuy23Ho2ueeDR7AWlvDTjWrT4jDEwN+GcgOtaxYr7b4dN3zVqnUbci+qrfYPPmJHQCzlVZYpVRdprE4JOhhvBbOz43EazKCFzUKcy9SquYHMg957+9Li1cP2tkR5bFsxqa7FDfpH7ZXsVVbpvLU2mfewMDDAwBRGbel8w05yoSWtYM2mnt5grCNAOpW5T8POZmtbJt5isWfT6rBlGzOhGYYB3rLYnctenDqU99kadXjoRxgOTpSfY594GCzlNuSv0e4wuDh5Sjja7Oibxw8we+c810jgDxFl0pqoWrc4Q2ae+evzdRewZAN+YHs71n+JcOx4DuvY98GELPD1YHZ9HyLfYooHE7ipw9r0uv82EP1v+04Mhgf0L076NletJmmnd1lv+tbjm/tjNWnhzOC8SV54zlH/OMmZH7OYPM1Fo9epTMhrAKrAmh0S47xsqh+mU/jHaV4Vl20ojEwJtLewnnyeYMn5AMFo5j1Ro7UhyzosyqgHWB4PAYLyP6FXEu1bGvK43LnvyBDj1Rljwp285J5nnvXs2TGtW56cdx8LruA2mAze9XPZgAf5O9UKpm/2Hn+VqtsVKiSD6Oe2Af51dhMnf2X+NZ+3uJ1gXtF+zFRuemz5DHgPA9sPP6FcXhUqNw6pYXBVakzGpTB4PA8GZn9CB+NrxubNum+NNPN48+s1FnhffJH9lXIECR38xv1gWEgelFwvqr+nSqddU9r8WJRzeD/GtnSuMfDmY+LHPVVl0+gJFKMFYhUHivWIGf20iwOSz0PWdOezZZMOx7yLYM3trUv7VRHIEyhMM5vOsnmeH0LLybeptB4kx5UGKHqe72TCTEMMw1w7/AONM8MaIxZl+XLZTZvHR5TITYYH2BvQW0/jexR2oVcbt86xbUrY9VqTv5M/3wdi6rDmRalDanU1/WjygA2T84Lg4WfB4Y8S6e/QHzZtq9ZJwpMHsjTeww+TrXeIYVFlrQnAyGHkXNa2Mv+w74qtDqEuyotKmVV/BhlqXM0QAO814q4wUGo2TcIUrixTabGmvgf8Ao+q8/wA4Hk+Re67w4kP2fF1LffpT01vrCceDIB85rxjxv4o8KLzuGoP+p6bMuN/D+dmXtaEyYeTPesBwKifaYLhmzrRs7kIk0ADl3T0TZ+j517AsP0tLLs9qn0qDLOTT342E+n1EzM4EoPIfWfWvKTNYnU12sBWbqjcweD7JhvB4PkNbJ6lReKjvqOJFpoOwgA3p0GAZ7/IZgvYsd+ur0ooNbn0eqg/R6xHpx65hT3jjSYweTBTbV4l2JxI4jHxYqUF6A1Tjz5ucYAAHhguBUG6qHatZrdKl2JDrbs1kGWXjZz5bZ1/7aqeN9t2raUOmTqdTtGRVwzNnPY8H0K9geNPFS2Lz4myLqvHkK9ICTos+oT0WXmGMMDM1h7z4isXO6Pq63I1NpTAfcIJvZgz8/R1qvgU2x5LWvBppw/B0Dz2ABrKTKDccN2FzVDqUaPNeAI0g4xgEkPOGfWoiw1G0JlZ16zKe/vc1pbY4hVGghLf9sDHgw9+GxNcS7SodBrkSnQZXLA+AZm6eeZqnepVKgXCdvvzpLIB0GyzmotC4uqq3Hc9v0edVXM48V7AI/eG9SDrb9yUl2c2xPDlcOTDPPYqSiV5iNIqDD9R/m8Aw3hsNG8BxsPsjXMI59YR+jNY3AZZqs4HQ8MHt+G9WcypU2G61B3nHzzwBUkaM+zN3v6Mc/bGlMmczNCJTnDedA/BeAOxRhbXQ62/R3QYoFSkxpEowA9I8NikfzUczlK5KmPfgmHnWfqU+m15qI/rshNAMJOAYBmnY0CIdLlhLfA5vt2Tz2YfP5FER1qR6lAodsNS4tZpRzJs3owMA2KngBLqUjPPwYXjgClT/AFB6menHTalJPDBnfhon51Sw3nNI3w2B2LsgXFSh3G8+b7kTOQ+ZmAZ7zD6EpNBueHS49clxGfVT4Z8wD3QrClVVi3qGFzttxnqqxM+7SOs2Q8n0LPnW6idBqdOfpzJx6vM53MHjDevGSRDqUoM4mf7L43zmrCZXoNVobreeEg/dKPbb1cpV20ypPvs6r+DBg0eZmwezBbCqwKbclXqbD8TwmAA42ieGfnU3Dbbn9yPTqxMhG5vPlgAzWaeZfB0HHG/CM9hre1uTZ1Nq7phbgTI7HsQM+tBDsmjX5pP0OWFHz9zgbzP61t45niinhhFBwD6zAABQpkblnTiGf3hvsWluHhRelBwl4etaeAYc3E34fWHYs1AgMTyPUlcm77nmAMAM/IunexZUepZnHo3sXXzwB4w2Ga01KpWEWoUeWwByOvA+w+8FSVuB6hdp9AqTH3hjCaZge8PkUv13OrdwvVJx8A5rwdIGcAD5/rXNYvbegMSZBm/ny7HtgA8Fo6ky+9AaiP02NGhMZ8nIMM8A+dZQAcjOuvxGM9+B78N6sJ8+sxo8Rt9gwCoBmyfYf0Lg7R0C27YgvUOFE9TnJm1HowPrXU+BtKls3K7OgxPWs2zWfuelJwMDPrM/P9C4pZ94NsxYTEqsnR8zwjSDja2HnPBbWlcOrtt6stRKbdsaTCrx5s1OnvGGsH0diy2PU1N9LSgWlUqPbjdlhAj1SpAzW50g94Pn3gC63xI4lxbSJoOeZh+scOQZeMAN4O8/kBeCeIt1U65KpNgzqPJ1QjADMjveMFRVi5LgvORCl3bLOfT2IfJU14zwOMAeRd0cyX6K29eGtb8is1zRZ0GTewB7MzD6FZ0GqsXCw64AYGweDwZ9B9a8D2xWHLepsQIlcn3DImwz5CC94Jw8/bd/1r0B6NN1fadqp1WhvzHnQBiLPgmGZveQwP8AJ1rpjmMPQRstn7tQpMMOY1FcYZjvAwPvA+xRHmdy6WOFU9GbMPZqnehuBIPTWr0VHejZmtotbKGzn7wwSkz32WgbU2pUqV1sGs1P5pl3Tczw866Ytja1AG6rTtCW3mC5ldVsMRpGvBY7966LDecw+RU8+G/Mmm3p5td67OG2TKvUqK9TjwY3+dZ9mgzpL+gDe810qTSmAwbbb2d6GBDbCfpsROhbfSw6WM+xMsA3ymTPyAq/kKrGfNtjsXWwoLE+Kem31qVTbPYjNab4J9J0ue0qNUpLDTmmbwGewwXQIdt02Tg4+GeCtWaPBZHTbb2B2IwAGXdRvoXHfN2OmIAdtwZL/eAeRS4dsQWT8PNBzmCNmvMGeDbi59tseDsymsAOGmspVXmGc2NTetNzJyXfEcWXue2zelAdNcMDM96qGNqdmsSoZ6eooNSr0vmM9RTp8Bumyg5oFWVuAxPYzpW8/IvqeMJHTbqMHd5rRM3bE/HXKns2ZBtudYJczgtPmhy/Tbtca6mJJ6e8FIkyW5LWxcfh1icA6eotBTbqfZIANxc1+m2jmaCSz4puKRGeCMPtEAG3Ma8PrUflpcN3M29ixbrCTJlm1rsN5qnCpcy+YONmCa9cGfgINBv2iYQdmPNgYb08zVW8QbcNVLxgbRqODy2wz22DM/NoHEYSVRRpJ8vp7E6y8+B+Iua4XtoAMFNAwVLDmA93qaDy5rh0pZyfvCmgb5qq97qK9hgBtAuZaskgoquJMbeqx5lzLoQApENRwZcUgNgoGnutIJOCN7eo5sqMCay8ClgeSqQDFSGTUYEs00Ab06Bp0F6EB4EpAPKPh8Ut4IJeso7zyAzUc1QA+tNJ4ARYfFWBBSGUAAnQDAlqJbPQnU0BqQAKQGslzXwQmAACgnJwJWhbc3+8Rc3+8VJzDiMHjNMC4B7Mk9kahQ9hhmrUwb0swUAmZmBhvUjWbMVnzecB3xEbLz5n4biYF2BgpGfwVSBmpGbgJgSzQGopyfnQcy2agOnvWVuE9xrRZ/BVtVhsPe7W8DnjwZklhsWj9SMB7RtR5MNgM28F3RbmuGJnhuNVTzOZrWvQG9XxG9ihPQIuZ7N67I5nNcM5yG3PMPoUV4MC01ZT2cD2KtMMl1MQZ/BGDzeajmaAD3qkrAzbPoVebLgGpsCYDJHqN5pTHgeMPDwTAhAln8EZpnP4KwsjU6NJ2dag5/BLP4IL6BMMHdji0savNg0GZrBAbnu1ICS/51zXCotuPWTDxeGauKabZjmC55DmOAa0dHrDhugwua4bRbasmeSlhJwHTVYzJbMPDNLmXAXBcOza7ZezNSANZ+NUjA/EUr1k2HvFjhe1xkCNk21n/XbAd6N6pG81gw4vcG1rPqsGMHiOKq9atyfZntVYFKbORrv5masGYGYZthgrZmnjPJAZ7FZhAwDxAVfJhuGfkRSnqTz5tabHWqF6NWdLNxsMPrWpOjuPH7fBOhSmADTMzNbOdz8DlvO4N5mr2j0R+SOvLfw/dYLV+rYoDsiACpKlP5MjYYb3+daxew1MpuiPgAqzRMz8MDUgKxLPrRvTH0Qjhs607m351CPUNBg+fQrw92sOcYZ9o4oj1bY92ainDc94ojzLYLbpNrDn3Hk0cxtV5vOKbTaa/JdBzBLhG0uNqH7Ns1NNl/De2tBAo+jH2No5NEN4NPUAFx3bVlz1AUF7UNah62JeBm2+BqlCMeWmtotNojLLnvFze4Yz9BvSFObbwa1t662EDesfxgpT/qSJUmPbMvLbuQ1rLzZgDgd6uIBqssCGxWLUhTgMHncN61bNNbZDoXHfM1RDPYoh9atpLIAKqnlLUQYYYNqvmagGpHQo8nxlTJTzJjfsG1FBWQU1jmM396jzIwAextdMMhxobeWopsNnOQq2HRJ089j5g151oqPbx032b5n9axshYAyANKKbLjzuDYZq6ZgZ9asI0BtnBxcd26cKqm0Qw8dxg81oGdgp3WbBo9RVr0xv3ax22WHOfhpc4qnmdyRzNqjb1OkzMz+RVMk28s0Otml1gggnpoTnhDaNxxzAFNCHmlMpUHlz9amYNfIoHIrkvO2AmPTvWX5MEB8abYrFGkUOh2rJnyHww9j4KznEsOHMZ06dQ2HnpB9/kXGpk+dTROlQZZstB5FzXalnDueVRKzIgny1NZYePNZ6sVtg6ydRYl8yf4uGCzVVeM5D3f8AOfWq8HsGljY6Nb18uBVGvWVO9agHRHPozW4k+ka+zNCIFoxg+fRVfZ9Ho1pWuFwVFgAl9+e9c5r1eocx96q07AJZ+QFcD2HaVVnVJiO3VYrJg/2GCxXH6zLSegHUorAQ5AHmZshsVlwcrdsRreaqtcucDkH2GYf66i8TrnpV5tHSqVvA/n61oPJ8Cmv3JXAptKM3mvxTNbr+TqdRHdCUByfnAF0CHaU63qMdRbo0OHh3ma5/G4qXHGqjptywktd4GC+fzBqpG/Dd02KPJ2d6y9bnzpMXCUwYYdAGC1tY4l3BUnwf9WxmQD86akz4lzgEuszozOHQGGCxixyozfZLyKbSnpZu5tq1qVpRTlHoVkHgDvAFCCH6tPQCXrfPgtuyGxT58v2bhqEb2sCkVJmUYZmq/MA60i2R02Q0g8QOtNZsMj4bYGo5mwZ+zQdfs1cQ1SOQOZvbcQZuQ/AbfMPoQ4P/AIhgi/vFQHWM+tIz/D60s/8A1fIg1mMutWJtyce+MsmO6xOqPqeI/wBkc1z23rwuOj1nXoD+tIm7PvG8M/OszGnvzHwYqs4zj95mq+fPOG+fqp/aHsTX1u5ztndUO9YdUdnVGnRvH6zAMAD8irLev9i3ubcl0rnJEoMAPyKzeCuTLcafnTjMj8655Ph6MgwNxR3CXPrDc8zcbDDM+hRM28VFzwLTTpgYNA4ouFpECY5Gf1wz2fInQk60rUwM3VHh1h+G06w254Rqws8KlJrOpBbZzDsNQq0WfPzd6DB0EFKmPg1Lbz3mGxKt+sjq8s6k2yDufuVCDYmCLbDhdGbk1R2cYeKx0KDc9wzrhr0gJbmxg8E1aVYOgzeaBvMHAwwQVjkXpUioxAw1zzwUYW20l71rTYnKuZhF6wVl9s4lSmeo3IOjRz/CBYKiT34eD7bnWpsNnmZR4OdaYC9ZQYxy6Uw+cmJngB54KPAN82mqbFbAPvOww7DVVWIz8Cfg+4B/OCuKbDl02RElyzBkOvf0KMMWjr1Vqtq2pItj1azJ5o8+YA8wUridWKdWLQtSXTpwSTYhgB4dhrJXPclRrFUdflPsm12GymmberB0h2sBFziN/PvNejot1cRbVk8FqParbnM1MA+8n3gGa5VDkhyrSgvGB/QkB4YKtrW096dTZQTmPBkMHmyfkNfolwiqV43zwjiVV++KbUnjAwmPHGzON8nz4L84p89+ZvPyYAtnSp9Vtim89aPEZ6mtVFn7zHZewA0sWXGP1BRKz9n7crHriODxnzwbAPf5FX23VWKPSAqTEv7wffgsqAU3VzqL73zvaykTORMTg0N85Jh515gSNGXXrvjxevmpgbF23jMFNo8C3KAw/nHAMM+8+hcss+lPhctCfluPM5zAzMDWo4uxmPtkERiVMk9DxnI6M/kU3Az8yNyzptt9ihTKxWZMUH5zj0xpvsM1ZwHmJNwxTnHhEfPA1X3hR/sxXDgtnJOOYAbJ6yjAsLevCdPwozmbNPDojg9vz+taiqwH69RmuRY1poZhmfWYeRZmw7Ji3CMipBUeTeimAfWhB71lzcQD2MLMbK27bt+g0uF/KHOkw8z/AGeOe8A+dWd8s0e258Kq0aVnR6jDzjeNn/v1rKW3bFSjOtVWlUZ6pH2A7vBWdw0T7Q0sJwUoKbLc/onRgucVlKuejQBm06o0bnGqozoZn0AtdW+DjHq2n69+U2BHmgAAzyBgef1rnNSCpWxIp51WjeFmZgy90PLXVW2Lm4tW1UOIdNuelMx6I9sozx4PM/QumIHN3rVfgVR6LVT+7sGYHIaPMFmWfGnnBgtvPH8gLob1KqVt1SlevJwG1UQzMwTVveqoDrrdsSwCqzZOAOu7AMF0wMzbbNOmVZqm1XZHf6zDrBegOF2nZkO67jsOqz4ci3gB/e9mEwD7HmVy2HwuuCTctPcNyMzHfe0HpEcMwZz7zXVbSn8OeDPFy4LVuq4Ocp9RgAEapu7wA8D/ANdWJde9IThXflUp53Bw5CNUJRgxMOOeBmfnAwWy4LVux6bcFTsq/wC/KrQThSf5qj1nxoRh58D715XoNtsV6/HYLDHPtBMPDszDNb7ivclDufl6c/b5hUITOgcsOsw7E7h2q/8AijxN4V3bNct+42YccMOQ5FkOWqTHn+deprM4r2dflpR/tBOjUqRKZwksu7N/n+deHLqsCnQ+DducX6O5PDQNinz6NI3gB+cF0XgJc7F20g7LviLDCWD3O0o3T9sBnvAzDvwWPb5Hbrqk3patGp862JcaY1S6kGZuhrAAH0GH8CsLA4x2zXry+z92037PXXCZNl5l4zwP/U/OuKXDY1zxuLFTp3CSc9AjsRgm8obxnGk4b9FbW3uNnDa5+I1El1inAcivRvVFeadjZgzKA8w3/nXTH9wek5NEYmaQTogPAy8D4fX51cABoKbSvU8cIPN6wB0H8is2QA+tZIcS4923XGbZqtSpXEKZRwqIaPIxIzODz/QGZnvw6F5ZoPCWlej8xRLq4mzoDwVTNmZHkRs+W8miv0QqtBpVVhm3OgxpOAHo6zOeB+deUuOXAqlWNQ4l91FuTUo8IJRzAkSdZl48NgYH0LPC4t48r0aDfLty1+1bcjQ6JFe8HWM8zD8/Qr7hFxRl2Bw0qtHtmqnAkPzM5MhlnWew8mfYqwK3R/sVIfiQQZhVuS+DzR7wjfQg4M3tblhnL9f0OpVWRVDA4wQWQMzw+tRhbtdhvTqVCt+sVinUr1hd8B9llmdmATMD6Dw6FyTjfb1Zsy75ES6pRs8lg96seezPA+vDZvXS+NnpA2XcNsg/L4c1iBXacGFEkS3gAGT8/wA/YjMOLfpt8NLXluUqiMhQZhszKhsZzPvWbHhcxolHg0dprixX6HAepQPZ+pnpJshJwPYa13EXjAxxg5ehv2X6qkP6Hq0JFSzZjB+5wDYri+aVdVVdOlWBTgr1v06MDDLMSNzIHKDswPo35rhNBZl2BxIolx8RqbMpXJPHrRHYx5s/k7F7hsduGT61herqrbmDsp7QZqLpmYbD7FY2299jJp3NRqrGqXq6NgbUhnWBXTzNpcQo9bua2KBMORSM5rJ1Z7Y8HyB2Auez7klVJiWw2AMnKDDBoNi8Ei56w/clel3A/SoEN2dhmET2OwOxQabMYB2QxL92GYYKPDgTo1mu1XQA+SPDA+9avgVQW+LVynR6k2bMeiQzqGcdnMzMOgDDvU4GMCmy6lm+ErWalexaaeADM/nUWGFSoM850Rv77TjNe6J/oncPI0imcSuLE6iQJtws+NTI5myGf7k+xeObhgRbPu2qv02C8FMizH4oaskzM/nTAj1upW/VZke44rZw5Eo8Jkc+gDTTNeo4Q5Db7jNNdMMAl4ZgH1qFJO3J82PFpwPMhKwwzDoNU9w29Vbbm+qqlLZOX5A7FtgXVKqUGlXAdOqtR5mjvhg8eGGas63JoEyHTIlv25zMKE9vkOn3n/jWco8Cx63dUelV+pSYEIwwB5k+h/5zVa9/NVzcrTalzMeFPzA88weDYtutk0teoNVgW+Fxv249GpNReNmGYd5goT3KVUafLiespkcAMDiR+w10I+N9Yk1GiS6i5D9cWvVeah07RA4swD68/wAgfxq7v/i7wyr1s1C46HRwtu/X6wxNOCyybIYfQGxREKc0psli1brj3AxBeeBhne1I+dHUnuZdkOMGfLzvHADPPD5E1JrFcuejBOrMvWkdmYKhrwNw5gsMbGjDMAUXC4aOqhR9AKbQ4IBIbDDWd3maOZf9wTIst9hyHA5XADiMhhsDvTtBqtOk2lUJdRfDnac8AAZs+4w61XnTZUylhWHIjOke83g8nnUYGrsatzrhdlOUNx5mWwAbHjwCYHkVPxLZqsCbHYuCqyXgnRglBHkMgBxj6OxR7Vh1KHcbXIywjSIvjsmfQYfWrig3DbM/i1a9Z4vzjetRuZ/OTR556GHZ/ArgM2rwuuC/7LO46MxrHCqR0/VkPbAAAzw/QB/oUeBTaUbQPxXAD6zXoDjHDoHAGKFncHbj9ZWZxJpXroDnH+zdmYH84H0LzaEZhmrtMUaVJqULPBl5oN5mfkS2SwrFScjMaDejv8h5ra1hlit8Lbcp0GdG5unGcoPGzzA+tZy8zYmOx6UxQJkaXS82JmqAGZnsWogRrVoPAR2ohUocmu1SqsA8Ed7M4zGfvg7Oj+NY3wtYUTP21ZjgwwD0kA6MwA8FYRrqqUbNxujyYwYYTHnthsgs4zcj9huxKrQJ0aZImh95ZPeEYz7w8is6rcNNuSkHQH3NaoSuiR0Ab/nUdPhTV2reEGiUusUrT9dxzNj2p7wPvD51nzu1iMUuC5TpMamVSZztNz9yGzYf60Eyz50OZH5GuaMiLAbeewjZnn5EdsWBUr8tqaFuVKmvTYL2gFGdkgzJe79majpHVbJk2rVYbPPVl6perqwAM8of3pljr2L0BYF2vz7/APWtqwY3KW9J8blwwOfFe2ex+T/IuCejZZ9Gs+LEvG6px0eq0isBC5dmNm9GfPzs9/yYda9vUHhvzN4NXVTZYM0r1ODGq0yAG8+b2e8PoW0cKe5rbbmPzGHTfzA888D7FYaO9S2aaxD1tD37xvGkbK7IRFgBkDUd6Hh4ikYOAgeMzHerELRb94qet0duSOHQB+RXp6iHR1vaLbY5+cB+G0YYGaiHGqX9EYNdI9SfIgepuHYtu5jcOSzJM5k9OU2fX2K9psM9LeG9aOSz4umgCMAJ3IwOADcOPs94ndbNOhTc1OZprYBpqdtFDovm/qNnsRvMvg1qLQBAYAfZqI9DzEw7E8Wpz+sVWosugxFYzzPf8iOjxpRvg5L2KbVYDYTQc6MFTz7hbgStBtzD511+EtQexS2XgeXP3rtfN3TYczNTaVc7jxb3N6dVs9rO86O/MhOvsdYLKM02XTYbrD/QYYZrbHVc/D1ERxmJI+IGauLRbirNNnHN0D3/ADrRRqbbjzQMP/todYLfBb1NA822EAW9SgdNw2M810/Sx6WPOiMPewBGzYb729uVJBdNo9EpwRQ2K7gQIIe4WP1NulkbVtU4A6h5n9a0ZwIhibbjAGCuzZbD2YKpmSW42a49tsOZV62+Tm/dG8AUTmWIzQBqeKr6t1th6o6ZuKhkxmHndRdnAyU5vOaumbmAJpmZg/puKwk01zDUVU9Gwd2Ls3DltdMzAAh1DU4JMQwNZ+Mzngw4prMYwLQMDWdz4XFrOlMsM7GHO9XrLyoqbGfDfpq1ZjPm/qLhuHTFrBk96s4b2B4KpBkwU6MG4HFzW2aBkGza8RNPRm8uhAD2AqIc/wAXDPescB02W01ot/1oOZzUjw9JRhaEYAmjBMzJhsv6baMJOYK8IGiBNH06iiHJTC1sBp0DVOEn51LCT86i4FwzpmSde08FWMydieOTmKjAAzTWfwQGaaz+CvAkM9alYfFQmTbzU1k2zwTCBgynQZ3o0Os3/Ure+EgGW80Ek9FOhJA1Hk796tVoRyTzTPWmpOYOqQyG3xFuyINPLTVhGjfu9nnULBvJWoG2Ef2izsHyzgKWyeDHiKsOpHiYaijszM/eKAUx5vVT0aTgCj6Lbx6jilRowZIpFeqsvLTAzDBSmaq5h4nWnjgMYezTPINgvAYPZkj3oAZwUgDDBBEzfA0gPJJ54M0OQIkElnMPDVY9Gc942rbPekbOYLSLGUmBgCz8wP8Ag1TW4eprjx9CqpNE8U/DXTw2wwwkwM81TyQwW9qVvYNG+sVPZ8VfS4eZzXCveDMVCVlyz59AZqK8zgXQuliQIjTWHxQK2RIUSSPLCiRgG9IwPNQ9INidDekzGN4wBtWwUoI3tDzTa0dkHMFIZkuRi1G0j0wU2Gyw81m4Cxu1Qs6DVX3pAG4BgHzq4eqTZksvMeOMP3XYCsKU8w8QG4ua426YtcGb57229iCSD4NZuKQc9gPDQc428scKU/Mv5qwgTHNjakBAgmZuaaMIzAF4aXAu4D2YArBk1TwDbDw1cQwM3fZrjt0eEoAzFV8wN5q4wwa8RQngzJZQ8tnJ8xyH7NV/rV81IryzRyXAX0eNja7ereiPWql57n3czUID/HNGzMYBWxWDMZhSuQbeFV8N7WNWzLKi2ooFvQTDUfcM/wA6ek0qDGDwAwBSGXmw94gkmDykZSSe421XvMuGexas6awZZ6aXq1vLw2Fr3JZ+BSn3pAADa3FNoL8BgDfBWdvQ2IDQPg34oKwnyXJPWua72vCvA22VHmPOPbG069sFRAeAHdPTzWTccbUxwNRDonj6+p+RXAACeAAVbFJ6qb/rVZeFq+u7ZlsfgBmtjot/1ouT1mHYrnQ+GBqNjlno3n/MNQox+1inmf8AGunGuI8NJL9q8TqnQ33MMzwXdjjLHe1KmYy4814aihTTw8TrWg0W/wCtMvYAK22i4UR0pw+sFCmMtxs8FdPPOe7VLJB85B59CuLRhUw406ZPDxMGlfPUQHtjjexfIcZwHQ02MFeH0K7swggy2yGgDeAApcONmQJk+tSIz2Hs1zXbXC2Om4BqaijvPYNYJ31l4WGCr3jcMljtRp6YfvFEOSjkh4WbiqXnnMlCFhrAaaN7BV/MOJo5LitabzmBKQzJz94qTPNSWdfP/wBaIX7LyorzOXyEiqhK/ZQz0sNhqaD2Cztz8SLStUNO438A/wAZqFvPle433xviOUOBDDznGzVFWJIBSWZTlHZmG/14BvWt4nXtZ1yU3XpVDzAO/PBVlmV6dGig+/Q4wRw6D865xx+fMCY/yPqbkAUKNGp0l8GAqX5zDYul3a9XKxWXZzcSGAH0AYLOTLeqtYDByjYYLO4UrDepUn+bn6zJmNeTsVfJh00DwitgDSaN5tmbymgDIedQpLz+rptueEmGyazpgWcTPV7MDVhTbku6mygODOksmHyZ4Kkwlsx9dtgww7zBSIFS+8fe3Dz+Rc1jcPXPc89jmrgqXOHhszNZVnTAnXA6zNOz5MWSQHBiYB9ajshm6YZh+dc6DUk8z3mo5xsy1NcA+tKeeDvtA/Io/M4KsLTT8ZrQYfAFHDTjGbeesfyKIEnNI5L+OxX0wgZvHJPBzoVfJZbA07o5gb+of5Eo0ZjPOVrGHkzW3VAisw3JJ+HglJZ5YtNTTjMZeBmAfWg5Nj3maoQsjRH7JPSWWGR8Mz+hRDMzBFhNM54I00e8VqhK4V8Db/ud2qwYNsQzj56Jy5Z4YHv6Fo7w9Da6rbtl6dQ3GZLrHWGtmeC7Bdte/ko4quy6/wAz6iAzOM8zswNcUvz0pbj+0tVctJiTyU38gB/AutkhcFrnti4aJN4eXixDjO4YRnj2b/8AIuKXtbf2YrJwXKjzjvf8iqp5vvTZEs2NHWPPDyKPh8VAQAx37EGGtKaYzPSM8FOojMU65HbqTgBH781cBTYL15AGYM09h7r86uFAuGxmLe5TOcb3NfJhgq+TSjprRvxaqyBfWtrdpsXzIiUC3Ij0yQHkBZeTAlW27LpVYoZ5mGG8OhdDJWhTazPhnVeVOS031mClUS3osxh1+4Isxn8HAFuKCzTj4dy2KGw89IfDB5kA6FuOD/BO55LAVWuXNJpsLDMOg9Ze9LztefZOhGfNhg82gUg2Ynqs53PGDrfQyHetVxRk0qpVl2lW4EaTyT3tYLOAPLCyWT5UwPYYdYKL4VxzNhM+yrNv0z1POM5p+2AzzwNUlVOdGlAxqGGYdeCRm3DitfcdHXDMHuxV71SlzNkt/MFjbZavaczxz61IqVYOZgw+3hgGGxVsaTg7oOdCkPAH4iyMIj0YNnKOH+dWAYeqAinLeDzhnsTQBiiOG+YHzcGSyYdhgqMKcw0Q0E10KWYbdRREC1t+opbLLD29wN6gqzDTZjhp9akG8Z+z08z7AXQGeak0bQlxIzMvR7AwWPZjQWWgnS3PFY34IvXE64Zh6nWfQC8sa6z6k/SmqVVdTPlXu/sXWL/jU65OHwXPTmA5iK8GtgHYa4Ib06NhEiBnhvP5F0uNDl1KzWaxSqyD1KPYbJhgea58Cbw0jWzcMAzYp386xQzeiPBsM0F20euXU0THqozjh7E2esDWKpt+P23c0KcDAMtMHg9h3ga0vGa4a5RJkQ7cnHGplXZ12Xg98azGNA/Ujrrbm+X51orGClSZFQcqsrAGDzww3vKtuq1alGfpgOS2eYfjZycPOq9ll+NquMN5mx3qh2qzOIoVishaVuGEaIAGZ8wG/wDIp1VquFeOhzn85Adi4oFSOmidcpr5xpod4Gru5Kx6nrNPqrc7nJHt3j1utclwN7eFVfhu09ycwD0Rg88D7DXIqbcL8ObW4NSiPG1WwwDDsNbrjZXucftrQl4U+bG5oww79iw8OsQTakA5BN5188wdDrZXTw2LuiVWzp9GrH2nYek1B9k2Q1QzwwWfo8B/1aAHszMASCmxXnXYMUPvZhmypcaTUgjgxBcjMusH4wSOvPyLpixHn3JeMM3aBFlVKMYHg8yyGGsCr6bApx1SIF3VGZDj5+MZ7zZDzqUdz+uK4FSnMYSMMDNkOtQqrPfqted0PGd7AV7HU69atwcHIcTiFZVcolVoVUwYZPDMwDz71kbtoNfoPKzqyxJBqrhzUbV8ihHxCrNetILAqtvw3jYe14zx9bJq7uG+XK3aVFtW4GzN2iZgy91ngsh3Km35atb4KS+GTlVjcw/h6tzPoP5zVFWLVl0H7Ly51N5CRCjc08Dp/tJh/wDZrJVKq2/W5tr1G5rOqUCj07CFUnY/WZ9i6rMvazvWlHst+qxqrCp0kDCpyD6IX4B/66seveEVHpRwPtMETRkTZnOgGe8AMNi4xedsUq1eKdbs6hxQM7hnwpsZl4NjJ55mea38+t33aVDo7dlS6bDjm8YMwZwZg8HWAA92K+4dXhQOIs+RWalQwol604wizKZI6wP5FrFjqsl4Mwbbc6AwTsZ5zJU/Mnq+P1qRGkt7nHHMAAMzNBowec6FkuMFsWzedkS6NdVzepKeftnvOH0Kn/ldtxmqRGGJ3rKFKeCLzcTfgfkMOxbKZQaPJdOszqaEmQwGYGakfnFxvtuzo1ehU3hrfka5KVVzfCZT4kMIwM4B1gAef/IvOjNtxIcjXpVSmRgY3smD3eunXbGfZ4k12scQpVSZkTZj4RnY4AAAGfkBc8rdViw6vNgsUeNJiSgwZN7PYChbR8V7ktW4YFv0q3LnerEiEBnPCRsAzNSOFHGziNwTaqrFjy2Ycet4czqhrBmGwMPIo9nyaVPpcSlTrfgTKq2GEDSPDWD51Q1WNX7VlSKBX6cD1PN7Wza35/QfYs8IbC2JNz3bS5cqKxckCuwteVzcEzAJ+Z5nsD/fesVxXr06q3BHfcOvcuxGCK965PN7MPnXTfR+vN8L5iUZ+dVYFKfZfhU0HjwBkzDrz70PHKm0aNfx1Kvywqs3YwciIebM/wCfZ0PAtmTllKufn4UiJL+7YQzZA2TMMzVnbcm44dvA/SqczJkB7Z53rAFIt7hLUbztq7bxtWJMjU+1Gdd6PIDr84AaoeGlStiq3AFGu5iqyeawYhhTuvNRcNYt37hvSrLqr7VpcQ7uN6lVGiHUGZZnyfIP/If8HyK6sam2JwWq9C4qWjw1rD1uVemv0itx5zxvGDGz70D2zvWF4tcQoNpXBRzobcY5Frs8ryM5nMJIfQlSuLVwcToVNofE2qvQLffz0YNMZBnM+wDMOv8AQrwt1L0iretK0naPcFNqVbuG1LhAH41MOZzJwww3vsn2YLh9b4r0Ofb9Qsqdap1W13A8GomAMzQPzmay9b4iuWHW5dG4c3dUno+wGTM8Aw8hgskFw1mSEinTtHl6i8b5gAdZpgBcIQYFUjsesgqVPivdYbDw8hqKD0t6qHyrcnSPfm93sKyOtv6oU1zlgw79HM1SsyWIciQ5O8bONoh2YH502I8lljmjBh/WDz4dasqPbz8mqUyI2f8ApE9E2o73jYK2rdgRKVD5uDc8Oq4Bm8Ed7AwBVP2knM0EKGw3sYe1o0vPxgNbbQGqswYFwSKPBiHDBg8NE+sEzy2tN0Ot3vPvNSJMaXUqI7fD9VOZUzmcq8BgHkzA00Z0oItMnMS5nrPW++H0Bh8igbOHbd3T7ajy4NKwhMM/tGsB5qvC0rjvynTa5Tm2TkUhkOciGeD35AUGS83Jpxv55u4Zmtdw3qsGz69zbb8wzmwzCYYdgLmsYem0p+SIMPuaOfQZ9Gams0G4IB8i5LzdzAGeXPYZpl6HOmRZpxYutHYPMz8gLVwJL9w8IJE7XCNIoMwGGTPvBerK2KrVbY4jU+jXU4E+E+fJSY5hswNbvjNe1q1iZCtyjU1ln7PSeSCQYbzBYo6lBZ9SVK4HAOPXsAeew3xjDZmBoKIyEy5mos6czGaenhCnyHQzw/fmrYrDiXeb97XC1UW2zZiRYwRYcQwwBkA8gfxrOc4bMdpjDRMOvZ1mp18wH4FUmgw4D2gZxdn5FEuQKbJt6FXKVO+9+w5F4/GZ+dZ298Kem1Kqw5rs5upSc3zzM81aszH57p64ctHf9tonsPDozBR6IdKmU2bSjqsaHUDPNnmDwBR5htw84kF+NUuSMDN3sTDfws/VrdYmtU21Q5x0/ddCTwMW9NhP6+fNM67J9CteFD06q3e6dO9VQ5vJv6ISz0QM/kPzqFdrPLW9RKdgZzQePDAPnUXAtaVfP2YdOdRqdJCa+GHMNPf5DTUCzHKlTftVKczm1GZgZge/PzmlDsz1k1EiNzoz0jDXBkz2GYdYLR2xPnBNNu1YMMI4dbRs5myazHaOBXFq4KbRpES7bOoNYoVIkxafVZARs6sfZFf+dexeFEy5ocqoWrXKlougHO017AzCZF+cD6DDvXAuA/DGlVXh3cd61xuSf2hNiK9BZDDROKfX+teygo5zJ7VcYDPCGDAfQe81vCbhLnxn40M5YN5mHYmjksGw04wfWGasJNVYgQJcqX7KKybx/QC5DDvmqvXDT4tKoEmTT5rJzQZzAD/j8i0REOkGmj61j6rf8sLoo9sQYJsnVAPN4z2bOsA862AMuGrWDD4qQyzmSMIydAMCRCWDLYCo8xnBpO5/BNTDcNrBBlKkGBmahM7yV3MZAw3qDyeHQ2tYswMHnADYjZqTmWmaQRnzDoS5B83QwYRCyZ3jmCaOM+87sUiHDNkcOtGYSmT2NqFql6z26kWcttVVS4Y2x7R+IZn87y2DITjSOmyj9pvXuzDk9V4bgbRhRth+dZc7SrlNnhF0Mw7zzXoiNRP3H8aN63mz9wuv6WPS5Zb1qt5a8pwzw6AWgehts+4M1pvVTDJ+GCiT8GY56ajuXjDKPM4LOSaweqbbfYa0c83DE1h6kDkMzcPoXTEMbXcatmyHiGrKHcjDODjj+C5+clgD1Ncz2Zo6PW23iNhzfvW3zwbdggVhiZ0ObFCrfQbip6JMBnDUV1JwmNaYGuJo5fJgTq3csgNDR0MFIOmzocjQPA8FuzjaPiAsZWJkoJh6ja6uG3LcJzLOt7tO+raV7+CGarKa9O1/3Xer5lnWDNxXteEcKPSsQbBtWEamxMdPQA0QQ/I2pDIYLHsa4NHSomGpoAkEYPIpZmeCZA169wEwb/DRACSMOtQs0ZmqyezL9owrvls06EDZ7RZDKxpM4H9N9xWXrLRDBxTXoDHkVe9R9Zo9RaaSaeeiSTzbfUuHAza9os/PgPsxTcb61SRq9VYxgxqLaIZ7b6Szg0s8Z7zUuNVX3o4ZmqSY8/qn4iiOE2s2Xsy01LDUWXjSXMlpY0zWANRXfCRaRngmjk4ErBmM28AaiI6Cw8eoCxwvav8AWUf8RPBMYP3ioaxb1VhzTcieyc/gSBmWDXibz86vCNr45Lfu0gmB51mXpj7Jb1HCe+a26UdzcBVezUSemLLwHn9XxDVjUpnLGDeBnn5FnhcLeNJcy8RSzkt4e0WSCpP6+hpmrVl41lhe1xrN/wBSQGq3N8B7FG9cYFpuJhLQJo5LiqgqufvFKZNuT7M1eFkbx5I40lvV3mj5Nw0zybme9ShZhJbD3ilsyQPoNUnqrM1YMxsGgBFpvMvp7WM1BDU70Mk8GvaLJSQcxsEzzypzmZkjZeRG1mb2aQKIzqH7RSwNsEeprLO1SABQuf8Aw0gkn51OFHT61EkuYGjeePBU8yS5mtoRs7MBt5rTWKrdE8XXbb2GtLzXwUKZ4w6a6eG3NzQx4Q3GQ09NVk9netW8y4BrP1IPvBrv4eZzXCn0VHPUy2NmauAPANPTVnDqUQBw0AW12jDOPU2dGFpx9jADQMsmZaYNrdsvQZ8fQf6FVPQwZdwwwBO5eFOFKfxBzYaT1NfAMzBXsNnN0Fo/VsQ2tRwFj3GGZo4RYw6j7Gaaektm+epsBWr1NMPdqFJtjNon8zzcUd214Ukl5s/ZqRAec0kf2efD3ma1FHpsWHF03HP4Fd2xwoghzpjWuxEMwQSfWtNDP1VJ+vBbhmSABpqX61wjmxnsNY9zpw5oFbnaum4tRomzACUgOiUeTM5tx82fkBDcMzloAMRHFf8AkQUaZzJ7HET0l8D01nYE9yMatQn5nqGnSeGjpXNPSAzbW6poYADawtKrbBgGzBaKNWMMHM1wW7ONo3o2aiSYeiGaYZr2ans1JgwXE9ZWt0o3hNzTWJk018HdPBdgeBsxWdn0pg3dTTW/DbO4c5epr+OGCkQKI5lqOAtQ9Tf3CUOGYH7NdPd/ojCEzRMDz00jZWgw+KjnDYP3ax22wzhmYGpEY8z01bPUps/ZtpQ6O4HiONq9ssCZh5qwZgAAdCOMyfs9NWsaNs6FjdtkJllwAR6KteW+RLl21iKSTDzBReT0TWj5dtRHozZpsU+fwRM6jykMwHAPepDMPA02sbLKmgbbKi9CRnkvEOCcUYz9scV49fYDCO+YGu8RnuZYaf8AOGa5P6SEP+aKfVW+tg/9Rbbh7VWKraESUHuwwUtWiM1XyT3qWajmCvbJBMHPZp5mAx1uKRh8UyZrbYTxsM+zVfJmb0cl4FXvb0uEDOZmpsM9ipw2KzhyW8AXNa1rn8EgNADzaF55sAULMzGXDFU5xjzVg9M2KEckAR5tENk8kJs7ERzAy001rN/1Ik7GZBWAaDLSqgeAEYTNysKYZn4ar7k4b2/c9G1J0HOR581ZazZkpwPZtaaLeMb5ojFHny6bQH5h6HYC23DSz3JNDOpVW583mAz5TPYC9BQLStWNI5o6UDx+c1YTKVR5MI4jEEGQPyAsbgeNa9cjh151zkXntDwABkFST75rkDDNs2c+jMMF67CBZ1kwpZsUdl7AMz1TzM15JvylVm5K3IuaVhDalexZM+xY3CmKmSYkmQct/rPsQGcE2g0DMDTz0DRP2YGohs716JoPTpmDD8o5IB50JxuWf1DbUcDNlI3s8NQ1y3C0s6q4HhsYACZhz3AlA/qZn5zTRssGOopUYIgBpm2nWGqlpvZyn3PF7FWZ5irCSDer4YZqK8bDJ+Q14GQMFIwce9mjeqTZx9BsA/QlDeBl0MzMEDrMZwA3tmmp9NnMta77GAKz1taQ1FpRnJdc7FeyabdtNp2vUbYeNr+5WQ54Dz4J5mYb3REA1Nn1IHj0PU2j85qPzmzAAD9C6YEJ4zyPYmcPw09JzMdRFDZD2jivAimy4kEZxWUOMxJm6bEreH8Cs5MZsw02N5rNbe8Qrz4m1W2Y948XuDr0xp88OYN4wAP+x/QuT35xIsCZYJ0a0aAcOaftjBnDA/8AOuwcdeMFv8Qqc1r8VKP6qDriR+t7615KrZ07xQpVSOY1n7UwwXa51xYz1iM6s66tHnTw2PdBrL3V6qmT3ZdHY0Y/YGeajgzmgOM+8GEQMz7AQVvhh1pc4GGm25gr2BYd8TPHYoZ4Bv8AG71IrE9gyBuoxYxugGB4M9Ctq7Hwih3HZNinc9KpVt1J18898/B5YeZxdlz6zLlOUaMzImhhymewFzTmYMZ8MHHgjh7oD2LQXDW7cuFqPylDCM6wGAGBoya22+LVcsB+QxLtWmvRH3tfDPDBXd4cVOIVyUMJbcrkKEfumlw/WceE2zfM/rWjZuqsvW4FuOROZjh8meALp7njrfo/Vuw7P9Z/bysQ2ZFRw0dYNmG9YbidPsNmRVYNDfCpc09nGltdAB5FiqV6mn1T1bX53IR/O6Cl3DAsqMGnbFSOSAdZmC17ls/JmSpkcGH3MwAMAURne7pqWyyD2w3MFKo9NbOV4+Bh5FwrRwPDxDT0OsPxpHhgB/WCKYDfNGDbewDUV4Az1G20su10zM5aKUvUwVnQZ9SuHWYnPm8ZhszWamSc4Ghp9CKmz3Iebbee8POoCqrJw5TsTMD0/Iq0zTr3Wo/WgWseadAzTQI8/ggsp8/mWg+QMFOsCMD1eOc/7JgNQ1Q5+7VnArHq2EbEVvAz6zUYF3Mn/f5b/RrqvZqr8DMKVUjADPPR7FT8y+86Zvmkz95d04LZvH34K8DVPVViqxcH2wB3zgt9YcaXdXq+3Ky/DOFS2dgHvzBc8jWxUTm8rEb5n5wVhAqTltygYfYzMOzPBZDYcZq23Gr0KlUYDZ0I2gAGHWCzh83TWgYlN4O94KXcl7UC6pVPqVRlgy9F/o/WaUw4tSwqvW12LOxT1VnR3tuZtPgBpWlRPtaM2C3OCGbHRn3q7r1sPnHaqNKbzjmGeHYCVt23zkgJzDDxgx7aIy9gaRYp58yc8PK1mpBJ5LwWTDyKPRJj7M8NBzD5z6FderbD++tsMTwCV0AZ72VX02lQaVnKfnHJw6ANXsWdStWpRmjffcZPeexk8zUepUGmszHfWU6Z6wPeGHkW9sCm2/MahXPVakAO7/uPkWjrdKptemncdAnM8xTvB0j7/wA64+4cHgSWKVPPXWlsw4ky5TOjcmzIDA/vYZhgt9c/CudDhRPtHSoxyJoZxtE/vOAdi5pyEGg1yQxBckw5GieervW0czV0CzJ9gM3DNP1GHrjW/aJYeCyH0LQXDxLs6fS3aHSqHDDX6JBhvM/kXD6a9VQd8PB50+szBXttzJcyttNxKN64qAe56ATLnp0uNVW/slU7OrkSTnUXuaA8NmCpLDo7D09mWFKCpR6W995gn1vAru5NeTCiU5ylAdQYwfCPHP8AzqPbBwbAlfaq4IkmM0HREePMzP5DXN3L8PQtt3JSrwkO2q/eU+lRzk/zbBnM9Bh2B5FZ8co1Dk0iiXcxcEaNUKDPYhVKXHe0XjA+/eudUevMcV36VFg3HGjVAAA4ASA3mH17FpTsbiMF7hKrMSg1WD77RDAzXZw8w77ZNenQ9Jv7RndVv1EA5Op9coDAN4H/AALZsmw8WBvvaXeAHguL8AaDUbPhVqgVFsIcd+ec2BHdPeHnD/AuqsvLpHEuM1pXPGaOdQ74O26PFjYQIkcMzk7+/Dv3q74e1u7rGpdtXNWX629LuX7k9Eqc/WNnPyH8/kXTZMOnSZEdypBGPlTzAHujNYy9qbOvO9fVzc6TGpVEjBKZ5c9+fYiHLfTzptvm1RLjYA3qr+ywIkdnDMz6zPz4LxDP9aszXmKzFks6B4cxIjGyH8a9fViBSvW/rydXLkh12CecZmWz34fx9C0fFFm4OKnCWg3pSrjpulgAT6TOhgbMk8/OpXt5k4Mz7OjVKusXHVaUybDIBACohsM9/euoep7gkw63BOnQ6bCrYMRWWQkhJM9/WAfnNcUvCz27zv563LZtyjgFOAAmS6Mz4LIec10jhdMv/gCxIoz9lhVWjmBNCrR/vIGwHWAGoZNhTeF1Vk3fR/Rz9a0oKf6152NIkAfjH+Bn7n61uK96P1j2NxQ56zm4FBrcIAA6NIP1lTzA+/PYYfWCy9V49wa3Rgvvhrbky3rmpEkzZeeAHgksPbDA813Ct2q+dEhX/wAYrqmT6rUYfJBEggcN7Mwz0QMPbAjo6XnT0luPcSg29cHCGC3Aem1cGAelwZOGieef51F4VxuHnFGlx5VYtGlUG87aOK/AMD0ecMA2H9HQt1c9pVxmjR/sHYFEh0zf4tckhJk9nWZ9i4bWLkptE5S4Jz8aZcsWZgD0TMKe8x3sB8is6cKbiFMuC1b6qsq5m6bJkVR433qezJB443R5FgK3W+fCO+ww9GkRT8HA9gB5MForzqVuPUY251DwuObP5o50fvDyLPz6bKgaUUKU9GJg82Td7w7FCGwgU2w7znx65XJdKo4aJhJjyHsHjPz4Ll9eNyHXDprbYYU57wcAwzBdD+09YqUWsa9DgHzTzABmGcpkwPrUTi7xLo/EW/oTnI8tQoTIQs2Y2Bgff/Gqi1sYcmLrtTn2DedPoZ8iY5NyfNBg3AZAz9s8eAAn41NlVWpHTaHEOYYHgyEdnxjBRZkOczACXv0jPDf2GogO162zo8p371AeaAwwOO91qqeNtnDTNOvPcybQPt4SMPBPDADTQQ5cnM9gAB78z6FeBL5x+fm+bgbA6ADYnTkuSYUfPA3Yuxk8OgEtam0p2Ryrhz2vc5o4bzcmbHYcbBnmu/yJgXFtxoNbqjUWs1HkI+jodfWa2/Cu2H7hlVhxiVg7b0PPM2QNl5ZC1YzEmvHSvs561kSmTZjBmAYGtLUrwbo9IOlWPTvUkjRCLUpbOw5J94GH+/WloUrzNRjMO1H1UYNSnjMAPvDvBdV4bs8Gq3Yt0WdOYqsaq3LJ5qjxzjZhTX2Q69bo3nsXJ2bwvWNS/U8quSZNPwMAiPbwDPyLa8NKl9s6DU+Hr9NhvSAh/wAzumyBmD+eeGf5FktjL2h3PZM2PQ7utyHs8aMbTwPAYfIYKXdt50q5Ky1Sn3AprQBgcgD2I3qxSpNZClXAwbLR/cpkdrN578n6FraJZlj031hBtK2K9W3TAAOnVOBonoZ7zBWhhJ92xJMqPRqbmYRWQYB0ww1gBTqbUqV6hrdHnMMhUKibEqNUXQz0dHrA/rW9jcIoINBdXEZ+BbdPYh6NKiSz/nCYfZn/AL96zkCw6dMtCbUZzkw6qcnQjNB/RvkP6/8AIucZWlQ26qxLlym6Ubr55hmGeB9iLlj13Yj8XlpEoPYgGALV02iVx5qbQAiSYf3MzjGYb8/kWUo94VKBVPWM5sJJmzysmOfQvdrWEalP6pxMM8PbM9+CrZJ+A0xKlAGgGAZ+RP8A2huBmO63TWGXgf2Zn1gHkVmEyjw5AalHjT6ebPjNOh0H8hrRQ6bPg29UqE5UaNMZ5JkAPMOsD71vZMCmxmJoWjUucOqSdjsfPwfID2azt+H6ykRH26rJk0qEAQgCWzg8H1n3ooFSnBAaietQh5nnmAYBsXLY6dw6493Vwltw7f5sJMfPM2mnjA8/kXfeDnFSq8Ub0kBeNf8AUMdiMGjq1LvPoMM15Mn1WDPoztKrFNhxjN7XZkR2czP5DSCvAfLzqlQ5hnhgGHsXvIo8cw/RvjfxdqNgcOar6ni85VX4egzL2aOGfXgvP958Wq/ZNgW/bFDcwqvLMMTJbrO8GD8gfkXnmBfNKn3vTJVGYeeAGeSjU+XngzKwwA/412vhXavEK8JFdqtcowSaVFzpE8Hns5rJ94AH5F0+OYabh7PqNw8ZqJSr0qr0an07mjgPSHt5ytmz5Pk+hewmZjYda8H+jeyxeFtOv1WL99pFbYf5uQZhmB55r1HbFSnRo4OVGqnJ8ma6Yj8ud1sJ7CdzzWPgVJuZnpnvA1qIb3hArwtLQPGGKM3gxUE3lKw4NmoklnAs21IM8U0Z5KoCjB4SLnDDZpgjA8BQIggnvxjz2GraHPYkhqOaIGqR5R0G1Aw+RPZAsjDmHGdVxzmfvFItgeAEEl5vFVnM/OoUmeYeHmgmnJbNZ+qyWN4aie5zA9TNZmsVJgJTviLo4YRZqZJiYaeaop8amyYp82/gqmpV6JGmnqBmov2hps/7rqYLvjhc66jUejyQ2RANC9arDOb8GIyBpmBVWI2GDiuwqT8mKZsNgYAfem1s1JoNVgFzfNGYYdAdikUqvVkJgRDYPf34K4gV5/VNt+JhvVq9rvb2MAPzgou1k9Mbx0z2GoRwG5PiYKhrcaqw3ZFRB83gzzVnQa8w9A139n1pEITQojZ9CtodHAA03FjJN5uMvm2wf51LDiFE/HwV9NjcaMQGlnJ8lxl08FVfa3/h0tTXTJ15ipZ83hgYYbEjgs/Kd64z94lz/wCG4qf+asQbbPYCT0yCyGgwe5bdDza6ZqoZ6bisGakwsfDmMYaj7m/yJHWG/cJ8p3NsFSg/iJ31lE7HFh/XbAdhmj9cAfvFj0nc2Rz2MDBR9ZZf1w4HQae9cHh7ROk7l7JDWHTVFVaPEe7EPrh/8RNnUs/eJEI2hcnoh4bhoOWM/aKW8ew3FC5x/sNdMIsjZwNED2CWBmGeCEGXMtTsR4tY1YfD2as4defAtRZnDB3DUA1IzwUYe7as62ElrTcVVMZPfoKkOS+HepAVV/2ZubEwvYJMPmR8TrUKMycB09cFZ8y2fs06GD3tFtDNX8zv8NahlkDAG3FkpMbRkbEXrJ9loG23FjcPYa56NBMtQ0AMxA9msPJqVRN3U1zNIK9OD2j6dK9t7sBNPRoskcHGwWS+0j/nU1mvZ+8V9JtZ+rcOhOsxnGSzURmq/OnvWTfnUX4NrpmS57xSM/gqD1x86lM1Jg/eLHC9rsDBO5/BU/rJsPeIDqpgoxa06TJNRNYz601z7D3QmXpOCddpSOWz8RLDAlHZn/Ohenh7tMISucbBLnAP2j6z8me+Bqvekvma2jheba71qwHs3M0vXeCw/OSgd1NRSPWTh+0NbfMjbVHXs0ITGHvaLKHM/DcRhUsPeJ8ptevHvTQPb96ghPz60jPPeowbTpmmbSz8+M48eppq1B5sy0zUgwbwAABbcLNjDjOAWmaEGWA9n1q4qUNz24KIzDzdXTtlhHjSWwdBtxxW0aSwZYOABh86jTKUwA+zwQ8gYNZg5mpW0EblD8gKyjPNgsLGnmy74ji0EOqtmGa5b4bX4tongbMfDUWSHhHh1pMzETzzZ+0WMtrUD0x8HcNNSmTzAE1PeYB3YCZ5wPOuhgvmYzh+0RPAACoUOsMaXiPpqZUmzDTzU9NvdqmqycHdjirzeckjgaU/e6aGMHhLphzCZZDPerhmm8zHBtjYqc+taOlG3yYYKLXBM0GdTR15TgYfIjCY4HhtqbDjP6WD7hmCCTTW95trmt0JECZv9oruNJ3rJZuRnQVwy8YCC5rh7FtUzP8A+dxG8828Gm2s/Gn5ngbimsyQUXDp2lnDTXLNgnQmZ+GkZrKCINYfFO8sBigT2fwTaTPJuavs1YMw9iFl5SOZbBeLIIwB7tSAAABRwmAZdakAbZqAWs2Hevus3/UoMnP3aiA84B+0QWxmCZNRwe3o8/ggPD4pk9iPP4KOe9ABnsQgaLRQmqGZ4qUpisWbNYPsDMFj/R7qr8mgyKUfWwea6RVYfrKly4Op1smuFcIqkdvX5Lo+pgBmYKR6A60sPigz3mj61aAn0KC8e9SzPYoMl7YqEcwUWSHhKQcltR3jzFaoVJmlrOf1JGmlncLWcae4HvFKOY2Y+0VFngkcnaouHm1hJMPxFWPScD61HN5xRTeSISlHJ+dNczgXtFCN5Bn8FeFrA5mfvEgk4e8UIE6ALFCaExS40/fgqoAUuP1otdAZmaqq9flo2f8A+0dV5P8AJmrJk1z/AIwcMWLzpoS2DMJDGe8PyLmtbkXEji1SrhrnKW5Owj+dYSfR357uuFVnz5BrqFt+jYxWIpy335OqAGa21n8EGKaem+xJZPz62a9U4JG4V3PPhuznGDZaYDM+YNYI3tGQbfkXto+Fdx1j7idc5ann1s4f7a5vxp4D0qlQg9R1LOU+eHihguUeZT15kwNAM81Ik0c4bWo+utW3wNfZaB+dKz/uVnbw4XXbSnzlOZvQuzNbrc3M8U/DZB4M35Rs/QrOfb06G1qSooB9aapVBqtYkaFOAMw37zXmwdKZYZd1328wBSOWckzOeg2+yYfPsU0JNRAgob/LADne11qwemQaIIQZzGs15M8M0wM/Po86ZHOdhDZ+QDU23qlQ6bC052Bun2GC7tbd52P9ngiW43AjZ+5d2HmsZM4J1WfUvWtVlQIYfIAGowOeQ5MFmrjVYuwwPP5F1Cj8cjBoKa5ECSfnMwWKuq3oNvGbH3aTgsI9JAy2AAfQlwN9xR9c3bKCsBTgAADAMFz/AJY43WCd9ZS/xzP61HemPn7RZYEKfrn9HkUEAzL2alvG4o4PYLYXFH0OYBh88F0Cm23QI2lOnVWZy/eAAuWAe9XAHWXhw9eGDXkM1ktwqNXrfkvgD9HjQA87KVSqVDBjBhzPPvVZAts6xnpvgzh51o6JbDdKMM8Jhh2GGw10OdCs+lVK6nZEGmy2eXihmYGrLRg23IBvPCax0GughXqrDpZtwbE5Zr3zscFe8NL84Qw/ul42q9JqBnmZuw8zVRYygXPb91UtqnVG5o0Cb+MZ7AWSvDhRVbYhPVyq3HTZMc97JxzzAwXqi1WfRlufVCuW+errHhEOm8tgszxa4wcGY1n1DhzYfDlnSfDDWejHgCvbWHkKAFGOUB1UzBrzqQ96uy04MvWD6Fo40ym0TS1ILL3yGqyvVhie7rxKVGhmH4SsQpMCmswDfbneL2NH1qVZ8NyTUgCVUjgR++QHWC6NwKO0gpdYqVx6L0gOwwXN5+x11zzvGiLWt82S/R6pClP1/n6VK63sN6z50c3j/mpv7upoG5PLXlnmZmAKzrFvXBavKa74BzvRhvUbGUn0p+mvg2/3o9HRBWH3qfWQYfivTP3LPWa6BVbPqtKrNCYg2l97lM/sgB1/QrHJ/EMk6YaIrpHF22H7eqUdydbkyj+5PmI2AGudSd/QgieHpZuZ5/wKOBmnvHwwTOHxRsR70IBirWlU1iSLrj85lkw7DSZZbz1MAPA0QrHoz7OGoGCEAM/Ztq6M25koNTsUWSfi+H0KFoPLZqWFKfNrXMwAE7DNgM9dtW1NhuTIB4MAeHQz0KRX0SNhNalttg9oHngYZga2VSuHoOh2lDpp9BmG/MFSvcjR4DrfRL7AVOdYnH3ohooE+c8E0NTB0AzA0dsGxXqHUIlRc/nXPNnMN5rOetX8djnWggSX40jXA0DRs6Mo2HOsOtahkKk9Q4kGlN6zoHmYd+CpXmZUws2G8810iwJJ0elusGGdQNk8FnYpAuquWfcbT9Hf0Y74A88zhsz+haCtz7cvB2PUqHnDlngExlkMAMPOrCHPC57BneuYoPS4r2sefYslPo84KzCYtlh4+dZ1mWgXN4G44qM2dQfVjFuWzGhyJ0MH5Mho+v8AIuaHAckxfWMRv7uHfmtBcNYfepofaNh71rS/B5Q+vD5FTvHLCED7b70aOb2vy4Hv+hbh2A9nUjprjf3d9k9/nVbZ9YqVKuhqCxLM2n5PKvZ94f7mok+sSwmhKA8DDoVPnrTHXPOea86R6t4kXhSplrnTebwqtu4Pw5HeC4vf9eo14RYVx04JLNVAwCSHRmawXQXWataPPbelBSpzGtHPrDoTpFgzPiyYeuwwYSw906Gw1sKrTal6ro942XL9VR5TOjPOOHsT8i5/UqP6tmuzm5zz0TswNdI4SyWJNoXLQ6i5nT342uyz5DUYFFbdVpVEn1BxipSZ59YSHj3ma3fr6jXnbPKVWjz58s/YyM8AAPkNcngUp+M004+31ns+hbim1JuBb9YpWg8YTQAGcOzr/wBdY3AC2NCZPN+lOPMyKcGtGaM8NbDszXcLV4/evqlR2IFGOlUKtydF45GBnGf84GudUTh7OvPh5ZlHpvLQ6g+c0wdwzN4FEk2xWYfA51xiCb3qipG/JMOgAzT9g94WrclDvmL6jnPs+sKdsPfvz7DBNXzxFcsNrlaVZ1buSq9jUFnZ+czXhT1kxUrZpVwVXmY1P5nlTkZmBh+hdgszivelHuuiUCVfEmsUKthoMyJB56OfZn3rv4bGlu2vcd+IWlQLutyNGpQPa3q6CYG8B/OfYuVVu236IE2pP3pMptYfk6FNpkSsGb2fkPyL0rdtsXPYFNCsWxUXnpz7wA8csNYMzPDPBeUuM1Ku68+IddYi0Cmz/s8ASpM63oGB59gGf5Fey4UlV42XrbEyXbl3VyBcjtOwAAeDWON8gPdij1viXxGmcOafasS34dNtI3u/eZn5zXP5lpVUJ7XrimzIciohmzrRjzeP6Oxba4bMvWlR2qjdr5/eo2ENkNgfQgkWBdtq02/4lSrM6BSqfCZw5eIZmE8/IZrtsCBB4i3HTLfl0qHAtKiQzq8+PTDPRN/PDDPvXHeFFKseTRjs65rcA3XzOUch7ZgCn0HjBcdnxZtuUOmxpMIDOKybu8DYXNfMl36pVXgmeZ1VwJ7sr7qHLgYBGDswDsVDfnGa8bGuuiXBWZcmvO2psZp3Qy8DwGGtn84f4FzeHXoN4U6b64tKmw3orIaMiC8eee/BRarJrNyVm351x2/zjVIAAeBk/wBpAPkWP0Ya3YOMHpD3VxddChv2wzbEKaeewN57O81zefMiRvV8Gcx6yoVL8h4b/k862HGavWPfNzRItlUOZbbv9JOW9gB/kWCtv7DwydC47fqVYrGsfg54Rl0xfYbRKrebE+4XbjocEIGfsQDoBEFznW6M1Sq5vdA8wkd6vqrdWcqQ+xbEamx5R58uAbFkZ8xiTUjfYggyAeTvXo1xnR7t5j1xzkPQZ8HlzzzP5/nWJqVgXBSnwbYY5zMM/C7Fcw7wrkbCLECMHja5norUHc9v1uaAVmIFvV2Kz9zlx98V761hsV/DT7F23xDoNZo86qvNMZxawZh7EzDrUq/Le4ZW9S6hLc4jPXDXTk+NBZDD6zBYUKrc4T+agv8AJumZn4Kqf2OZLOsN6zsrfn85rogbu+YESiWlRKVTWIwRzwmhrHm9n8ixlbpVVgR4T9SoZ03XZ2AfWYJT6rLmQ4UF+WZx6cGEYD7FFmVis1UwCq1WTMAOjWPPBaCDhmKlxpOicd8G/C2YPdhrccGeF1R4l3a1BYgyZMKlyWDqrLWw9DPz/OvRvH70bLRo9h0S9LHtif6wOpBT59OBkz5ljzmHY9sQebeFcl8L3kS2+WObFhm/GCRsA/Pv7FIn3PTbkr0uq3BBjM4Z+DHDAHsEVbZiWSFQqsG3JkP12GhD5sMzw98H+BbuNwutysQqU++YWxCOMAA9UAM5Uw+s8A61Fivqtq2zb1qWvXDA2ZtU8c2TZ+6mGGfWlwouG1aPdsSDRqcyc2br60hkOjZsw+TNZS+alZXPwrVocow9UADPPZmYPH5MD6FO4e1v1DVz1K4EB2cyYHOZZ1s/kUYYgZhv8N7javSpU1mSbEl84YPGYaL+ezP9akfb+/8AiLUgrNYnSXnYpmASIgaPLB8+C0vE6gvz7XpVDiNw2YgZv+s5Z+M8exYWiM1i2HfVWuGk/wC2w716JVSCuXDN56dcc/VhAZgdRPM/oBWdnTLu9aS6rEf9ZRwZzmHIewz+c1LmHZ1NjyOVqUyG7UcGNV08wBVlsXJTrY9YMOa0mPNjaAYB1mvn3a24smM/VatTIv2jjMnzOZnLPAA+g1yfiLb1VtjiXXaNUZUaSZyeazjnmGB7/wDOpszQqVO0323qVUGP7IeAH9ahM0pgAwfnG8fe69vM1cB0PuYR22HGWTA8HgdPfma2FwssW9VI9DbD2/8Aph4Oh4z8nkwWaOHEB2JKbl02G6DwYZn4xra1KqhdtciW5b4BPq1YeBg3jDANdFlclSl29MCI4wzPmwg9t0BgfRgs/PrHrKA0x6jjZh/ArO54FNZo0euQawBuwnjhT6YZ5va4eRZqm1Izn88wBshgYaJ9BopIOSxDfOU5Lwdf3s8u9sZDyLe16TotW/E1+W8ED0Q6APzrGBJiaWhUWz6DAwwwPf3gt3ckCvz7NhXP6/ZmQoUP7sy9hrYLG4Gos+t3xVeINCtW3KdbEmPNmAZux4AZhge810XipxRu7gVxYqFKs5jlvWkn1hJ2a0aTFeAMDwPoPYa5LQalKCVaVDqVzxqU7y3Nes2Q3xj+f5FpeMz1VuSpOzrjrlvT5tR9jIpm8MADDZ5Fccw0dK4osU2nep36BDpUeUzohOjgYRj8nYu0UGt1WlRaPRqrOjG6+HXHPMMPOvPk+q1XijQbf4cy4LNNkMmAPVvPBkwAPPgGC1tKuq4LMvJ026xSjdoOFPN7k+ZZeDD32C+lw+ym4ew6Vpxi1G28Fq4czwlh7en3O9SQfudilAGAPMyKfngYH9atYdVYefBhhxbIag3s00lDZOS1qAiOMYeIslmjPems/gnXg3qPh8VWwes3/Uj5htVxmo+sgszktoAez94oRmkBmAIhZ5AnecwVVzLgIDnsGWn3q8C4CeBnp6iTzzeKojmAyXtE6FSYe8NxxPEWFMeM2jWCr3rXXdfYgmZB51uHnmPxEy8cU/eZrs4f7bG3Eq9Du4/v0uCYR88E1Soz5ugZt5mfYC7U8zBeDBwAMPIaajQKVAw0KazsXZHsscMVSrVnHhLPZ8i1EaG/Dj6bgKbJn4ezAAVVWDqs9jCnSwjLG/1tDvMweY0Dc8VWEaexhp6ir3qVEjR4+g3vbDeaiPeCGxRgW1VZ5+GbDZ9a5/Ph1mA7yje/DvAFcQ62+FZaiSn/ALufWtBP5R4fANXxjl8xk4zWv71U+i4e9aK4Yz+boNt7AVKASvIYfWC+pxuK1hDqWDWm44nTqT/u1WBAlH7tINQFuy/J56ZLy9uaDmX/AMQ800Z7vEQZt4+GjPdpbNVlge9xWDM/MlSAYZp0JLYEnSbauNoPDpmeGaM4zbJbHM1Qs1JvAG9MFZxp+ftHFzYbQkHqMjmjZntmABp4GpGzFRwBsPEcBetU0Db0vaIMTQ6jH9SIHgBYYEuN0+IjMGA92Cjg9n7NFns8RxRgPZAgyBNawZ6biIEWHRb92gxNO6wAhemBmiDJghw+KdB5h4UeIKwyGYApIPOAvmbfvEv7tQs9mD2x9vYoj1NMN7HQjyNSGQc94grwjOZYONoHqUGHhtq40W/60Bgr7k4Zd6GbKa3gtG8q84wZ+zW20IPMmHengn/vN6A421V8kNE16hZnUnPOnWakYe8VP4mOog8T3arC9tX64c/ES9cfiOLM8w4lzDiYN+WpCsNh0IjqWsCz7JuGnj1AUdMG/K1Cf86A55g7mq0DPNIzMzUYg2tTmMSU0YYqs1nAJTWZ4G1g4rwBeAMNRv8AQoJmpz3VqNqC9qL1AdZEBppmM+97NvNSGYxoDB48lKZecwwUfkJHkUhkMD3ouD2Dh+I2rCNqGPidag4GHs07GeMHVhhScbOxRzjMMlqae9WfhmKiSQ2LFaknyT9mhzcCLg2pDzLZn4iWitotDOPA514bEbJuASujgbfDUWTD5YdTBdO2SbSnnGQ8QzNWEySBtbDVIzqKQouG0WjyZO01TnnkraSy5lsBQZLLgJEMbNBMPZpp3mXzDe4o+GCe7Fug7m2ZqWdNMwDQPBQYwYEDitmZIao6iz2GZNNOMGZqRSp/LO+0VhVWQkxdj4ZrNPRjZ9+BqVtgzW2DLTUvn2D94sZTTwkez/WrU3sA1FhcNotfHodaDW/DWfjTH8vabFagYYKOl4nAamxpO9U7JuZdamgsbhptbBJbRc5gq3xMPaJkzNY4bbXHP/OjCf8AOqLWMEXNfBMG13zmHvEYTP3irANwxRpg2uOZ+dSGZjgLP6xgpATzUYRtdHMzQ5/BVPMOIweM1HS0WYPB+IiB5tV4GngNRgTdZBn8FHz+CWfwTAl5/BR3kgPJI0wI4PYOrzvdTL9q8WgltnnrHn0edehcPvAf8S4v6RUPk5VPrjDni54B8nQi3ZgeB5hqUHQYJcz86yNh1j1raURzPoBXGst2KQ9PbyUc5OYqK91pAYGarAdUI5ObunqKUZ4R9nWqd48CXrI891plSAQnsUNUc1HMzUg0yYI8tCMzTRgrDR/ERct5FGD96q0XP60gZcVxyzYIeXbUbaK4AUkOhOmCZNZh5Jk9iZz+CNBNB5HrZhgbeYeRRE8AKBYRpLbIYMAAB8ikBJcVeAKQBpgBWKlXIwa9KDM+9cyufidBZwgzpbMyoH0GHQH0LZXhSaxW4vK0bDM+vM8Fy+pcNKjDmBOqsWlQ2vOby5xh7kqt4suuv024AjNdeALLw7nuOsPgxWKickPnUvidW6VGqPq2jv62HtjDoXP/AFlUvcIqLbi5M5LWg4ENkPOBqigVinUEw09/nWZOTO9o/wBfeo5hrFm4oXtvqbXqdMmhpsRmfyKpvAOfqUduDvz61RUrZNBdL/k6lPReeiXHTfozVraDh1JfokLUOh0RnD+kd69JWe9bAUv1xoBJkAGe9eLKlZ844TsuXXADQ8h9as7Svam2NC2S58z52uhBX8fpk6t8Qag5BYNlo8N596wTNHqQNdC1dyV5+7aic5hs9U+gFZUGTycc2J0EDd8h9i5xz9nU94jNautnTns84oMu+RpQaDaVfuR10KVTjNkO89iDMPdajGbfu1d3DatYt6fylSbADwz2GqrlsOtDwADU2NPce8OWewOxDGBgzwccwUpmMGa1XTlrNHYDfEcwd7DWlgRuTahc9sN88M1jJNSBmOZsTsJQdmC0frvn6RFnORTB0Aw3965RvZl51i1cH6a+ycfsAw61zWfflcerxvnEgMg+efgs70zUq85Pj6DhqlwbCQBpCLh0iscVHKrD0zY5aoYAHNsvYfwYLn9VuqIEM4Jtmbp9cgD60EnAGNdU7zLb3iLaLBhWH3oYRHIP/an1pQGYpu6lS2NKv1mGTU2Yei0GC6hs7SCnMwqxyPeGZrHz5OtNNR6VW36VNCc3vw7Ow0FyXJ64mnL9VRmfkwzWolveDD12+sDAwW4v+q02t25RK4FSjc73xwPYq+TaT9zyqJasShnAqr/vTPYu1XPw94XUGxo9uVyga1YYZzZNkMDz+tQhxrggzdUniXT6jbDcCTNimB8vIPDMOg16z9JCm+qn7fuaC3hcusHIM9ma82WxbFv0qOE5uDUmaqHRIA94Lotn1Jy4bjaqvEKdVZ9PpYAcMzPoMOhY9wuOPD3r60qPSr1lyQroGB+rMMDNeUrto9Zo9R0G6dywGeAZnmvTfpISabW36PdzEXn6fgcXrw7AXmeYcT+1mf1q4sFeFk3BZ8OPOqLjLwP+QOhU8aBOktc2EGTy/nwXaKVbdv3hwHl3HVZ0w5rGZgZvGe8DXN6DXq5RKIcGI2yYGee/sV7Wz/LOMhqOK4tiH61lAB4aXeqfWqLzp/cXs/IAZq4jVh9mjHEYAAd84KQq2zSo0o/Vrf1mHQq3R1vZgtlbdKpUm3mm5UTN0+/NSJ9mPwI+vEiZotgno3hYNhvVnGnsUqAGnsm+dR5OZu6nerBmlfaqKbjDjIOxdmkHWaDMvSX5J5vnmaMP2fU+dGzR6y8TrbEE9h954J+BR3wiuvzn2Wd/Rn1oIAbyUqNrm6DbfR51cMhEZgS2JcQAMN4GqqNAlTDwY2fOg0cPmoZtRDYPf3gGYK6MKrDqLVwRGM2mPbBnvwVxZ9S9ZUZ2jSgzdhBmG/rWrjUSjXIDVKYlAEiV+0tHs2fIuLuGUpsxuTWXfU8WT6qqkbAzNnoPzqzsalQYEp2deM6TD9XZ8m9HPA8FIegS7baOh82YNB0GC5PecaqhWz0H5MkDDPPPBRoehWb2ty4dKlUq1WZlPPrnVNneuK3U8/6+dYcYjMtdnLnmC0FHvB+j8Po9DqU4DzzzAFSQ+UqTRy32zwz2K0M+cPWkBsA8N+9Vkll86uZsRAAP4FralTYJ0v1rRswdc2GB/IqCjhKeDx+8N62hqZeZcjGbc5gwz6PnT0CS/wA1HCD+0Z7A861FNZzi69VfA2gzw1gzNVNvVJiNdHrw2w8D3IBgBr0aivRqBUo8inSmAhyNHXZMP401ZlbYs+jSKbOppvHKDAHgPoVFUq2/VZ7spzZmeeHkWot5mC9SQrFVb5l1h7wTZ7zRJme9TqJTvUFRiA9WDe2OmBhoqzqVNrMOz5D4OBMahPMczIjhmAZrQUqpMVLiD9sa5QI2wwZksyD1gZDzrZ8MbwtKzOJNYptNrMZm33+uC8zmDx/ICzhDS8GXqVwo4I/bisvzJMirvGxDZ/s2Z9YeRXfD2t06Zw54m2XTWAqWcMJUbW6Hs+v9C55MvasXzaFxuVm443q+FP1o0F4ADAM9mBrUejNxyse2Lfase7mI0PB4zZqBhmzv7DP/ADq8Mi4zWfTbV9FqiU1iDoumbBnv6D6zXHLApVZrFoU+C3OjQMJh6JvHgbJrtHpG3nbl+TYUS3LmpVSp9Oz+6czsz+vvXIpmnRKlHt+uMMnHpwc193DvUXzNYd7uT0jbxu2HC4cy4Mamu5hFqU5qZmcwA8nkUj0e6VbFYuO4Dob5g1FNjWN2T+04H3/Ia8ss29XK9ImzqbFk8jFMz0eh4/zqwCxr1sOuW/Va4b1Kp9ReD7xEk78PyLGOYeq+OpwaVcFVua46HMh0982IrLzR7zAAwz+hefOJfFSyrhmwtCdVWXYuzPDMMPOCPj9Xn/XcSlMVGpSY5hh9+ezZXNKrUmJNGapUtvoPMDDrBaWNtbdYftKbLpspwKlCfDP1iZ72c1X1K3rqrdBkXHatiTGaFCPA6m8GAPLAnJqtVnwYLB4ZmEXM+jf513OHGvF4v5Mr74jTJlHisschEa9iZrzqHN6VTZcNrUqWyI/7YPOrX7Q1WZy9ViVUw9XPYAABvw8hn3qOzaVzVi66rTos6NPCiAetIzwjACx8mpV/F2JBnMnEM8AAFjY6xbdV4beo7ouDiNTTk1CbJ+5mAZ4LnTNsOMzWq45UoYA/g+zmfWfkVIy9VTlHQ6lO5bXMAkxzPA/kWg4ezAs+qcjWIpzKUDx5unv0U2Qpa9JrJ1c/WMEGfO0CrJ7LATwNjZHlBmALoDJ2yzfnrKUByafNM+cZl7DZDzgsvWIFO9dmxFnBMaB48HQ61tHMsoECIFB9avh0PYGarKqcHlQPAJLrHXmGxbClUFiHAdfrlVjMxPfR+szWdqtElasio6kZ6FKDCNv9sHYliPR+VNpqU+rIHoMlp2j+pvveeYBy2eaiyaI/TQpUUGPvZs74+avaPXmKDKN9/RntPxjDRw35/Igy9YolKPx2G8APraDozVZGo8SHI/nLs62lozps6qsAwDBw3X+jmNmYLHvMmYhqe1710xY7Bwc4i3jY1WqD9pVyHSqfKMDn82GYGAfx/oXQOK/G+uSalR53rmqycD1mZDJmzT3g7wwXJ7Dsz7eQrgbffegSKXQfWENqIAZzDDsP5FTyZNO+zLUWuPvBIpwYRjA8N5q9i4vy4bfvCRbkX14ENrmX3pLsg8+WAzDrXYKPw9+yTVHvxydJuSuv1UPUlWiTDejPAfRmyfZ1rz/ZN20azLjauCsUr1q0EY2OU8/8C6xD9I2ms3Rb7bFDgM2vbU/muXgvGByc8+z5DTYwnHJmsUS+qxatcpsMJDEk6hzbLOGsB79iyR4epKfUm/a5ngthxdr1lXzPp9So9VmTKg/PPnDlwDjHg8ef8Cw9SCjevvVVvuZtPmAM/WpRg6zUn59UafqtRkvHnn4xma20+mt1JpqdSjDmJR4AZmsPWKDOoM1rnsAkB0YHmiqVVcqsCmRX2wD1d0G1sNc7bDZz6PTvUIPy+WOQBgDwG9gYGpYW3RqbaDtx1l96TICTypxI/RmuczJ78mPIbfcN518wPPv2LpEy4bYvOmtUqm+soFYq+EqYDx/dQfAFz3CMJ1Sr0usUZmdSqPTeXAAYN02fvIYdhmirdyU2+fvdZpoQAAAADZDA2T8+fkWFn0qXDzlxXzDfhhnsNWsCtsAAa7Z+ABngAZgf1qMmFq9QaHQWj+3FqnW2qiedNkRJJgYAHyB9aHhdUn7Gvek3BTZeHJPBKeCQGsBh5MFK4e16cdzNTnK/D8CG/CgPVPYyGYfQrilcGalW6Xcvqqq02ZLpEkMKjgejn1mGfYrhTL166vXF5SKrLiwJNPfefMAZZ0QDPyAHYs7m+zNd1GOveBh0GC6F/I/dtmQvtrcfqeNHhYfd4MwHjeYPDM8A+tZ+fTWJ9cOnRXDgbw0QqJ6Ow10CPR3nDuaObB55gAAcgNgJqSc5l12C2+8bTDx7Gjw/gUKZz1NfkUN9wwCK9gYB0ZqbGN+HPaYpp9D2YSPnU3AtaDmzTee05JtHmwBmB9feC6dwleod23bCtWXSrhmSjA3o0SjYcycrzhmtlwlrEW7bNrfDy8bKh3JCB5+ussvPHGCBKwwPxgXsv0eODNj0SjW5c1Ss6AzddIgAwcsAMJWBgfQH51zYHkq8I3HO1Z9E4eRbKOezcNElRYdPzAze355n++DPoXL6Pcj/AA0uFr1kwYVWKBsT6dU4Z4GB9hga/WWq0GlVKRCfnRdY6dJCVGN7rZMFxTjxwHoF28S7E4ovwQP1dMYp9VZBn2zBngB/Pgunxw4HPeCFbg1WbEs6Jc1VZCogZ+ozhmGiH7k3g6F32Bwig02Pg2/MM/xtmauLboLlKj0xuqyjmS6JmwzIdwMzY7MzWgmTzzXZw3eMWnCthwwhx9AHDP5zTphkiA8k8AZL0VMmNgCrTBaiTGA2lQzGWwJBWyQUQGVanGzVfm3q4KkDw+KaME7n8EjPMkEU+hUlS12d7a0GHxUR4GwLM10whjGZ9ceI9MzMAUgK3gJsOMGDvfmtXJjRIcc3GG8FzK5Kry1WkBqYLv4f77JavXDreA5myfzqikyZQezfeVS8b7I88xL1vOCaeqT7zWa7+mHL3NRRK3KB8AffMw+daA6wZ7294Ll51hw2tPTw+dHAr06AXhnm15DT44R9P+7os+e3JhagOZn5FXg82y6D7j5hh2LNBXjP2exO1V458Vo2NmAb1j8y+6GmeuR95o2G9mfeqQ68/q4PubFjAqUsO9Ln5HnXT80MfphuGeUmFmalsgbOGo/msTArDjJ7zV2zVdYPaJ8y45uxoHjzDU70DMmnMj48Fn61Ws1LZpuKaybbwmHYaywpCrclh4TCCeH0LLmeDumtadHz9gq961X9XXA1rFue4Zx48yQ4fFWE+iSo3iaef0KIyG7xFslHNNKyOM37tRTjOKxH1jBSI08wLxEybOxNYfFXhntoArz/AJ06Fe/HcWaBOgowbav1lmGbBpc+eGnms4DxgexxSAme7cUdLba7Znv+dTWZObWpn0LP8ywY6eoj1jBRhe2gCS28Yb0frJtl3QzWf5l8OhxM8yeeaYNtM9JbNM6wGq2NMN49Nw8FZQKVLmOgDZ7HO9Y2vZrNwFIB5R3mdGQbGeeB4JrI16hYaznu3EgqT8byfnUcJLakMstyUbbSAuFwz0+UBOhWHDdwwBRDpu/w1LjQAAtRxc7xLCSprLLckUHJsYAalw9Blc7fasejYF7NR+WDNaN5mIftEmY0Qz6E2YZr1b8UvUmftG1qggRD7FJ5NgA2Anj2TDEvUHwsNNUUmmyozunpror0NzPoUR6AwfWC6Y9lFw51y73kTrMZ8ywwWz9SRPInWaJEA8wb3p9LmwztNpr5zdNWT1KM5AA2xnvVhGh8tKM/nWgh6ZhnpqPpbYc9qTIQ5RxPIoOf4a6XUqPBqQeOxvWXk2kEY9MDNXHNsuGaNR83ANWEyMbJ4OKJo5rp8WxSGZjZhhqJ0NM+tVR6jPiN9adjSX8/HV4FmeuAeAexNZGiCYxjgB5owMMFkJDMlw1NBlzDUVbh+Gpsae2A6b6ztabsNRzjOKdGmU41YM8i9865mqsjPGBeIpbzOsCKTA8jaaZB8FDVWPQJTOeoCa6FqNEHvaNpp6jsH7ME7nmGcz+CRhzI4KzmUdwA1ABRwZcV7Z/hWctomj/dqa8yaiYGDvQttsseRshn4ajvRt5t6auGQzFHo707l4ZR6Hh7tRzZwWrmQAMPDbVS9GbAfZrbuhFwrQ1Pw0BvGBqXJN/AAw+RVh55+ItGKwCY4Y4GnQeAN7gAf1qtA07zXwXmAZvYdCLnHDDxMFHM8xTRpgTQewNWDMxUWfwToScDS4NtAEzf1qzjT8/DWXCTtTrMzAlGG22wCTtS2KijVJS2Zn/NqLjvhwvawMEykEkME1rfhq8L2sGXtqe1m/6lTBJwPek9Pz6FGHi41g/ER6wAqcHnFIB7MN5qMCw5lnzp3mQD2ZqieewNSGTzFBa843n7RHzXwVUZogNY4FwExHzOarDeSZe3JhquAeT2eYqsA1LZNRhWxPdCwXGyA/Psh3TbzwMDW9e3ioVw01upW9NiP97KnD1y/ghUtakO0dwPFYDeuhmGK4jwfrHqq6nYhuYa54Lu0kFsKeSe9RwMwNSJOGaie+WrnWAHmKhSWcD8NSGetSDADBSIIGo7x71LNk/w1Ek7OtVh7dor0nBV/rJzPxEEw95qEZ7108PDGWO7XTLzZ+IrCMeaomTVrA7VjzQuLTjBM4fFPH0KPn8FyOoyYJrD4p0wQ4Oe7WGAzh8UYJ3E0YMuKMAM/gngNBouf1owZTCDoPIzeQAyj0W/61CwA9vXJ+OtsVW7WmWKU3mYHn/gXW9Fv+tBy27U01FwPJ7Po68UZO9ujhpec3ljLntWq2lVDpVSbwdXuOSFVx+4hmuaV7g4xdtb9eXOweHyGowqHlcKVLe8Q8AHzmar3o2B+GvTF5+j9bEaKc6nS+jrDWXnqvRoNEnu07ow7EwtTyYz7OHiB+RWFvc3zuDB7/J51UnJbeLw08y84HsHDA/kVrei62dAZt6PBqTAMyH2QwAE7A4OU6sUYHH2MFyyj2NWThhWINcCS6HY70KzPidcZwORcnGy6HkNBqJnCJiHvp0Tf50LNn2zGaP1lOBmR3rl0niXdsZ8wYnf41n7hvm56k0ZzpwGB9gMrG4ZNxGsnC6Gj5tmY0+eAYLux23QKVS+bpU7RdAM/BPNeceG4VG5DkBLlvYBhsDZ512iiWkw9C8SpSdirCotw+qs1ipXLNfqRm8Zn1mqWvRmI0jTYf1l0i7aPWT1T9VSQAOt3DYCwL1Efnn4Zhn86m4Wog0w8QOtOhMln7MP0AnZlHqVK8ScxgPYahBJAOhxRgcVkmZlqKQzUpcaPym/SVenQewLoA1mtcMhEei803nsUST+7BC9PcZgHoK9tKGxcNLlsON4TWehewMu9Jf0tBvoVY884yPhq7mBycp2I/sdA1n5PtTVwg6EbmYuvmAJcy4bWh5EMk4OxuI49n35oGV2eBNgaDL+pOiHJa/BBS4f2ZmXG19xMIh+6M8MDVO88YHpprPNBqp9bnQ6pHCLOeB1g8Izue8AW2gUS6qk6FVuC46ln581x/Wf1WjYDWMDzwz613CvVV+Nb0JxhzDXZXBytWgtiNUgqjT9RlgcH59iXEK3oL2rO5ueAGYBg09gH6Fzy3rnrAVJp/mvyd67GzZl93DC+/W+8cc+89hrh3hNw4jflq1K0pEJj1rMeiTQM95/7+dYc2ZRnqNsZx/OvQdw8OolVkMxftBMkyIoYA1I8bAFySpWxdTLpMeqqkDQdAcmvocPsowhWxJrPqup0aLWOTjn4xh51Ch02VJdBhtzMz6ER+qtIGDbe5gOszBaDh7R6lXrraptKc3+Q+hb7Faz6yo831dno6+x4POo9VOjsyAYpW/DrXRQoj9gVaoP3GxGkx3wwwM8wVJXq9QwE5dq05mHh2AvRn5mp6ojvwWPFYPPrXXrYuGlXJPjwXzzB+Hv2dBrm8CNUaxQZFZzDNjPMMFI4ew5dVmesaVOAJtO8bRPvWQr6rSvVVyy2G2MGW3tgKa9TYoSgqMF8wdDyKXWwp1yV6Qw/O5bDvzVfR3pdNrMimxKjz8IPnXsWtQ3Dc86qzPV1K+5yO8Ge9RYEmVTYsulVGCEnXPMDPsUqsQGIFxuzoP5DVfJNx5/UVi+BkJkLQc71BA+QLkTDYp0N7R0tnR1qwu2GxJmtSqUxm1KDPNeiw4VvUZ6fW36q+DIMM+CBmrrh1UqjW2p1f1wB2nSTBkADsXLKPDfeqwRD/pQGGHnNW1NuSs8NAqdDcpv7b+N2LPm4R2CZc8Gq0h6ovsAzIing8AdC4/JvCqsum34J/OYKXVbknUE2uRYB6PUWd4OhsM1nIbOtFDU61jw8IkAfPzAnTqxDwD3LIYfwLa2r7WQ424zy4bAzMAWdo9NYmUaQ45UozJgeAb81BrEOVDhfe28AM8AV3A31KgMHWRg1F+MDWBmHjdag3PaWfMaDb0ZaC4eHVAsmw7cuqm3VG+1D+GdMDeBgfQq8zf9ZSJUuWbxv4Z5rm5YFPbFNia/K1Fzwg6wUK7Y1uU2pG3QG/u61tNk02ZKOI52d6x9YjMVLmM+sFcWBh0dw+Sl4YBKDMFuKPMCgxeR0Hnjfe8FkOjM1jJL0uTAhROeNlpgMDaBWEN596fClvnnyRgf6FttTqtesP7H07kbq5yHVZUYHt/vgV36NPCuvw+JtKv+46HD+z8oHwZkSDz8c+jBbC9qxRvSNtmsVylRHqVT7Qh68aQ9sN6Vh0H8mxcsmQLqtuyLcufXkxqfcpgD0TW2ZrLCW74l8JbVsY6xc92vgzIqlS/mqOzmYMxfoBcngX5atBs25bVCna1VqOwM2TZzDzr1bwfjOUGyK7Wa5avrW5QngBx9HWeeDDYG9edb/sy46ldFx33fdDZojvLa8OPnsDyLW0XDnNsXVOsNh2K/QKbUgqOD5gfYt3aV8037WzrmqVKn+r32fBkNBmGazVkwLcrdUrE6sW+EyPyGDIGeB5/Ia9JcEIdwX/NmwbScpUmn0iMDD1OqB4Bv7MwURwuerYK0rhg3mxLnVWsQIHuGYj2wzUKmwLRu3m7VuOs6NQhRjOm1GRJ0QA/oWU4u8Orn4J3vIlVWKDLUqSZw6e88EkGWD+cPrW7rz1v0qVRzi8MoFbOayE3Nk8AwWNuhyG5Kk/QadEtl+8abXmj6zaPW5Y8/OsbMkywf03OxbDiLUol+SpFcg24zQeSMAOCCykOM/XpEiXLlaMeKGbzuGeC2gSIBsPRWnGDzkGeHyLZ2TGfn3RyNwMZnFZN9n5DDA1z+m1imszKexUm/uQPa2fnwW4O5IlVujOlOcsBvaABo9Yd6gWFVrd8Xhw89Tt1g+XivYciyAAezsz71kaPbdcpU2E25EzhB45h5PkWrr1Y5bih6uiORoHghmf4x+dZSt3DKjVSQwxKz8Y1EDUXhJpwNR6rUaaEnlTzZMGdhh5D8iztVnxXnQfo7mbT4ZpUq/wCq0F06jBfB4wDxmnQ2GHkVlUjo1z87c1nToDM2UGEmjHse1/OCjAw/MzplSMJc4zB495h2KW8FSjPx51KwB2KewwBBDptRrdXabtynPTywA5IB7kz86U/7hg3Of1jA8MOj+NbYG2rFSgz44P8AI4YAYdfnWcoLNpQ4uvcbdVmaB4cvEewwBU5zKjMdwi7I8UwM1YHPOT90NtkA7AAEWtZMl+pVH1jZzE944vY69rHGBDaQGcKXcb8v+cKdJz5eQGAPLKU24ZdvVSQ/FqRw2nwwMwW1ptYocl3QcDWB/wDaQ6NY/P8AWmB1CqxqdxRi2/WJ0v1bR5QPxY0fDDRm4YfnXL7t4Y1GiVmRblzQY1KrHLHKpsg95yQTVHeqVpXpSmHH5kmExUglBBl9B/JmtXxyn1G/KpEvWXOeP2+cQ8MIbGewANdcDGcMQg23cP8AyquCq0QJ0bCHLib9Ez68/k6FL4nWe4y1QYlGfjVJqa8YMy457JJgax8kMCiS2DOTHzzktGe/BWVYuSlTKzCcptNk+oqdJ1maZIk/5wXt8wUOghGadfrEF5mQ9mzGz2Bs61mpIcs+bDnWHWu9s8b7HtKzaqxYdpVWBcFU/pDuDzP5M+xcXqUmo3bWQnXBUocN1/BgMwAAAFGwzArFR9bhUX388GdANi1dEolGh0TXnMBzsozf1T38sHnwVfdtsXBAfConToz1Pww5unnnGVhfmpTalSvUdSM6ecAH2WTw8HNeiirE9isFzZv5u9h+dUh7FLePebezV+RQc83dNUo8ybZ55tgezvVxTTchu80GeqYYZgHQqkAwdNvyGtNRKa/Pa02Nh8sew1FwyXDz3rWluy6rBkz44RsAOOeGB+c/OpFvUSdVbNPC35NSdlScAeZ68/n+RZ+AdzaEuDQAemG+zr8u0G/QUSHXn4YtMMTpMB0DzB5k8AzDzrmwtralaV+W3IC2J1snGkPnoQww1gePyAa6XYFe4cs2bQqbXH5Map0g3wrdJDWBmpBn7488D6FyqpVu443qe7abc0l6uws+ZelvZ4PgeGxemOAnE7hzMo0uh3VZUCBU7hPnZgSI3MszOzONnvDPyLbA4/xvvCx7zdokGzrRgUGbTgNiYcHDxg6w2B/jUqz4dgcRbSuC6r7lSftHRKU+bLseHmGz2JvK440w7qh1mt2/Y/C8KJCoL3rRmcbODxwjDDv6wzzP5F59+/Mn4E6ZGafZweZDMAe+teoWUPnq3g3gb05/YAAHWa6BxLs+j0qzbclsMctVYtNYhVWmBse50HzzN4PowWKoNwv0SPIBthk2nzYPV7wNk8wXeOJDNNoNWrd3XOxg7ddtsVCN5wflBvD+NZ21SLJsCo3/AGNeHFC37megUK3ji+sqfyxg8YGx1guocOuKnHQ7h+wdO5mBfE1kwZq1wzzzeihvAADozw3rm/Arid6ntS6+HL7+FCvmjmb0iRvegTWQ8EPoPDBdgue22JnBbh/xUu2pTJk2bXmD9YxHjA4dLNg/J0aJgseEdltjjrdVt3lR+GXF66ocyoVt4IrNRiUQ2fHP3Of428F6GkxmJMfQlt5hmB/nA8wX5s3JWOIQMW/fn2Ek1in2hWGJoVsHjeOSYPZgbzIdGa9PWZ6TNDrdBO8alR/VtQlBrnTDPWCSfnZ/BXUO+yOtQpOoueWrx+o9w3bHsu57cqVsVCqM81Rzl72akHyH5/kXRT01UBpl5zJWDMlvFQsN6DoWiVg9MbAPEcVFPezM9PoRPPOZIA60Asn7txUUmBUefNzTwWlBnNOhGzPeqgY0zlRj0zUhmTn1q9rFNzAj0FkplNnRmjfb6AWzFcAeSI4rbwaemqWlT8DDXWiZebeENNXQrZMP7k6C4vdVHfZlOvtvg99C7we9cfuSGEPnTbbwDWPNd/of5XNzMYybmCOBJbjAbb7YH9aj5uAaAwX23zkiS9FePwGwBGzDiGYc3UQZ+QOs1WJGtWcJz0PRP24GHnBXBzIr0AIgH4uCywHingec7FODYJLLjL5g52KPn8E69mZ5uJpUkYGnQNz8RNZ/BLP4L3DOFqzVXw71YM1XzuLOa2B708Dyxw322Uat/PgpYVhz8c1hwkuB7NTY0xw/eKMOjbcM1KI9scb601MoMGf4nQfnBZpmS/8AiKwZrbjIYOOLHBsEyiPxvnUKTAPHUbbWljT25I7zRvQGD9g4BgmzDCGojwLXVKiOYG422qF6G4A+ItotzXCnA08ieZTWHxXQk6B70jPMkKSB0M8lLB5QQNPAalW0rWRm8ADvUdXtn0SDcNZaps48AwPPYotcHqbZN61KAFVg0r7o/vB3PsVxR/WVEEGKqwYOguy0SgxY1IiRG29jDOAKnueyZcz73EY6F8Hm9x9KOFySYy+8+b7bfWovfga6EzDbpTWFVgmyHnNUVSoME3TfihhmtuH3NovhUQRoh+8Tui/7hAEZyHKOK++B7+sFpqkEUIDRxAADw7F2XzMceVOzzalHJcD3aON68BoH26O8819CUmfmWo/Bejf9jgC4+xthIZmZh0GH1qayeZIWbeqIQjnPsYNYZgokaSGGemYKNi7ZZzUoIwAqoJPyGC0B0GsA0DnKngYdaybmgBOg8AdaiTKbUoAg445mBmq96S4CgaPCJj7RVkmN+GCrefkedSo1Vb71YhHnl7NOsvN4aZ9anPHTnvYOeL5EDMNszzVs8IRycCTrNYiB1oJlNf0j0wVOcZ8PaNqteDbTBVWPxEjeiPKhZ37MFLAFL0plNiyQWamU19ktPTWwAEDzME/b71rFvMOfnDcP3aiSYb4e7XQ5jNOBrwGAzVObIH7RdkW5rhj2QPLTV3yD7Ig24+yf0KWdKb9oAL4EZtnsXqMEENxCcBwzU0DTpmCwu1q8IZh7RW1N8EkyG9TmQWPM18NBDweDxE7yDBqnZMwU1me+C53R4TggNgiAMUcaYBlg4rDk2DXOKo4GYKOFKYP3e9aIIzYAjw+K92pmvUJh2Ksk0fB09MFu8M1HehsH2L3Yw4Q076tcP3a0oUpjNTmYDAe7TaWPOlHj7NVj1Ec1dRdFNmJ+GoT0Nj8NV3PMOeTKDsz01kp7OEowXYJkNjHT01lZ9BYMz1AXZw8zmvhYQAcPoSMFazKbyZ6ar3gXZFua4NAkYYoDPFO9a3Ec0KkGBoMTRIAM1IA0YU3bqG+AKQEOJ2P5oDjZ5ArANQFBB5hkvDUg5gGsLanjkuKQy9tVZzLPnRhMb7DBZiaZoM/go/M5qWAbF7dh3t1EORoM92CkAzmKzINBqGpWDgAop7C8NSAkuYLJYt6lxtQ00AZhqKwpptgemoUb0X/w02eoBrS+BgqySywZaija1ezJ371YMzG8FXvM4DqAo4ScD3ph7tpo0kFYMmBnh51kQmK7gT29huLHC9vNVzxn7S4ltGw3gzrZ4L0Oz9/gR52nsfDNcd9JCMDNbplcYbwzXQOGle9d2LH372AwWj1LmB1qsM1ZzDzNQXgXRDCzUaTu8RXEZ4DBU5sqRDewTCNrU1Wz2c81KOSAJozzaRqy9SAwzVUeotRMZA1SSYwAWxtdPDbktHjHgSu4b2eGBqnZZ371awwYD2aXC4WAGZj4iXQo71SYZBQnqrrLk+Ztta5t+dHrRGfE1AVCDwfiJA8B9ZrX5kdzQBMY/EUtnTNZoDAPZp0Kq+yov1jupozZbTPgAe8wBUp1581H1n5hKPmO62gOZBD34IAmRfOqQIDmfs1aw6O5hqHmCfNCv1rOM8pAB+7TUOG5ng4rtmA3gC5uaG8K8Ixn0NpSabm14gLVRqUBhsBWAUFv3jea47tu8+3hw9odV1X37jnsyD/o4dAKkjcGbDprAVW4JUaT88jea7xedjUaZHN+K3hIDoXNGaP6nmadzS2Qjh51Gx5q4kW9wyk83OtVw2dA8DaANgLmlBo79VkHoMZ4da79xRtv7bVl2h8Pacyy0G953szPz/oXHJnD26rMqWhOfhgf991op1CqnbEayDYgvswHQDe1nvNcHe13n3X23NhmrusBWeXDngza+RVgPbcMFKTQAGWo4tVAtuJMjhytSh6p9hrK4czIBjzrYW3w6zkZ/aMA+hQLqg/8jGnXOaZekH7kFZ03jGxR825bAZn2Z9H8CyVYjRaDV9lS5zAN+9aCt3bYFSox6dqxmZZ9GlvwUqQbn4tP3JHOCD4Mxz7ABZqBUrfhuhLfi60gO/NYrfkrilPVJl3Ui0Y55+QAXu1re4a8xcjWhFimDvkNZCZSp0DDm2MM+haWHGr9VrLX8z8gffn0LewLVodVdCJWKrDZkdgda9Hi9kwNrUTWtmmmZLHK6moCjnUjMMGwAF5C1ke+PppUeqy6PWWp0Vz2HZ51CZecNreaA/xP+NSNXfIU6fpXBTm/bh431rGGeZIwmS2WjYA9h9iBVAWwwzQ9iQBvRPRn8NQG81tFiOals0qWcXmwcDS+tVh6new8H1gpDMl/DDM8FYvrPs+o3gchilVKMzLY6GXut76Fe3PW6qDtPtyqwXocinbDz71mrPqvqG6KfXNTDkjzWovC5/5Rb1aqIaMZoMMFHmBveHUmm2Mw7dVRcjTHTDBlrsTP8qnEOq1nmnKqfyNM7AAEP2zo0N31bEtWmnh39ClzLwtKjx+bpTB+tfkDoXyuSFHbVuess1I/vxmfM9/Yrit8QuJtt1R1+JLjSYgdYSGcwBYygvctSTrEpwGZD55hmaavCtvz7cN9uWo8QlSTDtWpVR2pXHVZJumeZsx2dgK2ti7aVSqoFOtW2ZJx3/2l6OH3l4Fjwolc5L1k+xnH8+asqVPbZpbWhEP1qHsZDXWC74Q7tb1mWdflx89dsV6j0+EzvCc9o5rm/EKNZ0O9DYodSZn0T3wRAz2KkemXVUoTrFVt+pGR/wBIkPGtXw6ogW9S5d1VWjsyXWDAIbJnszWwmzLnsClZxLftGpQ3ZQAYO8mYAar7PP1bUZFSncmyUrrMD2AH0K44nXteL0WlVKsy4D3R9xaZ6FhKbSolbI332MzPoBRfMKq87eCBWTrlNlBJhH3gs+y8+zNN9vZmuxs0GDJtybb4N4BFADBcce5o54RQ3uqNrSJ70oHdCX0dil0GM2BO1Vz2UUMzUSqvBVZQOTtjrAYYAthb1Ki/YOqg4HXmsxknjcB91xg8GjPMAWwoMmC9agU7Uwd7Dw6FQ29A58SiS28MAzz86UONKo9Xd0KbJkx9mbOanYN6mzqDPj1JxjM4r2uB9hqkrbNZvCqA/LbZN1/3IBvNbCfUp0yG6ESnG80G82g6wBZ+BArJyo9ct+VgYexOQe8PyLTYp58mdG/mqsOPAcXoZMOhNRnmwDByKbzR9gLoFKsmVdtxmd+SphzX43NMgyzgB/nXP9bRCXFBswaYPAAeDf8AnXXFiw5+JAihgAHI8h9ij1KY/WwCJhgDfZmqzmW3lIprxg7qA4lCW9DOGcfXAzdw2ZmtLTZ7mkbj/kVEZ5mBub8A2KIZys/HM8PkWPNA0zwToFUabg7yfDNSodNptVamua7wVCL2B3qLSj5kGn/wFI0W4zvN+98648CVw3ZYn3qEGW3mzomq+Sb8OszYjB7GDwzWrsmBTXqzqU2q8tLAM/Y7FlJ/g1mbp+NrycGT7DNXFqbv7Z0Y7BChhSgZm0s8wnA9vPPsMAUi6r8fCfQqNxCnGFCiwAlQIkQ8wz/yLKQLAY9cx4NVrLMaRNwzDP8AZvrW74i2H6qjx265UodSj0uHhzARsDAP1rdzrXhdnP4tUyoyqzclNo814IsmWzJMJQAewAP5F2j0t+FFlUGzWqbQ4kyfcEr71z06SbzwMB/k+Rc14S77IdqtccN5qb45umfXgol7cdeI17W/9jnHwgHKDRB4N8p6L5DNZ3+hbk7MCgT6JyNNqXLOv7z34ZmmuD8aoheDsGLWaxTZAZmzLgyd4GH/ANmqR5mVSprWp7j2PyKbZlVnQ59QqMR/Rwe3nguYd4o9bvWsT5Eu+6q9W9nKgFR8YDD5PIuWXhVZZyD/AJ/mBIiyThRo4bAZYz6AV9Y1bcelO1/PmXecDefQYfQuu0SwOFFeub7K1G+KDJqFUjesGeYMDBl/yAn72ryFUqk/Dal0pvofPMzPrVfSo2s+EQ33gafMGTwPrBehT9FdisVmu2/BuCNPmxT8GXBP7qyHkWK4Y8HOJNbrct+1bfzdoOYSTqOwM/J9a6YjD24YSsUp+jzTgm37BHJmSpkUJz8vk8AwZNrv86tbnmOVKadKrFD9VV2KZs1Lf1rJVIOWd0O1LRhe2lMppz2p14y5MloOjZmusTLMsq87ehVW3G8JtIMDmB1yjD6FxIDAGMG10CiMznqS7XKbUeQMGQ0Xe8z8imEI/GaBTabdsKdSokYKfVIwHosn3/OHYsfW7bqttvwqxUrckswje+7PPbAk/QvTfo02lbFz3bb/ABDr9x0cJtOefiz6ZUdhvbMAMPOt/wClpwxuOvXNbVzynIb1CpcYwNk4xmyyHXmfnXd0jxVbdeqVNrM2VSqa8HrAMJIAeeDHeaiXC9SjrLphKN6JnmB4d/kVzckaXGu15+2KNm0EkzDlw+7SQ+fyf7CpK8cufNOst0rk48o8wZ1s/wCNRYs2a948uJ6n0TlBgbr3epTNpXO9Ka5WhzDkb342AbDAOtVtNqT8aeFcbgsyQhdYH0YLqp0p+9oFpfZFw6PNfP1W86Bm8zGM+jP61kONVKt84+BxKbGjbMDDrMz86kUp59kTqLEF49DrPDoXYOIvDGsWrHp9q3VasONcELN+BLp2GjMih1me9ZF7QqtRBulVU41PfjGb3PBogB+Rn5Faw1ILguG3olx4f6Ok5snnvZMPkWtvmZb8z1FKivhJiTmeamRwP2MoFlKC9BesqpxAlfeNbNnZvw86uLe+zlVoMeDUbczainyoHno5rnFbMtJyS1U6zEiBGa5PXDAOsFQ02AxRM3KxECfSpXgmbL2Bh9C69f8AGi0SGdNocT7u/D0czPAGQPzmuXnUm6DAhUNy3KPPaY8c5GeZn+dAFw23OtXmApsvmaeDwYGYZgBn2Ksk0qpQwacrkTA3wzD5wWjuG8514Umn24coIxxewGfbH51Xhfj/AKhChy6VrTmDMOY68A+hagLbqp0SjVijSqbJehVEM/B6AfVfDOs3Pc0eK5OkssgzmZsngeAIalP1o+hEfeD6EVBmPwzj3H0U886XJe/BM+9WM7MmRXqo9ymYb+/r/OmejB/vzzVtdtYfrdbDU5PSpwckyceMDOsAd5qsDTedNt88NhmrC1txn51e02e5y+vBnHGmsb2TNZcDzJS2QMzwVDS0q5KrSpEs6a/o86zoGYdYfQfYtdwxrzFn1Kpy5zEM49Rh6LxyI2sbPfmAef5FhHmWIzUdh/MHevAwU3nHDa8dY3AlvQGLhuC46lBrjz0IHjlRnmgwB58/k7FraPf9z0p+2qxZVGjevbXk56Wibx59mYfPvVVQaO/ApcqW+w9Gjv4bwPrDfsUKNMYh1sK5SpZvTecwweZwDDyGvR2i5/Sc428S4cu0Qg0Gqm/TTZe9WQDZMA7w3mfQuSXOcE7Xp9D9aw5lQi+MeB+xA/c/XmsuZ1UKsfq1iSc3WM8GTWou2ya/Gfpjlcr9EORKNiKzHZzAwA+/eAZ/OahS1jUSsU2jOsW/QKbMkMAcKthLMMGZT3QAGf50deuGq1Kz6fKuZ+TVT5Zin8x0ctFZ6Gdiz4VWsWHWa3bGhGmA/hFqUfPeYB2B/rroHDG1addVErDFNiQKxChdESQ8bM0O89Hz/QvLYKqHAY+y8ec2bLMg4z7H3c9hsBh1+del7e4kTrPsOj8PaBEmU2RcMNhg2ZEYHgZMzADMDPoBecLwtipW3dAU62GMIRyeSZ5vBk3jwzP8nzqPR7tqVBr1Pr7l1Gfq54D379n4H0Lm3hrt7IjXDdp8QYnB20bq5mbVAP1lLkQ/vTMVkwNkDAPz71oKbwonf+kxCpUWUDNKBkKvJg4dD5hvAP0ZrmPDe/6NbfpIher9RmMx6jyrMaWZ5wgivYB42HQHk/IvS1vTwrHpBXhVWP8AoimsRTPyZ/8A2Cv1r/LJsOKnDGlcVLf9TPzjptVhHztHqbIb4coOg/oVTwuvmq3PTptDvhsIF52991qsTo1gDolB8hrZ85uXPeLVsVmZMpnE2xGM7ttoMOUDorELvin5/kXU1i3TQQd6orPvCj35bUK6rffzjyswNk+uM+HWwfzgrJk96sfZgYKCEnA1ZvbxVTJjbvDQ2sGZjf4isI0ls/eLKhqMmpsOS4CrA0p6bwKtnsscufhqP6yw94o8meDzWCaGPqoaMo8OhFTargQhqI6n+0KOEPvBdDFqAPJc8uSfBgVeQw+Gt5wBa4Jmj7RZ+vUqDPkHOw8U+tb+n/mRbllbZb1+aY6D7FEZ6FcTIxgObgKneB8HdM21+k4Xx+ZHMMCQ9alZt4+Ih5bf4atKPooAAwNWARnDw8iB6MbKbDQM5+0bUI2XGSwNWrPWpAMsG6eoGw028wz6s6VR357uo+Bg1hn0daIKbB9m+5mH0LbQ7nisxWmA6MMMMOxRzWRCDAjQWcIrkQD+sFIuSzAeaaqVDg6OftgzVgYUd6aM5hvR+TBaZmsU1mnaGmZrldeHL/ULYMGw+594PoBU+Bh4a6xMh06fC55hjB0FjK3R+uRFYWkWnDLgZ5qQElz3i0Ua2IrwA42Z/OnnrDCY6bjFV0fI1op3QxxaiZmeQ1ZRqw/GNZ02Z0CRIYcYMDYPen2XjP2itbXBUm3g3uJHGgyRWcA/w3FLjTHGes0wHZ9vAfiRWzNZ96M4y7puN71qwquH/qA0BsxJnjuBmaQYZLD4oMPirZ6Bh0KO9GwDoW22OEFGndFz+tHouf1q7SQK9s8J0avR6lEwzinvAz6wVOyGCvqIfJuhLBvP5Fy837HRD0hRHnHooG/sV0Z+FprntsVUDaazcW4hyc1+W5o/q+xH/CkuSiBWIRxX28wVJDtKlMyAflxAedAMAM1vjAMFE5ZnyKN/hbklYsmKzKkS22DDfmmreok6qz48FhvNoDzNdVmQ2D9oChW3R2KDKdOI+8YOHmAH2Lb6RcBAMP2pgM/oRnTYjw4OMB+hO6xmjee2rj7LXgmabEw03GwP8ip6lYdOnvtP7/APPDzq1jSd6m84H9SbswqnrbpRtafq1n9CicmbJmBns7FfPTGz6FBPem7MKeZR+faNj+NYepWNdTMo9DROP2GBrqAAloga2jmwxw4fJjVWA6bcuKYKFzM4z02GHjPyAC7rPt6m1Jg4r7GYGhpVn0OjtacWJgto9naPnck9Wy2Q1HH83fJgrWiUGsVLPxAZw866x6ng472ANIKbTo3iMMABpfsr6WVO0n5P7Cej/eqprFsVKBH132NZnvNkOhdG2JaLf9aj6TpcfjU1+Y7psRDz+cFZM2ZOez5pw2fyLobwAB7GwQbFf0nS5vUrVfjMajeazT2bJ6a7aDMQ/aBmuRXhTZcOuSACKZtGeYAHYtuHmMKQ3s00lJAw3mGCjm85ivpR5cFpGsgNQOc3ailMyWDPf/wqkAeCXjqNt7ELLznvFZMyWD8PsTwQ4p9Cm/0CIzJcBTYclwy8RL1b+G2no0BwHdRY3fY6vC6A29Deo5vNqczGA2tMwUgIbAB7Nc1ivjGeemrpmS5jhmqWeBg/4AIAkv8AnWC2lB5PbDWfjTz9mZqaElz3biLWeHxQG97tRweP3iWeZLJSRn8FIZBw/kUcMMlLZeBQD5b/AIPnTJxlcMxtYPDTvq1zyKNrZeTTX8dTBJmgg8Oo5mtcED8RtLlmwTaHMbnsadgb8RjP5FjwsO55JHhTcP708F37RzTJxgwXTw+5zMelwgOF1zyZAN4Rmfn1kFY4ezqD7SWEn6AXbeW3+Gq96icyfjsAa28fxHlY9Lhjxsd4dCj5t+7W+n8OqyBSHzcjGHXgCzJ2fVQ3sGBr6UezDG/WtRPPOdCTMad+Aa6BTbYisiBym83lNOiRMtRhtPsiD5rc1OBOkyAYCJJzPo2LrVt8KLZZig/WZcmTI7wzwQ0em5zw1+xbBkNEtNcHN78On57BGtKx4AmDFDjb/PvTR8PbHmNYepgZ+cDUo4ZvHqNp49RkAAFzd1tPwx8zgzQw/YajMD8+apJ/D2sU2PmxhJ+g10gJmZqayzEeDendY4b9m6/q5nTcA+tWEalVIA3sfxrqs8IIEqKZoe4W32WzwwkmiTjLU01EOmy2R3trUT5OGaonpLj3W4umP7jNTmb4F1mnQkmHvDUeYbeqo4GunDLa7CpP7G9dS+fz94s/kadCS37xRg2uNZwxVY89uUgJjGHtE1yD893wOhRhQAkqWzJcBSIdmPnvflACmnaWHROz/Om4e4Yfi1SvXFmyH9PM4pgazvAqsYMSKV5M1024befmW/Ngt94Z/oXB+GlS9VXXoZ4b8Fja4drn6gOqO89sDTVrWGcz1G1SPbFtCLRzkuKOc9wPeIXj3qC9vXZEOZbMz9bvUgJLizgGYGpATzTpNrszyVe8o5zH/wARM84mDYzTOs5/UjOSms/gtkgM3DSSzbyTWfwVxAMzTrIZqPnvVhDeYDBMCQEZ8/Ztp4Ka+fu1aw3mMdRWAPNrju1YVLNBAxzcVhDgMRvZtqXn8ElzbbZHyzGeeG9SM9ij5GiBc7VLBTWT2KvBWEfoWFtVnTZj4SAWljVUD9oax4G2Cd5hxc1wqGuekwXvbrBXhw6tG55rU6o1mYBgGGDJqfzDiHNg+ttRho5PfMmdTWvs/atq60cOuRhvVfefC62IdnhOuOJnVTDMz8i7ayDBl0KcdBpVbY5WosZgin543DVbRhu6EVuZh8+9YqMDEmqA443nHfPDBe2+PHArhzUreN+DEeCsMHmGB9a8j16xoNpEHrWdJB3yINxTbM4bc405SqGcmR2AB71Kv+MdEoJvwbVnw5HY88s5Z/FqjWY7zfqM5P1mqXiLxdnX4QNxHJMaO2eejmpGKOsTowGGAZnvPMM1E9ZSz7/4FNZoNwVVo5cGnHJaAOtRTps7l9P1c8Bh17M1OxEMwXoPgJQaHApvrWo3jGjSDDAw6M/4150Blw07D5FmUAObF4tvuMcDRuN1+DOOY15wkgoNjVKVDfAG2AP5zDM0bNbs5mOAHEe+sFYURk7knhFtGCZ/XsUDxOb29OhvTWHxRgeK6FprxuRgzNAcxxNSZ7j0UGD7FHz+Ci4FryzcxoeUb8XvRswHw9uGCj02SEYDzV2HjRwzTAahwAMPDb3oJgSoe9+Iel51YQKa5P8AurEvRPzrQMxokakBTqjOZk4eRRsY96tzpItNxG8GmQwzVS8ybx56YfkWzOfSg+6MUZkw85qRApsGfKCJoABn5E2Of4fFWdKZcjEEoG94dC1UaiMU2vBTaxTmTz6DBlR69Sn6JP5R9jDPeHzgmxHpsypPVkJb9H59ryK4qUn1k/r+puQ+RVVNrD9NfBxjf8i0ei/Uh54KacbPsNc1qUMyY2yPKH3qPJjPwIYOVFg2Qf8AYmhrAZvhqOAAB1rW3JUjk2pEb02eXY6PnSIGSgRp2/XfM4nYBmrKHGlA6D9N6w6FThMMxWjtWZy0r2AGZ9Bn2KhteJ11TgtKEwEv7xKAOZ2d6wp3tcHqgLcfrBhCc6wDvXQAuGy5NLCLc9KeePvDoWfpUmwKJWeepVAjSQc9jrdihLKSQnSXTf1ze+s1e2M8/Dqjrk7Z4Oxa6ZxOiQ3ZYW/bEaM7KDAze37/ADqPVfszJtz1jL/06f8AR+xZWL2zJ/JsS33G40kpWefyLTXtTbApVJiQarb8YOaAzZdaZwwP6wXIYdYfhxdNhaug8S35kX1NceBxw6HTDPYshzd63m40owib1op//sWcRv2pmDIfWtRMqVuA7yODMaIfRIAN+aizLeo9HoxnXKwBx23tbYH8C1ixSQOG94xocdhiLGgOsHnqvSeta2HSuQd15U4Dl94R3tizR+pq2XKU6VJZDzvHgr3C3IboQaUx0dbue9Y81hqtw6VAjtS24m997B48+sDVgdgW/ckKJKs6XyE3DDRz2GqKtw5U/DQMzAOwz6FST6lelEhg3R38Gs8MFj4sbONxCuCyasFSrEX1q1FZ5V5rDDA/rXIaqH2wuCbOD7mcrxgadWzo9z1WfAkUa56k9MN8883QA0FEpsuq1J1+DQ4z1EDNmYfQYL6vrWOT1KBOpUjQnMYH2fOmmTcB0F0i4WbfmVR23KM2cymZ+C6fuX/rWCejOQ3zYlsGBgeG9dQsjZPSNxjeYBmjNlgKXHnA+Zun1h5FLoNNbqtSjg+34XeiemOA7NiA2ARwPoBQHaJPYgQJbj/vwwBdDZtJ+HQYN1VWDJeoT/8ASI54GuVH7Az912LpFSgPw+GjUSJVZm/B4I5vbFhcC4jWqwyxIqtHDn48oMAPoM/9dZw7eYk16lMMPhGBjxsDP+BDR72nQ7NAHz8WLM8E+9dYgWBR7qhR7qfgnGMGQeZwezzBcylfwKrFNPjdU7VvGlRpLVejcqGrv0T69ivaraUWfQ6rblfuoKVQmKwZ73tYzAOjf/rrjkwAgV4Lj1HgnRXswMPIqK5GZ1Vddrj+8Hz1pP8Arrsj2E3C7evaWDp2dY9fkz4XP4MvO9gZ+RSrwrcs7gazcjBNZwZORB2G8Brn9Bje1rAPgAdmZ4Zq+OA3UqyFVYnA9ImyQMwDsWiMNLXrDfO5olOpUrWkPsg9g6fXmptscNJ0y45FnONvUcJUPWMwwMAPzmp0apetZ9wXOw5hOoIYQ/OzgrDhvfNRsauVB+qxecp9ehmwbwdcZctwA+zzEPh5UItDpsYJtIkhzjsc973zgsfM4b1Gm0Z24K5O9VTX3vucQ8MzzXWLh4tWr9hahSrcoEkwYAGAOR1nga4pcN21m8y+0dVcA5oHnGaZDAGcPIo8QjhduonFG2PR14cyLA9TVV666iD75mB+CZn3gf8AkXN6l6SfGKsUuPRqbVQo8LRBmSERnfJ+fM1hLhuqv3I161uOc9JqBhgZn5A6FX02q8hNhTsM+VPPA+g13cbYFSjTmZHNy6iD2fW7rZmmo0PnIEt99/fFMAWovaZzlej1yVQ40PmnmH+XjvZhgmrzrFj1KqS37OpUkI9RADeAOhkwWlwpkXjPl9nWtdMurn7UpNAgsGEeKGb2feayLxsdYb08b0tk8G3MGsM8FjcJWFHqTFHrLVVMAe0DzwNd4onHW+KbbMi7qrVTqtPM8PVL0zwWQP8Aj8684gy487hqAH1rTU2sHDgNU6o03Wg7+cCP1yQW0W5rdtvC86VWKvT6NUrqgW3H9WmYSKd+zbw71kjsnhDb1r1W3H+IECt118M4dRDPlYf1msjZlnwbngVulUOgTDdMOapsjPAGWA681W3tRJ1pFEpRxGQj1FnW2b8/z96QtEeqtNhwDi0YAM5QGzJdPeBh8i7Rw0uG7ZNgVC47OtkAat6NhUqyGzM9nZ34Lz4exo3Pw+xe3rMu235PA65eF9v2cdBCnW8dQPOZrZm9v8neou1qTjNW6cfAe17nu62Idz3HOjHFCo1EzzZzPMzAAXlz7Q60eOxLiMvaAYBmttdt/wBNjUilUPUCpZ4TXgkBsZ2dAeTr/gXPK8y563124ujEfADDeo2tew7zqtHLQpTcZkD6w0esFfW3UokyrxD0JMCOxs2HsA/OsDP2VcPIDILd0GMxSqWFZDW5eqbNY/cmHeCj2BcXVdsuZTZdKN/Ct0t4DeCRsDD/ADrBVW4ZdVdj804ycjfngzgrI2WKrccgK4/zjvZIZ3n+tKNZ8s7mhQYIBnnnvPrWMCikvRGWtTolB3o59SYOFH0+WknKDN57rPNTaxbEo69IY5E2Qw1sz6DDzgs5JNgHTbYc1mvPgtoDzIaLDr7hgYBvNQQnznoR0PU+5HJ5o2fnUv1bTTjyH85LJ57AaZzwDzmoRhUoGbDjDzOYZ4GHWtwj2IOtbOj8Lq/clBolct9wJ/rd42DaDYcY/nVJclt3BatxzbRrlKeZqFOeNiSDW8AP60FIHWpcN5jmGtd/RAzwM/IugcBLAo3EviI1aVc1galQ39Ew2feuwM+xR7ksZ/hFxEm2jc8T7xTsNY9+B5hnsVC1k2qxQbShX+3/AD9SqoD8X7xGMDZfzwDrWVZZlvRfWuh9yivYG9n3rrHDeqv3+xH4LVm4KVR7EYmBUKxOkPYHh5Az7zNBxC4b2/ZgesrRuONMo9Uqr7MCJnvANnW90Ge9c53eHPPVtOZo1Qb9az4dQN4NFkD2PAeeeapZM/OEDCuLqjVyHUTiOBv7MDAwNU9SOJMP7pB5MMAAw1s96saqgs1y26TCrlGYen1UwM+UON1mHf8AOujUq6r/AK3ZtPfv+zo0amMSQqdKrkgOWMDDeezv6Fj7VqtNqUWbVbu+5tUuHFp+tH6z39am8VOIU69qHRLOtm5+ZoVLjAARzDA3jz7/AKEZIl53Pateq/2goz71SrE2pc087y2iDLHk+jBVsyvVWHnUqO/6tmhJAwlxNgMmHeBrH0d5ipTQiG+YSD6MO81pY0/CjHRnGM2jPMzz70eOh+rbq4kWXWOJtcvhmTU6DGCazHBkDN4M8N5hhh/sLKcnRnrep8quTtF6oxjNnZnFzM1d2NyMCmzbVrmyn3rDwCR3smyeYfX2KRR+Fd1XDS5tuRZUM/Vz2syYPbM/n8gLlu1wqbMenRquFAqNS5CIZhrb97IZ9eHevfHouvTplZv2XVajGnzT5Vh6RHPMDVtwT4Y2PMhyL/ftyAc2rsxeZiPRgPkJrIYPYfnS4CU2LSr/AOKbFNiBGj+tdjLXQG81tw8I7UjA9E9ihc18EtZboc8rYP8AB+8pfEOlRP8AklcrwBcjTQfsEo9gTQ+vv+gF1aNva1w3gfQYdBgqx7lJkKRSqjBZmQpQYPR3ugw/3NcfoPF21eA9yyODPEa4zjUpnB+3qjIDMwivdDLx/If8CLd1NAbKiw6lBqUVmo0qow58KVvZkRHswMPrUjmG0EeYzgCpDk4F1q9km2YYLPz4za1gHz+zrUd6TmepqKIYPgemkeoC2wgMk9YFIpWZu6B9CZDepcMNEs1dgK3AfBrXYb2d6zr0/Dw310A3mJMNYG5AbZkOuA3sV+tf60Wp3uR1TTEmlQZg6jexRzecNEzM0dji+3DgwrzttgPaZqk6HTbBs8A8633MwZLWn1rL1KMYSkhjcIjOn7xS2WWHjBtR9H8NBzLkN/xOhbJA9TcHT008yz4SlBJAx1EJxsyUihksmB6YIYzxg71qyNk8z1E0EADd1NNai15x/QwY60odeMPDlnmCGGDYPm255FHOHounsWSmqptwsGOh2JqTVYgTTAOjDrWMzlsnscR5v6Rm51rPBtq6VM/nLDUzA96vWTwfa+tc6hzzB3NXfrt8PYPngsb4W0czo70NipQ5EF/2T4YLi9bpr9EmHBf6wW+ptYqT2HiKsr1HnV50224P3hswMDzSLXbFZ/BOhMcVqzZNYN372bMZrz55qSfD248QfiNhJA/IuzcODFqxmSrBl5RHrYuaGOpKpTzIfOmsJUb24GH1o0XABkgONnsUeNJc6FZxpMQz37FKlScBz8NNHGfD3a1oRmD9m4or1HP3ba87jCkjBEewYfb/ADqayy5Dd87XYaXqpwHwfcb2Z71qJlHYBoAgmBge8FF2uISbeqWB+G4ui23VfCDUPeuVU2HLjO+zW6o4Pgx4i+N7H7nfwt2E/P2jims71ggqT/NcqHWtBTalO9m4Gzzrht0riZGzPUUQGcCRyalswUQJiyFnngojzzhmo4ST7z2p1k2zNQtIZNPZ/BRwTuQKFDM0Sjm82CQSWwQSDSA01nreIgRKwB7BGEnMlX6zn9Sa5nBWLI5jgEmnpjfu1COTmKjnJwQSufwR+svgqSS84HiKEdSw94rF69PUU6kqnn2z94pYAw8GoqFrDmZmCsJLLD2DhtgezvVEyAMmpBzNHvUhTKPBe/orP6FlKxbFKOUbjjZ5n5NiuJNSfP3ij62fW4tfHJ+GOPDFHw91ndRiWbIfQo52HKjH4ebwedbsDUpldP2Wj5oclmU2dDdNhtjoUq1aPLn1H78Zsxw68wXQ5NHgvOnKMPFTWDbPs1r9lsvnkEyjt8v91b3rKPHOhyAYlNmya2zMzzpmfGCeINrnjmt0dMKVme5iriG8w8GbijhR22dmCkcm2yGo2Cu7REJQBBNR5lKiYajAb1HzwJWDO8VzbXhSnDwU2MG3erA4weRNGDfu21ezBkwRggw+KE3sCUbDvenWTwJQs96dA1A00CfgAaals1Vs5HiOLKBJcD2aLWM+tFtnMqrGh4bm9U/rhz8RUmsln8EGgZquaN6qsAGo4s5ngjzzFBYHWGM1Xz6rm74biiGopgrQdOpTvxzTQST3gfeg0UGHxTYkMg2alhGYNRI2xWGs2bvhtptaRGh4b1LDUBRAeP3ilhJwUA838NiJnXe9omgkt5ozkn0N7FKBcs2Hs0jefZDw1HzfD3ijz5hg14jipKFJkgBqEb2Zpl6TmSONhqopnbqecZzWRenuAu1nJivQuRfiMn9aKHR7cjFzbFHjA6Hfgu/1vb62Nw4Vz+fYf6FIZkrqd4U2j1KKbjkQAdDvAFydmMeemHnX1Y5o5HHcdaaZpdikM284Y6muaiTIE6Gen2LxCxoklgJoNvgBh8627J00PZsAC51A/aBWlZkvh3rk5nZC+OZv8PYCDn/xHFTnM2daiHMPzrgXtq4zzD2q35wwXlS8Kb9m+IzrHR95A16DZqT7Ps1w/jezhXgqve+zvSLS7mH3+nR5fnBVkmMonC6f61siJ4+ZsbFbTA3rsi2dwz8mMe9RAjfIr3DM/EbUgIzAe7XZ45mNwzR0361HOMtRJBjDDTUQIDGC27kYZp7YKjq7mUpzA1UvMuAem4r25rgOfwSz+CZMEYLZ6az+CLP4I8Pilh8UQWfwSA96WHxSw+KtazZmOArKNVfxAVAH7zpW1tK0nK3icoPunea4+aGsWCNVWDPBS+ZbP2a0FVsm34cfTgsYGHesjyz8Z3DPoXH+90xGFnnmjBMsnsTzJ71ja4TYwZkpoMnklDjOKwCMua22EfRc/rRgy4poMo8PiuYwj6OxGDJ/hqQDKdBlQ0GEYMFKZDBRwPFSAe2JYaqTJnHPkWAN3zmvJnHXhXWa3PAHIPLBnm9IzzXrsHkqloSaXIYfYZPMMN4KR+dVw2Hw2t6jGb9Zkyan9a5Zh8V69rfBPhX9oZb9z1IM88zaZPAFwXipSrKolcCDY77z0ffmZ/kRSy4IPTq3V/swc5lmEYb8+tdIv+2KHbFO9cMOPPTnzwAHjzA151oNeft6fz0WU8yfyLote4tUqt0SOD8uS9NYPPxVAz8mj8/HdnGxh8mjgs+HqCMGnVaUDzvnM1LqXE646k1oagMh8gLLz5j8x/XcRaa9RwmOm5Bbwjn0Aui8EIdRtu6o9SPY0wnret6359IZl0qJMk+R7DYpUatuW8RxH2OZw+fBB+fWB+7bM0GfwVhPBiMZsNnn86aknTjjg3E3yO9arR0gDejAE7h8VYTJ4Erhl5wGg1zBUm9LD4pf6xoI08wPNg1YMyW5ImDeH5FRMmDMXQbbUimvGy064HesbgSJLzbOeh7VWFvSZUZ2PVX3/Cz3hh0KnA8ySh83g6wYGbRmowxdLvmsMU2GE6lNgbr+HX2LLz78g16BHYrLgetovQ8AYbFQ1u56ycD1Hzf3fMDwVEB5n/sJha4psxiNMKW+4tGdyOVjAH5b2kx0ACykZnWDoMPrBaCHDONTtdxtRcLSwpTdyNSOU3mwGeCB57CzY7ep34LS8N4b7NJmynwwA81jZMZ+SZ0oPZAazUKlMsSSCJFYOS98i3Z0SdbdB55+340OQHtj60rYCjUGgut24AHNf9s88G8zU0Ka/JsWboGb0iaee81qMfUoD/qiRKOUD2fk7E1ZlbiQGjiSoPM59B4Z4KazQa4FO0JTnhH1gp1pQGIFRD+apMwO8GlkHbzpsGiPtN+uIz2uGfkWck1jn3Q354BgpNKsYL5vp2h02K9TY+fjOvB+zB5zUa7beiWldHqOhy5NYjh/SNHBe9InW8zOrzr0GC3m6lc8Ny3pGg/5ANVIeuKDVgw5mBINWd4T5depceJWGA9YMHnzAbP4FjYDN+q2+D7G8M1s5NznP4fQm6rKDmw7DPeax9tyYsalx4kveyB54J255kWe6bdNYwj9gKISUZ46q7/NxhGj/OpxzJdNa1GzzWHOfOpTR0oGDjfWCuLGqTZ1E6dWZeyaGAbO9Oka2j34/b0oJ0qJzMf3wYdiu5NbtipNG/Rp0aeD/Wy8GBxg/wA65/c7PLRZsH/g7AUaw3mzkaDn4JrHpF3XozARfWVKqMYNA888+tWcC4eQoMihweWMH/Gza6w+tZyHQawyw7Fqr8ZmIZ59eeCGiQ3KV61zb2B0LvgAz40KbBb9qe9QgMLhoxxZ0sAreeAG90GtFRKCw9SDrDlSBl2UBgyyYdZrLyY1O1zYYfjbPwQW0WFSp7lqynYk5zRqGGBoXgfk1s2Gw6wzWlrFmVGTSYN/+pjk4B95w3mYedUTJmyX2gcYeZjnsA8MAz+ReiDJjTgdNh/2QdC6mDNHmWRI+1UqGE1iH9waB7M81h6PGlXhHNiDgDueG81u+FdsU5niXT6jcYU2TR6IGE/fgGZ//YLO4GRjM60BpjyMgtRZlSiwHXnJzBmDAZ5gtNxU/koptR56zpcmSc17xmgA8AP5FWUSBaOhNY1D9ZvxswAzXHzMjvrilVWQ0dSYCHT5R4AHeqW9rPqtNrLU5hgPUT54BvzAPrVPXqwxrtUpj2UI81cUesP6U1im1zmaFKh9Evfh9CiGrCXhAbpU84jctmT87PQrOzNCBV6eD7mDUpk/G7wNUk+SwdOOgMaLxhJz1cP8CsKVAlasdidUYzLQHgzmfeum7F1JjVW26lUH4M6NJj1FnQk6J9AKTZk9uZXAg1VzNpwDMF0Bm0qbGtmdOuqJDeaiwPu3Lvd64/zL8+LHDNlnlc+WMA3gf1rEaU6U+FvVissRMKVzOgDpnsA/Isvb0kOVaYbwM4p559hrYWBdv2DqR0esyjnx6of3mPI6DNWd1cKGKVUZ11UOcEalVAM4cdkNbM/IHkW0IcxuevP16qOtvtgDwdGHeqUNTVNhzrY6wXYodjTresiEcWjHJrdRB8DAIxm8YfOCxsylVmw9L7VW5JgVCqRs3gPrMF1whBZjP1KLQor7+AczyusfQyHzrQV7XZm+rbfYA9M8+ba2As7MqsGS1hTaacNryGee9WtEmUY6M7Eqs44ciEyGBgpt0eFPMhzplWdffY8XDM1XzJhz5Uds8PItgE+3LnqjsSLFmQwfjYaLR5ns+dZKtwJdvNR6lnhIMz2eReiwqVHlM1EKUxLZku4AeYbAAPnUqNRJdBlR/XjAHHlM67OB9YLJeuK5JCQ4w/vlBg8fnBXvrh+t0a34OuAHToz7GZ7O9VDK4X1vcS6rwxqz3/DY7kM49XDOSzIjZsxj/X861r3EVi7ZVKoFco8b1OG+SzH8Y/rzw2AuT1Vlhma6ww/rNAGYHhhmpsaYxMadqLFRONmzoaPetoRho724dRZNKqHEK1Z1NCk+sgp4R+ZM8OvfmexaDhdxCv8Aok12pTm6a9R6oHq+qxMAzOKALOViyZ0PhvEuq3KzUqrE1s5kH3LJ+c1sb29H5uNY1M4jUC4Ocm1hkHmaI0eb28M+vNRcLYm56a/JkVP7MRAk2/Fe2SC62QWSCfLCFymub0fPvVrSpkGZSypTkvAzMwzPoBVkZlg4RtnKeAw6ADvTAOHr1KRptgrtmZKmQmqG/UtaOHQDR78FKoL0F6E1EltxqaYZ4On3/Wo8ym/eI79GlQANgMMw71jzQs1TalXKC7Li0c9E5R563eCuDeqtw02nvvyjn1VjYZgz96A1TsxnHpQMP1WNzB9gK4jXDUaI6dKYfkm0wAb2uv8AOsdi7jTOcCPFlSngwDADZ2GszW7bqVBLUgxHno/9JN1nPetBQZNDeaOXXIJ8u+GAOnswVhVa83ApcRugVV6S0we950PbB5FzbasFM+/0snIkpkMAzMO80c+S/WLcO4LgfOZIY2MutHsjH5HgV7c5sVuvUxy36Pg7KPM2Y/RmtQHDqsSbfq1uQaadbdqlSCVP5HeEAPIa6eHmc9tX6H8m/wC1eIbVSYsuqybUBn7+YRs+Wz9/8/0L0QHBbhXY3EZq7b44ms0eI5MlTYdMj9b2th1nv2LoHo/TOHNNBqzqNcZhcFOjcrJgzjBmU8ABsz8+xcR4i3Udk8fa7XLmuCGdrnWGGJkR6GEk/Y7Nh9ALswrwa/kTpUO8rztXhlhUrcqL0WVTatz4PSWZQbzMD7wzPf8AWs56Qln1ni7xapVqxalTZNQtq22It1VMGdGKyYGfXv8AJ/gXKuMfEhviK76tg5s0+FUn5UAw8EwYPoDD8n8a2Ho98RZ1pUat2BR6xTaVXbhqTE2HUaiGbOHQYGajbJ6D4UcAeB8C3Jv2fo0C6pDGErmJZ60nDDrDyf7CLjGdgVKxjtm4KVJnwZoA+c6JgYQN4bz8h71ye85PFv0crgqtxyq5Sq87ddNlUyHUY5gAAAYHmAB5M/41z1m82K3wvpXC6wKVXp82aZ+tZDLJvGYGHQf6NivhGB4kWxUeGnEE6U5UocyWxhNZdB7ZJA9/QsVGmA9MPm/BafMzPDoBb3iLW51euOmXVd2jogz6v5TDCUAMh3gsfTfs4frV+q6zP3M+QZA+t/s3pcLgbJweVdia/YZruF8+jlBk2DQuOHDysHcNrymdetnEAAegPh1/xrglHeb1XQNvPXZNhdg4G1u8bYfhWPQ7mk0ShXWfq+qm7GB4JOefQz8/QH1rmwWx9yPVJm6KZKbs443JUoDBkAwOTFPolYfOoVHeo0OiUSW+YPOynpT7zOe8MD2Aa9F+kJQbguR2Jxfty2KlR7chMwqFSnp0A2dZgMw5oM/c9AedZS2AcOzbwp1zW/Tar6hgA9GB2NhgZ5/416MlaVgVziRAdrNv2yFYqDGb/KU6SebOHnDsA/8AIrKzK2dq3Q6xLfzhVd59ifBe2HGPoezD9aXo63tcHCW7YlwUpt4IlRMIs+If9JDfh+hemJ/CKy+OVLO6m7fqVHrtbP1hzfXonsAw6/kWEQJ3oo34DLs2muVWfPkAfKw48jokxc9h5+cF1jg4GF78S3/PWMP8a5Twr4Y/yV1msORYujnGYZjOvHmYHhvMPJ2LTcK699m7fd73aieZu95mBmvpcPrXjY7gZ4EiB5YqiXgxVZhwdQ9XDMFZvVgGXdPUwV3wYYxbVMvZry/6VHCu6nrthcSrYocmsR5sAKfPaAMzhmHRs8m8/wBC9DRqln7NWUaSowvbzL6JFeuem3RIs71VVWaVNA3gjmHgsmGe/wCReqDNV0CHTqU0cWlU2NDaM8z0gwzNTs81GFmjecUR4806YJrD4oI5qO9vUgwUfWAC03OtdMIBngSs2dMxA1SHJbTzNScAdioXXYqefvLxOtWwGBgqmq6bJ6i8j96WSqsbRI3MFScyGXiN5rZyWWJgaaz8+3sPHYbX2otxXCk1nAd2dCM3tYPE60DzLkbrbVe9Mcz2bAXTDmtIeMw6E09M5kfHb3prnPxEX3R5dLnR83A9maeZqTiA2f3iaMFCtrAJLDwameBpZ4EqzeCkAy+EXXcb2I0XDMwPzqQAMSfzrOZGpbMlwDUCwmU1sGDcYYM8FXvQ3GY4vudCuGTc95KD6FZvW3XPVYTp1KeBoz2AfWsdtcMJhgrCm6hlp4KRMo+zUY6FLpT0SBvlsaxh2K0LWlesWXQPlOjvNWE+pVUM3AYA1nDrFZmSPAYAATUz1yB6bji5sL2s4dYfemhzWALotBqTDMIHHFy+iW3XK3IMHMGQ8+a2FNs+JTcNSXJMw+dY8y+ErzvODDlNNxHM8wzNYKvXIxVRAHHOhdK5aj+/iAf1qdGgUPHNulRv0Jw83WXDigPPhv0zAFI9ZRQ63F2N63qbPkAc6IBgB7A7FYRrVtwzNxylRsD+RbfZB0uLs1VlnofWooMa46xhyNOM2j7z2Lqsa0rcgSAfg0qMB/QrN57R9ngH5Fwc3t7dOHL5lt1WA0D9Vg6IHszzQMxnADBvoW4uEG5kLTWSAzZdNtxtIvZcLClA2ZBm2tBGZVZSuUzBaWMDGkC4eb9zaEIIzeephvVgyYAKAwz9mheBwAXM6QSZLZmgA01h8UYAgd1nAJIJLiA2XE0gswmOIOZMyUI3kHONgoFrJmNgChHJM1EOTmaEHm0FkzPwH2ikMzMy9os+cnckzM39aDRnJbUV6Ts1FTPVLA/aKPJqXgH4i8wLnn2/OkczMPaLI8+4B+0RhWPnWiGgkyXNJV5nvVedY2eI4lzzKAzmNgWxxPxqw4GwFTvGBn4aFk8CVLbAJ75ojkuH7RU8CYB7HFON4D6EBGaazwR70DwbFIDmdylxpO9Vh9aMNRBdm9m0qySebvhpZuYqPg5kgWbmSlM9Chd6kMqhYAe3rRnJYwwzVeZqE89g6m0WlSTbAs1OgPAYA4qR57ME9DmaJ4dqDRdajvGAdaaCYo8+SBgeDibEg3mD6FE63VVBP0XdNWEaSbzoIJGiiAFLANiWGCkR8TRos8EOfwQAgz+Cew+KDRQGln8EykqCePYq/mHFJPoVTJe3qRN5n50JvKv5htAcz94qE0JmBKwgPZks/rAas6a9gYIhqDDaBqPh8UYPZte0QAo2tNZjAfieRIzADQm9otKpemZkvRbHJbVPPk6yXOYComeaIRMPijA8CUjEEBgqBg8ZqXz77LWm2q/PBJ6TsQE8y5MA2zWVk0R8JB6Da0ASXPdpGuyOZjhSQwcZ2OdalGy49171Mwb8iBX2Iwr8NHsBR3jU15RHgS7eoRvOGmgPepWihNlYLAB5ksFxvpudsx5YdbDy6AyG9Qb5pXrK15HgZ4IMbwBqThtTac45meGwF1WSC878H6ryF1tMa+Gua9EPGtLtCveZwPUTQPNqa8GajhDbBbRby4A9gYpoFINltNGDgLbxzIuDJnsUWZDYeHoUp7oULWcAltDG4VXqd/PYwiCgy/w1dsvKWDyu+ZGGaeo85kNRxhQeWcD2i2utn7RQpMaK97tPHMdLKGCSsJMNsPnUfk9ns117Y4DAhy5koGAbzA12ujgFKpMdjTwd7wXPLMqTFNlABgtk9Uucd1Fw+26YaAMJnhud6ztetU4bpywfA/kVxSpOBBqJm4akZhs2L5e3UyHLveRWECGGebi+MgZqwjRn/wANdG3mF3DDYCdeDAVHjHg0kclzoXBbpPACkBpqvB5xSAPJZCUhN5tEyGZJ44eaCCZmnmT2ItFPMxgRYgNI4zExjQldB9aeBltTQZDFEOCcTvRRt+6mpdSodfkwJB9m9eR6xwW4jWfUvv0TCF/aMM81+m2ILL3zbbFz0k6M+5g0fyZoPzpPh0cylu1XXkyXQ61X0Hg/cdyQJFYDCNHi59YHvXt6icBLchx9ConJktfXgr17hLbHqaXQ2HJLMeUGHWg/MyTGfjSDbcTRhLPw2AzPyL1LxL4D2Pw3D14E56Y92A6eYLjp29SqlIM4tROMfkADRbrHBl64KJZTTdSihGDDPMOv9C55dt7cOZNSkA+wcyQfk2LZ0G8JdBgBBqIHPa+QMFj7/oNDreE6jUaNTd+bwa2eZqB4ZeZw8R/9al1L1UcUDg044x9CkVimsPSjfYb71LqVbfmMDEcYZz7MARbMgynlIOMbPtAwNTaD6jCYf2gbzj4fxrWLFOCMN5IqlyvPnyP7P2LR23SrLnxTcqtSqTNQ/B0c1Yzh7EYPPsj4akVIKdG1dRh4ADzroEC2Ava1AqtKYjU2JBZ+c80HP4H7UDh9C1th0ejXVNl01+7vU5h+zB+MayvJvzHY9Ki+1fPBeuJPo/cOeDNGt++35b3O6Ie236xn8iDzJXuG99hPNiLatSktB/SAjHvXUOFz1sT7GrGdq8hIis/tDQb3jXRYHpJ0A6HU6PqTJlTlGYM5s7GQ+RctC3rnpVkVCo2zBnnHB77yzE6ANRaHL3p9ZrcplicGxdFpXN3bbn81Uoz9XAYPYM4b1mrM4dXrf7RzqVB+5AeByD2Bn8iuIdt3HDqNTpVj8+AQsOczPr86xuFoNNqU6mm6DDmEdVXLTgmc8wxm0fQtBbDNGeM6bUYkyZNP2LLO/M1pWYHFSg1Fq2QsupRo5nhoyI28FzYGFt4K4zPClU6CEmbUXvBaPvNaIDqtNm+rpzE+BNN7xmWehnNdT4ncBKjYAUKXWZUl71oAGc6IzohGzXri5+CFA4i8Lbfo9GprPOvxsAqeGGiHeZmqU8CcRXqVShCDTai9zYMhsBXfD22KrAjm5ErkaNcFUZ8GOYZmALpHEv0Zq/wKhO1Jyo0e54UoAAzeDN4PoDvWw9GbhRRrVgTeJNxsBMqr7J8hTgDxmfJsWnSOKcE3rHtviNUD4mvvMwn8w5jsz+cEuKNSnX/NOh2VQ4cC3aWfOwDANE5P5/OmqlMcO/Jst+CyyHOaxxHt5grXiRxUtyfNhOVK0WYbQeDqxHgzTbBz28+bjQHZ3KZumYAGseeCp7Js+q3nKOVOqUaHT2PbG7/v8iv78jMM2+EqJOOS1KDMM1haDUn6O6Zsd4YLHm/W1u3SLhtXhzSqW0xQ6q89VTPADA1j6lRKzSnTCVFUQHp0kGq5BYPCKeesALUQ7hfrFsy/WTgBoBmzgGG9Y4GK+6ST+99ihW2y/VbyiaDfhMHnmretxqVy/PU0MNfrD51o+C3D9i7YtVnRKrJ9YRWdkf8A212R4Uz97T4NVqwQYrGEjDAzz601RLbCmu6lSfBk/IZqwuSA3Dix2H4MPm38/vEfYZ71WQ7e8J2XczE9mnh0Gb3tjV3AvqrJiSWo7lKYw6883s1BjTPWUeoQWw8XR6/Oo88KAdLjuUNuTG0Awe34ZqXRLeqVEobVYqLGjHmngyfZgowlT0q5KqdLhU5w/u8J7PBR5ltsQKj6yYc/bfHwANgK4ZprHrRqVp4U8/bPdgK1ZClTIZxXHM5AexPswWPcOh0riQcbkbRpVNwjxYewJ2wzWP4hViddVqwqMDEaBT4r2s8GfWmTgVV6RHvG4KlGBn2DO/rBaufbc6guxKlBfptV88fPzrLx5GNtiBzPLsWPR5kmPF/b5eHQHzrVVgINt2/CPU5yXWJOAA8G8w86u5/FoLG4X3nbk6DAgXBcszZHaZ2YGCl+jlwTue4RicRqy3AOlaOEYzPxgPzgu6IYsPJjQaVTTfl0r7wx0A9sMFSWNJYk35H5tzDWjHh9amzLVfrF9XG568meBJ0GQndbwfOslckP7PXG7BblMvHF6JDP+RR0hVVn70bj/wC0a2i8moE9iG07Q64B8vnmy6zvMF021eG+tQZEu4N4St55hvz85rE16251vNA/LchvAf4W9YwtmI0aD608Rswj/jKU89LOHyjGwPOpH2YuDQj1UwZOPKDPYfQngB9mBI0GM8wwM/Ia2tbQMzLVqoYVWVsYZ6MOtZ8I0STHly2G9GOweDKiGzhTWmO8zwNWbz1OjNFSqcebTfyLHAUyttzGgbcYZ1cMDM96uKbdVxwIbVOps4+Xi+OAH1h8gLPwKJUqxVAptKig9IPoA3sFcTKO/R6i1ErLHhMYG8CYHSK3flYtuqBKty4K3RJFXh67xy8JOsfyLlNz1K6rkmyKjc/OSaxK8/X+haa5Kl644g0p9inZxAZDR3/rWluem0q/DkVliqyaDIingcjMNEDSLR8yFcno5P8A8k9PvuwLqk1ho2QOq0yWABJZf78Fyme9BjG7EghJeM2QZPv35713i1bDlhCZ1+IYVL1jvgYHmbwd6w71n05mEE6DOk8wb2jpAe80+ltEOXzIdSojseUebLphmBgtXw6nvz2qrFqQMz9Q2DBqoBs61YVKqt0eswpUulc5FhBhonvezUu8KbBvB2Rc1HqMl6Xo74jLJ5nh5/IrjmMKm8IdOoNUDlaGYRJobIjT2eD594LGz2eWBp+XBAzinny7qsLYgPz6poMGECWAZgBnmrCtwKizCl1WqtsmDJ4cu8Gef1raLLVM+6mKlzzbdsRoBysOWz62Q+Q1trJsOv1tqFX7KowBLpwb3pB+DJP8Fc+r0+DWI4Soko9UP6I6eZsqTaVYuqGbVHodZkw4gSQmvHnsZw35rphi7Re0y+KbUpdpTpdBZkTYGb0GCexkD7PrVJQZnDm6rfdtiJbkmg1WFGz9YhMA5Jv+QDVPxIr1lshnb91etaxKezn7Nh/OB9ixUmGweEtt/PmgzMPItu1j0pR2HWYe99umg134TMz/AMCdjWfXDkAw2/DZDyG9vP6POq1mHOOKZxWM2mOs/ItBe1S9azaZOgsPGcWNgZh2LkbIVyM01mQ0xFbCMbftgM+s0dH1NlKiNsgYbAzPDW+RZ4z1pRv959aejTH41WCUYawAGz615cLaIJ7FvXNHlgx97A8DDPoUeZJbolytesnM8zzk/n6FTxmeckHKc6z6zWo9VUCqiDEpwOYfMAOQ8e81jcDRz4bk93QnRNGO/wDseB7MFTvH6qoLtuPhhNB7XZDzgpZg/wAm7UtfPkgBlk8PJ3/xqih0SpPO6bEQzlv9BmGBmCxuG22qnvWd9n49wUrnOd384y0exk/OCPh1P4lUGsyLmo1Zk0ePKwlPBI3g9F39nfh/nVZDo8plqRQ52DMjvZM0gptVgOvVWmnJkwtHlTanBvDz7EhFux+j3xFYo9NuunX5OZetKbm8zWQZweCaZ7MPr/yL0VdXozTuJ3Dm36Bc9cpVKuAHjqEmsmznJksYHgZ/kNfnuEzRo0uhxZTxtSnge0j6AwXa6J6XvFGNwoPg76tCe6cM4XrAzM3tA+sMF17RhScHOCdwcWr0qdK9ccnRKQ9oSamEbMHg39H14L0xcPD30b7etU7AlsUeBUJsYNE6mZnK2d4Gee/6F5Sti/65RLZl2PSn5kb1jJA8Gjwksn3gfyLahbb/ABmqjtSl1ifDrtIgZvTqmf3UwDo+foNTtGFLXuHt6VK4Ao0E3qlb7DxshUD8FkMw+ddgonBOpcK6RSr/AG77r33KAdUk8vTQeCM+Hn84LL1upXadLasO6q4cwKRhV6a8YAcKSABmG8+tarh76RQcV7Qujhzd06TR3a3A5WA9I2M6556zAPdnZgCvhsw86cUaqxcNSm3HWWOcqFwzOdh1AwwzAAAMADsWKN7kIUR82wOO+eGYGuvekJaV60er2/RrufeOPS6UARpDNNeCLr+QD7/nNc/ppsBTmvAZed7M02QvbAh8PK9RpdKud+MEinZ+rZB7Mw37z+fYChWrWAsO8oUu7qVVQhaJmDTRhmGfQYKbRLkpulytZoEMzAAAJYM7/wA4dBrcWxSm62bV8P8Aq16lRWeSNl7eeAfJ/kXN3LeuPRyqtH9Irhb/AOjTxJlnMdoMb1nomZsnoZ/dQ2deGYH+QF5Eufhvf8C+r9sq2WJMmk0EH5XNzmTZB6FF8hmHzrfcAeLX8iHH2RePEoJL0e5aadIgVaOGYRujA9Hv6MF76oOhxv4S8pf9OONUDelUipcv4OtgeB4fWGH602h+U9HrE46RIY2bMP2gMDP6PkX6V8AbVYmcLLUuqm0b1b60o7GtEw6z7z/P1rQXzwHsS56PRLLo9pQ4cdusRZRyADeywHWGfz7F1WBR6VQaXHo9Gi8tChBgyzn0ArizDmt1cPZdbadlgxg6DJmfzrzvYwOTLcjsAwZusdeAdG9e2A36rf7k/wDAvK/o/W3Ords1CsQZwRnYs82N4dYGvq+n7LHm4VTR6kxSq80452ArOsT2JMo5bDmw10iscPWKq1pvsMg6HsXQDoNcir1HqtvTeRqrGifZ84L6XDfjkcf+NdU28G4zWgYHsVxAvPmZANtrm+fwUuNMbZXTfpwx7rdwgVWLMwwNWoLkVt1sGZGo4+ul02txZIhm4viez6dxbs4eZanGzBRzZwUtmZEPocRPaZ+zXHh0q8wWfuc3GYYG2G/PrWmNU9bh8/CNhbR/yMZGn6x9auIcwADBxVUmiOBPal7w0AwPZ1/OlJNtmEcoD9mu/wAR2MGqCeAezWdrdYc1TMzVJ9oXAHPNVsmeczPUXTw+n/uyvmWoVvv1FNjV5szwWMPUSZkuB7NdnTDm7m2mQ2JnQszVaI+yWo22nodwmAYOOZq1ZqrckNTYf1r1G2HeBxk9NxNLYT6bBmBqaG/zqhk0cwPYtYtkrwef9mpCUMMJQa+wFImMnJmeBhhgr/YI/hma1FvMuSWgisOYGngtu3w9m+8Z/OptNpsqBKzYcwAOgFzc1ujwyleprdNmG2xsDyKFG5uS7oMBmZrd1u226w60+D+i6HXszzVxYFjRWaprzg1vIsfpnxG7PHrI/Dfh6AOtVyqxDOR3tGeYLtR0piqwtN/rNSI0CCA6mCkM6YdHQvicvuXd7p9TxDkt7cJaxAinUaPKB7eAcuAec/OsfWLAuqgzTYnQQMA96yeYL01m2cfTSBlg9ht5q+H38Mul5lgBBhyPvasj9TveIrbjHw9qtEqP2jpUH+bJWGtonmbJ/OuY+spYdZr6XDyd7juOtsJMY8c6VUgZPyGGaaeuSuQ44ajAGHnWP9azs9j5gjOpTnh0zfMwXThHc0QVt+ZI09gZq+okmcbvK4Z4d65+DxgWouhcPaw29P03wMPnXNzQvhvsaYAfAPEbTPPtsn474B9ZrXcmw92LOV6xqHUp5yjY39/kNcFu+CZuelMjp88Bn8iN65IL3eqk+HVDjRzlQeZhu/uT2KlgW3Xzq4QX284/eeax2ttWXm5/hgolVttzQ12Dz84YK7olB5PNwA61oGaC28Hjt7PrWPYW5VGecB0MFo4cl84+nmtkdmUbr5EANVVVtKqs4P0qJrNeTNXfNsBSnsGlLe3iqSGE6MWm+3ompeseO9c7ZKCM2aF5kA9mnWQU1mH+I2gixnojzGgftUD1EcwNxSHobcZ3XbTQVVzXMHD2IM5JBwDNtRzW4jUqnTCzf61l7hZiMumERBVHJbTPOKO8msPigM5m5AcxRJOvqpeJh7NEDenufiKI9McTR6ijnnkqEjPNLBw0AbFI5lsFIj4OAns/ggOTmmutBI1s1Ii/86ZgMt5+ItDGgMaWppqhXMngStox5tKIcPRd9mpcYG8FIlAjMMxSDYpAaahaFyB/h/8A6UQQ8PdqcCmxmWM/EbVbFIYOfhppXcyM3lsUQ4f7tSKnRc/rQ5uMmrA2cFEkh3qwZnkq+eBqwjM5jn2IJkbCPqIhnTeMEucP+pBMMM1F1m/6lQtfWWHvEB1IDE/EVUZqOZ4EgsAezdV1TXsFmmTUuNMwNBsOfwQc/n7xURzNnWgCYpGg5lwxSB7AN6pwn7EB1L8RxBfBM/eIucWaOqtownoNBnmgNV7M9v2bhqQEnNAjPYqyT/wZkrB49irXgc61QZCGgehmB+GCeDXA1LZe/EUoVgAamsngnnmW+tMqhYRpjmYZubFZhMbMPaLNZ4EpQPKRbTJm3TVScn501JeNRM/ggnaymx+hVQKwjPYIJRhsUUzUg3tigmgaM96aM96TxpkzVLGy9uwUvW2KsDrUtk9itAutNPbCUsDUKR7VIswjGmjRoFe0gUv1bmHibEDIN6oGp2s3/Um0GY0BsD1Fds01iTCdim31garM/wANSI0l/PT1E8LeTZMNy3r5PBvAGJmAAvSsb7zAjy/xwzXAuM0B+iX0bh9+8F2XhvVXKxakR9/sDBbWyXCAzBGYb01h8VioRmCHIEBoMPirixHkh1/8Sp5LmBq4e6FAeADLUwX0vWctvjLylg8qzMAPTUgHj/DNXcIWGfwTRg4fQijA48rGNGw7N65v8bVQvRjD3ZqEZ79NvrXS6bTdbBt/oNXTNpUf8BPsT0sFbFtuSS5t9taj1Voq1kxmIHhsN4AoviGS5r9m7bYSIbLeknZNEOe0bbaOMGAAriGeALj22wzMaguQ+xSwZwWgeZzLUUd6G37tNmFUYYCojwOK1ejbFWyQcDYi1fzOB71IjVIM1XzGXMTcVYEkwd60S3bMkFNCSBrGM1Xb7RPetXPdmg2pgwmjBUMasH7xxTvWvzopYZGpQPOKnCZ+8UgJPzqUrDI00YZkmjk/OlrIGXsAFVUwzeaNsFYTN4GqzeB+Gs7HkrjrcM46lLof2fM8/envXD6PeE61ZBtxXACQHnDNfoBdXD22LwjmFZinn+MB4GC8WcY+HVKol0HBt+XsDr1l6pgrhvO5qxIB+XVWdnRhGVFJn1WY/rvytZ0+9WTNSYo7psOMAf1goU+ZEmSNfUAPoBWtwdnTPr6FcVh6nRorUuDF1gwwNUUb2WolMkuY6ahavmHm6bigvKzjRm5PiP54I3qbrNHoMYB51UAKJaVSrzRuU1vPQ61pbABilTJEupOYYeAAYdayjMydDaCmxJZxuaewMw8i9J3hRLftjhZTAp3spTO94+t41Y5VXqDRplchNzpzLzVR9z3h9a7HDtX7GU2FZRxQClVQ8zM+sF55hw5z0r7RtgcmPFMDPAOxdjufjxbF20uExTYMyNVYvsTeDYBqxd0TgPblE4525FqVZZ9RG9rBzewAMPnXRfTGu2nXCFNsq2KrAkuxd72EkD0QwXmy7b2uC9uXbu27g8HDBloNgLK1Km0c5huQZ0k2vPmg0vDqyZcy8GgB8JMeL7Yw7816Q4acQqNaTV18OYjEmq1Otvfc4jIbHvrXleBMlUFp1yh1KTGdfDQM889iVt3Jc9pVT15Q6wbNQwMNY965x3Cq8NOKHBmhnUn5wHQqoZnJjwTwOMZ9gKEzZN48N7Iqd21GlctHqIYGEiZm9gf5FlJnGO9b2p0eh3VVTqUvmQMJBnh/B0LuvpGvzjs2iRHH/Cfh7w/QjFd+gNw6tWsUuXxDrlNZmSIsnCM8fZ869J8QreoF+V5rmn3meSDCNIj7DPPrz/QuWehzMptH4N8rOpTzOvJM2TANhh512PnLcPxMzD6wVXC4tcVK2KNW7VO2ak5DnxzAAM3etV9sc9aVIaobbAPR4uxnA+gFQz63Q4bRynJesAfg78FFjcSKMH9pP/sVHiEbTaxAYue9Y9Vqrbz0en72cAzDNK7bGpt2yOet+CdNqvZIAMA/OpdBrcSe07UgqUY4755559C5T6QPG+sUe33YNsAcZo9hy/fPB8nkXX4XFvPHHusUqTdsK3INpPM1tjMJktk9jxh5F5xvONVaDUuVqsF6NIM+g/J512O0rn9cVSXXL1rkamyIW9k5G8z+dYS6qkxedbKsnOBk4r2wz71N8LJio0+XMdBiU4ZtH2ZqwehsMtajC0VHoNcv+qOuQWA5Rs8OYAP8ipJMOdTavIo86KbJsHhvXHfC3NAD8aKf85e362c9hp6TPY5D7q57fZo96QU196YDbejh35pnk2zqh6DnhMH+tY4emqJM9W1mFKfbzjsPZvBn2LvfDqfUZnEtqJbEWMDVUDAAzwCSC4IdNlVWstUqC3m8+eC9AcNPV1H4oWO43OjaMWTgZ57OhdcCw4kej3Lqt78jY9K1pdOweqTwHsAz8i5ZxRptcol6/ZWquAEJjB57BnrXqC3oH28vziG/blckwKnRJOvGMHsAM9/WuG8VAO8DaqtVOZDuV8+V5cA2PB5/kBZsXJJNKYqs0KVSnwBo/fGa6befPfZyn2jE3x2IwGf5Fj+JfDd/hdTqYEu7ecqErPWiR+hn/Oq2wLh5aRWGKrOeNoIwaOf581jY1FEjMVuFCseCwYQn3gB4z7wWcuSiBbdeqFGiPvPBnmGa67wZpUS5LNrtSiSwhu29m94vQ99a5Vc9SYmVuXUuRZjSzePMA6AXLbaLa26rez4UUemxK5GORszid7xqw4es2rWLaNu54J+taQGDIH1qt4S2xXLzuPXpsWNpQs9adIPYymarTbgpU+oXAZgZ85oAYH1qRlb8encS7vNih0p4JAM+xd611rg/fnGKz+EtQ9R8OZ8+nhn9+zAAZD6O9SIzNOt6ygqtOo0P19KMw5g2cz3/ADrbVvhpcFw8KqfYluXpTYEeKyco8HvGeM+sF3cPMi3nG3qPKr0ymUqdXIx+u5nNPR+8POrO+eG/IXbULn9VSXrcinhrRz2Zqv8AWtv29CiPwQeCvQj8aXhhgefQCmvRuKl5x6PBpucyFVDMIccz3gexbbGVqV81W4Z+g+48FPDsA+tOw3qzXqjLYBuTMMGdgdeAKPUuHtx29dH2VnNshNzAAwPMDM10CxrYrFvXQHNOBGdMDzM/fB5Fx21ZeBXqUdrzmNfRkU7Yy1hnsVedS5O0gpz7e+ozOaP6OxWd/wBEiUSrxKxTQ/m+qGb4fWsfPqs6sVSI3OP7vrb1cJS40N96QHNeC151f2rTYLN6yG5UQJkeKyb2BqqrE8IFGgsP9ZyTw+hHArD7NRl1FjoOGYKBcUSfFmSpD8SmnAAzz5vPeH0KRW2XJ4GbBm86/wCc+tYSHXnzDlGNma2FEOqnNhPxIjxx6cecx5kMwBUL2w4bEyfXn3w2U6HgyHzqdTYdDrFErtOg9HLa8n+/BV9KqTEZipv0pwMAeN97DrWUpt1SwM6lB5mGb5ger9Cy6V7Xs9n1JDCDTXNGXCZzDvwzVZaVVfMZESUefW8B/OjOq0p6VNrk4/vD4YfQpz1E5ZhqoxZxyTMM3s+wEwpBCNENo37xnMyYhsnnh2H2LNUq4alRGpDFDl8g1KwzA957FdwD9dwqgxK3h0Mn86z52rVdU9RvDAwwnYbD+RXAgyXn6aYTmPagri57enased52QP8AWpFSZg1V0HDc5CRh42r0GfnV9c9t+reHce44lzSZMiLswzzAw8mHYtotzucvUGcy0cvT2Kwt62/XfrBtiucm7FZPWDDrU6pQLgqVGauCdUY0ZrDBmCB78POigRn3rUmsMaPhvdZngf610xYojjP8rg+YPHCDsDDYpEaMw81Hgty+WkH7549h/ImjebhkbBhmYIoxsSXdecGYJsa47MlUeREiSqrGk67IPTAiPZ8sHzq4n8OrgP1fTqGxJh1A9kk/6LJDDrzVDSo1Ned0KHRgN0wzMAPDMPIngvyuYabfM8uHsY/OHgyvRnAoNZ13WDiYOsSeVMPnV29YFx0fVlS4gfdezP230Ks9a1wOX9RwXg5U8z2Z7FozvCq3bVDcnVE9I9+iGwANYrM02lZypFN0I0OIxhgHnzVxGgNg16tbiQ4xxXswzD2wedZSpSalVfYGb0h97Q3n1ro1pcNJfr47VrntQ0DOQZ5gyB9GayESq17CPNgxYMCNmeZmfeB9ihUq6vVXOhLBmS0eAMyw3gyAK948Xa/GrNT4VwacEalUgAYjHHDfn51q5lh2zc/AJ65joZwKhTobD0B0D3vB+Ofn+jrWnUMlDtWgBcAViXUmfH8czkHgYAfRgtBXrAlViiVC5oNZhvR6czsBreb2C5zJmRa8xL5GLMk+rmQBl4/IrGg2xUo0LlX65PhnNA5TIMn4LwLkUwIRjjP6krBk/IZqbAmU2G1VZz8GScg2dCBLZewCN85+dS7qgfaG46ZTbZY5ya+z42/vVhM4aXNSqRLqUo4Bx4QZvHHk5gALroU7IXBTZrVVlBJZkAGcaWG8zP51d1jidOqtLhUCo1Iwjg9rGbQb5MrPPAzBZRmHnRpFcjvvPBFeBl4D6961desCpWrFo9VrkX7lW2QlRj6DD6wXo2FYuqDecg/5Q+ZoJxWQisxI57MAWJpVVwfj0246jVfsZzO86fh1h0Hv6+s/1qwjQLgv+jXBfDH3lqgsg/UjDrDM8AVm9atStvgtb9z1g4dSj3Cb/IQc8Dhhn7b5+j+BGDob3Hu1arw5rHDlh+thSpoerwjyzOT4H9qAzPYfyKv4A8B6PxasO86zOr8mHULXZ16aETeZ4AZ7w/IuU8N6b9p7jj2jEpRzKhWMIsMwz8E+88A6/wDYXqif6MHE3gI0N1cMq/JrATQ9UVuO0GZ8k9sM8Pzq1OG8JeHtKvDhlXeMVzPzHqPbUwIr1OiBg88BgG/6N6rLSDkOdrFKN4I78zlYbMs9kkF74t70e7Vs+/JDlnUplm17voMUJkfRwAH4WGB4ed7vXP6rwWsC0rGNi58JMSn1uo1RmmSJIMvGBnhoBhvP6FhcLcPrfDS9To1tVwJVKqVPrZnKZPmQDkD6N+Zr0B6M3pURLYdrFq8Ta5Dkvc/92qzOtg9s6MAA8z/Qudekbb0SsQqVTrVs6g0T1QyxUJjNMZ++xgPZvBcZok+VZ7rTlKqoViIEnmngNkA0cOhcw/R6T6WnD07yp9j2y5GuGoVcwCGcGezoh5+ZM/Y/xrsUasc5H1+bjPf/AE72YfrX5e0eq07j9eH2RuPk+YOGcqBWaZSghvAfWesHk6F7Y4LXO3Tbap9h8jnHokblWajBDOE9h/gNdMDtvrhhmPIfNzoZNcM9FeThZFdb/wDyl/kW+rEzOh1hzU/6Nlf/AKg1zH0aXtGwKg5+PJA/4F2ReC/1u4GYYLj/ABsgMYU+pMbMPBw866czJ1kFYprE+nOxH2wMD7DBacPs4c3NwvMQdOol3rqR2HR2YpwWGMADoWCuSgv0SYG/NqUGYHgv0vrez2Plc3Cjw3sOhxTWak/DMNNxUoG4CLWM11Y/LHf4dAtu5NaaHNGuhxqxBMfbrgkaY/GPUbcV3Sq8+Duo4/0L5vN6bv8AW9l12TUmADPUTQTAeaWHh3O5JdBg94GpsasYSDYcPvXBfrXDs7ttHJ08NRc8mHyECoRdQzB/fv7FtpJvyYXgHvNc/uQ3IebEtze+Gxbep+/+qPY/YoTmbU0FSAPDccXQOFHCiLxIgS6jWalJhx2HtAAj9Z/nXaLb4IcNbe/6D58z63Zz2sa7+b3+HgvDjjheYutAALsvFHga/AkeuLHpzz0Q89aIABgzgC5Eya24fZjkc3TaJn8FIB5WZ0R+Y00/FiGeYdB7MFTvA5GkGw51gr/Pg613GmMHHBsHERyVRMvYKPZ9Ycr1GkTpTeBxZJxfrwQ61w9DbMfD61EADZd3qQZtgGu50BvzTzLzEmPp80z0Z9aIwujqUGZTY7D7ABIY6JAdaT0+cDuu4/m19Czuspsae48wbD7mxRhe2tgG3PinLYl+L0YAhZrFwUHDw9YA7zWPZnuRndQFKOty5LWm22f61j87bxzPQdEvCnVWAEpg/kMPIas2ZmsvPlt3XLoJ+JvBdgoNbiVKAMph/rXx/c9Z9Lh9lqwkuGfiOKwjPNh7xZxmTvU0J+AL5WG22reNufFNh/oNefOKPCjk5sJuzqU88b5mDwB+TD/Ou0RqrsX155t481t63s3wWc3C8iT4c6lTXabUohsyGOsDTIPb16guGzLcuSpBWaxSgmSADDf0YKqOw7VgCfKUCMH0Avr/AMwcHxuGUeG49NDw13CzLbo4UlrNjeoVBtJsKkGuC3AQGIwabDeC5Ob3Oxfretg7GjQWfDbBHyzb2eDaaADA0QL5vY78IEyA492J+m0SIHj6G9SzSB5emEgGcFLBV4TFICSgsOtAYIAeDFGZtmgr5kOI97QFVSYDAHsVhMNzVVJMkuA6glhgHQpTMlvFZ/mHEPPn+J/+hQLp6SBn4jihSXomHht7/Oqw5jh96a1nD715sTfWUoOhxU8wzeLUNTkybLC0FObLiinsJXTzIezVO8zgeo4q2BwAz1Ey8exC9JAA01Bekr1D691pkwQ8w2vmfwQIwQmiM1HM9iBrPAk6ye9R8/gkDzaC7hvAGCtWargOnmsqEzAk6E9BoHqln3pMz8D9oqLnM/eJ0JPzqFtKzPzD2isGZKyrMz/m1FaszAwBBesm2pwGqFmYAKWzJ95qILsEsPiq8KltRBMzNQFJjfu1VSYb+fQrgzySVbFUEZ9kFYAyEmLpuJPBmKZOY5GHTBXsZKvQMJXhtqkPYtNVXnDI3DWcmHmatCPrIDebx1NRRTNR8jRCbrJcz86rzM0zrOf1ILjnMx09RGExU+bnnR5uAguDqW1NHM/eKpyNGDx5KxZ8y4adCSYKsB5PA84guGZisIczf7RZzWc/qTrMxwFA1fM5pHJ92qRmfmHtFICT86lawz+CEzxUTmfnTMmY3gqBnVWw9o4hCq5rPvSczNM85h0OINMc/wB4nWakB+0WP5w/OjCS/wCdENkcls00bzazXrKR+Ii9auf1JgabmdylMzFkvWh/iKQzUs+9FtXzyjvPOGCqQmb+tSAk7kQM3nEkjMEs/ggPD4ogNDn8EWHxRZ4DzJEYZgaj4YJ4DNBEeZTWGCmmCDlsyVsjQAeCd3qQyyAdiTwBh4aKRwewR62aZPrQh1q0uP8ApFU195+n1UPZMBoH/ArP0e6lrU12lTn/ABcM/wDGr3jNA9ZWVIwbzNgwNcs4LVvkLjaY1MOaAwW0Kd+eNMqW8zgemajgCxSaw+KLD4qXh8UjDYgr3ox4qEcbMFoNHNAcBs108PMwuFJDo7Zu+IrD1VFDsUsIzYezSw/eK9mEcI3/ADtqWyygUhnYSjZhdwzbAANWQT2/xFnea+CkA8uZolT5JvEmgNwEexAZgoWMJJ54KyhvGCgM9afB5QL4JO1I3s1Sc5gXWnQnt+8RS2AM/aJmezFZY3qOFSAPeKFVaq2cc280FJUpjAZg24s68ZmpskHEDMM3laVfkaezcBXQUf8AET3qSLh7NNinCpP/AIilBPf86M6IAJo4bgKBcRp+fvFYMzMx9os+yGCsI/WmhexvOpagw+hTgWQZNQjA8/ZqzNlBh8UGSvaHer1GP7HctzHfrLxJxRgVWMGpcdc1pveDXYvdd1c8dLNiKZhn5DXn/idw9s6BYEiuXG3nUPPIe3op40ns8zIzBSAtvAc6q5yzXnW94UW36+myJcShxnuV61LqrLFbuNqlVmmshCA83sPIi3j+NG3gwmqlDbjO6euB/Qa9JXJw3pvEKzaJ9nKjTY0iKyuO1u26VZnN0q4GweqAexdaPYfnXR0snPAkgypATGzBWcOlczSDluWrMmNd8hpa2lW9EuG3uRse3DmOh1yHg6PrToU55Ao9Vrc9p+nQTejxTB953sAF2bjlfME6XR7OguYQmI2utrw9tV+icKKtarGi9UDZzwA15kr1NrMaqOtyoknZ8idKU37Q1KNAOlRKk8EQ/dAoDJmCiMh+IpfYoa7FhrGCsDBiH4bh4I7So79Yuan05gwzN7vWwvyw7me4g/ZlujGyGeAGDKtjtj2QnPTGvV0F6Zn0AALcSeGlcC2vtAER4He+I9sMF6FPgJRrbsumTrZnSWXWAB94zAM3j+tah7gnZ1StwK5UTrcybKADePmcwP8AInSfS8m2lZMqZTQudypRmXYT2+IfWvRvGC4aVdtn0JilMPScIwG88HQz+dbClcB7KoNGuBxigMyQlRjMDN7PetLwZsawK3YYMVW340l3PB4MFHSje3RuEsmmzOG1EYpTjJgDPQB54f74LVer5f4a5fGsbhXb1U5FiqvQHT6I70/RVheEayrAt+Xc9wcTa9SoUUMz+85mf0bEptDfcnv6A/OGap63ACYfK8qB/QGC8z8GePfE3ivxDm2xb99nGokXokToYGeHZns+tekGWeKkPNg6NbFV/e8zo5/kUlwqbh4V2zXooQcJMOQf9LaewwXkLiLQb84i8SHbHptcM6VS/AjS9Hs+deta9e3Ea3mnZdc4V006fFDOS7EqW8AXlKNxau7+e5XDLhtUvV8548Jx+Nyx/PgtYtjDFUS24NEvL7HPmFbrrBmEmRUT8FkPOCz/AB7tijUqoteqqrGkyJXto7PQyueViM56+kPzpck5GsesGfetXaUbh5GlOv1xiYYaOwwe6Fd29FTeJ1cti3I9DocVmHIY/pwdapY1VnT33pdSc5yQ+eZvSN5ortCDDqRsRFSwJOEoG1jbWIXr0mVzAcp0I3pOj7zAzBOxowPMSH3H9EGFTw2TqU93U34dH0L57Z1LhLRKG876/luSebB7QBnW2Grvipb0Gg1mRKpVKwzAHngaDrNYQAYpUOPrtnGA895rqcC/6lAozVGqrDJtYYZvbzeBZNV96JFebpV9TYlVYMKfcrOiyb2wDMOxaj0qOKlDpVUj0e0qAdSrtLD9oDeEMNmz51jLPo4GcSDTa4dKp8qTnGlme9nz4LofELgtTY1pVuLbFRN52VAOVPnSDz5kw37FtFuS4eHKxJl16U7UZ0t6TIfPMzNaXh1YFSuG3K3XIMsGTpfYfervh1wcfuG1LgrNwc5Gdp3gwwDZma6VwHs+pW9Zdy0O9IjMCO/nnIN7eGzBd53IvoqcS7YtKjXLVb0fjaUow8LreM/oWM4l3UfEi8pd1RbZ5OO+ABG0Q6wDz+dY8KlBZq8ijaAM0oDwMw63g+tdTmVunSY9HtynVWNSoQYfeNHeH51x8pEOfw/WsBh1tiXUoGv16Lxs5qbasNgK5Cgv85JZPPoNdZrdH9ayIkGsV+NJpTIYBIZP9pVTwfs/1lxkj0qW+cOODL5syA9zs2HmsfwvaCYXPXr1p9iRKiEbmv2PW6MPnXobg/Sp0yqVjhzfDEZmbFjHgcTr+sDXCrGth+lceKq3WX5MkLaZffOQYZ4GAbDNdAhyYgWvC4hv1V6NVajXuSmSADrA10xwjKXhwft+57ortSrMvlrcoMblYciPsCfKVhwHuqhwLcp9KCj+tarSJh4fedHlgPv+ddL432xBn3fY/CuhxdalNnzrzOe8/rXMbt4LOWldsu3AnHTYXM69HlgzsMPIZ9hrs6xxe9odwfynXK+wwb2Ex+VgyGwAWztu9otHCO5VaaEmO+GYSMN4KRx+hhZPEGaxbjnJx6pTWDk9+Z96ylyX5Sq9RKVQ4tNOHCpYYZmftjXBzcK4toOJ1BwalVmC+EmjuM80cQ2djJ+cFw2mnzNUj5+ybPM1so1/1g6dU7cfl9GyMfZgH/2ayXISgdAADrUQJt5g/Pnx323wOP0MgHYpFVDR0m9TDZgq+MzycoAlbFHZAHo7py3z5gOvvW4jgbEOVgB5q2k3PKehO0qJ4Md/DNlrZmiZ+z9vW5HueJcfM1vPAI+Gxn6wVKZvnKOc+eZmZnmqwCnhsOWwZs+fA1u6IbB8G3fWTn/SWbP0ZrDnMzinBcb2H1n3q1599m2WqUe+PsewTAGToBD0G39Y+81uIBsBw+iU2pVL+cKo8eiGG/D61z+GGfL/ADngpHjhMiStf9hPNkPIuUdwtuNYd7cpTX4kam1hkOVeBo8NbBZefXnIYQrVnP4R2zzMz68+wFh850mrBLg7DN7PPyILkPWrPN9Hf1msulcW1FeCjnFkOToJyWuszAOhc85YzwYYz0s8+tKZJfzkBFN7B/rTUZ5gGtBzPPs3rp4eEu1hy1Ve/wCHUcgnMAOzsAEEl7b4EQMPI6o71YnQ4cilMP4c11mHkUc5772Gvv2LpiEE9pm7qSuv5DQzHmwMG2OjBOw3m2c322A1s80pOD2bhh+RetsLumyajAox3PTnMOVe0HvkNWFHZiBS5tVqT7MOPmGGZ7zM+xZyZM5OjO0aJLN4Kj472fnVhUgwhx2DPMzDMzRGFnP55l1oKbKeZjvsnhgs9QY05mbz2hnHDvWjt6pQYzQP6jxugBshnvAAQ1XAIZymzA/oDBciEiwHn6VdFPqQQecjsPG+cfDMzBeiOFdz8OTol10PiVdsm0qrW5LBxqm7G3vRQDo+gDXL6rwif4Y8PKVxNuO55ITX5IfzdEwzADDP+DvXJJN21l6QcqI+ZtH3yD1l0DqF+UGUFL5S2KrDqtPpxnKOpgeByTNUUO8L4h2Cdqy6q96q0c2Y59AfQq+qz2K2w03zwQABkOZMNmsagsw7gZ8RiDUpMIDAzkaObPWoHSOD9Eiyb0Ch+vIzzUqmvmbTQGefyfWnnqrRqDUZr9VtyqyaEx4MDl5+Bxj+fYu8WZdt6W9ApVxweHMypU+bAOUYBAjAcY8PxuwNi8b3hckq5LmqtzTnNE6pMfe0Q6AXmGR1msRHq5Va4xUZkCWH3qG69vPP51NmTACjPa9R1udDxthhmeaz9KkxGXznS9bZvB4AzwNdbh8Y7DrdqQqVeOscgGQCZzcDNkzA+vZgqa7cyOpRWaNHnUpgwPfrZ+xeXTeKNz8VKDYNucPOJTkOpNTWWKvGkOgHNMh2M5/79axVjQIN2yKnQ36kzRI9UB+VDZANn0fnVPPZuCvNQp0s5MlqKzyvMPPZgAB2B8gLxk9JeijcNs1Wzbz4HsBGgXBf9HfYOrOvHgZgB6IaOHzn3rn/ABUs++7YdsrhJc9VhyaZCh4RpcQ8wMM8z2bDDDMM81nbGrDdq6x8jGkumHgmYb2T84LoRvP3nVrXvy470Cm1AJMqVqywA2Tf2Bhh2BsBevHUPQw4P2rUghcTTlQAuCl1g2YABJBk3ooAYHmGeZnma9lvG4HzrxZwQuSx+D9crdx3PRqb9s6vM/5POtPazLLBnvPDsXtu26k5eFr0y45dHOmy5oZvRzDDA1utTz6xOhkEFiXDjTZoGEYJB+2P6O9ecuNPrGiXlTH5z8y6uJdXZwojTMblosP9+B/xr0RMsNt7jT9tKq3rR4VthFgZ7wZlZ78P1qh4wWxBh2RI4jTpWjWLNjPyqbLM/fvYBvDvQeSg4Sv1W36q5cd+HA4oA8Zz2Zz3gmx2B8+ezA/nXHKlaVH9aU+lRajUox+rQOT92M8JoZ5roHFG7XOM19NVWhyo1HkAzFosyPLe3nvwzA/JvV9clE4jcJaNHs64HLbjVBhk5UOcEwJjxgZ7ADyHs8ix5kczGMhY/IWoZzq2zcZxs3pEdkzAw8n/ANguwcBOP1/2HfR0C/IJvWlW6kGDzUPRBmU9sZ7NgbFxK6rnuM4FJg3pWHoBvsm/APBkzZA+vf1n+tWtjVu3LnqTth1y7pPq+ovQjjVCQ9vPlT6P41x+Crenbz9If7MT67YdVptSCuvm+cY5Z6NPBgw6APvwD/Gt9wBPk+Hjrf74P/1ILhPFFmDxCkUwLto0yM7ZTJsvRM97zB4YSgPv6A/Wu7cCoESBw0BuCxota3Rmu/hW6KzUv+MlahPbMPENZdl5hWwQH3gzDoV4DM828jwVFPhhMa0H29i0EmA+AZqveZ+Q108PJ1sbjbk9eo/ITDc1Nh9Cr4DPOTWoLftXzwBdrk2TBuGL0b/OpVt8LqHR8H9AzkeczX2I/iTgv1nH3rPrnO8jBgvSXfOyGxWD3DG+I0IJ3qoD/dA9v/QvR0ClRNmxWvqeIY+G2ub+YNPHpvKjNNrNHdB+q056MHzqQdYiPO5tuLud7cPX7hY0OaZjfOe9ckurhLUbehc3Tn+f84B1rbh9zh5FXw4Wdq1IJgaGfQtLMsOgVt2PLqrGtyueAH0b/OuX2x61ojrVZqtKkswvPguxhWGDjtOB3hmuP3P/APG3Eu7PCDbzGg22yGZ5ngyAZn/vgthDqrEx3AFy/nN2otLQZmGD6+b+904dIZkgA6b7nhLj/FGyaAzNpM6gMRoGZmDwMs9ea3frhh73iobq0JkUM288DzBXw2YZKBQYgHqYKz9Q0oyAzpoGfnwSZNO8z+I4tO+zrZTiEESgxaf6moYG7Ke8Z3DYywG814VqV/1w49YqNNqQMx36q/FAImwMDPrXt2scbLVpV2hZz/3wzDPYGYLwpP8AVt7U2oUaj0aNAkRZMqpxgj7DPB/2J/QB7PoTx7lvOt266rw9Q8PnW3H86ryAfIAGYbN687wL25NrXflzzmxf7OezBbisUq6qxRjl1I/vEpnCSz8jHQGHnXOarPpVKmcrSmAjYAAPYd5rt8e7bh+Z6GpXFe0To0TTckngGB7MN62VvVJivUtmpU3e0+Ga8n1Kj1WYUeJRnMzOmhKCOHW8Zngup8GbznWBHm2jcFO5N3RN43Te2Au3h9zbG/WdreZfAc8Eyy84BqqoN4MXDg2xsDDMwNXGjgu+HOMDyWgt64XKPKaA3PCz3rOM9akMvaMgD0+g1HNww8i3cKPcLExoJR7PkWghzAkhsXFId2xY0oGIlN8LPrM966HAmAA6jji/Pez6dxb6XDzNqBqWGosvDuGDng2/mavY0zWazXBcO/8Aetg3oDZwUdmTgST0nMFIhG9g+poTGzVJJM9XUSZk/vEUvtZv+pBrACr+c2e0VZMrGia8wL56e2ohzwzWMn3ObLu9xNfa0A9u28AeddPjhtltuAngnWak351kmaw28IG24pATMFzXw/he2wCe3+IpDM9jzrGBVQ/EUhmqt/iJhTXSXMwWWqrzbLpqT64z94qmqva2awDRzEs81VZ/BSGZOC1SsHtgqDrYJHJ1lWyTcz9ogteeQc8qU5LiZOT86sXD1S/4iVbJn5qC9J/eKvkvIDkyTz1FEOZ+8Ud541FN5WhN5ncj5n51Wazn9SMHkE/Wc/qR5/BRQNSGUCPoTJmpb2moTyANZIJKjmeSSCXzXwToSVXgaMEFwzJ/eKwjTFQsqWybiDRhM/eK4hvNmCx4POKwhzDAg8RFtL0Fm4ngebzVSFS86MHtYvDUC4B5SAebUFkDMdikcs5jvUCzDTNpQqky2bXXgkyZgq+pTOverGfrGwVmpHWrqpPaxKqMMyVoRNFLk2806l3qg1ywAKaNlvPYCmmGxR9iANFRzZU4HgBRLhksQLcqFR1wZdYZPRM/P2IGgZzQGGK4ZTeHvGmsYVGdUjgc115mAHh2bF3OHD5CA1E1ze0Awzd6zVgM/gngNR+9EB7FCEjP4I8/gomfwSOS2z4iCcDzgIgn4e0cVUdS/dqI9MzdNWLh6sfhuKK9UjP3ipzecQaxmgs+Z+dLWb/qVYbyDmHFcJW3MNoOa+Cr9ZLWQTeZPzpayhayWsqFgElxSGZO/wBoqrWToPKcC9Zk7/aKzjG4HtHFmo0nf1q1jT8MFCl3rGadz+CiAeYp4HgBQJrKsGYxqqZmBmrVmY3ggT0NAEYwJSDktpjmG0D+HxS2KEcxRDmb+tSLbu00jAFBZk/vFOA9iBl5nfsUfRVg9p4KEZqoNolYgMVWg1CC5vM2di8q2w96nu2I5/ZZOC9agfX84YLyvfNN9VXhUG9PDfmC2ix6g1ucitTtTPXDNNKmsypc/asJzP2AYK21kSmgjDTUQHnE8BqA7h8UCeA0sQVwxMoTRGBqOYH7RdQPD4o88BUc5LgKOclxa4YbWGRoNZz+pROa+C+6zf8AUnTZtPCefeae59vzqp5ljvcTRyW1HSvuaONJ1jVqzvWUhvOBgtBAezwXNfC2i042XFHwwVkBt6SivABrFsiG82qyS9v8RWb0YM1HOG2iVVkCkMyWwSOmt5dCdCjuH7NQhICY2j5zNB6q0Q8dwA/vTwRmywyJuHKjAAdZ6wKNrFnmmjBZ17ipwvjTfVz96QwkH2LS015ittc3Snwkx+wwNebDOHxUiMCr59w25R5GhWa5Ahn5HXsDTB8S+GsMPHvGlB/3kFpvwNKyeCsGXlzRnj3wkObyP2tZD5zDYtxSq3Q68xzVHrMOS18hqBbcw2gOS2q83tEvETL09v8AEUizeMPeLOXJaVuXVC9XVym8zH8meCnc5mlzXwRTOUThRZVvNGxQKNotH1hrLj8Dh6xJ4mn/AMmQZj++zBeg+cwUSZVZVKhyJ1Nb1pDAZgGfWg/JyyWaVSvvbESZUtmGASTDBdutjgbYF20aJWKjSjPmjwNl08z/ADrnNscB+JMN12XF5lnAM2TZDPW/jXob0crPu1l065c0rWjh4Glgvow51xZ/o8UOmu+rqqw8zQg/oOeYLqVNsyzo1LOgUO3IcCIYYfd2QzBXYBz7oMe670pNbp1Nwg0OnHUpfkA8F72qi3iKTR6lwN4tHQ7gcM4VRM8DPoMPkWM9Kt6h1Ws0em2w2D0vowjh1n2f512X08LSrkmmwb41M3QAwCOHQz3ryLcl2waxChVKKwYVCF1mfYtkpFw8FrqsmgtV+6tGHr54R+s8/wDc1jD2LtFYrz/EvhvHrl1VyNDOCyejBa6APzmmuA/CJ/iQ7IrlOfCZIpBgZx5GzMN6x6Wu0SxuGNYCjUfivQ3wntQntd6IGwww2L1LQbSp3E6s0y+6cAHHhe2Mzw3rO29bf2Jq1VoEumyYEKr7GWXgDDP5P1rccKKbVbbqTtD5oHqZKPNnDsXb44fDj7XSJINgQYN4YB0KDPPAQWm+zc6SXYCZe4dTpLoOOPngHZmsswrwp4cnWB2C57F9nA15yva5+KNh3W7w24SBrc744PaOZs/7716tetXlg7A+hU8Cz4lNmu1GC2DMh88zPNRiHR4cfsbgVQ7Vw4hcVLgk3DW3/HwkZ4MmqW/PR+ubjHXna/cdx8nSorP3NprDYtbxg42WHYzseDOfOpTWHv2eP0fnUSHxmqN+WbIlW5SodEp+if3uW9sD+BZdLOGF9BuyajQb1vN9/fH0ThA95zzXark45XPRObs62bVkyahCzA50s9GKyHnNeTeHvpG3/wAN5U2lUrkJkJ+pG/JlyGcze+hXvpLcYLqvOBRKVSjCHHq7PjR44YZhs2Gfes78Nts/XuKl/wBStWuxKrc5yWjePY0Z4Bn5F2rh1xCrPCvgFT7VodlTGZtRMzOc9gcYAPvNeMgok4/DbqUbmA/o+e9bM+MfFQIvq6XKB4AZ0Wd+GALHa0S87YoECPU6lTr4ZrFQlPaxgbOGZrn9KrFSps8JUR/B0OhXEkHzga8pvCQfWs/hg7qLHbZOmTH57/NSnPFNR4AOHWQ094KOb2s0rKgsuPMOy2MOYY6M16gp8xz2bbiOg1J+HMkONsA8eHQaqjDebmoG81oLeo86Y0DkH9tlHog13msLhaVz8uvV6POlngyAYaQdC00+vP150HHGHmdAMN6p5lvSrVddYrGDMhjrDNWtvG3JgHO0zBp8MAzBY3C4sEO5PVrrTESVslPAx9C79aV/sWTFOw+IcSY9SpT2HPRD3n8i8jwJnqqbI8AHsM2N6so1eqtSnwqU5LeMHpIADRnmALaIRb9RpPCuwGbQiQYreFP9trHvMwXhHjrWLjufirIti36k9UqfTjzjMx+gA+fzr2DxFO4A4RnbFuN61QYgMM7OvDzgvON223B4RWg7WaHUYbztUjGBvSAzkm//AJF2YceFJwKCmvXDWKleNKo50eV9ymBI2GyYL0BxU4dcMjsP1VSrYjQOawejVGPGDwQ+teOqJRD9R89O2OvmZ/WC9e+j9W7to9Barlx3UdetqUz+yBvOGe/eubrdDkvGO3r1t6yKI25Ro32aigAes473WfYthwNB+fw5rtZrjAMu1cAiw8Nh7A2fQrrj9Srg40x3otKP1baVrs86c4A2SX/oXGuGNjRI0qz6idxyQqEqpYAy8ezDsVxwPXc+CFqwYdpXXTa4+zGqpzNGqyD69AF55hw65efEsKHBfM6Uw8/KgNBsAAD5F7As95ipMVu2bjobLMiaZxQqEcOv61S0Th0xD41tOQWM4VEt7B4/nMP9hXhC14OWAxMpIXxd0qS9cr+w3s8DAAzwBai/7eYuG33XGHM/V3jmB9GCtaCEtmO1z0RmMYBgbLXQsVxIjVm0qlcHElhvnKPKZAJLTT2D2fnwW0LcE9LewKzMi0q46Vb+cKnRgANHr34Z5+fDBclonDSuVjhjLYqsUIYHnKhnhrPGfZs7F6H9ZW5W/Un8odKuSlR6uZs037+eB9GxR+LtKuCpWLcs5iqnR6VRI3KwGc97wB1ma26exk8sQ+Fdx3C0f2ZiA9HAAA5Dz2BmfesoEyqwJp02cefq4zDDyGtbb1+TrVtAKVEckyZE0DMzA8OW+hc/jScNXm38zMzMzPrM1zXw4VFriTPOpRXddiNmHQYdiakxqlAdaYcY9uCrI0k42eHetHJrbFbIJU6IAHhhgBrjtbOT4fJunrsABpQ5IcxqaebXkVnWGXDBpxs8MAwVFhg7p6ibWn4YDqL7DmMA7pymweDDDA0UbQeE2/IyahGH3dpzvWgm0oNEJbnfrbE8ZmrKybJvS/8AmHLYo7xxIp4PSz6M1rq3wcu61Yvrmc/An08PbSIh4aJ/OBrOzHll41YfprQA2HWGf51Uz68xUosRzkdGQAYPOgeYPKwtiHLrE1qL1h2H51n63R51Hnu02pN4SGDwMFPiAPOdbDGZmYYKIYPstab5hn8iWDgKQyyBhqGuhSJmbwAf5Efjh1qbR6VOrFXaodKY1pco/BDPD+NHW6VLo8h2nVFjRlsdYZ54L1JrDRzcc6zDYCZOS+ZeO5mfeo/MvmQOGe8Ej1DPPP8AOgs4EN+pSNNtzAADMzPsVnMZgs0iQ+w3m8B4a3nVTSqkcDmPOYYIz1DhBpubEUkUeZBZhYOS/vGeeGC1dHmUaZHagzoIAYM6GqB73j+dc8ANF1auyWadMq7TFcrHq2EeZnI8mCzuB3qHcls8TrGj2HWIOdQtcM2ZGt91AP7UeeB631rl950Th6dUBy0q/JqtPA8Jk442iAPrc0rhLaVycPqrc385U2EDJvxmgDB6Yfe/v9syfkBVljcRadQYDVvwbAps+j8sBzzlhgb0oOsMzXo5oFqy6q+6/R3IxxwDP2wbAXReG/FGXQZ8eVVYsmZHhSQCZBjgBsvMYYHs86s72o9h0fgVQuIdKg006xcNeffN6OeZxov9lMPyKvCw36w7NvSpOPU3XZCUEGJsPDzrnEe/OM1cvma7bNAb+zdpMSTMKeznGN4Nntv9RbvgJYfDK9rZrdKnQY0C66C8bzJzj1s2D6Aw7/8AbXL7qsmhwJVV9XVzWBgwNk2gw61E4P3a/YF9BdsWqxgl05kzCPIPAJ/yLq4R12t+jlX4edRuf1VRKJUTN6Z6uAMGTDvwXNbGs/hfP4hzeHMupTJgVRnCj1Az0QOb1sgYB0Z9GarL54u3xf8AdtTuqdz9K9Ys8qzEiHsZirKHSv5uiVh/A49Rz0TA/Iau7c73ne3oc3bc7FVu6dUbbgVWLSjZgUykw8wew85/5wXhx6lXPbFDhBWKHMgU+VJ2VEwzZewPfgvTfof+kCxYEWt0O/6zWDtw8GIEjRN4GX9/z966xflHse6vRcqVs0ps6lLpzL50dqohomcrX6wDNB4yeh+qq5CYzz1wM/yYJmxmWKxXPVzESTM5o+Vwjhma31h8K/X1v1OgXNLClXbFNh6MZhrSQYw2Bh3/AJFuvRF9Gmv1i7ZtVr9Yk02jgb7GEfNmVMMPIfYsekegPRatiwL84c3NQ7xtij+uKQYUh6QcDRMAw2GHzr0Q9Jg29a82WFVprwUiM+DLPOBgZgGf+ReOuCcm57Yql60CzrgpX2MYrGDwVYweN4w9sYH1rJVitsWxXrltyxLqr1YhRWQClR5fXGB7PWDD6Ni8XD1LavFG3+ITtMrH24o8DQZYOTBB7DB8+xck9Nu6pd5xYViWrccAKFS5IVCqyI5nJeN/5MPJ9fkXm28AqVmSqhY9DqTNYM4zEqZLOMDJsh5FHqV58TYFLolNgv8AMnvfhgGDxvBnvzBc/Y6GUsy3n7trh0qmng0APvSZBhvjAHeadrdb5CrxIluUpmqyLekv60t5nWB7sA8+/Baig3hSnqpNrFfqTJx7lo79Lr0iJA5b1afQCr+HVyWzbz4WzOCTq16MYBLePBnQ37wD8ive3Phiq3dVwVVqO/cGcnkmQYZ1Q2YfIas7StU63dDzcuDJjMwoD7zx4Z6J4bFHqVBpVNgVujTq4frCEecPM9jwHvVwzJrMCQ1WZc6MzIYZY2GeYSQMOgwWJb3nQeFFsTPRmdqtx8nWKhBoPrCNOD9qjGAAehn5A2bFt/R4gMPcGIT7+905j/8AjXhLhFxaue0vtg3OYmeqrhppwpjUdn7tmfQYB9AYL2rwN4hWPSuH1EoEu540aXNmP8mEsDjcyfkDNdPDY6AdHbCVqOZgtBGDBpRZh7tNSIExgxBjU3rs2QIwyU6BTYLzWD7HX3qCYeLqKQbzgBqAoLg8FNYo7uw881ICpMfiKvOTrNKnek6Lumo2jDaw5gK9gPZiuewKq5qhqOLVQKlu09RIW0D0ZuSKy9eoNSNh31bKNl09gGHYrv10H9SsI0xiS0BqUYcUr1BuqBH16k+c8+88MFT8yYe0XoB5mDJDeGa5fxCt5iAbUuCYHnnmB7F0d3YRClZPPBWsOS5G6HFn42YYaisIzzmSiltHGnyw9maZnzH3ndR9wzNR2TTzwZhmpWjgaKSGEV2VqBgwGZprPRPxFYSaIxW6SbDj+cd8MDw8i82PLXEi4aHRKpT6lBiw6k6xUjmng97EOj9a4vefEKmhxIZqsG1cJErx3me/DDZn8/8AqL2lP9G/hXPjy2AtUNWUGzN7YB+deReP3DqVQapCqtDocOHyrOjUjgmBg8eYYGH5DXowN1cRfXBwWIJ5gfNGZ9+/DPMP8ix9w8oc2I5EMDdfZ34Gut1ILDO3jnepqVSnWGXwOOyHjPA90GHfsw6FwyiQ/wCdIVOc2HngfyLDY6Nw3tKq1Kg3LX4NSks12lwweBkN+cXfnh863U+g/Zhqnt3dR5/rOoshNOJI2YB2bzR8KLts6xs4tZOSzNlPGEzRZ1tEAPYB/nXr30irVpUnhpE4ozmAknBZYfDMN+g9hmu/0/ZjeLc3NDy1Srk+xjsuI3TeZm1GYGAH4OiB7Ni6QHPfaaW4340IMMDDoA/8686XCc657mqFDs6hvZ0sDOSYPAYdn6F3iz9C2KNCtV+UcmaAZvAG88z61+h4ed8q+FowDej98pVNolcqs1qDTqPMOQ/0BgunQPR+fmBnUq5MhveTls/8617kdPlykD3eGrONXqlGaBth9TbzsO4LGnn6yY/mwzAGZ3Y8qINM1H58ci4/ttbbFVbkzwcqJ7/Ot8zdURmRynZ51yKlQ6zPkA3Q6VPmd+bLOYYKf61bZ2SzwdXJfrw6uHm8u1wK3EeLT1N6sOcbNcMpVYl8+GD66RDnv6XWvlex6mP+HXHM0B71HeZcAOhHGezNTdYMcFx4dKq1sD3rOXJJ26bDm9X1VA9UNNUkmHzOea6YQ5/UpNRCUDb7bxgfkBLk4LzBuS25OZ9BmfQtbMPlg0MMzPzqteZbeimbgb19Lhtx4VNErFVgHoPvg8HZ8i2capNyQ1HDXMpMkwkeHmCtaVXnGfbudCez6c3+tHDzfj97oZmg5lwFTxrkgycPHVhrAfQvlXwu+LS+fc94aA5n7xQjzySXOtYBvQPHgogPYJHJz94ge1sFBkzOreo5yfnUJ55WDek/vEByVHMwQGexEFrKO89uR4fFRDjOGfhoEZ5JrEFI5Yw7EgjOIA0UOjgrDlnDFLk3PeNoIOfwRgalnG+RLlfigZ1lHM1O5NI4aCsw+KkMxjUsIe5TWYzYAgrAjOIwgHmrgIbakBGbQU4Qz8ikBGwVnofIo5hgSCIAKWCjmkBmgsA6FZUcG3nQ1FWsntwVhSnm2S1DcRbcRmabGjgq+ZPYA8NRV709t4dMFEwzUA5lV6221STJJn1q15MMdQ1XzIyClknvUEzVnJZcyUfl21aEQEsPipGIJo0DTx4KDrKRI6FCmSYlNiuzpz4Mx2Os1QOZPi02E7OnP6IAqGHDl3JNarlxsYRw3xqdnsD5zUeNDl3DUgrlcYwjsb4Ec/8AGavjk4IJDz2buomXntqhPSc+hBzJ4K0EcnAkua+Cjmmj2IJByXAFRzmOGmje2+Io5mCCVzPzoTebUHWS1kEg3m0JvfhqJrINberEvWR6yg5/BLWRKdrJayg6yWfwVIWOs3/UvmsoQGjA1ImgaMDUQDToGqWnAe9SmTcUVlTWQU2LBmS+A6eopASXD9oahAiA1AtmZOBe0U1mfgqIHlKB5QpcHUtqH1lmql578NNZuZK0rg5LZprW3qIybimxobkk9iCQy84CnMvHgmQpT6neqpYD7NQrANY8E0Z7E6EZz3iRs7ESg6zn9S4lxsh4VePOb2Zhgu4Gzga5lxvpudGCpNub2DAFcB3ghPck0GXBfcz0DzW+XGeCdYbh152K/wBEpdre0APxDXQF0J0DULmWz9mmjkuLT53LtcaydB5tUgPH+Ilz+j7xX8xuF8BgiMAMFnwqvzozqThh7ROnB3QamHome9QTmI5JmZKIbP4a7+GP0OO7PHP2e0TPOH51E0X1OhxmPeK8COc9z8RHGefN3xHFanDg6XhgqwIxnI02AWf6Er6GfhBvNSuc0Q1NRVoQ6kAbAUQ4dSMt7ayuHRFtAFwufjp0Lh/EWfCA/wCRGFNnfhrm6oX3W0fr5v8AEVDXuLViWwH8+XBGD+6PW/wKgvZ6dRLZlztN7Z2AC8XVi4ZdSqkuW+Hi54LG+FcW9cXD6Vdj0f8A0PSpNY/JgudXD6YF4zPDo1Kh00PIYZ/5156erEt73igvTHA9ouO22GwuTipflyTObrl2zJPyAZh/nVUF4XBlpt3HUgDycyaogze9nh+c8E0bOBdYfkNc2Gy1M4P9OnHmrKBflZoMfQodz1KM0fYDyyR6h9biQM5+8AFGF4Xc+tv1WUcudOkyXT73ns0geDyB+dULwcts1AP6E1zJs+zNMDQGAPe7UuBclZokj7jVZ8b+6PBZZmsGBajnQrYIFfrEfm6bSpMkPOAKB6Nsn0rp1NhtUqv07nD/ALQb29bOseknnSeatxtkJHkMAXjKHMfB82HPah1q4CY/h4ZoO6yfSlv82jAGIzLvYYB0fwKke9J/i8H/AE5G/wDLAuOzHpTMc3zcV7Qab6ypwPuBmg9QcIvSHO6mmoN3GyE0/eh/qLtebfv3wD61+f8AGkxbbm896jeM2OjA0V4cUaxdUAILb8mHHDszW0Wh7AOj9jDAB9CGZPg23Rpeo+DMgAz864ZbHpIXHfNr1OuUPRA4TJ5gbPQaqvQ5vOVed812LccvnKg+Znm9vwX2+1zukcK7/rnFcqnRwfjQGoR4PH3muy0qjxKDAM4rfih77vNed7qhzvRs4kVC9fU0mfa9QAz+79eZ/rVxY3FqXe3/ACxuC5o0alZ4RonMgAYLBUOSelXQePdbo0io3jd1BjW5FPXZg097efznn3rzlb3Drk7aOuXi4cCEf7MYb810vjTfNSmcS5Fv1KuBMoRmAcwHQled7UO8KvbVnUZ83qfCAAM3gwZe/wB8EeuWW9YFSvB/lbcqUY8zwBkzwNeqPRj9EvihZ90BeNx1mNSoQYAcRrN45PXsNVXo32fOs/jw0FYpTIU+o56JnvA9i91GaDl91cN6NXrja5pv9lPMNiuIdqwaa/rsMdCuADOsuvq2wbME/IjwD3eIrhk2zFUh9XhrH3/x7sfhppRXwk1ipmGfKROz6zUxbPDosyAw80evsa865FcMC8arVHaVb7etE98fYYfWriwOItwcZooVVi1To9HzwwN7PWXQGaO5DHDmjZa+RdMXho/PzjfYFGqXFCFQGPubWAc+fXmaXFGlVWe/QuFdAbCNT5RgGABmvTFmcK+fv+474uCnMyd+EAz+RaiNb1AZuM7nqUXWkdi6Ppc7xPxL4CfYanR4NGl8zNfPA2tm8/P8isLno8GNZFr3xVWAZl0swivAAbz+derq9Ag1642pYMctHAMM3lX3bwctl6zZsSgUf72AZg86eeH5FjvwPz+uqvW/Urym1mIYMg/hgAM9aqoDL8+vabG/MF6wrfAHh5c/C+sVkKUdNqELecsOt41yKfZlx1K1aVSrfoB02PC/aa29sZz+tfI5odsW5ZXgcptX9TvhsNY+pA/GfNg1o7nDWaCW5WQnywDA5DL2YLOctKmeI4eeClsjsvYRXWPP3p6lPPxpQADmDXeg0fw0caM+boMN7NQ1aAVWG2zKNxhs13PhFw0Yv+qG5UqkFNdpEAJUY4h781hIFpRK3WYlNCsxsJQZn8i7hbfDGVaXDSt1WouZzajJ9VwHsMAMD7/oXlrc3mXg2dSqcU241YqBvclzDrOz6wVrdVwwZ4US0XIJwHYrO9ozzzXOrttKdYdUClTnwe1wzB1kNhqQByrhqPrXXADAABkzWOGTOTIxhNkeH741cWNSv+V9CqtRbwpXPhrOrqdgWBEvCbLtmsPyYxgzrvOxz2GC6HwQ4S0O265WGKk5zkf3LLoHsMF9L1eH8pvmelTe1psSuN/sRxgwPzgub3naVKuG/wBqsv0eM9SoobAMNmfnWthg/Dig2w+Z/Waaqp1KTTZDEVxnmDDADMOhfY+J8vveYvSBgBJvl2nRZcbmz5VgIkcPOu6/yb/YC1Gvsc48cgKbhMju9DwedVUbh1aNEualVWsv85VZRgbzzx4b11uTMoz0hptx/MzPAAT4+E73m6+fSZbuqm0yw3LVmW9bTzwBMqEgMMwDswwWaC+bOs/jS0/FtiZcNKixsI0eCzmcY/OAfkWz9I037k4u2ZYDFK1qfrBKeZZ78z3q6rE+3LV9Ju2n2GwjBKhhCM2gwZA9+Cx5eHxDp4r26tSrhfmTPWTEGTDjzQA8JAYGH5EUysche8SpRGNhxsHnc+9ausUqlHPPXPRP+Bc0uS54J1JpulcszQmD+81uWeAH8jPnV+IYulhcgH79OnXmAHw31j7MvPhtfk2RTbVuBmTLi9bJhgZh5wVnUmdF3TBX49biO5T8VKPTb2oMdvPCbTpgSox4frBeauOt4cXrtvKJY9Di+qqPUWc2QMA3gHWZr0VVYz8zSw9weYfWuW3tbd1XPdYPsfdnYUMwZkYAZ7+tbfNH4Zd7xpPjVW26lNpz8rWwzY1QDYf0KkDrwbXaOLVHr7NI9QRGM49OezmSD6zNcvh2rUjEJzjYaR9C/Ocj6nGrQPANNzrVxQTiG/pzmDNpzydig1uMDNSMGGMGuxPUGA+9UmsHwZaDe8Z+Rctw6Vrc8DlvHgtno+QzzwWc3menp711CpT6NVSj6cQAaAMOjDNZevU2JGA50QMDXNsZ95l+MWg+2YGYIXnsI/0ApFVZ0ZDTeoZm+zmqmYfhaDnQ4eC6YHqvh7Pqtq8KLaiUqpGy0+B1Dweveuh1uz7jqVv+rrutH13SqvDB7mIJ4PAHk/IqG5LVnW9b1Hg+rZIU+LTWA5jRzD/feux0HiFat20RpiDUmYciiMhnzB4Zh5AXyr/yv2fD63DyepDyZc/DF+z6Q7enDmuSalEpf7TT3QADhh51x+57hrl1SgqNZlm8YBgGzoBejrqmMU2wLzqLmx1yGbDP1mvNMbTOmg33rv4bfB/iXrR6t4hT6zYe73/Opxn+Hgo8xkAMNNBmYCYaf6wW75jZ8NJ9ZevKn02jP02M7v8AvE4Nir+KMluTdsjQiRvA8A5EczMJJ+dZR4HFYYZ05oFYTNKlmxrsMZqIBt+861IhxnJ8xqKb+iBnvMz2KdSrefn1Q4ubJtBvzz60FZ3rqvBCxrHvmueo70rPJ81gENrWwOSfkzVVRLeYjDLuZtgMIL2vvPBeneD9q1yBa4P3Bb9EOnnMCpsu8sBnD37/ANCDy7xp4dMcKOKFYs6Cck6ZFPOA9IPMzYP/AHNBbFKpsOEDlzWq9J50weprx/0n5PkWl9LeZTpnHCt+pqyE+OxoABtPawBszw/jSuS5LjsOfa9GuO1Y0z1QDFQjBIewPeGaMEXijfPEaf6s4eXrVYz0K1z+4R2g/ZgMOj9CxJ4HHGIDn3cPch0K4vm52Lzrc245dGjRps0882f996zoPN6QNo1izpxogNajDa2cDjNdsCPEo0RyN6vw5V7mGdYwDvWRgBrA98gZ4B1qv6PEWdwp1u87kp1V5uXQ3GeXfA4ptAzgZnsw2LmRs5yHW/IeCkAecLXQBh1+dRH6Ac+ZOntAc49YGA0MPk8i77edL4Qs8IPtrSYJzIhzIpw6ZLzwjPbNcPn+tcH5PMVsGbwnSbBhWA+w8ceFUglRng6GfkNaJuGwsZmgVi16nw5pUSpUGRcVViyoD04M2Y0UOvM/k/jU3i1Y13WG67wulS59yUqjveuqVOPMM4rwHnsVVcPEWpcRZ9bfqUEwOgs500KYG9kO/P5Ohem5PpD+jbXrDtqDxtvGNcNwMQ2M3repTxmABgeDwHh+fsXRFuRyHh1wxuM6DctzxX5MmsWUzCm0cJDOEkDzzeYwP3K9rV6q1K3raNy3GIzMiVvN3R6Mw3nsWcpV/wDAviLfjt48NuIdKeObTQpFYokvNmTJY36J4H14b9gJPcSKxclwzbYsrhlPmR4T2i9U6mfJxvyZ5mkKh4CrdSptn3lUPV0GqstBPCbAdeMwPlevP581va9clscTrwkMV+4JNtuvydaHXPbBG2dD3Rs+deq+KPBk78ptwacGHJkSobEKmtGAAAb95/IfzryDxdsO8bMN243GIFEhU7Qp5xIMzmQeM8+/vS+FanuThjd1sQ6rXGIPrWjxQw9eR/YzGM8APNNSeJdAjR6PBsejyXpDAeNOlsgzJZDyZh1qvs/jfxNtWkBb9KnQ2aVgYSWnYAPA9n9fQqelUF+6pVb9WsAYMMnNzA9H8i+bfC2i19M4kUaHHaocuDGPNl9iY61G34H51oqlflGeteFBp3C6mhSoTLEJms1ED1j2YZmuZU2jy68xNciU4J8h9nfn1n/rpU2sSoGdAnRD5fsiSD9j9H1qNr2arEBg3TguGEwn8NGQ0ex4zV1MCs02luv1WCYOgDAMmBgfyJSY1DmMakWjchIhGD5+Qw84edS4D06q2rHpT9K1td432ZfXhv37ERf63TeHT141iw7ju5xuBPplBNhk+YA8zzwDAMPrXVeA9Ko/GPh5U6Ncc6S87BqWFVjsngdKP3M0P8B/QuO22zc1k0aRQPtBTToVUNh8waez3+dewPR+4dUOzIp3rTZX3urwwYkg0eAGYH1rbj4UN7ap3jDpfqO+H2Z9QhexqccPBnxew/kPzq7B4wdzTJz2A2ZgqqZWG2QPTXfHB5tDURqk4Z+IatmXsxXMY12sBNBg3PzraszGDMA1OsFfT1o2lzDPV8DoUJ7zqw2GoU8w92sWyNrGHQrCNWHwHT+RVWeZKWyGaCygBOe/ZetbiGeiOCxrM9uHhoZgatY1YcNrrUiRVbqqMOYbEWCYAx1m8ueXPXpdVkG2+5n5Frql4zBt+dZyTSogeIYLyBn2XjMd6s4xqM8zh7NCy9gS2Q0bLykcztWJue9qVasBp+U54r54Ms9GZ/X2Lkt1ccr1en0py0okaNT9YwnuyHgMPo+TvUXC3QOMd4QaUES36xVZNBCo7AnYZsn8hrH3JxC4sevjDh5zMaPFhxWDN5kDZAMOs1l+Itz2/wATgo7D9AuEJZyQA5bwHosh0ZgHQtazcNj2ZFO0pV6zAhPgAGEsDkm8B7M8/wBC51u8GEqvW5IOuVKTDdis4Pcv1n5z+deZ+Ot+Wk9QaJ9g36VPmvvetD5gMAMIuwMw859C73RGYtjesPUc45nrd5gIZy3sDAA9yGfX3rw+9wZnT6jdtpVmvxmahSJmuzIMM2cHt/XnsNBHtXijb9V+0EquWX619bmBs8uyBnDf+s9//wBguTzIEufeU6C2xouuPbGc+hVNVprlEkco5vM88EdHN+fVwfcqIRpDnvXuheWLilRn5lSlucjJkx4XttE8F7DrfpUWVefCJq2LgiSYbRw9AI7IZhGBlkw8/fmvH8/15PfNiKxhzp5yeU7z+ddKsm2LfCnWVOqMQK3HlVUDmRAD2zAHvBIRaw4CWxdr1UdpVGOYy7cL3NMvGGZmAd69zcnaPBCm0SDLtgJk2UG+WDObxn3mZrH2rdXo33ne9CuCx84Hq546WYABsxejPo/WqzipxvtiZfgVK1Z0CvRzD1FAAHsAZfPrzM/0Lp72OHq2lBTpMKPOib2pQZsnhhmCuwwxXP7DuqgV6m+rbfczaogMQj+Q8OhbBl5ZdlujA69QaVcMB2m1WKEmO+BgYGuaUT0frKpTTrFVYeqWZ5hmZhgHk611jPYoT0nA1r9T3AqJSqdQabHo1DiBDhMdDIKvrFn2xUmJER+lMm1KDA8ADNW0OS28pGxZfUYcJZ4CeqpoP0es62HunQ/z5oZ8Z+iSMH2Dw84Au4BGAzzUSZQYMxo2JYZga2+r8mHJIcnMc1LzzJS7tsZ+26HLqVAfN4GAzOP34fIueQLnnPe4k/oWjJvXg1mlCCNuQQKkckA01OBl/wDDWSkd6Aw81p4AspWKPUox6jDGbS3cPeWDiUkMFtHMOY1JnWgG3oBqgGxY16M5r8rgYO+Q10KvUdvmDfiuGBfIsuzQeZkG+/mZgfevscPN+h83mhWvU1+Bg/KbAwWgptbiaQN9HyKPMZcktOtuMHsVDg5GS8c8G+t0BmY28HhuJGazsB4wdDTV0yDhh4i+VzRh2cPMN6S2Cj818FIOGZoAo7+fWuPDZXvGmsjVqdK3e0Q+p/ivRT4mgwc8i0oUfenjprfkQZQANSAZVwdKDL2aRw2A92giBDz9oj5A/wAP/wDSnQMFOjPN96CsCA4Hu1IZh+701ZgbBpHpgi0TkG/eKOcBv3alvSd6j85uRAORTT0bBWHMNqDVY0uTH1Ij4B9a1EUMMkes3/Uoj1NnAGbE5k/OgADDrczV4RtN59sOtL1wx7s1XnvQgDYe7W3TCN2tQqTCRz4pn1qnM1HN7AFjcLhoHpkTS9oq96e37Ns1Tm84o5vHkoNtGExxS2Z+BLKMz3wBF6y+Che22ZrH7xTwrDH4i516y+CQV5sOtB0L1wx+Ih9ZMH3rBeu2/OmvtC350G1mTIgBqAaqnqk2HtHFlJNbM/eJr1k491pgXB1JzNM88qo3s0B6+GasXBzANYrNy8KzzT4fzFSHtFlk94PH51Z1KpNwKXLla7IOgyeGZrOWlclsUS2qexOrgA6YZmABnvQbU5LiHWzWUk8QrYD2fOPfQzgoT3EVg/2GhyTDzvHgoG1M0gNc/k8Rarl4FKjAHkM81WPX5dR+zlRox+dmMg6gZqObywVt3nXJ9R5GqzuZA+gzDeC1ZzG1aBPSXMlH1k09J3qPn8FaUvWb/qSz+Cj5/BLP4Io7n8Ek1n8EWfwRJJJ7w8/ZoO9UABGnQBLD4oBBOgCQAnQBe7BgpYAo4HvUgDUCayCkACaZe92pAPNqA6jz+CADzJS2Wc/doI+9TYzeYKWyyAdilBpoIvIOGlye/wBmtBDjAYInofu9NBRMw/8An01o6VDwED01IgUoNj+grIGWw92gENMFIOSxpe03qIZqE89goEh7eSBkN6ic4pEZ4D94rDz0MDDxFzridQZ0+25fKMZ4BmulayiSYDFViyIL/Q+yeauB5Ftue/TazHfDZvwXoWTAnaEd/D24ZrzvcMb1bcs5hvoYk5gC9UWxMg16zYU79yutix4STBSAmZp2pUpxl0zDoUdmMvoRcYcSRrZph7rTrIb1YMwGPaGCncEKfE1IDUDrVxosB2AmXozB+zBZ7MWrD60OfwRmyeSAwxXTF/oZDDeszfN+QbAhnLnYaodhgtAbxs+zXlrjHJnVK9ZfNSwew6MOxYflrwugB6TmAeJQ2T/P/sKIHpV3ODunEo1NZa8mGa4kFHfmFgw+AfWpH2Scwz9cxl8u/Z5nZ0uu3P6SFfrYBoHGgf3ILHvcWrqk9d61X6GjXOnobm/70yf51XnGlrm+jmX0ukSeKl6SWjA7mk/WmoHEi6oDuv8Aap41zczMDUc3tFPo5nrtsn0h7/epztKOssyWjDDN5neuTvHL5h1zX6zz6FVBJfMtilRpL8mQEVhh550/IGax3zNTwbOtXVsQLOqU0wuepGyAdAAHWrCiWMdVmtc3Bk7PkNdDmWfblEpBv8jJZkMfuc16M5Gh8HY0gHG4Mx7D9yanT6xSqlnQ7ZseADR9Dzwb1mY1bfkn4jcb/wAEFfUGjwarP151wM01oO8z60GcrfDGdRGtSdODVP3IBmaz8mxrqZwfChzzjn3hGNelbVsnhy9WQrFcvyBUjY6AOZgtnVePHA+iTTgzrnZN0O9pkzBB4sk23OjbAgzD/wC7GnnrDuZmHz06lPMtfOvXEzi7wSrz+dNu6Mbvk5Y1h+JFw/acAiW5Q2ZjPe8AYYIPMgUSqzD8CmyTD6F1C2OIsGyaR6qnRTP5APoWwo9pOHRNeXOehuh7lroXPKxw0uq56ycG34mtn35oLazJnCu87gOlXHBBnmjz1gVlxj4LUqxqWFftydM7/BM/oWj4LejTVbVuVquX/BAwDsA9gLtfFSt2H9nHaHKwmAfQAPYKqgeHqDMqtVnhBixAkn867NSrPvF6K0wxb+8/wehUVn02gWTVzqrkHnI+eebzy9EU3jlwnOEctyXGgPfg5pcDhgcB7xZlBKqNv1UGg79ZO03hdblbqnq3lJOfefMrcXV6W9DhunTqHbhzGT96b3+wuW0S5LmuGbIrlvsTI2Z57OxeYHkemz5wNZwZ0lnPr0jwzWtsm4bq4e1SPeNuP6NQDvWceZl8wbegAYeRNTJk7Y3n0KmrqvEj0meLfE6l+p65WfuWGBtYBvVVwTn06myqhErNN5mPK6zw6OtY96S+FG136UYM/wBow2KFAkv6GbD5s5+RGDS34zSgrbrlKbkg15HuxXDNt1y86DCrFDghhTvbPB2edU9sBKN+FcFZptSqtKM8AwDYZ/WusVU36PUafUX7VuGiUozA5MTR8Ew7Ni6IsewLYs+LcNg2VOlvvBIp0NgwMO9dTjPawLkNpekJwZqVJiQWLnZgAwABhLAwNauTr3C0bESv/wA1H/YXt5/6ivYM+IVKpU2bSoMGTW6gD29qCGYB+dZ+t8eLLt6stUO8WAo818M9J6eBnh+RBflVg8GeFVVrNvxDOoPhosmHWb5n1ryFfPo8XAzwid4xXc/JmV2rva+ASczADwWI9S3b6WnAu2KJUH2Lm5moAz4MdoMzM1wKwOOV28RbauOd6qprzsXPPNnxl58tLgnxRvaY1Bodjzw1/fPbAD6zXaLY4hUPghNoNo1K3DgVCKfK1IDZ9wauEPQfoVXy/M4eVWIHLHNCfgesfX3rrVYrF41KQ7F5uGEfyAua0expVt3vHuex3AZpVeZDmWg6F1VmEwyAA22lh2lRqjyoNypeeHnSeh4Hhsw+QFIB7AUjeQVoQILPs2Mz+dFWKkxSoug2BnImhgykZ70NS058D1cbGevs6+hBT2rZ5nAC2a+x4QGb5h51509PniFTqJb8Lg7aIRgN4AOZHaPoDPZ/nWt9Jbi1c/C6lh6jnaM1tnA3TDMzBeD63Xqrds924LjnPTJr/WZqLVClgMuasenP7Ac61ZViiOUGeDDD+s1KDYarXowZ9/606dSfOU1rvmfK+x+RY4bXaOdNlxj022DwUilRjN0HDAwBS59yOPNGxp7z71KpU/Wi/egz39Cki06sfzVHp9Rpv7RmAA12GvdF1VKdxatWzLAlmdNvMIwTWWQDBno7z7F4ptifFDiDb9VnBnChSWD5fz716O4i8b6Vw69KCn3dWabMZpnq3PlzDeezZgrWznpY2ZOsnh3bVAuA4EmqhPzOREk5h0dGHYvO9EqUqHy/8zRj0Dzw7DUi57nO+btqdZncyZzZhymTePoA11D0frDYvC65FNcrgU2RS2edADZz1vkUIt0UIF6WM1Sr4uayoEOPW2QZ8I8HsDXoOTwxuqZSaffFOCNJhPw/xsMF5/408SLn4kP0yjUOjvBb9ENhmS8Ad/kXpisekJZ3A2LR+GUGlBJaCMwZ5vYaIH/jXd63t9Djvh7FVGoNfCKBy6MYNecDzXJ7wvwGa2dOpTmYMdZ/Ou6030xuHtS9YW4/Q5LODP7Q1vBebKDZ7F/3vVW2KqDNHYMzN4zwA/zrq/mto+FS1uTVa3MiVGX/AEU82T/xrW0e4arPhx6iDB8uHefnWztu2+HNKdBiuUp7mA2MmzvAARXbSrOr1r1Oq0bmWfUnjvH0bFh9PN5PmcHvbiQ/DqVWvG7pxhVeWOn0oGg3gHnAOwFS23bz9Y4GHcctsJNVCYcqHLA9/WsJxFqtV4i3K0bEWTsPlYcfDeZrvdtw2OHvB2Jb8vRgVh+GZnBM95munh5P/KL4cNvDvN+sWLCYrOfrMIwA8Ab1w30gbnfej0SnS33maYAYYAezMMF1Dh1c9uV6gyJUGmvRqhTmQ5zmD3ge/rXD6rTYN2u1O5ru1j8H7nEZPA2VHN7OEcLsfCXiXwZsmPT27m5P1qYZs1yPGPDD5/nXoU5MGpQ2qlBlBJjv9DrPQa/P+NJC2IUdhh97l5R4GB79i9K8JaDSqD/PPDysnPp74YTKTnmEY/OCvh9zZcOrSXokZo333wBoOszXPONPFSgcK6NBqs7mZhzfYx4J7zDzra3JbfrthpiCwHMNnmGq8YAvI/pJ2Zf8yad6345A5eKGDLMTMwZBd9+zjwy4eH+rnt1cSKlec919yIEOIb2uDOe9ZSsXJUTYCmxDwaDoQQ4b4CEs28AUKq6Zz3eUbwZ7F+e5H3II5LDxtNtuPPH37E6em8fKm5gB/Oq9nYSeZBw3QbBvM/IsnrRM1Js2j0PcbAR3DnMto5zYYGqfBxn2HenalPb9SQqUw5mYGb731rDCirH+kqe52cslAgRKxcdHpR9MqYAGq3mX3jBx9zoW44IQ2Kxxdt+DK9k2Zv8A5wBW98ePz5/D3bDuqlSXQpzdwZ1DW0eUkdGCl1vh7w2vmpSIrHJxq3ngciOZ5gYfIuCVL75VJcrzvGpdBuqv2lUWqrR3wCQx0awZgviXzR/3P30fwjl6YxSp9Ja2PsBw5kUb1xGqXO1JjeyBgf515VDUy016g9LG/Ptbw+tIJUUAqD8wzkvB3gHQa8yRg8XUX0vTxh+W/iv9Ob8FJ2GDad1mzPA0Jg+Z71IgT26I+64/TuZN9nAN/Qu18lNo9vQak06/OnBGa36OfepdVgUpnS9TRXgiH7F57rNVLM98Ggpz7YHC72cFbVi536qcL7qDMeEzosxw6ARgrIzMEAd13A5hz2J59CsNGJD8AKkD0hvrMA2AqR7Ued1OxTqOy/VZvqp+Xg04yZgfRhgoa7dCo+vUrXj2joYVCqSWIrJgeySBmvY3EKq06w+EFTuCmy+WmhSghAz3g/hh0H+deKeGkC47kuug0qmynngpD3rA8Nhs4dasOK9VC5L3uOuVivyZLr8zQjPdj2HYgx9h2NUr8ve37Sozf32rzAYD9a9fekV6PHCilNVXiTel4zJ9Qi0fCNHZMAjPPhsBjDr/AI15v4G3gxw64oUe+JcEJkSkZvm0Z4Gezs+ddK4i3PK9MDiNR6VatGqtBtekQH95s5hn1nmYfP2K2Lz1SrSqtbhnWYO/N4GAa/zqPW6JWbeqh0OoxcKgBgBs5+dbi87AvThXRqPUvWui7XgffZp7OyUEUOh81zypBVT5epVViZ96wfZekB7YPk86LhoKJcjln3lRK56mON6ueDPSDeYd6u+NnEW2OItxx37VtkKVHigeb2iAHJM/PgslJqrFVjm3O9qAeCfzqvjRuZd09RFpUMH3muUAEt4K2tigyqxVPV0SXy3gm8Zuh2AvUFgehyxVbXte/wCq1g+SqlHOVMiHDMwzMPk+RFPM7M9x6mx2HOhnPD86nwKrSmRCXLnaLv4WGYfnV9xgOy6qFuTrRiPRqhyZsVJqOyeBvh/9guWa2/NzoUJdeh1i7pPC+q2/YEvCIw9zVSejhhKNgz6P8ahPccr4qtgQbO9ZRjOF/N8kDhgZvB2Zn+hUnC6tzqPdUfQuA6VEPrewzDPsAwXpizL54Q170f7lqNx06z/tXS5+GcSMEaSe/wACUz8/XgtYhFw4ZG4Y8RrbapV8WxblSptdinsicsZys/x8PnXbbYpXHr+SM/SBse/7nekU6YZ1Kkzng0XooBvMA78DzBd74e8eOB/GmfbWvfZw7tokkwjNS82ecAA2Z9h9fRn1rooTOHlHochi34MaqtUEH9Z43sKeyeeZ9/zqUYeFXuP3GylX1WPsdc9bmQqoGcODUWTmGAGG8wzDZgolz168ZMKny7jo+tChSQhBEdPfJMw34B0Yb17IsOwJ1wwpHHO/34cOt1SADENpmNyzMCEB7A/P1/nXn/idb1vhf0di47xqRzZUaVUOXjxgZZpsUPo6117ZOT1iHY4c7Bi0CmwKgcPX0ZeYAHyB51gqO9UdWXBgxeZam4czHzwAwDzq6uS7adVYTVOp0tl7XZDPZn+g1kmdQ5Wgw+bLvnBfH5nRDRwzozNzOg3Bkw6U+9rYAeGB/IoNz8jUv5xlPmEhnYBgfZmlSoFcrbUsKcfMyGOsHT3n9Cj1sIsOeFDqLGBmGtmB5hgsVjpVyV+A1Hi2/Uc9c8MHQzWrtWlcSWYcu44NO5AKDMAHnpYYBm8eABh3guc8tR9D7q+b0gwPwegAPsNewLDv/wC3nDmsVKquQ4ct+leq2Y8iT1yoodYb9nYumBW3nwT9WzAupyJydMrcanBMpwb/AB8z1jA/416AsONLo/DejxYJyXmouuwHfgAHgC883bx4iXJake2KPTZLLTHKvhIePDRfZDB4MPr/AELrfAfjNSplJplpVXM6hKqsqKBgYYAGGYZr6XrXw8PK5219azs/aIzqr8kNNw1xescbHLboYN1WlPA6zrxZkgDzOMfQGYYd6p43F246VHagg/AqRsHm88AYGbHfsX1/Hv8AA+fjldzMM3VvbbntzIvhub29i5PYF2scRavLiUOC9y8JkH3nT2da6hAtt+AeowCc3s8POcMczWsyTMPESM1RU2TUtV1iUxh5FcMgZ7zXxbh3xZJ1l7AkfK/FHyahsLNs/eIwk4e8UIwMDQbwUi7CY4Yb0BgD3tFVBJwU2NJzQOnSgeDBv/hWanwzhytNbWBvVVckDmXddvrBebFIARDDBxhl76wzXLZnFejVKfU7DrnDzmYJ+CbLMbM/rWcuq5OK/P1ihuUd4DOSD8N1oMMMN+YGrg7hsd6yqhcdZrPqrwTimfvgf8gAmxX16xr1mWkFDprEyNTz/aTlh96ZYDoPBZKpW9Rp9BiPxZzxu0gOVNkO/wAh+c1K4IcYKxXjj2reN1ctT5WcWG8YADxn5DNdQepXDKgz3WJ0tl49n3gOs/zqLHJeLt/3xdtvUShzsKC1SKkw+94xgYBhgGBrg/ELi7OuGt1io0d84wVR4OZzDrMAXqrjTQbVNh2sXdKZepVXjcrzeGZgHzrxVclsRKPIdfiVLWiGZgyfXngmFqSlVIJNbjuXHKzazMzPDzp6jvHyrTenv5kz/QoJxvvEfyG8C11pRmHp9Hbcbz16rofkM1I0vBb+fq9LsRyJJ0quYGcto8AjAAbzPyBhmu4XOFDtK2anWbAqsCqwqWbEIMP6GxomZ/Wew9651dt4cPOHUN2VwntwKbUJR+q5Ms88AA+vrSmXzbAcJatTolKZea1mAksm97YzDDP5Ov8AgRFuc2xcM6M1U6NEqsmlQpQc0bzJ4dH/ANmuoWHwuYpVDo9VvysmdKmmc0INPk4GZ+cz/QuFBJnaAQYLAG7N8E8/IusWxatOqUKPWarEmHIp0yLnEB4zB4DM89n5OzzoPXHBbjk/DvJ2wKdaMalU9977s0zDwPDDrM+9eoGbhYP3i8/0T7I29QYVYnSoZzX2T0Xc/Yvn1gAdnYrCiXy+ciJBqT4My5oa0aPhvw86Ld7+0IAHWqSTW9Z3UBxYr13mHtEDNYwke0QdAjVVzMPEVrGqTmeeosKE/aBtuK1h1Xwg8RZ3CnQI0xgx1NRPcw2sSzUjD3imxqkZuh4i8F89G5xg2DDYYYGuXz7DiM1TX37DXS5NS0WtQFmZjzh5uOLeOYQo1NYZYAAbBOhAzUfmXASOfh7NxRtKXJgYNajaxlVmVWHN/YTMPkWrZnm97xStFt4N66YsclmV6JPd0Ip5umg2Hs96GwwU28LVpsaRrsBhn/jWdhw5wTQc0/u596+lxfsc3MsjAAA0cOlcyGbbYYfOokx7RkA32GtmzDbZYa0/IpSpQo7mW9tTWYe3T01KxNGALiu2sQierduoo7zLgK+ANiiSWWFjtsojPFEzJY71LOM2ZoDpTYYG33r0PAadAwSZjN6SQRtZ3Qbc3qEK+Y94vhqFJPNrf1qbMZNl8wcBRHvvLQN4bwViqM3AS5xOmAYKKbO9BKCq7Ujqp/iKqPPJRzM0Fgc9zP2iAJ6pzecSB4/xFQ0HrLAU0dV+dU4G5+InQBw1aFh612b3EPOKCEY80ei5/WmxIOS2g1lHe2KFzJgeea22lZmexQXk0dSUfnDM1Cjpp4AzFQTNw0YPYCiTphsUVWrLOtF1FIC25b0I3/IoUzhmain0KbJZcZLxFHwzFBEN5xNaykPMpnl3EA6yNl7cCLkHzPUUV7YWBollLkuquHUpEGLOOMDB4eCqJ6TOktffp0l763lKuEP57kYB171CzzFZNQeHyvhsJAzmIOaaXQaWbgCZtrza8De0ABJkwxSBmWY6/KHh58FHPYlmDpprsQd6e7FGwVEPlqvHf1MAzW4Nc9zcCUGHnXQNbMNRbQgGfwQazn9SHP4IFsySNZz+pHnsUdEqEjP4J4D3pkAUgAUh4A2JAG9OssuGpbMNAAM7EeipoRnEgjOIIRggw+KszhoOT3+zQQkQKzClOGHQi9VOf1oIQGpzLL59ilM0f92rONABnDUBOlCvhw3NUNToV2EZzBSGXogbNimgcUw61d8NvNQjxob7x6emrVmiZ+8wUiADABnmnjPD3ixaAjQAZ95mprLLZmoWsjB5wFkpqDNjSwbVe8exVJ1JwB61FOpOGfWqNpUmThmq0zceJSD3igDTBEgNnYhAHAPUUgzBIFY+czh1r6FSAPeKO8GxVj2mBLaIZ7ef+LUDkLwdNtzZKADXS+Btwmdsu0p/fh3/AELGccqa+EyJVW+jDA0uDM/CruxPkM12YS7mEmI97RR5LLABqACqTmBG94qKfPqplm3VQBryK3Ovc96eOpYe0NYfmZXNAb848O9OzDcPx23DeD5DWrJqDrDZ9D6NmpOLKQzPo71exmXwHoXqNJ0msNw2DflH4QLn9e4/cNaVmw5LkyZAe6aZS4xhXPse65R3ADDrzPBeVKbyISg57rWPM24fD0LUuP1v1WjO+rosmHI7NZcSn1Vut1c32388/wCBUlYnsc0YRG9FnsBX3D2pMQJviWzDmYedcfc6elImW9ODDYe/yK1gcJbgrbGuFx8sHkeWwmcTokBj+arcAJHeysfc/GysSWjptOpT0bPrMAWj1n7ksl+2JoRH6lGmZ97JqbAojj0XXYppvfQqI59Vk+JL8Z0+8+tOs1KsxvYbPoNcXNaos8FHtx4zbrDGYZ72V1DhXYfCesRzfctgNh4eMea4+88/7SUGCQV6VAz5F/r61j9DR2p6TwWZuE6dUrVjRo8XrPA9631N4nej9bEMINKgxmfnZjLyTJuGdJ/azzVf6y3rH6Hm3t5niXw1rboN0qsQwkdmcYFHuTiFasmlu2/OYhvOmGDJgYLxuzVW0nqlFP3ifQbbC57VnTKob4S4cZoOgGVTyZLcNjkXz6FnDr2HRKUJ6ttvdbhmadxtsqDSqbVYDrj7kMN/f1rL1Wm0pmQbEWpMn9AZqqOe/wCdIJ7/ALxxbbWm0qM5Amg+E7A1ro123BRz5uJXJmfyLnj0lwz1HFYBMcei6YGvR1ig8ZvXcz1bcdfONH857AUi5JNpMtBLs6/Hgmh36y4/Dtiq1Jrm4rYaR+c1Ckh6km6c7Y6CDrVB438RqOBsTrnCZEPvMM1j78rdRuSuc8xUsw7NE1CpUxiY1qMYGoUyYcapZ6Bn9CDrtEo+tQWotSz1T/PgqyvWfBjU039cPz7Fn2eJFZBoG9ON+haN7iFQ5NN5R+DrSPPh1qBkqPDoZzBzVk9dVctubhQ3zhtB2GCqjn60pp/QAMDz2LqsmsWxUre0JeD3kzDoWcWPIVsUqpV67YlvtzjZdN7RMw7F0i5+Al3Q71aty1X/AFrrhnzEjwVySS9LturSHIM49UzzzNTo3E7iEy+D7dzSdUOhbj0FM4FcW6VHjwZR0SfCM8OXjnv/AMC5/wCkPw9g2ZVokWhxIcZl8Azwe7/OoVN9JPjEFNOlP3GGkAZ6uiGZ/nXL7hue47wqh1G4Jbzx/OeeCNXvv0UbPqUnhlN4c3VEAJsJ71hGkNPA8eZ7w3gvUFtz23qa1FrEtlmbF8AwM8F+S/CviFdXDeqNVKh1h5mP3s5rpR8fpfOu1ysyzmEfYbxrO2D33XuFHC6sf6Vtm3njPvZwzXGqx6PFuRs3+HPFifb3/V4k8DA14hurjHetYnyH6BdVSptPf/o8eSeC6d6P3FqpcN+X5q32arHmngZme8PzraLFgzxmuq2KyfDm+wmV6mUiZmbzIYPHh359/WvXFH408EuKlGatz14dKdzzCJUWTA1jOIvB9jiQVH4h2U+ECsQvHDYGEn5DzU06Dat5wgtzibw5jQKqYYc9HZ2fwKxC4x+kU5w6dj02wLOO4cGf9JgZ4fkXhy/6rUuItwnc8Wm1LB8wB4zDMGTXrapcKL/4b8xUbHrEOsU+Kyf3SWG8A+TNZzgbUuHtYtmu8ObqcCm1WryTM8ww3/Wg7nwKk3NaVkUS2L/0QlnG14xgeYYLqv8Adqisyg/Z62Ylv1I408ITOEZ7DfgroJLYIFvSz+CaeebxTWs57txXgEAKQzMYhlqPubMFWnJnG/pt4YedJ6BTYYB61nBm+eAZ96xHhfj3Aua9r/mwabFzhVSeDOtmuf8AELhLL4Y3hEtWdUjkxJQAeto4AC/RCvUqlA01LgwYxx2D15J4dALzJxvmWxxp4g2/YduV8NWED+choNYMz7OxBweg8Lol7XMdsW449zABnmZ7FzKq0Sq0GuS6HVWNGRCeNgwz8i9ocK6lTuDlNuWzrgpQQKtTmTfZqbobJPyLxvW6xLuGuVOuVFzOQ/JMzNFFA9XMnrzm8z7FqqPQX58Nqq8oBtAeZxzPA1S0GzKrcmZsYRo4f0h7oWookOdb1NNiU5rAHeAbFzYIdArFErnFG3uetGzmaPT7XZ516XrABnh2AtV6UR0a/wDgnZV/88DNQOMELlz6zw716A9GPgtar3Cx2uTn5j32lZMHmje2MgvPte4P0CpXvWLOrlOZpXqcNaMAPH95Wi9vKjOoytHArdYo5xK5Q6icCpgeASw34fkUq3uGNz3Jf52PBY0XczPVMNjIB51X1iM/SpEi33/axTwzQes+LVt1KzPRwtoKNWOZduGSEqpSA3m8b2/Z+hR+FfA246C+fGnjTLONQmGdCN653vH5M016P1w/bNix2LunQ5kKkVIIvqx4z1sN+/5F13jldVv8WuMlK4ZAEmTQqXve5Rnv8h/o61XKyeZ+FFpP8SOOp2JQKjy1Cqkw5UlroB6KG88F7weCh23WZFo2rw5pr0egxswaA8DP+DrXmy5Lq4e8EPSbt+XTYnJx+QOK8DId59H+dbi9rwqVnyDrlo1WSFQq/XzDwPPaCjbPboHEivS7nt+2rntGlGzzTxxZkTDez8hrk9t1V+Nc1btGVTpOlK9z5zVR6PfE644HEs6HOqUl6lTTflPNAGYa/nXWrAuG46rUblOlQaaDsWTnnIDM020cctLgnLZ4nS7urLDNNj0h4NGnunv39+a5O9UvtPxauC+LnlGcK2g8EGdnZ2LQXhxjqJ3bXXzPORKPDMNgAH0Lm94V6dMo0huDhDjzcGDZAOsFlsR2btrFKnu1liXhT7o3yWewDT0k9ZrT1FhKlUpwNQqHLYBn1d0YKzhz/WptNtnsPrXNvy86Vlc8bRt+Q/5OhaPhvM+ytcNuJOks0+bDzPf1mqJ6fm16unMazSJ6fyYAcTBk/OZqt+TpdeuHiXOti1znUqc96wqknDN49gLnMyqsViGZ1yoyZkcDA3gM8zP6FnZlSOYevVX88OgA6AVTDqTE/PlfZdhrs+zmOkNYnxJM15iIxg15FlJ9NfyNxjA/kBaCt5vTwbiMHnhhsUeSy5R2jfcwOWYdGfQo32OnGGUjbFOjSeTPXY9rhhmqzBxn2nR3qXGe8IFoGXpgB4fve9CeBtA42mcM5quDo8Q4TRtzmQzDPA0FEZmZYB1rsfou0F+TdtwVx9v/AEdSjMD8hrlgPBDdB9hjxW+g+xd99HXONw+uCuG3/pSeAB+RYez/AE4Xb6X9z2YhtuW8XZhggOM4peQIDPo+c8F+W/5t/W+P+nhy/jxAl1u6rctmlMZyDpWEZrznmq/idwf+w1Nok6nOSZJzQMJ+qfsTXW+LVgVi0uI1C4hucs9T6cywBnhvZPsNZm+ZNzX5UWrZprecuo4Psh0AAf5F+w9D1v7L+U/xv2Y5vbuocNmQ3DwbgseKAYYedbq7eCb8O2aFVbSivSZDEDnalIM9hn2f51HCyaxTbtdtGcGjUAPAD+ddemXyFEss7ZqUoAdAwPRA8z/Wui/7b4kW8vvSWzYaff2SMzAw7wVlJpphb3Pf0g/Yh51o+J1Big+FYqU4PWsrDmY7XRmoUyTTjpcJx/8AooAuRp4ZkGcA8RWdB9Ww5vPVFz2AGbId5mrWHbD9SaBw2DZ+89B7DXc/Rv4S0CvSqr9u6HzlKN7APJGPozM/zoq3n2zLkrNBlVA7fbZOpVsDih3mAH5AUafMnT3YlKnP+FTsPu59YGujVut0exuM1xy7Ot9lmmUjXixgDeGfnz71zmMzOrFbdnOOa0iaZvvH86siG44V8K6lxCkTZxzjpVHpf7ZOwDr+RenqPYd48N+D1s2dw8uqHUqnW6w/rVaCH7MB79nnXny1aq/AsZ61bgrP3KqVLkqU0zJ0eWfz9u95wXtv0V7Sp1pcKIlsHXKVcM2FPffkyIJ5sgZ9gGq4UczhPEv0aZQXva7Fcv8AqVyTbhpsp+pesXuWDMMMPPgzmfQuoX5wNoF23RROKjduAdPtSgnRXqHJZz5blWDMDA+/rBQuIvCW+L24pwq4/c33KvPHT3ogbNGlh14eToBdVt64Q4XWrEse5qlJrFQhA/FgRADOVMYz2Gf+utrgi3jCz/RY/wCF+yJF8cTZfqeFJpr9XpseJGwkyTDozPPyLLM+jxccmK7OqNoyYBfZt+uw5Ec9kwAwzP8AjX6J2ZarDwfaO+IMYDYZMI0HPOLTWPIrPlrHr1NlzqMEM84D9PjS4/QAH5P996jLbb81PR7sn7YXlQgualMybaOThPkH1xs+jMOvAz2Gv0YgU2daVtM2rSmGQahG+EY2gwAAwPDZ+dcv4l1Lh5wHpNqVymvsn6keCiyQiRg5o2H+szAOvoVVTfTk4UVXVYYtWthNM8Ka09hhM+s/c/nUMlxaXDSnWNalj1KLbJvaDNR9ZNSGQzCVKMN5/oXhzjZwcf4RXhNtly6gnu1EAqEMOWPoM/4F7DjelRWabd8i3+JTdKh0o2TlQDpkbMDPD2Gt3rH02+bRk3Q76R/FCpQwhXXTZVIptOCNnohh0fP9fzq3jxjJpr9Erx0YJcOfID+yPZh0edS8KzQeYnH92CV4Lwd+CCsBZcCeB2XBnxmtENY5Gwwf354fIo70/nPAffN4O8DUQtu+FH2YOZUKa/SnqlNqjIRYDQBmZv59HyfWvVtk2rxD0qZ9quHNBptKt7fTYNQrDIM595vYZm92bM15dsOZQ6lb8ijTrVe1aWBsM1OOzhvPoBe2KPe1uWBYtvv8Q+Ffqqu1Q2IRx6czrBJPDrDyHvV9I5z6XVz8SXrfoUFy8ak9SquD7AR6TDBmMb4dHzmGGf6F56qR1Fn/AJR0OIdHpUpkKXPkB74+g2Pzr0rfNeiVviJSrqtiyqqzSqRGfZku1MMDAz6DAPkXnem1KsTzaoz8WNPokWqypr2sGAPGfnXNfN1ouHJ4EaLqyDBzkN+DOsr6AzR6UDvPS41Smyg8HRP2PzrUW9UqNG1Z1YYjHHi5sM0z2wAfnDYtbZN/2xw0h1auTrHCsR7lhnFpvgh92M9m/Po6/wCBc132Lhh7MNj1uDh1FmA6AGbJvBsM/IlxFh1KNcITosED0DA2TCNmBh84K1r3BDipbdw2vbNRoZwPtDMBmm1N3eD3nz+hdT4o8KOGtEn693Xw8FVYpp8/Bp0YwCS/h91w+TNareeodKbmTP5ypxhIfMAwADZBbCj1L7N0uaxEM2Qpcxg2ZABmYGew/wDAuqw/Rm4m2fwtj8SbqnRvWb7JgzTJYHmGfsQA+jWNZ+8IES3rXm0a46G9bDr9bN55mQeZvPsxQAwZ/P8A40HLKlTX2apyrlS5zwdcJGff9C20OfOgVmmSqdcejyoA9gz0BK7zUcKIw9TolNuM+W5oAqEAz3myBnhgarIdY9SVk9SLDkx4TP3nS7w+Rcq7ho3qPd1zhXWIM6TcM6qTGH5IR2TN7Bn6P9hSHo1YemlTbjp0wI9LqTDJy3mcAjA8GwD/AEL076IvCugWxTpfEO2bgmTwr0YAnsvADJw5WfQeC6be3DG1b5pdTp1ZpwZ1A9Y5AdYP4YAa7PHE5rhw30eJ9ApXEZqJBrL0N04xxarBkBgB7wwMD8ma9URpLZ+0XKbe4PxLYu06rSm+ZhSocViYZhgZmDGHX+QDXSMHA92a64XH9tdnGiH0YH9CQM4IKJG5kwB9wGfnNXfIPw5BsGwB/P1qNiq0VIZDb4imnA+RAEbBerV8mm+8BVTwOZGC1oA3nvbVbUozer4YIKEAzJSGQMCRgGBKzZBjBECgPYIax40KR84ImQ8XTWFtK56zJvysW5cfgx4sk+W1dmYAoFTeAP0qg1C46lOjU2QAGDISD2GYedeFbquGs3tINicf3fnNcI4dAGvUHpD8Tn7VrztHbvGMDwPaxx9jwAwfQGBh8i8uVWqzrtuFq4KdF0QN7CTI6NYz6P8AAaoQoFefo95UqJRn8NB5iUZ9gH3ruFE4uwQhzYjFqsyZHrI5UnM9YP0YLglEo9KeYdP1qzGrAZhy7wb8w7Fo6PRKiFIj1i3z5aRVGT9cd4AGfQvR22g8SIl4VGsWdfEuNGpTAc1TT6Ni4PxjOh/aM3KB6tNrYYPU72LzHnXULJsylXO6fN1yNA9V002DNoNmHZmuFHbblVuap0qCx4VOefAPkDNFqy4YH3iJEpobnwzAFteCZ0OHPp79RAwdYeflPGAZnsDDM1gq3JbptSapzbmfJZgfnDPYpFEnsUqZH9cvmzHNk/ZHvPPyKBs+IT1Oqt2+rbRnHJo8qSB6z3WcrvWXk/6LaYcb+c/k7F1KxrVi3Dw0pTlSqp0qbSJM2UbrsPPYeeGZ5rnr1Nua5JESj0qKDztUeMIzQbM8O/6NiChjH44HFc6F1jho9UpNZpVuTqj6tj5hKkyJDIGew9gfOptb4V0Ph7cEecFYjScMACIyB5g/sPPf2LQhW/ttAuCWxBBmRS9B6S70A8AH/AoQ6ldtE+zdv0LTYkvQoVVPA+ZwMwMPItNZ4Snry9cTtk3lsMOwA8i5/ZlHrF/0ahXiw+Zx4Mw9aC9JzAADoAA866Bb1mX+FzevAqv81Pva5tSA3vf6iDpWsYJcy4Ci1WsUqlOxGKlOCMc08IwH3mlzDaLWQT5f4inRp7+YOaioQmNo+fb92g2rNbz71dwKr4Qb1zdme5nsVxDmP+dFOkesnD94j61mYdS6G3Fdsz2zBeQHeWYPYq2fTXwMj2GrA3mw94mZMkDDw16Kpl4GfaKbGntmWnmqKYf3g0bMbWHUzWsC6k6D2epvBY2sU2XzBuRDzDyK4M5bPW4iZZ5k96vacMUz41SazYMMFq+tTZMAw+hVTz2i+r7toiE0wyRhGM0UN5s+tXEYAMFjtcK8GTBNT6aBx9dhzf5FdnDbwWfmG5kYZmpUqc/xEtbo+RNPamaDu00Skc4AAhCZ4uaa0W/618NnANigTnngePNxV8ySwyB6feoL0lwFAnvOGrAPPBmmtZhQZJv71XvG+eatCwemMAahPSQP2aiHrmo56iCQZoAPehBSGYx5g4qDoMmZbFYQ4D+GolGbwNWbL2CsAEB/yKPMjGB6a1cB6Dpb1IegQXvE2Gg5vK/5lBNnNbt6iRTlaYMdaP7ExHvZvmmxzk2XEIMuLpR2AB+wczNVVSsOswN+nmB+Q02MkAYqwjQAkhqKR6kqQe0ivKygQDZHxFGzBqmw22etXAGGGmnYdNYewzzV8zRIuHhgo2MJVaDFeaN9vrBZd6Bh7MF2sLYYkhp6fWszc9q+qi8RsE2OahGM/douTcD3auzgYdCaejPgHQrQpHgcAFXyW8zVq8y/+Gq95k1Y5vfJhGr2/vZBZ3nfIa1HEhlsJ8J/T3GyYGsZJ0wDYsmyQckPePqPz7jIuuOeRRw60pIOG0ahSz4Y0qddsypsTp0k44Mm8Dusez8maiUeS+9DDXWl4FfdqtU2POz/AK6pcNF2Qx5HjXlgMHMlI7dRLNsASUhow8XUWtgHnAaPzgso8eA6i3tq0dyq0aO+w3sBaxabRQToAtAFqv59CnBar/4C6Yc1wy4Mp3RWoZtV/wDAU2NavnYTZDJMw3/w1NZpss+htdAjWrBZHx3AVhGo9GZdBvXDNNvWHjUedh7BSwo878NdLCjxA9hgf0KQdK0Q1DbBQOeBb04w6EAUGWHWC6WzSmzDPBOhSon4aDm/qcz92rONRMA9nvW9ZoLB9DCR0fD3CjYxvqf5EyzSt+noGttyYeRA9G/DbVxYx8mlOMhqNtqseBxdIhxgePTcDYqeq2HO1ddh9nlz867OHmYc3Cw56gIwezJat6yYhjgE7B3yIA4aVgC19cNLzrs7ocfjhs1C/Z/+BSN6kBSn4Y4dadCNgvm8z6BRgD3iTxt57HM0YMp3l21itWGo7Ox1WZs5nsVe8y5qoDnhOBrUBtR4wVGSWAMLW0d5t5oAlIJLLcM9NjDD5Fehn3oz7LWbiiszA9m4tRJ05LBt6azjNBDnD13HsPrVIAclVRxpbzq1YUem+7cM0ycBhksFpHMhy/ivQX5NnyHwDoXHbJnvw7lhaHeeBr1bWKbFqts1OCfvmeteNKbJfh1TTYPB1h7YavuHo6TbdSed15bB4JmsV6m0Gmgw/QwP5wBWdNqVwVujNOMSwDPzp3kzNgwrDbLzvenc8w5rMueDUoEjkabJZkdiC3qrUWYv34NnnW4qtEpXJOtsMYGsvSjzjyIs5gIxsZ9Cx7jBoL/odElffs9nyLP3VxUrlYaMKbUo1NhH77DeuKXVdrlVqkthg8MDwWSkng6eaj6eY63XaxMOMHPVG4PWWfzrmUnlTN1xiI9vPrSjSeWY2PhgmWXnJkjQYDM1zX7PM6PEeEiA83Gka5weZ+Q1sKVVZUaPmxbgb1z+TMnRnTDUwUhm/LtgNBFiVg9LyGCzb4dWt6HaUmQ7UrgqoU13sjmaz9202nHVNS3KkzPjuBns7FlKVdUp6UEuq6Lxh3mC7HaVK4X8QoDrFSr70aq54cuyeAL3uY4cXkzG4Zmw57UE7SteS1rm2ukXJwQptt1sKbOckvA+eDMgzwBaiT6P1xxobstusQ5OHZo4J/kRhwq6qkD0dlhhgAMO8FkTmPh0P71q6wzSnpDuvKeAwPDAAWckhEAfDXrZHDmj6zQmbnnAEtZaiiRp2gfKWq9PAw35hvQZQHt2nqB+RNPa7IahrUQ7MqVSKROptK5PDfpHsWcqsk8uUc6wQV+sZ9fQjAwDoA0TMA3mjfb7OxaO0vs4zK07gM2fIC9wGgpTlQgacGJPzc8+9Z84dSZd09n610iqzJTMV2Xbk4whAGwDPeuYyZj7xajnWvcCzOmv6QOco8f0KEbLjJe00QUQJ84D8Oc8AeQDTrLPMmDb7mAedWLOlVWXTZGuxOeP866rArdpV6lgw/TY0mR53WVy8KbE9xLBSGTOGGDDmCx2OhvWlBhxzfprFK/Ia5vDr0ulVR19+Cy95M01Jk1XP9u/gTLz2/f1psDW68dSnnKCCzGz7GUEOtvwy1AzD58Ej0HvZgmgNgy03OhY7GtmXnEnwN8RnV84KijTwePT55RDBt4NNhvBPM0SXpa+oz+tIHHzmMTI4Snz8VMsvb81XsnmK0oRonqQ3O8F1ClemHzAafepBmcbNhwA3qJovm1rg3sVhGhuTWA02AwBQs0yb+ANgvlS1AaD51OZpsrsbM0dVprj2ANsGGHnUAKPSnDppsSw+hau1bhOmwwpRsMvAx0Z9irIcx/QBh/sTLLIM1I3A7+tB6OonHi6qDRo8pu8WXuV2erwDDZ8i9QcMb/g8SLXCsRPa54GHeC/Pyg1hilPg/KpoTGg6wXQ7M4x1mzKlIqVpAEMJR5yY57wNaxaLh7gMA1dBzo7wWSvnhFY9/wsH6aFNmh7GXH6wNYzhdx1fvl0KbcdOjQ5v9oZPYa6wcnR9mu3w5HGo188RuANSiUO+4nri1DPBmdHDeAf7mu1Ue56Pc9Nj1Whzgkx3wzzBRKqcGvU12j1yCEyO/1ga4BdVHuDgJWWrjtGpTHrfqJ747oZgHRsXo9L6zn9S+6yzNq3bTb5oYVy35ew+sD6wNWDxy2fbrUXwTGIzTst9zAGAzWXtuM/c9ZO+Kww9gweEOOYYAAfQpAVulBsnP5h+CHWadZrcqZFOJRoJmGGGq9sBctwOP8AGysXdG4dyziyzMLhqXKh5GQ+Red+FdpVW1eOtHo1VbwJl7YfnDYvWvFG2HAs+jt6mfK1XXe+QFX3Db1Nh3RU+JRxAedhU3wTPsMFGF+HBPTYu3nKzEs71VG8A+a5vv6OheYDDBrZ3rUcRbtrl83hNuCuP60gzwz+QFX02TEgHz0qmhMw90fQud0LCm1W8ZlGj0ZucfLgeDMcPnXVZlq3baQ0qm3jLhzI9XZ2Rw3mz9ayR3DRp7VPnMMMxnWDzNoA6F1K6rzpXEWr0fTt+Y9CokMNYw3gZ/kXmx1aw/STi8K7AasCsUOZW6qAGFKj07fs8hnhsWM4hXJX6xFolVqMWHQbzm56LIdZsH51x+q3hVbDuqJfFv0pk2sDAwlgq+g35c9+cXbcuO53MwlTOVZw6Az7FY77wNtiWE+ocSZzmDsUPV/KQQzN75815/v+guQ+Jtb9ZQXo33kzwd6wA+hei6ledN4LXbccGW29pPgc2A0Ge8/9zXm+5KrX7nKrXjLp0k+de13jw6PkWNiFAqT8aqBKY2YL0Fwf4wWVwxddrD9OmPSHw8Y44Z4GvPtEpXONamuf51cUSY+yGvSpxsyMzDYubuarrjNDg1vneLFSfmQKhXqrnAjvdGgjr1z1U5UKUcoHpBxgDYfWpb3De+OLTtP1KrA9XwvbPOnvAPo/ItEdn2jYfFC3/sdSnqlHis6L2e8DPzqYtlcO7eh/DocmxahVarFA6hz5gZn1hsVlGr1St6FxDrlKcAJGGYZhmGatrYsOv0prUo0X7vUfGAOxn6Fqrbsamz5FQiS6icyOD2DzTwbDNd0QjD87ZNt3xPlOuP0OSzzp58w8GAGlclYpUliEFKDwoPWHzr176SEZufSDr8SWABRD0AiH7YwwXky5LYgs25CqtKYMOaDMwM88Fzcrb8OZHPcefdluOax4d6jw6q+yYMRHzDPyKwrFHnMsO8rTdHAM3jM9iyTJ7F54HQmZnMtahvgAYYZrKVKGxGqjpvuPPRwPDMz3piGeGezZ/Ao72ZyPEWkDSwDp0mRHp0FzMnzwzz6FYAHJynaVoABxfJ3rH5sRnQ03MDSk1KdDmhLYlnmfnS4GremPsumbBmB/IqmSEuqkbb854CfPrVJJrcuTK19fr6wUlmsN56b8rBr6FcC4vCNatNajt0P7y6DJhJePYbx7N6zMkPCiB5GVOkxnHmAluBg0/wCx+cFEMFoHT0DlAbbfhYI3oDEzDXf0dPoQUo8JoOdeAZo5MCo1J050t/e52IGZOnlp9i9LcOqaFE4QWpTmN51QOd/WvOgQAZitOS3w8TyL0xZlbod1WHb/ACNfjUqbTo3K4SwwDMPIuX3P8T6/8I5o4Obdpxsyqbh6xiyY2fRqsmGf61KprPrWbHgsdZmC3dN4i8TY0flK5Q6bdVK6DN5nW2eTMFSXtdVHqUqoXVbFq+oY9LpRgYYYAb/nXyPW4f735fs/P8Xjr1DTRo38p121C1b4iRozTDLGHLngEwFIrdsQaVxVpTFvwQjHS7ePRMOzfsXmyHfNYCr0y/4MsGZsLrMzw1gQV70hLqqt0S7qfnBTZpwwp8PR3gyC/dcPvcPiMP5h7PF+ea/JV6/2KVKO4JcXn6wxJfzPDe8+sfb1bqNSlS51YiRjkTes8Fn3p+B6en1nrGZ9ZmlGrEuNNDBfI5uZUNnUplKekG/yMaS6bOiesGawlV1zlSD0D9XxXvGMA2B9atqbWJb1ZaltthmcnlTzDPMzU2541Ssy467ZTnLTGpWHMmHR8i5thoLnquObEvCQfydauodYvGqwqhBp1ZqTMd9n740zJwB761knmX3hafiB4OeYKwqVbYpsKnxYOyoMePJP/AsW0HYdHzowAxO5M4pmBtGeB9a+wKUbNBl1U4IHHhHg8ZqNJk0ee06/hhNlH1/Il9+ehyKc3zJx5WGsDXem/LbalmaD2GuxrNZ7wz6wXp3hRxUu2wOGlPY4T3Ay8D8k86C7DOSZvn8/WC4FSqUZyo9KYgmGfQBr1B6JF1WdTYVdtXmwpsiLJ1zCQeGZ7967OG3Hzf3HU7P4tcWOIucGVw5Ow9A9Cq1GRvNkPIyHnW9pTNsWA1zcU6lPqD/XI9tNkmueSeNNuTKzIrlOr4PW5Chnzh8mZhn5/oU2iXtOuSjSJdq8UKVMAzzBmPAzeZDyda78IiHFeP3EjihxLvR6x6HQ69YcJimvzarHqMnRlT2A37wDZh/rrn1q2Tc9EolEvGdekym3HvqcODImHmEUOg/kXpmpXJwdjSIkudXMLjoIHFekVl7xsO8M+gwXDON982dGrku4LRrkCtzaiyAPSA8YGQwAAAA/WuDlXCbWPSKl3hUqE3cFowDqDAAbMiJGAzmTegP8aPiRY13WTb18XbXLVtsHTZA5lRCAYaxmYBgzme/rXJ+FdtzuJ1cpgUqpBR5dBZN46jIDZn1hh/v2JcXa3dVSfdt+ucUZNyU/rPRezAD7Fl1LZq2OJD8apW+wfrJ6FRJJmGrv0QMMDVlJk0a6rgCJTTnwKF477MuWzvZ/1FgjNuBvg9HnXXfRjtWy+KPEE7EvjnD9Yxj5N0JOAMmHX9fYuiEOO1gwOry+Rf1oWf3Yz6zBSJkZ+NQ6Y45BjM64P+KyeZvHs61ur/sm/wDh1Np9KvGw6lyUI34UaW9G696i2e9SqbVArhvwHgi677MGXD1gw+jsXo9BW3e1HpVzUK+HLLrdYt9+24TzzXLGDJnFAAz+cANZr0gfScuPio7TKH9nHqJCp33rSdDA3n+w/oBWfCL0tOQsOXwyrHDWBUo4RpTAGzUuWPB7sAMFN4hWlRuK/Dmyr4bnRoFbq7NROSEgNmhF2e2Doww/jQcE+2dY9udx1sJodEjnz2fkR0SpVXSCsBVXtIJjDDxyHswMz78FVRoDEaRL13GZMfR2Oh0K4tWguVKhgxSojPMPPHhIDYaw5oF1xamQaVc0Kh0flsGAYN7l42AAZ9nzrR+kJwEY4dcS5dsWldUyfHOjsVeZzGzxzz2B+hc0rdHqsyfLbrPP+uzPDRMMzPDyLbViiXNeFUj1W6r0M7lOMDBwamz7YPJn0dGC5sDsXDf0irYvDhlbnDniNSpLM21JkKbSqnBM3jmGBnns/Pgu3eidw3pUat37WZ06TW6eHK0iAzUAzA4vtu/61589HW0n7S4p0ypXU4FEapwGBzo/jMmZ4YBhh/vgvdbM/DxIMpk2fOALrgXB0pgKN6gfbB6nhsCO6GwAz2AvEnpG+jTxYvy/DuqlQXjpVRqp8nBDOS9GB7DN88NgBnvXsj1lO/ERxpL7x+Ii3jfiFwcuCyavZlx3ocZ4zjHT2YjR7ANkM8/4+hYHjTwNuC1bPp9zxZxvTa9P5XlILOx4DDPMDD616W9Kt7N+0mPwHpR/wAut0emwXrXtxyXFZM4UNg2TPrA9EOhTgZHgDYzFpcHbUcfox0qqzaUHrJnvePzvec10UGQMkfWCQLRDQUTQjADa0EO2IM93Uc/wLGMz2AMPEWmolwgBaZmi2iCzKN+AjOiMe7bToVtvH2iA6xFw09RQIEmGxvVDMPB1XcyZt2LOTD8VAJmmXt6EzUTWPNeQBehuZpkwNlWYGDw/Oo7we7VwI+t53FyqsRn590NRLgfh+r6Rm8ZvSd5gfR/nW/uSTFgUl19+Xon0M7PeLz7xmoN3TKbEnNwZOrmD5yMN7Pz/AEL1DjnE6zLLjV6PeNAuOZVYUrXeqTOtm8B9fQqy8Lng3DAp8GJOZjclGAzN3rPBdeu2wINHty0r1qVsVLVCNo1jwTweYMP9z/OuOX5b1HmQwctyXGZdnAbMbRDeyGzYYLnGfupmKzCoVSCJJjVBhl9mYbx59HQqyHJOZCagt1U40I3tCeDXk7MAUum29fBw3fWLZhzQer2dUzNnP5z7FnJNEOlRaUxVfuzz8kzN3r2di3ix2Ct2xVYFOqtSsuWcZqnAxK5fremGZ9B+Rcvs+q1yBen2jqNOMwhG+eB7Az610PhdWJ1pWlcE7lDgU/MA9ZvbzMMOjDv6/wCNcvo9VfOBUzYl+E/vww6DM1otVVt7W/nKoxACRNefeN0OgwM9ivdH1bajTb8EDkTfHjG71gsfUodR8Jw97RyeVB3sM1rarb1R+1QUaI289UM8GczzDBYjZwLwrMy0PsjBghDjsft+bObz2e8DwWPtW7YlNqnrV+qyY3JQzZjGyGeBmuhPUG4Lko3qN+nchWHzi08DaPZm8eHWtHxaZiXm7/Jlw9YhxqVCjAYHHh4G8/F2PB5zzNBYU2+WK3Yx1yXEN6PFjff5Zs6JzDDoAPP3oYfF22aran2Zg2wcaOxgdSee/pLAZ7MPrwWBqVBvSzKza9Zu2nSYdCq7wRQgyz2Hh0AYfnzW74hWBSmZVy1WlVVmfIqLLDOAB7E9mZ/R1/QrHP5l/v14AiUqK9RGtbDCJMPAz869Uejxc8uTS5dvvufcqOZ8zLdk55mfYvKlNokGm2zHrNNM5kij1IDktAyeDxvdH+A1s7Ar0sLAuCDEcksu3DP2cuYZs/74Ih1OG99sL8l1W4/Go9tb4fMbAMz2Z4LqVNuSlV5026VL1sOs8DwULhpwBOj8PmpfPPPVisRgM3pZ7APsXQKDw3o1Eh8pE2HmZn9ahaiCM4fs1ICA/wCRaUKC2HQnQgGCsUTNNfU1lmUz7taFlnPw3FKCm7VArIwGHtFZsmYdDiB6A+CaeCWHQijr0x/PrTzM9sGt6qtZ/vQGb/4asFJkt6ueopEaZgqo9ROgatK2kyQNAzPwJQuxM/kUC4enuPBp6iqpnVqI8TQPA2Yb0QUY3AwPUVnGnuB7xUPQj59sB03EW1YVLMfDNRJh5gqWNVW8ER1UPxEUaeBvrTIBvSOSHnQg8H4ihKUAI9EzUcHlLZk70Ef1O+fYl9mDPsV0zJ86u2XohxdNBjJlsU7SAAbzVO9aTBn4EsADyLpfqqIfQo8mgsAGbascqetI8u81YQ+HrDzWo++ta9GwL2atYENjS8RtBig4bsB7NxNHZjbPYukAETFAZwTUoc0O2HAPoNGFtuLooMxTNH6tin7tajDhQWw7zUgIGHRn+daCZTTZ7FECMZ+7WexVcm5jngjB42VYPMuAohstrQS4Ezxd6vcwea8NY/PAlYU2pOB4ZuKBYVKmtvQj021l3qaAe7W7ZPWYVJPhsZG3goas+yzgrBmYbKiPeCWmopvOGaq0tGzWGwa696y1zz36rKw3nh5E8BmnWWWwNShmvVVSeLTBgw+tI6PUQ9owtUZ4oTezVjGSaO+Y4ON4KIFtnJ7wWuksm8fhqOEZxk97aDh/Gy3n6PDpU5xwN7xs4f7/AELlJs5tL0hx4prEnhydSP2sGSBh/gXncPZItCe2Nez3gmmd/idalPA3mooBoyDUKWXCuY/Gv/lAPwn2TzUuvM8tcFTY/wCsmf61n7VPluINPb/HPBbC/GdG8pvzgBqxRPe309NO92mgeZzwf98CRn0bO9Q9sMn2RrvHAqmhW7FkeHnyszBcJMNYNNehvRU+821ccH8CSB/wKoQ6FAtumsj7BWAUGCfWwhns1WG7mxvaQBW3/eOLbbHB77MU78FP/Z6mnH0OUw+hQvXbnnUiNVXD94ga+yUQOgD/AFqOdpQTd1DggZ/OtBzjZ+8S59sF6b8AjRtHBvQUsHgA/EbUfnmUGsDx6gKBZ843joB0IGWWzd1NNBGZb63HFL2MhqKA6ZgyxsbVe8b7w+wQvVJj8RRzqofiKxEkg4yWomjk5ipD09h7rUQ8DPYqgTYB4PgtHJOI9A0MFlwPFWbMls4+mbilaOEMAkK1CS/paGp2KvZDB3rzVgHQvLswiHGY8ir5MNsz1MFdhhn4ikaNNP2ivZcMU8GBaaIDWrmQ4OGm22qJ6iOb3GFe0YRDBt72aiHD3Kdho+0Ve9J8Xw1kHQZME6EZ/PPBHGkgpoPNmvNg4bMQB8dBUgif0RzrR4Zprls+hNiCzTWAd19Q0pjLB9aUk32S001rAftE2KSqvVlk3otKcwj+deRb5ps6g3VU4ksMHdboXs8Hqc9s5peYvSKogUq8I8uK5slBgtotGHW+EtYYrFlx32P6L1rWz+UmRfA6wXJfRpqr70Cp0Mz8UDzw/WutPMtg662wwYGHeYKFsbcIeBgsjGh60jQcfZAD6wM8DVnfnrisR5tNYbCNIAPBMAXHLJuSdJlOuTnzedzwzNBT3twZrky+nX7Do3MxH9/XhmaOq8KLxCODFZtkAd/6u9ngu8UFlyHFanVV/BpHXrwsOlOhUpdVMGvkRth5srb1z2rRjYlUemnBwMN8beuZRnnAXrXijRLEvyyjrNGf5kGDzPZ2Ly+zSoMmVylOlsh5ANY4RgqaGcgG5cH88gNi3B2G/WIGFOl0HVPyPb1j69VajJzg1VzMFE5mXSqaDlKYNnz5q8LVs+NUqDPdizgwdA1H9dzocgJUR/ReDvAFBekzqk+b8ucZn85qK8eB6axuEW0Um/LqnmGvdU8/zqzPiFf4U7kftVP5fyZrHwwY94tbbcmjPEbE5zBXCGf5hxNHvWgqVHYkzT9Xbw86jz7MrMCEdScbPSDrzXq2aB6pQ3ebYDeHQuxcN/SEft5o4t1W/rNbNF6P1h1rlkbQMPHcV3DqUGGxhyLL31oOw1u+aBxIiyKVbkpmmuv9fMbDzXDrhs9+2JANvzuZz6DzUWY9EB83+U61BmTH58jUM0GutiG+9F1IrGtvVZXqa/GmnrhgfeCCiVKVTSzYl4fImqlWJUmonm5sViE9Uj9m24YB9Cgmzrf0oFfQ5lKgMGfrIwPvDRVZz9NN3XcbA/yKBHegNstA/rggCZ+/AFHkzG3j8PoUfP4KosTntf8AtZ/kNLnNoN5mf1qJzLYe0VgdN8DXbUYARqk4Z6Zp45jGfiYAoIM70ngDvbTAuIwMdksE0cNs3c9RVQPYJ3n3wUYEt7wfZpBr4+ItBYdEYueQbEtzD51cXJbcWiasQH4x4d+aYHmyNAnBIDwFIqsnDOJmtWzTfVVG9a1lzlpp9DPesvUmYLzXNA2efea3DNNe8A4PnV9AptYpuDHq54wPvBU8ClOSYpymOxdlZh+sqNFfg+M8AYYIMFzXJ+8wSOq8z35rpEO3rfjRwfu6JDjZ+TepHEKAxDt+J6mt+HGp+YAEiQG8/oXIty89TS19NkFCeNvLNTZPKA7gbgH9CZOAw8eba0wFGNv2masI0nW9m4GHnBRQo74D2frV1TaPg1m4YAua/Ij/AM4h+wy5IfnXsD0fr2qtyWvytxvhzEXPDAMNi8vxgpwGHf8AQtXZ9VfCeFOgzno3kAO9Xw87muHssw94gk02m1WAdOqsFmTHf7DXItauWrhOqTcnDyOmuu02twfUwVLYcfzmC+1Fsnny+bVuf0e7hj3PZRm9b8rDWjgeYH9a7nY0+jcRaMFViVye8Hvo7J4ABqkqvEJiqzTtW37cC4XZXgnq7GQNed63Zl8cIrmj/aM5lNo9UP7ydMk7MPIugeqKlW7Dtgz56sQwd/BA9Z79ChBxatUMPU1OrdVa88eAeAKwtWg2AFGj1W1acy8EoM+Ye3mtBGew92z+gFAzsDiRw5vPO3H5TwOv9cSWybJmvN3pb8VJdNnu8K7ZPCEGBvSO8w2bF1j0k72sC27f5G8bRqUyQ+GcCoxAAAZf2d68HzDOqzznPvmZn3vHmawt0eEc2cAzc70b0PCHrmpbzJvNR22Ow81cM0GXclzUy2IP9Ke3rju3Q6F6HnDSDxL4jTqdWImdKYh/eT7wz6F3i5+CF98BBluWjo3Vb817mjiSNhs/IChcN43DzgbxsiQabKjU2O/Rw5wze2Z/P+tdz42SardVuSKHQ3wZjnGOU9O8gBv2Ly3PT8/+JdSnXnXpcuW2zTY/sAiAfkVratBlw7NCcwH3inTwfZNWFn0SlT+G1x3HL+8zaCZnh2GpUC54LLVJuB8M6VKPlZjQf5Fyw6Hol6Zat1VmmS2H4D1VfpocsZmGbJ+dedLzCVQbhqtsnXAqWb2u8bLIAAH5EfpAnLt666fcFDnHDmmyAM6XXh2b1qqlwNqMm1YVScr5nUK3gewMzeP61s5MPP8AUp4MvnEigYNB5DU62JPjhqN+EHeupX/Ydv299n7A1w52a998dDeZrH3hbEGybhl25BN56Ox1m91msLh0w1FvVhh66mrVB8I3Os+Cee/P/Ou51Ww/uFMg0bWCQx4LwAfWC509b32h4RQqzbNvxocgOuRI9thmui0Hi7bMCy4TFRrGjcEVnQM8M1twsbXs+87j4dDTwY1nuS62njMzwDz/AK0g432/Go1YYiTjk1CrvG+DUcM+W+Q1x0OKN1T6DLnOQTkuv5sHOMNgKher06xrFGuMcsdQmveD37FtYncS7/nVivA5r+KEYA5c/wDOuf1ivVKTFagvgyzHA9gACpqlW357vPa+bp9aYk1jWaabw6F83l/uOmLO3DJgzKQdKlOYOmuaTKPyDXKg30d6u69Mzmnv6FVBPYez57WPyaK6eMtEOTy0cNMDwzRHPzhaDbmxzrTDzzGroOeyUE3gZa02+tdKBvHmSTLznMC+5vNR/wC8UsNPSNvzqQyZ9em3vNSIwORnWpTHtQ3701gGr4aRzMC0+xWJvLTnnTqT7gH58EYaZlh2I4DJyYTviezUQMAa1DcQaOmxnKJUok5hw8MMHsA7FHmPMHUXT6wNFAn0eTANxx82XQDBnyKt1sGtPrPzoNFTbbfrE+FTorGtKYe1wj9hr0LwxtiLUmg9Y01kMDMDZANgGvPVn1iIzVDYnHJZ1wwA4/XmvVfC5nRpsT54xvn9a8whzfj3c79jXNEg2jVXqU6cbWe5czDPf3rn9V403xclBdpVVrgfWEYM3lH49z/XHFWqt6eGgAMdfkBc8h5syFHW2+nyuAkzo1NiMS+w8wUdmZE5gHJzeYeRKpBVc9ecxJBow8E3u9SaJbdZrE1piDB1pDgGYM59eCxwxu+wVNegzJ5sPxOZM/Y78MDV2zGpvMSGz2YMmrWyaJbNvUau3PdVN5nlXtAIhhme9Z9l6IbEuo+so28DPRM94LcgjhwQadYiue2PNNPMsQPEflgcg+s+9EzMtyfTZrhsPMznz+5gfkDvUWq02XGNrmmMM+j5wWeAnpPMxeUYc6PY/Io9V+4Hm/gbph2H1pk3okYv2R7V7HQDYpFVNh6EEHXB503usOxLhqu6xGYt6l09gGAOoTQz39ikTJ8F6jUd9hvCoGeD0fvDDoVJWw5maDmZnoBhvPNS6DUooNHEfiRtUAzZDDrBY3AuJMZ+tzAfg5s1Vj9mDPZmeHeqSNb1z87UIvqb+cIpmeetsM105lmJcLUKLSuWpUR8MJPnA1dzLe0apU35VDCBH5AGNEz/AGww3meCuEwy1Sr1wQ7SOzrgrMY3ZRgxm1vejMd7B/X/AJFOk8Rb/s9r1MxcEaHgAGyEeAAPGB/P+RAdYpVNsgKzOpUOqzZVSOLy7x4GyH51h6xWH69ctPqTEGSzHi9YOvAeHnW3Y2w7XZlHoF5047qrkXmY8UOakszj2A+B/wAa5TxdoNjvX/IrNs1WNTafKADCPH6NfvAA7Fd23R6/cMpqjsRZkOFVDMwlmGAPAG/v61irkmUo/UUFuXrPQgfN6RhvM8+hIceEj1rWGbfdo1NqRvUxgM5LUTsDzmr2wKJZd22/W2JcGSzVWIGEYAeMAOUZ4Aa1HAq2LDvMLgptVueHTedei8yBvYAELPN7A/nWH4o0e0rJublLHuqBWIms+ZvU548ADPYBrBbBSZNSo9SkUqoxA5iKeDwH2GuscAaDZVyXaZ3HfB2fUKcedKqMRnfnvz71zeHbblw0247qnTj1ac8xsw3vZ/OvXXoE8OoMZ2sXVXJdBmM1eBysOIbwHKwz8bYf5FvC1Pedq3/xssu4LxuPjFPmNWG8ABHdZzA4p9b2zv2b/oXmUIeEg234kzS/oxgGAH/qL9ILG4YwbP4VcSKHccTRhVSZKN6P38rgeAAYecF4xn8K+JMm37wnMNvPW5ZuEWScjZohgBgH1hmC0Qysng/xNZ4afyvhav8AyaAzZ9YsmGwwPA9nkz711X0bOP0uiO0/htc3qp6lTQfp7MjPxgN7P+DNe4+FFEpV+ei/T7Hvihs02PW6UcKZHAMD8mf1r8xOMfCu3+FHE65bLtGqnUqfb2GtIe6wP50HTbztij0TilR+EnqcOXi0owzDMOZfe35nn9Cw9Y4dVW2Lo9VUOmzJLTEYMJYHmyefWs+9xOuavSoT9wVWSbrEbkueAPvRseQzVnPuSsRrShNy7jPm4RgxG5c8NnkXLY0tnxgjXXT7mqs7WDeyy7nng+YYAsVJ+0wXC7Rq42EnCYYHOMNmfnVnbc9gKjEptK3nV3orB628AMz6wXWK3atv2HIuV+q0OpHUKDTcD557Y9Nf98H+/eowMTSo06lQHQiXpMeamvA9JjgezMDzBe6+HvHiwL5GJRgl6Ny4AEyJo7zNeD2ZnrKz5bdOi6046kc3BlnA+VZAN5/rWr9G+NxDe4u0SLRqHJeM3vWh89sDleg3v417C36ChM1h8PP6DDeCkA8nXtA/EbbUXNvJd3hDhPpRPZ3HaTHkgG//ABru1KPCg0z/AOgi/wCAF559JN7PiHbjHkpR/wCM132A9/NNPDyQ2A/gV4WtuZwFRHqkAKPm4YKOcbNQxSwqWZqXGquHvMFnDZcBGBvh7xF7dCgXDm0G9WAVVt72ji5uzMNlWDNYc86xuF7dA9ZBh4jiqpkxsy2GqL1q3j1pnnM1GFLsHsyTvLZqqjPb1ZhM2rQAYOMuoDePJPGesKgyfBE33OgAzNA8bLb3tFibhpVzPDU7gqJgHJPa8PCTsMPIaj0fjHZVSdkAFcDBjDAz2Zqqv+86He0CXw2bqTwHVI2caoQd/LPhvBGTL3PeHEKvXLTKVEgshTGHgZkx2pOsDx+f8mCs+IXCvhzMnuzp1v6NYYMw/m4NEzDvM/OuH+jredR4dVTUuqqm8D9Vi0/RdeDOMYGet/jDevYVSoNHrzvN562f9IjmoHnXg5bdqzKHdtnV+6gqQVcMGWjAwMDDPA/74Fwe6qJczMWXTpUHlrop2yTEkHmBhhgBsr3hDsy3KPUuRi2/GOI+HjMuhmbx+fNeZ/S9pX8nt0cP7ntidJekRXjDl5HQDAYPaHz571UDilVrfJ8NKVbjByXvWhhzgdZsgzsNWF1cJalatGj19xjRhTTYMIMd7MHmNhn/AAK1qvFGwLwvo34vMxqfNZMz5hnDA/J/B1qwuTiXVa9RqI5cD8aGzCpr4By7PjeTPDNWtw26qwx9mqfbDHRSKkbzL3eefQu1Urh79j+BlwcVKzXwmVOayxFZZPYYYPB0fOuX+qoM/iDZVKfb8WVJihJPzhnsD9C6hxLqr9pVIKHLqskIVLe0NY2c4zIPH3/kzWQ23BzhvUqxcDXFu6p0aHTDgBXQgnv0Q3gyBn9AGfyZgohvUq7eJs2ucJLgONSczlXJWZDPgwwPfsM+vPes/SrAu66qNEYsuXUmY9XzCYyZ5hGi54A9+f8AyKkti0n7SrlwUNypc/QoUAJtSnMngB+TZ3n8iDfT5n8p0g6Vw1inP5WMb1Sq1RmZngHvvkWZ9av0GpAw3EqusxMPWlyGcAM/kXU+APEu1Q4d3LatHgnzYQA5mo1FkGQzPZnmC3dmWfwnmQDvH7Yw7kqtIzzeA9mf0GrHN6bc9Dsy3prDlsPBm8e8zzekmuQ3IzUnrhP1/LgUeRUT140eIyZ5gfQ9869bVig1K56zblHgxYzNYhPPypkgOiGGHf8Ar/gUqlejlY/2miXjKqtbqVYpzx5uzntj2zDo7FqOJPWfxwZ0n2OKlSOhPvMA88AGybwYB2fwb17ApU/Wo0I3PamzvNUlevO2bVI6BzcaG6HXrBsD6zXDQuHi/PvCq3HapvV6gmAAE6X4MYAz7AD/AH6FkPSDL2akAzvWXh3bbB1k7ci1yNMqDAZnHaPM1IolVrP3v1q3h43g/QtRqADAlaxm8wWaCpOG71q1ZmOAPhrIWuiBqFMBvPTcSZk796OTpmepqKFKd5lsEvDPocUiSy4aqXgNlWkpIBmahHpp03sx3qIfWiEtk1d03TN3/wDcs4pcafyyDQSQzzbVU9Afw09PekzWAMt6dOqh+IgzUmHLAzbcbUEwcDrV3PqTBmqeTJzVrQpIOH34IAN9n35mjM0zn8EQkBMc944pcaY32Gqo3m0zrN57HEGo1sw2JRpgAXiGqBmZgXtDT/M5l7RQtqGZ7GPWrCNUtyysbU2K6grPA1EOpKW9Uv8AjFUgGAAlzIedeqKS8BnqaiR1hhketVsyYxhhmql481aV99p2wUQ7h3dazhyWEg3+IrGjZrzgHqaisI1w+L1rKApALEbn1lrNeI4mefih3rOMyZWHhmo8kJzwGqgWFVrDBteG4moc9uS14az/AKqnH1qwhsuQwwcWgmvBvSZzAlHOTmh5r4KEL2NUjAU1MnuGWeoqfnMEBz80Dsk/ef8AAo4PJo3s01vNBZMvMZp4zbPNxtUJvOM7DT0OfgagG88+BpnnHlYPackdMFWTIzjIaiB4KlgpHPsPB4izjxue0UfmX0C4u4SeFtbBv3AAf8a8yh0L0VdrzkyyK7Bc74ZrzozvYA1YakmxGDXlbATTJsTB5pjo7M1YGAPeG43sUTAGRNtgAD6FC1Iy9y10Umd/1kAXRuKgYXbCf88bD+Nc3rH3MWp2nnyrwPYLoXEKYxPgUKuN78wMM/P0IvwqvDMPyJnNv2aY5xz3iicyZyD3o3WBng0u8ehVM/5R3RRnG+uNrZrz1Mk4NLrHonV71PxdOI454VRhmBoyevZkADA23A2KsBmmw/DbggHzqdMqrZun4ir5Mlt4NiqHPhlL5psuG6FRpUQ3gPrw7FmodefDY+5ga6rTarotYS3FhLztLPVrFGY8Lyd67+FzcqtOvP8Au31YUqe/M9oag0GNTaxTeVfbzmrUQLbbhsaYLXm/DE0aDxw6FZnTXASZjedcO3Z4R4b04zwBTZgVEGOjYpAMtgewFINlx5rTBZDHnMfywUulQ5VSdw3h86dCiOM1IHHw8JbaMDGkHhqxT/YZ8xFz1qH50D1tvwxPUMD+hXBvPslppPTPCMMFAw8+YcZ3TRwKw2ftDUe4QPPw1mTkuMktR0ONPYNWDMls/ZrBRqq2HQasma24Ci4W00l7Yq8JOBbzwVS9W3PxFXvVjMjXo1ZngOfPB+tV517lvfrNPVIz94ojxvvKhd1KsBJEzYcVYExw/eKneeMPZprnHw94g1EaY5+IrWHJfNYrnwhtHKfcwAOtcsuH0t4tEnnBtK3wn6Gw5EgzAM/kUj02GupUYzw8RwD+heCr29J/i3drXKRKlDpUc+yDsP8AWuWRqlc8N0H2Lqng6H/XDQfp9VQ1msFn5J6Ow14ktX0qOMVnzQcqtYercEP6PLPr/OvUvCLjHbHGCO7ysF6NUGAzNozz/wB+hBfcs2E/XDNc09JCiOTKNEr7HXFMAXXXo2BLEcXYD9VsaoA37gM08Dk3o2V6JR+ILTdSc8KUGC9ZzwpvvG1+f9q1j1VUaZUTPAGHt69twKxEr1LiVKLLZNp9nsNdfKhnLzjMMwJFSisAybHkBeIp9e5nigD8TZnM6A6F7qu2Gb1Bm4eywwM/IvG52jSrVu1qq199mTT5T3WbK5h0i/8Ai7dtBtwNC2YcbfhqgZrg4XDOr0/785m6fer7jNdUSZMaiWxWOZpRgB+dZSydA69Hly3DBoPIGayXt2uq3nTbe4fO0qlMZu6JhpPAvPMypSpMg324oM5+RdprFVs43deozj0vk2LM1Wfw9hx3ZdKfkm6fyZgvLHKpkmcHjm2Z/Qo71YnY6eua3teoLcOL6yclRno59kdYKfGim7qRdYAPsNNiDzLh7EPQe9tEYYdfQmnuRwwYN7PvzWiEvnMOhTQuqohFCCAMgAd+G9Z/W36adUCcFVnMu67Ep4HT8hqR61rMkdN+c8YeQzVOBnmnQe87iNkvWwD2m9SAmBgB9HnVeclgI/h7zUcJOzxFCLXByWD94BqOZuAXtw/Oq/nA/qUiM8Ek8D3ohIZmZnp5gnZJ4fPmq96A2G8DNNazge8NeQ2SsNY0MmGbIah4KPrGHQpDISp/s8Pzmt0ImBn0KaFNnG1qaakM0rlpDXNGGBn2LcRvVzMUGwfPP5AXLd9YwVNB/mgciNgboedbCZMnVWBoHBjM/OAKQ9JbD2bDIfkSZeza0zXJ9Yx7MB/V03MAUepQH2S1MNb6FuKk9BOFpxGA1VSPPchDOdpg8C1+kZLNwOtg/wBCPW+Q/wBCuo18vn+yMM/nBVUl5s3TfPrM1tF7Ch1KdALUiPmyaOTVZb2eo/mZqEYGfQGYJ1mHLP8AoprqEiNSm72s+XdtccknIAM2WY/QyqyzLbuCpUuXUYnLaTH4wda9aw7w4bVK1JYUOmgbOj+zhDw/jXBaVRKxSrPrE5hvRaPPBrvBa9I5PGefzeDZsP3PQiB6U94bk6SAeQDwVhR6DLeilL5WTgZ9gKteZ0ZToaDwH84LIa3h6DkOa7VX3wk8qGYNSzzzNdTmcS2Kr9xfoDMYD75AbFwbWfBoGwPBX0apNzx/nGuSc14Lu5LzlBUncKVSnmuwwDrVDSoBzAzV2zUotNa0Awe+sFECpNgfhtrm5lpBm2z91fbD8iMzYPDTcw2KFzIPEgkyQZHUcXzcCwZNsDV3aVSYpVxxKrL9kxnmsM9W2w8Ntt43j6MATXP1iS1oGxoimB6WufipSpkf79XOfdf9jEjsrW8AbAuO82Arl23idNpmeDNEZPef1/wLy5ZlViWrXGqrymtI87p5r0XYBsRqyFS13g5p7YYHgunh5HO9dwLbodKi6FHpUZlr6Fn7t4aW/flDkUCvxdj/AEOh1galQ5Nx5tRHHwSuSg3NW4fq5yszIDR9ZxDwP9a+x63ss7h5aoN/t+jNddT4c8SZ0lmlYa8A8M9n0fOqy7fThtyBMBi2LOk1WP8Aiyz0c/yLd8b/AEUaA9aUi6qBLrEyuwvGeelzNY3g+hcXvC4eGV28HZZ1Xlo11wvACJ0Hnmuze3Phn+P3pM/ywUGn25SqNJpsIPGkg69nv2bP4FxQJOAqK9sPTQZ4Lmvmd0QuGZ7gNBgeBmeC9IWZw3/k0pbvEasVXnJvLZ5h0f7715igMv1J0IkVg3j8gAtEd4XjJpzVuViuPSaOB74meGYeTNc0frXbR3JMuD1DUL4uCnSQdrcnY9hsA/IvSFn8ZmKl6Mk2dKf9WyIoHT3jzN7MDXmTi7xRYvCm0y3KNRzptKpbIMgBvZ5rpc/hjfFE9HXQlzoARJTzDxxADB4/rV4criVNn3G9FqdpW5WZj0KaZsmAHgDwfOrW0vWtYn2/w8lGEOI/P34BvBeheBXCv+ToZVVqNOjTNeNvdZPMwzDow/36FfWrwxg3PxNjxaVSmYATgzkyDjZyWQS4abL0k7So0PhjHqIRPvEKSwyy7nvAFoLPvCcfD6j8jSgnu0uB7Yz2Aa6Bf/BOnXPw2m2JSqk896reOaEiceZ5h5z/ADrGcIrSo7PCB2m2rvqf7KbPfnn2LG4IhxT7H1KqyJfEO5gCZUPbgAHgGaG2LMbhzTvG7aqyBmefKYZmvRDPo61k7PkTrnuc4YMBnpNBhvXnyt2ZXDhS335cZ6nsHhrZ73lF26YZx68KwYmx6xnxqVmZ8ofQs/MqsuTFdguOfdX+t4OtaCfGocOL6qcg/eA+dZWsMhAadCI390DDA/n71EWi4bKw7hpQQJtnPtvPA+Bmbzp/5FiqVXnwrcd98JMmFCzZNnrBVlNmTmZUidBlaLoM9a11N4RVwKC7cdfqXq2Eebxh3/nW13teGZmPQTlOvwWzZiGewPIoUx5gAPT6wBS2axTWYrWnBBkADefXmqeZMivE6+G8DPYuPC4ZF43MzPfmfXmo+Rq7CGDzvZmZ7M069ROTddbfD6F0whmngfMQ2bzPBR83GXcDb3guh0SBTYcd1ww+8YZgfkWXqVNY0AnbMz8gLQV4Rs4fPOdGaZA96aAN2CMNMHfEbzVqAfWgMDTp4Gfh9CHYiUuNMfjeGDmw+sED289RSIbLckwYcSns6LQfXggCAfhOt/IjZDCL4jni+RGzDlaHhxXvrMFZQIGEV2XOlss4dAGfWg3cCfBqVXiU2C5m1y3sl6asZn7lEc8gYLyxw3o8564YlSbY+79Ga9S4chaVQ+SG/wD4EZPHV51J+pX1Vqy//Sph4H5wUGms51SO+57Jg8zBKqnOOPTObl60cGTwZw6MzU6ByMa13ZWGDz8nr86PFwdyetbSm2zVd/KyTlMu59/0KsjT5TLQS9cwCKzgej1msu9PfAz03OtaulTHKJFNicwB+t42AGfWH0LluGyhmSYsnDlcw0+/PeaOlMy/WUfXghMBs8+XPoe+RKg29OuGs+pqaYAfXm6eAAC6LZnD2jnZVYvyuXBGjHS3n4TMd0MAAw6MDVwKe7bnC4asDnq7kGWAwBrvUUzpwSonrJ/RjsHmCzk83JNvU+dpvBOlGZnIPoMA8ikT4c6NFp86quYNTWfB8+zrWiGgmHTWYrT8GnHAN8zDlw2BJDzqs58HpRmxomZhgHg4YGjks1KSwbbHOPGwGeq92KiZCWyLxtnh9C8saIP5tpcsKqbJyJR5hgfsVWss1ll0Kq+wHj9BhszUiMDFYpoUZwPvvQzI7zNWs9l+ScSjv0rewARTNlRZDd8N6bTTtWq3PWYIVL1cYGceOfWfz5n2LM1u4bgti8jrlVpTPO9gPHmBsKQdpQYbR0pusyeY/B7FpaaD/wBnqJUZeASOfOnnIMw1jM+jYuYhzq4axSpkx2qsN6OuAGYfOoXrWJAlRH/2nZnJZ6Ni7RQbJtWvWRfZ1WlAyFE13zqPeB9gfn8i4FTYfrKqRKbqYc09o5+RbRbbb2l6OVeuC9uDtVtg2DekUiScKA9UdmbBhmG/sXj25KU/Qa5UKHK/0hTpJsPeTNdIsC87xmcX6fb9NviGyZzwCGbz2jT3jANgGtBf/o/cW3rmuO8bqtxl4zknNeCDJDReYw34H+TyLsr/AIc1uH0qpToGbmeB9h/Oum03iixDpxsN0OmvesYHq+frM5nJ+f5FZ8S/RvvGxrSkXVyISaUxDYqByM98bW9yfn+tYelaD01qDO1qUy/oaOzeB596xsh6C4aejf6+sa8Dr87CXUTY5NmIe9kwADwPz55rvtmej9wrt5qmVJijnGqsLx+YiSTZMH+/zrTUGM+9DCKDZmcVlhk5Bh7bZ1qzrFYt+z7fnXNdVVZgU+nBm88a9SzPGz1PW7flOP3PyccAM4dON7AJ8oD6/n6+hUtVsxilWRdfDW5zBmlX4yE1mdH2PRjBgDez+jAN68qceOM1K4nXuEvhy3MCnsQDp8Y5YYHn1mYAvRHpOVK8b/4GG/w5qUMLap1NYm1iog9nzIAGHKh+featrFqzgD6UtOtjhBW2OJNxnJqFEelQqO6EbMJIAGwM/OvEk+TWYHrCJVc+Yqj2tJN48zeBasIF1Q+FDtuXBaRxqU/VWJrNTDrZfMA/xgt36MfDSj8XeI0SozpVNZ9SSTCSzLk4HPfwzDBFvP8Anhgx8i6Bwoep0CbUJ1cpsOfT9EAeCRJNnD5w8/f+td79Jb0YKbZ9pV3iNz0k6qczXkuxzwp8ADwAGADvz2fQvOPD2w53Eu4aPY8GcEN2sSeVCQYZgB/Qg9ncEPR+tifXKZxRbqNKqtPi/eocen9HNdmzswXUuNlpUa87UdnV+KB+qGX3mQw/aXzwAM1I4MwKrD4bUpi46UEOuxQ9XzzBkA5nR2Aez5FuAZB7DX34GBrwci9GD0abVgXVdFVuegMgdOpUKinB6wkvvMA888fz57Ni9BxuGNpUes/aCh0pmBIYhsU+Ny/uYodgIqI9yDuuxsN/23zrQesO9Y3AyVVZMHTzVZouGWGC1sk6dPHx3N6hSY1NZDYunwxeXfSTh8nxaoUXPPCggf8AG8vQUONhAif/AEwf4F5/9I17meM1M+SgsB/G8vRxm2zHiB5Iwf4Fa0QIz4diB4DZDxG1YMyW/eKPcNK9dxQbYcwdA8w3pDJVG8mgVO8dVhnhOiPMh5z6FKjTzeEAbcXZ0se1Z4bEHQaWbgAo5vb1wXDoTdZz+pSWXjMlWZ/BNHJfZPwzNStpWTwVrD3rjHELjravC7km7jckyZFR9jHiBmeHzrglw+mrf70qsMUalQI1KfDRgMugYPRg8+Ydahq9gV7i7wotWb6url/0qHNzwNo3g2JVK/7EqRO0anXrR5Mh8MAAJIb/AKF+XNSrDdVmu1Kc5nIfPN4/OahHPgrDuHsW9qDw5sCV/OrejIlZvM8xvih2Bn5Fye4eME6lVkIgVw5gQngeCREPY8a4vJuqdJwbqT5yWvnPNRXnufPU6E7hqK3efr6e7OnOBm88b54B5zWqsPj3f9kzIQUa7qkFPin+z+22Ljs8H8wNgEDMl8HQbfbwzWI9l0fjrXOLt3u24xOmRoJhrQzA8HgMOvNU/E6sOUqZR5dVqsmsVCkHgYTg8GSZ+T515so8mcEpo4M44chg82XQ6wNd9h8ZqPaVtU+nXNbkOvXAYa8yc9vAA7DSLSws82LYuqXVZ1NZ9YVQ9aNgzshv+TA+tM3zdTl2j6yqVvsxpoYAb0QP1gra/OKlOv8AakNnQ4cZ042Gs1nmBhhhh+hR7MmUa6re+ytZB46hCAzZl62GZnnsNabGBrck6PVqY+2/m6Ac0B+Q1YXVdtSr1sxKa+5nHfefe5s+uS/5zVKdElybriQZ049IDBjWPfhgo4HOr1ZaAPZBJ0AAOgA+Ra7HpWwK9aoWHVaVLqMmfN2SnjMzAzAGMA/J17FzyTWLqo9pXBBpVHPl6vPCFzBhv0A7FbBZlfo9wUyDTYPLNVsGIr0fPPWP/IrXjfQZ3DqGEF+pbzgesGd+eG/DD+NejlQV6XR6cdKobZ8vOwemAYYZmGeC9EcMbzpX2Fqbb7EalRwDZgGD0x89+DP0LzfbEB+7bhhRX3A0mz6DPrPyLsXAfhvWa87T7tqtKjQ6TS5+cbM9hmHWf+MFY9F8JWajbFOj8QrncqT1zVd4NaD0YYZ4B/GpD3FcIFBkQYLEz1xVJhvnIM8wZDP/AHBV9w8RbOrchpv14dNmmGEN49h/k+tYKq29OjSI9RpRw3nX8+ZjhvMAWrFumbhty87jqdSqU5kD8fBk+hk8MP8AIaUOiXj9lTpVuVyiBSmHjN45EMzPBZet29xCOEH2VoEOTHNkDPPDYZ9i6dYECo0ShhTqq+Zuh+MeZqML2zlB4b02lXb65oc56HNimBzDz/bD7wAOwP8AXXW881XgDeeonQk+7VrWAPKazPMO9URvYElzJn3qMG2tjVIOjUTx1Lb7RYrmHvOnwk7A3qMIaY6r86gyarB94/vVaczP3igzAiSWjNzr+tbYEg6qGrpp71lH/EWfMGw94meZ+dXhG2p59jS9oo3MtmqIJLilxjzFZLXGeBInnvCPTVfn86XiefYoWizJh5qCcxSJjLhqqeA8lrhAzmfvFHOZh7xQXs89ijm8YIhYc5v9oj5nAfaKnA83U69rgfyItcBMz94nmZ+j1uLOBPcBOsz3DFBs4dVYw1NdWcC4YpnoNvrngPOY+GkDz7J5g5vRDrfrtjD2irKlW/CPQc3rFBW38MHDTL1Sc86YF2FVqO9xxzYClvVUzgHoHv8AIsidSPSNvUUIKq4Hs3EwNnTXjktG457VWHMsRgwcXOXqxLD2b6aOsSj63zTBtuJ9wtgQNsGpEa4doNuGudHPcz9opDM/50wvbrUOtsGGeopPrhv/AH/4Fy6NXsA09RWDNbz9o4owuLdD9ZMGmjezWRCsBn1p47hYD2j6YQvXjUJ6SYezcVDMudjDBs1VfaHzmmBpXqk4HW4mvXHzrLya3mJuNmoXrX51sNwFV39alBUvnWCCq/OpbNV+dBtef1tmmChPagO7FUxp+anBJzBY4FxAnuAXiGrXWB4d6y4Pb1NCYpEt6Gwe9V70Bx72YJ3mT86MJjgIIh23LmRZbHRrxnw/gXlcOjD/AH617Io8wzkaea8fz3oMas1CCb/isSTAw/Oi4LsTRgpB6aaN5j8RQpmbkDCA6t3dX3yxbflt9gACxlwmxJhGAOK4oj1VmcNNSpVIJLUV7BkA7AViLon7zBADO7UUcKkx781K5kDDwIMww84MmpEnYrjg/VX6VxfoT7Hv3tH9azxxq5Ja04tv1J4D7wjGr7h7w9vuZeFPqXqOZTWoTwPHIkM4AALfDJ7TeBwHXdTzqIZ4qIdbced1NRCc9vHxFGBKZ68FcQAbw01nYc9jmA8RXDM9vsNXkWRwKayepEiABn1mh6FHZqTGW9xTWXmDFZAADWUgKb500ZtsjqAjjVtj365xI5MPIpoQ8BTXrKKbWu2aAKwwtVnTZY94Cj+J7hxDMnsG1sVJJquiWxzBVFoXDzxshqS+hV8mqsAOo2qSTchm0YOGqd6q5+zNeidUpPMgax9S1AM1ehM1vaKFVWdYTNtVAz4T8D8Q1NCquB7NxVRxjMkVSeo9t006rcFVZhtAHvu9bMl3z773WjDevLl4ek5dRzXYtvwYEOP5w60rY9KW+I2yqxYEwA6M2cDP86yavWEalPycNMFbBb0vHe2vK0D027qpoGxUeHtNmF2Gy8YIXvT5vSNIDU4bU0InkzPP9aD1E9RGD92q2pUqm01rnqrLjQ44d7r2CxkD0jbcvCw5twW45yFQYDM48g94H+heL7/4hXpf8o51zV+TJAPYs9AMgoHc+MfHim1KO7Y9iGEnmtj04+gPkXD3rGvSN4kptllrzgeaxkOY5D9gtrA4o3qb4MG2zJz/ABmV5saapcBLtgUELgbueA81hmYAB7FgZJ1WlSNB9xdAqtz3jTWg/nHOOYZ8uAYAotw16y7kYiRJc7lpDHQEgMA/WpFCFSuqAOucV4I5++BlXHBa8zszidT5wTuWjzT0JIPBhmC6gzYFHuThpT5eYRpDHQ670ZrFcaWa5As2mQa5SqO90fe4hqose06Jc9uXm06/b9ZjTDDrBk88EzXqU+9SZsR9jNo2d4L897GuqsWTK9eWrXzgSPkDNPTOJ3EqfVzqT991jM/wZJgCvjGgqUZ+HWahTXG8AYe2L1xwxht3DaUeosS2Y0jDAzwwzXj+H4zQSz3un1n3mu0cHL2rMaG7bjD/AO+ZXZyodyzlwGHYMtzwvxmnl4l43+taJeTtKnSjkx+wD/IvQFYrFZ9Za/rJ4JDHRgub8Y6kxxCpDNKnQQjVCF0SHtn61zDmNsQ5wUg5zlGB4AAzwM1rqbx4o0Ol+PQIz0gPdctgs5AoN/2xa81+UwzMpQM+6e3rk4G4Zm58+ayW1F1Xb9qqy7VdAGdT3LPQCp9aq+4gvaXnwVYeuyeYGCszuescroa6Bp6fLZPTc2H86dZqWcfQMAVU9MceLx38zTrMN+T7DegPwJIFmYB9agnGcAtieNk2S03294IN6sHGZpp752efZgmTZ3fIpDLOsWmmpgcmWGuaBZ+7TRqy9W0rlQf9ceKYZ6PkVUbOwzULgutNGg1jA/kUtmNm0DmfWqtCDngW9OgZ5+GjmMsMl7Tf5DBDonhqNr0SgePJPc42HuM1EZZbMfEc/QlnonszP6AXmBLM2DP9kBbKNZkF6ltVFuUZ6nYAYYLGsxqrM9nBeP8AItlat81WZC9RtwYx4dCeRL+z1A0tdxt7MP3yrzgOM58iD0lSKqcuN4kvADPsBUj0+cA67Di+RcCweenHHPXiPMm351COq4NaHQg9fVl5o25cTW8mCrzjHJ3rWOH/AHF9SobE8wbf3ga2XqGlMgDfK7/nXP7ekvw5gNuObF0jM3hzcXT0wIX2et8PZ0qN+hF6tpoezpsYPyKSkkf2wyDMUA024Mb9CHTY/AZ/QpARnD3tggeZNktNxtaLZf8Al1qIReVoduU1l3zrP1u87urzDrcrAGn+sGtgLBRpPLO6ikHW5xtGAOda9/qh0W1aUb1va7dYOAGeAADOeZqvn2TVWXebrPMgZ7w1g6wXS+CEyo1i0jByIyzyR6GYd6j8SHpcBrTlYGb54MmB9Coc3pUCXJzYo1K590OsFpaOdsxmmqbeNuRmZofOo9sXIxaQBKfcwaPr86u69c9h1WOc5iIZun/SFYfr0C1TohzonLHIDoZZ61zKt8iErTlmYH8ivvVsF6qAdHlcy6fZn0LQBZlx9bkGGH1vZrC2TmMlmJGj/ccGXT859ZpRqI+cXm507M/IrOq/zqUg5QBzDB4BgojMl9mODB9i5ra2jhGcy8nzoQCWz7zmTT8mZs02w/WoT2oHiG4f5FiJYBLM/YLptvcS6rGYZgzopsgHvQ3rmVNk7+9XHrJxnoSEPSvBzipSmbtj01y43nudMAzlngDOC9tzID8zfEDNrzr8t7DCJdVbj0OdBAGj72utei4dYvGydI+GXEOTP88GcesBguzhtth6wCiS/wClN4R+9flfxyo9KgcYrlg0ZwOUB7wQA/kXpq//AEw35NpTbfvG3KlQa7KZ0QjxwPRMPPn+deLYZvvT3Zb7+s6Z55n1rvjmRHCr/UjkyuamnsUG5Agwz5TQ3rUTw0Y5vqv+z04KQ1fk6CydPive93gaxhthdz+GlzW9RqfcduPhJele2ZDrAFs+CFmUOS1VadfdG1qhv+7vBgYfQaDhFflHqVzcjdT4RmjP7hmeAMn5F7LZoL71LjtsTqUzNfDMDCMG8FcIuHjSm8H3KrfMT1VQ5hworwPyQPfgAL0BxdprFbtkG2Jxg1BeBnSDo/OolEkxeHU2oP1KsazpmYSYjR5hn5wWapVz0qvTApTB1Iz1tY2WgzVuC4dT4Ps06m0H+aszm6wMGZn1hgtLR6xBh16q1Wq1KGzIms6DJs7DzWMo4VWfMCm0N9mGB9EcGczVxc9sMW3S4lVgwecuWa9hGB09mfnR06bOsT6VZPCWusVWqxtU2T78O9c69GC54MOw7jnVWWAAwGbJunve86808Zodzw7ll06v3qzVag4eb0SP0M/WskEydMhcjElVI3Q7APYouyLey6Pxsr9/2hIocqw5/qqVmwcuIeeYLkPFG+araRhb9GgnAafZ1jZdPeBqdwovOsPcNPs5E0YbtOeDWdZDeYGf8Cg+lXb0GiVyiVGJKD71DADZde3h0LjtG3D5lY0QN+c/4y0cY6dWOF4Nthgb5gH0LFXDTWJJ824tBbEasUpiPzcQHohngEd7oSIbbWE+j06HRIQOPgGh3n1mq+5Lk0YQA5VZjwH2G8pt1a9yXBHpVGYz5WNmYeTzrJXgDkl2lUdj2WxnPyZrRe1fJNx6P4aPWiHHAOUwwDDPNae9rPOw63SYNVlsnElMh0B0KvuRm3At45VNlmch8wAAPrBZ3CmXee5ab5w7Mw61Lek5+IfeoT0x8wabNwNJjsMFIjPQXmtR/r7FcC1CZEOlumbYMmB7GQPeaop7Muq6Utx9nDDAGg7EQSQNjXN8M1FOZgsxWSaa/GDUcbVUYZkripVJx6F4Z96pzNdALoBD3ogZceDBIw0T8RxWHo0nRd1E69JcM0dEpU65KzEodObzkSjwBdID0eLxD2lz0QPzmsPY9nh9WN815V1ubszJfn2KQ9Jcejm2uln6NNzB4j98UhkPOYGnWfRsr/u77pTzR+9Bk8F87+efw/8A/q1+blWvBl5+S7HY62mOj84LrvFqq+oeEtblNuYOmyDAfnNY/hpYYWAOgdcjVV0DzPRDDBaPiLTft5bgW5zQQGs8zdPfmvP57/DvH/6nx8zyfJ1HobU5v2WGAJPVJg6Q1TWOlg12ONwE5bovRkPohgo73o8MSS1DvQPyRgBPH8d/h3n/APVr/LeZxQ963EO8/WVvBQJcGMy1FDBkwDfms1WKV6qrM2jA5rcq9oZ4YZq1sN71bXgblUqNJCUyYAboexPzgvseP6+Py+d/w6BwW4UVziE/LnUCqnDqEKSxFZMGdYPGz3mH5FreN/DG5uDkqmUC5n6DXo75vvRjieDm/s9sB9C7H6LvGPhdwu4QV0ItN1rziyczggz96qRmeDOHyZ9nYuNcRbA4h02RcFx8RqUEmq1SMdX57mc9EM+gP1qRmbGuq2A9VU29YMamx6I9rM6UPWB4z8/kSur1dcnC+u3i2xGmVULqw1e9mEGeiAB2b00deo0C3godVtVmTNqLIPBI/GD5wTX2esuHYYQfWPq25apWMDA+gIXnNGSvqUOonRIk6dSqxGqFws6MOQYYRXj8gLL02M5GmSLflt4S2MwMPnXaLze4jT+HduWy3Fo9zwqCfrSHOox5nGD32a45J059ckXAbmBzjz/Oimr4aRgol5SzcgvPSNH7th0ZoKVVbnC45fInGZkPm+BuveRXvBCyale1el01i7YdN7DB0PvUkPIG9avivZlv29WbXo8F8HqfSwwku6OB5/rXLa3F4bIT7hB+shUpmg9/0dsM/oXfbksawKJFol4sQXmahNZ5pll57WAz8h/66zljUSDTbml1Vhgw9Vngy08GBgHeu03zSgvm0qPcFGo2HqesaBhI3gAGowvhcZpocHabDqtS4o0CfJPnGIp0yJJMGTPfmeH6POqeNwfoFz8Ebr4qUOpcnT7QnnCjOhGzk1LN4MM8+wAMF7As/wBHu2AuahP3PbjM9q3ob7ByDjAEaSZ+x2d+C1bPo98PKPw5vDh5Rgk8ldGvKksunmAP9mC29Yt4a4P8Cp1+cO3eJUWdo1Cl3JFiw4jsbZJMzDo3/P8AwLqvHL0gbnti76xatq0rRpNOkgzWAnM7Jh94fQu5+g8cWNYNYsqpU7ORb1YzwdD2OYbMP0LqF4cIrHu2jXHR51KZZ+0pgcyQAbzMDzBdTFkb/tsOKnC+j0OIbIU+vHCenh2cl3h9Gxc9r3ox2dc/EuoXNWYvM0+UyD8YI72AAYBgALusO1WKDAj0qj7IMVkGWWuwPoUuHAc36+xaiDmwy0EVgMGgAAAPyLNXnbFKuem6E6lRp7rG+MzIDMAPz4LcTKV7xtVRgYH0Il4JvCyaNYHFpquXHhGo5xps2HrHsmGGwNnzmZmtt6JcOq3/AMAuIFhsBhHqNYzOR+CBhv8A8C9S3VYFpXUxCqVx0pmYFIZfwB3eABh3guD+hnb1Vh0O/bHlMSbem1sGJsA3th6BgYZgtRyLjZxIfptm2fwPsOhh6vmgxKnwZe85J6/g7/n61Nn+jHxUsCjUepUaC89csKqnUHokFkDCAYbwPPv6FE43vU61fSTplKYoHM+qJNEYMzZ3vAz1myHz5r9A63D+9A/+OGagcnrBzvSB4H25Fu2CECPV3osq4WjPADYZPN7owwz0f415q4Gs8MqPxXplVoDDNHoUW6pRw6hOewB5gGejM169vm2KrVeH1bty2ILL02bDOLGjm9ogefWGfYvDV80Gv8H7jjzpcqic7RKlzR0QD5mNDM/n71Nw12/Qv943vaPoPzgvgLBcFuMDHGayI9ffiBDqsXwalHDsPz/nW15zcowyX0YzwWlZ0zjh9CykCSBtKzOsaMXBtRcNYspjMQDNxt881VPPGZAg9ZAeeaXMsH7xIHm/jT9844n+4psUP416DemdH0B/gBeeb/k85x4rbf8AZWYof4F3t5bCWzM/5tRWsapYCsfrOASlsz8FA2uDEyPg+wBgfYaz9w0TezLpzYBhs0ehJmsYNYaid9am97xbRaLI2QNhVsmG/wC7bUrWwFYf0h+Lv8jnCyRcETA6hUT5KB5wM+9LRtmrt9IGx7SmXBStfnJtEZAwaAw+8n3gH0LlVK9MB+t1TTr8F6jwj63WfGNeV/XD5hsbwA+v5zUd6pPsl4a4VtrxXu0L8uM65TYknPM83XnszMD6P0AsFJmVlnw9Az+gFIO53wHe4gC4X8vEcwBYNotHpsOVMkeIGYGCuAojAezl9B4daqp9eza2Hh9AKPGN84u/2XYrwpcGzBAgcfcA8OwDUpmfBeaBjoPDrWdek5+wz/ICkUo3IxSHH8PHDANijAuOgwYMDPZ1qPJZ1opsb8w9infWUTvcSOqth4bmAAfk71iI4BVYzvRswUjmTANR/Df2YI2alFe9n/tqQzPiH/e+dIEFmpbsNMwT2ebuo2+alG9B1ezM/Oq2sA/GBqdBwA894ZqxZBr5amZ5rodgUqh1J1rnpTOcLx/G2A8fkNcsCsS9motBbdYqrMrladOCHzvgmZ9GC2ix6WZrbFKoZ8UXw1qfSHn4sAGTw3/uQXHLn9f8Y69HnPv4OzWX+Wa74zAf6+9ejre4evw7XhRarWYc8zjbHgPNnQ37A/11SGzYHDqGB1V+mh6xMD1nowAcaLFPA10w53BI1sP02qQqMx/Ntdb36R7zZ+f8gLqcO9rfoNBqFOlXVGekUSAARjOZvkh5MMFzTiXcj9Nub7cUC1eTj15l9im6oYADAYAB/X5/rWNjQ7SOhm/OiVIKh+K0Hf50ttFtrZl4U2scU6ZOuCszKVSn3tkjM3nmWP0bM17Vo/2HuSkVih2W/TZMeKyHOBE7DPoz+deCuDNt0q6qvU6a+xUpNQ5b7hEg+2knnvAF2Xh7fn/o0netv8UAept4erWPUkcA1gMMDMN/1n/jV8P/ACxem6ac6M01EfiaIMAAAGeamm84uZejxfN28XqRVeIV3UpmHEMwi00ADrAOs/1rrHLBgvoMkQJJ5qQE/DvUeTAfz8AFC5ad+GuW1bWpz8+9IJ7Ye8VOfNh7RV/Pmcx2DqeKAZ4d6g20smpN7fE6zwTHrJvzrCX5M0bZdnHzmkw8xmYfOalUett1ulhVYofd3zMGT+hDbUesnPdqXGn5+8WaB48u/wDQrKHnirepcj2qkRqbOeP2ezzprrPerqHWGGWNDT2Lp3+jDDH69q+TAfh9eH5ELMlwPZrUUqTTjd139+ALSybVtg4oVKJBZM3wzM1zW6YhzcKlu9opZz4ml4hp25KJBgOgcVvsWaeDNc1tkp6pYEbeoo5yWzVbJB8HUAGrhilvb/EUF4M0efwUiNGA0EQIyM2X8NNaBmlbdRtNHAc94i2XOAaQRnw7FehAwNO8nh7tBVQ2X9LoNS4EP1kB6bZgfYHea1VvG3Dd1OxatnQMNRtgFI5edt1nsgmqeSBsumw51guyyT2aexZ+pUqI8BuNxWdXDrXmxy157BV7zy0VVpRxvdrPyWd66EI+s5/UhOS4gPoUczXqTvOH/Uh5w/6lHQdCpCXz5/if/oRBUnA9mar8PikCLXPrh/8AES9ay/xFWIs/gvcKWZz3MPaJHPcNVQGnQTAm8y4aMNQ1HBTWVCSAH81OjMvmkyG9WDIICZBwFYMm5jsSjM6ytoEBjMFF2pBDnlOZCo97a0QUeJ2KzZogBH1HHAWS2XZZf94ndFaP1awHYmnqC28eo2grKUeE0Fh7t9HWwLnuCRX2KrMo7srfJZiAGBn59/QuinRzZPw0uTP+tBzqm+jfYEbNt+sVifn55OGCqpPBawIfF+lWxyMx6lVGmm+cd6Tn4/YupG9yxrBXnVX43GSyn2zw14ZgvMKatnghwdDw/sJTT+vM/wDOvOlngxbdRqdDcihMiUut4A090YL1AzVW2XdRxxeYp56N/wB4N+eeb4Lp8Qw/L0gE+1YefK2rRwD/AOjD/UTL1yYfsMGNGDyNBgqKMbkmBHlOd7IJLvj1ocV+zazC4Z3Majh5h5Ey9UnDLUB8/wBar0K26Yc31Wso1ScA1IeqWapwNSANY360NvqWdN56TI8Bg3sOvBapmA/paiztHDZqZ4fnVgbz4d64+aMOzhtZ5tgpAT2A71lJNVcyw1E0FVw94sOm22/DYPVX8NQTmZqiZrbYbHHFNjVKCZZ6iywbXcB6U8GAZ/nUt5lwB8RV/wBoYgd6ZeuRg/fphez0mY4yarJMxx496ak1iIZajhqverEUD8M1eEJDwZqC9sTR1iLh7dVkyqtvezfTAsAn4GnjnhJawA1meZzL2i51xX40v8LnWqbFo3OSJQZsnn0fwfOmB1WvT6db1Gl1yqytGPFDMzXhe8K9OuqvTalKqMmSGsYMg89mAArC7eMF8X+BsXHWDOOf9HDYALPwIb8kDCI2BrITmbVg+qee9eAE4A9iar+Wlh0aP60Unm4zpsS28DDrUfWULHMh1VkdfXD8ihHVZzPvM/rTs+S5DYNzefyGo9EgV+7ZBsUCjHMdDrAOxZxYaeqs4/k+hbqxmeE8+lmd1cydT/s73esVXrPvu2GgfrlDejNH0GqoMz3q9j01avDfhzfMfUpVuSaVIA8AzDAFHn2qxQaydDb5ONLDoORsBcSt7ivfln7KHXPu/wCE6GYLZz+P068BhN1y0aa9NigYBIz/AMiQJHEWmzrPrMT1zKjSYUoM8455p07J4bXnA9eP3GzbwB1ge9c/uGZLqszmpewPIg9T1WZhoRNaP9a9E6ZJnWY/6qty6gnwuw2TzBR3r8ud6nO0NzlpkQ/cugmWWfU8rkZVNjRgPo7801VYDGk6+wZhn5DVQK/1VO5PmsNiCBJ0faK+t66n6VF5R9sHg7wNSqr6jreE6DTQjSz68OhXAu6DM1oAK9ptenW9PiVWC5vA8Fkre/AV3UmXOQNbb7B3iTArkwwnPxfu74Z5rnN/8Ga/eEw3KdVdHDr5h7AFqOCfEty6oAWrUYpsyKcGAZ94Lp1bkwTgGbkQGXewwUjj9jWZWeQl2VXK5DezZ9yeYGuX8SOC0625UhijxQMGPdNHma6a9WHKbXvWQftAKi4u3s5POn1liDMjSM8NYD2LW4HMaPwcu2TABx+hyTPz4KF/JjUgkPN1IHozQd4AvYtscS6VA4aNXA+fM4e56F5t4i+kCF2yjbo1DCG15DPNY3A4/W6C3AlBykvmY/nRUeS/SpWuxgZ+Q1On1LnzNx8AAD8ioXngz2KMCdPCVJka5xDZM/Oi5OWY+GwGH98ov2qrOlyvPHo+RHAr0GNH0H6MEk8+vWwV4W+HzcAujBQnpPMn4+9aWHoXU76qo9GPmz6AN7rVPcNsVy2JAMVymmyZ9GG9ZIRecYBrTbAM/OgCTmGGar3t5eGgADWonPM7s1HCS4Bh5EQH7tw0J6YO+zUjZQ7tY9THFlUqHJ2YAZhvBZLB/HYYfQjZ0wTrOmZ6feqGz4XMwalPCm1Kmxnmj+RdgOzLctt9qWxQ4wOmeC5Dw3CXDrwOaHhefyLqE+e/UquDb87WwPMNi2HWGfsWYcjOtzljP9zmuD8SLAYtK6o86lRXjjzegABejoz1NrFuNRDfAHcMAzXBKxR7jh15piuVIJkfWDfn0KLhkwVVenZg+cEw+tVj1biG1oS4gB9C9EcSLep0myJE5uCzmHQYLz1AuTk4+g5R4z38C4OZUWKNMoANeO3JzUU6k5qnyMQ8OzNVkme3JkeIwAfQmtZzPYeCzWmsnLCaBrqFKNs6a1prk7L3jhrmZh5F1Cgn/NYAtIgSz1AUST/w5ipBqO97I1eFslPN/nTV1SpJ6Q6ihTwzLUR0o9oLaBxEHmD9o5gngNt7w2FoKPYz8+13bjM/Ye5w3mqeNRK5Ga9ahBMGg61aHbeDkB+g0vnp06Y8GebMFnrNb25LeizGAu6VBNmOfQ09/nXF7V4nN0pqO/WWMzi54MxwXSKlXr/4iwGpzEGNRKSAYBrH7ZSMrW/sw8w6/Uabh5AaDYCxjLMGfs62u8FoKwc7mHabVaxGOOHWa0dBep1Bjm2cTmfnwzNAFpQ6zDaCXQ6HGhtB1yJe8FRV6q1yTcBufacDw/s+wPyLShxClwyNuVQ8KefZngazUkKNUpXNwYmAfWuW0GbkZ1nWpcWIyAYYbO9RYFNl1hjXiMZgClVJ4GYZtm4vsO5KjRKJytKYZMHPOuO2zIzz5aQTD8Q8wUR6YwfuP41YVU35PiP9Zr6FnzjDmnHwAEFdG1Adzb6VIzfM9RRXo0pnYpTNSbjYMSwwNWhqLeZ1ndBtx4DPvA969QcCoEqsQwYcpUamtRTwN7Pea8q0GtwYcoH8+hdqgekJBtizXX2PVQVPDBloNi2hbD+k5c86t8QztV+VGehUT2Ol15/OuXwI2chNRjl3DUpFZf8A2iU8ZvKXzPIHsYN53yAuna4Ta2GtANHGrfr63qZw80MI7j3jPZ55/kUqNyPKtN3G5JZjn7ZlnvVVdsOx4AhKtKsSTd/CPsSLeKG5LbbtuovQXAzMN4H5wXaYfpG1WpcPolDrDei7FZCKBtZ54LhMl5+Seb75n9ae1tGAAedRtkvZlb1pBm2/MZ+t41qLM4qVWxjd9THGkg+G8JDOe/zrlmRqXGPD3Zn9ALn35HSJ/Fe7ZNbO426ryzodDMcMABdDgcb6rXmNSq35c5h+EyGALzic9vPBxaNm5GIdvusMHhrhgZh1gvN+RtY1/wBOuGPVYNNobIRwDA5xh4xqLw9upiz6dUJbGD1TfPAMw6Fh6PUnLYoctt/+m7w+hKNUmMMz71tFmHRY3EiowI8vkaPGA3zzel94GsJJmVyt1n1rUZz0wPxjNGzJPmAbfbwa86lsyaVloOH1rFeEGTzehrvueEpf2hcmNNaeYGCr3njntOxcwAGOhV8B5x5g3G2+g1Q6RTbkfo9LlyqUxGZqD4YZ4LmnLVWtvssc8DLrB5nn519OpVVl3Q1E6GoHjn1n1mvR0KHVaqbXqapcsc6aGj6wkdYB5Fz+vQDpVRdprkoJOh70Og09Uqw5PFrXc6AwVZrAavajtKo5zB1zfZZDyOprk2A3udCaN5IJIGWpqLQKe9TYcU+sz7A+dZ+T7Jo+8wWiZprc90wN8Gdme9RY1N1pAOG2BtB0b0Gc0TP6EsGwW1qtEOpNHpsAAfhAs09R50Yc3ILwAgl0Sjzq9PiUqmt5yJR4ACvQtuq22/XWKrRmT9XM73ZEY8P1pWNMnUSvUysU6j+spEIzMI59BrpfF3ipLvC1/Uc63DolVcPOSZmGDzHeCtkx/BClUf7b0SuBKB49E35gB0Mgra53/Xo125+VOTHOeDMOWb29kFP9HWghGKsG5TmQaeDMHTZ3mG9aP7FcG+c5pyfGD90ck8F+U/insTHN+t9r0fxhCkx4lz3VTLZrcuTyEWjg9pZ9Zh3qkoV4HZkCqsRH5PLvzDYh7M8AW4rdI4Q3BIB+q1mGZgGAYScNitYdSsSHTvU9OqtH5TAw0c18T6+LGMU+r44oZLgi8zPpdVnf8L5vSDmeMZmq8+NdzNuutscOjeaA8AMtbf8AwLotq06j0ikx6dRIoMxQ8gdZ+czTtSueh0KYcGq1yNAkN9YSDwNcd8/D59m9cO17hxu8ONl4vUJ+CdjvUXX/AKXvDBbfhHe1fuajS5dfpZxuV/peGASVd1K7OG1YaCLWLmpUwAPMAde2ZqJcN/2Ozb9TNi6oB/djBlkD3meC7a6ubi6eL1vwi9fv24ZfNH5CuSJ3Nsvc68b2AdihT7h5xqkhFiaLtO9s9hhrGqIHnz6zzU2ADbzuh3mv6Nwf048Py/J/y958B7PsS/OFQcd7jpTJ1Vg32WWWQwZgAzs2AHf8683cWuLsrjSVCixHzjO0tk4rz0h4GczM+v8AgU6zPSBCwOE7vDJujPBUMJQMutZ4GbxrR8BPRmti/OH1Vvy8Zz0mRgbMOnx3tEAfDPee/wCjYvEu28LrboFw8I7Ul0OgUqTXcApcyWzGB55kA7zPPo/115Ev+H614tXRUtD7oxPMA+RdOsC0rg4FVGPcEGsUpm4ADRk0bmcwkgfea5e9cL9wu3RUZUQAzqT5mbXRmZ7ARlFukejxc9q2lddx1W8aiEOhHR32cPxs+wAXDHt78iKDBsgw8ejmHZns/gVmc9g6acSJ7XDmge8ilVKfEuGiNBWWNGWAYcwyeAfnXOOoej9Z86BHDiTUYsyNLYkn6hddDCnz/P43yLvH8gkS/LhdrLE6G9EheOzTg3nr4ZnmH1rk/Bb0urx4e2HC4LVW1aJWLX38m7OAz0Q69mHzrbW2F8W3w0uPjbQ/WrMh+Gb8Z0DCTFAOjA8N+aeBcRuDNcgQqhVaras+NIADOS9zOYfIGAdi67w9ptuVu0morcV7B8AA3WXsA2fIuZcOvSignZsS+70tiSbVLwpdYnRJPnPADNnDeHWvRFpUSx3uGjVVtWq+vqFN8aHOj7zzXR4XBQ/BlaCuOQfePUbBY+57ziWNFA/sPe1b/wDzTDN5eXJnpV02/PSMtIKNBqTNuMGcLlJ2cZ5mV5zDvww6FpcYLaL0hOKN1ej3x4OuWdSox0+vUqKcmPLDNmSYeTyLq3/pLWq9b1Hqs5sKbUKpG506Y6eDzILmnpk2Y/8AYqq8ULgiSalUJ0aFTKCEfeEZgN7z+z8i830Sj3NxRpIVW4IujNpdVp1PPNnAzB4O9YIw/R6iXDErcBqdBfPeGZgYYGCtYx59aquTg0cAixehsADPz4BgpEaYGftF0QyXGsohst65uONoAmB5094bwIoy8yw8Bg50HsMFH9T0oJUSc3EZ5uEybMZ0NhgHkUs2XAUc3gD2iseevSo4CP8AEiUF8Uqserahb1KzZPMwzw3gu8WrTa4FjWoFc31AKJF5k+/PBTTjMT4uD7eYGiOpVH3bmayH04b7PYvCnpP2GxValeF3U2lSXqgFz8qeiewGAZDsXvUK8/yfKS2A/Ws6dBtyYcg5VDhvHKPUMzZz3oPNPou8KLnsmAF3VU5lNOvRjPkcNhhn4Oe/6/1r0GZue8WwnssTwBw+sFQzIG/w1QhRp7jIdaM6xn7xH9npb3s3AUGTR5bJaZ4Zq8AzqThn4aTMx/VDxFTnGqIHpp1kJYOhqAZ70wlw+vPczx4u1/8A64H+ReiJ73i4LzaBnJ4z3b/+dcP416CnyfvhphRZGjBQjkt5p1k9/tESm54KQzJwWXvC7abZNvSLjrDck4kX2xshnh9a8+0r0rq49dpywgxnqFKPZEwDOMHkzUbHrOM9rGDa8k+mTxRsq/4dv2rbEtmedEkvynpwHsZ6AwWjsz0n7VBiWxerDzMirnKfDfsjAfQH0LyTcM+IyU1uC3g1meAeQM+hXsVTzzcmVyLB4ZqO9pwJWoc4Hj+voVUZmGbgOACQRp0xrCIxnh865sN/DQMyQeDNzlj+tUlSkxdXTDrP9CMAOMBsPtnmGwwBKNJbjdbYGHzqIh6jgD5j4aQanYatefYk011zlMNDokYdB9ikVKTTZg0+o6Ek3YsPQPV78FoK+BP5Y9Rvf2J16e4Y6jef5AVZJM3ndRCBuAWnqAAfP0LzClxz7YOhm5n9akRpMWNm43o5mfWCqmeRZN1vYfz9ijmy2Dubai4FxMBwHeebPYo7MxsxDU2H3gmQma0c2H3E0cbD3mxXgXcapRQfB9wAeMPOroJMSYOg42G/yLD56Jq4jTGzaBtvPP5FjcC+5CDm1qN54GlnBhyDbcAzE94Zmqlk5XtNcDDvDNS9ZuS0GbmBgfQowO10Hi1Q5PDSPZ06dUoc2kRjONI5nYZqkvPiRVar6kqVSprPL0TZTTkBnrH5z8+GzYuWazAezWoonEK/7SYiXrbnq02qW8AGciGDzOZ9mBrphNwevniRUr/nsyp1VOZHi5nGA+wzw/1FmaJAqTz7rDc43s8396kXJdsu86y7X6lBpsOQ/wBYRI2j/AqrnH2RzYcwNeIavhvcP2Vuin3rymt6uPMA1sN/ZvBaji1xIqXHjinHuM7YkhoRmIXKQXtYwYA/P+dc3Z/aJERvfHz8F3z4LW2ZJqtsR2rxps4DqpvP0yHTI73jPGfnD9GxaeLLe/eFcOJD4R0Ji3KNUoFPCNgzHqIYSg3960DIGHWo9B4Y1LhFws9f1y7vbxmJsyPU5/7MeGZgGf8Ag+RaCBZlfqtGj1yC/DmR5QAbOD3Ya+n4cFhjG3oApGzFSqbw0uqSYc8DMZr5HszVwfD2VADwHHnvrXLbbiYeqwH59OlsMN79EzDDrzXEqJfNNk3rQqjVYhw5b7Mpk8/fBhsNekDpsoJBsBBk6vYGC8L3PxCiw6tT51Gp384U+fKA4MvrZPXzADUNrh0P0h69WGa5TLKpQHGp/JnV57zodnzq/wDRyuGDcnDePBY2SKW8bMkPnPfn+feuYelXeFYZvKEDAMhIlW2EWYye8GdY9+C6B6K8yJUrXrEGlUbkIsJ5hkM+szwzPNEYddCN/wA7aWGCsPVRsjm4/wDoUSTDfzzw2K1ohvNgh5hxI4x+RNYfFEneYcVxDvCqw6adKiYaTh55n1gqJSmdBBLn1iW80bj/AFqnenv4q1Nlh4dNGFBB4FFwqGf5zPrTRvLRhb0TV8dg/wBacet5gI5vsAGzsUbMMlrOf1KXGn4Gk9DYMvZ7/Ionq09/iACvY00a4WwDT1FI9axHu9YcwcZ94iCe4HvEGxB5vNWUPQewbNYxmf8AOriHP0T1NRZLabRBk9MFYM1IGWgDUWUeqvkcUc6q4HvF5gas6k3l1pHPY86xnrg8uvNH61+dMC6qUOLMWfmWxm0fiAngrHzqQFSz94vRlztt8yNvBVNSo78D24YAugbMkqlDg1ijVCDOYzaOG/8Ak2KtjlRs4JrD4rVcFrkg8QuF7VSqLbL0uE8cJ7AMN4H/AKmCvnreob39EVxaLhznDYlh8V0P7GU572Zmz9CT3CgDYN+nVzN3yOs/7avaMOfpYfFaCpWTcFN8RyLrB+63qq9UVX/4bJ/Qr2tHw+KLD4qQECcHtIL36EuWlf2R79ChJYYKWzsUfRl5/sj36FLjU2uTCwi0aY8fyAoEhl4A9opAPJlmg1k/bsYfIakepJwe7VidGk71cQ5Ie8cVIFHqWfsFNZptSD2jH8axGuh1LZ7dXsOeBhg45msPDh1L2jjGH51a02e2y6Dbh7+9FNnh7xLWAFWevogNaeYJg62x7tR02vshbvTGMFXnPDNUr1Yz9mmecz61t02x7oS5jPMu5rlnFeG/Au2xJz+wOZwM11AJkXzrmvHJ7wLandkWpBmunhhF+W9kxn+Y0G1wriFGcpvFqpsPhhzsMH/4F3WZPwmG4C41xsP/AJfUSd3yoZgf613/AIcrpFtg5JtemSw6eWBSDBU9hvGdnx2DP2B6CuM/grhlZYfFAlvQZyvdsLbbmwMAR5/BVk87tjOgcGLA5fPAweVgyFVePTOKyH51FtFxANw2utSDNgGuveoUNl8B3uAi0XPeL5t/3Ldcef0IT3WgVm9DbwRxqVF0tc8zW24wj82rOWiPdbeZprkGALPerg4eAbGwD86Z9WvmGbe9cdtotWvG4HvFBOY/+IrafTZQNeICpHoxqG5o5m7wzTPMOIeWfy9miNnAd6YDXMOJayaM0ybzapCWD2BrgXHWGxcnE6mUozjSWTDAGdbAw/3wXcNZeLOJZ3BSuKFVqXrUOYYewZMAzwBHm3S+JHByh2Tbx1+m1GS8YGAYOguSM1KdzTXKAYefNa2m8cq4DWnczEaquh7F0wwMAR1jivBuSluwZ1vxgdP+kM9YAsrh2RbOVWpBMaPm5R6vzqijVKdAka7B9CjhgBeJKM/rQTJLANLGyFm9cJyfElYZfIC6LavGO3KVAOlMW48zrhgbzJ78/OuK88CQTAz8NeYW6lfnEidcmlb/AK5nnTGMN7vWaxsxltljUiywP8iz/OOPF4jikRs5L4NuPgy13mZqbhBGf4ilQJPLOg4ttJ4UUaHS/XMushPa84ScFkqbQZVYqkeJQ4uemeeGarAtXpmsOemYfWrWNPwtoMIknVDoew2Jq541VhtGxVaVyDvkStK6p1NodQoegBg+GYGfYuQU+DbzQPy52s6gMH/cSjBUR1LOQZt7M1NjSTN0NM14LAKrSgi6D7EYHQ78FWBVdF3NjPYHnUCSzovnqdeadjHE/pbeYLeIHQrSmMSWmp3ea2AHzMd0OzBcyptVp0lpqKwZxjA+jPrXRaIeZA250mC2gY96fOt6uesqbOehyA6DBdIt70hK48PI3AEOpZ++NnA/1qE9bFGrf3SUw8Bn39C5vXqbTaJVzgsS88O/BWOsRq9Br08332NED6AzzUf1a3W6bUKB8+uyq+0qVUplGOXRvvOHtgDrUiiHVfWgPwYh4BseM9mCvY3HCixolVtyRTqrOk6rAGD0fPYC5/WPRguCZVHX7cqsYInYDpruHC4GIcqq6jn7UGa6RR6PSmaRqH+RGTxv/wCjldsOaEGqvwDA/I8udXVY1coNXdprdHk4B0bF7th2x67mHzUR4PJmsPUuETFVqnK3VLkw2j/ZtI/bKFRbxVUqDWKU1ry4uGarAz951r1xxd4e3BbbH3Sx9alB/S4+8F5sqUaCc0/DON9YK17QqJVX6PNCcx1gthXuLtSuGjep6qEY8+s8OtVMawKzPp3rKnYSWvkWYBlg/aNqLEcwxToQ5Zsa7beYKQcZswV3bzNuSaW6xL5xmWHkNBl3gMA1HAMEGfwVhyet7eWePkMEy9GBkvDQDDMMXc3OvoRgGZaepgfnTJsmAaiOHGzf8fo+tB1OwHqc9I5GdKNkz6DBb37JMPVJp9i7owfI6uSW9bb9Ynx9Csxoef43etnW7VlUGO0/zcaZmeHgn0LaLZOh1U36UDQRKwy9qeTsXNLzqs6NPBzmgMz8nYprNKnez7O/eri7eGjDNtO1yI+88bHYYIg1cPEJiZZHq6DKwd98HnXHwZVqzJinTtNxzxfIjpsOlTC06jmHzga5uaNqhnzZYCUYP54fQrsKPZfK80/OksuqvuGHTYcr+bZZvB86qT3rJbXUe3rRrDRmxWTDDsM1raOEUMKdBPMwXLKCFVOsx26axrH5F2a2KVOZfaOoxAjO+QFazL0N8DwcBNPQ5Z+7W4+zDb3jg+ad9Q4KFuWz4D4NH4apGTONK3rqtVomZnsWCuem8sGbbfQtYYsl/KRLgbINDhxo/YAdAfnVTatsX/xUmVCm2jKprIMHmccHsAVbGu2V6r+z4UNkzf8Aem8t7wcgVXhFxVpUq5zjRo9RjGesb2zBdK2EuSz51BdwYpUz1hTnsJJ9YZ/Iu3SY143tYtMqLjGi0wGbzLJ9i1t51Wx5N2zarBuqlHHlBvZZ6DPzrOSbqozNGdgwa/G0g+dRfr/9XJ3ORXD6qelNRWDw85rttmXJb9Ht6PTTiMycO93rNeeq3Ppr08+RPoUuiXbUaIZm2APZ+dcfc61xfhv+vneVPBryKspXNAOo/wBCaOq+uJpzpx6IfrwWi+0lDeo3qqLBee/fKLvYr5Mxgwa8MHjz701PNwy1HGAjfIq955jMOVDDA01UqkxGIJct8/kXMsqlJbBre4rCmzwkxTbcfWEuSsOPOx9PMGgPYl6yzPXY7OzsTA1Zg2y6eZ5+Q0DzMXDUcbBSHtABAwwNow2b00ZxQA9djMVIrDgPyXw5TDA+8zUcwl6uB4H9CT0xsy0GP2fsBSKOYHNBhzZn3rogXFNh6MdWDMOnQGjqr+90P407WIEpmkGEUwB1ZKTnAa03zM14pZ1u5254fsOjh5FkXntaQb7aOfMzaMwUT2MAPOa0gSwBzDxOhWEzkZLsTlH/AAmA3hgqyM849FBv51OZgTg5vXiGAYd60e3Bl7xp7uAfd+w1YW9Um4dX5SXhy7wYHsTUCM/0S2zAPOYKwOHSmX9Tr+dYXaNMvW2YrL8hyJL3gewDQU3m2WglAB5grCts06SYOQd4B1qneecA8G3MAVIS3qw5vccYB50/fGmmXn3gNvTMwPvBIIfhZvuACdAGAd0GOpatWgmaEnS05fQHQaQMsH7R/wDQo8bDVAH2zzSn+C+bazwyWcMIhu4a5gHnBTXgp1vOhoAB4efvWSzfDocRG9rCDb+ZmmBdyYbcl033MIxn3qObL7MPxz3qRJNw4QeGBh86jnMzHx94Jh0wrzNv3aDPDPoTT0zN08GwBRzeceM29P2ferwzINQ+jealsw3M+hSIcPBoJen1qwDQx7Ad+deivAGHpX3vPDDDoS1nIwYArJmHFOObj7mB4Hhv71Smy+9h0AYAgso1SnMhqZ7+xRarUnwmO6Dn3Q8FHhhVZJug+4YAx50EkH9DU+dA0Ex8Hddh82T7MDwUiY9Ua3IN+dKekumGG9RGQVrzjeQHFiaIAGHXmg6BbfFes23bgUP7MRnjYjcqEjWwwD6FmqlXn69Ro9Hcp0NkIp5g6DO/9ahczs03z+sFB8Q+hc/48eXqqeprEOrR5coNaODwZgup1ILYqsCFLt/fNfPDSDYGC5/GPmX9BxbK3g5ytwvk2LfPh72eXpCw6bF9Rx2NDeC833yds3PfN0OVyc8y6wZmyYd5h0L0xQZPqqiTajp/ssYzzXle6qxTq3Hj1jZzZ56xgzhnvTPj/wAMey2fCiRJ9OanMMYb8NID2Aq+YHqph026U9pPmbAG6CsIEmJyoNtv7/J8incRbzi16REptv6wUyEzhvDrPvTxweG27Ydk1ZU2G/UpoQYvtT6FXwKbOqs8IMEMzNdOpUyh2fCCh07ljm/0moSGczZP5FtcYczIybYlQHzCpYcx34HmvsCTUmdIzuOYy0B5nBaewNSnqkcyeD59b54GaA6U/DqwVVve0uVrFnZNVCt145Z6zz3ebzxmas7VZtl52Q5Oqpw5EWTmEfseWKpsxwHZr4d7ym0p59mVLfixAekPhgGfYtItCdWKxUpNZmzpRs4GGHhM7AD5FKoM9iS6D9V/ZDAwUWSznRokF/2oHma1VqwKbJaplvyoOebJvHvSx0DiEbEDhZY4QaHMjNW9GDmZzwYA8b2ewF6L9Ce3n2aXcFuXHUXpLs0ItQCjAYHFOL+MGzZ8/nWXn8E7Z4x8J7fvFiuVKgwmMPWRvHrBoM9eAbMz6MFyfjT6QMWBddv/AMgkSvWf9l6V6o56WGD0xj6PJsUcMD2Xxysli+bDO0fVTNHjzs5r0tmMAGHK78Aw8688hROPcb0UGrKsC3Jj0SbWPWEadRp4GYRd5mwfezvVtw99Jaq3PDiWrbNVo9VkPnoAd0zNGSb57z6F23hj9v7Yv+oUq5qVR41v3CHNQI8GTzMYHwDewB4dZ/5F09K45nhfhpxv498OqXcHC6hy6wFdr0/PSl7zAzDfhn0H0Ld8LuA9lXtwEui7X4NVpV+2vMf1pEs9HA+sNn0f41130peDlv0eiVviU5KmVK869W4rFEeaDROMZnsAMOvo+Rdd4P3DaXFThY6/X6MEOqyv5vueDhgfOhsPNRcG+xq7GqUjiRwJoVNrgGDtRpTATAdDfmGCAOBtvnKuCc+2B/aGtxa1JAOjOKGAAC0FEZo9BpselU43uXisgwzrHmeAK+ZqsUx9oowhTzLegzD35gf1qOFqxQ6JRgtBmB9CWDfvFttDPnaU7+iywM1LjWNdWGbbkMPrNW7MwGVL9fOB0GmxnJlsXdGazcpoPNecHlTnTai8fjxDA1tjuTNjTcPeqSTVcz9omxT+pJQBsnYfWoj1NrLPiAbLwfIanPVJxRDqr4ezcWwiYTveKOZygLxG1OCYBl4idNnmQ1GzQR40x/LNxWoMsPNa7aqg0wUc5ODvtMEwNWy9yzQaaAIEGYfjtrNBVXz9+nQqsoPZuIL163qbq+HvBL7KsG61oYBvVEzW5wH1q6o9w5z2gfNB5CpTOtxmu3/88B/jXp71VR5NRNuVsXmGzJjc/iTcD7ffXv8A/svSE+ThPQaCNQaVDd8CIB/WrALbt94M36UyZrPxqq/mDhmrMK2ZgoEe5OD9s3tRnaMEt6A7mBgetsz39a8CcS6bVeGNpVCyqxZT0CqyqkZnU3g0QeAD2Gyf0L9DoFVfCQDi8jen5D5yr25O09HXgYA92GYH0fxmskvNQfYeBKwfrk+ZoBnvPYZrFT6lOueruyotKjQ2T9yz0AtXfNEty2IsSDTqidSqdRhsSjdMMOTM/cLGQJPIBprOLapD0ClBH0JTAa3eo8B7R8BvMA+RS5MznGtPUzMAVIE/RwiaAYAs1tHJh02f4ge1w6wUQ6DEMN7hmfnVZ610R1G1ON58A3udYINBR4xzIU2hxQ/bcMAzAGQMM8zNYwHpYOuxJzeEiKeDweQ1dw5LYSoktxz9leB9K4aqxMm1M2ADl6iAZ4BhvWozPMuAexGy9hI13wz3pp4DyQZ/BbB3YDpn2mkZ7E0GZkGfQkfWgWeeH1qUZvnHAPnTABir2lHEN1oDY1nT7EUqtE8NRxJkzDocwVxMhsA08+wweln+hU/es9i1hzNum44pck9E2nw6DBVMaG/JHwHAzVw9DqTMU2Jbf0YGpD0PTme0Wwsaq1KiesH2LcerFJDBmpMh2b+tc5psl+M/4gPfnW2pV21mmx5DEV/BqaGDwdhgvIGiv+fwLmWLbj9nQZNNut9435mDxvH0d+a5kEnMvvzi1dNptvyQmzpTgMzQP7nmfYszW6PKpVZGJOYNnmmQeZzDA9A+hepS4DPMygbpoG87hhgALUTKJXOHtepjlZgshUGAYqDIA9mAfWrCw+IVKsy35dsHb8aedReN85cg94Hsw/RgoV7VWo3VVHbqnPsmBgAa3QZ/QCsbrijxjqvEKl0SDOM3qfS43jRD6Hn9+9e/vQtmMHwUo9cpTGi1KzA47rJ4AYbNn17DXgHgCzwkZG4KlxDcpsmaxAM6VBnbAN8AM9h957OheiLe9L24A4KUKxOEliSaxcYW94ztMjZnRzz79m89iIw90TJLmeuGxR3qrybWZuYLypwh4s8aOPR0KvzW3ratK3gcjVIzP7zW5wAGw+jAA2fr+Rd58T+0H+tZ7IhoK9cMWHRpc6ceDTAZmYdeC8OT+HtuV7irIvG3KiAa8wzZCWex4wDr/gXrusSYh0ap+sv2diG++f0AvE9HvOgVKk3BX7H5mpSItSms0QAM8zDRz2B8i2ixxni1dV43JdYUOuVX1l6kPlebBnBkwB/rzXfvRLgRKJc11m5LM3ZUBiUBu9G8z3/n7FwS8DfZ4fUSjSnOWqBsnUHu/M/J/Gtxwu4ltw6jSrcrlSh0emUh4JUx4A8aZgGwDNRsezT1/fpbFS0evP3DbkKqvxDjHNA3wAw7Oz+BO5GumEJZmGaicmBnmlkadA0EeZRz9pEww8hqvksvwMNfDersHn1ErYHPig231h0K0oQTMFaxqq4Aabaz5w5QGjZN9lYqhpTmOPEriADmlvWNZmSgd8NxWUaty2T3msmy2n2fTZkN19jNmXhns71jHqVODw9NdIptYg8uBuOJo40GZK5rUzQckn0qs/gH+RVT0Oqs+IcV79C7FPOCBeG3h9CrzeD3arbFyfnKkz7SDJ/QnfXEr3jb36F0ZmSDLuogk8iZZuMB+hXtbBM1sz86P1w/5DWlkswT2aAGmvVtKMfEYBWhn/WThp0JL5j3q19VRAPwwRhAY8iCvB5xSAkmC0EC2+f8OIGZq1Z4evve8ZD69iyWxoTHFK9fMQ6XU3H+yA+f8C0x8MT95LD8iiV7hWDNn3BWdc8ItKmmYdfuDQeYvQ2vNs7evClOHsCeE0PoPZ/kXeguGJn7Rebf/wBnvGoFb4g3HY9c1s6vTQfjGB4B4J7/APGC/QilcMbLpTGh6nCT88g8zWcDkMCvQTw8RaONJceHYtxM4b2Ubuu3RsPkA9ijvW3TofhxGMAW0WhnWWZZlqamH0K1ewewcfbBTQpTaCYywy1oYb/Oo2FMprcZoDbiMmBh3gq8GYsktBxsGT8gApDMx8/APP8AIq+YejI1PepsW1KokEJoOGwB/WtbGZYDwzYZD6Fzf7Q6Je3VhDvMPZm4vRoKrY0GsSjlhLwM+zBZ+q2G/SmubcY8Lzq9ZrzBtAeupDNVYe8N9/NryLzY5u8bB9Caz+C2Fz0qLWMG4mEYw35tAsPJ1Ib5sH1rpj9bG/0LWNJBnDNPT41Nqoa7YBGPzgqFl594tMMzNXdKCIcf71LMHQNLjrItWBQalqnyjesABnsNQgeW1CguHC55icH0KXTZLjMoHG4kYD7zAOtbR7HWi+FhAP8AJ9a+63zgur1Kj0Cq03m50EOY7DwWdO2KMH9BD9Cv7PCPm8sTnu9oCw/GwNayAf8AwJ7Brtn2Yo/4B/rWL462xTY3COty2APVYNgwz+tPp4T50VnfAp7/AJ4wGuaceGeWkWvWXPZMPGwf58F22z2aNUrIoVR5E5OvDDeH0Ll/pPvU7+T6nsNyo3MRZ4Ho55ma6fp4T5kjhobj1LmseR5ag2WAANeVGZ/vXsFxpniFWbJPOj02NJCoxgM9U+jYCx8yq1K9rh9Y3a2zJ8jTOwABcd+/w8bp+C+e/wBDvtSvOy7edBiuXPDjGfkPP/AqKfxytGAZnSmJNSAPfYaIfrXDAt5iNIkPsNhm+eeBhmngn8sWnKiPM/3PQuP2f43/AKPpet/Bf92quHivfFwzwfpUr1bHA8wBnv8ArVtD4x3/AAGACq0qBVf+x0VjGalRvZxJeZh2GGBqWzUje6H18r+bczv/AJP6zpUP0gaMDoMVyzqxG85smDwLUUrjBw2rDWoxcBw//roxsh+tcS5mcHvEv2n9qfT+a8x/KPU/1d7PiRYGOy7oD2H9nezXPLzvanSWpE62L8n8x7mIAYAudPW9TpJbNn0Jk7YpwH+1yfyPLb+ZR/3sf5LwtNatBp15xZp3/wAQpMCbmGiDp9nnWMmSbn4e3QbFAup6fHimBsvNPZgf5FNk01+ZHOI++zJD3ObO8PzqnZoNSgH4FOMw+Q80+/iefyd0uq+kJfkaPEnOUqiG0YZm1ngpvDTjx9vLoC3K5QwgOyv2Y2T/AF/5Fzc3qUfh1mgMxnT6HgjGlDgUo5rR0auRo0sOg2T3gvfHuuf+VvVEmBybptqvk0p+Z7BtcdgccuIVvP6FSuOj1sP+ts7/ANa2FE9J8NA/XlqwDw99THt/6DXb49zbk5vVwvXrYrh/0RRPsZc/9h/jUiN6SHC6ZIBh+VPgO/8AWI2z9a2NN4i2HWMApV20p4z7OZDNe97Dp8sN9jLu3uMRMDAM8814Ku06rJu2sP1VjRkHJ3hmv1NPPPw215H9IT0XTpTVV4m2dOeezPXmU7R6PoNX3ouHlU4b73s20mQ0VYU2HUpODk5/1bEf9i68HWCsrksn1U1HlQa5Gng/hvwwWO20MvJ/duBmpcaTTgY+9Mcy75FCAGw9v1ojeYZ6A3qVwPmYISNf1UH51WPA2GbnzqQcwz9oorzwK1pDINvNeHgBglhn4bmCrDNwPDbcSA38NNxxeWhYMsvyR5XUM/kM1NB6uWrVI7jbhxnesN6qgk4dDm9S4cav3JIBtiK88cXv8iv/AKLbi7bknV4I86d7gMAWfh1KKAn4nWGCCqz2GWvVr8vN0PkVfGZbNcHlCvMDB/2au4xt6TR4YIpkCKbQS22Jmff92NI2WwjtaZn+heNT0xmLJIJRzmQA+sFUzGYgO6dOf1gVh6tqUlrCC2ZqL6tlRnd7eDode9dXrMhRqU/gD77a6xar382tLm8w5zIg3yMkPnw2LoFqwKjGpwOTgw8i6RuKPJnBMBuCAPOn0Aajz+HVZmVYwvSztaP2G0eGCOjvctVIT/75djkhrDmHX2GtsMmRt6zKNaXMP0YHmQfDA2jPNSDptRoLB1Vhhl6FUfBMDBTqq8+zANyWxgHefYugWlAbr1uAxE3mCjKtvP8AGvN+lXQcFw8APoXerMZnTKWD8vPDsXMuN9hwYFBC4MI0abCMAMMMMwXWOEtY9ccN4U7PNQyaNkMFSXsYMwmqrp5vMdCvehZ255PrV1qgRG8zM0VAqbMfu2Ey5Wd8fvj9hpqvcK+HNegHEl2rD+Qw6wWrolNg02ltMYeL3pSdNFuKXnw3btiiNMUMAkwt+bOjvD6DXne4be4eGw6+/Tp9Hqp9BvBsXuvw8tRUl1WNZV5xzYuC34z2ff0GsmO355PW9UdA5TEqNJjh1nngqymvPxpuDHtX+gPOu28YOG8GwKjyoTg5KVhgAB7FciZjMQKprxX9bQPY7hgqdK1gWZWa26DkvOHn3mzsR1Kz/UhG2dSZk/Qru7b89a0hpuDOwkYYGzgsM9VZ2PtEH2ZR34wZtt7PIq8AwJJ6qzvxFHCTn4nWakaWA8bzQMK7o8NgKvHCqzpMaEfWfWsvR6wwzMabfXTaJVbOmVGOxWGA5fvz3qotFw2rL1gQ2gYiTpkl3sXYaJQXJls8i+xmy+GC5odKob2B2rRo0k+w8F2K2JL/AKraYfiPMmHnV7Rh5amcMaw9dDtK55mG12ZhmoQcN5cOou02dVWWXQ7wXYuJDPqq7Wpf++9VQRs78iP/AI4ZraHrD03h7QwmtN1i4MHewHgWcv8A4dN0RoKjBqLMxp/yBgvUd22raVSpf88t8n/1iOG9eNL8NyNVnabSqrPqUIPPvWPSpZ2NUn7Vnuzn7YCeBhh4wdC0FY4nPvVlpzQCNh5Fl6DdT4UsGJUreHZgqSZJbkyjfXNa3q2jhrUmO/5wzTxgs/wuqrdVtdrCXrYLUcr8VK1JUmfCNtttc3vOH93d1F2M4Bn0N5rm/EulTo0N2XodaqGLyvDe5OqRGzYN4GDzMADrXQLhv+j3tctKiTrc0YjGDOeG/wDQsYEmj1WVLnS9aBmeYAyexZJ6paM834j7wEHRgtlvQE/i1wusl31VQ7A5lrv/ANzXN75u2gXa7zVDoAUp7fnh3rCSak/MPUf3p6G8vTpbWwGac80bFVY1nT71YXJb04OYlU2lSeXDo2LFQ6lOpr4HFcTsy6rjnu6kqcYNeQFiACZgGpmG9XEasMRmtN9/A1VRpjAZv8oB4Ks5yXMN1x9sAaDsBZ4F89WAP2BqnmG/Jd1H3wPyYdigg92ApcaBKkl4YbE6WqWB+tfbtgGhsBRJJxYHsGDOR3o5gOMjyjew+9QTDkzBxzvWguI0l+TpOP4bD7FqKlPcmU3UbiGGzDYsvSoes7m5mhCvTmZTtN5swDNZ4EiMBh1tmCsoEDWmNZuGAAahM5nIDUcM1paay2fiLMasDB5jTWKv+SwDoMMBhh1rWs+DFdfc9kwGZrnkllytnIl6mG9TwinBlx4wbb6zNR6l+0Ax2MbFbQIz8OZry94MBnsVI94zpuec12Mlrb2Z1Iw8gZrYQ6l/MxvnFM9/YCyVEe5B919tvN0wwBO1i6qlSpRxYLmj51FwvazevmlPeBUojwH8jKP1lbB/0o1nHnokyGdS65HeaiRpLAOg+a5sC7Nm33pQb5LMcDzzDvVVM0HppnFz0s9mt1qRzJzPDwBR3g0S01uHTe5n2nYru3oFDPOdOfMHTDYAdipo0+Iy0behnIMO9XNHZYCGB4b1kgR0rWqQOc9ogfeaGYzyz+gZg984Jk5LmemZhs8iF44rLQPg4Zn86lqRg2BJmpaexiLv2b01MqT5iHQCajPOPOq4hMLWgvbDYfcPDDZ3qPPPANBBg/G8QMw+dNAbcl3xGwz78FeBE6wUfWcB0221IeMw9p+RCyyByMzWikgKlKjRdBvBFz788gff6wSOG28Wm25gkFvVHLUiP7PoXOLYwOS0HzqJMZ0ZAedsFYGzVY0AG22DM/ObOGCT0OKY+IZ6x9a1Fecx8zBvUDD50BvMZ6eoo7IQTlaZhMMwPybFN5NsJ/3pvRj4Zh86D6FNMIuuYLRfZ6KdJDkfGd8ioq9JfZpsJhhzreVxPepzNNhG+5mYPbwQZ/nMHXQb9r0GkbzjLGzrRzIcs5Gu3BDB89mCRhgZsTmDZdDsWeAEAMwNxvrXQOF0N+ZW9N889Df0LCQOg11PgzD1i57vPMzWjndoqsmDD4eVVydL5aOcYwN7DPDYvG9KksQ3zg1mDJ5gzzz+ReqOK8z1bw50MP2qYwH5F534kT6dJuOE/TXPYRgA0FFXqbLhuhLgxD5J/wDZjDvBZkzcyPNdWeu2lHaB0dhxkDwwBkA3guXyQza3+171rFtota2lUpdNqWpFbAzMO9bimvRGazEfrkp5k8N+rvwXPGdSNiYOdatg2eJqGZqLtFtLSodGeuFoNTCI+Z4H5EdbjBGkVA6PO5yPFDPow3+RZzn243tFozhv0qPpzgMI9RDZohnmfkUWQyMx5+eTTD7EaMDe/ADSpQPhK1A2YH1qt1gB3DsV1SjYDe5vBwOhIWnT3mJko9BdI4RQIMyrG/XzPkqdAzMw2edc0gQ8yW1pv3amusOeyfDA16PUfCvi164t9q3Lfof82UEzCHT3jwB6VnnrvHh7EPIvJ9/3VcFz3RdrdYnBPqtUn4GFMZA4uAdgeQFsKDdTdvUaaANzAlymdANLrA/rV3wlt637brwRarUo3MTYZ5vAz2fOaqOYcZ4RUe0q3xLt+j3w/JjUKoyeVkvRzwMDPo3/AFr39b1nnwxoZ0f1zJqUKiVL1vGlyHszAPIvFMy3qNbdx0y4KNTalMtoKxFlPS5AYG8ee8F+idSo8ECZcw9uGa7I/uItRT+LtDrbEJw7LqtbPmQfgAEboMOh75FpYFbo5ypDdKghGkGec8IjOwJXfmfea45dvD2o23DvCuWPLN6oVSmmEaIfufoPNcfoPHLihwEt+FYFRs7RkRczMJwZgfzq8Od7QOpSw71W3zxIg8N7NkXjWJexjYyz3vH5FhOFHEK9btpsgOIduRqJU2MOWwDRCZn5Fl/S3vm3IfC+RY7dRA66/JYwjh1sgBhn/Asbaw7Rwu4303iXbTVZpQfew/aYmeZgthJvbbsiPB9a/OLh1eD/AAo4jUS4Dc5kGGQ5xqP16Br3V9vLVk2o7fDFSZk0JiMcp54PIsdrXx3/AE2NPiU6pVmHGmzQzZjm9vMFI+1sU/aPmf5F+b/Eu8Jd+cWqhetGfk8lzOcCRI6wD6F6l9Ga/JfEu1K63fARpM2kSWGQdANmBh0fwLZDvoXJEPolAk9XoOHtFlfVVudgGH0GgOlUf3bZn9Zq8MlrJrET8cFWvVjf7cFTvUpgPZvn+tRzpvknPLbAvQrf7wFYM17BrT1FjOQf93K/WnggTu+WBpgag63v9ogOe28sPVZM6lOtPy3+Wp4Zm89hn+hWdErbFYgBUYgSQaM8A1gwzTAuKrW/U8M5XIyZmHZHDM0zatw1ypQxOuUMIZmZ7wez2eRVVznOepbrlOfMJDAZhgfWs5Qb5tiHUTtj1lnVTADeDDrf8i47hbrefwRwzwkAfkXmem3PWY1Z9eWw5JCdFmPxXo86TmAb+jBddjXy5SqXIlXHKZ5thk+ZOOGxFOJcJd9wVB/z17/PmvRU+Sbz+a8z8JZ+jTpFfwzwnmeC9APVLWYaf08Mw6MwNXDJYBPfD3ieCtuMlqGazh1L501zwLoHQmbnppj4j64v6S18uW9QafcZ25GrdPD7qYSAzBkzNXwV6mnWfs/qffQZ5ow+RZHiLSrf4itS7OnXHyHq7xnmcw0ZIf58P86npPDxbfNV9fXHULu5QIceaeei10B9Cy8mSa6hUoFDrFG5SIwzDaAzA3TPsA9i5TM2GuF1eCZmGBZpozzma7/5wBNAD7xeA30dakaJgqerjOjvYa+sAAnf5uxNsH3jz86os3MlK0T0s21FwJPM6LWn9aiPPP5HqHsP5EBm4yYHn3oHnm3i02w2KsBHvTR7EeGC2XCu27Vue43adccE5kgIb78CODxs6xh1hmHyKhjc/wANJdyptmWPWGjfp3BapSWgPAzZqUk96jyeD9se0f4c3+z/AHUnZ/gXnc6fhtxfelrPs+wcwPzrrx8MbAD29NvmB9YAf+RAzwisCpTY8GDdt2hNlHgyEiAGGax7j4eZy9mSbzWD7hp0IeZ564IarTXKJXKhRnJbMzkpJxdVnoPA18wDAD84K4c1pMAzhunpn1griNVcz0zc2LPmeKA3nw9glwNXmxJ1WM/FDfgnXtgqqjPcs6b7DYb95gppyQMAc84Lm8KGy8GXjtgYeQ0FVCqzOXrk7N5qUeizIdezMwDs+hAyDjzum2pYUeovRWnG8OV9yGfQtw7Qar6krMes8qEnQz8J7oPZghhxqzcM1qm0qDJnzXzwZjx2c1ZW9Qc5oMVFgzwe0TAAzwDzrR2BXq/wQ4oBVWH9aOZ56p7weD/IvNiXwf4G3NxXrcihy6iFBNlk3z5uGeYAGHZs867Fwx9E7ibaXEE2JdzGFtHUmIs84LxxjmMde8DDo7P1p4PSuYrc86xyNHOqsHytKg6JgYZ7MzNbKj+ktfFHo1Tfvyx2Zk2kQ+dZkU89GN9Bgm2D1MdSY6NNlkA6GgDAABJmYw94eovGnC70iuP3EKa7WAo1Nn0eKehJpkeNhnmGewz6PrXqum1ht6nRKrOY9Va4ZnHlvBmC9UXFG+f5PbGqF1RaczUuSDM4hn1h3rxPwfr1sWlcFdvRhuSzNmzJpxqYyzrRWWMD2fx/wLvXpM8RbDDh8cE65PZmzWT5blMDB4Pn+TYvO8x6m2faEKsVWq+sgqgf0HojYB/9gqgGzQWLqGpwXwZePZKnyDe0dEzMMA39GAAonCu3pdw1St2dQ6HDeCFJObPqbTwHrB2MB+dc6t6ZWbqr0e2WKjy1PlGc2SznsPBdbonGNuzI8eDEtHlnWZOgbtMZBkJh78AMOvrNej0hZl80qex9nJVV1qrTwAJIAGwNnYtUzoSWtRs81wLh1YFf4UcN7lv+7oOtWKuyEoPvO8GD7z8nWukcEJ9SuGyPtdUWAB2qSXzDDowDZ/kNZxY2uG9PcttQPHguOekb6QN3cEPs0xQLOjT3aob4PPSwMwzDDYGHfvW0WOsGc5nsUhkNZoDXP7b9IHhzVbFhXjdU5miVCaGb1JDe8B/Rgs5J9Jy3KxNClWJb/MyD76ieAYfIHetkO16ODXs1XhRJb3u1x1nj3eoTc36bSnon9nwMP481ZM+kPWP/AJRhh/3k0IdWZoj8Z0DNTeTYPrbXDJPHi/5M1p+LFpsaOHXHNnPP86N70q27YqkeJeliZx5QZhIph7w/IaxuF7ddkgcZ3DsVVclyBaVEl3HOfwhQsNY/IBnh/nBHaXG/gLfJAwxdrMOaf9EqP3Y/415h9Jn0mTedvPhBblnRmaeD3JBVufMzeADzMwDo7FGFvVAVXmY7T7b+sD4AYGHQYGhOYsJw3qr8zhpaUt/2vqeKH8C0fOZrbDnSnpijnMPzqRGjMPGGZ4B51Zs0emyRBzDMDDMDUqZ/mHvOkEl9asLSpXkMFIZtWnA7515tbCHVXwuA7f0/FCGEozzVgyb5q1qQWVR+KtEpst9k6xW6abMNrPfgG/8AyGtQ9SqaftGE2M/DmS4wA4DmBqczWJxu6hv5mroKJSgH9l/jT3qSjH/RP41jsRGbhfMsHHFLrdep1NsW66jOfwjsUSUZn/2BqVDt62Q9vFksh52jUXjHDtig8Cb7luH4QUGVmf5Nn8eCuB4J9AOY/D9JuhAx/SoE1g/owz/yL9QDeYAvEcX5L+h/WJdH9JTh1LiOYc1MOE984Gwa/U6vSXGXzMDS2q1nzILLXt1STHtYDcVUdbMA1D3mqqTW5cktTPD5EiE3bUUo857OuezNWVejQc/AbWPptS5N0HM88DVsdS59eMShmxzAamwFZVi3qbPpocjhzZ78zVKAK959g4AN7AMA7F42c8rFsVGmyMJ2jh5wPNVp0d/3C0VyRn3nQfiPmfnzVVGkvxnxNzsNawxBAoNxvF4DGAfvjwVsFKvWMxqPwcGg78wVg9cOtHz6E7Au2VocrgBh3r0UkCpcy7pvv7FfeoaBMDXfY3msCExt6qOuMADIZ9ALQRpL/L+0VXAuI0OlUGZr0pjA/OshVWarSjkTjYwjvvGea08N7WLN9WFboPrWlhoLHaMOfxq9O/EVxSqrLOaG9PBZj/u8A+tRLkjfYO16ndU44z3Ks+CHzqtkOZcV/SHvij3QduWJOjMtQgDmWtEDMz7+tZ+Hx14vG0EsK5TZgH/1BledKrVXK9cLtZfM2ZDkkzA2jwMFcSbwqWkbDejGz68O81ycr7HDDvx+kVxbje0fon/awwVJeHHLiNdVtTaBVZdK5eaGB8vGwPBcBO8NE/Elgf0HmpH8pD7zseK5rHHM8MNEFi2xxf6OsQLz4jT7Zj0CJeL0aEwGAAzsMA+tZK4bbbgUt2W/40j8XWzNZq6rnfoLrUVjPB8M9h4LPs3tOkyOVccM2j68zzTxfKY4XYNbmaJSn3P7GAKCzMg02a05Lfwz6FCmT/8A7XITv7KGC5VGuquSWgf9Wxg/Oaewj1uZ6ICMw8GuGGB9G8FHPTZDMAZP84Lgh3hXA/otNAPoUf7W1w/Z+rT/AO7Lm6nf9Dupm291gyq2TTXHsPVtV5YA90b2z9C5J9ubq/8AyV/5ZAd7Xc8ft6b+dnNOnyj6XXufuOH7SdAe/uZO/wDwJ1mvMSXdCdUgD6zXIguG7jH2lK/8n/tpmTeFZgNG4/Bprx/3OCdK/odz1qbj+1gf0GjAGMfAzXCaVe1fqWq5EodNDQ6/GMFpqbc91SWs4jED/sZhqOlH0upAD/uIrx/kS/nX3beC5L/KddUOQcR+nYOh/wBZNO/ytVwPbsTA+iTmself0Om8zUpPgdaX2Yfk+I+wuWfy1SvxpP8A43+wl/LVK/Gk/wDjf7C6elH0ulnZ8UOw2foNL7Nmz+y1I8/OYZ4LmX8tkv8AHk/+ZRhxmqR+w5k/+8/7CeOG3n0fluqlbc6S0epBB4/OGxVXqr1b+1UqSCov5Wq4HtIMlOw+K9VkyAY054Z/Onb58MmzpVVOGDrlHrlSpTvn5kwzV7MvbiUzbMhiDf71Sjvhg81IDWzBZR6ZUak1pvnnn50o0CdGD2Gf0L3u8r/HD/oxXDSewYTaHUoLMzA8wA07cln3HWIHIxA5ZrPMNma0dB4esQ6u7XGGHmdfPwsMABasIZgfiOAH1muj63nx8LzvP4e3UDum3yz358P8ap5ln3jAMPWVAmA053gGt/gXqUAAA/oZ/WaA4cQ/dww+h4wT62Xx8LzFJtKczTjqrDmbQdYYYGCzT2mvWR23BPPUneEfWGtsP+BZescH7RqueD9NZ/gW0eztzX6zzbrN/wBSLMDPTXcD4D257SXOMGg748kD/wAas7M4IWyzWZFVg1w5hsM/s8gMANbd22N8LggeCvQfCV63zt7OjRXmZb+HMgff5FhLwjU0LqNyLaMaNyRmB5++R0SqyqVXmp0HRjB3xze2Yd62i0YdQrFHocYDqTFuU0HQ/co7YrEWS0bHqaAzp94ApAT4Nw0504L4PBhnsPNZe1ZLjNcOJ5wWNwjDa1utt0qkuvuMRsPoXnyvPczKdneddL4tV5imwhpRt+3DPNchkz2HmsG1xqPMvbV0XgbZjF/1yQxOnAGh0NGHWue2xMixq9T3Ki3nHB7eC/SKg0ehzLchVWh0OGzrshvBnA13+syc9pvDGJAaBhunMvfWqfi1w9qMC1/WTEEGQY68F2f1bL/DUC4bYcqtuVBhxvszXSyeRGZO+PK8h5r1bbFNo9So0SdhnmyC8hM/tDrHkPBeneCBzqlZod+geCu7Q6HTWaVDP9hZx+cFHeprdKmnWbKbCBIM83mvcvI9F8PdoDeMFCHJPSThsXhZvrGUHJ1il+2ZZ6Hg+f8AQpfowVKDMsA6Mb+DrBma2d52lBva3JdNffejS8NjzPWuD+j9Ujsm46rb9Z2H5/1ot2i8LwO3nTA07arLE8WrgNve+B4GuFXzJqN28SXYjh5wmDAwXSqP9raa61BY/Z/Ii3Szk7lHOSHnR+ralpZvsGCjnAfz9miC5htEEkDUc4csPdpBGfDrBX+hDnvG/hFTuIVDdlxM2ZsXxwMD614v9W1KjyDYlQTzA8MAX6NADns15l430ELMrkiqzm/u817AD86i2sW4e9WIsmObD8UA+hSjtKnVKgnUgnPBh2At9AtXh5WLfO4GHAN0AzeA9m/yLm/JvyWjiRTeBryZ9CLY/Rzd021pZlk1WNS/XAcs9HD8LrUiBZ9SB/XbbzW1A50mjO07QZjfOChbltKmRID+o5FCT9a0VKrEF6pR3/U8YNA81jZ8bkJ5xNTPBWtvSYPrJph9/DNEvUdmcZrDjcvRn6OcN3zhvXYIFVptVa5qlP6zS8zW99laJManSmGTDvM16WtisUCt0kH6HyYYdYMmreOdcdQ5NqFO+hZ9486lTKiHQYYLpXFS3vtDakhhtvN0OheZbbvycdZiUOpOdEnA1tFj2LDhwZ9E/nKKDzXeBrgXEKscNqbzcGgUY4cswwPAF2CpXhRreoIHOlAbR9gGvKlYpVRrFwy36VLZCJKPZmsRy+YABKdcYNR2dcC3rS3hZ9ctvSfcYB4HzwMwPoVaFNMy8TYHnWNwta0G6qzbZ69G61cfy03ibRsGwYZ94Gs0B02N/SgP6FKpsmjg7n3rj5hbSb5v/rYr8kFEO7bqqrWhWLjkmHkUjOlH1v4Amp8OlcvqQZYPGHXgse5bjO+A0B64Gb4dAdip3g3Zp7P4J0PGE219BSOzp+8UuNJishhmoJgEZ3e4o+eZIlfczE924o8mZ4R+GZ/QoIamYKzjMt7DPrQAyyeG9NYH34ACdmPGz7MDw86hZ6xeIrEuMyBu6AHn861EA2IelqZgHnBZczbgR9je9WFKZnSYX7UYNd6hSRVZPPz/AAA8IO/zqnnvct7NszPsU3MGXTYDsVgZsPQAYbAAQSKCbDMAJc7YXkVFDCmzJ8h9sM3gPrUqYzLnhHpzb+AZ4I41Hbo5cq51toLCGzrSAWwpsZvDTbbWcpoN7MFuIENgGgD8RYcor69p+qDYcMwA1ggBum6oU3eHQYZra3IYHKBhtzYAdCy72B+AogV4TH4DAeAZmZ9Cj1I4snSbcY0dmas58NvFpzm2QAMMANGbLDLXN+DJdV7WqaO9EOqR4LDaauGiSnqy70AHzq9pVNbk1tp9xgAw8isLqpp85qOeD/fK9oZ+j2HEea15dcZD5AU6t0eJQYBxORA/nNM/+v1MZgEzDsNroNU8aNKea1/XMz6DPNZiQFKc0AfB8N+/BQnswPBxI6230S38wUsKbTXjB/UMPkWlJFR4bkl2Q+/0AGxGzMw9malgbbInoeRVgMuaum2vTA88M0jk5taCdPQNro8VKBSpcwzBtjMFnsSOWo0yL90cMJbe8/nUSMejs709o8mebD+BfIaiGe83O9aIaCSbB0jAJbOQdirIwMM5+Pnmq/Nw1IpUwIDUg329Y3z70X4R+iRmbae60jk62HhgpoMsPOtYbAPYa82uyZZzjnh19n0KwemVKNRgqUGWYOgeH5EnobEbDQcMAcPYCs+WiM0PlHMMzPZma9eKSBWH6qJnUn5hgG9nWUgAbeIHOvYgk0qc90GANd7yuIElimsOtmYScwwADDBZ2HgmPzIRweUPwA9qAKkmAw9F03+xI6rVYzRhvBow6ASZeDldRxRAr3gCeTQNngDAYb1IZZiPZsPvvGYGGZmahcyHNAbjBnvUo5gTBdb0A3mgs6lMwdjsMGGGHQHYqypT+claj+88MM1FwwLUUdlnWmtMG/ogZ9eHQt/AkbF3jg/A5OktefoXF5lH5bR8fPM8NnWvQvD2m6MCI2ibZ/0nJj7NBtylMOftUk3sPPgC4PDgRJgnrywZqHzmukek/Vda9aZStfP1dAzw8ma45GM+aB9GK7o8amvbJzBvOxTzeMOwEpLNNwkHr4GG/o2JGbjLRuNvmyZ4AeHeo8YGA8BzoRsajQJU93CI3mQb1ZRoxycAb688MEMCSxDm67B4HgYKPD1OaNzXeB3DYYHhgaDQT7blnCha/wC0AGD2xTbkuGm/Z6PSmH88AzBTaDXqxpQqiw5m7omDwAG81z2t8qBx+U+f/GiFYHjGfiK7ZDRaZw7wVJyz4Omxp7wDM1dxjzaa0+zYa1W0tHDPBXEyY3qtBElA9oBgYeRR6OyHL6nyZqsmB/OTr8F/M5WZ54LiatnDZ1oAS+h1BZ91aJ1uJVaaE9rWAHnjPsz6FnTrdRjUtpt9sN/eakWZPpTMiFBnRA0tYDmSHujPsVQm11xIuG4Lka9ZQaNJh0eKGEbS6APzma9x8B5Nz3twsoVYrE7CacYAkx5B7/kXic7wiWHXgse+Lfkz4lOmBUAZjycOZDrA/JgvbdB45WPdsOJWLAl0F6XKDOTRpE/k5ufkDP8AOu7xzOO17W4E6A/EcY7zwP6Fn6lwfsq+Z4S7np3M1Bhk4saRn7ED+Rau4akEmRb7D8F6lO1GSYYSz354fIgerEW3qzUHJ2elFjc1gAZmfyLaBlLhuRixqDUJV90eNy9BZAIxgftj7MF5HNmlcWrjqt8X/c3q11hnWAMMwMFC4x03iNat1x6NxK5wKhW2Tq4A7MMzZAzPs/IueT5jGkcVxzwjXNdkJ1Skt1ifInRZ3LNYZ+MHWpUniFeMmygsCLWXnqVra3KBsAz+dYrNjmMDbVnGeYw8BRtbRVVmdR4VKbCcy80eGAB2L0Hw0u2zuHtcqc5y4ACPNjMcy1EPMDPf2LzVGny40X1bLcZCOYAYAYZpQ57ATwBve0fWHnNXFj9CKbclArciPFpVchyZEpk3wZaezMwVhvXgaiXIdsXVSq5FPA2JIH+TNeoPSH9IGnUS3qVQ+FZw5Ltws6z07f8AdmNnQfYfWttuanVt6W9Vno0z5188HaPUqyx4sU3ovMZ73sD6zXRnrYin2LbbRiQZlGfhuJPRqyyGeGxbYLGl4akRwEjtWqh+Cf51G3mHNK8BzKNLiVIMwMD2KqolbptHtk9CU9M5UDM8/P5F10LPrHvKVGP6zWHuSzKrbHrWsv05lmEYZsgBhgZ+Q02YUhyZd221LcbCTStdnwXgMDPDzrhtKuFgJrVZcfgHNingD3f1h/qLuv2Vodetp24INckhNYZ1s2ntjJ4Z4fQvOVVtL1UdPqMSqmdPzADqJhseMO8AXzfe8/8AhcNVR7nt+HfgXHWdkI3uaejxwzDP6FqLwue0a9FOsfbh6SbAG+zHZgGBgeHeuT1J71leh89OZZiPgGEh7owVnz79Namt0CdDmBomBun0GGB7F8fh9nm/LpWvDTXjcNJZtnhm8a0FsVWnW9AaqvPT5POz+VZkS+jD6Fnbb+58G5p+czD+NWVBuSlRjplSrFO9ZHFezZjm9gHzn+TYvocvs9Dx26vQKBQYVMf5s2edM5Ul4+v6AVhSmbck4TmGKlDdDsnB/kXD+IV+P3/cdHYprHLQoT28HnsDwXULevNi7ZE3Ti6MelgAa2eYPLt9P3I5K/SyuHIZ7N68OrhOdVaOFSkHJM5M5k82XoT3QHyb8FwK9rwrMCZU7cfo0amyAM2Hjj+Q/nXp29uMfJ0iJVbOYh1VoHv5yjyAPMA+ReZb8n0a6rol1mnUp6NT5T2vpOnvW/NzIwwIA/Umnc89JjrVbMPYZh0GpE+YcB82GHNip3j1g+fNc0OgUN7Bo9Q0Byd+o4Ci5/BIzXS9WrJsZA44pxvYB4HQqcDDBLWPzqLgSJMxh7xD/Qo7L2HQo7x5+IgTAmnJcUilVidRKpCrlNPCXTntZk1XgCkaJ4IPXfB+7YNNu1pxh/8AmK72edjM94Su/P8AOtrZ8aXc8w7/ALnlSTkPvSosaCB/dgi7w6PnBeWuFdSfmW/ULO5vCbCP1vSjw3gYdYAvRtkzL4vmmx7utK54GE1nQk0moM+DGfDYe/sXDMO+r3G1nat1ReHV0XRaU6dM9RU6MxV4fNmZnGz2GwH8C45xF4hTqJCl3/L2XBdGcWlRD2HGi97/APkWgu2q8/W5EG465z9PpB87XqgDOGeH9FAOwF51va5514XDIuCc3o57I0cD2MsB0ACvxDa+a+Dh/R/3KQDPDz/WppvBgOHkVf3o89mn3rr8PlDN5w1IjTJYDpsOKP1gpUOMYDsNei4hgegOfWnjewUpmG5sMHANSAhxHgPUb8UOxcSjUaqgZaGAMugGAH5wUjnJUZ0HAfzUeSDDwab7G8PkwSjBmANq7Gwh1XWaPwP0I69J9a0ubRpwMyWjDOHLkdbP0KhMH4zQAx7w8DQ1KY+yMeI4/mAHmBguO+Jk6BGpVlT6WFVgxDgGwzyQA09szDoP61kTqr7McKVOlfeDMwkhh5OhR3nmKbS9NwzD6DVOybEloDbczdU8PD5GwtW527GdN+m1WpMumGDwRHtED8iNnjHxGCneo2LjknCzM8HjzPM+veaw70neomsu6IHZq9cLFw8O2m4LckwB4zkg6fvwA1WXbdVGjW5b79uRIxx8z5mPIDMww71zxmq8nTXW233gzPMwA9hmosmSwbugwxh59/WauBfs1vkKydYobnLO55h34KTSq9dVYqPiVUOY5kJoSD2GBh0LLg9gptNk/fWvrzWg7HXuOt/3PAO1bjupl45pgxJN5nDRYBekrG4ncOeF1g0+yqM/UrkOnZ4O6IMgee/rXioIAPE7VXAA3u/etB/K1fmf/F9iqb/4xoPSd1eknxGntOsWjZ1Epup0SJD2sYf5P4Fzn7W33VfWdc4jVkJ/Kva0bPoZAPIHZ2LmX8rV6GH/ALD00/8AtjUepcSL0qVLqFNOxIEZp+MYG6EzoDzoOxsyYMmO1L5WMYPhmB4diA3oIf8ARsb9C5PA4o3rDpseK3Z0OeDDIMg6ckw2AoX8s128/wAiHDyNJkfhR5msa1ux1esXPSrepztVqrcwIjHWYM5qiDj3wkx/a5IH/wDRmCpaxUrjrFBdiXM3R7bgythnveeP5MFn2WbSoLshyh2wFbqD/wDS6hsDP6OxY7Xhvf5eOFxn4FVkn/3ZYeAbAOnBo056quyjzZZCTrYB/kQMzLZOZz0uhybeqp9cuCzrRsP7lWZwLqPfSjgVuOfvqfhn+hVswCfQQhx9e9KrQaJHDrAzCTNP8iwt51uzpNLBi2YMkzA/9IyHt5h5MFdBGpUwzbwjaodYPYZgs1fMBiBDjmAMhrnhsBUOh2TxgYtiBBiRaxVaC7hhmyfMxZJh3mHZ/trvVK49yzhvSnLVh3Jp++oM8Mz+tk+heVbGhxJlBlsP+4e8i0VNgRKO7z1K+7SPOyGCqLRcO9h6WlneK23ZVbB4GT1mXjADA16Ls8OZtm36q37KVSorwfoXgGYB1J035b+bp9+AAru3r/4hWk0EG3L7qsaIx0RDezZ/QiMP0NZwPvUjltubew/OvG9E9LS/Kawf2gtilT5H4zObP8C+2Z6WN6QLhqsqqsRp7VQ/Zojx4BGw8hrm5g76SdYrFq+lBYU5usRmZcKGxg7IZ2M5vmG/5MF6SO6m7Y5Ji+KxQQeqP7GcSG9m9+TevBvGC8Lq4l3kdYuNyMdVis8qyEfoAPJ/GvRXEvjq/UpFqfybcmcc8AN6WyD2i+GGw/J9az+geqDjfcjNRTPRYzXiqZ6Q/EOwLh9YxXIEz1iAG8yEk3mQ+jyL0lYfFGncVLPC56bFOG6Z4SWXjzPNe8N9i249ZAsV6RV1MUf0fbwl6ATAlQ+VeZe6DAzwVgc8wXJ/SirbYcALoYcc9vyoB9esuoeMeA9wxLP4yWJX6lshU6tsG8fkDoX6nV6qzjd1wc2H0L8fP+H3X98C/UsKwwdLp+pLz+5xf/1ALOxanVXA6zQM1ID94qKTJYeD2iiazgdDi2Q2AVL8RxX1KntvNYZ71znXlq2pUx9lButbxfaKRht/agWS9ayEvWv4ijA0p9WmqqpRvFaPsz3pqNPczRTJmzUczSIB1JmJGhg5Efz3o6DvlYKPyD8xjw07Dokpn703KDZ2KxT3JM/5RyG2wAMPICl0eYckdM1dSYdsQ6JLnTqNzNQcPMAXN7quf7DWv68fbMJD/gRmj2ZmlsltdvGawOHtUClVx+e89hmfKRs9H60EP0t+E7LABoXCf/8Ajf8AbXjqt3y+zcEiWwbL0d88zB4M9Y/nUqj3bZ1YwciSmaVUA64Lx4AYeQFx8r6nDw/+XqO7fS9pugbdlWdJkyA97O2Bn9ALhV/8TuKPEKFIYrlRe5LrCIyzgAGsuHEi3DkNRW6NJzfPAPGBMz+IVGjOusS6GbLoHgYHJBcnc7/HD67FSY1SZM9mCp5J5ytOUBh5D1lq6ledGmAbnI7PkMFkefiTJQNg5vM+9awq/wC2qs84T0rDDQMOxXr0ZwKvCwADA8DzUfBsIVTguda00MM4EQ/3KfsZbP3nAYnx4Us9+myujWTQbZpVoNVE6PD1QAzN42c1ipLOtSPE7F0C1Q5+zZETzsn/AIDWvhyIka9qUzUgp05iNGaf/ZnQDMHlVcRaJSgaj1ViCyzIzwMwWHsaTUqCFKrl3ORpNK3woxn/AEM+zNdQvxnWtR1zyGBrf/o84XIfs84Eo33DM2j34eRWYW3BNrUcivfoVh1x1oIbOcKP4mexfOh0+w5VcNHCM1H5RvvMDVCEbRlNG5n1+ddIu2GGgYA+B6D2eaxUxlvAHG3APA10sG1jRm9jjgACyV8wwYrINtn7cM8FuIG+O0Z94KpvZkD5R9zoD9a4ov8AvO9krPk06BNmsVKdy2uzs2ZrZ29clq0103KlX4z3k0mcFThR2/wAU2HR2M/Zgtu5l0pFb5Gq1451H3wnwDekdHivNabjaswZbBGexc+15Y87DOT0AYKumcOpUb2b4B9a7DR2QeazUifTWHvbsYL6vhxPPh29OZLx9FTaDR8J+m+5+ha6ZAAJptnvwNV4AEaeBg2ieG/1ps+jvhFzguGbylwIeEIAfb3q4AAwRYfFfJ5rfRVtSk1+BD56nVIwADwPMM8FWM3Pekn/AN4P/vK2B01ufQZYeQ1n4FHYhgYZ55qNJNBJv+SGo3WQw+cwQaN+H/0qznh5wR1KGxkB6aVsRmArgH2GCvDm7EfC/P8A4kz/AOMCCS9ekbDOqh+QwW9OjxDL9lBUtepoMlnoAH0LbBw8362POpXV/wDGUz64ub3dYV2cNh4DbcbUQ6bFjR9jaiG1w6dYFgN3JAhVKqwZkabv1jlvYMmqKvXhdsCqO25w5t8HjzwAw8bP6Ni9X8Ima5dvDGjucpGCOwzo6rxrYUqw2Iz4vsRYer5wAF08MOO7eH5/C70ibnih68s7D5MNE10aB6FVGqUOO/Ov+ZAkGHjR3owZsn5F66qVpTp8Iz12WcPOsFMN+M7yh1KNmHZrAvpQ4LcUuew6PwxpbsSXbGDPuZ0QNhn8640D2jdeuHsn+he0wjSzaNidFCS0fYfQa8Q8eKDUbMvWoUpjwY5gBsh8ijpR3MJxdrHri6DbYfzjxQwBY2Mbge0cSkm48/qOJ1mGwdSBs+hY3wto5tnjPYvbvoT8SG6xaUizqrK3wjzjfIH+4LyVJolGZgBLYcA/oNaL0cuJH8m/EiO++39yqODD2/o3/wC2seH+26el+jD0yIBbFLhvNvCbDZ9exVTzLD2D7DgG0YZgakQNAD3rqczxjxIpr9E4l1OI/wC8PNdW9GmpAzVKrTXz9v0fxrOekzR2IF6x7gb65oABqs4IVjk+IlMb18AlHguqGT1NJZPPT01WTAMD3roEmz3DzfCpMmCp5NBiRv2swP6FlHMz6WWp/tV5o9JOgnavEGJVYPgtVTeZ/PsXrMKUxq/dW1wT0ybYnBb1Kuc+hh7Qw/3+hXtlhSPW8xTX6PVW3M3pW97YuxsgATWlz+m0d+t21bktvsAAXSPVUs3Qc0021penWIptaGojBkDDUbWXkssUqUEqqzo0Nr96eCjvccuENvAbFSvymg6HZmaNGwNnBQpOmCwM/wBKvgXpG4xccmTh2MwzNV7PpS8Fpn3Q5U9nP3z0YwBFfh0DWDNci9Jy2/tDYpy23ACRFPMMz6+ha6Hxa4V1J0GIN8QDM/PmCrOKlKg3baEhim1WmycOj7yoYPF9BqXJxzbfYDSPMDVI9PlsynjbfMDA1LqX3N04gbNA8FnJLz+qXifrVuqGtZv+vxo+HPGq96/64fQ5o/Qqemw36lI0N/5DWrjcMeZ6H2fzzAXO1YyZJbekZmhBnfqAt1P4PzmY+uxOjBh+M8slJpr9Kf0H34z39yeaBAzn7+T+tdI4b3/OsMA0z5kOwHj6Fzdl7BOnMzT8j3dwu4kUa+acGmBsyw6wNeZOLVsMUHiM7ymAR3zzDA81zIKxVQ9hOks/3J4KwpUyo1WQ0dSqUmS953Xs15FpuG9qQSwig4w+bzXeBmqI6rQ2f2WsHGm/hGaCZGrkBg34tZjG15Fip++fqOAyGfkBNowuKrds6Y1oPyze+tUj1bl+zzWoCzIkyBzfPAz9a5/MPCUbabMLDmWD8R/rToVJhk9RtVobzwMDwUr1ax7sMFGBbHWHJLWm23+dKHVTh+zDNVIajKWs57xY4gYIAPHUcRsm4nXjY9mGaaAM+hdSxGAA7vwzRhvPw2wUTDOVgfQpwaYO6baAJLJm6DgdisJh4Q9dRz2EgnvA80DAdCAabPN4NCW4ZgfYYda+SWW4Zn0B5ABQIeBu+H2JTHjeLNxWpNgHrSAB8AMFYPMygDw+hUkbUDxAVnGqstnDXfzwUB2HzTzR6kQwU6BD5lp1vXAMN+9KNXoJ5sGwe8MFNh0dueYacsGc1nYiMxtHx9QDMPInmQ1ndTzqQ9oQGpETQN75wRU1lwIbT+gZ5n2L0WdBZc50Ni2APaIa7nQCr6bTWGY4P4b1YTD5akungBmfYawuxh58nmZkiWxngZ96iAcQ2Dfcf3h2KKFbfkuut6AAbBqJho5n5zSAnmWHpWu+AH9avYYQeVBx9Z3PMNTA0B83MaNgGzNoOxeDW0qNLCqR5zDGzWDRNW1yVJivVnkX3NYAQW9UqNyrUGDTpIOsf0iQexZe3nqbAuj+fHP5vfe8ZZLXcCqnb0gNTwaeZ4cuHQCm3VZlNgR/WNDfCTCMMwwR8Qreo9NqLTlDqXMx3wzAM+hZTCUHsHMPk7F6Kx6m6PiciAIQ8FWE+ZLNoGH8PyJmHAfnjIcbw8Dzro8WIRz8A09NSID0TY451qOzGYekG3Lz+TBWECHBjOmZtmZn0K7tCOembuuDZgHzqQzJfBp3B8w2didnxvAa0G95nvUWHDYkzwiS/Zd6wEc4GYA5BfMz8hqVAo9RqWroRD8DYeaszZbjZtxIp4dhqaATqaRt7MzDM80FL9mKqfu2Q+s1Hn0GdAADccjPAfXonngtVGnthCOXWcMVTnUuWlGxEfPS7FpAzQMmBqwja4fIC0fOMGxoG3mCr5NNfjCGGbwGCbCDwX4k5jxjDsVlMe9agbb7YYH2As/GqTcYNB9s8/Ip0CfKM9cNjXzrHYLWlvMYbwa8iejM4GHiHv8AOgkvGHs8/H86XMvsmBg2H51TJNZzjGem2B5h3qlP58MM+xTTece3qIGmbuhmAH860havk4H7AD/OpEN6XGgHp4Ab72GZgpczUBoNPA9PyKIb3QDi8UNkIMk3W388ADsTrzME8H4uwz681Hzw92CFPAuKPqSazCied4F6YsmBhy686WTAfn3LEw7F6btIMI7T/wCGGa6vDB5c4zPP17ixW52gbzQYRQw7AAFSUqNBpTRtuMPH/eqRPmesroqs7DDmphngq2fM36YdalujvahuumbmYdYAg1tbf3qE8ct4zcSh6hmgso2GqOfsu9WBssGPNRNmYYAqo9mHzqwpsxx5o28DMGN+wOxEtGyblNix32HMMOxMzHm7klNNtsAyDAYAAAskEx86jqOHsV3GOV/QZ2/yYLnFJJCcAOynN8iUZhsUujxjDDU61paaycDmAffjMyzDoPvVZTWfFNaRY0DJ6NGdlfhgq2HVZdSBqm6kZnYfi96kXbr020D0D9u8ALOUQ8DalOLBq1s8GHreBhgwN2KGAZh1rTcNLDpt21u37VneCdRqrHMu4ZmABvUegnQ6lCDQbzkSuvWDANnkXQLMh0qiSGrt9f8ALVClveDEjvYPH5969hz22vFoK5QeJ1buo7foNNp9OgMU+lSKiGsZgHQAB2Zry5W59cZN2DVWOWkBMN4w0QAwf+tfosfDdi8LVpTdYnc/IYnsVc5HWBgHYf615p9L3hROCuHdtsU7+aqXDBiT3mb+Zmf5N67MOdHZ4u3BefAyRcdzXPGeuu3qxTgt5n30YA2Z/PmC7Rwf9I23L/up2VdsQKbXYtK3wQPMHgZ359HyLwUzJqNHrMemv60CQ4GZgbO/DsVgzW36DKqE6lVV5mRNZ5J50OvBXf6HkcK94r8Ubn4x8RqnfdYfB6OB6EPSDAGWA6FnaOGcrU5UJOHYarKIbEYgivuYRz6zTsmYcN024LmB+dY22wkcyEM5r8RgAOVsP5E1n4uuwBmHzqJGPMVdxnmGab4/WexS2XsCmzq9RnZx0eSDQRjAz7wWajHou5qb9pKizD5GLUXgaPrw71CZebBVCF2bzBxwbPrMNnzpmNMls+PFMw7DA1VRpj4SA8TepZxjDN9x/PMz6OxSh03h1xUvS3qXItmm3VyFHqJhrMvdDO/r/jXtv+VGzuGNjUILnuN6Y6cPwXQDPWD61+d9NkuQAjy2HMDYPNdAptvXPc9EC54NND1PK6M5P+D5ES7rc4cW7km1u9bVu2vRo77wPwGWTwAwMF3vgbMrMzhpTG7unf8AKADMDCQf3l4POYLxzAvm+PsqdueuZ8zMMGTANgB9a7hwuojHCu1P5Tb0qslmoSmQA3aj7Y8+xX+Tw7HG+39NhtQXzgasJ43jnHMPPAzz6F56vy8LOmPwqHK4jTK3T4pmcyPgZ6z+zo/jWgD0lqHclIuij1x9kI5+BAlgzo5gefWC8zvBSqV9xpUuTMdD3wdC4ub2W/4dge4luUG2qhbnDlgIxyut55nB7BWtgcN4PFqw5tDiXG8zW4WgZy5AG8yB+QAXnyBPp0ashWKqwcwwAwwN5dQgcZqHAo0H7K0qpUea2GBtMzA0T8+YLg+j8pwyt4W9UqJdUu3K5OZ/mHNiSbT2x7yKVXvUbNLkRLfiSWTCNmbzp555qEd22qD802KBPCbUTzeDWB7NR6lfNGZhSKHRrVmMyJrJsnOnHsAPkBR63D/edroFmU313w0Clc2zG1wM9V3oDBZy2JNNe5uLVWHuvAzj7wZw8ivqCzBDhvCCo/s77wMH+c0VbsO57VmTXLRfh+r34YMm9IAAzD5F1+85VZ9oaNTaodVffqU8w2HnszBS+LV+RYfCOXBo0Tls6kwEBlk8HjYwzzPzqLQbbbM5rl4weQ1zYAJGieH+/Qnb84ewbnrMSXTmOfp+jgzLjvYBguf1r60W8/vQ6kzIdc54wz9sethvTr0xiSGnmAbFHr0yjMypcSK48cRgzADM+vBRQCpMgB+rjZB8Mw1gwzBdP52K+sQKaYHKcf3/AELMvA2BrV1I8KabBsBmfWax7x712QGngb9pqKOnTTWe9dkNUoEkIJ01aUTvTyIAzJIwQHG01Izw9mojPkUhnTNFLOg1WdQatCrkE8JEJ7WD/UXfabeFwWrIqDFpb4l5Mg/AMNmD5554H8m9edcPium2Hc9RpXCq45zGByKCf82uvBmbJvdf+Nc3sQ09b2elX8V7hYZFrhtRn9aFTntapSOyTN/2Fz8/8iX+pn+dLsTxC75u8CNAln8Fs5jylQHsHc9PNQgU2Gy2bR78FI0EOe4AB8is2akw8Go22AHn1+dZQJPhZudakUo33nT37ABctwpfSdR4fEz+sEzAecjeA/1/P2Jp6YcZgzfAz+QFBjPNm0c4GwZMz6M+heDVxpObunqdn8aR4SdXm89Zjo2dazT1V0ZnNvqwCpRJ7R7zwMOzrXuHOiV6pPydJs3DOOHfgqcJjjJ6gdaanz35mbh56XYeCdCNBAPDfeM/3q2iFno0/CUDhthhh1p157ws/nUR5kGcNRwHt/YaeeksPDpsBgtAy9Mcw089iQPb1CPU7FID2WoqXCyjSXPedinRj8XXb6wUIMDaBzv70GsYdCjAuvWUv8RPBPnH3qXbFH9akGxbVmzIIAuX8tWRZk1yN4bFVZBnyct/toZ8+s8nI1KqGkbJgYAzhmC2v2SiKquq24kO36hKb9qEYzBabeYZ+3pLkzkvWLcmTT8DziM7M/zrYUqTVZOqxb9Kh0SIHZHZ3n9ZqJwxhsTLXB9xvPxjD/AttbcBhnncG8PG/wAgLHbq8Qyh2rUTf188z88gM07g/TXTivzmcw6wah5rZz/ucV2d2MBmaahybSgaURi44Ya4a5mfWZqNsrtj9E6rIBhipRgM+yRDwNV71pTodZzgmEZ3zs7Ft629aNSziVK56a9oBmy9ngYGmoZ84NPnOdb7O9UjbFVivVXVALxocavAHRIw0ZQB9YLBXVMOSDIaj3L8z4LMjrZDyLuFVo7Eww1G1zzivbbEChx33G8M3lpFtGXtipPw4TuhKBnN4+tnPNXHr6pf/FWf/JmonD2NBkzXYkv8HZ9a3f2SY92wr28wxh3VVWT/ANKwP/JmkFerL3jhOhmH/wBMa3f2PYAegEAWYwfWwmzpYo6rXDaNvnoH/gmmgnzvxP0LcPWfBx9mAH85rE1iA5SnzA3GTa84Gua77GNkc/A/Oul2HTWKrSJD/rLkI9RMIQZn0GZ7zNcfZk+KBq7h1t8ITUF/fHDsXNDF3W6uG/qewJcFi3zjHSHjejVbnAwMD867L6NNH4c1Kw2oNKrL0CsMH9/5t7A3jPvwXm20rwocOyJFuQXJgTTmN7DDPPfmthb1/wBx3tVwt+6qjDZpVOM5T06nRgAwMOg8w610+LwPYEmwIoD/AKVkmuI+mHRJ1K4BSHKVm8B1WLzOZ9AZn/nwW29G+t1G4YddbYfqs+jsSfuFQqOz8ig+m96/h8BPUdvwQk/aWpMUx7Pybz/yLvix+b8x7Rprp96/VuwOFcW5LBtS45dzgHrGiQn9gfuQ+dflIbPMux4meGpgwZn0Zr9CPRU4lzo3CC37Efr4ViaxXpVPZwP2MIMD/RvUX+sd+h8FrcANRy45735AVZVbAo9NDNh948Ow1qAudiNCNjXwWfmXJEezBx/NSKEIEQN+mmjwAvAUg5MV7wNfDPvUWTTcB1GJ2sHyGuiGQD54OxOsnUng1OUM00zGlvHpsGZmrXlqky0GpENnPzog0EnR9opAV5sABtxsDwPNVUmHOePMMA/OkyEpn27bJotcBc4Mn4ikM3IwfvFn5LLEn3YB9CAKU4B+A/miGtOqgCxXEW0rf4rlCYuqq1KG1C6ORPAz/wB8FLeolfZaN/Uz+RZyHeFDn6vrGdGgGweGDz2eaDj/ABm9HKx7YtcK5QLnrcnxtB5qQ9swP/7Bc8onAGzqlwTrHE1+dP8AWsWSYAGtswzwwXoviFc9s1uzZdDp0vnJB9GzYuOWHPfDhfxFtV8AOOwyErAw6Dz/ANhPLu8W5/UrYiyaNSqr52QY1g61RPWNTXrjdpT78k8MMHjNdNhgw9w0iPm3noPBh+tY2ZMzuj1jp9gBh2Gvn+XZFsPXrYi0qacXrw71U0pnO4Y8QPPmttdvjVSbLD+/WPZP1bV49VBgHt+BgfkWTZtYGgYV1h/DxwzAz7FLpumzDabbPMA71EpWg9PluPmANaOwzNS4Z/d/EMD+hMMl1JPRoZn84AthwxmczSJETyHgsebzB2/I13wZw86vuEVSppyqgwxLB73+z866PDFEZucJ7VTtylcOZM+nsScJIcyGBmtRdTLb1uS4jDeAaIGALnmdAoPEat+ubnqtNaN7mozIbAe8/wDkXSOZg1Whuzqc/rR32TwPzrf/AKMeNz0Azjq4gHL5VrlWwPZ3gqdk/ASgXtRqCRxZ3Mgef4K+bLu5lhMgTjkZ8pG+jDYq+ZSn+XdfC3KaeAZ+xTr3Ei1XndTXkh/2Kaevy1ZLTzHPTAzDDYytWHhEpT2swDmmAH5AUiq8ocI+egnJDsVfSnoOLoRZRyQ7DMFZz9A6abbj5sgfeC+X/wDq7/CCzvignobJvSgiMdZ9CjsmwEcAYfB4POnYb2jV4TmfvluL0KO+Ht1cBAthlgG51Okm73m08jnnotEfkDNccrd81WZKdfY1o0c/6P5Fs4e5taPxLCg8ww5Qzea1jweNVtb4i85WdeD+xeRc0OtznndNx88DTwTNEF1xbj27lQabY95tHO9VVIJHeHPgGaG56PYFpRWpU6hz3tQ8AAJ4Ljlt3zOok0H28Dw7DBO1i/KrUp5ynAZMD7OtXsdbgTOG09rUYo9S+jnN6uDpVlAwD/q2sb+znAXEmaw/m1LYfD6ADBbig3zLqQR6U/EN7DoeBcFx2Ozh5m1jM2doGx6jqW/v5xR3qVasbBw6VWMD6D58N6hT3n4bWbCupMzO3I8rQA+gN4J0unuVT0Oxz9pTax/58FLgUGwGcJ3qaqn9cxFVaVTQpuuDGBmlTfGpEhj6F04cFtMbNovYPtxKlgYdjyj1Wm2XJpLrcqJUjAA+RKHpnDabDsDBOyWQOA7meGxUhj2aDY7x6f8APf8A4wJqZb1jsu6f89/rBO0c86pp+dKthhNNZYdPb5aOyb/rNhterbVuq4QjmeYRDwNnNaKf6RV9su6Eus1Vk/3MNnNYWjmGkGfnVhVdfV12Nma6YcdrKpXhUbta1Krel8nn2A9gCrY1mWV7dus3CzI8/M4IM3zpZmZ5pmBmbQOLpi2K+A34A6cHiTdTLQdnOAslf8ly6vV7FfuqfUmovgBrde//AOwUipftBrOVUNhueRXswCZwZ4Qh4nrK5Az8mBrNcS7Ds6iWlHrluNyXnYr3tZAYHguh1UM6XCnedlVN1QHKlw8qDDDebvLHgCiyHJaUFOqUXOLOzd/Cw61nIxxI1wR3Kl+zsPBmCDh7J5avR4jneBgjqsZg6vN1w98snTt+ndhyYlVtKmP0NvOPohgC1FNpsp6UDfKPB+RfnJavEK7qVSAbpV/1KAH4QPbFes8deLEbYxxJqq87mN8L1H6V1paNpRLj6zhHh9HQvOlpVv1VWaZWXDwBh7eqqpcUeI1zterrnvWpVKIfXHkPZgjpQNvRdBbRzIv9D9CrYvy0bnobT9KuCG87hhgBoZgZu9a/Oem3nc/D2qHKtmpcm758M1rab6Y3GylTwflVWBVWvwZbOxZG3u2HoRj1JT4MtB1mfQuP+lLeHDa4eFtToYXjTfWsXx4YZ55ngvK/Ev0kL14nOtMPwQpTWGBhEePA1zcI1sH7eo/fe8HVk9/DtFgeksdpWXEo32OemSIXsXWeg1VVv0tOMVS8OnMQ6O18jOZ/rXCuufyi3UCzwBrZKM8/Ourua9Kiqtz3PWyP1zccmTqdYHMPBVWjTg6zZD860tVo7dKa136OzJa7zAOhQo0mgSfYHGD5D2Gnc6elBZmQWT9vmrCNWIJ+wP8AgUg+VjBqNtxnlWyZ7Bn7AGQ8gKL5nqyCpRc834oPH86aOvTg/YdFkPIBqnemBieCjhJ864+62GDr2uZm45vMzVfJZim747Gf51sLYgU6e7oe2VhJtINc24IGHyGCfXa8MfQTo8aQ02/R3pn0GtQbNpH4jdvnGd85vZp71OxSv2+IAAfk2J02bfP2bhgf1rHutthE9SS6l0QYxtefnDUn7AWqAfzlKOMfkBPsyYkYdNjP9aF6ZE9o/KAPrNO6zCkmWfQ5I4UZ+Tn5zVY9aVRje0cBbMIbBhmxOZ/Iar5J4Hg44Zp3ouBW3bdvn/pJh43fkNWdbgUMIoPwYhgYd6hQ6q5G3sfnBSzuHP3YAndbaIj/AL0GBJOS1vAwwTUy2KVPHUiMPBI+RJ6ZgXh4AjZqWj3q+5jhBh2fUWQ0/vL319Cr6laVV3uAwzh9a0B16XnqNzjS9ZOPB7RRswysZmDGEGJ2x1TXreqswdemwc2vrVgzR+flB4Gea6RbrLFNigw3sW2fKMOFTIb8Y9B8MHfIojMN950GwbW7uSybg9c+sTcB6J5wRBAYZDxIoAtXM4kzoPZ6YGZ/IgAHA8MwMDVhAnt0cjbiYdHkQnJ5k9R8N66FmY0Pxdd9vwk1M0wkeH0KacmIDGnob/PmqqT1eG4gUkwMfDc3ppll8z7EozL8ktNtvNSGWdF023M80E2HQWHmnfEw2edUR/c3zgt7wDvV1GqrlNlA43vTzMNuqyObCKDJn14IpTw41VnyAYi01576E7JgSoZ4S28D8i0fjw6XI0JxsgHYHeszzLkkg1FAkQwchyGpTgAYZrXQzoczPnnzB0AzDDsVJUmW40VrDrMM0MPZ4neaCbJDWHTWgomuDAA2qGNpmS0dKeYZwzfAPrWA0sB7ANNzzpVh5yYObHQHWnmYesQaHQapKxPYhmYdaxsUVNt6CbstxvAAPeqR6Nonp7zU2fJMA1GOvsUijxp0yA7K2bA71cB61YDkypNUphyNzEo8MM+hbC2+FwY1CJOrDzNQzMwiNBgH+Nc3t6TVYd4U+pB1sPbF0W9r/qUCqR51NYAJAdZnvzwXJzWM/Mk1W25B0qpQQOIHnWcqRxJLpvsBgBrqFYOjX5bnrxjBmp++AzVFbdt1KBMj1kIrLwRfbAfRgqi1slDmMG1gbi+HJAFfcTjb9aRH6bEZCObOzAFijkruEqSYH4iepsxtl3w3MM+tVMmS4YYJpl7dgvcMmgmRogSD8cD+hNRmTkyAAPOomexGy9gowpesw2JMrQz6+9QjprYVE9A80zz7nu1IZmAHXnmowLNmfyY4aiUmTz8jX6zwwVU9MbeDBSIzzgB4agVlVkuBgw+3gBmogaYG0+3v35mHyLUBG1o7r/kVLz8XEPDMJAHve7F0wEc+dv0ImAH8itaaEqTFwffDDyJQJkGHTQOdEN6QHRIDoV1TaVEk0vmwfwMzz6M1mM/MZYjB97bVFPnsMsHyji1FSjMT2jYcfwd7Aw61jZ9HfjOmw57JBqGabcc+A1UX4mizhs1lXvScD0H3N6XrVwIrQBKe8DzmnfXcGZI138Dl/QvfCQaMvSdNtjZ51C1twOeQFLkycxPBzBVmfnNb+BLOTn4YdaZBl97PBtNA8DPtFNgQ508z0NHDDPeeC8wI4anenmZL7LuwAw+dOvaeOnpnq9/yJMgw8074+Bh2KvA3vCsG3p4OMb8Otd4OpHb1pTay3/RYxnmuNcHKa5Ginrh1vHh9C6HxjqTFH4USIvvaiYMh9Het/DF5hhgYOtPg535p7Wz5j7rrOn0H5FZSXoj2DbETRDBR4YNvOg+DnhZrmptBBbEE4Wvzx7wz6Fl4zPLOmxqAeB4ZgukBAiAw62E5nI+/PoWC5B+NUpDb+wGz61p4Ul6ODQMedSAqsuHF9WxMAw8ikGDcZ0IMp82WjDPM1XxmWAI/H1vzqGmEF483zNxsAP5FLok9uHVIj8pw8GDz2KbGhxDfz6zNOvW8YOx328D1wPYB9Ctmu7hepU+AFZixTAzPvVbR/wARKfGfh0hpjUPe8riiRm82gbDrXO0wZvkH3qNHpzDecjrwWKpUluNPwfcwaA8M1sL2mctPeYwzPRwA/Is1DtKpVKK7KY0dgZ4GfWtXjZUR5+ZH16OBvR+wwBdGt62HDpcKuPymZjR55mGzAFzmyYFVo9vx9Oqgzr9eB9YLo0a6mJPLxdCpcuDJsvYPbP0LlplcOi3UdctWz2Tg8QpL0eaH7OyeZ59gLL1vjZelSozVDpTAQ4TEM4skJfjcyHkWEqs+pernYkHDwHtcJBn4yqqPPlTJHKyzzA1H08zmw6L6QNeiz/sPdVNpsBmRKhsGbxxjCSZh2H8nkXNb8etmq0T1bblNjPS9bXeqB7JLx9/0AoPEi/7nvN1qXcc4D9XPaDMdoMAZBUUCpYLpjm7FwpDB8MwBR2XnNXTcDrWmk6EyLIiMBvw1/wA6p+T1vYbHVutIhxgN36FNDQedOD4OB+c/8CVKgPhm5hvAOtQt8mmnOcNkDA+joULbK57tO5KXTKVUotNZj0sMGQiBhmfnNZII0t72DaqmZOHvAV7DqQAwDeC8gR5IcgDWu5mZnvwUiM82DQamf5FH8SZKBs28M+9SIwOMnr6gaQd5r1EQtc8IYYLpvCi+daVRLRgsGbTng4PPfde894YKqsCHQDogTqrb/OVConng6GwA/wAi0D1Hcnux2IlNhwwYPZy57/oWHc7+H1g0qsYT5dqnBkg0EkwM4IbA/OpFz1K6q9KjxK5cE+sQoR5xglnsBOsw/VscGG2GQMPIq+SD4NHqH94wMwkdAAuDm5m3wFUqawzCaNhvB1/vM81E8dkjYfi5tf41Ss3J61kAwD5mfZmadrdw8gJgbgM4di4/6mPA5IUp5o36bEOMDAdBnmeCiMz242eGwD6FCjTJ0kQnQQ2GGefnBBDMJ8pph98AzPeZ/wAaYc6XMksRpER9huSEjls9XWT8YNam+vPnMNZSa9TXHqzHcpTH3Jg+Vzz6FImHLjA7B12TDzsgtvW/zNnRa8y//I3TG4jebpyWDw/OmqJxOcjW01RrugnVXadJAIxsmAGyHZgrKtvMRuFVP13OjQw+vBYejydE2nHGAeePrM13+zeGEQ6xZN58JIHMOVyv16q655+rJEYMEretWj3DfQXPEfwpUX21PCGYAZn5A+QFhaacFmuDUZTZvO+QN6iXhxal8PbmpjFD5wzYDmpkcPBA8+gD/jXNw32I9nhwqvSN4XWPbFw0Q7HqUmNSqprhJif2Y/8Ac1y+6rqOqzIjb4aLUKMEVkM89gAmbkvBy5Ko7VZTejLfez68wBQgOI8Gb/X512RDmNSZ8UxNhyLrZhsPPBZmeBsyHdQNgK+mGw8cd+CwYNYZ71WT95G45371txiqANY/aYIsMEg60g3kuzw1I0gBIwQgrSdBAZowPDNNGoAZ/BGBoMPikrUlga6BTT5PgVXZen+3Vhhj8gGC5z2LpEkOW9H2n/8AXa2f+dWlz/PYhPUBpFhgKaM1AWfwRgCZTwAoULP4KRDkuAHidijgGSN48BAG0Dp4PSjBj2WavaUbEaA7Kfc2dAKlo4a361YPPMPRcHHDMwPMN6C4ZmBPr1MprBnpSj5U/wA5qsk0qDGmS2JVSBkGHjAD+hRYB4VKnuOOYYTGM/1oq9suOqsOdYTDzD86xwyQntTPxHM9/X507GN/2bH51FeD3moH0An2TYje3ADzBMCXJ6OUfcZzDyIYcNiZKCKE4GTPfvUEw++mw3sDWR4OMng2+GYd60EiZGcjSjY0zNrsew60DJn7tSzkm9CBtucb2/eCics4bTuo4AHngHzqgIdCkBDPSBxvvUeMDeO88D7/AKFdSXqUzTpBtg8egGDPei4QjewaaY7wDejjH4qgqwgMucmcrTMwB7A0v/gb6z69SqPFPnn4wZ/irqrOhJjtPsOZgYZ5rkXDqmxKxWYjc5jNrPPBdoknHjYYRT/IvlNfCIDO9ZziiHLWfIlB2eCYfIYGC1YPZ/0WT+hU/EIGJPDyu67B+HG1wzDvSLdHhScE9A7SkMH1sTD/AMALZ0rZKqf98H+Bc84CHnRqw335gf8AkXRqPsmVD++/yKA7Ug/maof3KkTKbB/nDwKb7HzpPBzLBsedQjtuJjqS5xxg85xgwBXDOzsaBBBilP6FN3s+dVlKZ1otP/7f/GaoqCcGq0sH4j5npmYHgzn3mmqlGgzNJidcZswmww5SIeB9f8C6EYaqsVW2aU6AVK4IzPnDrNcn4u3bSq3Ap9GpT+thMB83uhaYPU0A/wCY6azv/pDwZvfrXPOKMwzqMTnm9aQG8HdgYB5Fq8LhRPYh3lzUo8GjZMF3UI2Ya7HsvOvOVmAD1X1H3MMAzD611Ojya5Daacgy9ge5PoWTVveV+KWGAG5sAwDPeqeNdpxmg9a0owHD2rJ55qzjXDQJ4aeuB6gYYHsNThV25ZXqxWZ9Z55uWAYM6+Ec9izkyNUpghpwTDz5+dai6qJTbS8diog81KAwBoA9iCzsY2HmNdtxLhjf61fhorW8NIb9bu2n24w2yfrQ+SPMM8ANYeTJbCQbbe9WdtnOer1Pbp37Q+8DDJ9gHmCxuGOHXbt4RP2BVAtiq3NTTqEozOMz0ZhswUewLqlcPXahEcoYVJ2VJBg8Ht7IfoVZAurlr8m1zipFk1iXFAwB3W2AfYq072YqtUmvxKHGjR5rwGAZ5gyAdiIfoxZ8ZisWlTKzSqqEAKjG1zjsvAYAa456Vd7W4zwiqtpMX5DmV1iTFmxo7T2sebOfQfZsNeZKCF3VKO6cSsgzEzw0mZ/LAhn2ZUgpFQ0wh/sZ9EwDXfw2vDklYqrFVrPrUNm8Dw+fDBd19Gni7QOHUCsOVy2JlSkSnsGZEQ94B3gvNrJ4EutcLqbUZNBN+JEzA3j35gtteEPUH/pXWll4dnXD+d4Fq7e498JLk5diXWZNElv+6qDOAB+cF5qCg1w/6D/9+BAdq1V72kEPzyQWNrw9qxqDQJ9N9cwbm5mIfvmnswTx2YxPojxwa5yzp+xlgewDXiEKDcdKzcg5xmj6zjzA/wBdaCz+KN8WNqt02uA9HfPM49QweBRHMjD09ap1yyfXDFx1X1wABzUZ7Pefn/yLSs8RZU+148uJQIz3NBmAHM34d+GxeLKxxOnV6vBWG35NKqBngciPJPoWzjXheNKtp2c5eM82jzZPMNbA1HdtGHqWm1VipQo85vvDMw72T8hqWzJ5/Pld+geB/WuBcIqlRplLdocHixJCoSj1jj8sGefyZmpB2rfE9qa+3f8AP9Z4ZxjB7Bkz+dbRbbHl3XkN/jxXv0KvqtNzj6cWU9Gd84Li+HpC0qnR+Uviee/eyEnPAPOlVT47xqzHoc65vu8oMwqYBmAHhngtDx6zqfITmbXm06pVyZPzZ3mZ4Lz4EBs+tXF21X0hbPoztcqt9xjjsPaJ6IAZ/wCBZqj3/wAXp9ej0rm6VVXXwzCPLjM7wWGnT8y6pTOE8InYa4/fkZxmtn/H9C7NGpt6PVmO5XKOzTZpnmAMnmB4eRcs4rsy6bVNdynPPfQojmL4etsLPPn+FToYewPBc3MDB021sOEtz0p60q3TZctmBhJw0ZB4H0LCVW53KbNNhiCzJDseDfmvFwUn2DoH3hgs6exbW2LYvziRTpdZti2OZjweveDJmfkDPrWKnvaJg5++0MO8DWVwva6ClRJINOPhvwVnGjBDHBjYCyh3hKgYRAgxjw781o2akxJaB/ozXO220rO+hzW/OyasOEsZiNcZgDeGuyapaI9zMKWx52TVPwiuSovXrEiPv7D612+HI6lcNyXAzeX2VguUGHnG5oHagHX8i0FKZnHCD1ry3MH18v7FZLiRbb8y8KZX/sqdw09iNg9HDADA+xai26w/UooZ2xMo/KgAaUvDeuhgylYsOs2rSAqst+G9Hfe2YPb/ANCy/gZ+Owyf1gup8Y6lUZNpU9h+mxtJiZnq/kXB3rqpsarhzb/3fvXyfZh9LhvbQHUrfjO6cuVDZP5wUKpPQZIhKpU6MZgfYub1iqxXqu642/sfPNWEOpUN6OEE64EN0+/DPBc2bR3NUFYw/BD6FOZuF8w09QFz+qm3DNpynVz1qB/gs9CUA6qDoGEST/4JrLps7m4C4aOyWFSlGyf9zmr22IbF21aPSrcrgc6Z5gEvYB/nXKqk9zMrM2DD6wV7ZNSfpteiVJjrYWuLO5veIt1XdZ9UdtyvwYzMg2ddkw6HgXJJlYcmO67kUHmj62T6DXrnijQaPxL4d1ByuQYxyGIfOwJ0c97J4ZrxpZNKfrd20q0qxUXoAOPeNIDsXf4c6RUpjEl3XYiMxj8jKhZ/iLqHGPghSuGPJVyh3+Fwx6g8bOkYb2fn+dc8ZoNZqv7JTpJ/kVwwuEdkIvvDwUhl5j3Z5qazY1f95BmM/Qo9VtWsU2KcvkZj31gqe4HrGYaYNgYfOtBQbk9QwwYiQTN3zgaxjPrgPaUqeH1xjU6Gct7YcSSH/Ymucw7bRLwo0yjfzyZsyz84Iwu02YvIsVVl6P5DgGsDbEB8PAnQXg8mYYLUAyAezVtYtoA4hMcvoSn9YPkhmnqVdVKkv8i392F/rN1nBUIMmCeNkHhwMANZLw6hGepsaLoevKaf/bImalTf/isP/wAZcneoMT3beH0LqFEtWh1KygcOCAOn1mC83ymFKdEPnebgzqUH/bInqJWZJajnqc/okrPnZO82wlAGHnBRwtXN3QN8P1rn3zLa6NQakz7eJG+TRk5rM3Pc9wUqeFKO3I0mOHz4KPMs/RaN9id0dmar41KfqUB58H+g8N6r6fYOloIFeqLzoMBBAM/nVwEOs/2GN/5n/YXNzpUulCZv1kHvkAFOgUdue1rt1V4PkWv08p0twdNqr3tILJ/95/2FBn0Sccc2/U3/AN+WVkwApRhqVJ48/IatYdpP1uLr02q5+drmd/6E+nmOlLpr1Yegepm6UzMjsdhydFSgptYeiuxPVUCGBgYft6wRxn2Z/Iv+1BdFZsylVuz5DbmYS32T0XgPeBr3x7PK5ul5ln23VbJu/larhqmZmBsnmGCdu37tUgf90+GZms+8zLgVQ26jz5yIp4HmHWtbeElh616fOY37AzXfd7Lh9ohsHHDTcjPfxqU9MYhyPHpWwO9lZK23vv4eBJ/QukQwtx6OBzqlybvkdZNcbxGZqtOkmDcSX4vkeDBbG3jDVDXcWc9SWjJ/6VjZq7poAzpfIu/hZWrL8Z0XTcDyLNUq3p08w2da6RUtA39d8M0EOZTTdBjQAPyJzHCiUqw5xtaYBGD59YFVVjgDUp886k3WGYwH14LehAgmeo42p3Mvw/AYf+7+RZur8ObwOCDcB0H/ALQZ4fOtlGt4I0cGHDzV2bKQM/v4360GfmM0OB/pV/lmvP1qiqUDhfM3yp0Z7D8iuL2Z0YGpqMn9BrmMx5aq2s62fDJmAblKB43Q7wNc6mVJjV69isKwebRrLmB5o5k06lu2dCM6l939mZqpNlSIfNngxFMAd7M1n0nhuLDk0aNMB/1rJ1e9pdDOpOPf6KnAEj51ze3rMuaNPanS6bg0x14HmulhPg+09VRlx+y71PUgrDzuFZfAw84KD6kg5/6Y/gVtPnwZ5eIAAHkBRHnqPD2aGa44RY+Tp2ODbgPIPs3TZLXNhUWQd/CNVUnTPN9tvDBQXpjYLphytNJmRIDWmxFBkw7wWfkz+Z8RU8yeZjp82eHkVeFSlsl4Dmz6FfSlrYFY5P28EJIfOolYqrcw84MQ2Q8mahc+29FDZvVecl9k104XsZyZYezcM0mfXhlruN+F51H5r4JM1X8CUf5EwbXRzGA/agzDyKWzVWDHUbAw+tZw3tY1OZMAa8foWa9tKzds5kNPXZP8idC6qrJ/peaw8mT96zYPYp0OZ4vtF6jbZRpM73754fWjkzKO9HwlMHs+fBZ/13h5/wAiiPVIHi1N4KNs3JzPAvE6/OpBhLBrXANikVKiNw9WWb4BmeYAo8mS5GpDrbbmGYYLvhJ7Nt5rM1XvG2DqCNUmGYGDmeqo/tloJGsYBmx1p2NPfz1H3DM8O9GyyxpGBqIYYO6aCRmBu+I3s86tmZMtmEfI7AMMEzJpTbNvG/73sVaFYlm0ERz2QILKMb/L6bjmw+tR3gYZdwb7DUiNJABSnxucdAwc2d/zrO4Us4zJz3Wn3N7WGCkPUpgJGu30NhvUcJPJwgb3/kUjWcezccP2gYLlFfRDfOsg23vBbU6JzjWuw3vBZW3ox8wfyGugQJOBg3pmf0I9Ow3n6Db5vzj8VjoXPJNYfmSDfM1q+JEl9mltMMOb5R4AudZuAjxYPSTeVwzVWLSdBjY9ElBvzNZQ3nFNjPMT4BwX2zOQG9nBaYGmZh6NS56K33gar7znmdW37MwA0DMypU2L4jnR51RVuqvz3wfc7AwWOBpYFeCNpcrFABAP1q7+1v8ANbub4MumfQuXhUpfnTvPyPOr6RoKxW+caCL2B0GqIzyQcy4ftEgMF04SRpd6QHvR5/Bemz+s5/Uvusgw+KDE0QnMzME7zIPEqc3t6RvLzC1rrYBqJ6NVXAPoBU4PZ+0PBOgbefWpGlZmOH4nQlWAgz6NoSnMDz6wVUy9uwUhk8/aLBTS0esQQt84IMYckHg5hmrCj1uJDpzUGW/vPes1GewAwb7wRs6Zug231go2Jcx5vnXdNUNwzMHQg4eKYZq9mb8PDBQjh1Ke7p+rWTw6Hu9NjPxpjcMMJbGs0rA3qBMaByIxyxggqVN0djj+/wAgJqmwAMzbcfZD6zViPJNzDNtwPzqqN7N1O1IH4084jn5EVNZYOVpv+RdfG0gbMNySeHZ3mpYfcNgZn+dIw0fZqwgAxgZvt55qbtGEu2A5kJco3AAGATJvU3lTcibDYPBNGYU0TOIxgX1mgejNyWo7neb3QpLd24aRjCFH8NL0kJmjRLco+nvlPG4H8C0Fkw9FqO2C5Z6S1SfevmlRAPM6dDzMPJma6Ic7l9YqWjP+6NgANh0I4FebhwNA4gHmfZ5Fn5IOHINTYdKnTBP7o9gxsUXDo2tTnwTadYYfMw7MwwTRvcyPjvnn51CZo9YCU1nBMGc+tOvGw8+bbHRnsUYNjrB+tZAOSnM9BkABSKaFOBjkXMwz6/Io8Oj1WZKCLoYOn51YVKlPw2ggy385DD2GAB0Jg2kMwOTntAw+D0fswVhAksA+1r5gGZ9Ydar2QNme0thPrDFbitMVV/PAPBMw2AawUzNeeYk1SI4wBhv6FprYjZvtLKPAB1kMDAwAAADBbq2GcD1PIBryxzW6pj71zVDUPMAewBWFEOpHFDlXzB0zwAPOCyNSeN6qSDDveM/41JjPS2WglMOb2DWzJ12AywcWOw/E3OBvdz6FSSQfjVJ03H3gDsBU8OpOTIBznAAJAdDJq7gVvnGsKxB1sMAAwNcV/oQdOfR8dkQweDoM3s09Go78ymu1WI2ZgHZnsNXRnwywBtigHMkGeGeeiAB501Met9mRoU5iSy0HZnmo8DllYN952XLltmDpnvBHAZM3Y/h7M9/0LR3VApXt33zDXw0QWfen6PhsdfnXZxtQVKS5Aqh6bnhIDqrbO9jrUepG3JB19zrVV1rqS0cmsOG01ymsyQBvP51Xsg+8BsMAZveQEyyBvHpt9as4ehDPU96s8YEIIxh7RveiDMMMOxSJknOQbjiEJnkXoQG+crXfPMzWojMsSWgpstwwzMM8FmWd7ufYtnGrzECFpxKdG50wzOQYdALCxu+F0xh5qW/OfN7XMAjAZ9gGtkzMwfdY1AA2OvNcdO5AZdByJs8+AYZmqxm534c85z55k51r5tuz1vcdlrEzk6b610Dkh2Az1mslcNVi3DCj07mpMbQz1gaewzzWUC8H5keQ2B7DPNNQ57Dx73N658Nub3EJ6sRKPNkciwDJhsM/kUF6qnP8B8De1w6D2ZqRJpUR50zlAEnyGGzP61OPxsMGwzbDAF1acccwIbz72DDj+joBgYZ9AKwejQeV14P3kM8OhUgPfeMNQA+fzqXJqoBH09fAGA7FODe1xRKa5SpDVSqQYO/0b6FYPHKkyDcl6OkfY0qeNVWKlCj675+AefQrBmq8/MdY0+gM804f8zR128P/ALmNHb88lj/9Qax4T8HQYbi6OffnsWm4nTGKPw5oWu5/SWP/ANQuWPXzTZmcRhh7Z3mtvbV67ds1hiAf85MPMmfQeGxV9816jPQPWsHWOqxQwBkA2PAszJvOUA6YOADXkMM1VSak3MiyJzHtYoYfr/8AsFxx+hjzXtQyZL7wab8SMH1s4foULlj92pDzOGGe8D3o8zZM2w2GvpQ5kfOW8BxMGdgZ+D1mqczceazNgw+Q1cBPYM2dBgNgYaveoRnrFp95781txihN75ABDGPORpq1egQQA8/a9ip2QNmUuzws7JDekCeAAed3p02WGc8D6DVqRD2JrP4KQ9v/ANdR8PFBtQkYJ3D4oADB0/kTy9DK6Bc8nR4O2VStP28l+V+jP/XXP3gXRuKJ8tbPD+C51sU0zP8AgWt2pgTNzHeoh9anAe7TUeSGDqxSaBSATKWe7BWpLDZ4iDrdNzzpdiQdahKRm4zsBvMz/QjAzeFqI22Gef600Z6x4H59iAHnIzubDh5+dSL2NGACZfnN7AeDP9ahXaeF6V3/AOsNKZVXOVMPnA07xC/9tKh84MH/APeAQUoHgfXhmiNl8w+hNYfFO5uYHqZnsVIS4cwNgdeB54H3o5gOG+b77eAGqwM8VIDsbQWEZ7DYHQnXjwMNRsM+9QmetHJzZPQcczMO9SCeNv2ikQwbeB3P+zPqv61IzcBrr7MFS1hMpoRo8LQDrjA8adpr2GfkPrBCzPMxBt888AwBO0RnOQ1p4dee9ZeRtrYptfPCdQw8XsXYKPMlSQNuXENl0OsDVPavNHKiOP0qmvYRnzDCNhgexXvrud/8v0T+MF8/y18Jx9Cx3F2qsU3htUDcbzOo4QgD6+9av1+5/wDLNK/8ya55xvqvP2UEX1PGh/fGH82T68FEOhS+j3n/AMo/7lj/ABmt69c9uW8/UDqtRZAze9kB5muRcN5LEanS+er8mmx5T29mP1yfItLAqTkaRqWlQ4zOfRUZHjPGrGreuq5qlD56h0D1JE/t1WPD+BYa4bqsrVdfrlVqt4SD62mT0Yv6M1CrcajvVI2Lxu3Oax2S+xR2YfDln/3jpv61qBo9yWUcOJzds1KjyGM/v1Oknn1+TNXUZmcbQMWzX4FyafWHsZX6D2Kt5bh5/wDMdN/8ZRjpVgHviXHDB3sMHsDBWzwvgrDEZ04NVYOmyGDAzCWGAdfnWSv95+Y7EbfPwjA9nzga1R1W56OwcSc/64hB7mdvWKud633osR+j0c6a6BnzMfWzD8nkR6C1Wc5TTf74zXRQN9lrTY8+f+Bc6tvXZi80wxrGwuh2YzWb2dqcSm0rCRTmQNkHng+8nn0IJByX+YDxzwAANAbLEnxDDN3PPPvUs7Vvhk//AGfjdGH7eH+ol9mL7Np3QtHW/upgLfbBVTJhw47pynweDDBkHgzwXOXvrXQJNgXxMaNv7Izz/wC8gqeTwov8Oi2DD65ILG3jOxpNNDCI/Tc/n1lIjGECQD8FvDDoUi4bDuq0qbHrFwQQjR33tHY9meaoecw6HFjcDZyZ9Nn0GW24xhNfwwez+dNUqiMTKQL7578zA1n4EnN0N60UaY+AtRYjmDXX+dQnw33De0qHMo10BUsNXkw0QM8M+tBaVq0qNw8r0uquRtbA8MJIZmGCyR1KogOnFYB53yYKpqU+7QiyDftKSAaJ5nyZmtot1eHP10uz2Ys+yHWD9rFk5ga5kD2YLcWZJCNTTzbzzPMN622jDuUCz6PzFEffY+77Ae39hrNQOGNcvDi07Q7ZpRyWYtVDmXQ6GWM96zgHOeaB9uCyYH5z3rtfrt/g/wAG2otDbN64K8yDzxhs/wB9hr5vs+z54Y/Q6+GNptV9G/hrAuZ31/xph03mjz5SDD1jD85n/kWfrHDr0XqbMOC/x+reqH/5EA1x05lZrbuvX8AkfOaqZ/tTYXH631+Z3Zvh/wC13ilcJeC1yRRl2/xzqu/oCXRAD9e9NVXh1cdsVKFzT4VihSnv2unHmGzz+TrXn9nUhuhKiPmy633gukcOuIVcCbHtmdVZJhK98Z/41Hsx7nDG+FfDfDd4t32w7Mtx6syLn5Qwm048AwPZvDyLcActkvuiy/D03Aarv7t5j/AtdANfof4Rv5J7v3o5joSaqZabnQlzNZZHBvD84KQDzaA3s+w19RzmpjPrWG7BrLEaY0/1g8zsNZy7bPg1incjSm2aU6B+DIZZ3sh+taZNPdaj8Nnny/KDVbJdietbqkz3XwM2TAz2frWc4nWeDz8K43Lunn6xAOsM9/n610Dj9MpUOZSvWIGebJ4YBn5FCuSZbn2KplSqOD0djDAzBflP4j+Y5nR4/v8AFTh8yzw0tT14D398GH+dR6bb1fgOnLi0oHo+GGrHNdF9fWkfRR//ALypdErdK+9hEi6ObP4OC6PHlyIPBk7/APWlQYtm6nreaN7xgNkDAz3+dZ2ZYFZZqk0J07M+ZMzNnvNbrh1VWJlwtMMAYfWCr78qtVpt21OKw/g0D3Rgtotzs+zZjDPs6HJkn++3q5ptKqLIabdvgAfOCqzu2qg14eiH/YqFPvCuBTdduXmB9GxRTSLaujxm4dRdY97hgYLL0o7Zt7iDR34J6zpycPoQcKKrzJ1Pnm9aRrBg75OtY2TDcpt1esdDDQngef5wXX4cz0hxdqtVt52jvwarMgRH5hsyXYm88FE4e3JKqVwu01+4J9YpQBmEiWzgbJgtLflvSrtptPCC4zqxZITd/eC0B+MS6PBDlt8ssBcsvXlyd+GzPYqyBSmPaHBZ+vBXvE6BRvXLUqcx4r7PXmsKb1RAsIlYeZa8ma+f7LshcViiS3pXs43/AGwKp+wDczxH2Kb+g1LOBXAHU+1WfyZqJ/OPvKlJ/Wud6kU22KlD8OC5TWWg7AZU16g1ySep61ZBUr2wMznPYecDUs6bTtIDcrjxn14AiD529UveVGG8Hzs5qtOByzvs2fyBggmRgwacgvvGHQebyaZDRL2hotdw4cUBBxvmQ+h40zUrYo1SMX3H3g39BqPzJgPgOKKcxzS0Jb+at5tqpnDGlTLZaqVNzZNg8zz3qvjUp8NgVHRP60qVcM6BDkQXH8474YYKqmT6V7M20NtLyc4BAPWpqvZCVR5kgAnG8EoM8zVFAebhugbEswaz3s9hq7qtYps/SfYiAyYB0Aa82bTQqT/vDS5x/sNZd64WGT35gHnVqBmvWyz598+s0HMmBKvAzTRz2GZDTbj4AeaLwsDquj1pMz3DUfiFMYwhTmGGQdMAA8FkvXEsPfo524598/eKPMqVZBr7i5gf1msTMreB++/Il67cOPruOax+TNGe2mCq3j79zP5zMFcMz53v3DzXN3rtfP2AYO/OiZueqve0cDNeYNuhyXpUn35qJTarK5V1g3Og1l2bhlGGgbiNmpaPtOhem1nMefeaPTShhXGYoPtvs/QombDx5tyjwUjnGGR09c/zrzC4s7Du3nx9mGw94datmbncpoBOiMGZgs49R24EX1k2YG0/v2IGZkST4Bphe13649d1L1q57U+tWsOfLA8+beD6DWfZpvqcGtN/WaM9isGXlbZbGbZ78GTPz4ItbZp4M4fQq8JKMHs/DRCXrAHuI3/goHpmex8GT+sFHM8D8RNGYGXhqV4SAeY/sjP6FEe/aANsEDL27BSOTckl4a19a/1sfZjxhLmaZtA+oR7B1AWigW2/VWuU5sGfnNV9VoLlBd0HJYSfnBdPM5vWVoVh9loM3MAVfWKxrBnFfz8mBqsueTL0gisMGefkVJSmZzMoOvDyGvm7d67C85YYMTmDA/rU0KrFea8PDeqKpUp+TIaf6MP41ZwKa2yAG4efyLHvZCmPOG0qp6muPDqYLUBGYktYNtqPyblNMHFt96GMmU3RE821kp+Gfs121l6hzMObYzNSXrSseYAalOAPoT+ajzo9prUUSyabWIQS3LgjM59ma6VW+FdpVVrTiZxvoVVD4V8thF5pk/nWv3/l5w8KPTY06mx+VCpcy0Hz5pqSE4yNwIMnDz4LQRqC3QcGAcMwDozNRJkyqxpAPwekOtZd35dOGM9YP/vkPM1KSem24YfWC1r11U6pYNv05lmR+Mmpj1Nk4YOM5h2AtvDgtnDCpQ2tkoPyKFJkvmHidfetLMoj5gbjBh+tZSTJbZdwc61th4HBs/fh+dTQpTjJaj4bFXnJb92pbNVlvRTiZ+EtQUwNHw9ir3nsGlIe5o1Xmy/np4LJ7gvDkh86QRn9X2exHGDxfHBTcG/dqjCvOG4Z+I4pAMmAaef6088gN7P2nWCnbMHiYeHgjDUAtRR+ZP3aWtn7RBN5xLmW88HzUI9B4N6WiGWzrV4HP3nn3j1HzMz+dAcx82tDU2IzDM8E6FHfMQccwBdSVeAH7RTmVdQ7ebZj+I4CaZoL7zp6mANecF5sRzZcAg0zA06YOMhqGrNmAxGf84fOq+pHnI0OxehrWOT4HeagzIbkN0G+9PPRnPaA5gpBvNm0DbnYrChmZiAH1qw6AUGlHrStRtg9is5+g97NYqGyDjwb+xSGWXzPUbQRp8Rml8q5FPV86kU16IbumGZ/Ws8PfC1pWgz4krYHnVmdVrL0XCjQTZaD+ln/AKiQeqocLm6l+QFn69XqjWHfElmzHDojh0JhuqjgPyZ5vvznpLoBnmfYqwAzdP5DV2EnloEjxN5rOA84DoA23nmtHOmhDceMMOjvPyK+ptN0YpzoNOekgB4ayqYe8sNQA+tdFtKM49aU1tvDYZ96yGCen84buzAz7FS1KNh4hqwMDjOuvmAbDTNVZCTS+ew3h0IKfZjnqJAYJowQBsXR4SlmaaM96SZNCEgDDBDn8EyCRqw9rJa/zqPn8EgQSgMM0eQKIkgkZtmng2EogJ0DXmC1gB7s0fM/OqzWc/qTwPfu15gXoT8+tWFKeYN3rBZ9k96kazYd6574VNQ9JAA1M0bM9g4Zt5+Kayj0k3gwbNOxgYDxM8HfOowLuBQYnNGb+sZn86keqonOusOAyzgHgn86jszMw1GJYGajzJj4HrtuZqBSVuNUmZn35sMD9iYKt5x9k9NsA+vBXFeecN1pxw/aAql6A/iDmG894LpgS6bJbedwc8ikBJBl0HFTsvaLqmgGbWuvUriY9meal2qfP16FF1NmeaogM3vaGtlwupsWTcubgdAGtkW9HWTDzdjgfkXm/jNWPXfEuuygczaB7RD6AXqK2AbjRXpx9EWMZrxPVaqxMq1QnMdD8kzBUjhJmMZyNMADWPzmtdW5OcUG4hmZnseMFzx6Y491rRwJ5vGDB9eClsKq1WXAi8qw+HO9B/ICr4DLARzB/vR1KSxPlA3Ebzd7z86kM0SWbrTkqWyAB2d5oNbApWFL55w/FBnPNUpvAcg39RXXrLWi+rWHMI/kWXnxmIE0z1DAD7+tctjUMyYj0XxGAB3/ABpqZrmYAx0Aq+NJb5eO2+Z5mfYrMAcNh1zM8MNhrxSDGDmaotxDk+raDLl9GAGsrR2c5QGf51dXJM9W2lLPZveAN6m1uWwGc556nnNSHmW2WjY+fNbqpUe2KbTo/rmlGEiaGsy8HkWaOHTvcAZq4tCPAeBkN5gB9mamsyZTJG4YHpYdeCOMEFn5DPrzDNTWdCSeDHeH8Cxu+xOE6NJpQDpuVWZpdhgyipsyIdUdDXPSAOvBReTbANBjoBOmzEptLPT9q/vzUYQOsQGK9KBiXUgZaA/BzWM5N8yPTbXt7h1ZPDysWpSWKzasCZNBnxnng3rzXxRZptNvypwaPTY0OOGGDUfoXZwrc1CA/JaMG2/rTR002YuvqZn3grjOIyBtg+DJH1oDZPkANvvPBXtAJ9vP0em0qqm+BtVTPDDswVbMPA/kW64uxnIcC16c3/RYxmYeRYeTGfkwI7DDebr54JtZGzrNA4mtHBe2KbbFuVu6IkT1bDClWvD8aOEYMDlGHXn8i8lXbVWKxeVYnORAjZvYADIbMAXopoeeDvie0V3AOCZG2+AGBh3qiADAPDc3GfQnQNxn2izsXfh+8U0LJuOt0Z244NGknSmDw5gA2ZrOvPHyby67Upk6fI4f2BBnZ0enMwnnmQPYZme8zXHzQQ5/Gt6pUdhoJ0GTk+GYHo9YKOEyCHv16rk3zb9mTam3zbMmQ/8As0cAzADNecZNsRZk2RLluBzEp43jwDDDNRS7/QpQktn7/NM4Vk5B6bgGBhmBmtGzZkTSAGHAD8bPreTUm0pbzQBz2iAbDMDV7YszDN+SJnKbzDrDBWGi29g3ygGZngGaOpPQYBhSqM/rAwGAAAJUeS+zUYj+BvGw9mGHRmqbRabAtK9Y0jXfs6qhE8mipdsPPzJp5wTjYHhv711Cq8YK5MgSILAAzrhhmALFUoM5oKfEMdth6QMZ+TYdtNsSwZeCYB4H3+CuGUqBd08nfVtsyawDB4PcpGMwA/yLuvHuS/Gt63wb+f8AwKdfN+Vnh7blpcObEqsmjx4VKYqFSejngcyU8HvvOo9vlw6Yef58kHpQS+RAMNmAbDz+dWVHpT8m2pFxhOzwe0J8fDf9a13EiM2F5S9TvZYfP6zDesvDeOHzrbbnhTWdEwTwzTYfDe/K3AhTqHbMmZCNnY70ZrLz9eNUnYj7eDodYLsFE43natuUy2IlKA49OZ0TzPr/AIFySqyYlSqLs5w/Ff612caVbngSZeeAw01Ikxm8wcbfDfs3pRqUD2q4/nmGwPItxT5/BCaspNKlsjqaBvBngGjvUV5l+MOpKiPAHzgtQVNtusXDzHqpgD5UM3szwwBDPpU6lRQCVhv6DA8wNdA4LQGJ7td1/ZGywBh+tZziiz6quWPSm3M2gjAYfnVqZdnq9mk8y4y6BsNoAk6J9GaRycwDDYoSkZ/+v50lCAz1QU0w/DQNGC6HxpZwqNuQf7LRwXPMDeqMSD+PJAF1XjSywd4OsG3noUdgA/WaWOT57s0bx59+9RANS40Z+ZgDER4zzw6EDSZ98px0qsxg15dHmAAd+Cjmy5hqONmAfOCvajqsKVTTqrrwA4AaAZqvyBaCzA+8TXP3IKEqow0Sw/jSBaCpUSXPf1IkEz2Zn5FHC26+GH81PHn0YKBn5+py5q6vzfdBP9j8OKYfoUKZDqT3t4rwdnQam3gy4EqmG530eL/gQUQHuzUt44jzH70P41E/d96QRpeBv8qeAdasaW1bMfudiQ+xL0QYPDozWgZ4XTmc/wCdYx4GYdC03A2BzlpTX3O+ZgtQAZg7/fGseUc8pvDepVWaFNixJMmWfQEdnP8AXvUu+eA87h7Szrl1XjRI3kiAebzx/IHeum2lcNx2ZV/WVDl/d3w0ZkQ+iSH19mCrOMFHtjijhWPUcmj1tjAObOTrAYeQwURbVx2iWBdV1RZFVtGgT6rCihnrNM4AYfn6/oUGlUeo3DUWqNTW4wTT6wkSQZwP867XZnFe+OEVs/ZVtim1iJCD7gfQbP1+dccnz27qvp27eJLD1V50zOY1BDljPyYYLaLThbXnwrv+wIseo3VQwjQnzw5hp4HgD68OhRLYjc5Uo7DezM8F226vSElVK14Vs2lbnqdqVD2HOwewYwwDAFjeHVpfzoFR0Ngdi5uZDqFt0R+jwgNyqyZh6JgGthszToWrKqroAxcEmm/3IAeauGY2ECJ+RSIGw9RfK7nZxqQOHtf5jQb4kyQ2GfQChXPwWqNw031VVeI0mS1mBh92A8DW1ZP76H9yakPTN6dzZyr/ANF2lPMBEnX3rB5OTWgpvCinWrChUODUucDB8wPD6Fswk+KDiUk86zC+h/8AyLbuHObh9Ga3LnrciuSrqnw3X8NjUYMNgYKKHoo2qH/v/Uv/ACbK7K8eEczUTmc32g84J3Dj8n0V7SZ9peNVe/7EEbPoqWO9n/yqrewM/wBmBdNuqtt0GnHLbc39iws++Z0/P+eNHPsjnsWP180C9/kit/386Z+Q8FWf+jHwdeDNz12bv/15qPRLwcpXO/zkbwAAGHMH4KtofGOCznrxI0n6AMFj9fMKz+Rmw7VuOj2rTWJhwrlAwk8xJzMMPIrj7AWJwxrkRi0p0/VmyQZkhLezwBR6rfNNuGs25c9NwAKCZm80Z4GefYCzVz3Id23GdYcY0XTZBgwA9mxbeOflHVZlpUeS6bk64+TP8HBAFBobPj/a4NgYYAGGa5PDwpWbj7FSPXDA3eZzAPoA1SvUS2NUwbcnyZHndmdf6F09x0uxBZn3jnvWpnn2GCl021X4xk4xUcM/3KpA4qRXmAieqjjSGO83tgfwKdRL81pH35uMDXnaezWP0selU3haVOuR+oUa4GOcjwofOh2HnhsWfszgPw2uewKPUqrSnjmzWdd6QzJMDT11X+fraXLpQbJsDkuv+NM23xUft6kUy3IlHA2oTOjq62z/AAJ9J0pZ+jNYkCLqU71kegBmetJzzWPmWxaT0ebKnQfu8Xoia2/MFoK3fNZqVUCuRa4cBqEz+ydhpVK+XKrAdfnVWMGfugDrXP8AR5OlRWrrwKjTHKBSnjm1DrjgeeH513U6a2bRsPy5klqUzvB2SeGBrikO/JxwuUpUuHGkMAAGejv/ACKRG4kXGy07EqNSekif4Ib/AMib8vW7Dgzwkz32BSv0GjmcPeC1KzY+x1NZw9zgaxLPEipBSHYLEsw1+s5DPjYKnOpQZmbBubz6z7zXn0+WqFVQpUaqS4tODBrPNdFvy4abao2lWKjSjqUJ+G/FAGTwPNconz6GGrEYYk6p97xrot+PUCt8PrcYfczNh7fs6Niv2eb9i/UZmq8VINVt/wBTS7EhmIPazMgOsN+YLG35f797RY8SVBhxpDB5hIjs4Gf1rSwJNKjasSJBAAYPDrzzWdva24tSdC57fiAyeH3yOz3/ADgr4fZY3wsYBv44OdakUeS/ArMec+GZ54KvOfFB3XYM/oNFDPnHQcB8OtfT/Ln8Re/0PTFmcRWOHs2WxVYkmZEqDzGs9n7HYu4US5LYuGEFStipBMid+HWB+Q1y3hdAo1YfuClVxiM9Hmw4p4PH8nYtrY1h2/YbUuLQ5xyQlHngbwHh/vmuvhvD6PsRhsAmQQAHHH8M06bzGlslgH1pMwzP3f8AnRnDNn3YYeTBfR7PDiNBynfLZz+tQbkqTdHoM2sxYoVKRCDNmOB9almzBPris/oSZZYZLUYYAPoUPcPMV23PeNz1SO5d1Gh011sMIzTzJgeCr7VqrFSiu0N+dhEYPDozWl9KI/8A7YNH/wDoFy+BsgSHA9rrHgC/P+5w/wB53en+vxTpFYgWdpR235TzzoZ+MyyG/wCtVR0e3KVKCUFcZ0u/sWXjevAawfD+NBPh8zHwfbXJ1OLytbYuSm23XGaq4+D2h2AfWju24abdt1y6rFzZalYbD+hZcLeqXu4msHyb00YORiab866HOuDBjlwYcPf507o2+zFabcYN48Mz37FWBRKq94Zy2Xmnw2GBoZkOvxuimvIChzIlElYUaDog+e/BSZlVYeim4DYHh15rPYVmeQMRIn3jPzqOyzWOfdiS2DDZvW3hOHsCiTOctyjv+eGoVKjVGMTvPSwezPMMA6Fn7SN+pcJd772qxAMGcNh5h86tbJmc/aVKlG/rGbO8/nXb4Qx/G+M5hR5YecwXMgjP97n6DXW+McYztlqUAZ8rJA1xF6pPszQiPxDAD865fZh0xbRhG5Yd7+YfWgk1JsOhxZeZPYhyAYcYmMmYbAPoUd6TODY++el3gCxwzaZmtt+/NnBR5leYDxNMPoZWRN7B09dt7DszUc2Z0mQHIwXjDzgr6UtgFY1mswzD5DUJ6sTmS1HOjzrP6z8Z82HzMwUiS8/pe02edOl7hdxrkfz03HA/OgmV6DJd035RgYeRYKSbkCaEvUzPPfh3q9jPBWzziADOGw8DTpSu/WUuHg5rmYdmaD18Bu6jnWoUPldU4suWGAHgesimQ6VAdDDAwPvA0wtZ/aHAPkRxqlzm+I+H51WfzdGa5rXZMPwj60maxaphqZmH0GmBpTCLMY0zVtAmN6AMA+B4eRYmHcMSAR+GEnPzrXW2zVbkkNMUChg9If8AcgYB/jUXDfjWEmTyzBvrA1KY5Jla5uLp1b4UcW3o+gxY888/IYGsZM4S8X4ftOHNYe+gAV4bcts/Mqs6Tg2+5ngky84atQ4b8VHv/wAHtSA/32AJM8K+LZ9djyQ/7ZMMcKc40sPEffA/oUfwHndA38Fo/wCRri9/8v8A/wB+BSA4FcW5PtKNDD/tkwdNsoANgpYPAC0H8gnFTzwP/GP/AFE6zwB4mn1vw2frM0w8+a2Seh1WS/qRG1LBmsxg8dsFuoHA3iUy1ptzof5MzVmHAq8ZMc2JzgPfQBpp78fM50zMz94gOfg6Z89+TBdNh+j9czLoG2GYeTRNBM9GOdJf133J8bPsDYC5u48enzKiS9rWeH0AsTDefemtaa7dA4FXVTacdKbceeaMMAz7FED0Zr4DfFOAH989gax7nT49a2XM3/UzQGxhoGo4ViI34bji6BJ4G8RmaM7EclU0+jfzKylB4OVy6mHXGLjo7JsHg8069vA1cWi4V/rKJj+3M/rUiHUon4il/wDou1wC1Dvi3mfreV3A9H6qssCx/KHQdivbJnJlSifiKserDB+HmtRVeD86lO6D960p7+5DP/Oq8OFcE/El3NJ1fOyzsXmBmgmSgfBzUVxArcvzqUfDrD2F2vH9cMFczKPSodL8CL94D32a6uGHOm2rc7kN3TfDNO3JcMWqmcT1MbOHeZrHxs8/DVhGe0SBw12X/wAMOH+35RXmXDPYpTNvA9H5vYDvzq2k1Wj6H+gwB7zgsTWLwcZzbiMYB5DNfm+be/0PqeL8LM5NKM9OW3nh0KsmGwDvgbAXPZ9zuc0bgPqP9p33uuWkenbHbdc45+P+hLBx4NRx81lIdbz2OPgH1r7MuF8GtBtxL9O4YNEZvslmBq4ptVceAG9cPzrmUy5HzE9N9O0evOG6DDhrL5bew6tValOBjTBs/wAiCBPN7DNs/wA6r4D054NP1kzh2GfWpD3rUA0H3AMOwwVRDs3gFyBUeV5qLhh86xvrt8/D1zUivPGZGGuexZwzxXfw8LG7S5563iJkDcx8RKHMcB8NgGHzqzkwGJIajDAA6feu/CMKeS9LDMwc2fIqd43MvEUs3pbJmEv9CiPb1TM6zm8HgOBn5DT0ae3GI2D7+9V+DgH0bEyb2sa1TDXQKlrGAON5tKwMG3g0zDZ2LEw5kuMHWB/kV3GqvhZvvmuW+FSwOGwDvhtqO9ph7NNHUmzd9oeCXNsec1jEW9tBekuB+dMax+dTZJgfiKCZh5F0eGYcjRazh7E1sSw+K6Aebnu06y9u09ADUcFLZZ2amoCyGPZDB8FZQzB5/TcUcGc/Z9anQwbjYPuRfG+ddCVlovmGppnpedNaM6S6DEQ8GT70gBupED8uUeAdgK9h1WnUqObbDYGR9hrnEeq2rEjUk5cSq7+8HlkjjLa2rW3Klc3q58GTaPzrQXzRGAp2uxBDZ5NimLHJwzAN6jmGs7pq2BnmT03GzBV8zQjSNMF1CRGByHnvwzUqHGOYfy+dUoahu6jjmxaKlZvQjbiudnWoUamA2zI0wcM0ocl+HK1MFCNmcZ6j7aKjvOPT+Uc3oNdrRKrTQflf8GAfgqO9RKjJa558AZ+QzUqmw+TLUlsZgHYotyVXwj13PoBE7QakbEZoGGAZM+/vWdCM3GdN/wA5qVTQOpTRYYVwdBYMMD9r2IpSxpn3oM1teEtVfeqlQpr7n3d8+jsBYSpQH4cjBwFbW9PYt5jmmNjprISrkwZqjrbnsjPetnf8OhhZ8KdTorIGfkWCrD3P5yvxFoJMzn7Bp+z2Cy8DAz4wB4gKF3eIthdUaD6phVLAwNwMzBpY8N4ai7ISkBoBm4o56ZlqKQzoe8cQHhq6ja9DQbCQPGpHtnfEc60pkZuN7N8D+hWIoB+Iln8EGfwQYmikjP4JGaaA07+RAgT3LOA1qIoZsZm24AKdrGHQqSrE6BoXg8VIFIlZ5o2UozLZtb+tTWaa4YZg+CgHsNrAHEgPFRDZfZdw0D+tG8YGK8uBNZk/QCnADZjqMOZgqeL/AM60UCHtANQFzWpT1WBOwOVp7FI5ls47USUAZgHkV3Pjaxh5A7FXz6bzn3p9wANXFvbVM+Aw8YPsMeEmsGwDDoBSwpp4akXMzPsA0cCG48PjsHmr2hWB7U2103g/GzmS3/nw/wAax4UffvBdW4UUpiNAM2+s3jVxaLdAuqq+pOG1blamDpxsA/OvG4QHzY2N969ccYA0bBCmthvqLwMAvP3qEA9gCjuIYjk3A620nmXwW69SeEbjjG8OxVp0R807nQpIYOcuZthvBTaJMNknomn7feZ96sApT8N3Bz9CnU2MwzPCW42Cu7TcKc57/Om5gfWieqWeZv8AsvwQVg9RG3nXXwfw+TBRTt58/DlsGBeRY7h4jwJ/rWR474BobFevG/GA6drgYJ2m29TocNpt9jeB7zQzMNXw07oFhRGc1NuGfBhw4kWcGbUo8E1SvBBM3nTeci0p/Qee0D7FFrZV6tzql+1vm9h0Zn0AgZeWrolBYZj6j7HWrWBb1Ge1s4vQeCju8Dm4SXHiWgprP3Pm94SA7DBXtYtim8qblOY8VSJkCC8bPhmHgh0Gnd4FaB54MPubD8igzHogNBqNh1h8+xbD1PE9TGDbGbp9+axVSoM5nVNzz9Cjh5otOHeKV6SdlUcXXwptSMwjYM7MN64VcN1BXq3LrjjGjzR56OeeCpvVU4+htRdHAtNxdMW8TQqTZ9DDJ/WCs6O9zMqPTtAAB+SHYqcAbZHUwV3Z/wB5uumN9mZn+gFunCy4zTNa4I7H4EYAWHZmOM8u+HXFPMPrW4v+HFmXpNbl9AAwH8CouQtiM6bHKAZ+cwzNedo9EWfxUtG2OHciDUarGerc0DlSTzw1nzDD/IvNUkzA9QFZSaaekb9Npr2YfIhZhsH4b7fi94J3CpOfK/EAPyAkExzP2gH+RW0y1alJPUgtgbRombY9VRzOuYA6CbFxR6PBmNA4+wGf0LrHD3hXUr8oxv0OqxgkRTweakBgAH9f+wuRWrJnSa9EYY/Ys8zNbgNcOidJZ/uXsFwWdyquqBKtsnX/AFrGmHmbGcfPADBVkOpN4A++5vUu9tflafR22NgMnNPf1/74LOBDfMzcPYCjCIvsaD1xKMjbYf0WsO/vVfUqw4zHPxwPNZx6SBvmww59ew0NKprbzvPVh8wj54A0HWa2yvBmGzLmTQ5VszWjgQKlAI9R8ADyYdCNl5iHgxTYLwB8gdCUapOTD8CC99bp7DTalwExj2Znv7Fa0Hxpiyj0ydnptxAPAMzwBXdsBdr0rUt9ilHgYZnLNI/eybP0gTwC2mPrP/AsjxLrDEy8Hjb9xGixf4FccUXrxeuO0qdePqffhgEH53gDesvezOtxVqdH/o/OMMf4Fj7bphL4hVJyq3lUJbbGGGgx+gAWcMHD94AfWlxOqr8DiNcESJsaCZ/kBZr7SHn47i6PCFrJNwB6GTULMD9ojjVLWDewDP0KDPeYZd6+tbwxJ409GqoMn4gAexQdbNNGGB6a0W3FEmQTtmafN/eIp81o4dYb1s+MFvMHw+ieJ+xQIspnZ0Znga4zGkvsumww5hrhgf0LsFS4hQalAapVSpoSY/IMU94DeDrBZ7X+D3CKAxGtI50Rxk2pWGBgeZ7POuY8Yzzv90PJDYD+BbazKrb9mUuRTYrj2k+ee8+hVk+HbNbrMipV8wekTTzDSPYAYbE+h783lyo0gBdKCiWBD3vsRnszw8V7PD6ASO3rAmeGDbMZ0/YvAZp9ML63PQZ8Vo/nVxJ09U3G+81oAo9o02O0ctiY87n2HsNSjptnScDb5xnPsM+tPpgwyVHjczcdMb/64x/jW14o1ViTfVYbY34AEL6MFYW3bdsyazSpcGL99CYxs/3+hHc8O1Z90Vt99swkHPfzwNPoh5hhKJbeBhKf/IC0YAbPhsHgpHLRIE8GNTwsNiONPtwGtCdzMmQHXongse7ZhPh1hsIpsP4GYdZmngqsHHTlsGYZ57w2AqmGFNk5yjwZPeGH+2nqaDknWYcbBl1gMwZ+RRvyvAKlatlVLN/QksyD7wPZ+hSKJasGBreqpWeoG/WNLk2JjQPy6icb6A600cP7uTASph/OG9N+TC7jUrAvEqQZh1gBopgSobGo3gYB1qopTxxvAlQXjDA8zMDzNPhXqqzPBvkZ70IOhmPGPem/KLgYVXM/aKRXqVQ6l6s9axNbNk8N/RvRnAlvE6/SrZqXMPnmGrDwAEqlDub1bEbg25VTdOMYZhDM8N5pvyjCP6hpVsMB6ui8t5N6Z9azvkP6wT0aj3rJgAEuzq38n3M0Ua2+KkZ3Ui2rVQheR6NgZpvyYDAuqdR2DgsaMaOZ5mAB1mpsO4Zb2/TDBVVVs+9JIa8Sx6weHX4Km0q277NjQOw6wB+TDrQw1tHn8zH+hR63PfZw09gd5qPSrev+Nm2Fl1IM/OCs59n35MgGH2cMHewDkgCiLUwkk5R56m9Ur1BfmSNcG1tY3D3ijzAOTqHDea8nPgCsIfBziNUs+RobOr+E1UgzNXFtVT6tCTFp/h7whgALfWGbegDHcweBrS296Md6z8PXN1U2lSIQABwY4c4Yf4Fs43o9sQGtOXeNY/7GHorG72rDPveyaBvrwYwD9aUD9lac+v8AxrUM8DbVDxH7jrbx+fPBXDPCizmWAY5uqnh++XNcGGCe2Ox3/Pmnc9ZoDWtmcLqMcV31U5JCQAeDzEnYZrBfYz0iYHgRbcok9oPeg8GBqMOyOFZgeKQPa1WhOOeyDPM1n3qb6QrP7Vw5gH9CZ/8At4Rvb8L8/wDsT/117h0/Nw/7tdJn9bC5jxg4hfZKkNQYMrRqEroPyLR63G3P/wC46B/kNV9Vh8Qp7uvXOAPOH59EzTR8nD/2OJT72cet8/WNSmTHX+85IBgs/GuF95pqUxszPPPPrXdZNKqJtGZ+jmGz/qZ/6ijs0qpA6D4ejmAGHQfJnmH8C108+Pw5VDuqpA6TDj4G0YYAAdiZpVzyqrVGqc4wyDr/AFvB1rsYUepPf/w7/wD9sf8AqJHar73/APDe9+SGajph78fhyqBWJbNU5V98JJ9mBqUdbjxps4wlYGcYwA/nXTgtiWz7D0dqkH/djQfZWqmf/wBwGZ+eMa2xDH4f/dyeNdRnn7Z7Dvw2AokOsczUdNs+tdwZ4dXwf7L6Of62TUj+TTiGe8/RsDPzhGNMwfD/AO7iU+q5tNNyz9gGAfQqwK3hFOLFlyd5556xrvzPCLiGf/8AD1G/j/11K/ku4qe44A00P+7M/wCumOE+H/3ef4dYlhH0NTP5z61HqVwy3mvVuuYYd4GeZr0H/JXxfAt/ASm/+TZ/yGpAcHOMT29jgtbwf3wAB/406YPh/wDd53h3If3ds5x+AGHjHmpfPwfaagfkA16Ah8IuOEkv/uc28z8j3LAtBD4OekSA6cW37VjB5Mw/yLzrPh/93mmA8w9I55jmTd8mijnw6/Pd14kWYz5DOGa9NvcIvSPAf2G2A+iYAf5Eyzwl9JAy2VKjxvoeBMNvk4f93m+NRL7k75dGnvH5wZNW0C0r4klmFnVsyDoBoOtejQ4Iekg8P/tjSg+h5IOAPpCn18SaaH5z/wBRc+D5PX/3eej4XcSZ8U5TdnT2ZHZrYAH+Nar7GXPcNtR7fpWASITPjG8ezPDBdbk8AeOEaOb9R4m00GgDM+vo/Qs/avAG6rnpvrGLxGjRt+BhgZph08Pretwf9zl8P0cuKh5m5VabGP681qKVwTuqHH0KrdVHB3vMGTzNdTD0UalJj68vipUgL/q8Mz/zqIz6Mdsf07ixWJJ+flk+dl/8Zj2eCFsg6Eqq3VG1fkjM4H+tXD3DThkzS5GnFo70htk/GwZBaNn0Y+GX9OvSsSQ8mwFYM+jHwSP/AKcqofXMD/UT9fhr45fUc0oNsUfDmrncAI+jgGZ9/nVwzG4TxvaTo36zWl4e2lbF2yqhSrqY5mPC3gHz54LXPcNODUb2luAf15msuHm8unm5vWcpOscNWfYVypB/cmf+uhk3ywDRsUC57h5js8Y/8660zZPCuMf83Wkzn88YzWrptBgxh+625AD5+WBd/wCfL5X0eu5YdjcZZjEd+Dxew1AzMHoYGnmbJ45sh/8AdXpR/XRw/wBRdqh23KmeGGDPz4bAVszw9qL3/Ssb8gGa2+nmcHc8n3h6NPELiFVAqtx8TaObrAYB/NpsgAfkXPA4IXqzcEix/XFK5sDwB0AMANe+P5PYoF96uDD6GVwyvQ2KD6QoRJctl6OZgZmH0LD2bdPrczh5+idxpZA3KVWaDP8A+rtTN5/wLH1vhXxmtjw65YFVjfPsMP1r3RXr84T234jlchg72GD2awVV9JmCy1IiUqD64juBgDMgNi98OPzy/wBXkqHb3FFlo2Ps5JAz8h4IHrS4hvZ81bD35zBdFOZOmSJEttjRB94z0QPYCNl6WHtHDWuFuVHYfEYy8Slf/fgVrGsbiEcLlXKaz+eSALopzH/xDTrMx/H2hpgcw/k9vsD2U6GH/fAU0LGv97fUWKaZ+cHgBdIzcP2hpH+dbYRtN4bwKrTbSkUOq6OqxmGbJ5gfWqKgxr4syH6gg3PbD0QDM8HutbC0j2ugffma5pSqJBO663SqjZx1XCpH97A/Yh9C2hjcN3f8D7Q2lNpwP4Ov4YGuRBwiquk02dxsmYdBmBruEwM4prPmyot45i9wiqsn9ruqMH/djUtnhLKZi8q/dpveQwZ6P1roARk6cbaow1cye4PtvNGw/ccl5o+zlgSZ4Pth0XVUg/IC6WDKA2VTPDnQcIqN/TqjJM/PsBI+D9vvH4k6fh5AewXQuV+KXK/FatsOdHwNsp7e/wCsj/7ypcDg/ZcAT5WDPDPzzM1vQZToRs0Y4YpnhpZ0Z3U9Rg9/fGZqQFmW4yecWjssn8nQtacZszQhAA1ztGXjWfQ4zoSm6VA1Q35myrJmj2qb5v1KzqPJd8+jgrj1aCXIB+GH/wDtBXnTbAMMPsWyz/8AT4IQoNqsug5TqxPpWHYEYFZcmHkS5NHmwhTTDByJxY5Y+w5DLwArJmm8TdA36HxXoM/yYTDDNV4Q8EuV+KG0h6f6UMaPrsGzMw/BeAwTH2t49gwD863A39bpmCdjHOgO68GUbLvnBaCBfl3Rnwcl1XnGu9mQyBghtlKlP4m15oAluBAPvMJOGajs0q+w/wDe7/8AuV0WNG4ZXbWdOdFn02a/+QP1oaxwluOB4lKch1JkO/WZA0Nsf/y10sHLtZZ+t4EHJ1x5rTPiMYH5M8wSMHGSNt8MDDYexJe4O6xw6O/GDxL8e1fOBngpAMvgXj35MeDyAChazf8AUpQQJzwhoQZJn8jK8O60v1k5A8SJdVVD8izlYr1x1LMJVxz3gPs1la1K27jOGbjFDmf+CuO1V65rSqUtusxdFp88wPPvXD7MPpetf/lsucuploGG6q9pB2Z9Cak1Wog0Z+sp7xh1hzJgsOzxFnAWmDAKXWKlVaxFCdSouB/g5/5187Dp26Vbcy36rAB+WxWNXPefMmotEh0ZmvVBuWxJ0dY8AdWH4b3DczOfPQTDfgGexaKTUuWqRy8F9OP+HByxtuZh2dAa1/UZvfQ8aoalcluM5vsQQjNfgnvQUSt+sndDXjMn2A93q4eo8GT4dV5B4PIvjez7N7xDr+dT0S7aHVXdDlIx/kV8YRZPs2AAfkWErFpBTalnbgaMc/zrW0etvwI+D9D5z866PW5//Lkv1kv1bEw9mmZNNiHDkbOxSZlw1V4wbptDhs4efegC5LqDBhyJSg+fRwX1Y9mHH83K5+zgzsPrA1Le3tZgoNSjPhMNvTPM9+AKXDjTjj+zX0Y9mLhjfGsJnKxqNrvuYAuZVWjv1vN+mhmZ9i6EcOcY6ZxDe+Rnep1KtW8axI5SgUCf9Bs4LhtcQ841K27jjSj16NJVf6tqQH+wvZ/QvYbPArijUsGJ0GAy0fvnjzwV7A9Gmq5GE64IAfOEZXHM2w8XhTayHt6bJD8ianmbOw2zD6wXvWH6OVD9nOuOe8HeAACu43o/cMmSByVSnp+HZIMDT6TD82nnlKpUlgHdRzMPrBfpQzwN4SB/7nQ1IZ4RcIGT32PSgw84J3eDD894cymveI45mal85WA2QXJINeTljNfogzb3CeBvg25QYzod4RgTvM2OyfhsUEPyAsaevzdks1V7w+UmG6fRnGMEDNn3dM/ZKHJNfondVV4XT6WcSvzqPh2YAGa4P6+4a2M7Ily+ZqUczzB3PBI5WeHBIfBzijJaBwLRmM5972wFpYfo08X5jAuMMQAz7Dk4LpcP0qOFbL/qqJb9SZEO88MFRXJ6T7+Bxbco8CMHY6D29b9uHrj9e4S3/bc3ka5Q9E/OD2YGqz7JPsn/ADrO5APOgu3ipfF4T9Sq1l7SDsA1ndFw/bvvH9ZrpGoqVvWlGim+xeOsYdiy7wRQP7q4Zh5zUqMyx+GGaTwAYYIyVQSTB3TPBWYTG8cFXHAfB3U6wTxh+GquNh05LaQSW8fabEIRnDBEEYPZqMA+ZA+9I9NNGy2yW9w0zzIB0OGaYD2e9Fn8FHN7LsQ6zmHQH1phCVrJA986hZGlkavAhazgH4aI5L5jvcUt6BEePU3gmjhsMlh1r1Igk6MVGy9nvf2JrOJ7Nzr8idNlvHf0KAEOS5Dqg1KK5vBdLpV/0evQ5FGrPghnhn15rmRnTg9gijAwDuZhmsMC2qr0GMDzFNbAw7DVCDzHv2DM1ZvaZnqBsBQpIfgLcR+vw1cU2Zy1NCmxGDM/OqkKVUpjvhgf5FZw405nM2APYikjP7qbnejsym/zzz0pjNV7wYStc1rbGjetSkaE4GeVDM8+9BcPH6ymdGAd4LmVyT3J9ZkA2HhAeAfOupT5LFHgS5znWwz0LkRhzMo8+gzM15wpau2IB0qL61P2p9CKqvYNaneo8Oe+boOP/wCj2QwwTR1ViTIM9DNrsU4UkXnsp1Pnae8wwNZRnXmTY8GK3mb54YLUV6YxVbcaiNufeGDzwUWxgg0qfIrlVcwdhBmz86sSrzgN0FiJSgP76fW15FZ0171Va4U6dszWP1p1z3UdV7D6A8gLR1tmdMlHEb3gHQCyepsm3jqsBoHKiAU8PIaCHVYNHDkeVjcp84b1oJNBqNEowRDimeYZ5gue3UBs4B51y+Hh253qNM8enMYLLp143HmlFwcBfU4QjNMp44x4m55FHVpSGXtyIzbNRw3kpAQ38NTBBH6E9n8ET0Zz3anUqiTqkZgx2IpX96d1nwVx6niMwtcDzMFC0c02GQNSGTbz9mmtH8NJll88MGDMT78Nix2lNB5PYS9LXCK9j5+xBo7FYMnLmHHinKe0uhSKoGZZ/ej6FKCM2fWp0yHyzugb+wA2fOo4HivNqTaaywz7c1IkycHdMD2KqN7MsPIaRvb1jYvWZmftHN6UmSbxaarAPAPmT0Z5vV8RRECayb7LuGmYK1BnAQcVaElw+tSOZ2rJaQ9Jb2Auq2HGMKbH2da4o8ZnNhNt/jL0RZ8AAitN+RdHrMrUXHKTm7SqUfQAZmuVcy2yK2HHKvNyb3OKx0RYbAYLlj1SMDTlF3z/AIWnrmagnMbDr86ojqr/ALxxAFSz63FGPIuJMxgy1M00E/BUhz+sE1zh+dXgaUKq4G/ydinBPbMtdxz6Fj2Zm8FNCTuUYGoCTmftM008bnOhg2okN7PxO9SGQN5/X1NiYatBDZzLTbVrMmPnP5TwQBjDrVVTT+8ArWfUqbG9v/8AZrHmAnygZm51qOdebZ8DXzD6FBmVKDpG5r/QCyT1Y8c9NcmPI2vrjxf/AN/hYIYdSB4vHw+TBZIKqwft94eREFSw94vOmxu/WuY+A3gAbMEs2JLvj9BrKMzHHgzcM8Fdw57YABg2B/WmOtqtTptNMMIkF4z+RZyZaVYM9SLbFVePzxwW7oNSfjYS6c/ol5wBa2NcNzPDgxeL0MPIYLp4beYcFk2fc3uLcqR/92NWFpWrfEO5qfO+x1SNoDwPZ512h46y97e+DP6GVGehm9+13xVQ/wDpwwXfsw5Rc9n8SZ9bmzvsdPNp948MN+xWFKt7iNGY06bwykn5zejb10j1JSgD/wBoLhP/ALZOswKbpaHrGt/+Mf8ArrHmXhhwgcVNI2Jdlz9/yIPsTfFSENe0ZmYdBnswW+ZO2YbWmFVuf8kkFU1WpW+DWpFnXPJPyGZgsW2PDNHZPFCBhEiUA2fnCYAGpB8Mb4qUdo50QNXyS54bFYBcLEY8GG68AeQFew+UksA+/LqQZ/imtmOGF+wd22qYTqrLpvKZ4YR3szBOmZrQPU2DWH3XAu2BTQA8NKWeZpBZj5+J9p6I815+Z60t83mXrPo38W7/AGKZfFuBR4dPOGARnp0/AzAO/DBWEb0M+MT0Xnpdz239AGZ/5FChvVhmHHpsWs1Ll4oYMgEzAABUleqXEaNPj/ZxupTMAzk/fD2AahtwsED1xvAYU22JMkwMwzjw8wUGNb18Mx9Biw6wf1s4LscCZUuTabff5Z3vDPBR629VTppgw+Zn1nmsmrl4W9xKe6LOns/WCizLM4jG1g5aU8/oBdNgPVUI+vOfA/JgrWBUnwzcfAzAPICpvhzW27GvisTdCqxPUkfvkTuhbC27POz5h01y4I1Y1zzzjgYACuJNeo5+3tGMf/Yn/rpmm1K3DqwPsReQZ8jUZbRbG4XdetKm3VdtErlSrD0B2l4aLQBmDwAee8+xVlb4Y2idxu3UxXJ7zpyQlci8AYK2qVz2qZtcjVYD2zA+YjHsNQfWtDP/AKVt7/wTUcvDttCqn8BOHN5z5dz1W7qlDdnPG+bMcAwA1TvcB+DMB0wO7rnew8jILqFt2ZWLtz+z8u1ZOAZ4BM34fRgmq9wc40vHytOtKiBh1nzjKur62105i9wo4JM+znXa9+dkFFDhjwZz/ZLqe+t4F12Bwr4twGgblWdRHj/dPAakPWNxejezsqH+RkDUdyNuVM8MeCQB4lKrYB++k/7amhw04Hn+yW49J+uet39nuLf/AMl//wBgCobhn8VLedFh+wHjdP8ABpoGCd7zaqDh7wyD2diMn/3wFNZtXhyz4f2HgAfkzWFrF1XVTdIKrFqsYzDPB2AYLOHeD8Z2Q+xOeZkPnmZmyYLm+jlb+LdojUrhWEgIj9ow9/WAGYIJj3CsKl6utyx4b2HtnpG8M/kXn968AN0wqU4JJecFcQ7noz1I0G33tL8INix/+R5dO3Y6PXuHM+G7Lg2dSnjYPDRCMrtm8+HMYM6jwdtvDzmGa4ZSp5g7oU5/lj7PGwWgO8HKbI5SW+zUvIBhvP8AOuHzxezBt12BxjYpT4OUOwLbCF2AcbPBS+Jd1WXxOGmOTrRZh+q9fDRjdeeHkw8i5f64ptVppgxrMu6OsBgzngasI1yMHCZ02Hg+vYq9bm/vf1cfNaW9QbHjSIkulWqyzNCSAap5pSabYj0g33OHkOS6fWbzJ5maj8/nNpXhvHnJz2B5AQevoJyAYbc8U+hfa34cd2tQh2xG9hYlHAP7lHnb5+0seiPH5zhgo8aY5Jz0HAPDYad1nwW8eYeJwVWKAabFq0cA+SGCL1w3lqeo6V/5YFWcw4vuea8Fh6+fDop0AD8gRgXz7W1UPZxIAf8AdlCw+KWirE77W1z/AKt/5YEz9p7j7JQAHk0QUXRTuigd+0lx/wDxl78gAl6+uA/aViT+tNaKWHxQO+sqz/8AEZP60Jz6qZ/t0n/xjS6EWQL3AHmZ3vJ0no/GNLmZ3s+emf8AjGiSTAaPXP277x/W8aRsmftDzD51I/IaX5DXgiaLnY2CaZjVmBNj1Wj1E4FQinmEhpWGfyH+hGH51lgbC2ON/Eq28+eiU2seQ3o2BgtxD9KL/wCM8PHsPOD2a44lh8Vdw835d1D0lrEmR851lVhk/IEYDQs8fuDLx/fqNXo31wFwnD4owBML29ABx14EGPsKx+eGl/LlwI8lV/8AJrg/LNmlyYf1phr2O5vceOB7Ps4NYP6IaAPSN4QRvZ0a5P8Ayf8Atrh/Jue4bR8sfvEwbdyD0jeE5/8AR1w/nh/7aX/pIcOQ9hSq9/5b/bXD+TbNDogCdZ2O1f8ApLW4Bf8As5W8P7kFID0n7R/+Ua3/AALh+IJYZph72O4f+kzZ3vLVrYfkBL/0irV/+HXIH5AXENFv+tLRb/rTB2Ozn6SFv+7o9yH+hRXvSWYz+6WXckn/ALzguQ5txkvWUH8RMDrH/pMn/wDi9rH55II//SXP/wCQKl/5kP8AUXInp8F73i6LSuD9lVWOz/8AbmpsaQ/7rljDf9eajAsD9Iqpe4sd788xRHvSEub3FjwD/vp/+wpcz0b7npQG+FSkz44e+gsg8Czj1h06MeE646lGd8j1NWOfB1eVsfpD3aH/ALgUf/z/APsJk/SNvT3Fo00P+2NQgsC3DHx7xmH9ENPBYdpB/wC81SP/ALmCb8Nfn5jYceL85053qej7wwwAzUz/ANIfiP8A/Bqao32GtLL/ANo6l/5MFICybRMP9M1g/ojAnbxNPh5g/wAv3EY/+iqUYfOyl/Lffhlqep6aH0AkdmWVlh68rAH/AHIJfYO1fd3bVf8AwQTt4j4OYv5bOIfvGIAfJopBxpv/APsMD9CX2Gtw+u7Zh/8AdgS+wdq//N1T/wDBBO7iPg5jMnjNxRmRZFO1IbLUoMDwZ7P1qPTbwuq1WApVKiw+XDo1mQM0NyW9Q7VhxKrBuOTPdfe0OXdDBWH2Vt+qtNTpd2z4zr4Z6PJ54fxrh7nR83NjAv5Y+JoeGHqrD/6YEv5Y+IYeIxFpoO+flgS+wduB0XjJD/uy+/Yah/8AzrK/8mC7vp4XP8vMA+MHEZ7rg0T/AMgCdDjHxKMP+jQ/7mCH7DUP/wCdZX/kwRfYmj//ADqAfXDP/XTz7PD+D5uZRW9ValTY9QqtNfBmXKMwe8HMOv8A20QXtfYHqN1zD5+TBDR4EV6RUKNz2GEkw1cFas2Nn/73Rg/7ma+R6PsR+bdHt+rzfoR/5SOIwDp/aN78jIAoj17X/J9vdtS/Wrj7Af8A86wA+uMYIT4e1X2cG6qJJPya2C+v49nhcHy8qkO8L7Z6L1rAfQ9gmnrqviYH3u9a2f8A3k1e/wAlHFB7xGKHGmNeePMA81TzLbuejtalZt+pQ8OvVjHgr2Xw+eNUn6xeLUfrFVeP55Jo6bD+/nm488Zh1meaQT/dtvhmnoBthVg1PeAuD+Kf/X34bfw7/Kg+p4Iezi7w70fJth7tWHQ6efQBmgXXwX+Yhz8n70Xlg8iXLfiNqUj2GulCFyYGCDkw/rVhh8UsPijJCCMnTjNh/qKRouf1oMD86sSKJskH9Cx91RqHSrjlvt8RpNEkTQB+THw71s6V4Mgzc/4exYK/Ga5/K1EcodKptS52mgHLzg2GtWrZUSS3MpAadc9cf9bw600YKwhhowgByDGhumGbzUfoA00bK8sQcPilh8VL0UsPisBH0UeGxO4fFLD4oGtFv+tBh8VIw2JrD4oyBh8Uv7tO4fFDh8UAJAeKPD4r5ouf1r3YHWwJLP8AD6EeivmDfs14BM96WsjxBIwbBe7ag1sEtZIwSh02dWJ8elU1vORKPYvGRp6S2CAKkwfs2De+gF2OicE6HDd165KemSG+zPYugQKPb9NY0IlDgM/ODKDjNmW3bEmRHqU6pViHL7GeQ1gD+NdQmUGonh6qpVEmB55DOj/nQ1u+aBb0hrnsAaM8M/ItRUqV9obXNy2KlGPnWfBdWA4pVbGtmiar943bGjZnmceP0AqSTeHBqj4RabQ3q26HeAGf8axVVti7qbXnaVdzZxnfxZGwDUJ6mhGP2jJ/Qa87nThtXuMzkYf5jseMyHYZ/wD2CpJnFS/K3C8CsQIGf4TO9ZepUf1q1oc0bIfIhjWrUYwGFv1nR+R4M81j3PcIleuG9abFOcd6zDaDr0TWX9Q/bOAFRcrGZv8A9oPetWYVE3XYlxhAPDrBBGgWi8ZsUqc8DodccD6FHc9Yc+FdSjAb4ViMfyLW28cSBACnS3MJAdmCtQZDHTYczWarcmmxpG+C9zAf0gN4LHued3lo5ODOHhnv7zVfVcDgO6jgBszTsB6JPhtORZwGeHQZqPrUOeZxakwyZh5Fyd3k7mft6qweS+/VEGXfIaGsTGzI50W8WQw7DNWFeoNnPQz0Hw/OqRmj2/PgcjEppmYec8Fo6e9s7Pu1+pQNRyXnge9aA7zp0bZr4Lklt29cdH5jlA8J88wDPoWgCM28Oc6LhLD51yXwto9nDVz7zYBg34M4AMPOufw7qfmVsJ1SczdV1G5GY6EVveZ9ix8+26lTaibjDgK+GFdzqE+56NTaWFV5TWe+tfbe4i0O4YTunR3gMPIuaTKDccyBr64Bh2GaZt46rR2j0JZsu+do104cnc63bFefeuFqIb7LMhg88D713iZxRo9HowPzpeboe66F4yjT53P89Upet5zWlqV5+so+g+4Dx9hvBvXu/LF6YtXicxcM/lA8EPPmumw2cwXgiiX/AFy262HKYGHkMF6StLjrBOlhKrlGmRs++J0LojmeuoVu7aHb0U5cp/o7Flw9IGwNUGH2JIZ964PxIuedW5Ts6DKwA1yqTVbjCRyOuyefeCbH6BUqt0atsc3RnMwQz6a3J9u2ZguKcAbkfgRY9NqTex/3ufQvQHWmxip9jQZMjXYDBcU438Pb4OGD9OowPRN+bsT23YvTuII1tA/OSsQ36DngxUmZAfjMms7VaxVZLWg/KPev0tr32OCluv3BBhmfZnGA814/vCt2I9cMvTp1Kkwj6ABldPix58CmxTPUNtSOQggPs126zOFfDziLWQYbOTTY7/8ASI55mC3VS9Bh/rtziEb37qdD/wA+a6djyVJhthvbbTWH7wD+hemHvRC4jUp3Tf8AVs/+5eVlROFdftJo3KrwyZmNecwzBO55h5npVKrM8zcg0aZJaDvAFcfZWdJDx6U8yfzrvzxy6O6fqO2HqbHPfy7PRmqGsVKpVuQDkuLhguD6TDk7NmYD97zzVfPtsAkbAXXjBjS1H29iz71NAy1MFH3nS5idHc7G1HOlH3trpb1E/cKP6mc/sq1/mEGHN/VRmfs079mze9gGZrox0eJJa/ZQA1H+zb4eJvw84da0++EYYH7MSw62DUKZSn4xYaBmupxoDbIaZ+N9aZOjwfdsGo++DDmUam6wZ6+HyJ06V+ItwdtsZ6mgYfOnvULAedX9nCYcvZgTjY12GMwRRocXLOoyzzDsBDrS/dvmi0T63F0OM1JhwczfYz2B1mqrmXz63FbvA4DSqMMCQSoxwQLT0HjP5FYZxOX094H2AosaS+AaeAYKQG8V7sQTefM9isGYbjIgcpwAz7ASjMsAepgmpnVqLwXdNnnAlNA2eDSuHqkxJM3AfZ1TWBCS+86Clm9FCQEuWZhgGGxe4FhPZzI9RRKac6lTWn4LZ55p7mTqTRy4vSCm02YxGmAc7eCzXhd3m8b1LaA/2gwzeWEhs5taji2dVqrE8DYYYZw7DBUOi2Hhtr2DCE9JlaXKsbGu8MEGDgNKW8GB6fegNkzW68Ipsvm1qH2JnDwjUt6M+HYmuWf0jbcAwzWTHw+UF5yNICUwug29VYLLsiVOYAwP+BYyBG5YfZ7FLA3Mz01Nup1W3r5Y1eRYPCIahX/wso11Dz1DnchI8nYuYgb4e8Vsdw18w/0xJ/WuTx/RkyMmlTqVIODOYMHWDw+tNcu2rCS8487m++Zn5zQaP4a645kKp5nMtDBRzhtgengrhmNh9aL1abxanettiljQ3wd6Fo4wAccWzRBSjAAcw61LCjubHM9/kTYiHRMxPT6DVhbzLASjiBmGzrzU6MGB6AOBmfYnQBtmR4bYB9C5r5koXqpiNPMN70c/OnZMOCEU8G2WcOjMFZ+GfWgeprDx6i4/psZxmg7AcwMNM9/zqRJjNnmeGDIdnkWujQDk4QYgZgar5NNbZdNjUDMPnW302vDLvGYCDegAACZZky2T12GN/Yaszps45B6bGYJ2Yy5yug2YBh2LbuFKZvyT0zczdPoDvNR5LM6M6bcuI8z9asmbkqtBaP1HEBmaZ5nLPeaixplfuGog3VX9Z0+915XDxXhqA7mpBnkgeZ9qHvWOvyKv5xsPeLdNrUHsDRsvYGDnWq8HmzFGDziyF1zLh9GxPAaqWXjPYpwA51m4a9wpo7VZ5m5qf8h5r0baUPwgXBOHUPWrwPn2L0bR9CNTZtVM/YRjMAW6XlfiFVTqV71uXnmHM4B+RZV55OnJlz35E7QPx3jNV8nPBQpEeezBNMvb/ENO8trd+CTNNlycG4gA8Z9jJq03Bp55NZ5q1Zs+6ngdccobwAAd6MLVqptNHEY1s9hhn0IKxneemraMz/whsPrA07JtudAkafWQditT0HnWsKabJgzvDDvWeGpRgfw6Fawz8JUrMmqvOiw3StEM95un0K9ZpUp49OJUgz84AowLOm6hzWm/Oaj1KTFCpOtvnvBMs0G5oErnm6qy8cXeDJ96Nmgy9c5dZ1mRlBvNlMOhn58zNrDf9YKkmTGw8NtdACg25DYwNuTJ+t5XdKeocCiaD9DZP6NiYHHOczJWEY3DwXYIFYstmOGnZwAf74/9hWwXnRowfcbODPz4f7CaHL4caczHw0DP5wWooNElzIuegYfWriZMYrzQG2+EZ0+sDPYCuqVJo8OPoVi5gmGHZHZzwXHzQIttw5cOVykpjDDv7DWlONE9aCDbGZ9gKqO4aBD8OI3Jea/tDp4YfkUiNcNsyXcwqIPO+QFERgTThsap88wfyb0mTgzM+RY0cO/BBMue3AINcJO/86N6fTeV5th842B5+dbLRwk4RTPQ8Lo3gnQZnG0csJZnoBsjmGxI6xb4O+G489LfDZmGwEJ3JRmWtNus4OsBvZaA0DrLM6ZFN84jOZ+6ZBO8sbLXiOYKKzJYeitHrgbxhn4Txp2lPUMM4PNm86+eZ6vQgaNl+MIG/UswM8PBRfZWDM1X64+8y0AdnXmrP+aua5SJODZ0AmjmMH+1VFl4NbefYggyaPSpL+hoH4AfIkzbdpPRXW9A4csAz1jDrRyeRA3X/BZ1/Iew/wAiTMmpTA8OnBMAAwAwZeNHhUqG+E3UYYZeaMDAM1Ycno/0GGH0b07TbM4qVJrXptqyTa85ngrOm8H+KEmab9SiMwM+szmYfwYKNinMA/DjH/2Jgge0A2Nt7zW+/kTrn9KvWB9YMnmrKBwKtFl0pVYrlVnu4YdeAf4E29cq1gD2+H5w2KJPepRjoMTjzPyGu7Bwl4XM9dvzHvreVxAtLhzAaFuLY9NAw97IADNB5hnmEPlwoGcx03g1gNkz2LVhQbjuF0GLfs6YccOswZwwP869KxjCGOnSmIbLXkZAAUjmXz/a3w/WrHneNwEvyS1ruN02H8kiZga0FE9G9gJDL9x3d4XfHgs9f/bLsZ6B+/D9aDNsPZyg/WmxW2ZY1ncPYTsSzqHGhm/hzMs97z35/wA6ujefAPaKPqN/jh+tRzZcP+nPf+CosSDk+dxH/duRvzmqx6G5/wDET/QoT0aX2TgD8ilS71pYdD7P/jJHVZwf2b9az52rWZJajdSeA/I8zgmgsy6pLugFSjB8/QiML2TPlyR0ziRng8mYf51QzJLEb2doxj/Iz/qJp7hddxvg+D4T3Q91zIACCZbHEaiRZEti0oxusBsDW1gM/J1rDbbClqVsWxcmfrnh7RHs+s3YwZrjPFfhLwSehR/sxzNNqeeZhEPNkADDrXYJl7cW/V2hc3AilSY4d4f7Czp3Pbh76r6PVKB3znrB/kSPZXHE8+VXhKD3L/ZyuPMu55mEsDPM/wDIoX2eqtBrn8+Psm1C8c9I8wNdgu0G7hqnN0qjBbcfDDl4+eALOHZ8GeEv15cb0nmjzM4/gmCXy9iOaGUkydYT9XSz+9GDAYbMwWrnnRqbQdeszowSP3IKkqQRKPS/s5bFHekn+MZ5n+tR6azUbnpfq26ovIYPZxncNmYLmuHHcLi258GZmEGWbxvno4Go8+iPxnXZcTWhxwMGQM2c8D70mdCz84tNcjTAP3ykQ6Vf9wsSOaptVgUqU8D7Mt7oZBR/VGF9JPRtIKjEiZ8qBvGGt1+c9ij2BWHzr0eWxTY0wHzfPCQGYHs8iVHt6uWNhEqM6jvNG97UOswPzrW0HgbcdbrIVW1bVmRtAwMJBvAyGH0GtPE+W2Fg9dvDWZIOJeljnRDD+l0Z7DD6wXQKrwHpz1GjzrLqsl6RKDMAnSdhh+QFZ29wTtK2JB1K9LqCqzQwzj7AD5M/Ot167zMPVT/3fAABd3jmMPO8/hjf9Kd036AZtB76O9mCp+WiU0jCuMVuM72A0yAf412CiPXpDv8ArESdX4wU9jOUzE7zz/8AsFs9Z+T4c7CS15DDYtu5eHm/mbc92/Xj/OyjN6jvYcjEn595yDD/ACLtVb4UWdW47rkSnerZp/0iP/nDvXD2WcM/rMFcWi4GjZBjV+9g9h+660SMOhasB/8AFQv7DUv/ADIJYUDvp1SP/vIf6iBJe7AaP4beCHE1Iw+KLD4qGqRGOh4YVGh1KYfnZmYAvudpf/KNS/8A6qmcPilh8VTI7zlqh7O0ZgH5zqR4KLJPWfM22NFrsDPPBEkoA4fiKRD5EHdSosPPNeRk8DUdECNVnr2j/wDLlV//AKklrWb/APLlV/8AP/7ChYfFJBKN62MPulDqTLvnOZmChI8PiiBA0pcB6CzKA6jBOY0Huge0c/zoMPilh8VIuPtDZf8A+KiN/wD1J5B6+s4+vhr+isPKnw+KNULA6xZ3u+GzwH/+eHlWH1m5hgHk8ieyBAalZrD4qUyDdNwfqtACe0+GwDeMP8CEARYfvFAtqVfNh02YDlV4Ow5LXnZezP8AQa6xat4cBZ+r6jYgW9UDDDCXDwPBcS0W/wCtRzpsQ+xB0+8+GPEONH9cWPatmXPHc6OXDRP/ABrlT1yXNQWA/lG4ZSaP5JGiejmpsCTXKI7nb9cnwPJovHh+joW7h8abtCPytx0am15rvCQGGajYyVE4u8nhLt+9JNNPyBJ2fxrpsPjxfhxQ14lBqrX40hneaM6V6OF806PVbntKg0rXDPNqSbJh+gAWBvbg5wZ31XhXxUCiTQ9jHkTDeZ/jWTV0MOJfBmTm5d1gTKPh1yI+9n+BWf2Y4XViGFZtjijACIfQEjfmvNNV/lUo7Ehhy47euSO+GH3d7AwTUN6vvWr6jO3IDLp95vf7Cxw2++no07StgHdP+UaiH+Q0js+1eziHRAXBTu29DkNMOUCAyGGGYSc/8ivQvN+BHAJTcM8/IC8wfzTmdgZsa2Xv/wAIVN/8H/bWPptHt+sXpUKHEvGGEenAAaxhsM+9YeNf9VefwiUOBGd/Gde2f4FnLVC4LJrlYnXA5GmesT1tGO97E/PgmHT99vRH8ntnM/8Av5R/+xZUc7GtF72HEOlfnZz/AM657PvaDAYB9g2Z+zPBp7emXr5oZw4j+gZuvhm81rB4J+RMH3W1de4J25W9L/7b1Hh6GexmMeC0bNjW+EJqI5xNoh6AYZ4GC5fGvyh46nq4A+uSpEa/KHJa1PVofkkpg+7mdFCzLVDr4lU0P+xNA9Z9ie/4oUr8kY1y2fxFofrFphiKel74NYF8e4i2r/8ACp/5ATB93M6j9j7H9xxQpp/9zNEzatnM/wD4Sab/AOWNcq/lFtz3cGeH1gH+uvv8pFD/AAJn6A/11GD7uZoqDRIMPiJNtiVcbL0fPMJzTOz9Ga3T1n24Hs+JtKD62TXJaPW4tSrk2sseTPBEfEu2D75P/grj9SMXcOnl97ldWCz7Z/pfFSj/APgmpAWlaPu+KlN/8ma4v/KFbBn7OSf5FIDiLbIe8k/+Cu/Dl+vmdiCz6AHiMcVKayfn0TBaOBUpVKY0GOMsN5oOx4MwXAj4l2dh7CT+hQQ4l2kZ+HBkmvcOf6uZ6Lee4a1g/wDlUxZ9Vd87MM4Z/wAC5pxjoPDK3qjQqrYD8YOd8CezHeMwz2LntSv9uSEf7Phy2B5vcweww8iCpXVBuGVECJTTjaD2Z7881h73/wBej1P8zsdepXDm2KXEuODRqbcOvhm1zJ55n3mCyj1e4T1uVnUbDqVHj971Ok54fkVFPuRgM6aww9s7zDBVPrsPZmwAfQuj0/P9mGPNH63c6DwH4bX5TTfse/5gO/PgZh+Rc/vbhLfHD2abdcgvSYnZOjhmGCx7NSixnQfgy3ozod7RmBrpFsekVetBzYqUsK9C/s8g8D/Wu1g5oB5J5d1oMz0bOJH7dBO26mffmEbf37w2KiuT0e+WaOdaV+U2sNdgOyQA/wBaPHKUGD4dClz6bOokrkaxEejO+R5CAeR/+BbRYZDrVbds+36PKhXHXGKkchgDBl2IzngHzq6APeaiT33mFIiedk1cWKqg3nQ7zadcobkk9AM/GZMNisFScLgwsWIx+A8+z/GtA8GCuw1h8UsPiliaLx8/EWAaw+KWHyJ7M/Z9iDNzyfnQBg55EtEwL2ae+fTzRAZ/h4AvYZImDnkX3D5FI1nMg8PYlrB3sHh8ihqiHp5g30Glh+8NS+bi+Q/0JADZ9DeCCJouB3mg0cy/av4FOwb94lot/wBaoQcTSxNS8GzL2+/yJaIZbEETBz2aFmTUqVPiVmlP6MuEeYKbooDZXux2i3uKNsXCenOc9W1DvB7o/WtgEN+SGcXB5rzga8y6OBKRAqVcojvN0aszIZ/IaxwOgcXeFdYuGBqU0N7Z54LjNBu2/OG8wAYqs+jyA/o5nmC6dTeNN/wMAf5apAH9oDf+tXD3GCxLkwiXrY5n5zZwMATAp43pDy6lD5G8bVh1Vr5MAUSSfo7XVhgxUrVkH178wVnG4XcD7qdBy1b0mUR38E3v9dUlw+jld0DxLcrkCsNd5meBqLhUWkBwHiVXNyx+JNNnx/JI8E1lKxwi4t0oz07Y5xoOh6I8BgqWsWBflBz9a2rPAPxmgzBKj8SL0oOHq29axGw7Dk5go6WipuG1axGm/wDKelTIEg/OeGaqqVQadb0g5dOYwMwwNbipcdeJslrlKrWYFVjn1tPRutU4X/bkl0AqvC+gvZ9ek9omafMM1MmVKA6dRpr56odjp5gsjGue6pjroVVh7Awww0V2iHcnAg8/XNgVWlNeeJMMwV3AD0U6k7h6xqUbP8UzBPjebeZIcC6ocjXYghJD51bQzfmT/Htw4xn5F6QmcLuGUx3XtXiUDOfQBgB/50H8gNzmwT9Ku6HP8ng4Z/oWPQPPk+iW/AzrE58zw/o59CsLeqtpVWVyrFDOBIP+kAeALpcn0fr7A/v1DjSQ7wCT/sKqe4e3NR9gWPMw8mGsnSxwy9SrDFKaOmvytF36FmWZ9VN3wJYGGfkUi7bArlSqhy6qxPppn5GTwNDR7PpUDBs7nk6odhhgnSYbI4GjTWqlBYjesAMDA2lW1KM/MfCsOP6MjDB6PhsVrAmOAAZubFmr8ZpoR/XDc7CQH4Rrm6TKJJOo1WQ7EYcwDzqnZt643s+Ub1lWeu6l7s1oKPW6rSmPWT+B/IBrpwzY8JNRjSNOVsPPoU6pVttmEeasp8mgXbNaOq0aTD/6xHPeq37PMUd0mzfeqUfswV9Y3VscRaBWKcFNrNKZkvdkgGd4Go9VvydSjODBqOi19CwUOtwaVP8ADp0kPyKbVeUrYG+xEDVPrMFt8wszu1isFoVWonmfvgQPHbMAgnfaOYZh0YMrJep5bL4P4dC0dNpUGfH1JYJ8w3FBvCCcUPVs43mvOthA4qXHDaBhi56kyAdma4+yFHjeHErBh9YKyhvA9/Swe+hXfrN4t2tn0gbnh9dYZe/voysIfpRTglA5OpUOS13tbwXHWZlOOObD842c/Iq6ZGi7HILhvB9CjpaO4X/x1p12287TmLY0XT+fNeZa2ZnUjfWo1nzHwGDP6FBepr8k99Kk/oW3Sz2hWZdVz23VwnUaXhh2L17ZnGyuV6A1qQY3MOLykzbbgeIFNn/kBaClU2cbAMaclnD8ijmhe3rB67bteDxLfZ/Ws1XrwuNmLpy2OWz+fNcRhs3jA8SLc08PkzU4K3fBnoSqybzXkNfK5retk9dVVBrUzA/kMFVT7kfmB48GN+hZ+ZUq5GYzbBk/rVE9cNVe/a2wD6FwXYm1t5g2s22wDeqfnMC9moU+seEW/NUL1YfPo2JgbDmc2tTTNRwPMln41YlZg2auIdVpTzunzYA75DWObFsYMGHsMEHLAYaYGjB5h4PD3/nS8N49RvrUa5FK96NgSZPTAdRxS5OzPNVjzzBgefQtoSMwM9+GahPG57tWECYw94ZuAH1qPMjNs5m3h+RX+ockZpsoA8RtHJDA9PsTrMkzY1+xNHJzX6d89EPYg5Zt73aTx700clsPZmrgA9GNkvDSZN8+tHrGfWg59gPAwPNUo944IHgz60ufwHT7/Oo5vOL2ICZDB0D7wU23qa3Uq3g/+zsBmaigG9XUx5+m29912SHz6Fonwz8yS560eYisAETsBTTjNn+yuZj501UqPLo8KI/Lz1Xwz3qLGkvstBh3rJdrrmW8AbY2YbEZz4MBozf3yD8nYqTWwA3ASgM+tZum/K0QDf8AWiF2cN8IoVWU4Hib8ARsnrFhsBE88BymogZmHQAAjk00IzRv4eL5O9QvYD04Z65t5/Io5vaxG444AKKZv4m450Aqw5gPBptr3DxcBJbPocDBOg8GHhrOgeboNq7jA4EfUwTCdHXj/DQczs01EOewHW4nqaz61fNiIBmeHYowvApNKfDSlHgDSb1mPxF9qVNqMaRoPsSTa+jNMPM8s1qOMGAfOmDB0NMy1A6FIB7BRIB6yuAgMB4Zt+KmzAWXsDA23DP5D7FasxonW42efyGq8KbyxamZn9YKWAS3mtRt/D8ii14A8GiZubEcaSqeZJcZPTM1B5+Xl4bix6TDTc/reGDeZqRA13nfvbeDXeaiW2cU33n5bmAMhnmmgrcuZP09gNH0AAJ0mGiZPAXXAfMGg6MEO+S19ff3pZuBFPTf0c/kSh6gNefPzrHC8CZprYNeOahHSmNU3Obw+tTXmajq64VKMDQdeYdChGdK/p1zAbveZgriHqqmQzelaGuzh51UzID7J6bB7/kVwYW+z47FwAZ+cEMaM3MLUYPWDzrs4bwzwygetYD+uwxn8hqz9WhWPHYgmEj8JXsmg8z0RM0Emm+pNJuIGi6YZmttmGVOA+y/obFNClRY2yW5rH52lrXnqMzFaYqMTmZGeGsAHsUWSEWM+DESVoxzDrwzNNo2r5JnPw+6gGAdgKPGZ0XXT1DPM+hWp72jYiTmcO8/OiptBqTLpy32Nj/evFw3PDSHm0csOg3jwXUrzkv0fhfWJbGwzZ0QP61jeHUPRitMLZcTjz4eBRuyU8Ga6GLy/TdfkAbbcw3mnvVUV725roAcPaGDpmFwBpeQ+xOnaVuM/wDvABn8jK5HTDn/AKk2+HEM/oTTMZ+NIB+JFwMOg10j7N0YD/ZJJ/WrKBatAmOm2/EeAADMzB5etHN4cyuauvOf2go5xn3j8D2XnXVmbStw9h84Gp0AfeCkBZNODwwibPrTuHMWY05n3gPfP3pMxp0x/DTzXS3rSYZd04jAJHZ8HZzbAZ/xq+5DFBGnMtcpEpWsYe9PeAJcnWNj5wXjP/q4LahRNED+6awB0AptNhy556D5ny/Y10KNrc1BmqvSA0GHvzq1ZpVwSYoBLinpB0Aya3r0Nt5oGNPRwPPADTx0qIbWDEsIx95mr2OeBDlRj0/UxvAZ4HmnaxGfjSGojdHkgGGfnW75aDAMGG6iD2fWAKIDNvzH3X3zeAOg+YUDn4UqqvHg3TZIfWCkHbdzMnqP4PNB2AuhgdHN12LFlYZhgZu9Cr2WYrOfNRM3T9jy59abGMOzLgmNBLYpR8v5zNSGbelxiwDADNaaYzXHsG6GxPZ/c6Oa1ECBd1SaCnMWHVdX5IZgsb/uDI+qpcMziagA8HX8iihR52RuagPH5w7F1UOCfE2YX+g6Uz8kuZgau6b6LtceE365eLNNkH7qn+MAfnVDiJ0p9loznSzP5AZUfnKUzm3KgyZIHvwPzr03bfo32JRM3KlWKxUnTDDM3sFd/wAgPB0/EnWwc93zyHszXo8mM1KIEgHHKMGYdjp5qzeBy4QdcYpX3s+g4kY164pVh8PLew9TWdSo2HRhGV0Zth+wsAz9AYIPLVE4b3HXo7TYcPZLLvfIMwAFe03gJeL2DkudSqUf/WD1jD9C9Acs/wBeZn9Zpp6A+97tY9w4+96P3M+HOvxl5rv0YZgf+NaWHwisOlNabdDZn/8A1bxmC3Hq0199WvptbPQ7VtiAepEsuiMvecGVbAEo/DY2fIAApb0N9nw9M8/Io2EoPaMGsdiPyDjx+O48f5096tcD2bbydDXw9gead5yqs+zYNY7EdyMYe4e/QjOM+H9Fk/oR+takbRm4B5h2IgqVRMP2E/1q4EfD6/0IzCIft306ck2Q1HwPBV7120dmK7LnVmNGjsdeZgr2JGFH/wDiTIfWaNmBTnv6dG/8ZZqZxdsCG0b4OPTzDs5NZ+Tx7o8xo2KNw85k+94zwBlXsdN9TyjE/VQczh14HmnY1sXHJj56Bh9a85VWt1i5KlmxOCmx++PTzwBBzN1Q/wDQ13VKGfn1jNY7Hoj7N1ID8eX+hTgtiUY+3mPLzTzPFB5o5X20mScO/mcFcUe6uOdKp2vSq/Jej/vTzTY7x6tfge0gmf1gl61lhsbgh/4K4vG428dKa7qfzbJPySAT1N9Iri/Pf0JdnUp5r5wME2OynW5R+3fUQ6k3+OC5ufG+uMtNVGs2PR3omeDwR/bAtHA4r2VWHdP7JVhn68FcWpoPXDee/A/rUsK9Ew0+UP8AICz8m9rEZ3vwarGDz6IGo/8AKLw1Dw3LgqTJ+Q4BrRLVM1ilB7Bx5nP608dSYP8ApZrMhfPD3LTcviGefZganBcNovBmxd1Kw7PGXgsuZlPf0uHJ/dSAULk+c9pbFKkn+QE6zJt94dRuuUo/omAngBh49Ng2Xv7negrPVsGMfj2HSvyMgo8y0rHku80/w9o7zp9fgrQBDfPZoPB9YYKFVarblBDCq1xll78LrNY3DXCnk2rw5mRXaadgQAB/r+f+BR5PDHhkzAa56mvA7FDAAOeYbFcQzrlwmbFAoeEf+0VFnRBXUbh1Q+a5q4H3pkv8FnYyH+urwjDNWTwxsqNPCuOW4B6G+MEt7W/Pgtdds+qw6bp0rY6+YAbwdbIecFZPSYMCOb79RZDDsz3mqV6t1WY+b7DAG15zTBhlrettgGjnTtYHT9tIln1q7OpQQ8Omv6zvnDoSk0o6q7qTs5If2cz2KJVbns62GuVqs6NGkB/R2d5phFwepVNYgap6jz0iUebzxnvM1IqVSo1BaCXWaqzDa7APrP8AIuX1LjNXJg4W/SmaaHY66GbywklmXPf5uc+cl3zunma0W6BdXFQ6xCdpVuMSY0c+uQ9hmYfJ5FhD60gB/DBIAc94qhzWMOjBtLE0YIlszDh8UWHxSSVBIwQI9iHgG9H0HggSRuSQL6fQme9GB3oQ62CSWHxRqdBHh+ImQPDw80aB1IEkKkOo0yjVBIDRpB+8QJJJApeQe70s96Z3o0ejRAmkaCRsRZ5qPvySUBaLBl0B+hHyzAe4BID3aYJZ71GFlg2Hu00YMGfhtp1RHgcyAG3wDXPDepCeZDPU7FFMAWin2BccCE7LlToB6AZ4c4GeCG0rDue+YZzrciMzGg6/GwMPrUKwy/elh8V1P/0deIx//CgP/wCp/wBhZ68+D/E2yaMdfnUqNUo4HgYQZOZh+TBFtRw6rHC6BRo9NvHhXTaqAe2nBnrH9YZ712627G9Gy6tlHtmg8x/Z3fBP+M14ptvi6xDlNQbjpT0P5w7F06j16lXDDCq0OcDzXY8HWCDvFz8CqbAdNyh8LrVkh2MnmBmuX1ILcoMjlK/whpVNkeQwNXVt8cuIdsRwic0FVjh2S+v9a6xbfGywL8jhTa4xGpss/wCj1MMw/XguR08PM4VzlnH/APgyt4/+xUflrYP2HCiiH9GxdtuTgJbNVfOo0OpPUeQHQyz+zGuX16z71s8zcrFKOTHD+lxN4Gsn1eH5uRmgZobPifySUT88k0YPUoPEDhRb30d6J6fzJeA+Bu+QzwQn5HPa96dzv8ejwqd7XjXMFwU20o1NDDA4gBsPzq6ZqtKP9r4V0H68NiRxnA9opDMbb7QFw/4/Lp+bhSwqVHZwfY4ZW3h5zDNJ6tse44bWqH0MqOAecMw8me9Byxn7NwPzro7rc3x8KQdbY9p/JzbAH59HNI6rEkhv4c2w98/LAmjh7t7hh8hhmH60jjbMNP8AOG9O6z4+FIZmUYA2WBbB/XGwVFdUYLhajsUq36VRJDB55xAwM1ZAGBI0/O4xZ8fCCHVacFOjwbgsujyXWAwN3A96l8/Y/wD+L2mmf6wUfRYPZqBn86XLM+RY/m/2Qv4+H/VNZes4MH/5OaJ/GCnBVbVD/wDBtRD+sFSAy2CMDAOtvNdPdaPg9f8A0WT0yynj8ThrRw+gzBHGn2XGPUDhzD/8yapz0DPPTMEs2/OndbH4PX/0biBWOC0/Ni4+HLzOfvuZz/gUg+C3CG52tSw78mUp3+zyN4foXP8ANs0gNv3ad1sf5VwruvejZxNpo69GYh1WJ+M1JAD/AEGsDMti56C/oVyhz4fnN5k8P1rZ0q4bxoL+vQ649G+QzzBbN7jNX59vVClXNToc9p+MYHhsM19L1vZfHv8AhXNDzzatNYokJ2CEvWA3jeDZ0Zq2k6Zks5w99RzLypUG6tb1VKe0HjZPAwXouq+jlZ1Ydzse/OWdD3Mg9YP8a7Obm2474etxTvQH1rfVX0e+KNNzciMRqqAf2d4AP9Cw9SpVYoMjlLgpUmBI8kgMFEWxwjmgyNAEmIZYaiPMPduK9mCAzR6x4JGgRmdz+CWs5iCa7EkapGt7vPYgz+Caz+CPYjISWfwSz+CFGotiSaz+CPP4K8MjoaaA+tCBojNeBeH5N6X/ABh+GhyBFm3ipalgx7xvYhONBP8AogIuxLPAUgRzpVKP+ggpDJzoZakGpTGTDyPIT/dpAf4ioXUO9rqh/wDSWf1hmo9bn0q5/wDT9uQ5PzhsNVuQJZAgz9V4XWPPI3IMutwD84PZrnl4cE7qmO6lDuCNMw7DDRXZc/glkCDyrM4acZaVIwiQakyf4seSBgiee4029DD1rSpMxrzmzrGvVGfwSz+CMdvIQcYKwy6bFStyAfzgeC0EPi1Sg383Xof/ANI8YAvQtYs+zq8JuVW3IDx+flgWPmcAeGUnDCnTI3/05obZqlekJWIzoOU6/JOr2NTmdhroFH9J+/wj6DjlEmfkwWKe9GOzpL+pFu2qww8hxgVe96KjgbIvEmGbvYEiMYf50ebdljekzmf842VJAPPHeAwU1nidwPu2VylzW48Gv3vRsA/gXEqP6OXFSiTQl02s0GpRD62mZmBn/AtBJsC+KUQ+tbRns+fRDWBe9L3b0XTfR74O3JDCq0OlA9H7DiT1ze5/RsYpU3TiznozR/24Nn8CyNt1W57Pm83bnrKm+dk4x4L0hZPHi1bqAKVXOTgVA9nKSPfGubmhrFuHn6IVRkuhKfrkPlz960B4I5PocxJIA3OvjAP/AKNdlvO55dpPmdHtyqmB9bLWDzKeonFezq9S9e56NUqa73nIZPAFkbckpXor0qA1yjnEl56P3s8mrNn0XeHv9LuCpH9GALssNmgVXNyh1GM8AfPh/jUv7POd7GaG3GYfowcEobvNy4M+Y7++kqWfArgmyfgWr/8AfjXUDh05ksDbTXLU7LY2resIHCXhl0N2PG/Wok/gbY8l3XptDZgefAM810vwwH2aD1lKDoT8vNuVPcCqcz7CCB/9iCjhwfp0b29AjfoXW/WUrBMgb73t1rt64/J4dQc/2EE09w0ie7qMZn8i7GdNiH2IvVsEPcJtbjX2JbjH/pFk/oZBSGbJck9fjfkXY8KafXFh/oToeow/BA/kTY4/9hgAPEivf9io52rTe9ufs+RdoOfSoweJUWQD61Tz7ntJnrqUYz+tNjjh02hh7eLPUKSzb+fhtyQ+tdVmVi33mjbYiRjz+RY25KPBknrxKaAH5wXBzcPYOb144saKYMGBgueVuY28PhroFeo772egwYGuRV56pUqaTE5g2fIfYa5PHp2z2gz5OzUB9V4T38TNjf8AIqKfJf1/AwDz4Kq9cTmZAabmC+pHp2dzdRrhgmem+wbJqw5yDpa7eDwLBBVXz9u4pEapGBamovPit66HGmMGGpFc3qwZqTge0cWPo8xiZm25sMAzzBWGt+/3+Q1yX6dtmgk1Vs+s1TvVIAfPBxQQjSzLxJYKPUoZsta+uC5/jthtcMvMSWgfYcDNRzmSgPCUYYLOwzfN0NNz+NSJMl8w03zA10fNZtWRmXDaDU6FEmRohlgw4YfnWm0aUzH8eogAKJzNlm1pg3JM/Ovrs2amBBjB4ksANAFNpUloH+aMz+RaLn7VA/AtkJJ/ivGnmalTsPutNhxg+haQ1ZrOmxv2pt7S+RSJk+hzIfK0qjvG75zDepsmS/Pziaei158FEZPkCwiuZn51oyU4W9cEwvApUlW1N4e3jMdBtiIyz/empoVWos/0p79aR1ic8O99HuFrSuGNSZmg5Va5TWQ+tWb1sMQJrM5uuQHmgzzzBYwJPvHIus7581Cn1J+Tsf8AZeRYM1xWKPbE+acuo3wG/wB010Ao8apcMqC7gDz1VMPOCxk8GDkZsBgZqWFtzghesalE5Zr61eHrR3DclKrcXQg00IYfIGGapIDPLSAfc6FX9fhtqbDDmSCm+deIailMsMkFSB/o6ANCeh63aff1jdf68wRMw50aaEFjDAA86nQIEuvVmOww2AaHWsrXC6ZsylVKuNRJThhHPsBVVYti2KbNNhi2HpjX9oZe2LV1ujzgznMYZh0YGqmNMlZg3KnaJ/QodOFb9m7Y0hfg02TGdP8Ac5qRGokFl3xM3vrDBa2TPfjRTfffjPAHWqpk2KrH1wiYH3mttsahUvQGw9hBjZ+fBRwPlhN9jwZAdZgC1dHBsH9Nyh1KZh1gAYYJTAinKwiUeSyH75Y7beIV9NrE7S13D8U+tWASW6wYBUYkYwDsMNhqazbzGH3pt78ihSYdNhyNNiDP/Qm22FVPBufICD6jgQMD6wBOswGA8BswN0OvBaOAGcfTfinh2ZojeiQPA9TQAP5DWO0YUsC3oMyUbD/gs95qJW7eYjNfzVVY3L9iuHsPWwZ/s5+2VgEO2DPw6i8AeQI2f+dXpFwwkawGKk6DculHJM+t3PBGdgUAHTYYiyQAPwluzn2qz4DFRkmHkeZwRB6q9ow+8H0JteHPQskI0eQw3rQ2nwwN2QGwFZQOG7EBjmvWsaSYdfYZ/kWre8YwY62u9Hy0SG0bmYB8me9NmGfk0Ghs0mPOrFOqRnns5cOtVnPwXpuuxSp7IdjJsrocPQ0teX+RfOZYeLwHIwfwGhhjeWqszxGLcqWl59FOwKPSpLWpKpwPZ+dbh78Nxyef0Gq86DLORqQZzLIfgu9aGGdZs+jRpAOciD3/AFc+g1cHGi9lqwGQ+RWXqqo5ZvuQz+hSzo7gQgl82yefYCu7LhSRqPQ5joMHQAN0+xpHMsO3AA3H6Pgfk71ZsxqjAkBLYM2TDvA0VSebmShl6ni9b2feax2jDPnb1uQ2hfYo5yZAfjGaX2eoc/xwobOr2NGr6TDM3QfcNkGgDsSeo7cyG0/zxgb578FtteFd6nczNg7fhgHnAAR8nTY0cAiUdnNs+vBaOTAYZaabgxMHegzdk5mf1qvekvw9XQc7N+G9TsNW2znKz85rS3mEWTDj00zDMOwOtVlpB4oNqPecwwrMjQbPZgGYLrtwx/lVsaiUYHQ5qnfRn1qPPpVsAR8iEk3W+sEzPnvgANvsPGb/AEd6sGTpWloe2m4ZnsXI7DRvZtAwxEZDZvPDejmUGlGJvt1HM2+hnoSeOmw5XIyjN5p8AMJDWzBAHqY5QAfOco+fjGAb8Fn3NVmcNgH6ZOCWZtctvaDZmin+M7qNmbIdgApc+lS6rIiMWPQ6xMaAMDzjdBq4Z4LcSnpTOvB5ZrvOQ8CvaLhj2Qfz3qQ8DYB4Ab10CHwKrmqZ1K+IFN84BG5lXEPghaurr1K+J8w/+rwwAE2jDjmcpn5FLjTGwPN+Xn8gda9AM2Bw5Z6LVB7++3qzjUqhwGNCm2/SowB2aIK9tHm/kJ0+U0DFGqRm+eAByxrRQ7DvifKOJTrYM2vxpGxd+5yX+JG/gS5lwx8R8P1qNjikPgVdU+U0dYl0qlNB3gesZq7D0dbS1wclXO9gHZHZ6/4100Dwd1Mw/WjN5sy6w/Wr2MSHAfhsz7ue99byuKbw34bUrDCyqa9h3yAzNX3NB/ZzToTGw64pn+hAcB6mwBBinRIEMA6NFnBTfWUv+3GoR1KjGHsDBRzk0Z4NPxv0LIWHiPHm/OM/zosP+tfrNVWjbgd+H1oMLf8A/iOiHfh3oLjP4Jo3nFCwtVn9lrB5/Oakc/Rg/wCko361WwRyf+ZxNc+wHWaA6lByPw2TDz6wKL6yYM/2Uz+jeli1CtwT8PU3p0Kkx5M/zrPzJ7DLWp6tN4A7AZVLJuqhhgFSYkwM+jVDrUjcBJz9mg5l/sbzWHO9rHw/9pg+hreaqpPF2gU0gbgxZ8wz7HgwQdC9avgfh9aeCq197ofXJKlxprgQHX4NnQ2fI6ckDP8ARgsf/Kvf9ei5zpZ0RkO8DAAWI9BzJ9Zhta9RqsaM1/1gwVTJ4hW5D/0lc0CZ8kQMzXnqSESpNc1z0mYYe+OSZ5qRMn1WfCCC4/AyD5OhZDrFY400aBm5Q7fnzMPxvBA1RHxjuqq4OMBR4wH0RO9ZSMGcOO2ex0AwM2dmaTLLHJmw4AauebLxn4wGtIEiq1usT9Vus1mSAP7NJp7Z+dUnLMB+3MGYH5zwNWDzLD2DBtmGZ4ZmCOHUuWadfnNhMD2AH5wWgiPHEMQfc1vA3+MCMHjOFvihgHf3pPSdYADA8De2B1gky9TWYs1uW/rOgGGwNgLy2ownv4abDYAAIgqT4Ab7+D2zAA7ARsw2DhuuPmYH5AQRgpwZ67+H5NijCih1V+HsfiPb+/sVgdeuOm+G3F9Wxz+TrBTmalaJg7BfixpLuzDM1Ko8yo1Imosu34z0Js89bMAD86YThnzqX9qceA3OgzDBWbM+sPQwN+3Hp8fAzjPNecPOtHXq3UabN07q5MIT5+Cy0eseCrGeIXqSG0dDgmDUX2OqeG/zmC9RpTzHvF15VnSQz/tGYYfoUgHrSe9oFSjB2HHA9/8AArsOP1Z9o5Q6a86HQYPZmtnR+JHF654+FN4ZRgaP30gMA/wIdzkMnkY0g226i8YfOahSalBZ/pTP5zXoL7DXBUvv16xLSB3s0oesYK+Zolh02OAfZWjmYdDr0MDNWjbzzR7YqVw5eqrVnzzDvANhrZ030dX5+rLu5xmleRpp4Hj/ADrrEy4dmhTcGWuwGgwBU5zKw8enrnpfWj1Ct7hFwnsxoZbcE58hjvkPGYH+RaILnYD/AEVTTAD7wZwVeFK1g8fP9anB6tprQMOPmHyBvM0DPM1Ez8TYh5NhmUFcfpUOZIY7zBAdSlzPApsHRz6JEsMP4EH2Mpsl85dRfmSXT7AkmAfoXlq2ek3yxMadcClSdnWqx6pV+YQcpnDa8hhvV6dEpVEa5qc5GpsQO+Q9gCwte42WrR3QYtmD67kb/FD9mD8/epGijU1wzz5FnVPrePeap7h4l2zaTpwX5XP1AP6PHD/P2LjtyXnfF2zwcrE4AhB0QY4aLIAq8IcQPdr1jtqK9xUu25GDitmzSo59kQN/61mQZ/P859aMAYD3akMhmurwvZaJ+Q0Wi5/WpPYlh8USj4Oe8RbE8aZVhd6SRpGjPRJdaXWl1ovBAl1pGkjAkOfwRZ/BDh8UCS7EuxL/AAI1IEBn4pto0HQeaoGzpmSLPBR8G+/P5ME8Bhh3n9aB3P4J3P4KPh8UYKQ6CXmQ5/BLP4KgSWH7xCkgLsSSSUsi1sBSz25pYefA0kaln8E8B7EGAZ9CWDfu0BZ/WvufwSAGwS2IA70ZmCdjBRuVM5VReZkdjIM9f51HDMwzNFjzcBMvZn4bjAH+RPIlArTjYezifwJ2AdYjEc6lSnobrHvo72CkIDBvPUbzWQ6nY3pDzobTVKvhjmfPUwwB7867rSqxTq3ACo0qpMzI597T2a8ZPMqRTarWKC7zdDqsmBI87JoqHpPidwf4c8Wo+dcY5aphnozomwwP5/OvLNw+jrxm4UOuzrYlc5CwM9WOGzYHvg7F3az+PbE90It3RGY0jsqLQbM/n8i6qzcmbAPhOAwPoeDeChbw5b3Hhh77pcdNeZkecF06BMpVej82xvBdI4hcCuGt81KPcf2fh+s2z8YGj0Qkrg/E7hXdXD131latiVKNC7JcF7WD9CzuEQ6rbHEK8bJkadKqpvQv7PL8YP413C0vSEs6vHylcinR5vf3xl4MoPHv3FzU3/vDX+ouoUe5LfrAByNVjHn0b1GF7eu7n4RWPdQhLgxWYzv9rg9BrkVycH7qt5o51NwqsIPfMvb/ANCxlvXtc9nygcodVeBrvjnvBdrs/jrblYBmLdTB02afvgDwTWOHfw+5zOKBMYDw5WbLvke2GpYeMGbZn+Q16Lr1n2jf8LXltxpgP+xltHga5pWOAlwUpoPsrVQqXnjnsMFzYfY9b3P/ACwjLLhtGYPsn8nemsHOgHz1fI8k88+y6EWuUo4chjZmAYGCLPWa19fnAzw39al3xzeLIznM7+g/1ghCS+e/YfzgiB4GXQ5R82XfI70JHJcA9PYyf7roUKGy829m224yZfOGBprB8Pbmyz5M94JZtyWvvbjMkD6POCdjHgenzwM5/jdCqwGD5+JgBgHe0eYJaxh7NsD+g1Oegbc+j99H6FEeZlaWgD7Mn6+sFL0ge3+IwH0O7E8b2Dofdf8AOogG+Hh6gMn+9DYgPXAu8D87XQreJps556bGf0Go4bwMAcjAf708DQnnJAHNQHj+fYaIPB3yzAP74MwNEo+8O/8AJgpeD7wg4wx+g1EM9+zZ/ddCMNR49T7tn5D2GCtCb45jqN4fr3ow08T1A6wwwMFCzfB3x3MGvozUhk8DzYPMPIyexecY4pSj0aoHkCZ/nXcJLOB80w5g6fvmjwXGp7LkOvTW3Ox7P+NddjScIsfo9iC7+X/h8r1+GPN3Fr2j8Qr4omDcSuPGAdDUjeC2EDjr0MXdbMaS1+KAB/nXMs96P+8XN+W3xQ6K9b3o9X5hr/zDUD6NLwVT1X0XXJPj2dfnMteScGZn+cFksMx2JQ5lVo8jm6VVXozv/V3k8ey4/P8ADYV9w8IuJNseI/bD0wPPEMHv4OtYzmdE8Jzb0Y/wnQwNdzo/GO9Kbg3OlszwDseDA/1q9e4u2PcLWhdtl5+cwZB4P1rv8ey4P5bbzvn8EfWu9Hwo4H3h4lv1X1bIPsZeww/Is1VfRjuqM1r25c8apfI6GH+uto5nNzcGHJ8PijV1W7A4hWx49ctKYzH/ABQwMP4Fn+a+CvbmwSPP4IANLBs1b0eeeDfegS7dRCjIkSFJUCM96XR7xJCa921Fn8EgNCkoDuQJZ/BR8PilvVMkjIEf/EH4ig70sjUiw2IVB1nP6k7rOf1I9hJSUfWc/OiCSjQ8Ye800kuc2abnQgzb/IqZEaWGaPZig1kQDRc+g/kUtmq1mN4cSsT2Q8gPKPn8Es/gpabWoXhczI6Zzgkh+9ZA009cLEw86raVNmefZgq/I0exDbYUHiRTaO1oN0OpQ2vIEwzD/AtAzxXth7Yb5h/9QyuX4glogC8Nusczw8rHj50Q3vID2iaso1Bp2OdGnSWfkamZ4LhXLRPwAR8sAeIw4bJ/IanD3bvoRqkz4frwz/vmUH8/+7OjyfrBcSZqtfjewuOeH/bKaF83xG6Krrf3oZphs7BzNcD9qtWMf/08/NPetYge3ocxn8ma5ZD4qXMHgTqVGksn2Z4KzDioAe3gzI30GouEN09WKGftJWj9YGkzJpUk/ulRhn/2yw9Sv9ufHBuDdpw//qI2aw9z0q8bn8SDeNqvGffgbJpgd7OA4HtH2f1qiqtz2zTXTYqVcjMn5M155e4V8af6JKjT4/kj1IN6zsmBdtBkGxXLfn7O/DNWt3CZdvD3I/58mfkBMhP4ZVJoD+3DzOp2GBrhQXDBA/HbeZ+sFaw6lSp4ftbJ/nRDuDNh2dP9hd0Z7/tsEMnh1BgO+BSjqTXYbMkDP9C4vycE+jf9CdZOdG/ZJ09n6DQdTk81Ad02LErGzyAs5P4kP02QbD9j1gPnMFSQ7zvSm/stzTPznmrBnjHxGZ66rGkh++jZqFqKq8e/VruH2ZM/yLm988VIN1NG2/bARvnNdz/laYmD/wAo7At6pf8AdsFUyalwPqrRhWODPi+ePJTCHjqY83JdNxtQvVsvPU09i9a/Yn0WqwbuvSq3RHfrzQPcDfR+qTGFN4jVKH/fLs2zw8pBDcPsVrDo7/vG16D/APRjtl4/+TnF6myT8j2AJSfRg4jQ2jfpVVo9Va/dPAri3jiUOiS4z4P6fhB1qb/dreyeDPEaHI0H7Vew84Hmq+TZ8qjyCbrFGkxg8+CxtqyXMywNVlVmOSfbmtNWDo7MIuVbPV+dYypSYhu+HKBREMgRp/LHm43mHejk1uI8ODcF7Dz5qEYZ+8D9aAIbfvHwBbYgVjJtm6GmrAGX3vZtq7eskIEo2zqutLDyAqw41ShzdA9id0CFy07VwPYnQeOMYeBmpExl+MYYax+fMFff6KgBL5EzIw8ii+aBntapPPhnBMA+fYnpMl/mM+U/QpHrWdvN+lGAefBRI1YcedwOK9+Rk1jFtQ6054v2TBRzjb9Q1oobMswNw6U8eYd4Kv8AUlZeP/R0kPrBXtgr4x7/AAE89QZ0xoG23AZz860tKtWVDhG/OY61I5Y/cRXnvoZNNjIw7SgxnddyVzJh3hsWu9VS6rb7rEGjc/h17PYgpAQ5YU2Q4xb883XwwwBk1d2NJuaj0PkZVDZ1TMz9isdupy+j0FyZNBuJBeP69ilM2ZUodwhm31rqcC1aqdR1323o2fkWgC0qdqg/OA3nQ7zNRsckrdq1mjkFSnMYNJqj0qcdUjhh7c8F02sW9RpgmEt+Y8B9gKLQbVbjCHKNnsV7Z+IOyaDWGTOLB5mSfk6wWXvaA5SqIc42NGWBgBrZgdckv6DFcCN9YIJ9quVhrCv/AH/5zNRt2eIVlnxoNYsWI5OiG9nnngqyq1sKP/N0GCbLQedXZ0SKz4Ddfepv/V4hqvmWlBnhhr1KT85nmai2NwsKPckusFyrFRZAMN+atmZkXS0H5wPYeRUP2DocNjUY5zV8hngpECguU3SnS4mDR/Pmrh6R1tiM+DERt7VM+t0EJnKezN+qg8Zn2LSzGWHmGtSDAwMNhyDUiGEGNACIx6tB3v0u9WMvGjMGWEtzWa8id9W0p6omEGmn+TetmzY1vvZ69Z8UGdYwj9ayUaNWTaBuDFk6P4zp7FItYzJw2ACXBzDvwZzNSAkwWfEiwcD+dnBUjJ3HJfBhis4efPYAKPDuTk6sEGVOZedDrdNlaiWbMQAdA4vf1mCiGDH4a0FY5GfDdY1wzPowVFGh8mAa+ZtB3n3qBZszKdMM+Rg8sYB1moMmGEkTfnMAZ+cNhqweoJstHOfimYGGzRNIzYgAGuw9h51KlZDpvhaegEN1g+/vV8YNmcc3Ahhv7EYQ7KN3UuCqmzszBkA70yEN94uai/s6JWdKrDlNnuuORGXo+GGsarJJ2q8+bhuSc/I10KOFSiA6bbh+E31qv5mLJm/dAw1O9BOOTFAAbiQZOw+t40mZj8l3lGA5MPxjNJn1Pyum/UZJmqx6qxYZhhrSfoRTSvW9t1DuNkM/OooW2w80YTqqB+Q4+9KlB64DUYtiqm75AjGa0QWBflYGOFv0B6Bn7Y5zPLAH50SpWYFKprWEqsVXl+/AEnjocjYxFmHE7MNi3VH9Huvn9+ua8YYYdjJmeC0EbhFYgNAxUq/MmH3+Nhmg4fJhsBN0GHJLMT8U3kqlDpUBoDYuA5LpnhgC9HBY3Dz3duU17++DMzWR9IH1VDgW/RqdBjRg1tZ4GmcPItfCGCtsMJANg3vPYAec1eyeC3E3nXZehDDWPPA3t4KLasDn6vT2A2ZyQXfXqVOMf9ORl0ew4o/e4p/IhXA/0zc0aMfka3qwo/Ba2I2b86uT5MjsejgAYLqB0ecf9OjPfnRhSpbPiYAud2spRLGsq3vEptH5l0O+dvV9nh7CnQGfojKw9Wyj92l6qlf2dZCJzMswwN/APIGxR9H8Q3jUs4ErDxATOjOwMG2HvyKhH5bf4jH61IZZDyJrmdEvHbez+hNHJ3ezeD8ikWB9OGokAMKp9asAfRmf0JetQMu8A8gILgNM/eIw8E/aLPhVcC8QN/kSOeZ722zD5DQXBvJrmcFRc++oj1bNk8DA1S2l5/6/zoDmPmO9Zf1xOM9PQP8AIkAV+Y7psMHh85qUNAc8w624361FequB4eD+RFDsy6nvH1Aw7800dEtylFzd3XNDjZ9msCJA9PlvdG9RwN97+gmf0Jp7ipwgokwG6GZ1Wb2MtM4H/GolY463BDaBilWyzSs+g529GrRxrPrkx0A9Wshn3m8pr3D2mw/9P3PGhh3hnguO1K/OOdySjg0O5qbGz6wggAGyHnWcrdVqsms+pq4/RJLrAffHXntbmTQdtevbgZbZhBcqPrKR5Ixm9mlJ4625Gd5W3LOPDsCR4J/4FydkKdGznRbZZN1jf93DDZ51HZqUufPZfcfAJYdeB54INxXuK/EqvAYRIjLMf8FkMFh5LNYuGf8A8p33sA3s578DU42X4c1qcxKPVzww6zUeS9VY1SOW/Lk03DoekQ8AzQ8K+m0EGZhv+spjIdjQbP8A7NWByXHmo7D7cZ6WGDHsetfGawxJkA43K5nDrDDvX2sSalqyGGOTPMMMwDeCxbnZ8OJGgHK8ikcgxUoYGAaxmGZmqwzbjVnUfbOTT4oZgB95mpcOsVgzdYgxDjNSgwM+zBBHCB6taN/Qw0DMNLvP8iaCqscxpt0Z4M/IyrV6B7xx8Ad7zNaWm0Sqzwaqrj7LMdgMOYM94Im4YpmZ96N/lJOwM8MFIk1uD+JUlcBSn5/MTvVz0l3PAA7HvrVm9GOG7HluQWaaYBgBywMwNEYZeBAqNeqkKm2+3Jk81h4zuwAVrVbVgw647bkSuVKZUGAzeaCGABs6961dNptwVWbrsXHTdUACUAROs/IoMzh1WHnTl5zI00HtcDPrZQZKsUqdSp/qqJbk+TN0df8AaQA8PoSjWlesloHztWYyHkdMA/zrQM01xl+RLqssOY3saxyd7IfIrC1QbN9qKGtPz3nLMzAMPzotgjN+N7dgw7N6lWrdVqvT5tGrLEk5odER5nPPD5+xdTue7bKprpvzqV6y5L6Ovv61WW3xL9cf+wHBZ7m++Xh0fnVrWHDqj0p7Nw7OjQ3QDMHnQ1s/zrTXJTaPDhyIsuIz8hgGAACKNbHFie1/Pl1M23HDraghm9+QwVtR+HXDmlYPy+cqsv8AtFQeM8/ydC5xx8LYr9wvtMUChnWzAMOYdk4AB9mZ/rWypXAesz3eevu5gOR0cvBDs8hmujSbtpsZ04kE2TAOjlwVZ67rMkvD2LVjdjt6zOHljRw9VWzDZkB/SJDOZ/xp2ffnimxF+84eRQqwb+kL8t8zDPA81EejU2Gwc6U4DMcOt508ABEgen1me6bmDIZ+RCcCU8f3tVMniRZ1KqMSlMMSarrhnrU8NYAVmzcNx1XZR7cON5HZfX+hWJoQGIzWu+YAHnM8FCO6qOzK5SC29Md7NENn600dvSqk6D9cfenmHY6ewFas0d9kNNiKDIKBTmdz1VrTlzuQjn2R+v8AWpDzNOtiAE5iDJmOgeHXme9V9ev+zrVEAfqPOSP7PEDM1yy6uJdwXa0bFN5miR/OyeDyqB064eNPDyg3a1ar9cZORhm8Ye5PyKhuHjw+DpsWjRntnROkbP4FxqNRIMbN9iJ94PecgzzMz+dTQ11so9VZlcuR/nrjrEme7nngfR+hFGZYANPAAEEgTwMpEJLYlhmpGjh4ZtosG8fEVs9o4aCl9waYYAkAAlsRJbEaZSVrI0kkkQSSSXegQJdiSSoJLsSSUvNhRIUgBHpJJJIAQI0GGaoNG8Ae0ToGo5hvTzJtgfnQSgP8RL/g6vD6EPf8iQHgKkOowMPzpnP4JZ/BFnj2B4iLu8NNZ79PsR54IgXel3prNzJGe8kBn0L4yhA/dpZ70DvdppIEug96BdB5o01nvP60WQID60H7tLNv3aWfwR5A0aZR7EaEeoj7MA60AdCMFADeeYKPyx5qQfVqIzRBmfSuQ0vvTL2uGfhH0KztK8KzZjupRn84/vokjeyYfQqwwQ6KLd6snijat2m1Fbc9T1U+uDLPAM/kNdLZDDw3OvvXjc4bB+IYLcWfxsue2Ggg1xj19TO9l32wfQaxuHkOl8SPRs4ZcRQdneqgo9Qf65ERkMM/oXk+8OCHE3gnVwrEujnVaUxn97pwZh+js617dtK+bcvync9atVN50OuDI2PAr1mThm24wDwH1gYbFG2jwFbHG+LJPla5FeCQHeALqdHuGjVumg/BnMvA55F1K/8A0XeGV+Ouy4NGZpVTf36zR4Ab/wDkXl+6uFHFTgtP1HKVzgGeAS4J6wGHnSx26m35X+HrXrWlVl5mIHWye8F2rh76Q9o3Ow19oGOQkH/SOw14qjcRdaiS6VetO5CQeHin0B9a1tgXDQJ9uR9Cqg9IAzDo61zXCtvdcyHZV80vQfiQKlH/ABgDeH51ze5OALZunOtGdhn/AEd7/XXEaPXqxRJoTrfqRxnfJ2H9a6DRPSlrFHf5G9bSCTH75cTZh+tRhv49zyzkyBX7Yd0LjpWiB/jKEzJgvO6gPsh9a9IUq9rHvmjRJ0GdDejzc8GZYb1lLk4IUapcxLtipeqnexrrZ/jUXwvr8H8S3+9yQGaVjqOMAEjzgmtFzPZovK9qXDe/6UWhOpRz/njnmCpZMA6b4cuJPgSPoWNw+lw88chn9m8Nidyx+TsRmzrHryooPfvmTTRvSut8Aea8/egB6LpE4xVeWPP2JmpXtNjPTgDUiTmcP7PINPAB+0luPRvIbXQqwJNK/wCkohvD5wNNHPYNrCjVxkwY62njQXucvobbZnh5wPeoRyZwGeh92+SRvwUH13bjIn6xqPLO+cHlFmXtR/Zt1gKkH0b0WtWQ6n5bH1ux0GyTsbYCYfkNUocRbYDZ98h/3zOaX2/t/P8AtP8AdBgrGiwON4em9Ga8hnsQhAce3gxGeDzgs/8Ayi24GzkakH5M0QXhBP2FuVJ76AQZe7Wf+UzrAeQP1roEM3OQiOcoYZshsDeua3OdYqteOXBpzzOAbwPvXQrYmN1KkA+/Fk0p0PBNrPYZ+davmx/lpLwfeA28GT+g96MGW2faOPfnRfdZPW4y8YfJgaWGHhtvvAP71Q7PCQz0eA2f1glm4ftHI2Hz7DUUGWD8TUDPzsnvS8M/b1Iz/wDqAUruEjw/ePvAHeAbwSwY92wD2fz4Go+i31g2eB97JovEzP72H0GGBqkfg88y4Hh8ro/Of+urWm3hd1EDClVyTpfhZ5qtDMGut4A+Q8wQBh7QMDD517+WN8Hh0Kg8eK5TcHK5TTew97orRhdXBa+f9OUaAyZ98tnf+tccM3/ZuZqPycV727kZ78617nNzfw7hdTrHox2PWGgfty4J9Kz7DeB4Fz+t+jlf9BHOmy4dba84HomoTJzoG+j1WZG8njZrYUfjBfFKwCU5GqoB+N1rp7nyv5bbj9Vo9cohmxXKHUoHzyIxgCqQmRXvYPga9UQON9HmZwbntx5lo/k1gUc7S9H6/wB024jcBmR38u9yZ/oW0czj5vWw8y9YIAXbax6LUuN/7K3iBtf2eWH+fNYW5+D/ABKtKOc6o24cmOHW7BPWW23Nhj0SaB4DMwczB0OsD6wRnpq3pZ/BJAkgPekln8ECBJJGgQI0j6vDSBHn8FeGQEks/gkmAOe7BFngOmlh8UOHxXgd1t/iIM/glh3mkgPP4Jayaz+CWfwRB3WR6yj5/BJBK1l91m/6lGz2JdAI9wkZAkBpoEg2IbSEG9NZGntY/OpXt8z+CFJJXggsAPwzbzBMhDifgAns80Wbf5148CAaPs33g+g1KZrdwRvDiVV7DyHvUVJBNk16oyRwl06iSfnegAqrlqGf7XZ1HM/O0GCkPJnD4qWm0WZQbcP2FGmQA/dPZqCdsUr+i3NJZ/vWVcJYfFMPVKdh1w/2G46O9/22CB6w7/Za1G6bGmB+5eBXGgx+GCWiHu8w/OgyUmlXVDz5q2Z+zrwBRAqWj7diTG+tlbs3ql/8cmAH1p9mq1IA0zlhJD96zmqZOdHUqVJ6z3/OCH7i97NtldFkhb8nxJ1nQHj84bFWSbesqSWp9nJMY/3LykYp6BTj9hmH51HCHLjFqQarJz/vlsPsTbj3s6rUo3yGear5Nkyw/Yaqy9+TBXAqmbhvyN7C45n61IZvO+2fb1jW+sFI+x91M9DASfoeBVk+lXPG9vQJn5AzUbapH2hp0n/TNiU2f58AwRhM4HyT/nnhfJZDztGqF6To+3izGfrZSZk0aT7d9Itk0x2Z6MtYE9OJUqbn9aZZ4CcAal7DiFMZ/vTwUJmNSg/+zUjRpX4f8ajYqoFHtWmx9NiXDz/FPrTrMOlauoHLGHnwWUnnEDNvlQBKBG5n2ZmAfIscOhtsKH7/AJb9CMAoeeoGif5FlpMadDa1+VMwDsVtRKbKksE/ObCMHk6FLVcPHb73iOQYxn9CjSZ9GhhmdNjYfIyqo41GjZ+tayf9yCr+ciVITCI28DIH3ggvguSgGO+Cz+RlAdeoHZTs/wAiz5wDAAbYYPP50bIcsf3uIH6FrtK6O7bfZ9vTUj4iwfZsU7BR2Qo2kD78FkA70bMO2Am81EpxvfWslXB1m/8AW9hTTP6AUj7YTnvYUY/zgheuqW9gxBbBlryAygOgy6k0dSbfD6EEj1rcb3/QbwfkQTKlOgYN1LBkzDYHWmmabUjDx6rJz8maqp8CdznsOZdDozNBYBXpR+GDDJ/PgoLz1VeM9Bt4DPyKwCG/yrWm2GqfYCGNDlxs3KlBMD+dEqx6mv4aj87CR9aj0qG/UpHKVKsyQaWjjU1usCbnNxo398agnSjCYcSDXI3zoqLL7N0dkfuNVeN75wRwLVrkl/X+1T0MA7M+tNBAlBIB9ipSc/rToQLge8N+cZs+QDQv9abMZAHdCozgeD5FCpUNhl3XffPS8570vVXJ/tUQwz781o5lsPw7aCVzZsh2ACqBTnAY/aua1mvo6FG+6syNNhv9AKL/ADyy0DEtg3gM8OhTQeqsB0HMAwDsNWjDQUo3Apct9h8GZD+zP5FWhsa0AgvavzmgmT84Gu44yDue8AQQ7kiB7MzhmYYeKeaybGghyow68qIGfkTsOpS9L/jYpsYDDzs5ocH5kg24L5zHT/CBWEa0r/nnoUq35J595hgvIRcIP3t779K6FIjMvveO+xnCDvWpo/BbijMwi1Xk4dPD8Z7eH5FqofAqhxvDrN8PH+6ZAABXtGHKjeYkhvrLIafkNV4T2GT+9vg99B5rvsbh7woprQN+qnpPzvPGrCHGsCleJRrVjA75z3q1vOUlmq3DKa9VUafJaDvCMa3tN4LXpPj68Wjz4398eALroXPVcNOKxGjNeRkEJzKrM9vUnv1qUuaRuANZe/05UqVADvMDzNaCHwE4bM6PrW6pMwGPctYABrV6Jn7RLk4newgiM2rwrgHp02x2Xng7zVnGqVNhhp062YEP6IwJnlmPdxM/zp0NSMGnoMgaIOHcNfP3+H0bEBz6qft50n9ad5lvAG0YaCCvCNmGbjhn9ZpaIYezBTczzNI5OAKBEBlwD8Ncl42VV+sXpCpxhg1To2H6967FD+8yADNcFvOZ644g1B9s8w1thro4BquHsYAr1PczDwMz3rrcmYbxamouaWBDY9aSKk/0RWfB+s1uDk7Vfs/vcsJwSXA9oaA3nD8TXNVhzzDsBNetezTAPoXO6lrzj4ewfMEgqVS/tzwKiOpH7txRDqr/AOIqQ1fricf9K/Wg+09RZHw6jGw8hgsqzJcklg2e/wCdA892a4GZ+Teg2H2wne0DljS+3Mo+tuGsfydVe/ZYhvfQkFBrD3t2MP8AGi2oO9mw64LJ/QgO9qUz1wWT+hU7NgNmWpKfkh/fSVW1WfYlt+HLncy6HZHweNQNUF4UqT12w8f5EbL1OqruAW48yubs8Zn4brrdKs56Sz7k3XsMPyKPP4zcQ5kDwOTpQ+QGcz/Wi3UJlmOTPYMGz9ZrKVsLDtVoHKreLL0s+tpo9YwXJ6rW7tuqKcU6rUnj73ecPAAVfb1t1Wm1R31bSoEyOYYHIndiDpEnjZaVKa0KVb8mpSG+gw6E0HEu+JmEptiNSo7gZ7AzMPzqPGjUOj03kWLVgPS8zM5HYH0fIs+FNlg+b/riSy6fQAHsBY9zzpS63cl1SZv366qqbXkZewBZR63mDlc8xOk6vYDvjAtBJPA2nzlnMA/6Qfer2mz50aLLpUGjQHmqiGDxmzv/ACJ3NmF9VPxpgsU6XrSDDM3sFKmPNzH451Wc9JOKGwD7DVhGh6MzXieC1mAYKScwAMwYADw84K4tGFZz9OmMacp8I0h/22AdYKwk021QAdCnc+657nR3pQ5jjLuDDmi6iOTozOebAzdDo0QzNNtUfmW3nfH2A+GwA2YKWAUqA01BBjxWDzPBRPWudZ16xTalGZDrkOs9Cu+WoYNG/wAjPZzDMHnj3mpTasOpSgmhOYPAA6DU6fcj9YY05b+s75/nTMl6CdGdiGAG73u4bwNVrMmlM+0pTL2fnVbQsgo7eq7L02QaAN7wH1pQIEHS1GAPAzwN14+tURswYxG4xBejAYbMDzwVnG1zihFcnGBn0awYJte0jNyN4bBs/QYZp+qyXAYhN+spj0gGfGw7AUA4b4TWsPvJ94NbzUiHDYZCQx7E3zzMJGwwXrI6884zDaCCBzHT3m71mAJMxuZPTzPP989gAGmnp8SM+bFOggyGeBvAfWrCNR5U+m5sN5s555m8rVtdQJnLCcEH2ZhsZvhhs3q2gPXVW4DR8ozJ51nxg7A+hV9t0EHqC63WWHs2DwB7PAzWioNw02j6rFGYeqTrGGEePvM1itXwIFxxs6VR6MEbPe9IZPrNW1EjXBAkG5UXzk7MAze6FXwOC3Ea4ZrtSnPhRAfPPCQfjYfQt7R+D9pU1o26jKmVWQfW6b2AfoRDnUmpXBMrIQbfpQSXT74jIPLRUrh7f9ewC9Z3JtB0BEMOv8h7104PVtKj6EQGYzQdjQYAql65/wCyOb0Eej8LrHojTvPU71rIfPM3ajv/AIFcSbkptK+4xAjA0HZHBZeTJqs8j15xhn5EmYGHtN6C2k3POmHhEi7D86r3ocqeAc2ZmrCHAzdDTbQVKq0alSggynzOQfuWQzw+tBHjU0AwDTVnosQA15T4MteczVIzcM6ebrEGmnDaw2PO7zNR/VTD0jmqq+9Pd87xoiEusVVusQDi0CJJkmfRIwwAFn63Y1ZuodC6r1Nlr+yRGQwWl5aLh4bAKLPnwaO1zdYnRobXYbx4LzC8+BUeg0q3oQ06lOAyH7lkFYGywy0b86q1IGg6zN4ABcvrHHinM+BbNO5n/rb2wFzqvXBXLqlc3XKib37oNjP6FcGXWrh4zWdR3TYoAPViQHfzJ6K5bW78u65z05dV5aP+DEMwBVLLIB2I8G/IK2wI4MsRsPDTubf4gfrUjD4oQhhnqONgmGQAZcUgIzikACdA8A2dasNAyYJ0AQp1EADq1EaD92j7tNABpI0AIF26iX/ad6FJWsSSSFEEkln+Gln8FQXbqJAgNJAQHv8AEX3w/dpkzSz2IDSz+CDvSw+KMjyBL+8QKWo/Dz+dIN6a7tRLPBAvE7DQb8kYJH1eGgLMzw1E92KOAYHnr5/InTB8CMOgEEhBngkGx3fgeCWDh+JgoVBGYAfWHWj70GDfvMPzpGfR4eatI/3neln8EBn4RpZ/BAeHQfnSz+CSWHxQHhnsQd2mkgw+KBdaPfikl+dUEGoHWjDMAQeIZJKQaWRoEkWPxD6EWe3501n8EefwRAsG8DcSMN/huZoQ3gaXQCAuxPf3aZ/OkfV4aLHsxUcwDPo/WpHsfaNoM2zzD5FGBCznQ5AS6dOkw5AdD0c8DBdVtL0iqxA0qbfEV6qx/wC1xwAJQH8/nXLcPilorG4VxvXVsXbb9zxQqVuVWNPj5+6PeH1griSEGpNHEqUQHo59YGvItpW85MmnOYvFmiS6dhgYHv3/AP2C7lb18y4cUGK5c8CvGx3xMAP865W1slxI9D+2L25h+0qr6nmv9bMgzNk15iuHhjxJ4RV52W/TZkOOHQ8YazK/QKlXJSqxHGdSn82ug/OBpTK9Q5LRwX9GT52TDMEY28D2xxgfDxLgls7PegGC6RDuSDdVLB+C/rAa3t/+jZYFzx3ZdsNvUqpn0aJ5gf5F51vDg5xesY3Tp1OmBHDodj4YLo8PXRY2vRJn82y3o2e89I8AM1oIF21mG6D8WszPoN48FwoOKl8Ur7pd1vvZh1vHGwNaClcUbZmEDebzOfnBVEIdtD0pTtvwLuocn9zIgyTwMFs7e408OeIUcGG7n5N5/oCdnmvOMyZRqqQMPvs7/nVVRztGsGcXPB0DwDZvXmDxyX4euHuCdgXCXrKLfgRpZ/hPbFCk+jG5JaNulXxDeNz8ZkDNecoEa6rea56y71MNf20R7xgeVlGuHipMfkSp11BAkYZgcQzAD/Jmsul1ePc5nY6l6N9901rUbrkOT8gBgqeZwWuOGxzVStUHvO7HeWKpvHji/DaBisVgJkcO82QzTsz0k74Zim/RqxoyA6AMMwNY9Lq/mnMu/secN0H4Nua2HWDuxRzCjBIBiq2+zAkfIearKJ6Y3E2g7KxSqbWIR9YOhh/gW9onpgcK63/p/hzyYefRA1HS2/ms/wDezrIN7+UiRpkc+vWZzTLx2qyWDlGOBI84M7F1uHxa9GmsRwcfggGf/VlZhVfRbmB4lRgB9ZmCYdP814XKWaxVTHUbYjVJn61Fen016RqS33qa75APYuoBR/Rh3nFvRmNn5JJqpnz/AEb4D5sP3/DNrv1t6YP5rwsaBn/Tn2Z8RMnUqOzsgxDjB5DWugWxwEuSRr25xRZjNH2czgAfrWzD0Zuf3sX3zLXnMAM1eG3j3uFyRmZLk568GM80HQ8yeBqQy97tisPRg/Bee2LqX/or1UA+43iH/g/6ia/9GC5vd3HSnvrZNRh0/fwuZfcQPUlxAeDsej9aJl5s/YOYf/VrpB+jTd0b9kuCAH0ZqO9wEvtnrlUp76zNMH38LFAEsPEfYZD52ehJ57mfaS8/rNbBngVfEbMwnRmT87MlJng5xCD2nqqT9byYPv4WKAAD2YH+RSAkv9Bts/8AbLZfyM34Y/deQjH5wk5poOAnEky/0rTVjg+rhYzAw8Q+hL7oZ+I2D31rdh6PF8H7euQ2foM8P8CmxvR4uDol1+A8HeGiZrbDH6uFzrNsA8DMPrPNLmd3iOAf0Bguq/8Ao3uBvC5zD8hqczwBpQft1ZkyT8+GCvCPu4XH+ZY94+9+tMnMifgAfzhsNegKbwZtKmmDnInJMPxjzV39hrcD/oaAH96oz5cd+3wuVcOrhu553TcfeOOHZIBdahz5XnTWFuQ+uXGDDyGo71etGN/0qzn9a2i3xPZ/Wj3Vw6sq/wCP/wAo6cEaX2To/gvLzJxU4dVXhpcAQZcrnKfKDONI7/nA16ArF/sA0bFNAHjPzrPhb1H4hAdKuczw62TA+g10xbHDzkAZjqAkYGu11L0YJcZrUtm8WT/cyw3rmNw8PeI1q4OVy2XtL8WP4wH+hXswokaiBPYP2mz5D2J0JLBh7RbQzGYJGadyBMa3i5tqGr4kkfWj2Yq9hpEhMPw0k2yLPekaSShqDP4J0NRNGkALbbIaSAwSXgNAaDI0efwQLsQdCPDYkaJLP4JZ/u0HekgPP4I880yh3gijuRowNM70Jm57tS9tKz+CWfwUXWS1vdqmaVn8EgPFR896PW3o9wka2aWfwUTNwEefwR4kI9mKjg82kBgizqSDWX3Wb/qRL4YIcPincgQbEeBzzSSw/DSw+KBJYfFLvQZ5np4I92X61NCpTscObPBRUs/gpXFpBz38DBzRMD7HgzUQ41Dk+3tilPH9GCWHxSw+KNEQ7bss/aWq8z/9O8h+yVjn0VGtwDD86mpYfFQOPgywcjUcYzDyLQQKO+Y6kWmxoYecz3qFAkxJLQNt55+TBSDgSsDcPWD5DPBYugp9KqPtH6+yDXkBV+tRvZ1E58l3yBsTrIViM6DcSLDw+fers6w/oGB8tn5FIpGabBx1NP8AWneWYjDmj0XJI+G+AH5Ego75+0cM/oQI4z9Vj6bDEkPnwQRreYpTWb77xu/OpcCBUoxePUnga8gKkmPMMyuU15J7/OgM/VTx6dR3x+9SJkxsMPV37P2bFZ02BFgP67jAH9adqtHcN3m4vsj6Ees+zpm7qONnmtHDNsI4LPvHWAf1AlAYeRW4A+bWprgH5EeJLx+EbjapIzwSZuo2GsatYfPA7pypQPNGfkSeqUFl/Q1AZDz4KBH5OqyZQaDBhh0KXMgVXEGJXZ3oDefe8SJLzD5EjqvLZ88/mAeRWBk0qCzgYRMz7zzXxmAxqg/gvsaS5WHdCDSqk878kZaKNw04k1WB/N1uSYwfjSA0QQV8OlPzHcG4gAHY8byjnAfADYnVZln6DWoo/o5XwchqpVy7WYYN9mea2H8g9sSQ/wCUF5HJD8GPgAIOO8nF52O25UQktAeZgbyta9M+0khr1HLANAMOXj711uBw64SUSRrxKG886H4pma1UOZb7I6dNo0aM18gIx7fLzuFgX48Hh0qe8PYb2xWcPgVxGqQhKfOHDa/fPbwXoD1w3/8AZggOpA99HyKzblTPAqjstf8AKC7mTkf9XZz/AM60tH4e8K6Vh/MwT3e85C0zwUp724GaDk6H/Yc/rRtsgrFGpo6FHtmBGD5GQBNHclSP/h9oAKy9VUp73H8aXqSjfgn+tEs68b8nPUlHv+dR+WY/DzWoeolGP3jwfQo4UGmmObDknBYs9s+yDYH4ikMssZaiuvUMT5/zofUjHu3FZtX6LPnRBoe7U71C57upRvzml6kf9xKjGfyGj1H/AOIPxE1zLGroa4Z+RA9bE7VzcimZ/IaB6j1UAzbo+/z96K2mmAB7zBI95KqOHXDPxIrx/kTTwVJnw3Iph+RErDP4Ja/zqkeelh7zD61HOq6Pt3wVjR8z+I4CB6SGOnqLNHVXPkwQBUj+RFtRGeb5d1/yAZrz5Rw5ye7O87xrp1w15+Bb1Qfb72TY/Wuc2eHhAu7hcvM6XbDzcaHI1Pf7FZnPbD3ioaVGqTwacEMD86swtW4JLuo429J84NAuP2f3r4f2HXq2wHs381FeqWbpuZqa9RKVR4+vXG2YH/1DyhPXDwyhwnZU6qszwb9zE/11kvAAqXzqRGhy6k7g22Z/QGazT3G+0abh9keGpyZvnl9ALL1XjBxKqubcGpHAD8KCzgt3rsB23FjBqVV+GyH/AFh7BUkm/OGtvarblVA3Q/sjOf8AGuT1Wjny7rj86Y9I/Ge2Jo6axJhjBf5kw8jKwHRXuN8UJgMW/Sv+8TjwBK5Lq4qU2Q03rhGAwzN6OGCyRvTteO45SowaAYKwo91P81yL/MzGov8AR+xGpl6TQ57oVK7rjqsx3vaDs/OlRLklhDBuhvssyw/CZzeUJ7TnlICW/wBZ5oKJDfjHz1G9rvYPNc6zVSenVKUD8t/lpbZ+MAd6dNlwM22GA0g7FIOHLjE6/Lgsm6/1nmp0DlGabKqtVcPyAAdZr2F4KBdR0rDlKPDAw68A61Epr3P78OWkGZnqu7ASDQktBK0MPOakBUuWDw4jJ/kXhgjptZkyGvAMI5nmEs+g01PDe6ww5m7hhrB0K1pUnWi6bjZmYHsA1E9d2rDlePgYdBgALJeB1iAwy0DBvxnnj/BPPAFHpTL9NkRKrp9+DJqJMrHhNQeRk6p5mDujmGCl0GS+cUG65Q6rM0/Y4QzwR4UOY+9VJEt/Az3mnecCYIOagRpB+9w2Ia8EvDZTY1Hz2cuHXh86Cq+roceO2DeZ95+dbYTo6FNiMsc2ctmTvADNnrDNIJMSBK+6OSdn4yj00242fKVUIAeQwMwNKYyclo5zctmS0Gx549m9Zm0g+bmBqa5vB86+xpNSjO67YAbTfXmCKlUSq1geUppgyB++M9iu43DS443iMXPAZd85qwFEoLlyNGxz0bVPsz3gCr5Nq02mzNN+q5uh2aP+2rCq2NceDWvc8Z4zPDNk07TeF11U2Y04d1QAafPA/GzPD6FMWu4ZK6qJKuGa0xSnJLxsHrMvNB2edV/qq44cgObqRyQ7wNddmWSxTcIM655jzQdARwAAVFPsOJJka9sxalMkH2SDzBXCMs/DjVXwmKawB4AZmYBgrCBSm6xWdOqyszlYZgyfRgC2FpcJeI0lr+dTZokczM9Y95ra0fg/bFEMHJU6ZUvJm9gC0RduTvUS2ecCDSrcenun7HB4wzWwpvDS7q3pfzMzb0cAwweMzNdYh+pqIxoU6DGhtfuQAE1JueIHv96z2jTPwODNss6Xrx+TWMAwwdPAP0LSwAoFsRQptKpsaBHDsZDBUp1uuST8BwATPLTpO98wM1oLaZdXig2wwZ5qnk1KsSZWAOYNYbNilBDbBPMw9Z0GwDeiVOzDlyXcHzNOswMC021NmT6HQWtSsVVll38EN5/oWdk3s5Ma06BSng/6xID/ACINLybcPx5z4Mtec1TnedKN04tvsHVXQ68NjKpzjPz3+bqrkmS6fnDMFcAD73sGzz+cEUiPfaOsHqVF8IcQ+uJH6E9DpsSH7BtTfU9wH7jP86orhvCh2l4FSnMvTf7PHPWNWNAcmdhvwVfUq9R6COvcFZjRvk7/ANC5jXuLVx1L/QcFmmtdjrrOZrBHAB585UvN54+948zQdFuHjY/JaOLasXRBz+lvdf6FgpMmdWJHNVyc9MkeczQAz+G2pDLOfu1qyNBGY8iMABSMN2CQMoGQBIAUvRDBBGNt4DMM9nnBWFhgSPrBGYYCo4PPmR+B4SCQB7ESEwcP2Gw/OaLxO9zM0AmZ+zSSw+KSsBn8E6CHsRAeA+zzQGaXdpoPKhPMPEDrUIIwXzoNfc8/aJK1kZ7EGfwSNAqQNLP4IEs9viIFn8Es/ghz+CWfwRkLP4JJJI1Cizcw1EkPnbQO9aDoSSUhZ5pGhxNLyaiofANzHxEOeHtDwSMw1cNiNAQb97aMD/ETWs3npnmkyBmRg4a92JBnvRAaHvS61DyBGH7xIEuxID932I9HrZ+HggD94gDrRoEHYjDrNNHqA6H1p3/GgQBgYAaXiASXifiZpAebumgW/Pf0JfvEfXn/AMaZw3oDA0vD94kkYbEC2IEku9As/gjQYYI/EMkBpZ70GeYh9CSAz08OjrQb0f8A68/oS60AeJqpGG9H4YGln8EeQZ8QATR5mB4dadPe6aaAHMTQ2hSaU3MDTfz/ACKIdsMAObc6SyrvE0RhksTakgRqzR5HN0a5p8Z35FqYHEi/40xpuuXG9Jhd+AYPfrVZhvSejZo6XWLeqtDuRo4kS/5nMP8AsWp3gnn+tVlYsD0k6Ux61o9zs1iEHQyHWf5Fy2TGXYuFd825R6XHYrL95nI87UbmWTD9Ci/7CXOplztzHfVXFG2XobvfnGNRJ/o/cOa9DOdZ1xw4f4LRmvUx17hzfgBb8SXM5t/oZqdKwz/1Fl43o9sW9P8AXNq0oDkf2d48wSLQ8j17gJeNKE3GIoSY4e+gvZgsYdn1EPEYlGy78i/QuTf9Nttr1dd3C/kP3sHeCoalbHA/iK/rgcbm3/OGi8CvY8JQLnvG25GBxNbD5M1dxuMdSjOhz0TA/Jor1X/6LVSedNyjXxDZiH0A8zmqaZ6G0GpVQ27quONJidj0GMYGo7lRwvN/8rVHelOg/Bwz/CVfUr2pvXFCSYH52cF6gmeglwy5N31HcdVjTfcm8AGGfzrh91ej3cdjVJpipTowameB55gf++adq+ll4FegyaW6573sDBTqOcGZn4fQmqlY1f0iYYlRj+gMFTM23c9K8TQnn87W9LRyw69b1KgnFDwFcHR4IB0LhjNz3cyemxUWTPsZPYakRuIt2xndR+Ln+dMDtYUqL7PQUeq22HJH9xZP61yE+MdcknpwWDZl+Q1IPiveMb28Ez+sFdQi2jjcPYMyqCb8XAD6wA9i6BasmsW216qpVx1VlrsaCSeALih8aawye+lYH/cq2o/GlwHdedSpMk/rwWuHr0Aze19xj8O8ar/5k1cM8ReIzI+Hc0l761wf+X6lAX/s+9+c1ID0h6MH/u/J/wDMgufCXc/5SuJv/wAbP+P/AF0QcTuIwe0uAw/IuKB6QNuf/Cpn5N6ms8dbLe9vzjP1sphTsYcWuIYeH9oz/Qj/AJYOJv8A81H/AOCuHyeK9qvO64VLRa8+GauKbflsyWgfiXdTTz7JAYGmB1gOMHEYz8S5zP8AIrONxgv8A/05n9bK5izcjEkPAlUqT/dGkFz0pksJctkD8gHmvMDpx8V+IR/9OB/5ZB/KpxD/APmNz/wVhAue3AHfVWfyJfae2A9pWYwL3CW2PiXf5/8AvNJ/IGCj/bm9D/8AeeYsZ9ubG/8AmCMmj4hWOyf/ALQM/kDNMDZHc98H13BPNr68EbMyqz2szqszL++WEk8XbSDFilOSarIPojx2d6js8crcjBpv0OfGd+fo/WouFOlssvmfjvvGpoAAe8P9a5rSuJZxsJ0qCcxo+gGutSpPEKuVho/sxbh/3rxqNjpbLwZfOqS568x4USj3dGptYA82d641cNevyS6DlViyYfkwDYq+Gy2eZno6vneTY7nb3pD3xDmHFvGhw5OgeBvAYMmuwWrxmsCveHBuMIcj8GXsNeNAnvs+HqIs2JPt21eB7krfDex7tHUrFsQ3j7JABgf61zSveipQzzftW45MB78GWGsC87w5lYgZhBrMyM15AkmtBSuJHE2g+HRrxn4eR57MP41exe17gVxRo+b7FHCfHD3sc81lLetW47kqQUamwTCafuZYaJrTM+kDxiZLxKjAe+uGCkPcfr0n4etaPRJhh0PcngYGmxYPejHxzDotVl75AmBmsJW6DcdpSjpVz0eTAnB2PBh/GtK9x74xSdjF4nDZ8gM5pqt8Y+I1z0Y6Bc9wetYTnY8z0H5wVxaGPz+CQb008ejvwz+QEz64poB7fR+R4FbJONAgZeYkgD7B5sn3ghM3wPAD2eQ0anTSz2JZ/BAgMN/Wl0GlkHzpeH7tWAz+CM9NBh8UkCSz+CWHxSw3oFnvQ5/BF3aaHD4qmeySSSR4WfwS7ECWJogsPijz24JIM8ECM0utLPYln8EAbwR5Glnn7RB1oHu3UQZ/BLsQIDz+CPP4ID0+xAgeyNFkaaz+CWYZ9aJO5n7txI3k1m3n4fQkZ5u7OhFHQNHn8FHz3ozQSAMEj0MM+9R8/gkaB0OtGaj5/BIDcz1EendiSDPYkB7dNBz2MzEpT+vOg63kDPBT6lch1ua1oUeMGB5qqCmud/QrCNGjsug4wvlO9o2arTnnwYCh01nPve6FLzoAf6VnU0I//VA3rGVI2DLQcbzTUN5tloGM8Gg6FpFjRVisUN5rChwZIfvnlXs6/tAfMPoVPMqrYO6DDbx4eQE9Do98VL/RtGqptecGcFoL0DAA8dg3k09PYDogsh8+CuKPwi4oPNa/q14DPo5g+ha2B6P1Z0gcuO5ozId4aJoOdPTKdMYBttwGXe8zNVj0NwPYXBn+ddoh8CuG0N8JVVrMmY7854LSw7M4UU13UYo7Mn6zNDueZWdTLx2Hj+RneZrQQKDdtVdAKVY9VNrzr0F61ptN2U2h01kOzYo8m8K494bGiH0Ix25fG4S8QnjBtygMstH1m890K4Z9HWmvf+0F6gH7qOyC1Z1W45Pv8FE5OWZ6jmChe0Wm8HODtEwccl1WYYeeTs/wK9ZpXC+m/wCjbYAz7DMzVecOX8iejQH/AMMERtbxrqfgDp0qlMww+RRZN1VJ4vvb4AHkTIM/iKQyDDPuAP6wQ2rzmTpI+G4eHnzRsw5b3tM1Z9mo2wz+hSNb92CtcKwKbh7tII2/BwDVns6DTJm2B7FCQhGY+dF4Hu80ORo9bZpoAwA+tSPAAfOoOtvS5yIHtH96tCcZtl7s0tZz3eaj+smwLBtg/wA6iPVJ/Lw3EWs+ZfAfImTny/7WeCrTmOH19aZek7EFmdSMPfpBWH/eOAqcHgMdRLmWwQWp16d7hiNh58Ema8/nqORI2fnwVLzgf1KOcxtBpXrhM/aNh+Q0vta2yGDkEz/Os1rMGGeuCgnUnz9mCIbMLwb/ALIaR3VSj63DBYoObknpgrWHbdSmdbaLaP7YW/h+1n+hMnXrck9cVmT/ANiooWZTqUw7LqsqHGAAzzM1hIfF2zmcw9VHJPPAAZVjdzAo5+zo0b9CjxreYn56FuH+QFiarxduMy5GyrZjAf8A1jA1irh4i8VDkHzc4Ibvey0AYLLwtuOJ1vVG3rXkOS24zIPmAA1rZmslb0bRaA8FlzCsVKQ1UazUeZdP5+hbKMzhQ5B/uV9Dh/4fP5lnP40/Z6A16jsd4MAw1ZG/P+BTYHEKuXDbTtVqrj0Mz3xo4YAzmuevQJ0+EESVOOTEA88zWoOAx6tDM5LwMdDJ71wez+99Hh/YzkaTnICdeIM1J0P6RrZh+RBUjtiZMNtulPAfYABsNaOm0Sq0qqR5z9uAcdjP7i91n+RXdVuTRqnrmDZwQ5p/2vDZ+RYNGPP7mWGGYYdCkSZMt4GmIODOe/AAQBhpOypzj0l1889+xKMbgZuPsYGHz5rVGDTJ5l9+nRtLz5p2qhBhifqqI9MMOgwwSDQCmuym4sblz63sN6iTzYhg1g+DOZ4b1httgYHrRwclsYSz7M1LpRhG5hxhj24YGptvAxDrMfnp0M4/fmo8nkaPMkadchnHfPMMDW0WYO1h6DMJqdyIRjAMAwVK9P8AHBiK54p9ikT4DFSYA4lYB51g8wZMMANR8NGKDE7lo0v8VemFgE+jyYU2dVWNF2D4LPLnvM/OkzAiQITs6XOzN88z1TUR435mlqVEJJsBgGHYCdeelHg2395keTvNFLapBSjNpij7GjDN5k+vNVkmm1VmYHgYACOH61mfdfU4BIPokGe8PoWgoNk3jVcHzlBynfIkPYAC52Vql6fFgU3XcA+d8gAoUaY48WpEzw85hgmr8eiWq+1FbuOBUpB4bIm8A/Oo8Csa2EWCxzIefPBe4RtenPuNkf5mfkgf7lSAgXcdNaqVVr8kzM8AZ1jMwQU2pMUoDl1y2alMw9iDJ4LZwJlRuHOKxY4UfDvlvBrGoNslMt6pPU05UqXnIPozZMzP86ms8NLjmNMuORA2B3vYLcUqY4DR05x8wOL14fOmjN9meD79wSXgcPDS8i2g2zJ2HUoeE6dR6abQe2AD3mo7xzpkhqJBpTIO4YYYda3f2nYZm8i+2uRQJL9Vu2qnEckzJfOYRmYIZhgvFuhU2mnb0N7m6jGkyDPM+xVtY15NX9YwaU881/SXQ861EPh7eNwwo7EqlM03A8zdlvYGf5Fs6Vw0tWHHwqLHPu/vuj9CDjzNyRPWTUS1bcnzJrB5+F0Z+Ra6NaXEa7Ra56DAtuOweYb8zXWGeRpsfQgwY0ZoPwWQBQXq3EZ94pwjbPwOGNDZka9VqsmeYfkWmZ9W0TNunfdmj7M1ST6q+81psbFVHzb3tHM1cI20FSud+Hg/zQGHRgqmpXDUpPsGwADNQplKzpbrmHQrg4DAO6mn2Zrxav5OXJLUluGadhwAA/ZqwZhvvHgwCF6ZQKO0D9VqOBn0NNbzRKRyzYezUjRcZi68pwIzXnePBU/2huCeOFuUZmG1/a53Wo4W3zjoP1yqvVJ0POexFLB656dGkaFKinVXfOGxn9ajm9cdS1m5z4Q454eDHVgzDYZaBthvBSAZWqVJGtW340gJYQTOR+M8eZqw5OJl7AFLmPUqlNc9WKlGhxw6zeNc0urj9Q2RepVjweckdk57YCsdDN7R9o2AAHWfYCxNw8Zrco75xaUwdVdY8h4M5rkVYuq7rn2Vyqm812NBsD9CrAjGHkQaOvcTuIVz+A/VQhwu+PH2Z/nWX5NgOhSNFz+tIGTz7FqyGgADM1Iw6vAAEYaivCgBrgODbe9Os6nvEYAj/eIEYbdRACPPZ7NM5/BBIzzL2fQh60uxLPDYiCQI0gRZZ/BFn8E14ee/yJb1YQH4qM0HekgP+86Esww1EjQIDz+CDP4JbEtiPNln8EBml+8SNPBYzQIcPiiz+Cu0ln8EkKWHxUBdaSXYgVA0g6s9NJIA3oC2AGaHPMUv7zvQIDRh0IEQdfYgXiGSaPq+tH2ID9gen19iBHs7EgPJLRA8HD68EujD/iUg+/A0gZcN/wACXg0CDPNLNsPzoJCPPeaDPYl2I8gfie07EkGe3w0kaDSDq8RB4gF4fkR9YIgXel3odiDP4IJAanekmkX7tAJ9Phos+jU60sPigPrVA19PPPNxMo8jUgs/gl7oOhINMy9okezvQLD4ocN6kA8wDDrZxdY3wwB3yKPn8EBAGKRnuH50g60s/wATegM88Ug6dTsQG9n4aQbEB4ZgbnYlsQZ/BLvR5A0H7xHnvNBnsR6HP4IutR8DA8+xPAeSBYb0WwAwQ96Lw96hZpnXjSGn28Nh54GGYGut2xxybo9stUqq0OZzAe+ibAMFykDSPrXNzev3xgdGuTjM3WIQBSqbMZkAeYSJBhmyfyKRbHpP3/RPDuqmxq3B/Q9+tcsw3pJw+v0LeraDxv4X3tC0Jb/qqb/Z6h/kPvR1jhjaV1ZvsMM59kiPh/kXko4bD3tG1ZW9c93WlN5q3Ljkw/3OeYfoTCHoP+Tq+LY8S2KxzjQdEd49/wCtSIfFepU3Rg3VQ5kDU73WcwWKt70nKlGLC7rZ5kv7RBMA/gXVaPxO4ZXnH5T1xDP91L2Gow82rzes7iFC5Gc5rD2cvJwNk/yLnV4ejfXDY5u2bneqph0R6geB/rXVarwWtie/zVNzpUgOh2P1rOSbe4sWfIN+nSwuGP3g97Zem3m+5LMvG0j/AOUdAkw2j6HsMw/Wq+GexeoI3GZhl8KdeNuPUoz7DZzD9aQWTwL4qZv02DoyGDMHnYPg4GvO428vvaDx+OwB/Wosmg2/M9vRoxrutY9FqqxmnZ1s3UFSDsZkYAuWXJZN62l4lftyTGj/ANo6w/WumLes0zaVuQ5DU6JSowSGD6z3q1N7W/a2Mw/chvUdl7NPGo2JD1KoBjmxceB/9bgYKCFND3EVnDv2AnsM0QG4HQmxWzLbpUwvEgsh9AJk7Jtx4MPVoZ+fvVsaj8y4CYAepGDY0NBnSw6DBR2bYoxxeUlUqMeHQeCsOZfNJl5wz3q3m1E9ZNuf/CgD6EAWNbnKusBFwz71o/Dy8ROgzmPs0NspGs+jRgAMHjw788M08dn0oy1PG/WtLot/1powxRKnCjsM+zM/zq1olq0asOm3VarGh+TW70kBhkg+1WxqHRJjQMS4E+O/n94jnmYfWmY1Box63NPhGwDMMw2KQAYp1FotHzo9WiVm3340abCPNk3QzW4PiRdU9o2Kjb9t1LPr+5gsY8zB942mo1SCM7qMP4ACyarB6ZrTDf5FmNmfsg6ASZOpMm65FlyYxvhgeieGaiPXC/6y9VADL3eGzeauMDDrbwNedUHcjnPrkljkZ1SkvM+QzUJ6BmQOYZ4K1wD3jaAwbDoV4hKvNkMPDA0YB+Ipef4iDw/eK2J0DBLmW89PUQ6LAF4ad0Wz+sEB5gHvM0yB4HqN9CRxnOvYhDXD6FYdzb/Ojzw6DMPnUfPejB5vegCSD7zoaks9nkTr2DwacoAeD5wTLJ5+73p7wz61YTIcs1psBgCd1tYfE3pYOY/Io5gGw3M9nkQPZ+cMER6ajmYYeG4hA8PaHsQSkks/gkBt+8UBrPAtNGG9BmB7MN/nSz3qwZngSDPdmjwbNLDD2iAM8yRpACDB/mPaIzsaSXiASRvaI6jip4SA+rw07sxTSII0vEMEaSPQdiA0aA9NHhZ/BJLD4pYZo9IzxSQ4fFEjwkgQgi6zRJIc96JCi4F0JZ/BCl/wdBq0EaQGfs0gSP8AEb61ChZ4Ic/gl4hgg60DoPNo9Zv+pRMPikgl5AkBgoniB1ow3hqIKQKJc0//AEbbFSk59Hg4K+pvB/iFUo+v6qCB8kg967Q9fOH7LBjfoVfMu2sSfZuAz9ALkdbnkb0b7nntZ1y7WYbXeEfNbOg8DeF9t+JOnSak7/1h7Z+hAdSqRj4k54/zpozfe9uealW2qAOHNHL7jQ4GfyMqS9fLDPh02KyH5FjgBPAyjNdSbqqrx6mvh9CqpMydJLUOc8aWDh+7Rgy4BajiKRAZ3+zRcu4pfQaeAEQg8s4aMI2zUUswM/YbEAB7tQAAE9sSSw+KtZJZue7RYfFLBv3igLPD2mxHm2YaeGaB7Qw8NNGeKtCQGoyODfSjzb0lB5nah1jxzRaQZ/vEtZv+pV7zyXiB1oJpzMw00Gs5/Uq83jySN5zDrUM7TjPJM54FvUXWfD3gKCcnN/x+hWzXwSYv4iiPSYoHpg4qJ6qxGRPxOhUP2kbZPx5zP61beG4emNqOcls+8FlWbnCZm22bwB+No7EzVaw3R4rT8rmTN88AAwWLxqNYwDeear5NbCMWm4auabbedL9ZXBWQpvyAeayP2k4V1usnQ6VOmVWX8mwDVqWHrUDDNgDP6ASo5y6kf3rRZz7HT3qkk8SH6DMkUa2LZjAcX30jAzWHr11XP4058+TM/wAHvXP+R1uTGpsaVg/OjB+daA7n4c29AAH3DkyPwQDea5Vc9YfptBpT7DGsEqNmGzNUMaBValnOnVLDPoa6DWXcOgVXi0/J0nLcs4KaDB55vPZmf5MFUV7ipeNYHTYnerY/4MQMM/rVJ6q93z0k/wA6ZeCmxmvVzDEx4/OYLsjmaq31rXGao1UvXEl7Dse3grIIdKqtcCpMaIO/hM9CEKU+fhthsRQKC/DlOuU6lZyO97DNR3PU6Syw87qMP7/kNSKaGi6bcvCZrhhvDoUSBDrj0jUg1ym6oe51gzUuBPfedP7RytbQPDSwwTbxl/AOvGDHslteWNmiGDneGCyUblJlwSH4rGi0b3gtB2AthMgTp8eOxzwQGm/bPOnsBfS8f4nzb/yqeMDjMUA0+hTYcyXGPQltmAH0ZgolSu20qVH9VfaP1lL/AOqRs/41LrHFe472ap9Kfo4Q6ZCDADwwM/rXx+b9z7HD+xHeOq1We7Lff1jcUipQ3IBu03MzkB3n2IKbJpzJBVX3DPQ90B7zUit1uC80FSbc8V88AZPrSF2p5LNR1dM9H85pH9qodGdnORQgNa2geG/NTodepTLuvWWMw8+aTwUp45Et8M2tmiHYqY4U9t0qvwJHrWnVzRjmBgfPPdf5EWATJrvrGoszJAdZtdCnA84z7Mww+REyy2Z6jmAfQCha7o8bhtMEAqVZwqB9HYueVs2IFxxwbiBVQDPMAWon1L9ni6ABoHnmHeop15umjvYM8+8OteinewOpR50u3J7IMf0c+9X9YhsGYSn6M9Jaf6MwVifrXCPBqTgPOnvA/ICZqXPSX3X5UuNpAeAZngH5Fv4UzsOTTQqUdtgw3+RXQPRGX8zNV9HODWAlvsUYKUcXrkSAwzUilHSpjunypvH3uh0LzmStmXrfN02Obkm72cuz/nU2BZk6vQnddw4ezMOYmbFTz/VoSP5tDRaP9YKJydNP28vP86x8DNVK26NJkR25TDL2ueGw+hXcOwIlvTY8r1k8zszAFZQ2bfZhTQYbA3TZ6DUJmquXDIBgKbMkuh0YB0At0XDUQ7qwaBiqt8y0HQGeClyb8fkx3XDpRmcoMA0ej9avbb4S3xUo4BUqPDpsfv5vYf8AAtbR+CFuUctOq1mfUg/CPYH6AWdwjDjv2qrlS5iLQKG9Jm7M+XDr+tXtB4UcVLnwnVJhm3o59eqfR+Rd4gQ7ft5jlaBTo0No+vRDBI624B+3TAxlvej3bNKdCVXKrMrDodhnoh+hb6m02h281oUalRoDXfohgarHrhf92eahPVWc97xaDSyakwDR676p5lY2ZxczBV4Rs2gc35/OpDLKBp56VJj7zNMnDzaDw+hWYM4BpqQzAcxzWQiRoebWaMIG/wAPrQzLhtijgATq5GZd8ix0m5KbXptQcbgyQkewgO4Zhh5wQbCsBydGly5x6McA3n5FW0Ge3WI51gJzPLn4LOrsPYuc0fhpfEmsuzrtuA3oko8zZOTmf6Fq2bDtylTTlsa37lnsD6FkFWHr7nz3olNqoBSgPYYGAZp2jwHKUOHq4HnfPnmngZnZbADBWsBl83fH2fQtog2PWr5/0EEggXGboBoAGfyKpuHiFbNsH9+qus7+FEPWNc9r3Gy6qqwbFuRDpsR/vdMDeV4Rt1OvVWLZ8Pm7jr8aN8gBma5dWOOtVPw7SiZtf2iQH+Rc3kg5PfOVVXDkyz63nutPMst4eGC6Xop8msV6Uc64Kq9Pd7Mz2AkAAz7NPACWHxSIQDo92j/vE7o5o9FWGfE92jADwR4fFF4f44AfzogLIOMjsRAeHQjzcNBhsVrH4gNJd6RnmSA1ANIMAPU00s8BSPeXs0QDMzPw0ZsuYe0zBLo6OhLNv2atZIA3ikkCBI8MxS0WzJJAkuxLPekgM0AG37xLBsw68EB/if8AGjPBJH1pJI9JJCl1+I4jyCSS/dpI9IEu9LvSNUEkkkpCRAmsPijVBGe7TSBBkCQIDQAj7Ej+TYgD/h6fE60kaSBnRcyB/Uw+RGYfiJrDAzb7M0QIDRgmQPMkYH2eRA6fR0I+xID3oNbATb7OtSDANniJADYB4YbAQAeSQbA370WPuNxBnvRo8G/eKACW8EntQwDT86PBwxBxWz2ADbB02zbNGgM8zHURo0LelsxS7QbSMAzVIAjBAewDcSz2h4iPNnfD92mkfie7S/vFL0s9gJf8HWaSWexAGH+ojzwJJBn8EBvZmeokkCSBJZAkkAb0eQLNv2aE/ICQaeRpHvDTwR6DxMtNxz6EYHii6zBzvBCYICz+CR5gepp7D/gQgGfh96Wt+IjyCM0QJH5HDR7ADw1DQfutRsEGGYoAR5/nTCAGCWHxUjBs0sG8c8EELRzTTzLBjp6e8P1qwMGzUfRAz01GFr22+JF/2k6DlKr8nS72pHjAYLqFt+k5BA9C5rRehh/aIO8P0GuL6PzoOWAww3pgey6bdVgX/FNuDLptYaD3T3X+hZKscDbYmTzqtAfk0Sb2G0ewPyLyucZ+M+EuJLkskHQbJ4Gt7bHHjiNbGkxKnBXojHupfX+sFjcDpsmlccLGaM6bOCvQg90eGakUrjrQ9Xkb1ocyAfeDzOYfrStv0mbKqvh1yJJo7odGt4wH+ddG/wCSt+UbU/m2pR3/AHzJ5qNjHybP4J8QjdqJxY3MH1uxPBP+Bc5vD0aZ0BgJ1o1z1kGeGjI6/wBa6BVeANK1ebtWsPUeafRo9Cz8+Bxwsk2nH2AuGEHWYBvTbVxqvWHelq+JXLYmMx/7QAZgqJmSw97NemqJ6QlAelBSrngvU0z7JAZh+tWb1k8E7/8AHBiADp9EiI9gabS8pPKD3rvdw+ipOZLXtW8QktH2SA/zrl9w8K7/ALVfP1rbkk47HXIjhrAtosUXh46aMAY0j86jA9mWn0YKWYeFpsbNnWrQAGd+DjaXQem2aMOeZ8N9wDSBnebiIH2IgZz3mh70tbMUWdMOtADP4aR5mCQdAabiBrRcMkYM706DzaNS82rpIPsu+wMwQmbcwdNvD9CuAMMvHbzDyKOYMGXht4KlxbLzKO+y/wCsm9+HYHWrWBc86f8AtcF5kw7z71amy2a+hSohh0I9OgDkkAltuZiaSqpNNnQ5GvBfzMN4AasHmZz0BqomGjn1x/8AUVoFhns00noYASjgyeP7c8yHkwTzJ4e8zRAAZ0ejoRhr+0bRBJYDs3po8HjzbzD5FAWRp0Dc94hw+KQArAYHq+O3szRyQDPw0jAwBM6zgEmw10D8idDM+hJ7Z196QPNh0GgXMvgew/yJ4JLjwnqdaZzYMfD60JngexBIRB0dChayl8yeHnQO9YJGyD3f0IAksH7MMEutAeHxQdAI88EHWrCZNHn8EAGjA8z8NA6BmAajaEOhfOg0OtgoBogPr1ATWeaQAavwDwAPeI/A86jd6eDTMNRtxXbPAO9HgeGpglnvS5lwPZqGhGDfY4gAEj60eHxRkDPegM3ADY2jRo1MpYfFH+RAgSWeCWHxS/vFTnLP4IUSWHxQCeogPUyDTR9HiZ7EX92gFAaRpYe8QHn8EsgQHqGliCA0Helm550CA9iXQkkg6KDLZu7zwTpgAHpsZmpJwGwPNEzsLwwXEtH0cBRgzmncHDNHouBvXmG+B6OzoTODge0BSM/C+CaM2zXqRgYGCAzw7M0AYZ+IjN5QsAGeXs08bz/4eCiczgfXvQHMczRCdrACA5jaiG9mKjm8rFhz7AITmAar+Zb944kcymgPtz/Qi0r1lvRnPbD3iz8y6qHGP3x/QCrHr2puXhxXkQ13OZoHpLf44LFSeITAH4bAB8hqpk3zOePwG2WQ+RWt0bmQ86DnHAPBc0+2E7vfTX2nrDx/dNbP5AQdNN7NLmcBWChyb4nl4bDx/kWtpVmXO8AS6lBmPfRsQHJqXkVecmsmfhtrXMw7YojWvVeTDDrzPPBFM4o8L4DGfrXWPyNBmoeWy7NKuaSXs1pYfDevzI4OGfX8ipw9JOmh/wCytpazvY8b2H+NZ+5+MF/1UcKVUY2f9J5cPYn9aJbgOEUGHF9ZXU/DZPvzeTWfCSA0DAMRp7vYDQHmuRa16VJrXrj8mpZ9kg9isIFyXVRMNTRhxA8gKlthf921JmA0xblN9Wh35guXTJNcuQAYqR5n51pqxVadXouvzxyXfIap5j0Sjsah5g75Fx9TVCCybfZ2VyuTHnT91H71EmcPbRPD1M+dNw7896s4dbivNaj4b1EnzGAMPE606hLtizPU7pvwaqEl3sOQewFcXDYFz3PD0HJdHA/ODyy/ORTH27wfQrWHc7FHpupyM94+gD61HSyV/OXrGhBQ5Rg9HYPlQ2bMFZ02idDE6qw4YB70zUSM9Okjrvt4Z78EfOOM9C2jharCq02CyAAxdRzDPsDoUq2JlKpWtz1Re34de9ZKfW6xDd0+R1v7kFa0SfX5LWuxEZjfI8zvW2BNCYe9xgDBrWPAzDrX2g3J9lX5B8jz7MrrB01U167bqMDgtuRj/wC7GrNmMxWLK56lZ+tT9jmai4O47MrHJ+I3Y8OmmfsTZjb81FC27mqsCRcc6DPhx2QzMDwAE7aXEus0qG1Q6zw1qVSqYe2e2YYfWpdYuHi3c8KRTjP1bR38PBkYZq/EG9qGzIE56e0/odavrqDCZofIGf1rD29W+Q52DKuYOeivaDOB4bFGtjiRTvt1IYqr8mpNRQzAOzNbdyOls4EDk2NOc2HnDlw6wUcwwdz8bR7AM1qGeItl15h1xuC8y6fQABmaz8+4aVGLnooMvYf0d4P8iyWrznwaafNOU0JOHYa0Zz+ZgZ0a1WaVL88sMz/QqxnijcZyvV3qaBAjn2BDwzU6Nc8RmRnOprx/ICG0FmTcExrOWdH/ACBgjh29EqWfrm4GYzXyGru4ZlGo8+P/ADV1hn4KzlSk19mObESlBzD5mbOZ9iGzMNmmwD5GJ7IFY4RD/ZDzTMOzH40XnpTkx6QYAZgz7EFLCAAYNxJwBI/CNZ3C4tRTNTAH33GTaPrDyKVR6qf3gKM/D5jDPOQzngpcCsROfwqMEDkd/gmo9emPhKPlIMmG0/swBnAE2jaFSrhrnPidfiAGz2rvgrW0Sq2qGbdZlw+dzzAz7A8ix4PVE5jTfKBMd7AANY1ew+DnGK4SCXEtmHAa/FlnomH5F6vazqtbcr0CRFcf1mTPZgGAAHkWS5aWyRsRd7ueBsrrVK9G+uPRz+1XFCSHkjxIy6NROG/Dm2IWg3Sufkf2icesazvhZOCW9aVRuH2dDqUxo++OGz9a3tvej3Rj+9XHzLLXYyEnf+ddbOpRGfDbPBnyKJJrcX8RbeIGdpvBnhJRzBw6Hzh/9YeMwWzCZBh5t01iHGa8jUYFmZNY1vZqJrS5PWa9Vto5lY5b2j/8CqnrqzINBgzAw61ECAGGB5mnWYzYDprzaUc3qlJd8ToRsw3MtM81IZZ+9AfYpeGBmvRB5ZsFLjM4HqYKbGhuPdYKFPuS1aIem/UTkyP7PHDM1CE1lnW9mCe0W43jynAZa85msyd1XNJa07fo4Qw7HjDNNBQX5Lpzq45JnyD6zee2I1Wcm9oLPh0OC9VZHnBnYCoakHEO5AzluPQI5+6j7P1rQM/dvDYiMsh8gJ7nH/wwQZmBZLEM9kEDPzmGZrQBSnGWvEAATwc28o9VqVOojXN1msw4DR/2g0CCG2ZhqYfnTr0OKyGb85llr61zKq8coIZsUClPTP30gMA/QueVut1+8JnN1yqvPNdkfoAAQdbufjTZ1BdOJQ2HqlNY68NgB+dcprfEW+LtaBuVOCBEPrjxAwBU/q1gOhsAUhmMbK2ZI8aGwyWow3vc7+9SwZc60sMCTufwVYEfRfP2jgIwBSMNnkRgHhZ4KxHw3qQAZ9bmCAzSB7PxG1YMEgQb0kB4N5eJmCSBFhgOog+gf4iL86EDSUAs/gln8EKLD4ogjDAtNxPB0JnelkatZH1pYbs0kfZ7RAu/5EB9aSQaeRuIDz2+GlvNLoSz+CAEu9I0uxAu9CaDvRH1ogfQhz+CXYl1oEgz+CW9AgPPekfWgSz+CA0efwQJKgYJH+7TO/JI0DySD94j6wQMgDbI+GjSQH4J+0BAZn+7S70gPDNJAs28kDMlgy8NzNI+rw0J/if8aBYb/aJd6QHvRApC6DTge1TMaM2zg2B7EQH4+n0B51QOSD5hptv6KkQ3nI2DjeGfzpo0gQJEmv7xHn8FId6PZpBhig3pdmHcirPZ/BACAMMkZ/h/8Sh6WxGmUYK0HP8AUXw0HWkH8fnQLvRoHtM3TMO/sR4aLvidivARpZ7cEB9CQdOmoCS7EuxJA6HWkmt4JB0HqKAgR96SQeQ1YDNs/EbR5/BB/dojD3bnWjJ9Dfn8iSR4YpBv8NGpAbm9s0QB7tCln8EBJrDNGl3oAA8NmmjPf0IDZz9ol5UWMDRdBe0TR6Z+7/Olh4Wv8+CB0Ht+9SD6c9T8ir+3UTwH+G4glaJmPhnghBlxR9ZwD7zRG85kahA9+SWG/wBogzPyJ7/GgZPfmCa0v0KWGn3pf4EEHlm3vDwUumzKlRHdejVGZAd87L2CANjoOJ48Hj1MAD6Fkt0WiekVflEaCJO5aqgHnDA/1rrtq+kJw8uHNiozvU83+zy+/wCg15a0W/60BxgNZ3C3sivWrY97QNSpUqHJaf6HmVz2pejxToec6x6lMpsjsDWXB6JXrntj/QdwT4fyA8eH6F0q3vSKuqmxwbr9KjVUGOt5kNE/4FGFLsKlxssl0GOVZqofxrQUT0iqUc/kbqocmj/O90K4tLjZYF2tBhOZgSP7PUfBP8i0s+1bVuqGDjkFl4Ox6OYGCuEsvWLJ4M8SJHPOMQwef/pETwTXP7k9GOpQwN+zq4E+P2MyMM/1rTT/AEdWGXTl2jWZMM+wwPoVU9P402Bvl/zrHD32tvWg45cljXjaT7QVmgTAaP3wBmBqpA2D6F6Qo/pD0oMKddVOOBI8j0ZWcm2+CfEhrnm4jPMf2uJmyapDzEEMDS5PBdrrHo0v+3tG8Qe/cyw2frBc8uHh1xGtgM6rbLz0cOuRH3h/Agymj7xBy2/2iPnGz8BxzA+8EZgAdD4HmgAGUug0ebmKQbw1EBJoEpJ4LQTLM0aX65o1fptVjh7bSewNn8it5CiM/eJav6EHiaR6aQYfOChI2ZOZYKhqVSqNt1R2W+5zlKM8wD8FXebjI6jDYfnTUl5iZHNidTQMDUrM/aGgT2ObYlvMh5JAKQzJizGvuj4GarQjP00dCI2EmEZ7I73WCmxggshqRWNHPr2KhL0XA8iSDsSwcxVoAaXX7RzsQ9aQYD7w1ALDD3hpI828fDcA01n8FYkMvNs55sA95M1ImSbfmMA56mOHNDZmyew1EAEfLNn7RBEwAC1GwRmYGCJ7TZ8NINNA11owQZ/BSGT0R09MDzQNYBn8iWth7NfDeADMDinn5wNNZ4FvViQB/iJH+I3+tJnQP2iR6meGpsUA9+l4aDmfkNGGwUYHigiRqxEeLQcB4HfIYJ095J4zYMv2UM0z1nnsQPADeXiJGfi+Hml+dIMDPerBJGCew+KiSZJwzADYM8wzzBA6CQKOFVpx+0fOMfzsqQybEk/AlsmH1qAaDxA+hI9QD8RvBLeCsHkaSDP92aEzbBAXWl/dpAbYHv6E8eHu+hAz4nvEOeadSMG89RGFkZ5kgRme/wBmaaNEEiX0OhfABv3iKLD4oQTrwbPAcA01/eKgksPil4Zmi/u1IFBh8Uf92l/eNgqZAADQKQmu9FuwHSqr+Ag5Css/0QFcBDnH0XB/Ak9GrgYcjWQ+fNc7tU/LVnH9h/gUc2a+H9FNaDn7xjAAAxDeSCq3P7+jRj/OpGY0aiHt2DBfDecjBqGBmtAcms56j9D/AI01z+H7dRzw+hQ5/DOesgPo60uZfNXHrKhn7SmvfkBLO3D/AOipKOhRZth4jj4JoJjBn1q4OHavfEk/oQHSrRBrU03voQUT0wAPTzUd6Z+G4tB6hth7xIMEA+d1R5NBtIzHnqwYH8i1GcekmoJ6j3tHFtgtWyj9nXHnj8gGnQsmjn+yTjZD96eCZ8Odzd6lcyftwD60bNpS3vdrpb1HsulYc9WIAO+TWWZ/ld4c0qV4bbzwB5AV4aq1nhE/M9pVY2/5Fe0r0eIrLWmdReP/AH+tUNS9JlhnH7I2xm753gzWYrHGa/63COVLrhs5+6jhgAKLtbtFN4S2BQcOefDVDsN5NVi8OElmO6jkvWMPctbzXnGBPnXC6evOmST7z1kghtvTeUbg5/Oax2O11L0maHviWjaRvPdjrzOC5jdvH6+Kw1yL9VOBIf8AcxwwBKMZ23MP1dQwkx/njYKPMh02pP678Fln6E2MrMjXHPhgb845MiUeHWrWlWBd0bwJcpkGu881cbIccG8NgI4Z1WZCOo00zjNB74wTYpPULECKcHUMwPr+dS6VSnwYzowfdz6EEwKjMlakusyXs+vsWjt6BOkgEGDKBkA7z602M+cOqmYa/OBgfepsmmtvRdQ50950+zPYpFSebDVYfqrx4d5moQHVWXWgYpsmY0fvgBXhqjhRK/AaabYcA8z7zWo+yVwSaWb86mvPGfepUmmy40Xmp0R5kA7z6FXw7tiTIR0Zh+YBv97J5mouBVM8N6qf9B/+/JPWHUoDRy6lTTZaDvM01PkymWDYgzqqEg+gzPNKic3MaNisP1J6IwGZhngowI+dAAP2oM/rUqiM02pT2ogMZyzPAHj2ArAJ/B0D05zEzV8hmZq4C6uHINfzbOmRvnjw9/616lbVugxKI6ESc+zJd78D6EEa0rcqTQSjggAH0GbyhcgdSYdrNNpTzzvnqZ4Gf5FWU227jr3MMXHWYzMeKGYQY4bM0G9hhSrej82xKjaIdee9ZULqpVer0iXFPwgDPNU9KjWybDoVKhvHoHgGfQaT1EtityM2HPVsJgwN4APrDyK1LKq3DBZxY1+cM/6PHDeoQPRalHB/Qk01r696uLb4kW5bA+polnRof4LvtjM1a1i4YNSa05TEYz+hEXbmtSZqr0h1husPPNbMMHuhVkmZc7LrTFAqLMzQz1uYezDf/ua3dNo9RuGO6+EuHGwPDAwwSt6NUqDceo3TY0l0GTzDoRG3Pw4dMV6O7Ouevw6a0HWEeNv/AFqygcK7YjeBbj5872SDDM11ipcN6jPdkS38KUD+89+YZpqiW9FoJg+3dVNz6DkfJ9CNGCC3p3O8p68mfWcYAVxSjg02ttMOUN56QZ4cxh0fQutz6lY8wWuarLL2geYYAmecsoJnIuaIGexkHTwMzXnaM5MuH19hblDoD3Mf2h3eAKtk2TcZ4ac6B+TYr2vXaxb0w4NHocA3Q6MJILHwLzuc4ToBUaaASjzNrDemxdz7Voc8GnDfeekMBhzGezNZGpU2sTKpoN1KGEjoZze7P0LQQ6PxCr0UIsWgfdzPM3ugDWugcHD0v5xuo4bp9kcAz/WvRz9mZ6hPkazeO9j2x9gfnVhSr5l1upcpb/D31q72VEw2GuoQOGlgU3B9yjs1KQHvZe9aVl5hkdNiLGjNeRkEGCo9B4jVJ8wqtsUSlB+MfX+RXocH7R1+ar86ZUt/QZ4AtAclxU8y52GXTY1OhQLuBGtG3g06HQ4EM/ODIZ/rUg6rrbzzM/OsJJuGU9ngx+hR2alOnj4j54K0NRUqw2yZm4/+hVT1bN7oz+s1TmyeftDMzUrlvCa+tBIOS+Y4JoI3i6malnD3J0IyLNAy2ZhptqQDLnYpDMb8PefyJ02XGd859mMHnM1ARhsTSr59yU3knX6M/wA/LYPAI/QBqkZjcSq20ZnUvUjR9ARA3/rWQ1r0+jUcgOs1WMzn0B51Xyb8pxyhYt+jHM/6waz8bhWwEgJc6qyZ7ofjK+Zs+Kz2LSABvVGqug/LqQA0fZ0YJ6MdGo54MS4efnwzTv2Vpv8AWadZtuIyemDAfnWYT11RA/6RD9CjncjcksGJef0AClz/ALMW80Eus1KmxmvnWEr3HW1YYcratHOqmf8ASPYgH+ug3bLz73tGz/QqGvcRbOt7w51VN6R2MxwzNcard53bcPgVKqmEc+uPH2AqLlmALw2AXR4G6uTjNeNYPTthgKPE+fe8sO9GfnunOqss5Mg+szNSw2B0IM/3a1wyCAYe7RYfFPIEAZuYp7xDHxEyl1+GtVDz+CPxMgcBww+hBhuwTujh4bikLxD94kAbtRPd6WGCCOy9+Iwadz+CdZBgy0338PIjhnBjZuS2zM+wFYaw+KLR2eGj6zz0+s+xLrPBQgHQlnmhPYRpkHnDD2Bh9aA8PijAMfeJIutFhAE7hvwBACeDUNACXbppd6DP4Kwj1PZoUv7xEgR70uhJJHlj6zQeGAoehLrBHpd2mlsQb0g9qAanWgNI+rw0sAz9ogz+CIGfQg6EaSBnPekkYb0jMABAaBLyeH1pIHs/gmexLP4I1QDvSz+CWfwSRkPPbgl2IM/gkjUee9B26iXWexCZ9H1oCSQ4fFOgGt4aAO8/ESBLDbml2KQIA2BH4nX2L54gdbaEz8UMG88074ZmGm4qCM8Ug3pSQfAD09jvnQ5/BA6Bt4d+xIHge3g2YfWgZDrNtzPMOhPMngBt+dSst6AOrTRmiAM/kQLPYjAMyQI8NuaBGGHhgCBIzwPTRggDs+dGHY58iAwSDo1NmxAaBGDwGObfQjAMwPT60QWfnb7EGfwQAZ4abjmaHfkjI6ZoIv4hpB+H/wASQKmozSDUQGkge0febMEHX4baAz3ow/d9SBBsJI96XdppIF3pIM/gjNAuxB4h7HE92IPEDxFkEZpB+H/xJdiSsH2JZGg7EYICzzHehS1s80GfwR5AP+HrBGeoB6bnQkeniKXYj0eGZafYmQZ3aZozMwPUBFrMfnUAdn50efwQYZlqdiWeBIDM0gePJBm4aQAGeBoJeeffuSzbA9PU3qJ911Tbz6EB7EEsz2+Ig1nNjfYo4GpDOn7xFjZzedw2AHnTR64eRAZ4b8Dw78EB6b3ht7DRCQBuY+IkYMdaaN7NofPggAzw+dHmy+6H7RsFcUS8LttV3Uti45kb9yZ5gqrw/eACRm37tsFkbdltv0qKlAa5G7bY1v8ArEQ8F2OzOKNo3+1hQKkzJd74jvtl40797YJ43oIewbMHfOCNHtKsWZb9SawfpTIH9C5lW/R7iRs51qzuQkB0AB7Fx+1eLt+We/4FRCfH/s84MwXULb9KK33vAuehvU10Ot5rez/gWS0SA9xpsx/QfYZrDXkz3mtdQeP0SM7yl206ZAP6D2LcW3eFs3nD9Y25VWalH78OtR69Y1DuHNudEzE+w1O0KmfROEnEh0NehwJgdkhrAD/gWNuf0V4r2D9j1g42n/R5xmf8aZuH0e36a6FSsSqz6bID3TJ7FCjXPxssZoG5zYVJpjvz3psYS4eD/Eq0g16rbjxx/wAWJ4wLK9HhmBgfkMF6Qo/pOWwZNN1yDMgOn2SFoKxD4UX4wEt9ilPeR0AADBbRY8o6OsCaZgOB1r0HW/RXg4nKtW5zZd7I8gzNcxr3B/ipQfHfo4TI/wCLEPMFexj+Wf8AdpoGXzDNxtSAelg7oSmDZdDrAwwUsN/Qgr3o2zefQexJkHPaaimvMuH4bij6OB6bfQgB5nbmlh8U8YHgbmnsRGfhDgrQhAG7BI9f2bbmxOgH4jaeANihaPo4JkOtS+gzzR6LBhm42rc6v/uAAEAZ56Zqw5b8NtNHGcAtRGqNrYFpudaPmvgjeZMz1E08y4CAusc3Eg2eH2JrBwEs/ggkbDQd6EDb/ERB0ZtoB/vEZhmHRmvgZ4aepsSDYPiKwgD8RCYZn4eadP8AD/4kHiASgRzMwDxE8yeY6gOJ3Ww9oAJrmWPw1YAzMDS1nP6kebHvEGw0BgDh+PsRZ/PvX3Rb/rQYOe7QSGXjwSN5xADL4B7NJ5nWD2mCB3mTe9oAfoUJ6lUYzzfghn5w2J0Ndnr6EefwQRPUkQP2Sqz2T+c1Ozc94gN5NazfvEE3P4IO9NG84A6n+NIJOHtEDqHDegOYx5N6DWzQSM9iEDzFABp0PoQIHt6HWiauBuYH5EjDdqIMG+9sPrRngfh+7SAMz9okbLGl4bmCQAhgRn7tDh8UXYjw2ajaIwj4YI8/glhgiVAT1EGfwTx6eem4gxBHhGYZoj00OHxQYZkiWr5CpB7w/wBaWFSZ/pzwKQcx8/eKJzEr5FztSZqVcZPUYnST/Og+0l2h0TjSB58PoVhGo9VmdFNPA+9BX/au8f7caX2nu7375mtRAsOon48owBrvV9Gs+hh3vSfyIMCF4XP+AyrOHXrte6IgLoTNpNsh90o2aizzt+iNa9crkCBh1gZ70GcZjXM97d+AH5FcBRHHg1NPNUtY4x2BRP8AQFKk1iR8/QsjW+NN/wA90HKGxTaOH4WAHmpaunPW3LjR9ecbMZr5zWUrdyWHbzuc6qxnnQ9zhma4Jc9w8Q63N5657jmcuHY1sBNW8dNem6lVYM4nnPvVLdNn8e6czI/5P0eAGHvXjz/gWKuG57tvx0336wEZr8JnYCtXjgnMBi2LVhn/ANYBn/IrA6PUQ8e4Dhxs/Ie/9CyUxtEs9g54P3BUtan/AI2ea0Vbs+3HoGpR3NFrzvd6euSpRHtJijy2YAMBh0dap4dNPS9Yzqrz5+TsXm0YU9HkzqCR/wAzc5H8iQVuLMqgHOob3zxw2AC0DNbMA029gKDMqUUz3xTe+hNtsNHDrFvyWuVNgI2YdYAsfVa9KpRG/Tjh7D2GZ55qz5aCfYH60o1KowSgPkWTd7DPsUWi4KHdtYr0P79nn0dGAKPMCqg6bcSlSXg8+Csgjct0AH5wzQz7qrgDyrD4JBhnw+05n/oow+tWEaTUtjEvYfkVhJrzB0s4P9IPrNXFgWx9rQkNuVLk6eB4PGHWYL104ZGfzzz5nTg8LsWgpUNt6k6j7hg6314K4vapWHZ/LxbfpwPSGDzeMz2fkWNqV4Vm82gpUSLGgQgPPwg6/rRjcHTpUFlp2Ux4xh3u78FYUeZc0Zpo9f8Am8OxJ6pOHF9WttshH8gAgCe/jpsMA8fzq0YWD1wyp+DFSpsl6F+C891qt5Z+lSDrEShnGa7HjNSIDNSeI5dRlMgB9GezBWVStWs1iLHlyq5/N7AYAGaLVTNzznnwf0GQw8iketX5Lpvvv4AfyJRqbRoYaD9VA/nZTs+A5TeXnQZYYMYaOCCRMuGmhgbEFkMAw3sqPyDlV+/N8sH8Cj16fcd1Q/vbHMgx2Ms4dajw7SnV7Sbqr8mHHbDYzh1oHpLJzDwOovfrTNNNi1YTug/JPXPM+9X0m2LRgRc26rJjSA+fNTbYqtqnbh0Y6ODzphgciR1mfnXOM0zP9ZNa7Dc8/PvwWitu3qNWI5wQtWSy6HQ9nmCuI1q1IKcDEE2WQPzrR007jo9LCCYAy13yADAFpAw58KHAkG/XDNkPk7Fdw+Gll7NOdM/OadqVe5yYVDg1wDkdmfQf0KbGe5algb7BnIDrwBaItfQLSYjYN0qJm0rWTw6tKSevVZejN/cycMFy+pXgxMnhBl1yTDjt9jOeZqqetK6rwrYVK3G6rJhdjxs4ZrnXDS3t68purFgvzKk1+K7vWfZuGmw6Hyj9KjBUA710aybAuOiR9O45wG12RwDo/OtUdpWi9vft+Ma0iy+ZwKm1iuTGmgbo0aTNPoOPv/gWoe4S8Sa3NB+qxKaz5zdPeH5F2VmNBh7IMGNGa8jQI3p7Eb25rRG3N4HA2Cy+D9ZzmH8jy0sCj2zarv3SxzZw/pDIZmrN65IjPeoJ3VrfsoGH1qDaw9fMTPYawfI71qO9Jih7d9Yq9qrOC5afyL+HNM70YBLe9u/mj1dPXJBZ2N71Tya9LekA23mGaQU35FICA3+GiAgD7wb3DQcm2ZHm2atWQAN6MzzDTBWKo6bm1qNp2BGbAdNSPHwwTtKpUvQN9/YDh9bqhZo4bZ9Cmgy5pBsTT1w2rR8GJ0rmZB9DUfrVfJvC45jWhb9KCHH88g0Q0fq0wj68t8GQ+dVj1yUOAeAPnMd/BZBU4UeLMwfuCuSZjvkz2K1jTLfpTWhBggCtatmT+I1bmg5R22behecwzeTrNmZuhKrlck1V0PxlZ+uwMPDbTJzJXYxmoFjDgUqB7CCGalHJzLUVZGCpPD0YKFWLhoFBz9eXHGZMOwDzP9C8wL3WPNSMHD7wD61yKt8bGD8C3Kdn/wBYkf6i55VbkuOvP51WsyTa/CA8AXo7ncPE6zre8P1kcyR+EyzmsFcnGa6q3HNi3IvqqOfeftlzkGQZ9mCd1nP6l0dIGTzc8tSqznph9+qjAGAHw2AToAlsBMeEGgPP3Z7O9O4b0ZycDzbbPNMsn+P+sAVg0s3Da1DAATWfwRM4PBn2KwYb97aDE0ejv8NSA1MdPBQgAB+IkeGSezwJBsAPZ5oqDoJpAng6dRHoDPFHnmGDnegwz9okZhmgMPah0JH0JAf7tBm5irBhJcwPT2bMEs/gmetGAHmgNLDM/ICWjm74jgJYgoC8P3adwwPxAQAH4aWtvVhdCPAwDe2gPs8uCQahlptoF1mkhS3oC60YajLqQfh/8SDvQCf4n/Gl3pOdXyJZ/BAjNAjD59iWH/q+RAB6hpAjQIgsPil3pYbc9T8iSAzQID60aBGgwzSM0YBs+hUyIHnMAbcPYCWxB3pYfFGpH1pJfu0jBGRfnBLu1EsPilsUtSNnb4iR+y9okhPYHiKguv2iAMDI8Ow8EfYlhggdee1jz096ADc94hA3M/kSzPP2aB0Pw/8AiSZA3gzb8maDNsPEc6Es93hoBz+CWe9BvyR9AmgWbhml0IA1MdRI3m2fElvsg12Zqdg2Tw2J3P4JoDA2jfAM9mzDvQQ9TSPUivRu8M+9FprPjNeJs+tDgGkHnQYZjmbiD/g6zUIPAeYpawZgxmGYdCWjgAH2GkYNmeoYb1Y+H1ow1EsPih/PgixdaHvSPPJJECw2JBqe0Sz+CWs5+RAsPijNnwtTsQfvO3yIc8wBxtvA0BGAMkGmYHsRgeSa6zzRZ5hmgHPdgiBPcyDMV1g2APPvw3go/YqBZ4FsBJCnUAd6Xel+8SBSB3gizDDD50njcMk0AZkgdBHsQh0L4G8UBmgSPrSQGln8EJ9C+Bpn9aAw0zDU00YaaANgaaWeZIEe/NJLvSP8P/iQIz92gDDPxEZgeaWHyb1APz/Wgw+KAwRoyB0AmjNz86kMng6Dh78E0enr5tq2pGYe0fDrDsRHpn0IXjzwSANiLF1sBgjZ/f8A600f7tF4mPiIHmT0c9/WoJnyxgeeGZ4J49RMmebWmbahBGGB+0TzMxxnNvU2Gome/wBmneWbNrUVrHm4fidiazwRGDgHp9iEA1vDUIEZ5tagOb0IPBinWYeffgly2BezRYAPMkZst96DDDsR5v4HnvBQAhvTqU7zdHnPQJHY6z1rfUT0iuK9EkNc9OCsRw6wlhvP86wJnkh/WsbgehaD6WMV50wuC1ZLIdhtHn/kXaKPUrVvClhVaa+zMjv94GvCSdpVVuegzQnW5WZkB3v5d7DNRgeyLh4UWJcLTrEtjAz6Dw3guY170aQORr2rXHoAdmD2azVB9JC/4EdoLgocOsNN9bzIYPLrFsccuHNyMA2/WfVU3vjzgww/OtBzJm6vSX4XZxTpQV6EHRIBnP8AyK4oPpbymcIt22UDMjzgeC7rmwB73wP52TzBZK4eFdgXO67KnRfFPvXOAo928JOMEB1iq0eAEs8PClmAH+Q81mrh9GOjPNGdo3HJhu+SQYPAsvcPou0p7OVQ6zJjSOwHt6hRqx6QPCJrXluBcNKDz7zD6Ni0ixn6xwZ4qW9mb9DOpRw97E3gsozUmI2yW2YO+QwXYIfpexA/0xaMxmR8hrS85YfG+nNG/WIbLr/uTDCT/vsW2x599axPxUWbZrvNV9Fq2JMcAodcmQJH77xgNc0r3ATiVbZ58oFVj+eJ1/oV7GXB5sGtNxRzeDPYohvYZxJzBxnmz3gYKQyywYbDQSMAew7EjDR8M8PyKE9Db8/60uQbB3w1aE3NgwPDsTWfwUfDAtRB9xMvHMwNQDN5JmSAH4iAAi974HmgNkPdqwnjiPZt6eCazbDw3EgBgy0zM0ZhgfyKHmzL2DIaiEPGazBzBE9oGGBt5pMg2YbHAz8iPQg84GbeuBqRnmPtFHMDzTJg5+OYKxODYSdM+xxQgzMfaZ/Wl/OXY4H50DviAWDjmYJp6GeWo2/s8iIHpfvGAP6EucDHDQMFYENTvQHvLTz3qWy9FM8DcwP507ycQxzMN6CIDx5afepGtgIZ96T0Nx4fDNR/Vr+OpqZ/IgsA+tAfWhZ8iR55IHQz943mCXQXs00bx5fIg1nP6kEgwDNNGy3mg35J4EDJh+H0IzBvSBvTQHqB7xHn8EDXLNmSA4zfzqWH/D7zBLr8TBBHDPJO5uB7xGYAfs0GH4iADeMOttIP4Eeij9jmHnQM4eQzTwan4iZBPdiAvE94ln+Gmsz945mlvyQO5/iJdfs0O/8AIlkaBYOH4iXQiTW9GQ9+KSW9JE4dIZs+pe/bwVgzaX7hU8njlQ9c2KPR5NV8hgGw1WVLi7xCmNaFNtwIDXzrnU3sa2H429tuMAed1HPqVAojWvXLnhh8jO8158ue6r0n+HXLxejAf9HZe2KnCSwft52agdjr3FoJPgWPStZ4OuXI2ACzh3bxGeH79xNjMu/hRw2AsVG9Rn7Rh54/kRvG/G8SDTgjfOYLV0LKq/yjXORthxikgAfjBgqf+SWjyXebuPiacmb34GlUqCcyGDnNPZn171n2dCBUQYcWTzDZyYEHk+V5oHvIYJqlQ9HD5Fn5nNaunR4JvSOzBXDMCcEDXqpmy9+EBrza8LJ451S/YYIYMbzN1VTwBPw5pgAw7wUdmsP48jTaO9pB74zRsz+ckcoxFPNRt61FvMzoYOuUqq+rWj6z71X1KHhNNxx954z7zQRplYge4DD51CmTJ0yRqaa9ebABxYHiOMMn9YKaFyPnH0GGAw+QFTzIBzDBxwHgAO9XcONTThBFlVUIwAeeDIb0WjxqDWKlF5sIIaWfRngjOBF0pfgMs6HXgfWqefDnGQcpVTZPPAAz2YK1kwJzYBENzM8MzMN6yWrKVR4oQzqM6LJAw968fQlMmMaWhRpZnIPoUsHmzaClSopvMn14IJPI02aHqqlSTMO9A0zTajGgG/WGKkZ+fPrUimw6bUoAS34MwPkM1LjVWXJqguP6x/QCuK3Dt8z5GK5M53vAzwWqGch1ujRj5WVb4PAHQYJmsTHzHnqdTeWhfQtHDokHS0zb8UEgwCZyINgZ/OCLi3N6rWKzXs6NRoujEPrPDrVtaVHr9HiyzqUXwgZ2Pdi2Fbo9SksOwYNK0Xf7Q11gmQOuQ4fq6oznj84Ggz8CewAmD75nvX2TJfOZyMVh7VPv8i0sajxXmtcKlGZ8+YdCdjURyS676unA8YdZvbANZG0e26Cw9K+91HA/3ys9bRM4rZ7EDPDq5694j8tlkPwQBXcPhLdQH/NwQADzunmZrSEMjJoObRv6kkP7oFCCHVYD4U2lRfvHWGZrcXVY1+U2GDdVqrOk/wC6iBvNZqHR36IHNm49zAefsWgD1PxRANT7nDa73sw2IKxTX6a0D867jkyDDM8A6E1XpL9yNNRK/cB8uB54AHWmp8am02l6EFsHvPmGZoNVb3C6dWI4Vmoy43qzvdA95/QrCYzblvaR2xbDM9qKfjPSD3mfkVbTWeKFYp0egWrRpjNPYDAzkZsgrqj+jrUjwl1y6ghmHuY5oKqfxIqtb5fnqdyYRTzzae/2FWVipV+8GvVtOcqtSw6I8dnNduoPDGyqCAOBEOpSA99LPNapnQhh9xgw4bXkjsgAIycKtvhFfb0LQfozNEA/6RI3mH5Fs6VwochtcrUrqemR+9pnZmuhPVLR6zAFXyaqwHvwQDAt63KPH0KbQ4zPnPrM1YcyeHYs5MupiN3qkmXO+97B8w+hBsHqq2HvAVFVb2gwJGhqAbvkBZwzlvD7RQv+Gm60oDf60Gjeup+SGmw2YKkM6lJd1HHzNWZxgjNG431gCgwHgMM3Os1AkGy5ypmrCHG8IE0DLhqwAMUGfuSNKC7aO/oZiEYwV3DZbw1FWVuTncFPccc7DDBWrMaWYez0Q+dFj2Ams8y2IJlw2VR9Jiq3AzzB9bLO81VM3nKkzcLYoZhH/t0jo/QgvTgSnvIH1mqet3baNtsG5OrLJusdbLO801PgVmsO6lZqT0lryZ4J2m23ToefKUYN/ee80AT7/fqscINh0OTDmmAGciczgAIQptzzP/aC4zmfIzsBXXLP56emAKWzDD8QFYpAoLDIhpxGQPzqQFKfDyK+Zhgfv2f1p14INNa5uqzmY0cOt4zQZ/1a4ZqQFBbPe4qKq8aeHNHPCDKOsO+SP0frXN7k4zXdW2DiUoPVTJ+TAz/WtR2qT9n6Ixr1WdGjNed41jarxjtGA7oUNh6qn+52AuLmbk/xKk+8875zMzRMssM9AINRXuJ18Vtp2Iw4cCIfWEfr/WsuEP8AEN4/ndPM1LZ3pHsRia8DBF4eOohPDJIzDZpuK8LO4BpZqOkAOYZ+dLwwD51bzZ3NsEs2D2G4Zn5AQsnn4mmi0WAdN9HpYfho2QcMcOxHJD7uegiCZm0Go4YHh3qA0cY8Dw/+wRxmTZHxHze+tEBn7NH2IgkkeHxQIDQI+xB1osu9HrN/1IMPilh8UCzbNLwAw1Ef7tABtvH4aIH2IP7vrTvh46bfWhDYHiN70WSLoPND/wAL2GHh96WeasEhQGen+tGG/oQEZ7/DTRhu/Ij8Pyb0gDegIzz6G8AQshoibgOGl3outAkObgIsM/D1MEsMA+dAv7xIzSSDoNAKDDejQH0eGeCA+gzbSQZGlhvRBJ14O8Og010CCWfwVBZ4ezbS60v7xtJe4AH1pJAkALwI88M3PoS/Olm4HhpBvQCiQn1h8iJAjRh1po0XZ7ToQGfT4aBIEYIAzcPyfImjDPrbzR/8PWC+dZoPvidaWfwRHpgJ4AhDfggLMMfZpAl26iQGgHP4JdfvEjBBhn7RAB782AMM0MM3Aj/ewwMDMEWjFMjc0N+eGaE9mYP4YIEZmBm4Bhh5ET0WDJaByU2B57AUdmMwBmAN9Zo+QDVAwcPYeeGexS8hLZ0waBhhvAA7ERvZutBpnv7/ACIQwxS0dH2bhn9aPRPSWIwA2+eCPPD3ftEgDJGZ4EiyM8D00gPemTPxddz86eA2zaBzPYaM8C8T3aE/w/8AiXwDb94hMMHc+xGgzNLt1EvD92i7EA4ZpaLhikB4CbnevmbnnRnsITN2hoGZmpB6eAbN4H1ppFn8EMBM9o/Ii70jS7EZjCSDIG24wB5h3oD5vAHGGM8zwBCe/wB2CdB7D2fX51AB4H4x6b7eDobDBLP4JPA+947m/M95ozwD2AYAragQ5/BJFsyD51QaM8z1EYIv+DoNCAZ9CkOgkYeF4fQgSNQBM3M8zSz+CLekYLUD2Ig60vE0vkBCCkO94OdiBCH4f/Eiz+CAwPeCPPefhpnP4JZ/BAjPejz+CWeY6aAECM0v3mohSAPFwc6MM0BGaFF2IQM2fEQEZgZdefzr6BtnsTJ/u0GBm7s2YI8hLwbZ94gzY923vSA2zQ4OAaPTuzE/OCWDZgB+fsQM7/D7090AgWGzTw2IA08jYDNH2JKAz/eI8MPZpJYOYqw09qMj4aLmXAwzBHhmGGogNnMFARnmeCaCNh/Ssw8hoDDFWAM/dfPmgEAz2bMPOCXIbz00mWWw8PoBO57vDQRDZAEvHMPDBS3g1izQA9gOnqb0EcObANM96aNkHi8cM1L60jDM8EwLC3rzu22HQfpVfmADf9HM82T/ACLrFt+ks29IBi9aHo59cuCGwP8AsVxcGcEnozZrmuFvZFEu23LqhhLtyuQ5/wAjJ71cAy294csM2u8F4ZjQ3IcgH4LhxnfOyeBrfWxxpv8AthoIM7CtwvwpZ7/1pgd4uHgzY9zuuvzqUGqfkDBcfur0e36VIN+gSpOzoDyLa2rxvsSsTY4TnJlBkeR32IfnXUIExiqta9OqUaez52jzSB5no97cYuGJgw5zkmmMe5l+MC6LbfpOUOT4Fx0eTDdDrMAzXWAokQw8RtZK5+CFpV7x3IPJyPxWlIvf/tbcQqcD8tilVID73WQzWCr3ozW5M+92rUTpph0NdYLD170frqprrtStKq85odDWtgf+NBROM3Enh06ES46NzjQdfMAef+NBCuHghflEYecbg+smQ7GutYQz5N04k5t6M6HY6GBr0xaXpG8Pbkf0Ki5Jo7v/AFjDD9a30yiWPfjGvOptKqrX4vWf61rFjxVqMf1JMgwHtG13+5/RXocyZzdq3BJpRn7p0NYFyW8OD/E2zNV+dQ3p8JjrkRMDBNjOPU2m8lzVOnBn3sunvVfn8FCemMA/oS82XfI9sUjDNWHQNzPUbAz+gEZvMH7Nt4DDrzBRAqtVoMznqVL0Xfo61Ln3JUa2DTlSBk3Q7wDeiAHJbAN7gAgAwwUKZJYw8cDMEo0ljHYexWJpmmjPemvWVOPw89/f8iIOgDb6EBg82jzbBAewUGtmmBLDU8mxHn8FBA8M8HEbJ5l14IJABrH86kcyYDpuGoTx6Lo96IzAy8NBOA8kjCWfs3FTHAfed1GKkbJ/WvuFcjf9JAasWv8Axn+Ggz+Cr9auG1nps5/WpcbXPw3DA/nBBIM/0JeHjsbwUfNz2Zt4J5l7DPw80DpgADnmCADQHvS7ED3X7xFgBjpm4ovenev2aA8HGfDNJMmZpAZoHvE95sTP50ZvGftE13oJAG3npoD1O8EAdaNAfuUAaiX/AKw/4fkSA0BpdCDr9mkgSQJM6eeDnQkeAHsQGgSzM/dprxDJHOdA8UGetnswQgGHhttp3/Gi2UP15RGjcptZDV84Aru3qJfdyMA+/fEaNqdmCr2WX3nQYbbzzVnJptShsa7beDQLhdDZQ7Mo1NYPnghz5f4xngnmQsOMIRarFjHh7lkMzXPI0l+TsbnPB9Cj1Kqv01/QcqUk/wA6NceHQ5lyUaju/wAzWqyHkeeBZGqmFSkc0+34pqtekuGHhy3nvrVec98Ot8wRGGoZh5003HJbOHY13qhA7cOVys7DmPnVeFencxoRG9/nMFYTGaVpR3DYM5Z7zwRbTUesQaJFkNhB1nXwwAMFWBR6lPlbG8M1EZMAdBxxaavXtKuFoKVBihTY+GB4df61Yjhw3vTDUCdAZZ85mpFNt6m29KCVXL4ojOHWGaylSjUrLCqzpgR+/wAY0Fq2xwrr1bagyp0/A+88wBc7Js696gu10JdDrgPBFDA9LvTVBpT9EdCq1XlmY/ebq0X8j9pUqOblq1IzkH5DWKCgxabIdCpVLmZHfv2LUNVisMTKvoMb4hoDjU4HWjptON6R2YKRAsBh4AqL9Vk5/wACs6bQa4er6qpT0kw7w7EWr/VtSAfHYgMn5OtLNxmLyoOYGadmM1WlSNCq0Cfn8gZgptBpVKueYER+DPZw3mfYAIo0zZnLQ+eqtwYZ9DLTO80YAxPaAGKaeDfefeuihaVuRpDTDjkk2j6zMOhBUrSsTV0PWL360NuNVKe+GbFO+7O54a3Wqqfb044/PTn6lJMPe59C7rAtKymZGg3m9n5zWlh8OusG4Icufus9iDzlatpMVh/fc8kPIy0eea0Z2ZdUYz9VMazXuTPrXeo3DGnRvEODy390ClhbzkYwNhsAwRk4ZDo/FDleVbg9ffmng4aXrMaB+o1WAEjvA12p6M+8e9gFF+ysHnebfquHyZqxzH+TfRYBysPxjPyNHngtnA5VmO0FKpUYDAMM8Fe1ilQTgHoPyXj7ABZ+3reuflcOaZjNeSR1qBcQAMHQccbwUKpXtFh5hBiPTHQ/C6P1qXJs+JUsAqMp4w8gKyptt0aiNAEGCGf4p7zQc/gXVxJrc0IgUpkPnMFFqvDHiVc8836k+yzHf6Hj7F105LgCgOZs61a2Kt7gbZ1HdCVUnAmPB1mGzNa2HbdnU2RrwaBD1fO8GZqFMrbEbvUILkbe6EZNVzJn7M1FemNgXiLGfaSpPTDY1MAUKTJlcwfjmidtlJr0GN4YHmfkVVJuScYabbGALPwwzdzcU3DNQ0NSZ86Tnm+aNmM5y+DhmaTwYJ1k/CDzqxCmMgAajhpkAz9mB4fOpdS0wimb7ewE1DeiHFNzPoBFijMnimtFznWnOxtIKkwCT0xxnFzAAz8650LCYbfLnh5FWQAAyBhxUVS4l2jGkcpzxzJHkiM5h+tR6rTb1uSY19n5bMBrsew3o1dDBk2faGAD85qvqtyU2lNahy2XvkaPNZSBwfMJrUu6rqqVYMPc62ALRwLJt+HIBxinAC1SyNYerl5uxH6BTno2iftgPDNTYfDS6pLGpXL4kh52s10kGdiWigwkPhcxDdzCsMn84Rsz/Wr2HaXLNaHrief1mC0bLO5SMPigz/2M1v8ApypfrRhZNNB0HH51Sew/6ygrfEuy7YDTnVkJMjsaj7zXPK36Qk6Y0H2Vtx4PO9IMFY6wzQaUz5/+2krNV6/+G1vR3eaqRyZAf0ePmZmuFVu6rxuqZzdwTntLsZZ2ACghDY/A/WtWLcXJxXCttAFKtwIzPneM8/0LCzIzlYa0KlLkvNeQ3tikaIB7RSA2IIMaMxGaBhgMABSAMwMGzbP8gI3tMB1D6EASXPd5/nWzzZ7M8P2QwPzmge0wMMN54JBrnilhn4mnvBDZH+76kbxmDXQmWWX3iPXzDyKQDPebmaPUX/s9/YnXgPZh1+TzqRgGGmjwDPUwUCIGx3Dv8idONn4h4fIAJ7D4pZ5irVsgAABElomYaiMAUJADwb8+tCYNnvSAG/w+9ECtZb0ebeKNB3qEDwzwSM8PDXzNz8i+9iAAPPY2lg4B6biDo6OtGb2fQrWR9Wmkg6zzSBAaMGW8DfBsAPDoQYf+v50g3l4nQiCNGB5IvD600fWioO7DQbM9QHM196CNFiChJYOYbADDNB0GjePb4aEOhWsGfwRhnnqdiQbxSRAjDbqJIM2/Zp4MNmogaM0uxIz3pIF2JIM/gl4meHeiySz2JYb0eGz5EAHvFLvS/wCDoNJACSEz3pZ/BUzwJCf7tI0sNiMyz3YIs8EYJo9i9tqLPZp9iNAYYdaQLwI/w/8AiSSSQAYN4aht7EYJZ4ChA9/s0ZPgG4Z7A6EMl59kA02M8zUg8w+tNG85qgxpmZn3o1LegZDDP68074hghAN6BeIBanzpZ9aAz3h/Gj70BazhgDHYHQosmMEzwHwPDNOmeBh9eCkGGB6fWp8COHst/Wl3ow09mn2JHsV2EZ7FBh6jzshx9zPfs+RS82zUSMGD8jxMzM814JBngJ6gYGB4JGYM5udYAhCSw8J/IadM4ulpud4IEzJYed023wM1IVeyEVnx9AAPzpPSeQYN+K/mHfv7FIs8/gmc3DUeNMbmYaDmwwTzMN9mZqetTea/B7FSzv8AedCP+7UTOVqmBgGkHQadWQPYl1mkbwAeHyZpAeKsGewUG9I9/vNmaM8w+tEFh8UAdaWeaXeixpZ/BLrQdZo5zvR7RvevrL2Gfh5r4Z7dPyJoANQEG/vSMNjRg5v7wSxBHn7vTVtS34BqIgPYhz24IEDvaDiAOtCAbtRHn7tQyIzSA8UB9aR9a1aj6EGeZJdp/wDF1oIclh7PXgvMuh50EjW6PkQ5uGaLwMEOG9AAG4BH/wAbaM/3aRpGH/BpajakAjDegSQGgz+CXejA8UCz+CID2IfDz9ogPrQGZpb0CQGgPP4JG82A6jjmCA+9xH4bw6D4ZgaANbBAEnAcABGemDvhtpIEGnvPDeadwPDUQhvRYbDQID0S9mjN5M55jpp4DbNoMFAMDb9n3ouwG+9R8HM9RHvQBv3/ACI+3UR/3iD+73grAHhh4biMHmwwbPvS/eOdCMwbM9QG9iAj1OhtNeOGxs0e9LvUBBvLxER9ajmyZ/IjDUwADPPBBIZNwOwE0HjF4jeBpAYJZ70AHnkjA0sgRPGDIZ6exA14nu0ebnvFa02HRqrm2xXIzLoH0Go9So79NmhEOWy8BhsdZPYi0QDxSzzJIGcMHDRhpogvAMtTQD86dpVbr9vT2qrblRegS2Og2ehNaP4aBk80HZrV9KKss5xL4ofORz/pEQMDBdNt7iXZVyYN0a4w5g+iPL8E/wCNeSnmdpqIEbM1jcLe5TDD9qYM/oTU+BTp8flNADA+sDXlK2+KPEq0mgiQaycmP3tSzMwXTbb9JCgT3Qg3bQzpTv8AaI+ZsqMC1uH0e7OqWbkRh6mmfeHQudSeFfEqyZnPWdWHpIB/YXsDXpCm1IKqxzdDqsafE87J5p14IJ+JOiaL3ebJr0cHonpJ8RrYIIN+UN6Th714DZNdTtv0geHlzvgx615CQ52SA2frVxPtW369H5WowWalHPsPYf61y25/Rjtzl87YqJ0qR+DI6P1qh0+6rMg3PTnX27VoNVdPoMwwP9a41/6Mdfkum3TZTNHP8KWesH61nJ9vcZuFcj7jOmPR+yRE8YFfWx6Ut1QGmol3U2NWA/GZAGT/ANtWMJcnCK/Lbm6dcocl6J/a47JmBrP3DaU63mo8rUZkwZQZhIDYvXdvcbOHN27ItZCNIDrjy9isrksOyrthcjWbfAw88fZ/Gg8PmzsS0In4a9DV70V4p77VuM4wf2eWBn/GuT3Jwl4jWlIP1zbDxxw65EQ9YFaGRCHEA9TQBHm2yGnp4YIznsGGGwD8iLkwe95mgj8y2fhuI9iMIzCLlg92jI0yAB2bAR7Mtjad0cCSMG/doI5pAYJ08MU1ogaNSzwLU7EGYH0J7fpaBuZgo/LaJZoJAPOAaLWby2bE0l/jQSzNs0Bm37tRwPdpowePJWJAb80s8x09maZyBIN6A0YGmT2b3EYHh4jb5oD1g+f86Xhn0L4b2fzr7+8QLE0Wbns0s2/eJZAgMOtIzQAbgIw3oA2IUR6aHsQJEYBj7dD3r4YNmGm4gNGo4Q2wz0HzA/JhsS+9h4bmCB3WwSA2zHZ1paPvM/yJYb0ZEbx/hgkybmCRm3l0b0YGgDmdFg34NKk6wKtCsXHPn6E6LJ0j7MFIMLxOQcqIx/GnaxeFZCl8jO/aO/euF0JHqHnPZywjfWqdm2KVMmHKqr4PaB4AALFTKwZu+POMPoV7AuS3IcUGObwWCttgzDpoNeHEwaD51WaM57xIMUHg+dcvqsy6nq4cqgBJkx/J1rpdhz+IUyEDB2w8yAdbpxti8W0E+gwaVQ2vupvVB8M9nYskASoYfe4p6vkBdSo9k1Ktmc6szgjecwDM/wDGoteCj2w+LFKqpyZvzrSLS54cOuaQSn4PJtH+Ke9RMHAkAbjma9HRreocyBHcqUHWkYZnvWfvC0qlPAINuU2NGjh359a0HIY0+VGLwKNzK2FKtiuV6GFRctjCP59Fa6xvU9gR3W7qqNEOQ+eYaxq1n8ZrcN/1VSmweM/7J0LJ5tRfZK458Am+ReZA/OpFN4dUqNFByU3nL71oo1yStIJcqmyQa+hNVWZc1YhHFoFvzAaP+kGG9EbVTNn3BJdCCxUoECn/ANnAMzV7DtWpQ2sKNK2d54daiUeG/Z8IJ1Z5k3e/PehrfFSnUpoHIvMh8mitWi1ZjVjLTly96aeoPLVHX56Sf+ooUO8KrWGgl0phk8/kTMyNelYz5RxlmQfRmewFbxZvSYkl31ODZnIPsAFZBGtiHA0JdOZP5z7Fn4djXizManVK540lpv3LOzNaD1CEn9qUMmXe9TPSOaobfJugftTVtGu2qwMNSqvGHyK6ZoNKjDsiApARmGfZsM/oR5tH+1VwVKFrxJcnPs2IYc+5pLWhOcMA+cOtTeZMPaGlzjfe+CtEWAIzYNYJBAghv0N6jya3BjHp66pZ92tgWmwBgrbYav2I6gJkHmD63AWK+086Tnp5qP67nGW9w/0KEYbeTVadGDUN8FAO5GPd71lMAMtQ07ye3w0ep0m4ai9n2KCEyU97Rw80JsuJMs7UVAJIOGCajeDvwzTrwHn4adZ2CoekyDfMG5p9aE/aoM9xuJo5IAXjuACt5hIA8U6Dyp41SYelaDef1pqpVWDGaM3JwAah6unnmz94no08A9m3msId1RTd5SnNyZj3yKwptNviZg5ygQIh9eZ70FrfNV/5LzmG8NU+wzwXJ6bdtVOedOb1sP3QZga6wFgUZ6aEqsuSZPnDPrWmgQKNR2AYo1KZZw+RHRFuUs0HiTcIaEWJ6qj/ANoePf8AoV3A4RU4HdS6rqqVSd/CzwBdFN6W8e9Lk8/EXmGNqqm0e1aOIBQKHGZw79HerMNcz6FKCN8ie0QD27mAeczV4No2i5/WpIagDp6aoZPEKw4Eg2Jd1RgMOxYqvekDSg8C2aUc8/xT2LbBt1XTc9npqqqV1W/RBP1rVYzLodi4FWL54oXJslXG9GifgxAwBUgUps/ElPvPSPPrYKLhk6rW+PcECODbkV6Sf0Yf5FzqvXhet1bKzVXga/CZ2AogRsPZo9Fz+tXgR40NiGQOA4AfOfWpEZlgPZtoDZUhkMBWoXgAJ+H1oetF4fvEjNsOxehrBw/dpb3u8NidP8T/AI0eYAOm4gaNLRbz1NRFh8UjDDwNP86KsYBsSPoRgBoA3koSDPeYeRH0eI4aNBn7s96tYzP8NI9T3beaDDD6EeGxEAzPoMEYH+tEHWfzpd6gHiaaz3p3MwIEsGzNA10J3NvJIN7qMDBl0222w6+9WFnglnn7tLZkg35KAj2CkB7Q8DD5EHiZaaeDUw3v5oGc/ghwDPUUjD4pnvVrIAR/vEZs4IHnsA8Q0B+GG9zrSzc37PnNBh4u9GiAAefiN70ff7NBngWmjz3o8gvEz2OYIm+n50HWaHoPYj06YYoUs3NLBw88Eu0NNFQSLWwazwzPyIdmPeZoOhHoAM9mo2j8T3iPWcy8NBm5nqGjPZBvAHEt+SQHvQPHgj0v7tJIEg60B9ZpJGgDUNAg60kvMl4ePz96AT6w+RJEmjQGeo8YNt7NNLP4JYfFIECSPp00j0w8RLxPeKggBJLsRIEYJZGhBLP4IyAZv/ifwIwPAkHejwcBA7q/rTXiG0THn70ksPkRqBlnlg09Qz+c0efwSQa2BIEe8tQ0YAkCIzwDBAJgkHQmc3Dx1HM8OhGB7/E6EBn+7XwzzDTQsnm1qdCDBx4tTXMAUgwDFNPQ4knVBzPYGexOnp+7czUTls5RuahgoBsw4kPPADMzDfvSBliTFOJgYdiMA+7n4h7D70mT2KwQajMM2H3ANrvMwVaFSpTxnFcYBWGYTGjbMD6+9RPU9KMgcciHs7M1QmnysBoAbDAOhSDPowPsTRs5tafQgZjaPRngpWkAbmGm5h+RIz26aQd6A+tA8GmYJBswNLt8PyJpl5sy0z6/IkAswAtM08B5hgmXujTbbD60gRgM0jP92jMEyf0ZoujwefU3IASSz3rVmMDby8RF4elm2o4YZ+Ij6Fk1Gy9olqIjNt42jbYw86EAbP2jiRgfs23EDJm+D+Hus070GmGTndEvDZ5EWfwQPA8+zmw30H1pAbYeI4o+D7L+evm0YdBh0J0HsHQc8iAw3+770Bmlm5kGmkgDvRm8Z+0S6yNAB71QWGae6EGfwS8T3aBf3jaRpZ/BJSEl0JH0JBvFABpdiSPw8zNvv7ECMMxS7MDQJe98RAaWG7NLPAdTvSzbxQO+ADWxxNYeL0IDwwwcRggANMPZhgCe61HwwA07v0vDQIzSzzSw2JAgDvT3egMEsM2tM0DoHhvb70eeexREPiZAoE0AAGvaZoGTcCRgx1n2JoDz8ToTvQYOKwg1DM9dvBPZ+FptoDPPNxxIOnUUBZ7EbJh+Go5pMq3kJvhmajh1m2Yb+zej2YpYAHiYb0emeg81IA2zM3EIZh7TeCZAzy9moDxhvSD8P/iRAeZ+J3poMw6w/WgaegNmeux4LvnBAy9UYw+O3zn0KaZ4oQVrAzMbmBrt5/QYIwPbppHhv018AwPw3FigebmZ71HzMCBPf8HWaA2W9LUNzBWEBuHnmj7EsD0tRtAAPgW9zYgkb89RxMvRohn86+maQdWogKBMqtHd16PUpMB7ztH1rrFB9I24IzAQbupwVhrvkBseXJ8/gmjUXA9V2rxI4e3m0D9HrnJu/wBnnHon/GtgZzv+rTI68ThGYeDerW2L/vSyZX8x1+Tpd7MjeC5sMnsgHqV8jJrOV7h1bNeaD1lbNNk+SQ0eBrnVsekhQ6w6EW+KN6tkd8uOHgmuwUeTSrhhBUrRrMaZE87S2hvDhlyejMDz5y7VrAYH/RJGzD8+ayr1Y44cK8IP85ckx0axm8Br1K8D7J/zjE2edkED3KvYMNzgMP7PL6FL1xW2PSogvNAFzUc2T87J5/5F123uJFs3bSzftmvxpOfWyf8AnBY+6uDlj3I6ct+1eQkH76ndC5PWPRvu6lP+srVnBUmvwuZ0XgVRY7m9YFv3nINi6rOo5tH/AEiIGBh+hc0uf0XZUbxLHrhzM/6DL2GH51gqDxX4hcOp4RZ0WTpfhS43X+ddqt70nLHqXh1k3qa75HjDBXsef7nsa9bMw9eW5MBo+h5kMwWdjVKKfvF7yolepVwwubpU5mZHPrADzD86zV1cHOHN5tYTrfZjPfixAADTbPDx0cxg+g0g3rtFe9EuuQ2jnWjcDMxrsalmAGuU1u2Lns+ZyNz0OTD/AHuGYfrBW9VrwBhp4JnDAkeswfsH8z+dR85WvswP60ZDPPDw0HiGOadDU3tmkAI1Rz6NQEYH+7zSw+Ka0fF1G3DBAWfwRmZgAOaZn9CDP4JAZh4epsQPINbBAa+n0ICDeXiJdBoM/gj70B54DqOImZP4aQagbwwMPImvyKxIN7ems/w06zp5eP0JPAGfgdChkDWRgZprD4oMz924rEsz/ERBp9iihqH1p0DwPBGp3ww2OJGzn8gedCe8NTUBAbzmOm2gM2XGT8NzNFrN4+O5vTWZ4fOlh39yBH1owNAgBEbSDNJCgeDYiwSTr70jXpVYjMxzVFVaPXDlHrhzhn1m0uh2ratonAAKrKk/3LILS0et0O3tWJQ7cMzDrdNcK3J6bwol3JCBiDZzwZ9ciR0KRJ4RQbYkeqnwB6QHk6DW9uridVXgCLSnOWAOvRRvQ6lXqHE5piSDz/WeCzwpX02pMUeODD86lQD/AHTO9WFSuH1k0LFHrIHn3nszVfP4dQYEc24vjVB/rN4OhTaDY0VmB9+fkhL8+a2w9PVWlXO9Tj/nKGH0Jmj2lXGYYHpwz+s08dsMRj8DnHvoNaCj1WDDihBnRZOsHyKMCveu1i3ouEtvmZH4LRrCXJeB1iab79Vkww7GWexdI+xlKrdWOdOiZs+RXYWHbgTOaYpsYPyZphi4jRLSodekHVW3KxUnQ688wXS7StWLQWgnMUZmHh7ED3mtg9QQk+Br6LPkZDBOw6PEhiEVvM2g6MzTArWY06TIOUZn8gdisDk3HJinBDrPod8imhGbDo6E6yy2C2eQxQWNWOd5uo1jW+TBXbNh2ifj1WnHPkH1m8rh48OtQXq9TY3tH160SwptKjf6Og6KdPXD2iy8m9oIF4esor15znj1GF5tzthrGCZeqTDJ6bj4AsK9W6rJ9ooh65+3M/1qWrdvXDB7HwNVz1z7jbBxZoAzMABWAQ2/aG4ZmoBya3ON32iiPSZbx6muaE43i6madAEeQAIxgeojNnfqaal4Z+HqYJaOHh5o0RDBzDDoTLISwLT0ww86nPdGDbgIAebMT8ToQCEZ/PxOhWB6YKm9ZMfjgor1YnSXeUpsU5J9+HYq2Lg5LCaOZED3iqfUl1H47gAzn2GYKqO3pZyMKlc9NZdPoa1t6kXsyqxA6HFVTKw57g1XvUSvsyMKPB5l3sMzVnD4aXHVfHuCqsw8/dM/6618cIqfXE4z02H8zPsDrNVtSrFwavK061Xpk3s1V1Kg8OqPQTB/UOTI/FNaAGQjezS7HH4dh8Ua86D9VOHR2j8j29aiBwipsZ3mp1Snz3f3r2xb3Wc/qX3WUYFDGtuJA8OIARvnAFbBDb94ZmpoM63u1T1u6rVt6OZ1WsxmT8me9eiXybKLk2w92uX1L0hLcB3Ch0OTPPsM9ix9Y4wcSqq06xFfCA0/5ADMEHoOTJpsCPrzp0aM0He69gsPXuNlnUrZStasO/8AVA2Lh7MadP8AEqtYemO+d54FKZgNs9Bh+taudtp/HK55MfQo9DCH9YZrGVKvXxckjXrlZeMPIAYAjww7w/Wln8Fq1V4UeIHYpAMtgkYOGXtEtH94akSAMPIgMwAjR4bEOjn4biAM/wANFn4XWGaXRs2IwDP2itkD5wSzwDejMDM/ICa97p9nnRqZB7MNRuI9+fYnj1EHMmBg2YGYH5FIwbz2I8gBm57xJ4/C9h0d6N4Gwd02D/Wl4mehqZoWADDyJIzDFLIEelrYBvSQYeL8UeHxRZeIDu/oR4e8QGGxGHTp9iAv3aWj8maHBzPxAwRZuezQGEZwyPBvoDM1HePAt/R506EnADczz3pYZjpubw60QEDAwzDeiA8BPTQ4N+7bwBEANh4aLHnqdmCaMHPdpAf4aLNwz8RHmC/Jgj7fEQd6M0Rstid9j7vrTXQGaWZm6Dh9CPTueAo9bMMPzprA8+hAZh7NQDOS+exLAD9oGaBIA94gSXidnUl0O6nWkZgfswwNWA6+vr70YdYfWg6DR4fFHkDP2pmkkkjQAHu/Il1+IkeYAl3aaIIDwJLMzPUSwbxQdAID70GfwSz3YIzDYjzRrrRHhihDURIYCl3+0S60sPihsgP9aQbPD/xpGHQaDPAM0aDS7Eg3hqJZhmX0dCM7fDZ39a+4ZuoNbP3eCQGYOg4j0b2plppJGfX9aTPXqOdCoI94efBAZue0SMwD2aM0ZAA+vUBOhvQ9aDoQHh8Ukj73EyyeZID98jZDxT8PrTIe1MM/nRmeAfOgNBm5lp6fZmjzcw2IM2wDPA0aiP8AEb6E0Z+K10IzM88OxBg/nsbz3/nQHnvwDsQf8Z/hpYOaR6fXghZBwANzU60DoAZ/QkexBG1wawc7EjMPZnns70Asg+ccNc/FzPNBDM+XPU2ZmeakZ5pPHs1FIAAAB1GwDA/ImgCVzQG/saD+NOs9ChVLN7DlJWiYIJslnPwMzBU4TJbMw2GGANp8Og+xXAHrADneqqZDl876xb6wDBBYMmeOZhgfkQQ2Z3NG+++GHYAJQ5nMtNG+B5mHeCl7AVAzPekyAM56ee8+9LwMTcf6ABV8mfOZdCKDYBmezYgsDZMzw1wz86XLaw+PvzSDUMOvDYi6w68FKyjBo5tm59CRg2e/v6M0utLWYZM23zD86A8NiWbmOYN/QgP8n5Ej2IDz2JrAAPU1MEYdC+GyBtIDz+ChSXmzzYccwUoDwJNPAw8Z4B1qmeEcJOj4fWpYHrYNudGfWmeWbz9mjAHMtnQpMJZnBP8AYc8APDegz+CaDTZL2ffmi7tRHo8/gmgBwOtP5sB7fPDPsTWG9BHZZlBIdNx/No+gPIpYAGl4jn5EeeHhgmT61QNI+hB0JZ/BAeHQaWxAZubGwSNSEakM/s4H+RNPHsAM9iTKoAAbsEt6E95IgPNr8m9SEZ5ikBmHiAl2I+xAgBI9NLrSwc94gAz25pGB7HPxEkjDoczQDg4ZgCIw/DS7tNI0CPe1pmkH4n/Gkl0DqGgX94lh8Us/glm3kgeMzORpv9CWIJSdA3dRhvYg7ECz+CXifWkGAe0BIwB4vDbw+QEWRh0JAB4agAg7tNI88/DR5A9bYlnt096DD4owPYiSDeSd6FH35I8/ggPPekB70vD0vnSR5BeIBG24iz+CEz3pAGYE55EeiA/FNPYAYeB196ZZCJ7/AGfOkbzYHsc3qGRGDjLqM3jMQzQa2ZJH1o1GZpB0b+tID2dGzzoEVA3kgSAMz00sMDwRJd2ojA8xDUS8MOtB4nu0Bsh4vh+dLPejzlh4Zh2bDTJ6ntHO9AjS8Qy0wbzNB2JGGzU6FAXie8bwPyJGbgJ48DDU7/Ome7xFYQGYD7TBI+rURpIyB4Z9aONJqNNkc3RqjJhu+eOeCQJ3Yg31t8eL7ogYVKWFVZ8kgN/611W1eM1gXbosVWUFNqZ9YPBs/WvNTyZBY3DV7gCNVWQzp07mWvOB5go+swBH6ypxhI/GaPBeQrSvy+LDd/5K3A9Gj98cwzBdQt70nAN0GL4twDz650c8MPyJhk7LJpUWsNGxOp0CsQj7Hutcyu30eLOr0jXoz8miSz7D3gui0S4bLu1oJdo3Gy86f5DBWZhVWT05TASQ+detNvLk/hRxp4YyPXlqvmeHW9T3gz/QpFH9KviNR3TiV+msz3Q69Vk2Xl6YZk04/YPyYEgPkVVW7Mti6j1LqtGNMD+1gGBmsUbVVk+kJZ12iDD+dKln2S3gw/Wuim8xPjmxLig8z3gYZga87170aac9mdnXHo6fREnAZ/xrKGzx64XRQgt84zE7MA5kFrFvHoC5+CHDK6mjbbofq2QfW9HP/IuRXP6Lt20o/wDkjXI1Va/BM8D/AI1Isn0nJQaUG9abreec0YB/Aux0fiRYdea16Pc8B4/wXT0TV7HkS4aDcdqzOUuejvQPne6FXhJYe9m+Br3a8cGYxyk6DGkh++ZzXPa96PfDK4XTltxJ9NkH3xJOAfowVxY8r6JmWo2kYP8AyLqF2+jTflvNG/blSCsRw7AA81y+ZAuCgu6Ffo0+H87zOAI1MmzgmjBzDoUtk2Hg1AfzQZ4F4DmBoI/92gNTjDMc3OtRDA1bBH3o9ZPgDfvEJsto38ADPJCZ4osPimjD3aA2Zj/4gGCd1s/ZqJgYezR54HsRh4TQkvgkB7vEbBRzezdRhpqG/hI1t3gN7PIgPQxHQzz71FDUA0X+NGB7D94kg6yzbSzcyVt/BGZo2TcNBn8Es/ggkaL/AOHmCDoPemdZ/wDEX3r9ogkZ4FnhsQG9maEDcBM62Du/9CIw01t8Or8n/wA5VW5zh08OswDA1Nu2j0q24QNsViTM1/xT6F006JOknpypZ4fIjCxqG94k5g5LvnM1wtnIrJjUqGbUudOCTIDrAF10K8BhpxQk/wDgqwjUGjw/YU1kPyKXots+w2K3Oy7MaqnKCXOYAz+dWrLOt1xQ/QrAwUU5LbPWrai5NjvYAEfJwfeMAf1qC9W4jIamainc8X3biPLtdeGHu0YPNrLnc7b3s1Hercow0w2LPadta9MYZ9o4oT1bgslvcWUMzP2jiaNNpaV67WA9g2ZqqmXJOkhpsbFX6JmjCNgm0wjvTJxhqPvmoOjmeuff51dvRgw6FHZZbz02wXqkQIeZKwjQ8A08E8yGiWo2mecc5jT1EapYQ9yaP2unoZpc5mmTe95qIJcbZ1tp3xPdqvCYxoG45KBVr1yABmwwe/yIL16NrAo4Mgzgs09W6qZ6jbmCrJNSlGXj1F7PyMgoW3pyQDrcVFUrkbB3QY3rL1ifOoMLnn2JPWAaLx7zWcn3JeNShGxSrYONn0SM96DdyalUZLRgwarHnqyz+1vrK023uJUyP6qnOcs0fefWf51ewOFcuSenVZbxh5AeQNHWILLR6jgTJHuY4H3oKbcnGyY6ESDTeWa/dMgui29ZNv0Ec4tGAHQ7zPNaiHs6GwD6EHN4fB+o1vOdeNcqT0h/rDNbOicNLKomHI0Y9UPevPZmtBkadZA3kA8s2z7AMEGsf4akcsf4mH1rL3JxOsO0sG5dVCfLP3Mfea1ixpWXnDUgIz59a41VfSElvR9O3LfBnP3shc/qt4XrcJH65uB7Sc7GtgJbnegK3xFsq3vDnVkNX8JreaxVY4903fFtilTJh9jroYLkIQIn4alww0fZraGkWs6rfnEavZhOnGy0fY0GAKkOlNycHJXMm75zNXAGmjewVJ2AI0WNIacbb2ACPw+wFHPXy1GMN/Xmnc8B9nmvcJLw/do3t7QBp5oOtO4fiIBAHwEM0HiZeIjzbA/DTqNTSIDwDw0JpAGxA6G9GB7zTIa+PgBmaFl7b4nX3oCPeh8fLswQGeezf9aMOgG+xA7kaPoQJo5Lh5hoH9ahGzrO9JAAI+hWsCPDAkDJhhv2GjPePtEBnvD50AM5ijZZwDT60WGAo8oPQCLe94nehDT70Bn+H0I08HT6OxNJkzcMdmGaNnUNoHHG8M0Dx6eGGpvQB9aWHxRoEnYwax6eYBh15pp7Z4envS/9WH1o82Lt1EObhpf4EWefu0ehZZ0QTueDRt4bzSzMyBvsS8M/EQBg57xH2I9hnqYfkQdx6aILw8DbPvXwPZL6YdBpZ7EWPxAwcczwQJH+8cM0CA0j3imnjSZDANiM9i3pBvwSPXxDyH2JI9Pd+o43ml4gbD796ZDU70g0wPPU6EeQPfn4DmBo8/gg/wCDrNI9P8NHpeIYIOxH/wAQfiIECM0YYYoMAxPNv6EIBh0IEYfrRBqAOmbmeHekG8UkWWfwQpYfFIMM/ERBdC+dZofE94jQLP4JAe7xA+sECQA3l86B0z+QA+QEKZPejwzFUFsM94JJGl1oEZo2Xt3Rmg6w/wACRngWbaBdYYJGkH/2CR4H1oyAaeDoDU3oAZb1TcccPf0Ag1tuxvM/IrCA88wQB1r74h7Og0WGBabihqPsNxtNJ0DAANtBDjOaTx6gdaA9iAzwFLrBCbLbwgD7eYB0IEe8wc8iD72bvgN/nRgbgEephh2Is3AQG97LTbPeaR7/AGfQms96kMh7xAmdiB7ozSz36enscQADmRuameaAwD3fyIDZ2aeofQmntQ3Q+Q0gDAj8Q96BAbGZhmZ/QCarGmzDzBwGTz6zTrOoA+IeZ+dG8APNbwA/rUhMg/GitMPmBmAbzBI9Q+hLw2QNw+wEo0yJMa1IjmYZqg71gDh9aWGarHq2wEzkdB7Pz9inMhL+QwQIDbAdDefnAwUoOVeJo5bGbTZ78EP944mXnvCd0MDwUvKGexGCiU2ZLkgZymw+RTfdaiNCz+CiyYESYWDmeZ9+adzbN3TBII2DueZoEyzywAxqGeCRm2B96dUcKlEkm63FMDNg96CQAI1XyTfzB9+L9384GpbIZtB86CVAZpz0j78+bIYdYKA8Hi6jaf5ZsEgBvE9RA0GefiKWehoB4Hih77zoABwPEM/oQGexEDNl/l9fT2edAGoCiyQOY1yIS3gDrMFKDYH0IHt+KiMyXzlGw4xs7DTufwSOS2BgAIDBAfWrOjyaGHMeuILz2fscDwwVe8CAT6F86ARn1o5OhiABvVCOCLw+/wDIj8jaBB9w2Z6a+fu0vcpB+J/xoF0Hgh3pPagO5+dIECBO57E0yGsfhp0NPvUgAP8AQkZ5kjANF8+xIwBA0vnX7M19Sw+KAs/gj8PMO9R/Ey00eYAXzoCP2p+RI94pBvazcS/u0A5/BO4N46gKOgOSwz7R8GTPoA1Ql+H2fnS7E1n4Wo2/+RLfigdS0HPaIANLXc9mpZBP2qIEsPih6PdoJBhvQZ4Fp9iAzSAMlDUmdfS1HG0j/d9SQPPs+GD54d4GkbOs0f7tWDA/doOjw80OZ5nqNpa2sGp0H5EDp6mOokYe8TQanf0JGf4biB0NQzRmDGemaZjPA9n5wTpgwftDMD86D6BgDWml4Z+ImXs89iMMAw1HOhQHQBNGDmfgOB9aN492oweCDPAczb61byLGyAH4+8EGDm8NTNH1pKCBHvwRh+H/AMSQYY+IkZ4PtafQhsvE6E6BgbSjvHg/8iPR95qdaPSMMD3ojNvSPUNCep59iDxAwcbbQCzJAHTbfbewMNh9iLrNLWc8mCE3kDpxn8NRAB700bx5ahmnQ3jqAgMNM3QbzSPPDw0jAPzoMTQGB/iIw/EBMpAeB4aaBGbgIweYe2G3vSP+NNAHio8wdA340gH4L5xnQ7w61uLb468VLbYCI/UgqscOgJYb/wBaw6W9Hr0xaXHKwLqd5W4JfqepeR08wP8AOukBTZZtA/BMHmj6DA8wXhd5nNXtpXzeNjOh9nK4bMf+zu7wP9axuGeHss9T2lSggZ+cNiTL0FnZBqR/JHd6Fwqj+lRL2MXpQ9aP3yIgYYfkXWrbuSwL5a5q2KzGk/ue/wDQowbNXJwxsO5/HuO1QZkfjRD0TXH7t9GapMtHOs6uMyQ/s8jwT/xrvBsymeiWZ/WgzYPxDB4JH4oGtEbeX2b5428GXwg1WDMOEx7mogbzP610Oz/Srt+pMAxcEGTTZHnB7WZP/UXaHoz8lg2JbEOqxH+tl7fmuc3PwE4XV7V+6SbekH/Zw2AsVOjW9eFKr0MJ1DrkOYHfpPb1NnnErDHK1WDGmRz6wkAvJ9w+jZxNtL+dbcnMz2g6Hoh4H/jUelce+KNk4U65oL0//wDOAYH+tVtq7hdXo08PLkd5ulc5RHT/ALOeYLl91eivxDoLXN0OXGrbQfvsDWrtL0n7RrcgGLjiPUE/PrazK67ArEqqwAqNs1yNPjn0GyexbRbJ4fmBVaU7ylcgyYbvkeDBIJLekva0mZLngbF42jGnxD6zNkDXL7k9HXh7cJG/blZk0GQfY8YYKNjzoZsGfhuI8Pit1eHo68TbSa5timhVY/niPb/0LBHJfgO8pKYNl0Ox0N6uLDvbqKPhvUvmWDDxOtNBvPTVgOW1gzYQYHvbBv6FIA3I3h9iEzzJWhBzPMGDYw+dPdCdQaOZKFhM8/eJHqJ0IzaA9nzq2oQPAkWe7PTS8M+hCeYBpoJoPNmPkUd4HwPxA2JoDNFrGBggX+ND4nvE6e/xO9NICSA8PnSzbx8RLw8PnRnb0w9PYZ9o+AKE9cMFn+lAuZT5j5lpvvnggZzM9PUM1zvHQHrzgslh1/Qoh3s57thZc42GCkAyoEuTc9VMjc1MFRSaxVZMrDmzwUieeBaaiMhvBxWLVkHD94akAG3BKAGzNxSz0wUNTWjs6EWinuhLMDDwwQBh8UYAGaDPBGDzAHqamaB0AQGGBJPVWLGDPUVLJuHM9h5oy8LXW92lmwHWsozJqsx09Nsw+tRZNKqWrqTp2DX1o6Gok1uCyWGuAKnO5IIOm4bizkyZQ4AafNaxpoKxFNrQptvvPH84ZoL2TXn5/wCyObE0frF6P49RCM186iQwrjwGc7RpsfD6DQQGYOqb8uhz6r5DDMAzRnhYUq25Vbf/AJmYkzGu90OgFHqp1y3ppxAowG7+MZrYW9cN9zPDChswwP2IaOAACvnreqte+93NUmeY8jTKPXIQrdzn/pU4zMf6FCk3tccObylvwWTM+h3lszNdVZ4UWdGkA+b9Sew7Dk7Fo6bAt+lR+VptNjMh9CLcPC2+Kl2yAflweWz63pHQC1FvcKKlSpBv1WuespB9nkXUDe1h8NNaL59C86RSepJfyKbApr7PtEc+e/Aw06bJkun0MtB1rP3VefEa2KSdcY4bTGYgf0iXsBMDYHDUKTcNq0GPr1y46bG/7bNeb69xFv8AvOQbk6uPRmj9zH2AqRmggBg/KfN761t8znd4n8eLOjOmxTW5k8w8jOCx9Y463xPdzodNhw4/0ZrD6OB5gpDOwU6UxZT7hvytkfrW5pOl+CBqEzTW2ez86lGGSQA57targYRmwa1MEjUgNT2aAwUowQYGOAdaEDczw0DBEAbAfbR+J1q1xBAkGpmZ6aWeZ6eBowDejR80XP60YRgMdTUQ8yAO4G/+REeHtOsEZBe37AbBFn8E0Zt5+Hv+RHnvRqSQPIM80YMoE8eZh4iAzwFGAN/hpZAgZwcMgPXMMOwFI6zSPDHw0IICPADwBDh3mke9LDATPeaBeJ7tECeNk9DX1AAM0CBoNfV8R8ME6yyckzw3gCRgGCWDeGxw8zPoQLBv3iTx4H7NG9mB6baBHmzoGfWCaeecMNdvMzSM3MNPUDDvTTOfR/AhggNw/kSwbPPU2Ai8QD08E9hsRewYNh4bYJI8Pilh8UegM8w0wRpYfFF/jQDgZnn2J0A3aiBmTo5+HmmgM8PZ4Ig6YYdfQjZBs/eJo+aePUfw/IiM9EN7f6EBmG/2iaRYOAYNmGGe9I8MkB4fFLvQYHlp96WfwQGl2JozwFEB7c0WWfQCQA4GeaRh0OAGeCMD8LU37+w0QADB5Hh+GgZAN+mHzo89hoDMz2edM4fFIzPPw3EZhu9pmgWfwTWe7BO7NXBBmGWn3oEyGAd/50m/ZAlht1DcSQL+7R9fs0GzLT7EJgYf9ogdMHDa1OvfggPwfyIQeMwwNvDA0lQWbmHh9a+Bnn4iHPf86dRkaPUDoRhmDW/rSzb9m51oM/gpajD8T/jQHvJLel4YdaBJZ/BADzh4eGkgNB4m9tJGCoLNvE9RIN5aYIDBszDUDMEjwz8NA9n8EBoADDYjQI/8iiRoeHt38zU7D8Nzeo+eAam80Dvh6oamzNAye/8AyIevyZ/OizwLMO/YCAzPN3MEj9qb7mzNADLgABuBsRoAPeBh2JADbODDbexIN5+H+dHnhs00AaOBamoaT2zD5+hGZ7FHkgGzTPAz2b0BgGDSQZgHidaXutP86Qb8NQ8wNAjBs8EGixnrmeBo8DB35EQG2ZYGgdDTy0zbzTXvTbbc6EYaYO6jjmAJeIDvk+dAYbM9Teo8kPC2GYZpazgOhnv3qPybmqZsOGefQglAeGDbnX2H500eAHqaiUPU0t/nRyXmGWjNzYHegAA5mOep3pMssUqPIfcPZnmnWXs2gfYcA2nw7Ej3+TBBCOZBmG0DE4M3+xSAByGOxxR2YcSM7psMMs5qRGe5mZIg6ZgyxsPNAZyeZzbY34daLP6M00FHgxpgTmM8wPfv60eG9S8o6yHhYd6WtmOmBpAgeZb0jbYPA+w0KCYHnqNuYKV1oQNxmQ0/qBmHX86lG8EkiccAAP5FDTwj54bFH5NgJDsttgNU1IDrNCb2irDMmS2y0ep5Ojzo4Z5xQc6M0jDPxE7g3vc2YfIgMDyQHvP5EgQH1qAZn4vXsSPTPoTQfO32JdbWnpq0HQBgC1/eoD619Mz5fU096Xie7QEy9g6DmGeHYlJeYydly28O/Z2Jp43Aw8NLDMd7eaA2TbMc23AME7GPxegPzqFDjMQA02GMM1Nh6eW/YgUnTZLT359+aaw0dnee9OhJbeE2/IaZe1zdacbDPQ6/oQGeoYeP0IwNvDU60BntM+zr/Imnjwa1GGM9ioOme3DsS/wKOEnWHNGB7EDp/wAHYms96M+hBhnmgPW8UHOhIz3oD/DNB2hpoHuwPOaADczPrUcANkz8czBHnhhqKRKR96jgG7PNG94JB35+RQEl4Zn4iAz25oGfGaB9vvVh4/3Z7Es2wDxEs9YvDbw7Ej3+GqAAfe30GmnozcnDXDPBO/ICA88PDUhAGj4bexPdCAD2ezSz3eIqBge350Rn0KOB7z37EjPP2alkl7DaDDrbSPpNs+9Mh1p557AAc3maoAYN46aQanYkZ570IagGpaQI9PPPeYd6EAbedA4hnhgkemeYZ70gDYqZmd55gkyyYNGw4ee/YakAzhvNC88AFqaew/4FLUmfZabm88MEzg2D+zr8ikGcQMDN/AwPZsSP8dAuj3e9IzwHUcSA8kYbz09Rn85oAzbNPPRohtajhn+RJ7YWg43vBAaADDZp6YGpDLL+GxwA+tNGDgdaDW/ERkRm4EjQcD86eAHDPrTPM+Lg5+tH3Z5o1ID3qQZ4dAKPmxnpubF9P2qgPhqPCfyJYMaXtN6az3ZtmiB7R9oGeaAzQAeZYB1pPG3lmDeAIQMOsECzPM9RtJFn8E1nggWDfvEYbOg8E6DzeWD7aXgZ7DQIDTUk39LwEj2B4Zgky9+IgZA3Hms9TNGEl/3m9O7EjZbDexn+dGQ/HeAH9MNI+g0jZb629iAA0Q0w2AZ54JozRpdjA8yReIfQo4HvRojaQyGZJGzge9R/EMPDcTus5hvcR4QafZgaWjomEqC+9GkB3tbEs3APUbSA8yQb62OPfEa3mAgv1JmpRPwZYZn+tdgtL0jbAup3laznb0jySzAw/WvMmiAKIbOZKE4e4+Wgz2gl0acElrzx3tiWb/Q/gY+QwXielVW47ed17cuOfA+Rk9i67avpOV+A0EG/KP6yj/2toAAwUYU7t91jeJTajMhyPIYIqrDg3JAOLc9vxqrHPrPRVPavELhlfLX8z1UGZf8AZ5Zgya0b0aVGH+bXMFI45XvRgsesOuuWdWHqO6fuTDWD+Nc6qXCLjLwuzfobFSktB1yKe9s/QC9OhqAedSiH/fACkAcsPDp1SMw7GndiqB5ws/0nLuo4hBuanMzI4f8Ajf4F0+3uNPC67ZB5y3qbL/ClhgH61o69Zlh3I06Fz2cyEs+t4A3rk9z+ipz/AI9h3HGe/wCry8ANB6Dh1vD2b4Gz2PAYGBqDWLetW7c/XlGhzPOfevH7zPGbg5PDXOqwMPIGsyt3aXpRHDd/5QWyb3/WIOAGkDYXb6K9uTNWVZdZOjun/R3czBcar3BDixarRyn6G9Pjh72IzmC9O2rxOtK82gfpVwRtX+zyD0Xg/ItgzUuW9/gtttXgL1k30PsGB94PBvUhk4rwammC9q3DYHD2+Wj9f2xDkyD/AKQAYGuT170UQ67OuMGQ/s87P+A1bJ588QCToPOYrQ3bwuv+z5GFVo7xtfjNbwWUZk4KBN3mmsTUjWYMPDQGD7IbwQDsTLzLmXh9HWngNzPUQGZ5q2qOGwM0jecPxEnjR8sYNA/2OIjBnNzJGgz+CQaaMyPxhRgG3TzzToSexxgMFHNlxl3NhzYim+kssA+b5tq4jA29g4GHQo5ssGeDis42mDQeIAAubZ4Mnsd00Ek9E9NCdSgvPngeeCiTHtZ3UbUbdBp5nM1IMN4OeQFX844B4P7DUSY9KkyNMDTY0B1VhlrU1OhHGrAPBqOLKMxp2/XUuNpsnm/KwBeoaU682CaO4XHh02MM/oWfk1uhs59Z/nVed4MB+w03MEGi1qqfvM0LMZ97xJc7D6zVCz9v6w7oQaO8z8+GCu4fB+7p7WvXJ2jn5DQOnUrcDw36yz+tV8+4aGy/oUNzn3fk7FoKVwctWHI5qpMHPe/fHsWogW3blKd1IlKjMgHYg5uH24qroMU6IcMPOrCNw9r8lrOs1XxfIC6QcyL7ttV711W5G/a6jGB38EDzNFqSlWHSoBhrsZ4edaMIdOZ2NsAjZmRKk0D8TPA+jNJ44jP/AKzfjfrXnarrGdNiHv0AU0DwD3P51j6qzxGrebdqgHL+cOtZKpW9d1HD1lVdY5rHnPNY/R5bYddzfMPAbA1WzJ7kb28uBG+t5c5ptK4oXJgESDPzPowPAFuIfBav02nO1Wovxp8v9890LHuOlcW9PoZxz9ZVmNJm9gRz2J2pXbatE1eeiRtUOwAzWFk2Nf8AWJ4N0diBSmf4/wBaurb4M1KZUgbuOuTJ+fe0fg/nWvhzdXkP8s0oywoFAOSHYARlqKVdV8VLByuW5DolPADOS9IDAwBBJjWjYdUDXnMsxw63nnjD9C55xC9IoHqk6fDlvmfB0DlvBhgf0LretXzl28Tqlr0eqxrPtxgwAKnO8F57DyKPxU4zWHZlgy+G1s1yZedYlbDlu7wD515quSfel+SI/wBtbqmTI7B5hHDYCKBTYNNY0KaxoteTNauc9TWdGOeu34pqQGmYb20HQnfEew09gLUNdAYJaOfvEbLLgZ83h8idDAD01LUAA3pI2dg5ow1D/IkHQgDB7+tLRc/rRnJ2aaZN7Z50Dzx9GAIjMA9p5E0EYz63E0ENgyMzcNBKOf4WgDAAmszD2iaw3eGiw/8AX86Mhgbhl7NLu3mjDYKAzb94aNSyBOs6YAZ4Z5qOeAeI50JHvLr/ACIHjezSzwTODZp7rBAW/wDImjZ39aM3jz2MZoAzMMzQGCRmiw+KEwbPrQFo4BngjMPdpZuGGm459CDPfp96Aw1Az8NGBgAec0H/AAdBoANswzBxFjA89mmgMN2oaFnlWRPQbM/OmXpj7zpxG9gIJZyWwaP8VAeftMN/kSZ2e0wTxhn4ihCOyD+Opp9akYYEHyJGb5nn8iQGjzZdaIEtbBNA9nvDYatodNDhmSAHnDdPUbwTpm2A5ogjNhlgP+NvA0gNzAPnQ+GYg5sMD6EWDnvEWaCS57tjNOmeYpGGABqd6AOtB9DZ50WfwT0Z44xm433hgmUZ4HgZ73+9AbOBo8MBFzU/IgPf0IYJLD8NLD4p4D0d+nnsQwiMvAZedHnvw09qKTMYxDXDDybE8AAYIvZlGfQkZ6x6huIESNnpw70BpdCXWexB9zb86+GHQaajAeGBp3t1OxAO9Fn7xJD4nu0DvhmfeCDDbnqJBvzz/Whw25oEGoB78Es/ggD2umkgM0v7xF4Z/kTXWeCoLRDLU70gSM2wPTSwzQGZ/u0GeGHzpGexINMwQJCfQvge1NvoSPZmHWgHZvPPsSwc2JGD4Hg2YYIgUg9mSRmDPiIM2w+tGe/8ioBGe2agfoSPeSPxDM80gw930IDZNB0ezSMMUHncQH2JozPZp7EDxvm1s/WjDTMdnYjIBgZn4YJ3YecTvw7EPRv/AOLoRGHvFIHDDDNwzNEBgkGpiH4ppGB5+Hh+dXhqEDb8U20s8wzS7s/OkeeB6beZ+ReAABINmbYYGfXgkz16jmw0s29XX35gvYAPB4WxgEmXm8xY0z86QG+Z/WaeDPPTNeAu/wBokAOb/Ez3pAfW243hvSw0c0DT2mHiaZ4fIiOZ0BoHvPDNGewN+YKOepls2IDeDcDiTzJyWjYBOvaZ4abeGwFWzKlFhnhrnn5AQTY0ZuMJtt5n9adMG3mj1E1AeCZANz500B1IJXh4HH7wNAzz9NokMG338GQ+RWDLzD0cH2z2GGYKOzGiSYoMG2DwfOpBhm0AB0B2IFsTRvYStPsUgGXDMA86iMnEmZuN5ngfkQSz6vkBIOs3OvyKPPht1JgA5t5nf2JU2HyDWhrm9v6zQS0Rhszw2Jp4zDDBvMzSM3MNNZA8PiiePAsPkTTL2BA4fQCN4APe30oEHVqJYZugkzsP5EjNWqCeNyG/hp/Ig7wc7EYB+I2nXntYf4ESA3m3gAww/IhDTPNvUwPsz70mQYAdPT+hAYfeGn/IizuGApozcANTDNHnmnTZcAAz8maIR+xLn4nu3A/Wl4nu00cCK9IzNjf50Es8DEDDoXwEAGDLXyAvusBoCBfT6Ey8b4YaDee/fvTuH/r+dUBDZ7sN6dZkuRi1G013n/Alh8VIKS9rE12bE0Dwaug252Ig60LwOB4gBmqCxBXFbjUAIEJ+huHmYfeQPsNU6QIDAwAg1HER7CQ5mHu8w70s2zDUbQIwz8RID6wQAZ4JHnjs6w71IAwA/aNoc3A7M807gZiDmpnsTW9UAPUAvk86dDZ4iENTLTRGGKB5k2NXZvTP5MASBlsDzb6zSPMHdPA8FIPDBB3o88C86R7/AGbeCoGCWHxTLIGew0j1ALTb60Bh1pqTr+zY6070Akb2Z+JgvdiP/wATgBg43gaQeDg52KQYBggAAAjb7OxeMiM2DDYZowDMfaYIOWM2jfbbzAO9JnUB3egeRBp96+gbZ7EBoE9sdXw9TEMEJm4bu9OgeieofR3o1IDcNrNtvYhwz8M074YAbbBngmlIDPrYMEbI9bmf5EWjn44diF6NmAH5DQEGp+Gkeg97T9CTJ5+0DrBNGz7tAeHRvzRYZ+0TQbMEeeaCQe9MngfhuNoWT1iwYMDPyIsHO/YaAdFF+RLNvFNZGgdmaYaQOd4I+c1hBvT6O9NdftEQACAzeiG6DjDeB96aM3DS0W8s0WIIJDLLhtG4Zhs7EgZiB4jj5/QmsTRgGZ73MFAd1op+G3nmmngAM8D60Rsthg51oDNv3aAzNw/bodEAPUSMzSz+CADPAkg3jqAkZgA70gQSzweH2mBgo4fvEnjbwDsPzgmQNw0ZJBmmsNiNkG0ntQESAAwHf0JGDh+IDiDWfx03N6eZPYaKM6xgGmjA8/aID3khN5wPaJhqkdCZDUAtRPRnjP2Gw+zNEZyszBzBAnpLZoNbNoA8iHRbz3to9Ew8TsRkR7E6YfoTRnvQffg9p7LsTAUlkMNRv2odBrW2fxg4hWZpMU2vyXoQf0eRvBZLNw0ZhimB6LtX0orcnlyN6UZ6jn/aGjA2V1eAdNrEIKlQ6izMjn3tGvDW9S6PVa5bDpy6BWJkN3sAHtn6FGGr2k8zVWdhhsUV6iU17CUx9wm/isrg9pek/dVKkad+U71lT/PHDxl3K1eItgX41/MdVZCR/Z5B6J/xrHAlnPuaN4E6KzWIQd4dax9Y4UcHb2Lnn6U9TZZ97WbJrofLe7bUGeywBh6xpWYfjM9YK4Hm+9vRavGiffrYqUaqxPz6361mqPxF4qcN5oMOVGpRnQ6484Ni9ZxnuWdD1bUj+Rp407VWYNwwzpt1UaNMjn5wzWOxxq2/Sitl50INwUOTR5H9oaPNn/Auy0SvQa9CCpUOqhVY598fsXIrn9Fe0q26blnVw6O6fuXd4f41yascKOMXCiecuC29y59cunPbP9962i2T2cE8McH2zAD62j6DWUuThRwuvBo26rasMHT99H2GvO9q+kzd1HkYXNBjVuP53T0TD/XXZqJxp4e1tprlbuZjSD/o8s94KNjl95+i1c9Hmc3Y85mpU8+gHjwMP41zG4aDelpYN1yhz4HzvM7P1r2hGqr4eP8AtMf8ZneCuOcg1iOcWcwEyO51su9CuLavAQTHPeOKQGZgvVdz+jfw1uHD1VTTojvniHs/QuSXJ6MfEK245v0OczVY/kZA81tFjk8lnbqJMm4Adew+sEp8auUd04tcpUyG7+9DBKNJiG1gasLBzHw+9Ls0zbToMxOhh8/oNI2T92gjvA+Y7EIG4Gw+hSg/eKO950G+5xRJNSlH4epgkkuRkOm23VZ7XNRA2edFUqa/R2Neq1wIzXyJJKGo6DW+GoQ9SVWTkn5zBZypXharM83KG3+fNJJWK1m8J1Yf5SjMHJd+QFsIfDe7qk0DlZf5No+zPrSSWq2ggcIqGDoPvvyTw+da2Hb1sUp3UCCB/WkkpFhz/wCA2AB8iA5L5+8SSUINPahprRM0kkWdCHrNG2b4Rg73j6AWFn8ReE9n1l1iCwD0gPbS8M8zSSRXGlSePfCuZ7eqvB/2Jgo7PGPglnqPvyXv+xNJJcT6vhpWfSK4J0ePoQZ1Sw8jMY1XzPSW4Vh+yUqsTD+RnBJJdXjw5NeVVJ9KWzg/0bZdVkn5NbBRw9LFswwb4czA+uSkkts+Ea8oTPpUVXmgfOwIxh5MzVfdXH6+7tY5Gmt+oYJ9YR888PrSSXufDFznkAM83zekn55DxmpfQCSS2cgPEP3ZpAEvAM2MEklaD2Gw0QGYNJJIBzbM0j6tRJJSsTMlsM/DA0szM+wGvOkkiSAwwSMADxEkkb+Czcw02z2JZ/iJJID8TSPBtAGo81qdiSSAT1ARdmo4e9JJAs/nX1nY6bnWkkgdwzRYfFJJAPiGYeRFh8UkkBgz+8S2JJIEYe8QYNmPekkgXh7Ac/Omz9qkkjyHzRM2jBtGyGApJI0O4B7xvNO54AYd/YaSShBGbmkmmWTyPUcSSRkdCNgHtP1pBp46iSSNIIwzFNAy+ezsSSVvTvhgANt9iQHvNJJAsMyzcR4ZkfkSSQLYCD86SSLJLYCSSIIzxS2JJIFg3sc/DS70kkAmyGJuA4aWGbQOedJJAXQCXhmJtpJIBAMA9okyGAaYdCSSB3Ng8M9ijg8bwajbaSSBAe0Db6DSAw9mkkgPPDPT70uxJJAGbYAl1gkkqC6ABtLPYkkgDDNH3aaSSBdBoM9v50kkDTJuHmZ9CNJJAHgGQantQ6EYGG9JJAQHmYI+9JJGRGGYpow2aGHWkkjUj/dpkGcPE7/OkkjyB55j7NGybmJ6nYkkpeiMMw1O/NDgxv1D7EkkZEGmi1vw0klTVH1jB0NPsTUaTOmSDcciYB1h86SSCzDAAPUwUfM89TsSSUhYGf6818e3lh8nWkkqBvA/7z8iQamkGZgkkgE1XvUdh6Qb7hnvSSQSADk2GnNA/wAikd6SSCJTZkV6Kfq5wzaYPBSDk+EeDef+NJJBVHUpbzoRMD39asowOMxwbcb9mkkgew+KZkz4MPw5T+Bn0fOkkgeMwAwcBs0s3DSSUgsD922mo1NbZDXlvnq+RJJBLDoSNk2XQ1PrSSQF1l8iE9iSSBdaL9aSSPIRQZwf1w3mpAGDOlnm9nnsNJJCzpva3iaeCjmeA56mBpJI9ADPhaZuGYJdAJJIDDnsR18MOxH4mHtEkkBAlg+Znp9gZmkkgT0Z9kg12zDMO8EetgKSSCJm4b+nppBqBq6/n2JJID1mzLT1N6WYZafekkgXekzqe8czSSVAfE2YdBpnN/I28PFBJJAZnt+dBzOeAON5/wCRJJAgPxQb1+zADNHn0an60kkDp7CSz63ASSU+A6GppGbij57/AJ0kldg83PZto8M2t7eBpJKABmgPUD86SSoCeYH3/Wlns8RzekkgIwcxzb6ELJ78HDwSSUgjntgGnoZmkEluSIG30JJIEep2J7eaSSBowcPwwcMD+ROgZ5pJKgvE92iDeBmkkpDRg+Z+Hv8AIgwfZ9u2YH5EkkB6LfWGw/kUgzbMQ12/F8/nSSQAbLeWo25sQ+Hnh3pJIE88GfibEWwAzSSQH1gg6CDxNqSSBPHh7A80jef8iSSgIDcNGCSSB3rBNJJIFyzDwYPngaANBnZqZpJKw9rQfeJk3m9XBhJJQHtFzAHNRRno2sWeoYGkkgkgGY4d/egwwJJJAu9GYZjpuJJIGcMCwBPeJ7xJJAjNMnnikkgeZ0z9uGCI3mw8PU2JJIBZktgXnQZ/BJJGRAakABpJI1I+nTULk/F5pjwXQ3g8GzBJJRbJ0C1eP3Eq1WgiSqj64jh0BL6/1rstpektYFwu8pcFNepUg+90wwSSUtXRfVtKqrXN0p+NJDztPZoTgVKN2bEklnYjmYd/WpDNSnB4YNxpLXkSSUjOXJw64e3nH065bkaM7+KyeBguNXP6LVcZaOdaNwM1Jr8E2QA/8aSSDCQ6xxf4RTTbM6xAZDrB4PuprptB9KKDJ8O5rYBk/wC0RD3/AOBJJVA63avEi0bqaD1Hc/MyD/oj3WtaFVw2PsPM/WkktAqrDoFeawqsGBPz6zMMzXJ7z9GCyrhd5ugTjoM7s0jzD9CSS1ZOT3V6OvEq22uaigFbD/qnWuayfWNNd5Sq02TDP96zgkkgXORT95vSDTNJJWP/2Q==" } ] } @@ -120,18 +68,9 @@ { "language": "typescript", "source": [ - "const tensor_data = new Float32Array(inputImage);\nconst tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, tensor_data);\n\nconst inferRequest = compiledModel.createInferRequest();\ninferRequest.setInputTensor(tensor);\ninferRequest.infer();\n\nconst output = inferRequest.getTensor(outputLayer);\n\nconst { data: outputData } = output;\nconst layers = { bg: [], c: [], h: [], w: [] };\nconst resultLayer = [];\nconst colormap = [[68, 1, 84, 255], [48, 103, 141, 255], [53, 183, 120, 255], [199, 216, 52, 255]];\n\nconst size = outputData.length/4;\n\nfor (let i = 0; i < size; i++) {\n const valueAt = (i, number) => outputData[i + number*size];\n\n const currentValues = { \n bg: valueAt(i, 0),\n c: valueAt(i, 1),\n h: valueAt(i, 2),\n w: valueAt(i, 3),\n };\n const values = Object.values(currentValues);\n const maxIndex = values.indexOf(Math.max(...values));\n\n resultLayer.push(maxIndex);\n}\n\nconst pixels = [];\nresultLayer.forEach(i => pixels.push(...colormap[i]));\n\ndisplayArrayAsImage(pixels, W, H, display);\n" + "const [height, width] = inputLayer.shape.slice(2);\nconst resizedImg = img.resize(width, height);\n\n// Transform image data from NHWC to NCHW to match model input\n// as alternative you can use ov.preprocess.PrePostProcessor\n// see hello_reshape_ssd.js sample\nconst transformedImgData = transform(resizedImg.rgb, { width, height }, [0, 1, 2]);\nconst tensor = new ov.Tensor(\n ov.element.f32,\n inputLayer.shape,\n new Float32Array(transformedImgData),\n);\n\nconst inferRequest = compiledModel.createInferRequest();\nconst outputs = await inferRequest.inferAsync([tensor]);\nconst output = outputs[outputLayer];\nconst outputData = output.data;\n" ], - "outputs": [ - { - "items": [ - { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAIAA4ADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDx2iiivbPYCiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACirdlptzfrM0CZWIAsT056D8at2ugXMsuLk+RH3YYc/gAf6ildGFTE0ad1KS0/rYyaK6iTwzYnZ5V7cLhQG3Qq2W7kfMMD25+tTQ+HNHWJRNPfPJ3ZAig/gc/wA6h1F2ZyvNMMle7+5nI0V2X/CO6F/z01H/AL6T/Cj/AIR3Qv8AnpqP/fSf4Uva+TJ/tbD9n9xxtFdl/wAI7oX/AD01H/vpP8KP+Ed0L/npqP8A30n+FHtfJh/a2H7P7jjaK6z/AIR/S/S8/wC/y/8AxFPTw9o5z5jXy+m2RD/7KKbnboN5rQ8/uOQorpP+EYtv+ghL/wCAw/8Ai6P+EYtv+ghL/wCAw/8Ai6fOv6TNP7Swv834P/I5uiuk/wCEYtv+ghL/AOAw/wDi6P8AhGLb/oIS/wDgMP8A4ujnX9Jh/aWF/m/B/wCRzdFdJ/wjFt/0EJf/AAGH/wAXR/wjFt/0EJf/AAGH/wAXRzr+kw/tLC/zfg/8jm6K6T/hGLb/AKCEv/gMP/i6P+EYtv8AoIS/+Aw/+Lo51/SYf2lhf5vwf+RzdFdZ/wAI/pfpef8Af5f/AIinp4e0c58xr5fTbIh/9lFJzt0M3mtDz+45Ciu//svw1/0Dpf8Av43/AMVTY7LRIJhJHpCttzjzJWYH6qcioVZvaL/D/Mx/tmlbSEvw/wAzgqK9H36X/wBAWz/79r/8TQW0ryjjR7QSdgYlI+vSl7Wp/J+KI/tr/p2/vR5xRXpMGoG1jKW9rawoTkrHHtBPrwaS7vvt0Bhura3ljPZlJwcYyOeDz1FPnqX+H8Sf7Zlzfw9PX/gfqeb0V2g02wRiY7OJQeoOW/8AQiaupcTRxqiSyKigBVViAAOwrW76I1nm8fsQv6u3+ZxNvpt5dQvNDCWRBkksBnr0yfm6ds1VZSrFWBDA4II5BrvSSzEkkk8knvXL3Fmtx4jktyxVZHLsQB3G44oTZphcwdWUudWSVzKVSzBVBLE4AA5Jqf7Dd/8APrP/AN+zXYR20ELbooY0YjGVQA4qWncwnm7v7sTivsN3/wA+s/8A37NRywTQ482J489N6kZruarX1qLyzkhONxGVJ7HtRcdPNm5JSjocXV630bULrd5duRtxnzGEf5biM/hTbGO2kDmaOZ3QqwVCMMCwGOmc8/8A6q7Gh36HRjcdKg+WC18/kc//AMIvcf8AP7Z/nJ/8RVZ9AvVuPKHlsmQPNDfLz3554+ldTRUq/c82OZ4hbtP5HP8A/CL3H/P7Z/nJ/wDEUx/DNyi5F1aufRS/9Vro6KEn3F/aWI7/AIHP/wDCL3H/AD+2f5yf/EUf8Ivcf8/tn+cn/wARXQUUrPv+Qf2liO/4Iw18IXbqGF/p+D6ysD+RWh/CF4sbML3T3IBIVZjk+wyMVuUUrT/m/AX9pYr+ZfccpNoOpQRNI8ClR1CSo5/IEk1nMpVirAhgcEEcg11HiS8SJUjtWmgZiHVS5LbcEH5gADyK5ari21dnt4OrVq0+epb5X/UKKKKo6wooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACrT6nqEuPMvrl8dN0zHH61XRHkkVEUs7EBVUZJJ7Cum0zwTfXE5/tD/RYVweGVmfnoMHjjufbg1nUqQhrJnLia+HormrNfPf5IteGWm1aC5EsiJ9mVMMQfmyDyTn2rRks7mGBJ5beZInxtkZCFbIyMHvxWvDp9lo1lcfYoVg3AsTkkk445Ofy9/esKuejVlVk5Lb+v66nysq8K1Wc4K0en3ahRRRXUAUUUUAFFFFABU3kR/ZvN+1Q7/+eOH3dcf3dvv1qGipab2dhhT4kWSUI8qRKeruDgfkCf0plFNq6sBNcQRw7fLuoZ85z5YcY+u5RUNFFKKaVm7gyW3jjluESaYQRk/NIVLbR9Bya3rrwowsjc6beJqG1sMsSjgYyedxyenHXmucorCvSrSkpUqnLbdWTT/J/iXGUUrSVybfb/Ztghfzj1kMnHXsuPT1JontpotMF5HDLcEsB5US5bGcE/8A6s/zI04/E9/5Ahukt72MY2i6iDbcDGe2Tz1Oaoy6hK929xEkduXxmOFdqDAx0qIPENtOKXW97p+WqTX3aEySVnF38mche65dGSaBI/IAJX5gQ6+ufQ/yrO+3Xf8Az9T/APfw1376hNKjR3CxTxN1jljBU856UkFtpd6xtrnTLRFfo0UYQg9eo5/L+tbupOKvKP3M9ClmNKlHWl9zv+djiP7a1D/n4/8AHF/wplvqt5by7/OeTjG2RiwNdffeA4ZJHeyuzECCVikXcM88bs5A6DoT9a5XUND1HTGf7RbP5a/8tUG5MZwDntn0ODzRTxFOp8LO/DYnA4hctO130tZ/16GoviK12jdFMGxyAARn86G8RWu07Ypi2OAQAM/nXNUVtYv+zMP2f3nSWevwyJi6HluP4lBKn+tdhoB8Kalp7G71lIbkMCweQQ7RyABvGGz1OOnHTv5XRXPiqEq0OWE3B90P+zaHM5L/AIB7BPpPhpnBt/FNjGmORJPG5z9QwrzuKe2m8Yoy3Ci1e4EQuH+UbD8nmEHoMfNg1h0Vnh8NWpJqdVy7XS0/DX5mlPBUoXst1Y9efwqZnZdO1OyvCqbiiuA5+gGR6ckjrWPe6dd6dKI7uB4mPQnoenQjg9R0rirbW7u3jKZWXJzulyx/nWvF8QfEURC/bFkhJy8M0YkWQdwS2WwRxgEVCp4ym9JKa81yv71dfh8zznlkm2tvmatFZw8VWd3l7qyNrL1P2QZRuSfus3y9uhI9hViz1C1vkleOeOIRkAi4lSNiD0IBbn8M4rrU9LyVvU4KmErU27xehk6L5kM0lnJach90kh/hx9368jj65rfrM0+4guNSvXhk3BwhGRjoCDj9PzrTqysZJyq3as7L77BRRRQcgUUUUAFFFFAFnTvsn9oQfb9/2Xd+82dcf4Z645xnFdpqWg2kuiE6JDbs2/zNwxIXA6gMc9wOM9iO5rgqlguZ7Vy9vNJC5GC0bFTj04rzcbgqtapGrSqOLj03i/VfqbU6kYxcWt/vNbVtJt9b01bcylEVt8bR4IBAIH1HPTiuPv8AwPqFvtNnIl2p4I4jYHnsTjHTv36V1aeIr9bkzyi2mZvv+Zbp8/GBkgA8fWp59ftmg2W2nNC46M1wZB175GT+dRBYyi1FRun2aaXrez+5EYetiMLG1Gaa7Nf1+aPMb3Sr/TiRd2skQBA3EZUkjOAw4P51Tr1aLWkK/vYmB9U5z/hSzXOlXqobqKKQrnaJodxX9D6Cuv6xVj8UPuO6GeVVpUpfc/0/4J5RRXp7+H/D11OW+zQF2/hjlKjp2VTiqr+BdKaRmEt0gJJCq64HsMrmj67T+0mjojn2GfxJr5f8E86ort3+H6GRimpFUydoaHJA9zuGfyrOfwLqqxswltXIBIVXbJ9hlcVosVRf2jqhm2DntP77r8zmaKsXthdadcG3u4TFKADg4OQe4I4NV63TTV0d8ZKSUou6YUUUUygooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAopVUswVQSxOAAOSa1V8PXhUEvCpI6FjkfpQZVK1Ol8bsZNFdDD4cjC/v53LEDhBgA9+uc/pVxdE08KAYSxA6lzk/rSucc8zoRel3/AF5nJUqqWYKoJYnAAHJNdDe6Aj7WsysZ6MrkkH3zzVePw5OW/ezxquOqgsc/pRc0jmGHlHm5rFCLS76XO22cY/v/AC/zqe40Wa1i8yae3RM4yS3X8q2INH8qFU+3XQx2jfaPyqO50P7Rt/02c7c/607/AMumKLnL/aCc7c6UfRnNuoRyqurgfxLnB/PFTb7T/nhP/wB/h/8AE1fsNPsi7vc3cTrGdpVSVUk9PmOM9D09Kj1iKxgkSK1TDjJch8gdsfXig7PrEJ1FSSd++3+RnOULkxqyr2DHJ/PAptFFM61oFFFFABRRRQAUUUUAFFFFABRT4YZbiVYoYnkkboiKST+ArpbDwPqFxuN5Iloo4A4kYnjsDjHXv26VnOrCn8Tsc9fFUcOr1ZJf123OXq5baVqF4Ea3sp5EkOFcRnaecfe6V6DZ+ENItoohLB58yYLSOxwx/wB3OMe3863q4qmPS+BHiYjiGC0oRv5v+v8AI860/wAEajcSOLwi0RRweJCx+gP9a37bwNpcRRppJ5yB8ylgqscewyPzram1O2hx8/mE9o+f/rVm3OqzTbki/dpngj72PrUqWJqvsv6+Z508xx1d/Fyry0/4P4l6C303RYGS3ijhGMkLy78nGSeT1PWq82tNu/cRgL6v1P5VlM7OxZ2LMepJyaStoYSC1nqzBUE3zVHd+ZMbmR43WWSVwRwPM4ByOo796hoorqUUtjdJLYKKKKYBRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQA5GCurFA4ByVbOD7HHNdXpcnhW8ZVmtDZzjkb5329+jZ9u+OvGa5KiuTF4T6xG3PKL7xbX/AZpCfI9k/U7q40m6t4vOt2+3QEb1aPG/byRx0bjHI5OelUIpY5k3xuGX1FZWkeIb3SGCo3m2/eFycDnJ2+h6/n0NdbEdE8T73iLw3YXLMn7uUDIHPZhwPXGe1eDX9vgn/ALQuaH8yX/pS/wAvxIngaVfWk+V9v8jlLvw1pF5HtayjjIBAaEbCM9+OD+Oa5fUfAt1CC9hOLgZP7t8KwGeOeh9+nSvS7jRb+0XcGW7jCjJjTa4POTtycj6c+1UVYNkA8qcMDwVPoR2PtXVh8dzLmpT5l/XzX4GUcVjsDKzbt56r+vQ8XdHjkZHUq6khlYYII7Gm17DfabZ6lEI7y3SVR0J4K9OhHI6DpXJ6r4G+9LpcvqfIlP1OFb8gAfxNepTxsJaS0Pcwue0Kvu1Fyv8AD7ziqKsXthdadcG3u4TFKADg4OQe4I4NV67E01dHtRkpJSi7phRRRTKCiiigBVYqwZSQwOQQeQa6HTvEsNtZeTeWJuZUzskEpUtk5w35nn2Ax3rnaKmUVJWZhXw9OvHlqL9PyOpHiKwaLcYrlJP7mFYD/gWR/L/Grkd/aSW6T/aYVVv4WkUMPqM5FcVRRy2OOeVUX8LaO9orjU1S/RY1F5OUjxsRnJUY6fKeMe1dJB4n0qS3UXVlcRTDqbdgwPHX5jxznjn61MpOPT7jza+XVqVnFc3oXqKzf7f01mVVa4GWALPEAFHrwxP6Vf8AOg3qi3Nu7N0WOZWP6GndHLOhUh8UWh9FFFMyCiiigAooooAKkSeWNcRyug64ViKjopNJ7g1fctRajdRdJSwznDc5/rVldamDDfHGV7gZBrMorOVCnLeJm6UHujXOsRSgxzW2YmBDDIbI9MEc1nT6T4d1BnzbtayPgB4ztx9AMqOnp3qGioWGgvguvRlU4uk70pOPox6eCdHlz5d3cvjrtkQ4/wDHaozeAJViYw6gjydleIqD+IJ/lVupFuJkUKk0iqOgDECl7Ksvhn96N44vGQd41L+qRzk3g7Wo5WRLdJVHR0lUA/mQf0qeHwPq0kSu7W0THqjyEkfkCP1roU1C7jXAnYjr82D/ADqZNXukXB2OfVl/wol9ZtpY2lmmOtZcv3M5afwTq8MYZBBOScbY5MEe/wAwArNm0HVoJWjfTrksOpSMuPzGQa9DTW/uh4PqQ39Km/tq2/uS/kP8azVXEx3jcUM4xkdJwT/r1PLZ7C8tYw9xaTwoTgNJGVBPpyKr168uq2jKCZCp9Cpz+lOnOn3UYS4NrMgOQsm1gD680fW5x+KBrHP5rSdL8f8AgfqeP0V6xNoGj3cShrC3253AxLsJ/FccVWfwhojRsotChIIDLK+R7jJxTWPp9UzePEGHfxRa+7/M8wor0F/AenGNglzdK+DtLMpAPuMDP51T/wCFff8AUT/8l/8A7KtFjKL6nTHOsE95W+TOKooorqPVCiiigAooooAKKKKACilVSzBVBLE4AA5JrTt/D2oz232kwGKHdsDyAgFvTp1pNpbmdSrCmrzdjLorch8OSFv386BQRwgySO/XGP1q4nh+yRwzGVwP4WYYP5AUXOWeY4ePW/ocvUkUE02fKieTHXYpOK7BdPs1UKLWHAGOUBNTpGkSBI0VFHRVGAKLnLPN1b3YnKRaJfSlcxBFYZ3OwGPqOv6VYXw7dbhulhC55IJJx+VdLRSucss0rvayMeLw7bKFMkkjsDzjAB/r+tWf7F0//n3/APH2/wAav0UHNLF15O7myv8AYbT/AJ9YP+/YqSKCGHPlRJHnrsUDNSVXvv8AkH3P/XJv5GgyUpTai3uc5YrFNr67QDEZGZABgYGSP6V1dc34fhiM01xIcGEDBJwBnOSfwFbMuqWMWN1yhz/c+b+VNnoZhGU6yhBN2SX6luisj/hIrT/nnP8A98j/ABqlN4iuHyIYkjBGMn5iD69v5UrGEMvxEn8NvU6SoGvLVWKtcwhgcEGQZBrkbm9ubvHnzM4HQdB+QqvTsdsMo09+X3G9da/JFeOkKxSQKQM85Przn61Vvtbmu4jEiCKNhhsHJPtn0rLooO+GCoQs1HVBRRRTOsKKKKACiiigAopyI8kioilnYgKqjJJPYVvWPg7VrqRPOiFtEwDF5GGQOP4Rzn2OOnaonUjDWTsY1sRSoq9SSRz9ORHkkVEUs7EBVUZJJ7CvQbDwPp9vuN5I92x4A5jUDjsDnPXv36V0NpZ21jAIbWFIox2UYycYyfU8dTXJPHQXwq541fP6ENKScvwX+f4Hnmn+DdTu5U+0J9lhZd3mPgn2G3Oc/XH9K6Kw8D6fb7jeSPdseAOY1A47A5z179+ldRVSfUbaBQd4kJ7RkGuV4mvVdo/gePWzfGV3aLsvL/Pf8SyiJHGqIoVFACqowAB2FNlmihXMsiqOvJ6/SsifWJWZhAAqcYJX5v8ACs55HkbMjs56ZY5qqeClLWbsccMLKWsma8+soARAhY/3n4H5f/qrNlvLiZmLSuA3VQSBj0xUFFd1OhTp7I64UYQ2QUUUVsaBRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFOR2jdXRirqcqynBB9RTaKGrjOr0rxpNbokOoRmdFGBKh+fv1zw3b0/Gujl0/Stdi+1QuN7cfaLdtr9jg++AOGHHtXmNWbXUb2ywLa6miUNu2q5Ck+46Hp3rwMXkcZT9thH7Ofls/6+7yOmGJ05aiujrbvStRsFLGI3kQ/jgX5+3VPqT0J6dqpxTRTLmKRWHXg9PrS2Xji6R0W9gjljAAZoxtf6+h78cfhXQG00zxHZrdwM0bMwLSw4WXIGNrHB9uPYV59SticJZY2Gn8y2+a/4b0MKmX0a2tB2fY5+aGK4iaKaJJI26o6gg/gawr7wdpN1G/kxG2lYlg8bHAPP8J4x7DHTtXaz+G5IrcLY3ZaQf8AP182endcY79j+FYlz9u03f8Ab7KQKv8Ay2hG6MjOAc9u3B55rqwuNp1f4E9e2z+57/ic3sMbhHeDa9Gcf/wr7/qJ/wDkv/8AZVS/4QPVP+e9n/323/xNd1FqFrKuRMqnuH4/nT/tdt/z8Rf99ivQ+sYiOj/I1jnGOju7/JHmv/CI65/z4/8AkVP/AIqoLjw5rFrt8zT5juzjyx5n57c4/GvVUkSRcxurjplTmnU/r1RPVI2XEGJT96K/H/M8Uor2uitP7Q/u/j/wDo/1j/6dfj/wDxSivZp7aC6jCXEEcyA5CyIGAPrzVf8AsbS/+gbZ/wDfhf8ACmswj1iaR4ihb3qb+88hor17+xtL/wCgbZ/9+F/wo/sbS/8AoG2f/fhf8Kf9oR7D/wBYqX8jPIa0tDj1I6lHPpkBlmiOenyjIPDE8DIyOo9ua9NTSdOjkV00+1V1IKssKggjuOKuVE8emrKJjX4gjKDjGnv3f6f8EhmtYLjHmxhiO/Q/nWbquhS2Ci4gZriyYcTBcYOcYI7fXvWk91bpu3Txgr1G4Z/Kq1n4rurGd/KiR7dmB2PnOB1wQeCfx6CuWDxcfeoq9uj0T9H0f4d+h5OCvdxnt37GBRXZya74b1V0+32DxyNgvLs6EDoWU7iO3T0qlLoOizxebY67CilThLggEtz16ED8PfmtoZpbTEU5Qfpdfej0XQ/kaZzNFbw8PWq2skk2u2CyKCVSN94OB65z+QP41huoV2UOHAOAy5wfcZ5rto4mlXb9m728mvzRnKEo7jaKKK6CAooooAKKKKACiiigAooooAKKKKACiiigBySPG2Y3ZD0ypxUq3tyjBhPJkerZH5GoKKlxi90JxT3RdXVbtWBMgYehUY/SpP7auf7kX5H/ABrOorN0KT+yiHRpvocFRXfL4J8PIoSfxI/mqMP5dszLu74PcZ71cgh8L2aeSmhPeIOkt1Nhzx3CjHXP+eKz+uKX8OEpfK3/AKVyp/I+pnmFGPX+vxPN0jeVwkaM7HoqjJNamneF9b1YkWWmXEgBwWZdig4zjc2B/ketdlaX7aZNI2mQwWiGRnQCFGZM8Y3EZ6cVC9xM901yZCJ2fzC68HdnORjpzWkpV5X5Ul2vrr5pW/BnJPNv5YmWvw319IHnvBaWUSKWZp5s45xj5A3J/wA80Wfgyzff9u8Q20OMbPItpZc+uchcdvWtKWaWeUyzSPJI3VnYkn8TTKyjSxEo/vKln/dSX/pXN/XQxlmtW/upEK6B4etrkApfXqRv94zrEsoB/u7CQD0+9n6drN5b6LIkS2OjRWoTO7dIZS2enLenP50yitI0IqSm2215u33Xt+Hock8XWmrSloRxQQw58qJI89digZqSiitznbbd2FFFFAgooqOWeGHHmypHnpvYDNA0m3ZElFZs2u2ULbQzynJB2Dgfnj9KzX8R3JcmOGJV7Bsk/nkUHXTwNepqo29dDpKjlnhhx5sqR56b2AzXKz6zfTbh5vlqe0Yxj8ev61RZizFmJLE5JJ5Jp2OynlMn/Elb0OiuPEUKqwt43d84Bbhfr6/yrMutZuruHym2IpPPl5BI9Dz0rPooPRpYGhT1S1CiiimdYUUUUAFFFFABRRRQAUU5EeSRURSzsQFVRkknsK3rHwdq11InnRC2iYBi8jDIHH8I5z7HHTtUTqRhrJ2Ma2IpUVepJI5+nIjySKiKWdiAqqMkk9hXoNh4H0+33G8ke7Y8AcxqBx2Bznr379K6G0s7axgENrCkUY7KMZOMZPqeOprknjoL4Vc8avn9CGlJOX4L/P8AA880/wAG6ndyp9oT7LCy7vMfBPsNuc5+uP6V0mn+CNOt43F4Tdux4PMYUfQH+tdNRXFPF1Z9beh4uIzjF1tFLlXlp+O/4jURI41RFCooAVVGAAOwp1VptQtoY93mq/bahBNZdxq80m9YgEQ8A/xAVNPD1KmyOCFGc9TZluIYc+ZKinGcE84+lZ02tJ5f7mNt/wDtjj9DWQ8jyNmR2c9Msc02u6ngoR+LU64YWK+LUsz31xcMS0hUEY2qSBiq1FFdcYqKskdCioqyCiiiqGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFb2heJm0W1ktzaiZHfeDv2kHAB7HPQVg0VhicNSxNP2dZXX9di4TlB3idfZeOpRKRf2qGM9GgyCOvYnnt3H410tlr2m36IYruNXchRHIwV9x7YPXr2yK8rorx8Vw7hK2tP3H5bfcdEMZUjvqesf2Tp7MzyWdvK7MWLvChJyc+lYd/4JtJVd7GV4JOoRzuTp09Rz3yfpXKWWvalYOhiu5GRAFEcjFk2jtg9OnbBrptP8cQuAmoQGN8geZCMr16kHkY49a8qeXZrgnz0J8y8v8n+huq1CrpNW/ruc5eaBqunMXktnKLkiWL5gAP4sjoO/OKo/a7n/n4l/wC+zXq1rqNle4FtdQysV3bVcFgPcdR171FeaNp2oEtc2kbuSCXA2scDHJGCa0pcSSg+XF0tfL/J/wCZE8DGWsGeXfa7n/n4l/77NH2u5/5+Jf8Avs11154FXBaxuyDgYScZyc8/MOnHtWBe+HdUsEeSW1LRISC8ZDDA74HIHHUgV7uGzPA4jSElfs9H+P6HLPCyhvEofa7n/n4l/wC+zTkvrqNsidyenzHP86r0V6DhF9DHkj2Jvtdz/wA/Ev8A32aPtdz/AM/Ev/fZqGinyR7Byx7E32u5/wCfiX/vs0hubhlIM8hB4ILnmoqKOSPYOVdgoooqhhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFUm1awSTY10o9SAWA/EA1lXfiGR8rap5Y/vuMn8ug7+tB1UsFXqOyjb10OioJCqWZlVRjLMQAOcck9OSK406pf+b5gvJkf1RyuPypk9/eXUYS4u55kByFkkLAH15NDT6HaspndXkjp01nThetbzTuiKSDMqb0yPocke4/lzVLUNfSK4eOx2TRgjEzAgMMc/LwRz/Kudoo5db3OyGWUIy5nd+XT1/rQuzavfTZBnZRnICfLj8RzVR5HlcvI7Ox6sxyTTaKo7oU4Q+FJBRRRQWFFFFABRRRQAUUUUAFFX9P0XUdUjd7O2MiIcFtwUZ9OSM10Nt4Bmkt0e4vhDKwy0Yi37fbO4ZrGdenD4mclfH4ag7VJpP739yOPqxaWF3fybLW3kmIIB2LkLnpk9B+NemWnhfR7TBWySRtu0tN8+ffB4z9BWvXJPHr7CPGrcQxWlGF/X/L/gnmtp4L1eecJPGltH1Ls4bjPYKTk/l9a6fTvBmm2YDXIN3KCDl/lUYP8AdB+mc56V0dFctTF1Z6Xt6Hk4jOMVWVublXlp/wAH8RkMMVvEsUMSRxr0RFAA/AU+kZ1RSzsFUdSTgVTuNUt4OFPmsRkbCCPxNYxhKb0VzzlGU3pqXaikuYIiweZFK8kFufyrEl1a5ljKfImf4kyCP1qkzs7FnYsx6knJrsp4Fv42dMMI38TNefWhtHkRnPcyD/A1Qnvri4YlpCoIxtUkDFVqK7YUKcNkdUKMI7IKKKK2NAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigByO0bq6MVdTlWU4IPqK2rLxbqtnEYzIlwvYzgsR17ggnr3zWHRWFfDUcQuWrFP1LjOUNYux6DZ+NdOnAFysls+CTkb169ARz+lb1veW13u+zXEM23G7y3DYz64ryCpYLme1cvbzSQuRgtGxU49OK+fxPDFCd3Qk4vs9V/n+J1Qxsl8Suep32k2GpbTd2ySMvRuQ3fjI5xyeK5698DQu7vZXRiBBKxyLuGfTd1A6DoT9ay7DxlqFqqR3CpdRr3fh8YwBu/qQTXSWvirSdQLQNJJbbhgGb5M8HOGB4/Me1eW8Nm2Xfw23FdtV9z/yN+ehW33ONvfDuqWCPJLalokJBeMhhgd8DkDjqQKyq9ih2GCMxPvj2ja27dkY4Oe/1qrfaTYaltN3bJIy9G5Dd+MjnHJ4rqw/FEk+XEQ+a/yf+ZE8Ct4M8nort73wNC7u9ldGIEErHIu4Z9N3UDoOhP1rnrvw1qtlFLLJbZiiyS6OCMeuM5x36V7+GzfB4j4JpPs9H+O/yucs8PUhujJooor0jAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAormf+Eiu/8AnnB/3yf8aqTareztk3DqMnAQ7QPy/rTsenHK6zetkdPfahb6exSdiJR1jA+bt1Hbrnms+TxHAF/dQSM2ejEKMfrXPO7ySM7sWdiSzMckk9zTaEu5308roxS59WbDeI7r5wsMIDAhSQSV9D16/hj2qhJqF5NG8Ul1M0TnLR7ztJ+nSq1FFkdcMNRp/DFBRRRTNwooooAKKKKACiiigAooq7YaRqGp7jZ2ryKvVuAueOMnjPI4pOSirsmc4wXNN2XmUqK6iw8D6hcbjeSJaKOAOJGJ47A4x179ulbun+CNOt43F4Tdux4PMYUfQH+tc88XSj1v6Hm1s4wlK/vXflr+O34nnVblh4T1S9MDmIRW0oDecXU4UjOcA5/D+Vei2GnWmmQNDZxeXGzbyNxPOAO59hVquSpj29II8nEcQzelGNvN/wCX/DnJWXgS1guA93dG5iAP7sIUyfcg5rodP0uy0uN0soBEHOW5JJ/E81corknWqT+JnjV8diK+lSba7dPuWgUUVDLdQQ58yVAR1Gefy61mk27I5Um9ETUjOqKWdgqjqScCsifWjuHkRjHcyD/A1SuL6e54d8L/AHV4FdcMHUl8WiOiGGm99Dbm1G2hXPmhz2CHJrPl1qTzD5KJs7bwc/oay6K7IYOnHfU6oYaEd9SWa5mnP72RmHXGePyqKiiulJJWRuklogooopgFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQBZtdRvbLAtrqaJQ27arkKT7joeneul0vxtJGCmpoZQB8skSgMTnuMgfljp3rkaK4sVl2GxStVgvXr95rCtOHws9Uste02/RDFdxq7kKI5GCvuPbB69e2RV9g2V2kAZ+bIzkY7enOK8crWsvEuq2UpcXTzKeqTkuD19eR17EV85iuF2vew0/k/8AM7IY3pNHoV9pNhqW03dskjL0bkN34yOccnisO98EWk0oe0uHtl7oV3jt0yQfXqT1osPG1pKqJfRPBJ0LqNydOvqOe2D9a6BZ7TUYnjhuUlUY3GCbkenKnI6V5HNmWWySblFL5r/I3tRrLozzy78K6vabz9m85Fx80Lbs59B979Kxq9lqpf6ZZ6miJeQCUIcryQR+I5r1MLxRNaYiF/Nf5Pf70YzwK+wzyWiu3vfA0Lu72V0YgQSsci7hn03dQOg6E/WudvfDuqWCPJLalokJBeMhhgd8DkDjqQK+hw2b4PE6Qmk+z0f47/K5xzw9SG6MqiiivSMQooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAOCoooqj7QKKKKACiiigAooq/p+i6jqkbvZ2xkRDgtuCjPpyRmplJRV2yJ1IU4803ZeZQorqtP8DXtxG7XswtCDhVwJCffg4Fa1h4FtLednvJ/tcZXATYY8HI5yG+v51hLF0o9bnn1c4wdO657tdtfx2/E8/q1Yadd6nO0NnF5kirvI3AcZA7n3FevQwxW8SxQxJHGvREUAD8BT65XmD6RPKnxE7NQp/e/0t+p5lD4P1l540ltfLjZgGfzEbaM8nG7nFdHp/gayt5Ha9mN2CMKuDGB78HJrqqKwnjKstL29Dz6+dYuqrJ8vp/nq/uM2y8P6Xp1wLi1tAkoBAYszYz6ZJxWlRRXPKUpO8nc82pVnUfNNtvz1CikZ1RSzsFUdSTgVXm1C2hj3ear9tqEE0RhKWyJUXLZFmismfWhtHkRnPcyD/A1Rm1G5mbPmlB2CHArphg6kt9DeOGm99DekvLeINvmQFeCAcn8qz5taXb+4jJb1foPyrHorrhgqcd9TojhYLfUtz6jczsDvMYHaMkVUoorqjCMVaKOiMVFWSCiiiqGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABUsFzPauXt5pIXIwWjYqcenFRUUnFSVmO9jptO8Z3tv5EV0iTwpw74PmEeuc4J/nj8a6my8S6VexFxdJCw6pOQhHX14PTsTXmFFeJi8gwmI96K5X5f5HTTxdSGj1PZaK8v0/xFqWnECO4MkYAHlTEsuAMADuPwx0rpNN8bQS4j1GLyW/56xgle/UdR2Hf8K+ZxXD2LoXlBc68t/u/wArnbDF05aPQ6G/0yz1NES8gEoQ5Xkgj8RzXM33gdSZ5bO5IGC0cBTPOPu7ifXua6u3vLa73fZriGbbjd5bhsZ9cVNXDh8wxeDfLTk15Pb7nsaTpU6mrR5Ve6JqWnxCW6tHSPuwIYDp1IJx171n17LWZf8Ah/TdSlea4t/3zLt8xWKn2PoT9Qa+hwvFK2xEPnH/ACf+fyOSeB/kf3nltFdZf+CJ4onksbjz23cROoU7f97OCfy/pXPX+mXmmOiXkBiLjK8gg/iOK+iw2Y4XFfwppvts/uepyTozh8SKlFFFdpkFFFFABRRRQAUUUUAFFFFABRRRQBwVS21tNeXCW9vGZJZDhVHevQNP8EadbxuLwm7djweYwo+gP9a37SztrGAQ2sKRRjsoxk4xk+p46muKpjoLSCueriM/oxuqKcn9y/z/ACPNv+ER1z/nx/8AIqf/ABVKnhDW2kVTaBASAWaVMD3ODmvT6K5/r9Tsv6+Z5v8ArBiv5Y/c/wDM5nT/AARp1vG4vCbt2PB5jCj6A/1q/D4X0a3njmis9skbB1PmucEHI71r0Vzyr1JbyZ51TH4qo25VHr52X3LQKKKKyOQKKKjeeKNsSSoh64ZgKaTewJX2JKKozarbRSbMs/umCP51Um1p/M/cxrs/2xz+hraGGqy2RrGhUl0Nmojc26sQZ4wRwQXHFc/NqFzNJu81k7bUJAqtXTDAv7TN44T+Zm4dat9pxHIT2BAH9aqf21c/3IvyP+NZ1FdUcLSj0N1h6a6EjzyyLiSV3HXDMTUdFFbpJbGyVtgooopgFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAEsFzPauXt5pIXIwWjYqcenFdDp3jO9t/IiukSeFOHfB8wj1znBP88fjXM0Vy4nBYfEq1WKf5/eaQqTh8LPTbDxNpd+jt54typxtuGVCfcc4NbFeNVo6brl/pbqYJi0agjyZCSnPtnjnnivm8XwwtZYaXyf+Z2U8b0mj1SiuTsPG9u8SLfwvHKWwXiXKY9eTkfr0/Cumt7y2u932a4hm243eW4bGfXFfNYrAYjCu1WDXn0+87IVYT+FmTd+EtKuIpRHB5Er5IkRjhT/u5xj2/lXO3/gm8t0RrOUXZJwy4EZHvycGu+pqhgPnIJyegxxnj9K6cNnWNw+07rs9f+D9zRE8NTn0PJLyxudPuDBdRGKTAODg5B7gjg1Xr2N0WRGR1DIwwysMgj0Nc9qXg6wu8Nan7JJuJYqCytnttzx+H/6vocJxPSnaOIjy+a1X3br8TkqYKS1g7nntFdDeeDtTguClsguYsAiQMqfgQTWA6NG7I6lXU4ZWGCD6GvocPi6GIV6M1L0/Vbr5nJOnKHxKw2iiiuggKKKKACiiigDrqKKK+dPGCimPLHFjzJETPTcwGahe9jXHlq8/r5IDY+tUoSeyGot7Fmisq71WWGUIkBUjr5o6/TB6VRn1G5nYHeYwO0ZIrohg6ktdkbww05am5PeQ26sXcFlxlARu/Ks6fWjuHkRjHcyD/A1lElmJJJJ5JPekrtp4OnH4tTphhoR31LM2oXM0m7zWTttQkCq7OzsWdizHqScmkorpjCMdkbqKjsgoooqhhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAVLBcz2rl7eaSFyMFo2KnHpxUVFJxUlZjvY6bTvGd7b+RFdIk8KcO+D5hHrnOCf54/GunsPE2l36O3ni3KnG24ZUJ9xzg15lRXi4vIMJX1iuV+X+R008VUhvqey0V5bYa/qOnSo0dw8iKu0RSsWTHbjPGPaup0rxnb3Hlw36eTMzY8xR+79icnI9O/r9PmMXw/isOuaHvry3+47aeLpz0eh1NV7yxttQtzBdRCWPIODkYI7gjkVLFNFPEJYZEkjboyMCD+Ip9eIpTpyutGvvR0tJo5i/wDBNncOjWcptABhlwZAffk5Fc1qPhjUrDz5PK822i581COV9ducj39Oe3NemUV7OF4gxlCyk+def+e/33OaeEpy20PGqK9YvtJsNS2m7tkkZejchu/GRzjk8Vzt34FR5ZXtLzy1OTHE6ZA9t2emfb86+iw3EmEq6Vbwf3r71/kjkng6kfh1OJoq9f6NqGmIj3dsY0c4Dbgwz6cE4qjXv06sKseam013WpyuLi7M/9k=" - } - ] - } - ] + "outputs": [] }, { "language": "markdown", @@ -143,14 +82,14 @@ { "language": "typescript", "source": [ - "const alpha = 0.3;\n\nconst pixelsAsImageData = arrayToImageData(pixels, W, H);\nconst mask = cv.matFromImageData(pixelsAsImageData);\n\ncv.resize(mask, mask, new cv.Size(originalWidth, originalHeight));\n\ncv.addWeighted(mask, alpha, originalImage, 1 - alpha, 0, mask);\n\ndisplayArrayAsImage(mask.data, originalWidth, originalHeight, display);\n" + "const layers = { bg: [], c: [], h: [], w: [] };\nconst resultLayer = [];\nconst colormap = [\n [68, 1, 84, 255],\n [48, 103, 141, 255],\n [53, 183, 120, 255],\n [199, 216, 52, 255],\n];\nconst size = outputData.length/4;\n\nfor (let i = 0; i < size; i++) {\n const valueAt = (i, number) => outputData[i + number*size];\n\n const currentValues = {\n bg: valueAt(i, 0),\n c: valueAt(i, 1),\n h: valueAt(i, 2),\n w: valueAt(i, 3),\n };\n const values = Object.values(currentValues);\n const maxIndex = values.indexOf(Math.max(...values));\n\n resultLayer.push(maxIndex);\n}\n\nconst pixels = [];\nresultLayer.forEach(i => pixels.push(...colormap[i]));\n\nconst alpha = 0.6;\nconst [N, C, H, W] = output.getShape();\n\nconst segmentsImg = Image.fromArray(pixels, W, H);\nconst resizedSegments = segmentsImg.resize(img.width, img.height);\nconst mergedImg = Image.overlay(img, resizedSegments, alpha);\n\nmergedImg.display(display);\n" ], "outputs": [ { "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAQ4B4ADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDH0NljmMUwKuDjBFdHjFYEwW31yIuhAY454xkV0LCuo1I2QHtmomQAYwPyqY0hGaaBsrspPTNMAINWCMU0pmgQinkU8VHjBqQcgUBYUClFKKUClcBCQOpxUJu7cHBlx+FSMgY81QvZpB/o9tESxOCxHAouBpDY6ZRgQfSo1QqeFrI3XOlgBVLyEZOOhpLLX5J5ZTNiNFBwBzzRcDYuA7wFJEBU9SegrmvEKBrQRrCWMYyGB/lWbrOq392Cod0j7AHGRWBM9wsYJuHJPVST/PNFwNWKB/sQM7HZ2B5xVMQvaF5Ui32755ptib+8eO1UHacnkdBW3e2ktusdnkHzSACO5+lTYopaHbkymaVmWPOUYngH0qWa2nv76RS+FBxkdDW5F4cmhsBanEgPJJOOvtWdqmi3dlaF7e5Z1GcoSMge1FyTFt4bY60tpeyLEitgs3Q8dPauyvtL0N9LaOCCCRuoMbZbNcfZadBqiSSTXLm5APyE8k+oqa3tzp0+Ltmhb+FweDRYoDHNdl7ctIqk4yRkCsu5WOwmeObFwQMDcT8v0rXupraNXuLe8l8zjgAkVzF1ILiYyA5zyT6miwxOSMsxOOBn0oXe2I1JIPb1qLJ6GpFO0bgSCOhFAD2hMO0kYB4/GlMcm4KCik9CelMeRpnDOckDikk+bBzjHqaQEqWkkc7SSKNoB5ByKs2+oIrCGUHy+zDkk1SjeaIErKRkYPNRxhNw8wEr3wM0AbZgubmPNwcwgExk9AKqWF1JY6kXjfAwVLY4Ipl2BGF8m4d4yM4PampMsVsxXGTjgjrRYC4v2U3ZmuJhIr537TzmiG3059U8syMbdl4J4ANaej+H47q7tJZ5Ee3mUkgHBB96yNRtEtNektUYlVcAZo5SbGh/bENpYSWBh8wI5CHII/8Ar1Te7a68smNYyOEI7+xrc0270kwlJrASX8QIYqvDD1x0rndVmju70vawmGMfwEYINFxGjapqKq0sMhMaHmEtkY9cVv2FzLqOnJYhC3lncxPHTsa42Od4U/dsQ3cjuadHf3gVwlxICxycdzRcZ6T9tTTbQRiyFuHIUOpGCTT7XQ7VtN8uWGOSQnJPbNee2N7FFcg3RlB6kEkgn1IrQttTvTqDT2ss0MDDoSdpPt2q7FF/xNDa6ZCtlaxtCzkGQYwCPXP510Om6fFYaJHDbyZEuCzjuT1xXCX+pTXGsxSSymZYnBy3b1/pXbWNlHbrJdz3Qa3BEkaZwFJ/GiwFxdIsoomeGEo7Dlie/qazLnQNSulkX7akcDDkAZBpLrVhqMJNrexwMp5HUMe2f1qjaX+qwzFbp8QuMMB0J9aLEmTHpgiDC4hUsCQCDzx0NNm04TN5hmAB6bqL+5UTuq7nkMnX0FVrq0u0tzI8UhA7DoKzsUOjO+4e0klMkZwFYDOBVC+sxbzlAwI9u1T28iRlShKuOoI7VO5le4iMiJtJxkevv6UXAxrcpHdorHKE4PtUVxGY7iRSOhrXliUawSVwvrUT273E5IiZgRkMBwaLgMski2gsoYEZIPUe/vVgxj97EvzhBwe+KLmyFkIbiB2BYYI7gikSTyskgMZF5U+v1ouAseof8S9IQDvjOARzVmS4FxdIpUspTAI7Ed6p2kRQSs3Cg8nsKu2cAW+GZgeMjA6Gi4EMbPaTTRycZXGPUdjWrY/Z4dPAaTd5wyfrUWpQeZbh5kBkAwrr0+hq3pNkrQlgF2vGVcnoD2wKSQrmnYW3/EoMshJkYHBJ9On9auWWTZxknIIyKgs3LWAtmOHj+Q++O9XI1CRqg6AYFa2EOAxSEd6XpSVQBS0lFIQtIaWigBhpMU7FIaAE6mk706jGaBiYoHSlxS4pgNIoApxFFACE4GaIpA2D0prjNEaHOcUxF0KCgIqJhzxT0JC4oIpDIGBqMip2FRleaQiLFIVqTbSYpDGBaULzUgXigrQAzHtSFR6VLikK0WAgI9aaRU5WmFeaAIcCkIqXbxTSuKAIyKbipCKQikMjIoxTytGKAGYpCKfigigCMiginkUmKAG4pMU/FJj0pgMxzRinYNGKQxtJinYFJigBMUY4pcUY5oAQClxQBzS4pAN70UtGKAE7UU7HrR2oAQUCigUAFKaMUUANFKOtFFIB1FAooAWiiimAtA60UUAKKUelIKUUAOo70UUgFpRSUooAWlFIKWmIBSikFKKQx1GaSlFAx1AooFAhe1KKbSigB1FJS0AOpaSlpgKKBRQKQCg0opBSimA4HmnimCnCkA4dacDTM04GmA4GgU2gUgJBUg6VEKkHSgBSaQ0GkoAKQmlptADwcUCk6UuaAF4pQabmlBoAUmkNFITTADQKTNGaVwH5ozTc0uaYBmjNNzRmkApNNNBNIaAENIaDSGmAUZpKKAA0lKaQUAAFGKWlA5oAQClp2KQjmgBlFKaSkAhptOJpDTAbQaMUGkIbRQaSmMXNFJiikIdmikpM0wHU5KYKcpoAkppNOzxTT0oAQUZpMmkzQA6lzTM0ZoAcaBTc0ZoAdS5puaM0CAmkNFFAwzRmmmigAozRmkpiCikopDFozSUUxDs0A03tRmgB1JSUZoAU0UlAoAXPFHFJmkzTAdSZoJpM0gF5pM0lJQAE0E0lFMBKKKKQgpwoHWigAoopaAFopAaXNMAJoApCaATTGO1uB5o0dBypFaanMSHvimkZpQMDFIAJ5pM0h60UAGaMUuaM0XAaVB60BQKWjI7mi4AKR5NpxUMzkDgkfSmS3MQhASQFzzjPNAExcr1FNAzJuqC5dJbJCX2uDyc1VN0VKIXDcjnvSCw67keTU44c4Q8VVuNMFremIDEMg546+tWL+QR3lvMeEJA3dhW4Yo5ohvXcMcHPQ0wsYt3p0Igt4kQFlIIJ5JrL8SaHBK8EibIjgbweBiunMYe4HzEYGOO9c34ubD2ocgxs2Dg9RQBUtozZ3Cran7QSOgPI+tVAbl/EEKSxkyCQEqDxiuqsNOt9Nt3a25LDO8jkf5zWLpsLpqtxezRsWyTHgZ5PagZ2kkY2ZLFQAASK4zXb3ULpza2cKqi4wxPJ4rds2untZhNkZORkdvSsjXNOu3Mdx54UMdpUdhx0pNCZwrtcW9wSSUlUkEjtVqSS5vo1867D4OfmrQu9GJimkTKqnTzOC30Nc0u0thwQfQipA1Dq0llCbQCOUAEb8etZG4MSRUrCEdBzTGBOMAcegouNMYAScGlKsTheR6Uu04JAzg0qEqQw60DbGFSjgOCM06VDG+MgjHBHekkMkr4wWwOwpiYaRVZioJAJHb3oAlt0Nw6xNyvf6VY2rZ3BxGJEHQkc1HPbm0fAkD5GQwHNQglgd7Hd6igDSuLGKaP7RFcDaOqYyQKzGVVbG4Nj0pY0dn2gnB649Kd5adsmgCSK5mhx5czKB2zx+VOTfd30Ykfl2ALYqMQuxwBn8aeLYjBeTYPXNVcDedY9HnnjQeYCARITkipNGtrK/gvLvUQSUPUHGB/kiufaMrj98WHuc1JGZlRhCWG4YOPSobFYu6na2qXAOmTGW3YZIPY1VtYBJOyKwWQAkA966TR9C+1aFIzkxyMSVJ6HmoLewsbu2MNwDDexMAzA4DDOD/OhMdjnmgluHEOP35OAPeu8twYvBtza3EIjmgQqQR36g1qReDdLsAt4hcSKN2Sc81x/iLVn1K5+zWaSHBIZBghsdM+netESaPgjRkv4bs3dqxWVMRyEcY74/SpbvTtO0vT5re/uG84AhUDckDoQKqaBcizjhhu9RnhRiSYwcBT2Htmr+o2elWhe/juzPIwOxJG3DIFDQ7nFJHayWzyPLtkHQHgkUgupriNUe6kCDpz0FWYL+wmvppbyzCK68AdAfWs+WMZyvCk4H0qRm1NHElpDcW8qmdBkn1FS22q6hdp5YjQbCDkDr7Vjy3EohSFscAEEDGauq89vYJPHNEVI2MVPOPUikwEvLlmMhMMYkJyWB/pVQSsFQFyOc8UTxEvK0cm4KM5XoarRvvA+Yn61LQGnaE3LSNJIQQcZA61DJKYZ2RJXAU44OM0tnOI3VGA2k8kVDcxS/wBpMArMOxAoQFhDwPMY/OScHt6U0pkbx2JFJNKHKbRjbwacFYqVBxxkA9CfTNAiRLkeTJaxp/rAMlhyR6UxAbWYxk4OD+VLbL5kgjlwvJGQORSXEsJJZJNzhgMkUAaTO81pGTKpBABHv6VespJIdN2yfLiTGepxXPo+6ER4+6c5rX8ySV45ipKSAAgdAaqO4mb8TKApGMkdR3qysmTjNY9tKSmTwemPSr8Lg9a6miUy7mgGmBhjrSg56VnZjuOzSg0zNOBGOTSaYx1FFFACUhFOA5oIwaQhhopSOaTFMYopaBRQAYpQuaSnA0AIEBPPSpQABgUzNLmgQ/NBpAeKDQMQ0xhzTzTSKAGYoxT9vGaAOakBAMCjFSACkI4qgI8c0lPIpCKQDCKQr7U/FGKQERXFMYVMRTCKAIiKaRUpWmkUDI8UmKfikxQMZikxT8UmKQhpFGKcRQRQA3HFJinYpCKAGkGkIp5HFNxQMbijFOxRigBuKTFPxSYoAbilxS4oApAJijFOIoxQBHSYp+KTFACEUd8UUYoAKXvTcU7FABSd6WigAApQKAKKACjtQaUUAFLSCloAOO9KKSlFAC0oNIKUUgClFJSimA6lpKWkAlOFIKKYDqUU2lpDHUoptKKQh1FApaYC0tNpe9ADhSim0tMB2aKBQKAFFKKQUtIBQaUU0GlFDAcDSim04UwHA0opoHNPAOKAHA81IOlRgYqQHigApcUuKKAGmm0402gAzS5ptFSA8UU0U6mAhPFNzQaSgBwNGaaKcKAFpc02imAuaaTQTSE0gF7UlLmjFADaQmlIoxQA2gdaXFGKYAaTFLigDmkAYpenSg80oBxQAmTSEnNOIppFADSaaTTjTTQAGmmlNJTAKKMUtIQ0ikpxpuaYAaTNBNGaADNFNooAdSg4popc0AOBzTjTAcUuaADNGabmjNACilzTRRmgBTSZozSUAOzS5puaXNABmkzRmigApM0E0lABmijNJmmAtFJRQIM0ZpM0ZoGLRmkzRQIXNGaSigB2aM0maQmgAzRmkNIaYC5NJmkzRmgBc80ZoopAJQaBRQIKUCjFKKYBQaWkFABS0lFIAozRR2oAOtJ0pM0UwNbNGaQ0lMaFozTc0ZwaQD6M0mc0UXGFNkIVcn8KceCPfpTZRlQD0zQIgjuI3maFxtYDOWOAax9V+z7DIjlXBzx3rVNiLi6SQNjb2zjNc1r8rDUzbJII8HIJ6E8cUAR/ayDGHZgH6bumar6lfSWTKIzhz+INY2pXssziFiPkOQQMGqIlnmlCbyTnIzUFG7Jr9+IDbzIjAgEEjkH2rZ0rxBMLZI5JFXAwC3INcldSy4RZipKdMVXa6kIKo2B9KAPTo7wzYaJkcjkhDyBVTxTGlzoQuGUq0ZyM1z2iCOSyLhzHKvcHFdBey2l9pJsWuQGc4yeSPxqkxGfpd697p6ol8I5l48t+9S3GvTadttL+IKQQUmi5BI//AF1La+EbOxtJZJ5WlIQkMDgAj6VxbxXeo3DiNWl8vOBnJAoA9BHiK3mtmSNWd2XIbIAz/Wsm9u0vLa2YXDlUOJFUHI981ycNwdOuwJLc8Ahlfiuj0bUpI2eKGzAhc7g79Qe9Mkta7a2Ulnbpa6oHY5yrvkfiO1cjqhBdSxXzQMHb0NXPEVt52sN5MQB6uR0I9axJlIYA9R1NJsCTyidmCCWOMCpprC9t03zWzovTJ9aqSAiMFJGJPbNaJ1C7lsxDJM8isNvluOPrmpGirGCyk4IIGTx2qNj8pI4INIzzR5Rsge9NJOMZ60DJLCWSO4LA4IBxxxTblxLNkAZI5wMDNRVLbMiTgyAlSCDjtQMYgAOTVzTooJ7qRJmGwJkA8c1Tl2+YSjZXtxipYIy+4AjIGcE8n6UAS28gjuZAmSACOByRU9xNaSQqqAoy9Qep/GnxWvnMRbr5NwAS2/gEd6fDpMjMZr0FY8Z3g8EUAUUDna8RAHYk0SHIKyEHuCOxpcLG3lhw0Q6MOpoMdoSSJGGe1IQluYQ+ZULAggEdQfWr/wBnurdlEJB804Vh1yfWqOLdFJVySPWultpI7rQ9x2rNGpIPrzwaGMkutV1HQI4IWKOCOhFUr7WtNuv3yxyBiuDhMAHvVpJIdQ0iK6KH7TCCMnoT3rl3kUrjPPcdhTRJqtreoi1+zw6pM8Lrjy2xkD0zVnR7u102zmModLoggFlyc+1c7nC7h1BGKvJqLvJEbgiSLPzAjFVzAdgbKO/8GC4KF5GJLsOuc4rkLmxENrGzySRlgSMjjPf6Vv2XiNrK+ltreLfZTYIB48sngke1N1u3tIbQ3EMv2kqQxJPBzx0NFrjOSjXIBJNXFWARlXYmQnII6UXE0EzKIIfLGOR0ANJbWc0zExx+Z5fJwcYFS0MuW+nxXEM5eQo8YyPcVBp8+2GWJkRo5D8wI71J9pmuJTCjKm87RnjB75pwsGtCEkYNuOAR0zUgRaWQhu13ADZgZ9RnFQQ2fmRtIG8tweDnj8qiuIXtrh4pBhgc49jV6IJJpbCPIYjJBOQCKAKoWeOUgISykEEdCauT3E1s0c5Ty5iMkY4cVXtJJDfW6dVYYceo71cv4nuAEt2JhiHIPY0tgK00b+THOgLeYCSAOQfSiYBY0cnBIAqxHdILVI8MJUPAx1qaG2S/iIzgjO1SO/1qwK9jE00crsAQAdoB5yPWrFnZJdWytKu3BIJHU/WqQ32kk8QbGQRnHWrdtIVs50Lc7OOOtOxI23iVldQwbDEcda2licafHCDyDk1Bpejjyo5pgxZxkg8DHbFavlCNQozgdMmtKcUndktlaNdiH3JNXY4Z9gIXgjOahERdgAK3LdR5YBHQYrduxJmMJojhgfwoS5I4JreCKRyAfrVCbS4XbIGKi47FL7T7inpcjuaq3VlLakHIKHpioYyznAGauyEa6SgjrUoOay4HczhT+NbBjIAx3FZyRSdwUZNKVApVwDzQxBqBsjIphp5601gewpgIKd2piZP3hipAKGgEozTsUnFKwxDQDzQRmgYXk0xDwMU4mojMC2BTi3FIBaKTNGaBijpSimg80oNAMkC0rDigEYoJyKQEZpMU4jmkoAQikIp+KMCgCPFNIqYikwKQyuRzTCKmYYzURFNiGEU2n4pCKRQykxTqTFIQlFLRQIbRS0hpgJRilxSEUhiYpMU7FJikMTFGKdSU2A2gZpx60YpAJSYp2KMUANpCKfim45oAaQaMU4ikIoASjFGKPagBKWjFGKAClAoxQBQAmKKdigCgAo70YoxQgClFJS0AOHSikpaAAUvam04UALmlFNpQaQDhRQKBTAKcKTj8aUe9AAKeKaKUUkA7NKKbTgaYC0gNFApAOBoFIDS5oC46lpBSigApwptKKYBmlFJS44pAOBpyjmmgVKoHFMByj1qQAYpKUdKQCYpwFKBk0uMCmAgoNFFIBCKKU9KSgBtJTqSkAgpc0CloAQ9aZTyKTFNgNp2aMUGgBaM0lJQAUlFFABThTacKAExSAc07FGKADFIRS00nmgANA60hpRQA4DmnU2gnigAJppNBNMJoAQ0hNBNNJoACaBSUUxDxRmm5pCeKAAmkozTSaQCmkpKKYAaBRRQAUuaSigBc0ZpM0ZoAXNGaTNGaADNBNBNJQAtGaM0hNACg80ZptLTAdSZoopAFJRSUAKTSZoJpKADNGaQmkpgLmjNJRQA7NHam5opiHA0pNJRSACaM0hNJmgAzQTSUGmIKKKKAFoxSgUuKQxBSiiigBRRSCloAMUlGaKBATxQTSZpM0AOzSZozSZp2GFFFFIRqUlLSVQISlpKSkMcDSO+1SxOAKTNBAYYIoaAp3Zku7ZghZJEIZfrRY6h9rj8mUFbhOGBqLVr2TT4k8kDJ6k/hWQNdjtC01xEgmIwAO/6UAdFc3EllGJRF5gzgiuW114dX1CGPyRDJjJJOSenSorrWLzVrGWSK5iRV/wCWJ6kfWuZe43YLM+4dCD0oES6zYjT73YpJBGcnrUV3YvZQxTlwVcZBA6Ust+90VNyxk2jAJGDii4u1ngEOSEByAR0qGUmU5mDnIcue5NMQlXqxb20k7FUAJHqcVJeWEtpCJHxgnHBoGNjuBHkbyCOhFWPnlthIHck9cdqpC2dolkAyCcVZQvbxgxyHPcHkUhFgatqSwG085jGexJOKqxNcW0wlSTYw6HNakU9u6AgIXPUP1rNurZI5gfM8wPjI9DRcZfn1czRssojmdsEEDkEVmtqOovGYg7hfYAGoUKwXAJGAOtWGu5C4eDpjB561SYmi3ba01vbCGa3EjgEB364rIkJZiSepzVtR5822Xckh6E8CqkwKSFCMEUMLC28LzSYjAJAzgnFEk+G2scEU1XdDlDg+tbMV3YCyJm03fkYZw4Jz60h2MVyZGyzE/WnFQB1BqIlWcsgITPAPpS7jnApABHPFPCAjlgKjLkdTTgkrxGUISinkgdKYDTgU9YXMXnxkDHUHqKeLcm2EwdTn+HPIpkUrwyKwPy5GR2IoAvrOLiDyriN/PYAq3TI9qqmSaNTC0ku3upJxXVfYIkv4mMTPbuMo+fuE/wBKl0yzmaa9TyPOjjBMb+vtVE3OP5HJBC+ppTjsafdyvJNJGyhVDEBB2HpUBqBlkC3kY5fAHVcf1qeRzGBCMhMZBAziqC43AE4B71cU7VA3Ej1NAC2t29vC9vJIyKTkMB0PpUQj+23qQoVGRgHHXFEzqzkZ+X0ruPBnh62GnPf36IS7ZRieVA/zimBxmoaTPp4jaQq0cnCkev8Ak1TC4JVxkeld54g0y5vroJDJFJBGdyhDyDxXNanHFd3AFrHzGArcYyaAKKandxEpHJiMADGOcD3/ABptzcRzzeYgYZGSCeKnWxCWskso2yY4HXmoZY7dbWGWN2Ex4kGehoAjRS3QVbtvMhSTBIMgwAD3qqHki+eMNke1aGEjs96knIBIJ70gGyFLh41TAkC4JA7io5ot1mZvOdyDggnjNMkIEgZDhgeeKlFtM0JnKkRn0PBNIDS0m4glsLg3oQuFwDIMkj0rEt5Nm5AcAk4qcSFY0A6EkEeopZdNmjgjuIoyyN6daBklvCATNnDxnlAO3qKvWEa21vcSecknnISAp5A9xWQJZULtyG6YI4IqZYTEwaKVT6ikwC1kCXoYLgYwBnpWxpizfYzMIy8ZkI3A8ishE3OFH3z0NbWn6rb22lw208DP5bkkqeQD7YppgT6/pMSCGaE7SeGOM8HGKgbQ3t7FpH+aaPHmITjAPcV0en3VlqdvIiAOvTkcgdqsNYPJIAwBhIAYZ4IA71okKxQ0TUpLo/ZriALsQFG6ZHTpWjcWqsSwFVbS2+z629uo/dpHlB3APbP4VsFccVSdhNGXFbhCXxg9Kt25YjDAD0qYxjOcUBQOlNsViVcY4oK5poOKeOlK47EFxCjwkMM1QhtERjgdTWnKP3ZqpC2WxVJisIlgUlEhHAqZ3GcVZzlRULRDkmhu4JWK7yxqcZpSacIrYfM4pFeJnwEwPX1pAIBQrjcQB+YqyQI+o4qu9xC74jU59aYIVsBTxzUJznOKkcHGRQjKRz2FADQScYoIIGSKRTg5Aqwgyhz0oAqhgRkGpVxjnmkkQZ4GKQEg4oAi8lVcsM5Jp1SGm0AANKTTe9L2pDEBpwPFRlgDUQmx3p2FcthsVKCD0qksmT1qyjYGaGh3JOMYNMIoLHNJnmkAucUZpMikPFADs0E03PFJmgBr5qI1M+MfhUTCkBGRRinEc0mKQ2NIpCKcRSYoBjcUmKfimkUANoIoIooGFFGKKQhO9JS0lIYUUtHagBBRQKWgBDQadRQA2jilpcUANIppFSYppFADMUYp+KbigBKKXFGKAEoopcUAJ2oFLigCgAxRilxS0gG0oFLijFMBMUAUUYoAcKKBRSAKWkpaAFFKKQUCmAtLSUvegB3aikFOFAC0opM0UAOzRmkxS0gFopKWmA4UopBRQA6lFIKUUgHAcUooHSnAGgBVFSDimqOKfg0wF609RxTB1pynFK4EoAxQRTQaXNACYopc0lACHpSGlNIaAEoooNABRRRSAKSlooAaaSlPWkoAd0FNpT0pKoAzRQaKQCCnCkFKKAFFBpaQnigBDSGkJ5pCaAFoBptFAD80hNNzSE0ABamk0E000ABNFBptMBc0maSii4C5oNFNoAXNNJpaSgQZpKDSZoGFGaM0UCFzS02jNAC5opM0ZoAXNGaSigBaSiigAop2KMUAJRS0UAFJR3pKAFpCaCabQAmaM0hopgLmjNJmjNMQtFJRSAWlzSZozQAZozTc0ZpgOzSZpM0maQDhRSClFMBcUAUUooAWg0ZppNADs0ZppNANIQuaM0hNFNjDNGaQGgmgQuaSkFLmgBc0lGaKBhS5pKUCgRp5oyKbmiiwC0Zo7U3NMY4UCkBFKOTxQBT1K2FxCpIyy9vWs7WtJt5NJWUx5kUAg56VtXN5bWcYa4cJ6Z71R/tIXcR8u3Dwn+IH+lJgcs2jCSyeWG2kBA4KnJ/KsW30eW4sZrkSgGHOY88mvSrNBDAzyMFBzyTgVizDTYreeFLyMGU5yDipEcWNHum0s34AEY7E8020snuIDgHcBkYrsYbeKLQJrVZ43wCQR2zUug2dtJoxjCYYA9+QaQ0cFGXLmNN27vjrUoTfJsu5H2jpk9DViExWWsMZoy2xiMDrWpd6bZXNjJdxko2MjHX8aBmYlqIYAZsSW45DIeh9TWedomKoSV7ZNa9rp9/NaGazikKgZPcEelZ0SKtwCVMbqRkHsaCbksjw22AYdxIyGBzg+1VBMTLknNdPf2Vlqemo4jFtcJySOhB6H9KxtQ0tLK3WSNyWzgkjg0FJmXKcsTT45CFCBgD2J6Va+2mOyMIReKorFuXLcD1FCGXZpiiKQytICKq3Ekkjl5AMn0FMCoOMH8TUw8poipGD2OelAFbeexpFRpZQoIBPcnFOZQAcHmm4PegCSWBoCAxBz6HNEcYkiaQHocYxT7SeK3n3zQiVcYweo+lWvkW1eaMxneSTGBytAFKNiGIChiRgA1r280dpZMGKmQjlT0IqjpNva3WoCG7lMcZ6EHHPapr+0EMskUMTP5bcvnOBQBSt4vPlCbiCSSAOn5U+ayeOJpAVIU4PNW7K0jju2iukYSFcxlT3pv2TUJ/MIjkW3ByXkGABQI09CvfsrC3uo5gz8K5b5Mdq6TSb+4huprMOptUBIKjkHHr3rlYdamhsjZPHDMACASAeKW31cf2eLSOExzHjeTgH3PFVcmxlXg2X1wHbJMhwSOSM1WYnjHriuml0Gyt9IN9NcGWYA5UHv2Nc2yVLRaNDUdFu9NSCWUq8cnIKHOPXNMvZoXkRoGyCORjoa1NH1mIWz2OpNutyuFZudo9Kw544o7qWOBxJGGOxwOopWBliCNpJ42WF5lUguqDJIHWvWbRLCTSY7mwgUo+OpwAfeuR8C2bWniKI3MgjaSNiiZ6nHQ1u+IdGkt9QQ2RMMNycEIcLuHqM0wMnX5ptNmP2SyQRvgefEeo7jA6Vh25sUmDrOQzKQ4fnn1rpfEUL6XokcNvBI743NN2B78fjXDzW7qilpl2uMgjnFAD7qzuomBlfPmHgZzkVA9vH9nYBgkiHOD3pwuJnws0hkWM5UjtVe7IkYSjnjBoAti/E1g/mSZmAwBjpUFs7OuxjkGqgfKketOQlehxSaAv585nO0ZBwSKlt7tlAhPK9xVYMApAPJGDx3qNQUkBBwfWgRoL5ck0ZWM/IckE44rWvdWjt2R7dWCnhk7ZrnHkeGRJNwbnoOtWLgNMpfIOe4NADr0+dMZFQKSASo6AUyziSWSQsxAIwCO5qawkQBvMKmQqQCegFRWMyQztHMAYW+U56j3FJjLBtHWMDco5A3E8CtG3torfVLMkYV8hiTkHIqjGMXZt2k3Rn7pJ6jtV6W1lkght0OCjZEncD0pCLEcZ0fU5YHjMkFz80ewnIwa6zTbiS4s42lBDHOSe9c5pL3F7d2yzJmeyBJJPEin09625fOt7iaCEkDAljAPUdxVxGaXlr5/nEfPjaD7U40K4lhWVBwwzj0oqwEIoxS0AUCYgFPC8UgFPAoACmRgiqhgWIkj1q8KZIoIoTEV1lUcVHcTKEwD1pJIyuagkAKYIBq0IjMsfQtk+lKMqcjj09qqzKVyR1FTI0r2OT9/OKuxJZDgj55QW75NRwtmcDFZqRyPh+h7+9XrJ2Mhz24osBfkUgcD61nSORNtzitTOeDUMlgGmEmcY7VKKI4ASRxVgcAinBAgwBimsQDzSBEcgwSagLc0+ZwM1BmmBIH9aUMD0quWI6URv8xBosFyxQTxSZpCeKLAQSfeJqsWIJ+tWH5NQSLkVSJZJG2CCDVkTY6ms9WIIFTBqbQJl4SAilDVUVyCATUwORUNFJk4OaWoQ2O9ODZ71Nhj6Sgmmk0AIxpuc0jN2pgbB5osBJSUtFDQDcUbadS4pDRGRTSKlIFNIpDIiKMU8rSYoAZRilxRikIbSUpooASg0lLSGIetFHeloAXtRSdqUdKAClpBS0AGKTFOoAoAaQfSkxT6MEmgCPFGOKk20YoAjxRin4FJgUANwaMU/HFG2gBuDRilxS4zQAzFKBTttBU0AJikxTscUYoAbijFOxxSYoAKKKKAClpKWgBRSikpaAFopDS0hjhSiminCgQuaM0lLiiwC0opAKWmA4UCkFKKQCgU5aaBzUiimAoFSLTQKco6UgJgBipAOKaozipOlAERGDQBTiKSkAoPFGabmlBpgOzRmkzRmgANJmlpppAFBooNMBKWkozTAM0UmaM0AFJmkzRmkApNITQSKaWoAXNJmkzRQBIKcKYtOzQA7NNY0E000XAQmm5oNJmmAuaM+lNzRmkAuaaTQTTTQApPNBpKM0xBSUUnekMDQKKKADNNJpTSZoAKCaSkpgKTTc0GkxSExxozSYpaYBRRRQAUUUUAGaKMUtACCnAUgFOHSgAxQRS0HpQA2ig000ABNGaSmmgBSaSg0maADNJRmjNMA70ZozRmgAzRmkopiFzRmikpAFFGaM0wClpKKAHClpoNGaAFJpc02igQ7NGabRQAZpelJQaAFzSZoNFAwoooxRYAopcUYoFcSlpcCjFCGhCKUCjFLigC/RSZozRYBc00mlNIaEAZxSq3zDNMJoBpiOX8dYkW3UHJz0/Co/D9heyxYafyVAyCDnIrY1myS9aEucEHrjNX7SGO1h2IMDigDmvENncxKireSTZ5wDg9qsWENl9igW5hRpDwQRV+ZQ1zKSAeO4pljaI8xLEHaMjHrSaAg0/SLRtQuMR/IOVGehq7BMlqZkMYQDkYPBFWbC0aCWSRzkvVTWIRNNbRMSFc4ODg4pWGYUMUOp6x9rmCII2IKAYJrX8QXtvBZ+THbo7SjAwMY6VkXennT9YWK3BZT0yeaiZJ4PEEa3bFUBBXf0xilYCrp2u3WkiS2hiyhJJBHT6Uywt01TUJPtLYkbnBGAR7VLc2ZutekSCYxxkZBHfA9Kngm+wamFvYy0Y+7KQMA/hRYEVr2xkt5BEtxvhyAADyv1rfXQorrTZLKR2cqu5CeoPtXLX8R/tGaZHIhc5Rh0Na7eICMG3Lw3GACSpIIHrSKRkpo8l+Da+XFbzRdXfqTWVfWs2nyfZ5JEbIzlDkV1FvZSGZri9aORTk5B6GuV1Aq1/KY2LLnjPagRCuNp9RUZJY8flS5xxTkQvnaM4GTQNIjAIHNOjjeaQRxruY9BnFCKXuUjJ4Jwauva/ZLxTDODxnntQFivPZ3FsiPNHtDdOc0yGN3fCIWPcDriprhria4WGaYEFhgk8CrNvaPZ6pAHuY40Y4MoPAB9aQEOn21vdX3lXTFSRkEdc9hWpcW93pDuoIIkXAIGQR2FWPEWhW9rYRXtteJIwIBVSMt7giqU1pqNjawz3EoeMjhQckCmIXw3KI/EFo8k20An73IPHQ11d6UvNRms/tBRc4MY6EVzEMVpfXjBiEfYCpBwQRRZJdXl/LOb3FxFwHIyTihAQahaQwX7rZ7gEJBJHGfSooTJKskMmARyCB0NWBqFxdXT29wQMnPmAdD602zguLi4uEtShIHzBuCR7U2Mob2z+8lYg8HmlkhwRtIKkZB9ajkjaORkcYweRS7wMBgSM9u1SCGOpQlWGD6GprWxN15myUIygEe9Nk8lmzyKktnjhcmQHy2BBINAFgXt2fKmM5E0PAIGCMV3mmapfeJNEAnKeZC4IkAwXI9fQ1wJhhWESxPuOcFM54qW0mmtNtxb3RjcnBUHjFAHoNtYpcEvJNdSEZB8w8cdq4OazjY3CxzfJG5GCK17PxXqVlAZhL5yEjKSr0+hHSsLUNTnv5bh4IdkMj7jgcjPb9KYDLeSMaXKE+SbPJPQj0FUXi/cmTevHbNN3fu9tR4oAFJDA4zU6cuB+JqEDmpRkED160ATQoZJAoGSaLiKSHBYYBOBVkgRuhjAHHNQ3UhcEHrxjikBVbmQCrMMoRzCeARge1VcHPvVg20ilJFG8EZyKAJtot5jGvzAgHnnBqX+z5nWOSMB/OOQAeQfQ1GGlZQ7x4IJGD3FbHhu5kmkmhJ4UAg9Tj0pXAq3ESROkcgZ8JggcmM/4UJfXELJbynMecB/UVJPIw1Rt4WN8kE54YdqrOpmLwx5fBzgDoKQWLceoT6ddwzOQSjZIAwSO9dPPqltcSW9+k223K7DjlkPYmucAe6ILRkxoAshxkgn0/z2qOOw+zCa1uIWOw5DjkD0z/nvVxZL0O8tbd4ZeLppVcZC+xq6RiuFs9bupGtHIdltWAZ8dEzjBrsxfW0vzCVR7E4q0BLThRjIBBBB6EHNANADgKeBTRTs0MYtNJp2aQ0kMjZQRgiqUkJDkjpV8ioyB3FUSUDFnqOacqbRjHFWGQDtSECncCDYMEYGDUSII3I9eatFahkGCSOtUmJokTk5qbzMcZrKku5FJAAH0qEXMxcnd1p8orm1vDdKRxkVli8dB1zTku2cHJxRyMLk0gyaiPy0CQnnOadw3WnawXEIDDimhMc+lOC4GKOc0gH5OKQmkHApGNJILjWwagcHmpC2KQnIOatCZBtAPBpRkfSg07qAKpiFj5Ge4qdcgVAmQxHapweKlopD8nFIHIpCwxTCRnrU2GSqxPfNO3Y61AhHmYqVzkZpWAa5BHHWkz0NNyCeaMgsADiqsBKDwKcKaBgUZxUtAh4FGKhMpBxmnJICSM96XKxkh5pMU4cikxSsAzFGKkApQBSGQlfSmMDmrBXNMK4pAQkGm4qYjIxTNppAMxRinkU3HNAxDSCnEUmKQBijFFFAAKUUlFACg0uaQUtABnmlFIBThQAYoIopaAExSYp2KQigBuKXFLijFIBuKcBRThTAMU0jmn4oNADMUYp9JSAZijFO20baYDNvtSEDNS44phFIBlKKCKKYC0UgpRQAtLTaWgBQacKaOtKKQDgaBSUtMBwoFJSigBwFOFNApw60APUU4ULTwKQABmpFU8cURgGpgB6UAOC0ppO1KaAGk00mlNNNABSCg0lADgaWkFGaQC5pKKSmAtBpKDSASig0hpgITQTQTTSaAAmkzzSE0ZpgLmkzSUUgFBpRSClBoAdmlzTc0maAHE8Umabk0ZoQDs00mkJNITTQCZozRRQIM0hNLmmk0DCikopAKaSkJpM0wFopM0ZpABNJmlNJQIKSlJpKYBigilNIaACkxS4ptAC0UlLQAUUUUALRSUoNADgaAaTNGaAHA0ZpM0ZoAQmkzQTTc0ABpCaCaKAEpKKCaoQGkpTTaAFozRmjNIAopKUUwCijNGaAEooooAKKXNJigQuKWkFLRcYUUUUAGaKMUuKBCUuDS4oxigBMGjFLRQMTFGKdijFMBAKUijFKOtIVxMUuKWjFMYYoxS0UAWc0oNJiigBc0U3NGaAFxTcUuaM0CsMYAjBGRS9sDpQRk0Bc0BYia3jdskZJpQLeyUzMQoA5JNMkvoLeby5Dg1k6ndRtKJIyHBG3AP8ATvSHYty+KbCFsEvj1AzVWbxXZTSg28bSlcYGOv8AhXOyXb27vb3FuCHGQR6VnSFrLdsIGTkYOaTkBvah4mju2XbAYWQ5BJGaq6lrH9qWsQ8k7oznIIrI0xklvSZQCoGeR0NO1C9SO4K26BR3GMDNHMB0egXcF3mMWwFwgJDEkk1HqFzZXsvl3EksbIeMLwDWHYX17YTC4CgKQQcjqKu3lzaXTFwjAsATJjgGle42dFDpiJAGmuFuo0+4CMAflWXqoMEwurU/uyPnGDgY/nUdprUaRQpFCXlB2MD09jW5q2rxWmlrFcCMTSDGAPlFJgmcRdaxJcXOYcRgjBwOCfWoUiSNnafgvyPpW0RaW17HJM0LW8g646E1Te5tYr5hbwiYE5APagCl/ZxkRXBO0jJMfIArSfTbF8RgskgXO4HgmomvrdHaQZhmxgKBxWdPcpOmC5BHQL0NA0R3JxwYSCpwXx1qtkEDPaniR2UI7kr70xkCsAGBBoAeCpKlyQoIzg9R6V0Go2Gk2gtJ7S4M27BeMkkY9654EYACgYOeO9XnlsvIX9yfMAxkcUAT3V3avKAw2qOgU4AFXpLqO/ggt7udBCuFQqec+9ZYt/K01bhlw27IHYioYUjeTdI2xT1AFAMsajYyaTdQyI29Ccgk859P1psV+iXBnli5Iwcd6CEIbzJJHiUEqM9DUIuIpLIQlBkdHHUUAXreaF5w9lbMzAEsjDg+9XdNRQ4v7Y4uEc74TyCPpTNBvwsZtGUCRRmM+o96g0u8Ed3ccETSEgKeATQBBfKk19NJHKCGy5BGCD6VScYOK0LhLKMPHdIwugTnaeCD0rM3ZpAOCGSYIASD1wMnFWZkTascALODgccfjUmlXqWF7HM6lgTtwOoz3rRmsrS81sxi48hZeSVOQDigDIe0e3YC4JjJGQBzkVEkxTggH61Lq0P2XUHhFybgLwJM5GKpg5pgbM2qpdIsPliHIwzjuKrqfsjPDbSqyyDJOMkVQBy4A6+laNlcpaOXEaS7wQQT0NAFB2Ibqhz3ToajByae+DIzDjJJx6U0ZByKQC4wMmrFrGJp0A5HOfpUYKEfNn8DirVu8SSOIclSOpOTTAdIS12YYRnFSXy5KDHIHNN0rTjIBdXTlbcEg7Tzx6+lEagNMxDvGG6gZBHrSYFIgoQSKs27q43PMY8dQBwKkuVj8sSQxEwsMgkVXhYKrjH3qLAWij28oLkMCDgg8EVZtLgxSxXUcWAoIkx3FZySM0KIxyEzj8atWjsLSTaMhSTnPWpaAuPe2/mMlymUJJSUDqTUGl3Pk3jv/C2Rk9BQQJPKSIjaSQQR0qa2tprRZFRd6Eg4B4I560AaNpNKZi9u+9QQWQDlgK0dU1J4rkzQrmG6QHGOQe+fxrHtSYL0XdmRmPkxE4yO4rs7DTLS8tkmkhXJJOD1B6/1qgMfw95SzX5YE2jqodQOAT61qTaXCLGJIFMgQ7kyecE9PpUtnZmyv3lh+44Ksvc46fl/WtAReXIVQfuycj29qpITRUt4/s/yxAiNzyhP3DVrFLtIJwcA9R60YqhWAGnA0znNOxQA7NITg0maQjPNAx5OaYRSg4GKCAaAIyOKYRUhHNIRQFiHFNIqQimkVSEZ1xCAxO01VZQq5HWth1BGDVYwr3FaRZDRll89TU1uQcjv1FTyWyEEgAfSqpGx8YOa0TJJI2IkwatLxVJeuasoxKjNJopMnzmjFQhxnGakU5+lRYLingVETT3JxwaiB55oSGDcoSOtQkmpzx9KiaPIOOKtEkYapBzTPIcjIIIojJztIORQFh6g5OBUoU4pUHy80FhnFSxjSeKjDHdg09jULkBqaBkwGWBFWf4D9KrROMDNTsfkzSe4IoyyGN8Hip1VshiCARSDDSDcATnvV3AIA7DpSYEIPrTHcdBSzZRTioN2RmhIdyNz83WhXIPWo3Y5pgcg1pYm5pRyAjk1MGBGaz4mJz7mrCuQazcSkyypy2KkC5qssgyDVqMZUHsazaKTHKozQ0WRkU8DBqQDipYyiyEHpTCtXHGeMVEV5oAg2e1NKcdKshMnpTzCDQBRKn0puKuPDgZAqAoaQyLFIRUhU0hWhgR45pcU7FJikAlLRijFABSikxS0IBRS0meKWgAo70UtACd6KKKAEp1JS5oAXNB6UCigBKKWjvSAQU4CkApRQAEcUxhxUhppHFAERFJSkUUAJS0lFMBRRQKKAHCgUgpRSAcKUU0UooAcKUU0U4UwHCnr1pgqRaAHipBTFFSheKAHR5FTjoKiQYqXNIAJpDRSGgBDTc0pptABRS0YFIAoopKACiiimAUUUUAB6U004009aAGk00nilPWkJpgITSUGkpALRmkzRmmA4GlzTAaUGgBc0ZpM0ZpALRmm5ophcXNNJozSUgFzSUZozQAUUUlMQUhpaQ0hiZ5oJopMU2IKTNOxSUDCkooxQAUtJRQAuab3paTFIQGg0tIaYCUUGg0AGaM0lFAC04GmiloAU0YooBoAM0UUnSgBCaSlJpKYhKSlNIaBhigCgGg0AIaKWkoEJRRRTAXNGaMUlIApM0YpaYCUvWjFLigAxS4oxS4oATFFLijFAhCKUClxS0WGJigU40GmAoxSGgGgmgVxKKKKYBRR1paBhijvSijFIQGlpDS0AwpaTBpQKALRBFJg1KRSEUkMhIpOKlK8dKjIxRcQUmKDxRmi4CUoOCDSZpM0AzE123JkklxkYyK5A2srXKgwNtJGSpyK9DvIxNAVaqh05E2SxEgjHB5H5UDOQvLIyXMcbiYxkADJ6Cm6vpiaWIwkpkV88HqOn+NdPrFoLmKGNXMcgOQQM1jy+H5zbNJcXDOQMioaA56OMEjA5PQetNMalyGDIw79q3hd2NnbBSA7DsRyfxqpHqNpJHJCLdpDJ2PWlYDKkuZpB5bybgOnFCNJ5ewH5e4p7QGGbEqFCegPpUbTeVIRsJHt0popluzvGsJCwRWVhhgevtio72cXUOZHLY6A9hVZ2yc03aWHHSmBAAMgnJwe5qXDFTIM4AppABxTS7A4B4pATRoJIyz8r79RTJAigbWBHsKYWOODirSxILEO4y55pWBFTGRntTSQCParIjja0L+ZiRTwuOtNjmi2gSQhx78ZoAeiJu86GOSRB97jpSrayzqHQqAffNJHeyQB0tz5cb9V68fWizuUtyRPGJIznIzTAmvZbkSxhjuCjkjkVVEgDkkA/UVsXWt272AtYrYhT1PU1jxxF1dtrFVPJx2oAnhniVgScKQQaYYY5FeSFgFToD3otDaB2M5IGDg47063uJLZnMIDo3UEZBFIC/bWZvZ4W0pH8yMAyBuAM960dS02LTpbeeSRWuicmNQcEcVi6bqVxpepCRARvYB0IxkHt+tdB4yspZJobuAExmMEgDkfWgk5vUrhLq8MqKVyACD7VUBxmlZdvDdaFDMfkXOOvOMCmUjUj0hxb291IQ0LkEkdhWvH4XvbmCQ6e9rJbg5Cl8s3GcA1Q024ihgEFw54cMgPKuCeQT+Vde2paV4f1SGYREs6cxxcj64pEnA6ksJIEcTRSJkSI3UEY/8Ar1TIAAAFdZrlpBrN1cajaXcUYcAmJ1wxIH8zXI5LDmmUmWrB7aK63XUe+EjB4yRUc7xfaXNsCsZORmoh0oHFMAAJ6daXaScDrUqtCYypQ7z0alOQRkDj0qQICcHBFWLXasMr9wcYqCT5nyoI45BqaAMQUA475pgXVCfY5EJKk87QeDVaGRoeYpCARgg8jFSCNzKCSpYkAZPGT61vnwh5Ns8k9yEfP8ByoJ9fT61FwOflm8zZGoxgYAB4z3pVEUdk5cfvSemcEU42/lhmDGQAnJUdBUZjM0QkCEqCQSOo+tUmALHvQuWAxySfTtV61VLm2gjEn7xjtI6ZPaogA8cMhAwgC+WBgkd6iYSWdzDJCwYbw6KOCCOxqWBtR2S2l0YZoisy4+Yc/pTrGHZerDJNgSsArjoD70271Z7pzeLDgsgBQnoR+FMtneW3jkhYDLgSA8FSD1pALNa3CXt0ViIMYywUZB981s6HNe2OmC6ckwzkj5ycoRyM+xqXRoJNS1O4XzSqyJiQE5JB6kV032RL2yktIZkVSu3BPIIxTAkjTzrW3nQYyN34mpyDjmrFvGBYW6gfdjAP1FMdOK0uBXwKbipdlG00xEW00mDUuKNlMLEJpD0pzDBppFAITNGaQnFN8xc4zTEKetApCw9aTcOxoEOI4qIipM5FNI4pDGGo2HNPNNIqkwGbARg0hhQDO0Ej1qVTjikcVSZJjzxtG5I6VJFMCmDT70YBqmh4rVakkjP+8zmpo3OOoqqTk0mSDTsK5oE5SowKrpMQoBJNSLMMg5pNDuTYJpCDTkdSM5p4INICFQQQD0qfYnXHNBA7Ckdcr1IIpFAcAVEQM8U9FZhgmmlSjHPSgLDWUkcVEynHIq2hBHFQzoRTTE0QqCBgVaz+7FRohK8VMIzsApMEVnXnI61ahfKAMeR1pnlHOMVJ5RAyKTZQsiCRSM4qmYyjYIq4Dgc1XmIPIpoTKkyg8gc1GqnPIqeQ4WmJ82cDpVpkD4sg+1SkkGmxkDAPrUr7dpNJjQiEA5Y4FaELAxjHSsoEspq9a/KMZJBNRJaFJl0Gng8VCGqRTxWTRaHbQRUbDnipQeKQjnilYBoApaaTzQDzQMcygrVdk5qcmmkAigCoy88UwirDLjNQsDmgCLFFOxRtpANooxS4pAMxS4p2KNtACCinbRSYoAKKKKAFoNAFKaQCUlFLTAUUUUUgClFJSigAzS5pKXNMBKQnilppNADD1pKU0lIBO9LSUooAKKBRTAUUooFApALTu1NpwpgKOtOFNpRSQDxT1pgqRaAJF61KOlRqMAVKoOaAJB2pxNNA4oJoAXNNJozTSaAAmgUhNKKAHUGikpAFBopCaYAaM03NAoAcTRSUppAITTD1pxphpgIaSnEc0hHFMBp60lLSUhiGiig0CFzRmm0tMBc0E0lIaAAmjNBpKBC5ozSUtAxM0ZoopWAM0UlLigBMUdqWjFMBtBpxFIaAEopaMUgExSYp2KMUwG0lOI5pMUAJRS0UCExSYpaDQA00UUGgQhpKU0lAxaWkpcUALRSig0AIabS02gANJRRTADQKWkxQAHiiiigAxRilooENIpKcaKYDcUUtLihAJRinAUUhCYpQKXFFMBaSlwaMUxiYpcUUZpAFFGaM1QBRmjNFACUUUUhAKUUgFOAoAAKcBQBTgKAEApcCnhTRtNADMUuBTgpp22gBgUUu2pNlG32oAs4pMUZozUjGnpUbdakPSmNyaYER9KbTyKYRimJiE0maDSE0IEBNJSE0maBkE8Iku4SegPJqjrckhBSMkEAA4PUVqZGcmoJreNyZHx7kmkByr6fawxxGQqGY5IkPIqsNTsbKRxbxAkjG8DIq5f2L6peiG3bMKDJc9BWVdWFtFLIInYsoz/jWYFGSeS9ug8hxk4J64rRMYt7YQsA6E43kYNRLaS2EC3TrtJOKpT3stywVsBewApjuNuRGs5SIkgdSaIiNwGQBjmoG4cfSgFSpyuSD1FAx7MuCOozkGoCcGnFSFDdjUZJJpAPJUjAzTi8jR7CxIHTNMApwODgU0AhJIwaFRz0GRSq6I43qSO+KlLDOYwdp6A0ARmJlGTj8KdGigh3wVPBHpVuKNGjySRxyDUOFDHA70ARThFfCEEeg7Usd1JFDJCMFX6gjoasSODDsEQJ7MBzUUXlLE6Sw5Ynhu4oArCrglRLWMiRMg5IHBwKrlQ3CDn0qWKyeaNiydDjGefypAaetahp2o2kFxbgpdIQHBGMjsevtXRJ4htZfCmZ5Ue52GMxg8g9Oa4poYEZSckE4IzUtzpxtQk2QYZBlSOx9KYNFORt8hNXtGsje3UkfnGJCCC+AQD2yKouMHIq9o6iWaWP7Q8DkZBXofrQBtwiTSpHsrhEljXkO68E+lR6lbPZrDcsR5I5AHUE9ADVWyu5Li5OnXz+ZCcgEjkH2NVrgm3Z4JJTLHG2BGTkEds0CKzXom3JM25Scg9wfrUDhB90fX3qW4kt5VHlwmMjvnOaZ558vZtUgdOKQIQAY96aRkHsR1FISScDOT0ApMgkHBB6Ed6oY4danB3LgAgjkE1XzzjBH1qeBfMkCkkDGaQDd7O4LHJAxTg7LnacZ60qqp3q4PXAINSR27tCTGM7eo9aQGn4c0ZdZmuUklkjEaZDL0yegNa/h1pEm1DTnDTMFITJyDjtUPgmC7eS9nhCmEKFmQ9Twen604wPaXUNxZJtuITl8nhxRYRJLb3Wh6XuKRlrmTCZHKk9j/Sqp0aKG4ntWeQSPEJEGe/cfnXUSoniDyZrmPyUtjuK56k85zXI69PdT6i92TtT7sZHcCmlcaYkGmwsI0N4MuOmMEH068VG+nQzEhZSkhB2EnIyOx9K7bRtMs5rbS78wAM5yUI4JxjNaq+HrJLi7ieIPG48yMEAFT3ANJxA87TTJ7exE5KywkZ3Keh7giiCA2rwgr5kZxKSp5A+ldnpNpZy6QbSQ5jmk2MR/AwPAJ7ZGK0T4esxK0ccZAVMAjqB7GlYDD8L6bIFk1OGJpsMQhU4wPQiqNxrmqaTqLqts0eHLlGGTg9vaugj0uXQvs15YFyrttnRjkHPQ+3Nak1obudi8YEkmCSONoHvTsIXSb8XvlSICqzIQwPVWHUfrWi8fOMVFYadHZ3MxU/K5DhccA4wT+NXHALE0xlF0welMxVp1BzUBUg07gMKe1RkGrIXIpDGO4oQFN+KhLYq68SntULWysCOR9KpElQuCMVE4GcipzZmMkgk+xpkke1QT3qkBWaTacZpBIT0NRTffI70xWK1diTQR8jBNONUlcgg5qZJcnBpNDTHkc00jFPJBFMY8VNguAGTUjr8mR1FQg4YZqfdlaYNGVeeYWGBmq7wFcYGMjJ+takiAkmqsqN2FXFktGf0NLinMh3HIxTSCOCMVqmibDSKbg1KF3HHel8hyRgdTRdCsNAKnNWYXOcmkMRA5FIo2ipbKSLSnODT+COarq4AGelWAQY9wqWNAB83HSnMuRUIlOcVZBBAxSsMgCgHgUoG7ginEEHGKVRzSuMQR46CnhTjmpAMCgVLYWGAc804jIx2paUUrjsU5I2AIFVTHJySRj0rUZQRg1CYRnkVaZLRlYLNgcmrAgMcfTk8mrogQNuAApXUNGeOgquYVijsY9BS7TjBpyvhiMU4AEcdaq4rEYUAcdqlgkFN2EinRxFegpS2Gi+BxTlOBUcZOBnrUgNYstDwaUtTQaQmpY0ISSaQGg9aTvQMdmgmm0vUUEjCajYZqYimlRQUQEUmKmKc9Kbs9qQEWKXFOxijFADAKXHFO20u3ikAzFJipNpo2+1AEZFAFSbfagr7UAMxxSVJtpNvFADMUlOIppFABRSUtABSikpRSAXNJRmjNMBKaTzTqCARSAjoIp2KDTAbRS9aDQAlFFFCAWloxQBQA4CloFAFIBRTlFIKcBxTAcKkUZNMAqVBjFICUDgVIopg6VKtAC4ppp2aax5oAQmmk0GkNABmnCmilBpAOJpM0hNJmmApNIaDSE0AKOlFJSikAooNFFMBppMU4ikxQAgHFIaXtRigCMikxTyKMUwGYpMU8imkUANop1GKAG0UuKMUCG0U7FGKQxuKO9BFBoAQ0lLSUwFpaSlFIBcUUUUwDFIaXNFIQmKKXNNpjFpCaKQ0gA0lBNITTAUmkJoJphNAh2aKSjNAC0lLRigBtGKdikxQAg60opBTh1pgOFIRS0GkAzvSEU402mAnejnvS0UCEFBpRQaY7iUlFFIAooopiA0AUpoFABxSGlooAMUopMUo60MQoFKBQBTgKAExQRTsUmKBjDSVIQKaVqgG0UuKMUAJRilwaXBNIQgFKBSgU4CgBoBp4BpwU08LQAwA1Iq8U4LT1WhANCe1O2Z7VIF9qlWIEZNMCts9qNuD0q35YA4FMZBzxSAr8UcVLso2e1ADMUhpaMVIxhNJmnkCk2imIYelMPWpSuRTSvtQmMgNNINTFPamlfamIipCKeV56U0igaIyinrn86jniQ2soAOSvrUxFJTAp6XEkVr8oxnOawNPsvM1O7EwwUOMV0yKIQQOATnHpTVQGYyAAEjnHeoaAwNWhE6QWZ45zisybwzdQ3OYoiyDqSegrrZrdWmSRVw6nOavElzkDrSsM88l09ZZSWQ8DgKOePaq01qIbAyBDnOAO4+td3qSWcCGYyRxzAE5z1PHFclJPLqkogggGBkkr3FAzn2PGKZW/HHpkZYuheRTgAZzWfeI1/dSTW8ZC988HIpAUkQt07VKY0jxkgkjII7UlqkkkhjjXcSD3xWhYWQgSaS6VSoGNucmgDOIANTSwGOBZUkVsjJXoQfSopGV5C0fCnoKbnFMBxkY4G447igP2qMggZoB70gLId0X5TgelOMoZCCeTUKOTwfSmHg4oAepIOVOD2NSB3aVAJiWJwcDGKgVsGrukTQwaoklwcRkEE479qAI5IUVz5pEh9RS3E3mQCFCwAOdh5Fa11p0aTSXOxWt5CSHzyp9qwZCPNJU5HY0ANKnABrQ0maG1kdnBDkcMOgqjnIpu6gCVJC92007ckk5xnmppntpP3oBMgIyD0YelVRz1pCM0ATXhtzIrW/AIyV9DQlxbxxlTACSME1WYYOKVRxQBJA/lTK+M4OcU2Z/Mnd8Yyc49Kkjgd4mlUZVTyfaon2g5HegABJp8TFZQR1zio1OKfGCWz2BpgTAgyYzyTkj0rV0e3e4uZokmWNwMgEfexzgVjk4diBznINdHbQW03hyK6SPF/DNw4OMjPIP6UgNvw4bvTtWMjxN5M42TdgD2P8+a3n0T7ddpuCLFGTgg5JGc0ukTG5vI1eAwSCP51JBBGOCK3kjKDpQBTGkxGaZXwYZQAQPYVwl/pKML+SJ2ENuQqA8gnPSvQrwzG3KwnBPBPcCq8OnW6ae9my5D/eJ6k1SAqaAS+iW8MigSWpCcdO+CPwranuBBbyXBAOwZx3rJ0WJ4I7qKQklZSBn0FaM0QnQAjIByQehoQHB6lpt1Z6fFqMJkCSOTKAcAEHg475r0SErNbQTDkvGpJ9yAao6naedo72qnC8HOM9Dmr1uf9Fh/3B/Kk2BIQGQo4BUjBB70IBGgRc4AwM9cUmacOaQNjhwc00uc4pw4FNPJoAU8io2XNOAJFLjigBoXApCBtoPBoByKAIXB5pEyDzUjDNMxTuIWQBl6VTmti564x7VeAJpCtFxmJLp0zuWBBH1qCazljAOM1vshFIFDAgitOYkxI7STYCVqKVTGMkYrceMIOOlZ80YkYg0+a4rFFJyTgmpgcnFQvblX4FO3bcZ61QE7IAMigHjFKjgqKdtHapYDCM00qPSpdtG0YpXCxA0UbjBA+tQm1jGRgkdqt4Ao4qrsLFBrYIxAGD61GS4faTkY71osARUEkakDiq5mKxEELrk9aa0YUYPFTrkDGCKQgEYIo5hWKgTJxUoBUFT0qQIAeKVgCc4ouFiAISc4qdCQQKQDnFPAouMlyCKAoBzjrUW7HSpUbPWpY0OxRS5BpKljEpRxSUtIApCMmlooATFKAMYoxThRdgVpoQHyFxn3pgQA1bYZFN8tfSr5hWKzAgjipUAIFP2juKcAB0obuFgUYAxT6QUtSUhM0pNITSGpBC03vS0GgYhNGaQ0maAHZpQKTNKtABtpdoxS0YoAiZOTxUe2rW0mkMftSYEAjPpS7ParKpx0pTEeuKQFbb7UbfapjGfSlCe1AFcqaMVOV9qbs9qAISOaaRUxU0hT2oAgK00irBQVGRzSAiIoxUhXjpTcUwGig8UuKQ0AJmkopQKACloxS0ANPSmGnnpTSKAEo7UuKMUAJjilxRilxQAlKBTsUoFIBAKcBQFp4X2oAQLzTwOOlOVOelSqntTAYq89KlC4FKqY7U4CkAAVIOlMA6U8UABpppTTTQA00lBpKAHUZxSZoNABmjNNzRmgB2aSkzRmgBwFOA4oFOFADSKAKU0maAEpDS0GgBtFKaTHNAwoxS4pcUxDcUbRTsUYoAjK47UmKl25oK0gIsUm01JtpMUAMxSU6kxzQAhFNp9IRTAZijFOxRikA00CnhaCvtTAbRRigUAFFLijFAhKaadikIoGNzSE04ikIoENpCadimkUDG0Yp2KMUxDaKXBpKAHinY4poHFPA4oAYRRinEU0ikA3vS0EUCmIdijtR2opDEPSkpaQ0xDcUUUUDEopcUYpiG0U6m0AFLijFFAC9qMUAc0uKAExRinbTSge1ADQKcBTgtOAoQCYoANP2mnBeKEgI9ppcYp5GKQjNUAwimkU400jmkITFJg08ClCkmgBgBpwX1p4Q04IaAIwtPVOelOCe1PC0wGhOc4p4WnYpyrmkA0A+lPRST0xUgAFOAoAUYAp2RTKCKAHbqQ80maAeaAuGKMCn4NBBFAXKtFNVgRjcCR1FOFSAuKMUoBPQZp20jqQPxoGMxShKeF79qULQBA0feozHV0rkVGyYouBTZOajKc1aIGaaYwe1CdgKhFNIwCfQVaMQPammIAHIyMdKdwMS5ljhhM9y5CjoB1JqnZa7JdylI7XEIOC7nGB60a3azXiIjuFDP8AKq9cetUNYjGl6fHZQ/6xxlz3OaGBeufFlhbzmIQySY6lTxmqEd7qmoTq9vcmGGQ4AyOB34qs/h9F0mKabcJJCBwM8HvW1b2X2W6s42HyADnvS5gMq9tLK2uWS5Mk0xHVjkE/TtWl4U05I4pLsjJkyo9hzUd9ai41SaYLnywMr6ir1he21nE8cRLR/f6YCE9qVxmLd2cOj6/5srN5MnzIcZwT2/Wsi5QSahcTRSbIckg5q5rN++sXqiQeTGg+XcOCR71z7SMCQGOOxFAx0DtBcNIhxxgGnTNNI295iwb0NRQxzTMQgzjrTfLfcQByOtADtpCknAApvPam5IOD27U4EYpAIATQeDinIcMDTWADEgYFAxeakQbwc8kVGACcE496cMhuCOaLCBxjkU6MxqUkcBwDkoe9GQD83TvUZGDgdO30pgat3rDXluIBCI0GMYPIFZ20E4Jx71GCSfLVSXPQCrlxYNawJI7jL/wE8igCoQQSO1JmrFvCZmI6ADJNLDa+dceWDjgnNICKIx7X3g5/hIqQoNgYHg9vSmyIEcqOg4pgODQBK9qTGZQc47Y5piRAw7+g71PFckYxg47EcU27aVGWJlVUIyABjFAB9pP2TyVJB7kjqKqkVNKI1YeW+4EemMVHjcRgAHFADcHtUsZzkUzkcYINSgFDn1FACkcD64rtfCse03Gl3AjkhkTzIwepOOcD8BXFxqZMgYwBkgnFWLS/urS9t7pJHLQHIx1x3FAHdeH9Zli1W3tLwFdrGPzG6AHOAa7txgkV5haXEGu+KQ4hc285wSRghsdfzFegabZXtpCUuLtpRyAGGTjsc0AWj0xTDUhFJimIrlQrlwME9cd6eJCBSumRVZiynGM84ppgWt4YFSeCMGpEOECjoBgVWjTByDke9WFoauCZMlOBIpgOKlGDUjDNJ/FTsUh60gHCg00GlJ4oAY45poFKxyD600A45pgPApKUUYpANpDT+KaRTCw04NRHhgRUhYA4prbSDTAZIMoaz5Bsc5q8x4xWffRuxDKcYOTVokjlG5dy8n0qpIeeeDVpAQvWoLhAPmIq0xMIz0OasBgehqgsg3YBqdJBnrVNCuWhzS4qMOMZpwf3rOw7iEUhGKUmkJ4oATNMPWjJBzTM5bk1SQEmwEc0wpg1KoyBSlc0mwIDGSfakMZxzU+BS4BFK4FTBB5pQanZR3ppiGMiqQEI681IOlIOflI5HQ0mCrHnimIkD4ODTwaiOMZ70Kxzg1NhpkooAoBpaTRVxDQKDRSEGaXNJRSGOBoxTQadmmDQ0ilopBSuFh1LTaWgBaMUUCkDDijFOFIaYEZGDSYOacetApiExSikJxSBhmiwyUdKXBzSA8Uo60gHKKeFzQBxTlHNJjFRORxUuz2pUGTirCxZA71IFVosqeKgK4OK1jGAhyO1UZI+TxQBXIGKYw5qYrjIpmwk0wItppMVOU9qaUNAEDDFRkZNWimRzUTR4NAEBGKaalZTg1GRSAjNNIzUhWm7KYDMUoFPC0uykA0CjFOwaQg0AMI5owKdijHFADcUAU4ClC0wG7aXaKfg0AHNIBmKcFp4WlC+1ADADUyJkikVeelTouCKAFWPB6VIFApcUtMBCBSEU40hpAIBRmikpAKaYaeaaelMBlFGKXFMBtGacQKaaQAaaTTiKaaYBmnDrTcU4DmgB4pxpBQaQDSaKDTTQA7NBNJSgE0wG4pwBpwUYp200AMwaXFOwaNpoAjxRinYoxQAAc07FIBzTscUAMI4ppFONJQAzb7UEVIBTSKAIyKAKcRQBQAmKNtSBaUJ7UgI9tBWpdvtSFfamIhK0zac9Ks7KaVFAyML7UFfapcUhFAEWymkAVKRSECgCLGRSFcVIRimEUARkUhFPxQRTER4pcU7FGKAGEUYp+KMUAKBxS4opcUANIppFSEUhFAiLFGKeRSYoGNpcUuKXFMQwikIp5FNIpDEpMU7vRimFxMUhFOwaQigQ00lOxSgCgBMUYp4ApwXNMBgU04KTUgQUoWgCMLSgGpNtKBSsAwClC4pwFOCk00gG4oxT9ppdtAiMjikxTyMUhFAEeKXZTsUuKAGYpwWnAUAUwFC0oGTQBTlHelYBQntTgoozQCaYDtoAoGKTNKKAHjHelyO1R5NAzngGgCZVzT/AC/cVGpIIqUZNIBpXFM4B5qwFB69KpzzBHwq5HcnilcCyCCKJBhc1Qk1W2tovnDu3oBWNd+IriVwIozGgz8pPNHMgGx38a4EbxPM56bxkfStyKOXYCwryQyZ5CMh7EHkVsaTrWsRSCO2nMqf3ZTmkM7W4uLy4uDBaSKgHUYwQafFonzbri6lkk74fFc8viC1u5vLvont7oEgyQ5wfrV+21uKGYW80kzZ+6xPOKANk29zaKTauJF7xuefwNSWl/DeMY8mOYHBjcYIP171BDqBkJ8lTLjsOtNu44b0K0lvNDMpyJFHOfekBqkAdaAMisDU9WvbGyAmtfPUnHmAdK3dPnjvbCGaFg2Rzg5wfSgCOWMg5AqEHFaZjDDBFU5rYg5UUAVywFNJBBAqQwt6VGY2HUUwKyW8KNv2At6msi60/wC13DSvxk4AxnArdIIqIpTAz763/wCJeqIOEIP5U14zcJGyjgAEGrsyF4XUDqMU1FMcKgjBAxSaAw72cWllcSk4ycZrBmvLd5EhikkCkc7B3966e+077XD5R6E5NZktvb6bdFUj/eFOFUck0rDMvX1SaO18kA5BHAxnpWFNb+XkOQpHY960Li+kmuCbwmPYcKAMDNUNQuPtMxcdD0oGQwGQMwibBI596bkjkkg+uaslYTChiJWUdc96gkRkbDDBpAMpRSkcdKOaAAcEH0pX5JI6GmGnDpjtQA00oBqQQtuAKkA+tOIy20EAA9OuaYEb4wMdehFBBBwfTNWYBbJMGuELJjGB2qO48kznyCfL7A0ALZNDHexyTlgqnII7GrWrXgvrkMpUqvQgYJ+tZ/lyHkDIoAI60AXInMdu5HcYpLWQw3UbepwfpT4ED6TLNnJQ4PHWoSPMhiYEBuvPfFICaREFwxcZXOSKpSlDMTGCF7AjmrRkEsJjU5nPQdz9Kq9ev3u4PWgBrZCGtS/+wvpFnJDLm4IAcdSPUVmkZFKnyEHAP1FADVHQmpo/9amRxnmhmjJ4BHHpUtqpc7wD5Y6k8YoAJkJkyBkdzUeKvfI84CuGGQCB6etSTR20QmBjQgkFCBkg0gIYLe3e1Z23LIMcDgZNK8Nzppw67TImVPUGrGlrNJMgtl8xicMpHBHpVrWR9rmEYDI0KgMjDkGmmB0Pg/QpLPV7W7mkVt8ZJUDhSehxXoLdR9K5rw5BNZGK3J8xTCskbMOQO4roRMHAPTigAIpmKkIpCKAIiBTGjBqQ03tTQrlcNsJ5wKkWboc8UkkYIJqP7vXpVAXUcMMipQ2KzoyV5B61KJWJwTUtDNAMDSE5qur8VKGzSAXvSHrSZpAaAHU4UwmjNMB9LUe7BpS4UZJ781IDqCOKMgDJPFKGDDg5oAhkTAyKhOatMMioCvzGncCPGRVeRQVII71c2N6VHJEdh4qkxNGdIMDI7VUkkBBDitBo8jGKikst4xjHvVJiaMd12nIPBpEd9w449atNZylgGj4HvUwiAHKAGteZEWIGlIQc0JN83Jqe4tiI8qARnsaia1+UMOKLoZIrg96fnNV9hWPJOKakpAwTSsFydj2qF87gQcVICG6daY4796dguWYzhRTs1WRjkc1ODzUNDHdaMU5QDTiBSGQOSBmmCQEAVLIPlNUZAQciriJllhjnjPtTWIKEjqOaqmYjg05Jjn1GOadhXCOYOxB7GrGMnI6VSaMpIHUgqRnirEL/ACANRyiuWAeKXPNR544ozzUNFIlzRTATSg0mhpjsUlLRipsMKBRijFAC0UUUDEpaSloAUUA0ClA4pCFFFFLTAjbrSA4pxFMY4PNNIQrU1QSelO4OKljXjimAzOOMVIozTvKyc1IsYAHFJjAKfSnKOalwCtAUZqWMlQDIq1HjaKrJViM8CpAkcfLUDx5GcVOSCKaR60AU2iyc4phjx2q6QKicgcYoArlOOlMKY7VOKaw4oAqkYNNI4qRjzUTHmgCNlFRMvNTE00imBXKmk2mpitJjmgCMg0mDUuKQigBnNBFO5oI4oAjxRinEGgA0ANAFOA9KcFpwU0ANxQAakCmlC0AMANPC04DNLtwaQCqnHSngYpFGKdRcAzSg0lGcUAKTTSaCaQmgBTQDTSaBTAd1NBFKBS0AMxSZxTiaaRzQAhNNNLSYoAKaetOxQRQDGDrTh1opRQDHjpQabS0AKaZTzTaAEFSAcUwdaePWkBIBxUgTIpikVID0ApgGz2phXmrHGKjbrQBEV9qaVqUmmk0AMxiil60YoAjI5pMVJijFADB0pcZpcGnqmaAIip9KAoqyY+M4puz2pAMVeelPK09F55FOK8dKAIStNIFSlaClMRDijFTbPak2e1AyEqaaQasFPamlRQBXINNIqZhUZBoAjNMxUhFIRQIjIpMU7aaMUwExRinYoxQAzFLilxS4piEApcUYoIoASkNONNNIBpFAFOopjG4pcUuKXFAhpAxTCKkIpuKAEApcUoBxTgvrTAYRxTcVLtFJt9qQEe32pQtSbfalC+1MQwKacqnFSheOlAWgYgHFOC0oBpwFAhuz2pCnPSpcUYoAjCc07bjtTsUE4oAaaQA08AE07aB2oAhINNxU5Az0pm0elAEeKMVIVAph4pgLtFJwDSFqjYnNNIVyXIzTtwqDfxSbqdhXLGaXJqEOQKUvRYLkoalDc81BvA6GnBiTSsFy3GULDJqwFQ9wPrWcWKjOKR7ohCTwAKTGmXzJCHxvyR1AGcVYRFYcGuRbWTG+EQkDsRVyHV5ZOASv1FRzFXOlaEBCxIAHXJrmtZ1GK3cohznqenpVe71aaYmJZi/TIUVk3IjI3OMyelZthcZJqEkjEg0w3MXVxlu/NUWkJc5AA7AClADckc1IGXZJ9vmMUrHaBkY7GlsrUG/eMuwROjKcGr1/pX9nSQm0fdIRyvUk1oW+lKbJXMOJjy5JIH1rQoztIsEu9QkaYl41HU9QfWqN7BcXGpyQwGSfacLg9BXTiyNohhtJFYzDJBPQ1f0XSI9LhknnwJiMkjmmmSYdpJqGhLHNOGEbkA88jn0FdRP4mhh0t7mA73AOFYYxXMS6pbXGpyTXrEQg4QkcH2xSS6ppk7eXBEGPbnGaYCWfjGaPzRfRGYseAMYA9MVEuvmGR7ixuntZGOTGF4NVNWubNDGbcgSKOQB0NUYbuF7kS6hE0kI6kUgPRdF1fWLyySc3FnMp/v4B/GunSQOieegRmGRg8Z9K8huH0yYg6cLiPu6jJBFb3hvxNaCM2U0xEg4Q3DHBHtnOKAPQZIY0G5wAPc1Re7sD0lz9Bmufv9WS3USXLlYW/wCeb70Yfh0q/p/ifQysaF4AH79SCfUUAXWMMgzESR64IqF4/mxW40C3ESvDMgBGQyEEEVjXsWo28EkjTWwjQZLkH+VMDC1SV4bjZZT/AL48sh5GOKqHxElrKI9RjAJ/5aRkEZ96xdT1u7lEkcghBBxvj6Gse4srsRJcSDMJ/iz0oA71tb0ggEXQyemASawtS1iK1uTKAs7OOCD0/SueaUxQrGAvIzkdaqk5cAdScCpGJe3f2y5aYpsLdQBxmoEXLAGtBo/s4KXEQYvyGByMVW8kI4wc596aYxuwEEjqDTZnMoHA47+tTEkIVLAY5qAFtnXjNIAAGCO9Ckcg0owOSDx1NSsEcARjoPXrTAr7CTwM0PG8YBYYz0q5Y2pmcs2Qq5yAetJPaEusiv8Au3IAz2pANh+zvb7ppHDg4AB61A21WypOD0NTXtobNwDIGyM5AxVYAsOTxTAkkIIQnjIqMjBp0hygHpUYJpAP68ZI+lBJOc8mlAzyOtBGDkjFFwJLeWZbWW3ABRuTnnpU9kkt80UEZC4JOccAe9T2FhLctGsUoXzQcgjj8662H7FZWKQwlHuosjgYOfehiKuix2mmRzyXUcUk8ZyjIuWIPtXNa4beS8+0wCRRNlmRxjBz2ruf7OtrTTGnuIBNI2STjBGewrmLvw/qV7aNdl4ykYOIwckgelA0zItY0icG9iZYZB8jepqqSAxAOQDwcdR61bvdQ+1WdvCY9pi6n3AxUVqY1uYXlGUB55oAm09TNKyFSVA7jgfjU4hub238mDDlDyiDkD1NaEV5BbtNcRoJSThs8BR7cVHBefYtQ+1REi3uODkYAH+TQBBZWLw2S3rKrIzmIj+JTTpbSTy5AwBkblcHgite1kjbS3TAIF0HGRnA5pl/BbR6iZ7dHmDkERxqSpJ64NAG5pfhqKw8OHUEcvdTRgqO2SeAOeKo2RkGo3VxfwqFkXyZFxkg468Vd0S7uL/TTYSRSNZxSDdIODGOpB+ldZZaFZWkpaCSSRXGSGwRmgCW0s0hkjuF/wCeIQYHHQcioLuIQ3KSA/u3PQcAH0/Wqml6ssl7cWTjb5bkR8YyB1FaV3F9qtQqHADAg47igCwGDqGHQjNMY0qcxjtxTSKAGk0maU9aZmhAKaieMMMZxUmaDTTAYoCJg9qhWTM2B0qWYExMB3FUbaOSLIc89apCNEPggE1MH4rPZieM4PrViJjsGTk0mhloNQDUQNODetSBLmim0ZpANkbIwOp6VmC/eO4jtioeZhnB4rTKbsEHBHQjqK52/wBLgE7XV1O4kRCUfODxyCD+NMDXv7+3tVUzZPTAAycGn296l3p8k0G5VwQNwwa48a5Dd+U14ZS6DAkQdR2z61RN9K9yFWU8NtDhiAR2yKm5Vj0eC5ieyWcygRkZBPGamCZUH1Ga5WPVEudKayitiPLUDfjgj1Fb1lqPnRQhkZcqOT0J74/w4pkmgEGOlDICMYojcSAkZwKcCGGQQRTArG356UC39qslaAKAKElv1FUprfGRiteRepqpIMk1aAy5t8ce1Rn2pkRLJz19KuyRggnFUlBEhBFUmTYbNGTGcDJqm8ZCk4rTIyKgZQSRiqTsJooJIQcGrC4YZqG7hdB5ijIHWoorj5QKvcnYtqME8YqVeahSTIHORUynAyDxSaGmShgBzQZPQ1CXqN5ABwamw7k7uCp5qrJ3qMzmmvMSKuKsJsZKFwTnmokYqTg9RigsT1puKsklSQ4xnirEZG0g/hVRWqeOULxgE+9JoEyyjA8UE4JqFThiT0zUhbOahoaZIDnFPBqqGIbGeKnQ7hmk0UmPzTx0qM5pwPFS0CH0UClxUlIMUUUhNKwBSimind6YhaWgUGlYoUUh60DrQaZI0k0xkL9OtSEcULjNMQxUIIyKtIMUwCnBgCBQBOoyakCjFV0cg9M1YVwRnpUsodjAoxTS1OHTNIY4GpUYACoKcDipaAshhQWGKgBNKTxQArOOmajZsmmt1pmTSAlFBwRUe73oDc0wI5F54qJl9qsnBqJhQBDtpMCpCMU0igCEjmkxUhFNIoAaBzzQVFOxRQBGVpMGpsCkxTAixRipdtGKAGAU4CgCnAUAJijFOpKAFHFKKQUUgHClzTc0hNAD80maaCaUmgAJozTSaM0wFxmngYApoBJFSAUAGKKXHFIaAGmmnNOxmjFADMUhp+KQimA2kxTsUlIBMUAc0tGKAFzRRgmgCkAYpcUoApSKYDBTgaaaUDigB4JpQTTAKcBzQBKpYnrUgXIpsaHg1KDigBhSmkVKT8tNxQBCRg0hqRwO1R4oATBpQKcBS4oAQCpYgMimhTipEGKQE4QY6VEyck1Mp45ocgDgUAQhfal2ZoB5qZVzigCHZS+XVnZ7UhT2pgVSntTSlWChHaoiOaQERGKjYGpiKjYUwISKYakI5ppHFMRERTCKkINIRxQBERRTiDSYoAQClApQKcBQBGRSgVJtFIVpiI6DTiOabikMQim0/FIRTENxRilooAKWkpaADFAFKBSgZoAMD0oAFLilAoAQLShKeBTtpoAiK+lKEJNShM1KsXAobAg24pQlWhD7UeVjtRcCttNKF9qnMZz0o2e1FwIQopwT2p4Q56U8J7UXCxF5foKjKNnkVcCgCmsme1CYFUACnA5OBzTzET2p6Q7e1O4EZTjpTSuOtTMQtRFgeaaEyNhgVCx5qZmBGAOaaEB5I5pgQgkik25qbaKQgDoBTRLRCUxRwKc1Rn1piFyKQmmk00mnYLj8ipFlAAGagzkUwjIxk/hRYLmiro3Wi5jQ254rLDyg8Emqd/cXpjIOSnYAVnIaKcqsbgkFW9qmaaVEOWT8BWUFAbcxOfrTi8ch2FyAepJrGTNEhJZn3khyPpUDXDZ5PNStCA4SMliemaspYIIWeePLAZxmoYWIbC0F7KQ7YAGfrV66sYLeLMYIbsSc1RiZS4KDbjpjio7m5YfKWNMCggaVgYpXEnd3PAFFkJrjUkheWeeEZ37CRmqF1qMt7djASKMDGR1rWtNZTTLMwoEd+cSD3/nViOpMek6XAlzJEYsZwCSSK5/VPGizAxW0KlD0J5z+Vc3fard37ZmfP4VWXb1ZcmhFGufEN1NEbcQQCNhg5HNQw26TOF3Ksh7scCs1iByBSOkwjD+W23scUwNbULM2CqXdJAfTqKqWpBJKuQvdT0IrPO89S49iTVmGTYpG0H60CNJru2toQbVm88nBXGQKzATdTDJVSR1xTSC74AyT0FTTWLwwrK7EEnG3uKAJ7V4rS4IvY5JIeCAGIAP0zTjPp8l7uYOkJIGQMkVmggjHJ/WpbaMTXKRlwmTkE9KAOrh1I6baCfRNSmiA6xyHIP4Gm/8ACRanrzSx3E07jaSVhXOfyrNmhe7YwllaRRhSo4PtWr4WsdSsJHuoXjBUlJreQYZh6igLnOXYkhYo8bIO24c1uW9tNf6Ip+3Qxxgcgjkc/WrHiWMa3Pvs0eJoxlxIuMnjgc1h2wMt3a2lzGIsuA5J4I9KYrlfycqwRC7jPI6Go44pHXeB0INdrrCaZp1tJb2BjZyuSoOT9a5CKOe4iaIzCMDGQTg0mgDULuK5ESrGwdR+VVthCb5AQO/NWVt4bZ90kpDjowORU6xpcWk0spPI4xwCPWgZSkeAxhUHzetVypPerCWxZkCkYYZyeKjKOl2Ihw4IAPYUhiK7BNhojGSv4mrd7YvahCWDFuoHUVSJaJ+hVh6imBJK7CXEbkA+lBd8hGcNjoRUbMX5PJ9afp8AuLxoySCRkYpAJMhMXmFiSOxp32RxZrcBgykZYDt+NOjXeSJA3l5IOKa0JFyYLR2I7DpmmAFITGpiJJPYnNVyMGrptDFbmSZZI2BIyT1qoBlqTAvCzEdrFOWyJCARjpTpIYw2I5Vf0PTNPtIBPIEkZkjUEjceM1EphF5IJ8BQcDA6ntSAhkmmt3MYkKY7A8Vfs9SCxRo8KldwMj55IzVCcxEl0AYdsipdNktmnYXG1FKnB6DNVcGj0m/njOjGTcB+7BXPUnqK5STVJr3fDCBaoRksCc5+lWdMu1h0WRwEnkt3O0PyAtb9jeaXqWmS3Yt0BQEOm3kH06c0mB5bKIySUznJBJOcn1pCcIAat3FlMJmmMRSF3JBJ5ANOkDPEI4ypVcE7eCDQBELgpAIw+FPUY7VI29bBGE+Y88IRyKqFCBz2oTBYAnA9aALbFpvL2714wSTx+ddBb639lsUNu4DodhhIyCKwrS5ktVZEG+NzgqT0J7ipdPSOSaaN4WMOMn+8Pf3pAdppetRJe28sTiGSYYlQj5JB7nsf8a7awvba5jkkhkjGwHdGDgofpXkV3DFZXsME4MluyBgSOQPWtXRre3ltprmJpA2ShAc/OB0HNNMDprOwIVrqRP3kkgkDZ98/410S3lut8bYklnG8ADp7VxtpqDDRHkkmmdVyDCpGYx/WtDSzbx38KxyNcSPHvhIcZx3BoA6eRlUEk9KhWQPnFZT3V7fSlbe3EYB++ZOv4VqngAn7x5P1oAQ02loNMBtFFFAC9qjccVIpG7BOKeVBU8ZoAqttCfWmx4HQ0PC5PUgjpigRMMA9fWqEWFYHpTm5Xpk9vrTUQKKJCuCrZIIxgdTUsYjzFAwZ8EDOAaEkyAQWGeme9cxqmpPDMQ+JLdDh1BwSe3Peqq67dG2ieJwjOShRx8gHrUsDs7i4FtatKSMpjIJxnNZt/cWmpaFNJcxlY87ASeQScZFYZ1GFba0j8wSyICGC5IyD780yXUluJRYuvl28mHAIIGfyoAz7MxW9w+yR5rdfl2FcjB7jnmmrbWU99Lb73ViB5XGMn0PpTr21Ns4ktWdDzgoMgnFULGSYyvcOnmSAFfnGMHvxWbKTNuKdLWKb7RN5chTaU3AnPbinWtrbSeTK140eSCqmQ8sOuaw7ewlvZ3kBULCpYuTkE9sflW7DYo2mte3bSFwQCiJnr3IHIoTBmjFq8NrNHu1ERB2JC43DPuRXR6frFreyiCORXkIJHljII9favOp2tJ7OWRF8uSHBjIOQT7eldl4XmE+npduiJcEEMFGCwHciqTFY6QnmkoDBgD6ikzk1QhrjOapzkJyTj09zV5l4rMvIfPi2klWOSpHYg1SY7EbuqKdxxmoWAGDkH6VlXuptYXcVrMoYEbiSRkAelWI5hHMSVYQPyhAyMe/pVXFYtYpjAAZzzUmQ6sUwwBwSDVeUkHGCKpCZOgV0wRkGqs2lLICYSFPcHvVq3bCgGreOKLiMH+z7qJiMgqOnNNDshIIwRWy5z1qhcwK4YqPm7VopXIaK/mbuDSFcqeahDgHDAg1IDk4HSqsJkTcHFNqYxbu4FM8lh6UxEWKTacU/kHBpcg0wIsEUAkHIpSOaMYpgPDknk05HOahNKDiiwFk89KkSTaMVFGQQATTyh61DRSZOJAe9PBzVZOGFWVHGamSGhwPOKeKaBzTqzsUmNJwKaXGaJOFOKhJIGTTSFcmBGKUuB3qFW460jFj06U7BcnEnGaXzAe9VQG5I7UoYg80coXLIbPel31X84Dg0GUEcUcoXLO7NKDiqYmYHg1KkxPWjlC5ZDUm4Z5qNpBt461HvosFy6p4FSqeKpJJ0GanRuKloaZaB9RmnZwOKiV8in5zUFC7uRT81EOtOzSYD80FqZk00mgB7EGmE03dSZpAOzSg0zdRuoAfmkNIDSk0DGEU3FSnpTCKQhhFNINPIppFMBuKMGlopgGKAKKU0AJikNKaMUAMxThRRSAM0ZpMUUwClptLQAUmaKMUAANBNGKMUAFKBzSU9B3oAcoHHFPwKQDFLQIU03FOoxQMbijFOxRiqAZikIp+KQjmgCPFJinGipATFGKWlC0AIBSgClxS4oC4gFLspwHNPCigCAr7UoGKm2CmkDOKAGAA1IqUm3FSopxQBNHGCvUCmuMHFMZiM4pvmE0AOzQTxxTA2TSk4oAD0pvFOoANACAcdKdt9KcBxSgUgAA4p4GKeAMUFfSgBFNOIyOaRRzTyOKAGKOasIvSolXJFWlGAKAFCjFBAx0p1MJwKAIZFqBl5qWR8E1AXBNOwCEVGwp5OaYTmiwiIqTSFcCpccUxulMCAimkU89aaaYhuKTFONJQMTFKBRSikAU006mmmAw0hpx60w0CDNFBGRzQAB0GKAENJTiKCKAEpaAKcBQIAKVRQBT1FAwC8c0YFPxShaAGinqM0oTIp4THagBVA9KmQYFRgYNSr0oAkUd6COKAeKTNSMbjJpwiz3xSHigk0xCsgUYFMxTiSaKAExTgM0lAoGPwAOlRu2OlO61E/WmIgkJJNQPk1YcZqJkJFUtxMhyQ1PBY8AGlCANz1qXOB0FNgQE4ODSEg9aVzlsnvTCCKaJGSHtUVSN15pypkdKpaCISPSkKHuKuJbgkZFTGAAfdpXQWM0LinbKuNCACTUEzJHGWJAA5JNK6HykQCg8nFULy4DoVjOR6+tUrrVwZCkalwe5GKozXchjOAB9KiTKQ4w5OWIH41XkMCNtVA596ql3f7xJHvU8QRckjmsWykOVXL7yxB7AUsssjZBckehNJJIuQQTUY+ccVF7jEifY/FEgeeYKEA9T2/GhcK2SM1L9oQdOKaQHLSLiRhsK852nqKChIwsTAd811/jCyjtpIZkADYwcDrismMXmrwsF2ARgHJOM1qQYToV6jFMAZmCqMk9BW3arHd3CWt3EXwcAIOc+tTatYHR5kaN8xnojAZHSkUmYl3avbIu4glhkAdRWnb2tkNOBuLiQOQCADxVZd+pXoJQdOcelX30s37iDTSxZRlg56/SgZlrIGjKDJPvTIlMkuwDntQ6S2F2YZUwwbD5HQVeu9OAgFxaksCMkg8YpgZsscls4JGD2rVh0u91TRnvhOuIz9xh2xzz/8AWrMNrcuciN2/DOK6jw9f2semyaRegxtLnDnoc/y6UCOQimNvPHLggqc4PcVoTRxGUXKHIIzjGMV1kGlwm3lWYJNCgwCVGcfWuXnurVZHhjgbyQTg55H40AJZvdwyGe0JjwcFc8mr9rrc0+qM4c4IyQD3A9axRLJHOGjd1A/halklMkvmAbW9QaAOotTqfiKeS3iuIonQH7w5x6fpTW02ESNa6jEY7qMcPuwD7g1j6fqq2ELgwh5D/wAtAcH8aoXF1PeTF55WcjgZOcCmFiQk2t7JgqxQkZHINQzSlp2kHDHHNMMmCQKYSScmpGOlmeYgyEHA6AYFG8+WFBI56DtSADODTWGH2igC9PP/AKLHCwyw5Dg1QkYs+4sc+tKRxTMEnAGTTAdFJILiNt5JByN54FXLtTMfOLR8gDCnOPxqhsYgsozjtVl5lYLti8sgYIHSkA2NTJNHCP8AlowU/Suj1DTRoV/bS26GTzFIYEAdcVz9sC9zH5YJkByuPUV0UNvquqFZJ3yIXIAIA/AH8KoDCuI5be48xxjzSWA7D2/Woi7GbzFfy/Ug8irl/BcTXMpJDeWduOhH4VmSI6Ha4wfSkBrz3EjRoLtftCEZVzwRWaR6VK7XDWkZYfuR0J6mkQDILDKg5IHXFAEzXs0kYTeEQDBAHJqNIhIcAgD3Nad+BcW6TQ25C4+8Ov41mKB0HWk0BFIAjlR2rWsZRDZhbiBWAOUyMYBPWpNLWS4QQpaGYhsknrim6xdAXvklGAQAEetCA3rGxt4dclgUBYbiHgE9Cai8Px/Z9Wv7B2GCCMHjJBqDSxNEFvLqFpIY+Yjn5gAegFXPOj+2y6o1kskLsCryHBB9hQBJd6S5sriOZNsajcH6knsK4iJiJlCKTnrjnium8Ua6JQLS3kIjKjeoGMnuKzjZR2caXEFwBuj5BGQc9aAKaWUl1c/Z4iNxGQTwKgltTbqY5D++VirLjp71p6ZLEl3JcTyAMqkrkdTWfNcxzyySMv7wnk5oAjYkAYOMEGrtkIjcK91O8KuMh1OOfeo5rSW3tEknXYJBlOc5FI8CS6Os+8bkPf60wNPVJLi7mgMqgSgFRjuOxqgk32S+RJWKxhgXAJAIqxbQ3FzYwXCzBQpwd/UGpRCJs2s8JMhfi4UZGe1ICzqRt4dVfUNJcvGuC4PIOR096t6fdJYeILLVgM25OHVT93IweKjkEN7oi2tsiQ3MZ2vGpyGIx0+tZU9pLZWMMkhIDHBUcce1ID1u8tg6G6tgOAGV0PBPepz0B9RmuL8FazBDbS28zSgKeGJJG0+o7YrtjgqGUhlPQjoRTTAjNNzTiMUlABUbsVGfyqQUEAjBFFwK6BpiAcjB64q6BhajTAYVPn5TTAjOOpphwTkU8j1phAFMAFBAJyBk4wKM0oOCCKkDz26gikkmhOflkIYg4wQfT3qG1mIRluIfOhAOwgAHPvXSaxaWSyyzQjbIDkt1GM8j+tc5q9k2kzo8d19pSVMgjohPQH68/lSYyxBC9lcJJbssUhByZDkYxVK71K91GEtN5KQjPIwAenbFVLWRZ3Mt6pnYA/KOGz/XtUNzbhbcsgdAOiE/5xSSEy7b6zNFppiVwXBHlEjoO/8ASpra7uXea0mh3SSYySeAcdT6VLoAii1KzN7abI2QojOOCT0INap0+3stb320wImQlwzDGOwBFFhXMJ0khu4Le3mwGOWi6gEHv6iunIu7KeOZri3hQqDcWwB5GcEj8KwoGSG5trbWHxbyElJIj8wOehI7c1LcRb9Uc3cxe1BEcZY8hOmQe+DQhogtZLKU6nCk6iEjdECMEjPBH0GK7PRrGaOOyaWNQY0AIB5Ax1NcPOlvFdrZRljMrmMOoG0qecZrq/DZj0+8AMzNDckiMk8AjPH45osM60kIpJOAOtZ7atDDf/ZX+QEfKzHqfSrzMCxA5I6iub8TpJNFG9tb72U5WQHGABk5PemBsrqMRmmjjkWQpgscZwSQMZqvqDiK5ilLAMMoeRnB6fWuHh8QXZ8pcRbZGAYoOWGc4q3q11JqF9FcLnyyoABGCSDnGPxApJjKPiq8W4lsgAZJoQRMQOoz61VS5uI7eFRO5UkHySefp9Oa2BEZtBvJ5Y447dUPlkHBLD1/H+VY4SGYQ5cLKACY24Lgeh/GhsGjr7C7FzIJQAm0bZFxjJwORVknzWKn8KzdLs9wkkgul84kBlZsgDPHTrxWoYZhKcRkYOM9iPWrTJaFjQoRgVbByooVBwCOaXbTvcVivKMZqjHIWJzwR2q/PGXjIA5NUxAYlZnHQZq0JlS5hyxdRVdUdcMeKke4J6HirMaB7cEjJrRMhoq7zTlyaRgVJJHAo8zPTAq0SI2CeetGARjFN2kn+tPAx3JoAhKEEk9KQgelTnmomU00xWIitIBUhVvTimkU7jsOXNWEY4GT+dQA4NOzmpYFjGTketTqRgVBGQSM1Ou3oKzZaJe1IaUDinhMjNQMgYE1FJ0xV0Rjviq00aqxKkHNUhFYNg4p6nNN25OcU9UI4PSrYiQAYxigLzyKeoAOOtSFeBU3GVpIQ4zkg1D5TdjVwox7UnlN2FO4WIEjOcmnFMD0qdY2FSGIEe9K4ymBgYpwUkjHT1qYw4PSpBGDGAaLgQiMgjGCParMaEimrGF4FWYl46VDY0RYINSKDUpjwDTVwODUlDlBz0pxBAp6kAUj8ikBCTTCaeRTD6VICZpM0lFABmkzSUlIY7dSh/eoiaAaAJt/vRvGKizRQIkJBNIeabmgHmmApoo7UmDQAooNJRSAKXFAFKBVANxRg0/FLihAR4oIp5FNxTENxSEU8imkUAJijFOxS4oGM20oWngUu2gCMLzTwOKcAKcBQAgFKKXbSgc0AAGe1P280qinkelAERFJinhSaQjFAEZFNIp5pOKYhmKTFPpMUANxTgKMUoHNIAxS4FLRQIFHNPApoODTwwoKE5prEDrUhxioXBJoAQOKmSQAetVSCDUig4FAErHI65NNI9KUDNPVeOlIRGAadtJqTaKqahqEWmwq0g3yP92MHkj1PpQMn2mnAEVkLrsrqGFmAD0JcVHJ4pitlQS2TMxGfkkHH1oA3BmnhTWRZeJLO7njhMMkLOcDdyM1v+U3pQAzFFOKN6Gk2P8A3TSAAKcKQIw7H8qUKQeh/KmA9OKl3ADrUYBHY0pBI4FIB3mr60x5ARgVVcFOoI+tME3vVCHuck1Cx208yDHNRM6mgYBie9GajOSOCBSDI6nNMRKGprHNN3GmkmgQhPNJQQaKAGmgUuKTBoGBpDSmmmgQE0ZoxQKAGmm4p5HNJigBMUYopaAExRin4oxQA0LzTgBTgKUCgBAKeBx0oA5qQCgBAvtTwvPSnhaeAPSkMaF9qUL7U6gUAAT2pQDS5oBFADgKMUbhjrSFhjrQhAabxRuzSDmiwDutKKTFIWGetOwDjxTdwFMZiQQDVZ3dTyaLCLZkAHWoy4NVhKW6GpFbAyTVWAc2T0oCnvTGuAucDNIJiwzQBIFGPemMp7Cnxgnk1MAuM0AVACD0oIJ7VaJABwKiLDPNAEIhDDJFO8vb2xSGYA4zSecD3piJl7UpIAqEEnvTsnuaQwYbgRWdqMcQtzvO7rx2rRJGOorPvACG3AFfTNQ2ByJVu3FXI7izjjEbDfJ3GKLpCvIQjPrVYJms3IpDJIopJCyjaD0HWjycDjNPe2kjTIX361CJph6VNxleSMqxAOaEYAYzzTmLO3J5PWlWLIyOfc1NxjCo5OaYFBOSeKkYAdT+VRkheB0qkwN3xbA91dxQr90dfbip7HSEsNOuYQctIgIOa1Z7Vbi5Jfv0q2bdGjAx0GK2MznfDOiwwRvdud8xJHXpVDV7Y3sshuA+AcRlRke3Nb9lH/Z0kqySAxnJJxwKxrvxOkNyYbYqI1JG5uR+FIaL+k6Hb2FsZVGZJEwQQDg1L4csxbJPIeWZiAcVgW/iJbyeQXFyyRp1HQH8K1dF122lEwQEQrkiQng1QFfxrpqXFoLtEAkQgEgckcdaqaTaWh0JHlAKr94k45NY2qa89xqLpJM8lsCQAOO9ZdxdhlMcMjiM9j0oA1NNv47XVplhwYSSQCcfzqLWGtHVRBCUkDAnJzn0rFX5Tmpg+RUjNWbXbg2oggTyFxhiDkn8+lZS3EiAgYOTnkZoJzSKuWwCMnoCetICd7kXQAlZUZRgDHWoSCMnHHY0yRCCQRginC4fyxHgEDpkUJgNLcZFNDcZpzJOxLeUee4qPDKcMpB+lAxzuCfpS4IUMUbae+ODV7RrA3eoRgsFz0DdCfSuqvfDV0LcHegQkZUjAH4YoA4Zcu4AViScAYrUl0a4gjWRreXJGcEdq7+y8KWTpGk2BtwdgwAD603xPNZWumm3hlP2hRgKDkgYoA8xcAsSBgemc0xHMMoZeo6VuWFjca7BNHF5aNCCckYJ/H8KisVm0e5ju5oCVzgg9CKYGcMzXAAUZcgBRwCa14PD5M7o4kViOOMj86kuLU60ZLyzQxshzgYGBWpZSSCyWSOZ5LgDBOeQfcUgMyygvbKGQt5YMZ4jK8kHuDmta0eOGyuLkSK07/8ALMHhc+1U7Y6oXeS62yKQchsA1iyRzXF0/kswUHkg44ouBDdy3EV7MfMkDE5yeKpku5LOST71dmZ/MUySebtGAe5FVX+eQkAjPY0AOEkhhEJ+7Uke0OnmfdLAH6ZqPy3GNwxUiQ/aAYdwUkEgn2pgbWpRppu60gJZZlDLngqD6VlWqZuFDg59AMmrtzfmewgguAVuoDgOFzuHarVhaE2o1JLjbMhIIYZ4/OkAywupLS7mUM0QIyMr1FaFrbrqUgSKMGQgyF5FJx7VS1JjdrHIFKMOMnpn0rtdKtxZadDCRhgoJ9TnmgRgzSlTJDJDIJApUYGQPepmiSbwlvkGPLUiM9SWHTitpIVillIAJc9SOgNSWsENtamIqDGDnB7UDTPK7gGbkrJ53WQkd6WKFIZ4yG8wHqAea7S/0iN9OvbiYZmY5TB4A9MVjtbxXFoJoYVAjABJ45oAypPOibegwPQjmooo4JiqGJvMzk7TyR9Kts8InEbOJ8nBIORinXEMaSgxQtt9uTSA0hLpupW/2e4ikT7OmIjnBPbApNU0G3ht9OhcCO5dwHVTyQfX3rOspWF9HIGAniIZCx4OOxrq9LaLVb86jdxncJNgXOQgxwD/AJ7U+YRyGox3OkTS2iuRbsMgH0NaOm2ZudPNq8jQXBw67+ARz0ra8Q2sM11DbhFILg+cDyBnGD6ik8a28NtJaMqA4t9gB9BgCi1xtmGI5kkUwgRTQ5w0Zwfzq3c6Ld3FnZoWY3U0mQpOQAecn0/+vWp4VZG8tyELTJnBXkEdh+da+p6e9xpZvY3aOa3cvHg8BeMijlJsc/JpMnh0CeWUSmYGOSJcZHoRW3qCyWGgwTIcSDGCOCM1ia3exyx2UMZLScFiTkkkjrWpLYzW9xPCQZYZoN8W89DgZA+lK5SOmtrhJbS3LuBI8YJB7nHNTYrmbE4tdLZ13IFKHjoeldFGWGVfscA9yKdwJR0pCOKAaWgBhBprGReQcipe9BGRgDNMCEXGHw/A7mpXwOpxVGe6hgcq4yfrWZ9qujtS0uAIxnPmAcD2z2oA3C4IJB6HFAk+UnqQMgetU7WQSKGIlfPWQjAJ9hU27nikBk6jqgnE1jPItuGUZUIWPPY4Fc3dj7Pp9yWXfCGBUZxkjOOD1610N9YXM121xHKPMEZAUdSccVkNZ3l1qENpqO0SSRuI2A4BwOcdc03qNHOKl3ef6iMyMgDHHYVD9ovC5uJnZhnAyOAR2Nb50yTRgRcQSBTw1zG2VI9x2qqLWH7LNDHdI25ywVepFQ1YGRSajPHbw2k0+PLO5Q44X2HoKbZXcSTQtM6PGcoQrHj3xUNzLbYADAkAA7hyCPf8aZoxhOr2sky/uQ/JA9qdxWOhttNca5Zg5mhnjLISvI4PGD9K6yTSUudLijuoQJooiFIP3cDr75qlYKbrXJrgzYNvGIwTyCT3FV9fF3ppt/stwHaYlSG7YA/Tmi40izb2tlpkSPcCMK6kKqLksD1Prnp0rJW+tbezEEUsLhHL/MCrKCeME9SKLDWIxBJcXZkFwoIAB3KoB6D0zxVRYIvECzPaMI2EwYxOnz4PcHPT2xQx2Ohh1y9k8u3WJX+Tc8wHzeXz1965q71G9+xqizSS2EcmCrnBPqCR2q/9lutCv9rMGtJkK4I5IxyBWZrVygjhhEX2aGQc4bIAz3A70kyb2K03kuyzW8AgYEEKp/MU6S5uZJy6I2H+UZBOD6DHeo0EocyRzxP5f3UHQ54wfpToI7qPUBLcSxiPO4hZOAfUDFJoaZfu7qW30yCwuoW8suWcEc7h7/jWdIy+WXeNhGhHlk9QPer13NFd2kzmRpghDHdkE84xmqUOIr8FhvhJBKAlsEjjjvQO5vWN+ulLDJDEskbkmTYMkA9we30rYj1lL0sYJ5oWTIAdQA59BmuBmQw3jTW7PHCzYCk8j1FSRTTtNDLId0MEmVB6g8cUJ2EzvbXULh0H2tgA+Gikx1GOQfxrZCVx1pqURnguJnAjQ7NhGQAe9dfaXUd3AJYzwferUhClOKheP5Tgdat4zSFM0wsc3LpkwZjGMjOcelTWkNxHFh42ABIzitcjDYp46Vopk8plSWgMZ4OazpYGjGQK6JlycCqk9k82cdD1FNTFYxAxHWlD5q7LpwSMkE5xnJrNGV4PWtjMnVhnNSlwRggGqoY4xmnZI607DJfl9KY0WTkUKc1MvGDSuBWZCp56U0Eg1edA6ZFVGQ78YoQD0OSOavxxqQMk1Wjtxt3EgY7VJ5wUheuO4qGNF5VULwKUELmqYuCSADwamWKWQEAgA+tQ0UEjAgjNU3TJJFXVspMksQwP4YqZLAHqRRewFCMHbwKdgjqCK1lsYwQQOaf9kWlzoLGQvBHFTBlPerxsgTyMUgsI+uDn1o5kOxVABpQtSNDsOAKaFxRcBuylCmnAU8D1ouA3Z7UbD2FTADFOAHpU3HYr7DUqYUYNSbQR0qJhzxRe4IlJyMVWMmJMGpQSOtMKgtk0DJVNP2kjNRjjvUgfAxSYDCh9KjIxU/U0OgxmkBUINNwasMBTCKQEBFIRipitMK5oGRminFcGm4waQhKMGjNJmmFhwpwXIzTBUinAoAXFIRTs0hoAZinAE0uKkRcigBgHNPC+1SeWOwpdlMBhFMIqUimkUwIyKQinEUhFMBlJinEUAZosITFGKlVM0pTFAIYo5p+KFHenYoAbgUAU/b7UoT2pAJjigDmpNnFKE74oGNAqQAEUu0UnI6UAIQAaY+KUkmo260IQxjUeakYVGelUIXNKOaj704HFAXHClFNBFKGpWGOpDQKD0osMQmjcaaTSA80WES7jSF6TtSEUWC4uMnNPUcUgHFLnBxRYLkoXvUgHFRq3FSZoGOC5rgNYu5JtUnlkRlAO0ZGAAOld8GxSPtcZZEJ9xmkB5l5gc8ZJ9hmrNvY3N1Jsit5Xb2UgfnXoKQwBgwt4gfUIM1HdTvGCFwPpRYOYw9P8LuBvvJTERyqRtkg+pNX5NOdTgand/wDfVL9rkPVjTTIz9zVcpFwWwm6DVrsfiDUq6bdfw6xcD6gURbi4wM1aVjnkYqWikyAaXe9tYmz7gUv9l6kOmrP+K1pRjIBqUdKkZknT9U/6DBH1Wmm01cfd1ZfxQVrvVZzg0wMuWHWV/wCYnGf+ACoCNYH/AC9wn6pWq/NR+WT1HFUIzfM1joJbc/VaXdrP/Tsfwq86hOlQSSEIcGqJKpl1kdY7Y/jSfatYxzaQH6GrCyMw5OaX71DQ7lUXmrDrYR/g2Kd9t1T/AKBw/wC+qsrgGpgcilYCg2oaiBzpY/Biai/tS/B/5BTfrWpmguR0OKVgMz+17wddMcfnTP7ZmHXTZB+JrUMj+tN85/WiwGZ/bRHLWUwHc09NcsmIDrIh/wBocVoiQnqAfqKa8EEy4lhRs+oFFgGRXlrOMRzKSe1S45qhPoltJ/qSYmPpyKqE6lpbjJ86L/P+NAGzijFVra/hul4O1vQ1aANACbaULzzS0opiEC07AozigEE0WGBGKbuApT0pFAJp2C49MkjjipgvrQoGKUsAMmk0Fx44oLVXNwoPJpROD3osBPnNG7FVzNjoaBJnrRYLkxcjpQHz1NQNIACagMmepIp2FcvGQAcGoy5xnOaqjcehJpjzFTjNPlC5dEpzyaDdIg7/AIVnecxPU0uSe9OwXLxvARgA0sc28E8j61TFTRjgCiwrk7TqgzyfpULXKsMFKcyqFJIquQAelJILjzKSMACnJG7jJ4FMBAOQKa0zkYB4p2C5ZEKenNOCgdKpLK4Oc1OsuRkmlYLkpcikMpHQ1GWXHWozyaAuSmc+tRs5bvxTSAOtNLAdKYXDbjrTgQDTC5PXpSocnAxTsIsBz2qTORUAYDqaXzcjGcVLQ0VbmQgkA1SuLhxlFjYj1NXLpRn5AST1IFZeoyGOIgAgjqCKxkUV7hzIpJI/GqJOKuW8XmR+ZIOTVa5JLkYAx6CsWWhklxKwAB4A6VAWI4I5qQccc59KEEZkPnEAehqLg2Q4wcnpTWbcvydPWrMwtcjYcn0zmmHZsOBigCsIWOSzACo2AViAQR61NJjGBTQRwMA+5q0B6EoGcnoKoT37S/ubRS8h7jgCrM2ZIymcZogjWCMKorpMzGuVks9NuvtJPmOMAE55rJ0/w3u02S8uEYsQSqk4x15NbGuWbzSRSg9SBjtWuH8jTQEGCUwB7mkM8cnBE8gIwc9K62x0U2ugPcmZhJKOE6AHPvVe48M3r6shdVKynOc9RXVaspGnwW4w0gAGAeuKkDzqHSp7jUfsq43EkkjnAp9vpLHVTZXD4wCQy9M4rqZkGl38U7Q7mxghOtYWpXxGom4ijaJ+24ciqSKM2+sks5Cgl3kdfaqmcDFS3M0k8xkkbLHqahz3oaAASKt2ccTqSz4ZM44zSLDLcwA+XsUdx1q1YafLcSFbckMBkkjj86gRUkYuxYnOe9CzmMYCL9cVcvdLmsivmyKSTjCjJFVJ7Z4UBkGMjI9xQA5JpHIya0Wa3IyDgjpms6NGVAcUBlL7WpAWnneOeMkARA8sOMj2rsILwQERG7UWkq5Vpmzg+grjVdGtjABnJyPapDcOLcQNggeo5FVcZ6NZRHTrV5jMJAQTk9x9a4+Oz/ti4eQAqXYkHPA+vrVAXt3NEITNIyDIwPSp7OOQyHHmKAMkii4G1cG08PWcyiQGaZMFAOfr+tYB1GaS2jjEIaNTnLjmo7u5lv7gC4ZmKjALdQKdp9vDNcmB2K5HGO59KLgbWhala28DJImJnOQFHBFV1u7qzu5o4YECyEneoyQPYVmtJJp9+QGwyHB46itVfEccEyTvZSFlGAykYouBBNfW8TYtmkeQgg+ZwSaW3024k2SJLCokGGSI8getVLrUob/UnvUtSgIAYEZH1rTkl+xWKTworydmxjBosBRsrK1h157O9y8Y4UnjnHHFP8S6WlkY7hJVMYO3ZjkDH61XuH8xGuHKyPJgk9x9PSptNlWS4S4v7gTDGIwSTg/SlcDEmmjmIKHkDGMUkciI6+YMqTg4q5rMdt9oD24C5zlQMBemP607QZrVLmaO72CN4yMuMgGmmA5xCXknE6HYOO4I966ayvrW60kzyIkccYAIBGScda4SNTHceZHxtYkZ6GtW22Xt6oNuz785jjOAx9TQBs2+rQX+JhGA1vID5Y7r0zXYORNElwvQjIH1rh7pbKzvrQW6GA7iksb9QDjqK7iKN44BDGMIowp9qYNEdxeW9o6LM5G4ZzjIH1qRvnUhOc9CeBUc2npNKksnJUYwehqwnCAHr3FAGH4lkks9OhjSUqXYo+PQiuVuZHWwltYrgvbq+flH3vqa7y/0+G/Ja4QMApCE9QaytI0NorURzDCiQsCQCSOwpAcRM0arGYUKFCASep+tXJrlldTC+cjO3HFXfE+lTDWJXhRShAOF7VU+wXNlJD9ogMeemT1FS0BRJeSXzsD5GHQYxXZGKW3htLm0gaSG4QNJCh5yB1FcytqZtQ+xRuqCQ9SeK77RLadFiE4YRpGI0z37E/jRYDmrXVGsEZhAZgWJV8gEeoIPWpIrC48Yan55DQ2QAzk59iBis2aC7mv7iRYQ3JJBPAGa7uzj/s+3hEMbESRghFHG4j9M1SYkc9aRQ6F4oMcK4tYxtIJzgkcmuwmZLjTpooDu3rgcY61yWpaPdaUl1qlzcbnlOViQEkEnoT/WoNL/ALUt7eK6muCgmcCGNxksCcZ9v60rjLfi6z8h9NmijJYEIcc5IxXR6pbmZLcqQGjOcHuCOasXdh9pkt2k+7EQ5GM/NVl4w8qNjOB0+v8A+qgDChtCltYbOR52CO2cmttx85qtBHtiCMDgOSMjGCDVojNNAMpRQRimkgNgnmkBKBTZEkkjKxg5J5IOMCmsxGMAnnnHpUkZE8YlQggH16UwMc3cJ1Q2y7TKmdyOOT6YqTUJrcWhSQYmIICgcDPWsHVJTaeIpL1gqYAYByQGGMYrIuNfjmmeYofMbG1l6qAMY560gOqF3qVhDEUt457YjALPgj0zVsG4ls3n8lEuCM7FORXLW2qxQ/u9RuLh1wCqIMqQRkZx0q9pmszTav58iyx6eV2xt5Z2g+5ApXArnVJlu5xJdqJjhQAOAQen6Vk38sr3cVzDfTl2O194AMZ9Kt60txpl0FlRZg4aSGdBknJzyKotA4toZEUukx8wADJBHUGqTsNFe5v7qTzElkmeEYWUE4BxVP7bG0oKkxgjAcHkfhUd5epIJMMY3kIDoBgYHeq5ieZGmjhCxoMEjuB3oauBILglWjOHMhySRyTUVvcNHN8vBwQVPWpY4jdzp5BKkjAIqzEM2MxlhHmxuMSEcnnvUt2A0dK1C7tx9ptZlaaM5aJhww9xVy619r23kb7OY5JCcnOSoJ7VhPYS2V8QXM2VDF4xwMjPJq3buiTRhmCqPU0kwRHdzNLLDGjJGHwDt4wfettNYj05Y7U28NsCm1pY+SSc8g/Wsy8uLQkeXCPOQ5VlHBPvWdcrcJL5kyxkOcEKcgYobuM0muriG/Ie6NxGqlRI5yBnkEeh/wAapFmmkiiunJBQxoQOSc96IYIJDIHk2qqcRjgginROhvYVtAAoUnMg4BwaRJoaiYbLT7DKoWCbSFGMnjJJqC+ktTaQSJFmMkiQjgqRjrVpXimst9wwUW5IycFWP0rB8y3MjrjCscg5yCe1MZeke2isykLypIGDIrnIkGadp0624aVptlwDwW5BH0qoCIwkc0QwMkc5I9CKbNEu9AedwzQM2JFa7UyzXEZBJYKowATjP8qpPbRokdwHYqwJwTgg5qq0ckcTiA4jIyF7GtGyVZLaOFlkdojudScBhnOKliEEx5AKmPk4PfgV3Hh5/sej/OMgONpHTaQCP5muTvLu0u71THZeWiEYAABz6HHHpXQ+H5priZIZYykcmWCHtj/IqkwOpDjvTiRiqkwaJjgnFSI4K1dhDXJL52nFTIMjmmk0oYAUwF2gNkUhIBpd4NRty2aaBoz9UlKgY4BHNYj8uT61t6jCZYWA6gVitE4PSumnsZMYCQcinM5brTQD6Gl6HkGrZJNH1qcDgVVQ4wanSTOATipaAmB4qORCTuAqRHUcE0pTJyOlIogMjAEdiMU1OTT3TB5FIAOwoESRErICK1IXHBrMQHcKuxnArNlI0gQQMU9V9qqxP0yatBxis2USjpT8ioPM9DQHPrU2KJiMmkIpA2adkUWAjaMEdKhdAOAKsEjFRNyadwsQiMY6U0pg8VMTjio2NMTEAwRT+gzTARmnbuMUWAN3akxls02jNIY4gmmkYNP3YFRk5NAC54oBzTacKAJAcCmljnFJuOKaetACk0hxSHNBFTYBh5pcA0hpy0ANK8dKbsqQ0Ac0gICntTSDVkqCKjKc9KYEIBzUgBxTgntTgopDG0U7bSYoEIOtSKaaFqVVxTAlB4pCKcF4pGHFAEZph604000wGHrRinEUuKYDQoNOEfoKcoycVOqjvTAYiAdql8sEcil2gUcUhWIigHAFKEHWpNoPNKFApAR7Ae1OEftTwKcKBkRBFA6VKyjHSmcA0XAbTSKeRUZznApgNbpUbEVKVzyaYyHNAiI8ioyO1SEEU0g9apCIyMDNIWHrT2+7VVjg/WmlcBzSEHipBIMUxVGM0EAU7CuTB804vmqxfHemmbHejlDmJ3fAzUZlA6moHmJ71A0hPeqURNmnHMrAAZp5YAZrLSZkPBqcTl160coXLQm5pfMJPFVkIA5qxAVYkipaHcnjYkc1MDUYAGM96lTGMmpZVxQDTwPWmlwOM0hkx3qdQJQBUE8KuCMc08SZ70hcdzmjUDPe1YnGOKfHABjKmrmc9hTkA7incViKCE7+mKtmIAZxSxgAAipDzSvcaIgD+FOJpaQ1IxSc1G6bxTqcDQBVMDewqCUlQRmtAjIxVWW2AJJYmrQjKklbdyajLkjmrklsCcmoXtuOCBWiZBXD45o870oaNgcVGUbPFMkfuJbrU8TkHAqNEOM4qWNRnmkykTg5GaCKBSmpKG4FJtFOopCExjpRTqQ4pjEyc04EEYIBB9ab3oFIDNvdH3ZltD5cnXGeCaZYamySfZLsFWHQmtbdiqV/ZJdRFgAJF5BAoEXiOcjkU2srTdQYE2tx98HAJPNaZOaYxWYetCjJ4qBvWkEhHQ1ViSScsAAOKiVzuGTSFmY5JzUZbDU7Cua0ci7Bk0yaVMEZxWf5x7GoyxY5JzRyhckd90hIPFSKGCjFQxhSRxV+KHcME4odkCIAT3qQNgUssZQE1SkkOetCswLEkw5ANVmkOc5qMmgmqSFclE7gYBpCSetR9KN+aYEg4NTxEEgVVzmlUntSY7l4sifKAKdG3OSeKgityeSCQKtLCSAACM96hgJJIhGM1XLZ6VcXTwfvOT+NL/ZaZ+8fzqbgZ+4mlFW5rWOIZGSarHaBVXAYTgUxQ8jYDAD3qYBCMvwvesy6uzDMRD931zSuBrKgHBNS+X8mR0rD/tJuA0y59KvR30rQhYQGPfNTzFWJJ5BGMk4quLiI/wAYH1qHVJnS13SoyuOnFZ1sxkTeVwD3IqeYLGyJYyOGBqJLnfNsXp61nGTflFUsO4FaNpbqsYdiEx1Bq1K4NGgqEgEio5Zoojyc+1V5NSSLIBD49KxJLiaWQuTj2zUuaQ7G3PqkcCgrDknvnpWZNfRzElomye9UHvGOVyPqTUBuCeAQfpWLncdjXjRWXIGKzLuZkmK4J+goju7gLtDYA6VWd3acu5z6VD1GmJukc5EbflQbS4c7vJcL6kVbTU5oVwiIB9KZLq9zICpKgegFPlHcr+X5QywII9aYZMmms7Sck80zpwaLBckyD1OKMA981EcDk5zQHx0p2A9CIpRSlSaUR8V0GYhAYYIBHoaRkDAA9BTipHalCknoaTHcYsKAguQAPU1lyXFvJeSPGJJtozlRkA1LqqTTXUFvkhD1xTZpbDT7ZkjuIoWxgE0rAcXLeCTUpWe5IwxAB9M1WZtOlkLXM77j2JJxUMzxzao8YmVoyx+YdDzS31jAko+zkqQMHHANJjRWFvbvqAVNxtz68U5LBZL0ohxGOgJ61GxmQYY1Nb/uQZWclmGBntUjNJ72ztlEUodvZaf/AG0CBHZwiFPUjNYTuZHLMeakhYEEDqBkClYC9GPOuTJJIXwSDntVS/uTcTlScqO1O08tufcMZBOKpSh0mYspGfWkkBb81TCoJ5Bqu7ESFgMg1HG2QVP3uxpzuygZHHrVWAQzbHznFPE0+POCbkPfFTSQiFo5ggdDwQeOKszXkYhSCzGQRkgjpQIbHHNDIu8bcjI9xUwvJoSQjYz14pLjWRNCsM9osbJj5lHJquk6XBZY42JxkGpGdB5NvJbQzTHY0g7DqfSsq+iNjfoyk4BBGe1S290Tp8aSH50fj6Cm6v8AvrpOeqg0gLGrRrcTJdkbkdee1ZZmOPJjGV7KTmrdpKfKNtNyoxg+gqlcWbQXIFvyCMjtTuBMIpkt28tAmQQSPSl066NxKltdXIjgPUsOKbqV26QLCq7XxyMVTtwNyOygqpBII4IFUgNeC0ik1F7YSmSNc7XB64qV4tOeN1uZ1idCRhAcn8qQzWdzMRpxWBVXcS4wc+gNalh4ftL7RQbm/hgYk5YYJP1ptAcrK2+1JjTEYOAScn8aqKcVJMDZ3c0EFwJIwxAcdx6+1MUcc0rAWIlV1J9BmrmgyEaoj9WQEqOxPpVSMBIC1Pjji+xvMj7LqM5wD1+lAFvUbe7udTcyw5d+QV6EV0ml+IZDpItZJhFcxZG9+AQDx+lZb6XqEdpFqaTrdkqGZAcnB7dapaJItxNdW8gBDcrnoCKYHZ6Pd3uoRzGSdAY228p19+tauCG5rF0mOwEL+XIyMeGVemfWtVN0UxjQpOuASB94Ci4E2KcDUazI7lACjAfdfg1JtI60wGCOKNjL5YaQ9SRkms7UNJjvdPluLsuZEBZcHGDWrS43Rsh6EYNJoDi9JtUvbGSKWMC7hYSox4JGa7yIhreJvVQawtMt8QSk8SRttUjqBWraM/k+W4wUOKLAUYNLZJb10TPnkAdgBkk81uBAgQL0AAH5VEvFShulFgRI0aSAbwGA7EZB/CqV3Yi7mjLxAhOVA4ANXAc09aVhkaR+WGCZJJzgmq7EPAyRrubO0kvgj1xTru6eNwiRK2DknocfWsrWLq1s2t7iFefMxJh8EAjqc0WEWbeWaw2LNMLqNjgEk5U+voetXbS4+1eZIMeWGIDgYGKw7KS1vrJLU3EcExJaLEgY/jg9a37FTZWdvb4GCSHJHANCAbMFjlBZsZ4AIyCarCWJtzecpAODjsa0b2yFzF5ZkAz2B5P0rGkW3s7uz06M8sSWJ+negCw07pEQ8UgjII8xCDgHoaz5Zru20eZ2Zbe3Qgk5O5h7VR1HUJo54ozJHC0Nzs8vPAAGencEGofEeqQ39p5EExDEgBdpAYnjqen/ANamBVhnhuori9uLpLlihVA4ywHpisXTJtPivUkvULQEEEEcVC9xNps4SMpJIBk5HA+hqK7eEyRSQQsYyCSOev1qWB0V8mm2rxSW8Dh5QTHJ5oIPfkHtzVabxFfrEgQeS4G0lHG0j0K9Ko39tLBZWjzyLh0JjHoOOM1TkSSLAkQgMMgHuKgDsHuTDaK6pbEFA4diSQx6gDoKydT1L7XaW8ySGOTBEmw4IPr+OKxhLEoAKNsByc+tWJrOMxI8M/yHoCOmaOYozb2GYhGkOSM49SOKo+Y0YOwkZGCOxFbscMl1YmNnO5TyMZKj1FUZdMa28u4EokiOcHGCCB6VSkDQQ2VxGiSQzBcYO8dAK13tXmCTJdqFkOCvQH1/GseK7llt2i3KuGznpmrsxiha3jt7rzoSQTGwwVY/hyKGriLa3kkNnewWjIFIBckZyPY1lF0mdMjJBzkdMUwqTcSBiQpJGQe9Wbe4RdMms3QDGWRh1x6UrDQ+UEWryIAdpwQR2pn2mJ7KOMxl353EdAP61CgLTElise3GB0PrmnJIY4DDIpEZOTjmkDGLIY846EYApMZjjmKsFH3TnGTT7oW8SsgZkkBBCkZBHsaihkR0lBlCAHIUjg/T0pgWLgedCoKtu4Zo1PA96rSgIQyDEJwR6g+hqdZ3MIfziHxtGB0H9aosHwQHJU+tMC2ZvtDB5owEAxwfyqaKGM2ctwRKTEQApPBBPX26VQBzgMMjAGKv71bSZkS72OcZhPQgHIANDERxzRmQK5Yk9D1wfWti1jjMkyM2JGQFNowCKzIbeG8jcFzFtABbGQPWlnCQzKkM7ybQPnPA/D2qGhmnb2Ekl2ohJkjQB5n6ADPr716NCdNtfIASTzgpZU2Etk8kntz9a4BL/G94ZAS8ISaIjOQB2Pr3q74evr+PWIYUmcJKhOCcge/PXFUtBHdTIJCc9+3cVEItvAp1lML23NwRiQsVcZ4yDjI+tWDHWidwKxU0xyQDgZxVorzUbLTuIpCYjrkH6U4TNUrxKRjFQiMgkc1VwEaQsCCMe1VJIRgnFWzHjmnJgHkD8apOwnqYhIByRTggnBCISR6Vrz2MFxyy4PqKWK0ihUBQc+tXzmdjnypQlSMEdqAc1qapbbmMy9T19zWSODVp3JZKGPrVmGYA7T36GqXPWlBqmgTLsyENnBIIqPbg0RXGAFY44xmpMg1IwQ4IFWN2KiUAHNOcZFSxplhJQBjNSLNg8mqQUgZqRQ5IABNQ0UmaIkGAc04SCqwBVeaUMR1qLFJlsSgHINKZs96rAgjiml8HFFguXPMyOtNyDUCsTUgOaLDHmo2HFPPSmkgDmkBHS5xQWWjIPGaokTcKQuM9aaRioTkHFFhk+8UBhUOfenjBxmm0IlB5pwOaYeKQHFSUTYGKYeDTdxpRknmkAtIacBTtoxSAhI5oBpWHNNFIB1KBzSU4YxQAE0YzTSeaeOlAABgdKaVx2qQEGg4IpDGEcU0DmnnpSDrTEPAGaeAM00Hmng0ASgDFMcAClDU12oAgPWm049abTAUDmnbaaBTx1pgKqkc1IDjrTQTilFADy4A6UgORmgUooAeBxRQDSikAYpRRQDQA7PFRsCTmnigjigBnUYpoQ5yRTgMGn9RQIjKe1GwVJikNAyBox6VXkQgdKukVVnJU8VaEVGDEYNV5AT0qy7cHFV2BzmrTJYze6jANMMhPU049KiYDPFWS2KXJ700k00nmgGnYVwJpuKcaAOKYXGVKjAYFRkc0A4NJiuWVG7HIFWYQoOAw9+azs5FAOBSaKTNssOxBFMebaDis9JzjGeKcZSR1qeULk/2ok4waeCWwSTVWPlqugDaMUmikORiG6mnnLEZJGD2qAkg8U9GJOTU2C5YUjIAqYLxVYHnNTqxqWUidT2p9V9/PWnbzUjJCRSEgCoi5J60mSaLASF8UK+TURzTSSDxTsBZL8cGoJH5JNMMh9ajZiaLAIxzTGGRzTiaa7gA5NUkQyFowRio/KUcjOacZBnFN3Zq0JoDxTQeaUnNJ0NICcHilqNWOKeKTQ0BOKTJoPWkPSgYbqN1MJ5o3UWYrj80maaCMUpNKwNjs5FKOtN3U1nxTsBQ1SyywuYh8wOSKt2k63FsjAcgYNK7b42U9xis/TZPJuJIs4UnAFVYDQk9KZtJHyjNT7QTzUiRg0EtFMxuOopjKe4rSKDHSoXiBPSi4WKQU0hU9hVownPApREAORzVcwWFtowSDxx61optCjFZyEoc1L9p5wOPepauNaEt0QYzg81jscsautIS2Cc00qrYyozQtBPUp03JrS+zpgYFH2dAckAYquZCsUAkh5CkinpaTuMlNo9zWlGADxUwjLGk59h8pkCzuME+WcA9c0+G3kMgBQk9621BxineWKnnY+UqxIUGCMGpGbFSMgxxUJqGx2HBzmpxyKp7ualRyRjNJjSHTFApLVlT3MERJbgDvVq6li3FCrZ7msXU4IfLMqFie4xnFLmHYjvbiS5GLcEL6msqRpA3lyyD3Aqa3eVlAQ4J6elVbmBTcFRIxY/hUXCxOVt0jCBwSfStDTrmSDiBN/OcnjFZC2rxP8y5/GpEguncBCVU9MUJjOsu7hJLJ84dgMgAZxXIm6lcEE4IOCK1njurSywDnPUkc1lR+UoJkHJOc07gWtOulhkKbsMxzirOovKYyVI56gHNZDTQxOCDVyG9tSuSfwouBWRnAwcj60s6PFESWBJ9Knea3mbKsAPeqVxcb92AQOxPeoeoFKTOeTmlUAHjk+lMJyc1NbeWX2yA/hSAkAfZuAxjvUTOSeuauSmGJSACfbNUWkG4kKBQFhrMcEYIqMZz2x9adJKTkA8VDkg8UwLAYjgDJprAjkjFRBmyOSKeQ7kAEse9MBN1KDn6U0oVOD1oB7UAdvPe6oIyYbaEe+c02C91brNbI+eijj9c0hv7+OMmWxAH/AEybI/lVFPE6GZ0k3xonXIroINCS61qTPk2Mcfpls4rOv5detoBPNOsYzghQCAaztW8ayEGDT0Kj/no45/LFc09/f3DAz3MsmDnBPFJsZ1s769JYkvNb4P8AEQQR9DXGvPfXMxgYG4cHGfWtCTV7y4gaE3HloByPXFZVreT2bl4XwSMHIyDU3AtadZMbsmZMeXnjPIq09xC0zADOO9VI7+Z2fPJfqQOlVXco+BUPUpF1zHMxJwQDxVe4YcKCSKDIAQSADjkdqjYbpNwOV7g1JTGEcUiFo3zUgxNOkKDOTg1Zv7WO0jRU4Y9apE2KRkJOUJ3eoGakkJcAMfm71piNItPQoADjPPNVpLYeWJGBwf4lOQaSCxnsChyOKnKNNbHA5U1v6XHA9tMCFyvQsOxrPjSMxyBMZJ5IOQaYESFbiwIZwpQgEGojDbpbpItwTLkAgDtT7W6kgDwtHGYSeSR0pjBReoHQrHkcH0oEbupaVYT28T6e4luMZIBJyPSsqW8AkijNsY2Q4IAxzWgZpdPmN1bFYhGPwNYs17Nd3RuJCNxbJwMU2hkl0zLOQpwMZxV15BOkLj7wGDWWxee7QdXYgAeprbu9O1KzsCzWgwv8SsDj6ioaAhKeZgKoWTvikDSW8mZskD7uB0NRm5eGBHZFJbgkHkH+tMFxJMwCAEg5wRRYAuIZTaSzOCGzkEimpvNqu8LnHbuKsXuqSN5QwvBwRipri3htJRdIuUcfdzgZosBSaGxa5iDyGON/vEDOD61sSWWjWlt/qvMX+9vOTWA6B5CwJXJzgHgVau8LZxqA45xlh1ppgR2+ni9mlkgXyrcHO5ugqtKBFKUDhsY5FXbRL6ztxcQuDDKQXTuee9Gqz/a2VhbrGVHOD1p3Aimj2xRfMCCMECmC3bcDtDDIyAeSKkVDJax49STVxoBNZRyIxEiEjA70rgSbbrT3P2ZJzblQWw3TPcEVVjuZoboTWUZXIwSUyD7nNaehag0Qkt7gIIyc75DgAc5B471X1JLSM5tLsiPOPKHb6e1O4FdNS1GwuHm3BPN+9gDB+ldT4dnthEbm0Pm3L/fE0g3YrhpyS5+dnXtuOTTIZ4wwwJY5B0dSRTA9RfV7a7nNsocSRnEh2ZAPoCKsuZIQxtVeUDqDggiuP8G6jb2V9cm4vERZAOH5LHJzzXbMtvcxGfT7kEAZPlnIA/p3oAhF+C22S3khY9PM4B/Gpw7/AN39ar2+pWV6n2eS4jkYcEHg5+lTXckWnwiaRwsIOCCMk/SmA5dy7yI+XOSc9TTHF55Z+zmONu+7nIqhH4m0+VsRpdSD1SEkfzq7Fq9i7Ab5QT0DREUAMg1SVM/a1j2Zx5sZyAR2PpVwXmYftCmN7cDJIOTj6VVa3jg1RWRR5N0CGQjIyO/603TYRZ6jdWoJMLAOgPYc0ATSa0IQkwhM1q3Bmh52H3HatZHWRA8ZyrAEH1FYAtE03VYzCSLW7YoydlfsR7da30DhcOAMelSxlK/sbuW3ka0WJ5jgAO+AB3NcPq0EllciPUo41kIP+rl8zePT2rvbvUTaJmExGQdQ5wAPc1xOtanp15cyHyo3uTj98DlAR6Z5qQOfh+0C5ke2tfLRwBtJwR9PeuysfFVraCONGLRlcPHISSjD0PU9a5ITPG0wluF3HBBHII9jV0xLqh8lVjBQAmREJ59CRTTA6G68TJNJDIDDMsZyRGSrKPTFPtdRs9R1Q6lIhjEZzvkIABxiueguobWeeNrNJi4ABIzt46//AF6i1q3t7VInRMNKMkZ4I+lFwE1nVLe81Zri68udg2B9nOAQMYJNULzUBckeXAscIHyITkD36dajjjtGGUQpxg54ANVHgnQpuhkVXICsVIDD1GetCYCLh58ysfrmrNtfvAHPUDhOccUmpWkMFwiI20EDJJzk1CYo47kwl2AHQgZpMRNdSz3NtC8hyqnAXuOacZjcQlwHG3GwZzxUEM4VpAZP3YGORwaVgVx9mmwp6hemaloBolcEcA845qV5EQhOSuc4FQKrCTk4U9TQ77pAqEkg8EDtRYZcSYwkCF2VmIBwccU/92X2zM59CPWq0kkc4Q5AcDDYGBmnLGphcMQX6g54x7+lJKwxkkcNsrrLGXkfpggcVCgaRs54TkCnyeT5BcxyMcYHOMH1qBCdpI4HarQiaRJyxJXLEkgZ5I+lDQXCRLMyjySM7gelSRSi3txcAKZg/wArdSB6fSq5mkNvKhjBWV9+QSOaBDkkYZXP3efc1I86S25GcMe1NtzNFexSEjJwCAckg8Yq7KiXE0oK8AArg4z60mirmdcF5pEJ5O0AmqpUggkcE8GtCZrWOYkQB1HQE9aEsor7BhYQ57YJH51VxFd+EBB6kACo3XaSpPzDtVq40u7gw2PNjAxvTnH1HaqqRq5OW2HtuBAJ9Kq9xksMmSsXQk4BI4Jq1FFhJImALdcHse4qCaPyDHDIPmGHJB5HtT/Oea5LkgbuMAYA9/rUMRYt41ZiTnavUA4qzICYwpjVYxnYxGcD3qoAVJKjPODzjmpHklVUBBAcZB7H6VFxmhb2yG3jXyi0j9MHrWtokTC6LovmyWo+TDYIJ6k+v0rDtLsBIw0phycKxXOPU4q9Fp9zbzhY7lWjmORLGSMj6dqVwOrj8VQ2ssMAtRGskgEzseQT3ArY1HUltCBvCeZjYCQCR3PsK8/1C6juZnR4myFADdyRVea4mvGRrljJGE2RknBUD0q0xHpNvd5tmlmwMAnAOSR9KlglFwpIGCDgjPQ9a8+tpTbxIFme4Z0OxDwUz17/AFrofDl19pEskJckALIhGSfQg/h1qkwsdEVB7UxkG7NTYyOQQe4PamleaoViFlBHSoSh3HFW8U1lyapMCoRnuRQ0hAAp8kTdQaqybgcHOPWqTEyYASxYYZFZd1bBW3IO/NaSE49qhkQvIRjI71aZJTMSmI4HPrVcQOzYA5NajRAYAHHekRAJMBelPmFYpfYmA5YE+gqLZIrEDtWwIBIhwOtPjtAowRRzBYzoldlBwSCeCK0Et42wSM4qZYUAwBwO1KAAcjpUN3KSENujN3A9KelsgPGaTfilE4JwDU3HYV4wBjFVJSBkZ5q1uLHk1UubclwUJBJ5poTIRMyHGajknbdnNSSRiNxuFRTRiRcx8n0rXQQsd0QeTVkXa/3qyDkMQeoo3Yo5UTzM3FugxxyKezBl4rFSZwMZqzHdFSATUuBSZYYck05WCjrSAhxx1pu1kOSOKQyVmYjIGagEm5iCCKTzieKTaOtFgHOQCOaUSjABNQsQQaaDTsK5dD5UGlDVXVjtxxTgSDzUtBcsAingioEcHvUgNS0UPLfNTwWNQ981YQAgVIyJhTKnZeaiIOelACU5KaAacOBQAh60BsUp5phWlYCQEU8HNQgYpwNFgHmmgc04GlFAADinA03FHSgB+aaTQTTDQAHrTacBRimAgpwoApQMGmA8dKdTQacBQAZo3UhAAphbBpgS7/el3+9QbjShiaLATA5NLk0xOCM1KQMZFIBQ+COadkEVXJIPNAJJ4NFgJwRTs1ECadkiiwDzSGmFvejcDSAU1UuDyas5qKVAe1UgM1hk0Y4q15AHUUxkAyMVaZDRRkHJqJhk1cKDPI4qNo1yeOatMloreXxnI+lNIwcVLIMHioiM1QhM0AjNNJpAeaYiUnPSoiCDUkbgE5GaHIJHGKLAMFGaU03NABUitx1qPNGaAJ0cg8GrSzgAZNZ4J7U4MfWpaGmX/MDHg1NGQRkVnI5BqzDMSQKloaZeBOaeGwMVEGBHBo3EVDRdycHJpwNVlkIPNP8AMA71Nh3J80Aiq/nAd6UyErgGiwXJZJUUcmofNDdDxUQiBbJyTUojyMgYpiEJOaQnipQmByKjZCT6UAQyElcDrVcrLjJAx9atGEnvinCIAYJJpiM4FicDJqeGEsMsce2KueWoHCgVXkfYSAOfWqTuAjx4HAqLBJ4Bp4mY9aC5pCGjjrS5HrTDk0mCelOwXJN4HU00zDsaYUPemEAVVguSeYTSFxURJqSONmYHFDQrj1yegNSFWxyKsRw4XgUNCWGMgVDZRVORUZyatm2bBIINQBTnFNMTGAHvWZcKYL+NwMDPNbATmqWrRN9nRx1BqriLisCAR3qeOorBBNZo4IJxzVsRgDpUNlDCeKYetTMoAqE0gFGMYFRvkU6mtzTAgdx0FNFSCMb8nmkdQDwKpEgqDOakQZeokhdzwSB61ZhtzH0JP1pMESAACmEEnAGanCZ61IqAYNQ2VYijhI5IOasDgU7IAOaiLjtSuUSb/SjfUO7mgvxSuBIz5PtULEZzSZzR1FADDikMgQEk4FSBM0PEm0+YSB7UgMa4vLcSF9/4YoOs28sDQw27SMRjpxVbURYqSkIJY9/SsR5HjBjRiAPSpbGSpctDOyHbGATmq80wM5cNuPrjFVJSSx7+560wHAqWBpQXpEw3p5oHRT0zV9vEJDhRbKPfFRWkMVpaCaQAP3zzWbPPC7NImA59BTQHZ2kryKokUEHsRVLXrO2MZkUBGBySBxUmkTW62wkmuAWPYkf403UrmO8Uxx8g+/WmI5COM3FwI4ySPUmtD+z3hOGBb3ArQS1e3gLiJEx3JrOXVJ1mYhgw7gioYxJCUOBE3Heqk0jsmGBGOgIq5NqM0jBvLUAfjSM63IDSuox0GMVKYGQSQfSpIixbIqaS0UucSggdwKjCeWcBs++Kq6GDyN0JphORTpAxGTUeQOtJMQpA2jkdablQOADTSQT0o69KaQx2wvyDSZKcAkfSkwfUil+tMQmSetBPp1oz+VG5c9aBj7jXdUZSHfyUPZTWdb3c8M+YTln4+bkZ9aqq7OwEjkr3JqORyjHyzwOhrW5JdukukmLvGvPUgYA/Core/FvvLR72YYB9KlbzXtQXckn1rNdMMQTRcCSSYOSQMZPSo8jFNzg4pSCADSaGORyrZGePanBsvnBJpqTFAQDwaktA7T5QDI7GkDGO+W6EEUkTsA4zyRxRN5hmcyAA57UwcUWBMuaYoaYuRyOlMubh7i4YMeAcUlrMYJNwGQRjFJLsLs6jGTnFKwy8x82JNpyF6ipftiyP5RTEJ9BWfC5TBB609Buc4PWiwB5iKXjDFkzgHOKamSBGoA+bgjsahmVo5MEg+4qeNWjZGYgDrz0osIszyyWtm1v5Ybcc7gcin6lNHNZWjRuGIQA47GqdzcvNMSzAjsRSC3lMJlC5Qe/JoEaV1qFu2hQ24O6UD5j3AzWQjDYKjJB+lAOMU7jJpC8bhxwwOQfQ16RpMl1e6Kkhu45CQQ5K5K+3vivNJJC/J9MCr0UkltFvtr4xiQfMoOAaGBJq/kwz+TBL5ygk+YOAefSo7eUQwiQN8x74qoBHnLlvc5pz7CCkJLEetFgHYae9Veu5xxW7rAjijtoQcqBjPc9KybSIi7t2YkZcZINW9WUfbgoZ26Elumfak0BXddpIHao3mlZAXJcDsTUqENcoXOEJwaiu4fs1wUBYqQCDmlYCW3u3fERyIx0UHnP1q3LGZ4BsGZBkZPUj0qrY2S3AZxJsKkD60gPmb1B4FIC7b28MEMf212TJ+6p5A96lvRFbyJJE2YXTK85/z1qpb20ikSRxGQ9geRUtxD9ohDiMRuf4OmKkCKK5iQPG8W8SDGT0FXJtIHlR5vI0VwABsIOfrWbIHtmjMkXGSQD0NW5ba41W1kvUuIgsJ4hJwR9KpIDMNqUmKyH5QSCwORVZeZCiAk+wrRMUlvND5zgiQZJFJb+WhIt2AmdsAngEVaAq24R5gknCnqR1FdFpmLNLuezkdGgAPJyHB7EVkppkzXcZJUKTgsoyBnua1rSSx0fWri3uJt8bxgLKeQDg/wCNAD59f06+mjM1iFkYgOVOCT6girujTWkdzLaXt5LAUb90s3KEeuD3rk4LcT6iUA3gOcdsjNaGqXENztjMJEiDBcdSO1FwN2/1O/0u9kMM4RRjZsA2yD1966uz1S1vrJfOdYmYYYHjn1964O6sI00O21OIuMERyQtyAfUVp6FcrdQm0u1QODviJPUE8gkd8Urgbl011awRvbsriOQZJGQAehH6VLYavbz33k3Mf2e7QEEEnB/w/GsK4srhdZlj01yihQ+wklWxzir9vrNhc6pA0qAtMvlTgrkBgcjn8apagdM0IfAdQQCCPr61MAaSOLyVCbsgdD7VKAD1pAYeu28wtJnjvXhDjGxFGCegyTzjpXJwaVHoKwXF+6ESZDqy52+mK9IlgjljIdFbAOMjOD61yOt6JHZW6XjhpFQMXDNkE44GD0pWBM4ydUvWupreBUgU8ZJyR+PSrGk6m1hpU0VvIUYvlmAyceme1RCaL7AxVAEmYgg8hfpSaPewWJfzoZJWkIKhACTj60rDNyxZ4o7eWWOLzLpCAHyA+D0OOlZGtW8tnestwxGzB8onJweuOOa09Z161uLQFrGZLiMYhZyAAfX37UpGpeM9KtWMcIELkPJwMnvSFEyoYUhA1KaGMwg58ksQGweDVzUNWGrbYTa+Uxx5YaTIUe2BxU97FcSkx2UYnt41AARdwLDtg9Oc1gQBrDVIJ76N4thOUKnI/DtTsMW4bzU8uSDBY4EhJI4PapLc/Y5DcRSLJ5a4IYZFTkW2oLPcW8LlosuDMeCPYdhWc9w0iuoABYYwBxSAdczG4uHmMcaF8ZCdOB2qONwCyt2GRimojrYmTAOw4we9W9EgGq3RikBCwoXyoySR0BHeiwFIRtJkhshugBAJPvTUEkEhdR86E13knhayVor/AFJ4UeccxKSBn2PauLuEW0vZjGhESuVGWJJ96LANmkhldZ1BRmOHU9AaRZ4gjAkRkjAfGQPrTGMEkiqgIDYxkdDUFxbyW8nlyMC/oO1OwFiKRIrkpJJuhIwTjGalmaF0iWGDciHlie5/nVWGO0mvFjmdo4yMAg9D7moj+6u9scm5UfIOcgjinYRangkjthOYCsMhIQjuRUZ2yiNl8x1AIKL2NaR1qVpYGkKedby7kjwCrg9c/gP1qe/1bT57SSeGIW+oGYOYwCBj6DihIDLjYWt4swQkAchvelkO4sQTtfkAnOPakaaa5gDytlu2RVecBHAHAIyBSaGizKItojhQBgMZPJNK9/M6OQUj24BQDHA70sEqNZSM5HmRkAEjtjrUZjZ4RKVGDySPT1osBbspnuC5hJDgDgnAcelQ6kJI3UTSMQ6hgrAAqenam2qSJdDYwVl5BPQj61NBcW76zay6m5NoH/eA5zjHb9KEAlrpk1/YGeJclJCmWPAAGcfkD+VNjjjIBUgfU10WrpDoSC10ufzLK/i84GQ/d7ZB9welcwFCzARM0iZwCBySfShiJJpCq4GOfQ5q9MBNpFsiOu+MlhznIPWq14Q5WMQurx5D5AJJ4q1GtvB4cLiRGuJJQCFOSoz3Hbp+tJoaIB9qCgDLAdMgHFSLdOuSImUYw5PBAqsLg2JSSF1dnHzA8hSe49KllnjuITCxzI3RugJ9aVhlu1u0himjx5ykjqeQe496rm7Cl0MbLFI++PPYccH86R7R0ddk2GVAThcnPpS21hJfWjiCSMyIcCEsAx78ZpWEa1k1vKg3ymTy5gBsPzAdeK6KwujJqPmW6LsgbnaMF1PHT2/pXPeHbSK0RLq4cwzRTBNoXJUn1Hf2x1ruoNO3XouI2Ah8kDIABJJzyPpVJBcuW7s6knIOc4PapdvNOEYTOO5JNBFUhJgADTSmOadgikJJHNUBHgd6hmhDDHQH0qwc0mM9adwM4oyAjBNMKv8Awg1qeT7UjR47U7iaMd2cHDA9e1WI0O3kc1ZYc4pAoFFxWFjAReO9LnNKI81IIwBii4yvtJbIPFKQwGasiMAdKYUyCO1FxmdNK4ICjOTz7UsSsWBbinyxgSA9MVDJcCN8A496oRaPFPBDVnG6JOFOTUkVySeTzRZiuS3kLPGSOoqmI2jjIPQjGavGXPGaUqGHIzTTEzDEbl9p596srHAQAf8AWDqK0Bbxg5C0C3jBJIzmq5hWKRhB6ClFiTyGYVqRQxhBxU8aIO1LmHYp2tqYxk5P1qyY1IIIBFTkAdBUTsFzUXKsZU9vsk+UYFM3BVAzzViaYGXBNV2UE5q4kkJJzgnApA+GwakaM4zUJXDcVV0SycOARk1IGUg1WUZwDUgUg4INJoaZLEAOAe9WFaq8akc4qYKS2ahopMkB5qVDg4qHBFSKOQallFgAEc0hUZ6UgOBTDJzjPNKwDiopu0Um7NP4xSsBGQKQimu5DYFKGyKdgFoFIemaYWosBMDSg1CG96eG96TQEwwTSnGKiDcU4tkUrAITSZpCaTNOwDh1p2KYCM1ICDiiwABTgOaKMimAoAzQxxShgaa3PNAMYWOabSNkGnAcc1RIDGcVIq+3HrTMDNSggL1pDF2kU8HC81EZDgjNND570AK5GaVWwKbgE5NPVRmgYwytnAJGKeJTjnrSlBjpSbBQAA5NLzSBcU4EYoAZkg0A5oYjNGaQhGGRxURU9xU2eaCMimmBUcYFVnHJq+Yyx6VC0PJ4qkybFBxnNQsMVoSW+ATVGQc1omS0REZFMqXaT0GaYRg9KsQUGkxSUxC0lLRSBiUtKBzQQc0CAcUo5oCkkAVKItvU5pXKGgGnBipyKDgVIgBGSKTYIlglJYEgge9TGQE1Vdiv3eBUkRBIJqWrlJkxJPIHFDAgZNKXA4o3g0rDIdxzUiOeBThGhJOKAoHShoCxGcgVIDUMZA4qZASelQxjgM1G45NTYwvNMYZNJAys7lOlR+axp09VSxFWhMnM2B1qEnzDk0wH1NKrgUxEgUU/YCKiQ5NTKKGMWO3QjJJP405okUcDApwIHekYhqQFRjyRURBJ4q2YwTnFL5YzwKdxFaOIswAFX44DGoJFS26LGAQORUjsW61Ldx2IwQKa5J4FK3ApgIDYxmkMVc4waYYfmzn8KmAFOGKdwIPKFRXdr51o49BmruBRsypU9CMGlcDK8On/AEaSI9VOT+tahrC05ja6tLETjJxW/tpXuMhcFhxTRGcc9asYFNbAFO4miDyieoqNwFzipyT2qBgSxz0ppisRIrvIOcLVgwg8EcUiKQRgYqc9KGwsMAAGAMAU9FyRTT1pynHSpbKsTGPAzmmk4GKXzOMYqMkk0rgIXPemFqVhxk1CSc0hEmQaQnFRbjSFjTGSb8GnBs96gzmnDOaBFgNUF4W8tpA33RnGODTw2KrXOo21qMTnA/maQznJ9au+VMKIPUrmoJmAhDGIOT1wOat6le2txFmOHIHfOKis53VATCoUdD61IGPI4dtnleWKjVUZgBJ+JHFal00005cKgB6Aiqz28kowYsYpNDIiY2/dmVnHp2qJkQHCgAUhIEm3AA9aYxOcA8UWGPGAcrnPbBqSO5uY3BR2BHtnFQYYLkAjHcinJJ8/zE59qlgXzc3Ei7pn3nHGTVQYBJHUmnSMrEFFwPrTAMsRkfjSEIxyeTTSuTnIH1oc4PUfhTd2KLDHnkbQQKaMKSM5PtTA2aCxxxRZCFJLHB6VGwANLtyCcn8KFVc5bJHpmqsgGhCx4xQw2nFPKjPGQPrRsHfNAEfNB+7TmAUcE/SmEkigYhpM4opDyKoQ/S9EvbkzIluhXOC7nGDz0qzd+ELi3tGeEhiOozk4rZup/wCy9YLTbvs4JKkcYNYd94nn+1TG2DeW/wCAH6VQhmjXNvcW8lldBFOMKTxz/SsO9t/s05QvvPf2qFyTIzEYyc49KbikAADvxRjLgZOCcU+EKbhRIQF75qYRob4DIEYPX1poYlxZC32Zcnd7YxUbRGMErIAfrV67IvWSGBS7D0FVWRrcvHLCckY5HSqJIhHK8Zk2lgOpFPht1dSZlcemBV6ARnTHEKksRggDpV/SdGuGUSTTtGmMjoc0WC5zrYViAcgU4qvlF95BHQDvVvU2jknMcAVth6xjANUGB2EHgjqKTQ0y4/2cW0XlOS56gnODUEpdXAyRkdcUEhEHyYyMg9qjMjPwxyKTGTHD8nrTpJS+ARjAxxUStg4PSnED1pBYYVHG0n8akGPJCliPUZ4pAMUpQkHcjAjsRQFiEjAxTelPI4zTKYBnnNPAB5I5qOpRgKMdaQCsT0xk9hWiNzQYZVD47DFUgqAB2PI5xS+c9xIc9T0FDAuWkjRCGTOdp79q179UuNNFwgG5SM4HY1z5Z1wqjOOT7VpqjSWIljlBhPBBGDmpsA/TVt7iMkJ++UZKEcE0XUU10CPLJUdCOoNUY75re7jcKAAcHHcGrWsXE0LoYHKxSjII7mkBSB8kkHl/WrNkI2aQyNgA5xjk1FdWrq0QLDcVy2PWowCuSBkjvQBuWeoCacW0BCoASdw5/Cnyy4uDC5yw7VhiQxgyoSsg7g1PcTeTPHIH3t1Jz1qWgNC7lKGMuAUBzg9jWPHcFJJ0kUkTDAx2NX9Zn3ta4bEbruIx34qgkyEMChJJyCOoqkwJ4ZbV4JvtALSEEDIzjFV4Yz5QB4yQKBGrEoo+cjIp6tIFCoVBB5Ddc+lUmA2S5ukJhVpFIOCAMZFRRohmQXMjoueSeSB6083PnXAkcYbGCQOtMlkMtwccnsKdwNWe1m0hEvbSaGW3kwAcZIHrzVO7gmg2PKGAlG5c+lMN/LPZCykhQkHKk9QanuL0zWUFvMCTDkA9TipA3o763m0F7AyLuOPLyeh9zVea1aD7Kzx+WyLuIJ+8R/8ArqnJLDNJaSXFrJHDHhJCvUntWu97a+dDaGRZY42BErHonof8aYHZaTFGY/tAXazvvAzyARxWHeW0drq89rCoJnkR1BHAOck5rRea8tLeEWjRopJAjkGQR1AB7VY0+7h1CRpZIRDfRkK8TdQfammBrMRkAHoMU5Sc1DuOeetOVhyScADJNAFgE9KqatbW95YPFcXHkxnqfUfSoP7WgEyKH8xGIXenOD6EdqvPBExMroGYDIJoA801q3tVuUjsLxbiGUkPGiBQMDqAPX+lc0LdUbMcjqByCD3rVu1I1S4lvWkVnchSoAAGfQVmzSqk0iCJWRhgE54FIZZ1S4t7iO3jguDMyAlw3AJNP0vWb7RlmFowRZsbsjIyOBj0plo0bwpG8MbzAYjwcZHvVeVZrV2hmjBjJzkc5+h7UrCLts1xdROyi4juEy29CQJMnJ4H+eao6pO8typJn2hQp845OR71qaDeEagkReWOEgpGCcAEjrnvSa3HEuomSZhLJwCyHIf346EVQjKiufMjZW+XCEAgkZNS2zTpbAxoGYdSeoFOt9Ke8tLy6tldY7YZKsOvqAar6bJby3IiuRK27AQR9c0mhpnQ6dFaSsLa9uSYpITIHJ2eWfY/p7VPZR2ejzW+o21hMbaWMxTK5JIHHzA8d6oarfpa3MJhClrcbfLkGQw+lEWqzakkUN/IUtznCRADJ7AkdfyosM1vEFvbWhhmjkmuLScArEX3FBjkg9sVgzapC9tJaPbmW1I4kIAcH1JqrNqBsbh4rG5kK8AEnAx6EVTFxKwZHxtkJJAHU0WAS4CRzKPMEkanqODj0NMDMZjtDYPOT3FSmY5EZ2jHfGTUAYIzF+crgdsH1ouA1gN5AOR6461LDblpolB++cEKecVNNYLEm5LhJcDJCnBAqH7Q4txCBwDlXzyDVXEJKEjuWiRSgBxg9RSbcvt6nue5pzK0lubsyF5S+0ggemQaQmMJE4Z/Nz856DHtSAupb3MlorJHhAPvZBzUYtJ76KSZApaIDehOD+AqNiGiJzlsZNXNOlS0udwLkuhDkdhUsChHEWABOM9CemakEE0Z2FsnIA2ngmmlHdHKrlQck+gq5GxuNEZ8hWhcAE9xQAW0sltqkcVwRIhOxlI4wav6xe200kcEUYHkNsDEckVRMiDyJJiCs2ATjlSOMg0QgPdhXcKDIEkYjOPc0xEmpXhvbkOAQiqFRCMAAeg/Wqu8hQMYI68dTUl9GUmcAg4JXj8KZcCNrZJo3+fp5ZPI96TGQxySI5cO2Sck5qZXLk5G1T1weDjpkUyExvFJGZFSQnI3HAprkJlUKybCCT2osMl8sTOI7cbye3SggW8iHOdwyD0qbSy8t6TH5SSbDgOcAn2PrUd0u22gjwTICcYHvQ0BNFe/ZiXiRhIRjcD/AENNSzMkX2hjl5HwSDzn1NCWfmBFDqWxkAngkdRVm2kcSEW6IFHUEZINIZuaJqs0cDLc2sE1vEyxysFzKeyn3rtNLe4R5LeaTBA3xnBIdfcHoR3rntD02OXTLm7mDHzyFMYGMFT1/Ou1EJeQTAZwgA+h5NNCaHurKhYDJHakLAqCD1GakaUJG7N91QSfoKxkvZDcxrHCzRuC4GQD+vpTEkaZpD1qlLfsLuG3RCDIDkk8cdQPWroBNAxMU4DJpQtKBg0xDwABTXGBS5pHJIxQBTkGCTTByancDHNM2Y6CmmFgBIHFKJDnBoCkjpR5ZJGBQBKORkUhUk8U5EIGOtBDKeBQBEbQSHLCoZNNt+pUk+5q6A5oMbHrzSuFjHl04EERcH1qqbWaOQLjI7nNdIsPt+tKbcHtV8wrGTb2ozliTjoKsFAP4SateUAeBTJMBTii4WsVGXFVmmOSB2NWZCSDVCQFCSelUkJk6zFRyalS4AwScVnFgDnJPGaWGYEkHnmnyoLmzHMH6HimTdCahhcDGanbDjANQMyWR5rphjGMU8xujYODir5XbyKpTOwc5FUmS0PAz2p3lx90GaijL7vbvU4GRk07hYaIo8AAVKsa4xgGgJ6CnKKVyrCGJcZwKAo9Kec4pBQFhCB6UoFFFIBCTUUgbqKn25pRHx1pDKis4bBNS+ZgYNPMY9KjaHIOadxCEqxyDTkjyOtV3QhCR1qBZ5FIGaaQrmgwwKrk8mnLKSoyahcnJ5oSC5KDk4pwzVVWOasq+QM0NAmPzimlsGpAoIGaUwAnIpWHcj8wetODg96rzW8iSEr0P6UAMBzyfWnYVycsO1AcetVi7A80wSMafKK5fEvbNDPVWNju5NSSPtIGCc+lKw0TKxzzTywx1qmJDu24NTBjSsO5NuFANRZIHam+dg4NFhFmmljUIlz3p4IboaLDAk5pVYZ5NLsJpuw55oEShgO9PDA9DUHlZNSBcACgB+407JNMGe9IxwvWkUOLgUm+oC+TSqaBXJSc0CmDJ608ECgB4HFOApnmelAY+tFhinrTWODQScVC7HNNCuK4BGKozQ85A4NWt1Mf5hiqTJaKQQgYxUUg5q4VINVpB8xq0yGiHbTTnPAJqYHAxipUkUDGBVNgVTG6gEjANCgk4Aq+Cki4PSoimDjGBRcdiARHAPBoaMgZIqwgyRVny1IyRU3FYqwhVGSM0hYFjngVMYyO1RtbZBOTk0XuOxAxB6U5CcUv2cjvmrUMaomCf0obFYrhHcZCkihvMjGfLb64q+GAGKd5uFIzwaVyrGYJn3YNW9pEYakMMTSbiSPYUlw+2MBTT3ECtuPBpWYg4qsjlTUofJyaLAWYtxYZFaEYxgVnxTAgcYqys2MHNZstFkrmmsmBmmCfNSiQEVIFOaIkE4qk0bBsYrZYAiqzxAnOKaYmjNaM4xinRwnOSKtGP2oRCD0qr9gsRiHBzijFWcU0oD2pXHYrHINOXk1MYgegoSIjkii4rAEzUgjAHSlUHpiplXjpSbGRgEClxUu32pdopAV2TIpuzBqzsFMKg0XAhzSrk0oQg804Jg0higVICFplBOaYjn9TVrbWVmAwpINb6ncoPqM1keIU/cxyDqD/AIVesJRLZIw7DFIosk1Gx5p9NIp3JIyD0pwjHU07FNJp3ACQOlRs/NKxFRHmhoQu/NSIeKgHFTIwwKljJc0A0gIoJAFIY1wSKhKnNSF+KYWAoC4wqc0hXilLjOKTIoEKoFSDAFRAgUofmmAjknio7jTobmDLplvXNS5BNPByuKBnE3sIikeOEucdhV7TbQtbl5LjJAzszwK6OO1t1bcYwT6mpHiiaMqEAB9BUtAcVPcE3BOwnHAAFQSXs0eMgjPTIxXYhLWyjciIHAycnJNcffRS3M7XDYQN0BPak0MpOysxY9T2pCUKjBINKUwegNNI5oEOBd8AsWA9aCu1skU0EignOMmpaGPMpHC4ApqSEODnJ9TSEKRmnqFAwRRYBsmDliee1RZyKkYDdwM0wkA+hoAQEU7BPSlMgK4AH5UIQGGSRQA4KQOQabJG4XJGBUucsFjJYntVho7mOLL25I+lSBmgkU4OT0UGnvICceVj3NN38YAH5VSAYScnim49KVskZpUA6mnYBhBoCmpUUNJgNyP0qVlBGByaQzQv7y/ltFutT0suCcbiSAPw/Ksi+1GyfTTFbQlHPUgYwf61s63q0N/EM6jD5I6ovU/WuQmMfIjkLjPUjFWSTWRsxl7jG844PQ1VuvLeQtEML2Gc00LmkKkjCjJ7CgCLgdaN4xgHFWEsbt+RCcDnnvTppASA6qSBg4HSmM2dJSey083EcdvISc8vg1RfVmed2MShnGNmeBWXuRWGCQo7A8VYuJoLhV2whSBgEGgRdttVmsGYNbxlCc4zjFT3eqXtxbhg3l257CsHcSCCSfrVkXUptRAV3KPbOBVXA2NBms7TzftkqK0mMZHGOaoak9mGlSEiTccq46AelUovKeby5n8tfUinXCWqjFvIWA6kincZXZ2dQCcgDApi8nFPAB4JxT4YwX5wR6VADAccmnJMVbgA/UUrgbyAOAaYQM5AoYXJw+1C2cVLBJJcZDkkkcZqq7ZjxjpSxuUyBnketIAlBRyuQcelRE0p603rQAbjmnAmmilzQBI77gPYYqSwUG4LnoBk1B7VIkvloQowT1NKwE7v+8c9M1GJTHkRuQCc47VDuJYkmgfMcICT3xTsBbMolTBAB9RWhYq115cEpQpGOAecis1bZzJtUbvcVLHIbdwGGSO2cUgLmsTBbhI4gRhcAEdRVY7owAwwe4p9xew3TxySMAV/h6mhysmJOo7UmBDKuOQcggGi0h+1h0DhCOme9Tz2xKh4xlSM47Ci3tt7BwCQOqA4NCYEEjuRtlkDbOAR6U2FyJBg49z0qfyrP5wBIA3QE8io44kiyxctjoDTuBLJauoJJB5PAOTTZIIw58x38w8jHpWhYRwuI7iSQA8/u/SrM0STyGeFxuTjB7/jUXAwI2EUhzVmzKvdkxbAwwfnGRitG50t0jTz41LOMrg/Ngdqy9iQXDBCyNg5zzTTGaNm9mLlz5Q87P3nHAH0qxcalayRNDHCgz0Yjkn2rCjMgPGCT1JFT27s84CxedIO3QUxGoso+xS2sytlzuBxxioLGEGQMIxIsZ+ZD1Iqe4y0aIYwZBghVP8AWm2xSwf7RMrKB0QnJJ9jU3A6O3uI7tjbm6kiUt+7jkHQjsPSpdbWFoILkTKskLhJHU4JB781mwzjVGhVJ1WQAFAw5I+vFWjZXwvw0qwSx98DBNUmB0NlO6YHnm6t3A2S9WBA5B/SroIJwScdwDisPQoHtI54XARTIXjUnkeo/lWuDVAYWsWlwoLw3Zt4VXEaKMlue+O/NT2E1zZQ2txKZi9x8hSV8kZ9D7+lajJGzKZAp2nIB6ZqlexveX+wOywwqGG0857UAZPjmOEiCcAmb7qIoxknqT64rhZPMEhEqsMHG5lIH612UyR+d5zzXCXCHKhx3x+vSrOpibVNGt7uOePGAHhkQEMc+tAXOW0eS1WW4E8kQIAEYkHBPPetXyZmSdCiRxzAKAGDE89QPxNYd3aC81IwW8EICAB3hHAHqa09Me90JWiNqJQXDiZfmBA6gGkIux6ZI17DofmRCPzd6swPJ9M9vrV6fQbSy1bfa+XBOgAMLHzIyD3zwR9RVWXXUmgF5YQPb3UTEgkAhgeCDmt6a1Jt473U7h5JnTYEjBQ5IzgEdRQVY5rxHrqwW1xpiBDJKAC6NjBzn8aZpa2Wpwq0ttFBe25UxkHG8gcH6dK0Lm0lEC/Y7KFIueZ2DN26k9qwJrhINkzlXulfAKZEZHcD2phaxDfvNa6hM1wI2aQkmMMCV6elZ003mBSAVZTxg8AemKs3kkBgIeHFy77jIvcelV3jaPCiMqQcgnuO1IRdjjs7yRZpmihGCGVjgk+uKypyUuCgAxGeMDGRWj9plkSbdDGdxAGRlgQetM1bUYtQ1FDs226AJkLgg9/1oTGUiy7g5BJPQelN2F5ACQAT1JwBTljaWUpCpcg4AUckUx0cRhucE457GgB09uYXPzRkAjBU9ahJAxg05juIBGGxwcYBpoRmyeAAecnpRYB+8vkkjgdAOKUsWjXOCV4Bx0FLmOJm2kuO2aEIaRQQBu7+lFgJrdUmlCyvsXGOvU1f0u2Nw8xDYMCZyQCDVO1UNcGPyPNZgQoyBg1ZkuxDCY7RPJbAWRxwWPcEf560MRAQ6qX8sgMSQD3HcVr6cNLm0+7tXEqzXDboVK5EZA656cnisgXd2sXlNMzR4ICHkDPpV7TpPtlvLZFELBP3JIBIOc4z+FICjeJcWUiw3MCccqQQQR7EU+6vI7mcRkiMAYLA8UpmjacRzAgH5HUZJ/D8quQ2dpH5iW1vPMSADHKmDjPJFMCg92rOsSZIUAAkYyBT45I/s88TgCSQhlkIzjHUH61oLpKAC4vilvGExEjn9457Z/z3qsljG9lI7lzMWwoH8Psfr/SpApxIJVdmERJORkZwe1LtO4qV2sw6AYFW44ZiskIVk+QlSRzn2qpDdyRzeY4DEjaynpTuMlWI5K4zjqO+KiY/KAzAYGBn0p32iYKRGoIPGT1A9KlDxIwzErxkcgjofY0AEciW8tuXicbAAcjqD3rRaNFVxbSbzI3BXPHoDmqt8fMZCJGaFAEAcYI+p70scjiML5gTJzkDA4qWBqafrtxpVqYdwZc5IBIOfauh0jVJNTv2F1N5ChRjMnc9CM1yLypJAY5URSTkMoyT7GgTg7XkicnGBjofSlcD0rWtVew0ybyl3zFMB+MYz1xXO3mqzWWm29vCcTbQHcjkA+g/CucS9je/iaIEgDYsb5wGxgH9a3NLtb27a4kmiDQrmKQE5cHuAPwq7gWtPeS412CO7kKxpuMZY8luOPb2+ldmHA61wHh4C7tCZF+eKYHexIyDnNdXbSOqgvIW9M1SQjXDinZqlHIHzg8g1bQ/KKLAOoJGKCRimE0DEwDTGGDkU8nFNJzQgBRxS7yOMClBwKSmIA5U54NTJIGGTgGoDTKQF4Ee1OzVNHKmpt+e9AEwYCkYjFRbqYzkcZoAeWBqvKw5GaXfg5zVaaQBzzVJCY12XGM1XdY2Q7jiopJ1WQ5GaZ9ojf5c4q0iSdYYmHCg0htQMlFANIkoXGDU4kZkJABAPei4yq0EkfzbiRjoO1OinlDhSDz3xU0c7ZIK45qZsnkYB9RSbGBcYweDTDGG5xVeZZELOCSM5qWCcGPJ4+tNIQ8Qg9KmSEAYNUmvCrEA/jThqC+uKLMC/hQKrSMQTioftfGd1IZxJndjBGOKEmA/zs96PM9DUP7vAA6CgugGAeaqwXJxKM4NSCUVSRxjJPPpSmYdqOULl4SJ60vmL2NUPOA7E0vnA96mwXLpkGCKbmqvnEdDS+ccdaLBcsMMjFV5YVPajzT60hkz3oSFcj2YHBNJtJ61Ix4JqPeexqkDAjBoBxRgkZxQFOc9qBEqzEdKlSdgc1Wxg4yDTs4osO5bMwYYNQuDzioCzDvThKehPFFguDJuHPWo1BjJyKl3A9KcMHrTQEW7niragEAGqbLhuKXzCAADSaBFwqhOTSAKOlUGkcnO4mgTuOppWC5ocCkKqwwQKp/aG9aeJ896dguS+XjpShSDmmiX3pfMHrQwuTqx707NV/Np4kB71Nh3JxilzUPmAd6QykUWYyRmNM3E9ab5gPSkLYosxDtueaMYNND+9IZB2osIfvAo3g9TVdnINRliTTSC5b80Doc0vnYqhvYHOad5hPU0+ULlsz5pA4PWqhf0NKJMd6OULlgnmmg880wPnrQTnmiwXHvgrVeRSTnFSgg8GnkDAAFNCKJXBwaAFHTrU0iHqKYEyaq4rDVYAgE1KrAnBAI96a8QA6Yo8sgZBzQBYXafQVKpAqgrkHk1YSUEZqGmO5ZIBHFMYcHHWgPSkg9aSGVy5BxinA5ApHIB4FJvqhE4UnrSsABTEmGOTTXkBGM0rMLkUrYPBqLJYYND8k0ijiqRIqgZ5qcR7kAHFQHrVmIjyxihggEDxjLEY9qcHI4FORTjBJIoaMckVDLHI/PWp1bmqeSpFTAkAVLQJlwP60EgjAqusmTgmnhhQ0VceUpNoFKHzxQTSQJCYpdoIopaLiG7DnpUoTikBp24CgYBQO1OAAFNDgnrTgQaQC5A70ZFRtntTASD1oAmJFIaYDzS5oAXFNPFLmmmgBCeKAaMUGmBV1OITWMgPYZFUtAlLW7RnqDmtOVPMhdM9QawNJkNvqLxZwCSKQHRdaMUmeTS9aYhD0pjHmnk8VGx4oAaRTGHFOLCmk5FMRETRuNBptJoZKshHenlwR1qvnFBbihoLkrEetRM2D1phY00tQkIcW96Tdg9aYTSZosMkL570Bsd6YKUCkIkD09ZOcVCBT160DJgSainvra0/wBfJs/DNSg1natpovIgwJDDPI/CoYzG1HVY7i42wPhfWqDwl2yJJHY1q2/h1ZkLEtkAmr1poojOCGB9c5oGc+ul3EiFyCABk7jWeThiPSu5OmTzfIZtsZ6jH/16zNZ0OOKMeTJlyccjFSBy5y7jaM5pzRFBk1sW+iFVBds/Sqt3plzExY5MfbNUBmE4p6AMMliPpUslu6DLKB9abFBJM2EAyOeTRcBYgA2SMgU7aWfekII9+KeGkBEJ2gHuOtSl0hARxkemcZosBXkid1L4QewNPt5IY48Pgk9iK3ra8tPswWARrnseDmqT6NJJL5kjIg9gDSsBnIyCYSLwQc+1asWtEAIVDH1JFUbq3S3JHytis8sCeAB9KGgNHU/Nu3EoQAAYGKztu3qKXzG9SfrTS5PWlYCOTJ+npTAMnpTyTTQcUwJocbgCcVox20K4d5H29wBWSDzUwMpGPOIHoTSGYCzwswBiVB6iiSSELgHOe9RJbGbOCBj1qzDbCIjOHI7EcGqJGWkUl0zJGw2qMkGpcJbsBnDjoa0RPIkRCWe0dyoqfTr7TE+W6ty0hOSSmTTTAqC5huoRG9wscnqTwKqXelyWyNNJPGynkFTkEV1lqugXORNCc5OEMe3FVdV1fS1spLGzsRgjGSpwKLjRxsYiLgyEgetOOzOEbI+lWleOHGUB9jUU8yucrGqEelMCNkQRkh+ewPWn2iFpQGkKL3YdRWlohthDNJPgsOxFZknBJ9SaAZNe2ZhlRjNvibqcc1WMJJ/dj5akGZDljkkgVLNbz2uzJA39Mc0rgVHiMbAHvRjAqX5pJwCpf2HU1oS2kkU9uEtvnYfcA6/SmBkck04jaK0tWtjbyqXgeHsdy4BrNbnpQAzjGTnP6U0E07nGKbigBTzSAYqWKMMCS4BHY0ADOcA4NAEZUjGRikAJ6CpyQ7jPamMeeOlIZHtzTxExGSQBSoQM5FTRoXjOADjoOlICOFcSBgAcHOCMg1dkuOhhtkjPQkc5FQHZDGR0fsKhMznvTEWY3chxnBAyDS2xWe3kVz++zkZHJqr5pxwetCMVbINADSMOQeo61bUSG3RIxkg5I74qBgznIGc1p2DGGEgjMhBxSYEAuprS6BiOFIBIxxn6VYmkgu2V4co5wHAGAR61IkgudOk81QXU5Oe1U3hcTxi3BO8ZAFSBf1QWsHlCC3VGdAWYHr+FZhQsnmKPlHfNWLiZjEPPU+dHxsPXHtUJLCMEEqpOdoPP0pgLGcymMj5SDz61FaTSRXYQMSC2057j/JpkkzCQMDgjpUOcuT6nNFgOu1G7je0KbsTQYKN3FYl/PFdok8YYTAgMOmTWd0PU1NC4LiNxlT1HSiwEgdWjyAQ47EcGrssUnkw3Vo3lKww5UdD6VnSReXIXDkp2wa09KZWsrqFzmMrkD0NFgK9tLHDJIRIznqGJ5JrQ8+K8tdskUkjno2cAD2NZCRFQCR1PH0q9HKI7aaPBIcADHbr/AI1LQBbYeQmMkNGMqCcZx2zW7a6958sKpEYreZsEtglT6g1mw2D3mmWUUe1JCXIOMkimNbSp4fJCE+VISxHQDNGwHf2txDep5LkeYnB55z2IpL3UDYjbHazXE3YIOPxJrgfMElpDNJuWPdtLZII/Ktmz1S6hvIIWu2mt5hgMxzjPbPetEwLV1PrGoYhuYFWEHPloQSD7ntWRNbmESObp45i2I4kmJOfQ+ldRd21xYRCW3kLOSAS4yMk4ziuR1iK5vNTuAsMb+QAzSW8eDnsCfwouDRBLrN3bO8FzNHcFMAAjJX2B7U2bUb59Ljt1hSOzJ78kn1NZz2sgkHmo6NIMjKnJP07VeuLO7iUSXJPzLhAOB9KAHWF1bx6kkkrxxRoMbUJIc+hNbsaJqF1FC0aR2cKGV1iJwTnGM96xdLitGgNrcQAkksWPGBUkGrT2iSQQxq0YJUE8gipbA6GSXSTkyEOW+UbQQFHbA7VBfaxdWV5BNKzTm24EfQEEEZz7j+VZaTpdxSebbRoVAw0ZOc84psrS3E9u88O8RAAgH7wHtS5rDbE1bX7nVmEJtxbRuc8Dk8dzWY7qvlo6+Zbx+hxz7etXdYntL27RbSF7c/xFzgH8Kz7b7IhInhkmmyeM4WqTuFxkt4JLkzwoEz0A6ClFyZoBHNyQchu9Ty3WXYi3WNWOdoHFU5HVpSQgAHp3oAuExXW7zd6YHG05yff3qjJYTRMABvyM8dqnS7mXCqFHOScVaNzDM4Eqi3uFHyOvKn60rgR6d9lt9Tt5YnlIGVmJHQkdaffW9hbwyMb43FwW5jAx9SKzxLcCTch2EkniovuO5lGS3Ofc00BfvY1hsoI0ChThxk5OfaqU0Ukaxl4THkcA9SKJJWeNELErGMKD2pjzSykCSRnA6ZOcUwGYyKerYKkDjjB7Gr2j6Y+pXoQIzJGwMoHBxn1966XXfDttDp0F1aW8nmGQRvGATuHqR2PFAHM6YxF+zDaZFQlQ3APrz2p0lzHcXDyTIoxnhRgHFLMFshJIkDp5wwm8ZOO4/lV9dMgmjhJItkKgAyAlnPU4HWkwI5bW3t7O1mIIkk5II+UjGetGlz28N6iRIDI+csB044x7ZqneyWvmR28LEeUAN+SQT6YPSpNPn8iY5mCFwQZAM59qLCAIdOuhdPGGIYlASRg54z+dO+33uoSiWV2JUkBkGNo98Va1KAyWkMKhFQZPmueSeKz4RLbHy8jB6470CHSCa4k3vO+UBIMhyfoKmtHufOeRT5igZcscZ9zTnNrGrbXdDJgZJyBUVtcpbeYDlldcDA6ms2xl+yUyzRL56g7skucAfQ1kahbyW2q3ETsrEtuypyMHn+tSPiSLBBikH9w4B+tRiIDguSe5PJNNAKPkCgEAg4IJ5yau3AFvMsIHX/XEdCT6emKqlFBRg0aEEYyeTV6SUXVwkEIDzSkAkjAzQAXEjW7hSA8iDr0GD0xVd5vMjA8lcj9KkuUQQLMkoJQlJIicnI9KrRyEybwCBgjB6GgocWCMWLYJ5G08AelaE7YFuudvAOB0B9apBlxtkB6EEYwee4q/cxzSWMdx5wdET5QcZxSaEW7Sa7l1K3t4EtmV3BJWMZGDya0tU1O50TWZI7UbfMbzG4yrKQMHB6Hg1jwSMHs4XuFiO3d5qjlT7+1WtYaS5lLzzW7yP0aLkYAxx6UJjLMWpiOLyjCkSsMB1BCn07VtwTSRJDFI6kkdVORj1rnZJZNTtrexZBGwIBnzgEAeuBirkV1NZ3xIliJhxGTs3AjHfFaqQmjs4sKcgYq2j8VRt5Lgwg3AiAwCGjzgg/WpUlBYAGmItls02hFLDIpSpHNSMaTzSZpxHNMxTuAu4Uu4VGTTc0ASlhSA571GTQCQKBEuRS78VFuIpDICcd6dgJhICcZpWYYquXAPWlEgPBNFmAOSQaz5/M3EhCSPWr5I9aaxU981a0EzDnS5PzshC5xTYlJIJGSewrcIQjBAI9DSLHGmMRjirUibFGK1c4Y8e1WlQouCKkZ8dABUMxkkXCMFqXqMduTdtJ5qRXGMZqMxKqrtHIHJph4HFFgJpRvQgHrWbIkqNtHOO4FTpMROFY/KetWJNp6GhAZTgqu7vUOCeas3CnkAcCoAG9CPqK1RDJEkwME04yHtUQRj2oGRVCFLtnqaNx9Tmmk880mRjigV2PErA8mpA+TUAIzTgwBosFy2uGGCcZpSoB4OariQcDAqVXz1NRYpDjkDNKJAQBjBp3GKaAByRTGPBG3rRzSZFKCBU2AevTmggDsKaGz0pc8cmlYBcik4pMjODSigYm0dqTFO3CkLjNAhpFGKUEMKXApgNGQKeCRRkd6PpSGLkHgjimGMjkdKXJpwB70ARhTnBFIYhjgVPgUhFO4rFRkIpORVk1GVGelO4iPcR3pwk9+aQrxUbDBpgSmQ+tKJCO9Q84zSc9qLBct+b70ed6mqu40bjRYLlsTAdKUyZFVgSaU5FLlQXJhJ70hkIOajBOaCSTRZBclLq1IeKiyQaeHBGDRYAIGMj8qjJqQ9cio2zTAM0A0gUt0GacFNAChjmnKTjFN8tvSnAY60Ah2CeRUi5I560zBHSlUkGpsMeRxTSoBzjmpeCKYw4pDIJGPSjJCYFOIBPNGKaYisQeuOKASDVgx8cUxk2jOKq4h8TEDkk1I7AjrUC5p9JoaYxm4NQnOamYHPAqNlIoSExA54xTtxI5NMxindqoQuQTTzHkDBxUajBBqYMNwzSuA1oyoyafFJtPWpJQGTgjNVSpHcGgZcEwJxT94PeqUZw3T86lJwM1LQ0ywcdaM+lVlc568VMCMUrAPBqRW5qEE5608Umh3Jgwo34qLnHWkJNTYdybf70ok96r5Ipd1OwXJ9+O9KH96iBJFFKwXJ93vTg5FVtxFPEhosK5OXzSZqHcaUEmlYZKGHrShhUQNOBosBJmkzTM0ZosA/NNNAOaDRYBoODXN3QNrrIYHOTnp610ePmFYniBNjxzA85wPbpQBtggqGHQijd71TsZvNskOegqbdTEOMgzTS2RTD1oBBNOwC0wtk4zTycLx1qBjg0AOPWkpwpDSAaaaSacaQigGRkmkIqTb60u30pWDchwaADU20CjaKLjIwKcOlKRSGkAtAPFJmloAeDRnIwRkelNpQKQEitgYAAHtTgxqMCnA0WAJZJlGYxk96yrnUkGEdg8h6EdB9KvXcUsybYsZPXJxWVJpsiSB5FiQepNSBQuJbpSWjmCr1wKqJczzMBK5Ye9P1KaNZdkRzjqR0rO8yT+GgaZfuMsMEIB6g1XjmSAjHPrVYs/U9e9NIycmkO5fjnjdxgKPwqK7G+VQnOetV4uJBWp/Z7FN6zx/TNMCxp7GGPJhhGP4u9dNaG3EPm4DMBnmuIktHKFmmAx6HrUtrepZR8NI/uOlAEeuu82pSFAQDjk96zxDIB0q3cTm6lLgHJ6CpYG2KQ6An0PapAzxnvQatzGM5yoB9BTILSa4JEcZIHc8UAVT1phI7VPcWstvJtkABxng1Ftx1oAQGnrITwx4HahQCcE4pwUZpjZlCFRypwexqzGuwR7+CTjNUmkAUkPhx2xVnzvMhRypBAxz3qRGg95La4KEFewI61mSX0xuCSsYBOeBzTZJzIuCagwAwNCBo05tUMseCu2TAG8HH6YrOlulEZQgknqwPWhsBd1QsA3NUmACYlApT8T1ojClsycCo8gGnucAYqgL1oEEc2zuMmqUjZkNNimMUgcc47djRc3HnSFvLUe2M0wHHhMjqCCKv38sc1rBMJF8zuoPFRNaG5eC3WExzHuTxW7c2GnwaesE0OZgMggYOfrSAxNFW5bVY5LcRtIpB2scZHQ11/iKPymt7hBi6yPLHbNcxbW0MShwkgmHRgeRWnaSG4uhJevK8cYBQk9COlK4E2uHz7KGO7ZhcAg+VjBNcjdQywy4Ee0E4GTmup8QskzQ3IXzI8FeuOwrl3K/3ifrTTAW7sprRFdyCD6DpUKo7LuCNt9cVtxW8N34deeR3LjJBJJ5BrMgnmhtyigEE557UXArbSBk1PbJ5rgHGO9QZcsfkOfQDNTrMRAVAAPqKACYRq58sfUjpUW3PQVdt4o2tgGXJPfNOkszGuVXNIZnsvGAOalWQRRjHEnrTWyTnvUgi+1ISCAV4wOpoAqlixyTk0o+7n3pRDKSQEPB7nFOSEhCXIHPTPWgCMcmnLkkAdPWp1CiNwygEcg1CqM5wOPegCym5CFIPPcDIqciRJRMq5A6jPOKmtJPMgMTDJQZHPWriwxXIEasAzfeB449qm4FSNw058pW8mRcEkdD61JZRJG5e6dk2Z2FTg4pxRrcGHcQB0IrIvFkFwcMzAjOc4pXA6MXsFxiOO3DxnrJKOaw7pj9oIIUDttORVmK7MOmrDI4Oc5AqBNsgLEHGeKYiuUyw4BxzzUTAmYkKAP0q5JGhh8yLIJ4IPtVaEMRz3HNNDEIKkhwRnp70sbNvUJ97PA9atxjKZkIIGcZGTUNvIFu/NIHHYDANAFqdYZFZGARsZBH60lnMLSBo3jJLDAIPSq8kxlkLHjJzj0q3bhDCJZBuIPBHc0CGuUhj8l1BmJ4JBGKlkjlSyYghwhG5lGQM1ZikEmp/apoV4IDBjkAetXdNu7az1SaOOVRbnqhGQT7CkgLWjmPS9A+1ys7NKSEH93J6j0qewmR9L1S1jAkygZc9Dnr+VZr3st7ZXJlnXy0kysZAGBnjBq34b1u0trYWlyqpgkiQjI57E/1p2EGsWkdr4RgQJtJIJ56HqaxbCKWayjQOseHOCTgg1t+IbyC+eNYLiKSNM/Ju4z9e9Y74hlWGYAqg3fKO9JsaOguPEN1dIliyLGchZJA+S4Hp6U7QIrea6uDCSFUjJLfewe/sa5QW807SPGreWpJx0J/GpBZXVjcW8kxMUbkfMjc4/CkmB1utlIrma4nhdIyQoIPJAGM/Sud1HVLW4ePDygrxnGRj1FGuzn7QkYkkZSMfvDkVmSyBoBGw6HII6imwL1vMbWR42IkjIz5hPIzUUkFxNbtPb2bi3Q4MpGAazy0kskaA4yQuT059a3kW6J+wXl87wqBsQdCaLAZsUbIuZOEPUetS/aJH2yLIR5ZwABzj0J701bS4mvJo1dXEIOWzhQKpNJNgqjgoTgAVLA2LaWw+z3c19GWkdvkIGcVmC2IkEpdADgjJ6n0qANIXMUj7ckBlJwfarOnuLSbZKpeEE5J5xTuCIJ2lMx3oB6gVE4AkBHCsMgVog241HewLRuTvD8ED1FVJkj+0FVcOATgjrTTGEcai38wjocE1FKU2A4DEdcjirsUASMmaRQndepNVpYWyz5UowwvPUdqGAyHaQGNSgowMXlfPnIG3Oaa0JjEKgfORyuanhnEDknEgKkYxzn2oAqzQxnkDAPUDpmolhVG/eduoqyY3lUAAoT03cZFUiCQM9e9UmBtaRqF1ZTSG2mSKNiC+8ZBA/X8q0dU1qZpYX82VsHIYEhCO4xWPY2f25LgEmNo4N6BAMuR2PtULMn2QLMSGQYUg45NFwJr64hu2tl84INxLFjnaCR1rahsPsghvC7XFwZR5MyOSpB6ZB7da52yu4rO6E0sfmgKRs9f0rXTxCgu7cCGMWtu+7ahILZz29jRcChrSyQ6hNbzRoGDGTeBjIPPFUzj7PHIOuTirmrT2l7JHJFI7yFzvLoVOCc/pVGQRfaPLhOQSAPrQKwokaSYGR2Jznkk1ekjEgDxkbmOASaozQPBIN+Aw6YOaJJTLHEpAGzoRwakqxdkij+zhm2lgQCCcEGni3ijsjPKSzBtpRemazXcsrAkkkg578VpvcW95EI08yOaXDOCflBAqWhWHyTtNAHjij2gAEkfMMdiaWa4S9+aWMRgAAEDBB9c+lUJImTLKSOcYzwalSZQBkHgE4AyD9aAsTGCGBT9rtzMJDmNkYggD2H1pNMkNlqEM0bY2EMQwyCPTFP0+dzdBzMnCFIzLwBkfSpotHkmiuvLkjd4mGJMHGepGe1CGVZ7rzr5pGWNoySQAMAZ9AO1VskSHI68gjoRWj/ZNzZp9rn8lVTHyo4JIOMnA+tV5Iw9wY1Jj5GBIccGqAbESbtSpzkAAsOBTWLglASQCeAcfpTH3xs0JJAU4IHTNPUlJFEZ6HIb3oaETQZEW/DEHIBIPXuK1NKMV3ex27R3DuQSqQ43FvUZq9pUy3djPZXVolwgJmAJKiNsYPIrtdA0e0hgtrh7WMXcUYBcAhsEHoPxqbDOPu11e1kgsltS4nhZUjyCTznJ9xnpWVDcnTrkeYCJlBEkcqHBB7EGvXpYI5GQuuTGwZSeoIrD1zQ4bvVbDUGQHy3CSgDqCcA++Kq1hGbosySulqtxKBICfIKEYHsSOldDHpKRrgFyfXjNT28BiWISMXeHIDHBJHbJqw8hzVpu1mFiJECLgEn3NKRmlBzTgM0gIWXioiKtsoIqu4ANMCJhTAKmK5qPIzimIMUhFLmgnNADT0qCTI5FWMUwgA5NUgKQkmJOMkCnCbAIKkHvmrbKqKSBisu5l2zMM4q1qSSm4zwcg+9QMzDoTUJJHzhs+opDISM1fKiblqGZgwBJI96smYnkcisozErjGPehJ3jPByPQ0ciFzdzSeQNHkHJ9KjDBSCSRjtVUTk9OKdKS6AjjA5pco7otNckgqOM96gM5zgniqIkYd6PMb1quVC5kXxtc5NPAIxk5qjHMQeTU4lyOtHKCdywTkZ70gaMDlB9ajEnGDTwQQR2NKwxkzAghDj6VUJwcVcMOelRm1Ocg00yWisTk0mKkkhZecZ+lMA55qhCGm1KVHamFTQA3dinLJg80hXim4p2Fcsic+tKJ/7xqqKcKVh3LfmZGQaPMOMVWBIPBpwfsaLDuTiQ+tPDZGc9Kr7gRjNLkilYdywGBI5pfMAOM1W3EdDSbjnNFguWiwNNyDUSuWOCcVKkTOQAeD3qWFxMkU4GmkYYjOcHFJzTAk3HsaBKV9PxpoYU4ANQO44XBJxtFKJiTjAphj54p6xgHJqQHhqeFDCjYMA09MCpHciK4PSm7RmrLBT1oCqT0ouFir5dHk56irYRT2p2wAcCjmCxRMHGMVA0bKcYrSZDnpTDGD1FUpA0Zu0+lKFJOMVe8hfSlEKg5A5o5ibFaOMmTFSmIlgAM81IqbXJ96sJgjOKXMOxnSKEcr6VHn0rUkhSQcjmqjWgU4BNNO4NFU03JBqV1IODTMZqriHBwRjNKMHrURyOR1pVY55p2ESnIHB4pMmgOMYBzSgjFIY5WJqTBxmoselSK4AwaTAf1phU1IroakGw+9SURqSDzTiMilZPakAIpARGNhnIptW9oPUU0wg9BRcLFbNBG4YqV4cDIFNC07iItuDS+1SEGmYIPSncVgAzxTSvJGKmAyKXHNHMOxUKY7U3birboCOBUJUAdKd0JojGfSkJINPYnAGPaojnPNMRIHJGDShgOSAfrUYNLuosAFsdKXeSOcUh5FNNFgHg4NSB6gzShsGhoLlkPz1qRZM8VVDcUofBpWHcuhuKKrrJTw/vUtWHckIpKAwxTc+lFh3JQ3FLuFQhsHmgvnpRYCbI9aXIFQgmnBsjk0rASblp24Doark4NOByKAJd4z1pd1RGgVNgJg9LuzURNAPNOwE4alzkVEKeKVh3A9Kz9ZjMlgcDOCDWgeRTLiMSWzqe4pWAytFk3QmIjkDmtEjFYeky+VeFScZOK32FUBC3WmgkGnNjNM/ipiJAcimMMHinDrTiARSAjBppPNPIPpTG4607A2MZsVH5hzzSOeTTCeapJWJuycEHmpFOarg1NH2qWhpjyKTFKelNzUlCGkxQRRg9qVgExQKdzQAaVgDNKDSYNKBRYQoNKTSAUuBSGIDzWRrdtJdACMZIOf5VsYFJt5zik0ByI8P6g3IiGPUmqVzayWsxjkGDXdsJMfIM1mT6Qt1cedcA49jSsNHJiJjycAepNRlcHiunvNBt1QujdOozXOTqkMhTpjtRYZCysMcj8Kkt93mYB59PWoiwJ4pQSOhIPtTA6WYwi2VJAAxAwBSppEc0AJGKyYbKUxiVJgxHY9KlOpTmPYXII9DQBafSQnKLz60C0t1B3uA3esptRuVYgP/Oq099cSAl3BB7AUmhF9bLF2DuDgnAxW+baGKHdG+CBng5rmtOElwWDMcDHA49a24bVSnLtxRYEzClEsl25ckknqagnUK2Ac1p3cMpyfLYAdTjgVnGEueCM+9DQyAYHI60odj0H5CleKSLlxgdjTAwHQ0rAYbEk5pwkZV284qOlBwegNIZOApTcM8UxvYUjSERnFWLRBcQuCMSDpTQFUsduB0qEkgcVYcbHKngg1WbqaEIcF3JnIFG4kY9KRinAUnPfNIKsCRMBsupYegp6fZ3uh8hCHsTjBqAkg4pM0AW3mdJVCuQQcKc8gVejhuJGEk08mfXNY2TkEDJBzjPWt2eUrbIQcZFZsZZtlkEwLsDH78UX9ujZfc4BIGAcD8qzbe5lEoO78O9bQs7y4T54SVPc8GovYGjCvrV7RkHmOUcE8n/PrVEhicgZX1rorjT1lYL5zsyjADc4FY8ltcgkeXIAOg2VopCsMtml8mWJZdinkj1qNI2ZgAck9KU+XgAg7h1JFWLCGSe8EcZ59D0qrgRDfDJ5ecZ4I9abKYgwEfOOtaQhNhNIZwrKRjBORUE88IBa3QJjsKAK758lSg5Bz1rYtriO4kVCcgpzx0NZkavNbtLkZGcjFPsEaWTzI3AkTnB70gIpYvKunAGADwKkMahw6Egj0p8wS5uWBfbjvmo4S0c7Rq+9B70JjK9xcvK+yP5G7gd6ZGzRo8boGycgntT5owl0XT8DUbEls0ATgB48HvTAfLOwjinocY46dakukDSBoxkMM5oAl0sxGScyEAAcAmp9PkeYSTZAMbEAAdqyYULTBT/ECMepqaO4l04SwlPv+vahoDZe5SWEuQAynBA6VitdyAkcH3Ip8tw8BGwAq45BHBNVkGUGetJIBwPmOHeVMDsBj9KvWvViCNo4GSBVaGMPAxMigg4HOaZMjInzDAJwKGgNCKMGcI5UDBI561Hc2ud2AVqzcafDZadbXEdwv2o4zEOQQelRknzWZmJJxnNS0BBbRru2yHgdRTLpYI5SIR8tXI2R3Knt3qlMok3Z6immAJET5bYwGGRV6FxAmzBJJ4A6ZNUmLNHGu8gAYIFSISZI2JzsIP5U7ga09j9ki2XG9JmUHnuKn8OaXMmrQ3s8KfZ2BAZjnk9MVdvZYvENpNNGpijtUyrHglsdD7cVkvHcW+n21xlljuCAUzxmlYDQ1HSreyM1xcsA0kn7pRkgL9BWOl9bwWN1biPM0nAyCMj1rrtJUwafcSzW/m3QcArjJIxwOa5u/s55Lu5vLyEQttyi54HpTYmjMtrp7FSphjkD4JB7VoWt7H9tkuHjfyyOGAyM1Vso4JppnlhDrswATg59jXT6Kk1/I6WxiaOJQDHIcDntkUkiWzPtbhLxXeSVE7BDwTUccdtd77eeXEiKTHIzYAP0qpq2n3GjagzSKAGYlIyQwAPuPrWhOYYnhK2CTFwHyDgYpMoxriQwRJbm6jnB6kHO059apOzBsHtVzUJFvnaZIBBsIBjFVEUzszM2FUZJxnFNAOQgoCDlice1XbJWku9kwyVBI9iMGs6OZBJGHH7sHOfXFXzcrLd5jO0E4Ax1HegCWWa7u9M8oSnapx5YAB47Z71Sht5opEBXMY5I9Park823VtilY+Bk+p9apzXDLMwVs8mkgLd20YCyPGG2nIIHBHofSq0rqSDEcgjNEV/JAS6EEgcgjgj0qWQxXPmXFq6CRhhoTwc+opWAobneUhnJBPJHankOrK8eAVPBApEjeaYCBDIcAsB2J9aJPkwHOSDjHT9aqwF6aRJFB2YwCOvrVaAWyJmcSvg42ocYFQl3c4XhVIJqQyFvlIAHYAUASsxkl8y1VyV7E5Kii1BMbzlv3kbZ2sMA1TjuGt5mKyFARgkVfjmhZsEZB+8OmT6/WiwzVlVNTS3ldvLhYFVXGMPjH41lXWmvDO0FwixTbS0bHksKbCZLS/hBLsgkDBH6H2zVzW3e+mS7ZyeuUOMIM8AGqQFHTQlvc/wCkTSwh1wjpzgnrn26U/UrQhbdYisgckB1PDEGqTDBRgSy5ywJ5xUs1zG86GONvs6NkRM39RTbAEgCgmVCGOQueBx1qqw2sQeo610I1q0tLGYWdtLHcSfxHBH4Z7VhyNJdzh5pEQnAGQAAKVwESZ/OEhOcDA4q3DDElvlwPMYk5PO0euKju7aaNg5jUx4xvjOVqS+zHLD5MhMZjBAOOM0AV5pBMdxbJ7H1qA8U8nkjjPtUectimMcCDnIB471PGSh3DOSMZA6VCBgkehqzDGXGBwdp4NJoknJ82EtIjSKFwCpxg+p9adbwvLYnELSFmwCOuff2qshuNrpCC5IztA5xTUnKAKHaMg5BBwMj1qbDLklreW7C3e3KsThBjIJ9Aa07CexFjbpMWWWIkTQjIEgz3OcHpWTJNOvk3MdwxuEzuLnOCDjiuo0LUrF4HhuLSOOWc73DLuD9srnkZ9KdgMXWru0vDAlrbJA8YIcx45HUcD+dPtEs9Qsri4vGb7TDESCqZHHQmp9ZS5SeeG008QxwnzA5GCUIx36jOT7Vznzg8O6gjBAyAfrTETJvmwMEyHgADqa0NStIorG2YDbMsYSWIcHeCckj6YqjBcGFWAAIJBz3BByK6DURHBNPc3Awbm3Ei+oLDkfrUsY6ysJL/AE+91CG4MdvAV8yPaQSCOorV0/VNXNz9jj3JfuCBNcOclRyAB0zjmszRNS8mzu7EnFveQklm5KOBwPocYrZubcPoWnajcyO7vOD5iEgpGQfTpgikhm3ba3cW99DYancI8kxCiRISOT2z68iujZQy4YZGQfxByK8yuJr0Lb3n2Npo7WYOJwSSxByCQOma6mz8SQzW5upIvLkYZMROQx9R6VQjoW61G2azrXXYri9W1uIJLaSQboS/IkHsfX2rSOKaAQE5qQMMVHjmimBIXAHJqu5yTjpSknNIOtIBFPY1A0b+YTjFWguaULk800BSJZTg04NnrViaPIJxVN43UFh0FUImBzSlQRjFQRSYIzVkEEDFMCJk/dkVh3UJDkhgfpXQHmsa5QJ5hAwMnNXT3IZSUnFCMFBBAP1pMkGkIrYzFYqTwAKUIpI3OB7DqaioNMESFMHqCPUVMXVowoPzYqoDinAntRYVxGBDEHtTc0rZJyaSgBRSgn1pM0Zp2EiUSkd6lEvqarZweaUNU2KuXVm98U8TH1NUAxHSpFcnvSsO5fEingjrTXgSTnofUVVDH1qUTYGCaVh3EeEr71G0fGQKtLIGHJpSgPQgii4WKBphFXJITgkCq5Qgc1SZDRADTqCtJimAoPNBOaKKBijOaeDTAaUGgLj80pYDrTKsWkKXE4jc4GDnikwQsdldyRiRI/kPIOe1Tw74QBICCK2oYFWFFA4AwKhubJn+ZR0rBzNEjHcEsSB1pvfBrRCCJcSIQPU1XkgQklRjNUp3BogCqe9LtPakClHKkg89RVqQKIwVABx2qnImxCN1PLEdqF84AEREj6UNJk5KFfwwKm5VhwbI6EfWpByaBA4jLkYGMimKwxnBFK4E4GacFAqIN7EVYMEoAO04I60hiAUoIHWmPHIgBJyCajLEUgLOF9aiZfQVF5jetPWUd6YDDnPSlDDGD1qQmM9Dz6UgQE5oFYjLYNOEyjrSPGcHAqEqR1p3C5aEo9aCVNVxzxingUgB41YVVeMg4xV3FIQh681SYWM4oT2pjIR2rScRgcAZqAqD1qkyWimAc4qfy2AAJB+lPMQ6gUBQvamFgCGkKE1IDSnFS2BGEI61LHwaQU8CkyiymCOaXYDUKkiniQipGSBAKAMUquCcGpNgNSBEUyKaIge3NWQoApcUXGVfII7VE0OCcCr+KaUB7UXAoBKd5ZParIiGakEYHai4ikYjjpUZhO7NaRC+lRlB6U7hYzng4ziqcgw5FbLoMYxVSSAEnIq0yWjPAJ6UEVM8ew4qM1SZLQgoIxSE4p1UAw0lOINJigABNOBpRHxkkCnBF7HNACrnIqTkVGGAPFOLg0mMUsaercVFuHrQHHYipAkJpM03dmngcU2wF96MmkzzingZFIENGTT8ECmHjpTgxxUgFPXJpAMjNSRkA4oGG0+lIcg1Z+XFRMoJzSuMjDc81IGGKjYYGRTQ+DzRYLlpWFSAgnHrVMPU8cg4JpWC5zFyptNVBAwM5xXRr+8jV8cEZrF8RKBcRTAYzWjp0/naevPIGKYD3HWoialc5phFUhMarc81MpyKhIpyHFFhXJjUTjOaeWAppORQMqyAjNQnNW3ANQMoB4FUmSxqnFToc4wagA55qZAo6UNDRICSOaOlNMigUwy7qjlHcmyPWjKjnIquCPWgEHqafKK5ZDj1pwwarAgdKUSlaHELsskCk4B5IFQGcmm5ZzS5QuywXQdxQHX1quEOelTJEcZORRyoepKppwHtSIhzg1MqDAqWhojCk9BQ0eRyKtrECOBUogHcZqGyjnLuwilyTPIGP8I6CoF0ezjUSTMre7cmugvLKJ1JUYYdKyxF5L4uGAUetK4HMajb2Db3tyRg4IA4FZkEJlY4Gcda6DU7f7bOYbKMADkntk+v5VivYXFnLhygP160DNWU262BCEIQORnk1gHJYkHgmp5hLtG8ZHtUQPGMUCGgDOTVuO3V1G2RMnsaqY3MB61dt9Py2fPA+lICeD/QwTuBY9hU0eriHIYDJ7Z6fpVOVVgm4k34HPNWJrqykgOLdQ56Y5xSAjudVNwpQEBT2AqrHJChDFcsO+ao85qeIyKcrEZD6AU7jJricXAwqkH0NUniePG4Yz0q0izSzj915Z756VoR2sMrBZZEDdh1oA4kEFc03OaQMNmcimmQkYAAoGSnlcUQytDOHU9O3rUYJI5NIfWkIuXojfE0Y6jn61RJzSh2AKg8HtSU0AvBGaTtQBzQVOMgZppjGmniJim4EY+tRHPcEfUU4McYycUwLFpaPdlhHIocdAep+lWLmaQGOCRCjR8HPeqtpL5F3HNnGw5q1d3P9oX4kGFAxilYDR09o7JTcSFXJGAO1N/tS9ln3GQ+wHAApPtkSNsW3jOO/Snvd20K7owfO9h0rJoY61uZRKfnJO7v2qaa/v7eYlWVkHUMMgVRgbbCZWIDE5GTSXc7SWpIaiwiBzbySmSeRiSckKOBU1tdxxTBLe3YqfvFR8xFUhDN5fmEZX1zUsTgQjCnzh0YdRVoRv29na311vuVMMaDkSHGazL9bVL8iGQPB3CDPFQF7iSMiSGQk/xMTVvT4RbwvcSRBiCAgJ4zVASPc2UWVhtpEZgCDsIBqK0PlytI+wFupB4A+lT6le3RSGSVoz0/dgdKz44lmJJXJPQUmwIry3Ec5mjYNGe4qsCRISOM1tCBGtZIQMBQCKxW3GQKOTSuMdIWBw3TtT4FAJkPRRk0yVhK4L8EDGBV23iX+zpgR1zSApkkMSDgE5Aq7AyNZiPOD2OOlV7dPMBVhjAzn1oRWhmOEZl4yM0XAUxvBIshGSpyD2NQTCW7mBYAk9gOTVx5HdCFQkDkgdQKrpHKXWWFsEdCx5H4U7gQuzr+7lJBXoCOlNUgDBUkHsK0YrJru6JvGcyFdwAGAfxrOzgOoBABwARz+NUmBJ5ixoMAFvQ9qbI5mAXGAO2ai3A0+MkHINDAcUKFcgk44yatRyHBLelQE5IJ5wOKYS2eSce1JoCyweOYBOSRmnpGkoc5IkXsO9MiO4A+lOwFO7v61FgH6cA9+EYZGDUbEpPIoPAOM1cso0afMcm1wM9OKpvxPJjnLYB7E00wND7ZEdNEIjAkjORIDyc9iBTrq+Ikt4r1yLdYw0aIcjP9KqR2A89UklCs+MjP3frV/ULHylUTSJIsaY3BcED86YibTMvrMUjS3EcLkKzhiGAPAB9q2/Fel2kFiI4Vd7hvm3uxJAH9PasvSubAyTEkPyST1xTb3W769tvspIjLDAI5Yr6E0noBkBIZLfYkm0nk84yabpKyC9KLLNGwyQ8bcgj/APXUBDRSDPbp7VJZyukkjqcYPJxUgb8M13NIzXkhm42gScgj29KybuVix/fOGViiqOABnoKsWUxZzNncd45PQj6Vsw2GnT3f2Z7uBpJF8wbiCAfQUbjONkkKB4x0JySetRxLlgpJAJAOD1FdGfDAmnuIUmV5FPDxn5QPSqOm6RfzXDm3hyYchjJwM+n1q0rDaM+aIwuUI6UrOzoHLbMDAI7+tTXLmSQxyw+VcKSJOetU5BtOO1DFYntXjMge6ZmA6cZrYeztLy2jkgGJIiC46sR9KwgQFwK0YQ5hM0b+WQBg9yfSkhDNYRI72N41URyKDgHv7jtVOa3kt2jleBghPyk8BvpXUeHLW3uby3vZp4RJGSskUnBPGAR61o+KtNnnu7W4YobeNSCCpIA65PrV8oHD288kc7tHGRvGGAOcDuaZcGMzkhiUzkHHf0qe4VlvWNvFkBiRtHysPf0/+tUE5Z5DKI9iscgZz+tJgSCf5nXysFhgk96etrcFxthctyVwOCB1qKOQrIJggYJ1B6YrXMRvY7P7MTC5PlkgkhSemfrQBiSTb2BWNV4wR1JPrToiQC4QnHU46VtahpstqsdvcW6LcJkxvHjDqOpPNUziWUCOQrGVJPmDAB9B7UDEkE1xbJPj/VtkHPII9quXrwv9nZSGRxudQejCqkBQ2Eqhvmzkcc49amt/Ilt1SSDIU7Qc4zUgRPaFhLKqhRsyMDqKrxxiHJlUPC3BIOCPpWzfqsKGOFflKYyTgAH1NZRkEEaQmCFwOS2ck/jQAlxbva7hG26MEYJGQCe1RNFIgBmXBIyPcVZuLx7uGOAsFK9gOp9ajF8fs4haPMgJG7rgfSmIS2lMME0TRsY3GeOgNRoZbm7VS7AAZJBwcCkkk3LhSR9KWByhWfpGcxsfQnvTGVndTMduRz36/jSdMN3zmpbqYzXAzswg2AqoGQO5qIYJIJxwTQAueSfWp43O3KOVccgmqoOaeoJOKYFmK4kiZyhxvGCR1H0Parmmzi0llZwhWRMEsuSO+QPX2qgQFCg5B64Ip+8kc0mgHmMXFzcyJMSgJZSBgEn27Vchv7iJrWW0iX7RbtnGCTntke/NQwQlIXYgqpxyD1HPFMVwlwJo2JffjBGBj0NAG3c+JNW1FHtgkEpMZB8qMggdxyT0rHuShtI4fMR5F5OD0B7fXNVSZBMfLDF8k4Bq1dWcytEZpoSzEKFGQQD35Az7mkBMsMscBEMMbsAUmDkYDHoAT+NLPcSSWUbXBaU7RHu6bVHQcVWEstjPPb4VwcLIueSB2H+NaOm2qXVvMEVJkTojEhx3OPX6UMRCiD7IrggMVI+U8EDHX1rqLfUXtNOhsoVeNp0AIZQQCSASCegrmbu2kt7sR24whbYN+AScZP4e9JDdvBcRzG4J8sg888en0qb2Hc7Rbi5OpJpdtcbnkB8x2T5goIIBA/HmrEelv/wlUcasBCAJWjx0JHIH5ZrL06/it/FIuzI4WTaFcnKBTgc46D0/Cunt5BN4lvZB/wAsowpPpn/9VOLEXdT02PVLbyi5jmQ74ZQOUYdD9Kh0y9kuYpIbsCO9g+WVOmQOjD2NXt/NZuq20rvFf2YzeW4xsHSZO6n19qoaZqCjvVe0u4r61S4hOVbIIPVSOoPuKlU80wBxiow2DUrciomXnigLkgcetSKwPeqgyDT0YiiwFo4IqKRRtPFN8zHemtICMUXApyjDnHSiOXBAzSyfepoTuKsRaBzWdcukczAjPqBVwPjrVaeJJGL45PWnDcTMqYDduHQ9qYvSpnUgZIqEghsEV0IxYwjBop+Rjmk288UwG4pAMGpApOPSkKkUXAQDPUUwgg4NTDrTgqknI4NFxWK1SxQmQ5IIGM9OtKI06E5H0q8lyoRQOmMYx2pNjSI41QYUqD9RTrizBAkhTGeozUpERkDgY9sVZE0YixgmpKsZXkAKQT8x6CocEcVruiSR7wMEVSmh6sopphYqAnNODHvVlbZSAQT7042O9iRJj0GKLomzK4f0NSrMVNVirxswIIIPNOBJ60xlwSAjk0FUYVWB9DT1cjqaVgFktweVBNVyuDgjmrYlx0NIQr8kZNNBYp4oxUxjx0pjLgdKdybEdOpcGjBpsAFWLQOtwsi4yp5BPUVCoxU8J2EMBn2qHsUjpoSSgJ4qcnjFZ1tKCBk1fRs1zNGqILiETRlSMioEtIwwLKCQMAmtAgYpm0elK4zHmslDswBHOaS3heWRUAyAcmtZ0U9RTLeJYHJUkgnIB7U+YCYIR94DP0pTGrDBA/KlyTSk8VN2OwCNcYIB/CoJLFJGB54OcetTK3NSbxSuwsQm3jK48sflTNhBIJ47VYLg9KjPNO7CxC8PmKR+tUJLK4DnGCvYg1qgUYFNSsTYwmWSMkMpFR7nJwAxPoBW+9ukilSMg0kVpDCuFXFNSuHKZHlsBknJ9MVLDBLJnkDHrWv5KY5ANAjReQADQ5D5SobVm+4cfWopraSNckZXuQOlaXFG2jmFYxVjLnAU59xUos3OdxI/CtI4B4ApOKOYfKZklqVXIzVY5U4rcAU9Rmse7jZLhgFJBOQB2qkxWICc02hgRzjFNLHFaIhj91Iaj385p4YE80CEIbGQOKQE96lVgeO1KEU9KHoAwMRUiMSeaPL9BSqhBzSbuUTgjbzTSRUgUFcEU4IAOlSwIlJzipwxxjNQSAhuBQGPrUjLQNLwarrIehNPDHsaBkuKQntTQT3ozk0hjs08An2pgxmngikAu360hWpguRxS+WfSlcCq0ZxnFAgBGTmrgj9RRtFFwMq5snwSoz7VTFjcMThMfU4rocZppUYqlNk2MAaZcMwGFHvmkm094OrBvoK3dvPFRmHceQDVe0YrGAxXuOlNyO1aEmnygsSVI64FVjaSDkEGtFJEuLK5J6UKr/3TWjHbKoBYZanmFc5Ao50g5WZZR2YAK2T04rYttLtwgMrMzdxnFEMeZBntVxRtOKhzRXKwW0tEBAhXn15ppsLRxjygPcGnlCTkUpyAAKjmYym+jwj7juPxzUEmnyxrkYb6GtMPk08KpHNPmYGD9mmzkx4H1qRYpAOR+ta8gQGq747U+disUGhcnOKYY2UcirTtjNQMxbqapagQEsD1NODEetNc800GqsTcnEh4G6neZnvVfNKGHelYLk2SRUZPNODjHWk2F246UWGIGpwYinpZk8lgKebTHR8/jRdBYoarF51ixxkqQaraJNhWj9M1qz25e2kQdxn8q5/TpPKvMZxzipYI3HyDTSemKmmGTkVXPFUgY3caaZCO9DHmozzVJEkwkzTgxqsCQacHNHKFycnNRtTC59aTeKLAKabuNKWpuaoAJJooyM0maEhATTgM03vUiEDFOwDgpPQUojPpUyMMZqQEVDY7EQgBGTUiRqvQU7NLUXHYNoznHNOzxTcmlFIoeKeDxUYqRelSwJY3IYVaWUHqapggUu41LQ0WyyHrWfd6fbXMgeSVwQMYBqXcaTIPUUrDMi9Z4x5Nvb5UdWxzUd5plulkHnXMxGSfStwAE9KkMEcy7XGRQM84uJbZDhQ+PfmqKgNMCRlScYrutc0SxktiUUiYHIwetcdPZJakeY7A+lAF+OzsN4McJZuwB5p9+phtyUt3RuxNVrTVYrM7vJLfU1BqGrPfEBSyqDnGaAKJmdQRgZPJyM0zzGPf9KkEE0oLIhYAdaYY3248sgjrxmlcBhIrotCghSLzHulViMEdM/rXNgE0qbA4B4oA0NXTF0Sjlx6hhTLKRkYAAH3IyaBNahQCp+oqSEG4kC2yE/XigDhyeacOabilBxTGPJKjJpC5pGclAD2puaGgJtocDaOe9AjI6jFNjYKDmpxyozSsAiRgjgc0jhk5KnHrUkcZf5Q2D61ZCqsISRw2PSgCkZnYAKMADGahIJOcD8KvGSMfKIgR6mnJGjuFwAT6UXAzsVLECpDAcjpVtYRHcCOWMHPQgU2eIwybSMZ5HuKLgNjeQzhjFvHpU0jeY2fK8v2qKOYxsCOfarO0yDeEK57GpYFd3Cjae9NZSkYLggHoaSUZYZIAHWrlzIWs0GBtHT3oSApor85JKdgTUqKwIKdR0qEOSKs2r7X6Ak9Ce1AF7UrpxZRgN8zAbuO9Z5vZvJEBlIQ9QO9aIuLVoQtxGSe46VXiayhn3xwqwPTPagCowdiTkn6mp7JikpL8ccVcfUlQuIbdVLDBJ559abL9na18xv8Aj4P8PakwLFnJsVyQrFs59qtXsdlFCiSQqNwJBAxg/UVjJMUTAq3BqJdPKnwVHQkZ4qQMw24VyF5q1J/x4FR1JAH1q08kAbZgKh6MBzmmPbxQwEzSgqDngfpVJgQJp10qKoVYyDnJbrVxIvLbLOC/cKeKqnypjtRmA9ScVYxAhCRjp1OealsBJkjRQwXknBOeoNSGwhuI0a1by5MYxng1XmRpMYJIHYnpUEkl3DGBEcDOMUrgXVv5rKYPKvmhRtIxjB+tY8o+13LuPkLcgGrsNzJJG0Vw5ck5yQDRDG0spKQqYBkOehFaxYGRJG8TYcYPb3poJBFadwIXmMEQLxZ4J7H61nlSjEMCCDjmmBKQcEryQM0pAEKuCST1HpToIxLKoI470pcguoACg9BQAsMgSNye4wK0haFLeO4kRjbn+JTg1kn7pPbtWlIjJpQVZH5wQpPFJoCdbUBWkiG9WGAehP8AjVY24a4hAIUDnBP6UkN66WIBPKvwe9a8dhFdRrcFCpABGDnIqAI9EmjOvy291GrCZdozzg9eKnltVkt5oJrgRW4mJ5OSQOnP+NYzgR3AnyRIpyCPSq9wHlYzHkE5b/GrUgaJzesCbW0mZ08zAJ7DPpT7uZjcjJUOMAtHwSDWdAvWUEAdsnGasGMSTiQOCzsCQO1MVizPYn7WiRtliAcE9c1Jbac73TWpBhDJkkYIB9TUiyebJcXAOJIRhPUYqTTr2SyuJDIu+OZCCR1WpaAT7OE0yRYo1DxMN5U8n3FUn0544GmmfypGPyIcZOa17jVbf+z5I4IWIAABbqcGsS4upbw+fIQXByoAwBj0pWEjch1O38P6W1l5Upu3BJIPBJ7g/wBKzJPEWqSxLFHIIY8AMEHLe+TVC4uprhfMnctIRgk+g6VHHLskjfGdpzg9DVooSVXDbmcHPU5yaRU3xuxPKkCrV4++5WZoVTcQdqnIxSXktpJM5tY2CyAEgdARTaGU2J28dauPdeZZwwoCFUZOe5qmSOo5pSWBwDgYzik0IkhkEM4kIBwc4NdBDrd3HaNcySGWMnHkl+AD+vrXNAFjjIH1qzHMUjCOmY+d4Xqwppks3bu8jmmjikuI7dfLJDR/d5HeqZstMt7SaA3sc1wRlJBnan1NU7O0S5jnjhhckjdG2cAAdc1FewvaFIyoCuM8c5/HvQhjTLGkZWIAlgQxPII9q29NuLltNkntbcAQLiSYcZPHbviudPAJ9O1d1Z3ULeH7rT4bUwCO3LnL5yTz6d6GwINYmjPh20uLm3S5uXUqJJCcjJySAK5T7RlVDKDgYGavXd/GsMMORJnDkMOBx0Hp1/Ss6cHzshcIQCOaQydLySE4jCgHqMdRVi3kV5kOGjUccHgH1rPk4mHoAKvwKIoRKM7ZOMnsR3FKQE91ds8Txk4njIJDcDH9azpbhpSu4gtznAxUpAlumEx3nsw5P50LaMbtEQDOc8nrSQEDFQM9HHeiSQGNcbWLDJPU5qSa2Y3DDYQMZyehHqKrMQCQDkeuKaAUDCkkggcmmCR2jMOf3ZbcR70/y0KscsDngAZwPU1GRJHkEEZGcEdaYAaSr0OmTXFvBNCRJ5pIIHBU+9QXNtNa3T200ZEkZIYDkA/WgCAdachG4ZOATgn0rR0Kwi1LUxbTZAZDgjj5uwz2ptxZHSdTktrhfmjxk84ORnimBM1qsFkl6P38MgK/MpBBzgdaqAMU8zb+7U4Jz3rX06U36rpUs0UOnhxJNIxwcegz3JpL/TobMb7adXhklIjTPIHHU9CeakLmb5aLBIPMkSQkYAPBBznNQM+YwKnulmSUqRz2wQQagkKuflTYMAEZzzTAuQLLbwRzRKZJiCdhXqR3960orq8msYze2qrEGEkU7DaQRyeO/SqdrLHIjyXPyCNFjyvU89afqmoPeW8Frb3G63jUAKRgk57/AEoJGXlzbzzedETJM8m4nbgAen0xUbzyJmSI+W4YEOnABHcGqUJEkgUkhj0x3NWlkxAYiMgnJOe9AGj5dxqNjNfzXYaWFQ4UAEkZxyRjH/1qqbIjbRtK+DIpI4yuSanstiRPbzcR3aYDdwQcj69qdFpdxcRSQKyHyzkEHjPv6CpbGiGzZ1mELvsQkZ55Az1x3r0HwyXefUGkdZHO0FlOQam0bTbR0a9MCF5Qu5CoOxwMHH40aFGsWo6sEUKvm8AdByaaQG5QDg8UzdRmqAzZgdJvnvY1/wBDuCBcAD/VseA4+vf6CtdeRkcg9COhFRHa8bRyIHRhgqehH+TWNBq1vod02l305WEYNvIwyQp6An2P6UgN40EUxJEkQSRukkbchkOQR9aduFMBrjAqAtg9asMQRiq8iimgDfx1ppbJzmmEEHFByKqwAxyKdFknB6U3rT0GDmhiEmQgZA471WZ8cGtEkNHWfcABiQOKcXqDITsyaY0SOMjimkk0ofHBrZGdiM24HXNQdCQAcD1rQ3Iwx1qpIpD0ITQ0Y708BSQKZj0o3FG56VQhDHgnFKBxTwwIzSFcmkMgYEHApFJB61MQcnNN8sE5xTES7ztwOtCTkcMcihAAxB9KaUwTxSGW47gEY7U1pVDkDpjrVLLA8GjLYJPWlYC5E/73Gcg81ODhh9azUkIOan84joTik0NM0ygkRkboRisSaMwyFD1FaEcznHNRTwvOSAnzAgg5oTGyjmlDmphZSk/MQo9c5px0+fAZQGB9Kq6IsyJWqUGmG2uEGWjIHvTcMv3gR9aYE4GaQrnimqx6VKrKTzxSGQmM+lIVI7VcCg9DTDCewouFiBQpwCPxp4BRvUdjR5R3AkcZ5q28IAAQgg8ik2NIdbyYPBrStpeBk81kxoynpV+EELzWMty0aAfPU08c1niQ79o61YjkfoRx61DKJnXJzTQMGhpOMU0PSAkzimliTTQx7nilBBNIY4GnZpopcikApNFNJAoDAUAONANJnPNJQIkBxShsmo8mm7sUxkpcg0hcdqjLZFNLYoAfvxS+ZUDEjmozJjvTAsF6YZKi8wHvTwARmgCVHyRUjAHBIB471AAAacXx3pADwof4R+VU5raMuSQcn04qdpCe9N3Z6mquTYonT8nIYgfSmmxZTxkj1q+DT1qudhyox3jdCQB0p9rE0kvzkhR1yK0WhQksRzSYA6U+di5UI8I2/KOaqEujAMCDV4P60kihwBUpsdkQCQ4qZCCMmmiEDjFO2ADIFNsEhwCGkeJcZA5pucGpBzU3CxAUxT1HHNSFR6UhA7CncLCEUCkxQTg0rjF70oODTM80oNIC3G+AMU4SgtyaqBiOlLknrQBdeUbeDzUPnH1qDNGaALAlzSmUAZNVs4ozkUASmZc1E8uTwaaaYRTAUyP6mkDHkHvSYpMUXAeADTgqmmrxUmQTwKLgOVMc08ZFMDHvTw2KQBlscUq5PWmhhmlLHoOKAF2gdKCWA4puSO9NkcgcmgRGzAGmE5NNLZNKuM0DK10SM1TMhFbhZTHsKg/WlSGBTuESgjvitIzsS0YG/PY/lTg1a13HFIhJUAjuBWQFOcD1rVSTIasPJopwtyecmmPG6HHakBLCwDgEAj3q8pjHQAVmx/eFWgxHepZSLBfnjgUnmepqEvx1phc+tQO5bUg5HqMVyV3H9n1Qjp8wNdGJCOlYOtDFwJO5HNCYjeH7yJW9RUTLTdMk82wTnJHFSuOapMGiuynmmBfarGMnkU4KB2q7ktFUx/WmlatsFxjFMEYxTuKxVPAptTvEcGoSCDg07iaDNGaaRSimIbmlzS4oxTATNAPNGKMUDJVcipVl9RVce/Sr1raGbBYfJ3NRJDTEWUE4p+4HpVmWzhRcIMEd6p7SpxnpUblJWJM5NOFNU0oPNJgiReTTwDmhFNSBallWEwaADTwKXFSFhuOKUA+lOApQtIYoUYp4GKaDinZ4oYDZASp2AE+prkNb0uWaQAptGcls5rsQ1EmGhYEA5GORQB5vcWNhbwEmVml+tZOK7KbRtO+0ubhxnOSAcCue1SK1huAloSV5yT+FAybRS8032cuAhHOetad/bQ20XmgkyE4AJyDXNwTm3k3qxB9q0p9Vjmt1BZjIDnmkIrND5ilyMe2MVWHkqMSRgn1Jp8mpTyDGQB7CqruXbJoGPMIckoMKegrS0VJLa8VzwBTre3hkhDRq7ehxxTlmNuSpG7HvigDzvB7AmkzUsgCkgHPvTWKFQF5bvTGNoA5pQKXFMABwamBIAyRUFGKHqBZWQg5BqQMGBAx+FVwQE2gU6MkAkd6TQDmIGcdalt2ZWWQtxnkY6VADk0JuwQQSCaVhGpezCOMPGAScde1VZL5J41EpHnL0IGOKrzXMpj8nd8uQcVXByf8A61FgJo3CuWJqybgzYBY4HQCqijI6EfUVZRCsWSKTQx4iFwG28kDOKQnFioz3xVrTkIhkYjAOaospYmMdAaQxYgGIVQWPtV8wvb2+8wqjDqetLbCKC3IgAMh6kjkmniNm0+TBJZznk0xFORD5LMWBz6dqbZzKilWTdnocZxUiwTCLDHg9RT7RAko/ds47gUhi3kaQsB5qnIz6VWabzCOc4GKdFZC91AwxqY1zySPuj1NNurdbW78mFmmUfxYxTsIktw85KIMmi5Q27YPoDUI82CYY3Rsalu3aeFVlA8wHO4cfpUsYmTLagjkZq61yZNNjEjDeOxPJqnbsqwqrcgHOKW5dZCRGML2FCECkyn92Qq+9SF2jGQc1QMjxAxhSv1FTWUgMpSVuHGBx3osBchvjbuHZdy9xjtU7TW8gLROsgPUEYKj+tZ1yNqOnoKbYsC2D6Gp5QJ51ATzI5FGDnOetSx3Hl2zRJtIPOR1H1qskEoUrIVCE565xSQoYvOyOB0rRAA5jkQdTzUYIuICrsBPnAJ6GrMMAMJlMgBYEAEdTVVlj3EKV49BTTAIpDauVc4kxg0hBa4IA6jNWprN2hjvfKLYHzY5JHrVcEg+cQQp4BxgZ9qAGMrgkHoOlaoWJ7BvtDIJAn7sA5Oaowq12pCYBzjk1f0u2RdVjknEbQwjEnOBk/wD6qTQFRVzGF9AKtWcioSXBIAzkVa1T+zo5d9qzMXPIAOAfaooEttsgyfNK5AJqWIXzo5WBkASNjgDvUF7aSRzhwB9nJwOcgfWoZ5huEY6Ic1NFMcSCObdbsnR+cfSkhlC7QRSFQwb3HSpbPEc0YJwGB57g1A7AxGEYJDZzj+VSRI25Q8igA4GT3qmwJmWS3lkKOrLIMNg9BT7OQPcBJDkEEitAWsa2kj3CoQsfy7T3rG3F0UZA252kDkH60gLRiYW00oXEO7AJPAPpVW3YbABglTnPY1csbr7DKYpWMiyH5lboTU11paxSyXELhYXGUUDOT6D0poRlXM5nmIIAI6Y71AM7iD1HUVtJZPb2CFYi08gIICkkj3FUXilscfaIGSSRckHqRVIBiqZEt1Y4G7bk9APerM+Q+yEA4Od44FVXlRhhIyg9Cc81NC8RgKyOUZAMEUMZC6O8xJX5sZNRu5kdQcelXA8FzMVVXQFcYByePeqc0bW4WTOGJPHpQBJJCwlEYYMcA5HAA96esDQOvnAFWGRg9RVPzpmDEHlhgn1FWPOM0FumQCikZPHehCaLFvqMmmzH7IUKyjLBlyFP5+9XDqAu3hhmiXyRywXk/XOOBWRKAJCAcgDIOMZp6uHBcSFcjGO9UhWLN7p6tDJe27xiHzAgXcTjrzk8VY0zUL2GQyOIzDINkqYGSoFV5rJ00tLiCWSVM5dOwPqauXughdPivoZt7ygEQA5PIz1zSaGUbmMs0v2dQ1up4Y9QKph2Ee3JK571NE6PCYy2CSRk9BUSgFCCxBHQDvRYBUzI2AKmV2dBCXyo6AHnFPgKGMKwWMjOCe/1prx/MpiaMEDGR3pNDEjkmgLrEcFjnPcVMTJcRRksZJhwSB8wNQhSXAMi7j2FTLcPCTGCxAA5HX8aVwJ1feFVmIwMAjg1Vmt3gOUUlf4iRnmrMDRFS0yHaRgE8YqSWcRwoIZCwB5JHUelTcZnv+8hJVgMDJHc0SMZrUzTEuw4BB4U+hFWLkia5iMMWCxyQvTNWxp8rW00CIZjJIGk2chB6GqTJZb8JNe2uphxaStaAfvCFztz39/pXSDRtOstUFzd34hQuzpEvU5x1PPFaGgvYxgWsU5FxGu1kkIDEAcZ9eKw9QujZeI7ia4mQ2hmAdCgY9OOD0FVYY3+xo0vr23sMSW0hVo5vMBYMOSQe4yefrVbX7SXVtZhtlkjaS3twt1KBhQQT159P5Vk6vqI1A+WmRGkhaMjggHoMfh+tXdA1B7SCeyiljiuJ5A6SSDIx0IJpXEdFpehaTHayeTEl0ww25zlsY6j0/8ArUurmyk08280bPG4BMiYIj5HJ9DzWReNqXh65mnaaKc3MbRpIpAAAwcgD0z+tZq3gm0mHT7KOd3cnzWAJJBHQ/lxTiBn6jbSadqRjLo7jDgg8MDz0qirgud3AJJOOgrQ1CZ57qK4ucYA8vZjDAAdxVKPyD5xkyPkPlgHqe3NDQIVSmwrnsTW7e+HkbTrfVrKU3FqwzMUABjI6/rXPwkZIIzkEVs6JNdWzJaQ3DQ29ydkpKghs56D36D61NgZSuDIt3EwtSuyIEADBZT0bHvTITElvAxIJYsSM8jB4BrpNfgmuCmpwW8kNsgSGIyIRkDI3DPboPWqlsCbG8S4hjl8lAVBXGCc/wA6AKlpYTajGZYbcTSDJ2RscjHqOwP9Kks5ja3ZDNmOUkPGeCp6HI/Ojw/eTaVepNGCEchXQ/xDnH5V1Mmk2utwm4EMkVxMd+/rg8Ajr7UkgH+Fr4AvGZHdgdqK3Rlzwc+orX0gYv8AVD6zY/nWTpem/wBlzzFVxlQFJOSDjkj07VZ0yf7PbHuZDknuSCa0UXa4G6eDTgaoQ3Ylcpk5xkVKZgpxnFNxsJMuA5rlfE+l3DXsd/bwtMrxhJABkoR049OT+VdEsmelSq1KwXOX8KT3Ed21r5cohcEhSOARnn2rrCajjSOIFY41QE5OBjJp+c0rDEJNMJzTiKbigBpprc04im5AOD1q0AmcGpRggGoCwpwkIHFAE1QvyeetTAgioZcA5oW4inKuCTioNwzyM1eYBxiq0lvjkCtkyGiDJB46UpORz1pGBXqKjZznjgVSJY4kjpSF9w5HNJv9aPlaqJG5x0NOEtIR700ikFyUMCM5waM4NRcinAEJkjigZMHH404AN+NVsmnhiDQBI8YCkgE4qMoQoJ6VMCe7D6VKbebyQ7xkAngHrU3KsUMYqSPJOMU54uMjpT4isfLDJHamIli3gg7enc1JJJIMkAGqxmlduFAFI/mg4JqbBclSYtIN2BWlBIAgJrKht5pmIOAPXNXY7RY8ZZiR71LGgvLxEcBTnIyaz57kSgAnpWnti/iUH61IscOMiNfyoTsDRhgkc4IFO8xR1NbRt45GBdQQDwO1SLawEkmNcH2qudBymGJQOhq3As82NiEg9zxWstpAjApGoP0qUnHTA/Cs3O5VjKe3kjAaRMA8ZzQFIGB0q9cAPHiqYJUkEUJ3BoliAJGRVhRUUW3Iq0oGBUPcpEYUZzjmpVIApCM9KRgQKgoRmBNANNxSgUALkg0BjQQabTAlDmk3EmmE0m8CkBMzgCmFiajLZNAIoAlWTA604Pk9arFuaA/PWgC0WFMLcZqAyYPWmtJ8p5oAn3j1oL5HWqfmEHrQJvemIssxxUZPNRGbjk0vmCgBS4B4NKsxHAqFiCeKQHBoGXQ7GlLE9ahRweDTyQelAATSZpaQjikAm7mnq3NRHrSjNMCcnIqJjk8UZOKbg5oAMnNPHSmd6cKAJAeOtBZcYzUZNMY4NFxMexAOaehBANQE5FOR8HHagCzTGIHWkD012BBwaLgOJB6UzqaiEmDipFYkigB2KAKeBxRjFADcGlozijNACUmadikxSAWjNJRTAQnio9xpx6VEx5pASbvekLVHuFIX96oB4fBqRDk1XyKljOCKQFsjoaZilByOtIKLgSBQefSgkA0hOBUJfJoAmLCoZGzRvwKZmgBmKUHBp2BSEUAKCTT/ADCBgVHnFBbigAILgg1UaEhjgVZDHtSGrTJsQoCOD1pxUnrzT8D0pKdxWI8Y7Cmk0800ihsYwkmmg80/FIRSAQHJrP1qPNorDqDWiBzUd7F5lo3GcUAUtCkJEiE5OOBWuwrm9Jl2XgGcZNdITQ2IjIwc00EZqQjNNCgU0waEbBFIKUgUhBFVcGhCeKY6A9qeelR5Ipoloh8ls8ClEDelTg08Gm2KxVaFwMkUzaR1q9nPWmMqt2o5h2KZFFSMoHvTdvHSruTYREZ3wBkGt2ECKFRjB7is2zkEbgEVdMm85rOZSLIw/B71WntShLAg+1TRNgjNJcSEjjis7lFLafSpEQZyaACakVT6U7hYnQcCnMMCmqcCkLHpUMscBThiowTTgc0gH0hIpQMmlKZpARkmnA8UuKUKKYCg0FQ64boetKFFSADFAHPal4Whug8kMzRse3NcfNo99aS/OuE/vYzmvUeKq3tuLmExE4B9s0AebHTy8Jk3MxHWo4NJnuI2lGFVc9Qea7qHQ4EXEhZh9cVYOlW/kPCCwVhjrQB5cylWINIQx4AyfSur1HQ7TTh5wcuewJyKxTbxyMSrlT6AGgDX0czQ2AEihRjOR1/Ks67vbFpWBBdj6cVdgu2gjCOC49hiqV/BDNh4olj5yRnOTSA4MjHJ/OnyeWUBRCp6U6aMFyQO9PkmLqFIGe2BQMqgU6nFSOowafB5Ic+cMrj9aaYEIpRyaJNvmHZ93tVq3itZEJkkkEnpjNMRVPFKCQOKdIEXOQQB61ox2wvbMSRhY0Qe5zQBnR/fBPSrljDFdSPGbnySPuj1NVNhcrGvUnFdg2g2OjwW94WPmYHXnJPtQM5efTrwSFVt5GA/iCnmtXTDbvp82bfYyj7wHJNaUfiKE28sWXeViQMjgD2rJFvcRWEklukhUH5gnQGkxGUzyzOA44rSi3XVr+7jJ2Ag4GOarWen3d+C8afuwcFjwM+1TJbzpLLHabwExvyevrSaGRxyPGSAcLUW1w+8DIPSrFsIiTG6u7noBzk1aEeowSi3FpIqk4wy8ipsBQtxMsgjRAzyHgHuasgyRybHEiSE8gdBmtbUtCewFu0rMfMAJdBgLmuwudFh1DSLeKJB5hXAlxjA7kmgDz3UDHEAkch3gDgVPYW0kakrMq3Eg4UjJArT1Lw3NokZkLw3MbAAkjJH0HernhvS4rWOS/nUPMQdkYHI9OKdgMTRjaW+pyG/JCHI3ds+4o1OR7+Qw2sKJbRnfGQMFvx9abI5OoyMUAG/JQ8kVNqOpwSSRl7YIBxlCM0XEZt5uWMvtyxIAyc4qGytJLxyzuqRjqT/AJ9qsXygWwZXLBhkZqhBIYSSO4xSeo2zTuLWxihAhkJmJwCDVKSGWIkMtMBdgJkBwpzkCraTma1fzCBgZGBjmlYCj8rH5u1R24Mt8mBwDnNTzLHt3xjGeo96s6PYC6SZ1kbzFHC//XqkBWvJElmCKMNjBOetJDbCM5kIB9CakuYwiKCibzn5l4J5qJLf5S1wJBGOhJ6mm0BYlZWCmMY65yc1Gr+YsiAc46+tNcQmFTCGXAwecZp8NvJDbiVxhXOAe2KVgIYrmQwxoT8qHOKa9uI5d4P3+cAcCphGPODYxGep7CpQI3jKk5YdD2xSuBpRaiV2W0aYVU4D8E1Sv5nurOOIBY41OSM9aQxyFluppFA6DnqKtyW7wMkiGOX1XPrSApWybtq2kTsq/wCsfHQe9W5gltbIc73lbABHJHrU76qLHSL2B0jS4uH4UDjBFP8AD2jXFwEvpQhhxhSTyD6irSEUGVIoiWj+YdAeCKgsmVtRXccZU4+tSPambULk+a/DYAfqR71TuE+z3RQMCV6MP6UrAEo+ck/ezg0iOEBhmB25yCOSK1LXTs2zNNyG5ORzn1NUZ7d7cAsUIPpzSQFRVTzuQQvrTyWMe1eB6077NNtWQgFWGeD0pQCI2wM5GCfQ02Msh7eUYkbgDpjrVYKrK7AYUHAppGIgO5ODUhKKDGhyB7UrAK8wcAEDOMEnmpo7qeNAiOdq8gHqPYVWSF5phHGoLHoCcVO8JhlCyjgYJFFgNOe+lt5g0E00LSpkl8Nk+1ZFzJcXEjPcb2mb16/lVq4k87UoSI8oAMc/nVq5jjvi0okaBlOC2RgGhMXKR3Hh4/2PHeWVw0wIBlicAMD3xWS5RSVQMSQAe/Oea6C1sWEYzeiTzOUwckjvVA2kYjDo7bicYB5NHMUkZTpJCyscgkZBFW9PcusyuBJkggSDjrUkkohnRmj3ogxg8nNPu40uy1xFIxfHKAHJx6+lNMLEN2kcEw2wkI44QHOCe4qk42gFkBKnO01LbRmSbCkRuBkAnNSTI6o8kgBAONpGc/WmmDIXuhJ5gFuqFsbc9QPY1esrGaYJNaxAPGOSx4Y+lZ08iTKGVjkfwE5Ip1rNcIRFDKyIGDk54GOc1SJNy9e7jle2doAzx5McZ4APb61BA9jdWxt1gaCZFz5gcFifQGoNRntAMw3HmzMcvxwfcHtVFkBwwOdwyR6U7isPNjKnJEYHfD5P8qVbSYsACgHoTyfp61EEcoSoyB1PpVm9k82SJ0BJVcEjtUjI7gRhgFAUjqCeppYc8RqACOBk4z7VXJy5Pc9aVXKzBiMgDj60NDLIdbe7VgPnBwRnpTXYQ3Y3nOTlvx6VCo3sWPU9TVryoZQAxG4kAsTyaloC1Ihc4dcKfuYPGKgJ8q3MDDEgOQPUU8g7DJnOwAA49O/61XSFycBSXPQkYJFJody3IbX7Ms0e/fzvAPAPqKXT5L+CdriKVoVbDENyCvPbvj+tRJCwDQvgN3BNAjkRmkjLMmNpEg5HrxQgZtaBqAhiu0vHBs3yRMBghyeMfX+ldLdeG31LS7eG4miiuAS7TEZZhg4J/A15yHxA8KsSGIOD0GK3IfFmoLo50vyxIShTzCSTg9RiquKxBpGjTarfyx+bsgiOGlC5BHPT64rqLjT9Ct7M2TCFJHUYMpJbjuCc8/SuStr+aG0e0jLrvYHAOGB7g+1XhbNrExkaV0uIo8mSU/KQOnv0NFxWIJ7C6kuREhMluCQJDwBke9bMOjSaXDDei8n+SMyNtjBCkevqKqTSXJhFnczFxFiWMkAowAyOT1q5YeIBqlld2Ny7QmaPahbgZOcgHt2wKaYWOa1OUXEsk8o3yTvvSQjGQABgDtVEt5caEgFScZBrZ1+1u4ZreK5JKxxAKwjIXPoD39zWdGQIhwCe2aVwRPYJZTwPHcFQyZ8tjxkc8n34FMtZhY3yNcxyhMEgAjIz0Ip8NzHjbLChIAAcDn8R0NX7aITEXZ8swqNhB5OB7f0pXGdj4eli8QaR/YN+xcwr5mCSDjPyjjrjIP4CuOudOvY9Q1C0twzQwgtvkBAKL6Ej3rQ0LVf7F8RtdX4YrcRmKOZRkL0wcd+mK9CgxrWjbb2MrIS0Um3jODg4+ox+dK4jySGZzCw44x94YJ+ntXp+hWqvpFpcRxeX5kIymOp7n8etWL3Q7O5ggtIrZEUTKxYDkAdRn34rWSGOCJYol2ogwBnoKaYWMy609plLAYIBJ965uyBe2UAEkdcDpzXcjnI9j/KuT0G3ea0klRwpWQjkdQa1hITRDDIIrgE9hUszhnLA8GtOWwEowQAR0YDoaxp4ZLeTZINp7e4rRO5GxPHdhRgg8VOl5uYAVmZp6uFqnBCuzdSVXxg1KKx7aYBsk1qRzKwGTWMoNMpMlK5FNIxT1dT0NKcHpUWLIsVXuSQgIHOetWjUMyeYhFUgKKvk9amRwBg1E0JEgbkYGDx196GIEZYHpV2uSWxIB0qtNMckk1B9oIGc1E0hfOapQ7ibJhN3zT1nBOKpHNAYjpV2RNy86B+lVZYSDkCnJcEDBOalEoYZ4P1piuUGBBwaSrrxo4zjn1qu0JB4ppiIwx6U6hBhxngU51LPxjGKNgG8E1atwWAUHBpwt4R0JJ96fHGyPkHAHQVLYypPGI5CBwPSo13McAZJrQmtxMwIOCOvGc1NYWSibLjPpS5la7HyjdO08BhNIpLdwTkVuGJZUweppyxoBnFOXA6dKxc23dmljIvdKlRC8TA8gbQPU+tUprC4gkIdAQO4ORXU5BXFAUHgjNCnYVjlowiN81SnyjzU2r2EkMvnxp+6bGcHJB96yvMYdTWidyGrF1lOMxyAH0IzTTcTIoyAR61T81s8EilMjkYJJFVYLlkTF2xwM1PCzk7cZx3rOBIOa0rCYNJggj3qWgTuWgCByKb5gB5IH1NXNit2qtPZQySFiOe/oahmiAXMYGN4J9qU3CHvUJ0+FVLJuQ+x4qBLeYzBCMr3OaVxl5SH4FMltztyDn1GKnhg2ZIHWrIgBHI4+tTcGZCkgjFWkYlcZq6bOLrsANQy2sgwY1yPTNNu4BEcCnHkVAgdTgjBp+445qBjwoNIQB0pwFPCeooAYrKVwetBhOCacUCnIpolO4gnigCswIJFNNX1iRzk9aqXAUEhaAIiwpN9MNJigAL80F6Y2c0c46UxClz60wuaQ5ppzmgBc5owTQOKduAoAbgilzSFs0nWgB2c09KagGeasrGNucUARA4NTKcimFMHpT1AxQA4UpGRQOKcMUhjNh9KUJjtTxUigZ5FFwK5B9KSp3UZ4phT2oAiwaTJBqQjFMYd6YCk5qOQGpFGRntSOuFzQIrFiKN5ocjNMyKYE3mY70hkBB5qEmmk4NICQHJqeM4qsDTlfBoAu78Um/Peq5fjrQHpAWNxIoBwOahEnFIZPU0wLAf3pd9VTKKBJQBZzmkNRiQdCacGzSACeKibk1KTxUTA9aYDQlIUIPApw3A05W9aAIgDUgOKcQOtNoAkVzkZPFShwR1qrnBp4agCZ34xUJb3pGJpmaAH5qRelQipVOKAHEcU0mnE8VGaAEJ5pCeaCaaTTAUHnFPzxUQ604HigBetIeDTgaY3WhMLDTTTS0lO4gp/l5HPFIAMg1JuFFxDVQA5qcRhoypHUGos+lOVjnGaBnIMht9QOBgB8AV1C/NGreozXPaxG0OoEnvyK2tOlM1mhPYYpsRPSHFKRzSYpABxSZFIaMU0wGsKgc4NTnpUbAE5xWkSWANPBqLIBxSgn0NDQiXNIQT0oUFqkVcduanYogKkdjUZPOB1rUjjzgHoanFpF6Uc4rGfbW5Y7iKteVipmUR8AYFM5JqXJsdhyAYpWh8xSBSqMAVMhwKi5ViqsBTtTwMVZK5OaaUHai4WISMCmEGpivFRMCOKBke7B5pyyDNRuDgmogxB60CL4YVIGBqiJeOtO809jQBeIFNIqusx7mpPN96BkmTTwTUAf3pwb3oES800jJpC3vRuoAa2AKickggVI/INR8g8UmBx+t3DmV4fJJz3PNYUN29qxCkBh6jNeh3Vhb3akSqc+oOCK4jV9PjhuykLcDrmgDOuLy4mYFpBx0wtV2kkdslsk96lEghJUgH6imO6u2cgfQUxmAMHr0qaYoqBkXIxg1XXpmh2OMUhkbnLE0w1IqhuTnFKY8g4GB60IAhtXnBMYzjrVmwAidmkOMcAY61UV3RQisV3HBI9K6e7hhttIiCdGHJPUmmBkTwRPcIHcESdu4+tbSWv2OKO0KgQyHJJ6iucRHL+eAWVSCcDtW1c65b3cMYjR1mXoSOAaYieHQ4IfEFsryj7OTkb+ACPetLxddpcCK0t5EYrycMDgYrmbu9mvdoubkcYwAOBVOSOIuSjsR65oGWtPsme9ABDKvUjvmum06/itRd2KhpZZj8iAcH61ycbtACYXZSRgnOeKLe5uLSbzoZSJMEZPNSBvS6bqGj25kLg28hJZYzgqT2FRiyutOsJbmSPasgwQz5OD+FVH1e7vYlhuJDI+4EMTj9Olb/iFnNjApPBTkflQSTeBdPt5oXvZkDsrYUnt7109/bw31wNxI2DCsvBOeufyrJ8IvHDoe14yMsSCBwR61tb4Dzkj6ihoaZNJbRTWZt5CkikAEnrUdtvtYRCFBVeBg9BVeSeJAWLZA9OcU1dRiH94/hRYVx80Yub9ZJASqcjAyM0XVlHdNvhQxzdmAwPxp8EyuDIJFKk5znpWTr2tSw2xS3BUHgv3I9vSrGmc3rssbXqQJbETjIdweCR6VzV4skEu2RCrE9D6etbVpc+dM811MsbJyC3JPvWfdSC8uDKXAKngnvSaEUVkZyAxJB7ZqQoAMrVqGCa/mJQDYDjcB/Sq7I8czROpBBxzUNFCAFUPz9eozwac0g8v5T14x3pBGWcAYx3zSbAZjg8A/nSsAkL+XPGxGVByRntXQae7vqoW3VQJBgDOAwrnzG0s4jQZJOK6HTdkOrWBDrhWwTnjpVICXUdAaW/2WkeXTBkIPAJ9KydTjlhv/s8hAjGCcDrXVW6fbtR1IwTNHLC2VIOATz1rB1QG7IkkLpdE7doHBHr7CkIx2iEsgjjIAPcmtS83/ZY7ZeVCgn8Kp6jpzaZFEGud8jZyi9B/WorC42tMJHJAUYz+OaTAtQKJkS0RSI2IBJ7iq1xCLe5kiUkjORmtnR4luLG4dWCGDJ56H61kXMge4eTYFck5A6CpY0y5dW+dGhjWZS3GU7k1JYC3mtSLhD50QwAetRaVbTXl1mNVwmcyMeBSSxzRSSTEgnfgEHrSAqXxfUr0iGMhgOh61saTfapaaNJ5NjI8Yz+8yAAPp3pyrHb2AkSJPtDEjcRk8+9XptNmuNHjs4LuNFUFjg8knqK0TEzmbeFp3ijeZT5z7ivcetS3uneXeyXHlsbZTjKnjNR+bDbxoUyLhDy+MYOegqQrqN4sKR5eOQkIpPIPFO4FSS9kuJNpJEY7A9aVDLPK4AZyBwOuBTZLCe3u/s7gCTIAwcgk1o2VtLb3Y3EKSDknuPSoYypHPGbSQZw0fAGM8VGZNlkIyOZH3H6dqlv4VhmSWMfu5CSPrVKSV5pUDn5c800A9UJYbuB61ZtY0F+wZQ6qCcGoZpBHBGD1LHH0pUmIldx0KEUgJoZFd2KxmME5355H0p0wMgJBJJ9T1qgk5I2jjNXITIZIyqkqhy5AyBTAsWKB5LgkcRpgD3p8aQzW9widNuW+oqOKQKspjIwCScdaqR3TAl03ISQc/SlYdyxIPJQJGcOgyO+M1FaSkhlY56kH3pTLGzvM5+YjH0p5h2qHVyxIyc9hSsBGFUgm6cMhBzjse1VoriSEMIW8sNjIPJ4qeM+dHIG5HQH3qsbWTJyMYIw+OD7U0AxiYyHHUVNc275V/UA/nTpAkpBJ8tsc56E+tWLm38vTFnW4ZmXjGcgj0x2ppkmYYHALY4qW3t/O8wLNsKg5GOtPkjmkgEzyKoxgRg849aWNS1m4GOD1Jwfzq0wK5U7MMQSg7DHFPVVIVA21j3J4PtTSQhIIyRSqQxy4yKLgXDZtCyK0itkAuEOdo96mfT5j5aQhkkPDH+Fhjrmq8SxscQxAkjJAOMj0pwvpsYG7aOi7zgUAVhBLuIK4IbaR71ObCeHLMo+Xtnr9Ki82YbfJQjacnjPFWTdyXUxLyHB5wOADUgNjiy7R4VEGMD1zU6xgDYFRSpyMjqPWqcjSS9CSxOOT1rTtdOb7QbebqMEsTkAHpmgYyWfCuipGuTkk9we1RxXXleYGCsDgBxyABVjXLorPLpyIFhiAClRzn1q29jb3Phw3BhKSRoChB5I9T6/TrRYColrCLkSs455JY4IB6YqxPYNNbyXCSoVjHAHJOKzWdZ1fYrt5YABPpUkFs6x7TNIhcFgAeCKgDPClWy2AfQmnxvGizOUYsRiNwcBfc+tPuo/tF1FHbjfIRzz3qR9NuIoXkYxlUGSVbIAq2BCBNHIJGDBgMq45JPvU82pPLDHC8hCg5JA5Zs5wSKqBMwNMpJCkAg9eatz2D2qQyTL8kwDKehH1FAF2W6S8Y/bd0BUBQinjAqjFLhljneX7Fu5MeOo6Hnr1P51Isc1/BcXY+YQgGQjqMnAqU2slvoVvcSlJFnJ2R5wUGevv0/SgRonXbeXS5rEGYQuPLCuS3H94Eng+1R6DocWq6deyvMySW4zGE5JwCeR+FZGnR/abpbZYy8kuFQjPB7nA6//AFq6yTw1qGhAXGnzNMHHlTKBk7DwTj8aYGDpVhHd6TcapcFzDbuFMaDBIIHP05qO1GzzJYyQpfagc8MK9Bt9At7TUWNrGBa3UCh1xgApjBx6nvWdLo9la6eRcYZEmeQRMwBIJxgY5P0pNDMGbTrswWswaKSOYlgdwHlnpzk10XhvxOtsZre/mRj5nyzDODx0wAcn8qzfENus0cKW9rBD5QDuIh86g8cisSF2tCDHIJkDbiCAMY6VIHpbeKrI30dpblbiSUgIY3GB67ien61tLNvXO5T/ALpyPzryqGWPXb37NPs3FC0c0UQQg9TkenSu50a5EdrHZ7MrCu0SIMocfyNUgN3zgFYk9AawfDDY0+4H/TT+laEz5t5jn/lm38jWX4cONNkPqwP6VSdgepukjFY2soMRyDjHGPWtQNupJow8RUgEHsRTUrEtHLDpmjvWsbGIIUAwB0rPuYDDIOchhkHFdMZXMmhiHHQ1IshQjBqAEilyTVWJuaNvcZkG41orMhHWufVypyDU8U5ByT0rNwNIyNhpABnNIHBFUEuSxAPINSLNhipPeocWi73LLYxms1zsjkXJIPPPar7EtHweTWbcEpkMeSOKcNxS2IC/FNEgHBNaOl6Wuoxu8sjIoOAF6n8a3LfRbC3/AOWPmE9TIcmrc0nYhI5XrQBW1qeiGNvNtIyUOcoAMDArHBqlJMmzGZpwNSmFnAKqTkdDxioCCrEHqKYWLCuCoANBaq4bFNtJjPAzsMFWK/XFAWJygI460wAg804kAZPQc5pwIZcbh0z1oFYmMiPEoIAYdGHWgu4OScj6VXzT1kLKQTxSsO5cQiRCwbnpgUgmmgxxkDuapBypyKeZmYYAP51PKVc6OG7jljDA+xHoalD5rnLe6aA88itmCZZIwwbrWU4mikWwxJ5NSqwHeqobmpA+BWVh3LZxIhB6Gud1PS9kiC1jJJJBA/DH9a21l4oYgnNOMmmDRx7o8UhR1IYdQaaDzXVXFnBcyiWWMOwGOemKhNjbxg7YVH0Fbe0I5DBhQmQcVvWdvEIRleajgtQJRkVfEYUYAxUOdwjGwqqg4ApdoOcCkAINArO5pYjdCe1OjhUc45p5oBoCw4DFPFRh6cGoAk60hFAIxQSDQBG6KeoqFkAPFSuTmoHYg0AOGB0p4YYqvuNJvPrQInZgTyaYxXHA59aiLk96bknvSuMk8xh0NQOSTk1JSFRTAgINNPBqZgOlQkYOTRcBMAnNITxSFgBimFqYgPWmkUbhRmkAEUhpSaaTxQA3ODTgeabmgEUwJ0IGKlEuBjNVA+DSh6ALJkz3oEmD1qvvz3pQ3vSGWg+R1qQNVQP71MHGBQBYBFPBqurgU9W75oAnFGKjEnFKHyaQgZfaoWQ56VOTmkp3GRBSBUgAZMGgjIpC5UYFFwKc6YfgVAatSkkkmqznJpiGZoJGM5phNN5oAfuo3e9Rkmm5NAE+/IxmlD1Bk+tLkigCYycUhf3qHNKCc0wJdxNKGIqMNSgmgCYPUiPz1qtuNKHIpAXN2aC3aoA+R1pwb3oAkzQTimbvems4xTAUygdTSCXNVy2SaTfjoaALJfvThID1qlvNKGPrSAulgaQsKq+YfWjzaLAWt3NPD1T80+tPEme9AFvfTGJIqIPz1pwbmgAJNFBIozQAuKUGkzS4oAcDmgjINNxinAmmBGwpMVIRSbcmgQgBxS804ADtQQMcUDGhsUuc009aQdaYjH8QRktHIPugYP6VJoEm6Jo3POM/zqxrEfmWDYGSCDWVo02y6AzjcCKaGdATSU4jBwaaBSEJilxTsUEcUAMKnFRlcirG3NBQVSZLRAkIJ5qTylHanBQOlLj3p3Cw0L6U4LSU8cGlcdidCAAalDj1qtupwapAc7EmkBIpaQkUhihjnFSoSKjHWnA0gJw3FBOag34PWlDjvQMmAz1pJAoXmmiQDvUcsoKkZoAgkcDIBqsSTT2BoCE0xEfNLkipxD607yVx0ouBAJD608SH1pTCBTShFICdZM96kD5HWqwGKkXrQBYX1p9Rp0qQUgGmmEHPSpSKTFAFS9S7MB+y7d3fNcNqaSKMzzZk7gdq7263mEhSRn0Nc9qVhappzTTjMnqx5oA4lxubIpwt8DMh2j1rQ0u38+RmWFTt60+UCa6EcsYEYOTj0oGcYq8gUkibTjcD9DXT3OnR39jB5EkasorFmto7PfHMAZB0IPB9aqwjNDAU4OCKlSLdCWNu7juwq5FbrcW2y0gLkdWI6fWjlGZscMs0gKISqkEnsBW3rV6hhhtUOIwuavWFqYdHmtxgyEZwDXLzxyrMQytx7Ucoh/2iRYzGshCHsKjBIpgHrT+1IdxepFSEBOCcUtpC011HGCMk96uX1jcHUvs4iIGcAgUxXKYDlxsQvnoAKvNp0wtPOCkHuh4IrozoUVvYRPbuwIAJJAyT9atnRrWS1EzmZ3YAk7sg/hRYOY5C0smeIXBkUFDyh610ur3Ed1ZW4jBbCgkjoPxq7FodpBBcEQhgykgk55qzo9lZTacBJArHOCMUuUV7mlpTRvpcAjIIA6A5x/nFXPLb0rKWy063m2CQoT0UvipLtbSwtnuJr+eKNRk/Nkn6cUMpGhs56D8RmoZkDnbtB+gxXL6Prt/qmpyW8N4VgXo0iAnHbPH1rpQNRTIMVtL77sZ/CgTRFcaXbzoEwyMf4wcYrjdQgvNQ1Q2kcxMMfCvjt712E97fW4LTadGY1GWKScgVyK6rc/v2sLCTy3Jw552n3xTTEijDbpDffZSRNcAkM0h4A9RVfXbaKKUeXIrM3VR0FZ0yn7QzOzFsnIz3q3arZK5MwcjHBB6U2wCPUpra1WGFQjD+MdagWV3ZmkO9icktyaW6CJKQtQI2HAqWUkWCzbht6UFsd8EinKoKsScAVCgMkhzzjp9KgDV0qGEt5zFt4OAM8Gp9Ut0gnZo48ZAJAHU1QAESLkFQc8mtWO/kjgEUgBGMZPJIpFFjwpOItQdZARHcDAJ4BI7Vb8T6pDFMsVtCZLiMfeHIQcce9UrSEZREmMUbNlXJ5HritHUNGjWznW3kJZoy0jsc7iOeKaZLRwkzNO5kdizE5JNWdPsHuLWeZGAMfY96n0/SGuLO4lm3qU4QDjJrU0O0kt7C6hu1CKc5YnkcYqwuM8L6jb2kF1JdFcMRx1JP0qlqN0dRvnuFt9ikALgdQPX1qkJEWZosAQg4JHUj61rPMjLDBHIsUYx82OR+NQxJGcnmIpAaSPPXBIzUlqgFxGh3spz0Na80XmsiSzK0IGAwP3qh0m08zXFjZiigEhh244OaQ7jCLie/js1cLu+5npj3ro9Iid5prG7CiRVOCnX6g1gWVsYvEUwlLMLcEliM4IHBNaKMotI70yFZpJ9jsB1BqkgKd3pMNzd3EkrbbaFdqMvAdql0O6hjtY4xF5s0TnHzY2g9/etPWrZHvbDToVzCDvIzyfrWXdaObW9eAOY492YXA4I9Cexq7AYl6k39rXRAJw5bAHAFXbe9WEKZEDKRkNjkU7XUFlqTiA7FkjBbvk96qXF9HPbwwrGUSMYyT1NQ0NMsalBgPKhDQkbihHAPqKwYzumXPQHJq6t/KYpYC3ThT2wP/wBdU/LYEADrSQEl4DJIpBBXoAO1OlGMDOOMVGo2OA3FNABUlidw696YhoIR8A5qVrljGY14U4yBxmlHk29qtws+6fOAuOB9RUBJLlyckknNOwxZBwWBI9cGr8JU6GfMP/LTI+maoF8oUI4PU96m8wi0EZ5Xg4osAjYCYBye5q/GQNNSOST95ITgY5x9azkGdvucU7kOjZ+4cgelSBvW62d7sjKrHMBtIBxnFVHnKBLdzhQcknrnsKo5dpgycEnOfSkuDmfd079TU2GmW5xEUYuhYdSQOlZu0nAGcZz1odjlgpOD1pqlQMHOe3NWkDZLtkPJQuB27AUjHjhRj0NNMzojRg43dSPSmmQnGeeKpIQHBOW6+xpHIBAHTFORguSAM5zQ2GySPwoHYmjZ0gNwhxtOD7GpIQoheSRgi5GMnkk9qru+yAxKxIfk59akkGI1BOSRkmkFiR94IEbEKQcYqvAriTfjKjvVm3kRQDkkgEDPIApJcBCwIP0GKQh9gTFdxuE3qCSVxkkV0el3Nibe7hv7lrWaZgVlK8lQOn0BrKl0k6bpkN/PcMHLD92mMgEZ/TvWQ13KWLKSQe7HNMDVvoGEO23kSWNCWMoOCxNV0u7tNONu0h8nGQp6D6VHLIJlA3iMADcRxk1GEmHISRkBBLYyOtIRp6TCrX4h81WDRkkAE59vrSmWKCVzJBK1uOE2yYKn34roLO6ureOGdLF5I3jLEBFBU49ew4ri7u4a4u5rhzgyOTgdBRYBRMrXE0wd43HzITyc+9SO4EBy+d454Iyc1WiZQxds8cggZwa2E1ezms447rJYAB96ZBIPXjFA7mWZFECvGCDznPQ1qanc6jBp1tZX5SQOBKrEDcB2Gf8APWqNlGl00sJcQrICyADj6fjUMgmnEbsWYKNu4nIAHYewoEdN4WuLeWxvdJAVLi9hIMxJwSAcDGPc96ztUtLy2ax024kRokTCuhyCM5PHBGMjOarWUwtcnYrEjgkcg+orSJN5NaXk92I5AzNlwCCeBjHYcCgRq+D9Jt5Al+WjFxHMRGAwBKgEHIzknJrtSSPeuI0W4tNJuJ57iKP7bK/+jkHIAJ5OO1dzbyG7tIp2iMbuMlSMYNUMheV0IQMiyOCFDHqfp3rmtY3w30RcvdapKMQgLtVPcH9a6N7EHXftUgyqW4WPPIDZ5x+dQatbIlg187YmtVLRuT3OByO9AHHjSjLbTGe8KaqCTIJDwR2HvnjB96xZLSLzo41eRT5YLfKThxnNaOp3R1jUFkhZYWAWF1c8nnGQfTmrFzDfaVAtrMbdZACySBw5IJ4A9Dx6UmJlFRaeXaEvMLkrksoJBHp/+qtrQtdvbHUDDeITZzSDBCYAY8DtwOKwrq5nMcKXUpQkEocAkA9eep/OprKaC5lNnNct5chQrIx5O09P1qAbOpvPEH2aS4s5I5BcEkqXOIwCOgPfA/nWhoJ2aYR7j+QrA1MJqDRC5idTaAgpnkg4wwPfoPzrf0RFj0oBBgZ6ZrSIzSEnNSh8jk1VBFTCMkZHSiwDZCMnFQOgdcMOKsNGQM1GR7Gri7CauZE8Oxyc8HpUUa75Ag6k4FbjWSXCdOfWn2+mQxYOCW9Sa1VQzcTGNpL5mxELH1A4qQ6bdqgfywfYHn8q6VIl44qbyVI4FT7QfIckI5YiDIhUe9OMylsg1vXtgbhcbgvueayLrSnt03IfM9QOtNTTG1YktZA4xnpVp7GGYq0gztzgHpz61lW3mQkSyRsI/XFbQmBUEdxmomNE9oEt1wABk5OABk/5xV1JVdsVlb+c1agfGDWe5VjSVgBgnisbU7KESQvCFjySCAOua0POB71XusOgyM4ORTTCxTSBQc4qXyIyQTGCfXFAp271NPmYWKd+FgSPyogSx5OOABya4KS/mKzSRyAKZSoCcDBPWu6m1m3ivRan5yRngZFcDJsvYpIooljZWaRQvBOD0P0B4+lHOwsbt1d+RppBOZtg9gCRxzXNx3mwZLSGRf7p4xV+aK5mgLOfmYYYew6DHrWbLJHE+2MBcAA47mr52RynRxapbGBMFjgYPGOau28gnhEicgjNcjJFI5VYjkmMMFHUknFauj3j2CyW0ybDgkkngVanclxNwgjnFIpOahguxcYA4GMkGp8YrREgDmrNvcGFwCeM81WHWnA4YHHQ0mkCZuxXAdQx49qso4bpWGl0quAsfGepPNaKOAMk1zyg0zRMvA08ZqolwmcA5NWFfIzUNGm5MOaQrimhsGgtkUgIycNUgcGoGJzmgN70AWMikyBUW/jrUbzYNFgJzIKaXGapPckHk037WB1BA9aqzFcviQUokHrVMTBgCDTg+Klody6HHrThIPWqIl96esvvRYZcY5FVJSATT/Oz3qGVs5qQGl6M5qLNODYqiSQ8Co84oLZqNic9aAJd9JvqAsaQt70DJjJzUTSZphb3qNjQAMxzmml/emEmmlqYh+7ml3e9RbjShqAJNxpc0wGnLQAHpTSaecUw0AJmgNTSc0UAP3UoaowaUUATBvepFeq4p4JoAsh/ep0IIqkCakRyCOaQy10OTSgjNRCT1pQcnigCYNTgRUYBI4p+045pCJRgio5ACOuKASKjkfrzTGVpuBVZutTSHJqIjJpiGYpdgzS0d6AE2gCmkDPAqQjimUAGKaVp4IFMuGEdrI+QCAcE+vamAgXNIRisGOw1WbEjuU3dckA47cVvImyMLknAxk9TQAmacDTO9KDxSAdmlzTM0FgOaAJASKUPjqahMntTC+SaYExm9DTTIT3qAk0ZJoAk3e9LkVEWpNxpoRNuFJuqPNGaBD9x9aM0zNGaBkgY04Nz1qHNOBosBYDc9akUkdTVZW561Kr4xSGT5Jpc00HNKCBSAeKlCmoQ4zUyuMUABSkCkGnFhTdwoAdijimF6YX560AS98UECmBveng8UAIRzxTcVIcYphNCC4yWMS28iHkkcVyVsfJvUP8AdbFdgD19xiuTvY/KvZBjHORVJgdTnegfOcjNJUNnJvs4znoMVLmgkkFKMUwE0oNIY7FJTgaTApokSkNKQaaQetUAYpc4FNLEU0sadhXJeaTcaZuoyKLMLkgc9zTt49ah3L3NIWFFh3LKtk1KOaqISMVYQ5xUOI0yQqabjFSgjFMIBpFDCRUTHnmpSozTSgoEQ5FOVgKDGM9KUQk9KQh4cUu/NJ5WByQPqcUpUAEllAHU5FK4xc00iqx1PT1k8s3aBj2q1GRMN0bBl7EGi4DcU5RUb3EELYlmRD6E4NNOpWKDm6iH/AhTuBZBxUoNZY13TS+z7SB7kcVfiminXdFKjD2NAE24UhYVETg80hcetICYkd6rXFpBdJ5c0e5fTOKfvzRuoArQ6XaW4IhiwD1Gaxo7ANqx/wBHAXvkV0W/FNeVoo2eMZYDIGetAHkNkI4vmCvJxjAYjFbltolldQJLJGTuOCCcn8azbbQ79GLLuGBkEDOf1ro/D1pcgma4bKjjGK0RJPaaBDGdkgItx/BnIrVjs7VYTDDAkaEY+UDIqwB5hA7d6Rpo48JChkf0BxRcaZwrQyaJrJhmJMchOCehHtVLxQYZZ4Y7cAv0wo6nt/WtrxxaTNFHd5yQCAo6DvXHXF2k0aSKCJE6k9qoQ+40a4srcTXGEznC9Tn/ACaonityac6lpazXEyoUBwg6A+ppuh6SdRLTIQ7REEq3GRzU2KuMstNlEEOpQkSBDkoOCMcV1kFompTxXiAFU6knHNVre3+xTTQtG0aS8AEDGfb86vaXHJbymHcDExyMdquxFzUYAEYGMDpUchwBVr7O7HsKadPZiCScDtmlZDIUbIKHoRg1zV7c6hY3hsNNGd/IOMkf55rrTa7R2H0qFLNY5DIgAYnJOaLIZjWWiQ2uL3UZmuJzzhs4BqC+0G41e4M08+yFR8gGOKuavrNnZMqOTJID91en40xNYe+sWaCNIY8H53PA/SpsJFDwVZSQX96x5XBQH1Oa3LjW7iHfa29uzSJkGRzhQPU1x9h4hvdOeSOPY8ZkJZ2GSfpVnxHq1xeRwRxkIso5VRjI44J70Mq5Wn1S9ks7hZLgsCTwCcDPpW5p9/LpfhyO3htHEkhJLnBUA9zXEiFzwJF3D+HPNXTq+ohNjMCAMDnGBU3GNvLaGNZZEuxNIxyQRjJrOimeOQMpwR0qZgTHlhhj1qvjBzU3GPdzI25jyabGCZxjkU0nIqWAEqWGNw6ZpiCRz0BogkKOxABOOhqIjknI5NWLeF3AKffY4A7mpaGP8xp7hXY4AGMDpVp5zOQSCMDHNQvbtakiXAYdRmpbchoy+CARgZFJoaYJceWQFbhiB9K6C0vxZIbO9VzCxxvQ8n2rj0fypG4Bxkc1Ks8kkkcZYkFgACcgU0hM9VbS7IWSIoxH1yeSRXA63NPc6w1vDIZI4zlQvQD39a7LUDMNFNvAMyBAOOuPUVzV3bppNkZYZEJkUglhlif6VdiLEOiBDczSXUcJhb5HDcEEV0OqafYHTvLjt1TdgrIqjgfWuLhhP2fe/BJJ+orstBmuYbcTT3BntWH3ByUPPNTYox9Xt7q3sIAYl+yqAPNU9T2q7ogMml3EswAMoCpjg8Dj6VPrsU2sKVjPl2duN5kA4Y/SsTTbJVeyczsJGkwATxjtTUQN7RbVEsrtJiFmL4lY9cCucRJrzVRChJhBLIBwAB7V2doRIs9vPCAzkqJFHX61BDp4TXgUGUht8E+5H/1qLAS6RYB4Rd3LMbo8E5wQBnAq3f24uLYkHPl8kHpipYAwUb1CkDBA6VR1FZbWW4v1G+FgAwBwc+uKpAc/4rsJXSGeOHKIoAx15xnPrjFY8OmzTaS4kUIDlkOMkntx2rpPMgm8j7bHcRLKSI/nOD04purRTSafdOJDDDCu2Nc8kDqTTtcRyaaZPcA/Z1BUAAsTgk96ph5Eco5z5ZIx6GrtvfPa2QjUszOCSQcbfpWcrY3bjkkkknqTUtWGmTNIZEOQuR0I7UjK8ZAI6iolYrnHerLTCYhnUA4xgGoYFaRNhO4AGhGG7OMj0qSYEgEHGBioMYbGaLjJMYGaEcA4YAjGMGlXBBHoDUZHyg96YEkQwHPfPFKSalsrK6v9xt4iUU4Lnpmrk2kXNqnmuUkjHVkOMH3BpMLFRZjGAAOoz+NRSTiREOzDAYJByDUlsjTOF6jsfWq80LwyFJBhgcEUWATf1AySRimEEDBIz7UYIpwAIyaoYzJIB/Cl+YdakhieaYQxjLseBnH60TRNCxRxhx1Gc4pCExtyT1I4FNLEnk5Pem7iSCTyKDknOfxoAljQyNgHAAySe1SuEELEDJBxn1qGKQx7vUjFKcmMYPFMY+F0EeC3zZzjFW4XidQjoAQMZB5J96zcYNW7IRvMBNL5aHJLemKTQjoUuLfUtPWzlTMluMhs/KB/eOcHP1rKvIbIzA20zSxg4eQrgA1fi0q2uNNmuP3kcYBKgDBc9zz1B9BUVlqCQRiFLKOSHaDIXGCWHUZNAGWLVpWJiKlQM9RwK0tO1NoJFaRWdUYB41AIIxg8etS3sNnD4et72NIzNPOSSpyVX+6R+FRixMxku5CY8gMI04OPWpAbfaxNeubeEfZ7MMSIxlSRx1/wq/oVjYXtpPG6LHdwkkFznIPQY7//AF6yrqyhjebZNkAgggY60zSLo2GoC5WRQ8YJCscB/arQGzN4emTL3HlQwOSX8sDAI74rLsrTT5NTksWkdxIMQyE4BfqAQOmemaivdWu7+9luH8yLzBtCIeAtVDF+6SU4KyZwQfQ0NiPQL3wjc3KzXLvbxzLEQkUKZBx6n+orgzFcW1uglheONm4kIyDg84rqfCWvCwSeG9lmNscCNsEgHn371r30NpdeEZbeMGR0BMIkGCWz1AzQBxJTyrhF65BP4YpLICa48sKz7jtwoya0LHTPPtpYbhhFeKQVJGWAxwMd/wAKv+FPDk8168k0rRwgkYXIZyPQ9qVgOi8M21nfaXdQ3VvD50RERYx4IGOCPeujLJb2jsJIyIlIA3jBIGf6VxmjNcW0t9DazRfYhNgiYgkkdSD1qnNMLa5uoLO4nmjUARK/VQc5GPpxSA6u11OG/MUv2uFMAFowcYJ7VkeM7prxI7O3nQW8bCSVlJYk+2PT6+lcxdh7N5LSGQTEqGdyoBA9KbJeX8cMCIdx5KAYJIzzkUrlFSztzd3BjQ4ABLMRyoHc0s03lzIsEYlaBjlyMg9gc98Vagu4zNJLNICtxCY52RNvlnoKj0+4gt2Fu4bMykBycDHPIH4UXuTYozXU0oUzZbYABkcY9jUtrama7IZGUJGSTjODjimyQRxxzxPMfMQ5TJ4IPNTBpUYSs6hgBwTkMCOhFIGegQaXbv4UMk+yaRIPMVx95SADjPoOOKv+H4wdCjJ5Jc/zrgdJ1W4tPtocP5M8ZRwo+XJ6ED6DFdvol/aRabBC1wqu7nYHBXcfQZqkwNEwgPk5FWVGBTHPOKdG6kAZ5q7ggIzUkcaMuCOvemHrmnEkDIpA0KI1hPBzmnCRfWoy2VqEtg4ouKxeRxU8ZyKzklORk1bSTtmhDLJXcKqTwSFW2MQTwCO1T+fUiuGANIVjDnguEXMhLnucYqHcR1roSEYcjNZd/bhCGQg5zkHiqvcEiBTnFSoxXoarrkYzUik5pMZZWRh0NJI5JySSaYDTiMjNIY0GlYYQtkYAyabnB5qRoRNCVJypGDj0ouBymo3EMM0ciKkhEhc4PQdPzrEvL9BqgkS3wzckd8Y4z7/4V28nh3TnVwLcZYcZPAPrXHa7p7QTJJDCibRiQoQQTkYI/A0AZ91qHnGMIcg7iT35xnI/pVK42mRCpBJHODWxILM2xfyooiAQVA5IPQjvxjpWDCn75EPBzg+1K4Glp1rJJb3UyOwuI0BAHOV5zj3rQeD7MIxcxSebIA5RuMDtyaNLu7WyyspYSMSHwM4APAP412XiC1jbSk1BwGKAEZHODjNaQkr2ZLRykVx9jLqI90juMA8YB44rTG/7U5HMYxgjoD/WuauN9zdSQ2sJzGCWIII7flXQWmLaCO3LFpAMkDk5PWt4yMmiyBzR/FT44ZpZAkcTlj0GK1E0EuMyTOh9Nuf61XMLlMkHnipVnkVQAafeWM1lId4/dEgB+xqAYNLca0LltKGkBc8+taAulDbe3rWNEksjAQxu/fIGRipPNA4Y4aocUUmbqTKTjPNSbwawYpm8wYNaaOcdaylCxaZYPNNIIHShTk1JkYxUWKIs4PNVbhuwPNWJQcjFQMm7OatCM6RnDgEEg+go2IVJYNk9CT0q452jGMk+tRMAyEkc1omRYihmkQ4ZgR29qurIGGSay2YhuMipYpyOp6USgnqCfc0iaTcRUK3CNjmpNwPSsnEtMfvPc0hf3qM5zRUjJRzSE4pgOKC2e9ADs4qNn6800t71GTQApb3pC1NJFBPFMQZpjHmlxTCpJ4oACc0mBTtpHagKaAExRjFSbSRRsPcUAR5pQaeV9qNtIBuaaTUmygpTAixTgpp4TmnhQBQBEFNKIzmpggp4UUAQBD6U4LipdvtTSMGgBgFPFNNAJoAkHSpYQCRmogeMVLEQDkmgZeVUVRUbyAHGajMgYYFMxSAV5eoFQOxPWpdgxk1G60AQMeaYTUjA5pNooEMFGKdimmmAjHFMzTm6VG7LGhdyAo6mmArusaF3OAKrojXEgmnGFHKx54HuaaqNcSiaYYUcxqf5mrBbFADictmmluKYWz0pNxxQAFsGjdTTSHigBxYgU0uTTSeOaaSKAH7vekJFM3UbqAHEikLelMzRnmmA7NLmo80ZoESZozUeaM0xEuRRmmA0A0gHg0oNMBpwNAx4PNPBNMFSAUMCRWIGM04MT1NRilBpATK2D1p4fFQBqcGpDJzJxSeZmoWb0pMnNMRMWBpM80wE1IqFjxQA4Eing8UgiNP8pgOlIdg3HFNJ4pwU96CvFAiPJxWFrKYmVxxkYreIxWVrUeYBIDyCBTQDtFctbuhOcHNaFYujTBLgqejVuHAPJqgDpSg1HuB6UhY0+Um5NupwIqAE+tHmY70+ULonBFKQCKriX3oMhI60WsF0I5xmmF6GJJphHpVpaENji/HWm7z603DU9FHenYBpkPrSqWJ5NSlE28CowpLYApaCJ0PA5NP34Gc1EEkA4FMKOTyKlopMsi4PrSi49arhD6UCN/SlZDuy1549arz6rZ2w/ezKPoc/yqtes8Nq74PHYCuJmuGkmdiOc4qWhpnYXHii0h/1UbTfhis248W3T8RRpGPQjP8AWucMrN3qNnI61DHYu3GqXlxJumuXb2BI/rUIvJs4E8gHpuNQDJ6Y/E4pCuD1H4GpsUSnZ/G5zUqX0sC4huJFB7A1TOT1NAGe4FFgsTyTNK5Z3Zie5OaAR6D8arkbeMg/Sk3EdDQBYIB7U+O5lhb5JJF+hxVQTEHJ6VMI5pl3Rxsw9QKQHS2Xil44xHMm8/3ieauzeIsw7oAA3oQK4lHYMQeo61OHbHBoA3m8T3pBACg9iB0/SoD4l1Mf8tl/75FYzlgpJNTwR+ZECRmgDqtJ8QG6CpckBz3H+FbvHcgfWvO1Zbd9/kkkdMGlu9TluowgZkUds00wOyMXYAD6UPIlvA+SAwGfWsC28RT3tpLNDtBQHII6GovCN415qFws7b5CScnnFbXJNPS7+bVPNiBWMIcE9zW1FCsEZKjkd+5rnLpH8O6pJd+Uz2jgn5euT+dTWWqte/6VNcKsOcKm4AYpAjH8UQazNAz3VzAtspyEjPJ9znvXNW+n7LUzXRKRn7pHOa1NZvZH1VoZJg9uSBuHSi8vYbua1tYiTGgAJIwD/nFAGTb2El222B1OTgAnBrrPDfhXULS7F1PKsUYwCgyS3Xg1D4dtHtPEQEsYEcmcE8g8V3xNAGVdadFPdDcPunI4qZLVI2yB0qcDM7GpcAigBsZ55qcEEVAevFUr/XbTTcKQ00pGdidvqaSYrGk8YIO7getY9wl1LMY4RlO57EfWprDUJtYQSC3MMOcYJzmtERFBjcQParTsM871qwik1ZIR8gwN565NGpxSO1vp0ICxsQMAZrqLPS/M1G5u5kDc4jJ9qtLbwi6NxIuW7U+YRw+o6F9iiVIm3SE4I45Pr7VJcwoun2l3IoDxkKQByfeutnjSe6DAbVAxk1HdaRbmxdYYvnAyCTnH4UrgeeXU8Ml9JKpABxgAdahjBe4wOciuvm0KyudImlEZjkTkuOpNYz2c8lnDHDCY1T7054GfrWTRSZkzgxzeURwapyAqxU1ZuRkBjKHcDBYHIqttZ+Sc4pFDAcIR696dESrgA4Hekx6UqqSQBxk0xCSoA5IBre0nThfzEySCMxRhlKHnNUEtVmnSMSrhhk+1btvprWmlTySHMjt5cZxgEHv9KTGZj3gMsqkLNITs3EcfUVNdXCSCC2KFCo5BOc1m3Vo9jKI3IORkEDg04FriXzMgEAAE0rCKzqRI3Hc1NZRf6bbySDEPmDJrWsLBbuR7eUspAySp4IrR0XSobe4mEh3r2BB4IrSKE2dQTmRJh9wqMH1FZl5aR3GpCUxKYVHAI4z61cTKIAGJ+ppspdomVSNxGASOlbchnc5bXkDagUVl3naAij1rf/s77BZj7KSWEeHU9CPWoV0+2hu4ZJTvmYgkk45rYZ4iwBOSTgCjkQXOZvfEguoorM2729qSA8jDGQO2MVVF7a2mugrbvcQquFVBkqfUD8KveISbjWrKyEeY8hiB3yeanmkgtfFdqQAoZAhIGADzik1YpO5rRXBd/MCMiuAcMMEfhSvNsv0dRwVwTnvVuWKIyHJwf0rLublDKBHtFuD807nAPsPWixJqC4B70pnAHBqnZ3lhfO0dvMGdeoIwSPUVLIMHAp8qDmINUhjvbdRnEiOGU4/MVzGt3ep3d8lpCvlQuMgEDkDqTXSSqXxjscj61lXttcXN4CPlKIQGwCeetPlQuY4p1kt5ZIy2cZGQOD9KrjrgVuarDMIPJUZVDl2PUmspLWQgOQMHpXOzVEQOBg9amgKlsOCQfTtTJlAlIAwO1LBGTKMEADkk+lS0UTXMe3lAcehOcVV5JxjmtaSSKUrhQABjpjNVJ41UF1GDU3ArkFTgggkUhOF+gp0owwGSSRmoXPGD0JxVIDrbCSS10e1WOQgEGTjrzWlNaTyW2y5tvOhlQHdGcED0/Cq9xavb20KeWwjWMDdjI/zzW1Bf291bhUcI0IGdxxkegrJ7nZGKcEchc6abSE3VjM0iR/ejIAKD1rGubia6cSSsSQMDjoK6W6cR6beueCUIH1NcwuDEB3q0zCpFRdkQ5A7c+9PJ9MU1wARikyQCMfmKoyLumySm+jjiMann5nHFR6mwa9bCrxwWUkhj61UINSYzEBTGAiYrkDNMBHfrTkUu4UnAJ5JPFSRW5klK5BA5znrQBF3rW0WytL248m6l2bsBBnBY+mahhtwoe4AGEOeTiuq0i1mjtAZoYTGXEgO0Epzz+VAHK6zp40vVprVCxiU5jLHJIP8Ak0ltEiRg3FuW3kGMn+L29qteK3jfX5/KlDqMAEHIHGcfrRcXE9jJaRT26v5QEihjg8jNBIzU72+k8qyupFMdufkUD7oI6flVHgqFB+Udh0qa9uReXEk7RKsjnOR/nmqwIwBQNMcVUDIFXY9YuUVIlK+XjadwyQO9U0GQfYZwOtR9OaTQGxeXCS72hI2kFSAMEnjHFZRHzMPQ4pwOY80DHX1pLQAd3cAucgDGPb0robyLTBon2uNC6F1KRPnCnjI9/rWBsyKuC7dtOjsyCVSQMpHQexpg0XbEQzWktjGskDTyq0ZkGQqjrk+360/VbK5sWOns0lxDEfOikORlSDniorjUH1CScyIQYRmMRDkDvn26V1DeINCn061TVrpbi4CDJt4iSAMHBBx+PaqTIMfT9NnNvdXClmmtAjwhhhgc5Iwe1dxPLJb2pMAUM3JOOmRyeKrRX+kahqJurC9iJeMRTQPkMw5wcHrjngUHUZbi5kt7WwkdUODLKdi/hnJoQ0eeTSJaX0nlpKAJA6Ekg7euffNaM9xb6lesJpmt2LZSfqF46Hpx711mp6Ob6K4wiMzIEjBAAHPJ9j71x2rWN1ZkzkRwxx4QpG+4EnPfvQ4jILjTbm2SaYJ5sKjHnr0cZwDmkbUoVWFLSJiwHLuAGA9MjrUdprV/awiGN0EOCGBQEHP16VBFAbp5/LAIAL5Bx+FZtDTLD6jEiiFkU5BDkLzg+tWZL6I2iImnxiFAEE0gOTxjJrKjiadXKoJGI5z1P+NLHM0eYXU7eyMen0+tK47iTICShIcnGGB4JNTuJY4S0iEEAAEEH2pGWF1ysWxkIJ9CPUetPjZ5bNYzHnJJD9cc88UCepq6e11Np1zckRyRQkA7gcnOBgY+ta2hxRavpksU7sSkmJVBwYj2cfyP0rFtxcWUDQ+dGbeQgkA559a7LQdPhs0N3G3zyoAwBwCQetNIRo2puki8m7IeROkqjiRex9j61OGIOaaXA4yKieYAHFaKNwLSyZPNTBsispbsCQAn8avBwSBnqKdrCuPcnPHSmN61JwaZIR2qRjMkdKlWYgY9qhzk09RmmBIgc/d61fQ4GKpCQJjGQalWYkdaQDpbp0kIVCAOpNZ1zO0rEE59KuScqR61WaJRyRQgK4JI5qVTTCMdKAcGmBZBp27iqNzex2sYLHknAHTJ+vase61u6MkJtlVY8kOWII+nt3pNAaOr3aRBIZZGgD8B8ZB9jVK51DUvtBFluVVRQSQCAMdTVXULmHUhCDDcBywBcg4A6ZA6VcFxaWaG2a7cIQAQ4LEg8Zz+VSBvkNPasZpGQqMHb1PqfeuX1u+tjbwfYzFJIT5h3DAIXgZHqeldBCFsvM8ly/mkBC5wQB2GeveuFOju8t5bSzKJInyGIyMHnrng0AMtdThl+0NNa+b5pBG0AlD9Tz/+qsh42e+kQDBJ4GelRSxmFtp5JziiEtJMCXCse56UMZPEpeVzsZlTrg4rs5vE9peaKLeZWQFMBQMhQAR6981xsnnOxCjG85bZ3PvWnZW0IisXdRMrSgugHUA8ihCZJoVtctMY4i4M53AkZJA713uy20WKBGtw8jDlwMknuSapWt1oV5f281pmPyyYyACF6Z6fnUWqa1bvqIktnSdSPJjAOACeuSfyq+YVjrogjRq68hhkHGMiphjFZ9jdQzxeXCciEBD7HHSrgNTdlWFngjuIzHIoZSCCDWZDoNpECJFMmTkZJGB6da188VGWwafMFhYYo4IlihUIg6AVHNaW8ispjBDDBwBmpUYGncUuYLGCNC8pwYpc47Ef1zSSKYWwQceoFboUE5pjwI6kMMg0+YLGQjZGadnJp93ZG3t3khJIAyV749qzkuXPZvyqhGgRkVGF5oSQsBingH0pANKAjGBVSaF1OQMir6cnBoYYppgZcgzGRtGQOKpFTu24IPoa0Z4RuJUkH2qqINzEnJIPetU9DNoiMbR4LAEVYjmXAHT2prgsCCDxVfBWh2aC9jRVwRwaCarISCMVOASOayasWmKzAUzdTyhNIITnrU2GRk03NTGLnrSeTQBDg0mD6VaEPNKYx6UAVADTgKnMQz0oKAdqAGBM9aXYfSlBFPVh3oAiCEdqeE7YqUEGg4FIYzyx3phQdqeW5pu/mmINlNK4qXcKjlVmXKsB9aYDBjNLkUwxuBkOD60gBHU5p2Fck8wDrR5w7GozzSAAdqdkF2SiQUpdSetQE00tgUmhosl1x1qIyDoDUJY00k5pBcshzTxJg1UEjAUeZSC5fE3vUgmHrWZ5lKJwOtAGj5w9aTzAe9Z/nD1pPtA9aALzuoGQahMgHU1UaYnvSeYT1osBMZDmm76iJzSHOM0wJi4NUsm7n3Ef6PEcAHkE+tSSSBIXbIBAOMmq1rcW8NrGHmAJGSAM80AXixozmqbahbjpvP0GKYdQB+5ExHqTikBeJoBrObUJM8RqB6E5qI31wejKp9QtAGqTTSaz7e8mkl2SPuB6EjkVbLimApY5pmaQtzTc0xD8ijNNzRmgYuaKTNGaBBRTuM9KTvQAgpacBRimAgpQKAKcBRcAFPApoPNOBpASAU4Cmg9qcCKQDqM0A5p4Ge1ADKkUZFPAA7U8YoAZ5ZNGznpVhFBFKU7YoArhParMSYAOKdHEODipQAO1AAMCnFhjrzTCaYTikA48mkUc0zfTlIPemA5kBHNZmpQO9q+0ZwM1qZpjRiVGQ9CDmmgOOt3Mc6kcc4ro2jfapx1Ga5u4Xy7qQDoGyBXW2zpPYxv7VRJS3EU4PmnSREEkdKYFrRNWJHbs0hpQOakVB1IpXQkQ4NOGR1qbAHYUhUHoKLhZkR60maUg5pCKpPQQo5qre3yWEZZ8ZHYirBJHSuU1dnkv33MDjpjtUlI0R4kwOYVP4/8A1qYPFFwDhYowPTGawhCznAIH1p/2U4z5q1k5MqxsXPiGaYDBVPoKpHVbhut1L9AazSh5+YH8ajKtU8zHY021S7ZSDcN9abHqNxGc/aCazSSDTCcUczA3W1+9MRjMoYEYyV5rIJbcTnqc9KhDEninqxZgoBJPoM0rsocOOtT20drI5FxIQB0AHWpIbIyyDcjcexrSe0ghhJ2MGHtmgRWVNLVgQjnHsafJNHJmG3tEAPQkc1WWYseQv5Cp4IUlky8wjA7k9aAK02mvCMu4yewGTVdrG4GCIXKnuFNdPa2Vi04lmvI5COgL4q7LrmkwuUe4BI7gEigDiGt3XgI5/wCAmlNjcBN7xkD3rsX1bSZ2zHcqT6bTVDUbj7SAsEIcdyBjFAHLCGRzxGxH0rVttQSyh8t1J9gelXYbQm3yzlCOw6VmzadcXM5SFc575oAms3068uTHOgG45yKk1fRo7KITQO/fgn6VZ0fw5Ja3QmvUBA7A8CtzU57P7KYWw4PQA4oaGcLA8ksgRVDH3raitLooAsPJ9OlQWkcNlMZCm9c5yTXRx63pxjLFlQ+maGgMEaHdBwzwygDvmnR6ZBNNs2tnud1XrrxXChMcMBcHuT/9asmG4uLiRpoQ65OeO1FgOPjkcLlHYZ64OM1csri4sJluoDiQd6rENuIwBj0pHd+BnpTGa2o+JNS1KLyppf3eMEYHNQ6NIkbyLKm5W6nHTrVNmIgyYyB/exxUcbHbkEjPpQSWr4Ri4JjDAeh7VOLea8t0lhQYj6kdvWoLYMWSaWOSWEnAwOCfrWvKTDLG5t7iGEkFkxwR24qkwOztrRbjTbF2JDIgII71rKcisa11/S5IUQXAjAAGHBBq22bgELN+5P8AcPJ/wp3AU38cTyRojTSA8hBkD8arTa5aW84iugIXIzguCcfhRfSpo+jzSwqTIRgEdSSetcbe+H5hop1S5LPcSnOA2SAcVIHWXXirSLa3kIuN0gHCgZJNc7Y63c6ha3L+XGSuc5HNc7aaNqF7IEhtHGe54A+prbtr+HRZLe2kgMcinbICOxpoR0fg29L6ZMo2lxJg5PXvWxNNdSMV3IF9BWXDZNbagtxaECGYDcB0rWCAYAFDAdEr7AGbOPWgpg44x7CnA4FBNAEQjQdFyfelmkWJMAEs4wKCeaSTDx+WRnPHXpQBDa2hMYt5l4BJI9a5rx1fpDbJpdsFBIBdQegzx/WrniPVbjTIR5L4cDBJGSRXATTyXchmncu56k0mNECA5VDwD1qWaEwSAA5DDg1EyjPf86UyEuMknb09qmw2xpjZTgA4p8SkkEggU57gkEY5PenxSZT5hnnpQCZJN+6WN4/vZAA7Gu8upH1WzsrJiY70KHAAwOnc9q4e2kUalbyOMojA7fXmum1DWo9P8Wx3MqOIvLztI5PHGKBlXxTZvZaZawzGNphJksjZHTpjtXOQyMgX90pwc47GnXNyb29llfcS7lgSegNaug2Iu7tozMI2jG8AjOfagTNIR3VkIbu4tURZgBwcHBro2024eGO7jCtGU9cYrndZ1G41FoooYiLeEgMQO/pXUTa/a6IkNgkYYBQTk4wD/OrjOxDVyFYJggLRED1BzWRd3wFwUjOQOp9634/F1lJ5kBhYYH3hyK5iC0F/fzASAQgkkk4B/Gr9qw5CCZpJpEkb+E5B/nVyG4kkRXAO0dz61dt7exiIE0Z3DgEcgClu4rWe0lki3DyeSenFTzMXKYF7qLJLNdXLkTbSkQA5A9QOwqC3tzN4fM7ANMHLI4PPWs/UJZNQugQrcHai45JrobdBYaIkLYSZkJMZPJNUmDVi8l4ZtPjEufNCgEDmsHXrkssCMSIgMYB4yMVq6fcwT27MiFZIwN+48g89awpY0ujLcXOTx8iA4IpOVhI2tK1LS7JYxcbPOIyJ1U4x7+9dGWSSMOjBlPQjoa87VhbRqATtY4IPPFdRpUEcH72ylMkZGHhzkKfUU1O4NGsxVQSSAB1JrO1nVIdLgjkfc5foqHkj1q9cW/nKAgG4HIySBXHeIrO9dzdXhTaowAmSAK0chJGddajJeSElQiE5AzzVSa5cqEU4A6UiIwAYjAqOXBkbaMDtXOzdAWBIAJJ78U44J2k4B96iHBpwBJAAyfSpAsiQEHHbgUtxl7UuBg1Dgjp3pZHH2eOMHJBJP1pWAJf9bGe22iONZrqGM9GcA1FuJIJPSr+ioJtat0boCT+IFMDvkuo2YRibMmcbG6Yp81hYXsrKNiz5wWUnII9q56T55nb1Jp8F1NaSiSIgMOmRkVi2up3qk7KzIvEdt9g0xovNWTfIOQCD+NckM9K6rxTffa9NswygSFyWI7gdDXLKOc1pC1jlq7g3BApdwJwaQhieafG4hYkx7iRgc9KszHw26SKS7hRzjPenyxxjHlKQh6E9TUQcgCMgGPuMVLLctKY/lAVBgKOgoJI1CAHJG49DnpUmFTgSAsOpA4FQHJOe1PhBlk8stgEEg9MYpDuaUOZLRbbGJJGCgg8MCa7O/lSx0SWaNtriIIB3Bxjofxrh9OSe4vLeONiREfMOOCMdal1SUXF/czSzMxL4U9jjtQBSsbJ76/t7aIfPK4A/Ouy8QeH9NiE1/dXTvIsOFUEBSRwBjr+tczol2NP1aG7ZA6RZJBOCeO3vWpqFy3i3VIY7aKWC1iQ8kZGepyR79qYjm4rSSZDKnOSAB/Wmzwy28phkXEgIBGfWr95YXWlwQv5mDMCRGOGCjoTWdIJDteQP82CCw6j29aARYhuDaX0E3lFfLIzgckd6n1nULfULpTb24iVQcnABYn1xVNpRKpD9QOD71Gq7jjNIY5ASNoFHIqa2gaabYrbeCSSOwrqrDwiJbS1vZJTskhLOhQkZI9vamM5cOTEoPQZx+NSxyxgBmfB9MZH41PqxtZRbPbKVk2ESBQcEj/8AVWVnnJ6UhGwk1y2kzQ2TYQHdIVGGIJ6fzph1q7l02O18xSU/dsCgJI7ZP5VBpkzxXi4mMSHqcZGewIrqLO90yfw3dPOlp9rjfGUUKx54Ye/XFNITRgLpt9biG7t4JI7hTwm0ls+uPet22i1n+xTrVpe3JZHJkhcjBUDkgd8HIroLDXNJ1mS1zeFLyFiFD5G8AcZ7Hr0z1rTD2UNuwhRZRCDkk4QHOT396AscCdd1aLUJvstxM8cgykcgLkAjkjI4xTLme6aONp4sojBAhPLEjnA6Y5rtbGwe4jbV70ok8kYCALtEaA8D8ev41zupW8I1FRPdSGRlaTaqgCNR9OtXckyJktBvRYY45CmcPkAew9az4Wk3OiLuD43LnAIHrU9xdRyxqkbA5Azxn8jVNcl8AkH1FYspFlDELsgIyQk5wDjB9qjudkn7xiQw4BB7ZoijmmDiM7mHUE8n6U2YKkghdcEjOQcjFIYsVxNGFWGTOTjBGat2sV+EedE2CFwCXGBknAGO4rN2xbflJLEHjoAexrs7G/8At2lzSSFEcxeWFZurKOo547VSAivNG8uQXBXZFMqB4xzzk5IP610NirQ6XCqFiFyB3wAcCucu9cW4s1t4o2AG0hicYIGCMfX8q2ND1iN4YraTJkaVlBBGAMZGa0i0mSXvNfPWlMpYYJrEm1k29uBJGwIyrsDkqegyMd6gXVp4lCApIQckgYJHfitfaRM7M3iMmtC3cOnB5HFZFhdjUJnWFDtQAknjrWrHblDkChyTBJlwMSOaCagjaTJDDHpUwBPJrFo0TClDYNLtpdlAwyD3pQ2O9RkEGjkUgJw5I5pCAetQhsVIrZoAUxAjAqtIhR8Vej5qG4Tc2R1FFwIAFIwQD9Rmsl9Uikkls5rLdGeCAuT9arXVxqXmTQmIglgUIGMY5yDUxuLQ2Ek8svlcFSe4PoBRcCOeyu3shDGHWM/eLj5gB0OKpyW8UluhVyTENpA7+h9TT9F1aWcrb3VxtjbKoSACT6E1qmKwgkIdgTx8w6n8aTAx9Wv7u6toIXxAIpATyQQMYGDWBf6s9xcTPESokI3ZHUgV1usQW5Uy3LAwyrt34yQPeuHuLZYWJV8oSQD1zigZBFIGuFM7ZGSScetLCTsAxzuJ/KmFfmX0JFXLVQZIQRnMuPwJpCLOj/v7l7Mq2JSCXBwFAHJPoMZreuRDa2kstlIkscZCDH8AwSfqeDzWbd3dlp6FtOgEcjHy2c5wAevWh723GjTIsYIyAwJ6kjGfbr+lAGZbXDqJYlkaKNhuJBx0/wD11q2OmCK3hkvJSYXJcJG2CT6k/lWAGfaEQAl+Dn0rYtrVJEWWRXLRuuUBJBBJzx+Hb1pgdho2tlL42SWyxRk/KAmDjHUnvXVC4B71zsP2a3t0ldkMhBwc9CeoA7dqlhvSWRHIDuMquOcetIDoPtAA61A0245Bqj52R1pBNhutAGispyOalWQ5zmqAfgEGpkl4HNJoZoq4Izml3CqIkI71IshJHNAE5XepBHBGDWU9iomzzwa1GkwuRVVyTkmqTAjWMBQABTgmabuIoL46GlcB7JgZFUZXkST7hI9qthye9PwCOapMDHedXO1Tkmk4PHccEVJd2satkDGf51WRHDg4+U961WxLJSAAaVItwyAMe9Mc4YDsauhAFGPSgRAITnkVIqcYxTsUoFZtjSGeXxmmkEVYA4pjAVNyiuTilVh3p5UE0hiAwR3oAcDSgigKMUBcnAPNAiNzzxUbHI561I4IYgimH5gBjkUwISSKXfSkDFMI5pAPEvFBlPrUJzmmkmmBIZDnrQJKgJNAJ9aYFnzMCmmX3qEE+tOAJoAk83jk0m+owpzS4NFwHlhSZph4pm4g5zVXESk8VGaQyU3eSaQxTSgZphJNKDgUCFI4plTAbkzThbsYyfSkMrEmmHpUjKQeabjIoAYSaTNOIpu00AJmlU8il8sk5pjcHBoEVLi6mMrIrlQDjiq5Z2Hzux+pp9wP9IbA681HnIpDDjZwKAMgHFHQ0ZIBIpXHYVsAUgIxSgMRnaceuKYeKGFhxpvajvS9qLgLCdsynOBmr5rNyQ4x61o5yM00ITNJk0ZpKoQ7Jpc8U2igB2acDzTQKcBQA4DigDmlAJp6pQAgHFLinhTQFNADCKTFSlKNnPSgCOlFSiIkdKPKNADAaeAT2p4h9qlWMDGRRyiI0Q5GelThTinAqOOKkBUjrTaYXQxUJOMVMIc98U6MADOaUnHepGCxgd808AE0zNKCRSGWiV24FRk8VCZMDrTDJk9aAuPZsZqMkk048ikGBQIQrxSAEHNOJFApgG7HWgSAd6aw4qM4BqkhXOd1VPLvSQeGANaeiXBNqYzzjv8ASqWtxkOkg6YwaNHfExX2JqrCN7cp601gAMgVCXC96rvJITkSAD0piLOeaUyY6mqG5t4Jc4705yTyCSPY0xFozA9DSiQ1UQnp3qdQQOlAiRpgikseBWdPrthFkFmZh2Ao1cTfYiYiBjrk4rko9gcb+tJjR0cmvQSwHYrI3bNYbyiaYkNnP6VXlcbyFGB2FWLCQRvzbo+PWouVYc9u4xwefSpk0qaZcifaPQ1cfUlRf3cADdxVO51mVgY44yuepApjK1xZNbSBTIr57g1IkJKZCE/Sq5kkbluSe5604SSr04+hqGwTFEMDEiUZGeRWrpdjpsykm3HBxyc1jMT1YYoE7R52Hr1pcwzcLaULkxvbqqr1ODzWhHqWi2yBI0Ue4WuQa4dvvHNReZzU8wXO6XUrCYgRyoG7ZUU24v7doTC4QkjAIIrixKKDIp70cwXLlzau8xIZFA6AVCzBF2E9KrGfHRqjMwPUkmi4XL0EUcsZJKDnv1qpLHGGIWQH6DNRGRvWgSN3NO4x8SlJAQ+DVxbuaE7lmfPtWaWJOTUocsmAaYGvBrHnP5c8xVfU8CnXDWwAa1vCJB3zWMltJINygYPqajYeTJh+CKANiDWr6EEPcB0PcjNUr6aS4uN4kyO2DTInDjIwajdysucE/SgDZhhzbhZM5P44qOe0RYicj8eKrLqMoAGF/KrJv4Wi2lMt6460hFOFIjIM1K1zNbyfuSUA7EVCXy4OAMHPFa7TW8lttbB9MjpSTGcbbRST3qQhyCTgkdq07nQrlL8QW583Izubisdi1vMxRzknOTT11K9DAi4bI6UwOjfRNSiVUYwuhONqnn+VZ2v2CWcyLCqKCBnB7+tRx+ItUERjM42gZzgZP41lXFzPdzF5mJPuc4oGeheFrSRtJksbhQJEO9WBBOTyORXVW7gxBZWAkXggnFeQaXf3OnTCSGUhe65rUOutvM0rFyexJoZJ6FPpenzf6y3tyT3GM1iTeH4FybHUpLf/AGUkBBrhbrV7uaRjDcSRxn+FWOK1dB1WTTtu6ESq5wSTyPxppgSLrFxbTmxvA88UT5JAwTjvnv1rsIdZ0nVIBB5xiOchJAQapahpI1Ew3toQkycjgYb2OaeYLe8QQX9iscxGPMUcfpTAZq/iA6eVjsrU3GB/rQTj8K4O/lfULk3CxyYJAJIyAa7CTS73Ttz2kqTRqD8jjkD2zVbRJLKa0uLG4IjmlYk5GOfrQBvaI1xaWEFve4DlcqQcjFa30qCzg+z2qQuVkCDCnHOKnDAUAFGaaSMUmT2NOwCgU4OEOSeMVEWctgYx60FI0A8xxknAz3qQOC12O4vdSkSNcxySAZzWdf6U2m3qW7yFkYA5xgCvSJ4owAyIpUHLHHQVy2tPb6zqVvZwTDKA5YDIye3agDn4NMW9ujbwE7gM5J4rLlhkguHhkGGQkEZ9K7fS5I9IiurWaMRzRgkSkcN7VxU0zXFxLM5yxYkmgYJsB3OMntVyGAvGJNoIByVJwarwWclzkjCqP4j0q3Cj28RDHIHcDipsCNGaGbU7bfbWohjtxvL5AJx2FW/Exiv9Bsb3eBIVCbT1OO9dF4b0e2OkNM5c/aAQQTwBXOT6TBJfzWs0Yi8oZUAn5qYXOSGVqzHNLCUmhcxyg4DjnH4U+3024uNSNogwck5I4AHrUcymJmhPVTjNAHX6rbyWfhe1EUu4zsGkYckk88flTdL0WeBjquqsVtwML53JPpmm6DcfbFsBcujxxSBfKJOcc8+1bOt3UOq65Dp4DNbx8nYO/ofy60MRy+l2jaj4hNnC+23kcsw6AqOTiu/Iitp2trexjKwrkAHBP6da5i4urLRfFluyLsXYVIA7np/Wr97dyWjGa2kYSS9dxBOKLiuaOoztc21rcW0ZG4lXTHI9jWRbyst3PbMjYbt6modA1KdNVMLuxhcliAMjPrWxYXE8st0Y0jBVs5YZNFxmLa6Mw1Z7mUCNYiMRk88981kGT7TrNxd3DEpbjgDjt2qxd6vIb24JOWY4yOAB9KzLud3gYJhFfAIA6ilcBou5YpDKGxHccsOwNKxyMZqhJI4EcLADZ0xUqP5pAB4PWpuFiW5XFsx9OlWdOf7LcEK7CN0yeepquZMjY4yKUvsAK4B9SaLhY2bjUntrQvG58yRsZJ4FZjyiaMmZ2dQQSCck/SqzyFzmQ5x0A6CoUkEmdvTsavnYcoTOrSEKMD0qm8ZySMH2FWZsmQBQc4xxTWBhBJwXI6Z6Ur3KtYqLxT1bYcjrjGaiwR16d6ep4FMBC4HHfvSHBUEU3GZKnMKmMEOBkZwaAK5JJwOtbXhmAteXExH+riJB9DWSCEIIHzDoe1dD4fyum3ExH+scAfhSlsXDWSRe288YxSFTT8ignp7nFcvU9VGVrkbTXltbxjLGLCj1Oai1LSPsUUDoWYuCJMnoa2NVsJbTVLe9O0xxgAnHIPY1VvWuL6UW8Yy74IHQAf0rrhHQ8utJObaMF0JwEXkDGPWtC70Ypa28lspZgm+RieCe39aaLKWO9Ns4xIDgH3rYe9ENgbd2AIIOAcn86p6GKZypYFQTw2SCO4qVoyLbf/Eeg9as6lAoYSyOPObG5R0zTHaMwoT/AAgVAyqBgc1LB5aPvkPQEgdyalS2MgBII+boeDW94d0qGd5vtkO+EnA9FPTJP40Ac7Z3EsDyGEAyzAqO5APoKbI7yFI3PEePlPUGtKaaKy125a1hAiiyqgcjPrnvWaoea4Lk5ZyST70wSL+l6XJfs7lzFDH9+TA6+1dTDY3WnaJa2tlcJJLNMczIPug88etc7aytHp5t5pf3ckmyIBsbTnqfUV3Phe0S00ZLczRXDpISzIcgE9gaaEzA1Hw4wv7VZr2S4knjYyeYdoyMY9cDJ6Vq32iQ3V3BqIgBjtoDCYGGdu0Egg9+opmoaVd3urxzG4/dzExlBxiMdcenQVrW9wNMs0tLiRppEBVEAyzjPBP+NNoEziLTwxusGu79vJjaMyxqi4ZiOmTn0qqPD87IXktmQ/ZzMjKeHAxk/rXpFnaqR592iggELHnKxj0qXbaTxO8QQ5jMauvQA+n+eaVirnmOgWX2u+txcRhrUtiRj1XPTI64J4NekpG9rai3jAAQkKQMADBxx+NZWpSWWhw2k0ZB8kiFgijcQepIHXpUMfjXTpchbaYSE4jBxh/qe340CJ7TTUsrOwkW3JwJPMDAZDMRyfyrg9Z0g6Tevbm4DmQCRBtPQn9K7NfE8sd60N+IkhILRmJcgnHTPeqcd7bNdnXdQkQR3MbRRxhc4GOnv9fegDiWjMNwYgySMP7hyOnrT8SwbnPyhuCO+KSYWqSA2qOowMluCDznHtTTJv4JJHcGhAX9L+zl5IzGZJJAFQAZJOent9a66ytb3bF9osYI4YOY0kmAGe5OMk9uM1yli8Mls0T2xzGCBKoxyegruYb2Cw0+3N7p3lXEhCFYxkMcdR6HmnYDN8WXN+1tbobqQwyggLCgCkjp7kYz+Vc5IXH7+FTDCwEcjDuehH41097Os2pw3FvayiGJSGMowQT0IHtXNxySyFYiqyQLKznIwCT61DdgaMiNVyxB8vnAzViMRRA72WR2HGD096tW8kS5eUKVXIEXUA+o4q5Z39vpyTTPaCZbhCsfA+UnjnPTr+lS3cEULMjzgS4QgEgkcE+lGoI63IdUBwQQQuQR7ipp9F1G3ubW3eEp57gRynkH1z9K1tT0uwhkzc3ZEwjPmJGpAY4+XHtmmM5xIg7/ALxCGJAwAQKuQyfZ4XCkgRuCGAyQTwf5Vrp4bv7TSFv7h180ggRODkZ6AHpk1Wu41t7R4p4TbEzEkMckkKAQPx/nQBlSRkTbS+/jIbPf6VeR3jniaOfG0A4HQN3NNEKtEkc527gHjJ5IBOMGokm8mc5VGVB82O49qkbRZMNzci4CO1xJI4LBQScD6f8A1qcVlZzHOjhY5AC5GAoI4B/Kup8KaXDbRPe28zyCZQJAQAUbPQ4rVvNNt72GWOVBmQ5LDqDjANXYhowNAkhi1QKkpQlSsqMMA8jBB9M11qsD1rIt9JW2vTJGNyMihyRgkgY6/gDWngjsapDWhOVU9MH6UAYpIV3EAkD3NT7GRiCAffrRcCLFOA45p5j9qAuKQyJo+4qIg5Iq4AM8iopFGeBQBXAyacAQaUDBqUAYpiCM4ol+ZG9xQBziqFrcytqM0E/Cqx254yBSAhuwYreSeR1jYAgBjwSPWuCuriW9Yhz8u/IUdAa6rX9Sa1uTELpQQcleCAD0GCPauUlle7uRNGuAThm6ZJ6fyNMBiTmG+hWI4wQxPYHvW7DqyBJFFsGbzCzZOR+WK5+GGMqT5gWYZG0jnI7VZhhcQrLCdrSA+d3AGelIDdg1FbuWa1u2VYQN0Z6cVz+rmH7UTD5ZHBBj6EetatlZx3JO6ZY/LjIJA4x2zWAbcy3csaDiMkD2GaBkVwnzIsY5IyBV7RjCkkZcEEEsSBk8DGTWfMwjlEYOdmQfUZ4p8LiJ180kKQeh5OfSgC7flJb3y7Zy0LMDk9S3eqjf6oAj3Pt2rWsrVbjSoTJIYpImdiSmeDnGTms4x3FwyRRqCZCQoHGcd/pxQBXU/MCp6Vr6aZGnhgd/LXIZmYAng8D3p82mQ2FyriVWxgBADkHg557VZE322O4YIA0eCx6AgH9KBGrdw/Z7a3wGMaSnB3YJBHpVq0DG+81+JNuMdgPSs6zhlv4Le6UkqjnMZbIAHQAetaNvZ3ouvOEn7knJDDk/4UAam4ijcRTJZo4igdwpc4UHuaNwoAlEjetPWQ5BzUAcUeYO1AF8TZ71OkvA5rMDnPFTI59aQzT8wnvS1VSToDU4cEUIBdoPFRPGQSeDUpIHemswI4oAhBA61Irg8Zqu5+Y0qrkZzTQE7YOc8iqU0bbiVOR6VMS46mhRuPNO4WKS8yjIIxVvrT2jI+lRFsNTvcSQ8jNKFJpUIPWplAIpXGiMAikeMFcg8+lTlBiqzk5IyaQEOfWjPT2pDnNJ3xQA7eAKA/OaTApCMDikIkJBOTUTsADjvTGYio3YmmMQkZpMio2J5qMljmmIkLgGmFgelMOTTTmgBxNIDzQKcFOQaAFAJPFSJGcZoUYNSg4pgAQ+lNdSDircZTHNOKIeeDQBmNUZXNaBhUvgDrR9jU9CaLgZpBoANaZsAehyaiksZY+cZB9DRcCkBipVjDDNO8mQdVNSohA5pXCw2NAvWpwRjFKkYOM5qwIVxwKLgZ8sCkEjqKqlMdBW4LYMMY61VubXyjyBRcDLCk9qXYR2qcx46UjKQOlAEBBAqNhk1KQfSoyDTAzL0hbjnuBVbf6GrWogCRDjkgg1SbAHFSMcWHdqb5hAJPpTR1oYEg0AS6ZE93JKHdioBIOTx+GaZCxKDdVrQ/lnlHqP8agxgsPQmhgJg5p/bNGQBRSAaR82auRn92D6iqjHAzWhaxGWBSBwKpMGMFKBVkWpz0p4tT6VSJaKoWlxVsWp9Ketr6ii4IphG9KkEbHoK0FtUUckVKsMQIGRmi4FBYXx0p4hf0rTEKjpg/SnGLAyQKQGcLdyOlIIGHUVqCIEZxSiJfSgDM8kntUqw4HTmtAQA9BQYsdqVwKXk+1IIucYNXdg9KCvoKaYFJoiBkCo2BrTRQTgjioJbF92QRtPrVpktFA5FKGyattZKRgPg+lA06UHORj1q7omzGp92nc08RFBjrS7cVmywUDvQSM8HNAWnbRUjIjTRwalIyeKiYHdQAsgcDIFNUOxwBVyEggBqRgEOBjHtTuBXKkLk0wOOhq22GUjFVlgHmHJOPrTEIWqEqxNXBDH2JNNMYBxQmBlapAWsmIHSsWykKXabe5wa66aNZbSVD3HWuKjYpNgHBB4NO4jpWt5CcsDikmnSCIAwg+4FSRyTTQAhgM+tO2EqRKAT3ouFjLe5SSNtkbBu1JbyyBPnHHrV+WGPYQBg1UiOVZXAUjPSpuFhBfwwv8APnj2qvdapLMCI3WOM98c1h3V2ZZnAOMHFU2OCc0czCxsTOVG95vMz71lttJJCnk9aFbavBGKQEu2AMmpcmVYdGQrZKbvY1cilZVyIBzWczupIzini+uUAVZTj0IpFWNa3S2Zi80gjPZSar3caGbMDh1Izx2qpFdMXDSYJHcitm1i0+/jIkmKzZxtBwKLk2MRnCEg9RSxbmGSK07jRUt7gRuWIY4DE4FW20GdULCVGx2xijcLGBdSAqAAAR3FUy7Do3NW5hGWOWIIOMAVWYKBxQMaNx6mkJPqBRuq3Cr7Ttty4I5yOaAKYPOMj8KQ5AyatpZvIWeOPZjnB4qrKxztPUUAR5J69KUY7A0oQkEjt2qzaeQHxMSPQUWAaIjJHhFfJ9earlHBxx+dacrsELQOQgHAJ5rLZyTk9adgJTGcA7SfpUZBB64FMEjg8OQPQGnBdxAJwPWgCSKVo2yHJ/GteOe2nhAMas3qRWUI17MKcCUGAcUrgaRtERSYxF+BrMSdopiSgPpmms0mfv8A6UhPPPWi4CTTmSQsEC57ChJihyMj3xQcHoKaCCcHpSuBce8WSPlRn1FV1cE430wgEYAxThC2M5H50IDGLh1DE800HnNRg5FWgq/Zye4qgIC53DHenElcggc0zaSMgcVIqF1GAMCkMRScACiTIA96kEbdgTRLGTgAEY9aQhIYiYiGH0q5a3BjQRkAgdM9qiRztAPamgBZSR360AdLDrlxBArC6DbePLAxx7V1em36ajaCVeucEdxXncEwiYFkDgdRWjZ6vLZys9sAgY5ZTyDVJg0d2QM4PTuKp3uk2l+mDGI5B0deoNUtM1tr0iOdFR/7wPBrXLY6VZBiLe32hSpDeL51oTgOo5A/ya3IbmK5iWSFwykZyKZKUniMUyB1PUGueuoZtCnE9tI5t5DypGQOnFAHT5NLmqtrdR3tuJoW4PUHqDUhLDrTAsBwqlicADNVLdTczm7lBwDhFIwAPpThNGOHOR6DqacJmdCsSEjGMngVLQGNrM1yumOVYkTybR6Ae1c3pdpJa+IYYpBgg8H1HFdhqdtiyhGc7Zcn2FR3FvGl3LflQSkfBPYilYZz3jK63zpa+WvB3b+/TpXKkYHHereoXU17evNMcsTjPsKijZYzvaMPjselSUSRy3TwLEHO0HAUe9a72tzaCKO6ZHWUcKOSPrVM3EUgjcAKQckAdK1rq8j1CaHELmOFBkjkE/hRcDWsfES6XpospYXmmAIiWPnj0JxxVK/uJpkgkdUgvXzgDqQfWsaW7ksbxLuGMEYIIcVFBfXF9rVtPcHIZ9ox0Ge1MDodEtmEkl+5wVGzZGMk++a56/gKatPvQr8xOD1APSukkvE0e9uUYHBBeMDPJ/ya5m5lmuTNdMjHeck46e1SwGRyFZgw4xXRaTq1pppMpjcsRyVGcGudhi3jOT+NTQuQMxuQ2SOKm4yfWESbfqMheOSaXManpilnuZC6MWBYqBwetPOnXeqtHmSPy06knkD6fhVk2ltY6tb/AGWMyKowc8gn1oTE0b3hJIW0+SSRQZN5BJ6jiplnkt01GWMgNjIyMjNS21jNEMxL8r8gdh9Kt29lHI0itIXUHBBHBNWkFjzdre7dyTCw3nO4jANFxNGyoIxwnUe9dl4iUPCZlYAQnAQ9SMVyFxbILWOSMEbhkgnOKljMsuSxYnJx3pqSkEBSRn0qSaFwp2x4wMkk8VTB4oA0Q+4ZJAGMZqpIgWYkklQcZJ5pqHGeOP0ppyW5poC0hjZlRDkk4znpUgGxzHgAr6d6pZVSMHBoaR0kDBjk+tDQFsuQSQSD7VEwaUkFyCT1qBpmZ8569RThMM4LYH0oQE12tvGFEPzEAhieCTxzVZhwg9BT2UlQxGFPT3FMIpgOOC4IHGKCgfGTjHSiI4kB64GaGjkkJdjye1ACNjOO1dPp8Yh0S0QcmQb/AM65sRhUUsRz6V1FnNDdadb7JlikRduHGBkelTPY1pNJ3Y8ho8b1Zc9MgjP50+MebIqDqSKvx6hfqu2WGO6h6EkZ49MioL26ikeS4t7fyFjiIIxgE+tYxWp2e1VtC0q/2leSW92qqABjacBxTprZItYhEKBTHbnBHbniuZS9lE0V6jASJ1JOMikn1+4lu3uC4jcoI0xyAK7lNWsebJasJ78ROZmXzJgxycck1Tt5nkd3lVSz9TiqxfBxjqcknqTQszLIMVk2CLsjRlidisSMHIzVCXJdjg+Wp5IHA+tTRzMZwwAyW2nIzkmn3KyWd1cWp2uGxuI6e1TcBBcyYyGwx9utTpNdSxyJHLIFI+cBsA/WqbAkAqOM5FSSTCOONU4kHLH+VSNCpDmAAPsKkggnB60RxEWzyFAVQ4JNNZonUnGJGPX2o+do2QbirYyB3ouVcgfBxkZGeRnqK6rS9UubDSoxpswILHNuULEk+/UVz0URLrGEIz0BrqvCl1axx3FvuEbK2SGOMnnmrTIeprWmq6jqGUaxNjg4lkbkgegHrWhELewG5TI8h6t1djWa2swPO0yTA2yId52EjPr9KkhvXuIGa31CJwTkBY8kD061YkjC13UdQ1K/a0hhnsYxGXlWRsNIBzyBxj/Gs+1sriG3gununjueZERnOQo6H2rqJLjS1ZGebFzCCpaY847jPQisDWr21W4ea2mSaSQAFhyAMAAAfnUMaJJvEDXctuJrZDIACGRQS79B/Ol1GyubK2v7ma3twSAXkCEZJIGBk89ayNLt31K4iEcgheEEmRhxnqMf57UatNcSMYZtQa4j6nByAe1KwyrbaiVlt1PmGOFiRnnAIwalZorq5Cxl0t+SHccj/Cs4kJynT1rZ8NWtrqepGzu953qdhDYAI6/XtVIRizEGZ9hzHn5SepFOdStvESijIPIOSTx1rQv7K90+SOO6s5PLQlFcr15ploY45hMTGQuSI3TIx9O1AHRW97FFdW92bWaa3NuhI2kAlQBn3ANVde8Sz6o0UPkGGNPmwRgk9j9BUuk+KvL057CWwjkUKwBEm04PYDFSX9pFqml2N2HWOeUSFgw4wvHUdMY/WgDn/tkvUzzBx0bzDx+FEMkmBKJDgOASxyCT3xUSxhWfJDLjgjpU1rAZLcCNRuJOGHBqWgJ9VdIrtIYtuAATtXABPb3qzr+hDT9Ve3trh3UwiV93HJzwPyrLmhkeRxLv88nGCMk49KvTQ3F3Mslxdk3RUAxyjqPTPTpipsBtad4gt7vSbaxvo2Els6PFKhJLkE54/HFbvhbTo1uNQld2mjG2KMSDIK9e/wBa53w/aG11eKS4IhEYIMi8gk4wMY/ziu9EmOUYEeoFWgJzEBB5JAMY4CkcAZ4FcL4h8OalfaibiNCYXlOxBlioOMk44AzzXa+Y/rSqxJ5pAcVqGkTWU1lPdFSSpQIDwCBnP69Kz9Z0Sa1so7hXJeaTbsjHBBGcgj610/ig5azHoWP6CtiGNDaWxZQSiAgnqDgdKLDKehWQtdEtCYjFM8Q8wdyfU+prSABNL1FApiLEOFAFWUtkkbJ/lVEOARzVqG4AOCaBlkWcXpSmEdhSiYY60GZcYzSERsg5qu5wand+OKrOeaAEJpDzSE0zJzQhiFDmkIIqUEEe9NI7U0A3PqayJVL3YWYp5cWSSW5IPT+taNyyxwklsHoOO9c7rEFy8SOEbOQS2OR7/SgRi6lZ2q3C3UM7yo2TIM5IPXpUV3cpcRxorhdigknqcVs3dgkNrZ3UlvJkLibg4II/yfxrFvreJ0BgZQXBC4HIHHBFSBWugqpbuFZZACHJOenSokYvGEEhWMnDgenbAp8dvdmM+YCNw2DJJGfc9qrNCYkhEnysWJJ68dqpMDZmtpEimktGKiMBtvUuSeh9KyrSWaO/890JCEnB4GetaWmTPaWVw+0xx5A808kjHTHfr+tZUMpMcpDcHnGOhJoGQzHd+8dQGckkjoQTxVjHl2YBQFn5UnqKpSJJwTyC20HsTVyW3f7YIlBMmcDJyMUCLqXcr2X2ZECKPv5GSc8g4qna3axzeYZGXYhCkDOCa0TBNcQeSU8uYlYwQeMk461Z1ULeN9gsgiwooIKpgkrwR6nJpASR3om08zMpKqv7xyMFyOgHr3oTVreWz+zpblVGDIT/ABAZ4x9cVnyQXVnPaS3MbJbykKEc8HHQEfjmr9/YRh7qSOQSNIAMAdDxk/Tr9KYzOe/M4CxqYRnGEc4J9a6zw/cs0Lwk/u4idzls5J7VyccKR2qyxku0UgLAA4JPT+Rq7YTsNOuEUsDO/G0jI/zigRrIfteovJPzDb8pu4BJ4zitWO4jnJEbZx1ODimaboRh00NvJmlUElzwD2rRg06KFNq8HJJ+tICuFJ6U4IfSrQgA6UoQimMriM1IAw7VYUZ4NPEfFIRGoI61ICR0NBRhTSGFAxxc560ocAc1Fk96Qk+lMAZhnOacr4qI5pQaAJmYGkEmDTO1NoAmMhIxmon65peaQgEc0CBSRg5qVZCO9V6XeAMGgZbEmRwaY5yKhWUYoMo9aAEIHWmgc0Fh60Aj1pCHAUuCaaDTw3NMBvkk9qPsxPapw3rU4KlMUgKL2yYAAzUJtQTwwA9K1PKU9KY0AAyKYzJNoc9zUiWAIyTVwrg9KmjQY5FAFEaco6GmmzC9q0wFxSEoaBGYbYg9DSi3NaQCk0vlqe1MCgIAO5pwTHTP41YeMjtTQpPalcCHYcZxSgkVIQRTCBTAej881YyCvFUs4NSRyEcE0gJZIwYzgVUMYHatAHK1XkQZIxSGVwMVIHIph4OKaSSaYFkTADrzVS5kMr45OPSlBNOCgGkIq+VITgAj60GFx1FWycUFs0wKLQkjBGKYLct3FXGBJ4poUg8igDC1m3MKQuSOSRj/AD9KySMiul1yMNpZc9UYEfyrmx0oGMbgdORSLzz1p5AzTAMMaQEuluV1HaDwQc0+cbbmUf7RP51WtTt1KMepxV2+GL6T3ANMCu33sYpe+KGGcHuKCenHekDEboa39EjE2ntxna+KwSMjFdH4X+a0uU9GB/SmgNFLeNR0qQQIeookEiHI5FIJj3NO4hfsyelO+zxlcbcfSmecfWnLKT3oAT7Ko6A/nTTaoTkoCferG8HvR5gFAriKuMDFPBAPIpvmCkyCcigZNvGMDpSKoLZxSKo6k0/gDNIQpIVeBUZJI6UhkHrTTKPWmMYwIOaQtkU5nB60w4J4oQD0OGFWmKmPGKqA4qUMCuCaAGhAGqUMcYz2qMDB65qUdKGwsMKj0qJkBOcVOOvNP2oetFwaKJGDigGrjomMAVXMJ5Ip3FYYQD0phTmpMY61EW+bikMcARShTnOKVWFSBgaVwBAoHNJIF/hPWndabtz0ouBGIwDnJocA9aViQcU3cD1ouIglaUErGcL61x17G8F5KrDBz0rtgUPG6uW8QQiK9VlPDDFUmFjY0qYTWCkfw9auSbXTjqKx/DkrGOWEnkHOPzrYIAJABBHcikMo3A+XFVFTLYJAB6gnBqW+82ZXjACsBwQKxbK4dnJdiTnGTQBDe6PM+oE2cW5Dz1xk0S6XdBQJbcA/7JziuggBRA8hwKJ7uziIdpCB7UDsczMbi1gIaKMx4I5XmspSRXYanDZ31gZYjuAOTx2rlREjPtjYD0BpWFYIxlgGT8WHFXzYmWPEbQZPoeapTyu2UkORTdzRRAxgj1zRYZE6vBIVcYINN890YMpwR3AqMs8jEs5J9zTCcHFJoTLLX1xIRm4c/jUpv70RbPtEm30zVJAO9XbdoiSHOKaArbjTTzVmSENIdnI9abJZyxoZCDgdc0AVQ0iHcByOlbWna+bcFbiHI4wV6jrWSuCOTU6SIi42A/WgDZmvYdRRo4GEZPXdwc1hXFobZgC+7PQ5pjlQxO3rTHcu2SaALlshKZVc81FPGVkO4YPcUkMjRnIbHtSSTM0pyeKYEZkPQEgfSmEZ/iFTo8aKTvIPcYqLzELbiAfwpANKAAHIoD+4FNZwTx0pmaaYEhz/AHj+Bo38AZJ+tM3AdakMfy7hSsALKScGlLjPOBTAOaCB3FFgJlA7MKQoCc5qENil8wilYB546UDOOasWMIuWIY496nuLdYcqCpx3zRYDmFRww4p0rYyuatiPyoPMlO2Q9B3qrIEI3AHPc1QCRn5SnrU8ccseBsJB7ioY4iyFh2rbCeZAhTkgYxQBnbtnfFBl3d81ppbwqoNyqLn05p9/GEtk8qFFjyAGYcn6VIzJOcZwBTCRnNSNtBwSD9KaYwTkUWAFI65qRWz0Ix6imCFgO351PHDgZJAqWAnzj7jMPxrstBvZLi02zkblzjAxxXKqEBHf6VbtJSJBGjlfQDvTTJaO1I70NHHLGY5EDKexrGzNa4eQNj0JrZjmTyBJwV9SK2TJOcvbW40C5W4tCTbtjKg5B+tb1lJFqEAkWaQjuoOADUEt+JZDbwwC4LcHPABrnJrO70m6Xzy8cMh+YxNxj0qgOskms7YnfKgPoDk/lUY1W3GPKSaUeqocCpLWCyECyW8YYMM7jyasqcdh+QpDK0eo2N5mAsQT1RwQTXM+K9UaOQ6dbnCDBLdyOOK1/EV7ZW9tsuraR2IzHIgAAPHeuAcmWQuSST3JyaTKGkYGT3pSmE3GnEFgoHY5qcQNcXcVun8R5qGxmj4S05NS1SRJVzCE+Y9xnpXQXOi3mhbzbYurdzuKNwR7Co9OWy0TXkSNljUwjeSeM+/51vay0l1atDCQFKli/oBzxSYjzzUpHvLh2YCNegQH0qa1gZLEOB80bgg1JaQxvpdzO3zSQknHY09LlAIZiMwsdrgf0qUM6Qvb3U8TBkMxjG0kjIPrXNXgaC5mtzMJMnJIAAB9KXXi1veRzQuUcgAY647c1bk0R2s43MxMk2DwMkn60ybHOyOAxVQQo9DUls3zDI4Heta/sYbf7PZZHmOfnI5Jqld2yWVy8CElR1J6mk0Ui1bzA3gtwQu8cHPOf61vS2P7uJIshhwQD1FZpt/tGipLbwqjDqzdcZrSg1a3SxRZJcXCjBOM00JliS8n08RgZOzqCSTgev50v9tQrBMFctJKSQFGdvsaxBqdw9u7lCxOQXI4FQNO9lp4mG0yOeO/FNgP1K/ea4Bz8wUDaf61nTTuyBCAFB4AFQyTGQ785J601pdwAx0qHqUmOuGR4TGxwTWW8PljaB071PO+ZDz0qEODnfk+mKaExhbaowDjNKZMx4B4PWmsRnB6VGSAuB1qgFJyaATuBPJpv1p4xgj1oAaT1wOTT1BUhh1HPNNwN3FBfBx2oAk2uxMhIPrigYJx2pYwWjPPSmDAGSaALMamGVHBOMYOB2prkGUnqDSxyRNGSSQQMD0qLOFx1PrQBZjtzNIkajLg5C9jXR6bbLIBvjAwSCAOAa5y0lUTEOWGRgFeua63TBiJPdST9aLCMzXblrK7RLaQxErk7SRnnvWdLrF3cW5jkmH1CjJpuuyedrEwxjAA6+grNTIaixXMThnWJFbscimh13AuMj0okEmcuGAI4J706G2lmkARMsQSBnrilYl6ixlHkIK7ienOMGpwse5geMA1LZQ29vBcXFxHu2nAQjJ5quCpV33ryCcE8imCAogBCnqc0hATksCx6nvQHgeKQkESE/ID6DvTJY2UjcMZ6e4pWAUtuTaD06e1Ml+Q5OCSOx60hZVP3TnsQOKdKQUCZBJPUdqGhk8yi3hjAAMjjPPanPIjQQkDEhOCvcY6VBMN0gOScDHJzT4JFAKlVyBkDHUUmgJmUzSApkTD7ozxk471Atvcb5F8r94pJzngmtQBbgRrHtiQjDeoNWHt8SyloRGvlgYJ++RyTimgRUknmSyNrNKpZiBkclR3B+v9Ke2oXtoPKE6pgAgKgBIPv+FBmjjsBK8aSyNIV2k4IH41QmmM93HIEYKvUEg49adx2NyzhhvIjcTLuVRuYOeAQf1rJ1aC0OpNLbyLHGwBCr0z3AHap7eGa4cRBXRJCSHIwCBz361RuHjP2dA2SgJLY5Jz0oRNh3myi2MSSExAZYJ2HqasWENrdW04ZGEwTCgEgFicA1a0S2s7wXEclwke8ruBOAEzk4PvVHU4bayu9tpcJMmSSYycAZ4BqQM5mkhlaORRuU4IPY1r6FBaXF6TPdm0kQ5ikQc55z3rMS3NxFc3DucxkcY5Ofeux8Daeima4laBxKm1EJBbGeeD+FUhkN5a3us2FxdT6o7iyIAUjIKnqeO/HP0rlgmGIKvj+EgYB/wr0yy01LTR9ThnXCSOxK99uDgAj1FcTJpd+1teuATbWmFYtxgYBA+oyKYio2k340r+0xb/AOigkeYCOCDg8eme9a3h3XWhMdhceUYXBjDZ5BOf0zXeaXDHfeEo7S7iEazRFHUDB9M/WvLNX0uHS9WurW2kMkcGMseoPvQBq3ltFDq8Om+UNqxEZGRuJ5yc/SqE2nyW155cKOwCjDg5Bz1qudSuJ3jM0jEhdnmAfMR6E1JJcSrZoGnO9CAu0449KlgWrRQt5HcSPkcgHOcEjAqi3ni5MUwDYcguRxn1qW3kAlSOPkylQc8gEnqK15rWGxa6MkMhkhjwd54LnuP896VgKMSvFGQt05DkFlB4yDkV31hrllehIg2LrADpjkmuAD+ZZOEXMhkL4AwdoA5P51b8OrenWoFihYknzDv4G3oT+tCGeih8jjP0I5FODUpweQKZkZqxGD4mObqzHpGT+tb0Rxbxf7i/yFc54iOdTth6RH+ZroUP7mMeiAfpTsMm3YFMMgFNySKaVzSEPEmTT1lx3xVYgigEjvQFzSjuMqOakEoPU1mByKkEx9aTQ7mj5gxyaidwTwar+aPWm780rATg5NO25qFTzUofimAhBBoJOaUnIpjcAk9AMmgBSAetUbiK4IlmkIGw5TDcEehpkOr2khYCYYGME8ZqK/vIb2N7ASEGRcrInO0jkUCKlzd3s91FGqARAgMobIJ9fwxUt/pdi8hd4cTAkfuxgkdyfWsLw/eSafNm4kJBlWPBIypBOf5jmuzkginO7Oc/xKaAOb0i3t3t7y1muBIJRgAgggjOD9RXP3UNwEeNk23acMjHIIxgEV362cEUuxYVKEcgjJJ9c1y/iyL7Bd6dcW7sWUkbW6ADBx755oQGHLNs0qGAFj5hG8dSAODUl1pUlrAsxGI3IIjU5BHBP6VNLqdld6gSu5Y3BJ3DGD6fp1qS51KSeCAzFUCRkDaOfTOM0DMG6mH2SO3HSKQkHuc9K3IrD7J4fuNRlmDyuAoB4IwR096yvKR9SsYyOWZQx9RngflWrqUptJRC0jCOM4yRlQCe/wCGaQi9pGnSTXI1K4dUiMYmCHnA5ABP0BPtkUwmO71Z5tNmKw5LXEzDhAeeCeuearxWFzdQILVpAsuQ4JyFXOAfx/pUFtam0uLiEyeZbogeSQHAPpx3PtTA0JH/ALSYx2CmTapMk0j5OB39qq+aYJQAsuQ5y7DAJ9q1tC1K3GmXVvEh3iMbpJAAMnjORV+ztNNeM3X2pLiaLOSDxn6GgZmx3MNnbSA25GSeScljWNcBzcnzmjhaQ5VUBOQeh967CaB7me2iRVEyEs7Dogx3/P8ASnxeHrT7Ul00k0k0ZOTIeDxjp2pgYRtNWGCNRkNuSASAQSMDt+nNdlFJmCMnqRyagnvLe1Jh3KhHXI4H1NYIuNTe9mntyZ7cgAO/CgZ7Af56UgOmBzTgOaqpd25nMCzK8gGSoOTToZZfn8wY54+lMC0Bg1MoyKqiQk9alDkDikBNgGmOBnBoDc80NgnOaQELACjg9DTmBNQkEUwBgMmmHFKTkc0w9aBDwanjwWqtT1k20AWGGcioihxjHNAmBPNKZR60AVWRgSCKYQR1qd5ATULNmgZGwJ74pASO5NKTSZpiHBz3NOVx2NREikyM8GgC1uyOKFcA8mq4fB6mnbsnrSGWhIMdakWTmqi54qdKVgLaSU8ycVACAKN3vQAMwJzmgzADrUTuMYzUJOaYFj7SBTDcc9arFhQOeaALInIOc1Itxz1qoKcKBF/zMjk0nmKO9VQzY4NNYOQaEMklmBHBpEcMOKr+U561IilBg0API5oGQaaWzRvoETrIQKR3JOc1DvxSGTNACse9NBpC2abyaAJVIzTiQckVXJI4NKkmDQApJBpu81KcMMConUgZoAUSYp3mAjmqxJ603caADVsNpE4HYA/rXLjpXSXRL2Fwh7oa5scqDQA1iFGW4FICHG4dO2akIB4I4pmAAQAB9KQEAO27hf8A2gK09UGL2M+q4/Wsyb5AHxnaQcVo37iSO3mHOQRn16UDIuCPwpuR0pu896ZuJY80FEhOBXQeDX/0q7iI6rnNc47YFa/hafydaKk8SIQaBHZPGCCCOKiAjTgIB7095QSeajZgRxTRNipfRshEkakg9cdqqpORwTg1rRy4GGNULy0zuliHHp3q0SyIznsalikL9TUcCxzRbSMyVaS3CLgVTEIaTkdKlMZFAX1rO5YiFycCnuHC9OKcAAeBTiCwwKQikXOcU+JGkOOR70ohIlBI4q8oGBxTGRfYiQD5g/GkNsUByQfpUxJBxQX4IxSEUHcqcUscwPU0y4BzxVYsQaYzRVwakDA9Kz1lA6GpRMRQ0BaY8VGGweTioTMfWozNkmkBbJwM7x+dRmfb3qqZCe9NJJpgTyTBgSDUQcnvUBYjpSbyO9AFpXPrUyMTVHzAgLE4A61k3HitYZCltCJMcFmJAz7UgOpGaepOOSD9K8+vPEupXY2q6RKeycH86yVkuEIIuJAR/tmmB6pKNwxVZjt4NcNa+J9UtJAZJTNGP4XPX8a6vSdXt9WU7UKyAZIJz/npQBY2gSZGay/EUJeBJh1UgVslcGqGrIZdPkA7DNAGR4dnWHUlDnhhiuvkCdxXnlrN5UsTk4APNdykyzwpIrAgjsapgVrxQI2dQAR6CuFefdqwK8ZfoOld9dITbvjpjBPpXFG0jtb0STENGx6kVIGnf6tcwWoxbovOMgmsAXDzyfOck96n1i6V3C28u6EgH1qpZbTcKzEgD0GaQXNyW8S300xxjJwRgiuceRmYkKBn0rbmltScu5x7cVWlkskUtGWJPtkUMDIdnHJBP0ppmfGMmtCeAInmFlKnstZ8iqTlcgHsaLgM3E8UnQ8ilIx16U07MYBOe+aYEm/HSni6kCBAAAO+Oar55xS0hDxK4bIYgn0NO82VhguSPQmoQTmlB9TQUPzgdeacHGAenrUZYBeOTTQ3HNIRMWB7g00kjuPxqPeKcrBjg80AOD5OMilY4981GYwOQTSZI7mhDH4yaRkIGTim5I6U4BpOmPxNMQzBPSpBE5GcU5YtrDcRgntV9fLVAATn2FS3YDOjB3gqASPWrru8seCij3ApxYDoAPwoByMGp5wKQjOcHApskZBzjP0q/IUMeFA3VAW2IXwCKfMBT5HUH8qXPsfyqdb0n7oH4ioWIJJPUmmncASR4zlWINK0rNnJyTUZBPQZFOCMf4TVgOWIXtk9zMWLAZAXoKis7aaSF3Xbgeo612CXdhJZuIUBGPuhMfrXPxQyxWUzgYBzgdxTsBjqTkjjg9ulKCx4LsB6A4qSGBihba2CewqIjDkYIPuKQFywBSQyMQ20ZAc5ya1n1ES/IYQoPdhxXP5IAAOKsLIJB88zZoAnubxhKcRxEdiB1qvFGXGanEixjAwfqKaJADwKljHEgfKQPwoJBxg44pm4E0jMAMmsrASAgGp7WQRXSSN0Gc1RMwHABJPTApu+VhgjAosB1Fzqkbr883mE9EUVb0GwnvFE1zdGOLOBADyfr+lcrZyra3Ak25b1JzXSWBVZw+SNx4IOKtMk7GO3hiTEUagfSq91p0N9btDMvB6EdQaejT5CkikuILiaPyzK8YPUocH861jITRykF+PDd5LY37sIcZQ4zx9PeorvxpBHIBb2rSr6ucZ/CtDWvC0JsmuIWme4Tkl3ySPpWHd3Fhd6I5k2rdpwE6HOau9ybFfXfEn9rW8cEcTRxjlgTnnjj9Kww2BTDwcUmalstInEhAGDgk4rprPTv7NhN9LJvk25yOn+ea5VAZCFUEn0Aqybu6aIQSzFoQeUzjI9M1K1GyzcPN9nku5kYGZuDjgH0rprTWBJ4Ukdj5bKDGTknINcvq2pi7iigiiMcMYAAJzmtN9Nu4fDGGeMIxBKAYJ+tFiTCjknKS20ErmNyQQDgEe9TWnmTSW9kxCIZOcDkV0eiaX/AGeHkeNXyvJByRkdMf56VYtdNS51ZVjjVA4yzFcsBQ0O4eIrWJNJVwvzIwAOeQKsWl2502HZGHMadSeAa0b/AEZLnS5LOOQnyyXDSHJyPU/jVLSbWJdEKW/Mv3SO+c9qTQJGH9keVnvbgB5OoAOBmktrMJIbq5kAJOdmMmukXw/KbJnuLgoAM4AxzXOTWcxjcllMYOM55NJspFZruUgjzHWHJOw9KrvKzIUJ+Q9SOtWJFhRPLKfMPeqkwEYIUfIMYPv3pJiaLtjcRiOS1IJBBJJP9KoxTkXCsQzImQR1FRxu4dnRsEDrVuPSZRbGeaTy0OSR3/GqbuOxWcoXJQEITwPSo3IAOOoFPEyBBhAAByeuahd1JJHIJ4qLDRTJOSecnrmm81MEBPbJPGacYdhII+lUhFUgkDjknFNyQcEcitKGONFJI+bGQfSqkkY2h+Mn0FMCILlN56ZpAeaQDnFAwDyM0xgetIQaccE8dKTigQ9XK8A8HqKRuTmnIAxANEgwB9cUAEZ4I9qAMJyefSlCNt4U/UipUTCFnYDHQE9aAL8ciSTJGhyNvSupsh+7Q+gxXJ6dC5uUkA+Xpmurx5dnJ7If5UCOLvJDJqE0p/ic4PqKZGMzKT0ByRRKXKxbmyoBwMdMmnx7FtC2MMW6+tAE5uPNs3t5OdrFgc9/pUSuwAbJAUYOOpqozkE4PWrcTmFCHAPmrgE9R9KloZA7K2NuRjvnk0sQbzVygcA52nofakgt3uJ/KjIB65JwAK07OwiNhNeTTKpjJQKRgAjpg0ICC7uRcTA7PLAGAO9MJjDpvbCg5FVpCWto3wRIxJLHoQPSnyI6pG8hwHHHrx1piLDlFQFIyhJI2jgMPWot4LkrgkjA4xg0rCRlIG8kDOT2quoYBiDj6UMCyP3cLiQgsxyMHpUQEoYSEDnoRxmnqBNEIiP3nQN3JqWQFikRj5ACkikwRoadHGbOa4lQSbCCVU9T75Paqs1xNbXxlkjG/sCcginG0RAYxK2707VajB+zQO2A3mGMsSMknpxUgjNuJo3cyAYyASPeo/NVHRvvcZYdOK24LK2nsL8yRgCHJMncHsPx9K56NPMmSPONxxn0qkyrnceHp5r3Q5rcglomKIZOMgjI57VxdxEYLiSFv9YjEH0zWnYXl0+tRwpdoCXAQk4jJA4BqxfaDqRu7m6uIASWLkRsMEY5wfw9KolmFFI8ZJzg9j71qx6mEiIEMZ3x+W+Rkt7+1Saj4durKza52BoQgkLZ5XPY+v1qjFtaQI+Yg2Mccg570mJHRad4e8/T70zPh3I2BDyCADg+uc10NnoOnW4ikERWZOdyMQQe/rVmBSUCgElQAWI68dalmmhtLaS4uJBHHGMkmgCtrPlTWzk3GxQCUjJwJGB6+/XpUEtmIrC7sLggQ3oDiReCpABOfpgc1yWuaxHqV+GsQ4jCFFLjBz1JArpPEkl1f+HybGRBapGHmkBzuAGNo/Hk0xpkWheJkttEnF/OWkhLJCQuQwA4GfWuFkaWPzFkzukOWJOSRVwR3CaMYJrYrCZQ4lHUEgfzFX/DemxatqiSO0Y8liGDtgyHGRikM57OML7VoaWY45JHmRHjwAQzEY9x69/zroPEfhpLSyuL7exmL5YqcRxg4AAHfPH0rmrCxfUbmG0RwhlbaGIyAfpQB22i6DbyXEWoB4pY1+ZFj6bu3HbFa2s2kV5Zl5lB8oEgY+8TgDNO0dJE0uETxhLhR5chAA3Y4B49qvgA4zzgg0AY3hrw5bx3l1JcQgGOJITH1DEgEk++eOK6NdNtoZ/OhjCMECLt7KOwohPltkcE9ferPmd6VgKcoIJzUWCTjFXGKSDk81GyoBxVCOV8RJs1m3XOcQA/qa6JFxGn+6P5Vz3iE7tdi9oAP1NdKSAqD0UfypgMCkdqQggcipFYd6bcRecgAOCDkc0IRCWpBUJMiHDqQPU9KerkgAGrsTckxxR3oyQKaTzWbRY/JpwJJqLNIWIPBNAFpTipk5rDv9bt9M8sTlmaToqDJx71z9x4yvS8wijRYSMRgggqPXI60hnZT6rptrJ5c15Ej5wQSOKJL+zkJiS7hZiMABhz9K8qkm81zI5yxOSfU1GZEpXA7S9gsbB/3gwzZI3cqOwz6VkXGrPFOFExcIQQyHgmsRrp2wHJYe5zTCfMOelK4Fqa88+Qu5GSSTgepq5Y67e2TxiK5kEan7vXisVwcggUisQwBGM0gO1h1ubVr0wB3VCMoQcEEdc1DqUpikhaSRppIjghxwxPp71zELOHBRyjKcgjqDXQprEVraxpcQJPcEZdzyAOxoTAoORbXjyOg8yQ5XA4Q+mD1pt7dG6HmPCFkGASg/MVNfaol+GBhRSVxkZyCMYx+VNs3iurb7PKCZEBIfOMk54NO4GfMxhmiIbJA3A+hqS6u5J7RIycqSTvPVj6moDCzXiI7nAIGTzjFNBeecAdA2AB0A9qdwOnsJ7cadNG0jSPwxJJBIAwPw68VnNNcQ2VwkcR2yuE3Ec4HapRZzQ3MSRptEwClc5yf6VNrUD6egQyclPMHOcc4x+tAjJE7QxGOEHa+C4IxkjOK6PTbyP+z5QQsSgcYGC5POB9K5m2jN3cxqSMA9Cep9K2tD06Wcx3MkapDG+VyeCR1P8AMUxnS6UJLaJb24MhupSMx9MYzgfrTjqgjt2RA/nSOSWJyAM/5FR3GoWszAecY3IwhPB/D61nS27qyvGUJOdyjkgUxGgLiC8upXdwDzgHoDjH9DQkN19jMcE0IhBJJZCTiqk1vemMfZ4UZSATnHBPatWwjeG3CSEkj1OTRYLlaDTo4r3zYXKSKQXOfvnuAOw/xrYzmogBnNODdqBkganhyO9QE4NG4nvRYLlxZB0zSmTjrVHcfWnBuBzSsItGX3qNpU7nmoy+e9RuFYEnr9adgHGUZxTvMX1qsQB3pN3vTsK5a3jb1pu4Gq4Y09eRSGTZwaUtwcVHn3o59eKQxjuc0wvTnBNQkHNOwhS/vTS+O9RnOeKaSRQBJv560u7A61CDk0pyD7UATh896USbepqsJCKVXJFAF1JRjOalS4UnANZwJxxQCQcg80AbHnDHWopJ/lODzVETHGCaaZD60WAnEsnJJ4FPMpMZwefSqZkOCM0wSkdDRYC7GSwJPWpNwUYNZplYdDSGVj1JosFy89wAQAactxwATWaZDnrThJ70WC5sJMCM5p3nCspZ8DGalE2eppWGmaPmLTSc1UEwz1pTcKOpp2EWCajLEdDUD3K4wDUP2j1NFgLRkI6mk833qo02QSDUfm0wL4l5608Se9Z4l96eJfegC7vzxgUw5B4qFXzUgbIpWAmSQg8mptwI5qoDzUgekA4oDzUZjJ6CnbjQHIoAabdnRx0ypH6VyY6Y/wA9a7SFyWxmuNkKLPIhbkMQR+NA0HakIpxxTSR60gKtwMRmtC6+fT7dh2AFUbghkIBqeEyPpWZJAwU4AHYUwGYPfFIBzmmiQdzT9wI4RyPUA0gHcVNpMpi1q3I7nH51XKzMMLDIQe4U1YsLC8e9jk8l4whBLMMACqsI7cggnPrTCcUwzEtnNBkGOaLAOHXFToBjFVkddw5qYOOxosBKY0ByqgE9TSdKaJBnk08EEUgADNOEfrSEgDIoWZe9SA7YPSpAmBTfMUjINAlFUApA7im89jSO4I4qBpcHg4oTAmJIGW6VG0oAyKga4JBBNQmXPQ0APkbcDVOTIJqcPnrTJRkEimgK4kweTTxKR0NQlSTSyGK2iMk0gQAd+9MRP5hPWlFcpd+JLgyFYURF9R1otvE92vEixuB0yME/jSGdcsRbGBUwt2xyK5NPGdxGCHso3PYgkUh8dXasM2EYT0yc/nSA6swg9qikiSMb5GVFHcnFUk8QwXenSTQHy5AMlWPIP5VxF/f3d+5e4mZgOg6ACgDe1fXI5FNpZkNu4Mh6D2rCNldLywAHqDmqSOU6VeTU7ssAQGz6ii4FmTQrlLcTC4jIxkgA8VnsZImwTWjLc3UYHz5UjO0DAplxPa3CorPtYdAwwPzoAgElwgyVIU9wKm0a8Nnq0bh9qucMCMZFaosIrjSo2yFYdCemao6yJo7GJJo4T0+dDQmB28NzBeAmGVXI6gHOKSeImF1K5BHIrzmyupbJ/Ot5jG3sM059Sv5JjIbybJ9GIFNAWZFKTyRkYAPFdhpqC4slcMFbGCcYzXGJyAx5J6nua3NIvZVjMCn3FUwN7LIpRjx6g1wuteZDfGN2LL2B/Cuimll83PmEMOmKzNXkW/hEboFkTox4/OpAy7ZHEJcxAgAnBNW49ciSHmFSw7bcVWSC9trRywDwgdjzWQCSSffNIC3dXX2qcyYAz2HQVBmTshx64qM5ByCKkNzLsxmgBC7A4PB96VZMrggVEXJPJyaUIW6c0ALwwOSB9ajKkHinEFTgjkUnNMBVEZ5fOe2KaRz7U8DJxTXGw4yaADPakNS+XHsB83kjOPSoSOCaQ0HWkNJuIPtTwuQDnrTYiPODzTgTnihwAevPoRRg4yKAHAnNO3gds0wAEcn8qM4PGT9BRYB5IJ+6KurZoYQ4YnPYDGKpBZH6IT+FXbW9kePyQinHSgB/2eHGSDke9RmMjOwFqdKWXlsAnsKgLuBkGsWgJCXKnKkEetMMuBt6UnnykEMufTFRlS1NLuBYiQOQDyDV3yIwANvPvWdbsUkAJ4rTySMmqsgI/IhHSNfypfLjHSNR+FOooWgCAKBgIv5UmB6D8qcFJ5ApCCDgiqAq/wBtyBNsMEYPrVaa8uZ1IbAU9QOBWerbTmnGZyCAetAGnaxE22RKYxnAAGcmonspAd0u4E8jI6itTRXeayIKgbDjI703US0Yw2CScAg9KAMyKNmyIo/MI6irUJt1AjuoFEg96bbXAtAGJwD19annubOVS4Ukn+KgB08duYC67Sw6Aday5tgfDEg+1T+WhmzE24ntnpVn7HP1KIPqc0mIy2VVX5MAn1PU0iwkpud8n0qWX96WLAbgcDFNDEKAe1SxsaFOfT3pAGHfcacz8YA/OmNkckn8KQDgGJ6VqW+pSKoR1IA7jmsuNue9TeYR0oQHUaRqka3qxmctvIGXOAMV3Lxl+VGR615TYhbq4WF0AB7jrXSJNdWWDYXrP6xyHIIq0x2OvELfxDC968m1uGOPW7pIiNgPAB9q6m/8WlrJ4bqCSC4YYCqDgj1z+NcQhJkLE5JOcnrVpiSIvJL3GccUy4CJ8uOatyDCk1F9ncQi8dAY1PfkGpRVid9OuLeCOeAhmbqB1Aq9otnCwmS8izJz8pGCPoaTSb6KS62XBCgn93k4APpXaiAmFQHiDkZBCjkU0Jo4qPSTLqCeXC5jUgsDzgCuh1aMTWoAcgIQMDp+NNhZdPkkMkuSSQyA5GfUVViuY55BGDITnJAGaZm0a2kiOO3/AHeTJkAknqMVZhmRLiaSSRAzjAI4OapQiR3CQkID0UDJqa5thbwpIib7pzhQTxn1oKL00kdlo1wJJFyQe+O9Zvhq5RNOuXkYAAZBJ5PrXL6wlwl06TXYlkJyUXoPrVMO7x7FaQkdgeKTYJnaxazNf2TQtZyeS2QXQ5yKx9TvZLUiGJCgIyQTyDUml3kraV5C4QxkZIHJBP6VH4ot0huIHVh8yAEE8jpUMLmE82AS5+arKlJdIAAwSQPpVG4jDfMasWyyxKu5QyE4CnpTSHckeGNLdASBjuepqO4ucRgGRyD2JqS63XFyscQztXJHp61TuwWMMQ6cDPpmgdyNiSvFGVKgbcYGM5q3e2hsbiFJGBRgOg6VHcCAWxaNiWJAAPUUNAVCdr+o7ZHWnls8nvTGckAEjA7EU5SjDJ69qEBKHUwkkAEHgA8moJA0uGJGMYAHagMCuSRmmF8UhkTRlRkioiMmppJCU4PeoSaoQdBSd6UAkYoIweTTGOVsHNKWJNLDE9zOkKDLMcCtMeH7rvcQj8TUykoq7dgsZgdvXinFiVIrUPhy4HJu4gPUg0o8OzdryIg9wDis/bU+4+Vk2jlmKjqB0/EVsarL5GjTsDgkAD8TVLTbEWAwZllIOTgYxVrUI/t1qINwQZySec0e3p9w5Gci2WQOOmMCgyAwhB0BrZXQtvS7A+iCmnQAxybsfgoFHt6fcfs2YZq+l55lsIWRQFGAQOc1Vmi8qd4gc7TjOMZqaxPl3ADRqwYEAkdD6itTM0NG0ua/Z3hkKSIwUEDI5zyR+FXNa0240h4obgwTqSSpTjJ46g9K2vDOr6fpmiXAVM3qtkoB80hJwMe2e3asXULC9ja4nvow00imXzN2cDPQfnQBVsrq3HlR3aLGsJyMJkEn19KLrZcaTcXQCvMLrGe4QZwAO3NNM8SWwhktg0jgEN6j3FJ9ntU04Jv8u6kmwQegT1NAiORHNujvHMsk4wjEYUn0FVY1Ku0LDDjII9627w30mmW1uFhuY4T5iPCclR3zWK2JLhpicFzn8aALemqIb5yUJbHy46ZpIpbgXT7CoYkgk+lWNFspLy4eMXKR9iCPmYeg5q3qlnDbz2kSEGOMYY4wc/nUsZiIBJcgyiR8H/lnwT9K6G4srKFILoIRI43AE5BPof8AGq1lCkd28gBHlnABGCB3rbvYhe2UM0UWPKmwQ3IANKw0YsY0uOOWTUIXY7wpiRiATzk4/L1qBdIhudAu9Rhk2R2shRSFy0mSMZz2AIrs7TQLcXVubiASCBCCxUBWJ6cd8VbXQLKHS72yiDbLjLMCcgHtinEGcFpOiPfaYb9XxJHcKqIV4Ykjpz7/AKVr63r1xbXs1vbx4hRgJhIOHPcfSt7wWVXTprWSPLQTZwR0yOMflWrd6TaXUFzE8YH2gguwHJIORViKd/bDVNJhhUgRzFDIO2zuPpxWdP4btbnVZLiVd0bAFQpwAQMAVvJaiCNY4uI1AAHYfSnpGec8UwGZAAUDAAAA/Cqt5bR3MWHjWQjlQwyAfXFXni7ioiCD0oEefXdlFYayJp8LCVd0yeHI4HHuSTV7wqkl/wCHNRswMK82S3oCOf5V1l1YW10sck8YcRA4B5AGO4rn/B9vIlvqFowa3kmAdCeDggjIqgMfWdRMdjZaTZwjy3AZ435LHPHPv1p7+G9RsIIZIkJuklMhRACIyOQc9+lN1opa+KYoxDu8poQSRyQOpA9816HMnzg+ozSAyZS+vaBbLcoI1lKtcAnAIByemMZx+tcvoi2EOsxSQgQ263TFJJDgEAdMmuyvbaSXTZ4LdAXdCqqTgHPUZ7Vwd7BNpF0rs0O+GTcYAdyoT796TQ7no3uOQeh9RQKz9H1caxYLMVCTLxIo7H1/Gr2/mlYROpOKtDBUfSqiMCKlM2EwKGhpg4UEkE5qEsSRSeYDnNLuB70IDmdZ+fxAfaNR+tdEX6fQfyFc5ftv8RTj+6FH8q6E0wHB/eplkwKpZINPEmKQF7AdcEAg9jVe4h5DIAMcY6UCbAxml80t3qkxMCAVqJkPYU/OKoeINW/sjSGmXBkc7E9QT3oYXK13r1paSXMed7wgEAEfMe4H0rIi8WmabEyGFD1I5Ncn5xI4GAevuaa0hB4rMZe1S7F9dGaNWzk5JOSQen5Cs9nlHGCfoKcbkgcmkFwc8nApDTGxoztyMgiphAo6N0OOtRST5HBx9BTVJKc9O1FgJiEBBJBx2Bp4dCAOhx1qsWz0z+Ap0RKliccjA4osBN0IBBPHWmsMoRzkdKXzF7mlMoHBwAfTvUgMAkU9OMU7cQMnHPbFKsinp/8AXpwdT9fWhDIxJzjBFOzk5DGnkpntk+tRTAqA6YBzyM0wJRnOcnNaNhFDIRvYZTnngE+hrJEzcZqzbzSB9sbhN/BJ6YppiOnWZYrc6gRmOIlUAOOfYVi3PnavcK5OC4O0d1A/x5rprewKWkaySpISvBByMc8D/Gq5Flp6AyGMbyDkqAVVTg1SEc+tsY5UiX93cDnB5I9/wFayXsMFtJG1wpaGMBSX5YemMVl6lcGO7+1w2+xZgRHkYAAwAfr6/WqSpbG3JdZBJ6gd/WhjuXrO7jm1eJ5pXihJ4bJJA/LjNdxD9kuIZobQxsqgbwnYnpn3rz/R7eO6mljIkaTb+7SPqxzyBW1p99/wjhvodQBjvfLHkqBkEYJHP1P86cRHUxl1AUrgAAAZzTyTWXoF7c6tDNe3MYRCQsYA6gdT+da+0YrQkYGOacJMd6a0ZzwKZtf0qWO5KZM96BIB3qE7h1qPeTIUzyBnHekFy00g456nFN8wVn3z4tC534BGSPc0+GYTQiRR8pJAP0oC5a8w9qesme9VQTnv+VSpnFMB7fepyxuT049aTqeanSYBcY4qr6WJtrcjZGTrj8KQMR0q3EyFsnnAqy1rblBIqAkjJNSykjMEnPWnmRdvJp1zCiEFR2qqRmoZQ5pMEjNIWBqFgQaAaaEOPPNMIzS5pyqDQAwLSkHGKsCLjIpDGe9AFQxmlCkdqnEeDTtmO1AEKA46GnRp5gOAQew7mrluQhz2q2MEZAFIDKNvL2Q1CwIJB6itpjxjiq8kSkEhRnHWncDJJxUZarMsRXtVdhzVAMyaCxpD0ppNAhd5o3mm0nSmIf5h9aUSEdDUWKBSGT+afWjzW9aiozTsBKZDjrQXJqIGlFFhj9xNKMmmipBSEADZp6gmgDmpAKAFAIqQE44oUZqaOMZFJsZGN9SASdxVkQr2qUQgLkkUgKgB707FWfLA7U0wAnIoAjiOJBVC68P2VzctMJHhLcsEAwT689K0jCQeKNhpAZsfh2zXIMsz59WxioW0eyTW4bfY5hkjJKls89q1idprPvJSuuWJBxlCKLDLY0XSxx9jjP1yf61zdoBbyywlQ6RzYAPTFdUJQDkmuVkONRvR6uSKuxJ0wkt0ztt4QP8AcH+FIbjH3EVR6AYqupLRqx7gUVaiiHJkouH3ZJyPSkMhJyCfzqKkquVE8zJlkINOMmagBp4NS4ofMSx72bgFsdcVbCHGarQjjOcfjUhJHeoasWmS5xTg4HeqbSnOM0glx3pWY7l0y+lRl81AJgOCaesiE5zU2C5OhYjAz+NPIIHNR/aFHemm4B70WHcczkGomck801plJyTUZmUHg07AOPNMPFNMy461E8oPQ0WAlEmDSmQMMA1V3ZPWs3VNYOmERrFvZhkHPT9PeiwGvO8dvA80jYVRkmuCu53urh5GdmGSACcgCpLrVru/BE8pKn+EcAVXRCwIUA0hDxap5O/zgJAOhqPaw6Y/OhtykhhgjrTd1AxXSQDOR+FRmV175+tOkYopPJ9jTYY5rtiIYi5HUDtUpgNaVz7fStCyXTpISbjcZf7p71RntLy2AM0LKD0NQjJ5p3A6m106xvVzHA0TA4GRgU17UQTmEbFcdC3ArCt9UvLTiGb5fQjIq7Jrr3YQTW0ZkUEBs/0oQDtQje0nTzWVo2Gcqc042VheR+cZxbgdQeazrh2lk3NwPSk8mR8YXK/WgCR2ezby4LgSR9iDkUw31wYjCdroexFIB5L7GjVQenfNJLGMEgkZ9DTQEXlPs3Y4ojbHWrFvdGJNpAI7g06XyZsOkYVz1x0poCxA+YxU8c728iSIeQcVTt/SrEgPlmne4G+0czkOV+UjOazb/R5rtyUkxjruOBVrRtSN1GLd1IaMYGe4rVmZDGSVAPYikBi2VnL5b2k0yHI7HINZWo6M9u7CJQQOwOTWq0xjuN4+8Kr6temQxyhHVs4yDxVNAZcWkXLRgmFifXFR/wBmuGIcFQO4FdpbalGmlCYndjt0rmNQ14XbkRQhB6E5qWgMaaAI42tuX1pYWMT5GCfQ0+STeSSAAfSoCRniiwD3DM2SpBPrS7GI4Ax9aZ9qlxt3nb6Usc6KuDEGOeucUWGIdyHpimFtx55q0mLo+XFEd56AnrUFxbTWzBZoyCemOaQhu8AYAGfWkDZGM1GeTxSAGqAkK85pgYgj0pQexNIcBulIC8l0PIKtGjcYBI5FU8HHBH0pRgUowTjvQBe0wJJII5I1IPtWybOC3YMIVBJxWNpwZLgHHHr6VqySGSYAvnByOKoDXH2QjY8G0n2zXP6jYC0vFeNWKv0AFdMpjmtQpIBxgZrn5op0uAs0gdcjnPSk0Iz5WfIJQj61E0ykYZQPpXR6jbo1gzhBkdCK5yO52LgxKf0rNgmKrwheQ2aYZDuOxTjtmomcM3KgfSkyc8HFAx4LBwa1oiDEMVkA/MMkkelakB/dAUJAPORTG5FONNPQ07DKbk+YasRMdozUcgyc0sR4FNAYQIPU4pQQeBVmGyMloZyenbHJqFYZlHmBCAOtMRuaQhgh3u7kZyIx1NaFxbq6i5ZCFPQH+tYlrqQiCmUZK5wFFack97qEYcIsMIGBk9aQFSb7OVLSR49ABwKogI/HUdxVmYvuMckqlR1NWYDHApBXd74yaAEtUlQBoYVQDqz8ioJ5ZmuSftAOP7vA/CrP29kJDQ4jPbODVZhFI+5FwPrUsBtwMsrKoAxjjvTEjaZcqMgU+QgIQTQlw8NvtjAIPrUsZTkO1iCpyKazg9v1qSUluT1NAtHI3EgCgCNchsjpTsknNMKsOKeJAuAwwaBFq3GTgEgnuDzXVaIjTIAY1jCnBOeTXJQTIjg56VuR6+ltYkjyhLjAA4qkMoeJbl5tTNuWVkh6Y6596y41y1NUtcStKfvMSTT93lngEn0FVcaHzDdEaVZvPtorHbhSeTnOfwp67NgE5YKeoHeobtLRAGtpWJ9D2oTEV7i3FvKyEZI5B9RW4niGSTTUhlGCoCgjOcVgMSxyST9aduxGB60riJ3myxILj6k1as9UlsifKKsCOQwzz61lZNPU47E/QVNwNKTVLlrgziTaR0CjAFaSa1JOuZLy5I9AMCuaLjODVkXIS2IBxkYJHUUXAvLfpcLMkcICgYLkcmmWF0tpFIwwZScDI6VRhkNtbuD/AB8j6ULIMZPeqTCxpLqLor7IlBJyX7g1QZ5Zp/Mdy49SaAx3AEYHrTw0ecE9akdiNt23JPFP+0FwMZBFRkmQFcgAdKjQkqSB0NMRpx3BhhdowokIxnFZe2SZgN4DA5OfWgySA4zSjI5PU9TQM0klkI8qTaZHGPMbqB6VnTxmKUxlg2O46GnSTFwMnoMVFuBp3AWKEuMkgD0NJsA5PSmlqAwJzmgAkMaIepPYe9V26A9yKsiMOSCQOM80xY8sCQCB05oArYJ+lGAKvSwmQHAAHoKqmF1GShAoAfDC88iRxjLMcAVYFvJbtcCSIHyxyWU4/Oiyd4biKVIvMZCSFPQ1p6tqjXdp5LwGGYnLEkYI7imIp6LFF9vgmDAnBLgdAKmuW88XFxtLKXARyeQKk8PwBTMSgAIyCRyRzVn7Hpe/cXUexY4rlqySeptDYjZVubyK3mZvLWEHGepHeoILs2ccoUttLkJxnAq/NDplwwMkqEgYGGxxUqSWaReUkkWzBGM1jzK1rM1sipopDwyvkli/JJqM6zcAkCxJAOATnn9K0rWOKKFUhUBB6DqfUmlkuYYHKSTLGw6hjg1DkuZ6XHdGLd6zdG3ZDaGHP8fIxV7Sb2a5gdpoyu3+PGA1TyXVjMAstxE4ByATxmmXF/aC2lIuEPykAA8k4q3ZqyiJ97mDew+XcM+4HeScDtTJLjeIQq4MfU4xk1XBJ6nNSIAxx3NehHaxzM7/AEO0s77RxrE8YMwJAAGAgHHAHf3rmtV1ZtYNuqkqYwVJYgZJPX9KfZ68LDRjYCIiTDAEZwSTVnQvDdvfabNeXTlmwQkanABGeTz9OKBG7plvDcaLaNDDE1xgRu4UEgDuTnp/jXHX6ebrN3Jj5BIQPatSwtZtElWZJYhcAYaHdkMD3NZTTm4a7dlAzISSOmSeBQJM0/D9zb2l5cyXUgS3MJGPXPYCsE8sygEAE4yO2eP0qUuDEVXrjcD6U+SRbi3AlGHAxuBwPxqQNXQbR0UX7q6uGPkEjCSevPtW/wD2Et9cmUOhRORGOTnGTkfWsjRvFl1YadHpUlvDNa87C4JwOvGPer1uLu30q51aHzQxQlSCGUDpg45zQBMujzIkkklu6sASx3ZHsMDtWzp8cE1kFCnBABIOBx7Vlaf4mQ2KXl3bsRHiOZ0b1OASMcjrXSWsNodKElvJ59u/KOvJzVDQJw+2pvLLHIFU7m8WyQH7Jezf9cUJrlH8UR33iezESSC2BKbJMqQ3qR3xjpTasDLOv6ncaB4iM1rGpjmiUsrjIYj09K1/+EjtjbQyOBHJIu8xE4IFZfi+zP2CbUJlaSR1SOALyFA5J4/CuZhhuNThEky4kjlSM5GCQR3qRHpcFws0YdCeRkgjBFTKc9ah2JDhV6AAZ9cDFOVxnrVIRNmmlRuJIpA49aXgigYhAIIPQ8EU3yYw6OFG9AQpHBA9KeVIppIHWmBznifQjqLi7jl8uSCLIOSM45Fb9rHMNPtBNzIIV3HvnFPKh0wRkGlMknY5pABQjtXBeJbESy3tzHGxkFztODwAAO1egCc7NrAfnVYwQOWLQoSxySRnmgDmPDOl3FlGLmQvGZlJ8vHBGeM8/X866Ik96uyAOAT1FV3TnigCNXIHWlM2e9L9nY9CKY0LA4OM07ABkJPFAc5HNQlXBxSgMCMgnmiwjCnO7xFeH/bH9K6OQ84rmQS2u3n/AF1x+tdFI3zmiwwyaUVGWGacp560CH5xTw2KqXd2llbNPKGKL1IGcfWuci8UzG9LBFNux4TAyo9M0rgdgpyQK4/xfqdrfx29tbsHMLFi4PA6DFWbPxLbBXF2CGlLEc8KD0H0rj7iRQZAgwMnA9BnpTuBESGfYDjNNOI3yXBP16VCSRkggUBXdcKM496koshgRk7T9agkZd2B1P5UAFQQQcjggUKwXqAR70kgGgEjigZ7GpvMDRE7cY6Njoe1OkZHEcmGJVMHPfFMCOOTacjntSlyRkZ/AVExJOaQEg4yAPfpRYZP5gBGTn605WVckYyT1FQrsBI4Pv2ppAByKTQE7gg7weKaHBAzwe4pofKkE00rjvxTsIsLIoYEgEj1qcMrjBA59KoZwamVwQAM59qloCx5aZGRnBoyiMQQSDyMmogW65BHcZp2QwGTgg9KLAbkGqwtpS2rvIjxKSrbuCagvNRkl8h5EG2HiMsM5PqfXHHFZWQOlWob+9tVS6g8siMgEsgIye2DVIGhb3UZL+QM8hdVyVB7E4/wqrDHIWIDk5yeadcXbXk5mkSNGPUIuP0qLeQMg4NAi3p1x9lu47vbnYcgZxz25FW9V1GTXNXWc27DChNkZyQAfX8azB95lHK54Prirlm0lsgukcGYkxpEp5JPqPy4p3BnoWloqaLbiCKRIwuAsgww571ZAI602DTZNJ0jzprnqod1lk+6cZIGf5e1WEs5pYFmQo6sARg9jWhDEUjaKdxT49NuGI3gKPY5NTmwaMcEn61LGihLGZInAHOCRjrmsKG9ja/t5JFKOQwOe4xwa6YxsGICNnsMVwVzfqk0bxJ+8jdgY36g5yAaQ2jR1+eUXEVpGCsewyyEjt71Z8PXCXGlqg4aMkMPc85/HmsrxRdyi+QAAM1uFcHkDJ5xWj4YdZLSZI4tiIQBnqTjJzQFjYC+lGMVJ5RAyT+VNZDnOOKYyMkCjcaCp9KbigQ7camS7kSIxrjBOcnqKr08YoAfJMxBJ61CXOKmKqRigQAik0CK+/PWmk1ZFuueQfzpxtwFJAHHalcdinup6yYNBUE9OfSmeWeeQKdwLS3AAxmneap71QII70ocjvQBcBGalTBwDVIP71Mj4Oc0gLWADgVIJAABmqhl9DTTKR3osBbMgz1oMg9apeac9c0vm+9FgJ5EV6rPbZB5FOE3vThJnvQBVNuSSMVDJCY+owK0O9EiJNBIjjIKH8OKLgZJGKTFW9GuEv8ASQ7gF0JQ4GOQf8MVYa3hP8NNMTRm44oxWj9jjPQkfSg6WCpKTZPoR/8AXp3CxnUYqzJZTR8lcj25qHyZP+ebflTuMbijFOEbjqh/Kl2t/dP5UhCYxTxxTdrZ+6fyp6xzOcLE5PsKQDgQOtOBpBBKfvDHsad5LjtTAkVuamRh3NQCJ89KeI3HUfrUgXEk461YSQEYJzVBEk6kY/Gpo3CsATz3oGXcd6MiovPUDGRTTMO1FmF0TFxioy4zUBmz0pN+etPlYroe43HNZOqIUvbBzwN2Ca1Q6+tZetH5bV+yyDNUkJmiytuwKwL9THrMoIxvQH9K33fDkisTWT/xMYH7shB/OrJNO3Ba0iYdNopxFQ2JJslBPQ4qbNNEsMUlHNJluwqrk2FApc1FIblSCiptzgg1IBITgqB+NJjJkJI608kBevNRoCByRS4Pes3qy1sRnrRUhQYpViXGTk07qwtSPardRk03ywDnmpimBwAPxpPLJGRzUspMiJI70zefWpZI2A5FQFTSGIX54NN3Gk2tnpSkYHNFgG7jS7qaTSEimA8Ng1z2toLjVooyVYEYAzgj/OK3s1w+pGaLVpn8wbgcAgZwKBXNTUdIhsrYzRuxIIGCKxxI+8bQR65q5HrcwGLgLKR0JGCBRNqiXMJR4VBP8Q6gVLRaZWlkDg7mOfeoFkdGyD0pgwDyxP1odgBUsCU3BblsZ9hWla6vBFGYxARkYJB5z61h7xQHGeKLDNW+1F7nEPmyGIY5PU1ScALlWB/Cq+8k8mnLlmAJAHck0NCAn1p6NtINXm0uJIfNaUSD1DYqnHA0syrCucHOM0WAlZ8jOCPrUyyYtRhWyOhxxTblZEBWSPYfSi1unjt5IcAgjIJ7VIEOAQCz5NIQ3ZiKgMmWJHGaerEkYNAEoljCYIUEd8VEJcNkZ4HrUbDDHPXNKpX+IZFNIDRtHDAP3NXQdykdsVlxyowCglSD0z1rShOSAehFUgKZke3uN8blGHQitK31+YjZMEkz3IwfzphtopvlYEE9+lZk8ccMxQNnHfFAGus6TyEkYB6DOab5Ymikh98io7WJ3gLRfNjqB1p0Jk80FFOBwSeMU7gXtLslltWSR2yAQVzwKz5vDUzzFoJFCdgTW7pgCPNk/eGa04YYxDk/hQI4r/hHrlJAkhjIPoazbqymgmMYibA6cV3yW3nOdykemaoyaSJZttwzID93B60hpnDyQSxDLLjNRDPfrXYatYTW6/LaZhH8a8iuZkVDIeCv1FMLkcMpicOOoq7Pq0lxB5UgU56nHWoVsJXi8xMMvtVUAHqKTAaRilCMVyBkU4qCKntxA0JDbw49DQBVIIGSCKTNSbM9WOPQikZQDxQAiEYOT16UoGTjOD600ggZpUXLc9PrQBrWBQtsdiCehFaH2UGUEXKj2NY9vbmaRcSqmfXvV2a1aBQdyvk44PSqTEaUpMQAWUHPp2rMvJXWQHcCT6dqeIn6du/NTXenAWpmUkkdiKBCXF+HsNiNg9x61jAVKGUxYJ59KWNI3OJMj3BqWrjRWKgOQ2cfSpxDabNxdgajuERH+RiR71EeakZbit7aYEiUjHYmrkIUYjQ5IrJgEhnURjJ9K27aJwwLqFPoKYCFCDgimlGPar/2YHkE0vkYpAZTxkKeKgUlX5rXlhyTxWdcx7RkDpTQFP8AtFo+EhRV7AdB+NRWtte6pJJHbNGADkqDgVEt03k+SIQSe5NaGkJJpOsQtcFVV1JyTxirAz7i0eA4Eb+ZGcMeoz7Vust1e6fE5GABkgHtVy8ltGvZJEuIirDkDoT61Wa6iEBRJlwPek4kXMa48suFBx6mt2zuYYrZYyobHc9TXOTSRtIdh6U+G7eEkgA59ai5ZNfFvtB2nA9Kii3AZPSkMvmuXc4H54qz9ohMHlqhPvSbuBEzggcAnPekcknJAX2qNiMjaMYNNkkCkMxPtSGLIwA5NSRyBkIJrPuZSSuMgA8UeZk5Hbt2osBcIAJyc+hpGC4yQKccAAjBBHHNISoByMikBEYyzDbjB7k00hs4OD9KGcE4H3ewp0JBkAPGe9UgJo0wtSBEjBkbkj9adNGwhIUgGqbZQYJJoGSzXIcfcxj0qmTliRSu+QSKZ92MeppoQ8A456VI+xim08AcjFRKSUA96eI3G/KkDHegGhDzIcD5exqS3kCTbWxtIweKSNT0YED1IqQpGGz196TYXKswUMxVuQeAaSPcFDAHIqSYIxBTkDrUJJBwDgUCHmY8kgEnuaaCSCMEg9xQE4ySBTgADgdaYyw+Gxhug6GgKD1b8qauMgEHNEnDEUWESoFLY3ED1FPIS3IwAceveqeWHSlLZAByTSsBOyBiScKT3ppBVOTzTmJMY4BHvTS+RzyKLFIjJHakzjPSkL5JwAKaSSSMdO9OwgGT05NOCHPSnomFDY61IMY7A+9AEYAL/NnGMdKMlRgVKEUqSTg4OOe9VyCcdAQKAJVkcDOee1NlkIc4PyHFMQSMSCSAPWhgdufegBocg5BIPbBxTnMkzEuxYkY5poFS7xkFVwAMdc0AaNvqktvaiH7OpIXaGzjA+lVZJzPAsRRAFOQQOfzqPdxgn6imcnpUgRGMJMrMMqCMitWQW8saNDzITjA4GKzlO5sGr1uN9xH7cVQ7nTWMa/Z1GORXM3pt7nULszOQQSQR3I6V1EDeVbySY+6pOa5O6mSZVl43nOSBjPNBN2VhCskQcDHOMA8Co3HlKxEZwSQCRUiMuwAHn09qfqF4s7JHDkRIMcjqe9HKVdlEGpY0MjhF6npUaRvLIEQZJrUieG0jEMe0yfxSMMkH2qmrElNrZo2Ikxu74OaI2kGCZ3AByUBwaeZC8gJ6k4JpDEUmEg5FQNMVpRNcFjkt3JJJqW1FuSxeQoytkL2NUY3IMhHc1JESHdlUFiMDPammIfNNI08jsRgjHA4A9qfBIGIMn3CCKYy5gRD1ByauWscbCKFkzkEnmhgaOoER6RYBIXUQKNzkYBJzwK6TwbbkQ3EE7li4WQQggqV9Rxx7+tVH0a31fR7e6E0kEa48wk5GB1wOMnpisjWNeVLy3/sdZ7T7PF5W9xguPp6cUkgO21uyF7pxtvLEKvly4UAjbzgY9a5wQ6yvg4WtlA5R5vMWSGQEheSQe45qWw8RyXMaW9vJDKxOAbt8MSeT0rd037bbajJHcRwrbzjcio25QQOQDjqf6VdhpnBabrWs6fDc6fC0wuJpM4fkgkc4z0PSr+maHaXvh27uSksWoW7nLOcYPUcfT+dbPifSIYbee/LPJezTKISBgqSeAMdentWxpNxbappBM0QSZv3dymMHeODmk0F7lyykOo+HreOYEGSIBwRzkYpBokJe5cgHz5lmYDplRgAVYhEUESxxk7VAAycnAqwJVI60rCIHt0c85B+tNFqo6MRVnIPSjA707gVjav8AwsCaetlcYyCg+pqdXAp/nkdDRcCq9tcqMlAR6g1CY3J5Ug1eNzlcE81A0uT1ouBD5LAcPj60wxyjkEEexp5kNNMpHQ1QDMP3ppLA8inhwTzTiNwyDQAxXOcmpgoIyKiGBTS2D1xRYC2DtAxSBEc8iqolJ704SsOhoAnNumeORR9lBIxgc1XEzg9anhuMyAE0AcdEM67ef9dh/Oup8qJpSG4rlrNxJqlwR3n/AK100jYkoAsLBGh4UH61ILeEjJjBNV1lOQSalExIpAMuNJt72AxBihyCDnjPPWvPdRjk02zktJbQxzNISZSMAgHgg/SvR45SGBrj/Hab5rZ8YymAexIPT9TSEcwPskb4M0j4GeTwTVF5HuZiyxqgPYdBVu9hgtkREcySugYkjGwntVJG2DFJMY4xxhcMBu701Dt4GQPanM+8YzkgVAHxhcDAoGWmSN+R1x1FMMCkckk+tRebgZFPJIHJ6ikBZhUukkKj7+MDIABGck1SBYEq4wynBHoanRgHRifukGi4lDySkAbXAzgY5piKu4g8UoOGyRnmmkHNJmmMdwCT2NBPFNGSRnpQetMAznH1p5JKge9NAxU8RUkAjJPagZFtOMmgZHQ4qd0ABIBxn8qg70rgTI/GCaexwQR0IqFULDgjNTFJAhDD6YNIQ5MP1q5ZSyQ+YRAZoRgSKO3PWs2NmVuQfxq9FdyxqwU4DjBHYihAWb+TSH0+2NqjR3ZJL4JJ6d81lhsn5zVuOOFg7sQJAfkye1VZoWinCuCNwBGRg4PSgQ+MbnAjBJxjAFW3hmsLiIyoBIAJAAcgfWpLHUI7O2e3MKuZCSXY8g8Y/LFR3sr3UxuHIIIAz0J+gpjNDU9Xkv4YEckxxryh6E8816F4OdToMM0YwGyCpBwCOOPrwa890IaaBcSXpjZxGTFHJwCQCeD3PHSujt/Fkw0G3s9Ns2muRb8mJcmE578cnigVjvXY5yOKa0uwZJxXJaTquq64beZwbezgBWQk/NM4A4PTAHH5+1dBz/eP50rgkWJ7hUgd3OABkkdcVwj2EE+sNdQOBlyQHPBIHX9K7GZlMEvmfdCEn6CuGhvIZIbia03SMsjiEAnJGM8D2qkwMTVbq6ubwQzSeZ5J27wMAgHrmuh8KosN3dksSWjDAnpyTz+Paufu8jTYImO2QgyHvk+n61f0zUQksME0iQxREM5A5fA4BNK4HbHPejioIZzcWschUqXBIBHbt+lLmqQDyRmmbATmjNOBoAY8J6rjHoajYGPGcc1OCaZMDIgA6jpTEMD4qZZSBgVWKMKVSRUjRaLkmpowcc1SDsDwalWZgeTSGSyWkbxlhkPjPHeqRiccYrTjmTaCTTSqO+7OaAMeSKX0P4VEUkHJU/lW05QdBj6VGSO1FxGRvkHVG/Kl85u4P5VpBgDmhthOSB+VO4GcJifWl84+9WmCHjANJ5cZHIFMCt5hNODEjvUvlKDwKUIPSgCME04MRViO38zhRk1MLBj3A+vFICiHNP8APCQyk9kJ/SrR0092H4UyfSwLK4lycLE5I69jSA5bwheA215GTwHDj6Hj+ldALhc9a5nwCsM2pXNpNnMsYKkHA4PP8xXosWm2sS48oN7scmhAY0c6HHNWlYsOKvvp1oTkRY9geKYbdE4UYFNMCsAxOc4+lTHBwSBTxEKHAAxjn1pXAR4woBCggjuKjAVjggA+gFPDk8HP4VE5w2e9FwJooUDgkA/WrigDggD6VmfaMHrUiXg6E0wLEtkkzlg2Ce2Kry2JiXcRx61YE4IBzThKG4JyPSlcDMJB6UmauXMSy4C4UjnIFUWyjEHrVrUl6EqsBjNK6pKMgBT6ioASxwMk1PEFK/MxBBoasCZEIHydoyAM8GowavCAlN4cfSnxsQ4IVQe5A601KwmigPy+tGfcVryRQyxbnQbuxxVY20Q/gH5U+cOUo9+oqhrIzYA+jg1ufZovQ/nVLW7ZF0WdgDkEEZ+tHMg5Rg5jjPqoNZeuDa1pKegJB/HFbtoIpLC3k2FsoOR9KyvEpj/s2MBl3LIDjOSarmQco7TSTDIPQ1bIAAyyj6nFYgv5bI5ijVhIoJyenAqk8sl7c77kBvQDgAVDmkV7Nt6HQyXlpbkCa4RSfQ5/lUD63bRkmMNIB3xgfnWALcKzEAZJzgjNOEm04ZSPp0qJVuxpGj3Ldxql3cSAxt5ag5AHf61Mmr3sagSRpL+GKoiSLorZI7EYNPEmehrL2rNPYxNRNeiBAmtZl9SCCKtRavYTDImKf9dFIH51h7nHej733mo9qxeyj2OgOo2WOLlDj+6c1m3l7GwZ7e8k3dkAwKzTbox44+lIbZAfvN+BqvaLqL2KLNrAl4jm9vWjfIwCe3rVJ2uNPuyIbkuqkEEHIP4VI0bOpUsGHbI5H41AIJIzwhI9jmjnQeyNSXX7xVRzHCQRkjOKk03XPt12IJoRGW+6Qfz/AKVlkxniWFVJ6EKaVI4zIDFMquOhB5FHOT7I6xo9hIqJoi/QVjR63e27YknhmH+2Ofzq5D4lG0+dbocd4jz+Rq+e5DjYsG2lP8NM+x3H9z9acviLT3YAs8Z/2l4/OrseoWc2BHcxEntuGaLi5Sh9jueSFwQM5zXn90ZGvZjIMMW5Ga9XOc8CuP1/wyYllv7VycnLx46fQ0+YTRyRQnoKFGKkjSRsFz5aHoSOoqW4svKCskyyA45xipuNFVvYjNOVkC/MNx9KYAB160pYDoOaBoNyBs+WPxqIgDJ96eXJ60wkUDHAArxgEUYzwcVESRwDQCcYJpsRIAWG3JPsTUgM1rMpBKnqOahDY6Hmnos1wwAUkr39KBl67uHnCu/YYFV0kUA89RiklcAbC2SPao1ANQIjIIbpU6kbQcYpXjUgMA+e/wApoIAUYJ/KgYrhWIYuAD1FROFBxGcipPLkYYQE0zy2U8jBHXmqiIVYjgEite1P7oVmOXAA2MPfHFaNrG6xAuMelUBehZxIAgBJ6A019PleYi7tcr2IOMUsJ2zRn3raYZGR17GqsIp29nFabjECARggnNOMckCmQAGN+CCKfKSIyWGB3PatG0QT2oC8kUrBc55bwxXZQnAPSugsw7wgtnHasvWrFI7cTYVZEIBGMZFa+lTedpcb5zSEWQMVBeECMSYyR0qxVW5bzSIVGSTQNCxubtAZeV7r2NJPpdjPGVa3T2I6ircMaRwgY570NigZh3mnC2twIQGj5yMcj6GubuLeyKljG8Mx6Ejiu+461BdWVpeKRNCpz36GkTc84NvJtLBlZR1OcVFGxV8Dqeg9a3dW05LCXaHHltjAA6VjhRHNlTnB4OMUFEsdnLMQWyme5HFLJaeSSDIG+lT3d95sICPhsYIxVAyvjrTAHhKjIHHpUYGKGlf1pobPPU0gLMZJAFWIkHnKJHZUPU9aqRTASAGtSGW1eVRKBt755oTBovg2SKArux7Vswwb7XYRkEYrLMUJwbeJWPY4ratmPkgFSCPWncVjk302U3Zj3hB2yM0wacySlHkAI7itrUR5V6G/zzUIXOoofUZqkBRjsIRIBLNg9gRVa/08QgOjhwfQYrqru1tpIf3o2f7SjmuKviVmMcckkkY9eamwyWykNrIXNuJARjkdKsS6kTODgLj0qrBdEQhWbkdsVA7BnJqWB1sQzCp9RmnEVW0yUS2gw2cVb20gIJBwQBWZeJ8rZraMZPQZrM1KJ1QtjrTQHJodkyAgkA5IA61o3F/Fe3UKvBhBgZxz+VUQ0Uru7ZjycgA8VTMmJCVYgjpimB0T6rp9kfLhs9w7/wCTWZe3UN0d0MIiPOcd6oNIXOTzSo1AWL1gEYESDJPepLi3cbmjjbaOnFUUkeNgVNK91O5yzkD0FIAD4GcjmplmCjBODUKuBk7QcVHvZySQAB2FKwE5lB6GoXyzZJB9MdqjDdhT1jZjwOKLFDwfN6gDHAprFY+gJbvQ4IG0cHvTCNhBPemImVi20nHB7VbkkLxZCkcY4qpEmTk5pBO4cpuIGaVgHqCOoIqRI8uMkgA0wZLDJJq1GAeaQFvIK4qjfsAQAMY61cHCEnoBk1msDMWbOOaEBCASQB1Jpsn3gOw4qWNSkmW5AGeKgPJJ9TViJrfJlI9BmrqSfuCSpPPYVThPlsWAySMClmupInKocetJodyU3sZ4kUg+wpfMtz/Eaqkq6F+rdzTVYAgmpsBORCXHLBQc5HeoXwXJXOM8Z607cX4wKaRg4piHZ3de1T28cJy7kgkcAdqgWRQCMZYjvU8IAQHHNIAMWZQd+Ae5pHG1sEg+4pCxzgkcelISoAIJJ96BgQAaSTHAXnjmkeQkDoKapJNNIETQNwQScY4702Q4GKTBXkZHvSAhjyBnviiwDOopuSCQKcSR1/CkABbJpgOEjKmBil8wyEE9RQUBOAcUC3kzlTx9KkCYgsB70xxhh6gVIRIsYABJ9SMYoKKRyTk9aYEZckgZGPekJGcZpoCF8EOSD6cU/YBJ8wwuMj3oAURkJkirP2dTCNnJ9KrzsREgB6mppCgiQk5IPIoAr78EgdehoJIXjrSujFshBgnjFIRgkOCCO1FgCMZBI61o6Yhe4wTnHPSs9OhrW0dNx39zkmgRtysiaZMXbapUgnGccVxcTBGKSo24nOfaus1R/L0vGPvOB+Fc3qMiNdRlD0UA0AV542RgyKdh+6R3FVSTk5rXN3GbIxAqDjAAHIrKYZHPXvVJjTJrWRo5cqASR3q9GyidDMxBxznnFZwyuCD1qUcc5JNJsTLUSRG5AzhCTg+lLMoVpDE+9VGemOfSq28L1qyUMS4cEK44wM5PpSYIpuTIQCFUDnANEQIfI4wetRZAbHap4iByeQR0oQx8hDucVp6TGjzEzE+XHHkkcetZkaZNXo/liIPQjBoEdXpeq+dbCCGH91CSEjJwC2c5Jx0HpXIX91Nc3d4JXEk0kmCIgCuB2HoKuwXQt4JAA4dhgY6g/Wp9Kt4be4CySLudDkgdvc00xmJpMNtNqtvFdllt5G2sVOCCenP1r0O3tDptuYvNaRIZPNV2OSB6Vw728VvdRTRJI9qJlYuwwSc8ivSJIUBU46jNWtRMgk1aGZYybWWY7gUAXoR0PtVmOaIuwjQKxOZAg4Dd8nuaxbqwe3S9mtGJkkjIVD2+hzWNBreoaFbJZPa4ZckiQZB96diTtjIw71He6imnWLXUrcDgDuT6Vn6Xf3d3EwvYFhlGNuBgPn0qr4rvYE0hrQSA3BYYUdQARn9KljRt6ZrUepWoljHzj7yZyRV1r3jhSPrXmmn3baXqkExO4ADeF64Nd79ut2szdiQNbhSxI9KVxlg38ayIkkqK7jIUnkinfalPUk/hXmmo3bX2syXcRbZuyjN1A+ldX4bvm1GzuBdhWeJgAQOMEdP0pgdCLhT0YUGdMdaqeVB2BH0NIYouwJ+pp2JJWmX1FRGbnqKhMQHQn86aY/RzTsBYE3uKkE+BjNUvLPZvzpRG/dgaLAWzNz1pDIDVCVniILHbGMknGfyqWGYTRh1DAE4GRjNFgJpZvKjLbGfHZRk0lrcTSRgzQhCSeQc8elQ3JcxExkhgMjB61WgvbdJTb+ZmYgEjHU+lQ0M2M0qHDA+lcvHcyrP51uWEiuVKu2QOemK2FvTFCzTsN4B3FRxQMw9K5uZD6z/1zXSSMS2a5fSnxE02M4cnFdEZMqDjGR0yDTRJIHI70omIOSarGT3pPMqgNIXMZ6msPxHem3t45zAs0Y+UhhkAk1YE8Zn8nPzgbiPaqeoRQ6gHtXn2eXyRkYYf1x/WlYDiL2Xz7qS52hFc5wOg+lVGY1qyJDNBtUBACQST2B4rJfg1BYByDmmk5kyfxApACTwOnWnYNAE2YmxnIAp3yYIBJz61Xyc0/BxkUWAduwMfWmEnJyeD7UEkEHPekYhjgDiiwAeaaeKXGKu6Xb29zdFJ0LsEJRQSMkdRke1MCln0orejs7SYEx6VIwBwSJGPNNbSLfqbG/H0bj+VFyuRmJzRkjocH1rYOm2Q6x3qfUA/0oGk2Ukiolzdh2OAGQYzU3DkZlBiRgk0oTJzkUksZhuJIiwfYxXI6HBowMA+opoljkJQnB6ip1lycE8VWJxQSR0oaAt5DZGeRzilPAqJTtYsAOeSKeWBAPqKkBQRnkAj0NJKJH2zPkhjgMTkkDt9KACWwKcIZCgIxs7DPSmA6CXyZ1l2hsZ4PQ8YpEWW4kCRo0khOAqjNS28GZAHBODggDOB61ZsJ5tF1YSKcqTnJ5BH9KLgP0jRLjVLhoWcQEAk70OQBjtx61tab4Wv7XUiGnItTIFcxkqXHXkEdO350o8UCaQy7ITMDtijwQRnjJNXYfEd3DBKby0DyRJvDRnC/Qii4jrDIOmAAOgAwAKA4PGa4nTPEGs38hlEUckKnDRKuM5GeCen1rrY5gYkkceVkZKuRkUwDU737Bp8lwqCTYMlCeo71w+kz29pc3F0Awkd3KxAZUDB4/X9K6DxJqFmNNKGaQSODt2YII9/biubcx2lkksknmCQfwdFwP8A9VCAUQC6EqEAnhnYnGCSMDnpgCmaXbtcTT2sMKEIxd5QQcjsB+NZtu8t1cLbrJtjYl2GeDiteHVxZqqLbbSGwTEAA55wCOvU0AdNZ3sci+Q0mZowAwA4HHargwwyDmue0+wm0vS7q9uUzNKAw+bkA9z6da09FeS4sPtLgAyMSMdMDj+hpJgXsc0u3ikY4rF8Q69c6L9lENqrmQkEuCRkY4GO/NUmBsEuO1OAyAazrbXrGXT0urhxDI4yYRyQfpiqzeJIJpBHZw7mPeQ4GPYd6YG5jA6VGIWPasUa7diTJjiKf3cEfrmpR4gl/wCfZB/wI0AjWEJUgmn7AeorBbXL1nBVY1UdVIzn8aU+KBbTKt3Z5VhkNEeR+BpNBc2GBU47VFcXAtLd53bEaYyfQE4/qKW11rR70gC5CSH+CT5T+tcr4k8SEm90yC1URg7BN5hJIBySB07UrDOsEu5QwOQQCCOhBpC9UNOlL6VZsevkqP0qzvzTsSPL00ufWnKoJGTgetSiGNgDjIIyDQMr7j60BjVsWsfoRThaoD60XAoGUi5MOOQgYnNSAk1LILSHWII2IM00ZCDPOBz/AENWzFGeoouBWR2UAg4NSCZy2ScmpxDGB939ad5MR/h/WpuAwXBJwTTpp0j0+7dzhRCxJ/A09Le3HVWA9QaZq6W8Hh6/YngQNk/hx+uKaA8/8COyeLLcD+JHB+mM/wBK9UJUHk15B4SmaHxVprKcbnKH3BBr1idiGJBoZRNI6Adagc5BNQmYgZPJqFpmY5zj2oSE2W4jmQZPGalnVM8CqUcmwg5zg1MZN9IQIRuGeBUs1vG8Q2Y3nnJqACp94MYHAIHamBnS2zxth8Y9Qc1EYj2q1cKSQVJPrmoFYqwJ7GhACQTk8DA9zipRFdquSmAO+RUhuMrnpTo7ptu3AI70wII5NxwW4qx5EDjJXk1nhw0xIAAz0FWFZtvWhoCdUjgfMYwfWqUqyRFnIwpJOatI2Tk1JNB5sIxSuKxnrO/rU0UrGQc0osz2wPrTbhfsNpLcOVO0cD3ouCMrVPEF3DdmCzdQEA3DAJJ79arprepkBhNG4P8A0zFc1LKZ7kykkMWJBBwRU7Xb4IGFz1x3NSzWKOhPiDUl6mH8UFQXet311aSQyNFtcYO1cHFc8bvB5YH6HNO/tFiyqclScYwKRVl2NeO8vntVhW6KxgYAHBA+tU7i3CQljy3rnJqrdXJgIUZwRng4quL12baSSD1yc0rsLI2c7reEn+4BUYdI3BY4z0pjyf8AErD/AN0YrJW6mYA7FH4mnIUWdGFBGRjB6cimnAGQAfxFc+buYfwxgfSm/apj08s/8BqLGnMb5IPUComjJx5cm0DsTx+VY/225/6Zf980hvbknrH+IzRyi5jZ3zp1eM/Ruf5UonDHDuB9TWOLi5I6xf8AfH/16a13LGCSkZP0xRYfMb2Ux94H6GlAGOM1gRXs0mSsMYx15Iq1Hc3DDKhPwc0couY1gD2Un8KP3nYYrH/tK4RipTBH+0ad/asw6hx9GzU2HzGpukbjrR9mLckVk/2w3q35/wD1qP7Yb1b8/wD61VYXMahtFHYj6Gj7OR92Q59SM4rK/tlv7zf99Uo1iQ9Nx/4F/wDWosw5i/JbuwOUBPqOKi8ry/vRsKr/ANqzDqjUqapIzAYkGfei5JeilKAmKaSI+u4jNWHvb9bRgl6ZFIwQwzkVTLvIME5z60LG6jpn6Ucw9OxR02RSHhkQPg5ANOuLSeaPYo2jORxmrMFgEmMwBGc8YwBVsIQeSB9TVc4ciObksLgHA2n8cfzqF7S6QjzIXAPcDP8AKusAAH8B+ppCinsg+hIo5xciOVa1cRGQHIHUYwRVZsV15tkOcvweozwf0qpNpFtLnBjH6VSlclxOZyKMgnFbx0ODqzkAd1YH+dS2ei24naRJS5A+6wwDTvclo54fLXR6UYTbZiUhzjcD39Kz7tYxeErbKuwkHPelhlaK4DphR3UnjHeqTCxqywwqDIIIwR7UttMrAjykGO4FOEiXERKEEYzwc1UtWIuCvqKloVi9NMIoSSFx9K52c7nL+tamqziOMRkdRnNYzOCuBUAKDxWnolmL+4YO4GOgI61m2zqtxGXGVB5FemwQwvaxyQwoMgcgYNaREZsemqgACA/WoNVsHS03hAAOuK3PLb0qO4tjLayAjtmqEccG5VvQ5rrbaOKSBHxnIFccPvEehxXU6KXksR3wcU2xGjGI0P3Bj3FNMYikMtqBGxOSOxpcEdqQkikIx/ESC7sd7DZNH1A6Ee/5U/w1Ij6aYicEEmr15aJe2rxklXxwR1rn9BkNldTQy8H1/OgZt3d2bckGnWoDgTEckHBrAvWe71QqTlAQRWnD9pjIQfd9KBmoW5ppYetL5cmMkEU0oc9KBBuFAYGmlGHagKR1FPQRna1pKX9uWXIdeQQetcR5bwsVZDkHGBXpYB6Vy2tQCzuGkcfK5wD60mUmYJmVlIKgfSnm0SS3MgcjHYVoR2tlNbGYEEgZIPHPpWZsLAqpIHpnpQMp7ctgVaeykWHzRtKj0605LSQNkDNXgXaAx4C+4pAZMTqjZKhvrVmKZDKp8pRg5qjIvlyFc5xU1uyeaATjNAHV2esWa7YjEUPqOa2Y5Y5RujORXL2/2eGQOwBHcmuotpoZoQYdgx1ANMDM1sbBG/0quTmWJx0IxWpqdv8AaLNgBkjpXLW985nSGQ9GwaaYHaIiSW/7xQR3Brnr+awj3pDEUcjBwK2ZLuK3tgXYEHsDXJTRSTXLmNgEY8ZpAZTgByQaauQeatXdpNb4JAIJwSD0qIRknngetJoCaC6ltzmLrU39sXRUggjPcGqoKL/ED9KfG0QOe9QwJWvb3qJmFNN3cSjEs7EelOzGepwKa6R7cowJHXFK4zF5jAOQSR0HaoSOc0uaUcgirAaMd6erKoxmoyAp5NNzk0AWNy9jTWfg8E/SoxnIqVQOCetAAAcc03B74Apzkr0Bx61HnJ5oAeqgnaDn3q3GVTGcgeoqoSI14HNSRB2j+8QO9ADpW8yTgcDv61DIdvQEntUmQGIHapCQYwAAKAHQECMM/B9KroI3kYgZIPWnuGcKgOBnFKsIhO09RQBIg3MKuxqOgFVYwOMVfjQAAetJgRz48kgkgGs8AR5CcjoRmr1wQXAB4A6VVbB4pIZEHMajgkk9KbIVbAIxxmpZEGAdwAGMA0pAA3cMaYEMJUzKgFJcQsZz0A96nijDXAJAGPSpLqM+Zk8fWi4ivDYqRlpgPYVJNCsEZXYD7mm/8sCQHx2I6GoVViufNf6E5pAOER2ggjnnFRnIODQZh0Y5FPEaEg5I9qbAWFCzMT0A4oD46GnggA49KjAOcCgLBnGaC2RilOCOnNKkTOSAMii4DtsTp8pIccn3pinHHelxsOQcH2NMJ5J70xFliDDgMMjtUSgDPOc1HkmnROEDEjJJ70gG9GyRS0Fs44FSAAsMcA8Gi42AGVOOvb6VIXkWASIxBBx+FBQLjBIBPAqXaot9pxknjJoAgjmMoJcuQORmnYBIPXikaJz0IA7mp0YRqQSGyMAEYpMBQ5eMptPA6gVA4BTB7UGWRQRyAR0FAI2ZNCGRsBIQAcADHNOUKcgliQRkk1HuG8Egnmnlw4IwOTQBLI+CoBGMdB2qKSTe+TycYzTMYOaaFy4BOAT1x0piHV0Gkx7IR69KxHh245zk44610VhHtjQUwZX8SORb20YP3mJx64FYCRq4OWAk9zWn4llzfxR5zsTOPTNYqk7waQieFYzw4JKnJI7ChhHhjnBHPTigkgEgkE4Bx3pigDg9KAEWNnOFGSOalVS2AOucYpEYI+QcHBFNTO8nJBxwQcYNAFmS3Yxpn7wGDxT7i4j+zLGGzgZFSQTy4jkBycEEAcms2baCu33/AJ0ARDknmpwMAY7ioNpBIxyBk1OpyBjtwaYy1CM4qZ3GQFYHAwR6U2IDbn2zUTj96SjZLZOcVIy8g3RhuhpLS6wZ1kjEgyAST2z0qsZnWEAgc9zTrN4wyI6jGQXY9M9qEDJ9RuJrgeYkTJCowuOgPqTXd6G1xe6RbyyvhyoDKx59q4c3a2NwLS7hZ0Rw4CtjcOoPpiu5g1u0ukSWyaAuwy0LPsfPoM/jV8xDLEyPGyEdzg/Sq0mk2l7IGuI90gBVWz0B9quXEgZrcFDEXYjDnnOPakMy288hfOFXdgDJPtVICpcXAsraRryJdsIAUg9T2xXHbY9Vupru9uPLIGQMZBFM1eO+tbxYr/eJJlMoBckgEnt+FZsjjBUng1LYIkkYTSM6vtGM8jrT2v7prAWSykw5zsHAJ96o5G7BFSqwxxSuMsyq8McIDgg4wB2rotNu7WwuJXMwCuo3BDkE89q5hZGVPLYgKQCARmhHAkAHIPUepoTA9FjuIZmVY5kZmBIAOSRUlefQ3BtryGZTghgfwzXVa/r0cNtFDpxRmnGTJz8o46HsetVchmvRUXhx3vdEhklHKkruzycHqa0jbKe1VcZRAYng0FZQM44q8LJsZUig2sg9D+NK4WMucF4HVxkEHiooZkhtDhi+0EnPr6VsC0l7xqfqao3FnJbedKyAIRkAEYJ9DRcLEBZrq0cgNFkcEEE49awIrgBxKTGZFOAe/Uf4Vv8A2WGe0MyTMJAM5B4BxnH0rmpbTyjG6yEx5AMhHBI7gVnMaLcNzCmoieXhCdxVRkZ+lW7u5tp0Mv2ssQCQojIIOO9ZEh8y/O9wEIGGPTFS7zGHELo4wQSehGDxWKkyiXTcrpTkHGSas20sdvGJN8jb5NoZ+mPpVW3+TQ3PqSP1qWC4jUxPKnmFTkKTge5/DitHKwG7PHDBHEdxG8lmJ6/QVJEIGw4EiEdnH9Kwr++N/dQhBtRDyCcHFatveC7Z8LhYwBnOQauE02S0Yzrd6fcl5IhIxYlpAcgoeg9ucVz97dyo8sBiWNgSCV9D711N7q+yFJLVUlAP7xWByB7Vy188V1dvKkZWNjnBPNNyCxngGQHOcDrUTngkdDTpHKMQDxUJOR75qUUKhwDk0hbnJFMzQTVATAjIJp5OBx0qEEYoyfWhoBzOp5P5U0HHSmk55pKVgJCxp0UzwzRzRnDxnINRgU7acUAdjpN0kd6CD/o90N6juG75/Gr1orXLm9uGYsSyqgPyheR09xXJ6XIXtpLXdiRD5sRxyCOoFdLZPd3sS3NtcR4cYaGQcKRwee1QkXe6uS2t0un3d3bO7/Z0USpvJJXPBA/SsXUL94Ue9bi4uMrEh4Kr3P8ASrN1LvuGSebfHEd88gGM4/hA7CubvLl7u5aZxjPCqDwAOgFOxTbS06lcE49frUhIwMelR96M8Y71RkKWJpyuwGAab1FPRSBwaAJkB2jPWlLYp4Q8EEGnBFIORyO1SMRZQTjABAwD6inb2Ugg5prAEYI5HtihRkAUMRdSXIPH5UTt5sMkTgMCMo7dR9KgIZQAO5waSRyAqk5AOQRUtCNFYrSSESIpjIGwAHjI6H61TMpCiN2+Ykhhj06U0kRw4JI+hqEEMAQcmkkBctbkWRJjkkBIwQhwD6Uo1e+EXkidjHknBOTk9eTVAtzTN1WkBtz3AuNMAQMQCSwJ7gGoru6iW1tzAqlcncrDJGO9Zwl2REBiMnJAPBNMZgTgDHrz1NNAWBNsnMsJ2nOR3xTop7iWXmQbtwcMeCCOlVQ2KfG3zj65pgbU+t3tzGbee4BLkBiRjAFdNZalY6Zp0drEZLgx5wcAA5561w4jBLSEAnvzVn+1bzP/AB6R/maAOmuvEV9ICLa1hjz0Zjkj+n6Vm/a7yXzZr6USbTlc9AB6Dt2rL/tW7I/49Iz+Jpsmo3ckMiGzRQVIJD9B60AbSsjKG2qQRkHHakLIP+Wa/lWQmp3SRKotUkAAAJYjgVH/AGxc+ZsFkrN6K+TTbA15bmO3iMkgcIOpAzUA13TcfeYH/cIqCaSea3K3Ait424J5JPtiq4W2gLGG3E0h/jk4Gfp2pXHY0P7c08niRj/wGqEZAJSJzKWOQA2cD+lIHtzJvaFreY9XjGVx9KlMdweYzHMp7x4z+VFwsJJAEXN1JBCo6gkM5/CqF5NatCBboxIP+sY8kemKnCxuSMLkdQcZFVb1FjRSAoyccCgDSstWFtHGqyywNjGQdysR3I7f/XroItdYoWNulxjvbuMn6g9K5KyRXt3B7H0qzHGsJ3x/K3qBimmJo6AeKrXkC0mBAOQSAQa6W0G60t5B0aJSPyrzxwZCSxyT3wBU9vf3tqAkF5KqDohOR+VAWPR1we9O28ZHB9a4uDxVeRqfOt4pG9RkfpRZ+KbqO5maQK4f7qE4C49DUsBfEU0tr4t09xKodEGCw4GSRz7YrpzdC22C7lgBf7hRDk/hzXn+rXdzqV8ZZypmUbQF6Aen610mpa2ZGtPsGwqcAlwDgjHB9PrS5gOsK/uyaYTtXNcQ/iC9sLnzFMb7wCQGJA+npXTWOppqlkLiNShJwwJyc0J3Av8AmCqPiC6EPhq9bAcMm0g9CCcVIZCKyPE0wHhu7BPXaB9c1YHE6HcLaa5YTPxGkwJPoOlerzyuTkHg9K8a9PqK9XEwMMeWz8i/yFJgSmUjqaBID3quzAjrTMkdDTAuiT1NTxOCMZ5rNy1TRORQBoZ+brTscfeFUvNNHm+tKwFrviopF5B7Z5pqyHNK78ZOaEgCRVVAVOeaWDl8U3yy68U5IWHzBhx2pgQXL/6SwAAx6CnwuWGDU7Jbpbu7xbpCcgVmXVz9is/OIIY8KDxk0Mkmu9YsrCYRzGQnGTsXOPrSJ4r00KBi4P8A2z/+vXFzXpFyzAgqTkgjOT70+G6tZsFWEUg6oTgEegqWaxR1d34sj2kWlqzMO8nAz9BWBf6lqF/GwmkPl9QgGADVQajAWCiJsk4HIpHv4lJDQkEHBBYVFy7RKLLICeMVCxy+GBHoc1bkvInBOzj2IqnvV3AB5J700N6EWcoWxjBHapypEyYAIODmm4ASVD1q0nMaH2o2FcdeIJFRjzgVpWUFvFZBzEmQCSSM1RYZh57Vo2o8yxZfUH+RpkjFvYxKI3CqD90gZBqLUIYwqyBAGzgkVQsmkgEM1yVaHlFJ/gPbNat8M2bH0INUJGN9nIckkkHnHpUot0IyVP5VJ1WrKDMa854rNFSMi4hChdo7kGoAuHBOevrWndoNpAIODnNUHAwCCDg1Qi+qjgkAVTvUAnAB6jOKvpyoJ7iob0A7Ceg/OoT1LKdoyJJIHfbkccZq7b3FvGxMkyn0wMVCIR6CpEhGegp3FYdNsluC8XMZA5oMKsMEVIABSnilcLFM2JboCKifT2XoQPrW1CARmnPGD1GK0IOdNu4POKfBDiTBP5VcdAHIPODUYAWQECgE9R7wkJlCSaeiYjAYc1MAMUuKybNCKRpo03pIQAcHIziohc3bf8tv0q6Yw9u49DVZIggIznNFxCBr1hkSjHuRSYvD/wAtBnHqKWRBkHFFsoFwD2Ip2JuNxef89B+YpGN0uMyD8CK0DCpP3RUE8YBzgD6U7AnqUzJcf89ab51x2lqcoCCCKYY1VeBSRTRq2FgLiON5EdX5yXOAarz3dzHMYLGEE5wCOc/Tiuu0kTXWkwnaoUDGSauxWIVgQqZ9QBVJENnCPpmt3KDzrXHtjBrSj8GxSRqXvXjYjlSoyD6V2Mlq7xk5Ax61nuSp2mRcjtkVoiGYdzYxabCVa3wvaRBwT71ihsXmR0PSu3CsVIdQwPY9DXC65BJZ38kY4UgED2osK5n6tN512QDlVGBVJSR1NDEs2TShFMoB6VLQ07jieK7rwbqIls2tZG5Q5X2H+RXHtDEIwwIP0NWfD2o/2dqikj92+AeenP8A9ektCrHpRdQeKehDAgHrxUJUHBBBBGQacmAeaok4nUYzDqsqnuc1r+HJAJpkJ69P1qt4kiEd+sw6uADUWizbNTiGcBjirQjq2BzjFROCDzWi1oTkhwRULQKv3iD9KlMVipH1rl/EUBtdSSROBJyT78V14iGflFc/4vtnFtDcHoDjH+fpTuKxAbdY2hkByW5PFbQAEgrOjhM1pbMOwArT8piQcUXGywZlIxmgAEZFVWAicNI6oPc4pp1vTLcEPeRgjtk0DLhGKY2BWfJ4o0jBInZsdghNRjxPpT/KWkGe5UgUDNDIzWP4kt/tGnlgQGU5GT16VcTVdOkICXcZJ9ciotUiS7smCSRtjp81Ik4iCTYpBAwcg1AXYOSCQQafJ8hKjjBxVZidx5/OmUi4t/Mq43mozfzHocfSoI0MjY5/A1cXTd3Qj8XFSMouwZsmgLzkVfk0lwuQ6jHqaptGYmwSp+hzQAAZ/ib8609Ov3sQMHcOwJ6VmA4pxfNAHfaZqMV7EMAhx1BrltVthBqh24CscjBzWWJpB0dh9DipIneVgZJGY+pOaEwaNCQMEBBJHcE1XMsI+7KVk9CaHWaNSVlUj0qjJzJkgDPoKLisTS3TuMFifrUBmbpmrYs1ePdvA+tZznDkUXCxJuB5PWnCRQcioRycEHFP8sdhilYCYzFhgD8aElKdBmoR8tG496VkBngHGTSqTSsR0GaQDPSqGKQAecZoHJ4ApmMvg9KkGAcCgBGBJBHapXOEzTDwaJCCAB0oAbG5IwxJB7EdaGAQnoPQCo0wTx2ocljk0xkkZywBAIqUhgOOlQLkcipVlYYyc4pCFTcQcqRUkabgRkDHPNIs6HIIPIxUiQhyMMBmkwGBcc5BI9KcBk59accRhlwT7iiNSEBwTk9qAJYAfMHFXQcDJ6Co44wFBxzUj/LCTgEnsaTYyhI252YZwT3poKlSxPI7VGJixIwAQabjGT6mhCEIBfJAP1qwgTYCarZyM4NIdzqQASB2oGXIlYTK4HGRg1NcSCefYTkCkt5ItgREYEfxMeKq25jju/3x/dk80gJ0lNuwzxGTjaOgp91Zxou+Fg0ZGRilv7eKOUGGTcpGQM9KqYYdDj27UARGPHOwCkHFSu7EAHH4U1Iy4YjHHrVXAjL4GMU+MrwT1poUFiGz7YqRERSSQST0obEMOCcgED3pwZgDgkcdqdIvyjA5J5piIGkCt070hjSmQChJPoafHDJJnCnjg5qUgLkKpx2NPAeMkcZIyc0AQfZpD2A+ppHgeMAkqQeuDnFW1kAQtLjFQGTa5CscdqaEVQpBqVcj2FWd4K7SMio2jKgYyQRRcYg4ZHXkjtUrt5oIIGD2FV1kCjBBz6VIjsTkcD3qbiDLFccgelKowRyefWhiR0zz60biCCAPxpiHjKk4AOR3qA++MZ7VISTzTBgnbkA+9NDI2wegP405CyxnGAScZIp75AGMHHpTCegNIBVCMSDnAHalYIcFeCeuabn2FFAEsOWnRfUiuoskxtrm7JDJdpjtXU2gwoPoM1YjlNYJn1id8EgYUY7ACoIlSIEEE/WnO/mXcz4xucnFRSPzgdaQxpyWJJyOoFGd3PeoyWJJoTJNAEq4yM9O9SEAjcvGRgVCeMe9SxuSCMEgc8DtQBZBMaKQcY7U1yLhwAAABgACqYcmXJPFTqW/gfn0xUgQMHALHlmJHFPhUjGetWowY9wJUOR0PeooxyaaYFkHEBb0FRJK0gEeVHB570673R2RwepAqtCcEMaQy3IAbYAEEqMDI61a06xju7i2t34Mko3HGSAOabAYZIxgZZuuRgcelaNmkcLLc+dtkjPCKcE+vNCJL2qiaDVp7gwwRxxoI4mkGSQOgA7ZrlZpJgSkg2sHJIwAQfrXpB04XdnCJX8xhIJS3UEDsfzrmPFmluLg3Nun7mNAG7knJJ/DmqsSNGrTXnh9p7i4VruCZBbr3UDjPvkVt6T4hgv7wtcqI7hYuYwcggc56e1efhninWM5jYjJBHOO1SLMYHkeOQhnGwkdcU3oCRPqmp3Gr6pLeSkFQcJgYAA6VXiGXztDY7GoYSFIUnCnqaczlGIQ4PrSZVh24IXKgAtwfakzzkAke9MU5FTqQIuep4pAWEjeeAuYmACkE9xVVTg5p/2lwmxZCAeuO9MBApoCckFQD1I496RXYcqSOxBqFXIYc808qRkk5yT07UgNTT9TureFreO42QuRkHoOev613P8Aadrpun24uJy5KcEDOR9a84jYxhWBwQc1ox29xc24uEjHkt0y38vagRvXI1K4knu7e5nVSQYwDgEEV0GiPK+lRC5f/SASCGPzEepFcYl7d/YzB5sj5GAQOAPrW7pkI0uz/tC7kYSMACZOpz2pgbS/bY4whKZQkly5zgnPSucvru1dkha+eaNSS64JyeOn61ZHiOG5hu4piAp4jcDGQc9RXLkRxfJGzOR3HSocijZOpGC1kgsQFLdSRg4qaw05NV06SJZyJ0wS7AkA+gFc4kiLOJZAXIBGC1aqaxDHBH9njkhkAwQHGD65FRzCsVLu3eG8eCZx+5yGIPB9KfP5IhZYVYELkknOc1Gbu3DSEQuJHOSMg5psl7EI2hit3DOCDJIeAPYUorUs0bOPztKEe4LkE5PQYqtbNGd6yA9cEryBj0qxAEGlxiT7pIB/E0TWNxau5tinllACWAGR7VcySL7RFHMZGMkhHBzxkU/Vb5U0V0iXbmQCMA4JGM5PrTILcEyG6Ty8kANg4/z0p19YJczo0Y3x4wHU4GKmLsJnOlHDE7yM9TnHNOLhhjIHFNneIO6qSUBIBJ64pgDqAfLIBGRkYyKrcCKZEOWJ5+lVmABq3IcREEDJ6mqTHmqQCEDOc02lNJnmrQx4opBSmmAzvTqUDJoIoAVcU7OOlMHpThg0ASwSvBMkyHDIcj/Cuhju5rVpBbcpdAFCOME5zg+3Nc1itWxuZItHuXGC0J/dkjJBPX+dTJDjKxFqlwABYRHKRnMjdmf/AOtWcf6Uf4Z/GjtRYbdwpaSjNMkWnxnBzjNRipEAKnnFICyjkAe1SCQEZAAOevrVQNxk9adESSeeAKTQFhskc5+opEYrwevv2ppcqpJBPsKYpBBcAAk9M9KQFtWycZ7frQcNndnI6cdaqmXEm41IJFcHk4I7daLCGzyFsAklR3xUAcg5HWmySF8k5x2OKcFQDgkn3qkgFWTDgkDGOtOJ4z70wgDGSDz2NKWBGAMUwELnGM8UZphz2pw+7mgaJFY9+1SKecjqKYMEA9+9JkjpSsBP5jetKJHPen20PmkcVeFmgFSMphpl4WQAem3/AOvSSPL5bZkGCCCAMZFX/si1DdW6pbSMOoUkU7hYrW7F/L3hmjwcoOM/jV2JpGyIY0hQdlHJ+ppmmoHtASM8kfyq9boB5mBjn+gpXKsVDauWznJ9WGaXBjJUuMjqAmavP8iM/YDJpqNbR4UToMjJJ6k0ribKeDKwAdQT2ZMGozaOk+UIU+o4q9M1tJlZLiM4GQc4INIh3iNz1I5piuUZp5MgXUKzgdGxhgPqKz7pywAydu7gN1A9K3ZYQ5GRWbqluI7dSRjJppjKttIUjOGAyT1Gc1P58n/PQf8AfBqOwVGkKt6cfWtD7IOwouFikbqQH/WJ/wB8GgTynkOhH+6av/ZAB0FILMHqKLhYomWYgjfH+RpA7+v5Vfa0THQA+5qhMhiYgkEeoNS3cTFL4PrWnYxiWBj5mxXIQZPQk8k1jBuQanSZhGqHlR2qUSb91p3k6a6CEqYiSs28YIPrW34cisZNOCRymOYH95vOCSe+K5i0u4U09oELhy44IznnNXbe/nvZxDcOghjJYvGoBBHQ5HWqvYDsmsFA/wBYxrC8XQvF4cYx5IMq7snoMn+uKveHZpLhLgAyyQhv3cknH4Uzxp5yeHfJhQN9okEZz6cn+laJgeZucRE969bsNLW5020na4A3woeB7D3ryQjcyrnGcAk9M16L4X1J10S3szMJpBO0YweiDB/Lmk9QOhTR4AMmeQ/gKilsIoxkEnHY1bFyFjIziqz3CnIJzQBAEUHOKacA8U8sp4zjPemNHgZD5HsaaEId47UoMhGdpNIFYnAJJqXbIAMqRn1oEIGx1p4nAABAODmoWRycjA/GgBh1ANAyYXIB5pwuQe9VmUN2A+lAiIPBzQIuGUCqOoWsOqGMXEkiBOnlnBP+cU4wzAE5z7VWS7hfPmOsZBxgnOaAMfWPD1pbWgmhuJm5wQx4wf8A9VZsOhWsmgzX5d/OViAM8YzjFdLf3NvNYvCjb2PTjisWxkI0nUrYgFQA2COhz/8AWoNLmdJbK0EMnqAMjrVc2UbXRjYscYwSa1EAOlISM4Ix+dUnfN3vx2Ax2NQUmUJ7ZYpCvXHeoohm5VR65q9d8zSMPrVIHy5lkAB5wQfSkUXo8EXAOORkE9qfHgIADkDvTItpkckgDHBJp6H5eSD9KLCJ2OLcn3Aq5pr7oWX0OKpkg2zZIGPWp9JkjLyAMD34/GmIaLkOJYI7FpIw2GG4YJq1dAG1dQMDAIFZ2YYNUn824ljBO5QOAfX+lae5JbcuhypBwfWqJRmgZWp4y2wbQDx3FQA/LRHexQEq+4HPpWaLZK8bls7V+mOKieI7SRBGcDPSnHUbcnO5h+FNN9bspG9xkY4FMQyI5UHAB9BTpdpjO9Cw7VHEUwQrFh2JFSvgxEEkA9xWXUsYOUFKgJcKOp6U0EBQAQR60qHEyHPeqAsCEj71TiO3CgOjE9yDQ5wpPoM1izXsjuSuVU/w+lMi5eh1IQblMJIycE1FNqG+fKfc9KzDM5bBJwaUPtFWmRc3oI7S8Bfy5A3ceYBmi5isrRAzwucnAAkFYtvevC4IwcdiKWa+kkkLEAg9utO4Gwj2DjKxSfTfzUxitQoPlzc9t4rDExyGBH0AxV6C9aQLGVJx0IrNq5aZeUWu0jyZOe++mmK3XBMc2D0O8c1HISgyKnZ82qtgHoORRylcxEUtD1jm/wC+xT44LJcP5Up+r0ssSCLIGCaSP5oWH0qrGbLRFscELJgjsabLHatCQyyEAe1CYKADsMUrAGM5OOKYFMQWhOP335imvb2gOP335inQnM2PWiYYkNTYq5Zs7+WxHl29xcBSchDgjNWZPEF4DhpZQfZBmqEJGBn1qSXOcjjNWiGSyXb3QzJdXpz2BwKiWzteoluA3ruxSZJhJJzSJkgGmmInBMYwl/dADtvFVL9jdeWJriSQLwM9ef8A9VOk+8arSjgn0p3Cwj6Ppg533Az6YNVdSsbWGzWaAMSp6sMHFaUozCj+oqG6jMmmSADJ2nApMSMeIRyJlXyfTHWqylVuVL/dBGRSWDbbhVPcEUsqgzSZHekVc9TsWWWyiMIyuBgVajjYuBtI/CvNbW/uYoQI72SMegPFWBreorwL+Wi4mjqvFNpiySfqUOPp0rm7Wbyp4pScAHmopNTvrkbLi7kkQ9VY5FEQBTFUmD0PRra+trm3BjmQnGMA0jjJ615tHeXFhMWt5Nh9cZq5H4u1aKQFpI5R6OOKkDvkwpyxAA6k9KxvE93YXGkSwi6j85eUGc5OK5PUfEV3qRAZBEMYIQnBrMC256yfvO4NIDbsPEZtLBIvspdk6EdDUU3irVJOECQj2GT+dYHWTbV9LQBeGJz61Vx2IJbm4mJ82dmz1Bc4qHCDqQPxq1LCIhkxBh3IHSo1aFuhUex4NO5XKMDoD97NSLMh6H9KcdqjICmomcE9AB6Ck2BKJFzkqCfekM7j7mAPQGoC4wcU0N61F2Kw45JJPJJqNgpPIz+NXLaNJDjrUjWg3EICPYijnYWKcBiVgDEX+hq0RbHkQlT6k5p3kiL76gA+nFKVhPQkH60rsqwzyWk6IpHrvNP+wW4H7xip9BShlUYGfzoLr1LAfU0XYWIHtIWGIi2fU1EbV16kVdCAjIcfgajY4OCSaLiaFt7eE/fBJ9jUkyQhAUUgjvTElK8j8RTzcZ7AUXY0l1I42LDkEYpHto5BlQQ3tQXweMClEmO9O4rEaWkgGPmP16VHJaSckAY+tWDO2chzR5hI60rhYqKEUAPwakNvI4yiZH1qQReY44zmtO3AjQAcU7CsYDoVOCMH0pgQlgAK0Liym8/zCQU9RQI1A5UCmSYS4OcAk+1ABHBBBqRJBCSFx09KQtuOSOaoYipzkjimvgNx0qQsoXGOfXNQt14NAAxBHB5pFVie1CgscAZpwGCQc5oAkSAEHnHHrVc/IxQcgd6nWUxuCOaUIJW3BQCeuKAIUWR2AWMn6UrIyHDDB9KtcpE2HIA7DvVXcWIzSAcgKMGIBGauIYXzvJBAyMdqgkAVBjqRmkTjnuaAHsM8VZh3BQBVdcE1ZiIGMkD60gLUZwMH1omJcZHQdaVUyRjoahmkCZHWpYyvHboS5GADzUBXBxyafIxAyOvanQq7xluOB3poQ61jLSiMFdzHGM9Ku2+mDEivKRJkkIBgfzrMt2kS+jkHUHitK9v5I5leMAMOpPOcVLYys7SW7GORAUHrVaQqzEgYBrVmMV9a+cuBL3BNV7e3eN1lCgheoPTFCYFRHBGCaCwFT6kR5yFFAUjjAqiWqwHsQeaWNwDwcZ61EzEjFNB5xTsIsuqhjyD9KaoLMAPWm54oDYpWAsKis+M9e9MMYEpwc03ee1OVwOuc0rASh9gxmhm8xs9TjFRFwRinKxA4pARSsRgEYBNMGAQRzzkj2q0FypPpUO9cDghgeT2qkAhd+cLgH2qaMMyYJGPShHRIgXUlh0YdKnjiVodwOCTnpmgCs6hR8wqB3AU7TVqRRICCcHsMdapSQlSQelAFsRzvGHK4GOM1EWwcE80eaQgAY8epp3nI7ZOC/wBKAEw2CQOPWmZ5B9BTmbIODio8+ppiHFs8DrSAE5wKQEDrT40eQnGMYzycUWGNGe9KGIPAGPenHGMYOe/tQACDzgjtTEaGlgGQEc461vmQ29k8o/hUnNYukRlUOR1Jx9K0dXkEOjMveQgD6d6YHLICCCD3zS5zu+XJPQ+lSsVOAFwMU1ACQQeM1LGgFshjzvPIz0qqo2kjIODjIrTEahSA4ye+elZ2wrKwPAB60wH4wAPWnCVkTy1wMelPICkIxIBGcmo1ABPOfxpDsRk5YkgA+1PhkCTIWJwDninqils9SacbcgqRg5B4B6UxE9w0bxiVVIJPeoofWiRWSEDJ5NTQqMgAdakdhL0EwKgGW64qhEwWTDHABxmrt6+2RhjJxgH0qqlpJIhYY4GcE9aYF6El1zECV7ECtK3tiYo5iwcHOSOMCs2zjkhtlxIBnrg9RWkt0G2riTaAQcHj8qliaNK6M1rZApesVcfdByc9hVWbWbuSAQxgJGEKsH53D0rPlkk8oqmODkMTzUMMjO21jkGjmZNjS16dX+wXCIgZkBJKkMSOx9vSsy+NvLb7II1L5yZDwxPf6Cmajf3F4Q07g+WcBQMACq6SYqk7jRAQRkCmgnOCOtWWw6MoHOM/jUOzPTg0wHIoJ+lSDBJTjB9T/KiKMjJxyB1qPloi5IBB6dKQF65ujcQxRyLGFjGAEGMn1NUwrHoKhDY7irCSAKBihDGsPLAycknnFOUgAZz+FN5dwCMZ705QQc5GB3NAkiXOEGK1dLvcvBbICQeME/L3PIxUNgkJtw8kO+RznBHAH9KsmEuVCoiAHjaefpSuaKIkU2JHtyjABiCYxwPxp1zJcTuqzTSTIhyoc8ClCeWoAABHpUbAgHJ+bBIboBWbY+QJIwEBAwT3JzTOQSCuR/OoBc+awAYk9smnTXGwEEgY7VAWFYRkExqVAHQnJxTA4XOOAelRq7sA6DgjOfUUIQ7gEgZPJP60WJHOyqyEBg23Oc05RmLzvcjNPnjJnUxj92DtznpTnLKCmQR6gVUdyjSnB/sOIKMksDj8abDqRW1EVyhlMbAKQQCB2xUsxC6PHk9MY+uKowtggkAk9SauTsSka9leaam4zTTy5OfKZRii3tYrjUBcKcQr1jCEAk+g9hWfGUFwHYEn0HNMu9Vawu4hDvJA3Oo4Bz0B/WpTuKSsReIdMtLa5gNpIywyZDJ/dP8Ak1lXV0ZZEBGAihQM54ApLm7NxKZGGHJz1yBUYKkZPX1q0iRrSKcgrnI4OcVWkBDHI4FTuQSpQEDGeajk5JJ781SAhAyeuKXFA60Dk1RQGgCgikFMQooJoBxmkNIAzQDRiimA4GtGM7PD1w2PvzAfgCKze1abDb4bj/25j/WmBnZ4pDkClxgU0mkAuaAKSlApDFzTkYgc9qaBmlJwABQApwzkDpmrERVYyxPHQVBCM/nUhIKYJJIORzQBMHD3EUYJwx2n8TUTRIsjhnAAJAP0pkZxLGScYcZ/Oln4upgeoc5H40rEjDnPJzz19aVSeg/GmEd8j6CnAheoByKLAPbptJGR6UiIHcKHAJ55phH7wgcDNGCDgEZHemA91KuRgkdjjrSAntTyxKAByeeRTNpIbJAOcD3oAQdKeEO0Ed6YoGOTg9/pU7GMRMQCcDA70DQwnAA7gc0KeaZUiA7C2CQDg0MDQtJ44UO8qM+ta4wygg5BGc1jafGs06BxkZzittiFxhT+FZjGAc1W1MbbJmHbgj2IIq4Dn+FvyqC/AbTLjIPC5GR3oTKINGwbJgeoc/yFXouHl+o/lWboRzBMO+Qf6VpQ8SSfX+lIB0g/cSfSnPGn7z5Y+nrQRuUj1pht1xlnKj1KjApoTFVECwnEfI9aiiGUj/H+ZqvAUlhBUk4JBwM9zSSKj4DzkIBjYhwev6UxWLc0tvEQJJlHqOprI1a6jmjjijbOHBJ6VaHlRn9zGOf4iMn86zdTcmVN4yw5B4GB6UwDS3CX25jgEEVvhcjK9PWuaswDNknGBkfWtWJpkAKNwOx6UijR20YwCeAQM81At0VA8yMgY6g5zUy3ELjGQcjGDwaLA2ZM80sk+8MBgZwp4qs6yOBhCPXPrVq6hS0+YOCGBAAHQVWUgrkGhol6kfSrmnIZr2OAAHzDsORnANUGYBiBzUtsXNxGI/vEgA9gcipaFY2bvSTYTC3kuIzIxJUdMjjFNsLprBpFMIkLMAcHkD8qijutuovNqKtM6ggHPAPaojeiWaQrCqq5BAzkADtQI9JtFE1lFLHIIxIuSoIIBrF8UXsA0Wa2F4j3AZXVQcnIz0Pbg1y0AuZFJWUBM4wH2iiSzkEMmAn3D0cGtIsdjHmlEs/mDjkHHvjFb/hzVodPjmM1u8jMcBkPIHcVzAODWxpkcjWxKrkEnnIp3EdV/wAJTbZ4tbj8SKt2+u6bcbQ0rQue0gwB+IrmBBMf4P1FIbWQ9UH4sKTHY7hYIZIvNS43Ie4ORTjZiS3JSbaT0cHgGuFEE8WSmVB6lXH+NWLTU7uyyI5gVJyVkwRSTFY6m1M1l5wnk80AblbPJ9f6VaXUGktFZYVO4ZAL84744riJtSee4EoLRSE4LKx6VdW7uorVnN05ByDkZwaV7hY6uOUSIrjuMkdwfQ09W35284OD9a5/SZInhMKaiwkY5K7RnPtk042t3IHIvX83GVIOAT71SY7G9s55U/lUcseVwrFT6isTGtRRLtu5DzyA2cD1olOsLOsL3HysMiUDIBxnFAcpreW4tJEkmd8jkk4rnBGD1qe7l1u0gM0l4pUHBwAT/KqsN/qclyse6KUkZCuo5FK5XKTxDEgXsaxr5Stwf1+lbSx3RnUzRCNycgA5Bx6VlaoGjmyUJ+lCYNWLlofM0cjHQ4rMIIJFXdKuY2sp42YJhsYY4PSs+W5MbkBAw7Ec5pDQN90g9xiqx4q9bW15qMTy29vuVOvIBJ9BnrVGQ4IPvjHcGk0FycRKwBI5xUqqEGBwKqG7ZMKEU475qyJAyg9M0irloc28g9Qak0pVW6IAxkGoITuRx6g1DpNxIb9FJ4PWqJNa4uJhffZ0MCZXcDIOvtViIOUHmbdx67elU9Rty97FN9nNxGFwVGAQe1W7eYyIM27w7QBh8c1RBTmsZbWESMUKk8YPP5VV4zyAfqK1dXkdrKMGNcB85/CsA3UazDcfl71lJGqdywZIVOGZAfcUyQowDRupIPasyaVTMSDwTmpEkhKhDMEJ74zipsxXLYlx6D6VILhiMZFZ0pCEGObzQfQdKIzICCFb8jSsw5i+LiIHDsQfpmp7ZBdTLHBKN5OQH4B/GsmRtz5II+oqxZSGO4SQdRTsw5jQ1C6ubSYwTIoYjII6EVjvMXOSoIPUHoa7DU4ItS0yQzIpYJvR1PIOM1xVlEZr2G2lkMYJ5YdqskdI6schQp9BTM+taur6LHpvlzQ3ouFckYI5Hv71mrBLL92Nj+FNCaGgL3OKcGHY5qQWU3dHH0pstrLGhbY5+ophYXJIwACPerEFx5CAKhJ9QapDzR1jkH1U09Cx4KsPwNSFjdhu4ng/ekhz6igXZCbBICvoUNZ9tGRw6EemRirYUDpTGmWBfjbhmyPZDTorqNm2D5QepIxVcAilKgjBANSOxqqyKmPOjP40okT/AJ6J+dZBgXsMfStWG1hksASgBPUii7CxAYTv3I8Q/GlaGVjk+Sfo1VjZckBgMeopotcnBI/OpuwLiwOOqr7YbNVbm5mikEZgVlHvimvaYBIfp2zUaxGSNiD0OOafNILFiOd2IAQDPvU4SX+4v/fX/wBaswxNECTKD7AU9IQ4yJCPanzMLF8xyHqgP/Av/rVG8LlSPK/WqjIIiMuTn0NTJamZMxyZ9Ru5/KjmYWHRmUx+UIw6jsWxTxHKUK+WiAgj79Z5UiTYeorSFnHNZMDkOQcEHkGjmZNjl3t5LK92yYySSCDkYpbv5ZQ3YjJNVyGjmIfzCynByOtXLtgbSNxzwM1bdwaCEgqMFT+tOLhG5j4HcVUtz+8HDflWmggKgu+w+hBqQGCWNiArc+hGKu25GRk1W8m2b/louanjAGParQmRXwwSR6VVit3kI461pyYLZIzQjoSBgD8KGERkVi5GAFHvkVDNoUjyGQShQeuK0AiE5Ip+4pwD8vpSKMyPRQhB87OPery24VQCc1ORQF91/OgZXcQx/wCsO0evWq8kenvyzqcfhU16uI85B+hrLc0wuSzGwEZMYJI7g1mvIM9eKkmOVNVCDmggkMnPHSlMny9CahIpybjgKQD2zRYZfsWiVwfMbPcVomQn/VuA3vWZb2dwsgdo8AdcHNagdOvlrUSLIZBKTiUgj1FR+Smf9b+lTPIkh5AA9BTCYk4xmoQmLsToCDSfZ0YbhIAfQ1C2DkgYxTGcCqRJaZ1QYCgEdxVZpN3NQvISMbjj0qMSsDwePpTsBcjm2dUDD3pk0oc5RSB6Zpm8FBxzURYqaqwXFLMOhJoHnE5I49abuoEv91j+FFguTlwPvDI9KcJQRkAj61XJ3GnggLz0pDuWhduBjIP4UoupG/izVBm+fIPFSI/zdaBXLqs/cnH1pWeIrhgePfFV/Ox6/hTDICc8ilcDIJweevrTiGAyBxTpIQmWJAycgU1mKwkA4yMVohC5BXJqMkBqRZAI8HOab1oAduIGR1pyyHOSSTjvQAMEGmHg4oAfkE8jj1qUMwjOzgEYpGiAtie/aohMxAU9BQBKpO3BPB600gBsDsacrACh13kEHjv70NDJVBkIJ5GMU4xANkdAOaaG2IBz+FOyTkk9RipERQkmcAcirxh3jIHIqpbqdx9jWijYIGCfpQMchaC2Jc8jpWa0xdiSat6ixEIAPLHArNyRQIkLE1OJRakDgow5yaqEmnqQ8ZQglhyMUWAtBMS71HcGo7yTM3PGQDSB5I05PT1qvNKXYE9hilYC1HOFxtUAAfnU/wBr/cnJCknpWUJG9aXzG9adgLM028Bew6Gq5OaTcT1oBFVYANHegHmlzQFx2TRmkxRzQIeHxTtwY1ATzQTQMmzgZpVlIPQVCDnqcUoIz1pCLQcnnpRMEkgwxwc9RUQPOKcDnrSGWYZkFsUAxsHGRmpYZ1SIIx5PNVVOAQO4pRgsAOopXGOdh5hxUFw+CExyRmrD844FRlHkOPLBx0bvRcCsrhBhhkVKTC4BUbSKSSPHBPPoKbHGCSCQPqaYCMTjII/GoScmlkBWQqfwpYwC+D6VSBChCxx27mnj93wMn8aCMdKkjAwSRnNDYWH2w3B2JAAFNJj2ErwQcUhIjBKjB+ppGUMFPcnpSEzf05SI14o8Qvi3toscsSR+lWLNMBQKyvEchOoQqDkomSPTJqkIyZpMSfKAAB0pY5wkeCoOT29KrsCWNPSJ3B+U4HFDRVyYyIQQCSO2RikLbhyTn1qMQyhxlCBnrSkhmIHTPFKwXFmPmsCxzgACnxhAuw5GevpTEhkdwuME+tSSRFAEY5YHGAOlFguPEeyQAEFe2KkjYBhnIGT1HWowCJBVx5hMgEjZwOCRwDUgVZyGmQgEc9KtWy5YVUIBnGCCAAARV+2XBz6A0MDMunJupMnIBwKkhMhQbSQScAeoqnISZmI7kn9acpYAMDyDTJNmMKUUFeSOTnpUDArKSSQOwqBJC8ZcgBh0BqdJt4xKmcYAINQ9AHGSLHCkEdCTmlWEvEZFBIHbPBqcmwwAIS7E4znAA9aRzCG2oGAHbOaAMmUks7MCCTyKVASV44zz9Ks3SR9SSM4wKrGTHA6+tUhhIxjmODxSeaByOtNkIYEnrUPWrAstMSBtyCByfeowCQQASfQUigk4HWpUwhz3pWsBGFI6jmlGRjHanO2WJNIH9KAAEl8k5Jq2oDKI2JGSM4qqOTntV1ZxGmFRd5GSxHQUmBoaY4IcuSckBQT2Bq6HwxGQCOuaxTcAMCvHrgYyaiFyUkLk5J61ky4zNuZ9kXmYLDsB1NU7iVbiNU3MuM5AOM5qoLsurAHgnNNRwTyeamw3MYZlhdtgAI4J9qjMpk4ILZHQ8ZpzRKSSwDehHGfrUh+bGAMgYFWQmIhJwCcbRgjPQVIypsynzDOOlQBvmxkD39ae0oC4zgAdqVgvcnhjMTCSQYP8P0qQlmYlsYPYVAsokjXJPBz0qUS+Y5GOgzmhbjNi7/5BMI9WH8jVISYIAXGe+eKtak4h0u3yf4h/Ksk3qPlQDx3NVMcTQEwjPzqQT0OOKjvZ4jH5iZMyjAAHBFVWvGAwCAPQjNQtIHRnHVRj8/8A9VQtCW7kDMSMFVH1GPyqPae1OIxjPIPNLkgkDg1oiRuWIK4HAzx1NQkkjJBHsamDgkYA4GM96jJycdzzmqQEBPsBSKctipiiAHPXtUABD1QxzDmgUoALc04gDOD0NMBh4puacef8abjkCkAopcUAYJ9qWgBK0LlsaJYx46sW/LP+NZ5FaWpnbaach6iMk/pVNgZ5JxzTD1qQHnFMYYapAQU4UlGecUwHjjmk6sT60dqB1pAOyRwBkn8qUEkBQBnP500nJwfXikBKnIJz60gLCqAQXHAIz+dMuji/uP8AfNDynYR7g0t//wAf8nuAf0FAEAOD1xmlIJH0puKdk4OcnimIejjgdcHOD3ocEsSRgGohnFPHYUAPU44HSnMcEZAz3pg60NkHBOSO9ACkjrTkAIOf7pqPrTskL17YoGSPGFVMDqoJpYzjPoeooEhIAJzgYFLCMsMY655pAXraOY4eEc9q2YXZgQykEdQahtdxdCY4zhSRhcYPFWPOf/njD+oqBjz0qnq0oj0uQkZL4QD696t+ef8An3i/76NZutS+ZYBfKVPnByD1xSRRBoGf9J+g/ma0DcwW7SGSQAk9AcmsfTmCxPvmaNWPIXq3pVqOQq2baFRnpI3JNMC2bq4kTfDD5Kf89Jjj9KoXF1a7iZpJbtj1AOF/LNMmWIykXVzlx2ftTQliP+W8f50wEhuLUxput5IWGf3kbHPX0zU6q5AFvNHcY6jo35Hiottl/wA94/zppisjys6A9iDgigViwJgrFJAY2BBIcYHX1qnfszlATwQePcGrZluIVKufOQdpOao3JhKIYojGQTuXOR+HpQMS1GXA9ya0gSBgeuf5Vm2+Qm4LkitKzEt6ZVSPDIAQCR8xz0oAcWbcOTgAGkKhuSMnOc96ebW7U/6lemPvj/Cj7NeFTi2z9HFVckidyiksQRjABGcVmN9a0WsLt1I+zSH/AIEKhbS70dLcj6sKliKytGMKY8++acpCMCgxjpTrixuLWJZZkCqTjg5Oar78dDSaAus6PbOCMOcYOfekihDwgk85INVkbLDmrKuwAVTgdfxoA0dOtYXgug+N2wYBOM9aS0tY10y4aQrnBxhhkjFUzI4GFAJ9MVFI9yEYm2YDBydhNNMozq1LQLJYEHqrZBrKDZFX7NgsRyM5ORzTuKxvJaRboCR8vAPPY1VTTZrvWTDbxlgso3MOgGearDeQCEBB9TzW55x0nQwsIJuJgCSOP88Gs5SstC0rkkvh2wjuj52qpHuOdiJkj8Sf6VWl0/w/G5Q6zNkf9MQaxS8sxzNgN7mopOpFRHntdhdG/FpWlXCBodXl56B4QPz5pJdPntpU3MJrdj9+M5HHr6da50ZQhlJBHcVp6fqEwdbd5GIbuT/OlJTSuhppux0NjZwGdrjaQ8ZwMHjkelXwWB+WqlgSBcexH8quIa6KV+VX3ExQ0hOD0o3SgYGPxFOBFITnsa1JEcebGUlCuD1BHBqtd2iTRbIwIiDwwHIH51apD1oKOcvoJLIp5lw0hIJBBPH51W1K0BaOc3LneB1GefXrWjrrxo8PmAnIOMDPpTLh4PsETyYKjGCRXJU0Y90zBe0GM+cD9Rj+tNjt5oyWWMFcYyprT8+2PSL9KdDNH84VcZHpiqJGaOb3zpBb3BtwTyCAQTz61WewlE0gd8ncSSO5q/p8oe5AAI+oqO+lkjvZVBwAemKaZJWFmB0hZj781PHE4GBCAPcVEbqQDjA/Co5LuYRZDZB6cUMaZbhUJKV74wRVWI29vqUJQ5JbH0pNLl3GXeMtkYPp1qkyGO834xhwc/iKok6bVpZLcwlJHjQuQxTk4plhcNJcmMzSTQgZDOMEEVavrdruKMIRlWD89xVg/MaoEZV6ALt8s3OOM8VEkQ6lB9cVPqccXnhnHJHXNUCZAcLKQPTNZyLRNLCxfov4iovsAfkiP8jTikwGftGfbNN/ed5G/OkA6O2kThDGAOwFPMEzHPmAVAeBkuceoNPMceATMSeuBSEONvJ3dCPcZqIx7T0H4DFDqMAoSR0OTTQMHqaYydEUAEbh9CaSS2ikIJJHPQ03cQODTC5xhjmgLlt9Nje0EkeQQcnPNRrERwHwfrRFcPGjITlSMYqF5I+hFAXLWxwAPMNRgNC7AOSGGcmq8ZCEEMQM8jsanlmjkwQoBA6A0XC5IJD3NJvPY1VNwAecgetSgmmMm8wnqaTcQajBNNLgMASAc0DsSmXHWhZCabfuuI3AAJABxVPzmHekSXvMJ7015JQPkOD9TVF5sHv+FHnErknJ9M0CuWhLddzn3JFTB37k5rMN2T0GD70ouZD1IzRYLmkxZu5pkcrbCCehqoLhiME0qyY69KAuSuSQcUIJggII+lMypOQxxTt4UYyfxosNMcl35g6Dg8jrUouTGA6qSRVYwhE8wEEHnikDq3BosFyfzvOl8w9T1qZHYHO4j6Gq4j8kDByCeKkBplEpIPOAT64pc8YwMfSow1AOeKBD847L+VIXzwQD9RTScHmkJBPFAWHAj+6PypjfeBAoB5xTthbpTi9RSWg98EA1GeBkVZS3Mo27gPc1HLAYDtLBvcVTJiRCVgBk4FRzS5GVOfTBqK5ZsBQCc+lQRBw4649DWVzQnF4wwHUg/WpBKpHGOaryRFmB6Y/WpY4wACTn2pcwgckiojGSM4q0FDDAFN2GMg0+cRSePAORVN8Z6VugwvjcMmnG0tHAygH0o9qBzRxVuGyjmQMZlGe2a1JtLtpRhcr9KhTS9uF3A+9V7QEhsavGu0PuA9801g5JIRseuKsrAIMAEkDpk0x3kVgU6DrU3KsUvMPvSbnbgEj6irhukkwGQBvWkco2MEZHYVRmyqRIg4YfhTGZiOeverTwkgkEfnVRmAOD1p2ATAPcfjTxEQckcVGWHaniVihXPFMBXGOOKiJwtPO41EVOcYpBYOGHvQFOenFKo+bkVJgdqYWIihJ5NOAIGM/nTjSE569RSuITnHGKUZBzTdx7UZz1oGP30bhnBNMOCOaMDPHWnYDOJJOSST70hckbc8UpGTinCEkAnAqhEQB61IKnS3AXkimiAljnAHqKAGkEEYINOIIGTUoRVb1HvUUhy2O1ACZLfL3NMdChA70pU9QcU4kEAHtQAISQAetSdBTIjl8gHipXw3SkAAFhz2pwUk5FCuoh2lTn1p0ZUnAyfrRYZLFgctwPWpTLKUxEhAH8Z/woHlom6T8BVeed5TyxCjoo6UWGQlC0hJcsQM5PaowMsfY1OG2xtzyaqgkEADOaCSQIWIx07n0qxHHhC6IWAOM1CnJxkD61pWqk2UgGOCe9IZnl95PGCe1QSLjk1KQVJJA4NNlAaHfjkdKAIOMZzQCKQikHFUIeTSE80UhoAUEYozTRQaYDs0bvem5oFIBwIzS5FMopiHZBpw4NMFOFKw2SA85o3e9RZNOB9qLAWA+etSRMpPUVWB5p+R60mhlosAM5pQ4MZGeTVQsSMA0qgDnOD60rATxwLvJOST707yl8wggDA4PvTQ+RkMCaa7kHcDmkBBMsgf5wMHoRUW8g4AH1xU05JIJPUVEY2wDjk8iqQh0bBjg+lPDAEGoFODUgGRmgRM5zzT7U+ZcRrnjOarglupq7pkatd5I6A1QM6ayTLKD6VzOsTedqtwwOQDgfQV1VsAqFz0VSa4eWUPNI46FiRQEQCktgAZPqauTNlAFJJPBIrNZyetWY5CSAeuKQxZZWjTaCPM6H2FRoAFIPeiRg7gKMnufWnCFiQWYADt3NAFxIsQ7yeQM5qAkFic1P5mU8sHC+lVXUI5OSAe/WkwLYZSnIAP8AOmvkkAdBUasNqgk5J7VKASpOTjHBpDI1G6ar6N5du7dMA1UhGXBP41PcP5dk545IHNDAyUXMhz6mnEAAj3zV+SG3jiXzYyGcZBHpVYonYE0JiGxkAckA9s09WYEkg4x1xSqEHsT1yM08bWOB3H6Um7hYkVowMGR8diBSxupmIycAdcUzaAMDoKUhY4Tjqec0rCFmjE7gNIAAeM1R2Ek4Fd3p9lZTWcIlt0eQDkkc1zGpiOPUZUiRUUYwF6VaGZYjLAgD60hjITOcnuKm+UAgMAT1oKnywR3OKdxCSW5hihkJBEmcY7YqJzg+1X9WUpHaIP4VJI9KospaNQBkk4ouMCMgGm4xXcx20E12i7EENunKhRgsR1z7VyF3KJr6ZyoXJwABxgUAQJnB561OhQkggEEd6gAIHB5J6UoJHWkwJ+O9SCynmgM6RMYQcbgOM1VYnYa2ZHd206yR8wxhCQDwSTyTUtAjOW3eFQHRskZBx1FNDoO9da17DZySjcGY/dUDIBNc01sryMzEbmJJwMYzSGyAMD/Fmm4lLHBBBGQTVkWa7QAQPXPU0jWrEAb8AcEg0ElVCWBLDI6jFSYBwNoJJwM0shRCI4jkAYAAohYiVDgkg5GOmaY0ySO0ulbJtZQnpin2xLucoVwcc961JdWleNkAAyMZAqlEMyCiwrlzXlLadagMAQ4OD34rBijuZCfLt2mAODsUkA/hW/rrFba3A9/5U+9vpdPtbOxs5GhVIg8hU4LsR39aU3YpHPSMC4bYBjjA4OfepYYi1q04fODh1xz9auaioF8+e4BP1I5qohKeYAeHGCKAHpp15NHG8NuzoRwemaqvlZSpGCOorZh1o2trFbrGCsYwcnr+lY8rLJKXJ5PWrQiLODSEgjFOZRkEEc8c0LEDknORwPSmBDmkNStEwGcEjOBjmmEFRllIHuKYCx28txu8sA7Rk5OMCkkieJAGxz0IOQa0dHRZDcZ6EAEfnVbUx5V2sYOQFBH40wKg69KCCCCBQGwemaC2QMcUgHZ/OiowTkVIR6UANIrS1kYltk/uwis3BMqJ6sBWvrIBvSCM4hAH5mhgZGec0E57800GnKpfACknOOlACUn8VPMUqjLROAO+KaRxkgge4p3AWpYozKSAQMDNRcVZs/vOfYUgISMHH60CrMkLSNlUJ4yfSmi3mGP3ZOemKQFaTO01Pfc3ZPYopH5VG6OeqkduhqS7Uh4ie8K/yoArg85p5KlfcfrTPbvQFbBO04HWmBatbM3KsQ2ADjpmrA0xxn94pwSOlWtEj32Tk93xVsDIP1NJgZ0enPLII1Vmc9Aoz+fNOvdDewhM1xdQr6IDkk+w71qWtxPZz+ZC3ykYdD0YfXtio9Wht9TxL5LQzjA3lsgj0IpJjMWGwuLpGktoXljUZyBgEfj1+lRxQvcSiKMKHPUMwGD+Nblnql3pNp9nAjmRB8h6EfX1rFkcXWoG5vwZd5JcINpPpjFUmFia80u9sEV7iEKhONwIIH1x0plsu+VQOMnFbt1r7SWiW9tB5IZOC+DgYwMCqOn2n74Pt4HapYjWtoWhjBMjOcEDOOM0otWlIAmaP6AHNTBcRp+FOj45rK5aIBYTbsC/YcE9BTLnRpLiLy5L5mGQR8oODV4f6wfQ04vzRcZkf8IzGVCveZHpsqzHpaWsccKSbxgkHH0q6G5BoY5nj+h/pTuBm3HhuC5uGma4kQnHAUY4GKYPC9uP+X2T/vgVtnhSaZuywHqKLgYzeGLYdbqU/gKB4XtDn/SJuBn7orVuphBEWB57Vnveu+f3uM9lPFLnaAsf2TD3d/wOKi/4RvSyMnzif9802G7MW/8AeEgAEbjxUyaugzlVb6Aip52BF/Y9na3UNvGHMdwCG3NkjHpU32Cz024QWzvl2AYOc4FNlvY7ie2uI8AQkkgnBOewqrc3Bu7oykYJABAPHFVzMDWe1iYkvPsPpigQQjn7SOBjAGM1kJiLJKyHIwTuyB9AagaG3yQC7N6l+v5VVwsbQs/m3+YTnsRT47UqSRJjPtVcaopUL5ZVh3J4H6VJDfZb5woHqDmp5hWIru0S4aSKYb1RN47HOOKrWeh2Fzp0MkkZMjjJYMQaW6vz5ztGOHTZ1/Wkt9UNvDFAsQIQYznj+VHMFh58N2aJlPMOASctnNUntrYq7Onyr0TPORVma9lkmEyzFAg+52NEl6ZYyXkUZ7AdanmCxXtcpLEYYyZJOqg5x+Nb5jBBBZ2DDkFjjBrDS+cx7Y2RWAAJxz+FOXUZwCruWB9Bz+FFwL40fTc82cX5Glew0qLK/ZYxjtg1RGoyCEoGIz1LDnFQGRHyCeT1Pc0cwxkojWZ1jGBnNaV9cR2os5XjMkZQqADg5rIkeEZVQ2T3JrSvjDNptsCckHnjpxTk9hwK0uqJLbeU1mhAOQw6jnIqjfX5vEVWRFYHIZRgn61ajaNcqqAAHHXOarXtushFxCoBx86jv7impEtFIE4wetOhZo51cjJziozIoOQT9DQh3sCCOtaE2d9DqLPUBYO4kVnRyMnPTit2G5t7iMPbyB0746g+hrK0yOKZriOYKVdFOCfbtV6ysYbEOsLlgxzgkHH+c1adjSSsXA6AAk4zTiRjhgPrQEJ7f1oKEdhj0xWlyBBt7sM/WmXEghtpJVUSMgyFB604qh6qPyoAAOQAPpSHY5a6ubq5mU3MSRkDCggg4qO1lEiGEvhAcdM1a8TH/iZQ/wDXOsqPiNiPvZOBXPNalw1uacyWuFBYkjPIA5+tQmGCJwwmGO/aqq+aFwR+tJIm5cEVNiCW2uI7e4EhIOOwPWluriO6vHkXIDY4P0qqLeTsuR7c00gqQPWqJJiF2gE8+tOxCEAIJOMnniohDIeCwII4INI6TL0jNACo6wviJMAnnFOeUFCQAcdc1XxLIQFX5s+tNCy+YVYEcc0wsdlC++1hPqlMiV1J3sDk5GB0qtaEyaNyTkRkDHByPeprJ99lCxOSRyferEUtaU4hYepFZYU9z+RrY1dSbUMBnawNYRkIkClSAfWpkikyyF2jk5H1pGkA6GqruEYAhwSOAelNLOOCTjuBU2EWhMO5GKY84HOB9BVQnBOQcds00q7MNiEj1FOwFwTZGRkexphmcHJ6etVslWIJJFPYnHXj1osFiZbg5wSPxpHnRjgsQR6VQYlJA2cnPOO9TqRMcqAMcHBo5QJ/MZMHJI7ZpPPBOT1pibclWYYBwc0rpGhGMEHuDRYCT7Rge1Ksm/lSPxqL5FG7II9D1oE1uRnJH0NFgLRCuuDU0bjaFBBx6VRS4VCeA2fWrluslwyiGEFj2BA/nQ0UiRm2qTWfI5Z8k1qTaXqRXaLSQ59CDVJ9K1NOtjMfoBRYbZXeV2wCc4oDE1KNO1E/8uUgPvgUDS9SPW0Yf8CosKxCVYckg/Sm8E4JxVn+x9T/AOeP6inDRNSbrEg/GiwWZUAApwIFWf7C1H1T8z/hSjQr89Sg+pNFg5WUykjNlRTwJVHIFX00S/AwHT8MmpRol0ykOQfoDRcORmaHz3oMmCTv/DFaiaDcBgQMj0waR/Dbs2SZFz2HAqbhyMhY5sh9BVFCTIMVuR6JcRxGMEkEYGe1NHhu7HKmMfU4NLmK5WVCW8gAjGDTRKo4JrRbRL4QFS0Z6c7qqQaRNdKSJ4QQcEE8g0JiaIvMXH3x+dOSRfWn/wDCMzA5N3bj6mp49BkCgfbYOKdxFZ5F9aiMwPGaty6S8RwbuI/QZ/rUY0tDy1w2fUDiiwFUOwYHNTJM3rTzp+OlyT9UFTPDGkPC/MO+apIkktbko2CM0txcLKSvlEY7k1SXOeKlU4wTVslaDCCTxTxbgru4B96laWLb/qQD6iqM12RkKMD0Jrnd76GtyUtGThhnHSonIDccCs6S5O8kGm/aS3VqFBiuX95/vflRgkZJNVEmzwSB9aHuCBgGhwaJLJJU5FTRykgDI/Gst7gkHBpYZySATU8rBGvLI4XAB/CkjcnGQfxqNC5GPMGOxPWnHzAMEgjsRTSLvYS4D7Ny4x71S84njJp05JJGTxVYnFWkJsfIc80gJxzSI5DDgEe9StGGGQACe9aWFYhYsMkHj2qFic808lgSG/KmNzQIUZI4Iz6GnK4UkHv3qLBB6cUmcmmJFxJMkAjIqQgEYI47VRR2UdQfwqdZflySaloZIUAPAppwOlIZAT1OKN49TSSYMjLEfjTcn1p7EHmmEj0qhBzRknik4oxTAXJ7UoPOMA00U8DjORSApAYYVKhBbBpoGenWnoAuCV596oCXBIzg49aTDsQFOAe9AAkILMcDsKnSVIlIABJ7GpAbLaqsJZZOe4NUytXrWYyXewgEH1qzewqIshBx6cUJgZAyBzTSMnFSgbjggio3wrYFUA5QUzzjNPRS59vWoBknJPFWYsmMhT260gEcANgEmhGKPnFRkOTkilhJMm080AXMrLEC3A9KaYXYbyAPYmnRpsOWGQO1MuJeDk/QUBcZIQoAABPfvVYKFYn1NPjBkkAFTGAEYPXtQBAr/OM1e0qUmaSMn5SenYVQkQo2CKmt5BbruXgmkA64wJmB6E81dv0hFkjooBPpWfMfMy3rVln36bHx0pAZzqByKZ35q7dKnko+CCRkgVSHIzVoBwwMmmnBOacNvc0hxnIoAQcGkJp33jyetDqF6EH6UAMA9aM0ZpMGmMdmgmkBpfwoAUUu0gZpUIyQQKfuI6UCIqUGgjmgUgH5zQKFAI561IIyRkEUALwRgGkBxTSCDjB+tDEEUNASBvoKeACMg5FQpVlE4AyKljIZUfBbHFO3AqFYDIHpU7rkj0Hao5I9/wAxIBppgyGRASCBxSYAGOgp4jOMrkk9gaEQkcg5ouIiHUitXSFzI598fzqkIeeRWvpcQWMkdSTTTEzQupfJ0qds4JXA/GuLCMV4Heuw1YbdOCAcuQBXO+QB0FK4Io7COooIIq/5PBJHI7VEYSaLlECA7SQORUkLkFlx15J71IIihwfyqSNVEgYgU2waIDI28nB60pkzknp6CpDCCSQce2KabcnhlIPpU3QhiP5rckDHFWCSoKZBFOjt0RACOQeTSPjPFF0BJCM0+4dEjRXGQxxSRcCkvI96QnBOD2oYym0zyfeJOOmT0FANW4YAFyR1qVLeI5yvQ4pcwGaGJNWIx+73chh2IqxNbJsJQc0540JHBHA6GjmAiBzhWPB9KjcqAMgdR78Vd8lfIIAyT3zVGSB1yT69KSkmFjoIvEVpCCQkhIXA4xzWBcXQnuHmIxuOcZzioPKc9BTcYODVJiHiQHoAfqKlhO51jwACw7VCAAM4qe0+a8iHbJP5CqCxLrD5uVHooFUA5G0jqpyPrV+/RXv3DdAAP0qvst1JG0E+pGTRcDpLTVLa20xkkkUzuCzHOMkjH9K5diQcipWjOCY4zke1IEB4I57ii4ERdvUD8BQHOeoP4VK9rIxygBBpRbeUpM2ARRcCeGFHAJUZ+la9hpcl9ATDIoZTghhgA/X/AOtWNas7XCAfczkmr4yOjsPocVDC5DdRtbknzFc5IyucAiokkGASeafe52RxAcAFzz1/ziqwQkkngUrCTuWfOYkhTgY796jkmIU8g5qqWBYgH68GkijBO+UkLnAA6mnYdhEDPINoJqyiSITlgB6Y6UoKpgRoQPYdKFkLn5UP1J4NFwJg46E89qmg5eqhd84Cg4GTgVPbC5L5hERwRkuaFuIu68cC1H1P8qp6lMHvSR2VV/SptTN0bqzjuvJ5xgJ7kDmqt6M6xLF/DvA/lSkUh1/IZb6RgMYwPyAqsQT3A+tGpSmPVLhV4Af+gqt9oOeTTESsSB0BpmQetKsmRyAPpUchAPXrVIQE0qygHkA8UzdmmkYOKYy/C6G0kO75lO7GOo5q9q1uDpqc/cjVhx0ycGsRWIJAOMjB+lbMl+kkYjkQMuwIQSOopXAXSYwtkXUghsYIOTx61l6uc6iR6IB+lXrOWGzhaNScE55PSonS3mnaSYgs5yMHgDHFHMPlMk0AVpiGyTkqpyccnOPoKU29k/AAUnoQTRzILGaF+YH3qZsZJHc1ZENtGoLK5Oex4NPMdq2CN4z2J60cyCxShXddRD/bH86vanKrahMBzgBPpipbe3t2nhZF+cOOP8/SluUtpLuckEMZDnBo5gsZ8Ntghj+AqyARwDiniMJIAASuOOKm8pUiDSKzH/ZNTzCbS3GpMAhBwSOpNOEqYwwJGc8jgVEiBssVIPIxj+tS21tLOWUIAyjOD6UXG2lqyKS1tZMnDBj3B4/KnQ2qR58ts5HOTVn+zpnALNt+nekOnT7SBuPuDmlzruT7SHcYsWDzIMjqAaVwyLkYIHWnxWVwvDRZGDknOTThb3YkA8tyg6BQeaOddxOpDuVhLk9afPFDJ5XmLnIOOenNWTptySTHbPuJyMrgClk06/8AKQJbSElSMhSccmjnXcXtIdyn5Edso8tdvpzSea/sfqKtLpepNGA1pL7fKaVdI1hTlbWQR+hGCaOddw9pDuVUunhUoMKpOSAOpqRLhjzgYp8ui6kwytlKcdeKki0jU9uDZSA+nrRzLuHtIdx8Mm5fpTZnIxjgdzU0WlamuQLRxn1NSvo2pPGR9nAPYFwKSkh+0h3MdixznmoGgZ2yBWuuhatuBeCMj080CpF8NarJnZAufQSjJpqaH7SHco+WGSPjkIAK0LEjaB3BwatweF71z+9mSJkABjUbyP5VdXw8UGGuJ/wTFDdy9Cq33QB1wMD86I/uA/X+dXBoUI5MtwT69KnGkW6qB++OPeoYaGaeCp9c0ucgGrz6UhQ+WHDAcbjwTWf9h12PhbaKQDuDwaC0kPBxQGzNGT0GcmoDBrq/e09T9BSbNYXrppP/AAE/40FWj3LTSdRWXq1/9khCI2JG6H0q1jWM/wDIJJ/A1HLDfuczaIXPrtJouHLHoYL3pNsd8juT3LAYqutwSAw4yc5z1roGgmIJOgnj/pmf8KaIJgQRoJBHQ+Wcj9Kdw5UY6XUgJBYEEYAHakiuWlmCEAE9SOtbYt5j/wAwI/8AfB/woNk5/wCYA/4RmiyDlRjpMwm2khj2waeZlWSQhsEqQD71rCzkHTQpR/wA0n2OYn/kCSfihp6C5PMxluic9TjvjgUxJt0uAetb40y9P3dBP4qasroWpuob+wV574NLQlpLdnNPLkAMegwPpUYmwhVWbk5zk11K+HNTP/MDj/M/40//AIR/V/4dFhH4L/jReIrw/mOVSZguM59z1pslwxXy8kY7gnJrq/8AhHdaz/yBYv8Avlf6Gp4fC+quCX0uBD2yBn+dJuInKC+0calwflBc8DHJzT96dcj8Aa7NPC+qMf8Aj1hHsQoqwnhXWQMKtuo9OP6UueJPtafc4eMgtvG4n0xSyJM5yquPQlDXcnwxrIH3rcfRgP6VNH4f1YKBmAEd8ihzQOtTWzOBWG8blonJ9QDUqWt2xyLWYkdAB1rvx4c1Uj/XQD6GgeGNTPW7hH4n/Co9oifb0zgzpl+6FhauG7ZwB/Orf2O4uLVYY8BkHJJ4zjFdi3hi/VSXvIQAMnr0/Kq1v4a8+HzBdRrzgjBNHOio4imjko/DuosT+8jU/XNXItBvEXEk0YPchTk11aeFIhh/tzqw7qhP9acNBtP49SmY+uyht9ES8SuiOZXQIwQ0kvPsgwfzqR9LsUhbKRbsHkgCukGg6Z/HdzMPTgUreG9Dm+/JKcerD/Cjnl2GsUr6o5OC2ixuuCAuMDJ7+tTBdOXq6/ma2LDSLK7kljuHZlTkDHfOKtjRtHjBBtw3ucmqjK5vLEUznBJY5wksv/ASf8aVrlgpEMtzu7cn+tdNHYaZC+6G1UN67Sa0I12j5YkHvtFXd9DneJXRHLnTdYZVZdQYZGSGjzU8Gk6v1k1KPbj/AJ4j/CuoWKWQY3Y/DgU8WDn/AJaL+AJquaRk68mjidR8JXl9MJX1CIlRjmMqAPwrE/sW7W5a082LeDgEAgGvU/sCjO6Xt6VyF1GLbxRsdlKkgkj6VLZtQqN6MwD4V1UAmOWB/wDZD8n9KpTaXqltxNZSr78EfnXoLXenROCZU+uc1FLr0ADLHEJlIwAw4phOqouxwcVpqR+UW7An3xTzpeonO63/ADIreDMSxVdoJJ2g8ClDMOpNOxk6z6HOnSNSJ5jH/fQqVdI1Ex7Si/iwFbxc+ppwc46miwvbSOe/sa/B4SMf8DFPGkageXERPqGArfyT1NB/GqD20inY281vZtDNjIyMqcg9aqwLd2aeSlxbFAScHrW5AodGUjOckVzqW6i9uIpLUykSH5weg+lNGkJcyL1/H9osnQHBOMGsmPRZpMATqSO5BrfMDvHkYxSQwFMknk0MJzSWhkpoEjt++uVA9lNWY9DjiH/Hyzntlen51phacV4qbXMJTk+pmHSYyCC2Qe2wUDSYx0kYfgK0gtIVpcqM7GTJodvIQWZgfbApE0G2VwxeRh6ZxWtto21di+aVrXM86LYnkxMf+BU6PSbFAdsBGfVs1fApwXNLlRFikNOslOfsqH65NBsbXOVt0U+3SrhUE0CMGpshlWOzto8EW8RYc5K1MIrMsTJYW7H124qTy6PL9hRZANMenkY+wRj/AHcUggswwKZix2CCpNo9KNtHKguAjQYK3qqexZSBVhY7jZuivrdz7MRmq4XFG2hxQvmTl9YVchFfHpgimfatRCgvaLz1JxTFLocoxB9RViO+uVYFpN47hgCKXIg+ZWuRcXaBWHlHuVOM1XFhKP8Al7b/AL6rXjjsL25CsrxyN+A/OnXWjXNuflMcg9Qy5o0TtcpSklozHltbkACO6AHfcRVZoph8pvufQHIqSaSRJXUkjHB4qGqsdVNztqxUhKjm8OfUE4pwDA83jkegFR5FPEbkDCMT7CmaXY/zDHytxKPwqtNPPJkNO5B7ZqaS3nKEiF/yrFlNxayuJVwCcg571nJGkWXd9yFAEhwO2elNaWQAnzHJHUbiKojUHBwAKfNJJMgeNcH0z/Ws7FXNO3eGWMFlmznk7jTYUiFxIGDYycA1Q064uBnehHOBnirLSbZS2K1RDVy+5tYxnySfoTVeS4gGSECj0PNJDN5hxlQewPepzEjcSeWR6VjKTvZFcpBDdQytjap/CrBCt0AA9qz5rQRy5gGFP41chmKLgxb/AMaqMiXEf5a46UjRqY247U57iQkCOFBj15pBcXAwCsQ98YrVSRHKzOGF4PUGnnkZFMkUhyMHJ5wKeiuV6VopJoTRI+1YMk4FZUsJmy0YyT2rSKORgqT7DmnxWt1M22GGT6EYrNgkc1Jbzq5zE1R+W4P3Dn6V2Y0TUJMB0QA9yc4qwnhyTJDzRj3C00yrHECOUdY2H4Ujkjggj6ivQE8PQ9HnkI7gAVOug6eCC0Zkx2Yg0cwWPMi1PiYA5OR9RXpw0TTR/wAuqU4aTpgPNpEMeoouFjzlHQ8k5NP3yjhCwHptJr0gW+mx8pBApHcKKXdaA8CAfgKTA8zYSHja5J6ZUikFpcv92FjXpF1Lp8kJWZ4cdsAZrA8+wsizNukUnIOcUJisc+mkagwBFs4z3PAqynhzU3UECMZ7FsVqJ4n04N5awyADucYqvceJTgrBFGo7EHmnewGNPpV7bybJocH1ByDUX2Ug/vH8setF1ql3dyZklOB2BqrgnqSfqaoC3Jb2yoSLrJHaqhCg/KSR6mnqB6DNDAEYoERBiDg4qUOMYqIoQc9RTiPSm1cBSwoDDHXikCkilCjpSsAu4HvQcU0gA8k0m4DoSaLALmlzTSc9qTJx0H1osIfuoDe9R5NGTTsMZkg8UpYkcmnmNSc8imlADjrQIUNhKUHPJ4puV6Hr6U4gY56UhCIxSYOp5FacV/FPG0UvyjOM9c1lkoOlKoAOSM0rDJZWRQQgBHY1ACO4JNSnBORwKjYelMQ3rxU0b7Ygigk+tRCKRzwD+FSorjJAPFAx2fkJ70tnH+/3sM1ERh8mrlkvmlsOBtGTnvQBMx8x+mB3FZdxIZJ2AHAOB71rSMIY3c9QOlYxG5znoSTQgLltGYk8w9T0olOBnvTUkJIJ/wBWBjFIZQzE4yO1FgHXnEUb45IwaqLl5FRRkk4xVqdxLahQfmBzim2QSKRppDgoMj3oAdeIIFSMH94eo9KljPlWgjfjNU9z3N4ZOx6D0FWZg7uVHIHQUhj2tzLGAXAjHoaElSEbNq7Pcc1YaB4YApUnIzkVnXQIwPWpAW5MT8oMVUpxJIpmCK1QgJpKUqcE+lNoAcDzSkg00cmnBDjOKAG0uaUqe1PiheQkDtQMj70u5hU3kqI8g5IpmM0XENBpykZ6UmPSgAnGASD3xxSuBIDS4bGQpx69qTHFSgs5VSxx0pCIQrH5j0p4UHrUjptbBPAHHvTQaLjHxgDqaczYOAeKhJzx6GgnmkwLAfPU80MxY4qLOB705SM80khjwWBxgipQMAGogxPWn7uKQDiw4Fa1ipES8dawySXQD1rpLSMBAPSqiJlfW2y0MZ6AZNZO4Crmtzhr8qvRUAxWU0hBoYE3mfLjJNMLgdfWq5lPc0CTPU0rATM4JzmkD4qAv1FN3n1p2AtCUjn07U8OCdxP0qmH5FSBuaVgLQbJ65pGJ8wYFMQ55705QWbOeKLDLCDJwKmdyZNvAAx1qGM/MKmkkjXr/wDrpMBDtGSetNM4HGcj6Ux5UwTn6CqZm+Y4qLAXvO59sUiSAnnHtiqYlB68j0pRJjvRZgX/ADcjgYA4xRkMeehqoHJGSTip0kAAIAP1otYZMY4yMKhJ9qrPaSk5W3lJ9VFX4JCuGQ7T6gVcW4uCMC6KD0IqosLHPtaXHaCQ/wDATUlpa3aXcb/ZZCAcHj1rbYyn712T9BTShP3ruUf7oxWlwsZFzaX8lw7/AGVyCTjHPFSRW98q4jsGPqSvNafkxgf664P405Y48Y3zfmf8aljsUAmo4Ia0k59qT7FdyAZtnyOhPGK0AbdBgSXP4MKilkhAyr3LH0JIpDsVTZagmFWEj3DgGnHTbuRQXUZ9HkHFSC4VTgCcD0FWE2soJaQZ9TTFYz/sNzakPI0ezOMKckU4k1YaNJmJFzHGAcYc5NAsyeftEJHru60GbLA8O6lfrFdwCFIygCl3wSB3xipF8H6oU3tcW/0BJ/pUamURrGssm1RgAPgAVDPJfLIvkCR8DLfOeAaQ0ZwM7AhLdmIJGVTIpi292FwLOY/UYrajd9gBO09xnFNmMhiIBJPU5pDMoW9+elrIPqKY9nfFcG2c/QVqI0gXLkH0xUySEZJBIHoKZVjLt7K7mkxIvkr3Z+lXbe0No5jMyzZOcqCAKmaeI9bZT+B/xpI5IDMCF2L6BaaZLRPPax3V7BM8pjMeMADIIBzye1RTabbG6NwJpCSwbYQMVNJc25K7JEPGDuU8Go/NhP8Ay0t/yNDVykQvoVjeSPcSXMiFySQoGAahbQ9LRiDc3Jx6AVq29nLd58lrVsDOA/OPpikn0jVSdqW0Ix1O8UN2G2ZR0vSR0e6P4gU0abpefu3R+pFbEel6kgAa1hJ9iDTjY6mvS0T8ADRcVzJGm6SBzHMB7t/9enjTtJP3YC31krQ+z6l/z6f+QxVe4fUbcgGyJJ9IwRS5guQiwsB0swf+BipBa2I4+yID6ZrPmuriPAkWVSRnBQiqxuypYhyGJySQRU8zKubixacGCm2TnqASKRzpwl2QWiHHUtyM+1c6bsEkSOGPqKnS5iMOATj0HFL3irm1DPYyIWS1iJBxgLU4vLFRl9Lt8epGawIpCGwh2ntzirJuzG21iJPQEcn8ai0kFzYj1cRMDDZW4j7ArnFWLuW11h4g1sqeVnovXOPTHpWL50csRAyDjIIGcGpUuRsG0MP0oi9TOd2tC7Lp1ksiGG3VJAwGTmk+w2Zk3NZo577lOSarJcuk0LLkndnp6CpDqjsduRk9OK1uzncai0TNCNbeFdsdnCF/3ak86M9bWFvcoKzVu5WzhgccHirSTjaMsM07RMZUmtWWhc4GBbwgeyCl+1sDnyov++RVfcaM0cqM7IsfbJR0VAfQKKP7QnHQIP8AgNQYoxRyoLIn/tC59V/75FN+3XXZ8D02iocU7FHKgsiT7ddf89m/ACj7ZcnrM351HijFHKgsiT7Vcf8APVvzoNzOT/rG/M1HS5FPlQWQ/wC0Td5G6eppDNKw2mR/zNNoo5UFkISx6sx+rGggnqcj3p34Gj8DQMbhu1KHuI5FkilMcinIYUufY/lSj8aXKmKxftte1G3zvWKb0JTBFX08VN/y1sjj1BzWFRihxQHRDxPauuXtZQfQIDSDxFppPzxTL9Y653FKBRyoNDpB4g0kjpL+KUf2/pP/AE0/74rndoNGwUcqK0OhPiDSh0WU/RKQeJdOXpFcf98f/Xrn9h7Cl2nvRyoNDoR4l08/wT/in/16P+Emsx0jm/74/wDr1z2wGjAFHKg0N/8A4SeIH/UTY/3RTh4pg/59pvyFc9xS9aOVBodB/wAJRb97eYfgKP8AhJYP7lwPwFc9gUYFHKg0N4+Jo+0dwfyph8TnPy2tw3/AsViZC0eanrRyoLG1/wAJQ/8Az5S/iwpD4kJP/HnJ/wB9D/CsYuh71qW+i29yFC6nGHP8IUjn65oslqFh58QydrRvxemHX7jtaIfrJ/8AWqaXwvdW6Fy7OB/cUGqTWKKcPPIp9DHUqz2CxYHiO7QYFnF/38/+tSHxPfdreMfiahFhARzdOfolKLG2H/LxIf8AgAosh8jHjxFf+YX8uLkYwCacfEmokYMcdV/s9ln/AI+X/wC+BULRwE/LLLj/AHRR7ppHDyfQt/29fn/lnER7ij+2rwnPlRj6VSKQ5x50oP8AuilWKEHm5lI/3RTui/q0y5/bV93CD2xQNZvf7iVW2Wx6zyH/AICKCltg4nlB+gouifq9TsWDrWolWQFFDDBwO3501L25tx5aKm3tkAmqNwVtY0kS4MjE42kYq1ElvOitJdOjkZxsz/WouVKhNIn/ALXvxwPKx/uij+170chYwfXaKPsMA6XTD/gNH2KH/n6b/vgVfMjLlYh1a+PVIf8Av2KcNXvyP+WY/wCACk+xQ/8AP03/AHwKX7FF/wA/QH1Q/wCNHMg5WV7eWSNZJIyA7Eg8ZHX/AOvSi9vAcibHvsFNi8sySQeZg7iM4qcWWf8Al5Uf8ANZQktSpwloN/tG+Ax55/AAUxr28brcyfnU32D/AKekH1Uipk0a4kjLRzwt6DOM1pKpCKuyFCTKRu7wdLqYfRsU03V24+a7mP8AwI1d/sHVjyLUOPVXBzVaazurb/X28sZHXcpxQpxezBxaK53k5MspPuxqk6f6U2SSSOpOaveZ2BGapSEC8Oe4pVPhubYX4yW2hQRcLyD1qbYB2pLbiPnoCalqk9EZVPjYzaPSjb6in0cGqIGbARSbBUmKMUxDAtKVA/wp2DSYPrQA+Hhj9Kx9TjggvWZb5oGkAZ1x3rZi4Yk+lYmsCb+3IjDHHJviA2yDg0zWj8RbsZCyACbzQf48dakg3ZbP3e1TWkYWAHYqMwywXoDTxGFGB0oZVSabaG4oxTsUYqTEbilxxS4oxQAmBSYp2OKTFMQmKPpS4oxQAlAOKXFG00rgJnBoz6dKXbRgdKYCE80ZpcCggClcYmcUZoIoSN5pFjjGWY8UxCMwFIJAeik/QV0dl4ZVxvmckjoD0NbNvpVvCpDRx+xUVhPEU46NlKDZydnb27MsjyTI/YbMgfrWncRSrGHjihkGcZYY/rU9xe2kFwAVIQnHXpWjJAJ7P/R3QhxwabnboZuXK9Th7i0t43d7q4VMnJVegqs13pcOFjhMxHcAn9ai1CwvIb6UXKFBn7zcA1TMYU9QfoatSuj0YNOKaLx1gqP3NooHYn/9VQPql5NHxKkefQc1Uki80Y3ED2pFtZFBEMuPYjOaVyrDZ7i6jQubtyB1waqeR9sjDmXJP9481bIckrOIzjqKFS2JIjcgjqoPSjmGUDpcigkSqfarluVjjCMcMO2KlAGMA5qtM0atyh3D+IcipuK5ZbAxwee5qOXBjOSBxmlRlkQFXBOOhNN3RSErIASPSpuFytbyp5fzyAH0NEzgkut0BjsTUk8FqUOCPxqAQwyR7FjJI9TimVzF20ujJHktnB5qwbxF4zisi2t54d20cE5Az0qwFDDLrhx71DRSlYtveAKSjgEetZ6XRe4DyHJqddjkKOSe1Unt5I5SQRTSDmNWS5ijhEm3J+tFvqENxGcREEelZbwTvHnIGOxNJbmSEHaxB9QaqxNzYtpybkKSAwOcHvW++pxQwAu2SO3SuJV38zfI2fU1akvPMXBIJ7Ejmi5J1FrqQuJNo4HrmtRBkV59DfzW9wNuCPQiuntNbQwhponXPdOlUmM1ZruG3QszdO1VBr1lkAhhnvWBqNy8zl0bANZDSzhtmQc9xRcD0SKaKZd0RyKHjDdQSKw9CuCiKkg4PfPSuhouBReyRmyBisPWrC7MYMcQKc5Kde1dTgUtNAeayoYM4EgYeoNVpZpGG0sea9Pn+yiEmZEJ7ZUHNcbdzWZuXwkTIegAq7gc4I1JyRTtiAdK3bPS7LUJwAWjU/xKckVfk8EnrBek+0if1zVAcgyAcgU3HuD9K6g+E76I4Plv9DUsOlzWgJksA49SMilcVjl4opXJKROwHcCp/srsOYyD710BLQk+TbmNTztHTNQTSSTMCy4xUcwWMgWeB82c1HJbgNwK2CFxkjiq5jBOcUucLGUYT2FNMR7itQw+1N8k/wB2q9ogsZnlEnpS/Zyegya0zCrD7oBpn2YjnnHqOtHtEFjP+zMOqmmPEynG0mtZYwBg8/WmmFOwNHtEFjLWPIznHtSmL1q+bcZzgj3pfIUetHOgsZQRyuQMilVFzmRjkdhSZbsTS4PU1RA1kTJIzwOpqLcT1NTEECocYNAD1KA4wSfapMrtxyD2FMViBjAxThyKdwGEknipAhABYgZ7ChQAc4pr9c0gJ45DG4AOBU5kDEkFcms8MSRTiVDBmJGBjinYCVxknNMjLxSAoDnNLuMgLL0FSRuFcF+RSHYnvCTCAfvEZNZ6Llcmr0sokBUAY7EVXwBwKEFhhZsbRwO4xSYIFPPBx3pCCaYWGEEjJ7UmODTmUjtSbTgggjNIQQEqwYVo28qBmZwCD+lUUXb24qQE5OKGUa1vejOwHCGo7/TIrob4X2N6dqywSO9TG4mI/wBa351Aim0TxMUcEEHH1pNoqRiWOWJJ9TSbfSrTAhIycYppQA4xU4XH1o8sk5707iIFQg9KsqAVANKIiADjrTxCeDnn0ouAww5Bx0NSW4AcqMjjrmpFGDgEZPalAAbgAfSpbAj8oLIRyVPrTmRAhwAMdMipeD1pDGCc1HMwKwg4BwRg8+9OZQcnGAO3pVxYy2EUZBqNogpIyMj3quZjsVCSABgACkDMDkDnsalMbljhcilcHZgYGO1O4EBJY4JyT0Hc01g6khlI+tSi4kgB8lQHJyXPJpqvNcSgSNknuTTQiIZBzTic0EdR3HX0qLeB3pgyYHBoDYIPWowwIpQTSAn3E9OKUVECTxTwD1JNFgLNqN11H7HNdNaJwK57T0zcAntXSw4WKSQnopIFUByd/KZL+ds5G7A/CqhNOLNIzPg8kmomzikAxjkUgPPJp23PfFAjZsBQCT2Bpg0NJpM5qUWlwQSYSAB3pRayEAquc8EZ6UARDk4qZRjg9QaVrd42x1I7VMcEjCEEDkY70rDBQcdKmQ8VXDSEgCPAzySelWFiYnCuM+oFFgJY8mQD1NNkZRKQTyKaILiN94kBK8gHvSiBtxaXIDDkiixRXkfK45+oqB3A4FaAggRcEM31NTxGGO3wYQfpxRYDG35NSKScVspNahRi1APuf/rVKLyJR8lqM+uP/rUrgZaK4XGCfcVaghZ0zgj61M7icAghSeoJ4FTxNEi4luA5HZRnFS0A23Rkfawxjv2NWSq+cABk9hURuIU4UMR/eJxj8KctxbschwT6CklYB5QbjvB9uaAUfOwYx3xQ9zACMhufxoMkezcCVwc+tUMaGwhOOOnIpQHILBicDhSOKUzQg8ElyOMjgUhuIgMCXBA5ABoAUB3QkqMnsBTtpA5OKYGBQHIJIzwTSxGEZTcSScnPSgBCCoBL5BOOKX7Kj5MxIAHbrmpP3e/arjjoKQuD95wRnk9qAGNDGzYwflHtQLe2KEYKOBnJHWhtgJPAz6Hg/hQGkccIHAGAQCaBBEhEmQAVIIGak2Y/gQfTmnR2eoyLlLdiPUnFSx6TqDOTIojz1JfH6YpXAgIHop/Aig4HAHJrQ/saX+K7j+oBzUseh2wYtLNK5xjrgfyouMyMgdcfiOKbIYyMBzk+hreGlaeOsLn6mp47WxQALaRgjuwBNAHKyEJtEOXJIyCCeKtiCe4IENq5UdSBjB/GuoUhBiMIB6AAU7cT94j86YHNroV4wyRGnsz4NWYfDyhg09zx3VB1/Gtk4PcfnRkDow/Oi4EVnZWthGVtYVQnG5zyT+P41OSQOtNyPUfnTSCf4z+VJgOLeppfoV/E1EUP98/lUbK3ZwPwoAnyw6EfnQZXH9386rm1lY5EjA+hGKaLO5Y4Eij36UhFhpGYYKqR6ZH9aruwXpbKfwH+FIdMuSwIIkI7bgBSPbX0KMy2ykgcDOQT6daVyrEEltb3GfNsoTnqSozWLqmlaSY1+z7o5c5IQ5AAx1rZe91LysXGjxMo7j/61VjcwHmTRYgfU5H9KFIEjnZdKB2+RMQc5IcE5P8ASo/s8kFx++IITk4OQa2bsC4m3RxC3XGNq5wKrG0Rw/nTltxySvBFDdwaKjNuB2MfmIAxxkVbkMUdvmV1DewqCQLDD5FvEWPqTk/nTYw9zDsuF2YOVOOMioaIaJreRHyEYkk4wabJCVJZcooIAJGcHvQMWmVQq4PenpFe3CtuSVIWIIc9AKBWJ2OLISKudoJIz19TxRpkpN7GwjVwWJwwyD+FFtaT2zCNmiYFuo64PrWxZeFbqW7WWK3ZApBDbgB+RpNpEznGK1ZpyXGnPlbuz8jH8cJxj6irMnh7fbrJZs7FuRvbgj8BV6LR7W1BkvLgSuMZXgD2z61OLyRcCGQhOAKUJStocNNdzmZdLvoG2vbnA7qciq+1YyRMJlPYAAfzrbjkvP7TmEtwuwZYKepz/wDqq5uZuHww9COK2TdtS7HM7oOxnP4ilJiONiyZ7liP6VuTaXazKSsflyH+Jf6jvWCoxn6kVSYmhaUAZ+YHHt1opR0piF/c/wByT/voUYh7pIf+BD/CkoouAmPQYo5p2KMUhjlMOMPDI59Q+BS5tv8An2k/7+03FGKYhd9uOls4PqZDimMcsSBgdhnOKWigAx605NgOXBI9AcGm0CkMsRtZ7v3kE232kp7tpu07Lefd7yf/AFqrYoqXG7vcLjybfHywyA+pfIqOlxQKsBKfGUDgyIXUdgcZ/GkxRikBeFzpeOdMOf8ArqaZJcaeV+TT2z/12NU8UtQoWd7sdyQzWvawIP8A12NRHqTjA9PSnZpDVgJingCPBkhEgI4BJH8qQClx70hFyy1DTreYNcaUrjsVOf0NdTZ3nh+YkQLDBKRjDpg4ricCkCAHI4Nc9Whzu9y4ysdLq2hX5zcWNtZXCY4CDaf51zRubm1/4/bB4CP4sHGamS6u4TmG5lT0wxx+XSte38QsItt7bxXB77l60o89ONnqDs2ZEWsMPmivHjPpv4/WtWLW7ttoZYJVPdhya0WtPDWqwRzTW9vbhhn5W2kfkBWVqPh7RY42n0zVRDKo+VWfI/Wkq8XpsxuDLk19prQt5tk0JHVl5H6VRe3tZLcSQ3yrH23d6yGm1KBWVp7e5RuPlOCKcJmksvIe3Tn1P/1qpc0RQrOGjVy2bS3DY+3Qn8DQbS37XsIqt9odnVfssS8YznP9Kf5caD5nXn0FVzdzoWNXVEwsrc/8vsf5f/XqnHDDNfyQrdIFQAZI4J71IihmwiqrepPH8qz4NOu7G6uHkKOrncArdPfFCkjRYqD6mx9gtR/y+Q/gKabK2bpexfiM/wBaz5boQqDgP3wDzTDewlEOCSRkjI4PpVmvtC3Po0E2P+JnCmM8BTirK2UIQKdQhOBjOCKylvocZ8sD6tTlvoWGfLH4NRYfOzXSOFFx9vi/I0pW276hD+CmsOTUIfNACnHcZFB1C3/55v8AgKLEadjb22/bUIj/AMANKBCP+X+L/vk1hf2hB2SQfUD/ABpf7Ri9H/T/ABpWDTsWYYlXVJIWuARnIkA4/LNa6rCvDX8WP901z8MyyXDyj0zilOpW57t+VRBWbRc3zaM6JRaBgXv4uvoa2rWfTpQQNSiTHqhrgPt9uT0Y/hThqFuO7flU1aCqLUzStseiAWI5GrQA+uCKsLeW6Lj+2IWA/vcivNTqVrjo35UwajbE8IxrFYJLqLU9HubvRjEzTixnbp8ibD+lcTrMNhDPBLZsvz8OFJIzxWbJfhtvkjbg5O48EelJJdJcOgWMrg5POc1tGkqcXrcdONmdXp66daHzjHDOGA+Usep7kVbml0S74ktWgXu0TdPwrkWugjbVU8dyMVJFfb32sqqPak6HM1K9jKrF3ukdvF4U0zULYtp98+49CSDj8KxdR8Pahpb/AL1DIh6OgyMVmpe+UV8udk2/3WIrctPGl1bcTss8Q6qTz+dFq0Nnc57J9DApa6q31HwpqzbGia2mfjdgLz35HFV73wsq5ex1CGdcdGkAP51cayvaSsKVNo56kwR0p8kUkD7JkZG9GpAPQ/pXQmQIOtUtT8qN47mVZCyggFBnA96vgd80j/NE6+qmmmVGXK7ozLS7S+yIGf5RnkEcVpDgCqGhDFgF9GI/WtEjFNl1X71hMUYo5o5zzUmQmKMe1Lk9O1GT6fjQAmD6UYIPSne+M0An0wKEIZg+lGPanZORxxRkdwce1AxhxkDoaMe5p+5fQ/lQAD0GKAGYI7mjGT979Kfgd6MCgBnNHNPwCevPpRgZ4oAZg9KQNJFIksZw6HIqTFIVouB1ema9bTgRTHypB69PzraUeYu5CGX1Brznbg1NFdXVtzBcSR+wNcdXCKTvF2NIztubuvaFLMBLCOhyRWFb3l3p7jbK8TD+EnIrTh8UX6bVkVJUHXcOfzqeTW9KumAubVx67ACKUVUStNGc4KWqZCNeM0fl3dsky/gKglg8P3qbfLe0ZuuKuw6ZoV/za3skDD+Fjj+dQTeFr5CTDLFKvrnBpOaWj0IUJrVGbJ4SErE2GoRPH6Pwazrrw3q9spYW4kA6GMg1oT6fe22fOtpAP7wGRSQ6ldQ48u7lXHbdkVqpM0WImtznLqynik/0qGSJj68ZqCKBLdiyDBIwa6yfV7yYbZ3jmX0ZetVhPZMQJNLtjnrg4JquY1WKXVHNO8kZMkZOR2JyKprc3LkiQHBGMYrsWh0OViZLOWBf+mbkipE03wzJkCaZSf7xIqrotYmBwiR3CNkIGHvUqEvJzAVJ9K7F/Dls7Ztr9RnoDg/1preE75Y2dLmOTHTA6/lRdGntYdzk5IYUzK5Jx/CelS28ttK+0QlGP8QOBWq3h2/B+e33DuAf/rVE2mXcP/LjJj025p2HdFKSURAxlsH6VVDyE8MCM+lOu7CaSYtIskZPoDg0Q2kaYBuGyOxGKdh2LpjxEHQL5gIIIqKRS7CUnDYwVxxUqOQBk8VWvggXzQ+GHoaiwDG3ysVBwPWoVt5znaM1F5z9jViGaSJfMOD7A1dhFIO6thuDnpUkk4VDmpXaG7kBkiZP9pTzUX2cQsQSZF7YosBfttQhmiEcsYY9mA5Bpst88RKI+B9Kz0mSKThGH4VJLtmBKqMnqRVcoEpuhMcSOcnuKQm3jIfz3JHTAqn5LBgcdKsxxI65YUcoF+C7QoNjkj1q5Hqc6AAXEgA7ZrGURLwspH1FSoQf4gfpQ4lJm4NeuE6yg/VakTxM4cF40YdxyKxg8ZUguRn0qJ1XgoSR9KOUZvX+uJdWxjFvgn3zXLTEmUmreWI4BP0qNoyx5jb8qdhXGWd1cW8weJsY7V2NnrM08Yyi7jXJC3I5CP8AgKsRRuVAwwx+FDQXOvN3ckcwj86qz3c4TDDbn3zWEguk5W4ce2aeJrsna0pI9DWTYy8bqQDOQfYioZLguOUX8qrvJMq5AB+tVzcSN94AfSs2wJJiCMgAc1BvwelMkm4POartMT04osBd3ZXODTQcmqyytkA1MksZONwB9DSswJiAR0xSbQRgGlBBHHP40cE5HWldgRlcGkOAMmnNxnNREgg56U0ApBPOM0wk9qkRweCQPrTXUDJGPwp6gY4jYDkUMMHHanBiVz2ppbNdJkNPFJtB7UE800sB0NNDAqQeKASetLuJ60nmAcYOaBi80EZ60eZgY7+tNJNCQhFGGB7ipLeMSXGD90DJpgHNTuTHbfLwxPSmIrux80hQAnYU8qD905HrSSQtDGhbOSM80xWIAx3pFMn3DAA4xxSmRIwSeWPp2qDdgEiiMea+C2AOfrQInKEIJGI55wKUHJxwKCQXCjJHQAUrRhQTjn070DuIcIckZ9qaTkkkgU0k4JPQVCXBGBRYCcMD0IxSgjHFVgckCp1BC5xRYVxSfSk3cYppcDqaWNfNYhQScdqVh2BomGGOAKTI9aWSORWwQxH0zTSNoyQQPeiwWHDBOR0pwOKYhzUwjA4I5ouFhAcEEEn2Papgq9SDn2NRiPac5J+op4DFcg4/CkwsIRgk8UqtUDsQcE0ze2eDSsFi15meAMmnR5LfMMDuaZblSxLHAAzmkEzPJjgA9ABRYLFkHAJBIA6YpOWH1796OQhwcZ9qEyB659aVh2FEYA5NRmIZJ3Y+tSFX3ZEigDrkdKjJj/juAT3JFCQEToS+MjHrULxkHAPPtUxEI5EwJ9RQqhzkHI9atOwrFMeZG2QM+xqXyxNyqEN6VYaDd0XNI0fk4CjBIyaq4WKhjIbHFSCJV4Y5PqKuFoggDrubOMgHimMFVgFbCkdcZNFwuRsTJj5QMDsKao2sTknJ6VKeVIVxjufWljgkDbiOD3pAi9pyZBYdCTita8Yw6TMw4JGAfrVLT0wgFXdSOdMEXZiM1QjlY8+WADjk0vlKeprRFhCCSJhj0PanG1gH/LYE+wqSkZ3k8cKT9KaFKsCq4I6GtP7PED91j9alS1hckFSABkkGgZmI82cueBTSpJ46etawtIDwd4z0B7iniyjHAXj60XAygrr3B9+9AV3bGM1qG0AbCgUG0TjcBn9adxFEK4G1Y8kdzyBRsl4JQk/7Iq8IcA/LkDoKfGjSHBJ29h0pXGZgEhYYB/GpRFMyAMpwOgBrQKAgDGMHOAacYlIwGCnuTTuBmhGU48okE4OadMpVgoiYDGfWr+1IyAHBz1ApgELsSSQOh3UgM4RSE4CMPqKebe4BycEDsK0QYiSqtjIwSelRgKM7lyT02nrRcCkbOZwGEZ2+pNOW3ZTgYBNWnWY4EKyD225q0kdzIBGLOXPshFJ6gU/KZCVyAw6+1MEL5JyCfUdq1xouoOf9TEPZ3wanj8MzEEzXQjY9o+QPxoAwjEQCXYn2Apu+MZDIzA84PrXUW/h2zhyZJZpCRjJOKn/sLSzy9uZD6scmgDkRIoYExDI7E5qUg3AJEfznoUU12EVjZW+PKtYlx0wtTkgfcAH0GKAOTh06edQBZMD3YkAVYj0K6OCzxRH/AGjkj8q6LaeuSfqaQoT2pXAxToO7h7wEd8IQf51aTSbOIYEIk/3ySKv+XR5ZouMrpa28ZytpCD6gVKAx4HHsAKeUI4wc+lNww6qam4DfLJPJJ/Gl8sjoGpwzjoc0u+QdAaVwGlSOzflSlSP4W/KjzZCCSDkdqUSSEfcP500A3H+cUpCnqaUsQMkHFRm6iCFnlVVHXJFFwHYi/vgfU0CND/Gv51WfVrJASCZCO2yqza7E6kRWW49yTgCncDV8piD5Y3Y64OaVbadlzgj61zUs8txLkOI17rGcCk3XCf6q5kQ+uSaVwOk+zyA8t+VPFsxHVzXMbtQKlvtTtjvuxU8N1q8UWY5mK+5zRcDf8sx9UJ+oo81hwEH5ViLrOrxnP7tj6MKWPxBqbthrWIj3BFFwNozMepppkHqKzDrUwAeW0hKZwQvUVZTVLSY4+zTD64oTAs+cM84P1pwnXGNp/AVXa9sxyUlUeuAab/aFgODNID6FDTAtiaMdCRn604yA/wARqqL2yzg3aHPbBp4uLYjIuYsduaAJdzH+JG9mFM2b+tvEx/AUqtCwyJoj9HFOChjhcH6c0CIfLRTzZxfgBTXtbRjuNlCSevFaAsbgj/VN+IxU4tLa35uJlzj7neolJIiVWEVuZS2NncAxLYp83X3/AEqUeHrXCiWMrt4/1h6VsQQs25baDap/ikGBVtdPh37piXf0HAH+NZub6HNPEu/ulHTtGsYJBMtsMgfLuOfxxWhdySJFiPgkgEjqB6ipHdIoyxdeO2eTVI30jsSEBHqalRcjns5akC26iEu+Qx6s560wyIOEOT6jpT5gbhgXBYf3SeKhlubW2XbI6qw/hHJrWEbIuMbIWKMJk5JZjkknkmnSSRQKGlkCDsD1P4VlSaxM4xDGIx2JGTVBlZ23OSx9ScmtCjRutUMyGOBWVT1Y4yR7elZ560AHpQAe9CExR0wKMUopaokbilxRRTGFKKSl4oASl6HFJRQMKBQelJ3oELRnFFGKQxRS49aaDjjNLTAWgUUUgFpabS0wCkNLQPegBaSlptIEO70Z5ptLQAtKKbS0AO4pc5pvOaKQBtBPQflRtUdhQDzgUZ5pWAMAdqQgE8ClphByACBk45oGKQM57Uw4qzJYTxoWZ4zgZxvGcVJY6TeajEZbaMSL3+bBH1qW0tWFiirFWyKtqdyg1of8Inqp/gjB/wB7/wCtUVzomqaZbNPNAHjBwdjZI/DFZTlCWzFODtct2f8AZZgSO90yKYDq/OT9RnmtaPRPDt6AILaBH/u9D+prkLXV1DKs0ZUeo7Vqwzx3CCSFwR2I6iou0Qpyjsye/wDDJtiWi061dM8DkE1hyrDbttn0uOJvQg109vrV7bqFLCVR2fr+daUeo6dqahLhEjfsso4/PFUqnc6qeL/mRwu+1P8Ay4W5/Cm7bc9NOhP04rrbnwhBK5khnMLDooHymsG60nULEs0sBZF/5aJyDVKSZ2wq0pbMzgIRz/ZsP4saUGMcjTbf6d6GfceCM+hOKD6Hr3qrnRyIgbK3QmjtljGMFAOD61OssZ+9p0H1xxQVI604Lx1FRsVyoeJIhgiwtsepGaDMO1haj6CmgeoyPTPNJtJ6EfjTuyeRDzMOv2G2B9cZoMqsObG2PvtFNKc8kj2IyPzoK8Yx+I5ouw5EODxAcWdsfquKgulFwFEcMULA5ygwTUgGDS0bqzDkQiSxiJUmtIWIGCcHmn+Zaf8APlGT+YpuAeMjPvRtHpS12Qci7Ega1GD9hh/UU8S2w/5cIT9RUAAFKCB1Gaq7FyR7ErPak82EI+hIp0U9vC+5LJQfXcarnaTnBFGR60XYnTi+htRappjrsutOJP8AeL5/Slez0y7XNlcNC391jkflWJkGgEdqS0MfqtPsbZ8Nai0XmwCOWP1DgH8jWXcQzWbbLiJ0yO4OPzpsVzd27ZhnZPbORV1tYnktpI7hEkBUg9ia0hJ31MHhLMxtNRYhIoIwWJAx0zV1sE1i6c0X2+FZy3lMcEqcEV1b6RbS/wDHrd4YdmOR/OtG7mdSi73Rnd6Q9a05PDmpIoaNVmGM/KQD+VUJbae2OJ4mjb0YYqVJHLoRGjmkDKTjNLkdjVXHYATS5OKDSUEi5oycCk7UUFDs9s8UmaTNLQSFGaM0lBQvFFNzS5osIUYoPWkBpSaYBx6c0fhScUuRikMMDuOKQqh/hFL2ozgUIBvlR/3BVhbu5T7lxIuPRqgPtQD60nFPdBexbOp3rDDzFh7jNRyzLOP3sKN7jg1BxRxQopKyB67jJLO1f7pmQ+u7NZ91pty7DypVYD1GK1M0ZFLkRPKjnHtNRibCrID6qwIp/m6nDH86M6+pGTXQZo3VPs0Ryo5oaxIGIeGM+4OKspqsY53Tp/uEgVrS2ttOCZIUJ9doqq2j2bYwrr/umk6fYXKLFr8qkFLxs9hIODWjF4lvNm0mF/wxWSNCtieLiVfqoph8P44W9QnsGUj+tTyMXKzbXWYCf3lgQPVSCKcZtCvCUntdpb+Irj+VYtvot/FJuWWB0PYPgn9KufZLlD89u49doyKGpIfPNdTXj8HaVcIJIeV7FHrKvfBYhJ8qVhntIOP0qW3lntH3QeZH6gqcV01lrlvdARzbEkPGxu5qXKRosRU7nDy+C7iPDySJsP8AEoOKibwirAB7vA/3K7m/m8jIS3lKnqowRUMc2n3EBM0EkTerA4FUqnc2hilb3jkYvDEca7Tfkr3GypB4Zsv4ppD9MCuoOmo6FreVZMdun86ifTpkUs8ZxVqSZ0Rqwlszn08NaSh3MjufdqedE0kHi3/U1qlEBwRSbEzwKoszxpWn9BaL+dMk0S0Y5jhEfrgZzWpjA6UnmMOlAXMk6JGOiA/gKaNIjXrCv5VseY2KaCT1qrgYzaemfuCmnTV7Oo/Ctoxqe1L5aDtSuMxPsYU/6wH6AU4WRbrz+FbeEPVU/KlHkj0B9qdwMX7EAOVP4U02sfcSce1bheNRy4A+tQvc2w6upP1ouBimOEdVkqNhDngMPrWu80LKQFU59qo3EKMcqgB9RWbjcDMnKqhAIIrOmcEcVozwk5wCDWNOZIpCHBHoexqORiuMkbjINRCQ4JHPtUDsd3GB64qHznDDBxWqgwuaC3CE4IINSb0xkYIrPErHqacshBzmjkYGkrgjKnmpFkI6mqULh8g8EDORUmffn0NQ4MostKD1NQGQBjg0wKxPLCmyIQM5FTyMm5MCGAYEZppZgcMRiqyEkjB/WnMxIwSDVcrC5EqkgZ6Ux1UnAJH41axGq8uAKZutSMAMT61sIquEUcsAaBHGwB3En2qz5luDxbhj6k0qyJj5Y0UfSmhlXKL94HHtT3khdNscRJ9SOaezGTK4wPXFMB2HCnJ9aBEIt5nPEbVLHYXTkAKB9TUglcfxH86DM7Dk0BYli02QSAyTRgfWpTbCNw4mQgZzkVSDdyuW9c1G8hbg/d9KQE80Nu7lnuxz2HQU1ZLCA4BMpHqKpOAWyBgmni3cR+Y67R9aLCLFxcRzJhECD2GM1BGNrAnpUfXgVIg3ER+tAFqIAYkB6dAaDjzgTkk9cigI6yBBjAHrUkaNPOoAAx1pMaJxZxyXAViQp7CoZra3jkIFuXH94Hirc0LjLjGR0waiV2yAz4P0oKsRfZrfAZI2Un2zTlhQHnJ+oxVxpGVCWKkDrUIIlXIXB7mncloiMYHRFz64poO0Ejhh1IFW4QA2DDI+OoAxihwpfCxMB71NyrEccz4yTyetSBhMQHVSB2I4NPFuuPmB/CmMkaNgI/5U7jsQyAOwTyY0weoFOCAcAgkdcVZQZXBU47ZoJWPjyowfY1NxWII7dHcqeB3NNmtwo/dyLt7VMcecM/dPWpAluTw7AegXP9adwaKC2IkIDRliepzig2EIYgKwA9K0C9uOBIxHoRigeX1BI+lAWM4WYVWAygIwSw4FSx6cI13eYrEdexP4VbPzEDqO9LtVATkD2zzQFis0EIhV5Y5Cc8bR1qLzEL5EcgHYEVpJjblvwo3AngqP0NAWKW2R+RBJj1xSpFGwy0YOfWr59CXP0NRGBi2UcAeh60AVxaRKwOwH/ZPQ1MVXtbxqPapfKfOSUP0p5hIjDbgc9hTbBogWKFyAYQSewoexgAJMWD6d6lCvGwYEgjuDSyEO4bPPU57mpuFisbeBACIizD1Jo+zwvyIRu7A1YZCSCSAAO1BhDoDvIJPOKq4WI/JOSphQD1AFGyNVAWIZB64q00agAIuD0JLZJ+tRlimcHtzjmlcY23GXz6mrV4FaNYyRkdh1qK0HzAU28cidsA8YGRVMzW5EsMQI3J9M9aa8VuCdgYkdRTZHIABBJPTvUqmPG3q+MnipLGk5AAUDjk45pXgjIJEmSOg6UExo+xiSCAQw4xQqxs+MOUJ+Ygc4pXGSFAGicMSNvIHGaV/mOQSB2AqWS2llZBaW87gDByvQ1MND1EuMxFR3LEU7ktooAHPNPIAHA5rRXQ5gx8y7SL1G3dUyaLb5y93I5/2UAFFx2MXLD2pyuAcls+w610QsLEdLYH681YiEUKbY7eED0Kii4M5cRSSyKqxSEscAbTVyPS7+STYlqdv95uK6ZLoAAkLn2xTvtikcn9aLmTnPojATwzdO4MskMQHcck1KfC9srDzLvA9FXr+tbP2pQcj+dV3kMjZJH50XCPtHuUW0DT4+jM/41JHp1hFjFpGcd2GTVjeP7ppynP8AAT+VM123JIpxEoWNIkA6YXFSfa5f7w/KoSVI+4wpjGPod2fpUk8sWTmZ3B3MDTMf7X5mosQDvj60mIf+emB3x3oKSS2Js00k0zFsPuynPuadviH/AC0X86dxilvWk8wDqaQyJk8Aj1yKZ5gJ+6T9OaGBMJkPGeaUSL6Z/Gq7uAM7CQOwFQNdQjAkDR56ZHWkBfDZ6Um49hmqBvbTH/HwPoOTUTatDGQEV3J7EYoA0fNIPHWlEszdDWPJrEwjJS1QehLAn8sVT/tS9nTLsYQO4IApAdE7yoMySKo/2iKia/gT/WXCP7IMmucYLIN29nI7lic055JJIwhMeR7dKkDXm1mKPJhhkfHrwDVc6vcS4IEKg9E71UUZjUHggYJHGaFA2EEDOcgk8g1SAdLNLJkSysAeMA8fjUG0D74JB9Tg1IQDgEEZOMkUqSbVJcBx0B9RTAYSpAJzxzyKUElOVGB370FsgDBwTwOooBjCOGOSBjgcCkxgHbGAAAKUSMAS2DxgDsKUIChJJB9BSKEGcnH4cUWAElZOCp579qkM88fAXy1Pt1FSCS2IKFVY8YyafC7yEK0KlAc5yAPxosFisZP7xIJ6EjFSh5TGCYDIuCVI9R61ZnmeOTFxsCE8AHJxUQv/ACUBhQgL0ycc+pFArkLtzlrVhn+9kY/KnA2x6iRR2Kg8/pVlNanfOIEdh3Bya0ILnWLvCx6aAD/E4wP5UroTmluYTbFYgOSPc1G0iD+Ifia7mHRH4kvFtS+OybsVfFnp8aj/AESAkdCYwTUuojmliop6I4SHTbq4/wBVZyye4HBrWsvCLTbpL7EI7ICCfxrqXuONseFXsAMCoS8pOMnH1qHUbMpYmTVkVLTw9pGnkSLGZHXu7Eg/hV4TRj/VwgA9wuKaItw5z+dPzHEuCxHsOSai7Zg5N7sTc+eeKTYA4mMaOw7kVCbvd8qIRnozjH6VC9uskhd2ck9gxA/Kq5GxqLZZl1NOcRMcdaqSXVxKcJlE9COaeYY4RvcrGg7scCqE+s28JAt085ueR90fj3rRQSLUUiyqMT90ZPUnk1Dcalb2hKFvMkH8Kj+vasW4vLu7kBlcCMdI1GABUYRR2qx3Llzq91dxiP5YkHACDn86phfx9z1pQAO1OAzTSsO4YPoaMGn9qMUwG4PeinGm0wDvRSmg0CuJml60UUDsAooNFBAUmaXNJigAo7UdqP5UFAKQnkilpOhzQAowTRnFNwO+fbFOBGO5+tAC5pc03FKKAFFFGaM0wCjHvRRQAvaiiikITOBRnjNGPXBooGGacDxSYGelGB2oAXP1ozQABRxQAnelJFKoi2Es5DdgB1/GmjJGTQMXJFNOTwVB/CnUtIREVx0X9Kltri4tyXgcoR3BptIQM5GalpNWYzqtM8XcLDfLzjmbgH8a6iKaK6gEkUiyRt3BzXlhFSw3VxbENBM8ZH901yTwibvF2NI1O52+q+HbHVFz/q5ecOvUH39a4y60DVtHYtEC0fPzKOOB3Haul03xVHMVjvFCSf8APUDjPv6V0DSrLFywKnow5FY+0nTdqiG6cZao82t9dVhtnQhvUVqRvHOu4citbU/DFndTLcRRIZB94ZwGrmNTsLnTSHisZUTs6HI/KtlaXws5nTlE27a/urJsRyEx/wB1+R+tbtrr1rcfJMvlP37rXn9vr2RidP8AgS/4VrQ3MMwGyRTnpzSsRc6q+8P6ffkOIwjD+OLgGubv/DN1ahpYCJoh02nn8qlt7y4tHBhkIHdTyK3LTW4JgFuQY3PcDg01Jo6KeIqR2Zwjfu22yAo/o3Bpw5GQT+Brvr3S7LVkDSBWz0dTg1gXnhK4hGbOUSgDlDwatTT3PRpYuMt9DBCkgkMD7d6TB6ZOfQ050ljwJ4GjK8Zxgikzlc7t4zjnrVnUpJiEuOeh/MUBieeD7igEAjaSD6HpQWIOOAfbpSGKCDkAgn3GDSYI6lR6Z5FGQw+YhgenqKVTg43gZ9elNgJgnnAIHcHIoyR0AP0NSFOM9PdelMIbGAQ316igAB55A+h4pxOCPl/rTASOMgH3HFIcg9wfUdKAHkZzgZ+hpo5BAKg+5waDlgDkE+/BoHy8sQPqMg0AN5Hf8MU/BIBA/I0wnnjj6dKUZJz8ufQ8EUxD+SMjH580oxg5HUYwRTMkHk4H0zTlODkHI9AeKEBhxHEo9A/9a3WGDuBwT3BxWK4KXLg9jn9a2FbCL06CrZlFK7TL9nrWoWYCpMSoP3X5rQHiuUtie3Qp2O0f1rAzzS/Ws+VbkuhBu7RviTQb9R5qGGY91+Wq8+iI53WlyWX0k5J/EVkYyOKEeSFt0chU/wCyaadjP6rAuyaVfxLuFu0g9UIb9OtUyxRsOGU/3SMGrsGtXsAwZN4HY8H86sNrNtcDF1bE+pwCPzrTmMHhHfczM0da20tNEu0Gx2jkPZTjH4VWl0OTI+zThwf7wx/jQpGEqUl0M3FHFWZtOv7b5pbVwn94YI/SqxJHUU7mQlLmkBowDVDFznA70lHbNFBIlLSUUDFJ5o6d6KQ0XGLmgGkooAXIozTcUc0CH5FLx61HzRk0CJOKQ0zJpcmgaHUU3cfxpQ1AxSO+KKN/GD0pMj8KBCmjGaOMUZpiE2n6H2p6yypwksij0DU3NGaQ7kwvLkDBk3D3ANAvCT89vG/rxiocmilZCNGDV/JGBFIg9AxI/lVqPW4ScMTj/aWsTilwBS5UKyN77VYTc4hJ9A2DVhEgZcxuV9g+cVy21f7oo2gcgkH2NS6SFyo60REceYD9Vo2Sdlgb6rXLie4X7txIP+BVMmp30YwJd3+8M1DpMOVm7LbmQgyWq8f3JKb9lth1gkX9ayk1u6HDxIynt0qwNbQfeR1+hpOM0PmqLRMe1mhJ/ehR7g1B9nYuQmGx3zUsmrI6gJcFP95c1TnFxdjKXdqSe/KmnzTW5oq9Rblg2U46pSGCMD97OiH+6azzp2p/wyI6+iyDmqxW4hYiWF+O+M0OpIHiZ9C49va5P+kn8BSCws5AD9vxnsQapC4QHkEfUVMkkbj7wP40vaSI+sVO5KdGjLlVmDgdwcVFLpRgPMDOp6EMCfypdiHpz9KNp9XpqozRYp9UVm3IcCzm49BVZ9RaNiDaTD3IrWSaaP7sh/HmpRqN0OpjYe6Zp+1Rf1pdjmpdd8s4+zk/hWZe6ol0CDbhfc128t1DcY+0WNu+PRcVVaz0OUES6W2fVXp+0RSxMLanm7kMxIqPy2znHFegyaD4dmB2JPC/uc1C3hPSpFwmoMn1rRVIlKtB9ThghPapUhPcV1r+DEVS0GpxOR2OBUDeE9TQExmOUexFUpJlKcXszn0hZWBxwOtP+lajaDqaNg2b49RzUMljJbtieF09yKGy7plHcwNRSuW6mrUxiCHaDn3qlIyk8MKSQCK+05IyO9K0ykYCHHrmoyM9x+dAQdyBVWQEQIJGKkCk9BU5sgjkGTLj0FRFZEkweKLoBm184PFKCVI4zTnBUjGT65FWP9VGG2Ekj0pNoCvmQsMoQPfilZjuzt/Knea/JMZA9cUxZiTgqfwBpJjE3OT93FNK85NWkViCTGTkdxUXkyk/cYfUUXERqeeKUwO4ABAz61aitWSMlx1p209lJ+gNFwKaWiK24tuI7jirnlNLbEJF5mOvHQU8IwiYiGQkjGADU1k1xDb7GhGSSelK5RlQwF5AFQn68U8WciXIyOta0drIZckFc+lWBax5BcEkdyaVwMia1liIkcYFNhiczKMdTitWa3icEMXIPYUyC1CgbQeKdxWFaCUEqm5j6dRVS9jMVuXIw4IBq8DMzYWYL9RSPamYYm/ee5NK5diK0VJtPQupOc5xUUswh/dohAHrU5hUcCYx/wCyhqN7RJBjMje5OTSZLRJDcNMdocAY5zUwdcYLg49Kr/YYUXI359CcU5IDHh2XAPvmmhgZgrAKDknqRSEsckyAkntVpwCoyiYI4LGnIEWMKPLB7470wKqqpOGOR6U7y4zKQiH8OauiyhOcy8gZIXrVNVlIARWx6k8UgJVBRQGTI74GTTgyLyqYPuMVADOzACXHrngCmpcbJgjOCR1JFUA8hQCCvfqRTCB6VZm2OhGRk9MVXVNgGckDue9ICUOjk7E2kDqajZAwJcAn1HBqUwEAuVJBHGDQSIwMg49aAIkj4xgIQe/erBAJUkIOe1AS1JzNIRxkADvSBCTuX7tICWKYxyElQVxjJqJjbliSWz6DpTRIoJBPA61HuVpPlGM96AHllAAVG4PUmgOWO0DYPUmhRHswXcmozIEIwGb6UwLJt+MmcDPrTBbgqQ8gPoV5p8ETXP3LaUt6bSatjSdQmCiG3MeepkG0D8aRLkl1KgSOMYaWXb3wKQmFuArlO2OK1YfDF0fnuLlBjsCTirMeg23CyzO3r82M0C549zm2QCTALBPUmlkSNACJixJxgV1x0exwdsUbH/aGSaytfEcaW8KIq4bLADHpVBGalsZ9uMMAByeAPU1c/wCEf1NmLhUG45wW5FM0+IS3cS44LCuuMT4/1y05GFSo4uyOX/4Ry5H+tuEU+gOant/DsCgtLNIX7FABit8wuf41P40oiYc4FSS6smULfTLG2X5IQ7DvJzVlVVPuRRr9Fqby2Pajym9KRDk2M3MRgnA9BxTcepJp5jbHIpMPggA/hQIrzW4fkL83vUYtX9B+dWt2DyDn6UhbnoR+FBcakkrFf7M/QkYqUWkf940vmgHpk/SjzQT3A9BQN1ZMekMYGMZpwiQH7oqES4PI59KC5PIBHsaCOZ9x5jQjG0VD9nUfxGjzDUE935Qxg5oLjKV7IcIzuwxwPc1YRdi/1rHa4kdskHPtUnnXLsAmQMetBrKE5dTVMm0ZIX86z57gO/AX8KURXbgHdgY5zUJhgiO65uEXPbIoCnT5dWIZGPTmmgk/wE/SkOqaZC4EJMr9gBg/rTJtbmQAR24iz0MnNBuWVtJnIHlgZ7k1IbCNP9dcKg7jOKxZL7V7hykNxGueojABA9arTSyNP5UxhYgfOSc7jQBuG90i3IQv5jeiknNDa3Ap2wWpx2DcH+VZAEa5dbcEjn5Rjj1pokaSRSSA464OcUAX59Uv5wQqgL6AYqgwluJP9IJwORnnBqQgpIHDHOcY6mmsZFlLFmjx0LJgZoAijgAcne4HYDj/APXUhYkKCFLjA6daUTBmBDbsdRjvSTNJlgNhyMZA5FIY50VYy3pTtgkjBAySMk1ESFnyQWjUZAPcmnpNKSQilQwwT2xQA0R+WCcYwSMdz+FNEo3YERGfQVMU7kgHuTVqOGRwJCQFAxuJ5FANFEP85O1uBnGKc0yeslTCIybn2FjnAHY/WpSpQqxQRkDALgkGgVipGjzzRxwhm3Y5PAFTS2qJcGBZpHkAyQEAHHXmrccc0smRPHkAMAnU+lMfT5WJbLq4OQT1FAFOaJ4pPLWCRnxn7wBx9KFtbtgCbdwPQkD+tWBGQzNIw3cjJbkD2qS1ALBRmTPJckgY/GgZQJK9QR25p1rdW7SSRShi46IRnOPftWtc3VrGxLx+Zs+nXv1pLO8e6cLY6S2/u+On40wbS3DT4Yzkm1VCBkEjOfxq1cRRqjIyD2wMACtFdD1CRf3159mUdRGMn8CKu2eh6bagM4aeUHO+Uk5/DpWLqIwliIJaHJxadd3sgS3gMxA+8zYAPbJ/Otm38JPIA9/dfvOhWMdvQmuha6jUlUIIHTaKi86VjxxUuo2cs8TN7aEdjpmn6XGBDboHH8bLk/rU733JC/Nj0qOYnALEkZwc0wrGil2ICjqScAVDdzBtt3YF5ZCTgDPpSGNifmqCTU7SCREw0u4ZzGMgU8X0kvEUJX0L9fyppNhZk4jCjJIA9ScVC97BG21cue20cfnVZ1klfdKzPjsTwKcImHRQKtU+5ah3Fe4mlTG/YD2Xr+dRsVt4gwDuc465PNQz39ragAvvb+6gyayrrUprsER7oV9QcGtIxSLUUjUuNZsoL0W5mBbGSR2PpVe41wgkW0R46O3H6VirCi5IX5jyWJySfenjNWMdK81w2+eVnOc4PT8qFUAYwABQKdihIA4oxmnYxwRRgY5piuNGKf3GBgUACigA4pabRTGBooooEFFFHegAFHaiigAo7UtFAXEoooAoAKKKKAEpKWkxmgBCQOtKDTSOacCAfWgBwPrR346Ud/agHAoAWlBH403NGaBjjwOaO/FJnnHalzigQd6O9Jk5pTyaAA9KBQD2ozzQAvfFFJR0PNAB0OaWkzyfrRxQA7rTfalyO1JmgELS02l4oGBzS9sDrSDpSikITk5FN2nNOPXNKaAEeLy8fMDkZ4PSrunatcafICjlo+jI/Ix9KokUYqZQUlZjTsd1p2t2eoABG8mXvHIcc+xrVaJH4dQTXmG0ZzW7p3ii4tgsV0vnxd8/e/A159XCyjrA1jNPc19T8H2F+jvGDDKe6gYz9K4u90TUtCnDzQmWMZ+eIZH/ANbrXpNjqNtqcHm2kpY90bhhVrggqyhgeoI4rNYiSdpBKlGR5Za62jnbKp3DuBWtDcRTRAo4IPpW3qvg+wvcvbRCGU8/KcAn+lcbd6dqWiTESQ55xvjO4EetdEZKavE55U5ROgjvprAeZHKQg6g8ituw8QW1yo84eWx/i7GuGj1ISWzpcpsY45PQfWrthcQvariQFgSOnWhozud5LFZ6hBskSOVPXHI/GsG/8JK5aWyfDHoh6fnWfDPLDIHhkKn07H61ow+J5YW2XdsGXu6cY/OhNrY0hWlDZnNz2l1YybLiEiq4ZC3UD616LFeWl7Ajo6FXzgOOayb7wtb3LtJbS+Qf7o5H61op9z0KWNT0mckFjxkgBvUUmDnjBrRn0DUoHKmIyqO6nIqg0LxcNHIjfSr0Z2QqRlsxn3eA+0+nalIycsoPuDSEt1YAj170gK4JEm056E0F3HqXAyrjH91jTgD1YlfQjpUQaP8A5aKSPUGmmQEYimBA6gmgCxlugAceoPNMLOCcfL7Nzio/OgAO+TafUGmvexdBKJB9OaAJQOpYfUrScNwAHPoagGoW44+dPqM0fb4c/wB76DFMCzgrxgqPQnigRk8gKR6iq39oQDjZIPwzSi7Q9IJD9BQBUux/pRA9B+daKE+Wh2kZA4HNZlyZZbgsiEYHIPetG2cSQglWiI4IzwT60yFux+CQRgH6HmgKB1J/Gl+VupBI9sGjGOASB70ihw6cA/UUZJ6lce/BpoAPORn1B5o4PVyf94UDaHcdyQO4HIowvYA598Gm4HUA4PcGjnJ+YfQjBoEOII4249z/AI1LHd3MIxHM2PTOajGQOpA9jkUgx1GCPegTibdj4pubVQJo94H8RHStT+2NCv8A/XxKD6uvP51yBLdDmm7FPUqfxpNJnPPCU5O508+iaXPhoJ5oc9uCKzLjR5oBmJxOPX7prNBdOYpHX05zVyHVruLAYrKB69atGX1REMkM0YHmQyID3ZTiodwPQ1vQeIAoMcsTIp9sikZdL1FsBkDd9rbD+VUmzGWGknoYfUUgro38JHYDb3gII+64/rms680LULNS7QF0HVkOaFNHLdGdS0meTnII6g9RQcVZQZopKKAF5oozSUAFFBooADQevFAozRYQUUZoosAmecUZwMUYoxTAXdzzSZox3NFADs0ZpuaM0CHZozTc0UAPzRkU3tR0FAWHZFANIKBxSC46k5pMml3GgdwzSUUUWBBgHgjIpAi+gpc5pcj8aAEAx0Zh9DUiXFxHwkrAenWo6KAJmu5mGGWFvcoKYJY8Ya0hJ9QMUw0mKlxTFZEh8gn/AFLRr/snNIVg/huHH1WmUYpezQuRDjA5+5PEfxxSi2uj0RXHsRUW0egowO2R+NL2aDkQrJOmd0L8dcU0S46hl/CnFpP+ezgfWpEuZlXbv3D3GaXsyeQhMkbdTz7ij5D0AqdpEbl7dCfUcVHtty2TCw+jVPs2LkZGyRnpkfjTAjKcpI2frU4igY/fkQe5zTWgA+5KD+GKXK0Tysas94vSd/zpDPct/rHD49RTvKlHTDfQimMk4GTC34DNLVD1QmLZv9bp8MnrgYpkltokwAk05ox6pQzheoYfUUK0T9WoUmCnJa3GHQfDkwODNHn60g8IaFJ01CQfXirIWMf/AK6diP0/Wj2ku5ftqnc5qOG3jXCsmfU9aUJHnI2keuKqSFRkbQKI13dCQPat7HqF7EPfb+VKBFnIwfwqoyugztJA7VNDGzKS4Cj06UDJmMJ5KKT9Ka0kSDJRcewqErEufMlP0FR71kBCggA9xQBYFzCRyg/AUhnh7Jn8KrmMgABTn3pQNp+ZR+VO4E5u4V6x0HUE6CPFNAiwCUAHegJbiTcsZP1pA0KL/PRCfoKf9rdukR/EUhumOAgAHoBSGBpFLgj6UAO82c/8sSPwoeR48B8AkcDrTRHIRzI2fTNRSRvv6biOmTQBIJ2PAAPvimMZCTgEE+lSBDsGAMnsKRUZcl0IPvQBEYzjJfDfWmxIZG2vKwFWVjEwJ3Kv1NMMREhVJl96ATD7PEB8khJ9xQlrMzZ+0FAO2etII2DAiRs/WlEc7cFyR6A0A9SR1AbEjgj2qOJADkk49TzR5Wz7ykZ75qy9sUtA24gdgKEBCUH3t2R9OlN+UNgD8hTP3oADAkE46U8GSMg4GB2NMRZiJELkEBjxn2qIcDAQ59zQ75jySAc8gUiXC9iUJGOTmkUNCMoyyjPpTkkbHEagj1Gad5MrsQgZyfQVIthqDnEds5z3IxSRLa7kPzH526U5QTyVyg71oW3h7UpcCXZHEP7x5H4VpJ4btV4lu2PsMAU7mbqRRzhIYcygY9DTUOWwfmPsc118emaXCgURFvdiamRLGHmG2QN69aCHX7I4xre4uHHlwSMB3CmtCPQL6RcrBIv+8cCunFzJjChVHoBSF5H6yH86CfbSMOPwxKSDPLEg745NXE8N6YMeZOzgdhgA1oYJ60bF7igh1JPqRpZ6bF8sdmpI7mpVeNBiO3jT6KKNo7Ln8aUZUYwAaCA8+b1x9OKDJIert+dLuGAKUYoAjC5GSSfqaNox0FPycmgtgUgGAEHiuY12UzamqEYCDH5810sj4XHrxXI3j+dqUjZzzwaqJ0UV1NXRVAnU8fLk810LOSc5rC0ZB5kjnoo4+prYLcU5bmFT42PDEdTSEk85NRGQjsKTze2APpUiJt5HQkUCST++RVcyHsaYZT60CLnnOf4vzpPtLgcOuPQiqgYscA8+9IT23Ak+nNAFz7W/UbTR9tY9QlZ7syg7RuI7Coi8jA/IQf1oLUJPZGmb0DqgP0pDexr1QH6VmiFycsxH1aopby3t+GcMR2UAmgv2Mi9LqI8wgQHH0pI7oSNzAcVjDWWVm8u2Zh2LNjH4U2TWL14+NkQ9AMn86DdU42szclRpB8ilT7ms24Npb/NLdBnPUA5IrHlmubpCpkkJ7necAVHb28kcx8uON1IwWftQVGKjsabazbRDEcLSMOhHSmjUrt8MAsSkZ4GSPxpqrDDFsFuhfJJbsPp7VXEbBifNYE9ADwKVx2HzXFwz/PcSkegOBVNrcF94ds9geRVljggli4P8R71Yjd1R40iQhxgkjn8KOYoz/KKuAjZYjJOKe7B2UyOWKjgHsakVMPleBkDFOLgEgAHHqKExWIt6OuGIVj1wOoqVo7YAYTeT2xzQjlTgHBoLYfeASR0wMmi4xu4E88AjgDjFPAjjAQDkHJxTPNzPmVJFA6sR0qfbCATskGRkEnk0AyIyMHDg4A6GnvcmZcMcn196RmQwFSAT3OOQaiDRjrGDn1p3ESiEZLYAAHJB60RxpjIBwTgknrUBCKSQhUEcYOcVIuSgUuQT0yMUrjuPyV4BH0IzTpWIWMb3LAc47CoyhEgx8x7gcmnIigMOhJyQ3BFMQpJCAIC5PJPUgUKu44yc+5wAaQuqsQiADOCQetSLC0kWVGRnOSaAuTo+0FAQ5GSMcc1LGbiaMHaG3jkdh9Kjt4AbciUHIOAc4Jq3ayBSy26NMy4+VOTSHe25DGk65jiiC55LA9TU0KzIxLktxgZPSrEXhXUrljI8iwbucN97H0rbs/DllbAGRnmkxyxbj8qhzSMJYiCWmpy4NxNciK3i3sf7ig1rQ+Hb6cD7XMEA7Jjr+B5rpAkMK4REQD+6MCozc/3TzWbqPoc88TJ/DoUrTw9YWqnzU8+QnO6T/CtFp44fkRUAHZRVVmkcnLkZ9KBHjrzUNt7mEpyk7tkrXLucKvB9ajKM4G4k1KkeSMCklmihba7Hd6AZx9aRIxYwMDFS4WMZYgD1Jqqb0sGEabRjgnkmqxy5zKxc+pq1BscYtlue5SRCkQZiejYwBVG5hmuhtnuio/uoBipNqnsKY7pCN0rqg7EnFaKmkaKCQ5I0iQIhAHsBTyAAWeSQAdSSAKyptcjHFum7/bPArNnuJbp900hb2HA/KtEVY17jWLWEkQgzMO+44rKmvrm5OGk2r6ISBUIAHalwPSnYY0KFxxS5HqPzp2KQIM5IFFiQCmnBTTgKUHA460wEAIpwFJS0AA65oo9qO+KAA0UUCgBO2aPx70UUxhRRRQIKKM+lGaADtmgUhooAUHnmjjtTSaM8UALRmk70YpiHUlH1pKQxeM+9ApO+aM4oAOexpOc0ooPXigAyTjNO7U0DBznPtSkEEjoKAHUZxQODzg4owTzikCEJAPUdaXvSYHfH40E9OM0wF9+9GaQng0ZoAXHQ+tGaKMUALjPFJ7UUmKACl5xRR+NMAGR1pRkCk5JopALRzSUUDF5PSjPHvSZpc0CDAwTQRzwc0DkGjoKADtTvpTaD7UDF4xTSBnp+dO6dRSZByPalYBY55rdt0MjI3qpwRXS6f4vYbYr5S64/1i4DZ9/WuXxRisKtCE90VGTR6daXkF3GJbeVZEP908j6ipZYorhDHMgZD1BrznTmkWTKXawlCOe/P/6q7G31aNUCzXkU5HdMA/jXm1KM6b0NlJNGVqvgaG68x7GbyXI+42SDXIXGn6ho10fOieIAcMRkV6vFcxzKHibK9D6g02WS3lRo3CvnqCM04YhrSWpE6SkeXWms55ldTjuBitVLhLqEFDkGtPUvB9nODJZEwyY6ZyD+FcpdaRrFgWPkyKg6MuMV1xlGWsWc8qTRqrmF/wB2xXPJwcAmrMd1KhBWV/oScVzw1e5i+W5iOR1JXBq1Fqdu+OSM+oqrGRujxMbfi5hb2ZGOCKuW+r6fqChRMELdA+c1zbvFKQCRz71FCbaYlc4IOBxzSsF2dLP4X0+7/eRXgjkJ5weKrN4IZgRHfoSfVQTWUi3FuN9rdEZ6oeQalW41F2ZnuBG2MgoSAfwzTUpG6xNVK1yzL4LvYlyLlG9gMVRfw3fIu57Pd6spq3HrmpoAJZQ6juQM0r+IrsITFLhh0BGQaFNmqxtRbmb/AGRcIQUtCcdQeKrskaSbJYBG49DmtqHxdfwcSxxzRnqCMfyq/D4q0yf/AF+nhffaDTU+5ose/tI5gDrtRXU9crmmn7OpwYjG3qF4rtk1Lw9MoJiUZ/2KlB8OOOTEPrxT9p5Gn1+PY4oSyEZCrIPrTC6FssTGfQHiuuk07w1Kxb7SEJ/usRVK4sdBR8JfIF77+aamjSONpvcwQT/GRIlNMkS8IpUehrZj0TSrhswamig9t2APzq8PBMj8i/Vh6kZNVzRNPrVLucyHZs5RSB0IODTgewlKj0J4rpP+EGuAPkvV/wC+f8Kb/wAIRd9rqA/VTS5olfWqXc535AcsoI7FetKCD0OP9+uhPgq9X7t1EPpmqreGdSUkYRvzpppjWIpvZmWAw5IA9x0oJ3dWz9TWkPDeorkgYPqGoHh/Ux1jRvqad0V7aHczBgdAfwpwY9CB+NaX/CPakR8saqfUNmkHhrVSf4aV0HtYdzMwRyelHyk8gH61qjwvqJ6sB9M4/lT18L3nRmUjuNpNPQn2sO5kZAHGR9TmjdzyQfoMVuf8IrOORIw/A08eGWH3zIx9cYo0D20O5g7h3J/Oml19AfccGuug8LxKFYjJ9G5q7FoNsufMjjPpUuSRhLE0l1Mfw9eXjs0bMxjUZAYV06O3rTFS2iHBQY9KDPbL/wAtBn61k5Xdzyq0lOV0NuNKs9UGydAj9pF4NcfrejzaNdBHfzInGUfv7g11Mt+FBEeCTRFDb6pGYLrPqpz3pe0lB36Dp2ehwYGRkUEGusufBTA5tLxQv92Qc1h3mianZMPNtmZf7ycg/lW8a9OWzNHBoz6WmltrEMNpHY8Uu4Eda2RIEUE07Ipu75sigYUUHrRxii4CUtIR6UU7iDPNBoopDEzThmmmlAp3EFFIRRTAWkNJk0uaAF7UnSlxxQaBBmjPtSd6KADNLnNNo5FAx2TQDSc0hJ7UgY/NGaZmjPamIfmgHFN70Z5pBYfnNGaZkilzTAWl4xTQRQCKBjqKTNGRQICKTFOzScUAJnNFGPSjFABRR3pM5OMUguFSCV8Y3nFMozQNMeZGwQSCD2PNIGTGDBEfwxTMUYpWQClYD1gI/wB00eVbnpJNGR+NJRilyoLI5QAFskZHpViOFiMqioPUnmmRsrAAZz6YpxjbBJyPYnFM9IJIpOpmAHoKjzF0cux9BxSgSqQFVMe/NTmU7SDtz6UgIBGmM4/Ol2hRmlwWHBAPpQISepJ+lAAVMq4AYe+KRbcRDJJJ96eiSKeXIHoKgcgPtyx59aAFPlk4fle9OdwMbPu9uKljjVGyVB+tLLCSdy9D0oArjBOSDmrKEBRVcmUNkMCPSpgCRnIH4UAOJ+UkVApDSZAyamTeDhmBBPpQZEDYyAPXFIBmyRnGARjpT3STADdu9BJPKtke1HmEZ3HIHpTARokGCFye5zQIxkHFKu6Y4SGUn2SrK6ZqEsfyW7qPVhgUBciSIucBQB2JNNMZAIeQD6Gr8Hhe/crLLcKg9M5rTXw1anBnuy4/urgCgh1Io5wRZdfm3qDkjNTTh7hh5PGBjavNdWumaNbKDDG5cdQ2asI8KjCRKo9hUp3MXiH2OOTSr+QcROR2LcVai8NXsihnKp9W5FdV5o//AFikMgP09qoh1pMwh4atwv766y3+yM/1q9BpelwADyRIR1LVcIjPUE0myH+5n60Cc5PdgJYoxtit41HsAKQ3Eh7gVL5UZ7frR5MXofzoIKxJbOWPPvTdo9M1aMMR7kfSmiCMjILYpCuVwADzTgFzmp/IX3/Gk8kdjQFyPA9aBt7VJ5B7SL+Jo8k/wspPsaBjfxpNwzjIz6UhtnzkqSfY0jQyAZEXPr3oC5IQB3xQeTUJSYnlSfwppEg4KkfhQBLmjd71ASw74+tNMuOpFMCzu9SKQsMYzVUyn2xQJD7UAPuCPKz6ZNcnAPMmLerGt3UbgpZuQRyCPzrEsuCp96tHRT+A6OzVYomHrxUxkA71RSclcRD5qUfaGOSm71AqJbmUac2rl0zAdDmomnXccsKqsBEN0pEf+81QtcWCozvKJPZP8aRoqHdl8Tp/fFJ5oZ9oBP0GayDrVtHj7NYFn9X6CqkurX8uQjmMeiDFUUqK6nQsdvMhQD/aOKpvqVnAWzN83+wM/rWNLCdpJdy3qeKaYwyBDuI9BSNFCK2RpNrY8zEMfP8AekOBRcXWoxsBkKCMkqMVTJfcpMajAxUkN0d+w7nC/wAPagsQtC5ElzPK7dwO340Q3DCMCEgOPQZNMOHLBj1OaSFCp3xdeQc1AxJGeRwWO1weQO9KQRkADA7U4oyklkBJ6nNSJtETySE+gA6mmgsEd15WNsSAjrgdaZGfM5xtYknJ4FAwwDYx6mnCTaOFB/CkFgMcrMOCFJyHPQ02QckA5OMZHSpomymCCSDwDTPOt0fnBHQgCkOwsyAAAlST6HOBTYgY2STHfANMebgJsbJyQcZGKfAzFAJoZXx0whxTAEcmZmOCeTS7w4ByFY98cUThscIsOeNo6496SXYiqAMnufWnYBRGoXcWDcgEjqM0BlR/lLcetNjIXO2QIPQgkGh13AuGDAcEnjmkIcdzjOSR70K0inIAIHXIpYoZJhtjIAPcnip102deRcRg+ppgJDAbgEb1ye2eQKja1jjfBkyR2x/9epJbKfAzcKSTjINLHplxHICbiMAnB5ycfSkmNoqXULXEgEZYkHIIHb1qLyp0YbpCw7g1stZCPCNO7DsFAApW0OSTD26O7Hs7UXS3JlJR3ZnIsnAjAOASSBipEiE0+JGyWxkA9MCtmx8Nak5/0grBGSTuzk1tW3h2ytlActN6Fjik6iRjPEQj5nJrZxSSBIbZpCehBIrVi8OX0xXMaW6DjBJJrqYkgtU2QxpGv+yAKVrlR/FzWbqvoc8sVJv3TPt/DlnCB5u6fjGH6flWhDHbWa7IYkiX0UYqEzTMeCBSbXbkkE1DbZzuTe7JXuvmAAJzULSSs+AcDHHFPCAUoTJAA5pEkIRmOCTShMHAp7ywW4/eyqrf3RyfyqlNqXGLeM5/vMP6U0m9hpNl/aE+Z2Cr6mq5v4SSsP70jrjgVReR5jmTcSfbIpQCegOfcVoqfctQ7kkk1xI/zELH/cXpTBx0p3kzH+HP41XuLuG14kcGT+6pya0UUjRRSLBZ8c4qOSeKAZmlVfbv+VZc+qzyf6lBGOxIyazygLFmySe5OTVDNK41ksCtuuAf4z1/Ks9meZt0zl29SaQD0FOAz2piECj0pQBTsc4oAoAaBQBT8DFIpBBIzx6imIMYNL1FKRgU0EknjigBwPFFBBPTg+po57nJoGBJ6UUYopgJmlFHagHA6ZoAU0d8UUhyOR1pCAijoaM560UxgTxSZoNJQIWjNJRnjmgAzRmkzRmmIXNFFFIYUZOM0UeooAXrSUUUAGc0Gjmj0zQAAnHNJnHU4oJGccUtAAOeRSg+tJkZwc0KCSQTRcBxPNANHeigEBHvQKO1APbtQAuc8YpB70DrRQADtSjqaQ5BH1pf50AAGCAaOQaOfXNAOTigA5zz0o96Xrmm45oAUGjjvRQRxQAcUlFHegAzS0mMUvJNAC0Z5pM5A+lFACnGOnWk5pe/0ooAOc0hHNLwDRmgEN5ApDkg46048k00A4NAXGNEHGDn8KYbYDkOwqfmgjNSFwt7i5tXLQ3MgNaVv4hvllAupi0XfaMN+dZmOaCuaylRhLdFqTR2VndW16u2O+be33Vk4OfzqO60zXPKLJcCWP0HU/hXIAsjBlODXZaJqMItIzMbzef7q7lI/KuCvRdG0kzWMubQxXghc+XqELoe+VNNm8LWMsBaxuEU9gxrtjPY3wEKs+89BLFjP+FVV0AW8nm28YLf3Scis1iKncThF9Dzq60LULdmAi3qv8UZyKoLE3HzFW9q9ce+jt18u50/Z7pyKz7nSdE1RgV2q59sGuiGJi9JKxnKiuh5vHeXFu2CAcfjVhNXdSN64Ppiusk8EzmQ/ZrxPLPTcKoyeD2M5W5m3IOhjQit4yhL4WZOi0YX9qxFyCmM+lRyXsfVQxB9RiukfwbaeWfKeZX7EjIz71gXWkXNlKFkKDOcHOQf85p8rJ9mxiTo0JPfsMVJEUfPHSqskVxggFD9OKhUXMXOxz7jmh05CcGdBbxIUHFTmFAOlc4uoTA4WQE9gRg1MupXCnJAP41Li0KzRuiJemKbLbjyz8gP1rGOsyMcJw/oaU6tcr1XP1oaEy0tgjzAlcA9QDxWjatLbjy455QOwDHArCOtSA8oAfpU0OtYO50LH64qrAdGLy8U8XUv/fRqZdQvlHFwx+tc9/b8QP8Aqj+Jpw8Qwj/lkf8AvsVNhG//AGlf/wDPY/r/AI0o1K+HWYj8Kwh4gt/7jfhzUg12zPUsPqKLDNoarejjzz+VL/a1/wD8/B/KsFtWtSciUAevWp4763ZQy3MZz2YYNFgNcatfE83BP4VKurXoH+uz9RWULpGHDwt9DQLuEHDOoPoDmiwGqdUvT/y2H/fNH9p3v/Pc/lWcLq3A5lX8KPtVsOsyCiwi+dRvT/y8N+AxTftt0f8Al4eqH26y/wCe6fnSHUbNT/r1/DmiwWLxubs9ZnI+uKUPJIMmR8/Ws9tXtRgRsZWPRVHNNGuW6jBidT79PzpNDNQAk8kmpAAO5/OshdXEfzMhfPQLTW1eeYH7Pb/iTTUWxqLZthgOTVG5v9si+RcrHMpyvNYV3eXspHmqyemBxVdADknGfU1pGCtqaRjbU6yz8YXiSlb6BGA4yuAa6Wz1/TrlRsulRj/C5wa8wDkcZoyG6isp4SEtVobKbR6tNpmn3+WktonP98Dn86w7rwPblWa0uHjfsr8iuMhnuIMiKeRFPYMav2uvapaE+XdSMD2dsj9an2NWHwSHzRe6LNz4V1WBWcRrIo7oc1nW2n3dzMIUhYSHs4wa0h4w1cHnyj9VFPPi++kx5lvbuR0O3BBq4yrrdCtEc3gvXAOIEb2DjNY9zaXNi5juoXjcf3hj9a1G8XauwwJwg9hmql1rmo3sBhurgzR+jDofUVpSdW/vky5ehn5oHNITt5xn2FN86MDrj2IrcgkNJQCGAIOQe4pCSDgHj0NAxTRnijNFAAOetHQ0ce9HHamAZoOKMUUAFGaMUY5oAM80maO+KMUCuFFFFMAo7UUc0CDFGeMUUZxQAE0UZ4ozQAnIpeaM560nWgBe2aM0dqSgBc0ZoOO1JQAuTS803NGRnrSEOyexoJpMjPHSgnJ46Uxig0uabnmg0AOBFBxjPem5oNADh1oNNzQCc5oAdRSZ4oB4xQBzahYmy6Z9BnFSSXJmkGIlGDmoRGe/SpFUAgisj0y0sqMwUQxjPc9KfmEf6x4wv+wOaoyFScEZpEICgZwB0ppgWZpoSMQow9zUYz1BI+lQvKAcAMcegqaGyv52AigmIPcLimDaW48EAcgmmmQDogHvitNPC+rKm9o2Udt/atG18JAxhru+jX1Xaajnja5HtInNl0dQAQD3JNRmNv4Zs13kPhLRCBIztIfXdircenaBbv8ALGjFfUmsfrMXokxOqjzZUfPKOfYDJNWoLK9upVjgsZjnpXeT6hHG+23ghVR0O3rULatdfw7B9BVqVRq9iHXRzyeFdXZgGtVVT1JbpVpfBsA4vNQ2N/dRQa1DqV83/LTH4VWYO7lmOSaUVVfxP7iHWfQba+G9BtZFaSeWYDsTx/KtOODw/B/qrVc+pzWaUb2pyxn0FKVHm1cmQ6re5P8AatjEQxKi54xUbXTsfmIA9KQL604ADsD9RWxnciLuw4Jx65pQjHrmpe2QB+VOz7CmNEQjx2oC84INS8dDSEgHikAgUe9Lx2zSZNLnjFACYB6075QPWmZ5o3qOp5piHkg9jRk9s03zADgA/jTGkOeDQMl3ED0ppkb+8cVGXJ69aaW4oAlMhHegTHuRUIIIzRuAoAlM79guPXFAnOclVz64qDcKaXFAFs3BPUD8DR9rAGChP41V3DGciozIT0FAi6Lsf3TQbqM9SRVAse5wKXk4wN1BSi3sX/tcOPvH8qabi3IyQrfhVJ40WMlyiDGSSayU1e2GR5ZY5wAKZrGi3ubM1xDxsiU/hUauJM4tvyrJl1acnZaW6g/7WDVG41DUSx3OEPcADFI2VKJf1OCSC1csFUMRhc5NUrdcKDiqpEsjCSV9xPv0q6oxbsf9mtEDJZNY+zxjybQjAxluc/pUkd/NcWpkkLITyqjAGazSjyRhWcsgOcmrZRfKGSxA6A81nLc0WxWVssHugJCP4s5H4Ukht3cgRkHsAODVmOGSKZXMAKjP7s9T+FTy3GJvNS1COf7+OPwqRlI/IcYyMdKczMQAmBnnAFAxgs5LEnPPFCkjJIwR75qhWGg5PzuuPXNOlCID5alyOhGKBgRFgq7T1OOaZIQgGCBk45qblWFByoLDDntmnxELuIHUYNPtwEnXe6Fe+aa2yF2xMhUnIwapMQsxRyH2BSBgYqAyfMAp+Y9qdIgkUFZQSDkAjANNxhAH2q/rQOxIJImSR5BgpwNp5J9aUIqRl2fJJycmmEl8ZkDEDAx2FKSxwB8zenc0ATSCMlRFwCMkHrmomjkEg4wBQnmP8vlAMejE8j6VZgsrqXBLDZ3ZjgCpEyEuqRZIO/0ApiuScrnHqRim3xW1YKJ0kY44TkD8abHLnCou4eucUWFcsF5wP3RYH2pwjuTEJJJmJJwBkkikjkEQLTW8j46AHFXkeS4yotBDju5GTSC5Te3cxFmbLHpkEk/jT102dwCVHA7nFX4nIUxkkFeuPekJIkBMzEE4x6VSC5VNjImHeKMgdQDyaaS7sFSMA4xjHWr/ANpAfYRWZDBPcXcrxq8jF8IIxkYpXBtLcuRxm3Q7nVmJye1Ry7mn3pGWH8RHrWxbaBeXMaGaMRYOTvOCfwrXh0GwjA3pvYf3jx+VQ6iRjLEQRysaSO6/ZbZ3cc8dM+ladro2oXbA3EcdsinIGck11K7Il2oiqo/uqBUZmVe9ZubZzSxMmrFGLQ7NJA7kuw/Cr48uHIj+UHtmoHlJGBxUR3N1Oam9zncm9yxJclMHcCOmKikuJG6AAE1G8WYScdKnMYBzjtmkBFsZjliTTkjAPSpAhJwBSM8MK7pHwewHJoAdtA6U7G1MsQo9ScVT/tCaQYggCD++/Wq5hMr7ppWkb3PFaKm3uWoMvSXkUZxGvmt6jgfnUBubiTcHwinHC00KAABTgK0VNLcpQRCLeEPv2kv/AHicmn7Fz0FPcxxDfLIqKOpJrLutdhAMdom9u0h4FaFGkTjqAAOp7CqNxrEELFYwZSPQ4Gaxprq5ueJpCR2A4H5VEFI9KALM+pXtzwZAid1XjP41V2AdKdtNABz2piFpACTTsdeAKBmnYAGQMAc0oz3pQKX3oACOM0gpc8dKbmgB+cnp0pOtHajOOKACkpaBQMM0ZpOM8+lHNMAB+alNJ3ooAX69KMjGaDSUCFzSZo4o4oC4ZpM0e9BoBimkoxRmhgGaKKMUAFFHakoAWgdc4ooA5oAOAM0ZyKPr3pKAFoHSigde1AByTSHr9aXtSH7px17UAB47UA5owDgnrijpigBe+DQAS3DYApM0ZA/GgB1Lnk0meKO1AIXnr2opM8cUUDFoHXmk5B49KXqKBB3o70UmaAHDPeiko9qAA+1GemetGKD1oAKDnOTSUvNABmjsOlAwT1oPHegAxRjmnAgKQVySMA+lNzQAAYoJ5HvQOtGfXmgBTnFA6Z7UhOeKBxQA7GQT2pOKM0negELSe9Lnk0meKADNFNwQc9qcDmgAxzRwBijvRxzSGIu5WBGODnBGQa6fS/E0dtYLBPE4ZejJwCK5kGg9axq0VVjysadjpr3xPHJEv2eNy+c7mIyp9qLLxrdxti7iWSL17/nXMY5orKODpxViudnpVtrum38Iy4ic/wAMn9D3p02m211kgDPZlx/SvM8e54q/ZaxeWTlo5mHouePyrnng5LWOpSqLqdr/AGfd23NvLvA6KTz+dEWpSw/JdRuuTwWXIFZFn4zyxF3b/wDA0IH6V0MGqaffRgCVCD2fg1zSpzhpJFqSexGTa6hHsfkdtrYIP4Vh6h4QaRQ1rcFyP4ZOD+ddDLo1vI26PMTDoV61Wa31G0YlGFwvcHrSjOUHeLB2e5wl9o17YkmWFgmPvYyPzqkuQORXpC6uoYR3UBiJ7EZH51C+laHrB3CLDrkEx/Lg1108a1pMzcOx54cE8gH60xoIX6xKa6+58ESAtJa3Ydeytiufu9J1CyBa4tXVB/H1H512QrwlsyHFooC1gRg6xqGB6nmpS2fvDI9hzTQc0prS5I8xQkZE+D/tx4qMRjsox34FLjNKCR0ouBE9vG55QD6CkNnARjYM+vepjTNxFFgDyQVxgYx0Ipotoim1o1OOhxUm4mgEk80wuV2s4P8AnmB9KcLWIRlQMA1PxnmlC5HSi1wKi2kSgdTjvnGaU2kZOefzq1tFIRigCERAdCfxqWG2imJEkip6Z70UhGaACWyhhkADJIpz8ynJH1pFgiO7cQuBkZHFOAxS0ANhzDMksLKsiHIJGavHUblwRJDbyZ6/IKpEJ3FNWQKcg4ApFEpfc5OwLk9B0FALgkqzKSMHBxmmG4Pm+WAD3HHJqbBHUYNFkFxpeZl2O7FfQmmFMkHGcVLgdxQQB0p2QiLAxwDSgetPz60nHegQ4EUm4ZxmjAB4pcA/UUALkDvmkBwcjpQVPXikGR9KYC5H40ucdCR703PNKCOaAEYMSMsePSlOCMMAR7ikBz25peD1pgAG0YAwKdncOeaMHHtTCBwTnj0oAdn1GKU4phIxwaAcdTxQA+ijNAI70gG5wcUo5pMg8Y59aM80wFJwaTPOaXANGMdaAEzk0tAFGDu60CYUUcg0E4GTTAKD14peMUlAgNHJFFFAB2pDS0hxQAZooxRjNAATiikxS0AFApBS9TQIKTPNLSUDQdKM0UdjQAGgE9KBQfUdaAFzikzRyRSUAOBFGRTcUUAPzQCKZyOtA5GaAKEWlX1wgaK1lYHp8uKtR+F9VkXPk+X7N1rrn1wocQRIF7cVC+t3TdAo+grk5qz2ijq9r5mNb+CbkIJLu8SNe4Ga2IPDehW/LymQ/wC03H5VUlup5c7pWIPbNRHJ6nNHs5y+KX3EOpd3NO4g0m2C/Z4oyc84FO/thowFijjGP9mssClC1apq1pambk7l6fVbmddrMAPYVSZ3Y5Lk0YJ7UAEHJqoxUVZIG7gpYcAkfjSYJp/Q0oFUIj2k0oXjNPIJ6cUgHagBAKdRRigYUZPajFGB3pALnHXilyCMYzSHGOKQnFMQoyowOlLkbaj3cUZOM0DHE+9GRURNHI60ASF8jFJk1GSc0EnHWgTJCc0mcHmmZI7ioy2W56UCLG5fWmFlBwDVdpVAPPSoPtAB5cfnTKReLimlge4qn9rU5G4geuOKglv44UBYOSTgAikCi3sjRyQOTmo2mCnBNRqSY98kgjHpnNZxvbGe4MUcskr+3ANBqqMnuaD3aKMjLH0AqOG78w/MuM9ieazn1FoZGjggXK/xNgmqE91ccuTsJ9O9SaKjHqdDJcRKSMgH61I2pWsEWGO5vQDk1iXMpjt4SBkMuRxmq6xySZd5MZ6DoaVy1Tiuhoy6u748m28sA5yzZJ/DFQT6pdTDAfy19EGM/WoPK7b2P40hEarsAcn1Iq0zQi82ZZhJ5rHHY8ipQkctwHGAfQdKQRMeAOKVICjkxx5bucZouA9lBbIPPsadGMEhsPkY5HSmxpMWykseR2yM06OQsT57ZwcYxii4FXg3BA6Ve2kW5B7jFU12vcsVGATwB2FXXjeRVG8IB1JPArToZvchUEIBjpT0dlOGBAPTIpkl1bRL5fn+Y/8AsLn9afNqk96I4zEEiQYBxgn61i9zZbDSZJZCxOSadIhQtHklh3PakjaMESEk47A8mnTTIQJAeScAHrQgZCwkzg4/E0H7SkBcqIxnBxzmpEnjDZlGR65oIjJZiMjjA7UxWILeKaNvMSbCkEHzD1/ClwHc75A7DqR0p4JHQjHtSgAnJwPoKQE8K2DgB5cSHp2rNmIS6UBRKBnIFW5JPurgDBzkd6YZxGOQTnuOtAEJwZVdoHAH8J71YmQEhjEWU9MipT5mFSQgk8g+gpsm9mJZlwDgZOB+FUMrI0YlUKRz6VOCobJNRwlJg7CIRFerMMZp0RjdsbST3I6USEShoSxG5ie20f1qSOzeeM5JTjI3PxUMnlhv3YwD+YpmxD1bP41IFWS3iZlDAHJxwelTpYLbureYRxkCp4o4ykgRdzbelIiy3UioIZGbtx0FO4m0iZLrAAkG4DoM4p7XxdSfLJLDAx0/Otiy8M3ckeLiNIh/tcH9K1LTw1Z2pIkkkmB7N0/IVnKaRhLEQWxyO67l3LBauz8fdHX61p2XhrUbrElwFgQjkE9PwrsI0ht12wRqinrgYoMxB61Dm2c8sTJ6LQyrPwpZW0gkkd5mHZjgflWvFFDbLthiSNe+0YNRm4PY5qMyue9S23uYSk5blppAFOTULzcZXJFRhcqDzn3pwFIQ0lmXkmkKZA46VKBgYpwjOM0gGKmRmlCc8dabLc21uv7yZVPpWc2oQzySEI4bpGcZGPUU0m9hpNmlMNkDs5woHJ9Krx3CuvmbwVPAzweKxlsryS4L3E5MbHON2T+VWBZQxyFxu9h2H0q40+5ah3Jbi4vJXZIpMRZ65AzTIVaIY27j65zQA+eAMVLGCW54+laxilsWkkLmY/wCgRzkgYAz7VFcX9vbH55Mn0Q5NZs+s3EqkQKY0PckE1Vgua08q2ke6eZV9gMmsqbW5DxbLkf3mH9KzGBdi0hLOepPWnADHAqgB2lncvNIZD2yeBQAB0pwFJihIA6dqX60uM0uKYhvPalAOKXFHHqAfegBACBxSg46UuSaTHFMYvIFHegnJpDSELQMA5xRnAoPJ6UAJkk8UpBx1yKOnTpRkdKYwpByKKBQAUuMijaCaKACjtRnmigBTSAjvRgEdcUh9aBWCjvRRQAtJRR15NAIKKPaigAFHejvQaACiiigAoFJiloACecUCkyKBQAtIKXtQfbigBO3PWilooAbg5Bzj2pSPWkxgkds0CgBaUU0HJpQe3pQAp6dKXtQDzSZwCO3WgBQOOaAABwOBSA5oHA55oGL3JpM80tLgd6QhKORQckDHrS4JANMLiAgEgg0tITkjNLQMKOMUdgKCBmgQlKKQ8AmjPAphcXjtSUvPaj60gDPAo7mijPFACf/AKqXODRSZoAU5JzRQKKACjNFAHNAIMjpQfQUDGTQeRjFACc5wT9KUHFHUg9xQRQAuaQ5Bzjg/pQBnjvRn1oBATQKD6E0vAHFIYvbIFJjIpBS5/GiwgIoxTsA0YGM4oAj25oLHGNx49+aeQDTcAnFKwzSsNe1CxKhZmMY6q3ORXQWXjGByVubZogP4l5H5GuOxSbQRjmsJ4aEtdilNo9UiurK/T5GjmX0PX8qpy6JbvIZIS0L9iDwPwrzlZZo3V0lcbemDityx8WXtsAJm89R2br+YriqYScdVqaKaZ0TRatZAmNxPGOxxmnR63CTsu4WT1DLkfnUNl4usLrIlDQEdN3IP41sf6NfQZ/dyK38QOa5pKUXZlJ3M6bTNF1QFmjUN3aP5T+lYeoeDGQB7KfevQq/X866CXQo87reUwyHpjpVaRNWsiCQLhB1IHNXCvOGzG4pnG3Wj39kxEtrJsH8YGRVEMDXo8Ovws4juEMZPZhkfnRJpei6ip/dxBm6Ohwa6YY3+ZGbp9jzg1H3rs7nwOACbW7LA9mH9a5680HUbB8S27Mo6sg3CuuniIT0TIcWijxjFAAwfWghlOGUr9aUjjA4461stSRAOcEUdDgGlG8cEg0Ack0CF7UAZ5NHejORQMUjrSAelByRQOgwaAEwSaUDmlBFLQFyNgQehIpCQ4wMflUwIzyMj0ppAJ4GKATKrwlW3jnHYdalS5eT7yMCO571MQDQIlI6UAKAWAYHINFRNG6NlGyRyAakIcxhyMZ6r/hTAXGeMUFADTADj75A9MU4HHfNAhANvTpSjPUUoYDtzTTgnIyPakAZpwJ70mKAKYCYOeRxmlYDPFBBApNxBouA3oPanDJ6UHjr3oBA6GgBNxB4P4U4MSDnrSZBHHWkJweKAHUo6dKj3U/ccetAC9RQQD36UBgegxRQAYpOgpc4pKYADS5pAaUHJ4oAcCQMikHSjoaTOKQC0oPXIpuc0AGmAuAO9Lx60zvThgjINDFYTvS4OM4ozzRuI6UDAgdjQBQetGKBCZ5pCSBwKWloGNoxS/hSUALRnFGKPrTJDNJS0YoAQ5pDnIxS9Oc8Uv0oASkNBox3oAXNGRSHJo4oAWk70ZPrRQAUUUUAaQAJ5OKcQAcDJpxjAOaUcHgVmMbjAoAzTsEmjaRzRYdhccdKTBHUU7Py0hINAACCKQnHbNIMZ5pSaQwBOelKSfTFM3YPXmkLnNMRJkCguKYWyKaTQA/eBQXBqLcO5oLoB1P5UASeZzQXA71Xe6hU9z9BURvY88KaALe/NBYeoqi1+AeAB7GoWvXJ4AA9qYzS3e9G8g4rL+1v60n2iYn5Q2fYUCNQnNG7ArPQ3j5O04HtVgRP5Qch3b0HFK40m9iVpPSoyZieFNNSVLf5pGUY65OcVbbxLYRoM/Mfbmk79C50prYr7JjnOKlW3Z0zk8+1ZzeJUZy1vbZJPBLY/nVW51e9lGI5Fz/FtHQ/WmaKiuprfYwieZcCMHvlqhM1ggwCHPYAHNYmbuQZmLSZ7MeKkS4uIcZwiD0FM1UIrZF3UL2UxqIU2euRWS7TXAAkOT61amljnTO8sfQ1C5WFcnIPpUWNCMWUI4mmck9l7017C2OPKJjx3zzUiTKRkjmmO4BHPWiwD7az8liUkDHsWPAqe4sLi5TBaEH1BqpvUjqR9KmS5EMWdjk9AetFhEW+7WMQsQVB2jjjFSxw9A8iIB3Jpql2GSMZ5xRvI6U0hkksaAAC4Lk9h0p9s8cWd8h5x15qnJNKhxsz9BU0LzMMhQvsRzTsA8OeSAQMnBI60QXH2VmOzzA3UE1FPdXBBQFT/wABNShRNYb48+cemTQ0FxXm2ci0SMnoQvOaYLe4ljad0kRQMkHAFOtNSliQQy2Ekko6njGPrT5p9SuUaMny4TjhsZosF7lezRzIDjrVi6GH2+wz9aoW82zzEa4HmKcDBxxTbbUY/wC0GEhaQKMgds1VxWLqJsXDgeo2jqKaRg55x2BNWhqFrOpIQgnoAMmq73EaneoBx/CR/SkMjMiRncYw2OxqyZN0eYrdYn9XGT+VRDU5y/l+UkansExmpFuVDZeMn2FAXIw0zrljD+AxSpbrJnzZgo9jU9w8UMi/u+ozxVaRpgpCxjcSSMntQFxqCNDsXoKkwp+6c0iWZVN7FyxAJA6CniMDAVwG9DSaGmV3zgEkEHqPSnwyn5hEU3YzlhnFPjmXzMSIC3fg02Zm3nYjKDxgLgUriuRxXE3mAzKBx1PFXIZbcZErJvzkE9h6VWjjnllCrHvbsAMmtG38MavdfMYEhHYucH8qHJLcmVSMd2V5ZjPGyk5BPGBgAelVBG+SqAls4K11EXguUqftOqMPRUSty00bTbFMJCHfHLucms5TXQxliYJaanF2WlXF7ysEjJnkgf1rdtPCFsr+ZcFyvZd3866QyKvA4X0pjTL61Lm2c08ROW2hTg0LSbU5W3BJ/vEkVoB0TIjCKPQKKqtNnpTcs3U1Bk3fcsvLt6n9KhN1kjCkgjrTAgxg5NOCgDFFyRpLseelAQ5wc04D5we1PxgmgBm0CnKMHOKkVCeoqCS+tIThpSzf3VGTQk2FicDPQUpwg3OQq+pNZ0uoXDjFtGEHZiM1CY5JCWnZpGPUk8Vaptmigy62pwjiFWlb1C8CqdxLfXXdo09F4/Onj5eAoA9hS7z6CtFBIpRSK62uOq59yMmpxEQOQBSjcabLIkI3SypGD/eNUUAQEjOPxpxRQMlwB9aypdbQZEMZf3YYH5VnTTTXb7ppCR2XoAKANi51i1gJWEGSQdccAfjWRNqF3dABnCIeqoMCofLA6ACnBSKoBqoAcgcnv3p4U9aMYNLmiwhuCepFKBTscelKBxnFMBmOacBnqcUE0A55FMYCgUUUAGBnnIoooxgZoAUH1ooBopCDNGaKMUABGDg04dKbzRk0xgetGOc0UvbrQAd/akPWigYyTQAueOKOTR0ozQAlHeg0dqADvQaTvSnrQIKM0dqKACkzRzSUALnmg9aSjNAC0uaSimAooPtTec0GkA6ik96XqKAGgADilopDweooAUn2o70A4zRQAZGaQMCeDmg+1B9aADHPWjvQDzQKADoaUfepFULgA8UA/NjoPWgBWBIwDinISuCMZ96Q0CgAopPrS5oAXp0oGMUUnbHegGOzSCkGM0p9KQBxS02lFMBf8KDSdaB+vrQAd6WkOCSR37UuMHntRYANGeMUh6UDpigAo7UdqKAHDrRTeRQOhzSABS96KB6GmAmQeRS5pPpSkdj1oEA5z7UUHGKBzxQMATyDQB2oozQAUmKWjvQAgOOMUp56UhGetHpQMUGjoetIcHt+NGOM++KAFB55px6Zz+FR9s04H0NAD8Ejg4pADTckHuaUk5NIQc5oxz1pMn0p386AEPORTcflTxjvR/KgCPaD2qzBe3Ns4aKaRSPRsVAOCDTjgnOAPpUSipbod7HSWnjC5ijCzhXI7Y5/OugsfE2m3oIMnkv/AHZO/wBDXnWBRjnIrlng4PbQ0VRnqU9raXseZI0YHoRWdJoEaZe0keNuwzXFWuqXtowMdxIB/dycflW7Z+MJlGLmJXC/xKMH9K5J4WpHzNFNM0/M1ayIG0Sj9asw+IIzJsuIWh9z0pbHxFp1/hUlVH/uyfKfwq+9rb3UYJQEdipBFYuLjoxrUpXOnaRqrb3RCT/EvBrEufBO3LWtyXXsrAZ/OtZ/D4DFraVkPYg9KiL6rYct+9Ud881pCrOHwsTinucdfaPfWDAywPsJ4YDOap7eOhH4V6FFr8fEdzGUb0K0+Wz0fVEyY1J/vJkGumOMa+JEOn2POggNGzFdbc+DAzFrS8GP7rjj8xWNe6FqVlktbs8YHLryP0rqjiKcupDi0ZWO9Jt5607OeDkHuKCAOhBzWxIgFHQ0uTigcjNAC00UMcVZezxD5sU0cqjrg4I/CmJFcnvRn8qTnacUDHuKQChsnFV5JHt5ixO+EnIHpU+SBkAfjSMQ6kPGCDQMb9ohkXcGIHowpwZXHykE1EFMYwoDRk8KeoqRQgGVGM9eKAH4I9KKTtRg4piA0dep7UnWgYHc0gFxjuaKMjHBBpM0wHAgZyAfTNOdoXUHyikg4yDwaYBS7QetADMAHIFKSCKU4HFAxQA3rSikzTgcDGAc0AJgZ9qM46UEgEgqc+oNNzg80wHg+tB9R+dAwetBznGeKQBzjijd7GgcClBxQAxZlJwQQfQinHk0pIJ+6M0nU54oAUAZ5oJ54zRQME80wCginYpjMUIBBORnIoAcKBTRKh6kqfcU4EMeGBH1pAFJyPpQcg8jFHIpgLk0UmfY0EgUAFH0oBAPPSnHHbpQAnPekzmnUhAzmglgTmignnoaQ0CCloHSgAd6BhikFOYccEGm/WmAUYo4JpfpSASkxS/Sj6gUxCAGkp9N70AbJik9KPLlH8IqYI56TfpQVmGNko981BoQ7Zf7n6UwiYfwmrO+6UAAIaBLcd4lP40AVcSDqCKCSoyQTVgtLnJh/Wk34+/EcfSkSVfMB6daNxNT+ZCesZ/AUmYD/wAs2oGQZA5JFNDgnrU5S37q35UhitgM4P0oGVy4Bxmml/Q1Z8i3PKIB7mmtBbEjfKQfamBWLGozk9TV77NaYz5xPsDUbRQKcI5x7nFAlFvYpGLceoH1oFox7Vd8yxiI8x1B9N1Zz65Zq/cgegp2NI0pPckaxH/PVST2p6aSuzAZs/596z5PEgGPs1vlvUjNVZtYvZkLNMRnsowBQzdUoo6GGzt7Ygsw3DsTSzaxZ2ZyXBI7AZNcjHI9wTl3Y9zmgIDJtCZ9zU6Mr2cex0Vx4tEq+XBb5PYlcVi3eu3cw2GQxseyjAoUm3kOyEMvuuKa6JI24oB9KFZFJJaIqOs7xglyzMcdalisLleGYAdzmp+EUDHAoQyOhkQlQO5FFxkPkBEKZJB6+9OiiIXMQ+U9KRw7vlpWOevarVujsAiMAB3PWi4FYpISM7xg96kaMFMl3JPbPFOkIGQZCcdyajBkBAEbOD3Ap2GNEM0YABBye5q19kmaIs6Ek96e0bKm51IA7npUaXavGYgXBPcHJoaAhGnSH+D9aDYyIC0iEAdyaR2YKQjyhj0JOaIdzgiUyFAMkZxSsA3MIH3hn60+EJJIFAy5OATwKkEmlg4cPn0JJqYXViB8juvuqc/nQBNNAsJCuQx74PSkW0gkAYoAD0JNM2GRTLHGSfWU4J/Coo7eefcJ5VCqMiNRxmgDQQR267gy7R1zzVQXUc9yzKeAM5qCJbcqRJCTg4GehoMNvM2QfLjBBIB6j0pgTS3CDC53k/wqOajBWRQcNGPrzU1vqMFsPKW1VPQ9STU01wkgwwUn6UCbMuQSFiBKSOMYPSo2e4BAhkD4znccjn/Jq/HC9wpIZFwcYIxRbq8F1kIrEA5HSgLmcNPE6l7iZIwOoVefzqVNLt1+WAnzOzEZNa8mnPIWY4iB5PORmkht1gIIuI89C3t9KBmcLd/M2+c/1KgVPFsjuADESxON2On0rYkktHA3Sg4ORgU3faCTYcAngAnBJpXArPcefiCGE7v7x5AqJrKc4w6fhxVie6W3cpFChI6YYVTju7goQHj+Y/dxzRcCeS1hkAJJLAY3Z4zVOSOV5sCRA3QZPb8q04dK1K6iXbEApOSTxWnaeGSjbp7naewQD+dS5pGMq0F1OcD+Qdkt1yOp7D8atWcst9Pst7LzP+mpHFdbHpOnx4Jt0kYd3GatoEQYRERfRRUOr2MZYrTRGBbaPfPL+/igjX17/hWlF4fsYnLyF5M9ieKvFjUD3IBIz0rNybOeVWct2Txx2tt/qYI0PqFGfzpxlzyck+tUGuGOcD8qasjuOScUjMtSTAEkn8qiaYt0z9TUJU56kk07bwPrQA8sSMU0L82c08pzShaAEABIwKcAe1OC+nJ9qVvk5kZUHqTSGBHFJUMl5D5bNE4kdTgL0Bqj5uoz5O/yFPQIOfzpqLY1Fs03nggYedMi56D1qF9ThJxAm8/3jWaNMG/e8rSN/tVOLRR2rVU0XyIeZ5pTlpAFPbpimqYojgMmfXGaX7LH704W6g4AH41aSWxaVga6Uf8ALQflTTchjgNn6AU9/s9uA0skaj3rPn1u3QbbeIyk/wAXQD/GgC+CT1B/KoJ9Qtbfh5CW7BRk1iTXlzccSSEKeqrwKr7QDwBVAaFxrF1McW4EKe/JqgVMhLyMWY9STTxwOlJn2p2EIBjtS4p1JQAmTil5I5pKOvFUMXNLzkEEj6UmOcUuMcGkIOT3oA5zTu9GMUAMB9QadmlAUnBOPSlQouSwJPYUxiYoxxxS9TnHU9qOpxSEJRnNB4JpoJI6EfWgBcUoGO9FFAABS45wKQU4ZNACUdsUd6TNMYHPSko+tLQAppKWkFAmL1NJwBSUdRQMO+KKKB1Az1oAWg9eKMDPWkzQIU9KTpS0UANzzRQRzQSAKAFpKPTjrRQA7NN7UZpaAE70ZozRTELnjFHakzRSGLnmk7Zo6nignp9aACijFKBnigBO55oFGOM0dqAAAAnnr2o5HUUhPIwM5peCRg0ABOKBzQwIBxwfWjNACgjHfigEHkAj60gHUg5yOlKDgEetAwoHXFBpQM+1ABnilAyaSlxxmgAIxwBSUE4OKUUAJ296Udj7UhFA6Z44oAWkpQQRkdKUDIOOtAgz6jtSUAnGCc0nOaBCk0i+poHpQKBimgZpDRQA7HfjFJ14FITzSj260AA4NBo9qKADvRSZpTQAdqTk8GndqTkc0gAmgelHaimAvajmk7UooAXORzSUZzmkzQCE7ilOQcHpQcYFHagBcZOO1NA5waUkg5FLkfjSATj8aXNJjJz2ozg0AKTQGOaTJNAAzg0APznvzRkA4zzTPlyRnpSHigB5PHNJk8DtTQacMd6AFGSccAetIcj0pCcc4OO+KQ4PA4NADwTjmkIHWkJyo9cUgJx70BcVWCsCMAir0Gs6hatmC5Zf9k8iqPHcCgkdgKiUIy3Q1Jo6+y8agKUu4MY/iU4zXRWGrWmppm2lV/VT1ry3vyBUizeWQUJBHpXLPBxesWWqj6nqc1nDIMGMA/Ssm68P4UtaSeVIOnpXKWniXUbRx86yJ/dcZFdDZeMrSYEXERhI6nqP5VySoVIa2NVJMfAdXsgFk2z/AE6mr0Guqp23Mbp+B4q9a3ttfRebbSrKnt1pJ7KG4yHXIPY1g27jK8ltpepHDQRuOzKAD+lY994Ijfa1hN5eOqyEn9at3GgGMiSzkkjYdgeKYtzq1koDgSAd881cKs4O8WJxTObvPDWp2KhmgLr6x8isogqdrAgjsRXokPiS3JAmR4yezVPNDpd+odo4W9GwARXVTxkl8SuQ6fY80xkUgjI612154KhcM9pcFZCeFYkisOfwxqsHzFVdfVDkV1RxNNq97EODMbae1IASMkVIwkjkKSKVYdQRilHPSt07kkZXjk9DxQAeuaeQTwabtwcDpQAEcZoxSkHBOOKUngYpiIwOcUHPQHilA9RTgOKQxu3FNHWn9Cc0uFIyRTJI/oAKQZzg1Jt9BSFSDmgY3ODg9aN9KQSc0jAigBeoyaBxx2puCKM0AO4NHekBHrSjpkUAJ9aCMjpmgZxjPFA4HNMBQPWkIyeM0p9KOQaQDCSBzTgcjINLnHUCk3D0pgISQaMmlyO9JwaAFAJ54oz780YFGD2oAcpOKCTSAEDpQRkdcUALuJ6gflTDFETkoM+o4pRkdelLmgBnkqPuySA+5qTJ70hNJkd6AH5pO9ISQM/zoDY60ALRjmkLj05pM5oAdnigHNIDSj6UAKDzSZXOCcH0oPXNJx3A+tArC8dqAMnrQQNvBxRQFhT6UmKXtS44yKBWGYxS5oxilpgIc0maccZwaTAoACRmlOKTFJjJoEW/LkHc/nRiRf42FOLk96Zub2qChRJMDkOx/Gk+03I6OaAWH0qRYZH6RnB70AR/arr+/R9pue7E1bSwkI3NwvenmK3j4yXP0oGk3sUxd3HoKlSe5PRRU4yB8sANRy3UcSZlkji9eeaZfspDgs5+80Y/CpRGG7j86yp9ctohiJXmb36VRm1m9kIMIjhHpgHNI1VDuzpJ4/IUF2UA+9ZsuqWtqxLtGx9CMmuWubi9mffcTvtHYcCm25QyZkBKep70W7lqlFLXU2bjxGHkzDFGvuTn9Kzbi5ub4kmUKPQcCpSUMgFvbof9oD+lSmFxzMUXPoefyoNUktEUYbQGQGaTMfrnNWZrSAx5iOB6nvTriRTgRMIwBjp1qBIzje8m8+nalcLEMLPAT+63r6UCZXmBeE+6jgCrAmIGBwKY8ik8qT9KLjsWUmhYbSAuR1AqlLO0RJjKcHgk5zU21D2H50ixRBwdgJ7E9qGJoEu5Z0+fOenTApriQEhY2I9cVKF29APxGaSS6mA2gihBYrj7QT/qyPrUitJwG4PpUjTgwlP4j1NT2Ft9rDAvsjBwSOpFBdinJvLEoOO1WIkBhyxII64qe9ks7TasMYLA5JJ4/CqMl3LeARqqxoDngdfrQS0OMSAFhyR3POKkhe4UA5/djtQ0hKeWAAvoBSCRsYUAn3pisSNcNJgPGxT0J61FtMTGVYSo7EmnRiQks7AA9M8YqWS1lmRWab92BgDNAyIXLlgcAY9Kd5pYkk4B9qFjiQYMgPuKWSMx7XRhgYxigBz3CDBCAYGORTdhl+cbR+lNnknuk+YbgOwGOtNS1efAkLIoHAx1oAVgXOC5/OkjItYztLHJye9TtbW0aZEjKw981JbS25tTEYgSRgs3Un1qQKqyeYu4Bz684q1b28UylBbsCOhzkVMtrIIgEIAPrVmMzwxBCAB3YDApoCgdLIYtMSB7dqnTTrXjDv8AiadJPvkMSTAt2z0P0p6nbCCQSw64FMTLCWgXAjXIqZtPtm5kbEns2MVlSXQklEbTMi+gzk086Nf3k4kgWRo+xIxmobS3E5Rjuxb3zY8qheQep5qsLiNLfaY1Eg710ulaFNbpm8YN6Jjp+NaZ0rTzybSM1PtUjGeKinY4q1iurlAUhRn7Ef4Vojwzqc8ymcRKvck8j8K61I44lCxRqiDsopWcL1NS6jOeWKm3oZEXh23UgyoHI96tx2dlat8lgFx/EoyasG4Ud6YbrP3QR9azuzDnfck88P0yPY9aaWUdTVG9lcXcew43DmlAY9TmkInNwg4HNQNOxYAZGaBH7U4IPSgAAJHJNJsBJyDUoAHNKTkYFMCIx5GRSxqAMU7nGKWKM7SWOAT1NIYhQHpUgBwOKY91awD533N2C9arSanO3y28QVPVjVKLewJNl8ptXLsF+tQtdwR8B959BWe6rMQ08rM3pninq8MQwiAVoqfctQHS3d9M4MAECepGTTDbGRg80zSketO84EcCkLt2GatRSLUUiRI406IM08tk5qJQ7Dpio5riGDPnTqCOwOT+VOwyxuOadgnuB9ax5tZB4gjz/tN/hWbLcTztmSViPQHApgb1xqVrb8eYXb0AzWfcaxcTKRAvlKe561mgAdBS7jTsANuc5kcue+aAABwBTgKOBRYQ0HPY8d6djmlLYOQDmmg+v5gUwFoySuSAKbmlGCM9qYxRzyKTBpdvPFOGcYxSEIB60HGaXODRwB0zQCFFJSU4dM0AITilzkYPekxnrQSM0AA+8OlB6UA+1GTimMAxwcccYozSdaUA5oAWjGT6CjGTyRRikITjtTsYPIpAPSjdzTGHSl5A5FIe3pigZPAoAOpopKKAF60oyGoHpSd6AEPrR3oPX2ozQAE0UD34ox+VAAcmgUtJQIMUd6McZz+FFACmkpD1paAA0mM0E0oHH0piAE4AJ4FHFJ3oxSGB60Ue1BFMQfiKO+aMUUhgRxzQenWig8DmgA69aQYJOOxxS9qMYoAUnJzjmkBPegE59qMnPSgBR6UAEjI9M0mQOT0ozzxQAZozzSc5pegNABkk0UgzjNBIHLEAds0rgAOOKdmmgggkDPHGO9CZwcqV7jPemMeORzx9aMDA9aTqMk0nc0hDgc0ZGQMjI6UYwAexoIBOSOaYCHrSjNGKT8cUDFo70HOaKBBjigZ60ZoyfwoAMUpHGe1J79vSjOQCBg0ABABGCDxSg5pOpzR1GaADPOKBTtwCEEA5745FN7UAGcHgUUUtACd6O9HvQKADkUZGMe9BJJpAMmgBRS8Ug6UDkUAKaSg9aKAFozSHpQMH60AKMEZxSjFIOBijOTQAHnNFHeg+lAAT2pBjPNKQc0Y9uaQC+v1pMUhFLQIToKQk/jTgcEE84pDjdkUxiEjqw6jtSnB6UhOcUAcUDF6qMUq+/wCdNPtS845oAcpxnnrUZO0g5xk4pxzTScjBFIQEYPWlDkZGeDTc89KXaCuaYC5J57UmcUEEHHagDPFIAJyMg80AjFKEz3xRtwelACA5NKQO9JjHalycHPIpDHwXE1q4eGRkI6YrZt/GOqwyL5hWVB1DDk/jWEeaKynRhLVoak0dzaeNYJpCs1u6+hz/APWro4Jba8hEsTLIjdwa8iq1Z6jd2TZgkdfo2K5KmD6wLVTuel3Gl2dwCGGCehxyKy5/DgLZt5jGO2Dmsiz8ZXagC6gSRR/Eo+auitde066QZnETnqsnGPxrlnSnDdGikmZgute0zKmMTxjowGf6VND4sKAC6tQreucVv5X+8D7qciqV1pNjebjKmc1l6jGxzaTrKHzoIt5xw+AfwOaz73wdbSgtaTtG2OhIIps/hmIjdDKyt2B5piza1pIyxFxCPXkj6cVcKsovRiaTMObwvq1vkmISKO6HIrNdWgYrIpBHtXaJ4sUf622cN7GrLGy12AB5E+b+EjDf54rqjjJp+8Q6a6Hn/mr60uQa7efwdYyR7YWaJz3PNYF54T1K1b92onT1Xr+VdUMVCW+hDg0ZAIAwaaWGeKdLBJD8kqFWz0xTAARwa3Uk1dE2HYBx2oIxwcfhTCg9fzo2AHiqEPyCDjtSZpuMHNJ8hPJINIBSaAwB5pAF7kHNIQO1MAJU5GMUmQODQME4JNKRg+1ILjWwBmgcjIOKGwRgjNAAI4Iz6UAAYjIyDTs5HWmkHNNIPqRTAkHBpSexpgyR1z9aP3nYj8aAF5BwTkUhQ5yDx6UAt3AP0o3jGMEUwAZ70h5OM808FScE4PvS7FIyRzQAwE5x3p2cAZ70FCRwab5ZxnOfagCQfWg9aQelBzmgBRnuMijoelISc+1Jk0AOIGaQgZpOc04UANI9OlKQMAYpDkd6XNACbQTSFR708euKOvOKAGjOaXJHelIB6UmPWgBCSOooH6UuKXpketADcehNOGfWmindqADnvRn0pMnuc0c5oAXPrR16Uc/hRzQAYJ5o6UtNoELziijmigVjSFo/fipRaj0rHbXItxEUby+hHQ1FJq984xHD5Y96k6PYeZv+UIjyVHuailvoIOZJkx7cmuUubu7k4mumUH+ENxUAYHq+aC40Yrc6K78QGWLyrKLn+J24FZxu75h8+oKD6KOBVFfJPUMT7UEleUjC+5FCVjWMVHREsv265JA1RgB6jFQ/2VEzbp78s/fBokgLxg7jk9earjEcoBoKsXWjTZt3A+hFJEm3HtVZ92cRIS3bFTCNxHmQkH0BpXCxKxeT7iDA5JNRHD43ADHcU0THGyOI4HcmgSb22hTmi4y3bh0BMcnlg9T3qORMSEkkk9zSK8qdhj3qN3d2zigVwBVOSAfqKeLglcADHsKheMuQSCAO9TosZQK0gUA5wBzQMasEsibggxnpnFKY1w/AGOuD1qB0ckbZCDnAGeMVMyOAFJycZJHNSMiihUIXdWBHcnpQ7jGImJY9KeGBAjZSQeuKRtkcg8uNiR3oAaI5FjJlEhPrnrT40jkjDFHHsTTllZpgTk/QVPMkJOxS/mdwTiqEVkmiU7WhBA6EU2ZyRvjTbH9KspCmMEcigYD7AAT7igaZmSzSz5iiXCHqcdamtYZoUcyLwBwe1XZoZGUokeD/AHh1FNBmRNjuT6g0AV43ABBJPNDMxk2KDk9/SrSxKRkOo9cjpSrCWJ2OCR1J4BpBcbbQBn+aTB96l3YJUHikXT7ifksAPQCp00q4B+QRgepOSaaEU2gyCcsPoKYEkRgka/N1GTV66sryOMCSQYPZByaqpCYRuJO4evamAeTqAGfkQdzkcUk0ZjAL3JZiMnA6Uk7NcALNMdoOcAdaXyP3JEKb8deMmgG0tyxb6Y8yiWRl8ruQeT9KlcQW+Db24kCnkseSfSljs9XmiWG3tpBGBglsgVo2fg9+HurkLx91DUuSRnKrCK3MmTUZJtu9NgU5yD/9agrdagpSESy4/hUZrr4PD2mwAERtIw7u2a0oo44ECxRoi+igCodRdDCWJVvdOKt/DupmPBtxCD/E3JH4VsWPh4JEVuZS/PQcZrfMmOpAqNpVHcVDqNmEsROSsV49LsYVxHbIvv1Jq3uOO1VnulXvUD3JPRiPpUttmTk3uy60oHcVXlvURsZBPoKrEsR1pnl7nBPWkIsm6LDABFQEyM2SSalKhQSOoFRxkEZPU0gHEHYTUiLwKQAmpAMUAV7hWF7CcZAUip0AxmopmzcxknsRipQrEdMD3oAXgUmcnike4tIQBLON3cDk1VOpZkxbQkr/AM9G6flTSb2Gk2XCjH0H1NQzXdrbqS8qkjqByapzma4fMsjMvpnFJHAiZ2x9e55NaKmWoFibVd67LKJlk4yzjAFVitxIxM05bPYcCptrZxgCnhB6itIxSLUUiAQAAYUA+tOERHpU4QHuPzpxCRjdI4VR1JNUMr+WSacIAeTVeXWLGE4RjMfRen51mXGsXMylYx5QPpgn86YG43kwrmR1UepNUpdXtoziFTKfbgViEmTlySfUkmgBR0FAFufUruYFQTGh6hev51UCepJ9ycmnrzQeKBDflxS8YzSHGaCRxg07AO425ptABxn1o4A96YXFyBRkHgkk+gpAc84pcAEmgYY9KVQSMdqVh8pxQHyBkkHHekIQqcHH/wCqlUEDkk/WgE9KXtQAUUuKSgBaSl7UnWgA70uRSYoxQAZBo4GM0vtSAhjxQAvak+nWl4xgdaBwORzQAlL0OaTOMcd6M5pjFpKCcfnQOelACk88UhHP4UcenNAHNACk56DApAMAkE0d6XrQAUmSKXGeM4oxge9AB9aCaKB0NACUY5ooPTg4oAXoSKKTJoxzQIKUjuOhpOgFGaADOOgo60fUUUWAQ9aKBQBQAHOMn6UUZI4oHNABRQeo9qKAA0o60ho7delACnpxSUClFACZJ9PakIz1GaXuKOpoAOetGaDgA4FA5xQAZGOlAo7ZoBoAM0de9BFJjPWgBDzlQRmkQkL8wwQSKXCkk45zjNIeMg4xQApJBJBGPShlRgCwBzwKaFAJAHU0uwZBBPBzjPFAIeMAAAYA7UE5IGDz39KBjFGMdCT9aAAsFABOKXOO3WgDNBODQMUnBxQDzTSecn8acCCAc8GgVg57Uh9KAR3oIwc9qBgTR2zRx2pe1ACYpdpIoBwCe9GT60CuIH5xgkmnHGBxyD1pKM0CsBPA9qXvQaO1AgDAAggHI70h3YBAzk4FB57ClBx06+tIYhBU4IwRwRRmggnk85PJpTgdBgUxiUZoo4yPegBCcnNKKOxoAz0oAUUEccdKSg0gAk5yaM0UEUwDtQOtHOPYUCgBe4PakoHpRmgBQeRS55PFNzRmgAJ5pc0ZyMUgoACaPfNFAHOD0xmgAJoo7UAkc0AKSCeufekBB4pD7UmCTxxigEPwB3pMjsOaAQaMEGgBeMH1FGAQD69qQc8d6d0FABjjGOKQYyQM0vaikA360uMdKKMHFMBDkDijcRjIpcZGM0hXIpABOTikC4/iyPQ0hGKkA+T1zQAAZ44x6ijZycUAAcdBS554oAYQBRyRwKeRk5oBwMZ5oAYNwGDzSHnrzT+tBGTiiwF+x1q7sVASVyB0UnI/Kt+y8YxyNtvYfL/20HH5VyIGKGUGuaeGpy1sWptHqUF5b3cYe3mST2U81OADwwyO4ryeMvFIrxsVYdwcVt2Xim+tE2TL9oX0Y8/nXJPByXw6lqoup2Fxo9pckl4xk+gxWNdaAYmJhZuOg9Ks2HiLT7t0DM0Eh/hboPxraRxKMpIrj1BzXK4uLszRO5yceoavpkoQRu8A/v8AIrWt/EkLcTxMhHUgZrXEKkciqlzottPyU2N6ihu4iciw1CIF1ilB7sozWRP4QsX+a3PlnsOoqvPoNxGTJbSb8dBnB/nRDrF/p5CzxbwOu4HP86FOUdmFkzIvfC+oW5by4/OT261kSxPbvslVkb0Iwa9DtPENlcNiQtCf9rGPzq/JbWWoJmSKKVf73U/nXXDGSW6IdNHlWVpAAOoruLrwVZu+62leInseRXN6j4b1Owy3kmaMdWjwQK6oYqnJ2IcGjMMcezcjjPcE81HmiRHiP71GQ/7XFJjNbpp6okUE5yAT9BSkg9AQR1yKYJZIH3xtg/TrT5Lh5gC4BI7gc0xCFgByQKQEYpjsMfMCRQrDHB4pgPJpCeaTzEPGee/tQOgI6UAKCKXIFIeBSZzRYB4z6cUuajBxnBpQcnrigBwGT707cQME0wnBHegkE8UASA5oIY9DUBQk5DkH60YmX/loDTAm/CkzUeZiM4GfrT1yeCQfcUAOJ/KjjHAxTcnoRilBxnjNADiABnIpAaQ80dqAHde9GARgmm96Xr0oAXBHBoppJoBNADue/FNpSSetJ3oAcCM4pDnuKQdaWgA/hoGaOn0oBoAWik69KKACgUDGcHpQcA8UALSUZJ7UnJNBIoOKM5zxikAxwBS/zoAwj50IJjlGfUCp7eG8uFBN2q57YqMAkgAZzUrRui5AwBUHpF1LOKNTvCO/qTilAs1AWRVOOwGTWcrFuA5H0pskpjbBkY/jQOxovcxQn91bgehIqnKRI24jk1EWJHDE/WozIR1JFArFsJmIksMdh3quDAX2vjd71GJ33YUc+pFSOI8KSpLnk4oGWYZkhRgEySMAYqMQu78DGaYCAQTVqe9a4AjRRGuMHHX86YDRp13jIdAPUmnR28du4aa7hGOozVSRY84kdwvfk0lrbadPcBGeTB7nIFQIuz+TdkNDMCFGDjvSQRGEiSTaF7k1Z/sm2iUm3clj6GqIgWNiJJNzd+eKoBs0webA5Q0FYwwMaEt2xTo7AECQyNn9KljgmOfLjLEdx2oGR+XIByIwfTrRkhNoOCaVxJE2JIXz7DIqSCKO5kClHGOSewFADVs9qb5JsZ6ADk0oAcACM4Hc960hawKwBLEHqSOlJJa2eceYfzoC5iySEZCfKc4z1qGS3crvcyMR3z0rfS1tA2Bk59TVpNP6gINp7Z4oA5q1tBM3Nww9ADnNWTZ3Ck+WMjsT1roF02NeSm36Cni3KkEADFAjBSHUNm0J175pRpt04BeSMN3BrcKsx+6KPsJ37iWHtTFcyP7OwoMpUn0BzirqbQoEcaggYzirU1sTGcCQnsAKTTtNuJsq58tQP4hzUtpClNRV2MQEEEjFDzsB+7ieT/d6fnWmdDhcjzJHI9K0IbWG3QLHGBjv1NZuouhzzxKXwnJo+qTOFFvj3IqaPw/f3she4IRD0J7V1hYgUhfjrSdRszliZNWRj23hTT4GDuS7Dqc4zWrHaWcDZjt4wfUjmmvMF70wXAPSobb3OeU5S3Za3E9DTSwB5qj9okZyM4FMZm3Hk0ibl1p0XgHJ9Kha4cjAGBVdBk5NPxmkMRnds5JpQp24JJoYYpQeB60wGOABkmmgZ6A496fJgISRwKRGUoTnoKABQcUmDvB7CgSAUFyMHAGfWpAkcjacelRIASAary6nao20vub0QZH51DK9xcSDyWCDscc00m9ikmzWAI6kAe5qOW5jiXJYH2BzWUNOYyBri5klx23YFTJbRowwuK0VNlcjI57iS7dTDGy7T94HGaT7JdOp8y7YDuM1eA4oxWigkUopFFNO2H/XA+4XJ/Op0tNox5sh+pFWQOadirKK32PP/LaT86UWUYIJeQ4/2qSbUbW2GHlDN2C8ms6bX3cD7PAR6liKANcQRj1/Fqqz39hbqdzlmH8K5JNYM11dXT7pnOOwHAFRhB6fnTEX7jVBMAI4Ao9STn8qz3UzDEjMR6E8U7AHWnDigBiqFAAGAKcMggEH8BSnAGT0pAx7Z/GqC4uTj7pB9TQcAjHJxQMnFGM845FArgfbrQSQvSkUMSc5HpTgO5OaBjPw57U4g8Y6+nrTsDGKXAznFIBg4OO/pSlc8nHsBTsUZyKYXAAAUUYJGaUCkAgI5z1pCAeaAB6d6UUxhS5GKWk70hC4zignHFGT+FHagBAc8CjBBwaTp060pOelMYHriik6nNAoAWlAGCQADjpSY/OgcnnpQIDSg5FHHWkoBC8GjjOQc0dCaOKQBg44Axmk6GlY8cUg6UxhmlGc57UDkUUCAjjNFGR0pRjjNACE0dqCeaKAE7UUZo5zjvQMKM8UY5oxx7UAB5FHejsaKACikJ5ozQKwUh9qU0mOKBC55xRnFKKQ8UMYZ4x2paQjHWgUAB9KKKKAEIGMkcUoozikB56UxACSeB0oYkAYGcmnHI+tIScgYJJ70hhSAYz9c0vJFAHNAByDn3oz1pCeR+tL3oAMkgDsOlMZQ/BBxmnE4I+uKcRg460ANH3eetHelGOMdqDxQwAnio0ySxJzzx7U/INMUYZucknNADicA5GCDigkDJ6gUgYEH2NOJXGD3FAAGBOAQTTqjAUc4APrQW8tSVOR357UASZpMk01XD4weCKUIQ+fMJHp2pjHfXpS/SmZbJBAwOhpakQtHU0EgHHtmgHFMYp4FJzQee/GaU5H1oEJigdaM5o70ALRmjrSdTQIXp1HNAbGeM0E8Y9KQA0gAc96COAQee4owKXPbFMYc4GaAeKM8YpKAF7A0DrSAc5pc9qQgJoBxSHrQetMYtJnJo7H1pEYNnKEEetADs9PajJJo4xRjmgBASCeKU+1BoI4yKAEpRzSUUALSZo70oOKADNAPFHGetIetACk0UlANAC5oJAGScUh7ml4IwRkGgBM4pA2BgClOAeBRQADHJxyaXBxmgc0Y4NAADg9KUtSZyMUoIIGKQCgjp3pewHemYOc0vNACc8+1L70v1pPpyKYCHGODSggYB70e56UpAJyBxQApz0FN5HANLzR3pAA5PNKetNKk+1AzgAnOKAHAkdhSDk8jBoBFGeaAEOc0oNGRSkgDOOKAG89qXJ71LGkUuQsygg9DTZITG4UsCCOCDxQMaDijOTSAYwTSjFAhQcHcBg1Lb3t1aTrNbzMjr0weKhx6UA5pOKe4XOstPGxUbby3Lg/xIMEVvWeq2d6gMFwNx6K/B/WvNCODSIzrwrED2rjqYOLXu6GiqPqesEY+8CfpSSIki7cAg9Qa8+07X9RsTsMxkjPUOSa6Gy8WWlw2y4jMLD+IZxXHUw84GikmXbjQLWTJUGMnuOlZraXf2T77WUsB/zzODXSxSiVN0MiyJ6qc04hDy64Pcg1kUc/D4ivrYhLyEtjuQQa1rfXrK5YDzNjHsw4/OpntYZ12ugkU9jwfzrKufDcG3NvIYm9G6fnQBp3djHdxEm2t5iemRz+dc43g0ySMUbySf4WOR+dElvqmlt8juV7MnIqxbeJ7hAFuY1mHqAAf/r1cZyj8LE0nuc/qHhy7tDkwuyDqygnNZlzp89squV3RsM7+lemW+s2N1wsoVh1V+KkurC0vIjHNACP9nj9a6IYuUVZ6kOCZ5ORxRtX0rtrnwRCzFra5ZAf4GBP61z+paBfafIA1u7Jjlk+YV2QxEJuyM3BoyQig5wKXIAxjGKe2dvKkfhTdgPfNbXuSN3A8Gl4pQoo2jtTEIAB24FLxngUuMGggdqAGmgEUpxikwDQMM4Oe1JkHpTucYJyKbtwc0AOBINGRnjiko/nQA8kGkJHamg84pQTmmA4c5ozkY4zTeKBzQAtKDTTxyaUHHIJoAXI9/xo4PSgnPvR70AHNGT0oyO9HFACjrQTSAkUooASig4o7UAFBAx1o70EAjBoAKWmhAM4JB9McUfMODigB2cUAgjjrRjvn8KMc0CAk+goBOKCRnpzQDQBmbsKSkbZFRCad5MOrYPbFOIuixZV/WnTXcoh2P8Ae781B6I7yN/Rgv1qEW0byFpCDg4AFUXlJPLkfSp0uIEQDdipC5dCRgcLgD3qLa55RQR71lSvcG4LQhmX061qWMl68YBtyAOpK8UDLDwJFbj5SZCM8dqpgMg+ZTn0Fa0NlJMS8rhfUgZP86ZOIrZgI5Cz+9NMDOKTYDFNgPqeaZghgSc10q28LxqZEy2Mnmq93aSSAJBGqqO+etMDGV2U8RbquRW008Yc2+F9cVdsvKsFIuZISxORk1LJrEBby4wCT/c6UhXK/wBlneMjYQD60+PT41QFhl+9WVuWwGaNgPpTZXuJoysMLgH+IjmkFyIWkzEIJEjj/ugZNTpayIMRNx3OOtMhQ2kYeXcT3zzRNqiRAFdw9sVQyYLLnDNzTWg2y53sf8KYl1LMoaNQc+1I0d5OcJgOemegpiJCysfKAJY9gKlC26R4aMH3PapYdBvMrJLdKf8AZHFW/wCxd33pP0qOeJi68O5ikRs26EbWB61bgNy3Ic7Qea17XTI7Zy3DcelXAir0VfyqJVOxjPFW0iZ32aSSPKzPntxQlhMy4eVgPcda0txHU0bx3IqeeRzqvU7lFNMRCD5h4OelXQiDnHNNaZFOM1BJdAHABFJyb3FKUp7lv7oyKQEHqRVH7U7Zxmm+e5PJP5VJFi+0qKMlhUZuB25qpgE5NLs44oAe1w5z2qMOx6k5owaFHFAIRgSKReOcZpWBzxSjgUhigDcTjrSHrSZ5JpCwB5IFMVhwOKUGoVkBfAz9abJKiqSXANIZOSD3pyuB0GazDfxbtqFnb2p6tcsoKqEU9cnmq5WOzLV7L/okgGMnsTisdLxw5TnHsMg1eFspcNKzN6jPWp1WOMYjjA/CrVN9TRQ7lHF5OuEXyl/vN1/KlTThuzcXUsh/u5wKvksTzRszzVqCQ+VIjRIIwBFEox3xzTxuJ6U8L7UuAOpwPUmrsUN2mnDIGMVA1/ZxsQ1woI7VRn16McW8ZkPqeKdgua2D0xUUl1DAD5kigjtXPTXuoXHDTlU9EGBUAiB5Ykt65xQ0I1ptdQEpApY/TH9KzZ7u7uuJZCB6DgUwLjpRg0WARUCEEED3PWnKAOgpCKcowKoA4APHWkpeO9BIHagBME9qOW7jilPrS5AGDQA09KXAznNGKCMfLj8aAYoHFB6UoBpByaQCZ5I9KXpyTRRnseaYxSfSg57DNJjH0pccUCEyehFKD+dA6n3o70gFwaTPNLkgijAJoASlyM0Dk0oIBIAHXvTAM4ozntRxmk5zSADwKAeBxj2pOc4pwzjk5oAbmkwM5p2KTvTGAFL70EYoLYHJoAXgcnrRk88e5pMc80tAhAc8jmjv0ozg4ozzQCF5zwcUDp70dTSdDxQA4jFJRk4wTnFHYYoBCUucDOMn0o4x3JpOlAxBnjIpee9Lk54pMnOTQK4DkA0c5oB5pGOKAD6UUCgdaAF6mig0gyaAAdaKKOMe/egAPUe1FFIaAFOSQBxijNGKBQAUHpig4HNHPegAAoo7UUABFGTSCjNAgJPr+lAOKO9GCKAHZ/Ok5II9e9FGPagYgXaMZJ9zS5opN2DQAHk5NKBQKCcDFAARQOlNyTjJzjpSg889KAFPtQTkYpAcjPSkwSc5IFACgYppRWyDngZ4pxx2OaZty5OSKQChFTOASSOeaAAyFcEdqAPlPJ4PegHimAoyIyCQR3JFRCSMkqVFSZDggg9e9N8mMkEqeO2aAHnagAAwOlOJ6YPakIyMdKQLjpnFAxwJxg4/CgnjFA70h60AOGCKBxg0duPSmqwJwevpQhDsgHBpQcjFIemAB9aBQSBoJ9qXFIfpmgbFHrnmkFFGeaYCgjPNHGMimjGeaXpSGKDg5oJBIIGPWgAHqaCD0BoAaSQ2O2ad0NNBfo2OPSjNADgxGQOh60Agcmm4IbOcgjoR0pQcEH0oAUc9u9ITRk5GKKAE70pJPWjqTSA80AGM06kzRz2oAPqKDRmigAo6UHpQORQAho7UUvGSR37UABGRR2waSjvzQAtGOc0ZwM96MjFAC8AcGkxz0pDjGDSigBBgdBgU6m4wDS844oACaM0Y4oFACd6d3pCKMZGDQAoOOR3pc54plHORSAeAAOuaQEhsDqe1IDnnpS9CDTABkk5GKdn5cCkJzkmgdM0gDPFKCPSmmkFMEScE00dSCOe3NLxijAHOOaAG9DmnggkmkGR15FNBOelIBxHNA9KUHJ5700ZHUfnQA0xgnI4PqKAZFHI3/Snk4oFMYgcOMjP0IpQeMUHHOKAQeDUiFycnmmZIIp3c0hA25JxTAASc5pe1GDtyKQAg8nigB3OcmkKqT70E0DrmgCaO8uoCDDPJGf8AZPWujsfF8ixBL5PNx1YdTXLZpDWNShCas0UpNHpNlqun6givBPtJ/hkO0/rV8l/9l1ryqNip3A4Iq5Z65qFjKDHcMV7q3IringpfZZSqLqelAx+wNVp9Pt5wPMt429GBwaxLHxdbXRC3sfkyE/eUcGuiheK4QSW0qunqtc0oSi7SRqmmYNx4bBYtbyjB/gbjH45qoZtW0vCfvNg6ZJINdWQQfnXj1AoO04AcEf3X6VIzDtvE6MALiIg+oOf6VsW+o291CTbzK2eoP9RVK60i0uCWNvsY94+lZE3h25ibzLdxIPTdgihMDdfTre/yl1aQsh/iUYI/KsK98FKW3WU5wf8Alm3GPxqKDVL3T5Arq2PR16/jW5b+JLSTiUtGfQkYq4VZQ+FiaT3OI1HRrzTcGSByh/iAzWcrDuDXrsM8VxHujdXU9QDkfjVO90TTr+PZLbqvugANdUMY0rSRm6fY8vLqehoHNdZdeBZATJZ3ClOyvjNc/faZdabJsuImVf72Mj8xXXCvCbsmQ4tFMgYxim4waXIPQ5PvTctu4wfrWxApzjik5IzSjPINAFAxp6ZFKD7ZoxSY+bIJFABmlJIAOCfpSZoBI4zxQA6kzikNB6UAKOTzR0NJml70ALnAyaFb0oGRyMEelJ+FMBxPNJn0pVxnnpQQM8dKQgzQCaTFJk9jTAeT60DHamjJ60oODigYvA4NBGfYetB5GcikJOMCgBSCDwc0ZGOTzSZOPejHfvQAHrSg0lAoFcUmiikI4oGZjGYtmOVQpqCWGYucjeT1IrRtbW2MYEjN9AKtQzQ2+VhgJI6k1B6BkR6W1xGAlqRnqzdKc2kpbN5ZALD06GtC61KQgLGdoHXFKUknt03BgT1OKVgIo5BCoBeKM+w5qWS48wARSg57njNRyaekakLzIepI6U+CyUR/OWD+uadhjpYrgxH94g+lNhtZhGDhD9TTjbBTxvP0NWIZURAjq24e1FgIzdi3TDDc3oDWfcXZmkJMjIOwHatmDShezFyvy+9a1rolvFIZGjQt9M1DkkYTrRgcbDawzsZAZpCOucitS0tVgAcRBMdAeTXVGyibjaoHoFxSpaQoAoQEDpmp9ouxl9aXYw7a1kupGw2MDj0q4dPumQoJFyehx0rUWJE+6oH0p4AFDqPoZPEyb9058eH7jzNz3Kn2xVmPw/b7w048wnrmtYnHWmGdF6mlzyF9YqdwW3gQARxAY9qdsA/hH5VXa9QHjdTDeOTkVFznuy7kikMgBwSBWeZpG60w5PUn86Qy+bhOxBqM3PJANVQMkAVIEHUkk0gFaZy3WmFmJzk0FfmzmlAoBCBSDmlI5zin4zxnFGMcZoGMIOMdKaAwOMDHrUh6YBFICCDz0oAApzz0qQ4FQ+YPUUxpXY7UUse+O1O4ExYUhdR3qvsmPJIGexIqFo+fnuYgey7uaLMLFp5VHQ1E8x7GqTG4D4hTcexNSJa3UvM0ipnsP8a09kyuRjjckNjfye3c1BPdyj5YrcySds1disY4m3k7n9anChelVyLqUoIykg1OcgyFIQfQ81aXTwOXkeQ+54q5k0Zq+RFWRAtuqcKAvuBUoQdyTUgGe1QTXVvbqTJKoPpnmmUP2Cl2AdqypNfgBxDCznsTxVKXV7+UEKRGD6AZFAHRsyRrl3VQO5OKoT6zaxcR7pj/ALA4rCCvJzJKXPqSKcEA6EfnTJL0mt3DLtihCfUZqlJPd3DZmlJHoBgUuMdx+dGaoZGIVHalAApSCT1ox7mkA4EelISATS44pNueDQAZ9KXPHUZo6ccUAZ60yRPcUZwOaUgk+gpv8WO3rQUIDkZCn8eKU5o3EEAgkH0p2BnigEISe9BPHTp3pSADgH86Oc4zmgTEBHpRSkYoyKBhnA5opMfNS4oGHIPPSjHegjigdMdqAF9qMe2aTBzyMUuT0oAUKSTgdBk00nB56etKGwCc55oxkYPI60CEBBGRzSg4BxSYHYYFKABxQMXOe2KQg9qAfSjknmgVg/DFL25o70GgLhxS9O3Wk6DNGSSCelAC5wKXORj8aTBz0pCR0pAKWJ4owD1GaSgDvQAUc9utHQ560Eg9Bg0wDr1696B1H1o6GjFAID1JooooGAPP4UdeaDkCj2oEAODRkk5owMUnQUAL3pM0uecUEcUCuJ1pTjFIM0UDsGKTv1petGKBXEB/OgccfzoI6GjOBmgoBS9qQcjNLkZP06UEsQjnrS4yaTdntigEgg0DBs5xS0E9frQOuT0oAQ8j1xQSetBIHSg0CAHrkUo5pKKADFFB7mkByaAD+KlA5PHWkH3iM+9KTge9AC0mTnGO2aXJxxSZAGcGgYp9R0ppPI6UpJzjtSYOeBnn8aAFzzgdqTn0owcHHXFIAQCc9aAHAE/Sg8Ui5Awe1BI6HPHegBACVGTzk5oQnac8ZJzTs5oJ4zQAgAAyAMH0pAG3gngD9aVelMkycbWwRQA9hnjJFQh2DkAAgjoe1TA5APeoXRvM8wdQMUASgnGSMH0pED7ySRjsBQj7gCQckdxT+BQApPNAAGcZ5PejjBJ6AVE0jghQAMnjigCQqScZGfWjbkfNzmgZI644peo64oGCjGQT9KCAee/TNHWjIBIJH40ALjijJxkD6Uh/D8KDQIXPFNwAc5xSjpQQCKBi1GxByCcU8HBpCAScDrQKw0Nt4608HOAemetN2jPSlAOeOlAWHkofuZwDjmkzSDAPTvmjvmgBc0gBHWnZA65xntTcc0ANAYMSTkHoPSngDHJ/ClzjgU09aAFoPSkozQAuOho4pCTwBQaAA04fdB/CkJ4AzxQKAEA5xRQeTQDkfhzQAE5oBI5FHal7UAAFBxR1owe9ACE8ZoIPB9aKCOhzQAYJIFBHpR3xQaAA8jBoHrRR0GTQAfWjFGaMjNADiSWwelGKGwWyBxSdqADNHP1oGB1FBAJ4GPYUABHSgA4yBSd8UHPagELnijPGOaTFKDxQADk0tN5zS5oAXPNGeaOMe9IaAQvIJBozQTzQBkE+lAAD8xp2ARx1700Be/HvQWAPB5pCAgg0pJIGaTOTQetAxSaB0560A8dOPWjtQCA0CgDJxRjBxQAd80oOQM0cDrSc9qAFA549aM80uWHBHbg0056nvQAGjknAGTSdqCOM9KQBz3GD6UEkU44Iz39ab9aYACQOuKD1zS0UCCpoby4tiTBKyE9wahFLUyipKzQJ2N/T/FV7E225cSqf7w6fjXRWmv6beMqPMscx7Hp+defNSKSDwa5qmEhJ3WhoptHrAWQDMb7h6g5FN3AE+YhDeoOK85s9d1HT8C2m2qOqkZFdDYeMElUC+hAbu6nH6Vxzw04K7LU0zo2iWYEOkc0Z7HrWXd6BaztmItC57HkVo293ZXo3Wlyrt7cEVMRIDhgGHvWLVi7nKPpeq6a3nW7E46mMjP5U+HxRfQkrMgkI65BBrpwyHozIw9qims7e6Obi2Vx/fAwTUiuRWWv2t2ApzE57ORj860W2yqVdAy9wRkGucn8OIcm1nxjokgJ/WqhGsaYgQbwnbA3CmmBv3WhabdqR9nEbHqy/4VzN54Juo3Js7hJE7Kev61pWXiRhhLuPPrICB+lbUOo2c4zFcRk+hODWsK0ofCxOKZ5pfWNzp0uy5iKD+8elVgwPRga9cdYpV2yRo4/2lzWXc+GtMucnymiY90OB+WK64Y3T3kZun2POMEnIoIPtXRXvgu/tyxtpVljHQYOaxLmxurLH2iJ1HqRgV0xrQk7JkuLRWIxSEHHSnggjIOaTODwcGtRDfpSGpCMjJ60wg0yRvNLmnADvSECgoQZzQTijFIR2oAUOfUEUu7PSmYI6UucHigkeGIoB55AppOTSjFIodnngcelIcYGM5700ZBo/nQSOx70UnU5FGTmmUBJpQSaTNGaAHYPpkUnQ80mT60detADs4OccUE5NICRSbsHn8qBWK1vp94/7yS4KRjqQMGn3UMdvGAJWfPqelb66VcO4WQ4XuRV9dGsyBvi3t6k1i5pHVOvCPmchYRLlWJ3t3xXRxWzlgDkL7CtWGzt4ARHEqg+1TYA6cVDqPoc08S38JnrajcGeJSfephbRHrCv5VZIphYDrU3ZjzS7kf2SDvEopfslv3iU/WlMygZzTDcr2NK7Bzl3JlVYxhVAFOBFVDcg9KaZmIwOKVyblwuB1NMMyA8mqZJPU0houItG7A6AmoXuHYYHFR4JpQuKLghpdyMkmo8ZOT39anKjHSmgDOAKAGhMmpFTAxilAwcik3ndjNAx4TmmnrjGaN+aQt3zQA9eOopee1RhxtJLCojcAEgHn0oAnK7hTQAMVVaeTPBxUTSknmQ59BQM0CwHU1BJcAHA5rOuLprZd21jyBgnk1UfULuSMiO2256Nnmq5GPlZrvK5BANQtI6jLMKyhHqEw8tztX36n8alj0tmOJGJHoDVKn3LUCVr1VB6O38Kg96bHqGrudqRBR7AVdt7KGAZWIAjuTmracdAB9KvkRXKjMXSZJ8vdTSFm6jNXYdOtYcbIjkdycmrPNKATVDE2gdBikyfSn7T64+tVLjUrO1wGkDuey8mqTAtAk04KT1rFl19iuIIQM92rOlu7u4J82Y4PYcCgR0M2oWlvw8oz6Dk1Rm11OVt43c9iRiscRr6U9BjpTQ0yWW+vp8h3IB7AYFQGINgtuJ9SamBpCcUxXECqrAgcAUvHYUw7s5GOeuadnA6ZosAcdqVuQBjNJ1p2PWgQ0AgDNHOeaMgHinUFDaUHA4pDQBxQA4c0oPJpozjgZNIG45696AFPNHOe2KQnPHP1oHQDtQA7JooppYnIwfrSEOXmikApelMYlLjBpARjng0p5HWgAPI96AMigLgY607GBQJjegpeTz3pBjvQT6dKBinp2pKaSSOMZpRkgEjGaBinGMZ5oH1oxRQAtKoycZAx1zSHjjHNHb60CuHvSZJo/lS5z2oGIo2inZwCMcmjJJA7UcHmgBMHvS9qXgnOPwpO5xQIOMEHvSDpSkdDRnigYcjBOcUUH3JNJQAtB5FITQowOKCbhQOcUHOB6HtRQMd3yRmk5HB780DPekGAc56UCQvOeDg0uaTuaDj0oGHJFJ2pfxpKAAmlGMUmBg5H0oAx0oAQj86UZAwTnHegciigAzRRigYzzQAdKOppOe9LQAZoB55H1FJQAM+9ACk+wHsKKaeaXGRQAcE8iig0daAAmlB56ZpOo/lQTg5FAB1GKDQP/1UHB60CENKOgzzQAMkknnoKTPHAyfSmAA5yKQdaOTx0NLjBwaQxexIptOBABFIinDHI60ALxSE4FHUUhAIAIyB0oADyQfSj5ieB+NAJBOcY7UuSKAFbpgHk0Hnp0pueaeo70AA4pG6ZozzjHBpADknOc0AKB29qQjjGT0pGySPY0AYJ5PNAACMkZJ+gps2AmQQDnqacMgcnJ9aCAV5AP1oAAGVACQSByRQcnpRwASewoV1cZU5GaAHdQCetGM1EZlEmzBz69qeA3sRQAAgDHJ9QRUg2kgsMgHnFN+poJ4OMHFAMDxQKZG7MCWA9qk7ZoAM0xkVzg5ye+adkE4FAXBzk0DBV2gDJOKCQD3paaJFYkKQSDzQA4ClqJicglfl9QaeBkD3oAfGELfOSBjqKjI5yKdtAoAGDmgBBnPNPONo45Hf1pACOSfpSE8UCFIO3OOPWkGRTGBcbAxA6kU8cD6UAO5xTAxLkEcdjS5oLAEAUAKKQ9alhaEbvNQnPTBxioyKAEPSjoKU9aVsYAHNADRRx3/Cl9BSUAGOM4o9qP4aB60AHQ4ooOQ2fWgUAApc8UgGTxSjHegBAfyoJzSgYY9qCBQAlHXoaKMUAGaXjI703nOKXIB96AA/ePpQeRQORk0fSgAzS4GMim0hYL1IBPQGgB/Hb8aO1NzxkH8KXnFAC0bT1oBoyelBIh+9SijFJ07UAOI5pM4OO1ITQBmkUC5xkig+3WgEjgE47g0EZB9qYCg9qTpxmjJycijO4Z6H0oAU5xmgjvSDPfpQT6GgBRkmlIGcGmqwOfUU4gHqSD60AAIC4o4PNNOc8UowMZPSkA7FNIOeCPrSk85BxSZwMkdaYJgADzyKMHkZzRS0hAecUo9KBjHNBOGGOlA7hz0pwIIphOG9qXb3z1oACMHmlJGDk0hz68UnIwQKAAMASCDgjg9qOpoyfTFBNAClTjNIDzSEnOSaUcjIoAUYJAzQc44oIH40mKAFB9aUeoptAODjFAASRSgg8Ec0H9aQDmgLEsM0lvKHiYqR6VsWvizU7farlZIx/eHP51h0c1nOlCerQ1Jo9BsPEenX+RLJ5Mvox4P41sBGZQ0ZDKehByK8kIrR07WrzTAFhlOzsrcj9a4quD6wLU+56Uc9XQE+o4oBQcJIfZT0rlbLxqZXCXsA2n+JOP0rora6sb4breZG/wBnPP5VyTpyg7SLUk9guNNs7nme3Ab1Q4NY134bcAvazBh/dbg/zrfIYdGJ+tJkHkghvUGpHc5UXuraOwSRHKDtICR+daVp4ohkUCZGjb1ByD/hW2VLKQwSVD1B5zWbc6Fp8+fla3Y/3RwKQGlb3cc6B4Zkcd8HmnyhLhNk0aSIeoYVx934e1KzXzbYrLg8FTg/zoi13ULPCXCF/wDroMH86d7ajNq68KaZcHdHGYSf7vIrndQ8E6hC2+1ZJk92wa3LTxLbTMBOpgPrnIrZSVpYw9vMrqehB4ranXnB3TIcUzyu4tbm1YJPGyk9sVCGGK9WdmkBW6tlkQ9SQDWHeeE9MulY27m2c+pFdMMZ0miHT7HCEgng0uK2Lzwjq1oCyxCVPVG5/KsqWOW2fZKhVvQjmuuNWEnZMhpoZ2zTcc0/cCOetIOTitBCbcjIpMHkAfSngleO1ITk0xEeTkAjHvS06k25NIYE570HNKFFIePemMAcGjPOcUcHpQcgYoAeCCPSmkEHkcUgJoyQRQAfzo570p5570lAAKAce9GRjmjjHvQJnbFwOpAphuEX+IVlSOScEnFAyTjJNcZzmgbxAcdfpTTensKqFcYpwFIB7XMhJOcVXaaRnxuOKe5wcUwDkGmBMAT3NOA4xRGOMmnnApDG7eOlLtpaMgjgUAGKABmjOKAQDnOaAFAoIwaDKqjOaga4yeDmgRLu7UZA61UDSOxwCPrUbxuDmR8L9aBltpkBxkCoTcIGJJqg91BHxksaYb1Sm2KBifcZq+SRXKy+05f7p4prF9mWkCr71QQzsCXIjXH0NIm3JLRSS+hGQM1XsyvZlyMNM/7pGde7DoKZLczQOVEI3f3iamt7i8fgQqgPQYwAKnNvJP8ANcSDd6AVXJErkRjm4uWOZSoX6VG17cJJthUEnoduSa1hpdqrAkyHHYtxVmOOGJdscagfSrKMIW+o3bAsm3PUt0FW7fS3iYmSbzGPb0rVJ3Dim7SelFhkHkt7VIkZHWiRymMRsxPQAdar3V5fW0JmFg4QfxPwKLAXSlRtcW8C5lnjX8c1zM+oXt4xLzFQey8CoBAAQWJP1p8pJ0EmuWqkiMPIR6DFUpdbu3OYY0RfpmqOMHIpw4FHKJMHuLyYnzLhsegNRiMDt+NPIzRg9qoaFCgDOKDThnpSEUgsAwRgdaATnGCKAOARS89aYJAKBnJOKM5OMGgDmgYu00BQRnNG4A4LfhSnHXqKCRDzwAKM0hIzxz7UZ5oKCgNRnNAWgAY5I5oJxQAPSjigBMEkHcRjsKd1NBxjikFACnAOBSY7mg80YwCeTQAc9qUUpB25yAM0UAIN2eSMUoBYnHIFBAxRgY4JyT0oAMDvQTg9KU5BwKSgVxwJ6imkkjIyTQScYyMd6aM9P0oCwAk+1LgHOeBRyDjFOxxQO4mAOAKKMUYoGBORgUUYpf50AGCTntSgc5pA2M8ZpATjpigQpGOvSlAB70h3E5OPwoJwOR+VACkc9aSjBBAIxnmg4zQAuKO9Jg5x3ozQAtHamk4FKDxmgYZ6CgAjOaCOhAzilB4zzz2NAhAQ1Lj0pABzge9Lng0AKSePWm4oJOeDSkc9c0AGaTPOKXjOKTIzjvQAKMDv+NA6CjHGSaKAD6UvXpScZx2oII/GgBSCRnrzikPH4UAkjBGMGigAyccdaQZzzRnn3paBCHI6UoyBz1oyOh60maBij1pDyaOaOB1oAKM0gJOOKKAFpOeRRSigAyMHNA5OBSEAkZGRQcZ4oAdmkNIBjiloAD/SmKmOpyakx6HmmZwM8mgB3G4Z4zSA8/0pOvpn3pc4OR34FACk5ORQepJ4zSAEAEjiloAQ8gjtQAFwoHFA5PH40uccYoATGDnJoPGPfpSk8U1gOMHBPHNAABgUDIHPWjtj8aBzjJyDQAEA4owM5JwaMEH2pQQTg0AOGM4IzTe5APSlGAck4FHIPp70AKOM55pjDjgkZoyQRnnmm7DkkEnPSgB4OMA9ex9aQ4BzmhM459aGYAEngd6AADcpz3oAESsSeM5pQcgEEEEdqDz6YoAYXRyoDjJ7U4AoODTVRVbAAGacp3SMmCAODmgALbsgc460ufpmkEKLIHGcg889aXHNAmKB8uO9GcjANApCBggHB7GgBCDnIOKfQCQwORkdfenEhiSQAfakMbnHFN2AMWAGTTh1NITimMRmABz6dPWlQ5QHpmgjPNLgcnjHtQAoOaQ8n2oFB60hATz14oOD0pB7jtR1XGKYCgAHPeg9aCTtzjmjntQAA4IOM47UMRkswx347UhJGOKMZHIzQAoIIyCCKVTz0H41GiiMYAxmpExnnigAbAOOc980mMcdzzShgQR6GkOSQQM46/SgAOSOelKCMZ60hPBPbr+FIxwMgZ4oAUnjHaj+VNDZGaUHigBT+nakzzSnpSYzmgBc8g9KCeaQ+hpOwxQA7sPU0gJyetNAKk8kijOMZoAfS96aBznNKeCO+fSkAUcE80hPGaQcgEd6YDj7HijIA5ozk8DHag88UAID3HQ00qGxuGcU72FIc44oAANvA4p3SkB46UZ55oAUHj3pSelNB5PPFBOelBI7ggY6ig9CD3po604nAB5JoGBAxik57UpOeaaMg0AhTjOeSO9IACQVJxig4ORnmgDimAnJyKACAQTnng04DHJoLAHOOD+lIYDpg8nGKbgBuOvpTiVGCTgg8cUH1oAOnbmgnAyaAc0o5OMj8TQAmQacVUjJJ/Cg8HBHIpDQAEcYwDTgDjggfWmkEdaM+tBIhJDYI/GnAEnrTd3zYP50vfOaCgB5pxOOgpuRnB4oP3qQDhkg+1GBt680mecg0A7eozmgBTSA5OB1oJGcgYFAI6igAycnIoozSZoAMDvSjjocUoIzgij5c8GgABpGJxxQeBwRQD60ANBLDOc0oY9+aWggDkZ/GgQvJAOBg9DQQOo4pANowOATnFITQNsUHJo5PSmg80tArjgMmgjB5pvJHBpcnHJoAB7YpysyMHjdlcdxxTckHIoByaVrgb9p4uvraNY3IlXuH5P510mneKLDUBiT/R37hyCPzrz3aBTeQeCRXNPCwa93QtSa3PWdqSANE4YeqtxRk9Dgj0Iry+zvrqyl3wXEig9QDxXSWnjV1xHeQF1/vjAIrjnhZxempammdX8q8pI6N6EUSolxGVuIVlU9TioLTUdPv0BhlAY/wuQDVoqyj92cVzlmLP4atJiTaymEnsRkfrWbLpOqaZkwiRgOrRnj8hXUjIOZFP1Ap+WHEbkjsDxQgOZtPElzEAlxGHUfn/KtW31jT7pjljG/o4wPzqzPZ2dwCLi1Ac9SBzWRc+F9/NnOp/2XwDQB0STY6EEdiCCDUN1Z2l+CLiFJB3PeuPI1TSJBkyx49BkVoWviYo3763J/2kwDTi2ndASXfguyfc1o5hY/wkkiuXu/DGrWbEiBpUH8SDIrv7XUra8UGOZc/wB1jgj8KuiTb3xW8MTOHW4nBM8gfdG22RCrdww5oBUjOBXqV7pWn6muLm2Rm/vAYNYVx4IgwTaTFf8AZkJ/Q11xxcbe9uZOm+hxHINOBOKv32halYylXgZl/vLyKzyjxYLqRn1rpU4vZkWYvWkwadkEcUhBA5FUAU0g546dacCc5pCTmmMaOBmgknmhjS7SAD2NArDcnNLSZoGKBCnkUAcYzmnBuxAxTCCDkHigDeZQGJIqZQDgjHSmkAnBqVcBRyAK4rmAh4OKRjg4oMiFjg5xTHOTkUXGNIyacRyD6Co95BweDTHLM2AaLjLJlAGc9KVZgRk1UCvzupy4ByWwKBFozgUhuCRgYz9KrNNCM9T+NQvfJGPlTIosBbzIe+aApPLPj6mqQnupmwkTD3xipTp9zImXfbn0NUoMaiywZIBwZR+dV5r23jfbEfMb27UsWmQo26TMh9zxVhLeCNsrGo9qtU+5ah3KLy3c7KIlKD1oWyuHX97Kc+grSLr2FRm6gX7zqD6A5NaJJGiSRBFYxxkZGcetWQiDgAUB1kAK5wemaCVHUr+dO47CmNTzgVIDgdvxqlKL6bItwNvqOtU5Le5hG+TJcepzS5h2NnJI4ANRPIV6si/U1mxxahcYCpJk9MHAq8mjTRxGRyrv7npU3CxPbvCVO+VWfsFPFLJdW8Od6rkdgM1QayvZpAIhHEP1/Op7fR5HlAnleTPcHj8aZNhP7YYnEMJYdgFq1FdXcmDNAkMYBLFhggUMttYzDLgKOpJI/Ks2/wDEAMpNiN3GC5GMH6VYy3vudSlzFItpbAgCWTgnHpTdU1izs9Oewt5nvJm4LnkD3rmLh7q+ZftVw7qDkKOBSpGka4QYHpmmSLGNqnI5NOGCORSU7k4xwKoBOgxRjPegAjO7HtThgHFIYgAxSjgZoGT+FA6UAGDSYNKW4xSE8etACk9MClJA6+lIFJ6mmhFJJJNADzJxgACkyR1pMc8UY/OgQoJJ6Ud+TQOBQSO5oGGaVcAE4zmmnA5PSg8nr+FACk5ozikwDS9RQAvP4UhHPWgsc8DNAyRk0AAoJpcUhAPWgBcYGcUpHajJIwT9KTPOO9ACjIzxQCAPU0djSAgjINAxQc8YpCOcmkG0A4BPrSFySVHAoAeWAB9aQ56459KBx1xTiM80hDADjOOtPxgj2pCSTn2oBoAKUUZxSA55HBpjFNGMmkBJJyMUpIAzQICQqjjBoBOB70nBAPBB6UuD3oGIGPYZpScigjAGe9IOtACjj1pc0qsVJI7jFNoFYXBPJ70EYNHQA5/CkPPSgLC0Y9KTFOBxzjPFArDAwJ9aXPOMcUM4wMjHpxTgARQO42lPSgnJyTSUAKOmO9IaKOp4oAMj1oI6GkUHGDS9s9qACjPeijntQAvBPcUmOM5oHOc/nRjjNAAMg84ozSDriigBTR9aOD+FJ1OKADAznvQKCQDijGaAFJ9qTOMe9BPFAwRQAUh6UDqR0oPGR1oATjk57UYPFBBBwCMUooAXjNBIHNJkD60p5/CgBFPGR+VB5NLySc0DHbpQAoNJ06UEYpPU0AL2ppJ4xxSEkjj86UYI47UCAgk8CncHK98dqTpz6UEd6BCYxjJJNKCKBnA9TQQc8Y/GixQgI5IozkZo759aDnBwMn0oAAKBxkDBPXFC9cng0ZGd3ORQgEI44AoBGQMH1oBJP1NKM5waAF79aADzznmgHqCMc0Y25oAQ4HODj2oL9Bg8nGaU8DnIppznjigBSOQaCCykClODjAxwKieRUOMnPoKAJFUKCBk/WlIDA5pIyHjJ96aC4fjBXuDQAnmJDGATgD2qQEFQQeCMimhVZACAR704jIAHQdqADikJw+O1OAJIHrTAVfJGTg+lADz19hQOpPX0psiCRQNxHPaiNPLGMk89TQA+gjjOOKQkjGBkmgk4xSAXFBODj2pAcEE9BSsAeR0oAB1zRjJFA4PtQTTBCElGxj2o7g9qUD1FKTkfpQAhIYAjH4UgwcjOD2z3oAAGMfSgj5gfSgYuMCkJIGcZpc5pSCAM+maBDe1HmL2I/OjntTSilskc+tADzggEdKBQCAPYUZBoABQelISRjAzzzzS4/OgAHHYc0oYqcik7n9KMUADHJHbikBGcA9qUdaQgjkDNABgVNMsIjjMJOSPmB7GoaBQAoIBGTQeDRkjtkd6MgjIoACM80A9RSAnFBzjjqO9ACEA9RSZI7ZzTsEgHOeKbQAHIPt60o45pBnOKUjFADgRnjmm/hgUAAHI6mg5Bxg4oAXGKTvS5wfWg89BigBRRimqCeDQcg4HWgBR1prZ6DrTugoLZPOKLgM5AwRg0DjB7U8gYpAACR27UCAkEcE0oGR1xSbSQSBkDvQMg80ALSjHegEHikNACng0HOBikJJbmlBwcnp3oGAJIyBxRjPBpeACATikoATPUEUoHU5/CjGeR2oK5APoaAAZ9KDg9fyoBz1HUUhXtQAuOnOaMZ60g4xS5zQA48004PBFAOTgEE+lGD34NABiijIxSZNACvgYB7il35AGOnek69aABQApKkggYPekJJowM5o4oAcoJBJI47UAKOST9KTBpQMnk4pALlTwM5pCAM4PWggDB60EjtQAEk9aMAHNBJozQAhODQORkUEgDmgUAPOCOuCKaPegkYHY+opoJNAhxNJjilAFByKBCAYHPSggnkGkycYPNOU8GgY3JAxSg560h5NBJHWiwx1NGQc05ST04PbNBLZIOKAAsDRnKgelJtGeRS4I57UCA8UpH5UhPNJ8469O1FgHl3C/K5yOla2meJL2wQIZnaMdFbmsbOaUis50ozVmhqTR3dh4xtLl/LuYmgP8AeBBFb8ZjmjEkLh1PcGvJQWUgg4IqeC+u7V98NxIp7Ddx+Vck8Hr7rNFU7nqBEi8EcU0wocMPkf1FcpZeNZI8fbozInfYOa6ax1Ww1OPdbyjP91jg/rXHKlOO6NE0yYvcLw6iZB3HWqc2l6XeneYzG57jINaO3sKZIACN8eR6jqKhDOZvfDN1D89u6yp+OfzqrDqGo6dIAZJFI6q44rr1O0jy5D7AmllCXCGO4iV1PqM1NwMW38TW7EJNC0Lf3gcj+VbUM6ToHhkEqnuvase58MW0xJtZjCT2PI/nWRNpeqaXIWQHaerxnj/PNUmI7YSDGCCAeoPQ1RvdG03UIyk1smT/ABLwa5618R3MLYuEWZfUnBH+NbUOsWUwG25Csf4XPIoUrO6A5698ESQsWspPMQ9Axxj9a526sL+ybbNbypjuV4/OvUFlI5+8vqORU+9JlKuA6nqD0rpp4qcd9SXBM8fLMpw3BpRkivRr7wrpl6Pki8hz/En+Fc/d+Cru2jZreXzR2ABrrhi4Na6EOm+hy7DjNCkgdeD1FPlguYHKTQyIw9RimggDDAj611JpkCYOOO9HbBFKAvQE/Q0EHtTAawJHFICRwelPHvTW9aAN7fTGkY8ZxRRXEcwsdvJINyjj1okjMK5kmCj2oopDHQTWAjy0pY+pFU5r+1jlPl/nmiiriruw4q7IRqPnNtiyx9hVoWN3OAS2wdxnrRRWjio6mjilqWItI53M7ce9Tm3iifLICfeiiiDuwi7sd5noAB7UhYnvRRWhY05NJgmiigY7ZkEEhR3J6Cs99Q020nIQAsOr4zk0UUAh7a7pz9ZCPwIpo1fSc5JY/gaKKg1LQ8QaTCu1Hkx6BTUT+I9OH3Y5nPsMUUVRBE3ia1H+rtJWPpnFNHikEYFi4+rUUVQEY8TybwTZKR6ZNR3Wu3l2uyMeRGeoXOcfWiigkzfLBOSSx9WJNP6CiiqIE5PY0ANgZGKKKYDscGgEgUUUAGQTQeuaKKQwDAZ4BoySewHrRRQIARiggDmiigoMnGAeKM+tFFAC84OBSDJGe1FFACHIpe2SeaKKADPvQOCT1oooAXGaXFFFACckj0pcUUUAKF96OKKKAAjvSYBHeiigA44B/GkPWiigEJgkEClUYFFFAx2B3GaXOAR37GiikICTikAOTk0UUCFC4HX86BjGaKKBoCMikAJ47UUUwF4AAHagHk0UUAGMnJpcZJ9KKKADpSfjRRQMKOBRRQICcUUUUAGBwfSjvRRQAhAwSCaXGQD60UUAHQUcEEUUUAAGB1oAwMDpRRQAuQcZ4poJIyBRRQAA8AjoaAR0oooAXOM470dqKKAEyAKOooooAOgAozxRRQAmM0vfFFFAB0NJnj8aKKAEBJyT0ooooAOCRnqOlAI5oooAUHJFL3oooEBGRTSOMY60UUDF+lNAxz39aKKYIXOR0pVJwc9qKKQARkZ75owOcntRRQSIMUu70oooKG5IIx2pFZ3YkrgdR70UUASjABzimZOc9qKKADBP55oPJx7daKKAFIPf8KBnAyRRRQAhqMwgsSSeaKKAHAbFBwfwp3eiigBsbqUOwkgHFOLcHAz/ADoooAhMjEhcHnrUqgqoBHSiigBcUjOicMcE9PeiigBSQCCAaMk0UUALg9hSLGAMsTn0oooAeOlIQQRn60UUAL1PtQeKKKADrR+dFFAIYFw2RyaeCBjOTnPBoooBik55ximk4Gc4NFFACBflwSSKOgoooAUb8DOMdqXnHWiigAFGCScdhk0UUABUgjIIyO4pc4FFFADMktjFAyM59eKKKAF3AnGeaMjOO9FFAB3oGe5zRRQAnPGOhpMnJGORRRQAE8e9G7OARn+lFFAADyBntgE0Z6Z/OiigBx4NGepFFFACjOCTTc8+9FFDAXJ6ClxkcjBoooAQmkOR+NFFACHIPf60Z45PNFFACkHGR0pAecE4oooAUyADGMmgMGAI6UUUABz2p3JoooAaQTwCQfanAnNFFABz2oHIJoooAQgk8c+lJgjqCD6UUUALtHUcH2pxIIGRz6+tFFACEDOQeKOM470UUABIzzxRwBmiigBeopOhHPFFFAATjoc0En0oopAAJNKKKKAF6ikoooANoIwTg0gwOM5oopgOynemkjdgUUUgHYOAc00ruOckGiigBwGRjv3pMYNFFAB3pSMjBoooAbjBwKdz3oooACaac4oooAcMHqMUEgcZ4oooAAwB9aTNFFAgBpwBoooGB6Yojd4iWjYo3XI4xRRSaTVmI3NN8V39qvl3EvnKOhfr+ddPYeKLC94kQxn1JGKKK4MTRjFOSNoSb0Nby45RujKsPUHNIY5F7cUUVwM0Gkjv1pwkccAKw9KKKAK1xp9leLiaBVPqDgisS58MTAF7aYSD0IAP86KKAKCTanpLkEzIB1BHymtSDxMjcXFuAf7yHn+VFFCA2LXUba6A8m43Mf4D1q4JccEEfWiimASpBOMSJHJnqSMmsLU/CVlfjdE5t5exU5H5UUVcZOLuiXqc7e+Dr+1jLxt9oPonWuecSRMUljZCPUYoor0MLVlO/MZTSWwb1PfmgYNFFdhmf//Z" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAQ4B4ADASIAAhEBAxEB/8QAHAAAAQUBAQEAAAAAAAAAAAAAAgABAwQFBgcI/8QAVhAAAQMCBAQEAwUDCAYIBQALAQACAwQRBRIhMQYTQVEUIjJhQnGBBxUjUpEzYqEIFiRDcrHB0TRTgpLh8BdEVGNzg5PxJTVFVaImNkayVnRkZZTC0v/EABsBAQEBAQEBAQEAAAAAAAAAAAABAgMEBQYH/8QAPxEBAAICAQIDBgQEBQIFBAMAAAERAgMSBAUTITEGQVFhcZEUIoGhMlKx0QcVI1PBQpIWM0Ph8CRigvEXNHL/2gAMAwEAAhEDEQA/APmvg98cFQaWrYY5AbWIXbAWXIVQjoeKIHywuDZDbXS1wuykC/Zv1CB8TXdLqB0YAtkH6K07dCWgrUJMqUkbjtdRAEOV0tso3RXRAxu1CmbsoMuUqdguAhRwOgTt2RNRAdApZYXPY31myrOxLDmGz6q3+yp5ImSHzrHxerqB/QMNpS97zYvI0CWW228maPPDIHtPZQRxPjd5I1zubE+HmtZHGZpSLuts9NhHGs9ZPUGsywxRtNgNdUsdLXtnlpXQ1EALDuXbBcLxxEyTD208dIZHQNzh7Xf3LG4r4kx3E2mNs8sMXQNNswXHVUuIRwh5xCUk7sJP990sdBTUkv3Y19dK7ldAdbLNFNLhjpKqGl51HLfVBg7sfxWSLC4wcr7nUbBdTi+GVVEyHB84d4kgAjqfksU0y+D6J5nNXVSSRw3vC8nRp7KzU0FdjOKTRmazGG1xsV1NPwHV0mFNwt1pmnzlxdbf2WNxFwni2EUBnw/EnysZe8biLgeylsuXoKbDncSswrGamOmijksXv2Om3svTcY4e4IlwN8FDQ0cz92OgkvJdea4RgNDxFHLPV4lK6uAP4ZOpPcKzQ0T8CqsmLSSUr/6uRp0clNHNPWYkZMOfJPGxxtci4CwcQjpsEqJKettWuAs3O4/h/JdDiVXh0DJMQoMYqOfp5Q0kLhcRnFdUmoa7Pn1J7lOLYdXNzSSE20F+yePmyWgjeXA/D3UFz6SpY3cscwEhw2IQSvpX02R7xYP0+qd0E+drGPijc/YnZRyzyVUgkmfcgaIZvxLG9rdyoLEWG1EFU+eojbkY0+ZrrhXaHHImPFJVNPK6PGpcVmQTVdMHOjqnjOLHVQwiLO3ntJZ1sLoOodR4nXxXxA3pwCYSdgFnYJiNRg+Munp5srbGMvtoQgxIMgazweIyywkXynoo4aqOnonmO2Z1tCN0oaUZwt2IOq8RrRMyW/NyHW6VJRcOy474czyOo5G+UnQArd4W4IgxGvw+qrqiKajq43kgGxafdc7juGQ4ZxXNhkDyWRyhgunBz4tj+dNJhmFS4EaLnNilIidmBH/FZk2JSYjyC+mjhLNIi34vYrq8AxPhR9O6CrwJs2MUoLJCyPSQd7bLjOJKunxLEnTYXSGkhH9WWWLXpaW2sOi4jjY+qpJ3mKI60xkuLd7Lr8FxCrx7Bo8DELpOQeZIXabdCvMYayemj/AkLX9XDqVLBjWMhkjIcRmaZDc26lLae3/e8OA0Apm4M2ibKRGJWOFi8qTDuD8Lfgvhqqjgnmebl3S68awfF6WnrWnFH1IO72uJcCe5C2MP4hxl2LvrsMqqqmpZG7Enlk+3RdKabH2hUuF8P0zMEwuCSkfK4GdtrBw73/VdlgOB0uDcMw0lBPmFTkfJIOpO9l5LjOPVdbxJT1FVVGrZTSh93/D3/wAF6jg+EU9EyXFq7FQ+jYRUQR3sIyfqlDTj4YwamgfLSUjopZBq9x69ysGu4J4kxJs0f33DBSSN8zWi4KbEeJmY7Tl+F43BRvjdqNxIel/4rKwzGeKqaoMeKTZKaUWkA2J7pTLnoOH207XMxGiY94cQC1/n02KCpwJlU7xDq0Br9s6bG8QjFVJHHzJZjNv2Co4jhmLxUZqJqWctHQbBcKbS07+dVyYVU1RnifYRvAvYLIxjC2UVU6Fkgc326K1QTQwlj4SY5RuCOitymqmq4DPFFkebZh39+ytjmqB8ENfHHIbxk2Psq1bA6CrljeNnLpKilYOInPfHZn5lWloJq2qc8UskjSLh4GhTkI8IjpcrTJGyQFt3NO7ff3Vx0Mf49LH+K2Buh62SxDCGYUKXEKGaRpkFnN6ghNFN4a7iwSGoj1Ye/wA05B4Mb/8AhEdE1rubTmwI1V6eubXV8cb2GRj4rAjoR1WdhtK+MTySeVgfq7oFpYXRiPFG56trvJnFhsVLFaB82GVFVTTnLeO1u46FdBg5w6lwgNkqM/ixmd81Xx+j59IJq2EOmYMkcrNvkVo8MYTG+mdIxkeSWAxzF2wf0sFnGE5t3BqD/wDR81U5JlkacpJ7bf4rSwjOcOheX3DxcKrhcrpMJbhshtJT/hH3t1WlBGIYWQM9LBYL1UyMCyRHVPoAhW6CSSSWWYmiTO2TpI0jdsmylHlKFyMhvc2StqjTEXQs2UpxsisU+ULTQC1IBGRdI7IAc7KLp6edjyDso5RdPDC/eyrDVDWujBCryN18qkiJDLJEWUbU5GlQEWVyRqgc3VZRBlKYtU2UpsnssqiEacR6qYM0SLEEeT2QmMdlYylCWK0Kr290BarTo1G+PVQV8re6AjoVZyaIHR2QVyOhQ5Spi1CWIIS1NlKlLUkEWUpiOhUuUJEXQREXQlqmLUGUoBylDlCkylNl/KgDLqmylSZXd02UKNI0kdmpsoUpKoKVtEWUJra2RfMwHQJ8pThuqLKVFRpI02UIG6fVMpLd03T6oGbsnSTN2QOndullKZALd07fUnSQEkmbsnQJGgRoEkN0kkBN2Tj8qZuyJu6B0/X6pkkWBom7IUTdkQ6du6FuydA7dk6ZuydQGnzFMnbuqtiTt2TJ27Ih+n0RN2UaNu6kgkkklJigaJCiWlk7d07dkKJuyIQcjbsgbsjbsgMHVSN3UaJuyCQbow5RZiiBsgMGyTd0yZuynoJm7KZnoVdinGye8OT1KZ2yTtkKgSEm6JCrIMGyduyFPmKqiuO6cGyDMU4N1hEhPUoXbpkJN1sJ26duyDMU+dSxLmCWYIMwRZiqGSQ5ilmKgcmyB26Ym6Z2yBnboXbJO2QqhJJJIGdumTu3Qt2QINSyhGkBqgYNTqTKUBGqAEk7tkyyBduo0ZPUpnbLQjTu3SyhJ26MzJkCd26ZGj5imTZQnRgWYJ0CbMEEiOPYqJu6ONyCRCXI76WQO9KAW7JZgmuUkBJ8xUeYJ0Du2SbsmSQEizBBmCWYIGJukkkiz5EmzBA7ZOiWSHMUsxTLYSSSSwtkkkkraHzFIGybp9E2YKh0kk2YJYdJJM3ZID30STZihzBUGmzBMXJswUD3KbMEKSoYmyRNkKSBJJJIwfUlEkN0kCSSRoEkmDkWYoGJ6lIDoEJcmBctNpOLaOapiinhHmjcFuxuzQRnrlQOF04FhZZCJ1TZihd6k6BswSylOkrbNhMYO6ZrWhGlcfE5LCHVDJPyzZVquZ7W+RxHyUVRXUopw2KdpkOtr6qNLTpHs3Cja3NLzFUxCWGowyN5m5cgdqbqgcRfG6OAzsk8w16olDxOaafG4aTPaM+RUK7h1mHYmaVgy0tQ3XTfurmNTsgxGjq36RvIGfoF1b6aCqgHNj5jbeU32KFOXxPAqRtLR0sEDC+Mghx1JWDx9wfQ1EtJPByqc2HOB0Fl3RhbLVt/ELcgtp1XE/afJaWgEzw6CSTI6x3CKz8PgfhVYyPDH+Nc9uzTq35rOBxKXi2lhqqd7pRMwuYDpZd/gmA4dgFJLJhvndI3PzHDUf8AN1zOAUk0WPVuN1lPI99yYcov5z0Qenzws5eZ8hjaAASF5jxji3EGIyOwvB6SOOOO1nuOr9F1mFvxOWhqmVmZtznbcdOy5zjLAcVldBiPjhG2Q8ssb0GmyzMJLyiWTEKKsL3PMVRGSC4dFoTzYljELPG4q2Wxv51r4lwq8wVU8OZjINufoZPkVw8fLe/JMC09iFFbp4lqMIpzhLGQVAAI5lu653O1ziQrEjKQegaqN4c+1mjTsFm1iUIa4nKUT45Cckeo7Isj8jngXsUUTy0iQbrTUyhLHwyNEoLbo6iEwSZLgi2hHVNOamply2MmQdAo4bSSsjlkMbXuALh091BYoonV0jaaTVnxfJXOXHhVWckAmhGxI1UFZRHDJcjJ2y5xdrwNVXDnSNPOkfn/ADBBt1uD0tVF4+lxEZB6o7XLQsGSONj8nMbJbsnginkk5bHuyn1W7KXkQ9LlBLT11bS28PVyRtHw30/RHDzsTxSFlTNrLIA59lC2mmkNmC/1UooHixmn5Q73WrR1srKfhWqq4IBzwWh7ZybvCm4UoMGxmlxLF+IWOc6I+oG1h/yQuOdC9lv6W+Qe5upoHVjI3ikMg5gs7L2XKZZpqcQYbhUVYDwzWvqKOQXcD0KoYdRieqfBHII5mNLwD1XccLcHHEeFZZJyYJZHExl2x1VOiwbBMTojSYgH0uKU0jBI8GwkF7H+9WJapxslHVV0oosv9Ke7IG+69coA+n+zeuwvEaQQVNFEYy0jruCt6n+yrhjBg3GIjKJYxzLk31Xm/HXEs2P1v3bhEM77Eh8bbESW2v26rviy2fsg4VhxqnxE4vhj3MqYbQTubpbrb+CsYngfDvD2EVNBj+IP8UARHG2TUgbEBZvBNezCoaWjxXiGspIpC8mEOsIz0HtdbGPYVwrhZkx6nxU1k0jTyo5ZOYy4CkwtvLoYcKnopJ5qrlzDZp0JCYYjV1sTIZsUmEbdtdgrtHjOAVWKVFVjGDiJk8fla3YP7rHqIGXzR6MebN+Sw06eqgpYsPp6+gqmGriFy7uFPh/EnEGJx+HZBEOUQbgb+y5upraoU8dJLl8gBDgLXWnHLXUWFxV0FXTFhHKkLDrbuQpIWKV8kjpiaSASvNy9p/wWaypkDI2vmeNb6JVdOXS1ElPU8wRi92bFUoJec0HmF/zXKYRu4aX4g+WSpqS0tNrgbqrPUGkq3ww1UrRGbaG10+FVgglZDIwZHu1IVfEaaq++nsbG6RvQgJiLkR8refIfxSTlPTsgMWZvOb0JCarqGTGPI23L0KNrJHMdGH5dLtB2J7XVVNFiDPDzYXBD/pAGZzxqR2UcIfhtQYHmzsh/RPh8fPlbTVWWPUjMBqENbU0hJkhn5kjJALkINySWarw+B5qoy14AI9+y1cInnpsG5dR5Mk2S+5suNil5lOIMv7N2e66XnVFRJDVvjJinABDdg9dMP4nPJ19O+MBjxa5G46q7HPc2uuaw+pcYsztDtbstikma71L6M4MRk1MwTA2UTZBbdEDfZceMrY8yIHsosyNrm28xWJiW0iSSSoFAR0KkDdUxZYrDITumREapspWmjt2TpJKegfKU4ZdCjDlQwhBOuynaxrBYKLME+dVE2YpO3Qg6J1FA7ZRSDVSu2QEXWRGlk9lJy9MyQGqyGDLBLKVKGNTFmi0Iba2TKQi6YtQRkXQFvspsibKVkVy2yjcOitOChLUEBagLVYLFGWoIU2UKTImyhGvNDlCWUqRNlCjKMiySMtSLVQFtE2UIsoTEWWQxBQFqlI0Q5Qq0jylMpcoTZSgjTZQpcpQ5QgDKE6LKEg1ZU2UpZSpC1NlKCFNlCNNlCBiLJW1sknylAyLr9EKfKEU6Hr9USSIQanDUg1EgSbp9EnbJ0CSSSQF5eqJu6ZO3dA6cGyFuyJu6BkaBGgNJJJFgkbd0Ld06INO3dRo0CRoETdlgGkmbsnQGjQJ+v1Wwbd0m7JkkWPMeYpk7d07dkQ6du6ZJFgYPUJ27oQbpd1JRIDZO1CnaqUkDkTdkAGtlKGmyA2mxUw2ULW2U7DogSfKU+UJ0EbtlGpHbIUAp8xUaSwJm7olG3ZOtr6kTohSQKeiDBCe4QN3Rt3VDp8xTJIFcISepSJ6lCXKB+n0QoswSyhQRJE9SiLU2Uqhkw3KdJUJNlCdIDVQJPtsUTtUgDZA2dyEl11IWKMjoUAlyAnqUTtkKBnbIHbInboFQkk+UJ0SwkdCgUjt0OYIpnboMxTlybMUZLMUySSISkBso0+YoqQG6J2yAGyfMEQ6bMEGYJZghYm7p8wQN2SzBATt0OYJsxTItJMxT5go8xR5ih6FmKHMEswQohJJE9SkgbMEyfMUOYK0HRIUlZiw2YJZgmzFLMU8gSbME2YplQWYJZghSQSZilmKHMExN1KCzFMmdsmduqH83ZNmKHMEsxQFm1SzFMkpQSd26Td0yrBIgLJZQibugZO7dEhbugZGgSQJPmKZJAk2gCbMUtSUG+SnuQhcmWmoI7p/L2Q5glmsVlRp7+wQE3RaAJYWhCimLI2Z3/RSnQj32UVSLsAdtdBWgr6aWpfRzN5bwL3ebArmuJPAcozxTOjkYb6fEuhfg7a2ujnbJk5fS9rrheNamQY2cNgnEGU3BOxOmiCP7zcDEJpJGiXbPtdUsexipwpzGU78srvqCuYx/FqqqkFJIR+EbhzRYrKbUV1VO2Dnucb3GZc23Wz8a46ylNBWwRSNeAWuI1B9l03DfG9a2ijgnnjjsLAv1BXnWI1FVaKOrfGTFtlVOTEZy0xwyWHyQe7w4oaq0lLJFKRqWxnUBZ32ixQ4hwq3EXxmN9ObtuuN4REE+GOmbO6Goj+IOsuwxaownGMAOBvxINfMbZjqR9VqJYY3DmLS4thDIIscbBUR+TkyfErNdxpW4EW4Vj9K2MhwfDVU+rHEf+6mw77McHwfD56muqpKgsiJa8GwBHyXmEtPi2OVcjKaOSo5F7NvcgKj2Ucd0FXRPgp45JZZI7tfcAX/xXOYticOLUVDIMRmcyI5J2MBuPe689pa04FiAFRQO8oIkZJou14Tx6pp3yUtJg7W00ruYJJdweqrC/wAZYbg0+HUUOGcTiV773jlluPqOi854jcwysMhj57GZDk2K0+O8P8XxE/wdKGO3lLdnjuuVq2Pa8NfuNyszKpvDvdy8jw50htYKzU4LjdBFzqvDZYmbXPdZ8wcyIPhnkLj8N1tHHMWqMNbRz1sszZBy+RINPndYWGfA18jHGxBAudOihkd5HOboQU0ktZBeGQFg91G8m1s260qxg1TPDWGRj7ODTbTRR4hKyoqc7ALkebKLC6gU+HyRQ1QNQwlhBBt0UbRwhjXXetHAqehrK+aGrlHKEWcA6arOqOXzi6KTMzppZT0cJmzhjmZmC9idT8kFiimENbMIbuaAWaDUhW6+qwmanZHAx0T49w7d31UlPh3inuZh8fhaxjS9/N0BHVSU3DNRI81eMsMcNr8wHQhBkxCUls1K8NHQkp5zcGOpex3UEdCnLY4H8gTCSnGzxuUxhwkuLxUSNv0WQ1C6kZJeqiMgIIBbuD3Wv4DFKJzRRuDvEm0bxvc91klmHRNL4pnOI7ruqCenxHhfmP5cdVBGXg99dCpImxHiTiLgqKkopHxTAt2IWZjHFvDWI/0uOnna98dnWisAeqvRT0WOcPwYo+E+OpGvZd2xPVcFLMwtyZteo6Bbhh0EnF/EbaD7uo+J6uWllZbkPtdo7XV3hbE8LwHDap9UJYa8ggF8dzf2XG3ys5gHmBFlqxY5LLNA+veyaDN+I0iy6ch6ScKp8Z+zZteYjLNI8vme313vZecV2DspKGKSaongMgJbmGl+vyXXYTx5JhGKVGG4fS83C6uxAOnIJ0JHsh4tocIpMPNfSVX3g6NwkcS7R19PSVKsedQR5wCXFaUbKFsJjlkeZibgjZKuqqGqewUNJyBl1GwBQ0GFVdW8mmp+dyPO6zrWCxMNtOiwOlraarfPUGKSnGce7FUwOs5dPPSyQxSRVB/Ea4dVP4+srp3UkL44uaeWL6ZT1ujbgsmGOEE8jJOY6zXN2uuYr8OubE7EI+Yxo5Vm37i9lTpML50L6gScmQO0N/L+igraWXD6yWlnFng3t7Fa1O2CbAnsguHvGdzSbgPCDPDK6CcsZC9z4yC0t2JWpWV9bh74a18PIqXtuRbSUKlhs87sUo4d2SDJKO7Oq0sbp5a0CHDpC+lpm6g9Cs+iKFVBN4aKuhY+TxALyANWnsnqw1kMMrzYkAK5BiMQoYoMkgngOjbbqzSYfBjMDhnyOZflscOvzXRVTB6aSqhqJpGgtDTywDrcd1dwvCYMRomSVUeTISCRufmspnOw2arpRJa7Cy9t1oYfUPjw6rhdJ5uVppuulMBoaeN7JGCQSZZCPLuumZTTswiGkYfMw3Kq8PcLM5EFXWtkc+UXc12gt0suh8OIWCNmew2uV10YRjNy5zkoQx8qM/vElalPSV3La9kejhe6rNp3yvDA1dXQsHJa0jYWXtymnNhyCrpzlkB+iUWIEaFy6xkUbh5mA/NZNXw7RyvuxuVcuTfFl+PP52qSKuHV6oYjhVVhzg64dGdrKtA6SV2VguuvHFh0kVQxw1cp2G6wKOSY1TY3/VdKacta23ULhsxdImyYLuTua0ImZWnzJPIK5tygI6FA7ZSu3UUgf0CqGbujI0UcWc+sWUoHQJMEBT5ipE1gs00B2yQOqRF0hZurlplMBZOTdQGrYX2YjL9NlGkiBJJATdk6jDtUYOqhKVjU8jdE7CLJ3m4WRC7dDlCIjVMgYtQkdCpMoT5W90EGUoHNVktTZWrIpPZqoiLK1I3KCq7xdakREWTKTKUBFllr0AgRpsoRkKSSSzQBJGmdstAU2UI8oTFqjQcpTI8pQ5QsqZJElp7KygUwv1TndPlKimTZQiyhLKUDICOhUmUIcuqCMhyWUoyLJiOhQAnylPlCWmyAUk+Up8oQOkB0CSQHQIGyhOnylIC6BkR3SyhLKEgk6du6ZJAQ2SSSQO3dP0+iFE3ZAeYJ0CIG6Am7p0zd0m7IHRodPqnF7aoGbupUDETdlIBom7IU4NlVEmDk6ZuyIIOSahBunWC0qSZuyJu62CTt3TJIHzFO3ZCitpZZ9QYPUImDVCB0CsRtGi0Cjb3UwaLIUTdkDZQjDUQFyitYIBbuk7dMkgYtQqR3pUaLAUCNJYQzdk6SSBHdRHdSlqHKtSI1JmKfKEzt1kEnzFMh1JWwySbQBLQhSPIOibshRN2SQ2UoQ3VHlCdA2UJi1Eoy7VSAzt0humTt3V94kA1RoUidEiQiepUTinLlGXIBduhJsk4oS5AJN07d0ySoNu6WYocxTE6IxRE9SgRZgoyboE7dMkhRYmju3SbsmSRBJIUSBJJswSzFFOkmzFLMUQswTk2QlySAk2YJZihLkDg6p8wUaNaoEmzFOhWVmbJJJAiCLk2YpE3TIpswSzBMXJlaQ+YpZimSWgWYJdPogzBOgMHqE5N0ySzYRPUpswTFyHMFQWYpi5CnduqwZJJJASbKEQanyhRszd07dk6SR5hJJm7J1QWUIUkkYOTomJ6lNmCbMUD5gmzFLMUytNkkkkow3kCNAtNQSSSSzEqcGyaWXlsLy6wCbMEnND25SEmBmYmZ8TontgMkU0REkfzTYPjn3pF4SqaY62LSRpUHE2K1GBwReDYMz9yfoudbxjBhjn1eI0sTKoizWj4v4IO0xCtnwiJtUyl5zb2IXAcZS0nFGL0sHhG0ktrkl1ydtlBiPFOMcS4XPPS4lTwsj/6qfWR81ws1bzLGSSXONnA7IwscV4KMDxLkRve5pF7u3VbFMInwmngrjKC2UXBA2T1GMzYiWHEpDPyxYEixsnrcTjraUUZJEbDcAjZcpbiWdVSCV2cTulPUlRQvfHIrdBh9RWPMcTGkjubKfFcGqsMpxPPlsTbQqtoqeubDdnPcCNiFd/FqKNtQyaV5fvboswYfLJA2pYLh5srsTpqGJr4Kk36tOoWRdZxNxHHSnCPGSOhd0c4myoU8mIYfUtqoankvGxut6mrMOlia9jYXSndsu6xMRw+GnqA7xHPEtrj8pTkNet4ofVwvjqmQ1ckmUtIZ5wQsOTHuIpoTTMmlDP3WgFVYXR0dW15FmjdXJMSqDK2eh9NrHXdbiWJxaOH8WS0NEKSrw4TSgECSTey5yYve9xedzdaUbPGVPLquZFOdidAs+qD4pjC8WISWqPRUk9VNaBocQL2JslNWZX8uR1nBBHLLC7PC7K7uulp8TwJuGF1Xw5zc4tJK2UF9+6FOXlL535pJCfmiMYDdwVC50ckrpIQWx38gPZPnN7BYUnM18qkbExw1kDFC6Us9TlIIqqWndVNhc+OM+YgbIBdlCkjpZzB46AtGTcHcKUUDzRCrEsZv8F9QoqeolpZWSB/kzjOOhC0NeOsFbS+Fr4JvFyAGN/puPZZ5mrIGmkkqKnJ1YXGy7/7lpYsVgkNNJLRztvFJf9kT/grPD2E1clRikPgfFwQB74ZPzey258nmvnGr2EM7lEcnwlSYnUTTVM9NJGGMEhAjHwjsqjtlydF5jaCd5vNkaN2W/wAVbnlfCG0jMwjtcEC+RZMeXOA85Aeq0Yzy2Acxzh3KA8NxOeip5MPqKiSJjzdr2jY9lC2D73xOOkgfG3OLB1t7IamaN0hF/J2Xq/2UcD4aMHkx3H4Yi6WS8LydYwP+bKsPL8c4arsEbDJO+OSKo0jc3v8A8lZobZxjmbdvZet8b8P4ljFcIKSemmpKc8yNsR1B0XEcQQUuJVgZhdPrA0RyaWuVoZMPEGLUxMEFRliAAtbWw9/qgxCup6yo58DZG5xdzXHRWo8HZDQT1VUMk2XQb6qvUQYdHQ0tVTzyCpf5JxfYoIImPk2C0cP8RSRTZHuBqG5AAeqz2yz0/wCNAyS4/dWzkigw7nxvLrgEgnqsNgmdBWyxRwWZMyOxIHUKGqp+Zhzqzxcsrg7IQ46XUc5AlbJC/K9h10U4w+tkpzXGMiF/Z2hKyNzhquoanCq04yIXSMjyNdK25I7LlaKfk54g6zS82VsTmOGNjNnkhw7hFUYDVwUsOI0tO6SKTtugnoKRgJrb5ZKc+aMDp3C1MGgjoKStn8XFP4uIkBh1A9wucZVVULpZPM1/osRoQrMdI+meJKWqjPcKShYdMyLEhIyPK21gL7LpOHm1v3c6sbTmWF0xGcHVi5yGLPKGN/aP2K6jBOJMOocDpcNrqGSbkSvLix2oB9rKxItca8M0sIpa2idkc/ySG19Dayqv4PnosLfUTfiVVPbnxk2sD1C7XA8QwTiGjmghaJWejzDUDorsmCzzytbI1r6YsDJBfRwA6rvEJxY/CGP1GIu+7q+hZHyoQ+J+1xtstqtw1jyZGMVDDaDwHE0lBGPwYoLxDqAel/oulMdtFvGeLM4sGnoWROM1rO2WjRGQttKwDsrBgZe9k4Y1vpW8804p47ZdEiy6BhspBssWUq11NFNTubIy6yaTDIInuyM3K3KkWhcs6kfd9l0xkoMODOinFS9nlCtSyi+VXr5mBVJKdmrnpM2kRSpLVQMda6IuRimw1n4krE0ctLJJkZDYd+6gYCyaOUZywN/UK+9rIR5xoqMldRSyZYI3X7qkCkyBh01VZ173spZQ7JcJ4pIy3XoFoRgl1sqdwc0XIQxvs7MArsQzRnPssqoCRpFw5Tx2y66ppomZvKLIASDZBD4SOOQyC9yVIpHbIUlYIOTlyDr9UXT6KKEHqjDtFC57QdVXFWG/ErwZtoskyqwC1+yzI58x3V2F+UZknFbT6WsVER0KYyG6bPrZRSvZLOmuExFlgHmCYuQ30shv7FAEt+iru2VqXLbXsq7gghLUrXRkaprWWSUZCawKNzdE2XuhIMpQZSpEJagBORZIiyZF9CST5SmRo3X6IUaBYCSRpun0QM3dMnbuiQC7dJ26JJACfy90SfKUEZHQoS1S5EJagjyhN5e6kTZSgjSUmUpZSgjSRp8pQB0+qduyLKUwagfKUspSyhOgFEBZJJA2UpgLok2UIDbsnTN2ToSSLQhCkgNO3dMkgJJJP1+qA+n1TJIm7IDSTZinQPmKfMEKSLEDSSSREjd0m7JkkBI0Ld0SEDYNESQ2RNaUDxhTN0Qxt0UmV3dAWhCljGiiBN0TXFqliyA2yRagBuizFUBlKZFmCFYCOyF26JC7dAyFEmdsgZJJJAkkkkAu3TJzumQSelqjRHZCtrB8xTJ3bpkQLd0bd0LdkbdlgE3ZJ2ydC86LYZ2yFMTqmLkDJwbJkkEmYoS9BmCYuUCL1GTZImyB2yBy5CndumVCTZghSTkgswTO3TIEU+YpibJJIzPkbMEyd26DMUaMnzFLMUyMxAswToE+YoUfME6HMUsxQo+YJZghSRBoEkkCSUmUpZSgZJJEgFJN1+iFAaAnqUiepQIvobMEswTJKofMUsxTJLQJJCkgJPmKHMEsxUCzFLMUOYJsxVB5imTZgmzFQEkmbsnVD5SnDUKNu6AkztkswQlyAswSzBCXJByyyfMU+ZCTZMNlqWhZilmKAGyRN0YPmCZJPmKBZgmT5imRskeYoEQajDdzBK47qJJWlGmzBLp9EGYKtDbsnQghONTogzsew8V1OwuGZ8fTusXizhrDp+H2VTqbPLGAQb7Lp6/FMNwqISYhO2Ltfqsr7+ZicB8NQCWmf8YP+CkjgZOFBPhstVRYbOHBuhYbn9Fy9DwtVV2F1WJMqmtdSXvDfUr3LC4hSUr5ql4ja++rjYLl6tvDdPR1dFBjEIdUm9wbLmjzAcLYo7A3Y61jRCOhOqjw3CJq2mdYHOBcWXpVLQ0tPwnU4UyuhmsCWkdLqxwZheGz8OOgENnsB66goQ8igfMZTBDzM/XLurAi50vKxWolys2udirlIaXCeI3vq4C/lSEWbuugxPAMGr8MlxWB5ifa4tv9UVgxYcylpWvrbT0Y1D4z6T3Kx3mNtQY4SSzpcrosOwPHqqgdWYPSzOYBd/Vrh2WLTxMZWBz4zDJGRdruhRm1ieWjocrDScxxFw8G9j7LNFU9093m67zGsJwbiDBYpmUww+si1JHpcDsf4Lmsa4dgwqkZPBM5z72JcNCsNRLAqXZnlylgnIYImPa13RztleGLmDDTRMgj0WZHS52Z5NG9wtQ206uqMUbHskjknDgqFbPUTymapDMx7BRNZCzSx+pVpvhZIHRvZZ3R19lRR5r/AIShZFJUTiNjwCepNlI+NgByO1UdnfEgkqKOSjcGSPab/lN08ELJ4Hzsf6Da1lLhlZS0FVzquiFQy1rHcfJX/wACOhlrYDAeaSTABrGsDMgkIeWMjZI54sAV0dBVU2G4Y8SPY6Z7fMx2xCyOGKLC8RxdtJi1UYYT6SDbz9FaxrDGUs81LSUr5eRJ5pM97BBmUFP4ycQcwtc8kgDb9FLVYTPDA+oa+N7YzY6rRwjDIIcQfS4pC8SmO8LmHqo/uziCs5rxTzx0Ydd0swsGhbYbvB2Lfdz24disFWx8vkjldJ+Fbou24ZxuvpK6pwcTRmgiDyHMGoNu/VcBScWVtNhhwWaClqmsaQ1xaDoiouKG/dAwmCkMNS/yc0mzD7nRW2ac/io5eKVjZpM7jM+ziNSLqg9x0t3su7qODcGoeHzjlbXmpqgDmYD16FcQ+JZmHWGvjvCWLYDHSVVU6OWGfzsdGb273UeK1dJLNG+hku1zfM23pK6DhXiulbRSYHxLJzKJ0eWN79eWOy5SshpaeungoZxPEJDypAPUFKJW6OGSaqhkjpJapkbg+VkbbksG6+h8NiwKfAIMSwChjdFLbc2DT+8vOfsbwuTDOMad+JVIgfPBIYo7+o22K6zjnhWoosXiODF1JT4ibObGbR8wdxdQc/xtVVuAVB+6sEhZDPYeLpzuOosNlytC7A4agTx1xa+SMslEmvn7rt+OqWXh7hmGioKGeWW3MkqejT10+q8qqqKaONjpKuPJK3OHN1stCXEcLxSme01Ut/EO8rb3uFUmoYPBvDJGxTRG9j1RivrJbR1dSZ2QG8ZHwqnibmzvFUzXSzkF9uMiqwqTxNRepDbAW2VfD5nys5EjrtKzWy3YWd0cRez0GykwNfP4p8r+WLg2JCnosTkYBRP1Z1aqTZGBjmB+rxY6dVAwPilD2Ose6g2Y/Dz1ETo6d/4Ts5BNtF0OLcTQUT4psOjkDD5JIul1xUs81LNFPzRJ5vSN1crmyVLDNna6/UFAeLHxVQaiOAMLwC5jdgEGGU0NRLM6SQta5tmEdSrOCzxMD/EvjMxjLGl2wCr4RVwUtU+CrYDSyfhuvuPcLMotnDZo4g3mRjUDOToFt0NBS0OOYaXMysluyQk3abhZMAy4gcPknzRH9m5x9Q6LWqMOqp6Wlw6E5TE/OJ+oHZYFuCF3CuN1FDNTvmpcR/Eg5ZN22K9FwCtnrcOikqmODze5PVcTw1LX4viNDHWwXq8HDy4uPknjPb3XU1PjaCsqqGjeWtsKmEA7jqxd8Bt8iPxXiyPxMvLB9lI7dKOVlTTsqoR5ZBe3ZMuhMhcLhLL3RJAIzMhDFK1uiYNUjQgRizNykLPNGynuWd1qhRTRhzVYkVGVDALKCtq2CIBp3TTQvjuqk4a6LKQCumDCB1TTekyXPZEC9hu0W7eyoVTXx3ezcK1DJUy4Xnd+0vZdqZXBKC38apDn9blQ0r71QZlWJFBPLln2d191rYTNI6V2bporQ1qhrwPIPmsaeR4n5d7LdvfylQTYM2SobUZrZei5Q0r0YLiPKrw0BaibCyIZQLIXkNOqysIJmgElVHP8ymqpgLqqDdbRKJe6cPB2CpulcPSlTz+YgpxLXEnO0Q5imcdEpbVZ/WSqD3Fjj81cm1cqs7LhdIYlPBJYggq4yqy+orHjcQQ1WRJ3CswzEtYTNeE4kWcyYtIBKtMfdq5TDpErTTmRZfdV2SW+JGJL9VzptIkhLkBKoGRyHNdM952UQeQ7VShIkiBPVCkwBAumyI0RGtllYREXUZFlZLQoyOhWVQkXTZSpSxBlKAE2UIsoSyhQRpJ3bpk9VJM7ZCjWQJ3TJ+v1RIH6fVMm6fRE30oCSTN2ToEmyhOiDUETgfypZFKmsSUEOUpW0upuWllKCBPlKlyt7ocoQBld3SyKTLolkQBkcllCPKE+XMgiylOBZHyz3SLCEDZShyhSW0SQR5QllCktomyhAKSSSA0kkkDt2RN3TJIDSQu3RICbsibugbujbuhJebsn+iSSlBKRu6AC6JUGnbuhbsibugcCyNiBo1U0YQEGqWPcoALI2DZQWmgWUwboo4xmsp9AFRWc3KU4aiIuhWAYOiSjzBEHIDzFLMUOYJ0DO2TZiiUbtkDpnbJk7tkDpJJswWwswToMwSzBCjqPME2YpZisBy5MTdM5wQl6Ak2YIcwTqyJVI3dRN2R5gtA0EhTF3VA4lSwi5AkmzBU9CzBK/wCVCkoHzFAXJFyF26BE6pO3TJKhJJIev1U9AnbpN3TJKh8xUZPUp3bplAkibIUlQiUCd26HKESTpJsoTolEkkkjRJJJIHzFMnylEjIW7qQCyANRjZD1JIjoUkR2CCNJO7dA7dEIm6ZJRu2QEXJkzt0yBJJswTZitQF1+qfME2YpZigWYpZimSVBZglmCdApASSbMEswVDpJJIDSQgp8wQOTdPmCZJGD5glmCZJAktQUkkD5ihzBJ2yFWIbGgSRpQSSawT29wozZIhsmsE+ysLAS1EBZJPlKitdJNmKWYq0HQFyJ2yB26QyQNkUb/OLqIlJpVVwf2yWmZRRsN3Z9vooeCcGxupgySV3hGMFwWm9wuk4qwiLF30z5n2Idva61sMpKfDabkwMsNERxPHGF4lTMjjZjE9Vfz5QbHormDUuDfdlJHidFFJM/yFpC1qljH1tQ57AfJ1CjwfDIZakuleHcsXFu6zMCpgfC+EPxettB+GNYxfYrUo6uDDTUQmARNZq3KdHBXMGwyWjnmnmfcyrO4ppBVVNDSyFwZKbOymxspTTlaSnouIeIvverbDE2CQgxgWLl0nG2L4dR4d4Knw+KV9SLCzbW2XNYngj8D4iZS4eC9j/Tc6qCSOuo+LoY8WkMcQIMfM2tZTiKGA8Y4rw0JsNpKW8TySWuHp+Siwaig4ixeb7yktLJ57EWBHsp6/Cn4lxXLBQ1pgheLhw62HZWqKr+5cbEeM0xkhH7Ooc0WB+ilJip4tg9RRTCljxHm0uYMAB1j+a6+Pg2lxHBZcEqJpJnRx8yJzt2v9lwONU7/viprYJiKWV2eJ7dnLopON3tLX0D5aatyhhc6MuDgO6Nw56LhafGQcL5FNRVNN6pZNyVz+MYdW4HN4CoqYpLi94zcLvaHCZ3VL8RxmSGZj7vzB2xXn+OGOTFZzBI57M2l+iMqsdsp7hQue6Q+X9E+e3lUkUL5b8sXsLlZaxhC0PaPOjhhnq5hBTx8x52F7JQxmatjgJ0ebFakuHfdmIsfSVzTpfXootKdZheI4fFHNV0/LbJtrdRUsM80loITI7qG72VmufX1VYyjq61rg+QWcToFeosMmwvHaQTYlDBFIchqA7yAHugr4HQYdiWKeExSR0ZeLgjfP0C3q2ixbheSSNhD2zx5Q4NuHDoFd474Nw7DcJp8bw3GIZnggFjHi8nuCFl1WGcQ4PQU1dX1TJYXjRjXXICrIuAqhkHF2HzVFXy2hx9eoOmxXoGLuhxXGKnB/vEwszWMA2cFwlNT4RjGIvEhEUnKD43A2IITYVFimKYrUVz8ZtWU3kbIRcmysIqY3hdHR4rJHg/MDYiWFzhpfsoaZ9RUMlop8oI84c0bFWxjmIYjXSYfXvAu6/PaNj3QYVR4jX1VZBhT4nuYz8Rr9C4eyuTbJMsmf8AHqnuadD5kp6Wzm8t4LHi7T3UVRDJDM+CZlsh1CfmgZWSNJbfp0WCEcsZicY5BY9irGHYO/EubyKoQvjAI90E3hJHX84U2Hy01LK41APIkYWEtKC4MXxU+HrTXEVFH5GOaLEWXrnD/EeOcfcMhlc+Ln0coInAsZSO/YryB1LSR07aqlm5jr2Md76KxhlXW4Zkr8OxR0MpNnMDtLIPZKDB4a0umqK3Ep3MuHc93k06LySrwqmkNbHT1f4dPK8ZXBdFhf2lcS4TSuqxVeKiLheKoj2+RGy5PHOIK7G562ehpOTTTy8x+Uai/T+C0I6KaBmB1DYvwqi/mc7Zw7BZU1N/RzPzmadLoOZ+Fy1BlCB43Oa8Oy3VqLWVrPqVWa3VTjM0hnfdUWaWJ80wjYLkpq2nqKXKZG2DzYK+4CGSN8ADPLqq2Izula5j/VpbRcxnSayhiv0lSyKR1I7QPGUeyz8rr+6unD6iMxzxjmh4vdqCzy2UVQaeP8RrwH+bWxVj7krZmQz04bL4s3DQdQexULZKqRgnmp7FhIsfiC6XgGunqpaike7SMB7TubdlnkM+tpoaeSOCcSS2isQ3Uwn/ACSixivpHx4fVG8V7Cb8wU1ZUSDHX89kcMmYsJvpIzoqUrX1RkpKa8tnXsBsFCmjBjddgOIUtXO8OdA/OQBYlnVd1WcR4bXTUeOw1uSjfHynZdZIj0JXFNE2IkGSnJhiDI5nZLkE9v8AnooqfBvu8VWF4jRSHlG4lbqB2v8A89V1xlzy8nreH0UtJUaYm+oZKLtZ7FaZFl5PhfF+KTPw6YtlkZhkgZJJl2ivaxXp7cYw2q/EZVMZ7ONl1hVhO3ZK1wHtILTsQbpwboDA7Iw1C1HmKSHQkokztlmGkEjQ4WIWZNTOEhI2Wu8KB7W9WroyyjS5/UNUUcPLbky6K46IDSyEtC1aK3KYQW5BYqtDE2CRze+q0S3qqs7MpLxuukSzMJoQScys87Jpdc/PidRG4sYAPkqor6syE8zdXglup5zXelDKLtWCMUmib6roosSllBzOsr4UnNanZdygPkTNnLvNe6Lyu9SVRZENc3RRiK3m7KRrcgypvNdES5nEJibJhoEDzZZiFsD8rlVkYdVO59kBdmBzLrDEqpjAPlKcZx8k/ZGNQAtyyKHztv1CtMJaFVhuHkdFZa7RYmHWEtzZMJHBMZBlUbni/qXOlTRvcet0ea3qVaF7eblU8pu26lAJnB403Q32co84J1SzAvDWGy1QsNdoFI3ZRgZQlntusTCwkDUsvzVZ1S5hy5lJFOHEjN1U8OWkztUBClGo1TZSs0IsoSyhShqcNCyK5b0aonNN1cdHdRuZZZFUsKHKVZc24yqLluRpEkpS1Bk1sshnboW7KQjoUOUqKWUpZSmRoGbsnSSQIHqE+YpkkBZtU7d0waibsgdMRdOkgbKE2Uo8pTEdCgjyhOiyhLKFAGUIwLJ0TdlQsoQkaqTKEnboI8nsmyhSJIIspSylHlKWUoI8nshIF1Pl0UbmoASSI6FJAkSFO3dFnzOjQJIgweoRN2QDcom7ovuEHJ27JkSIdu6duyFE3ZAYFkbEAFkbN0EsYRpo1KGoEGkqVkZ08qULQVaa0dlATWonbIen0RO3WQBN0BcndsgdsgZJJJAQckmbulmCAsxTJJLYFO7dMksBIETtkDtkDE3TE9SkT1KEuQEXIc2qYmybMVsEmzBCkgQPUIm7IUQJWAWYJ8/umzFBmKCQlNmKC5TLUA8xQPKYlyYm6sAcwT50KSB8wTE9SnzFCTZA6SBJSw7t0yEm6ZUJJNmCWYKWHJsmzFJ2yZVgkCIuQoEkR0KJ2yZ26KZNlCdAhMjSQI0QkkkkCSSTg2QEHIgboMwSzBAYPUJJsxSzFAJchSJ6lDmKBO3TFyRchQJJJCXLYTt0yd26jQGmzBFmKWYqBkkCJuyodJNmCWYIBSSSQGknzFDlCMHSTA3Tq22SLQhCkoCzBOmyhOrTJJWKPIllsogMru6fKU6SNmylLKU6S0E1vuiLfdOkN1mmbNlCf6pJ8vutLZZSllKJJFXsxTg3TZSmQEdEzUOZIG6JJ0OUp8wTZiiUCQBws9twnvplaNExbcp2suhSvJRU8z872XJRAUGEsdVyERtA1JKinxihoqjw9Q/K5c9xDiNM+cVFM8TNI5dgf8OqytNCp+0bAaZ+Vzpcv5g26oVX2k4JVztNBBJUOjtZuXf/ACXGT4nNQyy0GIUAInF2kdlizufhGfkkC5uLG6Tkrrcc+0GnxN7OVQmlfA7OHFwuqOP8U/zjoaceCOenN8wIXN8PyQ1GJk1bGuYBn1GxR43i8ENY6PDoAxvxC1hdZ5jteC8TocSzU7MNaKyIEh7iSXKHGq/BMXn8NiNRUQPidpaPRpXKYLjGN4NUivbGGseCDcbhauLYhhGIvdOyGQOkALpbaAqXyWXZUnD0UVKJayvjxGGL9k0tsG/osHiRr6OqbimFP/ALPxW5TZtv71DhnFkEMFNDS0hlqA7lSA7exXV8S8T0uGYE2lxBkAqagWytH4YUkiXleI8U1FdW56TLCCLHKNCe6rQ08MD5JK7Qy6t+S6gswjD8ShqK2Skkoqhu9tnlZsuIYXT4o8YfRNqmuNwD0VRl/cTpoWThz8jxcmHUALblwHA5stMxz4ZRHfO06OKgkxjD4pH1AvSVFrBgGixKzEIKyKz53NczYM2Ky1CHEDbyGkLXRmxkt6lRzBwGbopRPNIwQyzEs91HJEI3gCQOBUVI0xucx87yGBwvY7jsuxx3BuE8Nbh1dhOImr5ljNA4ki3uuNBZla1sQFjfTqtSapwbwrf6IeaG2uNEFzEcSwuadrJGcto9IYbABas+JQYzR0mHYtXRNp47RxOYfPf3WAyg8PgrMRkjs/mXA6EKtSRQTScyok5TDuGhEldx3B6jhqvp6iGTmxPN2uJ1v2/io6fG4Iqw11VS+Z4s7L1Sc2Eh/Pnnmp42kxi/pKritpZ8NFI+EXHpkG4RWrQVdJLVCfBsNkkeATJG8aH3WrgEMbZBjuGPy1sUp5tMdQ5nyUXBeNsjiOESMAljbnhPce6p8OYoyDEK3Qtqpy8NY7RhK0KuMRwVWKVM9NVBzZLykEWLT2WXK2zrLYrosEgEsGKQyCvY43yHQg7LCzErINsT5qhsIYS072FzZXaqKDI2noWukmBsNNPqp+G8XiwbE4a2aMyB55dm7i/VbNXhOE4pxOaduIeDZU+dzmG4Bsg5ybDJ6J4ZXvMDiLsA1uFBFVui0LA75qxxLSfd2Ly0bcSNc2PQTXuLLMa66DqKriSDEY2Unh20lxaSUdQqcb/ux8lJh1VG9k4uXZbkLIY7PIGN37LawjEIMMlMwp4qnmtLCC/0lbGNK97XbxG/WLYqIG5RzZDK+RmmdxOXshZma67VgFawuVcw6EVVVEBq3XN8lCHREfiX+hsr1FLSxTSMpMxYW7uNygknL5K80lEL2U+MR3dEy3mY3VR8N4E+doxXFJzHRBxYch1079kqeNgfUyPbLNCJPU0XBHdSRlua+Jwe5qvUUrJRzJat0OTcNGgU1fHT8ps9JSl9NILhzgqdI8RslGX9orQvuilopw+Z4kBBs4HRwV/Da19PPT4pT0uRsYIny9QsWKokfTRwSPuIb2+qvYZNI3D5+WLtjcTe+6xMDUlxeg5z4cShvG9xfDUNG5KqcOYh4TEZZ/gkuzMdgk4Mm5EFKRkeSHAjZWKHD6vDGTRwx82N5Bs06Ea7rI2sMq6p1Q6agm5zAWGSIDWQBbXEePTU1a6to489NicIOW2rT1v9VzWGl9HiQxXB3tzQaupybXHUL03BOH8IxWijraiijzEk2O4O/wDiuo5zgfwrKjFjIC7DpY4xKwDQE91v1fDtG3C4IaGMztidzIbnXIT6PkrGF4U/CsVkqqT9lKDHIzq6236f4rWFNyJXRQs/AJuP3fZbiGJhQoIPBfh0oLYZneeEn9kVeyhFy3hxsbNO47p8oW0owNlIDdRa3RZQijzXTl1io7gp3C/mVgE511G8XRNNhlSIBUEBGijI6KcjVCRdClYhAQpnhCRddIGJW0oDi7llZ74wxudm66WWNrm2Komkj6tXbXk5ZQwDKHepytUTmuzM67hW58PhILmgD5LPLOTJksbrvGTCene5suRyvt01WXHvnV6KRxaLrMwsSs5wUWUKqJW3y5lKwg/Jc6WxPNgoHOUkxdl0KgDtdVIhTP1YS3dQOcdlYOh9lE+G4OXRdoZlAJLKVuuqDwcxbnDwQlCXZshBuESkjGnMcoVgNOVKJnk1TlwvlWJUD3aKBrznylTSFVpHhj1YJWWtu8PCun9kfkqVPKMozKy93kusz6rDLqJnQyWOitxxyXEhYQCEDckkreY0E36rVytLQ3oNklPerB1/UopZR6Qiqs0TDkVTmXF0iG7Qyu/E9SUcha71KGaQ5t1E2Uh267cXO27BO1zdSrAkYRdY9PIXX/eKuRyuaVwy1ukZLrXZnZVOI8ypMmZmBV6EZmB/QrjlDcSNkYulJT3F2qZrbFSW0XOVZMkRDtkBjWlKy+llXdHqoKvL9kBh02V1kVz6VKaYFBklh6hBkWlNS5BcBVnxFRpWyJi1TGM7oTGpIjtrdOjylDlCimST5QllCBkYQ5QnSAaSa+l06LBJ+v1TI0JD1+qZJJEJGgRZggLMU52TN3ToBSSS6/VAzdkYahA6BO3ZYDkaKNw0Urt0DxotiuRZJEWoUWrJJJJEO3dOmbunQE3ZOhbuiQgTdkTd0Dd07dkUadu6FuyJu6IkbspI91G3ZTMQSs6KZnRRNCsNbogOG4Vtvpaq8TbKxmKAS5M7dMmdssBnoE7t0yAEkaaw7IHSSSQJJJJAkkkkAu9KB26N26A7oGLkBOic7oS5bAlyFO7dMi3RJ8xTJIWIOR5gogeoTg2RDlwSzBCSlf2KwHsUsxQ5gmuVsHmKAnqUkkCSTZglmClh0CSSoSF26JC7dAN9bJEp0IborLBJswRoVGyQI0OUoCSQI0DZghSSRmjO2SdsnTO2RZCknduk7dGSzFLMUySBIweoQt3RIEkkkD1CLR8xTJJtAEX0MXJk5cgWoZJJJM7ZULKEwanBuk7ZZDO3TI0CsAEkklQWYJZgllCFAkkkkCT6kpZSnyhAsoRZSnyhPY9kCST5SllKMBLU4anyhOrTZJAdEbgEnAKpRCyZ2yQPRImyJYUkkkaJJPqSiQNb3KXVOkjPqdyJC5EhkSSazkYFkX0XS1w6pEFWC1CWqQqqWptPZWHR6bKItt0S2SQ5SmOibMFLQyJhyuBTZgmRXK8Y0T3SzVWS7ctwvOH4dVSVrGPoZMhcLlj7hey4rAyrpTG9ZxwKKHlVVKSHMt5TqP0Rp5xiuEmorYaeZlUYiAGXOwQcUcPw8OiERVRmZLfyu3G3+a7zirDBiEFNBHOYJg7O0gXXN1HA9a6jfPX18kpYLhc5gcZBAwuFm+Z+w7qN0EZkc2QSRPZ16Lr24ngeF0QjeBK9nQt1P1WdBjuETxTUTMOkndUdDusUOfnxCrmHhp5+YBtolE+o5PJa/wAnUKSSjNLU2qonROOwPZQvqvDzEclzx7bLcNy0cLxSTBpnSMhjkZILSB2/tZRYtWMxGmzzzukt6QegVGV93XQct7xpsqKYYMwcbmx6lWMshY6dl7MCBzWtNlGZZGmwdosC3BE2eEyT6s99woZxDGBy5AW+wUbpH5dDZX2U0LcLE0zbyP1VpIZ2W4v0QEsBHsrwgp5MPM3PyyxnRlt0EFXS5AKmibKPfS6ipYooM3jaOnnmjH7TTZPHh1VWNE8L42g+900OL1FG2WHDjyYZd2b6fNNheIRUTnMroBPE69xdBZxapxJk8TZHcxsbdXN1CoCdrJC8sB+YXS4jxdh02FDC6XDSxh3duVzMFM6ZkknLkLIzq63RBbpayljeHPNmPBD1EaWnnZLPSSBrItgeqLDThAle6uJDcpyG3VHRVs9A+Q0QZLHJu1zbghBrUGFPxeqp5OFYZedTgPnD9A2/VbOPYBS4FPR109RHJXvdd8LAbOZouZwDH8Q4dxps8DHDmyBksZFswPT+K7D7V8Iqp6mlxWhY50T4QSANR80YcTj1dDiOIOqoY3MuACD7LOBtdE+Pl6SepMxskh/BjzW31tYI1Dfg4ZmFJR4rO4SUspBcR8AXRw/Z1jeIUszuH5sOmo2OzsYZrySaXsCsfAa+lpKUUOITHSYSRNOscoJ1BP6L0h+P8K8EY7S1baUl88Xmgp9R87Iy8h4gjoi5op6WSnlhuyeN+4It/wAVmkNa0NAXofGGG0PFdfWcR4Ti9PC2UAmnkjyyEgf3lecFznDVGsZaOCy4bT1/MxWDm0xFnaXIUFZLS+NmOGscyF5uLqAbFO3RbUIY47bp8pJs3dWI5KR0JifEea7Z6J12luZo07LAqF+U2IVzDuXHTzzdQbWVWf8AElzxsI01BVmjbI4GFjfL1ug1I2xfd00Lnvjc/wA+Rp0Ko0k0lL56WdwDxYtdqLKYQTunaS+MveQxl3aXPdde/wCy/wANRST12JCKW/8AVuzRgnv2+a58xx9RVc/lwRstkFgAdL9UUYpYMNkMzPxy7a9ixG6g5LXyMkfMA45ixvpCgfTvqYBUMhLmAkEjcfNdIyCZBzYjOZA3LqSe3Ra+HRxYhQ0lOyp/GkPLLdrnoqzQyWKnnLBliAj5IFiWdVBIJ8LrKaopJBIOaJ4mDQgjoVjIdPDhMeG1zqSrpXR1MdvONf4KTB6Tk4kyknq7NqZAI5G7A+6DEuJZsRldjEdJZ0kQY6Nx2I+iiw+Wapo4p6SQC8oEwOhjIO6yHq8Nr4cSrzHSlrqdt5AwXDve66jg+qxnB8EGKTPc+nrXFn4xN4SNRf2Kn4VpJ8fxutj8WY2Tw5J2k3LwdyF3X3bFiuGy4TSVcMbDHy7OOoIsqJYYfFUNFXQMy528z6lW3A5fMrdDAxuE0bGD9nCGH5hRzQ6LvYo5QhyKflp8jlthXyOTeZWMjU3LHdClZ2yR9KORtigIQxDeySEusUHOZfLmWgTvUk3ZCXjugzjoURI4aKFw6KW9wgeNFlUV1E9gupHbITqtxIj5LHizkD6SFovkBI7qeM2Nk0o0XXGWHNVkEkMpcPSpaeqaY8hU2LMsCs2I2C9EfmcUz5fxc11ZhlNr5wqDjmKG7gVeBbYJJj0UQF1TiqnsYA9xKmZVi4ddZnFbWMhKEtcjimjcL3UzS1yNK0YeHAO2VrlQb21Tlo+EIZWXZuQQsqc5QFXcBm0UkUckgyuKB0b4nnPsi0CSN7m+VVpIzbVq0onNc3yqtWROarEszCrGHgWCv3/BChiic5misthfkACkrCjK0k5hutCkkvEGvd5huo/DHNYhTeGc0XapMtCmjbMwi9lluhMT7PC1A+w1VKqLXG4VhMmdUxg+cN1UUbDm8wVmc5W3UUXnvZuy6xLmmp7td7Kd7nB6CEtaQ091Yl5eQuWJIDGWNOaQ2C2KSRpiFtlgscZGlamH+RuXMSCVjPHydIlptUoOirh11M1+i8sw6QLKCFDIzzaKyHaIXM10WaEYY1EgLtUg7VGhyRtcxUpIdVbLkLg0hBnSR66KJw6K7JHa6qvYboIMoTosoSyhQ9ApJZPZPlKyqOzk+Uo8pSylALdk6LK3umylAySSSA0ztkg1E7dAyBJGi+h27pkkkQkSFEhBJ8xTJ8xQMhLtESAuQRndAjdugRYJJJO3dCJMiTN2ToSdu6JM3ZOiQSkb6VGjHVFk7fUFK3dQjdSsUhEjd1IxRt3Usaomj9QVluyhjbYBTRg3QTM3CkJshA0Qk3QPmCEuTZihJssBE2SbsgLkbdkDp3bpkkCTu3TJi5bDO3SzFBmKTd1gSE2QpJ3boBLlGd0SB2yBO2QoyNUxGi2IzumSSRZCkkks+qCSQpLRZ8wTk2QpnbIETZLMEzt0yB8xT5ghSQL6JvN2TpKUGzBOgRZQr6BspS6fVEmyhAKSItTO3QMkknylSqA5QllKLKUspVAJJyNUspQRpI0kZiaAmyhOkgZ2ydJM7ZEA7dMndumRqBJJJ8pRkSSSSBnbKNEhRfUkkkltDO2SbsnTZQshE2S0ITptAFQsoSyhOkgjLUKkdumVDZU6SfKUgMnylEB0CSz6MkkBdPlCdaQkk9illK02ZPlKZPmKlBkk+YpZiqGT5imSQJJCiRmYozdkTd0waiAQEGogOycMRhqIANT5W91IIyn5bkaRZPZFlCkDHIuWe6IjEYRZFNyk+QdkRcylDlCWYpeXssNgfsoH7qd3pUTtSqID2QqR7VG8WWmZCXdkOYJ0BPUpCQTzdCkT1KHMUbVKylE+IUz3+lh1KyuL553NMFM8hwaAbHcLezNvcqrV0NNM51RNl9ySsjz+bA8KpYqd05jD5DciY6hURxBgeESyCgpQXEW5jRcLSxnB5+IsTFJh0mamiF3SO2C5/EcGw2nnmZSzyF8Yv/muIyp6yoxeuEtQ+1zYney23wsoaJtJI1ssTzbmEWKrx4bVYLSsxWaPll5ssusxeqxB4jkyhnRoC2tgxAU8dUYqV5LRuSlTkZ2szgNtqqkmkjfknBjLDeO7mncKNJJHR2czcXuCqhNkZjeGCToVCSSVkSOLC2zLqQzVEkXKMhLRtdQtHZSh9jlC1AAkuFilHDM70MuETJYIpBzoy5vWynfIzNenach2BWRC6lkjFzb6I4Yo2ubNNZzDoR2WjTwwuhzl7hpq0qtljY92VnVBXrBCyTLA8Edh0RQYlPT08tIzKWS7gjYq5PKH0/JZStLujwNVBTeFjgkhqqLM950f1agoN3WmypiioYSyoizMdnc0aGwVN7BJpAzXsrFNhE1TC98kHoNrX1/RBu8WY5w9j2H0mIUDXxV8JDZQW2uOh39l2cfHOF1HAN62pimruUad0APnadtV5dJS0MTmE3IJsRdWa/An4c2Orzg0tQ28bh0PZEmGZPJzZiVrcK4S/F66Wn8WaaMtIM2UOAPS4WRKLHMFrcLNbUVFRT/eMtHI9t2lmx+aK6ulFRw3LJg2IxRVEMerZpY9Ceyg4gw+XCmU+JSPb4Uedobu0nYAqjhGJz1ta7hzHJefTPu1rnDzA+xVGve+hfLQ1FUamKnksIHG4cOl1WFCTFhV54auXmMebtd1B+aqyiIfs2/P3U1bNh1QweGojA4db3uo/GHlcrlxlo20UXEgG5fdAWXaehG4QvLicovc7AIbhzgbEO2I6rbSRvqVprs7MrWkOGocVTz+a1iPmrVJHz5hG8kC10Ac2SWQGR1yBZGJZGX5brX3Txxxu5scwO9gQVNBRTvpnPgbfl7jusDc4D4TZxZU10U9VNAKeLOHs2udgV0nAr6iKpxnh2VklU8MeyK5uHZOirfZHR4tLNiddSMjfTsjEdVEdzodv4qQ0kuGV9NiODRcuspDeXMdJQnFhLVUGK8H4FzDDA52IzWizN88ZPQ/4Kg/hSlpaurwuWWcTy04qIhn6/GP1Xe1EUPG3hq3EoPCR4ceY9l/UTre6854zrMUrMZkxUnlxfs4XN6sCsY21Eho8AopGwQuxgZpR6bWLT230UMuBUdW4sjqjDMQeUS64uOh7L1PhPh/B6uiwHHXUIa+V13RkeUm1rrfZwLg0NXiNLLTCWGcc+BpABjPUArM61eMRcP11Fhba57mVFM8XzsOx6ghKjo3YdLSh0fOgfapc5h1DPkvTeGcMwio4edhFQbw1c/KkI/qpA/QE9Liy2jwNhDJ301PAWtjisxw3aPYqcByv2d8P1LWT8TUVNJV2kLIix1rM7OCyq3i/inhnGJI48PkhyzGUxvbmNj09l2MHD1Vwd4HGMCdKWSycushe67TfY+2q3qrDDidU8zwASz5HvcNOWB7rVMH4axlmLinqYGGNlXERI07xyDcfxW3NBrayrYLgUGF1tUYz5JSJQy2jX2sT9VqSgFxK02yJYrHZR5FoTMabqoWEOVsRGL2UJY5Xmx3CEwttq1IRmy6BVXPstWWnYT6VWkw9kgI1HyW4ZZzpWuFlXlAzXarTsKdC4vY9x9io5ocjA49V0hIUZKgRm10zKgu9LlXqh+IR1UccjmrrTm2IpCRlc5SFZUczwQbq1HUXNnLOWLcSlc3VA5tkb3AhRuOixRZAXKmlb+HcbtVZj7PF1azXYqTDn8V55eMouqktGWZcjLXFz81vTwteSVn1EcnRq7683OYY+zrorBHJE/ObtsgNx5XNsvREw58QvB7ILHspmx5zbqi8JMXCzNynKDiBoew3V2llN7lM6mLG+dqZgyhc5lqIX4zexUuh9SpRzAWzbK6CDFzAsS1Bmjz6bIpGZgq7ah2ayvNIc0WWaVUEYadGogzmaPajLXA5cqJjbFZtoLYA3Zqlax1tVMG2CYG6xOS0jDNfMiLLjL0Rp2rPJaZs8MgaQ1Z7oajzEvFuy6CSNrhlKqmlbm1C645MZQ5/K+R+QalXG0b4IvTq/UrUFHE12cMARSxh0TtNguniMUyOS9+zUXLdazkcctnltlIGtLfLutWUhDQ1vl6KejmF0HJcQjgpzHsFM/RcWs0aKVhsCoKcnKL7qdrl5ZdYGCiLlGCkTdYlYCSSUwckd03X6KNCzBMXIEW4RkDioHi6slqEsF0aUyxDlCsmLXZDy/ZQQZQnR5bJ8pRUYFkraI8iLLosiJNlClylPkHZBERdICylyDskY/ZBFbRJS5QhyaII8pTIiOhQFqB0kCNAk7dkyduyB0k2YpZigZRuNypEDmAhYAISLKXJ7JnbLYjSR6kpO3QMEikEikBI02UJwOgQEGp07d04FkCUjBZC3ZE0WCCRimjFyo2tsp4m2sgma3QKeNnRRM2ViPYrAfKEDt1Io5DqgYuQE9SnduhdsthsxRMQt3ThywJCepQ5ihLk2YoHLkzt0yRNkCGySSSCRu6Tt0yS2BduhylGRdNlKAQNEyLp9E2UoqEtQ5QpSLJkRHlCZSEXQlqAUkSSCJJGkgBJGks0I8oRdfqkWpO3VAO3TJJKhI0CJuyAspRJJIBylC7ZHmCFGDZQnT5io0bGhJumSQJCkXJi5Ai5CT1KRPUoC5GRZgnQJ8xRBIEabKEApspR5QmylF9ADdG3dC3ZGNyr6eSDbsmIsiTO2UEfX6pi1E7ZCtQBS16pJagpQQNikTcp290nd1TkZJJAs+gNJJJaDO2SAsnSQJM7ZOkgW6duya3uiadbKzDIgLJwOgTgXRBqiGyhMWo8pTI2idumUzmhRli0BST5SllKAEWUJ7FPZxRLMAjDEg1SBl0ZAGlSBhRhhspGR9kEYa5TRx6KRkfZHHHqrAERfuo+VfopxH7KZlM0i5QUOSn5dj6Vo8lgGgUUkTdfKo0p+Xul5e6m5SLl+yFI8oQO2UibKFiJVGTdDmClLGpsre6rCF2yida+qsOjuEBZ7KxLam7dAQ5WjF7IHR+yqIExCkczX0oCLIQgMUbt7/qoaymhfQzsANzH3VktQorO4cp4aag/DbbPe65DA8I5+N4gyrZYxOtZd1HGymDms0aTe3ZRshY6odUBgBeNbdViYHI8S0rayOkwc6ea9lhVX2fYnS1uelpi+Jm5J2C9Eq6COSojqY47Sxm+ZahL5TnaN1mh41VYHHUzufJCfI3QMGunsqVXhrKXCXVLITmzZAOo+a9ax+LB6ON1W+eCGpYCb39Z00XnU1ZVcRTtoaGhGVl3uLOoRtx0jtLKJdjDDwzA575oTLNGbANvdYuKQyYzXzVdBTPDOubQ3ClDNiiMno6Kd0EMGXO9ri8XBHRDh0VRPM6Cmj5ji09bLYwbCG0cVRUYpHG5oFsl7lUYzw0FWqmjMNKyqgqY5Li5ZsQeyrVEkcspkg0YdggzEIDdPIbDmG3UJCX4VCQ4DMnB+JBebLPE38N9h2RGoEkZDzqVWilJ0f2UT9DZBKwlpvGbO6FTCWd88TBVvLybGwtZVWyWK0+F6uio8cjqa9+WEggm3XoginpIWynxTxMe4T19Xz6VtFC+RrQb8t2oXQ4jgUEVRPiXJjkoqgktkvrGfZcjO4c8mM3b0KATG7KGla/DVXRYdNJLOwiUt8rxsFlZ7hDzD2QTwzvlxB9XXSeYuJva+qsVcuG1H9KDHGVhGcHaQdlQGvqTFoKgs4qaB0rJKDRrxdzPylPFXUEMLozQhznixcqMjbOsnjGiCWjl8PUMntfIb2Q1U3PqpJrWzm9uymgo5pYH1UYzMjOp9lXl5YddnVA7S4qWmkcydrxveyhjNlJCCX5ujCqLAcDLbN5nuuR2XQcK0M1dW1NLBWxwyhudrXD9pbWwXNF1pHkDW9wV22H0eG1fBsGKQQZcXpKryyg2zsvqD/BYHUcBnF8C4gNRNSyeFrW8qq6AHo/+/VdfLwl98YhHzGRMgp3mxBuSL3S4XqX4hiMUc9CaOUQfisJBBFtCF18MJh+FWBmt4apTU1Uc9jS1LAwtHsF5JjfDUMjcWnpZpG09A8RxA6gm+y9mxV1a+jMdE/K5+hd1AVKkwGgiwiXB5I7tl/aE7krriKHBRfNwxR0VRGGzYW4ReXbrYj6Lpq2tZR0c2IvYH8oXt1XP8J089HDX0tQ8kx1BYL9gtmqp21kbWEXaDcg7FSB5Hj2AYnhWEQcR0T52w1EpfUNabBjw/yG3W69opXsqaKkrGameCN5PuQCsniDDPF8OS4ZG/KzR97X2N1rUL/6BT/+Ez+5ZykSuax8boZmB7HixaeqUTGwxNhZfKwWbfeybN3Rt1WEmRN0N0L5TeyIaBA/UoqR2oULm5kYa4hPk0utiNrbBM5oypHRIG4UFWUHVDFmB1U8jAVHkcrbB52MkZss2qoHSu3tl9lqhpKYsTk05apwGrlkMjHtI+ap1WE1UDQ6112Do3BCGMe0h4Xfmy5eHDajlAujVeoY+nFyLLqpYWxDTZY9XAyoe5hWuds0y4qxxNiVZY+5sqstE6OTRqk5nLtfdaRZfE0C4SadLIopWPaEQYPhWZWUbtUBYOysZQllFrWWbKVJKanlFiwfNVzh0AuzISOi0LNal5VvnJTIkoGROLWCx7qFzpmyct5uLdVtSMa8KrPTsIGi6c5Y4qwifMy790D4Wxix0VptwLWITOAcLEJzKZ7Ig42up2gsaY3bKYRgHQp5Ggm9k5pSs2NxdfKrULi0hqFrdbWUoYVOSpyQ5qIRtDr5d1XL7DRWIpLjzLEtwNCju0pliWgogLIUSyEhcy5RJIAyhGGttZNl90Td05SKVVSsEmZjLX91GyINO60JG5gg5MfZdfEYpSexwcLhTxtaQFNy2dQiDANknLktHY2zRlRpmp1zlvE17JyepTEpnbrCYiQdfokndujZi5NmCZNmCILMEQahzFExAsifKLWsnSRLV5IdToouWO6viNxQmHuFmVVWwu7IuX7K7HDpsnfTO3soqlkPZLIeytGA9k4i/dQUyxyWT2Vt8fsg5fsgrFuqBwVoscgMXsgqlijI6FXHRNUTm6oICLpsoUxj09KHIgjbumdoitZC9A2YIUkQai+h0SbKE6IA7KN26lfsoyLoBT9PqnyhLKEDZdE+UJ0+UoGTgXRZEQagENRgXThilEfsgFrNVKxumyJkWuynZF7II2R67KyxlgnZFb4UYagTQpWbKMDZSN3QJ26F26dA7ZBEkmdsmWBKmJsmzFJ262BzBLMEOYJZgsAswTJswSzBBIGowNEykbutgCLJAXTu2TZiimQI0Lt0lDJJ3bpsuqAU+UospT5QgDKUsjUeUJZQggLLdE2UKxlBSMaCvlCbK7upuWmyKegiylMjQ5dUiQBFkkSEiyoHKUOUKXKU2T2UEbtkm7KURpGP2VDJJvN3SbsgFJGkgBC7dHlKEtQBmCEuRkdChLUZJAT1KkylRkdCjQE2UI8oSyhW2EaJPYplQaPJpZC0aKQDRBGRZMjLUJFllQdfqiTEXSbstIPKEun0S6fROsgTsEySS1IjyhMiQoEknylLKVQySSSBJJWPZJAun1T5SnDdU+UoByhOiyORhvsgjAUganEeu6kDVYA5RsnDCpMjlI1nlSIEGRyfLlUxFkLm3VEJFkBapHbISNUQOT2TZCpQOqcRklGUYYUTWd1KIiUQiKCIRqRkWuykEXspGxrSgEWuaykEaPJ7KSOO6yiMNPZSxRvJ2spWtAUgCAhlARZh2QoSEBZ0JN0ycHVGrNk9k+VikyOSLCES1BJBHIwty5wSNwjXOQ+UpZSiYwn0tujyEbkD6qtIspRCJSCPr0RBqCpJAd1CYStQx3ChkhsljNfFqoHR6rQe0XQOgB6KxPEZxagc2zSewV91OD8KA07ADcXFtlbHL19TT0lM6uxKYtYNmt3cVmYRxnUYnUGGmwu1Kw2MkhtYd0uL8OrcUjihmmbG2SX8NjN7d1j8UQM4dwmDBaT9rKLynqbqSNbEPtMwGiqnUopJp8m7mHS6yIcW4oxqqZNh+JOpaeoNmi40HXRUpeCII+H6etrGyCacgeUX0PVdPQ4V93V2GwSN/DY0ebqpzHP4rhmDYfWvhxIz1dQ9u73XBPy6Lc+zXAoIIJsXcLunvG32GqhxjDG1+O1NWyLN4cC7O4WrguLYdhcElPSuL4f2u1hET0WbHL4lhdFwrxZ4qqkk8LP+JEbXsT0/iucxCFk2LVlbSz8qluSHXWlxVjUvFOJsbOzwsUTfw+YNCR7rj3zSNcQJDboQjaSjlko6t88L7aWBRVb6uofz5q0yCTsVDSw1dW8shZntug5M+dzAzUboC5bwwuNgAo/N0TXINn/D0RgiyAGglI6GykidleHKORoDi5rbBZUvN2U8LTK05tSFE0AmxNvdGLsfoRqrSFKzJ5gpITBGY6mZglDDcxnqmzMB8+3VRPblNm+np8lRv4lxTJilG2hFI2CJlrZTqAsblse7KX291C0vJ5EcZdI/YBaVdg0mHUsNRLO3NL/Vk+cIM4hwcR0TZgrdFSPq3ubs0C5Kelw7xVX4cG2hN0FenMGSTnNdm/qyFOYmcpsjHaP6dlHPEIZDG3Zuija6zlgTy4aTEaoPvb4baqOKna6n5+w6q3T4gW2tlNujhogxJ9VC9lLJHHHE8XDWi1loL7wf93+CjeQ7qSNwqBFlZqRTMeORPzA5va1lDbORZgBsgGx+FTQG92qPVvlsQVOwGI5u4VBEaD52XqP2b0/LNZwtiDYJ6aeHxEDXbk21sPoF5hCx8+YC1mC5BNlcwzGcUw3EqPFYJ5XPojcZd8nUIPWOCOKqqmx6iwnGAWcuR8HOfs0G9gV61KMjiF4PhlbQ8Y8dtmZSTGirTkJcLESW3/UL2LAMJxnDacw4ji0lQNQGvFzboboLztrKI6KZwsU2X5oypFjGSOnYMrn+q3VSsmICeWK4VJ75Izly31stRKL/ADGvDoydHixU8JyRtjGzRYKjBFY5w6491cjvsUyi2olZj2KkBcFGw2CmGUrHopZgh+P6o7WQndZEjdknbIA5OXaIIpRqhDU8huD3QsabaoDA6BJJPlKALlM4lS2CjeFSgHKVA/yODgpi4NdZA7I4FaEM4zRlZEw5UputVztLLGxeGaTLJG+1jcrriwiqBnbnj1PZZ0zvNroVfjY8M3VStjDPxCF1iWZKnOzsyuh7T6SsiOdmfK1ytRTtvutzDNtAC+qWUKJkrcuZEJP3lxpqyIsmIsE5N0zjoqocxUb902cg3Uee79XLUQkyl5QcPMo3Q2KsRi7Qnc2+6kyQqGEk+yYwm2quZQmyghTkKAaWnVEHqy9g6oH07bXC6QKw9fmUzDohF3fhvZqNihyOjeddFGE7JbOyuUoddV3ZAM3xJRyEOyuWabiVkCyQCYOTrMw1yJM3ZOkshswRZigSWVtIDdLL3QA2R5lpJgGVOkmbspZxSJIEagSVrpJx1RZLTullKJv+CZ2yL7kJGUocjron7p1pk2X3RN3Qk2TB7b7JQnGyezrpmHRENysgmhShl0mt0UkY1WZaFFF5hop+X7Iom3Nlbjp7tHVZVQkprsOiquZkOVdCYA2M3b0WVNB5jogpFgso3DVWnR5bhRcolyor5HJsnsrRi9kBiKCq9tlCWXKvGK41UMkNigqOFlGrEkZsVCWIInqMi6mLUPLPdBDlCMCyMRo8nugiAullKlyu7oCwoIyNUsoR5Qll0QR5E4aiAujDEEeT3T5GqbKUg110EeREGqQRohH+6gjDXKxDFmITRs12VuGOxCB2Q2OymYwBP5e6JAJaEBFlKhdusAALp8wTJICUbt1IgdsFtZRJJ8pT5QiI0+YoiAhdusBO3Qk2TkWTO2WwswRN3QZSjaLlBK3dG7dCxJ2yAC5CnduhdspdA8wSJshRAOKoDKEQa5SNjbZFkcgDKU+UI8ru6XLcghyhLKFImyhAg3VGhA1UmXRBGRooiLKV/VAgjyDsmLVOBZA5qCAjoUgOgUhamDEDZEuWe6mEaMReygr5EixWOX+6mMfsqyqmNBkdfZXOWmMY3RpXEf7qRj/dVjIhLUEHLQloHVWCLoCGoIMuYICwBTkWUZHQoISLoC1S5QmIstMIspT5QjyeyWT2QRkWSylSZPZLJ7IHaNEeUpkWUKWALUJF1MR0KEtVFfKUspUhHQpspVEaK3uUWUJ0EZFkJF1IWoCOhUCTZQiy6p8hVOQMoTFqPKUJF0AO3TI8pThgugFPlKla0Igy60IgwqQMJUwiARCNBCI0QY5S5PdOBdShGGImstspGtRtaSrECPKEsoUnLKfloiIjRBlCmcyyEjoURFk9kuWpMoT2QAGIw1EGpw1aQ4aiDcxTAWRsb1UoE2L2UgYAmzpw8oCyABOLdUOYJ0EjcvVFcfCorlIXvoCqqxHHdScj94KGMvBCsC5WRG6OyDQHVWxG077LOrKpkUmSOPMOpOillrrSCN00zcjM6yZuJMMw+D8Zksr/AMrQuZxTjmvqJWspacwRi/kJ1TniAgxunjs2nnp5amV23NFx8l1NPDVcoGRi+djPfzCKSE9CDqF0nDXFvGNPMKfDa41Mf5Kh91lp6jXV2MVtYaHCqiOJo3FrEFS0/CHn5mI4pUTS9bS2XGR8b4XidR4fHKWahr2kgz017H5rYw/i+lpKgUFbPVSX/Zvc7WyDpn0OJYawvwucTs6wzHX6FT4ZjNFijzTZjBVMNnQSixB+fVVKXHHzE+CjNRbo3dBicFJiwZJPQVVNURm7Z4xrf3WR0BDWetIMzBcfxBxLjOD4aGVeFeMY8257RsurwOspsWwqmraSQSZ2+axvY9kEVRAQcwCqg2W6+BkjcpCzqqhLTeNqCkXsCBxBBAUxpJOyhNPID5gtChFQ0kT+dyQX9yudxHA/vOsfVTaXNgLXsF1jmkKuYlRj4xQ//CGQxDSIg/ohlgfXRQSRjQAEFadVC+WnkjA3FkETHwUzA5ti0WUmByuL1jcNw2sqnuy535LrkarFcPmljpKWpnDC3zcodfdd3jGBfedP4V3pc65WFUUNBgNcWQU/4pi8rGDUlShg8axw1UOH+EaDcEaC19lydXRci7ZniNw6HqtitxierrHPxgmDkOtGALC6ycarfvCpdMzY7KNqtG+oDntpZMpI190F3jUuId3urrmUb6eN1KTHUN9V+qqTRSQvySNsVgBqSnbsnI09KexWyzgWIPZPL5ySNionboxtbogB26cAqdtLJnAdGQ090Zbd3LY8BoO290EMuSzbb7EJEOabHtdXaNuGxVIkxGEyR2tYdFDXeEfVO8CXcroHIHwiSigxOKornSBkZuC3oVocS4qzGK1skZjcyP0ua2xPzWPyJzqGXCQDm7oNGmlfDSSlvUWQ4dO+kroJO5sfkpaSJsuAVFXnu6J+Q6bqs5vPpoJA8Mfvr1ssCzPFCKx75WZmXuQsupMLqhxgBDOgI1V8zCopjBGb1Z2b1PyVDf1+vqDugCS4iK38b+45eHsNnoqrNWkBkrdy3uFiFlwih/CIfkB+YWwEbdiVZgP48dx5c2qaR8Bd5WkadlZw1jpXc8MPKbu46WQDVxPM1wy46lQ5StX8CWqDY5myNuA4Dt3U1VBhtO2qY+niIJBicBcgrAr0dDQS0D5n8yOVltBoLlPLTYlgLrTx8t1RFeN24KucPNrJqiEYdHz3vOSRhGjh2WhxW37zqBAxkkT6OMCSN41aVYkdn9lvB1ThfEFBi9XPHJz4Xl7GjSN52Nl7I/dv9lcLwHR1uFGnoHnnsNJHNC941aOrF2TaoSgHbRaDuCjUpHQoS1BA4Cyikha5TO0UetlcUtTEnKcdbBTR1Wz82iaaFpBcoP2fr2WmWtFKHjM1TCTKsaF74/Ow+pWGVEhOVzliYabAkaUzzdU45dFYD7opkx3CbMUwcsA0aiJuizrYNJRZ7FO6YRtuT11WBImI0SuA3O52idjmuHlddBXmisLtVd2ZXpGkhVXMOcrViAszBU52Mewtc3qtPlu/KoKimPLOi3EpMMWcZG5h0WfPOxzSyZq2H0+YZcqrzYVzW5ctvdbxyYmHNyx5HZ2HQpopJc4007q+/CqpzwJINB7qyKdrR54gCvVzhypVkqCIx5kMVV5tSrdbh7mw3jaCL9CoJMO8gew2WbhU7JWnqpNSqXJeyHOXWQRVDmixcpxW1x5+FVZb5w4GymaWyendRSt69Vqi12E5WBS5lShebjVWg/VccoVJoQllCKMAoy1qjanMSBcoGThzQFPO3yFZUwe0ksXbBmV2Rpb58wv7IJCDGXN3bqqBrHDQo4ap1+7barXBnkUFUJXEHoVcy3NxssySF0UomjcCwi+iuU0x5YD04Ja2wiyIO1UIf2SzarlMNYrGZIi+oUQc5G13ULnOLUSLKUySfKFmmzJ27pZQllCjJ0kkkbJJJJA7d04N0m7JwNEYE3ZOkkgid6kzTZG5vZROfkOq1EB5EDGku2R+V1lZhj00WxCHW0spmC6kFPc5lMyAADRYkJsb7bKRrdVNkBYnEbbrnLaeEC4V+C2UKhGrsDtBqoJZR5FUlhu3NZXC4FqAj8ywM6Slub2URgy9FqEMVeUsb5cq2KZi02UZit0Vpu6jkGiCi9uUqN40ViQ6qtIdUETowq8keqslyAjoUFMxlDyyrJahtrdBCWFLK7upsoTFqCKxSLUdikW6IIspTKQtKYMKAQ0Iwz8qMRohGUAZSnDHKUMJRBiCNrSjEaka26LJYoHZFpspQ3KhaC1GpYbMEQcmQg2WQZN0xNkibIC5bBO3TByEm6Td0B7lORdIDoE+oKCPKEwNkRcgI1QMTZA7ZOmyhAyE7ogLpOagjb6kfxJ0kRMNik7dRo0UkCNAgZuylaLBAN1K38ykCVrdFK2K4QRuClDtgFQ/JUbo9Vc8uVQv9SCsY/ZCY1MT1KEuQR5QE6SSCJw1Q5SpcpSyIIxsnAujyu7qSOK6CAxk9EwY1XXQaXsh5XsVBHHHrsjLVNFHrqEbo9NkFQsQuaFYMaRiVZVsoTZFa5fsh5XsEaVixyFzSrRi9kBjG6CoWOQkXVh4soXNKCBR5Spi1CR0KMoSLpspR5SllK0gcoSyhFlCWUKAMpT5QiylLKFoMB0CfKEsoTlqAUk7tkykiMtSDVIkqI8oRZSiT5SgAtbZRkXUpF0ORAIanyhEGOsiazutCMjRDkU+UJuX7LIr8v2RCNT5D2TiP2WmUQY5SMjNlO2PTZOI0aAG6IxGiDXIw1GQcr2CExa7KxlKWUoIBFqpeXboiylOTZBG7dCAVKwBxUnLYOiCq5pQ5SrTwL7KPIPyoIcoSyhTFjQoyLLSFkCawBTGRQvccy3EJae4uizt/wCSq3M0TZyrTPJbzoswVZshaE5kUo5JxIjEmuqqcxo2cia8vKUcmjCYS4XKuCOE9QPmsYyOjF7JpsTeyNxOgAWJajJrmeiZJk5t3DcAXsrcUcUg8hXnL+LHwyZYoSQOhC0qXieqm8oeY/mFx5N27l9KGRmVzwAN7lcPxZj1JQyGGF2fNuduypYnxPV1LnUsdYZdrhgXPV7ad45k4zS9lyyyLRT45PO8vD1EcQpd5hd/XVZT53vkNwAOgARNYyTUjVcVYOEQ/fNQaWqkOQC7bdCiwjDmOxWSmdPI2KLZ7DYrUxnhv7hlpn4VNzJnjzM3JK2aDhuM4YyZ9Jlqn6yuc4gfNelti8M4LDieLzyVZMsEY9Ttwe6ysWo8RrsblpKF89ZyzaPKdgu6ZhTsLiNJhU8cjqsXIcditjhPhen4cppa6us2pIuSNVYlycnhs3EHBjYauuZIIpSAddRr2C72s+0GipMCkxKhPOlYw2Y8WsuEqOI8Nrsbmq8blc2mYckRI0PtZDUcRcM1j/DUNKJD01y3Wmiwr7VKuHxAxyldVmQ+VrbWaO1lXj41dSTSV+B4pNhs0huYGx+Ryz+Ja7B4jE+gLWzxjUAbFZFJiVJLWtquIKWSelG5asj2rhPijjDFMNjrn4jhdSx/+ss0/Vd3DOyWOPx0IhfILjKfJfsvnCtm4ZqnA8OsrYOsrG3IIXXcA/aFhTIjglZVlsrNInVkhyke172Qexz0sETeZMA1vuVky4ngB2qs39kXXH4zxLBRMFRiUz46aT/Uy82KQfTZa+CfaFwOWQwvmo2iX4ty0nuEGnIaScXpXl7e9iFVlh81l1klHFXQMloquINeLtfGQQQuZxen4joqWapkrMPbFELmRzT/AHLQ5PiKompazk4NXf0o6yRnVttFnO46gw6dtPxFTtaX/wBfAQRf3XL8QcXYrUiWmnFKHA25kGxXNV2FYsII8RnZemf8d9kHr0nFvCRaHjE2XO1mElclj3FFLh1aaoCOsfK3ylp2/guNfVPp6ZlOGx6i+Zu6oE5pGsG7zYLIbFsT+9a19a+HlOk3aBpdU4o8zw0rYkg8A0w19KyR0vna8G4sqfhmRSNyPvf3SJbR8phaS3dhQVUr6kN8g0691YJcIjG57RbVVgZOXvpdYDsaMpHVM0s1D0Tcg1LH6blTyNhla0QM2HfdbFTlPcfI26UsE0ADpBa+y0sHw59XIZZLhkd7gHdDWYaXyMqI5/wZXBgzfAgCl+7paTmVlRKJGmwaDuqknLa68ZNjsVaxfDDhMjWOnEmcXuBZUQHSN8x0QS1BB5RdpdqhIs5HO68TWfkUIJWBNvpnI+SZ5cb31KcNJ1G6Tm5DmeLLViaiqauOhqMOYxphk85vrsreEQ1WMPgoYHNjyEm+XQD3VrBcFqsQfFHS1TY/EtNw5mn6r0Wk+5MJwuOkonxS4hTXZoyxv7pLChwnBhXD0VXU4pT089XTuzxPjjvIQfZcNxgaCbEvvKhZNGKu8kkcgtkN+i9X+4sOwzBH12I0LauWS73G1iL9AuExTgfiPFaB+LPmhdHTtflgBuSwdlGolzmHQw08rX4zSyMpp2fhP7lUCWB5DHZmg6G247rRxfHPvLDqKidBy3U259wLKDDjTx1tNNVC8QdrqtKs4HG+rqHxGMuYG9W6D6q0ylxLFqPwVDaZ0TtY4xqB3K2abFaGifU18EIqHE2ffyiNntooKPFPurF/vSlJFHX6G4sAP+SgqYRg09JhjMbkijkiklNMW/1kb0dThs/JlEjAZpPPHY6ELpMNnppMClhs0tGIiYXF7DVRYzR4ZDjJrsOhmq2ykFsEMZMbid7FB1fDn2f0uDcGu4ggnMtfVwB8Y6XJ0A10WVhBqBjOIV+P0kYZUM8HOy1y19t9Fp8I4nX41gpwCelnkw2mnZnlGhgZuWH5L0PCODMEwycvoZ5pmSi5D7EXQWcMwqCklhr4/wDsjIhYabDUKnidOKStjqGO/CndsNA09v4rP4c4njmxOtwSUcvkTPEGlrgbhbeJ0/3lQhkT7ASB4NuoQWw8SsbIzZ4uonm6eLWFvTRCRZYAOKHMid6kF9bLUJJz0VaWEPFs1lNmKd2ysSqJgbFHld0VeOfNUWGymq2PfTvYz4gsrD4KmnuyV2u63DDbbNlcGlysNk0WQ+R58t7HurlNI7li7rlZmGl4SIgeoVcOUgd+ZZVLmKZAizBZEU0l25B6jssQY1PDVxYY+MS1UjM+U6LcdFzLOabEbOG4XF4zw7Qtqn4rildKJYYi+KTPY6agg/VB0mNY3QYcxj60uO1gBc2KkocXhxTCZquhMkbLENzssV5u3jCjxLkSYw+oMsTcjJ4huzpfush2MVUtaI46o+STlslEhAczpmCnJ04vbKOupZcNZXPqmiF4uCdLq0Irta/uLrz+DiGHEMBdg1JhpHh4wObbyEdwutwnHvFQUwkgkjvGNXbE9bf5aI5tlsTLbJPhaRayeCZk7S5l7BEwiQZmEEKikaLXQJhReyvlqTWqjHmotwsyqorXbZdFO3UuWfOy7iusDBqebDDkjF/ZQ07nSR679lqzwNeC6yzGMc2ZwIW4lzpFVwPdEbNuVnSwva0my3HtzNVWRgJLcq645UzMMiKocDlKtsAkGZV8SpJoW+IjZdo3UFNW+UBdfVz9GixuUnSysM1VSKe4F3XCsxuyi4OizMNRKyx7WjVIzdiqrpfdQy1DWjRyxxbtclkYWHVZ8yhNYVHNUue1dMMaZnJFUCOxdfVQRSPjJyHcWSc4v3QZQuzmsRTPy5L6K5TlmQh30Wc11lchqWM8tgT7rMwRK7E9rvIEnOs4qqx2V5edrqYyZ7rnMNxKdpzWspA5UGve19s2iuROztzLE4tRI8ylbsoXA9FIx2ixMLA0kh1T5VzbxPlCdAkT3Upok7d0wT9VWBI0zdkj0+alNRFHQncJDcpyLqshcSopIjL6d1MW6JR5brYjjheHDM1XoW2UQF1IJAwhqItsbcqURiyqRzEHa6txyhwvsuUqO1gkhL0Q2uo2NpsFPFI0AKqpGmyxMC6HgpF7bKoHlOXaLIKSZvpuoZJM5QP3UWchBabsmOUhQ8z3SD9dlsQTR66KvJH7K87KVXkasCvyz3Q5W91O5tlGQtiqW6psvspiLIC1YDBuuqYsBR5QnWxAWpsru6sZW902UoK+UJ8nsp+WllKCINUgHQJAdAjDUA5QllKkQIWICyJu6FuydATdkWYKNMTZBLmKHMEAJTk9Sgcm6ZCXJZigci6kYLAIGgkhTAXQPlCFSZdEDt0Au3QG/VFlunyhBDlKZS5QgLVpApspRZSmWVNlCYDVEkgLME6azikBZAsoTogwJy1BE3dSA3Ubt0Ybogka4og9yiDUYGqCdj3uO6nEdwghido5Tg2QRmHsoyxWXHyqPKUFYixQu3U8ob8KhyhAOV3dOGqQCySAQ1WadouEDWGymiFlgWhELelQSQ+YlWY3aapSuaB5QgrMj9kXKuk0+ZWo2XstCryT3T8j3V7knshMPsqKBi9kJiVx0Th0Vdw1UFdzbKGQFWnDoq8gVFYi6jf1UzhYoHjRVhWcOijIspnAoS3RUQkXQqVwKFSgwFk4HQJwLqQCyCEtTgdApuW1CWrQjQu3RkaocoQCWoVJlCEtQCkkkgSJCiQNlCcNThqIC6B7N/KkGNRJAXQCI0YiUjGI+WVRAW/lTtiJKnEN1YjptAkyKmSyIRK8KX91P4a3wqWKXLcnEfsrRgN/Sly/ZLFcMCMReylETr7IxF7JaUg5PZqidG6/matBsbQEMkV+isSqiMrETXZzZuqlNM93RSxUnL6K2K5i02QllvUrMhbGq7pAfMrDMopBYKrIbFWpJGkZWjVRCIHUjVaRXBJCbl3Vnls7JiGjRoC3DEwqmKybQKSRQn8y0yPO3sgLkJKAnXUrVFpMwU8dS1oDcyp3BCjeCRlzH6JRbaZLA/dKughNI7yrBbLVNPkJKzMbrcZMTmPzOj6ABefNqGbURvNYXMfG/wBlYkq6qKM5pIvoFgtYGvzyF1/mnfLBM7kmUtB3JK8ueTtEFU1U/Mc5kxHyVR9dJez36qxJStEghgeZHHa6uQ4NAKd89dT3ewXtdcZOKtgmGsxWoc2eTKAL/Na2I4PQ0UGenDg/oS+6yaeSMyh0I5dtsuiixDEJG/hmR6rTJiElQ8GlqpWz/FNKdAE2Etq63Go6SaqrKylF+by3EXWTiWO1WL17bCKniAtcbrosN4rh4fw00kAimk1tKPf+9ehl3ph4U4cpY8RqKU09r2BJJC47iP7WIqsOpcNomGM7Odrf9FxOL8SYrjL71s+b6KlHy95I85Uht0R44xWrgdQNoaNkUgsbs1ValoIauUR8yOOY9XmwWJI9o8wahmhqxEJvDvydDZUdBjmFOwZjHzTQztf+XcLPw17SS+OYhnVjtiFjnnO9ZlHsSVdpJ+Sws5Yd80YbcmJ4bh9MDhckniybFlrgLCBfiNQM5jjJG9kzg+aTIwXcdgp6rB5aSmZVTSFrnm2TqEFzDpaXDKwsxmCeem0LWtkIAPyuifWcPzYlzJGSw0zyBdouQsQEEW85/irGH07KutipnzCK5uHO2QehUnEBwDDxXcFcS1VO1nqgndcO+hUf8++JeNHVFPX1tZK3lkllKy9/0WLV0s2JvdRvkjfNG20bmN0Pst/7OcH4lwSWTFKSWBro3GKrophaSQdwhycVibamkeYJaaSJvw5xquqoMPrcZ4YY778pIIg3VpGo1+at8fwji2q52DwzU74BeZs7Mt36aDVcth4NTiFBhOJUwpryhkridHDsqlqYpLteyGAyysv5m7FQw0tRKznBnoIK9Q4pi4ZwKiloMAfA+V8ecsDrn5rzmlhrq2B9KatkDWWu1xsVJhQ43iVLXtgjjp5BLGP0VLklkXOnBDeuqvR0FJQS8yepIlHpeDcK0yCGtw+qq6p51Hly6AjuisyeWhdEI4h5+6pPY4/ErsWHmR0bY3ttIL3OihfDPFXilZpI0gA9Ao0aOWRsXJKUAzOZ9StDF8Hmw4RPMjZHSbtbuFmOdJTybGN7O4VEtRLI2fJBK5rXdknSz5hDJKJLbEKJ0hl879Xd1JgdG2uxF9O572OIuLIBq4iYOe6QuI6FH92TDDmYiyQSMIu9o+H6qWBnNcWVDJORch2VRvpHisNDhU8jm/CNroE6KkfCx9KSS/oTdUnDK5arsMNNSOnrY54HsJF3HdZ2S71JGo3C2w0EFcZLicgEW9KOelga/JT1UcvY7XUuGUbKyUQVMkkMUbSRnOl1XjNEMRnbXWaxjsgsNz0WRWnqq2ikdAyd0VvhadFr4VxAGQRQzUTHM5jDPJn87hdZNYaUuMsQEg6XCnwCbDJKpzMR5cbDGbHYXW7SYe34zWQO4cdUcwM/BD477k7hefT8RVeLGWipGDDoni5eCb3+Su8O4nHScNSzBkVZNQSnltk1AjXX4PinDGP4JPiww6JromkSx8vUHttqpKvBKlkDiTFmzXIeSb3PdCXWiDCtGtwmtFS+rdSmKmnlL2lx1AKKdsksAp6d0ZZHYuyaEFUQNrnxUrYGTWYdxbopn8+PCY5GV14s2kThqFnGJwHm6JR5C8NebN7oNGR0lVyuXzo9LEk6fquwouL/ALuwuN2HzgSxHlOpiLhwXJ4bXz4ax8MLObBKbOY47E9Qp8FignqKmCajkdT2zu/1jff3WB6fw5xbSxYlR1VLOKaarFqiJw/CmHuej/8ANeqYNi2G18Us9FUwjkNPMhBsYj8l84YlS0uE4nTUNcx09HJEJGuI1aO66DhShw+ooqrEqWScPzGItbMfxQNhqrEjvMKwV4Y/FZ4Px55xUNff9+/+a7OPFMPjxQ4a8kvlbzWADb2XmWGY5I3hmSonrKmVkd2upmEZ4R/itnhx+HwYrTR09TJWzSwc+lLZRe3UFaHdzSRxtLnO2VeOdkt7Ln5cRxvGJzHh2HCFrHftTNv9FvnQAn1nV3zQMgRoXboI0kkkBqGYeXVTNczNZ7rI3RgsPlugoP5Yj+aGDI3YpS0sz3esgjayZtPIywO/dbYXWSNd6UcmrPTc9PmghiEYSqHx5THJch4tYblYybBLVmEPbJNYsF7NKeKfM0PBkGfbN1XB8RY/NS1BbNlno4nZJWB1iT016qjHxlijqKCWllET5nGJ0cg/CA7rnI9Mrq5uH0L6ouZeK12k2vdYmN1+EY/wrUT4lTmOHPygSdWkm1wuUOO0kdFh1P4gVE0AeJAy5FwffVR1OPR1s7cDlZyaKotOGkFov+iox8LNLQVknJqZaujj/D5ZjzCx6jXVBHh+CVmKz0HOmjeQPD6Zbv7HsixfDX4fKypwySWI62dGLgmyycHmrXVEmIywc+YAx/iDLY9dFxluJdTT1kGGwVXj63kTPh5Zj5gJv00UmG4dhs3haqTGJIc5YY2GY+aQb3XJ0WC1WL1UtQx0bWUkZkfKX3Dj0t+i62mweCTBX4zisk7pGODDHFFffqQNQpEpk2afiejw6oh5nELadsshLY7cwX9yF22B8VYZi04oYKmOWZwLxyBcEd/ZeLVkmEVmHT1EMfJnpMhhINwT7dl6Z9nVW2swiLF5oYoqwtLJAxtjIwdSF1jJKduTqmTNkEjQ/uEN8xWkDKLgrMrCyLV7rdvcrWkZosLFKXxkHLeXRvfcsI6EFbiVpDNPHEw8x1rqtI0CxzA/Jc/i/EMmDV9PhVXG2RrxzC4kZwB2VuCrZBUkvjkFJL54i0XFvfst2lNC1lHIGAXvqpQRMHGC0gDrEgqpUl4NrELcMytRMbLFkc27SqFXw22YF9IQw9Qeqv0D7MaCtG2i1yRyX3JitO8tztLBtqo2yyROLHts4Lp5nZt1j4hRsla58bPP0XWM+TjMKfPz+UpnsuwnMqzZWsOSRhBUodc5WbLpTMq8mhyoFadT8z4gFF4WQdlpJV8iblm11Lqx1iiu0rTKtYjdOHOabtTvBumIIVEwlLzq5SRTvzKqRZO02V4qvnXZSxTcsWVeAhwAcVM6M7rlOLcSsiVh6qVrrqlFo4K9Gzy3XLKG4SA62RN3QAaqQ62K406RISbKN8oulMbMNlWcXNGYrUQza01zbbJ3StHVVY5DbdM90h22WuKc1sT6XT89p+JZ4bLqQdkTXvB1KeGcl8S3+JPzvmqfig3ykpOqQW+Up4ZyXc4Kdhss5tXIDo5TxVTjuU8M5LokTcxt/MoJJ2ZPIdVFzipxOTTjfoFYjdosuKbYXVyGTyrnOLUSvsPcXR57DyqvHLcKTMCsOg+ZqEZdpuoG7o8yxIluUxkUeYoS5BI5zSoybIMxTIHzBOD1CjzFLMVgS5ihdumDkibqqBw6KPKFYd6VGR0KiIi1A5pUrh0QFq2AyhLzd06SLRJwLpkTtkPQ2UoXbJ0kRHlCNuydJA2YJZgmt7hMgSNAkgSbME6bKUDhyRcmylLKUDIwNUCmiHVAcYGnlUlh2QsFkSAnbIMoUiSCLKUspUmRLItiHKExHQqVCRqifRFlKHKFIksKHKUspRJw1AIFkQDeifInylEswan5Y7o2t1UwjCKqGP2ThhCtcoeyjLBeyCNrGlSsiTZLbKxC02QWIIQ5m4CGVljlUUkjm3sg57ygPMUidNFGJLlO51kDnZR+XuiSAcgdrdPSiy/lRhuidoUCY02UrW2UgYLJOZ2WQLSpCLjVAwaqUjRBFG3zK7FHsoGR3IV+NlgEBBosnLWW2ToHHKFsVZmKpIzVWJprEqo+YEpxDPaoZGqRz7qJzsytMK5YT0KF0dgrFtFE/ZUVi1ARdSO3CBUBkQZPZSO3TIARBqdO3dQMhduiQKgHoUZ3UbtkCzBCiIuNUwY0bCyAXbJlIRdMWoGRpAdAiDUYMBdEwWSDVJG1GyDNNU+VvdSZQiEaAFLGLpxDcKRsVuiAmNHZWYhZQgWViPZBMxvxJPGiQOiHMVgBkuUYp83xWTHRJzitBPiEYyhBlCIlxToBy2RgXTJIJLMaNlDLJbZHqVBL6vqqKs7iSVVlzlW5W3UD4iQt4+rnKoCQ5TMc86BpSbEA7XdWGvyjYLcqrPfY2KAkEeZFKbvueqAggKwygmPRQqaTfzI44cw9K3HkwrH2QmN/Vq04qAFwuFYNEwD0KcoOLEDLI+WewWlJSMALiqlS+GCJ0jyGhupJU5QvBAGRg6myyMVrWSxmOmdcd+6zMR4nY6V0VPGZWn4nCyyKvFKgxHK0N+S47M3WEj6S7s0jgPqqc76GJ+WOFsrvdUDNNN+0eXj3VqmZDHcvGq8uU03iOOOUyc98pYegCVTUVElw6ZxHYlNPPHcFryomjmjRcr5NBppeXJ5QnnbLV1LYmRAd3dPqnjyRvu5uZTeOibtotRA4GojyyvHJdHrfIdwkYnkWjpZGjrmXo/2p4TBh8tLWwsa19rGw3suehGMcUU7xHymCnaH3JtdehzcnLE+P1tso2iRzxHGLk7BdThzafEquPC8XpTLY5GiMa37q1xLgx4WqIpKabNCdo3tF27I1EuVxPDp6CNnMIJkFwBuFu0GHYI3Bw/EcQnbI8AgA6KiznY/iTS6Fu3mt2WvLw87GpRQ8OGQvjbeRsh3+SjTn452SRGFmYn95R07Hzz8hg83RPLFVYNiBpKqHK9j8kuZuwWtieAsFK3EcLeXtIuSHaWVGFUQVOHyh7hZ3RdBScPY1xHw3Ljja6PJTu/ZuZ0trr/wWE7DcSmdmFPNJ9L2Xe8D4zhUOCzcI42DA+pvaV3odf+7ZGHm9NVvoaqGqsWmM3seoWzVwUrp24jC/M14va1rL0Oj4don0c8da2Gqp4hYExi9vmuCrMRwxkslJT0MnhWPOU31H1QDhcuL0srq7CXmCxs5l9StjDuLqusxx8zJjYi7mg9QO65cVFTBVCSnmljaPgcnqKh80/iAOW/u0oO9w53E3HdVNh1LX09PLAD6x57dv4IZMBohM/C+I6UwV9ONJeZYO9wVzOCcSMwankY6kEsz/AOvDsr/qsmtxCuxSpM9dUyTFujcxvYLRxTOc/DsTlyOZIYnFmYagqrV1LpKp9QNHvtqo3zZCQxRFznm7lybHUVU9SQZ3A2GwFgn5ruSIw8jXYdEDQy+UoZBlkyNQa1ZWf0GCkkZmezVsoKyJ5C+TmOkObunI0UdiTYNuVsSU887ayGTnklhuOYdAtLE2vqneNMkOoAysN7fVY/JkcDJEL26K9NVxyNZy6XkuAs5o2QDAx89TDRD+vkEZ+S7XHMBbwditDVUEJn57CHggDey46gD5a2HkMJmBzx27hdpS0HFXEZZUV02ZtJKQGuaB9AfotDk6+CqoKvxEwy+JJkDeg9v4qsZZDUeIjn5PdwOoWnjVHiNXW1DyRJyHcu2xH0WHPFLC7lzNseyyOjrK2pkhjGLR+NjIuyQ6ELDI7KeV9fJQQOkH9GZs47lNExmZr5BdjDdwG9kFmTF6uaJsXOZDEwWIA1KrxU4ndlYWAe5W9jTW11JHW0mHEMyesb/VYTGj0jdZmBXmaIpTG3oujweobSYcI6+hjkaw54swtYE7qTh2OetYKSLCnVbmyXcTvZNxTiIGJ+EMEgbA0NI7qwOuwfB8OpeKJ6GMBtLX0mjSdiVX4Ig8DxBi2BTSDKWEWOmYgqpw82sp2sxjFKSSelg89O7N+I0A7ALS8XT/AHnUcUyYNHNSzyAxyTuyuB9gqJ8T4amOG1lPVw8uCMcwS7knoF5VTSPFQxkEZOf1W10Xd/aLxi2oaMKoKgiExjmsaLXPULFdhEGFxRYhQ14HMh1BFwb7oM2LCZ8Rrfu6lcM5FwXaBVKjDjQMME7/AOlRvMcjLbe63OH6ilixCfEK6cNfHGTHcblY9XiEFZPLPJH+K9+rroIZC8BuQ2sQVqYS2lNYyfFK6WlZMLtlYba+6hqsNqqLD46iuj5QnF4tb5ghlo4ajh1ldzm54ndfmg3uI56/E6ikfVRgTsaYxbqOhWPFU/duKRQ1Mjo4mSAyhpIBCu4fSYhiGF0mIx1rWCN2R3M3aVYbTMqr4XXUZM7pdKxguL9EFzH3YdTY9JxBwnO+WGPI+YHUPuNvdaOB4jDgvFuF8WNF6Im0rGH9nnFjooZ2UmLcMswvDIYqSugPLlgYbiQi3p+a5+sw2qwnC6aoqHva2R2RzG6aey5j6LxTD2TRHFMNaNAJI5IjoT1Vxx8rT3F15f8AZJxXQ0tFPh1ZJUARu8r3OJHLPcdLL1F2VzGyRvbIx3pc3YhaiRE7dBmKkcLIVoJQyyOjF/0UoFknNa4WeFLFOES1bgw5hlO9lqgZWqGLI14Vq/kKohdk3coTlJuFMR3UJDQFoE3ZIsBNwy7rWCbMU4dYghZHjeIUdLNNVUjs/wCHO4SEG1iD291Vw+qe2N8eI0ni6UNPKLQAb+67finDMGjnqKukZy5gc7n7i1/OP8VxPFOEycNVUM0GKfeENTFcFu0TzsD89f0WMml2jpJ8Jq458Okjp5iw3MpuLWWZiePY3jtO6Wt8JFStvq2wB26WWdh1RHWSOqsbjfWPYw+QeWS/+PRVsQomMozJAJYmj0wk/wDNlmIYlp0XFlZTYMaWKYOkY5nhyR6R1/wVnD8TxKWWowiso+ZLUWzEnRptueyn4KbS0+NYacZwrlQSQviifI3Qk7EFdC7A6DCeJudhtWC2rhe+UPkFrdACE4s8nJyRT0lfS0GH1eUSG8lP6mtIPXuF3Zbi2E1UFXJiNFSROjD62hDTqL2JH0XJUUkNJWUWG8YTWo6h7zFLTn8QPvsSOmqsVlPzsckfi9a6agYWQwOedRBtcHrYqw3Crhs+DVJx2khroxSkcynaRYuGfQj5Cy9O4UwesghwuSqp42mCIMc0HUC3rK8prIsPp69mDU5kfURymFszAOWYzra69C4CfTYHiLQat76bESWQOLtARfT63Sh6K4siYXvNmjdYsnEtFS4t91zfhB4/De87nstd8jC9zG6kbhcP9ocNRVwRTYbh3NfG7PHM11soYLm56qwsOnZjlK+pqKemqY5zFYyG17EkC11SxqUU9bBUuka17LxHUXsdvmvKaXjfFTyI8lPknkDHmMeeQXvZaXEuIz45ikFfHn5JjADXCziQb2t9QFiMmmT9pGKMrp8MYGumqqRpFU4DcX7qhHX4jDSUsba6UsLw/wAM46/L5arpW0r6vhXEq6qp4IKOOJ/ILTZ5kHf6/wBy5sRUVUymzzBk7AwmF+hlYOx+qsyTD0jBcUZiEoqmBsXLHLmZa1zYahXXHxLy1/0WJw5hfMEtTQ4pH4l72CRj5Mway+m2+i3jS1bag2geLOtfoR3XXDNiYFDE+IizVoNN2BJkTdGuGqfl+63dpSpUi11kQzl7jn0I6LYrIHywuYBqVmto307XyTN2F11wYlm4hTZnuljCqRxTR2kdoppq0u2dor0MTJaIPcLld8ZcphQMhTtzFPI0sJJboE3OzbWC3DIZMhPm3TBgItlQ8txd/ipGNyfGSqisYHglxOiTwB0Vl2oVd7HLUSnFCWIQLqYsf20QOCtlCZmursUhyi7/ANVTYbFSg51jJVu13Zx3VxhGUKnAWki6uM5ewXDN1hP0TO/wTgaKRsWYZlzdFV4cVWn7LVELPisqNXDGxxcx4N1uHJRDrHKpmG4UfLub5VLHE8aO2XWfRJTNY3LayTY9dWqVjADbdTlnlC522oT0rJRfMQVW8PJ8JWk6N7tgm8O/4QtcmeKpFTuvmcEborDaytsieFMadrh7rPNpmtbYZUYjJIsNO6sPpSDspxADEGFOYrNheHC1iPZXoYnlqjjgEegV6nZ5PSuWUtQr5XsKmjD1YfDZpUceUaOXNtLG199kT2uAUkZACUuossqqlyAnqVI4KF3ZQLMUOYIUkDZgmzoElmFgfMHZOJf3lAXJByCzzj3S5gsq+YplEWC8EpEXUScHVbCdumRdPogs5YBN3SdumSQJFlCQajAstraLIlld3UwabJ8oTFFe3uUiFIRZNlC0wiSIupC1ARZAySLKEeUKU15osnsnDVMGp+We60IBHqpg3REGNRhqgYDoETdk+ROBqgcNv0UnL1RMFlK5nZRVYsQ5PZTNa4pi2y0iuWoCLqV26Gw7IyDKEGUKRJGrtHlKNoSyhEBqoyWUp7BOkpYdo1UwFlE05SpmuCrQbFA4gbqY2sq0oJKBhKCrEU7QO6okEFTRh1ggme/OPVcoHM7JwLqZkemyykoGtKPlkqbI1Z+OY5S8P07JJ2c2WX9nE06kdz2RpcyORBjgudj4xqpWCRmDtDTsTMFBP9o1Jh7Y2VWDSSPeL/hzN0+aDq25lIGOXO4Rx/g+J1UVI+inpXymwz6i667wr/yII8oTojFJ+QpuTJ/qigYNRt3TNikHwO/REGPB9B/RBLFop+YAPUoQCOhTlryPK1AfiY/zKKWcOGULPlD4jq0j5qNtV7rTCWY3JVaR2RSOnZluVDJLGUaISE9UOdQuuR5HgJhcep91WVgSIJDdBmd2QEuQC86oETgUKBnbJN2T2uhylAnboXbI3bKN26BE3TJJIGduo1IRqllKAMpSylEkgbKEsikylLKUABmqMNCMMRBqBg1SNZpq1IM1UzW2QCI/ZSNj19KmbGpA0dlAAj9kQj9kensk3ZAwi/dRAORXCYEdEEoamI11S5jbepC6RndICdumuEHMJTgApQfQogLpWshMgvurQIiyHOAoZJHkFrSqUsk0Z1KtMtF0wA9ShdK0qi2oc7YqZj7C5K3S0kfmPpSEZ+JROrWx3s26YVLpBmUZTiNlvdQvjd8LVNCCdXqwGx2zIM0NcD6EnNc74FfeWAGwVdz231WhCyka8Xc1EYOX0smdVsYbZkPimv8AiRlZZuETyAFWBeeqO5+Jyy0GRucELHx2ClbRnmnmb6dFslwtuFj4oxrmv5jQW9rrGWQ84cyT4NFpw1uDwQiCQc2XqMqWIwlvnbCW37qgIc6807HTEE9PSzTGSMcsHYbpvCADy3UstBUQx52R++6rCprW7kLPJpTnhdG8tabpROAFr6o5DJK/V2p3Tx02YXZr7lc7Ebowbm6iEbXOzE6KaQNZ6j+iiLw0ZRstxK+rqvtNo5sRr4KSP0Df20VvB+F4cEweupAcz54g9rrroazDY66tc+brstB9FFJCGW2Fl6nFxn2fcJUVHDJis55tSSRv6Vj8T4e/F55317JcoNoXMFx7arr8Ip/uGWeOeoBhN3k20C5nE/tChpax1Jhro2wxkjO/UfRRYa/DPB+H4NRGqjF5Z4rODmA2KscB4WzD46upOr5JCAbLj6HjmPFaqVuIYlJFFFu30h30XQ8J8Y4bUCpZECKWO7xOToVtVL7W8AhrsPGLQxATxEAkDUjTdZ3DOG4UeFo5qpgcyP8AaEm2pXNcR8aTV2MSQ1FXLPQtJAA06rArcTD2Gnop5RCfhOyDewDG4MOx+qjpLGlJJaC6396g4okwqZjGUNEYZxIC67r37LmI/KcystluFgdDVcY4iaFtDQw+DZa0hBuXfrsufjramIFrMhub6i6cuuEMceeTICLnYE7rItTVzcRDW1UkcT4xZrbbqqWvFzl8vQoJ4ntcQ8WITium5QgswgbXCsSBc/TMELZNLo5Ia57i/wAMdeoVfLJG7LJG4H5KtpZZQT8k9nhgkdFJkPxW0K1eFMEfieLxB8jWX2a/Zx7L0DF/s/xUUbTzoWxucLsIsB9LIPKG3lkaxkchJNgLLfqOFcRo4mTyUFRci+Ut6L2DCfs2wSWOKKryjl2PLbYAHug+0OqwbDcFdh1FVHxkbbNYHXLRZB4RKxrnksZYdr3UcUr6WcSR7jZdXg2D4jxjS1UFLyIn0jS+5Fi76/RVsHjreFq2HFq2hc5l7FrtiEGK3PV1YYyMXnIAY3QErpKPgkmqkgmZPG9zdPJcfqpa3DX8WOlxjB4DA+I3yiwsFv4TPOzDGT01bLNWBuQm+rT7hBg4VR4zhNPK6TkB1O/ywOj1IPUG66LDJqekwytxIVMclXL/AFDTpHf2WZh7+J3SST4py52Fpu19mlcxPBW19dJ4J8jWMPmc020WbFbE6nEKfE6p/PmD3uvc6LNcZpSZJ3lzvdalU+fnMfPUeI5YsHdSFQm/FlLwwi/QrQMTzmmFI70KeDliSPn+gvAf8rqDkystzBZTxUnjmmk5gjcQSCfZB03EEMXD+fCqF73sq4xJHfQxg9lz+HRZqxjJmPv2aLlalfjT6zCqWhxEOjxCiNmyNjvzGdFewbDHuoW8RwV/LqoiQ5r230/VYAYJiM+GYhUxskkpmvFxmj3C2cOoo8flEFLTNdMQZzJLGTb2WXj0j8UZBUMjMT2eS7tr9l6hw3RMwnBqWkcLPEYJ7m+qrDkqqpdGZ6SoopmzCMxjKLtHurMlNFVfZ/zqgW8PGRAdyZBtouoipI6aeoeGBxnO5GwKmw6jpMPoTSmMGAG9j0Wmol8/1wfVavjn8RvOXDqnp6SCkqoXiTnNO7QdV6hjPC9PLg+KV9aL1EhvFldoGdrLmZKGlrqBtbSUTGtp2BhLtNUVz8/i6Z/OgZlH5XN1VemgoatzIH0snNvd2R2pHyWhJNSNqm00k7azObOcDcWR11JBFODS0kmT93UrA3W1HDmP0n3diNLNF93xZKc3sXdLBDxDwXh9NSYLSTMEFdPKwSsYdSw9/dY2E1UjMUgqGPAq6ZwkiLzo63wFeg8PS0vEmKu4jxandnE/KDL3EQtoD/z0WuTDzjHafEuF6mowiKYijkGcNPYrZ4fwp2IYQ7DJp5KOsfaWPmaB412XT8cYbR1dfTUDYo3h0rH+JB1aL2yP7hN9rVFSYfNh0kcTDkoeUAezLAJXJqZcqIK2KVhpGNp6qkvlfA6z/wBVoV/CWLVuHYbAZJH19ZPdrC64DDrd/b/it/7N5IJOVK9kT31kN7GPzNLOg/VdFxBglRXYGcZp5pIaqglM8OU6CPS4TgxTj5uGJ+BGtrqmqbUOrAaeenZa47ELqMcjqME4UpKuJ9pWZMrhoRdctxbi0FTDhlHAXPl0fI5xuSSRut+qwetoauroyw1FLV0fNp+YfS+wuwfJYtuHd4fXQ1NBRvnmAmlgY8g9TbVWPquEwd+WhwGSePmRsjfE7TY7LtYDIzNHN8Bs09SFq1WBskRoha5GtFoi1yCR9RH5mG4VhM5mcWaLoKwrbSZJdG9SrM1mbmyyKzEKSjlMczbu+axPvLFTkiwnEGNiF789o0Htfog6oygtcWHY2SbN5HO3cBcDus3DpmTMEjxUS5/XMRlYT7BWOZr5UHO47xEKwVWB108dCJIxdjYjIdehsFxOJt8BhFc+WPm04kY+MXtd4vbQ77rs8YwXEqvEH4hTVQ5wgIawbuNtFzcmF4xiWL0uEcRctstRBKIHtHlabDW290y81hxLIcWxT/QYHTPiAkNugVbxuMPlOI1s8kjb5G3GgI6FdeeH6jhRpbX0M4Y/yOroJLxuHuOioMw2j8DVUVPikUnMlfIGM3IXOY4kq82P10FJS4TV12Xw55kYkbpH7DsEGEYpSw1NLJVyxSwvzwOayQ6e9lXxCqw3K0NkD3BgY7OPO149/qoeFDSP4hoJ61n9FEvmLW+yvJKdlQ4BMOKMNa/NU09dC+SIlmrdD5LH5L0SbhmKvwKCnxSiAqaanLIyD+zsN/e6y8FjfiPFFVXuq7HD4BTtLtQS/qFT42GLYA6i+6sREz6xz43Nf0sB/DVLXGF6iw7BOH6eObEGQBksZEbIY8z5Adz3vtsudjxjC6HDm0NLV0kzYpXy+dr45IwTpYnchPg3FUApZsRxd04rYw9ga08yOMA7N7X0WdHR0vG7KmbCpBC9lUyR1PJD+LY9Q++3tZMl4uypuMMam5NBHSMm/C5ktUB+JyNd/dcNiWPY0cOZFHWT1GD082QskOUu7gkdFsfduKcG4ry3yCTDqyIx5SNSLagLC4sxCAQUtIKX7vp52+bLJmaBfqB1WYlm6Uavwcr2VdBQijexwIYx36hHNiGJVFU6aKGS0v4bcwJynsLdVBGKpspnp66mm5H7ONuzr6WPyUlHDikGLiqxGqgZFm5haybQHuBZSYWJa+KYjVUWCUmBYpSScl8pklDma8we/wBViTSR8kzy08ghicOQTuGe61sTqaXE8PqZ3TyVQicJDnuCdbWusqkyU2Kh0jOdSkgmIEyZCRpp1ReTr8HxqLhttLUUVLHPDK4mbltuWsPUHp8l0sPFkGLl5oa6rpHxXDRNGGiU9hdeQVMRpMSfV4fJLBSySZAwnUdwpqerrZKmmqqh3MpqKfPGD6gdNFMZpJevYbjdfLEPvWRrBLkkp58vrFtQfqulES80wzHqV1VSV9XOBDE7lcstuAD1Xo+GYjT4nStqqY6P911jJBuiNlWmg8psN1oZbpnxXWkpw9Tw9Vhz3U7Ltve3ZWsNpK+GC09PIGgkXsukeMj8qkb6V3jYxwc/PhjHRO8pusWoo5IBnDF2b2Zjlas+swiaqvl9LtwtY7WeLlRIR6k7JrrUqMBbFE4tcc1r3KxBdnlO69TiuRyNvmU5lYRZwBWe2R1rXR3ezdapVj8P8qjkp7nMxJhzKyw5bFZ5CjJEYz5tkIe5pWrJCyWK4WdJEeZaysFJIXZ3DVbFPBG4DO4qhT0QymQvAt0Uwq2xuDN7dQuWa4tVkcTW6NRtIZdZja1ziA12hVllPVTghjwAe64TDoUsjSC3Ms2WIFxIWozCKjMXSPDwfpZWYsGadyFefEZEDTk0COzhuCF0MeDwAghmqm+62KeJC05uMWcNFZa5h+Jar8Ka46tsmGC0++U5u6nOFpQaGuThndTSUnKNgEAjspag5SJkZRBqlaPzK2I+QeyXJd0CtgMsiAH5ViylTlHsp4srRlcpjGCNlXeNfKl8lhYLrtsqLpss2QqcF43Ub2Nc+5UVPGdFPy3ObmUDNBupmy2blWZVE+J35VC9llc9RSliFrqDMc0oMjlce1qjLVgVS1CRZWSxRll1VV0kb48pQWsVEJN5u6WYJZihR1K2PMMyhbsp43WCAsiEjoUeYoXboIsoRhhKLKVNFHcII2t1Uwj9lLyG/CE/LPdbEZagI6FTFqAtVEJHQoS1SFqEjoVpESbKEZakG3U4shyeyViVYZFdO6KypiijGqkyJMb1UlnIAyNSDVLy/ZOIv3UCy6IWs1U/K0RNi62UaRsF1OwAhLI1N5hsoBLQCo5MqJ5JUL/UrDKOQqHMFM8WULtlsNmKMC6rj1KVr7ItjARNHRCHtRB3ZSkEmdumBsk43CnFsJN0+dyiJ6pA6q0ifmHumMqG+iEi6lJY7XN1JGBbdAwaI81jZKTknaz4lO0WChjk0U2Yo0MNJXj/ABViU9XjtXVVMMkbWO5bcwsAwbL15slk0vLlGaSCJ5923UHhXPEx0u4+wurtBhGI4lLyqXDqiV/tGQP1XscVJQteJGYfTtd3EQuocRrJYGlsVm/JOBycngf2dzgc3GKo05ZrHHBJcsPclbE+AysdYcT4l/6if7zqDu8oTPJL8ZW+DnZR4NWbM4rxIfUFTswHE/6vjCtHzATUwkMgyi60I5H31ZZYnFqJVRw7jXw8YVV/cBP/ADe4jG3Fs31jW5ALtBVgbLLbnjgfFP8A/GDh841G/DOLm/s+LY/rEF0b1SmNig5+opOMo/8A9poXf+SFVcOMW/8A1akd84l0EvnUPIJ3Gi2wxOdxhsKqhd82J+Zxn/8A28/RaszRFsqk9Q8Rmzl1FA1HGbfXT0Dv9pN948YW1wmjd8irjJ3uGpuivnWZhLUBinFY9WBQf7MlkX3vxT//AA63/wBRXo8oO6ssdcJSsiTG+IgPNwwPpISoP5w4+D/+qsn8VvZinMrxs6yzQwP50YwPXwzKP1QHiusHr4bnH1K3jPL0ch8VL3Shg/zteNZMFq2t6lSw8Y4LI4Mmjnhd++3RbLZ3u3APzCGWjoatlqqkikzd2hShFT4phVYLU9axzj0U2TW6yazhDDZ/9ELqd57ahZzzxLw5IM58VT/8/wCaDp8iWRUsPxqjxFnkfy3/AJXK+AUA8s90gzXVGnbutMmDEWVvdK9k4c0ndKaIiyHmNHVO86IY8r3LVJaWPM4jTRWWs7pRgWTueGi5KzMFpAbJy9UzWxMOpTira7qrxVbzApZ7Km+qy7FJlRn3TglrBmI2Tia/qKqPmAaSqr57+p5CcGbajpgBoVEZXWve6z28w7OJUctW9htmW/DOTTZUG+pTvxGGIbn6LE8TKT6yizPf8S1xObWdioLbBhTw1XNBOo+azG7q3ANAFOCWtyVkcIzav+SqSV8cgs6JSSMjDSXhUyGg7LMYljNSSLMaFJFBLKLv0CjBY05gEElXMRZp0WqLXW0sXbVGI2t9KzI6qdrr51bjqbi5cpxLTmVwQmocNnKJ8rLbqAkkpRad1Ye6gdKZOuiEhg3QmRg2Wiy5dt0bSwFRGYnfZFE7OcrQFac1xsruilLrhVBI1m7kXiMwtmsszDpChXTvDnBpWXXVswvDHBI8fmK0sRjZm/BYXufuQFg47M+GAsDXBw3BC8ebopVsrpgS54+qzC7KtChpudFz5x5iqeIEmUjIBbsF5ZdIRz11W9gYx/lA2VMvePK8aqYaeXW/ZKJlMZT4twA7Fc7JlWtY3OyCSTmN/B27q9VDCsw5L7nte6jPI5ZytsiKLKWR13ySNAVeQMjeWh4I7q3NltYIAW6MyA+5XSCHsjAL5zsFk1mNSVH9EwmMzTH4hoArtTeaIw3tdPRwR0cIjjave4uYxCOfC8Gr/vJ558rbBpN9VzmB8A58Fmxivhe55aXxsLrW31K6XjDC56uanqmO3cGZei6Rs3g8FayAWcYrNHuUbfNNWHtqpQ8Wdm2XouD8Jvw3hKXEn1sjZqlmkewBv7qlW/Z7jcuPxumjjLal2e+fcL0DiZjhhFJhzMskoDBYHeywPFaXhuurcY+6o8ud5JJGuUKWi4ZkdjxwSvltYEh7Nr2Xf1UTOHcWp66Si5j7WIi3XI4/jBZjBxClp5KeXpnGoWsYbYuMYRDhcxiZVc0jf2Wfms3Kp8Qq6isqXVFRJme/cqrfrdSYDtcWrRwqGlmY58k+V8V7aXQx01ViFK0+H5TB8Q3WhguCVVbM6OgJa9guXOGn6rmwz5pHSuMjze/VKOsdTtsyGP52WjivDtXhRZ4qpjeXm2VguQs+sw+ekjaZ2Zc4u33CAoqueVwuVtSPw5zc4NnDa6xIYZGRtdZO2Rhk5ciy2vzVs8FVAXsAgDvM9ulx7L0qjxWOjc2lOKxtw+pjvG+pkzWPYLzGOWKSidQgXubt9lO6unFG2hkylo7jULdj2vCYDgVBJWvrRM14JzHqPmvN4MKHFNXJO0FjpZCQ6+jfn3WM3F8VqoG0j6uaSNlxlHZW8LgnMrsvPjawXJaljqK52EcD4dUxsnYaqrhsY2jX5/xXIHHqyeiigFIJIYzfNINVBidfVY3WBtfJJI6MWaX7gI8DoaOrrTQzyPjuPLbqeych1PBvEGF0VK+CoitUyuzhrBoQqbMTxTCq+qgpKGJrKhxPNYy72j2CwnTVOB4sQySz4TY6bhdBHx5T0dTHXTYJOXxiwewiyWKtXjOHU7smGyTyzPBDufoSU9FgNfNy6iGppYxUC0kdOfOB3VDEcfosbxqTGosLdEHtAe0i4+a3J6n7owuOuooY5Zfhfa1inEZOEYRhdNxXJg+M3lhHkjcdPPbTRSfaBw5DhLoa+GqjMIPL5NvM0W/iqlbNzo31872Ty1FnvPUfLsrGAVMc9ZHX47iAqhlyQhxLsh+Sljl6qrgqXNMJ1AtayaCohjlZzxdjjY2WlxXDh3ixPhzGR3vmY0WDNrf4o+DKrDIq2qgxblCGWAjNILgFWJBSikMk9c2uiPKb5eoI913OE4xhWI4A6unhhgihADmhwuTbdeTwMdBV+Jg05chIvsVv4fycWxJjX4dJLzb5oInWEh7lQdPQ8S0OM2q2U4a/D5w/lD4o9rr0mR4q4I6+PZ4uB815TiLcGwrFMOZh0LqM8wwVEMu4BtuF6rTwzQ0raSBlooxaM+y2kwirsUw7DZI46uUt5gvfLcD5qaT8ZhZB5s+xOgUFVgcFXPHVVPndGLWOxV2HyxtafV1CK5X7QJ58KwelggqiwzyGKW35CF5/XzTx4VUYZS15loo5b+Rv7T5let4zgdHjJdJXwCQMjIiJ3BXP8M8HyU9C2nq2WYJjIHEAkjoFB5VUyU8bIX0kJidEQCXbn5rTqq98cjHUk2bML5LaLV+0PhurZxFUT0kEZjeAcrOizRg2JYTNTfeFC+DNtc7hc5gZRM00/jco/CeNhay9NNNVUVPh+JYTQyT02IRCSemjOtwNwuDjw59Xi/3NTTRxCd27jovYOEKCuiZAyuZIIooBBDm69CfqpQ4bDeI5cEifIKE1bTIXxy5gHN7scDup6bBa/wC1LG/HPZJSYWAL3N/YgWWHVUeK1eK1k8dIJPM8vaTo0XXrWFQfclHTtoqaQtqIWERsGnMI/hddIlIcZhlPR8Hccugoo7UFO3lua43ykjUr0qrkhrcHqaWhfzObHkbpbdec4/wtinDcdfxPiWIcySpOeOnjBcQSdif8VW4cHFFFR0+K1mImJtXKGUkEjbvkBNr+3+KWtND7TsK8HJgdXS05Lw5kLra3Isu04jon1UdG+N4a+A3serCNVexPBvvCaikn9FMRKRa/4iuywCWeOTJmyDb5/wDsoOTpcNfFRYRyvM3xNnN6XuV1UrfxCqNFBy4RBIw5RK97bi1iCr5F1qBEibsmIsUJc0OsXarInAumqIp5ojHADd51INrBRSSOFsgJ11t2U8DhWQNqoHNIB77LY5p2JUT8ddhsfLNRBfmRyN1Paylxurw4UBgqBlqXAhrGt0F91yfEVScN4wmxuRkcNgyQCUkCQWtZc3XcawVdRLWPgdzpLct7N4wBa2u6wO/bifEmC01O+CghrKEjI1z5cpb2utJpr6jDpK7wkUNaRflsdcLgKDiSlpf6PxFiFbKywMccYzRuBFxe2y1+HuKq2q4h8dUx1EGDPj5cD+SeWD7uAWbFF3EVXHX1YnxWNlUbRhoboCDt/Bc9jVRVS18GI0WOVjpZHcuXmANMB7LR4sjxDh6uEdVDHVtlElRS1cYuXXN9QsuSinFFTTwxvmjqz4hoaLlrxuHLcTSwpV+NYpNzoaqoq5qYWjqAXWDrLM+96d84fG98ALcglB1b9FDimKxTtlySGGWoeBNG1lhYdVTdTzVcT62CkZHBELOc3qB1WpjkiUVz3sfTvtM6c3JI1JUNBWyQVP4ejrEFh3U0FO/EqqLwJMbiLAhXqcXwuqfVUQ8RTyi0rhq7XquUzQ2+G8cxehH3lhVXHJUwG76d7dJB7haOJcayYvRzSfdzoZahxuc2YsBPwrk5sGqsIxQsfM6rzxiR0kDPIM4vqVo0MsMVRC2SQRsZ3KzGS4IcTq31E9LTxSRQiezHZNLH99dTDxTBgLIsLfQUmHh0XLkqINSSb6g/NYeKVuEOcPD0Q8TEbxvYNHH3WJXx4jFUeJq44C2Y2IYbgWScrVuSYjiNJirhNiprYI43xtmmdcNz6gjsf81lmSSrmgpcUncWmJ8ERa3Um/VKmo6Gd0wmqMjI4tIBoQQjppoHYnTR4U0BjIybzjRpsVGWzjposKwnCLxxF7IeWQwWzHS5JVTGJsLOH0k8NLeEkidzdHMeLbq8yWlq8N52IyCNtASMxsY5D8lyXiMPfLJHksyQ3Dr3BPRaaa1RLhtPhzoaKaphlZIJImSG7JhdHgVZHRNfVSVfJrGO0L9QR8lnNLYGx01XStyi5brcjsQo6unZzIw/XmC6DpZ2SYlGamtr4HAuMgYwWAJtf+5ZcuHwRRRYiyWRzJATYmxD7qjJDPBTyihfaIjOGdCtrCI456KGkkjnmfTHmSsJsJBe9lzlDCrf5gx8botX2d10C9U4Hl+6+HPxRcCUctw25bwCP7yvPMUxPCsTxJjqfBORFERka0AG/Y207LsuCKmrramOiqoDFDUZ5Gxn4bf8hbwkd+JB1RucLLPqmSUzzlebKaKVr2LtSAlLjJfIbKzELi5QFyISBgVBZAH5mqN5Aci5rSoX6uzLWKTDH4hqDGG5NARquWm1kL+66nHKV9RTvazcNXMPppWH0r36fRwyQte5rrhSvldJ6lGA7sUV8p1BXWXJYhN3K4xt2hZ8TrWKtRTZrBzrLEw0ssOiimiJOdgUsMsY0JTuhu7OzZYaVXzyAFnQiyCLUqWWKx1amaBfytRlNTl7JA9q3aSVtwViQtOcLThdYLlk3g2wQWiyljb7KhTS7XKvtmZlXnl1WWDRH5eyqc7s5OJXd1im1ktBNkJaN0zZLo8zUoQvhY8elVZYQPKArhIsoJDcrVpSsIG22QOisdFZJI0UL3Ks5Aa2xCm2F1G1wujz6ZUoLmdENruzJk2YI0IscUBFipOZYKEnMUBZtE4N1EpG7rNkTaYGwQF5vlTZjlUbvUqCcULrJG/RMWrFKA6p7NKZExURuj02UXKVh2yTRqsCq6L2UTmlX3RtIUJi12QVg111M0GyMRfuowwBVUaSkylNk9lECN1NGUAjU8cdkFlh0QlqkZH5UEg0WxEojupHbKN26CN3qSyhGRdLJ7LSBDAUYh7NRRtubK3HG34lRHFEB8Kn5LCPMEWRoSs3sonFXMLBo1qdkTPUp+Ww6pCNoWRHygfhRCH2UoHQI27o0gLSEm7FTvYMuyi8oKWGQEXUhaoHXvZq0iN/pUMhCsFl/MVFJEbqsoTqFC4fCpnMIUdjutQIi2wuhc8fmUj/AEKjI6zvmtRFgnVBB8qmbUNso2xi10xaGrVMcllst1IZbqiZrdUDqm3xJ4ZyW5ZbC6gNSB6nKpLVuPxKs6d56rca0nJu09UyQANupHSBozWWBFVviOhVoVhlZunA5tEVeqczlzvKqUT2huqtUZZIS4LEw1a5A4kaqyCoQA21+qnita7lzlvkMNejDT1QmRg0zIDNb4ljzRZACqVlNHKC3LqpBPf4kxkHxG6eYxpcPeTly6KWGjY212FaWe/whFE1vVq1aUho6Y8y2Sy0TTNaL5UULGsAc1SlYu24QtB+iInqUkllTk3UUsPNCkRAqjOfRyewVapL42EXWu5uYZVQqKBrC4mUlbhhgT1EmfVygdM548y05sPaTmKqzYf5fIQF6MZclQS5dUvFdknQPabKExPvotMpM73v9StU8rmOytUMULrXyqxDGM2qzLcLYNxdIjoUk7t1zl0Blb3TcpifUFLUFZQ2QjZOiTG3VFDnN0YcHjI9oIPdB1+iTdkGJi/C3MvVYS/kzb5b6EqPBuIJIpfurFWFj2bErouZZZeN4TFiUBlDA2aPztICMtVzdbjUJtgudwDHJGOOF4h+0YbAk6rde+620N8jO6TBnPlVN/50zJ3t2K1TlaasdI0BrdFXjmfnGYpnPe45nuuoXPyPWqS3QwzR8sXKiqqmHKW57LG8U/oVC+QyOzPN1fDW000ueUkHRTRiQMFlXhYx7hoteno+Y2zjZSahIVAX7OUwdYIqmB8ILllzTPzbpFSi5NVs1DXKlJO/Ne6hLwmc5dIxS1hlZMBYFMXOd6lBfKn5t1S0oOUq3TljyGKhmLkTHHopK21TJFD5AGo4JPNmedFVp6J7vO8EgK/HSlzQ1oIv1XGWgzTxEWuqT5M2y02YIx37SZx+qL+b0F/2jv1WeQxi9xRN1WhV4dBTNuy5KouyALfIRvNgo2NnnfZkgA91YY2F4zT6M6rCxLEjSVJZSejvdZ5Doo4gzR5Vjk/h5xsuU+/pdBJWR37LVgxeqkpmx0TWyHrdc/Eb4p6yaOBuZ5sqjK6lf/WgfNVOIqqWKh5lVFJHINtFjUD3Tx890eVp6uCz4hxdMKmneNJQVBFiHNqeVH6fzLEfPzc0MbDI3qAtrDKKOOJs0jxFbcFbx2Wk4tiOIkBzgoKiqpac+d1/ZVJsfgprgPEtvyrlpq2rqJTM829rrGe6Iap1NZxFBRsD46TM49b7LCq8Ypqoky0sl3dVkS4pIbx3b8yVUdXPOjXg/JeXPdzXi6OCJkjM7BZYWJ1ckNQWZC75BPBide1nLEmRrdlQmmnfVGaZ1+y5z+ZuJNnqJXZmU7/0SdhmIzHmeElDO5C0YeIqulZkhhhYP7KiqOJ8SnaY3uYG9gFfDW1PkeHF5GOa4fmUbpsxQvmfPq86qK4HlKtFprtPqNk1mHrdQHINTe6QltsrRb2QtRN3SdG4p2w6L2uJENkFntBHYoXxMe0MOwUhjI6J2seT6SpK2ijpIA4PlcAB+YrBnrsPnxGaeBk9XyxfMwXDSp+JI6yrrqTDs7mQv3sgq6nAcDonwQV9PSPtYErNK8wqMVE2MzyT4k8ZHvAB7XVKSTh2omdJiVdLnPQkmyq1MtPU47JAytjfEZH+cbHVFjOD0MU4FA90bmixtoCsy1CkKKgmxdscPMdRn82iKLBY5sSdBC+0I2DjuoXmriGSR6s0X9Ea6qklLnvFhfosK25cWwfD2NpapksnsxS/zsa4NpsHohSR/mIuuTllM8hkkOqsUkrHAtZ6gLgLFI1IW+KrTUTzmXKSDfos/GsQfXVJje67GdEWBl+eTmC2cE2WXUtmiqXvkjLL90xgaPiIzTsaTqCqc0jhMZGC4KhhkDmmN/r+EopZZY2tzjTutUpjVcuXPmspW1NdbxYg5kR62VmalZSOiqxEJY36OB00V2rxSAU8dDg7LtIuQ4bKgIIaullZzWcvOLj3CtDFa2lcWwSWvvomreK2VdOykrsKjhfFbzxjUqlDWQVpdHT00jja4KwOy8Jh89HS1tW/lPqB8I3PZc/jFMcHxWKSN5yscHi/RT0WJE4RDDO78SCXy/IIOKP6XXR+b1RgrIt8TU8dfURYqRzIpY9eiwTVOy+EphmZ0Y43Wjhk7+Q7DavVgtkPYLMrcKko61ow7zB4uOi1YsNpquKkfyIWRZ2kEjsnwLEn1s8eG4piLYaQ7l40Q4/ic0VK2kjjyS5fMLLMogzPHLLGHMjeHuDhoQFuB0dHhtLPjEmGiqM8Ed+XIDvZTyU/DssMkeJ10dNLASLRA3d+iRqcHxCoczhx8dGyOPmOdKMpv2BW9gvBOFYxw01+JY7SUby593tsSfmrMDz+ofzaFz4IbQg2Dibn6rOY6ymq2PwrEKqhocRE8TJCBI34h39lGxumqlC3TtjlYT2F1p8GTFuOxT7viBLG9CeyzYWthpXSKaCGl+7pK2GXlV8BvlB3+Sg0cdosVxDG5H1VHeWXUFmxC7fhzjmc8PjC6mrbT11NdnNk0BAOn8Fz8vDnEEOHwcTQV0eJuewSSRA3Nj03WZwjNHXVOIYdUsBEnnjzbAhUencLYnjONQ1Bqa6IOp5OXrF6vfddBlIdquX4YhwIU8vIqJIXnyyMZtfuugiL6aoNPA+KsZYEtb+0aFqxayhE0qJtTDLIYWtMTwPRJoVNkcN1RGIaWB5qvDCSV27iLkrExvhmDFcIqK/FjKZommSOzrWK6BFbmQvhPpcLFZmB5fwzh0OLYXNS1UAZiNJIyphedCWXXrlM4PpIJPzsBXJ8P0NqWoJ8k1PJy43DdoXQYZJN4fkSixiNlaGTR8OPhnxOaGG/jnBg6AC5J1XWMiZEI2x7MaAP0VZmmqsNk2SiEz4YpwOcwSAfC4XB+iy8TwdmJVEJlpQWw6xgaBpWox11JHZSlQRQcgPbDmc55vYlUpCJaV8NPHzH35ZLpbFveylxLEJqeVsMFLHJY53O2Nvmud4qxHC8KfRV9JHrzrT5ZbEAjc3SkXqGoq8F5cdXVjEYJH5GkuN4z37HdauG1v3iJ6htuSJHBsoFhZcphE+FYxhsWGPxCCjqHuMlPlnbI762O67DB434ThtFh1mWJIlcRoCkCOqEcM4fJJkvoARcEqiKmlkzyeNjeGGzsvQrZxfCWV8Hh31LBn6NOp+S5maOgwrEMN4cpj5pHEyE/LqoLslZNFA5k9LM2F4I58bgbA7FY1RU4th/DlTNJJHQUcDg8nMeZIPZZWO41VwVUFM+ohpH0mIcrk30DAL7dQQVW484jpMbw/wNDWEPLgGs5ZDZHnTc7f8ABBQpaykxGnrMZxHFIsQeYnxxCUXkaO1lzHD9Xw/TYjFPjMLpKUghzXN0VWWurcAqmwQOhnmYLuuNGfIqDE5aIy09TQ0khiIJc3Xf5rGQ7PGIuHMOlgnw+hlElSwvgn8QHA9dQemqoVPHePxwRMhHhZGN5ZdHKOW5nYx7LJxnD6qjw3Dp66ePLLETC3sNNLrOmhqKfKKmF4bILgHqFyHpUuIGlw9kscOHuaYRK2Z5JLJDuANgud4gx/7yw+iq4JzBPYsn5ZsWv7/Wy5oVFK1rWOhk5bDc37q5VYTA6CKekrvwz6WuG11eTTDxekqyI5Kl+ci9u5Giy/ESQA8p5GcZCOhC6uGknxHC3QSTnPGfMLXfGO4WTU8Pvw/lYgyqE0BvZ2XKQ8Dst47CYFS4RXwRxT0VWI8tn80bALopsNmqmx1kOKxtZUGzmbB3f6rmqfE6qoo30ueOPI+99rrVq3U1K+igw/FPFUzyC6B4sY5D9NQkxbLRjxSopcOxOhwiSJjCAZi4XuPYrnnTRVckeZl3MN7jayiex5rJhIXtaSRcHqr9BXQx4JU4PNCxtryRPG9uylNYjqWPbQS1MAB5ZsQR0URxCllw2KnfAZZdeYRsB/iq0bXyVBJkcyLl5Mo2d3ujimMFI6kqYyIXm5tqsNSjjnfBmyekiwCa2aKKtMcjWD9m69rlSYi3D6dj4WySRTMIIY4XYR7FQU08EkU7HVQhANwwjQ/LstsrlaPFU7AY5M+kkkDD5B7qjUBkLmyQstTPs9vcHsVajrJjTifxZZJl5YsNh/ismRk2UsErnMPdbGo+r8a8T1dOGxsGTQ/op6Wlp3YdUYiWVBNMWAMLtCHnf22WS1+awkbmbYCy1hLFJgFVDBi3KkNs1K7YsBuACkiOnqoDKI5nyEn0u3yHuukw+ngdLVRSSZZZIg+LILAhYVJQ0WKxSMM76flhgL8twO6esENJUMhpK6WbltH4h0H09lxmBuUWDVM2IMZREzwwBk9VJsGi/f3XtlI/hvDvCMZDN4kRmSOPlEyXfqSemvzXj8ONZeZNSVLS6WkEVXTlt8wA6Hv1WnwPjGPwcR01FDWyiKphJyuNw3313stY/kR6xVRMmJv16dQq7afl+UI8Iqm4rSHEHDLKZHxytvpdhtcfNXjCvRGdoomN6jkL2g2F7K85mqiexW0ZYqnje4PyRipep5adjhayrCAscRqt2oJJ3SAsc23sqE9ILF+VaDoLedHHlDvMB9VvDPixPm5ZzmNdmc1GIxWNLIIXPcOy6Oswehr9ZIrO7hFT4ZS0jA2Npv3XbxnGnHPY+NxY8WLeibNfZdBxFh/Mca2P1H1e5XOjQ2XbHLkxksNkd1cr1LVtB5b/AItiszW104PZbmCJadVE8PvYlpCh5dilT1tmiOR1tLXU9wVyU0brODVc5irtawG6OVuYLGTUSuQ1AaMt1YjqrHVyy2xvAzKVjJy4AAlc5huJbLJ22Dro2ThUg0sZqiZIR6lypuJaDKgA3aURqr9VSDg9ugUTpMjsqUttPxFx6kIcCqjJCVKw3UpUr+qhkGildsgJAHmQRaAJ82VIviSuw6ZlpkPMb3QulbfdRlgCrm7XZUptbMo7pCQKtmP5lI3KbXVmGE4OqlBuoibIQ8BYaWbCyidoUPMcnF3HzKAkztkQF0eRtllVcjVMDZFINUKAkmjVCpW5barAEm6fJfZAT5lK30oGDbDZC+O3wqVpBSOUhVURGiFo1UrvShbuoiUNbdSMY26Bh1UocgsADKo5Q0NRByjletio7cKNSHdRoCDdVJk9lGGqQbrSDjjcNVM029SjBNkTdkEjpgG7Jmm4zJIm7IJWiwTpA3SWVJO3dEmDkBX0sopGEm6kScPKgi3FkIide7mowLFSbhBCYv3UuUFMhdugqPgH5VUqIngelaZCo1Zc06LriwzZGyEWKpVDST5Velk0NlUkBvddcZYyQ82aNuUOUZnc71FSO2VeRov5V2ZmTulceqAlyEnVIG6tMWRd2QZSjckALLQhtqrETgLNURGqYGxUlLX2DmW84CuUgja7IJB76rGc64TNflCzOLUS6kyM6PBCjlquW02WNFWG2W+ikNSSN1jivJa+8XuOWxUrHuks9xKoQed61GtGQZVmYbgcLnh/qKkN3kZnEWPRVC4tOhUkL3E3K50trrHtuGhWms0VFh1zK1HIViW8Vxp+FSKnzdfUj5ju65tJiWpi5oCrGUuPqTZiUoTOlskyW5UBv0Ub3OadFaRedLpo5VZptS5yiM7u6hfI4pQZ5uo5G3GqJzlHLMxoOZy6RDnKvJCx4yqHw0bdRe6d87b5U2e66wxMGdohabFO510OxVFpp0CJqhY82UoPULlMNwROUps7knepM/ZWmjc1yXNKiJ1SzpUs2kzBNfqhD22T7KUTKTNcJDcqPP7oZJbK0jI4jwgF4xKlb52G5C0MNrGVtFHI0eYCxRyvEsL43fELLGwGcUlZNS3swmwC3xGzN+W6iMb3egZlcLGE6qaKFhVYmGY6nmG4UMkb+oW4YmW9Kry0wJ9Kcl4ssRlCWO+Fq0DS66NTima0ajVdOZxPQQAkHTTutqPltYLLGhJiOa6mOIa5G6e655Rax+VZxIh8Js7Vc3IbvcFpvnc59nPugcyN1rsF0x/KzPmzdeyDM9bfgYsoysS8DCHXLALLfOGeLJENQ7VsZIUkWGV8ozmLlt9ytyFrWu8isshfI5Znb8GvDc2MKxCxf4Y5Qd7qaloakyhphJPVdSxjstkXh29ljxJa4KNNC+IZXtsVI+SynfG0Dyqu7ZcZlqhNmfdW2+YLN5mqnime5tsykrEDqjEGEyLArMQoackyaAdVexKppc5hMcl+rly/EFJScl1VA57z1ba9lnkvFDi1bPiDcmHsLWdysCeScO5FVOPcBWaGWpkYGQvsTt2WfiFGw1ZjFRI55+i5cjitmPDooRCJgXHstnAsQqaLSih5ut8ztLLm48NmppfxI7/VTw0eKSyBkLjGw7WUjJt6DiVdBPhsme0rwLhoF7Lzo4lVStcwus4GxC6KWDFcNw2zHXz7kjVc/AKWMOM7fM83utWL+A4lHSSmLmWfIb2V3HZqp0JdG8a7gG6559VRU0geHrSpcWwxzMxf9EsUIpJw3K+4+aOsilpoC90gJPZWZqvDqp2eOQAe6y66t5ufKwhvQnqucxaMue+bU3TxtYD5dT2UbjnN1Zw/kGTlztP0WFTBs3L5gba3VQyTPcfVdaVUaSnYQGk+11lSTszEiMBCgvebEZSFCzPfpb5p56l77safKq+ctOi2LjZCPK0XKB4eNXCyriSS485CkLJZSAx5kPVagNzETTf5IHxOjOV+6dp+FQep1mLcUiImjw2lH717qOjxfi311eGxTX2Y3T+N0n41j8MTjVYGA3/+nkzD+5ZMP2hQmokgqObBHF6szV72GzUYjxpPfwmBwQds0l7LDxqo48oKUV1XXRwNvkLWNBAKxuJftaqXtdQ8PxPjH+vkGv6WXDy4zjta4PrsSqJ7G9i7RYmWXodZNx7NhZM1bQ2d8bgQ9vyK80lrMcxCpdQyMNbKDbN+dbE/FGL1tK6ifiHJjDdR3ssDDsVrMLlM1FNlcRlNxcFYtpewPCZDXuNXBl5F9L6haE1dSSVDw0Xt1WdBjVbK+XPq6bctGyoSSviksxcp824acjoKl5e8ggHRU62QWEYJLUjOxpaSADbUdFC4cyXmNN2dQVhrJG8HKhidLBJnU7QKqqjpIW3zmxV3GsPpsNgjig0ed10hmmWZiXZoSc3cC6nqHGVoEjvP1W62CKnwiN8LGh1r66qhPQM5TaiRpsfjYbgrMFMaRj4jmGituifV0TsrdYyuw4dhoZaKqDhHeP0ueOhWNBDTvilZFlzPf5i03BVEEL467CSJJ2RuiIBBVc0tBFSRTx4iTPmAcAOilw7EaijbJRSQQmnLvM5zdlFIyNuJRsmhfHFmGh7IOtx/hrAqyjp5uHpxUVtrloJOcdlz1RijBLBA7DTA+I2cALaraNVVYJUHFMNMdO2nb9Hrl6vFazEq52IVDxnfJnNhZamBNiMj46osjdYWvZaUsorIqWZvqYMhWFIZquvjZ65JCGAdyuoxPAuJcLwp0j8KbaP42SA2+YXKcBAY+dYRxhs3WyZslRRS3rblo/Z2GxUBxCakpYpZIY3uk0Lmu1B/xULa2pq3tZAwOIN7EKUCr6WpdQT1czCH3uCQo4ua+hZzmx5rfD1CuYtxFUP5DMkehs5tlZraKkw2duKwx3ilHovYXShlvpMDkrYGS1LoIpf2haL2PddLPhXBuGUX+i89n5+ablchLEyWV0jS6POb2adAr+KAR4dDGxko1td43WokQUOBjF6monoY/D0TDfO/YKjUMZTTugZMH5LahauGR45hVGK+kma6mqSDLF1OvVLiSs+8nslZQRwOjGuU+pWxWq6fJBT/AIoIIsQFGyhkztdyxI3MLta7UhTxwunoYbdyStKSiZVYZDURSFssRIsOqzYlyYrgcjvu2CsNGYwZMsnpv1BCpQV9ZS1wrMEpzHnbkLjFcP8Ac3W3wbjclM2fDq9sTYHHPzJ3WYBrcHTqqePRYTC7PhOLkRXt4cdPl7LpYpw4/wARYLWSVvMEXif2lmix+S7/AIErMNbAcSwp/iK6X9q2pmHMsvJ6x5Mh/HfKzpnNyo6Wsp2vbYVEMrdpGPLVB75NxRhuJ1TsNjZKJad9pjyrhp7AhXpXT0zXuwuOWoA3Bs4ELzX7Ksew7CMUrnYhjEMTKhrPLNqZDc31XqckeHYhAa7h7EgWsF3ch1wB/h1WxWbjLHP5dTQT0jztztAfqrYll/1f8VSosfwbFo/u+fEIJns8jg7Q3+StYnPS4JTisqJ2x0zDZwcLk/JAceePmEQeaU3cb7lRTDGOUfu50ML+ufXMFkQ/aDw/UPy00GIzN/NHSlw/vWpT8UYHK8M51Qxx2D6chBFR8RVUN/vSOHlXyeIgddjSOjuy024tnp/vGMwy0bG3c5pubfJZ8lFBR46ySGMeFxNpEkRFxnHX+KHAKRmFYxX4U0uNNI0SxA9BqgsTcWMpWxVYojVUEmjqmm15R/eHRdFFLHPE2aB+ZkgBae4XHjDIcAx6F9E8toMVkdFJD0jl6Ee267CITsbklDRbssSssrGcHxeopJ5MKjp5ah9mNbJLlAHUryriajnwmtFPxHBAyV4P7Co53NHb2Xr2J46/DI70T6d0rdxMbMA9yvLOLOIOHMUrZn+Fglrn2/pLX5omkdr6rKOOpPHitlmw3DOTHMAMjjlLfl7r0zB/tIwrDGwU8MpkhfHaaCckmGQdjud1522qlgdVNqsRjzmz2kahw9itN9PHxC7wcccAdEAXTQwk69iQpEjs8R+0GKrmpakPpqpkBzkQOMckY7WUuHY9hGO44eI54XQtgN+ZMQADay42jxGkw6qq6eTB4qt0rQxri2/L03/4qtxZQ4dhsVPLBBlfUi5bfRw+S1ZZuKuIqDFMffiGKcmseJMjfBnKx7GWsSVk4pjjMQcORQxwU7B+DCXXDffbdQwwYRI3PDAYvLkN9ACs2WjronRcyknjZK4CN5jIEg7i+6zEho8klVnqpHf2rrQw/Gp6Nsj/AFhvki1togx7DaSjq44IZOWHgXLje5VZ1PTw1ponzSNDdi1t1JFjEqiuxCippqg3jjfYM6jVGat9fTOmayYcu3KF76KrSVjI3zMfP+CBbUaFJ7TGR921dmHcM2usTAFlTMHDQO1tqrEs8ETmw6ll72CqRse2XV+Vh9RSll5kojgcXuB8rgOiUtNOKrfTODKKeSN8hAOU20Rnwxl5dXJK/wDKR3VKeeCsbG/O0SgWksLC6KOGN1PIJHgy+tpvpb37KRFKjnhpMPbJHVU5lll2sQNFWgEk781/LF5wFJN4LwrpjBPIbZW62ynuqsLjkLxoOi6wLU8Vc95eY7veSQL6kfJKSjr4qdlXJG3wzxfOH7KamqWUVIK8NjdUsl/DfuWjt8lTNXOaOeB8DSyql5tw4jVRkcU8gvHf9n5/cqWWshqKRzb2e74UFA6rpsSp53vbmfZhDTclh0stSphgrqmoY+PysALMptfupMNcmNXOlqpYnHU8sAlUHMeHBzh5SdCterfhkFS5zaESsZ6QTumiwmlxjK+kkFLf4bFzf1W+QqTC0YLTuQAFFIzluMbj5x0V+t4dxWjtLbxEAFuZHrb5joqEMLJibycp3wZwQCey3dixTT3LKX0OebBxGhKu09PaOalkYC/ex6HqFWq4fBuhop2edlpSQdR7I/FTVda6cuA5nkyhtgPf5rEi5RQMkeS++RnrANlenY90LY3wRxwsvynkXsPdZwDo3OfEy+tjrbVTTT1cbIw9hAmF2nofkuPIbdBQQupIo/CmSWfax3XQ8I08ja90sMXiJcLb+FlksQTuT3+S5PDMTDY4mSVRpbm0bzHmt3Nlq0+B4lRVTY4MSjkirDdtRA4jOPl0WLHocH2kUWGz0tC3ChCyonDauV51Dz1AXSY7j0eGFo5zYufbktcQC4dT7BeO45iMFfUSwzUsmYRgNf1cQqdVX1uKPikxKQzQMi5UDibGMDsu0Zsvb6LE81E+qq8rcjS6wfckfJWKKpZWsc4CxYbOF9juvG6CqNFBGI6uatfNEeTE7ymK/r6/NdpwHiP3g2oqKR0r3AMjmiIuXdiD9N10jJadoWg9FFJEM91atcagg9QeijczVdGaVnxgj0qq6Ih5stENBQPjuVqJVnFl/iISkmcxoClnppPUwqhNzWOs69u63EsSsgCogySC4WBiOHsjfniZ11W5E91vZVZ4nyyluW7eq64yyznU0b4DZmvdUm0c0j8gGpW9JTtFg0adU0MQE2Vkey34jPFlfdUgGsrSewUPJqY3lrOi6VtG2aM2G6kgwxrG2eFOZwYlOyeRjTlLwToQtqKggks57b2VhlNE0ZWjQdE4AY642XOc7biDGhhkf1A7KWKghadLoeaQiZWMJsCudt0KWBgFsqzah7GXZfVaGdz3eYrOr6Fz5AYHEEnVXFiVZtTJEcuZRT1kufNmUs0Ip5BzGqvVwsmZen1PZejyQ8GIkO1crzMSj/OudddryDuEuZZa4Qxzl1ceIskOXUI5JBI3RcxFUzNFrq9BiL2kBxXOdTUZLkjfMXI43iMepMCJW6boBHJE65Giw2nfJIW52C6qifmPIIIQ+KJ8ibI31K0lileARqibUCwa4qu9wLSoxJ7LVM21GzXaCkJCFUY92TLoiY57T5liYW10FqlBCqRStf8AEpg9c5h0SmTzqUF5VfrdW4g0tC5qgkCjVmRmqgc119kAqSNAGuRjQIAeNUg8joifrogLVKBscFIDdQBtuiNtipQkdsgDdUYN0Td1AmmxRhyDKU2oQT5go3OumJ6lA7ZbDHdMnAunyhAKNiYNRAWWxK3ZEmBujAUA50uYOyZzQAojJYqoscz95PzfcKpzXImvc5KFkOuUdyoYtCLqyQMuZqinbJYjVHnBCpFzg7VOxzyfKVKRbBHRHmCrglFdwShK7dMozJ7oc7Soo3arOrTqVezBV6mJjvhW4RiPbcpWuFf8IwbtUckTBcWXWJc5hkzDUqu5tzutJ0QzeYaKJ0MdzpqusZOcwo8nS+YfJRuFjlU8zbHyqAi66MmzFOHtuoybpA6rQmcWnZQlpBUsMoaTmF0pXhxGllKESSd26HMFULzdlLHJpuoswSzBBbhkIdo5X2VjQ0XcsYFw1CkEju6xMLEtbntcfK5WYC0jMFixSOa5XqapcXBi5zDUS1mF11IJLCyrskBGhSzkarjlDra211yjDlSjncD5lJzw3qs0trV2p2lqpeLYOqd05e2zClHJYmqIYx5nKv4kSek6KuKYF+d9yVMyG4uBZVDOJumLtFOIbDUKGSJxPZFVZ3PLbM3VR8dVlzEC3zWgaYuPrsiFOwDKSStsMYOkc6wuVcpqR8gzSvt+7ZaBgjA8sYCqzzcq7GDXutRNgZYMo8gVfKSdAUbaqQ7p3yvVZRgEbhHp3KiNyms87JS2m5jG7lRvqmfCVEYT1Ub2NC1xLTc9zkJlHVQlzuqmggfI4OskwzaRlzs0qUtfbUK7DS5GeVqd9I5wtmAXKZbZ5uFC7MVoOoJLFzXgqqIzey3EpKNrT8Swq1j6PFYpmizc2q6QQ6rM4npn+EjnZ6mOW+SNNr2kAjqrMLlXwWJlVh0UoIc62q0m07WDZcZyaRF2iidurL2NAVd2yNnGW1mqCW4Kk2CCTzIwqyyj0NUQ3VgQMz3fqgljAPlaukMnZE291NE271XipJpj5XkN7q9TURg9LyfmsymKYNAaoy1znZWturTYb7qeOENsVymXSleClcPOWG6us0CWZoabqB0rfhWbaTc3sU/OPdVuZqkZdFmxLJLc+yryEXzIc990twqInZUxmETS4mwUoiunlp4ch8QSG+yyOar8Vw8Sul5/0smPFWH1NK+jpMOfM94t6fIqOOjA43GCkY57z17LlpZpoAaeGQtaOy5zLaxFiElJVPhPLgaxxuqVTVsNUZ2Scw/mtZUKh5L3dfc7qEOytWZRuUWLkVDefB4kDZh2utZ/HL2yiNuGxj3yqHDKWlw3DxWzsAk631WHWVlJK91RBYSHsFYV6dhtTNOxgqI2kO6ELK4ywvDXQmeNgieDclo0U/C9XQMom1NXiALz0JH+ajx/EKfFWmmp9QffdaHnEMD66rEFOS5v5iVsDA5qR2SRpk9wFsx4bNRUpnFLFFbqSsaPiKujqHESCRvUELjmgJy+J1mUr9Oqz6uaZ8dpGEW2BC06nHq2d4fyI2gfVBJNHXgSVU0bbbC1lmJHOkvYeymp3SF1wrE+GxmQ5KoOA6gKIRcg5GyZvey1cKaaaTZxUZfnCOdkhFyoLgbpEgiwZRqN0N2MGjQUBLCfSh9WysQJOSZfMCmzGLyNeR8k1n9yE/XzKgc5PqSJ7bpX/RNnZe91oT13GPFEjHMmn8LG/owrEosTrqWqz0T8z5dPPqL91nslfK8MqZSWdSVDNM6J58O/Qekr0cmGriMeKRVDpZ4I/PuQLAfRV6LGmUXMMtNzXyCwPZTubVy0LXzSkk91izQ5XkFyzyEs1SJiS1trnZRZhZR3AOVEQ4NBUmGhwyGN+cX09kQkzSXykuQxVT4muYD5SpsME8lVeENuOhRZRTTFztiHBNTyva2Rt9XDRKr55qZHThodf4VGNFKIlpcPxiSodM4eZmyhxCtlrax4kdow2TYbUmjl5jW3BFrJqrkukfPG22c3spStSR3iII+W64j3Csfekc8vhHQ5aY/lCx6SZ0VntdupY2CSQ2O6UUXPhjMlOx5kjvYG9kEJeWtgjAH4mhHQqtUsfDNZ5B9wrMEUkD45JHBo312WaRfrKiow7Dn0HIbJzDfODcKTH6qCqwzDpIJhIWRAG3QrNr66eqqHPke0t6FqFlFVGmdVCO8Y99SqNzEcdw1/CtNhwdzKgD8Q9QLrnYpG8tqgeQfkkDayti1O+aB4mZ5XsN2nsV7bw1PieL8NR1D8WgncQRK4x3Mft72Xhs8z5RmPawWnTzVGHwc3DccdA2dv4jGusCkixxR4KkqvBUNV4pgcTzhoDr2UNDUMpaYTsk8562We1tPmvO53u7OjlEBBipHl5HdKBZZKzE2R78yUaLreKBBSw0NEx92AWv1Oy53DKZwr6N8jnMvKLkOWhxLGz70EbJJZNnkv2v7KTApyx8txA6KKWqqnxh8xdK0dCVNC5klbGZjaN5sVDidL931ZiaZCwgFpzLNCeixOWa1K67YRswO1v81oVEL6ylbyWZpRcXO5HZUMHwqOtD52z8p0ZA+aYO5/MjadGKDUoaGio6eL76mkiufQw6ge6nxdtNQzRVFLLelnivH57/8AO6zqHD543tqaeldOegdqFPW0njacSimEEjv6vay5iGnxClhEsE9LzW1DclzsFpVfDDORDnxiCFkoADeSQb/NYc4nw98Lqml8tyQ12zlo1NBX8SUE2NQYhTtZSP0pXGz2/JdIgYbsOfDUOjqT5GEgvabhUWeaUwxNc4+wWy6mnoaim8XMC2cXJCah8PC5woJAKmWSwc7QELpAo0DYJalsFRox+5G4XZ8PZcLixGuweomifRAP1dcSg9HtXPRcP1slfAS6NjHuyOewXDb9Suiw2bA+FuJa3DsRrebDPCBHUO1ANj/mtCSs414exiohNbgYjmkIZKWGxJ7ghanCtVhMFbPhWM4xU0Zik/o7KrzREd7HqvO6OhZWYw6FjOc0Sm3S4utjiOvpMQyQPoyJ4m5DINyOinMdZjPEOP8ADuJSuoq5sTBbk8po5c7O/uvQsK4kwvGMNZ42WOnfI20jXaa9/deSYjgsEXC9DxNSumGQsgmpX6gHuFucHV7MSpjhWLxRCQO5tOXH1AnUEjrZZ5DrMRkxXDqWGbD5I5W084uXDMADsR/BT4NxRQVmKeDxKDwWIxNLHNcTY/5fVcnWYRiEfEdRBw1M6FjGCXllxMcltci1qPizAsQx2kkqoAX1kfhqxro7hsgNxr9VuPzDupKRkuVs0YcGPDh8+6stDuqaCn8KwQ8zOBsfZWGtB9SwOU4xoKtlBUz02NS0wnGTlRxss5+wuTrbZee0fDdNwYykxHH5oXCouyVj478vtlXtlTR09TERNDG/KDkzNvY915zxdwhT4RSR4xOHzsiEhlD5MzXG2gsdlKIl5lWRw4u6vrMOoY4qSN3lzE3I+uyucM8QSYNgNRS0FSYXvlvI9rMz7dr9FXFXS/dT3xxBsVXI8OadRH8k3C+LUODmTxtJPUvqCDGIWgk2+aUOrwh01PFRVVVBTc7EoXsa2W4EtjsbbLmuLaKqwvE3xYjIWcrK/wAO51zY7201W9xZxphVbQB8mB1UNZCLUj5HABp7+/RO4cSfaxgWHyOp6Roo5SyWfRtz1WUx9WBSU0NK1vEdbSQupg6/hXSFoksdCtPHOJ2cTZKN+FeHe/JyA+e7GD92w0VnF6fEahxpsGgFZRQRhjGxs5gMg6WO2t1yFG2TBcdpK7HIJaflOOaN0Zu36dEpRVsniIuRU0NnSOyNncSRoeimoXfdUpxClqI5+RHYte3MFbe3DcaZV19BRSl9NeVpqXaOHsOgWJNXyTskYAGmQWs0aKiTEKl9fWSVr6eGJ0triL0aDoooJWNc9knQXFlHFDMzC3VGUHlG1j1WjwhRjiSvNLOC1lJEZrsZckjYEdVmhmCCSe7xJmbJ6WtcASfdRxCejmM0Y/FgJXrk32cYJG+DHeI5aSGWtb5qdhLRf909F5hWwx4ZiVS+CFwgjlfGM0hJPulFgq56OolZXRgxPkNpWHYFAyspWxPa9zYHEWElrgfNRSGhmlZHAHsElrXGxVatoqmhl8NPI0ydh0W6Fumnhpq0wVE/MpXizja11PVvopYoI6Kh5kUT9XuPU/3qjSwYRV4iymrZ5IYiLBzTs/3KgePD1/Lgn5jIprg3uHDRXiL9ZR1ENEK59C6OmncWREdSFC7l1Ihkj58rACDGzoVtHi2qknpJJzF4rD6jmRQZQY5Qd7/QfxVvGeJcArKCeupKUUOMvqmSmFrS0W+Q0SIGDTyMw/EWVrInODG6tf7p53cxzywnJL5wCb29kMlTV19KJaqXM/pcKpWBsUoYzRpFwFJhYXajwuQQUkIDwLZnakp5carZY5HsMUPLsDG0W0HVHR1MEmGzyTvHNgeACW/BbdQGCSWnFU6NuU6lw7d1mkaOEVcta6R1I5zZGAaONhKOyrY8yohlYytqJHCaMSBj2gGM7dFHh0U8Vc3kyCN8fnaTsR81ao62gm4kw+q4mmLsOEv47Te+S3T+C1AHDuHq3GcKNdSszmKcw5nu0AAvb9Af0QQwUxaHxuA+ZXacTxUXBkYwvhiu5+F47TeKBlP7PpcH3B2XCCNjKlrKWSSeK9muaNST2UlE1VOWMyNy69jdatU1lXw9QwQzM5kBMg817g7qjihZK5lM2iljkp7slzAEk6LRhjw6j4OdUNnikrKioYHBjrmNl/jHTb+KzODUKbTirGBjA54G1wDZTR4jPHcimkYLWlc7QtCotrX4O6Ooopo5Xyj8Rp1EZPUdlPUVsFdTGie7NPJs/YF/dTirSw7FIaSnqafL4thLPUdQeo91TOJsYZIXQPjgqJebBf4Bpof1QS4ZNFKzk1eR8cIe60dzfsnw/Bp8YoJRQzwPlidkFK6QNkd10upwHR4VJh1RE3nVLp+RVAN5Z/EazfRdnguJPmxnxGHxR8uik82QWM0b9PR7f4LjuBMMpcMjjxPEZTS1FNVCLI1l3RvPdnX2tuvWaPAuZibcRgkDabwoZma0AueX31HyWoxTm0qCR8jHOfcG97HopsmqlbAyLNk+JxeUsvuusJEma0FA6K2qOxCZ5JGq0Icg6qtV0oeLbA9lcdmTZM+6tjFML4mkWJURZP8A1bCt3wh7JnwW6LfJiYc1LJK05ZGnforkEZy6t1V6RvmshDAE5JQ4Q2JmnVFnunEF1KyAAZVLVUyPc+4OidzHhuZXRCwDZRuiuCOicmmLVVE7HBkbL3Ovsnpo5HPDpNFNUQgSg7WVWatEMmRpt7rqjRJACka9r1iuxJ5dlYblS09eXHU6qcZZtYxSlfJE57Nws5kEkELmP2Itdapqc3lukY2SDzC61EpLlWQTGXI4391eZBQvaGPP4o3C2RRQA3DEwoqcOJLb3W+bPFmGkY70BO3B3u1Erwt+lpIBGPKrUMMI+BZ8RrizcOw8wi7rn5q8YoyC1zAQrZaBsFWlkDLrlybpgVlBypfw22Ci5jI2hubzK7V1bHT2LlTkja5111xYlVLjms42CFsuV+UhTSQOtdV3R2doulwxK0yUNIzFTB8bgVSjbezHKVjHA5C0qTCxKxTNY3Rh6q41ypQMe3zWVpsb3PuuUw6RKVh1VmJ9jZVg1wUsY1DlzltdaAW6oXxtvsmDrBRmbzWzarFIJzGpsjUPMuj8uVKVEQ1AR0KGWUsfYJxJcJSHRN2QHbMozIrSrINk4N1WbJbqpRJ7rMwLDACdkTwLKFsmiIyXClBiQg+iRPdDnVoE0aqTKFE0i6maQbJQcMRBuqSWZqqWJrRdKQ5E4eChk11WiUTpDdDoQhkuHI2jTVaZIWvZSsZ7ad1HlF1YBAZ6lJC5ZClabM1VczusQSgZLfqoClcMyKOSwQZWuNypI4xdGkbqiS+UOIyqRtQba7ojCy3pQ8kBA4dmKdMGgIwW2QRXeCkDdJzxdN5eyMk9tx5VA6N3VqnvrdOW5grEjOlFgqUgGYhaz4HPOjVXkpPMdFvCWaY8ovdVpBZbE9FZpcsmZvmXoxyc5hC4ZgolPy3nYXUbm2Oy6Mnbuk7ZMkjBIEaFyLJkaQGqRDroENETdUmRueQArIphH6jdS1pGwGyNryw5mpaAKWJrHtuQszKwsUVS9zwS0ge6svnBKoSuLB+HoFNTPY4guXOYtuJWS5x1A0TSBwbmKkM0Y0Tcxrlni0q53gqaGU2AUghhJJypCNjT5UmBbidcBTNKrQkDRWYgXO9K5S2kAuFFK3UqxazdVG8XKzCSozyui9Kg8Q8qStVAvcAu8M5Lb6vIN1We7nOuVED+cp2ysaqymbGxSckOChjfnKsMZosy0KGihIu9xP1RSU0MY8jbBG1zR1TPIeoM2R2pChc1xOi0jTsJvZLkNv5WrfJlRhpnOeA0LXgo3wsBcFPRRMhaHhvmCnmkMm6xM21SAENQSuL9AifoFGHAOta6w2Jl7WcozSefPf6KwAOiIALVir4dqgxPD/F0EjPyC608re6fl5mOjds8WKzyHPcCn+hzUp9UZuf4reduuTwF78Ox+eke61zZdgY1m7FeVrnN0UQgNtd1dyt7oX2AWrSYUzTE+pqhla2O9lbc4/Cqrw9zzm2ViUpWiZNLMNbNVx1IH6FuiUUZa4WbZWjstTJSINa1uRrbAKSKO5CE+pGx1tlzmWuKyYSBe6je+zcqLn+W1lC4klZsM6U/EozInkHluVWc83WRLmBQufZQZ3Ji9yom5tijbJfqqoN0bM91Rca5U8UMnJfUCT9mL5baFSh9lRr8dw3DxaufYf3lZVxlZxZi2sZpIYh3Md1TqZAKZshpWyl+9hqtPH8Wwuugz01JcDreyr4XWTRxh76SMMGx7rCOZmlEr+T4XkhQxsgkeGCf6kaLexJ9ZVVTpRHEAdgQqMtFUVQymltZZmGlcup5PwH1Ukre3RQvigBtGAGoS8Nl5eQNHdQyPfmytd5VaVMyw1jvm6WKmgr8TgkBhmkaR7XsqoEjWZ2tIt1IRxT+f8Rxv7LnKw1nV9fMzmVs/NNtLlZzLAucNyUU0kbyDDHYfNA0XcW5h9VhAvNzq5AWZjmzgfNNMcrtx9EOeytLSU3c3IxwCAWYS2+Y+yjD7pF77eVXjCEXOkNjsoXsAKLJcF+Y/RKONl7yZiO11rjAjbEZDpZKRvLOVSmNl/JcD5pcpnxXVEFynPp+iORrGDyk/JREkhUgzt0JNk6F3mCok4e4RxnEHVMUVBEWXyGSU2sddlexP7L8RoaB0tIWvczcZrmy6bEqz+bnETpK3meDBJjc3SxXK4x9oVd4+oOGsk5Uv0A/gujCLhXEMPraSXBMUZFG61o3HTX/AAXK4tQfd9UYjPzXdfZVpi90r5CzLnN7dkGUoE0M66JrZpGsucpNlLSMjNWxs5AZ1urQghdigFw2Fjt+6Q0GtwhlFy7zF3M9rWUMlMYGl8dS0H5rVxIsxd8dFQxuleOwWe+GSgdJT1VIbkW1Gy2ygENVNEajlmRo3IUlJRRysc+tjlb+WwWvRsgdgkjKRjnveLOaBstbhnhOvkYKmrr5IIbXGxzK8U5OLeWRvLGG7QpCyLw5m5xDhs0dVocQvpp6p1NQiN/Kd6oW2DljyNdyyDoRuFJxajJqSfdzaKn8LMTKfUCb2Kq1JlY8MuRcb2SJbFG38HLnFw7ooHzyS6SPuFiWlh1pfOd0c9U6WzHi1hbRQRvs7Idkbg3uslI3RjTlk/VTi3hhGZHDuL6IQLJzE8g8yGRpHQhaKVSA0ZENrKUjS6hRYLP5rqRrGO1I1UKsDKxgtusIN5dta56BbDOZJS5JI42yZegss1rIWtE0jvMzWyXiZq2U33OwVkaeGTvp209Re/Ld16Lo8Zjgr8FFfAwZ43DNYdCuNL5o7Rxi9tT7LeZHJPhbaqnqg6mOhaRY3XOhNgMdBWwksg/pMYu6Nw0JTYlTVeItLPDksHpLdwVlQYw+hr4pmsDWsNnW6gq/xTW1dJLGaGYx09S3O1w+IqDKB8I5zTrJ3V7CBTPfM6pksGm9rauUGI4dPG+AOkbnfHeS3dQta+PMWC5Z1VHU4XjYqaoYbQkMjAJOca/RTVFRasNJK+7x0XK88wg1cDjHM3qCrVdVGlqoahsvNf63HNuucwNnE6l8ZgdKwPjYb2PQrmYK18UtXDPG4tqxYW6Fa/FdZzH0GSS0EsfMIt10WVFUwlr2uhL3ONw4btW8ZFmkqMLmpar7wY6Sd7SwZhe1lTpYn8gA6XICQgje50MbfxCLtUjJJwwMhMbHMPmD979l0iQM1ficZdRRyTxkGzmtFswUEEVOaiMYlPLEy/mJ1LR3UhxDxNWJ5WWfaxLRuoaiZ9RWOyeZ3QK8h0NZh1bwvFHjeFVdJUUdRZjTa5A76rNxKjrKPlzVTHhtSOZHm7KI41VVmGjBqmiieWOzscd2lW63F3VeG0mHVrSXUlw125ss2OtgxjDqvhSTADUR5325FzsfcqnVYdJR/d8k0HJfFHzHBx/aEf8Aus6epoquXD56/C54aWC0U7mbk9F0UuLYX4ilwp9THURQSAioedovyH/NUemcM01M6Hx7Ysj5peaBfUAjRctiuH02HcQ1eGUkYJrZopY2uGjTe5N1sTVeMYbSUowqSCJhcQ2GYXDhuAHdFdwPFKPHJn1U9GKTFYCI5ad+4PstRI6ORwuGtOwsije66q8w5vPujjkbqXGwAuSqLoc7ZZvE1BQYphMlLiOIeEhPqd3HyVX+ctC2ojYybnwyOEfMj1sexHRaktHSvcaqaAPewXBKwPDuLKHC2VkUGA4xHXU1SXiWCOIRhthuAO/+C4dlBHE/NTzyxhmrSHdVv4kx7MdrKrG5J2vlleI3MAAAv2CxauojiqZYWUsckcgs0uvoEVe4jr8ProaKmocQfVPiBMwfoCSj4c4rxzhRtSzCJGRMq7Z8wzC40Fuyjwx9PNTx081FDLUgWhym2Ye6p1Edbh0j6OtgDoS7Pdut/keilI1KCSvxKnkkjZXQ1kWeTmQkgTXNzoP+dVlcR1k1RWseTWZGRiN3ijd9x7rf4LxR4xeOlfLUw0z2PigDjYNJG9+qbi6Cljxkz1sgqJdGF8Zuyb302cFplztNX86J8cn4doiwFpIuVYoX10dEH08DJHj1uduAjouG58VoMSxTDo5Y4cObncx437gFU8BnoKitFLiTKiTmWZEIN7qTDUS7LAqfCql7cNxvEi6nnpDM2Unlcl/sf4eyt4RBg/CtTScRYbgVU6hqYH01WyZxcQzT8QO06rI4kxqHDa2lNI2Mvw9vL5MzbiQfJKm4krMfigpMfqHw0T75Yadobc9ASN/0VpadDxvRYbhrqWtgnq63Dq0B8dOZeYYhbV7T0suSq+I6SaimwqXDjU4eR5ZyA2UHuSqFXjZwerkpcDxKdzNA0k2FuxCzBW1Tw+CXLkncXkAblOIatEMNQxnPE8Mbtxobdiow+R9QeWH5Trd3VinNW/MIDyxbrluVUbIyJ7zN5rx5B0se6WWGRjOYQx+Yd7bqelonyVEEbT+3OUtY7z2VmrweKni5kOIRVFhdwY6xAVbx8raMUjBox+eOS/mBWrQ1SyGGtfSwxmIMNsp3CHl55cm7up6lHIySekdi76ky1Bl5bgQO1wULnUwjgmZLL4jN+Kdhb2QacVDiU1AySKntCxvrzA3UQw2uxiCWrgDS+maObGTZ/wBAoJHCSAvzXda5WngVTDhlZzA+UmWIiUjoFzkZEFO94DHuy32J2upm0dbCeSZLuuA3IdCULoppY5THHmYw3J7BaMMj63hl82cRvo5QxpPUIFh9RUUGOQUuIkTRPPKkYRpYrY4pxbDqqWKhpYGN8FJyg8jUhZJnhb4SetcCyssHOtrGRpcFNSNEte2OWZkbXzCKZ7he375VE+PYo/Fq1s4aWxxxiOKMiwaB2H8VR5rmMay2UjfTcqXF4Xw1ErWEOyEx6fRR14gkooqunm/E9HJcfO33WZRVgmqYpXTNqJLvNybqyyR8zjnHLY/1ZTobbXCCkNPLBLTmojimJu3ObBBKRFeOF8c/KIJd0VptOadtTK2DDhzXH4dkzgyilifnvzGZ2nZWuHHTVGJuMHh4puU/KJTlBPse6hxNnLoqWDKTKHm1h7qTAsU+L/d7jNSwSCV4tna7/AoYcLM8H3jIbyzy2JB1v3KUWF89scbZo3PtnDSdCRuFdw+aYSluHQxBg3aW3LSoOr4Q4krYKV8eJYXR1VHTSRw1DxHepPSN/uvT+G5a+KSbDqufK4DmwOsSJY/cHYjquN4OwCnqcErsVqxIfGlkboWi2Uxnf9V6iKUyzNq2NvaIMHyOpWoSYSTRvjiMgbcjohdIxzGuYdxdTSVLIYZJJPTG0vPyC5qLF6h1bBHT0UkkMzTKG3AP8eyqRDcdsmO6y6jGZBiFLh8UJaagG7idNNwO61GtcVVDYo2NuU4jRAWKiJA0NCCVtmokMpJbZBnztsSVE0XOytSsBHmUXLynRq3ElHDnAeVO2c3sUgx5b6UuS8uFmILDdRcITG97tEcURYLbpyJGHyhBWdhbZzeRqr1HD9B6nxkn3etNomckYJT6tVLKc3UYEC0il0PdZ5w2rgmEeS46m67eOk/c/iifRNPwLrzZ4udosOaTnkJOXYK6+INOjCVfFOxp0Cins1hyqciuLPezKqD6k5iB0KvzFxBWRUAxEk7LcQzK0yrLRqVYirmNs4vssYvYDfOTpdPS1TXEsdrqt+HBbpoalkuztFFV7EqrSShtsytyWlbYFcVc++GWrr3gsy5LIzBNE/IbGy1yzL5gsuqlkEpuF0wliYStbn6IuRT/ABxC6ggdNn/d6q4xmcZnLVrQBS01g0NU0dPHa2QFOIuwRsBCzbfExpo7XyBJsY7IyTZMChxCQD0TgXTIm+pZaCS7ooJmybsVvl3RNh03WUZ0b5g/K9yn5+UWKmfCz8qhfTZ2m61bIHujkOZr1JFDdvqVKaF7YyQNVVjrKmMhl1uIS2xI2zd1TJ1KOOpc5guVVke+51WYxLWGPzGykGZZ7Hm6vxy5wLrU4ESO5CEvsVM2MOAzJzRsebhY4raHns/MiErD1VOqoqiKUmP0n+CQbIG66nurxS1xz29Emyt/MqD5XNOoQCZ56LfFObWFR0zJPl6LPge/N5ip55chDbE37LNLCzHI6+qlMgtus0Tvz5LFWGuKzxatZzN7JBygu8Dog8TY2KUi6hMh2VYVN+qka4Sekq0E55unjkGbUp+USh5Tr6qCwJAOqkY8HZyq+HuVM2OzQEEvMf2/gizEqIZuqZ5s3dZaGZWhDzlVMlyna5GeSwXXSbsom3O6lBaEErW6Iw1Rc63pSEh7pTQjuo5DlKT3myrSSG61CchStDm2WVV0nmztGhV/mKKTzCy6Yy55QzBEWjLlVeZuq0XNcCqU485XbHJzmFbIgN76AlWA4gZcqsxTxgWyBamUpQdBNGGueywKZjSTYBa7XwzMyP2Vd0TWutlsE5rSs2mfYHQpPgeBctVyJt3BXuRGW3cFnkUz6URxjM9l0LpGl5voFZdAR0UMmHktL7m5WedrSrI5rtkcLnZU/gnj4rrQpaeOOOzj/BamWaUxHNK3OyMkJpDUwC/h5PnZa7ZQBlUnibMLL6OWObpTBFXLmsVoZC2ESJjSUr5eYXlvsE1bLkhDIyteqEyXmHQp3SPByqjDMYzdWRNc3cnEXqbmPeMwWxTi1mrGpqthA0sr8dVls7MuEukLr2XUUkWQZkDay6mZOwhc1Z9XTl4JyrKkgeHWsumcARqqM1OwuzZVrGWZhiPgfa1kcNIb3IWg+D9xKKIjot8vglIW0ljfKkWdir2RAYmHos8mqUSSCjjNzZWHU4PpalFTEauarbNCZFdSthAGydjTtZWWM02WZltC1rgE+UqzkHZNkasirJFcKPlZSr2RqjfG0pYrZvZO27kTISDqjbFYpYdrVKCGoNAExN0HH8Qxvw/iNlaxtmPIK7BjuYxr/wAwuuc46i/o0NS3dh/yWrgtQyowyOQdBZRpeJuoZDqjchLQVbZQkH02RsiZu5FlKYmy1YZxYzZQyS6p5HBQO1SYZoRlupYjcKq3RWIniwWJaT5gnDkwcEznABZAytJCrFhupnS6KIvARbRlhumLdE5lF7IczeyIKNoUwyNaq4cAnEuq0GlJPlUFdgVFX0t5obv73VjM0lSh922QeXYvSMpppIKN8pydAtXAcMdJSGepr7uYL8u+gXaQ4bh0b+YacOPcqeSnpZIjGyEMB7BYmB5dWV7jWOPJc/J5AGhVJsWq4bXBbfa4svSxDheExSOZSsdlFzmNyV5tjFPVV1W/EJLRNk9LSeixMNsqV8cjzI/d3RM4wloyEgonQ5TsCoyzVUStfNJZj5DIB3TOiyPzEKNrnNTl97XK5zAlNSR5WZQEMU1pQ+9z3KAtYRmUkYjAsQlAZ7OvI93m6KC+YKaQDNo26jeWNPYqAQR0R2Ltk7qhpZkaB+iUTgxwzOIVDtY4bgoZoJmtzvbYKxmLpGx05dI53RW5KfE4IM8+HvLf7CwMUOcETZS7aMFTTThxyeFy+5UZl0s0D9F0gROJudENvyp5LkXTxNHqK1QjLXJBhsrEUbJJcrJNR/BTyRtIszUqDaxrFMfqKBmJ8TcMOla82zlxAH+z+i5vGMdwaXBjS4bRGKY+ohtrH/FdPxfxPRY1A3PxDS+GbvGzd3zXm1WYPMKecytv6iLLqwtYQ7B2ZpcRyc420dsVn4j4eWZ0lKzKzoL3QBl0xje8ZYxc9AgguBum5otlBsrkOD4vL52UhsNfN1R1UzCQ2eOMuAsbN2Rt0/DUVdhOEGvpqahnc831ms9ZEvEsk1VJI6lja+UW5d9AsHPFG8WLwwfCDortbV0Na1nLpBG5gsCCjDToOJKvBnvZJh8Do3uz2vayt4nxDjddSCQScmjPwtXI53OBaXk/NXm4jVOoRQuj5jB7XsF05DpeC6vB8M8R98VUTXz2y5hpbVZOPzYOx9RDSPE/MfeORuwHZZNP4Wao5FZNyWd3BSVsOFRjLQTmQDckLXJpTfLJKwMebhosFG3zOspGNDjYuspaWAGTzWI7LiqIG2pRx1T2O0APzCeUN5hAboCo3AXzNakkytsl5cZkvZT0c09bmZM8uJGl1RlkvDltsngmMVw2+o7oFUMMUjo7g27KAmyJ26j1JQLOcyIE2TN3SzBBNNNzGj2FlNgzGvrDM/0sFyqmmynhqRBEWRtsTuVmhaklPOkftnULah8NxTzkAm+Xoq3Me5xL3JN/EdaEFx62VoaDqhlRHZ7QHdwtnB2SYjyaGqfEYqdugOtwsSPD53S8uMZ/cKaGc0MgY9ly3peyyNPimrbHWRU9KCzIzIARuFTPMgaGSCzuoR12LUeIyQzzyBro/g3KeUxz2qN29FJFWpZk8wddrwCmw2k+8hLCJhEWbX6q5WYe8sbPTi7HC9ugSoaDmvErGOIZ6o2usVIkVJpZnDl1U4k5XkaR2QUkr2zDI63udlb8Pg/4rWMmAk2BOrVDT00NNeR8xktsCrYsT4dPG0ve5p1OjTcoJ6OBsrufNLzjqLdlr4NDRytir6ioAdr+D2V6rp4ayU11FM3PB5Mp6/Vc+Q5CGRlPMcyv4WY5a8upeUx7bH8QXFlsYhw7LFFH46njL5hePKfxLDosHkxUdW9kLpIn5TfNqtRlbTawqbBm1sp8IPFZvXIPI0fJXK7H8Lmp3UlPSRDPs8jUn2XJwOqQ7SznHckK3QySS1bRHTeJmHw7BWmJb8dQPu2fDKuOS87uYDbSyqYPSsdM2RtOJ2U7vxIju4K1XZ5Io4DTAzMs8MYf8UGHmHBpPvGtjkjaNo3G5J9iufNXa0NdT4m92HOxSanYZPwIZWbEdB2Vni1lJJTUmJMrWRz0czIp5GOyuIPXVYVLWt4jfTxRV0bJwAYQ8akfPRX3YRjgxYSVUdHUQ/HlFiV0xzHZ4TWSxZR444jRTAcqo3kBA1B/gtVpY42L3ZeoBsuV4Mo58Miq6OcCJj5jLCxx1Hcf3LomuXUcnxThuIRtM1JixoKWOO0MbBcya9bddVbwWrxLCKegxCqfVukr/wAJ0dRNmLb9j79lvSRQPex04jPLNwHbXWXi0E2K4ryBLJHTUkYkbkOt+ig537YYKItpa5gLqj9nDGxtrk7k97LyebxDZXMqo5G5DbO+MtH8V6XVQ0/ifFzVddFWQm8bZG9bfx2V7iBtbxFw3R4rTV0GWwE1NNECyQ37rJbguFpsLjnrG11TTNLGgQicaE69V0Hha2SKriMEUEVWGRtaJBITruB9SuVxPDG4pjLqHD6GlAgAEslK3yNHcrc4flxngxj6V+FCoaZRKKln4gLBuAUZacHD9Q/E6bgfxFOIPEc2N7wfMfyX+D5rXrOC8IwjiDnYW2Gjq4gAaV558JB630I+YWfU8ZRVdKMYwGhloq+mkJa5wDhIx+hBuutq8PeaSLGeJ8RlmqZ4uUI4QYn3IvkBHqCN8HD8ecYx0dFW8MwiF8tSGB0sMlspvf6qLh2PBeIKdklVhtPR4pQGN8JBy80gaH5bLYxDDattKz7nwakig181XIJJOm5PRcjVV0FJy62V8ctfHLYOjuIXM6sHshxpVxqWtw7F6mSvEEj6h5e6BsgcY9uyxaur5wY9jHRvjPlsdAO1lexWehdSls1Jaulm5hmZ1HZU5oJIcsYp3xlhu0u6joiNOGDB8UmZV1clPTDIRIx77PJ72WBWZoqwwNaLQP8AJYWuFtfeFVPHVZ6SE8xzALi8jCDuo+JcepccxmJ3J5dHE0RXayxB6/xViV9WUZI87ZnsL3HZvZBynTShhIaCfU42ARsgkqZzBSRmUg2aGM85CjlimbCJNcpNtehWYIFWUJppHfiQuaCLFjt1Xc5rbWKNzuYWhzbPt5TawKARSSXdoADrc7LXEpJzXzXe5w0GwGiIyGSJl7F0ejTbYJ80FO5/LJmb8N08TmySsYQGczr2ShZoY4audsdVPyWZMm+5Wtw7QPrZKpzJLOoor3LQWOWbh8bJKw03gfEPkaWMFwLFXZ8TFLTGmwmDwj8ojnkboZD1BH/O6Siq9s7GOn8OQ2RxIB6jqF0WBM4Wq8IxDC5mVMdRXycylYY7iB7Bvn21Oi5xmJ4rHT+Fkq5JIbECN2oF+y1cBn+9KOowR8ETnti/ori0Eh972v8ARZgZeLRYhhMrKTEqGLTzxlrg4EexCkxLFKeuqm07yIGgWLwdE76qnkqhT1rCxp/ClY27n/T9FpUmF4RBz4sNw+sq3EAGCohynJfUhUZM2JRySMpYLkRtDA4i2YBSwT03g6ullY0TTlkkc7hfJk3B+a2Y+GoQ0Yjjj4aGFkWSnjkP40p6X/56qjDg8EuGSzzOlNSX5I2j+r9j8/8ABYGbTRCpZJJI2mLnm4uL2PRPyzndG+PI+QekCwWjBR1b2y0bY3xfhExkjW/ss6lxOeCo58oEhLOXIw7K2LDKd+Yx2vb1N62VeQ+RrZJAMgsL9kfja1rHCBjXh+lzuB2VgS0sTxmpY5oS3zNcNj7FVo0E0NDPSGellZymgG43B6rafDBHHKMMn5pqJNHMv5ewddUMYPPfG8VEklNEBEBI2zx8z1RQzyiFsfiBFc3uBYaLnI3sE4xxHhuhNFzBIy9y1riDf2XZcL8RVPEGKvGJ1vg2MjGXNP1OxF15zNUQzUrqeqgijJfna9jbk+xSFY05Jp6SUm1hb0O7Kcx7jxbxLPg2B1HhY+bUPiyNk0yWvvZcZivElZhODUWH0jrVHLYyWRzdQw9h9FxEOL082LU8lKxzgG8qOCS9myWsD/FdXw7h2N4m+sqKulElPHemmDn3laeoA+i6choYJNPW8U0lPitQ6OCDmGFz3amTTT29vkvTWytG68f4FazFKBz6iP8AFpqtjua8kXBvdehYfPNGwGeoMna63EMukErUVyVmwziW9jqCtCJ3lCtA0zi2ydzhZQl6jRANKikbY3CO9tUzjdWEkmDyp+aR5coTg2TKoQmfGb6FWYpmPGZ2QFVHoEGoCPZFdqzYpTG5Webf4lgWQ5oTPeLKvzP3lFJM4eW6CYvBVKpezUXR83Kb3VCqnYJHeZdcYSQSyR2y3VSaOCSM819lWnrI45TcXUfjoJvw72XaMXNajpaaQeWIFM7Dmtu+KNoKaGpZHbKVaFQ+SMljQQD1UtpQfR1EJ5nMJFth0UlNWVQlEZYdetlZhrHlxD4raqy/O/Vlge4UmQzpW2ynQqMwh/msqdVHUROfOHlwvdT0dY10Od+nzSIEopQdlYipgBYrLkxUseWsP1UoxuP89lrjI1ssYaqU0ha42Vb7z8t86E1jJ78y1iLaKRjIk8Vfqn53Zyq/0ewDToEnSxNGRh1W+CclttQL2KmZOxZkUrLZnu17JGqb8CeGcms2eH8yLnR9HLI8W0bAlP4oO+JZ4nJqmZliLqPMs/xLhsUXinW3TivJceM4sqlRTRO+FN4p/wCZCZ83xJEM2i5WTYlCWEnVSuduVEZH/C5bhmTluUpw4hKziM1kwab36IqdlW8elWIq14OayoZQ11rgo8wHRSjk0TViRuVyrytOuRVC9w6qQVDtnHRKWzSRcwebdQsaYXHO1WOY0+lELP3WoRBzNdFoMaCACs17MknlT897GgByzMENN7IScxTBsY9KyJJ5y7NnJSFZKN3pxW2zdoQPZHILOAWb493dTNrL/ErxLTci2yNkZYbqNtT+8n57fzKSWuNkd1CO/sqXiz3UjKhh6rFLa20tR5gqniAOqY1JCnGVSySOUecn1Iecx3pQuktqlSC5ebzJWsULZvdM+ZvRWkSc0BPzWn1OVF8zwVE6RxK1GKW0fEsHpN0vF2WTzZA691J4gncrXBLXzWX6JhKx24WeZuxTie3VPDLW3O1Qh+vmCiEwPqSLs2qlFppbFqoyxlxvZWc4OjkZDbAALWLLLLCDlcmDWDbdWZ43etqjZHdy6XDNBjkDXBrirDJGE5XAEe6CWnYBtZNySG3BujS4zlu7BTxuA6rIjmLHauV2KpBF1ynGVteeGkXaonjym26dsqTnA7rMNSpOleHWyqRjr2QzPYHeVqHmhaYXGRF3qTuaAFHFVMy+Z6CWoaRlunGVtXqZLO8pUJe6QWcmm1cU0Y8q6Q5naxoOpVpsHMYAzRVTuFepyOUMqzKwdtHNAM8hFvZJsjhoFNHG/JZ5JCF8DdS1Yl0SQya7q1HJqs7MWOAVlriAFzmCJaLJu6d7muGUKnHNc2JUjZBdSYbtIYuqblgJxLm8qclZgiAZSiyNISRZgrYDluzbKZsWiZrlJzAFAhGB0RgABAJQTuiBBUBZ2j4kswUMl+ijD3A7oLJIQu3UYdqnzFA+UITonzFA7VQMTomBsnyJnbrQocRU7KrC5WHoLhZnBVQ+SjfTu3YbrdqYufTyQ33aVyHDU5ocYkpb2DiQoOzSTX1KdUMdlE86qQnRRSP0QRkdCo5B5UZeLoHOzBaZVybpZ3JO7prDqpMNJmTEfEpTICN1TBy9UjJpspMJaeQjuq75LHdAXlRF6RCeqQv/AHkPMsfUoi9NmCtNJzLf4kg/L1UTd0QasMpRKpY5tbKsBZSR7qtLTSSVBWYxhuGf6dU8r6XU7CsbibAGYpAJWOIey+o+i5yrmMd4kp62r5dDNZndY81K+Z2cVE0zyugoOBGVUZke6TM0ErWwvhNkBs9kjD3zXUacfHw7XzROmLCxrBc5ysYuyvI7L1g8PV1V+Cavlwndtv8AisHirg+mpoh4Se8jzbzCywjgTnllGQXujkpnRNu9dJQ8IvY0Pmkv/ZVHE+HsSp3mQ3dD0utqwibKSJgcLvkLPkrE1DNE28kYHzQ09FUVL8kAFxrqVLD0zWNdne24CPIZJedDRNI99FKJJwRSP5YB6t3UzpYaQCKZmdva9rq0Kc1NNKwzWib7AqainpIIrS2Lj0IXXUOKYR4IR0IhZf4XaG6ypeFKiafxFRJDEPYAqcRixSRNqRUR6EG/suhpeLXBogMYkPckLIxGihoiWfhyWWS6QE6AD5KTA2eITU4lIKoQAACwssXIY9wi58n5yfmgdK87rNCGbOfl2ULRc+lSvLlG19lsWqXJnDHmy2YKGiZlmmqJcnUALngdVaBqnC3iyG9iVByMdZRSPAfSxwju1KeekDLMde/VQQ0Bqr5Xhtu6vUmHinIvaUjoRoVplDhlPPiLnwwSNyRi5BU+SGgeBmtKzYraFbUR05bFg/Lb8TmBXMBxjhmL8PFMOdJMTcl0VyrEozm4hRYjTtgnr2Qy/mJ0CzcU4cqcPidWVFdBIw6tLDcEL0LDm8BYhmFXRHNmNozBy7KjxJxPwtHhk2B4PgbcrxbM6M2Cttw80hFKZAagkN7o3ci9oZMw+SvRywUmXNC13sVXrKpkzs8dOyIj8qoikhgbCXiXzdGndHhkRfUASTmFnV43C3eEHYaKeqnrsrnjoQsGbyuc7u8okrOL4U+lqIpH1vOp5N3W1VM0znH+jjyKQF0xzyOuSQFYqaKtw7l53gc3a2qlqzpqd8DwH9U+XIFP+JNVBj43S/ut3K2anDKinqqNkOG/iSN/ZAb/ACVHOauKItyhbvEuHvop2GahlpfgOdlgVhya7IItMt3Xv/BACUXntlQ5Qgc6pgLKzTwMkDi+ZrSOhSY1t75QbFBA6N7bXbZMATsFaJEsgv0Uch83l2RUGS6lFM8tzuIARRFjb52qxBEZYTZgNtm7LIgpGWlbIAHZDexFwVqz12zqTDYoDsSNbhVH8mlhcNpOgVZ1TMeqiLsM0zxKL2cBcFPh5ZWUk0c7v6Te7bjUqj4h9vKd00Mj2PztKASzLIWHdu60GCd9JHDTjM4G5HWyqObJKbsF7rewWR1LTuYReYtNlJFUYjV4ZXNfSvyseA9zbaX+SuVc1DibmT0l4pDYStaLAjup4phiGDS+KjDpI3Zzfos2ammFVEygY481udrQsDV4ibhdHyGUNBHE+WIPke07/RYJhMkfPjb5B1urtbUvdAPHMd4in8vKO9vZVXGQRB7XujYXZ8gOvyWxJC+85gc3yPade6r4ZUz0+INhZIS18nLdfqP+SopqmQSiQGxGyq3zyuPc3U4j0fHcUp5cPMHMtU0NnxP6hcrjNZS4nHFXQNe2pBAkG1ysbY7lWKaYOkFPMzMw7jZOInbNHJFnYwiQfC4aFalRBP4elxTCpPDskbkmLB6T2WJPTciV0wle6PpYrd4bkZJhlfSTm8D487W9ilCpQVFNSzTFk8kx3EjjqStfxlLidBy6ilmmkPpfewA9iuchpnsa0vbudPkteCdsNFVU+RxEoAbbpv8A5rEwBw8MlnL6cua+AZ42k2zW6XXXYdxl4yelZDSmmo6uTI4vsTG/uCsOkwWbFMEwulg5cUzzKQ61y4KKSgqouEnFkRf4acvkI2AunoPYMOrqTFo/CTPZzoNDrrfoQmxfGzg7eXT4XV19T0bC3T6kryAzsnw+nrajmRw8zll9yCP0XT4VxHitLiNJRSYvJVUdUMjXvN8t+l+q7YSL+JVnGGN5aPEqGOOnDs/IiILwfc9FzlVQPpBLO/FZYKp8mSCnjqiX37Hsu8xLD8QwaAVWHzudK9wDjIMwuTa9l5zxTT4nimN1jIqOCbwQEj5aKGxv0BP0VsmFao4rxXD5ZKHEquGvdBYAOGYx+wd0QVePY5LgcOHR0UUGGl3XUk9ysWXDagTN8VBLE+cXbmYbuPy6LVrsLxWnY2fEnn8SO0TRoPkqDwbEsOgxmOepmhpoYm2yRkkTHsSurhghxuugopKeKHDaSI1MzKcnKX3ta/Vczw5TYRJSnDK+hBc8mQvdpYKaj4nrsMjloaSCOSIExtLtQWLlOQ7KoqOEzc1DhM6T8MZAQIx0sOip4xxTimEYjSVtVJJWOw7RsGzHB4Iz39x/csKKshxOCXxWGwROjaMj4XG99bKOofVV9VRTV1HzW0wAcGn9oB7LPOlmTcTcbYjxK4Uj6BuHxSm+g1OnUrCmljj5MUzOfR0/Y219u61OKazCMXr4o8KpJaB3xmV1gfosWh+6Yi9tdRT1VVmPlvaNbibLR1GKMmrXV1JCIb+kDYJxXmrpW09Xq4G4f1VypxG8j3sw+OBkhvkA0WbNKyScvZEGgduqo0iaXEs/iubFkb5chvc+/usmfBq2meAxnNuL+XorUWJVbLRxiMebOTlWga+jq5QKqMUVZG38KRmsbvms2iHAfuqhxujqqWapc1l46okekkbqXGKLAKKmmkOOPrawyeaJot8yFjipxATcyF/KcST5VXH4UshqhmdJrf3K3isNjF4Y6TDaSngZGGG0ozG77+yyqumqYWRPnpDBnboDuQlNUSSxRQvkJZALRg9FHLU1VSQKiofKBtmN7LSoctwpY35Sx4Hl0s7oVrcLcPz49iTYmQySRU8jDUtbocl+/uu54y4Ew2lwekxXCcPm5xnEE0AaTzGdyOjtEHCcPSPGLPkHLMscRfGH6A99eiKbEIK+skqK2GNtr+VgsHWRVbY8KE9RDQyxeLGSLmC5t8Y/uWuzh+hqoqd7yMPiMYAdOCZJTubDdZlJQ1GHYfRYdh9YQWS1HnLS38Mi190uHK3DqXEo4aWBpllz5ntG2mlva6zcXqMK50WHUchHhgGc25Ice1jspsEqzR1JvViF0rSDK1ua/ss0hmxPwKubis8DJCyR5iDiRlffS/6o/vnGcbnFVVTSOdGSA+MZeWPeyv4/Rvmw+mpIxEyMXf4iQ+dx0WRSNqcPd4fOMr/VbqqJJ21dbLzpq6bNECQZzc/IKfDJsS8RJUxP57GtvKXutf3KkldhkDHiOeWJ09mZnG4Cgw+ugw/nsOaRkseQWG5XGZGthDH1FVTx+OjaeZcmU2A+RXO45RVOH49WUs8kchMnMuw3Fjr/AIqSTJPBZ4dTTM/1ZsD81G2mYBlfMXnq52pKQCH4QY1ha0g2cHHW5WnWsZRVDKQD1/6U4bOJ7drKgYow6OQSwRODha585WrPUjEquOhogJqmpcGFxFhnVU1dPJRSiMsZNLEPVsLHaypzVXPhazwkdx/BT4hDA2lZVw1QLonGKanJu7OOyoQTkzc5gLBYjKdiipDIyJ5kdJZz9W5H6NHZbNY/K2ijz8vyg5RsD3WU2SPLknB2IItY69QtivhrZsLixDxjZYoovw2utnsszAv4ZVYvU41R4dQwYfIyaUEuZCLix1K3OIeIcS4Q4jmp8MZy/ESc+TTNG+N4FjY7HQrnKOeQSYbRz18dM7l8zxDRrGff2V7il1RXTumrquimln9L6fUWAtp2VjIXqfiFlPB4V9HFTskbkEzARGe3RdVR1dRTR0tLUzRuc8bsNxbuuLmqaniCjosDkibTvYQHVd7MIA72FlpU+I1mF4o4tqqcuo7Ql3K5jHC3x2XfHNJh6dT5YzmAstGKXyrJoZq99MH17KYCwe18F7EH5qxFUMc8MYVpGgXXOqBKJpe24TmMjzLKhcdUKJ4N0P1WrKPmCLmBVyboM6CwXtTB9+qiJskCQERP5eyLmAdFW5hamM7CcvVKFoTgnLdM57bKoZWtO6IVDHaEpxkKV5c0rGrPEZ3PZESR3Wu5zPzIHGMje664flYlylZHiR/GkiIZeyGnjeXAltyegXVObE4WIBHYoY4aaG2SBvlXWM2aZVNh0xtIdPZaDInxMykKZ82X0gBVqp1RMy0EgYsz5tC5kOfIXeZTRzMta6idTRxsZkGoGpUTvINFKFmdvOiLAd1izRVULuWNbdQFaiq3iqbHI/yHdXJuW4eQqwOfla6Nub4lVLXHVXq1j7uAboFUAk7EfML0YuUp4p7Ns4ojO/oq4gkPRIXC0wN8sl/WU2d/c3QE6+ZK7beVVLlIyplB1cpmzXKqgi6ISAFTiW0I8jhlJtdO5jWnR11UbOLAWCnjmDtysU1CQ3aLp2zNIAy2KPy2QANGrgjSUObl9Sax7Ibs7fxThwHRYoSx7a7pyGDoEAff0py428zlKB+XsmuO6HOL2cnbsihyt6JvN3RZ2hM6Vt90QBamylEHNeEVggYZgEYeQlmb1S6aIorh/leNFG+AjVmye5RsaeqCuGG9nNTOp228rVayhM4K8ime+JzUPmarzhooDG2+y3aIuYR1Rib31TGPRQSDIVWVgzu/MibO4fEql3Wum83RKatpeKd+dLxXdyocx3dLmO7pUJbQFSBsnM+YKkxziiOYLPCFtZE3uhMxDr2UIJukSSUqEtYMrH9EDhZQ5yCpWzAts5WlM8C2YfooibKV+92qF91WDZ0QN0DY3v8AS26kZGUU4cbqRjjayDkv/KjaLHVFxFYnzBTRkka7qHK4bIo3uDlzpUxbogLGNN7aqfykKOQaLLSpO8+lMCRHZqkexpOqbKbrUSyovB9VtE7C4FXDDpoo5IuWL2XS2UtM8sHmJKnle0t0KpszKRZmG4kD5NCqpvdWHsdfQKJ7XNViGJCJTpZFzHkalBlsiWkFdpKlMBIGQ2UMYsQ5WWSDMLrNgZIDGMxR083Ldup6hofHo8XWe+Mt+MFUabKthNlJzWHqsuA2f6f1VkvsLrnMNxK4cnqsmz/lVGOV999FZDhZTiJQ5TRyXKqtJJ3UyzMNWsiRqfm2Va7rbpi4rHFq1rnDunEw7qlnIT81OJa5zbfEnEo/Mq4cSNk6cS1nme6NkpGip5iFIJipSWtGa6HN7KDO5OHOKnFpOHt/MnD2qAHuiBUoTZ2ps6jzBLMEoS5ggcdUmuuk7dKEYdZy4nEWPw7iQSA3zm+3ddrbzhcrxvFypIath817D22QdSHB7GyDYhLmfvLNweq8RhsZvsFZzrTI3TC6AvuED90g4E6LVKdRmS7rXUhNmabqq82KerJ37oVI3dM7RZVG7ZCS5E7ZMRZCUJLkJFlPl7p+X2Uo9VbI5IMcrXLaE3Lb2UtpC0IxsiLUKgNMw6IUk9RMHJ81xlLbjso0QHQLInjeGizAAPZGJCoAEYKvENUz1bBnpxc9VgYhj8LLQzSNlmOxGw+S1MTpqqrj5dLa53ubLn58BnilE1RHTxN7ly5jKrqnFGuc+CsDG72CzosQrql4ZVTmQe6lx+rpo5+RSvz29RGyxRPP8CLGTWrg57chETR3BVOGpgoyMuvdUTJLu/fqgcMxuVFtrwVkEsoysjb9FXxQc6dgh1vuqdMbShb33JI6Pmx10Hyuqq7gkj6SK5pKRtvj6rusMdh4p/FZRI8C+q8rnwyUxOkkqwMnY7qxhuLQYRF5ZZpfduyCHjKWarxqZ0LC1ptqeqx20s4bstGurH4lOZmtOY7BT0cnKYWSwgu7HosDFbf4kS0aswOveNrXdmqGjw2trnOFPAS0dTogoO3Ubi3ordbh1VRTcucAG19CqxZbdAwKlZM53lkOg6JMDCcpNkbYxdaWXPtpWDWM2d0Kvwx8psXN0Lza6ynzhrC5k1pB0sr3i+dTRzGMhwFteq5o2JMUqsPs+B7SzoCN1gzYxVurC4xwtDzfys1Qz1jpmZHFVSAHhyQTDdquIjURZXM5c1gOY11v4WWJU4jG2IwuaS47vB3TyZWszqq9rX6rUSEKp7ohG6L/AGjulC2MuzT6NUGZrSppTlaLLY1cNEDIqnk9RcrMmkzSlBT1b6eUTN1t06FNX13ipTJ4eNvta62JHeWLONwQQtbGaiCroaSsE8fN6sB0UMmGvr5KTDo6Qw1LviJ0XVYhgvD9HhDKGro81Sxt2losb/NQcvwm3EZMehnw9sMksZByPNrjYr0fjqA0z6KvibavzDkt6XXC4fh9FTsEohnbUDZ4OoW5hk5ra5tRjc1RNBTgGIk7EbLHIWOMHeMw2lp8VkkFYCD4e1iV53iVNVU0+RsGQE2Fzdd9x0+CrfS4kyPnQ2Me9ugXAymP/WE/NaiQWJ4TW4ZEyWcscH9hsq0cMr28wQyZO9l1FNQ0eJ8HyV1RNKZmXIJcTqCsKjraykpDDEGkE316K2KWRzRmcrWHxeIkANsvVVc073H8F1+wF1ajqntpTGwAO7hZCq208ch5DfmRsoMuf0tWpQ09M+ia2SO7j1ujmwp8LM8cd0Vjuj8tgNVPHMynhGXSXugfmLs3VTtp/vGMlhaHR6ZRuVRnvkfIbvNynHoze6JtLVOLgyE+U9TZHDSvEbnyva3Xa+6CAalSx5y4Nbt3VloibDIySMAjUFV44ZJTlZp7oL0XMiLYiw69QLhWiKiKdtbGy7W+oX1srWGVHPpXUsgu6EXGu60Y6Wlrw2mZIA+T9o06aey58imdBI19U7wscnh6iOxJbse6nwimhhkdNik0kXIvynMNjZSOhkoGmk5hDW7ELnMUZUirOR8kgIve9lmynatxahrctPT4c2WE7zVDdVyuIuf4xzHMja3pkNwrlLib6bBWUk8wN73AVWLlztMjgbX0RFHlZ3jQG2uqhkY81JLYwB/BaM0EJp/EUtw52hB9lTpRI4ebqNV0hoLmGMkTMIvt7ooXv5jBD676DutCBt489S8FovbMLlV6KdjMQ8W5o8nwgWBVF+sZRzsfBIwRPy52kfxQYXVswylfBNAXGQWDgdlUnq31EzpDpc3t2WhRMhfTCqqBzHMd5S3qVGQzOgpIPBzxB1UXaOIIyKeogqosMe8ESticzmPYLgXV2mnZPjX3pV0UehDJGvOYNHdamAYnhuF47UwQVUbaJ+8Tm3Dj7BZxF7hd1Pw5wn97VT5ZH1LyyJv+rudx2VvBKuCXA8fwqBgnvEJI82zr7/osaXFqnF8MrjVVzOTFNnjicALC+litDgLi7CcPom4TiTI4rPJbORdmvQn/ABWqYNxThcGHfZ7SQMhyuJYTrsdyuYwamqqnDIYRLHDllOUuNiwrquOMUocYlhjocQpp4YL/AIfM0v8APquZltSTsoqtjCyAczyDqszk3Dsa7jjE8SiiwOSGOB1xHPK2W5lA7dkfBVNh9VXVpo3kNjLM5dJ+0sevsV562iq6x800EcnKjJOXZ5+qmbhOK4PV0VRWF1PDO4eeOTW30WYyHovF5hpq2or66jliheWRtc06kAWv8lxmPcQ4VWyw5Jahro9L2uLdwlxnWP8AFx07J55GEW/Fddiwqidj6RtNINjcEbhWRqUNS/DZZIJCJ4ni/PJ1bdQT0OI1dG+uw7CJWUcJsahwsHLHdJU1E0MLDa5Edztr3XXRMxRx+4MYxuWWljazkxt9BKcRh08EkTM0+kb/AFDup/G1MpZUR1BHIdYADW3YnqhGG19ViFTBHNHMKQHM+9owFlyT1tnRxTMMZNgAsSOkoKjAfCYhW47AZJpZPwiBeyw24eWStqzPEA+z23O57Ko11SZDSTzcvOQJIybH2V3BJRhlRyqqMy04cbuOuVWyFSsfVGpPPhDe7QoJmsEwczRkguAtlhoG4xz5AZIZSea2TQtHcKhVQweLLI5RK1rjZw3WoyU0MMYo/EPGzrEqCpMPLBsJHM3uNFp09GyKFz6upjbH8TNyVSqaSTM+fNG6KQWj19Q6JKQjpeWWtkerAdDI11L4X8S9wOXe6jkpHwCmjDPxC3Vl1bpaxlHIXvyTNfGRltrf2QZ9VSUx87G2B3aNrqCOmiif+P03arpgmqGBgYYnP2z6XCzHNJAv6uq6RKun4Xx3E8IqZn4bVxU8MhBm5guCB/H9FtcR8W1b56WbxVTJY52vaS2Fw6iy5vB8L++Iq1r3uhfT0fOibEBeUjofZVZJIPANjq3uD4BaMg21Kti3jFdRYm+hj8WIm8x75HPN+WCRuumpsFGGtpsYdNJXVj6geEqY5S6NwO12npuuLwnEqXC65tbVU/iGiMs5ff8Aguji45gZiFEGUkLcPoJuZkhcQZL36exSxkcXNqaTF6rDqyCISMkM3Ma22YHXRZpLfCQzj1XNlp8S1eFYvNDPS1Essz5jzTJCYzZ5v/BZM7aXxnhqI3a8gN+aiUds75qhr6ieRxvfzEla00Ana2ancM8hsCSsmqo5qOVvOsHjaxunnqHVMMEb2gcjYt0Ky1xak1LB4MPk5ZeCA4F1iCpBQ0sGGOrqp7pHiTlmNm11hyzPkY9r3F7nkG/XRbstbh+KQNpoOfDVVNpJQ4/hh4C5zilJp6ySqpWzU9LBkADC4t/EFuhKVXXQYv8AiVUAhAAALRYtPe/ZZE1PJFeSN5GtrX0Ksw1bABnafICbAXB+alFLLqOko2n72w41bZzeB8chBAHsPmh4fnfhGL01ZBJblOEjg8ZgR2spcErJjXtmdWxeSJ8ULqjRguPkrNPwtPV09f4aoglkppBaexyX3Iv0VxVn1mI+KxR9RJHDJA9zyA1uUC/YDoqN3tlddm+oI2IW3/NjEsKi+9a7wsbIreSKUOLmG1zYfNUpqdk1WYIyYdRlE5y6FdBHSuJr2FhvcAAvGgTPdM1zoQ9xaxx0abfwUUolge+ke4gRusQNrqSMvimayA7OuH+6zMC1R5mwc60habsBIO/ULf4bfSYliUWHSU9bLIQXxx0tuYZO4utThuqjxLC6vBMUwqKuiDn1jWueYxDJax8wXqPBPC+EUlLQ18+GQtxGmhDDIARJYg7D6rFDzbFGcXYdNSYJHhRmbW0kkcUFwS7W9z+8L7Ln6WvfgNa3nsIqIwWTQVERsQehBX0hUUdNUPifNHmMEgkjLt2ELk+MeDqLEsewfiB0IPIlZBUNDPUwmwPvZb40MXhKrhqJY8LjxCpaJwT4QxEZB+6XjZdlDwzDAzKHyk/m0urVDRmmZAKmUyyUl2Ne6xJZ0uVclmN11xmaqUpBFEImZASfcoiLp2uuiAugrSR2CgI6FaEkYLVTla0FBDIFGGqwY7qC7c1lpD5QmI6FPmCROYoqM7KpPmb5gruUKN4ANyt4+TMMpstY8mxJAUgq7Ascwh3W60XsjiYSwWXP11Ry6mQXsu2H52Vl1bm8jrtPuqkkkg9L3qs8vaOcyTN3CB073tuu3CHPk0aSrkDw17yR7q4alx1bqFgGqLm2tb3Tw1k0J0N29inhQzz+LbmmEkVw657KEOaxwe55FuioCsJ20R1DzNG0s0sNVng1yhedXOe0sbpm6qoax2azzossVEg+JLnP7rfCGfEhrt5UpzOUrQWWzPusiGqLT5irbajMN08MjO1xxuL9UzHwNHniZ81A2fSzlMzK4EdCs0qKrkY4FsJt8lnE5XZVpGmv6FE7Dn5s4K1EsTCi83KbKVNNSSR65b/JRtGuq2I3bIVYMY6KIxndEkGYhGyYg6oS3RDb2VpnkuiseeqIVn5ys8GyNrvZOK8mj4i4uwp+c62W6oBzg7yuRiX4XKcWrW2TO7qVsl23vsqWdhFroszhss0trgka8jVPz2tdkuqWd42KHOb3Si2g57SgzhygjlLzlc6ysQ00kjgGnQ9VmWrDcjZGHoHNyvLL3ymyG5VZS5ndHJxO6Pt9UAkCNrGyI3YhXEm3LCJtUS62UKMwa+VSRwgG5XJEokUzWNkCHlMsCpIg1iw3aB7LHZR5BmV9zIz6kmxxE7K2Uo8hqbwgO7VoiGM9EfKaBoE8QplOo/LaypyQSRuy2W4+I30CjfCx3qC3GaTDDyP7ImseTay1fCR9kTaSMHMG6pzc6UKeB5lyqw+nJeAGX1U8cXLkLvdXYspGayniN0xJ2iKQx9lDmvst6emhqB5marPkw0RmzSdVYysyhnu2QXIKsSRlpylRZLrpbI2TNItdEC0+pVzduo3RMe741aTyWDnA8h0QXKISstYG6cObZRBtkJUoabXUBHZSxzACz1mW0uh6oCxylZLAVM3lO91zaQxuIPmUjmZgnkh/KELQ8KKgdFI29wg0C0Mofu1C6mYdgnIpRzBJwEgyqeWlIGYNQBhVtmkHLylPp6VK5pUdiHbLVs0duvlQOZqW5VZYCQny6pzapnPitplQFtt1oywtI8oVd0YA9K1yhmYQDN2TFzgVJIX2Dbeyrm9/MtMphIXDKUQcB5nAH5qAHqEfMPZKCLrelPzXEa2QE3Qu2ShK19ip2yqnmCISAFJgtcE2vqViOe+izxJoibNYqU1bUEmiSpxzqRkvuuU4U1acg9kBv2SEgsm5nYK0tpmP0R52qqHgHVJ0wOynEWszfzJFwCqh7kYeSNXLPET52fmRcxo9JVNzrFSMNwgs81v5k+f2VYmycHqFmhYEqLmXUBf2Sa/VWmloPHdFcEKu1SNKzRZ3bLG4rhfNhLsovYgrZdqFFWwCeiljf1apSsDhKfNTOpnDzMGq23Cy5Phmp8PiLoybZzZdfIFsVXnVRgkFSPy3UXx/VVhODmChkbY+VSN3RkAhQRByjcdVI5pvso5NPUtUTKN0llBzzfzJpTqVETqt44xTFytscDqpmHMqjTdWIdws5Q1jKUjoUOUojso8wXJ0MR0QfVGR0Kax+FZoCnbuns5E1rlKCzFOD1CbKU4apTNiD0RehDEWRqjQQ7Vc5xfh9TiTWspxcg3/ALl0uRqHl63spMDzlvA/EEnmbSjL3L1l4hh1ThtQaacWcvWniot+CLrBrOF2YlV+Lr2G3sVKMXngp5XakgDuSoXMsdF3mKcF4eyMzQSbbjMuKrI4aSZ0G1uiU2rPjey3mH0U9FzObZh17d1WMgcfKia5w9DiD7LQ7irNE2iZDOwNe9osAih4XgqqUF7LLnqbCKoxCqhqxI4dHbKc4/XGHlGYtcOxQaEnDTI9YI9e6FuGUEbTz5gx/VYEmPYkx5DJf71SrcXxCdpM0wIPQMWJhhrMwm2INPMbK15sLLrzQ0VNT8ynmyuAv5TdcPgQmri8SSOsLaDTuuppMNY6LzTv0VpqJclUNqp6+V85LyTuVVrI2MfZj8y3cTpao5neHkAG7raBYrqR8x8rhf3UmFVRlGo3RCWQ7D9AlLSz0/mmZYdCog8DZynEcq9xJujbPIxnL1yqFEHWPpBUFpojdHzG30UcnsEz5i2Ehit4bEytp5GObaVmykCg578uQbKs97mDyq5KOVI6N+jgVRfuVqBII+ZHnuAlzC5uTsmkMOjYy6/W6Zuy6CWHI1+aaMvb+UKSLwEtc38EiM/CTaxVV73A5U2a6DQmq5oqhgjmcHMNo3X1AWpBSYjO4VFbXT373XNBz8zSxuYg3tfddZV1L46KJzDlztXLJpdoI5xUNfPIDD76JY1RROzS8yYAkCzXWH6LFosQqhO1/M+nVdOzC8YrYvxaJxYep0K43xSYcnjOHT4a+JviZXRygnU/891klkhdmay7O67StwOOoe2Pxsr3xiwa/wA1guanw/EWuLPDTho2HKXbHYlI6CSp8PUUsdVymHzkd1FHA+RwY03J2SPh8oY5rs43JCuYJSz1mItgpna9jst2IGc+ll5F8ufRw7oKg0rHhlPrbdbYpH4NUzPrmMkY8WsTcKrWVlIAZMOgbFboFBSlv4Zj4meZhvuulw+tp6+ZkLzcPi102KwoWT1VG+quLsvcWUmCRSVEvPp5gJYPPlPVBDUU/hq+RjWWaHaBTOgjEgnieQ4dkdWIK6sex83Lt1uoaV8kFU+COfnQj3SJVTra+apl5FN+E/qG9VHC+SCOSnnhEmc3BPRSVMLIa500P0KhkLnPuqq4GtliyHqoQeSeSW6KWJ+TLp6d1PiUTHytkp2XbIL3RE3DzqV01W+pcGBjfKCVbwOonq2y1mcB0DyGADouepYnuqRGf6wEW7lWYK6qwEVFIYP2v5uimWI6aWvhqKZ072Bj4zZ4Gy5iTFKlri3yn3IUtRXTUZbymBzJ26hw0JVGJmaMZt1nHAGDzpRLNVRWHwtFv4LWw71PLXMyDQXICo01OyWleTPG0g2Gt1DVRSRRfiNsCbBWYSmzTQsNUIZ3xhtiR591DiGG3z5GvjV2twOiwnB6HEYMRj+8H2vTjUEHZQuL+e+SSQuL7XuueUCrQU8eflznyjcKLEmUME5bRDyLSp5IJZDGenVZlVGyfPfcKxJRRUrjypLWEguFrUsoo4+Tkc4vd5Wja5WVJJLJDFHzS1rBYtCnie980UjzflEH9Fu2nRVmD/dkHKxHmxVEkYfr8QVvgPh2tix+mxmuo4vBSB4a95v5ztZaWK1NLxzQVVXTxup4MMizxvdoXSW2PtouelhxGhwmhr88kcFeQHR5tLrFMtjHuG8Owg1VfiTwx9RP/R2NuQ2P5Bc3DjGHUWF1+HCDPUT6C7S247r0bhmM0mE1lVV4d4ivEwBZlzPcLaDVcRjOF10+IV2L4xSNpHcvPEy+g7Lckww8PxGbB2OjfRwTiezyD0WxhuLwfeUtfPTTcl7fI9ouLrPwmGhq6iqmqqMSs5Nmgmxv7Fd5wnFWYzLNFhrqeSCmjDHQTmw16XCzGLnMsbDa2HFWSS1NVDD8DYnaEqGCHDMSMmHV1VkmhjJgne/KAfks/iXA6/hPFnyVMYY2SQmKFzg8NYfcfNa1a6ip5aYx4BDVmVolu02FliXRzddO+jgjoH4pBWNO5ac3LN+6ypZJA/K7otTHJ48YkfVxULaPlEAwhZscb6x75JJMjIxdzrXstwJISx0bSw3eTb2WphMb5sQ5Na25jaXt9iLFYsFVAyWFk4/CDs9+9lrmviqcQvTnlguyAZdx1WRNUVeLYngvhW1RyRvtymgA6dL9Vm01DV080QMd4h5yO3stGsquXxDyI3Mh8oufzHus6rrZI6h7GSX8xUgX8UkhDWVE8AfyzdpDdCOx7KjUTRucH0pu14uip8ZqKNxnieHEDztcNCOynnNJiHNxDC5oWzSCz6U6Pz9wlDJzyyzlskpIcdSOikcJ43smp7B0Z0ICGKCerqWihgdMbAyAfCT3Sm/Bs2V+Yg2tt/Faoa1TPDMwP5NrAjfuqNGzDYo89cKmXIbZI3WsFVMs8rrR6MjIJUxmMn4bgwDoAFBZke+efn4WyZxj6OdnMYTYaCYpK58n40El8jxYOWbBWyUVQ98c5ia8WJC1oKmke7I4Zw/9oNsx7/NWh0FTHBxBHRVU0nIpZA+ONlrZZbW+qwMSwCekqn0NfDHT1XLMkDzqXhBSunw3FaZhfLJEycSCGTY+11o8WzT4xUR4rJM4+u8ZtaJl9ACuuKsvh8RUNb/8RraikEsdopI9cpO9/bZSY/hhY2jjpXxztmcQ2Rh0kIKzJBYxyMJkZe8jSdbKepr6aWqidBBJ4OCTO2nfJ/iEnNCioxG1z6qFzXuu2O+g03VCQct5Y7cbrsWcWYThuF1DMHw2ohraj43Wcz6X6Llp5J8SqhNWTxROfZguAAApa2CGqn8SJ3vvZuQaLSpKSlipM8zBzZCX5jryx3soMSw+theJzAx0FrcyE3jU2MZoJ6fwk5MJhD2tNvLdBUqpmVR5hfd3Q91UdopXO1LdM3sob3dlRRNIN7gHTqrUBMTuY2+Yi1wNlXygOI7FXqSB8zcrNDyzoUmGVpzvE07pKiGSZgjsCw2se57o6KkmqMLNqKSd0klg5u9/f2VKE1+SSGiDpS9mfI0a5FHFWPiDWMmkhcDcOabC47rnRbSnw3GKF4oJqAxveckQtmDj2BW9g1ZgbcLo4Kt8kc9MXirphnDJxf4zex2XOz1ddH4XEoK+R1ZFfmOkdezwbaLveDuIMDlpZKTEcKhhqK082UPj5jZel476i/ZbocxxZieEYo6khwvDYaOWAFkpht5huNB/epMMiwbHMNrcRxiSTx1JTvLHMiuNPQXKzxVFiMVVV0WEcPikio3eIbKW2eYiLddxe59lxf4zD5JZY2vbZzRcB3zQWYudV2bYulfoABuVsY9hlLT4XQyMZy6mOBkVRTjR/NDzcvHyssyjrn0jHhrWFryw5uoLDcLrsdbBR1NXiWIMs7EaFk8fcPkGo/ipLSTCsGnxnCMT4gosQdDR0Rj58HLIeQWbhdBgXEPFzq37ng5kOLStIZU1sxu6MagAbXtqsLhHH/C4biOBvfajxelJc9+r4ZWDyD5G1l09fQMl4VwXiLEp5ZZZqxh50biDFTlh7bZCFjEdTh/F+I0OKUuAcTYjFLPVuEbZ46Qt85+C/wCbULtZI2SMySC4uD9QbheFV9VjQZRYx90SVUOGVTJRVhxcXkPuC9g2uu/wrj6jq6M4pPS8iaQZzATmEh7t/Iuo7KTc/NQyZli4dxlS1uJMwqvoZ8PmqG8ylMurJx7Hv7LbdZSADHm6nbI2yitrdNqCtImdKAPM5U5n5ibbJ3uddMN0DMPwlVJIZ+eXZbK+1tzoiEdzqrBLLJkjOUo2vvoVcqoLgnIs2WCaNpe3YLbKyDdO6NrhayqU01iM6vNcHAWVkQSRfhOC5XEaVzJHPa8O+S7A6rma+JsXNc0WGc3XXT/E55MppNkUMjWAteAfmo7uBTFq9bgN743E5AAnbHGSOZUBvsNyoBuUnbrSYpXRZT6wR3CtGWN0IjB81lnhxHREHu6JSWaRha8h3RRk+wRvuTmcgQENUTS78yHMEswVpmFltS8dVM2p/M5Uc5B8wRB9+ixxb5NWOr97KQVR/OVkCRw9KmjlceqlN21mzxu0I3Qy0cU2t7HuFQY93dTNqi0ZSVks0tK+P3UUkJtcBX45hINSndEw+gghLKY591G8WWnPSGxIaqboiB5lqJYmFUORJ3MQ5QtoIHVJxuUySAgHXUgcog+/RE1yHJLnTl4A8yBW8MpIa2qbBMbCxvos5NQKnwnFZ4RUw034b9Q6/RWqXn0gDKhhDguopKOOOmjja3RjbBVcQwmSX8SNmy8mW13jFzcrXueXtG6jvrlctpsQp22qYS0dyqk9HEXF8bbXWsdtpOCoGRnYp8j/AIUwjdFIY3vB13CvziMQtMYANui6zmxSszOpC8jYJ4xVhoe2le5vySklucz4XR/7FguVt8RskuNiPmpWm52SZRTiIzPZZtrhRskba9iFmxbYy6kEYCrB/sQrpo6oNB5ZsRuo2ANRBzRuo5YJ4QC43BKhdI4LIvWituoJGdgoOc/8ykjqR1WhAb39KNr22yndTPMDvQ7zdkzYmk3VSkJfYom1UY9SaWnflNgqpiePUFbS18VLPzJF0b1UZrpZSAXUU0kEcg1VKWnew5cq0w1C5sTvXqtRKUxTE4m1lG+J46LblZBl8jBdVSxp9RXWJYmGY0G+VW+S9oAc9p+SkNO3cBIRtb0VZohE4pjCSVK0oyR1WJlpAIiN1Zg8hQjVSsCxm0uxWcPMj5LCqrCQpWzPCw2lELQkBZPHK0nK5T8phXMVzDcIG0zD01V0MAGyJWxQ8GR0UElLZxsFrWugdGx3wpYyREi5BPRXhTsupmwsHwpYzDTm2yrupDmvdbhbH+VROiZ+VXklMWWj0vlWbM3LIQullibbLZZ01Gwk5mrrjmxOLIaHE6JOarMsHKNlA4WXWJc5gKciyEmyLUlaAu3TIiHJvN3UDAuRg2RCDS5cAjEUfR91Qo73CnbmCha5jD5VIZQ5ZlRF5KNrtFBzGd04lb0cFlEpKa6DmX6qQDypMqIbZkiShB1sjDLhQgIueilsQFGdDoiEjrKNC19lIy5QhtxdTU5AOVZDZXdkJuCr/ksq8jGE5lm2kDX66qdkgsoXssMwQCSx1Si16OQKZpaTbus4Sq3DMNCVmltwdfG/DceaWNs3Ney7Zv40LJraPF1zHHTGtq4KtgtdbOA1ni8IZrqwWWxJKN1CTZTSm5UTgtQzIY5NdVaY4EKsWI4nWSmeSd2ygmZe6kc8BCTdqNqE4IBVY3V+VoKqyMANg1bxlykEZylW43XtYqs1utip4wwelJhYTgkjVPoFG6djQonVGdc/DbtYzN7p80bNbhVA4d0g4HcrXhpyXBKzupGBpVAEDZEKh7FJwTnK8WN7ofIDq4BVDWPKDM+UqeGvKV0ywj4wmEsfdVRC6+ysRUrrXNwp4cHmsRuRgeyCKN17OVtsIsFjKG4QBjjs1J9PdvmatGOmBGjVYFGOouuUy04rE8EpKjM99dMx5+AbBVI+FcHgYKitkjk936ldjimEUsrC+MWeNlgtpfDS2xCVoYO6ljg8dosAk5k2HOLchsWgaBYNHSvqHnIy9t12XEND97VTqTBIGtaNXO6XPf8ARcxLguI4XPkmfED/AGt0bdBUGgZhJZC9sLgNW31K492dz3PB0JVupFVkHOF2+yrh2lrKMBAF8xWjDQxysHLnizHoVnWzvDO606HA8z7+OA+Sgt0f/wALa48xr3n4Qp4OJ2Ut2yMFz0vt/BZtVHHR1Ok/NsNdVdq8SwaopTlw6Nsh2y62WW0OIcSPr2GEPDWHoAqEM9FE4SPjzPHW6ytb7qzTunY7NHTGY9gFbFmtrGVzckcZDuxWZLTyw25jLX2V+KOsqKpv9F5J632WzDh1JUOEdVURNf0G6o8saWlt0OYFCyRnLvcIDOSMrWgJCwnOrLJUtRLS1TZoz6OndRNe4t1KY91EaOLiCbLWwD1DzfNZROYohLI1pYDoeiZIWC8rhdD0SA1ROjfa4F10iT1Ru2UrKeQx8wEZfmq5zdWOHzCNkj7ZbmyIuYZhc+Jl7KadjZGbNdu75K5iFVUh0NDUQuifBob9VQwypNHiENXe3KN1oYniBxzFWzjLG0WspS22MDkgwhjsRndHK4izW9E384sbqKrmOqD7NboAEP3pSxP5EeHQG3XZSy4nhtKzmU7D4n2Gy88wosOxCqbOfxiTzOvRWavGsfoahz45I5IxuHtuAsujfy6Y1Ujg17zcXKHE6t81CXtkUpFWU4fPUOnrqmQuJuWsboFPh+JU9NUCDDqCQsf+0cwfiPCzRSVfK572XZ3upqeZraduSM+Jb6Xt3C64o6+iwrDMYrudiUbqWGJmomdlusPGo8LixUspJ2zUnxCMX0VR0uIzxOZUUU5J+N7itHA6QUNPJiNRStkcwgRNJ0utCaXEMFprx0eGzxOkAIdyiAVBhjuRO+eXlNMm5B0A+StY/iuKPjp56qWF234LW7LIgpo6sl72XLtgk5CtitE2GqNXBIJIj1CpMc9spe3S66dtHFJQy0QbYRgELmH80zCMauUsscz5A7JJt0UlHGAXVB9MYuVHUuFRIHTaOYLWC06Knj+56gOG91BmuJD3OYbNJuAtOjkhdhwgzWd0NtlUooecDHI21he/dPEySlqXZIHvZpdt1mwnQTUczJ3MuY3ZwehVSrbVYnUB8jWlz/hA1K1JpppYnCOAuaNS0bgKlDBVGRlXRSWI9JedR9FrkK00kzP6PVOcDHs0jZBG4AZTGXNPQLap8JkxKuLsYklMr4+YwNbYH6rFzZRLGAQ1hsA4a/VdcZVOJo4YxlAL+x6KOolfVgR2sB0uoOYHlHC4h1wVJISPidEWZwS62lyr9PM7KS/sqZNyC7Ww0UZMl/O429lnKEX3NmhqGiHUvF1JFTwVDZTncJo+g6qOnPMDX/kUmVsbuZ8XdcqWx4E1k2KiKQXbkKie50dVNGw6MNrrQwqCB9VeCo5cgF/Tos2by1UuXz55LNPQlaiVbP3pSnBhSCmDZac3Eodqb9CAjxHGHiajpcblIo44RJDHGbi/+CzocGZ4pkNTVNjfLa4v+z+a2McwfwzGCrninZTxWziOxA/VVlY4fvNxJBPJVV0FLK4RySNkIkAOgB9l1X2k8OYVR4W2CkjlmrZPxObNIXPDB/h7LA4b1wl1RVuL2y+cuJ3sgxbi/HMVovuwvEJkGQOGsjo+xKk+Q5tsNFNSGGCflufqdbXKDhhk7cTdDHVVUDxcskik1BH/ALqq9klNK2/wen2UuF1E0U088b8tnamy5jsKWrxaqmfJjFQ+r05YE/mBHt2XP4nUyF5/pkofHIYo2DQNZfYK5hFWXyOrb8x3NGp2I+S6Wkwbhysr/u6fF6OSaoZz25yCGv7BX1aeZz1D4myU7Ro43JO6gpmZniMveGvIYbHcLtj9njKmqrKOKtjmljPlkiP4bR2WVw/wvjtXVyPw6iu6juJDPoL9vmtxFEwx6mnfSymFw9CeSWSWMTPk5VhZpb17qziEpnlNNVUnh6yMlk+u6zJwY3ZOiSlLuHSwGVsuKSPkaNtLro5cLwrFKKKooRaWmIMo3kI+S5QEBlgtilZK6mdWQT8khgyu6k9lIQHFMMEGJRS08cYgqIwcrT19x0WXV0NTQPiqp6GRsTnfhudoJPku/wCA8NoMQxKixusrqUSwOfHNTz6F2lgR3W19pGA11ZX0FfIYnUdOwgtMZLGje57rtwHlNDWT09TLJTQOHPFpADezOpUVa6mNU4iQujvcG3XsrdfHJHiLnUFLdokJGQfhyD37f8FTrDJNKaoU/KZIbhua/wDFSRYbWeeSPw2UyCxc7qpG4biDpG8uklL9XssNCBuq8E745hWCFsgi3B2suiNM/FYcN+7XGllefDucCXMjJ2v81kcvPV814McEceliNyT3UlO57AZ2QvOTc22XTY5gFVhzIMOxHDoo62K74ZILZJYxuTqs12SonDaepMcD4yXc4ZQD2Z7KqGcVtbRR11v2El2m+rSPZaeLy0Uvg5I3iSOVvMlYD6ZAs6jdC/CqiMSefPdumtu6s0XgamjZDPQ3bGeWDfLdZRXlw0vbUVUcYjbys4sNwqkELKS5qoxNTyeQlrrEfJdLjMcdJEYKSPyPiyXJsGg9yufM4o4YqQ0NLM1nnL73J+qBq2hlw7OIJOZCHCxIuAT0UElPPEGuq47F4uPcK9W4rNidNBQmQRmPoG+o91CMYf4MUklPmlYSM+9h8lQNDUGkpaqlkgkdFOL+XYPUMRqsQr2RmaRjQy5LTY2CaebMzJG9w+SejlfEWV20BvTyO/KT1VVRlljdUO5dxr13+qHaz+t7qziVU+rqxflZYBymlkYZmA6lVxZzi15toSqHzal3dXIJjkzwzGOVmrSVng3UjQ4nKtC9T11RTvkMD8vNbkJG4+R6LSwCsZhk88kzIiyeLI4vjzFvW4Hf2WQ9jGNYx1w7exCl5ri3zrEwUkdCytra6eGrc+IOMkbmiwc8+3RaVLjOIU76Cqwqlj8Zh8l8uUuN+lx76qvR0r4aeSR7HxsfbUHca6KKOVkVWKunkL5ebazm2FuxQdViHH3FePRSYaIaOoL4Cx3h4S0gdRqTsubxAwnD4aTxEUs0fnNj6Qfg+d1nk1AqTyGSGXMTZpWhiWFVrHwGrraQvkLI2sbcEA9dQL+5UVYjpKqClcyiooJXsBiqxIRZsj9gCfqnrK2onwyGSve+oPLZBn25cbNmaKkKiqweqq8PyRyh9o52X1IHQf5rawDDoMRpKpkEcNVFFtG9xbKOpyd/kpLCtFCz7vZM0ta8xvZ5DoWC2/dd7RY7NhmDUuCUcUsD62JjC18YcGkkAkE7BcNieHz0GICCgZaIycpvNs0uNrn6e6ipcSno6yGtdiJPIeDrrp+T5LF01b1GOtxI41HwxhuI8yWoB58j4vxGxsILQQPrqrsHDkv8/YqaOQNpg0VMkNtnkagfpdYGBYzS0HHQxV88rGT8tkchN4hG+w89th2+i7yimFVxpidQz/q0DIye1/8A2Vwllp8Q4BT8RUXhXzGCpiPNpaho1ikGx+SrcP4vU18EtJizRDilF+HUR7ZgNpB7FavM1WJxJQVUssGP4Oy+JUAtyhtVRdYz39l0aiW+3ZN1+qp4ZidLjFBFiNE+7JLgtO8bxuw+4U7TqqFKLKISWKnfqLqu9mvlULTtkHdTRvB6rOGZpUsUjgtUWvkB4UEzG5Dog8Rb4lHJOHNypZTMqBlkNtk8FTYgXT1HrQCL4guiNAG6xK6aGGpex7c/cBaYly+pUqynhmeZreY7rWr+JjJz1W0Z+YzY9FE30q1KwjUtVV7Xh2UtXuxebMDhYplJcW1S5eui0yjyeyYNLSpxGTbsmewtSwDWE7hRFjmGxVlu6MMYXHMNCpZxUVYpqV0zruaQ219t04gh9L3XHyWtFiEbI2sG1rWt0TKVxhFDHCy0bowfmEddhQc1s9JDlv6xdTEUrpRMwZPayvsqoGQZLOK5ulOf8I0MLHu852Cq2IFl0csUE0XOayzgsurpd3xMViUpnhxupBIepV1lBG4BwcfdG/BxK4llTl7NypyhmpUmy/lKsR1T4yqJbNC97HMILDqja4u3WlaYqGvGrkiyGQKgD+VyljlLdypQKahadY2kqk+MtdYjVaLam2gckWRy+ctuUgpmZfdNlPdWXQ22QOjsNlu2OKBGnylFld3VkMrmFiaOtZPHa8Z1BO4VVosrlIeU4SAX9lzy9G4d1SOc6MOforTicuVYtBUNLW3cteKS6+flD1QqV1IKqIxvFwqkWG0zXh8kYe4CwJWyQLKPKxS1c1U4VGyR8ojI1uhoqWapmZCxl2g3K6KWNh9TVFQ0zKORxjc4gm4B6LXiC0ISP2jBf5JGCNwsWD9EeclO5wsuVytE2CK1i0H6KrPhEEz2vsfIb27qzG8XUvNanKSlZ9DTltvDt/RR8stJaTp0Vx0rTsoTYpclK0tLzmln8Vkz4RiLZDkylnQgrfaOyWUFajOmacm9lRC4tkjIUPMmccrGPJ7Bq6+aigqGGN7bgpqfDKSlbljistxstODm+RIwZi+7u1lYpKOqnv5gy3ddH4WG2rAU4ggj8zGAFJ2LwZpw18n7F2X+0q9VQTwszvZmb1LRstzT2SyN7qeIcHMMgfK6zIzf3CsNwuZ1+YS36LafYHQBNcK+IcWHPhz42XbdZ78zDlsutDInbi65nE4JYqt4EZLSbgDotY5FKpfdRnYIpAR5iLKNznWXeHGR50zt1DzdcykbIwnzKoZ4ktcDRM15+JTskYdOiMRRnZJ8hE2QhTRPJOqXI7BEyEh2ZYmbbWwW5NVE5wU7IwW2IRCJgGyxKoYy6+VW2yOta6qTAh/kCYPf3XNV8OR+UqlHMdiVKJHdHILGUoS7ogDndU+a5UaFmCkYC72UYy3UrHBZU/L+aExqy1hcPKUXJd2UsZ8kD7ZrJNo2uFzdaQhPxNS5YHVLGBiGETWL42X9lmNwfEJCbQW/tGy7IMzIXRiy6YbcnPi48cPV8jwLRt98yaqwSaj3lbJ8gus5dz5VA+kznzsBWvHyZ4OScWfENlHmB9K2ZsEqgXvLoyN7BUDhdSNWEOXeNkMzhKo5x2SbHN+QrZgw+NgBkF3KU0kd8zGp4sQcJYLoZpHhgjkudtF0tDw5QNjD6qWSR/UXshpoLzDP0Wo0ZDZcct0N8JNHhuEQgtZRx699ULsFwmVtvCtb7gqQxF5u0onXYAGrHOWmZLwtSD9jPKPrdVJsFqoGXYGyfIrdEtypWtjeNU5yOS8BW5rup7D5qaOnnaNWfxXRTCEFU5cnwLXiyzTIkpJic1lE6CVg8wWjM+11TdIXDUrcfmRVJeDuUYe4dSo5SMyAHqF0pi1sTv0bnUnNv8Sp3PdOHt6qUWtZ3EKB79UYlZb1IOS+Z3k2UpTCRSMkIUkWFvOr5AFKcNttNf6pcLTJ4kp/FYW99rmMgqhwjVWY+n7XXQVtE+WilhHxC/6LjsCn8NiOS9tbLErDq5rhyBz9BZWKltzdqqv01W4SQF5uozMRs5Jx1ULtV1iHNZZNm6oxIeipBxBRiYqcS1pzrqByAyu/Mh5qtFnchzO7JzIhzBUIkndMlmbdASStRBYnOKNoJUd9bqeJzBZKBhjj6QnbA862VmJzLZlOHNXKZFdlG0i5U0ULI/SFJmKZYtviLIy+bLqjvog83ZOsNJG7qVpuFA3ZTs9KxKysU8rw8K/HUA7uWY0tan5juy5zCw03yQu9ax8TwPDcQlbLPVSggWs0qXmuTZmncKU053F5JoG+Cw7Ds7Bu+2qgxTh/D4sME1dHepIuT2XWMDCdlKaOmq2cudlwjTxStqMNidkjbLb31WTGGSVALm3Y82svWOMOEMDnoi+GNwqmG4sd15rWYRDhpHiJpA7sg1YMLwHmtNPSGR/QA6qXGY3UlGXw4dNE/o5yo4ZxJS4W7meEL/mVUxziWbGCGxukjY03y3WRlGqljBblFzqbi6j58h6/wUzaOtqWmSKAyNA3UZglyW5DgRvpdLEJIXa8G0dJDB4ifFI43kWI2v8AxXDta4oouSyQA6INniiHLXOfDMZW9xIFDhE8kTwAwH3IuUTKvC2sDTG/5hTUjXV04jw2En56KDyou1RDVNlKQNl0WEry6MXchMpQyTF0YY7ogv7LMweqzkErRy2+bqnZC9vrbZBA9sYN1bHmYLpxKBFC0jyt1TSiSLzPjOXupoYHTfhskynurrY446YQTzNkt2UstmPq5pA0RizWC11Xc0uN8o+i1DNTj8JlK0juUcMEU0gjyAE9kstiqxTtMZEgGo2WjHSMgrBBUwNN9iGoKynfSTct7LX1HuEssEMs7qoSPpec3srM8nPfn8LyfZV4ap8Dw5mvsruV845ogMd+hWJVUllDByj1Uckb4Yg6dhaH+kpqkXeLuAA3WnXTufh0bbNyM291IihmQsm1zvJj6AlTxMkDg6DcbKsJHEK9h0vLk9AJOxPRaGtj+IyjDYmCTzyAZ9OqyDi1Z4YUL6oiJ24HVbIrcKkpxHiFO5x6jZUqd+DUlVzqeijkDvRm6LLLOkEz3F+cu+ZVzCHviqHGbTy6LSlx+OJ0goqCON0gsS7XXuo6jwD6Hnyf6Yfg6LMquYXNymSPIjkMl7+yv4tBg1PTRxVFFGOYCWOay1j8wuaiqnxR5WLRo8efLH4WusWDZxF9FlGE6ibHIRHqr03/AMqMY9RIYPmtCWehDuTZkcZ2eBrdRy0VLS0pdV1ILGuz6D+C1EqqQ4FikcTGMjZC5hvmc/daUVPyXZ5JgZeoY7RUD4WrPLgke0d3GyuWoYnCGnZtu6+qxnIGripoWNkbHq91nG+4KmODUVdFHJhcnJltbLfQqnVxSTWyEkDoTsqs0+K0kQFK+zb2ss8ktqR41W4TUieqj8S2NvLc21rH5rmqhv3pWyyj8IyecNctSlxCpmhfS4hO6UvN7uAKakglqJ3PhpI3UguyU7EL065Vzk8M1PJklZY9PdAxzg4LdrW0UtQ6hpWmWC/lcfhf81jujdE8skYQWm2q2Jy05SWakC6ctYKdkzXkuO47KWjgFROwPHl6p3SuDpYg0BjTsFFFSTMhhkc/4xYLbZhr4qOHEaiGR1G/42GxXOH0E/D0W7PC+LAWxxVEvms8MLtFiYZWmYcxrH1FK3nMkFgdif8ANUTRMfWU7GPEYZ5rE/wTUuLSxYWGvd5o5fKeq6SHBqXEYmYi+ExkND22de4WGkHCFVAeLJ8PxOnjkbWR8sZtcp30Vupw2OakqKGsxEU1GyqJ1dnJA21/zXMSgQ1grrvE0brgjsqdcyaoc6rfqHnPJ/musZpMLbsWkDjheE1sk0XOs1zugv2UuJ1chrW5nRiVlmF8OhcCsSji9VUHgDpc2urhgE9UKhkwc+WQEgdFUpoVmEPNfHBTSZnvaH2cd7qWgwCWWvfhhDqUSRZyRYgHuVLHP4iatr2G01GLRd22U+BYvPhFXO+oj5sNZEWFw3jXOYA+CbFgk8dJBGJaaQc1zDq73CzJcCngpXVtZL4aV7vwoza5uujruJMO+6Jqaho5CGAMBfubFcpXYlVYofHVBBlBvG1osG27KUmLrKTiGg4HwN+DeFqXYjOHvJB8hJ6g/wCCw5+O+J6qnZTQVApYsoZII26ye9ysetxGtr2+IrpnyTkWJPYbKGnqeTJFNa/LN7HYrti1AZ2TMfzJJw6+7s9ymji5sMj3v1jICv4tLzKxlXJSRxcxzH5GOuLIcUqsJnqJH4XTyBk4BcBs0hWYVmuJyabrSlxHnYdTUULCGRi779Ss15Z6m6oi6Rps02ba9lmYSk9LOymqmzuAdkN7FdhScXYtBQPxKpqDUQuNvDOl8jQf491wwYXutcD5q/T1RihbBPBmi15oZvIFqJYl1mJ4rT1VVDSz4jDQs5BIfB+z1HVZpwrhmiw+poX4zDV1jxeKcX5cXzKzMLwyHEIqumpKKUuI5kD72DWDe6gxakmw0x05jaGTtz6a3+vVIUDqmCKEx0oBMgLJHHUEey6jAa3EpMGnrsLoAG0Udp6oaXOnTrZcW7QE/l6L1jCsRopOEq/h+iws0YgojOby5rl+vbqpMqq8U1cB4Pw+vxKgixCumjMYnnJu25uSAF5943MxjJImuyCwutXEsZgjpqekuJ72leHjRmmw7b/wWLWNPic7Y8sbwCNVLW1uLFKilOSnEbAdxl3CuUE8ctTGckkLGaaHQHusefSpHYNC16ONlPTiqGbJUaZj8JHUKZi1iOJSSwSU5farp3guD9Bb/FY1RXSVDmcxzS/W9m2VhzWVFc8Vb+a7o9up/VNHhkhr4ooQL3vqd1iEVHujY2+0g6p5p2OiZblyGQXc7c3UtTh8hrHs5JaLZ7nYjuFRkLA4tYcze9luATRkY55cCBqVEJpnxGkzfhGTmFvupORAWPfmkab6Brb2HcqItnhuxzXNuL2I3WlI6JtCFqUvD9bX0dJV0ThN4lxYWjQxn3VSuoKzDq6XDKunc2aBxZIG6gH5pYrDcqSJzM7c78oJsT2W1wdg1Lj2Ntw2rzBskT8pGn4nQX6KOvwh/DWNzYbiEfngtmOtjcX0WhYkw5lHhsWMt/plNUB8fnjILX3sN1nMZK6PxGT8KN1i6/VdJgVQ/GWM4Vqq2npcIZKJqqZ7rG3YX6kpsbwKiwoc/Da+OWlqKh7IY76gabu2J1XM5sTkQNppx4iaKYuGVoOjgb3uqkk14gxWcQjq45zGRr0sQQVWnMcp/Dh5QsARmvqqNGjZV0NNFV0rHTVJBPLMe5HX3W5T4jjNXhcL8ZwuOOnZIKinq3jlkEanTrsszDqmCeOWoxL8JtPFHDmZudd1LxDjUuLUlJhdBX8yjp2ABhFi436/JER4riGHVlT42le6eqln5jncvKGs7fKyry1lTFeelfyJhICJI9A0jqCsyleyeURl5DztbqVfZNalNKWXaTcm/VBt8jEMdwqqx+rxdslRRxiVrA0EuF7akWt/wWaYqV1FDJVzZXTxkt0vHclW8H5UMEuHVekOKxWD+rCw3Hz6KSm4exGup5aGKSI8h2dpDtL+/YLnMkK2FOmjqRRzz8mMkZtdWi+9uq9j+z98stVjMlROyaU8tjnsNwVY4UwDCZYn4y+hhMtSyPmRujB5MrBZ9vqn4NgjpsZ4hZBGGM8Ro1uw1K1jgOpRA5DooeYlmC0MWrD+GMUkxumj/wDhte8Cua0fsZDoJR8+vyC6KPVucag7EbEKu7lyxPpp4WSxSCzmO2I/5K5mj4lw7g+vfwvjlcWUzLPop3i5Eb9mPPsf4IOudskWqOKeGeNs9PPFNFJq18brgj5o+Y1AEjLNVV0ljurjiC2ypTxhdID87TdA6S5vmURDwctkjmC1SGkOYI6fMXZHbIW6qWIZTdWVDVwvDc7Bp1VJ01tHrZc5kkSxq4Na9zgNFdc+bGSs7k5io5KaKUXGiEucnbLk0K9cONITQtG91T9Li0A2HdbIfC9tt1nzsIkSGZhG23xKRrGOIagyflTcwxP12WwzoLOOVE1nlUgkBGZMY7lYFR7SDlCaN7g7dTlpucyEQguvZbRY5j8lmbpRVhGkhuE0QaHlp7IDFkcdFhWjBWsIydEMlREJSBtbdZeaRp0cnu/KSd0oto00v4+W9wdVcYbPb81hxTEOurRqnt9LzZZnFqMm86Jk8T4X+l4suVq4H0kphduFswVU77aqtWUs1Y4hsPnaQQbqRLUsq/siEp7Ky3Caou/ELWN73ujOCV1g+NokB7LrcOFSgbIpmuUb8PxCIXkpnNHuhtJH62kfNVtaAuhMd9EDJHbKwySMnzaLArGF3ZCWPHRaQYw+lyjfSn4Wq8ilSMRusx7fqpWsdE7u3oUvDkPD3DS+q0JaVgaBCQQdQpMrEJKKosfK5bdBUeUXOq52CKSN+y16UPDNV5c/V2xbLZrjUqVuoWM2d/M5Y3V2Gea2Vw07rjLosSx3N1GG2KeSfSyjEqwLF7KNz3EqMSO+I6ImlriipWlFmKBu6K47qKcm6JRlwCQkaFQTt0gbIb5/MhUplOHWTiS5UOYoeZZVU5lcChdKBsojJcIDJZFSc6yfnqo9xGqiM9uqqLrplEZ1X5wPxKQBjhdaFiKW5Cme1jrEgHTqqbAGlSGXL1WQ0tLE7+rb+izqrD6cyFzmm57aK3JUOPVBnB3K3yZpknBS512SFo+SA4PLGfLdw7rYBUjFvxJZ8OHNSwTROLAzZSYdSyTT/jEtYN7hbclLC9xkI8yGzWela8WU8ODS0oyfht1Wc8zRSBkgLCtZsvdBNGJgAsRlLXGFRsxsrUTmOGYoBStZpZHygwZmhWZSISBsRUctNHbMxuqG4BUzSHBYtqlQxEKWNunmVgxj8qEgfC1LKAWp27Jre6YvsVOSj6/RO02KivrdEHKDQhmsBZSNqAX+YrOEhGyfOT6lRqS1DMnlOqq+J91VzBLMEF1lRdO+oYBdyok2T3uEExqWXUE1Td3lKB26iIugIzzfnKFsh1a7qmyIcoTkJGtaVII2FRsNlPnaXeUJYOOK2qkFx0UYefiUgksgV320Tszu3QiRt05kdsNFgFywPSEi57B5UF3jqo5pSG+YrbCKSRoKic+5QukuU8eXMo2oYg8tus58xC6ovjfFyXxtPzTxUtDGeYyljDh1su2vbTnMOQ5xPwn9EbZF0WJwU1RGSYwHDqAubbGb2HdemM4lzmKTE3S6fVGyiJGbOVHLDNEcvRRlNSyMEgDgCPdazTAPSwBYcPrCvtkeOq55OsLhl83l0Cbnd3KsZdN1GZXd1xW2jG9j8ze4svOsUg8BjjmbfiArtWTvb6VyXFjbVgqer26pEp6OuH4sDJO4Vd7FHw9N4jCYznuWaKzKNV0iUmFKRjtVGGfuq3bMfM1GGMHRdeTEwomD5oDGr7wy1rKMQsst8kpQf5Qo7FW5ac2Krua4GxVjJiYNmCWYISO6cbrZQcwTp8pSylELMUgdUspSylPMWGylqnjqe7VSHvstXDcNdV2MjfwupXPKG4k0dQwmykMjT6VdqcKoomZYWWI6rNEbo3WJ2XL1dIiljNdO1Cw6IgTdYlYSx+YqUMN7p4oypRGsS1QMjkQYVIGndFlK5lAy6Jw13ZGGogxGjtYLKVgsVG02UgfopIGdhLDyWAu7leb8X8O1VXMAYeWL3c+916WHpT5JKd7HsYbi2oUR4fW4PgVDSkvqpJJ/mudyhemVfCnDvjZH184ve5a02C47iKmwqkqxDhL3uZrcn6KNrPCbpaup8A6ZrIiNb7rdxnD6TD4PFMLnSvNgHG4K4ejrH0U3Ojke0+y3KziSmq6RjXyyOlYb+ZBRkpecx0xZb2y2VIeCjFqmmDndyVLPj9dUNyZg0ezVnTSvlfnKCZ1KJXEwizDsFucJxT0GIsndo1iKioqOambJTxSydnW0UkdW6iJjezmW97IPGLH4WkpswU0wZGSxrr+6GQwFgbHq/qttATBuqcNRKsk05SrTXODRncFV19kre6T5i5HMQbtKmZIJAQLfRVGkMjyAI4HFjXFvVYmFSSOa2+T1Kaikkjcype/y31FtlVBJKUXNs5jgS0lZpG9i9UyCITU7QXPtv0WfNjMVZCxlUR4iPZwFtFTq8QqjD4TmeS4NlTBuf+CtC1BKyOUyvcrxr3VNmvkdlZsAs6NuYbEfMK7FEWQZ3NWZgGKZte1/L1LBeyZ77YWwX62V/AontppZHtsDdZcjHyE0w9IKNDpmskIjjYZHey1zSTUNHzX0UcTx6zulh4paOjc2haDK/wBTnDUlSiB8mES5CXPlN9SqMuohf4d8hkDr9uiDC6uOFpjkh5l9ja9lM2jqxBkkPlO4UuGwshnH9HklHUNUBYpBFSPa3xUbs4v2VKSq5zh5r2FlJT4Q3F8XdRwRugZfzOcP2Y7lBiVFFhuIeDpJJKpg+PLZOIkomy1jnRQi7kq+I0T8j+wKrDxVHUi3MheVYxOaSsp2R1TBzmG+cafwWJDXfU0QezUXWq+vM2CxNqZBzB0J1Ky6GSNlOyOTVoN7IsQljmcRAyzOgUhCjcah34BEbPdSmWSBuZrrrJM8tO004YY/mFawioBnMFVJpKLDTqnBGlS4w+ikE0kfMZ8Qt0VuSroJ2l9LLHMH7tcLGMf4rFr28uOWL8oQYO5pfkP5SscFpbrI2CLn008YyG977qeGt5NG+kh5ZD/Pdu4+aox0dUxjo6h8bYyb73smpIjTeJuNBsu2KGb5opYW+o6qIObW0pjmkAq72BdsVdpKNjqY1Tpw10gIa0jcrPkjgzljHx+X8oW4lSp5nYdI6OZ2We1imcHvqyxo3F1fqsKnfTQ4z4UvsPxLakjuqjCWnxpY5rDoDawv7IIZI5Q4sf6Rsuha2llwl/3jJEJWRfgtDrm6yaWOTE2FkNg69tStfh2ggbj0M9c2B9LSC02thc/+ykwjOYzPC1nZgWhhU8cLnGZhIYL3Cv8AEX83IJ+dhckkhld5mgGwPsq9HDhuSZmY898dwCVyyQXiqeoe0zsEUEhsB1VTFsMqIKpsrGDwbjYa3A+arVlUzO2nZ6YjdWaWqfllZBV8yjki2k1t8lIahj4nCKeYxtkZJ7t2VjCskNTCHus2Rh83UFVZpGGA0bMjyJL5rf3KanhkzMZNPG1oNm3PVbmUW5I6mgnmfDNHIyduSTKdgjwqYS1YhqHXaQSFstw2COglmxGOJ7Y4fw8juq5nmPmjYLsby78sgag/NZaaDqd4oaqqZFam5mQOJ0B7LOopBy2sFiYze/QrUwbEvuec0tVIZmVB/EY/YlT4jw4ymnlxGkmEdNOLxMaM1z2HZahlgYhWPrKhwe0Bw2t1VRubMWHdm4XURYRNRYTE6OlMlXOHggRkvI9wsuSnqsHy/eNDJDNUR3cDuQumIhZG+eOjje6w5nLzHZo91erM7ZeRRMBym/NboFQkqIpG2ggMTexN9VZpJaU0ro6iYxPiaLEKS0rSxTSVLnvZ5rXKgllM8jA63Zagmoa+odHHHLEHx2ytNzp7rNq4ZKJrJ81nknTsqJZ6aRk4pmSMkdYG40AHupI6SSjkZ4tgLJG52WO4Wd4qrkDyx+sgs49wrYqX1dJRQ5wDBG9lzp1VhmYW6LHqnAKl33QYiypF3tfHdsZ/X3Wk/HG4lJT0VXSx+FGsjWeY/O9tAudqGMbK5jH52gXBta6lZKyVrp2TmO7cmXqtQlL2LYHHJTzY3h00Ap+eIQzmE231udFc4fxrGaSV08wgfS1A5FRHYXMYCpVOFTRYFHiNDVT1Eea8sPwNPcrTxbgtrMJp8co63my1TQ9tI03fqL73UmFZOIU8kj5/ARCSijdo87tCzRLKIuXnLmX6q1TyxS05pzJlJJFzsFXY1hiIMjwRsB1Sg8eeeTK1qtNlkkibSPnzMGwadbKWjfC6JsUgjgIvZx6/NRywedj6WSEFgtcdVnKAME9XRukjpTlMhvm6hWS6orYIXvkM1QzQkM/EBVZkZdIGPqY856BWo62ekcadj3lrANW7/VYsW45RIGRySPFhYFuhVCroZ6M3ijc5n9YXMvqrlHJSPaZKuI5HiwcdLKaorBDTxiiqHPaw6ucPUOyxbbHlvNTl0cjBYXI6lPNI+poTW1rzK9mjXNOkZ7OCuYgWVdZTuoqWzpDctZtdaAwOqfRVNDDAat1ROJJuTqIR2K3jkxLR+zJ+NYdjbZ2YVUSYcG/jER35d/j9/ku2HCvD2EY43EsWx5tLGZZJYqdm7s9tzrotrguXA4A3C6WuIrYI+W+KYhsjgBpfvouUxvEXYVxlWVeIVsRw81TGSxuiEh9Gmh2C60tg/mrTRYpiuHYBaehndHJBU84OkbINSQeoudfmqPGuGVXEvElNh0c8Ek9BQsjxGoDMsbSCd9e39y53ifHRjbuRDdkMU75ISPKQw7C30/itTgrHJsNpavBqWqgpqytnZLFPOLttsQSpaO04d4N4ShoZfBUsOIvZaTPIc0lrbjt/wT8TnBp8INBW00k0UoDzNHYiHUansdVzmKycScD1tTXSVdPWOxGCSninYQAALG4A7X/isRuKsq+H6bh/BqasmllJ8Q9rS4kEbH9NFcBjY7h8+A40aczxSyMtK1wdpIDrssqOUOlPM8rXkk22C2cbq5qyup8RxLLlDeTy7WkAYOoWZB4E+JfUZm/hHktB3f01TKCCaYQx0efoSutxfgaGTBqPi3BKo1uHyNz1ZjADoXjf+K46me3M4EXztLF0/CNXieHvhwikr5KSjxE8moLow4PvfZvvsPmsUSy6907cQp5G4YY+VTghoFjJGdpLe6hpXUsVJSSOIe6R0j3NvqLHQFdxxrR1te6Piahw+emoYmxUdO6aEtzMFxzBf4dh3Wdh4JwvE4K+igqPBwh8YdHaxN/70GbhuC1eOxOqqOgFVMzM/lwSG7bdx0B/wVjC6t2HYg5kj7xVLnsmhdoYzs+4/VLgjFq3hvEo62nDhFMRHNGf6wa2/Rd5Nw1hXF1OcRbRT0tZVnn8zfKdARv7LEQJvs6xgNdLAaiaZ4PLiY/aSO+hv3C6PhcWxbHn96m39653h7AP5u1VU6OPLeNjGOcbkG2pHbor/DtZ4Cid1dObl3UkErvhrmrHWudZO14KyaTFGVMpiuc1rhTuqgx1s1lqcKZiWkx115/9onDuIOxKLHcPo5KpksIgmaBcxEbadtT+i7WOe/pU8cizS24L7NqvEIMQfhfhqhtNKC8MI8jCL6+y9DJsoYYoKdpjp4I4mk3OUWuVNmupShLio3G5TkIbKgFG/VSEdCo8wByu3W8UNexVhliAVULwETZy0aILZ2VWXU+bdWA4EKvUWabpHqyzamPIS6yqcxt9RmWq9rJRlVKehy+dgXqiXOYVMzg7TZO52Ya7pntdHuFC+V19NAukOcpHEj0oHy8weduqHm/mT/huXRhHcjZyNs5TFvuhIWVtMJGOF72KWaxUGoRhrxHncNEVabKPqjAZJ9VRu5SNkIKKmlga1hLGE5VC6MtYHu2Vlpd1kHyVh1DWeHE01M8NJ0B3WLWmRaymguTayOWm0zM2UlM+OHWRmYjoiLNPz2uB5W3UqaaeoF3BgKouqqqV/kYAEMvigbOcs0tp4ap7pRzLBblHOxkQcVz9JQVlW8h1mjvdacOGRwWvJISPdYyaxPiuKwxSNERvcXKxqyuZUABx2W9kpfjjDvmpmQ0lrimj/RMcuKTDlA9482UgI/ERjcrp3UUEzwZo2kA6Dopo8OoSSXU8dj7LfiwvFyzKljdnrQo466qtyYCWHqdF0TMNooXh0VPGD8lYc/L6bD6LjlstunPy0NTC0PqIsoOl7pmRuAs3Za1a1ssVlmguY4ghSJsmE1OGlwuFdjYoKfl5gr8YZlC5Z+reKEMGa4GqmaQAmIvsmcCAubZpHglMDZDlCcDoEBZ3ApB5CYtcmRU4lchzklRFybmAKIsPla0KIyOKjMlymDwgnbNYepSMlufUqRk1SbLrugvF4CidJpmVZ1RY7qOSfyHVKFrnN/Mk6W43WZziD6kQqvdEXJJDlUJOqgNVpqUua1AZlAOhRx1RGgKrOIJ0TNOUratMTPKcyE+pVYZQdCpi5p2QInqUGayLX2TPGiyGz6qSOTVVzuE4ugtl5LVBIbu8qa7rILG6B8xupG7KJGxaEzTp6k7pGWtdQlyhc+zlLSU0hAN1LC4EAqo91wiilyG3RUXtSUDyBugEqCaQEHK5SxI57Dso93KsJsrrKaOQucFQWUpw1SAaJEWWAFiiSvZNmCBk2YIsoTZEBJ8xQJLa0Z7tFFnd3RHZVpHarCJ+Z+8mLlDnamMvutiQS2KnhfcqlnaVYgfYhQaDrWBQJB92+pC3dSxM2MHXskSAUxflaqz5blQWC9qrTSZkubYKPNdAGUJwbFHZqYtWg4cT0UnOe1uUKAGyd8miBOa6UEFZ0lI8POQK6Hu+FMTZdYyc6Vog5mh3Uha52+qks3smVtKQ2ydAo3G+qldqgeLJMiEuJ6oAdVJkTFiy0YG53WPxZT3oGSN3Y9bLBqosXpufh7/JeyDL4NnJZLATc20C6N7VxHDFRycRazPbOV27irMsoHNsbpg9qle26jEQC1EkwTy0tQN2UjmtQkELfNJgBOijliY4bIzso8zgStQxMK/hn30YkKOXeyttddSh3VWckpnupZmjMWKHlkbha+cndQvjjf8ACrzODOI6JKeSMD3UfL09K6WzRoYpJZAxouCurpQKamY3LZ3ULEwqdkEgDgtV0/NddcdjcLgAl0d8So1mHGJxka8H2VqnksRmQ1s5I8ui8/J0ZeR3ZTwxC93JmtJU8cb+y3ZS3ENAET25QgjNmpOkdsuOToICyJuVQhzkYN1hUiReEmi5ujMV0EJJRsOiWRGyMIEDZOWMlZkk2duiDGqUMFkHHY/9nNFiLZJ6Stkheemq82quFccwyf8AGjtF+e17r3ywWdi9CzEKY0r3Wafa6DxA4IZad1RnkkcN1DR8M11dC+qbaNkd9wdV6zScHUMTMk5kkb87K27hug8LJSMMjWSC26DwSSN8by0oSJD5WC57L0LHuD8IwIeLEzpXdA43C5g0VPO8mOcsPYAoOj4WfW0mFNE8YjFr3G/6LFxLFsDkneHsMrz20WpR4nJRwiGdpmb7CyzMZo6SrtNS0scGt3DNe5QeRubbV/6qWfwxjDoYDGdkdVAx0hewdVJPVvlYI3MbfpYIqiGpKUxlu4sVLR+EEp8aLst/FbiRVTjUp5+XzjyfR0V6hp8KmjLqmeds35ct0RSdokHvYNFJOIGZszHADutmHDxi2HCop2RwRwt9zdBjQ/tA52y0sHpaXEZZIH4l4Yj9mPzFZ3KfK5lNH6nmy9Jk4LwPhelosYfI7m5B6tcxPsg4OswLGBMWR4dPI0fGIzqt7h91BNhNVmw7kvjb62jVxW7Dx3RGkqKW8stRISG3bo0ey50UVfT4VPPh8MxYHfiNj2BUkc+6aqq5GsmGi3KfmYlQ/wBHpieQCH2bbVUMLwPFcZaZqeH8IGxedBf2VmKhrop6inwnnARW5tzv3WZhZRwTywFwY6zFWyTCXnMZdp2VzD2UriYJ45ZZT6Wt1uVfZBxFRztoBhU8bCbZXx6hc6Rj0Qq2TCmgiEks7vK09SrwdUwS8iZk0Mpd5mt2bddFj/B0+DCjkqpJHeIAJmjblEd16TX8J0WOcPUVLSwN5r47CotbKOpJWmnjeOOp6cCKCd3MDRoFbwTD6mFhMdXHHW1DfKwi5AW7j3AVbwhE6cz0uIRSAAlwu8fIdVp8BcO0uHQzY7XMEtQ9p5MAHnb20V4jleE3YRQ45M7H3OZE+4z9L+4S4hnlxmU0mFUkUNDTnmwkDIZPr3Qzyk4xLI+FrRzc5jdqQrGO8Q0M0sTp8NbE1vkzRuF1LZYmKcyOF03Ku4kAZjeyq4ThlRikhkmnjihZ6i7/AJ9ldxiNjaISRyl7ZBcXWRRzvpXEs6iyzl5rMtytw7A6enaykqHOqSbAgrMnpKqncRJGgDpZA2rhYbRm+YBaEVa+qoJOeQMgu2wtqs0eTJHLkP4nRRUDX1GKR5G+VhvdWaqOnyc6AWz7j3V/hXBmYlHUzR1EnOjZoz/iusKpYtNDU1IhjZZ9rE33QUtAIHZp3hp7EqeuhEUbGPhi5j7+dmhOqgiovK6SvZM2EbEv9RVmBaqXxyNYadlt73ddRRyc9k0IHmy790EwojTsNIJI8gs7W11JSUU9JSNqp2ZWSmzT0spTKtT19QaeKBx8kTr2UctCyGfnsd+189gNArTIGeIbJa0J9TugVlop5YjGTd49J6WWOa02qfHSzlYZTQWZHFoJtCVmY1VTYjh0NK1kcMEbs7xfdCYal72YpWzxhvobruFozUE1G6OeF8FR3Zfusqy8Ph5mRmE0ssjI/wBtLbYe60aoQ0FFEc3NkqpLAOGpHdW5uJBhHD+K0M0MMNbXy6Ma3SxCk4G4Ur60R45VCE0+W0ZJ84PcLrEMsmSOGngL5Kfzs2DtCFUwmRkmMM5htnjNvmppcOfVYvXHxcvkkyNEu7x7rMrovBVzoRI1xj2e3/BTiHqWfiFzvXmyuQQzMia6kqwcl7tc3UhdBh+BB1E+St1EmpuNb9ysmsoZaJofI6JwP5dVmBnRxw+I1BDPzKV7pDFy2aDuj8BW5GVLg0skF9DsiAeyF+Rl7ixPYrci62XDqkWqZNGN2tuqYjjkZJIwZWMNmqEttA1nVxsVYe6BjTTQOu0eyxQeWrErQHMZmtYk6qzBiNdDE2CCY5I/OAdx7BUoaSeqqBBTRhzzsC6ysy0r6WdsdSzyssXBKG9V4xU0FQJKGtq6R9TFmcZLSZj7Lna+fEa6V8+Ic19VJ33/AEV+un8VjVO9kF4g0Zdf1V7EIKbGC+qZUyUb4zYvuMgKRJ4aKv4Gf/NyHGMGxGSqaWg1FPIAJGv62XOzPhjLo4RI4uaGHrrfVdhh2DyCJufGxPz9YbG5eOqyXYZA2ITQzSZy7JlB1Kc2ohz0sU9I5khu1zhcELSwOZ8zamOdrZrlhDZxpupp6kUtVFJLT82OIWynV91LicEWJufX0s73SZdYmtNzbv2VjNeKricUFHUDl0hEcw0ja69nnqFmTN5Ya+WEExm+Rymw+F81RkYRDIBcAm6mq4Z2xy1NQGkMNsjhe/zW4klXlxFk4lAoI4TJbl33aPYrVwrB6yrbFW4VSgSQDVzz5JD+RY1XNDVRiSOQ5h/VONy1HhtViMRbS0lVJFGJBK430bbW63DDqcWlxannkw2aSja+WG7oojo0Hp81Uo5cDxGidQR0D6OoijvzxKDIX9gVWx2rwpovRYj4ipkdebTQ+4PRZUkTDaQPvzBcjst8mOIzg9VFq8QBvW0tz/cijwyrLwwPiaOxdqfl3UDYpnRl0bLtZueyu4rP4mWnmhY4mNliR0XFtFXsgY9rIwIyPUCdynpr6U8QaCNBc2zeypk55C7qd07JXx1IkIzADT5qzC+q8JmUVeyQM/EBsRfZBJI2krm8917m8n12VWNvNeZDudytDw1HUAMkIzvIBe46lYmEXponTOyTR5GP/ZWOllWefDUbqF7bSh2do7hSkP5Tp89+UAxpt26/xVKKlnc6zIyZH7EixIWZh0tpTOwvwTK2n5vN15rWnRh7hLA5sdo6p9fS1UlKyS0jg/UOj16dbf4qCKlkY19JNZr+rSUzYKiFzqiAySRZeWWzDUd9EhnJ1PBWOMpqfEYMYma7DZbvbVBlniUnS3z/AMF2+I8BS4/gdFRYhV01PWtcZ31RbeSRljYn6FeLCXLSyUkcjy2RwflOwsuqpPtM4gj4cPDHIEzjEYueSS/IdxZdeSUqcL8KVvEmKz0/iuVSUzsj6gR3Dhrt87Lva3BeBqLDjg0rKWGaVgymoJMmnUE31+S86w/GKykoJMIp3yx8+QGzTaRp6g+y1RQP4oqHTyVU0VZTQ3dLUH8Mgbe+xWbSlWtwXFZ60UsJfPRMcWCY+VguPddNScJz8O01NjIxes/ChNRJkgDhG8d+4WdVz4k6nbg+I1ZlFNapgcQDFIALjU7rSwTjccR4ZiGB4lNJSuq4eXC5+jM5vnYH9OlgtYyU4jiCpjrZ5a6qZzZq6XmxTEWuAALAdFkl/JijeQCx5tcFdLxrhuK0tTRUuJPcWU9OBG9sDxHn7A9fcrGpyxsDfIx7ul05ELeDRYJWUslNiD4w+C/IedLjXU++gUWHVQwfFIpMSp6gRZSQ1pFxfYhS0ldBl5dVRxEgACQN1+o2K16CmbVluLv5DqeNvKLXamw9v8FjkPSeB56Xjfh7+YePSmV1HH4jKSWnJf8ADGm9rg/QLzXEMBxmHF8ZwqgbJJTUYfJzJWFodFH2JHutng3iT+aXGT8Ux5sjmV8BpoalguI9rHJ12svZKPJxbw3y8ZgMcxdJTT5PLmsbG3zFv1TkPnWlqpjTPbppb1ixPy9l7vwbhzJeHsOxGCl5HiKVmeK256n67q7i/B+D4hS0mFUuGxRMbVRyF4GrWDcX99F0UNLT0dOympY+XFELNbfYJElMHEMElq2ula2zg0k+64nCA6WhYwMJczew21Xq41zD9w/3Lzzgyhmq6CeqhmDHRzFmo3BXp1bGcsVamqGU9Y0u6BWKqZkkhlY7QrcqsFZUtyvY0OHpcBsVzVZS1FFLyqhmU9PcLvjNuX8K3BibWNyEHRWYcU5jw0LCzeyljla1dJ1Qxyl1sNTHJaxVgabLmqGra1+YvW9BVxvAu5eTZqmJdcclgx3CAttuFIyWI7OTusdlyp1QkdCqVeXMiBaNb7q+7ZVquPnRFi1CMpk2c+pWYpABZyryUhEzZNRkFjpv7pSFrIjI12y71yYloicDZUaupdmJJVTxpaM2ZQPndLfMt4avizOSyKvrdSsrQTlWWbpMkI9IXXjDHNqyxNl2VCopHtN2hHFXWGVzrqwypbIL6H5ozbJcCx1nIFpzQQyjNk17qo+mIOi3EsoGufspDslELSDPoEcrTJL5LWsr6ANCtCiY6RojYbFOKGjGz3k+6mp6eSGTMx1gNguects6sgEEpDNB2UUfMkdlYLkrXq6AVTmvD8jhvpe6s4NhETajPMM/ZZ5xVyvAOBYIA5tXURkv6tJuF1Rp2VMVn7lSMghAvZEyw29K8mW2Zm5d6c3i3DdVDGZ6WQO1AyAdz3WZVYNiNHKWTQggfE03C764LMqQax3lIurjupni4GFsMT/xFYPhXaqxxPgtTST+Op4fwJLZspuWn3WB4iQepy74zzcpxppvYbZqecMPYi6F9fWRMGZgI7rM8TLfR5Cc1Ezhlc8kLpRzXhVPlfa4F1cpJJi7l2vbqsUOc05lt4LUtfPleCPdc8oXCbXwHgahDzgw+d4HzK0uUx/RUazCKSeYyFmvXsVxl2xJlfTsGXnAn2TurondVXOB0kbDJDzInfunRVIaCtNSIXi7Opus2rWa5s2gUNTQnLnYb9xZXKSk5V3Abq4yja8edunzWORLnY3EOGVXopHFlsy1HYXS+rkgFV6jDahln00eZva6s5Wpqd9mqR2oVSISxnK8ZSpM5tqsKkDAUnNA2RMFlK2Lu1BHG6N7Mh9SZ9I6xKN0QY7OEAqHZyHHRBSeCCQgdutaOmglOZ+6z61sbXERoK5kCDmIHbocoQIy6pjKo5M+ZLzW9KiDdMe6jfKUBv1QHNdbBZrprOKYGykzgLACxCLMUxfdCgkz33UjEELW31V2OFmXNZbFdpylWYzdqjMWV2ykYBZYEqcjMEwNlILdEVFyfb+KcR26KVSxsZfzBWxVIPZArcsYvoozF+6oK+UoblpU5aQo3j4lVIm6hmBU0bbi/RNLHZmZRlTLiEua5NKW3UWZvZbFnn26pjO0g6quTZRk2KCcOu5WoHWVBhupI5bFBpc6ybnX+JUzLp6kwlWBd5jnBIOsNVVE2iYz9yguCX95PzVQNQEQmQXCbplC2YelxRiS6BydFBJqVM46KB4PqWxGIkzoiDoEQzgqVr/zLAgAPRTNNinc0boFoTxyuuLnRTiVpHqVHNYqQPWRYll0sq5k90D3FR5ggmzqWPZVm7KeN1kEhGijJ6lGXaKJ26BidUJOqTzZATdbDtdrZSZ9FAN1I06IH1KF2hUjXKN48yRKUiOyFxKcplbZN5lJyCR5tEmhuYFS5mq2AjhAN1bbAx8ToyPUCq9+ykje+9syDzeSJ1Fi5sLBktgF3sf4kLJPzi64zimF9Ji5J66hdTgVS6qw6N7ugstSzKykS3snI1Q5SstESOqa7UybKrEgJBuqsjrFWnbfRQuDXG+Vd9bnJMepA5VwQDlKka89irMInzIHAnZPGHPU7I7dNVz9GlN8ZHwlRE65W7reggzWa/Yq0zDaX8ieKzxY2H0LpDzHtWh4fIrMkbIdGCwUepKxOc5N0ONrcqKSkdM0tCdgsArMTiAuVt0ox0Zi6KQNAVxzbm6B0Y6JZxVyLBRuDlZdHoq8gI0RpDzLHzI2TtuoJWGxcoBI4O3VYa7JAphI0rKbU6eqyPxJOzkGqQxCRdU46o/EVN4j3UbTebspA8qsJf3kYk91GE1yhIuUJk90swQC+wCrTOc5pa1Ty6gqubg+VSR5vxfWzGeSk8ETf4jquSpcTmw55EbgHjuLr2bEcEw/E2FtVGb/AJgbELyvijBKakxAw0Umg3zI2xa3FMQqnh8lQ3y7WjVSSaplfnfJmceqnbOylcWOYD8woppY5X57gfIIOPblO+ytVToI42yRR5haxVNnpulLIbZUaQyG7iVE9Txxtk1feyJ0GZpyMsO6uKe41Jhs9Y0mBt8m6v4MGU0r5Jza3kAtus5sssTRBHKY+Y6xI7LusTpKLD+HoGwemRurju4rQ52so6WWriEszXNn+HqPmuniw77qgiwoxgU1QbkncLioopnSeOAMjIyCbDounr+MMPxKniZBDLHUx+kuGgKIt0nB9DScXUMc9U3wZdnHM0AI91t/afiUFaKfCqCphkdHq+zwcosuFxLFq3FsgxLEh5bWa0aBZ08FM6UuhmkLe90F7A8JklxNrQ8SMj9RHW67rAcapcNbiOBxskqKird+FG0aO+a88hlko2uNJPJG54yE3volQ19fhtR4ukqi2axGY6rCutqMB4g4WpDO+Vpo6gkyMiNjGT0CiZhOJ4FhM+JT03LZOLEPlu6x+izpOKMVxaBlJiNQZ5OYCHk2/hsuv44fMcLpIy/yvi1H6Ii19jeB4dU08mN1cDZXxyWjcenuu7xqiosYrG8x7m8oWjezQm+9/wBFz32YSwUvC/Lmp3MzyEtIGhHddPzaE65iPmFZgiVqegpavDjQTmKZhABLt1BQc7DaZtGGBzI9GWOwVSarpImmR0uYD8utlEzHaUf6w/7ClJaaqhZiGKsqKhr3Mh1bYXF02JYRBiT+bRQmCo6PAsPqpaOrjla6cTxljze99lzvGfFtTS0TosPaY2nQyfG4e3ZdViXE8Y1VM/EoqGLDXMq2XEsjTo4jsuGxSOoo5+XUwuY8nY9u66jDa/xVRLV4rVxwPi1aX6k+6xsRnZilWap0wYY3aE9VmcWGVHNJK4MkcS09Lqd0bGtzMV6loqvGqhzoWDlg2zgf4KpJFNBUvpZoy0sNtVznFsIDmRn8f17tvoUckzOT+GfXpl6phTvfKGjLbrdDygag5HeVh/VYoDSS8iqike27GOu8X6LscDmnlx1sdBHGG1AsBewkC44wS1FU2niF3PNl2eA8ilx/CSJo8sclib6bLrCp8d4KkqcW5WE02aWCz53A6Ansue4hgq6TFfu6oIETLPfZm69AoYfvjGMbfQ1ckM9JJnjIdYE67rkeIgcTLampMsVe88vIBo4d/YLCObkp2VMop6d4DT8RK3sU5vgYcNj1YyME/RZmPYE/h6CASYnzZpL5o2bN/wAVBg1by31TKiZxaIxlv9brMov0kbKuKLCYWERPcA8nqFSrqQUFZNSxvc4XuLrpeF6aOuwusnikETqK7/Ns75rna+oZLVyT8lschcbgbBYlqJaWI0V+HKWCOrjL9Lx9XFTYK3DqqgIxCE+IphZoO6g4bw+rxSuzwRx5Yr5pnnRqGogrKeaatcQTzcgIO6gzcYdNj2JllJTvDw30u3XS8MYxxPhnDU3hMDmmhF/xbgBo+XVGxsFDhQqYKWLxkhIzltzr7rWq8Bra3h2HB6HFYIWRtMhs7zuJ3C645pLhqKmkrJYKeWrjPi5eY5nUd1Pi+BcnEp8Q8PI6hjNszDpdQ+IoqKKN8IcKyI+aW1rG+wUro+IcUZSwwEyxVBIiYTqDot2M6oxeprZsj3PEI6A7o4nVVZPIwNklIZoN7BR1GC11FiH3dMGiW4AsbgkrawjD6qixAcxwjcQbk/EOy5y0z4aynOHzMzZXwaNba+igNRysNEDxrPLzD8uisYzSR0lTHVQD8GoJePmsuaplqqiMTHyZtUhlNHE9zxzPK3urmHwQtxV4kjErI2l9iq9VM2GlhY/cyG3yTw1T2zyTs2dEQoLNJNHLI98cBhBN+ZfUfJHVsdMCWEuc/ud1kRVjyOWzS606Q1BlifHG4sgN5XNFwEFvB4mSzVj3jSCKzR7qaCKjqqSsgh25ed/9sKCnnZGyofTEWDy91t1nQYjICZ4eZEXkHN8lODXJcmaaSIQwOyyRNuOtrqvhtS8h8Uhvu8H3RGopnSS1cx87xb5KV1Jy2NnjmMhcLuv0CUqFscZaX4pK2SMtN7dD0VGnrZ6Rr2UknJbJa4Op0VqE+LimZJqNmn3VN2HVGY3blsRaW2h9lYEMjnwETM9QVnEKKbMyXu0H9VLOyGpcCTyX282bYnureIUPIwRlbHXvkfHpa9wR2t0Wolhhuo5mNMmXRT0VD4vnBlXynRtOYW3UtRBWT0ra2WeONtrNhB1t3ShY92HTMZl0duTY/qukSKpY7l5XuDjEOgtojjYxzWQiTlvPxOOh9lGXCIlhFyE8ZY92eYXCWNM4XJSvjikqI5M7Q+URuvyx7q1NgdaeRBSMkimOjz/VyC291Tp44HuyUlKC4i5ANrjsiGMVdso5mQelnNNmqCkKOqzuYY7OZJyyPdWnYNXUuaSSIfh9L+r5KDxFWMnhInjlm50vorzsTqMSqC6ac5TrlGgBUAQU95HwZI4omWsO91ajhAbyBHFGY3XFx6h3WdUPnqPQS573ZNTutvDcBk8YcOq/U3IS8m4aDtdQBU1lmTQxwwx3NyT1B6KGnxHw/OEjWyNNg2UahoCucYYk+Oqn4digEdNTAMjLBrfutGXB6DEODXYg6kMM0ETHwuB1cPznv8t04lM2LDqIVoqpJ2efzkvNiAdrK7WYLJVUk1fDVROZA3QN1LrLDfLHWMk5UcsnIYA1x7Kejw+eOLlvq5ojKDI0NPkcFzaY7Y3RvzSWaexKkilgibUTPikLyzJDI11hH7nupcRh8bXU8GHs5sr2+fXqppsBxCnppJ5DCWRC7iyS4AXWRWYKyCVtRIJGPAvHKNST7q3VY/NUU8VHPOQwOzktGskl72JCzmxg0r6tj3OEbg1wO+q0azBp8Ojpqirj/Cq2iSM7EfMKjSqcQhxV5+++ZRmNgjZGw6WCyqeos9kFdPUfdfM1MFtxsdd9z+qmjhrcZpK3F2fiNo2B85G4ubBWHYdPQcK0VfVOinZWl/JhvYxC/r99v4LLLadxjh1RgdVgbHVYppRyQyQmTyf6wEnQ+yg4N4PpeJMHxSqmrZIpsPbngEepNgTqPoudwODx9czDY6cyzVNo4iL+U9TYb/8ABehzfZ/j/BzRiOAVslUJR4arY0XPKfoTb6rQ5LhvBafE8ArOJ698r6WglEboIxZ7gQNflqoMNHJ5tVTlwY+XlxNkOkgXsdFwVh2GYw92F0zGYfidHGJWZbAPitY27v6rFqOFcGw3CCzELSRwVc9Q2nfIGuIJtkFtT8liYPRyVXgOKmloKsS088FWTI08wDknbW5XacBfaHHh7qrDsfrInu534dS3PZ2m1gDc/osPjiijqoqaDDsLo6TwzWTytp2fisB01C5ekmlwxzTTVAqoxJzHAsAy22WGnuUn2kYKcUhwigMdbNUkCIxTNyDvzCfR/FdPHVc1mfmRu/sOuP1Xz9TVEHGeJ/dtdys5iMkNVT04ieDucw7bL1bhXEGwUEOD8m7KSPltnhF4nW/uK3iy6zxTGse8nZpXI/Z5JbCKwf8Af/4LYqpb0lU6/wDUSf8A7hWDwG/Lg0zvzyA/wXWJpZ83WktsuZ4rhZaCdmlvJbut5r8yapgZNA6N4BB6EK4bKc8sXAN2ukd10JwelbGYWMsBsseuo30ko1u2QXBsvoa9nJ58sQROtsVKyd8RGVyqBzm7BPzHFdOLPJtUFfmlHMK246qEj1rjo5XsNwVap6x4ddz9l58tTthm6V87QM10Ila5qyYq8yODHagqWOqs8sceq4zhMOnK16S1rrElPJhniuSH669FqyF74vIdSsavLorskOrxot6/VM/RTMuiEThuhctrhzhyPHYZJ6qeSJjHZAGbn6rqaDhPAqL/AKpzid3TOzFdst2OE05xi4G99LIQPZdTxBwi+F/isJgc6M3zRgCzLBc009lrDZEufCUWYKQOU5pHyta+OIm42OllWcHRvLHbhaKW2SsLA0FIyKox9lHhlU6spXzSNsY5DH87KFLT4gR5d1GAWu1RktAznZut0TXMkZl5jdr7qpS2Z4ZYGMewB7NnjdJ8swdnL7t+So5gpWSlzCx50WaW2lCWzRmRknm2sEzaqso7eXMB1KzGzGN1wpTVySNyhp/VY4Nc3aUmJwVMIkYfYjsVO2TOuLoMRlozrqF01HVxzwiRj915tuDvhsaIkcTq5TMeB8SotfqpmzWC89N20XFs0ZY/Yri+IOHOVLC3DKdzi8kOA+lv8V1UdRomc4ON1rXsnCTLF5rNFLTyugniLXs3BQhwvsvQK7CqGunFVVU4leBbXayrHBsOhB5VFGPkF6vGcfCcjTROfKNF1uFUNKKZt2aqKjw1rZxnC1mwsjFmCy5ZbbXXroTI4m6AJ8gffKEwaQdkmrhbtSCWFzvhUkFJGPPk1UrrJB42QoQZZStUAkUgkQTaEISLJBwsnJBQQSxRu9QVeSFgOimlJzaqpLIQ5BKLDZSMkFlSzuTc491LRbdICfM5RSOjtoNe6rulceqHO49UtUvPkGxVWUl5u5TISxiqqxaVGdCrT2j0qq9tjcq8kNZpN0DjomdIAMqidIohP3QlqbO1LMUDEWSSJuhLtEAXsUTDqgSa4ILcTgLKw2oyi11niWxRCZBdfPfqkyax3VPm36ohJ+8ir7Zrj1KdknZZzZfdWGyCwQXGkKUOVNsoClZJ8WZBabullKhE+icS3KiFJH7KvJE+/pVlzrp0tbVgxzQpw1skdnJnDMEJlMYytWuRbOrIcsnlCqO0V+oeSS4qjK65WkBnQlwtfMoy5B5uyglzBLOO6hJchzFBZ5txa6cSqrd35kVyEFkz6ITL7qvdydrzdUT53FEJHBQByIPcgtMlU0Uuu6o5iibKQoNDPdIyfCqrZrjdGH+6yLGYJnOso847oJJRZbBGoA3chFRfqqbpLkoebb0lBeM3VE2dp9SzOa7unD391BqF4KBzgqPPf+ZP4h3ZKGhn1UjJVm8890bZ79VRo81Rvc4hVxLrujEmqgcucmTkhLMUBJA2ST5SqogblOW3BUZFkYLuqIhexDlspXBNy7lGDAGyLX3RsaB8KTgLeVG0YfZPnvshO6YbqsOa43ge58NQ30sGQ/wU/BVRngdTzP8ANa/96ucUw8/Cn5RcsIK53hWr5Nc1l7cwELcNOyebIFK9tjYoA1ZZCnylSJEaIIXMNlEWZgruS6YwgreOTMwqRUoc7UKbw8Y+FSCMN2T5f3lbKAI+yka1CjYLFSylyIgAFTiYd1R5g7KQPWGhzSFxQguCfT3TEtWQQkdeynic4KFu6MPQXA/RIuuqnNsd0QmHVRtZAv6kMzY2s1QCdo+JQ1FQCwtugpzysFwHKk4kqWQOTMiL1WENyiuRsrYpfzIvCR22VsVhO/8AMpBM8dU5pAEJjcFBZjmJ6qZslxuqbBZTs9Sli5GOqNRw7fRSt3WV9Au2UJBvsrBamRGbi0WKupT92cvP1zLyfiGGojF66rzTdQ3ovXsR5xpyyMkX7FcZj+C4XDgz6uubebu92qNvLJm8x+YKQUNhepORvdbHDlB4yV8sdIx3L3UtQ1lXXNp6qBoiBu+3ZB5nHHqGIZ4msdbOD8iu6r8CgxrC6TwM8DHxtXL1dBTYXzKatAdOPS5p0Pdb4sMUSBvRGJQRsrEVPzKYyOw6WVvV7VpU9FFW0XJwihMrhu9w2+acW2JDS1FXM18EJcyMh7ndAF1PF2LwmnpcLhNomR51q4Lhr6Thypw5mV85bewK4Gsp6plQ4SRyaeycWE3jZ44TTxzuEbvhCgaS1RtH5lIjdnsHEKchkWjnWT4bSvqq+GBhFy7qtPGMHxB2NeAFKWi9gQ1GbZjRM6VvIhdLfYALWkwKsFB40RODusbtCF2p4OpaDCoJaCaRrmAPcSBdx+a0HcKYXPQirndVyyyAFxz3B+icTxHnGG4TJLAMQdPG10TtYzuu44nrafEsMo2UzHSWjBc4bN+q06fg/CqOlrSyibIJIyQS6+qv8L4Tg1Xg4ZUUTJHXs4WU4M3bc4bkglwOkZTlpAZsDe3/ADZaPIk/Kufjwjh6iqeSyodC47MdNlU2KR4Vg1FLiFbj9ZTxRi5/EuT8tEluGzy9dh9RdVauESHl8sH5Cy4PhbjHH+I8blw+ixcspY9nyxAm3S+nzXctbxFFdjqXD6j97Plv9FDKFet4doKyMQ2fE8/1jXWsvM8co8YxvHXYTBVk01P5I5MvT3XpNZiuOUTXSVfDsBhjF5HRz6gLzlnEmJf0uTAMCn5MrzaU+bln3stRLEMqloYqTFPut5FXWMJEj5z5GjuFS4xw+lpp2+GqY5HyepjdmLEqYz4x75pZC/Mcwv1WlhseCMkc+rZKRl0IdstTKngx+rw+hZSUkbYns/rRuqjKmWaR0s55r3m5c/Up8SbFFOWRqpC+0gCxLUQuOfJnHL2Tuky9bEhHHGHMe8vyBiqxNM8zr622+S4K6DhukpHu8ZKZOYHZA3NoVb4hooaOrfJT09rgPcGjcrHAZTxMzgxg31K6GHGp4aVtLUMaW2td2pcFG137N6sU2Lyx1DCIa9uRpdoCR0V/7Q+IqSnqGUuG0RnrKcesaiIaae6zMMpgTHDBVmngkkvHITq3vZbWN8KQR4bVx0E5c6SEyTTPN+YRrotRLnMPJal8lZI6eWRz3vNySr+B4LPW0NXVwyhhp+h6q1gfC763Dq2qrebG6DyxAaXK3uD8MnosKr6TFY2wsfe7y7UaWXY5Ivs7x6gw2lr6nFXx5ZCPLu4n5LLx7EXY7icmIsw/lMeAI8o3A791miohZUvpcgbTA2JG7h810EtXBIyloYKiOmiFvPl1H1XLIiGNEKmFjmslnhz75Xlt1Lh0LBVxQv5sjTfYrpKum8RJHDVVsclMwWD2n9oq3DGGc/ihlPI8xMDXlrx8OmhusraItr6zFYcHjnEfM/ZZtre67XhmnlkqKrA8XZG2WOM2Me/zBXH4RQPp+MKltU+SQUDXvLyL2IGhK2onxjD4sbfUOZUz1nKleBuCtxgM/E+GKKvxCrnqpOXQ0cfLiezQTSKbg/EKOGhhpxS+IqKaU2/Ey8sHr7re4soIZsTwjh2kjzU4PNc2+p+awMS4WOG4lJQiYwRczPSyBuhHYnoV1ocpi0VZ9/172MLrSvks0aALVocWjpgw1EAkY8XD7ahHxnE3CsamZQnlMqIGGTrc9VnV2MU1bSU1JHAYoqcWuT6iuWWKxK7j9HZslVE8SUpbzDGW6NPcLkYDzKhl/S03K1WYzVGCooXy7aRnpYf+6zOTI1wAb6lmETYoHzTMe14LNmgdEdQMuVt7aWUMbeVIBLoo2hrmOdI85xv1WwALIpLNN1ZkxCR8TqePysfa7W6XTs8HRULK+Ou5lXewZbRvzCqEvMhmebkkm6UtHmGhkYS3vYrXpCw8Lu57v6+7fldZJlvGYSNDueqs857aBtOdWaPslFGfkEWQPzHqVrQlgwWOCef8aoccotrb5rGiF8nubI/MJY5M/wCyN2jssI62hZg+LcuB8ccFSwctwabZrLOmrDEIsOmfZgOYk736BZV5pKkSQ6EuvfshrnZqrmbddys8GoloVgpTG8ywmRu5IGyxeWXWa2+S990pZH3eIy6z90EbmBuV179NVvDEmU3LqHeYwmUDp0ASe7TyRi3ZyjdUzRRPpmPtzNyOyAzPdbProukQhOsXXk39imle0EBm1kUTmsu9rBmvdJ+V1yR9FG6W4JJ4aU18Bty3ZHexU1M2MU8tRO5kTLi1zqSeipSycqldSxylwn87r91NOLRMYTckXJRKTzc5jmiCR7WPabWVKjjmbLzcl2Dqr1FPDG0Pu4uALBfUAJqiwiMgIPyFllgeDPfT4hDOIeaxji8stckLt+Ha/AzSYjSY7iUmG1FXIwx1DmaujA2+QK5+o4afgGCU2P11fIJXyD8CO1wCL/w6rm5MSqnPMkbyWnq85lpXQYxRyCn5VBURVEMBMhqAbGQlUosUxaLBjh0lQ7w+W7WHYfJQ1EzKtjW80QgNGcjTMVE2KtbqyGeSIEEvy3buiN3hmkjfiopPFxubJA8lrQTf2+aJ1RS0c8r6ihqJKNnkhyTWMZ99F2OFYlitFDTV0OByzwywmQgQxgxm35ug0Xl2J18lfX1NfM7IaiV78o2CzQNtVE6rqKtk8sMg/Eic7U391LLK1tK7PPm5o82hFzdUaeSNrzNJm01DgL2K6SLijB6vDoqbFMxeGgS82G7CQd9LKrbBM8baRk1Owg65r+ly3+IK/iGjwahwTHTFO2VrKmN7gOY0dGX/AOd1lYRDDiT56R87KRlQHyRNA0+X1VaZlbWNimkMkjY28vO51wAOg9gojuvs5raCowvFOEmNjhrcZpXsNS5xsSAcgy29z1WNxFhmMYe7CuG8QqIpIIorRyRm4Ivc6aEWuL3VHCKpuHZjyY5HEeUkatPcLaLn4pU4fjFdiogmEkkmaQAtL9Ba3QaBB0P2WcM4dOIcfMkIraeqLYQJA0ujAINxe5NyvUXkj3XlXCVfhHDFXV11fSwfelTL/QnNdmY1hOpt0XrFBOcTw+nrpKYwSSi7mEWsV1FWapliIhZJFHLKCIw8+o/LquH4q59JikD5nS4jj1S21I1sfLji/fB/iu2lwdruKvvWpGZkVCI4b6hkl9bfqqfE9BDFhL8cmky1WFxvkgkJ+N9hqOqDzYcNvqKKoNdjBh4gDyZmzO8hZ0HvfSx91zE+G0viYaeOeeM8gGT8Mm0ovdbPEOJHinF21FJJHSvDY6SWOR2p1tcHtqrlfSY5w3TMwutNDHMxpkimEoleQToB2OnZc8kyZbG4TycOc6arZXGO7nsaSCO3/sun4O4zxnB8XNHjEJdhtXOLObFlDZH6M6aDRcniOIVxhpocVqnwl7C+E2aS0HfXc/qrGEVdDXzuwerxKTkzuiMc73anlnb+K5ky7/FeNvu+aswepp5xWPLzGZDlhDCNgeth/etngx3KwNzf3h/+4Fx/ELYcbfTjEqWWN2FNLHx31cw2tID12H6rsOEIY4cBAhZlbm2uu2Ktts5urLZQRqVQa5isCF7hcbK0AmIubKpNE2VuR40VySF4F1C9lvhK6YTxZnG3OVlNyZSc2h2UELObK2Eep5yhdW/Coa2P0691JQcP0lLZ+Ql/clemN7jOtzLsMq+byoYnSO7tGimdw/i0cQm8OD+6H6/ou5hpo9NFY8LGR5QseMvhPO2wVVK4PqIHRj3Rmqie7M0rrsWwV9azJzGx+51XN4jw3PRRcyB/O7gbrWO3HJZxpYw+cSjJfZXpMHoqtzJalmfl3sDtr3XP0HiKRzaqpp3ti72XUNqmFjXDqLrlsbxW8MbFRMyBrRc3NmgXP/NlpxVLZHZQuf5uuZX6Oa1nrh6t03GSADK8+VczxDhNEyWmlomRw3JDg1m91seJY47qliOSWMXF7G4WsZKZ0NHGDeyn8HTkhxgBPeydru6Lmd3K85KZuNNjo44fC0gLpHeZ1tGsGpXkM+M1ZZVTwThrH1D4wI9BYndes1XFeH0+JDDH/iki+guF5DNyMVgmpaWljhfHJJURhmhNn+g/IHT5J4slOsxHE/B4K4Ofep5I9gCRpquJhxXlNzvlmMsf5DpZa1VTYjU0pknPnkbaRvszYW7rDqJqeml5dOwR2ADrdSuviy4+G7Wm4jw00seV0hsLHS2q1aKdlZTtng1a8XXnE9NUylkdK65MAkDBu4k2XQ8LYpNgzJsNrYOU7KXlxfoF1w22xODq3se0Xsha8gqvR4mytsGaC1yCrOUhdoZk4N1doq51LI0F3lvqqLD5kbH5Hg5fSVMsISJdbTVzJWiQ6eyuRyh4uFysWJRskDI4PLfcnVbUUoaLly8WzVMS74ZtYFSDMqEVbDezX3Ktxy5m3XGYdvVYbqmcyyBslik6S4UEZfZ6lErSqkhOa6TJPdFXMwTZgFBzTb1KCWqLSlC26ZqiMzbrMmxBzXalD95AesOA7rfGWeTWEzUTZx3Wa2pa8BwKMSFqxONLbUEw7o2zM7rKFSO6NlSO6lK0pHXCz6h4a4o/E36qtUOzXUDGVMTdQZiia+y0ynfoFDntsk6TMoJC6+6gs81DzVVLyUJk/eVVZdPqoJJrqF8nuoZHIHke691EZfdA5xUZeiJc+qfOO6r53Ig9BNmKdRg2Rt2QOdkBcpH2UTt0DZ/dISKNzrp0EnMRCRQhydu6CyyT3U8cqpsUjS5BfEvurMTwQswPKnilII1VVe2NyiDhdVxUX3Th+Y6KC0Ho2uCiYCRojyG2qiJxYtUU7AW72SaSFDPLvqqqjVaBUJNz81bnfmKrEXKIHIlym3ujTdfotgeWAEJa2+gUpGiDT2QNlKAsUoc0KOtkZDQzT5w1zGnKT36IADLpiLLkoMF4qqrTzTmHmb3IBt00XWxRcmFsecuyC13blAOYIwbqPr9UQOigNPmKizBOZGt8yCUPITia3qKrmf8AdUTpbuKotOqvyuUbpyeqql7kxcSgm5nunzjsq5cmzuVhhZ5gTcxQZ/dLP7qibOfzJs/uos/uln91RMJHKRkmvqVbMUQcs0LrZNd1PG4j1OVCN+u6sxzWso2tlxKLMVG03CJrgFBK3dTMjKrslF1YbK2yBOiTCNwKkLxdR52oDylLT3UJkUZl13WRZ65UiGqFknupgdEAub5tEGX5KZ2WyiJK1CWjqoWVFHNC7Ului86w93hsSjP+rksvSQ7cdxZeeYvB4fE5hltrcLWMjv8ANzY2y3vnF0CrYXPzsOiN/QLKznSWUrdk7cqjDndkTTYqA8oTIgeoSsOysMyFC7dO4FAQd10BZSnvYIHSOCAyOWuLNpte6bMVHzD2SzjsrwktMJndSi547qvzWdShMjVngvJdZJmKsN8yz4nOFlchdeyxOLUSmLHIALKcFuVRuDSsOiIvCrvfrqrD4xfZRmIIwguEbZGhIwNvsiFKXbIJBKE/Num8NkHncB/aNk5axgJdKwAbnMFmw+a6EjoVRdxFw9HLyH4tEHnor8D2VbeZTSNkZ0IKWAylHGCoZq2hpX5KqrhiPZzrFAcewKIefFKYf+YFbF9hsp2uWC3jHhoy8n7yYPcjRa1PV0lYzmUtVFI32KirGdqReAoC7KdULph3UE7iOqpV+G0OIxcirgzs7XspebdLmI0o0nDmFUTSyipcrTuM65mHBGSY+f6AGs+K4Xac2yjlqJKeJ80Dc72C4F90HzjhLaen/FZHLPpawkIsurw/hHBsSpY6qenJ5hsWuNz9ViYfwfjsTnSR52WF2lovm/iu04HwzEmuNXXyZ2DyZbLvDlC1hnBVJA7kVLHNpB/VXuF0UGF4ZHTmio6GKGNwt+G0XCtAc5wZ8PVKSrggtDRwGeTsDZOSxLyeSln4R4kNJWkmKcmxOxHssv7RXUlRVU0FA0Pk2swbnp/iuo+2DDauSCLFr3cAQGDZvVeaV+JQ1UUU8bCJotyei2JK3hXEcKo21mI5Ys97M3N/+Sst2i6mqrH49gTKvEayOIwtOSJuwPcoeD+Gn4659XA8SvpiCWP0uNVjitosIwCqFLS8R0jmzNidndGNCLaL0Kjw2DH6qDGIGgsi9RJtqqVFQ/dNTU0UkEkMVTo1rgLX9v1Wtw5BUUM7qTmB0Ehu23RduLnybsgAIyttYbKKY2AV/wABLIegQOwOWRwc55sOl1ioVWifmDoSfK9liuGxav4gwfEXYFw0M/N84dluW/8AOq9GdhwYOg+Sqw4XHBK6eFoa95uTdSobcxhHCFJh1sb4hrZK2rf57PvZpVXGeDMQ4orHVtdXcqmjb+E1ttFpcT8WYPhDmQyvM8zHehm31UcXFE+MYW+Whp4qSHKfxJHaD+CzwZhj/ZJhM9HiuKPfqzKYg7u666uu4ur6TmYXQYdI+aK4MshyRtHcrzfBOOMZwKSWnp+TLE+cvkke25d8lc484mxHFIaSmpyImVLPMxgtcaaE9VJa5KdZxDjM+HVkdRXmRpcdGk2F+y6rA8aquHeDYMOo8KlbLOSTM6xjAPUry0Usx8onjzj4L6rUPFHEIj5EkgcAzIzW1gsW0jxSgooWTzwYu2qnkfmILbXKxqaqngmEkb7OGysyB5hzyNs926pWs7Ms20lllfM/mSO8xUcIcaoW1CAuztVijaXMdIy2dm11EKaU7NKejqHxSvIYHG2xVdw1JzDUq5RUs0rQYv2shyBvUqTAlE0lXWMlkNmgWyjZX5qx9Y4Esc3ILaqrLQyYc5zKqzHs3F1ZoiHwmXKQ14sLhYmGokMddyHtZFJpI4M+S7PDcaZhMZwfG45XU0j7c6M6n2Xm0MvhpZPIHWuzVTx1lTUTRU5keQ+QANJuAtxCS+gpOHcGbhkcUYtD68x1JC8g4vqa7EOIn0FFO+eGA3jazYD37r0zHHVo4aOH0IzTMhYzTe3cLh8SoYuGsMdVUk8TnVEZBc8XkL/8F0pypV4QEDq6qnxSnpTSyfhSh+haQuz4iwTADg/hqfD44eZZ8c7Ix5R815hS0jvCc6XRzyT8wvS+DKvEqajbV12ImsoJG/shqYjrqsU6Ob4oosVosJpGmlj8BGAPEMduei0+EQ+bA6yqq2BrqkCOK2h0GnyVzjKmrOKmOjp3cjDcPbzTMBo9/wAly2AYRHHJhk5rpBNJPYNcdLdFYwHW8J4dFFhuIwVb2x1BlyVLzvkC4qKKsxXHhSQvJpmOfJC0aAAey9Nwx7J2VdBXUjGvmJjE7Bv81VpMDZFxW10LLxUlFZx9yP8AgrQs8L4MyWmGLYnJI+vfoXXsQBewV/GqJlbRucw35HnIO1lZoxI1jedG2MgWLW7LJx2Oqw2orcdYObSyNAka11n372WoHHfaVgtVLHTV1NRXigjAGXfW1797WXN0mA1dTgEjKmIRA3kiNs7yemnRdvz6Gr8J99U9dTsqSWQfjGx20UfEtPWz4RXysqDS01JHy4W31cBuSt8bYeeRcO11a0+AjD2AAF7n2JPVZwlqYZTBM6/IJFuxWlRYxLh2GCmjc+R8oJJBty/ksWOS2fmPuSSSTuSuc48WolZfMZ43Z2R3GxHRM+OeFzWFnrCrMkMd7DdX5KtlWRJNGAbWsCucqozRcp5zsAKUcgz5rXb2U9U1xDSDawsqdrOy3SxPlsMyUUrQ60jQ8WtYp48jgR2aVER5GkbqqlpxlEh65tEZJVjCsJxTGc7qCleY4zZ0h2utGs4XxLDo/FSuhmhHqfEbZT7grMlM9lU+BoAbuL/VVpq1lRHGeTleBZzgbh6nw+KSqlbHuOh7qlV0s1LM6CdtnsNiEpDGXdjLkkWUZD2NyvcL+yViEbWAjMVppHdzgHfRP5xupaWmmqqltJTszSSHyi9v4p6umkpXugnZlkZuL3sqgbZbuO5GgQmR5PnNz1QZ3kgk6hI3Jvf6oqeGN878rTYAXJPRWJWwtpnvY27gbZu6rU85hz93Cyc3MQynRQSUssLYsrpfPe9rLRppKWVjYpoQCG5MwOrj7rEAyOWlhLYJalrKuq5ERuS/tZJgdjFX0GP4QzB6qK82Hi7X5vwwP9Yb2Ob5rn8UpMENQDhta+ohBtLMY8oD1rU/DeG12C1Nf+PBCGl8bQLPlPV+vraewUGEY5BRwtoocGgmpuWDMZBYukG4uUGAMOkqXufSlhY0X9Y0C28C4hko5mSVEckrIpAJYmAFjmWsdO6sYtS4PS8IUmN00MBqq6se8uYbmOP/AFZH0UAwd9U6XFZy6DO0SCKPQ27rIbGOKavF5XUFE3wOGskJEDbxl409f+S1uDsHwDFqCrp5omQ4jRuL2mY5rsOwt1/4rAxHCqOGSp5FZnDCC0tFt1FwziT8GxcYlHURiSBpIY82E3st4DpqvgasivPiHh6SknJdLyQLNI62WBhGGcPzY3Lgck8soqG2ppicoMu7AQNr7XUGL8TYvjOJVGIzc6n57eW2OM6NjWcab8COqfYsnvlIPYqzLL2HFvswxLEGVOJTT0ENTHTlkNPTRXDrdz/iF5I+mr8PpIhVUksMMj9JyLsdY62XffZlxozBY6ukxmqqjQmzIX5S4Mfr79V0mMU2E4j9n0+H0wM8kDHmlbOMpMmfcC6Dyx8XhquJl75wT9LIcIayqq+QyOSXmHl2YLlbOD8O+Mop6KvlFPiUZY6MkZpAy2gt1+i1/s14DranEpqitqpIKUF7LMu2SUjseixxR2f2fYfg2MYHX0mJ4fS+KpiKZzzDlIFtCPddu6SGiw+WQVEDhTMeGt5osSBf/BeYcKSV9BUYtRYXW033Wyps4VJDy4j1kHdZlTVsw+sr6HB8RrKqKNoFOyTeMPvnFvloiw9Dw7iGixp1PVfe1LDkawyQh1rPPRc39rGIyYrHFg+H10Io6eQTVD2EyOL/AGt2+fZcJiYnwuSbCaSdlUTGySWQxhpaOyCfFMfhp6SCF/MOr4gLOLhfW4WLaZ2FUL8SqzTwGzQHukeRrGB1KKrquTUxx0NO2ofRSPzyObnD+gN+tloUeJ076iWqrZ2llfSvp6x8cPL5B2CgwOvoKJ4oJRJmrIyBI42bk11A+it2zTKqsRrahrH1t5OU0MbmGlvYqxh2HGrxBzZIZI2wwve42vlNtFHNR08ENVSzVZ58JvFc6OB1VpslVC9tVLLG17Gs0JuJARsQskvYaPhzD5eAnVFdyqqeGj58co/aRkAHJfsNNFrcEQsfwtE9+rjK/wDvXkPDXElfh33mJWS+HrYDFK1jfw7nYgfIWXqnCWN4TTYNS0UleyOSWV/KEgMfMPYXW8JG0aUCTM64V1gs1RyOF7I4ZWOAZm1XWyIERdSwwQubZ7N+qhI811I5zgMwQmDiBlK7R17qQVDPzKAyZmqq6TK7Ks2lNWKUK7C+7Vhw1BzC7lpQT/DdIVdcwSBZ9ZRzljuRIWOOgI6K14oKaOVsjQVEpytZR4jCzPO8zHqbWVbO4brs3tieNRdc/jVEyEtkhIN73B0W+fIiKVWG9lPHI5mzlUjzC2ZSsc7MpKr0c0g2KGaV7nXe4koGnqicLjMooAeyKQWjdJcWYLlBfKdVNJSMq6Ysc+7Hixt2TkPPcdraSkqIZ4Y4p3MnMps70jb9VyuKY3AMdbUw4dZ8nne3ra2l/f8AyXqk3AvDszJWDDhmkGl3aA915rxlgclHURVNJSRRctuScwkEONxYj6FFY2I434oxMhdcHmEnrra9x/gsyt5ZljdG4Fz262K6ScYOaIzeFp6dzGvBY0edwfsR10tsuRpI/wCkRQO0N7H2WbZbmBYbU1FHX1kM8jKyniDw0a3j1vb3WxNR/d7YRiVLNz52CUxP0sOmpT8OYlheEXjqjI2WRxEuVubKAdAfqvTON8Op34DHxBMwSGFjHi41yPtddtWyLqWMoeeU1d91ukiEHMlnlFgfLlB00W6Od4+QjzRNtYjYH/FcRWmWvr5qTC6R16cEyEOBHT9F2GGZMPpYcOfIZJQLuA1Nzuvbjm884roaLovi3UkFLV1ErYYKWUvfsMq34eDHyi89XLE7ty7/AOK1zTi51p82inZWVEbQGPU2KYPW4RMeez8AkBsvRypjK5Z/iWPJpUFQ2SYGc691stxGNj+X07rmaeOqmeG0lNNL1u1txZT+Ia3yyOs5c5wh0wydVFVxvNr6qbmgrkqeqk5wIetyKZ+XdefPVTrGS8RdRua4DZFG65UuZtrLlTavmAOqoV8nwsOqu1AdmFlTki5hOZbhGLPJOJAHtcQewT8qF7CZGyXOxJ2WlKeWMlrk91A9gfGXOGq9GMuVK1JU1EJyPeHDp7LUjnEgu4rAkkIf5bhWaesLPW7ZM9UT5pjl8W2SCm5jgqsddDJbzKbM07LzTi7RKTnO6lMZfdRHNdJYVM3VM82KjDwEjJfqgLNlUUku+qjMnuonOVDvk90JkQEhMToohZ/dA52qLKVEYyTogRN0Ngj5ZHwpBjkDZQmyW2U3LJCXKPUIIswTg9QpDH7JctAOdATZS8r2SMSCvlCkZGVIItVM2MAIK4YSnEJurQiARhjUFURH8qkEdlPkP5UBFigjAspG7oEgXIJxsp6YBzhdQNOllNTua05iVVa0bII2BQyzMBtdQOmDhZqjtdQPLU7taqsshO6scoWuVDLGgqSHVQk9SrEjXXUeRqIjSUlmoDugTzZQ50b9lFLJHBG6aV4axm5WwUs0cETppn5QFTiikr5W1dcyzBrHBfQe5QMikrZxV1bLMZrCw/3lWzJZAbnXdmQufoonPvshzusgRksU/MQO3QEWQGZHAIDKShLtPMgJCCTmfvJi5qizBLMEBl4TF35VHnQ5tVpEmYJ8/uoUs/ujKbP7pZ/dQ5/dJUWMzeyWdQhycOUEocnDlGDdODZUTA63UjSVG3dStCzInbI8DLdGJHHcqJIGyirTX2PqUzZrKmHKQPUaWTPom591Wc/8qG5uqwtGQFDn1UTCVLHE6TZFSteQpWONkIp39VL4eQD0otGzGyEnREI3fEkW6IyhLiuT4ritUsmGlxZdc5liuf4sgvSidrtWEBWAfCUxko5IXm+Q3WwuX4UqWxVjo37SLq35AfMVoLUIgeqi5gPpKAyHZa4MWs5kYc3oqoc78yXOyfEr4ZcLYI6py0EKkKjpdEagkbpwo5QGV2QnVQmVPIXEqIt7LtjHk5TIzNpug5rvzKPI9TRRs6q0IzOfzJ2PeXeZysmKHL5Qq4jJfZjVPIXYj5R5ipObkF7qu2KcbBRmKdx8zVmYaiV0Vp/OiFb+ZUhA/wDKnFPN+Vc+MLyle8Y3uqlZxJg9AP6XWxj+yc39ypYq6akoJJrP06ALyyqrZJ6iSR4817LM4tRL0mt+0TCKX/Rad9T9LLDrftNxSXSlpooB2Iv/AIriXVUrzuo3Skepcpapp13EOMV0vMq8Slk9gSP8VWGJ1t8orpwO3MKqDM7a31NkJbY7j6Fc6bWCYf66U3U8OMVNGzJSV88bT0Dlmm53ckGZuoClLS3NVvqZDJNNJI49XOukHD8o+qpvHL0uD8kPMLfSUoXSA7Zqkhr6qkf+DUTR/wBk2WeypINzsrIhrKpnMgp3yDuAsjuMK+0aaCJtPWwc0/nLtVqVXHd6bmUIYH9iAvLopXh5Y71DdWRK+2hVHXyfaFjJaQGMa7oQNv4Ko77QOJh/1uP/ANMLmJXyMYXlyt0dPz4A8i6g9B4a42OItbDiTmiU/EP8l1RI+N4HzXjMckdBLzvCOJZtYp8T4gqsRhEIfJExvS63Eo9NNL0YwD5JpZoaGlku8Ne0X7rkMP45rsXw+erpMgMTTcFuxVf7L8UkxTF6yOul5s7yTd2tl6+Tm3eHcZq+IzUUrXRwtiNnHqV1FNSxUcJMbfMPi6lcTiMcvAmOz4r4WSbD5wT5N7n9VawjiSTFv/idbiEcdNe0cXMAFlhYc19olHxjV0r58UxOjjoYznbFA7U+5v1XDUWB8qgNXibjDEf2ZGt1vcVYvPLjz6KerEtG4gZxsliuLUmJ1NBhdK8uhiABLhZjv+bIrnqLBp8Tfy6GeM3Ng0mxXonAP2b8QYZiAxOuqo6eIWBjbdxk30KrcC4ZNhnGDRVU7RBPfKTqDovXi5Bz+I4FS1lc3mN/Zm40VmLDoYH52M2VoC9U56sWBCAIXa+ZW2uaQqh38qy8Z4ywnAS2J4fVTuF+XH0+ZSJZpuSQse059G91zVbDilTUOp6IZ4/iPQj5qzg2OVvFMYqGYcaWlvaxdfMtptM6IW5hY32W4mmnjHFmC0s/EEVEz8JthzjvcpcQ09RM+j4eomiOGQgWAuu9wrh3nYzXYtWwMk1tCT7LQjoaJlca+ePM/ot+Iw8ox7g77ogZDSycyV5sW6anv7KevpYWYTh+LVDA2SnIjcANT7r0WshirK5sgZy2NFruUOJcL0D8Lljoqb8QC4c43t9Fmx4viNZRVGKS1MZDA61gGbqvCx81ZlZrcL0er4MwTEOH6mqFMYJotTIN3Fc1PhddPh1PTUVEadkX7Srdo2/zXmyh0iXO1gMFT4V40KzJw+N5YVexAZ2iQ1QmkaLF7XXCo8uSXzE3so2ja60bmd+qKne+OQAOs3qmyflTxxvLgwaXKIVTEGyEsaV13DWAsxmoJnnEDqaESRmM63WRDhsdXVRwCqjtILn2XWUOASYbgNXUTm808nh4XWsCD1+SsqwpcTBnnjIjqpi7lZ3N0+YVnEa2GYUmGGEwujbq0m91h4lhs2D1AppXh2cXDmjQqQGStn8TnAIADCVimVGWNwlfp8RVnCKf/wCJ0dROLU3OGdy6HBsGjxOWTD6p8kZa3O5zDoQtvhPhukoKupZOeaz4WuB0IXo14E5O8Ls0sdWP2RjFj3CwsVw2nrcZbVPpY3U0Y0BGl+60og+KMNY8n5lDUmeSB7IyzORYEjZenwnm5uA40hbJi7oI5Gcw8tgjYO66/wC4vubDm/dZeXtgtKx2zh3VaPBMNpK+nqap3NqJCC5zjbVdJJLSue0F1yTYBPCxObhsX49GIwQYOcOloqB7gJZ3i1wOlrKgMXwvDOKmvjoJa2mjZaNkTbmM9wPotTjkvruJcLwZlNmhzCRzW9bnVW6qahw7j6gcwBgkiERLRZgOtlnLGnTGbdFT1r5ZeeyGSJkoBs8WI+ieWq5OLRzxs0LLOdfqtGqp6YzHOch/gsGuxCEztbTctlGw/iVchsD7N7q0y3hXg/GiNYwDR6zMKxXAcYlfBh1a2SSPdpFiR3CnnbkdYK8MTkrcRU0GK0bBe0sEokjNv1C4Pi/FOJsSxSPCaSPw9NO27QQNQNyV3FQx8uW3wG4+a57FqHEcQxEPZ+G6KIhj7AnXdb4QzyeXTR1NDPLA+TNa7MwGh+SqDewXVcSU1Y2m8HGy7IH3ledyVz0WHTlomLRlOy8OTvigBsMp3VmjMZfaZhLXduiiq4w2chjLN6IqOF7522eGNGriey5zDpKxXw8vzxNOXsTeypXJOXLqugnmpagsyxgNaLbWus+sgjjaZoxYrnaKTmvjOR4IJCZz7M+QR1Dcr2i5Je26ryu8uQ7ONluFh6Jgs1Th3DlBHTzlrXgz+XfVbdXhldUUXIxLDfF01TEHZ4TZ4Hb6KnXYdNRUVLFyJBDHAwZ8tx/zquoosbw7EqRrIp2xPpGi+c2uOwXCf4n1MNeGWuHm+IYA/DKZ2K4HVyTx0/7SBwAMQ7rmMQravEZBPVSFxAsNNgu5xGVkGDYpOdHGIsZ8yuEjymAN6rrjLyb9ca5qFbMB0191KT2AQSsa0jKmuQC236hbedp4DNVOxSGClfBG7XzyjRQcQSB+JPyRx+TyF7CSJD3Wc8FT2vA0IE2nlLM7GXUYLeu6OKN00rYy/IHHUk6KWnonzVBjuwtGt77oK/X6ro+EsIwnF6vwmK1XK5lhE3NYyHtdVqSiYwSV4YLROz6my7/hnDqyHDw+toqQwmUVDXcsExa6/og8+4qwRnDnEFVhcRkMEZvC55uSw/8AJTUFNBFEHV+HOk5pD4HH+s9vZX/tJlgl4tqvC1QmYzIAWuzAaXt/FNX11dg82H0tdh0cvhgyeMPdY6i6Mg4gxfHJvD4JitTG+LDz+Cxo/Zgjb9FknKWCMHyD4RsrWL17MUq5a6SljZNKb3b/AM6qi14ygIsSMxxht2BakXFGJwsjpY3R8i3LdnbmIHVZkIzB3sL2G6iGnmSYV0eKV0FSZJKQtyPBjLQyxJ0tosFzLvcOxspAbxZkItv3UjyDzSTTNBmdmDBkt7dl2WKU/DLeGfvWniMsZljMVPJe0b9M49/muP5VwtNuJzPwaLBnscWRTiSNw2Z7FVJhp4Q2iqcPqMDpo56N9bURyQulF2xxje59v4qXiTCMSwdzuH5JJq+mpn+KppjcXjeDfRV63HJ8bmq3zwkGjbeAU41aOt/bZd8/jbgSswegh4sxSOtrWRMu6ip3kgCxs8G316LcS5uawPAK40dfXxvfJVYU2KWlD22kBvd7LH4F6rV1E9FQE0LI2Pk1Lsu1xqdFRpsa4RxzGHYpgWN0zjLAKaqpJbtkkZrkNjvbXQJOx2qr62WgwrAZpWROyvqKg8qP6XuUhcXjVXPBhmKTciKoa0TCWFziQeXvf3utisr8Px/E3sra2Sgc+TPFV+oR6bO20916LxBwscYgrcsMUj5ImRQNIAA11PsfdeZ8S4PieFl1cWQ0kUGSExxS8wOJv16pODSrX4BiWHxVNYyHxFLGLeLZ6JWXsDdC/HqONlLDhNLI97B5pZGhsjR2uN1DhnFmP4dTCip5Ym01iHtdCHB1/nsqtNRvxGSq5DQQxpluDl+i4TisSuS47SxMbSSQxm7Xslc1mtj3V6fGKV+HxRQcPwCmiayJtVODnOlrlYEFLJWMlMcAme9ut9z/AJpQVMkN6OaM5OkTz6fl81LasNVCwuMJcJS+2V7To4lW5W1UFO59RCQ4BgaQQfZKSOklZmjpeS+Ih57Edx3UkL5qjDmUz6fNncXtk3trrojM+bfwN2KVWD12JuEM0FGWMOcG5vYWFvmui4PpqXijBKilrppHOintUsabGnPwSj+4/JcvQtxDCaV9F42A0dQWPIa++vdem8F4HSYXGcVgk/EqYgyQNNgSDutRgjYw44pFT+ExZ7JpofTUMHlmj6H2PdWw5wdeyEzMGlwq8tUGg2XbHC0X45yT5irLX3CwGYkwShhd9VqtkYSG5twtcOKckspObybKJ3dTaFRTEdFloGYjZTMqXgW9lXvcqRougmhEz/2e61onZRZZbJhFbJcFWY6olu6wDqMRnilLY4iAzcuWNX1klQ4h7r9loT+ZhHdU308YFyFYFJji4eZTxlA5ttkzXWK0LzXXRczRZeIYtT4dC18h8zzZrdrn59FzWI8W4q+andhscccOYiZz3Aj5e3VSYG1xPicNOI6KqqH0Yn0E1rsPsVl1+NcR+MIwTmRsjijYS5oLQLblZ+OYhRY+KVj6KtEhkAMjgcjRtcDZaTK3CcLjOGyYrKIngAiQGQvB0vf9FkdgRJWULzVzyROjbZ2Tc9z7rg+L8Yw11HSfc76eaV7vEHOLAiPQXHc7LsaRseEc7wkxl8S5giMjrEAfAL79V5K7haaafEsNqq2Ns1LLna8i7bP13voVAGHcQUVR42SrwrxHiSC3I0ExP+Z1/wDZc5LDJNiksIZlcXaNvsq1RAaR/LOpN7J6YvmqQ904jefidskkrVNG+WeR3KfIyL1ZTZenVf2h4VivDTaCtjkiaYsgY0XEYYwjv1uvMpvFzPLI2W5pvJy+p91vYTh9EIMJlnjFWySpBljA9TAdQpBKfg7D8SdUOp6UytdWu5jXEXJA6r1vlYbwnBSQyYeJZZBrIG3eT1JKzMOxHgbFMWo63CLw8hxpyAC2Pa+36qDiLi2glxgT4dNDWMI8HCA6wa873J/RdObHF6LTCCSJk0erZBdpta4VsEW2WLg+IUVZByKN120gZEfY22WmxwWeUulHrKOnrYXQVEYkY8EEFYNJwXhVM1zKhjp7m4uSLDtuukuLKJ8lit+IUekp6ejgZS0kYiiZs0KCqw3D52PjfTsLZBY2AurEUgepNFjxCnIM4O8PKH0tVmt8Lh/jdNNG+kfZ7DbuAutEbScyjlo4ZWlkguCteIU5qKS4upM1ypcSwh9DSST0Ty8MFyzrb2WJDXzO+CT9FpltOGdqhEeqaGoMgFlKGu7KKjfCxzctgs6qpZ4zdjLtWvESTZyeQWWoyHPzszQuGQZgNFlujOfl5SHdituspRnL4yQfZZ7aPO8vfckHqvTjl5OGUIHQPhs+QBwVyCrjygbeyCVhka4Fh0VOxjSazhL4tpkrXDRyRKpQuIcLK0wOI1Xmyxp1xyJ7wEHMPZSGEnomFI6+6502gebILlWTT6+pD4X3UFbzd01ndlfFLqiMDfyoM4ByMNVo04v6UnRMHRBGIr+pPyXflRAjopY3N+JBAISPhRsi6WVgFpSNh1VaRclvxIDCPhUj5NUHN1UZPyT3QOZZT52qGpjkkZeN4HzWxGMt0+ZvZROgmAuyZp7pgHD1OurSWl5zRuE/imfCVCQT0QhoHwrfGGblYbOxO6aJx3VUlAX2CxMNQuuljy+pQvmbsCqhe5AXuuoWviQqRk2UrOZM8BLnottdtT7qYVLO6w+f7JCsA3QbXiWdwlz2HqsbxQ7pvGt/Mg05ZIwMwKrumA9RWfJVuPVDzy7dShaM5ug5yrufdMc9rqiyZQVlXdidVzHj+h0zsjWnUOPdTzzthp5JMzQ4MNrlUsNrsPpKCFktWA4i5AF9UGqZCmz32Wc/G6Abc1/ybZQuxxh/ZUjyO7jZBrFyQcsV+OVF/JTMA7E3UDsYxE+mSNh7tjQb5cgLljUOKVc0/JqZuYDsSNQtIyhAnvddR5/dC6TVAtMJczeySDMEswU8mvIWYpkOYp8wT1ZOkns2+ybr9VoM3ZOiA6BJA4RAeyEC6MNUsO3ZSBqAHVGHKCVgsjAuha74VIHBQGkmBupGsv8ACigUsbbhSNaB0UrcqCIQkpcrX0q7FGCE74ulkFNsXsr1PFYA5UcNONH5FYDAOiBDKNkZkZl9WqjLlE52VEE7zFM0aqLnI2OB+JAb4gR5lh4/RzTUMnLZewut7Oo3wsqI3wv2e03VgeaUMz6eqY9umtl2skM2Rj7esXXE1sfIr5mN2bJcBeh4fLDWYXFN+6ujLNEhb9UbZbop6dzXEjZA2Nd4mKcpHmugduiaNVMyFm7gpcEK1ndkYzDdWsrG9Ahcxh9IUs4yrn1Ji5O5huhIsVuJ8kFuqGL4xDg0RkmtmHQhXXOLdl57xRJNPisnMkDrbW6LK4tocf2GtI0/X/goh9oteHWjpYGt7WuuTFM+U5WuA+aP7tNr+KjXnnPJ14umr+Oq2rAyGOH+yFlv4kxGTfFaj5NKxHRHX8Rp+qgMci588l4t2TiLFXtLTiEnzQw47iMLs/3i4rCJIKAuypzyV1j+NsZdA6ndVNkaRa7marnHmTO459zfZVhI8nyqRkj5HiNjHucewus3k0Mabq1h8OFzykYhOWgbADdT0mEGolbzIn6exW1LhlDSUxfyXtez926KpRxcMRvDmwyut+6VLNVU896SgwmENOznDVUGVb5DqI/90K5R0sNRNnmrWwNHUndBRq+H5qRuaaVuY/CBcqlJhGINs8UcxYeojK7zDsJwN1UKqrxiGct2BlstSo4w4SpJTDNiDC4dWtJCDyuShmj8ohlP/llE/B8QbFzpqd7W+69Jl4l4TrH3gxKMu7csrJx2t+8AI6GkbK3q4C1kHAikqZT5IJCPkugoMchwmm8PNGT7A7LTpcNcaTPJM6Jw+FuyxarAcRxCqMNFHnv1ugsYVLw9ilaaevhDeYb5grHFHCtPhFOK2hml6+Un5K9wrwHU4dXtq8ZhBA6A6BdVxFV4OKF1HJaUHYB1kmB5LRy1NRMIY4xIfddTT4Xijo2sZR6u/LsqeGU9HhVSah0PNZe93OXbQcXcOGIyGSOF35bpMDkhwfijJBJPRVIaOuZFBw/Q1dRyOW+/U8xa2I/aTRxOMFJQmVp+Iv8A+C56krq+ulfV0TJY7m+nRKHm0E0wbmhmkbffKbXWlhVbiOC1DMUoX5Zh1VFzZM5GQC3ZBLLLo3N6VVdHjvHvEmP0/hauq/CtYtsNVX4UmggkmjqoOYyTc223WW6R4pc76chv57aKKGR+S7Hlt+yML2MMp21bjTtkDezuitsoavFaOKqpIRaD1OHTuqtAJC+KtqoJ6inJsLDQn5roqgvpp4J34dW0lMSDJHl8hHTRdIlHp1Bhkdbg2FTSPcHwxMII6roo35wuYw3jXhaemjhZXthDABaQEFaMmetaWR1v9HP+pdqf8lbDnG6enllp4YZKuYO1bELgfVUqvjDCqKqbSYowUsrxfK6YE2+ibGKiHhbh2pqqKMmZ4yMI3Lyd15ni/BNazhp3E+JPklrKl2ewfcgGywPRMS+0jhHD6Sd7K/mTBvlY0XJK4zBuLsSxyhrpvDwOdHe92+dcXhvCnEGLSthpMJmGf4naAfMrqaDGqThOajw2ehMM0Z5c4LfgK1A7T7KcXfLgtRGOWZRNY5jv1XS1VVik73RcyIM7BYNLhElDi7MQwhwZTVjRzGjZdEyJjQAGpIKmZPyw2SS9u6d8VjbS3sEYdYJF6CuIYWellz7p6qdlPHkAJfKLNSJ1TT5ZoeQWXz6b7IK2HYYTCKCsZ5QS8juuH+2TG4KShi4Yw0RgvAMrGnYX0/xWlx5xJX8PU48JNllDbFxFyQvH6usqcSldW10zpZX7krORiqwtOZkD9A7dT1VIaOYMY/M2QaFQPjbfr+qIzvMjc7yeX6fZZpuZRmCSM5Qw2UlNGXODiCApJq8vaWZdT1UlPNmj/EF9dlCJS1X9HZDPB67gBvQr13EZ5uJMOwvBpCYMUEYla0CzNup6LyqgmjGNUVRMLxRSMOTvqu4xviymwP7QIMSqoJWwci+QjU6aWVaUftIwubCcEoKKtMMlSJrl8UlxttbouJpJ5Isn9FYchvboUeIYgcXxKoqpeYTLKZGFx2BXQcF4OzE8RfA6rED6dvNALb5vZRmW4IcVwhtNi9fhUMTKtoZ5TZ9iu1kwDEZaaHFoBHJE+L81rLi+KsdxDHXwUtJSuFFSFjJHAdey72q41wvhGOlwCGmEjRGwm7rZQf712w28HKceSvHR1ojBkpSG9wbrnMTxgNqzBTOuGbn3XXQfafgs/OoX0kjbN9bdQuEo8MZjOLVLWVIbSsJJcTYH6rp+IlPBVat9RVyxzy/1Zu0/3rSpa2omiZOGHIOp7rUoaDA6ZwZV0zs40aW6gBPiVPhdZh89RS8xvhPO47aLHPI8Nx+LY6+KeoxTEpiKjlmGnDRqB3A6BVaCifVcImulaJKlspkilB13WPjlTUY3XtLI5NDy4mW1JXY0MTMF4YjopMsNU+IkxE6kreMpOFNeLFH1WEQsqr88RgOA1XI8aV73spIJHubTgWsDpcWXQYHiFDWUb5IoHxzQNHNznUHXdclU08OJOqK/E8x8v4UbTYtUy2cUxdRw3j3C2EshFfyvEkXbVsjNre/uu0c+GeJs8MokY/ZzdivGo5G0ETGMe7JIbEHXRd3w3R09H/SsFqjNC8Wlpr3EZ7hXHbaTDopHRsaXveA0bkrE4q4iouHaWKol5kpl9LIjqR3WtXUHi2NZEwZ2m4zOIC8147wvGZZTiuMOhyRizWxXIaF2nOmccWLiOOz4pK55jEUZdnDb6rNqa6csEEbrNGyUUTwBIRYKGosZncttm9F4snsgjIxxa1rnPPXREbOPLJsD7qFvlKJjXFwAFz2WVXmzAtOT4NAnrQZaAzAWKrWLfR1RTzDwkVMx1yCXu+azQVT+3hPTlpQwx1VdTUztpJQCoOY55Be7Za3CcTKriWiik9IJf9QFVevRYjTyOEArbzZsvLftZS1eCYFi8742cqOrvYvYTcEey42f8Wokl7uKlo8RrMNnbUUrwHs2zC4XknKPe+xj02XCKlW48w/7mwN9L4qOfmzs1aCD9V52M18q9A+0fGPvLBcNEkQEz5SZHDqBsVwMbfNmXfVVPn9T/EUmhARZwTZyYh5OqOGYUj3F8GcvbYa7Lo8yWlooZ2ufNMI265L9VJUQ07cvhY3CM+lztyq7ZnhogeAYurbKxVV76gxfhhrIm5WsGwRlDG2EB2dwzu9JvspssUXkbOHPG5A0CqOLnOv0UtM19TL4d8lmuaSDtayLbbpc8+Hsw3JaaokZG0g6SAlen41UQYPwzUVsEnLlFOIg3qH2tsfqvKcBhrq7EaOngke4UzuebaFtt1PxHUCvxauq6qtkkc+XJG7o63RBl4PhE+MYtRYbSt/FqZQwfqvTON+CeHKZtTjuK4pLNNHS2jY0gRueNAy2/wDFcLwjibMD4gpcWkhEsdNd5aTYnTp7rfxzEJPtNxylpsOpaijw+mhfqWXF9zcj36IjiafDairiNVDrdzWBv+KCqpKqiqDSTx2mBALb91r4rg2K8O0tLP4jK6sD3sgbpIIxs8rEnFQck9QyX8Sz2uePUPbuhC7SV5wzFKSr8KWchwvlGpHVWuK8cw/HK5j8OoBTsjBu7KAZCe9lmyVDKhhE3qA8p91CyPO62ZVpJEHuby2hLzBWcPo5Kqo5EcnL8peS4dAvQMG+zBlRh+H4zUVR5VRSmSWMxEi5Ht7KDgmTufAxjtmXt9VNDUUzAJJJsjvy2uPqrnE5wqpFDNhkbo5uUWTtY02Lx/7Lns+uY7IjpYqrE38P1NHg0lo2O5k7mC0hYTt/eon8XYtUYNDhfPjJi/AeDECXjpc/oqvD1XLS4izJWmnjO7rXF+gIXe4Xi/DNZwZXz10GGfeNPNa8cYjkOvkkZ772WsYSYcjHgGN0DabFqChngrIzpHyyZL/nt7rq8Pp+Mf5tHjTCcaxBz4JSZ6aZwyOjA1IHWxuF2OC8YcJcVT0GfGDFiVJIRG2S7eaANL9DvtfdbYlwSlpHso4Y6htGH5nF9oWm9z191Ep5E/jLiymxep+66+rliqBeKGdplIBGpFxpZR4hWYpJFDJXU2aKKQRCNx1kJGthtbVeoYPgstbE/i7GXxRVdRCGRNbHy2QxA6D67/VcZj9DRDGWMrsTnMsjJJsjIw1sEY+W662y5ypiwgc2KOighmMWfLJcAew7rGpnT5pIYY+Y2W3MZewIHdW6/EoKiJsEEjXZ2i+l/wBCs1t3S5GPLXdwvLk3C7EaVte4Nhkipnuz2BtY+yixDkz/AI8jyHs0BB6XSpoaurbIIDzHs3DjqfkgqxHFMKOdliRmuDcWWWj09dWwsZHRT3zm1nC60cOp8dbFJXQwckUcoDnSiwu82At1CwzHS5Pw3lzyD5dgD0K9NwfGfvjA6qeoMUUj6fw7WPk3kjG4106LeLKvivCnIlGIuj5UFXHAJYBr57nOQf4rtMHjkpcCpo4i9zY87B1sAbBcTiXGEdfhzKCmgka1nLeHuNsr2Czxb5/oul4P4pp5aaDDam5mkqJIwQRYC1xdd9c44ZMNXxM1/UnNS+QWcVy1VxWaCkAqad4czPHK8G5jOwuLdVVZxLXU7Gwh8M5YcznAWJZ10Xq8bByrJ1xF3LYoZmyx6O1Gi5zBsSZjlTJHSQuyRND3OOm66CGhfCbsCZbMczGMmk15I1ScVUp5J8zmSMt2VloJNyvJMOsSdEx2Up+WO6flo2VweqdsluqhIcCm1CwLbZXEapiA71KuH2Uscl0Bmna8WaqE0ZikyrUh1Veuh5js7dwrYqjlEWc1rvmLrnpeJKWeafB6vBeZEfIWtjufmqGI13EfOqqQ0rgTIHxOaLWtrcFWjW4S/Cpq6qqvD+Qxk/EH9gEsQVmEYrLhopIGSxwn9oZB+I1g2NlnT0VLNRxvjmcXUw5ZaOvY9yj4S4mqqwsw7FMR5cEl44nEAPJ7EroXU+AUczmTSMcdPONz9VmRzfEuM4tiVFS0ktqNtNOx7vMQQLWFiuPxriWatq6qeleYxUOHMuNyAvROKqPDix1VicjX01THy+Za5A915VXYfHSvc+OfNGSQ073slLSpTziSrYa6S7bkk27oqVx5bW215hP6KIx+dnZzgtLDY2PmpWkXz1GT6Eoi/wALf0yskwd0cmWqLSZGmwjAGpPYWuutrxSYbQT1WDVENRFTlkQt/VMyEn5nQ6rDxPE8EwOJ0nDlCIJpD4eSU3sAd90pMXw8cN1EEdO1zczBI0u9RItf23/ghLDoK6aNtRSxVElNFIOYXNNtv/ddBg/D7KekpajGKommlJlEMMliT3J/RcgJJsghiYC6byG/ZdHh+HQTxMqqiOUvgljvGHEh4JN9Pp07oPSuFeLnxYo7BoMNjp4Hu/Da2KxtbcnqvQG1rD1XGUn3ZRUcVVLJEZXtOV1/S87gDp0U1Ji7y+OGd4ZJKM7GW1t3Qdj40AbqrJV5nXDlk+LuPUmZVWfug2mVDrjVWI6h175lkibQEOViOo8o1UmGmzHKwjNdEHtusls7h1U0c7i4aoLr4+awsI0IsVz02DxtqM+uhW8+oyNuFRkeXXLlqMhCyBjWBrWhEIbqPmEJGa2xUtlLJDZtwsuokqYpf2JI9lotmL+qlyh48y3Eq5qWsjmdkjddzk2h0+IaEKXE8OgY/OwWzf3qjFFKJWut5D1XfH0YyTkNAKKKn5gu1ot7qKV+R4HRy1WxBrG27KIqCmdfUKVkWlrKUApwLrlMtRCPki10DmkK2BoontYs20qE2TtkZ1UhjaSmNO0WI6oCB6hEC1JrBlSDMzsgOqiIJX+byqKQ3bruppWua8hzVE/8RoFtQgrkkJuYjIFlEW6oJRUaJGoPdVjmugJKCwZnX3TCZVXOKQce62LnPsExqPdVWk/mRAEoJvEaalNzv+bKIMN0+UpyEhkahzqN+m6i5hBvdbthYJ0UT0JnUfNJKjQnbogLhREkpB2UIykI0USstZmjujFDI6Iv7LLaiS5RnZSyNLDqgy3CCMvchz+6NzUORyAc3zTsfqE/Je45lE7Q2KMM+vxGrM74Y5jGGG3lVN0kz2/jTSO+blJXN/pclhvqor3Cw2V28vysSDbgOsm0DkrkAkK8loT8gamaRZINlcM/LNu9kJNkko7t0PT6Juv1RKWWejPLqWPzWF1sO3WGC4SjL3W1muMy1CBLgmzFMktsHzuRZtECSokzBEDqhAsjDVkEBokBqia0uUrYlQwbonyeylDHJBjlBER0KbKFOYk3K19KCJO3dTinJHpT+GKQIweoUrGPPRSMpfZTsha21wrwEMUTswvsrYYbI2OiHl0UoMZG6s4ylwCOJ7jlyqwykv1sjhDQL3Rk2PqWFCyEM+K6mY0EqHMU4e4LDbSLmZbBQOOirmcgbqI1Did1tLSySWuq5JeUZ1CYZWogSzRMA4G9kZI6pN2RDmS26YThvxIXDRV35QVuIS3GcSQcnE3Oa7SQAre4RrSaB1O/XL1+Sy+LoHiWOoG1rFLhea1S6P2JXWkdcJI3+pBIxgGYNVYytZ8SqTTVBNxUAN7KsLd9dkZnt6isjmScwF8xt1RSknzhxcPYrTDQNU0+l6dtQVnRF23VW42PA2QTPqmxML5D5Qsas4ywKmuwySSPHwtam4obV/djnUpAtvc2XnkHJEg5u6zk3i7WfjOiqKV3Ijkif0zLk5qhtXUl4fe/8FUqZm8wtjblb0Ct4LO2CXzUEUtu65c2+I5aKYW0OvZWIeG62rZnbXcsdnLUlx+OFn9HoQH9WrMxDiuqkaYIKd0d9yAqqlXYS+glEb52S36tKmhpHPjzsgLvkqZmqZPNJ5nHqd0TZ6qP0afIrllJEiFLQuJFUy4vq1dBw7g/DlUwvdh40NvMbrmXvf6pBlSFbLDfkv33WeaurfJws2tME+HRxsj3NjqtiDH+DMPiEVPDGz3bGvN5K2WT9obqDni+yzzW3rLMewKrcG01TEH9LsCCuxrD5Kd1FMyJziLNIIXl7ahqTp4j1TmW08Qw6aWoLxJFG0bBqqyPETOS87Kkay20ihdVBx1cSVORbUo6eCohcXuiGvXdZ9TBTNeWRzsPyF1XMz/zJCZ/Vy1bSSnjMMoe2axWkzEq2lPMiq5b+yxXSOJzFTCVzo8ocg6Oj4p8XLyK6tMbO50Ckr5MNa0SYXjDhKOuZcxFh9TO3mRgZT3KikHhJbTaOCDpKPi3HKYFk2ICWM9SLrMxirnr6vmsnuOmUqGnlZK27LFRSyGOe+Qn5IOmpKUOo2xz3zH62UFZhkMcBfnH10VNuO1QaBlj/RXXY1SSQct8WZ/e26DNpYqQyi6nfiFXQy2o3mJo6EKuZs0jX5ALG+i6KSqoJ6LJJZ3a42WYlHmVBTz1mJR0QmLXF2QkdFu4hwdiUWKtocOf4jOL53+Vc3I6ShqXuhmOYm9ypY8fxtjw9te/MNlpXay8IcSU7GQyGkmiJy5GHX+5YvG2CQ4VUxx0kcUbXgXs7r3UUHHfE4gNO+uGUC+bKLn6rn67EK7E6gz1sjnH3N7Kq9l+znC55MAlwPEYwJYnc+N7XBxudRqF6BRTB0DY6mRrJY/IQTZfN/DuN4jgVQ2ekqnNZ1bdbruMpOa6rqpDKT0LipLD2Wt4c4fqv9Iw+icT1ba65Wq4JoY7vwPiOai/cjmBBXlGI8T4rVTPfRYjPBC/4GSGy3+C+JJ8CycyiZUMlNiSdR9VqJRK3inEaCqOB4wJaynppblzRZ5t1v13XpVLxVwnxFStofGGnde4jmYQVl45wwzHDTY1hTxDVRecaC0nsbqU0eHYrEKHH8DjhqXC3OY3T+Coi4o43OBuZBg2Fmtyt/0gE2+i8kxmpnxytNfHBPZ5AeSLhpXpM/DmM4FnnwmqiqoI2n8KQage11R4RnwWqoKzA8RIgqamQk3Ftfmg63hGTEMNwmkw/GcokMeeMg3Fl0fTRU8Lo/BUEdFOY5hE20bra2VsPAQLX3SzFC9wshzO+ErVBwLqRsrIjmedLKuZJi/KLW7pOhgiA8RMLvNhfqsq8i4xhr8VxmWGCO8VRMGZrrGxrhuTAMTjw6acyRyAHNlsAvbKynpg1skMUZYw55DbYLguLJcP4qxqiwehrRmiD7vaMwuenREcfR8PR4tXnD6EuzgZrk6LAqKSoo6uSkqGZXxOLCL9l6nw9PBwvBX4ZW0whqYGF7Khw0k9l5fV1UlbVz1c5u98hJKjZoeQ12eYXPRaVLRvmibUcsFoNywmxVSjwuprrllo2D43bK/SxTUUBZIcwHUDRYpIbVVS1fEFFzcMwtlLDh7Oa6TMATboFo/aC6lxnhTCsZ5wZMYxFkO5t1XacAcKYc/h51XM6V3j2kOaX6NC4us4Yop8WqsLq4G0/hhnjAcfxFVt50y7Vehq6qlMdXSTmGcGwkGtvopKLAMQrsZOExMyOuTmI0aB3UFSx1O99E/1Rm10HpHElDPhfA1A2lquY6tkEk7xqXF+un6IOHuEa6jf/OriqUx0bG5I/FavPa6Hgut+9G4SzEpopYaacR+HcTntrr7LpeL8RouJOKKbAAJJKOn1fy29ex/TdXJhwfDmGux3i84PRT8ujqJTJI3YPjGpsvYHCjoKp+G4dgcDmUcdw0GxP8N1wldiOC8JcfUUkEXKZyTG4NHU7f4rWxbE58MeavDaiQTVO+dwc/Is2nJtY7WSYhRUGIYbTlvMcY5Y7as9iucoah8dfV4bJBJlk+HuVV4L4hroceNJLPI+nlL5HNAuM/ddJg1dXVM9eaaGAOjkvd4uVbVzGG8KyMx+XEqqNkDKV4ywuOuvW65x9R94cSVuLYg8mGgHkDdOnRXMT4nnOJVj3Ou+Q2uNAB8lhYnVzS0rxDaJktmFoG4WbANxKqp5nVTJbQYhrI3oCjk8zbZljz1E4bFSSMDeRtZTxTmoLQHaHdc+RxWMQiy0T39tlewKQ4dVkRzSNgliuddyqb5iW8iVmdqd0xiAMdmHuSryXi6atx6bD8PMtPK/nVElruOgWHLUMqoiaueSVgILwTcn5KjLUGU56h97bAbBV4p2TX5d8vQrp4uScQ1U0ckrmRss3ss+WB9yWWPsFdq7umDYmG9raKORppWl5sZCNr7JdulUzo9FLHJyjnZ6rWuoLFu+3VSMd5QqoXStGnxdUzrFocENrzKyaWMxNImaMwvYoKhJJsN10/2fUb5MSrat4/YU5IPYrng4ROD2M8w2PRdlwRePBa2rcP8ASJgB9FnP+F01eeyIa3L82lrITGVLcJibW/eNl873v0uLn+MIZKvEaHD6dl3mnsxvc3UGP8M/dEFLNA6SQygibMfSV0vEmC1OG45R427I6GBrATbVp6FUMXkxDGJ20EAvJPZ7RsAP8F9PTr/K/OdXsjLbMw5GWJxsIWeZotbutjEuFHRUFHU4bG6R7IebPIToT0/xUYwqpgxJ2GyjLMDYH3XSy4u2kwo4fPIA5pByg3P6rc/leWJefukBY179H3II6hWJICKLnfGfSO6vY/Rxh4qp5h4mS2djdrqKWSA08Rf/AFYC5NM8NsNVYouRFLzpz6AS0dSVYjw987QSwt8+x0K67gXhuirJKn74pObTl1h2jO1yfqqS4vCq6qo5JjRBpnqwYx1IB7BBPLLO6Onmf5YLeQ7grbq6ulwnimukwuia2nps8cYGov3v1WJG2Wpq3TE53ykvcfdUiGvw9w7Pjb5ZTKaalp/2s1hv7LvqbB8UwLhigwzBMRinqKuqfmqYR6AddO64vD6h8OEOw6sqvwqiflU7WyZeW+/rd3C9Y+zzDYMN4cjoDWU9bLFM98j4jdgJ6Aq4pk5DHuBJG4th7KvGZ66WtgkfPz38sXFrd7Nudl0GMcI0eJYhScRChBgw6jNK6ke2/L5bCQQeu4UOOcNYtivEMVW+v/BrHGF0Y0y043t22C6Khrhw9h0eE1876qeEPjhjAvJKy+hP+a1MGOTyzDPs8L8Jfi2Py+FikgfUwMjZaSQja5v2We3giufG6afDZIT4F9ZE9h0lAtc/xXt+F4cx48di0LAWNIjhveOBnZT5MJrIJJqURG8L4I5GbAHt/wA6qU1bwbgrCvvPFKMV9O2SgMlpnneO+1xvYnQr3CGCXDaBuHU7GBsReIy0WAFjbT6rn8enwTg+mw6rge0+EeKWQRxjmFj9yQN9lXg+1vhypzMZh1WJSbQNda0vzPwfVZRaw3AYMIw7CZ46AuyNn57XtFxJIRqf0XknFnC7+GsTmoHYiJnTgTxDlnYn+C9Oj+0OqgxN9FjwpoqdzDJCaeO4Jt6M/VZkGL4a/EHcc8QTxCLEYJKaCAMvlFtvf5+60PLJIH0lYaUSRTPb/qnXG3dSgVVGHzH8MSeRw62TVQwqGYHCoZmNyjMX6EP1vb2QOm5vke8uHUFSBr8OeAMs0D6Z88tQ0RwtAuS++3t816LhWHY3lg+8cDo4Kai1ginqWBt+pda5f00uvP8AB5qSeikpZsOfmpwWNqGNtqdgvV6XFqHBcIon43w74asqCyIsgbmEhtuOx1V4jC+0rEMdfRUcJxOd1PUh7BHTRBsZeNvci1/0XFTmdn9OpIzS08jRTzPHxnYs+q7zF6yOrxumxHD8KqG09NG9j3VAsQTsQPZcRBPVTObSvjjmpY6iSV2YWDie6xOXFJhzkLI8z3B3J1s3MrkLKWnDudJHPLIPLlPp91oUVRSx5pqlkZZHdjaf1AHuNFpYTjVBgMVTVzYSKplfEY4PKPwydNb7b/wXOZtYZGFuZ4kEzthcASwuGhPZLHI5460TRxA5CCwiO4I9wrNZwnxFQ1uH4fPSGHxsoZBUO1Du9/kug4g4cwKkmz4lizhUMgPOhgjIEj7fh29rrQ4qKnEsv48BD3kCwBaFp0s/gaeVkZLBTysLHgXIJ0P9y6KLgLH8M4eZjuIzR897SG08gNxf0AHbOVTxOGOiw+WlrqR+HufVl7mvNy57IwCGfX+9Uc9PA9tRyzPzfLmD79fktWKaaCqgkgrsvLAfZuwk6lAKRjoIoK48vmATQk6loJtYqCKq8JVG8cUjIWefL1HsuTUwvPpcSxAVjIZn1stRKx8gYwl9mfL/AII3MqXymCuglDKedjTI5thGHjQH9F3/ANmvDtFh8EmN4fWyzCsjAma4BpikvsbLfxbAMNxennhqoBec5y8bh9rArpxYmHI8EzUdPjjY4qp8TjGY6iJ4sDqLEHtdehskB9S52h4Zjw/EjU045kUkUbJSRYkhlt/oCt3KR8JXWFjyWyyI7WPySDbIaSMSEB7g33Ktcl8TywsB991LFfKVI0aeZSmH2TBlkVBJB8QVd4dctWkAL+YKCeNubRqCmBcomggogLKdgZZRDwusmqvNE/3Cdo81lkYbiFVJjFVQ13kZHIeXm0uAgrYmH09HPXTzRwPAIaHnQkd15DiNbVYs8smd5ObnDBsCvQONsffh1Y6lbikYcHZizRwDDsLEey8+qaibEq1tbBHlBdaR+2Ynb+4oIoax9NilNHSvtkcyQnoD1XW0nEsIiljZhzZH88ySXOYfpZcdSU1M5jj4hsdULjI4a3HRXaWknFMyqojy31DD4rqAL7IOto8cjxOeqwzF5I46Zg5kB20XHcTmk8cTRcgt0IdB6HM7rocJwunxBx5lWyHw8BYS0aW6XXIGhdU189PCzywOeB7C6qq9bD5444Bq8XAWtwoaSOaF84Icxz5HEC50FrlY1XIIJ2wB1+VcHuL6I6SZlNKzxTy1hafSdTfsoNXG3wVGJcjDZjJSySA5nbmTqs+T/R2scPc+3RdDhGHR1uA0xnqDTy00kshc6K+hva5usR0FfXPjpaeMOdUOIjaNL26/LRBTjPnBjdsujwF88lVTUM0/IZcSSPe0E6HQe6lq+HaTBa1koqmSWsBG0G4fob69FdFX97Q1sjIQ19PkdI7YPAP8FEdDiVJ4Gio7Me6KKoNjzLEgjsr+GCR2KeKl0l5drdAOyxcLpqrGaWjxRjyWRSnNE6S4AGwA7raosKxkV/ixUf0Z785a8au/yQb2dwSEjgo6mqp6Z0bJ5hGZTZgPUp87UEwmk/Mpo5n3DrqmJWp+cOiDXZV36q1DUeUarDZM6/lVmKV/dGm4JyeqLUlUIp+hVtszXBIBcth0VeaB4JOhU5eB1QySAjyoKzHBnqUkcwJy3VWU+cpNZmF7rUC2/I6+bULLqoJM5MZuOyskyt3cnY0yHVW0plM807bsIstHdSvhcPkq7n5Xq8rSITEXSEZKeJzTurTA0tWbWEIa4IZoAWZ2O17K2YhZUpSbluYqKq37pZ9vZC+901tbIg+aAEwl810OUJFmUaKIle4OOYqGWRjQcvVROkcFBM5xVUz3tuhzNUMhfqoXl5uiJnSsBUTpAfSojnKA36oJS5MHapmkXRsYbgrYJrS46KeKF1rpMbYqdrrIGELz0QSxkGy0YXQ5dUb4YXa6FBhSDVRFl+i13UsRkytbun+6on7PKtjDLCmDCt44O0+h1yq8+EVMOtrg9ipYzWtspmQNeMyPwlQN4nqxDC5g1UsoEEbWepWg5trJ4oGPte6uNpI7aBSxkVFJE5peNwqD4gPSF1Iw9kgy5d1QxDDvDnzNCWMMMJ+FLlEfCrZhtshdG8DZUVHhwChkFyrDmv7KFzXKjCxciOs16tCo838pWhjrWiaJ9tS0grKksBosqlMjer0HOcwOJ7KMbpSBxaVFT8P00uJSzslmkLA1zg7MdPpdBSyPdEM6vcIeSpnZ3b/mqtsrns7OKkhsrro+l0rtASVAFtnZlpwm8LT3CznnK2618OpnVFKx7BoFYlJRhEG6q8MOff0qQYc/8i3DnMKAYiylaDcOf+RSx4d3YrZizGxv7KZsEh9LVssw6Jo85CnZS0rXBucXUsZEdLLb0KQUsw6LeFLGPRY/JSGnyi5aFFYgopiNkwo5BuFvNpwReyIU8f5UGF4Un4VYZSWHp1Wy2jYdmJGlt8CloyvC/uoWU+tshWtyh2Quj10arEjMfTlouAoHhy3Yo2vNiNFVqMHlzZ2Pbkd3XXHJnLFkuzBOHXK0XYTERlbNZ3ZMMBqQc+cZe668ocuEgh/ZqTX2RinfELbohHZcMnc0YHxJPIvo66cMRZGrCoHbIG6OU5Zc6KFzXZkQ8wmDbhqCMTyGwYtKmcHtDZE0jREcrLW9lbFJ0b2tzOUbZR6StCS0jCMqotoxzTnLrfNaDGSyrlkjnLRFLB8JJQmFjTZWMmXP8R0jpMMe8DZcxhUzo6+LJ1NivRaqniqaCeF3xt3Xl0D3RVGVhs5j9CrzLdzJQTudnkYbIamsgo4Ax9ID7gKeCetq6VrmSAX7ouUSwiqaxzuqcimFLiEM8L+TBI1/RNRVM7Y/xhp3WtUUlPynNYyxWfTm7HxzMEZZfZZ5FAGM0lJJ+Lm09lSxHiKrqmkQTsgiPxW1XK4jiRqaiRjHWsbLMkNnG5WeeRTpqqUxjnT1nPv7rAk5ZLiyN2p3Sjk5bNHiyFjzK/IxtysTsydKFC4RvzmHmexWpT1EsbMzKEarGklljcReyNuMYjC0Rx1Ry9iFGqdFRRYbI909bUCB3RhKpYlBAai9DO2ZhF9OizqbEZHSCSoyuI6kLqMNpuH8bhcyetfHU3tkabBOTNOVklERLD6gips8jcxat2v4TgoKsQTGRwebNeTYLQk4Mro4nSNqon5emWyerPFx+IzhzGsY0AjqFmmV42fqtGqbTue7PI4EG1gFRkbGBojYBzXeopi535gEsxWhSRy5Dy8OdMCNbjVBnB+tsw+iF2douVoRYVPOXzQU/KtrlOio1EhvyzuEEOdx32ThwGwKJkBc0vb06K9hooGSWrSWdgrxEYp3TQ5Yo5ru76qmYp2utp+q3qiR7I3SUMpEQGgJ1WBJI55zHdKE5p35QeW4/JRFhad8oUYmmB0mcB2BRNZnID3WHdaE1PUyQPzslcfquihq8NrKcMfBHI/u5q58U8fwSBSMJiFmGy52Np2GxRMLoGU/0KwoqySnqHPfEx3a6aR9Rf8AbfwQudr5t0sDV1bqiYyCFsd+jU0VW+I5hce9k5yP9LUILCbO2WbGlLisc0Pmjbm7hU2TB5tzlGQ14ysblRNpJMue7f1TEcwZWSsEjz5kDXa5lC05gr4jj8IT1C2Kr5HZxl6oyTHdjmjVR5HFucDRTRxGVgysFgjQWF9g1qU9w0e6lZBJ0BKKpgL7AMIt3UZDTUxMBZI35LRw6tMEQpyxjwza/RQRSvyBj+iFrAycuHXdB3FJxhiNHSskbijXcvTkAW09l6DgGMw47h4qovVexHULxijqWU7w+SAStbuFs4VxPVYVO+fDQIhIbyMOoK1Ekw9bIGbI7bqFm4vw1hGMxWdAIJR6ZWbgrK4f4vfi7hBXQMil/O06FdIZMvpXVxctHi+OcGTx0mMReKw4mzZWDUD/AJK6ulxClxCBlTSSiRjxe4UVQYayB1LVwiVj9wVxmI01ZwdVNrsNnlfRTnVjhcDbRB3md3ZLOqGHYlBi9IKuik0O4O4KmcZGetaVdErI2ukebBgus+hjfiFUcXqmOsw2iYRYAfJG2rph5Zn3H5RuUTKuSWMx0sJItbM7QLnMDmeK6nEo8EkdHISK2fljs0ey4nh3DajDuL6WlqG2LH6HuNF6RxBQEYZSjNfl1Od3sFDW0UEWIVGPGMPdFB5SehCU0437VsS5tVHhnho/IeZzOu2y8/Is3TqtDHMSq8XxOWtq35nk2v7BQQPjhPOkgEtvhOy5tp4KjFJaVlKJjkBs1g910UuHYlhop4MUkilZUt0YNS35rNNbSzNhmYxsbmG5aBsuhxHFKbHKmly0cr4qSIZiNQT9EsdFg/HcfDuDNwaqpJaupAIp2Qa6dibaLLxuuraqOkqJ44qPFJb5WjcsPdcxUYnU4PiMeLUVO0tsQRIFDR4xiGMcS0NdiBuJJeW22wv0QdlwjQSCabHZjZ0Y5PLiFy73uuNxmjMWP1fPhdH+ITZ24B2XbT4pBwriVdDI1+VwMsLRfU/8lcPX1NbXmpxSSCQ812Zxtt7KSIYah8dQJGC1l2nDPE+FYA51U+nlc948xYL2K4ukpua3NnP1VqklewZ6eUsfcjRc+TS1xTFFVmXiOd8sM1ZUXhY7bInrK+pMkMjpA95jA0O6lfgWLcSOhzVEPIi9TnHUD5fRXThmG4PxBRfddO+eONuR19QT3UiWZh1/2ZRUsmET1NRGDNziCTuNFYZWT0MWNVdOQH2uLi4urFBg9bTtzUsfkm8wHRvyWjQ4RBM+eOScysDrOa4aErtEJTxGSgxeaRxfSSN5pvzHCwKevqqd7Im048sW4916Xx1G2amNbHIAKQ5RGfWRZecV+HwtoYqinYRzBcgm9lzyahgGZznukccxt1QRVD2kMjeRfspqqmlYx3Lgy5RdxJ0Wa06KI22y8xuYvAFrXWbPEyOocXue5gNrk6oIza+mn8ED8xf5lqBfhMEj2QQm5ebXvspwOVI6nyAGPt1WXdkbhZ1ik+eaKUSskNz3SYPRoulc1xcwkH2VeQSVBLXzOBed1UfVyySZ8++4Rtqhez5LN+S1AtYnHh0DWNpPxHBpEjjoXHTVUJB5Yx2apZIy5gkIs1/p9woyLqtCOQyAtb5bJ3wsltnfky7Jqc2lDt7C6eSGedxmkfq7oiBky3t0XeYHAKThnDoGamoHN/VcOIA2NrpHjXsu9wurpMRwei5VZHTywR8u0gsLjsue3+F6umyjDK5TOZJBbnxyR32zNIv+qkgb4iVkLNyQteDG8fjZy6ukgxGm2Jc3Pp2uFVxbEaWokmxGgw7wbKenIItYF/debXh+Z9T8TFXC/HH9/wCJTYdi0ccbWNZbIbCUI6vD4afiKmZRQiM09EcpHTXRcJFi9UKmnxmGQMli3JNswTVnGuI1GISYi+UQSmIQxZdQ0L6+O7Gqfn9mP5pKsxllPIa2SPnVTJH3NtXPWZRVc9RLJNVRxl8u5sqT5rHLbc5yTuSkypkjlFl5sslhqTyU73l/KjkcW5DmF1kVAeZHuyHkxu8xA0HzVmnqpHVTZA0XMnLNxe5KlxCOfC66swp3LlbJbmEbey52BFfUWzMks8+26txVOKVEU8MFVO1j2/itbJYO+azHte5rXxjy3uFPPVsgihjh0mb55D/cstQeKmvSgMm5RjJBaTY7pQ07mUclQYgWRGziUD5KWZrn2tLId/ZK8z4nwDmFklswb1S27VZcj7Z2Zm31F9wu/wCHOIcSwbAYWcOVrHB8hvRuiMhLz77hcbTUzjIynbCRm2BXf/ZtiOGQRVmHcwQPjkzkPNrnXVdcZcsvzOiwziXiPG7wyYGcHyHJUTv1LR2YO62KZmH4K3mRmeaZ+7/VLIViScVUMlU+rgrQ6hiiPNPKJF+/yU1Ji01dSvkw7iGnlBNw1kN3NHbddqZiHKcZ47xDj2KuwikpKzB4mQPlqWTyZZJmDXUDS3+axcOwnEKSkpMUmxWWCu1qIoXym4jGx9l3tRXcMMfFLLV2rqMGNz6p3nt1F9iFyPFmLYZHVyVmG1cNXLO0B7x5g0WAAA/VccmoTVPHEmJz0ba3DYTMwAtfHGCZZdh/enx3CcSwmixbEqvD6AOLQZZxCRmJIFm3Ou65zh6hlx+rgbTzilko2l5meNL7i3/PRLiWrxGd7qKs4hkr4dzldcA9FnirPoMdfHPRMPPdFSSEjNrkBFirL30uI1gjgM0NH53tklbq3/JYpIh1h27rp+AMOwriDGjg+L808+M8pwksGkb/AD6LeKOWqi01MvJfmiv+GTuQjljeykgJhjbnD/M03LjputjGcJxnA5YafFMHn5URfFHI6PfVRYY+ngqBVl8LhHne2GSLMLfLooO1oMWpqevo8WdhVXVUT6GJ7m5CGExgC/uAVQ404/ruInU9J4F1JFB+JlcLFz+h+QU/DX2kcnB5MAqsBhnYI5GAtn5Zs/oBZS41htLxHgeFYs2aOGrqmTmQSDTJHp6hta38UHH/AHrVes11WJRs/nHT6IqSeoLRVCodlErGOL3XBJ62VZkLI3yZyyRmTRw2VnDqMz0YZTxtzvcbPGhWc4FviSSKmr4qSlyWYGF2SOwBPT3V7jTg1uB49Jh+G4jLMw0rKmXPp5zfQfosGrpqmWaQVXO8WTbKRcm3Za1VSYhidQyoxHFSa8xhhiqGeodr7bWWKHTYHxvQYngFDgeOU8jJcOlilp6iElxlIJvp9bLrPs4wKnZV4xVSzSVcI5dNC2cXBj9XX5ri+CMNfhvEME+IkUjYAQZmeZhJtYWt/wA2XrrZreaGRpb3AXSBaNOwUvg3tDoW6BjhoBfQLyjjngPiPGMYOI00LjTz1B5UQvI+MPtd5toBfVeoc+T8yeOR7jqqPL8b4XrcJqcLrsVdG4mMwsjadAWi9/47LG4q4RrcOwyGvjmLpaybl8uFujgRe4I+a7z7RHXfhrPyPkP8AulpoInYfQmSNpMUTCwncHINlKGZwbhDMN4Yw5z6U01TNTjnt6uPd3crcDASn3CTdlRdpMkYAV2LD4ZnZj/cstszARqr1JWgGxcjS8MLpfyJ3UjOgRirFvUhNTHbLmUZQyRM1VOU2crcsumioynzIGJ7oX66JiVFnOZIDOiN0JBarAIePdA8fCrCwjz9yudqo3zYg2OtfFyKW7iXP1IO3+K2a6SOGmc98mU7N06ri+KaPEpYI5mwyZrh5fbVvv8AJEczj+F4VHWMxOirpamKTO+dua7gd9lXxPEIa2GGKOZsfKjBJdubLpcSwaGmocNxWegnzCPJVeU2cwj/AJP1XMYxRUssQNDJGx0oLI8o1aNNCFgUcRZGyKjnEcjJmNeyUuN9tlDFIZImwioMcRdkmDe3SwUkFFixidzwRzByW5iSy/ueioSUjqaOnZUfhudISXb6dFuJHT1dBUQwVM+FSGNsAZJk3fKSdj2XP4ZU1cOK+OngJERebHQX3W3w/VS4bhtbNyzDDcDxDtSRba3Xf+K5+lqHmGcsk8r9bW2JKrStVvzfjzxgPlc9xcNiCdFcy8jDmtfEC+XzRl24WZPFP5SdWl/LDuhK0qiin+8RSxte6a9m3NxZZZakOJ1UuGfdsUIiYz9tdt3uvqDZZmHYlHBUeIfUSR8qIsjLRexK2nUdZXUvhHwcmqeY4AWnS7zbdXuJGx4o77gwRkUdNFGCCyKxc+PR47m5VE8GLsq8INXLGXMjj/GkLcplI2A79U0XEuH1OHeAhoDGxljO539YwX0t87LGno8Vwqqw+qxKCSKjqXCMQyHQ22BH1utfGsGpmSV9RT1DZnztYywHpOlz8t/kisaXGX1gEdPG+kbmtaOU2J7r0PgnEJJKeSicfwqYnmSOkvcnovOoKSKCgZVQEyvppwZGhps4v2/uK1cFrJBg1bDGZGOrZtMhF2/82RHQxO+88YkqK7zUtBrFn0BJ0vZdFBXU9Y4tppM1tzY2UOA8GOpcFbJznOqqmMEukOgPRbVHgVLSRcuPQ3JPzQUxGTsjELz0V8UYbsibCQiqjYHqVjZG/CrrW38pUgg0URAwEbqw0kbFM6F4QuEg9KrQnyvvuibM0N1VbM/qmJf2QKSQZr3UkctlWdmRg3VRYkkBTNmsVD0+qH/ZUFl0xeLXVeTfMnsUzg0jVEKMuFjdWI5iOqp6gp+cGizkVpCe40co5TcKpHUNsnNS38yKBwG6EDVJ0g/MmDh3UZSgWSykoA9SNk1QB4V5+FL7vJ6K21/dWmOjMeVBly4fBlADbqs7DWE+SQAdlveGjOyjko2AXaq0512Gm/UqaLBWObme9abo7H0qxDEzLqERktwJg9LkJwsN6LdAjsmJiKiME0DgfSU4oStwNjcU/IjPRUZIow3q5GIbbX+quyU7m9FGIyeilitynWvZOHuZupntcFG5oVEkMvm1Vy4c3RZmaxU0E5HlcUFieBr4jlCz3QBvRbDTmYqk0TLltlGlFjLKZkpao3HIcqjLiSrKLrKoBu+qz8QmfUSW1OXsiBKJrA0oij4adxsGEfNI0s43YtEmyYvugy30ryLOFlE2gMnULSkaXHyoBGWnVqDk+K6J9LFTzEjVxZb/AJ+S50tu1dzxhAyTAzO71RSAj+5cS30I0jfo3bUIGa+bdSvAuogMjygscPSvjxnlg+V7TdSVjOXWzs/7wn9VSw48vGoW/nNlqYwzLikvuAUFJ3rtZETrZM5t7P8AiCTjtp1UgyNJ6XLsOEIBV4Q/S/LlsuQIztyrtfs7/EoK6H8kgP8ABWEbcVBAwehTCjiPqYmmbUxOzM1ambVv6uWrZof3fB+T+CPwUBZk5VvkovGHujjqSfiRQ/dsY2af1QHDYS67oQT7q5zQfiS5wCqWFjMthkUocAdQg5rUOcOOZqwqzzW2yDZMxjS7NlTRtG5Kk8rBdA5IazQKF5c4ehM6dn5lGagd0EcgLXZkxkuET5mO9SA5SdFqBLEbPCuvdG+HJZZ4NlO2RpZZzlAIhDXqwJH5cl+ihaLO3upgNEmSgGNn5VA+IE3yq2LX1RFsJ9RVsmGU5tjZOCVpSxwWsAqb6Q6litpSIgO9KjMeqltk9ShdJ5tFhRBpHdGGOveydkjVKHgq8g8TYgPOmnEX9W7dERdDkvsU5CBkDA7PmKUrWHdPIXtNkOYHdOQqVDqppdHTOszuvNMXgmo8RnikFnZtl6mHwu05i4DjikbT4mySM6SCy1Es8XS8N1LKnCmPZ/V7rSm5csfk3aub4DqXuhnpCfMDe36rpHMaHOaxhBHUhRpl1o8llmRxZn5C9oB3BNirGMeKqmSwMAjeB5SAuYwmvlkkc6Z5c69rlBVxbharlxdz8HpeZG/Xe1ynqeHMUDAyqw8B37jr2XYUbXRRtmqH2anrMUwencJ5KkhvsqtOEq31+HUpZJTQGKxGrNVgMeWr0jiCkwfGMKNVSvzhhudOi8/bTQyScuCRg7ArnSUeAXeBJD9XjRaxwd9VDaCWjzHs7VZlZUTyXiqHXCjzyU0AdTMLe91riqvNHPRzujlFnAoPFyxPEkb8rh1AUL5Jp3l8kxJ9yo3m3lCxMJK+/GMQmIz4jMfqpnY1jIg5P3jNk7XWbEGdVo0MlK4lkzrKwKWdyE6q7PSsklPI1HdBNhdVDEZyDlG91RQDp4ncxjdRsunwLjV9C0x4jRZ26ZHM3G656PIR5yrcU8MTMvJY75oOmq8Xo8djfTUMjYHP3z6G65Ktwx+HvAfNzL7G6jldGHl/L3UUsr5n3JQaVBG98d42ZtVXrIHxynOMp6hNSTyQG7JLeyGeqkknN3aIInTnZriB8lEW5v6wK3FLTwsJ55B6jIoOdAXZ3NB+iAHQtY0PzhJsv74Cjkla4+XZR5rKxIlcX/6w/Qp+doBcn5qLmNG6nMHkzhZoNHUEnK5EZWX1sFCG6pOA6tShbjazpIEJiDnZrqqHWRc54ShM/wAuyYZ7aq5g9IzEHlkht7q1XUMdJmjD4zbrdShwkcMoePIjqZLXjutJkHh6XxNUeXKdm9Vnztie3mAG/UrQGB3kMPdW4YKqCzOQ4g9Qq0NM6SMyM6LqGxc+ljfF5nNFrIMfmcr4rJGpz9brdioqKNgfiUcUd+2qPGoWRUUfhaKKOG4Ae8an5LI585suezQonlt7qV/LDrFwPyQmFjjcK0FGRbNdTskzelwt3CiFK9o6fqrcFLZtyQFzlYB+O39jJIPqvTODMWqa/D+XXPGeO9rC2i8/jEAItr8lpYZUPbMIIZnR9gOquOTnMPUXD4kL4IKiEwVMLJGP6Fc3nq8OtNOJLdnFdLBVQ+FE+hZ3IXrjJlxeL4dX8FVrMQwol9FJbOwG4PzXXYRNS45Sipjq5nj42NNgCqlRjbKiU4dRUIrXSeQ5tGAriKvCsW4ar2eOMsFLUH8Q08mluy6D0OerwfDyedVRB35Qcz/0UI4kw4W8LBV1De7ITYKfDqPBhSsqcOga4SC+d2pV2N1vhb+gWBShx3A8VvQvkeHP3jkaWkrhftJ4hkgmdw7QG0QsXP6kaaLo+O8WwahouTimGzyveLwzxAANfp1Xj8pNTMZnvJJ6uNysy6ALLC56p3RWizlSOYXtY1nQ3VptHJXV8FBF/WO1XKZbbf2Y4DDj2NzQVUV6ZkXnPUX2XYYhwni/BwlOG5cRopXcwxv0LfYKHA2YJwlxXHDBLHAx9KOaS7S/v+q67iySoxGhfSUjw1hjMjpewGuiSxLxjHp5sUrJZJA2BnoEYPZWcNo5IsLEzB54Jg9pU2GUlPNgVdXSfiS0ZJt0Klgr4WNp614vTSHlytH+C5w27Z8uHYjVwSMfC+pfAOWSRdp7riMVElHW1NAasT3dncWtAAPZPxmZaLEYK2kmMUpaA3LvbpqtGfhGeTDopzWkzVdjoLlx+a058XGTzBrzHGCGjsVLQSecZh5R1XQ4zg9FReCwXOObK78Vw1JWXieHxYTWy0MJc5jNy7crEw3DRoqpjsRbhzXiPmt8pvrf/FddUYP+DBDS5g9nkcAdwsN9F43hqKqw+ijieN3v9drrco+JcPhwqJk9VkrY25CbXWsWZW58UrsDEDWZncrdriSbDv8AqnHFtFHS1TI5jJPUuLw1gvy/YrlxxBiM1HJMYS9z7sMxGgVN1ZNhGECrZyzNK7y9dFqVTY9jUtTWA5/M2MDIf8Vi1VZPJG2F4axgOgAVWerdM7m57udugkqs7Wi2y45fmaiR1r4ZaY08hs4rAlpeS3ltbt1VyslvKfNsqwmY+/OzHtlW8TJEZOWwWBtdOZrxZAdDugc5mbI70qIva1thuuiHcblJrnZw86lB/aUgy5S3uoBJ3sNSjjDmObIz1DXVDYZtEjLY26KiXJM5xneQe9k4sTl6J4WmSJ2vpUYsG3JQXoI3Uk8czHG1rOsOiCVzDO524KeGalfCS55a4Czeyr57Ny7nuguwUD6qaKCNmeRjs4Z0K7bAMPjnaOfAwZSQWgaAri8MqYm1BZMZGZxYFm916Jw+wMgj94y8/NKRhcY4g/CK+OLDah9O4x53ZCRfXqsWo4qxauo3U1TVj5iMXcg4xmNVxFUi1sgDd+wWJFma9ZprxFoSSxwRsk6G4UbZY84dM247J521F88rJA0jyl3VHSUFVVStZDDme4Eht97LNMzPI8L4ZZix8fMJ9OtrFWmRwZ3g6WYVZwqkoKKlrK/EYOZy3ZBGRc6qkx0ZZJPz49QTlJ1C2QRihDXMjd6zdC5rIfM+QF53PVO2WhmgmJY5kzz+ED2HVRVEEkZbzGWvt7hSg738yPlsO3p9lHUfgG77FxHQ7pi+Nh/ZuzdHAaI6ksfEIc4e4u3HRJhpaqY2UVPCxrAZpRfXopJZ4X0tK9gtOTZzOotsqtW3mSg5icgtqbo6OeMNMT448wF2i24WJgWpI31cgfDdtQz9mL6XNuqqx0WIc2aPwv40ZJvm0JW81sda2KOm5dPG8Wk7gq3JRZaiofJSCFnJDMpP7UjUmysJDPnrK2LDThlbVRl0hDLt1fGzqw/P/BSyY3jOGN8KysjisAWBkIDyD7/RMammgwkVUtPFUyyTmPI42LR9Vk1VS+sr4Z2RSNZHuHOBt3W7bp1eFUtHikBxGsj5jIxzJGzHQPB/iuc4lo8IdjL6qgqI4IZACGM2z9QB0Vuhpq2tkbSsjliiqCSJCLBwGvXdZNdLTnwcIkzOiDy59tSb7JDlSTxFU2idSwTl0DBeRsfQdyrmDUmFYlRVbJIZG1LIbRgOIBkJsCtDhCgwfFRWwVGIRQc10fMBdYCK932PusjiGlw3Ca/l4TiMNVHmeS6BxsBfQFYVjSST0s76eeMZ4zZwPQrpODKPCq/Eia7Fzhk8BvTzxt1vrfqsKKhNbBXYjNMc0DmaW1df3XpX2O4JCx1ViNXJRytqYeXFGXAyWv59D9FuFhVxTDsZ4rwqtxOu4nmlbg7gAxzbgxnd+nXTX5LgRFZ5D45cv9WQLA/5L3TCMAiwzh3HaSujyRVEshezry7GwBHcLy2bh3HX0WJzMa59Dhdo5C/TKLAgfMXCIzZOGcfbgP8AOYYd/QASzntI0INjp2v1XRcCcZSUjocCxDw7qeUPhY+/nBff+F163w5SU2MfZ/DhOL0jYGVdOYpWAWPa/wA14DxRw7R8OY/X4VhtQZ4KK2Z7twfdB0GKYfS0nENNw34UZI6ci4uOY9+tzf5LJqsDqMPxDw1JBLI1kYtKDdhvuqTsfr6ySJ9bUyFzI+VzgPxCzsSrE1dVMw2JslceZCQyPIbadljIX8MY2PEYa+omzDVjXXvZ5FgsqQ14rXUtWBJaUgykaX7qegmYJ44KbU1Lo2HNqASdwujq8OosHfXvqaSczUcFjzn6Olf8Y/56qUMmnZNTQubHisr2yuD5GA6XBuF69gnGGDYuI6USZa+wEseXUleQNl5+GSCCPNKZzLZrbHlsA1P6rQ4FZjbuJaSOlpJHEu8QedoOXsX/AMVYHtLZA8aX+RGoUgd3CJ2QjM1qjuLrsOP+0FwNdhrO0Jf/ABXX0zgKOD/wY/7guJ47dmxuhZ2pz/eV2ULv6NAO0TB/BWi1nPYKN04CC5IQGO6yJBOCVKyot1sqJa4dUg54+JUttw1t2jVTeIDzq5YbZCzZSsqnfmWJhbbfOFtXKtJICdHKp4ltt0PNupSrQdmKPJdVo3aqwJdFQxDmuQueboyc4UMnkaXnYC5QO5gdusqtp694nrZyBynZ4rP0I7FR0vFGFTueBVizbWJ0uq+M4pSYrDJgTZ3g1Ed454teW8ahGWfiGJ43WV9PTRxNEDHBkjGyZg49/pZT41w7gcszpZqLLVMJH4AykjqT3XJcEYpPglRmxGoLg+pjhyucLxkE5/7xqvTp6OlrHcy+a/xsKg4jhihw6WkxLC63EROKkWa0gggi9j/bC4/EaTEGxyQSQ8vEINJI3m4ItYFi9hjwuhpp+THRRmNw8zXC5ce91wP2mU33JiGC19BNI58biMj9gwWdk976pA5Woq+VgNPQsdI7xBHNG5aGaFTYjw3Ph1KytLMsUpYRCx1w5mhP8FZqOIMGxPFy+PmMhlaSc7bWPb+G6mr8eqKylpDWvjibFA8DI3z9r2uqrkcRqWeAhw9m1NOXtd1N9l1dPgn3ZwjW8RVVaJZ5msja06EWeNvdc/4eGbGsKp3t80kkYkPcX0H6LoMeqH4bOKSSokEVO7JmLLxtDz1+l0Rq8L4FPVVreJMRmjip3QisEJ1yjUMBPyBPtcKMup8Sx+Wr4brTHTXMldVPb5IgddCd76qlT4NiWIUsbMKknYypuJWk3Ecd7B/1/wAFVw/DX4bV1tI6fnUcUIlnlabA9tOp9lBsTSff7zTYFGZuXGXz1M8tzYfH7Kh4l9HOGCOpzMlOaR7LAn2XQ8G49hzcEr8OpYTzBCOZPO0NFzpe4WxheGcOSwnFPvSKvqaa93A6X+RVaYcFfSYXRSsOHuF3HUm75CuZr2TurT42SGlfOc8bI2k3B2f7r0mqo58QqqGmhjjZVROfJK8bRC3X9f4KSn4Hwjx8eJyVFXPVQON3TO0dpbboto5N2GcWsyvZxDOaN7mBzgC0uFh0/hqvTaebPSwl3qLdSqlZiuH4cTRcxkThvmGg+ZXIit4mmxOprsOLqyjcABNL5YwL9AP+dlhXdMddSBlis+LEqA1RoY6uOWZguWNNyjpKiq/E8SLeby/JbRfAsVZY24VATuLt1YZK4DRYVaygqGUC9ik2TXVO8tJzZlBVe0BLQ+lyORjiqzgWKoUgFyojbqiLrjVRO9SIlaVagsXbKipWTctBdkbe7VWdC+1rapNqgTqiNQ3ugoyRSAkOaoXBw3VuadhKrSSXVVDIHHrZMC9vxuKcnqUOYqIMSu6lSRyjoVXLwgzC/lKDRzXGiUcrQfMVSZLY7lHzLndFaDZmEbqaOfVZ8d9Fah3UpGjFOpXz6KmC0DdPzG/mRo0jgTmzJGpY0bqvJKy1syrvNwqi394AIDW6+pUTIxMPN5kF5tYQb3U8db5t1nI27ojW5+ZvmchM0Y6qi2SS2hQSCZwKQsJqmqY5vlchimbI3yql4eY7qeJrohYqiVwsUm3BUZkulzD2URbjncAhlmcTe6q82yYzXQHIeqjDkJeCh1KCw1zLoiQbkKmXFvlKKKaxQO5zwUPMcpn2eMoUEsZaMygIT2R85jhqqLi71IOY/t/BAuJcsnD1W1vwAH+K4EbLuMScZcJrIT1iK4dvmYHKhpHMjGeTQIGlko5jNul1OQ12jhoo7BgIYAPkoKrHcvEKab/vAFucRC2JRP7x2/isKp/CDZst+W4PstvG5WTQ0dW3XMCL99kaVtCPohu3ZBzT8Si5hLzqq2mJytXYfZVL/TsQpSPVHnuuKlfZq6P7OKzwvEpjcfLPEQVGXpksLSCCNFXDYIvK2ED3U0tQC46qCR7XDRWGKZ2MQSROE9PGXA726KjFWPHlebFdDDUZW2kKyMUw2+aqpWeXt1XbFzyQGsd0epqeZ8u5UVHHBVQctwvKtCGhETLNWsmQofOPSVOadwTtjPxLjbsCJ0xNgpZROGenRGGgHQKQtLm2asjKMr75VJTRSTutqPdGKQsnBePKtWMMyjRBX+6HkB3iR9ULqB0QOYg/JWi57TZJ8vlIsgyZpTG7KnhqmndyjrQb6KiZHNK0NlkzDup2yA7LHZUNGxU7apzVJgX5H6KAPsdTZV3VZ7qB9TmJUGmTYX5w/VQms5fxrPdOT8SjeXuWxbnqhICWOVcSk9VWe8jZDzXj4kF5kru6sxSPKyecImmRxsBuuerftKjpJjDhtEJsmhe8kC/ssDvhnUkZNvMQfkvHcW+0DiTEm8uOeKmYekOh/Vc9HPXxOD2YjMHD/vSg9/qBnbZUnnJoV5Rh32h8T4ZKDUVT6uIfBId/qvQuGuJ6DiZjuXC+OZguWk3/AOdkFzlgTZxdYPHNIZaWKsZvGQF0ro7FZPEsL6jCZgPgF1YHOcCVkdLjbROfLILL0iYQfEF4xh1V4eeCcmwY/VesQ1UdZTxzxyNLXt6FdMhRxRjGQvnjYGlnYLyeasz8QB8el5dhsvXsSic6jlt6bWJ7Ly92G0+HYk2orXtkhkduWrA3MZ4mxKjoRkoIo9bZgSuPFdNWTfjOu49Vc4pxCKWVsdBVcymIB7rOwkMNYySRxDR2F1kt1VTikFFgrqamZd2UjK4LipZ5JHl4jDb9l1NVUYWXZ55jl9tFQqJsEiY6Sne8uPtcJI52V8w85BPyQOqprWzlbFZRNij57pY3MPRix5o4y7NGXAHoVLEPMJ0TbHUInC2+yB3JtZhdfrdUTcy3pUoxGcRiIBoA621VLPrZEglFRM12dkjg49ij8RVSCz5nEdiVWBN0Qf3cipC+w9WqkbKLA7d1A6RgZ5dSgbJpqokrRew9QVGS4H1j6qHmtRseJDY6qA2y3OXMEUht73ULoWjUEocxHUqwQktmKGSIsGY2QZ3DZG0STbW+pVEYBOymFPMW3sjZTct7eYRYnotZhgbGGteb+wWJniMeAOEgdGAXDutOWSWohyGKNvuApHPaNmNH0Sa67bFc/EGW2B+axsEE8D2G9s3yWtO6ExZY2DMqjncmIzWDgteIM7UbtP6J83sf91Wo8Xe7SNjPqFXkcC4vO5K1E8g0U80JvG8tKeSokdfM+5KiIcfS24RMikP9WVsSR0wxbDJMSqzIXgXY1mzVXwugrZ6eSePl5WfmG69JixPAp8OkFJAC3L6BFb+K46mpKunwyqmY3K03s3qFviObje+7hpofh2Th0jtDNIB2Bsp6ajkfGZOW+zj0CruZlkcMjgfcLA0sEBildUPeH8sXDZDe5XQyY8yo/BfRMjDurxouPzPDQ0GyuRztmH49ZJdBbrsVkE7stPTub0IG6qU0JlF1abPHA3I2z/mFGJw06NWMj3jJaz8N4H0TksNrG2ii5jXFNJIGC5XHiJmkAq3hs7KaujqJPSy91kPqwPKGuLjtYIedVSNyOZlCnEd7iHENPKz8ar5zn+mNjFo8GYNXYowVeJYoYKe9m0jTqfn/AAXn+FVMeHVbajl5n93G67fBiyOqE+dw5jtCDZdMcmHpcNDSU8eSlp42N+SpYlgNFjFI+irY9H7OG4KkifXAtjLwlX0dfVxch1VLC07mM2P6r069iTDz2jxocBYjUYHj00jaa2eE2vp8vdV8S+1qhhlDMPwuSpZ+aQ5b/RbHFn2dUb8NfiNHJVS1kXne6SXMXD5LlsTrsAxLhiQ1PLjxGLyCPY3uuvK2KUeM+Pf5zUcNDT0skEQ88gc6+umn8FyofYKJwDTa6V7dVznJ2iFlszg0WNiTZdzhWBfcNO7HKqo5svLvcbf86rgIWvncI42Fx7AK+cTxR8DaGqq3yUoOsd7XHa6xHmsrtdJW+DmxatgkDquTR1tAey7rDOKWT8AyzSP5D4wYHG5dcFcFxLxCzE4IKGlpTBTU7QwAuvdbs2AYtScEZJZoRHI5jjGBZ5+atObk4Jq58c+G0NVK+KYlhANg4e6s4b4iqmosEkIijfNrYahdrwhw79xiSongjlzx6uabkXG1v+dlcw7AIcQx9kdPTshEwvI8x3kaEmGuRcd4bSxYAycReeGRjGuvqArmGYnMcGpuTTCZ1PD6idAVtYzwpBiGBS4RTTud4dxlD5jc3Hc/VZfDWG0zeGXQYdrP+zLet79FmYSIcp92T1L5MbrwJZ/WADYXSw/C2xSnE8TqGgk35drldqzgiqOGPmxCvMQYL5Wi2q4urwurMUj3yRugYbZr6uUmXSFJ2J1RBZz5o6e5PLOypS1Ekkboifw37uG6uTR0kUfhjF5x7rPqmtha5sY/DFrH36rMSkw1MHraYQzYW9rnh4JLnH/BZNNWPFWx72yPiiuwt3Cgp5JWyPmhkyuDd1pQcNVbaN1dWz8iI3cR1+q3M2tKEr4jI58LS2MnQdlDI5oBy7gKVtVA2NuWINAGp3uq0ssbi541DjouVLEsxxNydbne6C5VpsTXO6XJ0uidScpzg8fJdIRRcHkDTUmyjuWus5uoW3SQwRMc4jz2uD2WfPAzIJtLnsFRCI7xc47XQg6oANbJ22DvMLopHdC4FGbE+XZNp7qokjlfHoDodwmdqbo4mtkIYUpm5Gj52RTQnyuHsnaLR+Y+bsnZFJk8sb/mQp4YbRukmka22wJ3RGvDNDPUxwRG7eX6V32EM/Cjd2Fl55gVLM6tjnazybXXoVuThs37sT/7lEeYYpO+fF6mqf8A1kpse4UVO29Qx59LDchKoMxZBzJMzA02bba5UsPKjw90mWznyb90Fo13icNloKjXlvMjHX6/JQRzSNaJc5AjbY5dys90zwTZ260aeV1JGWysB8THYE7j5LMwKcskcluXcW631KenZJz2Z4RKAb5Ds72So6KatqvCwOAO93GwAW5hWC0pwqqxirrY4zTufExjhYAjaxVghVxLEBW1IdyOS1gsG9VGTAJI+e/KxhuFRmJkooZsrxLISS87EDsjmimjjhmqTZsrPL303VF2UwMja+GAwlxIyDQSDuoBMHSEsykkWHltYpSMnkYWs5riwXzO6KowSNDi02+Ski8PwKeUVBYXyG4sfSoGsqmuFQ9g8+xGl0bAyqgFK4fjbMf1JViZr5HR0r6bVgEZLUkhs4FBAcOqK+qhE/JIJYw7n3ueiz6ytrcPxQ1dRTt5vQONwWKT7tiiaadtVJn/AC9FehD/AAVLPJYSc4wF5IzknbRcyGJW1NPLK6oYMmcAke6h8RHDJG/9ppeRu2i6qjwnDqzCcYNRThopM7zP1B6D69lxkEfPqI6fNbmOyX7LUS1yercD1lbi3DFTh5YXPppDFC6fS7CLjXovMK+nfR1c1JJ+2gkLH9rrdwbFMUl4mho4MWiaTMBEXOywuIGgKu4zwXxI6vrsUxGha4mQyvEMgyOZbWx+nZdZYlyVNPNDd18p6H3W/BxCyKAsbSQP58PImzNuZPf2U+PcDYphGGvxHlCSmZEycvvrHn+A9/msmmyOlbDNmpmvyZdNQb9ViSHaYDwL4zCMTNbLaScs5TYzq0gA2Pe912WFcF8O0TYJ2UpjqIvPnjkLSH9e6v0cb3xCNoJMbWMLyPVpurFVVUWGUUtfiNQ2GCAXc4qss/ivwtXRSOfX8pgBMUBfYTSA7+++yq1GFMpsJxHAsQcGU+MNErZmaPjIYC+/ysNV53xjxTT8QYsJMDbKIWQmCMyCxvuSAu24+nxTGeES/A54hQQQMlqpw6/MAFuWPrqVW4lX4M+0KDD+GatmO1xkmpHSRUrhHcSADQX7ryiaSqh58dRfPUOzyFxuXBaIhxGLhx1DW4aY6Z9QyVtQN2vIH94WxwBgFLxLjkc8skDfCSESNkksZn2uLIri81rM9ltcOPghlmmq4IpocoDg+Qtt7jv1/Vdjx59n8GGYbWY5zZDUGXPI5htDCDYBgHW+nyXD4Jg82PVtLhEMoidVScsPIuAfkivUuEuDKCarp+IBPT1MEf4kTINuZ006WXQ8V4bS4phzpq2MHwzHvaLftHmwF0fC0NRFgVMyuphFWRjkTENA5mTQHT2WsGh1s+tiCiOa+z/gPDocRxCoxCjaDBTw0pi3Ej3sDnvPvfTRdpHgGG0tV4ykpmwvZEyCPJ8MY6BPSO5Ls7NC/wBfurvO6rMwjNqGEE3UGUk2yrSkMEw87tVE9kDR5VsefcdxcriSjjve1GD/ABeu0ijtDH/4Y/uXGccuz8U0/tRsH8XLuSQ1kY7Rj+5URCN4GyFwLR5mqdsjeqjrabxcYaw2cDcapDCuXoQfdVnGoiNpo3MHc7KRkxcA0OXXizyTkaJtAU9yAoy65XHKHRLmd2Rtc4lV8xQmR7T5SVlWiw2ViLVctjXF+HcP8oVzpJHz+lkYube642t+1XGXSVTKWnhjpnjLC1wIfGO9xujT02s4l4cw6XkVeM08Ut7FpcNEp8ZwecupYMVpZHvFgBINfkvnyepFTK6eY3e83ce5UJmiWeQ9RxaiwPBpP6Q3K+S72Z9Yx0F+y5yt4mlp6oRirMoicHh8Z0cVykmITSWE7jI33N1G93ON9k5C/V4p4yZ0szhd7i82HcrRwfjLGcJlibS4lOIIz6PVouYmD7gsamZI8ODXi11gepUvF9XxNibqFs0scRGeIg2cCN7qpj9U6mlpZKipkqpqY2ImHlkJ7e64WlkmEjTDKYnsN2uG4K7OLimlw2ghgr6GKsrSM8srtQB0KRKMeZzMPxGSpmgbzqg547N0if2sd0GL4icSHPnomslbYF0Y/UK1jHEUGMte00kUbjHbM29wRa1v0UeFS0uI0X3dVNcZogS2TNa5N9CraMareaWpge193AcwHsVPiOJT1lBHA83Y9z38w7yP7lVDSySYjHDNMcoIZmOtrIA6asqmhvpEmQAbAey3Y7zBqzDxg9TTyTyTS6SOJJBIDLD6b6LEkqsRpcNrYaelOSpmEWcjXIOisjC62lrYIYIuW2rDI3MvfMf8FY4so5sEiEL59TDz263tra38UHOiskpYDT0jTkls6UEWuRey7bAMVpvuicPZHTsA0sLOledbN+S4bD4X4lWxRvcMoOxO57Lp+D8Cqqx0OJVFPHFTU8147nQkbn+8Kq7jhts+HwMxvEDO6vqXDNFta17D+KN/EYho3wwsl8VUSl5eTcNF/wDkKGtxzC6t7R4wwSkWicdD9PmsepopmPZPTGJ7n35jBqQFpGuyuocUrp555mg+ezTs02t/gU0VLin3caahrKQUzHEuL4iTZUKuixoxD7uoonsLQTe2hPRb+DQzUlIIKh5Lh+Y3KlFqVHgUFNiXiqSZ0UsZBlN/2p6gDoP810ma6gAbe6MSfCtKlDxupmzEdVTLrFLOT1WaLaUdQ3a6I1Gm6yeY7ujD9Bqs0i+aj3UT6iL4n6qEy3+JRSiN7STv81ugRqG5rIucz8ypEAfEh5n7ytJa/wA5mXdBzAVUD3KSM3Cwq1msU7neU5VBm/eKWvfRFRyyOuojIVJKxxVdwddapCMvugMtviUTs1/Koy9wQT83X1J+ZYbqsDmddE7O0+yosiUHqibNl3cqQmIRNmJCDUiqGWvnU8NbGTka9Yge62iQe9huHaqDpPFNtuq89V5TkOqyxVvtYlC6d3dKFoVFRqXHQKR1Q4wnIdeyzjOcpF1CKkj0lKGpA50jSXepTZ2xixWG6plHpehNVKd3lKS2vNWgEBhUkdabBpKwzOb+pG2f95KXk6SKrYRe6PxIXPx1gAtdTsqr7lSliW1z2FA591nCpbfdGaxg9T0pFp5sonPI2cqktey1mlV/G9ylC++dw3KHxX7yz5Ku4JaVFz/3loawqNd1IJ/3ljip91Iyo90Gpzs2lgoX3DtFAyW6lDwRss0LMMxB8xVjMHjVZ4dqphKoJHxMPmUDoXP2ajzn8ycSOCCM0Mkscjds8bx/Beet9Nv+d16dSyuL8t15nM+GOpmhL/MyQgj6osH6fRCR0KN2VC5zO6iqFc20LlsYj+LhFFIOgAWVWlkkRAcrVI6olwG884kbG6zQOgVEeQ/FZCGa3QCdnxuUhkaR5IZSO4aUB2HZWeGaiSn4mo3s+N2X9VULKt7csdHO4HqIyreC4Ji8uJwz+ElgbE4PL3tsAFqmXqbg4OdfuoybIHVRc7NdMZW21UoG0a2VqFotZUopm5x5lZbMOhVoTmGBhzRxgE7lNso2zsv5ipmvYQshAZ0Yp7boCQ0XCKOqZ8awD5Q7KURWCHnxludpTCqYtBy1nxBACbeQpSzMLfKqclRlOjrKxItPcWi8mygkqGAZgqsleS0hxVV1Tf0lBLO/mArNnzC91bEub1KOobmBLVYFETWPmcpRUuHpcq7oy4p53UtDAaisqGxNA+LqtMrfOe7dO3VefYpx/iJldHRQwxM7jdNh/wBoWLR6VEcMoG122J+qw09HjpnyWs1WBRSW1avPIftYxGAFk+CQSnoWuITO+2TFY3i+BQCPtc3/AFQegvpGHooJ6aCBvOqJI4mDq51llw8c0OJ4PLW0J5MzBcsedQf0XleM41iuMyGavrJHgeluwaEHXcUcYQTsdhODkSczR0x2HsuSfhGKs1kDWt7g3WXFIY/QtaHiDFS8McGyX/MxSxen4OxKGjFaMQhc21yADosd5qKd+R5WzUYhikDR+PdhF+WBYKOtrMKr2RRyTZHs2DxYfqqKYnxGEZzG4MPxBqt8KYqcKx6GUTctkpyvDha4XQNwalrsBgkuI3s2c7a6yeKmVcOFwQ1dPSu2/EjKRI9UpK+hxRrn0VVHKRuGm9k1XTvfTSxvZdpbqF4phGI1OEyeLw6sML/YXRyY/j01SZ34xVXd+WQgKwLs8b4qqaBzbBjtF6VgETa3DWTskbG+1ibWuvMovM0SHVx3PUrquF8Wqo4nULH/AL7V1yHW3khY6GQ+X8zXLyjizxFJijqeaQyM6A/RdnVVVVz8/PeHs2ssLiidmNUzaeaERzRbPdp+q5jn8PimFMZnUocACbErSg4wpYqfz0cbnj4eXZUoaPGcPw+V8jGy0wb8LtVzgLiS73usi/iOJfeNU6oyBt/hbsFWzVPwROy97Kuc7TcEKwcQquXkzoBfNI02Oh90TZ7syFoVZ8pcfM+5RMifJ6NUBeSQG5A+ahLCDoiLS05Xt1am190DxsgOs179LISzX2UjW5jlQSt5RtnKBX+FMVPyKflh/ivMRfL2VdzdCUWCNiLoHbpF5HyRsZdoN91ckRXsdUTSb+VPK1jDvr2ITZDa7VBIHm6LmtHwXQNY0jzH9EvQdAT8grQMlhP7MLUZhULqds7ZCb9ALWWW2Ook2hefotXD8XqJYvCCJhtsoJPBUWXO5rrt/eUJhLb8oOejqebH5pLAnoFUdNKBnY5eeYE73SlhzxuaW91CaghuT0peMqntIljz9rKB0ZkF1Yx+IuU8TJiA83DlqeDpmgDl6+6xaJ74pQ0nRbt3OGYrpxgReBom+mmj/RLkQD0wMH0UiSseQFrYwMohj/RNlZ+Rn6IxG4+YBM5habEKjP8A53ziPl0lDAx3dUqvFMSrGOElg1+4boFjMk5brozVylpaHboNzDqYuos4qjCL2ADb3Khmwqoa7mVXMBdqMw3C3uEpZ6nDSHRsbyjkuOqjx18kLcsliXmzSDsqMOnhkkuylp+c4bhX6U4fG1sGJ0LGyj3QYfXMw0CR5s0791arMQweoYZmxkuPxoJKyHDjSGWPlue3Zrd1z9VyhJlkcQfZXeRC6oBpZOY49L7K23C67d0MQ+brqSywntjjZ+FZrndzuUmUjyzmTTXPZTz/ANILzI0Z2Gwso2yOa0MPRc5alEGG/b3SaJGn1cwo5JdMrR+qhdceYuP0WBMBIT6Fv0WPVMbGwzRloHxDVYEEmvVWeeWbKwO94Y4ip2YkyB1c5/NIF5TYNsvVpIXy6xi7e6+fMHEeI1bKSWEBp6t3XcRVOJ4UGnAcbkm7xSnMCF0wlaejCkl/rBZnVfPPF1LTw8T18NKRyw7ygH2XfYz9pr34bLR4pQz0dZI3IGMByEd7/VeVRF7pnSPfmcTe53XaMkjBD4R0tXfLooq4QxHl5NVfmblYXqHwUopm4xLC0wxu+LUFZhulubAa+ipYa6heJHSeto3AWpwnhVJI2ogxilzza+RwsR8im4axilnr+ViLxG0n8G5sGnsvUWUb307Gsmp2yvFwRGNQrDMw8ug4YdUYvH4aklMUbg+QHWwC7TiWBlXQBrJiGwuDMo2+qipHx4HLM+eqzOJIkjabi/cKhT19PWSinYZyc2YtaLquUw6LhhsEFH/R7mXMGEk7iyv0tVDFWVNRUTxNfK3I0t0N1lUoqZpRBRvZEDswNuVZr8PbQ08dRFDza+V9ow46X7o3bVqp6fCuG6xlRUR5i09bdVh8AYhDFg9dLUSgBgu0uOru64PimOvir5IKzFW1MxN3RM2b81miSWWLlRSTlw6A6KTJEvUabiusxnDH0kuDzeGkuwyRm9wua4gxeow0iipYjC17c5a46gqXhzFKp+A+BjyxOgeMzmjUgn+Ci+0SihpKuknjkH4kQBa52o2XKUtyUtVlBfM/zK9GYKrh8ANsXkD5LKrYGSHmFXMPjqqdjOZGHxk2DHbJENWnmpqeKkia54GTqdyoa6uDYgDUyuB6FykxHPXVrKelZflx3I7d1m4mHSOp6Vnp0ZftdaW0Mhc5nlT5oiwDl2sLXutHFsMOD1dNDUSNMcjRsNlBXsoW0RlgkJe8gAHcKTCs8v5cvcdLjdSuffzHqoTK8hoJGVnQhSRuhc279+isCcSxmncSAwg6NB1KpzNlqS2Rz22tYNHROJAWZy4XUZlsoIZIHMGYhViLlWp5y6LQ9VVLloLYJuv1ThjnCyRGQ6lAccmV19kTpHEp6Snmr6qOkgbd8hsFujgnFB6q+kH1KmezHXF5TRTCZLJ30Rukc5hC3jwHXjzPxalYO5BRN4ErPhxemc0/EGmy4fi9P8zXh5LPCznvcxm7WbfULpeJKjwfDdXI02cWhg+pWZgODtwYZHVcdQ4G5yi1lexuD74oRQ8wQtvcuOt1Pxmj+ZfCyecyXdE2Yem1gk+oYaZsDNmFdOzg7l+nFWj5RBA/glkhzHFR9GAJ+M0/zNeBk5N2q14sU59EKKSGNrYxZhA1uqFVTeHqpqVpz8t+S9rXVjB3cisAkpmPEjSAXD0nuF6XBtcK8OVeNPkmoqgxTQyMjaQ3MPPfUj6LS4swCv4Xkp6OvfR1jHl74zF5bv09QOy6f7PuKOH+H+GawRwZsUjkuYg38Sck2Zb2v06Ll8bwbG4H1tdjlOJKmojNTzuZfKL7D9UFHCMRw8eHgxWFkDKR2duWLMHE9+yWI8iu4frMTDI5akYjbN1bEL5AB01QmspYaIUlThzZJZ2h4f8AmHuEPgsKiwcQ8/kV9RU2IOwi7lGUE8U5pI5pqeqjmrW5InkWjcewWfAwxyvopBaRlwR7rqsUdjk2CUNAI6XEIqM+IimpTcxj47rl5LTVb60mxmN/qjTSwGMUmKSOdC5z8v4dtrpqaorxXSckxte8vBc7srfCWEz4tWSQMxKKDoQ4fiSDsNVpcR4XRUVTh9LC8PgpxaR2Wxv+q5yOUiaJq0PqhPLkd/UaE/JdlXYRg1JHSYoyFzZpW8xrXOzAnsf81Rwikhgr5KljCPDmzWuFiB1XV4tTDFsNpaympbeGqshD9QAVKXBysA4YgiqJ+IKKaQ81kZp45CGk63Nv07qtHwxRV/CmI8RUk/KhwyYxRuEd5J7uFr36AEL0zDOCsPFfRvxChZM2iiewvMYEchPo062WkzgrBKXA8TwSlEnKxDPJI1xuA/pZa1kvIuGeEJsYwR2PRy5ZqeujjiidHpISRtr7/wAF0fF3GlfQYnVYdh1PlpoJAyqErdJT1HyXXfZKY48GqsKngu+iqr2cPTcaW/Rb+J8NYTiVLXUs1O1vjyDK8DUkG4W0Z2M0I4i4fpaSMtEFYYnzDpyuo+WixazgHC8Qx6evqo+ZBIwPjDHWAIFgF18WHMo4WU9LpFG0Ma3oPkpIoTrn0WxDmYxojYLNAAA+ioYpQU9fBkmp45nM1ja8XAPey1paf4gqxaQdkYeO4nhNLg3EjauvtHTGOWWLMdJSNBp7kkrW+zeKpxng3GsHYLMnqrl/5QRr/cvQsRwbDcRZFPXU7ZRTNfYO1AFuoXH/AGW0VRFSYzhMjJKKWrDJYS7Q5CCLhbHNcV46+DC8M4SwekHIlDJJoZNTIc/k1991LNwDxFg1LSz0sT3V8VQZ3xxNBEJGoN+uyj4sdBh3HdPTso+Z4aSkYSW6uDNy0e917PVxfiB/5xdQc7UmbjThKhjxKEQsqXxyVrSbAsYbv2ta+T+K4PhJmAUvEcFRRMbS0ceIyGKeZ1g5gZtcr0zF6CoqMFq6HD4mOlmiMcbC7KDfcX6LyPF6Ss4YrmSySUnNpJ+YaQHmRxE+/VZmG7e19MzdWu2PcJm7LG4V4nZxThLKx8YiqY/JOwdD3+q1uabqUytsJAWiyxYPks2F4LVOarJHlapMNRJStjBLg83VcuJsh54N7pw9h+JIHDcVfi8Wn9yCMfxXaOl2+Q/uC4rGZObxhVt/1bY2/wBy7F/VaEjZfdWY57BZmYgqRs1kGrlZKyz2Ag9CqVbSatkgaBbTLskyqs2105qC/wCJaiUk5a0sVd8b/hClz2CyONuJf5r8PPrY7Gac8qHuCeqSzyUMS40wnDZa2nz82WkaCGgj8Q9QPkudpvtNfV1GWthdSwndzfMV554p5GgsDv7lA6d7T5VwbavEeItxiuNXBG+9zdznXJB2/QLHfLVN0yE/IIziDwNSmFa++psFlYlHBFJK/VtwQrIpGD0ybG26gmrLt0NvkEDC8x6+notUqwWRAgvINugKkZNE5oZsbbqk5+b0E/QI6fNGXufbziw0UoWtiGEE6boJG5oyzW49KLnx/E5I1LRobAHt1WAAFTG7bSyLmFou+2vSyJs8bvT/AMUbZoz/AGu6YiFs+trEIr3ddrypS6HN0ue6r1QfGGyw2BvqLqiw0vve5utzBqekqHN5sjLxefzaBx7Fc4KmTTMrtBVVLJOVBMIud5CTtZbiR3jKtlNSHiF7c0NK58cIaba/uBcxX+M4orGSvfZ0rX8tvWNg/wA9V3NDgj4sPijqKqKYmPRwN2ZNdB/mqhbg2BxA1D4BzyDmdGAY44zYrUMuOZh74KiKlZ+BWN1ynUt9/oF0MWLUVFRzQSYjG59LCBGTLrIO1rLBx6vfBX/e1Fh3KZWNeyDMLAMFgD8+/wA1lxx4aaMvljnE35mjr3VlYlq4XicFVxBBNW1UtPTvdo+5c9rP00uvVqX7orqaqpMKfBIyNo5oi6E7X9145wtQ02I1E8D2TyTcv8GOL1yG+oC6nBcY/mGcVo+IA6DE+QzwjAMwIsSNfmf71cEd7AZo2tjfHlDAABe6mLnLA4JxfEuJqapxvEqdsUZIjgAG4G5/VdHyxZdmEYkN0Ymt8SCSF97sCiyS/lXOTksGa/VITtHVVjzBuoecTK6G/mAvbqotr0lQ3TzbmyDxDe5WRjEuWgdMeblY5lyPcqSlqxV04qYx5Hkhp+SFr/Pd0Ukc1/iVAON+v6KxFmstIlfujZTzOOo07oNzqrcVUxrMltFu/Kma87QyQvi3t9EzJHD0rQpnwF2d+tgr8mHUBjE8cLSXi5KxLcQw+f8AvIzNHl1cjr6SGFwdGOiz3i65y2ldPYlt0BeD1CrvDw9MHKwiV2uqhcLp8wUkcYKCMRpyx9rK62n0zNQmF3VUZ5hKQY9vRWxDYouVbogqxNfl2Klhi54OVpB6DqVo0RbE7N0WizIRmDAsDANDU9IiqsgLXFh3C6iQ6W0VKemjcCWxszW3V5Dn3OsoXOV2ppzH8KpyM1W0RZndkxkKTvSoyepVZHzCh5pUaWgCok5x/MnE7h6XKHKUm7oLfin903iJPzKDN7JZglNLBmNt0jMSq4d7Im7pQl5jiU4zOQN3UjFGTgPupY2vKTBqpmhATAQp2l1vKkxudTwwsJCzMtohzlK0T9Wq8KaPorDaQBly4LAz2Nf1RZSr3IYOiB9GHG4QQUxtKFk4lwRg1fWvrWVMtK6TV7YwLE99dltupS0+VPynIMODgXBo7tdVVU1+77WVeThXBouJqag5MrqaeAvLHPv5+i6Eu5ZWPilQ+PijCng2zxEK0rQbwlwwNPueA/O5/wAVw+FhlDPPSGMSx09XYNdtZegtqA11yVwExyYziY7zF4XSnN3Qmw+K/Lw6laP/AAh/khfXW/YxRxjs0WVJjnPhZI7q0JLrGEOc5ysCtlz3JuOyF05JzB5/VQpre63whz8SU8c5BUj57qqCjB1WZ1w14ixBznv8jC+29loNheW5lSpR5c17fVTFz2/EuWUcXXGU97IxM0dVnPqHXtmQioy/EscZbtpvqOyiMpKpsqmjRxU0c8JObMs8S1qF8jhYX+qlc1wGqg8bGOqF1cw/GlLYpJXNKgklc46oX1UZOYuUL6qNpu0q0CcCdlE+4Qmqjt61BLUh/pelCYTEFEZhI2wcqPMufUsPiPip/DzmwR0vNfILtN9v4e6UOirJoKKlkq6iTKyMXJXkWJ1kuI1ks8k75BmIaHOuAFNiXE+LYyCyuqiWH4BoAqcMTpAWxgFZEjcOh8NzvFgSgekqHlyDbJ+qeTmRuLJRYjdR50BSxVLRnzt+ihNRMzc3+akmkMTC7U+xUdJDWYk8soqUyuG4HRYiQD6iY+3yWxhDOHJqcnEc5n/I7qsqswzGMPaH1dI+Np2Kri51Vsd9h2BYHi7L09DJTvBsLiwUc2HMo6o0jeVHINi/QLlKHiPGMM0pKvyflcLhac3GcuJiIVeGwOljBAff/BIB43BNhlVH4qSN8UgvdhuiOE4FikPi31zaIDcHVYtbLJUS82TQdkvC1MtskeZnzVEs0k2Fv8PQ4iJouhabhA/GK90DqR3LljPwuCFrPDScmSCNgO3W6CphZlc9hIv2KQIfDTcrmW0SgkyDVW6LEX08fLe0OHUFS1Hg6sCaGARyHe2y1AtUcmaEK7BWTUU0dRCdQbLMoifQrc7HckrV2Oxkhq5SJXx+R4vdYeM8LVuJykwVOW2+d9gtDhTHnYjCMOnjLXwCwv1C36t8RhJMYa7oQg5jCMLquTJhVXVxOu34TcFc/jvCk1DI9lLGCGfC03K3nVToKznj1hVOJcWdMYapkMsb72zA6LUwMCm4XxKSEF9JIT3sofuCcPcJw6No6gL1DD8ep4cBbWvPMy/DsuExzjMYlIRS0gib2JusTA5ero2wyDlyZ2d0VK99NJnZYnsVNNPziS9oAPZU3ubfRShLM2SR+d0RaT3T8qQjRgt/aUf3jVZeXzjl7J4a2FjMj6USG++aytLZjzIT6bKJ8nMPm1V+PJiLvD0tKeYdgXbqrW4fV4e8Mq4C0na2qygOYwNs1ov3SD7i11A7U+VMAVsSObc3QCRwI7JwehKY2Dtlga0WJM8KY5KeKTSwJGoWZZ9vKR8k7MoRssTbqtjU4eZDPMIJ4I3NPsumOF0NA9sjKSMOJsuawISRVgdk8vfsugmmfPUgPmzWNxotDom/dRHJloeWT+7dchjuDMw3EWTU8bjHNsAF27HQVVC2MvAdawuuOqaauirGsq5xKzONb7LMwwx6h01w8wkfNQPq4y3JJGB8l22O0UEmEvlbE242IXFQ1/LjyGmjd/Bccmok8clGG+cSXUZndmPJjNul1BJM2R+rAPkgzG+hssqmaZBKCF0FOWmBtlzbX+cZySOy36M/0cBaiBKQQo5NQjdsgd6StUM2Yv5pVumecouoZhc5kVMdAtQsOTDmHc2RAtd5WK7S4Q+bD3Vxd6PhtqVWjpKuNviRCQ0brSOq4XhfR0/OmmlcL3bC3crZrqKOVgxKSEsYdmu/xXK4dj4p2sfVMuY72awLdnrMZxyFszIY6SmAsMx9SwM6r+73sc+eC3YNGgWW1kU2m7eoVyqM2d0FTVRlg3KvUboKNhBj5nva5QLDYqqJokpKSOJo3fLqFTrKirkrS77wBt+TQfRXxjckRLZKS0J6XsVQeKWeTmQx2HzWJA17M7myRxtAtbTqooYJKlmeNlwFLUODYiC5KKvnpKTl07GkHuucqzJjy3lj4zcKN8rD8H8VPUl8nmfuUhhkxHMLwAoiCO4dduyO7ybqN7JGaKRs7Y7MkFiqNGibmdkBcCeoOq7/AIQhkqogw00cDYzYuvqV55R1cUUgffZdVDxrDQYW57PDiotla0aLcDI4/r5qvGzhz5GOipfTl3v7rn4Y7vQxmStqH1T/AFyPcXKXPyT5WF7uwW7axS1bc0JTx1fjKKnwTJZhd5nXvf6KWMw5GtrnSNYfU1vVV8SiwiECTDaqQu/KeisSKNdQihndC4XI1B7hdVFxxU1GCx0lU3K6NojBbe9lyMjnyHM95PzR5ssIHdZtlblqszyQ+VvzeVoYVxDU4QXeFLJA8ah7b691z2ZykjcR0J+QWLG7NxFiclWa4VHLcNmsFgFtw8WVNYy9TjGIEflaLBcMZ9bFXm17YqJzGG2cWJG4Sxqx4zBWsqYYKRoYBYykeYqPBcRZhkE0jLOqHmwuNlk0tQcPpJA7+t1HyTRzstmPVaiSm7Hjs0LJOVSxgvN3SdQVkSS1dXVeJnmdKPzEpmyOzgPFm91K2SnvkJ3WVpFJzMmd7vKpPGmVrctwQoXOMzXR5gAzZQwuc5hcG7FVG5T1zqankkp2RsmeLXssHl1FW9rOcGOYbm/dIz1DHZMyIXHnO53Kg3IqmoLfCz5DLKMnPfuB2WLWQmnndA6QSZPiGxTz1Tpg3OfSLKvnaVu2klPSmQZy5rR2ch5TBqRohLkhICc2ZUKZ8EUZ3J6D3VKT0td1IV5kDZnEF4ZpfVRRwZ3hzmgtG2qClkJ+SawAWtU0hnabMAH5QqD6WaMXMLgFQdJSzVk0dPA275DYBXBQ1NC+sZUUrTyG6ufGbfqmwiWakrIKqCl574iSGHYre4l4ikxPD/BzUJpKkm8hJFnM6hGWbwnT0v3tSVYkDjlL5QNmhWK9/jBV1/LMjHTBkUhdq0KfgijDDVF0DQ1wuHFupGqvfdXC/N5hmjH7pkNl87qM4jLzerT6IpI48QxGCgq5JOTHSh2W+5HVVKPEzhcNQyN78j5SyLS9gtarpeGq14fUVURIFhaS2isxT4PFB4WCopeXYjLdeTxMaqpenjDN4Te2anqZs5c8y+YkqB3FdeHODMDL2g2BObX+C3MOgpaWmZBSRhsY7Dc9yUU9fSUcphqKuOF43DzYrlOePiTeFrcOWxPivE30b4ThDqTP/Waiy1+GsWrK+lkkrKcs5f8AWWsJFbnxHAqlojqq+nlDTcBztLqKtxnCW0U7mYjCfIQxoOpNl2njljxxwZm/W3I4vS8mrfLzGu5zi+w6KKat5radsceR0Hqda2YqmHPOpddSwtD3ZOrl9zD+Gnz8nsfB+F4PjHDo4wrqZhqGF7GtaLMhDNNAOvuuF4k4lk4qNJHG8xup2mNznuDbknf+ClwrjQYLw47ABSuE1pA1zb2JeVe4O4CoMYwWpxjFJnvfYsigY7KA8X1Ovy0UHVcPUNHW8NYdJSUVNJWWFPLI2MOc0DqTfb/NebYzF4niTEJ8n4bJiB7LewXDa3hCdlbDVU7a0DJJS8y4kB6lc++tfWuxCeSMC87yS3a5OgRmJbvBNfh2G4jXVGKTiKjdSvZb81+gC5F+r3xBhaGOOS46X0/grBmYYDHH6rcwO7KSeaOtpGiqZkkAtnabD6rA6DgvDJoWDHZ45Y5GPPg3OFoZu/m9l2H8zo8YrXVTJYnRxedsA1Oe1zcfNc5wr9pWKYLg8XCtRh1JVYfrynSgnKN9Le61aEYtQ4DXcWUniGvfEXxuBEkYG1jbW6CzHwtVwxTVNRh00bwCZHZ7j2Fh0XTYLBQ1eGNjbG6zwAXNdYaey5/A/tBiOFx4xiuHyFtPanqpo39zYEttqN13GG0mEOwFtRh1T4ykl88UrNTdbWCi8smRWDC9xuAs3EMUjwiMO+6cWq//AOWiLl57L9okGMcc4aKWKdtCwmLlzXje2TuR1tbZamKJX+NeIcR4K4wNXhdPGYaynjMjJBdkhHbsui/n7hzqKlqJmiCaoj5pp3Gz2hYP2oYU/wC6qjiCtiknmljip6MM1EbBq9+n0XC0lLiHEFMKitjyy09RBCbtsSHjqsJT3OkrY6uFs0LzqLkEWIVhjrnVV+VDSgRx7NAF+9hZEyVt91uGFrOoyxucuITCVvdF5XBGwvaxwIOx0IQeFphJHMI2cyJhZG4aEDspCwhAXAbqjiftE4OfjsgxamquRNRU92m5F7ahdhh0FYMIw5tXrMKSPmHreylMbJo7PFwU7p5/hddYCMT29F5D9oGDsqJ8TxKCnkdMMQ5ZynQMDB0XsQrH8rlyMH6qkaOhlLzJSROMhuSW31QcJ9n3DmIYTCMSqDLAayMnk20Iv5L6/P8AVdo4nqtOZjJgCdwqcsOui0ImTFo3Tmpv1T+Ckf6SFFJSyMNja6tBzOSdEmyuzDVVDHODZEwSBwuCdUphylY7mcYYk/8A70f4Ltpneay4QEycU4l//MW/iu1mf+KU4thzHunbsozI26Jh13RhMTZE15aqGJ4lBhNE+uqhIY4/UWi9vmuLp/tFq3YkZGwxuo5DpHYXjHa6lj0aN2chq83+1DiHCsZiosOw+RsxpJHyOlB0bsLK9hfH+HBkjMVY9r6kyPGukYOw+S83rZomGUQizbmw7C+ytis5wfJyWG10DrQyXMwcfnsqxJF3BwCQZNK20bL291inSl1sjXC55Z+aqTyR5rDc/oiAMYLHtNxoQEo5Gx7gEe6zEIjAeR5UhfoVZ5zJIHHl2ybPtseiOeSCUQz5JC6OLIc3Wy00ihm5ZuNeiJ8ziLi/0CrSOLnXumDnA2zAD32Sha5rQ4XN/mjjfHHcjLcncKuwxNLm6H36ICxodcKTAtStIdzWnRRskaQM2h6hCJczCx7kJZb4tEoWmTxteHlocR3VoSRyjIQNeyy75SrDJAWgNvf2UmBb5MN23bexSvFE8hwJB1Fyq7eZvnBHUXUt2yNF3WIOylDqqPiSkfgLMLllnilpoyY38zQlVMUx2pqfCTzwNyUukBeL5z3Pe2mi5/OwbLQpcaxnDWR4rQ8gtp3AEviDm3PSxWoSYPi+Oz4zM2SaoMrI7mMHoTb/ACVCkhnc9zBKX3u/VHXYlJitU6tnhgie/cRx5f4KvzXtF2OsVUaWBVww7EIcV5ebkG4Ga2vTULQ4kx2fjDiJlcaCQZI2RcuJ2YhgPf6rCb63xjVl/I7vZaWFSVFAxuKQTNNQXPp4qdjvO4nuP00Vsl7Lw7FHFw1SMoaWeGAMs1k4tINequtDgdUFHgE/DXD3javEvXGyWVlRN+zNrkC/93srkWF1tRSsq4XxSskALbO6Fd3HIzCMoUgyqSDAcRkcOa1kbfZ1yrRwWSEeQuf81iWsWRUwvmgkYxuuUkW3uuUpMXgkxWjnqIzFI9sjDf4hbQruTBI15YIZM3QWXkeIY1HFUwTUsH40M0gMMu7TnuAVluYbXG1ZVNq4MKpwY4eUamZzh091d4GrYq/AmQs0fTuLJB7nW/11XP8A2iYlUtxSJrGtD5KERytOobnOtls/Z5JHPh9VDT0vJjicxgvuTa5uiU6YR9kiLKbwxYLl/wCijkife+XRVUReAhzuTmM9kKMnzuVuLFKmKA08dspN7ncKmpG5EgSzVUjmkv3VV8zwFYLGOFkYow8KTDUKPNLt0JcdldFFHm87D+qJ9EwML2AadFLKZ2Z3ZSRzWKTomE7a9lHyDrqArYvx1oAy3R+Jjd1WSQ5nVOJiOqK0g4XViLI+wcsts3urMU2U3usC/kDTlCmbOGtAus98/YoDUEdVaGgZxfdIzM7rM8Sb73T8/wB0oWp445VSloLtPmCIVXujE9/iQUTQPJIyqtPSvh9YsFsXF7J544qmlmhmbdpif9NFbHOuZbdDl7rR4Vroca4fbPOGvkicYnWFtQf8rK2+hpH/ANWrEpMMS2myVltfdUD9iWfJJ3DjSwvgq7u7Ob/xWrZ4sVPlKuz4TWweZ0eYfu6qt4Wo/wBRJ+ittI8pT5QjEMzfVC79E/Lf/q3fooya1lK3RBkkv+yd+ikZBVym0dLK8+wWVGx4HqRhyFlHVH1st7FH4SUa2WkTRyaq1FIOpVRtNPf0KZsE43Z/FZGlFPp61bimaRZzrrJiin3LbfVWYJmscA469VG2pl6pFzQq/jIw3LcIDVM6KcZOWKy+VllCZxdVXVV9kPNzbrXGWecJJW53XXPcRxPhxLCJn6DmWJW+JY97rC4ud5KCbpHOLrpjDMtmSN+fIFyGNRmDiSoY9tubEH/wXXyTWlJC5biw/wDxmkm6yREH9V2c27QB0mHwSDblhGQq2DvJwxjC70HIrNwrDMllCbL7pa+6G8nRi1bnR2j3RZgq8pxKNwMUcOS9iHKdgqXmzo2D6qS2swlxb6kZLA31aqKJj2jUhFkPxLhP5pbj0Qv3TKd0QITx00eXOblauKTzQZIn7i5Q8lgN9VZMWUaAD6oeQ8jMNVzluJQOJHVRGV/5lZmgkDdQqj4yo2B0mvlKbO5DkkvsnLSBqpQHO5LMEJchc4LTKRrrHdcbxfEyux+CnLo3tIsG5rEf82XWZl5XjxrabiCon8QM7HWaQL2CFt/HeF6TCqI1kE8jyCBZwXNNnm5jeWCO91pQcXVjW5a9kdQ4elxFiAiqeI4a+ndDNRMDj8bdwFiYdYlQqZ2ysPMkOb3VSOeaGTOw7KPyg6yE/NNJIwNWZIWHVpf5pLX9gtvDuKKGnhNOyhc3OLFzTrfuuV5rUmyNvolK6DGMdmrstF4qY07Lau3KzJWhrM0cgP0VIyFx1cijvI8Nc8Nb1JKkwhF35lLDJy3By1n8OUsVP4qSqEze4kss2CikqahkdJHfKb2urQsPlzC+Uj5qzHNagFo35hs62iHEI6iJpZUU/Jd2Sw3EZoKSekyAh4uCei4irZrmh8k2dyYh/wAEhCpme7yRpdTMkJcLFBMKmmEeR7Iw4dbKuKjK67L6DuoZG5Hm+908Zj/rBcLcQNvDZWPa2bqVqNPMjcOllz8FTBI1sbCYyDtfdbtIb2DtiFuBmPmmoqvnwTOieNiFu0XGtW8cmtEU9/iLbH9VE6gpav8ADkY4E9dlhVkEFJUmFkl7dbLQ6OOrhrJi97MgOwvdB4dtXBPRe+dqgw2nnlpTJS/iW9QG6OkdUeID4YzYaOJ0sljW4dwiOooXwVM0mZgIcy+gWLVcAVstQ59DUxiPoHFdbw+GRSVN3ftBdbtLS07aa5+iMvMP5j4lFKIah8JB7OWHiOEVdHUugbSyWbtovX4qDxcp5kTh2usmfhplRUcvEZJImn9nlPqRYl5TPRVVM3PLFa6rjN8W69K4lwWtoGfh4Tmpx/WM1C4SdkJlOhj+YRbR0lQ+mlErN2rSreJZ62m8LUCM33Nt1Xjwaqmg58Fnt9lnZWHdqkgSLIhFIWZwLhGYwQrdEyhkp3Mk5rJB2Koz3ggXIISzFTcrN65DbsQgfG1h8qBoiLOud9k4Fza9j3QlpAzJ4o8z/Pt80HQYM6Bz+TLIWE7ELZ+7WPna9mJRj2cuboqB9VKzJVRxX/N1WnV4dJRsa/mMlubeU7KxLDcqC+nDRHVNdft0WDilRNHMDzASe3RTNp5tunXVWcSwFjaB1XG9zyzoQtAa3GmS4TyIZLO+Id1zAYrLZIzBZzvN2TwRU8ptPce4K5ZRyXFSLWCQh97fJWxS4Vy+Y+aRrlDWxQRSfgSFw91WdqsNNGlosNqmksqiLdCVpUwjFoITchc9RioNUwQMzHsupoKaZr2meMRu7BaAvieDlIQuikPRa/3eHecPKLwdlBz00Lw06KowmOTVdFUUlydFj18GQZmjZagZv37JDpDSRRs6AbD6qvh2H4zxFLNBhskDQw3LA6wUEeJSeH8E2kaS/wCIuWxwvDUcNcRU8leY2MnjJzF2ll0GPXYZNRusynl50DrSHcX9l1cjMUxbCKedzMjWDM5rT0WlitRhEmJTVEOIU5ZINWt2J7qlJiNKyldDFWx5W+6zODlyczWmndI2Njrdyuswqvo6WiZAY2yW6u3K4qrmgfMeSdlJSYlPSEloD791z5OqzjDnmsdyzZvZQ03MAu/ZCajxMpmmOUfrZXfHUbqXw8cLnfvLMzYgklYQ3yh5v1QzFxOYsEfsoXvbccsWsUM87YyJJHn2WGinkAbq5TQTCSMtL1kV1UXOZluGg6Jc8OOdnTp0Sk8miQGuNzfsUzmx2zOARuyAAixaRpqhJjAOdlwsiAwvkeOXax6kqMiTNY2PySdKHHIz0dAjpSHShjvLfqukIswRZWKdkUELTUv1cP4oqmGRtMRG4ByzX5oW2eSVWk9XiAmH7HLbss578zy9qU0t2ktQeiEdyrAkaDbzbKaXlPdHy3+Vg1FlAxznRge6lbDMOZnjIFuqpMBec8zrDydCpqGcRVPLky5Hix0QwRv2kaQO5CmdHTMfm391iZS1CqZG17zHJqDoCmg5jWiQA3apqtsDyHQ6gbqs5xBs02CqJHVR1JYHOPxFC173AjKSD1CQi8uZ7gEQDA7IzdaaXZckmW0mw2KYNYfU/wDRBHbMA9pulN5HkWWaZWIhGXWzkDuFK9sFE4ZGg279Vm5njYpy/MAH3JShafEJHF7rRk9UBa9kXnOqkkJMQ0BHuozLcefUJTpCF2Xom2vsmdLdxygBRlxcS223Vapkm3O2pUrYjfZHFFZoky7qYZLdA73QQgMfJ+Je1rbJZywWCnbFGWFzzY2Nteqqlj3W2BAVE8c8zRmvr0UdRO8SuyO/DNkEQqJC4PJAZ3TSB+S/uoAEr2uzseWnpY2Ukrp6t5fNI6RxFtVE0KzzRcGOPKALb3QbFDxHVUFCKTwDHlkfLD89rD5KhPWPrKVlKYImiM3Dg3X9VFzNLPPzChsTss+QrugZFUskkGZgcLhdDOMPqIYpKLWV5tlGgssaN3MfkK1KIc2ri120WqXk7rB4I/CMZk1C4XFzQV+L4g6rlexzCS0jqRsu9o5fDUks+X9nGTdeeYjVwVbGVOnMN8xDbX1SoY5ZKQpI5oGzMZbW2UHQKCUeHY5wpn5XksBcFPC+Plhofr29lLjeKR1j44KLMKeJttRueqnBq5ZLXBWIInzyiGP1HZQQwTVMwhhFyVv00tJhkQpIOWZf6yd7blp9luY4ss2TD5IXkT2z9bG6UMk7crnV0rGg3MLXWKkdUGaYPdu42JTGmfFUipb5mrmsSKSoFXWGQ53u6lzySp8ObQOc90tSYnxyXDOjllQSkOleOrlNTueySR8cYe94sL9FqJRLVVU8lVLNI5tiLeVugHspaOZkjg+o/ZkEKGRl6WOF/qBuVpYdDA9sFFJFe7S86qSNnGS2Dh7CRDSSxtomDmSuFg4vvoF3H2U0T2U9bQ107pHSiOcUoIMZj/MNNPfus+bhSg4o4cosUZVz0cTLc8uOYZGb2GlztZc5xVxnHFiNF/M6Oswz7vp/Dc6QWdKz5dtFMIHqPF2FMxfBzhvh2UrJryukbGARy9bC3dcUKTjGP7OW4Vg1DK6OWq58c1LMCRHqSw9WaqxgvHlRiEUeHUFRS1L3nIDiEuWQvOp2XWYB984fjM1PX09LHRVo5kLIn8yMPA1YDbc/4LpxWMnkWA8WcY4JT1vD9JJVCsrJr5ZNSCRra+x2Wxw/wfhWLcHYhib4qinxjD5X5nynLY7jT5f3rpftC4Xoqakq8dMks+KVlXGykc0ZTGSdALb7ey6Xhitw3iLh5z62lEVTJ+DiENrHmjQ3UmC+TSwid+O8IUcFW0h09OwShw1uLIRwjRGStmeAfG1cdVIBteMWACuUjaWjgZTwF2SNoY3MbmwVxtRG4epSkVpaKGU+a4PzQDDohtIQrtwdksreqtikcOl/qpASpGYTiFrtdEPmVabKGqTxjhs5WxRloMSjbd0Ac3uHKsYJ3nzxkFapr7syl2qqSVFz6lLFbwkgGk1vmo3QVTNQWuHsVK+cqM1Lxs5bEVpuqAmQHUKYSgnVEW5xmDkEccjycxVkMY5ucKuLBRmSzvVZOI0mu5bRZMIYZT5gqAqXn404qZG7FBcdRQZtNQl93sLm5LDVVG1cwO6tU1beZoe5B5pTtz8U4l//ADI/vXf+GpZJy2TRcBhcrZsdrXt61n/+67maS0yC6yjp4neSMH5qYUNE8XfTNJVOOpfcElTirJCgCu4YoMWpXUokfC64IOfS+u68bx6CowDDZsKqsKfDUSTkmocMocAdCw/Je1w1Lw8OXm32yxc2poZsuXPDYO6Eg7fxKyw4IfdMMln1c0uQX1OhKypqibEKl0kdNHE0/C3YLSxekocPjjignM9RNEyQuItyifgWVC/kiyzEtjdDThmSRgzdUEL8vkFwPZSSS81tr3ICqCbLaPILBBefFBN5h6rbhRmjjI8ziT3VfxGQXapS54GrtwgvUrDLFLSRj9rawuAwEXuSstr5A50cws+M2cOxVqKQCSOQn9m4PSrahkstQWAZJwL2FtVRQ5hafKna6z87xfVC4OvomzBaUWgcT0KROiEAuIvsk71IFe9vmpCXlgHuha2yt0xjLmgszOPRRVfI61ykwuGzrK1LEwNc9jDlv+iq9UsWYpdLOKkkOQteNiFXjifIPIRdWXRTsjLJB8rFQFDllPmWphFTPSc57KF1VTCzJ2jprusKnkfG/wAwd9VrU+JVUDHsjfZsos4dCEgXcZm4RlwihfhcMkGIvcXy2cXHbrdYAkufxnLRggopBLNI4NlB/CueioVdLJT1QjmYW8xge24sch2VZSwt5kgbAC51rWAWhLSVeC1lOaqFgmYGTNAdcD5qbB8ap8LopMPNFHMZ3F5kedQdLfpZRYtUT4jUOxGZ7SCAM2xPyCK1+IOKKjG6ekimJdDTx+aM7OfrqvZvsrlYeFKWrpmZGyXBY5hsCNNPnoV4zwYzhtorZ8adBJKyEmnhm0BeAToep02XbUX2l1o4Uo8H4awd9VXCi87qdlzSm/XTU6KpT16V5vnGijfUcpuYmy874a4k4q4xNHWSh1BhtEDHOSfxKuUAaHawGn6+y7AE/nP6rNkQuVlbHFSyTTGzWC5I3svJpsFoaziJ+KUM4GeUlokOjiBv/BelVT4zSz8/0Mie8/ILyilxWinpq2swjmTvjnlZSAE3IyX0HstRI5biTEcUr8RFJV1HP8IeXzA2zCA/e67L7N4Y6SvxFxkJdJCyQF22pOv16Lj8TL24LSU0juXOWmd3W57fxWtw/jwinp6GrnipYKZwklcB55bDQEqWPUjn+NPp7KpS1j62hiqHRmMyguAI6dP4IvN2W4EhIuo+UCbpebsiBsgjlpT6orW7FQyNfDbPbVWg56jqwZowG7jZaYRiWyssqSG5WqkYpAU7C9oWWoXjK5xVqFpy+ZZTZZGu8pViOrkadSsNLE2GQSxOey7ZLX06rLfTzDTKt2CqiyBxKExxSycy90Jc3NT1X5T9FWfHUM8xjd+i6iYxA6C3yUJe3oraOc5s7N4ZP0ReKkO7X/ottsgY7MmkMROZzB+itjHZVk90/iX9ir72RHTICh5FOR5mBUUue4ohI8jqrHh4wdAnELB0QQBxRiQhXYqHnG0YuVZZgj39Wj56LAymyuUvjGRU87n9IXn+CvnAD8Ug+iirOHQzDK2qzm0dPKSN/gKDgfsvxRposTpydBMJR8jp/guwFbGDuuF+xaOiq8arsJq816mAPjINh5Dr/eF7TTYBhVOzJ4USe7zcqQObgrYnW1V6ORzhotaXAsJLs4pbewOijdQwReWNlgtxkKTGyE3vb5Kw4tdYvAUwp2ppWsY3JbXupyClgbG0FsbSCOoUAbFIchAYewCkbK8+Q3+iglOV9/iUsWKelhEoJYD81pMawaFjR8lheNyH1qeLFB6S5BdqMJhqZDIJLE9LKlU4O6nbzHM8vdXG1jC0HOjbUMd5Xvu3sljDcWH0oVpYhTx1VhHaMt1u0LJfeJ5Yd11jzc5/KsxvDbXRTRwVAzACM9wqbXPecouSrdOIiz8SQhzSpMURKBtHPmPLbnAF9Cog7stQUTjFzmTD5KWCQskBEcYPUgbrcZ0k4scH6fNPm9wuinpqKog5ksQz9DZUTh9KP6ofor4qeGyr6+oLJ4rGbCQ/8kzCur+76b8h/wB5ZXF+HwR8NVcjAczCwi/zTxMTgjbYwwv7xgrB4wYI34fVO9LHFh+tl1mFtpZ8Jo5+SZM8Q1HyXP8A2gPg+5YGCSPPHMDlvcldPExOA8BLnU8zOzloFjABnkY3+06y5duNVWEm9LBHIJ4wTmO2gWXLUz4tW8/Emtk7NboAFyndji34M5z5OynxTCqJwZV18UZPY3/uVSbi7DYSXU7JJwPitlH6rkRQsje9zGi7zexF0Qm5ZyyRuZ/Z2XLPq/g9GvpPi0a3iPFq6YPppOQwG4DevzViLijGYWAVFNDUf7OVZbJ6XaOS5HQixUjZy70vXn/E5u34XW3ouNKUODKvC6qPuWkOC0KbibAapt2Vhi/8ZhaP1XKZ5h1SsZP2j0/EZH4bX8HYnHcFt5cShfb8jrrExTFaeRr5qDGJs/wxAWCxHUUEh00+SA0EAP7ST6OW/Hj3sfhcV/DqODFY5TjONyQy3GQOPTusuWSvwXECyixF8zIyC1zXXB+imkgfKwxve14+C7dR9VVbRzwnyQEj2N1PGxPwzdqeNcYjZHM6mpC0i5beymwHjD74xAUNXSCF0n7MtP6/4LCLqY6VVE1jjs4RlKKGmMrTS1bI5BsWnUK+Kx+HehyQ8pxaoZKZ8voC5eHi7G6J+Seupasf94zX9VqUn2gDIfF4fCbfFTu1/Qrr4tuWWult2H1Z/q1H91V/+q/ijj464fleGvkmhd+/Hp+q04McwiqsKfEqdxPTmC6c2eLJ+68S1cyKxAve68dxJ1TJiVU+oZleZNRdfQpvfQLzXjX7PjTtqcewuZ77nPLBk2+RV5Mzi87MT3bNSb5VPTxTyWdM/kRv9LnDcKevwnw4ZJDVxzNdbW1lm24Z8m2hF1JFJAGfiM5juyiAA9e6fOxuw1UWD54Q/P4cfVQOAFz7ozKT6lE9wVVI0NezSwITZb6GyhcXDytKYF9rEpKLDGvkHLzE+xKlDqvDqhhaTG7caqs2S2x1UscdbXvAZG5xj69lpWtiVdLWBks3wCwVKKeMA67iyaomY1vIfJdw9lBG0FedEBDg/ZW4yMrTayeWGMtEjWS36/hlItaGNsT+irQpmxyESGZgB3CryNjBtA/MFMYJ5G2hBKjEEkbtRZw31XTWyKOnfYPe1dHhzvwGrClMzQByXj3totrDoZ44AZhbstjVppJhKBEA5x2BQTYHVS1JGK4ZmZ0LTayKldy6iJ/7y6eQZhcb9Ct0wzaLC6XDc76UOaHixaTdSGCejYahjGOin8hBCkqHPZCTK2w6notnDYRWUIZHqQpTVuNjxV9PiBhccoOy7HCmSy04fJe3RYHFmDww0YrbRxyxEAi1rhdHw3VeJwKKa91lleYLFVMVIZC2oy3czZXNQVRr5PEubRRi5cUXE8Er8Sia6q1Z1Z0KGs4ewOshMcmHRexG4WjSQQwU7WW83VKTKjTlMUwJuH0jWUgEkWt25dR8iuJraLBCxz3wTUtQ7YuGi9e0vmVTEcIwrFWFtZRRuv12Kw528UdRT5DIySORg3N7KCB745crPU/Yd11fE2BQ4NPyxMOVJawA9K5psbIajPG/NkOjrWW3RZhwqqq3B0l4r9S3RPPhnhCWmdknyVrEsY8RTNbFLZ9rFtlkuqJrboBlpXxi4GnZQgWKTqiXugbJfXcrAvwvL2hit0sTBUsbUyyRxHc7rPpqljJWtet+kqMLlnYyqYMnW+q1EpMNZrsGiaGRSyyO6LpqSjMtByXsu14ssE09G+xw6ljkPQ2XT4e93h2sfG5pb3V5M8XnsuAVTsQdTc5sTelxdRDApIp3QS1DGuHULp8dZ4fEmyf86qBsd8Yjf+cXWoGRBgtIJWtqqyzugcFRxnBBSNE8MzZWv7Cy9BxLDsNnp/6UOV++wary7GC6OpdBT1E08Q76rPFtNhE78OmdK+gEwIt5hsr1Tj73VTTkEduyzqPEXiAMkk1HSyqySCSQvWJHotKM1Mx/cXRlqpcP1IqcPblkzWWhy1BUnb5SAFhYrF5HXXUGEn0i6wsfp5o4nSZN0geexP5VRGCwuDDcgDdbNbjNLi1fTRy0OWNlmXtr+iyxJS1Mkk0maG5uA06LNdPlmL43uBG1l0HZzcScP4U7w9JgudvX/krCxfEqLEncykohTu1vbqseSofKcz9UcTyhxamDMge0sqWZnHqpq6imGeSCnkyDbRZUU8sDwYyilxGumdmkmIb2CwE2WwzXGqsx1TIxZ77FVo5WC7+WDZV+bJKXF4AaOgUoXX1IPoKqy55HXe8Htboog/oApY4ZJDoNE4tJAfEesAZNAo5DHD6GEv6p5QWjlt0PVREcohx6qiyx75Mrn20PRaE8plgzCMjS2iz6eLO65ukKyZsjoOYQLqUHjBGhBCsQw5pW5iQAVC3MXi5JV+BjTqoNIEPZlWVjL2AhjG2tutJvkic93pYLlYj2mrzyZrarOIqhrnuDRuSgn9YZ0ZorEMb45c8moYL6Kq7zOJ7ldGVmhuZyOwzLTjn/AKKXmMnXoFm0ruS9z2i7iLBPVYjPTyGOE5e6TC2suxenf5J43A+zE/iMP/1hVBzo5YjPu/qVHHIwOD3LnSLZZROkGsjWA3uOqrS5HSl0d8t9M26PmGXSwQOGQ2WwZdzPV0VqihpDeWZ5DiNAOirRzRtaRlu8jqrVK1giBtqshGnzTh3Oyg9Smlby35CQ/wBwgMhvYkadkzjGxoeCSfdGjuDQhmto2PXTVDLO8gbBDG8vcrEJCxRu0LHuNradUExyjIms+PzC490AIkdqBfrZWgG4QZiHEBG4kb/RMxgL7lVRCeSOPI2ycTPmIe/cJOiDzlabJCinvmjdp8lkWiDI0e6jlZleO4CmLaiOEAMJPcttZJ8UZGpOZ26ohMryQLi3umLmXy5kDRCZMrhKSD20UvKaJvxG5WWuPdAhARHnLVe8FGaYcnzO7KnWSPZBExjt3K1M+BkERebkO1CChzbOcB6tikXOYzTdHLFIX52xCzzpZJwsSyZha4dFKAwi4Lhutrh+J0tXZ5vk12WPB6Suh4WiznndTclGXVVMkMWCVDppOWwxkF1r20Xl9M9kTzDVQyZyb39l6HxHLyMDyW/aSsH0XE47NBJXRPgd6IwCgpVkEkThJDGeU/8AZkdQqJc65uuifiVO7DDStLWm1g0DULn3i7dfV1W4lYlZw2eSCe8bQSR1WtTujZVRvq5HtNtc2tliNvHYtdurI0F7klSZSV+nipXVrRe0bybHslVsDHzGlm5rIxfa2vZUuc2PdXXRPp2WmBDJxplF7nssyQzZHPmLWPZHGBrYFKmDxJduljuoMzQ7L0VunLBqdQ4bKwQlmeyWQ5Fu8NQwy1JfWE8qCG5I07rBhiuVqwfhwOY70vFig9B4d4k8VRNoaKj/AKPRkiKBxsHyXvncbegdl5zjOI1uIYhiTaqYTVNRNYinaDHYdB2C06PERRUsoaJRJI3IMu4PzVvhuioqGsEdRPHnliN3BnT3KsZDluGaXDavH6Kmxd8jKOZ/Lkcw2IJ21+a9mosMdw/SGl8VJPDST+JZJI65A7LyqWipaCup6ylgnloG1UcjpXixcb6he3T0sILXW9YuuseZKnNxNR1bIicKqKs8wPhAj2I2f7K9DV0pkeKaERvJvMI26CTrc9SuYxLBZ6GLE6vCJC+aogIjiPw/I3XMUfF3EHB1FFg0+F5Hx3JEouD7q0w9U58jfiUGL47DgWFPxSqk0ZoxvVx7LG4cxvFcSgeMboY6SoZbl2GUS37Kh9pWL0MXDz8IbODWPkZZg3aARf8AgsS1DquHuLafHqBtVTD8QftIr3IWnJiwI0jcPmvDMExN/DmOUlaTzAxo5rWb5CvXvvjDpMOdi7J2yUbIzI5w7LNqunGoI5o4J6qKOWUXawu1IR/eUZ3eT9F4Xj2JyYxxJNitK+Tlcy8L37gfJehcBYxJj2HVjcWbHJLSvYwOA0sRt/BaHZtr4ztKEnVkNt1m+HovhaR8imNPTdAT8ytUwsPqY/zhQPqtfWFVfTsHpef1QGDtM5aoXBV/vBTMrLNtmWXyX9Jf1RCGbrKClDQNVr6kxmDlk1EktM5r5H8uFty51r/op6SrZVQieMSBpNhmFrpQtVFX4aIycmSW3RguU2HVtXPEHVdIIiSdQ6+nZVa8zOp3GB5D2C4sd1So8Xw+Kc0HPvUkAvFt39lymB0mYJ4jZ4PZcFBiFVHVeLw8vEscr43smkuBrtZdNHi5p6d8ldIzmMaeYWDRG3J8N61sz+9Z/jddvM8vfdcHw3NlgfWWvaYmy7N84exr8trja4K1DCYTPHVEKtzTmJVEz+6HnBaG23EICNXrlePMXNFRw1xoY6uFv4ZDxcNJKuCsgNV4K/4obzCPZZuOU1FjjZMLmruTyPM5txkkH+Nv8U4jyrF6jxldNiXLETJTfK3YfJZ8khXQzw0lVS8uNjImgkFxPQHRc5LoVxdCbKQcyEm8ud/1AQgPcfINt0eQhVVq9K62fMAEX4FiGvcb91UJN0eQ5cwUmESczK3L81E577nMdD7JElpBv1TOc1xytborQRF0DtE+Wy1OHaHDsQrnQV0Jle2J74WBxbmI3Fx7IMy/5U31XXQYXhFU0vg4VnkaDYls8h1QScM4fu/A8Zb/AGZNP7k5Ongy5Xzd0s72+h1j3XTHAMGHrgxeH5gH/BM3hrBp5WQxYliQlkNmB8ItdZ5HhZOfbIXMs9xTiK5vnCVTAaSqmpTI2XlSGPM3Y2KawsHdwkOcpISYnGztwrMdTc2LtFTJsExe8ehWYJhoXZJmZfzDWyJ2gsq8b+W4vYBrqQpTICA7uFhRNc2/naCOxSqBUS5Kua7myHI2Rz7kgdPkhYC52UKUUs742kW5fwi+y0Co6jwlUyq5YkyX8rtjpZNFHVVsrYKaGSaV5s1jG3U9FSXlDJ2E2fkIAvYd1dwatreE+IBUMdmY43zHUOH+CWJOGOEa/iOrfSSTijLWF55kRuALdNO66jAPs4x/DcaLJK8igM7I5jC4xmVm+oI26fqjH2jMq5jVcmlNSw8unhykEX0uStSl48xalpah+MYS2WWmi5rXwHJH8iFOTL0E1DdrNYBs0CwASbKx2l15dw/xvxljUrqoUsE1NGcklOyO17i+hO3zXokFSHwR1EzPDZxcskcLhULiHF/uXCJsRjgbPyhcxk7jqvKeGazD8NrazFWNkbLLLKY6drc0bWWOn8f4LsePscwcYKYXVczJZWHl8uxDh7+2i4iV0GGYZFVVNTzxUD+p2jsP/ZIDso2YiKiF4a46STPL8uUkiw12sAo+HaGStqKvC6SjieIpDLNUNcDmHRg+qw6KWqxGsZh7J8kMhMsjb6Gy6Wk4nGFMZFHhuRzZMhdTtDRKdbAjfcqjusLxammZ4GSpzVMAAkAGg06LRble27XXXG4Hg1bw5gVfjOJQ5qqpaJB+JqGHqe263eE5p63CfvKdgDqiR5FtrDT/AAKkSNXqj5eiF5suX4440xPhLwDKLC45nVBeHOkBIuLaC3XVaiR0hMzOiNozNBWNQ8Z4HU4RFimIyspJpRd1MNXA/KyoycfUNVKKfB6LmPPWc2FvYdVodVksNlE2kkd0XLM4xxUS3fBTvj/JYj+N1ZZxtVW/+WxD/wAwokOhbSPjeC5S8tp3auRk4wxmSVro44I2DdhZe/1Tu+0RuH1DI8Vwe7JBcPpzqPoVmYW3TSAxut0Vavr24bSSV0r7RRWzHsCbf4hPhvFnB2LkMZiTIpT/AFc/4Z/iuA4949LnYpwzQ4XGyEO5QqecSXAG5IG3RSlp6IKjmMa9r8weAQRsQUxlWPgVS+XAcNkf6vCxj+Cu8661TCV0qjdKfzKSONjiLmw7qdlLBIA7LcEXBWWlPmO7pCRy0RhtP+UhE3DoA7urYyTUvFaaK3mbEJCbqZrnlWJ24VS8RUkEj2mqq4CyJt9bDX/ArQfTwH1MSxSilkjAcDYqZlVKXXL7lWxSU4H7P+KLwlMf6v8Ais2Im1rybEqSrrIIMIxGeZ9mMpJCT/sFSx0WHj1xvaO7SouJ4sPo+EMYkLvKKOS5+mn8bKwPHvsalfHx9Rhn9ZDKw/K1/wDBfQBcwHzFfN/2ZVUlLx3gckbrcyUxO9wWFfQ1ZIWPJBUlpYmkhY31qpK8OBKrurCBc6lVn1ckhve3srEJMtCmN5m5zpdWKyOG/kasuCflODr3sVZM/OWaQoizOL6BT1VFBNAOTbmHW5VQC6uc5hhA0BA6KjFqcPngfabLbuDdVzTP+BXq6N7nB8bye91XjkfG4F3QqwBho65x8jLD942VgU2KxszPhs0dbhTurs7M2yKHEpMnLsCOqCrDPndZ79FcFHRSjO9mpWMJQ6ocWAMF9grrJH5L3WpgWY4qejlz07LHusypbUU5fMWWY9xN1oxPzm71NV0fiKcZFi2OLFjrJvzK1T1MhlFyiGFv6ZR81HXx/c+H1GIzGN3LZ5R7rXJcWBxHxvi9NiBocHmjY2IDmNygknruqUXF3ExaJBVwSg/9y1cRU1BrK11U8lr3SEgtNiFZkxOfKWtyx33t1K5ZPTjDsjxxxJHu+k/2ogquKcXY5iNBLRVElPklFjkjsbLjDieU6yg/I3R/fr3OZG7OWE2tlCy1WPwdJDimOTUDKKPFHMiYLAN0IHzWbW0LYad0j/M/82a5VDEcQfRubGy9ni+hsqTcWmkfy3Elp3ubpeS1i6bPzKSme7/VAKJssMErTI+19lFLN/8AAxL/AKsWXOR4jVyNDuRGPqUzZ15O3DGOGdtrHbUKMkNFw1p+oXHHFKwacuAD5IPvOsO3IP8A5a58XbxHXkh/qDFBJA51uRU8sD4S7T9Fzf3viPan/wDTTHFcScfVB9W3TiniOl51dFvLC7+zJr/cibWMkdklnA+ZXNCuxMj1U/8A6X/FBJilVC0l8MDz/ZsnBfEdfngt+1B+RSAZbyXXI0+LVk+Yx0kAyb+YhXoMQxCRt42Q/wCzKVOCeI6INf8ABG8/RL+k/CLLmvv/ABGJ5jfBZw/fKL+cdWPWyUfKS6zxXm3+ZPJ5N0vAPk1exc9/Op/5pP8Ae/4Jfzrk/NJ/v/8ABb4J4jeOFxjoW/IpeBcz9nUG/ci9lgfzrk/O/wD9ROOKZz6M5/8AM/4Jxk8RsT0E0jTmiDj3GireGMH7SnkCqfzkqxvDIij4jqJHhlphf3Tkw1KepMQcaWrnp3d+YRdXJcWx1tA9kOMvnY8Wc14z3CzXSzztyvN790o4ZmD0X+Sc2vy/BlYDMwiWknhbLY3AKKuwyuqoeTGMjb3Gl1eo8FZFUuq2Mc3Pfy2sAtEROB1LR8yuninhYuJmwTEQ6zcjvrb+9VZcLxOEjn0UoaeoGb+5ehAAD+qPzKYxRn4Yh8nkJ4rPhYuAkw2ZsBqWOu0bi1iFQdlXpBoYTfNN5TuM+h/gs+q4YwyovZ8DP4LUbLc5wcJmb0T3BNl1ruD6HeSYho6skB/vU+FcJ0DKp9RDWGUsb6HiwK1y5MTi48DIu04bdRGivSxubI+3MB69lj4nHAMRLo8NjZyiQb/EnpKiSnrGzQ5Yx1YXaW6rcSU6CppqSMGdtDAHD9xFh9VHI0s8LC23UBG2aGtgcYnh4tfQ3WfhzyyrMf5gsTDNNWrq209M57mR2+S4usfnkdN3W9xJWMgiFMR6he65mSZj25Wrmo2u0W3wjhbMZqnsmmAybNI3WLQSxx1kDp23YHahe6UVNSS0ENRSUkTM7RqG2K7a2WHBgEcLQxsDXfNVuJMFnhw/nsiDQzey6nkSflUNbh7qihmYW9LrbNPNWyask7G69Fw+npZ6WOa17tC80YPO5nY2Xf8ACZmnwsdchstTKNqBtPEf2Tbe4QOgbTymqwpoheTd7fhcnyvHRM55Cyjm+O4mYnhfPkbyqqn9TW7OHv8AopOAJ4ZcGNKX2cwkrUxTDYcWoZIHvdHJbRzd1x/Bc5wquqKKp0Pf9UV1OJ4maJxDkWHNZMG1pGrwbFchi8k+JY66NxvEwghb1L95QObEz0dkG+X6oC9vdPyJ8t3sIQGF9/SgWdqcSAqMxSDonbG8bhXyZ82HxZw1BjVI6WO7ZY/OCDuvLTBPSvLJIjcG1gvcAHelcFxbRjC6t9RK3ySusD3UlqJck+qjkYWPjA+SkOGwT0ZnErhboFsw4dglTRGtYQXAXcDpr2WFynyNMcZeG9r7I2zMt3ZWq/LhNRHT+KHLcwfl3UkOGTtfnaLrVBlfSugyNj9wg52mlihfmMYk+avU9VC6dj/CxjIbrLmj5MxjvfKrFFJDz2se+11B6DhfFODx5KV9KYndxqunhqIKhvMp352rgqL7upJWzSMaR1JXeYfVUVXTB9JyhbcNKow+LxymxTfJUnG88E7diLLe4iovG4c9gbdw2XA0OMTGqjpJztJYrcSPT4ooZqT8eMPb1BXG43VYDBzIaKlMUhFjYLpp8TpaKjBlkBaegK87qqaeprZH08jRHIdLrA5+UASucwqNheD5loYphlXQ5XFgeHmxIOyrinJOug7rMwLFFiNVQnPS7q0eKsULSwtIv1BVBpgj/rAfkpIX0odm6rlmLMmLYzuytkCjOI4jUNyVVfIR2R3pzu+wQzRU+S8UgeRvZZ5jl9YWg52kvGw6Ks4a3RZiiHmBauzSNtviUkckbRa6iIDHalR5rlBczx9HIJJfKfKT8lEL3CnY0aOO6BmsJGqGzutgEUryzYG3dRXzHUqoljYHOyNN/daEJZFlzAgdws8kQs0bqp6ds0kX7QhvVRR1EnOm8g8o691Vmdy9mknopbhriwdFOSx0IYGgIDoyxkIkm0PZU4hBLM94bdwO6llZJMGQB9heyUdK2lPLduEE8TMzwtKnjFrBqpQAC1lqwxMDQO6zkIay3hixxIBWO0CDMINRsRdatcQZAwHQDZZ5sfIpAhEj4WDykknZBOY5MoLcumZTzRNs081oAtYFOWsa3meSRyWK9K6M1DIWNQ1tJI6qdsB7q3TQiSra8sAt2U+Iwu5uZwy/2lqxRpsHje3PJVsHsFNVUsdHEYuSD7lB/wBVLgJbdC3YqtHHI5ufxUvyJuoDFOcge1411son5gbFI1Y2kfcKQQQPIfcj2Vlk9LE6Rz3v2A0Ttlt6SpgWtBydlXDDmyhPJaK9rpzJduREQxzdvMmhppZSQGXCzaDyUssf4ZIkGp91HG7Lp1RZOUbsfY+xUZOpd1WxdkLDTWbIy46KCMMZfz3uoLuKkp5RC15eMxeeqgj2fchFuU5kzEaBShrHvbbQHQpbUnY27Dbfp8lM6WdlKJ4ZCHA2+iTomMtkJAJ0Cn5cbKTlm1ydLlBVhqX1AJnfKQNW5k4AeQ7fRPJTyu2cA3q5WIZGQMcCQ+4sARZSQQlfLEYeWfIPUAqsga6Mh/RI1NRG0jUNI2CTHjl3KkCB4ExaGmwYLao2tjddj3vJBFySos45gLmE6qQyiUObkGpQWZ5bOYxhFrbDooJ5ubJmfqbWuo7WN1GxuaVrHPyhx3tstA9N11/DMPKpm99ly0tLy8vnvc203Xa4LBlhjCJKnx9K9tHQ0zD+0kL7d7BcfFDHKDnkDZ/crd+0CpD8Vgp89+RDe3a65eNx5geoi1TRwO8szC50Zu4joE8jYLPOexGu2iRJY0kPLSbA26qOMMHkdsqBjhkmdaMXI1ViON0lg3e9rIYZGRS52GxsQgivzC7O4OtoQbWKC7NQyGKLP6wLO0UtdWweCZTsfewuFLR1lVlinYbuykOAGpWLV8sFnL9/70EA85PmVpoytbbqFV5bw4ty6gXKtRm7W5emhWlaFKM1lZllbmaI5A7ILEdkFK0ZM3tdQSj8dz4X3Mlzey4q1ImZoRJs5DheI5TVx1EAmbmAc4npfZUnVc8dO0PA16lHhU1O18UM0Qy5gZXu2v0WoJW8dra2vbz4aWSKljFo8uwPclet8Hvr8V4eo6qqltKYwJGPOvsvKDiceD1gwnFqKSaOCUTBrJLcwbg9rL1ej4uwjEoo6rBpKN0kgvJSvm5Ut+wv9V15ueS3VwzQvjczqbH5KnPwzhWLzCSvg5k7GGON9/SD7LRrZxI+iY+F1O6eQi0h1vb2TOqY6GqnM18scfMsBcn2W4Rn1tczCKOaTGKWPJRtAjIPqPSy80LKfiSuqcWxmv5DmNzgWuCFFxRBjmHYiylx7miaraakB0pJaCT0+ixZpWZTG53lK5zK4pZ3iqmfNHLy22v5hupJMaxSTChg0dU51Pmz8saAn3WVdmexarEb2W8izbS7UsmpYqYCZr2m1gOi7XAcSwvBauomNaAyWNnMbGbgnXouDjmkjj5EhYGEAgEXSimY2YBurXbjuVqJHtNPXUVW9kdPVxPfI0vDWuuSFNqvHaSvOH4jT1cZsWSA/S69A4240gpKKno+HjFI6tbndLr+GzTY9DutW5S6LX3S191X4DmlxfhilnqmeaMvjz31dY7lbjsPiJ2W7aZQbIT5XJnR1TBe2i1hhMtrxuCRw6pH5T9Vm0pg1jXS0skc4uCDoq9JVQUtAckjpeWCTfv2XSDDKrrTsPzKya7C6mg8RVPgYyIi7QCLE9ilnFVc+TEqCRwElPnb5XAgm3dcjTVrBK2qL4TLGbB3Xcf5Lr/u6krKB1bFVyCVjM92u0YbXt8lw1RhvhzBPHUEw3AM5GjiOoC47WsWjTYhRxYwK6q0iL+Y5jBcX+S0MTxDDayM1X3s6QsBexjYSCDbqucndz8VPOla2N4Fnu2sp+c+Bsoo5opRkILjsRY6Ly47MmlnAc7MBlLTa7iruH1ENDC2p50z+bNy2vk2t8lRoPwuF5j+YkfxU9HXU7DTz1UHPMbrtYX2Hufpou2WfAdZWQ0VHFA/mFvNJkkcd/kFPTNoZLTMZPE4dJR/guSxrGH4zXUrIGcuGJ+oc6xsugosUZiT5cseRlOAM17hy7atsTPkkw5qZmK4HXGaopRO8yEyStN2OifsPbWy43FsUqoZZ6F9LHA8EscWdj7rvsW4n5VNHUYWyKoaHfjseDcD2XBYxNS4jiElVBTujgkdnyuOq1lmnFjND6hrr3ys3VeU6Fw2cpJpXQvLGHRVXnOPe6xDQonWablMZNcxCizBOXLoqw0tuC5Sl1h5NlVDhZPmPdZmD1FJKx+p/RA11vSgeb6pWCUJC82R01TNSVENXAbSQOztKhDUeV1r2Qel8M4lDBiTXMf/AEPEm82NvUSdb/VauGRyYhKcZr5JC97pI44gfwxHqNvcLz3h6d8tFPhfMtLEfE05tqCNwF3OEyYti8DMSw3EIbSsySU07fKx40OvRcYh2mbi0+HYjHgeIYhhs00vg4I2VMXMJJjvoWD+C5fHMampIpMZk0rcQvHTxnQxx9X/AOCu4lUc6rfDXVfOhpjzaycNte39WB0C4jFa+bE619bK3LfSNgOjWDYBWmpynDHy96oCbd/mpi8WFuyg6/VPfTL1XV5zuc4o45ZALMch9TUcUZA0KgsxA5BfdGX2UjYjoQQUYijcDmb5h0WFJlQCclg1wFge4T82RjgQ+6aQMcLPZq32shjGawVkakVTmafJ+iesk8TTy0szWSNIvFK/dvyVMh7GgN+I5SmqJHsDIy+4BuCFynFltR02FTU4qYozCWN5QDXaXGx+azHVL2sFNLJ5ySJBbtsge9kFPlLiPkVWYWPaC113KY4jUw7EG4Q4vgqJ2ucLOEb8oPZJvFGOCDwjK6QxXJs43NzvqVkvk1UWYrrjA6msrWVuCNbEJCA4mQOPxgFV8SxGljoaJ9DHGWXPMY8XIt1WKyp5UDmte4XNyAdCVFJIxzsjGW767lagXG1fJqjVUbuW69x1spKesxGqn1qRn5glDzoQRss8PAUsEn4rfndUdRWcX4ziEJw6uxFrjMQyQubbKwLusIx/A+H8GhwqlfPXmC9nZQ0G+u68qEIcXVDgC7rqrv8AOTGL/wDyqD/eKDu8R47xyZrmYbhdJBfZ73ZiP8P4LD+8sYqPEVeOVQm5bs8d9mgdh06LA/nJipH/AMpgP+0UE+O4rPTzQHB4Y2vjILhLsO6DqWSQyMbJy4yHi4NuiEvhbpyGfouch4hxWKBkbcMimDGhgcZCNAoP504lzuSMEje/8rJcxWpkdJVYhT0UDqmpEojZuQ26pjjHhu37SQH/AMIhVqqeuqqN0deKWhhk0J1c8+1lSYzDKNz3UlAKud/9ZPoL/LosWtNj+eHDxPkqZD/5ayYXMDjDSzOqHSG7Wh+aw/wQtlw8y86Skkoqk7yQtzR2/sqwYcQOtMYath+KC1/0WuRRpqNsTM+K1FHSMbuCRJKfosfFKvC5KcMoIZCQf273akdrK2I6eUluWPMNw61wqGLwshiYQGjObaBQbeE8TsoIYoo6moo3Wtdp5kchHUjp/wAV2FPxjKYnSOw6Kut8VHMLn5sOy88wiKOajkY/4Hdlep4Y6Z3Np/w392iy1EpMOxH2kYX5gMKqw4MOdriAQV2+GDmUFFUt9MlPG8fovGpQZ3F8r7uPWwCt0WM41hrRDQ4xURxs2jLrt/RVKe1sseqPl6XGh7ry+k+0jGIGnxtBTzP/ADNu3+CfC/tHxWGtqJKhkczZ/wBnG42EduxXLJT8d1NVh32gYNK2qjbJFEyzns0bd5GvtZd2cQGH8pmL1VGHT/sjHE67vpqvHuJsUxHHsUNVXGM1EbeW0M2A7fxXb49xe+d+HfcPKLDYF0jQ7K8W0Pb5qcx6IY/wiVETkZdeVScbY3g1dz43Qy88AuaJC5g+XZd1g/EEHEWGCvgjMTibSNcbm6YzY1ue1ZXG+IspeC8TkyCUSRctzXbEE2UzpiFzn2g1YHBmIMJ9fLA+edbHlvB9bHhnFGD1k+kMFWwuPYbL6GrKiYuztdodl8x/l/tBfQoqWGnhvJf8KP8A/cCSLJqCNymbO0/Eqb5GOHqUecjZy0NQT/mKt08zXNtfVYmeVWKaV7UGxm826PLp+0CzPEP/AOSl4j8yzQ0DvZVaiO72npfVMyZ17J5ZRa5urEB6hsccQdG++qej1ksg5L5WaIoqSRv4glGnRUVa6X+nPAaBbsFJSyl4ylW5IqCKkkmmpeZMTcBYWI4h90Yf4t4Ie/yRtOlyksLOJcU4NgtQKerfM51rnlx3y/NNF9pPDjYwMla7/wAj/ivMKvF3trXyMLXscbkOF8x91JS4lhdVZ0cjaeYbwuNgR2C5ZPRhg9BxL7TIMhGFYW+R4+KXQX+QXIYzj/EGNRPZVzu5W4ja2wBWeMdoS9sbaWS7zYecIZsbpY3OZJRua4GxBkC583Xjgynx1DCdLKrIbyWkBHY5loz4pSygnk6exCzedHLKGh2pPVahZ/KrZrxOktbIR0VxzHCpiygEGxugytEU8Lt1fiAMUZ/dT0ZseKwsmZFK7XK1beE0dBT4Y2c0sWYAkuLbrKkbmpvN0W1hw52Fvj7tP9xW3NGzFqds4gmZHG1/7NwFw5V8cpKcNZUMha197EhZGEvnoxTVmJFklP5ooyf6o9LroMYZmw5zuxBWjBzRoiJC8klp1t2U4oYi25jd+inOrFcibeKPzX0XCG9jna2lEbWctvUgqmI8sjXOvv3W5iUQyEB4OR17rKlaLBzXA2K6sNaOMXDnNAWbi8QZVANPrF7LWh1ja49Qq+KtaTE87D9VwifzOzNwx8EMsrZ5uXnbppdalDXYdA4unrWO7ZW2VYUg/KFLFSsvsFvmzwHV8morDNS6xPA1SNLG9uUhWA1oTk2WLaZhwcyDQEKGXA5GbPA+a6ala1zbo5oGO9bbL0OLijRTNPnyqajpbTZXu/RaUsIbKWnWxUAAjmDmhDHLzSz0zxHeEkuUsMVog17dVZDRZEV5spdled9bDFzYJyADY3F7KBuIYq//AK7/APitMwNmo5B2KpQ0rIgRmvmUtAiTGXi4qhb3ITZMYP8A1ht7dwiniZcHKlh7GCrB6EK8WLR5cY/17f8AeCaR+KstmqR9CFsGljcf2YVSsgDTfIB8lujHLzZhnxH/ALUh8VXjapVsxscC0tUZgjYzyhZhuYb+DYKK6GKeohljm1zGR1mlU6zE8ShqHUOB0Qcb2BHmv8tF6PwyyrxLAKY8uMMYzJmcVp0+DsY8PZHFm7gBdMYcpl5JNw9xviDB4vDLe1spW5D9lVLPGx02MywvI87HRi7T2Xpc+GyzRF2djLd1jyl8b+WZ2XHTMF3hxlylfg9Lw/Tujkw+zfhljGhPuuXD8uI52+l+y9UDJC0smjEjT0OxXk3GFHPheKz0zPKxwBaPZTinJkcS1XicQLWPuyMWCymOc31OTvLnPuU7YmGcNOyzOLUZWMkW0XrH2UY62pw1+GVMmsRvH7D/AJC83kpaVkIlYQfkVe4Gx37ix1j3t/Cnsx2u2v8AxWMfyt8Xt75YwdFJE4OBYDvoq72sdZzCC0i4KOHIDqujDy3HYH0mPVEb+puui4DnDaipgefXt/FUePaVkOKsrRvKACoOEqnlY3Ttz2EhsukMPQnsN8uVQStcDqFtSYWTd4naQqslJHH+0IPyWYyTiz4fVsuC47ozh2NR1EPlbUak++i9IFMzN+G1cb9qOHzCipq87Mdkt/z8lbZpVdRMgfS1AN3Sau0XTtAErVjQUrqugoZB0AC3fDSFwdlS2pWzVRluW6cMBGYLPkYynkElTLHE3942QO4t4ZoQWT4xAHDpco01CyyhkyhY832icI5S5ldJJbo2IlQN+0LhWX8N0kzb/E6MgI02swzLmePqDxuEGQEB8ZuLnfZacXEnDs7gyHF4ST3uFX4ip4sSwx7IKiCS234iObyyin5TCHtGU3BVR80jZXFryCCpZ/wnGIaZDZUZHPznzfqjpDSZjNZHHbmlQvxmrOxy/JVYI3zvya/QrSjwDmbPZ9ZQsNMqV4e+7kwYL3C2JuGZWMzsmjFvzOWbJA+nfke+N/8AZN0DBl/jk/VbuBYzLg4FjzB0DjssNr7IjLdB69w/jlLi8AygtkG4K4HiSgZR447l2DHnMLG6wG1VSPRNI3+ybKanknqXtM88kju7nXSJSYbE4kEYcx5c3qCVUNTSN/Z1Rjl/KSmljq4WF8dVGW9llTC813NaM3YJaUs1GJzStyPlLvmqjquT03WgMKjlh5nNDfmsaU2kISyk+djvM/dEJ2MOZqrjU2LTZSeHZ0bZShZNSZG2a36pRTmL0i6rAFnRLMeqxUDHANsxTtJROLPSLoQL+ldFOQA7zWunGp8rQo8t5MrtlKModYIGkaS4EdFNKbRZ0DtCmmcHNDBsgaCYuGSRxId0I3SkY2InYdgFDFlLtOiaVxeblFTwnM8B7QQpnNkA02VWO41CnZUSMtnfeyAouY5pzREKWKPmNcM4bbXVOyshN2Fh1FlJHSiYjLIG3UkAyPJ57gkdkTRndm7o3ZIWvjyF3uE8DSImvyE3PRBPRtIlGi0mvyjO7YKGCBjWB9tVPIeXTOOUFx6FYmRkTSZ5XyMvZx6qNpjLHPL9R0UQq3vc4ZACwqMDLmPcqwE5rHyZ3gH5q5EIeWHPVG9xexTHmStLA0lo6INKnZIKhkzGaZxlKs19QysquS92cBNRT0vLbDDBIHM+N50WfROghxD+ln8F7vMsKtQ1LqJ4v5ISbZBsFNiOFwQs59I8PiIuLJ8boaaCdppJ+Yx4uBfZZ1pAfIbe3RBA6ny+blAJh5FNNLIWhj7fRDFC+YPcLeTut2iIzWGWyOF8YsTugaxjnkSX9rKWKOGNxc4Ek7LUyAOUuzhpA90bJHhrrPI06Ipo/I3I3UnVRxRMkmEcnp6rIAw3AdC8k9ipIaWefNkjPk0N1YLRHcRxm3QqUNlgJGlyLm6Cp931J6NHzKaajlhALnMeDvlN7K/HMBEZKq1lXM/LkLI3nL0VgUA0gqZmcewV3msczKRcKCSB8YFruDglhx5XxzM8xHRTyv8AEgh4Fj0CpxztjGR7TfspYZpCc7dG+6xYLNI5ltQ3sijbYjzHXume9w2v5+6bmPaQQB9UZTNvHewBuOqqH3ta/RTOeX+ZRDKXZLtB91qBBJlPoafqjifIyE2s0vda5Cklu1oy2OXsoy7YOVaO0RSFzX3sB0RObEbPj0J3ugvb4QhG6C3TXfVRR/mcF3mEw2yLiMKhfNXx26LvsNFmNf2F1pl5/wAUufWcR1U2QvaLRi3QAKrTxw07S0scf7SOaXn4hUTZbcyUmyrzS62G6NAdcucS643ATZ82vVROMjiXJRXJQTMy5hf09VOWsI5kfluLBVjpb3U8Epe0tsSGa6DoiLrSYI2PYbW6IZXiuka1rA0MFgAFmiV5nzE6K1GZP6mbXtZZRVkEoDpDq+QkaKSljLbX3V+Bhhzh742SEbHqoIG+YqxK2utOWldJ2CgiqZJw2C8bdD5uqPEs8GGHIfW4BUaQ2LZCuatKYMdRBjCC6MWFxur+A4PBiVVRYdL5TPUM5jrXIA1QURpJ4hkbd8m+YWGnZbOFxU9I9uJeN5c1O/yxsdZx76qwzLV4kFXR4/V4i6jo4IYIWQU75xmJA2AHS68+q5qtpdDUM5bxKXEZQCH/ADXt5wJmJ4dTCql5z2TMqS/cEDof1XCfaXw5KKs4lQQf0eniDJOpL7kn6arrxYRs4lrcU4RfXV+IMfiNFVQCib8cYGl/e4XVcMcc0OM4i6TEoxBWR0+sINw4M1vt7Lx1kk9NVMgfmheRcgt1t0UzKt9HJPNT1DmvlbynOG9lZ8kjFc4j4hr+KMcqMYqnh7AckWUWDWDZUaZuaS/LEluhVekLIyI3mzDuUb5TE4iJ1j3WJboecRGZ8bADJofZDfzZ2Ake6jjNwrTHMbB59zoqq5FBLWUrpjSyBojIJ6hUGHI7MpTXTsi5Mc7g0726qFrgEgXC5hYGncjT3QslkZ543EdCCq8crw8ebVSGMi73PvcnbosjfwPiHFaGnfh8GI8mlnIzNds3Xf8AivWP5wYXgGEUYr650rjF5XAXzD5rxKCR0IZIw2LDdbMFFX19IK+GAeFk2vJ/d7LTDr8QHEldLV4rh2JVkbHuD4WtNgQQux4RlqpcBp24lN/TQSCHn8Rw7kLzCHF8X+7jQ+KmluLNIGgHzXW8P0jOHcO+/wDFamRs8jQC6f1m/RFdOz75gibC8w5onF5lMpvYm+y4rGMTwuV8VJJjktXDGSZWWJzv02/iro48pK+mxClq3tDD5IZA3LcG+4XAuFPT/g08kkrh8Q2XLLY26d2PGjoJqHA2NjMm7nNs+ys4NgUXEmDy0kdc5tXFkJkeC5gPYBcXDNBHVCqqWGUgEWL1vw8U0kNLF93U89LKBYtbKMp73C480pnYpQz0mIyUNXKz+h3ZIWv0d2UtZ4RlO+OjjkaRHcucb3uoTieHh8pZRTCWc3cMwddBPi9KyJ9JS4dK18zXMM0x0A9gprx/M6tnC4PF4CKfmNjzAnM7YWVLD5IHcyOoY/exLNQ23ZW6NkIwKJs/oe4MP1Kerwevw6WU4a+LkviDC54AuPZdNzmg8bSwVBqHvnmI0N9LhScSYxHFw1JDSx8u87BC1ps8ste57qOjoWudK7FIeTnLAH5Tb/nZHjGCQ4hVxywM50OWzZWPsLLGueKS4p8c7XuImLb+o5rao3SskFrgaIKySlbJJFGXmNhIBJ3somtnaA7kFgcLjMLXC36iKqhgIMjn6/JZ7wAVpTm0BYWC7tyst51XXEC8De6FO7dDfWy6w1KRu6ZM3ZG7ZVlGiTgXKRagePKjzZfSo2dkbcp3RpYo6iajqYauI2fC/OP8l2UGJ1uHPmZhuseKND4SNLPN72PtquIyrewfEJ6fh2umZYvoz+A5wuWl+/8AesZw1hnwQ8R1zGBuBUr88UDs07+kkv8AwWMf8E3+V/ql0SlnLmQ2TphslmC25wdSwus6+W6hbspomNLT5rLBC5HKQB7Kds7HC7QAb791QEnlud0VOXvcddAFmYVbku8a3+YQwuMflfv79EL5TGwl4J9goY3AtMrWhpJ2vsg0433dbN0/ikbPzcy+Zm2m6oOqMkvMephPHM06mxHTdSmUdZO9+UOJLB1sqglc03G6GaZ8tyb5ehsjbFCBo9xP7y3EB2TWkDiBa26Nz/Lf3UT2tZbMQ7XoUT3scMrBZAzpDa19Eg7zKJ2bojGjbrawnZIevRSxnzZ27hRjKWh3XqmzFuyzRS1z5O6MTSnqjw+l8QRotZuFwgLmrMa+rj0ZUNDe3L/4ppparlPzVAylpBAba4Wt92x91WxGhiiop5B6hGSFq1pSopTJyue18kNjeNul/qtSnkqJMzKOmipI29GN1PzKjwCJkuHh5F/MR/ctWhhYzm2bbzf4BYtuma7Dpy/Pe57vF0Ya+ncY3zNuNw2K61Jvwo3TdGC5QxSYbCGxsrohnGck7kqWkyzMhqXhjJ4wT0fFYqB+GTRVV4nCN3dui1qt+Gz3jnr4HZBdrr2IKGJ3NEMrt3t1VZtlVNZUZg3FKSOsA2fbLIB8wsfEZDIGtu7JzPK1+7R2XWVNKyUjM1YvEVCyGkY9zbXctRLTPoJ3xROySBl3Hdt7qz4yo/7Sz/0yo8EbDJK6OT8unzWx93MOzFbSmWcRqGn/AEiH/wBIphWVLvO2WIj/AMMrY+7GAbBMMLYd2KWvFlGorC0jnQ/7pQCaXv8AotZ+GRW2APuVlVUJpnkFzS3uCsTNszBGax7rdweBlRTPfz+SyciIXOxJ1JXMsk8wKtR1bxG2F+rB0XOGHY4hgXhcFkhZRGM0zi+Op5osQe66jgOlwOfB2w09U+GqYfxuY6xcT1suFw3FKSLCZKGEyiUyjQi99brSosar8WqRR4jPEymgJkdLBGAQRsbjddLoemvwaMC/iZCuU+06klp+DXmmu4Goj5lzsLn/ABstTgaqnrYqwMfUzUrZPwZ59PoovtZ8bFwd4SihEnj52U779tT/AILtEjwyZ+SBx6r6LwXh2OvwbDq6TEAOfSRP0H7g9185lvMcyO9r2YSdrr2r7Oselj4ZosHfWCqlZWSQtsfTELH9NVJ8x2UfCtCBmNdM76BQVGDUsAux7jboVoDEGRxFmeypSV8b7gvuoKghjHmsgdYO8ikMkb/JntfqopILC7Jsw9itwwTucOiJpneL8slC2ORxysJJVnlzsaLxll+6oASZd1IKxoAa5oNjdVpIpnG4sPqkxsjfWGFFWhiAadUbK5h6qlI1knQD5JhTuB8j7ojSNSAsrHMNouIzEzEameJsW3JNif8AmykdSVrWl97+yoxYnSTZufNHCWG1nOvdBzPFPA2E4fh4q6Kvq3+bI5r3aWP/ALLEpODMLqOFKrH3zTeJjkIAzaWvay7jG6/D6vC5KSCTmvO2mi5fB5nDh/HMOe0FjGiSxGxv/wAEdrYs9BHJS09R3aGZhuqb8IgdXOp3vkNrWcSt+NrHYDG8tvkcLfqsuSW+Ic+3QC3Qrg6RLJrMPjp5THvbqq1Oy9ayMd7rWxLz1Esg/trMaeRUsqAwP1sQeyy01IcpFYx9vOLgnopKezImgG4HVBTiN00heQG5NCSpIiMmpB+SUi085aQu9wFp4BLzKZ8fY2WYXMNE/O8Mt3VzhqeAyzMZIHfHp9V0YRsxATNqKGmwN80DJLSDmCxK0MRaHUMkTBYZQQFi3o6PHavxWIVEDS/mRtGgd3/wW5zIaikdNA/Ox7TY91pmPViNF2K1AZOW3ltB06hVWHyJ4MVpaMmObmA3/KuEOuSaWGYvvy4/lbRQS078jnihgNhf0o347hz3Zs8g/wBlA7GMOe1zOdKLi2jVphHTvzMBygHsEdTyzEedCZB0UNO+GzmxyGQdCQp5shgLS8tDuoXn/wCp2RM1jCOJpfII2bnZA0sDA1jw8d0UTstTE7N8S2LYpXj1q0IaBrA2aCQu6lrk8xyNJ/KLrmKvF6mWRz2Zo2H4Oy05c2pS48KPOw0hc3MbOKgq8cEtVni/Zdlgmrmc6znmxRiTIF0jJyt1tHBhGKNM3hpw/qOcBdNiFNg2GxtkmpJnXNgBMFzFBi0tJKHtsbdCEVVi9RPMZCGkHpuraOkhkwKZt2U0/wAubqrRp8KDA/kVWvTmhcmyqfdsjHj5AWWtR4vJUBlO+Muts4LlMcnbHJrMZhmQs8JPr15qjdTYdHYup6qx2POGqimc+Jt2K1JLmoWSZAdhqFODfNA6LCHbwVX/AKwRw0eCstN4WoPzlRT08DYM4ZYlNB56Z7PkulOMr5bhrrPbHPYjo9BUQYU+mc2SOcgD2SisYmtb0FkUjAYXXNtFaRmNo8Jccv8AS/8AeCGWiwlrrf0v9QjpTeot3SqxaUrNN8l3CsZqsHbyMOxGtDCbiM2LLq9NxvjDXZJKqoaf3YmXWPTEWF+6nqc+bOzS63DnKefE58SbmqMVxc36B1goI8Lwr1tqq1j+/Msmu805LnXQQ3LQbrpEouAyQi0OO4i1o6c0LNxmR2I8llbiM07Y/IM2+v8A7I5/WVRqAbF3ZW0opeFuGW+bn1wv2sVn49g+F0mGsq6FsjnRu9TxY2W3Ui9PFN+ZirYjC6fBZ2Mbd3LNgpkQ5ynEE8d45ru/LbdUYzGytYZ/Qx4uE2CycusZGeoIT1DGGplzj4llvk99weSOow2B9ILsyiwWjBBK+QDluH0Xh2H41iVNTAU+NTwj8odorbOL+I4/K3HahOTM4vQftGw3LhsdduYjb5bLiMNq/D1VPVF1gx+qrT8QY5XjkYhis88Z3Y91wnpgHx5FqMknye14fjGG4hSNfT1kTnWtYFNI27t14hT4rX4LUGSgn5Tu9rrSg+1DiummDpKiGob+WRmiyW9eiDIzmkcGtG5Oy5n7QsTwGt4eqKQYpB4iPzxC97my88x7jrFcfc1j4RTttYiN5sVhCPDz65/xeocsjqcG47OG4VHS/djpXxelzdiq9X9o/E8/lgbFSt9mXP6rkNTNylrw4aA3SUm/ddOTXBUqcQxCrJ8VXySZtwZTZVssA9Tmj6q/UUzadud9KyRvUgbKKOSif6DGPY6FOTpwQsliafXdTsqYj6D/AARkxxjMGscq752uPoDB2Ck5CwJ475nxh590JrJh+xyNHYFVXytsbKMP7rlzlniN+Ykk6klQyNiLvOy/1Wnh8EEzsnqU78MbnIiBHsQniStMyjNLG9ofSvl+RWgWYafM2iMbu5fdP4VlP+2jAB7aIiyiOxIPzWOUtUDwkk+0MZb35pR/cuHAfjyGM9gibJHGLMv+qTpY/U+QD5lXlJSpJhlJILUr5L9yq78Nnj3cFqiNhF2yt+hUEhscpJKc0mD0NDRH9ux5d7FWKuKkEYfDGQR1UMVSY9WfUKU1ub4QE5S1GMe9FDIZG6giyGXD6aYXjY4P9knS5T5bBO2fL1V5M0hiwydot+I757KCfDanUhrbfNXjWSXu2Ypc8vHqUspmxsijAZNo5SvoqmUZ4Ibt+anbTc6QeS91u0TWQRhjdFqkpyEsT4zkcLO7KNkT3OAAWvXYVW+K55IfH3CdsLGjzRgLTm5NuR17Ak+yYBw0IIKmhmFKS2O23ZMX5zd41W1DHF5s7x5UMuUP8uylL4gy2TXvdV5N/K5ApC0jQ6oWMeT0SY18hs0XRtblcWm90VNFRse12ttO6qH8J5hbqB1VqOpMEgI1RNiFS/mCMNJ3sgrRR1EzwyOB7/kikhkiNpBY9le88VO/JMWgdB1VDmGQi6gOIGJ7ZCARdaURpJb855DgLi3RVZ2CONuXci6aLTzdSgN4ziyu0mcMDQqceUlXqdzG2u8D5rIvwusMp7p6lzpRdmzd0TIs5GTYqrVTMiJbusSKcNFEXSEWAdqqbmZTl1KmmkcBdm/RHTMmlhdJpoOqsA8OhMs7adhjzyG1r7LToeHxaaOWqeya5IjaLD+9YVFJURYnDON2O0W3iuMzw1DJoGAPG5Ot7LnlIpyvqKF5p54QYx3VGcxvcXsbYFdBVGlxih8Wyzaj4gSqdDQTwysqhG14j9QO1lYlWdFKwts5yYyAK5j5b4iN8EbAwt0sFlGRdUG9wOqOCVrXaG191WkeSLIWO1yq0yuSsjDz5wfkhY10jwB3QdN07TZSmltsbJJMl9+qiMDROchug5x6I2Sgb3us0LLZuUMt0pH85+fc2sqzpWvFkbHkDRQV6mRwsx7bBxQiwLXjXW5HsrwjzMc/sq3OjsNCHg6u6LcATNNrkisD7KzAJJI8r3i3ZKGWGKAGaMveNnjZW4KeOSn5gfYk32ugpSsZGPxGqnNMxrDy3LQnjZM0sc+zugtusualfG4sd6VBotgrpoWzvjyttpmUD5LHI92qfxBEbQ2R/k7lP4uGV+d9jJ8lIZNlkyuLWad1Fm1B7BSPkuDY2Ve/croJDJfyjdCGPffKEwc1vqUsMU0xOXLa19TZKKRjN1Rte9jtALe6J9rWynN19kmhjmu89iOio2eHQHTAs1tuuwdOaLDZaof1cZN1y3C8BjjOcbuNvktviidlNw4+P4pyGD5dVoefxBwc14d1ujz3z/h5nHY9lO90biAyPKLIIgHODwfLdYlYOKCIxZ+cdRfZZ8bOW4szA2NrhbohjDHNbMy5632WNyXxzvD9ADuqqTLZoZ3Ripkij5EQAt2RlrWOEMjyxpF7lQxsYCfPn+qjVInm7ySAD7KSkmEVRG+QmzDfRTRxRl+bclE6iIcx7bHODoDsqytVrqaaEVUcZBJ6qvTDqlMx8VM1mY6uVmkY27QG7rDVGxcPdSsgYLv3ssmmkEc1nus0G11p4tLy5nMtc5bA9lRiw2eeN0jMmgvYndVGpSPfKzPStLmdCAtyioCaeKrfIyVpvcjSwWHhMNRTUTLVAbn3sdwttmIMkyR5Z8gYWOs7T9FiWZhuYiavDsMaYcbkcyUehpub9As+r4rxWopW0lOwRRMiMcgk83MHZZFRNNyHRxW8js4eT51WpZpJX8uQ3BU8TJji3ONKyOb7pxCCCFj5ImEuMZEhI6H27LBxh1BU0nIoYI3SZs7pzpI49fkFFjuM1+KObLXTA8h+RrGiwaFThnstxlyWFQhwuAga52bKRur0mSWN8bBrbP8AVVuVm9GjltRxMBd8lMMjnGHy2Pc/3JU8Lxd1tQN1FrJAZiWgg7bKDUr8SNdTwU08cDGU4s0Ri1z3KzRHI70BVmSW+IK5FOAwDKkAHjkhud1y462RxuAaL3+iDzSyAOFr9VJGHMOe4yjqUIhYzWiFlv8ADmL5pKTDYWEtPks534fU6iyrYNFRGkE1RRc2ac3s4aAf4K66ldM5jY4Iogw6ZDr8ljk7Y6zU9VaaTDjFIGiQgmEaD6o6+fEayRkdXWzVUURvGJToETIhAwMaxoI7KGQPDTd3nsSJNgFxyza8E08DWxNLBZz+pN1H52EsfHdv96qNrvEvDA8k9LlHV1vJBDiG26LkcRSCme0vgiMYYNibmyibM2O+XQHZRRyzSASxDQi9+4QxETSNY94FzqT/ABSmEsr2RyRvYJA/l3zZkcYzQeL9yMyOsgc6rY6nb+Cw8u99kchkjDoc7SO7Qt4fxNtytY/+a8AjF3GRht9UNJj7mUDaXEoTUOgkAjLSAWjpZT1b2R8Ow53bZLfOyyaZ+Qtc5ge525K7ZzTEQ6PCcV4ahzmsrKyozm/h3xiyVFh1LW4uK+N9qaP1QCIgEnsPYLHgMLKsTyAud2GqjxTiSTBa+nbSc0lg5krB5Qb7A/xXPGbZzxpW444ewigraQ4RPJHT1GcSR/6s/wDJWBiOImeWNr25WxRiNovfQBDXYma6odUSNyyPffe4CiBjcLv37rrEMAkmjILDHmuNDeyoTgse7MNArkpY4sfCwhtr6qvPqSXfFqtYiuBnPqsny2SG6Q1K6w0Tt0g1ItQt2VZSJibJwbXQu3RQ5inDk2UpkVKD1C2oHcrhCskt+1qmM+gIWH0+q3Xjl8Fw/wDe1Z/xW2WLm0smNw1PlsEJNlkLME4HQIEYHQLLR8xRxPcB5uiAC6dxsAAkAjlfIWs9N1cpiyOF0jzpsFUpRn/VTucx8dnEkg3GqCy2Vs1ZTwMJyyHln6lQSU8McsjJJw0McQD8lFCbTwucbWlZf9U9ZpXVDDuJTcfVYplG8m+pvrv3Txuf6WfVRvHW4+QRtLI/WAbhKEkm3Le5tx2TRRMlkEbZg0nXVREfiljdBmT2cw2DxcdVRLLGY5CyxLejrbpmk9FIXl0Qa2Uu11Cj5bnNdmcAb2HutgRsjERygt6oIw22psevyVuR9OyB5aHHILN6osIc9mtZ1A1TsPmUSmiaeUZLEgOsVJGxhlZT0sZ5z4xf8y6JmWRjXsdcEXuuawSCOqqoxMy7b3supe5kdrRn6LztQAM1VDiAcvDHyDp5SPYghaQdf+rk/RVcaax+CVmdh8secXHVIltU4UyHDXsduyU/3BatPpJP/aH9yxODjelqh1uD/gtyl0ln/tf4LKjnH9Fn/so5YIbTeSD090nDmMczuoTQx2vJMWDuYxYLUMyJkELWUz8kGre6gp2Z44f9v+8qlRuhqKcPjeTYkGzb9SmnZDLlZNXFkIFuXGbHf+C2lNGqqaCmcBPWsb3G5XOcS4lTVcMFLTvzWlDy7ZXx4WE/0SBuvxuF3/qsTiCVxnj5wzPGodoLDsqhuHZmRYpzJDZpaQuvEdxnZ6e64fCwHVWZ5tYXHzXQ00lXE1phl0b8J2Rps8u2qVrAu0BAvqqseIljR4mnIFvU03urEdbRTC2cHMLWOhWaWZc9WVVVPVc4SAWbnsw6KjNHPKBliI737rQxGkgw3zsnDmyAgNA9IVFhY9mcFJhifNBlyLSwGJ9XiUNCxrD4g8o3F7ArJkkAeWt1VihMz6yBsHre4MYegNwszDNOkxLhp+DVAw+pr4DNISY27XGlkGC4jJgj5o3UgndJIGGztWj9FBDiPLxiar4ijkqpIwQHZtAeirnFmVFRK+KkjjZK4EC9w0Dooj3HDI2VWG09VTVAhE8ecsa4EArmPtExagbwzUYa3GIpKxkkcsbGuzG7L7HpoVwFGMSnY4x1QbFe2Vs3LCU2Fztppsoi/ZHaUFd8ZWnN1VSyoqvEDTUG3vay6/gPiWiwOGqdV4fLO+R9mvjOoHULg2usV0vD8FRJRl8cdwXHW4WrR3//AEi4bfy4XW/VwWlQ8Y8N12RktVJSSP8AhnbYD6hcG2jqz/U//kEjh1Q7eEfWQLMrT1VlHRTQeKir88Z+JrrhOcLZNSOMVXy3H0Sg6ArycUddT3dDeNp3LJR/mrmGcQYthBcIKsPY83LJ7OCzGSU9Aw41eFeJZXVPigBzI3X1Pf8AwV9uNyTYeySKijdzBcAy62620XldTxBNWVgqg6SnmcbF8ch2WozE8TpqB0xxSYtN2G4zWKnO0p6DBUNniZM34hcjq09ipWSc4Hl65DY/NcbwzPSy07qSLiOQTSHOWZBe/tcqQ4di0zZXjGZufa8ZDrMJ91uJap13J188b/0UNTBdlo5Hxu7hcrbjWngZy8XmOurRJew7pVJ4vjqmUk2IeSQXFQBcA2vZDw3Q8mVmHywT1ks126kmy4sQtPqVnEqnjbDKV1XUYxGWMdkOUAn+5UaXGuJpqxlNzKeoc8XDJY26hS3TgtUzA2YR9CuZxiMtqz/H5LqI4MVfVMNXStgmJuA03Bt2XPcRMkgqM5gc75LMZJONNPDDzuHXNt6DZYZBDiFp8N19O/DauCWRsNpLZXmx2WRUV5gkLGQtkHRw1ujUFJ6HA9RZUD5Stagw/GMdgkqsPoOYyLfUNJPYX3WTM4sId+9kt1BWZhbWxTRvDXPbrZWI42xCzNAs04nLDaMQxm3W6vNnZI0P2usN2vt1pJR3YVNw3GxlcQG2ztKq0juZFIzuwqrw1XTvxWON79Duuzk6Ktrq1uKfd0LqOLNHzA6cb+yu07ZjCPE8vOd8noWbjtC+XE6es+7zWwsjs9gsCD0WhQVT54xfD5aXlgDLJbVaYZtVg1Vh1MKmR8T2PdpZ2v6LPOW/mY0/MLoeKJ534bC18EeVkt830XHuxGBlSOa/ydV584d8ZtdNRRRutJJEw+4UU74XgSU8sZIPRYVTURvqXOD9HnMpopqRzBC6sETj1tey51Kc2iKm35R8lK2te4WuFjVFoi0wVfiAfyt2Sh8SHBzY3/7pWeMnJriupWm08haf7N1boI2YlUspqGrHNJuBJoD9Vzs7uZJmcwj5hXMJnfBWRzs3YtVJybOOYhiWGVDqGthY15ZnaRs8Lm5akyuzujD2ndp2K9K4go6XHsEndVwxl7IubDMw6tNrry7Cad9XiVNhtVO6EF3meOi6sCnlY92dkbYz2aos3ddBxRwlTYByqukxkVrJ3FmUjVvv7rEbRVVR+zgkP0WoYmANEfV1kbXs6Oupm4RWfFFKz5KOpw6qgjMnKld8whxPmcRYNBHurtHXeDiDI4SXdwVlNFUPVTTD5xlTRGV2hjkH+wVheLrKTE6WWl/pRLZD3CQxJzI+Syoa5nYwlY9BC4eSWJw7XFloNYBstNRK6MaZkySPzD2iKOnxCmkfyR+GHblzbKk1rgjLQ8ZSAVladAx8DI8ni4D/ALSdtRB/2iL/AHlzbqOL4Rb5LoKTD6OfCgXQgOO5Cl5FKppDzeZDNTj/AGk7qSpebnwp+UipHCdSBKBl7hAMOu7IXj9Vi8mmlHRzt9cbPbLJdUMQr6ynmFOaFkjB72QS4YWtL2S7dLqGOmfPC54dsbaq+JmnFchrJnvDBEBf3VpsVT/qmf8Aqf8ABYRppacEvqQ72AU0NKJm5xUOHstc8jg1jBUO9UTT/wCZ/wAFFNSylhb4X/8AJZskIpyM07zfsVZiw2SrjzwVF+7eZr+ic8jgkgdVPh8KKdkrGdDJlUgp6p0bo/DQxAgj9sscse2bkvHmC224XT1eGPBuJHtORwOoKvPJz4uCmoajCcU5dRbMSSC03FkWJHl1Af8AC8XJVJzZYagifnF8ZsbjdaeKPY/D4JmebQXXWZsmDUhYWDK6N38VI+VkT/PT6Dq1ZtC78YeST9FuRChcwGaflO7OaVyWgNqIJHBscnm7OFlqURbnGcqj4TDZD/pEd1bgAYWjsu+LEoMYblcSOyoU9DLMR5d1u1GQvzvF00UsBcGZAPomRihp8HmLbNEY98wVeq4NnmmM4qmxg72WwIYSbuapc74vIx/k7KOjDh4TELg/xt7e61I6IRsDXOurZYkGfvx/qgpSspIf9Jfy2991Unh4fl1kljdb6K1izcsN7tPyKwJXqraxVnAGwk0zXFw6grDlnZm30U1S67Ss8g33RzSmo102Tmo8mxKrlo3RxGR1mRuAd0upxGtg8lLHKH+JkzdWrbM7nf6PMA/3WFRYXiEczZpILNZvY3W8JofV4eNcdjsrTtqnOtVPBHcKAUkN/wDSv4K1NNDM7zNAHYKJzqWLTJdc4ZkXKgtYEOQ+BgkbzBO0O/KVXksbvDbWUL5QNF0hhoSSxQtysjDSOoVKSbma3VSWdxFuYbdlCKiVp8jtPkrwRpw1PK9cIkHuo6qoEpvDGWjtdRc5roxpqoDI5h3W6Wzl8rdnuKTfGE5yPL3QcwpNqfySH6JS2tGVg/aC47KRtQxwzAEfNUi/MVM1wDfPsoWvNxGZotnafoiGIVMn9ZdZEkn4mZh0U0U3m3RLakb5vjebfNPJLSvZlkYdPeyp+Lt3+ijdOHHNqFLRzZNjrv3RkSNbnaNEc9IIs0heBc3AQSSGOmcGutcWXWEgVw9tyoXkByaOdjYbOvmUfrWhJnIF2bomTPvd5JNuqdrWZS0qM6OyoDuC7Uad1Ya+RsR5WgIshkpw2iL/AIuigFVIWiM+kIqxGX5MpOh3Ub2tY6zehRxyABPNHzXAg6dfdSYE0bTM5r3attZG6mY1+cbAaoBJyogNfopMzn3cTuLLAgpC81QaNQtU0nNbnYNQs6ijdnPsVtQvsQ3KT8kBRPfR0RfMfMzZYklU+V5eStHHZHtp2sY7WQ2CxLkbIJnPc5WmVDMNcGaOjkGtys8uKlY5k0Jhe0l41bZWhfZHkn50beoKgxSYmp82lwChbLPBH5j6e6qVdQ+Z4eegssUL8Na2PLy4wAB+qt/eX9Hdd4a4nZYAmk7p+a78y1xF2pq+a0R9BsVUJuh5jj6kgR0W6Q5t1S6pgTdLMELSZndks/umyhNYoiVstkfMDiqpdqkXJSrGawujjqXA7BVQ6/qNkQIvuoi+yUnXZKpEM1LkkNjfcKs1+tkbTfdZaX6aqiFEYQy3KHluLqemq44oGxSP1OqoRuABA6hE3KXBo3CzYOV45rrKnWy2cIcvmIursovbyhQmGeZ2XkNNtndUsUo5RELSMztUxdRStBjZyyE08GXRz9ewQwQgkhz2j5laAyE2zNcPqqxddyOoD45jGfolA1hks/sumLUHbEZDl6dSpQOToLn6piMvpU8IbYl7b3UmUpJh4ziSQkAMCFz4OWTFoWGyEkQAmNlj8ymfGHhh6l2yiZOuwFhETNE/HMuWkoaW2sjiR/BXMJiyNY0Ln+PJ3vxemjabmCK5Ha5W4Zc9VT5Zvw2gADZFDWCKHI6MG56dlSeHOeVLHTzSAjluszRSYa5LJmhLXMa8kdLiyAv5g87zfuom01UJG5oiG33RPcxzyGbX0WaWz1R8Q8OkN8jQAjgEAZyTcX37IIqaplkEeSzj3U89O+Joikfd7HWsBslFjbDypmtY8OZ0sp4ZGB7c9wLncbqBoLJmrTmqmVcbWVL72HlJGgKyijWOZJURuYCNdlew+O72rPeAaoWIIAABC16BtjfsCpIwcRle6vnu64DrBTUhnMY5byHE2A7hZs7i6peR1eT/ABRxukY0SMOrCtsumhY0xsa+PVw1dfZU35453Fz3AdAq8U5lhMpaA8bNKsw1fMbaqhzWsAQVxnyBGaltpGQ4bEuuijpXywOqI2kgdL6FWycAsGsojK8m175QB3QSvomvyQMka0dL3UHP1TnvdJJICHE6hFCwlzPLpfX5K7iMNP63vIz2yhVHzZfKzfuu0NBnkMNQcrvKhNQG6s3QTkSBzzuq2pK2i9JVEtby8zSBqfdQtD3gsY0l3YIWtLjZu6sRZIjf4lKpEIjI9Q1Ti7bW6I5X3eSUwl7IpAvdJne65K0I2MkYIJCRci9lRbq6/RakdYyGLLHBHzSLl5GwWMka/D8rHtkfM8uzkBgJ6ArVbLZ7mXALN7rlzXNa4GPTvYWuVA3EHxTGZ5uTuuGTtr2uoqpRFB4jIZB0Ddys2tqY66JkHMkjyXzBr7Xus4Ym+VjwDoTdDFMxxuXarFNZbUT6qOmlfymBpHlJ9lC6odN5HAvzjY6XUklPG5xMgEnYjS/zUp89rNFwLBdLc4zNE57rMc/JkFiL7BTOZFy88PnF7bKqH+fLmaPfujkqAGWz2DB0U4pfJbpIHUz2zzizv6v5KZ5ke8uky5T0aqsdQyeJmd58hvsp21HOlcy2wvdMP4mnS4p/8gpW95Gf/ulZgms4MEeW/W+iv4/KymwSjzn+sZ/+6uddi8Et4mMdp1K3sXBsMqmQn8djmk7G2igxespXQ+IizGojFg0DRwVGTFZALBwDexF1WknEsckzPVGLfr/7LlH5WMptTfI9/lfHGPm236KMxnoiLMpF9QdUVy0kDQrtDCO8jgY8rNBfy7lVXFzm3cwj2KtCZhLcjBoLZuqicc5y9TrdbxFQu9gE0Zu/KrDoYQDm9XRVmAtkXZTyDVJu6IAOdqiLWNvlOxRfUBNkGYKR2v8Amgt5g1EO3dEmAs4/up0KAtrEH5eGMKpsvrkfJ+l/81iuC3eIDkoMFhO7ICT/AAVmVY5LrWKjO6ladbIJBlcoiNuykQJ82uVFSjy+ZN6nE90un0Sb6kQVyzQNuT+icEuDYw0Xv+qEnMcp76Jg8xuuxxv3WRcjjaC18zdA4X/VRYkbYrWf+KUpah3LI9wUWNf/ADWb3DD/APgEFUGx3tdOWPI+SFFd1jmudFpEsUrdBvY3seqUoJeXvbYFQC9kYOwQSxvtoNkTjYjMBfqombp5MzTkJuR1WAbi3dHEGvDr/wCreoEdyG+rpZbVNLA2NkWQbxhxRQOtfsdwmbMSAHm9hYIqRt3ttbe+qwrVw+CtNpqQeboumpZZHgtkjLXDcFVsO5hkjc+mgdZjyLR2sdFc8XN/2Kk/iFxVMdll8S1DIMCmLhcz2iA+fVaRrD/2Cm/9QrD4tqedhQj8LHF+Kx92neykNqvBV/6d/ZZ/eVsOxChonzmonYCX+kG5XNYFIyOCTnVskDJHatZvJ2V+Gocx+bDaSNl9p3+Z5WkaLsRxCeLnUlF4SP8A1tSbfwWRW4jhWZz6uoqcTed2tOWP9LqKrjpXTlmJ4leVnSTogbFgbP8Ar0H6rQalr8KMUfMw+emey/4sEhvv2urcbZS0MoKyGvtuPTJ+h0Vfl4J/26n/AN5RmnwY6x10Qd0IdYhUpcFUxjjDUsMD2EEiQWG/dZuMvfK6Nrz5SDp7grRNRX0zDHM/xUQ+GbVZWIOonRxvpaUwOaTzGZrj6dkDYc28jW/vkrbBe1tmd7/3LEoM7I+YxmYtW1hbarFnVEcFNZ8DQWhzh+Ib7IJC9+cec2ABTFrX+Yi7r3v1Upw7Fmn/AEKPa37Yf5Jfd+MFrsmG5v7MoVthXllMTHGR4cLWaHC9lhP+a2n4Ni0rS04bMf8AzAqsnDmMjbDyPnIFJFGOSAWjfBf3zKSMiF4fC21tkddg+I4bAyqrIgxj3ZNHXN1S5uXZyxMDVfNBNRyNLLSvtZ1/dDT0jJaYPedbkFUoZLuGqvRyvAbHGbN3+qg2cDw2klpcQE9s3KGUE2vumw3DqZmCVklQY81ja0guRZZpnnAyxsDndrKtPNiYje5+GyAZTc8olajJtjLewxsc2EuYfVHJcFc+H3C1sLkbHAbtvc3Gq1aU66HDKUvpHvZ5NA7XoVRhwCrxTiR1JQU5kbHUDmOGzWX1VEGZzQ9sTSHdzqur8W/hnhdsdI0uraxoc4jT/nQrhs2cY8nTGLS1PAuBQ17vGcVRQcw35cUWYj6k/wCCpVOCfZ9BKYX8ZVeYf/0gK5gy1VW7PW2D/cqtN6ixc9fiVcl4+52FNw3wrXRiSi4uqNdhJSAfrqhqcDrsPni5jxVUcj/2sDrjTv23XGNvE4SRPLXDqFu4HjdWJWUEtTI4SfET/es5xtwi8VwnGZqXZYPhVC+qkr+URLAbCx01HZawMjHXjWfgjiG1ns5n9y0oTqvd01+HHL1TIQfUE5TslnqmCwt9QpA4bJnOv0K7sAlb4iJ0VUyOVr9w5uhVHEsMhqoOTThtO5p8j2t1aP1V9C7dG3F4xR1GEuj8RiMkznglhBOn6qjj+GNc+KuOJzHngbi+vfdbXGUtNFLTc8E3abWF+yir5aH7qp557OYy1iQvm7/LJv8AjxlyUuGNy5vFh39oW/xQQUVZC4yx0wcy1szCtzxmGEaUv/4qSkq6b8QRx5bt/LZdHJDws7GfETsoMRdRNL/MC0EE691SlwaqZUSiWW55hJLepWvgdQyWtaxgI+YUGMVNRBiVRGx9mh21lqJYU2YWxu1HJIf3tVZgpp2CwogB7hV3YlUBnlyj/ZUU2KVYgztkuDtopLUS0KWNsU7mfFaxCz6c4fRY1SvhOZxkt8kuHKjmGo5zcz8ws7tusqSJ0GI8/JbJMDf6hdXJ3XEtRUUTqV8NRLDG+UskdHqbIMErpJ6x0D62aqpgLh8jLFpCvYxRSYlBCISzNHIJdeoV0+ZaWHPYuxgr5M8j/NbS+ighpmXuYWfOyt4/DS+LbJMzzPbvdZBfODljqnNb2uuObrC1U0kjpPTH/tBV/uYS+Z7IP0KkMVWBf7xv7XUf4/WeT9VgHBQTxaQuga0dA1Suo6uQ38S0Ko7yi5mdbuCpTBBlBNY4newUQZop/inicPdt1AYeW7Zv0Fk0zBZpie8jY3cgaMh9RRVqKJgAcOYPk4oZ8PpZyHl7hrsUHMIHkKjMxy5JH3Wi2jLw/Ty0DZ4LtLDc31UDKZ40E+U/NPT10sMT4nPux4tZVpZ6fYtQte5M4AHiCoGtlppXhsrnCQXuVVhe2JwcyQht9W9CrdRVQTZXsjDSBsCpZaQTv6uS5r/hKz3VrGHW4HdWQXKtLHOefUUOdwKhDiEJmY17QXgG6NUnNRlHmSbM4oMalZaKZjGhxABss3xUg+NHNrc553KCWeqa38E2PzKyZauzvi+iXiyWZ3OzntdEtfFRinxuv7khWmTTfG43WC/Enn0CzvdOzEKh3qIupRbckdJJ8RUcFRJy3Mcdis5tbIRkLk7Z8u+ypaxK572nKmiFWyMPD2/JR3a43bIbKTmsaLB5+qlLEnixLnD0jQ6jdWW4gYAJo2EkKk+lEMfPa4Fr9dEzZY5PI5KW1rxXi5/En1HdWYppAb8xw+RVJkHhQ3K/M0nRTNfZVtYJadbMJ72T5ha2VtvkoRInDr+VGUucD4I/91M+W/leGn5hRk2PmTEgnRGqGHM/1bf0UTvWC1qZj9bI+UZDotYT5sbI8ksti0PUJ0BcFehoH1DeXzA33KhqKN1G4sdIH+4W8nPBAKl7Wi5sFBVVOYXjfftYqviEkmURsYTfsqlO2Zkg3t2K4W7rYxSQWZMwg/NTCoje3S2qpz08kj2v2t/FTwwBoBJv7LPNk8rnFqrugc4ZrLQEbJG2aEHKMBDlrxmWXLT5QbtWZNa/pXWNdSS25sdyjdhuESgXpwPkn4hHEOyrQpMJgqohIa2Nl+l1vVfDuG1DbR3Z8lWi4e5do+Y0+614y44ggjmgZyxPzGj3uhkbMSXCGS3eyux0baOzASQNrlRSy1Ebw+HZu6zybpmc5/76ESTyGwJHzC0n4jBPZr4Gtf8AmQyOgktZzbjoFpylRLZ4m6SD6KGR7yNd+q0JaR5BLHD9VnSPDXWdut0hrNPxj6qUU5acz26KEyD4VI2pldGY7+VaQ8rcumige6zVI8yFQlr72yrK0awePdIRvzbaJ2erzhTWHwqlK5icT5ipAwgWv+qN1rJi6/q3CloHzW8tk7cwN0HMPwpZr7oJeae6XMF7PcoiGOGpS5YvodVqhiuc95zPJJ90jI8tyX0TkZjZEKV5ALrBbEAB3UzdlaiogxmpCFlG9zjewb3ClpYCwgixBREFozFWGQsY/uPdQTm78nRUDmMnk6lRSxOicG9UTmHdpsjLwWgHogeIuIAO6m2Cip3Z5MwYdFYmyO2UUzQ5416I2MeXXCdk0baflujObujgdGXWFz81KE9OWt1k0HdWDU1To7UsRa0f1p/yTjw0UXMn+gVKsrJ6p2shaxuzBslNq5hfJMXvmc9wF7noq4F3H2Kttfy4X+bUqiHEOAaL3VYSCJziLbdT2VyCDJGZoYHSBptmVaPU2uB81uYawvw2YNy6E9VBjPm5pdpYnoqs7LalTEGNznlo0KGoaH0/Oy6jZBW0te6QIQlqZui0gybpiUw3THohAwW2TZggbsk7ZWyx5glnH5lHmKTd0pUoIun8vZRpIDuCE40KBu6IGyUkpgbG6fme6hzFOHfupxFsS33U9M9jnbhU2O1R5mjquc4qvvkaBfMnZMwxEX8xWc55eLNciY1g82azu6ULcNHHzCX5yT7o/Dx81zCGsyjyH3UbJbjMyQEoJZXg52uusitVMnZL+MBZ3oIUHNew2AHzsrNW4lzXE+oKu6F9g62p1C6QDgkDnWd2RtkDHAqsx+VylAu3OjK1K+5upMOPOrIo76XuqYcXblanD0EclfcjYFaJd1hMV3MB7LheKarxePVcjXXaH5B8gvQcPAjjdMdo4yV5RU1DJamaZmz5CQqmJMY4vsAMx7laVXJeMCNxJOjiFiOlc/dXoZi4hh3so2eoqJIY+Wx45ux9goYWsDCH9U88jJpA2MXd1PdG2lkLmmSRgA6dSoNKGntT84u8wbe6qlzS8vurfPzR8hpszss+WNkMpNyAeu6xLK+2SN8erAHf3oZc5IazYKCOQZGNcTcnorADnMcbm1tCq0jYOZULWik5FHJLtYFZ1M28gJ+qt18vIw2Q6avA1WZHOwNvMc3cqR7Q1pZ73WtPS0EEDPFU5D5Rna4dlQMcHwAlaiQELmsHmIB6XUzZJGuLiDltvZPGIWexO9xdStySGzOo/gsTPJKSxyU7RldUS5ehDU8EsbqhwznKBvZBymgZGbBEWRQU5y+p+t1mkPVQsrJAyScMaD5brK5TyTYL1nBMJwWqw6mZVYdDLK1vnc4arhOIGwQYxUQ0sEcTBazWbLtiQwRC+RpAb80JgLY8+a56hWrxtBAeGE7piw8kOb1NlbDTUT6WCnqHPBbUXtbpZV5TY+y2OJWOihw+Af1cZJHZZEkb3wsYwXc82S1si3M0OQ5Lbr1eDD6GrxCOPkRCmw+LzMEYsZCN7+y82xKoZU4pVTGMR3dYBo0sEVVivZ2u6twmEkh4BBHVUwCBodSdkQLm7qSyt2b1UzcJrquldXQ0shpmG2cDS6oueeU5dLPLLM/BcGhlvSwNie5oOhJOpK5ZQsMaOinpWNE0Ul3i4OXcIBLCPjXokmLUWFS1A5jXvf8As2AXAJXDPw+OWV8kjhnkeXmwta6iz5KrZGH47obVJecrgQRcEq43Co8oawgfmvu5DJhsrmgc7KBoSClsqMRe8EyjMNxZT5A6zeWCSbC6eodDCRT0r8wYLAAJqWR7J43WLix1xba601EpocNxVj878LqRH2yqTD3vllN4Syxtr1XQVPE1XLC+FgDM4tcBZVM28oUpm2nxox78HoGskDXCUGx6+RcjTQ4lMXcigkqgw2dyoyQD9F1/GUj46KiA9/7lNi+MVWCUOG4Hg9RJSsipmTTuYbGWR4+Pupsyp0hxc0jXyCTlAW0sNDf3Vilp3yUD65s17OyTMtr81pY7GBikmbqxjz8yNVnxOdFzWtd5ZW5SEZSxYFjFXDFNSUEksRbo7a6z5s8c7o3ts4bhdPS8WHDqGDD46cFkDcpud/4Lmql8c87pXO8z910xZQZrFA5wIspJGC4LXjXTVJlO1+YvvcaDstqrZgmdup5KaRozZC8XsMuqjc18YvJG4D3C2FT0NVW5/DMB5Yu65tYJpqeamjDZLa7EG4K2uFYWTOq8/pLWAj9VR4gZ4evZTg3aGAj6oqg3f0pPYWuBYEwkyna6RkuBbREHf9U6hBOYXUpb+VCgEXW3xW209DD/AKumCxA1zp44fzyALouK2MOJuYW3yUrAP1KkkOdvrdO4366qIGylYx8tgyNxN7bKgEHxqY01VGM8lLKAOtkBY613NIHuEtRqanp3VDnAOAyC6huO6u4UPPKf3QgqkZDb+Kdu6uT0kkz80cJOlz2QCgrRb+jON9rLKKU2bIVbxjXEC/o+KMj9FDLFO71xuHTYqbE2ESQE9aWP+5BUB1upHGNzP3h/FR/u9UgySxfyzYbrQvYfhT8QY97JMoYbbXVxvD8zL/0mM2JGy0OEoebhsrz1lsr4bcO/tuWMhiwYFPUSiCOOSSQ7Bjb/AK6qTF+D5sFpzV4jilJH2jBu9x9h1XQYbW12FVPPpJPI8ZJYztIPn0sq/E1LQcQ2qvCSU1Wyw5hfmBHYhZiVcxSYLiOIxvqcNopqiKMXzNbYEfXf5KGnpZ62dtLA2MSncPkDbH6rq8L4jxbhqg+7msgqo4h+Cdi3591y80wxHF3YljrHVHNJMrYhyye1rLcScVnFeHcZwaNk+I0gjicbZ2uDgPnbZRYfHzZmMGlzZdZiPGkk+HxUGG0PhWyRaGWzrMtYWCy8Dw3+kCfJoOi55I6CgpH0sQLqh8pykDNbS6IYdJUuAZWyQf2QDdWmR2hj+iOHym68/J1xVBglZnyDHZBoTsFFX8Kz1sHh6nHJJG3BH4YNitVh/FH9ko3S6pyac7/0f07mCOXGMw7cpXYOHIMOiipIZ+aLPINvktQSeYFKQ3qovk//AAWuYxK3gKhxCrfWSYjNE59tGxi2gsoh9nWHj/6zUf8ApNXUuNmEqPPd7R3CchzEn2eYa3fE6h3+yE7fs6wh1/8A4jV6C/7MLfxGrFHAZAdeiyJsXlmv/Sst+jDos+JlAt/zaovjll+hsoP5g8MOF3eLLv8AxigpMTNNzfxy4AAjOfIrEXFELb54o5PkCFnxMhB/NfB8OrqXDoGSmKvBEmeS5Fuys/c2D4BVxsw2WbNLIGPEjr2CCoxeCtqqGvgsBRklzSbE36BUMQrjiVcapzMri0MIB00WueQ6KXDqV7iZq7lH8tkwo6Rvn+8hoLWAtdc5Flp7l7JznFi7mXA+QKqupMPzEAzSP7ul3/RdORxdOMKOfm+IJv0IUsGHPjJLJ7X/AHVUHEUb2CPwxjezqX6D+ClpMYzv/GbGG92vuseIzxV8Tw2CufNS1rOayKLmjob20VLC+DsCxDBqWeopnGWVudz2yEFFiONHxMktO3SWHler+KGh4ifRU1PQx0oLYm5M2fT+5PEOKQ8BYPDHmg55yAk5n3usyXD8NcyWSaHyR7R5tbhXavF6qeoFXHVmFsTf2fQpT4s6ohc+aojF/hA3WPEOKlh2eGendRU7zLPuwG9vquvMALSx8ssjZG6h0htYrlYsYlMXLppIo3sABOXX6I2Y7XNa6Oed0gP5Rr9E5I1xwtw1fXBqb9CilwThWnuz7rgbb4bFZLMdnFM6JkhGfcvb57KsZ4ZbscdTuepTm2hqRTR1EkcAs291uYxXQYcMNqp6YzxPifGA02N1zk01IA6NjJMx6uK3MYdRVeC0LHm5Y7XTbRXZl6LqUKjiKKoovCyYPEQHZ2vG41uFl4xjL8WjZHJFEx7DcPY2xPzV6GSmjzRxxABhtve6pYtQxzuGIUcYabfisb19wrhsZnFlAvtZ26Ome+GqZM8XN7KEzRh2dpPyKeM81wcHjdehzqb8ne4XjjMFlkbUxySxzuZmdf06LraWuoK2IT0E4lj623B7Fc9w/DS1T62mq2RuZNFGbOPt0WrhGD0WDtkjpJjIJDexcDb/AJuumE8XfOKagliADi+10RczLpKB80mRk9P8U5jLPhFu1l25OQRy+srb/NQ184paOWqjjE74hdrAd1K5sJ3jb+iZjWtOZjAPkotOAxKvxTEKhjsSpYoHAWja5pBsocOqGTxupHy2jYbbXV/7QT/8apf/AAVz8Plhe5vqzmwXj24/mdtXnbcqocLysa+RznC/mawa/NVzS0NPIJBVsy9eiz4/FNbZ4/immiEjMrwuXFxpYw+ugoattQXh2ToDunxKtgxLEZKiO7WyW0PyWeKGe+keYe2qEh0ZaO62wslrMgY4690eWiZG0Fjnm1zroq4pKl+jpGua8aEFNLFWs2gegKKWKkky0sWUPOtkctSx0ZLWg23uqdqqYhkcfnv3UbWVXOdHI0jTVbSnptJLzaGlf3iUVNHOwu50gdmNxYbKlhpfPw35nvzMhIZbQ3HurGEy87DaeQvzEt1PuuozOLY3WpZB3IWA2N/V36FdJxPGTQNkAvy5AVyjp3slEb4yAe655w3EroZyx5n3HzTSTgbOWfLMyJ4YWStJGgOyFz5Ro95y9QFmmV5tW343NsglrWDWw+TVmk2cc4fbpdAY5pHjkwuI7hXijTFVmbcXHsVC6qmacztu6p53seWPJIRyPfl307pxOK3HXuvlLh9U0tbDI6z5CCOyxpLwyiS9zfW3VXIyKs3jAbbQ2KcRa58sVjnJHS6bxYLru3UUXLzGOSQWBscyeWKnhcLWIPUFKVP42w9k8c5l1jePqoPwI28zO0j8p3SZVYcRmzEfIpQ0C2OVmVynhlbkDA8G3ZZUddHEToH37rSoGVNe9rKOkDnv+EED+9SYbhPI/IwvWNUSmSXO4rfq+HOJXsyMwiY37EFZcnDfE0XqwOqd8gFaaylSlqJpLNe69kmuJVkYFxE//wCiTg/vWCZnDvEh3wmQf7aUzSsY5Rq94PyQeVzshdZXv5rcTf8AYv8A8wjHCHEr96WIf7aUcZZwa0KQODeiu/zO4i7w/wC8f8kTODMfO74m/MlKOEs10VTI+8YUgZVMHnAWxDwjjrG5RLF9LlTjhDFJGFkzg/5ApZ4WTDbLfqmM1nE836WW/FwZiDXAtFx2ylNLwDNI/O900d+g0C58zwslaRwdhg+QWVE9zpW5V1cPCGIwQGna5z2kWF+ijHAOLjzRmEf2n2KzzdOEs4l/hWgstkKFtVENCVsycI442ldGZIDtrzFnUfC9XiLHFldStLDZzXO1BViUmEPPjtfmt/VHFPH3Un/R/Vg3OLUTfm5W4eDKlrAz76o9FbZUZZo/zKB9Sw+XMtCo4Zmp3ZH4rTu/si/+KgHD0R1kr5M3drdEoUBLIHh2ZWoauT8ykOB29GJuPziCtS01PFT+SPzj4rrpjDmlw7EDE6zxdFXVsdQTH4Utt1JWXHmv5VPG/IQSuksY/lRuaSdFI2iD2czQO91ZkqKXJ/ogDu4WTVYo5t2xssOxK8WV35PRayZKYm0gvbZV5SwO8mgWJNiDuYS16Dx73+qVSNUs21+afz/ollLxmc8rOiq76F4HzSlrXNbkDlZ1zDC8XPYbgq1BUFwAzj6rn5a97gbPR0tY4uDCVjw5WHRVM8oZla0/RDDMXWuD9VDC6Zwy89tuhO6N3iAMj3AjoQrEOl0VcJ+XzI7W91l+LedM5R1j3EkF58qoE21K7Y4szKWY5tUIvbVNFIQ8aAj3ViSBjxdjAHHqu1JStI6QXIdp7Kq8m+qkL5GkiT9FG/UKsCZmcPI4X7FFHMIyWO69VDZwdtogc7OVoxacM+YgEXapiGvGUjTosqKWRg3B+itMqPLd7yuc4qndGwO8oQOsNkLpwXbmyXNZ3KzjErKF0hH1QZz+ZSyEHVQkjstsnuU+cnRDp7JsoWwiT8KJrtbZAULd1I1uma4UGY0WeFPEQ59nIA2+26mjAjs8x+b3WkTZHkXynL3Q5JpHBkZs13VIATkPkkNh0CuRVEFOwhjQSehWUDU4dHHTGSOp16hyzDGtbDqsz1/IeGFp7q5i9IwQZ2Qjy9tFmJPRzguBqgIzOyqwGZzYtIUMuSN9guipIw6K+trqWKMyn27qoMxdmLtFeprviLY3dN1FDK1rH5QSUopHxSXsoS2YnM9qeme503LOqDSzRVEAfJoPyoHUk0jea8BvsSjgi5RzSMuB0UddU+U5z8giWjnLI2hjAwnr1VERiNxf+YqSAGeUMYrRo2EWPq6IqmyX8QXWrw3UvdUTwPPkedugWRPC+J9i1WKKZlEzmM0cVBJX2ZUOB9JOq1cZipBhkUsEbQT2WNVO515O6uyS87BofL6FgY80YGoUXXVaeIsh8NFPYgkXIassai66QiQZBdyA5ScyNuT4nJjbNmCoEaFC83Unrdqd0pY2s2eD8lREB3SzFMm83dRR5gkXIQeoRf7KAm7osjg26eIsuWuAUudw2W0VkYPUJnDzJN2WSEma6duyUbGluu6lZASLh4UD+UtsCnBsonMc11sh+adxBCTAma+3YKQBpF2OuFWYr0MWgFwucqrVEM1jJbRGZAWNjkAuB2VuZmcjsOigmg5v4jyAVYklWmhY4h7GeVDZoFtgpBAbXjuSegKeGJzx52G61bKD4iFv8MMvLI73t/eswUuvmaui4cpmRwkjcvKsSS2sRqPCYFVyZrOMdh9V5e2GRzNB1XpXEzcuDCADWdwYFxfgw30BTmQyeUR6gk9rwtjwvlJLNR0Vc0rypyaVIgchIGoUtJKWF0eX16k9VOKZ8TrH9FLCxjZhIWhamSYVjK7mk2O6d84Ny8eX8oUzqQOc54fb2so3UUh0kYQeyzcAIZvEP87wMmiuOL4wYM4IRQUUMUTWvbqDqU0ts2inKBNSNupa6aGKKOOYXbIbJqfyhBisHNjpn5HOyHopIznVcs/7R5dba52CZr1o0lGxrMz27qxDQ0r8149jZTkrDEheVdgb+FzNQ8dCFbqcPg5ZMDfMjlhicW6EeUbFOQgBvZj3aHsopXRhouBuPfRafhYvCkBl3O63WVPRytzF3fZTHKJTi6+m47wqlDniCckR2bpbVcjW4iKyrkqyzJzDfLe9lU8NMdgo8tjlcukSJRODsxp+YVimdzJGQZAA+QdFWAa0XsreGWkxGnb0uT+gWkpPxTLmrWN/JGAslspbkeN4zcfNauMxxy4rK2TYBg/gqnJw+NxZywT3IuU5Dt8M4iw3D8EfDPURuq5QZJDmtmeRb/BcG9xBzNU8kDspfBA+49kzY2Hyvb5uoTkKxmk7gfQJCU33B+isy4dPIbxAFpTtw/w7CauwcFLFqmpoZWgvYL/JdHgnDtRjFKX0lRGHxmzmvFgD8/8Aguaw6SaSsjYz9je5K1gHjaaRv9l1lyk5q+IxSUBc/wARHKbll2XsCFXinbYPedVJi2flwUoZoGGU67/82VERPJJOgUpInku+JlJIY/K23XqoZ6lzGHzg3VF8l3ljD89CmpoA53OqnkMvYNG5WuK0CJskso5YJV6GGeEm7wB2tsiY5kdmQQuA9hslHOZT5IXfNx0KWqzzWeku16KzR+d6zZJZr2EYNhc2Ct4eMSc8GiZTmxFzKUj+Jlq8amwoGfM/3LNx6qZLibiPgjjj/grPELsUfXYbBinhdbWEXu8DVZ+KtzcRVFL8HNYz+5Y2OkJMandUYpNI1lrZGfoAqJBPUD5pY/UPhxytjj0aJf8AAKh44385XRlZeXAbNKhzNO6KOfOPMwN+Shmc1jt91uGSeboo6hrT5mg6KLMChIscqqteklhNBKeZ54zzMttxqtXieiYcFj837KGORum1zYrlmSva4sYbZxlPyXTT43DPC2mnpxIzksgcC4bhS2hcNQsjw0zRlpbJaxBudO6wOJ3Xxlw7RMH8Fq4VU0eFU74Iy/K831OygmjoKuqfPWkOfMbjKdALaKc04S512yQFlvCkwWLV7I3XNvM+9vkEjQ4LL5QGRuPpcHFOcNUxAzzNN+qsyZcxcOpV4UuGQMa6RkrnX6O0KkNPhclnN5rb9Cd05wUzKWPmV0Df+9Z/etbiCpZJi9U1mtgIvlZTUNBQSVVNJDH+KJWaf8/JFiEWHTYhVve0h5mfexTnBTHpKCxEj/oFeALfKx2VWY6GbmDkwSvjtYFrCQtHBOHa3GqnwFPRTPna0uOa8YsPcrnluxiLylx2btWnGc9mUREetyoRVIEZY+xI3JRCohtlkYSL31GgXTxfZPxW+8rqOBh1FuaP70oPst4tfmY6npWOYLhvPGy4R1mifTOPu8Ud57dP/r4/eHIT4dhU935JGvPUHT9FJSYdDDm8PLe41zFdb/0VcUStD5PDx/KYaoT9lPFmQsayN3uJwU/F6f54+6/5x27/AHsfvDnGU9jrOLjcAp5BJEzMLEDddPTfZVxnH5JMNjIsbkyG5Rj7M+O2zAfduaEbNY46qfjen/nj7sT3ntv+/j/3Q48VAJ9SkrKejn5HiI812m2u2q7E/ZFxu4ufTYIc7zcZrABKf7I/tC5EbYcFcXGMi4ubalPxvT/zx90/znt3+/j/AN0OL8HTYewciPl9tUPiZfY/MLs4/se+0eSECTAJfbdPH9jP2qRuvHgbhF2c4AlPxvT/AM8fc/znt3+/j/3Q42HEZaVhhZljYTcgDcqaKukdrYWXUVP2KfaPIM8fDshtv5wpqf7GftJLMruHHg9uYNU/GdP/ADx9z/Oe3f7+P/dDApZs7Pkgq5nstl0HUrqqf7IftFju0YI0X7zBTzfY39okkJb9104d0Bq2BI6vR/PDX+c9v/3sfvDzqQyG+bVVXUj5X5wF6HH9in2mZwZsOoXN7eNYFHXfYx9oFDSPrZMMgc1lgWQ1bXPP0C1j1enKajKHTDu/QbMoxx3YzM/OHJiASRw+XURABbODkZAzqw2K1qL7MeJJreKf4d8QAMLIXSkfwC1I/s7rIW5ZJsR/2aYt/wAFuc4y97388PjDLf6Whu9mWH6pQ/s2u+f962GcBNGrziTz3yOH+Css4MiYwM8LiBt+67/JYmY+Jzw+MOffo5j+90V87QVtS8HymN3h6OrDwPLnDrErn3YTxxTuMbaCllDTYnMLFSPP0dsOOXpKcGyQfnqYi70i9yqToONW/tMDhPyQ/wD6Wx+vh+/+wf8ANWnXw8fi0pJt2LA4nxr7tpmxQyZZpNj2V7NxXf8A/VgH6FQ1MWNTOz1nBvNPfKSlnh4+5yc2KudRHnzyyuf1MgFlSjrXva2RmmY3vfddfJTzlpceBxp/3R/yTx4dXnLMzgB9vhcIHaf/AIrVpOuI9ZczFiM4cWOeC0iwA6IaevlqahsDmNDn7uG66wYZXv8A/wBgZP8A0Xf/APKJuBVs72xt+z+a7jYWgcpxhJjCPOZctDVSMqOW94kPSxUpq2RyykSWLmEA+67ofZ9xA30cAyD6FN/0ecSk/wD6jH6grXk8X4zo/wDex+8OAjxEm/rdbrbQKOKp5k9gd16Q37MuMz+z4Aj+t1J/0X8bnU/Z7FfuAU/KfjOj/wB7H7w83mqQWtbIfQLD5KuKu0Zjjkk1N75ivTm/ZPxuf/2Dg/3z/mpf+i3j74OBqQfRn+afkPxnR/72P3h5hFVSBmS9/c7oJ62RzeRnIt1BNyvUP+iz7QgdeB6T/dZ/gUR+yj7Qspkbwdh4DRc5iA7+9OOJ+L6Sf/Vx+8PLoq4+QOmPkFvMbqTnReq4+gK7iPhHjV7iP5uUzfZzWBa2F/Z79pGIudFR4Xh8eQXIeQ3+5KejPw9ePLPOIj6vNoXNe/nMMhd2ypTRVszs8ccrOxMRXtmHfZHx4WiXERQRua70MlbYj9Fvs+zjHidKWgj+Tgo+T1HfOi0ZccZnL6PneOkxiTWSmmce4aVYhw3F5DduF1ZI2DRuve8Q+yLievawU+JU1Jl35Z3WPX/Ynx5FG11DxJFK8nUPJAA/RYb0d67fuqJz4zPx/v6PHzw9j00ZlGFzMf0zWA/vWl9119bQMoqewfCzzlx0vay6er4G4wo3yR1mPU7TF6xrt+iz8O4MxHEIOfHjjI9bEWJVp93XjriLifJz8XA/EJu41MEZ+d1oU3CmIxMyVOI0wd1IabldCPs5nezPJxFUA/uRE/4qNnAOH/1vEdVIe/LTgz+Rlt4SoGuElTiMeb2Yyx/VWnYDgDKd9oqVzww+azArzeAMA/rcVqXjtoFMzgDhQ/8AW6gfOUf5J5t8tbBo8PpbcyvcAzLYXPXutCiocCqJmwRPY5zjbQuVnBcNw/EpJqfEWcxkWoHvey7/AIJ4RwCGofXSYaOUWloLrkkrGObzdx6zX0fT5bffHp85chJwnDE1zm1bgGi/keb/AN6wZ8SMTnU9NW1zZgbAFxH96+gm4Tw9GfwMNZf3jJXN4vwdiNVik1VRYZTZHm7XENC7RL8z2v2gjfsyw6qYiK8p9GPg/wBm3EL4KatqeKTzD+IYzE1zSOi33cG4y6J0YxeiGYWv4QXH8F1PDuF4ozDmQYg4CZpPm3AHRbLMFqHf9YZ9ASniZvzPW9x6jZvmZ2XU+VT5fo8L4g+wnFcZPif5ywPqGaND4Cxob9F5+/7PMdpsefwrLW0bakOyiS5aw/VfWc2DMZFI51blIaSMrPZfPmMCdv2gubWSGZz3guNtTopnlb9N7P8AX9V1fONmdxHx9bn/AIYg+w7jYm1JiuE1BO0bKoZnfwVOr+yX7QcP8tXg74/fM0j9V7BS4fR4VW01ZLNA1rXB+YS3Nl0lRx7Exr46eLxTCLBrxorbydx73v6XOMdOzHO/l6fu8CwT7OOJvFtjximkjpnA5jC7zey6B32XYS6/M+8fqGrrzLLK98gbkD3E5QdAna+QepxXLPVOc3ymPo+Lv7x12/LlGycflHlH/LyCp+y7i8zyCGGMxZjkLpW3t0Usf2YcaOi5Zhp/rM0L1x0zvzFE2V1tHFdfN7I9outiKuPs8g/6LuL2nSGkH/nhSj7MeMXaztoie4lAXrlyfUUnfVb82f8AxJ1vy+zzuj4SxvBsGfHXNjdkJaTE7MDuuco48WwuLwcNfh7owSbO3XtLz/8ADKsO6xvK8LpqSI4jV00+FmotOfxAfSPkumM2/Q9m6/b3DXlO2IuJ9zYxmHx2GzQB9nPtYqjhn2Z4zilGypp6+AtuWtLyQdFuCjlnpZqhmXJF6rusVt8CzzvmnpXSuMTGBzWdASVMno7n1Ozp+ny26Ji8au3OUX2Q4jLMRiuKxMjtpkaSbrVZ9j9EyPlvxud3Yhg0/Vd+2JEY9N1jzfjdne+u2ZcudfR50/7HcLe0sfi1S9p6ctquYb9luFYeyoZ46eXxEfLvI1vk9x7ruA32QlqkxyipefPunWbMeOWyaefj7HsD/rq+pJ7+UJnfY7gDj5q6st2DgF6DkKWQrdy6/wCb9d/uy89P2LcMu1fU1p/2wuuouFeE6OBsUfDkDrNDcz3XJt1WqGIhHdc9mvHb5ZPJ1HWdR1URG3OZr51/Ri1XCfC08rZBw9SMsLWAJBUTeEuHWm8eD07D7N0W+YwSmbCCpjjGEcYYx3bcYqMp+8sOPhPAI3CQYRR5hrcxqdvDvDReXz8O0D3d+XZa3IalyR+ULVtfiN388/eWYeH+EiMv826dv9hrUdLgHDNLUMnhpmxZPgMIsVo8sflS5Z7KTFxUsz1O2Yqcp+8gq8OwqqYwQ1VJTFpuHGJwBR0OEVNK509LXYVV6WALiAUhHbomyLEa8Yx4R6Mxu2Rjx5TX1lYml4p5DgzAcPfcEXYAQFi0tDjtLTtbNhEUlySZH5brUjdNC7PDIWu7hXYcZxKN4dJUc1vVr2ghMNeOv+FrDqNuvGcccpqfnLBnwyrrGgSUUEJ6kOtdZWJ8FT1d5o8S5DmMNg19m3916BQ0WC4/icdGI5IKmodlbmOVt/7S8/4zpuJKLFanD6aaIspnOheI5WeZajZjz8O/Orr5Pq9qw63qtt6c5ivWZlxVVT4vSzPppMViuw2JEgIKh5VW5tjjhB7ZrhItLSQ8WI0OiVh3XWn9AxyyiIiZ8zxUr4x5sYdm7gmyNrHg+fGJXDsAobhSiCZ4GSKQn2ai8pSc90PmixCoH0VKqrK6e4krpnA9MysT0Nc6IllJL/uLl6h9fhs8jaqPI15uDfquOyHo1y1ebiDGhjal2UdL7IJKicNJ58zi3ccwhZLcbmBsGhS1U9TUxianjsfy3/xXCm7btBLRVEIfIypzX1PMKjpIqVtZM2RkmXMbByyMCra9l+dCRrYX0V6SflzmTKvRDllFteU4XC3P4Qu+Tyqc9dQsu9kQjb+U6pqSr57smdjD0DuqtPpYpPLU8lw7Ly7Nk3UOnBVpMSpKh2Tlxn6K4RHJswAeyyKnDRBPehGVh+q0aarfCyz6Tm/Vb15uc60op47bIZIIzE/TojlrahxAgo4m276phXYiLMMdMPfLZeiNkOXh5MVtmaHcFSO1bdqinjeJS3KbnWwUkUcxZ6V3jZEwxOKeXlspc7zYLAqKV9WS+AXLui2zFMRYxl3s3VS0+HYnUycuio5vkW2XKWohw9RQ10chz0sig5E4P7J9/kvTm8IcQT2ZNDC1p+JxvZW4OA6i5E1ZCPcMVjJqnlggqh64JB9EMxLNHNI+YXsUXA9JtNXTPHUABW4+C8AYQZKd81ujyCniFPCXv7KWmkYHXNx8wvdm8I8Nj/6ZEjbwzw0w64TTC3cJyKeLRSwO1c65UvNqhpC6QN7csle2NouHIdYaGjjcOojCPmYQw6Mox9AsyPCpGVD/AC8uUuO14yEzMMxKX9nSSFe24jU8PzU5jrZaa3SwF1yHjMBwhz5JOZOwm4deyRklONi4X4gkaHNw2Vl+rtAtCLgPiWVgcxkIv0MllvRfaHw61/h46KdoHU2sqdd9oDrGKhpYYx0cHarXKhzNZw3jNDLyaukynuHXBVf7te0/0mXkjumxLiLF8TmvUVTso6AqkWuPre4/MrqL89FhscZezFMxHRZzxGD+GSR3Klja38ouk8AiyjKsJCHWNlYErbWCgMLw6+4Rlv5UmLBF4CQeLerRMI3OCcRj0qUHztPVObIC1rTq4oc7R6SSlILrunQF9/hSzm17D5q0JMxSDvdQ3KVylCHMWnROXvI1cpHQxuObUITExpy7qhxJljTtdfzP0Q3j9J37Ii0W82yjJRSGKoE8Z1C3qfGaWsifS1XkF7X3uufJgHoTxtYHZiLrNCzUPhjDmQAEdCqYez42ElWH5Sbt0ChkH5FoDvorUEvLgEEbCT3VZtPPK7QH6KxFHK25Y06I0O/4ZPVPhcH9K50jLqBwtJnK0sIZ4gyZJg3li5v1QWXnny+mw6hYFdMZqp7Wt8oNh7rop5GUsMkzt2N2XMkcyQ32JJUxZho0EJpo/EO9R2SqXWbfqmjmeXAv/YsFrITUskeXZLt6JTQ8U0ghmtqRYrObnllZDG25ebWWhWSsqKFsYPnYb2UWECGnmfV1Js6IXb7qg8VhbRsjpg78U7t7KxA7w2HiCbS6zM01fiJqLaHYdgrtWyaWQxt1A2CgnkojUwtaZwIR2KaKohphyeVHy/caq7JRz0lKIzGTcXuFiYiHMsO65Aq99LL54GWWcjcS9qisQvRiHLkKIxmxPZCtJdDa7VIlpCAalGIn2zWQBoAnzFO6M/CpqekmqCQzooqDr9UWZ4VrwsbYs4NyFFkulgQeoRtLb6NQ5eyTWvNrMJB620WbS0ociyy5c4jdbv0TZNLqdpklLIzI7LsiK4ZIfxHbKRsbTupZY+W7IXaNGnuga6yltJYWsb63IpJLOsDooHuubdimL9VmRdbLf1O1TSSF5sq4dYe6ONwzeZZiBK1z2utYhWA2wDlA2Rx3R59FlUjpBoF0ODsIgZ5d1yziTLEB+ZdrhkIEbW9l0wZlT4uku6mpzsBcrnjI1gWnxdWNkxYxs2jiYLLnnzkFMiFrneW2clRGQDfuqhqXfE5MKi+5WaLWZJWE3uhE2Xoqhm3CHmnurSL4qCNe3RSiYE53O+SzGy6hSiQXUpbaAeSd7oXl3NFmoInX83VG0Oc/Pm0SmlyNtzYKzLK8zcvygMtuq0J84VmaeBnr/wDdZyAnli5dugNYG+TPcfJRSzw5S7P8gs19T5zZcqGr4rzb+WyaKcOPnt7WWaKhh9eo7JxUW6pxkbHibjyNsBpZMSx7vPsVntkLhmJNlaimAALQD80qmll0EBFo4XE+yoyYbVE3joKlx7sC16OofHaWB+Q9wFpsrq94szE3RDsQt4yU42TDK/4KGc/+WVPhuHYvFXwTfdc5aDY6d11LjVO9eLE/JqjdEXftMXqB/YFl3tKc7iGGY9LVyzfdczmvcbW10U9NQ44xloMBee5dHqtzwlMB/plaf9pE2GC2Tn1f+8f81jJriyBFxHlLJMKm19k33Vi84GfDZbjYnSy2mnD4m2FRiH0kCrVM9EG3jlxCQ9iSFhqlA4VxBDaOOiLfcSgFGcAxadjTLGM3aSYaKcVkbDZjawDsFciET2B75Jxf8xW2eLHGD4lhxE1RJBy72sx9yE7i5XnQQ1L3EYlDAAbZZTcphhTz5vH0jm9+ZukuGS63gXiTGWU+LULaWKF0QEbpZrEgdbWU0f2W8TuZzpMQofkCT/goon1LYmQR1U+SMWaBLYAKpWT44yZngWzy2F5PxToCo3gxw6ucCIMPkkIJF2RXChjosWazIzB6o/Ntl08Ms/Ka17+W7qL2Q1b6gwEMeSdzdZVz4oMedthczfmFFLhWOFtnYbMfkFvQvqQzPM8HtZWIah4u54JA7BG6YdDhGLVUuSoj8Izq+bZaVBhhwuUwOrY6nOb3YCAFbfWUp9eGxn/ZP+aGCehdUh7I+S3s2NbiWJharMNgxHEqSrnqnQup7ZWgXDgDfU9FBV4BhhrnYiyrmc4yCTlOAsrE+IYcS3k1MLtLHOw6FReIpD/1mi/3SplhbcK03B2B4pNJX1GJTxOlcXlrALAqq7g7heFxBxPEHW7NC6CgwqqxK/gpMOkyi9hLrb5WQ1nC/FTjy4MNpBbc81qszxamWA7hvhRvpmxJ31YFGOH+F7/s8Rd83hdNDw7xJC0CTC6Rx/dcCrEPD/Fk8ogpcFZJIdmtYCVObOWUYxc+jl28P8KAeanqwP3pP+KmGAcJn9nQuf8AOZdT/NDj/wD/AIYm/wDQas+swzjWjqfBO4ZndUZcxY2nB07qc3GOr0T6Zx94ZIwXABtg7T/5oUrcPwNvl+6IQe11lV9ZitFJyK6CphkHqa+Eiyzzib43PeyZzXvNyS0hY55PTGUTFw6plNw8HiN+Gxa7gOITSu4eE/IocJidb1OfqL+y4t2KAuLZ5hIe4VqLEKV1Nka92X8o0WfzuluopazA5onSQ4XTuLDbKGK03FcDjGafhegt3IuuSp5yHZIH8s9PPZXHYm6B/Lke2fsCNT9Vx45wW6WHihlO8Oo8GoBF0BjvZT41X4RxE+lFTh7IGwFwu2PobdrdlzvioKiAhmdr8ucENvYqZlewxNsxw+eiuvL8zjnNu3wPhPguQGWCmiqZWStbndduVar+D+CXvLv5twSOO5de5K83ZVyslpxG+UZpMxyabBTU2P1FLVRywVL+aDduZxI/Req35vqu1dXszyzw6ifp/wDr+z2HCW0GBUoosMwWjihDi6xjB1KvNx2RpuMOoi783IaF5THxxj0l8k8RtofIpI+NscZI0ySRloIzAM3C5z02nKbyxi/o+Ds9meq2zOefGZn5+v7PWf50YjlLG09MGnTSIKia5xcXmnguf3AufpOKcKqYY5H1ccT3j9m46g9lrNcHAEbEXTDTr1/wYxD4mfSZdLNZYcf0WPHzDaKMHsGBO3F62NwcwRtINx5FXSyey3xiWKhr/wA8+If+0s/9NqH+eHEfStAHbI1ZOUIspXH8Lo/kj7Q5fh9X8sfZqfzv4j/+5v8Ao1qX87eInb4nIsvKUspT8Lo/kj7H4fV/LH2an86Mf/8AuEqY8T484/6fJ/vFZ17ap8wV/CaP5I+x+H1fyx9mh/OTG+tfJt3KY8R404Fhr5tRbcqhf2Sv7J+E0/yR9oTwNX8sGdJM4kyTSG/d7kiZT6pHEe5T/QpfQrrUO1QG8vwuP6p6abEKKpjraKqMNREbteNU+b2P6J2/VZnCJipgmInyl1eB/adxDhRea+jpcQB9JdEGlq6GP7aqf/rXCQt3DgV5smyheXZ27p85ucXmy6bVlNzD1IfbJgErLzcMTNPYRtKZv2t8HvP42B1EfzhC8uy+6drfdT/K+m/l/eU/B6fh+8vVB9q/A7h/8rm+sQWA/F/sfq6oy1GAzgyvzPdl77lcZywUuU1ax7fp13wuP1l6dOM9Nc6cpxv4TMf8u4rar7EaeEvpsHnmff0tYVRGO/ZNH6eFK3/dP+a5XlO+AJ+Ueq1h0eOMVOWU/rLtj1HUYxU7s5//ACn+7qxjv2Un/wDZes+rT/mqdXiv2fSyB1JgdXC23pyk6/qsLlNQ5GhdcNGOE3Ez95l1w6zqMJuNuX/dP92p948KA/8AyWqt/ZC2KfjrB6akjoYsJqRDDfI2zdFytgla66zjE+pt6vduiI2ZzNfGZdX/AD5wb4sKqR/sNS/nphO/hawf7DVymRvdLI3ulOPiS6k8dUfSmrz+iidx4y/4eFV8n/mWXNXbGn58P5kodH/P4/8A2Wq+sgT/AM/T/wDZp/8AfH+S5l00TviXS4NwNDj0rKXDsfjkqXgu5TIyDce91zzyw1YznnNRHrM+hETM1ATxvP8ABhDvrKo3ca4gfRhEJ/tTf8FuYl9kGO4NRvqqueQmP1MjjDrLmX4PBGcs1dPG7s6BctO7X1GPLVNx8mMZjOZr3eSyeNsSH/0em/8AW/4IXccYr8GGwD/aKhGDUJHnxSU/KJEMHw1v/X5z/wCUF2t18PIA4sxTmmbwNHqLWF1M3jTGW+mipx8guIx7EH0OKzUtPiUgjYdLwNKpNx2pI/0+Y/KBqt4vvavZrrt2uNmMxUxfrP8AZ6H/ADzxw/8AVqcj3al/O3Fyc3hYB8gvO3Y9Pe33jKD/AOC1L76q+mLyf+iEvFv/AMK9d8cfv/7PRf52Y18TIR7ZUw4rxn/Uw/ovO/vyuPqxi/8A5DUvvur/APvD/wD0mpeJ/wCFeu+OP3/9i4truJ6aaaUPi5FSfMQza/TdYuFz4wyVmHYfBG6/pDmglScQVLpGw1k2JvqHl2QRluUfouy4Q4Qw59PDjdTiswmmZnjYIr5B76rlyp+m2bt3bOgj8RMTn6RXx9y1Q09XBTMjqYaaV+5dkAVjlhvmZh9OHd8gWx9z0I2xN4/8tL7ppP8A7q//ANILp4mL8Nl1vW55TlOyfuxjFm3w2j/9IKKtgqZaV7aaGnhktcP5YNlvfdNJ/wDdX/8ApBJ2EUjgWnFQARbWI/5p4mJh1vW45Rl4k/eXknD+HV9ZVVAonhri9zZn5LgDNv8AxXplJXYvSwRwtrB5GhuYRDVUMLwnBcHq56eKo5UlTK5gc43zHfRbjcIv/wDUox/5RXm1bIm3v7512zr8sOMVhHp5fdH9+44Bl8c76NAUb8Wxp/rxKf8A3la+5h/91hHzjIV3DuCcexjOMGdDWGP1BjrWXXb1OnRhz25REfGZqHwsdWeU1jFsY4njDdsVqh8n2QuxHF5R+JitWf8AzCujqfst+0amgfVzcNyGCMZnyNkaRZc9LQ4hTNzVVHPFbfNGbLOnqtPUxOWnOMoj4TE/0XPVlr/iilZ3Pebvqqh593lcTi1ZS0PFDZm0hfJEbvcXXzrtueGg+cCwuV5tj9TBUcQyywyB7HAWIWt/8Fvv+y+mN3VZRlHlxn93okEUEsLJWRC5aDftcKTlNHRYfBNZE/DjTPnzSxyPJaTcgdF0PQrpjPlD4vW9PPS9Rlqn3Sj5Y/Kly/zBGi0JW3mRGMEWulylKWpZfmoyiEaIxtH+SPI5Kx7qhQxMlD4pBdr22I9l41xRSYfh3EFZHBjUlFmcC6O2xXtNN5Xlx7LyH7RMMrqHjds1NHTVQroMwjlbpZdMX6f2Z2xjvy1zPrHlH/z5IaSXmUoDaw1N95LbrpuAxatqf/DH9652IZIgDDHE4i7ms2BXTcBt/ptTf/Vj+9XL0fou8/8A9HZ9P+YdnlKWUqTImyhcn84BlCe2iLKEg0XSiw5W902UI7aIUZNlCXTRFlKZQJIOASSyuVsNnsUr9tk+T2SsPSoGLtUs5T2HZIgBatqjF1kswTkJooJqmZlPALvkOiyyF8gCYTsPpYXfIL2Phv8Ak+PxTDabFp8ZiJnZnDHNJstes+yqHhx0JqJKSWN51EcZBXwsfabtWe6emx2xOcTMVU+seselOu/Rs6fVO/ZH5YeU8NYDBiVXE/nVcMg1/Y5g0d91wnE+HVEWOVsMENLPkmcM0jct/wCK+lZsRw3DCxpYxjdG5gNQvPPtK+y2oxGmfjPCJE7qv1Rt1cXH4h7LXS91x3bpjPyifR39l+96dfWZYb54xlFR8L+cvAKjCKCkzPxTEo47m5YzYKpJifC9LaOCkfVuHUAn+KgxzhTijh6uFJxPh81FJICWGcZQ4LMdTiM7sPyK+3jsjKLjzh/UNeeG3GM9cxMT7482s7il0Y/omERsHQn/ANlVl4hxiri8lVDDf8rdVnT03iG5OYWD2TR4dPGC2iqsvs4Xus826RVlbisEZmOKylo3ylZ/g/vSETmquX/nOq0jzy50dcITbcJo4sNeSynleHDdgOynJWQeHaiMF4qYz7LSojHDCIJDZ46WVkMbbKw3VCrfBG/WF+cfGNQs8k5Lz8rbeU69SoanKYXZnAaXTwujmiaY5gTbYlBmo5iY52NJb2XLkclKiqYeV+NOGO7FNVTNcTNHijRboSp6yjwx8RyOH1VRtNRzQ8mOnJI7my03yamF4k+eHM6S9jqrpxWCPy57Lm6GirqXPyx5Xm4F9ldDA8XmitIPdc5xbjZTRmxRgYXQygEd1ixYi+WrE07ruVtnJlcI26k9FlzUM8E5LCExheToZsQpYKcVHKzu+aVFjlJXROy0rgR2WDLR10sOfOBboShojU0rTklLXd2ldKcubo8PrHurWxuexj2G9j1XYy8Q01LSh80t3D4dl5bHLLzudPJm7lX58U57MjyHHoXDVOSO9w/H2V03Lb5R3ut6NlwvHKTGaugqxyrEdiF3eG8YQmnElZSysv1j2XSMkdBV4lSUUZkkft0WeOM8GzBj2SC/VcfjuIS1cjpoZLArnZKmuD+VnYb9Qlj2inqqWrZzKV1wmmp2v9YJC5Tg2vfDGyCduj/ivsuzvfSyWrKnwiGR+djbLk+LcFxYxB8FKHR63dH6+i9AIb0SWsR4fUxOowbMnY8fmaVRqKqokbkfIdV7vWfdgp3PrYYiel4wbrzPE6vCHVsmWCnkiOwDVuxxYp4ycxCPkRAbLrcL4dwTHKoMaZIGP+NhuQtef7In+qhxpzv3Zov8brpY82kiaNQE1v3gfku8d9meOUzsr+RN/Zcp6Th6sw1pNTgDZW9yLhOaU4Kmpqqcl0VLLI0dQFa+7ppB56Z7T7rsnGSlcfCYe6Bh1yN2uqdVPPVvBkjtZcvEKc4zCrD8S91DNQAP8oXSkMy3eNFSdA0m9lnxjiwTTHo1RmnPVq3nUn7iDwp/1S340JTC8MSdkXgC70C5W4aWKRv7MAoPAOHms63cbq+NBTG+75B6mFRS0zozbISuhjhDBYjP80LqWHpGVnxoKYEdPmF89vZEab8y1jQMvmyOHui8EwfmWvFxKc+2GYszsZcJ44o73nlNx0CbNJ0cU+Q7uXRxC+KG5ey+g3Krcx53KtPDg1VstigkYYQcuRxPsprx5Lag9Ao2PeBawsjGoUsRFzydFMyJzQDIQL9AkxjAb2Qyb5lRbgmMMjQDZqtOqI5CXNezMVjCR7nBSF0YeJJCRYW0ShPM0Em6jgMtPK18IN7os5naZI9gpoJWRygzahZa4rWKPL6dod6yLuWPEzM3MVq1NQyYFjGNt0IVPKBo1IOKJ0kmXls0b1Fk1iGqR7bG3VMWE9VtqkTmPLcx6ILeUqV0bx0Q8t+UtIIuo5lRuMbxIxbVFUwtc+SZoIP8FlxR5B6dFIC65yrMuro6LF2ZuSw2jcocZ4epcRHOpJeS/t0WAHPHxKya2sI/0qT9VzYZj6eWmeYZmEOYbfNNy2lTSOL3Xe8k9yhydl0jIQPbc5MqjMTQbWVljbfNFyS83tqt2KrInh3pV2MAsAKcQEAHLupBTHQ317JYiNJmBtsVPRMYJDGLjTe6lYLHICLu6IwA1+gA+S5zkyh8OyOYjVzD3RPiiEZs1rbbXCseU7oXQMcbrl4kik2jNg6xGU6+6OSMG5tZo6dlpRwmS0MbbgqF8DWOLLi491vxJa4s97yABkAAQtlkac7Ga9CpzBKXnKy4RysPLyAgW6LXIUyXyHKXXcdh1KB7Zo3ESRub81O2uqKNp8JEGyk3Mh1KCKWtrZwKh+dx6uctQIRdpuiJumc3RzfiZv2UPNaPiWklYDrFO19iDuoA4EJw53ZQW+YTtojBuq7XuPlUoB9RJUpV3Dmcyvh9jddzhsXlC47BIs9YHnou4pckcEtQT6IyQFtHnuNVDp8Wq5M1xzLD6LOc6yIySTPfLkPncSoX3soqJ7rhCx+upRZM3xWSbTySWEQDyejSqkwFzkBN1ZbheIuDi6kcAB1TjDqkta6Nma+hF9kEDRc2VmNltDuCnfQTQvtuR0Vk5HubaAsIbqLdVKaKMPtsrMR8qqNkqXODBTZRfUuOyuNp5HHLHO2/cBZoTQZjK0dygnkjE7g92oTNo6+GTnNqGuMeoaeqdtHJnMlVmaJBqWq03SjNLdttfmFVllA0C2hR0MTLESSfNyt07qSGkyPpGn5aKUU5fm3KnYXOsulhqcKYwZcLAP7x/wCCsjFKWMfg4WL97f8ABWxz8TJmMtkJ9wtCkpJJY82Qj5qzLKysaCHiNztwToFcpn0sUeSqrxKR0Y29lyzgRUEUscnLkZa3XoVfdHH4kAMuegVc1tHFpGHub+dxtb6I462gkdmE4e7sFmI4kJTEzMeaw+2qTXRS35LMlutkMtfQgjOJNfqndNBy+Yx5jsb91poLZCIycvl21CdrZS0yCQnINGEaJzVUQd5XOdI8aXGgTGupWNs2qs5g1a0FA7GzSxl7o2XPwtCPlljfM6yjbIx0bXZwXEX8riip3Ujbw8wvc83ObZAJa+MAvnuCbeVP93Qy5n1b3NaB03up/wCjczlxzDTYITKw/tJ2OGbU9EEL6ane/JkPkHsnbQ4a6NwyGKQC+YjdPJyQXP8AK3P2Oh+ilhgxGpYHw0D6hgFmuZE9wRmcox85RU8b2y3Y1rmkEC6m5eX+qiHy1VmDhzi6dueDA6kt7lpCsQcFcWSSl89A+G+5LiP4ZVLZ8TD4wziG9oz/ALJCZ5YNGjUrrcO+zDHsSDyMQhZk3L2uaVLX/ZjFgobUYrVyzGTyDK+wv+iW5R1nTzs8GM45fBxWYD12+o0UczqcjIyY3PYrrxw3w+31UUr/AJuVmHDcDhaBHhMAI+J4BKPS4GYtiyCivK4vGYFpOi2oMDxzFWtdhmA1MsI0L2R2Ad9V2bCIhanZExvZoAWxg/EuIYQHND2yMI0Y52g91XDqMt2OF6IicvhLmKX7FuKqmBs5fRwZhfLJLZwXSYX9huGNjidjGKyF9vxGQDRx/tLTPH1c7/q8H+8U38/sQG0EH+8VLl+d3a+97vK4j6TEL9B9mvCHD1NIaGEwNNubM593O+v1XNVbsOjrw2n5vhWmzrnU97KfF+KK3F4hDIWRMG4Y71fNYhaT/Wv/AN1H0u29J1WvGcur2TMz7r9P/ddxGXDjK37vEoZbXmHW6hjlkidzIZGscNjmsVVfEf8AXn9FE5knSYD6I+rhhGOPCfP6uno+MMZoadsEckTwOr3XK6XDOMaGsl/EbFDKGDM95Av7LzE4dVPOYTvB7ObZCMLxF7sgnjHvslPldV2TpepiajjPxh6tWV2BzxSGUUUl2m4Bbc6e68uxCSjZUPNFhg5V/LmDL/3KB/D2JF4eHiZw+HmABDLh+OUkb5GYYwuYNBnzAntuucS69v7ZHQXWc5X8VWow/D6+/icEpHX3LoxdctxHw3wm6Jn3fzIKi9yIzdoAtuunkxbiTkZMQ4QppGDqP+ComvoT5qjgqna7ucw/wVjN9WMXFVHDYfk8DVvY69yJATc/4KLwVRR1f9Me0th85ym4K6fEg2tqObT0ooWWtkZewVE4XDMJBV1z5OYbks8pCTlyZyhnSSZweRKfxCGC2lwtGY0sFHnqpo2v/dCqVDY6Wn8Fh9K55/MTc/qggbPiFPyMQj5NnXY62lwucw5zC1QzQy3EMhcXnJYqOalfG50kYdEwEMBLb2PVO0Mwy8cDo5QfiRxU2M1rH8yCohppHh7ZXbNCJS285cNE8cd+WC8jNv3OiDBql5rGSMgZKHl5s8XB07IqLDq/DXtp3vp5c77AsF3OB7r0PCP5PXHGLmLEsKwtkLXBr2ufM2Ozfdrlz2btemp2ZRDzdV1vTdFET1GcY36XLmqSXC8UrG0NTgj4JnHR9G6xHuQvecG+xjGqXDI+TijJTIwOjEsh297BaH2f/YRg3B9VT47j+NOqcTiaBLA3Ly7na5+Jel1WM01Kx0jJwGMaPKDr+i+T1Xd88tkYdL5/o/nXtF7SfjNkdP0VTjHvrzu/d/f328SxjgviLBJhHU0RfGP6yN12rJyRwEirZVxu6BrQP712tVi+N1uP1TJ65jKWMl8cI3N+v8FNnfJ5ZrSN7EaL7WjPZ4ceL/E8GqM+EeJ6uF5lCNn1h+rERfSutyY5r9S8j/BdbUcLUNdBLPDRuhc31SxjQf2h1XHsaWk/Mhdsconyh0yxEkwMzfiNfb93dGmGy3dsDtR/6mf/AHwmtR9YJz/5g/yTJJaGyflFk1ijylOGqNjjNJa09JPKe7ZbBLNhv/22f/8AyEOUpZSjIubhw9OGyg9zObKOQ5nkhmVvQXvZOkoGt3RxckOvOx7m9mmxUaduyrSxnw3/ALDU/wDrpZsM/wCw1H/rf8FFlCdAZdQWtHSTsd3MtwoUWUJ0ukqwKWF8LJAZ4jK0fCH5b/VNlKWUqK6fCMQ+zhtE0YzwtVOqcxuYqpwbborNRiH2VugfyOGcREuU5L1jrX6Ljkl83PtsZ7J2eLnHndRlNfb4O0bqiuMfZOarC/hwFwP/APNOVc7l1rDt2ReXsmX0nE2UKRobBZ9RRCZrxoC4j+5MGp8v7yyOh4YxvgKjldJxNwlLPlcDGaeTp7hy+guDuJPsa4gpYjhdFhuH1co5bYKiBolIC+XcjU9OX0szKmlkdHKw3a9psQV+Y777N495jlG7LHKPTzmo8vh9re3pusnp/LjEw9x+0D7DMQioZcY4Nw/DsQkLnSFg8pI7AXXic8/EvC8jDxDw5V4VM4eSTK5oK0YeMOMKR16LiXEIe2WU2/TZekYD9s+HVeHsg49wWnxSeABjHviJL2+5Xj6fHvXZOn49TH4nG/d/FH38ph0znp+pzvD8k/s8ri4zrJPxYseqGF24MxIP+8tuj4wxIuYaqKlqYb+Zzm+Yhe1PwH7CeO+H4qufDMJwttU3M2RknKlZ9AAuB4z+xf7LsNwirxfhLj8Q1cEeaKB9QHsce2pU6P2v6Ldl+H2a89eczxrjP09Y8oNnbs8YnLGYmPW7cNxhxlhAmZU03DrqeBjQxxYzNr38qwfvGrqohVUGIQiM7B8d7qvUfzhpmPYa6ir2PFvI6xCGJ1a7DvCGhhY49S7/AIL9brwjXjGMe506XuWrp9Ua8tOOVe+fVlYhgdVX1j6qethL5N7RusoDwo/pWRD/AGV0bsSxVz2sNFC0WtcSX/wVwYo+FgEjYjfsF0fXw9rt2vGMcdcREfOXEy8P08D8lRisUbiL2cy3+KyIKWjqcVmpI8TiDIABmI0J6rv5MNwriCqbJXF1NUegEEFlv0VSX7F34SyrqaDEPFVE7g9rDIA1vdy8m3rdWnLhsuP08n0tXth0Ucceomccp+U193O/cuGM/wDrFL/stQOwjDX+jG6b6tv/AIqrxHTVnDFUaSujbI9oveN1wfkeqy3YvSGKJ+Qlzxdzcw8p7L1YzGccsZuH6LV1eO7CNmubifSYaVZwpQ1eX/8ASaliyX0bGbLqcPxabD6CGgjx2gcyBmQEsIK4SPGKS2bkAfORHHjFHI2/hx9JFqmd+GHU4xjuxiY+b0Kk4ka2oYazGqTk38wbGb2XVtn4Ukja9/FtEMwBtlNwvC5sbpOe1jIjl+MZgk7G8O/7NN9ApxfI6rsujqJidf5Po915nCvwcX0R/wBgrLquIuGaOZ8DcbjkLdA5sZsV479+UPwwzD5gf5pfftJ/q5f0H+azxcdPs/pwyvPKco+zoWYpO/i17psQY8Rvzxva3yD/AGbr0/C67BMQp2PPElLFKTlLXtIJPsvEKarjnq5apna9kbeIaJjw9j5A5puDl2K568amYfQ7h2zV1mvHCPyzj6V/R9AjDKD+s4ipf90q1SRw0WbwfFjYc2+Rrhf9F4jT/ak6GFkTqfmlgtmde5U4+1gj/qzf90reWqM445RcPyc9j7hjP5Yj7vbXzVMrHMfxmS1wsQ5zgCpqGobTSRtm4oY+AOGdtr6dbLxA/a2y3+gt/QqvJ9rAlY6NtCBmFrgm4WcdGGGM44RV/DyY/wAk7jlMcoj7vsHGONfsZwngwzVseB4hMITE2FsQZUAuBAJt2XxtxfTcPU1dT1uBuY1tQ5wlYxxIBJCw6zHn1UjZYCQ8uzSGQ6OHZBPiMVbJEI4DHkdc63uvj9s7Jh2jDbOOzLOc5uZyn9v3/V+07f0uPSxUevlb1r7NH/Z9gfEdFiWP04rInvZywyYgB5PqkHVq+kKmP7IOPYzRU9BHCxrs7n4dKC4W9gviCauYLwMY/TqRZafCfGeK8N4iyTDqh0HMOR2V1hY9V4+8ezm3umzHq+n6jLXsxjy+FvL3Loo3ROcRc/B9o1P8m/g3GcMkdwlj9SyruMpmeHBv+yvMuMfsT424Lcw1VK6ujkJAlpWFwaPdc5hH2pYjhNdHXMx0zGMEZHzEAr1DC/5XGE4JQxniSd9W95IMUTc1v9pfFyj2q7JMVMdTh7/fP0+NQ/Ia+n6frsuGGM45PFXNcx7o3tLXNNiD0Kde9cL8e/ybPtYnqsOho24NWvjL3T1DG0+p9RDr5brmeJfsCxDDYZ8XwjibC6nDW3dDLNVsaXN6eba6+90ftT0+zZOjrsJ0ZxXln5XfweXqu09R08+lvKzfoms4bI5In07uXM0gna5uCkG9n/wX6uJfMmK8gA+ZcX9plLBEaXH6oTHw7TEDG25AJ6ruA34syr4lBFV4dUU80Ye0xPNj3A0WsZezt/Uz0fUY7Y/+RLx+ixSkxVrjRmQ5RfzNI0XofBuFCkpDWyMeyaXykO7dFzf2YYRTV1EZKgZhSyOaG20PmXpDmBgDQLAaALWUv0PtD3K4no9f6/2MklY9k/nvquT8kHKErDsnuduiVz2+qpRrO/KllIPpRe9rpwT2sEhlFZ3ZPb2CPObjy6JZh8TDb2UaRm1w3YpW/eKk5sf5T+iQAd6W2QRZXD4imy3P7T+CmsPiSyhUQ2KVipbAn169ksov5QgisdkzZZ6eaOqpn5ZYjcKbJ7JixW0p9E/Zx9sHC9bRUHD1ZnoquKLI58pAjvfv9V1XEXDuJ18kuIU0rZoAAWNa65+i+TGh0bw9psQbg9lsU/GnF9EQ+k4jrorbASL+f9V7GZaut/G9tzjGcr5Rlc3Mzc18Le3dvw6zp/w/UxNR6V5PVuKOC8aqaYyx0crQw5jcbLjaPEcZwKUNZUTUrx8BNwul4b/lC4swGl4lpopqdkYaCGFz3EdytKv+2D7KsaiFNieCVMRvcvihBsvnx/mujb4XU9LlPzw/NFPkbuxapx5dPtj6ZeUsCbialxSNsWP4HS17W6NzMbouTxn7L/sj4nrXVz4azCZZAA5kT7MH0XrOEcDfZlxVTRVXD3FE1NJMwuZC+QB31B1WO/7FuOIpJDHPQVEV/K4S2cfmrq79o1ZZYzs8PLHymMvyz+kT6mjou8dBHidNM/8A4zbwmb+TPi9TNI7AeKsPlp8xyNlJa63S64TFvso+0fC6uambw5PVNhdlEsADmO9wvoyt4W4lw972VWC1Qym2djC5qqU3EOJUcnJgxiojezdhkvb6L9Do71ty8/LJ9PR7Zdy6eZ8fCMo+cU+VMWwXEaCrdT4zQz0lQDZzH+UqlT0dPRPMkDLEixX1ZidScSfI/E6WirHTesyQ3LiuGf8AZhwnPMXVGHsPMcSQwhhX09XdtOUf6kTD9D0ntv0W3H/6jGcJ+/19HhMss8LjPA85h0cbhZrMQxGVzhUsdYi1sq9yxb7KcCZVZcMwuZlLlF3uqPiUUP2O4PUOtzmR3/NMQvZj1fTZRfOH2dftH2vZhGfjYxfxmp+zwmKHEYn52RCQe6sRF8s3noTGT2XpuPfZ3T4RWSwR4m5nK3cGBzP1vZZX8za8sL6fE4pu3ktf9F6MeOccsfOH19W3XvwjPXMTE/Bxc1JRQ3qpnk2+A7KeiqMNqZOWykMLz8YNgt6TgvFwfxaSOQdQJP8Agq7sFr6XQYRLbtbMtcFpnT1LKdpgfJld8lQbNUF3kkBF+ykxLBqueoMlQyaAns02KGlwumhs018mYdCLJxWmoYcsDZ4WM54IILVBOx8rxVF+V9rOZbRWIZnAC50VDGGwBnihLZ4/KVz4FI5DPUPdGw2HdVmUVc6/KbnVfxc/Ryu0tXUU7Oe+x9gV0plmNknY/LJob7KWera2I3ViZ9FiUrTU0skX77Dqq/gmUri0vdOzpZa4jYw/G6KqgEFVTse7o8M1BQVGMS05MMM+RvyWNFVQ003lgkH0UtTy6sF7IxmO5C14aLBxJlUclRObn4gmcaCEibx0pI2s1ZvhZGvD7bK9BTQzMvIE4K1qPE4TGORMXN7rTh4iromhjMQnaB0uuZaKZnljqiPmFNE5rv6wO+Ss624l1jeNK+PeqY7+1GpovtBmEgM1PE9vVuoXMMlgLCx8pbfsoJY4tHQkuHyU4tOtxni+DEqJ0DMPyOP711wNW5xnL1oZnkeRpPyUTqd8h1p5P0WuLNosLxHEKGpE1LJa3Rel4VxZV1kLbxR53LzptARq2Cb6BXaaCYsDLSNt9FMoW3ozsSxJw81Ez9VQrMTrmx2kbkv73XKRtxSHWOvmHtdStqsXJySVRc3sV5spVquxGpDb3B9iFWmr3SDzwx/oqcs9XGy4a0/NUn1tS/8AaAD5LjMiarcwtu0AaqrzLHZRTVXlOt1UdVPO2iUNLmXbexQBxJVKOpkuGkq1FUUz3ZeaA7sVmpFghhHosm5bSLBycOY4aa/VLRxuN1LkQPZlKRygXcik0vdROewg32WoCIJ1tdROcfhU0Msb9C4D5oJY2suRb6K+Y5psEgHmankFjboibIXMz9EJkuvoOCMiyHlh3RO52qAyAekqwhnRlp0SYXndPmJ3Tc5g8mU3WmhecJnNvukJbC3XuhLykQGYLOB6hTUUDZ6vK70MFyogNVbkc+Ci/D0e87KswpSyHxDmRsAj6BSlgP7N1x3TT0slNFG+S+Z4vqoWPe1ot1WWpW+Y2wDNLaJzNDC0l+rz26KpmsC4JQt8RLZ8mUDX5qsrhjeIxUyEa62CdhzG1gE73NMjY23I2ACeSBsbS+3m7dVlrkE5YjnIv7IC/MS4uAQHNYk7BVzI1wsFaFkSA7EWRhwt5VRBu4NVtjSGZsqlJZ3H8qbmaWQGaNu7kUDRUvLIwSbdFKaopKZ4yyGwahzM7op4J435HteW/K6BzeW25aQPdKSjjKTmGyMPAUcJzK0IWjQjzJZxM12UhzXE+x6KyxkW5ab+xUIg5Zvcn5hSBsjm5g+30UlqgvblJdonY9VpXlhylyh50l9HLPEpf5ubQNuVJCHud+ILN6lRUJiL3PkdYMF7oRVySzZbgNOwAU4lLzTYOIeQ0bWTavb8+vVPqIzZ+W/smiDg3vfus0UdsDQ3zOURpmZieZb5qZzJ82cTxho3uNlCTTf1teC7qSFYhVaWIukyZ227qvLC9pysOvsrJbRN87K0E9wkyMSm7TnHddcZpKZw8TC/Oxl/Yqfw4qvOyIh/5Vbko+ZtHdNJB4TK2NuVxFyt2UzzA9j8milFNGzSR2Y92rRc6lbG1s8fMfe2YA6KN4jY8MjlysI3tcqWzaF5dNb8MCw6BRsblc43JudlZOrSyOVtup7p4KOdrjK9mj+qNQ18Biu0yDYvNl0OKSPpeH6mRmhLcgPzWVgkWWNrFq4+b4IKXpI4XW2Hn9Pn5IANtSn8NG/1lbYwSkDi4Voy9j0Tuw2hb/10E+zVzhuGL4TTSMn5IWxvjeHxx2I2K3fA0wP7OQ/NTw4dRyuIfG8AC5Icq0xIpqvNnmfoEBY9x8np7roW4bQnR3NF9geoRjCYBoI9PmpyGC2OZnUO9+qTY5pX2tdbzsMYx2WNgSOFw6cxgv8AxWuaMkMma3lx02Yj4jqAly6rR5hcT+4FqikyA/h5gNgpYIpJjkeTk6N2UsYQbUPeMjHfVWW01bJGBJGcrdg0rXdE17QzLlsb2BRGnjLbMkDD1JVtbYoikjOXwpcCbG6Kqjex7Y20sgFr91r8uGEhgnDr7gKMNopXue8uA2OdRGMKeoebCnkHzCM0New5n2c0dAtpppS50cclswsS7ZQsbG2/Miu4+nId0tWWcLrZWiRlMcnclSMopIzZtgSr0rKt1hSMmb+7lutCGHE52iBmD1Ob2iIWJ/MM3w0kRMdwHDf2UYpprk3Dz3HRdEOE8flP+h07faSWxVqD7Pqt4L6vE2wPPwweYD6qo5Q0z2tJmkJ9g1R82nbcSQySA62Pdd9Q8C4PSXdPVVU7iLXL7K3/ADM4YOs1AZnd3uuUHm7J42vDnUouOjjdWuTNiDXPho3OkOzoonFej02D4JRW8LhdPHl2tGtmjxqtw6Iw0BZG0m/oRz2TnGN64iZ+fl/d5nScNYrWMa0cPTtd1eRYLosD+yTiPEzFPUPp6OB51c92ZzfoF6JTcZmOBjZ6ASyNFnPJGpUh44uLNwwgezljnL4/UdT3PKJx1aan43E/2c4/7DKeTSbicub1ywEH+9XR9j3DtFC55ldM1oJPMuQtb+fDf+wH/eU9JxbSVr3U9ZDyInNILib/AEUt8zKe94Ryyuo//wAs7A+AuGgY6+GkpZ2EXymMa/NdfSUkdJCKajiEUbdmNAssmixDhrCmujpKkMzalupVj+c+Djerb+ixOT5vXR13VZzcZzj7rj/iPJfmim5b3MDpHgXa0utcrm8R4jxXCpWxVWFsa5wuLPJ0WnLxZhMcLnxVAfIBcN7lcbj+P1GMVEc7KZ0eRuW173WYers/bM9uyuo1fl+M3E/1hfreNK6eB0UNKYHnZ7SbhYldiOJV7WsrJZpQw3AcNlW8ROWkuBuOiQnncP2J/Vah+u6foOm6X/ysIj+v3Bl+f6JyIz63o3PLRd4NlA/EaVsbpJqqONjN7kLVvYLLTf69o+ZTthgd/Wx/76oy8S4NE0vBfMR05SpScY0srSylwTmHq4mwarY3/CyEHw45lt7G6OPD66RmbIR81w9RV1VdPdkwgZ1ZAbBNzMQi/wBFxOeI98xKzY7XwM4Pnk/RTCgkcPXK9cJzOIHNMn3rLJbrzLKzS4jxdTQZqetkez943Sx1/IfDvCT8wn8RINGwj/dXLM4r4ugdm/AkPZ4RQcccTTPySYXTub7ghLHUGrkPreojOPzhYR4tq2NbPVYTSujvZwZ6wr0PEeFVTrfdtU352ViRd8U2/msfmpBWR2tyz9AqcmLYOzV8VRGO+UFR/fmAjyurZ2nsYStDRZVU49DnszfNGZ2H+tKoDF8EvY4tEb9LFSitw1wuzEqe3TzIJ+ZI7+sik/deFFyubvQU8h+gRMfRPF21dOflKFKIXPNmNz/2QSiXEKvIijPnwen+jQglw3CZHc1+C0rnnfyrqcL4G4nxqkFZh+FTSQklocRl1C1cN+zHFJKkwY1L93sDSczhmN+y8u3qdOu+WUeTw7+69D00ZTt24xXrFxf29Xn78OwOZjqc4LCA/f3/AILosP8AsdqcUw2GuhwsiO1mZ6gsOX5L13CPsvwN1PDE3CRNLEAXVE4LGucDvZd1T8P0ERb4kl7miwYzysb/AJr5PUd4j00R934zuXttjERj0GNzfnOUeX6ebhOAfsa4OwCOmr6vB4qqtgPNjlnOZzT8vZeg4lLUxQZafRzyAXDdo7hTPlggYXSVDG26ZtSubxLjEwVRjggE7R6nXtYr48Yb+uz8rmX4Tdt6rue2c9szll/T6fBYpqOOKJ9TOXNda7nyu3XG4picdZWPmhtnOnl9Oi0cVxw4vTOgfTvOb4S/Rc/UYhhlA3l1MzGPHwN1K+72vocunvPZFS9/Q9JlpvLOPNJQ00Mcn4sj7yuvI8nzEq9jVRg1HyXRVLY42ssSRZzj8lyM/FNXKLUVO2AdHOF3rIkbJM/mTPMju7jcr6s4XlGV+nufQnG8oyv0dHV8ZVxw+ownC5ZoaWpsZr284G1uy587hIB9sqQDraq44Y43OMVfr8zKZn1ONrBKx7J27p10tk2UpZQnSVCTt3TJ9PdAGvuiGhDUJBvskAb7I0JM3ZOdkKMi0ATZiEyfKVWhN3T27oQbaXST1SRpJJKKSNAkgSSScWv5kDjqhRDcoUSBdfolm1QpIo07d1GjQHp7p73Qa3TrKWbKwnYfonyMHwBMCL5QlfWylKcAD4UJDCdGolG8G4aHgZzbVUJ7Be/RRlrVcmwauhidJJNCcgvbmi9ls8NfZvxjxfh/3ngGFGrpiS3MH2II7rzdR1OnpMee/OMY+MzUfu3hqyzmsYuXKldZwnPNPSyslkLmxuDWg9BZbX/QH9ph/wDo8YP/AIg/yUNd9nn2jcA0L8ZxTBOZQh4bKIpMxBPtZfJ6zuHQ9Zr8HTuwnKaqOUX+nm59b0W7ZpmIxn7N6HCeB8XpIKbing6jxFsOvNcHZvqL6rJ4n+xPgXF5mVXBuD0DHus11Ifww33u4qvhHF1NKI/ERFjXbEdF0lLWU9bEKikmDm9HDcL5Onqt3SZfln090vkdB3jre17Iy1ZTUf8ATN19njfFX2UYpw1UiGXhOgcXNzNa3Mbj5hcTOKGjfy6zhmngf2IK+sIeI8SipzS1DWVcTmlpEoGax/eWLX8FcI8TcmGjfFhTwCHx1Dc4cfnZfX6bvOGXluip+L992r250baw67HjPxjzj7esfu+aObhbv/oFEf8AZUeSgPo4cpD8tF7txB/Jna2lnr8Ixh3iY23jgYyzJD2XlvEX2fca8IQiqxzBpWwE5RNH5g4r36us0b/4Mn7Doe9dt6/y0bYmfSvSftPm5kNpG+b+bdJ9ZCnD6cat4covl1Tvlzus1wL+xNkxts71dV6eT7Xg4qrs8deK2nw2OAWsYgND3VttTTH9pw7R/O2iTo3DcImR6eoLj6N8MUonpW2ezAKG3ci6TqtnwYFhw+TUAb3Fx2vqh5ZOzh9V05Sx4eKQ1bPV9x4eD3y3SNRE8a4Hh7vflhCYtdXOHsRcfqmMelrfUapyk8LFI2WlA0wbDz847KniMba1rG09FT0j2G94xYlWAMpT39k9YqTwsTRVMAgZDW4VSyOYLF1jqpOdhH/2SAn9QgysPlzC/ulkb2WPP0hfCx+CVrsMFn/cdJ/EKYVGHD/6DSH5hVAwBOCBu266c5TwcPgndLhTjrgNKPkSFu1fHmJ1+DRcP1rHTYdDlyU7pTlbl2/Rc0cpN8pCV29yuWzXhtnHLZjEzjNxcek/GPgxOjXPri7bhLHeE6iuhw3iQ1eGYeQQ6eIiQs+TV7FwR9l32TcZz1MfDH2sV7pIWBzo6yJrGtF+gO6+Z7go4qiSEl0Er2E75XEXXg7h0nVdVEz0+/LXPyjGY+sxMX+753Udl6XqImJxjz+Uf/t9MYl/Jz+0enle7C6SmraW5yStqGguHQ2K8p4hlfwtiU+DcRUdVSzREskD4nZT3s7qFHhn29/a1g0MVNhnE74o4WBjGuaHAAKX7QPtlxP7QaSj++8OhdUUtG+CeVtgZ3E3zLh2vZ3vX1EYdfGGWub88biYr0mYn4+nk/PR7J4Y5fmma+U/+zmuB24dT1mIx4fI3kPfnjFrWv7LrZCHFeL8NVjKbGqV1RI9kLn5XOYbEL2GOXDJKFkNFVl1Wx5LnvfmaR0Frr9JsymaeDu3Yeoy2eLo/NdR8/KKv9k3XdMRrddyz7D/ALRaigpsRw7DPGxVETZRy3tBF+llzuO8H8S8MAHiDCZqMnbmiy+fp7r0O/Pwte3GcvhcX9n5bPRs1xeeMxH0Y7u6a57pmyxONgU+YdCvoW50QJT5ja10nbpkQ+YJ8xsEKSNDz65c2ibN80KWnujIkswTZgmG6NC0SQp8yvG2RDKhd6kwNkRcoFp21Su38iby9k922Ro5DLat0QmOF39WER9Ka9gkCWhqJMMqm1mHuMMzdA9u4W237QONYjeHiStaR2eFzp9k4PdeXf0HS9Vlz36scp+MxE/1hvHbswisZmHo+BfbjxThNJ4etAr5MxPNlcSbdtFzWK4xh+NPrMdqWNjxCaa7YWC3kO5uud8vZLy9l4+n7H0PR7ct3TYcMsvWvSYu6qfKI+kQ3s357seGybj5rkc1MzOXS1ZLtnF97KjWPxKedgikhcy/mPpsEd/YpXC9Wfb+m2TMzi+dn0WjO5nFQlnxOnfaM1LT+ZjgQrdPi2Pcgvcx8rGmxL23KkuO6cPc30vI+RXkz7NpmPy5TEvJl2zXMflmbUqzHIcSgkw7FMNpaiGTRzc1rqpFgXAg1+5Hxf8AhFwC03U9G+QTvpojI3Z2QIBQ0XNY57ZQwG5aw7rzz2vqNUVpz8vsmOnrOmjj0+2Yj5TMf0HT0fC7HAwQRZujZozYqpNwxLU1M0sGJQRxSOu2JkYAYOwV51DhMj80c1TEOxaFA/Cw3SPEYy7oHtI/xXLXp7lrm8b+8S69P1/eemynLXsy8/jN/wBbDHwbRE/jz1QHdgYQpWfZhwLiUnLr66pGfq6AAfwV7DMJxGLJOK+ieHeqJstj/cugdhWJwhrp8PlaCLnKMwXPZ1XXaprOZj9Ibz9pu9a5rLbMfpH9mVB/Jc4LrohU0eJzPZ0MbgvPeI/5NvEOG4hOzDLvpGk5JJyBmHezV7BQVVbhkvMoHTQd2lhsu6wnjHDcSApqsxRTO05T/iK5R3TqcJuZv6u/Te2HdNGXLPKMvlMf2p8Wz/ZbWRzEVNRGyxsSGuAQv+zCOQBsuL2H/hL6y4n4S4clrpMQkwGeSSXR4YQWFcbH9muGV7XzVVdLSBziQHwuswdl9bT3fRnH5/KX7To/bTt2/H/WmcJr3xcX8qeC0/2eU8LeWcde5nVvKU7fs+wT+srZ3fKwXsOL/ZJidPO1mAyNxGMtu57fJlPbzLlMQwGtwqqkosQpnRzRmzmnoV79XUat/nryt+h6TuXSdfET0+yMvfXv+3q46L7P+FI3cySGaV370ikPCHCjT5MO/wDyK6AxQMOUhDkgv5Wr0PcyBw3gGwwmP9VFNwjhMjs8FI2HvYXut3ygelNz5BsULc67hCBvohB/2QgHDEEfroo/0XS8+SyEF7vWtWOXkwOK/wCxCF+Ax9J42/RdQYIz8KfkRD4Etact90iM/t2n5NCNuEufv5vouotAd44v0RAUjfyA+ycinMfdAA1jd/soDh0HVs2nsuqM1PGPNOwD5qtNX4a3eeMn5pZTl3QUg9ccyikZRX8oePmujlqqJ7S1scZv7LKrqWJ5zxQAHuFxzwsYVYYmRkMIIWJVyhw8q2qylc++RhBXNVjp6aUsmYW9j0K5eFLNoZn6ZmvUImfYlgv7KnNI/P5LDvZV/EyteLOsvRGqV5NeOthJs9haVPzYsucWcFjCpefWVJHO4G+YJ4UjajlYReN2qnZUFu5WXSysluDoWi9wps379j2K5TqltdkqAfUVVdOA82cohHITrKEE8RY3PnCx4UudrTXMe0PYRdAZZAbSEWVGJzy4ZT/FHJI4jK9wK34craCNhLRm2UUscZNmEj6q/lp2M884AUefCi3KBIT3XpYUJRDGPNIAUwgppGh/MJPsrvOw4HyYeJD+ZxRtqILfhwRMHyWobUM0Ef7RrsvsjlmpJYuXTUri7uRqpZJHzXjy5W97KNh5JtGbnutMqwoq2U+SmkViDBcUlcGsja3+0VM2onb/AFrv1SNTK4avRaT0+ATtlBqauBo+asPw9kMrZm1cL2i97hZQk6mPM7vdRTTvk0f6eyyys1VLQTSmSfFhr8LdghjnwGjdYOfUEdwsyYML7sFiVIKGURc+ePlt+atC5W19PVx5IYBEPYWuqsLeW8PdsoN/K1SxDmEQd0ZaNO1jbTh+2wKY5PEte/OXP3uE7Ypo5BCy1gO6mhhkrKtjGNAybrEtYrbMKp56tscjiGHoFWqsPoIJSxlA6Vv52u0WhV00wvMy1xtYqtHLJcCSbKfksulIfA4flD4oJGOP7l0bKSJrtbu+YstJ8z2Rl73scG7qu0sqGZxHY9St2xMK7oQPRDHfvZRg8sF7PK8bkBaFMAH2dSTy23AFrJShhktHSyNH7yzbdIqepky5y7zHdTB7Kohs8cZA6EaFSsomW/Ea76KKSKCJ+VkU36Jya4q8wEzxD4OGGx3ARNiaPICC5u9ldhF2WfGbdLpy6OHyeFhB9isWlKsFFDLIWP8AK3qVFV0LY2/0aojydFadl8SL+g+pTiKgJ8s7gOwjv/itWkwyGYMydwElOZC7d17JHBqIOLGRyAD8q1zNhzfIyd5HZzbJx4bdrnD5Ja0xRhQjY9jc8TXixc8aBWIcDZCzmeIjkI36E/RaDgHEM3b1T8uKJpdcD2vqlnFTkpKRlMyaqp5yb6ZBuoOfC+XO2mmaOjS1bUQZlzyfRMXsefIYx/AqFMzJUy6soZ8vfKihpqd7byQB1+61nDoTMfkVAaKQvvDM1g/K7dClJuF0sbw7kh37h2KtGOPph0LR7Kfw097vMR+Sk8KWxCTmNOboFqZJhVZS0krgw0YLj0anlwahALn0tj26qw1k8LxIxxaR1BSneJZBJm827r9SsWlKRoqGJoeylMjx+YlLwVJN5xSNzdGlXJIiXB5LQ0Dok+lEsTX84guOtluykHhjcsNFEB3ACflQRsDYqVt2neyvPhY1rWwx2dsS6S5PzULnvizZD01tqo0GhbeTN3Kv4oI5Io4C4XHQbqvhw8zQgxSUiqfkB0sLhdJcI/iQMpKUOHMg+V91HNT4eCeUJC4bhDNM8ANexxL9uqnYabLk9UtrnRcnYJfdoYyNo01NtU8tHTkF4nuW7N2ScYIpOVIS5rwCHt0smHhTIAeby3nzEDWynNU5jY19PMJCW8vVo0uim87szSWjoApJqaSpfGzCKSqlaBY3j2Kss4Vx10jc8PLb1L3hatJhmNa++qke1oHkGq2YuEKvMXT4tDB3Aj5isxcJ4dmzz4tNKf3IgApaU5jM9nspY5WA5nS39huuybg2Bt2w4O/taqeOmpIWZIKKnjA6ZArbThuTNNI0MpZyXmwHLKvRYRi00hjgoCW/mfouy5kn5o/4JZyRq8fqpY5WLhHEJpGmqkp6do6g5iVbHBGGZw6SvdYdGN3/AIreBs7NcfqkXAncfqtWMocH4E34Zn/NytQYFgVPa2FQPt1eLlXOY3/VlEJWjeMn9FA8LoIQGQRwxAbZW2U3Pk/1pUJqKUj0EIC+lcMvn/RQTauN3yk/VPl/7z9Sq+WhHW3zTWov9flHW3VBZzBC55UWXDm/s6o39yj5tKP6+P8AVA5k/MhEzBuUxniudGkd8wUfPYT+zJ+WqCyKuI+XNqiE7O1/qqUszGNvyC4DoGqpJiNILCdkkN9sw3Qawff0puY7o26yTiuE2/08fJupVeTiWigIEUc0pPRwsg2vEvB8u6IVFa/Z65ufiqrELnw4XEzs4yAn9LLM/nHjNZHeWQ0jR1BACyO1lmqom556mONv75CryY1Qxft6+GX2jFyuJkEc7eZznykfEZCbqSWaomiELnw3HtssDo6riqlhu6koppbfm8oKpHijEaixYKWMHaLqs+MXiYDo4CxLdLpmsZyiwgZr3a4nzgrUA6irqpswqqqQB+mVrtPqqvLYP2zCQe5sVM9rHWYQRc2uQnin5bXPmAlHpB7hURuMZAec3k18wSDyYvNGLDr1ROfmAGU2LtBuEzXQMjlEj8zmi2g0CstHEz7WY0ABOJ3gF77P0sB0CdkTDE4vJB7BNGIG3zvt9NFniFFUvi0fG/Xr0Uxra6DyiPkMPtuFM2fDSHRPjZI7S1ypKWWectjkoo3RNN81wB9UopSM/wDrC4E7EiynZNVPiBfQumZYmNze47q9WVc8EuXEeUInnytacxsoG414SJpo4SGx+nMba9yFUtWlf5s8mGSC/wCe4t+iMOw13qbPGOhYDr/BXqbivEKqZsMGHxTzONmBjsziV23DXDn2j8VSvpqfhYUjQ3NzaphYw+2y57NuGqLzmnm6jrtHSYzlvzjGI+LzSTkskIbO4j3KiknhZ/WN+pX0fgf2ByTQPl40kw/n5vw/DMzWHuvQcK4E4IwXD4aFnD1BNyRZss1O17z8yvnbu8aNc1h+Z+W6z226Dpp46YnOfl5R93yNgfC2N8TPkjwPAauvfELv5bdLLteHf5PfFGOVUkeN0/3Q0NzNdJZ9/Y2X0xDFh9AXDC6OCmDhb8GMMBSMtU45c5y/NfP2972ZXGvGnwOr9uuozuOm1xjHumfOY/4/ZwHD/wDJ/wDs3wKmppqummq62CxfIZnZHuB/L2XesiwpgDabCIA21gRA0f4KQU+cee/6qUcinaGOeR7DUlfK2dRt3TeeUy/I9V3Lq+sy5b9kz+vpfwQxNdEOXFEyJv5WtACLlNZIKt9NFK9nUhZ1fxHS0zQIW5i82DnaEfRc/i9bHiD/AMOSex1NpC3X5L0aeg3dRMeVR8WdXTbd83Xl8XTYtxXSUMLpJKd7nfkG5WG/jd8pHLpJom9ja6xJo2xAS1czY427OlfYLErOK8OpnBlBD4t+vmH7MfXqvs6e0acca2ecvp6+26scaz85b2JVsmI1hqYInRZmgEl1ysavx6hw55jklM84+Bg/x6Ll67FcWxKYOqpgIhtCwZWAKERRj4V9LXrx1YxjjHlD2YY468YxxaFZxDiVcwxgsp2HpGNf1VAM+vud04awdEbW30uu7pZsruzkWUo+n0SyhER2cPUlp7o3boButB7a7pWPdO426JON+iJZJXzJAByRAKNUYJ9CEnJKSwSbMUr+xSylAyXRLp9U/wDcq0Td0xPmISTbG6AmWJT5rILDrf2siBFup+aoLMUSjyhE3dQEkmzBLMFaLOlb3SSUCSSSRks2UJX6pW72KSNEiB0TWF9krD4UD3PulmCQACWnsgFOSETBS8smSd7X9Ghu/wBUAzEZigK7ghdc6FgP0RpKT5CEx22j/guk4f4+4r4cohSYLVRwRxOLg4N1ud1gDZMQ2923Xk6ro+n67DwupwjPH1qYuLbw2Z6p5YTUvojgX+UXh9bysM4pjMD2x3krnua3M73AXsNLV4VxDhjaqkqIayjqG6PY7MHDuF8KOYtvhzjXiPharjq8KxKZhiaWtjLyWW+S/nvd/wDD3Vsmd3bMuGXrxn0/SfWPP9IfX6bu2Ufl3RcPpHir7DeDOIKWSaj51LiTcxgnEhLWu/eb8S8MxvhHjjgWUPlw6oZTOc5gqAAGOLRu4dF6pwN/KFw3F3U2HcTwNpq2Q2NUxtogff8AKvWZX0WIwRuqGxVER8zHFoe35r83Hd+6dg3eB3jCc4n0mZ8/h+WffH1/5Xqe1dF3THnr8pj4f8vlLD+M4pmgVcTgXC4c3quhglp6yPmM1C67jv7BqTHcYjxjAeTG7NmkiL8jXm+wHZea8YYFxBwVK8u4arWRRHK6qp/NET7BfqOl6zpO4Y4z0uyMspi+PpMPx/V9j6rpPOrj5Orw/G8Twl+WmqC6L8knmH8V1VHxdgmJjk4lStikG4e0Pj/ivHsN43jnjDquIOaR+0Z/kujpq+iqgOTUsN9tV3qcZfJjKcJ+ba4h+wjgDiCSeuio3w1M2ZwlgkszMRpp2XiXFn2D8ZcNlkuGsGMQuuS+m3b8wV7JRYtX4ZIHUlQ4N6sOoXV4ZxfQ1QbHiLDBKfiA8pXs0dx6jR77j5v0/bfa3ufQT/Hzx+GXn9vfH6PjGtpKrC6g0WKU8lNUNAJjl8rgCgHmFw4/Qr7B4x+zbhP7Qoo5cSia5wN2zwnK92mxK8Xxz+TdxThzJ6jBa+CqijzObCTaTL0HzX2em7np24/nnjL+j9o9sui6zCI6jLhn74n0v5S8oY0lpc17T7dUNj6Q85uzlYr6HEMMlZBjOGS0ctvLmZldZQXzNz5+aL213X04mJi4fr9e3HZHLGbgiZma7H9QmEjz5tD7hOHhrhy3lruztkjI4HLo0/u7KNnY4Pu1paT7ixQ2ePWWN7X1Ca4kb+IWyA7dwjYbG3ODb/m2VkNlcfNlBA6tNwkHFuzQfkVK6HS+37zNlG5smXIHsk+e4UCDtdWD5O0RF9nD8P8AxUYLx5cwYf3homOcHqD3bstCUtvezL/IoBqCGmMH942KY5pADcOPvoU48vmkcB/aFwUZBqOv0spLPcAWM/QqMnXTT+zsnGZ5zfh37HQhBJ5yMwI/XVEMtjmG4tYhQ3eHec2b8rqRpsczDcdmnRMRylOctQOwl/xXWyMseYw2cfiabLlpmuirJWno6/8AFdMyS0bNvSF2yebDCLmJeo4J/KX+1rAKGnwygxam5FKwRxMlpw4WG2t1t4h/KcxXinhapwDjrCY6qaWoikiqKaNsYZG31N11uV4nfVP01Xw59n+2eJG3HTjGUTdxFTcTfrDE9Fpn/ph9VcJw/wAl3iTAaWesxN+CVbxrBNVkPafc2VHjT7GKeopIse+yrEpMfw6V/J5bPxHhw3Jd2uvmO1x5Vt8M8a8UcH1lPV4HjNRA2nk5ggZKeW4+4Xz/APKOu6TbPUdJ1WWU+f5M5vGvh8fpNvj7/Z3pd0zlMefr/wDJjzerz/ZT9pVLEyWXg+uOYXIZlcR9L3XKTOdSVs+H1jHwVNM7JLC8Wew9iukP8rn7T300lLO2ge2RhYTy8rxcWuHLz88bRYvVF2LQyulqHjmykh4JPUuX1O17+67JynuOGGMR6cZmZ/d8WfZKcomYyqfd729uluV7PwH9kf2Q41hFHQ1PGEDsXnN+XTVQ66gBvdcsPsqq+IcTxCLgc1E1Lh4c1z6llg97TYtaddVx6f2n6Dftz1xMxGHrOUVHrUec/GfR8HqezdTouYi4j/5bgspSy+63avgHjqg89XwtXNZ+YRgj+CxKiOelldBUwujkZo5rhYhfb19Vp3TWvOJn5TEvkzryx9YAkmBulYFd0Pe9h1STXG6FGSRoEkBF2uybbqiQu3Vtqj5gkDdCkN1AVwnUZal5vZGR3HdPp3UWvulcoJ9PZM5Q5ndkZce10IFc90gTfdBmO3VE2RGhOGt8qZPzdMp2TXG/RGTO2StdP5bJs6IWR3yPsrDMQxKMZYsQqmDsJCq+YJZgkxE+qz5+q6Maxdos6tfIP3rFWaTiaspieZRQTBxu/M1ZN3dv4p9Pdc89WGyOOUXDOWGOcVlDuoftKpqemdFT4ZVQPtoeaXNv+imoPtOhEh8cS9ltAWWN1wHl7pZQF5Y7b00Yzjx9XmjotEYzHF6d/PLhuq8/hoS7sJA0qDFOHOCeK2Q1dZK+GRgNmxz2cPnYarzPJH+QIuWAczCWn2K5f5Vqxm9UzjPxiV19NHT5xs6fKcMo98T5u+i+zTgV7xDFXTvc42AMp1/grP8A0R8NdIBJ85SvPYquvgcHxYhOxw2Ictii454moI+UyqE1ze8ozFct3Q9VHnq2zP1l6NvU9zj/AMvqMp+sy6o/ZPwy39pg0p/sVhKrYp9lHDUlDJHhtHU01SfQ+R5e0fMLNi+03iAeSakp5Gnpstem+02mETfGUs7JOuTZePPR3HVWXKZ/W3n/AB/etMxnG7Ka+d/s5Z/2N1Z3x6lb84nrDxj7N8cw2qbBRNbiDS3NzIdh7ar0uf7RcOmYGw1ssX9uO6yMQ4kkxDWGuonk9cpYV1w6juWM3lhf6Pp6PajvWvPlsxjKPhUR/R58eAeLhvgk/wDD/Na1F9l1bU0jaiuxamoZH3vDK05m/ouhp5MbqphFE9rmEdJ2gFFz56ebk1Eb3O6W1U39y6zDynDjP0XqvbHuc48ccIwn41f9bcjL9kbLn/8ASSL6ROQj7F6edoP88Kdt+hieuzFdCD5w5nzCsRT08w/aNP1Xj/zbqvj+0Pn/APjDu388f9sf2eZ439i1fhr4mUdczEhICSYLtDfndZMn2V8Twu8nDNXO3oWuaT+i9l5UTttfknaZo/2cszfkV3x71uiKyiJe7V7c9dhhGOeGOU/Hzj+nk8GxXh3H8CY2Wv4VxCnjJyhzm2BPZYE2OvgeWPwmqHuQvpmaeqmYI55zM0G4EgzAFQvcHxPidR0bmvaW3NOCRcdF2198ivz4efyfT0e3uPH/AF9Pn8p8v3fLtTxjyHW8AT9FhYvxFFiLS1+HiP3K+mn/AGfcLzMcavDY53WNtLarjZ/sgoqgvbLgNNucpZL06L36+59Ns/6q+r7/AE/tX2vqL/1ONfzRT5uleHuJaouRJfNl0X0P/wBAWAVRdneaR39olUca/k+4dQYeauHHHyWcG5GXLteq9ePX9PlMRGcPVr792zblGGG7G5eDiInYKxFSv6tXpH/Q1iLj/Qa58h7OjAVKu+zTibDZHRvMD3AXLbgGy9OGzHLyiX0cN+rZNYZRM/KYcbHSSRvD8vlG6l6eVbMvC2ORPyPw51u4N1A/DJKWQtqqWSMd7JL0M3mSAqCokMnrKv1RpWRHltOb3WXPJGXaSBSIZCyYRm5Fx1TyVcThYQut3uoi2/xD9Uwib1eAtVArtcC4ZSpw17vSFbdhQhkINRnkHYKu5k8UuR2icoEWSbNY6Ig50ZHkujla+MjLmPe4Vz/R4RJyiSR2UnKBTzTveLwkD30TySPz5uX+ik8RNqX0xA72UTKkvdYxu+jSsxLQc8zj+zsgLNczleibIQXGnccw6hQ+EqXn9hIPmFq2EEZ18iJ1HLK0NBDL91oU+HSRRF8zN0fLPwRud8mlTkM2LDYmOzuk5hHUaLS8NJUUTmRUvOtvp6QpBFIIHllFMXPFrBpVvCJK+mpOVJSNzEk+lZt0c9S0bpZQ2OF5+eikZhdRFWjM3ddFDh9QZ872ujzdlcGHQZg+Vpc4dSVmxzVXh1VTETzMs1DS00xqGNt6zZdDVUVLKCJHyuB6BRUeHNjA5YPlWrSjyUVU0mOHmSHtuFn4rAaekMzmZZAQCtUOrJH5GVYj+YTTYe6pblrfxvclZt0pXwuOGpwiN00Rde97KCpqxS/gQxFrR3Vs0kbfI2sfB+5GVBLhkM4tnnf7k3KSzMJqWukqjy2TtAtrdWGyx2yPmDrdlU+56SJmZnNzdibIoaMwZZpI7NPvdIUjVsjeGRh2YndwTEyOuX1AcSeivysY5jc0MNiNC8o4hCyERs5Ad1y9VRQjjYTaQ5m9kXIp3zkRQH6arVbhFE6+eq8wbnIZusyOOqLQIY5Mv5nHRBYjYYmASQ3HWzblSB8LfNHDlPu2yqNNdI8MZVW730AQRV/KqRDJM17hu4tVEhbGA4Oj67kKMhnZXKrkzROZnFztZVI4uUBnuWjqeqgsNlglJ5MPLIG5UMkTZAXysBPcaFTuoy1pmfGSCNMpSJZCBmY+3dBBFB5bZBE5h69VbIBLCRENeicRYUXXragt0uGgdUIie88yP0IJ6eqdBM4uja5lrZioJDhz3kkyX7N2QCeMOIcfK3dQ8yOSX8MWzdUEpkjADY4ZNDu4pMlfI7lsHKH5iUm+F5dnzyEqu+ojiItmk+SKvuotMxrmi/dRihY5pE1QD2LNUqceKF2UFQXdhGSrwwbGKoMbR0Tob+oyt5YH1RFRsNPA20lVUZOtgk80j9GRymPpbRbFLwVWn8bEMTiFujSTZXY+GsIDQyesllPXzWug5KSJglyMMjY/zFyU8VNC0FlaZHONrBdwMIwT4aGB39oXJWbxp4aKGipYIY4xmzvDW27LbDHoBZ4AbqdAO5VyThXH+c6TJEM5vYu1Ciw6HnVULG6XkC7N1PMf+txrWbnHq5T+aVWP9Kr44z2bqpqXhWgju+armkf0cwAWXQGlmP8AXRu+qcU0jfNYLLoz6TCMKotYKXmOHWbVXL29EELPlGpuRIfhS8NJ+RYVFzJCLOfYdhogy9y8qQwyW8wQ5JrFoY76IoeXrqz9UbWjshz5T52uv8kJk12cPoiJjta6QDFW8QwHa5+SXiAT1A7BBaFj8SceQ+pUm1Nj5hr2SMxPma0j2KCwXoc9lT5z+yjfVlps4FGmjzvn9UxleR5ln+KmJtkP0SArZXWYw29yjK4ZnD1CP9UDqixt5folFhWIu89xbrdCaShpjzMSr4mX6ZgiBdNI/wBOqAF7v6on5IX8RcM0kobRk1EvRrW2P8VFVcX1sTQynoGU19jLqja9HhlXK4DkMF+pcpnYLBF/pmIRxDqL2XLz4xxdXSGGkr4I77iIAFo7qjV1FTJVeFq30kjmD8VznZuYUHWOxbhGhIhM/Pf2jJddKTi+hjdy6HCzboH+U/3LnGiCO80eHtLma+QW07oGzyTTNe54Eg3sb2Qa1ZxHjtYCI42tZ+VossmRlVWzf0977DVt9bFSlr4pWyskOa9rblBI6ojnMj5HwW2c+Kwughgow2Uv58rR0aNP/dTmQvaxjxG6QWZ6d0zalkjw5snMtuLdUqqSfM9jOUbi1wNQstCmiijhMnZSclk8QLW5yRclVyWx1WZ4MkMYuAepKkiqaolzIYyxsgsT0sgAQchpfktkJGXqfohFQzPZtK8X7NVh0PVzwHdSVfgpKiYNqS9rWMFs5OoQmGU2X8Qv5cmgvayOSrh/NOrLad82ebkOkdewHR/zVh0ZicyQwtgLRYGQEgolM+GGesqIoKISScy3mdoArNRh0MVW6hjq55ZmC72iIAab6rRggraiXOyugzACQCPc9lDLgdU5xkvLHKHZwTu1EZtVTzU03h46GaSXJn/aAG3ySjw3FZGh5w6Vo7OIH+Kusgc175KmQZ9W5jJq0eymw4Bz2xjNNfUykkC31Rpjkvj9bCOmqlw7EcOdNNS1LJDKNo3Nve3v0XQ12I4XC8mSmNQYthp6uu6s8OtxfiGsbQ8M8EzGpcCeY5haAB+8mWUYxeUsbNuGnGc9kxER8VXA6WnfdxwuOJwFw5wzX+q0KylgDHQcgOc7RuVttF6dg/2D8UV9AyXHeIjhEmueGBge4fJwXV8IfYfwhwxXjEayomxWZtjG6qv5XdwNl8rb3Xp8ImpuYfm+q9ru2aMcuGfLKPdET5z9fR4DgnAHFXFVc2gwjB5J5GsJ5tQ4xsBG13EfNesYF/JnbPTwVXFmPSNrABzYKZoLf7JcvaX4jTxuMcJYQNsjVX8XVSHTRfJ39337f4Pyw/D9x9s+v6ua0f6cfLzn7z/Zl4LwBwJww6GfDeGqOGpgtlndHd4PfVbE2MeYsj/Et2UVSX5Q+R5IvY3UTo4ImGaRwawbucbAL5mWeWc3lNvyu3ds3zy25TM/ObM+aqmcXWYL9kxhkefxFUn4iweimZAc04c3NmhGYBVRxlBOCKWncwj0ulbr+i3r07Nv8GMyYas9n8MW2RCyNud5AHcmyoYhxHheGEBxfK4mzRG2/wDFclW4rilbUmaSpkLRoGF3lCgnNXUuzzakC1yvraOzZzMTtny+Xq+jq7ZlMxOyfJu1nFks1OeRI5hJ0aG20/tLFq8Zqae9VE6Ykm2Vrr2usqtxrC8OAa+fmv8AyRi5XO4jj1ZiTSyDmUjO7TZ6+t0/QaNH8MX9fN9DV0mrT6R93QV/FeC0uJtoJKxrpCMz3D4XdlSreMHhxZhtK7TaV+n8Fy8dJDHd7I/OdS8m5J91KM6971iqJayvfzq6qkmde9jt+iKNjAMtgAEzd0YapEIe7U1gVJltoWp7C3mVpLAMqk6jKLBIAdEtPZEDp7JIUloO7ZOkkgSSSXX6IGbsn6JJKhrHulbTdOkpZYUSFOBdAySSSBJJJiLoBLwPUiBuoyNUbSAe6okDvzJddNkOl/ZODZQOnBH1Q5ilmKAzoPMn+SG+uXonJsgdJDd1051KBzskzomB6JX1sgK2tkySWx1QLY3SG6a+p+ae47oH3umPZPca5UJNkSDpJJIpzfqn6WG6YbFO3ZQLU3CDlm6N290Tt0Ec1PyC38RrswzeU7LtuA/tVx/hLEGOmrJKqiIDJIZjma1g6hq4kjoU2ULxdd2/pu46p09ThGUT94+k+501bc9OXLCafXnBX2lcMcbx3wuc01S12V1PUHIb/u+y6+qoaWugdSV0DJo3etjhoV8P4biVdg9R4zDqh8Eti3M3ey9a4H/lDYjhzabDOKIBWUrW2lneSZL+y/kfe/YLq+h2Tv7ZPLGPOIv80f3+Vefyfe6XumvZHDd5T+zuePf5P/D3EcdTXYFI/Dax7fw4oWtEWf5LxLG+DuMfs6ro4scwuSoYwEmejYZIrdydm7r6p4X4vwHjPDW4lw9X85pveKTyvb9FqzQwVUMlLVQNmilGV7HN8pC+R0ftT1vQT+G67HnGPl5+WUe71/4n7sdb2TpOux5Y+U/GHyJhvGNPP5KgEvb8TdV0dNW0tXAHwzNeD2Xo3Fv8nrhfGpDU8OgYRLlLjHDYNll6Xv6ey8Z4k4Y4s+zkwvxqgLXTmzZIDzGkd7r9j0fdOk7lUdNneU/9M+WX29/6W/Gdf2Pquhm65Y/GHWwYxW4K3xFNVObGN2nULqsF42wyvY3xrOS8/H0K8ohx5s1E+LEoeU5+Uh52HzWpg1bRzULMtQHPBI23Xryx90vj3T0Li3gzhf7QcNFNXwwvLSMtQxl5GDsCvIeLf5NFTCWT8G13OuCZGVLg23yK7GlrKqklEtFOY3duh+a2qT7QqmlfycVw0SM6yR6W/VerR1u/p6jCfL4Pt9v9o+v7ZEY6M/yx7p84fKVbgeP4LJJFimETxtYSHOfG6wt72Wc18T3Zg9o+a+3n1WAcS4ZyZeRNT1TXNdFM3UheS8Qfya8MqjV1uBYy+lebuhpwwGMdhcr7Wju2vZ5bY4/u/ova/bzpepvHrY8Ofd6zE/t5PAAyntctAf3CAtIPlyuXX439kf2gcP0jqvE8JMsTSAXRPz7+wXK1FHPRWFTS1NM87ZmEL6eOeGyLwm37bput6fq8eWnOMo+U2ivy/Kyblnt0TluY55Iw795pQF0m72hze/VMHR5SWVHLN/SSuj0cksbpgM0czbfkeUQDvVI58fYt2UAkp/6+MvHcFCZmFtqWraQzdrioq5mk2AZMO4OqiMkwJyfh+z9bKLxdCwHnz8t3cOUUuLUuzaoTj5aqiy0bvkZ83MTaP0awSnsVUGOUA0/Fi/tMul980d/9Z/ZFkF6xZpZ0bexOiYQud5gyN47hUvvyhGnJnH0unGJwn00M7vkEGfiTf6e5g7D9Vsxk8qM8si7BoNVhYgaqorDJDA5thqD1W1QStnpw98clO4eQtvoT3WnHH+KUlnuBFmn5HVOGNbuXfVP+G/ctcR7WKWW3la9wH7yOiRu3kB+YSzE+ox299Cowxh1zC/dp1S8p9c5P9sLLUwPy/E9wHUDUJrM6MDr+9ihyjcNNj1aUrOufxB8iLFaZW6OtrsKq4q3DpX0tRCc0crTYtPcOXb/Z99tvG/2dzzvw6tfWQVBc51PNIS3OTcu06rgQCG7uA9jcJgRuLEe68fVdD03Xa519RhGUT63/APLc89OOcVlD7AwD+WJwlHh9PJxBBWNq8g5wjpSWtd2B6hd5g2N/Yv8AafSxYwaHDXy17S4eJaBOfmL3uvgQuf6XXVvBsWxHAq+PE8LreRUxXySNdqLr8Z1XsH02OM59t25a8/r5fTyqav6vndR2jRsio/d9q1v8lzg6tJmoMfr6UPJOXyuaFwfFv8m/irBS+owCoixKkjjzOc94ZJfsAvG8F+3T7R8KxOnrpeJa2pp4n5n075LtkHYr0ym/ln8TU9my8LUlSB+aZ11546b2w7fuxnDZG7GvS4r9+M2+Bu9mscorHH7S4qpwHHqQllXgWIw+76Z4H9yzn/hvMbwWuG7XAghfS3Dv8qn7OsWpKSHHmSUlXU5WvhbEZGMeTa110XEH2dfZRx2yeVkVHBW1ABNRG/lvH+yvVj7c7+k3xq7p0uWvGff5z7/pFx9LfB6rsOeiPLLz+fk+R7XCZo6L6BqP5KsAF8M4wc5n5JIx/fdea8X/AGP8dcHxyVlbhDpqNsmRssDuYXdjYL9J0PtT2ruOzwdG2OXwm4v6XVvk7Oh36seWWPk4hGndDOzMZYJWFu+dpBCA5V+gjKMvSXnLMUySSqH190yfMUyBJJnbJhugTt0ja/lTtTq0wBJFmTZfdKDX1snuQMqWUplFFm18ybMUrdSmQFmCWYIMwSzBEHmCfP7qPME6CXOlnb2QX8qWwQodwkCgGycCypY02qC7kWYqLZZk53Caw7plYgiT2B8pFwhEUf5AnzXRXb9VAwGX0vcPkVNBiGI0sgfTVcjSNupCgSSYifKU9fJoycQ4y8Wkmhk93RBQMxSYPDpaeF5Bv5dLqs42Q5VynTq/lj7J4ev+WG1LxG0+jDeSP3XkpocfYZmiSpMcd/MS0k2WInyleee29NMVwcZ6HRMfwuoOJUx/Y18Tv4JfeEugZDzcxsMtlyuRv5QllHw3H1Xn/wAn6f5/dx/yzR83ZvdURX5kDtN7EKNlfBcNMmUnpouSL6j/ALZKB81LT11XTvbJHPmc03Bc29lwnskV5Z/s809qivLJ1hnppNzr7hMOS7YNWA/Hal+s0DHHuNFA7Eczw8snZY3s1+683+S7v5o/dw/yvb8YdI+GA+i4+qy6zhfB6+oNXU0kc8xABe83Ngg++qZw0ZKz2L7oGYzIZGtHLyE6nawXHHt3V6/zYxX6sYdL1eieWu4n5T/YbOGcOZ6KGL9EbcAw5nrw2ld841Z+8YmbSZ/kQoJschZ64Zvoy65cOs+GX7unjdw/mz+8o/5tcHyf6VwXh03e0dlhYx9lf2Z4zXGrlwSegYWhuSnAy6dV0sOIU08QmYXta7bM2ylZUUMps6dpPzXPDqeo05eWUxKae6dw6XPnr25RPp63/VxR/k//AGT1INsQxGC/7qo4p/Jl4GfRPdgvFNS+qFsrZyGNP1XprI6cf+6PJTfl/iumPc+qib5vbh7Ud1wyjKd0zXumIr+j5MhpcOgZZksV/wAx3Rsip81xyyO9lmzOjFxywEoWczZxA9l+vp/b6atqT4+X+icCkvcZT9FnyRyxNz8skDorNJBJIwvlAjHbZRpZeaJ/mMMZPyQSTUsQuYI7ezVXLKVl/E1R/shQc2OoBEYeGg9QirYr6IjWJv0amNbR9IL/AEVMwEANYw3907G8s/iRj9Fq0W3YlRs9cCc45CdGQWUbBS5Q98LQOqdsVA2XmRwF/wA1kmBNxnN6ICfkEf3nM/0Up+oTOxGR9mQtDW9g1MaOSdpnDx8kUfiK5/8A1Rw+iaWeaGwns0kaDdCyCcjz1El+11Wmhm5vo5jhtcoJxWSHyhjD72ULn1D75GuBPZTiJ/LbYDMegTRxSR3M8JB90RA6B9rvms/5oKeJ87+XPVSBqvMpxVAnmRx/2ioTTubKY4auP3RYyLwNK0XhqHF3uE8OHVcj8/3i6IDpfdCIZA8PZPJf5omw1r9Hykt7AoT+ZLK0B2SeYOHsoqeJjXZ3vOXudUvD8r9pERfrdXpcPfFQCTmFg6AKwiqYmftOZmb8tkF42vysb+gUVqprQyRhcCbbKYOqIXg2Fh0KqUvUxIppHseGvfpf2UA0bkELs3uU0s14c5LQ6+oCGOujA8pMRIt5jdZbAIpIxnkjF+yOKeQt8kEYI7tuhs+V5bC8yuP5Qp2YbjMxyU9HIb9SLKwzMIvxXfjSDRHG17/O9l4h1V+l4V4gltHUcqKAfmdqPotGLhCkj0qsXcf3WgAJaU50uZIPNVNGXsVCJmNP4jw75G67KPBOHKdoHh3Se7nFTRR4NT+alw6MO7nVVXDyNqK2Rvh6WaRo6iMrYg4VxWZmeOlmj/tGwXTDEKi2WNkcbezQmMtTL653fqgwY+DKp3+lz00I6kG5VyLg7AWZfE4jJKGfC2wBWllJ9SblR9WIAbh3DsJywYQ1zm9SrEc8EQywUEMXyjCHlt+GO/1RC8Yy5GAoyc1tafjt8tExmqD65pP1Rcxtg1OMiCER3F3En5lLKLbBS3NykX2CgjDSDouZ4rqH1WKwwOFmwMt+uq6iP8R4bdcdikvisbme03GbQreBLRwRgFZC648tzqukfKXHNmWDg0TPEPnftG3y/MrXMmiufqxCYSEblMXOOucqAzOHQIPEfDlA+Sw2s8149DyEhUT/AOtcFTM5+FyjNQ/uiNHxUx/rP1TfeE7RpPHbsQs5j3SGzTr7pnv6ZwSe2qDS+85dxyyl97yndsSzOVUu/Zxl3ySFHVO9bLf3o00TioG8LT8kxxWnZvEw/JVWYM0nNI+Qf2pFBUTYPQeWSbmOHRlnFEaAxOmk3w9x+idjoKh2UULmLCbxS+JzhTYW6RvwlzrW+iCbinG5YfJyqcdg25/VGnQS4W6X0MLPmVnVYwfDmh1RijHSHdrTmIXN1NXiWIxmN1RO49Xc02AUNFQ1EFQ7kU0MrCLF8vRBuycV4ZTtyU1HJO8bEbIRj2LS2ka2OnYRfQXI+qCNlJSwcpmHwvkuSX9B8vZUhBKHl/ipGOOwB0CxyTikqq/EHy/jYjUFvZrrBZ76Jhk5zJn5ugd5grkhsWvMhlB+M9Vbgmljjkp4aWFzZxZxLdfonNtkeHfHKGQSZnkXLrKSVwlew1MzpDGNAehU8cWWXPH5W3AspDI0EhjQcvcKxKUr86CVlpHiN7/VYbhTSQYcAMkHOcfhyapRSuY6zHZXJzJkl5wBLhtlFyryUHMDnefQPGgGllIBTwtbCGeZhubKM1N6rPVQTxtG73N2Vrl0YaX8mZtxcOcdSokoHTyCUTMNgNipZq99Uy0j8zu/uhe+E0roy0F3V1tQVA2SmbvTNdfunJE4pBmdJlaGgauDt08MEOXMxpsTYucd1TLIoySyFzARpY3sp485jEbpiCdswsnJbHd0ejC35EXR1EhDIm8+Vz2t81ugUJieJW2/EPUN1KOKJjA9noLzch+hCIJ7yyJoiBlcdS7cgJMZnOXMb/vOsAULpo43lkEQaL2Lgd1MylkmguwXbe9y9aW1uGXl3ia9spZd4tpqrML8Rq4Wnlsk5rPMOg+SgoaNr6NwqmOuw2Dr2JW1gVJWVr5YcDw+fEJIA0mOAZnWWJyjGLmUz2Ya8eec1Hxlmww10d6elpRHfVz2ncqxSR1sLyZ3mTSwu7Zd7Q/ybeN8Ua2tqMVpaDnAPMcoPMaOxC9FwP8Ak98E0FNAcUfVVtayzpZBOWxucD+Xsvn7e6dNrjym/o/NdV7Wdr6aPLPlPwiP70+faOjxrGcUjw3BcONRUS6NETA669J4e+wLi7G6Jk/EuIjDZA4gQNAJB7+U6r32DDsJw9rfCUNNBk2McQbb62SdiH+rdqvk7+87c/LVHH935Dr/AG36rfUdJjwj4+s/2/ZwXCf2EcIYBEX4xAMUqxKJGzTXsLdMq9AM1BRuyU9PTsDf9UwBU5H1ExOeYi/ZM2G3q1Xy92/Zvnlsm35XrO4dT1+fidRnOU//AD0j3LMlfLKcsceh7qB0UkwHMJKnihu4WCrV2MYTh04paura2Yi+QC9vmueOOWc1jFvHGM5TUQkZABYZVYLWQjPK8Mb3JWBV8WU7IHmjAvazTe7j7rmKnGK+tYW1s3NJ+LYr3dP23d1HnVR83q6fodu7z9I+bsMRx/DsrqWGVxk6OIsGrlcZxOvxRvJlxN0Ia70xtaWkLPcGP8xGY9ybqKaaClbzKqaOJvQuNl9nR2jTqnll5z+z6unt2rX55ecrLpX5WxifLYW8jQNEDmsa0vlqZw0bkuACwKnjCnZ5KCDmf947QLDrK2rxGTmVc5f+6NGfovqa8IwjjjFQ92OGOMVjDo63ijDKVxZRB1U8deYci56rxjE8QOWSo5bPyxkgKBjGj4U9m/lC6U0ANZHbRFmb+YfqiylM2IXzOaEpgwa7ujDHKQDoE4dYaboEGEIg1MjS0nyCBc5kSH91PfWyEmdumRoW7qxJYLaXT/7XVMkqokkkKBJJ79ksxQNbS6du6ZJA4OuqXluMqEmyV9Lqh0k3VLKFGTpJfNJGjeW/unGqDrmSzWQP5vhch810Td0jv5UD3JtmT9PqhAsb57+yIh4JGwQSoc1gkNHeaxsmsSL2WTEiQDuN0/X6prN+K31SJ20utB/3uqWYpi7ylNmCArbHulmKZPlKByL6JuuVMmyhA+hCfzWTJ/8AaWvRPUhmG6cXATakpllST3KZJAWp2T3090OYp8wQPYWJSI10ddMNQUtggXT6IuvlQ/7SR30QFpZAQL7fqi9O4TZgbj2UoaOCcUY9w1I+XBcQkpnvFrtOoXt/Bn8o+mncyg4sgcxjIxeqFg8v9+6+fMqbJrqvzvePZjt/eYmd+FZ/zR5T/wC/629nTdZt6f8Ahny+D7owXHMLx2jixHCK2Kqp5PM1zDrb3CPGMIwviCjfh2L0jZqeT1A7/qvlH7MMaxqHEG4bTcUUuGMa9rs84Ljr8I7XsvoqHj7BMOk8DivEtDUTxGzmscGuv7r+Ld39n+s7H1sa9EzlPrjOMTcRHvmo8v0l+j0dXr6rXeXl8bef8c/ycvGR1NVwXingyGXZRyBzw89dV4ziWH8UcDYoabH8MqMPLWBwfIM7Cvs+kxClrYm1FJJmjcNxuCquJ0+C4rTy0ldTQziRpY7MwEgexXs7d7YbtMRq67HxMfj6ZR+vv/Xz+b5fXez/AE3VReEcZ+T5Mwvixs0bZqidjmkXD2i110EVdDiNOHwvzArt+Kf5PXDElBLVcIPqKPEmi8eeUyRuHUZO68ZquFvtI4dM0uK4LVQUrNp2Ma1jfmv2vRdf0fc8Ms+l2ek1WVRl+kX5/o/I9b2Lqej86uPk6uPPSS/gSPjvqcpsCVchxKqicHx1UvyLzZcKOOZKf8PEstxu4ssVsUnEeG1AaWPcA/UEjde2dWeH8UU+LOGWPnlFOrPH7aRvKxahc4fC6N5LSEFazgbjmKKKvc1jm3MYfdpH6Ln5ZqaoIY97dfdVqV2G1RMd7OBsNNUwyy1zyxmpdNPU7unyjPVlOMx74lTqP5M+FYhM+qpuMIoS5xLWtBsBfQKpVfyVazw8jqLi2nmkDSWtMILnHsujhZiNE3nYVipGf1xO8wcrEdbxFK98k2IiF9rgxEgH6XXsjuXVR/1ftD9Hj7Zd4xr/AFP2j+zyGf7APtKgbdtAX+wACqy/Yf8AaBEzmT8KSu7uY8L3CDjDiaJoZVVQlYOpaLopeO8WbGX0tVkeNgRcFdv843fyx+763/8AIXcP9vH9/wC75txThLFMBqxT1uDPjlDQ4sk00VAxwB4ZUUAhf7ar6bi+0LEo3ZsRwuhrmEWPNiF/4BatLxzwVV/6ZwlSxDv4dh/wXbHvXl+bDz+r6Ov/ABFiMI8XTc++p8v6S+UmxnXlwNlYd8zLoHHDmHK6lML+4bovsRtd9l9ZFaTCKRoeCP8ARm/5LnX/AGWfYdUudI8vaXOLj+O5o1XXX3rVl/HjMfu+j0/+IXRZ34+ucfhXn/Z8zNqagjMGRzt+ajdNA5+aR7oHdgdF9CY19iX2VzUoHDuNeEnLrl0s7ngj5LmpvsY4fheWP4woC3rmYSvVh3Pps4ucq+r62j2z7Ruw5Ts4/KYm/wBreRhx/rntmjQmelb5YYzGOxXp9b9iGFtpZK7DOK6ebLqIYyQT8sygw/7BOKMSrIqcvnjjkdlMsjAQ33K9OPVaM8eUZxT6Onv/AG3fhOzDdjUfOv2nzectlkkvnhje0bOabFGx3RlS6Mflc/Res43/ACY+KMIpBU0WMR1zi4N5cMRzD30WB/0FcdfDQOd84imHU6dkcscod9Hfe3dRhz17sa+tf1pw34IN5Ig4dHM3Ttc0+g2/8RdufsO49j/Z4fIPk0oHfYz9oTfVgznfMFb8bV/NH3df816L/dx/7o/u48CQeZ7GD3bsk92fd9/mV1zfsb+0GO5GETtPdpSZ9kf2ij1YE5/zTxtX80fc/wA16L/dx/7o/u48AD0tP0Rh7/SQ3/aXX/8ARF9oZH4eBSMPcG6AfYz9pBP/AMqf/urHjav5o+5/mfRf7uP/AHR/dyNnDzHZL8MnUB3zXYO+xfj5jHSTYfLG1gLibHKB+iKh+yDi6ugZUQUrqiF+oLYi4Fb8XXV8o+5/mPRzHLxMa+sf3cdmAHkuPmbpczXzOB+Qsu8/6EeLBqKKsH+wVM37GOIh+1wyukPfJZXxtX80feGP816L/dx/7o/u4OmqWU9TFUFzzypGvtfsbr2rh/7YsExFro8RcaEsAawuebvXPUf2P4vHLGw8P1Ru4AueLgX6r0DB/sCihroZsVlpJKYG72NBa4/VfN7hHR7sY8bLzj0p+X9ouo7J1WuPxe3ziJrjNz7vh5fd6Twbxnj9M2FrKh8tGPPlePUD7r1nDccosWDYo5GPkIzFhF7LzKGHDqSFkLZGNbE0MFndALLTwHiDBcGrvEVFQ3KW5Sc3p91/NO89l09Try3aca2RcxXv+r+ddr7zt6fdGnPK9czXn7o+TseJ+BeGOMsOkoMYw+KNz/TNE0RvB6L5j+077G8Y+z+QVLKrx9E8F3Na2xjHUOX0TU/aZwmGlkGNUjye7kFDUcOcbxy0L6yGrY0XIabhp97r4fZu8939np58MvCu8omJr7zHlL9p1HTdP1flccvc+NA24uCk4Huvovif+TJR1EfO4VxlsEz5C+QVJLrg9AOi8q4n+x3j7hiojikwiWtY5uYy0zczflov65272t7V3KIjDbGOU+7Lyny/b93wdvb9+n1i4+TiUk9THNR1D6WshdDLGbPjeLEFCHsI3X6XDKM4jLGbiXimK8pOWpy5Pcd0GbzZgq0FON07h5k9m2VAJJEflSS2T31uk7dMklNGuEYJQO2TgWVtkkkiLpKBJnbJrlPf2KBuiSe2iTt0QsxSuOyZJA+YJ73QpvM1FGCeqQOuiYXQuLtwhKTMUsxUeYJZ9bIiXMEgbIL+ZPm1QoWYFLMFHcjZPmKCRPpZRtcEgQijSQ50s7UZoi1DlCO4Tae6KbNdJK35U2UIp0k9tU17m2VE5G/VS8+a1uYbKNODoixKTxMzWlmfyn4XapmTRtkbJ4SFz2m46KLKEsoUnGJ8phZxifVoHFCd6Zzf7BTfeIO1fVQke11RSXm/CaP5I+zl+H1fyx9nzuGsL7uZcdldhpXkXjgZEO5OqihkjkaAL37WRmGSxJzD2JsvU/spTU8+761gb2ChzUvpnM0juw0RNFVG4COOK3vqrRqn5CDy79lkVWwQ2zW/VFy2RjMnyukGjwD2SFK87kn5IEY31DLMY8e9kzKJlO273uLvdSQwzxnzzuDewVWV7GycvPIde6Byad5tPqzqjllaLcj0dNFYghjhfncwH5oqmlcXcyP0nZBRbYuu4G6vROaGBU3GqD7tkBHZWWteW3zgfRAbj5S5qqse2SXMG5irEXNDsskgc0nsk6eJr8lw0d7IgOVUSSDIwi2yklhqLBknTqmL3u80clx7JGoyX5z7gdkDSU0LbERXPU3SbCzMH5Uo3uqXZIaedzvaNXmYDjtTD+BQyMH5njKEVDFTvldYRgDo4uQGF4BZNUtb8itCl4HxcvbPV4k2IDpe60/5n0Dx/TcUMg/KywCDmOWzmsa6cSNBuQXKxWSmve3wkgGQWyM1XSQ4Jw3SPzx0jnOH5iStKKWiaMsFNHG32CMcnEDBcYcPLTTOHQu0U8fCGOTgSPMUTf3nahdp4of+4TGoD/l7LRbnW8IUrW/03E2l/wC42/8Air1LgvDtPb+iiZ3UvV9zad3raSm5NJ/qb/NG7IVVLAMlLQQxj2aAhNdOeoCn8NTO+D+KfwlL+Q/qiKLi997yHX3UfLZ+W60X0lKfiePkoxRwOF2Oksss2psaAfMjaxt7q34SP3+qbwrOhQtDlb3TjJ8Kl8Gek8f1KXhH/BJGT7FUR2b+dBzGZsmcX7J3YfNmzGMk+xTOpqkC4pde/VReSUgDrZI6lVjFVk6xvP0QvE7fKYyPoqidLP7qo50g62+aA1OT1vCou5/zOCF0gta6oGoPtZMJ3W6I00I3DI5/YErjKYcyZ03dxW9W1j4aKd46sLP1WJho8oXXFyyb2HuEcT7/AB6Kd0zR1VOnjncLQix7qwMOrZHXcHSdw0Lnn6tY+gnVTB6X3Ub57uLsyldS09MzPVhsP9tyifXYDFE6SapZMB8Mf+awtBE/7ykjikndlDSfkLqg7izDYLfduBGSXvJsFnVHE2PVFxDOYR+WJtl0R1BoY4xepfE0fvusqkmMYDRZg6pBcP8AVtv/ABXOVFMcji+aVz/zO0QmBkkQhfzCOzVhW2/iyMShlFTf7cxsEq/EeIoHtGcRgi5cwWWaXzZ2F1PGMgspqXEX8zkv5krY/g6KNBdJSTOE+JV1RK7q0dPqlSV0giApHtZIPytu9QutMXiR+5umpInxnnUvq1YbrDQZ3TTyB8j+XID5gOqIsc24YwZR0UhiljLnywtLn7m6lh5TYJKmocewA3KsLRQ4g6nty6WIEb2G6jgdztbct5JOZ2gSGSRokyW7lSCfINI2n6KFGdBUyPb5SGE3Eh2KCYC7mMdd1rZhsrVPJnjylpJB0BUfi8Oik89iNiAFlaNUwsa0ML43uP5TewQU7HwPjqLdbNKjlqfK2HlSZjch2W4spKKR5jAq6Solt6bRGyrJ4pXuqHyPsTqUfNEoDswjefitohrBJbywR0t9Mg3t7pqnkRMYGi56nutUohBG1nMMjH6gEt3F0hJHDJ+G6TT8yCAiO/LqBCOxBIKUrTI0zCVkjRo5x01ULSHmSi+cvHulG+eN2cAFo3uEqekqKocuAhoPxE6K2zAa6PzMr4Wu7lENSUZrmlnOjzHpfUBQSYdBTy2fUXcOmX/ip6jCK6zc9fG4k2u0ooOH8RglaTiMIa82PnubfJIlZhm4jSSVsrWUzpHFhzNc0dO6g8NXxPHNnMg6grp3YPkc2mdWzS9GtiaACuji+xPinEIY63CqPmc5oc1s03T3HRYz3a9PnsyiHl6nrem6KInqM4xv0uacFEyo8rIGA5QSSBZTw07aqqy1ElzJa4adrBercKfye+OZq+P+cLoMPocxc+Rkge/9F7BR/ZDwLQtYJcJiqbAAF2gK8O/u2jTMRj+b6PzvcfbDoOiyjHX/AKl/y1Ufq+a+G+A5+Kal0GEYcXgC7ZHOc1rva67ei/k98Z1csXObSUUAIDwXuLrX1X0HheG4PgFL4PB8Pgo4Ll2SJoaLqeSvjHx6r5O3vW7LKfD8ofkur9t+t2bZnpojHD3RMXLi+GvsS4OwBrhVRPxPOACKmxA+QXW4bhOA8PtMWE4XTUTSLO5LMtwkaurkPkICHlzSavIJXzdm7Ztm88pl+U6jrup6rKct2czM+vmsS4j5g1jCbqtJPVSSWBs22mikEbQibHmcABqubyqrYpJHZXkomw2NgnmxDDKJhdVYhCxw+EOBP6LkcQ+0F/OdFhFBmjbpzZRY3+S9GjpdvUTWEO2rp9m6awh2MphpG82rmZEzbM4rPl4mwqz2UM7Kl7QTo6zFweKYrWYzIJK0vIPwepvzUQD3egG/uF9bT2W8YnZlUvpa+13jeyfNrycWY9UvJmEMcROsbBos6rnNZP4h7Gh1g3RR+FrT8F/qqlbidJhvknma6X8jDmK+xq6XTpnlhjUvpa+n1ap5YRUrhkmtrZQz1lLRjPW1Ucft1/RYFZxJXT/6JC2BvRzm3KxzCHPMkl3OPVxuV3dm5W8VvkaY8OjyA/1jt/0WM+SaqfzKuZ0r+5KYM/K1G1t+i2wEMZ2TgBSW1skGIAAukBdSZRlSjIeCRfTuECy2KW4REWQBzyT5PKgMHROmIJ9Gh7lP5upuVYkCSdkkklQ2YIm7pk4Nh6boDdslfWybsmdceYbqASOhS2KVyd0lQ5OiZJJAkkk19NUCuEswTZilmKMnzBOkkq0BFc2zJ0G92pdp6DTaAJ0lFMDdJ2yaxS7ZlQmk21Q3t6jZIlua2idISTjXzNCcO7oczb5TdOwEktLktRk6lLMkNymJuokCI/eTN2T9PokD8PRFPmvpZMP3kI3RIGbuETfUUJuHD5ov/wB5AgLEApagpa/mukHXdZAtb67JfvJ97ocuqBw5Ly9Uyct0QLT3TJJdfogbME6WWyfUlASa+tk17gfJMgI2ttum190/X5JaFAhmzFC4ao/KCmzIkAs4BCcxBynVGTdx0QgOsULRmDMQbuBabgt6FDPBJNKamWrmMjtS5xuSVPY9k5F1z4xM3Xmcp9HS4F9pfFXD1Iyloa4vLCbOfq5djwt9vHEU2LR03E9S00knlDomZXh3u7svJ7apOjuvz/WeynaetjPlpiMsvfHrfxe/X12/XVZeUPrnCeIMAxt5pKDihss7xdkb3gOv+qfEeHuNHU7iKqKupj/VkXzf7K+SqCurMJrGVtDM6KVh3abXHUfVfVP2Z8XU2J8NUeI4i3EmzytuRGzmMI/Rfyb2n9lM/ZuMN+rPlhM1fpMT6vudF10dXeOUVLi8c4A4cxuudJxHh81G8tDSyNhDRb2XLY39kNPS1TqPg2saYGQGYGc31Hwgr6WNbgeMAUUckvMds2op7X/yWezgplFN4rDqcF/5HG4Xg6H2r7h0tY5588Yioxy84j5/onU9s6bqorPGHx1iuBcc4XC6onwciBuhkYDlVLD8T5/LjmkMU7tCR6QvtfEarDH0xw7iDhdnKcQTk8zSVxvEf2R/Zlxm6J1M2OglBJPKjyvPzuv1/R+23buojh1mmdc/HHziP+Xw+p9mtc4/6X9nzdDj89C+zpY3W+qv0/FznkOs11tLBq77Gf5L+PsriOG8ap5aItBaaggOv10VCL+TLxnFUZKmupDDe/4LyCV93HuHZ9uqNmvqsYmfdM1949YfGz9m9+NxjE39HKfzkpXSOD4bX/KoZ8Wg3jbIQe7bLseIfscqsAY1roJBNKCWOZ+I0EfmsNF5tiUGM4PUNjqIqdt72OYkH/m69XTdNn1muNvTzGWM++JiXz8+09TrmssaltQ1kUlO4/F0FlNSmGW+npXJT4xi+UsZDTn5XCPDsbqGD+mRTMeXANMYzBdNnbOpwjlOP2cdvb9+EXT0ShpoTGPIrRpYQNlxjMZqmmzK+Eno0uAKOPiWqjdme+M/+YF4vDy+Dy8Z+DrxTx7ZEFRQt5R/BafmuVPGL5DaKRjZOxcFIeK6yP1uYfmQrOGXwScZ+DSjwWGWoBfHYHcA6Law6SpoW+Hp66oa3o0PNguPPGbmnURg/RWaXjOAOzygSH/xAFrhl8Eqfg7hmLYww6YnUf75VlmN440aV8j/AJrijx3h4P7Fv1kCMcfYaP6kf+s1Y8PL4HGfg7P7+x//ALU7+P8AmnHEGODetcPouQHHmE9j9HAqeLjTBpjbmOv8rpwmPWDjMOqHEmNjTxx/RP8Aznx7/wC4u/3VyUnEeHOdnE+Vve11agxjD5Gh8WJQG/R4sVmkdGOJscJ81eT9FPHxNjIH+l3+bVz/AN5QW1npXX7OsgGMYc02krIAewkBWowmfSFiJdG/iHGZGljqppa4WI5e4UcGNYpSxNgp6kRxt2a1lgFjNxjCQNa+H6OCMYlhxGYVkVj1CcZiPMvKIr3Ng49jJ/8AqEn0FlH974qf+vyLHdjuBsOV+JQtPYuAQHiPAWH/AOZw/Qgpwn4JxlsnEMXO9bMW/OydstRM3MamW/8AbWFNxfgkbbQVPiX2uI47Zly9f9qlMRJT09FLEDoHnT+K76Oi29TlWEO+nptm+axh02J8QVVLWugge2RgA131WZWYvWVj8z5nMFrWa6wXJwcQPaefLC6YONwG7o5MbrKpp8BQO/tOK/T6Oj06IisYuPe/Qaul1aoio8/i32ubf3U9FxVW8P1sVTg+Lsp6tjswYXeV3zC4StrcYkcDUxPi7WGihia11ycubu5dt2rX1GudW2LxnymJerHKcJ5Y+r6N4O/lHVc746Li2gj8S6UsMsdmNA6XXrOF/aRwXiDxHTcR0rJXGwZJIA4lfDrZns8t0ccuWRsoJa9pu0joV+F7h/h30HVbJ2dPnOu/dHnH/tD6eru23CKzi33XivAnCOPOklxLh+jllkveblgPP+0vIuK/5LdHJll4QxWSCRziZGVRzNA9l4fFxxxnDdsXE1dG3sJiug4b+2jj/hx8pbjE9c2QABtTKS1vyuvB0nsx7Q9midnRdRjlP8vnU/fy8nXZ1vSdR+XZhXzbNZ/Jv+0Slu9jaSZg+Jj7rmMG+y/i7GsdfgFNQuinZfzztLGm3Yrr2fym+P2nzUNA75tW1Xfykp34TFLHh1LUYiLZmvgLWtPWxXt/He1/T/l2accpy8omK/LPxmp9Pr5Ofh9Bn5xlMV+7Gd/Jg+1Iemmo3ewnF1wHFnB3EHBFecMx6hfDKLWdls0+wcu+d/KZ4+k0Yynib2Fyuf43+2Hinj3DocMxoxOhheXiwJ8xFri6+n2jb7TfisY7jhh4U+tTFx8/KfNx3x0XCfBmbcLcJwboHnL5rX9gh8VAB68vs4L9k+amdumTNcx4D2m7T1CYl4OUHTsUaElfRJJEsw1GqWxSuPdOMvwrSmzFI26pZSmQK57p8xSylNbVA99U1z3S65U2UKs2dJJJQNmCXT6IU9iiFlKe+lk6DNZEg5clqSlm0SzFFD5morlK9/Uh0IQHfS6bMUun1TIUfMU+YJjl6JkKHc90rlNmCWYX3VZPc9CkXIQRfy7JE3dpso0IHqE+YoM2qd2yAwWpHJa/VBmCTtkDjdOdkOYJAm90D6e6Vz3SvokDpZB4IxsdO/PNFm7C9lNPXGrlbkpYxlN1XEDuuymjjY1wLFyf2FebUQOeGCkgbfq7ZSXoh/pE0AZ/3Y1WXOWE5CLoYnBrQy9mjZaiReqqqkc21JDIP3nKBuf1B5HyVaWoAdkYHm3YIoqXFp/2FLUFvcNsqLgIA87C5C6ZjdoWj3srNLw1xA9ufkOBO2c7LSh4LqsoNdXxtHUZSisR8kErAGuDXdSSq7oXN9Fbf6r1rhb7Ifs4xPDmYjj32jUtBVOe5pp5muzADY6d1BxJwjwNw7iDKfBMWpcaiczMZog4AHtqvHr6/Tt3T0+N8o/+3KI8vnMV+7EbomeLydua/nY4+zdSVdhosSqXAU+E1Bb3Xa+Jgg0gpIGjpoo5MUq3eVmUfJewtz7OG8bcQ00bGNO5c7ZWW8EQO/03FQP3WNC0TUV0nx2UfKkJu6yLYaPhbhWmkjE09S8ZgHOL9AOp2WzjWGcF4XiDoMAPjqcMaWzOBaSSNRb2WSYpPZFHC/sFicZnOMr8vh7v/kJfmtR4i+EZaambEPZRvxGd5/FeAOyEM7o2NY34AfmFstCZZpBoTbvdJsUj97qz0zBjP0RZvZqLiriC3wpwzWxBU+mxQktB0RDCNnun8nS6a7k+fTKgVgd0fkA7qLNqlzYxu/VESlzT0KWZ3S6DngGwYfqo3Tvv5SqqfmPA7ITNL/rDZQtfnka17g25AJPRazsKwm3/AOsEH+6VHDb1GGmuV+fwiZ/pDNM7m/GkKl/xEIKuOCCZzIKhs7Bs9o0KgMjQjrjlGcRlHvWTWTfAyO3eyUdXIXX5TC7vlVTmj2RU9c+kmE8JGYbXF0MriJr1Wn1pOjmj6FL7yawWdCT9VRdI15Ly9tyblQGd59LUVqjE2/6spHEaY+okLJHMkOUKzFQzy7hVV77zorftD+iE1lC/eNsn+yohhcFOx0lRJFGAL3JWRFxNhbLjw5kN7ANVGtKKU7Usf6II6Fk18lCfoFlVPEtcTysKoIwf37FZVbjnEReeZMIndWtAsstNbiCinosPeZBG0PIAbmuVm0MeVgNlnuFVPI2eqn5jj77LVjbakef3F2wcMk83FXgoW+EwhwsLZn63/gpYcbq62gdUVBdETrGwWDLrFfDNNEI5JTJGDe5V90LOQLukcGbNOq45+rvj6KMcl3iXFA2dw+PNcfRNOcPllLRTuB6ADQq9BSVFPUMmfQgsZf8ABdufordRXZajxUOGCKU/622n0WGmYfwja1xbZG+SV4ayGzL62ATC2V0kxdI55vrolGXC7nssR73W0oDTc/jTR5e90dQ2KIHw8b5SNiLJhlEDpGxx5Du62qjmLYg2zwy5tqsWtCaS5gdIy0h6XUtOWszuYz1ixUlEGxVTOdNEWdboJOTSyvtVxFjzcWK1ErQqp0MpbNyRGQLCyqOm84ZGfMeikmhZOwGOqDnMNw0iwKC2SMMl5ccn5lSkwmpZIpZahmV0PkbkOpPdJsMcMTppJbl5ucxUbi+XLmnEhYLC3QInPkNmt/Ef26lBYnFMS1lLo0i7mne6gkgqGSjyWATxeIl/D8KA87PJ1HyV2jwnE6izzIOX1e91gFhmVZ80cMGdwPN7AKJkpcc0d7dyLIcYdFhz2xiuhnebaRagfVBDVB9o4mcwd72UpLWzNXMH9FfID+6pGw4kYGz1FbISTYNzEkIYJ2U4MlXh88tvSGmy1IZZ628bMIFNbrI4ZyocmbLRTvgMkkl3na7SSfqpWYDXStaTGNB1dZa9NKQ0wOeQY97e6Fxc2YPdWyOBNsvZbg5KBweeK0s1LAWj1AHUoHGaV7Y4adoda1rbrY8exsvJc1VMK4O4qxyolrsNwqqqmySWhMDbtLVzz2Y64vOac93UaunjluyjGPnNf1KCB1FE7mTxyPcbnoo3QVFbiEbaOldIZXBjiOrivVuFPsH4gxeClq8fjZQNZKTLBObPc0L0zDvsV4AoJI6g4c+WaJwc0vkNgR7L5+7uujVPGJufk/N9d7X9v6TKdeEzlPyi4v8Ab9ngbPs84u57Y8O4flllYb+V4tfsu94U+w7GsQFLXcRiGhEbi59KfNf6r3dvKgZkhhjjaPysAULquNvxL4u3uvUbMeMeX0fh+q9su5dTr8OJjH5xHn+8y5PD/sj4IoKuOt8AHzQuzMcNLFdezw9IC2n/AA2npdVZqh722Zoqx5j93XXgy2Z7JvObfmt3VbupmJ3ZzlMfGZlcnxB8Vn8wEbWVeorZ5PQ0AEqKWnvTudl2VkwsDs1ul1zcUHKkebyElFFC0HZTNie42YFTxXFKDBKQVlVIXEuADGG7tVrDDLZlGOPrLWOM5zGOPrLQ5YHpTktjZmmkZG3u91lyh49qZmltBhXKb/rZjqudqfE4hUmpxCtknf8A2rNsvp6O07tkz4n5Ye/V27bnP5/J2mMcW0WFPENHF42c62Bsy39pYdbxZi2IQSQZG0rX2/ZnoskMFgBf6lGGL7WjtmjVEcouY976GroNWuI5RcqrKCkY8SiNzn/mc65UvLjv6QpJXU1O3nVU8cTBuXFYWI8ZUjQ6nwmHmv6Su0C+i9raL8m7QANz0Cyq3imhpXmOnYalzOxsy65qqxHEsQ0q6kvb0aNB+irhhB6IL9Zj+N1/kfUiKLqxml/qs7lMb6UeRycMdfotoSQDiUVt/IAnaXK0GGcDK1uqJt/iRAdAlruoHI0zJk99NkOYIJL3O2ybQhLp9Er5fKqBSSTt3UDJ8xQaX83ZPr7LQQPmTpuv0ToF89k1xlunSQPcdk2cJaJaItkXaJr2SvpdOiSZ+wToLIswVkLME6BPlKgWpTJ+n1TLV0lWScb3smSA1SSBaAXTZrhL+11TKKSIbIUQ36KykF5iUB3+aLp9UJ9By79FFI6dEgbpZQbOO9ktrKAuuUpBpL/JJZoTZrpBzW/VAaROpSvpZMdkSC136J019PKnRSTi1/Mm1afL2T7hA/X6Jdfoh090kBtzdUyFF+6gY7aJ821902UJjuqCSN73chSuVA+YJ/hGyQyk7pO0+JA2UJW1sjDmNY5pjzl4sHdkGYoHAskTqPdM31J7/m1QOc1khtfohLr6J26ICAuHO6ITZOHalCdwiQdL95PfUps2iKbMU+6CxBv0RA3QK2tk/lAsh6ovLqsgW52PY8W0N7EXBXuv2Zfbhg+B8NR4Fj1LJA6lFmSxDSQLwwOSJ1Xx+9dj6bv3TfhupuruJj1iXfp+oz6bPng+guKv5QWCspIjgFPJVTl5uZSGmP3CDg/+UziFTiDaLiHDYfDv0a8O8xd0F18/W13SBINwSCOoX5/X/h/2jX086eMzl5/mmfP+3l9Hs/zXqOfK/L4PuOj424WxeLJJWRU83+rmeP4Hqpqrh/DMRu9jWX6PZb/BfDjaqqD2yCplu0hw853Xf8KfbVxLw/I91VUyztsBGxpADfovxvcf8Oes6bDn0mcbJ98ek/p5zH7w9+nu2vOa2RT6X+48Ww/XD6rmtGzHHX9UcfEc8GWLEaSWG/Vzbhef8I/yg8JxhkbMYpmw1UryxsbHjMe2i9MpeIeH8VZy/FxH92TQr8L1nb+p7fs8LqcJxy+f/Hx/R9PXuw248sJuFeVuE4/AaepAkaRbyPsWn6LzDjH+TlQYrGyXAK8wyMcXObJu6/uvWKjhWgmfzIL0zxs5m6ovoeI8MeXwSCtZ1DvWt9B3Tq+2Z+J0mycZ+Xp+sekpt1a90Vsi3yXj32Scd8P8+evwWRlNFciW9wWjrdcm1kkekkbmk66iy+4G8Tx5hTYtQOgzbtczM39Vh4t9mv2YcfTNqqjDQJoQ5hdTnl2P0X9E7Z/iTnE8e4a7j44+v2mfP7w+Tv7RExeqfu+OXkOPnbf5qKSlpJT56Zh+q+j+J/5MNPMwVHCuLNYGA3jm1zHovIcS+yf7Q8LkcKvheqbECQJQAWn3uv3XbfavtXcsOeG2MZ98ZVE/v5faZfM3dDv0zU439HHNw6jje2aOBgew7k3Vkvc79o24/dGqUkcsEr4JmFr2EtcD0KZ2y/QRlGUXHo8lUN8FKRdleQf+8isohD+Vgt10CIi6cEjZWxBJRU8p80TR8ghOF0Thbki/fqp3IOY4JxDeGBZks3LbYhHRt8I0tjY1rrWzAJcx5Sa4k6pljjlFZR5JNZRUjdUVH5yPkkKifluYJ3C/VDpfzIg249K5fhtP8kfaHPwtX8sfZXma6paxlTLJIGagk2uoThsBN7v/AFV/I1ARZdccMcIrGKhvHGMYqIVhTtZq1zvqtXDp53NED8SbA1vpDuqpJiLqZ68NkVlFmWGOcVlFpMQooxVCSSoiqBJcGRjruHzUMdJTOzcyQR2FxcaIgLKRWMYxioWIiIqENK59NUx1VHIyOWI3aXC61jjuIzNLJ6Khnvv+EFlvbD1ahjnax12PsApTondLmlL+Sxlz6RsEmmZpcY5Hxl4scptdRvrX8/wwDXdRpqVZykbixV4wcgGarkZyZZ5Ht7EqJ0WYg2vZWLD4mpiANkqEQlotoCnDfzKS/wCZMMvxLTIwQlzG3y3Q5GA6I8rT8wge7R1uhBsbt2SMZ30TDOPkgK7fqnvbYke6C+qcObqqBkD3OF5Tp2ROs8ZZAHD3CFpv01RaHdUJo5bbMbYIs+YebVKzreyAgaE307ICzdxZO7KgJFtHJgbbnRBMkmzBIOHxLNJQL5TZO3VNcHTLr3SvrdaU5NimvrdPYFLLb1IFe51TpNahIfn3RnISSbzApFxaMzkDoTv5UWlkCJEHdum8xCNJEsPT6oXbJ0zsqKWYJ02UJ8t0UxNk6HKUSBJ27oW7ItyqwZNfWydJRqDaAJZghTjYrcMk7dIE7JN3SPdu6w0LNZDmKWpCZAYeE+dqiyhOiJLtSBCi1G6cai6K8WFJXzfsMPnkvt5LK3BwxjU7M/hxD7POq6p2L2/Zwx/ooJcSqpNiG/II/r7Ej4Fr5m3rMSbE3qGXWpR8JcP0Pmmlknd++7T9EjPORrM4/VCXPf6zdZW2iBgdKfwaSG/s1G7F2N0gjaPoswNRBqMrcmI1LzfPb5Ks+WWQ5jK4pWJ6Jw0g3Kqow3XZWKmomqsnMawctoaMrbae6HYogOgUYnGJmMp9YQcslOI9MylIJ9GiYD4UaINRJJIpJrn4U6Vgd0D3tvonuCLWumdktohJsiDGZosNk9xlUOfRLM611VGT7pZm9lA51ktRuglMtxZNmd2UBc66cvNvUozKUm6G9j5lHmc3qFCZLv8APsqyuNkj7qN8kYdZrlTdURsB82yp+Oax3nmb+qrpDWfKEBkafiCzm4gJbhrnAfmy6IamqbTRtfJnJebAELKNHM4DU3UElWIzZxVqChvT8+tqhB7A3Wb4/h2rqjR080tRJ7aAoqXxDSLsBPyCVMZJz+Jkbfo46qo/HX0cr6WgoIwY/ifYlZNZiOIeaZ55RP5eqwjpJGQRyWfNGPqrpxDA6GENeTI/8oGpXO4hUvgo6Z7GZhJHcaXVSOGonvNNPa+zdis8xs1PEj5MrqHCxAGG93OuT9LKrWcQ4pVCzJeQz8sYtf5qr4bpzpD9ULxBG3kMZK49yF1jJtX8RVtqGz+Kkdbo7UKcRU9RVidmUO/K3ZMKd50a3RFDRuikc6Cmu/q611OQlkax7szH6+xRwDK4iS0ucW1Gyjhiq3vvFVwZh8OYXUkMz3uPjpM2Q2y2slls/wAjqwtZ6VrZHNpCD1Flms5cta98bMrS7yNHQLTlhmmYxnOELR6nOOgXf/pcMv4lVgc2MNy+lTRyyMOSRpAO1woqjEsNp2eG8dz5P+7jv/FSVXEddizIaZ9KIoIhYG1ifmvNl6vTj6BcamomdI9+YlHURGEuguS8dT0QwPgYRUvcTk+EHUqSrq4XNE7XeZ5sGndMWpVZGz5rHL9SkfvGKldMYxC3NkNtbqaKsp2uz1TLjvdJ7aZxfI8XbplHRGKVKCnrYX+Igq8sZBB5zt/onsJZXc+dkrxuW7KYOczYi3snawE3Nh8girVNHgUoDZ6q0x26LFqyyGuYBGKgC9wFoTT+iPIBkN7jqozWNgGrCb9RugqusZ2TSUMzQz4D1V2qiYSJH0rpGv2uFOfE2ZDO4OcdQewQT86R7nySR5QbC5sPottKUT4GzsDCNeytB0bX3JUNKYaoSPZSimMe73i10dOaaV1uWXHq4bKZMrDX0TnFnMkLumRv+KmhwuasidnJi0uM8uiqzcgP/Ablaf1Cj5UB9cl/qsihPQUsj2CRjXZjbQ7K1Fg0dFKyTnuZpcBXKSmgmElPSwmWaRtmsAuSrlDw9j+MVkNGzBa0ve4Na50RAaPdMs8cP4ppx27tWnz2ZRH1lFFiADQyoGdo2F7KR2LyStP9GLnSjK0M2/Vex8K/ydcSqaVw4lDKZ2by5XW097LvOF/sN4Q4Y5jZ3zYjzCCBUAWbbsAvmbu7dPruMfOX5fq/bDt3T8o1zOcx8PSf1fNzcA4yqmuFBwzWSyODTeOP1D3XqXCH8n2uxPDo6riZrqSWXzENkLcgPQt7r3yngocPiENDTxwstazBZOaog+tfI39237seOP5fo/G9w9seu6zCMNX+n88btwHDX8n/AIJ4cr2Ym41NbMxpbkmk8lj+6V6DR0WHYXEIMPooaWIfDEzLb6KJ1a/4TdRPqJXfEvBs3bN03sm35vqes6jrMuXUZzlPzlfknYGnO9VZarS8dyFCI7tDtb+6NrVzeYDnySM1JQmK7R5dlOG2FlI2E2usCKOK7bpxBrpuqWK8R4LgjWCsrWB5+Abrm6rjjCa/xIigqGylpZA/LcW/MF6dfSbtsRljjNO+HTbdkXjj5OwqwIaSWWd2SNo8x7Ln28W0Je+RjuYz0Ndex0XnVNQ8SS1LpcTxeZ8Upu5pluf0VpuD0UExlbn/AHW9B8l9fR2bHGZndN/TyfU1dsxxn/Um2txDxDjuKTmCgqTDRMNwQ4NJKzoDPE3I9rpD7vLkwZLfQCysQseXebT5L7Gnp9enGMcI8ofQ168NWMY4wfNWn+qCQhry4DIBf2VetxugoD+LUZnfliOYrFrOK8RqGFlDGYI39XEF6703ydBWVMeGRcyurY4/YC5XP1PF9S7TDYrt/O8f4LCkBmeZKgmSQ7uduiaxtvKF0aFM+qrJDNW1DpndLnQJANbsi3TEXSIYNtsE/wA0WS6fIgbzdE4BsiyhPp+cA+6ojaHNHlTg22RXcU2XRUFqGpJE3KZ2yyHTiwOaybNYJHU7KgMxJ8qItdb1XCW22ya7dlQSFuoTJ27oGRWuEsgJToEmGyV9bJ0DuTC3xJtCN7JifiRKEhO4RJIpIXJxuUJ11ciQSSf91Minbul1+qXX6pO3VQySSSikibsgyhOteqeh3HWyTd0Nwk3ZQOmbsi6fVI+2ivqBO3m3TpJJRYcpuH3t7JyO6G1iR0unbuook7d0ANynB+HsgM7bJ0gdU2fKCOnqUCA0sUgABoNAma66Q0brqgLqSmvrZOnsPiWQyaxHVO7MQLDqns4gGy0lhBaHEEFOmcbuF06KWvulpZP0ASIF1QyQF0ibAmyWbQaqJYvL0QpDN8KXXVFPfQJdSmT30sgbL/knvYpkkDuuTdMN07d0w3QONymJCcblMBrZEgV27Jj2CQy3N0jqMtkU2t7OPyTtNin3IPUJiLICDt0Lrg3tof4JNF/L1TF3dEgibp2pz2cU+gHlWVP8N2tTWuEzSU+b6q0EWpZSjs0pWFr2UEcZexwfG8tcDcEGxCtR4xiUMrJWV9QXRODgDK46jvqq5aDugygnKueenXt8s8Yn6w1GU4+kvWOFPt+4gwgSHFnyT3IytY0ZcvuvSOFf5RfDOLySxYrh78NbGARIXAtd8gV8w5PcpuWCLar8h1vsJ2jq+WWGM4ZT/LPlH0j0e/X3PfrqJm4+b7ypMUwPHoA+lmp6uIgOaHbkfJZ1VwjQSzGpo3yUkvQtOg+i+MaLifiDDamCpp8VqrQEFjRIQCB0XrHCv8pHFKCAs4iaassd5Gtb0+YX897p7Adx6KOfT1tj5ev2fV09007PLPye1yU/FuENLoJhWRN+E2ujpuL6TNycVpJYT1Dm3H6rJ4Y+2LhTiOnhl5ppZJRdkUjg5xXY/wDw/GKW/wCBOx/xNN1+K36dvTZzr3YzjlHul9HHLHOLxm4clj/2ffZzxzCRXUMTXF2cyUwETyfctXnfFX8mTDW0BfwpiEjqjOAPEvLhbrqvXKjg2nzczDqp1LKdsuypTQ8W4UWuewVsQ3IGq+l0XtB3Lt0Rj0+7KMYm6vy+zns6XTt88sYt8rY/9lfHPD1ZJTVGAVU0TNqiJl2O+S5mrpKmgnNLWQPhlABLHixsV9s0nGtG6QU1fC+Ano8XH6qnjXAv2e8ZxTeKoaXnTsyCoiNpAPYr9z0H+JO2Mscet1ROPvnH1v416fo+Zt7PjUzry+74qeo76r6RxX+SzhTIZH4Lj08jy05Gy7X6XK8h4p+yHjnhKeOOtwiWpY5pcZKVplaPmv3fbPaztfdM/D07Ky+GXlf0fM29Dv0ReUeXycj5bZU4DMp7qSpoa6jLRWUssF9s7SLoCPLZmmm6/RYZ47I5YTcfJ45iY8pMG62cEtjYFOOc3R7gUgzUlaQun1RBt9Sh6/VPmuEBEbpgzskcxCQ2GUoByOJThmqIPHZPf2ULQSB7XegkJFzZRlFv0VkEX84zDsgIYToLKtRLPlpntfzxrboN1Yhr5pv2kT2kdT1VktBSbTxEbICAdIBIHXBTKCSCaJ+eF9yNQCpnMmfC2cty5t2f5IyK19LJPiAKANNv2zmjtZE0263QCG5Ntk4zeoJw9g6aoTlccwuPZA/m7Ig4/EmSA6BUDldm840unkaL+VOQ4BBnIKWG2HsiFzsk7TfqkHtGxQLmPB0P0RCQuBvuhuwjy7pibHRAacbbKLMVJzDbTVAW4SLA/wCLZMHsOzbJakoFlKbYJwbJlQmnqnzFMD1CcG58qAwSBdqTfSm2KbPbZQOkDvmCEm6QB6Kh8rR8Sfy90CMFpFwVZZoPVPZ1r2Szapcxw2UaJwb0KYC6TvUU+X5oybNrdMS4DQJI0aAkn/2UyBJZrJrBL+0jJZghRpsoQMb9UxzXGVPt5r6J/wCyiBSSSt1sgfMUvL2TG56pWHZAkkrnukhMkkkkg88DGl2psiIAOVlyjMIabp26HyhR/XEeWwThl0dnEpw1w1upxboWXTZDZw9QR38qElpRCBBCYut0uhFr+ZEXooQTfZEXP/LZR8yx9WqYyuuiJ8wCEytUZdcIC9BNzmBMZWlQ8wfEUjLAB6z+iqpOfqugwLAIsTjM1VViJrtGBrgXE/JcnLiNJGfiPyCGHH/CTNnpuYx7Ddp7KPN1Wvdt1Tjoy45fGreuR4Dg7Y2sNBE8tAGYjUp34Hgn/YIP0Xl8n2iYyDpVkexVZ/2gcRPPkrMo9lafmJ7J3Gf/AFv3l03F2F0uGVTJKZ2Vs9zkAsG/Jc9zSDZZldxPieJFpraoyZNr9FS+8Kpx/CzX9gq/TdFq26dGOG6byj3t8uulzLBY0UmLTHyscfotOnwvEHASTwyu+WiPUT6jsoHSVRPlatNkVBSNz1HKFt7m9k8vEHD8LL+Izns0XWSVBlNXyH0q9FgVbKwOJ39lWHHcA/8Al2G5ndHF1v71SxDibGagWp547/1mQek/NEaw4aiij5+Iviaet3Ib8NwtDAyOZ3QNBuuaz4rO3PVvknv0edFNDX4jSWzZYox2C0rSxnEp2wtZQwcgdbhc/K+rrgGTm57q/VVEFZHn5xkd2KrSujpWXdcO7LlxaQjCaJnlq6uV7j8LOqjlwXDTbwrzBbrfVWIquNzbvGqimlY0jXdOIlw/C/DOL4akSO6F50CtVuDV9fFkMlKD3Dlnc2Mj1uHyViKvZSwX5MzzsDus8WEPNxWOIUkhD2MPLGmllYgpNmTVEUQHxEqON00gzvba+tk/NLfStxi2mqIIWgBmImUnoNlJh8tNT5ubO7zW31WbNV1UTsvJzf2QrNJNWSNzsjbH7ObqtcRMJTq5gIbmNiRumo6/7ue93K5zZNw4qvWYliBBhaYz/wCWVO2NtThXNpr+JPpuVJg5ClquVq3CYoCfSWx63UbaGvqIX100M0TGC5BsAjw3HqqnibSVWBTz1A9btLW+aOpreJcQifA48ilfbyvtdWi7VMKilfM1+TdXMRFpcnsL/NZFFV8nmxSV450b8jbG2iDD8dg+93sqHSTtjFwOl1vknBqQw8plpmjuMg3CAizr+bL0BK0GY5hVYxzmwua47AC5VKatp4zzow11vgcP8FhURmhgPMMAkt0KumcSQ3pcOZTyd5Rc/oq7eIK4ycjwsMLHdBFa6mjr42vvLA4+wReSJklZK28jqX6Nsnioop7+KrWRt9irVbJS0szP6NuL+VUZ31rWFkdOM7yS256IWGJsEJ5MfpCmtGf2Zuhjwp8cfOkdK95AJDfQFIIQLNjmAf8AlKkwRKpKTYPeWFp3HZSU1SfOKV0We17vbeykhqoudlniBf18hTVJmfK4U9NKxrtLNZopaTlSCmrqvnA1sYGnqd5VpUlThwu2qki517gnoOyp09FidVVxU0VE+aWRwaxoYXG5Xd4H/J/+0rGK5ja7CY8OpiDeeRwBaf7K57eo1aYvPKIeTqu5dL0WPLfsjH3+vn+ke9yFTVOrIXxl+ZpOlhYAdlQhpKyVzoqSCSaQGzmMaSf4L3mm/krTuafvHjuUdo44F6pwx9mfA/CEV8Owlj6l0YY+aU53Osvm7+7aMMf9P80vzXW+2nQaML6e88vh6R+t/wDFvlXhngLiPimr8HQ4NVPcWl13NLGWH7xC9a4d/kw4bJSwVXEWI1ME2hfBG5rgD2JXuvMp4j5GtY3sAgkq4+6+bv7tu2/+X+WH5Pr/AGy67qvLR/px8vOfvP8AZzGFfZR9nGCTQ1VNw9AaiG2WRxcbHuusEsUVxAyJjezYwqMlUHbKPPJJuV83LZln55Tb8tt6jbvnltynKfnNr0lTy/U/+CrOxG5GRhII3UYiFspBKJjGgWXPk4gLp5Ha7J2xOvZ11I1n4gPRSWsSgh5YCkjbY3smmmhpY+bUuDW3ss2fjDhmku11eZJPyMbcrrho27IvDGZhrDVszi8YmYbLGl2wUFbXUmFs51bKGMvbe5XAzcY8RyOcMPPKZc5XFt9FRqTV4hL4nFJpamW1i57tF9bV2bOconZPl8n09fa8uUc58naS8fYKxpFEyapktoRF5QuSxHHeK8VmdN4mWkicdI4jYfVRM/D0ZG1o9gi5juwX1NHbtGiZmIv6+b6GrotWmbiL+qoaGSSXmzDmybZnC7lZFOWN1ACJvMcgqJ4KRvMqqqKFp/OV7YiMYqHqiIiKghECRe31ROijaLvma1vzWBUcXQi7KKmdL+88WH6LEq6usxSXmVdQ97ejNgAqrpq/irC6NxjpGOnmZvbQD6rnavHMWxJoEkwhjO7IxYKryGDZoCNrC1aYBHGxpzMGp69VIGndLLYp8xVAZHndwRAWRW07IgPLeyCK2t0YF93WTE3SD7+YLQduyTdk2vumQFZt9bhOgRWsLoCDvzJ/9pMHJlkGkkknmlyEixsU42Ta+6VytKR3Sy65kyLpugWt/ZMd0ycWuXIHvp5UvMUtAEswQCn6/VJ26XT6oH6/RM7dD1+iI7oH0ATZil0+qWpKBk2YJa+yFAWbVJ3qQp8xQEnzFMkqHbumP7qHW6Tt0BJJv3kXqaraUAANGiJJCfIdwopyf3Uuv0SBtdOga7bpmSMJ8rrpG1/KmPdAra7pJA6p27qBbFE31oI42ts0HROD58uw7oCkDyLB+RHE4x2LbX90Lt0m7qhkaj/tIsxUBHT0pNy2THZIbW6oSLMUyYWunPZZCSSSWg/+STt0yQ/j3QJJM7KXFzevRPlyOuTsrRZ3bpr6WTHZONrKBJJJIDG6ZDqEhsbqQEl1+qScdiqGuDqE+YpvknLfhO6MmGt/ZJObWSGuiNECdQU4HwoU+YoHQ2ukl1+qAQbaZUR19KYtvul2QOD1CfY7oDY6W+qe2mf96yAg7XVGdr3+ig6ZkYd2KCbI4jQ2TNaUGYg9SnL3XKgfW6Vtd02Z35UX96iBPmu1Dl/RSC3VL+5CD0VVU0NQ2popnQys9L27hdBhP2h8XYRXRVkWNVbhG7MYzJZr/YrnG6Oa4ojZxvYD5LydT0HS9Zf4jXjlcV5xEzX19XXDbnr/AIZmHvvDX8pQx0duIGN5zXWDI4ydP7S9P4Q+17gzjFkvg60080bspinGVzv7K+McjVbwzEq7B6oVmG1DoZQC0OHYr8L3P/DzoN+GeXRzOGc+lzeP09Lr9ZfT09224zEbPOP3fc1Zh2EYtDeenika/ZzViT8EwRXmwmeWB/QZl8uYD9rXGuC1sc7sYqJYWnzQ5iGu+i9c4b/lF0ElAyXH2Rtma4hzWNIIHQ3C/A9y9ie69uiMow8SPjjc/t6/tT6unuOjd5XX1egCfivCXBnLbUD+Ku0nG9OZuTiNJJS+7tlX4X+0zhDi6m8RhuIxNs7Lkn8jr+y6GbDsOxGIOMLHDo5hBC/MbdOzp8517cZxyj3T5S9uOUZxeM3DBxzg/gLjVzZMUw+mncL5XtGUheYYp/JboBJJPhXEE743OLmxuYzy36XXqU3BDGuMmG1UkR6EHZVnTcVYNrL/AEhg+LNqvpdB3zuPbMZx6XbOMT7vWPtN1+jlt6XTum88bfL3GP2T8YcIOdUVWGSSUfMyMkj85cOhIC5R9JUxtzyUkzGjcujIC+1afjSlNqfE6flP7PjQYpw1wJxjRugrKGKRr9S+MFhX7noP8SN+vHHDrNUZfGYmvL6fH9YfL29nwmZnXlT4pEQclyrL6Jx7+THRVla+q4f4hFPC61oHx3aP9oLzji/7F+NeGHPliw2Sto4mZ31DNh30C/cdD7X9p6/LHDDbWU+6fL9L9L/V8zb0G/VEzOPk88ydU3L13UvLnIyOhkB6jKUL2ZCWk6noQv0sZRPpLx0EMslsU9zZIai60h0zdkpDZXJcLy0/iqWsgqGD1ZXWLforMrCmT8SWb9E3mymyQt7hRBNkubKnPPPQVDpHnm0xNwPyq3csF2NH1QvcyVhZNACCgDxtFMzmMke0dnhGx8crfw3glVxG+AZI2iSInRjtwpoxC0Zo2Zb76IJS1w7JJun0SsbIBS9XqPRLQphlHxFAWW3xFMiuLaOBQZgrAkY8MvmYHdro5ZKKVgd4UxSjS7ToVGBZPywfUEEdgDmaE7nAhO6zPKk3KgDUlO3ZLMUbTlGWwN0AWF/ZLNbZIuAJBjN+4KG9j5lRIHfmSPdv6pNyH1JHNe2bRQPrl8qbP7FONAnBsoI46mNxyFrw7sQiOpREsJ/Zi6Hc30VDhovqkT5tLpf7SQsTqqHSI6FFlCjkkdEQCwm4vcIDbuk3dAKmA7vMZ92I2lkh8kjSPmshktR8kjcHzCyWoWg9ymSuOxTFwG6B0vkkCAfNsidbpsgY3+JCDdE5Iht7oxJycxQoidfSUDtkZCjTDZIAfEjZZQmbuicNPIQUPXVAyfKUtCU/9lA6HKU/9lP82hVgIB6JkaHr9U9Rwpp6n8qbk1Tf6sKyIpjtWfwSdHVi3Jqh73Uf2JX5dV/qf4IC2tH9WVc52KRgAMiekKmv+OljP1RGflnHrYQkXGNty0lXS+qvd9J/FDzrftaU2+SjEKPPB23S5jyrXPpD6oHfQJXoT/1eRRtTu0eYvCESMJ3VoxYd8Ucn6ITTYaG5rO+SopvlANsyB0v5XK54PD3axQge7kElHhpI51UQfZVVF8hUTru3K1Rh+FHarc89gUQwmlP7OUtH7xslObEfT5z6wPmnbhsjvhW66mwqntzqqEO7ZlQ/nNglPJoHOA7BappXZw0+Xeoj19lcpuCY2tsZ3H/n5qnPx6xtvuygu7u4XVCq4pxmriMklWW3+FgsApMr6Oqg4bwajtznjMOhcgqsT4bwt13SZiPhbqVw0M01a455ZZD1OZIRh0vLEN/crHIdZPx7Sax4bhpc7o5zbLAxLjLF6pvJfUGF7/hYLBDG40Mp5FIJGe8dkEscE7874Wt+SchnysrpogXzGR8hturNPg2JR+SSRob1N1Y0iYBl0CeI1MsRngJjaPiIV5Ct4NkMZhuSDv7qSmp3hl6VvkOyGUTyyZpap7r79FeooZpAIoZA0Dqd1OQpGKocRn5oseqmkga+O5mmc49L6I53tGZj6hxt1JUINQ1zQyCSVp+IBWmgCkrIWtDCDmPUrQ+7a2SnLpYHOJ6qSSCSOPmTROYB1OyhixKOWI0rHygv6tNypMCszAqk/wBT/wDknfg88LTLPAWtHUlBNJI1hZFLUB52JN0qTmStLKp87o2C5F7LNAL0QH7QX+akpGQTzNjDLyE2DjoFOJuGAcszJc3YklWRiOBhv4E0sfuyLX9VoWaujipHCOV7ZHdbHZNHhtDO0SGEAHYl6i5Lp2OqoKZ73d6g2J+irwUFdWZ2V1VGxkYuImDS6DZiFPQs5jJGZRvfVZwxGnrKx8kZ8oF7qrTx0BY4T0jjkNhfYpOpKCrfdjuREwguAO47IqxU10LLMz80n4GDVQtdHOwPySQN+eqtUGOUOHjwseGRxfld6iSrNVXQztyyMjJ+SMzLBnZVPe5japzm6Ws/ZV5JMQY5rKOdsuS+bO+41/5K2IKaetY54kijsbWIslRMno667YI5HBpuNkS2KMDZWMdNiFZFA0bhkev6qeHh3D4/JQvPN6PIuV0c+BTzOfI+1OH6nW4uhpKKOjIeMRgvsX+3yRpjNoZuby/GS/MxgK1TmKnq2sdSOc8m2e23yXTTT4RKG8yqa7IbiwQczChLyXBoJ0aHGxJTkKEtb4y1DSUbs/53agKCTCa42yzQ/TRXKzEWUUpipaSEuG1nhZtNiOJSsMUU0F5nDyAeYlOSeizNh1JMGlz3PewWz30us2eCqlqMgniD9mXd0/Rej4P9jHHOO4bDUNjhp2zeZzZGlpHzXY8OfycWxvlPE+NaOaOW2lOVwPuV4dncem1xN5ej4XUe03a+njK9sTMe6POXhrJfBnlVWKas9R6D6rsOFOEeI+KayJtLw48U8zC9lfIy8buy+kKHgHg3D4IohglJUuiaGh80YcTbut6BsNPG2GngihiboGRtsAvl7u9zONasan4y/J9b7dZ54Tj0mvjPxmb/AG/vbwyl+wPiqd5bUVmG04/MWOv9F3nBn2M8PcMROlxSU4jVSEh73aMLegsV3JeSqstexrizNsvm7u4dRvx4Z5eT811ntH3HrtU6d2z8s/CIj+iSDDsBoLeCwmjhcNnNhbmH1Vk1OfU3J7rIfWyOvZn6IGzzTDzPNl5Zm3wpynL1loT1QYSXP/RVn1Zd6b/MqqWuvuSXKXl+VunVRlIZHkWQiPzXupDFqiEajYQxpIsEYaeiNsf5dT7KOerpaV2Ssq4oDa/mcrjjOU1jFrETl5QlcNLoSQBc7BZGJ8VYXTUck9DUtqpmHK2JpsCVyE+PcXYrA+KSpbRRv25LbO/3l6+n7du6jzqo+b1aei27vOIqPm7p/EOA0jrVeKwMOwF91z+NcdBta2PAQ2eLKLyEX1XIs4eYJBLNUyTOH5labhkbei+50/atOnLlPn9X1dfb9WvK58/qtV2J4ljEjJK2rGUCwA8oCgjNLSmzJIr97XRfd0HuiZQxsNmsH1X0McMcI44xUPdjjGEVjFQT8RjH9eP0QGvEhsyS/wAgFJN4CiaJKqeCNvuses4vw6IcvDqU1BPx+kD/ADWmmu173+oH9FTrMbwui0mqC5/RrBcrl6vFMSrfJPUkMO7GaBU+WwHRgW2WxX8U4pUnLh7BSx++r1kvY+ZxmqJTI87klSN0Gyb/AGVukMBbonylOkgDM6yIXI8yZNv5VoFmCfW4cCR8k1tbIsttCshanqkBrdH1+qbLZBG135mFFmKJoYTle+3ZFEYmXMgJPQLQWUJ8umifc5su56JbmygZK90x0JQh7iPQR81A2UIgLfEmRaEIGDUVtbBM3ZEMx2QMmtpZLr9E2YrQY5tkk39pOgJ2oKQFkjsnRcg7lLygIdAEtwiF1ypaeybX3SHqAvugJOd/KmsL7pswQE70odAE6SAb63TJyNUiQAgJAn7abpkEmYoOn1SzFEqB6pZilmKSjJ82mVLp9E2YplWhZtU1tLpbnRM47fNAkkkTRn0UDdT5km7pW0ul0+qBmhoJ136JeYbtTE+YWF7ovKSMpVDk2TAXSeHgHLo7umQOC23XRIOD/MGkfNM0bua69xsiabAt7qBa+yYb2Tu2TgX9kD5tE4GYoUVtLoEW28oCFOXWOVO3ZAPT3RDofZMWphtfTRAaSQcCLt2TgXBtugRd+ZvRMmBNrF103mujIiepTR93JDsk3ZVo6QumdsnQPl66WTb6BMTqnHtugQ0SOqXWySBJJswToF0+qWp0KLp9E2o1WQxPUpDskkqF0+qVyknbugcG48yFPmvmQk2RII7hOcwNjskbWCSKK1zbohDdcpTkkHME+Zn1UA6W90+YJWub9Er2KBy5IPN01yUgBezlBJe/XVK4BtfVR/h5iL7JjogMnTUJs50HRADdE3L8SA2Xc62gHdCc47JibebKbdbJnWf5RoVQQLrapFrPUhL7tHeyYF1ttVEtYpK+eilZLTzOYWODgA4gXHsuxoftm4/w2Zr6PGS1oIuwsBBC4ezfiAScR0AXg6vtnRddMZdTqxymPjES6YdRs1fwTMPqDhr+UZgE+HRux8xU8zh5gH2IXpPDHGvDvGNJ4vAcQiq4yS0t+K43Xwt11AWhh2OVmETxTUFRKx0Ts4aHHKT7hfge4f4b9Nnjll0eyccruInzj6fH9bfW1d3ziYjZFw+56rC6Kdtn07QfksCr4KjjvNh03JeNgDovnzhb+ULxVhFdfGDHV0mU/hcsb9F69gH28cIYjRRVGJTMoHvbdzXOuG+x0X8+7l7L907XMeLruJ9+PnH9/wBn19PXaN/8M/duQu4qwt+R7G1Te19StGl4xhBMOKU8kV9wWnRaGDY/g3EVIK/Bq+Ktp3fEw3ujrMJpK24mjuD0K/P5Rlhlxyiph6ImJi4UZsG4Ox1wz4RRyjo9sbAf4LguNP5OeBY6Jqvh+fwlZKQbzOcWj9F1lbwW+BwnweomgePhadFDHiHFeENDZgJ2s631Xs6PunV9v2xu6fZOM/X+vxhz2ade3HjnFw8Kxz+TtxhgdFLVl7aoxtu1kDC7OewXls0MlNK+nnjdHIwlrmOGxC+2aXj+gcWtq4ZYXHo9HWYJwRjY582F4dISb5+SwOC/d9q/xF6vp4yjr8fE+FVjMfs+Zu7ThnXhTX7viHLmCFsJG6+puPf5POG8QU7anh6tZR1sflYx18hbfW4HVeX138nTj6hBkfNSSsA9UVyF+37f7b9p6vTGzds8PL34z/eIfO29t368qxi4+Lyvlv6IQ15F3NVyvoa/Cq19BiNM+CeI6te3Lcd1ENdl+twzx2YxnhNxPpLwTExNShdHp5jsdEmA+rMpXtJ0KDJY5W7LaE5ul0spREOsTbROT5RlVEIGtkjn2B0RAd2omjS6gDLZA31KU6E3T5GEXIRlF/YACYXvlcpuX+UISwg5lWgZ8pyndPzB2TvY4nMhcwhA+4u5IaadENiEsxQHoU3X6pmkfmTja7UAddURFxtdIXtlzaJN0HmVDhv5kxFz5boj2TagqCMkgaomm4zNcizZfUAhzs7KgSSCnzO7J7s+JDoUBAE+fRPfpfVLK3umyu6IDY42SLimDXgelJ7cw9VkBcwv9TR+ihdT0pN3wi/caIxnbvsnzFBH4SMfs6mZh9ypbn4kxehzD4kEuYpkLnEDN/ek2S3qQEmtrdMZWdtU2e+6A76Jg64TByIfJAg7VNmizWLrHsnO90OnVo+aM0ezT6Ug259SctZk0NkhohROcPSmyhP0uiy6XBRKRWsnzFIgokUJv1TIzlvZyHy90Q5IuidlQ5QmtcqsvL+TO34j+qVp2/1zgjMjj8Sj5kncKP6+TZ6thuyaQ/VMa/ExtMUg54+SmjpamXaA2PVC0P3jin+uKX3hiXxvJWhDg8588hAb1VxmGUg6vk+iIxhidf8Akap4qzEnbRhbbMNawfhUt1HMaOkbnq6uGG24J1QU2x17/W+EfRWRSOcL5bqpVcT4NSf6FTyVT/fZZtXxVjMzg6kZBSj8tgbo03nUMjGZ5i2NvuVnVdfg9E7NNURucPhtcrjsQrcbq5ediFdLkHRugQ0RgdLepYTH3PVGm/NxjA1/9CpYRb4nG/8ABZVbiGJYw4vfVCNv5W6BWHmIyhlBh0R/fDf8FMaWceetMUd+x1/RZVmUmFsMwfWz54PzXur1XhdC6G9K7K3u7qirp43ZWUsjYQwW23VWKnOXnzVPOPbols0q0sk1GT/RebH2SFXHLUAy0jvdg0AVxtWQMo0CilqIydYy75K23S9FVUUjeWWCO43AWZUVklOS+AxaHQk3urBZCfhH6pMpqUSB3KaXdCeizLMwaPE6qti/GvfbawQyCoDiI6aRw72U4Zy9gPqLppsRqwOWx4SClIfeBP8Ao5HzUzHz6Mk0PZTyVjDTmH4zuVawag+8g9rp+VCDZxG5CrpTMm5znkwN8vRXaeJrqa7yQ4b2VrFp8HwzJHRQBz2m7iTp9FlVGJ1WKtFNHGyGIG/lG/zRiYSGnha10jPOR1drZTUstexrTn/AHRM+dxj5Aa0M7AJhM+1mMDj7onFM6tkmsyeB7ovyufuoOW+neaqKkMbejiUcLZ3EyTyNAO19LKxPh1VVRskkq/wWCwF0aVmYhM94fkaLdkfiHyOL3vsD7JRwUsQyPqAfdqKaEwZJYZBZlstkBy1sAsWRMFhbVqDkmo/Gbyx/BBWTV2IxfiM5gZ0ay26CLDZqzKKh8kTANG23QFIwynKZ3fqhgLMOidkfIc5uequSYfhsMdxUSRvHvdS0FRhxoTSmlDnEWL37k91hFBs3PbnY2Y99bK9Q0VLVMMIw6Rrhs69wrMeHTiAMhLWg91egNdS04hcA1vV4FgtQMk8OEPL6slg9uiuRYDhWmWaX6lPPWc2U0kNWC/pfY/JTMdy6cF7CXjewVSVuHDWRgCnju1WJMDw2Q56mTLL+7JayxBWPr6uOiZPK3MQA2MOLiuk/6FvtCxapbU0FG6SEi7XOeBdefZv16f8AzMoh5eo67pejmI6jZGN+lzTncW8XBmjhfLO38ztVSbXQRUnLfTxiYdV7NwB9hfEFJXiTjB0baWKzmwsIcHHsT2Xqh+zH7PXav4ToT/sn/NeLd3fRpy4x+b6Pz/cPbPoOj2eHhez541T5X4c4e4k4jngpqLCw6SYEsmynJZd7QfyduOKzEoXY4aKOlLvxXNku9o9mr6JocPoMLpo6LDaOGmp4hlZHG3QBTulZH63L5m7vO7OZ8OKh+S6z2363dlMdPEYY/efrfxefcP8A2I8L4LTinq6NtecxdnlsXfJb9NwbwdhcgfBwjTxuYbiRkd3ArbfXRt+JRHEM37MEfNfMy37c5ucpfltvceq3ZTnnsymZ9fOU/jGSaMzD2dugdJGPW9ZWLVEwr4eU+3MbqnAkd633XJ5Vl1dC3QaqrJWSOeA24unEHsjELeyMmAe4auKblAk5gVYaANU5NxYKiAwXbdqKGNoFkfntayaBoa0mWVjbnd7gpETPo1VkYmnZShhyjRUsSx3C8KpHyumjnlAu2NjtSuck+0LFpW5KLC4omd3vK9Wnot/UY8sI8nbV0m3dF4Q7OQxQNvU1EcXs82WfW8SYRh8LpRUidw2ZHqV5/WPdic3icTrZJn9G5vKB2Txy0VM3JDCAvr6+y41E7MvrD6WHbIqJzya03GnFNXMDSU8NFD0JF3rImo5aufxWI10tVJ+/0R+LBGjUJlk+Fl19XV02rRN68afR16Neqbwik8UNPD6IhdSGS5zKvGJ3Dayiqq2io7+Lro2EdAbn9F2p1tbznMjAJ6gfNc1V8VsPkoYL/vv/AMliVNdXVj71NVIW/lBsFR11bj+GUXl55lf+VrbrHruKcRq2FlDH4dh6n1LEDWt2aizO7LfFTSGSY5qmZ8p65k4yAeVgTgdUtApTIQ6/wnTqitrdOZLG7QboWn8/6gKhJXcW5nABDmKdtnC/RaBjXUJvN3T5ddFIM1rZVAIb+ZI5bo81ihFg3a6LiJuyFApG7XSJQJNkV7izuqG1/UkSLqBN9Q2SOyQP7qbMbLQcSGxy6aWSzBCnAN0BJWuewSy3dqQns1ZA+XojtY6tTAdk2fWy0H1BT2cB5moT0vtZIZicoQPuUySbX2QFqSnGZrkh2TdfqgE/mS6/RJ2/slmCBFyFEPfRK36IEcxSbsnSQNlCXX6JZdL3+idA7t0yE7okDOTAEpONynA0+SMkHmwaToEtPZN1+qWUo0TvUUktfSkRZGT69wmvrdLKUtPdGiLNNUj6fUmSOg1VC33TCxJt8JsnStZATn5je2qYOPxJgTf2Sub7ICH5UmAuGZva6a7Rq7ZK+vlQMlfVNrdPsCgVySlayQva5SLmt80j2hvS6lhmm2iLMUILS0vDb6aW6por5TeN8fUX6oLDfM3XT5prDKO6G2YXLkviKgcHMEs7bhlxcbJslgD0KctaTmI1VAu9SJuZLKUP1sgPQhN1+qRzXTICtpZIX3SzBLOfogVvcpy3y36Jv3unZNe4BDbFA5AaRYg6JA3TWzHMnzXF0DZtcqduyLOGxuY5gN+ttQg6Kh72OgToEaTBE2br9Euv0S03SbsoG1CVxa3uncXEoAMxQG3ZOmGyQ1CB0kJ3CJAkkx2SGU/NA4sRfKnFuqYaCyV7lAjrdJJI9kDk9EwtfVOQbprdLarIc9fmmTEJ0ZNsEBLvqpGGzgTrZCcue4Wmgkjd43HROcp2TPN7JwNEC3YLIm/v/qhP7qXmtqglact9d1CTyyDe1zZGb9EBddti1QCRY7o2yuZcZtCor6+lFy2luZUHcnzdEINk5BBy9EwGbRQIm7bh2qYOFkTYr9bJZLHZQMDc7oi0dShtbdqe77HNqEGxgHFeO8NVMNRheITRCF2ZsYd5T8wvUOGv5THE1NV24ijjnpg3TKwlxPzXi5cSl+q+D3L2b7b3S56jVHKffHlL06er3aP4MvJ9bcMfb7w5j0Uk08LqZrHZRzHgE/qF6NQVuE43RsrqGeOphl1D2OuCvgFdhwd9p3EnB0ZhoJHys0DGmUtDR2Fl+A7v/hzOGM7O253N/wAM1Hl9f70+po7vc1uj9X2NW8OYPWtcyRlidjbULArOAwX58Oq3QjpZ115Twv8Aymq55ezinCoBExoLXwNJe4+69gwb7Q+E8Zp4pI8WhgklAcI5nBpF/dfhO49k6/tWXHqtcx8/WJ+kx5Pqaep1b4vCWS3EePOHrxmnFZENnht/8FZo/tJkZaPEsKDX9wbLr7sB1eD7tNws2t4dwavc6SaPzHqviu6gaXgPjXPLimCUUlS4NB54Ad9Ddcxxf9gfDeM4ZJBgbhQzjzMPlIJ6A+y1q37Pqd15KSqkjf0DtVFHVcacNNzyuFbTDvqR8tF9DpO6dZ0meOerZMcfTzmvs556deyJjKPV4TU/yd/tJorl9PTTsHxRSXC4vF+H8V4fqZabEKOS8XqeGEs/VfWkX2mRj/SsNla/2KsVlPw7x5QGlr3wPjnFnRPFnn/my/b9F/iN1+vbE9Vhjlj8oqXzdnaNU4/kmYl8XeJj/Miu0r6vxX+T7wNW0UlNh9I2ime2wky57e41XkXE/wDJx4zwipa3BcuJU5bdz8wa4HtlX7jtvt12vuGU4bJnXP8A93p9/R87d2zfqi48/o8uDwG5XIC4X0XQYr9n/E+D0clViNC1jInDOGuzH9AueaxtrA7br9T0vW9N12HPps4zj4xNvDnrz1TWcUksH26JEZNDb6KJ0TfzfqlyWh2i9bCW7CDbohQZbHMh/BJ87iCoCLkmSNB1TAR9Xg3TFg+FUM4xuu21kNwNCkAwmziU7hY+yiWF2VgzJh5m5gbJ3ZCLEXSa1pGjhfsikHOFxnBR3uPUgIN0BB/OQqJm6FET0coRcj1X+af8f4XD6oH8wNibhC+I3zNfp2Th0nxMB+SXNba2QhUM3N1QnU2vqpWujJs42Pui5UZFyNUEQcb26qTPYDN1SdEXDQoOQ+181/ZBMPmmO4TM7JHNdAQv1bcJbHZCXm/shzuQSEC6EtF03muibsgEj8uycgZQ2yY5h8SfMUA8sEpjGPdSD81kt/NZAAvdFcj4k7mg7JrfmQMXkbtSH8E+RP6bjugG3YlEM35kDdkfT6IH16pX7Ibnqbpea6Ar90hrsm1v7Ja9kCs46pbBOh190ZPc2SuE+qSJTztuGT/GLKZuG/uKtJxdR5yylpZKjsQNCq8/E2NytyQUIhb7o/rjZZh749WiMDu5PNPRUjc9XXxD2bqVxWIYjis3lq8UdGD8DXaKsJGH1zXUHT1nEgf5MIp8zhvI/QBUTiWOOH42Pxsd+Vg0CymeEO7HOPsncXx+aGAR+5CrosVIxzECQOKJAB+YWVX+bVK93NrsfMkvWxSnpHSxB3Mdc76qk3LDOGOWUpqyQw8rl8wO7EIaeLJb2VGXmZstLCXv6WVpsMwhz1BLHflBTktJ3maf9jCLM1JcqzgJrcxgFuoTNqn25MFK7KPiJTtm5r+WyM3UtV6iZNEHOpqjkNO56qGeK0pJe5xPUpo5aqH4Bb3UUss0r75VUsIMcPmcxp+YUwr3lmRjBb2CqywmUgkOAb1VuKOAxCOSoEYBvZo1RQR0dVPHzBCMt9r2TmGPLJ5Gtyb2O6qzRTEjl1BYb2AvpZWZIZgBGXXNrkjVYaV6eljERnmjeCPicdkpZWZclLIS87KQOaWinkjL2neyZ/Jp5R4eneSOqAWwTxwl9U2cnvfdSQRQTwiR8Mo9iVJHUyPqAX5j8grNXFRE8mMy83qCbLbKlFV0sZ5clEHAbEIaqV5HOgg5cXyV2Kkiy2LfMEhYS8oNBPuEaiWFUVNVWXpaWPLGdzbdWcNpa2ljkM8flDNHdFp1dNPIx0MNNkd+du4Qg1cUXInlce4KCnDMwAh7ydUnyPMvKjY/MevZXo6WNzc4nY3vcbIo6R0jnciUOI3LtAVhLBQ0bHyfiT2P7ysZshMbTombgeIVmr5GtH5QFai4bxEH8BsIHdxuStQjNfR3aX5nj+yFEIqmF4gp4/PuLla2I4RjEEQFRUNyv+GMalUIqV9IOYXPzjv0VU3heIAL/hRN6uuNE1VA+BofLiRkeRc2GyGskfXtbHW1pyA3sButaDgvH6rDmVOGYLPUQyC4eGh11nLPHCLymnLbv1aI5bcoxj5zEf1QUXD81SwVU8jOR1cDqfkppW0NFlNBQMmbGfM551J7LYg+z/7UamBlFh3Cte2BgsS9paP716Lwl/Jmc1zK3irGGljowfDwktcx3uV5t3XaNOPLLK/p5vl9b3/t/Q6+eeyJ+UTcz9v+Xj02O1FWGc6DlCM3u13/AAV3D+HuJ+M2SQ4HSVVaId2RtuvpCj+wn7OKMAigqJ3j4pJiV2mDYNhvD1Eyhwmjip4GCwDGtF/meq+fu71hGP8ApR5/N+Y63250Rq/+jwmc/wD7o8v2l8u0P2H/AGkuiyP4fFID8b3AkfRemcA/YjSRYXJHxfE6WbmGzQ612r2N0+X1OAUMlQxvxhfO3d137seHp9Lfmuu9ruv63T4Xlj88bif6sHBvs74M4eqG1eE4FBBM0EZ75ibrpOY63RUJcRZH1VWWve70PI+S+fnnlsm85uX5zd1G7qcuW7Kcp+c2031DR1Cp1GLRQvyXBd2CokyOHqUXIzSAv3WHBediL5BZjSFUJne67nkqwYxG0kbgKGFwIzHcqKkLTyyVNFH5Qha1xUwFkFSujkGJUr8lwIyFaiaLZlBVyXrYXF3QiysMjkI9OUe6NH0CG9zooqzE8Jw6Evqqtpe1t3MbqSuY/wCkUSvLcNwmVrSNJpHDL+i76el29REzri6dtXT7N3nhFutcxxNi5rfm5UsQxrB8KgfPU10f4Qu5rTmK8/r6nFMTqHVFdWSvDtwHZQghpxGHBkTiHdXOLivrY9kny5Z/s+hj2uZq8nR1f2kGYmnwTCnsfYHmyiwC56rkxKvqpKisrnP5jr5WGwCk5b75coCkZEO4X19HR6enm9ceb6OrptembwhVFKGgAMAPe6NtM8dlbbEHfG39UTmxQN5lRM2Ng3cSvS7qfhySjFGHalU6jirA6U2hkNU7szb9VhV3FOJVbDHTjw7T2sT+q206uTwVIzPUTMjb3cVl1HE+GwuyUjH1B/d0C5Ul03mne57u5JKdjWN2ag0azH8Wq2ujYTDGdwzf9VnCL8xcfdxuVKzVJ2iMA8lkVxbMgNrpFw0yuVoSeXLdBcJg02v3T6Ab6qrZXASuw6FxJ7BMw382VPlYHF6Ie35U7A4jL0TyDyHInEt2i7iDbqshjGbG3/sijaWjzPL/AJpAu2RdPqqGSRZQhVsJJP0+qHQhSAuv0RZ2ocoSyhQFdpS8gtdPpshBa86Kofp9UP8AZ3R6Wyt3QjQajVRSTnynMhz2t5eqfNdaCSQk5f1RDXZAidfKhLdfon07apBuqByb+kWCZgygkOKXX6ItSUDJruCe1/LmslbKPdA/9pIuTJ29UDIba2RITt5XWQF6SWpJruSy6oHRPHUbFDsAU1/Yqh81tmptCErdwnVoCd0ydu6QF1kOc1rn5Jv9pK5GiQF1QySR3HskoHdunb6kKfpvsqHO3lQp27p27IehruPb2QkZtxdP8QT7lAtd0yc5QDZqYa2UBXFtkzd0raXSBsgZLfqkR0Ka191Qx1uxpF00RIZ+I2xBIT5YyScmt7XTHQkPtZA5JBJBFuyT44ZGgyAG+gQsjY0kBu5TGEZgQTob2vookJW5Q0MY2wHROX3c0ZTr17JhayWXLs4n5opOkZGA15snzW+HdIC6cmxQETY2SB1QF3mzn6og5paDfQolH83RCfypAj4kxFnX6Ipy5LpmS8vRF0+qBst0sjiN04dYE9Uszu6JZhJrkykkoyBYaag7oE+b2Qoi7Qeyfqk7ZLp9EQ4kDQWuaDcdUx5tgWMvc2CY6/CEQdbbfushnB8Zs9tnDQhLMEnB7vOdbnUpzlHoFgtNEhzFMi0uP3lfRLsBNzmRN3T9ChAvsoqRu6RHl02TJKEhJdfM5LMU+vskRdaDdPqkNyn82X2ambuoC6g9EkI/KnzBA4OoT31OiHMEswQInVOle4smbsgRNkv3roU4Hms7a10Dk2QjdF0+iZpczzIHJBPqv7pgWnRMfZCQS7y6WRISWazqldnRuqQcDsms4FFFpY9wlYEA9+iZmvl6otgge2mXLomGW5YLp+n0SUAdNUrAbJ0rGyoZ2Zo0T8xwtcJ7XFsyYtzBQIm5shEdv6y47FC5tlOGfh97oGAvppbuEuTqcqTGAeXYIr66KCMtAS85GgUjxmN0wdYWvqqIxzALO1UsFVU088c8crw+MhzTfYhR6lORc2WcteOcccouFiZjzh6Nwt9t/E+Axv8AGTzVri4Foc/ygdrL13g/7fMDx809Ni8Taasn9QiGjf8AZXy41tkQzMcHxvcxw2LTYhfjO6ew3a+victWPh5z78fT0+HpX0p79Pct+qombj5vu2jxfC8SjD6DEaecn4WPF1daxrvLILt6hfB+H4rimGVsVdRV0zJonZmnmHf3C9Y4O/lD49hkbKHiKD7wbJIBncQ0Mb7lfg+5/wCHvX9Hhz6XKNse+PSftfn931NPddWyazji+hazhbCcQc581MMx7Cy5nEeCn0zy6ikk02HZWOGvtH4T4jdmoK/IYzZ1j5GH5rrYZWVDc8E8cze7TdfhM9WenKcNmMxMe6Yqfs+nGUZRcS4OmxXifACGHmyQM+GTzBbdBx9SP8ldSyROG5AuujFJGRq1ZuIcJ4bWed0PKf8AmauarUkfD/EFKW1EVNUseLHOwXXB1/8AJ84DqHGegoRA+98tyRdX6zgvEYHOnw2o5uTZuax/vTUnFOPYG4RV1LzWjfODf+9erpuu6ro7/D7Msb9amY/oxnrw2fxxbxvi7+TxxPhUT6jBWnEQ6Q/gsFixq8+xbg3iLAad1Ri2HS07GENJcNblfYuG8c4LXvyTukpnfv2t+q1K7C8C4kpnxV1JS1kLwWkkZjr7r9t2/wDxD7j08Y4dREZxE+c+k1+nl+z523tWnO5w8nwZdqZoYPU1fVGIfyaOBZpubh7qqnLvhdIXBeUfaL9hfFfDtQKrAsKmrqN3lLYLPLLDckr972/257V3DfGiJnCZ9+VRH3t8vb23fqx5ev0eXugg5XMgmF+rXHVQq7inD+NYMwS4vhk9K1xygygNBKz8t1+t079XUY89OUZR8Ym4/Z4cscsJrKKGCb3AJ+QTucw+lrgRvcKMVNRRy86mkyO+W6kmr56sNdOGlw6garqyRkAGpATAtsoZZGW84JCUcjLaHRUTE9SgJ1Q8+B3lvr19kQ2BbsgQcE9wExuAmLyUoSi/bRLMFEDa9iiabneyAwMx90fMIGVxUTzkcOqclpOigkBukRIfS5VTC97rsnLD80rVcf8AXgrQs3d+RDmChzVZbfK2/wA1LHnPlcQfcIDJ/RLy20bZBd2xbZEx2W+l0BEADNcJg5M7VLp9EBb7FPYEZSVF1+qPf0oHsWaOToHEpAlAXm66IP8AaRF5PqQdfqgkBF7Jjm6tQt9SdAXwpm5kvT8kg5ASbQBNv6UyBJ27pNtfK7ZI5QdECO6ZK5PRDqSjIwbJr576WTAW0ARf/vIPDT4ykaTBVDN3AVuhpMYr2B78Xjjv0soWMe5wYG3up5KeeJmdrbNCy/rzUiwulgYec2KaT8xNkTRg8YEdRHGbfC0XKxWSPk0Ezh8lHPUvp35HTyH6o1Tblr6Wld/RcPaOznBZtS4Tv5jx5iq75CRpI53zUJmeN3kIlNBsV4CXSMt0b1VQGhMnLmtn91AKyXPkjGvchTSspsrCWEyO1NkVfpaqKkjeBDmc8WAsoBTTzSaNtdRtIDgSr1Zi0la0U8MYgZaxtv8AqgAYFitszZoWN7ko4KKCikElXi1Iy24us6ojp72qJpQzr5imw7D+HayrbFJNNY9TcBYYatZ4LEnCSkqw4Rixy9UNHTupHCoqOW1nUuV7+bOG0zCcOnJeexWSKOOB7hPPzH9ddFtQ1VSySpyM1jKYxwBzTBAXP6WRw4MxwE76iS/8FYgo6s5vD0z5COo6I0g5E4HnZC09t0rubHy2mxKKVtTTPyVNFNf2FwpqOnp8QlEb4ZmW1J6AIAbhXLi51TWWvs1rdSiDWTNAZAbN6nqtsYbQse1hMhadyRsmnw3B82Tnu/VFty88zxdkH4br2zbqtNRSlnOmfPIR8V9l18OG4U1+QXdfuVfiwPcCEZD8N9ERw+HYaypfriEg7Nab3V04ViLCfDszN+EnddjHgEDNTDy/7IUoojGQ5jQLIy5KKl4g5fLEO/W6IYDisrQ+eohD+oK6t0b3nVgTQ4OampAie97zsxmpT09UmYjzlzf3FlYDVPYT2ab2WrDy2saKanjBAtey2arhzEzCcmHVzj0Ahd/koMF4N4qrZoqNlFLDzXZQZoyLfMrE7MIi5lxy6nTjE5TnFR81WEEOBIsnZVS1U3hcNpJaqfoyIXv9V6/T/YHg1TTRfeeJ1QmLRzGx2IB9l2/DfAfDfC1NTx4fh8Tp4G5RUvbeQ/Mr5W7vGnCP9Pzl+T63216HThP4eJzy+lR9b+D5rhwP7QquURjh2oHuY1v4F9hnGvE0jK7FnMpKZ0mV7ZBZ4aOy+mHSFo3Qul03Xg2d63Z41jEQ/PdT7cdbu1zhqwjGfj6/1eY4Z/Jz4CoJ46mZ9RM+NwcfPYOseq9HpMKwbD2tZRYdAwMFmjLshlq2R9VEK8O9K+Zt6jburxMpl+U6vuHVddX4nZOVely0c5d6So3SgHzLL8dO6UsvYKGR8mc+crk8VtSSshj8rTc9lWkrpSLNZYKlELuu5TZbqKGSaaS93lO2M5LEkpOFiiafKO6ojlYALuKAC+wNvdSz5RGS8aBDE+Mxk5tgo0UbDZDldzWnoEhOwJOlc2zrAX7rDCaUtyHL2UEIaSGFUKzibB6V2SWqDn2uAwXB+qxqriJ9bOBRSOhaDo7Jqu2vp9u7zwxmXow07Nv8EW7QNLd3AD3Kgqq+npWF75WOA1s03Xm1TS11TWiorcWqZQ3QMa/K2yliY9jhlkeAdDqvrYdl2zETOUQ92PbNkxEzLWxLitlbUNZRUso5TrcxjrByycS+98SI5mMTRRt1yh1ipw3RLL819XR27RomMoi5+b6GrotWqeUR5s2PCp2E/wDxIuB3IBJ/VTRYby25PFTH5kK8xuqPKF7oxjH0h6oiI9FL7rzf9bn/AFTjCYA4OfNO63/eJVePYVh4tNVCR/RrNSsSr41mlaPu6hcO7nkKq6NtHTN7/wC1IqFZjOBUTHcycyPHwMuSVyNXiOKYjLzK2V+Xo1ugChETPyfqto16/iMVbQKehEbe7ib/AKLHljNU3JPJI9vYu0UmUD1FGNEEDI2MaGMFg1SAuBAIP0CJ9gLkaJhI7oD9VpLPd1v2ZB7lM8AEZdTZOM5sla+uXUIWY+26dznBuyFrXuJz3HZGG9SbooPpr0ROB0tv27o7DLZPYXzZVBE2wdbr2ROYCLnL7AI8pSvcKrZANARJspIzIg1ZRGHDW+6YgHVIAfl6p1oFr7IszbIkPX6qJdnteyRNtErn6JdPooIwb6NSykHKU2226IuzbLSmO9kyW5ukgNINFi8NANtk1v1SGp12QO9O03an03QnZFxFoUtL5gbpbEpWHZRA2dbQC103pcncdNEw2VCzBEL3v0SbqEyAiNLp0OZvpTi2l0DFyXT6pidUkDdPonQ5ilrmt1QMnvpZK2tk9tPZAx1CXX6pDYpkBWv1St7oS7VIOF0SiSPsnO6a2iIROtk4NkSA6Ky0e+mXoiQkEbpN3UCPZIbpk43QM4C1yNE7d0r2CYHXZGSBcToNk0j3NAsy9yjOYfNCXHMGZSSeqrRJmi1/ndFqQmA1QLUG/ulfdMTqP4p0BZnEBnQbKJ8Yl8jwbXRE5SPnZGRY23UAD0ebdP1+iQtpl6JHRWSCJ0UUWZznl7r66eykuCo4xZ7/ADXJN0Bk2BuLEGyRcG3O4CYSMcD+6UTjHls7qECbI17rB4JUigaI2+fIAe6TpOSwvjfcddeigmzBDcndAyVstsh0IRNie2W/iC5v5eiAx+9si/sqK8mYgtGUbFOsh9PdLcpFwBt7XSBstA3aBDr7JHX4tLojcfNANvcpm+oJE3S6oDJN90sxS3KH1ORkW241Ttfa+l0idLflQBrlkONfiTEaNIdr1CVgiv8ADlWmi1sMycG4TX0smQF0Dkw3TBut0r/CssnLkgbIXepJ3qW2hagoc1yl0PdNFIx188TmOHdBJn29k13EpeSyWXVECC4E+VOf3U7t0iPLcKKZJuqSSBJswS6/REDZAsxTg6JvLf1IXepARclr7pkwcgLMUi9oFybIT1KfyuGRzbgoGz2TCSwsAiNg7RqSBhbU21KOzrZkLdUWXQoHByn0py5DfMLIgQWjKoHBHp6p+gb1QWN8yWvugbXX2T9LpddUvlqFQxtbyuThzRYO6pe52TkAnMBogI32ah848rSn19kuv0UCHmPmTndA5jj7JxmsATeyA2kjoEI8x8zbFIFqWbVAxvdO0pXCdxDBe2iANeifM7qrEEVLUXDKuNrgdignpXwSiMyNeCNHNOigAGyV8xSDbWJTi3RUT01fVUgPhKiSEONyGvIv+i3eGvtG4s4VqHVWFYg8yOtrIS8D5XXN5eyZhuvH1PQdL1mE4b9cZRPrcN4bc9c3jNPo7h3+U5h1RLDRY7hkx5uVr52ANDD1PyXqeDcXcOcQR8zB8YinN7ZScpv9V8OOboVZoMWxXD2GKgr5oGONy1jrAlfge6f4c9Jui+gznCfhPnH931dPds8fLbFvvEtt+0aT8kM0EEzOVkBB3BXyhwB9tHFfC1QafFMRlrKOTV/Oc55b8l7Nwr9u/CXEcslNVMdQviAJkfdrD8rr+f8Ad/ZLuPac55YTnhH/AFRHk+po67Tvjymp+Dq63grDJ7mNj4Ceo2WI/h3HcKl52F1T3gf6l1iu4pqllXEJ6GqjnhOocx2a6N4hOs0eV3UtK/N+j2OPpOO8coCIcYpHSW+JwLSuhoONMExB4Z4jkvPR40/VWpsOoqxnLnibOw9Dof1XPV/ANDkzUE5p3/lft+qg2sZwGgxqlcKjB8Prrm7eYwXv815VXfybcGxKtmqo3yUT5XZjGZLsHyK6Sai4p4df+DLK5nR8fmCt4f8AaFiMLWx4lBHVD8zQGn/ivo9H3PrO3zM9LsnC/hLGzRr3fxxEvBeNPsF4mwGWoq6SmfUUTNWZGue5y4TFOCuI8JgFXPhk3hi3OZSzKAF9uUPFeB4lpHVBjxuyTRSYrgGC43SPosRw5ksTxZwaLAj5r9l0H+IfX9NhGvqMI2V7/Sa/5n5vnbe06s5vCafA5Zolkj/KvpzFv5LvD1VUSVOG41U07XuLuS5tw32BXmn2gfYdxDwlE2toaKeqpmC0rmHOb30sAv6B0Ptp2nr9uOjDOYyy+MVF/C3ydvbt+rGcpjyh5eIowb5AldrBly2srxwzFCLfdlQD25TlVMOpa46jQgr9Rhtw2fwTE/SXinGY9UXMafK5LT2RCNiXLb0W2AtDR00CLS+jUWXKUiG9EAu3SBCI2shyNKKV7HN0TXB2Ra5chNwo+XlOZFSBxBT5hfTRAn//AHlfQGSCmJb0QA65UmvN1RINbpXuLaXQ3CQN0DpA2THTzFODbzB5QPmHv9UtDskXX902u6BWKfM70p7t6pXCBxukXJgXBONUDaeyFE7Kh6fVAkRAt60PX6pEAjKUBJxugEQF8jyD2tol+INDZAWeyQLSNN0snxX+iWXVGSL3dgk0myRIv6dU4KDx3mZGF8NPJmCgFVXTTZJY5Mp6WRkYoXmSNn8UVVidU2n5U3r66rL+vSPwfN2kEfzVZuH08spkqXh2Q2ACyZakl3nmI+SuQ11DFGG82ywW1GxQBukdmj3VfLM7zQxhw91z9TLiL6syUQe9nbdb2DzY1LEGnD3tA3cY9EaXZqOGnpG/hl8zxfTos0CSIfiRnN2C6ClwqerJmqpgzuQLn+9BWCloHhlPUmSX3ViRiGKryiR8PKafzHVRWIeCXXXcx0VJLCwzw5n2udVSxPDZ5gIqGCONg633WhzTJpYz5KXmLTp8PrKyITuw+zO+VaWEeFwZjhiM9IXvNxmKszcUUJf4anAcT/q9lhm1P7urpoS3kuYD3UkGCU0cYMgvJ1V2OukyiSSCQN+SGplr6qIx0dFKGn4yNULVm4ZWyOELJ4YYfyAXKuRYdPE3LSyadTbdR0sT8MiE1VzC7rfVNV8RQU7Q6PmD2yrbSwyOqvaWTVC6j5c+fmyH/JRRYlU1bRJTRtcXeyUlLjdTflxlrztcGwRJmI9Ur5I3u8KAS89AFYDMPihySQNPueiqRYFjLJWzT1Ye0fC1pF103DHBGIcVYg2jjjcyNpBlkI9DT1WM9mOvGcspqIefd1Grp9c7duURjHrLk3eFc/m0g5TgfUVbp62tDgyOWWQ72YLleus+wOGMaYu3/wBNdpw59nvD/DohqIaOOSrjYWOlI9V9zZfN2936fDG8JuX5jrPbPt2jXy0zOeXwiJj72+fs2NzxZ46erv0/CK6/gj7Osb4op31OKTS0dLa0bsnme4HUEFe6cuEbxR/7oUjXRsFmlrR2Asvn7u9bM8a141Pxfmur9uuo36pw6fDhl8bv/h5gPsMw8Nt99z/+mF0XCH2b4NwnK+qaTVVJPkle0AsFtQLLqn1ULDbOqk2ItBswELxbOv6jbjOGeXk+B1Pf+5dXqnTu2zOM+vpH9IaPpFwhD2HchZP3hLJfLdB4uYnVx/ReKnxaa8lTBGLl4UBrmdNVnWBNyi5Vh5VBNJWzvv0UTZZHeom6EtckxuiLBpA4hBH5NbXROBv5UbdAgTAM5OXdMfV9U19S5CXtB85AVKGDZEHKrHOx0mQX+aCoqoYmOe+YNIUFxzweqKOYDYXXLv4vwYSGGGodI8GxsgqMcq+RmpMrQ7QgnUe69GHSb85iIxnz+Tvj023KYiMZdHi9R/8AD5mC2Y9CbLkZuJo6GYU88jxm6MbmBVUVFY6USVVXJJ3HdE58ZOdkTc/U2X1NPZsuX+rPl8n0tPbcuX+pPl8l5uM1FaMkZdTs/O4a/os2rZLLVF02L1LmjQMa6wPujL5HnVPyr6r6GjtejRlyiL+r16+h06p5Vf1A58DsmSJpLNAS3UJxnJ2UoZ7IsoHrNh3JX0MdeOMVjFPXFYxUI8pRjMBbKqj8aweF5ZJiMYI6LKrOM6YeSgpzMfzHRdKW3RWO1lWnxGipAfE1MbXDouNqsX4gr9JK50cf5YxYKo2nB1ke9z++eykwjo6vjGEEw0Mb5D8rf4LDrMTxXEdKqocG/lboFGI7elPkcrQjjibEQQ4D3O6ONrBsExapGCwWg3kAOm6bdP5eqRIGlkA2J6Jau6jREfzJ7tAsUDPASDRfNdIgW3TkW8lvqi5HA0SOycA9Ew1UQF9SOyfbzOciQ3+E6qgifypHN8Iumtb5J7aIFc7EJwf1Tjc+6XX6KArFBfW6K7gQns0lBGiu26Q1ciBDXFrWjfqqHJsle/wpaXTea6iE7QJgdB5LeybzXyoxmtq/Miok1he6PKEKoQHQJ9d07mWTOdYalAeg1O6Vzrp7lDbzapxugTTe7hqkfV6Ug6xskXaokH1vobJ27e6bcodj5UUbm2SGyYk5bF18qQ9IyouJk+ezb2ueyHykdSU2xRCBOmYJ7OO6LO4HyhNmde7giWYagOS810gddkzjZFLromTt3SG6B/U5OmdsmGY7IEN0yfulpb36oGO49kkkztkDnM8gN0skkkgSR2skbDVLXqgQHQJJJxugThYpZj3TJIyYl/5v4IgbFN1+iViN0BZv1TalpZ36pkreyrRms5Yy5ifcp0k2axUCOpzFOB0CScmwsqGI6FIbIbk2u69tk4OuuyBz7JE3GVM03bm2TWLzfOQFAQFkLoopMwN9BfRGcvwm6i5d5C65CgTYo4r2BJI11SDWyRmOxHROB5DqdD1SYdFQ4uyIse4FvUkKAT07iYywKa7ZWkFp36qPw1OSHOjOnS6okPLhaAG2GykJ2seiAtu3LsmbHl2vZQGCbWNvokTpZIdUx3CAxlLUhoQUjt5eyBjgTY79kgFcA5XI2uuLIX7ZQ0fNM3dGTuTuNuiRCZ2vS6LJx3zapm7J02bVWkECL+ZPcZbhRi19U+wUaE12U5k5IeWuay3dCAHepyRB2BQMS8Pt8N0WxUbTNtJbTsnzFAQe9l2N2O6cOA8xUdntffPdpGxGyIPs4O7ICGvTqhLkrm4snQD1+qcvcfUluSmB1VCAuj1BQ5gl5uiBbepqTtkswTqBJtAEjskNQgZ26XT6pkfluS3r0QMRcJdMpQp/i8yAk1tbpXsM3VK7bID8gboUFvNsmNrWKduyBhlGwsEepKjtYFFrl0QIuT5rpsuiTdkDdfqj6/VCWpWu3KUEjTbUdU9ydFGh1uFkSgAN9V0zSQ+zNz0QA312R7EOWg4zEnOLIr+XKExN7uKQ2uoGvpZJpHZC5MzoqkJvK4oBuQRr01S0sns0a21RQ7G6kBBJcmGZvq1CEE32UBFuqQ/KnBufN1Qi43H6oInQgnOzyu7hJj54x52835KUmyYbqhNlEozi/wAiE4OlkjbXKkCDoVkFd1zqguQQi+Ipi0ZbudZAgSb3T9PolY5bhMA8HV2ioPW9yhdHGT7pFyTd7oN2l4840w9rWUHEVZThjQ1uRw2C9k4M/lHRNwttLxkwzSx2aZmAl7x3K+fg4IXL893T2W7Z3TV4ezXGPnd4xET/AE/q9WnrN2ibib+r7Z4b424T4spIa3CMVa0Si7Yp3Bj/ANCugJm/clYvhPDMTrcLqGVdFUujkj9JvoPouswH7aOPMCxCKb74lnp2OvJA8Czh2X856/8Aw46rDOcuj2Rlj7onyn6fD9fJ9PV3fCYrZFS+ww6n9mlUazA6CsaOfQQSdntNivMuD/5QvDPEQZBxFA3Dat8mQBurXdl6vSTUmIRCowqsimj7sK/C9b27qu27PC6rCcZ+fv8ApPpP6Psad2G6OWE25Ou4CD3mTDqoWP8AVv0t9brNfVcW8O2h/H5TNsxLgV6G4OYfx49O7Qhdy3WYJQR+STZeFtylB9ocT2htfSlp7tN/8F0tFjtBiVOX0FbHJfdp/wAQszEeGMIrnGR+Hcl5+KDZc5VcC4lTv5+HTCdv5eZleFYlXWvwWixVxZiOF0pY744xYj9F5RxZ/Jnw3E691fgGKyQNk3psgFndTmPddFR8RY1gcwjmjky/lkj3+q6qi4+wifSqLoHdnEWX0O3916vtWydnR5zjM+U//qfJy26de+OOyLfLf2g/ZHxBwO2OpFJU1NI+wMrWXs7tYLh2QVO5pZ//AEyvvunqaTEIc8L454zu2+Zv1WbifBvDWLxujnwuFhcCC6JjQV+56D/EfqdGmNfVaueUf9V1f6U+Xt7PhnleGVR8HwmZGbApN1X0NxH/ACWHVNbLX8PY3HFTvN2wSAXB6rybjn7N8f4EqWx4hSyvpn2AqAy7c3a4X9C7b7Vdr7pnjq07Pz5R6Tcfp8L/AFfK3dDv0ROWUeTkXhtstkOWxRljj6GvJ9wobyZ9LH5r9DExPo8Q3Xtom1IuiGbUOSAWmgHbM1OD7XSylBl81w4hA+YIi4gA5SfkhzBIOI8ubRBIhz23Qu2SOyEQJup8yWxQ5gi6/VAr2GZydj+yYXbqLEdkI39KokLtUN/yom2v59knBt/JsssmzfNOC5DlKa7ujloSE/mTjL0UYzH1IgbGyNC0Gjki2/sO6Y6jNmCYudbKEDlpYdDdPnFvO7VBc2909uvVAjunaUyFqJaQuSQ2PZJzdEV4691Y5+emqo2sKp1FLVukOcc0nctW3h2HYaYQKiST+y0K/S1dJRZo6ShJI3cVl/XHOQcNyV0QZFhbhfd79lJJwzDh7/DvAc8dtitjEeIKh4EdOeWBvlTvjnrKSPmMkDn7myzSoYKhlKwMdLTwn91uqmnrue0MpaoHN1Ol1BNgcMLCI/NM/cuGymo8IjZD+M+QSd7rXFR1FNXvgP48Q+SCmw2rbEHZYj8yiOHsjPk5r/kVcpaiGOMQyxyZh7KUIXYmyhjtK3mP/K0rHrsUNTKXvqJIh0a3ot77qp6upM00d29l0/C/2bT8UVbvu+mjjY0EmV7btv2usZ5Y68eWc1Dzb+o1dLrnbuy44x75ebUmHUlY81DTVTuG97hb2G4fHRtEzKVsVvSDqV6q77C+IH+T7wpWt7NBCuYP9h9bBWwsxKrifRtd5wxxzW9l5Z7h00RfOHyM/aXtWGM5eNE18HljY5nvMhJ9h0XecO/ZjxPxHhpqXVDKMPP4ZlYSSO4svaIOGMBgiZEzC6YtY0NuYwSbBaUMMULBFEwNa0WDQLABfM396yyxrTFT934/rvbnZs18ejw4zfrNT+zw1v8AJ6x7m8yfiSld7cpytN/k8uPnqMTp5nncujcvaXG26hfWQM9T15P826r+b9ofJ/8AGHdv9yP+2P7OI4Z+x/AcIZHPXx8+rikztewkMt0BBXbHDaIb0FN/6Tf8lBJi0IPlzKN+KTON2Lx7eq278uWc2+H1nc+r6/Z4m/OZn+n0j3LXgKAf9Qpv/Sb/AJImMo6Vx5MUMROhLGAX/RZj6uok3UZzn1uP6rjOUz6y807M5iplrurYejwVC7ENSA5UALkAKYRC93OJKywKSrlLt1E+SR5vnKRj817pwOyEGEZBukW65sqktfTNZPltpdFREOtbZC0SA5cot3UztrAhMHNIOuyARG++uynOUbqrz2f6wKN1TNI7lwRmQ9bdFbFovYgMsY+JVX09cxhmlLY29czhoqBkpS+0+NUbXHZvM1Vxwyy/hi1jDLL0hqy1MY2KrS1R+ArIxDEBRROko8s8mzbOubrHirsfqPPWTQRZvhaP8V7dHbd++OWMfd6dXRbd0XEfd0slc6Jhe6UAWue6zZ+IqfNy4GNll6Ziqj88jGiZ+YjXMNL/ADQBoj9K+jq7NeP+rNT8nu1drvH/AFJ8/khrq/iesqRyHw0kQ0Jvcu9wndDVzRubVVkk2be5sFPmd2Szr6eroNGqIiMY8ve9+HS6dcREY+ioyhji0jAZ7gKwIh8RJUobm+FVqvEcOomE1NVG09r6r1PQk5bU/KaPhXPz8a0IdlpKR8x6E6LMquJ8eqWuZG8Qtf2AuFR2ckkELM8sscbR1c6yyazivC6fSmzVTv8AuxouSbHNNrUVT5Xd3OCkbC1mzh+q0y1ZuLq+RmSlpBF8xdZc9Zi9c/PV1TyOwFgntbqP1SzBaaRCljHRG1rQkQSd0svu5QGCOyYkAlPbRLLfQoAzflT38vqF0+xy6JAX9SrJv3mpA2HmTkOJ7BD8VundGjNdcXbG/wCuid1+qWdwIBBIPZEQ2+iJBru+JO8+X0bdUnBodlYf1Tm98ma6EhBb+VJORZPcIoc1hqkmt5rIspQN5g7XZFbXMhLdE42t0QP7JZfa6azr6iyK52QOIySbDYXKBxsddu6ISWBdmvqla4sdRugEOBFxqnBsDZNYfC2wTgAaIDzX+GyEg9Eh7JG5OqLRW9rJ7aapX1TuPsjNm09kfp6boB5RmSuS4F2yKkvYJ81xb6obG+yYluygRkedE9gdxdCnaPiSQybzdN0+zr7pEg+kWKobf1b9U3xD5p/S5LL2RII+olOkkikDr9Et/MkcwCbrlQIHKUruJzJWbZLYIFbXdNc90+bW1k5GiJYdynIFkIJSQokra7pybpWQsIP6pDTT+9IjYpr2bdFEn6JhqLpyRc/LZEkxbrula7k2e/SyQJDgUU7r3skkTv8ANO3e52VCOo72Qku9STiBsnUZMDvcIhqmS0AQJJI9ShablA/xJ2jzHTdCPUW3905Nh7oHTXdfLbpdPc20TXAF7FVo57jZCT5m7JyTe3RNZ99BfX6oHvrYdE13fkSscpy72TNBAJzboCAJ+ScmyFmcNs74UiRsb6dUDNDywZ3ea5umiLshvpcm6kBumcdLqBgABcAWPZCBJzA5+jR/FEzZRT3fblyZSEEsjb+S5CrCWVspYxgLXjY9FZBzAO6qvLFJzee3cCyCZhNszhY9k0TZeYXveLdAEopeY1peDcjqFJoFQ5OqTQG3y31PVLyWLn7AKGSaZrhGGgXOmiCVzCTbOL90smYefXMkLkeq2ifcb2UCjGS4J+SRAOvXa6SWZjSQ8j6oHtonzOtdo+SY/T6Jjoge+lkIADr5rJxskWgtQHvqonuBuxzrIw6xQuDHk2buiUAPy+XdSg5rA7X3QZBf0pwDfTZCkhMJP4N7A21TZghGVp9PW6e+t0U+YIQCPUjuweu9r9ENtbIAa2QPc5z7tOw7KUBuXzO+iVwPK1Cd1QSY7JtQUsxQPbYpaeyYk6AJO3UCdupG+gH6IXHQC+iTEAgXNktfdMdU4N2/TVAibpAkahLp9U/T6IEGpOypaEJWPxIGJ0unIOh7oU5GzroGsSQE5HZK2tknboEdW2KQ7pk+wzFAuuqWUpZilcXQGSS/K/ZKzU0mQuzMbom6fRA2Yp9fmmFhu1IgOPlbb2CAnDZJrTluAmO9kjfoiwWfRK+ltU2UogdEQzdSj0Cj1unzBAV9bJg7VOMuU90yJB9QSHJ7hCTqnAuC7sinB8xRWBHk36oWiP49PdIuAOh1WWTkFrk5cSBdNmzFI7o0RN0hbLrunB0207pkWDu3Sbuk0XNk2WxsqhX1uiBuBmS8o3QjN8KAmjzad0s2qe8g0LemhQm+56oE7dNdxOUNuU3T6JEaZtlA+vVtj2SJIRGxGbr3Q/NAwJA3skd7p0kZJrrODg4gjUEHULfwLjzinhpsjcIxJ7Oa4OJd5jf6rARaLzdT0mjrMPD6jCMsfhMW1hsy1zeE1L2ngH+UJxBFWGk4rqYqiGU2ZI5gaGfMr13BftX4Bx2eGjGO0zKyY5RGb6n5r45cbpoZZIXtlhe5j2m7XNNiCvxndP8AD/t/XbZ26ZnXNemNcb+Nf1p9PT3Tdqx45ef19X301lQ0ZoJuY3uDcIM7ATz4CH/mabL5GwT7cOO+HoY6fDaqIRxtDS18ea/zXr/A/wDKAwrGqFv854oaeoZZssuYMBPcN7L+b9z9je6dr1+NnjGWN1+Xzn60+rp7hp3zxian5vV5KeOpaWSwQ1MR6O3WBiXBOF1j89K+SkkPQ6haWEcRcM8Rgv4exqCpePVkdq1aJFS02kYJB7r8xnrz1Zcc4qfm98ZRlFw89m4c4qwB/i8OeTbd0Dhf9FJS/aJjlM4x1sDZnDfM0teu9Y+A+h8kLx7KtV4Vh+InNiOGxyj/AFgFiVyotXwnjXDMSAY+9NIekrhb9VrzR01bGYquljmYNS17A5p/VcbWcBwPucLrstto5gT/ABWcW8Y8PxiEc1sfSw5gXTHKcZuGXUYr9n3B2LQSQHBoKd0jS1zo2AGx9rLxPiL+S1i7a6WXhvHaZ1K43ZFIDnA9yV6rhPH0gyw4rBm7ytIH8F09NjmD1jc9LXwuP5XHKV9jtntB1/aMssul2VfrfnH2lw3dLq6iIjOPR8c8d/Z1jPALad+MSNIqHlgs0gCwv1XJCRrtngr76raLDsTjbDiNBT1TWm7RLGH2XL4r9kvAuLwyx/dIpXStLS+GzbX7DKv3fbf8SJ1accOt1znn78oqPf8ACvdD5m7tF5TOuaj4Pi/K4m4SIf7L2XH/AOTDxbR1M0mBYjT1NIHfhtdmz29159xT9nvEvBrGS4tSyGN28jWWa35r990ntJ2rrs8dejdE5Zeke/6fV83Z0e/XE5ZY+UOZLLISDbZSMLHC4fdNex8jrFfceZF08qZ26lIuLndRkFVgOvsln91IGjqmLAjYRe6a9k+UpiOiBNlf3BCLNf0qOxGye9jojCcSPCQOurQoy67kTcqjYs2vkGnZMclhkvfqo25gd0/96jmK37yZNubtSzOutOhOJRNJKHMEswQSZX9rhNsdUGd/5k+/qQHexvbRC51zukHEIc9na/ojNPLaHA8Ym/HqK8xQDcgWKlxKlp6GIBlTJLn/ADHZdtgfBPEHE7pG4fCXiMXcScoXYcL/AGH1U2Itk4lp8sMdnesOz+xXk3dXp0RPPKLj3e9/S+s7x0XQxl42yInH3X5/Z4phMdPEWyTTCR43AXo3DnCeOcUQyS4XTlrI7eaUZQfl3Xt0f2YcBxejhukH+yVv0OHUWFUsdFh0DYYYxZjG7AL5W7vUTjWnHz+b8X13t1jOqujwmMvjlVftLyDhD7IcTp8VbXcUMhdHDZzI2uDg89Q72XozeDOF374BSf8ApreI6FRl7W7r5W/rd2/Llll9vJ+R67vnXdw2eLszmPlFxDJ/mXwp1wGkH+wnHBPCW7sBpD82LTdVxtF7qM4hH8JXHxtn80/d4p6/qv8Acy+8/wB1D+Z3Cw/+h0v+4tHD8Ow/C4jBh9LHBGTctYLC6gOIB2wQOq5CMoFljLbnlFZTMuezq9+3HjszmY+czLTdKxvqKifVwtOpWaXOduUJ0C528687EmD0AlV5a6aQWZoocrinay3RW0gD5ZSLveVFluc5691aewW2QMaL2AUVGIrlTMisLWRMGU5moeac+W6rSQRaoT6rZLpc26Eu+LMgkj03ai16KESsyEmQKu6uAJYw69kFx0ecIA0Nss99XUE5gbKrJWgnz1mvZtlIiZ9FiPg23PaNyqk9cA7IzVZlTWeDi5z3HcDK54uVyeJ8b4u2WWlpMHaGtdZsufUrvo6bb1Ezjrh11aNm6awh3Ek88jSGFVZal8By1FQxp7FwC4qDiLiOVnh5aVkbT1Ljc/VVK3DZsVqzPVlzRlAytf2Xv09o3Z5Vt8o+72au27Msq2eUO0nxeihgkk8RDNI1pLIw8auXNQcecbyuEcVBHG391oTUWFUVGLx0oDh1JutCLTYAfJfT0dq06onn+Z79PbtWuJ5+apV02JY3HfFaqYF3qa126KkwHCqS3JpTmHxOdcq7c+yJoLl9DXqw048dcVD2a9eOuOOEVBuW1vobZDmde2VScs97fNZ9dj+D4bYSVImkPws1K7RLa8xxdupGxvO65ao41lcy1DRBl/iesaoxPFa0nxVY7KejdApLLs6vHMLovLNVDN+VupWTVcZQeaKgp5ZT0c4WXNCGPspIm5dQtwsSsVGMY5WXbNMWtPRosFUNMJLGTmF3clWQ72Ql5CqWYRxxvaQ3QBP5fhagdnvdltd7or2HpupQRy/Cnfq0C102pRZfzKhgHgC6bW/mT3aDojRoKQNhomdskBogMG6QOpQjPbyNuUzX6ebfqgc6pfiX6WQk301+acbBvRBJc90tQUyAvcbjKfmss2JuqdCGotAFpoKK1iha5ttdCnOo3QOdR7pBtwkxthbdEW2CJIdgl5na9Uhl6oXHtsijO3RAhJJGlrp2Zi0OLbXQEbWtm1TN/tJ8pToEjZZxtcC290LhbS2qXT5oWbpdNdxS/uT3v0QM1uUIr2aW21KVySG9EvKdUCs7qn6IvKTmt9EPU2QNpYg9Uh6UiNilfSyB/MLE3shTn3cSmQFl90iLhM5yTRYeVEstfdIa2SOew7HokN0UfW7m3TG40d11TDNrdNoDfNsiQfW/kdYosxTfEUxy9kUvM4Jun1RWb+dAgcm6cZbJrCxzD5JgLbIEQfqkMwFi69uqQuUkD5imSTi19UDJ9ymGb4kkCSB11HzCSYAX90BOPsB7BMhdqntcKlloTqE6Z2yfQhAibJMdrtdNuP7kibHMEDnUEJnbJC/+STrHdRknbpxsL6pgxuYuc467BNn00Fz2VCBvcJm+pLU6bFPbKcrkaP0JamTggAtTRRuyuNxugLT2TE5Ql6moS1rgA9twNkCOpB7JvxC7yD6p2kgnNa3RPchA7/TYO1KR122Q5tUbG9UCb5UztrpX1tl0KYA3Jve6BwOnsmLdMuY7IXZi4exSAsT5jqgQLLkZifkENVlbFmBDTfcomXA8xue6dwa9urQfmoE0Pjjax5BIGpCRzHZLRgJPQJRyxytzRuuLqgtwCd05F1XfVsEvJyOv36KVok9iECaQBk1PcEKUctxY6Rl2g62Qj3KZ7vK7JY2USTO0Tt2UcEskgLpGj2Uvw5kUlHJDHKbOvc9bortLsoSEeV18xQJreWAy5NkiWg9USATxyFzYyCWHVAYHQJKKQvuHvj8ncFSNF2j3QSwtgc/8Z5YLbhQOHmuEeRoSAFjmQMM3xKU5Mg8nmHxd0IBGpPyTE6IHLH5M+XTumGYbKOQGVvKEjgNyFINB/ZQH5rKJkjzIWOZp0KLMU5eAQGoE3ZMd1YpZKQZ/FQudf02NrKB4sgR2CWwSd6kT8lgBqgjT+Xr9EW9moEBW0vZNrsn+FMO6oWxsmSdmDr90lAkWbRM0Zzonbl6oGB/RJxuU4GV56JyGoAS32KSSB8xT+W46oNb2T3APugd3qPZI6hIatzFLrogZGQLZmoEJkY31PDCdgUEmnT6pxsgv5bh/0S1VBpsrt0wKWc7KQyR9STd0spTbdEEpGqG+U26Ji5JrbqNGbny3ISPtukHPZo15t1BSLMzT7Kh2n4U23lumubnMEs2cX2PZAZva6Zw+JCL9dkiezkBDMSicGXsUEbg6/cIiGH1Eg90BAgNslodUD819EQyi13elQPYICDezSPmnc7XMw2SvYZi3dVIk7AD59QmIOozXT73TncLJBEXsnA6JC3VIus9uUaIWWvpRggtQONn+yfL8WbdVSIsdU5Iym7kxv30TeYWIaoGbIA4h4dYjQ9E+5SDj2smLkBFj7ZkwOqAvN8xKMai4VDtsXBt0je3lSIH1TWPZQOD3Tj8wQpA2NrIGNwnDmO0cNUif1QgeZEpr8O8R4nwxiDK/DJnMLTdzL2D/AJr0rBf5SnGVFPDDiFNTzUod58wJeB7FeQJa+6+R3D2f7b3PPxOq1RllVX73p1dVu0xWGVQ+w+Evtg4I4thdLJiDaGZjspbUODc31K7eJnPibPSSslifq17HZmlfAbm32K7vgX7YOJuBqTwNLL4mnbYRRSu8rR2F1/Ou8f4c568Z2dtz5Tf8M1Hl9ff+z6nT92iZrdFfN9hnN6p4QT3GiTXRM0hnPsx2y8V4L/lJxY5XDDuJcMZTcywjkh2J9wdl61h+NcN4yA7DcTppSdmB4zfov593DtHWdr3eD1WExl6/Hy+seT6urqNe/HlhPkOuwDB8QGeuw4Nf+aM5SuZxLgKdrTNhdW14/I/yn+9dg5kjdpCfmmuwjMWuD/zAr5zpyefsxfizhZ4hqIZTEz4ZwXM/VbWGfaJRTsDK2GSB/cPzMP8AkuqdG+RhZIyKojdu12t1h4hwdw/WZvw5KJ5/INAorcosUpqyITUlXFKOuV2qWIU1Fi9K6ixSjhqqeT1xyt0K89reBMfw3+kUMrZmjZ0Zsf70FNxjxBhNoMQhdN/44sf1VxznCYyxmphqYifKWtiP2K/Z/XO5lNgsVI4/kbcLx7jT+THxVT1zqrhaWnrKZ3mIklDC09gOy9qw37QMNq3hldG6jPfNmYukhqZKiET0FXHMx2xadF9/tftL3Lte2dmrZM3FVlcx/V493R6d8VlH2fEPEXBPE/C8scGLYfIHSNLhkaXAAd7bLCDxaxFl97TudVRvhxfC454Xgh2ZgOi4DEfsJ+zfEHPkosOjo5X6nMRZfue3f4k4xr49fqmcvjj7/wBJny/d8zd2ebvVl5fN8jOLCdHJ8pXqnGH8nLjnB6+R+B0jK+jd5mvZIGuF+mX2XnmOcP4xwxO2lxekMMjhmFxfT5r+gdB3zt/ceMdPtiZyi6vz+3q+Vs6bbpvnj6M++l1HbWyl5jCNd0I1OVfWcQ8vMLsTWOoA+SMEx6dExNytIiubhhbb3RI0OS5WVMXE/EkboxGEJ091pogbFK+t7JaHZMcwFkEwcCOyjeHg+YaIQXIszgQgb+9Nr1RnXXqgQE1MDb3Tgtt5k3lt7ozL3vC8NwzAqRtFh8TYYm7Dr+qsOrYWf1gWBNK8nK95shbcm2Ylfgcspym59X5HZsz25TnnNzPvbTsUhYbb/JRnFT0Ys8x2sjDVzZSSYhUEk3sqj6qofJl5hsjmNjlUTBqCtCw0E9SpANLJQjS5UhsOqjQcumyfKEWoKV2keUKBk4AumvZOHMBvmugIBIixSdURxi91UkrrnQ3RmFjP8KWZo3KzWyVErjZpHzUclNPmvLNZvzVbaD6uJhtnAVY10IcSXLFxjF6DBoGySZpcxtYFZB44onRllNhMzn20LrHVerV0e/djy143Dvr6bbtjlhFw6yStdNpG4WUU87o4HTVFU2KJouSei89fXYtVyvqKmbw7XC5yktUMUhcXGaKpqgdjmcG3X0seyT5cs/2e7HtM+V5Ori4g8TM5tHRSTRg2zg6KKo4xqaKUxjCbu/MXrMoq3GJdBSNiB9Iy2ACuOoqis/Er525+zWr6Eds6b+X95e2Og0fBHUcbYnNC5joWQ32dvZc3LjeJsmLaNoLnG4dkuSugZw5hcbw9z53W6GTRXYIaKnZy4II2j5L06em1dPExrj1d9WjXpisIcoKTibEnh8jXR33c86BX6Lhyop5DJU1hqJHdOy3y/MPKhyvOy78Ih2pV8JL7KWGB7PUnmmfDa0D5HHZrRuqeI4pjmH0xq2YFK2MfHJoFeI0jEopK3DqNmerroI/9u64WsxzGcVeTNVujafhZoFUbRhpD5Hl3zV8Nl2M3GGFscWQCWYt7Nssuq4vxeZ16SCKJnyusnLY3CNvlCvFIkpq3GKsnxFfJl/KComwBvT6qQi6QDvhWlg4jaG3sk7dGA70pi1ROJaOGUbpml17FhCcN0Dwnud1ViCbukC7MTZK9zbKU7W6o0WRycRgi+ZNzAHWL/onNt9wjJn6+VoCLMUBIv5dfZPm1Ro6YPTE3ThiBPNyNUxOUJ2gflSuEA2JIdnIt0CPcpG1tEzdkDnKDYJrfE5J2qVrAnUoF5uiJu6ItOTPcAXTIAGfN5ntsia0yE21ASIFkrC2jjcnZAso6pOOU7InZgcrQmRLEC7cIXOcRnbclMXOtluLdULCfT/BCia8n2T+U3voE/mBy2RW0RqwWA0AST5QllCIYm4sESbKE/wDegVnE5uiMDXMhbJlv5boQTbayAnDLvsna1p6oTzHG77fROTkGrf0QORr6kKcAggFts2qRtdA+UJdfomym9uqWYoGT9PqhcbBODpdA99gkARe6cjYgXskD5c2uvQoGBDk9vypmhuth7p76FA5J07ocpTkm+hSI13ugWYIb62ReXNZNdt7dUCaLDr9Uh6QlbS5KZA/XRFv6UOl7dExBH1QEQS2++tkx8n0SDyRYi1imQPd1vLumF76pE6+6dGQnMPSnFw3XdK42O6bMEaOO6Y6lPr7JvKN0DpswTB5NtEkBIfNq1JO3ZWkuyuLHMkNTlCYgEi4uEjlvoiiTO2TAW8qJD1MevyUccVvW+5UtvynVBfKM2pQFpmF9LpmHX/BNv2v7p72Nx10CBybuuEj6i86XTBhABI0RIBOoI6JABlmNbokNTp9U97aZUDZLHNmKTtLe+ycnRBIBoQbE6aoHAs26TcwGu6VvLl+qQ1tc3BQIhpsmysvnJsU9iHeycEONnICFr2Iuh+Ita7ZOLNdmc6wS1Du3ugcaXzeZA8eXQkXSzEOF9dUHKOYljib7IJAbWa7foe6E2BzZkor5de6eR7GNJdoOqBgOYw36pNaynY95Ol7omOu0PY4Frx0SOvayCIywyloZMLv6IwDENCgbFHG6zGNZdHG7mSvhykNZoboHMnMzNZrbdFf5XUbaWGOUTMvcHXXdGW+YKJJ2DS3VPmuLApmpPY3KQw2PQoSFzTfMDZSakpgXMe19xcb+6lLxISXAA+yyqK9vKg5TA90jWDM5GNymLsq0BfI1rTm7bd08RvGDtdItvqisNTpb2QEDdMdT7JN3SO6yHJ82+iY5T6UI/eHRLdtsq0CAYDn+JI7piTkvbVLXogdrrOBte3RKRzLukkbbrp0QuLhbypZbjUXQO0tIzAghFGfNsPqooo2QjK1lrqWLLfXRA8mVpy6363QWy6dTqiEjXAjsULs5c1zRfJv8kDnMR59k7S2190xOhd03+iF5s3Mxl9FQROluiX9yASZhdODogI/w6Ib62TnZNa90Dl3mDtkidUxtsU3QWQF0HcpgTc7oAHNJ85IT3ta6gmT9fqowNb3Tu8pHW/ZSAyWhPmTE2F0zPM0Pb1VBH906JXaN0r5joLdEjqLKhgeo2chdG2S2dt7IvYJje3lUCAy6N0RJgdNkr66qhwdPdOTsgDtTrokTceVRlLoWi24THYh3VMN07nWAdq4oFYWyphfonJv5kIzAosCOW99SOqYAOcDG42smdY3F9UgDZEDqbhJjSGljjfXQow23mcmc4A3tof4I0TPTldqbWQ2aH6b9lITGLEvsQdNEx/OgW3TVImwzOTg3SGptmb9Sga7SjdHGW3JP0TP0ORzdQmPRAxGmWwKNjX20IHzQua4bpZu6MmJIfkcPqiaHE+pNzPNlP6pdb3Ro4OqkJtsFHdt8p0Tn1KAxdwPslZuXfVDfW4cnD8m4vdA7t0wNzYbpOLb3DbBMCPUECu65zNTpIM1kT1PYfEnGmxsiDhezwl5b6FFMDZM8vy+RI6DQhMx3dAILntvmukJH/Fqj09ki0DVl/qoyfzvAdYZTsU5a0ajRMBkFm6Am9kBPUos5CBunsTsohuiVSxtGYpFtjqg8xGjkWY5fM5QJjshDo3ajqCrNNiFdRVMVdR1k0U8JDmua4jVVruabtSBusZ68dkVlFkTMej2fA/5TGP4ZRQ0tbReNLW2eZDq497r1bgz7deCuLxBTVFQMNr5hfw8rg6x/tL5DygIWOljkD4nuY4bFpIK/F9w9gu1dVhPgROvL1uJmf2n3fSn0NXc92E/mm4ffQbT1DRJSVDZW9436J7v2fYjsQving3j3iHhDEhVUmK1RgksJo85ddvtfYr2TBf5UNI6aKjxvBJ3RPcAajMwZPcr+edz9hu59Ds46MfFxq7jy/aZ/u+vp7np2ReU8Ze4fhR6wTyxP7EJ6iKGuhMdfRR1DDucio4NxHwzxFTsqcMrWO5gzMD3BtwtJ8ckY/ANl+MyxnGZxyiph9CJvzhzFZ9n+E1LnOwuqdSuPwkZh/FYVRw1xRw/d9GyeRo3fC7T9AvQBcG88Z/tAIgZBpBO4jo12iQrhsM4/xKlAir4GysH+/wD3LoKLirh/EnnNI6CT8sgsP1V6swvB65rm4hhbRId3AarnMQ+zvnefB66N37klgUHaRVdtngt6OBBBWVxFwtw3xfTmlx3DYaqMEOJ2dce64J7eKeGJhndUw27DMxa+G/aA6J/9Mw8v/fhsCumjbs0Zxs1TMZR6THrDOWMZRUx5MjFf5N/AczJH4RA6ke4GzXPcQCvD+JPsF+0bA62VtFg8uIUrTds0LLtPsvrLDsfw3FWh9PWx5vyPOR4+i02z8v47L9P2z2y7p23Ocpz8SJ92czP287ebd2/RuiIqvo+A8Vw7EMFqfCYrRvgmsHFkg1sqrTG8ZsoX3FxZ9n/B3HUAh4gwmOZwdmEsfkf+q89xL+TDwkYn/c1TPG4tIY2eZ1gbaWK/fdD/AIi9Fnpx/GYZRn76i4/rb5G3tGyMp8OYp8u6tKIONl2PEn2P8f8AD2IS0jsEqKqJhsJ4W3Y75Lma/BsXwdjJMSoJadshs0vFrlfttHcuj6rj4O3GeXpETF/b1fNy07MP4sZVvMUFnKTMwjypiHsGoXuczae6BzTfTbdEHOvmTFzrrTQBoLpFxPmTudbVPkIaH9HIlAubpXPdJIWRkj5xonA0te6IPOxYLICxzXXYdEV7HIxgeXkK1GGus4W2QFrCbFTx2DR5gAv59b8eAizsqZ5yGyYzwuebOvZRyuzOu1LbA5lypCNQewUPNcDlfoVFK6SR9gUsXTUMY3NfZPHVNcMxWc1kuudSx2abvksERfNY0IDWlwystf5KnJV0jb7n6rPr+JKLDY+Y+IlpNrDUrWGGWeXHGLlrHGc5rGPNr5ql3W6ZsbneaSa3zK5scUzVJLIKaZptoeXZZMtLxFXSunqK10DXbNY7QL36O2b90zExx+r16ug27J84r6u7M9CNH1Tf1WRi/FGFYXUNp4g+d7hcZOi56mw4xP5k9VNM7952ismGAyCR7GnLpqvbp7NWV7ZuPk9ertlZXsm4XP52VFQclNTyxdjZZdVJjWINHiqvllp0Eel1bMsfwhQPxGhZ+0njDvyg3K+jp7fo0Tyxjz+b3aui06puIQx0A8rah75A3o43VkRQM0DAnbLHO0PjvY7XScY2ep8f6r2RMYxUQ9cYxHlBzTxnXIFKCQPg+qzKhuOVd24cBk7jdZs9FiVKOfU5jKzubqc26dNd7h5ACq8szo/XJDH83LDgpuIK+zY4ZrnaxsFrRcK1sEDqid8c0n7ztlnkcFqimpCw8+qjfL0DDoinxLDqTNzo48w6AXWRJhGM1UwFKyGmb/H9VboeFp5ZwK6rlmv1afJ9VWOJv50yHSiojIOgEa0KfEcXns6roYqSEAmRzxYgJpI8NweoGeZjWDdznkfosTG+N2vnccDHM8uQyOFrH5LsjR5uJY/PmpahmGULCAKiXyvdbso+IuKcHwvBpMCoKuXFKqTQyO1A91wlfNiuMPZ964jLKxhuGDQJ4YIYGZIGZW9royKBuRhzjzFGLEatTagotX2y6BaA7DKllv8AEU7WOF+Zb2RCwNkaMAMqdugunGY/RIbIGs73SylOZLiyEu07oCcdrBOSBv2QiMu3KFsTCSXOKCQzeXIGAIbuG6G2uie2u2qMnDnE7JdfMU40CYkfE5Gi8vZEywBOW90BsNXbJHU7/RATn3SzWQ2B3RbhA+v0QluvqTl5v5WXTC7hmcgduyRciylCQD6kD5bC9k5HwpXcRZx+SbNrl6oELi+icEAdym+EpgQRcFA4ObTKmI1zFM3ltByAnuhdK9zjG3QIJjIA0/mTG+9teyTdN7IiL+ZQA1r7Xtujy2I9ki55N/ZIOUSzbhJLNlQh19RoVpRu3TWuUwc5zjcWREtAugYljWDSxSBdYe6bQgHQg7J7EboBDz0ZdHe6YiwF+qYblAQFu6fMnjeYyXDqLIESh5XHzPO6YssUrWAN/omJvshR7e6WXshRh2TXLfRCgMe0nunLtcttE75GWGcW7aIg1pCNWjRHZOTnN3FAiCbtbqmdulqCludECu38yRGxQsBtYoumbogZPfXMmS16IC8pPUJraXukNb3/AFSt1QMMwOtkkw9WVOgSXzT+U/RNubKhsovfqk3ZOSAbJEXQIn2TXtb3SJ0SFiEDpnelMPURskdLjdQN5dXZuiVjokQ8GwcLIm7oFpdIkN8xTXa35pzr9FQ0btMw/RI6lP5iTdIWt5dkDtPVLbYJiLJhpdyAjsgLjpbRM8vLdP1TjK4adFGTEEnyhFo68fW3RNtr2TuGt0DWta5JKcEJAGw18xSLTfS31VpqzNI8xale4ul1v3SN7GwueyBgLpDS4Fid7JM3udClcZs+twrAZw8ujAkx4uGZT3SBeT/aKIXvlKgf4vUkA7XW+qQO7XC2qVst0AvsNbG3snMuwyHU2unOg1uEBzX8uiB3t1BSc0vaWBE6xtlFtAoJZ44jlzm/YIJY4xGCG3PzREBzTdDC4Swl3uhBnEnlsWdQUA86CkiAe+zR7KZrmOYHg6EXCBkcckYYWhw90ZF2ho2HRAtPdC51pMvRGGkkN/Mo2mOW7hc2PZBId/YJDcnfsgmibOwN5jm69EoIuS3JnLtdygkREaXy6IXEi1hclOSbWWbCylO42NvZCx9iCdgicAdRsgYb3StchJuh9kxOllVgnF0T7ZfZN1B6JwO7Ubn5x/BEAXNcARb6JhY3bmsel+qTWsAtl+SYjztf2QHlsEJJAva6e90ZY4AX7XQB0+ibnRHYj9U5zfCgMMb35izXuglNiARsmbumBDW+wSzAoHbunOwQPLxbIL666orfqgYadBqiZI6M3CHqf4JIHkdmLemiEOGbI13REN0LwR5gLqh7BWatlGIYn0hNyPOD0KrJm7IHBAIuU50Ka5HS46pXBFwgRF9Ug7cJgTZI3tpuOqgYgH1NTXI6XuisSA7NfRDr7KhjcH27oh5fMmGa+VORZATSzN5dUP0sEgxoNxuUjcOtY2UBAWQ9fqnvY90jr6RZUOkhaCfKUjcHK3dA7d0Mhd6Wbo9gmL7nzWSygeZos5tikPLZ3RGQLJgACW9OijJOcwjRxRAZhvZDyyWl4FwOqTbh2qAk4y9UgQdEyBO0ck7NYWTEuLtUQOU3O3VVo4JLbtGiG19Cj0aC1hNkCga+7CE7Bu6/0T5L+cdEzo7gO7FA4v2SOR3q/RJpvuNwhLeiB7ba3T2vuhGlk97oDOqE2Ojgmacxswgnsns74tCgbJ7J/wDZSuLIfN2QFLYZWnqE/NzADLt1Q77pwAgIujLg5jbHqhJJTZRfMn8vdBK1pLS4kadEg2Nurnn5IbFOBc6usoCzRnQXuheAL2O6ctaLHdMXD4UDkuPrQ5QDdOSUswQCTYpxqLhIkAeZJu6IN1nDexaox+8ncRYdD3CAFxURIShtonYGpOuEZCBYa7Jy0nVrkOZ9rHVEwmxRo2YgWTg5t0JNykXuHqKtNJNAEDcwN0Uby70aHpdETJctNkDPkBQ5rtA7JZRfVqWQt83RRknaIyP0Qk6ofxhv6eiULzcaxeCFscGJVDBGLMAeQAF6BwP9vHFnDHhaCrrH1FBF6w8ZnH6rzG7iiIsvmdd2Xoe4651dRqiYn5VN/G/i7auo26Z5YS+r+E/5RHCHEE8lJicMmGGNodzZHNyH5L0ymlpK+mZWYfUsnhkFw5jtCF8B2PZdVwv9o/FPCc/Po8RnlY1mRkMkvkA+S/nvd/8ADnGp2dsz/wDxy/4n+76/T93n03R+sPtBzahnlI0Ub6SB1pGfgy/mavFeGv5TWHiKBnE0Er3vAEghiJs4nv2XsWDcT8N8Rx5sLxCB77AmMyAOHzuv5t13Z+t7bNdVrnH185jymvhL7Grfr3fwTa0Zq9nkmjbVRDqN1m1XDnDGLHmvp3QSHq27Strl9AopmsBHPp7j8zdwvnw6uFxb7PcUpfxqCeOpj+uf9VQpsb4iwKUMM88bhuyUaL0djsjhyJz7NcUVQ2GtiMGI0scrD3F1mxy9D9oNA9whraSSlf8Anabt/uXUUlZDVxCekqBUMPVnRc1iH2eYbVuJwurNK4/C7Uf3rnKrhzifh2YyRB2Q7yQO0/51W4ll6kJm2s8EA7tOxXP8UcBcI8YUEtBi2FQ/iAgSRiz2+4XMYdx7iVK+2IQx1bO7jlI/zXUUnFWC1bW8vEmxvPwSHULWrdnozjZrmso84mGcsYyip9Hj+O/yXaKmZJJw9WyzgtPLEj7WPS+q8XxvgXjTAauSjrMCrWmJ2XmCIlh+Tl9uR1Dx5v2jPzN1CtiWKqYY5mCVh3a7Zfs+1e3nc+gnLx58WJ/mn0/V4t3bNO2uP5fo+AKuixGgy+PpZYM3pzttdQjMQvtDjL7IOEeM6LwstKaOQCzJotSz5BeT45/JbrcOoZpcFxaWslDfw2Ftrn3K/ddr/wAQu39Trj8Z/p53VVMx9b/u+Zu7Vtwn/T84eByM0ukwkDfQ7hX6/AOIcLqpaKvwesjkjcWk8l1j8jZVHxSQAMqaeSInUB7C2/6r93hv1banDKJv5vmzjMesAs63l6pumUhEGx7MefkUix3RdWQPDyPKmDi3Q7KQfvIH90HsnMUUk8h0zWSSX89fjTwUFTM3mRjTulPA+lZnqKsRt9kkkaFR1eBNivJVGQ9yFz9Xxnw3BUvbTSkOabEpJL6Pb+lw6qZjP3Pb0fT4dRMxl7lX+e9BUkx0cjpHgaAN3VGSXiOskMslS6midswOSSX3un7do1TM1f1831tPRaddzV/Vo4bhOJ10h5FZM8xi58ysYhQGkqR94MzSW0zHokks4Zxh1ng44xEV8PNMcox6nw8cYiK+CvzvyAAeyYyPPxJJL6D2mdmKHKSkkinbFnaQXiMdXHYLHmxzhzDKpzIWBzx6pLXuUkkXFK/jHh2X11Dh/skIG8T8KXu98jv9kpJLk9C+zjfhSlZkhmnt2bGVBLx5w6P2dNVSn2bZJJdXNXf9oWFj9hhVTIe2ayAfaO0izcDlHzkSSWqS0LPtDqeYHnBoyO1yoMR4yxjEmcmBvg4juGXvb5pJJTNMTkgm7y957vcSpNmpJLbmHzH4SkBJYXZZJJEFbQpwSGpJIFcEpHe6SSKTJAL6Ap7knoG90kkCBFkiAPMkkjRXdbKDolfukkgfzZTlCZt3Nv0SSQMbhP0u46pJIFf3Sbo4u3SSQFluiylJJAOpI7IspSSQIN/eT6eySSBiOqawI6pJIH00a76oHepJJEg2UlpDU7RYJJIqSw6i6K9gR16FJJQMS7Kha03N3JJKMjayw3/VIZbXSSVagxGYIQ158vRJJUF5QAG9EgdSkkgVrm5RWuT2SSQD5QmSSQJP5QkkgcmybT3SSQKw0d+VLr9UkkDFosXBxStdoPdJJAWwTaEEJJIGAsN0miwyt2SSQFdhtfRAHF4zBqSSBA6AjZyQI2SSQPe17dUkkkDXACfcJJKhtgAlfS6SSBst0V9bJJJIbYpr6X90kkT3haSbl2yJJJRQ+QkX9Q2TtLdUkkDh1yEXVJJGTOFwhI0yW3SSRo59kGW3m690kkIPe42TscbHN0SSQO4XF+t01ma5ndEkkZJuVPn/ACpJKtAzuDhbohjfNK8kx2G490kkE4sAb2QZjfN0SSUCsT+t0n6m3tukkqHeH9fokM2UZiEkkDO2UL6Zjnl7idUkkn1BgcpjTkP0R9fokkgjgljdGeQ4lrDZGZPKbNv/AHpJIK5nke4R2Ou6njDmsALdkklAWUIZJoYtJH2J290klZhIESAQ4NKV3FJJRT2PwtQsgDRnkec3ZJJBINki0scM3zSSQFufZCdEkkD6kpH6pJIkI2ts/O3UqQEMy3u699CkkhJ3Oz65bICbC97FJJFMGeWxcSE/pakkgQ51hntbon1tukkgdu6WV5Jt0FykkgTmPaRnBFx1CfPYJJII7kvtlSFxmz99EkkD5gTa+qWYXt1SSQP1+iZub4nXSSVDa6Zdihu+5FvMEkkkOXae6bmXsHC/+CSSBA+YNz9LAlPfbN+qSSAjoUr7kJJKAhmyklBfX3SSVkPc7NT2u3zNsUklAibITmH1SSVDG4PX5pX01dqkkgch1szdkzTrlc6ySSgczNaMuS5SEjXgEbJJIE7N0R6lJJABBPlBIPsiBddJJULXonGoJSSUAkPJ017JrPb6wQeySSB8o3Gh9kbiCBnHm790kkCLRfM06IdL26pJIE54v5tE+gF0kkD7hNsR5tEkkDvNvQ66Rc8fCkkpAcFxKdu6SSJJ9whSSVU3LY4WebFMMjNL3SSQSZ4eqAuGazEklAWV1g7MhczOb5iCkkoCAuLdeqa2UpJKwH6py24sUklAFrGzUYzfEkkrKT6ETZAc1kkkWBNyu9bbIi4DTNokkgFsgB7ocwSSUZEHIwD0SSVaMdrK5hOOYxgb3zYRXSUz36l7NEklx3adfUYTr24xljPumLhMcpwm8ZqXrP2d/wAoTHMLgdQcVV7qtrLuZNKLvd7XXuuBfaNgGNQRzyxPgEzQ5rnEWISSX8c9vOx9H23LHqemx4znPnHu9PdHufou19Ts3XhnN06Hw9NUN5tM+N47tddCYZ4/h0SSX84l9cBI67o2zzDQNjkb2SSUFGuwPBcUZaroY43fmabELl8Q+z2ra0zYbWtnb+UtAP8Aekkgx4qribhqUgmqhaNw4fhldBRfaDDJpX0AafzxnX+5JJWB0uHY7huItHhMQzvP9W7daQqLaPY9nzSSVA1MFDWC1TBBNfcltyvMvtF+wXBOOqmnq6XE34XNAwtAjAcDc32SSXv6Hr+o7dujqOmy45x7/r5e9x26sN2PDOLh5hxH/Jl4hwiglqsMxB2JTNbdkbGgFx+a8YqYqqhnfS1tPJDJG4tcHNtqNCkkv697Dd963u8bsOsy5caqaiJ87+H0fn+59Nr6fjOuKsHNjPxapCx6pJL+hvlv/9k=" } ] } diff --git a/samples/js/node/notebooks/hello-world.nnb b/samples/js/node/notebooks/hello-world.nnb index dbc46d196345ab..83d4ca8bec29f5 100644 --- a/samples/js/node/notebooks/hello-world.nnb +++ b/samples/js/node/notebooks/hello-world.nnb @@ -17,62 +17,10 @@ { "language": "javascript", "source": [ - "const { cv } = require('opencv-wasm');\nconst { display } = require('node-kernel');\nconst { getImageData, displayImage, downloadFile } = require('../helpers.js');\n\nconst { addon: ov } = require('openvino-node');\n" + "const { display } = require('node-kernel');\nconst { addon: ov } = require('openvino-node');\n\nconst Image = require('../image.js');\nconst imagenetClassesMap = require('../../assets/datasets/imagenet_class_index.json');\n" ], "outputs": [] }, - { - "language": "markdown", - "source": [ - "## Download the Model" - ], - "outputs": [] - }, - { - "language": "javascript", - "source": [ - "const baseArtifactsDir = '../../assets/models';\n\nconst modelName = 'v3-small_224_1.0_float';\nconst modelXMLName = `${modelName}.xml`;\nconst modelBINName = `${modelName}.bin`;\n\nconst modelXMLPath = baseArtifactsDir + '/' + modelXMLName;\n\nconst baseURL = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/mobelinet-v3-tf/FP32/';\n\nawait downloadFile(baseURL + modelXMLName, modelXMLName, baseArtifactsDir);\nawait downloadFile(baseURL + modelBINName, modelBINName, baseArtifactsDir);\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/models/v3-small_224_1.0_float.bin'", - "" - ] - } - ] - } - ] - }, - { - "language": "markdown", - "source": [ - "## Download an Image and Imagenet Classes" - ], - "outputs": [] - }, - { - "language": "javascript", - "source": [ - "const imgUrl = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg';\nconst classesUrl = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_class_index.json';\n\nawait downloadFile(imgUrl, 'coco.jpg', '../../assets/images');\nawait downloadFile(classesUrl, 'imagenet_class_index.json', '../../assets/datasets');\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/datasets/imagenet_class_index.json'", - "" - ] - } - ] - } - ] - }, { "language": "markdown", "source": [ @@ -83,7 +31,7 @@ { "language": "javascript", "source": [ - "const core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'CPU');\n\nconst outputLayer = compiledModel.outputs[0];\n" + "const modelXMLPath = '../../assets/models/v3-small_224_1.0_float.xml';\n\nconst core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'CPU');\n\nconst inputLayer = compiledModel.inputs[0];\nconst outputLayer = compiledModel.outputs[0];\n" ], "outputs": [] }, @@ -97,14 +45,14 @@ { "language": "javascript", "source": [ - "const imgData = await getImageData('../../assets/images/coco.jpg');\n\n// Use opencv-wasm to preprocess image.\nconst originalImage = cv.matFromImageData(imgData);\nconst image = new cv.Mat();\n// The MobileNet model expects images in RGB format.\ncv.cvtColor(originalImage, image, cv.COLOR_RGBA2RGB);\n// Resize to MobileNet image shape.\ncv.resize(image, image, new cv.Size(224, 224));\n\ndisplayImage(imgData, display);\n" + "const img = await Image.load('../../assets/images/coco.jpg');\n\nimg.display(display);\n" ], "outputs": [ { "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAJBAyADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDL3/ZJ2aB3LSOSsZTamCBjIzx3/SrCXdyB5LusO3lQHPB/Hn2qGW5C3UMU2AuwJtMeGjI5UKc/MD68VQumNzDvkYxtu24BDjP4dKzqzkpNIiK6mhDPBcqJJk3S7mXc52gdOSf61B5Tz/OIcSP9zewB68jrz681mwXG2Qx7My44YdR9K1InE21wrSK2QE2H5T1waxZoijcXUX2d7OTdvUnPHIOcj9f8KrNGlzbSKJy/lsN42MSv49P8mn3st5Des+cRyLvLbV2+jcfj+tVxuFx5vnB45OVeMFTu6EZ/xFCQEcIg3BjKyjb1ByCffnp71pwWd5E2/wCXKncIyR0zyffg/wCFUm2+SWknjZgxcAjDA9/zx0qa0uywZju+VcrgbtvsR6U3fdCLcV3mCa2mj+ff5iGfY4+U52844/2vSq7mKW3dpgGiYhMIuzDbuQuDg5Aqyb+OIsk8ayx4ZkB6qSP5H0qvbvHbJNGu4lOQ6qNqn0we+DRd2sNIgEYdJkSB3iVOS58x1X/OK0tKtLaO3llbcpwVeNwGwpx8278+3BqnFHcpJuttr/dcnPAzzwBj6H0q2jOlw3n24eZFd3wuwgHodw7dOOazkhlS8w0gIaORg5XbJkbx14/Lr2pzXsS2scc0b2zOo2BjkMoz6/h61XvUvBNCvklzKqvjBbb2xuI5/pjHaoIdTlkt5Z5EZndtxkV/kHzHI+jA4B68e9Uo6CvYks4Le40uH55NqfKIy5BXPVlPcZxx/hSJcXizpD5axRrz93G4/wB73NVtHR1uhcRsHigVsuSRu+9tyAfrVq/fzpNhkYrGWVSmThuuMDjNVy6gSLceW8zPIhcY2Bl+8fTI46U+0meGWbzAAqgNsDZG0jP8qijf7PYlJUaMHdvZlAYj5fmXsfvCkiktvOWNkkuYWRVXYpD7cE7t3Hf8BzTsFhVkNuoaNBK4yvDDHfaW46DufftU94rRuHCSwkhA4JDHkZIA7Dnv1qF5ZFLxs8s2Hwwkf5s9/mXGTj+LH5VMsJnhWaZ32sxjVzgu+OdvtnGOTx70lvoIjS7RJi00SMY3JXflS27+93OPfpVrRbq4V7jLQRJKcDEeeM/MOnTFZjwRC+WUvJgv/qnAc56KknIINX9FRoftKywyMqqGMMb7Sufry3XHr610Ln3T1M/dtqafnSuptPPDIiGKMkkcNjo3rgYrQjddVM0f2hlSNMCdJSMZ+8WX5eeNuTkjJ9awIyRYRRKvyj7/AO93ewyD0+vStK2unmtka3kJRXAEO1QrsCDu+bjPb6+1deFqa2k7mNaLtohZ7E211smtHnhkPBADDce5Py8dc/xDA61GNsFlLDdQmeNlKttPmFTgAMScHa3PKnjNX7q+luIpik9p50bsYopGdGYKMfOFJHMnTHGBms6ICS4k1GeGOe0dmZwy8Mh52xkc56g9VOR0rWpTjJ3WiFCbS1M6SD/S7+zmLtCieZGzdAGAC/NwflORyPWtDQorF7MabqEkcMTHfDdBfnjkHT5sEj/DNVZQ1xbR+WI/tM0QSZ7lcK+PlIVs/KTgEn17c1Ss55RaywylhPBLuTA+ZD0IP9329c151SLhJ2OqnK6Orle1tYmuZH8zaxDKko4Ydwc/LnGfxrjdTWNtRvri5EfzSMWG7eWU8qy568YHH1rZ0wub4N9riK3A3O2fmR+NoZccg9Pf8axfEE8Y1q6W4VZJSImAClcN5a84I4yK66tV1aKl2MYQUajSIZLKJdOkEErbkXzHMithvx9R+uc1s+Bo8xXc20lkaOHk9QwZgB6fNjn1NZkFxIdLltw4McTjyyE52k8r79QK7nQdKj0nSkg2qZZP3k7A7g7n+gHFaZbTbk5dicTJKFu5laneQ3ISKZykJXMZY+XPDN/tL2I7MOCGI5rQskv5NSmmkXyYwC0sZbIMpGchc4xg7i3qTxxVe4sL+G8ElkkUU7SM8cjHzUI5+Vgy4U7TkE9cFc5wavzyywyzyvFELK3+R0ZeZlwGLKDx8vOB/ENw9K7IQm5OVRHO2lFKJRkt43117efMsclrsRDJ9/BCsD6Z3YGP7rc1WWS0gu7m7ljacwoF8qNQ/lg5+bnhjjHXsQed1M1q1nMyOnlvDGJYvMVN3VR8u/oc5+XpypqPToZrmIC3i8gC42AW6hHtkVMHzM9dx28Z5+97VlKSnNxSTZpa0U2y+0Uc8AeDP7ybz2xFhNzKFIXp25IGPu5+tWNoJZjD9lhHlvtDsvBKlvut1XCsx2jr71pLYRwxW1mztvMTrK6SH5QducFueW9eT36VX1mCaayO/wAt54JI2Ucx+cozu2/XJ6HqKK+GbjzPddAhW1siBLN0ktJoZpLqwxIfLVl3FNpG0MePlPbjp9a0LS5tDfC3meWRQVMUfKjj+DDDDNzyAx+7/DVTRp7eK3t1gn8uRZty7MgFcfMGU459c4POeTWkbYus7o+6RlUeXIS0asAOQDyuevHPzVLwvPFKGnUPa2vzFeKARSSi5gSEl9zSiHaq9CCHz69c8Fs881XvtLjdLh496+bLEwRG2JG+QNwKjPPue/fir8sc7NA8aCUg7xBPMEIB+8pwMOFGOv61Oz7FTz2jiYqJEIwy4zgMrf8A6vSuhUVKLpy/4czlUs+ZFGz0/wAm4WXzpBdNFtc/eUnPLcjHHbn61j3rzRXCRzKJYstI4RSEXAKk/NjBXJORx82eRxXSBUlUuixzgqybwwIyccfLxz6daxdTsobi4jgmLwrcZj3yNkiQ4Cnb2HBrCth4QXNFao0pzk3qZHnypBLJDakyRvAkrbF2rwPmO77pbA7fdJNXba4ZLdLK9t1KwsI7cqkjCUgsQ0X3WyDjHfCgjpS2TW0dubxiR5k+4m3Q5Mco2MoJ++PLIbHB3IcfeqRb63a3EkjRRSRBfNUKcGTDLsUY5zsLA479snPJOlzQUoysb81pamMzqjvFuZzBI2wtty2N7Asw+UsT16ZFRvcNcWUapNIkytymGwMsF+Rs4+vb5vwqzfiGO/1CANGASGEoH/LMpyx/u7W7dsdqSApmS6KJP5fyOqqzCRSMA5+6TnkE8cduK53SblqXfS460Z72QIEjeQRCCJBAcoyg+WD2LdVB5wCpNSXdhLcX4htoHuJC6xlFl3MGxnb2CH9Oo9KtWcxtbBY7ZhH+6WVwmCVOwRltpIzllI9t1TWLw3ltOuGlWZW/1vICbvlDycZYEk4z17Y5pSjCUuXYLtK5RtpI7e8Q+ad53Qyyb2RW4wuMDtgY+XGaR5DDqbje8jvKoiJwB2Kx9OMALxnpuzioZ7WNLGJ2k83ZNJGYj8zvjLeauMdscAk5DGi3PKW6M0jSvzII+dxY7MYGemCRz3zxXOpOJdjotOvgtxJaxw+ZcGEFPMjKgyBWVtuPuntjGORjOKfZPda1bTpHMiZj5kMTIythi4bZyAv+z83TpVe0VWv2j2wFdgit5C/zMw25VeSMrt3e3PPWpraSaG6jltYmuZCrRPeGVVMK9cORwwz1HJxXRShNx5ua1/xMp2btYq/2gbXyN6O0Z+WdWViCnADr/eXGct7jI5pzyLq1m5s7RfNI8t5ZI9kcZX+HJGWPPAx8vfb0q8LeGZJbWe6lllZmdoBKyxBW5GE3fNg/389RwKsw28dtBHDEuI412qDycfXue5NezRoTkuWo7x8zjnUitY7lS005IEjaVmmmXnLnKRseuxei/Xr71P8AZ4vMkcxqWkRUbIyCFzgY/E1P6epOB70FSOtdsIxiuVdDByk9WMoxTsUmKsgbijFPx7UmKAG4pMVJikxQA3FJin4oxQMZijFPxRj2oAZijFOxRigBm2lxTsUYoAZRin4oxQAzFG2n45oxQA0ClxTsUuKAGY4pQMDPenbNzdM4qPzEYlVzJj7wQbv/AK360AOUZ5P4U8AnoM03963GFQep+Y/kOP1pDArD96Wl9nPH/fI4oATzo9xVW3t/cjG4/p0/GgmY/dRI/wDfO8/kOP1qfAChQMKOgHA/KgL+dAEH2dWGJXeb2c/L/wB8jAqYKAMAAD0FOwT0pcUrhqNxS4/KnBckDH3vyp0myXCRg56M2OFyR+fSsKtdQ0NoUnLVkZzx8uGIBx6UjbI0Z3ZUQAszucBQOSSewFSBMcAZPoBWcu3Wp8J+802GTAwflu5VPQesaMPozD0XnVSsieW7v0HW0bXsqahcKY4kBNpDIMFFI5mcf3mHQfwr7k05pN8YmKMySnZFCOGm7hR6L/Ex/oOX3Mi3G/zPns0OJQF3G4kJ+VB6jPX+8cDoDU8MT+Y1xcYM7jbgHIjX+4p7+7dz7AVSJYQQeXueRlkuJMeZIBgcdFX0Vew+pPJp7DKsP7zhf5VJimrz5fuS38/8adhbjiM5rlbK2e41HVE3qqf2jcHAzuHzY4/OusI61xsMkkGqarMscpU3t0N2eA2TtXHv15715+Y/wl6nXgviZ2fh8oZ3mblHvZpDhcYAlIUY+irSqMjJ6nmk0dBBIIsjakrqCfy/wqUCssr2l8i8d9kbg1Der+5RGco7sBGMcsfTkEfieKsY4rO1SBzavhgkbMqjy2BJP3sMDxgkY7/hXRmFV06EmjDCw5qiONvSplilcxywMeDGw2/d6qOo6Z+oNXIxa3ocxCNNysuXPJYcjgDjP+c1zyumyTcpDbtwIbIUZ5yPf9K2rHTmey8/ezkNteOKVcjHT6+ufyr59nooqwW6O8TusglDdSPkb071euWh2rbi32MvzLiJfmPcHHU1nXEZlcqoGUAVyj5Rj13D0+hqS0S7trxHkSUMuDlmIIHqKx66lE8I86Vg6QM3/PQjO4f7S98cVRkEyQeW8QQbsFXUhW+hzwenStaCaNzJC0zyIrsBnpSTKyIzpbiKaRlPnRyYTJH44yoI+tHNZg0YTooaO5Ulhuw4HzA9MfpWxZWlrPK/2a5Mlwo+SKEDavcHPTpn/wCvVGe0meWZbqASsuQLiH5kmz0+ZcAcHrtotru907YhhDKWVuVwz8DAzj+nrTk9NAL94IxcQukZDqDgKOVyOCPXkH9agvpgfMa43xW4DM/lJhd2xVXJ7dRg9+abdTSSN9nTISQAkvljhicK3TbyOnsPXFTNZTz6dGHYXQEnzxScbvTLewIAPakvMCppcn70XEatJ5YwREcMR357flV69Mz30F3YagBbqAr/AGhwpXuRjoevvjj2qFNNtbCZo8FlmIaLgllA69Tx+J9OaqJJ9kmJt3uI5c9JHViCO+OlDSeqAfb3kP2HyRKytnEYLfdG0g5P8+BnFZjyrPcrE8iok6Ln5AoI9fqdv51aSGO8tZUkeMSRRsUG/JDkHafbPpUJgMtnHc2AeMI6sVLAiQFRtOMZzu4x/KqjZBcmf7N/bE8puDBCcbXgGVVcDgLye+PpVqGW3iu2ZSBHA4CMjEHPJy3Zj/D07/SsswXRtIJkSVNpKRvjaN2c4z+PWtKNb7zhBcNGyNtjnDAyfUkfwnnk4IoGLerGsrhDAwTbJ5rYxl+mxRwTt5weMevSqt3EbOeKRUkiSRNyZPOM5Yq2cnknnrSQZubOU+cx+bbIHj3Nnsc/1XpgZqe1uVAgQyymEbgyZ3fJ3A7jHpVCZLbFHnafY0kJH3sZyx65zwPxrcMEMAWJrdJopArq6N86r6EdyDWBa3Qt7bEKDbJ2k7c9V/T/AOvU9zdzH9zMU3ldwkRAS+7rz15pxmo7ENXH3CSyTm5ls1DbhHG6pnj/AHux+hqO0vs3Uv2mWQfIMhfvsB0A4zn3qtFcsFaDCMkkZi2lN3fI/iGPr2re0qOWK2mv0kEzpjzWKHgd1B6A/wC0uewrsoU1Ule5lNuKK0VvNcFIYola5RmwCp2spBbZ7nGMfSpTZMLSaCWKRHdgYfKkO1nJ5iK4+U9+f9rrVmS1t4PNvYotzWzgi3mZmZl3lVKMpGS28ptxn5hxkcxyzXM9xOJWMFpKcSRkDdGy5Cq6jIDcDuefyrtVGNON7XMedyZnQTRxQTQXBIY4IZOeGHLcZ9MY65xSy3Vy7IwXFtId/lh2ZcnocY5b3qKO2iMqyQvMTJIXAKbdq/dYc8MwY89BjpWpdWyxfapf3c7xqHS5jZVU/KMY+bI+8R/wGuCUZJNPRG91uZNvbNHN5s0pNrG6hwsg/eMfugcZyc/yqbUYYbe4ju0PmxZ8iaOVmLI+CArKQD6c+1Vb3VI7mWe5KjDqPlPAI6Z4PDcfdx34qy8k2pWUctw2ZN0YM853fKoOAAPXPzZ+me1EZQ9k4vcPe5kylKHunNuyhWB3CVcqQcccj2yKoz291d36/bcvNKuGuHGWYL0J9+n6V0JWVZDEXZlK7SI85B45BHUdvxrPutmJFCuszN+93NuKnvx+VcsZtaGpJo+mnUdYghd/9GVDM5jbBVFUBFz7tt/WvQAm1QMk9sk5rhPC96tleTrPPFGHC7mJ2k4zwpIx6Z+grftprw6mLkTmaylOUliZTHIpwVBH3lIzx09693AzjGkrI4cQpSkXbjVre3uo4CkjMW2ykdIlI4YnoefT1/Csi9YahdRXj3bNpattVJU8pFkG4MrfxZ+VeW/vY4BFO1CCXWkQx2AZ0l4xcFI5odpKsu5eCCx4IAzk88VcnsblIZFknsrazMGJQhdRGBzwuDkbjnqCOnpTqVJ1L7cvcmMYwa7mLZu4t/tMkjm0jKo6RxlljPzAZCncB0Ib39RiuksHhurKGXKNLsAZ4xg5x14H8XB+tYerCCbSbrUWumu7SBkRCqne0bEYTPHK5AyeODnkVqaNbJZYSIsUZdsuZNx3hiN20fdByv5n0rLBwlCo7qxdZqUNGI7N/aZtnBmtZV3ADaJIyy8oVOC2R0GPT0qF76NDJJE7yWNuI/PxCzjDL94EnII/iHUccVsTWdq12l3NDGZ4htV5D93nPfgH3rmLy7hstTvvs98plHyyRou5ZOfuSLkYIzgOvI961r+5rJ9SKb5tEbkNhaLdrMkMc80sfMindujwM7ecFenA6ZBHU0huTZyGOVJJCQcMpG47cLgg455X61UtIpNSOZ3/ANHkj8xIoIwI13ZCt2YdSO/oasK5s9QgWaS5xNcNDAs6iUlRtG4OBlQVyBnnua1hUbgmlYmUVzau5otGpO1gG2nIz600xssgkQ4cDGOxGc49ue9R2M1xKsy3ds8DxvtXcwcOuOqsOo/CnR3trLfPZpMHuYwWaNeSoBGc/n+dbOatd6GSi72RXuZoIpEnHmb1yN4VhGgwSWYD72Qu0de/TrWPqs1tdwRzIkMkYKJm4jLn5mAzjO44UMcgdG+lartDcRm1sbaOVzcbSZImEaM2QZGG3DfNjJGT82eKxNW32kFq0yxSzmSXzwqFgqZ29FGdqsWx0z79vHxFS0m77nfShoilFdXNzILUxtHNauGkddzlQrHcCRnC8YHX9am0u6bStUeSMukY3hUAwxRlwnphST/IVpLLDaSzq8SvEgA2l8PIS7Zf0x83fuOgqrcNBqMpmuHaO2VGddrq7Z2jKemTtPTp05rks7JXuzbuZ64VrYI4t3W3xMI3yRzgqvcHaT17cVWVkMaTI8KSODAYU4DZHXaecYZT83ODgdBWpcyxw6hIiIAIHcqQApjV8PtAGRn5h2xgHFULyOXc800rTec27LH7m4gZYtjhie3T5T7VaqOEbMLXdyVIJpdOne1knF15rBURB+8BIBO0fdIOTuGeR64qe6lv7aC1YTrHB5jOCgWIpcLHtbIVeWHTce+e4NZUaMrSwyPJLDGpjVzk/Nkcr7Hbn16dasy3NtBbF1jjklilZoWjURo6ZIy2085O7+mRWVSUZdBpFq4ZrPbbPdH7HKUuUuABhTkoDt6E5XqPl9O5qGBo472x8m2SMxlf3e0ohBxiTjoQc5x2A+lQRxPe3Cwz3kSP8se+TLDhcjp/D7jjvVqSPFzaJdeetts8tXjYI27HyPs5+VeAPUL/AHq5ZS0syhZdQdNPmsPMzbFNiPl1DJubDbQck+i8ZzzW3ZWL29lBBDFCfLk3KWkK/wB3Em4A4zx8owevNU4dCjnRbaEXAmZUiuIpflDMwJCqp+b5scDAIO3k5qa4uJ7OWERX0bFjH5NuwbHzYAA3ehzkZx0zWlDEKF5WuKcObqbSyQtCri5DDbnB+8Dnkbev+fapAFXa7sNhTfxzx24rOuIpo7iGVESKJWXda+SS5JJVj8o+UDr+PPar2nN9oVmgUW9rjaWL7Q2MKRjv2wCBjFenQzKT92bOWeFW6JY1UooDO0m4KodAwYjPJ284Hc8frTGG5RII2VQFHPzFc8KDj8KatvItuzhfMlcNsjVdwPU7eCOxzn1p2QAnzEBgAvz9eP6810Uqi9s3F30M5x9zVDcUYp3WjFeqmcYzFGKdijHtQA3FJT8UYoAZijFP20YoGMx70YFOxRjmgQ3FGKdigCgYzFGKfigilcBmKMc044VcthV9ScUzeG+4rv7gYH5mmAuPWlxx7etNKyt1ZEH+yNx/M/4U2S3RoZNwLnY3Lnd/Ce3T9KluyuNK7FEiMTszIQcERjdj8en60HzCONif+PH+g/nS2m82y+aTvzyWYcnA/wAKl28bu1ZUqyqRTRdSm4NoiMSYw26T/fPH5dKkAwoXt+lKF5z0pWHy5I4wc9xWjkSkJijFEYb592CQ+APbGf609lIQsq7yoLbR/FURq3LdOw1RknjkHGKXADKM5YICSPf/APVSxj5flO7jOf72abGX+bru3sBv9uAB/ntUSnqilHQkGd7DbwP50jLxuHUdqApjjWI9QRvfGc5HB/HB/wA4p0rp5YGSD1HGD/kVzVMT7NeZtGipMhuZDHF8m5gflyi4/WnxuY2IXlmcMxzj34Hpj/OadcYaN3+6o79jt5/HpWaZ5nRLLTQTqEkYYzzD5YV6bj6tnIVf4mB7KayTXxTNHdaRJLtpNQuZrC2aSKNT/pVwB8y8Z8lSCPmIPzH+EH+8wpZ5ooLdYoykVpGmzcowuxeMD0Qeo6/dHWo44IYLdLcFY7KFXEkjycSBfmdmY4+XJyzfxHPNS2tub6RL6dD5AKvbRFdu7H3ZWX9VXtwf7uNotylYmSSRJa27EpNKmwKP3MJGDGCOWb/bbv8A3R8o71cp23mlArviklZHFK97saeFNIq/OP8AZT+f/wCqnP8AcPvxSqPnc+4H6VRIuMc1x1qZDrOoQuwCtqU7A/3gjOSD/wCO12Z6d64rTgT4g10TEBF1CXy2b+AlznntnjmvPzH+GvU7MF8TOw0pm+0mFlUr9okZm78EYX8f6VOKq6Y27WJoUQkLO53ejEqdo9eGBq5wuNxwCcZrDLGkptmmNTbikNKlhgEqfUVm31ilzI7tG0bcp5iszNgfx7e/JUeo7VdkvIVlIxuWPJf5NxHo2M8AH+LnrWTNJcNJuLzvHnc/mYXylYfcUDO7qGx19wBUY/FUp0+WOosNSnGd2ctNpiDf5DyTMpOQVALL0x0xnOOlVdPtEm1KPZHKSjKVKDO0D1BzkVWe6umXyQ7Bl3IzbskHOSfx/rSwXLwsrs5jLZy445z7dPwryrO1jrNn7fCl+9nNDGgAJ8zZw2Tzk56jpn9O9WbeGAKfnWQn5cvjkAYUH3xxWLbgXkZiZ083cxDE43exJrQVZNnkyKQ6BvKK7cluv155/wAKyaAvwqsTuDbrGrcruXaV9mOSGHHB7ZqNRDLPLLCgU8iRpOMY4wQM5H51GuqWyRqEiIDhciRsAY69Pfjr/Wq/n3F9O0SiGLCtsXaMR8DkZ9aVikTXO+OFJYXaK2KBnRORwfT+71OKx5Hlmlhtwx3x4VSdw3DPB/GtK0vonhBmh88KMlurIB3H58HvinSi2uZ0f57YIWVdh4LD+H27fTg007CJIbabe9q9qjSQKDuQBgucgjP1Bx349qZH5kEnks7uyPyD867cgdPTOPpzU8kt/BKsds0LK0WU25yyl1GBx69j781FPKWmDzzbJFLbwjhue/Pt3FTuOxFO8n2mRJAAHQ7gV3fMB02k8emR6VlX/wBnFw00OxIZVCtGM/Myn05+vpxXSl0liE3yohjDMdxYgjjpj/ODWZqiRTacZIbTfIzZ3NHkA4+Yj/a9M007OwmU7eezjt1mM8f21P8AV8HcuD8p54H19O1NQLaKsyW0iKeU8xRjccn+vtWYcoo+T5WG7Ljj8+9XjC09nLiHzIUTYzRs+VDHG73UZX3FaWAvJdQ6ja+VE3mEICBvxn1GPrxVma5/dFbuZ444YfkUkYYtjC4IOPT0rEjnayu1EDv5zcLJvyrL22/3hkEfhWja3O6JYpiJflzuBDtwuB16jJH5VEo2C5nCSd5Jg7IsLc5Khc4PHTvVaFpYJF8t2SU8j/JrUUz28jobmRY1XCny8snfAByOMnge9PsbYXczxSuWkZtyxSK2O5ySuSM9OMjn8K1gnJ2Qm+pnQBJnVQ6BmIVlf5STk9x0HephK8F2xk2jbwEYbh/9erWqIICIdJO+HazMdp+UZ4CsMc+3b8QKpWbqbdkmQxkAgAr8p9Dn9KqUeV2Yk7q5NbN5V3G6hTn+EDg54xz9a0rIXUGoxExie3QOcIW2oDwxHv8ALjp2rGKiNkJYrlcqX+uCf8a0LbUVumhh0rTXkusKvm7SCWbKk7jyu4nHbp0rfDNRnczqXcTVkubHVb+CX7SI/syNIzFUZ2lb5ApDAKuBuI68kHjrWvNa6YybbyRLsxlcrEu5o/m2qWKfw59R9c1zdrpt5ZatbNdRyi5aTdm3jjuZAc/fZdxGAcD5sZxjFPuJ7W/Hli8uZ0b5fPLOiqfTZFH7DGT7GvTVblXvo5pUrvRmldWyWM8sYQNEVWKUc7MA5+9zgnj5aoSXKJJcB1hKhVeHlVRQVwFUZyT8vbnOayTKbQPZpKPMGRKY43Y+uGbnHcEe1Xo44rmwAaQkYBZSSoJ655HOfcehrgryUpbWN4Kyte5Wl8ue0QzxvhsZUICQvPzY6cc81LaypHc27wySRxpIoeEpu577ckbs4zg/Sqpnw8ka2qCVV3LtQKWI9uh9eB06dafIpmSVDP5oT5sR5Kt3C468dPwFYP3Xoy9y/cS7GdoIk81+TIcIuGyMH+736Y6VgTRSx3W5AhOSu8DAz3H1q+48qYpHmEKeARs684x7UC6gS38tGV3GAVbtWUr8xS2K8Nw9spujD5qL8jFt3y4wT0/z0rtrCXTpFeSxtryKcx5YlCnpkbThePoB+dYmlTLaafMI0DhpFDbiSVyNvTPIxxV+OBYbNvOmd/MIgVAGXaETgE/xdOO2D9K9DD1nTj7nzOerDm3NuOdn1MxIwhiSIM6BBhyT68dMe/U1Nd2bXY2m4kjQdFj4zyp+b16MMdPm9RVKxSZjGEuSvmJvZhAG3EYAUsTnPf0wSOtbGDivXopThr1OKd4y0OVa2udC1CeRTM1nNuVWihV+GyxDE4EeCW+bgcdGqtcaUtrHZyaci3VsyhBJbtwTtP8ACp54BPXHtmuovNOF7cWsj3NzElu5cxwybBJ0+9+X6motXvYobeVJZHikMTSxz7cqki8p3zu3DhQOxrCphYOLvp+hrGs7qxzeqteRt5LXMjN5gllt5ZBHIUZjglcsG2nuM/e6cVrRXGleJLPyZreEuvz+Q5wA2Nu5XXt2zwemRUN5qcd8Xt7iM2dnOjIZVnUKJMKY5AR82MgjPboe+NPTdMkspppJgk9xIuZbhI1T5iSWGB/CT831Y9qywylJ76bO5VW0V5kOkmGTS7dLOOWFrVfLe2mb95Ge6v7n7wPQ5z9JY7901J7ORU8yTDwJ5mwumBlTxjcMNznt0xV8QxiUyiNBKw2s4UbiPQnvTsD07YrucLqydjn5le5Wt547qAyR5GGZHRuqMOqmqUVnqEN3dXEN3CHnuBKsZTKlVGFDdOcHB+la59T196bG22SOXauxWzyASvGMHsayqwU1yy1sXBuLuupzdjYzW1tZ3P2+d4plDG2UD53IO7LegAbPHBHHWq13cQyCK3t2nnnuJGRxA5R3TqrBCOOm7PfaewrpRLFGjFbaNrlZGacRZbDNjdjqF5xleM5NYV55F1fWdtZmWRoWlj8uJvLbyCEbbhh6koM8c1x1cPBxbireR0wqSvZlfUZmjgktk2uqyhopDF+8bc20Alfv/KBn5e2cd6rrteyhtLRy9wqqqmNeTzn5cnBXGOQcdc1oXcepQaWXimZbtIWnAjbLsG4LL91Y8jKhRnJOB0NULyCFSkazxNd4CSBQ0gd/kXHX5dvzjcemK5J0EleJsp9GS2if2ZK1veBvME0U5ZGOxvlAYdNxCgrhgO+cd6yL2QzSJaCaGFQzOzCJ2JLMMBlPfnr9M1vpDc29l57SWrbpAXKT7mQj/bfd2J5XHU/NVPbHDBGY7ycI+6Uq6fvAm7KkZbJDAFcLn7rVyVrJKN9jSC6mB9glEUr/AL0LtUSpGhCnOWC9MirdpBcajp9xJI5ZYy4iJBHmEJu2sV46/llvUCrF1dbROLaULbecAskSZR92GxuJySf4vp24q4lxcw6dOjwho5m4DIyeavBLYGAc7ef1znNcybsaWIo7SyvLJGtoW82NldfKYZbd8pzu4Gwg57d/UU+3NrPa/ZEMMKTfKT5i+XuGcDI/iJHOB+GTSQyG1mc4k814RE3LMZecI6nbtb0z6frPNGdT07MsqvcbgxjiDK7OwO8bRgKTjv79c1bacRW1B5V1TEtzerqEtx+7WK7+WR2QgcbSflyvQgE1bjv4YLyLc4RLUlzKqiRirDau/LYK+nfmssadcXVq11Jc+RHMhGZ0Zd33dpKr1+6G68da0ILS7D2zySRiOERLJHa/vC7M2AjdBzzk9u/WspRSYzSXzZoTBEnlOEbcqOSyKkhG75e23k5Geamhf999mWPbOVDRjg7d2QZOvTnB9/asqW83fu7wPZ2yr53/AE0deNq72A+bA9c/U1M89zNdEQvuDKBMkjiN1xkqAxX3/Q9+KTqPmUkrDtZGnO1ssBknZTFbyDeJMKsZ7nJ6DntxzipGl8yGV/OcYXcp27jlTliFHOSvNVWnlguIpIdw3/L5KqCMMvyruOcqPUH07062uPMjnzsYojOe/mDk4I7dOnP481rTrTjpYmUE1cuyW8gJIjkwcYO316e9Q7MHDcHqO9RRkpPLHKsiCTEuS7Om4EgjPX39Bn1qYLGhlVt8Pl4xxtUfLx+eenuK92hjpOK9p1OCeHu3yjStGKldCp56+/FMxXpxlzK5yNWdhhFGKfijHpTuIZiin4o25NAXGYoxSb0zgHcfRRupfnPRAv8AvnJ/If40AGKaxVCNzBSex6/l1pTHn7zsfYfKP0/xpyoqDCKq/QYpgR5Y/dQ/Vvl/+vRtc9XwPRBj9TUuKMUrgRCNFOQoz/ePJ/OnGnYoxQAgFI25Q5DFdqN8oXnd25+opWjdthUMQrAkKeSPb1NLIjhQVXlTkjnjA4+mGxXLUqK7TN4R0TRXtgy2sbOSrKN2X53E455+tTL93Cr8g67s5+v1qGxKC0QEnIO9uOoC/pmp2ieGHykfhUCoTjgY/p09a4qNS0UdU43buOKnGSOT/KmY2li5GzB+uPT8elPGP4EPOFLHvx6/5/WkcyZk+YfI2Ww2duORx68nn6Vv9Y0StqZ+y1uRx7khQv8ANKwZuOQOcce3AH4VMinGQzH/AMdVvSkijC/wqrAlc564qSPILnnHUZHt39qekYXuLeVgjkTZuQYGc5xz0pNyABt/3chVKnHf2/2j/nNQyNvjlO4ltjDA6DcB/IGmecNpjVZyz7d4VM7QBjPH0/GuBybna51JJK9iwQss7NKqeZndvPzfmR/nijYhlR0Q7FORx0yP8BTdwTAZME88MW3cdvf3rO1m5js9PYb/AN9MwhSCN+SxzkZ6Zx+XXsa2q35bvYiFrhdSDyo7FEinnkBRY/MwJHDHcTjpGvBZv+Ajk1NaWcggYRCWRZGDzyu4Rrh+mcDov8Ix0A2rxkmPS4VWN7iS5aV5FCSGLbt2LwsSY/5ZjnnPznJxzT7qX+0Haxt3eO1UeVdPFhC3byU9B2YjoPlHOcYttxLsrkaxjUJ3mldJLAPtBBLJcsrdAD/yyRh/wJl/urzqDBDeYC75O9ie/wBM1FEim3aIR/ZFGI0SPaflUfKFx+OBxTn2orc5RABgAnfk+n+RnNdMZRtoZNO5KrMwVQ25/frnHX6VIvIJ698imyDB8kbGb+Mbd2e4JJ6/4/SpVUhADj6jvXRSvfQxqcvUYeqj1aljGVz6sT+tKeGHoATTkUiNc9doruRyMQj5T9K4uHyZdU123eUASX1wNmwtzlmGfThTXbEZU1xUEnleIdYi/dq0uouy4+/95Rn6df5V5+Y/w16nZgvjZ0uiOWmS4CbjMTOT0zuKt/6Dtq/cQmVWRdq5PLHORz1GKpaY226Tgf6oOfb5RWow5I7c8Vy4CCnGcZeRrjJuLjJHO6xAbebEDRQkoUjcqCdzEc5P8Wf4eeD71ijeY0shKZCsTtKxiP7xY12+uFwCoy3Jx9K6HWYmjhOyPZBJhZnjj3bVHdh3wM4Fc1NZ6n9uRYftEtvEI5Qypt2n5vupngjP4dfSuDGUHRk10NqNTnVyhJdWqQ7TaxpdKpJIHyyL128/dYfwnPseDVRrje7u6YbaHWSJQpHHYfkSO3Na7QQpLFMbaMjaoHG713Ar+NZmpWyxT+VZqQ0L7hHncVcd1JOQcZBHIrFz5maWSIIpJUEUqyAT5xwMA/pUuoLMtvmFQ0ZO8M5G7cB03Z96qLI73BWONFDJuRQDjPT8Me9aEMcxtJkcYkjU742xhu/5VMhWKunmZZEe5jZ4gjZQxMdx/u5HT/69SYkW4hNmXRnd1ITghGHf1P8AhU1pJDLcRea5j5yFXKE8dsden/66S7uFuLxTujVCvlpIo2cd85zlvei+oya2v4LRmjjhk3gAYAxkgcf7opJZyb5VkjKQIPNjO0naT/e7nnGfT0qC1ntNP1JG3uwUMFk8vdtI/n7j3p+panYz7fKti0z58w524Y+2Dz9DS6gXLabzZrplYANFGNrE5Mu5myB1687s9frUtxP9gtIfkaVnY73U8HCjqce/y/jXNht7gqpKs/VJGymDxjuCOatzahNPZymWSOJxIIwplGHVj1G0/K2QOoI47GnyDJrGWKOKZnXzSc4U9unbu3IINatuPs9ncpM8TNt3t5krYDYxyB2x1IrnkuJEON+IixyUOD/3yQDn3q7Fb28aK/2uL/V7vKcEBs5ypz26fQ0pIRn6lbpBeB7Rv9HYDARWUK23Dbd38OeK1rOSODTzNhmcWzYyxGI84Xkd2JbkY4FVo1W4ukmRLqa4MbMIiAqsBkthepx/dVs8ZrTtp2kt4pPtP2ZmXBSNFy3+yVboCuPl4pt9xmUsKzH91HFB5SjaPugAZxjH8u1X5YjEPNVvMuNiuYmCkHOeMd1/zjmq9xcr56M7tK6jY3y8IO+Md/8ACqa33nwkyHy2hIZRjAPPB6+lLcktSXcdzKEW3kk+T7qFy64xkLzjj+QpLON4Qiboy20oUfIUdcjtyec96ht9ViOY/uRyyBg9vGMv7Nu/kOai89WmeWUybtrF/wCEY7O68AkZ61ouwWLD6hLFbfY3MbWxIcQwcLHz09U47Lx0zUV7EHjkeKDDp8rochgp/iK9+fT1polgidPtILJsz5jx7TjaQeg3fxHjJHT2qk00tzMbgJ5UfMac4JBz1HfgfhVNCLX2dvsYTIv5I8rGjuNkH+03I5PYdTU0Om3Oz7HKq+csoZWUnc24ZLf7XUVY0p7CaRPPDxW4ywESLjOOWOfwqbUZha3EGJTJbCIPtcqxU+oK8kc9Tz157V0Rh+75rmUpe9YI729sreOOHbBbxdBFwS3oD0J9/X3NWJ7n7Wn2p0t45bhWeSMK2JJMlTuyfvHA9BnOc1nRSJdiVpLYy7cMkRkx1ZeefvfTPQ+tbKQR2xgL27SW02ZWzMyNMrDBBVuhUgde46nPG8NY6vQzlo9CGYXls+HaR2RsxFX3jbgEjA6jj0xTb21NxHE9n5qxXDhbhpD8qN03MOoXseh71Yn8QJbs8cNrGvl4SAqqsAv94n+92x7Vfv7i4uPDsV5bzATMrR/vUEfmAsCAoGRuG3jvg9++vs6ck7PYhTkt0c/KjW7mwuLjfOV8vzB/Ceqj69qy55ltpS6GRHkUMBuz82cMf602dTFeMksJLKdx5yGHsehqykUM8ImVHJViQjA7yDx8oHJPB5x2ri5JSex0J2Ii5mj/ANJ2+dHIWdkGDk4IGDzjqfxqKe3jjMcjSOwkO3aWHpU9/C8EMJLoxEajfg/Mp+ZDz6A7fQ4qrbNuljDgBC3zZbGfxrOas9S1qjVtYXFsjdVYjb8oxu/E9OnHfBro45Ev7EvbNCsavlQFztdvuqFz8oPp27Vl6eSd8EPnG2CgM7sAqgE8kd1BPPcZz05pdVuI9TvJGhtrw3McW1OAZJFyM4K5yR6HI6GuuhH3L/gYTetjq9LSxDZt0jabZta4X/lpzyAT1xxxjjIrSxXnWn388bxzxTyM1m0hkjlUv5ajjI5w3HbI+6QK7zTrtr21WYx7Rgc54c45IHYeleth6ilG1rHHVg09yzimPDHIpV40dSckMuecYzUuKMVu1fRmSujhPEmkwWF5C1rZXDtcxyfP5jOFlz1+uOcdx61s+HYpbuyt7i7hkhmtW2RBoiu5NvHXqvP5j8K6Ejp14ORg/wCfWs/ULiSzeIRrKTdyiFCmzEchHDHcRwcc+/ua5Y4aMJua+42dVyjyl4IT0Un6UYweaoahpb3zOQYU8yJoZCryqSv3l5U9m79cH8Kn0y0ms7GK1nlScwrtWQbgXX/aznB+nFa+0admtO5nyrvqTs0gjcRbA+DtYrnB/wAimtAtwpSUcSfeHT9etTKgWMlpY3WNtrSIwI+v6j86gS9DThLGL7ZKGK/K6rGvUZLt8vB7Lu5GK55Yig7u+ptGnUWliXYwiCsC77yoCIcLzxjsOPxOOKxNVkjiv0uIT5l1bHFxABuBjI5LnOFx/tEcVo/v7y3j+237zsc/uLRmWJmXht56+vynA746Ukfkw2EdtJJGFYMiiNQqEtnCjHy5+YD3rCNZzp2NOS0rmXc6fPfN+9uLaGNVwZopAyCMcsSwI3KM8nhQcctVKDR4/O86Helsdv2eQfIWyexU5PAyGAGMirosxHp7Ws0x+zCWRt0CrsK/MS2ApB6njGBs6VS0a3soCkkTeQZYgBdTy7Sw3fIzAcBTwAuB7149TErqrnaoW2K95ZC0kvUtAwRYSzzq7J5jZwV34PAA/vA+vXNVFuWijhgkjikdXY5fDBFbJ2lWHUk9B2zzxW7cXUEluyzl7SNldQVj3Aqq8lcDO05AXI7r61y728l3Ha+UJXimHlIGBGHA+ZefbacgY/GsXPmlfYpKxYRE8p03eWsobbDJMOGCjcduOAefU8Z9KpW8aPcQo5uPNZ1WIOzLnKYGSTlcEL2PAx9bskjrcRRRthYwvlEqP324lduT+R2njcfSi0dre7cXgRp8K8TvCrYIVhjqfm+foP4V9eKL2KE0hpnjSCe2VVgkT5HYqrId2B0OVDdP+BVbtbC2R2GpxvBCs0kS5+ZAF+bbucfMWHTv/ScXhuLNJ2ieRFR4niLkP5uNyqAFG35s85zlueKa16kkYEGWbzEliDptO4HYRlVwCAcHHUkHvyN3VthbF97aSS8Fu9tIjSplYnkDwLGp28My8MTt4B68Y4pu827xO8rQFiW8lUOchsYz36ZDYHb1qnahWtYmnxLl2uGidymCA3J285HIOMDk81DPDp8lnmBJozKseyaN3EeDkAIpJxyTj39aUYxUbAacDC+jeWNZIxGzfvSoKzKMklRnoOm4f3eDUgSYjyJljjWFv3ckqg7h97IHJ7446nHamxhYPKjt44lXeXj8wgYXZtw3GeCNvy+nrxVezu7XyZ2uJoWEb7xI0+NzKcN+JbOCBk464BqHG3QpM0ysQuAYp/mVtrGNVJjP+1zjdx9Tk+hqoWhkkXCM8jkeaqn7w7ICR0x3+uOtJItvFL5uyVCpQxRlpIVxjAYYGOmff068JvaOaVHFskZj3HDEb14G3dz8wbBHs3vRTjdtMGXbbckc5klNw7BeGddgXPOODkc8ZPXHvU0sTQWu3I3cRlVUKM7ht+nI544/KoIpEWNfMi8qeRw3mDo43fLynZffP3RUxDCUl1VSjs0TFQHVOmS3r9456DnrXXSqRpayTZlODlsTIGIGFIXPUnqOfbP0z61Js+TdlSDkcH0ODSpK0jhrba33sHjaD/e/Q9/yqKKJ1YlpN+5eXXgn0weu3rxXuUKzkkqadjzqkLN8245sIMsQv1OKbuz91Gb3xgfrUixqvKqAaXHNd5zkW1z3Vfpz/n8qDEp+8N3+8c//AFqlxSYpANxx7elJinkUmKYDcUYp+KMUgGYpdpxTguTTFO71AKg+h+lS5O+haSsLtNJjAP8AKlA3uDwdvqOlMldFUJhsMvADct/n/Cs5TkkUoq4OPnUYLIVzuxlfYfzqKZ3Vgql1OD13Y6Hv65xz7GoXnmRpFDgnbnqMj8+3XnrUd1K+YwYY91wuw7ztR2OOh3c8DOPb8/MrV24u+5206VmrbEscrweYsjYZCMeb83zbee/Pyhfzq5LF5gZ8rgqP3Yj6ntnHU44rKslkkCwwujqxLMfvMYxgKCcYG7v/ALorQ23EUaoqws2RgnJf35P9axoqcoXS2NJuKdmxwdIWjRnUb/lUM3PqQaSUeUDIucRjcqqcHsOPxNEKuypePLISAyEbsjGeWJ6k8D24pJWyg8tlYeZ8rb+JH5+X2APH1GK6FUtDXdmbheWhaiXy1VpWG50LYDZOO+M9qrGRrooxdxGC2XbAAbHY+39fxqLyHmwMKULFDImMnj5sbj93PHQ9M4pka24Im8tGMg2YRR6Hhckk9Dkk4HNc0m1qbJJkKs5ligXaPNQyvt+6ijj5m75bHU8gVdiPlgx+RkbSpzhcZ6988DjNRRq6zuHELFmzsjkwqHHyr0wO5wQxNXZCkcTSsVSHlnJIAA643fnVQjd6oUn0Qya4t4raW5uP3cMZO4v8oHOD0znnAAHc1i29u9xqy3F+PLhjs98cDuPLiWV8fN74Vtzd+APlHMjSCSOLVdShaCwSQyW1uykljyTNIvXgfcT8erVBHaagDLc3LwLeXCb3Ty1IghBOOBxlicY6Hp61rol7upFn10NGW+Zma2tnSABQJZY3VTCvZR2DYxgfwDnk7RSWc9tBbJDYRCZF+T9wpbb2wW+7z6fTNU4tPkdkVraZYU3bgzKnm5O75m52kjBJAJb7vFbNvByr7lZk/dtHHGF2r/dxtzj3/KlGMp3lIbajaKGRxyySK8jLIUwCsaELzz03fNnNSRJMuxg8ZGcY8kqM9+/Gf8KmCHyUZx2CsPMKkYHQ9umPSn/u3YrjlSo4PTv1H4V0wp05Ja6mE5zi9tBiAjk4C7ySu7Oc9Py4HNTKQ3de/Rs0Iozv2Krc4wOcU4qDz39RXTTpuK0ZhUmpbjHU/N9Av5mpcc0zHzdSfmHU56CpMcV0JvqYsAuSB68VwkGJ9W1C6ABdb2T5sc482PAz26V38K5mjHq6/wA6840UzSJdMMNHM7sxzyCZFZfz+navPzF/u0jswS95nV2sQMqjA3SRKjYHJ+Xj+v61uEfMe3NY1hk6lbLgYMSk/gprcI+duO9ZZW/iNMcvhMnVLcfYLppb6S3RoyvmhdzR5649tvGB7muHubTUGWC5S5e3jhLSxbSVfzFBDLtY5zt253eo4NehalDcT2LxWdylrcOQsczx+YEb3XvXB+Kob6C8lW9uWuowq+Z5MXl4ADEs7Jzv6HpgZB71eY0lOPN1RlhJ2dig2ofabAMsa7o3+cfl0qvd3MLwFmaMuGDB0GGyO/Xjp6VUt5yjBHHmKDglT0/+tTWk/fIqxuT0x1+b3rw1E7is8/lz+fazfNyG3Y5Deo/Oti1vHSIoZ5HlZApk3heMcBgQenrWfcq77WZiX6gMeWHPNNsh58s8Uu5P4cK4U7v4T+W6m1cDSCSG/VLgSOhwU8kgDt26fy61dma1ZJmZFZpZm2xtGPl7E7wfbv3zUcV5FZ2Daff27TRr/pEhVCpU44zweO2eMd/WsvPl7keSURzxZjLfxc5G5hnp3rK12MuIUtpS8eHLIyruw6jPoOo6dfwpUgiurT7RDAsLygskcchYsy9R/sk56fTFZt9aOi741UK+5hG/HQcrzxkjBo05Ll9ORmtXaJ13rKW+8ucde38/Sr5QNeO1SGCL7TBMkMeT5m8KF9d4yG4OPxrN1iOJVW1N158sMiu8rnBY7QOAfbuff1q9a3djdrFbXEzw3KPuDPGjhm64Vu+cdDS6jEt3EbqSXM0L7Nj/ADpIrc/gQe565qVLXUBtxBb3drE9rMZJ402zBgVDHr35yBxnpWfBl415Kn+Ebufy9KtnymgOBl2A8w7gPk7YH94evfv2q9pklusF0IZIZEbMP77aGb0Py9RnuKb0Ay2vHto40lQlof777RndjA9PX60pvmnxc3E0caxlgzMnzbf4Pm/ibrkflmpL7ZsDT/K+wkBFDBscYORz1P5VSm8u3ZMyPIIlIWJskKp53euT6/hxTSuIdLcW8lxGsM8irIdqq4+bb65zkHI600K15MVMZTc3y8bRj6HHvUVyrLFFcLcxAkb3KfNg+g9eP/r003MYd/nd0QfunCj5j/telVYC7DHFFqG0bYod/wA7r83IHP8An2p6Ssk0nlmP99FgfLkMc8/L37k1kxvC9wzS3KoBnhg3zD2Hf3FWZZpE06OKJ4vMYZBIxzuILd+xAzVJAMV5JLfy8ZhCj7PG7fIpB+8x6tx90fzrSW8RLNVtpt5ZWEqlT8rAZOwjGe3vxWdbr+7igR/LVvvZbA3dOfaprLzEdpAhkGQ5Zn2OexwTk/j9K0jYTFgvZI5S3LM/zEq2D+Har9oLma5Ecbi4VcrtGVdcnLFgeM/NjOcenAph1O7jnkkmNu6sVjl82281kHTbv4zwM8dfTNLb3t+t3JbxzykRq6OogUsIyxz8p5PDdDmtoRitzN3N6ysZTbCS2v51VEbYt1Eh+XbnYG6/nwfaqii9ubbzRtgbqnkw7QSD827k5/DJHerNrr7jTXmngs7rDYkRCyTeX2YFgVZenVs/hVFNeeS6uLGzW4MM0nnSReUHORnonOfl79flBz3rqn7OyMVz3ZCbm1u9lvKII5pGzHciPaFyeckH9SDirAtmuLF0sXluXVlcQSMq8f3jg468dj3B7HJ1Sxtob5X/ALUEjk42z2ssb8DhTlfmOMfmDUuiagLO5PlXLmKVdyPDEDkjo21h0HIOPWsorllyyKltdG7Jp94uli4+xQywr5u2NrRneJNu0qzBhyvXOD6io2Fu11ZedbFWaJIdjEREliSSZGPzKWC8jByTyOavazqtvBcI+nzvZxfLvdfm81Qw+V4+OR+fUZrOtdQshc3gawjlZkWMW0c27c2Cu5cEg79+epbIIrsTh0Zl73Y5/UrjzBFC8Tr5SeWiyEscZPvj7xPTt+dVyNwVlXbuO0DtkdB9eakvY8vLslV44z8pUHp079+Oev1qvYxySzonyuGYfeOK8uScpXOuOiNmznVJwoRIwCys4c7/ADDjaDwe4xx2PNaWjvqgmleyQhYi07oG3E/wsrnIYZUnAPdfoa1fCFhAsayXM0f2ti6ogbDGNdp2nnJ+b5uRnjg4rol0OxieV4IFjaYky453knLEn72W9c+lenQw2ik2clSturHJ6VpSWOsafcJJdJHcucM7KPn2gjj5gy9R2+91yCK7GG3gtJG8mGONZm6Iu0bvT+dVbLRhaalPcGVpRIfNVnC53nhs8YyRg7gBnvWoUzt5PBzXbTgoKxzTnzMZjNGKfj1NLt7Ac1pcgjxSEA4yoODkZGcH1pst1BE5jZw0v/PKMb5P++V5/PFR4vJugitU/wBr97L+Q+Qfm1FwsSTSR28XnTSJFEP45G2iqxuJrmPbZQsN6nE10hSMDuQh+d/phR71YhtIYJjMAz3B6zStuk/A/wAI9lwKG89tQ3OoEACkMHJc4xx04GSeM81zYmUlFWRrSSbMeeGK8sp5ZHvrq3CFImfKRcFdjrGpweuAWyflzyKtXF5aRXRtgjhUO9HlAXdhv9WrdwOOR/eq2ZyGcMGPGwIZMfKMNn/Z9fesx7uOO+ia42NAFlEjOn3MsnzBeSSy++fXk8+O8PFOzZ6Cm2tiZHt2W4a3uFCNFvZ94LoAp+Zj1Gc9OygdadLeG40toFjYNIgy7KIlViuRhW6gHbVPcGvQ0cKuly2XEe1nLg856YzkfLx8wq/PfQ3AiYWkbu8ixtLPCWVgELHP+0MHPp71hObguWP4bFqKdmzPkl8mzmV2gQeVulM+7aWDZYNj+HqO31qvaWkPlGWKON7iVC8kgQAqMgLtA/gwFC9qkv4me3FncCZQrCX5NqtjBYOWPBXnOOp24PpVa305bSK7Ivw/2gBLQqobb1K52nnn3GAK5oXW+5oyO/gs576T7S+Io/lJxkImfuhh3J/LgE9qxYrlLcTFswzMy7tibn2qfkVCW69jj+Va+n6RZG3Zrm5kG4yY+YRB8HO52J+8DyeQAPXiqS2iGwhus+dIu5zHG7bGy21VjXGWVcDH48YqXFrcCS/unhKXrXcKSzR4LNsEcbcFkDEfeOcj0I68VS1UBngtxCrPt2piMfvxklSR2PoPYZ4qJo7va15PBtRl2tibeFYNsBXP3Rn5frxT7m4tkX99CXlbDEtKrk7XYAMQO+D+lNQ6i5i1axXkirMXS4+1EuoTZliFB5467uMH8xVu8s7iMGO3eWIFBbKC7FT3ZmbHbgZ65APvVGC8uXjVba3jdYmbyUC/NgDAO0dR94e4C9aqtNc3ckk0btJNDjKfJwwUgEKOG7cdeamz6juXLSOIwpMNiySHZ5kSsSg4GxevLEMTnvn1qe7tL+O2tEgsis8dw86SSRLEjHZgOQDgqABzxkkDBzUGmX8DQSCW4SOWPd5JlRn+Y/KCV6cbm+6Dj0qxeyS+RJcxTGa3IK/vJA2U2nnbj5fz4yDwDRGTTHYp6ZbS391BdTSq1o0ipNIkpHnybX+UhuWXr0wq7sD+Krkd+sQZPNMAESJI1oA/ykEbFTdwAwxx1xU+n2RW32STRyQyCNUlA2yKuNpXrxzjg9jnpWh9mS0ihMBeK9K7ZJFff5ZyMblVQcnDAYzyAacqiuFjPS1ka6lls0aIeVtje8RwWZudzDG0Njj2xxjOauHTriZY2guUjG4GJynyh8nCBhxnPoO3OarmWRbpTC8bRlt27aByBkrno3XOMfxZzWnay+asjSlXWNWVuG+Veu7afvNkAduBzjmrTV/dAjs5Pt5VJkKkuE3nDbZMc8HjHJHT/GpILeZAzJC/kLyqsibVXj5c92GPwqveR/MWt76MA5kVA4WQ84255P3s5b8Ktwi5VTMZX2ytsXcFbcMZU/U4b29OK1jGM73Jk2tiyfLEbIzPu3bljkYMB9Mctwc4PSrTqdzcHAP1xVZGGpWjSQfaQ+3zIpZN0Z6+jdO/BHSlVLya5kuJG227HcqqMsoYgbN393aOo77sV6GDnKm+Va3OPERTV3oS4oxT8UAV7RwDMUmBT8c0uKLgR49qTFSYox7UXAjxRgetPxTSATz+fpUzkl1Liridz2qJ1fLFMHjkE8Hv2pWXZMrGVsEE4bacqMHsPr+tR/al3KxuEjBXciPGVY8bu5PGO4rnnWSTNlTegrO3m7EH3slyxPygYwKf5QdlVt2d24k5Ht/nFQsyiTzkTbGi8sSRkYLEHjkcjk/0NH2hSoZGJz2bgE9h7fjxXG6rtZnSqfUhIEkytjew7bcn5f4eegP9Kp3ZeRjIqSEoBKZY5cYGPm3e+O3vV15Ea3kkkQlH+6rcAc9M896qtCkkTWwSNQyuiRLJ5iMdrHdxjnjiuaTUluapWZNpBxbLlwWPMjAZ2n+Ec4BXAGTjOalS4a6dkQTl3QrlfmGB/Fj8ht65YVRtZY4rN1lmeURnezSYyXKhhtHXqSOav28xjborznjlhtjVfXn5QPyPanRqNLlFUgr3HtH5qku4EMTFVVmznacA5z6/yqtcl3eO9iuY41gkUsCQoWPBVnPPADbfXOPerMX7v5A2HC71lZVwuTln684yOfoBVOKWISzCaN0jQqqfOA+5T0K9iMZ71VSpdWQoR1uPEZSwZGkYW4Q/NbqWbkn5dzDqx7ADrjoppj2sccgOyOQEkKBukLYC5X2469qdDcM/m3N1HIHVN0UkqbEXd8oIJJG4g9fTge74L2NI0lSNt852SXAgl8shTwEAXcw98Dd1qKajKS5mXJuK0HyXKLt8xYxJt3Kn2diFUclicEbRn+dVY0gvmXUbmWR7bftgV5GQTMCDu8skYjUDhPvMTubAxUcs8WqXEqxSNd75PLSMx/LNIo+5tXlYl6tn73TPU1ZlE1pbma4mlLAFUQMYpJF+8QrgfIn97bxjHNdE3ryIyitOZmZr13ajULItHHIxke4zJLtU7V+63975iMnsF2qMkmrNhdPcGNnjSGRf3u2ctgMT94nHzNzheyjgck1Wi0qJ7mOAWsKkss2otJ8mxRlo7dRyQvRmXjP8R+atuSaNGMj3c4Jcl0LCV2bGMqMlACB0HHentG6YdbFb7SibYJII3jb5WHlOw5YNk57fX2xnFTyXjTyLbkxgE53xS7iq/wAI698fdH1rM+1OZpN+9lXAxNIe/cDcSMeg79KLR0ilVE+eTf8ANuTcSpP90nk8HnpXnVqznonodVOmlqacMtzJCzoZpck5Xbgn175+ner0TuSE2OTnbuOG2nqR04NZ6vHHO1ynyuzMWALEgZyp4Pr3oU3AuoWkjEh3Pu3HcVAbjCnk5GfrzWVKsqcr9Spw5kbCy4B8xQgZsKMHd196kUNgb/vdyepp5h8vlU2I3IHrmlKMoyRivoqCXxXPIqt7WIlGWH4n9akwKbGD+SgH/P41IBXUYDouJoz/ALQryfwhJutEyM79xBzyDvTivVXby43c9FRm/JSa8n8Hn/iVQHC7S4yQfmDboj+RH8q83MX7qO7BLVneWDtdamnluRHAiklU5dipJXnoOB0roWHzt9a57R1a3uFdxtklYv8AKef4tuD+FdHuSRi6HIYnHvz2rHLZJOReNTsivcfaDERa+WZcgYc4GO/Pb8j7CuJ15beyvo4ov3CxtIkyxLtViyNn5XzuAVccdNzHnt2d9PDZxie5nMMIO0kx71z78dTjAJ4Ga8p1i78q8kFvceerdJ96yI2eXVSTwm44A/EGunHVLQt3OfDRvK5hwSFJg8oaIZwHI4qxJFPLebXTYxI59Pf6HrRbB7WUu4B4IyT1B77amkvfMYsFXLMdqD7q+2O1eIegT2duhmuIblhsKNvK7Wbj2b3/ABpmtQ+VcWaWcLuqwlEkeLZ5vcDd0I5PoQc0r2juEnt0kuCW+eFEJbb6+h/+tUmk2FneXbrP56QCLcJdwRY8/wB5T+dR15hlhLsQ2CxizDbW+eK5A2SfUA565+UnrmptOt4EWUb44lyJFST5ti87h6Hr/Kqt/MWuojK0UzPg+akXl+YB8oyp6fdqUXEF2EaaIbEc71H8XPOcVL2EOtbveHfzHby2McYK5Cr1Az3A/wDrUXMr3reS9qgON3mIpwV6FvwPbrV37BH5YuUWBYc/Ptdd/X72P4jj2xUUziS58xXCwLyoJXDNnsO9RuMy2Nwgnt44k2QcsdgAyByfY/jVW1unWSWNFWRJFHOc9+SO3t+Na26Ge/cTM0cMikh0Gxm45B9e/Pb0rMiZUuHeNg0Z+dZVyQfqP51YiMqbJoN8eE2thSOWwec/yoikiG3LhFyRhCfl7jaOnTt3xTNQvZZZo4pRGwj2tGn90e/Pv1qMPvuYyYsqzYOHxke3FXZ2A07uecxsQccfKX+Yn/dPbNVYreGR5Uup5FjWMSeajq6Ju75B6HnK/wD66sm4EVv5tzGwDYeQRH5hnj06dBio7zW55vMtolQRH+PZjJA4Jxzx6/nSSfQZrJZ6etuTAls85fKcYRmGMn/Z/wA5rm/PVLly6YDltwBx37elLaX7LGYMqAGycjcM0+WdLmNiimMq/UjAz14/nTSsIryxRrGjsAF3Z8wYxs/x9/51DaGW5cOWaPaeFAB4JHy5Pfird3bzSJbQNNCEJffNuGz2zjnPtj0qivUxuTvx0I5/KtEBakQJboTjzCT8w65Hr7f/AF60NOYI8LvJHvZyU5BXGOVI7N94Y77qy5FZog6qzHIVVC5z61O9obVg8xTDKCyDJxnp9aadhM6ELpV7Ni8inEa7TkEq/wAo5Dc+vfH9czXEElvJNbWhVoeUiErr5jFlBKlRuUkEErt59xms2APc/uoYkllkbanz4+b2NWrRJbaFHdZILeVV+eN8bz1B3jJX5SMcY5rqp1e6MZRI5dMa0EcrOZZ5W2bNgZW+QMrbs4bPIKtg81VghzIrRIYJQyiJULxsxPQq3/fPX+tbUiaLdacytdF71N8CXFx99gTlWb5vm43cDHI4rNgbF5iSPKxhlWXy8iX5Rj5W6tg5xnscEVUlHm90Sbtqbemyz6jYz2WrTsq265WPCsxUfK3yv2IJxz2+lNlEGn3155Mn2/7sMj+UGDLtBWToMyYXDY4YDPXINOS9k0+VSYbZmUkNgkpeQ/xMFbDDt64qN9Str+7eay/0ZEj/AHcchbCsCNpLc9iQe3yitXUsrPchR7CNaKsgkVI9r7pM+azMvUqQw6Kex9hnFR37fb1ykLokIWIBlBW4JC8Kw74688+1XLXVrqy02SezltWdmEc8DLu+9znHZd3RvUc9a7LwnYeFpvDsV5q2n3AEszxEvMzKGHLgBex3e9S5KMC4xcpHmFnouqarqL2OnWT3s6nDrbDKoc4+Y/dX8TXWR/DTxn4evba8EEU+IncGwk3tFJjAXDbTnk8rnvx6+/aZDYw6fCunQwxWhG6NYUCqQe+BXN3XibyNTNrJcW+IZSCGGC2PT0wfzxXLGXK7nQ4pqxyFhpsVoIBcAfbVUt+8QLIMjuOvC8ewrQxWrFq9lqG23vLPfcx8xm4t2yrMcbgwGACO9Q6zZJaJcT6cZJRGfmhYq21QcFhuK9/Vvwr1KOMhL3WrHBUw0lqtSgBknHJHJxVa8uBbwSETW0Uqr8rXEgVAf9rufoOay7Wyu728S9GprJZyxgiP5ZGwctwdoUY3HHytjOM1fi0PTIpjOLKNpyMGZyWkx6Buw46LgV2XbWhzNJbmfpF7q11I7vaK9vsCxnz12btxJJYjc3BA4WtM2skxJurhmQ/8sYcxp+J+834kD2q0ibVA6nGM4xx2p2OaIppag5XehDDBDbxGKCGOGP8AuxoFB/LrT6fioykm7iUqgwVVRg5/2m9D6DFEm1sriSu9RcUSfaUty8Jzt3MqNt2SNt4U9zjG4Ac1Cs0qSNHLEzrGVQPvDO3y53nngHnrk/L9KDLm4t5LeRioJnJJwrLgjcrH0wfQc+9cOIq+0g4pa9v8jop03GV+hjvqAWRX8qGMsqvslnHzEY4C5yfmHC49uhpsN9Lb26SOu2381/MeU+Y0jDG1iCQc85zxjaOKtMsYf7GIIUlm+eK7bc2ZD82VbqGAG5QDj5eOnNLV4DBqdhLJcTmLckqbU3lIwwGW5G7DYO7r8zbvu15s8JLlUm9TrjVTdkiexMFpeMJ3RpXYebcwKMMxLNt355GGGDx07ml8xLW9W7gmhkkRpFgnkLZwy/MzKoHzHyyo46dM5zVeKWaSM2EthKvyOAkb43bgzLESOACQT3OP4RVa81KO2vgL13dJLMF3EoVWdVLMrNjoy8BcDG33FZuHLOzd13LvdFswyTPZzISSY2McH+sDx/Mcg/wgE/XnBzwajghEGpiO6Eks+9gEiRSGIHyttzhVYHpyfvZzVWB7iJoNSnjaQRqk90wVkViqNuQdRtDHOAOTgmrl1eyQXipcPEzXGT5RVd0a4DeWdo47nnnrgUrKacYopXjuNZk1B/LMa+SrBfvKzq38SqM53dePRvasrUb0QTTpceeHRkJlVijSKV3Bg5wP7owOpGc81u3MheK7jtoZGlj3gDDKpkwAx8s4PUkc+tc/qkUL6czqrIV2OsbsQd3yrgE8cBm5GFBzwKiFPkdrDvczH1Fvsd7CbeKORpTBvjfdt+bAXbyuDtJ7dMj1rSlsp7y0tI/Kt41YN5jTSLGI1Y/IFXavzFueN2fvE0zUNFudNWZJ5bb7Et1CsiBtyxSbtrYX++F+8x6q2MZrXkuIoYEhVEW3a43xofmdstsBLEnYWYAdfunPFaxjZO5LW1jmbDUCLPy1RpVt2O0QybZPM7bePmH3/wAOPSn3ssMt2rxBBMFDEREIGXOd25T8p5IOeT2NWrfTLPWIZigjiuYpdhh3fPGd2SWwB8xxsVVHGRnvTtR027tAUhVVe5X5JIJVwERgzdAN0jA4CElcK3Paue13Y0szEt3a0uxG6Rjb1RsfMAezL7jANdPYzNLpsBlhEiTNhPNl2M24kfiPlHy479xXNzabNBZy3Kq9wqtuSOAghVOSBtP3RwOma2rbQNkdvNNHIZCFLSD5Sh/hOOitnHJ3ZyKlxtqJPoaNlctcSn7aGt5Q2DIMLkZ+6ckbWyGOOmCKvfbVVJ4GimQxsFEaJyvGcgAZB/U55HOazb6C5S8kDSwPcSokci542nd97OcgYIxjuT0qvE1uLeSIWyxbj8yktkN/C2QThsKAD3Nc8o3dzQ1JA73UTpJGp3BXVz5O9SN25vQqQD2HPvTftKTIJfKkj2sIItnyHbgDawxz97p+vql02xpJF3B2RA/lxyENGV5KIOvOTknBH4Glha4uXgL+YQ3LJcJgY6AnJzyD26VcItiJkMBcW0iyxXUYKiRm+8cZI4PXnkCrUErzHy4rYxW5jVSX+UBupCsOvX8j+NYe8SSZDI29VVm3OxJBG4cnn+H2z65pJp41nT7JOROcRtslRcuWypB6Aruw2PbvVS546ktrY6C3uHRJTkSSqTkryueduG9PlH4H14qx5k0mx5ZvmxhhG5aNuOOoHTOePbtVW0YtLA8y/v4/vDAfcR1AYD9ffNLa+Y217zy1GDwB06Z+bv1HT09K7aCatUvsZTtL3SyXDOqKec4PvxmpAOKiX76uDhMk4OTyRjP8/wA6sY717WHq+0TZwV6fLZDMUYp1FdRgNxRinYpOO5pXCwwsq/ebB7Ad6rvkZAY7ixCs/wAxXjIwPp61JJcLh1jPzKdrF/lUfj+fFQMvlkP5ZOSruGGSxPAH06ce1efiZJvRnZQi0tRVZ7jhoiSyjd83y/iQPbpSEyJJtD4DKx5+9nd6dxj6dqliUhwyOXYuU2lSc4HsPrz+FVH8ySMSMoRpG4HLF92fu59Rx0P5Vyqq2rHRyWJTGHd/njY53FpBkHHzc+2QP6YpLySSK44VQ6sXBXOA3qcjsf1qSG5ysifLE68NENu7B/Q9Ov8AKqEkkzytLFiLJXAdM7v9knouRnnv61NS7jZFR0epXuFeOF3KoTwzb8FR2y390kc5IxzTALmX9959p9lGFzG+4NgdgPXgde3StOAQs26NhkqCEOflTt34Xt+PWkgsjMzfZk2xbw85wuIgp5dlJAVRuyfXHHNYOK5dDRSd9TPS0u9QvINPsLd5JSUErMNojRTky84HI43A8H14Fddpvhmy1JfLtLiMXYQyEyRb4ZBnGV6N+JyTzWZYXsOmzxizvCswdnd5wzGRlV497AY2jByF/nXT/D02ccd1axFnuYlBMrrtBViSQq9hu59yaUfdKeph6l4T163kANr9pUKq+dbFXIYdG2sR059f5Vzc2lvaTGK9heE4KpHMjIz7m3OSOrA+gznHPFew+IZ9Ut7NP7MtjcSyPtbjOwY64yO9cP4j03Uk0mw1PUYF+3RzvCzYDMY2+ZR8vA5HahiRzPkv5UMMNv5DybZmjjUM0Cf7p6t1wOmSOMCltYGu5ktYdzISFnkSTYf+uYb+LOfmYdPujnJEikNdTRx7sMRFLdJxhscRI3btlu316Oe8gSNbW2S5mQBYBBb/AC7v+ma+gzn5vZjg9aunOOjW5Ek2SzPbW1rNfXCDEqpDbRMflIXOxUB4GTz7DGaoPbGVYpo1KTSMcCPn92MHueFX72M/MeT2AvafGLq9/tC6QTTxOyQNtUwxDAyY8846gv1YjjAqK5uDKbmZNrrIPKtSWZDJxkrt64PX8vXNVWq+55/1qKnH3io9lZzTme8L3chw7SSqvXkljhepP8R596LwSTzswiVbcw+WjfoVIPU49vypk6hyJN8DS4DB3Ztir9044YL09Bn1qvJE9oizTwTsIyJGnaXyYWHcfLuPpkH9K4eeWzZ02Q6W2ghtwuwmB/kBVG2jnjd/EMbu/WnsqNdRu0SuUVm8xSAD+GeDxnr60WXyXDSmdkIZQGR1kxgZwWznHXqenSrkCKbYOj8MpQMqYB45JVePT8+lRcY0r/piqku87QEyxIbpkrzx9atllKxzPHu+Thg3Cj6nv3weuD60nnrL+5R/mHSQNtBwCcZ/GoWuVjggkRhudsRHdwWwWUHjJ+7/ACqJjNCzvWR5YrdGMAPlwEuMEDqw98HgcYIq3HdhreeWYsgyD+8mEkgXOzc3oN2fWsG2nhu7NNpMltOM+Yz8qnO0fN3yOB0Az3xUVxcTiO6tCsat52ZDtC529MHsM4Ppz+V0K7oyUt7GNWMZxcTbt9Rik1NLZ5VkdUmOIj95g4jXjoTgE/nitFZ08945EMeGVUYsMPnp9OeK5LSLlLjWb24niaKV4ljeIDhmJ3ZGPckj0LZ9qt6lq8c1x5NskkGongvMfuxhlz2+QseN3UYOK76eYz97mlr0OZ4eGjsbuozJFpl8yupKW84GD1YRM2Py5ryfwsWFlwcKzLt4yMjZmu2url08M6rh086KzuFLMp+X7yytzyW4VfxrjfDAdrBVRyhDvvGOMFVOD+tb4iu60E3oGGhyt2O/0wPFfQq53A9GPXhm4+gGKs/bhb3dyji4n2yfPAHBCu7EjcxA78bT2wRuxUOlr5mpQgseWA+nzYp1zr1rLLcbba3luEmkiAL7+MlM/L1JB6dRnHbnmoKPNduzRtiL2sle5ka5rkcN6EuVMsyLJF5NrdcLI+35A/BzgDcxGPmK47VxWpwQWdzIE8ue0Fs0zCHLbd23dnkElTycZUZIHSrOpadAHvJlRUW2R4yvmkETchY2/iyv3sDp0rnDHJeo0mxIImZY41ij+VucMv4cHnv9a3rVJSTUjGEEtjotQtreIumyONHG1Sh3Ae688/8A16wnIkuCsJVWJyBK4bd69qtS3MrW32fduJA2KYgMf/XqlGiKX84MHXofQ1xRNjTs7qWOQo6fL6cgfStJZ4YVMqNFlcgCRS42Y5ByeevQVz8biFkYDduGa0I4/tF1sBReOFk6Z9M/0pSAqXExvGaWO3VNx25Tgf8AAh/DVy1vw9slq1s8jMMbw23jGOfWnSSCG3l3+TIBhSCzBlycDp0PcZ44qjYyG21H7Nd4AlO0GVeC38H/AI9jkUboLm8bq6s4Aomkkto8Hc21jgjdwcZH0zRfO00cU0ioF+YiRedy44J+nP4Vjyaw01whdERe8SphfQ5Ht6/jVie7ilhQG4C26KqIYvmAA44+vvU8oEcSwzXV0kFwsKSKV3TRbmUjkY64Pv8ASqMouNOMtv5rSxHCyMm4KRt3ZHbipnAtbx1xISDzscKD3Vj/ALOD2xVgQG7siy3/AO7CAFZSW3KvIHH+fu1ewXZjSIyuWCJ5bRruIxlTn29cVfVnNowSDbJINozwEx1PXPPH5ZpyRxQGP/j0WTG/eufm79Mc/lUpETIDcy7F2nccHJGPXp+FO9wGWnmvMqsTs2tkSj5cf1HpS6xJbRhrSFLWSP73mKPmxjIP1+nTpzTj5IjiIIQZzweH6fNnHfjrVDUAjv1ULHlWVDkHv+FJbgU0VYZRlwF46oc/kKmmlfDMEwjHaFB6Y7ev51Pdm0aKKTYxmCqGRUK0+eW4mssJIJMqoCkYKqMk5/Tk9qoRVjgVW8x5BEjcbyvU/X0q5C8ryRwqA0Y3F3YBUCjnG7+n+1iorqNLOL7J5LCZX3ecy9V2j5STyAPbrU1hcwLdJ5xVYY9uI59xDNkcDAPvweMZqlYNS2bDUIYoZRYhbeYMYHOAFbvnuv0bHbNMkmuElQS+UXC71VVyR398ipdTnV78CKzSKL/lqFQFWHRX3Hr2H/AcdqljvLW8nCzQwrLtCLJ0G7n8uv6Vfut6E69TW0eWzhsmmmhubhYii3JijDOsZJb7235WXnp2BB9RU1RoJ5UuIbny4mBkQeU0YZPVOB0bPr1JFQRNf6TdO6yeS2GAkibOccNgHr/9evX7fw54DbTrRZJHVkiiYbZ5PlIAPGPXv61rOvTUOWbsOFCpUleCuePQ3ryweXJJtQY3bPmyPU57/wA6sT6dNpktvLenAZ+csHRkwCR3wSrdGHGRzXql14P+HVzKZGe5Rixb91cTLyep4qZPDngFbOW0ae8likADeZPMzcHK4brwazjiKC3kavB1+kH9x5xqCQXFm8ojgMPm+aIgDgfKAoGeRnGd3XnHNYmlpPJfjY8rsoOzam7dxyM9zj+XNewf8I14BG4Rz3se7jCTzdMdPpWjY/Dnwrcxi7shqCLIGUst1Im8H72R6Gj21Oq9JfcTLDVKS96LS8zzPw/bQXVvefLdfY4vnZYk2+ZnHy9Pu7fm28fXpXrfhbQbG48HWcF9aW1wpaSQboxjDOcEd/u7ff1q3ZeC9I04OIFuGVgAUknZ1IGcLz25IroI0WONURQiqMBQOBWtSpFxUYmUINSbYkMMdvCkMKKkcahUVRgADtXil/P9p1qZyDlmycj+8xb/ANmr1fWNdttN328rHzniLIF656CvHUAaVpccsc9TWSVzQ3ku5bbTbiaKVlIWR054zuCr+tUrWQ38z29yiTR3IO/JYduvXivQdI0TTm0G0WexgmMsKtIZYwxYn5ufxq3HoWkQuHi0uzjcdGSIA1pTajK5E1dWPHm028sb0NY6ddSGFgVnEibgC3KOvO5crnsTnI9+vhdpoI5WTYXUNtznGfwFdn/Y2lmbzv7OtvN6b9nP51J/Zlif+XSH/vmu+OMiuhySwzfU4vbQFrtf7KsP+fOH/vmj+y9P/wCfOH/vmq+ux7C+qy7nFYpP4lX+JjgDFduNL0//AJ84f++aadI00uH+wW5Zeh2ZIqXjV0QLCvqzhlt1e4kkRHD4GJE+9uA7AjsOvtjtWdLexJLFKj20oUBB8ijargk4XqOMZOOhHrXZand+F9OuFhvEsEmbnDAEjjGT+HFcnr3inw3AwTTNG067lYZeVoPkX9OTXJUlVqqysjqhQ5dWU5rqG9t/L8qdpJGV5WgwzHa2VX5umOOe23Pas1kBluIGR5cW7Q5JY/aFYuWCofvHlsY4+XsSK29H8YaPI3lalptnArHdujt8qTjHQe3Faus6/wCEorCW/sotPutRjUrbq0R3AtgHqOB649KzlCTkueXpY1jQlZtLQ5KEK+YknaZ3B2zklXkcbNsmGxwCO5P6Umq2aXkFywmzIZCSd4XZJnarMCDnA6nqc1reB7uPVtbMF9a21zJHDuilMChlxgcY9sD8K9Dm0jRlRprnT7IBBuZ5IwNo6kkn86ThKOidyWtUzw9bz7ffbFWQiRTJLAy7AWVflVmJGVJI5x6+laGqyx2ptfJkR1jt5POQPsBIaLBHH8IGMdcFfWtvXvEvhyG4kXR9E06eZgqtdyQZBC9MDqcds1zCarcT6jaSyhGCT/u4xCoVdxCsFUDAz09aI4dpa9Snqy7dXbs1qHZCmBkDrE3cLjHzbe/qDjrQ8ksllcC2xc3QRWLplQAWzuywGW4K++AOa9Kv9M8LaVZSahf6dYwxrIJGk8rlpOgwB1b6Vx6+J/Bsc8axeFlSBeBJ5agr/wABDewrOOHk2O5zLTm0soFgYPa2kqqlt5R3M7Ow3FsDlipO4/Mc++Kzr6JxPDaW7m9hfdOoaLbtCNt+XaQw3SF/lHAwuADXtNn4d8L3wTUrXSrJjKuRKikbh74NEPg3w3bS+ZDotrG4YMGXcCCCSO/qSabgyeY8wtUaxhzNHiOSDzTInfpENox1VQoAyck5/ipsM9vLGlzFD9mkkfzZonysfyEgfeGT8q42jjtxXrEfhjQo12JpcAHGBluMcjHzcdaqzaN4VtYDdTWllDCCFMsjFVySMDk9c44rP2bKckzy1oWs4BM1tcQytGHLkH5mbc20dx1x+Jz2qO4a2/s+aVCEmMX2fZB8uWZQuz/a+8cD27V2p1D4bPKsf9iROUfj9yCAQfr6iie/+HdjMksnh5FmQh0UQqWB7HG7j8aiM4xaTeps8PUcea2hxNy80c1st+LgBZAq7ZuIW2jjc3LLtzlvXPXNR6aylgs7sJc7PKEW7cW3ZVkIOei/KP0610v/AAkXgdbl3TwbDhySzsE3E+uPX8a6PQ4PBPiG5drLwxDvGPNeSJAVz0zzk5xiidFJepCi3scVZ3KGNU8iVYoTvMMxzwCWK8c9uvI/q3fP5Z2O0q7QznO35sk8HucDHt3z0rvbifwnpF6EXw7biaNvM3xIh2sRgjOeD2IqGTV/Csp3N4bRmzncY0yT7nNc0atGm2nJHVHA4ia5owdmcdDFtuJHQNHGkofdsLHjn68g+p4HQYqG7t7aZlmngjM+VRXeQquPl2sFHUjHqM4wegr1jSNK0PVbY3Y8PW8CSHK+bEuX9/p/OtL/AIRzRtu0aZagegiFdkOVxXY45xcW4vc8rt2eWNQhkUY6DMYQdTnbkjntnv26VKIraPcVh3z8lQX5YdcZY/KOma9PHh7RVdcaZaBkOVxGMinr4f0hc40y0ySWP7kck9TWsWoqxDijyy12vEkqyICzBdmQShHq2eeKvbsnHH/fYNejpoelRoUTTbRVPUCJRmlGi6YoAGn2oAGBiJa3o140+hjUpOfU83xn0FHH94fnXpf9kad/z4W3/fpaX+y9P/58bb/v0tdP19djH6q+55kxVRksAKikVpFO1wg7jdgnHqecV6NqI0fS7Vrq6tbZUXgARLuY+g965Y+MLADA0GDH+8n+Fc1fNqUPdnodeGyqvW96mrnPbkjQKzIB0BVuSM549DzilxDJGm9wdn04UHrg+tb/APwmVlgj+w4MHr8y8/8AjtA8Y2IJI0O2BPXlf/ia5f7Xw/RnZ/YuL6x/I5V5YpLpmfCzIp3AlQMHkfN2H41DJLFBkeftk68S7dx78Z3fd9Pxrsj4wsi27+xbbceMlh/8TS/8JnbH/mE2v/fY/wDia53mGHf2jVZVi/5PyOLubqN4Uj87AXOwF1bb7jJ6e9RxzKLJhFJG2W3AF9jHg/p0/HFdsfGcIAX+x7TA6fP0/wDHaUeNYSSf7Jtc/wC//wDY1msdRX2i/wCysV/J+KOHMtqFQI0cdxgjdBtD4HBG7/647mozPGEe1NzGgZRuUszBl67cHrgk/wCzzXdnxnFjI0u0+m7/AOxrotDvrjVo/tMtjBDbFcRsOS/04+7WtLE06j5Yu79DGtgq1GPNNWR4uL8TTr+/jcbvmCtn/wDX2rq/BOofZ/EVmwPyTq1u+P0/VV/Ouk8eWCCys7iMQpGkhQqEGSWHB/DBrh42a2lWWHCujB1IHRh0rptpc5D2/GayvEekHXNCuNPWYwvJgpIOqsrBgf0pnhzWjremm4ZAkiuUdQeBWzSEeOeKNJXwittb27+fBc/IEL8q27pz0HzZ3Zzx71QjtZTC8qIkDJlSZIztiBHzBFzuZmx97rz0wK9wVFUfKAPpS4oVk7g9dDwnUrkWejizsrW+hLfu42a1MeR/sZONxycbuetQvZRJAkQjcuq/Njdg9PvMc5I7+pHtXvZFRzSx28LSyuqIoyzMcAClWftNWFNKOiPBJZ3vFVbdJP3K+WBhsvggY3behHGcetJJaXlzFEsoYoqL8ghciNgeql0zxxg5r0a+8dyi6ZbGGI244Vpc7m98A8Cqw8eaj/zxtfyb/GvMeJw6duZ6HqxyzFSV+X8ThbTRrgJ5f2SeAruXc0TZxjGQQMD2PP4VLFa3karavaXaFTlf3En3SuM8DBz357V23/Ceaj/zxtf++W/xpD481H/nja/98t/jR9cod39xX9l4v+X8TjYLSdoXC2d48cSnnyJAX9MfL7d+aivbXUJ7SPbpWoygus6yR20odSq7lONvr2HNdsfHmp9orX/vhv8A4qj/AITvU/8Anna/98N/8VSeLwz6v7g/srFv7P4nnJ0/VRM+dG1ss7BRGtjIqfMfvAhewCgip9Qh1WK58ye1uYD520vPBIiy9+Cy4Pfoc8eld9/wnWqEcJa/9+2/+KrmvGXimbUrWxt78hY/PMgEClSSq9eSem78an2tCbtFu/oY1ctxFKDnJaLzOe+1lLRZBkPv8zc5LY27iGJOOpxnt7VU0typJjMZO0LLK7MjyyPnHQYZUXgKePnzTILuO4hR3ZplklW3+zIesjMRk5+mOlSJbGV1tvkk8kbZ3EYIVzuy3memflLfd4HNXBNbnmsf4huGuvD+rXD7T+6L5R9qM2NuQoODgVl+FSqWc4dcfvm5I4/1a1a1qeceG9Th2NEhhihZCxRiTIFZgOflYjIHfvUXh5fLguY8Ruq3LfXjYOPau2nJuLuVBHbadME1KFejyjei/wDAic/Sub1vT5rpGVrf7DAt/vM0q5zG8jBWDfeI+bcQR2XHUVsaCzHVJZXYsrAKv0VyvH1OTXNL4rvdLv8AUbZjHKIbuXyIPN/dszSkkHZyxHB+ZvlIHpit8O4KT5hV02lYyry+1K6t7bzrdFubWJhJO5y0xLHaox0OV+p28474015POxdgDMo+aTkklj7nv6Ctm8hvWRNQW/t5DeTSJaCMMskqb2VptnVByRzknc3rms+HULeyihhlkzGGVpYD9+PhuQcEFfmDbeDxjHeqqxu9DOLLdzFbuiNG5SQD51bJ/L1rHNukLD/SA4YnO3oB659+a3mKQM8U5YThQW8wAAH8eT+lU7iO3Vlmd/MLMzkOpTGfauVM0KcbBm8vYeORjp+FW5LoKkDxPtZRtPOw5yf05rPlWKRlWGSRu/Izg9sYp8ySRqrvw7FlbPyk45OR78VW4Fqe9nvWRjuwiOuzoSoUkqT/ABA4qvDZQ30Kr9pzcNJtijdiPl9zjHtn6VKIN0KzbPmwduDwc8Zz9CarAlZkcRk7WztUdfw/L9aEAsW6C1d0eaO/EqA5YBQnJP8AwLOB6fWn2yxqiB4ZJok+8FYL6H0rod9g9osKRvv3b5iDl1YcqvJBxnvzjvVINBdOnmRFgoKsm7Bx35HBx/WlzdwJ0EENrCk6DLZbepP3OhUnoAOD+LVWe3jumlitt6bCySMrZAHZvow69etV7lhb3JlhvGBUMojLAbfbA6DH1q3Fd6fawiXcZJpsbECcKOAc4ODtx3AP51IFKGUWuDDz842kjJUj+76dxVG8lLq8SRkgHPA61sXipcbblJnZ0X5gFzu/H261jtBcSsBBH5rkjAzgY+vaqQFKJ5IpEj2lQp4A64Na+nm32SxTmP5hgFxjBJ+UH1H/ANeq8d1fWMrwuUQqDGUAV+f7v0oN5BFZTmZT9rLDygh+TgnLFv0xV7gakUcF3Gf9Gnd85cMMY/hGOCf84plyWiO6K28qGM7MGJmZD/sk4/ziodAe71a+ci9EWB8ueMf7I9+n1rR1ETRZS6drqX7kUjlkVV4/hI5z3wcg1D0YGXcPPcbXut7hQPLEo/hPYHrj61bj8Qyx2cVsqxJCN0W0J91T68/N361jXUsyZCtGQRu/d8jPII9agR41fPQY53djVoRqyedOnl+dGxZ+meWUDjrnb/u08WgwA7gFhyCd2PTPGPXiqVtOkzDY+0ju/IAokuHmmwAoz949icdTQBrRxo7i3jiWZ5HGzLAk8crjvnjvn0r0W0Yixtxu6RKD+AxXl9msTBZZnZTvC4CkD6k9ua9Rtp5LuztriV0kd4hlkHB2/L+fHNedmaTpxfme/kL/AHkl5E24n+I0u4460m3itbQdCl1i824K26H95J/Qe9ePTpyqSUYo+irVYUYOc9kWvDPh+TVrjz58izjPP+2fQf1r0kBIYsDCoo+gApttbxWlvHBCgSNBhQOwrP1a6cKbNAyPKvyybgB15H9Pxr6TDYeNCFlv1Pi8bjJYqpzPbojTjmilXdHIrD2NPrz+GxvHvpRbpKH34wMrj611EEE+mWE1zeXckpjjZyuflXAz9TXQjkaR594v1DzdW1OdTxFiBPw+U/qWrO0fSp9XuzbWzRqwQuWkJxgY9PrVySygudFu7m7i8x/vDLsuXJ9vdq2PAFvtuL2XrsiSME+5/wDsa0WgmdlZ2/2Sxt7bzC/lRqm8/wAWB1qU9KdQOtBJCsgLlRnj2qrqWr2Wj2bXV/cLDCvGTyWPoAOSfpSafp1zZ3d7LNcpJDIw8iKOPYEUd29W9/avKvicl+PFEU0yTGw8hY7eTb8iv8xdc9mPH1ArSMbhY9Y0jV7XWrBL2zk3wszKMjBBU4II7VkeNfEUnh/SEktjH9qnkCRBz07lsdwP61wGheMf7C8LJZWUQa8eWR3lk+5GCeMD+I4A9vrXNXd9c6revc3lxJNKf43bJ+n09hU88VLQ7I4Oo0n32X6nX23xK1ZbRozDbvPn/XMDj/vmsXUfEmr6mW+1ajO6H+BW2L+S4FYMJxJg1M5wpNc9aUpTPby6nRjhnUcfeVyF3yxbpk05F3fM3I7VATkmrKcKv0q6j5Y6HDltGNeu5T2WpHIvlsCvANEspZFXOe5p0o/d1DGNz+w5og1KN2PFQnRryoUtpE1vIVk3KxVlGQwOMVcvNc1O/tBZT391Lag5ETyEg/XP/wCqs5xtkYDvUqrtGO/ehz5ffQYahOv/ALO1ZRerNbw9baA95/xP5bkJ/Bt/1X/A9vzf0r0+w8KeHUuLfVNPs4AUB8uSBsxvnuQOCw7HqMmvGycDPpV/R9c1DRrnzrC5MRzlk6o/+8vQ/wA/eqjiakleWxpi8voqap05WlbY1/H+tPqfiE6fAWe309vJSNP+Wk5wGP6hB/wL1qzefDm9i0T7VBcma/RN72uwbX7lUbru9M8HpxmsrwVBHfeNLaS6kTIaScB2x5kpJIA9Tlice1e1hcDI4xXVJxSVjxJJxPL/AIYa75d0+lPJmCdfNt89m7gfUc16NqV2LK381mCjOOQT2yegPQAn8K8f1+3fwv49kktV2Isq3sCj5QVfJZPpuDD6VW8ReKNT8SRm2v5QlsWDJb2oKbWHQhuWLf49KqVNTaJaO21/4lWmmxm301UvL/HzHP7mI47n+I89B+debXOo6p4i1iBrqea9vZJFEaAE7fmH3VHCitvRPh7qWqyedcxnT7Q8gH/WH6Kfu/jXpNjo2leFdMmngtgkcEZllccu4UZOT3q5RhTjbqUrLc8MVtrH5mDKx/MMavWWmzajulMhSMnmRuSx9v8AGnQRvqN8DKADKxd8dh1xXTxxLFGFQYUcADtXz+Pxbo2jH4j28swCrtzn8Ke3cyf+EdtTHtE04fs5I/lisiynez1BQshDB/KbacBhuwfwrptQu/sUGV5nb/Vr/wCzfQVg6fafaLxF25Cncx/z61nhalR0pzqvQ3x1KjGvTp0F719TpQFHTvzx3rqvCvhc6my316hFmDlEP/LY/wDxP8/p1Z4X8MHVpFu7tSLFTwOhmP8A8T7/AIU/UfinbWuo/ZdNsfNs4n2NcM20Njg7FHUcdTj2rPL8vdV88lp0NM0zPlToUXr1f6I9JUBQABgU7iqGkarb6xYpdWzAo4zxV+vYcXF2Z8wRtFGzh2RSy9GxyKkpaKQDeKXilzRQA3gVS1LU7XSrJ7q6fbGvAA5LHsAO5NLqWpW+lWb3N1JtQcAAZZm7ADua8r1nWbnW73z5zsRciKEHIQf1Pqa48VilRVluejl+XzxU7vSK3f6IZrWtXOtXnnz5VF4iiByIx/Unuf6VnZFOIPtRivn5zc3zPc+0pUoUoKEFZIaTSZFP2mjbUmugzPtS59qdtNBFAXGZz2o4Pan7TjPaup8L+FDqTJe3yYs/vJGes3uf9n+f0660aUqsuWJzYnFU8PTc5sj8L+F21Rlvb1CLLqid5v8A7H+f8/R1VUUKqYUDAAHapUQIoVQABxgUtfQUMPGjG0T4nGYypiqnPPbouxxHj+bEOnwYIDO8p/AAf+zVwz/dr0Lx7aGTT7W7UZ8mXY3+63/1wPzrz9hwa61scZ0ngLU1tdXlsXbC3C7l/wB5a7G/8UaXYBg0/mSL1SIbiPr6V5zpUzQJK8IAniZLlSOp29V/X9a9C1PQrHxDaR3SHyppEDJOg5I6gMP4hUlGZZeMpdS1H7PDaGNP77fNtXnLHkcDj8665NwjG8gtjkjua8zs9Jv9L1Q3Aa3jmt3ZU8zdhuo3YHOP5/rXZaXqDw6RJeandREqfndOEXgZCj0z070m0tx2vojWubmG0gknnkWOKMZZmPAFeZeIvEkutymKLdHZKflTu/u39BR4g16fWp9v+rtEOY4s8n/ab39u1YpXHevDxmO9p7kNvzPqMsyxUrVavxdPL/gjPwpfwp2PekrzT3ROfSlGfSlzSg+9IQ3n0pefSl/EUuR6igLjcH0rD8RyNCkE6s6yRK+3GAqlyqqWJ9x07112kaRc6zd+TAMIvMkpHyoP6n2rG+JaW+iatFaW0MMiW1gkjGbnJaR8kn1O0e3Fd+EoTf7zojws3xkFTdBPV/gc5p7Pa6ktvLHMzKqy+Y2BJj+LnH97I445qqIJmiR7ONFlukW3gWQNIXY4ChsJ146jjOakkmtkup8K20BI2bzEYSZ4XnPXDY5+varUDMktvbOrM+ZLZHEpEYlK/wAQOC2Bjnkc884r0E5JnypX8WJDFpM9tFcBxHcRA4c87mxu2kA8sG/CoNDVIo7poTiIyMQM8g4RiBWdfReR4cd/KG6e4t1/14kCIgZgBx39ck8e9WfDErbryPbwsikHGcjaoP8AKuqGkS4He6PGv26JDsEewoMdFG5Tg/n1rzzxBZsJZrwvE0c91chZVlViHV2JXaOf8V5HNegaeNt2sfXKFSfXGBXA+L9TE0rw4jWXT7meD7RsVHaPgKN/fnf9AAO9TrdWLnsYZ1S6igS2Rxt80zqRENyuy7cq2N3TsOKtBIdSWWS/jeMWVquyNPl3RoQqqF+9nByepPWsgXDuEkyHfdlCSRuORyT3+nvWrY2cWoyCK6a6F6ZN775DEmVOCmGyc9OcYGcYrWUm1a5ikassoXbNLtYTHZG6MuMjGfXaevHFZF2FaYv+7OchgjHr7gmp/wC0Liyu2GyEqV+ZGhU7h36flVC6mjuQQF4bJ2Mwypz0B7ioSKYyLEfz8qd33eorRS3hvJWHm7DtB6ZXJ47dvpWErMPlycdMGrcc7hSeVXaFwpPzc02hG1eQvbL9lDqFxs6dG7Y9qhtkhS1YzJvkmuY1VSpwQoYsp92yAOlIbxisccyxy7GYBv8Aexz9RzTrO9ns7zzYCknUNkFs/QDrS1QD/s7wrFCjtC53hJJCOeeBz1HPH41VeO4jluBK6ER/IzCUInA6kd+nYE1pX2ozXbI7KrIj7QOn4+3071j6jcO6SyomCyc7VwEOcZz3pLXcCGa4LxhvJMqqfmKru/M445pbSVo5WURJj5pQkq5AXG1vy/Os4ptl8kuW4AkKev074q5amRQiSuCJFbEhGQ3sD6nhee/0zWlgJQzmchPNCqdsSI2cKT049OuO+KSOZtN1ELNtaQHIDrkfU4qu0s0cjK8YLqxVFR8jr0B9vzq1qJQeW6gspiUMwbA9sce/8qYFG5heWTdawyF5WJOTnnq38xVl4JPsUEPlkTFDFsYgY+Ykc/j1/wAKhhnneEJHPJGdwAwxAx1AH41a1eOe1uBBM7eZGkaFi4P8IYZx069O31oQDLeJtLliSFo5ZpfuzIwKEf3V79e568Y45OtcajFqke3OwkMGV+g5H5HPb6VysMuyfesjRynlXXjnHGfatO8eW+iRzBIFSMKW+8rEe3bjP5VLQgnhi3OZZjHMp27Sm0fSsqWfadgI28jPrV8Wy/Z4pIbiNnwd+18HHpVaOPZPvfawzwpXpVARqdhym5V6cjNXUnYjDDLFssR/OpJ5PKtQ0SwmKVuQnVOPustQW8imMqFxls5HGB6U1qIvxTMVMZYlXYHbnvXpXh+XzNCtyh4VnXoQPvds15XExZ9u7B7N0r1LwFDcappcVpEN85uHHtyAST7VwZhBypq3c9nJakYVm5PSx0ukaXc6vepbw8DrI5HCD1r1WwsYNOtI7a3XbGg/E+596g0jSoNIskt4hknl3PVj61oitMHhVRjd7sxzHHvEzsvhW3+YtYviYrFo0lxx5kJDJ7knGPxzW1Va8sre/h8m6iWWPIOxumRXajzjjtJ8WeU6wzI7qTjCfMR9B1rU8V6gv/CMvs3j7SyxDcpU4PJ4PsDW/b2lvapst4Y4l9EULXHeNJvtGq2NkDwimRvqxwP0Bp6X0GYF/wDutJtrfvITI34dP1P6V0PgePbplzL/AH59v5KP/iq5rVHEmoNGDxEip+PU/wA67LwnF5fh2Bv+eju//j2P6UxM2zSim0v86aJCRgsZZiFAGSSeBXj/AI68Ux62r2FiAbJHVzKRzKyngj0Udu5+lanj3xX9pkk0axk/0dDtuZFP+sb+4P8AZHf1NcAwLZ75rGVazsj38Fld6Tq1Fq1p/mQBv9HwPUinRLtSq6kjCE96te9Kr7qsup0ZYvay539lWIJPkmyOnWpZW+Qf7VMnXKBvQ1Ez7okHcVcVzpM5sRN4eVWj/Nqhp4xVlJF8sZOCKilXCIfQYpqxFgCGGKqXLJanPhnXw1VqnG7aHPIZX2gcCliOGIqQJ5aEYOcd6hTmSkmnFpG0qVSlXhUqv3mxZD+8apPNXbkn8KSRNw9xUSIXOM4NSuWUdeg26+FxEo01fm2HFzIwAH0FPjbhvpSqoXgDFMf5W3DvQpRk+VBWw1agvrM3d9fmOi4r0nwj47Znj07WZsk4WK7c/ksh/k35+teax96Uvgcc0nzKfumlOOHngkqzs9bdz0D4pWRFxpmoBTj95aPx6/Ov8mFc/wCFNcg0XUFM2nwz+fKifaC214wSBxx0/KsO6vb2+8r7TczzrAmyJXckRqOyj/Jqt5wxlXIORg+h7V3wu7NfM8JxafkfRiAbcisbxckjeD9ZWJtr/ZHIIOOnJ/QGszwX4ui1y0W1uWVNQiX51z/rB/eH+Haul1C3W8065tm5WaJkI+q1EouMtTPqfPdncT2c/mxoG6jDDtmtCTXb5lxFBHEf72Cx/XisWK6Pkx5ODtqxBeQqw86MuPZiK4q2HpTlzyjdnfh8VXgvZxlZMdHFdXc5+/JIx+ZmOfzr0jwV4PN4iy3GRZq2XboZ29B7ep/AVzOja5oC3cSaiXtrYcssaZL+2R0Hqa9a0fxp4c1GVbLS7pZHjjyIY0I2qOwH9BWaw9WsrzhaK2Xc6KlWGGj+6lzTe77ehtX0RXSbiK3UIRA6oF4A+U4xXzBArrZwnBGIl/8AQRXvHxN1p9K8FzCFyk99ItojDgqG5c/98Bq4zwN4Hk1GHTdevJk+xsTKlmY87lGQmTnp3xj0rvpxtG/Q8rcofDfxDNputLp8zkW1wflz/C2Ole6K2UDe1fNfiCyl8KeLrmziYhbeZZbcnnMbfMn1x93/AIDV3xF8QNX8RqlnG721lwggiJ3TH/aI68/wjj6121cN7VpxZVrnsWp+P/D2lztBJfLLMvVYRvx9SOKwG+LmlMFMdrdAMAcOoyPrj0rg9L+HXiPUYVkkjis4zyPPb5v++RXP61ZPomrz6bJcpNLBgO0akDJGcc1UcLQXW9gsj2ix+JukXlykJd4mboZFwM+ma6a51u2stPe9ncCJeBjkuewHqTXzKs+7Iya6vT9XuL23hinmLiGJUjXoFAG3p6/LyeteXmFSFGk5QWp14HCrE1VBuyOh1nWLvWr03Fx8qLkRRA8Rr/U+prOw2aTzKA5r5Kc5TfNJ6n3FKjGlBQgrJC4NJhvWjfgd6XzB6VBrqJhscGja1O81doGOc0nmCjUFcTa1IVbFSCZVHK8V1vhfw0L3Zf6hCRa/eihYf6z/AGiP7voO/wBK1o0ZVZcsUcuKxcMNDnn/AMOReFfCT6gUv9RT/RBzFC3/AC19z/s+3f6dfRRGQABgAdqBIqjAU4HtR5w/umvoqFCNGNonxOLxdTFVOefyXYXy2o8tqPN9jSeb7GtjlMrxHEk+jT2TOBNcqUgU/wAUiguAP++f0ryvOQGHQ817Bf3CwWNxcFf9TE75PbCmvHl4RQTzgVURMlspxb3kUjfcV9r/AO6eDXoHh+a8k0GewtHiF1aSGJTLnGw8qeP88V5u5+bHrXY+Er8rqlrIzcXURt5CeP3icj8xmk9GUtiGXQteutRe3eM8MN0wbEfPOcnk1b8X6Qllo9j5MxSC3ynllSQ7n+I+h68mu8rM1+w/tHRLq2Ay5Tcn+8OR+orLEx9pScTowdX2VeM/M8jKnNJsNOWUYHHWl8z2r5U+9TYzYaNppxf2pu40D1F2ml2GkzS7jQLUNnrWronh+51q42oTHbqf3k2OnsPVv5VL4f8AD0+tzB33R2SnDy929l/x7V6ba2kNlbpBbxLHEgwFUV6ODwTqe/Pb8zxMyzT2P7qk/e/Ii0/T7fTbRLa2jCRr6dz6k9zXhPxUuLmTx5f/AGa6khaC2tIsJ6OWyWHcc9PevoHGBXzt8SL2f/hMtamtERGSWOJpZF2+aqomY93H94cfr0r2anuxsj5WTcnd7mBcFp5TIZGktm2shCFgzKcOFz3+X04J702/XyJmiLzQxeWzBhh3k3sM4IK7MMVG0c06GBpb6W7kSf8AdEl7cL5UiyY+d/m64GCdmeueaWGe31SaP5LeOFVYm6l6h/lynOeTn8q5rtMm5F4gs4tO0tUead52vFQb8EOqo25sj0ZsfjjtRoTeVc3iPt8p2jUv6HIAxUHiGKM2mnoTDLOJTGWi+URpgnbj3JDVP4bBle8G772FGepYbGrop607mkdzvNPJF+mRyEcn8xXn2q3cmj+NdTlntFVUuZGj324bzsHK/wCsB+Vj1K49q7/TZc3cuEBEabmOf73QD8B+teceJhHH4w1b7TP5sou5N3mFioXdlV4PPXBHbtSRUyqPKuElut2mKfNXMckm3c33t21eQBznGB03c1UsriaScRRwksWYyFjznHUsf94/nxUTyW0ckaRgqXVgAu75mzwMdieOP9rmoLtXitd8EjnAAkdQcDcOAfTJBHP92mldGR0l1eSXivewymKVNzSjzNxOe/3fQnPNY0NjJcxkxw3DlcDKRZ5P3fwrpYLG4sRcRX9vFe6UZhA11Cx8otwQpPVSc8bu/Ga6+bw7bN4dub6K/wBt6mZpIzkrNGi8ZU9Dk5yMdD9azdRQNVByZ4/LEFmZA54zzgjB+lKk3lxqd5y5xxz+fpV2fTpUvJkiURqrZCnP3T0p+ppujjLIv2kqIvNyRuX3HRueK2uZEK3Jik3o4IxwBg1NFfBJ0l24O7dgr1HtWfHaTC68g4jmzhkkGOnarkOnsbx7eSaCNfKaSLe/yN+PahoC7PdhpDNb3jF5VYtF5eCqn+Hd3qHfdRoUMXm2nl7smPzNrHoOnB/wqK2i8pv9IjkjOzq3OQe47Hsanh3hJI4rmdbvkbt3yADncQOc/T1qVoBnTWNxaL5zRFYpAVG8YP8Ate/1+tW9N0iW5eKQzPa2TSL591KuUUZxnGeSPz9KUXBmhD3Mgf5NuCvOMYqNpVhJkifgKPlBOOmfxp3Arm1ex1icOFlhjc9AVWT06889fWp5kQqkULSOhXam7qPpV7WkUaxPJE7iOZVcEtuyCPX+76VXNsrZRZS21M7k759e4qrgRww+d5ccXzyKSzHpuP8A+rinXVpMzlpCqp3PfjrxT9MPl3SyxxtNGrEhgPukdzjpVrWLqZICs1vOob5t2Tgg5wBkA461PURyl1IokDRIoGOwq/aalF9leC7X5GztkXO7/dPPSklS1dkEabgVVclMlfpzzVIiQFXhj49ccVdhF6HTZbhwIUY7sFP92knszCzKjmTbxgA4YevNLaT3Vq/7re9xOCjAAk7fTHv/ACpryTJdKZYXU8YEgJBPv60rATRtutnmk2GTPy5HzA/4VDtjWQDBU5xye5p8ty86jIiRYxyEXbn8PWpoIyFLJ8wPDkclfwzzTsA6ON2YfdbHHHFe9/A2OFdAvxt/frcAlj97aVGB+hrxeCykt7oSLPE6ryoZT93HB/PFewfBVWS41cYjUOsTFY+mRuFVYpNrY9gopMCjAoAWiiigBDXm9zOl/wCKb26PEcb7Ac9k4z/Ou91S7Fhpd1dkgCGJn/HHH615ajG20S4dv9ZIPLz6s3X+tNDiVRKZ5GmPJkYt+dekaGgi0KwRenkK358/1rzO3Ydew5r0XwvKZ/CejytwXs4z+lANaXNiuW8ceIjomk+RbPtvrsMsRHWNf4n/AKD3PtXTMyopd2Coo3MxPAA6mvB/EWtPrutXF8xPlv8AJCp/hiH3R+PX8airPliejlWE+sVry2juZytxXd+AtG0rU9N1CbUbC3uXhmXaZU3FV2ZOPyrgFOK9K+GB36dq8f8Atp+qMKxofEfQ5peOGbTtt+Z51JHaSeIzHak/YJrwCIqCpETsMYz0IB/Sr+vaNPoGrS2M7BwB5kUoGBJGc4OOx4wR6isMsyLE6nDKikH3A4r0r4iql94f0TVQAGlyh/3ZIw38wa6atpwTtqeRgaroYl0ujv8A8AzpfCVnZ+ALjVNQlKX9xCHtsscRseY1Cj7zEDnPr7ZrgQTuX3IFeqfFBjDaaNarxGGkbHuqIo/RjXlU+VVyBkr8wFa0ZK/s7aHHiuacY4ibvdsvOvDIwwwyCD2IqGJsZU9BzWrroj/t++aE5ikkEyfR0V//AGasiUYbjjNc8dZODPTxl6Xs8TA9M8a+DrCx0RtR0m1WA2p3TohJDxn+LknlTg/TNcHottFda5aQToHieTDIc8j8K2rrx7cXnhK40yRM3EsZiMmegyvP4jdWT4UHm+I7M44DMfyFehDD8kjxpV6k0lJ3tewurWottevrO1iO2O6eKGMHJxnCjn64qC4tX0TxO9lfwRzLazKJUIJV1IB49eDXS20CzfFkxOMj+1Wcj/dG4fyqp8Rdp8YTEADFtAGx3O0n+v6VyUHGDk2j0cwqSappdFc1Ne8MaTPoja34bkzDGpeWAOWUqPvFd3zKy919M9K5VIbGTQ9ReWdY9QjaJrRSx/eLk71wODxjrXSfDe/B1WfSpmBiuk3YPcj5W/8AHSPyrjwhjXyifu5Q/hx/SomlTnGaW5rhZyxWHnSm27bPy7CWVuLu8SB7uK1jYEtNL91frWhqWnWdh9nW21aDUXdWMrQDCxkEbR/P8qyUO2QEVZzn3q8RWtBRijLK8NTqtynrbp0EqTSBGmsQxTIJLaf93Kh6Mp9ajpgcpKjr95G3D60YCs4TaexvnFOMYxaNPXdDvvC19FcwzSG1Zs2t4vVT/cb/AGvfow/KvQfBnjiLWRHZXzJHfrxjoJP9of4Vf0V7TxD4aWG4RJ4JI/LkRxkH2ry7xL4ZufC2sQ+W8jWckqm2nJ5HzD5GP97+Y/GvSfJNOMt0fPxim7MyFEVvcJE+PkbaePfFdVB4StbvTLzVZbxdN0+1jLSXTrvTcOwXufp3IHWuc1yFV1O4/drnzGxj61e1jXLrV/DGmaUwS3tLYNJIsTH5mXhMj2+ZsepBrzKWGqVOWs3ZLddzunOFJTpJXd9+xh22LmdYoUlmnkJ2RpHudv8AgK55/lXX+EfDurzeJrOVrK5s0t2LtJNHt7fdAPrXZfD/AMHt4ZsPPmiiku7tQ7yryVUjhAfQfqa7uOEO24Jhu9dlSdpX3OF6M81+Mc3l2uhxu3ymSd8f7QQAf+hGu78Lxw23g/ScOFjSxiO5jgY2Ak/zrjfjHo8l3oWn3QOEtbo+Y2M4DrtH4bgK4OXxLqz+Do/DImSO1RTG0yk+ZJF2jPoPXHUcVMYc8Er2EN8b+KLPxL4mkvLUKLSCP7PC+OZVBJ3n2JPHtz3rqvhNp+nXcl1evEj3Ub7Ududi4/h9Pc1xln4Puz4T1LX5LZPs0SL5BZsFvnG5h7VtfDOSez1026FQkg+cemK73SiqclB9LFWPeFSNVxkYryr4h+Bo2uJ9dtLuKJpCPNil6SN0G3/a9v5V6LqeoWei6VJf38xjhjA6DLMT0VR3JryTU/EjeI9RDyiWNFyIYcgiMf1Y9z/QV4VXE1MPFzgrnTgsPGvVUJuy/rY5ePR5QPm2j8a17K1S2AXIJUZz65JNW/s4Pc0fZxnO414FfG1K6tNn12Hy6hQfNFajw6jvR5i+tN8hfU0v2df7xrj0PQF3r6ijevrSeQvrQYB60tAF3pjik3rg5Io8lcc8V2PhTwatyU1HUUPkjDQwOPv+jMPT0H51tRoyqy5YnLisXTw1Pnn/AMOJ4W8Lrd+XqOpJi3+9DCw/1nozD+76DvXoPmx4x6Un2WPOSTz70fZY/f8AOvoaFCNGPLE+IxeKqYmpzz+XkL5yUnnpR9kj/wBqj7JH71scweegoNwlL9kj9/zpPskXv+dFgM7XpFk8O6kqnBNtJ/6DXlLKc5r17ULBJ9NuoUzukhdBn1INeMXmoRWVos82fmGFQdXb0H+PamnZNsai5OyFupordEeZ1QM4QZ/iJ4wPzrQ0u4eMypGSJY2W4i/3l6/p/KvPL67nv5Xllf5yMKF6IPQV11hqA/0S/HO4K5A75+8P51lGpzt2OipQ9mlc90s7hLyzhuYvuSoHX8RU/UGua8G3QawnsScm1k+T/rm3K/1rpRWpztWZ5F4ishp2v3UGMIzeag/2W5/nkVl7hXd+P9ODx2uoKvKHynPseR+v864by1r5nGU/Z1mj7jLa6rYaLe60fyG7hS7xR5YpTGtcx3XQm8Z9xXQ+G/DcmsOLm4zHYg9ehl9h7e/5VZ8NeEhflL2/Qi16pGeDL7n/AGf5/Tr6GkaxoFVQqqMAAYAFephMDze/U27Hz+ZZqo3pUHr1f+Q2CKG2hWKFFREGFVRgAVJuHrS4HpRgele0lbY+YvfcCeK+bfEd7Z3Hi3Xri6Z9vn3KNsQurIrbQfROV2k4Oa+kjXyPqTeXHfeZAIZJpZJHlkVnCMZGK7sdj+OOKzqq6SAV7uzj1hL2C5fEbFIoypDqewLNuHBzjHsDWzcLFeTW8tqotfJmAlDjJTdk49FJPr2xXG3Np/pD28NyJnZVcHGwZ27myTgYXn61ZsJo5fNjs3KsELGXO5mAAP3FGGAPOfSsZU72aZKLeuDHkSJ55t2djE0yIpJ2rn5U4A7/AONaPhaYpFc/ufM8q4TA3bS4ZV3gn/gKkfSuZnmhezgWGTf5ZdpZBzmQ7c4/2eenua1fDjut1OnVGVc+m4MuD/3zkVulaFjSO56LpAkUtJIDvdmdlzkLnGB+WK828UC3Hi/WhMrFmvJWDxy56/dz9DnI68V6VYsY2+Z2+6d5/vf735VxGp2GnXHjHXri4njN5Dds8enSIGFyMDIHI57/AJ4yazT5dWXNXRkHVLGPw61iNMDsWG+5zucMy/MTuGBkhQMY+7VXT7WzmhgiQhJ3JBkbczSN0Cqo/DB29zWnqTptEdr4eFkszlYEO5Zih7yyZ4Hbb0rmh/oyTRKu3nEoZMNkHIwP4cf1OacY6GTNeW4mt8wRyyi1uCsrxhyA7AEAkeoya6bRNd8y1NndBiNnykM3PY/oa5ry4ZLYk78RDeOei96fYPKz+TbmQSKWwFfb5keMsnv0z705wTKuzpr+wtdQjklgnSKUcoZDy+0gE/KOR6ema5aKGF3aO7jYq3ytlmUL+f8A9anxyS2txHdw7CsvzFDnBxwQffj9aY181wUXOIVDBU2jPtuP8Rxxn2pRi0I0bzw5bqrJbXk4n2boPNYYLryEJ6ncMbT2IIrKto1eCTYAbm3k82RTw0inhlHbjA46g11VlIt9pP75c3FqoEh/vRcYb6jP6VS1O1ZBNd2yAMp/eKP4gB8/49GHqN1O76jsZV1HPOw/essmw5AXh2weeO7DIwfSkt50kT/Uxr8uUDHATPX8KqyXG+NCfqrDqKsQWbBEu3YGJVZ2L5bO3GRgd8sB+NPoSU7gTNqJgWMeXdtuVGwoJDdQegqGWGZS8Ls0bI3IHQ5/pjpWjf228Qwm6US2r/u50Viowc/L7fdb8e1Ovbgap9juwrb449lxPE+/5gwAbHHqOvXIwR2Y7EbSxXUNsy+bJcMkaMAuAeMce/A/nVUzpHL8wIOSuParF5bPZzxtLIJY5wZY5omLJKucE4PKn2PSobcpLcvG2398pUl13FTnO4e/H607CLNubWGYz2Mknmom7dkgr8vr9e1Ymp3U804laZmZ87l7A+3t/Ktu005v7O1O6SXdDbxpiQcCQM+3gd6zms7W4kdRKQ4XOGbk89VH9KEBnx3bpOkq7cKQdjZxx+vUdqtSOsV/JIw+bdkAnO1v64pX03ayrFHJKxQkgr029Tx+FMiiDHJ5IGCgHP1qybGppl3PFqUF9Gd8ybni3sflwDnn1wDU95dRnZvicArgbgT9Kzora5EkMg85SMug429e/pV8R77d7ljJv8w/JJn5V3cnnOevT+dKwzHljKAkN8vqFPNWbU7lLJkNuGDWvJNNd6WISkX38koCGcL0UnOP8+1V7FLhS8tiYULFciWPI2+x6+tFgN3TNEvbqGFEuIE8o7sSkgIemdwHTBr0j4OyOniDVbV1I2wD5h0JV8HB79a8cuB9njaGc5WRt/yZ4PbHPQf1rvfAuv3OgyLqFpardLJE0bxqeRz32/d5HfrVXstRo+iqK89tfiS9zGd+m+TIP+WbSjLcZ4NPT4kq5wLDPIGfNUAHrjOeOPXFZe1j3KUGzv6Wsbw/rJ1y0lufKEaJJ5a4OckDn+dbJq07knL+N7ry9GjtlPzXEqrj/ZX5j/IVwesTNDb2lshX5t0jBlDD0HX8a6fxhcefr1tag5WCLcR6Mx/wUfnXGaoXm1aUjlIwsSj6Dn9SabLjoiCVzFpt5JgAiF8BRt528V6j4aXy/DWmx/8APO3WP/vnivMZwkdrtuUzG+MpuI3DcPTnHbNemeGbmK88PWs8MXlRsXATeXxhyCMnk1HLLnv0saOcPYuPW/4GV8Q9V/s/wxJbxtia+b7OOf4ern8uPxrxomu2+J2oG48RQWYOUtIASMfxvyf0C1xFYVZXkfV5RQ9nhk+r1FFek/Co5XVl94f/AGcV5qK6Twp4pPhlrtvsYuRcKgI8zZt2k+x9aVKSjK7NcxpSq4dwhvp+Zy0vy7EOQQNuCMGuu1zVlvfBHhmy3fvDE0zr/dABjX+v5VjeKdV/4SHV/wC0TAYSY1j2GTfwPfFUEUhVHoMV21pQjStF3ueNg6EqmMcqitY9J8cK2s+DNJ1qBS6xYklwM7FdArE/R1GfSvLJWUs2CCD3BzXY+GvGV54fha1aL7TZsxYRl9rIT12n39DxXO63JDfazc3ttC8MU7bvLfbwfw4pUXCTcr6mWMw9anT9k4+6m2n6kfmGQI7dfLRf++VC/wBKY4DClUYApGXcjL6giuaU7VebzPchRVXDKEuqKaK291Ixg9DXWeA7Bn1sT84jB6e46Vn6tdWN9JFNa2UlvLtAlLOpV/lA4x05FW/DOuLoN+8ssDzW8ibXRCAwPZhn8fzr1PrdOM21sz53+zayTstTSuLpdP8AinLcucJHqa7ieykKp/Rql+J1i9v4hjuSMJcW6qrdi8ZYMPrtKmud1vUI9V1y+v44nSO5k3hHwSPlA5x9K6mx8c2tzpA03xBZvdKqgCZUWTdjoWVv4h6ivPhOLbTdj0sVhanJCcY392zRy3hK9+weLdPuCQERm35/u7ST/KqIYv8AOerZY/ic/wBaWeK0jvZXsfOEDZCCUAMFPbgmkrXGTh7sYO9h5XhpU4y51qysc+dt6c1aU8CnTJatDA6eaLlFKyZUbW+Y4wc+ntUY6YpYpwcY8rFl9CdGtK+w8moHJ3GpSaaRk9KxoOz1NM3pSqQjy9GehfDS8Rhd2fSRcSj/AGlPB/Jv/Qq7vUdNttUtWtbuJZInK5B7EHII/KvHPCuojS/EtjOx2xNJ5MvP8D/KfyO0/hXt+DnDcMOtdiqNu58/iqTpSi+6PnzxAr/2/e7flXzmCqewycVnbJTG6BxllIrtNVt9B8S6pJeWN9LbTTfvGhdVIzjPoO3asmbRLWGSeH+0yJoVDfPavsA/3h/SuurXg0uV6GLfM7nuXhXUrfWPDtjdRBcGFVZf7rKMMv4GtwIB0FeH+GtQv/COoFPtkMtrKBJJbpFMWORkMPlxnH513A+KGimREU+YXJAaKQOuQu4gsOmB61w1HG+jJsdhf2NvqFjNaXUKTQTKUeNxkMPSvMo/hLAmsK76hcSaeG3C3dRu/wB0t3H4ZroF+JujtH5gimK88gZ/Hp0pW+JOkqSGhnUjqGUjH144op1+TZhqTePY4bL4barFFGiRi3EaIowASwAAryTwLdpaeIxcTtiKNWZj6fh3J6AV1njrxhZeI9Jj0q0uUtt0yySmVHfcq/MANgPfaa5nR7XRNPkcahqdx5yjLLFDs5ztwPMHJzxiuiGJVPDySd5O5rRUHJe02NvxRqd34mWTe3lRgYt4s8JjoT6k9zXHadBdLqCB1ZWRvmB7V2rS+G1cqZtVzz0RG3fTCnNNWTw06eas+qbM7d21Ov8A3zXgRWJUZKVnfzPanWwDnCUbrl8ikA2OtL83rV0XPhvAzNqvTdyqDj/vmpAdAY4B1Qn/AIAP/Za4vqNXy+89P+2sN5/cZuG9aMH1FaYGhf8AUT/F0/8Aiaap0BmwDqX1LoAf/HKX1Gr5B/bWG8/uM3n1pCSM5YCtCSfw/GSCuqMR2V4yT/47S2ms+GrG7S5NnqNyIznbKVKA9cnAHSqWBqN62JlneHSbV/uOk8KeEmnMeo6mmIvvQwMPvejMPT0FegAEVwH/AAtTTvlH2OYEgHB7Z9aa/wAV9OV9hs5twx3Hr/nivYo0qdGPLE+YxWJqYmpzz/4Y9CwfWk2t/erz6P4q2ckqINNulDnCu2Av55qvJ8YNPjjaT+zbxgDhsL0Odv8AOt7nNY9J2t/epNp/vVxDfEe1Fh9q8u33Yz9m+0jzwMZ3FPT8agt/iZDci522Lg2673RmyxUHBZQM7sd8c0XEkd9tb+9QVb+9XDJ8Q0l024vo7WMpAcPH5h8wdMfL7545qGL4lLPIyR2Odp+Ztxwo7k+1LmQ7M6vW9bttDs/OnfdI3EcKnDOfT6epr5x1Saa51K4eZgWWRkUL91V3HAUeldnfeO9C1bULi4vNMvmkTKkm82rgHGEXPArn57vw1qV9JOLPU4DMQ23zwyjjGc59q5Zxqyk72SPRoVcNSprfm66fkYQroNCl36a8PeGVl4/ut8w/mauv4f0yKzjuWtr9VflQ04z0zzUcFnDbxu9grx+aBkTuZM4J6dMHrV0qbiyMRXhUjZHc+DtQMOo2Ts3yzBrST/eHKn+X516b2rw/RvMi+0wq5MrYnRj13r/9bH5V7Pp92l9p9vdp92aMP9M10I4pEOsWA1LSbm0PBkQhT6HsfzxXj5BBIYYYHBHoe9e3GvIvG+u6J4X8RyWl5oFxO86C5SWK5Kh9xOeM9mB/OuDG4SVezjuj1cszCOF5oz2fYzeAM5rsfDPhIz7L/U0Ii4aO3Yct7t7e351wdt8RvDkEwni8N3LPF8677rcPrgnHFbkfxyt5WVI9EuGduAolTI9zzxWWGy/llzVNTfHZy6sOSjp3Z6+AAMDilrzD/hbaggHSpM4ycOpx+tRn4wIGI/smTjnIkX/GvY9lO2x8/wA6PU6K8lm+NUcEzRtpEuVzz5q8/rTLX40vetKLbw/I/lKHctdRRgKf4vmPTjk9qThJdBp3PV7mXyLWWXA+RC3PsM18mwwxJHvvEgms7lAHlLNGzPtBA5IGFPYc13WrfHq5uNMnistCCGaFlWa4uB8pZeu1RzjPTIry1/FGoC2SGOxskjVFTKwdcDBPUjJ+nYYxzUSp824M37iQzwRzQvbz3EZCGJQWUr2aPcccEdOQT9Ky9asX1MJaxXDSyq29EWM7ULH5zK/AzxxtBzxVSHxRJHG6f2bb4dcMBbx4YZzggAccn8zUP/CTpF8selRQgAjNuzxE5/vDcwP9O1SqPLqmIk1Kw/st4oPIeI+SrB3k3efzguF/hGeAPatTRo3g1hG+Yx7tjqOjfLyOffFY134hsdRKveWt+XSPy42W8DY5zyGU5+mRXReH7iK6ha7Dg7mY7AV+UhehPX0/yabuo6lx3O6sAWzuYFisgbt/nrXJas9ivj+/KwT/AGlGy0yoX/ebM4VO/UD5uOp9q6aK6CsRwHUFlUEA4xjp9cc1wPiu8hh8W6n+5YTLe7y2T8w2r8uPTgHPWsUrmk9hl/qUv9kvMjFYZS0QdduGDdV6ZzwRn25rmY3Qv5ZkdW3ZVjyAuO/rV7ULo3CozfIATtTzNwjUknCj8f51nxxqZC+Fc+v9a0hGyMWbAmmhcMsrjB7MRn2/EVMWa3vPtERITfujOedpH/16r7g6bSOQevtTo8kFC3APr0qxmk0Xm2ySrhkkdun8Ljr+YOfx9qzzCpPmKNr9G9/erumzOJGsyCUkdSBjnePu4+vI/GomKpcMGB8rcQcdcdakZq6XfoI4Zhnz41MMsbf8tIz0Pvjp+VXNQJtbhgrkwuI9x9v4H/mp/CuatAvnrHMpZd2PkbaT9D+tbFzDqbWpXyor2OODdE6HZI0ZPBIPDYPXFS0NFOaCO3kkSVB8n763wOPvfMv+6R+RHvTbB47kzWG11jYl4A3VXHRcfTj6CpP9fGIX+eWF+FPB+625cdRkEfiKpR3xgdUmJdB9yQnkN1BP5CmGxYkZoZmt5UdEZ8kSDhWB27h6AZINRW1rHDdT5yE2HeoPyr04b1AOQfwI6VcnVLhEd5oy7RvKFJYBdzZwP6/jxUsuy3totQjcmZ1JVY3Vkfbndu/u5UfdPWp5irdSK/uLdrXS4FiXda7lkJ6sW9fyFZXlSNFdKseZFIlaQ9oxzwP5095Wu7lZN7N5h3Hf3J5/+tU1vc+TdHzlbKnh+hCH1B6jH9arYm12aV1DNo2iavpM6ESSTwNnPAHDY9jx+lcwHFzchbYSIFzgsQWHHzE+tbupWzXV5MkMkjxRR+bLLI+7A2jr+HQdea52wtvOjmdtwhT5nbGASc7RnsSatEmtaNOLhpYyjL/qzvXO5cfN+dV9MVPPdsZITIJ6/wCeatwEwWqK/JU4bHf/AOv/AIVV05PK1ie2dgQFdVYdDzkH/PrTF1LwgE8aDcy44OD94elTzrts3XPAQjrUds2Y09eRU8nzRN6kf0oBlOwlItVY9UmP5cGtUIEdlAA+n6ViWBzBMuejr1/3a2kbLKTQD2INQjR7KZtgLCFmU+lXPC3iSy0XTZQ6Sfad5dgOVlGPlXrx+XvUFym63dOxicfpXLK2YFIznFEoqSsxHS3niG+vX3F1jReVSMdMdDk8/wD6qUa1ePEN0zcDLknqAc/0FYSNnp3Fbnh/TW1jV9KsAM/abhImx/dLcn8BupxhHaxSbPprwZay2vhHTRcBfPlhE0u0Y+Z/mPH410FMRQiKqjCgYA9qo63e/wBn6JeXQOGSM7P948D9SKXoBwE1yLvWL+/JyhlbaevyrwP0WsJrq1tjFJd3EcT3EmyIOGbzJDzjC849T7+9XnH2bRtmeZMJ/j+gNcZr87p4gsx5txEtvbAAxRq4+dt0gYE8ggIM9vwpSbSujToal7DcQIZLm6iunlZMyQg4AHONv8I9BXoPw+uEbwdaIzqHM0q7Secls9Pxry3w2YpdQllhRfLZJFH7zcVAIwuf4sdPzr13wdj+zHGBkXJ5/BaUJXWpDPJvFVybzxXqs+cg3LIPovyj/wBBrI/n9Knun867uJT/ABzSP+bsa9L+FcIXStSn7vcqmfYJ/wDXrmjHnlY+0rYj6phVO17WR5jHFvPy9foa0INMZsbsjPpXuWq6xY6Fpz32oSbYV4AA3M7dlUdzXlev+P8AVtXZ47GX+y7I94z+9I937f8AAfzNX7DXRnkvPZP7H4mXNoEgh3EBTjI8xgufzq9B4cd4Qwi38f8ALNgxH5Vwl3qOkW0heUPcztyzM27J9STVdPF8cMube2hiHtnP51Toyatc5/7ValzKJ6P/AMIxIX2+Vg+lVH8Nv5uGGB1xu/rXPWXxO1G2QKl5cKo6DzMj8myK6PTvi5OpAuIIbkE/8tIV/muP5VCo1I7DebOWjQsXhiZvl2g47lgKV/C9wNwWHOOflkWur0/4mabezRI+lLsf77RSqzJ7lCASPpmu7g+wXMKSwiCSJ13KyqMEGsJ80Xqi45tNKyR4dJ4euFH+qbvkcVlT2UkLYbr6Gvon7LaN/wAu8J/4AP8ACopNJ0ubmXT7Rz/tQqf6URnc0jnFt4nzkRRivoZvDmhMcNo2nn62yf4VWl8K6CGx/YWmkf8AXqn+Fbwp8+zNf7bgt4P7zwIg+lLtb+6fyr3M+FPD3fQdM/8AAZaYfCHhw/8AMB078IBWn1d9xf25R/lZ4cUb+6fyo2t/dP5V7afBnho/8wKw/CM/400+CvDR/wCYJZj6Bh/7NR9Xl3D+26G/K/wPFNp9D+VIeK9qPgbwyf8AmDW34M4/9mrK8Q+CdDg8P6jPZackN1DbvLG6SPkMoz0LY6A0KjJalf2vQqtQcXqeU43BgOCRjNe+aNe/2jo9hfk8zwJIx98c/qDXgeecjp2r2H4ez+f4Ot4+pglmi/DcWH/odOjLVozziilRjJdH+Zy/hu0YWUDXVtMHkQRiRYHXhh2bGMHuP/r1VWLy9eWVflhmTlSqupYDywAB0AVUz77hXWaR4ks5INNge+aOSGyjt1AViWYp80i44+XG0Z77q5mbMDWsU0kqkxhXjTYF2DsW/wB4L8vf5snmrqSvGx8/Fe9c527gu3mnvHiiWJD5e4rtVcHACj0z29Sayr+1kkheEJvVY2EUSYCID9989+dvXPGBW/OzS3Wm2d5Ky2sSNdTYBxJ2AC9zyeP9qoo3Oo2t/q8KeXaXjApGD91AuGP+zubnsuT3qJJRSZSbbYs0A+zkMcxYXzCG4wAeeMY5wMelF6Ra3BhaGRcdnHMa+v459+tTGaSVrM+ZJJ5iKnJGCwOTuHbr+hp9yii8a6ubpWWS4kMUjRlvm4A+XjnPAz7Vhy3YFFLaOSTy3jIYfe53BV4IPIwM46/403U4Vs9LuisksjRbHMSuQGjHzM2M87cD/Iq/9m/tG7ayjEdxMyrFbj75kf5dygk4+XnOV456Vm6kEi0uW6ht91tbNt3wTcpG3EoZTwytnp1XaDVpWdhoq2kk1zcyzCJIZfmRFEh+/wDed2x8vyr6Z6eprS06cQ28mclTtYiTII47n8c8dcmsvSiLKz0vznkd502sx5GSvAyDhRwgwevPpWpEHRm2qFjgc7ijcA4PB54POPxpySYD/tX74J5UJYDPyoMoO2SBxkep5q3bblyQAwboc8YyfzrNH7u1RkiuH3H7zKM5H97nlun5irMU3lqu21uSo4XCDp68n61m0M0D8ikNjIOfu55qCYZiCgKXJyFkGR9fpTJJZDG3+izg45yF5x7bqrTXbxxGXypQp4yWQZ/8epWuNsfMTcXAUL82cjjBA9h/n0qBtu9kiOWUbjxvB9z0/WoY7p3iGE45GfNVQPTPWpY7OVwjmOLK7SrmX05Gfl55NJK25IyORvPaP5GOze5fAC845U+p7H0qO+WZNPkgjuAlxswoEYHPTgdsj69+RVuNJRFIJ3it/K2sZSC3TttGRjv+uOKjmnvEtXMbxnIZdhtDv28HGAcBSD1xxWgyvbWlmq2TxCISSfJ8hbcCFO7cw4BJxx1HtikuTskaKSbzbUzoJpN7KIhwBuGeevA9etE0lzA8I+zRusysJRcEHzio3bydnQepYntSadA2v6dqJtxDbu06qsbMwQKm5iFOCdvynHQ5bmqbS1bEadj5CapHbKC8kieT5sqqfl2kh9vrwM59etZuqxXdjHHrciy2szv+7kJ2sx3c8Dsffb1rttJttJWyM6RRyXMnlidnCicPg7UAzx/FgDrzXCeJbKAXVwX1C4lZ5iwjlXCElmLISODtHGBzWdGv7ZyVmraFyhypM6PV/K0vSzcWUaS2NxJbuke7727IyzdcjkDn8K5q61VbWdYrd/JuFYPJlh91hyGPH8qt3MM0HhPTzMwlRroPGrBTtG1iM5+99TW3BeaRPbWyPZ2kl0IzHGBbqdkjcFun3ucDnofy0VlLUVrnC/6NDY3l/Km5ZG3xLvVQo8wDjcCMYPp06VLZSrPbrIsaGWRi0Zz/AKtQQOMDA/CtXQ5TZzWsaJAwLGFRcJuXkFSCOvrVe0sRDPcvFvUx5Y/Pu/jHKj0J6CqlJbE8vU7a83x6bbWxijWNnYIFXBG0HqfzrJjSdUG+Am3KKqsOSzHORtp/ia+fdF5G5mtpg7AN+JU+nFa1i3+jbF6I7Lx9eP0pw7EyIo7eXTVtrqaZZ3Rh5hQfw+/4HFei+DbofZrnT2bJt5N0f+43P881xDRebE8R6ONtaHhbUTbajYzSHaJM2c2ex/h/Xb+tWG6PUK8u+NPh59T0TT9St4meezuPKbauSY5cL+jBDXqNZuvaTDrug32lTnEd3C0W7+6SOG/A4P4UyT5RsbWBdauLe8ucLF5kOMffI/Hj6d63LHTLfSopLh0ZpDyqyYLKOy/X+VZ9taXGm3RL2u+8jcqIiuAkuShJ9htLce1a8pEqRtcTxxhecKvLH1xnj6c1tC0NepnO8tCB32IzStlz8znPf/PFVxJyq/xyN+v/AOqnPNZNg7ZJV67ieP6VCbiGclI7c8jGFIy3/juacpuS1YRjrZIyNRuA93K6tld7bT2xntT7PVL+xt7q3tbmWCG7Ty7hYyAJV9Cce5/OmyW808wc2TohOAwk7np1FTw6dOq7mtZEHcGWNv6iolWj3OyGBry+FECBowrxuyMOhXjFRSxKylmTLY61qsunwRuLm6ZZdmVRLfcucnILbs9AMEL168c1ny3FsufLk5HIyppxakrowq05Upcs9GRvpRiMKCLe7KcLGCSKik09VXLxsh98r/OtJdEa+0hNSbVdPiw7AwSXTRTtjgtyu0DP+1Wd5fk/KmpXUZHG37Sf5UGZXWzgC7tmcdc96v6RerbXeM4ikR1OB0bHDfU7dv5VFCpmO1LjzzjONqlgPXIxTGtnjmV44zuBBDbuhB+tRKzLipbpHu3gLwfpGu+F7PVtQiuJbmcNuIuHVQN33QqkcfKOtee/GHSdO0jxpDHaW0Ucd1YxzSHc3MnmMpPXqQFH4dK9P+GvkXHgmyAZC8LyqUST5oyXJwwHTI/SvNvjgixeMtOwHOzS0wPT969Z6dENp9TzDyS8mxOcevQUkEZR9jD8jTVl81mKHZ6cfzqwudoLHOKEQW/U0p+9nHGKv6nbeXdSSoMKx+Yf3W/+vVDGM/SmUKshV8gkEelaLkXtq918vnLPtlA44bGG/wC+sj/gVZf8WfaprSZYbrMgzE67JMf3T1/xpMBxj+V0+7JG+eO4rp9Mm8+1t23rlvMTnosh6qf9luv4n0rCltnGpTW0jDzVcLu/vE8q30YYNP0uZYNRe0vMxxTOYZT3jfkK/wDwFv0zUvVDRo+IYY2miu2iD7o1Dk/eAB45/HFYjRCO8xDh1YEBJORtK4IP+fStzVDJNpEsM/FzbblfHJZVPzfqP1rnkuEYfIVcK4bYw68//W/lUwLaJrdrmKzNvGn7uLDMQdzNnByR1x9PSpVuFnSaJpUDyIU3XDEKRlWXOP4vlHaq0xZ7h5opUwcqEQYK4b5cH0we/piqt4bseU8QMiMAoGMlAOcfTk1ViTXsTFeSw28ttuz5uZoj97aCSR74rPuAHv8A7KySNlmjDqowzY+U/wBan0y6ubC4ieWNFW1uQJdmN7LIpBPXBGO/Wq+qtLa6qj20jJIAuxkPBwT835EfhTsIvS7xZzQxspu5E2yomd2SCFXP+etVtWgj02yGjxSMzwbZbvDY8yY4+XH+wOPzq3pcN5ea80FkEN00iCIy/Ku5UJ3Z/Cs4Wk0SXtpeJIt4Duw55DjuT3/+uPSqQFgJ5lvO6NkIokUnjch7/h3/ABqvZhn1VWHUB1IHpim6fcGIRyY3IoHyH68r/MVNpyeX4stII8skjrsBP3lZen17fhTJ6lyH19XzUzcrTZ4Psdy0Qbcvysp9jTl7d6aB7mTYEZuR0Pyn+lbUTAhc9eCKwbc+XfXUfP3f/Zq1I7qNNu9tvHNHUOhoH5lHHqtcjGP9FVemOOK6Q3qRXDQuxWRZNu0qcjGOtc7tK+YvTDsPyNMRNF2PtXqvwV0r7Z4lS+ZXMdlBKdxTAEjNtUA9/lZvpXlUXQV9EfA3TmtfB9zfOCDeXbFMj+FQF/mGo2RSPUB0rk/HNwRYWtoOs824/wC6vP8AMrXWV5/4nm+1eKFiGdttEq/8Cb5j/wCy1I1uYOojdNbWvQAbm9u38ga4W+vvtGo3N7DzG0ZZ1K7vlJ9MEcACuq1e5Yf2hcofmRPLj/3jhR/7Ma4L95byXLuVW3uVK5D7doznaMdBng+2aiWppc0vCJlk1y8Zpw8JhZkQOPly43YX+FQwIHbHSvZPBZzZXS56XCn81/8ArV5H4WdDqV7shjii8pWiWEtsROBt5A5zznv39a9X8FyqsN9uOFV42JPbhv8ACqjsRc8cZfmbcwXDNktx/Ea1bHx9N4X0iW004QM7ymZpJBnnaBjHpxXPeI9Qt47q4igHmRLLIY37N85w1cbdXZIaRu3rU06fLqeljswddezj8Jua18QdV1W8F1qTxXTqMIHTAQeigcCsS88ZX88YSBY7UYwxjHJ/GuekkMjliajHWtDyydp5ZWJd2Yn1NSBSRS2sHmsMg46mr5t8L0OKpCKXzD+L3xUqXUkQ++asG2H3ulRS242k9KYi1a6wwYI7MP7rKcFTXpfgr4l32g3aQ3MhntJW+YM3H19m9/zrxthhuM1o2F1ykUpGG+6fSpZVz7Is/EEWqaZBfWJVo3l2MOrr/wAB9enHp0zU7arM8EjRWwDBHYb27rjgjHX5untXz14H8cT+FNWCXDs0BwkiZ+8v+TkH+hr6E0nxNo+sael9a38JiYchnClD6MOxqbIYrasUkkBiJCEKvOMn5s5+u3ge49aWLUGubkW8sHyM2Acg/wB7H/oB/Srg1KwPS9tj/wBtV/xqVbiAj5ZoyPZxQIpSffNR4PrT35dvrTa0uQGKMc0UUCDFQXUIntJ4SOHidD+Kkf1qemnhWPYA/wAqHsXD4kfOMIJhi/3F/kK9W+FzZ0S8jP8ADe5+mY1/wrymBiIIuT9xf5V6p8LCTpd+SePtqD/yEK5KPxH1mbf7r80cdZyu+nQ3XkwR74FjYyEOqt8xIyc+uff2xS3lzYSeGgqy2F3cLN5/lQzoD8vA+VevA71xcUQTyUexieLyv9e58uQ+oHPB/nVq9lnliWKxsobSFYlVi9vGXkYj5iW2naOvQj8TV+yuz5rnsrWL1rr9rFp2oLLbyDUrgv8AZy8A2DC/MfYenrmm6G0dx4JmEjW1m28wwiWYJllIPzL1IG4jPTms+7a8kZvLe4kRAPLVZsKMAZIXqM+9N07UdShjdbcXG93VgqlBuIPOWbouMdat07x0M1PlZpaKIE160Imid3CxkW5BYfdOc/dIBzg89D6VsyanfaPrWi20nljznjuldGz+7aXC4ZuRnIyOvP0NZptb+2uo7y6Z3vIZmCOky7o/kYAncf4SRjHX05qlPcWttc3LqGa5Db1cKCcjGGJ4APFRGnrZjnNPVHV6pqU1pafabe3EF7qCuSh3eXD0XBBOTyG/P2rI1yYwaLqiGzklQu8O6Y4TlEDMvoQuOB13flyVxe6jqGz7TNv8kFl3sr7AOS3c9eSf1rU825WzfzLW5f7VbsjCNj95sfvNv+6gAOP4cVoqTi7k86Y+ymM1u9tJHuX5XiK8rnIwG9vTHI7ZrR0whpbczbAftAtyd4YMT3yDzjJB44C/WqWjJdWFo8qadcTkSfNGX2yMSpAkX5f4eeDjt1q6nnWcq27+JWtHOGSK10beY88jDMyjdzyfU9aTh0Y+bsTRkXMN08cMh27jGdjATbDzt47cfgRxTUvUVhPujK4yMv8AfPHA/P8AlW1ql3Zrostss8l9FbJiR3aSJ1klTbuYMWB+ZU4Q4VSwPWuSttQuNYR4fsoAkjj3ObgjaoxtKhe/y+4PNZumHMbSzu8Z2vhAvTb09KikPmRGOQky4ztTnHpUtpYp/ZwhBVgHL+az7trAnbjoMdufertrb2qRSMhMhVd482fGcLyAob5snJwe2O+a55OMNblLUyQsluuGjde/K4BxUyXMqsA75jYnDZ+8fpUt3aLKImMbABBvMe+TcOP4N23n/wDV60fbYbFknkuoFg3KJXuItrD5wepwqjsOM5FaRSl1JuWFWVVlVoZ1ZVw5ki2gR42spyO2QfUcVPZQ3FzfzXX2eeZiTHcTSurSGPCbOTjO07gcDkEU7XPENjqGjSfYbtHSPA3wXKsY88fN1Ck9ckYOK84ltZdS1aS4mLybxkTzIqiQn5QoPOBkdugFbKlrYFNdTqLy9C3aQTGTT3tfPSE4EjCMN98L0AJ3AFvT2q14eebToIf7US6hgQxuYI9PmnZtyk7ty/Kw27eec7q5K1i097qCKR3/AHzhceVj5E4YMM8Zw31r3Sy1CCw8HS3Q1Swt9Q8gSPNHJ5xLDoDGrLvbbtUAd60atsJs5RNW06203/RNStZLiSQlDJbywtGScgOWjYe3OAMmsnVNWvtSsfs099DPwAyC1jGzPUK3LdMcjrWDqN3c65qx1W6mV7cblSLzGbyz/cVjncP65+taWneFBqhmn1LT9SgYRCVJLW8i8sR7Th/3iYI47NzShRhG/nqV7WRc1WN59C0qztLiESqxZmEgYRhUYZOAe59K5uPYl1b3S6xa7YsuokimDFsEA/cxjnNbGn6ftzBpWnXeoGbjbvXOwrjMjquBnn2HQE1e1Hw7FYWdhNeWU+n3bgCG2a9E6+Uo5LKV+TlgMZzn8aFCC9Qam1z9DDNm4tIYTEl1Pc3JC70J3c+mVGSSPTHNWriy1XQJLeK4021h85v3RWNWDFTnghzjBIPNdD4ZSylvWGsafeT2UnyLnT3MAbs3mhuvqT612g8PeFm/1WlTYP3vKnkj2r/39FaRtbYipCSa13PMUbWtakmaPSFkdSvmmHERyRwD8+D3PQ1t6DdNcW5LqUdlyyk5KsrFGB/ECu4bQNAknld49RtN3G4XTqhIA6Ydh3FcxqenadoesW0OnXrTpIjtKkkis6Mx4JwoOG65I5IPNS0t0iVcsDpUAXbdzwA4M6echHZ16/40LNTLmXYsVyv3rdw/1U8NSktC4vU9Z0m9Go6Ta3feWMFh6N3H55q6elcn4KuxtvNPzxG/nRf7jf8A1x+tdb3qQa1PA/ijZvp/ixlCqLS6T7UM9Cx+Vx6D5sH/AIFXFW/+kPK7PGIovlIQnOf7vA6/SvZvjToh1DwlFqcY/e6bMJGI/wCeT/K/5Ha3/Aa8Ku7oWloihAruS5QdBnn/AD9avcRZvriOKPy4Ygig8ACqI1G5tovOs5JEulcFZEHK+tRxXa3VmzTYR93v64qCaaGPcsfz7eMjpWt1bUjW5NbeItTWdJjcxySqwdS0CZBH4fSrt74n1G7V9/2eGaQHzLi3gCSsD23ZOPwwax4WjlzuiwRzuJHHP6CtYaE05gEN3pJeZSdh1ONNnJxu3YxnHHXtnFZ8qLU5LZmK4mvJPnlZv9uRyxx9TWjoOhR6/rsGlJOtu0u8mZssqhVz0HWqtxBPp8oimSNWYbgFlSQEZx95GI7VNp2r3Gl30d5ZM9tcxghZYJSrAHr601oS7t3Z6LY/C3WLKeVNN1+BXRdrgSkBs8Z2tGR/9cfSvMPEnh++8N6zLpV5JbSzxxpI7QyFlO4bhyfb+ddfb/E7xPGOdau3PX96sL8/8Cjrm9XvJdd1OfUb6aaa7nK+ZJ8g+6oUABVAAAFJsDHtAY5kcfeHzYzwfbPbNdXYr4dv7SN9WljtbgSFJfOyTs7FSo+b05IxXMyKls+7zlGRwHPNOhlLBY8o7Nxuc4X8azlBS3N6WIqU1aL0PbPhRZpaT6hqenia4tbgLbIIEGwsjFuSWGMKQP8AgVc78a2Y+MNOeSB42/s/5lZgwx5rbehPqa9B+FMMcXw9sk4DefM0o9HL8j+VcD8b+PFenHOR/ZuPp+9any2jYidRzfNLc8t2NmTDdunrToxuXDdPpUMSgSEhgSvGQOoq1FgYz068UIg6e5kS7d7lQcsu2ZB/6FWNJC0Oxs5BHX1reuFEGpNNbDIZS2wj+EjlaoSKqbEbmGTPln09vwqEUZbqfkb1FN7++KtBR5hgm+XkqG9D2qo2Y2w2QwbaR6VRJdMn2mzEm4/abZVwc/ejz/7KSPwJ9KszIdSs5rlRi48xWOP4mx/WsqOQwOsgGR0I9qtRStAv7o4SRQVb3HQ/zFJoaL9xfvc6Ul4gH2iNGSQ9d2B3+qn9KoXUcL4nt0CxNzgH7tEspb7Vs+WO4Ull/uvg7v8AH8arRuy2hdTlCAG/KpS1LvoVIWeN4jhSG86M+/Izn86sx3Lic70Xy1wfLJ3Ajn168YrOjZl1FH3H5ZMk1oZgluJp/tCq8n3QWHQDuPWrJCaSJIDFAWCsd2GYk/T8+9Xxbw6laxzRMsdxaAHDn5WTdu+b/d5/A1n3lr5U4P3cfKeOoP8AhVmwinuZJpvN5WLaSVC7x93HAxnbTWwrnXeF08vWpXkCo0NrNtJG4Kx+XOfxPNc5rkVxLeS3c22N5H8p9ucblUKOffFdL4Umto57w3MqK7QJHCGbG87xxzx6VX/s2G8jl0lDhftLRkAdMScGobtK5W6RyqN59vFOU8tzGqONuNzc4Yf7w/lUT3BttY0+8UZa3kiZh64bmq9kGieaFydyuO+fpT7kx/aoWGNsyDv9x1f/AD+daLYh7m/qjIdSfyw+10DAnp9R9aiQsULDOVBb8KqGZmmk3vkKSq+wp8V0scgJwRnBB7g0o6KwPczpCBrNwy/ddWx/OoJJn3KVcjgcikmkUakxDfIcgH6riq9wHjjRDjf6A5piLUG5d7GQgHk47VM7mTrj8KzI7hv4vyzWgjl1ztJ4OaYiSL+HAJYdh3r6/wDCmkjQ/Cml6bjDQW6K/wDv4y3/AI8TXhPhP4UeJn1rTLnUtOit9PWeOaVmukZigIb7q5619HDpRJl9AJwMnpXln2oXF1f6mTwzvKM+n8P6AV3viW8+w+H7yZT85jKJz/E3A/nXm1z/AKPpCRDjzCF/Dqf5VJcEYOpu0drZqX2K8rTyt3CqNo4/3mP5VzRjknR7NpQUl3beOM7Wx/nNdPqdnOb/AD59q0Qt0j2b/mX+JgwIAySexPasUaVfDV47kSW/2dfmZWuF3Aj+7gVHW5TTtsV/CSCPVdTYbVPlRh4xnIbPzN+JFeiaPqMdjoniGaU/KtquB6kllA/NhWToXw/1WCe6u2uNLjS6RCqtdEkYJJzhcc5FVPE1lc6NIthPPbuZlEr/AGeQsu0HgNwOc81aaexm4tbnm2rByV8xgzbsEiuY1KTDeSO3Wux1EKgYnkIC+a42a0nlhkvn+WMsQD6n0qiWZzAikp6xswzjipIoGklCKOf5UtQOh0Kye4G8bVVRkkgnH4Ctu40gShmWMjavA7msy11M2lqlvEwVUXqO/wBasJqcskIVEOVGFI4A/CmFyvNpckKqZB8p+YEVWksjKrNxsXpWsbmaxjE1zKPKY48vqWHtWNdap5spZIiID/Bu2/XkUAVG0xQhlmlWJScKvVm+gqSDTklsop4t+CxDLU1rPZ3G8OoZ8/uhK5G1u2cA7gK3rWCzsdKSP7T5j5Z5AI+DnqM5z17/AIUh2KeqWDzafBfxkeYiLHKAf4f4W/p+Vdt8ENTng8V3GnyRTTQT253IqhlRgRh29u3HqK4wanPpjeQckM2Bu/u91Na2kfZDdtJZO8bspRk+6QD2BHUZH1oZdNJySbsfTD2VswwbCDHTBt1/wrOl8L6VKMHTyme8Zdf614X9puVyPPnBH/TVh/Wnre3a/du7kfSd/wDGsfbpdD2lkUpK6qL7j6FWMqiqsbBVAUDaegp2xv7rf98189rqeoA8aheD6XMn/wAVUg1nVV6apfj/ALe5P/iqaxC7B/YE/wCdfce/EH0NFeBjXtYHTV9R/wDAuT/4qnjxHrg6azqI/wC3p/8AGn9Yj2J/sGr/ADL8T3g1R1m4Fpomo3JbHlWsr9f9g/1xXiv/AAk2vL/zG9R/8CGqO58QazeW8lvc6tezQyDa8by5Vh6EUvbrsOGRVVJNyRmohVEX+6oH6V638MYfL8OSORxLfOR9FVV/oa8lB5yenevb/BlqbHwjpSOMM0Xnt7F2Lf1FRQV5XO3O5cuHUe7PAdGs77WbBo7edCsG+SQErHt2fd68sc4/KqVibSZJ7i6W+uHZd2XgBYSdevQD6HNegr8BtYaaR5tVtmjYEJEs8ihSe/3Ox5xUZ/Z7vQgJvrWd+flkmZR+kea6Lny92cLBqatbqglL3bT7sDAPl7hkegG0kZqLSIn+1qsE0cW5ZCkEduxJVEZgznoASMYz7kCvRl+BWprKrvPZN+7EZ/02Rcjpj5YhUkPwZFtJG41LTreePcuUuZMgMCCCT14OPzpcyiFnI47U/FIubxpo7ORYbK3UyOZZHd0K7FLNJ94ZIIx8o7Vm67cXH9nQKuxiJAj/AGZw7yfL/EBz1X+VewW3w2sYzEZ73TpDHGsKs8jSkIOijcentnFaZ8KRouyLULFR0J+0FePbbjH51DlG+5XJLseMaXHrzae8VuVt/tePtDwvIWkjP8BRVKqOeaveHotb0u6uZf7BkmDowzPK0YdugYAMCD3wenXNert4VU/8xLT3P+3cO38zR/wjDZydU0xQOm2QjFV7RdxeyfY4mebxFrl+95f6JYW05Xb57MoDDPRlGcnuGzms+TwTeXjv9o1qSO3YIDb2iNtk2k7dxLep65r0yLQFiHOr6eTjBbzaefD8Ryf7Zstx7+b/APXpc67h7OfY88g8BwpCyS3uoSIY1QISkSqF6epPJ6962o9FsYWDLDCp6Yji/h9OwxXUDQFUcaxp+fd//r1Nb+FprhWaHUbOUA4JTLAfkaOZPYHGS3Ry32XB+TzAOmd3IFPEfkLnzrn6liv5fLXYf8IndlQDc2+B0IVv8adH4WvYTuivbdD6iDJ/Onyom5ySxzmFjJJfkEZXbLISPrxVSSKX5vP+0SIe0p6+mdy84rvH8MXkjbpNQV2xj5kZh+RahfC96h+TVCo9FRsfluxQkhnmOpaBpusqvnC7UqCv7omHOfcDa3/Aqwp/Aej21vLDbT6ra3DAZ81A4Y/w7hsHH0NezyeEJ5T8+oRE/wDXqP8AGoz4JkP/ADEU/wC/B/8Aiqa0EfP0vhDW7eUSR3VtKYwRG250Y+pyw4OCe5p623iKyNtPcXF1qEC/NLbI28huwXruA454r6EPg/MagXojkXqUiO0/gW6/SoLjwwtrC81zq0ccaj5neLGPx3UOVhpX2PmtdWjtdOiglguBcRvysi7eo7Z9D7VpW3irVILJraSJ4bGQ4kzdNhR3O0MePwr3I2WjgYbxJbn/AHkB/rVebRPDM6lZdXsHB67rVGqPbQ7o1+r1X9h/czzrw944sNBvrqZNRSF5GbnqGXORg9PwpNf8UxeJria7juQyJBHEH27MDeWfH4nk13X/AAiXgxCXj1HTI5T1dLGLJ/SuevPhp4Vu757lfFsNtuTYIraxREUYxwPU+vej29P+ZB9Wq/yv7iHQPHtpoNsbR9SMYyTsEXmKf1roLT4jaSkoZJ7RSAQxFrIhz6fLmsf/AIVf4H+zRxnxIyzocm4ij2tJ7MMkfkBVf/hVXg8Rv/xWFyZCRtbyVwPqMc9qf1in/Mg+q1f5X9zOrTxzpN3GPPuNPc7mYBpZUxk+49AKzde1TTb+2g+xR6ck0bhy9sQzsADwzdSOa4bWPB2g2t2trbeIZUEUahpF3P5rEZ3YI+X6DinaRpemaXcmX/hIHuRsZRHLGyqCeN3C849Kv2kWrpol0ai05WdWsnpUhl+Q55XowPp3qGytxdW6yW06TR9A4Vlzj6102ieFodWjmV7u4heMDIVUYEH0yKttWMbNMh8K3zWep2Tu33XNnL9D939dtep15Tq2mDSfENxZRblimhWWA56ED+hU16Po19/aWk210cb3TDgdmHB/WsUay7k2pWMOp6ZdWFyu6C5iaGQeqsMH+dfKOo6W/wDwkQ06ZyrWxdJ3dSfLKHaxIHX7uffIr64rgda+GFhq/iW61tdQubaa52eZHHHGVyuATyM8hRmtIszPnifTxC8sLykSQzur4QnK8YIFUJLd5ZhHCsjE/wCztFe/3PwTsLm7muW17UQ0rFiFihxz/wABot/gjpsDZbW9QkXn5XSMA8Y7KKd0Fjwm1s4oruNUnLzbtgxaiVM8gjBYbh+HarwsbeOR1dLK5TaVBe3ljBOMbv3cgPB/A17VN8IfD2nW0lxNq1/CEU/vMp8g9ht/CslPAvgVwFbWNW3kj5z8uP8Axys5VacdJSSN1QnO7pxbR5CdIQfdSyUHn5JbiP8Amp/nVb+z/nlAAIXH3LxPl/76UZrqPGtto/hXxLPo8L3t6IlR2lW5EZUsN20gqwOBjkY61zCajpuxix1BGY54eNgB+Qq7mDVgGmsAPkuP+AyQv/IimPpUxb5Gu4yP7sAP8mqYXukv/wAvl2vp5lqjfyekWWwOSmpIM/37V1/lmgQipfQApHdPGpOWDWW7J/HNQ3SsYQHktzMpLR+VZLCzN025GOP61a3QlcJqNq3sfMX+a05PtUtpts74oyMwKx3GAc4IOM80tiox5nY9T+Edw9j4LkeZ3YXOpSBAecbURf6VgfGof8VHpROM/YXyf+2tdb4HbZ4Zitvs+7EzswEyoUfIJx14B9PeuS+MZkl1PRjJaxxym1lbckpb93vGFIKgcHJyPWn0I6nnCByy7iW9zSnAcgDHPrTV3BV4dfYikw5JcghOvA5qRnT3N2IJI5PMBaN9rMvIx/e+nNRyorwuquqRs+9ec7fp6jrWQZ1dWEkTYb7rlf5+v4U23uLnT4HhuIoriKQYC7+3Udwf8isrS6FMde3ClU3MBKp+YDoR1zTH1G3urLlsTw/xYx5if4j+VTPNbz3P+j2ttFAE5VwN/wBPmPP1FVBDaJM42bTyf3bkf4iqT7iuVYtQj8sB+OTU0epRorQt8yEHkdvSq0sMCndGoI6Zaj7MWPy+X+HBp3C5ZttRIuk6MGZeD0JqNbp7KaSJdhBUowzkY/8ArVE9i6FvukdRj0qFlXO3YTt/2qLi5mQ3Erecrjafbt+NaumtD894QkUnmgRsBvCtjoV9DWb5cbAlwRn0PWnxxwLjeJAO2BmmO51GoR28Vo1p9p3TGcOmEYYXbyuT17flTNHuJESW34ZI5STx/eA/wrn4AUkUgsqq3QZANbmlkJJdknAZkwe3f/GnFWQN3N+M6bEziRIZDjOJFLBiGyG4IxxwaI5Nu+XTlM12riV1a5zz6AY+bn05+Wub1JcuMbQ+wlfzqratcQTLJ+5dV/gkZiprOcG3dMpTsWNX0m+gna5e2NujP98N8jHOcDms66gnJUy4weRtbP6/0rTLmR1Mk0GVPy4XAX1oxayApO65YcywRYZD+YB+mKIqSWpLMtpWZHTDIHfOEIwB6d6ckcbRr/rN2efmp5t0LM3nEj1wOfypDDCNuyUtkchuKsRGIom+4rAY+YM3U96QW8JDbhtx/dqwIowo28/U1YEkCIFFpyP4mk3E/kBTEUTp7IwdiV2nHzLz7GrpnSQZSKNTtwRjINWFuYdoCWahvXzjj8sVIk7MSEt41c8bsu3/ALNimho+s/D9yk2jacqnL/Y4XbHYFRj+RrWqjpNkunaTZ2aZ2wQJEP8AgKgVepFs4/x1cbksbBT/AKyQysPZeB+p/SuSv4/Nvra3BG1R8w9M/wD1hW5rcv2/xfKoOVt1WIfX7x/Vq5+SYSXV5dZ4XIU/oP0BrOtPkg5PodGGp+0qRiurOB1CYXGpXk2M+ZPI3T/aNVgA427Rzx0pN+QGP8XNT2a+be28Q6tKo/WvHcna7Po4xXNZI7TaFwvACjBPpiuP1DUDdXMkoA8tiNgH90dK1fE+oNa2gtowd9wDlgei56fjXFuXOQGOK6MsotJ1JdTgzrEqUlRj03IfEM/lWAOR+89fSuPmvZZbeKAn93GuAPxzmui8TNtsYd3zFnKg/TvXLRRSTOEjUsx4wK9U8InEuFAVeMc5q1bgyDEKnsWYdfpWmnh6GN/37sQAOFBYscfpzWxBawrbqEiWM+jIWbPccd6Yjn41ePAKd+/UVpJG02AFII52hMVe+x2tsqvcnfcEfJERz9TzUJ1GOXURCdu2N+vXnjpQBV1uInyCOVVAvQ8VkQOA5R2ARu/pXTX0T3J5QDavPYVRh0QSsH3AHPb5sigCnb6XHJOjRy4BOCSOnvVwW8qRqjjHpWt/Y8L7SpZZeh28k/n0qW4s2CRjaWKfIXLdKBnMXcbS+WijHlnpmnbp7blkKns1bEumSJ8xZeffNTJdJ9nMM0YIVeARmgCnY+IboNtu2+0L0+fhh/wKu18P2uj6+/lDXo7C47R3NsSp/wCBhwPzxXGSWtqSWDqjDs3Q1lLc+VdcOY8HqhxUunF6tHXRx2IorlhKy+89DutPa11qbTDKrtHc/ZvN24BO4Lux+NdmfhVqWSF1W0ODj/j2k/xrzTR9VAuIftjYRZEZZVXoAwPIFem+N5bvxBdadqHh77ZfWZhkVpLEuQrbwcMF6NjsaxVJJNtHsLMp15QjTnytrW66mJrvgfV9BtHvJDDc2sf+skg3Axj1KnnHuM1zOa9X8Bw6rBo+opr6XEdj/wAs0vidwG1vMPzchcY6+9eSR/6pMdNoxn07VlUikk0ehgcROpKdOo03Hquo/NHWkFFZo9EsWNlJqWoW1hED5lzKsK47bjgn8Bk/hX0GqJEgjjAEaAKgHYAYH6V5b8M9JNzq8+qOv7uzTy4z6yuP6Ln/AL6Fep120I2jc+Tzuvz1lTX2fzNYdBRRwK5TxT4mNgpsrE5u2HzuOREP/ivb8frFWrGlFykebQoTrzUILUZ4q8UfYVawsXzdsMPIP+WQ/wDiv5V54VBJZgCxOSTyTTyjkkncSTkknJJ9aTy29K+dxGJlWld/cfaYLB08LT5Vv1fcbsX+6v5Unlr/AHV/KnbH9KNjelYX8ztshvlr/cX8qNi/3Vp2x/Sk2P6UXHZCCNT/AAr+VJsXoAv5U7Y/XFa2heHrnXLrAzFbIcSzf0X/AGv5VcISqS5Y6sxrVqdGDnN2SGaF4fn1y72Rjy7aM/vZsdPYerfyr1WwsLfTbNLW1jWOJBwB/M+p96LGxg060jtraMRxIMBR/nk+9Wq+hwuGjRj5nxmPx88VPtFbIWiikrqOAWiikoASgcVXsrn7ZarP5M0O4keXMu1xgkcj8Kj1DULbTbR7m5kCRr09WPYD1NKUlFXY4xcnZbjr++t9OtHurmQRxIOSe/sPU+1eV69r9zrl0GfMdqhzFDnp/tN6n+VJresXeu3fmzfJCh/dQg8J7n1b3/Ksvy2rwcXjXVfLD4fzPrssyuNBe0q/F+X/AAROaOfWl8tqPLeuA9vQT8aM+5/Ol2N6UeW1AXQc+poyfU/nS7Go2MBzQF0c7rwZb6J8HDxf+gk/41Y0XRXvdtzdoy2vVEPBm/wX3710tz4Wl+zWepajCRAXZUhbq2RkF/b5en5+lWd1e/gsO3BSmj5DM8bH2so0n8yaHChUVQqKMKAMAD2rqPCj7NSkQ9HiP5g1y8b5rf8ADs6pqtvnjJK/mK9Gx4dyz4+sz9jtNUjX57SUBuP4W/8Ar4/Ol8GXapNeWGflbFxF9Dwf6fnXTanZJqOm3Nm/3ZoymfQ9j+dea6FePYXlnNL8rW0xt5weyk4/T+lQaLWNj1WmO4jUs3QU8U0qGBBGQfWmQL3qrfX1vp1o9xcyhI179yfQepqHUNUtdLtHnnk4U4Cr1J7KB615pq+qXes3fnTsFRf9XED8qD+p9648Vi40FbqehgMBPFSu9I9yTWtbn1u6zI3lWyEmOInge5x1apdI0J7y7jy0UkXBOJCAx/u5x1wM9KxvKNWLK9vdPlMlnNskA6YDdvSvDhVUqnPV1Pp6lB06PJQdjy34rxQRfEnVootylPJVw3Td5S5x3xz3964qS1mRtpR/xBFdf41g1C6+IOr3OqW966fa2RpYbUncqgKu0cDoB3qniwDc6rJbkdReWMifqm7+VfURtZWPial+Z3OaMMcYzM5J/urx+ZNSRTWqf62J1Tlf3ZztrqRpjTJui13w/cFjn5tR8sn2xKq1heItJvdPnhmuo4hFcL+7aG4jlVivB5RiPSmTYzmbzJyIJHKD7u/jFa2noJmETw+b8rH5gCAACaxYgFXkY+td54Kk0S0jea+innvZGOzy50AWPb/dPJJOc/hSk7IEemfC0258MJLI8aIsrqvzADseKrfEW0tNZ1jT5kugyQ27RERSrzlwcEVzjarp0CfPdTJHkJFF9iVAi9/uvzSN4g09oo1hu7VMKAztA0TMff5T+FctSrUtZIqNGpNXihq+G9JAGI7iXPXfdtgfgF+lSReHdDkKv/Z0mO+66kVf509NYhc/ub22POVUSxgj27fyq2l47MRBksB99VynfjI68VzOpV7h7Cqt4s82j0XVZ8yRwXM/ukW4fmDRJbXqL5d5HKAOi7NoH5ChLhixV3ZF6nCBuO/y1fhksoUjEzStuXvHtP5f/Xruc7EXRUVvLVR9lhHH/PLBP48/pRvhZGRvsz5OfmVmI+hXGK1GvLCQuYRcqduAqRhR7tz/ACpRDZrII0CySYB+Xgg/72fwqfaLsPmM9Vt1wy6fksMFjI4HvxTIbC1Mx3zQQKegifzMfmQa0L6GAwyK9t5Rwu0rOrfjnq3+c1Wj0m2aGOR9VgLAElfmOfTt9Bjn60Kog5ieax0ZYiBq8IkB6m1lXd9eWFVJoLFYyIntyx5UAtn9QK1PsllFvQ3doBwSWRm2tjOB7VnalNFbMbeF7Z1xziEgL9M0RqXdkgcjNVoUPzqM+m4CppDCoV0jUdCQdrZ/EAUyK5mjx5cwVT/dUdKvvZR3Z3WxJbOcu+wfrVOTRNzNkkXbkIi47LTIpLjDfIURjjOwkH0Ga1JLAWiF36cfxKR+Yq5HFqF3bb7S5ghRR82bvb9Omf5U+aXRDMKeznuoY2ynmR8k+aPmHbA60gtnZgptpGLHt2rZih1ae7eJtSs3l27j+8Zyw/2Ttwfwp8V0txIbe6uJE+sQA/PdQ3PsGpkDT5CoU2o3KSxy/wAzA9O/QUrxQBR50EsZA4MWB+h61avLW2hk3JJM69AWAXn8P8ari6CpGRahlzgtyCf8cGnyyFqJDCZWItklLj5lxjkCmmFQx8+TyyPvFxgg/TFSy6pdOwYmMv8A39oY/jnNSPdSybZFmj8xuGj3rnI9qfK+oakK2kJH/HzHnP3Nr/z24pxghjAG5W9dpB571NBbXk48mKKZ0bOBhsD/AAqRfDesyn5LCXb2ZhtU/iafKu4+VkLWu1VbMewjILSLz61aijdLmJrNsSmRRGG+Yhyw29uecc1bt/CuorgXkEvlHny4riJDn/ec4X8jXYeCPB0kvjbTJJBZx29u/nmKOdpWOwZBJP8AtbaqPLfcFF3Pf4FkSGNZWDyBQGYDGT3NOkdY42kY4VRkn2p4rF8V3RtPDt2VOHkXyV+rHH8s0ijhrW4Zob7U3+8++X8WPH8xWBqUn2Xw9Pg/MyNg/wDjo/ma3p0EOkxQjrK449hz/hWHrtm93Yi2icKzLHjKk5A5PT3rkxilKCjHq0enlrhGrzSeyOF+XpWnoSKdWjYjhEdv0x/WpV8H+IZDmHSppl/vJhR/49trqfhx4Ve/1C7uL+F4oLfETRSKVZnydykdgMYPrXE6E5rlS3PUWJpU3zyeiMvxPpEw0my1WRQIHkNvtJ5dSNwcD0BGM/7VcVLp9z9qXagVOme1emfGi51TTbS0MN6p0+Z9v2X7OARtG7O7uvHT3FeC6j4j1C5UosrLF6Bj09K9OhS9lTUF0Pn8TXdeq6jVri+Kr2Ge+it7dt0Vum3Pq1VdJ3Qq1wARhsBvSso1Yhu3jQRnlM56c1qYHUWuo/vMBmyTyT836VtRTjZ5zKYwq7mct8uexx+Yrjbe9hiff53ze6dKs3XiCSZPLVyE4+YfL+lVcRroDcXjzh/mY8IV3fnUN5p8VlbyOjiSXO7eFx36AVlJqtyiq8a+Yp+8vX8zV+yv7lb/AH3G0+Yvy45CDHpQBBpmp3MTOkm+XnlXbGPwrfh1QEmSSaK2XoeUGPpn+Vcxe3Aa7eRd8u5uB13VJbazd2rARWWGU9PLHb8KAOoa5eeNUtJAzdXIyN2enNVEsYJ5cfvRMOryNuxVSDUNVuJC98/2OIkbQ8R+b2Gela8SOkyu0gcHo2Cp+lAx9rJsHk/eTrkA9KWSSzLfvYivHXtTVuEYvLhMdCrjGP8APrU9wLaWHMke9FUOZNp2/hjqaQEF5psM0O+N+GHU8fhXNSW7Q3WxIonOME7ev4nvXVyXEX2hLdApQLnl+V9OKo6rborLMn4nOfyFFwOh8KQaKNOuEvrpEvNoWO32tlSx4ZsjBGAcYPWrU1zN4E1ZJtK1ONnkVWeMSLKCv92QDg/zHtXEzTMkASKQpOmSpJ4bp8p9OmR6H61RtYb7UtQjaRioVsMFOfxz0pt2RUYOUlFanvvirU7jXvhva6vZu8MDuPtkCnhlyUIz12h8H3B5rzE/rXbeDvEGlWug6joWtT+VZTKfLJUtww2uvAPs31JriiNuV3iTaSu8dGx3/HrXJVlze8fXZXSnQUqM1s/vuJT445JpY4oozJLIwSONerMTgD8TTRXo3w48NEsuv3afLgizU9+xk/mF/E9xUU48zsdeMxMcNSc5b9PU7Pw9oyaDolvYKVaRAWmcD78jcsf6D2ArTPSne1A6iu/ZHws5OpNyluzH8UeKV04NZWTBrwj5nHIiH+PtXn5mYksxLMTkk8kmmtIXdmJLMzEkk5JOetJkYr5XE4iVad3sfb4HBQw9Oy1b3Y7zDSeYaTP0pcr7VzndZC76C5pMj2pMj2pBYN5xzR5hFG4egrY8P6BNrlwTylmhxLKOpP8AdX39+1aU6UqkuWK1Mq1anRg5z0SDw/oVxrlz1aOzjOJJR/6Cvv8Ayr0+0tYbK2S3t4wkSDCqBS2lrBZW0dvbxrHEgwqqOBU9fRYbCxoR8z4zHY6eKnd6JbIWiiiuo4QooooASg+tFZ2r6vbaNZm4uX68Ig+87egqZSUVd7FQhKclGK1Yuqana6RZNc3T7UHCqPvMewA7mvLtY1u51q7864GxF4jiByEH9T6mm6rq9xq92bi6YccRxj7sa+g/qe9UNw9q8HF4x1nyx+E+uy3LFh1z1Pj/ACF8z2o3k0m4e1G4etcFj2LAXNJval3Um4etAw3GjeaNw9RQCCQM8k4Ax1+nrTt0B2WrAyYGTwB1rufC3hRyUv8AVI9v8UVuw6ejN/h+dS+FfCX2YpqGpoDP96KA8iL3Pq38vrXaCvYweBStUqfcfL5pmvNelQ26v/IxPFkCy+HZyRzGyyA+mGFeebQK9V1KD7Tpl1DjO+Jl/SvKVGVB9q9iJ86yVMDFaFlJ5U6OOqsGrMHBq1BJg4zVCR6mCGXI715tr9ktp4quon/1F9H5igHGW9vxzXb2+p2sOn2bXE6xmSMbQT97HWub8aS2tzb2d1DKrSwy4GOu0jP8wKyZpE6XQb7+0NFtp8nft2Pk5O5eDn8q065DwfOUluLf/ljMPPjweA3Rh/6Ca6/tQhM8/wDHEEkWo205LGKSMhRnhWHX8wR+VcrvHSvS/F1l9s0CZlGZIP3y4Hp1/TNeZZUDk/TGMV89mNNxrN9z6zKKqqYfl6rQesu1xlQQD0I6/WtXT/Ed3pwKQwW5yoVQycj2z1P41nxXFuE3vCrsR8uDhOO55yffpWppWrO13FFb2emiRctkRqrfmTkn6c1jQupqzszfFWlB3hdebsTzPJeTPO1+gkdiWSUvGF/2QSCMD61DJYXQj8xrZ3j6+ZH+8X81JqQWRuZnk+3QCV2LOAzBtx65Ugd6Fsbm2lDR3USOTwTviY/RttfSrY+Olu7GXJZ2twcSRxSH0eJH/wDQgapS+GdEmYl9I0py3UmzVT+a11jPqM3E9pBfqB/EiSnHruXDVAy2BOya0urWTH8DEj/vmT/Gnck49vA/hx5Vm/sO1WRCGUwyumCOnGcV0Fwsd3Dsu7SOaMdpYVcfpmrxsY3x9mvoH/2Zw0Lfrlf1pkmn6hEN7WsjJ/fjHmL+a5o3A5yfwt4XuAVbTYomP/PCeSL9M1myfDjR5STDfajDk5GSkox6fdFdW1xhirkZzjDdfyNJmE/8sVB9Vyv8qXKi1OUdjiJvhkwjdbXXY5HJyouLUrx9VY/yrOb4aa5bNutrnTJXwf8AVTtEw/76WvSgqN8yySr/AMC3D9adtcfcmB9AyUuU0WImjwS80fVbJBJc2kkCnoZGTn8mNVRHJn5pAB7tmuqS11C/0vfb21uyMv8Az1Qcfi1c5Np2oxuyvbuu3qQAw/QmrUUcrjYYsI2fO5PpgU+GKxDqLiS62A/8sdgP/j2a0LHRGvIFaa7a2bPO9BjHrncKlHhWEqf+J1atj0Rj/LNFoDUSvLZ6M0Qe2mvXkxjEsqY/8dQU23isGiYfZC0o/iaaT+QYVpWnheJDu+0zyn0itWI/XFaAs47XDxabvkHR5/LjH5eZ/OjToOxgwf2fP8kdjGk6nkFncH8CxqKS33OyR24jI42pF0roVvgZg9xZ27leB5arGB77lq2NT2n5JbaA9eZnkb9VNGvRBZHKHSNRmPyQMr4+62Ez7jNMHhnXpAW/s2bHqXT/AOKro5b2Rx/x+SSZ5+WJV/UgGmrqFxHgoik+pG4/+hcU1zBaJzsPh3VA+028cJP3vNlRP61tWWntp8R+2IMdP3VzGB+tXYtRviT92MdysWD/ACqU6jLuy5kP0gj/AK0Wkw0RkjTbL7cZra8jVAdwVnLYPsVUfzq7NZ6fcv5jr8wOT9nRm3H8WGKLq5jZS80Ux95LhFH/AHyKiguLeT5IobdmPOMtMfwXBo5WFxXbTIoxB9jvHPTYZFTp9Nxq7DY2bKjJoMkrY27pFxkenUcUkOm6rN/x72F35bdQln5YP/jorWt/D2qlBu0qNfebaD/WjlQXMvy3U7ZND0mLb0MtqrEf99E1btJ7qPK2v2ML/dggjz+G1K0G8K6i/wDywsV9d9xIB+S7akj8Hyov7yeBWPOILZ5f5tS0C7KcmqalGpDW+oyv/eclVH6f0qqJb+5wGs4woxx5E0p/Lcq1cn8GXEr5875P9uLZ+lL/AMILNgY1BCP7gtyR/wCh1VohdlV/Ntj+7MVux+8zpHCR+G4tXovwrtJpH1HUZrjzgNsCESMyj+JurH/ZrhU8H3YYAXJgT+8iqM/ka9j8FaKuh+GLa2EjyM5aZ3k6sWOf5Ypu1tAV+p0dcZ43uDJcafYKerNMw+nyr/M12ZrzvUZxqHi27k6xwYgU/wC71/8AHialDRlaxLicxjpDDj8Tz/hVSyvLpdkSXMyID0VyOO9V7mSW9a5dcq0z5G4fw7v8BU9lp18x+WBm4IyFPXFTYq9h6yvKS7uzHk5Zia7G21RNI0LVtXlUM7XGFUnG99qqBn3auZGl3VnJAtygVZWCqwbI69DT/Gt5JYfDuxuYVjcy6kCVkXcpDM45/SmlYls8y13xDeeKbiV725e4RNxic/KhDHkKvZQAPc15vqdoLUoY3BV2b5Owx/St+5kubWSJoGEjxIBsOPnUcFaxvEcflaoYOAY0UlR/CWG7H60yTGbG48Y9hSbeM1KsqhQkkYZfUcEfQ1NDELiNlUElc7VXk/j/APWpjKYpxXikZdrYIpe1AE9pdvatxyp7VqZOotGd4WRwWPbIH+TWHVgXLxSAxttwu0cZ7f8A66ANMGS3jVQ2wvKBv28gYq3d6FqCuXeYGf76RlwokTuVYnGeelZkGpbg0c/3W6kGui0nVJrZEXyUu7MSK8cbk4DdipPRv89KARBp9zFIkSCYxr0aULmRP94d+fUGte5vobXUlttokjlQGN1w2845yPWrd5o1n4hjiuNIupJJ0T/UtFGk0W0H/d8wYz0w3sapLpfiTSrq522kU0tsm6R45EEjDPTafmLc5xjNI05bkP263yyb0dOAARtdf931HtQ0zxY8oeaCN2YJOT/wH1qjdReT9p+1afeI8nzKUTKozDd1Xg5B9e9Q6etjPdpbyXJsWLD/AEht3lqf9rjIx600Lk1sX5Lxd53oueoEkRVh9TTJLrzPl24yccZPHY4PT6V1l78L9djs0ubbVINRBG7yo3KMR6gtw3HvXM29hNa6n5d9EzFTtZJkww+o7Ggv2Ek1zaGWVubllAiYZG6upsYGt7VFfBkxzgYC+wqdbeOAlUjjTnPyrjNO71yVKt9EfT5flcKD9pJ3fQBx3pe9HQE5x712PhLwJca0UvdRSS30w8qvKvcD27qvv1Pb1rOMXJ2R6WIxFOhDnm7f10IPBnhF/EN0Lm6Vl0uJsOc4M7D+BT6f3j+HXp7GqKiBEVVVRtVVGAB2AHpRDbxW0EcEEaRQxqFREGAqjsBUmK7qcFBHxeNxk8VU5nstkMoHUU4ikqnsci3PIsAZ+p/nRgUnb8T/ADpcV8g9z9Fh8KDFGPalAFGKkoTAowMdKMccVueHfDcutzebLujskOGcdXP91f8AGtKdKVSSjFamNevChBzm7JDfDvhuXXJ97bo7KNvnkHVj/dX/AB7fWvTrW1hsraO3t41jhjGFVRwBT7e3itbdIII1SJBtVVGABUtfQ4bDRoRstWfGY7HTxU7vRLZC0UUV1HCFFFFACGiisbXdft9EtgXBkuH/ANVCDy3ufQe9TOcYR5pPQunTlUkowV2yTWtbtdFs/OnO524jiU/M59vb1NeW6lqFzqt213dvuY5CqPuoPQf55pt9e3Oo3b3V3JvlbjPZR6AdhVUgkHHHpXz+Kxcqzstj7DLstjhlzy1l/WwpA9KTaPalYAuxVcKTwCc4pMY7VxnrINopcDvSUnPcUALtHqKNoHpSY9qfDBLcTJDDG0krnCovVj/nvTSbdkKUlFXb0ESJpJFjiUu7ttVVGSx9AK9G8MeEk00Je3yq96R8q9Vh+nqff8vex4a8LQ6PGLifbJfMMF+0Y9F/x710le1g8Eoe/U3/ACPk8zzV1m6VJ+737/8AADFLRRXpnhiEA15Lcwvb3U0H/PORk/IkV63XmviCIQa9eLjhn3/99AGqjuJ7GQc0Bip704kZqNqoRLq+p/bNKsbVkw1qX+bP3g3+FYIkdo3QuxCkMAWOP8/40ajqlvDeRWYO+RnCyEHiPPr79OKYsE5JkSMshBU4/wA+orK8XexryyVm1udP4d1BbaSyvGJ/0d/LfH90/Lz7V6uDxXhulM6TPBMDtn+XHpkf/WFeueG783+h28jkebGPKk/3l4/lg/jQhSNRlDoVYAgjBBryS6tv7I1qSExpJ9nl+VZBlSOq5H0Ir12uC8c2IivYL5V4mXy3PuOR+mfyrz8xp3p866HqZRWUazpvaSOcuL+4uZkkmkV2XOwFQQuTngVs6VqOrTSj7NaJNtYfMsKqI/UnA6kZ71gLK0akKuF6scnn0z9K7Pww9wthdma1W381dsLrFsaQ7Sep5P1PHvXm4NSqVfiZ6+YONKjflT7HPl43GX5zzQIowPkcqPQHFNGQAM8gYpDDhg3zH09BX0R8m3qWVmvByl5ccekpP9aWW7u7iPZPcyyLnO13JFVycDOB9cc04TvkjqPzoEJkryNwp6SvG2Y5Np9UJU0qurckYHtS4RjjIz7igCx/ad2V2PO7r6SfOD/31mmC6U5D2dlJ/vRbT+a4qDyFH3eM9xQY2Hf86AJR9lZjmxZM/wDPG4I/9CDVKtrZSAlLueFvSeHcP++lP9KpFHzg5GO4OKcDIh7j60WA8kuJbJGJF07kewP6VQknSRQpacxjoCAB+VadtpkUkwUWd/L6iF84/wCBBcVvRaN5sf8Ao+hTFyMb7mZmH6sPyq9EI4tJ0tyGiZ0cdzIBmtGDV2eMLJqV856ERuWxXVW/hm93gvbW0Q+8V3qg/wDHAzVuR+HyItrrZqPT7KZP/Q2/pRdAebSZuN37y546tPKFH86jjtrYn5p7eZ/7izsSf/HRXqUXh6zj58uB/raxKPyC1ajs2iA8u68lP7sUSp/Kp50Fjy9dKuZfuaUiL2eSRv8AA1ox+EdWdci1sh64d8/X5tor0JbQM2+S8v3HoZ2RT+AxmrUCNJJ5dvE8rDsiFjS5+w7HncfgLVZEJe5tIT6b2b/0EVatfhxt5ub/AC3pEjAfma9Jj0fU5D/x67F9XdV/rTzo5h+a51Kzg9Rv3H8uKXtGHKeeR/D2zDAvMzqOqnJrRj8E6Ai82Qkx1JdsfpXWOmiRA+bqFzc47RRBR+tNGoaUmRbaQZefvTSFs/gOKOZjsYEPhvw/bcR6Pp6H/bhVj+ua17exbhbKxwB0ENvtH5gVabXLpP8Aj2tbW2H+xEM/maqS6nqU4YzXc4U9t/H6YpXkwsXv7Hv8ZlRIVP8Az2lVcUhs7OAnz9VhHqsMbOayN6s3zSlj9anjs7mf/V208o9o2alYC4zaQi/LJeSv7bU/oapSTwNgRWsmf9uZm/ltFTNpt5Hw1uYc8gSFV/maF0fUbhl8u2lI/vDCj8zT0Aq/P1wo+rUm1y23I+gGa2I/DGp7f+WAHfzJckfktRvo32Zj9u1OzhA7JIWP5YougsZ8Nu9xNHCGfdIwTggdT9K9NRFjjVF4VQAK5PQrHTW1FZLdrmcwguJWOIwenTAyea6/tVIRXvrlLKxuLp8bYY2c/gK8shd4tOmmc/vZs5P+03X+ZrtvGt15eirbAjdcyqmP9kfMf5D864i8+WOGAdWOcfpTQ0W9JAt4t56tWul7yOCex5rGlLw2MjoCNijkfw54zWTa6lDpn2me5l2QiPLEnqc8Y96TVyrnSeItTjt7O3YuABKWz6YU/wBTXlPivxc+pSWWlJKVtrLG4jPzNj0xxjms7xN4yfVZSImLIpIji7D6isRprcQkXdyszYHyRfex6McdaaViLkd/bPLdZhYEyYwR1A61zF6XW4L7nbdzuY5J/Gu78Kz6fc+IreG8kW3s5GG89lwpCZ9Bk81n6p4bEf7iZlEkEvl5DZDjtz+FWot7AlfY44FZTgjDU5rfj5DyOxq7qOkPYzqIZFmRk8xWQ54zg0ulrDczlLhtgx97+H8aLW3F6GcsbyOEWNi57AVGeDiumg060nt1edWBfcysDyFzgcd6pT6dFFD5aATSZyJFYrtHurCpAxutPSMuaVo9vUOD6Ypysy9BgehpAMaPA4qezvpLOXcoDL3U9DUyWkl0C0WEXcFCtzz9a6X4f+FNO8XeMRot7dSQwmKRlkgA3MyrnAzx6n8KY0ijaap9oO9Qqt/EAcMfxP3vxrdbxz4hsrZoEvbiWLbt2SNk7ffr+HPavR5PgTZ2eH0y+SZwOFvozz/wJOP/AB2su6+FuuQQ+XHpkbqo4a0uEb9Gxn8qhyt0Oulh1ON1UV+2xx9lf6lfgxzy2kbOMqpl27s9mYDA/HinHS5obiKW9tI2uFcEus6MVbtyD7deRWnceBPE6EhdEnfIxgxN/wCy5/SsC90HxNpzlbnw3qMQJwuyCRkP0O001rsTJyivePUvBXiUXc93oV00IuLb54GhkDpLCe6kd17jt+FaXiPRLfWLFkl2pcxjNvPjJQjsfVT3FeXaL4J8aXuo219p2kX1tJCQ0dxcgQAY6ffOSOvGOc4NezG11CGztl1WGGK7kX5vJffGXxyFP64/nSfuu6OqlVVSHLLc8su7Wa2cxTJiSPAIHOV7EHvU2k6LqOuTeXpto8+DhpOkafVzx/WvTbDSNPv7yGO+s4blA/3ZV3DOP/rV20UMcMSxRRrHGowqIAFH0ArP2SlK52/2pPDUlSSu+7OK8OfDqy0x0utTZL67XlU2/uYz7A/ePufwArtuaXFGK2ilHRHiV8RUry5qjuxMUU7FGKZiRkUmORT8UYpPYaep5Fs4P1P86aRx0NdJI2jB2H9jQ43MOsn50D+yWQONChIPfzJBXz/1FuVlNH1azmEUrwkc3igDPaujI0kED+xIBn1nkp7R6SMg6HBn3ncVf9mVO6F/btH+V/gVvDnhuTWZRPNuSxU8sODIfQe3qf8AI9Mhhjt4VhhQJGgwqqMACuVh8TyRRLHFYQoiKAqKx4Hp04py+LZyM/Y4x7GQj+lephsPGhGy36ng4zGVMVPmlolsjrqK5P8A4Syfg/ZIiD1YSn5f0pf+EquMZazjX0BlPr9K6TjOrork/wDhK5sn/RosD0lP59Kj/wCEvn3EGzjXHdpWGf8Ax2gDsKK5EeLZ26WcX/f0/wCFA8WzcD7LFnrjzGH9KANHX/EUOi2+3iS7cfu4s/qfQV5jdXNxe3MlxcytJK/3mP8AIeg9q6O6m028upLifTIZJpDl2N1JnP8AhURj0kHH9jQAgdruSvNxOGr15bqx7WAx2Fwsb8rcnu9PwObwfejB9a6Py9JzgaNDjHX7XJSbNJyP+JLEPrcy1yf2ZV7o9H+3aH8r/A5zB9/yowa6UxaXsUf2PCQPuj7XJ+NNMelBSf7GhxjP/H3JR/ZlXug/t6h/KznNp9/yp0sMkMzwyDDodrDII/McV0UY0yGRJF0aENGQysLiQ4IOQev0oKaY0hJ0SEkksT9ok5Pen/ZlTuhf27Sv8Lt/XmYFtZ3F9dJb2sTPK54UdvcnsPevT/D3hy30S33HEt24xJNj9B6Cud07VYtKVxZaTbxFsFiJWJPpyc1e/wCEvu8kfYoRgd5D/hXbhMFGj70tWeVmGZzxPuR0j+Z2VFcaPF92efscOPXzGo/4S67JA+yQjP8AttXeeUdlSZrjj4vuv+fOH672pR4suv8An2gP/A2oA7GuD8ZxCPV4ZeB50P5lT/8AXFWx4tuScG2gHvvYisvWb641gweYkcBi3YxubdnH+FNPUTRhkZYeua5/Wdd8kva2Tfvc4eYdEPcL6t79vrXSSafJIjRvN8rDDfKw4/A5rMl8NafFBI4tYSEXdgFx+H3qiq5NWgbUHTjK81c4R1JUgcH1967bT7gzWyzp/ENx/Hn/ABrl9Z1LStInETaKZcorhluSv3gT059KueEfEdpqlzLZwWUtoFhMiK8gcMA3OO/8VY06UoJ3Z0V8RCpblWp108Zktcj7y/Op9DXVeC9QAv5bYn5LqMTx/wC8OGH5EflWdLYM+lyXqfdiKBgP7rd/w4/OszTbptPvElXO60mEn1Q9R+W6tkcz1PXMc1j+JbD7foVxGq7pEHmJ/vDn/wCtWsjq6q6kFWGQR3pTyKU4qcXF9RU5unNTW6PJLMoQPMuBbxjDho4g0hI6YJ/xxXW+HLiJdLulmu5ZEfcVFzgEqB8x6k4/QVV1GHRLDUJbZ9FDFeciYgMDz0oGrWEVlcW9tpvkGdWVm83PX9a4MLg50Z3bR6mMx1OvBqKd36Fi40C1lUPbyPBu5A++h+n/AOusyfRL6IkrGJl9Yzk/98nmoILma05t5ig9Fbj8QeK1LbxBIPluIFYf34jtI+oNd+qPK0MMDkrggj/Z6fnRsBGW611hudN1JAJTFIR2l+Vh/n2NVJvDtvIS9vKydtr/ADj8+tHMKxz5YluC31oDEd/pir1zpF3bZZodyf3ovmH+P6VRGC2Qw46+tVcQ9S3Vfr1pwkyfm2gfrUYBHIwQacCw7YH0pgPynHJH1p209juqEH5vfvTlXbyoXOe3BNIB24AYy2OwJoEiAfMp9+c5qqzOx+8R9eMU4byMEr9etQUTtMoyvJFN3YPA/MVGQxI+c/hxSGDcDkfTLGmIlJP99R680zegPf8A4CM0RQ+ZJ5UaBpD/AAoMk/hWjHoOoSsFNu6qe7jgfhnNIDPLA/wSf8C4pN0pGzOF7hWJrd/4RyOMKbq8aP1Cqq5/76b+lXY9N0pMYt726I6YBI/8dwKLhY5MjdwXyT+FSQ2ck4/cwvM3+whb+VdU8F6mfsGh2cIPAeZl3/lVWay8TSD99dpGvoswQfoKLjsZn9iXkce+SJIF/vzSKg/U0i2ljET52rxHHaFTL+oGKnk0yxgJN3qDzyd1hKlj+J3VJAuiPxFpl5dyg9Gct/I0AVDPokRA/wBPlYdMssYP4dauW7GUZsvDgc/wvPlx/wCPACtG3t9TQf6Hptjp69iwBcflmrJ0vUJ8G51iYjusCBB+dK4FG3tdfjXdLd2djH3HloMfkP60CW1RwbvX7m8k7RQcA/gBVoWGi20o80rNPnpLIZXP/AR/hSvPqQZk03S4YYxwJJFCZ/Dii4EX2+QIfsGlXje7Ls/pzUTPq7Ms1xdpZJ6EZP5Ef1qSTTdeuzmfUY4x/di3AfoBVWTQLSIlrzVwGA5+7kfmTRoAjyaW6g3uq3t9gfcXcq/kP8aibV9OtjtstJiU/wB+UAn64/8Ar0pj8OWpGXuLlvXccH+VW7a8AX/iWaCwHaR1wPz/APr00M1NBurm+tXuLgjBbaiquAAOpFbFQWvmm2jM4AlKjeB0B7ipj0rQhnCeL7kXGvwWwOVt4txH+03/ANYD865wnzdUI6iMY/Krdxdfa9Tv788q8rFc/wB1eB+gqnp4YrJM3VjTKRYvp2jsJIgcLKVD++DkV5f4+kl+zQ/ORF5m0IP4j616fcRJcQ7GJHOQR2rz/wAbpFJe21pHtIghZyN3IZuhI9T70k0JnmjSFAQB0qo8km7vVqWNjLjsOtOWLdEGVTtPTiqJGWkzpIDnFdlFqe7RZIr7dIFCGO5BBKhW+649ACfmrjVjO/kfSrcNy8alCgYYxTU2ti4TcdjXbTrcy4CKUlB2OGI8tyc8EdmH61ot4djmiubmzhCIECSqqna4Pr798iuZsLq6sJNkUoaJjuEMg4x7HtXqvhm6srrSNsEmLp2zLGW55/hxkA+1XOakjO2rPLpr2S122t5GwZfkUr0OPaozNDI/lwJJPM/GEBOfoK7jxLZxRXOoJcWv7uLLGJhk4Cgr0/zzXJaYNU0x7c2d0bSLUdjKqIHyeex52j696iwGRNp13HJ++s54T/01jZP5077FKrKWVdprstQTXbrS3tLjUrIwylWzJCVLbTkf5xXI3Gm6ikfn3KCFZ3ZV2AH5lwCuAfl+8p56huKQ7FK41GZGaKFvLUMTlfy/lXa/BaJR8R7G6kl8qG3SV5WPQ/IQB+JPT2rEm8IfZJBFLdtLMwG2OOLkH/aJNLYa03h8Wpsztb7cJWx8paNPl+bHqS35UrWQ0z6qn12JSRaxPJj+OTgfl1quPEE27DQ2+B/tNzWCJopNkkJLI6hlI/iBAPrUn2gIAQy7VGM7s/n/ACrO7Gbf/CRkruW3hfnHEhH6daRPEkpYbbQAc7suykenasUyoWxsw3T5/wClMBYk7kwDgYA4HHShMLHQp4hVz/x6kk995/ntqO61m1u7eSC5sBLC/G1pPvdwRgZH16isUSM5PBZdvLY3D6Z/zipIrS4n/wBWuF27dx4VT7fjipnUUFeTsUotvQuadd6Vp7M1vaytPt4eWcuT+LdK1F1+KTiK0mkYdkKnH49BWbDp37qP7URLLgbhGSqZ74B5x9akkuIbRNiJkKM7Ih3/AJVxSx93y0Vdm/sW/eqM2F1JGUFoZEPoSD/Kmy6xbQj5w+7rtXBOPzrn2nu7qQxwbFQZDbBxj6nFTQ6ckeGkYu3oowB+PU1LxU6etWS9EP2UZfCjWj1y2kYBIblv9pUBH55qx/aMPpJ/3zWRLPHbrtxyBwqDH/6qzLi6uZsqF2L0KqefzqoYjEVneKsvMUqdKG7uzo5dbsImKtIxYdVRdxFRnxHp0fLNMMc48o1ye3DcKdvuMcf57U1lkB4OxupJ6ge1d8W0tTmdri7l3M+9gf8AfP8Aj9O1KWPyq7yHn+N923/PNNSCT5lO3B53DqaVoXWMd1z83BJ/zxUqnBO6X4FOcno2A+Ykc4+uePanLs3N+8AJ5wwyKaI4wAyo+1Tnlcg8U7DEkL03cEgjriqJAiH+8xy3HB/zijKGXIZfl6oPr/8AWpdkjLvXAAwfbryOtNQOsaq43dBlYwufwzxQApK/P8uf+Ajv7UjN8+3qMcYBbkdqczZLITjk8lOv4dc/hTNzKo/eAfUc0wGOVGeTknuMDPbtmmYQf6xFy3GR3qVgVI/iQ/eL88UxIm3D5B83XngfWgQ4mEDd8n91Tj+tSAZ6DJPTPr7fhSBiAQqEE9aejM3yfMF/2tvSgBq7RgHLYHI6/n/Ok24ZtwA7Yx/nil8twFAKgenJx7imFMDC4OcElcdPXmmAMMdVPpnbimgsAADz7k04Jxu5Yk9AOeev0+tIuOAvyn1agBu5Gz8ysMdeopwPHynBz3FN2kfM21vf+lO289cNjbjdwf8APFIBCCGJLFD0wDTs8/fBB9TTFXYNxZQnp6CkIfJI2+4zQMl+7wfl9QO9IZFVl+bBPQGmbc9WUCmkoE6OSeFwpNAiXdtH3sZ4Xjmmn5ht7deXqJQQcR+WHJ+YleT+NPK5PG0ZOeTn8KYDkY4GY/mAx1yPzprAHqz4AzkcYpDBG38IGe+eT+VIVfoY84weuTQBNuZW9QO+RxUm4gZLJj0qBGVct5bZXvnIP0pnAO9EQEDB65J/nQFi4CjgEMUxgHIzkf41BdwJdWVzb74z5iMgz69j+eKRN25SHdT6f1qU3O3Y3lttJ5YLnafegDyDxZH57283A3wL+G1m/wDi/wBKzvCsq2HiK2myR1Vh/sscH9P5V1Pi+wUrdInIjeQoR/dYbl/pXDaGTNfXMy/MqxqNxPQt/wDqqugup9SeGY47/wAM3NqxBYs8L+v3eP0rhWOy5hkYY8weVIPf/wDWP1qTwn4mfTpEuShkiuIF8xA2PmHcfjmo52F3HLjhmJcY7HOag0R6R4TvTdaJHGxzJbEwtn2+7/47ityuA8Gajs1Py2OEvIun/TRf/rbvyFd/1qiWcr4us1Y295tY4PlPt6+q/rn865rZu7V6Hqlr9s06eADLFSV/3hyK89Vgw+6Bn8cUCI23Aj7v4kU8Nx8yj8DinMAByRx7UgVTlSPfigBCQ6YwPpVi1u57ZgUdgPTORVXyV6jKr7GnINo4yPeiwHRWmuIci6BXHR0XI/HHNaDQ2epRbysVwv8AfHVfxHIrkMlRkc+9KkjxsJEZo2HRkbaanlHc27nw4rDNpcFD/clG4fn1rKuNLvbVWaW2baP4kO8fpV2LXruE4bEwH/PRRn8xWnDr9oxAlLwPjOcbh+Y5/Slqh6HLDZ97dke3NIMMcDPp0rsZdL0+/HneVG+7/lrE2M/iKz5vDK5zBckEdpF/qP8ACnzIVjlml5+WHB9TzTt0mP7ox3FBi3KR5nPfgU7yW5z85647UgEEjMMF/wDvkVLA8ayI8kBmUHLI7Fc/jTRFtyPLUA84Jo6k/uwwHfdxSA37fX41i2LH9kQfwW8Sn9WP9KnXW7GXPmJqEo64dgo/JSBXNNIgbJ5x1wOtPErdMEZ9qOULnSjX7C3YeTpzD1OFBqGXxXcMSIoIkGON5LH+lc4ZXDHgfXmlCyFd25cH86LILmlLrOoTA+ZeuAR0iAX+VZ7nccyMzkd5CWNAQZGXGO4xzQYkAHzbu+cU9ALmnPpkcha+jlb+6qcr+Pets+JLWJPLtLV2QdBgIo/AVy6gdEduOw6VJlAuMZGc0rBc3R4hllJ3s8IHaKJf/QmJ/lUMmr2bAeZbTXLHoLi4PP8AwEcfpWMHZ+MN65Pejyn2bvlC47HmhRC5srr13CmILezt4+wRKgm1vUJM/wClcD/nnhazfLJHz5z+OPamhMknKg+uMU7ICw91LKN8k8jntmRj/Op4W0tQvnLdyMvJVWVVHtxVZI933gAegXPH6Uux4k7ADjp1FNIDbg1jTIBuh08of7w2k/nVyz8QQ3t5FbpbS7pGxksOK5VEBCl2Cuf4B97n6VveFoC93JKVAES4H+8aOVAdaOlZuvXhsdDvLhThxGVQ/wC0eB+prTrkfHFz+4srFTzNLvYf7K//AFyKoRx0ipBpwicblYbSD3pjajbW8QV3Icc7QpzzUOqT7Z4oR/Cu4/0rndTFzPfzqnyoDsU+wAH+NSUaN/4kMVuXt7fdIxCRIx5dz0H+e1cP4ivDpmpRQTSedI9uZrhweZJHP8gAMe1WJr37DH/a1xtdE3RWaHgyH+Jh/L8PeuG1DUJtSv5LmZy0jnr6DsPoKpKxLZoyRCS1EmAY5fkzjjnnira2sbWgjUqsu7HsBj9O1T6GBc6YITy6n5Rjt60rWEkZby2+Rwd2f1qhGfJY4cbTkYXJ/PP8qRdOcqZQAFwDk9K17O3edcbSAW5I7VpT2W+N0jJRlGAv3cr/ACFIDm1tIZGKEYQfMY3O1h64NXhAsP7kfNLgMJA2CV9COh7f/WqYR4lw0mHAyrkZDfXNWwo2OzpujxnaOx9aBmXc6jq1sDLC3mOuB5cq7ldR6HOQcds81S07xFo63CyXGnC0uAT+9QFwvrx1X8K17l3GHXDJIM4jbqv97Pc+386wzZWFzO0l0j7W43qcMh/2vWnzMVjpW1jQ0YzfbrY5HUEs38s1lz65YzXomSbbb7SsvUg4+6WGOucdPSsjUbNNNMdrLbxz2khLRysu10b+6XHXt1rHvZ5GAiWHyIVOFjGcZ+p602youx00+s6agmnt7w+c4Ztm1iruR6EcHPcVyRmaWZNxzgBRVc09Adwx61LbCTvqe+fDXVjqnhgW8rb57FhCfm+Yr1Q8+2R+FdaI0DYLz+g3SBv515D8I7iQ+KG04SBUvYHXB6Fl+YfpmveY9GgDB5maZ/fgD8ua8zFypQlecmvRnVRlO2iXzRk+QM7LcTSTEAMZCMfkB3q9Fpc0kY8zZC2eWxnI/wB3sfrWi9xa2vyeYiMf4EGTn8KqnUWnkKW6twOdq72z9OgrnjWrSVqasu7KlGF7y38iwlrbW8Zdl3Duz8/hj+lNlv4g21csQOp+VahWyubmTfcHy17ZOWH4dBVuO0trePdsB29Xk5x/QVhN0071Jcz8tik5fZVkUy13dqojPyNkMfuhffP8WP1qeOwSMjLvnOSEOBn1/wDrD1pLjVLaFRtYO3Zc7Rx7n+lVri7mkLDzAiHgBBz+vP5VrCFep8C5ETKUI/Fqy7JPb2ybFwSn/LKPr/n61n3F9cSM6xlYU2/KSNxPuCPaqxdN2wcr0xwv/wCuo06gbC7D0AxnI/nXTHL4Jau77mf1l38hkjksxO9iCRkRPn39OaRtv2dXRi7f3PLZV3E89fw71IY0V2AIwR1f+ZphDxtuwxGMbtuPxrenh3B35mxTrKatypCg8YIw3DbSdx9/8ig7VCjO1eNwPT2/CjerkIu87c/Lng49KF8rH9w4zhyBnpx/M10mAw5XJy7BM5BAYL+YpyqPlIc/Q/p+dDbW2YOSD95Owx370HDklsbmwOflPTr9ffpQApJMnAYKpGRtByPQAjg//qpYyrxruy2AflTIzz+tKFWRgm0KNvY7uMc4NKBmQcqGYY3fdH05/wDrUARM2MspAP8AdB//AFUNsLEtGOgUep6ev+elP2nfuVlKt6j8yDTNwJ3Y+X+Hdx9Bt7D396ABuQVAkUkHIJOV+uOKaBtJJPHPzY/z7Ck2xkt8rJghhhtw3Dp+I9aPM2B9xJP3iBwfTrk5/wD1UxDDncxIKjG3qe/T/OakDKuPMRmXuFx83P1oQJvRwSq8lRu/p3oV2RQG++TuZGIztx/+ugBV3tgK65yQo29P8KFBLnc4O72wPqP8KRiVRt7n5sqc5P59hj6UuPkC4BXqoOfy/lQA3COhw+4g+ucN74o3ZDblcZHQ4xinkZHy7i3TOM496E3MxUdDn6e/+e9AEe/jcFBwO3fv+dOfIVtuMNzmlZgFHDLjPG3aB6U0ruUE7g2SWBIx7UAHBYeZGCvYEZx3o6qAvAYEqAPvU44UfKMq3K7X6j8aaWXHKcnoGH4/l9KAG7s9QopfkzjAL9ee4pMIPuFU2gDHenYd8fMQT2OP60DGk5bgfiaXahGR25waYA3JO1mx1Ukj86dyE+VfmJ6evvQIaclRg4PoVpTnjO0EcHFOwuGyilW4yT1/SjaGOG3nPHPHNMBpPGRyPZsfpThtBj2hjjPQUvlMFwsZJ6AHrj0oxJtO1lG31HFIB25ePl5J7mkAVuxXHoMFaRo8hsEDnGMdPwoBKN/rMDPGWwKAELHgsqfgO3enEhcHIDA9mwaaW3AshYjnJHOaPnbBBXtnf9KYHnvxOvXtFjt7ZXeS8g/euqk7EViM/Vuntj6VynhOwkuNL1e4jeMCDy2ZT1IVWbAP516v4mtvtGmo27b5UikHGMBuD/SvPkuZbRrgWF0tvNJhZ41278jpnPQ4PX0NUthMba69Pp0ZMTxmL75SRcj3x3FdzFfwo43ZVTghuox2rzqK2tE2xoQD93B4/DPSumheX7FChQ70QIc+3A/TFEojUjsLK9SOcT2jhzBL5i7ehKnkf59a9et5kuII5ojmORQ6n1BHFeA6FM8M8iycKXVh+Pyn/wBlr2Dwdd+dpBtifntnKY/2Tyv+fapRTOjrgNasxZ6pMozsc+YuemG5/nmu/rnfFNp5kENyo+ZW8tj7N0/X+dMk5IlSdvOPY9aP4QOg9DTnGMgjtwR1pm3b/Ceec9KAHqWUnqMUgRSA2RnOQcUm7phSPf1pc5BIZh6igBzIxywRQD6VBsV1+ZAR6b8VOF2nIPysKaEU59OxPagBEUJ/yzYAf7WadGdrbw3I5GOv0oG5B/s92pS438u5HYHnFAD4LiaKUNCXjPUujlT/APX/ABratfEMkXy3SLMP+ekfyt+XQ/pXPhiCMjbu4GAcH8aUkeVv4U4wyNSsh3Ghi6jAxj2xxR8pXqCO5psY+bbnLenelaNh8zKMf3uvNSA7IOSP4aRU7iPnsaVCD8obO3tjFDqd7Z/A9vyoEHlsBk4ANPKAAk4+nSmhR93bkH06Uirtk4GVPqDQA4KM/fGT2IxSK2SRt6dae4UlecHb3/pTeMj5WII5ycYpBYbucfKPLHoQTmpQvTIHHWm7P3jZAGPbOPwpoViWL8nsdvSmAjLG7KTJKOfu5wD+FG9sD5XHoeCT+FOXcuTksvYkbSR9KEiWQ4ZcAk56j+lAweUOpZ+MdMqKAXLL3GOMH/OfrUwRTtG3D9On6UzZJ5h8xISAOoLBv14FMBCqqBuJLei0pQHCnDOBlSopzAbjwGJJ6jrSbySy7c9uCaNQDauBkcn+EN+tKdxbYFU56YbdigO/zZCLt43H5sHpwKbvGDgZ2jr0FACgY/iD/wB7bzj2H0rsPDcATTPNHWVs59QOK40pv6ZII+UDDfyr0S0h8i1ii6lFCn600BMWGSK878Q3P2zxRPg5S1RYhj+91P6t+legTypbwSTyHCRqWY+gHNeURSu8E97L/rJnaQ9+pz/WmBQa2S/1CQ/NnO3IPYVT19La0i+yNKU85DJczg48qAffb/eb7q+5rXtWg0/Tbi5nVTGF+dm5+Udse5rzLx3f3MZ+zzr5VxdYuLlB/wAs158qL6KMn6mjcDl/EGtNq18ZtiwW0a+Xb269Iox0X/GsGJvmOe9PYmaXBPy+1O2hCaog6Tw1fCC4KcneOx6CuoMcV1HtiIw/TeOMeledW8hUgr96uz0hpJoQ8jYCjg/0oGaLQwW8nylXO0K2AMfn61YTDLczeWG2t5RBPJHsfp0qmHjMe+RgfVgflGf/AK1OtGd5sEnZ0I7KT0/lQBK0NqJmUKsiuTjPDepwfWs7VHlkglihDHGHVUbkr6/WrhiaWYwtyME5/wBrsRXLalceQH+cebgo4A4J9x6HNICxHeIkOd8TM+ThuBIMdf8AZbqDWbJqyOrwxOWhbDKkgz16gHtWNc3TOAAxz/F/te/1osrG6vSwgiL45pgad1e/aLNbTzTjZ8uR1IPH+H5U6SQ3Hh5kcBJLaXzGQ8FmOB0+mau2Xh5xcILpWMccXn/d4PPI/Q1naxH9hvp4lIaCcE8NnLfWgDcl0uxEMD/Zy6bV2k9ieQM9/p2rO/sr7QJWjCqjOwTaMnjvmtfT4ZZNNtRIRsjiD4+6S+3A59uOaiSV7a7KbifKBADd+ef8+9IZU8K3U/hrxrp8kgAe3uV3A9GU8flg19JNHqF43+s2w4xtZdueev8AkV8+axGl/DFcRP5M8PzRnPBX0z9ea958O+J4dZ8MWWqSMPPli/fIv/PRflb6cjP415+OcotThG7OihZpps0oNNjj5kLOewB2j/GrJnt7ZcHZCp7fd3fT1rMl1gCPKeXEvGMncev6flVYz+a4bep34XBbDDk8n/PvzXJHC1qzvWkaOpGPwo0J9WUEpFF84zxICPocDtWXPO12d0m/j58HIT2A5wPekYK4bqQ3QcnjP+OaRQEfewy+3kgg4/Ou6lhqdL4UYSqyluRmZYwf9FnLIM7Qnv2y2SetWTIiyBREVCYxJwvApg2JtfLA8c5DGm7ncurxszbcBsdCeuTXQkZiiT5zj5SOvQ8H69KjV3lLNh48YypPB9QRn0/n3qVtgdSjY39Pkz2//XUbJkbFTOfRAD6454//AF0wBrjIXZgDjA6cUqMrL8gHIx6/r3x/hRLteIsrPtGDtAx+HTkdO9BX5d0jOB2JYbW79eo6UAxm9cDjdHvDYz8vpn3pQ2AyI3K46qCPrxTiuGHKqD/CEHAz7dKaAS5CsJSF5HCgc+vTsaGFxA5+XKfM3Jz6etJ5yqoXBC4zlf4fwpdmVOSuGYHBHX/64pQGjfaXyBnAb5sD0/8ArUAKsqGVuNwOF+UYI/8A1/40NL+7xNGSNw+cnYG+n8qUBJZMqGXd33bQB+Ht3prIh+VeUXCn0Zv6duv5UAHmbQj785OA/OMfT9PWmq2AvzMNoPzbTjPsP89c0KoJ3KWJA53KOnTr9aJA6AEOjFThjxjnPfI/yKYCcsygCNxuyysSu3296RJd2MspGOc5/EU4h1jG4lAox2+br71CLqJlbLj5SVOOffHHf60mA5FR9pDxY5GCvT/PSpGZXyN6YUfdHp/+v9aYxyQy/MSaVZEH3d7AYK85NMBAwGOcqcMfLxwO+M96l+YY3vvDcZQdvf2pkkcYYK5B2/ONv38d6EiIyS+T6/jx+NACvvXy8pvIb94M4xx659e3PWk3NJHlk3r8zEjIx7daiEShWKAHp9Pz/GlZHbGWyoIBYHafz9KBEjCQOTuxu/u8kcdR6inkNv2Y5ychc8fjUKp82GfO3tnj/P8AjRtJDH5cY7Lwp/3h+NADxGxkyyE5IX5X3YHfI7UoT5shmJ28naT26UxQcHaipGnGX6dP6/SlWJmYIhZ2UhiSoyB2/wA96AAQu6jyyB6blz/WnlNiBmOSMfOTx+dR+XJGPmbGOCV6j8qGXaAuSevTGPcUACud2Ock5AP09KkIZWK7VAz2phLdScDHAbJ/EccUjD5cNvVmGVOeo9fegCXfuLoWAwduR6evtRFEsahRMwTbj7xyPxOc9aZGjHcrouzPUruA4/WgZRtzcE4wRnP5elADgpz8vzdic8j/AApCW+ZPKGOOT37UjKxmO9/MOc+wH86kDbydzLnPzcfe70AMIwq7Vj656DPufajysryML6Y/nQWIAO8KV592ah33SKih8nlgQOfbrQA9VzyACQOqjoaa6k4cZJA6Fuvv6VF5gdCo+U9PnQHa3YYqRsFgTFjB+4Dzx70ARXdv9psbmJioMkbAc98cH35xXJDTdN1W0hlvNPtp22bd0kYLj23df1rsMyxsjmUAZzkx4Hr1rm4FENzd246JKWX/AHW5H86qIGO/gfQJZA62s0JzyIbhlDexzn9K65NEsDbJHFEUCoApLsWx/vEk/nVEn5W+hrqtQtPsl2ImOB5UZVdvT5RnH4g1UhI5K50aS3V5I38yLBBYDlR711ngzUfK1SEFvkuk8pv94cqf/Qh+NVsBlzty3QoCM/zwaybXfZ3UkcLYZGEsJHYjkVCLPZh0qrf2i3tjPbN0kQrn09P1pbC7S+sILqP7sqBwPTParJ6UyTzMKEc+ZnfyMDqO1Ip7DeGHc84rX1y0+zapMwC+XN+9A9+/6/zrH2oMMDyPToDQArYZWBJ9sUgDKONpP+9mk+bcc7OTg8ZxTxLtb5cjjjsaAASFV5IAPXHU0B9oPy/L9O1AZSST8pHqMU4AEE/Nt6cd6AGk5KujbWz/AJ60jKoJLA54PAxTgquRnAHpnH5UPEpJZcZC9BzzQA0bAo2cOODhcEj+VDuRxwRnGWUdKcYlCruxuwCQTxn+frRKhjwyMpj3bct1+tAFTzNx28qT2K9aegYsTlWkP3s9vb9akC7eVVduM4PQ0KjoB5R6cbVbBI/rWYxCkynczhFUjocj/wCtTlKc8PJ33cgUzdk7nB5Uk5y1LtMqHLEg/wALLj8R3pgLIzkfICD9Ad30psLOrYwF45+Xk59qj8kJtLF2Ofl4/Xp1q0ijZgEluwL8/nSARiMrjc3f7tAaSTcDtVe4x+uaGCLwiZkHYNwPU1KQwUKVjzjqTnBoAjjgDIHSLdtHOG25p81uwdSQnryecUsrOH+Zsr0PycHj9PrSSCMISwI3dSOf0pgLt28+ZEGOcAnBI9elNDyO4wod9v8Animl8qFUO6DgAHrTvLckERMM5IBPIH+NIAeWV2+ZlGByOufoO1RyKm3cnBx0HPNSspzkKTtxncu7kf8A16SOJQpLFiCM5A6n6VQAVcqRsIyM7iuCaZIXiUMAY1/n7VKUTcwwjkKSWKdvypomJbCnJPykgH/OKAIzG+SzphWGRnr+VTCF9uNu0f7u3n/Ippn2nYUYMfunuDTcgsDlg3fJ5YZoA0NGtln1O3QYZVbzG3YzgDP867uuX8Kw75bm62IOiBk7/wCeK6imhHMeNb17bRjAgH+ksI855x1OPy/WuJvv3VnDEByxHH6103jWTz9W0607Rq0rficD+RriNaeTUdZh0e3cqWQNcyr1iiPXH+03QfiaYyW2UanL5+FOnWJyjHpNKo+Zz/sJ29Sc9q8V8U6r/aus3l4M7ZZcRk/3Bwv6V6h8Qtbi0bw/DotiFikuARtQY8uLpj8en51487Ak5AJHc8049yWQogWMfL83UnPWmHrUnmZ44PaopDyRimIYkpil3dcVsRawxGAATisTqwPapIvvegoA63T9Re4nVH2nuAemfpXWbgLVS5Bkds+mBj26nmuB0WXF/CGfaCcHnGfxrpdc1N7a4WGHgJ29OOx/qaBo0wySmWRvkUAYPTH+TXKahpk9/M7Aj3Y9z1qey1WTbgt+7T5jx972x6VQ1PWZf9UkmAuTkd+KAIJvDsmmxLeXiK0bMUWM/wAWOufTrjPqK0LTXltHi8iGJY1jEfKgnAJxzTtKT+0dOaG6cgOdsWST8x/l9aw7uzl0+7eF0I9qQz0mxntLyxSdkwGyG2jpuHP4/wCFc5qxOn+IktpFjkhmjXG5QRtbP/16NJupl8PXnlHBUq3XP3eenuM03XXW5020mB5jYMHI6hu3r1HShAT6+5swwSQbVKLx12/wjNc4b3zTv5Jb1ra1SJ723iuFddrR7wBj72enHQc1z1xY3FvvLIwxyARjNAEy377BEXAGe1et/DCVLjwrKjSSLtvHHy8/eCnsa8O35PHQ17T8KkdfD1ydpIa5BBx6LWOIoxqw5ZGtGq6b5kd6RGRtM1wgz0T5WP0w1E4iaMx/ZEjf5TmRlJ6f7P49zTUVThTDJ2KhUJ5/n/8AqpfLdMokJAYDttrlp4KnTldX+80nipzVmQR28kQUvI/zcYI+T8OST+lTJu2jJG8HIQclfbOadskU48ncG7IrAY+v580m1pIwGR92cYMX+Pvj9a7TnHLFumVh8+PmGBgH/H9KQyKoyz4ycnJ/SmsplcALIS3QFTznsOBntTFQiQDy2IxgYkY5XryP8mkA5WYs5hXAPzZbg9t2M9cU5ZCHWIjg8YfBH1OPTih42G7Cgc9SnA9SfepNsqSMZGbknhIuR6fl9KAIS67yzgMwX5lfk+vX/PvSiKJWkOFGTngcAe/9KCzspwWU8D73AH94nH171HgkfI7nJIZSnGfXkZ70ALLtDMQS7FT0PLH09gf0pE3kBN6lsdQ549hn270/bJkMByPmwq/Mo9c0iB5VXYSVb5B8m7Prxxx9aQDldY3bzDn+pz2qFMBSu/5d4Z84GfYdv8KmfO0/K5CjKgchf/1dPxqGVZtjOjyRHb8uFEg6/Q59PSmAxmVw/wDqyT91GJ56YHoe1SFM5VEDsMKcNjGe/wCPSh5LohNkUKbEByyyN6e4wc9qSJp0lxKgDLyTBnaOevzZPP40rjtYQF5lnEJbMTfMUGCrHOAflA6Y5HXHfmniSQKxZWZiD8jADP5dvSm3Fv8AamTz7Ys6nKF1LeWfrj9fagqxUNsbHKgbTgfh+FGomRO7MsZLxr0+6T8o/EdaaCed0mNi4BJzkD3pxSZ5U3eYE6MUU5z6VIN8YVzFKdvCrszj8cc/59KoQ1fLRw/mPxwMHlce5/GneYyxKgOF4+XjseN36fSo/IZSGWJ41X5Qq/KQO2PX+dPW2cTPiGQsGBYHLZ68cj/9VADtsQ2qWGP4XbjLelNy6Rk4yTnlzz04Ge3bijyZC5KQyF8ZEjLtHHSlMcuH+Ri0Z5B+ncf/AKjRYBN0rIpYM5UKxxjuf05o/wCWxDHLY7D/ACKGSZFRG3qzZYKAw3Y6/wCf8alZWKglCFDdCn3gffp7/pQBES6um3cQT94Z/AZ9aaFQYCkEjj5n6/1FL5Eio7CGTzn+9+6z+v51R1g31jpk15ptrHM6fvGRlJDDuRyMd6LBc0AA0rZ2yMW4KqATz707J3Z2SP8AOo+fnj69a4lfiPYbS39m3zNwRl4uP1+vFOT4hac6SrNDqEe4YGIYZSeMcN/DT5JCujtBkDBOeDg84A/H05psjCPJV+uOGfGB71xkPxG00yqkmmXMFvnHmLIrEL7r279M11Fnq+n3hR7PUbeQd1jkTI/M5z/KizW47ouffGN7IvcgZP4dP8miZo4XKqcuedpPP0GB06UvlSKrACQLt9OlAicM+5CQ2chhgD3wMdqQDTzKzCZlLHkZO0eh9qGbdjcVbPIbHNAx8wcHIJYfLy35+nrUflyvgQ3IVs84iEqt/vA9PwNDAcOE5JZTwBng80Y2gbiw+Yc9QB6DP86ncAMcDDbf3gHp+dNEG1VALsw5PGPxHXP+fSgCMqVyMyAjnhz1/wAKaifKyyspQMVwjFtvqD/nvUqrIGIC47beB+dNJZZNgLruxwTuwOnYcf8A16AAhWI5ZzxgMecD8aaYkbaApKH7vPT604QZmKllBBByw3fn3pwVgu5D5gyeMHt1xQFw2qItxRgoOBlju6dMDv8AhWBqyiDXYZO1xb4P+8px/IrW/wAhTuG35OC2cY9vxrH8QrixtrkIUNvcDjb0VvlOfyWqQDtOgN1qVpb/APPSZFP0zz+ldv4qiJmtblZPLBDRk5xjByPb1rmvCMHneIoW7Qo8v6YH/oVdprqbtGkfAJiZX5PH90/zpzYkcYUZZBtWOVG6Bjgk9/8APTpVC/HkS29zlvlOxvYdq1AjpIFJWRmLA5JG0/Q1DdQNdW8sOQ25flwNu0jpxUjOt8G3e+znsm6wPvQf7Dc/z3V1FeZeENQMOpWkjcCXNvJ9T93/AMeH616bTGzC8TW/mWCzj/lk43cZyp68d+cVyYjLsFYqpA+Y8c89a9EuIUuLeSF/uyKVPHrXn5g8kyRSbmZSUb5uAV+nf9aBFSTzdiqkLSO3QbtvH1NQm7ulY7rHaV9ZgBn8Aa0FibB2lj3KAAlfc4NRqjoMDK7u6ru3DHWkMjR32ZwA4+UqCG/WnnK8N6/3ev5GkePP+sRCQNvCjK9/505CxYKCv0Ug4OOP/r0xDQ5VOSvJxtJ5p5kCOjPk578gHjpSSR7eCuf7gfq3+fal4AXfkluMMc4Pf9e1ACb+M7t27+FuTTVkKBW2j5sgkAZqZ4HEmzZt2jpjlhj0/wAajjjDZ4PzDPAwD+HfigCsjOfkdlZiedjZX/HNThTn7hLEfLx/jShACzEorjBOBkkezCpAVLGT92/JBJXcfpWYxvlgKfM+XC7vvdMfWlVI1JdfvD1OcCo1Ku/l/dzn77H8qn3BAWjDKm7r6juf/wBVMCBfnRlDMEYkFVTjg+9KqlFLM7quwqCnGP8APvUxLsf4cr0AzmkV2jkxuVD93OOKQWGNvIyBtY4OSozj/Io3bmCDJ57rtNSE+azMHY7W+UEHI4H+Bo2bhl3w3YnqfbOeKAG4TczgM46EZ/Pmk2Zk2LCFXd6j06UuwtgkF/4c/LjPr0pQkcgQBVYnn2Hb8+aAGqE+VsBOcYcgk+/tUgI8l8Fzu+6VU8j0A/rTWEaKY33rzwCu0GoLgorIA27GD984z6Gge5OxXcSATjjBOf6cUPIxdhLGVYjoj5zx344/+tUAuCColaIAfKT8w49PcYqXeI4y8jgJjn5uPrTERMCMbi/AIbPU8dz6UrFwCzAqM/MTn2pxu1VTtALMBgY7fj1zSxnBDnbhVJ5yce35UXYxAQnHGR8zk9vTmo5LiN8HyB0JAZcc5qYnZuRWdth4GMkE45/lT408+cRoWLSMEztGT2/lzTEdf4fg8jSISV2tJmQj0z0/TFax6UyONYo1RRhVAAFVdT1CHS9OnvJjhI1JA7s3ZQO5JqhbnnniDVB/b97OiedMXFraxZxvYDn8B8xJ9KytCgWyNxeXE3mSAtNPOfl3t1J9gAOB2AqSztZYrSfUrrDX0yMFGciJSfuj3J5Y9/wrM8Xzx6X8ObuQsVnvJBbQDPUfxH/vkH86iU1EpRbPJvEetnXdcutQffskc+WD2jH3f8+9YDMdvHT0FOuWO3FMVTt5rRGY2NGeTjgDrRMeelOAZDnoKbIOAadwI06/4VPEpLDioBjn0q5aSFGztRv97kUAathFIs6GFY3f0LYz/hU2t3HlPtkZGnxlgDkIPrU9mvmMrcA9TxgVka3C3n5jf5F53FcHPvQUVxcIFLFz74zVAyiWbrjPc1WfOcE/rTakR3ekypLZLCmeG3DZyRU96kV7YFJmErQfKkyJnaPRsdq4KG4mhcNHIysOhU4IrqLWa8uDFeRXK+cMIxCEFv8AeIHP1NMZLo0pt9UjtpZCttcfLJgK29fYH9DU2vXFtBpsUNjO8ltNIfL3xlWQKeU/DNbCWZCxPKLaN1LO0czgbiOTww68dq4LUdWmvo4bcki3t8+Uh6/UnuaANmDxO1nZeSI1LKflLchfoKzJPEF09z5xbexOTv5De2KxiaVRzSA3Y7q2uFYfZYIuP4U7/nxW5DdXNlYW8MFzcQg5ZhFIy5z06EVz+jIr3KqzYyOAfWti4cSXBKLgKNi8dhVwWpMmWW1S+k+/qF64znBupOP/AB6nrq2pKONRvwCd2PtcnP8A49VAA7uelKePatLEF7+2NRG7OpXx3cN/pcvzD/vqkbVdQcYbUb1l64a6kx/6FVLdn19eaXnPT8aLCuy6NV1AA41G8ycc/apO3T+KlGrakVwNTvwP+vqT/wCKrPLDJwRkHkCn575/SnYNS5/aeojlr+844y1zJkf+PUDVtSGMalfcdP8ASpMD/wAeqlnqT1PU0vXIxgUWHc0V13WFGP7X1FV9BeSf/FUx9a1RlwdS1Agf9Pj/APxVUeg9qDyAMdfSlYV2WU1C8zxeXQJ9J3/xqf8AtjV1P/IS1I5GOLqTp6ferP6qfr0PTFdwuo/D0xxeZoo34G4LayYz35380padClr1OSfUb9/ne/vTt6M1xJkfrSpqeorjbqN8D0DLdSAj2HzV1qXnw7KHdpkobpj7PIM89ciTH/66UXnw6XH/ABKpCp/heCVm/H95/n8Knm8h28zkG1C/fO+8vG9jO5B/8eqNb28jUIlzdKo4wJnH9eK6z7V4AJDjSbtj0KGJ8frJj/ClS8+H7MN+jyKSf+eLn/2enfyC3mcodQvwv/H7elB/08SdP++qP7R1Jl/4/wC9IPrcSf8AxVda174AjVgmkmRsdRay4z7/ALwc0z+0/ATyYOhFEH8QtmJb8PM9e1Lm8gt5nKNe353Br2+IbOQbiTnPX+KmLc3aoVFzdDIwcXEnT/vqutF38P8Ak/2Qw+lo/wCGP3lPN/4AUIg0cno277I2M/8AfeafN5BbzOQW+vjx9tu/p50mP/QqU3V1kg3V1t/67yf/ABVdedW8BZ40ckEfx2R+XtwN3OakOreADHk6Mu4k/wDLiwOPwai/kFjjVvbzkC5uwrdf9IkwR/31TVurthjzrrPXHnvg+/Wuv/tDwB9oXOkOwJGD9kcBfdhvGfwqSK7+He0M2nYk5z5lrIR+HzUubyC3mcat5ej5VursDrxPJ/8AFe1Buro8G5uSMYx58n/xVdil78PMHOmMfb7FIo6dc7qT7V8O8APppDeotJef/Hqd/ILeZxxu7vH/AB83ePQXEg/9mpJLm8kXYby6weSPtEhH5bq7B7n4eIrbdOkYcYAs5QT7g7/50v2v4eMyqdNlwerC3lUD/wAeo5vILHDfMAOCxpw3HsT+Fdx5nw7Jx9nOwdzFc579s0qS/DrbhrQ8+sNxn+f0o5vIXKcKSe4xmmsd24zxxuAAFwDlAPoDmuu1GHwPNCfsd1fWcpPWC1kdfxVz/IiuXuIreCYpb3f2mPs/kPEfxVun5mqTuK1hsV26DbDcTLt/55yyKB+GasDUr8McX95z6XUn/wAVVTnNT2ckEN5HJc26XECsPMiPR17inZATf2jqWdv26/wO32mT/wCKpv8AaV+P+Yheg9f+PqT/AOKrsfP+HTMrfY0w3VTbTA/5/wAOKpXtj4DuonNnqlxYTc7cwzOhb3VlP5gio5l2Ksc8ms6oiAJqmoKF4GLqQY/8eqRdd1VcY1jUsnnm6k/+KrM5BpDuPT5T+dXoTdmtD4g1uLBGs3/GODcMy/kxNdho3xC5WLWI2yelzAP/AEJf6r+VedI5B5ap4bmKJsyWVtP/AL/mKfzR1qXFMEz3CC4tb6P7bbzRyxug2yIysuf9ruG/X2qeSeONQwc5ZVYHceR05wOleW6D4p0zRphImhTxlh+9MOou4f8A4DIMdff8a6iD4h6I8iGQ30fX/WQbgPfcrHjisnBo0TOs2gjCYkLcL82cr7H19qg1e0a60q6hUHc8b4yMYZeR+oFU4PEOi3j4t9as1d+3mKpP1VgM1pR48z91iXack/e3AdOR2NTqhk3w6AuYLi+UZBijjB+uSf5Cu4mgNzazQMOJY2Tr6iuN+HBW3s9Y07YUks79gVPZGUMn6V2wOCPahgcE0lsyr5vlgbcfvHXO4fqaQKUZioZiFyuQeKl1K3jttYvFCjIdiu4buGG78OvH0qkkriQuC2RySUwP93IP8/woYFCNDb6jcW6HHmfvYiOOf/116zpt6uoabb3a/wDLVAxHoe4/PNeValF5Jgu1QqqPtPzFgQf9o123g27zDcWRP3G86Pn+Fuv/AI9n86OozqT0rjfElu0GpeYCqRSrv3N03dD/AErs6xvEdsJdN84AF4DvBJxgdDTEcZACw6lNr9Y/u/n/AEqfO1sEyktkqY1PBxgnGf0P9KhZ9zZeUMxQ7CrYyv8AgMUh27doORndkEfMB6HvjrSGTKAInC7QM/Nnq3fFRysEkIIf0+7xjrTY5ot20yeW/wB0YJA46cjr/k1IruWUBABuzu7tj3PSgCJhGpZflPc56CnbWjkB2xBW/wBrO7uR6dqmaN1yz4Ub9p4X0yfy98/rUSSSZVFByeQCuCeeh469OtDAeJMn5I9m3oE/T9KR9xfJO33A+96UkoBLBnVmzwM9/Ue/+FM4Byyh13ZYgjA9OO1AhyxITwB82Nu4nGfagt8yKixovf5cH35xj8RTljDMu4tjopznbTYnKnluuGQGP73P5moYwV5ml3QxsuBwufXFIzsSpdty9Blf8+9CRdEXJTdsA/iz/OpFQRsdgX5uchgOv0oAijEgUKqgL/E3zcn9KnLPs+aQJljksvQfj601nMiv0+XjrjH58jPFNMscm0OnC9GNMBw2NuzhwegxuGO9NLjb8gzjnCrjFP8APBbAT5echR+Xao5J3xtYkAqMDnp2/D2pNpbgk2NKZXmIucZ+8pwMdxQMjbGi7F25OOn6mlKqwYkBR6kj7x6cZ/lzQCDGTmQ7vlYYAGe4xyf1oUojsxWkVQVJCluSA+fzqJmdtuDlV3FSrA4OR26+nNTBMAZVx83R+MexH9aD87E+Zwx9P5GhgkKQXYOjSEtznbgj0xTIgVH3kYg4O8Y7dQexqVAXiGGeP5suwwpP4GnynZHtLLub+ELuz6ZApXS3BIrGI7j5jfN0Yb+uD0JqRkAJyG3cAKHzx71bg024dUdlWNu4ccj+v4Vai0qNX3ySO7+q/KP8a5qmMpQ3ZrGjJmMTI+3LFE9F5+n6+9bXh3T5vtyTSiUqvI8zHPofXqauwwxxYEUYX6dauwXMNpDJPK3Vgigclm/ugdz7VlQxjrVVCC0KnSUI3ZpyzJDEXc4A/X0rlvFYLaan2j/XTyqscfXy1HJ/4EcDJ98V0MEUspE90u1/4Is5Ef8Ai3v+Vcz4ll+0a7BbjlbeLc31b/6wFemznRz2onZFBAnfn8q87+LGpgyaTpEa4jtYWkYhgdzMcZ/IH869H2fa9dji/hQhf6n+deP/ABU2xeOb+FGJSJIkVf7vybsZ/wCBVhzRdRRe+5bvyto4KQ5lwOgpw/KhE5LE0HA781uYiyFSqnncetRzHgUYzn1okG5A34UAQkZI9KtW77SM8fhmq/Pap0AIznn0pgdPo8qPKANu773zdPwA6msvxLdHzTD5mcMTjHSrelZEZkfnauQBkZH5VzF9MZruV88MxIoKKxNJ1oopAOHWuo0GTzIHheNGVuu7P+T+NctVu2maFgQcbefvYoA6nUrqaa9htGLeTH84KqTtAHVc5/nXGOcsT0yeldJK4msr+8lYpmFIUIQNubPru46dRXNHrQAlOWm1JEu5setAG7oKEPJNk4Ra0GB4ZsDd0rt/h14R06/0GW61G385ZpdigyMmMd/lIrrn8A+GmCldJXYO8c8q9OP7/wCveqjNIlxPGwPfNLs55HWvXf8AhAfDsmR9jmiYYxtupc/q1B+HeiRDcFv1YnaCLyTcuf596r2kRcrPIguCSCCB1NLyBj/9detL8ONAztiju26YDXxUKP8AP50w/D3w8z+XEl625tq/6Y344p+0QuVnk7Dcv0o2kjHavVl+HOiFYihu3BGdxvMBvTov9aefh1oDmN1F9gEBj9sbj3+7zR7RByM8n4DY4yOmKTkE5/xr1Zfh3oTfKPtobkZN4Sc+4204/DzQUVOL9lLDk3LL+P3aXtIj5GeUew5/HpShNx/z1r1h/h54eBysN8U/hL3rAk/gPShvAOgbBIILvHQlb2QAN7ZH+eaFUQuRnkpHzc8evGaNyxg72RSfVsV63/wgHhzyyPs86uvd7ibKn8xnr0q1p2gaVpEMotQHDsGYSozjjI4Zw2O36Ue0QKDPGDJCeDNH+LrS+ZHu/wBdEGx13g/TvXvwsLOMc21ofmVtpt4x9O3B9qjWyjVyPsVuBjABgjH4kYyQeaXtSuQ8CM8Wf9fH9N604SRnBEqZP+2P8a96ayQNuS0gcbB87WyPnrzjb9aha1sIlYyWdtx8zOYVPHvx+HTpR7TyDlPChJH3ljP/AAIUheMY/eR/i617mLDTlTZJYWEg2YDeQg4xx/Xp6UwWtq+Fj06wRGO1c2yg59cEDI96PaBynhvmQg8TRc9tw/xpwkiOV86P8HFe5/YbAxSyNbwncRvzCuP/AEH096ctlYeYu6zhljGF2GGMdR8uMrn04NHtF2DlPCjJEuQZovT74x/Ok8yEDPmx4Hq4r3UabbQFSbaED7uPs0a/U9PU/r7U5bSHzFIs7ZXbJ+SGPJPqO/5+9HtQ5WeEiaP/AJ7Rn/totI00WQDLGP8AtoP8a95FtaNbgNZ2+cjaohj+XJ6fd6c037BZgOTaW8bg43CJTu5wR93pR7QXIeEebGf+W0eP98UBkJ/1qfg45/Wvc202yZR/oVmSB/HbxkfT7vJ4oGk6Ydx/s2zY9Tst4sflj/Gj2gch4coQ8h1OO+4cUgdPvCROvUMK9ufRdKV1J07Tw+G2iS1Tk+vT+XFE2mWzBJTpGnXD7grgoq4Ue+0cgYo9qh8h4juXqGQ9uCDS8npg7evNe3NoekOu/wDsrTg6KT/qEJ49flweD+dRf8I/oTIg/sawKDOwm3TC+u35fX60e1DlPFcjJ7nHTFBZTj5hn617Wvh7SWXaNI0wDAJAtFJ2+vTjmmDRdGzgaVYhWPG6zTJ/r+FHtRch4vj36elGRn7yn8a9pGg6K7/PpOnONpIzZxqfbjbQ+h6OjAHStLCEY+e0Uk/8Cx1o9qHIeLYweSBz0pT97JNe0P4e0cRFf7C04HpgW0ePpjFZmo+BNFvMCC3awmIG02vyj/gSt8v8vrTVVByHlGe/QdqUcDHBFdXqHw/1e33Gykhv0X+4fKf/AL5Y4P4GuXmt5Le4NvcRSQT/APPKZCjfkf51aaexLTQzHejv6cdcUbTk88e9IAewx2piAHjHX60p49fpSg4AJwPpU9pLBDdxS3NnHdwqfngkkZA4/wB5eQfegEVmyykEEqeNrU62+0JOEs0mErcKtuWDN7AL1r0rTtC8HavYteWdkzKnyyIZ5VaNj0BBf369OOKvDwN4aZmb7FJsUZ2x3UwO7PXqf6Vm5ouxS+EXiKaPxfd2Gp3EwmvLYjN0SrF4iCoO7nIXcOa9T1bx14X0MlL/AFyySVf+WMcnmyf98pk1zgZNgiCJMiDcfPUyueO7Nksc1JGY1GEjiVR1MUQVD0+YY68Vk3dlosz+ItN8TWdvf2MbpFJuRZZR5bNg9Cp7fr+dMRHbBLFWyR8zjp349ailJnj+cRzYHG4ZBUemf881FutIlQyYgz8qqGyD0OR2OOlAx81utxZzA7t5X3O70+nP1607wtfm3vbSVjgBvIl57Nx/PBqGNyIwYrh3XecB+TtPse3+NU4UEGqT224BZRuUg8A96APYBTZEWSNkb7rAg/Sqmk3n2/S7e5P32T5x6MOD+oq9TEeb3cDQTSxzGQtExVQigEdjtPXnr154qusM0aKQPMXGAWO4r+v6V0PiSD7PqiXKkr5yEZ7ZHB/TFYJCEHLqE3bQuTn+mPpSGJF+7TzCykgbTtx3459vrQrtGyhkkQ4ysbdOOPp7cUkYPnABowCn8YxnqOOMfnT4n8rIRGaNtpY8Y3dMnBy3btQBYDKHd3fy/lwFB646/KaWJnl8tGyCT8uSGU+5OOAfSq0iqsjSxDaCQo+bavqc/wB4Uq3jQx7pSg3cghMDn3HXmhgTeRmMhhjB2omzIUY6cdD/AI0gIVf3hbPzcdu3Ude2KaHmuFXPz52yKXJwf9nr+nFDyQkfuy7k7l2lg2w85B7ntj19qBGNL4n0KBc/2xZnPy7EYycfUD9azrjx3olu5ktTNdyLyhSAp07bmP6YryvJHAzTjzzWipojnZ3M/wASLhnYW+mRoGPBknZyP5fWqFx4/wBcmyEe2hQdBHAOPxOa5Xk9OnuafyRnHOarkihczNxvGGvseb9+uRlVz+eKltfGGtq4Q3iuueVeNce56VzwC4DdCeuTWnpeh32rtm1izGOHkP3RQ0khps1p9dv747p7+VIz/wAsom2Rgf7o/rXXeG7u9v7dkTUb35OuxlYflg4rItfh7HFtN3qbITwVii4z6ZNdXpOk2un25itUknKrtLA5JP5CvMxOGU1pNnfQxHI9Y3NBrHUmH73UvJgYdGhQH+XA9qrqtvbBbi4u2adFb5YS2wrzjIzj39jWlDpLypiWNY0xjJ5J/D/GrkWjxIiq8sjBc4C/KK85+yp6SqX9DV1JS2jYwLTz5pI5oPNx95fLBJYHpj2x3OK1Y9MldmLiND2L/MfyBrVhtIbcHyowmevzEn+dDyLAcyOEB+7upzzCcvdpr/MhUluyqmmwDmQO59zt59RirUcaRLtjjVf90YqrNqaBA0CeZkjlvlGM/nVJ727nfajMGz0hB6fjUewxNbWb08x88I7GrLPDAP3siKfTPJ/CqU+qxqpWJSW9T0H5ZzTIdOnKhpZAoUbiZME/X2/Gs0XB1B3t/D+yfB2y6jMubeL/AHF/5aN9OPerjQw8Pid7fcLmnLbQZqWu3Mcy2sMUtxezbhHZQ8FvQn+6o7s3FdX4c0SWwtIZ9RmM99s9cpADyUT29W6tVHQ9BtdKykG6S4mbM9zN80sp9z6ew4FdaBgAV6GDnGd+RWSOerdWTYtcAZRdahf3zH5XkO0n+6OP5Cuy1a4NrpVzMv3ljIX6ngfqa4eWMW2jiM8lht+uf/rV2SM4jPD4R7me7lZRxxuOPmY14R8QrqK+8barcwSeZC05Cv8A3toCk/mK9i1m8TRPD9xqRd1eGFjGmcIztgL+OcfrXz/OWkAaQ72zkk9z61lGl+9dS/Sw5T93lKpBHtxULnHXmpJDj5qgZu2eCK3MhSRx70Bxkx8YPSovxo++49c0ASKnOM1fgjWGPc33j0HrVEtuwfTg1oWpeVCi/wB3rjpTAuRXG3TLjy+G28ncciuWbqa6pUJ09lnuG8schA3yr/8AWrFW0tpbtoRKepAOOM0DM2lxU1zAbeUoTkdjjrUPWkMmQKqhmPXpToyZ7hE6AkDC1EqlmxXaeHvBksqnUb8+TbQASkk4yOtAir4jtU03S7SCHy5VmJkaQKvYYVf7wYD1454rkyOa2Nf1d9W1BpTt2R/u4yq4JUE4z61jk80DENTW52yA9cGosZ71Ztogz4PfimkB9JeDovsnhHTEJCyJbo53tgfOd2cfjW40SMzFg7IRx8uDjt749hXPaJc399p1vbW5s2hgVLbc3YgYyeeeO/HXpXTKLJYTvuGF2OsAYujN0OGzyBg+lTOLi9RRknsQFUhjYNGFiA+VdueM/wAPHHrRIzjy28uQ53AbGHXjG725+tPYKhd9rHy8Zxlg39ee1MmjikCyYVmACsG3bvr6n/PWpKE+Rs7TGXwSBGef/rj0IzTUDk5k2RqTzhSvb3709nPlooJTcQVdYtw+6f6ZqcNu8s+QU+nt6UwKiRiN4UBUhhg/N83epm89I2CiNsLu2ueCMHkgH/HmgpKLjzndSH2/u0jyOmM5/p7U2SYRMy7X3sAxAwoUD26kGgBFEiBZCI2DJlgcpwOnTP5U0klNufmbnarYzz/9elUeZ95huHG0vjvyCelOUbVJ8pj0OHIcdeNo/wAKAI9p3Nxc+SFKsh9eucZHb1p0cygFUUJ5nzKin+LuMDPbPIx6UMAI5AEk3r91hGTs68H8/X8ajVtyOxmikCL8zBTkH1/kKAHjyUIVhGXw23nbgdyFPTpSqC0wY8u/zIhB+UZyDx39aezw+blVk+QBTtyXPzZ+X8/wqPZ5kUgdQhkO8jaR68n1wM/0ouA0yJIwkJeQBwdobaN3pj+tNCyTMZpljWReFjIJ3Z57fj/nmpHAkj2BlY7Mr0+Ucjbz6DP40bYpE2zbUGA/zjLEdMqo5HXuKAGoo2EyPCcJkPjqPpwMYOf60zywXaOBSwy27jGcD14J/wA9aelunmAl5C75wFGBtxkDp796jUsibm+RnXChvmI+bufb2oAYqOQrbWPO4Bxk8H6j3o+aWTZ/qwWxuzzj1479P5UkgAm/eZLsxV1Re3qB+H401SjRcO7oV/1m4OMfX86GIcjFJEJC91G5l698dhx6D1qRH2Mrqd5f5S3mbgR6+3B7+lQ7P3xd0YcbQTJnd2+UdCB/+qpowAcPvxu4jBCkDH+7x26+nakBXJcl9wUBVHyu+3Iz1HH65/OpomVoj8iYLlSCegOOfp/h0oCy7C0O1PlHzbcZJ46evvj0oEeXKLHJI4zxgfK3faM9O9ADWjaYESYwfvnlcDjo2fqKcqMTjhgCPlxuH0PP0/OpNw3fNvVTwqdz7DB596ryy5JMZLKAQhlZuSD9M5/TrTAsGNZPuCLdtJ+U8D8eevApJWiGcOBIr55YEds8+uc/pVeF7i4dUwuW7N3PY+w5P4/pYaZEKrEHQt94AHP+c0rDI0lwjxuFHZdsmdy+pBHH4fXNDt+84EhY5yAxAx6dMdj1p7M7Ssv8LHcFIKle3Tt1pkETnYuUO0fKHBHPXB+tMQ4SsrZiZCuOd/p6e1LtdmMMfl7mbaA2V28cDcevYZpRHt27yGPzbV3dTnkfrTQvlK5GSBJ5eEfOR3zu98UARBB9nEk22TBLAHIXPPQdfzxzjrSguIwkqspX76A8g9fz71KyAKCiAEDd8w79uKYQrHghcjaXPBH496AHEiP+FmRMl9nTrx2yMDiiOPbI8JBDKq/LkgjPc8frUeI/Me52BggZtqsMEe5PUD8Kas00QLlVEakchju+p/8A1UATpGvmN5hChfl65yD79qaqoYwPJVgT8jOoAb0yOc/hSRvGy7VTynZSikR/MR36njr0phAxyqojYx5mV4H4UmMlZCXCfO5C7imB+OR26VWu7C2v7NLfUbdLiPBO2cCQH/d7g847VYAky0ezBKb9oIU/UHjAxzTgvyhPu8gEscbPrk+/UU0I4bUfh5YykSaVeS20j8+TOnmw5OcLkHcvTHO6uO1Tw7qujkve2jpCDgTp88f5jp/wLFeyvK0ikQsGIOFiLBMrjtxk/wD16nhieLzdse948FkWT759yeNo9PrVqo0LlPAGHyhlIx2PY0BDjnr9K9Y1Pwdo1+zv9k+zXZyBJZyBc98soBU+nI9a5TUvh/qlp89jJBqEJ+5tHlSfgh4P51oppkOLRgaZqV9pN0LmyunilA5xjay/3WHcV6roWuWetWy3ESR/aoU/fwuzAxjt9V/2vz5ryGe3mgmMNzFJBOOsMqFW/wC+TU1re3Njcx3VrI0E8f3ZAM49RzwR6iiUbgnY9sZQmJJEDg4LEEZB5/rR9oWNgnzlOpfrtOOmM965DSPH9tMAmrxm3n/57wKxjP1UZZfr8wrqLS7sr5jdWt1HcIy/MY5Q+3jj3HpyBisuVrc0uSxuk582J22sRtAbIJHpkdutSuJIn++GCrxwQPyoKMbhMoyj5SWb5xt7n1zwMH60BHhclm4PJzk8jkgevX8KQDZJSfvbVwxxl8rn+h96pXsckK290zR/I/RARtUn3/lVolZkIKFk9fMPy/ljn9KdLYmVJYUcOu0jYB0wO/of8M0Adf4RugRcWpPXE6fQ8H9R+tdPXmfhe/MEtpMx/wBW/lS/7rcfz2n8K9MoQ2Y/iWAzaNLIqlmg/eDB9Ov6ZriZAXieOWYndhI/Nxn268fjXprqrqysMgjBFebXVo8Es9vcxlY45SAsuHjkxjHPXp0HagRmNDqVpIwN5AY92FLR8HjH3ccH/Gr7xyPCqzOiu3zLtbA/D059aPLkMJ23Pl7m/eg7BnnGPyx8wqKItDMx83zCuOFbCr6Ac8DH1o0AmnEzBVgjeQYywjcIQPbsfzpAn7wbgUIONjHgdjn0P9elR+S8sjSrlweAd/GD7f1qXMCRK4TKsMkqwA/Dn29e/SgBZIfmz5SkZz87fN19alGwYhkjkeZztyiso284AX0qLG9mVm2OCMF+cccHr0+opBPMUZ2nbLFfvDhueo/rSA8IXH0pTyxxmphYXYb5rWcf9szU9vpWo3UqQ2+nXcsj/cjSIszfhXSZWI7G2+23cNsH2b2C7tvT/Gu/tPh5pyoGubm9nPXCssQH4cnH+FaHgv4ca5bTteajZx2r7MIsk4Lj8F3AV38fhYhmM7o+7qqDaDXPWrcmxrCF9zjLXQtC03Z5OnQGQn/WTJvce2Tmt2C3kvJFEbFFClXXYNnvzjrXUR6UYP8AVRRrnrt6/nT2tZVUlgcD1NeZVxVbaMGdEacOrMa30iNG3SOxHZYxsH1J61oLGkahY41QDoAuMU25N4iYt7KSRjnGeAPeqn2XWLkNvikReygqo9veuT2GJru83oa88IbFpriKNmV5olZRuZS4yo9cdarz6pBEPkPmt/s9Pzqunhy9ZndokWVuGfcoLfUjmpk8K3Lt++uY9pUhioJP9K3hlsFrJ3Idd9ClJql2xVcJEOh8s559M/8A6qFs7uebzRtHy7cyIfmHcZrdGkWlggkeWKPAx5kmF/8ArVkzeIdKE5t7Kd9Tux/ywsY/NP4t91fxNW41IaUafzEnF/EyaHTYo0w+XA7dv/r1WvdXtNOuRaQRyXN+V4srVcufTd2T6tVldM13UlzcOumW5/5YWsm6dvZpMYX/AICPxpf7PfRYDBp2lYjJy3lgN5h7knO4n3NckqWJm7TTfl0NVKC2M3+xrzWMSa9KogPK6basfK/7aP1kPtwta7y21jEsR2RIi4WNVxtHoFFZ8x1SQMs0EiEHIWOFtrfXvVZIbvcrPbTRtyeI/mz2GOa1hl9Sf8R2XZEuslsdHpNzDd3sqwsjrCPmZX3c/wBO9bvasnw/biGwLbWDSOSdw5OOK1q9WjRjRjyxOacnJ3Zg+JpC0NtaKeZZMn6D/wCua5vVwWlt7dM924/If1rbv2+1eIG/552yBc9gep/mK5HXNXht7XVdWimRo7S2YwyZwC23amM/7bDFW9WGyOS+LWoxRadDpccqeaZx5kakEhVBxn05IryFzxWtqySJBbl1YCQF1c5/eAcbs/xc96xXJDAdau1jNkMg5qFl3H3xVh8EVFQIrspUdaW2+abHPQ0r9BnvRb/LcJ78UAWWtPLw4PB6irunReYxjU4LY5q3Z28d6rIWYED8P5VJawJZDzGSRnzgEjhfpQUVNRXZH9nHIB3O+04b0GKyHdYWZYt6HPIPUVrateo9o0Q2KwKrx1wP61z0j7mPOaYrBI5dskk/WkFNpwpDL2lWzXmpQQgE7nGceldz4z8Rxw2CaJZMC2F88jovoPr3rjbK7XS7d5UJN5KuF/6Zqf6n+VaPhDw9L4s8SxWjsRDzNdSd1jHX8T0H1oEOvfCNza+A7PxIxcie4KMmOEjx8jfiQf0rle9fS/jOxhu/AWqWVrEvyWuYoEGCqx4YbR6fKa+asc9KSGFXLRGLg/lVaNN7gdq1okG1QOxqthH0z4Rm0628K6Ui2cj+bbLKXeTlmbljnA7/AKYqzLEftUpfyYEfGY7dcZAHH17+9Yng9g3gzRWkALG1CjIOeM9x7VuPbgn975m3je474HykdcAe1RKTk7tjSS2GrsYAGNt/3fnYcjjkAf1xSICEOZ2GeCm3LD1JPpTkgaGBA8sjK2DhuQAfTjvg0mFkAmwMKeGZSR+A6A0h2IWSY4+SKU4JTLbQo9j3PPXHrSBhuk3plnUn5Bv/AC46+3ep2ciEgqybizZZQGP4fWmk7k+VQoHdBy3dh1xz09txpgRmRInRR+7LIBkn3zt/lwPWpSnnTINm1jk9/lGP1P8Ak0kZWKARxQYQcMEXhfTbn+fvUiLM8Jh3xiNRw0SFefXHp/8AXoAYVVVDMzspAyoUHy/Qbcd8dqGcRojeYyu/KAoefx7f596hmSGAGV/LV2OGcLjPBzz3/CnK4XzHlkMQkT7x43N2HJ54Ao0Acy7kA2DIb5jgfMT047/jQyFYx8saJkHCy4BH+7t6Dn8+tFmvO9DghOp4XjHrweuMU9LdkO+MAydcleAf936H/wCtQBVkjmYkRsEBGAkSFgT3y3T196ljmEQ3rLkjgKXJyegJHegK8bA5QqAoLkgFuOw/+vUpJx5mxUAO7IYHDH0P0zQOxWj89nIWNpWjONj4H1J7gYAxwakYRwsXh2KWP3t2wgk49Cce9RxRshRfL8uPglZAPu5wMgc9R+Jo2ywwF44zsB2h923Pp7jv/wDXoEOKuS+91Xa/zhFGOcfMSfp19xVU7Lht5TdGu4D5B+Jb0x7VdKylFWONlX0PHbPT8/8APFQiNGREjI3Km1QqdeeV6H/IoAroYMlY0DkBeA24MD03E/Qdc0zYT9+ZHkP3gi7Scnv+OPzqzEnl4dsBsgqQd3T6euD6enamlXTedrMi4+YruyCff249qBDtwA7xnP8AvHPp2wOO1MhknBITbwP4gI+cdqWNXkZXdLdUReJDyxb/AGv8+lPeGMj55I2CKNwfk8d+mMfr60DGNmeQp5BL4wcE8HGSe3QetRqGVSQgWQH7sbHC/X/9fep5YYCyyq5G2Papwemc/dHv69RTI/NKmNJHwW6FVG0d1oAYPLK8qsRdgd/PzDptz3zz/wDXpzRB2XZ5qqODIvylc9F/L6cmpGXIX99lW+b5hwOMj+dV5YQ8QkkVGGzzQmOF/L/CgCRF8tQjM0iLliS3BPX+X06ULMVt1wquW+UAEEtyO2fVuD6U7ds2OkMoTOcCL+LHt/XrSwuIgPMV03ZYBo9pOPY9gfp/KgQCOIOofAUDa2Oh9s/n+fvUBdSjbZB5gAY70DHb/ex9OP8A9dTrEh8xHPyv87JG7Zz1yO1RMBIoXzGcIP4DjIxxnHf6UANyzQAwNjGCIwNpHTaQPeplmcqsfmR4xhcFj8p7Ar179KYY0eKKN1B2tgHc2V9x/jU0ihZDk73X5iMZLc9P5c96AGt5m4lEyz8YVvujqMe3+e1RhUdNqTYVkGSBznP6elKbdRjPlj3yT/Loeo/KnqqRzRSNJuHV0OBt6/07e9A7FclvMUJ5koOd0j7QeB3/AMKcSoU/u3IKn74K9O/4/Snptkct3Rc4zu55P3eKQyEY8tYmPA2uxU+44znp9MUAHmfuwkYODhVUYALHtnHoP8KRid/nI251GPN5UntjNWEP2dcq21lXBBbG3J6qMDPpxVQzhkwr7tse1SX79/09PSiwD23jbEgt8KSxwdx3fpz7e9I6IJGlRG42tkt0PTucDvxz+lOlmd1JCYEnyYV9ydstj8uf5UjK6tsEqhgQMKqnBP1FADXmii2hYw2c9CpGRyApP5/SpQzsm9ZmAUH7oBZunHPGO31phaQod4EkTHbyOTjvj/EU7Akd3uCzM3Vlbbnj7vHOc9Pp0oAGLNxkO24s7uM5+tVpHWGVWkkiAAbyyWC7x3+Ujjr29am2nO4Db0ITcAB9ev58D2qdQwJjYo4z8saDIHPXqQeuKBGffWttqcIguIo75NzZtZkUqvurEZH/AAE1y2p/D60kcnTbiS0fn93ITNHj/wBCX82rsiiC4IL7LhRs7AgdQp446fjiphGCGGyONQNwH3lYfjj8/amm0Frnj2o+FtY0xPMlsXnh258+1zKg+uBuH4ism3unhlE9vM8Tj5fNifaw9tyn9K90Ijgk822CptyQA2wL7nGO/oevrWRqXh/R9cbzLqwzOcDzbc7JTn1YDB9OQfwrRT7kuJxFj451q3cCd4rxOjNKNsnth1+ndTmuq0/xto998ty72ErNuKXAzHnPaQdB9cdq57UPh9eRLJNp1zHc26qTi4/dsT2VSOGP1C1y95ZXWnSIl7aXFq7cr5qbQ30bofzp2jIWqPaZZnmt1uIEFzGflWVGG3HTryPX86coKQlfKkbdtQofn2+jEjgfqfpXidneXOnzebZzzW7/APTF9oP1HQ/iK67TviJdxhU1O2julHPnw7Y5B6fL91v/AB2pcH0GpHZ2mIdUlgf/AFdwm5ef8/5Feo6RdG80yCVj+8xtfP8AeHBrxY+JNN1KaCezmPnw4aWKSMxvjucHgjnGQa9P8K3gaSaDJxKomT/0Fv6VGzL3R1Fcd4otCl/HcYLJImCue4PP6EV2NY/iO2e40eUx48yLEg+XPA6/pmmI4XyfmPyK0YXOQ/Q59OgFCTNIoVTMGTH3Y25bPAPBH49RUoJZkTZ+6RjJ5mMhx39/bpUEjIsrgKV+bOYwHz3I4+lJgTskciBXL5PygjjB9+BTkjfzt4LsCNv3uPTjOfTj6VF50hRFVnB3HarOMZ9DilhWNCF2GMbSFSQ7gqkZ5/2sE/SgBGBRGWXzNu7C7hgM2OME/Xp9aSCVHaPy3L8HYMFSQQMn/wCt2xxSEuW3o5KFTkEbnOByBjgDNTHM0bBYiT93kYz1529xjP50AcRpfh291u6Frpz3CNjLOJPkjHq24GuqHw58Y6KTPoPiu3klK4Zbi28o/Tcu79VrvfDPh9PD2mi38wSzO2+ZwuAT6D2FbLMFGTWkp9hJWPEb7U/i/pOftFpcTp/z0s4orgfp83/jtY7fEzxZaybL28mtn7i4tBEf/HlFfQwy3bbSNGroVcblbqG5BqLp7oq54LD8SfEEuMawG5/gSM/0q2vjzxE3TUZSPQxR/wDxNeuXHhjQbos0+iadIzdWa1Qk/jiqh8CeFj/zALAf7sQFF12Eeajx74hRQW1A+5MSf4VVn+J+tL8q6kmfaNM/livVU8D+GUII0OxJH96IN/OtO00nTrEg2lha25x1ihVf5CndAeNQa78Qtax9ih1VweAywiJP++mC/pWza+EfiDqij+0dcFjHnlTcvK/5Lhf1r1nFFHN2A4Gw+FWkxsJNXvb3VpAckTSbI/8Avlf6mu1stPs9OtxBZW0VvEP4IkCj9Ks4pAT3FK7e4C4oxR1paVgExRS0GmBn3epiymWOS0vHQjPmxQ71H128/pT4tUspkLR3CEjkqThh+B5q5ikKKxBZQcdMigDzyx0HU9XvpLm4aW2jmdpizDBOew/CuosvC2k2cQj+yRzHIJaZQ3I6cHitymgYzyTk55pWHc+dfjhcpN4vtoVj2G2sUUjA43Mx7fhXk+7LFvwr0P4u3i3Pj7ViDkRGOEenyov9WNedL8qc0yXuMbim9h609huOasJYzSR+Yqkx+tAFNosge4/Kkji2zp9a3YdGne4ELqEZgdu498cVGNLli3mWMqy880AJpcpW8TOCgOPYVo6i6rHKyszIV3LuOQvuPX/69Y9qds2cLkc89Kn1B/3YC568knrTAxLxvMl8w9cY+lUe9XrlR8p65NUu9IAxS/dpQMDJpOSaBjslm5P516d8M/HegeFIZLe/0KSeWY/vLqK4+Yr6bDgfrXl5PGKVHKNkUBsfaGhXfhbxZp7z6Y0c8ZGyRCzK8eR0ZSeP5V8ha/pbaN4h1HTGBzaXMkPPXCsQP0xW34L8WXfhvXLfULWQ+ZGcPH2lT+JG+v6HBqx8UpbK98d3OqabIslnqUUV3HjqCyAMD6EMrZFA2cpBGAue9aUK521RiyQvrWpAAAh96BLc+ovh/pVlP8OtEJi3eZZxltzHBPPXn1ro10HTldn+zDexyx3Hr+dZPw3R4vhzoKPkN9jQ8+/SuppDe5lrodoGGFYIO2453euaeNHtQSArkE7mzIeTWlSEgDJosIyzoOnZz5HPb5iTSvoGnyJtkhLgf3mzWiOW3flSgqTxzQBlR+HtOWJUeEvtJIJOMflStoOnKijy32JyF8w4Fa1McbhtGOvNMDOXRbVHQqJCoHIMh+tIdDs2PKNgjBBc1p7hkDuadSsBlrotooA/eDHON/8An1pRo9kAVKswPGC+a06i8tVYsirlmyxx1oAzV0LT5QreVIACSBvYc+uKG8Pae23MJO37oJzt57Z6Vqjpzj8KU0AYjaBprPK7QyIc8vvKjGO3tipG0GyYbh5hJXH3+K1guM+tIBsXnn3oAxW8NacVG7zyF5x5h9fSoW8L2rAGG5uUUg/u22kfyz+tdF1HFIPrmkBz7+FrWW38tpplyOQCMdvb2pq+FIQpxcvuJzkRrXR4pPx4osBzp8KQFiftMpO3au5F+X8Rg1E/hBHkLC6Cnr8sXv8AXiulOd4xjA60pGRwce9IZzEXhbyyCL4eYu0lfLyFI6d/anjwrETxcqTnJ3Q549OtdJsXJJUZIwTikwp6HnpTA5w+FTsdTeqwY5IMH/2XFQN4OUDKTxM4zjfFwD+prp2RZhtlRSvIw3enkME+QBj6MaAOUXwtducvcQruPzrGDt9iPeg+FbuNh5N1EO/I2jOPQCuoi8wgmUYJ7A5Ap7AsuASue/Q0WA5NPC95GuPNhILBiCzNz+I/pSjw3eghklt9vHy7m5Hpuxn8vQV1pFB4HpQBx7+FL0wNGk8Y3KNwSVl3N37HHbmmR+FtQjhUNPbM57ktke3A5x68GuxAI6nJpsjbSv8AdJ5OelCA4+PwpeKvmBbXzfu8SMMr0wTigeFL3zwxeyAwVCqCCv0ODXYo6Om5XVh6g07YpwSMn1oC5xU3hTVHjwlxZAsNpDbsEY9hz+I9ajk8MaikqtHbWcg+UEiTB4+q/mK7CG1t7I4ggjiRidxUY5NWsUWYHCXPh6/bl7beC3JVlJU9m7D8eKP+Ec1JY4gsKM2fnd5s7s45/wA/hXdFVIxigqDQFzg28PanJPskt42jYYbM2C3PXOc+n5dKUeGr9FG20GduM+apIP8AnFd3tAIpcUAcWPDupXEDBtkUhHzEnBP028H8aik8L6mSdqQAbuAkpGBXcBeoowRgdfekBxP9gaqyMEjSN1wMtMu1l/2cDI/GqbeGdWBOLVXPOG85PmB654HtXoR7cVDLIykJGu6Ru2cAD1NMDgV8P6mNqfZpYfVIyhwOo/vZGf7351ZXw9qjxN/oqIWY7izLuxjqOwrtki2ZLHLNyxx/nipFVQSwHJ6mlcDg5dIvICDLbIqY6CdQAR65PpUL2Fy7RpDZjPO4QMvzgc9m6/5FehBACcd6cQCOeaLsDzlNL1RZ1k+wzYYbdwGHUfy/EU19HuJYp4G0+f7OzYeOS33Kx+hHOeK9H2gDFQRAyzys+CEbanH+yM/rTTDQ8V1b4d2ksRe00+806Q5yYYmaL/v2Rx/wEgVy2pfDzxNp4EqabcXsAGd9rC27Huh+YfhmvprbnnmmhApyB170/aSQrI+TUS80fVI/OsrqKZMq0MkDI5BGDwQK9J8JeMEtjYNcxyJsmC5K7dkbfKd2frn/AIDXsl3Y2moQGC8toriFuscqB1/I1xmp/C/Spy0mlyy6dIeqD97Ef+Ascj/gJFNyvuCVjvRTWwVINZtnDqlvpUFu72r3EcSoZSWIYgYzjr+tV7fTNYF+Li91tZoe9rFaLHGR9SWbPvmkBzb6fMl3LAodnTKKREcNz67fTHHSqtvDnc6QzqVO4Hy26dQM9T9P516SKWgDy6ZUkEkT3BLAB1wjZZs+33f/AK1JFHFEZi0pZXOTs/ebf/HhtOeccj3r0qRHD+bF1H3k/vf/AF6hksLG9TMtrFIGOTuTn8aBnna79oP2lY35OZF646Yx0HapI4m3OxuguXKJmQncfcH0Pc16HHYW0PEcKIMYGFHAp4tYkJfYDIerkDd+dIRPUD2qvcpOXfcmdq5+X8qzF8RWqLunEik9FVd3t1HvxTB4otFVnmikRBj5uD+H1p6DsbhB7Gg5ArFh8TWkqsSkq8nbgbsinv4l05FBLyZx93Yc0BY2Acil4rD/AOEjtWlIEUpRf49vf86WXxLaRxuwinYr2C/1ouI2jTFkRyQrAkcEA9KyU8R2ZX5t27G75cY/DOM0jeJdNUqzGUbjt3GM0DsbVIqhVwP1Oazotd0+aPzElfb6mNv8KmGqWhcIJhuPbFAF2iqxvIFQs0sYx1G8cU2LUIJSgWRSXGVAOc0XEWGQE7u4HBpkbrvKFh5g6jPP1qTdn7uDVO6ljtZklwu9/k2heW/HsB78UDLtLxVe3nklQtJEsYzxh92aBN5km2M5UfebGR9KYixRTQwPQikLdgQTjpQA+o5pUhheVzhEUsx9AOtNklZGRRGWLdfmA21n6lO7adfq8kIj+zydGyy/Kc5oCx8j67qEmr6xc3chzJczNMxx3ZiazXjGat2cDzlGVWYlQB+VbS6IgZWlO3nOSfb+VUIxLPTvtA5ZlA64Xr7UyfW3sJXt7VUWNWPbdmrOt6qlqotbNAvybSe9csQzHJGTSAvyazcSSB97Ajpz0qe11e7lmjiMzEMdmGx0P1rKWMsTwR+FWLOLbdQluBvHbNIDfltvJm27WXvg1XvUIbLEHbwcVs6mJIjE7L5i49cZrMmPm7cADI6HnFAzDuxt2g5z1qgCM8itLVPlmK9BtHSsygBxbNJSUooATFKKKKYEiMVYEVYmkaYRFudnyj6VUBrb0TQb7WrPVbi0jBi020N3MzdlDAYHvyT/AMBNAivChDj3rSiQ7F5/i5qhDgxqPStCJ8RYz8p5pDR9V/DW4+0/DjQZSf8Al0VP++cr/SurDAjI6V5b8K/E2nWnw8toLu6Akt5poxGFJO3eSOB/vCtm58fmRc2Fl8ufvzn/ANlH+NCQ2dzmkJGOa80l8b6tuU+fCgJ4Cw5z7d6h/wCEy1m6VN0qouQSnlbCRnv3FPlYj1DBcDqKcBiqmn3a3losq288AP8ABMm01bpABpabt9zSAkPtzmgBSuWDdxTqKQnjjrQA1kDkE54OetOFNBbHYn2prS7Ccq5+i5oAeRznvSg8U1WLDO0r7GkO1n/2gKQDxnHNBoHSopHlVlCRhhgksWwBTAcFK5x3PGaSKTzVJCOuDj51xmlL4KhuCfyz6UpUc44PqKQDulFReYVzvA2/3qeNp5GD70ACnIp3SkzzSAnOMUDHHFMk+7x97sfSkPmAnGCvp3qIOpuFUyru25EZXB+tAE4+YAkfnSk4FAFRSN8wVWG/rgnGaAGx3UbsoAky/I3IQPp7VK5IUsBkgUwmPeiuU3H7oP8ASnMyoMcZ7D1pAJHJvhRmRk3DO1uopxZQm4nA96ATu5xtpSQD1GTQAobIzUXLyDpsX9TUo5FRsEVSzNtHUndigB+1cfdHNKD2xxUEiOwysjDBBGGx/Sj5oSW3yyBnA2lc7c/TtTAkkBePbgc9c+lPbPbFQ78zANGwIHDZGD+tS5bHQD8aAHdaaTjHWk2tnlvoAKNuB0+vNIBJV3oVBK5H3h2poR9+WYFRjGOD75oi85k/fKitnojEjH5CpePWgBGbaMkH8qjhuIpw3lSK+07Tg9DU1RSjCMy4344NMBzyLGu5jgVHEjF2mcYZuAP7q+lOUpKEbO7byMipFZWHFAhaaw44pkjAKCTxuAqQe9IBpYj7w/EUCQH1H1px4HSjFFh3DqKgO+EDClkJx7r7+4qwKOtFhDVIKgqcinGoVt0jZ2jGwucsV7n1x0pS+DtkHB43DpTAlooAooAKKT1OOaNx9DTAUEHpS1G0qJjcwXJwM8U8HJ9qQAQSOKieLLb1+Vx39frU1JTAhSQMTHIu18dOx+hqUAAcdKa4UKSRkDnFR4mLoVKhMfMrDJz9aWoHnZnNsku9bp5W3B9qL3Pc54/CnRwvJKpQIoXkhX5i79uvH88UhlJ8pJEO1XwkoHGB7ex7mp3uDHuAlljDcIJCGKf3TkDnDbj0qUiyKTcQ8KFZXVsYX5QARj884596lEhkUyvIUZD0RgVHbg4yabEbgZaeZpp2P718jIOOuFxgj0xSzxhmKNjaifIw/jbjjjPPT+tAitM8scwSJJAkZOJSpJOe2cZxViSQSxRB3V3YcbuSuD+WPp6VFKhE4CYVl++5iLHHfvgfp1qQmNpGe3lT7OMxkNIGPPUY6f196LgDRLKpklEckZ5ELdWHQkj2x3pXaPBFvZhVxkEjke2MmkjJQb0QyyDo5bAVfXb2q1tkEYRwG3AgpG4x9eOfzoDURPMWBGSKQO33vnCqfU9c+lIPmZfkbkEtkdB6+4qHIRhEf3ip/dTaF56Ak8/WrQkV3y+WjP8AyzYn9cUhj1McaqsaLHIq5JRNqN9aRLpGSTbIMHDNs6Y9BioZ1jXAby2fPyyKpzn2OP0prMwWTafLz1288D8aBFn7dPE7CJ3SM8MQx6545zUj3V0/ztO5dAFDqSfw46k1lyyGBsu2TuK98Dj2/n71O2ws7Y2qAVKDnH5cE59aAHb55JNrCViw3EuMgVILq4k3BCxC/eZGwOnpUWUfzBmZAx4yCACB3poDCIhIwGX5owG3b/U47fl9aYakgkkj8sRuUyNzMRknu3ANJ58wl2RykSsOpbaMnsaZJcZtxKnmIqYf7/yZI2+nemyugxMxSPLY2ou4ZxwxP50ATPLuwhmkzgNkt0z0BPr1rF8Y3Mtp4P1nbtR1tXTC98/L3+tbMgCs4hG/5tzSKACenU/0rI8S2s+qeF9R062jdpJICoQryxX5jg/8BoQHiNm4t2QIVG3ozHrXQgxzWWV/ec/Mc/pXGmYmTfjG7nBrY0+9mit1bzBlgQq5xgVqSUL3w/dhzKYG6nPtzVQaJd43GB15xyvU10p1e7YiI3L47BUXAq4mqeTAC4aQ7vnyTzSA5P8A4Ry/wcwv8vUVoQ+FLhEErMCcjowrfk1h1ZF/gYDk8E/596yLvWJHYhWRU3Z5AJ9etFhEOuKyhwm0NGdr7D0rDjkxw3Bq/ey+fJI7gs0mGzgelZ1spErqyAnoQe1IZR1kk3nbhQODWb2rS1kKLpdowNvas0DJoASloxzSUAKKWkFWILeS5lWONdzn9KYhkMLzSLHGhd2OAo6mvSPD+qX2haE+k2k6RRXDF7kCJT5xxjDMeqgdFrC02xhsYmON0zcFz+oFW1kIfqApGMHnP4U0guc5GhgkkiI+4xT8qtI2Qq9Kj1YGPUmYjHmKG6Y56HipLYb5Ae4A4FTYZ0Gg6hcw2UsMM8yIsm4qjY5x1P5VqHV78yIz3cjMmSpc5HT0x/Oua0391NMm773zcjPf/wCvWpwOVZix9ewq0Jmuut6kilPtch2LjO1S348Ura3qAyDcyD6YX+nFZBPC72JbtluB7f59acJOQysT3xupiOy0v4ieItJKp9sE0KceRcRhgPx4P613vh/x9feIYbg+VBbSRc4T5hj8efSvECzbs7cKPm+Y8Vo6Zrd5pFw8tsyKGG1lcbg3/wBepa7Due8/2zfMGRpED/whOpX+97UDX7tQrO8QUnbwmTznHH4V5xpvjuwnxHf2ksD7dzyxsZMt9OCOcetdBp2safqEaw2uqQ7sbggfDdzgZA55/Ws2pIu6OnfxDd4O7YiDCllXOCRwev6CmxeIrt2jC4fcBy64DcdP1H5VlPEiKrscFhlUduB67hjIHHT86kaLZGJke3+TGeGX1HH0pAao8RXOWbYhHBVQv3hnqDxgfWlfXL4jcv2cL3bkjHqB/nFYhMcPlrNP5PmHjzAWz6jdwAaHCxnCybiYiDhjjPbGM4z+PFAWNVPE1zKszCIxKhwPNgIZj14H+eop0Ou3R2kzI69TkDjnnn6c/wCNY88jKyZiPzfOO/bkY7d/50l2iTSCWFNzn7rRqUO3svtj+nvQBsHxPdGQJFHGwP8AEw4/PvTP+Eovdjs8MKbFLcZO78O1YskcrGTyY1lIYbyhyOM5JYdhxx/OnKXknxK37xv3uxCcAAnHPfPp3ouI2P8AhJLlCCtvDukx86j73PQ89aUeINSEqo6Ip3bSojPX0POfyrBRm3sFTZGCuyHbuKsT93PqAc49qmhzCoMmSQdpJ+bK/wBPz+tAG0PFFw8uHt41i4IKybWbt/Fx1x61YHiWQMoe1AJ/h3cjrwPXpXMXDSR8QrJK/OSQMqvbp1p0NwzKTKXfcQFRsDB5HO0cg+nFMDoG8WMsiK1ntBOPncjP5D+dT/8ACT5mQLAuxxnl+Rzj/CuZZ5kiKdA+Ds3feU9Pof8AH2xUTIsQMit1OcFOAQQTk5x+VAzqh4nO9AbYZbjG8g5zjGCOtJ/wlFsZUL27lgPkZWyOfc4xXPW0h8xT5zTHdwSpzj2/w9MVEsuyPCMVC4jwuQXPoF7Yz7d8UrsDrJvEgjLBLWQkLncWGM+nHP40z/hKBuJNqQgH3vMHP/6q5SeJfsymMbFVdrIMctnn3A7dPehCkmXjd51UYwow2Mf59aNQsdKfEsCt54t0V2OxiX5wOg9f8jrQ3ieKSMM1sxG4FYw/zev0rnh5rJGZojtweCwODk9sccjjv6VCqyrvHmNEoz8hfBPtnn88/rQhHUN4sUzxrHaNgnbJufBXjjHGDVh/EdrJbnYsqy7eEkXYQc46/wCFciyMIldSRJJtkVfKG4enP4mmksVR4IyF5VipJ3E/eYEDt7dM4pjOti163cGS6nkSJlGFERG0+5HrViXV9KEaCaVmVDjDqfTqc9a5FSYWK5YMQWJYBR15+b0H1/LmiaRRF5jykyMu4rGOQB9M56ikFjsW1+yxiJnlbj5VXn9alfW9Oi5ku41/2SDn8q4WWTzLh0D4XdzsYAFQv0BJ5PNO3SSsnyfK2NoYlufQcjGAO1FxHaLremyOp+0DIPRgRj3pya1p8yhluguDxnjNcI5EDsVSNJOocfdOB3Hrx2pzK8LAybldifmCjqT8pA/HuPrTuOyO9i1azkYgXUB2rlsSdKlOo2XDC8twP+ui8/rXnLCUyRI0xXZl2Sbn5u3b681ZlkRDI8WxPKTcyjGHb2PbHpQI7iPWLCWXykuUY/pU1zewW4+d1JPbcB/M15+VXMUquzONoO5Fz0PXnnPoPX61HKIXGJFQxAt821sjnjoTz9M/nSuB6JHdW0j7FuYmfj5VkBx6VOCDg59uteafO0TvGiTZcKrCINux79eOOo/Om3M0qoshDFXQFsDD7s84UHHU9PrjpTA9O6jj8KhiljMroGPmD7wYn+RrzqC+uIBICj/MeQSOp9cH0/h+lSbpHk8xA7eWSCAhbI7+/wBf6UAehyPFu2O6gnnBIqCe+hszCksgLzPtiUdW/wA+tcJFJDBNmEGRQeGb5Djpz7j8veoGc3MjrBJLMkWd0bsWzz2xnB/w7UMD0kTHKgxONx9M4+tSbjj7tecxXd8I9v2maEqx+Vtw3Z9centjFRR3V2ZlmjuXk3fcLbyR0+6e/agLHoxnIfHkybcZ3nAA/M08u2wHy2z/AHeMivP5b6a4jCTTugcbR+8Zlz7gn+meaT+1XtZW8q5nVvu7fNJyPXnP6UXCx6EWIIwpOeuKaJlLbGBVvQiuHfVru9Epja6HzYRC5UgDr+I/rSPrmoAZaZ0VWAX5ie3GT3/z1phY7dn+YohAcjIOe9ShgR1rhjrGoI6n7RJIGJBChlz+ff8Awpq6nqEcwK3cqowJkHBAPIzkk+nOKAsdxDN50QkVXXP8LrgipCfauEOr3yMdl3JKwPzFATtA9vw9KfNrV7cs8MlwyBV6LHs8w455z93vigDt8qwzwRUUMol35jkjKtjDjGfce1cQ+p3a2628c0u4IXVYxs467iV6D2/xpV1bUmRGNxckY2pjbhj7HnP4/lQB3TbsjaeO49adk46159c6xc3djbeZNuMY3sOrs3PTA/8A1VZttYvorWQR3Ej8cbhnnIJxnpgMKQHbbgX27hkDp3pqsrOwH3xgH0riV1rU1mJeYiRFIJMakf7p4/8Ar0Lqt6Zkl+2zFojudSQox2yO45zTCxmrLshYMoZ3IKknoBkcMD+B9amElw8R3L5ZYhNiMWBx/eOev0Feb6N4nv8Aw7cGy1VJxGvy7JfvR+4PQj/Gu+hnttSshNaTQtC6+g6cDjP+RSasPcuGaZXHls23kRnG4SNkY9McfypzRXExCzM4RsjC/Ienp0z7+/WmkqPv+ZsIx979D68inStH5ol4UqNzAnOW74P5VIDQ8ETtb+SrAYHBOOnIPHpxSGDlSoV1VivIxtweO3v39Kky5VkMxY8ts3LsUHuP6U2aYP8APL1kOFUH7p/3e/SgBFeeV2haQghuiAHJ69f84qaOHKlzsRl4+9z19fxqvE2cqVO9l+UAgdufxpPMkkHRNhfcFJ5x2Jz0/GmBNcSC3faUD7hkH+9no2c80KY5SQ3mqsf+szhR9PqagadlUHchXkKV57fxdv8AD3qNkJjxKiyBQcLnIH0IpAWWlWNGCgvJkFHUfzJpwnJVW84/N8rjYSB61FI7gBBuZ256DK0gjkgDwyuxZvm4k3dPp/jQBPN8zShsKPlbeVx+vamO+fktuYzkhV6fp3+ppJnnlCSrGzBlKkvx+HrSAKSp2MVXgHaOP7ufTmgBsTRxtkRlnclc7uQ2TwO2PapUmhZkcocKWV+CwP8A9ao43LK7KQqhtqsD0PoP896kZkaBmHkh1xhtmCf9mgbGGQqN6xNtClfnU/NT0liE7Oy5kwrNvII57n29qZ9oj2/PH5mDlAD94+h9v1pyzBkVw+95FYhlVWKgYzwM/wCfpQIfdSBsAbNnI4J+fjOSeuafJsgjQJyJBkZO7aM8jmmAxSbZHTEbptLBjn6c+9JJJBEwG3zFcFcDp9O44ouB8661b/YdfvrTaoEV1IqgdAu47fwxiq0kxhkwG+7xkd67b4l6I9p4ih1DZiG8jUsQvCyqMEHtyFB/OuDmRmYn3rREFmHUPLIfj29am+3mQ7m59s1jMSDTWkwBTA3Jr0SLvLZf+76Vn/aWkc/l1qiZiPX86BJkf4Urhc1bmQrKu4c+Wuee+KbaeW10CVOPrjiqQk3xRc/dXbVyxwJkL8gHp60DKGvxmO9CntkdMVVtbfdC7/5FanimIjVlzx5g3Y9M+3ao5Y/s9isUONzdeKAMRgAx5zTcVZaDyhlwN3oDURHPApiFij8yRUHJJrqLG2S2iwqgY6kHlveuWSZ4n3I2D61cXWbxUC7kIHqgpgdQHYN8hbnnJFC7wd2D6nPSuY/tu+wR5o59FAqMaldeZ5hlYt7nNO4GzrSBoo5AMENj6A1PZ27NbNc4+QtsX3wBmskX0l3lGYkZBwfWt2W+8pYIUVUSKFEGP73JY/mTUsCO1Ypd5DFdyleDir/DLyQR/n8ayUmH2iNxyNw4rUIcuAFOQcdaqImTCQK2MdD0PQ8dPr/jT0ZgxfcVIbALeuPTvWjoOhXOs3BihG22jYGWTOFHsPU/416E/gzQLmLclltkKFtsMrDaP9o9PQf40OVgtc8sXeSowVPrmmAYbgg54bnvXpEvw/04swjkvCFON4mHHXtgnt09CKbJ8PNOaNfJv75jIfkUMrf8C5Xkcc896XOh2PPSxGOQx9B6VPb2t7fOwtLeWVl+YmMfd/HoPzrpJ/h3qCSsIL+xeMfKC+4OPwUY/Wu50bS7Dw3pjR2yTPJnfPLJnII6beg7nA/+vQ5roCRm+GdM1i2ti2qXLtE6qBAxEgGDn5nx69uea2ZoVubhpo3kTbH5XK5ySefbnrU0U7NIqJDukOSMLyCR0GPxp8Eu6Qhi8kbYd1crkKG/lx29PSs73LHpHGyYkl3boy+6VQc8ct+Q+nNQxTJIytIzzIrbtsan2J3H1x6H+dTCLNt5TzlwzkykS5O3dnd7fTtio45UUiFf3ke4hHJ3DvzuX+gpAPJbdJCylJVQLujOAoI4b3bnGPTNQB8zu4kWV96Kxh2LjjqfT7v9KmBcXiBGAITaqk4AXP3fTGKpzKYpIky8w3sdkVuECkkbdxI984x6UATvEkR6p+8+Zv3QXA28ZB4z/tc1Wyx8x45MEpgRqA2Onyk59h+lTPsguWWUyPtyqt5eS5/3c9PX1qGOOBZREIy8gXAMgA+Ufy570MRJFLJKoiZQZFcgvyPMPORn15/GmSbEjlFy6RqH3Rq0nTGMd+3P5YqbYYZt4Q+WFO4blI3Z6YHv2FQ/Z45ZYjmNlAO5NqqCfbHbH49uxNAyWCSUTosKOpV96OMc54AK9CucYyw5xQAGlVfvjqpYNjOP4QCDyAO+KeuYmLGTCY3ZD56cc8f7o79qdvja1eScybdgzuYKDjcOpPv0PpQITyfIaSHG1FbPmBOCvYgY4OR/L8Yz5kciHahYL/rMfMx9O2Tk9s4+lSRuiyErOp39NgHJxtH3s5/+v60x5T9nRcZH+zHgEjr+HYkf0oQEUULx7hvdlH3pHOAp6EHsORjP19qSJSsZRdzK4wQ6D5OMgkgjj/69PjnVy8hT96uADH8o5/vY+oHv1NNCwsCrLA/XrITuzzkDjPfqO/0oGPji82FvOjbcD8gwAGLdeR07f1qOCCWeQMsw4GTGBjbjtnPOMD+VPVWeFTbAvsG35WAIyOF249xyPekCkOszSTPGgYEFQyqeP8596YiR41lgkjlcCMYCkrtLYPB+U598Zxz2pjOtmpl8wh/vl4045P3gR0/x9aZJJHFvUiJSyeWWkPBJXJHXk5JH5VJv8pGffIJmXAKqQhbpyv3fujp160gHyQLKx3kkNuZkMm1t3Htz2/zmo1xLbjcXZ0IKys4DYAwox6np9OetReaVt0zhVKk4UYVTnr9f8BTljW3uI2YCQYzjdkqucqfT8+4oAdJGAsh8oljyiKclTj06/wD1qVV3o/lRMCG/1bYOCAAWA9fw6GmOR5KQgLH91BvG/PU7uBxnj9aihu0Yx75AZNuUfygCrY4zkd84oHYlMkcsjEIsyt8qv5ZO4Dvkc49v8KJJUd0/f4ZjlBtOB7jJ5OBmniTyhtaRo8NuU4xuYdMenUfnT/MVUk2SSAScMT05PIDAgHp+dAhhhMlwG8kurfKh3hNx/X29KW4T7ZIflVm2qqfN86/U49Sev51Wg8q4xL5q/cUYJUpJjPU/5PTilk2OZCfMijdVJKkKG9WH0J6j9aAuSk7YtyhBuAf5crtz0PsevJP1oVUQqXIRlO07WyF+XgMfXH4UKAFDQrIyx5Q9JEyQMnOM5x1qEsshiuX3TSAnqWGzHQ8YHr04xmnoBI6GJQyIVb5juKZPTgkHIBPP4UjzpbxPK8ToijyySVKMW6Yx90ZH88jtSzskk7srROWIyN+HDY4zgHrg/nnNM85DMUWWMfOBtLAcDg9fccD60ANhYSyLEF/elcMmBISO3bFWiESKNZWcwPh0CvtLLnAAx26/h6UxElMbOithFJMKc7T03AYPQYA7+tPW48mMOythcliI2x79Ocjnr70AU5re3FwzKnnXJOQ/l7iWI6k9+g59+vFPe3PmJcI5hK5wzrkOBjI5Od3/AHzUhmQW8ZTDlh2HzNyTkFRgYHY9DzTY5oPkhiWcKyhtgAJK44wuevrjPFFwLHkTQ7iIlIHBVmJH3uhP4Hqf8aie2xukfzC8Zw8iHZnJwOmfptol/wBWHzGEkPpzjd83AGTk859qrtdQtcYK+WqblYsCqE45JbaM9RnPrigCWZTFFiQSPLu2onQ/TGaYxRw6MJY5pCp3726HdxnO7mpIcsIgpRhnBYTDnrj68Z98dKUyx3LpGeQ6EBJ3YgH+LJJOe/p+FAEbnBAEqxxlNu0Ab+cZz/dyB1/Op38tkDyqh2KFXnKeu3gDGenX1NKyCGFXzOrbM/PGvHygY9ucjLdR61VhQIZpY3cQomJNpVWC/dIPTHPbnPShjJJNoTeGdvukSO+3jjI9zzg/T3qONpywV/siorAFNjASf8B3c1ZVRLCP325W+4SRmTn7ysORn3xgcUu5Ed5WkdY5G2PG53EL26+mP5HvQhMlVRGzed8oMgcsELcNkrxwPp6VAE/eFFSMozBYxHjL/Tn26808BNhDpIPMfdhHIG/A6/j/AC4xSNGZZNsKeZyq7U+ZCO4yfx7e9MCOdEclkeJw/Ow9DnnHXqOlPM0hZIjzJtGxUYK349e3X6VLI370yfKkTZ8vaDnAHPz+ox+vaqhkhuFEKrIHAON4OCd38OQD/n3oAc5gKEqoDLJu+QtmNfTJPuf8inSRmRY1GZJDhQ6ruPfPPbtwenrSoqMqRllCNv8AnQMGI6fe5wPfpQ8iDBD7CCyk28eFQZ9/vCgBibJJDIsbR7wU8pEYPu49+Rx/OkaeaNHh8xDGCcOkeT153Me/GMeoxTysJUfuZJGVfmikBbLBT/jnt2yKfEHDxBZFQ7QEkCqm0HBJLemR7daAI1uXQsXieRNisWOZNvPy8jpn9M0sZElutybiIxMRuLEqqhs8YI75P40qfeYrhJcllk4Rsjn6fNz+XNMR3aGUbBuX51kYH1zgjAHTjpwKLAiLUNJttQtVjuYJLtDiNCCN+exQ4PAzWPHoWr+GLqR9Ldbu33gyWU2dzH1G3qwHTIH0rpkeOeNlwy73H+yDx39RSvF5EjLsxlSAxlPU8nG3r+P41KbHYjtruHVLcTWrbUX/AFgkjwyv/EMfw4p7s64LEAD5hhh16DgVzPisadYxtePfz2d50je1OWPoGXow5/8Ar1gWnj3WHjZJEtrj5s+ZKCjE+vy9f0p2vqgPQhE77DNEVY5zKytg47A+uO1W4I5nDr9n2ovL/NtRUxzk9O3XrXml14v1KbkQ2iKefkDEH65OD+VM/wCEt1l8YltcA5A8nijlYXPRQ8ciO0DlJCSe+W9vpihYzIp6GNlyRvAH09q8+/4SbWmmL+aCG5KtFuA99xGfxzR/wkesmTKTABgV6dPUg0crA9AtpoXJJUEdGC5UjtuGc1I0kQyTk5+5x156D8K85bWtW37YdQKKevyDk0jeINdlO2XUYe5LgLGfXsPrRygeilFW4M5RjJHli7Ybc3+1zjvTMskYMPTbsymCvX1/CvPJtUvyrI97JKjDkbhj9KiTVLm3Vc3VwU7bXJH1xRyhc9KRJeWcOo6t8vVTxwe1Qvclm5ZiFyqqFYDGf4uPb9K87bxBcRyMIbmcMx+YFmxTP+EkuPmWK4uU+bjbMzfpmnyiPTIgkOOhk6dTgemOnWlZniZcz/7IXdkA/wC76/Q1543iXUIowsl1cEL1DScfzqOXxxfLMrfaZQV4B3LkevUd6OULnpc4dbtopHLSFlYgAcN1w2M8DoOabLN5cUodiACyl4x8pAPOT2Fec/8ACa3rxskc207vuHGP0FVW8RalIAhvC6sRwg5+p2ilysLo9KFxazMWmgWKBPmUvwMjoRntWPfeOtD0xsPdQloz8sdvmR8j6cD654ryrWp57lQ7MXK/eIO4fn0rn2ikYjg4PNNQC52viPx9Jr1v9hhs4obbByX+d25yDnt+FcXIwweg5/CmzKbRNufmbrj0qk8ucAmqsTckkKn0qDGTnBxTi4Tntjoai+1qMf0pCH+Xz2x6U4wuF3bCF6dKtW1ldSwmd7ScL2Cx8/Xnt+FRXupTofs6xFJF6u7bnJ/p9KdhkUf3RzV+wYrcIw/hIODVRbaWMBHBEn8YPUGtDS7dnuOV+6M80hkHiKUNqFtPuyBxke3/AOuqs9+gUYJqzqqq9gc5yp3rXO0wLJuSTwF+pFMd2YVEOtTpEZSAORTEVzRV9bL6042XIyOT270coXM6lAJ6Vt23hzUL4p9ksLyUN3SBiM/XGP1rqdN+EeuXrYu3ttPIwWFxJ8+D6KOv50rWGcLa3LW4kVUVt4AyRyCDnIrXtQLj5mySe3euvvPg/dW2nXN3baj9taJN6RxWzL5nf7xOOgbHriuOtrfyQgBODyDmjfYC9c2sUar5Z3DHOa7vwLpWh61ZzTX/AJkuoQSY8hnJVlONrbRy3fIzjj3rj44Qy4JyR+lTWGoXOh6vBf2qktGfubiA6nqp9jS6Ae7wiFBIIoWW0RWKbWCbuflyvYN/nFSGUAqsKtGudqo2VjYjqAv0xwTVLSbuz1HT4byHe8M6K5ZuoxyVJHQgj8MVa8oRhI0yAwP3lO9m6jnNQUSi4LSLMo8tNjAncGAGQTuP8P4/0pGmYMjrzIuCshC9Og9Qc47jtQY5Y/JEjmBCSpyCQT13cNn1qWJXwkaQ+WwOBOwbsp56+uaAKgvDa7W2sRGqiSXcB/EO2P16jNPkuISCxMXJwgLEhlP3gcdf5fjU0Soqqx2bU4TD7dy4JZs8cYxxj5s02F4xYkP5Um5RtJHI47k9j/7KaQEclrJItskKMmFbywi5QN/F7kLUIiB2zSTrlh8pccf7x57fX8etWWkVHkUbSnGPMk+TcF4+729+pqG3aZVaWVYQARwnPmHHGDjpjHXgZ6U0AIk08TfMQdq45PI7An9akMpiVp2+Vlc/IH4HOSMZ6cg9MnFMBVZMfarnLRAFimAT659PU4pot1eTyDHlvM5O7OMemTwc8/hmmBYgUSRRfMEh+aUEKF38EEjrnkf/AKqimLxrG6yyRAbtrRPy3oTz9BzUEjTRSbFuHlBGdzbWJfnKggctjpSMcNujLksm3yY2Dg5HUj2yOntikIstAiYkUqSFA3cDjI47YH5d8U2IeRcRiK4Zi/G0yd8dgPbiqpud06GQ7ZFkbBPZ1PPTqfr+XNWldxGuUi+Z/M37Dn2GOenPTuaHsMbPCsz4M2G252Bdu0+m08+vJNOlfEflXG52Rcx87ZCQu7p688//AFqgEblwUk3Qs+5ecYGe57dvwoARkyglmmDFeV4HGCMn65ODTAfbz5s0Nzai3lAbKhlkA5GG3Ln5c5546dqdFNGybVG1sE/vFOenp/EOn5VLEzmNIjtWN+cM4O/0Pf8AAHmkkkKuZQPLySdsacMBx65Oce+PSkIaGIVNy7wgWJX3/IW5zz1xhscHHtUcRd/mYs3/ADyCfMc7hjK9h36/40950MCzC2MaSkcqvp14/wAaWZNxihUxRnblwTnt0+Uc59/SgdyvbyuZCJY5FjQ4WR5crIG56nlSQBxQrW85Z/tDF14cjDMpzjHHTj39KmkglikaNypwvG18EDHoTxyRz708F4VVIUjaX5CqoDwducbR95vcmmAj+VHlkjVCx+ctjDZwST34x/OoZIy8oMiS7VY5Z23bV7EEdOmB+FQrKRsEkexzy52qSM/xY64FOjErrDP5yswO8FUO0YHDc8c4/WkBNI7gN8uXh2g/KeAW7eoz29cmmBlZACgSSMbxs6txjPzcHnp/k0k8CeYm5gE3ZOEwF6Yxx/tLTmmaN/uH5Ys+ZxnPb5SfT68ULYRC8g+1bUEcqZw0JZk+72JHp/8Aqq3FvgkVmYMTl8SQlQu71H8PPAbPao5AXdpHVkdgGVM4ycZH3j1/HtwOarvdIObhvnLf6wfNtI454/Q0PYAZRGsu9vKIjbZh2A3DnHqR/wDqqdxNH5SyXPzNlt5DYYtzkA8tjPOKIjvKTQopG7928jbXHzHBUbuc7gaSVZGmZk8yXYWJHyBm46d1IPegCoY44rnzVUxzfx5Q+3PoPUf41M2JS4SOQLgAuDgEMDlR6t2wfT3pJIxKm6Zy5V9x3OygA53EDp1xnPTAqQqjiMkNlidznGGPr37cHPrQgFVVKk2tuSCpfCqGAXGPm454BPb0poP+j4RwCDuc5y+MYBGeON36+wqQtGJczvMqYK7gxZs7RnJ6ewB60LP54Uuq7JG/eM+flbAUYI9hTAaX2OuxCWjbfLI8m49vvdOnQ0yWF54BaecZEc/LnLFD/eCqe3bkd8Uk8MsMjMWhKlur4XAPXjqTwBnGOc1NEXMaN5oAztyqnnH8IYYJ+U4z6+woARYiggSYRo6xEvgEh+2WBAbPDcnpxSoZB5kyxRMIf78vRM5wcgcknA7GkCuuw4UQoCJG3ZB5znv+GevNI7NAiXAXzZjGeWj+ZV3fKevU5J9ffkUAOjEslx5kkmZckI6YADf8BPpkY6/rTJPM83cCiM2MNuwMADlQPw+nFLC8vJ2qVWQr+8wV+bgKR0LE55pohBiZbfcZFXbK7o3BzgrjuB7YP4YoESRSSgQs8hTYCwPmktz6j/61IrOLORnySzBn3AHaSMZz75Pt16VTgmAbFxH5ZdMNg8DHPAPJ/PvVtTumY7/m37l8tlBUjPP1wMfhQMSBZUjLRLvAIZIwgwSB2BOWPbjrUha4gVkWTzNjZVT8wHf+Zxx2qOeIeYizW8cQDDa7RngevXnGcfjSqHR08rLbcN5e8KMH0z756gcdKAESK4czZm5ALNtzzkdgPXjr0qYB1Zd8e5ZkbYQoX5iORjP6GqkB86GQyjyt4+WOONlbqPlbP50pvTFK8O1fKkH73yyPLb/YHXn5eTz060wHFjuaURO5kU/Lhwxx8uAuTn09qkjnE5aVRIX2+WQ6bcD7oUqQPU/lTZLl51ZQ+1UU7neXCs2eMr+f+TUsMiRO7GAGNwVzG3Unrgc5+h4pARyvGiwKwYCPJ2KmXPHBJ59uPxp7ZMrxyOrAnLZQJu46gAZz2x7dKiuFuHRpYBDGo2K6mXnI43LznBx/gKdaTeVM+IpTMFHl+WSQD83G7jnn2/SmAiIrKU8oqQu7cOVxjrjPTipEMR+WVUVcLtZ8MSO4I6fgOnFR+SQ53pHECzNuPIUdRwCCFIxx9etNlSQKVeE7j/rAnKt/F8vTjlfyoAmi8sFEV/JYsVIDc4xnqOAeg29aQMJJZIzIDjngEgHGMFfXjr/OmnY8sPmLGpKF0EjBQ+Bztzg7ulNJndmE3mOsSbUdPvNnIXBx9ePcmkMnEUaySoqMG8sEYcjJ64/UAZ9KSXAVJCrn5/llPQ44xkDJ+vGKhhiS1gOyaSTYu7BAYrj3644+71HWpAzRuUWZ1cL8rHgqNvIJDH3yOP8AAQhzI7XhDNC3mEGIF8xucdAenp+eKga4X5WOQGZdyIxKoeuRgcnrj6VDGylXdg2QcPIMdD91T2P3fTpSwxyAu2QpVSuxpOSOSO/t0+gxTAnCLLIdjFocFXEuSeOB6bjz0HvUqeakwuPtDxhWXPlIoDHGM4688jNNiJlVUeR1YgKFI2ll9iB14PIx+OaZ5yrL5ixyS4yC4VcBv7oOe46/WgC2RNGkkaiORurcbdv1P+TzVeZ5raF/szMCdxRWwSx/hxnt2qaSHzUc+fIzM247CsbY6d+h5/KmSbpSzfuwVAY7jnb789eahFHjHiM3T6vNJe3Nu0oJHlxN8kf+yB/D9KzI7jDjEhAxxjmveDCtwqpNbLcxbjtjkGQ2RztGPp09BWDqPgnRLu4KTWItZWc7mi+Rj+A+UVSmTY8yhvdzbUyW7lxVtZgDv3r+HOK6w/Dy2QMbe5O3k7ZX3Y9u26s//hCNViKeQ9m5z6FcfgDVcyCzKMcwmhHmSk4PGT0oa8hVeZGDfT+lWpvDPiBThLWF1HOd+OnXiq0mka1CIxFp07tjP7plVT707oRGZHDfuSXY9Cikce/YUsgcrnO1s/eX+dD2OrfcbTrgNjlR1HvxUMVhfKoVdKuAScKI7VgWP1P4UXAk3FIwBOzSZwDI2Dj65zTVfzF3AOcDqcbf5UTWd+o3Ppt0u3lgqHIx1zioZpI/LWS8WREX/lmFbHNAFe5vIQdu8OF6ABjk+4qkJJJ22wDao6sVyDWpBd2G1Vt58lsjYoZz7jCr/Wo3mtvnCmRiOg8h1H8qdwKYiAkYF5HKf3ZPu/lnFPJygVIYyx+XMjE/jmp90gx5EE7Iy5UiAhefTIxU0WiapfSBIbC6K+6bAx92c4/Ki4ama1nIhLSSxxxjnO4KAPb1q7ap5iq8e9o+gGcFvYDt9a2YPAt9HLGz2cQZuQZ5lYk/Rc4FbMPgq+uGdLjUIrdfSK3wV9RuY/e9OKXMkByssZwqTnzW/giH8P4f16VFLFBa7ZZhBFNIoVIyw9TwOpP1rvT4CsZIEiln1GaPg7Hm2K/r8qrk49SfWud8Q22i6TeTWWm2EUT28SvJMXd3J25ABP8ADjn8cGlzIVjh7/TLu4nkuXVVjJ/H8B2qF9Ato7cTvq1sqEfMXByP8fSuiTUbN4lS6wrysVXKblK4xnNZN/bW1pIxHkG5B7JnH07UwsZUlvo8Qzi8vCOMquxPr3/nTItWsLJD9lswkn95vmb8z0qY32oKHjF5KIyDwpxisuTT0Y5WQr/vc0gLFx4kvJWLRnyzjAJbcw/Gsy3kP22KRssfMBOT15qX+z5T0ZD7Zpn2aRDksoIPHzZoGdXHAZ7jeYgxmbdhj61qpKtu8SomAx7L1H1rO0nEqgPjJ5OeorYnhMrwujbSPmGTxQByniMgSSpHFtjGMFRx9a5xULniu28U5MUcbEl9vOTnqa5mKBAVyMN6+tNK4m7FeG1LHLY+ldj4d8DaxrKRzQwrDaMpbz5jgEf7I6t/nmrHgjwp/wAJBrDy3AZbC1G+d/U/wp+P8q9ymlSG1gis4vLhVVVY0h6ADGWHf+QFKcraIErnB6Z8MNMtlWXUb77UzAhQJPKiVvQ7SWP0z3rr9N8P6Jpsf+h6fZ21wFzJJHFvl2nHRsE569OtX9kXlorJE7jGPmwN2O/b3/rUtxI8ql3K435ARcfXBA4HHFRzNlpFSKOK7AMVvmFmCHLlnJ55VfoO/Gat70GMbUzuyFwRH6jI59uecZ60puRuUwDKcMeMA+nPXgdqrpmPyvmQs2fv5yDnPp796QWEaRDeqVm3SKMMjNle2R714DqUdu19em1tpordJ2ZIZxh1XdwrA9PXHoa+gWaNvLV4wrqPvgjcRk5b8u/tXM+MPDH9t2hvrbcdXtVZoZFG0zIP4HP57T2P1qouwM85+0R6lbxzQwrG3zcL149vTNWV0lGWHzmURsm5sN8wOPQ1gWmqPa3LTbhJG6hNh4285J+v1rQui+oxG4h3iLqwHJ3VRJq6bfz+E7sgX1wsLnc8Xkbo5ffGfz713Ol/ETTb65hWQi3KjodwQt6jGec+teRXml3Udi1y4/eJyys+GUdtw7Zq1o8VtJp3mSD98WzknBVaLIZ7vHLFNDlLmO4EqfJPjcOB1Udc54+tL+9hbCZZ0bcGPWZj3yewxXmel65c6NqCGGaSeGTkwGMsGOOR0xubsfWvVbbY8UN3b2o2TBGV3J+6RkHg+nY+nrUNDKzwzSGTJ2tG5kxJyNoHPzdlORz3pyznyygWWRpGHzb8beSBkEY606UzsiuhEY+8Xz1PcYPf7oOM9KXywkcjzM6eV+8PHAz/AA5GCR/Ft9sUAQW5dYporgRlAC538oDjGSf8OmKndp52Rdv7mP5o/LK7yMjn05ODz25NCXCM32hWk3ZwJMbdhx3GOfXsefanuYZt8hjxuAIby8ZAOGxleT9emKAKamdfs80kSQ4LFY/m2hcgNnvjPrxSeS9xL5nlKIpPnlDHa3Yfr27DmrTXEjzSRhgAXy4ADMw44Jx7DgfnS+dDIV+WJAUZmWQYwc5HJ+Zu3A9qAsVtlowLSMFUjGI8fKufvMCPX09sntRbeSXaFgF8x8/XnjOD/nJPSlAa5VBOwUYw4DFQwHIHzenHTvx70nl26ho5IkZhtPKEPjGSq55HemAye5mmRVcqg4jMWNu4D0Iz8vBHqc07ZvM0pdvmX5UjkbhSDkY9OOppsqbE+UbOCBs645ONwz+OPanvEBII2k+0TZw4KYGevTgdu/pSCwwIs7qXj2KwLKFB+UZ9BgcsfXoDxTp1CyZbCAo2Q7FQMd29Og4+n0qVZERIy21jjEZRQy7eRwDyDz2H8qiby45mRmGLeMLwmeFzuY/hjnHSmIZNdoq7UaRYgAx+UoSp56EevSmqWltQ0iMkqr5pRyW7jknjA+meOabZo8E5l2u3mYaJmbd0yc8dAf15qVbgwTriGRJHzxnqAeOWPYf55pAQFRMzbrhSxARecBTwOhz6ewq00CiOJ/LkKRIHVwx4/i7YOcc8f0qETyeegVWzKqZMq7gBtPzAccYPU+pqTDxLIUlU5Vf3gORn+LHHAzjj86YyuombLRRzJEAFwxyGx1+Yjv8A/qpzSSiVonmjbd8scWWGOeD/ALI/SrEaQ8TSmM5xlmY468deTzwM9cVTQzJtka2Mec+VLtDBsDnpyT7+1IBSohVm8rdIjqigIBtOACPy749amhZ0QExP9ze2flHC9m6dB/8ArqJYJPsEiruWIdSo9s/KeOvpk+9QeXcpGWh2FN+0uG3nkcKPX/IphYvRkvDtmZy7MFQSg4BIB7+o7fxdKhnuFVsXSrGGYiNPLwVOemevIPb156VBCZFm2y+WUQk5+UEckED3PAA7VP5rxw+a5RGj+Qo+1vlbsPc85P8AUUCGIspWELFKo3H93sKYXv2+bPY9c0SrHJFK7xySBCEb93sOCBlc+u3r3qdLrerGRAHU4Zn5BwRxt/qMZ9KUuRbiRke3jjTcjRAKBkgA8DA/HrSGV7e4BtwEZAEGESQZx1Bx/Wm+dFLEBtVgqN1ViCPw+p49cU5yWmWbzXYLgl+p29Mc9j78n2pBMyRqr/u/KTccp82BwBnuenX6UhlqVVij8mYbC2QzueSDg/d9frUbSo8DKGAZQWOCdin7pPPHbHJoaWBmiWQPGS/+smG4gjkjnsxI7A4NVoFzlJHljkAztD4yOcgLnrTQie2eYXLQlZPKPIzgKRzgc8DkdueKZPJFcqpZcqqlhNtLfLt5JU/45pqt5sQaIrHlscYPBGDx3/un+lOkWF5V3RbIVRn3ShSD1XAwP1JJGOlMQ4R4u4vMTzkXbtcjGNw5AXHzf/WNSyRxRxl/lQth2YK3IPTGT/EM/Me/amGWOOLBklQGPCMSW2Y4z/u9ew+tNE8JeGRpVMgT5SWZcMDwd3b/AHenNADfNjSIyx/KWYKhCkDByM+o54HFSsPJYKJJIXVfu9M465x1/rQQRh5R0ZiQ42s3qAcfgfp61GyXRmdJbcGBU+UoQPmI4OcdgeCR0pAUG1ERs0LrPw2d5TagHYrg/geK0f8ARmbCRW8XnMGYjLEMckg5PJJ7UjSBZvOR1R0+aN/u+3p396jldCywsgJWNmj+TqAenpkYJ9ePwpgWLiZhCi3SAqjM+87SpPcZx04XkjtSMYUgSWBmijO0kjco3Z5+o9fr0qOzuFKgtJtjkb5hEVK4z/CMcfiTTjJMX2wzM5ZWddpB+U+/04x65oAfI4MZ2SDDkg/KAN3YZyMAe/c+9K19HEqssKCRirLuiUDC9yNvzYP8VEoVxkzl/J4A2KCMYPQcnjqPaorgCW5We4f975heUMWUkHpnr279BkUXAbHKjH92sKxsQrPHjA6Fj6A9KfIjLK6vFK7IpXbgr5Z478cc8DNJcpCqxM7Axx/LHAFY7eGyBjsP59elJ8sTx7YQqsoO5lBEhOeoycj2B4NFwCJGiubZXDqQW2PxxkjB9/f8qk3IJ90W5BvGPLJ3DBzn6Z5//VSrviuU/eLvlHzISVyvqxHTnPB5NQhDB/pEc8yHOGxu4JX5SFPT9MUwFYziUeTavL+6G7cFzzzkbucHHr6dKbAkRBPlybsksCSrNnuNvp2+lWGnSS4b5/NyrYB4VcckbcdP0zUf7pJ44tsksKoAXdfvDAGOxC+nP8qAsMCzLI8cTyLLnbtlIjb5R8ucD/a6DvQ8QkVRsCr91ihyzFe57r2xxU1ohIcrF5cpwgV/42BJ6nt0POOlPa4nUhIpRcbl2ySKS4wpxnHGBSAgieMS+XslMcgBed0xkYJGMZ/InqBUe67jmkRZJEb7jZyDnr835/56UjLF05WQnJzMflGcjrlenp6808xhWmh8zzA4O7O7arZ6YB454+p60ATgw+So81tzbdpypwp6buBj69TzURSFnVGdwvlblOCRjPAb9enpSw7AJEtwytjcFfGDgAfMDxzx6YIq3Du2IBDCJDHuyGGXQkjk54YZP1oBEECSBo0lSWRgccocSAHkcjGewOPSmo2yLaj/ALphtdQdpOe2WyP+BAdj601N9mwdt37luNsux39MHoOowfz71DM29onWeRJ1Zl3vuZSvpt7NxyN3emBavHRgNvniJtwOR7A7ffCqOn5npSXCl5JJoJcMhULIrFQxIx6Zzjdntn0ojiE8yoFVY2OWdF3NwpAx29PYZqBvtCTghXa3VSyoY2yu7Bz6egPOOetAy0TKmxYgqBurFF6888jOafFEhSMnbHH0Z3QNuJ9Bxk0rSyyRLCjKEJ+Td/Dn730z/Sh2czLu42/d7biBjoOh561BQjNHuaHbsxySDk7efXGPSoII4o5CIEPmspRsDnB6jrj1pxCeSjjIVi3O/cnHPI6+lJGqlXXdlcDkKfck/wD1qBClWiL5KiVergN8v+TSKUilmkCCBJMYIxgqemfT1qYSv+8kZFXhY9rEDGSOnrn9OtRZ8qRZAH2dVLL/AOPHPHfigCMXCbo0aUKTnADZB9BkVOzb2eVEkBbkgqp38Yyef8801RKiK7zOu75Rlfv+pDY/SnZhhZUkBlRWPJH6H0+tADIlldiQxaHlfkG0n+71z1p21GbD7/KCbmIGT7++OaIbhX+SGbzSr5I3YZR1Vs9/yFNjjmnwwhwgVyMnaWOff734UCBY3lVmSNuxVUj2Ee5Jp6QpuyYoZT1OfnDH+QqMiSbfvVgcZ2BOPcgf5NKvlJIY2Yb9rYJ/hXrknHy0AL5cZPywqcc424U/gO1ItnEAAkZOPvEALnPPOOo4qQNJE2UUPIygYK4GMY4x0H86a8rwrGjCaOLG/Iyi4xxyc/57UDQotY7i5P7oF87l8wfKv9afbl24TjHzEb+PbPP86rRSySlpNw8lRnL7lzjp8p6/SrSmL7JlFj++C2Uwceg/HmgLDkhLwGT/AFZQFiPLyDj6UsEnlqsgtScNtL8/Ug/lxSwxrIjPNtVfvCQrgdeBhT046UfZBGELAQpvb5kZgGz/AFoAhcOjFVWQqwwEbk9cnFcX4u0W+lvzf6dam7zEIrqKLAf5c7WC98AkHHtxXbESH5UeRyDuXK8EZ/hB5/LvSPHE6hWhlZwSrKw3L+P+fWhOwWPBbqIRXTLPC2xiD5ZQq8fuFNXGaBrcGeAPA3CzquQfx7Y9DXtdza2ur7YL20hlt1/hlVSoBxwo/wBr25rBvPAXh6VjNbQz6cc4c2Evlhh2+U5B/EdarmJsePajZLBMxO1U+9WWYhI/fFen6p8MriaETwa4zgllVbq2AHA4G5WH/oNcXqvgzxLpMTXEmntNbrj99auJF/L7w/KquhGHPYFo/kbLe1RQwpErJMdjAZBPet3TIIrqGKTzCJmJAR+FODyM+tSarot7qUpuILXJXI8lBzwuT+gzQAaFAJLpBnd1GPUVth86kNnAjUBcrj/Pes7QInjZDGuf3TEFgOcjFdHbRxSQw3hdsoAMnsR/WgDkdbt5Ly8uHiUkRqznJxwKw7aCSe6ihgXfJIwSNf7zE8Cuu1xFtZDLuJinVkbBAwDxS/D7RLnUfE0NxC6JDY4nnkK7vl6bQPVv5ZNUnZCerPVvDPh1fD2gQae7xxybd8sqxnEjE/M/PboBz0HatcC4DNlkSKPGCygszZxjI55GePSnTRqSqQxzo4UFkKryeu3GeBVe3l8uF7uZwArbkC5fJz7Vj1NB8pd3SKTMakh9gwRsPoOvI9aWCDbGx/fJEzLtyR8gJ+9/hRGryTC5RlbO3cHXgMevvjjp1+lPRXeHEbyFS23727aR9TSAaY0imjjkDuq5ZoivGTzx69qeI5AYo03lSv3Cnc4JXrU7W9t5bo7AH7olRWJ55wcdOn596rzp50Y8xlfP3lBy+f73p60JgR7Ny4GEfqxWPG7HAGT9BUIhuIZfmUyBZCArlsN9MHr19anFqEcKzqse0qAD5hbnjOcev4mltYmeEvCzzKBhvMXbuCkkA56dOue1MRy+r+BdG1ud3n32Vx/FcWkWctnqw+6w5Pv+dcjqXw+1zQsvptwdRtiM7YFMU+3ufL5U8ejfhXrjLbo8kzs4faWVmG4Lz27D8B6U2DyoJ0MlxcOemzyuvfr9W+9TTFY8WsVubdGtodJ1J5DuyHtJM7iMDOVqzNpF9K8Ns/hm/tZIQC6BCilufmLMdq554zivV0tQFRkDmRMb90hYY9eOPwNTfYpllG+3aAmFmGCQcZ6j5h8xXP0qrgeZ2HhHxL9oN3DqNnZyOqq8L3Dkuv8ADuKAqAPx/Cu+0m3u7XSbaLUps3ijd5sWGRdzHHzY+Y8dSKtww+WrlWaOLChkkxuRODsJ9e+PpT9sMaunyFZCF/ebs/d+6e/RiRSuCITM4SMfbcvhipPyqynvxSwtJsb5l+VflJ5zxyFycZqeNlMEb+RHIuPld2xuHQY98dvaqp2RXvCBnB+YyqyjPXleh9ce9Idy8jXCYWLEUqjuF3btvPGeuCP15pp3yNCh3MqZR9qAr07Dp7561HHNus3mH7tgFDIQMEZztPrimOJSltcgI0WdxQJny2I2nPtxn0NAE0m4WqoqgRlSPMxkhc9/1471XEcogkZJQfMxkgBdp5xzjpkHn2qm7rLOvnxO0QyMEYyVPGAPr0HFTNJ8sby46ZLZOeeOOw5yenWgLluzDtYib/UmO58tHWT5JBkhWyeVbpwcZPSo5UuZonZlV9ilpHnfbIuWxn5u/GB+lUpYLS4j2PBFMGx5iOhUMM4xz16dRzzwRU0KeR5SNLNP5EaoplYsDx8o/rnq3UnOKi00XeILbOkihEnfJAYYIAz/ALxzx6nH4U4yTFUVIN20hg0UYXGf7xP3vug9PWo90cbxPJvhOcKk6ck+2fXPVv5VIUCWreZlhhmDluCBj+IegqyBtvBtvU83G3cu6UMDszzub6/1qypUPlncqW6Pjn6EcgAgcVGbgy5dLzdGWCg+Vu2rn7oGODxxz9KhN3FFIkMZymRkuwGwnnGPTJBP1z2oAlkH7lJjFG6vJ8pOCTgYLY6+3NQ+fBLKINmZWJz8nK8/L0bnuenpT5iGYxtEzKqlfLIweAAfXnJ9eKjTzBBFGzofMTI3KFG0++717/yoAnnfzJnKBl/d8RlBGAQf7ucEY6jpnPFBAjuUeU/fO8uM8c8rnP1JP61RZT54nWVZHUdeSc8YU++M/pVi4BeJZnuJmRUw4UgqFxzjpj72OffrTAkmUGLf5hWYso8tnzzlvm/DpzyOKQyFygbCF22qVUDd07/T09aGV5LWScZMjsVOxcErt6qSMdf06804+b5zsTCUUKChLcqQMDn5j+PPTFIBlwqojIrRtvfezSOF3c9D/L8KZ5kkrZmJjRCI9yqMnrknpj8PzpQqeb5sOG8v587D97oFAHfjue2famtIi3G1CGZCFCliPl98cgf45piGDYxiW3Vo0VguSxDDOMk9eTjr2/WnLa/OpQkSDvnaWP8AeH4DPvmnIB5KPFvRUUcB+d2TwfbnHt+NOuGhe4EKSqybN3O1sPnH/fP496LgBBLKZIzGvIUbwNuFwQOcfXp/giKEmRWEuwIShXCk8/8Aj2716cVHsnx9wL0COG+4WPXPAGccDsPzqy0KIPKR0i3nayOzOflHCqQxyO54HvxQgKy26tdECOZTuG5D/Dn3PBboetTxW0iQODDIOVDc/dx2I+nuc81GjLABGQGZjtUsQRj37dO+PlzSW8oUBmtsKqluEOSg64/DufyoAZJDLGzfeLEYOBywBxz9SelQiCGR2LvE+QTll9eoyeQfXHp3qdopmhRGjtwjbuNpB+91+XP+J/Sm+VDbDzAjgNw4dmxKPQ5HHsMA0IAIKIiEqNo6puPy57Hqfvc8d6e0Uj27SLG5DqWcr028dMHI7HDY7VII23bB5bDLKAhxuXtjILDBHXrQWeWKRtjwq3yFWJ27sdGDE54XHNAESrIJVto5BGdwPzZUqRx1bp1PegtKH+aRGYbvlGGztJxjPTocHHOaa3ClcNlj84Iyx/4DwC3T61EpDSMkkU0gkycSYAbndj68HpQBPEYY83BVmJB/eByd3JOVx9Bwc459KeskEojk2wPHuXI5X1IG7Gdv+HPrVIqsO2RnSQ5+XZCY1HU9OpPsPzqwGEcxwcQ/L8hYbXHofU89MdqAHKyK43BpEk6smRz7hhjnHSlVoxDEnlr5gLMEYgcjk/QbcHqe/FRhGeUDylVUibBaTbz65/p3xTVAZli/dzREg/NEMde456ZOOfwpgSShjbBTxlvlR+QzYPXHX8Oh/CnRyRxSIhd1Xld+3aGZvVuPlxkdu3NQxEuzQKuNr/JhmYE9cqexBHUf3fWn71EPneWisFbd5e3I5GcL6cnp17e6AlYukmOTt3fODwMY7jrxxyf6UxpljHlsofnDKcY9gSBz6gZ9KhKrOyxsoZAxZQj4O7Py5GP7v/66UQ/O67tx24DYyM+gHTnNAFpzcNbxvMqMHUAyFuN2SQ24E89fTvQshYRIESVyAhZpMDHQ84x1PYj9ahUN5n7mAS/MFUs4DemAv149MrinyLMIVyp5OFj3DIy3ZcZbPsfyosA0yLCSqqCm8jZsOQue/rx/ICpFZWchVaQNll2LnG0ZBx7j696bIkhugI1aOXfxku0cjYOflx+vHvRHbRXWYRHHIrNwEVVbpwd3Ttnjn1ouBJ9oEjIkgk37lK/NkLzxlsZ9ev0qORAsrNPGGG77oQtkD7uD3B9PSmI5hUvH99cs6ZPPHOc89OPw4q1NI80sWSHud5U/NuHOPuknj26UAROkgWI4kVlyuyNjlBu4xyCAc9+c+1LGj/apUHn/ALyELKxTeAc/MG9Bx15pEZXZcFTISoCqPmZu/Xodw+91pjM6RpthjkVF2RibaVVieTjoBn+Lrx9aAHvJ5kriWRRFOvQ7sPxjlQvXOG/HmmuLgbJkhUrJgqCPlznBXIzg9euOgqKRIzG7xsT0XBXaMf7x6DJHYH8KnaTzQZsR4A5AkwVPTDA8++enPWmBJD5xmJuUudrDpu3CTH+3+AOe1VnZ0lZxn5yNgT5gAfl2r19sD2qXI8qKOLyh+63IEG4yN14I9vlPPSmmGdZjtCKhGflVTnjkqOnHp2zSAci+WIpoI4N0f7p2Taqn5cMv0/22746Y5j8xjEFado0UlWi25ztboD0JPf8AClZkRsXB3btpd+ASuPugZ/n06inAB1G2DD52vI0jEd8c465BHqaYEDhWdwkis7DB3JsDDHzZ54HHX2HFTqWTy4xAuwOV3LHtGN27G3sSOeev0qEMzeSzRSrNIiq0gIYEL1wDx/d604QbZEvPKt1UrmPLL8xyVAK5IPH0x+lICdMbin70Fl+VXwSfmGOOPSrcsmzgmCMJ/ACSdv8AtLjG78u1V44vMVRJbSOX3KoWRkOf9rj+RFChEiX/AEPzWRtzyIpLA84x68ZyD6VJTJsPJA0mD5fQEY+9/vDuDwQRwKbAzgLHOjBOuUHByvqO3TFQbR5Yea3dlZQwAfkAHtz3GOvPFPVBFMEMcpTd8nlDJyBwMg8dvSgRIwjmVmhiZ/JTezFuWyfc9cn9c1BA7BlDjYZGUkHAA68lumSaWV08sx7ZopDwDs24X3Y+uW9+lVVWby97PwrlQI/m2nrniiwy5JLHMcyTSY/hRPuqxzndj9cVOg89WE0b+ZwGkTkM3X04H58ZyKo2+0ybQhKb/mR35J/L8e1WHhV32Oxw33dxGNy+p/x7UCFA2Cd4RsiO7dhlYkE9jx1GOnal+Vg2+YBNvyhyTyO3sORULxSyAN5zh9y7k8sAAY4PHP4Yp0sscCyhpknmcHopO3gKAOMZ4zTAkjw3PkYCKBKyOe/qw96RhgRyRh0LgLvZMDjr9evU9KYkeUWRhiSQ5IGF3nqSfQdOOhpVkScbQpkdEVVGNwVeh7AY/wDrUhkhYW7uySgiMBUDZX/vnrj605pyRDK6IUALYyeCOcjsf/11GVmb5JUDbVBXfglD2JB/Ol8gr8tw5dwGAVcHnOcDv3+lAD5kmDHYJ1lwTsUIxyec4+uaSWdUwUY7FXYy7gFZvx6n1+lI1qr+VnywXXPljkg5wflxxiq16VlUK1usoTCBCuGHXnHHNFgL4RUQNErxO2M/P3OfXtioiHYgM24x4LfOWwO4z+XTOagQSOXILWir8oRgSM45yR/KpIY2WMReQR5m0sI8LnnjJznnnmgCG7t7S4wAk2cZO8lS6kdDz0z9KmtkhjtfLhUeVhUCSHcV9AOeBz71JdN524OV3INkikghccYPY/8A6qiWRHl/erHHsx8kQBDAe/Ge3agRbISOdEZl2biFkH0/hxwPxqJzcGRXwmwN1ckkjPUjjA/OopZE+VxGHVlLYZtpTsOPTPFRPNMkXzwKrcMxKbmK5+XGDx19aQEq+SJWMREisdrBMnj/AIEOSKmjl/eHAh2eZ90fwD6nP51E84WNg7EtJklNrALtzgZGePpUsjGeHe4RmCALh+Fz93JboO1MDndb8C6XqAFykosb1pDL9ojj8xZGwc+ZGMBs9zx35rjdb0XWtMaMyrpUMUoaHzLaSTe25clgrD5RtyMZ716ik6rAiJahdvzbkyfmH3jk47Z4z6V5t8RNRVbgI0kSLb2gVAvRnkOW/ELj/JqkwZz0mr2MVnJ5OB5aeXGo46d6p6V4pa0HNtvBJypYdK55BJdWjzICIweWxVKxR5bsh3O0Hk1ZB1Y8QSXpltLiNBDLG0Xlrzsz059jivR/hdEIfC0zMsaT3F22Qww7BVXGd2OmTjHrXklrB9g10Q3EKzx7Qw9GDDg17r4XuP7Q8I2bowfy/wBw29d3zIxHynqDtGamWxUTbZkkuGLieJ2wqvt3nOM8r7e3SomCRkgNEI+hcg/IPXt/k09EjNurPNhccMF5zz/9bg+tPkiKSSSLLiLIKiRO3br27+uKgoY8b+Ud+YF5fezbdoz+px/nmi0tIYZJ4ZZmLEBpCwJG3HbPc7hTSLp5TNJIq7lOMIGOO3H+NPtb1o38kot2Xw3KlTx23D3xTEWJRDFdyBZLjGCFIjwFYcfMB1Pb/DFNaBJMRW6RxyIAD07H0zgmmh83MSJjcWLbwpUHJO4EA/hnj6VDIHIdIX3/AD8CFPl/3if0HNICx5Gy1CzSIpVhuRYvnIPHPPJ//XTlQSyyPIFcfe2A7d4Xvjr6nFVyLiNndJdoixtR13An+LJB7ev8qmiuEgmjZIvLZApZIlIEkg6Ajjof50AImFk3QvcMdjbmVVGMnAHb6dKeqTeRC0hCsoUiOIg7V3Z7HgdifamzoFulR2jYqBuKocu33jkegz361UUMP3gkeJwmW3dcdWb269aYFk5CAIVl/ibaOC2epPUDr+VQXUjzygL9njdRjeSx35xyMnFWDCkwMkJOWHCbSCx254I+9zjrxVedbqSMoJhHLJ8rKMFlA7bug6dKBFsPNHbSF13SRtuV1IIC8Hn1yTxn071Hd3KOwtkjaV4iW3Fgdqd92evfn370y2kRbSGFpo1xuH7nOZDz97H16nimW90siuse4MwB8x1ysJxnOVzk9vyoQCkbhut3ISXG4+WRtYLjhSc9CvsKh8hlhBLAB32bMF9q4+Y+x9APQ1IYEhfdtCzMo+Yx7y2Mjbgflnr0qKGaeSRn80I8XzuhyrEg9cZ6Ac80wRI1k1zZ/aHdopVbEZ87BZcDoe/JyfTHvSyJLKw/eT+UylnkXcC+OeWPXg+3pzTn2sBLPvkjToxU5k53HP8A30Pzpwa1VU82MoSioFdtxbp90evTLY6UhkHkL5TLHMUZYzh2XOV42g46Ed+f4hTYyiJ5ZZppS2GYf8syO4GMg8H24qd08meMTeaFbAb5QgD7efrjgfhiqdu6R3UvlzNLvRgiKu0tz0wBzmmBZFmQk84STzEzIzliPlJGMdAPXd9Ka6vAqjIKqpd3jfcMYGCeOBk9/altrj7YZm86Hyo0Xchyw2kHqf7wxzmnbpVjVIbrEPIEkTli2fYcYPTBNAED8TK17zJJncJEbLDtz3A//V0p9ygjjMgwZFPGU+6mRj5emeR1+lPt4mMEQg8zfu+d1BC9ei/3sfh1pkFnIplEs32gMrNkNkocZPXufb+YpAQPiJdpkZHfO4J0LZ5GOM8Ee44qe4dI4URLGZ9xfH7xQiDvtJ+bsMnk8daaIY7p496F0wpkd246Mv1/hx/+qpp5ow3kyKoJARvPJVwuVOCeijAXH40wGQojkeQPLmIDBuuc5J9+OM5PPFQPJcNfmx+zuLYx71k4O5uBymOCR75q4k7+YUM0kcYQqr5ZRk87sDoAP5dahj8+TUFNrJH5Lbfuq2Q+MbSGb6+2M0gJJkVUKyIVXfuC7yUGc5weAGxk/lUDt5SKqSJErH91vG3KjcQ3p0GOmBU73MaXO0jzZt6nazZBPUL34B44x0qNEi3SuqA53HY7HO373X17Y60AQtc7gu2TI++VVvnbHUrj/PfnNK4l+0K6r+9YKcvFkqTyCM5xwP1HSrDCaYBoZIltlGcwtwgXuemCTjmniQSKzo5h8zkNsZQynoo7Ecf/AF6EBWiiSJXk8krGjLsLSndnvkK3v+tOZApWUykCTO0tKcx7QeASASQcD9OSKfDbQG1aECQjLMuVbBK4JwxPI46e+e1KstksAkTEkPmBd8pAHmZOd5z0xnHHX3oArzziOSLfMFjYnGSfmHTg4PPX35+lPA2MJR+6WRFHmFNuSM87Seeu3P0pWYmIiCEusiAMM52jPQEe3c88UoETwlpcKplUSon3thB2qvYjDdc9u9MQyAbJPNnCqCcb3+ZWPuy+56j9easyAW8hX7+wngDGTsXJVuf4f7x420QyF5EkabeuQ7r/ABZx8o+XqeRwPfBqFpFZkEILqVZTktu55Xr8u3IPuT0oGMaGUncctj7rBFzGOjMRnqPlH4ZpJZhIXa4zIobaqQn/AFi449yuePbJxUss/lt5rxK0UOFBdCRyenPTbj+L1NV59Rme2SBHkkjZlLKTu+Yt2OOOfugUgHNJE1yUG5QjYbMQOw4wQMcDngnPH8pYHQqZolYn5t68OEX+8cn72P5GnPJLagtsiG4sW3AbUGMbc44I6c8d+oqKQzPIXRYHZmbzJM7z0wVDA+30PGTQAzE8k7JIqzHO5lIwVXbg5Zew5596njt/tcIit02HbtV/vKASeRzgjAx0JAGe9R2kpfzCUXy1XbucYxyT2OelPhRJWR8F/M3rKxXJLDGPlz93+XPXimIrRxH7PluELdAGw49cH37j1p5tY42kI5jMZDSEY+bBAOBk9h7/AIVJEiiLfH5cPJyysdzE9A3sR6D0p6KVL2xLttG4/wB5lVR8xxnp0H970NICo4+1Blg2l41CoURWbP3Qo7/T6mq6WTvHvTzWZYt3KHHfrj9fTGavxECPdtUu0bFVLY5bp09i2AAKbE8kVs/3ELbmGX5EmMZOOny5FO4Fdf3cRG8EvkkH5SuOmfUf1NNKOShYx7XbJL/IvI5z+vrng46VaVFmn8hIWiCxASuASFIb5v4t35dM+lS7cxDJBb5mdxH3GBn6YyP1oAoTSMA7OfOkU7lGOwGeBjCqM8//AKqW2eSZ0KwTo6vtAhI4zjgYHbnrz2q6scKLNseaVWG5cZLMP7xHyrxuIHB6VDLCIYxc20W7KfK4mALc8e3X2z0ouA6UzGRY0dvKyB+9O4sOq9Rx3OBjrg9Ko3TTC4hjSNncqrYVlcEAEHAHGM9snHrVq2Fw1ixYRRjcEjc9VxkfLnt835+9WZkIhjkPlr+9BUDDP0+bIHbrxRcCnZTx3LzfZl8vcQEV0wcgc9h0z0/I0ATySOskSFzgK+w7cccdz6Y55/WrKWsHybPLEi7YWkODgAZOR0PHfhugppkZU8uLeGRt7RRnPGf4WOMDnPrzQAyaL91MDLIzKuI9w56cZx/DUTyx+ZmGRFDKGViMg/Lgn155+tEitdxQhvKURqUYFBsUfw8/99H88mppokltso2ZwN7OicFMfwttIwvGc9zQBXmu1t5NotjIqg71WLcgTqM7gcdRn6DrQjosbNl9hXPzABhj5e/PbpjFWAjK/mDy51bOHKKQzFcjLKBxnH64PWo1hQMgMimN0KshfJXuSMcdf/1eoAguHVAjpNHHIuwo2UPrgkHPPB96kMaPDMZUTEh4AKs5x1GMcY4684FDQfLJLGzRGTMR3JtG3I/i75Ptxgc9qSKNkiRSGLyKHHznCqMBiwxhl6YGaBkN2o81cwkqzFT5SjDHvjJ6g+tJLZpFFbyIRKkqdZhyzHsucFu3rzV9YZ1i3S7njkZdzKFXaAvb0HH1qnPexrDvFwUUK37lyW8tSTjnHp655z0oENtreO11BZEuU2PIqsOqHqNxQ46Ho2f4TVgvIIxskjUqp2MF+br03DouM/wkUzzLS+PyE2+5TGqxr8y98hQNvJ6kfSpI0hlIjmEYuHdo5Ez93jA+X+EZY/U59KYEHmMT8ryeYzdWDD227s9+eeOoqQvvjfafLjkfau58gEdA3bofvfnTQkSW+/KmXcqIAy7OMbsr256H68Utqsk7KiJtO0nKpvXpnk4+90HPHSkBXhMjRsm0+WSyRMOnmKT7fUA+xqdVd4pGQJOCjR7og2Oeeg/HHbPPSpms/LmRHTaq/LsLjjBwRt5yMkDnnIpj27OcIjtIIwSHIBbP8QAHzfxfTA5FMC26SzsHuGGwkr8sW04/vL+XfORVNnLXZ3mMSqA2c7ApHQDJJJP4flUo8ySOUh42VV3DkNhTxx9SentUrMkoa4luVkkYENGw+Zs8blxz68H0qLlECSgypkK4AZMKoI9SMd+/SpFVnk3s2UaXc8me5wAcfp1qa2MMXyO3yKvEe4pvweCCcgYHc+9VDLbm83LIrQscA7gSwJOOT/ng0gLWM2iKibvm3LJt+8T/AAn6Y4607DGaQRywQOrEjzMAqeG29xz/AEqu82GIR1lVV3btuQre3P6+9TgwjEhXchxuB/iY9xz1474piIpBKpljeLhS6K5jJAJ5wNp7+vSlIWSRi7tt39kBZ8qflBHb/PNMImVSREp2hV3puGBnG0D/ABqR1GWaOEgyDlY+in0+bnn/ABpgNtLeWZZAXhkO7hScEE9+nNOlhk3EMI1MmBHHEynaR246D1zz+VLJNKbVQNo+cAkc5x0xz1qB1O8SBd7yN80kxBwcYwM9D780rjsOVnVzbzQxpFjDCDCuCPc9R7Y70/euI0Fusn+1uzlc5GRjjgdvakttx+RcqPl2jZnJ6dO/U8e9PWK4KJHGofBKoABhsnnIJ46UXEQLKZpt6XDANkFJkzH9fp6d81IJN0pf7RGdrkKd/wApIHP3h/OmgwSS/M0jllwd52c/Qf5xVZLKW4cxzBorWMFvKjQqG68lupB7UAXEDM/z/MC25kI3bfQ9R/hTlwPM2N5uejhcbvRuR0p4jhkj+0PiTlUjC4Gx8cbs8kY/ug1BKsixHY4jkQE7PmZlPTI7DFACLAZNkbsVQnJAzz75HP4mpMmO2WPzGfkllyyj2OOtC+fa24ZrNgAT+9lY88c8A8/Q8VTa9vJJpI7eBlctw5jwfwP09OKALU8048xnwzMdufL3ZPscEf8A6qUQJuYTfK+M4I+8TyR/s5xTl8oQEGRi5O5jH1GOfp3p6MbmR0Zz8zBsyMPmxnOdvFAEYto5AS/7rd0CDnHb6j3J7U6dnEPlKvkIi/KflVsHsvrmp3bJ8xDHHJsCoqdG75I7flUG7Ym1/mZTuWQMCQOR19DQBIqG2mKW9znK/L5YbdvGfxPp2HtUSou7ZJIXRE3KAuTnHIP51Xa4tYcRJs8x/k24bG7OATn8+tW7eSOVTGDIkuwhgh3MzD1B/lQBJbefbDyS25du5Y84yR3K14t45tft91NcyqfOa+kt1hBwS3J/BQMfnXr6iIGXzYPKQrl2YYZvxPNZt14X0vVr+S/uYneaRfKWUOSi/LjIU8hjwu73NUhHic9jKZprSFMRxfPtToOBmq+j6LcOJ5XQxp5oQ7+McZBNen3ngDU1uDCk9nv27DIA5BVvXHHt9arv4V1e9nu0MsCfa/JtvNZ1G0LjdMo/4D04PNVcVji7GIjT7hwm6+mYQW2CMu2VAC/m3SvZ/D2mS6JodrpakyNEm2RztUK5+Yjr6nqfxqpp3hez0WNGsDDmQ7PMkbezYJGdxGMHHRa1xHJ5BWJBCy/KzuWxyfTrj/EVL1GiztyhRwqL5gLOTjAxj6HnvTAVuIz5O42y7QUIbO/OduevbqMUn2mSS1/eGF5Q2N5bjaOnrk+1PTyYFUzOxLHKZiLP09Aee9IYKwUsGi2NyRjKgjv7kDinyTRlUmaQyIilGByN3Q4+o9sUgurpmj3l4wq7gjEcDtuA46Hp9KYqmZJcPv8ALbJDxcrnvnuODjFACvPD5Bd5HjcxgKEK/wB3hcjp/OppC8jFVcN5saruJCrnAP3cDt3qpaadbs9uPLQXLjcm4/JIM9OeR7+uO1TJbfOzNGXd5PLTY/Cn69+h47UALI8bKYWQeU3zRRcfIp9vc022kuB1SZI2wEjG4Bzxge3HPXiqEs86gLFBIxj43lgvfjPXvmrBZz5gYTsQvyjH06jp68ZoEWYYt3ly7mUPlmOB0zxgHpkD6nFSqojEojePCqXYF2cLGPfkkn0qsIlYF5o42VVAlEnU+nRvu+9StC7XKCNJPMiba8S5znBIxzyOBgUxlaVoWmh+1NtWTG2I5Zz68L2qVJBiTzsxJn5dnv8AX+fXHvUck0ySlxFG8rBVeSRuc/MTgdsH04/WrERKxsl5FtlY/KgO7ecY57/xCkA1ZAslz5X2coIl5Lbt567frgc1BIlukTnzAp+Vn/iAA6fQZz09RUoePZM48vcY9xwvzSH09uQOcDjNSQt9oYEPEjNkvIX2g++R0xzn60wKrxebG8wa4SU8fLIGJU8sWA5yem31qaRo2idG2TAy73yRllUHkdzzipQsYZE3w792UjiHGSpyRnk9hycfjUcCmaVTNGYSP3mIA3zgrjaOcfQjryPegQsysbNzM0ojZvlZuWVsc4HuD+oxVdLaJZWbyg+xVVhK+0oepJPBJzwAOlTS3CTzNvErhXLbGbyz83fGPlPHT6VHLPNsD5cncQXBUDce7Hqx4xSGN8xZE8uKPyodqlmKkKFx1A4z/wDr9qsoyRNLDMjLE+MN5ZXYwHGwZ7889uKUXkUsz/ZmtS4wEnb5zHx18vpnOcEk1UithH5OwvcNjeshlA2nnPQDk9MfjSTux2LCrGY5fLRi6bWkcHBj9c+vBH86hu7lUUmG3uS0khRX2DDZOFy/rjv+HFD+ejK8pjLy5XYfmPJ5LE57fypwWNoha4QzKuFCuThs44HTdxyAOMDvTF1FNxJHfuYlXzlT5CsW4gr6c8gZPtTfs3kXTK7oRtARAuxt3QAjB474HvSzp5dilzBPEzK5RgyndjJHG77gHXaPU0QwPGFaS5MkY3SbY2wMZ2llH8R9jzQAkEUxVdkskvBLngbQQxI3dNx/kfwpHkiiheWMKJEdnzIpA5UBt2M9+4x0OalSCVZ0zbbCqkDcqt8wI9Txnpj1pljNIs7qw8rlQ6R7W4zx8v6FeenSgLEMUGX85SMIGfzD91vTv16Y7fyqR7cXNxLtMbuzbS+CVIADY46+/A+tNg85cGZDGV3AOpGegzwcheuCPfFWF8tpB5vmTFgU5UDPHyjp7Z4/woApxN9lbdvbzWI5HyblPr6fT1qVoYV3PJL5WHUq4JYBgeWzjB3fNwPpUb2dreupeD98oypCtGMcZIZjnAyTnnmrX2aFYZLeVMBCfJCOx8sfeyc+mOfrnvQBWlV2mkbY7C43MSVwrce5IAXjP+TRNIY5xK/7tkYCQY2gHGdxH8PHPI70/a6wpNGqpK3l7FA+XPvznj145xUskT/am3SnfBFnzE4b5uOeexDDnJ4oAWYqVZmdiysyl4nxuwOcc9Se/wCFVF+z27PILc5G5m+Qt8vTpgcEnr1NL5nmbA4kYiUht21tucAe/wDe+vNTR3MZfyEZzbh/LLo/b+ELnOB/QkfQAayjy5ftO4QrhxFFhf8Ax3+7936kc8iniQxSEqT8hzlnGc55K7cj+nTvzSXMxRwzRxgy8Elwvy/QAnPGccUwTIzeRb7Csm7Krh2ZsbvY4P8AnpQBO0IgcRfaAu4hZHT5uo6HPH3eM+5qFZIkbYyTAbWJjfKgvtzuMhPzDd6j+VQSia5iRrdDa4Xd5hl2lhjaOfcg9PSpJklEsYjSLzF4lMpwzMzd8+2OaAD5ojIxkKSL5bwz4wyNnAIBzxjjA601rWDyOHZzyBGy7twbb8xweOc9etXI4Jov3lx8kjbW3Bclc/c+UDK5OSPccVWmO0TNO7M8jrHGgAwu3nb/AI8GmBDc72X/AEe3EkjIGeQ8ALuOFI4/Klii3MyBTJIrIzxu33DglsnA/iOepqRbS4t5XG1xIh+XGUI/2lJPHpxz2p9xJ9qmV5JAYQdxEmAmPl6jg4ODyfegCLyWLolrIEuWOEAjUY5LZOc89xz3po8s2/zOfNjQl/7qnOORnt/OnnISNdrQthkZj1znqeOOpxx60PGluxWRfNT7gfLBmJ7Adu3t+BoAlkif7UHDPKNnltG4PljqDk4HB9/Q1CkMVoD8/kJkjYm3J5wcDcQOvXH0pIo0Rd3nG4VyfmHIP+zkk47k/XtxUEjF7lnhRQkeArojYxnIPTOScYzQIkaQQuX+TymyJGztbbkjJPr1H94Z96bAsTtHDGcFsxuE9j9efTHTipy8sdzFFHvEhzgKmAWOGIIbj/8AVUQl/ceUrNgnflWLMox0Htjg+nrRcGPRZFRQs0RUqW2FtoZhkY5BXOMrxzyOtDGV5yu4rCflx2PPykk9eg5OO/Slt5JlmYow8tMo5SInG4dScY/D60kvlRgSPN5hWQCNzuycgncB0xx6dfSgCwJYsRICjLH8xjMh28c/dI4/ofWoZ/LlhZoy20oA0kWBhuoGe/PY0sbstojQtOIpSN6snG7G0/8AAlznGOn0zVKS6tlvIUKzB3KvE8cnlpgdd+PUc59u/NAFsxIsjJIZZfMClVldf1+XPHX/APXUlxDJFqEcc3kNH93ZuHG4EY4/E0FEeM7WaSFYyIjtVhHxxgZ4A9Qeo/CpGXybSNZnhJX7oAJ2ng9cZz1bLdRxnNAEYk+UytFGQfleM/MZBkfMx6YGOmfxomEU0JRJFRvNHyMSFxz+Xr3/AK0kpeJJZHkLIw5EbEEZBIyvpjBx7jr0pINRtxDIfOia2ymGjiKKEK7QRx33dOOfrRqA5IoV2Otv50YRgzxLtZmYdcH72PlwOn40SrcN5Xl7MR+ZtVdwTjG8gfwjp1z1IqdGjE+xE42YACkoecYYr/Cx6c4FV4iUimiSU57RuocY4yQecMMY/HrQFhywPHK/nO3mW7YXOF2FueQRxjdnPTp1pFSLdsJZv74XOOeccAgcDjPP6U2MMh8ppWUNnCIQPl68fNnA469+lSnzIvMtjOD5fQxvkqepLevTI4H/AAGgBolmjmGySdXfKoo444GMjJ9PWoF8tbQLLLIgETER+YHQMHJzt+vPvnoKUyHywIw0kM0eVE3Oc9Spxxk+vJ5phdZUhaFlRlJCg7S+wHg5/j68k+4oAoxtfwhCUM6ModQUIHc9FxwMZz6VqQQvFeRSCGSMD5mO9mGD1O75cjrxznNV2tgMueAG2u4ypOf5cfnzUkczBreMNGqr8ys0e7aDyfoBz+R9qAG/Zktps20H3WDxTh125x35wD3wOnHpT2Hn28Ue6ZIo5MrlW2/3ux25G45yO4OKeYHJCxKrNMrZLAfIvGRjnI47jPvUq3EkUBZzLvb5WfaSH24w2CDk/XtigCOaAvOSpeTaxyuMKH6j5T156Hb1471Z+3XP9nRWaX0qRpIYwI8LuXaMDcuCc5/TmqlxvkdpVVWDneQM5RcnByRwBwfxFQrG1xJHFb2wdlXIUN94g53A9NwA7ge3WgZYEmQ1y584ooVgoUNnbgluCD1/z1pqsjboPOmO3aoLKS4JHXJ79Rg96Rorg/M8agHMiJnBHRmx3xg/l9KjW6adViLuQqny/McKFH3mz3PHQE/zpgywBH5DFonD7QVfpkH7px9TSsott5laTzlH7squ5VT0wT36VGHuEDxNMkaMVbc6lcEDrt6/TtyahSJd6QtNCIWbGxCccdB8w9SR1xUAGwHEcc2FRc/MDh/9nj+lTRp5NqQiR5X5WKKPlwSVAJ5HGTinIsETHdbbMqNsX3Qo5+vGc8jmnRiNYt/ByflGQwHy8nP97p19KBjyEjjSR1m2OfvMquARzj03Yz/9eo8yeYfJjkjjP/LZ/lLEA5xjt/hTYojKjTBFyNzMq/KBnHOTx+frTmiNyreSibh8uEJfd/vbe/XkntigRE0tuFRHT94EAPmBhhe304NC3QuJHZCGfaSWRMZxgYGOD9acbaIWwnJL4wrB0YHjkgcY7jrSK4YxmOOFGjOMyqgRfT8RjrjOTTASFXLebIECpkAJGuR6k8dvz+lSTrC05eIvORuSEMmwnuSPbrjNBaW4kkcFnMi7mdMDcB64FS26CaGNYvmTYTIHHEYH6dD9eTSuPUlhaGV0jllePCKMnGwf1b6cdOtMj82G3yj/AGpfvNyGHHAGByB7frUv2omGJFkeNBJtKvEu0cY5H3vT2qAMY7Xa88jIAxB4OB/un1z26UAVVnkaaMrG0aYziSPacA9sc9fWpg4Z2wTGVT7pUYPGNpOemPalfASWNh5KquHD7Q2RkfL36fWmxwMqyRRq5G/lR1Y4yMHnP0pgJDLOdzwypGp+UbH8sKcc4+g7il8wkjYjZI8tNj55Hv8ArirCmOOIR4cOmdsbYcMv90gd844FVrfzpQ0kh8uMZ+RVHmHjPtjP5UhErXEjtCEjnecqd0jncNvp1z29KsBIYCzSWxUswyY2Aw23knr7cdqhhmUrGkhVlzu2nLc/3voKa0EVwV3ys6Fd3zMFVf8A6/pSSsU3cib7VLIA8MccDIZEUEnbzwTgVPBEVVXYxtMo58tiSeevP9aktY1nkWOF8hg2QxVV3dvc5HfFRwwBplWMyttPdflI7jd/kUxCxjCnMgBkO3ueveoZZzcS58yNn/hXZxnjqOafcKpRY2lji2qEzy54J6A/0qJrQFWI2Mow6usZDtzznmgQkgcM8jTREIwRg/BB9Mc981PbhpGfLxoi/wARO38e7etCxwx7JJYElUS/J5hyQvGR14pLg74i63DwRD7kbSL0P0A4xigLCCRLd0by/MBwFMbnPHVR7VLbkRjDLH5rHDGLB2Z5GT0J46cVPbvsVSreYzIV+YZ3fgelRM2YlWVGaPdwjDcM9l9ifWi4AgKSl1SEM6lchiBxyAccY/EninQurQpEVjQkksrouMHuW9v61DPEcsk8exG+ZEV87d3eoopZV3wbkVg207Nqn2P144oQFxmSS1Ito1PzIgQ9T83POOe3HpUBaEQOJ5GlVnZy7FCfTHpjocD86iUpbrIqIUDKcRfKTkjqzN0PHaksVi8rdMh2Fd37t2L/AIHGB70wLKSi/QW3yG3VSxDAoMD0Hb2FVoYINplhcKyDaS+7eT6Ke469qvHb9oEZlMgQ+YskkfU7e3fGT1HpU4VJpfntJZ3SNhuY5K8cfT6Z5oEQosccPnJEGh8tSA7Lye/1P09+lRM0kcxFs07rJhvNb5u3Y578imzbiPNaEOqgcOoVs8d/XNQXEEIu3ZJJGMmFWRAVC92we/PegZJKGllnjS2hMQU+ZvcOcfn1GD0qaVG81fOYbpFzIiSbWVsY2kcdvSq05t5Y4EO553baCDjdgfKcdP8AgWck1OJY0dfOfK4wXwGcdeSD356+1AFho7OIRySbIo2mEgj5ZgO27noRj8qplssJGZnkXMjAbW+Xr1HA47U5JY5V2MjBUVnyuRkdyx7UsRK27tHcnchPkSNxnPbHUccY7+wpiJGiTdPDJgSSIxjBK4VjtLEg+3fNJFIw8z97M6Jg7QyrnsfXgevvUkTg3RwIQ8kRSND0Ruv8Q5HvnvTSxihUTeUZVG5wVwqe3GfT9aQxJsO6QCcLcOoblQqj0z68Z6YplqJBbIrI5jf96xkI+c8EfUZHI9/rUSoTHL++2ZQKWU4JPHKjbnk09W3xtbQuSW4IK8ejbiTgjAz14oQC26kSqyW29uDJGu5iqnp8pHqQfwpWVp4vOTesY/gRfukng+nLY9frVOaUx7PtOC8hLH5zI7fXHHb9a0IG+1WsRa4jZFXmF18w/N9Dx09OPen0BEkccFqwhF/83Ctvi+/zzt9PvYwcVXkDxpNBsjjC+jOV2rwWAbp9fbFPjSMXDT/ZpJtzsRG8YYEnpj0Dc9OmKiRisAgTbLKSpLIVby067TnnGTn8KAsKY/MwrQiNt5jy784yD1zwBuXA/pTDHMkZVYiJC5OxgCvYfNxjPA9amYBYizLGpRpA6ZLMuMZLcc9cnGOAo7VWEr2zxTeUnmI+SSpG5SBt2nqcEN9ODQFivb3rTRyxTWgUl2ZpAnJPQhif89auy3MoZfLKPeEAx/LjaNvBI75/PNVHgiMUBilnZpOVK8BWH8TD6d/p71JKn+hDzXPzS7QJMYIHPI65Pr7UkBDD5k1sGDYC8Oobo5/Dp16d+tSb1lJZ4wsSjqg3s3I5bnjJx836Uzzbp5XOxBErAKCu4Z5wFI6Dn9adIwmuIA/75g+ZWST73PJ2jG7k+vfHejqBNcS3kcr27K8KBgjyjlT09s+nNRwuEkkhllkTy428xol5JHb5enYZ6mkeKd9zJIm1RwjHaTjJ+79BnmpIJntg0tt8jOMnb94ZzhvYdR3NACNdJLcO022Dcp2xKu0p32ZyQSSOf901XSYt5lyZ/MCqrmI/LCpxxzj2HSrQe4igWPy5UkBVQhIJ8sliQu7kbm6moViEOJXlkJjU5B4xzz6kkf40agQQXbMI/Mi3KyFiCMmXvgHOfbP4d6nt0Z5wvm/vE3OokY7mJxtAPbr09jUk0cy7iSQYztIAU7vlVQxHqQRjB/Cny+Tat5fns7LMVYnqp/oOP07dwBqEXELp50vls7bMk8YwFP8Au4b/AOtTysha4eV/MaU7AfNGCSRwMZwTz14pmZLeESSMvnu+IwsZc4GduePr75p8IlZOI5I2ViVyvzIWJ/LrzxnkdqAGJEsOxZVcPcJ1HGFI4zz/ALPp3qusd5FFG0Ilf5/uBvkV8YG1fbn8/apisCyNMoXcrlhjnbyccdznvnpxTbAzx3TmZWypUSrLFtXdnlWJ9Bk5FMBz2xLFJIUA5VXIHGCOM846/nxUi2cNuzebI+4gqqh1PybtoJIP17dxUcwju2hgt1M0DrsRAQSFGM+/PHP+BqO00+3F1kQNEnKujjJX5mHOOenGMfSlqBaEczTKIlTc5YGRYtxHZs4684JxjOc+tQy28jyB1mUbP9ZK2wEKOM+2T+ODS38LxJL5EkaksrMpJ+YHOCcscYxTUvA9ilnPPCg2AMTuO4n1HA69fpRsA5mbYgEqyOCQ2RtbggY/T8hUMkryz3IV4mnMolDEZZDgnb+RzgccZ4qZFeVkSYSOXyZM/Nlgw3bicnP3uOQOtPYwpHIrvht7B9vIGF6bs/MxBwPb3phZEUMjjyy0TDcxRBsJ/h3bvTp03Z6+9NilKpcERJGwLI4XcRjno3UD5gMeo61AZWVIrXcqlW3lg/XLZxnnt/Fxx2p4M0TTOCnzTbVh3A5OcDC8ZHIOTjGCaBEoW2ulWW2cTyY+YPGSY+DkLk/7uDnP0qCPmzLqRHKp8vzGIZ8sODk9O4z/AIVLI8fBcW6yRnaQspLL6ruPDc8ZOetO8vzFkISSR0iyuMAAjPTPH3c44/CgZHHDDEu1J3Llt0hlwxVR0+U89RwOfx6UCMoY4+fubpEDgjAA5HOQO/HPPbpTmkazVHmVovMK7W3bnkUfMyjjjnPJ7n3oWVZ2/dqT5wOPkMZBzzkdB+ff60BYigDGxZ0KRvFk71b5gME8YA3Z5GeuOPekgRR+/EbtKx+VNzNjIJ45/L6VK0ANjF5ltGqhceWVxuxj39R04P4VAjolpIkTkRMx2BEy2OvXv1x9KALIwUkfz3ilZA6md3KKOeBxk9+PfFKZDKjMZgqShQzmPIJ5baRxg89/enrIrQRMscUMccW5o3U5ZgOSec4PykHGOvtR5Ssdu9UdZD8pZWLKy/LuJOWOc9M/qaQhzXUMcaOquTGCvmq+BJjGN34cnp6dKpMBPOAJ4TDJGH2E7Bt68H/ZbPH881PczGO3Rp7pAEOxUJwjDb90Z65wTn1z61Hbsrxi4+zwN5UjFxHCCucH+DsvAzt4z2pgSIWnj2lojJkln3fICq4bJz2wPw/GmyeZdokflqJFYjMRIZuOT8owc/L8v0qbZtkJulO1idodRiQkfN8w7rn6USyxLbJ/rI5A4VpCdpJGPmHPVcdOKAK7RMzTYufMf5uZIx5bBemcNgt1wOOv40XMjbpHEY89SF2tlJNo24YdjwMY9MZ9aVriO5DwpcrIsB3SouHcjIDHOecfQ9abIFnnfc0kgwpZkj7cbep4b72cn19KLAWoLh2hWWBZIxI3lRDaqksRkqMdOAvy+9NdsQO0DQecsbLlGDbQG4xg8Hn9R6GlSJ/Jfy9picM0SM3mbznapzjGfoecetRzfYwIlMdwyxjfLIWwCDy2VH6UAJNLc2yEQhDIdqqGmUbRuy2QuPlO7qSOtEiszNJGHf5yx/hZx7DjJwx4x71Iq+VeZDpHvUlXiJ2/iMZz3yfSoTDLIi/feVVO/btyueeuOpwGwDzjNACWuTGEz+9h+aPGGznndkYzxt4B+gqwyllZQzBkVU8pYx8vQ5XH41QudOmuXEiea+1A3759se05K7SM5Pze3THYVZhQ3VtbvDFtYt/qm+VkzyzY67Vwc5PGaAGhPtG+V0kmjkzyG2t3+UbevQdQMAYoWOMs0XmTyInAUSEKV/vAYz90A9O+RU9mgjPzsYfJYAs6llUHPy8EemevTioZ7jcZmz5amMHBfGU288/ljvQA9o38sPGFaIq0h8pSwjI+98v5c5+veoftAj3MVQLwRG4DA4Gdw6Y+ox04qcyvOUAThXJP7pmxzjlc4A+96cH3pHtyNqPOGaN9hkRtvr/3z931HpQAlvN++TmTzPMVXfrhePvAHPqT/jUVt5xgbygWmiVyy4VTgsAWBP8AwEYwc5qUpADGzvIN0TBjswVbOApOPmU9D60ohlZ1CmVcfP5eOzcjHf054HvQFyB3PnwKkLTM4XysyLGEKr/EGzgHjB74/CmG1aOBI5ZXLA7nYhtpIJ+bH8XPGP8AaGak3oxkklhtl85yu2VyRuPpk8bQfbmgROVyq7VjO5HST7xDDGO2O+Tz+dAD45HnuJ32jzYR+9Me35N2Mk47ewppAdnLxRqudiKzMyAfKck5HIz3HbiqqySyxu7xyA+YW8ohkcZ/IE5x3AOcCpxCHjCmBlAjVS21sfN9/dnqcflj3oA2X/5D0n+7H/6LWjUP+QVaf7sn9KKKkZUtv+PG1/3j/IVnab/yHrb/AK+l/pRRQMvL/qpv+uv+NZ2s/wDINj/65f0aiigRuxf8hG3/AOuU/wDKs6T/AI8bv/c/pRRQgRWvv+POy/3E/wDQRW5H/wAgzVP+uqf+hUUUhlB/+PSf/cP9Ks6j/B/1wh/9CFFFNDK03+tvf+uh/mKbb/6yP/fk/kaKKYFlP9SP+A/0qdf+PaD/AK6S/wDoIoopIRSi/wBVP9B/SrWof8elv/13P/oFFFIQtv0uf+veT+QqKH/kCj/rqn/oNFFMaLEn+uufq38jVfTun4r/ACoopiGr0b/Pc03U/wDkNP8A7yf+ixRRSQGjc/8AHqP+uH9BTPEH/HlY/VP/AEBqKKAMWy/5BTfT+gqW2/1o/wB9v/RdFFNCGW33pv8AdH/oNaUP/H1bfSP+T0UUMfQj1P8A5DQ/3/8A2oaml/489S/7Z/8AoQoopCKJ/wCQxD/12H/oZq/e/wCrf8P/AEEUUUwKtn/x72v+4v8A7NRb/wDIVH+7/Q0UUiixb/8AIw2//Xc1k3vRf+uh/wDQaKKoksJ/yGp/r/7LVq//AOPqf6yf+gGiikA646XX+8n/ALNVCP8A1b/7i/8AoVFFMCTVP+Q2v+4v/oNWJf8Al1/69x/6DRRQMsQ/8eL/AO//AEasjSv+Pu8/3P8ACiikBsW3/IIX6f8Asi1Tuv8Aj9P4/wDoVFFAFmb/AI/j9JP/AEJazL3/AJZ/7z/+hLRRQDL4/wCQCn/Xf/CoNM/1Uf8AvH/0ZRRQwF1f70f/AF9P/Okv/wDXR/8AXmf5miihAWJP+PO5+i/yqiv34P8Arov/AKMoopgjo9L/ANQn/XdP/QWqg/8AyOI/3YP/AEEUUUgI4Puy/wDX6n/oVJF/qG+tFFJjW4lr/wAfVp/wL/0E1W1X/VD6Rf8AoMtFFNCFtPv3n/XvH/7LVTV/+QJq/wD1wX+ZoopsBw/487P/AK5f+zmkk/5CN5/17j+UdFFICi//AB+w/Vf/AEbUup/el+k38zRRTA1JP9Ro3+9J/wChiqOrf625/wByT+bUUUCI7b/j+m/3JP8A0IVcH/IyP/vf+0hRRTBFa+/5FW2/661Lqv8Ay3/3k/8ARVFFICO1/wCQTN/12P8A6CtW9a+/p3/XOD/2SiihAMT/AFh/6+E/9DWq8H/HyfxoooYGpd/8i5a/73+Fc1e/8eGo/wC7/WiimA+7/wCPCf8A3V/9ArWH/HoP+u9x/JaKKQGddf8AIZtv916ki/5DR/33/rRRTQM1j9x/+vr/ANmNUh/qIP8Ar5T/ANDeiikAzUfvaj/1+/8AszVdh6yf9fK0UUAQ6V/yHV/64yf+ixWhL/x+6V/12uf/AEKiigCtqf8AyA4P+Bf+gmoL7/j9n/66j/0BaKKAZNZf8jTd/wDXZf5rTT961/66L/6LFFFICC1/1Mv+7J/7NVfxD97/ALdbX+VFFMC2f+P1/wDr2krAT/j/ALr/AK6D/wBAoooA25P+PKH/AK5H/wBCWrkX/H4P+2//AKBHRRSQH//Z" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAJBAyADASIAAhEBAxEB/8QAHgAAAAcBAQEBAAAAAAAAAAAAAgMEBQYHCAEACQr/xABkEAABAwIFAgQEAwQEBRAGARUBAgMEBREABhITIQcxFCJBUQgjMmEVcYEzQpGhFiRSsQlDYoLBNDU2U3JzdHWDkrKztNHh8BclN2OTovEYJqPCxNNUdtInREVltcPiRoWUlfL/xAAcAQACAwEBAQEAAAAAAAAAAAABAgADBAUGBwj/xABEEQABAwIEAwUFBgQEBQQDAAABAAIDBBEFEiExBkFREyIyYXEHFIGRoUJSscHR8BUjkuEzYnLxFhc0U9IkQ4KissLi/9oADAMBAAIRAxEAPwCEmUnL9RckUWVIW7NkLWxEXFLUXQtCdGpvUNAPnuQTbym/GHONmKvoQKVMlNU4skrZQmSsltz3BXdYHGng+vPbDfPrKGazT6bVdCWjGRGLK4m29EWgamm2VaiHUKHGo6LWsRiM195dbg+KnvLiPF7aKUqbkNhwX4Gg3Qbdz25N+2MeI1cscroo9AeY5rJBE0gPO6lVNqlIrbaZ1WjpeneIdjh59zaS3cp87jgNrDsFm/HoABZt8BKqx8YikhEqXxF8Q8lLhsrzNnzDWSfMAo3PvyBiHUqr7MpUFMLcmaLtuo4U2QOSkXsbg8ix+2JrTpKKiG5iI7s1p4rbbYLCgGljzbaiOeDexvcj04xyZNOXxW5l9lGavXqeKXJyrPLpkMLWVnbGtKyouI54NgsmwPIBKOeMNMiJHrlJlx26y5IMN1vxKDFdUpji9yr6Da5Fgbjz8emFOaZ+aqZmFyTrKIkxoSXHQy0GObJd8oHNyoH/ADj98NSA+3VTU1VRuRDnjcafiIcYd3wNtTZUTzYm/mR7W7YLYyeajrDQJHT00fcS6uovtI2gAttzUlTgIsFWUPlkc6u49sTOk5dzRBcMwbGtpYdRDW4i5QHPMu9yFgpWLi97C6CCeY26WTBW/PrUR15t5yW2lxvS8ly43L8clWkAoJAvybHuvy/mFbqHXnA6NporZDaQ8WuOUqTceW1uf9GLJDJ4mJQn6n5gK6dUKDVIP9ZEozIy6kY0loBpwKDRLhSdHPDosdBN/YtUpymzqU+/VENvQ3loiBMdnwxbdDtnEtFCghZU2jgkWuQbC/LyvNsGnrciVqA1UIZS67GQ4SXGHHG7AC5uG1C3lNxa1u2GujSoNGj1GGxvrVEAUl9lpG0w4RYILbg/aaFAk24A4BvcDtHhmS6LWNvm5psTDblMT4sajypUGPGJWuSozJLUcEWPAJsCE29RbgYluQcu0CHSp1QdLrTgS4zJivoQ+EtLDY3t24AuAoEaAULAF+cMUGHX4ssSMvbEokMSnFa/KkLBUClKNJsOUrsLJvY9wcPkd6VFqrorFCRJqMZiTKk6WvDKShdtKw+gCzdy3dJ1kjjknGKZmlrqwJhzIQ/KQtL0Oa8iQtjama0+IRcqAbuAeQknWeU9ubC6h3M1NYosSFU4EiiuSWUCKl5ZcQ6wjUAQXLEA+W5sskWte2GjM8fNSJ0BkUxyUqoNNSikJcfDAICQkuuJAWCCASD5dOjum+G2nZ5qcymTqzPhvvSJTweVLakjwjZLzgU2ASPI624G0K4WCkeiubooczQd0C/IlWXaXRKvkynq8TLLUT5SYa5LiXGNw2W8wr99vWUEJPIBJ5KcAiVjNLNRj0zwTEGLHIcI2g2HXLcv3PDjnYX7+nvhp6cRpLFZRW4DzciHSGHS4+takB3l3aDiUKBNxqFibWuCOb4es2yvxGWYy5z7jcJTsdpyMFuBuQfMUhKAUbhNiQT+Z4JxcYssliUHd5KWquqG/Pekz4y30aBFbdauHXARdsKbug+T1JB4NjfCnLlQmU6ZUUzkIbajpbe8M28XGww4kKsL2JG3fub3J54wjhyjR8uOQ6jGfhpWXTIdeaS26tsBoF5rkocHzW+RcC5uBwSXT5mX/HNQZESXW4EiMwwx4ZlaJRYDalF3dITY7hA7lCQV8WuS+QkIZCutS3KO2mRBhonyGwtoBt1stAebZW8C2fI2eVuEi5ULBAw5ZkYkRH0y0Q59MWtuMiUhbjbzt1thSkNptdCBqBGrlQNwb3wglTp7KpEF6ZPqRRJCHkypA3gu/ms61pC1BFgHSgXIuAkd17NNdq0FmqVSZJLLzrkNqQ5tqkyC2Cos3uNsK0lN3F2SRYBY5wrQM4LFW7mksbMESPUFyKtTYzyoUlxTXiQppx0um1nb3W4EAWAV+zv3sAcPvTCu1tl+rKW9SoLFQWEoKIgUSgK+a3wkgoLYsQbcHkHi0LkUunt5iZqC5ctKHJNzCkoblL120tRJfzErQsjgkAkCxAPGJR0xiyKZ+Mx6nSZj7LTTby6fDk7LrQX3NlkLdI1BBtZYAusAG+O5Gau/axOs4fRYj2LmWeFMzUKjKaVllVaQ/Gix106Gpbi0AIfLZO2921lLe2b9hew4xJ4cmPn52oQE1t5liBFS2mpRqgpOkOG7rjrF2fMNOyHFBS0lTgBuq4q+GtxvLUOmMR7NIJMr+v74J4S2XErvtnkeYEIufW2JbRq5JqVIYeok1xUdmShtNN2GksyXkLQ4JAL5LYWbaBe13COSjt6XhysBkLZnZr73J+P6rkYrC7IHQtAsjqplZ6iVkRKrlmRVafNcGhaENuoD6xbW4olm7Y85cNt1OhsjV3KRGzScvzKXmalOVSLIZW07suGU4yvShtp1xSyle07ZQDjTgKQogmxOJNXc1VKsQ57kOr5dTPgSn1QYUt2TGefQ0nbtIbZU4i6pdtspOgNtlZ9DiJQAmZVJefaxS4NUoMl112Ql1obTsRz5haiLbJc3AdaVnztOFSAdsgg7q2ihmkEkV2tB5kEW6kBU0tW+GI59SonLpYFczTleqqkP06JFE6I89YobQ+2lLN3iG13aXqQCpF+V3uecS3pRAyfKoKMgZ3nwqdBdWZdMraGv6zEmIIKLOhKloNiSeD5AoEWGGOaiRWaREMJqGavVKeiNUpNYZKWJBbu0pLTwcO044W23FuEklfOkhROI9lyq1BFFm0mpOut1SkVAvxdDZ3orgJSptdyC0QD5CLhzUbni58TXwPpJnGM6XuCF6ihqWyxa9Nbq7p7+W6DDezDOlKl7TriXWY09BCHW73W2sq+VqCddxcDXYn1xnzPLMJ7NeZa3mBEMh+a+48ku+LW+04dxp5orF1jbLafKOQdYsAL2JkZyS7mEPnMsFTNZQHn3Q4N6LLOktNusafmIUQEki2ogEcqOK36v1WEM/wBbYrrLEuctFPeSlLSmNuQILN16Vp+WHEXBtbtY9gceixHEZMWwpkx0ykDT01J9fouNRUrKStc1mtwT8L7JFMyxTWcrSk0eov7kVnx0lcpp3Q+DbgKPZbfIAIs4FBYsQbz74WIl4NeqmytTkaRCpZ1LFnG3UvPNoSe7ZDukBX9twXuL2iNKrE9WTJlCbmNuRKfKQYS0RzubC3LqY5JK/qSkX4IFr9saS6UZBg9P8lxqPsMLmzf65VnUubyZEtYseTwUJRZsC1hY+98buBKN76h0wOjfzVXElRHFS9lzf+yoVnjMdLrYYptXlOxqeuOHYi3lmJVqbVADw60BdtaRbQ6m6FIeUk6wDplWWY2cpmbqjUprJpsRptb0+IqQVtOVBxGrU20FbegtrDq3bi61LBSSnDXVsp50plfTOyjGp9Pqb8x+TCmPKM6O4gagWnkOtFDCtlwuIWoHVpWzq17a8SSqzqjTptTqcqm09OXKNphyIzrRDtSZCUuuSGW1+QiOSrQkD5qN5AHCL+qpqerkqHz17bWOnSwB352XAkfFFCyOm1ve/X4KPSaPDkdRpFCrSlz4s2hGLFYXLP8AWg2pLLzaj/iyreCUBABGw75hYnDLHl5ZpNZq+Z6lAdqiqVGbYEGG0iQIiHNy7pKxodWG9u5UeEKbXdQeNiepeX60uexMi+EkU6GioQBLajl2wcabO0ZIG24XA4A0PJ8xlwnkm6TJtNqlbhoaoVMFJbarAjtIpLDceTRY0eJtueM3ASovuFhWkLu4SXSSAG8cyWdtbVOp42Nc4EkHfcarf2TooGySuIFrfVSl6DBqtNRIoxV/X6iKs8UwQmIXX2UMKQ0LoAGgBa0t6QAxuXFxrZYkikTqgqkqy5TU+DlbLb7rFm3HGnHQSw8Bra0suvOFpPCwB9XIxK2spw6bDo+VH5TyZC4EpifJizFAR0LDOoocdBWdTpABcutYN3DZAs19SKVWallxZlqiSqnSJkF1lFlxDUmEBwO7Nv7W4onbWPmNkci1xjHD75IfeHWLmDRvI2GxS0mLAPMTCbHmm+JluSxKoNVpVVlV3LATNUYrTzReVE2lJLLbzlkEMuAgtnbJCQQTZYMry9W8srzEih1WVPmtIcZcgwwVsJ+X/wDo2mQ3odkHUS4226u5aBBaJxH+m9WocCmUlmjVkQZTNSLrXhg4htyPtkOoeaXpu4DcLLgQslWsa14l66MqUipSo81Lsx5mO2Ictxx2C06htshaW1+doKF1XSdd3ybgmwrfw8aynjiphlGjrg2AJ31GunIao/xVsRd2+vLXn5prg0tunyZzeYKPHpjq5KnnZrdNDLDJulTbiZIUSLrADm5dC3Qs6rquWnNORIL7FWlQfEMfiNQpshDDDojRoUncbbEhtTSQ4Sq5J1Lv8w8LOkiUVGLW33abLp0NE5Ta/FN0ypVNuK622sEPtKLaS3JQ0gtgBQIHJus8lxflGO0wa5Ig055xlE2KtKkPsBsrKG3mniAD6g3DZBui3qezHhTJonUNSSb7E7uuLE32+iwT1xZIKiL+yjmW8oJpdTbqCavLbrj9PLUlRG+w4sOWU9ZadBLZ+gaxcElYNr4gWZpVWg1NiDV2G6hB3HZslMZlbcZgIQppTh8QU6FtBxTgW2QizoWdaAEYtppiDUW1TYceDVkrZkRDIQ62porcKRtna8hDnIKSddwPtivs75WpVZqsWj1VcmmM1kLhiTKdDim57mlLCtrshFkrI5tcG4NrHkYrgtLRsM0DAHR3ANzz3uNrkc+a34dWTTy2kNweSghq1Ti02bPpOXFLlwZFIizXDGa2mjtJO85uj5SnwhNxoBLS3F2F8SGh1mRHpcfKGbqG0tmlupg0ZxmNOdTPcQt1SXoHLUgOJWU7Z4WUMJWknSLDyy/l6FS15skuKR4yqqkKXS4zmtcKoIMd5pLiuZLZiKQ9tnbcD8VWkXdBKljNlEfpTc+c/T4MqnNteOYS04UuTS260IzTem7m4YrjqHNP73dIUsL81UYaKmmZPBLltyvb1XbNS1k5Egv9VAHpbEZ+RTRIdlLo8t8RlPFkOvhsyHUqeeQQ048pZIWLIKkcAjuUUmsPVfL0VqLVZ0aoMSAVxg26Wmw462z/AFd3UW+DYrJIQN4jn6MPGa00uHmbNdHQ9DSha0OpnIbv/UnI13HVc3a2nbXSOElIBKLG/KUuGVSsyORY9UMMGNJZZZedalx3EFttW5ctOOByykKV8u7Y4R5TjiSYfK+YtlNzfQ6n4raHjsw8bW2RuXnpmZpKIyI0ORKbgIo8GOmlK3I77CVGElw3CFui6mG3Lr0IcacWCDbCrMWUJ9XzMmk5fo8irynJLUJUdmfvvNvhO4Wjy2iM4L2sSUDzoJJ0APeW6k7QctN0+gPohFMBifJRH0OKacERuIp7acW3rDjrDjfB+WXSQAAML8qSqVmWk1OOGn6gzVWHQPGguNtxg8S0mVMskLeStSlBOu5WCQgoJWRPBTTzCAENNtze1xdB0j2R5rXUZos6JSK+w6aksPrDtMnShJkRWHgEltgNhtu129KNs7OguC4tzfkmaunZwkJTLkTZMqoR24K1aUIAulxmCbpJbKUNsWTrI0F0L27Ww31WgQo2XYUl+eJwiVGbBVAWN2RI2yp4z2NGkEBGklLbi1laXVgWtblIcIVGokKQ9McqMoBcxEQhzdcdc8NthtJWbo0KWm6rgqDgKDzwmzyU7iSTp+PULSWZt1bWTM0tM1KXluFTDNq6qYhyKJURbLTk1tp5p0MbZOysgBOkICAFJ0lWk3PyxJzH1OpNSjQatHjByINctUF+M806G3VyW3RG86UNC3LNneUkhAIGGzLjEd7MzsAs0pxvwrdPo0pyUS886gNFxhoBSkBxgs7wFyUgrGo2cOHCiTKpTq1FqOW6a9W5amX6bJzAuoNMuU1i+rbkKbGh1sOcOJs44UEEGwIHaw6lrJIRP2uUO03PevpqB0voVzawxyOy5bkJjVm1yhCmmbDkPRVDw9UadaecSqMAlCZbIIJdaDYcLjxJN1IC2/NgUmXH6gUF9eU8tMmcpswpM2ZEEeFFU1cbJW42VyHLK+WgN3aChqLX0GTilUmpsz8t1nMtQnTn3X5btNROdap6GpB3ElEYOkvFtfrJKwNSLJRxh0ptKp9EpsSkwGC3FhMiOylR1K0Dkkq/fWSSpajySST3x9UwzB6ydhp612aC1rO3uNL9PO/VeQrq6mj70As/ryTHl/J8SlsRXqk87UqgwNYW+4VR4jqx8zwrH7NoemqxcsD5gDYOZpFNVJlTFwWHXZ0diNI3EakuNtFRbQWzcGxcVzb29hhy0A6Am2pxWlI/tH0A+57Ae+OLZUhWlVwfY8f+eCD+uPWU1PT0zBTxDRq4ks9RMe0kO6TkX8yr3Pc45ovg7buMcKbY2WWS6J0Y8UqF7d8HaT/Zx4pviFQIgIUfqTjhb5wo0nHNH2wqKJ2//IxzR6YOKCMdDfHOImsk+2nHgj3weUY7t/5OIpdJtFvpuceCMH6MdCCfXEQa1Jg0oH7Y6W8H6TjhQr+1xiJkRY49pJ9MHhN8eKAPzxFERpOPbVjg7RzjoTziKItKP7Pf3wII7YMCSPpwPTyMRAokpTpJPFvXHkNhCdZ4V3Pt/wDQMHCPvOAaCoNm/Hv7nCYy4rzio8crllsAuojILp57AkeQfe6x6e+Ih4fCjGm1KO6tP+9gi1h/44PShbhshBUfYC+C/wD1i6ShKGIqf7Th33L/AGQghH8VnAVUll9ITUXX54/sPqs0D9mkWb/iD+eImQfxOCHVMMvGU8O7ERBeUPsdHCD/ALsjAVuVd1QDMSNCBHeQrxDp++22QgW+7h/LDnpQhpLKEBDSLBDaAEoH5JHGPNsq1X03Ur29PtieaibhRo7qdFSkyKj7pkr+Tf8A3lsIb/K4OHBDSUIDTaEIQOAlIAAH2A4wdtqXfQBcHHQgX73wuZEiRF6L4MDSSNRTdNiePsL/AOjBzUcuLQjbILt7EiyexPf72sPubYPliJNSiNAacK/2bzuiyGgtaQQAf2hJTYX5F/S/PGxLGI6QBjdSV1qHDH1N5JNgkK9xOi7O2+ttCig/4sqAJv8Ale2Ar8LFYclS5DMZhttbz8iQ4EoabQCpTjij9CUgEk+gGFQiaVaENlS/7KUXJ/h3xE20sdTaglqLeXlGlzA2gpUCzmGpNOfSg9nIMd1PP7j8hFuW2SVb46gRxgE96yoNMZJC8+Afv5odEiP5kmMZ4rbLkSDHbccy/TpTZQqMwpshypSWz2fdbvpSeWY5t+0cWoGuTvFxk1V2G+9HqS/CwaekhL1U4KkMJJ/Zs2Bdfc7WHfbb+Yor8xireIVUT4mgRHEpnJS0X3axNWsbUNtN7uNhYuof41zSi+2lwl0psKYZLterZbXVJKC1obUFtQo9wfDNKHCrkAuOi24sC1kNtgXR2Lbg3VMp72umX7K9SKUqHuyp8hqZVZhQZctDZSghFwhlkH9nHaBIbT3uVKVdThwc6ncbdb0335CGre4ukH+QOFoQSQq+CGhrMMXIK3Fvq572CrfzcGLg2ypc7tO8jinXr8wHc3xSmV6LLrGas6RfGNMxznOuL0N691AMgNnSbWFwog88cWxeCk/XxawvigKVNnUrOOd6mzAnqZXmfMaA8HBtJkbjgaa0g9nLhV1D6wbHkjHiuORagb/q/Jer4O/6h/p+a0R0ecjOVGTVXrrjS80VSYopaKQltFQUllISTyA200Bb2vbnAGQVDW79avOSfUnk/wB+Dem8ZFLlN07WgNRJ0plCl+wISDb7eU/ngxLVtQ7W9PbHN9nurZz/AKfwK1cb5mmHL/mQNPB084QZiavCZivSFxZMt9tERsIGp9f9ga21ItYk61WQPe9sOgQjQAU3+1u+IrnylyjQ5Gh5uLDefaYbEV1tx1xZu7pdQ5ZAQ4tsNgecgdtJPPa42xB2HYRLI0A3sNfP81xOGqX3vEGhxtbXRZ8zQtlydBqMxcSdTZDlkriOtlgXauVtJI1tklJcB4BW2sWuDh/ht5czQ2+umtRYu6060FyVEqU6gFSTtoTdvVxaxJHP1WxVDEiMWJQeYWl7eLrbiHipLCCobmpFuyrDkEaLdj3xYuVcmyJGX/xnxLspaHS1Iiwp0cqTo5bAF/ODwoKB9SEc4+LSEOOy+itb1TLSqRFkyIMmWzLTObePmUgGM8OySDqBF+Rb3Hf0xJa25TA03RUUQxXWB4hkIgtHdXcBxtRRy4sCxHrwO2IlWIbtRkLZZQ3riNpZkuMSSuO84bq3U83bBBA21AEC17nnCjLkfM9ErzEioRJ7bzIbcCnXVNqbQDwtJ9bW9Li1r8HHMc27rvV2bLonWnI/Eprwlw6U+9bmYpvUHW+3zWSRr02Sbj7jEXlt1SLTfAzKaiKneKFtSWVpZe54KVBVm3AdAIT9h64ntLqUCSZdLkVSTLjR5DqWy4LtG9zewNyOeCeSLg9hguox3o8dyZDoaKfPmvNLFQizAmMFrbIuCQoo1NIU3Y3sscc4HbNY7RAtuqyfjMJXEzAwtbqN0NyUps8hwXTp9bE6L+trfliwcs0DLtVmPmg19cyqx0gxoFPbQGWgLuJXuCyASjULAk9j5+QI3VMvVaRMnsZkorc51grQ3VaeC9HqWu5QN2OG0JshwWVtA8AHkchouYM3ZO8PGXSW32nHmHTraCXpQ20lKC5pJ7X7oPO4CODh5nlzLMNii1tlKMyIgJqlPlxYC0yWUL0IZQC4wFtkpcbBFl3W2ojnvuEDnDZmiotrEp6upkU+koTIekGDGKWN8RmmWgpy/kN1J0KP7Q6gARgvMFTmTHTQ4qnG489KHHHHyp5wNuuKKWXeE+HIW2DpBNwlNyNRRha7lisVbKkVEt5uutoljxMKUNO+CTthx24JKUKSltZuEd/fFbe7q8qO0KZMhzB41FbgsOzBAQUOIguFLriALOHV3RcAmwRc8D74kuZlVSVmSm5myXnhtFJjtoakfislLDjQIKlNhs3Q5bUeQFFJ0CxIRhuj5Ky3lOc5B0OPs1VSJFPIQtTzCEE6hZxVkECxIUsXIR5gDfDHGmKy9OUuhyaxCnayduVJadWlxBBKym2i/APHta5sDgviZIc7SpmSikZlpH9HBS0VB5l0OBuI2p0HYbEdSXCpwmxNiSsBKNQSLm4uIdInM1art02VOYixqxFa1XjNtJU2bgOG1wFqDVxf9+x7dniLTYWZaPOiz5cNEunwnVxUmQVOIkrbcLTg7aCo/ukm4JBBvhC5Sl1CgQ69kxqTDbivsOuNOPNuNy0LaSWnCkp3CoOkt6eQLggosb3QtawlDOnCSaCM+1KorrblKgLLYZfpqNTDUfS2S2lk7i7AqCbEcN3NrWu802fRKfXHHmFoTFpMpKIzsd1SFa7qUVukAIeWDdojRYhQ4NkYhaqVmRdEpdThxKhGDC3IsKTtllIkawoJDhFrnXfVc2Hv2xLobOcTObo9ckQnY0gsQqq262qZe9txxxJA2VgKBW4EKRY345tDa4ddNmRmaGYLE2S1DdpbyIgjzDPd2y2XJBIT4dpADa17Q1BLg0BAvZRIRhmzDAcy3UIc5iHNgxpsbfjlbhDhbKip1TTpVrcGtxRCjZZBHcWOC6WHK3Qpjn4q+580tS0Pwg8+XARpUVggciwDjX0hLYWBwcOVBrjKEU2Gqoz3KegPtyGAovWjWJU22La0FJA8oIvyb2xpBFt0Hus1LqG5FlVJ6sGG/NpziDZ4J1FyQu+rXrG2g2/tWAIHqRey3KTSKUlunP0SLUYU9DElqRHfAksMA20KRbzuJcsQSeSRwASMVXQa+3R6Tt0qGkMzgBtzOyQF2K2rE25CTyeOwC+DhyrWYaqpRpVWXG8StovNymIqFLkbtyq6r6wFWJPcnkWHfFkFVHABlGv4emm6yvY5+hSirx6jMqK6/UcrMJdU83BhSWo4VdFyOHedCyRbyrFrEegwly7mtK6xOGYKnMTeKhKm2bCW822QUtt3SF6yR+0sSbWva5DTT63ISy9RtuO6xPiLppZVGDtwHA42R81sIINyFEHTYG49LMyFCqcCkVHOUSeipSYwQJzrkVZLaCbrZS4QEIcAIAda3ONCOAQMenwijjxGXPmPmuVWyvpY9r9E0waRVKwY1MptMaeq8R94IStlZZeYcbU74Ww4cXoCdBPILZANibLV5ZeTRKhRqjTJcaTKfQqmCBLcDLslbgCqetrSC05yVDULcO/WOcOsuhUWmqn5qptML71BlIdbpFRfeffeYVJU0yuK6w4kqceMhUYtBG5Z1Fk7iSFp5tTr9UqdUaqT6qTQqgpSJcVTbYfhvMakNMymkFTaHjttgWcUAs2JudGPVNwunw+DOG5ibjSy5TquaokaAbN/MKJUupQ6fTalRa2txLjm24h1gBRDb7fndG3qFgW9JQSFhejjAqhXa9LeYeaZLdHmL8X4VuS86yHFiyVBOka3bXOq3buTbCOHRKYqa1OpUyouKmzVyUpXF2izHuWn2/ONt15t1XzLhLei+g3IxNK9Q49PFbqJ8JVpEBpuVGrMN6O0wo7KdBb+aFouXXGxa/LQFhex8bNDM1ro5DlZ0BuT+a7XaREh41KgtHoj0ScajVqi45Q4clpuUhqam8t9Zuy22NOvW5rJPFx5T7EL8402lUiqRczQ1+PhJcNJqUOc88p+LL0OJbZeacbbWBykahcWTe57hlzRnun1ubUswux0aJbLYDK/lpcQAGwsbajoeAbN2iiwDl0Edi8yZtWzrl+JUK5ICpe9BQ5U6mreJYaQ4G0tpQQAVFV3S5Y9kBQBCMGKel/hz6aUd/cHmkcJu3bI06KOz0yq++qhSI6GnUrL6JzWplTbgTdvzIB7o1p5P9+I1VqNmTMOZGv6W7kifUmgl2qyElTzrbA8q1G99dikWPcFHpe1rrj1JmSqnrmOusuNbK0xArdSslI1oWgAuIAukjv5h3xFK+qLplMoYkM1F94mduvbq2V283l9CfL2txxjzkFVIwGM7dF1T94pV06yWvOGfKbS5cpRpDMVdUkKhultxmM00luO0FkGxU9tAHubLPHBxqdDBZZQ3rcUANAccWVE29ye+My9CcyxstVyqMVqsU+ImS20XXluFp1wI1EttKcSUAW0lfAJ0psSRbFoUWp5pXm1FfRWV1PLlSWFR5sFxl2FMZcCVstuI5daWgOWQQEAgHWQBYfYeEqmCloGhjbkk32XiuIIZqmpNyA0DRSKsdQKLRqzDoq4st55x8NT1pACae0tB0urVfQ4Cuwsk8BRJsbIMDzK8M4VeDmiZmV13JDD+00xPiiDGamoLyHmXbfPCwWmgVui93dsBtCkEn5ypdQ6lxozkHJSXpEScNvRV3I8OpUstLUy+1vsgtqQ46slKkJb1hS7q8pD/VMsV5iBLZnVjLNEy85SSiemO5JYaiNtgqISwUqC291QcuVIWkktgkFBL1lZU17ZL5TBfR17f72OhCy00ENE9h1EltlXuWpEpNM/Hp86SuhQlsRJUaLEceZiOfOSjcbaUH0IBCXG37A+bglTejFtZWlUuuUGBU0rjuzjHQiRJht6XNwpF3LoSCN0BDhBHKyTa/OK36gIotRyRW88vZjfzBQqO7EiRXGmVJlPQn1NlMYL8l3GCttGpwFFkuagFp4mnTehxMrERKYp9bD7RZn65odUZbb7jZfLKL7DbgcZIFyPM4eNPODhimloa1xkjDLjre58lsxiRtVSDI4mx+S5JefGb1ZfmIXUqLUmi6ltBjtzYTjrJC4zjLmlcgONghtIQSSUC4LYu3v5phRDKn02XKmZYorcP8VCaY/KbLbrRBeSpaitC2xfxCRZbYDZ2+L4nlRy7lt2uxszVelwl1Omt7TEiWsDYBc1AgOHQhdzYL7i9h3xTmYcwUrLWbsyqomcGVzkfJmRWGA8zNs5zEmMamw2pAcKESWjuJH9q9h0cYHuX82dxIzEjy56n1HPRY6B7as5IxqArHpuVMstVpurR6RDqk+pQrKlMuF7ehlDerZ82hccHb+W3bSFIcQPMrADWV5bfVAqcObOUpCyh1lbZeX4fbZKVtrKbuEKZN7+e97G5syUKDPzuddal2pU+EJ0aFTITbcJgOhTbLw4S62SFOJv8wEANrta2HZqSvLGZqYzU51bCKpWXqXS2akymcpTDewnebktpLjDami42gOHWB51nvjoUlbLLTNeyMx9b20+HMG6zz0zDMRK7NYfv5KVvRGlL2X223Q05qRqF7LHAIPofy+45wSuI6zKTUIjpQ+hCmwjkJcQV6i2bGyAV2OsC4NvQkFLlio12oNVBnMuXpFJkwpW00XX25DUhopBDjTrdtab37pBAsLXBwdCzJlydmKRlOLVESaxDbW6/EYBUphCFpDhcNtItuAEC51kC1uR1H1kbYxLKMpOgvvf0/Jc1tPKHmJmrR8k11ipUemyWKw0JYfYK2zJQy+3DjoKVKcdebRbdKkNbSOFkArvpuF4gOfqhl3MFNi1aHGp0yKlUaIV1aIuSsF91tvWUhe65paQ65uJRbQ6LkWSDOHnaZWoq8t5Py/EqElVY2FqlQXW4MV50qbcmvJ2tDo3gnWpJUsbus7Y5Nc9QxLy/TqK9U49PqFScm1EVVDLC3UsxA5tAFLKde0y846Gz5C4CB5yQUfMccrzDK9wdq4a6fCw5eq91hdLZjXW0G35/voo1T69Xq3LTltyC7Dn5flIfmPs7shbSGHlb6FOIKttoBsJbACgbEXIUCHLItedyBnGRPhLkRYiBJbZjoQG33Ij7RbjAnylDSlqNwOCA2i4BOJe3UKZl6ZVGZNOalQojaE7LkopkzFuSXdyWSfJoJetZywDibFKL3wyVd6jZznKqlckvw6PHiuymg1Kjy3wsMtlUUWsgOOBlz6eE3CPMBc+ZLHuY1jX5nDlyXYAGZ2mh5qKMbTDtIbiS0Ud+PRtNTRDlBSm7q23GGibrbUWluElRtoIbA5vhpZfjLjMVaLMp0aXKbcpCqfGBbS8HG76y0saw3tutOfOJWW3AhPCUkTStT4FNzPLjQ4yG00iTJW0tKW2HIbMstytpKUFxvWA6kC6NBbSoo54xFsyRagXpFWqdRdqYqb28HHnARFLriWwt1ToTdDrjnOk3bu0scWRjQysfRwmOUWP4XUMQkfnb/ul8elVOflSpSstzao3WxPdQzHYjtgTULW22tey2fkqbWFObrZcGtsXAXou51+bnWiU2ivtVpiHSzMfkpVHbap7seuMQQ07uNtMjW8kHSHVE+cL7LbXaCxI77D82lz5kudT4bS4TMlQWs7+4kBxoHgNq2twXssnRfVYgPNQreXqXSFSWIMKXOp86Q9TXYrSIkSRGDimw48WlDXqWHbAWsblGpskjnV08NULBtiOfn1VkTC25cbp8rDz+XC1l+bmRf8AR+oLiV2PVkobCGVhTkVtWyAULXrYPmTdg8lAuVrDZSnYMXMOWVUqgxoioTjH9TDS40VxDgTtzRoJDa23CsrDdgUNoAAIKC2w4ErNFTZpdYzVT48hO1CMiZrdSAhkqQQWzwyL23EgINysc3w+TImir0GLmQ1VijmMYTL8N5MR4uhFokkRrKG00NLbdiNxDNyd2+PP1FQcgbJrodeavaM2yHUc4SY2VahklM8uUhyN4aK+HJTLTsQuuhMgMNqLi1kAltry6tQ3AQeLHy1labR8u0uj0um0xwQppfaW7NcY4AZ25u62lRbCgUnZToWSVgq5IMepvSiBVGG6BSmqwme6xGgVeFNBaQ866lxSY7TThD5DumyEhtC0rDQLiiq5cqzWavlqbTmabnGE8p5UI0ykOtyAgbpSlKG92/KXNWtOoI4bCwSRjbg+NspGumy5wBYa2sTt66qqrpTPYZrXVgNSqS7T2pqK8h9vZLhQq+82rXZTez9ZN7dhc2NxdJwpbajMhqVKeT4dxgSgUEKJbNim6QbgEe9uDcX5xF6zBq8Kp0+oxY0aBDjvx92ifhzi5Dji1qadV8hN46Ek7lz/AGjuA+Q4k2TXRWW3X6PHRR6KlsNOPLk7SHyjS0pBbFiuwCShDjaSnbI5tj3+D8eSyEQVLxfQG4sdTqbjTT01XmK3hlnjiCVwWI5jstNPyXpW8hllEiK28l9xAcJWrZIWW2wQVuHQL2A+oXTyGlPNInJgustIS03ZQLrrOs7bSXNF7EgJFrfnawsWzRprNLeltsmZMltyBFiMtb7TnKlBk7bjdhpXqCr8r5NxwDStKBG/rC0JkNttxwZJJcugG1zblR1jgC4A55GO1htbH/Enuhlz2budhc8zf4ajdZKylJpQ17La8kmKVXtwMBKOcKDZXmTpIV2I7YCUn0GPozX3be9145wa26T6PvjgTg/RjhTz9OJdRFaPvgNlW9sHace0XwVEQE8/bHdFj9XGD9ke2OFvjlOIjqidCf7WOaEj3wcG1D3GPBCicRBE6Me0/ng0oucdSgf2cRMEQUXxwo7YUqSb9scUCO5wudFJwj3x4N3VbBq9DTZdfUhpv+2tYSP4nBAkod/1IxIkjjzIb0t/85ywP6XwyAQwg2OrHdJsVAeUd1eg/M9hgtTFSdIC5EeKj2ZQX3P/AIjgAH/wz+eCplGiPQJaXkOSXFRZAC5Ky+QS0oAhJ8gt34QPzxS9/ZtL+gKuYwPdluhiVGcUpMUrlqSrSpMVG7pPsVXCB3HdY4OOKTUFJ8iY8cq7A3fc/lpR/NeDqEiU5SGPxJSy/uWWp51N1LDaRwOLW09gLDsOMLNi6A9p8p4ufT7Y52G4rHiELZG7n8ei1V2HPo5XMPJN7lOilGiRvzD2AkLum/2bADY9+2FaEbbSGVcJTyBayR6k29MGJZUVa1XSL2/LBjiCWiXGvIULDgICk2sRz9rY3vma3Y6rPFC5zsuVFaUptc31KIx0It6W49TgUNt1JkpeSha0yC2hINxo0BQJPYfXc29SMHPx3UMLkMMmQphC3Q0m9nSBcDjn3/P9MY4cREoJAW6TDzEbFAab3FrGghSFben37c/z/lgYZQlxtsO7jiGEKcUnsCs3tx/uDgUFCtmzDgdu2HNfbdK+b39CTx7jBUVUv5xIWHTJfSgSSO6CEobSRcH1P5JPGMtRU5XtudP12WiKnDmEhqWI1pfdSWbIQALkWuv2/Mf6Rgp2OvTvNhOps/STYH/z/owNppcOKzTXAdaXECU/p3A4FoO2snuNRSq/qPXugk6fKjCKlpTi0rI1tjRpUbcE2F7BJtxe/IxwKzHv4fGSQS6/+2q7EGFNq3gXsEgrUpcKGUxFOvpWQwVsNBNrg8hR4Btfk9yOBfB0WSuE6tuPdxx2S28+4FlsDgKAbSeNGjgHt73WcH1gNvxJMn9i0gn5ndKwwdzkEgrBKQD27m3IGIg5VatJYjZSyClas1TIbbzlSqCdTFNjhKkb6r3DjuvUhlns66lVyW2lEc+N0RtUVZ0IuB8dF0JQ+Npig0SjMT07N9Xn5Ky8/NgRmlWr1WbT85kbYV+GMKQpJEhxCrvOA2Ybct+1dTYdVqVNplKagQXI9PoEOKY5fabCWBGaATtt25bjNgAFSeXCNpu+okpYdJpdLpkaiIW1Dy5S2ZLc2W/Mu3LQyN6Q888rRdouKLjzwuXVlQ1WBOFlCpTuapMbOFXjuJpjamJVEhOMlgv7dtme+zxbkhTLRA0+Vwi4aCOlTySTzOYDvoSeiy1MUcMQJ5cvNKaJSXnXI1XqEZUVLDZFMp6m9KoTbibLddH/AOFOi4XbhpB2kf4wl+Cb+nGDtlRPKT+Z9cCS2B9Khx7Y9nTtjjjDIzoF42qL3PzvG6Tr8ralJ9Af7jgLDVpCE34aj9reqyB/9rg+QlIZUlX79kX/ADNv9ODGW078hzTxrQ2L/ZIv/M4vuszTou6EpGo/nig8uuzXM+ZppUt9CWZGdqu62ogXdbjOyFONq7XF9mxPrbGgVDyK1arW/hjO+SkOOdTepjdVW2mMxnGomG64RaMtctwuXUfo1AJ81+1seK44/wCib/q/Jev4O1qHN8vzWg8gvvCrKpT7LTjf4tPddeJJWChxsJaA7WUb3v2Df3NliRyR/f3w15Fe38+z6VFhrUlmqy1h4n6XXHGlBpsXus7brbhPAAIFyeMPNkNKSp5aUJWsIKiCADf19hjk8BSsiiqXvNgCD9Ct/GbXSPp2tHX8kU40t1BQ0440o/vIAJ/S4Nvztx6YieaMrQq3JkSnoDsN5QXF8Y0+86+Et8+J2edYC1NNg2C03JRckAyCXmOjsTVICC+1A3HJF4266mwGl4J1AobSs2Ltli6gLCxxCKjMrbssPqk1SRD3N6QJQQwYLDqCREYS2FF79o26EcOEgXcbQmxzcZ8RUFbQmCG0h/C3MKrhzC6qlqu1k0VJVDI8VBk/g0mZU3I6160qZS247Htp0kBIRuBRSSE2sB3wx5Py7GqObopjU+ctcV9pxpyM3qDCEWN3G1hRcb5twb2uRzxhpkV7MjzZpKJTyXWg7Gfe3gtSHNepxRtcHVfue+o47Sa1Lpr7Ux+U5DVIKwp9sFA16zyQ3bQL35SOLX49fm7mPa3s3br1hLfsqwVZtpkXMsnK9VpkWKlLa1mX4YlL5WrzBxYVfW2fKFEW9Nu4C8PNGpdGDSkqktS1rAa1vlu6m0J0tJcsACsNnT+XB57V3R0N5kjKpzsmP44vOuNvLcKQ/wBrpUtYN+xN+5HubHEpZYmmP+FT2Vpkxm3fAuNBoKU6RqAB4WdQChcG/YaTcnGCSOxsESbqU01iPBkPtOUFmI06A4yXWthxnnlDzgUpDqLpIQu/lKibkEnCRlqlT6nMqFKjIYWA43NelWbKQ2dJS4lBUFouD6K45474SM57oEaO0iNTnEoltsFxEp0NpRo+q4Rcm67pJCx6XB8wDZ+J1zNFSdprIp0HbakeGZ2GymEShPzGw4TYqtckdwe1hilzDmVrE4VvxMSnx6lTZb0GjKjIfkxow1IuhYsQmws0LqVpHAJv2xX02VUqnMp1CaecEmGUMMuK3U7resltyxPJVyASbXJFxiYZfzVTpNPQup0g1ZDCAtx0eZ6K2gABbZPa+qyFG4UEkelwOoNZerVRYmXlUVEZx1lnw6gG3H0EEtEG5QCNI4+khC7EE3sY4s5JT3tktptEqxkSMtycuRn5dHjtrD0ZtDrTBWHEqb1X/ttqKLjWQki908pYSZ9Kl/hT0uRKcjSDuNuEyGQ1rSkEt+qCvbBPZN1n0OHOXPznSZrMLLz1NdakU/WwWgvceacktJ0N3SbEr7pURY6iFdxhFVai47UG5NZqZjTGnJAkojSUOnWT5jrAHKb2cSCAQOBcjFAc55N7I5Ujqsmf+LSoc5DaUSYyw8hTW/Z1CTdsNOKsgW8pU2f3bkG/MHzd+CN1R6rUwx49PqDSGHYbeqzz7SxyU3Va4IVYkIGnj2xcLsqLPgt1PU1FjLiIeeUHVuqQtADdg2Gzwe4sTYJVfjEMz1GptRyoufSMsCVKddLm89D1IbcCSHVJNhd0gjQVAge98PG8scGJXpho9TypDprNXcrEI5jicxCG1F1rbXdlR1kNo9QXL30WskkYIjJj5dZZqsXL8uOysa4/i2myndcKnOAOw+Z7I4sOMQ9WuO0nVGBZfb3dyQ38s82I1dl+hPtc4kaqY/VaBNDdLMuBFj+GddiPSQpht1zb3Ta4W0kqaIJ87ZuOxON/Zhvi5obqSRq7Sc6UU02nPCYpuK2ttvxJSXDa6kbdrA67ptySL8G+HupVwmEtnM9UkQ4lJpoMVpxTZQ84/tltkIcSootbSbeT1t2xW0Oqv5ZrbKKNLk+PeG2zKEkOMPsk3bLQHDqCtKkm9zdJF7YldArZdhNU6ruInWaCg8lbch4hDO2kWcIC0BbiOCedIHuMZJoBHsoH5VFRNrEuTPRJkR2abIKXAtTSWA4G3CUX0AAr5INuefbDNTn6jS5LXgpj8WcvzIJB7jsfP2IBB9j64mrCq1RpcmGuvzGYjLW20rwmt6LYhWhKVhSPKFKulJIF1AW9FWVqG1mCdJp1Slrelvu77UKWw6EAAqcDi1tBS0bg8pCQtu6hybBGOjTROncI27lI9+UF5URpTcOqPtx0S47brrjbL7Mn5KlOFSiCFIBCEcFXvxwOww4NT5lIrbrs8MtmP5ER3W91JBte39vj1BNrnnth6z5GRSlopXTJZlU7akSHVFlVmmy58tDT6AkFYHGkXCb9xqSgRzLsphylvQ6tGciLbQ42lDjRLLhuClwuHtwdN+eQL8jGiaAQuMTtwkY/OwPThRZCYVciSWEMKKyQGkpBbc1gp0ELPYhQBuRwe+JhlpvMdJzXBccgt1akxkyXNEdx7ZjNr8rymybEr+SE8oIGmxtwcV2ppmI9GcddcYC2C4yt+3IC9Klc8EAiyx29++JZRc4sV96nUjptkGTNre3Ha8bsLbU467uNKcD6yFx91agkAhH0glPIOOtgTmwVGc7+X5rDiLXyw6Kbza3k/P2ZKZUk5hTCNBiyJjzy2Yz0h6oOgxksqS62lpopTuqQfmWWptV0/XidVKhdPHo5j5qnRsxqhraCmYDRcfifN2WlPrjEDaCwQdxA7efXYHFSUDJGa8tZ3pD+YoM5NYdll0KpUSFWpqFlZ/rLzIeU2UJcKW7uhGoJKNskkg+sVXLmbE+CazRXKsxIG0amp+TGZYWQBp8NT4gJAKEhAcXYCyVEAY+gsxU08d6pmp2H70XnJ8N7V7eyfopbWqHEyrUZlPbhofhqaaptQSdQj7aF6gS+NRQ4ry2aF+De1r2jU2uRo8qqNymKYppDTEmmEKZZjNIcY20tNJKtxbgLIF0nWHAsm18QZc9WX0ycrRqi2ZaAtuoriRJT67klWh2QdWgm6krbN7lJufeSQ4dNreWUoenOKRoQt1pxxxlDiyNwuHWnzlVyDqRa+lfFiD4zF5o6iXMGZL7rtUMJgjyF2ayZ6h4KrUSM7WIMjTIKNbTcVK1Jas4Q6GwNFk3UNw+nfsCFtAnxIdWpMylzpkKJGltJlU5Ubd85vq2tak7+oJ1aFW48lyLYZHKqUPzIMfLUdM5lrfZ2oqGFOrQQAAgANrBHmOlAJQLo/aXJ8xt2pMTIbtY8emIA6ExS4tl8ghSWik+chJ8vPIKU+hOOQ4mB4LHLZla/dSisTjHdkSKLTo4myQXFy1huOwW3SpISoc7RNlWCQggpueCL1bU4FSiVjfhtR3Fa1tGQlGlIcAOpse6rDg9jbEpfaTAnORoG5TkNKJQlxsR7BdlaNsGw0kC3p973xxqvUaNSjBivMSn0aG1tOi4bA9jbgDkWvcXGOfNnEpPVaWNuyyaqfWJNBZVmRykmfHaIjPreLp2tGlSgQggci1yeAdB9MaNynOyHMbfqGTsv5lp9Ucha3VLjuRiTdIU3tOFLBLZ54ShB5IvqxXnT+qMZcyvPagwkyG5EuOl4vLcWpkOJLRIRq+YgoITbuDYknjEniUuNTKA8mqVWRKE9SKQ0w2h1rabjRrpQ4si7pOnyc6NCr+qce0wXE5KCL/0ova2a4tb0K4GJUoqHXkNuisOLVX5OcHKdGfbp0GLBbekMIioLclxxzglzygFGggcr+pfHY4XZgy49XkCO7XJcOMi2hmKNOo7jSiXSSdwWbdTt20WdubrSDiP5XiVd9cVuFX3GfGRfFvvJpDbwdW3tpSyp1ag4Vixct9AQ4tHCxcz9LZKRe1/XH0zDGtq6S8tiHakcvT0uvG1hkp6jubjmqQdomYOlOZqlUWHak9l+ql9hl2FTI8g6HQp1SX1LLaIZbW46Q7dCCEgaHTwGqsZBZoEbL83Icdiu0h1lMZMykvANqXsuEDaZUNd0trVfWEGxBSF2BubMuTkZmqdDnSMw1ynsUSQ5JXEp0wx0TSdFg8RzYaPT0Uscd8IuouZ6dTqbOh1GoSKfLXBdqMKp7QUzFmMEORrWVqLpdbuhpKTcNrvYXxyK/h2lkhe15ygG4N/D1t0HkV06bFXtcwsFydD5qos/O5rhvmlPV+U84JjdQnUmfMbhTlRn3VlKltBbqHQ0sgamyo2dAKbNgYnNPrPTfrTQhSatRKc5Ijnxaaa+4EIQ7pLW6xJaFw2T5dY0uABBW2LAYQZlz1BzUp+iVyCvLuX6vFfjqnM1WOllucUtOQ5yVNgPhsrbcSHBYpHkXyFhEwyZkeflmoVCdVkRarVZjSVTqvHiNRyHVrW4+1ttgENOOAP3JPneUBZIAHPwFs1S65eDH4XBwB1F7HUnfQq/EgyBt7EP3FvPU7aaJB0+cpMzJ1Ji5WhTqa7l5rwcmj1B4CdDWblxiRfspwguoXYNrCgoAAnQth5ukxc3ycrTmIpkzC3LpMbxfhVyIhbTuMuXSUbrZbfJVuG+kWToxJ0U6nomOVFECMmY+gNOyUspDziBayVuWusCwsD2txg0tJN/IPo08i50+3/0Y9dJTOdH2TXZeWn5dPIrgipbnz2vz+P6JnpNTgV2nqnQQ43tuuxpLDv7WLIbNnGXLcXBtyDYggg2OGGnZczxTa3Wq3TMz0xt+r1luexDcjamHGGm9tlt4WTdZQrSsj0bHJuTideQAvOqKiBxqF7AD3uAPT1GCYbxZlRKmYzRjx3Qs620rU0Sgp0qHKFm5A4vyb82xzsRp2VcXu813FguDqNduX2unJaaOR1O/tY7DPy30/RVFlXLFRotJy9mH+m9VkQqoyh5yjNNpJlS1tuF8OOgkbbbaXg58u6VpOg3VbDVmCtUee3BotCeqlWqVZlyIspNMkriyZUUXcYdbirSA3bQXgu4DhZUeUJIxb6J1MiR3FsUGI7WI82Q/VEQdx8Iff0l0Nk6kNDWEFbJCCsqWb+hrPMIo9ezFl/LuU3KhLXS36lDMWA6Ibwo60x3y0UupIADinIyA4NBCiO5tjy9fg1NLTufE0tI0ync6i9r338l6SlxGZsvZyEEHmPp8AmvOVRkQ6dKoETYksx5zbsGUuABMf8AEO7TaXFNC8omO2nWdomySspNgsNTBYk5fp2WMsy1yqtHjx2mlRWiXXDqKiWdbgQtooLdlpWEEBwrAxKq/Dz9SsmqmU2qvR67Gp0iqtpiuhyS6h0htTzVtliEVIDjCGW9wqccKEEFCjiL5jpVIZVGgs1inu15SW4stDLb0xMiSPCNbYJWC0Wh4pG659Jb9CRbzVVhLGNLoRpvqQPS2u/ULrx1ezJD8kty7HOR5ztDzY0+JiKnTqsp5h5YjPfIS26g2b33G2kKZ0PpSRZWspAGvECzPMcqUljLTVWplOZQ7IlOutwJLrilvupCW32lmxXdYuqwuQkrJNyLViU/MFHy8aw9PoD2/NQuQtiql1+KtBAF5UkOk2bU4Atnb/aKs6bkGOhmBTabEVBzRVERpe/PUzJjDxjcYuhTS0hboW428hKmCloOEhp08kG/nMVc2KNkOexbr11PpyXQpYmkmQc1Vv8ARKoNxJklXj0t7MdM6NEjKbZUV6nW2idJWgAjg8d7m3Iw+ZdpFbznlarT58xbzcBySiCpSVtmWtEXdDLrjQDZO5wLi6Sp0XAU2jDvX6+lpFUay/UW2KOKm22zLhRdyLILoS+UF9xRcW4o3LoJvdNyGwUASGLV8wUzKtUiy6Sh6JVXflpeiuxvHxzpU48UoKUOBzausgEWI3NwKK8cFkjyxbGsF7lIIeXcoZky9Hey/SnfGw3mJTJguthx4yCGnA4XQUIMdbags30EecdlIwfRlZaqtHOWYy6dTI1UAaW4ZrPgi+2FBtJcbIs6pxPnLbY4JOkOKIIaZNdoM+SsomeMlU5EB75jrrk4lwJiymlbW06f8Xq7FsWBsPM6VOCvPGVAupVJqTVd1t1cOEh1qQ7JkJV4tIaRtoZUopHLhuPPysqJxrkcyWK1v2OSTJZ9wuSp0fPeipV3NrWcJtbIhswK/dqdIkRFJbG2WlqHhytqwSpCHFEC1iRd8hZrpNKr0FT8lEaNl9Tkpc1phua84063ssGUXHdDjItZBsF3VYkkcQoZLrlforuYp+YDS4lUjuNldTivM7pBaDK3GY4IWAGkujzkt2DlrkHEspWXsyofo0qbPhoi0lFOYmxaIDNdkOvulKYzoJbRZQDhWom6TyseY25csUcL8x09CLJ2ku5KYsCo1OCqi02L4GQiK6JDUaStT0diPNcbDxDR5b2gFLDiNZLlvuV9NkaZ4y+zB2qm4yiRESdtQYDupLk4nWRoJVoWTYhYIN0Yg87MYdPgs1tyMuUaO1+KWPMyVHASGmvEupbO9oSOSoLte4Wrgr5FUzFUa0tqlyd9EhptFTjy5KIktkN6lMoS+4yASNQIIJF21nlYKMVPrXulbNG3KRsrgzIyxUzqjlAbpq51aeYchUOY2JCJZQ0zDWQQpRU5whFl8FI0AOBFhwQrfnqlwZskVOS2W2S6yosl13WwrU6tLSAF63GvMCSeALEG9mVyq1GlVODOpG+ky7MimtMoWgofaAaaDqwouNJvfU2skeSw13ODqLV/FxqkFGK+uJGdlKAAUJiLuK21JsS2BpBKLLBNgAolZx0qLFqincIi3nfz/uqKijilZm5p+n0Sa2pbjcCWULLZSraNvmctg/v3IItxzx68YbzGDSih4lCyNSALKB/UcYSwlmNUpsGpsTIzc7RUNbkp+RH3ULUlxIcuVm1g4LeROqwGs4XtxoLCpzEhMineDS2EfLDTCQGhptawuorNk8X1JJ57/YsI4vlfAz30DvGwte4A6jkvD1uAtkkcYeSSqaIPmTbHAi6beuHCXFWwopX9QNiFeU359PXt6YS6bm3bHv4JmzxiQc15OaF0T8hSZSTby++PFNvq74U6Ej0vjhTc+UYvzKmyK0jHjb2wbo9xj21qJ0puB3+2AmzojTjxT/aGOGTE1lpDinlp7pZQXSP4cD9SMeHjF2LcVtj7yHNSv0S3/wDl4moQQttN7euC3nY8YpTIebaUvslZ85/zR5z+gx1UMuWS/LfWLcpQdlJ/RHJ/VZwNmKzGSURY7TIPfQgJv+vc/rg3QRG68ojZhuH/ACnrND+Buv8AljgalK/azAhP9lhvTb/PXc/wAwrKMe0HEzKJI3DitLDqI6NwdnVkuOf888/3YOVc8nk+5PODAnHttX5YF7psyC0yVc2uL2ue1/T+J4HueMceTIaD60SFtbTDo20tDUXe6CFdwQtIAHbm+BriSniwqOh9xMd5DjjbKyHFNg3OkDla+LgC17ccjBs2LMaQh1lq62Vbi03WNJQ2Sm5IugJd2ybjsDf2x52srG5pY5Da1iOXX92XbpKbMxj2D1TVRkSGaPDfkLcYcabS8XJJCg6tenzELA5uvk3ve9+9sODTetCUsMWjIFnC6FFRAIGrkjz+oAsCTze2EGVlRk0OO04twuIc8S7duxUhtoEE9tGomwHqLk34w5vQZdOgmmx5ADbEZDUZS9J0oLZAvze6bBNvrJv3BvjyeFVgjgjy8wdOV77r01bB2sr+02uF1xh3TuOpstZIHH7l+B74J0KZU85JUgsbawRfzaCLBFvUqN0i3r+eFKA19MaM4S4UNLdUOVgItcrPYX7d/wCKr4DJVOCpakvNgxXQt0Jd1FoN2UiySLbgKlHWeL6eDjrnGw6NsYb3r6+i5/8ADCyQvJ0SSIl+NT4y5nzZryHXbpJcQ2CvSQkiwCAG0J/NJw4RmXCgLbkOr5uAAWmXbcova3a5AufU8cHAYURti4Uy0062tbRVquF6L83sOAPTte/rhXDS6hch1SVhuxU2FJI4CLHVc/szcnsO1/bFt2QU3aF1+dkCHSTZAEKHMiBgPREaU6wrWEELHlBI5He9+ee/HOCkuQ0JTI8STs7iWW1tL06gVWBOk2JLqr8Afrrw3y3hIiTVpecU8qM62EN222y+lNiSL3KUKuPfgGxwT+KNba4LDNVW9L2PEIZY1lptCCkLsggk2QLAcKIva17+NfM6Wp7PtNDc6r0jWMZFmybJzW2zOqLr1QYjmWVB4SF3e5sR8xxAJtxxcd0kAc46Y0YzGJUSIrYjr1IIb+krSBxwDyhN7ntx6nBW63HKUPRS2pYLhCHVOl+zYHywD9fbzAAXPFhiJdSq3Cy5ld5Bkn8QqjqKZFpcWSdTkhwKDjYURoLm2ByAAkgL5CVX6WIdo2AOdfLposlIWOlsG2Ouq9XpiRCiZPhxqfVapPQ5EZhiWENy5Lbqg644WyCiE0QlyQ7wT+xbsty+F+Xstz001xqnJqE1mdIbk1Sc/ITGeq8kfLLgS3cttEgMoDZIS2gNNeQKWpLkWmMMxZFdnV92fImMtxpi4AZ2DGaAS1AilFiYSSXPMV3lOFSwmyjhTXZxzhIeyhRJMmHRWmzArsmFojOPWskUuLbltP7r7ieG0XaQdZWUct8r3wkE6k21GwHTotwY3NYbJGzDRnGoyapUZceXldqQGW1ocU7GrTzDxIbbSux/Do7zY4AIfdZHJaZsuapDbiHVT23JMkrWZDq1kHc9SU6iAeCe5sO/OEEGMy7S3KaIJy+22G4MWPFDKhsNIG02yWyQACFaGzt2A49Tg6VsxmXlFYXGitob0pSpzxJcWBwjtcXuOSjWVk8Xx3op4ezGQ+g8+ZXMmZJnsRolzbkh9DbLTpekEkDWLqK9ABWLW8hueLDvY4UtIKkKd1BQ+sqQbj2PPrz/AH4LnIUhX4UhMZ53gSkbIeKgDqS4pSz5x6Djld/RIwrabUhlCFpQCOdSbgqN+L+gFrcDg47eGmQvs0a/RcevEIb3ykywoqZQQfO6B+Xc/wCjHIadTZdPdx1xX6aiB/IYNcGh1ClcJbbccP6DBsZlaYzIWfNtIJ/Mi/8Apx69p6rybt+6glPy3B28pxnmk/hk7OPU2iTKi2hMzNVZb2PCqfu4FOvp1AW0AIYcIJ72IHONGKb1Nqsm/Bxnejy00/qjn6nnwbT1QzlLeZKRaSSHmmw5aw8g83N/XR6m/iuOf+hb/q/Jeu4O/wCqd6fmri6XyVvz2K6iMXXKo4uruKAKdwyFMOj7gloNd72tbi2JLVacqa09EYLTJU4St1Rc1NkLBC0hBAuLXNwewtYi+I/kZ9TNYjnQiyoDclYuBpO0n+8gnnnEwfR8xxJTdN1gpIvxzcWx57gyljroamnl2OX811OLap9JJBPHuLqqepFMVSJwRR3afTlrYXEiSVMtqUX33GwXNxdrPBwAFolY0KJsQo2rtJlORo+Umqi5LUxBkvz3V09QMxiFHDVySoIZKW3Gkhx261aQCQAnFq9SIL8OA54OEYtMnJbaqL8WHvlhlAA1OtkgL22wrQ2Lk2JJte1P1LLXUFOZIzFKNZqFJpyYc9DrMUMFldnQAxGKjoW2Vgk3Om24T+zGPGcUYQ/CZ3REjIdidLfL6rr4TXiuiEnPmorMrmXI8Ax15biRq1HacW4tCCGZrV9W0Q4SWnm+7LgXa521jQrhieqypD78mVEKHNluW1KhNIaUm6SCUt/clKlpP0DWDcjE/fpVKjToVVXQYi0llhtBDe6QRq3kOM2FwQvgjtb8sQvO1GjwqmaZlZhbbtMkh9uLuF9TMls/tGXFqC0KKNaFpIW2RYHnk8t9X27gOY0W/Ixje6m2BLqMUQqkzMQmphWg7aClK+3BATYXFxxwDb0wtzezVGKYFUtlDsVxfi0vSHGy+H20myd3VYC6uO1jYduzE1MmS6oqPCgRmUOxi9GZbbc2td9NgQDthBIB1CwsASLg4ldMg1VdDqMWS2lEuEyvxUR0pKHiCFH1toI9iDe1sVSuDTdJlumPJyqoxKjycwwnpEFuK6XI64Dzm64CTs6kfQeeDcglXNiDhWEz2qrTnMruyY70qTLjuJjAtrbiOtkWcJuVrue1gQEjnuMOFAl0yfVIQqcxyIVKC22mQuO64Akm7ZRcLN08X5/3XbAcw1hmsV5l0PRGIzjQhxZbSDHJbv5tRXqK3SDcqPJJueDbCZwXbI6hLqNm2i5fdcgQaRKD6EoQUpQUhxaE+Qgk2abseAAbc82OOTaqtzMbLM6C5HpkZv8AEIig0tW0ty9w6L612WUhZt5b302PLZQarljJ+bI8jxMl1Edt1tmZ4QO7TiLXIANnLAkLTcWCu5wfnTO+T6uWvw3Lzjs+SXDMXuFoofX3O3tq85B5KV9gBYEXwpH8zQKBSOh1BM+dW347yEtvQYaQ06pYUuoF110qSmwWCV/M3SsWXz2UcLKzVVZRolNvEdnvSXXBJfaUAlwIaSLKWE3IG4Q1x6KtbjFPpdMl9tTLC1tSJAOuNMe3Y+hwbYSD8xDiTqBJJuB72u/VHONXq1CmrqU+HT325aILbS57emTHdUbKTsuHadKm2wNTa0eUk6VDBbTFxuFYl+VahTokOoPyWTPU4VhtpR4bI0nlJuFukqSpCu1ja3ribUdtVHoFXh1aVAfeLQlvCXOdKUv6Cn5jaACEFsHcUk83vYixFVMVmbGIbMzbguOr3FRl7boNwQQ0tCVhYBICrdiQMSCn0ehQ47UlWZ4AIiF0wH21JQ6HA5rac128gBSDbhKxfsbYWePW11Xbqotnajw6XXW5mWHiKS+22UJjMvtNMPlrS+GQ6BdkrujkHgnvfE8y5LhUvK6qqlp12Q3RZBRuOrSEwy5ttEuIuCt5xx4aklF22+SLg4Z4TLNXrLFThxq/UaquE683CWhLDLzbYU47ttXDjhSLDaadCyE6+exmVDqsibS4U5OYVUV51oocYixmg68RYFlTDp8jamgkFqyQO9ycPJICAHG9k4FlBmqbHqZV+GwoFLFOab2UcMIbQgqKS3o5PNzpH03Nu2JPUIDsBIqUd4y6qIzUlUJ5tpaHAvUA3tgedoXNiLAH93zctVZrbP4jHflyX6hIabEZ8FoBEVAIDm2EHly/JHAOn0xHms0iqwFLmqMN2mLbdYbLehtR1kIWfMbnTY8HkD1sMT/F1VZT5NzDBrcxMNmhzJo8KQWo6pLktjRpKkMnUGwEgAAWF0Jsb3wHLkWXTUxoan4a3Q0uK4xJDiWW/wBoVN2ukharqC7+cXI++G+jZ+pywuAlQjRajNQ83JpURJckm9wh0PEg3JI0pAWBb8ii/FmHJ0mpVByXuhl1cq/ymyixCZMlo6W1uJKgArg24JI4xrbdvd2AQy806yc31KDSRlaW5EdpC1IlJp9NOhiJdfCACdcU6LXS0NBskrBB4Q5opzciNLkU6j7b8S7MqMsrQ8hhw3DqmgSF2USTpt9XIwBuoUWnvsf0gQt+OIu4Zb8TZUG9lxKuW0h8gF9d061IJCQb2RaNv1Ko1qcqutxvAxUhcSOAotuqbWFfUmw1nQm97eU2F+DjQ9g0cDdKSnw0d8UJEZLiM2S4ZcZiR5MlAi0oXADzoCkgLUbaEElaj/DC6nZJzCGP6J1KOyJ7U9t1l1pai+8XU6lSLi5dJ1p59BYW4Fnbp/KyZUZUf8ZbkwKUjceSmFHaLWvTdTzhXxyQnkj8yBhfnGppoNVpyUVNyXRkQEywxIcYedYXYEuJcaAW4k7nClkr+oahYIHaipwaM1AdqFzJ57TCG2hXIWZs35ZpUSFStil0mm/s0whoU4/ceVLhuhaxawUCbLtwVqw7VWuJzCwcxSodHhTK01IkzIrbUgJmTw4ppwulxVt1QbQSAEoCwvXrvxEqfMi5hTMem5fXPLBbkRYS5hb+t9rzgOEB0k2BbKxdCgb6xfFgxqVBoiqauZRH5tGqZcqLoVU3oz1SYdQG1IU06SULaWlFws3K2z5nNR0dmmIfD/NfZpte/wC+a507uzfdg1SCpJzVRJBakyJkp6I6XILjUgSUhrQlSmyhFgtACRY7ejk2NwMFZnoLlZiwZOVfxBqFWZCGau9KWSxHfNkiQ63YLQzc6VjyLvdYAuRhzqfV6HRnZEKk5ciM+DCI1JU0y06lDXFnnHLkh4ABsoBFykG/cmU5tq9dq/SuFmyiVRCKg8w9DtNitwvGIW62422022VNl1JaARezm2ojz2OvoGhoapjuzeCWjb+5WRlVURWztsCVVc+O9SHlZJrdcEqpLaMLxTZ4aWPM0kEk3WCdB9LWtxxiE1SpMUWYuZEVMjSJrLbzaQ6FAyNWl1w3Nzc3XzwQPe4wVVY64Ffdh1GluOOMubrhDmtp5sjslwEoX3IHPJ7A3w9RqfSKrARV2Y0hxxp1xaI7zazJcQslsFpKLrUu6HLL0AEpNgSbY8maWapfo1d1sjYxclIDJXUY168lgT4ctbsl6O2W3NbgbUlspWde2PMu3I8wsbWwhqtIhQ1xJzs2Q8mY5tbTjrdxZsHgAC/PP24v9nTNlOl0qBAcXMjPKbiMN7+hR347gLsVd1gW0tqLRI8ii3xfk4ZKG+p6bEaltIbjOOgOhbob1juDqPpfvbmxOMVUx0MpD+S2ROD2DKpxQaZKRSGHlXU3IUja+Q2Gw/ft51W0WCDpJGspV3xbsSZGzblxUrLr1OaiR5IcaQlrc2ZbpIaabZCrtIcJI2zwkkaDc4hOT3HVpfotMVUV0hLLbbsmS62lhlttawVuJIs4y24oldgFo3NZGgheBZ9rcHPVelvUnLuZVViHTw1FulCpk2PuN69LjGorcSQBtq1IAKXL2IGPSYTABSmUC4P2eq41bJeYRONj1V35Ej5QQ8XaJEhPT/DBl2rMg2meclbaVLsV6CU3QEWRrQLnEw0m3a+Ml5SzZWIkmJWqbWJjz2VXZi5kWeyqSIbCLtlxsBQbdGi4Le4g/KcQjm19OZNzC9mmit1VyDspKWwHAsaZC9PzHEpBOhF7aASSQQfXn6RgtayogDQ3LbT5Lx+K0r4ZSXOungt3wRJptPmNLYmQo8hpxe4pDrQUCsJKQrkdwCQD6Xwv204CW7398deRrZAWPFwVy2l0ZzNWZetfT+jZTr9Pfy5lCryna7DmESPGPyENVDUSXObnWG/mFNwFoHZdiMWB0cp9RzBl+k13NFJl0yoZddMWC27BWwXomyA2TuElxqyiQRYhxsjgDRi2VIuUq1LG2sOI0rIsbEeh54Ue9x9r4iecatOy4/CTBbnqVmSc3TY7jBjluLPcSdDyxIUkBtenzEG5X2BWoX4EGBU9HVuq2HQ/ZtYDb4rsPxKSophTAXd1UlbiOOWDLTi78AJFyT6dscLZSSlzgjuCORiM5wyJKzQuQ4hymxVTqe7S5ampU9hTjBIdaAU04D5Xh9RAXoVbtdsuGRaBVcu5ah5crFRj1RVKaEdmYkupcktAk3eDhVoWO10nRYAgC1h0ffHtkLZIzkAvmvp6WWD3dlgGuGc8k7OPTm4j7dO8O3JLa9l5xrVpWbEHuLcp9ORfg4KcpjNZSuLUGwETCN5sHbFrg21Dz+gP6d/TC9uChmOt2RVIchuG8WJEqM82tq97hZsbAWU2DY3JULA3w2xMytPVJEbJsAZlmtvrZGzJjsQ2fqSHHJMg7RKHByloOkLSUWJuMcKbGsIfnkbIC4i9r2ueS6sNBXR2aYyAD5JcqO81DQw805IkCStptMeOvbaGslIQSdCPln1OtzQQgE8YrnPkuFBzLFrVMdE2t0JRTV6ahAdS5AW2dSpSgoIYDduS8435PawOJaDWczUuGnN+dZFWec3CKZQnXWKe/IaGl3xLhs4QLOEtOBtsX1lBBRcEL8KpmWYlBnz4TbDyJEZlMVltqO44+XA3HbKAGi581LYsLuH0F8cqDEn1lE5l7Aa6a2ttY+vP4LomjEFSHjW+nT1uoXW8nVjNToNQrdEp0RhgJcqMCW27GTCb8z6nHW3Gw7HSFkrUNMdLhR5nrEiOUrptBTP/ABSlCTFo7haNIlpBjqf1rIAQtpQW4QhIUh9tCCncSSb9pEMuIh5Xdy3Vau6aO3NnvbtLZaMdbQLynHdtDSkOEBxQDe2EJEUnTzbEe6aUfKNKUxPp0j8JXUYCEJrlSnBpT7e8fCSHW0WbQy5ZtttrQkGx13Itj5bW4+2wbK3OQbjlfnt0XsWUeQ/yzZNeZcqoy7LzFFyy06mNHpjjsqqNSno4mOhwpcZ8SUuEobbTxeQhdwdXCteGBiuPQItOos+BT5khiQ+5qk7b7ceO7qVtOMupJK3HFglCTfbKyFDbFrMq9fo02kus1p2Rl6LIZkstqah7zTkdholbjQQkL2VbiG2i4i3maFvMCaVk0mfmCNRPw1uoSodUT+HRUOtuJ25KEkvNfMJJs2WHC42go78ayBjluqhUT9oBlvrb1WljBEAGp3jRY3g5MTxAiM1Bt/ap0uptkpfbaSHVlkt3Q2o7gBBW5dsLPZF47RokWVVKfElrrHjXZMdmA2+860HCuKUp1OOK1tFK0si2hQ0JCCALa5BOmTGKpCpsB5KG4DbXgFuNJvUS+4praKnLEEXLayyqyA64TYpFu5dlSKNXJLWa0R3qlpalQJL9MaeCHER3khsEOOHdAkW0JH7JoE2WAjAbI1l2uVviQenTlTkRI9HrNBaaZo8uMTGfecZYdjOB8JSbpVuMpeuG7A2O6QTcDD5Q8p5ejyXW+ocCTSqdHqM2ntazuxkNsXe2Q/JSS8463bQSC4QBxySh0RmVdZoEasP02VMjR40mnyoTklxEn8QDYdYZSlDSTHG6FHXq3C48CsBIIwU7miNMiJRRS468ZcaoQG5DBaV4htfh3G9xpotocS2spWE3CnFNr4CjrL3ieLsr5Rulc0MKkr9DnTK8miy6DNiu1CMVR4Uia3IpLMOOoM/Leda8jyl7R0ocBKzo0+UnBZkrpEiFJl1F2lKdccdFPairLpcbeKS3q4K+UlxDu22Bdu5s4RhhoIZeo8KRWA3OK5T9ZfgSJSo5bcbS8C4rYIcLjd1tr0htuyljcNzhuqtOyRMoG5RYlThqqDUIR6jFlSkwwhwqS23GYWpzbAWpRQDyV2NlWOEhghbD2drlHNlNyplSnm81Q36lAjzIaYb748a4ylxmpR0BxSnGk6v2aSSnebBPyrocsScK0RamtH4NVWYUJmkukwpk1pCg8g/N3G27uLJG4UeUfMc0j6CbEwkM0kQ4VEgwGGTJXKiGYtKdLAjFjbdOkr8jiSz8kHlIIJWSjDZlvMOW/AVKRXKvTHW4cnxTct6rBsvvtKCXSbAgrceLhbUlvWoN21BCVjGSSF0JJLVax4IUycZpzdVQqnVqzkd0tPORGmVORFnkh+7gbD3ywTyFual2ACVjDA69TJkppKIT8mVLcR49plYIdbt5YzalpJKNA4cAJF1BBJUbcms0KnzTUvBzoymnIi4MNTsunMlBSW0upDaSj9mHBydzkFBAcJQAyn4dQnQ5jdEjxXIm84EvrQZDQLaQ0X7qAdbdCXEd7Nvc/UbGghMkjmylF/kpHREyYsWpuTqiusSXm2rIdksmMiPuXUE2S4Fo8xLepZBWEWsColfPp8ik0Uxy42Hvlw3GmWkMNhwvJ2BcCzZKk3c8nlPbkJw2wZkdiO0mdTfAVOfJbkeKSohEhG6C0CqMR8tkEkatdyygXAFy4LbkInOOzGWGFxX3X4LqmUpltRiA3qcdJICyd1RcF0N3XyokDHp8NrqfCgHTtc7kdfqFzaulkqP8I2SyOh1xCEtx1ttpc+pTli435iLAp12BPkCrEBR9uVPhSI4eDjSkrK0eVfIKFlKgR3BBHY9wQcHMTXpkhMigBp0/N0OAp2W1jkO8C9gUqH1+hPkFjhBChSmnVOyqh4gPNAOPtANrWbDbCXACvaBCjoJ7ng2Jx9dwfEnTxRtoWOLOd7X8r+XReAxGjbE9zqggP8tl10tR0a33ENA+Ybiwm/8AHBYc122Iz7o/tadtP8V2/kDhW1EYYOtllKFd7gXJ/Mnn+eBaSVeYY9jdedSItS1d3Gmh/kDcP8TYf/JjioLK7byC/bturLgH6fQP0GFxRfAQjnthULIkJskJ7JHZI4A/TAdu/ZPGFCm/tgJRz7Ya6KJKLADVj2i545woLV8dS2m3fC3COXvJOG/TgYF4dZTf0J44wchnUpKdXfBLTinwPKtCFtIcJuUuckjR9vv/AC9cZZZnh9mrZDCxzO8vBhY+lKj9wMcCLJUVC+kX0+p9gMGNoMp5LhS24WibFSRZJPsLC1haw9uDgifIjR0JiKQ6EutEtttu/NfsbkAHvbgknj6b8YyVFXNHGS0arVHTRucOi7IaG+0gNOOx1tBwu7ZUyeSUoPqP3zcjjt3wjqEuQy8lmOuQy4G1i690tAFtViV2Nl69JDgPGlYPthvk1aqR3ZjKJTallouE607qfY3csA2DqOoXWSPQcYTV6bI1xGnKXDL1ZZEZe+vZiSnVhsWSrdJXZCNwpsAA2Ra9wv59ieLukgeHi7jzAXsaLDeykYWEZUshVGVShKYmyS27GUgtmeQ/d3aBUSColY2Us3PpqB4BF36oQFTEOydbQQ4y3eKmECVLNtBc0XK3NA08cehBFziFZYYny0N0qlSI0pp1Tj0hwAOvOQ2w2llLitJQjdPCwQDZpsEXFsSoN1qBGaisMU55zcbCFqK1yLkeazixYcC11XPIFz3xzcLjqZqYPa0kNG/6LdVvhikyvcAT6/ojRKh012LDfltJ8SCy0l535twApxCvUHsALflwBguchUFKp7GsIhtl5llpzSq922xpJPcrULc8ckg4FTm5T7UfNcmpTlLSl2M42XtxJbCyFOuLsHFuAtpHbQQngAg3Knva46PAvNPJ8WAy8ZQDUuT5rsgkXQhLhKeQbrbKLCwx2Y6wspbyDvOOnJYH0vaT9w3A3T5BaVCaZkVKQ2XZcZx8Npf1qCCfmFvWQQ325NhwOBe+Gdcx6uqjvLkyG4ja3S5JeDbaUyAnjbcFxZsHnnsri4GsoTSpNS0IKGVRnHnIzktgNhxy6QHdIdVbaDgKRdDhOkrKeCCniNUJC0VMwozrkxsRgmO02SSEOEIa1qUty5bUFqcWEIsogAi44Ez3QntCbnpyXWY1smlk3tPylTIdFYLSRUIy6jKLVwzHYb8t3XgTubjukAOL8yEmwPpJYDpiJVB/B9xOythQWEMlIXwoW1Bw6UeXcsBc27chDEalNVGSmUinPrkOhwx4s0IZjrKAGmb6Q22DZxRS426tyxNjwRI5q4USG7UpC2YsAFx2S4txCGm0cqKQ6Tc2AXcjsB2AF8X0sAkcTI02PMfgq5pbdxhHxSSpVmh06kzMwV0KhU+EpZdcfs0lA3NLgsgK13XoShtI5ccAAxXdIpEyr51aredkGFToeXDLhU2TKbEOC1UJWizpPJcKGnd139+7baPlJupW7KTMjQ+pefqS/SssRJjk2i0l5hSnHljUpyqTGrhZKU3MaMbWuFkbjtw3Q8vZ2Q5MruYZNKazFWowmSo4iMrTSqY245oISi7et1bmnbN0OH5d+HCel3ImjsQXEG5/ueXwWVsby49oQ2+g/t1UqnZrkOOu5fy/Mj0pDbTaZ06JKaadpzRA22WwLoQ8UaShPBjN3cOpZaQfZZqmXaVSY9MyVTm6mw0BEIpbTj4a4CdDjxAaGoG2i5JuguG9zhgp+UZ8h2OzIoNSagRg6HW3XWo/jitZeG++dQaWpBbU4pLa3HSdoFFgDP6PS1BbcwvtOuRv6i9EiwkM7DFyQ1pLQWUcjzHjvotzhIopq0PqZTa3K23mmlkipC2Bmt0TEi1GXKakznmZa42htbUSKpDQ12ULp3rvFQVa5559BxhXT49UZDDzcyIpAWWygU1xpJXzuEgLG3q5F+L2SCMLURXFQWH5jSrFtDTyfFrYUkoRbQsfszZASL+U/ng8iG+6pkNAqacabJSsXbIAVytBsLeXg979sdyjoqGeNpD/AOYR15+n4rj1VbV07yCzuX6IqM242nU7ttsiStxxoSCrUV/SOxNkgJRZV7AcemFjS0P9lMjlY8joVyCfa9u1+ecejMp1iSqG0y9dYSUtgKCL/wAQPXn3wcptpZ3L3Uqw1JPPt3x3qCikp29x3wXFrquKo8YSaUyfmp0mxbQ0kH1K12wsU2nUUpVexwSWyXLla1fPQPO4Vfs0el/ucKUpJBN+2O2x7yO+uLI1rXWavIaDikN/2ylP8TbGaKUW6pnXNWZEobcks5lnkPFHm2xUohSgKBIQBtk88+vGNRU1oLnw06b6pDQt7/MGMkdLlVeXFrbzSUOw6nKkuvKLl1NuLmtPNEc8lQJFtHZNieRfxfHEn/omM8/yXrOD25p3u8grzoEFpc5prbb3JtPZjPFLfmUNkhIPrYeax+67YsR5tRfc/d8yuP1xA8p7rmbaMyptISuntOK4sBoZcBA+3PbFhvNq8Q5pTxrV/fjmez1+s49PzXQ43b3YT6/koLnmktpy3W3qlnCXR4zsRxrx6WA69D1kBZbIFwjaBb0pAtdSySSTjN1Zy/nl9um5hh5gk0eJS1yKhA2FKYleMjpcS80WnVhZWWtrXui9lNjSsXvqvOtMrtVy5Kp+Vswx6DVpSkMxKjIgia1HdJ4KmSCHOxtcWBscZj69U3N9Lr06Nmyuv1+K2w14v8OgGHtoQ26tTsl2MAvxIslZAbDbYUhdzqvjVxxh0dXAJrd5trLn8LVRid2bjob+v79VGJGcFVvLTb7EJrehSB4lNyARZPDY9ODYkd7Gx9mrMFcpkimOPSHoan23kSESWEbbwcRY6ydQ0EaQb6OSObkghio9VdjPIhzUCY0hwtrUy5fbBufQXKL839OcFvyiZ0ZlmBJWu2kIJCvm97KFr2sDY+thj5I2HvZl7W5TNLqwh1H8Yy3UwXSVtul7SdSHbH5iTxz5r9+4+2J/QsyTIsJcVdYlyZsiK2yuUJSGLthIDaHkrSrhu481zwLgYilZYmStl6RIWt8jcSl5fzXmxq+YDYDggAg8i4wDLKfxWZUqdUS7FI+UUNSksqD4F2VG4Nhtl4DiwPrix7A4XKgUwRGmrzI3GrbU2RGcCFRvw9aENXBSR5TZBAHBHl4Ve9wcSCoPZcdYqLz8Nh16oVN0x4r0RHyTbSpfim1C5s2bagAV6/XulhZjp2WstuZHznQ36lDZBrMtbMdTDrTm3ZJVZKhoNynUNGkEBQHCxCtQh7sObNnIiVeAXITjtiXTqLjYdeRqAKRbWDwb3ueDjmmLtX3JsAnzKQR1xqHNXMhBEhx+K600HtEhhsud9LdtaCCgfMJ5uUG4NsDjUynV2imt0yjs02TUEuOxYcOWt5x91g/MbII+Q45rHyyeCUlGIhmnL8yO2ZMJlhDMoyHW4sg6CS2i7jRKwEa3GwlQN78X9Dj2S42YJOVWH5GW5D8GUyqWzOU7YPsFYbsFA+Q39DZZFyi+NXYhveBUzdFPIlBh02mQlV+jVGPAgBbgleIbZS0SSVCSnUl3yrLfHPn9SDiI9SIdObaZy2rMRqk6ly2Jcqa+stuPLLLbYLaVkkANkXWq/Zd7lWJHQswZOzC3BoFbqcim1mLKL6HZESNKQ66AVFtp8gFwK0jyqB4Nu9jgecoDGYoa8yT6hqqNMkmNsPgyI0th0FRv221pWCdxXCgqxBuSKopSJO+o5yJrNKoeYKPCk5bqq5dShxQxU0vIW006sXV2cusrbR5CoDRYAm1uYrS1OSYzPzHGlk6WkF0lwC3cIsRoBsD73vY4fnPw52nOJQ2HJL6GxMc3m0ERgBtltJBs6kXGq5CgQF3snEkyPNorVOriKTNpkyPILlMAqGyl969ttYDX1o1D9o2RwTYdsO+8Y6pfRQp3McqiRYkOpQ3FPUrvvyCwkObunbbFvILguCwuFk88A4MOan6rprtcq0OI3CcdQ+67GJeLRI8N8+xL7ttWtsi4AGjUTbCnNIihht+tXZkCO4tDbDSHkOlA07bhcTZdwpZ5N/KDxwMR2pCDR3GC5PkTEU+OttqC6VrSwws7gdP75W4TfUO3KBpsb2Rsa/VC9kZPrFEl1OKzSKxOZbmKLTTMhsB3aHIcKtWtC9aLbl+CR37EtLT2Y56mVwHIviHrs3b2khv7JWUggHUbi3bj2wjrTDrMKDW4+YYKHFt+JkKjHeKF35Sgj6xoN/0IWBa+Cna3BD76lSZEmPGbHgZDbSLOrJuN0HlAAPpz3/W1rLDuoKQ0uHTYGZ9lGxBp5kASJDR3QFoQQ5wQAbkCx9dJsb4VR578adLTAVDP4pBKUAta2n1ly67tEfMBAWpYAuD2HriBQpVKkVN56o5hajJb3LJeS6S6gg/s2xcruSNaSRcc344eJ9SnRcqxKbT5VPEt9vU2442UkrEhxKnuyrWbcSkKtwbn14uaxzghdFNS5sylCBoKqc20gUeLIdvHaWhfLrqgAt2zZu03cgE3sq3EvZzJDj0JpigVQSVOsuonNONOAsPto1KMVxspKwSUEEjXdJ72OIjSGf6pBo8SYYjLpIeK3tCd3lN3CAfKO9gCQALc4ccspnxX3ZyIq5qSpElTzskx5KhbSotrcKlmx51AAHykkXtjoQlnhIVcoJ2QqTmeZBmreSlx92SQ+4tp7Q6bG5KSDouR6kWPrwTiT5fRX6lVUQIEpusMsBxnZQVtSWgtwOOqeSu6CsB0pDhc0X+klCcJ3M85lhVGVOqjlGkNSFtQp/j6J456O2DpLIkgthz5adXlI1AkFJWL4FR8z5zYrkqhwKxOWmG1LjSWW6THU+3CceUXPlLAW4Ah25SrcI4APFx1KSGCM965HSyxy9s7UABWflnK9QcpCJ9AzrVmGY0d/wAM1W4EZRLGzqMVLwAcBsQAHLoUBYaTYhkjpzZW6Qai0WqW7fVHNPpmw244hd3g8NTmu4IFk61o/eFrEutD6uSEZTfqtao+Wq+W3VIlsR1PRqp4AGzbyVuoVHkR/pJCnQscg+S4xHIvVmXMrNWyblePWXKdVphqcuCYDchSVoLhIbigOBdmTfUAXAWkL1EgLHfqRQhrcpOulh1XGZ71ndmtpzTc7W8t1/YolSRS4lQmPbkKtIh7SWg4slWpxtQ4uLhSm16bgkHnDu1RXaxlx+Jk6TPrj8d9iUily3o7F0WN3nAhWg2WCgi6FmwW2q4KFwPPuVaDTMxNTD1HbmyHFFvaqdBqEOWdtIKGnA4yA64ElHY3JcQs8G4WdMc3Jy7VnBS8wylwqgwH4sinwG16nEE7bpYkJPy03U2sJ7FQ5FsYKeLsJuxmFr8+YWmdziztI9bclZ83J+a2cmpry8pUydTmBUA1Eey+9Ik0+MGtpxl95t5PmauHArbUSCFoIPZFJRRX6zlsVKgOMPOwYlM8M8tMBbjj63FLUqW8v5sdTrbRDiShzWpYCkAKGJL1Kz7RqXVI8nJNZlZahfKMqSzd41Bht5shmTEukFxsA8nz8OI1W4ERoWbspN1jMCHslQ57kiKxDbosOoh4OPltbO+yUKUhYk+JDly44+HG3EEWAJ9Ox1MARG69rA3XKPbOFywi/wA1Vuc6wmYiFS5dNktCnRzDjMzHHHVhAU4SCSrQbOqcACQAG7D0Cy0rQXUNSGWQ0X1KYbSB8ouIAKEA99fm9e/bCvM0S8iamNUGJUSE58lTKHBdBukXDhJDnluvlY72UQBhsyrDn1Cpx4nyZDb77fDzmnubEfqL84+fTNdPOXHe69VBaOIBWDlqqR2KklluHGhpbcdaefbkOGQZrgSGkOeVRtrSEjbAOhwaweMS/pxK6ionTpeUIziWaat2sSozb4dUs22nmZKgpL6AppSghKhytkWAOheJt8OuU6O1Eam5gq0Q12Q5LjxoyHQh5cNotK8O4NYWuzvzhqb1gtktqLfe2WelmTYD86XSKO1EcqilKnWuvxSluannFOE7pcduQVFw86Da4x77CcBPZsne7VeUr8XbmdGGfFUXkHIUPKmfMq1uNUK9Fh1+Q4EPPvsoHi9ptSSGwHUPMkFbZB02L19zWlbeNA02jUfLsp5NKpUSEzVHgSmOyGkh+xISQOLHzEccG47EWY8sdMm8u5uqVdXUX56Jriag08+21uiWu7bxcskIK3E6XC62hsqIOu9gcTYx0qU0pSlgtr3LA2BVYjn3HJ4/LHq6KlbSsytC87XVTqh4zFEEX5SnHiB7YUbareYnHgxchCG1FRNgALn7cY35ljuEnUnnBa20r062W1ltW4nWgK0rHZQv2I9D3GCp9co9OkKhPTA9OTx4GGgy5hPtsNXWPzVpHuR3wmKc2VIjaagZfjEcl4CoT/0SgiM2r81vW9jbCmQc9UWscfJKanLgUaCapWJ8aBCRf+sSnQ03f2BPc/YXPsDhr/GKpXIezlGkOpEthzbqNbiuRIbbYIDjiYzlpMmwIARoaQb33ABcuVMy9TKTNXV0Ifk1RzldSmvF+Z2tZDh/ZIt+40G0fbA3TV3czCRKjtopraGnEPIkLVIcDem7fCRobC1LOnWSog8WNxwscqJYoW5W3BNjYX3/AHuurhMcL5SCbac1AKpTKbmbL1TqVQk5pr9JaiuRIDsjXHgAoUyY0pmI0rbXyoJQp7U4NrXdxABw+VjMeVqfWlZdRElIbiOCVFkTm22d7bd/1Cw6ANbafl+ZsAjduRbs+LqjjTslt5t9Z0eGbjrllJ2kFLoWBezQB+Z2BULAAgXxDZWYIMLMkJ+t+GfpjbNQRMdkxwDH1uxvnJZAUtbj7V7AqLg41gLUNXy6XBII35ZHg72O9wTYX6fBe6jrJHsJLfVOEaRQ3WqtIolbaTHfgGS7JMhDkiKhEdQ3X3CAtsuBVim9m2kouVm5J0/Maqzk12isQX23JkZAckOtIgtMSHGi43oadIK223C0SbAE3I4AxH0vpfr6H4NKYksV50LktxQ0/Ickocu4FAlOjcKkEtXQN1JFwL4lNWzRSqwiE+nLUSRIly2oLs6p0xT7DraIzjy9znh1vQrXe+nkDWSL8mrq30cZp6cXO1m+HTe/otsVK2ZzZHDTzUSmTjTqFPjSXqXHT4Av1BdTEjZW+h0uPtulFwWgdSbkIHJGrkjDXl7L1HRCcqVOgQpNVqEZyTLltxUpU0gKS21spQADF0Nspav5BYEDucK82092VTE5XrjVRZTHeRPPhgy09oKHH25TjrlkONXcKtsXcUWttdraMM1Gya1QYNdW1nNEk1ptEbLzjTTb20QFKa1eHV57rJAs4gJQ3wCb44NHnbdzgC/W3Nb5LGxakub6TlWq5kmJzDL0Q4PylqLYWiLG3Lhpt5sWK3Fg8g3bslC1EAIxXECtRKQmoOvblMnvvRy/4ePvSCw0sGMzGUt0DUbhLhbANr8FIOJ7lDpzlBVLdfr2YZbanXJ4SA83BakFtzVvSXluCzrayFODWltKBYhZKMR9rLkVzLUDMe7+Iy2t6SqJEkOiM7reLTUeEyUbjsdkNp0C1zdfl0nnM+B7XZnkG/Mfv6INsb2CU5tr8mnORs3PZnpkedU4mhx50Rkw4bwDan4yHlpA3XNWpHFm3EkhVk3xHeoCG3pFMoSKWy8+GizFDcJB/FkbjimXFIJshQJshIIuW0ld0AYRPQ8zhl7NVaopaYdZEd0JqXiGmnUP+FStncILTYUdkcka/IL2thRW6zQYrf8A61pipE6RofWp6c1IU4GJTqW23XEI4Kily4FrXb4NyS7KTNaQlKZtCnqgws0Tmm6muZFrAzG4uU02wYwU842ylw7nkuVh0lJS4OOPMk9n3MuWa1ESqBRJM6AlyM3QmULkuuMqJ+Y8867pBIbO2kODz7iW3LWGvEapeYa/KitsZeokOSzT3nTTWEskP7aEaUObCDZbYO82R3UhtojcPGGR6p5gzFLlVaDLfm1GlBsKjnwp0voacS2pLKAUSBwglsDX8zkEk4p7J7T3grM+neCkOXYlPcp7FWR4ZmXPc8IqZCjvLXFbu234VoWcGt9Tbziy5cFe5zZw3dMw5dztCpFBjUXKS2alDq0uqxZUuntQY7znhilMpSUKLbjSUISNRCQpakoCVlROGrIubKK7TpaKlXY8KbDLxprk5h2TeQ4S0lbjRIbBTuvE7SFhIBOmw4eMzzakabKzBT6qqpUtxDjBEqWl0ORC0q69kJs0bjizl2w4ldkIULCGZ8L7BWNY1zblR/JFDqWa6zTcx1SoMO0KRLai1GXHnOJNUmBiRZpTb4Dkhoea4b0NNB3bbsA6cP8AEzczCS5FNSXS0Ip0aNNeoSESAGHEOJMZqKXfIhLqNsaRdWm5ABIDlk/LDjVLEWfVIcyBNTAjxpqUlia0wE7LjIssFFllICVA2QorvoOJV+CQ8vQ6e7RlyKfmJyPtTZbUjxBhuFxOgvMtNJWXFaX0NlO4daULvbgWVFbE6UC/9kMjsqisagz3qxMqeVIjsBIp6Y8N/MEWSl1112yi883thhDu2dvgjSEHSGwrWZE5kyuVFqI/Ra9HiI3kKpz64tmUydxwtxm3UEIKwsH6WxbSdZWbjDOuoT2KwyukyojsRbxeLoaSjzoQFuNbg+W6bqCinbsd0r1AcYmlBnGoMy5FScYkNQ2nWpFkugNMXCt8NLB3ntbaUg+TyN2XoOs40sezOOxFyN/RK3Nl1SbL005sW1GqcVxhbkpuJvuFLxamFJCvluDQWxqU2PISDcW5KyppdHqkZDsiNSpH4ZHIW007FjbDTHlIaKjbcdbDdwDbTY2tbDXmOGA4t6iZwhtoWVzmo6JCWpivmaQ0VgOLFnirW72HKB3JD5TU19lpVUcqMot1B4xmg82y6HkBsuMKuTwtwJduDdu9tFkEW6cEEFcHl/MaDa391TUOezvMTq54BMV6I+7ILod3WIcx1L7bZJJBSUDW6ChzUW3OE8gkngPMuOsuOFLSwhtZHJ1aR2sD7CwAtx/HDVGfRnWhuz6Ma0iSWRNgz5Rdiq0lR4CXR5ADqGlSLBBvyMCQxmqoVeXW5zwapbqi8yyykKeYbdU2nwm7/tQabsHGzYL3dFgSD7fheploJBBG3OHeYBAPI9fJeRx6Bk7TI7u/BKihPtjgRe/GFBT625x4C482Pq5drqV4ayTFu3fHCyD9Iwo0Eqx3QR9ODmQSUtEemObKrXuMKQ3fHCi37uF7RRJ9r9cd20C+lQJTa9j2/P8AiMHbZvq08+wwAtIdVZXYA3UDbSLe4/uxmqahsQvmstdND2xtZFWVqUm5T2PHci/b9e2EkhiSHHVwlIWCg7iHFWSrurgoFwe/uD6j1wa/GMac1IdqLulaFqKHgysqaRpcuNCb82VYnkec34GEqa6yHmnna3Ghoca3Y7EiGpl5QLe7ca1KGgN3upPJ4AvxfiVWLRxtfff9812IsOkGTouuvvGYIkZoEvhbkhbqlfJbQEhLdwLAm4NvQE8gWOFQgIkPNx5CndzeDq1LC0k86ebcgdrBNj3sD3wgfdYTKNUhxtmHEaGt5xxxO42UuOuNqO2Q42CpslSr9xySlWOCsMqZTIivOKCyLoeOhC1kWbQTc6LgGxUA2LHkWx5mTEXZCxxOvXTT/dejjw9tw4Jvcbalz2ni0mQ62T8sMlxzU0D8ohZ8iFGxJuT8sEXJxHswrly3nKgzFnKciNoqKpsOoBvbbLZ3Q7cWCyjnSQT8y9wbDEkkzYjtMlTp0ZxbEonaaeIS0myz5dzzclwk3PoB2AJwzO02HOhO0BEaGwh1iXGiwGZgmR3XC06oPDQEjWA2Ag8jvzcC3BmeypiPe0/NdWOJzHg21S/p2sCkMpXMbU64NyW8hJWWnDfYbssJQtkISkLUEFZWTx7rI1XerjzsOG1VXH5MdbQcZs82Et3u8U2FgQW29oAua3W7iwviO0KfBgUGQzUqpInNwlCS69L291UlyOh1O02BrJKnHEgKJN+fW5klHqLkJwKLTMmqOXSdTzexEYaJJLllANNtgm4ACFdkW9LMMrZI2CC+mvoPOyrxCjY5/a2uUqeiCeytyVJbbp9PfcZZZcf1FzYWEpcKyo3sscA8HTc3NgGauKlyH4eb6fXocNqkS2lvoWttlpqGUONOylEKJbbbkbPAC9RTYEhQw8wEpiEQ0SAl9tkSWZrzLIbZC3Nx2TYK+YE7iRqNhyhCCbE4j0GdTUzJ7dXgSYsSK40zGvISmRvsLuUONCxQ4kJChcrIvf2Bur60yRCJp1PP02+iSlpyHmR2w2/ulDcR2Nll2M9PdTS24rhLtKaW6+dalDw5efSLrfcJs22hAGrQCENKOCJVCgwpSHPBxJaFqcbYQndmOvqQhoKaB4CBptrN9AFxxgynVl+UJuYMyQJyH48UuwJU6P4eMyXfkpcS4tSmy8pCrhdraLob4J1qKXmaFGiR6hGhOmRWHPCy6sik1AQ3ENL8rcZLbO+63cnzaG90krvYC2WhZFUzt7Z1/wC3l5rXUPkp4j2YSmdW47RZTPZhplFneYjijPKbaaQQpTriilTYab1A3sCCVEXIGGWFEouanWc+V6pTJFIEkNUpl+a7GaqLqFpcEgxHFJDcJpDY24wG++tW88UthsFHPq1Oz3U5rNNnO5iEmb4NiG5EOzU5rTfEXaZAXHpzAJce1H5xJRuEFxwvE9NWy7SXKrXanPccQ2tiLHQ65BmS2LB1xtmU2m8WLyN3ZARoCbuEkX7lXITKaRmwFzbYev5rlQRNDO3fudrqGdWswZcbzRlt16FDnPOS5NaK5U8sMubDJIadJF3fmuN7igSG22S00nWpay75Rr0ysLiSJFPj02XHPj9mpLkaUyHHBeQtwJO66QoIa5KGkeRv5ilWaYOQafKq8SjJy3TWFuvM1TOT0s+H8PHTqdh0Vhs6nG2h5H32hpChYuuXdtix5lSgx3HJ0rMlUQpyStcqMt5E+W+6EhOtlAUqMhCm0X0tgt2Ov1IA0jiD2uAGl9bfX9U12l4ZYk8uf+yZjW4sYM0eoUWFKivktPN+BlPJIcdS6XFhYttji5UObo0hekgO0vMr1Ukt0RbkNKFuat+FO3nWmO7SAdy41aQNpPtr73GIYa9LXUJXjFSHWmNtstz5arFK+NxKd5xbZbJtpSSdwXbBAGPZelRqfNaiRVeKlCT80vRw86ppa+AWXFAOL8qwHB8sWPe3HiMUxSSqHZsccg2PM/qvR0VBHCe0I1PVTKnTq9LgvSYqqlPDi1lbIaKVOWNlAnUFkm10D9oP1xJYMmUpSIgiSXFKcLAeWUOllwDU42fKNtdyO/HHp3xFGZEKJUXK9E+RJkOuuPNpcfUptGvcaUQhXYLB81zyD6Wx1ldabrFPfnwUTV78ne3V7ymENugpLbS7LcKm9VvRVl34sTzaDFY6CcPcCXA76DTn5j4LVWUZqYiNFPmJxbSpU9lEZt10tsthCt62uwJ1gG54IAAHrz3J6A6pKEyUkuJA1LURqWfW4A4t2/TCp6miH8yPGMeO8S4lIJsvXzz6W5/mfTHVxn2UbrjRQPS4t6X7fl+mPuuEMjytmEl77a3v576r5Pibnhxhybb6JC0nU4hSu6dxf8Vgf6MKwlNsEw21C+lPKGm0r/hf/wC2wrSnHoG33XD8TkbTxonxV+zyD/A3/wC/GGvhwmqeobGtBWZJdcbXr8yV+LigIF+3p/Li2NvPvJiRpExZATHiyHiT/kMrV/oxhv4b3B/QqmqDbO05JbCnEufObf34KgCPRCkk8n1SceD46daKNvqvb8Hx2e8+i0/lOU/X83xjAmLRFpEaOpxTUf5sh9cdxxbJKwdDY0JBCRcn1AOLSfSkSHSFXu4TxirOm7D1IqjciS0WZVQkOSjsrOoAh0NFCjaxsm32I+2LYccjTHFyozpcQ844UE/vgLtdJ9R9/wDvxzOA5mRvmDja9lp4zY90cTmja6aKwa4YakZbERyaVoSUPr0JKCbKuoG6ABc3CVk2slNzcZ06sNUTK+YotOphNJZhuzItRZgshph9ciK+XbtSQrdQlpkJITfTuurAUR8vQuaKrS8txkVqvVldNp6ViO44qGZDAWTc7gCTZSgktpU4Q2kq5te+MP8AUjMBgV+W3RK2mqMupu3UvFNS47255pDTLjijtxg8rShJsLjWlR4OO3xfWCGkMdtXaeg6rh8NQOfNnvoFWtJmOxZ7cupIegIKwht9SAWiTcDvxcntf174dJsCsT6/sS4wivOOIsomwSCLarjkIVYKNuQTgVETMoExUqShtYLa29S3ANTa7gL2jfgjgi/ryMLp2ZUzHXHkMNBTryyywkAMNCwG2E/uDsb/AHx8lJ6L3icsu0aG5PqlLzA+gR3I7/inGQy895DwQ26LcLAJ519/TCbqZTkwapl6HlekSpTTFNXGjy5MERjUBYOJSHxZtxuylEcJWleq4ItgcjLsqSmPWaHFmVhS3R4qnR461PbVrlwXGhwWHcG902I5GFfT3KWVcx1t9ms/isemIg+Ibn77cdmFuf7cy4O9vMCO1iLG4JzEASdsToOSa6do2YUU3LLcJrK6H9qRaVBrCE+GmqBBGtLay4POFXacWQFhZsQSC4ZMo9HjNzWvGQ4DJUiczGlAu+Fauout+iFgardgR5e44wyZsqK36zCVUHoVRdlFChNjwTCEttADTZWys2b/AGVjwLm3AwtRV6PmBMd6r01sMRJCzKZQP2wK7qCtABJuff2PFsUSju5m7FKjqBmFMhD8lU+Q8ITy4MJDjJW2w0PmJQFAWWhsk2IueNBv3wGtVCTmZ4UyTlyMhwo8QJUZlQSpgkpVINiANKyCU31i5PrfEjOUoRiIr0RmksU4KBklqQ0JN9Ys+W7jdcKbC5RosO4tfCSpSW5dX8exMQzTWvMylS2tL7oXyW2+y73A4P2xnBDjoU7QoU+5W2E1KiQKbF8NRyVvLEZCGy4hACnAedCyBySv7G4AwxUKvS2ZM6FFZYlx5jSCHNzWCSu6nE/uHsUAm/CvvxOy9TKrmWS1VH3YcCaw443IjNiM86A2SptziyyQVAKABSQPKQDiGU9xiNVH5UB9D8RwCW1NZClJcNzfW3wOL+ewHN7ffVvuEpSYsuZZdpfjIO2wWHihpaRqeLbnn1A8EgeX2II4NufU+XTkbSnZLcZoOLSUxluDZP1N7TZugAIvdPGrST7YT5vzLUZ1QiU2pNQ3UQg09EYIIDLZFgXCFeuonV9wLYRtyDIq8VbtN1NPvhtzRJDZU2D+75eBYc3uLm/F8asji0XQU0r9TrSojzjbhTZFmlyRuurAtw0o/s9QF782Btf1wyU+j0uW/OjZkrcxqLHiNzjNjSWpMSOX+ytxCrhCvOFtW5IvYGzgeVVlFPpQqOYID6EyCiTMTBc+cnX5SAdJsjhtJTYEcEX7lFmTqhWKiJWXqczGRCc4MjwwSVuIbOlxRQAu7dz5jc99dxbCMY7ZqdTyNlzJDVLU5R4tDkVJyTuRiEFuK9IbKS4q/wDij29bWtrHFxT/AOKR4tXkuzIxQ3KXIS+lLgQPr5DfB0C4Pe/IPfAsv5tfYjKoylMtpQ9uOFSA+2V8/bsP48jCqfVodciuuxmFwlsSgApTZS3rB1DTybHkrHA/PjDRxGMnVJubpqnQIDMSPKfQ2loO6zMSEBJjC4ANhYqBI81h7Eq4s3ZfVUq5IRLW+7E2FfLZQ22shC3EnZCl8a/L3tyfb1fcwUepzY9Hor1UpzbC3JIkVDfbMQm/kDhQC4HO90FHco78kRlopC3IUxxwv6AC24gBfvYp9wOf9ONkeXLqonubFajUyMp3bMtxxwbqQNQWgc6ze5R7G1jdYHIOJXk11EaRT5kqdDMh+Q45H86VNaAka2lpuA26LOp0cag6CORiEzGX3oSJLEd91QcQyy021rCiPq4HJ4sePU29b4dJOXnKC4mXVHIwQ80248wgqVt6wNIBJ8/ccfoeeMXRO7I3Vb9W2KtZtnpzmeft5pptUREj+HXrbU41LIabIcbdGr/bCbKKL2sTyF7jhV6ZOpMmoUDLLrD9OJXEgtzZTRnOrfjtqVHcYQH2lrS4lS2i0dd7DUnUBiH0tuXWx+H0qmx586e8Y8X+taQXSQLpcPJ/Ii9j6G2HzL0SpUSBHkzI8yl0qoNNAyYsnQJDhuptQko1LaIacToAQWyFA8Ec+iosRLgWyN+I0XKnpnNIyuv5HVIp+RX8voiVJ6SudVKg/wCGEcRUvMPKMVLrLu+VlDoUdxtbLoSsagQeDZnpdNC5bb1NhuUmal1hNPbjOSYbzrjlwlxh8ji5DV9Q7kcjWCbCmxuktfyq4y9mNyTmOKqXSY1XqqiJDra1hTTr43bPEIL50pKbuIJTe9jEqW6pNdKJ8DU3AbkMMzhEDjc+zLek7Ltwt7Q4FFvWAA24W1ItfFs0cJlAg2/NUxSyFh7RWJkubW855cqeUOptYeZaojOtiGG2nn1x0HadszJBJbUhStA1CxbFuCjAKgKRlDMNfNJmHNoBj0yW/wCBbeS8xspUzMtoRuTtDRbd2wW30Nldy5qQpgmZmm5QmtuLpVDfcjqcbdKVqXHzHTADuutsu6HUX+WbXc0kW49UsjO1AzXXH6tlL/1IxFhXiRJTjxaYkIcSGluO2c4La1pcsAj5TdrEDHQlrzDFkeQHDnzWWOmubsvYot3L8diSiexEhlmUHZuoTnn3mSNSmVtutgBtpdjtqN76WwsoB5R5rdVm5orjUqRHjUxDUBtLrSVMVhxaWrIadQSA5otuXWS4e+g3OJFQeoeY8s5TlVfK1SoL8l11uHV6Y80HiRIAUVlsgBDRkAFDvIDiRuDzXOhfh7yl8N9T6VQc0dTMkVlCKhUpcBxyTUZD7KZDZ1SW0paIO2ougH6wQLcEHGeSWKClLjoStUEElRUBo1AWL8t9L+o2f80v5OyHlKTmeqMr25TFHQHGYqwrQQ+5cNR7KufMscG4uMXnE+Bz4s+jmYKRmtqkU+qaIMmUheVpgkPwZgbKWmCl8sLKyFKO4zrHCwUkfX9Ucj03JtMyvAYyHSaZT6E62mREapsdLMdSV86glAHJ+4vfvziocwdcxS83Ky7NrdICaZUHEuNvJKVSNFx8s8aClR9bhZQRxfHnoqj3d4eBey7r6dsrDGs85TyTT8uimorbbf8ASNhpbw8ZHQxOSHG+Qps+e6WvJxwlAt6XMr06jqNh9sT6D1DyjnLZoma8piTV4ACojlXo7utl11zTutvoTobbU2bagRe4ue+GzqblaHl1iqVfIaptQEFf9Yp7q2XQww2oocdQXXGQkbhHLjpFhcJOPomFcU085EMjcpPyXiMR4dnhvJGcwUUShThUltJWpI1LCRcge5A/LDNmSsNUemy1tVehwJzTSi07V5jbMRtzuN3nWodrpSCux/LEMoGWMz5mr7GbWuobUugVKI24iHdma8G1lboDbmylpGgur0Dad0hwt6uAcSaB0t6eU6aurNZShu1JxG2uoSFOPy9NwdKHibtp8oIS1toHoBcg+nErpWnIvOvjZE7+aVFenmaOp9ekvSpeWWpFJEZDENYqzIjbpdU444p9aTIkWbU22NtoiwHINyZm5QahUlqVmOvPPsKFvw6m7kKL/wAooHfkfkpxLf8A7s4e2IxYaSjUFKCAnXpCSUC+kWAA4BtgwI5xIInMbZ5uUks+d/cFk302lUqiwlU6iUuHTohNyxDjJYbUfchAGr8zc4UlJv7nCgIsLnCYx5pfKkVJxphG2400ygJUVjkh103JbUf3EhBA4viSyPjAMbcx+Q+qVrQ938w2XinULdjgcoV6PSVSaW4VFjeeajuloR5bu0dtpwEFa9spLqEpGskkcA4QNVCoMSXoVTpzshmApiI3I8Sh2Q+CwHDLUAr5baiXAQrWsbJNrFBPjMP4pSp1Emvrbbccq61qc0MPNbbjfiGnlggbYSoWBQgBQ9FEnyWNYiK+kfTxtvIPsm4+LSN7b3Xfw6gdTVAmce4Rv5eYUAkZxQxKbmfhtOhqkNNSdidVEfOUgp8jbBc3HLupOhgNkknRYpVYE03NNSo1NjzpTSmKUqdLM2RPcE12XIbDZZdcStTa9d1BRVdBTtN+Ui1np5EJt8ZXao9OizamrxMGvuh10OTFnf3Gn7biXkttl5ltKwglohs2TZUe6h0tVMzdlepzq3VVwQ7EnxgzHEhcWEh9tsuP+ZsyA3IKVbt9YDrpdJDVx4Oq4bn7Jsz5e+SLb3AvqfUfFergxOOVxjYzzTnlV2j5ezA4iryI706U82Z9YpbSAl19xbrpaMoKO4jbdToVZA8psVrJuIS4lAzC1mej1WnTJcR2ezS6nKckFRbkMgOuutNNps6rwi2kHbuWzdvXqKw10+fVpcVeS6lkic0RGktpjw5RSXfEIeeagOON+RDbjiVOXAUsN/4pB5DRmPOsGjZiS3m2VIkRpuW0LlSEz0sMPy47Sn3WXXSgEIdaAShnbQWy0ASQ4nGOSmdTVQic7Mzm4i2v3R5HmCtIf20dxoeQT2unTai/QKnEcWpS4jq4lLv41uTAVvqDiVkksNtrUCR9fmKFFwlKwmpdORSc3og5iRMqFT8U+03HgMNKaeWhADDoZKi2yy624QEjW4bOlZc5IY6XIrdPepmf6zBemohtRKxXnW2pEdh5xqK7vRUjzILLbrgUG0oOpzQtwkm2JFX8zT6TmBmLW5cF9yt63BCWy1vw4+ht4QlbSQW7AqV5iXASotpuLGkNjrWSQwNttsbEC1vqeivbngsXlEPuw83v+CchNGnMSENG7rD8tp0EB5ppvUVh2+sbZNgh0WJDeIRnTMrdLqFTjVtNUS/GdjOLntPLjPy47kfdbkJlObbY7tNhLZGpaSsuEqOLNrs12TDrkGg0ya7NgGS22kokMsKm7baXVCI4EuXClOJu5YAuA2sLCrM+wKTKyo7Jjsux1RzGktQ5LrjbpkfJY20uOXQNtDrvzEhDSV6vInm1FNQ+5ydmIwAOupPp+quMgkF7qFSs5yE5fzFSl0SDElvz3KUJMWUHywfEaW2dklxgIVsKVcabFsLQbELxMJ+V61mOh0CCKdRojMhEgzX6lLZhphNOrIitsM7LRS+495wUbocF3nFAEYIzh0xr+SG6jGrFQogy3Hr1MjzY7bpfYgTy8GHQ2yLWlNs8POuEFbT+gJDgIM+m1im0+mx6SzEisUt2sGTEjKIekPhx4x0uKecW4Iy3XUtti67bSgv5Z4x0IYeyZIH220HPrf0VL25nNLSqZyhnFxNA8EzDentUN1eyinyy1M8Yf2YaOk7rZHiR3A0EIvfQcG5nnUyfW2ZVNRHbnoZQ+pMFbcdt1grKt3dYVdpYLim16jrVY6FG98P9GyPlfqTAqKojUGDWafUBGXTt3+swnA8FKce0JSN1W2YzLLSCU7jZWfrOB5yyRmfLyFRqUyyzIrrVosul1BrS3GjutuvHyJTuy3W3NtERTi2tDTxDhNkDhlge4Nvff5rZkc0d7ZVvRpUjL1bRBlRoiTHJLjDxbIdbbcBAS80SOHElKFAm1hY4ufK1TfqGU6Y7UaW3Mj1R3RH8dUBGfdLrim+AR52wWkjaCASFc60kkVBUsj1al0KZmGOzIq7LUgOxYlNcS40xHWXFpTsrHyUfLSSGwscggXBAsSh9IfCxqXVKvCmuS3EtOOykksrirJBacDZGhl0r2wFqDoUFJsQe2eWDsx2hCVrjfIpfleuP1ma5/S1DtGmNvBtUpG2zqRruGVBxSdp0uNuuFJsgNqSeSbYkgzSwxHqVFeptRjKhuoYbix4oDjIKSoKQhCSttR7iwDitwhabOaxEs00qvx67KQ/UaVKqtQjRIMxoOEI2Fh42eDmrWhOhxOnR2cUvhBIDTAdoiKZLp7VBYg77id1pantSH+S06VJUoIdDbSW0KI86zf1GOLUQds4SNOnktubS3NTOWiTJrUKVDmwmlB5DMlmSv8N8Uw4jdDrtgC2thxKFEjQjzAEgqIADXI1RjJqP4fMiFl1ujwdgiO5sFttPh3U7YDgJdFm+5t9XYrKrriYzsqaxvtyZEWMmSIcSW40/DW0NS4sZH1ncLiy44rQpFgAfIvBtNfrlckU1yYmWtL9lvsVaNpbLdglpbm4ouXU25+6Dtk3Ha400sMjhmB15JS7kUvjLo6n05fnx59PrUNtyOiY6+Lur2wpTZCFWDhKrrSk2Nxb1w8UufKqahAp+XnKfSnITTLi5JDTSX7anEtPIN3OFAHkkoUQBbzmuC+3NlbjUiM94tphp14uyXVKcQpvdQC4oBwWLVrEoK+2sKtgmp1SC1UmBlisLbqS9EF0sTozJVJcd3GXG1HyNra39D22CbBNjrBGNE4qaW0gJv8wqJXMIyEK2KPVpMWNNdK0TZjKl7imQVMlZ3A0G3b3LZDTZtfhChwV2QHMyqrMEaVU6vd4J0vNxJKn4ToDYCQA4lJGnWFXSReyByjDNlx95+ZTJNVZ/9ZQLF5sNokbriCQ4226hIJFhbUefma7mwODaAZ76WZmaTDYSEuHbQ2SUkFvcAd/xgu42CEix09igc+twhjosleXE5dr7fBcyqayoBgtYEfNOSnm332obLg17hQoXsVEIDlvysbm3cWAthQkWA5wU1w8zJbcKWCpxZQoLUStaQkLBvY/vccDzXtxhboJVq55HbkC35Y+r4JiH8QjdLfX5LwuNUPuhbG1JtGOlOD8dIvxjvLg5Um03OO6b+XTg4oJ5IwEbYJStQFrd/ubDC58veKZsbneFEF+OwRvPFC73QlN/NYgWv979vX05w3S9aS4lDzgdU84206+A8pmyNxOhFheyLEarcdyScKJNWaCH2ILo3Wlhp1T/AMplvi5BcPBNgryg34N7YbpDXg1JkiAtwrcakyUuo3HZC1jSlsD0QPKQnt5bX9ceKx2dkjiWOuV7LBoHRx/zBZDZek1gaX6ctan2UF0l35JAsfMpCe5KSNIHe3cC+OOOT40ostzChDzLqjqI3i5vDgtkkrb0EkklA4RwfVbBZWh9D0OSZDzklcQMraUoKCEA3shIINirzcgXCORhhlCbNiJnPsIjOzHhtp8zqpBdKv2G4Bcqb8vCFW4PKOcedbXvlaGXuV3TSBh7qWqiNSpEi8mI6rcD7jspsrS5tkOkquACjWlFz6kWRoGOZjmT4NTBbZYbfafXJbcZDmlEi1tThcSLhLhJtzdZBIFicKqZW7ty4aixBktHS9CSWi9oWQe1ghdgm2vuOeEesXlzKtJmuVKnJRADjjQbTIjBwOiwAacc5ba1N6rOEEK5uo3sKKwSOh7NhsN7fv8ANXU7Qx137pqqzMyJAkTFsx1LBQ88JJbUw2LlJcdIBDS1IuoLUgNjURc84ToRXp6fxP8AGsu/grehkuRJReQ+UJtdKWxYarpQSV38v0m98TKlN0x14PwZCCtxptbbCis7UbjbsQ5Ztq50iwsCrhRwGkZXXUnXVUCMpiCqQ3LqqttoNwG2lkKlOtLU2hplIe1ucfM2wWxr78d8DOxGTnyXRjmcX2conGy9mfN9dpeSMk0SVNnOKjJnOutlluFEaVqcn3cKUAqQS3uoV8tYN9RCEYv3JHQzKGdWzCyxW4aa83GcnKXLp/iadMbLm2HGbFDosbguLKlqOomxuBCspZmpOSqjFaytmhbNQS+7Lkv1NLr7suQxHkRPFutoCQy2EOFxDBsO5JXfm6fg5VlaHGrOWqYt6RWKa02pc2QyGkuMvOOKUlloEhDe7dR9VLcufQYWnaYdLqyVwk71lWudfh8600aSlLmWhW2Ux2Wkz6OpqSpLyLhLoYfU2RpBVYEOAE+4ScU9UsiTMvVBdPzdSZdNcUlbUeHUYz0Z6UJEguylqbAC3m3PVLZVqLZC7t8D6CdY6t1Eo9Ajq6eZfcrE+ZI2HBoCxGb0E7mjUm91AC5NhfkYzT1pyRn6LkjK3ULPdHZGZoNXlUx54pS++qC/85hu0chDYDjZFkEWuDySQQ8WFhyUYLgKmlUyT4SBSKZRPwqRPDFTdixGEPPUmMLA/JWbOSAQvbT+z1uJ8pQ3fHaDSX8wTmMuUxLr8ZxaGarMjzTHcJIP9Tbft80qCiHXW7FIuy2QvcWhYw4h2sT4MFT4S+4iBUK3HJSUP6bJgRnjYoJGkOP2AbBIB3DZs2RmWjxYrWXMvxq5UY7aWKU3TKQQ1vkAgQmSQA2grCgXr3AbdXpXysbaKqiaGvYLu1+qxTMe8m+yW1OVQaHR5+cK1Fb0VBqNS6LBecBYU20FeEjxm1kNthxd3CBYNthJWABbEYkURyc1Cq8KOuNPnvLKUwwXLwEbagLrUNtlg/NLe5d1ZK3L+RCZHlCI1XswjO+Y4yKjU6bIdi0t4NNOUyntltJU5DLhCw2BuNuST53VpOgIQAAjrlXXUF1irQ9iSzNbMCgOKddjqmAIClNFoAuFtRsokAAeQc6teLMWxAGlFtH6gDr1cfU7DoqsPp2iY5tW/h5fqmKRlfKtRqSqzmlyTmGUstynZU9prhYKnFPKDbRBW4sAbywXCBwq459mVE+rVF15FNaapLlM8FHdJBIJulTSkuW1r0diEA25uk2wnqjLUpaJyZdKenBDbyX5DrwjNNW2lFsFDqGj5bmyEarWKgL2aZtPmZeYbqlYo9UdRDW3OfqTs802mPoNtTYLW6sjhOtKiCQOyfXyRq5Q0sld009F6Ls4vE0I+dRKLTKU2x4RxdMkkxErajPBlJCzp3RYuoCS7f5hOoE2IHIOdZjO1iJMep7UlUZp14zGXEttOWtc7e4A255Qq5WO7gFiefZYKYtTdqC6y7GUh5ptt1iU1LDehG4UqeLhXotuABxdii+36jEgpUdldIRJjSQEusuRW3Wo2lLg03W4ppkBA/xYsCASq4SDjJ2gGwVu+6LcavXmmY1R8SosoRGCnlrbeHlK1tALs3yLBVu9yObjD644w4zFq0yFvkR7tvtveRlFuCVOXO4blYSoebSoEXVYgNVZqA/CokwF1sHRKQ6GEKDaXFFAcHBICh9yQTa4OG9+tsw6bTZ0V9velu6YLm8Ntx0pW602q6Stdw0ebm/ywO98ZKs7Ei6bw6qVZazM9HemU+gw3VU1LngqUpcpASpDdtbzZNhrKFAobsgJW2Rz6PsfMKHqZUajUlvx29xtwGZUkTJqWi4Ypee/sN7oVa5WAPa1sVfQ6pS8w0KMlha5tHq7ZcMp2QSpmISoNJIdAIcK2/InhCUFw2CygFDWKvWW41aywpmK07+Jlcte0ljVtCyS25YlKNYQ4Tcos4eL/RpwfGpcHqI6jxZbgAnQX3sPP6LkYlBBWQOgta9rnnorFo2caZMzfGoEqoNTn2o1UcUmC4PnuoltxGiE30LXttuKPIFirRcDEuYqEUVGRT6hGchBt9pqM8t1splbltNhwWzrOmxvzb3BNFdOq7Cq/UHMVarFPfgTZcBqHJhJQA08+tQeK2w2SRdxSnE8kpW7rB7oL9njqLDqdUNJy7GmUvNyhpVIqLn7CA280VkgJAjKdcO2XQCtOlwIPJx7Sh44qSJ2zS2cDdoAzeo+IsPVeelwKla1j8lxqCrIzjPjU/KOZJDMllxcakVZtASv63W6e86UAjuQjzcemMM9BVvJy8ChwttOvR9olGpsLb8OV373uD+oBxo6uVqZF6Q54S3JjqqFPy1W2lvPNL+QTusz3bLGtbp22WgCSAFgmxuMZ/6FIlO5aajxpTkdbcmSJLegFstrYYUG1exI1A8e1rEXx1sZxZ2MU0cjxlNtQm4eo20jpAw6XH4LU+RkSYOYqezLWp1Cwdt5fKhoedJbPH0pRoA59TwLg4dF5qbpVcrMKW1WasGpf9YpqJKFoakyXXHGw86tKbEuAp2lH6NC0bobwiyI2ZmbKe0t5YLjqE3NyEEuhu4Ha/J7d7W9sCrnVnLtRnVYx6DRp1Vi1GfAbQqSJFkbjkfWA0DuOOIUAGwQtO4UH6Tr4eEMgMxlkkyObYj59OfourjvadmGRszB2hUB6qdU4VLzAiHX2HKhPiNTKeKfQ68dpmZJ2QIqZQ0rCy22jddcbKCH1MluwKDnXPNLo+XKvMRFMSq0FFFfqj6IBcfLO6Gt0qupK1raX53AkraSVLSmwTh3zvk2iofr9WZhtRWaBFkwXGTUFocbqg1NswnR+1K2gC6UJB2yQgkkHFRLhzszR3qgIcalwX3o8KExAiEsPWVpfYBuCCkaVncuNZIv5icdXFKyaoY6OYje/n8PI9FyaKlbA4GPaytnOFEoUBUiKYsSJHlN7LS47hdaSSAdxolR13B4BJtqv25xW0hTUypuM0l1plxbhWhudIS6Hf7XOkAC/Nvc2vbDxPrNRfo4oSXy6txtAisuQG0FNwR3BJK+ffjEehR4bKnzVUPtyI99DgPDa7G5tcEn88eUiBAsV2Cpjluu1CFJVDlxhthVtIKkpuDYo7gfp34xMGarSqW0upR3oG4wFttpltOSWvDBHzEqK1fMJKuEp5+/bFVRJKKe4w+0gu7qA5c2tz3NrkfpiUxIhrFZEZC4zRU38tqVYNldrlOsXNybWQRbg3wkrdblBMtZqbmZHXahCojEYuuBoLj3bb59HkEEscAm1zybXNhh/oObUSKQxlx6gyJjj6S34hMgNEI06QFEkBwC/Prb3vgyXMRTKXOMkUyY23oZcQp11DzOtZbbN0AhCwbrQVAoukWub4jeVJTlDzV/R/MwbQmoLLKFTmQUKfIIjEm3HzSk6k9rk3HOGBEjLAKZlZqq7mPLlPQyipzZtHh7ay48Wn3ShxO7ZtW3rR2+gL4474FmqTIqMSFVZzEVLR3VJlNHUXo4RdLiiSR5SFA3t5Li17HFfzeo0io1SO5JjRo7NiHILUUpYBPlc3Eg28tjdXfgrNucOlTzFT58Fhp2uttUmKwxEjOQPmtNttkN2bt3Kie6vXufTFHu+10Sk9PbpNRrFbh0WttUyNLYcZL1Rgl99laLOILYAUW1m9gq3qm+IzORW8mLnUQ1B2dCcCGpbsYOoZcRtbutNyEHSCb34555Fg5SW0UKuyI+1NWttfzCxKbZaWeFNOuEiwaLahfSUEknkdsOTdJdzDl9b7OdrRERUJU1PW47vsNEuJTdHAF7Dng/KBuRc6tIwhmcq9lx3mpC5KY0YxZEJreWgoCmXC5YCyOSFFAB474lDLsl2hvIjUcMypiAwkODSmKUXKlghQWSq7YN7fSF83IwdGiQKSuGpKsvNSg2JJkMhYD4uHBdspu5Ym5OgW5HI5wuWinux0rr9REdsMr3lBKypSAjizgOi4BuEm4I49cMZc4tZRJqAalInsx5DixGS06FImoJY2wQSOOXEEi6B3B7G4GDOpEygxEuZapcShTIp+aZbKDvaCnU2o3P1giw0k6SCg67mxrppKY0FxpxuO3qK/luDTIJ02eCwgWKvKCFDji1sRfOKYch+xW02zC1tOtRnNaVXOocAANkdiCT374DPEoo9Haj02alLkxLbICba4y9RsLAaUXueLG5F/fDjU6hLCXJDcPbjOq2W2kuA7YQBZs/vg2APmH5E4cq8rLDsKHO8K+5UG2WkPMMx3GuwF+5va5BB59bH1wqqs6u1GgbUae3M1tMNoaUkJcaYRuOKLlxfvt/MVc6OOORi495yUJihUyO0742ZORBjukp31NXClkeq/7HI554viQ0yRUJEqJSGUNvREeIckPvJQzHbYQC5oD1hzwSUi5JdCADfCKuxImW4Zyz+FPIntSBI/EHWQNxgsp+QpTg3EISbEBuwXe5uLHDhlKt0VitR01VbDVPhlgNw6mHXG3ndaSUp0IcFyAq6VEI0aySL3F8Qj0JNwp3k9qylnqnQqdUG8mNs0qpNurpb6y2hLLosXNwg7jJP9l3TclsLtcXIm1Gtx5kZFQMBT6GRKaaaaC1t8BRHOoLQLjuOAOw9V2eKpHkZmQ1TcqR4EJNzPSzGQ4w+2CUtSd5d9wEaUkG3DWixKb4Ww8xZbzLUkR6vS6c1MLKIzUwgpTu2UPfyCzgA9tsdwSDrywvkHZaeqzu7Ro7wup303n5UpmXXKnVKTXKu1TVxmq0uDCQ/IahOLU7w6GLNOs3cJ2wflpUhVgA4mP58eotVmsV2l14Q4Lrbk2K2YLsJp+Jc2VGGlIOl3WCAXLFS1oIAsW2C/nTp5Wn5UeemmOBt1tEqA8F6tFku6EufWO3BAPmB4BIxv8Ao3Rj4K3sqUGPNmyWHo0GnPNlmrTLtOIaQsFOk2GogagBZXYj0xvqcWoYaQRVbgw7X/dkaXBq2vmMlJGXgb2BNvldfPum5mkzqcYE6cGY7YRu7ADocRcguKKzwsAnnnUSe3fDtV8mVTIsyk1LNzm2mRJAcDjzciM9G0pcUm3mCHFMvg6XW7pKkWUe43BXvhy+AytzVT351aiOrdckEQazUWAHF21KAT2vbt25JtycL4/Rj4I2aBOyxIreYp0Ke2226ZdWqbz1m3NxrQ8fmDSq5ABsLnGCDHMGiuXTi/LUfqulLwrjLgMtNJ/Q79FkPN0Sj1ihSKgiDS108TvHtwW21hpA8OltlCdyziNWkKDoIX59s6xzitshRqxKzMlcKXPkOsIcLBZj72+du62yocLWGwTYnktkKNxj6AK6LfBClTiIFczLE3gW9EasVEANlIG3Y3BSLXAN7Em1r4mGVPgt+GqtxUZmykjN8ZmYh5lx5mvzowlNuXDoUgqF0ruQbAX/AJ4Y4rRYo8mOUE/5bH81nl4fxDCmfz4XRt5FwI1+IWMukVHpFfpeYEJj18Zfp58VIahRSyZespu1fSRsloF0sXTcC+qxRbe3QTpPlCrdBKBR845aotZbdenTUFyGkoKHZLm04nUAsEshoXPnNvMSbnD1ln4Yel2TUyGqNHrL7MhLSFxplWekNONthYQ0Q4TZvStadPbSbYteFFjQorUWJHbjstICENNpAS2kDgADgAY6FdWQyQRwQ3NtyQubR0skEz5HnfZBptMgUanxqVS4jUWHDaSxHYaTpS22kWCQPYDHzizZVTWeoE+S42vU+6XF6kWBL7zj3F/Szo5Htjc/Ujqtl7JO/Q6hIWKjJp7j0ZDfKis3Sjj0FwTf0Ax8+4yWnpr1Q2zreUXBdSjYE3Fhew9DYW5xz2RiQ2K6RJCtCNmKpUXKlWqlNqLzKkMTJMca/KFl1thmwJsPOScRqhync3z5VDzBDjVGJXkLEkOOOp5CT5xZV0Cw5ANj7X5xq3pz0vyG/wBNaEzWMm0uorqFMYkS1T4iXlurcO9ZRI7BZuB6WGH2J0o6YU59MundOstxH0AhDrFPbbWAeDYjGyicymqBJbZY6pnbRFnVfPl3JGasrZgQ/k/IVelqpjzamqombGLyG3HfmRpLXzA60FtaibNrUFa08j5l90yQ9VKfFqT8XwqpbKHtkrLmnXz3KUn+IBxo49NOnCqh+LHIlE8dYJ8T4UB2wBAGofYkfrhWMiZLX3ytTj/yX/jj2MHE1Ow6MPzC8rLw7LIB3h9VnAtAf/RjiWh/5TjSf9AMknvlSnH/AJL/AMceOQcjEf7E6cLf+6/8caP+LIf+2fmFT/wvL/3B8is2bdhpIwCx3Wo6b7jy9LadB5NieT2HA9e/Yc40ynIWRh//AEtTL/7wMEnp1kFT7cpOSqQ48xq21mMFKRfg2PpfFMnFbbAxxm/mU8PDDw7+ZJ8t1mSPRmpNTlTosSQmQUo0yo4s6XUNkDSlxNrNtk7hPdG3c6BiKTcy06PLg1CNJok5tlDcZBEZpJZYkNqUotskhxF29OtegAIU36OAHQ2eMwfDpk2qt0rNMXKUec780tuJStaRo29agCSPJ5efTjFGdWevHQCkuojdPulOTMwzHklyROepaRGZJ7ADSC4q4BNrAcc48xXT4hibAxuVlySTcj09PzXpqXBBTO7R4JA01GnxUbqlepGaKSIKKfVXZc92PKnOUzS884Y7u40yS+Lo02SQ4TdsNFYJ0i8RfjoXMq1HfjSJ4RR3aYVKVIcNYYkOSHHW2Yqz81fzHtBSS2CyLBC3G7WT02+I/pTMcFN6gZBy3SmnXN4vw6NqaW5t7d9CBcHR5RcEAEgYm3Unq/8ADFBy1NzrlGn5NzBm2CwtikNPU9wPpW+UtqA3EhSG7AFYSRcN4xz08z52mplFwLNLelrG+q6UGCztidLDES3mQCQOm2yoenNsSiqnw6w7UX5TayzUlOOMSpclBj7U4JdLfkStsEbi12JsTZIABn7LkTMlOrMhFV1SnJq1uLEptgRpmossuutuNqCwlButwEOKLgsLCxn3wr5hgdQeojlHzflyi1uVBppkQZq6Uy261tlKAE7YH7ulNh6JH541hUunfSlhh6qV7IuWUJioLz8mZDbTtoB3FOKUrtz5rk98JLT1FMDEx4eDtofn6ql8QLmSHQr5qtZjObsyCIwzNcRMZcnVCmusmO0qQw1dlp95aklbSluJsrRyCo8bZOJX1AnQaGuhilzY0lmFRp34lGRI8Mhbjb0EtOpOk2LSW9JSAVlC2SSQs2snqz1u6C0+qS2ulvSHJtVqDqGmXa9MpSXELQ0QWg0k+dwIsNBVYDsBbvS8XP8AXarmyh1CpNRn0xKpeHERTo7bDJkLbadbYaQgIQXBZJI85sOeMSDBZImlriAXfE6/grZQJJAeSkVfzDJfeoiJT0dccNtlSUE7sB0gbjbG3pu6Gr3ctfWlWjlzHpEuoTMv1ZFA263WkR2HXH45cZSlDj+5vFx1tO48ClTJJsFaUNncNyNjZtyN8OPT/L8vPOdMhZVp0SPNbnOyjBO47N5S2W0oN3HjcgBPJuRwL4z+z1y+E6JUYrNP+HBqNTY5KESjDZSpoEi5DKHSbHQk257DjGOLBppHAE3A+F9ep/JO1w1sFTMirLy9l6mx6O83IomW5zTMajeBc3X5bst1sPuPBKbLdcacVuqO65qNiA5t4iWaafMTUaflmiSlZmp0ku1Zpt2DsBpuNIDIDIjqS6jemKkkMJshJS1tobXcn6MZd6MfDpmlDHUDLvTbLLqqgyHG50ZlbRdbItZWlY7DixHBGPU34a+gFFmioUjpHQIklDqHUutb6VpUhalJIO5xZbi1W7XJPfDSUbwSDvf6dLqsTNa5YroMZ/KsEqq0DRDnUn8SXLj87hJbgoDKdNgtppLTbbZWsuLUV/40glU2qUWfFjZgp9KFElzJRqVShSApqGDEW42kDfSVrAaa29hsBFyUAt9zumH0M6Nw2RGi9OaW22AgIQHHyEhBKkhN3fJYqJ4+3sMMtT6afDbQKcvMdVy1lemU9Cm46p0x9xlrWtxJQ2C4uxUXNPlAvcDGQUMoJcDrb5Kx8zZLNbdYmepkjLtObqr2X6xTpr8RuUuS4lZLr7+66GU2JW3+00AAAAqcC+dAwiq7uXf6LVCfDW3GqLlP/BAxS7tFch9lLQjcEB+++5oTcA7dxoJ50g7nH4AH5bUAdIqfJXEk/LvTUqQlxCxbncsbLbH2BHvfHKrm34EsrTo9Qn9C47NQiLTJjNt0xpx9KrWSoI3fIADwVWA4tjDT1kVPI2F8oLjyI3+i60mA1z4TUOiIaNzpp5b/AN1nCtSqvCn0hjObdXSmPLbaZLNTsmmu7KTtl5/zutFoLC3bG69QAUFDCTJLjKnUNViU8mYFmMISIJdL6390OsvRVtuBdy20Q02AbWILZuvFxnrJ8HbVXflRvhRp2iUta3X3Uxy+tRFi4U3I1keuu/3xbvSyk/CF1jq8mRlP4daaZLYR+ISJUCMhTJcBKS4NwqUFFvTcA3JAPfD1WGMY0i+XNta5sT+CzRxPlBLRewvy5eqzllytxXIrMUUeexCpihJXTqi4XBobWp1ccFsBy4CT8wakDgXPBWQJFZEVfhpT85rZQ6+4HAwBIDileVQuXFhtITxwkg7mq4RjUNdrnwwdMswpiNdDKO3UIjvjA/Bjxlll9aC2tJUFXSqw0qSftcYa5nVP4Zag4l9/oLEeeUvWXFwoupS7g3UrVqJ4HJx52LFcJw+R8ctQ2+3Pf5L0MPB+O1UbZoaVxa7Y6W/FUHTaeWanKlw234kONPRLLoiuOrAbu4SQDrGptRN9S/I2fK3pAw35hotAqLrNUrNFiO1PU1EjyH5bjLOgbRZdQyg2WtJT33GwrSULBKUX3V04yF0d6g0dWaE9DqRSmZiy40ZsFkqlBVyXbC/lJJsT9XJHFiZkei3SfaLKentADZKSUpgoAuCSOB7En+OPWU3ZSxMLTdp+o9CvL1MLoJXQzCzhodtPlosO0hyTNiNNRFzmUlJJQ2HITUVF9Thc2ipaLrAOnc51chIOgL0waDCLy2aV4ip2WtlK5XzXkE6igKeV8pBskrJAuOe4GNpJ6OdJWnmS30+y6h2Mrca0wmwpsq4JHHF+33we10f6XMatvp7l+63FuqP4c3da121KPHJNgOfQAdhjoU8rYGZLXG1uVljfCxxWI6EGZEKNUY8+Ohch5EfY1tqcirbNrKe1HWNHoRcAAW4sJApwrWEBLY1XJAfQri5597XuBxf7cY2HH6WdNYjCo0TIGX2Gl31Ibp7SErv3uAOf1wNPTLp40hDbeRqClLY0pCYDQAHt27Y7GF4vDhosGG/kfyXLr8NdW/b+ixzpSrlSkJPqDgJDXbeRf/dDGzR05yCDxkuh/wD+va/7sGDIOSBwnJ1Et/xe1/3Y7v8AxjENoj8wuKeF3n7Y+RWLHVsso1uvNtpva5P9wHJ/IYRyY78xha2ZbcdIICmy8G1OaDfzKGrQCB2AufUjtjX2dUdKsh0R7MuZMuUNiNHGhAFPaLjjh7NoFuVH2/MmwBOKJc+J/ILSNkdGaZpHYbsfSefs3bHnsY9peF0DhDWHITy/2FwvT4B7M8ZxcGfD2doBpfQC/wASAbc7Xsqv34kNhDLz0Ztu4Q2pp0BS0agq6PRC7q0kAc2472wIN0qdGjKkym1+GsBcoBTHbctuBtZIBUACTbubCwHNmH4qsjgKQekFKCVAagXWvNbt/iseR8U2RUqUtrpDRkqcN1HW1cn3Nmvtjzh9qOA3JZILeh/ResHsj4nIBdAf6mf+SpmVPp8ysvPTNDFRiMrMhKnGUtFtY3GwXQToQACbBZP5XAw3zZtPpIW0aztSgC5ZucWN9YsVDTrDpIaufLYH982AGL6X8UWSFPF//wBE1D3VApK1ON3INri+19h/DBg+KzLCrkdM6GCRY3kJ5/8AsWOM/jvA5HX7bXXkf0XTZ7OOJGtsaY/Nv6rP1arsOTBYgiqbbcfWI6XJMd7ZJ/xjetR+XwAV83JIubjCWFU2W8vPIp0+I/uPbqEqkiM84dtXe1zoADZ55KwmxsbY0K58VlBQlDJ6V0DbTygCSCE+vbZ4wJHxY0JalOHpjQwoDgmQL/8AVYwN4zwZrjap+h/RaD7OuIXDWmP9Tf1WdFzsvNtx24siFDquhaN2mbKZBbRdtTZf9b27FaAbuL9sJHKrCbYkZadr0SOl5lAdZcedcaeaACiyW1nzltxShYfKGq9saUV8VNG06munmXze907nv3/xfri4+lGaq71FhjMVTyfSqZSFtbcN1JK3JAv3SCkWaHv6ntxzjp4bxBQYi/soJMzv9JXMxXhLFMFg94rI8jb8yNT0Avr+i+dqc2N1CpN/+uoclO6Q820+HADxY9yV2JSb9r39rYvH4Xs4Ck9VcvyEOgR6wy7RpOmxFyLtf/ZGmu3+2HFw/FrlKIigUCuwGqbFiRZrkRxlEVIWtx5PlUCBayQ2u4PvjNMJ56iTGalS9DMiM+3KZUluwS62QUkgWvyBjv5AW5wvMZhew2X0oCUqAJAP54g3Wnpyrqr04q+SGaoumSZoacjTEAlTD7TqXUKFue6LG3NibYT9Fepq+qWUVV5+KiNLYlOR5DKF3QhXcWPtYjvziwyARY4QG+qXbZfPfrz08j/Dq3R6JRJaavTK6DFTHXIAUy7u3LRCyShsh0uF7WHLpI51kiMxKFUnIMmpRYcekuRCtlxcuE5sU9txA3kRmAouuuvbdg9y4SoXToSBj6UNx47KdLLSWx7IGkfywMpHfBiDIpO1A1+iLu+3Idl8xM61tOW8hIyvlHLuaacp/wDqUN56gKh7iCSLRS4ooDytSigugLJ1G3phBLyxT41OjU1ECQqRHa+aECRoWfln5rywoOOJNtw9nHEjtptj6krSLWFz9r4R1GowKNBfqdRltRYsdBdeddXpQ2kckknsMJih9/s95tb6D9FKJjacdkwEk/X99AvlvPq0rMbTbNFjywKWyYLaAh0uyi2tKdAeLQG2pBCQ4UXN1EgEXPJuXM01qJBZqLb7jEeOxeO3TZKm4Tra7BbTkmOXAWwEbbm4Oe4NrnV2cPjBnMVx6PkujwH6Y35G3pm5uun1XpSoaUn0B5tybXsGf/6snO6V2XQ6Hb323f8A75j5zNxJw/TvMbp3EsO+Un6819Ip/Z9xHNE2UU4AOti8A/Ecj5LOOXemdaTHEJOWKrSnGPENbrsB8q2ykJDiXG0BCL3JbUA5aw/Znusg0DNMRpnLsnLOYY6o7hUyTSpYBYW1tlY0NhCyo2KyF/4vg9jjQQ+MnOpP+s1Dv/vDv/3zHF/GVncA2o1FH/IO/wD3zC/8XYE7eZ39BV49nXEl/wDAb/WFQtKy/WHYD6Gcp5jkRKewslf4XLSqQLnSGxtAkXT2USskkk2OEWZ6DnqqUSJsdM851BtclqsR5UOiT0yWFsNbrTmksD9+40p85Bt9saEV8ZGfj9FJoY/OM6f/AOLgs/GRn8//AKroYH/BXf8A77gO4s4efvK7+lR3s54lf/7Lf6wsqO5M6kpnP7nSjqg49JebabitZVnMxzuq4kNrQyQA2hLSFtkgWAvcgkOWc6X1Jp1VE+sZardJc/EQyuRUqTLjtTuNRCHHmAhw2CgSleshN7aBfGmz8YXUBTWpun0S/t4Rz/77ioviP+IGqZ+omW6HnXRGiJq7sxtFMZU2pS2Y/wBZUpSvp3rgAea5GKm4vgdfJ2VI9zpDt3dPiuRiXAON4VSvq6ljQxu/eudwNAqoTmJcWiMzW9bcnxQml19xToSGC6pDzji9IstwoC7gItbygAYY8hyXGVrXCchrUlltmoTpDz0eTOmSw5osUJ0Osx2gG0NKsi8oud+yamZggVmFGmS5D1RamT49E/B2F8uTH3nEhaivgm6QkgJAPI9SQti0NyfIay+THl/hiS1VJCIiVIYlr3i494u/De5Zpx0fKJSj5lxjZStdE0l+gC8HK77KVdYqzIrvS/PVbmbCyYCpIcjSNmO7IKNouJabcCHClAJJNzyOL2GIV0BWzHoNSbkslv8A9ZSAFqRZJtCZsL+vA7D7YeupdWrCOk2cqUqK/T4y6ZT6W6wt1yO84tcxtp51Ld1AMurTqbSbahyvgA4SdHWDCp1XgBEOS21W3+CbuXb8O2QnuNBBsT7gY9ZQTPfTvdIbm6vpImtd3Vo/JlSajZtp0dR0yagjxMVr1sHnFbn2bFgL9r2HfFPdUMo1euNOR5FEGVaXHzcZKqjNZCyqHLnPNtOpdA3Vo+bvLbUkkBtrTbUm1hdJnXlZxmVCW8t9l1ttpixPDceWpkhN/RTm4q3b9TimWfiBzbkLM+bqC+7EqCKXmCoCl0wz7w3nXp6lKSoRRuSFJASsB10bS0p7bZQe1gklMyZwqNiEmNMkexhh3BUHzLmvP1fpdINXokZms5cgyEy6m+sLfqbjj7my03otoc3GiCTdxwtWcKbkLr6p5jrNVcXLfbbVUWW7vTBuKcUt1w3uVqvZRJGlI4sb+pxYWZqZm52NHz0xnajy1ZpqcyHl5uGh5qbOjeJeafqgik64qLqWnzFa3C9IJ4VrEVpmcKLliFT6ZUp4cioeZdnU1xI8RCu2+C4lRbcQtq76Xtoltd29spv58X4hBndmaDbfXosNO8NbYp9rdPoUiPHfhy1xZTaP6yy6HHCAfVIsCvtfg+tu4xAFUiNTXUqVW0SUuuOFe0CUpb4svUfVXmNvS3fFoPKjUp1+m1hb7dUbZbceEptKUJcJBBAWNazcEggIBFu57R6sw6Ew41VJkzxa5DzslxD7TkcoK+R8sWABJuALDgDkXx51j+RW6yj8J1t1zwJiLJbJcb0EhJAP7pA54vcDvh9l15LDFNl06SWXmWy0slzw7m5rV3FiCiyuQTc2PI4Ai1Qaps11lmkz5bx+uym9QS5c6SkI/Q8C/GFNRjz4bTMyTdEh9TrL2s7SlFA1KKkE8BQKQDwDz64sID0E8VTMtazM5HkOqdKI0WSzscJdcaRHUpTSlixdbVt2t6XuLGxw2U7K9JzTAaYTXtyqyJZagRJLqk/1ewN3HNJR6lJUbWIQbG/C1ukh2A1U/DWc23NnS58tQcBSFlR5HlUqxHrbj1wzocdZnR5LUJxXh3te002bOHi3yxe/ISSPWyu3fDsA2CFkKAXqXRpMqJKqMPNCJ8RtwqdSlpEQhSlG4BJd1aWyCdA5tqJsFFEYgNMRkSaVLqUKJfdbafSwDcpUD9JuLE8H39O+LVMjJMmiN0qLAkGRu+Jqa0KDkph9B3GWbuKSsI3Dyoa9vsoHEeQ7SK+/G8fT1vpjtrZeY3tLmixDh1IG24E8Wv8A2uBccVma4IcEU5RW6LTKNTo1ZjIK5Gt/xLK1XMSxSphSrltDaTpVcH954HvYM0uj0+vuTadl5MiOIy3Ykt5p0OJbbJ8jpIPKHUAa+VHzfkcNVbdbpFXVUqTml5Co7brLcRTyU7QJJ2ylFihBR2B1k3JJPbD9T6/kigwUVIPLmz6rtiPGbi/LabG2lzWUKDaw0U3upCVg24OoWpINrhRR6l1BFBKXKRdf9YQWlKQFqZcbuAWr8oH1JuD7jntiNZlqDj7UimxIbikoUFjSi4Ubeg9OOfyH2xYGY2odXDNeiVWS/JisndShoKD9hcEKH9g+fj8+OcV+9S63UHkt0eCJz61IDbe5tt6D3Oo8It35PocXs2BKG6jcGVOgyY8DZW020saG0CyghZvce3e9ziwMnrogjzqdWXYhD6C22uQkN7bi1ktNqJHnbsOb3HKhxwcNMSvZwypNkUqWuPHW025CcYShqT8y5G1yD5TyCPY249AuZjo8GgVJyqMOGuLeb8A3HUTGOhStx5x03+6dPY/mMXlrniyKm1PiUjMEReqgVSRIDhclNut6dAttJCbJUu1wR7AgINgQcJqyt6Asv06giBT4bgjbaoLr70Vd7AtOLCQTcEkjtdN79yh6RSMzdQMxyHWs2twChBU0FHSGyAQGmwbDXym1z5hc3FsSvOKKvT9yNmOY/XpygI1PlPuPxWmI4KbXacQAsuXIWErK0rHsbnO8dm611FDKvJrFY2ZmYvESENJbERM1APylk2QlXCy3wbFXufYYfInWSpwaDCy+yxAjQG/EU8Mpig7LDh7OWVd211EFVyeBexOK9r06rxitLMiGttxIkf1UhaQ5dSVJN/OCLG47YaosmAxIDgJSgIBXumwSsC5AsPccY1NblCRzlOZv4rVmBTzVIjzj8gkIKjrdaQgBsnWVBocghoC4Jvzc4Ut5fRpbaly0IXIR8xCnA9oF/l6gUhA7Kum5JB9O4jdDq0WqPJEKSGVoJuqSCpKEHk2tc/qO57YFKq0yo1DbbQ0guEB9dgEqXoAClm1iSPte/wB8NrmUa5TqHDhyZKaJApzFSkzpLfhdbqFKUSk7jWkkBYUQnssL48lz21HTHHmaJTWy+UlFPjtqGq4uhGg8/p64yBlximvpaqVVlPsL8UhgoQ0pLYBHClufuHWPT3+2NhRaxOzFSaRWqjJjTJEqntBx+MmyHFNXaBtYXVZsaj6m59Tf577SGRuoIXu3D/oQV9o9jchFdURHYtB+RCPLzzhuJCrfnju+skJS8TgtLNwQRpt9sWX0W6OT+o1cShxtxilxlgy5Vvp9m0+6z/Ic4+R4dQzYlUMp6dtyXL7nieI0mEUrquqdZjU+fD90VqHUCqJr1cS63QILgvcafFLHO2k+39o/p3xtNtuLTYgQkIZjx2/bSlCQO/2AAwnodGpmXaXFo9JitxokRsNtNo7JSP8Az39cRjqDX5LbLmV4zTsZ+pMjZmF1CW7ahuI5PBt5f88Y/Q/D+AQcPUfYx6uPiPU/oF+UeK+KKjiuv7Z+kY8Leg6nzO5UwiVKnz2w9BmMvIPq2sHCm6e4xlal5WzZJzFMbosaoJk+I0FCCpot24uo8AD7nF1UymVfI+WZ+Yc15nl1BUCI7KUxuWZaCElVgfqUeO59+2PQMu4a6LzT42t5rKXxHZxVPzvnKsR3Lop+ijxj3AKLMq/+yKe/hiI9Nun9a6iVxeXqC9DZdbiuS1uy1KDQbRp/sAm51C3H8MSCZlii1zp/XMwZnp/jZJIeb1SHWS5JWvuS2QSdbpOJ/wDCHRgxVMxVDSVCNBjQ0qUbnzrvyfU2aHPrjVH/ACwVU/urQeXKSrL2XKVl8zFyjTobUQvqHLpQgDV/59Le2F6wSkqwYU2GojA0hOoEi9iDb3+2Hb3lRdNrMxDj7kdGsFHuggH3sexwz536iZR6b0B7Mmc62xTIDRCAtd1KdcPZtttF1uLPskE+vbBWUcmZgy5XMxVCq1+LLp895sUqDFieHTGZSCSp7my3iTYqHcJueTxh746Y+dW+scKqVOHUV5Y/C48Ojyto+GZlkuqktahwh1YCDzYqQmwJscaoYg8uPID5pmtuVunp31Ey51MyzHzblaYZMGQ66ynWkoWlxpZQpKkHlBBH8CD64gnxO9ZZvR/IzE6gqimt1eWiJARIVcIFip17b7rSkDn0usC+Mq9LPiYPSXo5EydlGmJfzDNnz5cidKF40JCnfLpR/jXNCU97JHrqI04p3MWbK/nuvyMwZrrc2pzHAUmTJd1rI9E37BF+dCAAOLDjFRqY4piW6gfL+69TDwlXSMiktYy+BvM23d5D11PJX3Sfjg6mQ6A9EcpNHlVLXzPeSvSlP2YBAJvbkqI+2Kvzh136o5/dcazDnyrPsuW/qzL3hmAn222tKf4g4riMbrcYKrlSVA/mMDZ0pKnCnkJ068crEJZJqjU7220HyX2b2f4fhMHD8mJSwAzR5rm2umvP8kKRJCnlOgBOpV7jgn/vx4rSG990Fd+EAq7/AJ/lhEpepS1KA+on3wZI1qLaAbJShKv+dzjRVPMUYa06rxvs6weDiPHZZ60XY3vWO1ydNPJGvFLRQ/GulK+QDzgyXLUphtpDurWrWu38hghKlKjvJAOltYAJ/LBcVAce1H6UeY4MLmyxCR+4TcT0VVguNzYJhukdRl06X1/fkl1MmPNSC7DfcZeaQVIdQopKPyIIIxIsxdVOoma6C3lKt51rlQorSg4mHJlKcQpQ7KXruVW9ASUDggXxEZAU3KcDY/aHgD745IcLX9VSDqI+Yof9HAdU9gfeGHU7eSHDWDVnEJ/4eLA1kLiZH2F99APW1/xVjdGqZ0Om5hU31qqdaQyCDH2Qfw/7+I2vn/qLot398bWyl8P/AEJjVSk9RMi5VpKFxW3DEmUt/VDkawAHFISS246kA6F8KTqV74+cTKll8lKPlNDld/3v7OJr056o566YVgVfI+YHYF1hciJbXFlJHo8yfKrt38qx6LGLYsZq5Rnm8PUb/wB10uKuAsJhq2UGH1GSocy4YdQbdSNWk8rqw/i96nzM9dVF5Ioq3ZFKyW9+FRYrBv4usOaUvqt6kFSY6Pazv9o4ecx/Bbm6n9OjmSj5hcqeaYscy5VE2EBiVYanGY7o84dAvo1XQ4QUeTUDiEfDBSYWaviBpE3Mc6MVpdnVhtD7oT4yoLWpSUpv9StTrjgTe5025tj6OIYKEBTYKSkgg+oPcfzx3ppImMaGEH8h08r7lfGZ4nU+juaxV8CfVdMWsSemsucFU2rNCoUcK40u2u42kempFl29wca4ztmFvLFLFQeebaSpZTdxLigAElSjZtKjZLaXFGw4CSfTGAurtGmdB/iYmVDLbPh4zE9jNdKabO0hUeSVKdjdjZsPNvt9uEEYZus3XnqD1siroOdak1Fozj7b0Wk0VKoxaeRfQ428AqQp0AkXvYhRGmx4vlo21j2uB1tr1NtvmN/RZ3xl2o3WkOrfxt5XyVEXQ8gsRsy5oCAHl7hNMp6ykH5jg5dXZQIbb491C+Me1rOnUXrNnymycw1mpZmzFMmx2obAbUvYCnm+GGEeRlsepHp3JxZHS/4Oc+5/lfimYYLmTqC55koULzXE37JaXfaB93OT3scbDyr026adAcoVGs0agojRaTEcqM59PzJUhDCS4rU4eVk27dr29saZoqWii7Np7x5DU/E7AeQV0bWMd318zGnzHcUQ88l1h9xNwedaHVf92H/L+UqhmlLlTkTVxYi3FBcp1JccfXfnQD3I/tE2/PCmnQ5OccwpfqaEtmovuSpRQL7bZJUUi3tcDFoNssx4qWIrADLHkSgdkJHpj4lxtxS/A2NhpwBM4b2vYL7B7PuCocdfJV1hJpmusG7Zj1J+Xzsoiel2WFsCO3VKul5VkokqcQbFXF9Gm36ffEPy1VpOXM0NoYnOJeRIMB8tLKUuth3bUODcouL2+w9sWjmKrf0cpqX2k65zyf6kyDzz/j1eyEd/uQB74gOVKH+JVpptbJU2yrfeWfsfX7rOMHDVfiEuGVNZiryY7aX+pvzGoC63F2G4VDjVFRYLEBPmGa33biwPnoSfqrIc2Gi4sEKK1qUrSB5iT3+5OL++HH4enM6qYzvnCMtNAQvXEjOC34gQe592Lj/lO30X1J/h9+H5fUCW1nDNkZaMusua22iNKqi4D2Hs0DwVevKR6nDvnP4+cvUHNRy5kHJwqFAp0kxH6u68G23Q2dKjGZTytsabBaim/dIIsTg4G4GkxSRtfVMs37DSd/M/kNz6Ld7RPaB7qHYNhLu/s93QbZW+fU8tt9tgMttMIS22gJSkAAAWAHoMHEp9sRjp51AofUbLcbMlCfS4xIbDl0m4/Q+ovfEp5PIx9XfC6FxjeLEcui/PBvfUpI9BgvyG5b0Rpx5o3bcKBqR+R74U3Hrf9cGY9hLXQsi7pPvjvktjur7Y7c+2IBZGyJugeht2xHc856y70/y9KzLmSZ4eHHslKQNS3nD9LaE/vKV6D9TYAnAs853oHT+gScw5km7EZmyW0pGp150/S22n95Z9B+psATjBPVbqrmTqzmM1WprMaJFK0QaehV24zZ7/AO6Wf3levYWAx5TibiaLAYg1usrth5dT0tyHNe54J4JqOKqnO/uwN8Tv/wBW+fU7BEdWurlf6r141aqamIrBUiDASrU3EbPf/dLVxqVbnsLJGIMpwBIS4kk+/tg47oJCEp57qvgIQEjzHV74+C1ldLXSmolN3Hc9V+scOoKXC6ZtJSMDGN2A/f46lEKWgXCknAd5sdknCoML+q3HvqwHwxV/Z/52Mov0W8PbzSfdIN9B82BbqQPoODxHXbgY8poj6rD9MU5ipnaUmSQ6i5RcewwJJDh0JbP2wpMdQ0uJBAX2NuFH1xoD4efhuezg7HznniNpoFw7FhquFVC3ZSvZj7fv/wC5+vt4ThdRjE7aenFyPk0dSuJj3EVDw7RurKx1gNhzcegHM/spL8PXw8vZ4WxnLOkNacvDzRYpSQqom/c+zH/T9OLlWzGGGI7KWGou00hIQhKUgAJAsAAOwAwrjx247SGGW0IQgBISgWAH2Hpg02te4BGPvmCYHT4HTCGDU8z1/fRfkfijimt4prPeak2aPC3k0fr1PP0Wbfi8qRTByrR0oWlDsmXPWB7ttpbH/WnGZ5Gkt97DgY1f8XGXlzMsUXM7KSr8LnGK+AL2akCwP/xENj/OxlV5slCk3ubY9NG0dmvLl3eVvfCRnlmhZ6nZPmPaWq4xutAn/Htfb7gnGhs2ddunOUkutv1rxsprhUaEgurCvZVuEn88ZByDU36WxNlUpCEVKnPRq8wpPCnAwbONEjkiyu3+Ucayzx0pyb1iosXMkZw0+oTIrb8WpsIBUtsjUhDyOzqLHseR6EYpGpVuUCxIUNyx8SlQzrmr8DpGWFwoxIPiHQHyy1ZWuQ6kLTZCSEAgX+u5tbF7xTJEZAlOJcd0DWpIAClethjG2XOn2dMiZxVXGpFGiT6NJkMRxM3Sl82Und0ossNm/FyCsX9PPjQuRc4SadkWXmnqHmWCosOXkvsANx2iltOpppIudIWSEi5UePU2xU57Yrl50Hy/2VhZ2xDYhc9Bz8lO61W6VlymyaxWag1DhQ0F1151VkoT/wCeAO5PAxiLrh15qXVScqi0nfh5cZWC2weFySOzrov+qUdh3Nz251t6y1nqhPMEEw8vx3CqJECvMsjs676FXsnskcC5ucVU7HSgfUAfe+Pi3F/Grq4mjoT/ACubvvfDkOn1X6I9n/s/jwhrcRxMXn+y3cM8/N34ckRpsfpOOhP+Tf8APBxSEjlxOOApsTqTj55ZgX17NdEgLvwjBiW3P7GO7gPcjBqFix84xnDtUC4ooJcv9OB6V/7Vgdkj/GoGB6mwLh5ODn6pC4otDVlHWgXxBOrUxymU+m1iK9JZlwWpZZ0htLDTkhTTTTjzizwNbfCRyrnkC9706Y9LK/1RrSabSUlqLHIVNmuIu1HQe3+6WfRHr3NhziH/ABwxKB0rzrAyvl2kU6VGoWUI0x9VRAUVOPzpJU4tdxdxWykhJGjy24AAx7rhLA61x/iWW0befN1+npzK+Re0/imkhoH4Ox15X2v0aAQdfM8gNVUGUHpdAzY1RajT6i8600xPMp0pTNDZBDpKwkXG9uNgp8h1XAHfDKilVR6HGk5Xhxmp2Y4zVDpDMpt+auS84W0tNu7ccWcO2SVJsjcLlyDyFU6oUGPWKntMvlptMaE86JcZ5MwLBS0dwKJC9DxRZRFr6+AkHD3SX34s2k0CVHefka59DjSEVBTcJNRcZP7Zte2t4pbCBuHU2NV1gKKSfoMMk7X3cOl1+cHAON01/EFFpUHJNUy9Tq2iS3CrNPQ4W5Lllbr5RvbC0trAU82+QLEhFr9zhu6VMRoEWtPUx3RBclvqS2V3cbWExX1NpNu3a54J9++IjmmnqpfSh+WqmoD1XrNDYURVUTG4saI266lKSEixUbgq1qX8u1yFcPfQuoPlzMMFLKSmPLYcS4W9RU2WGUqAvwfp5B59cekpm5ackdVqo2rUPTaGwMyQ4rojoi+FcioKQA2w2JDLgQq3a2u+o9xb0xk3rFl11E2oZoXKp78SsV7MCGJrU9l5aZMeW+p1ksoJWLcW4AcautF1g31VlBGxW2YKfPuRnGXFf7ZoLab9ubg9/a+Ms/EdnhuozZNLSiE1NyXXKvSvxXwrUeU/CJbSwkSRy4dYk9jZtCUJNyq4oGcSNy/FbKsjIqyVn3McCnR8vw5qCz+ILrDS24De+1LdY2S408W924QOEp8gPIueMPqItMzs1Nn51gSoicp0KOmPDj3YDsOM42y0wloAOpc0KKnCSpxy5c4AxAW6tNlIjTStuRI3dcdalLQXXCtNnFOH9oBYAIFvq9b4nOU8uU/OkpFNzHIryMyLmGXK35qoMYONObbkTS8HFhYBbGsoDaSoN6TcHHTmnldHkc4geRXIiYL7Kc1CehotVOpbT6KooRokhh9ooLiAkKF/MGnAQo6LJ4Av6YgNfbYeqCpeqIoOa0uoYeUPmEkgKStRIAFhx+VhbDj/AExreWa2614OnLacYG6w9TmFbzYAKvoAN7nQTe4IN72xGK/U4NaS403HOh0LWGHXUFxlZWTpS4ANaOex7+/OMTIyCrnpNB0Q7y1BbCi6PlEhTfBvyP5/bEvYo9KzHNdaTUTHXtIcsUa2txfkBJQOEX5Om5tfFYsvvoJZC16ACgpV2GH6JU5SWVrUpxprabaKWnFjd8wPNr37e1uMXPYbaJVYmZKZKobIy61LaS2psxiNHCHb/LDZ48nfji974QUONSI1GcXVoolSqnWoTLLTjThaWiOl9TzSiOQ46XG22wCi/NyeRgpeZHXGYsOqMRagYjrrbbvawdLYCxxYrTZQvcC3J7YPy7metZarwqdFXHlg623Qptb4WSCCUpRwuxBI/l3vikZmBRKBSJVOZg0yJKfp0lZkojzJS27ruobbfn4cQQ55Da4soHnjDK/BrkOZVG6nJjqbhXjSHkzm4scaE2ClJBBcBCSSEoUvm9sS3NWc6nmB1iW+ww8xElFptsp2wCRwq9roPc6RYL49RiAZxq8mTHm1CNG0uPRwFBprSmO4VhsL3LjWT6GwsRzfEjJee8okFSqyn4iHvwtc5ppYMhTTW9ySRdTgT5LrNh+XvfBmXZz0Oa4winRygB2oNsTmtTbbWjadte19IJHPn4HY2xEXYymJppi5jjp20Ny1R+wXwSNJtrCSe3b1B9cP9AXOZQxEqMlKkzmXS3MUkuNPGwBbbc5O4o6GiFWIWQTbTrxrbG0aIXS5D8xypLRF8chplwx4LDDxUEsOKPksjsG/qDYuHNPJtckMSoyMk5qTHqaWHpSXA4lt9kqRYX+YrQeCDYADseeww0vVCqQ5TzEmAlT8d5xqO1GkamiQqxbSo3JKTY/2+PQ4e86ORk+DlsIW+yuAwh55DulN7XSUjTex1G57/Txhy0bKFRms02XPlF7LdKmrkVB1bjhWvVdywU8Ab9xrTxe9jfnDxJpU85fplLENbc9cZyniO842kJO6pxJKjwBdwkuE27jgJvhuplWrMinoiwa1Lhr3UIRtvqSnR9SEpIHF3OQe/bjD91EhVig1Vui1aS74uBFhRXH1yULNgyl5JUGyQixcNkAnSRydd8O3RRuySUiC9kSbCiUp6HPqNRIDNQjPJcjuIBF2mSBrJCwQVKsHPLpGi61zysZzp+fYgjqWY7jiHW3mZBO02Sps8XI0LKwbpHBAQfTFG06oCLUjJZnPQ5q/mNPsDTdzSdIVyPJcAHvxewxM8yyKjmuFGmO0WWhuLCbjreILrLy0EG4bNtv5es2F/oH6USszOBShy7VqbTdx9dRqy4lRjuBrZXHLSSLkBsk35vxfsL4g82qllZhoUgsgLQHLcOC97m3F+cSpuismmQ5tMrcJ6Sltwyg1I0ulAJ8hve5sR7dh374Y4cTwtSEyWlp9KVjS0toHSR627G3t64uBslckrK/Cr1wy60zfSNxAX+Q4AH6fl98SWLVXlJLchsreU7qecSOObWc/X2HbCqqzEwaKh6ms0xyDUHgFojk7sbyctOskW55uQeww1UiYy5EWw3H0bjuvWny7aLHy29PSxwzCHoeFSun1R9Ta4K3nFtSXUOFsOGxWO3Ha/JxrPplPMrp3Slw1kobcksABtaUgh1RISFkngn8j6cYxjAedckBgvhKgPI6fLc24JI/vxtH4UaDW+o2UaflqnI8RUVVeSkqA8oStDbilq44SNRuceH49pJKqgYyJt3FwHzX1b2TV0VFikrpnWbkJN9gAQblWx0u6c5g6mZjjUimpU0yPPMklN0x2/wC0fc+gHqftfG98o5Vo+S6HFy/RI2zFiosL8qWfVSj6qJ5Jw19OOn1G6c5cj0Glt3WfPJkKHnfc9VH/AED0H64lybEDSeMb+E+GI8Bp87/8V2/kOgXK464yk4nquzh0gZfKOv8AmP5DkPMoflxXPXNyNAyDLrpKfF0xxt2KD/jHFLDe3/nBf8r+mLGFvTDPmTK9CzbTzSsxU5qdDK0OFhwnSVJNwTb2x7Btr67Lwnos+9PPiDFPkNUmqxpElpatOhgby0n3SkXX+lsTb4gs4x//AEPv+D8Q2cwPs05sOsrZWErOt26FgEfLQsfri06Rl+i0BgRqJSYcBqwGiMwloce+kc4z38TtRNXzplnKDSriGw5OeA/turDbd/8ANQ5/HFndc7uhNfMquzcfw/JNHonlC5ji5joHsgHTf/PcH/Nxa3wrQ9rKFZqFv9VVQNXt6Nso/wBLpxTufpSJeaXqe0ryU6O1EHsF21K/m5/LGhfh7geC6U0x61vGSZcn8wXdI/k3gtN7oStVjL7YGjSBzguxPA7Y6DeyQm6jwBi1ndVBH3UOa621GW886hpttJUpSlAJSALkknsAOST2x8+fir+IWl9U1ycg5NDbmXoT7Uxycpv5lQfZXdtbd/2bKTco/fWeeEWCph8YfxGirSZXSPJk8fhkZWxXZbSuJbw7xUkf4pJ+s/vKFuwVfJrJEp9SyhStYUm6U6r4wzYg5rwyM6c1914M9mgmwqXFcSYcz2Hsx0uNHEdeg5DVdQr+oWQbWWtv/T/pwS2pLJbjg3I86sEtLWkCM6qxS53+/bAX3dT7sgXCkeW6sV1QEbcjT4tfgu77NoW4zUGrlGtOwRDyKNeUWJSXkCyeFi38/wCf9+D3LtNKGnyvL1D/ACU4RqSpcFKlFOtldv8ANP8A44E++XYbCCfMkEcfni+JgqGsk6LzuO1TuF6rE8IG01nN+J1t9USoqRp1XAIv+Ywr+U80hxbwQUJSlQP1HT7YLqDY2GFJ4206DglMUqQ274hoJV79x+YxbKYpo8zzYLz/AA5LjfC+KOiw6HO9zRp5Gxv8Ect0SHBFjNWbb5A/04GwFgLQlSkk4E2TGIZQy4nUhWpS06ScFNfMeXb+yrCNeySN7WbBdmpw3EcHxujxDFJLzSuueYGtrfJdkL/rLhT/AJJT/ox14MvJLjjukOd0hPOAqbS4C0eFpCtH3/yTgplkvagtZQQL2HNxitjYp4hn2anldjnB2PTQYc0ONSbtvz1J+Y2KMbWlZTGjNaUJPkQOfNhTFXdDoSbkJVYe/fCRLyUuLaaRthPcn6tWBF9CimUyq4Vqv5f3vUYjJYp3dgwWHJLi/D2M4C0cR17+1lJPaeQcCBr5X1RilONRllh3bWnSpBtchSeQfz4xsH4Z/i7emPRMg9Xashb6ghmFXpCwCVHhDUtR9+Ah/wDIOeijjwhS47iWm0qJ4spVsAYjNRSpxKitS+CL8fr/AGk/b1wr3StqD2QXTw+nwCr4PbFi7wx93Fh0Lwb8udvI6Fa2+PbLDqKnkzPSWVBF5mXZXl5BWkSWL/q2+n9f41Z8PnVSj9M8ztqqeSKdVTWZ0aJ+Kqd2pUJtaktnSSkgt3INgUG98VnXsy5rzUICcwV+rVZukRvDU9mTIU4mGygfS0g8Djgn67AC9hhoFRatrZkrbWFoKVjgpXe6SD73A/XHraYyPLJIgSLWd6fkviMlO+F1wLs5Ha/mvrmwhotBaeL84rz4ios6R0H6gs018tSf6PS1pWlWkgIAU5z6XbS4P1xD/hk+IqndVKI3l2vyGIua6ayBKZKrCWgWHiG/sSRdP7pv6WxcWb6MzmPKtZy8+EqaqdPkRHE+4W0RjLNFJTTgO01HxHX0WDK5rrOXyey7V6zl2oKqMGKh0KS42UOoNtsquBwbji3a+JE91IzSW1IplGhwFqF93Qp5afy1+QfwOIDBr6zAiqdXpUWgSP8Auw70rMdMZcT+JwnJTY4sl9Td/wCHfHksTwPDq2b3ueDtJB+/IfkvaYHxHi9HGcPpqnso3HX9b2JHw1RkWDmOvVBSv6zLkvKBdddUVE/dRP8Ad+gxrz4YPhrezS01Vq9rTlxl0rkLvpXVHwbFtJHZsdlKHuUp5uRTHTTqb0VTmCGxn9+VRqIhWpyPFjFapB7BBUjlKTfzK72FhYm43t02+JvoLnKW3lHp1mOPMkw4iVop8SKpBaaQLBKEEC9uBpTe1x74xNwXEcbaJa2mMdO3Vrd81uZt9kdOa71fiVNwxE5uGTdtVSjvy8mA8m31zHry9drGzTBWxkmq06iMpjKRSpDEVDIDaUEMKCAkDgAcWt2x8V6SxLZoFPcLS0BNPj3J4sQ0m9yPy74+oPxzdTpOQfh9qDVJlORalm6XHy7GeQdK2kSLqkKHqCIzT35EjGd/hV+FibnODk/rRmurRf6PyFOT4uXVQtW+ykqRGLrhVYpNistaOfl82vj2NDD2cIkOjb/gOS+a3LtT+/NRn4J+sdWyT1AZyTVJTiaPW1kNBV7NP6T5R6AH29/zx9MmntbKXjwCL3x8dusGWah8PvXKr5Xp7ziWKJUWqhR1rGrXBdO9H7/WEC7R+7WJJ1m+MDqf1nbYyrDkyKHltQbit0yA4sv1Jzi5fWixcursyjyAEA6jzj1OIYJ/FXRSQuGo1PO3I26lXOizmy+ged/i66GZEqLtFmZyaqE9n9oxT21SA2fZS0+QH7XJxVz3+EU6bPpZchZbrrSHUpXpfZSFp1DkK0kgFPrYn+eMw5E+DHrznKCzPmw4OW4jgCmxUniX7Hm5Zbvb14JGKo6oZYldLc8VPIM+vxqlNpJbblOxGlIaClo1aBq5JAIv+f54vhwDCIiWl+ctFz3vyCfsWNGq+imVvjk6YZjqsalKlSoDkjSG1S2ShBWTbTq7A/ni4q51Ry/lbKsnN1clpbgMnS2W1Ba5Dh+ltsA+ZSvT+JsOcfGhqrJdCm9azxzcdsXRl/qLXs00inUyq1dyUilQI8eGzYNpZaQ0GidIH1naBUo+c3t2GPnvG9fSYHhz6mjZZ97Ab/H+y9Hwfw5HxNibKSR2Ru59Og9evJWV1W6q5o6q5iVWqvqZhMam4EFDhLUVo/8AScNhqX69hYADEGWiUHQo3OrvgsVHQQgAlI4tjyZridSSpRSrucflmrqp66Uz1D7vPiH7/YX6+w/DYcKpmUlIwNY0WA/fPmepXS0+k8H+OA6HwfKrjHDO0jscCFQZtyhX8Mc+zvsLfaQclwtvlOkLVb21YCWJB7r4/PCv8Ui7CUFolQX7c2wSJzRPCT39sEtf1StdJ91c8LJ06UXJ979sAUxLCdIuoA3Nz2GF6KlHaNlsnSoElRTwAO5vjRnw+dBEZg2M8Z5pBFIVZ6nwHm+ZnqHnEnsz/ZSeV9z5bX7GEYVUY1UCmp2afadyA6nzXnuIeJqbhmkNXW7fZHNx6D96blNvw6fDVKzYqNnrqFFP4Kmy6fTnAR423ZbgPZj2T+/3Pl+rYyILraUoQEoSngAdsebmtNNhLTKgE8ABNuMd/EUjsyu/5Y/QGC4LTYJTdjA3U7nmT+9hyX5K4o4nreKqw1VWbAeFo2aOg8+p3KMER7+1/PHPCP8Av/PHPH/+6X/DHPHqsPlLx1e6vOKD9aoESq9P6nlN+WhE+usuxqW0q935jTSpKUJt62YUebDy4w6VbiUvITZDg3B+R5tj6B5srDNLy7VK44yL0yBJlBah9OhpR/8ADHz6aBbZZQ4sFQbSDb3tjRCdCkel2WKsijV6HOeUfDtP7UkA92HBpV/I/wAsas6PVPNM3prVMlZalQEVvLcxyAw5NKw2Iy/Oy55AT2UQP9xjHsgpDhSrgODnnGg/h5zctnOVFmSHrt5hp6qLLUry/wBdi+Zo/mpGsfrit/cermasSCo9KetNfzVJocmAqzbqC7UEPKTBGsatwLWAtwj1ABN+Dh3+Jvp3Gy109y4qkVZyNTaGFRRDLS1plSl8h5RTwlw2cJWq/JsBc41IAn2FsQ7q9lEZ16d1vL6GwuQ7HLsX3D7fnbt7eZIH645mP038Rw6WnaNSNPUahdvhbEP4TjFPVuOgcL+h0P0Pw5L5zrZfUrgXtgBjOEeYXwqTOS2o3ZVzfg+nuP0wIzx32jj8uC7Ceq/Zoe/kEk8Ksf8A/WOhhdj2wYqYpf0t/wAsF76zaw/lg9oQnu87oQjr7EpGBpirPqMA3FKN9N8CDqk6RY/7nTiSXG6U5kZ4NKrhwi/viw+kfRGvdUKnpbKolIjuf1qfpvoH9hu/CnD/AASOT6AuPRTofV+qdQTOml+HluMvS/L7KkEd22fc+6+yfucbcoGX6TlilR6LQ6ezDhRUbbbLSbAD/ST6k8k8nH0Xg/g1+JkV1eLR8v8AN/bz5r5Lx57QxgoOH4c68/M8mf8A9eXLmkeTcn0PJFCj5fy9DTHiRhwE8lavVald1KPqT3x8w/j9q+YJfxLZp/AsxS6a7SKJlqnlMdQBKJDj5Up1NruNjcAKRYguAgjnH1Z0BtPHAGPkj8bWaKwrr71EqeWYcaK9FnQ6e/PlNbQnNNRooXBDwKb8uoISSASSC4DoA+v1gFPTiOIDoByX5wmlkqHmWU5nHU+qqusOPVWaqc5Pfl0eQWH460x1vJekMKKZbbOu1nAWCLhFkrcPKgOS82tClVB2mLmVGmQfCOvtvNlEmTM8S+ncLam1MmMEyFNJDCbuEjng2BlOpcifmKdmWdHqv/q9bi5VJQ1+HzWZgbHipfzr6glBQpwR9fLhWQq5OB02qULPU+HeHR4lOjsyHFVudw6mT8oKjXXqAWrUO4ICBwSCAOA2R8UoaNgNfJZ86Q9XMt07JuTmo0uqVSTUncytRmzJ23ESo7ER4PvBTZPCZDwTbkguaDcpJwb0pd8DVq/DmFkQZL8Blx8E3bXrbS3p/M2ubcD1w09YKfBVQ8qw3XKdOqSag5DU9CJZbiRA2pzYKLAAqW4l0GxsABfDn0TQ7PkZgQl4EyA20nULqU6gxneD6Cw/Ln3x26Ih9EHE9VvpvEFqHJ7i0ZljlbXmbjyXFAEcEONi18ZRz/mCb04+ITOdSq+WWG24lcmOwxIpDbxqOhwqYJMtDmll9Y87jBRwToIvbGqMk1BKq5MU3FbcRDY3XXC4QQXzdKEAf5DZJuf3hbGQ+uTcKJ14z0nMFYE6cjMUwumWt1bLbG6VMMgIUNYs4ELRYaRco7klWWFwRdaazwhMaDTKuxOzFv5HYX4+OFxJcvYLzoG8H9iONbbbZ1BenbbILRd84Aww5Wq9VmVBumQaS4p5Tr7stTq7qKy2fmKdWe/zXL2NvmApuQDhFIlUGJIiRoCVsuSmXUJS0HSHXQvyNlP7ilANjSDcB0FZPo2ZhYlwaN4ykTZDgQhtMyS02vaQHW7JQogktlTiXEgK4O1e9zYWsjzMLVzL5bFXDX8yT8yNv5tpFSXAmxt92oIMsuqc13u4LND9xSgu6+xxXlMynPrcZa4FKrMpTRQnXHp5V53CQ0OSPKSOSB6Ed7YuSl5UreVUVSm50oVPzNkd2pIo71dpzqzTlO+VTbSlAbkdaiqyN0Dzgthz0xftS6M5ef6VVnOFPzrtZkjByqS4aipTVRhRmSGwplY8jgWoq3E6bhKr9gvGCSsZSWvzXRjpHTvIdyXz/qFPQzUHYbclzyawVlKklKx6FJufcevpfA41TMSI0vxSyqUrQNA1Wt3DluUEexGJHVsm1KLX6jEprDcNph4uNtOBdw0s3SARcGw4Fz6YNzxGS/FhuSIbP4u403AM8KcRus24DiQSh06jpvxYd7nHVDg4dVzikLFaXBlGTGmIWkp8iUaFW9yCQbYcYOakRqlGqQZDakuh0NraNlIvyEkWHp6c4iULLtXbrRoywiFPDhbejSkaSSjkoHHcAE/e3BOJFTcnPrrz9Dn1elxGjT5E2CZEoGO/YgWLgB2yTfvbm1+DfAewHVRSCq19D8tdUoeanVSag0+t6EIZbUww4bFovkAOA3NgRcm5vyMIfEZihMqiu08z6EIQeLi4QmBp9ywS2LNnQ5cHggkaee18IqJB8A5euwpcRxUbhx4hQU24AQpPdtwi6VC9wPbnDjTPGJjzIFOzBVGq78xBeLwMZLbZ1B5SW7r3LjuntuD2xUwZNlFEqjlSu5dZ/FXqatiFMQtpBkN6VccOixssW418d1exw/ZJ6c1KtPw566pJoGXHZjX4tXJrZVEab3NO4Gwobi0kDg2c58gItgaKuupwEya9PblARQ0EuNXcCNBSbGxubcEHvcH0wjfqDNNW5Op8ohCGm7NJW4UkhAUTxwsc2v3uDbjFhkchZNaqFMypnyptSUMVCBCkuctocaZm25b4cAcAcHmNwF2JtbDlU4sZTMan0p6dJjuNbMYugh1sAD6CAD2Hr2HBva+JP1NjR057qcymyZKYtTaaltqcf3txC08/MtYtEC6Li9u/IwzqojMgKjM1Fx8tRQsOxgDrKzayh9aBwRY29DexxZnFlCktNp34kIkKmjxMtpxbrqr6d1XJuPcaLI/T0wZXsvVSRIW/OcaajE+dV7qAQPOCnuLe3scKciLTErTNTgwH6jEYdcUl1KbhhxHZTgQQWwRzcEXHa9rYe+o9cqsemqaqtDqjCZBLu8Vq0qQvVpQkOJS5otq4PBHvxip184ska3RUVXpcdEtD1OiMISEi4bR27c2xKMuZ1ppo79FzPHuw6FlmU1r3jcfslnUBo9RwbHt3NwVBjLchyO1Bhh1LjLTRUqPqUyQCbNnUSvmwHrbEfcRPbU1LpELy+ii2NN/SwPGNIYCl2UjpuSahV30NUqK6oSChca44DRBJuO4AAIuRxY45VMuKprzrMaWuWWhpDbbawl1H9sFduCOeOfX8h5dquY6E+BTRKk1WsJXGeShta3A0LkpKRyCq4Pb6L4IkzKpGrLK6lSpLTnkLaJbanEuLAudRPC73vewuLYGVSycYcgvUd+rTvCrlKcAZ3GyXkOAHuLABBHN7/mOcN2xBakpbCHGVFYSA4q9lrNwB/owonVqVVmUbjcCOzDSAtMZkM7nIFykca784cKRDcDS5MO7qFWRJcSQ46yL82bKvmC32NuCO2CGqbI+HDlOvpCQy9oBT5bJIHpyeLi//AH9sfUr/AAVcKkM9Ms0oTGtU2Ku0px1zl3ZcYQUJ+wJSs8d/0x86KTlmdRqwmaxWIEluONbSHWV28OUHSvuQTr0AXP3OPoD/AIMFp6NU8/NFqGymWxTn1NReE6kF1N7HkcH+N8XZb7jZXxPexpaw2vut/wCO4Bto/wDJx0ISO2CghY9j2PYiiAvtjIFaq0bNvWXMWY1gIiw31REK1EjajDRq5HF7KNvvjUWfMxNZRydWsyuKSkU2C9JF/VYQdIH3KrD9cYljvu0Tp1VZb5vLmNiHqJ5cddNlEe/dw/phm6AlWwt1zJjbnO1WW9VHVFTkx1yQb+6ySB+gsP0xsHpZFRB6b5XjM6Qn8KYd47Xcu4f5qxjWkPI5cVwlsavyAF/7hjXHQioO1TojkCpPpKXJGW4DpB7+Zv8A7rYLdNFHxuLDJyU+7i+KN+LLrgOkuQvwqiTdrMuY0Ox4KkH5kRkCz0nT7i4S3/lrB/dVi633mYzS5Et9LLDKFOuurNkobQLqUT7AAn9MfKHrx1Xl9YuodZzY8tSIT6tikNL/AMXAbuGU/msqLh+7h/ycVVc3ZR2G5X0D2X8JjiXFw+pH8mGxf5k7D8z5KIMS1F15SklSAlA7c/8A82NY/Bv0y6XZ9yXmqs53yNRq4/TKkyllydFDq2mfC7iggmyrEpJt98Y4jSlRnSFJ1BQSDjcX+D+WJOTM/wADSABMiEA+yojqf9GMdBYTar9A+1FstNw690Tiw3j1abaZgCsryouWZ3VgwcurX/Req5kQ3T1sJWw4mnSH06NAcBU2pCHCACD9I73wv6ydMK30d6gy8mVl9MtCkCZBnIRoTMhqKwh3SPoUFJLbifRST6KGIU249Dh06RHdUl6PFYW2sdwttA0n+IGNffG7HjZt6XdM+pLTSEPT1uQ1kejMyEl9QP2DjR/icdau7OrgbJlAcD9CvkXBOLS8OcSPwoG8UjnN13uAS03+Buoo78OOUsqfC9Vuo2eZ6ouaa3TESqKVOqCIbrhC4cdDSf2rziUjWVXACuAEpKsZiaccDrSRYhxxCQByLk2tf8z3xtn4+JTlLoPTrLjJ0xhImvBCeBqZjMMo/gl1f8cYhquthEhbSNSmQXm0j1I5GNeGzxiQUbmjKCLnnqvJcSGprYIeIKt+YzPeB0AB7oHloU+PJKJD8F9KkuJUttSVJ5QtCiCD+RCsJ4yUuIVHdHlbOsm/+UL/AOnEm6xKgNdW8yO0deqFUJyKpGVe425cViT5f1dOIpKA1BxtRRucAj2OOVCz+a6lcdifxX0fi7tcJbQ8SUY+yAfQjT9Fsb4oPhkyLlPp4vqB0ty0zSXcuq3qtHjuOLTKguJALulalfMZXoXxa7Zc72GMzdMaJTK/1IodFrUVEiDLl7chhRUA4ix4uggjt3BBxY1f+LWuZl6H1bp7Pia6rUoa6c5LC/2SNTXmF+4U3ui3obehOIL8PyfxDqtl93QSlt91w/YIbP8A349dTYN7rMC4ggnX0tofivkk2NVtayNk8ucxlxab62Ou++nJIeodBjUPqrmfJ2WactLcDMEin02E2srWUlxKWWgVkk/WEC5P1YBV6DK6Y9YJGUM7UeFVGMv1JpqoxloUpiVHWhty6U3BWChw25HIH6WpQ6NFqfx4qp01IKUZ+kSlJPYhpgvp/m2D/m4Q/Gfsr681BbbaEFuiUlDpA/aLLLiyf4KA/JIxwsKkhpHyuewG5sL7Dr9F9A49r55I6GOK+ZsLZS7nrYaH6lSvrF8P/S+qdO3+s/w+yNcOnMLkzqW1IcfZXHQfnKYDvzWXmb3U0SAUhVgkgXoqgQcizOmucZM6rRoGb4cimyKBGU85/XGdSxKZDafIu6Cg3V2sFA8nF4fBPm5tzOdU6bVV5LkLMMTeCHCbLWkbDv6lpaP+aMZrhxnKdJFMJsmPuRF2VwpTai2VfxRjNUtbQVEVU1os7l8bLpcNVs3FmBVuE1crnZAHMdfUt1IaTzsRbr5pxyxSG8xV5mkS8zQKBFdQtTtQn32GgLW1EA9yQMSHP2VMq5N/CY2XOplJzrImR5Ds96mAJZhrQpsNN2Oo3IUs+b+xiGxlluQFo4J4vexxxSkKccQp9a1NHTpUrti/F8QMdO2GJgF9zbXfquV7NOHKDGHvnqjmMR8Frt12P70Q2ZcgzQ3dIb4T9OJB06bgRc+QKdVoiJdIrBMGdHXyl1pfBCv7wQQQeQb4i7SpG8FrZ0pvwdXfDm3JcizI0lmxdjOiQ2TxZYNxh+FsTdR1TmvJym113va3h0FNDTSRta21xp6eSmfVjpVnDoHmSDXaTVZjlFfdDlCzA1wtlwi4jPnkB0dgo+R5H3ujGrvho+Ken9SxFypnF+PDzOyQnSPKicLcut39eeU349LjEn6aScsdYukTVKrcOPVqZNieDmx5CNSXPQoI7gggc8EEAjm2MT9b+h1e6CZ8pyoMuW/QJtQjuUWprX8xsh5JMV5Qt80c2P76PchePaP93rY5KebRzbkfjcdPML4NDC2WVsT97j8VAW002jVRinSS2TFeVHcJSCDZRQTi8KN8PeXcyZQzD1LqWamck5Vy9Ecfm1uQz4qPuosA201cFxVyBZJvrUlABWQE1F1TpzLObqqr8PYK/GyVJLYuLhwgke3OJN1J6q5i6i9Hcl9N5CItIodAS/OmswXVfOfj+SOXEnjyjde0c/MUhRJIGPn1BgVbiXYYq+QxxsALm/ev5cxdewrKymweKqw1sQc/No8/ZA3t59FWlEU3WqgzTqVDn1KpTFrMWIxELsp0C/8AiWtR127gXCe1z3N+/Dn0a6n1Pq9l+pPZSrmXI9DfXJflVKGWCBosWUpWbnVex9u/Jxoj4QPhwf6GZZNZq1Pp8yvZjaakyZ7JKnGY6kAtxkqP+LQDc2sFLubGwONNw6aiQ54hEbS4bXJ7kY9TWVfYzGQd742ANrWA8l4+TuPzXusdf4SmpCHRum0KU+kNuTKvL0+hdbiNpSQPcB1X8TjTnQeJSaL0HyIUS0NQ42Vqc4XXXNKQ34VtalEnsBdVz6Wxnz/CUdNZuYumuU8xIdKGMvV9Xi3dvUW25TBZSfsndS2D+Yxl+f1v6nSOgsP4eW6tHhUOKyuC/UGVOCbMp9yUw1EmzbdzZZTypACOBe9UVOK2kjia8NN7n0ub/FV5bon4pOvOVut/V+ZmnLzTCaFSYYolMlabO1Bpt1SzKV/kKWo7YPZuyzy5YXl/g9cn5EzDKrebZdPjSK5BkhqLIdsrw7BR/ih+4TzrVa54HbGd8ufDdmVzohm/rVUKBGNIp8eOaUt58pdfHimg6+2AD8sJKhzYnm3BvixPgbm1fLnUd2hxVsIjTkEyEhXDZRxwfa/FrY9bJQRMo5W0kmobl9LakX8wtOTKO6dV9Pmo8NloISpGm3YkcYw58ZPwrU52o1LrPlzM9PpzsxxoVCDOuG5btghOwRc7xAtosQbX8ljjW+es4ZV6Y5LmZ1znU3IlPhISSG063nnFkBtppH7ziiQAP42AJxgbqP1nldbc0JlTxOhRWdaKdTiptTcVFuSSk+Z1X7yvtYWSnn5DiXEFbw9TvqqNmd2xGlgOp9F3eD8Eix7E2UtVLkjO99z5N8/PZUzDyDJQkJkKZT7+e4xL6JT4NFaTGbW2t1pGorSPqC3FL/kbjDgqA24fXn744KWyFXLi7Wtb9b4+E41xpiGPMdHVPbY9BZfqLA+B8DwCXt6VpzeZujEyI6eQoX/PHTLjjsRgH4Yx/tiseNMYt+2Vjx4I+8vX/wArqV4SWD3tjniGD+8B+mPfhjANt048aY2PMHTfC5G/e+qN4+qGXYwb0osSf5YBuI2ypSkkAXve1vucGIp0dSrOEpJFzY24xpn4c/heZqpjZ+6hQ3PAp0vU2lvo4keqXnkn93sUoPfgqFrDHbwjBqjGp2wUzf8AUeQHU/vXkvPcR8S4fwzROq6t2v2W83HoPzOwGpRHw6/D1GrQi5+6iRLU0kPU6mvIsZfql11J/wAX6pSfq7njg668fDSkJFrJ4wH8ChElSlukq5J19zjxoEE8Ert/usff8GwanwSm7CAa8zzJ/fyX5C4l4kreJ6x1XVu/0gbNHQfmeaF+Jwx7DHDVIfv/ADx7+j0EfvO/xx4Zeg+7n6nHWsvPrwqsNPt/HHlVeHzjv9HYH+X/AM7HP6O08eb5n/OwMgUUR6tzWZfSvN8dhwIWqiTLH/kjjDT7CwouKsNRxvrOGUo1VynWqXECg9Op0mM2VcgLW0oDj8zj52ZjzfTcsURus1dS7upCWYyDZ2S7a5bTftb1V2SOTyQDZG9sUbnvOgTxxPnkEUYuTshV2o0+jsRpNUmNR0vym4req93VrISEJA7/AFAk+g74luQ6vMhKmxIS1pnQnWqzAPtIjkaxb7pt/wA04yXmzMVYzdNfn1GSN9SS2wlq4bio9ENj0sbEnuTyftfuUs3o/wDUOdW/N4hMeS4hAJKtwWeSABcm5c4+2OdBXtq3uyDZd+vwY4XFE55uTe/5L6a5arMPMlBgZgp5Bj1GM3Jasb2C0g2/Tt+mHJXmSQe2Kb+GqvpdyzUsmuOal5fmHw1+Lw37uNH8r7g/IDFyJ9sbwbi4Xn3tyuIXzy68ZVbyT1UrdK07cSQ746KkCwLT11WH5K1p/TFeF9sBKyeTxjWnxnZEaqFOo2eWGRuQXDT5Kh/ta/M2T+SwR/n4yk5FZJ7WGPzjxbh/8MxaSEjRxuPQ6/TZfr3gTGWYzgVPO43e0ZXerdPqLH4ohUpo8aucCElrHUwGebJwI09kkeTtySDwB7482CDoAvYF0aCH0a9KD50+lsXV0G6DT+pL7eaMyIch5cbWNCgSlyeR3S2fRHusd+yebkPPQT4Zms1Kj5xzxEcRR02ciw13Sqb7KX6hr7fv/wC577AjQY0RhuPHYbabZSG0IQkJSlIFgAB2AHpj6fwpwS6pIrcQHc+yw8/M+XTqvinH3tIZSB2GYM68mzn/AHeob1PU8vXYmlwKXRYLNMpcZmLGjNhpllpIShCR2AAwtDrY/fGO7aB+6Me22xyEi+PsTWhgs3ZfnwuL3Znm5XlKBAsfUY+O/WrM+VKv1w6oVvMb8kt/itdiO7EdySy/EYe2Eq7hEYBbBacUUrKjzYAXx9iV+n5jHwTzu94SNmfx9GbpkurT50yTOltPSExZC5zqmd0tg+Vwgm5CtJLf6c/EYxNG1p6pHOsF6RmHKkPPkbN9Fr0nbhurjU+GphxMphw/S2487uoASvc0BN7gNoXbtixKs1Tcxz6VUMusN0IUmooRPQ+2HFxw+VOFsWBQ0tTg5Cj9Gix44z3Wsu3qciiUuvN1OS8w1JbXo8O0XPDh5/UpZSjQ0Cu976hYjDxlOpQZ4mQMqy3GHWo6n1TQ4HH3kNoQonwjSQh1LagVFRIs2Te9r45stAHlksbjcC3wVbU+9UwUmmzYYqiqVIkvqgO1BiKwtThaaKjtRgENtm+oX5Purk4l/QWoriwqun8LEs06sRChO8WFSUPstGQ2pwAkfsmlIsOCn74pqqVOlSqBS2aZNMoQ1yX58pJCguYsNBW2QBZqyiQkCw1LsBc4nHRWRJarVSjBRUxIZYKhzp3232NKrdr7WpPvY46rIy2mylbqd3fC1v07TOZW5OnpcEmW9IkvM7gUloL2wlsEWuNAbIHpc4x515boievHUVuqMOuPO5kqL6H4s4KIK7bIcAJACVlRcbsF+UDjm+xsrPuw3NL0x0EMuCUq3LxAFy6PuE39rgewxmzPWUMiVnr11PrlcrMFVfpmYZEmHlKVGbeTXGwlBU2khxJ3LWWBcE3UEa18HE2UQ3c7p6rVWML2C3VQFzPeToXSx7JrfTxuU6p5BkVgqL8pLrrRDqlF5OhAUttlKNOkAMgjnkseUKHlWpwqbToziI1SluLbVMe8Q+9LfAKUtNNNmx7p0OFoga1XtiaZ0kQwy3Ay30Lbyy1U5TjNKjr8QzVHIy7ALnSw55EnhAYtouebW5pxBVRWahTmWQ1de3UEOR9D4cQoKSW03+UWzew9lKC73sLqeEFhI0J13v8A2XMkJuFP51ZqtHK6PCqU9FDrLjFRkREyVpakPtpcS244kGxW3uOAE+/5YuPpf1X8bR3MrZiQ6pJikNOJeduoAaVE25FkrtzxyRwMU/4KlzKStbviAmnNmUglXKWuSr8wAb254v7YUZRk1J980ihLmJlsrkbaWZG0ZkIoLj0Y9tf07lifMTb2GJU0zZhZai57TqVcubMp5czlFlVGj1qPBnN/MjOS1DVIDCkpcUAwm60cHQLeUK4FuRSUCnUuRIdgZnguqakfKd3HnWksi/JAWL37H9ztzfgYURJdSoVTi5npKozjNRO6qOvcCHCg6VIUeLOEJ4NiLKHcDCd7NT1YXHjhemnx0OpajlpAcAJJSXnBy64EAN6j6JFsCGGSIWvdVut4lLMy9F6E0y7EoOaKompiOX6WJrzehUlo6kxFOHzrDrYSWVXslaVoJNrYhFEhsyKbKMNpC6zRpZqUtlR23prDgCX2kH6CG9KTpI1pWbgkGwvDK81nNOSCmqshdWy802mWq9t+nkp23bf206gSf8k+5xHc80B6Mio5noMZDb0dwGYy2Cd1CGz4mwN7OABL6LHzI3hzYWYSP2ci5jbZgoPXolaqjiU/iDrMoRnNxLbHypT5QolfkPC3kbjZSo2OkC4sLF0aqRJccn8LiMkshyKh5QbRGKwAo3NyEEX4vfsL8cskusmVEYU7ckguMut33U24JuP43/Lng4dqTlx5DDGZ5MhtUJpl2TIcfC3SraKQ4jSjkr1utp786vTgm37Kq3TBWW6o7mldHjwEGJmV8vsMPaGEOLbduFoUPKkd+RYG5FjbDdOplVaU/SZbzsNyK9dTbdtK9YBva3KS2botYkEYlmbKKZSKfSXcxtJmZeknwlSYZdUwkoVqIZItZA+S6OL+e3lvg/NFZbz5+AZlbZfD8CEIlZqkKSJB3W3kpS7oGkj9om2o+YqToUm5CLGuRyFJHp1OrtPozzCp82qvxocV9tDIQ0ohvb8vusaUgji/KyTfDK5VocKbd5paHC4trTb9y1jf1/MH1t7YdcyUWVlyoxHqlObqESrJXUIVQgOqdjTmivSpwpcAcZcuLFtVynscIKMqJPq0mE/tf+tGVsrW+1uutL1hwOt8X1jSexBso/lhnNzJU7UheW6ZPXWsnzphmxYxd3tTjamAGSCdRIBOskBNue9zfFcZ6r1YqNTRUJFWedck6t9oGyG197Ni9gggiwBsmxHGLHy7kuQcq5zzHGqQdgUaHGLcxIKWpjb8kskJTzrtYG3cHtb1ib+Wst1iXJjtVFxuQhkLCXnTqcOqxcaT6i3Ojg83PHYsGVAhRKHmKRHqUapRywW2Ftq8M9rKLoINzY6/qTe6TcX4w9y5TFPzNLmPMgPbxWhtThWGHSb3Fv2m3fgnvcHA5GSksuts02DLnvKjuOOJUyLt7VipwBBIAsU3ue5P2wRBp7T69x0Fam0KQthLd3AO5c/QA3xosFVlUzyLmGswM3UzOMFzxE+MHZcDxDyztFttW4NVwSvQlyw55NhzhyzLXIKzH8XTpKEuNaW91Liu9tIPPPaxv2GItT6JXkS4E1r8SaUgLlRm/IGLBwX1EH5dyAeOe3Y84lDcMyKXJrz65viTLXZiWV3ZY3jqcGsq1k6r6SeO91ei5U1lX9RgrjpU409dmwIW20oBy3sf1w6UBanm3H4xcS6HUBCgbAX9yOb37Yn8yp1XMOTW6W7Dp5tJ1qXHbcQ/IQ1cNtOKLmggXJIAF+L20jDXlWNWmlSKhlBymR1uuNakTYZUktdxoUDrFiVDn05vgNab6ouCs3I/S3NtegU6JFrdKjinueIKJrjiW4q7hOvebSfJoUDYg98bA/wbU6ZG6oZ3y7LYUnYpLY3Ug7Ti2pAQoJV2WAV8Eehx8+Kyj8GjO0usOBxue74slgrGlwHyhN1EaE9hcA2VY8jGovhQ6vV7pNKZzxljLbFeZmwHYUmG0oam/OCSraJMc62+NwWUD3vi0yNjZ3zZMzdfXH0xy3r9sZQoPxuSa1GX4nIH4dMQbCI9OQVO+QqGlRIHNgOffB8f42mJC9LeSw4dSGyv8QabbStQ1FJUVeWyObr0g2Nr2xiOIU4Ni5aW0kr72G2q1RcjAj2xXnR/qW71Vok6vinJiR4s0wmilRVuKQkFR/LzAfx7YsNRsCcamuDxcKgiypf4pa6IWQomXml/NrlQaZKBxdpr5q/5pbH64y51IqL1OplDoEZbFnS7OkIdjtupIFm27hYI4JcNxz98XT8SFY/FepdHy62vW1R6fvuAc6XXlk8/fQ0n/n4zzntUypZ1mra80eElqAyn2DaLufxcUr+GI8F1gFrgORhKa6jLXTsoZgqO02gtUuVtpabDQ17RCeBx3IGNr9D2hE6QZQgE38BSGIf/AMK7f+jGL6siNBohZr0TXDlFsrY3Vtl9sPNki6POEEjRrHa5742V0OrdOzF0sodXpFNFPiPKlNpjCQuRtqRJcQpO4sBS+RfkeuM4im98D/shtviTp9FskqqX+FOprHtS8Hyy5SN/VQD41+o39B+ic2jwpO3Pze9+DNEKsoR7a5Kh/wAmko/5THzNkOlWtKtI0LJFvvjU3+EEzuar1XpOTm1lcfLVIDi0W0jxMw7ih/8ADQyn/OOMoabAX9/4Yz1L+0kN+S/VnsgwQYVw5HO4WfN3z8dvp+KUxFBcgKdcANx+uNwf4Oh9t6BnyMjkByl6v1TITjDTaCsgpb7d+cXz8NfxCPdCF5gW7lFNYTXG4oKROMbaLCnCDfSq99ZHp9OEppGRS9o82Fiup7R8LqcXwGWloxmkOSzbgbOB/BVPKHh248NaVtqQktFK0FKx5iOQoA+mLx6ydRmMxfDp0Yyol+01UJ+qyGzyWm22lw2L+1yXCPshR9sQbrr1CT1ozt/T4UddMWuI1D2FTDI8jd7K1FI9/b0xAqhHkrDCQ4pKWW0tjlR49h9uT5fvjv10tNBQtjgdmzW16b3C+NcJYLPiPFr6ivZ2ZjJda99SLW+pW0vixYl9UPh5yH1coLCpTdPDU2oFCNRjRpMVLT7iwOQGpDTYcP7t1E2AvjFM59pS16XW3ELBspKwoG49CMXh0G+KXNHRilOZamU9Vfy+t1TqIqn9h2K4olKy07YpAURyhQKSq54JJNbdTajSM4Z9q2bsv0qRTYVXeD3hXy1dty1jbb8gBt2A9/fCYa6mlc6V0lnADTzC5XF3D+L4VQfwx8N4I5HOZIOYd9kjcWumWoTVzHI859V1IhRWP9ylpltofybGEkOW1MSWnElBUCpBP2x2oJU2UIA4KcN62bMrbB0qU24hKv7OoKTqxyJKjLVum55rr7ZQ4RDi/DcdJUi+aMAfL9UZGafEl9lbSkFtZulXcH1xe/wmZTfk9RE1ghe3AbcA0HglaSNJ/Tn9B74hnULNWTM2SIVVy5lGdRpoaSmoLektLakEMpTdsNi6DqSTc+h98PfRHrHE6QZmkVSqUiRUqVUI5afjR3G23kOA3adbLnHF1gg9wr7DHtjxDSQ1D3RG7HAD0Nl+fm+z7FmtkMUZz62Gmv7ClNZzFEyb8cc2uS1BDMLO7K3XFfSll1pplZ/RL1/0w9fHVlaVR+q8TMLjZTFr1GjsNOW8q5UJx1t1oHtq2lsqt30gm1hxRvVfPtPz/wBT8y52plLfjQ69NTJbYkFCnEAsNtkKKFWPmSTx7pxfOTPi+yxXsj/+j/rplOVXG4zSGk1FqPHl76UCyFvsOkfNSON1s3PlJCSSD5OCeN8jmPOUE3v8V9E4m4YxE0lLV08BfaHspGX71iBqPMO5KoPh5zN/RPrhlStuOpSxHckGTq7bAYUpV/bhP8hiCiW5ImofNgqQpyS4NP0l1Zc0/wDz4cqxHytErs+Zk1VSRTXy43GTOSlt5tpfdFkKVYWJA57YZnXNmQhZF9wpH5Y6HEM9M4ww0zw7IDc7A3N/orPZjw9UYZDUCrZZzxa3pf8AUoa1K8bsEaSF3H3GD5KNte8geZYscOcoZXlUukyISpyKxFYW1N1tILDvzlFJQvVe4bIHKMNM9YDatPFsUY4+nfFF7s69wL+R5hU8A4NV4JjVQ2QEMdoL8xfRF6yG2GvXUL8/fCqQ44HVFKrG+G/VfQtaOE82w6J0vpCym9++MGGvDH2dzXR9rOFzYjRwGHZriT8VrT4IMyxX269lRVkS46kVJs3tux3CEOC3u27b9HR7Y07nHJeXs+0V3LeZ6a1MgyXGipCx9K0OBSVIPcEFIIPpbHz06BZ1byF1fyzWH3dqFIlilzvNYeEl2aUT/uVlpz/k8fSkIUFhp4aXEkhX2IvcfyOO/FWOlOa+o0XwLiTDJMKmhmaLZ2g/EaH6r5TdYWZZ6m5iDKdloVKQlppR/Zt7itIt/f8Ae+IkItSVFfiokpCn2ltgk9iQcaKz/Sei/W/OUzNOT841Ciz6okzH6e+y0pOsp3P7KSCUclN72PGIHUul2WqbJqlJT1EcRPprCJB8TQZXhEtk2BLzZNxwfpBIsfTnHocQxWkljaYZLNsNDpqOS5csxnkL76ndfTDoHnWidSOleW8yU1LJQ5TmWHmwQSxIaQEOtH7pWCPytiy247aPoQB+WPmr0QzlnL4dM0ORDmqmVCi1FDc6ZSWIFSU6rWkrS62S1oCykG4/eA9xjSqPjx6SqkxobKzLclLW0l2BLbksBxDO8tKnUCyClvkhViOxF+MeTrpacPJjkBB5dPJUOYScyv7NmV6JnDLtRy1mKlx6jTKmwuNKiSEakPNqHKSP+72HtjGcT/B50SPntuXJzxWZuVW3t5FKktN7xAPDLj/dxv0PlCyOCTi1Wvjm6VuxhNbp9TcZso60NlQ0i11/T9PI57cjA3vja6YsLUh+lVVpST50uMKQUdz5yU2SOCSSQBY4lHjBog4RuHxt9FG528k4fFvCpmV/hLzxTKfBjxordGbhRWG0BKEKW8220lI+xIsPtjB3wpZii5c6rIrlbkFqDCYdefcsToFrgBI+tSj5Uo9SR7Ei9Piu+JHKHWnJEXprlrMMeiiRUY8yoKnRpUjfYaBdabbEZCibubSr+tgBcm2Ka6dUHo9lGTIZz11BrH4g0jU6xBppijWV7QQkzG7rcKjpCAASSRbgnHZpsdZQYNM2JwdM/MQCeoA1PQLqYU2lfUNFeSIydSN7eSnHXTqTmPrqqYJEk0+ChGijwg5dEVQVdCl24UtVgFK7AGw4veicpU7MEfMbAltPMuRHNTqFC2jg3Fz6e3vjRD3/ANTwxILC6hn/AFpC+G40R0Om5Fm9tpWskggAXJtgthfw9So/jm6xn4xdzYKyxF8rlgSD8m47j+PuDj4pTR8QwwTRTuY7teZcND1225W5L61V4twXLU0s9OXxiDQAM3ANwL33vrfnc3VfoakaiQ4LEn1wJYkJ5K04nzcz4dykFU/P3mbLtlMxkHRyOLs9/wCOFiYfQh5ZbbRnxxVrm/h0j9DsWP8AHHijwPix1zM/qXu/+bHD99S/+j+6rMIetfd/njhaX/to/jiz0UvocbnaztZJsdT8QW/P5HGCmofQl5zbaTnMjkBapMZKVEGxsfD27i/5YDuBcUHNn9X9k3/NfAB9/wDo/uq0G4kBW4ATjllJfKFSUk2ub8BOLDlv9BYSlpVDzy6prkpZkQ1KV+QLQJ/h7n0wblrqH8O+UK6xmM5RzhWPAr1hqYWHIyHQNQcWEoSDpse5tf3sMaIuB8QkkAkc1rfJ11TP7XMBijc5geXWNhltc8hcnRW38OXw1vzlRs/dRo39WBDtOpbzf7U90vvJP7vqls9+CfQDWyW1oACVJAHpjK6fj8yGNpH9FqmlTiEOaVcFAWbDV6Dng8nnBMj/AAgmRGJBiuZUqQdQUXG4j1UBb08wuCU9wObWBx9awfDKHBacU9OdeZ5k9T+i/O/EmP1/E1Yayt+AGzR0H5nmtY7bn+2/ywDad/27GUYX+EAytNmR4reQK60iUspakPFtDB5A/aau/I4tx64apn+EhyLEiuTT0/zG620oodDbV1IVu7VrEj9+3I4sRzjs9o3quAWlbDDT/q/gO07/ALdjNbnxqZcRlj+kXgKQXtAcNGNabFVS3t6y6qMezYHclfphuo/xx0utJrBj5MkocojSpUiO47qeVHQvS46y23q3Qg31hPnHPFhg5w3cqNaSNFqQsvX8z+OKYduFb5OM0RfjKiz8p1TOELLsJcekLKJUTxavGJVdIbG0BY7gWCjz8j7i2G+B8b7NUlOxYWTdQjqO86XVBDLaQCpxR9EAc3+2K+3jdpdN2Thqdld/Vbqpl3pPQDWK5LL0p0lEKA2oJdluDnSL9k/2lHgD3NgfkJnmq1iu5tq0urvNqcZmyIzLbVwywyH1lDbST2QOD7k8n7aJzd8VnRTqPmarV3NvTvNMiXFKmVLXmQtMlLaigpjtBQ0puCbcXvc8kYq2p5g+HvOmYpNWZypnilGpuNuhr8VQ602dsN6g4VEkkti45Nz2x56rgxOpqHCUtbEBoL636nT6cl73B8R4fwuia7vOqD4iW2AH3W6333JAJVWo1AeuLb6TT/FZRfpilELpc95jy3BDToDyRf8A5RwfpiQyujvTyDQoeYHqBm5lmWAtlt6qJCx5Ndlcm3FjYgH1wlpOW6VSIj8rJEeRDNQSjUmpyVSgoIWoApA0hCxdQAJsb824xow6hkp5b3Fis2OY3R4jT9mwG4N9lpn4bc3rpuastyn3rNVVt3Lss+hdR5mFHjuSEj/lMbNBum/vj5p9NDOp4q9KZluKmvbdXiursFGXHI9uB5NsWHADY++PonlDMMTNeVqVmWGRtVOI3KA/slYuU/mDcfpjvMGXReOm71nJu6kZSRnfJFYywpQSufFW20o/uu90K/RQTj5xyWVsynGpgLbjalJcRb6VA2UP0Nx+mPqGq3PH88YD+Kfqj0b6FdV5WWc2dDqtVX6xGRXI0+HWSy3IS8twO/L3BYpdbUCLAeYH1x4fjDhWbHzFLTWztuDfoduXJfRvZ7xvBws2anrQTG6xGWxIOx3I0It8lWhbSLqQo2t6nGlegPw2LnJjZ36iRiIpKXoVJdRYu+oceB7J9Qg9+6uOMZnoPxn9AqRUE1mmdAK06/TrSGQ9XA+2T3C9CllJ027kWBt7Ys+F/hVaFNcYjQej9YdfeJQllNRilSbclSjqskD3OObw5wEKKb3jEe+4bAbX8/yHzXZ4u9qZxSm9zwgFjT4nHe3QW28zv0W/G0IbQEIGkD0wK/6Yxaf8IlGQ4hpfTaXrKNxYRKaVtg+pOoDnm3vY2vhMv/CQxkvLbPTCaQkaipMtnyg9r+bv/wB2Pq4w+pDQch+i+JGqiPNbcx7GEan/AIT+BSp7sCR0tnFbJWNQnMWWR6fVxhNQv8J5MzQ7Mby90QmShT2W5Mlb1fgRENsquN0l1Ys2CghazYI4J4Iuj6WVhsWqxj2y+Erc9cnGl0adUkpSTEjOvgK7eRBV/ox8IqbTadGimZmqJSajQK7HbblTlOuwnn5ZabcbbGtSUaWlkkJT5yDc2PfT3UP/AAtteq+UKnTMp9G245qdMejtVCr1du7CnWrbpYaSQsJ1E6dab2F7YxHK68Z3RSGKVAybliNFjxmIwWzSrlQbQUuOG7ikanAReybDSjbCbqvmlohUWa42+B/JK89ArRq8xyqU6LVqRMo1UqsJTcVcBlDjzKmbgNOwg8oNjbWm5bupDi+Tw2RiFdTMrSc7txsuU2tuzprT5lx4rMJe1GW6u0p2oyiG2wsbYDYYQvUNA5JF2Sn9eZsOM/GOQaQG5LSW30t0aJtvt6irSpKG27oupRIN/qUCbHDcOusSBaPC6b0+mttpcbSqkPyqepxLnfdTuutuWPIuPIQNNrYzMwz3Y5mvuRslbe+oSzOmU/6CvwqMKPKgKVTWnUyZUvdNUO5pVKbaBIZSVAoQ3cEhHIBBxNOmcKTSs9R5BU+qIJHhH2UgaXvlXUnz/wCWEnjn0xXuYesWTs5rYk5oy7mxUmJCESG41mNLyW7Obl3EyGHCsEk3QFpAvcW7Ytzo9V6fmCA9mVMltRkPPOeHSpolhxLQBSpRAXb6SLWHe/1Yd4kbT2kNzZbICM4C0zlJtbwWZEhCnHGJiHRcJ4NyRz35VwOT9uMUR1Dk5NY+J7M7rFHqgrMZ/W9UWoypAM3wgVpaijhweZKLu+QDW5zwjFzQK6hhxTXy0SWG3HWmkOISooKSkkJPIsvT5u3IGMsfEDmOk07rfnNKqW6moMZm8U4+HF2ebDDQDJSeyLpSvUDrubWAscctkRkuAeS3VfdZdJ8252qQyO/VYby2oFRU7AbfZLWl5t3lxqxSVlwlKklQPGnz3POKZhyIqpPgnZkhh1T2tp1fmbS0GzbUCQVEm1ucSXONddq7bDz/APVUtuLLUfxe6mI044pW20n2GruQCSFnuTiKwYjC5K5RQxJcNxrsbC/71uxPr+uN9LAImWAsuS85irCbqdUp0hMhmoyUFCwQUSFtlQ9UXHNlIuD+eHBTr1Irv43TVuNxvEl6EoK8+0tHY27EBRH6Yag61Jj7K2rqbXfV2Og+n/n74NiFSkqjLeJQhy4JXwkm38uBjQ5t1Zmd9pTB+CmoUiNUGAh2PNkPj5ZPyJKLFQ+2pCgoD2ULfTiKOU5hSxPZa2JN9t48kKt2UfsRx+mJFkmpSUyHsqutOLYmvtONoDZKvFoNmim39oFbdv8AKHthC+qPGqbjUhpww99bawm2rbJ1C1+9r/6PTFBblcnPhU4yJm2MiLTqo0pYqcBlymVCK6DeXCXy2sei9uxTf22+Lg4kGb1uUKqONsyVrp8tuAXlW52yCYkn9LqZWfYpB74p3LyY/wCJNQKoy48zvlPyHS04on/a1DsTwofewxYFap3UJ6juR1U+DmaJDpJkQJEZwRprsBa7NuKbWC28Ur4WGyCOQOwxU+KxzBMw5u6mCp0uFSJcqHUoaCIxNTo+2kbVhIBdYJ5O04iwBF9taexCjgjKUqDW11HJamJLMR5xcqlJdHmZkouW2gkGxsghP3Qm5tbCu34pERS5hMmbSpV0MrGl0Xjuh1kpNnEFSVIIBH1pHriPQs1LpL7UaqrclRmyTHlqXdSX7hSVK9bnSnkdz3ww1aQUxswp1mOyKdOdoVRjSY8aRICnEzE+VmQhwtB1N+UIb1KSu3cd72GENFoECnVmppWXEx/DL8Q02sbLJBTdDo/fbS5rbWBcnyrQfLiQ1diNWY8aZMqsRb8iHJnoaUuQlpoOulWlPJuLghZJuCFHSQcLZ4h0ikQ88Qpji6hLZWtlqJKYejSDHDm/vgmzRUy3YNOD5nNu4tR7wB3VaWZbPSDN1XobtGyXRo9NY3cvGQ1LWoEqeW6OdVxYi6UG9gb34BxCBTpr0KtR48Hclx3G57spVvlw0HUQlNuBcgrsCbAexwfJnyMw1ZmcJjr/AI9zdWX7AuOOeYW97E6ftb1w40etinVlX4oy6FMKAS+DpUmIs8hSViziNJIA543P00i7W3VDWCR5Kl9dplV6a9PM/dMqxFWmXNqtHd3NYKEIOl4IA50LOn3sQ2T3timG5Ca1VksZfRMjJY17bji0qfSdHzVLIACxYGwPYWBvzizc60ORXq/UI1JnzJUGnwxUKhOmSi9pQGU2Kjb0QLITbXdy33FS5SoYqUafMeU6iBEG7IeLZbSpawoNNlY4QtS/c+/tjQzwlyqc2ynNAerTVTcqMJcd1q/gVh9nUH2tuzvAII1E2Nvce2GrI7UQ1KS9oDi24xW24v6hyRfjjkKN/wAhh9pTi6VRYzMqy1x1ht0tjlfNybf27Ei/+ThmyVGEDPlSoMl5tbaGpbLTqOUqssKbct7Eev8AlfbFoHdzKv7SkrdJbqsZhBeeZ27ocLa7F1vi6Se/Pe47Ww5VdkMUGS0FEpRGWi5X7Djn9O+EtDf3Isbi6iXEE27cXw4zhvQ3gnlSmyAPuUm2IBog5MGUagtFFZeXbVGqK/TgoJSo/n3OJw3HRHkuMNNISEi10gA27o/kcVtlFQXTagzrvtymjZXNrtA/6MWNGeU48ypZ+q4t+n/hiN7zk0vgCbc4wo0jLtRfVGQp1umvutLIHkIF7j+GH/oH1syl0xynNblQ5orHinJTyEfMZqCC38lo3VZAANj5D6r5OGytRy/SpEZNwhynyW+PcN2A/kcUmw8ldNZWjUVbY5t/34M0EdQ3s5BcKtpI1CuHM3WLOOZny87LaiMMkOMsRUmzZQbpWFrJWSOOf8kccYMb6n5qkwU79VdIbQFSVqWPOhDgc545B0p/K1h6jFZRXipQSi93GyL+2LK6PZIe6kZ6yRkptrWK9WI1PfCBa8dbwLiz/uGw7f8ALBio4AMgYLeiujke0nKTroV9nfhmoNRoXQ3JzdbSymqVCmt1Oftt6AZEkbyvL6WCgLfbFpHtgmMy2ww2yw2ENNpCEJHYJHAH8MRrqlmc5P6eZgzChRS9EhOCOR/ty/I1/wDOpOFygGzdkTusr1OtpzFnvNOdHHdcdc50tKJ1fJY+Wn9NLQxWb1ey5RTDm5lrkOBIrc0xICJLbrvi5i/MEFLQK9sEgLUP7QA5ViSykCi5C8JrIXMLcUX7kE+Y/c6Er/jjO3V6qSY/U7LzZqFZgt0WiobbXAhtSEFUl7emJebcUNaFNpithQI0nnunFU73RMLmbrcW5WWU2zNTa5SY6p9fzFT68/UX4xclU5Cg222glWktG5ZbAACEGwtx376t+D2rw3uglBjPTGm5LlRqDW0pVnQtcgqA09wfN64xH0Scp8/NM2pUuMwYr8eY03abvuMNoW3paLlhulI+WSQbHVz2xvn4bEoGT5KQhAUmtruoAA8tsnApJnTNu8WKyOavnh8S9cfzR12z1WrbjRrT8VspVf5ccJYT+lmjiuNpyW5dpCrptq0tlX/RT/0sP+apyZ+Yq1Uy0FKlVObJUSeFBUt5X+nG2v8AB2U5LfT7NtbCAlcyuR42oC10NxAbf85asc+KM1Eha5fs3FuIP+B+GIqtsWcxtYwC9t7DzWG6dl1ya4AyyoLHqW3Lf9HE7ovTZ53bEkrQVcDSDcn+yBbzY+oOfuo+TOlOVJGcM7VDZp7JDTbbaA6/KdV9LTKb/MWf5C5NhyMO9Xfi+6m9RnZMHJtSTkPLqhbVEVaoKR3O5IsdBPs1YD+2vBfh73Psxy+SVHtvqagaUtv/AJ3/ACUIn9HZ7NO3ltIYVtlxAlOoZ3Lc2AWQb4ktO6IPTILb4pniSUDUIrqX1JuOCQkmwxmXMGc+mFFkuS6i3JrlSkEqddee3ypdr63HFkk+v35w1R/iPg0yYF0PL1NgtpNxo3A7e3fUCLX+2Hkw2aSMRl684Pac+KoM7IbOPmtb/wDoAkqf2RSghRB8pNiPXnDM/wBDpYmhp9CW0fWUh21weAAr05/884qjK/x058obCWImZ6w02LFtBmlxH6IdDiCO/GLdyd/hFKvHWlut0im1pDqwD4qmtAnt/jGdPt6oxkbhtbTm8ZB+P6q2T2oS1TSyaK481yL0AqMn5LjCF7Y5Wp1CQfuOe2Ayvh8rI3W2KRrKBqu3LaPH6nn9MX5lH45MhZoqEGJK6bs+GlEiQ9AnMPuxxf8AaKjOIQ4tHrdvWfYHGm6V/QyuQGKlSmqVMhS2i6y6002W3G1d7cdr9/v35xyKp9TTv/mtWuD2o1kTcjI9AvmXN6I1xhu6Ke/ca9SboJFvtfFfVzKNWpbpQs+YK5So2Kfa4x9fhQ8tOA2olPX6XMVs8f8AN7YRS+nvTqpm9RyVl6So8EvU1hZ/mnEp6xznWcujT+1t0Ls0tPm+I/RfGhbC0X1JKSnngHHghepR0rP5IPOPsG90U6MvKKX+lGT1kjuqhxTf/wCTDPUfh96KodKB0ZyQpKvegRwP5Jx3aaL3s2afouufbXStF30jv6x+i+S2852LL5/5M4NLa16bx5Kvb5K8fUtz4dugar6+h+RL+34EwP8ARhO58Nvw+uCx6IZI/wA2jtj+4jG52FOI3H1VbfbVhPi92kHxavl8G5SCdMZ/T/vSsAdL7oH9XfCk/wDulY+navhj+HlRseimTwf8mAR/9vhOr4Wfh2cNj0ZywB9mXk/3O4AwqS1rj6oN9s2C5u093kv/APH9V8yUoddOksOjT7tqwojPuoBDragkK4NsfSh34TPhxWLno/QR/uFyU/3PYgvWn4U+itO6UZwqmT+nsSl1mm0SZUIL8eXK1peZb3B5VOlBBShQsQe+AMNlju4EaequPtYwPF3MpHQSNDiBqBpfnusKOLU+hSWlFKlpKEr/ALJtwf0x9VumWaU53yHlXOxWSqs0mJNcUDzuqaAc5/3aV4+TqXm1KCEHyrWS3+n04+ivwXVgVj4faPDN1qpM+qU0EHsgPKdbH/NfTbFmHyd9zTzF1z/a1hLYcJgqGDwSW+Dv7hU30Ty+8nL1MdzHQammROjIgtzGaVJZIQ62T5XQgI0OG5WkXBFzz5sMjNP8F1KZqcdOxT6rGsppxpqQyqQ2gRG20No5Q2201GC72+Zuo44xevTvrZlWbTsn0eVm92JLpeWIFGYbDLy1vSHIo35rQRdBDQQWUFVvOXSeLE0zUQulOUWnVWfUGlqhoalQ2PDNtGMgglCnrkWLrbA2rjcs6VKAUTiytmEkOQ8l8Hp25Zy9VNmKk5lkzqlmqZT6exCiOeCDrjGywxtr20NstkkaA4SQ2L2W4q1+bQbNlBnzYD9JRFTKZZhuogQo4QiPFbcB8VL3LAOWc2jdwrOgIbHBINpVV96oVjJ2Vs11F1qi0+NIzFUQhCtuXYaW0NtEAuOHU4AnuN4+xOEcKSrOdHzR1PpMZMOg5oebcjQ0OWLURDRS64eTtB54BQACWg4o21C5xjmYynY1wO6tie+V7hbRCqVJR+GqQ8sKhpQx4tSHvLtttuWcs3p0ArKEaefISTbAczraoVUVSJFJlsFHGiQ3Z2I1wQu173c1XFisgOc9rFxXUp853LzqZ8uaJkePGAcU3pcfQoqcDrdzt3DgAuAPKseoworcaO3XnMyZhzI061OrExyBMdhOOnds2lsbF2/mFZDaAo8ANmxAF+U6HO/fRO7ugqMx6HBmyvAzIK0PJIDvzN1LLQCVNqIWjQjUE31enIsdXBeeaZHy3k2tOsT50tynGNJXCZlKSl6E38597bKruFopbAHFwCB9IvKfwJOcq49lGA3ErM91qPT6Q2bSXZcn5RfaSpbhQNoBwL1NeU67bY5xD87phwMnTcyUmibtIoTxa36bUSHY8N35dRbdacsiQy7qBCR8xrZbWABcY0MYWPyk3RbfLcpky9Lq9bq8yptU2NTpl340Vlua5fxdy7JlOhFmjtNG1k67FvglarmX5LqyKdS5QXrUysR3Vpl6kqbBTay3LX7q1+UEr1LuDcAQrIC0ZZoWS1VeTLkSKxGDTryyFNa1tHbQFoVoaRdMVOlVi4AvglJvNqemZFde2WUNQ6PJXurjujabWW1XbcJVdCxuBsEceYW7YtliY+4KZveSk1689ENVNpi3kN7nyY6A5HbIskKU2kFvU3fhS7qABseTh9opeYKnEIbdS6DpUHDo0alW4/fsT6geoviGo/qVHjPQ6dV5G+4SHnWmy4VoIB3iHLuOnyd7XCkgcC+Hmn1NUNloR8u1xTKPKyURkEls9lELcBBJ1d/Sx9cYnxBrbBONCpWo+GbWh8NhSHA5y0FWWCDYj78d8NlRSpcRLKENLkOOFSGpTetI5/aAEEBHHF/5YIlzprkV6+W6qlQTdZcQwQoo5+ne47/ft+mGWo5gkxISqkmmz0NLGgKW7FRq47ftrm/tx64p7MyEZSrHvy63SmouO1mqNx0R7u7hU3dvSpKB30N+nY39APJ2HLY7siQ7Dprup1hsOrAQJCHDY3ccA0882AUbk9r9i3Q67LkQk7cUhJC0qcM5hpKfVOoAqJAPpb29r4cIeW6lIEaWuBTw5HLCmn1Trg7fmbKwWbrJUvk+wAHPcNjLDZ5WbNmSWHLe/EnYJ8O8sRjLfXJCUpZAc0+ZpdxdxdgEuXB2yBb0TZqaqkbLEqjQK23Hq3hghpCYaWxufsztptZGpHYHWCdVlIJGHyHFqSIUpFal0+kCnlp5U1xDj9igkhIaQFI0XJUQQTYbgT5bYTVGq5qYo0lcGVEUFtus+Hcy84ZZjkpc0FKFBttlSHB5tACLk25Bxta3UWT2HNNFFy7lRprLsuntU9uXOHhgIzjpfStDLhf3XkeRDillvyjzpNuUFJOAVxQiynKbMqonUVdUit1OWZDzTUBA222w82FfMIKjtti51ga7jkdqM3MNKfpzRoMOS3VWnUTU1VxCjUnGkl0yXFCMAENgkhTjri7HQbA3wDJ1Lf6vZVzc5QmaZSJEirMMtRHXnUxUNRd19bbKtta9qzSym4Sord83JsLZHNjPaOdp8/wSWaNVM8qGjxs4RKAyhyTKnR/wsT5zTSj4ctKcbklokArs2kLCiQNX1XGIfn6BmfKsWJ1dns1CgVCVKtElKUGXnnC6Aq6UXIQ4CeVbQOq4B7nRnT2i9NmaGazCp0ObV5phJqz0lDKas3JCFBmM22VXbABdCEt8L+YSSbgZn645XpCK1VVSc71eoOS6it9uHOY0RHHHH3XHoqnGyUObSPKG2xr45tYEYMHxsY2+ZjYnNERy3Ol/T99FvnovdmNkJvforc6i/huQ8nqruUIEedlmszaNLjRN0Dd3S42VuvDzlxuy0oAXYf7WbXNO17qBHoNRZptElmmVSO6iVM1OoN2H0jUh5zykcdvICOebm2H2uUyq0rollVdVfbnRpFfRKiMuoaUGWy0+42FFYO6QAPMo3OLHpOZel1VpFHiS8q5fl1tMJcKIhNHjqEea+Clx76QS6NW2j5hAQoXAIOjewxtlAcVTkbKNTZZk/wDrepmXcw5zqMPdZmOmVAaEhplLDZnJSAndSpso0K5BRbRwj3wuyvUI9UpjU5mBGXMmOuPwllYAhMIcS2SnQkIRe3BSPS55vec9LKgvLc6iQI0WlvoW65TGm6tFLzJ3EuNKbWkXXf6gCDcEXGGnL2VW6ZU6vKp/iWlQCp5wCUXwCZaSFtJ4GlS76EEgEelsaJ543Hszus/YloEltL2Wj8yCTCyjRqAunQ2Yj8h1EVLTOl1IaQ4fMogX4Krk9yRziDQo1aaYR4yjOKpTkZppl5B3FvOr3NxIaAuBYCxPqPTCjrlmmYHYIoyn3XKDUm5TyUSBYg3ccacAvoGgC9za4HNhic5XeH4SIsdR0xH3WQU9iNV0m/r5CMNSOzXYOSpm3ukkOkVLJLNIzHVKszVH4jqBMUw2QSweLquTc6FFBJ5PBIBvjW/w019IpFYyM89qVQ5pfic94sglQt9gvc/iMZsdgpnw36a5YJltlnk2AvwP52xK+gmc3KHmrLFWmrDYmbmWamFWGldxsk/koNfoVY1m7TdQWfHZbUsCMYo/wnXR2Xnnp5lXP1Dp7siqZXq4p7+y0VLVBqBS0e3fTITFVzwOffG1x2xEOrXT6mdV+mmZem9YUERcyUyRTi7a5ZW4ght0fdC9Kx90jFgWcGy+F+VqFRmOoVVomaa9ttU0TaZpLZBkLQCCAQo7dubJBJUSBbnFk5XyRQ8hQ5Valxn3Zbh1sszNK3mGx+zaIAALnbj9y/vc4ilFy9W8kVhbkvLficww5C2UQVs6URagFqjuLcI5LbZaceOnknb7d8T+ctufHiPVyrxIaWRuFDTRLj7norbKjo5v5fPz646lGGUhMu7uXQeaw1eecCMaDn1PkmqTJMVhx6oyNUhwl6UvVe7h9BfuB9I+ww1tzCpTLCbl+Y7c3F/Of7gEcfaxODpVTyg9pWlidUGySpLqnLJPvaxSLfphv/GKZVFrhwqI4CtGgIacbCnb8EXDWvtbi5++LKiplqG/zJAFKena1+SNt7qA5yrKJFbmyo7wUyZLpaWFXSUaiLpPse4/PCjLefM7ZWpdboWW8wVCl0/M0UQqu1FWhKZ0exG24rSSBZShwR9RHY4Lm0qq1OemWvKEqPHccS2h1Mwg61kBPC0g8kjm1uQexw4U3KVVYa8TIy/LiNpHnSuoQ3bc/dTaj64yS4nTg5S/ZerpeDcYnANPETfpZN0dL0NLcyBJejuoIKVs+Ut2+4wjnU+O8049IjBxwouVWuTx74mqm8iU6FJTX8yPMztjWxHj0hTrJcC1lTbjoe1i6Eo0LQ0RuGyvINeIrOrFBa1GDNUVIOtsqZWDfuL9x3GLYpGTNzM1C4uI0FThU5pqpuVw5f7JLJyCuAuBGFNTJkPsr0NRUuKUmwuQQOexvx98IpWT47TOuVBdiq7/ADA40f8A5wMTBnpe9mnI7GfXupOUYBbkvtqpkyvO0+qu6CG3HgVs7SEBZAF3eb+l7YiPhRTgGonUCvRFNEp2vxpVgQbEaSben/04mg3CxNd1Ka2ctUVDPiFRdzb5VqNwr/z/AN2JP07zQzRK4WlOhEKZFkx1BtIITILV23bAfWoNbP6t+owipjD1SWWIlb/FFlBcKAyyt5KAeXNSAk8cXvcc/rgh2iTIc5qXAhKLyHEONuboAQ4hYINgq9wR6/z9ctRkcbXWmCOdo7RrDbrY2+a+nXwkfDh0t6sdGsv9Ts80usT6xV25AecTWJLDCG9+2yhppaQEfJaN1Ekm/POMn/4SPp5kLp58QMCn5ZoECJEzDlWHUpii+7dUwS32VOWKidxaGmk3vayT5T3xtT4IRRax8PGXWkSI6pFLkVBlyNGmfNhLckqUEOoQbIKkEkAjlBBHGMdf4VGMxB6+ZSS03IWI2RIxQi1gn/1jLtb+BP6YwXZsxqtkbLb+Z8lis012TKEONZRb585ulItzc+p7fxx6kwnWHxGeSABwdKrj8gcEtz1VB5xcVZjlQujykEjuCr7X9cObJcDKHXnNZbAKQBYC2GZcaLKpCTcKdsQRxbse9u2OuC7hUlNkFA4GJVnqiCHWJVQitFLUhZ3kejTtxe3sFcG354iyW1JC799P92I12YXVx0NkNqW60/uoWtKkeqTYgjkEH0IPbEtlON5nosnMitoVGPVQ1PSgBJLcgN7b35bxUk+xdHoeIV/jQvgHTzxhwy/UmKbWCuegqhyWTGl6e5YX9Q/ThQ+4GA4aXURjkRQbfjcMy4ckOApNipsn09bg2P6/bF05FqSapRaW/wCJaC3zMi3VbSzNWSXI7guPkvj5g9AXFWvpOKzqFDkozbPoE55sTGpCI+92Dy3AFNPf7h1BSr+I9MHZCqLNKzVJyzmnchQapIXTKgv96FJBUlqSPu06bn0KCv0OKXjOwpmd0qXdZKZAdqEHMz1ORJEiHHTKWsEPJQhYCPPwQQVlJPqO/YYrh6nohV4opW3JZkIW22xMs40GltFKkquQTYE25vfQRyLYsvPip1SyNOpVYGmsUIusytB1KejtOWetbvZaR+jh98VPHrMZ5F4y2pCI8pt3YdTcuWXzYeoskA2Pqkj0xmpiRfyW17BlBKX0d7MMChKoUGJeHTih51SFB590LCXNTiSQstkAAlB/dHAOFjNYZqzFQpr1RipkTIrkUPVd1aWVNlxp1orDdjugtN2ukiwvwbHDTUlvP1N+rU+oxilZcaRHjNhCmtt07QSoEXSW1ADUOdOgm4F2XMqszI8DJp6HJseQltltAQHHY7aCVhAIBO2Cpwj2BsTbGhsQduFnLj4Sp3lZdNzHMp1EqOX98uCoByoQV8vbTanHHE+hXo5BFri3viL1hDcjM39G5ESW6HHXYaX2mk7br5QC053uBcBRB4ABsCezhkiv5hynVYMqpwYrDeXq2hNQ8MEmQ+xMZU244QFbbidsXCvrubduA05+eqNBzrHlZemuxpaG2jHejOWbWEOLAdIJFvIpP2KDY83vY1nJK45XaKTTzLboNRpMN5hyuzY4anR45c3itxtSWGQrsTwLAWvqHe+GnqFSYOScvt9LadOfdfo4an5iKHdImVRwN/KCPaM2QgX7ErPHOHvIdMzXmTqY5RsoNRlVt+XGRAVNJaYLseMpwu6iOLFJ5t7C+IojL9TgR8x5XzVCmsZhbUXSiQsFaZaCLrcX++CCQSDYhxJ/dxa1uXu3SgZk5pjibTKlJivbiIjDc5pSyEl2Ms2C+/dN7rH2UMNWWkPSs6NPIHmQmTHWlIsdvbPP5ev8ME5Qq6oKIk/QXmGUoIjOWII1nU0R7EakkHsThyyXFMTrdQ6PB1vR5kpoRkqVYvNOxzZBP9v9zn1Ti21wqftJ/p5SCXdIG5JC+O3IPbC93zN6b3JA7C+C6vSU5aq71MQ/vM/1d1py3OhYB59yCTg1gpJSr6xfn74dhDm3CD/EoNlJxoLrDdtK7x3D6Ad0nFiQXULQ0lfK7oWj9QR/pxV9FcELMdcghSzZo9z3s9e/8Diawq/AjlnxTwa+WNRJ/wDN+18Laz1HasUrcG6ynSi/7Ro/kQRf+f8APFCwkkUVplVkFIDZKQb/AHJxb6szxYNUepcl9TUtmWWi0ppepso0khQ9O4H54qUNLaMxnSUluS+iwtwEOEX/AIYtIVYPMJxgXIbdPojt+mNxf4MLIAzL1djZykR5DsXKVKqLu6tjS0ia68GmUpVfznYdfJuBpNhz3xhynklKO9lcXP8Adj61f4KvJb9C6D1nN8pLgVmfMMhyOFpt/V46EsAj83EO/wAMQHK0lXR6ahbVSLJAxRfxVVZbeWqHlho2XVqkHXB7tRxq/wCsU1i9BwOcZX66VM13rG1Tmiot0GA0yRq43XSXVfkbFofpikK6Jt3Krc5oS/UKNl3VpbSguunvpB8t/wBEJcOMyZpzV+NZqq+bqSrciyIa5EppTO98lxYFtJSpBCUNtm4IsLngYvHqJW32hmuvQ3buRo3gYR/96spYT+nLqv0xl0GfR5dXkyVstUmvMuNFxEkNbSC4FbSdB8iNwBCx/YKvY3yT991gtpe0AlS74dnKlL6j5hferDcmAumOvRY6JCSGCuUndCWRy00l1LiWwRoKPoPBx9DPhkWV5frTOq+3V2V/lqjj/wDJxgroLJjKzbmIQ6RDgwzBjuwGqep3w0WN8tJaO42m7hc8xWCdVjquRrO5vhjnsMwMyqkuBDTEqFIcUo2CEhDtyfsAnFsOrdljc5oN184V0t+TPlplvNsNIfkJUt4pSB89fvizMofFnU/h/wAgy8p5ARS3n35yqi7LmI1gq2ktlsNkgFNkjn3OKp60ZvocOs1enURAmQmqjMXEfsNDwVJUUugHm1jx+WM913ManEPTZAICQbBXb8sJS0vZ/wA3mV9H434+m4gjGHw2EDbfEjRWX1P+MPqTn6vozD1AmQa7IYTtxUPx9LUZu/KWW0EIbv6kcn1JxW+ZfiWztVoqYtGjwqEko23VRG7uKFu2o9u5xU82W7MfW84u5P8AD/wwlT3xrvbZfMrJxeq1SqDqlS5jzynDclar3wsQy4pIvf7A/wCjAqBSfHvIK0qKfqV68X7jEpcpCm2/oWUcCxHH5XxfEbC5VblHCX27aXr282knjCqLXqhBSf60vm3F+/2w7LoyBZ5SdA9B3J+18JJ9Ib21LCQkex98PZpS6hPVB6jyEuphy3nU2ILLzS9LjSxbkEc42R8MPxxZw6TVtil5gmKq1BqLoDqHHbJPFtV+dt3j9p2N/OD3HzzeToc8hWD6fbEuynX1FbFNqTqNDpG04f3T74zuAd4lcHL9C+XOsFMz3k+mZzyc407FkzvCPtk65DI9g1xdw+Q6TY6CSnVxdyfz9VJNOlvU+gIbeRGlOtiQ8eFNbY0qSEEhV3TdB/smxI5x8nfhX+KesfD/AJ3bjV6W+7S1luNMjhz9sxY2Fu3GoKbc9LW+hRGPq/0+65dLOpOV42cMt51pjkGQ2FKS9JQ07HV6tupJ8jg9R/eLHFRY1qe6E/1CcjSpaF01a0RnEsNWVoKnAXwoLJ4BVsjQn1Kkc+bgcHN8itVcUOoUYCM+6WkKLiVc2eKSQORzHcv2t5Pc2kKM7ZMXbbzdRVX7WqDR/wDtsLWqzRlpBYqsNafTRIQR/fiAAahAqNTv26/QgkHCYBV/qwplaVSHVXBuokc34vhOArG9p7oWJ268G1H74CW1BXbArKx7BQXikji2G2v0xFUotSpjqAUy4MqMq4487C0//bYclDAHPK26SfKltwn8ggnCud3StNK5zZ2HoR+IXxnpzK1QYitPKo7VlBKu+2jG/PgAfS50zr0FZISzmdKgnT9G5BZV2/zL4wFTpLjVOh6XV3TGZ51d/KMb2/we7rruQsyvPPKUj+k0ZOoqKv8A9XtqP8lY4VFczW8iv1V7V25uFy533mfiqEy3OmSsqwMxJpVLhCXSmoTzkpxuQw06S6443rWHLXDgcvwVEc6QnBuZqzkqZ0jSxHqOU8wVZipGqiBTqpGQu7A0pGw1YuHQnkOCxBJNzbGdKfT24xp8SZk6nyIQgc1OQoQ5rhFypts6vIu9h7Kt3w95on1qfDZp2T8o0zL9PagsNOuP0eIuVMfWmzqnHw04WkG6gNtaDYA+dRIBdh4kkuSvzh732cXZtj57qS0Dq5lun5VzVHqNDmozhW1SfwZb9JR4RO2z81Z5AbQRcIsLOFRsBwcJ+lb8Gr/DxUUTpFGy28JS6bTG59SRHDjrSm3LutE61tpDqkhRBR5j3AxFsyP5onPOiDLrEuNGbQYbTNSKGGyhCQ4tDQOtAUSSNwD1A9cEZLznn6lxZEehorBkSZLTrbTTkVAecQohzceePka29s3URYk2B7jZJQdtAchGiwtq/d5bOGimPTBNFi9S6EtqrwZEiSlqEtukONreTy0ouBY+UttKwooXZZIbXexSBiwpeeM49OOoPTvL88xECqyIeYGpDDwWTCdnlLIS6+CtsqDiS4kjX8wg3IQsRFVBztRK1EzTmR+RJr9LqTrcWRGqLRfiERnW0rUXXACGluNlAb+o8aQFE4jVVrGXKHV6zMYbfeq7bxlsyUNNqUVoLZbdUoFLaFkpJ5AJJI9MYoaPK/s3m55J6upZM7tGCwV557zvVMu0UZholDbpeZM6MyVKYVveDpxBS0G1JWoLcs4l61ySNy9iEgYgfVWouUrp/nWGvKs2oMOSZNNDs9zTGu5GjpfeaAALbiWi35E8r3eDwSmhqxmbPecBGTmCpiSaY2t5gPvR5Hh20EqceFg4sALJUtZt91Ymon15qgPpm5brco5gozsR5MV5wEvu6bTtog3JaioShzQB8ooBvjazDnQP7Qm6ze8sk0KVZZqDlSpUnL82BvNK2pMFbJC2S4Fp0tvjuEcXb0nW2fp1gkGW5GcbdmUl2pqjpWKwihrUZKH23VrBIWHEKusNhxSVnQdKGeTwu0c6aR8y5Socmow8i1iqrbmEuxXJAanOuOMqS3NY+RYlk6iUuBBFkcuekji/iuW5jFCk/EE9l59wNvRoND6a+IVD3LqSEvPvMI3fOS4sXGtR83NsK+lsS1+ifthbuJdFcRWoNblwaRNWWPEORHPDPJTUvCOEubF02skaSbclCknTYk4BHzNDYcbrG9DUyUBbe5IP9Ycun5aRex+rknn6R6jFi57zBlZjp9Oy8zWZma4dAj7cuQ+9NgSGZs+KWTIebeU624N5iOA3GWENNKdbXcquaHomcK31HYk0kZcQ23PiQy8+qsKSWGEFJZWyhoHzktd/OhXnPFhbC+gBBN9ETMArEj1WZJiqSzLCYzbQJb2BdAv5bk8kEdj+XOEcpZmQlQJrq3Jm3ubTF3Cm/CfS1v8AuOFWXcqxP6KIpSHGHUtyFyzOdk7u1IbUoNaR8tBQAdJ3ATyoi5taR0Gj5cjQpj8NxyYtpoSUGbVA2FhtoBSG2m3QHS44FuaVXsjQOVlZPFmlhogXBw09VdGO2IAUGQzPpDRakQpDV7uHW1pS4G7Xtf8AO/B5AF+2HCPW6kw6luTJCorq17boUAXVki/lHFwSObe36ra/l5moJhvrgvtobio8SuGJMsvIGk38KHS0Co3B9bcIH748MzUrKzrFan5ipLNMLzTdQfqsANPpvJbcsHHNLTDZACUeQuBbYJJFrboI46ghxduq3PGpCd47VQaZnR3qTVGHY7RbkuSqeWmkwtAadaUHEAeXWlV+XE/LItYHDllel1utZln5k/AqpUnFqXBq9QnSGnZjkItxRFspZbLgZXvNuFKCVIUk8kXwZ1W6y5OzfkGWnJ2ZosqPBKE+JpdbYeVDS5dADpJcQypVtQUpsoUGzYEkWyJPoVRztneVXKquTL8U2FpqVQjtNJlrX8ptptwattsOJ4CR5UJIt79VuGlrsoOiEVSw6vGyurMuaGma3Ho9UdmZPkZfNYjUxwttzX0w0Pf6rQybNobUveShTvbbuL7ZOHro3JquTKdA/wDSRGr1MpsZcOSqmxMoVSqPPl9lTgfL8ezTqA1tfMG4FF4emKHocHJEms02mzZUm1UkttBBg7d4kYFL7bzeoaNRbdAtcq47G9/ptljOFFyn0Em5kb6i5TpGavwtE6VUIcz8TWp1Bu0hyHHfZ8S7s7TKEpIOsjvbGqSMMOZm6R8ubQqjInUPIVDyoTlnqBQJdWnTXFR3JVGn016EtxW4luS47DdbBFtBDmhtOtfbk4gufuoecM7ZdGXqznKmVYFtCXYzdBhJEYrNlNsu3U6LoCRuJA1AkDjFX5yr9f6q52X1JzJVWpNJbLseLCEx58wlgk+GaecLhdQLmx4IWVkg21mYZN+H1Ge11Cs9QMjZ1pT6ICKhGlUTMcAQm4AaUW5RE2KUOIGkj5bt1EHgc2Slwqlp2uLdMxzG3M+qt/iM9sulk/dQIcqrdNsjZWyxXaciYw84+86mW28mIhqM6nUoIS4T5nBxoFz+uKeiGLHrFKzGx1WoAbpxXJZRMgVRt5bxbUltZtHKAgagq1/cc4sHJ2UNnXRumuQ8xZxXUxoDPimg4Ii2SkuS5TLIQ3qO4AbBCbhCFrIuZNnHo1TcpUHK9UzZk+q5NrspLaabRns0pqrIpzTYLi3WVsgxiFutt6S4Va7nmysGOkpmcru1TPiqXxe9AAN23F/gL3+irVeW5aKHT6SabHrtUr9aW3HD8ZThfus2sjcaQXHFuNkklsJBWeLGzzVstdTekMuk0+udPqBTfxN68FTUKO80+5HWFWS43IVoKVrSuy7X9L+lt9DYuUZ2YHG+qWR8yVTLk8eEYC8nSV0hLoN0SBUW3ra7A61LA+q3FicaIa6OfDg6B+HdNKiUqF3jAqsyFssi1zfx6PWwFv4Y2xGNzfBt1CyVtNPC9t5PEL6HT/dYvjP9XOp0uoPU/pe1NkR1sePXTwiC7uLbJbQv+sBDhIK1AFtRF/viyekteerNLW7JjrjyHWg4+ytZUpp9p1Ud9tVwOQtCb3Fx2ONKO9Ieic6pzZcmBnTL+6Q3upr8pqMpbbaCQnRIeR2Uk9hze9zimM9ZMyH0qz3R6TkTNjtWjz40l2dHlTWXZMV11y6VuBDTawl22oLUglS0q8xxW+OInM1oB8lRG2XMSdQnZvlNlYbEMqarlUo6F7aqxFFTiqTxoltH5lvv+9jrFSv3uBhNW56orMOvs/tqJKRJta+thfldH5WI/niuaI5Vop3jNlPNbv6eZnRnPJNFzOkWXUIbbrqf7DoFnE/osKH6YkKhdJGKL+GLMDQbzBkjcBRAkipwbnkxpHcD7BxJP/KDF7etsUt1Cj25XEL5bfHplqZk7rc7GSwy3QsxxjmJsHhC3VnZloUSQhNnQ24TfnfHfnGcKME1h+bLelREQqcNpxthxzVuXA2gUJHn9tJtwcfRD/CedLl5v6HwuoUFs+NyHU25b60i5/DpFmZHFudKiw79g0cfMnMNfRl6hxmW4aGX5K1yVx0322yu6uR7nvb/ACvQYvDi+wVZFtk75prUKBFEGmU9uKyhelttLYA/h/34jCc5V6hQxV8rTZcWtMSUKjyo7d1sjnUeQR245BHPOE8DMLNdoDkirBEeQHbknUQQHAk2uSfUE9+CThqqNTpcTdZh6pG15CpPCSb2IBPe3v646Yc0x2cdVms4OTlRus3UNmoxqovMMOZOjvIktLepMYKbWggjkJFhcJuBYcemJDmbrxnzMTcjxJo1MnzELEyrUmlpjz321m5b3SpW2CQLlsIWfU2xAaa9An6kvU0JUghYcUpNk2WLkn9xFjyfXtidI6UyKmaW3Ssz9PlSKo0twR1Z4gxzFAWoIDxkBsNlzT5ACvgpK9NxjAYGEXK2sqZ2G7XEKuZaarmWWFS6k+8QAC/KkLedCAOLqWSbew+/GJZ0l6TwurvUil9N4lYapDlQElxyoOhx1lpthkqN0IBKyTbkD19MMtXpVZyhORT6vGhNOPI3UpanxZqVoCym4djOrQeUEckHjtbDhkzqNW8h5iiZryg/JodZgJcbYn02c4y+hDgAcAJChyO4tbjFrS2MaLNKXyuzPWuMqfAV1VyvUZ0PIHWultSYjW1JSmoLSh7WNJc2HoTiBzb1uFpHNwkjFfW7o9nHoln6d01zTOos+pwokSZIep0tx9hXiG91vzLAJJSbkkfvd+cX/Rvjp+ImGk6urWYpK7hY8cxTZYC78EF2KTxxbn34xT/UTMlR6r5wqWfM51ao1Ku1ZTRly7RmgQ00lpttKGmktpSlCQAAB2wj3AqcrWVe0BhcaexJSlJcQdzaKzoXbkt6hykKHFx2v64vHKcboRmuhQ5XUqqxcv1VmWqPO8eVKIjCxQ5HU0geINvJZxSSm3c+tPTWIdFkF/8AFGEBaPlofUA4T2BFsG0yep1DMJTkeS46SkuSFaGAByNRsb8D27kfnjDNTMnN3HVdrD8er8LiMNO+zCbkGxF+tiCL+a+j3+D2yzFy5Uc1dQMjIqVZotcbZocdFMjpMVT0d9Tw1uOPJ0lDK20mwPLx54IxU3+FAedX14yjJnUiRCdOTgJDbjzbiS2J721bbUq3KnAQTjVv+D8p8Cn/AAs5dikIS8qq1R+c2BbRJXJupv3FgWxb8sZa/wAKbZHWvKThcKkDJRQCCAEkVJ4W/wC7DiIRQhgKxVla+ulM8xu4+VvoNFiJUaQTJCJA8yBZJ5LgHcfa3r6dsKYad1oIeN0J9NHphrgsIRIK0PIWpoKb1pbNlI9bXtb/AE4foBQgo3L6LaiEi+A3ZZWq6K1Mi5hkSK8whYU+yWanHSex4IeA/QXPuDivJtNepwjPKd3ErRwu4Ic7euLPq7KKTmxyq0FrU2+046YykWuwtBLjVvcG5xGZjDMYxociyqdPDhiOEfsxc/LP3T6fY4yRGyuKhEllZDEgA2Wgj9cFixUO4Vp7W7/+OHltloS1UWqksXUtpt4C4Su/kP5Xtf7E4Y3w5EeDboWl1DpacSTctkd8aBqqnZlI1zTWqAmfvL/GaEzHCVbn7eEFm/3JacUkgf7WpY/dw71KM5negT6+y0U1Qy2nlBHd18IAPHqVCx9rgYg8SYulyGZrSdaRdtxI/sEcj9Rxh6gz5FJaKqesJYmtNqadAvZaP2ayPyKkke98I9rjsnaeqlNYzbJreS42aYiEfi8CM/EmLICg8UNkAq9yplQB450j2xF69CpUkorFEjIZhOkLLba77NiOAPS3A/h745OnOvCteDSGolZYW46yB+wl7at0Afn5h9lEYaYcp5ihqkx1hyOttCHhfsSkc29AbA398UMjs9ajLdmVMNOflxJEF0tMuIdFUhLNh8wakhYV6mwcFr3seRa2HiJWpaak4qTFY8K0UKEVbm60pBKrjz8rGjT973tiJRH3ms1R5SXl3allSlAiw9Tx2HHF+2JVro06p1GsJrjDMmZYMIW8gEtoQLaki1l8gEetja+NTQs9yVypzKbFpyqbRXXUNvr39t51S1Ajktm/qFkkLuSQfSwxKE0il52o8OqU19iHVstNoWUSFnZeibwdO6RyNo6+bHyKFxxiK5koBg1Bt2+0WwGXBoJ1NuHuLd9JHPtcYeMp0+tVuVUar+JAragBlSlNIaEhsDZLYDaQgrDRFyeSOSTfFjfClL7q++hEbwXUCdJmtsRnKXQKmGlKbLrTTrlmgsqFuDuLsq4vf0xUXVSn1ydXpmZ6slqJImyvw6VslWkOtR22kkK9Q4GyfzPfFxfD7UsvwqjmBVeqUZmTIpUaDTW3XQnxDnikgtkL8lgNJIPcCw74a/6E0jMsWd0xirU2ymtSIKm0JILZRNOlYJ54BI7+4xme8Rylx6K8NL42hqo6M7+J0qDWlwzDkLiMRJSS1t7zoKi0+n33WwD+aScIX605RM+5VzYw0FO0aZTnXUWIDmh2znPse1/vhpy027T3qhS5jiy7HkoAsrUCLXSR7cgn7E4U1xUH8Xp77ZbDVUjt3AXzHltSbEfbmxH2c+2NbcpZoszrtebq1M+SIys3SRAakBmTFbebUrhJAPCk/ZQsfzvhBFW8qOt5AVrjoW6QDzoHc4YXKo87PlCTKLiY61stXA8rd7gX9gSf54UwK8zClIdXtqTqKFJWRZxCwQR+oOJCMkYZ0SSHM/MolNWlOfqq8yr5Ull8puLWuA5b9DcfoMNcupyd1lyPJW2ClF1oJHvaxx6py47ebXXUPDw6ytKVjmwW1p5t+Qw0VdEyFEYiuJQHxwUocDgta4tb7Ww25S+JPFJTIaL8hyepCF+ZwpPKRxe1/v8Ar3w5yZDszzLSm55JSLEjj+WITFq7wUQ8oElQ8mri+JdFlLktbqmVq+WvcAHPbv8ApbFoaSq2pZT7nb0IW44m9koFyo34A/Ptj79/D90+b6WdFMlZADZS7R6NGZk3N7ySjW+r9XVLP64+Y3w9f4Pv4hpHULJtfz7kODSMqsVWHU57rtdiuvORG1peADLRcJKrAWJFr4+vaRxhZnCwAWwaNXFqCUlSyAB3J7YxGK8msVnNPUFxZKH5MuejVzZsX2h/zENjGoutuYzlbpfmGpsqs+uGuJH81iXXrNot97rv+mMeVo/g2RmKa0Sgz3G2T92x8xX8dIH64oIIGnNaqVn2iqvzzKeh0egR3ZIjtyZ79Vnvd1IYjt7TZ0DkgvPOXPppxTjkOZVI7+VXqihUeoB0sWSAncDTxbFjc2BveyxckXPFsXRnnLVaVmYOis0J2C3R40MRvE2eYPLr6XkuJSjWtxz91xYACbi+K7RkHOCM9Ra83OpBpLVnnmnqzHL7biBYlrQkXBuQefuO+Mg1kzX0WiSN+SzW6nyKaPh3jtQs6Z0eaLTDngITcmInXqS7uDddNyQdS03vYH7WxrfptnODlTp31YqdTWS0zQWC2gG2pxxTzKUj81uoH6nEE6T/AAfdSaTUazmiRXMhQo2YIsVbLLtdUpxBSpSllQbZKAFhaTwT639MMHXPK9e6aSGMl1ms0iSuqMNz3/wiat9nabcIbS7qbbIUFgqA5HGNkcsbu60i/wC+iwyU8sY7zfobfVY+6hImKW2ZzyHXi7tqUn19h+QxSud5hbeFMQkEI/aevONC5yRHioddX5kRW1ySskWJ5tYfrjPVTy9V58CZnCZZmK48tDbh7qWP3QPYC3PvixZ3nMVEXm1oI1C1wD3wAWuPL+mD2Ykh5IWEHSOxOFUGlSJkxERlN1Hkkdkj1J/IYTvI5sqtnpPliVV0iSjaZZYbClOONrUE8k20o5N7enbjFkVnpy3OQ48xBcTsNAoSL6lDuD7AkHgc98Qug54Vl6ixqJT5CGmYzXCmwQHODfVbk3+3oAMO0fPNRl09DMaIsrjpDbTiTpSkn1Kb27AC3GHS501VHIk6mMtuzmjtOEuoUk3AHHf/AM/34aJeWVTm3HyQGGgdJFgAB/cfzxOXK3VsqRUVbMNQSIUhwt+EI1qdRa4DYJFhfknsecV7Xc+GfMXIi01xNNWSPDbpa7/USpHc+v54N0Uwu5HZTHVUqtUmIDK1bbLNip58/wDu0+v59u4wrpeS4s7L8KsU3xJSt5xt9oj1HYgdxx97YcKBVMq1kvtS47b8jcvBTPlONhh390qLaVbrafYgHnvi0KFS8qZUyXHgJzAmW+lx6VLS3CG0pKramwor1ghQNl9iDoINr4S5JVjWCyYM+5Sk1PK1LznBcQZUSK1BnIbVYloEhl2178X2z/mH740d/gsc8Vel9aqtkebTqlUqbWaM4XYzLCH2YzrbiS3JfB5DYHy7pBN1I9MZ5bzxWMiu/hDusokPbbYeB/ZH62li3II4sfX8wcTrp4jLTtaem5QlSoL7zTkV2OFFpxtDliENrQQHG9aR2OsWHB5xH2A7wutNFDHNUsilfkDjYm17DrZfZSTlnL7w2l5Lpe2UlOlVIaIse4+j1xFJ3QjppOQUqyO7G13BVDclMGxFrcKIA+1sfK6RmDONNWtLlfrCNC7XTOfSAfpKVefy2/s4E3nbO4aLsXN9bZHfyVaSk/8AWYxivbHoGn5r7PH7DqmoY2WGvY5rueU2+FivsIxCdjsNRmIT6W2G0NNpLKuEIAAHb2GB+Hkesd79WiP9GPkEx1H6hpUnb6h5kF/7NamC3/NdwrR1Z6qMp8nVHNYA9Px6b/8AfcO3FW7ZSld7CK77NYz+ly+uCmli6VMrH6YAEpH/ANGPk2jrX1jbF2+q2bbp/wD27N/++4Mb6+9ckW2+rubkj7VmUf8ApKwwxNjuR+io/wCRGK/ZqY//ALfovq+rQSLmxxGOptbay307zbmFcgNfhtBqEu5PomMu3/zFI/M4+Yx+ILr608UN9Ys52H/7VeIt/nE4TZg679YcyUSbQMwdTczVGDOQpiTFkVHW062fqQpJTyOMB2INsbNP0V1L7DcWiqI5H1EZaCCbXva46hQ5mI4iMxHJRrbbDZur2QE//a4+hXwC0owOjr0h1KgioZplrBPqllphn+9Kxj50o4dbXIUdsnUs3/dv5sfVP4X6A5lPoPkSJKa233qf+LSAU8pXKdW+Un/4iR+mKKCO8pcvYe2uZtLgUVKHeOQfS6+WfTPLecOpWWnoVErMZTNIEmdNStyPEDQj32gQsFchZWU2JsCU2F+bR3KrmWKjHqdczIzmurvyGS+XJFJSp5M0DUQFElttA4B0r1kEgjjGrGP8Ep1TeqEuXVupdEeivNuIjwGatNaaZWu9nNQj3ISslYbtb0JthI5/gdM2JjpdXnGhVOQAr5cupusNG/F7twisEXv3tcWtbnG5zr6L8zdo/mVmSmZ6ZfpjUVuoqk15+qJeCUlLajDD6Q4ggkoQktKUkKPNrE884Q9OadJNdaj0irQ4BfZmOx6bEoz61qjxor7yZElRu2hDi2wgt6yQCFqS2Bzrtr/BQdQm5rcmVWMsSLQ24bh/pNMZ1tpGnbIap6bi3qTfCumf4M5FDlRZTefsmUeqQQ+wVxq5N3UtuoW26hS1AFd21lJBA4Kh64HasgbYkAKBj6h1w3Nb98ln3PHXpFarz1WhZUmMU/KtGaVNkGdMkSJUZTQiMrddl2LqStbamwkhpsElJPcw7qxVq6MrUxhlUZ9aJqI0k0aS3JkzQWCCX20EuH5jR7iwOnnkY+gNE+CLJ8JUJdYzbk2UqBCapjTkma9OUiI39DCPEKvtj0bKigXPGJm58P8ADitiNTc8ZUaT9KnPxdxg6PRKAyElA7caz/IYyvqIu0Ds2o8wrvdJnNtkPyK+deQoXWp3LEin0RxmjHMwQaxJp0qWt6bBcPEZcVhlUdpACiFjg8cWF7ybo5A6wZErVZqH/oUl1JEmK6gqqk52EiU6bJbeSlDyXELNioNuGySQvUCBjdTvQBhfCs/ZPkKtqJk1iU7Y/bWo8YD/AOguQV6l9RcishAIRszHE6f4WH8b4t9+ab3ePmEjcPlH/tn5H9Fm6rVLrz1VzPJzVnbpFlKh1JxhTJqbrsdKHkFX0OtNhRWscKQ6Va/y4tFpfwu5ozG/ITXercuFS30RkLpVBjPFqWI61FndU47cnUonVrJ5+2NlQOkDEFJ3Op2UFr06VOGcSb+5+/3uMHudHqaoKWnqrlgPLPK/HH/8rCGriO7x8wj7lUjaM/I/osmUv4S6TFguxKlm7OE1hcRiK2wtyLBZabauW/RTjhC1X1E+a3N8WPD6ZZNpryXmaRTGFghGmJBFy1e5buClAR2tYHF0p6QR2U3R1UygVHuXJJN//mw40joLV6u07IpWe8t1BKF6FLjhx1KTbsSlXfkG2CJWS6NIPxB/NVup5oheVpA9PoqTFC0KKo3jEN/swoOkqSj7AC3pa1iMHoifhbYWKnXAb3KlPLa4HYIIaBH/AI8Y0APh7zOptLTmYaPpQLJWliQSP/mtg6F0EzdTFh6m5so0Zwc60UnUr/nG5/nhnQsdyVYcW7KhmIlZVBedmz82KS4gLa2ahOUpNuPOA2QL/f8AhhjmQakN4Vg1iXHcBBROcIKhxoKt1o7mn0BFvTGoJHQvNE1zen55akOFOk70d91I/JK3SB/DAWOg+bIxIjdRVMJ7hLUZ8J/5u7o/liNja3kjcu5rGGd+kOQOpjbP4u1mBhTTa2rQHHKaXCsgm7iG9h3sLbtj7WxWVV+EvpXQ6ZOpOX6xn6gVZ9tBWZ8ZMpt9wXLJdR4ZPywe+2oX98fRKZ8OVanKvLzzBcUe5NARc/qFA/zwmV8L09Q/2dxx3HFKc+k9xbexa05dtEOWUr5TT/hz6wUiWmfBzHRaguAlSYbu7KjOrBuVOFb6Rtr0qURdTlyeeLYUMUTrxlc0es1uu17ONLYIkTqMw94tSXxfbbaILgdQk6SXBpIAIA4x9YFfDeVxW2ms2IiSmeFORqe4GXE/dtbxsr7pIH2w2VnoUxQKfIqmYOp8SJDYRd2RJp5QEDgC6i969h+fAwHzZQS46DmUzI87g1gv6br45sdQINBypBo9QpFWbqkCVdbMposAqcQRZvc5ulZsfKB27m2JlRevnUSl0B2gzoEmm5anuBE0rrrxbaRxrUWUvK8pAAJ0gW4se2PpecudKEJLT3X6juD2dioXb7C6+2GqpdMfh4qzK2Kl1TyjIQsWWHqDEdBF783HPPvjL/FqVo/xW/MfqugcDxB+9M/+h36LJHRz4qcl9Jsx1uqxs9R6fJnvPjcN1tPRy4HE6VEaCCCCEi1rehJGAdXOvVO651Wo5ng19t+PGpUCnIkhrw22gSlPytIsLkqVdagR278c6ZHw+fCVFcVMhZ9yPCmuE7kmNlSnBSgb3H0Hvc3974qbMnwQfDTmHMUivs/E9TaLvxvCIhUfK8SPFaRt6SUtgkalC5Kv3iScD+L0POZvzCb+AYm7X3aT+h36JB0j+LfLHSWkuZYldQFwkFxbhjpgeLZUD7gKFx6dv4jFrZf+NPpnHmJeh1rLzC0JW26tFBlx1a7jyEs6uwHa/wDDFfD4Efg5/CIkJ34gXGanFc1rqsOKGn5af7LySpTZH3bS2fvhsHwBfCqmNIH/ANVVWlSlrQWnRTY4S2Be4WkIuu908kg8d+cOMdodjM35j9URw3idgBTSf0O/RXlG+KnplmGKn8XruT5Dm666ht6fPjaNauB50gX0JT3OId1az70/zdSKacoQsmRZ8GUiSuRRnG3ZT7aEKGl12+txI1X5vYgYzR1I+GzopQa6zlugddJ0VNNgx0Pymi/IE91aSov7bqdLJV222yUjB/TrIXTnIdXVUh1wlVpJjOx24s2I6y0244NAdu21dekf4vsT3xpFfTPbnY9vz/uqXYRiEZLDC4fAj8leLM7m6CoA8i/qMK11BOwsO+ZogtupV2LZ4c/kSf0w3ZYoqK7Smp9BrEeowwS2iQll1oOFHBtrAuB2uLi/5HFzdL+glM6hxaixLzLWaZJhoRqS0zGebcQ4CLpK034t643Pews3vdcjJJE+zhskPQLNL2Xc45ckynyNiS5lqoG/dtwgMk/54axtwcgHGGeoORE9PeqdVyjTg61BqtOj1GluFRu2ptIFgbk3SppXr7Y1700zWM7ZHpGZFKRvyY6UykpPCZCPK6P+cD/LHLZobLo1DcwEnVL87ZVpGesoVvJVfZD1Mr9PkUyYj+0y82W1/wAlHHwxzlkSUOqych1aWthygKkxKpIfZW4Ii4zhaeUpKPr/AGWof2txHvj70kXxlvqd8COSuonVyt9XWM8Vyh1CvCL4yLEiRHGCtoNpUsFxsrG4lpvUL9wT641RODTqsZvbRfJaq5Obp0idS5VRcTKpdUkx5ARGUvcaG3tqQn0Jub3PtiLy6NMqE5un0xmW+4sk8shpIH3NzYf+RfH1Urf+C6yZXK5UK9I60ZybeqT7j7iG4FO0hSzc2BZPsP4Y5SP8Fr0/pT27I6u5xnNALuy/FgpbUSkpB+W0k3F7ix72/LFglZzSZDm7y+YtBy1T6fW4jEOsLlVAPGMnTQEToxcIUlaNtx9IdRa97oI8t/QHEkRlWiQpcliVFyzW46WlsJXJo0+Glxwo0h3VDlpc8q+bE6FEcixtj6N1T/BzdC8mUmXXap1NzbTm4zCx4wuRf6ug2B20Fki5A0jgnmwxAY3ws/BfJCIz/VbP5kqWlIkLJaCSeObRwLdsYZ8To6UhlTM1pPK67MWEVNdmfh8L3sHMAn8FgdfTmI3cx42WGkOHURGn1iGCbeocacA9b3UfzwzjJ6S/NQ02hxLAQfkZjjANX/39lJXf24ti6/ifofSnoF1dqfS2kTM0ZnbprEWS7PYrjcRbKn0l0MLQth1DhSjSSpOn9oOLg4pePnTp/wCHeceXnGO684VXRJhPpSj25Q2Sf4X+2NjXtcA5puFyXNLNHbryMjyEJQUw6yexJamUySLf5jiThPIyDU1uf1Z7MMRbZBu1S21cffQ/Yj88OSM0dMpR0pzVmBi9wjxVBjO24/8AdyP9GAN1DJbhWuJn2O2FnkP0GU1/Et6h+eF8SpQGo2cqS2qNBzJIhsrcCnkPZYDoUvjmzm4QLAAgG325OG2vNPOQEtzZ9IcqDS1vRBCywzTJDrpASGitAbu33IFjZdjh7L1KWzpi57oDxI4SozGLj7bjIwbHGYp1E2cqZvcjOxX3kLZiVgIS5rCVNK2ysBXJIJsTgGzBdaYIXTyBgIF+uy29/g7KvKyt8P0qVVpMh1FdztObjoUSopDEWM0QLnvdBsB39++Kr/wnyT/6Vsjur0BX9FpIUq/b+vm38z/PF8/Cu94fpDDy9+CF7RU5L0hCKkzGVHllTbjoRYKOhKyQCkE219r4ob/CULnTc39PFz8uRIU5VDqDwdjzlvEQhJQEMrStptAKXNSgpPfcsQLYbeO6yWs8tWQoyJhcaTIUt0g8rcIv+uBrLQeUhtBSdZ/e/lgpjfSy1ZqQ1zyhbfe/rf29e+CtEtSnJLiFoYBUoBDZKhYWxSNFZmyq7K5mBqlSos/xranIEgNSHWgVJDd7B0D1QQrnCSoMsSqfJZYktRorsnxLPzA4WuQSUkW1o+rk+lsQJVVYlNvNzac/pfF2JCmjYXHBUCLuAHkaf9GE9HrFfyhTpNKrVNgVeHMbShLQlXsgXUmx1JcFiQbgixCPbHOLJRq1XPd0SjM1XjKbjh99CZsdR3UpN0qbsFBwHtYj+YwmlZzoley9qceCalTOS6UFPi4xIPJt+0buTzyUH7YXyanRKpVyaJlujU+mIijU0+hvxBPqk7qzrJ9VJ9v1wxN0/LMafJQmGGlDWoGLJWkE25I5UgenB9/XFzJDbvjVJmTNAzhAMVCJKgkFxYJPaxJthfCzxBjsvUp/57DjawFp5LZ7pIPvcfwvhmqFLo0de9DjpcRwgqdtwTz2sBxjhobry7R/BkgA2SSlQv39PTF2fqldKWp7oec3EVmPbQ6l11oFKr6VLvY3/wDNrYRM16XlmoSqcyI60ONLjOt7mpOjmxBHBKfQ4b5GVZcdbiSlpxu2tGj1R7/z9e2ED7DIWY/hlqLVxy6L39vv/DAz5tkvbutZIqvUHxPbkpLCuASi90mx7KH3xOckP0u0nNS0x4Ew1BCITqUGQll0p/ZrZBvpV24ve5xC0xYLyS5JS4nUedK+FfnhXDh0ZkoMpqYlFvIG2yod+b4ZxuNFY2VXXm+JQqfRHcsKr5dnrqjcqLpjSGwiPtfMZ3Fizn7vYg/L9DcYT9NqzOjx5lEs29HhT3HF3bsQXW0259f2f2ta2KppQWxKbW2480y079KAtKV34PFrdjziy8iKTFlV1bitKJDsYIWbaeAqx/XWR+mHgYWNtdB785urShudPqc6+ibGpkxZb17ctlx1p9aJG4275FJ0WQSld7+hsbY7CmhoyKlkWOuo15qQipSWX61qGsm+2lASPEAqIJ0jc+USRYkCoM7sFT6QhTSZAirU0Dc/vi5sfS3H6/bDFQnq3Sp7U3VTpTbXHh5brqmFA9+AQQfyOMlVSSOeXsdvyV0VS5gyp16idPs40qpO16Tl5dIjSJHlktu3jurKi4EpJUeb3IAtYH0xEq/TKwotOVDQErs42WHQsj8lDsQRyk9vytiZrfcmPsuTqpSg5HWSyENaWmSfqAAJBFrd+eD3vjxay3KQ5ErUtkKeTdc2lwNt6Or7WWlDg45SUAEHEhZMxoD1TLdxUIdqL7zEmIlt2OmTI3NLC0hKEHujkKIPAN7++DosOE7Gat4sOhZ1/NJBtzcX/n7YUmjxnXnHhVHFotfUUJBUPc6OL9vf88AXS6W3tKiVBbwcQS4l0abH7W7406qqznJIiBTZBKobL7aQ2C6h54eZdzqsR6YCmj0paHi8lTG2QSWj2FrH+GHRFPp6GkGOkqA4IUu3N/TjgYdkTaLHYSy3lhIWgn5r00OqV628iEj/AM84bVLlc3moyvKEiK4mW864yGnAizrRCrWBSo2FuxHOJMatEmDXGp0RhW1oWkICkqNrX+1/Xn1w6MVylbCWoeVGEvWADgqThSbeunTz2HBNuMKolWkvKU3EokJiQuyQ6VSXTc8Cw3Qgd+1je4xZG43AVseXMF93+kFbiVPIGU2WlhT5y1TJboTyEJcYTpBP30q/h+WJ2O2Iz08yzHybkmgZUjatqj0uLT2wbg6Wmko9fW4OJN2GEO59VokIubLP3xW1kusZZyYw4CZktyoSEA86GhpR+mty/wDyeKJzdD8dmOjUJC0bcdsKdST9JWQT/wDIn+eLI6pTjmzrrNaQpKmqG1HpzftrA3XL/wCc7b9MVTUasmTVswZl1AoaU4hlR7AX20n/AOG2o/kcYcTqm0dI+d32QT8eS9Bw9QnEKyGmbu9wHw3Kypm+rIrWbswVcthXjatNeSSjmxdXb+WGZSUSGVsJaTdwhtNk9yeB/eMFtygpCJCtQ3ruEd+VG+HGgIE6vUuAlNy/OYSR/k6gVfyBx8nlqZMpc46i/wCGq/QMEERqQ1jdARb5q/QlzeaYbWG0NjQo8aUJQOT+gGKZzfm9yvVaZUWkN+EdUgRUpNztIFmxx6kc2++JT1kzU7QKCmjwEOeJriVlbqFW22AqxT/nH+Q/PFBSlzF60IeXYi9uwvawPfv6Y7vs6wh8cL8SnP8AiHT06/E/QLwPtX4gjnmbg8O0ervU7D4BN/WOq/h+WUrStv8ArhNyo8lA54t3xn+o5nqVQpkOjOOARILRbSgc3JUVlZPuSf5Ytrri/s5bgB4bq3ZDjCFdgAgXKre5vxikoMCbUpCY0JhbzqyEhKfc8DH00r4zZOaJ+22ltiPZIRZRUfXD1R21zU7VMYUSQhTzqeFH/Jt7d7++JnH6OUqG+PxiW8ttttuyWW1uqfWUC4FuEAL4ucWDSqBSGKUy3EpzURwkgIdirde128yAEG+u9u/H6YKrcqoiMyYmhp2IL6r+Y+ZI/T9O+JlFhSKiUITHcQtshZaRHKePW/v/AOe2JIMuZbobLcmvL8RVltkx4C0EucEDWo6rC17i+G9WdIE/NaKUtTW1DlWLlysBdkg6fXuPy4PpgqZftJk6o091X4atHmZjxUNAltYKR3tzcepub39++IFSZTSJC4sl5DcZ0WC+RpBNyfbFz5pgya4oBcRCdhklQJKU2PqQDcn/AM9sRmndLkVB1uSp9CVblyW/mhSLXuSTxzcceo7WwEyj1IyJAmVGO/AqOlLjmhalp/Zk9le9h627XviQt0ipRYzcaYgoBF0jtce+J1/6N6RILK2XXWJhGlzZOpahbsNfKDa/bucLaxlx5tiKhMdb6owEZb7joAbue5N+bn7YiYKmMwQ5M8xY0dGgwlkBsLsCDxa2DA7WqGNb0ZxhRsUOgEG9rgg+va+LBnZHnRwH1vMEL5BC9QNuCP0w4Rq9FNLXSqnBQpLDJCEuI1Dn3PsLm354inPMo3lnq9mB1Qi5rcRWo5si0nyvo+wd9eORe+NAdI8i9I+r0s01rrbGyjUlHyQavRFuNLKgbASkyEJvx2UED7qxQ83L+W1qW83KZiuot5XRZKjYdvfv/fiDtVv8OrJDUxyJtuW1sKKbkdhcH88VupoXnM4L1mDcb49gMXY0FQ5rOhs4D0BvZaVr+Q38t9Tp/TH8Uakuw66KH+IbJS24svIaD4buSANaTpufp7nGlXP8HNnZWppjqfQVhCilV6JN5UFXsbOnGRenWd2FVenO5sdUhqLNiyGKgxHBU2hp5DmlbaO6eO/oVe2NjfFvOzn1gqmT87fD8rMeaaC5T57UiVllyUtDT3ikqCHksEFt3ST5FjVa+MIpGNDnPbe21l9cHtFreIpqKlw2rED3Nd2rpGiweACDrYa67fJVT1i+D/qj0foUjNrz1NrlFggGXJpgebchIFvmOMO3WG7nlxClBN7nSnUcUM7IW8nb06dPdOPob8IFM6qUvp7m6N14i1eJl0gGHGzMtRdQ1sveNVZ4lSWCgo/acEhwj1v87GNLe2ltRW2ptO3qNyUEeS/+Va2MtQxrAHtBB6FfQeBOIK3E6irw/EJGTPpyMsjdngi/zC74hOnTzcevrjpc1u9hc+uEw+o9/wDTgY3L6j/HCtFl9JMYUjyNlSZnzOtEyNAStUquzmaa2U/uh1QQtf8AuUNalH7IOPr+zGiU+O3DgNJbixm0MR0JH0tNpCUD/mgYwl8AHTRVbzvVep85kmLlmMYUNRTwahJQRcH3bj6j9t9ON5Hgcfyx2MPjyxl/Vfkz21442vxlmHRO7sA1/wBR1PyFlO0kFKbDi38MB9ObA/3Y6ClAue4Avih/iF69LyVHcynk90O119Nn5CSCmAgjv7F0+iT2HmPoFc/E8Tp8Jp3VVUbNH16fFfNMHwiqxyrZR0jbuPyHmfIfvVJfiI+INvJjL2SsmyQ7X3k6ZMlNiICSPT0LxvcD93ufQHHL6i84uTIVrecWVrUslSlEm5JJ5JJuST3OFEhqe44VrDrrjilLW44oqUpSjcqJPJJJJJPJPOCEwX9d9tRHrj8/Y9xHVY5U9q/wjZnIfqeq/WPCnC1DwtR9hDYvOrnc3H8h0CI0kDgJsPyxzaV/ZRb8hgxUSZf9kr9Tjhiyh/ij/HHAzk6l5+q9YMnkiyylP7iP4DHikg3BQP1H/djqoUw92se8FLT/AIv+GJ2h8/qmAZ1C4hgvHzaLDva3/djyktqGllYAHe9sC8FLSjcSi1u6cWN0c6F17qvWN3Q5AokZYTNnWBI9S21fguEfom9z6A76GlqMRqRT07S5x5a//byXPxPFKLB6Z1XVvDWN/HoBzJ5Ino30XzF1XroajqVDosJweOqBSCEeu22CLKcI/RI5PoDvXKWU6Fkmgxsu5dgNw4URNkNoF7q9VKPdSieSo8k4FlTKtFyZQ4uXsvU9EODEQEttIufzJJ5UonkqPJJJOHsAfnfH3vhvhyDAafTWQ+I/kPIfVflDjPjOp4sqrnuwt8Lfzd1P0HJdtfHhwMe5+2OWv7Y9OvGIWOEDHuT7YCdV++IgTZF3II9NWOJ8lx69yRhryzWxmSjM1lVJqVM31OJ8JUmQ1IbKHFIOpIJHOm45NwQfXCXOWcaBkehSMwZhnojRY/bm6nVn6UIH7yj6D+4c4qmlZAwvkNmjUlPBBJPKIom3cToALn0Hqjs3ZsoOSaHKzLmKoNw4UROpxxXJPslIHKlE8BI5JxgnrP1szF1ZrQUsuQqHGWVQYGv6fZ123CnCP0SDYepJXWDqnmjq7WTUKiDEpcRavw+nIXdDAPGtRH7R0jgq7AGyePMqvzClKRpXYfe2PiHFfGLsWc6lpHWhb83evl5fE9F+nvZ97PYMAjFfiIBqDy3DPTq7qeWw5klXWf31H/OOPXUPqWr/AJxwYIMzjygY74Gd/ZH/ADseEuOhX1nMzqEVf/3yx/nY9rP+3OX/AN3gzwUwi+hI/XHhAmAfSnFPe6/VTMzqEHU5/ty/449uPAftV/8AOwYIUke2BJgySbG3a5ue2CMxNtUC9g5hVz1J3265Bl6VKRKpwGoH95lxY/uWMSLpf0wmZoLWYczxX2KH9cdhSil2o/fjlDPpr7r5A4ucW3N+Hups0LL3UDPtMUmnrkPx41NdFi+VoDiHJA7hs7Rs33PGuw8hkIf5uoWv/wCf/Ix9+4PwKV1HFNWt15D9V+ZvaDxfE2vlpsMdfXVw5eQ6689k4UzQ2hmMyyllhpIbabQkJS2gCwCQOAAOLDF1fD/K8Nm6VGXwmXBUAPdaFg/3XxSkSSlVh6YtTo1VGGM60tLnlK1uM/nrbIH87Y96W5W5bL48XOdq43Kd/i6y0v8AAKD1EgM3k5bqCUP2T3jvkA3PtuBA/wA84M+GXMLEafX8lhy7LxRW4HqC24Al0A/mGz/nHFx54yxGznlGr5VmWDdTiOR9R/dUR5Vfoqx/TGPOk+Y5OUa/QatU/lPUCpuUOqJUbFthxZbN/wDcnn/k8ZD3TdbY/wCbCWLcWE8qQ1EZU+9whNrkDt98Hp7YLcaQ6hSHEBSVCxCuxGLFkXbAquEjj1vhkzZmyhZLokiu5iqCIsSOm5J5UpR7JSBypR9AOThBnLP2XMh0GTWK1UBojq20NMgFxx210soT6qI/gOTYc4xH1Q6j5s6pV01OqPhiJHURCgIVdqOn/wC2WfVZ79hYcY8jxPxVT8Px5d5XeEcvU+X4r23BvBdRxRN2j+5AN3dT0HU9TsE4dVerdY6s1y8yT+HUSMtaocNS7pRYcKc0g6nDbj0F7D1JcemHRyRmOvRXnJUGZCASpWmYptDi7m7QWEFRWEp12CeRxcHFVJprt1BSbn88SDJma87ZNmmVlGolmSkEkaEuiwSRykg+hP8A34+K0WJR1mICrxZplu6/mcvLL5dF+hq3CHUWFGiwWRsVm2F9vUne/U7rG3+EFp9Gp/xbZ6plOL7JiCltSEOklG/+HsFe2T5tHmFtVyTrNyLYznNoVWjvbK4ckexWhSQT62Fu3OL++KCk56r/AMUWfK/1HoOZ5LCsxPRn5sCgOL3o8dtDLG0khLZGhtHOvtzzfEfLWTG3ypfUibR1NkFacw5WnR7H3Ko29b7+TH6Zpwzs2losLbL8j1uczuzm5vuqccpkGGkrqkpxayAUtNDSO/ZSljj9OcK6fU8uxj/6yp8lmOoLaPhV6yySOCCbX/8ApxdicjP1FjxFO6ydH6wp9Wv5mcxDWoE30lNQYat+tsVn1n6d5uyfUafVcyQIKIVZYUIj9Oq8OoMPLa8qrORXXEXsUnmxIPGH0usxYoe694yoLTR50lcZKjsh/wApTfuLAn+/E9yfGbqbzdNmUkVD5TqyHkIUlKENqUTe1+wPrxbFcwWkMt3cRYdyVmwtjUPwwSukWXokip5xp1Vq2Yp76/C+DqkZLTELasAWFgrcWpZcKwSARoAHc4Sd5jYSBcpom3K2T8BK6IejsepT5sONGanyWmfmtto1ENueX07A9vbDP8aOX8r9Ss+ZUqsTMjbrFLo7sFxECeydRclJVoUkgm4sBcWtcd8VA7n3IlLY1SsyVKPFS4iLAg/0YajNRWiDqJLEghf52HAA97lvdX8iPQ4selZpoUYoYQl196lv0959dyfmWaUDx9Bv2Fje18ecr8Srwzs44iPO60QYTX1jDJAy49USz0V6aNpRt06sztQJWX8wu7STewGltoA9ke49OfRZA6O9HZqm5K8hyylRssO12ayzc24ACxyD+Qta2FEfqNSpCwKVnGhuXUlbTKajDbUnk3SSSk2BJtZAI9b98P0fMUx1xbVEUXXENgiSyzuRhyoFGpu5XZB5+38ccB+IYk03LykODYpDq6F3yv8AgsfxOmPUmqhU2BRa5Vjc3cYg7qLe5UhRHv7W98em0XN0dsQs0wKgltBBQyYu0lIFxa6Em/r3JwCLWHluKZlSX47IBW4W47b40WOr5R4J7HviU06blGmMQ0VV6oPmQ13VC2FAnm23e1gCOdfqOx4x7J9X2Tb2us3bMTGy+mG220cu05BSiwIghClcixCiFWIHF02uTzjok0t2O5EkCiSCtwqIejvvOp440qaLYR3secTN/M2S5i5K6WitsKSyEoZYiNtNgXBU/ZZN7i/yyeSeCLYEinZWZlNw4iGJcrbQsBkht1tZAIAfDnbkovYC4N/Q4zOr2kZiwotnsouyzRGQmQzkcKU6gNqeXNkoQfVVkk/3k2GCKdlHLjk5zxlTpdKacJ224UkzdI72G4ptZ/j6YlGaqfRnIEtmTQfAKKGA0pmqNOgEgjcCr63iABxcjk67cYZonT3L78CLNmdSKUp5CHHFsAPKKz3RyE8d0NlsajzcKwzK6PLqLI9u3xWTvU8q9JmYSkNdUaY3LbXYqXQqgxuj3UdTrZ9rjv78Yj9TpWTmYqmqfLpDjrgKmUNuOl37khbaf/N++Jn/AEeynAMiM7mbLzaRocccejSHS0+UhRbTY2KADa59z2xEc61Kn0VxdCpcuhyWS2Cst0xbbbXf9nufexwkNaJH5GNQlqb+EKItPUuKr+uMtlQNykPJbv8AlcG354Xy3KY0lqTDhNNgBCnG1hl0KvyPOhCT+d8JqfWqrCKRCqbbLayCdppJuji/A5PH3xKZeWIeYVpeoLrjjmrcKpEkR2yDx/jD2v7E41PlczVVdoojMlshkuNxY7IbB+W1xf17Xwlgzq4UPARnIkeQoNlZjKUlRI8rZUB3OJtLymjLrCpMo/LIRcl1pTR9T5kE+mJBCp2d8w0rxeWa/SaZHaSS6VZi2SUj6RZBURYW7Jv+mGFRN9hqZuZ3hVaVXLtXr0CI/uxjMgWWpRnIu63+4EpJ1kgew9cAbosp91Md3L0x9bq7ANm2k9zcf6b8YsGBTeptUrcinv5+y5KnBoPLHjXZC3WwL3aUGNDh4Fwk34vg+n1xiryVULMldmRrXFnICEpJve2rd4B9DaxwXPqiRZoRvIoGjKM9xlMdWXEBxla3XAuVZ55tdgnjV9CbE8WPJN+cdk0+ittpFWo0+IpDd0Lhbaf10rB1jn1N/vh8zLQaBTZPiIk2pSmjdCXHm22fmDm3kBB+9l9xhsTXENR4rreXG32y4ULeJcSpzmwIIIC9Kx7fr6YZtPM7VVua5EU2nuT3Ft5eiz1voBdaCQkFxtHcgHgn0tfvgKqayl1xVanGEpBIeVIbKVpX6Ap08k+gHYfpdZOz5mOU6h5xcVT4IG+GkuOkdvMXCrm3qAD98K5FdnzAzOZqcQy3xtuxfFNboW366Qb2t62t3GLRTyfaIRyuckDOX6WtAIzBBDgWAWNqUeCO+5taDzfgHsMGuUumQ0JbU808CAFFlxCrrH1nkCwP+nC+lUPNdUBpFPp1TlMOlZQ2W3S0km9ze1keuFjHRDqvOUPB5JqAbsCl55sMsqv7KXa4J4v9x6Ys7KMbuTdg93JNztB2G2pG7D2HGytKnpjI1AEBy1iTcE8ggEffD5ToUuPV4L2VpARMXNjtw23SXlplrdbDIBCfP8wpsqwv64fqP0Bz0wUIzTR55hL+YIcKs0+K5uGwuXpKihoEA3slRJxf/wALPw2zp3xDZNqE9vLcWk0OUKsqFDqjtQfWIySttbji7gnf2TcG3BsBiyIQBwObZMyncH3svqnS2prECKzUZCZEtDKEvvJTpS44EjUoD0ubm2DZcliFFdmyFhDTLanXFH0SBcn+AwoAsMVz8QFeXl/pTXXGXNEieyKYwQbHVIUGzb8klR/TCbm60DUrNNBrLz0HMvUKV+2lJl1AX77jqiW/4FSR+mKm6hTfwDpXUA2uz8mM9oUee9mEf/MpZxa1XYbp2SYVKRwqoSEeUd9tvzf36cVf1dyrLzRl9FBp0pDDrrMMpK2nHNxtKipYsgXN1DuPUY8vxYyapo208Avmc2/oNfyX0PgB1JBiJnqXWyNcR6nRZh+QAlKUpAR2A7acS7pkw09nGNIU2D4OLJfH2OnbB/8AsmHZj4aOustW5SunFTqbSuQ7GUhpJ/8A8ja/hzi7fgp+HyRm/NNezBnalSoNOohRTnYMtlTTz0pLii6ypB5SlBRpV/a7Di+PHy4LW1bTTxttmH7K+kN4gwzDy6rmkDms1039PiVCOtXTerM5By11HnMJbpsua5RthayHJDLiS6iSlNvobcQU6uP2o9O+d6hlCuqrTaWYzbMa5TqvdIAP9/rjZX+E3rnUfJFDoaqVm5k5VqcpTAoX4M2hbfh2w6pXiACVtDSPLxypPe2Pl7nLrTnetNLhx6m+xCPASh1RAR/Z59Pz5x9IwfDRhVDHRsNw0L4Vj2MPx6vkrpG5cx2/fPr1Quv+ZqVVsxwqJQ3y7Do0bZKwRZx0kkng+1hhl6el6mtu1tppaSl0IQ8D9Fhc2HqcQRRJNzzfDvTswSoTCYSyFRwvcsEjUD9jjo81xyrpoWciJe0iQ6FuKG4pRLxva/Kb/bFjwKsjY/E3mFxEx2t1+S48QyFk+Vei1+PMnvbjjGeKNmelQJAkiqEOKPJVGPlHsP8Avw8V3q/NqTIhR5i0sWQC6izVgBbhI9rD/RizMhZT6KhdZrsisImEuuuAoYUyXQTcAauOBwO/2FrYQZjyhT8s0qXJjSW5s4uB3xKWgkg67httIHNzz3FgLYg8XP8AXorTUuEyZrbn7ZskqNgeNShz3tz3++JRlfN2YWczeJruwoz2DtaRuNxW9BJG2e9+/Pr+eIl08KbMi56r8F1+NMTKn3WC40+7p0mxBsm359zbFqU7PaFLVOnVaBRWjZLnzIzYbBH7uu9+w8v/ANOKVzPV2365JmsiRO3X7ISTq3b8DgAXvb2tz2wsovUzM1BcSmn5SCXWVEaRETcae/JQe3ucRNZXU9XJNUiNRssTkOuAa5DiQtvdDg8tlCw55Pf2PFuWKLlWi1SYUf8ArBE9F9T8t0ulJvyBwLc8m3fuMMVKzf1IrMlUvOEkZcgrUjaS/AcG7ckaElf0fmf0GJ7AjTItQakyJzchDhul3Qpl0342yAQLfcEcHtiJkfQJaYqDSRZ5gkK1pSsjQO9+LcXHJPfA5svKynAajT1tDb4cPCQb8E/w74JYrMZ5b9Q0Ryn6FNPo29I4tcd7jnzdzhzrLeX6hAC5sEyI7DTcpcstOFm17EJ0EFa+3BsDf1wpKibMy5IplRg+MgzPI+gkOOWSOB9B/n+ZI5xTsujPU6sGHEp1Pkr0BtSy0LOWPfUv98/bF5zaxTzVGKJDQw5Gba3CHJVlNA/s7J9eDzfsRz64jPUCjxGnGqnDJIuQte4FD7BKPQc9/XED1FbXw/0npEnKtWiZyzJFi5g2W2YdKDLpUwt1XlfdK0htxBbS4EBK/rsDbi7zUa3U/hOztGqvTbqHDekT2WnpMNuY1OaWwSBsy20Hbc5JIsdaQLgp74zhUqm9GpyItNmri1OIFuNOOKOl0HSdpRHKACgOI76Vg/2jiMUKn5yzrmiM/UJDjDUd4JdSysr+5WXB5DYcj2Bw5kDG3WinpX1UzYYhmc42A6lfUb4js8Zh6t/CDSup2T5sil02XJbOZac2oBLzG6qK62V21lpuUEKtfzIV8y9rYwY4hxDoCFlTh5J+/rjV/wAMPXHphSummb+i3V+tpp2Xaq0sQlLbcdBbkNKaktAISpSTcIeHH1qVyCMZVlRURnHo4mtTEsOqaRKaBSiQEqIDgB5AUEhYB7A449XKJCJL6FfrL2W4fV8PxVWCVjMrmODg7Lo4P/zc8p09EjKlKJKz39bYd6VAnVmVDolLgrmT5r7caJFZsVPvOLCW0A+6lkD9dX04a2xqUhJSVW4tjbvwL9BHGi11xzPD0oKXG8tNKT31AodnEf2bakM/5Jdc7KSRVDGZngBev4z4kp+FcLdWzeIeAdXch+vRaU6LdLoPR7prRshRnGnZENtT9RkoTxJnO+Z9y/qNQCU/5KE4mirBJV6YPIFtKeMeSLLSb3sexx6A2jZpsF+GKqplxGrfU1Bu+Q3J6klV18QHxCRshsP5Uym8l/MLqPmvJAUmCkjub8FwjkJ9ByfQHH7lYmPKcmS3FvPOqK1uLOpSlE3JJPJJPN8BqU1cipPyHXFuuOvOLW44oqUtRUbqJPJJ98JFLSG1ISBY4/MvEmPVGOVbnv8AC3Ych+p81+vOEuFKPhuiEUQzPdYucdyenkByCM/EnCb2v+mAie5fsOcFpcRp+lFvvgYdY/soGPM5XH7S9bkaPsoRnrtpsk44qe7/AJOOKcZSDYIwFTrfqlHOIWH7ygY3ohmarTqdGtWOeNKQHCoBv74CFNaS6Ep/LFo9EuilU6sVPfeSuLl+K4EzJaRYqUn/ABTXuu3BV2Te/ewxvoMMnxSpbT00feO4/X+65+K4nR4LSurKw5WN/dgOvRB6J9Ha71ZrO+ouwcuwnNMuaByo/wC0te6z6nsgfcgY3Ll6gUnLFIj0OiQW4sKIgNstITwkf6Se5J5JJJx3L+X6RlikxaFQqe1DgxG9tplpNkpH+kk8k9ySScOgBta5x+guHeG6fAKfK3vSHd21/IdAvynxdxfU8U1WZwyxN8LenmerjzPLYIQx3Hsex6XfdeQXscOPXGPYKiD6cYCoG+oix9DfAre2If1G6kZd6Y0BdezFLAuSiNGSQXZLtuEIH957AcnFM9RHSxmWU2aNz0V1NTTVkzYKdpc9xsANzdG5+z5lrpvl1/MOZJmxGbG200ixdfcPKW20/vKNjx2AuTYAnGEeqHV6v9WK6qfWEGLEYuiFCQ5dEZs/9JZ41L9ewsLYI6kdUaz1OryqxmOSkJbJTEiNk7URon6U37k8alHlRHoAAIcp5KlBN0gp7W9cfDOKuLZsZk91pjaAc+Z8z+Q/Nfp3gL2fRcORisrW5qkj4MHRvU9T8BpuIzz2CTxxzgBlrV6WxxLjRVdSk4FvoHZYP6Y8Jk7T/EK+oZQNmoCpbhGki33wDxL3ucDLwUfMofwwBT7WrhxI/TEyeacNHRdTJdTdStRH59scMxy9uMeVIYA4WhSvywaw2H1oQlWpSlhCUhJJWSbAJA5USTaw5w4jLj2YKDsrBmcFxMt0nsAP3j7Y1d8PXw5SW1MZ36kQAhwEO0+lup5QR9LrwPr6hs9u6ueEuHw7/DYMueGzz1DiIXVuHYFNWApME+ji/RT3t6I+6uRpJCQCL24479sfX+EeCmwltfXDvbhh5eZ8+g+a/PXtC9o/vBdhWDHubOeOfUNPTkTz5aamt/iFpbM7pXU1qRzAdjzEKHdOh1Nz/An+OMmbKEkp1euNx51paa3lCt0kpCvF0+Q0B9y2bfzxhtpBW0haj3QDb9MfWoDplC+GPtulkctpKcSzK8swalGmIV52XUOjntYg4hjfyz3w90uYEKCCsi4tcel/XFrtkjFt5K0uthaDwoXBxjzq5lePl3rRXKbJBFOzdC8cyhLmgqfN7hJsQDuBRv8AfGkqNnrLtOytQJNcrDENyfDTtJcVy6WwAsj+X8cU98Tk/LtbpmXsxUuoMOzaZPLadChqLTjer0PYKbQeffHOlOVbae+ZXF0kzUM39P6RVytZkJZ8NKDigpYfaO2vURxc6b/riZW4vig/hwqrsWZVKIFXgVVAq8MoUNLb30PNW9CRtOAeouffF+jtiMdmF1XI3KbBZK+Lmk1CnZko9ZLjrlPmRnGm0Fd0MvIIK7D3WlSST3O39sZ+EttZDRBKvQ42/wDEhlMZo6WVJcdnck0kipMBKLk6L6wPzQVjGHtcVCFIUolwjyKTp0/cm/29vtj4H7QsOfT4w6a+kjQR+BDfp81+l/ZjiLK/Amwkd6IlunTcH5H6I+LVFsSkEsNqQ2q5SpHC+b2VYgkfqDixMlddc1ZMSqDSaBSXEuNIYZDkXzNi/CdYIWsE+iibk98RKn1qgxoiZs2jNSXlpGwlCgmOQk8qUNRUu5BChdPcWtie9N+o86XXYlOoeT8mNSY5U+FiIyw7wLcOKUVKXcggJBUfyBxzMEbJDVtEU5je7kG3P6BdniHs56VxmpBIwb5nho0531PknupyJuY6hJrD+dIyJUl1SnmJzkqIlkkn5SVOJU3oSbgeYfphul5TzImIZ71BkyooGsyodpjH5lyOpQH64Vt5XVW58icM5UlubIdcfkJS+626XVElYU0tKTfUT2BvjrGVcw0Wa3JhZigRpK1fLUsSqe+4f8l3ZAJ/I4/RMd8gubr8rVJaZXljbC5/FQiZlvLdYWGpsKBLWAbokQYsm36OtqOI7UOhvSGpvLdldMOn8hTvKluZbYaUT/umgMXw69nupXbrOWaXm1oI53o0aesI/tB1opfB+5w1us5LUoRqnlev5fllN/6s8pafvZmWL/oHL84s7RoVLm5ln974WOgkiazVx0doLUqI4l1pynT5McJWkgpO3uBBII7EEe4xa1XagZgg+DzLliJUoqLfLnUxqQkW7EFAURb7EYka8pwJISrL+caTI1dmKk27Tnifa6wpon76hgmZk/PNPbEl/Ls16OBcSIiBNZ/5zJVbCus/Ryjbt2VQVToL8OdZStmRkCBBdc5vS6pLgG/uGw4Bf7WxEZfwV9Kp61OUzOOdKaXHNxOtcWe2Ef2eWknk2N73xeb9YUl1TMtaCvXoLbw8w/zV2I/UYKKqSqxVSmEqVwVN6mT+hRbC9izktcVZUQDKw6LNtT+BiQ3GfYy71kiS31ubjSarQFMjQB2KmnlG/bnRbEQd+B3rBRXQ/l+vZGnSS2skwavIp7yQeCBusDv73t98bHSzGeG9HnVBkkckPB1JI7fWD/owYWpiRqjVNty30JdY7D15BxX2J3Gq2x45VQ7H9/BfLbMfTXqXllkTMwZYmUppfKVypEYavewS6om35cYZEQp4Ud6c2lFiPM6Vfnbv64vKNQM85tyb4uh5epDsZ1q9jOjIJR68LdHPB/h9xioanknPkKQ6zMokhoMXCloQh1IFr8FClDj88bGQxuXmnwFmtkmYprexplzHHB+6UIJsD6c/rhVToGTW5DSa3Ormwlw38AmI2s88XLwVyPcDEqyr0tezHTG5FTzM7RXNdlh+KjRo9FhReTwfywuHQGlLbV/+duhPlAFwiK64R9vllXr6/cd8QtpmmxTxwEi6aqhlvpQ/DTJy9VszSpW3thufOihHvxtR0m32vgijwMmPRHUHLCn5rYsHXqlLAI/3KHUj29PbEyy90Gp8ZW9+PVWcsGwRCoLrjV+ebr2/Qe4xKU5cgUIomU3IQkykcNv1LwkNJPe5SZZJI4Hm459CL4Q9nezRdXNia7dVhShkiqXiQsmwo9Sjq87bjsmSlwc38q3Vfw7jCObR0vPvRIFCbhrRdBaYgkBu3NwDcj0/TFsMZpQ5PRKrmV6PJXHslvwbMeI0i3Ze81cn24/jh8RnhLSrRqjQ6WskOEOVOVMeH/xGnLDt2729MQ9ruxiPZR8yqLV04z1U1DwlFfYkFsEMuhMbcB/fQFkYIT0L60TELe/oDUS2LgKVJjC/fsC7yOO/bFvzszzZSTbNcuaVnUdqnsNA3PbUtLaz27n/AL8EtZxrcPSuNDaWoC+txIedHFu27ZA9eBi1nb9AkcyHkqkpnRnqK3JLK6HCpinOXRNnxY449/OTf9MWNljJ72UISv6WRmwknQTCrURLRFje+sXv9u2JFT855xcWoHYiJTwpbVPDbgt7+Xn3wtczlUdwLkmcsf8Au6REPPPq4L4JbM8W/BFpjYoKnJWUP6RKqlAzZEZYQ4HW2npSn9KxyQlxppsA355WTiQ1HLWRq2/42UwC6hzWsUiLIfLyyBwSt5Ogfa57k3wKu1uC+2qZVKfUXCP3pdZitI+3ykAWt+eEFJrFClkRadSaO88ohZQVP1J24/stBLnF+wFucD3d5FnFQyt5BDlO9O6fETRxlXMclRs3sOS2Y/0HgkI3lnnnviR07KuVHmWHofRebUHg2Gg9Ka0hSP7Bspu6ACRwOfX3IadkjqRU/wDWPJeYRFdPmSxlswkufxZbP88TqjdG+pS2Eb3TaG1xbVUNpCz9iLKP8cDsmDmm7RxUM8HLaUGJvRzp5BDXCXJ1CYeUgW55ecV7AXtzh6y7V8yQgtjLf9G0NjnapdIglQ4sdIZjcW7fliWv9AM9yAFJo+VGje6xJrE1Kf8A4cfaH8b4UwvhtqEZq8+sUlp1Y1Fum0OVOsffzvj0PbnAyxc1M77phmZ9z9DZW0/Q85zpCrEuSHFtMpNuCDoJ/KyB64Yk1HO1aLaH8qwm2kBB0/hdTnuge4TustD7cYkVU+Gat1B8Oiqgx03AL8AxCLH0Tc/zP6nHR8KFTLadOeYy0Ei0ZNGcUkW+/iQP1IxYGQJTJIeaZJX4lRVf1FyBR3l/tXZMaJTlN8H93dU7a57XH64118AeXqrLfzbnqrVw1JKfD0iKpuY68yk/tnrBbrgvyzyDjMkX4b8ztuJaRX10qOO7rDEdsufkG1Ej9cfQf4YOmUfpV0co+XW5kqY9KceqkmTKILrrj69QJ/JGgfkMWP7Jre6owv3erdHbGePilrC5tUypklhZIcddqj6R/kDaaP8AFbh/TGh1GwxkvOlURm/rlXJqVaotI26U0ruAGr7n/wBlU5/DFLeadm6g3UadoqK4DX0UqmaSPZxzz3/gW/4YY8s5izEwI1Ph5iqUeMhy221KUgBF7qAtzY+2Gmtyqlmd2sSWErZdqcrcRvNnlrd4Ht9CQMOeWsl5wdUSxR33TtrQHENLsFlBAPIAIF8UOizarTmc0d1KWp0qeVy5Uh59Z1qC3XVOEGx55PfGg6JnuJ056b576n1BhDsl6r6GWVuafESQ0wy2kq9Ap0kk+gJOKYbyFmPLcqlsV6G20zUH0MtOodCkk6hdCrdjY359MKPihzJMyl8KuWcwUyNElOVDO7brjUtrdYW3IdlpAUO3Ypt7Gx9MPE3IFTI4uGqxh1X6x5q6+VSdKzbmGRV48QOuU+SuzUdTb6/OlhofQylCU2Butd+TjIOeMvNUFcd2FKbU3KfdAjEjS2EEW4POg3NifbFpVyZmGgy4cijvIlv0+MhvYUEASGkeVTVxe/AHPuBivOtUI0/OTlICUtqhxWHXGE3+Ut1sOlHPtrANhi0HoqLKvHtBdUQ2G7k+VPYfbnAS0Q2F2tftfCxmey22mLOgNvte6flupH+Sq394OHGnU9NYjOMxkLUpoKU002Qtdh6rHBIse6QfyGDZMmBHF7HBimiUhWoH8u2OOtFlwoWgi3vjoPlPm4PcYgCidMv5ilUJ07aQtpY5Te38/wBMTULdzm5DWqUhqVJbckLBIQXEI4AJFgAbK4+wtis7H6vXDumuTIMpLsF4tbTIYQCgKAsm1wCODyo37gnDc7KKZtrnUeK0wh4RnJM5CPE7N1Jb0kE3vb3POH3MPSjPDLy5kurtrqSrSY8RclLLUyILBTjT61BBWNQJb7gXJ7AGGUvO5eQ5CrVw3I/aLSsgD9LHt7ffFt9PM+1ehsR2TS4+YMvty2pUSHJWopTIBuhbDix8t0eg9b2IKMAlBnmm7J9dp02PBit1RURkEtvTkta5kXns6kGyxqPdSFd8TutZopdCzYzl9LCZsWoxUORJLWh0SXCgbmpux81+LDuLexw/Zk6a5V6xx4Ve6XZllzKjFi3FPep8SPVKeGEquLgNeObKNVgkpdBH0rviOsZC6/ZArFZMfLECoTKBHD0uRDlxkzHWw5YILKjuuO+YqLYTrAuebYQldFtP2zbhN4zVRCt2MmVFkMEobQhbe1IYsOA1cWcb4HlNiL2vawxx+qTKeEKpqBOQ4jd3KZMspQNu7R4CuO5ve3piN1+B+GfjP9IskZkjSJqd5hTEcOMRn3Uh4ELauhQUhQNgrgqPa1g25Qj5Oqtbj0GfmFeVn3HWz+LPF0RGVkXJeGkrb0k/UARzzYc4Zgzaqn3V2cMB3Upl5lYLyhIjMFf1JRLp62XmxYX1Ocg3uPY2498J5ddVNBYDIRuKCQUBxQDfGle2u5RcH6R2+/Nr0zR8CHWaFQmK/l7qNSc5JWgPGFDfXGeWggEKbU98t66CCDrHcWvfFN0fKFVy/nAQc5U115bThafjVCOUPNn2cSeUOC4IPFxyLi+Je9yFs/g87JWNm7odz5DzUIcYzBWXWwmmut6kboFrmxJHbub2P8MWtS236TRo9MdKFyi2A8UICUtarHaQE8D/ACuO+FsuCihvqgRY0OMnWpd47Ib1BXZWrvY9++EzRcbcKgCL+trnHJqqoyDKBZfpr2f+y+mwCRuJ1Uglf9jkB56/iuoJatqcUEgkKCVY42jW7dKPW/f+AwMNsoYUVu6XAL/bT/lH0xpv4bPg5rnVFyLnPqPFlUjJy0hxhghbMusIBuNA4WywfV0gLWn6AAdwZ4mOmcGgL6RxBxHQcNUbqytdlG1ubj/lHMny29E0fCt8MkvrHXBmnNcV9jJFPd0P2UUKqrye8RlXfbH0urHYamxdRUUfSFpiNGYbiRmGmGWEBppplsIQhAFkpSkcBIAAA9ABgNMpFMoNMiUSiU+NAp8FlEeLFjNhtphpHAQlCewwrCE28vbHep4m07bc1+MuM+MavjCv95mNo26MZ0H6nmk+OgXUL+/+jBq0JA+nALHVY+/+jFkpGR1l5KNvfb6hfO6YhKX3wR/jF9/92cJ0pQEDjB0r9q7wb61f9M4T6SUDjH5Pl7knc/zL92w+AIYaSRcjHChF7kAfnjqQPbHFJue3GMsj9Nk913bacG8sA3748G2VpOpICRwB/ax5CPKURwSTi2uh3QWodUqimrVkOxMuxnNL7w4VJWL3aaP8ivsOw57dHDcMnxWZlNSsu47/AO65mLYxSYHSurK1+VjfmegA6nkiOhfQeqdVamZs/dh5ahu2kSUiypCh3aa+/ur93083bcdAoVJyxSItCodPahQITYbjsNJshCR7f3knkkknB1Go1My/TI1Ho0JmJDiNhpllpOlKED0Awv7jj1x9/wCHeHKfh+n7Nneedz18vID6r8pcX8X1nFdVnk7sTT3W8h5nq48zy2CFjuPY9j0i8ivY9j2PYiiLVe9wBj17/e+PG5sfbFedW+sOX+lVGD828uqygRBgJVZTxHdSjzoQL8q/QXPGM9VVQ0UJnndZo3K00NDUYjUNpqVhc92wH7+JOw5pV1S6qZa6WZfNZrTpdkO3TChNrAdlLt2T7JFxqUeB9yQDg/qDnrMXUmtuZlzJKDjjmpEdpCrNRmtVw22PQdrk8qKbnnBGc845iz3X5GZc0zVSZb10pAFkMoB4bbTfypHtzc8kkknEc2n3ULdbFuCBx2OPhHFPFc2NTGKLSEbD73mfPy5L9S8DcB0/DEQqZ7OqTu7k2/2W/mea6ttBHlSOfXAUstEj6cGykMuSnVxo5aYWq6EKXqKB7X9fXBJTovZvHkC+3IL6M03Gi8WW/wB1WOlpk21FOC7H91N8cOseVacV6BPY9Ud4dsDSVpse2OeGabN1qRc9sFhClpKlJ4HbC+iZfqmYqjHpFLgPTZ0pzaYjNJup1X29AB3KjYAcnFkUUs7msYFVLKynYZZX2aNyioVJfqE9iBSIzkqTLdDLDLSNS3lnslKfU/8An0xtLoB8N0Hp+0xm3N8ZmXmZxN22wQtqnAjkIP77noV+nITxcqdug/w+UfpbCTW6olmZmWQ3pdkAXRFSe7bN+33V3V+VgLnBAJJ/jj7Twlwa3DwKyuF5OQ+6PPz/AA9V+bPaB7SJcXLsNwtxEGxdzf5f6fx9NF1KU2+nHcdx7H0VfIEFSUqBSoAg8HGDK5TZNHrVRo6hzCmPxuPZDikj+QGN6HtjHPV+nt0vqZmJnTZLspMoffdbSon+N8XQeJVv8Kgii563GOJeW0q91cc4MccaC9N+bXt9sJ3Sk383AHP2xp9FWEr6i54/pJkzLOXH4e27l9cn5+q4eQ7a3Hpp/M3xWTc2U/FfiOynVNsOIdQ2p1RSQRY8H2IH/OOOZ0z7QqZXoOVUr8TMflNszFJUNMIL4AWfVy5R5R2He3bCdmk1pxxc+NAcejKQ4y5t8k/p34Wkfwxyy6KYuDTey6fYTwBrntIDtldHRnN7FCk5czRIWu1DkqhySk/4hz5R1f5IBv8AnY43OlQKQoKBHuPXHzRyA/JjVCTRam2oM1cFlSSCNJWg+4FuUp/U43l0TzYvNvTqlzJKkmbCSafMAFrOs+Un9U6Vf52JELaBVzN0ups8w3KjuR3mkrQ6kpUlQuCDwR/DHz8zLRf/AEYdTplNcgxZgo81SmWZTRW0439TOtIIuNKkG1+4x9CTa3PGMpfF3lFuFWqZnRiOCKg0YMhVuA63dTZPuSgrH+YMeF9oOHunw4VsXiiOb4f72X0T2XYo2nxV+Hy+CdpFvMaj6XCput5zzFX58ep1SWzJVH3DGStlCkMhatRCUEWAvawPa2LL6c556o1ichFByjGqAjvp/rDNPYabik8qUopSDqUkKF9Q5/hin49TkwmSzGjhDSjuPrCljctyjUb8BNyRa3Pe+NMdBJNebytXnKtllqimoNBimSGoQjPTXUsuK+pR1ucAELV5eTZRx8+4QjqMRxIF0zhfUka6jzOg+Pw6r6fxs+mwrCi/3eN1tGgm255AWJPOwPmeYVXqlQJSdcsBzWNZJAN7n2seeb/rgSIEBCB4WW4wn+ylZSAfsBxfBCAtCEp12U2kJNrCxtyLf34Aqm6HkvkursboBN0pPuOMffQLBfmdziXap6aqebEgLi5sq9m+BpnOKFv+dgVQzBmerxvC1jMNQmtBYWGn5K1JBHY2J78nDSpe2gLUhF/7WgBQ+/GDkVSUFKQbLT6XsvEy9ELuQNTjRC0KfbBHpc/3cfywfFnyYTpdgz9pRP1R3Fsq/lY4GzKZfstbYbSoX8tx/IgYHtxX1aCtsq9lI5wSgnU54zOtnwsmsSJDJ4KZdpKVD/lQrBCK6y5qRMytliYT6uwAyr/nMFu+Gz8KZbuqPZOvkqbNscchPpJTuH7ax/34ACN3JwT/AEceWoryc/GC/Wn1dxIHvYOpdH88LGaDlKWlbkbM9UpjoJs3UqcHUf8AxY6j/HTiOKiygsNrS4gIJOptzSP1APODG1zI6tSS4kAW85BH5374mS3hUJusHVmoZSjPKWjMUmS4g2Fm0uWB9kgW+3BxFZlUhy2ksOvVVcVu4QhaW20n76bffE1oeRabLnJZayvm2cSDrbp0nWU8HnebYCPTuDzfjnFoQOmaZ0UfgnRmpuSVpDYk1qpvOp4A5st5PH+Tf3J7402ZG5VuzOas5RqpDo60vU2RJivo41rltpCu9uLcYmFK6iyJERLM/P2aZSgNCkRJK39J9BwRbj0vi76R0NzeH0uycvUSCk/NW14qPHSD62EVLrp9uSMWTC6QOphCPJZy4y2Rwg0JyYPyvJese/8AYGIZIuaUNd4Vj2bqrBd0zq18s2U9VKglhPPqQVE/wGEsKiUBS9L1ao9Skg/sGaq8p1XpaxaT9rcn1xtqD0cypC+YYFKkkcgLoMBpof8AJtsj+Zw+RMtSKelIg5kFNYHAahU9qP8AoC3a/wCZBxUamJP2bmrE7XT/ADBUCBF6bxozRtofly5Aufz0q9vbEvh/Dt1Mkta28uZYTfhWmVK3B/l3eLKAffi2NXs5fbeeMmZmrNkpIHDblVdjsq55+W2ElY/Mn3GHylR3pknwVDp0mc8jkoYjuPKH3JN/4nFbqn7qLYiskxPhH6kTY6nJNfy/TVE30iU86P12kkH+J798PdC+ClLJDmYM6bjwBIbgxXUtk2t+0c59fQY2HF6c9QZqyDlwxWr21yZTLJP6aif5YPX03dp3zsw5+y5SuwcSZRfWAPZI0i+K/fX7XCYQhZOh/BxlVLyFzKu/IZbN3Gl6lfy9P4/piXQ/hd6IxmxuZSRNKBZxTkp0JJ+4RbF6yI/SCnpV+JZ3rVa0G4TAp6Gk/oXB/O+E6M4dN45U3l/pe5OAXcPVKW49q+5bbuj+eB7xJyT9m1VfTeifQ6ifKgdK8nx1dvn01p502/3zUb4ntHyo+AhjKWT9sI+lFNowaR+WtCAB/HDy71TzHG4oOW6FRE9h4anthz7nUon+7DJOzxn6qpdVVM0VZLTh/ZiSQkfojSLYXNK9DIApH/6OM6lGuoQ4tMaUL3qFQaY0/oCT/LAF5cytS1q/GOpNMSU8KZp0N2UoG3uLC98QQSI77h8RUXHVXBI3P9HJw5RMuZgqg/qWXqtOTybtwnnbi1+SQBhSw8ym0UhkP9LYzIUzMzLPftcBAZii/wB7pURiNTKpRXtLdNy3MCgbgP1N5+3PNwjaRfDi7krNcM7cihrp24NaEy1NMX+3nVcnHmOnGequ42INBnqQQPnJLbTXtbU4bfwxMrBzU1+ymQmUSXNplsDkhTtgOfW17AX++ObEpbmzrRc86UoKieb35xPofQrqFslKTSm0KF1eLqBWpIPe+2yR/PCaR0zTQ3lqzj1By1Tm0D6WJi3nRfj6SkW/gcNnYNkuU7qKU6kSazUIlJbekl6c6iKAlaGwNagOLJPv642bFisw4zMVgaW2G0toHskCwxRXSnKvT97NLM2iP1urOU1K5KZrqtENDn0gBIQnWrzcd8X56fpi+PwpXd1NOa65FyxluqZimFIapkR2Wu/roSTb9bWxiSmyZdPypUKrMdtNqYWXHPUuunzn87rUcaO+KCvGF0+ay80tO9Xp7UYp92mzuuf9BI/zsZuzGQzGp9GbI1ur1lJ9beUfzJw7fJMxP/T5tqjQxKWm7josDbkDE+j5oupKg2pQSdKiVdsV3UHJNNy5LlQ0OJMRlA3EjhrWQkLJ9OTb9RiB5fzrSsiir1jMFSEWnJiFx1S1crcChpCfdZucVPa55V+YN0VwdZ88wKRQKU+uU22lueuQHCQAnbZVa/sCtQH54wz8QHxEyc7ysu9NYlRcao2VNsOuJLlnpARwCktgo0kqNzfki2In1x+JiRn6atqnvrejMOLREg3+W3yL7ib8kj3P6WxWz9RojcBbeZa+1UnihADEI/NDdiNDrmkAL7AkXJt34GLmMybrI593JJmyiy51aC6Y8hSpgQUKRYKS2AVG4HBv7/Y+uKYzQqU1U1zPESHg9ZW88srU5x3Kv0HGNO9Aavkat9VKVSM0TWKNl6Y+2JKrXaaLbSkx9wn6Gw4q67ehP6RbPvRJEO9Gqr7TUqjzxEK0O6kSGyCE/M7A+Um/I9u+NLYHPF2p2xOk8Aus9oVHnq0LQEPEWHoMGvUYFJ8M7daOSlWJDnPp1LytU226ZOYqMZ6N41h2M4HDthZSrtwSCMGZEZpdcqColbkeFSEpAeJIaKrgWUfQn09L/ngGPJpIFSD90qItQ5kx9MNmE64+s8ISnk/phMsqQdCu/Y3xcdLybleq0pqXWGX23JJdfYebWdxLAUUpu3cax2JtYg27i4MequTKbT6eYUVtuoy1LK0Sm31M7TfqHGnUi578gnFJCZrlXhGoDkfocKY0NchdgRbuST2x16GGhdbchKj2SW+/64NYdeavoQUpI+lY4P64XVBEPxdtF09/XDnlvNM/LUwPsth5q91ML4Qri1+O35jC+Nl2dXUqk07bjsl5DLbLp1HWbE+a3IAN7+uLl+ED4f8AInxFdeUdI83ZkmU6AuBNeamU1KQ+/IZa1BtvcBQBw4vtyG7cXwdb6JmtJ3UZy7nz8WcEuOyy08LqebQsIecN+AVrJDpueCq1v44s574rOumWKS7RoubKzUIRa2THlPFaiwALaiQoW4GjzkjSDwRjXc7/AATuV8thErpznWPUJLSToazRBUSokc3djK0W9rtKtxiC1v4C+tdIgfh8Pp5BmssIIQ9l+sx3TzYn5UgtlQJHYpv6Ypklc06tK9VhuBx1kPaRVsTX/cccp+ot9VQ2V835+zYhUCsVLL8RyS3qZZXUC0HivkNuvoSW27Gx+YNHuRg1eQ6tTapDqWbctQ36szKbWt9mqRXlMPd2yVNuEWsgkK86CPcWxMqx8JvxFRluIZ6Q1aRuJ0BK4D9hwByWty3a90n8hiq80dJviHyW+pnMHQHOlPStYRHMelS346gRxpcDTl+eQDY9x+UjAee6s9Q6opGhtSAfMEH8CVtz4YutqMxVGudGcxvUxqr0C8ukvQJqJMafS18hTSkEjU1fzIv5Qf8AJOJl1o6X0TqVlp2NUA1FrEJtSqTVNvUuO4i5DarcuNK7Lb5te4sRziTpn8LvxcZozVSM45F6YZpocylOIfiVesNtUptsovtqHiVBa08qGkoOoK0r7nH0SNCzvTKBRmOpNKpkCuTWAHTTpXiIK5IQS4hpagLEgatoi9uxXpJwr2mB2Zq9HhmIRYjS+7TkB42B5hYPz5lerUlRi1OKW59P0IWlB1hbKvoUhafrGq4vgHTPo71J6tzl0/p5leTU20L25E4kNwY4/wDeSFfLH5JKl+wON55W6b5GzjXIFPznlSmV2OmVwzOY3UBYSSbj1F0AkG44FwbY0bBp0KmQmadT4TESJHRtsx47SW2mx7JQkAAfkMZxSCZ5cTp5L23/ADRrOFMLjwmGLtJANHv2A5XA1JHyWZ+hXwPZK6dOR8xdQpMfN2YGSl1phbJFNhO2Fi22rl9Q/dW5wLXShB5xpsAk83Ku5ueTgzbHfV/PHQlI/dxujayIZWiy+M41j2IcQVPvWIy53/QegGg/d0UE8++O2+2DCkfwx7SMOuQk60G+ABHIKvf/AEYUaeftgJRqIAUkc8E9sK7RpBTsdlc1y+d8mMUPOkg3Utf7v+UcJCgaCFtr/hi/5i+k6X3kq6T0yweeQLqnA2BN1kAcAjn8+PTAEtdMHo6ZLfRilLbXwFmZLSCq9rcjve38fzx8IHBM8s5EdXGT92/5BfpRvtSpIYwZKWblrlaB/wDkqDDafXX/AM3HEoSofs1n8k4v1TPS9K0tK6P0lKldtVUlgn8h7W574Ocp3TFG4k9G6VqHquqymwRY+/8A5740O9mleT/jN+v6Ko+2HCf+zL8m/wDko10K6ETupk1uvVdDsPLcdd1uJJSuaoHlDZ9E+il/mBzcp23TaZT6PAYpdLiNxYsZsNMstJCUISOAAB2GKNp3XSfT4TMGn5Kp0aNGZQlmMw+uzaQLBFgiyLC2Dm/iIrCk3/orDb+ryuTXEkW97t/bH0vh3AKbAKfsotXnxHr+g6L43xXxTWcU1Zmm7sbfC297fqT1+WivoY7iiP8A6oOtXQtOWYK0LNnHEz1WaJ7AjTc39Lccd+cDHX6uaCt/KkRofupM9ZJGu3PksOATycehBBXlLWV6Y9iiv/qgasCoHL1PIQeSmoOEWHdV9Pb/ALxhKPiNrG4tDmVYbQSUjU7OeTqvbsNr7/xw1wor/wAcBuOMUK38Q1Ze/Z5Wg+vBqDgvbv8Au48n4hqtZKDluBrIC9HjHUkAi97aLkenbAuopT1k61UXpXSShW3MrstCvBQddrjtuOEfS2D69z2HqRhrM2ZK/musSq/mGovTJks3cdULceiUjslI9EjgX97nF6V+o9Pcz1mVXq306p0yoznNch5yvTgorFwOOPIABawt5rAd8I10/pilQQrpJSUqQkX05hmEaeebjv684+fcScP4xj01myNbENhr8zpv+C+s8G8XcOcKwZjDI6d3ifZtvRve0H1O55BUAEOEfUo/e2OjdSNOpX8MX4Kf0wKg2jpLTCkoJCzmGYASB2HN/wBcAMPpgVJT/wCiGAm47rrU4el8eSHs2xNm0zfr+i91/wA4sG/7Mnyb/wCSoMtOAAWXzc30H+XvgBQ7x5FfqnGglU3psWGkK6T01aE32UmvzbWJuq1z78nBTlN6ZJbW7/6JaXpCQo3r80fr3xHezXEn/wDuR/M/ooPbJg43hk+Tf/JUO1EecUShpaVqtpun37YPqdHqFHqD9JqbATJiOFp1KXEuJBHeykEpPcdj/A4vWEx07psuPPjdI6ah2E42+y8mszVhCkqCkquT6EJ/L7jHHYfTqTLW850gpq1rUt1a/wAWm+dZJKiOeeTyfvi//lvWZdZG5vjb8FX/AM5MM7X/AAZMlujb35W721vqfLWlct5Rrmca3Ho2WoL0ibLXZDKRYJHqpSj9KAO6j/fYY3V0W6I0LpNSAs6Z9dlNhM2oFFuO+22D9LYPp3J5P2qfJmfaZ0/bkpyj0yosFUhSVvLRPfW4v+yNSgo2+17fbElHxGZnC1o/ojT07adR1TXRx7/Rj2PC/CEOCDt6g55eXQenn1K+dcb+0Ko4mHulKDHB0Nru9bch0v5ny0KAALY7jPafiNzK4NQytTtJH1CY6f5Wxz/6ovMqlpR/RqnJK+eZT3b34GPbXXzVaE/XHApJ+lWM9q+I3MgI/wDrWpwBAOoynrc/p/LA0/ELmQ9qDSlH28U8L/fnsMS6i0ETxjL3xN09qFnmDUQEpTU6ba5/eWwsg/wDiMPzfxD19att2gUtIJHIlPqSQeO9sQvqZmmt9SlU0TY8KlLp2+G9Bdd3QsJ4NyABdGLInhrkr23CrVTZcdSlIUVBfAHrf0++Kp6m9WEUsv5cyi9eeFFuVUUfTFWOFNsn990dirsnkC6+1wTMoTpUd2DJqoLTqS26A06k6DwRdDgX+oIxDah0PyRAp0qYjLtMcTFjl0tpVJSLAX03LxA4/wBGM2IPmkj7OmNr7krq4LJQ0s3a1rS63IWt9Vl+Q26ptSWlKQs3IUSb673B/O9jjSeUKu5UKQzWIxIDzYfWLdg4NVv0JUP0xSfUvO/TfpxUE0+R0kVP3I0eSh1qtOM8OocUBpIV/tZHfEh+HbrXljPlWmZUo2Uahl5DVMXMjMyZjcht1tt0agkgBYsHr8+mOTh+Gz0bXdo7Q/iu7jOP0mIFohYQ4dbbfBX3Vobs2ja0Ku9HtJZUO6Vjm36gfyxd/wAMecGk5lnZfWsBjMMRFWic8B9sBLqR9yhST/yZxE5+U3ZOTZWb4oJZpq4zchKRwWXRbX/mkouPZRPpiF5Lr7+T68xU2tYdy3UkTQP7cRy+4n/mF4fqMdSN1l5yVweCt6FIKgT6Yr/rnlH+mPTOr01hjdlxm/GxQPq3WvMAPzAUn9cTyPIalNNyWHEraeQFtrT2Uki4P8MDcTqFvf8AngVdO2rgfBJs4EfNJQ1j8Pqo6qLxMII+Bv8AJfP3KbkBSCmp1lFEitFMpD0WCl2W44i2kJUrkXve2oJ4PGNCdEa3TGsl1pmqZonTWJQkONIrKUIUtltBDy7hSlFv3udKeALXwzZzpXR7KeZp1Bm9ImnltWUFoqCkpdbcGoEJvx6i32+2PJ6hZJgZfq1CoOQBSzVo7rLjwqAXbcFibkFXc3t2vz3x4fhrhSswSozyuaWi9rXv8eVui+i8VcYUGPUjo4GPD3WIuGWFvS7vjfknar9I8tzm0y6JPk0sOjW2j/VMVX+5BNwPyUfyxDqp0uzhAUtbMFFSaAJ1w16lWH/ul2X/AABw2UutVPL/AJ6HU1xk9ylp75f+c2vyfyxM6H1hnIIZrlHZfRYHfhKDSk/dba7g/oRj3GWRnmvm12HxKtUIGtTO0tKkdwWSCg/kvkfrbAvDocSFvpGscnm3HvfF6KreQM7x0oqC6fMcRwG5p2X2/wAiSCP81dsMVT6M0KYpcmiVJ+NqujakjxLV/YKBCx/PEbN95KY/uqqVurU8Ah10ni6iQQQBjjbzqDpLnIBKQk+nvY4lFb6dZlogW87SS/GAFnoJL6R+YA1j9UYjKNC3dbUhvycL4IUP09P1xe1wcEmXLuj2lvJAWzY913K/9F7fpg1uXqWC+GkpPuCFH7YSpQsALSpC0q9Dzg5LjyRpKNKST3buf0PphkNOSPLkYaCpbjdzwVCxP64OLLp8zTofB7WHGECFXcJSRqtZXe364NZZ2TrjttFWu5KfKpz7Ej+WFJURin0pQGtbukW0pUuw/THkzIyU2eZXxwrz6ir7/wDhhledlPLsHnE89lWTpH5dxg1AkqTpdcaBB4XcufxFhzjGRfdX2Tm9UY6AqOlLikfccD8rYKDuhQ0NAfZSeT/5OEa0PLUlKZbnPI2wEX/vxxVLLyXEuNXIN0a3VWBHpx+uGa26rSxxxy/mmNIsPMA4LH7d8JjJjJUUp1k37NIKr+32wOBTkS5P4dBjIflrtdmOkuOq/JIBJxLIfSXPE51DKqFKYZVxrkNjSgfdOrX+gGAco8SaxUVW80olIizB2BLpCbnBW/UlJMYO6GybLQ064oc82PP/AJ++LQHRWFCS27mPNLsO3Km2mWWir3ILzv8A9piQwsk9N41iiiZnrym7aNKVON/l8opR/E4UyNHhQyOVFqSp4bTknUpZ7WDdzbvze+FdOy5PqgIplJk1F09/DRluj722xbF4SKVm2KSMl9HMuU1CxpRIqDzJkfYlKb2P2JOGKpZZ+IaakmqZoiwmQPpZqKYyQL27NpHAHPfEEnTRNlUO/wDRdmqDF8VNpselM8fPqMtiMn/5lX/lgLOXsnwVq/FOqFPUEgnbpzDk8g+wUhIR3v6nDlLyNk+lrW7mjPMirS+7jNOVHU8r/OcLpv8AqMKqS10ikfLp/T7MuYJqHLht6Sp0k+9krtb9MNmzeJG3RMblV6QQSlBGbJzqCdIW6xDQvtfyjzgfn7nEgpDq6gjXlLoKiUoA7UipbkhN/e7qUpP8cSykUjqFHSDlXp/lbKDBF0uPpSqQkH30ajf9MPK8hZ3q1l17qnU1oA87VNjIjJ+/mv8AzthC8A/3/RSyjFIoHW6G0JFQzLlzK0X99JhxUhI7iwQm3H3VjyJ2XYslK8z9bq1mOVclECm2S2v/AJNtJv8AqbYe05S6R0WYgVJ1mqVFKvpmzHJ8hRJv+yTe55/s4MkVPPyHnYmQOnVOpkRJ20ypbaY+4n0IbOkj8iDiF99tvl+qKQDN05thZyX00zK+RxuPM+GF/fhN1/xwjfldUnnGqpXMzx8rsG6g2tBWu1r8pWmxt7FfH2wqm5K60ZhVqrOfIkNsW+TB3kp73N9tKb+nrhlm9I8sQVKezX1QSl1Kef2QUn8g4tw/yucABv71UQZMvpxJZSvNnUjM2adLdvDNB1pg88nSgD/p4b3+o2Q6GrZyl0zp7K72EmclC19uFaeST9isYNVE6CUJSNcus1x0Agq3nAlXHrYNj+Aw+0XMraWtPTzou8hPZEx9gJSCPUqNj/8AP64doF9lFNOklfr+aaLJrlcWgNuPbMVptnbbSlA8yk+4JPr/AGcWAOBzhuoZqS6PEcrKW0zlspMlLY8qXCLqSOTwDx+mF6uRyLY2jZUON3WWYfiMrSav1OpeX0O6mqHTy+4n+y6+b/8AQbT/AM7FSKcNQzmtP1IhJDY9roFz/Mn+GH2rVwZjzdmjOrt1sypzqmSrn+rteVP/AMjYxHsnIeU3KqkgfMkOev35P8zhxZXNsNE75sq0iLlqXT23SlmouMIfAsNQbVuJ5tfuL8YxP8XMypCkU+0lbcPxYabYSbbrlidX6f6cbTrMCNWaeYsh1bZCtba02ug/r9uMZW+KSPTZeYKNliBsLRSKY/LcTv8AzG3XTZtak2N1qsQNVgADYdjgRvb8UkrljV+WuMlTSEcJ7ntz+eGJ+bO1myV88f8Ak4e50J5yYW7nSjlQHYc2ODmICXoaHmY7hZcuUEot/EemL3FZ7Zkmy9U5MeUhSVlBHPf1vjREHPSXun0qmZwU9MS03EciVpDiXFtNtO8NSUdyhKFKAdHI4BBAuM8MxVeI87ZAJISQOQf/ACMPlLrsyC25EcjJeQUFsk88f+Tgx1DoTdq20dXJSm7Sp7IyZRHJxQmGyuNUkLMV9LykGHLWoKBbU2foeQCBcEa/SxGJfI6MQqnBq9eytSG4sdEdMSoNMtOFqQhYsAsG4Cr3XuJN+Oe+KZynXcx5TkiLTqkh+C+rdRTpibpKAQbtLHCb29B6ci4xuLobXspV/Iwj0eft1uW+HJ0Vx3zELFtrb1pQ4Ba6CQfW45xsqatk7AQFz3Md2rnDYrFFSzNPoCW8uZpgutux/wCrMuMgaHNAsDp4sbG1/Xi44wldqNKmyPBUeHMq0+UNIQw24oqJtcNpFyfyAA4ONKdbst0yn1fNkSvZdBh0/cdVBeTqcKG2UqaJCPU3vx21W9L4ofI7fUTI79JcyvmReXYWeTGdaajR25AcVZX+LcustNg+qhfWLXtjI5hFkGlQKo5NzPCk2quU6tTnOSfHQnY4Hvy4AP4HBpyxUGX2VSGWi0vg25/8CMaFzfF6y13Jr+WK5n/K64FQcYdKpVNcZU6GnAtINiQQSASAg3/LFCVnJGeo0Q1rMEVunNVeRIjteGQhQ3o5SlxopQq7J+a0oBQGpDoKSb4rcDyT5VHaxnOpxnXqZSnhEabeWorauCTbQLX+myRbj3PONG/4MSCyj4sMsZjn1I0+n0aLUZc55wENqBiqbabv6lS3Pp7kJPfFb1H4cRl2Uin1PMr86oyEo2ocOANSFm191TigAB7C5Nu2DMp9Tn+jzdFcyu7sO/0pTPdIIYW7DiEtAPFI7KccdPrYJwpYWNv1VzHAFfcWq9VqYypxGXKfImaLFUmX5GiPWyR5z6dyO+Gpvq/Uw4UP0ujhLfJO87ZVxcW54P2xV6KlT5YjT6Utx+PJZRIZcTYbqFtpULEKsSQbetj2wq/Fm46UqbeaDUdvQVboV6Wuokcdynvzf7YwZ3py3qFY6es61NF9iiU6SAvT8uapJJAuRp5XccXAHBNvfAY/W2pLcQmPlhLaQXA+FynmXU/2LDSQb978C2K7XUIrjwQIoS79HzzYk88pHBtye3e3PoMJ0uPurWp+KG0rKEBCUEJSdBuiwNwewv6c9hbBa8jcqZbbK2I/WNiUf9jrjil383il27HkqLRsLi3vhJX+peXcxUqXRq9ktNQp8sKQGHZgKniEhSVJCUa0kHs4DrSRcWJGK4alvSVKVocfaLQKnQ3upIvwkKIAFiebHyEW7nCuFl+t1UHwDOhssloPL8jDS7dm+QTZenkAkjg84oqa+Kjj7Sd4aPNXRQvlf/K3UhyZmPprlB51+hZbnu1Is3RImVZyQ4oHggLdFkdrEgXI9TfmaMdXqdMu3T8s1OU4nuiM404EG17FX0I/U/bEPp2UQYkX+kTjc6cEJ3kQ1ONRNwgBehKjuFu4uAomwJ74VS6rTKCwYcaMlaGW9QjwU2Go8+2gepvyT7cY8jNxmJpPd8JiMjviB9NfwC7Iwlzz21e+3x1Vgs52iONpW9SpUVSiflrcbUf/AJDbBc7qRQKcn+uNyC4RqDLW2pzR7kauB9ziqHapmavSVw6P4ZphBW26WGyU6e3Li9JHre3Iva2F9PyfEhlL851ctz1QwnQ0knm5VwtZ7C5I7dubYodxJWYYc+KTtB/7bWgn4nl8Uww2Co0pmH1JsP1U9idVcvTXEtxqTXHQbjcajNraFuD8wLt34v62OHX+mdINk7cu59A1f/Tiv59ShUhkMqSQW27tsMICQAOw44QPT/RiG1euV+oBcdphUZofLWy2vzGxBPnsCTbsOB+eLqPHMcxl4dTRCKPq4Xv6dfgknoMOom2lcXO6BW/O6pZKp7imX5763kGy2WGt1xJ9iAeO36etsJHOtGQ4hDkh+pp0HUUmnrv+Q578f34osMKQ8ChlwNEdnEFJKSTYi3JPP0Ajt6YKfYmtqIQsxXeVKWokuJRbnSLEkEA2783vj2kL5GsDZDc9bW/VcF+Quu0WCHvxy85MEx1KlEAAynBe5vcHXySCmw0d+bWx1Trl22ZUmUvzCwflF0NJvybX57qFxz7YJj0mYd1laWChy7m63cqUb+4sOeLW+/pgx2nSWYqUpGtoqBdulxSgTa17dzdI47g+/GKmUdMx/aMYAeuVt1e+rnlbkfISPUrifnKW0pTgbUL3CtV0emnmwvYgkj0wcwYwcdV41tLjgC9LrZU3Y2HBNgfa/fjtgpuFDSlL7MOSG2HNZ1s6kuHQDxYkDvbm3IP54ODchwqbYsGy6QhbiXGwQvSR3uLk8W+3fkYua3N3is6CtNMt5Xn1Bx0hBCV3PIHBANkevFyD39wIrjOTQtqQ0doErYTfi6ub37GyQeT257E4GI011vxLO2lLZQoWJ2wNd1I5Vcg2/wC71wTGbksRm48psvk6GwpmI2yFeh8pUQ33JBNx6WBAxGqLri2AX1JjBfNzdpB4XcXKb9uQQeTbi3bHHnkmQWTdbekFspQp07iBYJA7Dixvf+Ywa8+XFvRlrCDqcBWqMRqF7k6eF6yRa+g39eOcJy88y2lXjUIJsLKbOojsCRckWItwbdzwDi5RJpLjCA4nWvUtVrLb0pCwfLa6ddr89zwfSxwnDcRFlTYzRU+dAWgn5nFhfg3tf74XutraWngvsLuHXHwFAI73Jtx6D3APfCeNBeDqAIyPm3KwVgJTyP2hJv3IIt7n3tiJSjVrpjaA+VRwU/JacCRxxYnUe/b0+/tbCpKdZOhrUpZ8hcBIKwP3T/uOLki9sBbfcbSpLMRaFODzi3qLkEm3N725/I2wdGdedJhq30tkEje2eWzcXPJA57XueMRBFMBhvQlzcfCE/MSbK5HNlEG/bzcD9ccDWh50voQjsgJ2/Q9wOLhF+f77YEYkttLKEusJbHAbIWotkd1pJPPPJ4P5i1sJ1xihG0zoUXAha1M6CSi9tY1n/wCn07YZRdeb0kbsddzZGos6RbuCb3sLc39/TBSXHkJSlCwV9/mKVci/BJsbc+mBojXQXkhbqlrJ0Ibs4Cv6jcnye2rnn37Y4yEApQyrZV31Oi1j35Iub9+bYBURJehu6iJDTqNBAUDrRx39P0t+uDUrISCwsJUVgnU3Ygev8/Xt/HBYadbs89tOk28w4AJF9N+Pf2Ha/F8GFlWoqU4EPaNopD1kquTyRb/cn+POAjdBUhbbzjrryoygdBQlw3BNrg3v27jn7c4NK06hqloWlRuQpVh2ub3/AEv7A4IaZ8OkPLkNIYF/IbjbQL2/hzz64LW3K1KdQGgf30Fwg97An/6PfETJb+wshYLBIBWhNwFHg2uOTz298ccmssONaXttSyNLahwAOeLjt3/nhKWNRKnZDDaLWJHYX+4N+57cd8FrXGbYGpqQpTg0NBtpblr8C97kDtyfyxEpSzfS0jSl7SF+VqyLKPuCe1rdgPc3wBZ3kBlRsgjcIXIsRz6AAWt25vhC0laVlEAQ233HLuuONHUoCwuFWPPb7HChTQWu6FNJK16yVL3OOAUCwvew45wyCNjvrKEa4PzEJKQdzU2fY7gsDx7YKdShViuRICEJ3CpshOnnvfvb8sBXS4L50mOhOvkLC7OOW5A8g/lgtxiZYtOwQ4EaFAhwqUQDYdgbnkEfb8sRTdOG+8w4LeZKBYuamyEj7i1z73vhYXlpTurfjhFwQixta3Nj6d/S+GyO8yyVvCC+VtH6tQUlR/yLcm3r+XGE5DTavExosZC22y25w5uKX9ref1v+lsRTKn9Co0hKXG5C44QEJVqbKwpuxuAbg67evP5Ya8xUqNXcv1iiJmRHBPiuxk7guQ5a7aux7OBJHrf2xyMXw8ytqTIaXbhsDi/oskep9revvhautJY8M8Ibu2tfzHktBQaXyTuG/F/yPpziKLAfxCw01WRSavpQnxNKYvfjQWn3bg/kJRuP8nEQ6AT2cp9VKNVVLcRYLZeTcG7Dqw2qxv20m9vTTi7viNymw4zW4cRQWiBJmKjLbJIMeQ3vtWPcgeUc+oxmfpWtyp5iq9UZTvMx4jKQ8pXKVvkAWB9Plkfx9Di8d6OyTL3wvtr0MhwM2dH6vlt5aFvOOyaVKF/MPlAIJ/zCkj/wxmZ0qj1enTpCNPjUmBLTbgOEcE/8om3+d98Kvh765Ssmyo9fcjGbArlLj+MjJd0kyG7jUk+hC9Q59DhLV5CMwQ5yWiEPOuLlt6eyXCrc4/Im2MYdlIC2tHNa9+HzMyq/07hwJDhXKoTiqU6VHkhuxaJ/NpTf6g4s098ZW+GTOIjZvEF5e2xmmCFBB9JjAuB9rtl0f8mnGqR5h2ti8bLO8WKo/wCIvLbDppOa9h1RQr8PkFokKsq62jwQbBQUP8/FNGNu+YIBNuewH641rnugf0lyrUqQ2jU+6ypTH++p8yP/AJgB+ROMnNPNuoCksoSV+hOrT9v07YKF0ldD6FpSEtWPFlLQLi3oSb4PS8oJIkMot2GlYT/549MGvpaQk7i2wEcDyYChllYWy60TeywUm1hiIICnESY+2pDdzeySRz9sO1AzBWKK6lcaS623zZJcCknn0I7/APnjDIaaySXG9xlu5uUKJIv7c/3WwdHSGAFIS4n0CiRb29P9OAWh26itrLvVOIoLRmRDjJR9MiO0HEgd/MEHWP0BxK36blPO0PxTrNPrDSRYSGzdxr/ORZxH5HFAla2ka0J3Lj6iRYffjvgcaXJiPJnRZDsN5AJQ8w8WlX9+CPT0OKHQ63borRJ95WRW+i0d5Jcy1W3Iqv8A8Hmt7zR9/mCyx+oOIRV8h5voLTkipUF0soPL8dXiGgPclu5H6gYkNP6tZnpygiQU1JCQn/VbSA6f+UbIuPub4mtN6uZXeKE1FyTSn9OvUUlxr2PmQNYH5p/XAvLHvqmtGVSDZjECR4grQkdkDUCD98ASEPrLaCsAEtm6CAcaFnZCyRm5v8WNOhyN+x8fAd29z81NmxP5gnETqnQ1gq10evuIUjnblsXv9txsj+af1w3bNcO9oldEfsqkXZ11DZpZS531K8wH/hgzenlB8waTpFitAI/PvwP1x1UDfaU2meQskldm0Ekfe/pg4U54BRdUJKyQvQQNIBHpb8/tityVFomvuoCFyyOLENNi5P8A5++F9LkwWJLEufRV1NlCtTsZ95bIcHaxULm3b09LeuCkQAyVNJhNISslZQ4si9uw5PH545cKWv8AqKH0tkeYOjSfy9yPvhVFadH6uwWYhjRoBy+wk28NSafHWCBz+0eULn/k8OLXVDJs/WmbGzjPSPOUyH22W/y0tLSj9LYp52XFQ5rX5yjlWhvhRv8Alf1/iMHoqDxOgtrQF37IH6E35vbEdCCEc6uJPV7JVGeBpGRH0kDzuFDLagfz5J/jhBO+ICtvrWmm0aDGTp8pkKW+q/vYFIxULk6SlxSUoRa9tQ12v7dgL/ywNDU5xvxG80UrPNgdV+OL8/xwvZN6IZ1Mp/UvO1TSoTs3Sm23EGzcFCGLf83n+eIlJc31bs95+S4gX3JTi3lcHg3JJ9bcY6iOgqTuTEBP76Qk6h9yfbHF0+MhKTvl0Aa9ei4A7e/37jFoDGt2UzFSDJcjp3CkrezlCnO8AsNRhqYI7+YAhZv9jb3xZS+tmXKfGELK+W5TzDYslGluM0j7aU3JxSjDaCC3Gku3QDdKOE+vNj6/ywrCoyGttKApOrcJJt+nFvthSwFEPKsxHWKpT1KEqRKpyUGwRAgMknj/AG15xVva+3hBM6j5XeQkz8vVOtuqAKEVarqIXcX/AGSBt/8AyYrtuQ9KGjQ6APOFKFgo372PIwLwMosmQktNthFiUqusj7cX7f8AnnAbCBvojnKsJrqzmimsFuj0PLlIik2S3GjkkD2IuLn8sNlR6oZ4m6j/AEkAQgdoYQyfy4B5xDjCWpH9ZDhWO5svaFz5QAbXPv8A+RgtuOFqcWFsIUTYLKNNza36c3H3w3ZtUzJ1k1+ozkiTPrUuUsG6QuW8uw7EgL9eTwMOdNe6cspZNWjZglutedxpl1lhlBH7tkC9uO97n1wzxoRdGl5tCFAaEt7g2iQO40AenofXHTHlQWBfQhKPJYJ+pBIJFv8ARxh2tbySqyqV1J6e0lHiKXkZUYhXDqUsqX7X1EE4kGXOsNLzNX4NDiUGoFyY7oC3HkkNgAkki17C2KNjR0LQ25LeQy+u39WRy8dd+Bo4445xaHQOlKk1yXUnWEpTTmNscEkOucEEkX4CT/HB7FpOqYE2V9I7XxD+rWZF5U6c1+tMu7chuGtmOr2fc+W2R/nKB/TExHbFDfFNW70zL2TmXPPVJxlvJHfaYHF/+UWj+GL1WNVnyaxDpOVUU2U3usvoDCkKJ8wPf19vvhM/nPL9GhJYlyXEPo85aSysqIXyDe1v54QZ9q4bqUKmIIO00X1j8zZP8gf44qTPLdfquZakzDs1HbWIzSgeShDaUk/bkKxS5rnO02V2YKWZt61uU+mKk0SiF2XIcRGp7Dzl1SJK/pQAOLDuTf6AcZr6z5kXkfN0Oj1WeajKk0Zyp1mShfmlzZLhsQPRCW20hsXsEAD1w61PM39Fov8A6TK6WpMaIHYGW4znlVLc7OvpA9P3b+yb/vYzRnHONXztmWZmGrzFvzJKwCscaUdktj2QkcAemLomZFne9S2ZT0TaMifoSqLUAIxVtgJJX5iEn3uPTD6xQID1DRBYW01O3ggDslKNBtcD6LWTwfe/rhz6VJarmUG6Y6kLksLIZb02sgXusW59QP7sCfylOiKcVCkXYkpXu67AWAGq/wBuQLnFqraopNytofTsr1oCGtxQ9CQrVcD2Kefb1wFjJcxTa6g02hLJQhwqWLNgen6XsPyP64nmW6PJqrQRsuJS46C44m12ySb3Bt3AN8TGq5Z8TGkRoLior0dsNoaADOtoiw4+hFzYA3tyPfACZU81l+lTHlRnWwiOizzkSSssvN/29tdib3/hax7YkyaXHph/CkWfnJbQ63KS8UuuNdi2pu224Laee3a+m4JcEwimbtvz9MhtBcakqSHG3rHsoL54N7kc+h98P6GEeHfemRN6LthwtJvZtfYrt3BBNwQf07gQmya6g9bzl1MoraqlTJBlvxwhAiTmg8zIaQCAW3NWttQT+6HCFeliCDHMm9ZelTNUbm1zIScv1VDi7T4yVSGmCb6rNk62hyeEA9zxifVuTKbCJbKW32J7e4G4bxu40bkuhQvrWCBdJsfWyhfFbLyvkuuVJ6ZmKNK2XjoEhlQS9HcHJDoHCx25A5B9LWw7ZnjRIWNKuN/qR0cjuKqxzjRV6mwApK1uvkWHAGnWP5YhNU6p5MqOYEVSLVtmkllbE8ArWhejlpx1Om5VrKQNPACBf0xAs5Zbj5IVDy7UaDCqlDnuOPwpjzWxLjP9iwqUgXcHCCAq4seADfFe5nq095CKczShSqc0tSWoreop135JWskqPAub+g4GC+Ryvp5RByufNXPVepeQI7c+sULNaxPkIfkeGDL62pElaDYlDidLaio3K02v98UQ5UX585kvq1JbbQy2CO9v9JJJP3Jw0qB9b/kcHRm3C6gpBtq79u33xW6RzrX5JppRM7NYAr6l/BD1Bcz50cRQqjI8TUsouopbgD1nlsE7kVZ18EBGpPt8vnsBi+UQoyHS0udVDY6EF2Uh2/vwuxuD6WvwcYF/wddYnr6yPZGTPbZYzVS5LCkuEhCnWAXmyQAbnQHBb17Y+n8TpxR0uJlVN5+pyQbjVdCUEc3GnzXPqSeeBxa2PnvFE2G0dQXVc0jXEbNcR9NgvRYTUVXZhsUbSOrmgqECkt6/C0VuozqgtLaXXJam9PBuSW20n6vzHax9cSKFkepSIiTPESnO6/O5thwrRz2a/dWAe6iOebW4xLpVXy7Qrw1TmI7znCY7DZUorvbnT35PqfQ/fDKvN66lJVEosZ0ltJCy0z4h5KxzYpHkSft3IN78Y4lPiuL1MPZ4ezs2fflN/qfyVs8FKX56g3d0aLfKyc41CoVIiKkOMb6bDW7JJUDbkJDfbj+zYm/PrgmbmqnNulhvW8ttKTqWoMtWPaxPJ54sBhG3lqvViQZVbdERq4KNThU+gc8FAOhPJPc+vrh6iUGg0iMHkRkuBkeaTMsrQOPU2SnsP4DHHqnUEbw7EZjUS9G+H5/otEb57Wp2BjepTEpzMteZbTAUTHd1odWPkNNcW1hZuXSk8WAOojuBhwhZXiwym8mSVBwuKQw4W0KcNvNYkkcjhKSANRsOccq+eaDT2U7UpMl6xLbYVso8nfzrsLD2Tc8dsM9Wr9Vkl5oz24sdwlttEZsh0iwJvrus8EfTa/bjvjpUlLjGJC1HEKaI/A/Pf8FTNUUkH+Mc7k/y6pR6OyY7BbUuPwIUWxWCe1xfj81WOIzVs0VuU5JYgKZprBjjaW42XVrJN9Ta0cfSCLED0PfDO5Lh7wjtjU3YNlFm2TYkcm4859OLEg2FzbCSOPOhtcVcp1N7BDbekLCk2N+91Dgc34t647sHBFJHGe1cXSH7X6fqbrF/HpWvDmgZRy5W80mlynXXHXXBIfWlxSSpEKSFkn6yLlPn4vf0F+MBe2vwtqXHeXJeFj4fwjrDJdWvzEFwi3Oi3nINjcWwtchxWpDiGltlLjdgqTcm9wdah6WFge/ew98J3ETIjm+UPuJSkpDxaKRpJuFgXIPci1uw7469BgbqKQP95e63IkW/BV1eLsq2FvYNbfmAf1Q0KISW3UaHPJI2VObquxDnPoOe6SOOOfUKgyyhlCnS00NBdSoApFuGze30XII9fS3e5niGZWmMyJDmwXCWtY21FAvdsnuB6Hm9+eMFsinBAClCOvQFFt9aWw5Ypsn1JAOpR5twB3Ax6EC64zkQ4Fshxe7JdTGK9xLjaHksEm/76SQefyt+uDGGUANuIkrIN7JUBa/+LI4sSrngkcjtjzpjvGMlCypTa7l1gH5aNF/mX84JNwbA3sDYY84ESC46/t7juhslQ2lkFNtViAdfI8wOgAXNrjAzFBGKcdVLBQ2+22wpG4jZSrUgEXQltaQELPHNzceQA3GBQ1syIjPiFKeCELs1H3G9RKyTfmyzYjm/bi3bA0NMS3ExFMoZRtE2S4HbIKfNpUebnjki1gL83wJLYXKbSlbSHpDYb3Qdpo3tZFnLEAEemm9ue18RrsqiRPv6AqQyttKze7KVE9zfnlJuPTgH1OPPeGccW4/AQRpQ03cWdcvp4Osm47kgC1ygHthQWXA+X2XmHGXTyXG+PTUtKrD1Jub8XA9cJi80pYe2iWxcM7o0m1xpQWu6EEdlW41c3wzW6ZkLrr51IWyhEtpa0r1oUtepkH/bAgFH2vew9uMFJTtLW66sBN1gvbY457gEdvpTcjk/xwAsw1LcSGXYwbW262EvF1sOtnykAgWWknhXqee+OCWmMmQX3XFOAB5aGwUuCxLYAVqUFkk82ItdJPYnDXSolYcDrji2lsp2yzcKWSSsgp44APFvr7jthWl1hrSZsR19smyktFsF3z34ursRc+gH6YDFRGL7ExDi2WxrUwkug3NrEhI+sfcEi57+3mJUiO0lqQm0lxYdfYdcTqDQb44AtzZXII4PIODdHdDYMp8JQzJY3ApaWkhgHSbAEXN9B5BH5gj1x1pLi5ClPTELDp5GgtpvyA4kX97XSbEfcYC6p1lhzxcly7pcZc1BTht7rIGhGnk2KD27kXx3SrYQzoQtm5U0lwLNyBfbA9rhI4NuL+gwUEWERJEde1MDqkLtYq1aXQLHUUDgixBB9xc+oCXtaXd9mQgrRYJc0FIR3uLG3v6+/HIwpWjcSUs76nU+Xc06g2AbFVxbi5A5vz72vjsUPOurZRYocC7kC6eOFAAi9ybHtdX374iiRmR5A8iOhwoSD5LgK43ABc31+wJueecHSNaG3EsaNLvmLhN/Q2Pbv6ckfrgbz7SGkpUh1oI1kJ2S02kWGk3PF/ztbnvY4KW1vtNrXuodK1qdStSQk99IJIN+bG3obc4ii5dtbiUz4KFtC5bQpGoN8auOTc9+4PfHb6m0tMpCESG1raSlu26AAb/exI57cW+2DXC2ym8dBW3I8zRakcqbuQLBwc83Jta38MFOPMlBDsYBSyAlDrYFv3rWBsUcH6f09cRRE7ylWLjTCeAATYkG1+ObkHn0H9+BAxgvb2kKkg7hCwAVN9rgcfw5NyfTsEJjJuqGtiOGkNthJI1gm4Ha/seL+uDQiZIDel5aFLIshWgngjiy+38h3574iYIpxWp3ShBAsLrVa9yOxA9fb+7Ai1GWkqQB5PMEOCx49jzzz9r/AGwnQ28QtxaWnXNI87K1KTyL8qte329OeMGgLTH1R4w3XFWCLWCuB57jn1Pf0xEqKXrU0lKF6XDcFLjRuLe359vtgbmsBOstNqQShegG9x6YOLbIQ4HYrSmnzo1rVbVY9raTySeee5OOlhp1YTIEhwLukavKA4Pz4JP8+9xggqJOpwFIW0NaQTcJdKeO30/w49x64PQGUqi7LT6yjXYpbvY24v6c+n3wPwLzbIbYhOKVYNIQ4QFaBaybk82+3rbAQifsqDLzSS1z5m7p45ueRcfr3wFEeX2RpBZJWtZF1uHgcX4NvX1wWlDD99KFtaASdDZSpm3JJPYdh/o5OOOxA4h4tLQgatJSUEFIIv8ASb3P8vvjiFrjvHVPCUaiUhx8pSR3BNuPb+HB9MRRcW65dK3mY5FySUoPCP3r+wPsPXBq1NNaVqcbbcbWCCl4pVfuB7XtY8YIW9voU9EcdWgBZUpICtVxfgkXPa1+O/OBDxLwS42toFRQVB8cgaebXuCftg3UWTfjqzRJy81FoVAZkSZeaaWPHyGWHFiPGYeUkquAbKdJDY58oSTxdOKM+HrKM2r5Oz9XYMuIlukmG680sHU4hph94oSrsLAK79+2N09cqKms5TjPqeLQp8xlba9ASUodG2rm/Y3bNha/fi2Mox63UsuvVVvJeY2KPPmFtqqxGdgSNxAOnUHAShWlRAWP3FdzcY0sc0ssq37oqg9XK1kyKpdOkw3IV/Erjymg4mxF1FJBC0XHsbcXtjTNPzdS40hsyFKaZUUKS79SC2RdJPqAQRjIsCi5ajFqBFcbbUBtBtzyi440BVygm3bkH7YuamyqmMv0+G7FWJMSK3Gc1XBBb8qTz7oCcJPCHahWRTG60FljNEaLUUVnLEpuSukTxMYLJJStxpQK27+oINv87G+aRUYdXpsWq09wORZrDcllX9ptaQUn+Bx8sOk9SmU6oy2ZyghpchiQ3c8AOAsufz2Tj6A/DZmJNSyKvL7ixv5fkrihPr4dfnZP8CR/mHFLO73Vc/UZlbZF8ZY6n5bby1nKox2isRpLnjGAoEJCXbqIB9gvc/ljVGKj695e8XTqfX2W/mx3vBOqAPDbp8pJ9gsD/nYsVKodSmVLDOlRQbX21myvv+uBAJLaUKSEpItpUb2/K3v98GSEJRqSto8N3QpJuq/oSDbi/F8Ji0poHSwolyy9VikC9+f4DEUSphbzSllKVoCPfuf0wFEaOpLbxW2VBwuIWUEDv35PH5frgoOgBGhlab/vC5Cr9jf/ALsDDhcQt1p59FwAtFu//hxiKI12K8oKeRGaSlYPCAeR3JI7DDcYrMlm8iI2tBNi2JRSR79gb/w/XDi2wppWtDhLTqOR2PbvzccYKTFZWVauEqN23HAfKbAWFvQHnEUQIzKY1tUB9CW7gEPbg/LjgYNiKS054lp4FTfmRpF1Dj6e392Op3o6dRJ2QSVui/P6W9v/AKcGKkIL/nkyHEC+2hdnNPsPSw+9yfscRRH0us1eBMQ9SnZUNX1OSI0lbKj/AC+Zz3CgPzxY1C6xToNmMyRmqmkabyooDD1j6ls/LWR7jRiqEvrQpO62Gi+dtOhtzSq5t9Z7i/tfBjikmEZI22lbYbeYcBBJvySDwR9sIY2O3Th5GyKQ67JZTob0BHclvSSj1++PEsqZAStCkmxcWDcnnvfi59cExEpLoZLoU9zZJPmv2uB/pwN+G8n5zzCNIBIdI1fMJ/8AAj9O+MzhlcojtxCgpaOS0ASDyT6ce2AMRuNaIRKgToUT2BP3scDjLacBZbkFza5KAnSTf349PbHpLSy+9ruSP2aiTptYfu2Fjb15/XBalKD4KQ2gLcDaELJAt6n2wpcjpSlS1lF7CybabcfzOCm2kABlLJWlZ7IsU/Y89v54C0xtSSlpvU05cWUhYtbjgG4Avf8AhgucgjW2mtQUZaCtZPkU2UEewIItb8uMAad1LcbLJ8hss24BIuDx3/L2woktx1Lb1LKFhoE7hNwPXTxyBgoBAWgpZdWlaSFhay2E8cEX9z7DnAuplRO9MSDHQmEgEgpUlTmrj1HAHv3H54WJasEpWhAKBdw2JA9QQB2H8cF+GIkubjbaS2b/AEhwJ9CQi/f8ucEoYeUt1cxQWoi7awzYti/t2I5sL4Jbm2RCA6zAkONrXNqDYC7hrcKW1EG/Ddx37848ZTxQjSzJbtcIV5Fqc/JJIA/X/wAcHM+IZK17q32SflrUgNKcR2J0g8fx5xyLAjy1aHo4Slbi9QstAAI47pIIv6YZMhSJyJLS3pIKCgeUKZRwfXte17d8eQ5KU63qSXEFHl0LJABPawsFk2FlDC9EVlRbQGSl8nSDpIIP9gkeo9hgjws8y1qmxqY4hDYGptx5L1r2H1jQjt3At273xMrQoi3GWWkpD7qnHu+hkkWHsbk888/rgxcVpzSytKHZLaNbTjKQT3Fu/Y3Frehwc8lBdWraQ+4tayS4gHUL97elwO17fxwUZC3Fus7IWfpJQtR4IFyeLWA7e3IOG7yi6WGChOtF1LIG0l3i45KyPc+/cEemBrLy3vCNMsua+UaHQ6EgC5N7825B9eMcblSfmB1qK0Y90h1VntKz5fKn37AW9fT1wX4hASpLTZcLCLFQG2mxPe3IvYWtf3wVF1tvbuUvJkhJs6GgFab/ALqTbnSO/bi3e2NAdE6W1FygKigAqqLpc1+qkp8gufXscZ6cjKkk6CtaVNgtJQEu3uQLDRfv6H7c41tl+m/hNHhU0nUqMw20o37qA5P8b4Zo5qbNS9TqdSkHuBfGSusdcGZestSDawtjLsVqno0+jttxf/zOgf5uNVVafEpFOl1ierTHhMOPvK/stoSVE/wGMMQKhKk06qZtqVxLqsqROcJ8xBcUVAfexUB+mHQCiz1DjZtzTKd1OlWoNBaHLeRvjEf6vR8v5fgjLL1RXGNUjuTq3UkOBBg0dv8A1S6OCQ66SGGgRcrcJ5tif0J6j5OylVcw1plpcNto+JddGolpHdASObqWbC3ckDvjGXxY5uzBEUKJWI/gatmEN1quRkG4iNDUIFOuL+RlGpw88uOXN7YNsxsFCcqpPrD1Qf6g5lXVfBs0qkQmBDpFKaADdPhNizbIAHfgFZ9T+WKxp7tnFqXe7huAR2GD3nF1CbtqWQ2Dzp9fywZsojLUrTyAP/oxee7os2bMVcXQ/NbVJqi4ytxRkoNy2rhtsck/nYcAepxdaoVPrsQs051solgaDIQQkoIHlsOQfS1v1GMj0aW6ypC45s5e1/8ATjQ3Tt6fUoKZc98IbYSdtQsQBb6L8d7Xtz74RM1S96mUekSvkLakLDSI7xS2gpuCQPN3C78cXuL4dI+h9mr1dUBt/adMBaFr8ziCeyVWvYItoNiOfsMR9uVBXEMqa+2q5AU6hZDDYWDYA24Gi5tYHv74Py8/LlTtC1uCMTtuNjhLDi7lP5fSB+QOImSx6mZcbnusoZaltSVL06/K8CRqUEqv9Y7ix98RHPkmoTKbNptKQ+4EBEppqO75nGiSCu/bWDxbjuDxyMSBcB+oTl0yRZTe2tZUBYB0DyrbPN+QLffj1xSed6x+FokJEtHjktrjSEpbIQ44O5Um/wBDgUCQOxF/fA3Rd3U6Q8yQ4tPC/EwHnZIcXoeGlMtGkkLA42nR5krtbkX7XxD5vUOJKZk0mnyXH4DpbdaYlIDhAX+0bSo3KObEXJHcccYryuV16UlLaX1lwlW6b8O/5Z/yu4vjuWMq5kzQt1FHpzkot+YgWvc8jvzzbvgpbOUzr+aPxigt5Y/EnCnw12txFg4ttRLZB7AkAt8dyE4NmTV1jpa5EmJTHlUCf456Mr5a3X1lLZBSe9mwok+lwLG+JHljo5JRVI7eY2XlQ4cBNV/ZEpcBUC4kn0AKVX72/I4iXUmGnK2YqnTmFtu02spU4ND25qfBsfP34J/I4m6jcysmfkLJyafTJf4GqRG2mi04oAFLjg1JQFDlYtY6Tcp559on/QBNZRMkQEMssSJTqY4ZQHFWQD5gv+drcg9+MT7J9MqUzKVFTNW34aFAbk6AA0pyWWi2glVjbb8vzPXkc+qCPOk0SuLjJeWs09txtCHTwsBQCibjn1sfQqBthLJymLoHX6v0Q+ITKs2c2hEijVyOHkqHkeZcO2eT+4W3Dz98fYN2HnbMTt1TwxT1J0llxrZCxrFlg8nsOxb54N8fKXqPDi5sgw65TpIp1Tpl3Yay4AlTVwdG53uHBqB7Am3bH1A6N9cKX1L6P5b6jT5CDUqlTx+JR2rEic0S0+LmwR50lXP9rHiOMH1FK+Oqo4A+Q6XIuR6LvYPkmY5kjyANbbKX0vJ0CH82a47KUCQlCXC00D29LLJ+5I/LDuupUajMaHTFpzS1W022g6om3lFruG/54h8/qI0iKXInhIDZ0FKlLDqiSv3FgiwFySk/w5w0uVbx76ZCZTC/E7bRSt0JeTZSrqUPyNgL+y/PwMebp+HcUxp3aYrMWjpfX5bD6rfLXU9KP5DLnqpPVM/sMqVEptNBkICxomoUgDjyK0pudJJ9SPyxDapVH8wKL8/xBKB4lKVFaY4uSEoTdWhAtfWQD3H5ALrbMtLqQVqQ6QG0aVrJbDnAIJt9eoiw9+TgDKWor5lyEpU+WgFqQtKtBN7XK7i4vwCR2P549jhvD1BhesEevU6n5/ouNUYjLU6OOiSuVFmIhenLdVcdit6tlEYAAFRsW9x3Wtf19rAAc2JGHhcyGxJSymmuMojaAmWQ21dDZAFyLWsOT3tcgemErYiRgzJS48lZKAVlaXVAn1AuO4uOD6DnBZdlyFyGZkB91zaCG3SggtrXbUHFdgOLG1xzz3GO21pHNYXO6Ibc0l9aWrNKR9X0q8i+QAV/QOARxc9+MImpMyct58IkRC2Ea2nFAtqJ5WhSNXogAnkfUDdYuAvd8MmQ0uJI2/Em6AY+4SC3yTcAAk6rC3Nib2GEbsZSkiNHjFe4O6IzaVEW1FAKzosSDzf94WvcXsc5palXX6upxLPhNCUAo0I+khABHB7EWPc2Hbvgcd5l5nVGbbJcQUi43AO97KHC9P8APyX9SfTjHkQ3H2HpAabCFBptAbtxYpBKbuIHluNYA49xjjjFmd+e9JQjkJcU8gsukDUBqJK0AabC4A7Hm+Ebui9FCWyUISpovRRIQ6EBYLFxdIUQLBZ4A4I9hzgSX9tDkSI8NTQQBrZbWkA/4whA4v7W+x9cDWzocSrcaZbWSS0mO2NLevi+39B5I5FifU2OCkIdXIW2xIRUFIaAW38tpDY12B1AFAPlUTzcXufbDuH3VMyCmSs7IXEs66C4rcsQUA8q9/YD/wCnADUWWWkx9pxLZb1bjIuWQeSNNgAeTxzbk9+wzH1Mq1utFDryFBCmxdwg/qCtIsBz7YEht6JILK5mpCCsNodBdKUf2QL25HOgnjnthWtzIITFQjLmuakbqFluOC02W1Jub2FhcBQ4Nzb6+5uMefnARSipwVqbLzd31L8Ol49vKeOBykm/oDgbbcWdKK2Wn2g+T5y9spSi45sg8eQ/Ve978c3BbsaMqzLKdxiPoaV30OyOwvf6DbSfNfgcJ9cDL95RBM1LKY8wyS5uOBtuQde2Eeo03IsT5Txr+/bBbLyUpaO86gx0OAO7Ki2F88NpuL3v25PmKyMeYaQpe8y4+tbbZKy602Bo5T9RvYhZ44PoRjstMyKhDrcyM8phe28o6dslZV/jNSQbc34v5bW5xaFEWNx1xptDUSS3uhTrby1M7J5O2AANYsbG55sOcAj1DdCNbzS0BFl6yu4AuFJsAft9z3J5wcpMpqMnfWuKiOjSLFv5pIUCblR4HccWANuOMNzdeprzLilykEMOLaUU+YgE6ijyd3COBqP8MI/xaIWR0VmNK2lokwSklaNK2QQkdr3Hqbafbk9sKnXmZIca8VHDbDZIZSb/AC73NhcWuv1J4WfXnBL60qU3IjneU4ojjte9wAeDb37/AKmwwJiXDRYRxIfbQUOMkLLirG4N797n3IvccC+LUUFL6EhJK9xteh5ZihshKDYKCdfAWPyAub+pwtAfbKfFSRIQ+NGuMgElAPBULfRfvf7W9cJpcSEl1DElSHNq0psNAmRtgjUb/wAeL3PPHHHI0BbetbsvUpQtrFyQQo6QAABrF+Lg8c98C6iHKExjwpXD8Spt0eLQXA2WwEGxCtXYLAGkBX1Hj1xzekTI269DEhsB11a0BbYSAfpI1E2sbep4FuOMIUQGG23Vw0NqICCCSSm/PdQ5HKrepub2uQMDejyXSnceKmUOIbcebWWVAi4J1G/kvftfkYKRKnkzEPrXv7ZdJ5ZspxshJstIAIWgcXuew4vxg9SZBkGHoGvWsOJa1Ep5HOok29ODY9ucIWoqg8UPSdRaBBQXAE9h78k3A4Hq4ePXHCw4pDzulopDdgUs2S0uxud1B4A8w5tex/UXRCUIhPOSSuREWouKQ0C1J39KDfUVJNggXtxz35PHI0RlB3cS8+taWgHFllagfKLIIsAe/B+wt6WIaSsNqSzFajxYwLZXJ5aN03NyBcau/KLC/f1wJmC884iHGcdkPMKQ64pxlvcbbIsnjggegNhqAsCcS6hXk06TIbSIC0JJsUBxrXfvz9QJ455PFub9ieuOI0dDz7ocWgNnxLiwUgEG93COAebe4B/PCXwkyGn+sPFOghta2idxJNj2Re33A/lfHnWtlCGd1atOsEJKCkXHzEWsb354+17+5QXWpKi6Gk7gUtRWlCjYW08+UGxJPr6Wv2wrcQ8064zstNpKyQU82+/sBbkW/wBOEy3HidxxehOkFKHdxdrX86QEkI7G3t2NsAfQA1pe8Q066guMqC+VosLLBt57cX5B+4vzFEuMlTqn4rshDehzaK2yRdABJVwfljve47g/fHINPYiNtst1V5tjZ0XLzhUkEkkFSyrWCVE8c9ubADCeHFfWHGZkNgxyoWUpndSk6DzzyuxuTbsLXHGPIC4roeeSULXthDjevdBvySn1QObfa1+MRRHNsLKhskPG4S4sLspN7XNj9BHFwbdsBUt47sM05AT8uy1G+4TdJ4IAAuQe5vc3GC3mX1TlCTL8YvcLgBJDbaLgAc+ckX5HNr+w4VJfTKWveeY1hwh0hsgO2GoEW4I+/oe9sRREOJS220llmGLKLgAbbKrW8ywRco/Id79vTHjBK2rOoDbZFynQLc3+oegPHGOOPOISh0S0sKYIcA7qdd5FgeAe/oTjsmQXZTUaOiSVrGp5tSGyHO10AhRseTyQCPbEUR7TKVgrbabWptPC2k20rvbv6jv9uB74LktOqKJKNalobNkrd/aAnlVvoJB9LH34whMxEphbKEhhX0/1iMhW07fypKSDe44F7XJB9MKnShx1t12mlIQsAR0rsq6LHhQN/wCFhYn3xFNUizFSDWst1emvutIVNiPttkOXAXt3bXYHzkOBP29sUOjJPT/qBQ4FSzVkei1Z5UYMh2ZESqQkf2N0Wc4uLDXYemNA6qlDejyVVBCUlesKciBLXbUPMCPUevA/PFPUthFMq1coiPoiT3HWhcn5Tp3E2vzayhi6HVyDtlAJfwr9Ep8lMmPl2pUxWoKcRT6y+0h4C/kWFlVgf8kg/fF9x+l2Sl0mNAptOcjIjsNtsrVIeW8WwBxuuKUskD+0TwMRlSjsu2vfbWLD3sbYvDOGXP6O1tFNecCUGnwlNNbJum7CAopIHPnQ4T7cYtqNAkivfRURWumc+jMyJsCWJcLbWhx5tNnWkAg+YenNiL+1+cXp8M+cxAzjAbcf+RmCMae9zx4hu7jK/wBfmp/zhhmCW3mdYj6nrFK2EuIDgBva4KtC7i3BJP39MQbL4k5crMyBSXtDsSQ3UKctJF0rQdSbfkR2/PGVq1b6FfQ9P0jDLmzL7OaMt1KgSOET4y2Qod0m3lV+YVY/pgWUcwQ81ZZpmY4J+TUorcpKb306xcp/Q3H6YeFcg4sVKxolhEaQrx4WZF1tltFwpJ+m5PIHN+O/HpgLS7nS14hDiLgKUCrSL+3Ydrk/c8Yn/VXLpomcp7rSWTDqlqg22RYFZ4cJNwANQuT/AJYxXwaithLyFnUi5u3cJbX+fHpe3viKIT2h5lxpS1W40lAABN/W/pYemONokMJGgMKX6gO6iOex9h25/ngBD26rWI51q0OWbLhSeTb7Gxta3OFDc/ZeBZ1tkIO3cBKrdz2FjY+/BF8C6NkFMxbDd3VIShZ+YEmylC/A9h39DfjHUStpKvkEtH0DZ+j+f9+OIdZWpS1pLS0C/wAxIQSD6gj078nBzaEOIcX80Ni6RoJJUBYk29RyMS6CLcc1qblw39p4KFrcgj8lixHFre5wW6yw2tanmllRCFEpRt34uPTn/wAMHIYZkKSFlCEWvpKikH7psCb8/wDfxjkiAw4pTzKUFxDRBSkFdlm4seBwePfj78Yl1EWgRkNIMU6H2/lq0taVOIJuTYeQfkAD63wGRIWglCktuICtvcdZQbtk3A7AgW9+cHLp7KGm/EFAd20KUlxYLYXcAkdlkDz2t7Y9OjrhFL8Z5pyHvBkqeJ1EnjWLA8cnv+WIEeSYRMD6gypS2VrAOlTJIVY3ueLH7Wt2/XCqOl5Ti3dxp2WsAPFXJTa9kAEnjzE247k4UtsKYAWyyyWdAc0OH5Sjf1Fjzybcj1x5iLLjIT+HOcNjSGmnghSke1jysdgPa2MaszIDkeptLMh+SiO00tAulwKSe/PH0Di5JsbWwewuJdz5ciaLa94620geh5HN/YkWse+EweSpRfmJXdbS3FbgU7za3I7elr/ywINKnMLLkhxSV8Bp5nbItyFpBGsn7dvtgluVKjZb0lxIMRtbawLEBDag6B6JB4P9/GCqc7MYcKNttq6BqAaspwr5vpAuByT9v1wjFORHDS5Dkh1eqzQCSb+y+E/X2F7ADth7jstFgoQtxx4i7aXJJCifYKsSeO9u388DdHwoDymyppKEuv8AOqwaI4HIF+47n15tjzb0+WXW17TTIvrbCALG31FXcADm2OOojMkojRNyU3YhKXSUtgi6lAnsR/3++FykSG2ksLZhlwIHzFKKg25zcXta/bEQSWJSUPMNy4dOLu0k7hbeDQUe9yLk2t64PqVIeRIbW43HF7L86wXNHB4vcjk/l6+uDKg7MbfvIe1NG6VAxiEq8nYWN0cnlZ+3vguWiClhS3mlpL4GtSSFXv3IRYe3Yn0GGzFNlQywWCHPHU9Dy9ZShxWlxSL8KACSR29OLnBaJE+U+lSI6JD4auSb2Isf3eCefte+CVytbaWY7UmSwizaEpWbq54FyRzyOAeDxgzwklxaHUU59AWFrS2pYLiUDggj0X39+LXOF7zkhQpM+oyHNL0hpsNtkqSCVlwkWISm/kvbm5wjltRA0ZEYFtW2AEIBUNfHI73HI/uwtfYXcuIjrUI4b1h1oukLQOOOSQV/wuO9sBiQWUMqdkOPLS4ncK22z5nLXF2zyOeDa3GLWuzbori2pim1NGM42XEhwuqa0rVbk2HbsL9rCwwnmKk09lL6GnIjZtcAkhXrovYkEkD7H8zhcqLDDrrQajSFNtLWt5UcEBvuCAE34PF7A++CU1Na3i2w7qW5Zpa0trvcE2sCeEAX72/TDqJM5ElFan5MQoafQXEBQ8wvyCEDkE+n6YcE02SWi3s7SByLNbADhuODx209xwLm4GCVVQsrER2M+h5Ys0u/mSu/pY37W7c3wUFtKeSvW6l7kuFbh1PI1Am3JI4vY/Yg4BUUp6aUVmq5wpUVtLbrLL/i3d0J3NKEahwOPrA59iOcaeHbFKdAacmTMrOYlRYyP2cRLrAIDh5J4sOw0C5F/wAsXVewGLGaBKVTHxP5mmUXIDlGjISBX30QSvVZYb+pwJHrwm3t58ZvzUDAoVPpqEErdWgFI9beY/zti5fifm/imeMo5ZHKIbD9Rdt/lqDab/8Aw1Yzf1Okzc65/p/SuiTHGFvxkvVqc0bKp9PcuVBJHaQ6Btt+oBWv0GCNdE2XupZRGEZ6mCsaGV5Tye4XIzrhIaqVRab+dKUTwY0bkI9FOK19k3x84+vnUD/0gZ/r+bEayzUJykRFLFiYzYDbXHp5Bf1798bW+MfqjTumvS6ndJcnoagy6y242WmE6RDp4ASU973UfKD7BePn5KfQ4VlbaFKQRZazqAHqAPf1v9sPC293lVSHkm+LGQxFRpZIc+tSir6h6XwQogq81vMbnnCvxqVEoshXAQDaw/PCGY4kLUgItx+d8OqURHnLgzBIV5tHa3JOJ9T+pElaChLaFLCCLXIAH39P9GK4trcSvUCkYWQeHNXCUX7YiivbJ+cpdXqTMaWWnD9baVglIWBYEpsRwAObc4vUOtIozK5jqFSpLwVydGlsIt+5ytd1eovz3OMt9MZ+jM0BD0nabccDbgDgSXBftqPYfbFx9Vc8SqHVWaVSvImN+5a235OyVAk/5yueTwMRWMUyQ/DnrnTn/wCqtoQ2G1cp2wLXNvQhwAk+/wCeKMzfkWsZtnyHmnUXBIU6oWLiz5ieBa97/nfDnlnP87Z2lvWixgXVktgh30CAnjyX4Bvb1OIvnjqZUuKbFmaUtFa9bfBUdFv7yee9gMRN6ptqPRmbkqEzmvNUNp6G+65FZhqNy+UAbuq3LYGopC+bLRe57YlOXerUfLr8H8GpMBiLHiNwvO0lxW2hStJKvXgi59ftbB+QYys6ZVepeZJikIlrLUDcUtwb67XuO6D2IVexN/vitcxZcqOTq2/SpkNxIAvoIFwD6iwF7/YWv+WEKZbCypVstZky6xWHohQh0uNvFlNy2H0AKAFvrPNrDgpPAviouoa3cndVo2XpkaHMgVOFH2y6whSSw7q7kjkglQJHPAPqMe6fV2psdLMw/hqylyOuO9cL1WLVlGyb2OpGq44vYWPGCOq0pmtZRoNVQu6oDyHW5C0AFxD4Nkg/XYLbACTxYYLVE5dXpCsuIeRDnIDLDkZqzdtzaF9hBUO3uQPtio3MzeOX4oFalOgklZvYEA+nPe/54sXPkGXmilQ67HksbL8TxKEI2z8/XbQQjhCLK7e5xVFXyrW6QJK5EZ1sNguIQ4gpKhYG1u49e/e1sG5UTgzm6UqOmnOSUISXLjQfYEW/hxjeHwKzI9b6LTYz1QlsGPmWUmzF1A7rbSgLIWLWI5uOb/fHzT8SpSwlPKV/5P8AD+OPoz/g/wCNJa6WVl0sLcRIrTbiFhBIuhntcck88duCeeOeTjeFw4vTe71F7XB0NtvRdHCcSlwyV00Vr+YB/FagdZjLQI7tarDCNaQUsAtPOCwNk6Hr82t3tb3wKqNU12IuCcsx4chSml65j7TjoOj0DBJ5AULlauxv7guO0wsIZXTJtwW1tIZjqcBcNza9ws3PYEfunn0x0wZUYLhw6YttD7bY5SWh7iwtfgAcXtxjztBwdQYdMJmF5I6uJHyXRrOIquuj7OW3yA/BN0Ojz6ehlcyfIIfBQG1IAj8+ifMpaxYk/uckegOF8cvFlIddb8QhZcbYbGp1rm2gOaiL9z3PNxawGDPDz2FltNKL6XzcpYZeSkov21exJV5j2PF+BgJYemxUtvQ5BdLmgoVBPIAJuQvudenkAgkKtbm/rbBefKNYgB2cy83aQUAuoCUBtCvQXN7rtf2T2I4AwFc1lpJW9KCNa9bgW4eRzZItaw7EWHc9+9iXmHJ76GmmZi1O3LaVMqIc19kpOlIctZPFwAbdwMEsRnEy0tiC6tOnQnTLdVuND5l1J5BJPJJOviwFjhHDMijGnnnHZC6YyEoUA7reJSoghO6Uhz69NwO/Fhfm4wczNW3Iap7iAEuWbCJO241Yi+tZRz5fKLcC44uOMekRH2y6EMoQS5cKXHJSkXO445wPOCR7nke2FexUo8lxc952zi1jRHgnUni7dgRwUgG90EWHFgMJYqJvVIjh5b81CHXG2vmtSTrc7hQ845B7cX5A8/pjqafTWnZTmhhAcWVjQ35W0Wvdw3PAJOgEkcgW746t+VIaUltb7LhKEEB0lKUHu64rSLXGoAa+9yb8YSlDriT4aTJcK1LQ605GGkLuQFgrSVkWVxf39L3w9hlTOQ5wjNuOOoWuQ4tpZ8qwVPLPZsm10IUR3I8vsQMFxzKcQmMZbSnAmxcRIUAm97ISVgXsgj5nFyLgc4UlmbqQ8hHmbG7oaas6w3b6io9gbC1+PXgm+Ax0y5zTPg1rUy6RGT/Vg7ruSFApumyLEAar8X45wtz4UMqNYkx4j7vjnCvUObkXUsLIJSTY8WHIvcke9sN8VKEtKYEyzQkIkSNRbTqF7FKeNHoLEkab3uLjC6RultaSxJUlpsuNJSSptgkjjkXATwm9u6uOBy31Biqlh2XDlzIC9q7QbabmI+v2KVblwQmx8gsAQCTieHko1tzZEuPMSUSeYbinAQyw6tR1EaQlBP0LI8oBBtYe9rq1RtW4zEholPI0MrKX9st6zysgC91WCSb82HpxgMiZmRxEdMOnU6N4aMhRceYnOkgaTcWUnQsrt5bWseBa5wCC9WI8wIqUVtLse61LpZcDLfnuHBulTg1XtYahe1rAcwPzDZO5gZzugoXKqbdTRSnXwqnPWdWw2G1surLhShRLSUEBGkBxJIVp7rsohSiXOQ24t5h555bayGHW0JLhve40EeQcFBt3BNrAYJrFGVXXY6axQHH5LDhcjLkR3HhDct3KtItcC5UAAdIHI4x5xl5TSHlRHtIC2ko2nNtIIFvlkc209wBa/wDGNLnbpXhoOiQypTz7MVS5cRoeQ2ZW5ZlBIvbWkgL4N7+o7XNsFJUshSXpwQYjQQ2tThUVNtkggqBH29ByD3uLGri1WTNYU941LAOh1TDKi7uC4CRye5JJ7mxwqT4yEhuS5TZ6gyClpoRioJHoSrTZwXF7kjj7pJNtwq9UU0IUZ9MxM2R8sbaC2sh1nQLm6nBYE+bm9/W98GiXIahMxkLKWyGwGhoJ8ihpLt/X6Tckaebn1wm/CXmVoeZpsiEy0C0hlmzSktkgt6Qbhy5sSPr9fzUNUSSmc+lqmS3Hm3W1OocC3S4QVAtkLSbAAkkA2SQbAHnBQRuxTkbLLjyAg3DL7x2w47e+3z3HuL83A9cFByXFiKWlsOKcLh1SXCXCNuyU6uSgCyQGwAB25x40yeuQpcalTFv7e4mU80Wk3QBpHIHe359sdch1AIkKVEfU7CWAtLlgTdFhqTyTyT3KF37d8HKpZcLtSeYZW8h2QppDTqg2EADWuwJ7BFlmxNubXv7+AvPWh5wKdCOChs3B7XuboItcce1+LY4/FqcduPEeTJadfDjrbKG5De/oFnDYgkWFu1+bC/mNlrzUhbSXXIi0NNvXKVxgC8hYJN1GyAedQHJBBQfQ4G6myRLXJafjiMp9bbjl91O4QAAdKA52CuPUDufcYKbYjNhKGXUKW2dF3pFys8E+utHcAg+wt3vgZpcxhh+QilTBUJYIe/qJURY8EuAHiwUfuBYHEZ6kuZyytlCfmnp9luJU34gEx6M6y44h9v8Axi2zrb0HhR4J9eDYYIbm0CObK1SxCEuzXNzZlvLdBbUyyhtxXnsPrJ5I9+RfBhU7uF0xpcoGU02A+S58u/8AbBC+4INj6nsMZva+NXJZaU9/QHNLrxLa0lUinjbI5uCV9uVWbI4uOcGxPjGyJIjzWKnS84Qi8gIb0Uylz3FWTpJQ6T8rgk/mL3sbYZ1NLbZViVl9StFpLqEbbjpV5XC24CsNNoHsVi/l81re9ueMFy3moZWpiWCFlvyuyNIS2Rf5l7i/rawN/a3Odqb8anT9c1mJO6d1ulUtLmgS2ZjL7qGCBy4yLaALqPyys+nNhi7Mt9SMkZkXGk5Uz1RZidPmaiS4xUk3sD8xW5rI4sRdN79sDs5GDvBMHsdsVILeISUKkvxWjYOKSjcUSe4T9Iv7ci+rHKk/Apz62WXSuQ4SotLUdVgbFtsIbvoB2+/I+4IwLwE1lt1CETEtBomxQSUgcXHHoSLEccHjnHkQJiXHy9DW4l0L1odbLbSSAPPpQEjlHbki5OBZFEqAVNeeRVXWlvqAWgOLDCQLBtRBA278enfggjHX3t4pVIW0+FjWh0N2c55vYX9LC9xe2ONhqzqJiHNTbi3W/lHcdBNiQViw03HmN783sLYS+EqEgpapNeQw8FkL0U9ucw6SBYOtuEaBfmyVjsefZX91FveKPQdLB1rcdbWdtDZWdpVle/NiB7c4Dp2Eo3nH0jeQS4AVJbRblKddrDt5uT/HDpJQ0lxSUNaHtoGW22CQUEkX4UbcgkA3HFrnk4IRSdptpCFyH3WyXFnQUg8k7jf1awLA2+9x9NsHdQJKppbIWgrmJUjz/LkOX1+twf3OxAv7H3GC48ZRacZqDzS46HVtBLDy3i0SfMhzixPPob+YdrYWMszm3FoajlIBDe2ChNgeSVccknuSSL9zgtS3mJQhoXIZLug6FKLxbQPKLhDdm7c+vOr2GDugvOIjvKQNb8pQCNtt1fzQ2i59FAEgW5tzYWOCHKfDdDSEMuKYXYNDVct2AI1Hm38vW+D00tLk9TC3mm1tuIcKnUB0m5Jso312F+44/PnBqGJDbW/EcEtJWu6Ntw/RbUG/QDn9SO/riKZlwssoh7y4rqGkOaGwp5ze5T9BSgfXcXJ0Aj8sVb1CYRS+pFPnC2it0nbcAJtux3Cngkm/kUyb+uLTJW2yvfaDI8P5Fu7hSEAWskXsLrv2PFyTxfFfdZGVN5co9fRGcjro1WbO2Wh5WJALLgUb9rpZNrWueL9zbH3So7UI7JdKVXs3UOiJTcT6nGZXx+4XAVf/ACA40n19gKXPomYGJ5iIcQ9CW4HNOkoWFt2udA4KuTc8cDFOfDrSvxHqpT5BTdFLjSp9z6EJ22z/ABdxovqxGD+QZUnQ2pVOfZlgOKISBfaUSRzwHL29bYepd3gEse11ndyO+1KTsMQ50d0DbbeWUqWs/Vc3NzzawBR9HOItm1JpcylZhS678pQiv3FtKDwkEWFrG/bE1RGlxpSWXXWpjjq3W3A4txstOWv9CwT2t3IAJAAOG+v0mRXqXNpaltuh9m7WhIaLLiACm6fU8dwfQn3xSCmCvn4acw+IoFTyk8bKpMrxMYE//o0i6wB+TgdH5EYuo9vfGMvhzzeunZuoE2QrQioBdFmX/tr/AGV/vupA/wCUxs0G4Bw6Z26rPrlRjMyyzWUEXp0hG8CnUFMLUAoFNjrsrbNuOx5xRCYbsh1Ed5TTTjaLPKOklQ1my+bA8W/hjWtYpkas0uVSZQuzMZWy4NIPlUCOx/PGVlUn8NXKp0/dfdYU5GfBds2lbB7XRYa7kcElZv64BSpimGpCO0xDpT82S+LNoLoYsi3HmcNjwDcH3Hawu2qzFmRp5SZGTiypq9tVTbbaDhA7FCVDuRxcW498S1mA8ULUyt9wEFxUdtCXHWgLWWotq7A9zb1/KyJqPJjo2mtxoSADraaL4db0C6hwbdwDc2PFu2AE6TRpEvY1pbQ3JQdpbLakvkE+mogC5Hbmw578YUOF1m6ZCh9YBGzYqA7m6FWJuO1sBlQ0qAVOhx3FtpLQKWkB1kG6h6XF1GxFx2PsLDirfW62yhbAsQShpaHCleklPBJF72K+59Cb4ZKEW3KW0wCt1gBxYb2lrOoixNyAOwJ9MHqltx3470tTjgcIus6m0qsm2gACwANzx24vgM2JsENLZCwSDHbkmy3gTe9z35uPLzyLE4EAhCG/Ehbi3yU6XVBelzso24sNfGnk9rk8YmiCB4iyQ4mQXd3gsveZwC5tz6WPbAGpjkdDTyWE/P1tuKbQjUPcG/HI9bfzw4SqTMRJEYQ9rabACdADj6Ai/wBNwAe/1ehPPGEcSGh0OWaXd1sOeRspbVxfhIBC/IL8WI+4GBoomiK9LXaNLkNPPOLJX4Z3ca73AJJ16/QmwFweOOXNDDpWP6mtbziTtWSb3vbsvj17g378Y6mO2lTrzq47L6A244Ut61KQBfyutkC1j2I+wucK0KjqcVNSIkm7i21KcZL6rk2KBwCDxb0tYd+cYbp0T4RDbLnjzshDQfB3h8sINjcLsLAnubfzwNqPBZU5JYSNxs3utYUUt3sfva472P54TNLYkP8AgFAsBZcA33lm5FjtjuCeD39z3w5bqIqFyIDbrDAdtq7BxFrqWABxxwNPPHb1xY3dRNrF5Ed5hp51LD6nG1tMxbtkIUbjz8nsP0wJhhyMyp56TIZZEdxpC2AElsXA4N+b3t5uPzvhe4qS+sq1NamrhtCQoL7gDkenPr74C1IehyQ0l5iK4fkhwIBSACLEHi1uO/f1HphXORyJO+Ji0lbSCw8sIUFKYbCw2R3PfghJI5sf0tjwdL7iYyCtzzWOtraVckEEACwuR3He3OFTjgqDjr7Ut1ew6S0lxtQWm6EjgEHg6VHvzxa2BCPvoLkuTpdI+WpROpw2+gq1DRcAi/J+2IgidMYuOSkIekouUrQHLjudVlHki9hzbvgBj65Jhs0xDDReB/aJuAUfQCbXsb8e59L4GI63wl1xpyRf5IVdotFYAusgJ49bG5v397dTHgy0sNtstOqWNd+dtJB02IF7rsoe179vXETZkWy3GG1IU0iONegNyHEKUo24XYWCBbjubG/c8YVNrbECTockr3wNpxlpw7jY50JbABB7nUe33GCXkwIzaoEvxDHms2lTO005b29bf+QDfhvq64zTrDbcgOhAQonxTmkLufKoA3sbXJHAFxwTgEhu6Ibn8KcXy0HlOpaWrbARpW4VEm3e+nyXJPHP29belS3lPvNVCA404tsXbjyAoOAJtdQKBoJ9PumwJw2t1h1DjSKm9T0Nt2YUqzqLoFiUC5BWjQf4AcgYWmU1DjKlz5LaY+3Zz5x0kAk6+Rc9xax9fthmu6IObZIXm3GyneckAoS4h0qSdSrpuCpQPKLWv37+mBOqmISuQ+hxpAWN1xRWQLaRwBa5BsOfS2DV5gZZaUGUNqddbbKE7ZHkPNgVm69QFu45Pra2BxFba0zFhgtx2XHBr1L27EkpNxb6O3N+wwe1ciW3QULajnQnQHGzuyFrsCkC+k6hb19DY88Xwml1mDJKHFUdI8q1tpdaKTrKrC1jftY88f6VzixG3YbL0l4xlDbQEBSkrWEnUQLdwUkAcDkYVw434vUxAhuvKdnPoihRaRqUCdNr2/sHUO97c2w4fdBrbq/Oj9JVS8iwFux9h2cVzFt25Tr+kG3roCb/AHxOVXtxhPCiMwYrMRhIS3HbS02kCwAAsMMuec4UvImValmqrO6WILKnEoAut523kabT+8tarJAHcnGhVnvFZN6wZ7QOpmY63Ei/iU9UpvLtCghWnxUhtFlC/wC62k7rjjnYNgn2xBelNJZyyqrZqrdVEuWhTtTqlUc+UZD99SnATwhCUN2bTxpQkYV5boFRgUOpdQMwhD2ZarHdbab3NTdPYcXfw7Z/tqV8x1QHmNh2SMQ34jKrByF8KNenrkrZqeaZqKFTEBzlbZI31AWubNIWO4Hm9cZaiqjgtfmQPiVe2F0m3IXWEetXVFzqt1FreeJJkGPPkr8GlZ5bho4ZFuw45PHdRxVjr61MlSdRTfhKfW/2worTyw3tBRFzawOE7LS9vUs9hxbG4d0WWFzsyTRIz8mSdtJQlu2u6vbsPzwKprOoBKLW5N8HIQ9HUF8oQQbWPf3OCpaPKldweLkfbBzJUljKsoptwochI9cOVPZWt5ISgAdybcnDU3oAVpJCR66rYkOXpi47mosx3eLfNGpP8O2DomDbqc5Tgz2qjGdpLMOTJJ4bcdCQq/uR9H+nC/qjWfASAxNfjPVLb1vJbcC246D2Go83t78j+eHLLjXjHGX0ltKhdxfkCWreo45P5cYgfVCmyPxIuwZV2GDqLy2dLusgX1eg9gOSAB74Ct8PhTS3V4zbSnlSV2tdYb1A/lc+npc974jC56J84jdDYWfqVzb9cNEguBWhSyf864OCBf3xXzQy3Wnunk+NOy+3SIZUS28X0bADim+LXI7/AJ24w55njU7M2WlRam+ioPUc7MWosRy5st+qHdFyEHsQQQODcEc5cptYqlOfbegTn2HWzdK23ClSfyPpi66BU80VlUPNNOzC0Kg3ojPrTHcbde991TabLtcedVzyMHdMlnTWe5Rs5xcv1CepmkVobMwoSy8JDAuQUpc4PIshXryL3w49W6vQKVlKFTMm1mTNo9UmOCIZERbL0VDC7qjG4Fykq4PBIt3tiwY2W1oZhS6g1Q4b7S35bsWoyG0h9xA3HCG3Wx8zygjTYk3HJHOWs5dQqrmmLT6GpSkUmjFwQI6vqFz+0cV3Wsi1z9sTZDNmVh0rro9lrL4pTUFhT0dYLKnfmIYt2CUE9xe/Nx9sQ2X1gzJKqxqrkkyXlOKcWXwFIdve6S32IN+3/fivFKUSb8XwJpN1Aab3I4wNyirNh12g1lpxAy3SoACFAbUWxC+LEEqujtyfX2GLLp1fr2WMtUmkUbMNZpyXAt91EGa8wXCsgJJDakgiwPNjxxiqumsWPJqrTL0jQVIUG0rvYr7WFgeefy73OLBq8lEypKdjM6G2kiM0Qgj5aOARf8ica6ZuaS6zzuyiyd389ZxlnVKzvmeQnUHNK67MOm3Yj5v6fxwoZ6h5/ZTpaz5mxtC3C6U/0gnWUT3Nt7EXbbd3PmKVp7WBx1RUg3BKPXvfGvKOizZnfZKlA6kZ7SHS7n7NKvEeR7/64J4DqPQGz3PvY4Ld6gZ5kpDUjPuaHmuVBDtenFI/Ib2I6XS55bOEfX5v77W4x4hesaWbDsVW4/ng5B0VeeTqpI31BzulC1Iz5mULcCLqFdmXIR9P+N9MDR1Ez8tsoR1EzclFrn/1/Nt7cDdtb0xFVOt63EtvIKm12cSkgEH+632weFnlaV9/TSLYbKOiW8nVSE57z4k63s7Zlu2NGpytTCpII7WLvHp29hjqeoefk6FI6gZqsi2kfj8wpRxYWBd444xGtwnUtdytZspRvc/cnAwSsKQlGlI8oA9f1xMreidsjuql7XVnqo0nbHVLOjTfAKE5kli4Ha1ne3pxb0wlk9TuorrYQ5n/ADg4lFxxmOUb3ABP7YWPA5HpiMkFCDpHlPoRz/446pRUlCFN3C++nt+mFyDol7R/3k8x84ZqC/JmivoWsEHTVZQNieeS4TyALj7euHEdSuqTS7Kz9ndWtGk7dfmW0WtpsHb2tc2+3OIsFJU0ocX1g7ah5Sge5Hoe3+nGlWs5fAy5Fhib0lSZIbQXkM0CWEFfG55jIOs35BA5t+mK5g1lu7dXRlz/ALaoiTnTO8gCTJznmdezcNuu1eWVpuT2JXx37D35wONnrPrISWM+ZqQsANtvNV2clSQP3UkO9u3A+2L3i5l+BJbK97p5UEu3KA2aNOSVAr/aFxuWUA/kP3h3ODG8yfAczov01mKZWCdmRS57r3Y/WTLt3sDzcXFuE4oMo/7Z+QVgid99UA/nLOcoLMnNWY3wo8pcqspxKubXILvuBxax/TCVnNOaoTKY0av11ltHkDbdSlJAHrayxo/Ti2L1Fe+CJS25SOmWYXlgBLkdUGTtelzZcwtk2uRx5cGRcx/A+662JnSqawtbnI/DZSiOLC9pFh6EW5/jhxJp4PoEMjvvKjHM451S2QnNuZlMIF+azNtoHqAHTYc9scGc8/PNlP8ATPNC0OG5CqzOIJ+93b9rd+/bti+Hsz/BBCbdRG6YuTHijhaKFPKSs3BKwZjZCxweAQbfxTjPXwUSJYaX0ZXFjN8F1NDdcU8Ld9AmAi6+dJ5/hgdqfufRPl/zKi3s053c3EPZuzS4h/XuJcrMw7muxVf5vrb1wSxXMzMx1MtV+vpC2w25oq8sDQDex+bwL8+3fF8IzB8D3mWelzrYJJITl2Vfg2TpvMIF/W59+9sHOZt+CBpMeM10tWuwQ6XBl18IDh5sSZOsgW7EEEH72xO2P/bUyf5lQjWac5OfLTm/MFjYlIqU3T7C43ebgcHAlV7MpWpC8xV0tn0FWl8H7fN/j+Q9sX6vqH8FIUdvpStaXEDh/K6yGCBpslO9dYUCTa9727c4VL6g/BC5FDq+kzBcW4u//wBazqFaLi19DthxwLHi17XvgulP3Si1n+ZZ6azPmwhbaMwZiQ27Yq/9cyylSAe5s72vf+/BTVdzM+gtKq9eKgdRR+KStKuAQojc7/c88Yv3+mPwSfiral9LpD6VrQEr/o7JShiw+t1PiE6wfZu5tzcXthXT8wfAhtJkPZDCJQ3A4ZVAmKTbnlIDx/QE/n7YQy8yw/IIGPN9tZ6bzLm1sGOzmLMCUfWAmryyPzI3efpH8McOYMyKAS5Xa4tCkBG3+KS7d+37W9ja/wBvTGhI+Z/gTKXC709dUSLBIyzLaSPJ9Wre9zbgXt3v3wWa/wDAiUJbl5BcQ4k3K28vVABwe37ck8W5vbnDtk6MS5NfEs/KzBmUpCf6Q5gCbcJTVpqQPyAd+2ATa1muayYrmaK+UOXWtJq8tSTc83Tu2/li/wCRXPgTjtPbOQ5b6QEbaU5bnocVf99KjIsPvqtx2wP+kPwKOuMsu5AqG24RuPIo1QZS2PQEb3J5A4v7k4PbW+x9E7or81mf+sISlKm1vKB5Pa597+//AHjBiS+QFFtaj/uPy/78aX8X8CTqy0KE4Y6P31U+t6jfULbZXa44uQQSORzYY6xP+A3aCH8suDWbguUysl2wvwTruTwkDvYfyPvH+Uql1P5rMq1rBOtGkrJIJ9e/I98FOrS6XnKzCiyW0toQ1pbWFR0IFgQEJVrPcm4JN+4ti+s5U74OalT1JyvmLNWXJy3BdymUCdIYPBtuMyVcm5AulaCQO4vilKzAodLnuRKHmcVyJe6ZP4VJgK++tp6+gj0stV+cWsd2iURuZzRVPzHMjJLNIrlTa2je0WfLZbQTwSE6hbgAEW7WB4w6ozvnMOqUnOuZBuXBKK7LF+3Bs9z29cMWolV1dk+9z/HDnlubR6fXoc6vUWPWKbHkIM2A5fTJYv5m79xxf1HNucOWttsjm804/wBMs+lQZVnHNhQi3l/Gpthb2Ad4H5/fBSs8Z1av/wDXvmhCknVc12Xe54J/a37EjGgvxX4D3nm3hlWOEOlRUy5RaklYuRYcAgHmxsTym6DyTiOZoyt8F1dhPu5T6j1nKVRsva106oyoynew3GHWlGxN7qbcbPPAxlbK2/hVjmEbFVRH6ndRI7CERuo+cmG2rNthuvTU6e44Ad4HOFLXVfqUzpUjqnnUKX57qr8uwI4HG7z/AOGIaN1tZFr+WxPNifW1+bfnzgKy+qymjtL9iNY/h6Y1Wb0VHayKdU7rB1dp5Qprqvmy6C2dCqw+6yCDe+26pQIPtyOcX/0z+Ma62qb1WgP6lny1imp/hvsHk2H7zR/NOMkRZS0q0uPWtftwOfsDxhyptap0F8OTso0OrACxEkzWF/o5GkNG9/Ug4qfAyTcJmyuve6+ldKrGW80xDm6iVWJPiSYqNmYw608wFmxIe/fQ9a4IJ1jjy84c5lWgQ2kyESVlbzTT6Fbzh1NkaSXNCSQg97AEixHA5xiTpL196d9NagifE6N1SGt9AE5VOznKkIkkW5LExOg+cEg67j0Ue2LrpXxldIJEqMue7muFYrFplLDyGyQBqL0d1V0EJBtb9OMc2Wmew6BbGSh3NXsWUOICIikTVSPK0C6Vhxrvdtzjz2uNIva59uWzqNl2RX8l1ukRm1hyTDklO42WymQ1ZxsC5NwHG2x39ftiP0rrJ0kzHIDdE6t5bafkn6fFtMuqJ5upp1tsuEWHYE8evbEwhaBKCqcEVDw7gWpRJdLzaOUkqbsAhRuebgXxT327hW3CcvgwbRWqbVc4MoK0uQIUNtXJ5WVOKH52Sj+ONJ1OkrrVHqNFfR5KhDfikarHztkD9b2xnr4KlsUegZ/yGmMuLMyxm51tbKhYpjSI7bsUcE2ARcDv+eNJNuKQpJ9UEEfpguOYotbZqy07NoL7TIqSYSUqZDZEuS1r3W/Xka3PYWHew9ccQyuO66tlp91aGStorSv5d7Ed+QO/A7C/qcLs70eDRc+ZhjoYQFIlPOsbze8NDre8OT9F9QCCORp4xG406SmWqS0t4rbAcWtUYJSQSLNBSHLn0vqHceQ+wcgoxDYdo+a6tQ4y9AmWqEFxI02c7gj2+YL/AKY3ZkjMzGcMp0nMzAAFQjJdWn+w52Wn9FhQ/TGHc7wfw1dMzOxGcZaiSg04Q+t9Km1gdnVkk8/lYn3ONIfDVmEOQatlJx0HwzwqUQar/Jf/AGgH2DoWf+UGD9pNuruVyMZ5610h6l5s8el1qLBqDPii68CWg6LIdB9BwEHi5ue2NEYrvrTRW52UzVktIU/R3PFJWpZRpbI0ucj0sbkdjbnDJVnelpcfRq3Vx9iUBriG7HNwbqBJtexKR6n2w6BW06EOuT3Fv7imFQ2XLpXt6VuFvVwABcIVa/NwdIw3OyN1wrl1FDri47gjqZd07jRFwPS6EhJv3455HYCxHDYZSvUkLDuptaPmtovbSoG6ym+q9rD73widODTbaYUlDKWkJDnztdtx2xCggWuTa9/tf0vhNOfRHlqbW1IIJ0GzI29H1EC4NjfggHgWtzgiFU6bvBlU8RJBIZQEuLQ2AjhPmRYOEkkkG4J89+DhWzIkqcaaRDbbSHg4Xh+1eKOLhSxdHoeQeLXPOIg1qRuogsrcZUphVyFrC+EpueB3A9jc9uB64M2X4UpK1R6elt83IDoc3SfM4kj6LnSR7+/IvhydiSWNcmUpthvxBZWdDVjdAUq/JtptYlzULgcWULIY0ydrZix2l61/MQhbW2tXzLFCgEiy7hJGog25GI5SyPRMClDwcLwxaN20MG9ifoJJIH0H07i3fHJAfce3VubRSRZxLZBdBvpt7A2PHtf1HIJ6EOKdaektOuhR0N6wDrAIKkjuVkHggm2n2vhOdCVBb0dEpreC3VtuNlDduGxpsNHqbdzYkE9jEqMZgxlq0pabAkaCyHXFhJc72T3sb3t+XoMeceu9HjxGYUdkW3CWghw82curb0c2vqT9/W+D2ojTzrKpC3Qi5bZXuFW0OLggAgWNxcX5PNsJ4Mp1lel14fMKHYza4hIeG52vfWs2P5exGMjlblQmZVVfmB6kQH2dtslLRXyA4U8gG3e49LEdhfBb8l5amnJT5fZIKEBbRIPoQBzaxCrkckj749Hp48kZhS1MF3wyEC+/r5PPJcJIPcX78W9VTMdEJ1YjNsAvguBxDrabBYtf5Z5HB/ne18FKkUJMxLaWI7KENg3dc+bZSxY2uQkHm3a/f7YdVuSxH+dNbihbqytTjQGhAvbhw2sqwFzbBD0lctt9NkHYAQTr07dyPRfnRq8pNvyHe2AOVGDN20S4wKWjdLzlgBccccX7c37XFsEkOTNQ0+Ee3EupbkIWAUNlJdRot5rWINxb15Jvxa2CVSEFoqjNlwoO4ENslsNg3HA7nseb8H374UCqIceLTcW7NllxtlIAHHlt5fQH+FyT2wml1WUEbDzriEOMoLbYDl9H7t7C2gckJHocK57GWzm3roo1j5PCEQuMlbQUunOSF6NwgyGlFKAjkOJH8ffjHWy6kNwYjAjt7QU5psE9zzZaiTb+d+PsYphl5DinENsoHBWpxBIdWboISXABawsEHX9wcBQpDkZbmuasvkNPt6G0Nlw/tEaRqWQbD98ke5vhmzxE2CZ0Tm94ob0xhlLkdbrbSnQVONokbgAJsdQtcX+/AwidflPFlKFlbLO6uOtp5CtDgcTYhI84vZI1G4Nu3Fi4NxtIb1sSE3dJKHyUbY7hKk2vcC51WwJRMh5a1TgEPrJJ0DkWPDauwFwOT3BJ9jhXKMacuiE4hchxMmG/LWp0FWstBKk8eXTe1/QWP64TQGltJ1b8d5SVlKw+nSQQggOJcFyhYJJ9Qe2F0ZLsqGlKH5ES7pXIebKGVKF7EBKwfYci1wcH1Bao0Xw6nmt5/nZQzvFyw8oUlHHce3cnnnCGRrO85NHG5xTQqnubi/GvK3v2bqBJPzAhf0KUCPUj0ubi5wrdjJStRcQ+XgENIZTJKiWwRfUCOeL8enrfD1S8lVx9mPKejsw3TcqTISApAPtwVjj93+eHqDkOAw+Jk2fJkSAb6mQGkg8ci4Kr8Dm4x5+u4qwyg0e+56DUrdFhk8utvyVfqXNk7RW85FYAIDbR1ADkt344OsE218WtbtixujWUamcxsVSppnOMsAuJMoN+cgWSsca/qUSPYE+2H6n0+FCATTILbJACRtjzfbnv/H74klGrNLy5TpNYqcixU8iK022Nx190i6WW0Dlbh/sjn9Mc7BuK3Y3iDKaljszW5J5AK+pw9tJAXvOqms+oxabDXLlu6W02HHKlE8JAA7kngAd8Ul8QDbj+UmDWxaoVee1HhxfqENhB3FkHsXVaUhSh2CtI4BKrXpNPqVQcbrWYmQ1IHMWHq1JiA+5HC3SO6hwOyfUmnOt9Q/GOpVLoSDqZokEyHR6Bx03/AI6W0/xx9BecoXEjCqfOSvCw6bR4wF1nXb7IAA/mR/DGSf8ACEZ5Q5LyN0whNFuLl2myJz60OpUHnn1bYUQBxZLa7Dn6ie5xrwxTmHqPGpouWoqm2j9gBuL/AJqt+mPn/wDH4piD8RuaaXGecXHpsenx2WiL7IMYO7er1sXTyf4Y5Lpqd9bHTO8YBf8AktbmyNp3St229Vl6atK5gQlV0I9O2DEECwVdNhcWHrgMaMoqU8tdwe/HOPOFtP7/AJieBjsXXKshS1MrbZWNe4u+u9u3thLUV8ITcH1uMcKSvXZVlff/AM+mOy07sdD2qwA27+lxgJk3qQVOJt9PuPXD9RpCWFp3RpN+AE6v4YZ0lZVpQoAi3PbDpHbQpBXrJUP3bWP53wwSlXT03qEaVMbbQWg7cO3eHlB/yUt3K1ngdh7ffEK63190zF0nx2oJeU5t7f035tx+frc/fD30/K0RVTpidew0XEoTuJ1NgXvfTYgc2HqftimM01NVSrU2YF3Q88tSAPQX4/lbBJV26aFqvf05wH6scPJx4G2ERRrYGoJ4+98XV0lmCXTn6VKgR3mX7FW5uWAtbsDZfpwrvikxwQb3Pth9otSkU55taHdBaG6PmlF/sB74ICBKu7OtdqtRzFAyxIW8IEG8lCmmVLLKEJuXGteojuBcLtyQbYzvIXqdWsAJ1KJ0gWA59sW/OlN1HL2Z811B5Ubcp0amxlIjNvB53cFwV73kJCL7iQfuBinF3KjiHRQCyDg5ongJ7nBOFUBpL7oa7FRABwFDsrN6SxViRKqetzREZOokEi54sPTEqdbX5XZBS3ugFN+eP/P9+NJ/Bj8OmQ82dNJ2Y8+UL8TZqc8RWW3JjscN7ZAK7tKSe5PP2xfcn4Rfh+eS04x0zYEdu41RarPYuUeW1xItewJLl/MRY9ji+Gqjj0VL4S5fPRCFEatwqHrYW/K3vgYjJ1kLasF2HfkAnv8Afvewxvc/CJ0GlhTSsrVGnvNlBTs1+fuW9rLePPA54HPrbHlfBj0fp4EhEbNjDy1eHbU3mOWH2NZIBBsdfAVzYc3I4Ixb77Eqvdn/AHlgdLSkqUtohaEWC1W7XNgLYGd1KNCU27mx4UQT62vjeDXwV9FCosU2FmN0EtpbS9mpbKGEWsbkAEn1sOFDjixwQv4Ouhr0jwNNj5ndD7u00TmN32OvTxyfbiwseeDd/fokPdndVhN1CnW7f2OBYcfoMe2VKQEaVBJCgeeePtjczPwVdHlMwVw3MxSG1oLgdczJpaeubIA22QSb3B84vYcHCpfwW9EZK4smOzmrbbWhDyjmN0hFu6rFgldiAQeBxbn0PvsaHuj1hEJQhzaTo1I+gJJ7D8/44B50LVr97EhWqx/hjdLPwZdGXgY6P6UJd86NxeZFqVr72UkMAD7259sDV8G3RaMyxqbze80482ApdcdZ4NrOAFoXB557jA9/iCb3N6wuefIi6yfUq+m/v9uMDTGLpPFrGxuedfscbslfBp0LbVrZpOa3I9jtLkZndQ64sXJA0I7BFr8XPobjAXvhE6IiMieijZgLYO2pTeaJyUJfIuA3uNnjkEXsbBVz2xG1sZSmleN1hBaE7hS4pTZSbL8gUBf8iMcL0eGlSZj0ZlSxcFbobNhcnuRx3P6Y3mPhA6DJiraNEqjT7SSSqRVqlqYcNrAWcbCh5iCkA9rkji75kzpD016dQJjeXEJkokvIfeRNjOykgN7jY235LboQPp4HJGnsTfANdG3YFRtG526+dTkuluEtrqcQi3AVKaNh6kc9vy98GGXA3AlVVgJcUgEqMltQPqkDzexH5Xx9UBlDKUNADlAy6oB1p0tLo0RJPPlN9u6FkdknjuB9kzGV4DT62v6JUZKdvQ2lVKhJsB+8pGkrWhQKrXuL3FzY4rdiDeiubSZea+WC6rTdZUKxCIF/L4prnjnm+DUy4CtC2qnHC3OQDIQRf89WPqQ9leIl4vQ8sUuSkxkfPeosWSFDzAL0hrtbUAeDc88i4b36DkyC04ublahgo/rDz7lNjqG2RYlQLZNxym5QDoHre+J79f7P1U93818xkS4NtTtShq5sQHm+/secFqlQk6U+Oh27EqkNXPB5tf7d8fTRvKWRGY5hzcmZTmI8MW23RSozdmwkFvgg2A8xGknlJB9MJU0HLkjRHg5DyjHjvqLLRcorKVayCdzbWhOtBFiHLjuORfE99A5I9h5r5niZSkqCkVaBZY4SJDfYe3m7YPTNp6tTKarCJHNkyW7/AN/2x9MhlTJi4c2a9QqYoPKR4oOU1gNi4F7gsmx0W41m9+Owwe1lfJXiWt/KtNqERBQ14ddNgpFltktBIW0F3B0gJUOLgWJN8Q17fuqe7nqvmI5KprJWhdTg2J038U3pJPa3m/jgAm0tKdz8RhBKLX1SW+Lj8/scfTpvJVApS2Vqy9Tm27lrSaLEaJsfmLBCCQQtQBtx5iLEpNzmcu0kym3EZWobT8gOKJj0yJqUsg2WkEbhABIJUL2K79rA/wARb0Q93cvmGmqU/wCpVWhKPawltXtcW4v3wB2o0xS0ocqENBPIvLbFzY+mr/xx9RG6Lll2mJQ9lajhQcRtNJpkSzRWokN8tC6Dq73va5wT/Q/KzaH3VZXpER9tZb3UwY6g7ZwpUi+0DoFgb3IBv3ucT3+3JI6kK+YAnQVEhNVihNv/AMKb4/S+BIkR3Fj/ANZx7HnyvpOu/qRqx9M38k5SfbR/9aGWnFtpI/rNHiONgWNkW2rrXdPBPAIJB5IxxHT3p4rddPT7LT6yAtwsUanlu/ANkhsG3Nr3UT98D39vT6qCl8180WkRlXcRJZWUXIVuos329ycARKiG7yJ8YhKrakvIsD683tj6TSemXTRl9pbuRMoJfS2/sIlUKNdS+wcsEXNuOE+Tvx3t6o5Gy68lipL6X5Mq8gOttSEKjMtaWECxsoMpstKNIsRcWHNrXP8AEG9EzqbNzXzaVIZAK0Px1gHSdDiV2NuO3b2vgz5jhIbS2otWCyV9r9geeMfSV7pV0vkNiSOmmS0PxWVuAfhUVajoI5cu0ULGlwEkc6zb2wjPR3oy8xGQnpTlJTDYX4ZxyjRShrm6g0QzYnc4JOvk27YAxAdEXU+m6+cF0lS9J1LKfp0jt+WBKdaUEpLrYWByQsXPB/hxj6OtdHemLzIYa6XZHSnShxSG8vR1ObBPLhG2Sjzi4FibW73vghPS/pRqLSOmmVEtvuAID2WYpU52sbAbl/dI4vfgAct7+OiVtMW7FfOgN2BUHAQg2JSq/wCvGPBbRXw80uw4IUP/AKT64+jaOkvSaVIIk9M8mSkbK3E68twWlXBASQna7g2tfvc+gwGT0r6UxXUNu9NMiIjuIDf9Yy7HUtQuL/N2yNZH29DYgEAB2IAclHUpcvnMptIUUrdQg6uEjurn/Rga7bm647fVymwAtfv/ACA4x9F5PR3pWmG4wejeS0rICA2miRCnsSElO3ySPQ2tcc3xDc5fCb0jzKUoo1CdylPcQ2Gl0Q7TQBty6w78gdzc2T2NlHjBZiDCdQgaU+JYVCxytRKU3sk9h+ox1IW2nQChSLd+5P64vbOXwe9T6MXncpTKZmyM1bmMoQJR5PAZkKCHD2B23O/pik6lSJtHqi6HXKdMpNTTyYVQjLjvj7htyxI/yhcffGpkrH+ErO+F7N0kCbHX79reuOnTrP7t0fVpuT9vywIMO61p1q0+ursPywWlCydLaCkfQbH073xaql4OXQG0/M0m91Jvb3war5YHLiu1kkXt+o7Y6lYS2lbm2nmygnm2HTLs+j02twqlmDKsLMNPYc/rVMmS3Y7cpsixAdYIW24O4X2vwbgnEvYJo+85Mz2t1tSFtKU0sFBadBIv+R74OoX45FqLcbK0SoomunQ01SHHUPuk9koSwQVkn0secbHyZ0p+FLqJltzNeVcovutRiI81hdUqDD8OQsXS242uQQOFk6hdB03QfTElb+FT4fnnXn/6JzQww3rDMOu1JCt3XwokuK9QLG7YHYEk8YnVbG6ELZ2R3BUb/wAHN1nqsPrpX8ldQq3Um5+Z6K4jVXFqafckwFJU02sv2XqS0Xk2UewHtjb/AFD+K34c+lS1Rs7dYssxZrPenxJgnTfXjYjbix+oGKhS7DEdFNajR6kxGQHVfibSp8hwBBALjr+pby9fFyeARyebq4aoDCAiHAgMNt8KVAgNtRlEabup0Abnk9Ce3HbHPe/O+61MbbRPFU6y5B65UGk50ybCkx4cwOxmp85Ihvv7ayAlTTl7JHcE+exFuFYSx48x/Stx5xh5S3EfNktgBBA1XRb6+QfzPJOEVQWuqxR4lEKohtICd5JU0ppABGkL44NrE9iocnjCIvZagtRnJxRS9y8dppL4U24LJUFJ50LKSdNrHkC45wE6U1OjM1mg1FC1Oh9xokk7jhdJN0m1vJ578XXbUO+DegublUfMFBqb6y2lt40if5gLtu2SCfycDav0wghyVpiJXTq7IktCQ4EIfJW7tL7gJcBsgG3fmyrci2I/ToyaXnKqZeLyEtVFG80ptYKUuH6rEccGx4xEWr6AJ1euCJsZmbFehvglqQ2tpyxt5SCD/fhk6eZjObMmUivO8PyI4TIH9l5HldH/AD0nEjsMOq1kHMFLk0ufOhVN2Wp6nOrZaTGabS42fpVtKsFjV+0JCrqukemGlqm1WIwyttsTGinbbU85vKY4twSoXA76CbWsfti1+tlKFGzjHzBHU40alFWncv5NxICVAi3coKee/GKvUmMtKguWwmOHS021uKCjcepGnRYDlI5PAPBF0O6ZBg2iRxOW+0txDe04WSkWCzp+ZwbII76u9+L441JkRHGWno0yKsICmYbwGk6BpseSggfTdJHpccjHISFie22iREQhyMAPEI07hupN0gJ0AgcEqItcEdzhRAkqghTUaI67FdDTryrt7W+QWwtzQoOOgeUAaObXJtgBS6dEPspkSZUuYYQ2tDbTbh8wQPNdpy5JNuL8cm3YYFBfmTvCxpG4la3PlBS23WFEf4xxRT5EKB5bJtzYC4vhplsMsynqjTU7CVrbabBeLLAsNSg4LHfR6kX4J72PI2MyPUyKHqg7GSXQHG1IjlLRK7m2pF9ZKyTzYA3HIwXIpwFJSuMtp5Gjbc8PGjmMXEsthFtA0WCFgccgW1AdyTgpDjTLRE5b4cAdsixLdzpsHGyQsWDYTyLE4KbkVWrtM7qRKK9qY05JcVocJJG0BqIJAsdN0Dg9iOOyZdMcSVQ1yZLig+yWXHkP+FcBVqbUSNaxfToBNlG3KLYiX/Mq8qHXXozS2gtXVPLSysbOxGdVLsg+gcQn8zqv2Pf0xEqz8WXR+jyFz8tuVLMEtqy4zkaluRwCgWCd99YsP8kJ7gG+MPhek6W1LAT7cfl274GpRXZSQb9v1/0Y3NoIxzWR1W9aaqnxs1t595FD6dwoqXV/LXMqrslxNze9wGwOfMO9r+thiL1n4v8ArFUSpqJJodMYQfKmJSWyUfmtwqJJIuT6mx4xRqdSgEtpOlXbU5YcfbB51qTrLd1Ffa1uw9PTFzaaJnJU9vI47qzn/iR63PLIXnaQQHC4nU00XBcdgrTe353P3wsoPxIdYGn24q80sSGQvztSYTGkEg6l/TwT7/YDFTNtMBKZCrJUsgKKlgAfxxNMh9K849RHQrLdN1RUEIfluEhpPIv2BJ7+mBLHE1tyLK6OV+ZTurdV865qX4ir51nx4ix5YUGQI8NtFrcNIFiSO5VfF79F8x5yzZS3YkTO+aFGLpC/DPNPNjtYaVNq0D7C2IPQfg5gwdleZeoj0dxRCVtQoJDYc7lsuODvYji1+1vXF49Pen2W8oUxVPy3Gl1RTTRZU8lzcWtw3BIGlJ72uQOCbD0x874g4eFfGS2rc0+YFvyXtcIxwUbrOpw4epUnXlnqE8kmp58NNpryBdt6nRW3DwL2JSA2m1iU2JJ9sNrDNEoiGq3XszOu1KIy6CzT3JBjrY8wQFo1FB4AVewKV3xL6bkCTNjhmpQ2ojCkJTuOEOLV6myLAgduFfe2H6D08psaOzGk1KU+iPq0Ns2Yb5FiD3JH648O5mHYb3KmtMhHJth+H6rpPxCac3jhDQeqrHL34vUZcSqUhU4JPzWTES4pbza76NP+RoJupzRcD0xNoeSqlJcdcmIhxlpPy3H/AJzgHe4S2uwPpz6YmlOy/SaU2pNOhJjhYsoh1alH15JUT3GPSJTFLUFTZSI7aydsuqte3cD8u+Hq+N6yf+Th0dhsL6u+Swsw+IkySnX5BM8fJ1HRZUxEmUoEEBxZaAWL+ZOg3Hc+vbDtFjRITYZhQ2WBax2kBN/zI5P64Z5+dIzcdL1JjeM1qSNbpLKQnVYmxGs9j2GI/JzHmWpv7MV15twLBLdPSoeQkC3n+1ze57YytwfiDGP5lXJlb/mNh8grPeaam0jFz5KaTqlTKSkJnzo7DhsdBXZavaye/P8AD74YKpnyAy2pqnx3HHSP2jlttJtcCyNRXftYW74T07J9WW0mRUZiGUMp3XFTAlxQsOVXvZIsTfUQOL4iCK2vN0iTQ+iPh6oG17M7N9QaC6NAVfkR2hbxz49knbHcrP041QYPgdFrUymQt3t4R6nb4Xv5IGepn7rBl8yk2dOq9ehTWcu0mnz6xmKqB1EPLdOAS46AAEuKJPyGU87jrpCBxY3BGL06L9LajlGhU+sZ6qiqtmcxjeyyuNS0ueZceIP7N+FOm7jpFybWSIv0r6UZdyJri0gPzKpVnQ5VazUCHZ9Rcvcl1z0SPRpPlSLd++L8SgJSlIHAGPdcLVdNWCR1HEGxNsAeZO59BtZcPEw+JrWSOuTuhkWGMrrqCK9mjM+cHlksyZig0pR7MN+Uf/I2P440N1DrDtByVWaox+2aiLSzzb5q/Kj/AOZQxmqfEboeREwlWUqQgRyLfVuGx/8Alvj1EzrWCwQN5pJ0gbjSqrU8zVOQy2FIO2X3Am7rq78epsPa+PmF8Y9fp2aviHztXqNOE2A/VVNtSLftg02lpSvsAtsgDvYD3x9COpWZofS7phVc/rkyGH6VTn3IjAcCY78mRpSyCO5cCw2RbsNXGPlNV1vTEJkTXTId1lxxxXJcWeSo/mSTjn0+Gj+IOrnO1LcoHQf3VtRVEwCADQG6ZHELRdKfIdPp3/MYb5Cwkea6j9/TCuaooO8e/pbucNbrwHkKvKtH64665yEpaLpCeNZvyPTAkSU61QztlDhJQSexwgJSSNSzb8+xOPC8l9IuNWsWBHcYiiVtxtS9Cli5NsSqjwY1Ljbzxu44CG0g31C3Jt+XriMqeLpSvi7Z0uD2++JZQFyZ8dUZji7Rs5otpHt35uAR/wB+GASp/gVjayfVfAEIdDVnFl5wqT6cA3Fz+X8MUk8VFarknn1xeTMdassOR6xW3zEbs4mOl75LRPrYfuC9ze1/S+K4Yy/QahXHKY1UHBda0oc27IK78D3wSFa3uhRCxx3Tx64c63SlUiauItWpKfpVpsFYb/qtYc4RG6XxW2WG0yHnbblwkAXt9/b9MGRFLqlUjRlJ0NuOISUtC1hfki+ETDK33AhN7nj88aK6OfDPUKgyrPec3RTqPR0N1FxbjgSHGwdXf8xgg2Q3TJ1poEbJWTKDRqX4KczVXFy3ZbbTXBQnS01yfENuttk3C/IdwFJPfFFLbUFci2LB6vdRJXULNL9QdUwY8P8AqcRbTIQpyO2pQbKyPq4JtfkAgXsBivlLSVfb29sQm6IQFDm2HKjLDUptYurQq5HbjCLb1HVqvh2okFt2QEOchwaLHjvxh2tuVCbL7DfDdT/6OdDMmxXVoalRqPGkuB94pTeQrdLm2L3ACu9+/ob8WY/T4brjzj7ch1haPISyEqCCbJF76ygE30pFrm59jUfS6t5zzXlSk5foa8tv06ktRqFvO2IQ4hATrcu4C4AjjUSnlQ8pHOLpYbyixAV4yuut15vVqpiH3JEd50kJVpfCiFpTpUf3TwOLi+KKmB8DrSix6cx6quGoZOcrE2rYi0yK8iRAQ1CQg7DRa3LIKiTtAJJRcjc5ueT3AxyU/KaMV9MGWsubqWyw8gALATpLoBJCDuH012vwLkYPeRHirkS0sOr8EGy4UFbqXfyFty6uNABJHH5YIqMSny0tzlIaecQ2hl5twPB8A3JWSAVuWJ44HY312BNLcpWjwoNorutTK4jkkpW42mK5ZRI4FiTZaOCUKSFc34wRHTLWvVN8NEZWqyylhxrgt8gBZPn7cG4t3ucKH5C/DR2UOORt1aFsSG6eXUX2lAm5Fh5Ar2JtxbDoh1Lxiupo645sQCi44bBILYBtc2ue578DDWQKYY8RESRT4zbrC0yG9CzvXfAJVbgWBFx2Av3724cXjWIsRxEdEJ7ba3Q0+uyXEFCgFLS2ojngfveexPY2CqLUW6oapJktKTJDQEOPE1NgBsJCw4eb8CzfoU3vcEkuXUWoLjjIjyBIkJQ6pCNDTTLaBwCn61oV7C4AvyO2Dm1yooDSZ0ZDM9bcJ9LsfceSvXHAbRymxb1Ek8WSbji/c4KUpao5jlxQckWWGWnSkruvuTcAX13v373sTYdYb8WDvPI3kXSWlSQgiyvO2pVijg97XNrHi2DmG1NNrX+GuuG6FhMhTchJGvy7KfQkd9JBI7nscRRJiy6XHbN1wQG2nGXo7gPLhsoLKCpJIKLElRHfn2wZEqMdKFsxo6I3jwXmY7Tl7PkDUgJRq/c1eZIQOyPufPJbTGloRGlmRHI2XkwlOCOLqOlYIJAAVexcJBH1emEjDqX478hdWgS24zI3XkNLDiVkAFwjm3AS2SSebAetiAolKPwuOtEd5ENb5S6GTubJSgWKnG2nBdsgJIBuPQ2AtgTLbjtQbeWkuPyRvRY7jayWEbgcS58si6zwFnk3INycKXpVIE3cYYm3jNojuFnW5Jc+bqJa4AJIVY+ib2NwcJBGTLhympbLcZcxwSlI2FJA5UCtwm5cLaC4bjmxGi5GJmQCKcmxpjyJzq5MtKJCHC028GkF0EfLCSQSLkDUbji/HNyksTqm65VKmxEYlMXSzEcS4vfC/MLFs9uFG5tcHgfv4VSG2psXwyH2n1CNuM8t2YQApO157nythw3HdZt2Ax7ap0yP4arbEVJbbk/PSHHlN8JK2mkHWg/MBJcRYAX4B4WyKKjMIDC1zpNMcCIxU3IKPqQCDYpGlGjQrVcDvbzckYT+EaVIcgUdlx5Ot0vWRp3ChIuQ4SlbgNr88C/7x7qI1IhiW2tyXLU/JLhQhlIbaLWjUlIJTex1H6gLgkWtyUjCno0cPvAx3pbWhlLoDq2wHQAFKuAQkXN0ggEnvcWllEmajyXENPKYfWCouttvoDjhLbntqSCOFXNhwRwSMeIenSxDH9UQt7Tu7hDmi5JcugW18pPN7myOOTgM1KE1D+uqcckOvLYksx2h9AP1tt2uLls9yAo2wWwuK9CKUTJElhbR/rZdbkNbZ5B3Abd9QJNjcWIuThnJEOK65HksOLaaIs4ykOutEaxcuaQLtoJQeQlFgNeFkeT4d1qYysSVS/kuPGXupU333AAfJ5FC2oA+XuMN3hf685Mkw3U2bLDalyyrdudI2k3CFobt3tfnyAd8OUJLTa9EzxBRugIituJZU2gIN+S0S3c6QdR/dt5BzhFE0qckqMhT6GkJaZQA0/JLOpG4RrSC2eQALKCx7+cjC6C/HfhrtFjFLkpcdxDih5W16SFEixCL+t7jSbJsCR1tiobCpFMDEe7LYDxZKQ44vy3KRyXLAgqKDe6CeCbdbhFb7kViFLmyUBw6Q2i7D/O5sp1EBAN1XsQe9x2MFzooiXoj1TQtqelGlwDxLhDjWlBCSQl0KJ76k3J47g83wexGkOKCCW3koUi7JQXW7HnSo6gB/iyb8nVfuMKQ+nd+cqSy0shLLAFnF8AFLZQqy7g3Wb27cdhhqnz1KUpcFbjrTbbiGFznnvMtDnb6Q4HOAOTtg6+L2wXDMiHdU6Lhx5pJiogbm0tYLTnlbFwQdQ1EaiEpvyPyuTjk52nNlZRLQ3LYk7l1PJUkkBJcOvsVlZV29NJthppsqu1l9qMlDBW9Y6Xr2cXyEqAJG2i7iu4PnsT2JS6v1OHHU2zTWpMdT9t5KG1F0+hIsCQNdzwLmx4wrY0b31SaPOSmO/AlIYbBshjZl6y6wRytSVo8hFxcJB76wocX5JetJG01MceWVhxLby0pKCb6T5Q3YaVDzcGxv6YPfekuzXmVKG06rdbaW2ppxq3lvpJsggqAAJsbG9yMJ6XT5LhjMJcjKLSbtIfS4kblgrQ4CLgq73sQbC1he72QujkT5DTwdpr0dxotjc37k7YP7MgHyXPNyfYegGBBmY86qlQUwy869stpd1tBkFFkpDq+F3G2kKuCbAAcDA24W1sqluIdWA7ss7v1LDnzEk3Hq5a5sbe9wSS2z4FuSprWpKJYhFtiVq3GzcKKi6BcBYSbC9r8HtYc1LpG3HQKYmdVSxNKFuOttr1oY3CHAC2k3WLC31aDr0DzgHAkLktxkRqky6wpo3lR23BrSs2cNyDcLIAUeObG/vha/HQltLkSMhKm0B07qbHWRdsFsi9zyeQSb/lchaWHlgNONs62w0p9w6FNgAXsr98kWsALC3b1wVAjXFIiXKY77seMXHJIYJDQsuyTwjWgpRZPYjtb7eiQgzJfpbiHEPMMtHZLi0qbDgJClXTcAkfVwfQAc2SlEHxL+YfBofTES69ssvI0qRe1lOL+tCR6Ao9BybDBDNSqsBC5S2WkRI7jZ1pdcL3YWWoevNiCG+PzOIgnSNDZMlzxq22ksEM2LmoOIXwLOE2Rx3A9reuCWWIzkVCFUxp1C3P6u7IZSlp3gbepu7hX7gJ9QDbkg8hyYL7ZZjRjT5LrLkZhSIXzlIsCoDWoFA81tPJ9QOMJVNt6AHGWI0eRt6fGBTRKEEA28trC1xa3J9b2COTBL3mHHH0xAmTKWhkvKYCEEEAkKLib3bB0kEm3PAHbDNmLKVBzZQI9Ez5Qo1YhhtxzaqiW5aVg3ILRI3G1+Yp7oI4Nx6OqUzwXYHg9K3IxkhptSWnQO+tKhpCEaPMDe/N7Hvg1pq7aYxOz50NuOOuBIjixBLgcVyBqvqSeRzYDDMzNKDi3ZyzTnX4NMn1BSZvTbNFQocuWdz8Oqkbx9O1rCi20HELMiP8ASU2UXRew59c9576NdS+m61TM25alRqchzbRU458TCNja5cbF2wfQPBs8/bn6GyJz01txulPNvKQ5pagKeRFCmgi42xpC3AQLmwt8wGxOHOmQZkDxgjwRIfh7a3YzUu4krtyHHFjQWkm3lIJ+vsbHGtlZJHvqqnQtK+Vb7R20vMuoKDyhfdKhf0Ix5uMsJO4q6vbR2AH3P8sbtzz8NnSjNrj8tOWk0WuubiEzMvTUslRHmK3mW21MOcXT5kDgLJN7YorO3wf9Rsu2lZOn0vOMBwjYDKTT5t+/ljOEtuW+zgvxYdxjbFWRv30KyPp3s8KqvI2d85dPq0ivZQzHIp8xDdnAnSWX2vVp1siziD37cHkWPfcnSjqrlTqfSWa3T4UMVulxf/WtPfedQqGgkBu4Bs40SBZ0cG1l2XxjAdWo9VpU9VJr1NmUupoHzIE2Oth4EHn5TgBP6Xw5UDM+YMqVeLmPLs52l1KDcsSkpCtN+FpIWNC0EcLbIII/Q4M8AmF2lCKXsnd5fSJ1lEYomz4yJCXAhbq0rSVNr8wAF7AErAubAW9++OGsswXkxtUlccguOSBdwMrCOGy3qsSoD29D39KA6cfF9QakhEPqhAco9SIAFUprLrkJVr2C2m9UiObE3UN1Hc8Xti7cu5iylmtxeYsuZjhViO60A6uJPTIDVkktiwO43xZF1IQUkX5tfHMdC9niC3Ne13eCWwpUOqLFSpsx0tPqbLTaHSpLi0HkN60EeXlXBHb174WyEzYD+lUxt1tpkhILa20gXB5SDYE3BuO5Hc2x5yM8uqsBcR1pv5TjjroMhAaAAUon6yslKdCjzbVx2B4mPLp0hbkh66FgKXrKl/MRdSm0jusjUObeW4tfsFT3RcyoLcB3ywyUPL2wqQXGA5YC9iSULJP1E2/PgYjmZoc2ms0vMTz0P+qSPpjtuJ2WlrIIIc7e4T6cmwvh7cWzUWFIXGcejXBCzNcAaBvzZBSQsXN73R2JBsMGT8qLqEeZS40xuQyWlpLDaDdBQggbgt5FG/Fxc6Ss2FhieSl+9dX98PFfbUiq5cU7cOaKrGHuhzyO/wAFpB/5QYunn1xjPoPmxdJmUOqSHLeAk/h84k8lh2zZJ+wXtK/zcbLBuMBmmiLt1AOt9JXUcgTp0dpTr1IPj0BKiCAgELPBHZBUbfbGb5iVyIT8Oo1Zat8ojQhOKSrsS3w55CR2ChYngG9gMbLkMMyGXGH2wttxJQtJ7FJFiP4Yx9X8vyaTOqVEzBCU1EgTltpZmhEmFMDZTtkqPnBLdihNvLa9792KVQt+nZ+y/LcQ7mulORQ9tsuPQjtKITptsbZLbgJsbm11dvTEplRJsqntsVeTGakPndaLLuloDkHSP8Xddz5vTt6DHhCnLgr2cwGIXXSJ6HDGQHCV6Si5NxdGkF1JvcenNkUFb1OqDy01EzFMBBCGntthkC2lCSFHbb0Aiw1Dji/IM7qicaq3VX0tM0eDImoLYU6mLKRHdbQL8p5KHL8cFd+PzwU3FtLb3kOR1IUUGM65dKLAJcKh/i1g354OvlHGEhpkmdLeqMdLkltZCG3DJs1trJAukkAknnUR3B7E4XFdHjQ2paIpWzIRuOONPIQm5JFk/MINtPI1n6jdIBsAjdGTKZZ3dTTmlt69wmS8N8gOAWLgN+e9u5tf3wsT4ZJTSJkCXJqMlwNa2GnWmywSrbQlqwJQDYD1uOO98IgnxTzsd6QI8ltxspVIGoJJQdKvqPkFiLqQRa3e4wFurVRcd2S9WH9UhTVw63Zp0FQ8yTYfYLJsLi1+OFU37q+YTRQo8q0G3ci/P+jA1+dxSUayfccXwubyjmdDumRluqItcWMJzv7du+HOj5Az5XZsek0TIeYZ8yWQI0WNAcdeePfhNr9uSTYAdyMd1czJ3kkypQk5mrkCgCV4Yy3UMqdLV9NzYkC41kX7X5xqjLnwaZDYjoer+Ycz1RVgsoadYgJSfskanCg373BOm/bEr+GH4KusNEqTmbM+5VhUF9UctxWplUbckJBPcpa3UI45sbkW9DjU8ToM4l1xdYkRZBfHzGo6QyhY5vySTbn2xwsVxQUWjLn0Fz+i6dJSNk8dh6rPNC6UdGskhgUjIdKVMWojxlQY8TISb8oLjhUL/kL84s2k0mdmSS01CeXFZbaWxKZMVBjW/wAYQvTwsmwsLcAEE2OLoh5AdpQ1U6nQ2Cs2UWwNZ/NR5P8AHCh/L1SZaU6+hSkIBJKlC1vU8nHgMR4kxV12QUrr9SL/AEXbho6Vti6QfBV5SunsGI4H5kt1xKbFLURHhklVjdSlAlZPPuP5nEpZjRYrTceDDZjttj5baGgkJ4txbBdbGaYzGmh5SmzH1BWkqGlCfZV7839r/phj/o91TrKHvF02XHbN7MNqaYR38o76yB6kn9MeWOC8QcQOz1j8rPPT5AfW63mppaQdwfJOz1XgRnno0uqwGXo7YddZVJQHGm7Gyii+oA27kc4balnijQWyYivxB09gxwgfcqt/IA/mMN0fovmt1yTKepsRmbIG27IL7SXXrE21OJBWRzxf09Bhwj9AMwSHQKrmCIG3WlNuqZS4tXPcj6efvcdz3x26P2f0cNn1Ly7yFgFllxZztGABRmXnnMzzjbAZjQmyAlZirCyVm5CQo3P8kn7Y6zl3M9TnCothhsbQaK5kZV3UdlJCiSQByRYf9+LMHTvLWUYyJ0qpU+Ftp0eLmaGvTk3JCAT6kC59ziC1Lq/01TUnKFlGtys915N7U3LEQznCR/beSQw0PutYxrlp66gBZg9Fb/Np/upFJDNrPLfyThT8m0+KwG5RMlKbhKLkIsTfn1Ub+uGXNHUbLeTasjK9KgzK1md1nUjLtFZDktSbDSp8/RGT/lvHtewOHlnInWTOrIdrstnItLcFzTKJND9WdH9l2cUhtj7hhJPP7TAhkuV0ypy6PkTpqUxlLDrxiNpd8WsnzLcXq3XHL91ukk+l8eYmw/HquXLVtc8n7Ozfif0W+OopIh3CB+Kh46b5p6j6JvWipMimKIcayZRX1ini3I8bIFnJqh6pGhoeiVDE9kzqHleEzAUqLAjxGg2xDZaCQy2OAlDSBwOwAA5xGKkeokxLseqUaXHUhWtLUSmvbTwBuAo2KyextwL4aItMzMpxp+ZQalCdAWsaYRL2vslATZQA5vckcX9cdGm4IrKy3v7w1g2a3l6cr9d7qiXFY2aRi581bfTqr0uvV+axSno8lumJO68zIDoC1cAWH0nhVwfbFoDgd8QPpBRkUzLLkgx3W3ZkhalF5JDigjygm/5G2J5wBzj6ThWFU+D04pqYafVefq6h1VJncqu64TVuwKRlplQ11GXuOAf2G/8A+ZQ/hioOoqFvTKVQo4USdbvl5Po2nj9VHFj5ve/Hepzun/UtCioYUsmyErPzFknsPqQP0xQvVPqNSqTRs79TKZVozsPLVEkLp0vcCW1SA1tRSkrt3kvpKOxNgRjS675LBOBkYFQ/+ENzlTYGVKf06gVGMZy6o2ZsVpxCloYjtqKdwA+QlxSTY2J7m4AxgOS4QmxJAHIxOuoUadGptJcksvpbnpckx5LgWfGNos2Xg4f2oKwfNzex5xXMpbqHktfUDybDGzIGCy5znElIJjZKtI5uPb1w3ux1Oq7XUpPa/wDLDtJ21oKgOQOQMIlcXsR2PbEJCVNDrK2E3Uv09MDoqd+cEKUQQ2rm3b2wKUQEJ1gnXxjtHuxVGCONR0kD1GAmT0/l1MTTMbWSlfCkW74keTYPi3HILKy249oGoAmw9eAOeOMPWW6RAzMy7DdkuoU2k9x5B7dkkk/wwsoVLh5XQZ70SY7I1lttxTZCGjax03sCbHm/a4tzg25qxuyYc5tKjRRQWvM225uypAaXpeJFm0hPoAeD3ueTiCSpTNOcdZp6ZMder5iVkBSeO3b8+xxOuoWZo0mhuU1CYzTiFNNDbJCihHqTe5Xfv/dip5chTrqlFZV9z3P54N0uRelynJTxddcUs+6jfAEDg4KwYjkYVNZSTIFEfzFm2mUltpS/ESEhYR30epH5Dn9MaZ+JjrRCpmWo/R/KT7bj221+LOI/ZtAC7bfsVn6j7Xtye2e8sV+PkWlP1GG6pdfqLBaYI7Q2VHk3/wBsXb9Efc8S34cejdR+IPq5Cy1KfcRTjuVOuzOSpqE2RuWPopRIbQT2Kr+mAeqAbmcj8zfDpX6F8NNA6/PKkLRWKw5FdjBACI0IptHeJ93HEufkC374o8fV+uPsl8TOVqVmH4Z875Qy3TmP6pQddPpcdvSppiGUvNhluwJQAwo2sT639MfHIo830kA8i/tgNN0x0XgTewOH7L8V5chDtwLcovzY9wSPa9sNEKOl+QhCj5Sbm3tidU+O3sspQBZtywFrYsaciWwJX2X+HWoZDovRfJEdjKsyQKjRWKgqRIm3U8/IBcecKglP+MJ7dkBA74eahBcFYmrl/htKYk7Zci0pgI1ISnykg2DnGoC4Llzz3sK4+G51t74f+nL09DbjrlBbYb1tqCiUarWcRew2+bX44NuebJk0htS71JMvaG2JD6ADq0N3aUkgKKG0gixSSDfnnGeeeSokL5HXJ3J5lOyJkWkYsgM+DdS2hyC+ZAGzd91JKm7pIcbS2Cb8W8xRyCLd8FRkLbjubtYfSF2Q4xslTyTc6lKUL2QAOACOTwOMHRqVIplOjolVKY6y7odSl4FTSULHG35bgq0qIJN72sMc248xKartI0R1ApddZUpN+QAlJBQhfHHtxY3FsUh1k+RNz8eqLCVJiU+cvStyLqd2UsoAAulVzrX5hdRQARqHB7gS61uSxMja3ZTC1DYb8SOewbJSPPY8IBIVwBYC2HN+UtuAptbLsXfW+7rdaSl9wX5skC19fPcgeg4wS4tTrHyGUMJbIAWwghbthqdQSVaPMBpv3SHVkEHtdmulKSrlw6e9HYReG47GQ3rU5c316g1Y97eXypAI1YWKjpqM6Ogxtp5wuK5CwGmwkd7cLWbe97A68ciKZg05MGm0cpjNgoeRGZ+WwTwkNFZt2NgoAXKiDbCqKxVJMFdIMqGmKw2Nt2FGUyCs2JcCeSUAdyfUrBBsCSiiHGGWWkSH3pDrTjaNxlDKFeEFjpQGtIPmCSBt3J73JGPOyUQo7DxnusPyzuRkORlHcIPN1H6CQL29QO19Vm+qRaVSkLqMpUNh99wNvSUNFJc8rmo7n74NuAkX9eLWBzMhDQlSajOcp6J8YEPLBbD744S2StR1jQlPBHbn1wO6VPChusqejpbEZBcQ6S65tps6ty+n5Y+sC5vqNzewJIx12O4xFb/q8KMxrQ4EszwlLjYB7tbNghs7hsQTZX1DnHsuNXX4mGspWiOLuKulklBSLXXdDh8+kJBPY/c4UR6RIjqMmGhCpdwsLUx5UrN7fJ5A8ih63AAOm4xAomSXEqjy1NQHkRUrbKUsQYzjyXHO6tx42QRbVb9/1NwQSviVFEFIks1MqU2QhtlclxQcWRtpU42AS5YcHnmw474421JiOtqUuOplCGG1vqUlKnvLbyoAPNgLkL5PrY2C5xStHjzEaipbWHytt5K9t1dyNtRtcFOoAEWJJI5F8BzgE+UJnhisPyFoZgP1ByEsIEZ/bbuTypaiCFoRobRoGhy1we+FL6INMeck0zwzS3XOHt0RnG1rWEmwCVLKLWusnnta1xhLT4T0dbDPgTDiEIW4zLS2QGi4G29xtA13K0nnspYBtYgY6WajTqauTBhOGMhewmQHdouX5TxYOIsdQNgeAOFm2DukRrjMlapAly2mtiUTKSw02U+cN/OcU4DcWbA1WPKkEWvbDGsQ6y8JLsbdhtB1CSIyRe4s4t08BASL3KTYn7kgSJbVRUw2zAgusM8XQ5ZBsEarhHBvwq1jckccnRhAmFEfYjRoC07zUcMNNsxyQoBZ1NWKVEngEW9EgkdgCommOqjhS2YERuQtCGjttul1L6F8pLqnBwfKk2UF2IHFrXJEdbgKplUjSpbhIeQw0GHXCtVwXBY2GvTcAm2q45w7wIyYmmS9oS8FIcaW2ou8IPBGjmyglQNgjsEWBTwS4xLj+JdUy66xHLYLymt0OIcURzrv2SAkHunjgWOAlKO3m0IKlJXEUXBcftlFwfud0htA0i4TwbA9+yemzawlS24waIQ3YF1tuICsItdJI7WJJvfgC974HCakzHWpMuJSGY0ZohMpZCn1Oj/bhftz7EcJPNjhRIpkFSB4mdDeREaQHUyfMvyC5cHl0bYseCdZ4C+/EdsmSZ/XVJS4f4MtUgI21htxXy3NIcUokaTdKLElQt7HgXSsIeaaW61FbYltuAhmK+opYuQbrA4ItzYrP1XJw5TqdRlutVBmStO1E2mVlKhdBXq5abJABXa4cuVIIIBvwRCFSU25Ahz5BQt0XQpiO3tIt5mri4tYG55sBzzxgqJOgwnGbuR2KeuW6hXiLKs8jhIZDlgXNQ1HvwRbzd8GuwUSnG/DKnstoOhyW18h1orNktEX5sgXsSjzqv63Cx1hSks3q5Uy6S+N1F0pJbDieAeCQrgD1vxxfDTPpyJUNEyazGfQY5ntx9A0tG/azYsTftdBBt6c3G6iVxWPBtJjPvOzI7AW64446NKl/V7EmzYIFwn6eT6nzNSUxS20tstSFSDstpQ42468dSTfSVWHndIQoc6B244HvqjeHmRKVPQwFbmlNPAIdKb8BBBAHHKjZV+/pgymykQEtpnx5MXdC3W0vRCw6otgc6V8aEqItwkAHg30WiXbZeRCp6H2mpIQhlCdp7SAUqBN9vVe5/eHc8q72VhtVIZUw54ea34pttDzniYzbrgaHJdLfAPkBR3NiAASFYcWqfFcMqJLdG1LIlPMRJMjVrIKipJBKADySTYcX9RhE823LbRHTOfkpiNkEML0hSA3ZvcKLDWLi+3e5N+LjBUsiQ48/Tm3KNIDejQ4mIhBaUkXTtLbSbWKgAB/5OHFmpTFNtwDPhlBQW2glTrhDS+QlLkcHXeyrhPB5POE6okWTChQpMdCtp3bbXuva2bWGpIFgbC1lA2tcc2OHCYyhiUsrWJD7RLykhvU4/5/oIHFh5fNcBQBFjbgKWRLpnl5bkWMVOyyWyhl03aQBqb0ggeTj0F78X8pGEiWYcpjZi1MttyIqC4tKfMV67cf7WDYJJ9b2PAwe7R2EbYcMNJNiHNxxViSLX2/ocJCki4JN0m/IsYy1Dh1CFOkT91sArkR1BtBYIKgASQTcoFtJAB1EAnvieJPlTU4qQZbLUYTKghZcL0x/abdIbb7OD2BP08ethfuepTKWVkwJCkLZWQJKVsAhB+q9h+0uLkIsQRY98HxzHmyFvc64zW4UFwPXWdSrbQ024NrH3JBHbBa5q2ygQo9PdcOhssyHnGXCeQ4kbYc1kBJHcoCOwJ4xLoIRmqMZMOA04EL0NMsp0toU653Rq2zfyJvfnm2g8EEDrizI/FYb2/IabLYmjW064D5dGockCx4HPcC1hh0jufg7YWzJLTjDO24lx3b2StRAW0nSkuEgaQU2BB7XvhjVVUOx9DMnf2Imy0tyUD8y5CgQb2ui1insU+vIwuT7SiUPGS2GqZEapBQ04t1ZSovK3yDweUkOAchsgiyrWwGVFjJkvVCNDfJQGng4t4fLXbTYFattF/NdI1C3FxpODKhUpkhlTiIZbRP/qpQ1J3YxJCdbxbPYn5YCjxzcBGAvsSWHPDIqbSHkLQkpZZZWUrWeDZaeO5J72FrixAw9iULImTUadBLSGYSHS4HCNKmlp1oAcSllxdySACrsToN7YXJdkusCWzVXW22kuX2m0KeeB0kIGu6NsmyebHXew5wQt+e4w54pLc2E+54cBSQFOFHJcKQBc2AICkdhyewwMpblyH5NbW8667bceZd2CoFN9ohBK9esXQSeSkgpI4EUshOqee8gWiS6HXHZEh9sKChY2ChwOwte1h35wzzZDNMmtPTZ8BKENuiGtx5LBlIvdwFlaLIF1WIT/a5vfC4tOlYfQgNG6HERg62lpNiQdVgoEkD6vKjv5cObLbzalwXnY0pJcGzFjthxtsFXCyApxCx5ig25t6WvYIWUWzXQMvZ4gJo9cp8LNUcPO7lFqEdlbLV7eZp5betANgQWljsOe2KPzz8HWWJr6l5ArsvL0kBf9VlrXUoIQBcWuRIZAva+t0W54tzodUeG3VHGlyjHqrSPDC4S2pLd9xLSgEDQbJuTcFWnnjjC9uGhaXkKiRIbaEiQ22RusvINrWCykWsLBRvyk29sOyR7NilMTX7r5+5y6C9U8jMGbUsmyapTktFw1OhlU+MkD/bNCQ632J8zdgLG/N8QOjV6VTZqazRKtJp8hu7XjoEksvJB7p3WlXv/kk+nIx9OHEwKVJ/EaA01F8OVqShL3h0NA/4xRQWwAVkC6XPrvwvkYgWdej3Szqm54/MWSdypObbfj6Qoxqg5uHkuPtp212I0+ZDnY/TYnGyOrvpIFQ6n+6s15V+Kzq7RpDaKvMg5jjElDrk9vYmEiwTplNWAICR9TStV7G/fF4ZQ+KPpXmohnMEqRlKbIdDqo1WQXIQc1XsiY2ToQLXG4GzwkHFTZv+DzNlPbl1XIdeiVulMNLc01UiE+pwWKWmnG7tursQLqQ0O/Pcik8x5YzJkuUxEzflir0KRIGtkzoxaS8L28rvLbnII4Ub24vi7JBPskvJHuvo3Pqcqo0tmt0aK3XIrhDTM+M8jYKCdIBV5mz2VYA381zbBrKVx4C2fw+Y9ulqMuMomQWSblLqnGzoRybA/MWeLlBFsfN/LmZa9k+callOsVOjP83NPkbKHARyVN8truLX1IIPr2xfWTPjLzPCQzD6h5ei19tuyzUqcGoU5s3JT8q2w76WILRtc3F8USUrgO5qrWzA7rRWXNFNzlNo0kgRKzGDzQCzbkdwTY39f83G1enleVmTKFOqMhZMoN7ErULHfb8jh/IkXH2Ix851dbMgZ3qNMrOVaq5+J0wIenwZURyJIDZI1LDa7oWgFWkqbWe4JA4xtToFmNp2TPowWst1FlFVjAiwuLNPW/XbP6nGI9x/eWjxtV14z315y+5FzLErpbW7HnRg043qFtTawFnngXbUm/FrA97DGgx2xAOtFDlVnIU1cDR4unlExu7IcJSg/MABPq3r/li1ALMf4YC6tJjNPRW2i4XEyjdtwOdtu5Q2iw+xItwL3Ho9RkS2Ux2V1Ft6JtkBmI9dToUS22qzamySDYKI1ouO+FyFrddjRlRh4KM+uYZQSFJkNgEKAIJXc20gBFwO5w2zHobEyQhDK2bvBRXFQiSFADUtBKPcJF7cm3fgjCPU2Ti7FhTY6WJbsgrcBaSpPlLa7cFXkb97kWJJubdzhRHiSjOMxp2Q+haCyBvAJBB03SFlQB+WNBAudN/YYQ/icxbEeOw9KbWHXCyy7IbDYXf6VBANr24txwRe4FxU5iDFWhlMZcROy421HlLLrTDDjeoFXFw6UKUbWBTc8YAU3QH0ORY7keoeN2VO6Wg83pS8/pBSW1LI4uoXRexuv7Y5SahGkOxVQZjkg6V+FQEKZdcQ4hJUoHm/a+kW0hIKObnAFrmLd8TGkrWw4ysLSpO7JVob86UbY0IQFkki9+AbcE4cFFdSjOts01a3ANohxISVXCiFBoD5iAgq5v8A4wDtxgoLNeQejGbuqNZTl3IkusRntGt2QiZeNDbJ/aOl1KgB7Dkn0B5xeSfgt+KvpktVX6L/ABKUeZOca23WavQ/AKIH7gfZLoIJ/tNdwD+WoOhnR2L0dyimiGeifUZTviqhJQ1oQt21ghANzoSOBc39eL2xYrzzbKdS1WF7D7n/AE42TVGY93ZJEwsC+bWac8/4Urp4VCt5XrNWjfT4rL1PgVdAHbUA2N/0uQWr/livn/jl+JqgyjFzbmqpUN9J0qRV8uogOD7f1hhP54+siQt0ElG1Y29CSP8ARjjsOPJYXHlNB5t0WWh0a0qH3BuMZyY5PGwK7ORsV8uad8bfXGeEFPVRLwKhYsRoKri/2bOHxn4s+uz/ANOfag4k/urgxDc+37K33xvesdCeiteW49WukWTJrj37Rx+gxVLV+atF/wCeGBXwn/Dert0Uym2PZqnoQP5WxC+O9g2yW6x0j4tuucdtK5GenLD6lLp8Ye3f5f6frhmqnx2dXGfkMZ+ihfP0RIpcJT6aQkk+l7DG4o/wsfDxFWl1vo5lVSkm4LsBLv8AJV8TPL/TzIeVFpcyxkmhUhQSBqgU1lg8fdCQcN2jG8rpbFfPGldV/jn6n6TlKl5+ktuHSh5qmIp8bn1LzqWhbj90ng4sKg/Dr8cee20/066xpytEK/M2qtyqhIsbgnbZKGu3b5nrjdpSPXn88dAHbA7a3hACYFZZyl8APTOI83P6o5vzN1BlpVrUmoSzGiE+nyWjc/5zhxo3LGUMrZOpiKLlLL8CjwUdo8GOlhH5kJAufuecPWnjg4ClahfUgge+Ky+R2jjcILunSMcKR6jHQQoXBBGBXGBkFrKbbLmgff8AjjxFu2OnuMcULg274awUUVr+eUZZntwZ2WsxSo60BZnwacZTCOeyg0S4D/mYUwc95TqcdT0KuxVFAK1tOL23kgcm7S7LH8MSHT98FORo7ykreYbcLfKSpIJH5YmyOiyflfpJ1B6h5ik1+uPTqLEqcp+qLfdbKHFBw2DabezZsAfY4urK/QXpllyCiEcrw6mQpDinagyl67iTdKtCgUgg8p449LYsojBLbe2HPOtRUoqGo3t9h7D7YXJ0VhlJFl8j/wDCpVuJUeutJpceAYqqDlaMwtBbSC3vPPuAXSbdko49MYTD2t5TunUB5b98a1/wjGY2a58TWe3W3A4inKhUtA1XSCxEaJH57jquP44yQ0A0yCvj1JHpiwBwCzyC70jfGm6rAm3fBISNKQb6r2PHbCpxCnXLk2/IWth2i5WqcyN49mOpyKTysc2wEALpgfp+6lGoWC0cn+zgqJB8PUowP7q+eO3tizqb01rEmqJpExlEd51Lm1urAAWE3SCfQH+IvhMjIdQgGQuowXGHWAVWULffv6kD2J4ODZEhF5CnusVyNrCHI7aiggkhLZPrxzf8sS/OUqOzFnPMyHXY7jW+yXXCpLVjYLSB9djx73UMV7QTsTwsIaC0XWNweUff/wAcOmb5H9VSiPuAlfnWtQ8w9LDuLc98G+iI7oVbZleM2Z4xfKtsNi5+m3bEcPJxJK0ygBtfCtauSDfi2I4R5yB2wqgXtP54Hfatbgn19sDSjbbC3O3p+eCxqUrUrETI0OLdXdxZJPJKsbS+Bn4suiPw/wAGXRM59G5tUm1RYE2uQawd9xgG4b8K5pbIB9nBe3a+MUuODQEJTY+p98CjSVx3Q4g8j09MTdQd03C/RX0ozB8N/wAQmV5Fa6dyIlViLSY02Mp15qTE3E2KHWVKui4JF7aT5rE84+BHV3Icjpr1SzZ08eQ4F5ark6lfMIKihl5SUm/rdISb/fFj/DH8QmaOifUek53y3OUJUBwIfik/LnxCfmxXfcKHY/uLCFjth1+PeflHM/xJ1jqPkGc1Ly/nuDT8yQym24hbsZKH0OD9xxLzTupB5BOIRYpnAEXCoumQkIaCyLKJt3/liZUxnXtexNu9uMRqnhaktJKQVD0+2JrSUIQmOtRsdz8xbESsHeX2x+EHp/lKp/Cp02U7TQ+ZmW4S3y684UrcAUTqAULjUTx+Xti3GukmQmn3ZQy+2ZEhzcdc3nSoqIsQCVEhP+T24HtiCfBPFk0/4UOl0aYlaXRlyKuzncBVyn+RGLvHPNv44XK1yd5s4qFNdK8sNvIDTLqYyBfb3nCou3HzNRPsALW9B7YPT02y0hSkJZkKbccDroVLcupY7evtiXgWwFa0oSVrIAAuScTKEt1CV9JMiA6hRiHLEJJfUpQHsCbkD047jvfA5PSPJEyOI8+mLlJQLDeeKrD2t2t9rYlqBuOb5TbiyR6298GJUytepKgo9uDiWQUGidHcisQ2okqlrleHWtxtalqRpJvwA3YWANgLfzwJ3pLkRlhtrwEgRox1ob8W5pRwQD3ubXNr3xPMJpCd5vZaKSSoavWwvc4NkbqIsdMsux5MdbDc1TSEELSua4RfVqBse5uTftcHm+OL6V5UdWdcR8pW2pC0KlOEm5Jve59/0xMy6gLS2r6li4GB2SDewvgZQgoW10wyuyhLQ8YjQUqCRIIA03A9O3mPfvgaOm2UW0raU0+6hXyylyQVDmx7ena/88TLj6bYRGG006t6NGZDj7oceVpsVWFr3Hc2AGAQOiihzXSjJFQS0+afLQ224pxCPEOtgq5Gop49z+d74E90dyM9tBymOLDFtlK3NwMnUTdsLBCO5HFrDgWxOG76fPpvf93AlA2Om32xModqjdVu90j6fPSJ0l+lzIqwuzkkyFtIKNIFkHsE6eOB9r4WO9Jcpvo3mjNUpTQbBMolOn7C1h3vYC1/TE5S1pKlEkqPc/8AdgKUiO3dwlZHdVufzOFsEFXT/Q/Iim0b/wCKOJjnWECauwOq99AFib/b0sMNz/QfLjyUrpderUVpSVnwroaU1c+pugL7/wCXi2hpWjUi1jyCPXHEhQJuvUCb9rWwLZUbqqpHQTLs+l+CkVSpshaAFoQ42UXunt5b28vAv64IY+H+kttr0Zgkh1ate4mExwQCBYEG3dV7d7nFvFN/TASOQoLOntYDAyKXVSL+H+jLecd/pDOcUGdpjdjsks834WgJURa4sT684RSPhzjSJSnk5kQypVlENQLAq1Ekj5nkJBI47A8euLjUHC8nSlGhIurjk+1sDWgqTpaWUf5QsT/PAa1FUtA6BiGpK284tiUwGXHGRCCkNLQQUkALBtdAPNzcX+2D0dAacpZDeYGFq3AtZcp2vyWsGxddrc3tzzz3N8XB4dgqUtbLZUtIQtWkXI9j9ucB0sLJ0OeYXTcHke+CVFULnQFwsvsOZtYdQ85uKC6UD6HsN0BHc9rDnDc/8NrLaC7ErEF+S2FbZkQbpSvgi3KlgX7gH7i3bFzPxmakjw9RitOMqKk6XDcKsoW49b2vhQ4h5tg+GQl1y5IDrlgf1scGx5IqjGOgmZZC9cyuU5rfc/rLcVKwyAD5VJ4BLg4sbAD783E50BzNEdQaTmKnpH1kKbLTYWEAAhttNrcDv7fkBdcEzVJUqoNFDij+zS4FoSPsbAke98HOpU82W0KW0VGxUPKoAe2AWZkFRcboNmqG0EJqdNWlbqXnEreedBXYXJ1JsewsAkWAA54wJPRLN6VpkQ6nSNvyEMF96ymwAAnd0FYuO5SQTpR+WL4W2LAXJ9D98ecCUJvq0hPN8HZRZ+k/D7m1dNfgRq1CSHmm95DE59nedBuo3KF7Z4QAoC4F7WPOCIXQXPMOA23IrFFfkLIOtbjwUgEAFolCLOaRyHLJWST6AY0G2lxIJWoqUSOOwH5DBUx0sLZPG0tSkrUVW08Ej0+1vtfBaoqAifD/AJoZbM5LNC8aUlkBMt1JW0fKUrc2yTwLg+hJ4x5v4f8ANYqKZDsnLKE7amm2mEONqaHfyOFKjxc24HAF++NAx5USTHEhiYy8i9txKwRf88G+GZVZakhahyFEX598JlJ8KOZZyqfw/dRZUYpiVvLCVuoLLiXS9ocbKSLEJbAX6fUk91evOE03oZnmLNZehZdy1LSktJU43MCVeQEXGtkWB7rRzfjn1xf1NoNEywvRR6LEgx5Di1PLYbCLLUb3/Ikn8j+eHvQkcq/kOMTI9NmWZKz0dzm+dybl7xSXHiVqZeZW4ysm6HgCUoJB/eGi1r8EXwA9FM/NRoaGaVGdcLh8TIkVMOF3c0gqHNyLEkjvx5b406phlaCjQLd+Ba2PLYQ4BdIJHYkA2OC1papmWXnejvUObUTEn0OE7EkJU28V1IJW75r7gXq182QOBYaR5RzgxHRDO0ZpGzllIUGdAUKgypbaxfkEkDmyTbsLkW9cabLSUqTcn2H3/PBoSLWuPzGCM2yRZzR0Zz9V6c8iSI0CY4gB5TighSj3BSWroXb/ACha4B+2EU3oR1DcKizGpSE790oYqDiQhFxc8+vBP5k+wxpRLVgpIVcexGPBLiQlNis83UOLYRpcis3npF1LdjupiwI0N9oobCnqk1tOtEEEM7be4m3Js4bcgi3IDA90M6mJUvRlxmSoawh78SiWfbX9QWAhPskcegGNYrCrpO3weDzhunynmFpiQWN+U8NSUlQShCb/AFKPe35XPtguc4I3zLLTXR7qEnajKy/PpwJ+ZHiORlBKPqTyS6HEFQIIdHrwoW5dmujfUeTCeH9HIsYvur3XHnWQ8U6LaklJ0p57AggXPbGkotPMfW46rcef8zq9N7n2Hskeg+/vhUwywhxbyEDUv6lW74XO7ZSyzBUOnOaaUpDtSoEdiMUgFLdUYbSlxAt8wrVe2gGwAPpfnnDdIylXpL0WHTMqJ1p3C83S3WrSG0ea9kOm6ySQSLcEaLcg6xTHbbUrQLBw3NvfBikIcGlQCh98HM5DurIkfIfUhmpNTf6GVIpebLKnG2y3JZRyLcgovaw1psee3c4JkdN63OhVKjP5Gqxpb7pblRZdH3WHV8dkrSdYVZNze3HfGwA0lKNPoMNcFK51RnPSQ2tuI/tRbJtYbSdfPr5yofa2C1xv5od1fOXqF8GeWZ0NczLWR8y5NlLK9xdPp7r9PBB4JhLbJR6i7LiEW5te16Qzr8GvxC5OQmpRun1WzPTEJKg/RKc6Xw3fjciuAOoPF7N7l798fZwshR13IP2NsFCK0ysqbSAV2BVbnj74u98ljSmKMr4SRYmaOnGcYqqtlCvU+oxC407TplKkRZDjbjelwBtxCTyD3AI4742J8PHxHw6KrK7uYIMyKItSbZLi2SyY8N0Bl3fC7HjUV8Afsr83Bx9CsxZVy1nCmro2aqBT6xAe+qLPjNyGjx30LBHrjO+evgP6a1VxyodOKnOybMcCtUdI8dT137gsuq1tj7NOIH27Yd8wmHeGqjGZDotQpvbnvgp8IU2pCwCCOQfUYh2W6d1Fo+SqXQpkqhSqtAgsRHJylPqaecbSEl0t2CubXtr7+uGujZG6qpzKiuZt6vtVKngkLocLLrESEpBHqpxx1/UP7W5+mEujZU/IydVI1dm0ZlEh+RGLkZlTcFRaes4SAHA0RYoDfyydFzb7YZaPTdYemRqRVWFMK3m1iG7YNghSUBQGtZBP0AX4IJUCQNgtiwCfbBmANVCsT1JqHMEumzK64p1DaJTOmM6HX3ddyCECzRtzcWACbEYBBh02CuouyKg461LXuLMc+LDXYXtutlpZXZQbspsc+YdsbFmRZjb4qFPBK0A70fsH+1jf+0AOD+hw3TMp5NzOwXKll6nzEuqCl70YayR6K9bi/Y+vOBYI3WS2BJDSF/0hahSbuKKpTJBcKAQ2UFs2QgcpNr3I73JAWQ4MgOvyHcxIY1ylxo4VLcXvLsQdSVkcJWQSpw2sRcXxrGHlLL9Os3BpMWOjQG0hDKbISOwAtx2B/MYUN0Cmx3FSRGQ5KXcKkqQkPG/fzADAseiVOY5JAuPvhskUFmTV49ZdlyS5FCg0zrBaFxYnTbv98Q1rrNluO0H6wmawt22hhtjesb6SNTZIPn8t+Bf+OEyeu+WWGnplTp02LGQUpDx0qFyAdBAN9ViDYA3H5YmZjgnDHDUKzFIct5FgG/qL464VJTcAE4rimdc8rz23VuQ5zVlL2ghsO7iRzfj6SfY+4wfI63ZFjtpcclSwvSkhoxV6uf77X5tfEb5KZCrBQrUPMmx9sd8uK2PWrLr01bTdNqC4zJ/1TsiwVe1rFQIv6E/qBgU7rfliJFffRT6o6pg2KUsgDvwSq9gDY8/Y/a5zJcpVjKIt5vfBDU2NIW42w8hxbZ0rSlXKT98QKL1qyq4yVPbxdDYdIaKSkgi406ykqNrenfjvgLvW/ILCmnnnKggPuBoOKhqPfseObflgbohpVjXUftgDDKGG9tF7cnlRUf4nESgdWMj1GL46LVHyyCrzGE8OxsR9HvhxRnrK7khEVFVbLyyBtlKgRfkE3HAwdG7qZSpDa+PWGGleZaKww48/U4SdoedIkJJT9uMEwc3UaoOR240xpa5adxpKF6tQ/wBHHODnaNEMpTq9GStYeHDiEkJV7YTxJUcPOQ1PoElHKmy4Cop9FW72OFQd3BdkoVY2Nj2wwV6fBoU+PUtDRkSrRS0lgF19PJHzONKE8klXlAv62xNtQiNdFI/Lx98e8uGqjVSZPjrfnU9qIgKsgokh0KHv2FsCbqQmygzBWVstE7roRqQSP3Ab9+R74a4QsU6XGO4JbebXfbWk82JBvzgDjvdDS0qcCb6Cbfx9sS4QyuSgC2EtSnRqbAkVKY4G48Rpb7qyPpQgEqP8AcEy5z8dyO01BW8XyQoh1KQ0Ldzcgn9ATiK52qcl3KmaGpk2mNxBRptgh4qeaIYXq1+lhY84l02U7r4J9V84z+omfavmea5uSq/UpFUfUEfvvuqc/QAED8gMRGRDQF6VXFuORh8y5SpdUXGeZjOuqcaQhs83NkDvixmel8NDjUioObNlhzU44OLoBIPH0D+d/ti9UuzF11XGW8mCsout51pDd9ZQ0CVe6Bf1+54B9MJat1SlZRmP0HLjMZmIy84CAkO6r+pJHPoR+uHrqh1Ah0FpOXspxkNXihlxy91C/c3/AFA/8cUi4l55ZW4gqWTck35OKzuipTM6lV2XKRMVMeStFwkBZGkd7D2H+nDjQeouZ58+JTXaq+4h1YjBt0oPkXwbFfA798QZqI68VDaWn8kHDtlqnqarUByQnSgSG+CnVfkemBdTmrUqFEFMnlksvs384bVa9u3N+Pzw1ZmjLS5redQstAtq0g2Ha5ue/f8Aliws9Nz4C4MmQx4tooJIDmkOW7A29Be2IXUl/iBaCG0N602CVHVp+35D3wE+6rXMDamtpC9WsAqI+1sRhC0BV1JviXZ9uzPUymyU7TYsn19yf4Yh2IojHHS4fYe3tgF1H1xzAkmw4FzggXUXNP3wJNr2x69zjx4v/fhgFEojvFhxLiCTbDvUZz1RTCdeVqEYbSPsjk2wwoWQcWR0u6SZy6m0HO9byzBQ5DyJl1zMtTddJCUx23UJKUkAguEKWsA24Zc9sF1rJbFNdNjrTIRc3CwACewxMYEZYYaVrt80BXBt7EfniK0zSuI23blBtcffn+GJXT5SkQi0ldm1kKt+RxWnbuvuT8EVZ/G/hO6X1F1dyMvMRib+rRU0f+hi8kOocQFoN0q7H3xib4A+uWQ8vfCzRqPmbMLbUuh1GqQkQ22lLc2vFLcbshA7EOptf3xYNc+L5ctoLyTlGzJUoCTUnD2Btfab7ci3Kx/3xjDyTSWutMlQGC3FoCbq5He1uf4YxtUPil6mB1tX4zTIqVLIQhqnBZXwTptdRPAJ49sIf/ql+q1dZjqk1JiKyVpW5GEDw7riAsGzlyVt3AtYe498Wdi8JFtTQqQ2lStxsGxsDY/r/wB2D0pCRYCw9hhgydmBnMdEbqTFEqlLSv8AxFQjFl0fcA9x7H1w/wBu2k2GK1F5Rt6Y7YYLLXe61/nfBaVrS+GNwq4ve3bEUQltanUvAeZAI7dwfTB+OWGAqUopO3Yq9L4iiJfitSFtqc1nbVqACyBf7gd/1wck90g8j0wWhb2gFQbUonnSeLYIeniOV6o8lQHYoaKwfytgbKJQtI1agohQ44P9+BpVYC5uft64JYeU8krVHW1z9K7X/Pi+AubDr4HG8hFwSOwP3wLqJSgqKfN3+2OK/LvjqeEi/GEE2RUGnm2osJDqFJUtbq3tKU27DsTz7+mCVEehpxoOaFDzKunUSbXwVT5qag0t1EaSztuKatIZLZVY9wD6exwNUnSW0PgtqcAsRynX/Zv6/wDhg1TSDuFF0LIsVp74VRGDSj8jj11D7nCIzFsa/FISGgQA6OB+t/vhSgsqs6jQq44UD3GIovMKK0lSu97Ee1sGfT6YAXAFaSeSbDjHEKWSUKRaxsLm9/viJkNZQbak3J9METCdo6BZwg6VgX0GxscAWJrbi1NlLjdrBJHnB49b8jvhG3KZcqbUZ2os7xaLiYq2glz6raxc347fwxFE4tjdSla0C4sRq9DbA3FaUE3At9r8Y6lJuTcn2+2EUt47rbLLqd8nUG1LKQtINjfg4HhCnNEQ67ClOMttNzQqSCpBcjOJSO/lJIsk8djzhdKWtDK3mm9akoNgB3+2E7jkIPR2Zi44ecuWQrubcHSD9iP44OdeZjo27p3CCUN3AKvsMIdAiAgQ5niYDEl6K7HLrYcLTo8yD3IP3wc4+yhjxDqtKBySoEYChToe+Zp27etrk++BuOIQoJC0BSwbXPJ/IYPqghtu60bluD2wiIdlS0i6PDMXJP8AbcvYAfYC9/ufthaka0XNxfkgjn8sJXm4bTSnn3tpAOpaw7tjj1J4wbKI/YY0lJZRZfcEd/T+7A0qsdtLdkgCx9Pyw2zIst1BUxNeSEONuJ0OhNx6g2SeLfxwEh+mKU94moS0yJCEhpTQWGdZtxpAIbF+Sb2AwyiWTW1yYxjpaTdwgKCuRp9b/px+uFDu6bbakg35uL8YQ+JKp6WnoDza20EodK0aF3NiB5r34B5GFgW+Ry2lJvblV+MLlUsjCAoEE98AU4EaUkKOo2va9vzwUGnwslT1wSSlKU2FrcA48GilJBQCFElRKz3/AFwHDoogT2TJjuMIdcaLif2qO6T6H7/lgtuPL8QXZL6FNI0qb0XSsm1laubEE8jtbHKf+LPR71WPFYc1GyIzynE6PS5KU89/S2FoCAbJWPy74ljuogvPBlsuLSuw9Qi/8hzhNTKvT6ql406a1I2FlpzbXfQsdwR3B/PC61wfbCKoN6WHXo4bEjSdC7cg27/ew9MMd7qI+TLZiMl55WhA9f5C3uT6DCSnxXlvu1WYjbdeAQ22Ty00Dwn8ybk/mB6YMaVDqCGHgveDRC0FSSPNY82PrYnCpp5l1HyjcD7EDA0KFkYe+n3wW6g6ToVb9MJpjyG20OrV5N1DZBNrkqA4/X+OFSQbWUq5557YDgDopchEqdW2Dutki31JOBNy21DkKQbXsoc4NVdKSdJNh27k44Eck3IHtYcYXI4I5gUIeZIKTe/N8NThmU1KQ0wp1hxzQQBdTF/3j/bTfv683w7JNxexHpzjxCVevfD5ULothaFtIWysKRbgg3/ngagSfKbYQNUaJEdkPwEGK5LXuOqaPCnPVZSfLc+ptc4MVJLawxNSAlyyQ6n6SfY+xwbZgpdLBfspOPY4lKQAEqPAtxjvII83P3wGtQXrc6cdNxcntgBuCpaUAqt798c3VWF21C/tzi1TdGIUlQ8vbAsJXZ0SNpS++21rUEJ3DpuT6DByXNSrJsU+4OEvZReWlak2bUAfc84RyIAU8Jkc7UlAtrHZY9Aoeo/mPTC+9xcY4NRHm4w11E2xpqH1qp81nZklFygG6XB2ulXr/eML20IQnSj6fzvgqSllDSnHG9xKBr02vyPYe+EhRV1vMOMrZbY2yXmXklbgWbWsoEWtzfv6WwO8oslLqiqHGmCWxX5E50upklmOyB8xfO4oKAb9jpBN79sGRKZKlzGnYqIzKGgVKQ1IsuDwVD6Adfk/6RQQO+AqnuuGDGnxnA21J22JyU3SG0W7NgHlK7EuKJB9b+jnIrLsPcaRUJ8NMgFEZEpxD7kc2syrWhJCyl7dctoI9LkA4ytb3cy1FyQzA+pL9MjLYnyI8gN6GVFppCFoKSBybLDgbs4exVz6YXNzFS2VVOTPXHdjOWCWHkuMpAs2QlwJC3DYk8WP27YIgLrbZW9WKq7Uqk+r+vSNaC4lekjWW2gnbWkWBTosPU+uDarEbedVFe2w3FjgR3Ug2kOjSQgaNVnLFN7HkHz3xAkLtU0VN6oQ56YlNjy22IallM9xhS3HAsWCSrRrKO3BsOeSeMOcuUifChImSmJD7qbpDllOsaFkEWtthBJ40n92xA5winMLRUkNwwhp5gf1iQqC4+stgeb9/Q3yeLhFioXNsK3HoLsl2ZQ6hG/C0FcJaXZbbqvOCHElsWRxyOCVg/vc8NnU1XXaexUGlTqimJLiLJWiA9bU6ixSpakix8ukHzcciw9cdkuwShSaJlRDLe2FIccb8zdhYpCdxRA7cn724scAgrVGSZMOEuoSkDySFSCltpjkbhavZvvwD6emHsMz0RURZbaHfEIcQtiLJQGiQQdR2zr9SLKsR7G+A7uqd5Fx/GsU5h+NTpaH3iC9/WkssqPdSwdQWRbTwRe9xawvjifnPNJ8G+CttanSpu4SgjlQN/mI4v2t/PDcXERnm6etQmNxgRZqKGGmjrNkJU4rz+vmHqffD03LYkPlclLj8RZForzi7Hg2CtHcDufS47HCqxKmXIENppmDGYhSo7O4pTEbaju3BN1cEkkextyO2Axa7GeZlJjzWwlYQ674YAJ0dtCdHBHPcEc37js31VqC0EofTDffC7sy2mXC4F2sNtRSSBY/Tfgj9cEOvvNtSksr8EFgbm0SqzaOTxq4J5N+4BJxEt08/wBKaxAfeapsqTGiODQ6pDrguvUQk7gURe3Ngb+lvXCp+v5ikWlPViQuRFShlL7KlLOoi2myBda1jtf+GIZNlLpj27Mkbiw84z2c2kkpJBu2D3HG4f7Q++HF7wa3n39BabQlbSoyQF6SQLC7YKHF6yD5rm/HpxFNW6o0SazMlCO83PdXITurXJb1JbI7XF7k257DtfCxFerUveajPPqS1+2ejultIISCbpsATa3A54OEJchyhLb1VOMh1Y0bja22krQm41m17W7k+w4GCUodbhrRFgIQ5HIeiIbd3hINhqcKbgoAIHGg2t5wcPok7yVomT4giNwZLkUKSHXXVt6lKHKnQEtqsbnnjt68YL/Fqu3NMSFUnG5rqLFa3QwnWs3CFAEci32ABHtfBMyshdMbqUVMyKzG0SbiV/Vw4tO0bApBOoXAAvaxwTOkxQE1d9ceGVvaA3HZ3UFZQQl1xXNuCq5FvsMC/RMnGRO3QiOuryyoNodC1PElOvhtKlAEawdV+fv6XxXHxJ1qpZe6DdQ9ktRZDFBkxAlkABRcBaBIWeRZR7AEG3POLDlpaZekIpTQlDe3pEpltDalgaRdThFwDxds3+2ID1woVYz50bzdkOgQpL0qdSnI6WFsXddcYG84W13B7s2Nxfi9ucFt8wSm+VfNvLklqjPMIiuNJ2AdDrqiNRAAFvv/AOOLXSqFUcvl1m8u6yHXC4CL8jQO5N7jnjGel1NapIl6NO6ArSoWHI9BiwcoZmqsGmNSPHNhTqFtss69ISgE8mwv+d+TjeqM2ZRbNHR/M6X1VF2jvi616zfhICja/wDD74Ym+l+ZdJeco8hoBe3dxo+Zdxx/P8sXE51EzM8pFNdr0gIJulDUdrS2D3IHJAH3N7Yf4+e/w6mpXLQ/NVvESNalkOC3AJJ/Pgf/AECyKosdF85lLmulSAWuVo49r+n2xKqb8Ptcjx0VB55ClKW2QEOpta47H1vzzbFqS+pEllxhlVgw62i7iiEuqHv2t+ZVxxxiAZi6jz5Tq2o8iO1G3i4AptKne5VYKAF+efQXPa1sLkSJv6qMvspkojJaS7DcLMkMOXCQTYgi/rYXHvfFbQ5mmzT/AJFjvzbj2FsSbM041SVMkyWluuz9Egq0J5JQCCR79sRKiMOJmvsvRm1OE6VpWf2fvb+fOBysnUa6mOLXXRfR5WkNgpVe4A74h9rovxxiY9TEMorTIZaDadkAhJuOD/LEPQ2pw6U4CiBjoIx0psdKuLYCRbEUQ06Sf+/AiL/vYAgX9L4dqTRp1amNwIEbdfcIsL20j3J9Bh9SlKTU6mzKjLZgwYzkh99YbbbbF1LJ9BjX/SDPmculPTeT0zyxWosCDWnVyK2lEFpZqThb2w2864LraS3cIava5JHJxWWSMqUzK0F1wNF6oyAG3JLhtYD60N27A8XvycPzUtaH77rbba0hG255tR5udNue/wBsWtiFtUuZVDDjqpcqVTXEXEZ5cbkd9BtftxwBh8ivBxDbKk6eD/8AThL1BQqHm16QtvSZrLb4AQG0hQ8qvKOAOAf1OFNFR4iUh1SSFtobJSjjjuOf5/nfFBblJCdrla3SbN+Yadl6bS6RWKnFYZmeIcaYeLYCyiwUrkceUD8wOMTVXUbOrkqPIk5lmOvRta2nH1akglFr6dNiSOLqBtyRbFPZKV4CoT4pe/bAOgKbC7kK9L9vq7++Jp8pPnZedU6u/CiLJRxxx3I5v6Y1RDuqt51U9Y6oZ9jMuRv6TzVGM0UbhZZU+ef3vJf2H6+3IE/1RzwkLQvMMtIJIG2G2ie/f5d0H7HtxiCKUnS0JLy1vAEIC3SUoAIsi459+Qeyjfm2DES7qQ+zIWv98o3SOe1yPX2xakWhchfGX126fLZjJzWipwIwDf4ZVoiXUJHfhYCXAebX1kfbGouj3xeZy6xQaq4KdS6LKpwDmhgh5oo7D9oQs3JSLgce3BOPmst99ThdTHLbaSXRvOWSbd73sAASbYmGR+qWa+nVUfqOX5EZpEhOy8y+2HUu3HBsedd+b+n5YzyRAjubqwSr6if+kzODzb0WRNjIkHTtIj2K1NE/teQSj1HIN+LDm2Ao6vZlYS1Ikyqe20pwNHTH1LJXq2ypAN+Sgg2sBbm3Nsh5K+LHJNWKIWdssT6VILQdkT4chcsLfAHOklLiAF6SDdYB5seRi08m9ScjZviNUrLfUWnF4Nl5LDcotvXAU4EJK20nWdRFhxdR9bAc97JmbrS17HK6pXWPM+le8Y0ZhoNtOOsshzStafKs+Y9yD5UgkfocFwesuZn3IjcctyA8hshUhgNodJRym45vdaTe1rJJ5GILIp8aM23MeWptbzYWxHffJbTwSoOtlJWhHlFmxY3trPGFb0BMWMirRJNIAjbYWA26yL+ZJ27jgJFgADybHtxhc1xumPopy31mr4W8/wCFjqQAhTLKGCd5GojUlRKdKOLkr5tYgWN8dk9Vc5KTvMijtt+Yrd86klu1w4lsgGxBBNyNI73JGK4cXApnhGKpWhTRMWNsSm3HQ4Oymy75UIXcXIuDa1iBxjklDMJWhmYHVrguNuFt5wJKwQUhO3qKC4T6agUWPBPC5nIZFOo3XKvTm57yKc5AaiqUhvxtLcS8+u2oFDdwbAAHkc6084NpnVjMKtpa6rHkM/W4FtoGmzhDg19jZHmv6EEefgYr+qzJLTscLpywXbyUc6hwmykFPZFvmcWHPn7C4LzDHiVGU3UqZFL8hw3adiNOR1FgABLNgbI2yQefRNyDqvg5lLKwV9c8wqltxqdEhPoWeHXkHQQQbWUCNRuO3F7/AHwnPXjNwYkyJNLpscRWVvAp1r3gPdPBb/O57WtcYrmbEqD6pf4VBanKQ8gSVMOam7IKtxbjrYICE2b+WLDk/WQMGMuTJdRCKlIPi3f/AFiY8dxwpQhtxRbsrkuFwEWTxqvY8DBzOQ7qsIdba9FWlbVFpwemFs+JaRcO3UQEKuoEKtfsCPW/cYEjrDn5uY1ElQ4zCt8srbREWfOCPIs6isEg8lIIBHfvirorrxkPIjxDFhtraMenFndU0+tYu0FXuXEocKgkkDykcHDhTQ7TG0rn7iltuBpanBvao4NiCTfbHIIAWB21gkcrmKlwrHR12rsqaWpVChsQgUKbcbmbT7tzosA6NBs4UXsVEA+/Z1b63zUutol5cQ2pwqGyHwXEEBZ0JvYLNkEnkWvY2xS1YenQ7opMeZPknc1rU23qZYAGknRcrJF+eO/Avzg6m1h91pa6kuTIEhxCWozxbSErupJCtpI1oVb6boI4vc84tagDZWq/8QslqXGZfysGUrc0XflKQF8HzApSbX7ALA9Teww5HroHJ0ZuPRWvDyk6wXJXnSQrSTYC1j5bXI5PNrg4px1+pRoTkYnQiWELLAfHzmlnykEXLax3FwD8zgjSUFG/EYgpVOjvkFThc0qjWQlxtxKnNxQcDZ79k8EWJtcHEzBNoruT11UHmGncvpKnbN7YkKQ4XC4E6AlbYOocm3tY354AevGXjNYclUKQpxCD4Z1l0KSdfBGtYSEci35i2Kookpwyml/i79RWHgUuKaWXSi9/KeRwO6PVGm/vhFHmpjxduJIcZSyUQQlsuIVKctYNttD6A2FX5Kf39HcDCZnKFoIV61LrY1EU63Gy5LWptkL3FPNlBcvYtgIJX341EAcE9hhJ/wCnlouKWvLa0x0J1bhlt2Ub82vb6fX3vxijapT2E0ht2E2Y7bTW06w2UAqfKruWHC0NgeTlBPGsnm4FGMaYVSYMmRVmo6NspZbDbpQUG4HBNib83UDYm/JwC56JaFcS+ttJZcNZaoUVmQ+4IzrjkjzaE3LabWCiSLngW5TbXwcBd6602XFS/Iy+8pJeQ41ETJAfGmygSeEG/sDcWIN+2KoZ/EHWIi6pTVhnQsBtbzbhbWFuAjTt+QBxshFyFkAlB9kLbNQY8QgT34DbZWPDKkFLqrj6NQ1c3t5iu4HqFm4ZpSuarpd+IRhyow48LLD5bW4WZgekaXWiUXQUgJKFgk2ve3F+3OHeR1py7LpjiokeoNTlNeSNLZ8Optwq08rJsRf1TcH0OKDdjvohMyo7riZc8NTGGhAQHk8WSNzi4JU5x78dxcBUt5bMeTR4TiGgFx5C2nFKDriyQ88lSEnhIsCWzZOoI5uRguRIV6U7qxRJSFTsx1mXHgvtICGUQVpLS7EEqcbJJ1EGwsLW7nDtP6idNxFjt1WpPOtRHAkpfYctcJvrcCxZYHf15F+44oRlxynPKZLr7bi0OPKW822y3cKuqzpIGhu97BZIsOE+YY7UpkdqF42XU3HJkhoOuMxUEOJQ2Lm+2VazZSSePQ8DvhMzuSmUc1oN/q3lEjRTJEme6dBDLLNlEHkka7dgCbYWSuqGRIPnm5lhsEgfJWle7zex02v6H+HfGZJ8pU2pyYyJOhoPAuCM62lpbSGuCbpS4twBTg1AAc835ODg7OnOx9MazL+jabeU46AsgkpSNSSgpQkDy+3sAQ3afeQcFotnqlkGY80tNbbCkLsEPtqbKePrFxyLEevYjB0fqbkmpspfZzI2ztuHTrBSV2F/Y8Ec+9sZglKRTJDqmYkSPKvqbfSAGVBCSTuNn98BJsEgG59Tzg91iZTXkrmqfYfdcXd1DLZAWtV2nG0kD0cBGpANx5+bEFr7KZWrUUDqFlSa6ptGZKSrZaDrxRLvt/xA4se5wsXnPKRDbyM10dKD6mY15he3Hm9xjIrqaiqXCjPVVxoRNyS9GnguDdAukgBINwColfJFiDckYeZ0yDFXMk04x4xp8bdfZbDZblPkAjS5e7ZSbkt3F7G/e5NwhZq0rD6j5KnTRT4eYIrriuL6iElXawJHJ47DthwrWZaRSEgS5LS1OXAQHm0kD1J1qAAHvjKzjTJVCqDEl1+QgR0qD0ZrWLoUbuHVZersUpA+om4svCSamlykBFQZjuQUOOne2nQ4k6xpttqUAs2JBTrNr/v3wmfMplC1nErlBmP+GYr0B98BBDLctCikEeXgG/P87Yc0ONKspLoAF0kBQsD7fmMY41Sn4b8yBFj1HXJbZYebp7boe2+AAqwcG2Qm+pAPcErAOCa1Uqi1GbnutvrZkxW3Hg2gNy90qs4G2kKDYGpweW5JGvQPKTh23U0W0DpcTqbV3+k9x+eG6DOgqmSIiZC/FN2Lzbi1i1zwQlR7H7cYyPSc01ukJmNLhybvruttTiRZawLFRQoEDQR8ogA+Qk3FsLd2fKl+PiNSHvAOLS4lMVbu42LBVrAr7G67k3uD5LYPezKZQtZTZNNLghypTCVL82hbiQfzsTft6jDZVs1UvLrlOiVGalT9VkFiC0jlT5tfgnjgclRIH64zFBl0yl1DVSmnJrSFXS68DHdCCNIKhYALSSDxdHFtXNw1uyl1uU+zR59QqManFZehyXlvarqAJSGwoNrvcngghuwCe+C5TKFsNFQd1NIXTpSd5VgdIUEcXusg8dv5jCsOuBBu0QR6Dm+MjwcxZvai7Jr1Spy2nXAWXjIbLxWOyig3On/3ZTpvcg2Iw3Qq7mZU9mrwa9Ilh3mMp4SnHE/TcNKJOsgFFiOCLmxwGucoGBa7cqrjb+2KTNLWgqMhQbQ2k+x1qB/lbClch4MIdEB4rNrtAp1Jv9724+xxlOoZpq1ZioiVWsyYqJKAw2DLefa3DySpK1E+gsAjX5uQOBgH/pAmUGa6abmGqMPGzBbM9bgWg/4wFwqF+DbSB6g8g4OZHItYOPONqSlDLiys3UU2Nh/HBTdRYcd8I82409a4S42QD+R7H+OM1SOoOZczInOQH66kF4Jix1yHGVpbbA3CeQdaQQTzbzXvyLAk9U89JbCnqtIjNMuoSyS+pViU3SVqAOsEC/JPH9u9gdUMi0o7ISXXIsRaUvuNlaFFVxrHFv8AvwubdSpF9d7cE/f1xmZzqTniPJbV+NzJyH1OIUhlLrIUbXAGscL9rEC6SOecEsZ5zvCqCXGMy1BmPIS4uWklC0NuDUnWHFqVwNI1hINifXjAzHdTIFpWm1IVOGmczGkMhdxsyWi24kg2sQe2FTjhAJCex9Tbj1xmNfUbOMZ5Xhc0Tag8hag85HbcUGkIubaDfkaeSUWtf3JCmo9Ts3VtyTSp9cfipjsA6GIgj+LXosohZUDtHlQSQCbgffBvohYLSRUy8gLCkrSexBuP0wip09FQD+uBLhrZeLZTIQEFVuy0EEgpI9Qf54zXIz1mhmls0OFVqhuojOSWm4iPDHRYLDzimgNDYBN2+DwB+9cmMdQ8/ux47y69W1I0qZjlsNFLqxxdLgKtwjkXUASQRpvzgqObZabd8RdJZWLC+pJ/e9ufTBhUrTfVz2sMZSrXUnMGYMuUbxtVLq4SDIeQbqkOyE6hZGhIue1ibaSDcHDvROpGcINGlphV6XIAbO2XW9dlhSVOBJc5TpQ6gdiDe4PFiuWymULSJfSuQWQ+3qSm5R2Vz2P8jgph5l195Ldt9sISu1ykHkgX9cZwa6ndRGZynZlWcTMjNLQ4pURpSQQAS0ohvmxJ7WXewt3wFjqBm1yoR6j/AEuqK3Ke4XpTCnEtI2zYpDjZFlpIc1cH2HFsG6ORRJqeY0FxqQwh+RJcQtlxTnKUICk/LdSsEA321m1lAXtxhe3MrsmGveY8Gp9TcQR4zqnm1aL3DzhVYLtyA2gC/rzjH3TPrtnPozVV5S6lw6qmJHG1sTQQ/DFvrbWQUONi97XIOrvY2Op6bVsv52y8ir5aqtOdgSme2lPDZ0pugLIuTYj/ACe9gMVSRmPzCsDg9PyqnVWn0JgSHy3ZxuIoJ3W5ju4kpHOnQCi/v9I4Pqa9BrdRWhmqPSG2HdaAlkmOr6AOUg6Nz/3hBJCvq9iVKZSQmV40RnG9Fw8bC/JQoE2X50jtfgc3vwdUHYImN1IJQwppvddQtwqKn72cKVcj0SSCRweDyL1olFNyaLBkOUQ0xp9LehJLa3Ck/LOpCvLa+jy/xHJN8FuUtWtpbLTEplp9yONaNIaCF3SLaASRqJBUONI7Wwt1TFtOxnao4+olx8Rw+0IrCFgAKSBb0vot24ufTBNSqLUgGZU1ArmrLbTKXLhhfoS1fz30gA9ue9+MRBcak1ioPuUp+cttbbtwmO2lwqXfVcuAH0J450+2HCHTFONLkuCNGdYBbBLtlAa+AXARYea/a59ALYaYL4Vqjux3DIeaJaS24lsiyCVi3AC7jkHnvY3wWqdOmJCdqOYy5IdS0twbhQOEuK3CQi3bzG/Pp3wXHMjpbvJxrExujPll2KiRvJK0KANnSsDS7qKgVk8c9iceZVCnqU2/+IMtQ7GWV6GmuLHQAD+0UeLggEDvhskVWQ2ylanoymU60sraBUbbZJDpsUAi9rdkgnlRwkeirVELVSjNTER23Ahkr1pb7EBLiOCbd7cW55NhgKd3Mnd6ezCjutMoXIllxCo0hpPYH0U4s8Cx9Aeb2sMHN1ZxTLbyamsF4BmSkMOKSm5sq5IsQQe3Y+h74STZEtDaYzW69JfAULNoKmT2BBPe1xye1ybC18Ftw5lLQ/TKlJdW86PEHRLL/KBa52wRzzchfp2wNkN05VM7r01DyW2Uq2njIW0UW9RdzkIuBax5Pb74IlSkuWjUGy4jgW4hpo/KsO9tFhuAEDzLtYjj3BUZFYnJjVBiE66l9pbS3JN27AcFsggruLgA+gPpxgDaWXFsu+FfU00C2lW0g6QQNoLAI0ALubn3HGCjdEw34MRwOtQFvSZKlxw4HhqS6Fq8jYF0bdrHTaxN+cLI1RpL7rEt2K5paU6xJGhTrbnfgWtZHfseecJ4sh15mRIjrbZaQ6plp1Dlg24b+VJNgQee4t8wc4VPPxnac5IbTTm32NstveH21OAWAZsB35JseL3PN+YmckypTjSfEx6Y8WkNONAPtOWdufsBaxINybAemD486npqTsx9kuSwlp50SFpU0dZF1KsOUd7JFjza+CBV4GyfGQvGaHA5GQ24CH1gABCibWQBc2A1+hHaxrVRadYbkok+JfntPuJeZZaecZQgpKgW0bh54FwRwbnlJxL+SrCVV+c2/oabVH2EhxA0OOEyPJqLilcLDg5IF+LWt64OmeEpcVhqKAtExJcbC3N0tIC7OIO53FwOR6XHN8J0OU+aW5kyNpiS44aW63IUVAkklFlkgjWOALXH3uACZLo0B5CEsCW1LbWyEpuEkcDbuQpACfYk3J47AYGZNoF8kOp9I/or1NzNlgstITTq9MZZSkApS0JCtr1PkKCk9zhml1NymydCJFtj5epAsVW9RjSHxwdLpOXuq0DPCYu3T80wo6n1tskIZqDSC2pKuAi7jbSFXFhcKB5BxmCoxnnnFr+myza57jG5hzN0WR3dTxTs3mItElQbB7pufN9/y7nC8ZuXMUX3vMBbyFwgDv2/S1sV88paFC3Nxb74KdmlIQnVcDtY4a6l1ZVRzM1LbEt2QVPm1mbmyQBzbnt/5viKmtPS31HsD5T8ztfEXcqLgtwq/qCr7duMdEy6NKiQb86fQYBehnU4rctbM1pLyLKMSOXAFgnXtjn29sF5eMF2sNrcYcKTwRqCQUepJ9P05OI43LEiHCssANMhoAciwJ4GJBlYNtz2HZNlpQoEoIJ1cj2wE26i3VyI5EzAmO6LFoKbuUFJsLWuPe1uPvhkoFIU/T5EzsbWBH7uJr18gON53ZDgCfGth8JBA0Fdr+UfR6G3tbCadC/CMuN02lhsvPmzg2ybpPcm3Ppg2Uuq2dQhLqkles++Cggk2Bw8P0kQEFcttBc/sJcBt9jbscIlpssaEfYdzbD2Qa66FBh+MktxGwVKcWBx3xdmVaJGocIoZZQgoAKnErup3v5z2se3H3xSMeoyqe+Hob224OQoAYf2OpeammER/ERlpQALrjIJP58c4NwCpZXWmU625aM4+dfnKlNjueADzz6nmwwJozErD6UL5OpwKJDdrcEji454GKVHVDOJStKqimzl7hLKE8H04Hb7YSN52zJ4sTV1N1bgIsXFlY9gBc8D8vbFna3SuaSrE6mxUPRIk5trQtt7b45CEOC9rkc2sD+uHLLVFecpLtfS0RGcf8Kzc3Ki2lOo+9ucQRGa52YAqJIkuOJKkL0LII1C/Y2v9ub4s2oZrNPbptJistR49PpkaK3p7l3zOOuH7lxxX5ADFL7O2Ra1JaE6qNXAtDzjRkMuNEtuFJsRyLj3AtiU3Q83ZbiFtg3IF7W7DsQvt6YgUapINSiSWgFI3UHQTbi9jz6YnDjclchCGo6yptZbHnJt9zb0HHNrY0Q+GyR7e8nBuWhpwNlsHQq+lRG0o6L6L3+uxHuRq7XwdGekJeXM31sqS7pS48QLuae4R+/9+LfriX9JelGYeptVVTqQgM0iG625PmFWllsC1m0XtuOc2sORquSAQcavkfDJ0TrcMvxMpbEtcdbxap9QfSGmwL/NcB0X+lJ7Ad7qwslQ2NwCgiLxdYha8SpTTSm1tKHZwKNjza9rXP6c4IQ2UueRaF67peJWSNY9geLEDt3xsGd8H+RFuuogzcxrS0stmSipt2SSVXG2W1LBuk+Xk6FI5BvYmb8GuRH4jSaZnbNbypjh8M0h1h0did6y2fOgBPnJXzq+3M96i6o9mVlBb60FKi426u/KU9gi3BI7+9sOFGoObs1PvNZZoU+e40A8tyK3cNC4H7QkNt888rHOLhqnwZZ5jTXUUfO2VJUVBLTbj4kJkJtzfS0koIBPAKxccgWvjS/TLIOSeimUHoOXodRkS9ZlVWfMCyptaLaNoApRxqWEN8ng8k67LLUtDe7qmZG4lRLodkfqrRKS4/1FzDJehy2mm26W6puWlAQ5uHdklNyQsCzYKxrHtwbDqlNYrVTcq9PlTYpZhCAStoKLi1uHUAbaLqIDgA78mwFsLoFVfdltxotK3pThWpsts+ZLi27baQggEW1A8A83vzhRSp6XZKkPuyZcN0NyZTMhTIcS0h0c2AsUDQQS2CQE8jRjCXh5uVp7yVx4cJ+MG5tSD/iIbksPz2EKCrN2ceuOeUN2ABCAXDe9ycNsKoxprrT0+RIqMdh0O7URpZJtpU5vquSHNB4Da+wsbBRs4/h+5SDTplaXKTIlOOTlJna3SwHdwungaABY6RwkNkWHbCOFPiMlulMkTYgeW3HfUsOt2BUCovM97Wv8tFrkX5JOKi7kiEepT25MpTzK4sxiKhndiOFCWkLRdt2/Ot4hRSU2sUBwC4vhAzIDlQkyW57E5/xERp1ynCM0GzpvrcvygANWsAL3LZuTYLG1SkV2O1GfS2puOY7TSlFLTbG5favco0FBPIJPvY8YYqk0qDIhxlLkVFAkPOGNCpDcZDLji0hoPqWm9rK1bYRYgII7kCBQpfKgQ6eskLjXmguvgQEsBLZa8pU24SgLHPzfPwDciwAZ9bznipMKeErXF0txWkNu6SdIEdTmoEkhKb8XF0W4JGHKQIlLq7zNRXLkFgrZadMTcVIWAASWtVggEHXwdVhxzbCGLDorUxNNRBckykMbba5jbbZ2kWsTwdvzgkKuCbm5PFneUuVKoFQm1BtFPfZbclx5C21yBrSJqwVBSSo8ldlGwsNQFuDhNM8NEjTUV+THiNokb0Rl6WBpDYSUmxVyU+cW45TovfjDkWFUyeZaIrgiIZcLre8y40HQ5YoKUG9g4LaUgjg37Xw2mkwZ82CsOQ3m20uB9gstNIcXYCySgE6Sg89lk+Sx0rXhcubUJkto82pJqUaPSo0hlxqT4uPJbCRuBYCUpU0Loca3Cko1OoJXo55OPJbQ7NaYsJCOHGlvNu6degkbSW1NrIUlKbWXtkAk8EDChsOQXlPrn7ccILoU3KDgsizZ3Bp5uS0kjz3IbtyTY4SIL9Gkzaw7LDfhUBe86llLhRutglS1G58xOlV7aQTfizeaTyRf4cKW5KpZRsRmHdzxaIvy1Nd0qSjQChe4kcW4ukXsAVJHBOiS2HFMR1vNsXEoN/PdXx5QCU6163ALpK9FrklBwsgy4rMtZZrDTglBJR4dCbuOBvaSDulWsjg899RAOsnCeXOc/C47CkBxIItswtKXFoNlEk2BR3bWpIFzYj6ReNKNspSOn06TDDjXjJDzbZs/LfWEtNOctqSoi6EHWnRqNrDUePIcAgMOMxHIzIdebko23G5EZsiLdGpLilIUkhBBJBHchQPJuVEOqx5S5E9cYmaxoQlcUbLYDhJO7t9yS4hNv3gAtYsQcAQzS3kKjvM0mRcrNlzVrLoc8wcbT5SsA7l9SCbqN7XQSqiOiU7x8B1VUguh5CwI6AhCUvOO2DhKkfQANsDkHsFgg8IqVSqjVJaXmKogBCQ4uGlvSWQi40lWuzmkJbsQfUtiwGFLTL0iC27QEOSPCI2jtPNtqb3GyEtFrSbABSTqTYftPUC4EsuokM1d+bUZMSM2+lxK2UOsNOAJAABFwb2Gomw1XsAcWNddRxSp+IxPgTINQktpho0IZUpnYW8UKOlwFpe4ORq0hYQNRA0XOCH5EfLjK6h49aZJvJckRIpLQ3Fm7qVN2CCRYEcgLJuFdyRLmQoKn47iKewt2P4NTspXylrcaDjiQdVnF61OJ54A27WvfCsyhAjvSUyZaahIZLba2mltx3HR5bONEFoWaQRt/Xysjg8Kd0Bsj5lJZnOqTMW4tDofdfYVN2ni6A2OLJAcIskkflwBqujY251LSl52S8/EcQ41OdlNpe20I0tN6Tca1XCTbsgaydd7ojUHWKZHSoNstLZcUEMpKWWFhYBUSOQuxsCSQdKSAbYOahMUiqRJD7bc1O3rKd3W6y0XAppwkeQ8XN1AArbHB4wfJBDlwkIalLFOcU8sFUWO04VKaWEE2DfLnfgA3ugE3JPBjUfxEeT+G019CkPH+quhDm2tDaUuOttgAhy9+yOW3DwbYKlOtiAxSUIahizUZsPt+I1Ealbp22wEBR08AgC6jYcgoabmKE+uGmZObcmbQXFf/D0tusyAmzYUFt3JVqKbni3Nze2FacqayWOTYE+S6tENipNP/KakiE44XkI4Dm4gBYQLGyTYdgD5celz40mRGSayUOPLDkZBYcLaTYEKSXFXWoIQFcWte9gLYOTO8AksPT3Ym2+XWVaC2HnW7hITzZAupIt6aiLX4KjxrLLMoRJ8xLc4lL61XCSXFEuIbdQpKFglPckDWRcdhg+JL9pJ3KY5MqaHhSHJLLoLUdfiURg84DYnncI/dIBDZJHA4wbWI5zHLWrYadcDLEeKS/8A1hkXsNxzTYEqU4LKAF7ea9gGakim1jTUvxBoWjsN6VLZcjyy2FWDirm9ri5PnJ02SAbgc0Q5S5a1qm0+LLaaW4plaGm3RYanW+xIS4oHUk8X5C72wGtRzJY4tLEMvMtR0CQ2iWSzuMhjWCUuCxOhYsoanFgjkruDyJhqNGU27JW3FeacLLhZe1tNWa8rbrgAJVoJ5voPccDAWktIZbkUtmY+1C3Iq7FuXHLi20hxW5pCyso5WbWAN+fRA4/HlqhZgmKfqMxtSyNxTqTFDdttwFGlsXIcIKRoKCqwv3s7qCVSo64DaHocNbLl5Cw6qKVrsWxpWpCypCFq8wuf3L2B7YBIqsajQn6hLp8iNGjtiG4pa2lR3lumyQ2WyQ03rTwOCbOa0gkoBlWdjTKjJkMSIEhUhbYcT4jTKbdKLpDm2lXKglQN7EatZUb8p/xKGueqGzUIaT4ltJaU82nyIISq+sEfWk6EkjgLJ74FlEVTnWp0tmntMATVslDsYNtzHVIFg3aydHPPfgkcmwvh5W3EiwojNTkSF0uSESo6GpG0460HNttDYQLhsnUCR2RYAoJvhPGjVFcVyXEZfKYzS3F0+MNwsrvpDzaSlVilGlLdhrAN1kWOFDFZ/DIiJT0d8tsa3HXG4jwaJBAc+gFetPm+o2uVggAcNsoo/UaRQkVR55mH+I1lxwONyTDLrin1oB3HFAgL+lIDhvfcsVXScKJFHd8WxXI0pdOUwHNLr7QcTKbQUlxN3FFZdJNjy1buSQcK3KlFRS4q4yW5Lj6BYIbu8+StThU240koRpRYFKjdKyF89wVDqNIJYpFPj1RLLzCHRGS2hSnGtI07bQVfctcrKQvyEE2tiZk32U8Gl1enB1aKcy4lAKFMuvOKbA3R5HFX55SqwUvjse5WG6TRQ2XZstMxx+GsokzI6hGCtawlsAt6uQTpLQNrkE3GB1AgREzNyI2xMWQSGxrLYds6C2hGtepXzNwg8t3BNxZoer1MeqhQtkwm4wdZdccbW1GcXt3UtbxaSHDZxOvUSbKKLAWJVuyBS6psuwIe3ObmSJodDMdgHbc4AJQElXNxcngWPABsDhK6qHIQ/EkNVCHUZimlh8yXidtYdAbCioOgK7EcgAE9rkK6cVvJgtx1xn0lwocdTU02VbUW+xIcIRr7+cIPksLYEudBrT7EB0lTcmM4hEapSXVJSsAB0OKWpQcAAUAAU82Nk3OGQSWQ4lK0IbqDESIuMGtpKUGQQsJ3Aog2aKkJPzCSbAa+cOUgQ346JdSYjrERpDLIKyqJ/aDQ20pCAoDSQVgEBawRfkx6OimwWpKnKow4I258+I0SkFptsIsCQjz6m9ToGpF7a/VlpcVqK5UalAkyUwIsfTL2XGmHkxwC04lR8u3dZB02WVHyA35xHJw66WTNlLHiWpEl79ktEqRJ2vJ5Q42BcFa7q0rIIFkgfvHCKG/WVOIZmDL7TDDyG1xzGdQmXc3Fmt67hB7ckcci3GHhtlE6EB+KB5mQbRVLW2HZY1cutOoBW2Vc/UEBKAUAeuBh2JHfkVJ6fJYizHjElQ5Ci6pLQPlJCxcBJbAPBtZC+6uY1K5LWmkQ3XTU/lJcmtynHUR1PkIfDimfLdKAbk6BclJJv2OG1uKoS1xmY0JbD7rbEJuKG9yUSRbSdQuLptqGsD73NlDbcbw625UOa2Z8kPBLEpTafE6W7hQ7kaySbkE6SGygjkt+G7PkiPSohmjWwyGY93Y6m+NTYWvkkErH0dyVi5wyCTVaPEkqcfjS4EpEsBZYc4aUXPMEjzcLSRpJBPYWFrgnuVOc46xTXAFywy2IrLDqWHuwACiSojyAhYtc6RzcgYXy3rzVzdTEeE+XDELKVhwNtou4BKPIWnTybW81/Ib4j6plMrDaaSzHmpkoS5pEhtwJcWHbHaK0pWPUgWAPe5CgcRM1t0fJXR1R1uMsIS61L8QAw49uQmCbhGtavQqULC/YHjSMHS4TktmIyjXLlL0NJkNtF1QN1BQLlgUG5SdKhds281iLiitRnWY8F19puPIMsiQw26h5xsEJA3rKCG+5DgGgAAcc37Imw29C0TDFcQ5IYcVSoQQzHQHAb+f9q2bm5sLgXsQAcRKksYxpklU9mA7D8ShcYwo0Z1MgPgpAJAVdxHlta1rau9zgLtVqcNiTSlTYy4iFubcmPEDijZZLgddWR5zpCdIPK29HFhhQtqlKZT/6plzHGmiHoUpLjocdbacsT78KCgTpFtAWk8nCiCmYiRDaZnsRVhpCYsptlqMWkL0uKWt7nyhxJJPkPmFuSbRRJGq1KirdXKpsmbHEZqQp1wrmloBZ2QXEE6Aq9r9khQB7jA4TjUylt15dcgKhvuN7rjrimmWm39Q29taSAFFThA7BfYWOBRyoPOOsqRFmlxx1qXdEV8uIGoXI8nzTquCTfSCscgEmNJmvQZzaYiA6wDJamOoc4O4FFCm9KWyAm6Rds6UC1gDiZUWJDnDp5l/OFGahZgo8vMMdZRCjqQ4gyA5a6VxlaVEISXBzxfgEG9xX0PpP1R6EVqXL6cyWMxUkyG1y8t1EObzzhtdbZaIDjraCdG4hAJFik3GLoiSoFViOsJadZ8TJQDyWW13RbzWPnQeLcXNz7Y7Mgfhcp5jwYQVtLSl1U5VitfnUWw0bL4two3B5XxYYoZK8KzIN0kouYKRnumN1PLb21GYuJbcuGUvMy7kOtlvuwE8WBA79gBhRJemM6VvuIQhF3kFt1BIWLJbJSi4B+4N7+mKa+IJORMrRnc2TM7VTLeYTZMKTRFlT7luW0vs3230EKJJJB7HVirMv/Fx1UkxXI02NQ6z80ueMnNuR31L485LQOv0tfTY83OH7LOM7UpcButYNwZUvw66pTXGXFleqc807tuFHJbbcJICgi40get+4w/UqHV5KZDH4HsR49lyjuliO1E2yHC44DoBBSCF8rPNrXN8b174js/1GziKRl6M2shREdt1TavYOBxRQ4PtoF/XCcfEP1YkBBRUKFpbcLiUCnAJv6m9/5eluLYBgfbTdHOOa1siRAlsSH6NMciylrWs8OB14kHyAngNhAt6Eg89hjrUVcxtRs2uHIY1LQJSEpIsfLcHyECx4vfv64ymeuPVl+cqSak2tD5DimXYJdabtwVh5xO5cnnWVki9uccPWjquqUVxqu22h9txq2gAIBHmWld/W5B+xN8TsXjcqZgVquiVKlyVLcWy24i2h1DRU0ps8pDrZXqPbuOL9zhS9MpqdTi9ai5YRbtgayHBZCQQAbIBJ5ubC33yO91N6m+J2aVndyK2u5WPDtkKXwQbG97W9729Tgt3q/wBZpygzUs904i61uSEIZiOnkqPCE2sLqta3BAOB7ueqi1uqKyzVHKwuM8uXCC31yXNLpddPHzhqLdzr49R9zggrkRoyHaVYILPhNTBSpi+u5JcPBJLYAA9ByLnjJdRz7nVxl2JLzfOnR3UkOJ3Wwnknj5ZufU8/bnCONnzMNIbZK8yVhTF7Nhp9TjQF/r03tbsOPc++D2B5lDMtjx41TBU/MbkMJBDjp2SdxpYCRpcB8l+eee/oRfDbIrTjrh3X31IYDjDDKWnW06CsX3hpAJGkWJJPlHtzkl3rDXIclxFLr9UQ4+5d5KnXdrt7AgWt6A+vbBH/AKa66d6PT65Wo53bILVSfdF72Hl1WAA9Af7sP7uUMy2XTkxKdpOlDkwEN2DiyhNx5S3fSbqFwbj09yDgyQ9MgPNblXP+1JZ3tTaHOwBaABKyRzpXxcdsZLf64Z5gRkMTcxVhxDRO4h6XdrkcnlXe5JuAbnnjnCSd8VOcGp7bxzDPQ4x8tDgdaCk3A1G62yfNx7+uC6E/ZQDhzWyKqmUxXHqbOkuOynHmnnEIbR5ZBAUUu7YcshNwlHnHIF73wTOqfgoc1qS84hLbkiO4/FbGyptC7rCnCLIbHPrwD25vjIo+J/N8mK7DhVcMObp+QvbCCSbXIQ3ftyAeBzbvhkf6zZ9loTDXmpyU264gFEdADgB/fVtNgWvfm4798I2nfzR7Rq2ImsZbqLy3qpR2oFNiWdaXJASncQPI42XDyjk2uLjvfnEBzT8VvR3IzhblZjpzjsBd2IlJ1y5AcQASCUWbQTYWVuWQcYc6m1StVptMl99chbQO64hwvtg376iSgHixAPHY4ql6BPfUnShwpWNYsPbjn2xY2mA3KQy2WjetHxdT+rdK/odTMrQKZRg2suKknxMp0hYUlZcPDdj6NjknknGdpbrYQskoSS53J8o55/M4IqTKsvMBjdu66AVaewRcixv9xiPSKgpQShaza5Nif54uyBuypc+6VTHWTYakEC17HDWG9aivQsI7mw9MGLktxfmKUCkoNkrPH8PXCP8AH2EBGknjnyDv9gMLul2R4hgrCtKdJ/dBwoXTZSG97wq0tklAuk8nD3Q8sZinwVVmXlqqIaFy2hqJ5iBzqJcP0AX5CLYQZnzvWIyxRGaauLLaI3H5Du/LUsj0PZv/AHIHthsqOVI4gUWkqKzcE3/O+JNlJ5bNVYfQQS04hQSrkGxvY4ZGqJUoaG4ktDiZQ/1QlZupLh5I/PEqyJSJEmp2WyflIKgFdr24uBe/OFKtTX1mnNv5npFYEgOIQC2XE+pQQf8A7bDLVs2Q0tI2nXO3AB7jDxn5lmVllaXFK1MOeJYJA7cg3/O/8hipSSe5vgg2FkCLp4XW3FLJQho35KlJucJpMp51JUpVgTc29ThE33w4xYTk5SW206kgXJJtiwd5LlaE2qNzjmJQ1lpJsbKA9L+pwYrLSQpKVoOpYPlHCuO9vfE7EodoFE7YGhC1HSnFkUXornbNa4ycs5KzHUEv8BUalPrSVd+FlIbtb1KgB64u3JP+Dq6xZmcLWZZlEygpvbW63VpgMjQu37NlsELIv21jsfQYRzQzcpwbrMtBrj9ITKjtRmXRLQhAUtI1NrQsLCkn0PH6jjE8oSEVe7z24tS+Qg/V7n+7F/Zl/wAHBmOiZUrGZcv56OaHadG8VFhwqK814uwKiA8pRQLttulv1WU9gCMZ7olI/DEsIbcUW3DrSrcNgCBa4PIGBo/VhRNxupPWqDTobLaoDpdSW7uFZHBPoCPscad+FHp/0c6m0Ko1TOni5+bKPLDf4a9JcUw7HXpDUgMIst833AtJWEDSP7XOf4VOadZKHF7ikdiQSUgXvb7Eg4X5SzjX+lWeaZnTLjK1uQFElgurbalR18OMqIsQhwEg25BscA3LdEBuvp9TkUmOmY1TqW61Q4jTrkUNOpjF3zAtbjPOhDtrAAcmx8p7KjUGm1tM0qO/EaCy0zHdC2Ybq0cqS20QLkIKSG1LuB3BNgI70+zBlbOWVoGaKSJMmnViO1LceduXUhAClx1OIuUONqTY2Hl273sbl88CiGiNAibiUOtrI3WXBJfdI1Ju4FWsbXuTcDj3xi727lfy7qWJq6npLNVZSIbHhnW1KDrb6Ut6kqUXXAAWu/Gq5vwANNyB+ovIejzI6Q5KaLbjMtSWvouUpuLqQvUEggKR3SceXEqEQQG50xdKjrccaWVtuONKc4UXbodDnYq5JAIsOORhbT2ZKUR4cOmGG4hwNoqbyHiLoaUC4RrJAKwq9rWAv6XMLr7opiRmN2ghl5LLriYbTTcueHUJtZ5sk6Sm1+9lC60hQ7AYPmVilOIU+4uCdbgbjocdWpt9pw3dbVosF3Pt5OT++Dhxp7MZhluQrwoaiXRFCZRaDrQbUp50uDSCgN6Tp0XdCjxwMFU6RBTlxbctVPmCQwjZWps7qTpFwpa+dCgOCLW2lm1rnAR8STS6DPmt0eLSIz0fbakCGmMwVR23TbesT51oaHa5AFwSbnDf+HpcDVXm1poqkNksqfTZo8cvKGonyj01juBqsF4eFzWY0mYyhLTkazYbEqWTGEhto6bBi90WuQoWWogC3u30h6qMsu1Kpx6YlCHGwExgXDNWE3bCVaSAgI0WDl0JKrae+HYhzXo8erVWG8nfWhYaY03W4NxBPlQ4oEEAWLnYix9R3WqnrpzbtYfJYdYkLIjtyrtt3VuKRthQIQSpKj5Nai3Y3tYI0rjsydBzHXCqRAQ2465GKW3F8kqKiCC2BYLVoIFyBfi5SaMzJlGjrgBx4y/MsOle3oFyUa1HbcK7OXFgdsLt3w2VSyd6U03MhQ/npjwPnz0qbZSx4q6VJU4kHUVkrTcHt3sE3OEFRXMhtxJbNTmQEth8MvQpHndBN0rcOq47oSNRv39+WuY/VafLMSPW5E9DjZWXHiy8pUm6tTTakJs46UWCAbjgG45AC+6Wnd6AuQ4t+OWhT4ryJDagtF9TiT30lSb6fUJKCbYDm95LmCepFJjRiieytla0MJSHvlpGgqSNsAaShIJIuQgElei/FioLf4TU4jVNrb765J29pUu5Dm2bhLaASbIski5J44HOGRdaU9UIy5zoaltS3ilawCEy2lHcIDfK12AN1Hi5sk6gcPbUmS3EZC4dPIkSjOL/AIZQdFyNKdNlcJuq2m11q7jmwd4UwRNVpzNSe0OVPS9tbhjttbRZX2De0s6yR5vMpYF+fW2BzpKW4wptdS9JdjMlyJdQZmKcbaDo8twNz5l1kA89uUkYbxCmOPocjTy9TpEjdZJWEhDYWAdSh9BBCSALXQRYdiONohvMhcVqfUaih5bILjPym7p0qb1OEXA1a1ltZIBIAHBwzdlAlFGqwcoEZdey2ijzm23Q4yh2PNabBUktu70fcO0VlXmAQbp5CSLg2DU4D7AYjo2XC24sCUy4VDyEgFsW3UH5fNwRpF/W62nvTHIjFNWlpmJJsvbdlIX4g/uuXIUbf2EqAWQRe1iACZLcafXUkNCGXHFuFmLFO26hB02HmK17hRbnXpIJKQCMLmzKtFpdUlqOl1kSEREMU1mQJAMdTpKg55h8wthDoRZtwIBIsm3ISQHJkr577j73H9SRGAeUXN1OgLa7IRwVi7hta57FZPkVSKqms1ZFBchxqgpFlMtAXDdtXltcWBNtRtY37c4HUYweXCpLLsCE4GS5JSpQVY7d9IDTfnCiRYuW+kX72JaO93k+ZNdHqExctSKlBmMRYjgbZlPzgtiY2+CoeZyzjK3G22zp5BNhe1r+YeolWW7LVW3XJLQKJSm9t95lYc0hHyz5PIL21k302tc4cJdLqMGW9BlqacDbVk7UopU2gIHOlxQDfnWk6geC4SL2AwqbXJprTUOlRYjs4CMppphCzpWGg5oLKCQ86COXFOEkAi3ezudZElFyBTISXH40BqOuQsiSt7SG3y4UqcW4RdYCSgEkHglV+LXbZcJcqYhydEnlph1YU8+7vBhrnSpKkfR9GhB5AJb9zhAzPcbEZE6D4eU5819W00pSQskl7TcLKE27ggG9rHBsRuoSmqdWjVWHXUOCUlTTCyw2W03S6NZCPMEm4sQC563virveJROEqRLQ298gKkUosNrsysFCFu9m/wC2grABT31hayRYDCdL0d+OhC4aYsqGjxKTHsHX7p0hZDw0OWcA0AD39LrIKtSo3iI+9Ibbjl1K3NMXS20DtlJSNNv8c1e9gPY8HBj1TfiSLmMv5UEOeLOkO6+yQWlquDt8X+YQgggE2szPClckEqY0KwI8VEOcwlwtu05br0cENX+WpSL/AE8iwIP9g2vh+gmZS5bT0h5t5bmuSG5VNcYaaLt7BSRbaIX5UOhYA0i1wQcJJaXZL7s2Yw7HkSG0OtMaykqXthSSPEKJKyQeSvskhCQVWDRJr8Nu7tbePiHHb+KSN3aW3ZN1eUggg8pVwTY2xHeFKhOttQW5viX/AALjcR3w2iS623vt+YIN7OLSCTa5BJugA2uXKSmqxfAsz69d54uO760uhp1boDgUlK/O6UhXnLfqQBfvj1OWmSpiq0uK0psugxJEt0syE/OUEraSHjr1B1tV7mxH54DPYnu1B56MJk/wjjrikBMZL7pKSNsgamnG1Eec9iACLYXkm1TGYcCBVxUI7LkSoDiSFRnAT9PzLkaEEgBSOL3BIsvC97RPckIjQZiGUobQ4+hQS2pt1CgppsfvuggpLagbBJuDq5BNiN1BgPVOWuSpiSX1bsl5pKW1lzeUlu+i4WU6yrlOlAPI4UuMxpKYi1oe1yHFl+SvQEvuAkhd7qF9B0rKjxqFuDfDMUzI1pplTKl5boa1pcZcl6WWUPJS0EFJLt0kLJbbcVayLnyG17BO2oGlBEOShtxtQelK3CqQW9AbbW2FnRdO7YAC41Em4SnC1bsBuYF1iTUWGAhxndQ6t54L2k6i44TtggHShKr672F7E4CzVlVZLLsxhrYmunxr0jWdl3bbYbKXEXAIQnsAeQO9icWJUWuV4WQ2mLGWp6A6ZM6U/NL6rgpB3ydJAbuEr78eguRhPPpkmqU1GWfxRc1iWslgqC3VxVkcPJaZWLhN/ICtI5UUXtfAKrTKlTpLshb9Ocbce5ckaWQ224Lrsm5W44QlKQooCAHCvm+F9PckriR3jUm20FezrbZUQ4UE3aS83pWuzSgnUCDrte6E8QDRRBYp646abGqqIUWQxT1uSSlLimpRttlT6XEpf1+V0bihZJKDYcnBkdU1szKqzTae6ilcgSZ4ATG1hQS4HEpBcUtRSgjyKItYWFy0sy2iwtTbTcCMhbcx3eLjSjq1FdiVXIH0bnCzqHrjkh16lR2K22yKhPXEcJcdh/OaaLt2lW1EhaipTluVi58xCk4iiNhpqUuq+PmziuYVrbjyWAhCEvm1iS0sAAthxO3fXYEcXWSmmGd4zeacixXXyjbeDwS1obSnzspQDYWKRc30gti45uKnSakNbuy0tpiWtomZpWzZ/wCWlpTZuhbqllwlwcD1A7kpFOQuE6xQlOqlsMiPOfkxnflubgbcjlIJK20k3unQuwNxo0nAS3SuDNqKEU96TNXFERtx5tfjlKesuxs4m/cEAkhAuSObC+AMOyUUKW/M3FuSH23pW8lKtlxaCnXrIvdQUoA8osF/SQLx+l1FCHdFcgGIqXH0vBLhKUhB3AG212cWeBYBYuVfaxf2FF6e8tMm7okB1kxXWEraWjUSsEgDcDbYRwAPlk97YKZF0pmoxoy36cyJKG1oejRUxkBDjjaP3UrVrfXYFN03Ll+QATZUt6t0pl2GxOMzwjoU0yobrTdhuA8AkkLcCSEgeQ8kkjCarU9syo8ep0SHAbQ6jafdhKO2jmyz8wlZTqCSACBqJBJGBsolRZDCqcHHSwEO+FMlDLYQ4P3dZsQVhQOpCboJ0G9sS91EXHp9blLqG7ViVoQt90M6wHAtvjbSgEEOEJuXLabWse+HBtuS060qZBS61U47ojLSyhmzq0DUgJKiSebaVHkHsQQcMNLX+JwJa6i14ASW7sxIsN1p/wCpNmnQs2sAdQHrb15JErMztPmSaTstCFPSBO8KpsQ3hzaMm+oly7XnUNY8ourkkHkma6yOW+7vO1JFPkSHJrKwWimSl5Qbs0UJZClBd/pPFk2Itc8LIVUbqinaky1LU/tCEpMiMGttsWabacaWlIBJUqwPHl9CcEy65JqjTrLcnZbiNLL78idpYdd3BthTIBN/2gBJ5v3AVcLKbLhwX333aOhyLLQ40XIrxspawdwJbBUVkAjyquixABHFwlSWdIgx26ZHeQ6lMDWsR2oxXLUA3ZtalAOXvZsabWuAvsScKX91c2RAnTGHW1r3HtcVuOXzt33EtoSFhwg7ZbFySkgpuMIqyxXZLDtSordOhNtmI1KZcnAO7iLpDrQKgvQoJPABJJOhNzwZl+ppp85/RTZ66ihpHghFW4pLa/mjSXbt3cJVb9zm/wDZ5NlEGLHZdbXD/DnGFttboebJUyUBFtZbCj8sFJPra44AvhXGcp6wGKkxGZbUhotPP7bylIF9xKk30XAvZKbaRoJuThIKatL6zJjRIKFvPu7yiVttN21JJbQtLjbSkaRtmx5URrHGCahFnJacZlUpe44f62iNdbLpBLoDQskFvzNGwJNkggkEgHbVROMEwEKjxWJJprq3VsuJS6QoNlIcNnEEIQseVAatrBPJPGONvNzJsqCuc25o+YAltam0rCdOhTJsSsaACq3FjysWATueGkTKcJzERpbkVcmKiW8llqQUJIcDQWUr3QSnsLXHHNsFqXWJDziKn4uQzTo+1HfjcPOlYUlotq025JUCm5AKlrJPBCO7yflqnNunwmZMyMzDeQ74RtbeiUpsOLuHAi1uT8xKUBQtdu/IPIJ+hLMeetmSseJAZnOX2nNvy6NxCQtzsCVXQWxf0N8N9NgRqDTVJi1aZMMZkOlKm23lMFBuRuX17YKf2R87Y85sBhU269CfVEZqklh9DR2nlkJU02GvM2tSHVXBGoLSNFhyT30FqB3RrsWU7XXEPPUx7xq23IKVSS5BkObYuhty4buLo7mw1aDaww2vVhgbLziXEIkOtB1iM6tTEVZGoLToSS44buBBIAITySQMN0R1hTcmXJQ+VNq0yZidAJQs/KacFg2sjaty2DoJJucGUyJOQqRIK0MKaaWwWHpoClNkqU2RZy5tt8JvYgIRpuL4ZKnNuMxUJKkxpC3qeW1syETdbiiG/KmwundcAUQUpAIAV+QWRxUos5NcNbkxENOtBQgx2m0vubZSHAkDWdRKkFRAJubg+hcFbs9puJMmyGHHG0NJZWnZU+0RztqQm5XdKjqb0XFh5yrCX8SYZmePZgTahtlxC5LbUfbS7xZpKtVzqb4WLXIVwBwcC6ikK01SJHlQGERJrxu49ZIZDRtYFSwQAACbc67OAcG9mmpyarRIT5oLzyVLL7kVp4IUp9wj5WnWSCg8JJJNxe18OMunJqEaQtVYnPvPvB1wxlNRHtH03IXbQsFRAJI8nHphLOL09Tsn+qJU02h1ReWFbFyBrGsXcAX6W7/cXxkatJXzr60qzJJzzPn5tr9HemocW2IkB/8Aq0Qg8stpAAat/ZHIvz3xDIdXLb6S3OKGwmyC3ZQPb2vf9MfUFdNj1hpqJVqC1WoJeXtQ5TZWl4rQNwNJCbgm6fp9UoHa96vzj8LvSHMVTVDqeT26FMdlL3noA8K8okEGyWyGkG47kEi3ryMaW1OXQhZnRZnLGNOzPvulmHuKeNi45Ib4HFz72Ath9ZqSELEkSWuf9ruot/Y8cdsXqv4NqDHS87Qq84WQFqLU+TvBs9w2CdsuixHHfvyMRX/6lnqVAXH/AAWZluQsqHISprbPr5EKJP6m9+Di0Ts6odm9RqHUW6lT21TqgtYbcBbLiiQkcn1/uHPHbHH8x0hpkpdnvodHBu3a/t5ewvz7X+2HupdDOt7StqHl6mSG27r1mTpsEGznl5I9OTb19jhpm9OOrVNTERTciVSQ6Ww4fAPNNMqPbcsDcjm3v3PYYfOw7FCzkjXLlocCqStyS6v6FsMrSAgejhJCEevqeB64MmNyXG9YWGXCv9syCQBflXFgbXPPv2OAysq9TuYr2Q6ylzbBcZbF1JJ7LAQTxyefXi3Jw30/KOcmGUsx+mlZStbhbaREoLyVOr7AFxYuBfSOAL9/TBzA7qapWXXY8VLaKy6/L3A22qU7pXo+ytWsfcXsMFsyTMZEhpEhQQjlatGyLdj9PAuTa5xyo5czo0nfmZBr7G0Nx1LUVYUnQbK3NF7WIIuRfDdUZsLwzM/NbMuNGaI/qaGnw3ZfvexI4A5/PjBsEE1VvMdJSrw6pSZCWCdttDbqi457qTwP4m3HtiOplz6q4I9Ha2WUglx1bOtKr8ewt298TSl5gyYWmmaFWCpUgLQY7KZEhRJ5UgIjtXFxfgrP3wmkVOgnfQw7MdWj6GxSpLLQIHcAp5A55NgLYObVRR9FPQmS4hUqbJXGF7tTAAx97I1bZPoD/LChSw5HRHiUuGt5yzRVMeU5a3+M1XI+1iefYYcy9OTpFHo1UdjvshxpaaU4ljzi90haQ3c8W735PphfA6W9Rc1yURqRkuuqaAJAXFEdDqx21PyVBAvyRtjv+uJmHNS7lEHstzo61SZ1ShQoqDrKt1DDaUf5ItZf5AH298SOhRfGstyYSpDsSwSlAVpU8b8JbTxoAufMbnjgWtiwqT8KGcoU2JIlZWgIdeG4hdTqLD7qljkeVrVtt8k3JF7G18WFTvhgzjWXX4tbzvT6Q1e4bgUgtraI+pAedcsHebIsgG9ufZDMxu5RykqjahFUUtQ6u5490ECLCbIJaJFySi/bsCogIve3bCObBpFBLNRqjdKgVGe0hqNEU6jvrVZLY8y3OSLqAAI4F7XxqRXwk5OmU6PT6jWM6VOGChZjSKl4dmRz5yGWmtbmkfvKX/bIxUfWSh9JenlfqGUsg5MgQH6NBjy5lRckSZMtTha3G0JU5YhoIJV2BOooWeLYAmY7RqTIVmvNmRs0VipS8wSmWmIi3OD3VYcAJSLlHY2BuT64QSOkNAh0xNZmdTaG1GWgl1b6FhTZ5FrA3XyCngd/44tqLnPKkmEzGzIpDUioOuNNa4+8ypooCdRUDxe4Fva2IJm6h0DL0t5xCqW5WEOW8kXcDYPPlAugE39fv2wxKXIoPMo/SynpK9GZcyqTdG4wwYsfj9/gKJH+cDx2wng9Q8mZYYcOXMqJjSr3D7tnX+39pd9HJP3w4LzXnhhL8BrNdQTFcQsaGnNIbBNzb2v9vfEKmZQivK3WZq2dRt80ahe3v+d8ImTnV+tea57yn4LhhrLe2lwul51HINwtQ+3/AJ4xDqRLc/pBCmPlTqxLbdWVrJKrLBNz635/jhWnKNRXfbfiuabkpDlj/MYJFDmxl7qnmELbVdJDurkc+l/X+44iIKvOHSl1aqmY7TkPrqbxdKXVWA1kk3t+vbE4i1CPR34ceLE0NvrIIQyCHEDj6gO1uO+In0+2qg023JS3qcIcVqvqTcXIBHAxYFWprs9+ny4r2ytsbyApY0mxt3/gD+WIiqM60qS3JnRYNM2IyCjQptshJF+VX7c35GKgYjOSFEI9Bck40h18UtUKLAkOrck7PzCpWr61X4H54pqBSo7amkup0uC91eiueB9sWsiL9VW9+XRNNOoLjqgt1SbeqTe/8saE6M/Cn1T6lsRKrSaSzTKDIaXIFSqKtLamwSCWmwCt0XHNrD/Kw6/Cx8Pw6wZ9kVGtoeaytl5sSqq+RcOOHhmKOCbuG9yOQgGxBtj6ZVKbGptFpdOytT/B0+OwwyzEYp3KG20hsLdTay+STawQ2gDucVVM3YnIwapoo3S6lZdyL8CfTuhstVDPmcPx9x1C22UtyzAgMujnQraUp1wkCwSVgea5JAti/skdHukOSIp/otkfLlDqrbW5LlRKf4iobDgTyl0pU4Vm6zdNg4CBfnEn8LTvCR478OBJkICCg7pSkOlINlXuggW1G3e/1WN8LqxMk1BpcqYtrb8SVITHY0kEjzFK227pb+WNF73txYEYyGZ79CVpaxvRMNPhU/MCErp9CCqe663Gc1SlvSVOAKuthgXAslJuVXQFkdrXD6JMNGjQpqOVl0LSztuJh8/MQFN3WQbhN1HWUFZsoC+BrrLW62ujoDkcaHXBtlKVm9myVW3PK3wWwTzzze2GmMXYRh6noy3nS4fn6wpK9wKtyg2I1chzixAJA4NSJFlx6bDczAytmppfmMtht6O86FMgXTuJte7lz9uALi2PlfnaFRHsy5jXlvL1Rg0uJVHXYtPqiS3JYYDt0supWAWyR5gk8hDgHNr4+rDz8NxMRmTAQxJYRfxKHGw8psOOanRe1iEH6ibeX1vxTXxI9C09UKKrOFA3VZ+y4y+9TJTKNpVSjIAJiSHDYCwDhaUPocJAuFY007ww680sovZZDNXgZ3pkWq0qlNQnAX/ltftBoABG2Bbb1nj73Bw8M9PYz7dP/F5DTcR2MXntLtnm3NB423ObXtwOT/DFW5dz7MoNXdqm8ibEksIjeHcO2GBqClKsLHc4F9XN++JVXlTM5RF1uleJTBPmebSC4oPggE8d72B9gT740bHRU7qc5KzbWfh6rakNZzrDFOkrLsmB+F78GcbABRbKgOwAWQQsj1GNM5C+Mzp/mqr09ma4ijuR0W0qLyI7j57KbLYUQvWbkuAni17DnA2ZMh5jhZbezDMa/rcQanWXpWl9hBI0h1J5QVdwD6c8XGHvpvAoMvKhnT2z+IuPhzUpwocaav2H3PJB+xxHMY7VM1xavp7CqFNqkDXDr8SrpqMW7FTCN1oaEX1tJI16wu6bAGyyeCRx0GpUxzajFx9+M6XUvKJLtSdWSQsOOWIQ3tkEnt2AucY1yJ1Ur3TPNDDlIqs2qwJhLiqYqIt5Dqwn5iSNJQHngDocJBLgtc3xuSiGHKh07M1Ey22I1TTEfakyVuAFlxIUhyzaiRdBtpcAJ08gLxkezKrQ7Mmh+m1aYuXrcDLsOU5O0ygHG9hCRuEuiwQ0rW3ZVyFc2tYXMaqy0xFxG48+W9Mdbs74oN7IDikp3G1p0ABfI4sD2uTbBs5ysOx25kV1uGnh5UkK+pfZxAbcP1k7SV6Qu2ngcYH4JuPGlSqo9JjCnnxjh0AISFjhoOIKVuIHDoZFraSgEjAd3VLJvo6pbUKo02uIhqYQhyWrfIXHQ4EBOtSiLEkjgpJKS2TYgWw5SHaxVHI7Jjn8PhkuxDEcZMtTe4k7nogal7bnzDwiy1jjnketx33BXGZEsvbmluUWwx4VegjzNlJLlhdR+ld1DuU4Pku0qoiRPdhBHiEtuNveCCQpDay26Ua2iVrtc3UbNlsWHbAuhZR9lVZZFKqs6nRqYUKfcZhgulltrcbS6VDlwo1G11DbJuLC2C/w6XV5hnKprSYMw+JnJecLTw5S2CQLC7ljoAAQmygSOAXx+tTZM6VBTIQ2hcjcktobS8+62QkbbiggXuEJOlJB4HmBwYqpUuapopj0+MlUV195mW3tltZWXEgqXd10myflt2vdNzxfDl2imVMgi5aeQ49PeQ0ypvb0RAgbLQUTvOpWixBXwNIty3rUR5Meov4YqQ7SXkIaM2QVEG5KiF+XVoUABcdr2Gpa7FAFzW0v1xqM1V30MIDeiShLzjCHUNkKbbJdPIT5QQm9lgI5J14AIVDYS9Am02M+43tOWcirRI0adSmmiuy2yPN63ufbjBB+8hlCIqlZq9RjtMzHGIqflw1wg2Gi6hAF9txGq7Nm1JNjrVqPoBgwxlyF1CpLmOkOsANR4s10bbS0qKkhsjhBCfqUSSO3pYqoRxGYPh2/CjbW23sE7gR5nNsOoKgSTcLCQOdA5I4UyIDKJaID078ZqJWG5CVxQlsOElwEpulFxp7KvcJN73Bwt1MqTJis1aQy5MgeHafQ460202obKNy30o0oGp1XJCyQ2lwFPubVGEMyg7I247a4z6nEyHVMpb0fvO3B0WCUnTwLaOB9GFrEyJFjxVvJYdVt7cRyOy28xtDU3dKVkLQs7lrpbNx/uTZC6YUKoPRHnm9FEhIYumLueRrUHnnCQf3Nuyij6AbgkDDG6gak1RzFGYa8PGemNQW0odcsw5HU4wvzGyVp4OsHbNh634IsBhb1QoyH50R2NNaZ8euPIW49++mzjjhKShu3YJCrIJX2JsVlpiVS6iqohqS940tvQHXng7YIKnNZCLlttVgBc+bzcCwIXNVh2k1BtLVMmR5ckOHSFghSELO35nVWOlFuRc3sAbKsRbKplsmpbCKi+8l6utKcW23GZIc0pZXZKTdK9VwNPYlKCb24Ng9vUmOiPCkqhy3I9OjIktSEvOHSbF0CyCleoI8xDfYnm2k3Qoqs81NhDMd/XUGYwWqa0XkpRtLIebbBTdAQ5w44STuLJI74VJbmQUSnYdRacK2WgZSHCtsOWO6Gzo8iNZSC2Dze6yQAcP8AZTck0x01R4OSKbBqMaGltDJbdc1JdLfLg3XECwcBBIuLWAQQcGuS6kmY9TZVXiOh0hmJADjyNuzlkrtwGkGwNySi1jzexdY0alXRVqguGvVtlx551zaBCiGwNd1uHWClGoAL0/cWYY7lUi7U96guQi4F+AnhlL7TxQ3dRs2Nbi7c6gbeUm1u1e6GwXlsoprLkhNP3pcSU1EabRHbSGnClKVJsbkApAIUUXF19uMONOfmRmG3V02RxGMp7UNpJKGhfS7fQfIkWHqLX3L4QtUmaMsy2Wd9mEixU403xYJLh2FWSQVcnTrXckldh3bTDr0eM49SvDrjmV4dUlLviVALbGllPHzDYWNhbsjjm1l8ymUKUQnHJMAM1WRIcffdQzFTNbXtIWtCXCRrtYqTYBIuHRdHHF22rVlhhwt5kYahpfdW3EjGJtraWHPoK+VjUhRuEgABVl/TbDXTFzmJ4YqCoi2Iy3FldmkqSCtTakJPIC3DpShI4SRxa4OHPx8mHBNQlLjR3Yd4qo8jadIYd7pT3Otyyitwk9rcLTgZQluiY7VQU1Aaj02eyjecvDEVyOEMC+oEBJ3Qok6FDzlfAve2AT2oEqHNmS6fMmJiKRGdJhCMoocQkOM6hxr2gC4Pr7j1OHZivpksvLnw2232HNt16RdxLgQtslva4ANzcKb0arEhI746p91ulpnPRpFIiwmA7HdhNpZS2VrSlKvlthCLWvdV9wkgkHvUU4dlTXRqw2ulIREfjNoiIKI0eU2VlskuJUGwL8kCy+BewFuCQUKnTqhDQ0GGX0NRnyCpp5xpSDzcaB2upVmweVhHIGDJTrz9QbqxqUl5LGhS5F9Tu0CW9BDlyEKuAArzqABGj0KbqkiLGbZl2hCnx91WuON8IbAbS3qFi4sWSbqFgPJcYGU8k/dT9NbZgRRS6o0Iqnw426/IcOpxtwtqI2SP2gtYbl7diCcIXahGk012OiQ224w25IWEqcEdldtpxy6wUC4SU+ZYuCL24wJ+bR3nYbU9uTCW5JAEuot76m1o8ykncHDbq3EkHQ2stqI5HOGWltFRXDmyZ8KWhIc2m5BRrQAorQlgqsF2AFje/wDDFkeZVuTrRJNVbq7tLdjzPBOEuJCw2llxBCi22Au6GwXEkgpJXZIIvYYTVWXTq2204+yFNNMrfbqIZcdOwGrKWppzg3NzcOBd73uL2LYfVOhIfp7jEQuPaAGwlVkOJLazpAs4SflLv2uCAj0HMZpkqa34iniLT2I78kOz22lNODzNFCQhs29DqUta29NykkjDNckRyYmiuQVTIpqUZraDMhTZTtl1F1NpaCSXQAQLgcFKwCbDC2bCpsOMuYkNRlPlEp11th0FSF8JLZWq13Wwr5qjwv8AdBNsEuVCBChlDs6fFbciFMV5bjjpils6Sq5JO1bcBBbQTwdQF8EpqtMVIp86RUWFykRgWlKedZDchCxtubpF2+By0Ro8w7X5ZREidAjQ3KhAAYXIfbZiqS0tLQQvUnXe5cQCsaUXQQCSbgWwvebTTnG46JkynSY7QuyCElQRfXqCD8w3uLkgqPHpgDja0hMyptn5brq1ofQGHnQSQpCHCnkXASs2I+WP3+CS/FzIufIjVKiIcpjUU7LjDiEkSFou2vc27gJQ4QhSkW0EggEcVlEKKO5yahvO0uYzVPlvBZfcihmMlu5La2ShQBsQErBRe1jcXJxLSKA+4EQ6fSKf+KOtvurbC33G3V6nFNqLizrWpYHlHa/a3cL05DNQ/E4spqO/Gs9EkW2bWu2TbTcBXJuoEkg83JwlqEiMt5ulSIbalx4Uh2IPDW1NtuAaTwG9aQlTguddk8D9zBBugRlTnWam8iDGj5khtuNRnXZQfVtKZUu5KmwopuGwUtDU4i50m/e2AuO0iNTY1SorztPiuGO44tJkMtl8qG4Cb/MQCBrsACXL6RYAJctVhlbKFvTg1EnujfTCW0pnQHDbab2xoJ5/aLWQbDm4AOcmVZcgM0mrOyVOtOy2iy4ldo6xxdwgXATZst3JK9YBuDhs2VM1KJspCoikxp7ZbkuOIcOwltvd4KUbhUkIQ2LHzH61X7KwN/NcGCy28zTIyZT62nWQ9AYS1ttAXccbDQLulzndBPccWwGc2zIQXF1hcn8MAbQ2Y0dKklvSo3Sg63PJwtNuCmxBsBhHWG26hVmq1W5NpwmLkzkuuOsuKQsEthRsonyCwULoRuIvyDY5kqKhz4z6z4JinMRXXER3n4obKUDyuPrNwEIX9IBFgL2tck4PlxnmpkmPMptQlOxGltFktqZMNZ02OqyRoAVZCdwXuLg3AIa1FpjLUN6S+25DhWZhUxtl1wMEIdCkN7f7iSSbG/nN1304LIZp8iKmPTEMMvsIWHnWm1Jmrc1X1JLiitFwBtpWCldz3uMTMohQYr0Gq0hiUiQwpDjojSDoASVqSErI4CwDyu4HYouLYUl+I3Ud+mpfioMhsNiK44HmwhRUFC17Nhy6rAmx45CeRMmTAq0YqntCRUGzux3HFMBTB4KnVIACBr1DSrzqsR63w3txXaWfxuBWajFVrDbxSXvlrWyAyptlwgNi9rGydN+LDBcoBmRj66wmYhVKy3InjwLYeLqGSshYCi4kukrKFFu585tZJGi5wRSY1OWla1QJu8FrU8hS3GnnS4DZaS0bWb7oubHTYkAjDu9VY0yqOKVJE/W06W0KISw1oBUpsNFJ+Xxc3AQFggkA3wkIp0eoxacWZc+BHjoSqRIZN3UaUtlsjyuNtXA0XcJ7G3l5KORENtVdqS/ApsmWzOKtoNTnG4b/AMpHytwNt34LpJSn9/sTcEelQUTGWkCMhlm5YeXHc1vvONWu4u51sd0hADdiL8i/K/LsZxSH1s04wprhbiIZkE/1p9DilXDi7EouEqBcCOU2HAsVL1ZrLK0xKfUm6wZDIZlzGVrkp22HC3uFICdtsAkC9gQL9xhUE2QJUBuYIKos9cSa225KqciNoDiAlxxvSUarkg2KVLJ1pRe1hdKH8zQqhLiMz5sZ65jOhzWl3cF1AOjsQNVj7DvcAIIHWKcflguMzFuanNdTc+UgL1JN3Cpi5QeNJJGrzg2woVDQ07UaSmb41ElCw6F7uyy6FDyaUKBR57JF7ArVcq5F4onJDlIMBpCqi6XnfDhpzcZWUtLsUh27aQjkXLgAWqy7g2thEqLS35DcSRMkJZEEvNL21LSWyoaW3rW55VcpuLpJHe2O0zwyUS4dCQ+06Uh1tl8JKVFCUps6lwhF1EpsTo0rbJPBvh/pwfLEdtqmU5EtyGZBWh1IckxFqUklxQcJbdSVOE2N1H8jgIsTXSYs9LkOJUYc+Y62rQNcVwNTG0L+Yga2ygucBCFFBJsi17YKjvKjQwxGmDwT6dmQ0lYaUoOXs2VPhSDcH9qlBA0rBI1GxUXxWW3kyZCn70x0adqf4aVKNwGwlQJQ2bOJ0KBsbALPKiG+oveJegymK1Nj1Jh59oPyQ88ypoAkNloi6Hhp87YdJJVzwbl0Wp6zJKhuoQI/4qiE/vtuBSAf8WlQZseV6Wmkkhsg2sApZujHKy0uRJlVaj1AtuxnGkNTGnXGWnlrb0/2dYWUbusfQFngo4xyFARVZ7cNDLDUWQrcdkR2A8+dDKkpLdyUWsU2J8iSoXPF8Njv43GqSVojyX6U0y460wuI6FMB3SrWeC2QCEpcsvbJcNlc4Fk6flrqEcR49NbajpduHHVxmh8wBQKvOkuBwDkc9iLjjB8GDDcjw1ubMSGSUuyZDCXQ6tz1bSQkrWL8Ak8i4tgbs2oyobdLjOtNsOOERy6T8gLJDt7HyBVgSP8AJ79wOSXpSqgyl5RQGrhocpDzjadskpRfQs7gBXe3A5xjzX3VqA+/BLjtKEcxij5i3EuFx0sXURcLKS2b2buALe9wLt1KhU2FLUijQ3BOdaciulKSXChYspu5Vo5JUL97YNUiKIEeWlS0tPrdBWZJcj/LJV5k2DlzdNibi3IPfBUNplbT7PiNbOhvzJZXcGylKcF72FgLoJ7C49RiIF1katl+CqQHFMpmsfVJbS78kAjg35uF35va3BtfBbKo0CbPmojN0mPP0ba0aAlbSx5S4Afli4KiSL8Cw7EuCJ8sCXPehtNDbYhbTy0oDZWtJu3blwqsSbjy313vhCVGny2ZzSJXhjdbS3mDbgGzrgc8hI1XQDzxcc4iGZJk1eLuQ4z1SQ044VlKG3QppwWslGpA97m5v+WHF50vuyajGjS0LfJcW2tppzxHyykOOEqAN+bnvws+2C2E1GKw3Ml1SSzukNNhTNjIuRqcQ9p4INgU3Nzbvzg0O0unOsw57bk6Mw64SpSLXHNkqA4bBuLuHm1iAO2HU0SeC3U5LilofcegDcavHb2lKAALRJXqHmNr9wLAffBhajPOFEwyvAtxg88tLYccAIuriwWEXUL88EXF7jHqbWGZQMOl1Uz3GpIU4gO6XmkfUy7uA+f7kISAO9hguFDq9ULbzdLKIyGpK29bgaW84Fg8lyxdt66bCxAFzc4iRcaiypzTj0OC/wDuLaaYiGO6kkX3FKWL35t2FhYC2D49Oh724un06crlxwrIktPLAPccIRxewuOR7jCRbc2pB/xMd9C9GsMJijRe4Dim2x6ADn98WNzccGMGnRpaoUh5sydp0IUsEbTQGorW4UfK4AFhe/FzY3xFP9SEIUJawWaY05ou4W9nbZUSSfpbB8gHJJ9uR2wFnLlNQltuJAWsII3VtoSyVhdlXVoN1osnt2+2FiHpsB0uRmUSJT7TaClxoNoDZQUnTtk6Gzfta6rHnscFSJsuntxYbyKnChafFFaQ5HYCNNk+ZYUCeD2F+CNI4uCHItPRCRQYVXq7gNPbcklZda8UghloHuAACvkjtYGxvwMH0hyS8SiGQgIO64nxIDRIHlCrqsQCONRsb4Z6fMnzlOzUyGzAaSF63y81qDZOkhpfLhvby3sSB6C2Hxo00UPVEahi8pDjpcihLgbNgUJ4sBrAWBbkHvzgo5SjY1OVIpypxKYaoyFvOJMQrS5oHayL/qf9BvgdLliG0zOby6te28Glv2cNwTqcbVyCbaRot2sTfAqbDZmx3ZlWU000QXUSlslKSQu7bZS04SEeU2TwT39LY8aA3FRHW+hunR/EO/OjOupQ+VgHki5Dg4uOw7Ei+Edopz7yQSGpcdxxlhmWtp9GhDDx1OjzlStsgD34ABsQQO+M5/EX0yzjOzMvOuQ8uHMG5Bbp9dhQg2mV8rUWnkscFeltTiXAnnhB03JGNILROcIYiy5UlaHC80VNWbUgqF9ptZ12INrpB84HFr4DKhU2U22y9Sp78htxbTzLyS6wQL23CLX7AkEW+vDsfk1ChZ91fLiu09ECtPR61THSxIW2oRXIy2JMQkAhSWl2I4BuOxHfk4kDrtFfpSF1mjtyqa/8tqpNMlSVECxBcAugpHBSqxB98fR2tUDLfUXZoubssUyfSo9gGZrDSmUoc0koaSCbbpA+nz+oI7YrDMvwkdC5ziqnl6kVXJjhWG5K8rT/AATTyP8AFgsOFSF9iCXE3C+Ob8XNmbzVORfP3OmWWKVUHXVbTLBG9wRc3+w+1sQhcBuXIGoOFN+9/TG08+fAvW6nT01qkdYXpCVuSI7LddoqUJGhsFKN9l1IJJIFtoXtf1sM65/+Gf4gunkJ2uT8iu1OktBJNRoj6ZrNvVRQLPIHry329cXZ2HYqotKrOq5Tcdjf1N7U5a9k/nzyfthFTabDgMvRqq6Yrradbbix9XPt784s3I1Kp1fp8GcmYtM+Q4tDcZ8BtlwoWApAcvbXyODbuPcYWdQOmWb87TVVuj5dK1xytsU+O35rNtFxRHPohsqPvb1wVLLvSektTKzHaUvdKgtASLDUgC5PPpwDxix25W5mxCYg20Q2kNs7jRSAfzsSeNRv98RLpDAlRHWHYDIc/qLrqXHEo+ZuI0i1wbX4N/S2LdosSmzafTs1LkvlcZCEbjgHlWgkAWH754P+jviIqhOqNGnZlrtWk02O4tMNl2W4VK0nbbAvYG/34xWlDpU+q1mHTKPHMiVPfbixGRa7rrigG0fa5I/IXxffVSOxQJSqiZDioNXZdjOlKkp0tufLPY8gE3t62wL4Oul+Yc6dXqdXaVJjxqflDbq1VlqaDtmrlPh0pPBcd5FiRZGtfpjRG8MYSqntzvC3N0O6Mx+jnTKmZIlTIcSUGPFVCa1DUW5jrjhL0k67EpF0pR5rbbY+i4xPgittuupW/Gjw4JRpW60hT7r+vSWwpALhKmyqyR+4BfkXwbUYkdxTUalQapHfbabU/GcZZOpz6toJDlkIJB4AvwPtdspU/wAHTpOZqvJQhth0ux0NbkjU4XL8hv0AAuL9xbm5xynOzOLit7W6I+cqbKfj0+eHIbSlol+HbDakeGcAILaT5xqbH732AGPUmkliM46FVGPDfdYLQWtFo6HHAC6PUmwGgm1iLcdsChsTJc5uvRnmntZaLyZDQCUvuHzWH7QNkpA08ruB9F8HxmpMmAUQJMtxlbu0bvF/aWi5Js4okgX9TYA35xWRZMXZUUuDEgT4kGeiRKZaC33oKmiUbizqISSPmA2bJIFuRb3J7cKY2uHBjJkKaW0LMKik2cXpU4yLqsQDzc8mx79g6O0igeDkxpTzba7bTc5hl5xwlZKg2oNkBs/L4uba/wB482a6pFTU4zfj3mJIcSA6025qkawAS7b9mDyo3JAHItxchrwTYIFul0i8MHWihARHkklx9xqHoDu35UtlSyCTdLYuD6kDsMIG6bW6bMAeZXMSxMWhtmSt0Je9PLoUbOAlR5KyBzfDmmhIjPtsvSWGIYaW02ltfi1vWc8oVr0juom/AUsGxscCoEGRKp7kmmPSKi2hAQ8ZbRZ3UNOKcShWvlsHQSVBfISByOzbBV7Kleo3wo9KOqFRkSq0mTliqkEO1egQdwKd1AlbqeWJCBuKJP7S/r5rYoLO3we9Y+lJXMyDXVZyoziS5tUtpyDVS1yFr8IdTSzoBNm3b9vLcHG9nmaFGkyqnLfkJk7TjrDryS+lgBZADYuG0AkG5SjvpsLi5KpaabS6lGcnVuryVEBvw5gX1cBRsocA63T83sO4OLo5CEjmL5z5VazDRo71BpXTLOsqYsv60ycuzdYkOJKW9QWz6cGxt274d6n07zhOfgZek/D3m6gy6UlDkmMmKuOwqR5jvKdeVtNbh1XbC9FwSADfG6ItCQ21HfiIkuSomjxRdmOPthFvrOiyASSCUqueRb1OHA5YqjM1vxVEfpS10x15strWhYRrN3EgupvIU1quf3b8AEWxb2nQJbLGuUfh0+IH8UXmalZ8y3lqZLaaYkU+RV5K1SGudrdVGQphtCbcXKx2toNsao6eUnM1ByNRqfn6qhVfaSXRPg6XozQddcLZ3i2N5wBIu4pAII7Wth8plOENt9xh52JBKGkPx5Qb34sSyVeGccJJ1kHUUgmwLZuMKTHpcRh+MTGUzMcbaIll0q4aA2XABr4bdW4ggWHPthHOJCZqblVKY3HiN/0t1PlDrjS1EtMvsOAWcug3BsTwbEAWFxbA4D08MPWeaIjtANOLBVuWbG420VqKC4OSO1jbuSAHOJIacp0OWKNEltlADL8h8pL7fCW9snjWEWITY3DZ45AwyOKh0/MA0xUPyUKu6ue0+wkOGyrraF0L7FYb4sHACAeMK45k2ZSSK7XI+hmm6KfOjt3AU20Hw7tDUC2VfWEOJsfcKJUOcEueMlu0+I7uusxNcWQGoyXGSdNrtpJ2xa4UVCywTcntgiFUg/QpFVaHg3W0tIdYU2kpcbCioNOdyspIBN7+178YTyWqi4xSMwttR3oIXvOR0xioRJDiC0rcAI+X8sKJ5QrgD0OFsjdLpm+3RW4zDLaIrjK2xLCA4pDAVwFXAvcahpPLnJ5HIam4NQRTpT0OpNrM0thxSG22i0vzabOFJOgLQoFQv+zIAuL4jsqQxUKm0K1TZD0FvcRoUjSVONOWbDbaDx9XKUnRYn2ALm7NBahzKglslKQpbwcWVkOeWyTyhBLhUojQLrt6cFmtcUzjzT3lpEx3LiaskGnKg1wQ4slmaDFmt61IakBSxraf+kbbgRqXwm9wcJJ0ev1GHIkPMMyBGYW/LkVOUWZzIW7pLg3QSHPKUoJJsDZAPAMfnUrLNXjGLJo8CpJd2/GMPx3GUuo1lvbBcHntpsXE2WColCkei+mxxShBhvVCo1Y0iHHiNOTnlutqAbOy3bkcC6iokuOnzrVrCLZslS06EWv57LQXwEXtr+fVBYokmNKbTGh1SUHHENvI0LQlvc7C7rgcJSf31aSeTZJNic5Lqy2WGIdJLoaWh5DsGE21thYAG8pZO6LtJUAEWJ1n2wkD0CJIhzJ6ZFMVqLbMapR/mLWRa7ZWSQVFz6nRcW7BHdaqKiPRXRUEuPoLb7zclx7yLQ2E2IdRyQlFiL2tyTxYY0c+6sh8SJpFJDVfYVUdBa3Wi9NS62rwpXdwuvD0LhHIseVdjfh3ZU03IK35khxpx82RI2yXO5AS4i7iG0rbSQk3HAv3AwmcrLtQ1zIubA9EW+hltz8P3w01ruWkJKbNqO2Cga+LeTvfDcrMVMp8uPTICy4wVIKlSHW0CO45dRbDZJu3rUlS7jkKK/3cFzXOUBulswKECPVV02JJZkyyWHF7anXNDelTunlZA4buo8m/e9sN6qrRZ8wUfw2qc8pZUPDeZoBwFm4Q6AvkuKuUHko5NrA+orQ64uC/TXX2mmltCKpASsBDaUqBB3Dua1knz+UWNgOySOZzdOhwJEmO54yOXG95pLKQwvi4cLpA85B1+hvyi5GC0W8SmaxTnVpHjag+5DadaJiANw1Rm4iW3EOWHyCrQtGi5WggI16/Lga0Ih1aPLqSyfFrEpUhsOAp+YS40VBVibbilqAsLE6rgAxZ1hYqSKvHqDUyQw3cqAW4rWQnS056lwoCj7kaB2Iu9VhDsiE3U5NdqL8dqKW5LbKm3GEtBs6ikHSUD5ukar8hQ8/Fi5EpRUI6HIZmGctioLfaR4N2QVkuBx0l0XBJKeUgKJcTZJPrgK5i5K47TykxXJTvh2lNMoSHeE3Jc5PKO+kA+axNrAedalS6PKrCA45KkurZWI7OlxxoNABxpS2wi4WQDYXCOVkrtgxYqQnvyFrpjkdhLTbjDjjxCmFpTpbusF1wEi43POLJKLXACOdmRRFXZZjMOxo70J/xcgSXXpcptkPALvoUbk9wUk8fs+LWFyPFzqg7rqa3IbEZxEIuMstlarBWpSr6dBsL3Sb8k6rm+OoYimYahTC2+IQ8WFmMonxF9LbTbbZsVjSBZS7/ACws3towU9NiM1UR4y0OvRHG2m21PKbIauLBRRdaGyCALeigs9uXDhySeFJWxFdVDj0Rl2GxHdQ0FLdcQ8krKStxw+Y7ig2Lq5CQOByVk5mgpMhtURaky2wSFa9px1dh81Jub+ROqwACi4BYXuDoqECAxJpypMVuMygbaJN1b4cUC24TYlF1hNySU3ANtXB1YdpUiqIpUOpNPMCNugObT+3LC9s2JPDQAHJWeVE2uRZ8yVcWl1brLs+C5EauttlBkoSGdDWlxCbq0E3I1gabdub3QCOwiPOYYeRUDGRGWqOpnbYU4Q5Y2AuXd0DlwkIJb5vcHCVUeshI0xENABtMeQl0kR3HV8Oa7pQjUUkISLlKLi3Osu79NjRUCnw5UeniWvaejyHnZLl2kEttMqbdVrbBupY0I7nWNAuI1HMRomhijMu1hbaIVSaWHW92MsWLOs2tqWdC3baXAQsi5FrWNnOn0SfHp0ht2lzW/O026QoWZ0XIQpIsLlsA23HNVlWwmiPR6UlEFaG3nXnC00484lTWi4ICrHRcIBOoIJZKiOxtgFHqCGUIdeoBbZjsreumMvdcjNklWkEgABAvuK4sL6QTYq511LW3SabTKhCee/aqdW3oUUoG68hCtJ1WNyFLc4b4sBcgkjDcik0mW+65MlQJGttbm48yDfXcuN6l+dCzbzlJIGnuu2HN+DVHqfHhvwaQiPIEg6NlaVC7oJcBZ1WIvccharEgc6ME+BplFT45ESS2iQdEpt910pqCAbFKi4k7dyQUJ20rF7jkYLEF5aVxo7ERx1hvab+qNuuDY18htRIcWBu+caLgq9zwpdp86RTHpzMCQpuSyt2Spo/K2Lt/ToWVti+lWl0IP0H1vhUmG8HhFQITySt+O2mOop3WCbJDZcSp1sJW2PMfOb2IBIOOuPSZ8KU8mNIprL9oy2XVr2Q+Ufs3W3VKCxoaKfMSSb8C4BNgokLLU9ExmgwZqIay62v5pcYcaWi6SCp8AoBClE+e5At62xxx6oIfs9OjvONh8hlG28FBhStsNlzlsAJc0KKCXAqx4tYt67bbkdLb4cdcJkIWgOOuAeuzdKFu3CRYk6gLWPN0TDiH5LsSfTqjLROC3AmVpQh0BzdDY5tueVXCSbnse+IonKC5S4mutux3XVLS5aUmUtQdAccUFtBAtwEpulWvT5jYhN8KGZlFnIiz1M0qTF3mNxHzGbkanGm97TrDRNuQLnbssA+cx1TLNNDc6RJjzFahs+HpqoTKQCpRISAVuOX50pNjzdVhh2bdah1BYbdCYFmv6s46ksyEAgFKjYFbl1kFJQLaSDe18RRGtOxmX0h9p+axMN3HmCtsBZIJCm3Wwglek2TyORe4scCadgogwYaoLHi23H3kR3XEIstBClAWIKGw0G1X1q51eXtdIiO9JmIQqnNNMxoLpQt6XsHcNgVBweg5AbsCrSRzfBLSG3nWaaoxKjCccbcO7BbLRAXe6knUAE6laBrv6abC2C1uUIO7yVzm3lUlDK7oDjwDMd8FSXn9Dh8wb5cAt+6SULve50jB0KbAp8tiM7JkMs3ca3yzsIdddtbU6Q3ZrQVNkeQ3KPN3Ab4C1ynHaOywUCPIPhgl515Kli6g404RZtxLibFSRazR4CzbCgSWUU/8VMKOw821I3fC7QUkBxIcCWSPoupyxT9XBRax1BFLnVS48soTuK2g/aS2oaElvb43EAlwFslI1L9SQB5CE71RYiI8BIjNyLL0PMuBG2ACAlKnG0ncIPmQnXbhF+b3b1tM1ZxmA8yh6O26t5pEeQUOGQHBtFSdBAO0L2FwQPqIBx1NOSHn44e33AxttulsKaLlwShtJIQCoKPb3AAOIonuQutv0yLJqjMd1EthDa5Tjp2y+XFOJdLqFK+ZcrJ+ix1gjsTxmW46mHDREjT5LjaIy3XZulO2DpUNRRtgBbg4S4gEEix1Xw3tNvmWfwqjtz7PojsrdkpS+O6Q2lokAErskGwQXGggcYUzGaqmAzuMrstwttQ95BU2HHQBpa29bu4QDdK72A4RaxmVRFuTWaYtUeOw2uMZC2/D+GUFNtBwk6r8rsi55JICW0cHCpp1p6QpDLL81Mgrea8M0VFO02XEqDZJJ1IABuFEgKuPYqXFnKrTaIMd+JNEklJU5Jdhy3QhWoFopIB5BLh0XsQsi+PQqLTq+VUxuDEmNOvWbQw1HYfJ0EJVvXDYuElXlOu9gvtcTMolIq7UtyNGntyzILzSmiXg420dw7Zcd06za7gIVb0RfscJpkZDUt6RWYDbyA6PktxnHS4huwa21CxWhVzZNraLXNxymiynacyuTCHz44W9IYC3CHCG7LCguyx5AEk3v5SEEg2L3UpkmpTISXVolVjxS46/ml5vzhJJaUty6Bc+ThuxtzbEUSORGmttw3dqYy4yVtFiI64VR0F0bYSSpLiG1FQILg16zf6Bx2JHkmtTIbaaoTNpqGpzrkcyEpWF2ebeJACG/IfMSoAgnvbAY78eU60ptbC5a3Gm22WWwXnZHIVYrB0L3Ukl0grNyeRbCZ96XHisKj0iJNaiNGJDRUQwplqQtQCnNJJQhGsEB6xWNsn1UMQqJTJmmbNkt1KcyiFV2uW3C7tSvJpGtlDVi5rDbvfgOArt3JUlutoEerxqWwtmYUONJcQNkrLgbU1qQFFCx8y5UUkhKfXuilxoDkWTLgvOOXLbO2pktNFHA/ar+hJWpPJQlfH9i4w4vzE1BKqvtRNCG/mIRMKHGV329Lra7rPq4FE7fmA1C1sEKJbTPxVVRUuvQ62WnUDyF/eamluwI8UDcAaUq1E+UX4NsM8t+XGmuSkKcAkqQWExxvpbbcJa2WgNVjYJCE/5JHe5KwONGFBgU1UFP9QL0ZuM2XlS3z5gUqbAAIbs0sBf0Ag+owUunVpicvw7cZmO4gODZYaWHPlnU4ykgo8pt5SQW9wknjAy5nJsyNjMqiJh1WiwKXuwh4CQ8x4dllR2tL7QIA8hJsZLo+vQCU7fmSCa85CSy9WXYTDCnGHYOyXAosPfs21HyLWoEBdgSbo5FhYTr0OM7t1tZdDpYVJkANpUqPt38OlOsjuCLOXKTdaQTbBqEtyW0bNH0ySrZkynZjykk2Vt2UEga9aVpNzrct2vYlkqbZLcd+RIbjT2H5LqNtwvRfDIebKRu7gKwEIAbtqJAJbRdPcYcmVvRvCQW6MyYyJK2A61ELSC2XS7oDIIKFuIIX5gQr1sgYQIffeNPkPU+ezUZsVhl+Yhxt1Km2h5i2lZCCB8k3VYE9xg1FJUxJj5o/DqQ02trVC1vNXfcDjjSUKY1Kbc+XewIQEgA3NtAXZROsfQXFxtU9K5DJ2mnw2t1Q3WykFICQD8v7i/Yd7PVQlqjEJUulwkRifkIWtSy1wCXWg2EB0m3YoPLYsbXw2RYAlsMom0CZKcl77LDbUt6O7uWBAeunixBFkrbP5WvgLTcOPCbAyqJz0R0vyJkdpxx5LnmKdJFwsaNQWlQIskEehxmdur3bpwKZM2mvTihfhSC22pvbILtrWLqBcLbXdK0qQAlBPbBdJfmNpZgViI62wfOFx0Aoc1tWFlI7tmzYQLjta1iThtLLQjJk1SiSH2n2W30NtyrLShCxcpuoklSCkWV5/KSbdsHNRUU+ciGuDPVHD5EfwTZU4HEN3S2XG1Wb4LdzdBI7HjgpUrfRAqLLz9Jp78n8NjGS88t4broW4SBZbn1haueSBqK+BbDdSZUht5pqSjwrkx5hS0OBKEpPms4p0+QrUsC3JHB9sdnSowiKgKZqMCUsBttfh9oJYJAsp1YuQoKdBBuu2n8sMrTNVEXxT8vytSlsobiAO7C+VBwBs35I7i3Bt6YGVGwUgl1CDUV7s2rzQkAlmOwQGWnV6g4HdscAXuvT3N+CDw6R0CqsupqcCUJd20uymAVNOv3KrnyjQggix8x0BetNwBiMUgMqlllqKtyN4m7zD8gh1S7EkghJta5VclBFrdjh1kU5mQ+Isx5ZTII2N5TYTutAC6lHi5PPmAujk3OCghhKYyKnJpaPCwnA8Hdt5p5Sm1uEEJUNIupGnlIFkWseCMDsw6hxUqqNoYU0kNNyFOOArbIIQQPoR5k3/MnsLBvfp9RmoRIVVZKX0utbscw0ttttlI0qBbBXexAKSgE3/XB9QnQaU3Mbeqcaqz5LawS2wtWwAhLSG0jSEa7JLlweR398TzQ8kqhlt7zmkFtEVlCJ7saQocLANnHWyONYNiDex+98FvoU2iHPp7UiO5JSiOH3oxbaAQfNYG+4LqALih5b3IwRFhlxlufIRplzVBS0p0tGQsEqcU56tosUgJACFC/J9DGJ0Oqo2ER1y34kdphpJQXkstDyqAGltAQeO1zwj7nEVluqUuSGqQ/IfjVJtxMFDbUZLqXGCATyWratAvfzE3IIJBvwc5VlrRAqMqHHcjtoXICA44C2tACy4i9kLAJ4IFgQv74SuR6m7aNUIzbwjsoU0JAStUdwAbbikrJIAJ1WPAJ/I46aS4yTHrktyU+huQhtloNrBWVaihsjzi+u/oi/HtdL62KOX7qU1KPVGnFGMiqMzlJcWI7SYz7utz5m4EgngLDl/Uc37YDUKtHj7bkZ1QjMMiK+0X20suukXvdditZuQsXH08C2C5FBZkGDr8G2qWyVGGghS23A5pUS0EgthI5sTyRx64aMzuM1BtDMiiMTkxdEZthbOh5uxUArb8tlk3N7H74bKlccqlKIrMWOh6mx5UB93QVEyhyte5awcP0aDwPS1xbnCB5qW+pLb0gOqh6FPDxDjpbbHDiSoWN76bhIVqFr9jZsjJmyVSFtrdy+zHAaRGdbUtIXossOOIBAI58h7nv64V06JIbjIpyaO4gTS0p5EXba3AXLI1KKgsBQKrKtYDk98FHMHBN+YKRlishKERqkF7Ycc33HGHZTC2wdKgFfSFAgE6CbC/fDhQmKVEogg0xlsQg2xFTHlrLqmgTdLaQV+RAKrk2V7YVV1/8T3kSlNByIgxZbSnELaZ0At6FAfLXYW4v30ck90LE2JJmf8ArJmHEMQoAYhNoWl1CBYXUAkLuNHIbHsffC2S3yp9UmNDqLEN95gxi8tLUpFuQGzfZ0EoRYkX1C/f2thFJcrq5bUktxxGbetqkLWpSmwsHWpJCQhFjf8Af9exthHOlwzsyUQkSWn2XHgh13aVHsQlN2x/i9V0gX7A3txdFKqdWiwv63R2WXjodfccjh15TQc+SEFCgUA7ljdYuO9sAaIJYyKUma4unOomtuqLLyY4WoaOD3dQS4tsX7n1HYDDhEnESlqQ3TAwJZIaQDeK2RYEOLKiCP7RB7+gJwikVRDEZ5Ex9xbswLcWwWnUpZLQUEt7iAo7dh2SCSDyTzZZLfXVIAkzGo7rjcVDbOmUQlkLHyi447+zQPoBse3fEUVTdUPhQ6c5xSnMESpN5VzG9MXPNVhxBNZluhCtYlwkBLcjUCdbl0kXV5j6Z66o9MurmR3oi6lH6f06DUEyKWJlFmzPEOh2OXHJCWnk3YR4cKSU6yRqI79tsMVVlinRocbLaGixZ7ejhagX0Czqy4vSCC3ufLC7W0k82xjz4y85MsVRuJInQI7dEy6hiOhkgpekzHNx2wv9aY4bufufVWNMTi42Sva1VNM6jZPp9BlGlJQnwccQYjKQUjyADcIt2IubD3GI/kDr3Iy6khdAMhK1L1MrdbN0d7XNgCfz9cVOwmdX6JIqsNpbcNtRCnSkgC3YX9e+I9laPJqFaW3LkrLTSgFKIJt+g/jxjVbRZcyvNHV+dmhU3LNchRkQKjCkU/wrRCvD6xdshXoUr0G4xrz4DKe1TejFReejwo1UrmYJBcQ8kIkvIjsMlrUHdJOgqVo03vqNr3Ixgyg0n+iPUhFIrdJYq0TabdTc3Q8262SlQII7C5t7gj0x9OehFXTnHoXl6XGebk+CtSnS+yHrvxHXG7NOW3G1BlIcFzyDb2vRM7Kyy0QNzHMVZDr8aXVHlyk1SDId0NMPlkSFFwICvO0CLaRcXSCU2seTfCJ5uLEUptMinoiE6FSFIcvHQbWWbhII4tYH97gm2FEaLAXTGnpNULbYb8jrbJDgWdRHJIFyNNkqA+ojiwODpkByPJlTmakG4RcbWymVFFy3xpFlkgo5KrA69Fye2MatJuk8iHK8EtUkuUtsBckyHXQwWkBywFxbccKDe54sPdRx7L2XqVTZNTpdSqj63VoQ9LW624pOxp52ysWK1brYAJ5JAHYYAtrMUmYurz57DAkMrLZQw26rbuQiyQTcEXNlW45FrG6mg5kdhSPwhcNjMCpe2/5mlsOnRb5ZeQbW3Cm1rnt3J4sshmTpPbpUCtSkNT6zp23EtLRD222nUENnebRcuOfud/XkICSSU9SY00Jp1EjRIkuIhDblggkaFG4CdRQ45bsfuRyMEJkqNXgw4gQHFurdEhEdxlLmtxQeQpKFXFidvXZBsD5eblvmpmLTIh0iYJBEohtFPjfJI4BeUq5J48qLrBHtzhTtZBOwpYi0ZDFUnR2VR3Ww7HZp5EhxCwUgqOoa1jkm55I1jsADmo7U+ZKmTWmJSRd0MIcLXim2gQV6Qddj8xWnjsRYjgtTrVbhuyJMOpBpNO0BpiQzutOOEWd1ONqFi2beY3PI+gAEr6fWYdKnxH4tO8E7GQwp+NBZU23NmIB0oUkFJs2sk3J5KiTfsAfNTRFxy21LD1KlVd1fhnd55lhpITrXtpb7pNiPKTovc2t3IUtRKsKbT3pziGXo6I6m4kJSVFloOhRBKFXbbAAS4o3N0mwJtgmqR22aw1GmSIT6o6UbymozgVKdILrgU2SPIkuWAUbKIF7c3Y2UupvPTNkU+QiNuPl0efbIKnXrEjRYK/aE9xwMM2yJTw5upYQ3GcYnWvIdLKAW1OlwALUuwW23YqIvc+X0uMNldlyqrMbaY/B4T7De2JDjjrhkBYSQpvWrRawAva45Iv3w6OU2HU0rnUp1ZU+38uNtOJU6vaChpU2DunWU/Udsckn0w11RrMkuIuKiqtwpsyzTrLZbceYbQDdsugltF9IunkgW9sWqpPiZFUhUmWuWyX5UB4usvsuIU22wNKhuW4cCluDQHBfy/vWvhLmKtxZDqMvw4L89+nrceDrjyVbEYE7m8F/tLAqu5wQXCLqIsE1EmQ2KJApEiqxGC2H0n8OKyqY5dVi7tgkX1/U4CiwP2OCaRX401uSzBD6XXW21mVIZ1M05wNlzWHGirWskhsg+mi3OAxRdLW8N+hyXEsVENl5zwjidh1tnSdtlagu5QpocHQnvcG9kJpD7MBLi3m0IkyPDFjbckllop+a4SLbbl76G0i5Lar2AvhSqlw6bIMhTCGqi6yizqogkuPaNxIZCWxcG5KS4AXDdu9gMIqXVK1MkuzPxBEWTTiJMiOsLYeUtCwSop1cNpQdR1Gx7DucFzUzXBu6VvZYerlA/G5Mt2nzGHQ3EV+IlC345Sk2SqxDllqCli5KQni2o4MmR6hUHUqM+qeBeZW7JmMh1LskoIUdTrgGsltQBA2+wQAo9jZJjupRUKz4mZDin5bq2nC7LAcLyi5yAP2yTxyS5cA2JBjb2XGG46alAVGUuPHiIZkOl1TxJSLMpJN1k6Qt0o+gkAixATN3kyblUpjwTzMCqriusQ3Nt95oq1NeUNNqKLlC0gDWAsg7rZsMERlxIscQlyHanOW9oddQbeEcQQdaU6Attw7a7clBCeQCeXaTF/DqjFaqZnJaf20O2abjJbk7IKrC516RpT3tZvRfucR2lSokSsTfA1R+eZMZ1MaO0yWlP+YHbKW03cKiOfS1zccYZu6ATwnLbiGKlWExpfi4hXOdkOOrbG0443thIIShBIJcLtwbhINxe5EpiXS2mUFaHGWmly5D8OQHmtAbbCHHPLdCNxwAhRvYt2AtfA6JWP6SLqLwqtNEKBGaDsde46naWlwAqcItupDZKw5Yj1wcH6izEajUnMYRAJWhuVCkOPqeC7cbaAEaFDy7al+1wCBh1MuXdN0s7c9iRm6y5c3WXUS47pU62LhN1jlxtN78AXJAR9OFFajNw4zk1oNrlR1eUqi22ogU2EnasUBdnEmznpZBBIwfSafIcpsFqjGZ4kPWkyGkrSxYucIZuSXSkEEDygFVuThNSMtzWlzkVCq/jKZDTrxW2/qVHWEFSrFwkBbgAFm7i5IuFpxTzR5Juk7cBox3JrseTJDgeSxcNrdCyFoCfl6wUONjjzpsjm3BcaxIiwoEaJEyfU5IfVJ2x41huNHbJ8xaU580WCUlbh1r8ouqxFi0UyBmB+L4uKuSwEtOTZD71mgNLzRuLFwgFoJFuDckgab4X1epQEuGlzmGkqcbRGfNUccakJaDjSg244bIYQEJZCASASVk8ADDh2VQIimRoshSBRWhEqK0Nupe5WHA4XFOdwV3SCnWFLAcs36kHDbIl1t7Mq8nKochFGXCMpqXdKt6R5U2cjaQULcbJIBc1m5N7C+H6JVZYkKirq8yJFTHWw0+VPNJK3BuB4pQAW0JQbE3JukXVY8oIRrU3M7S8vT4YgOmPYMsuh1MkI29pxMh0m3KuTZGgKI74hGZK3u6pZU4rTcZcefFcaaL5dbZElxcdIXq1aHAEtoe29aiCTe6DzxhqlPpgsNsxZseC0+u0EyGwxqjoLqm3rWCCChJSBoKE3572w6yK3BjVYsOoE+o+JYWGnntaVOABxLJAKvlpX8vy6TdJ72AwljxKaXZspmG2or33DGkurKixy6DuW4csCjSfPzcemFc1Nsm5dcU6lrZnbiFAyVstOkyHtFtxxrbAsDzex5I1+fVwKU3UfxRqSyyRNkJYXqfg63WHFjU2psOaiizbfFxa6knyXGHR1FTqaUvUydAao7DZVuU526YqGuQtwWSW3FLKfmA3IJtY2upEtqa09Jiy3KaJ4Km3Aw8yl9hZJbZSblDjYLfJNuALqvgsU5pnhwI0FqROTS1tRYj7AjLdnuB/cBuoqSw8e2sk37lwEXtwc9GQ2tqou1JaETAsMqdnLLkTYQohKVLSlxbiVhKbg3JIQNa0ix9OotHXSHKUhEtbZW+8wFMvBKlsaXFFt1awHEApI0+zmv8AdsRsT8pM01E2IpubT/FoYL81xtLXjCtevxLgcN2w3uFshFy4RYheA095RNVUqiIkuF4mqNsQ3XFlvWpZLqCdJDatKjrILhIJ12UD20YPbQYrjdQbvAamx46RKVHDOpbYVZRYWs67hW1qIsbI7gWwN2QtcJbdHpTkpmfFQh5AcKy02FiyEuN8glAuFKJWQkEkm9jEIpsmCqRUwhplydHTOjMX3xEWhRYjs38jiNt0kq12um4Cyb4szW0S2y6pLS2lRZQqVYbYYStzb8RIO6w64ebKdaPNlqA1JFhcEBfnOHiYlFIlLYVeT4RSxtpQUhxzwzGtbTp3DfaJHzV+UNEkeg9TZjsmTGmvVfxTOtEmU13fKygbTfyLFayVJ8qQSPMEK4N0L8xh56O3S0qkNONOsq1uP7/zPMyCVjYEfW24Lkbil3KLjC5sqI7yIep1QcWHlpcf0CzLyYzWqI3bS66tvWbuJOy2LixCdZ4tguoVJqap96tlc1tt0tNRqe5xNj6LN/5a2gsBNxfSFLKDe111QqiYbgqUynNOwqVoabckxVKb87gJb8/CNotgjd4+Yr8y11bOlUkUiPR4cuXMiSHmFusqWXSXVuggpUE/L+ZYNJSbDtckghUEY7OprtWXDb3WExHQ27rp6FiOvbDakp0HQgFZDa1FwBJJuL8IW0uVFdbVVKcy6pyz/iGhplojtAD5ytbhu7oBHBPCXLkXFzZEqpUJCnhGp6C65IcdDqUbMVGgJLOoJGhbYOkFwhBvrBK08IJbtSky1S4selS3XXZBlytwyFElJbU0262sAfszcg6FEoDirEgxuZMk5RWZdQdiz2GqmoOGQ8y42W3Wmg0UqKnWbWQ3ZRChb6iL3Bu5Q6McwwU0+iRTGUWA0xJB3mUtrWrzNgOBC2y2nSfIpaUN6xyoYSZfqC5AlLditGJHZDW6+2UhJLinLgIVrJCASQbDm3NhdVTIkSoLjS0tLlGf4lqoOraLjin29JSAyFD5Vjz6oAWLrBTd/CFX4kywoKzTC69ZMdx0WSht0JlI5AVtuXA85BDiRwXDf2woXQoEJyU6iyoq4i0PS3Gy2S+EKSlwNoKnBy2kgEFfa5CCRhXBjMJh+JheCpxCnAp1p5RfecXfSh0XJDakE8pbsSEnixwpjMLYVJoC1yHjHQXlXID78dppJ3nCgquEkhKCb7traVk3EUTDJb/HUOR6QlpT8NlDUdyNFZdeLgJaS0kABdzyUXBtqc5vyGqNliVJiGZEFQedYp4e+ZGVt/vDzFBP+eQLpKSuxAIEngOITED6mGlSH4j7jDS3tuy3SQk2QSCShTuhtKEEkkAG2AQJM2BSH9Rjxlvl11G5K8zczTpDjgRbQNoON3vccg+oL5lE0tXhQloElClydxxSVkNKaKAdOoC4Wggi1hYLVz2sCnIslxUZchcItS3dxxx8iMwA4klwq4IFruGx1lXlWGwNOHtiNHqNRFFiUh2ChinttT5CUuKS0tDo3bkuh0G5F9u5b1f2Dhdta4SAtxC3QZDsl9MQga0BKSvvco0biQSLgALANgSHZkzVFanMfSiQ9LcFTlx3C8y2WyAG20FRLbZSW2mUlw6x6n0PkwKhyJ9SkRls0WqR5LEkx2005abp1lJ0J227kpAUbuee/kN7m0jag0mMzPEaTUZzMhBdZDZU4+6iwAecTZpgFveUlHkWLtjsbnDfUKY1TIia9l+m7u5G+S+3U20rd+YS2fVAJWLcoKwA2eCcHMo5GzlVRcpmBGmOiGVoT/X1l5x5BAcaBK0gIB86i2kJBKtC+UgiN152rN1SBAiQHZMhbTT4Q081IacQhCkqLaWwEBBWCA2VrKADdVxfD1RW669lx5x9qBDTuojQpCrF1jbCmztBfGgF0Em/1i9ivDxU460wIk9fg2LTm3GkJDb8g3bO9qS2LhsWV5bgci4HbEzJVH8s1aFWn56qDHMQyFNpitPxylQWhNlcBKR8vUfl3FjfQrjHUprEuVJZnU+Mt9ehDUkRlbIQdI0mwUsXsnQdYCri5514do9Bo58OqN4NMqOGqY/Lc21FKG0FStSbBtY0EnUQl24SgdjgDk15pjwFN8Q27EdEt+DEVuWQV2AZdWUhDZKgruVkq4BtcRp6qJLUYI8FPbdqM111hrbibzZ3SCmzYWEEgNWBNgew5PYYRSZ8LxIXSJ8ZlD7Tchp1bYU058rbUogkL8xDh5FlHsR3x6axIzBCp7chUBlMBpyM62uKgxmkCwa+YbG/Dqu5NtZWoG+HCpU+NOpO7EeLlSQkynX2IwDaowQLlp0tKQENeUOblxrUO/OJsFE01HMLNIlbLVBcmtx21+IaZgl2MmJ9SQ4XUq0Eam9fvtpF1ADHY8qMxFcka5JjONblnm0NupKLtEAuDXYhs+UoCLixsRfDo3FkNPia0mHVmnQ4G5CozS0PSFs6myp1lCbN69JPJBOsoUfNZOxTYzb0dt2cy5Fkx1x3WFStbrIACnHGw2SjhfYgggHhNwCuXBGiiLRWZTUdEWTGqcOLNZ8K4w9qjqNzqDalocKzqBSo2NljtYkDCtyHGkU+euow44TNX8tKFsvSVlseZAb0/LCSEjzefbSbcXOPP0tOxMqUF52AuYFwF7sbZa2taQAXQPOFLBA8h06UXcBGjAIMN+PBix3UOrkTGkykESVhDUdsJS64+gpLbrR8obTrtzYXBIxW3xJ+Sb8xMo8Y1u0pbjch1xpfgWkBLq7gOFO4o+dtY43OQABawBwXPy3Ggw6VMjLROYqMcALqLdnHXV90tBzS46BdsEnX57XuCCJQ1TqwxCD1RU9KizHY4edabaaLDbbQNkeiEAN+2v25AwwVXM8Bqn+JarbkVlDTv/q99bjvgmnHFFu7hTf9mSPMVjXqF0A4dDL3UVRaRAoOaGp8OvR/DyZbTLySdURZspvxCoyw2BpXcIdC/wDFKJIuQXZcmeiMjw0+I0uO0sRXUMgvgBwEJ3WwAhoNhVrtKRa4J4BCUTcsZrVpiLXSDIZcgssxWrvtH9oHEsITtHUv61N8HhFsK4UWmVBSYNUbht1SXKfhzY+4CWLt7bZDVgGUBx1wjm6l7nPlALbpU3CY8tZLMyX4t162t9t1IuCEhrdCzbVZz5lk21J4NhdU5I8RGf8ADumHEnyvDsh6QVJS42RtofP0W0uEbpHqSsm9sEtx6bGpQkqWwucX2I0dDb7Ij2QGw6FNXOiy7bblyCCshJsMGUJidVHWosaLsrDC3CtmOZDJ8m4ApzSTukFKbKsg+U3tyAominOTnorkTYcEVa34sF5Nw34xpxXNtIHNnEoVzfSogECwdGWJUqFLfhtRqohcZ2GHIKXikbh1W0o9blRbudGsFYsm4wvfy14OdHiSYZYZjkMbCpKTpLbgS42WvNuIC1NpOq6y4m17WuRJo70pZREhyHpaITalpkuJbW8XB/qhKUJBd7Om3dBSg6kgWxDsgU/SYlSqbqJVdkI8MpS2iWoOwrQAfmsgk8nSCSrXqRYE9iWB+U4/W3EyVw0TmkNvhQV4RDLiB8ttOtSluLUSDYaLDuAgXwtBnzI01aJEN5lhoPN2W2+EsLGm7ZAuNS1ctkA3TcAAYWPuxJyXa5UsxMS5TyXG3YbqLvP67JLrQbIWRcODSo2+WQOQBjNmWhwv4U1xag05NY1oakobQ/FCGmm3Gxa6nEFP+MI+YbJPck/msjx3pMnxkh4OR35xelSgsCziwlKVbZ4t2TyrnngDjDjQ1UuCPCTJIMdpm6Ie65H8QW3CW1tuL3G2ylFiVKIv5gcMSqhRFV4vR57DtPeUUpUHkqcdQ4tQbJcXcD0JAFjpXYXwl9bItbdPIbK6GxGiRN4B0usygzffW5yWXASPpKRoJCxe3qcG7by58tEKo0ulPtPuON+MKUuMueV3aIs4gly9+eBt29DhrlVFKX1NRpTE5lpkuh3aLjbT/YhNlcAGwCuLBQPPfDihdLSG5jrG8w5t7zazcOvr/fQSo+caRcqKRYgcmxwwSbOSSa3UmvHQZdNulpyTGZkuRHFIbcXZwoTtKA8177nKODxbkdWlmXKcdkynS0JNyURkKelBbShspUgGyOASfW9zrItghxuqssLWinsuCO2w0ZLBdSW2wvSGkpBNr3NtRt6jnCqU0glx6BSloXPQApqKSG2VkX0kPALGq4IBJBsr2vhygEVl2j1KotykOS6dLcDoLbK3AlaXF3Oq2m7lgRcA8cd7jBtQp1QLqm3kRGlzA23CiQn2XA0tAHyyGyAhu4ssq+ZcAA+Ugjm1KoOUZtCEsJPiG0OLTdW5o+kpJUbrubcC3N78izbIYcMhE9qMJMme8Q7KqLjatC9stlCddghYsbKuu9vywuZPkQ2X5cd9VCqtKhxoYb23m6aUMSUuI5B3HCStA/s6CPMQRhV4tkJiw2qE1LFgd0SArcjlwqbDiAkhvyJIGkWADd+LXBRUvEmIwlbSLtFpsxgrUu+keW4K/qVZItyr1woZg11bDECCy3JKHHGoyEpSUPFagVakrUCgEJJHa9jye2JmQzEJtZnrqlQMmJXHW0ulxLkaoRiqGQOznpdAN9AFlhYv6kYVIll6YuV+NxFlqStDSjKAaWtCPODupIPcEBVjyecFtuUaZMKnnpslyQyptzfX4caxYcJbJAA7AWtoHHOGWPlio1d9dPq6H4NFhocdEKLGcYad+qy3HiQtxChwjgEjm9+MGw5qXPiUgjpfffCpKt1tx7ddYcQHgyDwlweZN7WBAuEG3B5vg1otJ8Z4R4zi4SW30MbYdtyl0hxNyiw4HoRxe18Km4VLmxRXJam5hLjUWIhkttiPJ0kNh8rBWtIQOA0lXNgbcHDXUGZrEJfg5LcKXES44I93n32nL6dxJHkQUg9z3viXuoe7ugNUtcvw0GY8tiMtZcW2kKAcJ516m/mH7qV9/thZrdhUlqCZ7sj5i3HWdTzTRHG2rTcrPt9vXi+OMpq9ApiX38qOoDalkTp7zllEJsoaEK89geW1EIHHriPvZnzVMnyoVEo77EhbxDb6omh0XvylRPAKOSE3QQL+2IiWp6qlSrCfFyJKkOvSHPDlYiF8qXxy25oUggAWB4F088C+DE0uLuPNVUbMhTe4UKSRvrX5lI7ANFQT3tcE9sGM/hzdPcacmurkuLDrrkU+ZsI8xIuNsklQFu4vwe4wpiurrUmRGekrHiHkPhcp9sF8oDmrVtEIBsfXngW74ilkkbokKahSphFP3eG0x0nVotdIJJBWiwN1KXbyj74Nqz0punmmsMJpTEZoFlfyWnihZsEtC5Lmrv7+vNuXSW/uqFQhOQ4koR0MRWWB8p8cqDjiTbbI47IJ4vfDWXhGY2JRDrrLgdZloeQpaWyFJsFG/kUeQe4sDbEy6pM1tUraju0WeuHRMwhwraBZMNLpfElAUbWIK3L8pJshBP7vHKFmKzvCNOnrksRIoeaQlorWFlsa0KsOACok2BA5BPPDU9WMt0wop0Twvi5P9W2dDpaDuopSpRcHsNQ8/c+nAw/UiZBntrgIXMjTPDONvJjOB5959B7KSvk9yNsEE2F+2IillE/GaIk0xbwdZDW6zFKy3qWg2KlNG/pz2FwCL84+cnxVUIZsrVQzDUmVCfIzXPorVObUUrVIBccIte7bTbYbItcnUAScfQCOmnNuTTUaKKewtrckuvIKXnbEW8y/md73IJIIvY4h9e6D9OuoWZpmdK/TZMqozGjAZmpkqciM3aCdxDTl3ESHBttbvfzL4B7WROsbpHNuvm9VMq1BydUctUiMEQ6deSWWBdpv5beo8cW4HPr3w09NumVckoqVSlQ3IUbxyIyxIG2WwWi42pV7WBuBfnvjaeZfhD6hsVRdKjVrLe+GREXMbRJUhxh3gBzQCgkfSSDYLFuL4aJPQDqfmapV2M7UqVHOZRTKCZrshpIabaLZeqbDZPIs0LtkIWSq9uTjU14Kpy5VnXKsFacsVaYiMX8yVN1FKooQ43qlSNTKUttAm5sFOjyjgnnjH0S6O5HqXS7p1Q+nDK1zHqfGLU19ZaaaakrJdcbPmAuFqIClC5AGs3AsyZM6D5V6YxI8jJi6duTHBFMuW+ZL7+2tTYcLy06NCikkpZAAJ9e+J4iHPNMWxTYrdNdZAZdkPuOhvzuAkFu5WUDgH1GpJtjPKc+gVrE7lrVHXGmNNRmfGIU7IWoo0oKCkn0QsFYHm7AC9j3wS2pqrxnPwsuuUhoMIcYU27q8SVhWzuEhZBLYutvSCCQMBTWp8yjBM9dMlTEPFvxLjo2vDoA0gXKtbl7WTc2t73wqjqplKbaXVpT6lOq1xyuAt6QRoNyEoUNZvqNj6i5IHekBOSi2XkMKeRIpojuELcbCNbTSkEgOAnla0NkJ9bXte9jdRMqMBbUaqvTnJkaI0uM82sKb3/pUU3PJWk2+nSeSQQBfHEV/Mb7sUyXZEJEdreRGdcbO2i4DZebQSgApUPl3N7puMJ2WV1KPOKZhlGG6FqRIgHca3OAoqBGtACVBGkdyL34GCRlUXZVWpP4cuTMnSIUlyIhthEdbQI+SNMfWj9mALXtdw83IvhylrlTXFMR5LbwqMSPHLq1tsMhwtpUBs6U2O3cBzkEkW7HDFl7JlDekUlHgIzdYlp3YxeWDHlt6uEALAWgW5WSCFFJ+nDhHogEh16RBckPypfgYxYlANNLJIB1XAWLJUdJICbd+BiWUujJsmC62qlyIjYhOnfgQfIfDsLsbaRcXUvgnvwABfjCehy64knXGqMaI+G24sRAebTIX8sJSObtgoAUbrBSAAeeBGJ9UrTCUM06kTXXIV2xJcfSzc7l067BVyHCoE3JNhY8Yc3HpjnikPIqjykNEst7ZBuQkEqRcIAsFWSF8cX7kh8t0uaye6bA3zFqAkPNJkhyQ8sIbIDYc8ulCyAgKQkgd1qKTYeuFrTLcJM5FPmQtEdpcl1tUp6S2xAb7EueZbjjl7hN+LgWthmRAYeSuVVYMR9mOy23PTLsVOcHSbNun5RNgXL8kC5scLX6XLeq7DcGJKEunvlqTAZ1BZWUKU3pBcutA0thCR34JNhg5WtTJpnuUl6dAGYpAYZnFAZhK1OyVgkhRLbNjouCLXFvW4GFkeW1pmGrlyDGCxteHBsdY5BC7AXAuXD59Ha67AJplTq0ecqW3TYsibIQwzIlTH7OlZLrittsW2wlwWs2NBIABPnJdIC1txHouaabsTH1HaYS4HxIcDe2dy5C7gupI/IC55wjmqbIlqW2zKrAp34QphuC0ApToeEpdyosgWtrCEgr5PqLcm7bLjUKLDfcTPQyoiO9I/wAcltCOUnnltvXqBDYPKkDjvhYiTAEadMbVDDzkRUhwoaO9LcNhtk9kedtJKghACArvfCqmPJrLqVtyqfFefLi5MxUkMocuB5ypsEoKQVa72tq9TbFjfCimWVTxPjSamh6sx5qwEksy23XHGljU646lu69xRASGhYaybnggOEt6A9CkxpHhaklycJcjU42FOsNJUAtscOLAXtk88WPc9liGISXWIaZlMMjdLkWJARZrcW0rU4nWQtyw20kqXoJNx5iSEtJYcqM1pdVgOU1xBE0ppqXbym3Gg3sNnUEE2BKFJuVXcQCOV4IVSNqbT66FIXVnqg3EfeAYddsp5p0NgubaRY3UhVgTbhSQg9sNEai01qW8+KaiUYjTTDqJr5adjrsFKcUu6XFrKztoS2SW/b1w4z6vEqtQeMpqfKS1Kce8O8+YigXeNQTpIac8oskGx8gBNsJajVKuqOiZuSVqL621vocjtpDrnZx1QG46s6Qggkgg37kWrt3tVYk/i2prAg02AYEANNKdecZcbYba03LjafKV3NyCSASHDyAmzzHejQXJtJqsR1iDKDYS94JTAjPttgpMdsKuSoFXnI8o0Ei98DRmWm1GoP8A9H3qEp9sIbi1J4iU7ECWwSoQiQgLDgUELcWq1gbHsWOBRG4JgJjLk1h3SZTUtdQSgtOEq3CC22m61C6S2QeLrFiQBUx5dIRZMWAC906sMQVxpqoEN9T8Tw70uQlZSqISLuBVuHCUOJFj7BZ4OG/MFZZisldJoVaU9OlrisyBGQG39awlkqkWtqCATqPe2gBPFvSjWI62pdSXCVJqAcZEdwB10hxwBa3XHNVhoNhc9mz2BGDUMwX4SMuFuOqoMtKbaQzJW5ofDgSAhIJb3To+YlKDpCUD6ySLB3Ql+0hLrU+FmV9ymssePajf1dTUEvOJcaHBbIVZaElThsDouSQD6FChmk1pxmZJjrb2kNxY6WjGfLtwltCmylRCONwoSTxqBvjtWi+Ey3Gr1HrUB96PKXFeQ605vFBcUn5e6B4ZDY8wab7BS7nm2B0ylSoYaenV1cyIguzdqK6Et6NwMqeYSOHXD20qOskCw9MTM0oWciqVTqsttkRqhLqF21uSHAEJLSFtvOKQHuEB1VxawJ0OG3IKMBkyqdAgv1KGhpMqLIdk6pTTiEWW0lLodCCqx1gnWnTctrCzbjC6NSqixU4wdoIiqaacQneZadvIQtvkhxRDeoWRpIACzwSRYk5WqkxioPtSG/AWU0iTHihp8lBUdu7He55StklQIbA0g8iXzIZbJsgUvcf/ABOOpsNxG35RlKPyHxa6SRquFglOgizYHJJ+jCuTR261U5mw7ElSH3Sy5I21KYcbQlL2m6Lax3KyEIuTbVYgEulKqjAQuqQ3Ia2i+22+ytAdALbe4QlwKQzYuFLjZ5s4Edxh1ZMJ+U2moGZUVvpXGstptO5ZNmk20gG2nUAmwJ5vbQA2ZqOXqo/AcVQnS8JLpmOuN/MTaOXWFgkagSQg8303uFgE8Dla9TKXHLsmfUDT9ElpbUhtxb7bb6F/Mf1adCy6A6NKTwboJsL4SyMu5bzLIadk0cePab3GVpaeiNFALepxt51QWEJ1KUXLqBcAPY3w8iiUtmDModSi7aYy3DTEx5LyzCRy6Vublh8sJOu9+HCv97hRmtoimie1Meny31RJDqa1vuqUprSy6QkEDzqU2hDQCdZHPHYGyx2pzVxKg3U5doTsZ1tuW2Wy0ltwpKt1xNyGAEWUQpHAcF7HknlmY1TotUgstRp0gQjFaS3ZkuHuVjVr+WCBuHSSspwtl06Wau4ZFScEmjwS4JUYFt8F+7ZCiVWulaHU3VuL8tgLciN7pUQ6gplxh96RKdU6067HckQZJTvBts6w2AqwWpZA1ADtoFrAYYmfwSjuyJ4oTmpsSHnSIqnrtW2wQ3oTZtS1AFwjWq9wAb3MXNM0x0TGpr6m560SC6GndrWG0p9Ss8F0ggeY7liCBdfFrcBT4o0SRKXSW5HgnH2JNrNgjabaK9wIRY3sQSUOLRcEWSXfeU3RLrKFRZpr6nUU6OUS24UIIYuDwLNAg+H5Z/3xbdlnWkXUJmOQpSnI61kRHCrW7JbDgcDllLa2tTYIuB6osUC2sa8BrlVXGkJeehQ211CyXFrlIZPhxcX20NqXrOkKKQU8gG/ulTUobrxo9EMdbcwulxpotyX33S2XexKVlCrm1+QBYA6TYeLwoOF05O01ulSEU/8AHUMmQtDUuTHBeI1p5bUXDtgloFIc7gOLAJB4b2ZdOjveDfiVFKS1IW5EkhbKVSw1qDypbirOt7osQpFhYAfTYts5uq1qHGeokZyg7bIdMpVQ2lOt6dpsBy9vmONqvpt9IAIubqqjFqImRGoMOAJTHkqC5qi2+6+68LFRc9NGkbl7EEAGwN2a0uHeUFuSF8+nqlvKmrjymjClUyp7eh6O/uBKVIbWFDRpJb0puVEgE97lPUKjinakTHZK/OhuK6zvB1t0NWdUEKAbIWHB5gQo8gW5xIYlKqlPT46tpMeVILTodbaC1NFYHhgWUJ3GipepSALedslBNhdmqqyyioSKxMdfkTpLUSJHSlspY2rOBi4vY3trOhX27kCN+6im2tiW+0PwShtzZT8VD0qYs6Wm2i6ottKT5QCSLBIBNu9u+DIEFT7z0NplyXKjvRnZESS9cx1lClOlbgQkWDqtRutYIHe2FbWXq9Rpr7exJblRnLRykqjqSexdaUtY0WJKCE+ckbdiBworMwV2c3JnT0KgIVvuJlBKYpR8qwUkaVltwtqJcUL3Cri17uEC3VIPwx5x+NGy3Pbi1mQoNxkIhMtlIC1Olxwr1AOG5U2As/UOfTBaDCXTPnSVibDjrVJIsGGnNQbJU3rN9IF7WsHCPW+FCi6iPFZ2Haa8lDsV55Q+brCwQ4ohJDYupW3dvi7gvbt6REjUZ5TM1nxzFxFbkBbqX3VrFwhLZJLfdJHBQAe+hROIUS4BLpkGYawiW29JntiP4J6LIQ4YSeVNqClaUjQokW1AX0qHphujU2nZdQtJkilxy4tIYjlouqGrbVpSXlNoA1EhQQSARowGBEixmzJVVHKy3LUsB1B3EuAXs1uLWotiwUpd7kBwm6PIcNc112TV3ZNKjMIjwghtp+NHdDQRrCkrPlKw4pwJCAoAmx45OIl8W6VPzWqa+qUVRzBf1ty3g4WX9oLUnWpVz8zhTYAG62VAg2VcApTVNlPRaVBWUrdC4MpEYm10LvcnVZZIGnTco+WBc3uXFUmoxKtDpsBMhuW4XAlDLBQlTqwl1SFJdsgj1II7JIANjhEidammnsPOhDjniipl1bzzKAkgttg3O2EAJWLHR2Cj3wcyDkoitTWGGWo9Up7ja2lveHW6GEvvoKk7Z3EqaKwjU0dvzjUg+a3HHXKjKqS2UyFs09whrbJIaUQ5dlTilg7hBbbGpzQbazZIuCOkS6ozUHFxn0GJGLkWStiAtZSX0ghalBJb9L7ZueHBwDwCofh0JKZ8upiYpiWhuHJcD2pwrQpzdS2BoCBp4ARfc9U25iCd0zqaUQ4jSo7zUMh1UVc1ZYIbJcI2VpsgEEC5+lf9u4Ab6p4CfAeehuuhlcZtt+ZC206XydxLe5zueexKVCxA9DgUOU+xRGH6W7VkwagpBksux/KXQjZWATcF1rcCwnQBoNuCncxG5teoLNegQ3WamiRLcjyafJizRDjhCL7niS2LnU2NRUAQNIuXAVEp3syif1QIrUl2NPXUJ3j22FMtTpMfsCTZQ2iu7dtQ47ellXKur06dBzPDhVX8Kfi22gwXkWTuoUkoO2Qb33HLEC3r3x5cWNKjKMeQ/NgMQlogubMd9MQBPlKUlVkNpsQHEucrbuASdGFTzQplDis1aZTlqj8MoS2pRZcISojcKAsLuFO6neFIsgKKsOokaJZ2l1F+mxFoWPDyIqyXVS29TdnXVCyAhOkgJKwbm2qwJHammm1KAuJEntRXEzmyYzri0shBKrDgWQD+0PC/b1K8AnLkwI82fKnuPxpCCXExXnEuJDjbimy4xYENlGlW3bspu+v6MBpedKIinynfxWnvUfVG0Ow4K47LcZbIZS42NJIDgdBKboIWRzdWB3lEdFgUtkR5bNEFShojOtvvwmi088+62bOBtZG8G/laEg7fuCu5AagxXHhB8CY+iEJhaaaLrcezYT4lxLZJ2kfs+FbgupaAQADh1juQUVLwkaGAgRttCUNOKjqAUW9t5xq5DTyxZHnCE347Ya6eoxodQpsapOBRB24klpMhBQQ2HFpcJUEOtlISeAAFX1EmwXMplQmaTMiTXzU5jpl0Z0NslZQ14Vbt1HU2tu6NJdKioEIN27BY7cajU3eENbrr54EhDYWUjXdRSA2hSGxoT5A4As+pFkYKiJejKNNeqT7SH9zbjMLbSdgjUdv5usISAkjUbldwjtwrc8dBMugrrLaxCFm3IkgqdacILilvEC6z5dSLIRaxALRF8Ed1RBbm1WFOSIs6qMSJetqKyg7ZCCUthJW2FOCx27g6hxxexJbWjT2aI2xPqM2MhMCQ4mIJaJMZD6JSlhezwR5iVi4srXcpQCSPLmOGKlqAh+ZT6pCLjSagdRcC7hTjTgSNsuLsCFErUNVr3vghUhmoMQHqY8xHdjqW20leyqT4ZDgDay4bCTcqBcUqwBC0AkA4GUuTZlGYTuc6YhhbsZdVjyGW5LTa46m27eZQBbaCfIkJDgWLeQgk8Ym1Lpkun12FPapcuG22PEPLMl15sNrI3Fl27IcbsFeQawoKAIPfDQ/REJKpi1BKEP8Ah5D7YcaU5rJJ9Btgo7EglRC+3cq4lRebcpcFD8JlmOd5p52IXQwhyylA2N0IbBUTcWslRP7mGS/aRYocShzg7l6jm0d1EqDU0yo+yF6OyhqKEOA3VpTwk7fokgqHUpq1LgwkvVGNChyy40FtOlkC4eudDm0Vth5WsKQQQ4hYTckE9VKlLWhmmMMPPVNl8qU62m8do6QpvbGouIsk2Km9ZsCHBc4WMVmdApy35i6hvunw78kNOKTKLG3tu6VpVrXck2UbFBSO4wRZRJanS1yak45Hdkyw085uM6dLCZNypI2FjzgrICFFoXXZAB1cvRzTXhlaFlWLnGoRokSauChMUtsF5oNJLad9nStWoqIFhe6bLBuSGGrpmTJDlRjssPokqEpSUBzcjNBStK9TiPIhPlUD2G437E4bmYkisy4tNodBRIdjtBaGm3gA+tCtwvJcFmw8lCQPMhBF/Ibq4RwvoFbzTqmYVJcr0x01JUZltl9LKGUvBYZ21OPApU2s3Ve9ieQL/vgtl+G8XKOmq1Nza2Gm1OsrVLbcW2AXApw3C7lxIbUCAsd7WwB2DW13emQWkocK5sWOFhKm/pfdKbEuaC24LEgkoHPLeErFefqjTNPXLkLTHacEQypKGktNgl90KvdaxoPy0qWb8WsVXFmyV2ydEohCnOLep0ht8tIcZfB06kLuGlFJA4K1E3JBPpa18GPtNUQPrqEiaKjHReEtpovMsxgTZO24oAaiCkWNgb2v3wkD9bipkU16px4cd9bDu7JZca23EIHzCyAVgW4bF9BClHuRhujU+OX49MeqdORAkO6SxHcVpIQAUpIdR2K1KTwsoF+/AJyW1Tea74ZDmmBAqulmM1uHdQvbkEkXZ+WDzxe6T6Dt6uMJg0yiLREjQ9Ue7Ty2GU/KCFKcZQ244NbZCCpeknkD1INzWGaPBcVvUBMXcabMeAQWmmGxqFrAq8msK8yfmG4JPrgcRuE1DMwpQouKAYbDiHkJ+UQ4oqB/agbZ83YJNxzfB3T7JQpMSJEjTJLNT8NLXYOvNMyEtuIsoIubIDugOG5v6GysJiqaJTiqRAmworhv+ISPkKdcQhzVthu/kuebEghINwexECA7UGXKmiM0Vtl2Q80yQ0lsrLZLhcWSgmxFiq3CrdzcHuwF1xt78Khxg4gbITHWuSXRYEB4tA3cF3CVOOW8ugDuMFIkb1QojTUeJLjDxTcZtKjLbeTpasNNrC6AEK45sT9r4CxX2qvLkPQnW3ZAacWp1iMElzRpb20hBCFngXUBc+pNrYOcodOFJTWHHFySkoZeS/GeSoaAVONp8oQDdTYAUew5IJwFuU0+uGunwKdFcgL0hU1mMmMzYXSQCCC4kJJCg2F61EdjbB7ttFAi6c1KU7+IzW4yWY2ttDceI0FpuLqW4QkglNvssEADRxhXVGqW9UFyaY5Jqy2w7FpiHGDHccIAcU42ASNBAVoCiDYkHkY4p6pVmXKmNKfkuTGt559jQ2HkIJI1BCeLi/A4tbtzhfR46KnAiRqcN2N4VbkxElB0xEIJIJuCgWQogH6/MsC174XOrO8llNfpM6RHgVKoyYZRFYSVOFIjtgAXF/rd720jQfL9XbBEIVKnUwLiyxXWj814lSHk2R5UoCUWWhABvp9T+9xhemuuLgwozM2VEYbmbS2pEBjYbAbCSHEC7p5CbX8lgeTcjDc26uJRQxLrMx2O23IW24Q2opbAPdlztqKuAn6fUXIGJlQDeqYW6pPenxHGYDsSNt7hTLhBlRbQrunQQ5YL/tAXuCB64cG5DTsh7acXDcjxj8pxlBS4SgpLTiio+QoPPkJItxyDg2UUIjzYDyDTm2GgiSmSWW3ytAUkFoEFy233uVG4PINhhPFpTzLUqmwGZC0+Ks402CXXXNsrSUuXUHCCSSnji5NgMM4WQLVyn1CsKDkqk1CPEZWCygx5RiNsr2zubdgBdKCQVJHHIuOb98atxbaokR0LW34KMY0kqO42BwXLX7jVpNjY374dm3IUKG3B0SW34wWGYj2iQ0+0STtOIQLhZWG7Npve/Jwy0X8WnJenz1mFERuf1ZplPjnLthYuAU6Aq172KAAe9jhLoE5jZL36xNkvU5EaDVJFUcYWHZUlzdTtXJCB5tYN03I0dzze2HVESlUtT0idQFsKdebLjkN1CQ26GhqWskKvfynT3SSQMN9MqbS2ocOe4061r3dlRU8C4QPm+gCE8WuBc++CXaXTautkzKk/IYUyXruutsstAkg3AuQs28nBIJFgcKxgZpe6ulkz8kjfVmOdJS1JpUSJTZEZcyK0hxxzaAVZK3AhJJAFyeB2Pe5w5UqAtltqY85CdnsJ8/hXXFKc8/7QhYNyO9legPAthVQojNUktwaRLDjbqZG4lxxplku3sj0K1hSDbVo49r4T06kNvTWmoS6g6GF8BTJDC2xcOt7xsSCD6WQQBwbjFir3XoaEobWp2a2hU1zZFitVivkOdgCOCOPXi3rhBOqy6vMK0zoTsjgss+Fskr8tw43ZVzcnggiwt7WUVliOqO1BeqUKAY7SIgUdUl2za1fSlZBuQP8AFkA8XsMInsuoU08434Z5pvbksyWoTiJL11EK1EqNrDtwL/awxEiBNTJS4/OeqdPcbiPIiutyQUKSs8hIT5r+fct6i1z7YcqM1ImPP7kmJFjsAEvLc2ubCzgACneRqPc9vQ844zEpEPw0+pUWLUGm54EXxagtxLV0hTZGq6O47CwI7gA45WVGTDXJjVuTS4LYBixXpbRu2skmwQlPk0aDzxxbm+IHX1S5UBEyHR32JBhCWlZQhlyK+sKIRyppNxwg3PcX5+2FlIdbhpCJDEQTHXCl1cIIX4UuAqSXHAChxdk2LfywOLE3w50iQqKyytl4zHX4q2QHEhRf4IuG1ghA57E9/QeqN54rhNs1KG+7D3TpYdSHmtdiW2vQIcVa+rk2wMyOjVyOhyPMVKZi0xLsphxkuIdWlohBKm0qKLoCCb8a1LukX7XwdTZLL8KPT1x4cdS1uOOsyIzJb21jlbjwuCU8gEkEBw2OG+rU9ZU5FrMARozt3Y0diUFBjdIuogAEmxIIPPA78YQ06dUGC/RRIjNOtvBpXh9ply1/KvkgFdkkIPNr2IuL4jUSpE87FmUdbVBgNKJdiRG4zhstXzTquoIsvkp8pP7OwA5Jw2qdpSKa+3WZ7s9t+U7JU+8uMpZIGktkA6Nu5SopTcXNtV7AIWXItHamMRoa46XWXA3BHh1KK1oPzHXnb6HCUgjSADyBa5JBlZmmeCL9UhrEdbQeIhynXJJ5H0qKChAJIK+CCBz24sccvdQTtHnIzawigkRlUphlx1SXkLitlCBb5aRYoNgQhPPqbdhhoptLo4aVUqVMQ07ERtLVIDpkqWRYIZcsQtuwUSQji3qTbEkWWBU0Ql1ByWmI4JrMyZENlLDVrNkkL2ypX1JAJCQbWw6NsxqlM0ycsVCqSI0R1suPOFSmrIu2LX8gHFk67K5vbtiafaVabo7MKJA/Fo1OQ/A8Iw4luQ80Apd7OXIB1rFgPl2PCydAtdG+7Oh1BacvyKpIbmFDvj3kh3gIPKVajbUSpPIBsAOPQqo+IKDUnqYiS1HbbsmQyhh4uAptdQ7uayAATc89rCzVWKVSU1yQ9DnTX1TQ2yzLjIcYbaIJU6Uq4LhC7jVxcnjtcq2xTudlS2oNvVCXU4MPL1MXCbaWZpfktyFFA4P7xIcToURpta/a1hhxnx3vFtJqryQ7Na3JjEebsvtO6QktOJukAbYA8oFwRyecM9WXRJ0OlxFqfk1KS8WUKbcKS9tpO0opJCATcAulZWpwkW4uHJuoQY7zJq8oON7YbXICUvyEi7hK3G128/mICuR8vubgYLXDMp9lObsLK8FEWbPManxZFRRObh2ceeS2LlougKI0qRpI450k3HfDC4/uOonPPuyJjQXOdS3tPjaI1AhSPI2QgE7dr2PN+TgcadAqDJhvQ30txmHZIW0HEbjdjrU6o30cm1wDcDtzwbAcdYpUh+BXll2MtZpcp26C5rNyjQBrbGjyhPJUTyUoxZdIlb1Ohh6pUqelDcqZFdXEbcca2mn3NpTylJcNhZBADhXf2tguDNebVK1VGoyI0YIc2m3WWQrjS5YgKuhIAAc5uVck3vhXAkoXWVaG6al+bAcixIyiQmNIICuzqLrRYGytfdVgTYYKcfXAp7SKoYCprTZekNuNFDMU2PlJQVHgIBJ9NR4JBxW52ZP4UGeESH49HRWEM1SW028AtpLDDYIISVCwLgCA59JQO5NuxT0JE5NIjx340lcWWPHvLlOJAkLJS40TcDWnWmy0iw8wsfrOEEeOtUaao1QRguMhlx1lzQpazpJW02Wi4dS7W5HN7kAmyhD6pUR2gUqYtSnQULSpr5QuNt0uqWooWjbbDlysBJIPe9ixHmhUZp1MxmRGoIlPHbXLitB19xlpZBbJYW2B9biVWvayRfgYMeYeqsJVVhpkNQ21E+GYZtsrW5ZLnbRdT2kkEKvcHVYcR6p1BcLYVmFSFSJji3XB4pUuQ8ORZRRZsDy3uDYhwewxK6U6mvUWE49W4b8dholdPkMGW5d2xsNDgCLFNwSiyLfvX4c+FBuyWRIdIoDiaW1nIl1QbZf8TBH9aAWNQavwj9qW9tWgm1yTYYaZbcuExPo/hYcJLQuNt6SpgR2jpcdbbkHyc91XIO3otzyqiMU9NUerP4FMqW9KdcRFfhJdStxdwkotYNodJUSUgFvTcC+G+O8tmmoosZTE+ctxhxT0Zxp8xIgsoR1BwFwIC1FwixJ03BwGlANRi4XiwmO9Sm4bpkLhhUiTZzQFJcuHNR0Np3WQhJsOe5QTZMuFVI8RTLFOUiU5JcX4d5LambgtpO6NsIDgKW7crA44Atde6lDMJUl6PEaXGfmIkxw44680EBO4t4hs7gJcCllspuhtpAtpIDSmoyKI/Dqf4fG8XElFxa1tLbDzDiG9gMq+tRbW26Te2k7az6DEcVC1NdGzO/UYs2nVXLDbLjkl196YmKApawdKkuuL7G4vYAHlXpa0hqFaqSHGvAOx5FfW2hyJ/VwnYbDVkrcbtZeokkE3XrsBYG+GGRSqeYVNXTahVHnZg1NKaJS2y6gj5zqbXFkA3WQLgpPovCqfHBy+g1KYsF+oFpKJhbKXG2xqIUkXc1uAAlRI5SeCOcK0Zt0XHLskNMM2p0lMhD+2lkaJLSHh5JK73vdN9vhR8ptr+s4ViUzPUt+VAQ1CYQRqjp8S8750+d4Fd0BxZTd0drcJPOEpnZjkzH1JjRkQ47yG2m1sh1rWCoNttOIFkIGvi5tZQJuSMHy3m6nU6c3IvUXG5Wqc8xMPzfOS4osICQ/ZagCdZPmDYvqBEd4lM2YZkurFQzZDmyaJIYk05ht9EaRNbBUwogJsSCkuEEhsblwQSOebYT02SiLLlUmo1GXHECHIE12CyS44tvugbJsg8Np3L61E9rjBUmn1iUHnoc+OWY6SEMPOFpxRQHFE7VuwbSFEqN7+oBGFlJqMuhtuVHL+mK9JRrWGbbydzVpdvxtoPmbtda+55vfCNNtEw7yKersOdU33qrtUsvsuBmCw0GlRQBqEUuFSkLcU4kBZsSAysG174a41RceMqvO1gy0tMtSlwF3YpzKymyTq08/S3bSOATye4e2365T6YzC8HOjykuR2ExlLQp3wbjjq1Ns7oK0F10DcWOBYD3JQMU5FNKalLqMtS4DThW2q7YT8wBQJupbi08gEXvZwWte87ylwm2k5gkPIieOpofaejLeW2pvUqfcatttwqCyARpLnI7oB83LnSIr0mooZ/ErS4xkSmm5brgedWvbDSW1jhBOogItY7bnNhbCypxKs0HlrW4lyE5srbShpYeJaaaS8pAsNbiFNhBbcN+TpGDZ5pVCeMIVh2Q81U3GnnFEFbC734JHkbO2LWAJLZN0gXUdku6BGU1WIMiMqp1DwciU+IwUtfyy3pSysggXaCHrgA8gEBJ4GDlx5y3arLqMnxbtSWIyVmc2WnHFuJuhIb1FC1HcuVEo54BIFyN2fRoCJ095oVKXJ24bbURUhQbRq2dRKbWI3BYjXrAINucG0xupPMfLgS4jrTq3Gipqz0dx1xRJsQQ2PMdZCCuykWsjBzZUQio9PYpiWI9QZkpfrkWwUkhJbaWkbe4Cq9yGuSUC2o8DvhoZhZqgRIj1IRUJJ8SLMNujwjUvRtpLDRvcpJcB9AXLcFJIcnWaKxJdqrLbO6xIceQUHUGbLVpukcOLKrgqK+EXRY2sE+UV1mHWX11aM6VNONNz2J1P2mg6FeZh9xZBOlAcUFJuexNzfDt2UCPlUdbjq4c6kx20krZZfWlB0hDibtlwatBOrtcDXZHHJwpay3SqM48moT5JecbcaZaTIaUBGL2024pxtQsPq7oAIUnkgWwmqbUHMT0Ci0JhdRpslkRIsZtSXFpjtlsuWNwsBQKbqsARYC5bUcJMu5NoSK1rbozsCPZxmSxJRrW0S86klwIO4fIAkp0WPBQb96u8pontqJU3Z7KaazHLktbrTkpmAXXWwRpdCtAssbmlSwkJ1FRX2CsN1Ro02TLTLZqrDZjEeMnPGMlSWEWbCyLnbLiwCLEL0KFuL4NzbTZMCPOTRZsJlxx2PIdZU44C82vcDanNx1W3p27Cw45BPAwUxmZEjLsbK1XrNOjJTFbQ6pzdUHVuEkam7JQbLJ1k3N0mxPq7f5aiE86/sR226mxNktuLbeC2w0+dC0pCLi/HlBPqUJF7k4QS5sudUayiPLgP1RdQbntvLQFPR1hClFq5FydCg4EpJQdJX5LHDhGak1ByPGqrcyQqWVrmlVnQ4628kPF5Swpe4bO2SNaGz8y17YOdXSosaVHlyS28qS+3J2jqSkIatoL5UN91SHAlu5sEXBsvjDd5DKEkpk6Sgw3ZFNfSHnlRY6fDuKJAZ3Q8BfRYoACN0qHmsPqIBMKoOMs1RbVPjRHEOOxpSGUvKTtgqsUOiy22xutpCLfW2LqueWxU59iPCy6HmmlsPGSp1MgEq1vBRSVDVztkXdukaALp4AwpbVVKe9PltLj3kVIsNU4OocDjgWUpCWvKFoAcbUFKKdIbUs3HODlCF8qWoZoNdabqVBlt1aWEAuokxHFLhnbUVNshxQABIaKHCvXcC2gDDXCOqguSY7jcKbHV4ITHnG35BW62ClepYIQRZxsuc+gAGnldNkw7BcxukNSoawy4hqctx9og2U0HnDtu3WdsqcKuFXBsLkzwgmNynUQ5kx+NT9bW2EoabcRqBCSslAs1q0XRxYDTcC7J0mh06k09oR4lYkOSFul6WucW3nGGEX2yWXBrtqbuhvzc3I130Y4iE5GVEg/MuIodlxkSkOJ0NoSS42ArcQjnUAnzjcN9AugHOy5GW2Y0mqx34JmLjhp4vB2VLjoJddZQSk6LOFfzHP8Yo8jULcanM1ZwmAytZqbawgiOqI625ru5qQLtouCb2WblQ9Cs4CXKEkpTbzmW3pkZceJIpoWoyGnxvJQW1OWToSndKrON6j5wghAvwsgpUaM3etJhSHp0hV2mA88+GyttTgCSVgk9yg2udJPJuQtepTK8tw/HZeiMNIY0CI4zth0tlvknUCCFoN0gpWSAblAILZGkRI9Elw6bKcbhOuuCMmNG1vFu5USXCPOAVbfBHkvbgG41cpoU8JDS48qYaxIgzJEZuWy5U5MpUdlvzApSdG45ezhKbAWVo9Lnq5jk6O68uqpaYqKWEPvrhBxLiwVPBpxJCShw6gPNyAVEn0KpqWy7T4b7EKn02HDg7rsR9pzW+6hI3FK8+soUC0pCtBRfVYiyTgJgR31llUliPJjy13aU+0647HdZBa3XFq1vrKwqxbK7Akm+peFb3u85ByPdr1MhxGJLDMlS4SFtCa1J0Jl6C3pD4JJAKbKXcIsTtm6ATiOPoRVqkhpFYpy4E2I3LDC1mM1tGygUuXsNh0KJTe/Atq1Yc61UnIdMjvVjMUZKYqxFYjKcLcZ1BaJDTZWQFlRQ4oK4GsqB+rlHRn2ZURutmiUt8U6Y65JTFpiXGAvQrjwptoZ8re5tDQVi5TwDixRqWRVP1aIY7j0BUoqccfkh4CI2tprS7uKLljp0pF+ToJsOV4ImGdmGOxA8EwiWw+4jXCccbfeJbupXykaFhyzI2rc2RfjjDkI5ZluLzIw4WZDjhaQ+yjbluLRZ0h9sghxoKNiBoseQTY49UKhTmqRH1JmQpbclDDsxbhacU4gJ+am7nLjRSSGwU2Fr3IF4p9pNb0J996fozAJkgh26pcNvwb7cflIUG3dtx03c0NkIAKu/deOVyXILkqUiAg1OOptrZeC48wNNlrS8kkFtwBCQ3p9EBsKF/mHr9Xg1xMmkxswtTWqQ4JE6MzpkylNlxKXVlQV59NwD5FEFR9RwXLTHqtRkqkSJstIbaW89GiWs35Q0SFueSRfdC9SyBZZN9N8TKo5PdKq8p6C3UqKzNhJnvGBAQGY7DinVt6nI7e2LN+RLRLXF9zgnnBUl7bpkp6jvUsVBiFIj7jDrbwabQ9dsoKFEocBUeeCQpBH7NWOxYMsQH/AbSoMlt1+Cw86JZlObgaacCiko1gEkltfm2xa6wcJ6n/RZCIjK4NZdahtiTPlre221Nr+Y8XGWzc2N9FyLAXAvzhXeFRqKqU/MNFYUimNx1yl7LDSHqk0nabU8XHQ4hjTdpRdB1OOIIKu9rgdmNPOuPT4CJEgJkuOruQ09KR6htJ29a9Drg0hHBAWLgglYzH8DXwtuZGhGTHW4w/CcWWCACAVJCS4HLEqCnOPKSR2s2OUyoS2GVJEiTMaaWJGyGS60FkODzhJ86tLbulKxqCdY4HAy7FRyDQtSoyIiVnxtLG9D0lt4uBwBRe1N6dYDZaGlKzcm6E35w7PNLfadZQ+6h6Iy1GEFmIklm5bUHGi2bkklQuCSebk4itbyZVK1ITPjePk7EVDpFQkhiEI7mpTJZU2FFayXQLAIvt6OyUHD1TI7lfpFLmUynhl5x2xgOgMvxSs6nn9N9wNNaVBwuLG3qA+5YpUUmP+L79SkxpdSizQ4ApLxaeNyr5SQ0DuWCUkbiEFKE6DfuOMxIC3HKb4+qy2Iw222W5im2FMAk7yGwkrADTbaj5DwoLRhzy5FahLCpjy6aKW6hK3ZDLjrLSHC5ZoFtSewSFfWBoOi4sQW+qVkuqqL2sw21xEL0rkhvVGDQKiXL3NrJ0AguEWtxbBUSh6JM8ImTBaZfhradmuCE0463DWgDeJaF7/4uyisi99YBCzhuNYRDLrzrMdtsaHG4shtLyXC2kK3U2CQgkH6k6D5bI5vhyXOl1VUdtEazbUlbirQZDukBzbutkqCEIA3TYaTocFhdXIZFHW2WYkmtIedhyRGVKYe2TcFQte3yrbQIGtsi4Qb2OIog0aoqE+MlTkvxRltMyXx5glq7YIeS2rWOApSzbjv9d8I6J+LLpr4prTj0+nMyXXWQ200vQ46lKnkrWLpAIaSGw2or1AmwBwrXFo6VxHpkmWgyIDrbqxFCVNPhwpS0pwN2daUfKu9i4b3JsMGIplQdfaQy7PZ0AyhFDZJLbpCkhsgbgIsklRKEA/vDgYl7I502yZCxUqaxDpD1RclpjiCVTWojcdbDQI3W39RQ2q6dDgN1bZBvbRhK5QHodOYhVGoSVOtqD0l1TbwZWtClXkBu4LpKxpKTeweSV8XOFYkRXVSp9SpdDYFTkrZ258pakl9wnlOtQ0FpCrcFBKz2tbHW4ExxoOR2dlqA4X48liWDurQ8kJKQSUaLgq1KuvgHkKIEd1QR8OZMqlTqUxTKDMpjZNQXF2f6qH9IWtwIJAQALBKQSLC1xcgtxtEhySqVTYbDO54SO06889HSi7SitxwqbJWkqFtSDwnyWBuWRqZUZ8R+VMgTUq8WuR4Jxt2NLb1kgkElKFuBem9lpQoq0I7jDkmmtyoiWF0V1pKIjDLjwZkbVnT/AFkvBYOtYRwRb5ZT6hQOITmUVhyP/aZK/wB4h/8AYmsdzj/sKoX+9TP7049j2M6tTFQv9jlE/wB9X/1beIlkr/2lUf8A4/j/AN6cex7ChONlJGv9R1D/AIf/AKFYiHUv/YjD/wCL0/8AQex7HsTkl+yrOgf7LKX/AMArH/VjETmf7HK7/wAHP/VnHsewWbqN3TTmn/WLLf8AwWJ/1KcWTD/2IZ1/4fF/7Rj2PYU7KwqMyP8AWWpf8HX/AHpw8Zy7Rv8Aimmf9a3j2PYdm6iZal/qzMX/AAxf/WJwXSP9Vw/+EzP+g5j2PYd2yieGP9QJ/Nr+9OHFn/Wimf8ADKj/ANUnHsexU1Dmo7B/1JU/9y3/AHN4eM3/AOstJ/41X/2XHsewBuk5odH+mr/8Tzf+rThJT/8A2fo/4bF/6nHsewSi3mnaZ/q6r/7p7/oLwz5O7H/dMf8AV49j2Hd4UHckU19Dv5j/AKSsFZ6/9oEn/fY3/Y28ex7CNSqXVv8A1mR/xSP+rbwn6v8A+x7Lf++Rv+zPY9j2GUPiVc5Z/wBhb/8AvY/6DeF1E/1aj/hT3/YTj2PYZnhSu3Sei/tah/vLf/VYmFM/14pP+9wP+rkY9j2Gd4k32Ulzx/7QG/8AhX/3crC6d/rDnH84f/XN49j2KjuUG7KOK/2eU/8A4xb/AO0qxJszf6mk/k3/ANSnHseww8KDky5Z/wBbKJ/wdr+5/HaP/s0b/wB5T/0V49j2AN1aNk6Uf/2pUn/jVf8AoxBczfS3/wALV/1Qx7HsXHZVNTrF/wDaDUf98H/UjD1mz/Xipf7qZ/2ZWPY9itu6jkfWPorf++xv/wCLiLRP9TSP+Ctf9aMex7FjdkClWff/AGiM/wDBWf8Aqhh2ndqL/wATN/8AUDHsewysCdaZ/sdk/wDCD/1buIHkH/XzMH/BVf3px7HsVO3S81YFE/2Ct/7j/wC5m8Ryu/7IVfk5/wBdj2PYQot3TzUf9kS/97mf9a1iHZm7Rf8AfpP/AFrOPY9gjdR+ylCf/Zkx/wAb/wChGGvI/wDqKJ/vrn/bRj2PYZ+5QdshdRv2sX/j6T/0sBzZ/q2N/wDi4v8A6a8ex7Cs3R+ynWb/AKwVn/cx/wDonEZZ/wBUUz/hcf8A7Ycex7FpQZsreyJ/rdG/42jf9U9iMTP/AG9J/wB6pH/UN49j2KgiN0lpf7Gd/wDjNF/7RgEH/W93/fP+/HsewrvCmb40XQv9eaF+Tv8A1TmGjqD/AKjR/vdO/wCzzsex7FkaBXcv/t8wf8Tw/wC9jDJ1G/8AZ7n/AP4pj/8ATcx7HsF+yI3R7f8ArBQP+Lv/ALqcwXO/2VV//iZH/Vw8ex7CnZIFGJH+yCB/u2P/AN4DC7PH7Wd/uKr/ANYvHsexY7xBFTaZ/rb08/32b/2pvEY6g/6srH/B53/Sdx7HsQ+IKtJqJ/sjqH/B5/8A1ycSBP8A7W5H+/8A/wBwJx7HsOEW7pmzT/7F6P8A8Pwuz/3qn++xP/3dj2PYCZqSUH/YTUf+Ml/9U1h86nf6oyn/AMCpP98fHsewsexQciI3+ql/8cRv+1tYbaX/AK6L/wByv+849j2C/ko1THMX/soov++/6U4p7M3+xjNv+8H/AKYx7HsOErvEjswf7Gqn/vLX/ZTieI/1kR/xtWf+ixj2PYQKKJV3/Z9SP94k/wBxwqgf+0Jz/hEn/wC3x7HsWN5ovU7X+xkf8f8A/wB0LxG0f63Uz/jqN/2mXj2PYQ7IIjOn7TNv/wCM3/3Q/iR0765f/HTGPY9gclEgyD/7SWv+Lpv/AGJvEqnf7Isj/wDGVf8A+0DHsexComnPP/s5pv5O/wDZ1Yac1f6/1P8A4c3/ANmYx7HsK1Fycss/+2evf8Ys/wDWtYKP7ai/8LY/7E3j2PYHJKN010P/AFBN/wB4m/3vYaesn7b/APsWX/8AojHseww2RKfXP9kEj/iOZ/ecVbF/2T1z/hiP+yY9j2CVFZUr/Y7B/wCAK/65nD/A/wBfUfnV/wDs0PHsewjETuF//9k=" } ] } @@ -120,7 +68,7 @@ { "language": "javascript", "source": [ - "const tensorData = new Float32Array(image.data);\nconst tensor = new ov.Tensor(ov.element.f32, Int32Array.from([1, 224, 224, 3]), tensorData);\n" + "const [N, inputHeight, inputWidth] = inputLayer.shape;\nconst resizedImg = img.resize(inputWidth, inputHeight);\nconst tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, new Float32Array(resizedImg.rgb));\n" ], "outputs": [] }, @@ -134,14 +82,21 @@ { "language": "javascript", "source": [ - "const inferRequest = compiledModel.createInferRequest();\ninferRequest.setInputTensor(tensor);\ninferRequest.infer();\n\nconst resultInfer = inferRequest.getTensor(outputLayer);\nconst resultIndex = resultInfer.data.indexOf(Math.max(...resultInfer.data));\n" + "const inferRequest = compiledModel.createInferRequest();\nconst result = await inferRequest.inferAsync([tensor]);\n\nconst outputData = result[outputLayer].data;\nconst resultIndex = outputData.indexOf(Math.max(...outputData));\nconst prediction = outputData[resultIndex];\n" + ], + "outputs": [] + }, + { + "language": "markdown", + "source": [ + "## Display result based on maximum probability" ], "outputs": [] }, { "language": "javascript", "source": [ - "const imagenetClassesMap = require('../../assets/datasets/imagenet_class_index.json');\nconst imagenetClasses = ['background', ...Object.values(imagenetClassesMap)];\n\nconsole.log(`Result: ${imagenetClasses[resultIndex][1]}`);\n" + "const imagenetClassesMap = require('../../assets/datasets/imagenet_class_index.json');\nconst imagenetClasses = ['background', ...Object.values(imagenetClassesMap)];\n\nconsole.log('Result:\\n\\nid\\tprobability\\tlabel');\nconsole.log('---------------------------------');\nconsole.log(`${resultIndex}\\t${prediction.toFixed(7)}\\t${imagenetClasses[resultIndex][1]}`);\n" ], "outputs": [ { @@ -149,7 +104,11 @@ { "mime": "application/vnd.code.notebook.stdout", "value": [ - "Result: flat-coated_retriever", + "Result:", + "", + "id\tprobability\tlabel", + "---------------------------------", + "206\t0.7470666\tflat-coated_retriever", "" ] } diff --git a/samples/js/node/notebooks/optical-character-recognition.nnb b/samples/js/node/notebooks/optical-character-recognition.nnb index b7e8e109ff857f..ce450dbce0a0d7 100644 --- a/samples/js/node/notebooks/optical-character-recognition.nnb +++ b/samples/js/node/notebooks/optical-character-recognition.nnb @@ -24,60 +24,42 @@ { "language": "typescript", "source": [ - "const fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst { createCanvas, Image, ImageData } = require(\"canvas\");\nconst { addon: ov } = require(\"openvino-node\");\nconst { display } = require(\"node-kernel\");\nconst { cv } = require(\"opencv-wasm\");\nconst {\n transform,\n getImageData,\n displayArrayAsImage,\n downloadFile,\n arrayToImageData,\n getImageBuffer,\n argMax,\n setShape,\n} = require(\"../helpers.js\");\n" + "const { addon: ov } = require('openvino-node');\nconst { display } = require('node-kernel');\n\nconst Image = require('../image');\nconst { transform, argMax, setShape } = require('../helpers.js');\n" ], "outputs": [] }, { "language": "markdown", "source": [ - "# Download Models" + "# Load a Detection Model" ], "outputs": [] }, { "language": "typescript", "source": [ - "// Intializing Images, Models\nconst baseArtifactsDir = '../../assets/models';\nconst detBaseURL = 'https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/';\nconst recBaseURL = 'https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/text-recognition-resnet-fc/';\nconst detectionModelName = 'horizontal-text-detection-0001';\nconst textRecModelName = 'text-recognition-resnet-fc';\n\nconst detModelXMLName = `${detectionModelName}.xml`;\nconst detModelBINName = `${detectionModelName}.bin`;\n\nconst detModelXMLPath = `${baseArtifactsDir}/${detModelXMLName}`;\nconst detModelBINPath = `${baseArtifactsDir}/${detModelBINName}`;\n\nconst recModelXMLName = `${textRecModelName}.xml`;\nconst recModelBINName = `${textRecModelName}.bin`;\n\nconst recModelXMLPath = `${baseArtifactsDir}/${textRecModelName}.xml`;\nconst recModelBINPath = `${baseArtifactsDir}/${textRecModelName}.bin`;\n\nawait downloadFile(\n detBaseURL + detModelXMLName,\n detModelXMLName,\n baseArtifactsDir\n);\n\nawait downloadFile(\n detBaseURL + detModelBINName,\n detModelBINName,\n baseArtifactsDir\n);\n\nawait downloadFile(\n recBaseURL + recModelXMLName,\n recModelXMLName,\n baseArtifactsDir\n);\n\nawait downloadFile(\n recBaseURL + recModelBINName,\n recModelBINName,\n baseArtifactsDir\n);\n" + "const detModelXMLPath = '../../assets/models/horizontal-text-detection-0001.xml';\n\n// Initialize OpenVINO core and load the detection model\nconst core = new ov.Core();\nconst detModel = await core.readModel(detModelXMLPath);\nconst detCompiledModel = await core.compileModel(detModel, 'AUTO');\nconst detInputLayer = detCompiledModel.input(0);\nconst detOutputLayer = detCompiledModel.output('boxes');\n" ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/prakash/OpenSource-Repos/ocr-node-sample/assets/models/horizontal-text-detection-0001.xml'", - "File successfully stored at '/home/prakash/OpenSource-Repos/ocr-node-sample/assets/models/horizontal-text-detection-0001.bin'", - "File successfully stored at '/home/prakash/OpenSource-Repos/ocr-node-sample/assets/models/text-recognition-resnet-fc.xml'", - "File successfully stored at '/home/prakash/OpenSource-Repos/ocr-node-sample/assets/models/text-recognition-resnet-fc.bin'", - "" - ] - } - ] - } - ] + "outputs": [] }, { "language": "markdown", "source": [ - "# Dowload Image" + "# Prepare Image for Inference" ], "outputs": [] }, { "language": "typescript", "source": [ - "const baseImagesDir = '../../assets/images';\nconst imgUrl = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/intel_rnb.jpg';\nconst imgName = 'intel_rnb.jpg';\nawait downloadFile(imgUrl, imgName, baseImagesDir);\n" + "const imagePath = '../../assets/images/intel_rnb.jpg';\nconst img = await Image.load(imagePath);\nimg.display(display);\n\n// Resize the image to meet network input size\nconst [detInputHeight, detInputWidth] = detInputLayer.shape.slice(2);\nconst resizedImg = img.resize(detInputWidth, detInputHeight);\n\n// Prepare input tensor\nconst inputImageTransformedData = transform(\n resizedImg.rgb,\n { width: detInputWidth, height: detInputHeight },\n [0, 1, 2],\n);\nconst tensorData = new Float32Array(inputImageTransformedData);\nconst tensor = new ov.Tensor(ov.element.f32, detInputLayer.shape, tensorData);\n" ], "outputs": [ { "items": [ { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/prakash/OpenSource-Repos/ocr-node-sample/assets/images/intel_rnb.jpg'", - "" - ] + "mime": "image/jpeg", + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAIFArIDASIAAhEBAxEB/8QAHgAAAAYDAQEAAAAAAAAAAAAAAwQFBgcIAAECCQr/xABgEAABAwEGAwQHBQUEBwMHBg8BAgMRBAAFBhIhMQdBURMiYXEIFDKBkaHwI7HB0eEJFUJS8RYzYnIXJEOCkqLCJVOyGDRUY5PD0iZEc4OEo7PT1OIZdIWVJzY3V5S08v/EABwBAAEFAQEBAAAAAAAAAAAAAAMAAQIEBQYHCP/EAD4RAAEDAgQDBQcDAwMEAgMAAAEAAgMEEQUSITETQVEGFCJhkRUyQlJxgaEjsfAzwdE0U+EWJENicvGSouL/2gAMAwEAAhEDEQA/ALv1/D+mrU9hULp3EdolwdxxB7qgobOdQOVq9Yn/AGbfA7E9TW1jqr/pXq91b7qqW98vfUvOSA605Go22tbZ8fa/71u0J0gbRatBg9DSOJp4g2/QWR3YjUztAleTbqqJXn+yo4ePsli6cd4tpARlBdepaka8o7NuzTvr9lJei6ZdNc3GCqCDEesXAgnQ9U1U/LmbejyQRHxt2ArQCTY5pY0MTOXlje37Lvi/TBBu3HmHqkI5VFNWs5tD/I0sdBvtZr1/7PH0hrvcccYbw9X5hH+r3mWjpz/1hDfKLevIHMmeeot2LDNG081LjnmF4w1HoU+kvSNvLcwEl5ZJhLF9Xc6c07kJqCdieXOyRW+jP6Ql2LbNTwjxgtKVStbF0vPp2M95oLHIaz8be2a6dh0/asNrHigGxddyXO4cy7spSevZAG0O522KlxwvDauwBxAuB5X77wriKhGgJrbreZynX+dI8Nv0smJNQjMhyo74ChOpMg7wfjrb3dTcV2IENU6mo/7txSPuIsRvLA+Gb3SUXndrdWgjVNQO1HwXItE0juqQnC8PB61CApRTzJjWYnYeVjSVVIcbUgqUG0zEcx8iBPj8rex15ejpwbvYZavh1hpfPMq5KNR+JamzarfQ44D1qys4CudJUCO4w40Pgy4gWAaN6Mydo3XjxQvOhKmlPqH2rqTmBIKsx8OvP4dbKdO9UKZpEthUBgd6J3Agx0j7xb1Ir/2fnAV9KvUcKIpVKWVZmLzrkak6+264B8LNmu/ZwcL3UoRR11/UobASgNXw0pIgR/HSE7f4rAjpJI3XIVh9RG9tgV5udqsOPKIWpIIQQB7vdv8AdYBx57KEuKzSDlVlIMbaj4cuYt6BXn+zQusqUq68fXyyCdEuUlPUwPMONTy5WaN6/s0sZNoSi6cfMPZABNVdKmwQDMHs3XPHkbH4bhyVbMCqYKWorQS4ggGCDodjrHn15j3W6Qp1SlpDyoEDmQRIJnny+/xtaW8f2d/GmkWVU984ZqEjXKF1zZ6mJpoHxNmxePoMekJdqHPVcM3RWZ5ILN80zafCA8pB5DcWYsd0TgjdQGaioTT/AN6oEa97XKTy135fW4yHKtDqU9qojX2hsI/W0oXn6KvH+7cod4V3s+E/xUZaqxoJ2YUvn4WbtZwd4uXQpa704Y4wpUITGd25KpCTvuSn65WgWlTDrpqIWt9SgQmASCcp26+Gg+Xw7DrgCASMrgkBJkjToDvIsM/dd6UKEt3jR1FK4pRSUvMls94xpI+o8LcoacU4nOAvLuDrroOn1p5WGQUQboVLjhJSAqBqoAAxHkd/CzUvWqqWcX0QQSlRQkE7QIVoT7hpvoN7OtCVlRcSgGO6CI1Hx8febM/EBLd+UbqUhQUJCgrY5zEfAbdfG1eUXCLGdU76GtU7TD7QLhwpTyOsdP0sIagpAVmKhM6HfQxv7+nzsVutRZoxUgEqzwnUCNI+P11sKtZSlDAQo9p0AJ8dvH52hCLNsiyHXVbVUj2QpIA1OUkHnEe7x5Wxx1aU5t5IGYAHXff3ePnbSVIUtXdByawQY168v0Ng1KZLZW6gHPsCI1nQa+GvWxiNEJCLcfLhUVnQDcATvt8frmH2rmaXHezMlUeB6Dy1+HhbboST2crlROYgkDmdJ32J/rboM5kkmco7oGbTLO53Ez91hndSC4Ln93nUpCiQUjmPEbmJJ+duczql5lgpgbKIgDpHXzsL2S8wSCpStyRtt+Y0Fu22HUqJSQRlAiABtM6npPhaBcNlJYhS0biY1OxA5HXccrBPqU9TrSgkSkwCoxrET7wDYyEOBScyUgqMZYVqCdRPlpvbtxhwtrDgKwpBVAJ9xEjXn87BcbKYFwmRhmpeYXWoOYlRbKgBEjvEggeeh56eVnj6rTOhL4ZCAqVkEbp5nx0J+PnZn3W6tqvrmEtnrAXEEHrM7E+Pv0L+u3Mq7mDkOVxAmQdT16fXvsB3hddFaLtsk792MuZAplQJ5SdTpymenwtyu629crzomE6KEmfPnP1FnClpKEhRCEyARrAAjp7j9+lulst5R2bWYkTMkCfGPhZuImyJtm5/9iioUdBmBG35/jrYA3E4SSstqBHdzp021j3b+XhZ1GmaUSkgjpG5HXTy+t7Y3Slaxl1MTM7b6jp+ZNomXKpCO6a6MMKeAWpinWoQDqUyOp002+jbHsNLywijBhRMJX4bn4geVnuiiAjKkaSOkHmNbcLYKgpMAAxPOImfn+FhGZEEQUduXBUHRFI6r/ECJUJGkn+n4FH8PPpUFON1TQCtcyCIMQBr9RNpNXTlIJUnIUGRmA0P56H5+FgX6JzsFOBCjCTA10+6efztESlIQgqJUB5p6AM0EpIgkaGZjzO3gOeyi1QF9lClhtKuSp31289j8bavRnsalcpJBWYIPMkGfPT5ednphNmnqrkbS6w2qFrbUCnMd5A+cCzufsUzW8kx6rDLbbC3qm76VwJlUJbSSY8D7/iNptHWMbiLVKqqprqSyEqEFpvLpr0EQZHj7rT7f9LRFCqRphISDKykECZ9nT3/AI+EcYrogLnqUtp7qFhRO/ekfn5+VtKhlc1wN1TqWCxsoQLTzZlKFiOgItsVdYiQH3RJzEZjv1+6zkcpyT7a/tCdVDnJIOn1sdrFfVgtBQO9oSJTIgRJ193wt0YlB3WQWW5pFNbWaKLyjlgCdYtoVbs94A89RZTVSIyBwoSZB0CRvrp+nlbj1RKkSpjKNydhvtH1r7rSD2qNj1SaalZGoEnnbr1o5MhEgCBr9ePxsZVSN5ZUlSYE6a6bfX0LAPU6ENZkZt9ZtIFpTeJY447CXF5u8IBJ5W49ad73e0VuOth3vtKSmCie6lSB5hRP/ULFQyogmRoJ52cWKRuuxUlJzBInnbv11ZEZQJnYxYAtrHLxtrKomAk2ewTZijLVS2lQK0d2RpvYyq8myns20qy7ZiYMT4WTi24BqhW3S3MHpZsoKWYo76y2pARzHUaTJ3FtB5OoJEyNZixPWLZryFllCV0NUHNCpGvSwUzFtWy0homJQtNUO01S1UtLKVtLStJG4IMg2+kvhvik4u4d4XxU2sKTfFzUVfI59qwlf42+a8biLe+foIYmViz0ReGt5Lc7RynupV3rPT1Z5xgD/hbFuZ7Sx3jjk6Ej1/8ApaOHWLiCp+VUuRvYI1Du4NuTvbStbcmCtgNb0Wdu4T7Vs7dzUz8LcxztqCbGCnlC6YfcS+4gq0ISr7x+FjHbOz7RsTSSmoQdsyFD4EWHMzboaM5ogq8oAKG7Z06lRtoPu/zW4mRFsMnna0AhErrt3Brmtnbu8lbWD52zbnaQF026F9YdOmY2T7xecD9K8VnRZQdeRFjnw91iV6pWaUrbIlCkrnyNk4aJ26Fddu9Ht25NQ8P44i2gZTIG9uFD32YjRPsuxUukRPnYdNS6UTPIcrFI0MjWwrAlAHutXc2yI3VDGpeiCr52QcV3hetHh+9Lwbvc0RpaV2oStFPnKAhJVqDObbkBOtld5fZpkIKlKMJSOZ/AflZBxLhlvEVyV913tVPuIrmVtAIOVDBI0KUiMxBAPenUctrBcwyeEKZIaLqvtb6TdwIum777vC5Hq6rrgy+lhbDCV0ockF3OHVBJA1LaghY2UQdLN3iZifBF4JrUVGI78Xfq3E0jtGkLcNGcx7nYJhttSvaKHVoUEmTJ3auIeH1z4lwxfuE2r0dp8QYKfco/WnezUpTKUgrkhQBCyc0nUEZddU2rWzipVLSUGHnr6YdvO6K3KyENoU0wlDS1ZFFaSFpSvUEqKQqImYtnVmHiWBzSbaH1We6tkY4Ai4KngYnwGkBLlPifOBCovCmQJ5wlNIQPIEgdTvbLNxn0g6ltlDdVSXut5KQHFJSkgqjUgpWARPQAeA2tlvODhtbf3f8A9v8AhaPFh6r00fH2unU27bAUItw+T20RPW3aDpv/AFt9IO3XIsFghkjrHvt3AAtwklO9u0kWHZSXYEH9bbAjbblbkEKI193S3YPKySWwBO3vt0NtjbkRztsGySXUbWwTG58ranwi29Js1k4XQJne2wSeY00twPf1t2Dv94tEqS3MA6W34C3MmSAf6WyYmySXQI2Nt8teduZERMe+2TO+1kkujAOmltnvD5W4Jts+NmskuF01K5o5TtL/AMyQbBG67s3/AHfTiBulsA/EWMSAOnKbZmAHusrBPcom7ct2vI7J2nJR/KVqjfpNm9ePCXhve6iu88FXJVlW6qi72XifetBs7c2kZjbJjnItEsb0ThxHNRbeXoxcEbykO8NsNImT9nc1K375Q2k8zz52Zt9egj6N99uoqKrh7RpcbgJWzVVjMdNEPgTqeVrBzr52wq/mG1hGCM7hSEzxsVVyo/Z58Du+LuavigB1Cae815QY3+1S59RZtXr+zdwS8srujGl/0hCSEh19l8A6awGm522kWuQCImbZPLoLR7tH0UuO/mVQyv8A2a14pacRRcTXVqXr3rmQE+UipnkOVm1eX7OXiYktmgxdcjyWdU+sMVLc6RPcQ5yn5W9GQZETbJBM/jaBo4ypCpeF5fXr6BXHSgQ+umbw9XrI0DFc42VdD9s22PmLNS8PQ/473UhPaYBYdZbJUssXpRO6Cd0h2ddeXK3rbMG3C0NrTDiUqG3eFhOoWnmiNq3cwvHeo4BcaKdpVUvhPil1vdJprsee0Ame4kiCbIN4cPMa3PkbvjCV/UJBlRqbufZHiAVpHhrb2ZZu27lMJSugpiAIjsk/lbf7luoiRRpSTuUKUj7iLAOGki4KK2tA3C8T1MBLzheXBb2BV4anXXythpnCySh0qcXIToNR8NOWtvaerwtcVcCippVug6Qt1agfcSbNy8OCXC+9ZN4YMuV8k7vXVSOke9bRPzsB2GSciisrWdF4gFhdPfTzI1UtCspJ3Mjl9fKz/wAPhblz0yVKISjMQdUlRnbeNvhb1QvP0O/R5vZwvVfDO4e0IIzt0QYMHfVkoj3WRX/Qf4Glo093YeRRNme6xVVYVr0Kn1D5WBJhsxGiKysi5rziSD2WctaAADuwCeRjz+UeFhEUPahLJKte8VHyEyNxuOdr+13oD8MnVZ6C879Yyg5Ui8GsvwVTqPzshu+gPd1Mpxd3Yxr0FQyp7Snaeypjb2m7U34fUDYKwyrg6qlibto1OEqQgAAgefMkWGZu1lAAUkdo4ZlKpPu6jz62tzXegrfzYQKDGTbwSc3293hsTB17jy+fhys3rx9DTiRSl1ynrrlfWYCAE1Qnz+wI687U5KOpbqWlWWTwHZyrQaNtx09nmSGkgFSj7W5kGOXOwaqF0iO1IUuSMyY8tJHytPFb6J3F2myhvD1HUJTqpabwYQSfJakn5ToLIFVwB4qUJccdwBey1AQgUyA+B1nsyoHrvy91q5gnbu0+iIJIzsQohcu5RWpKMhABUpWUgDwHWRJ8osXdpXjTEEN96RouCdxG3OD8bSZX8LcdUK0t1WCsR0qUpCnFO3c+kdYkpj4aWalfc9TQKAqmalkpEJQtGVRPM6jx3iwjnG4RBl5KCsU0zjL6kqQjNmClFKpgRqT0P6WXMG3itm5nmwStaXjl0gQQNfHbeweM6dxpNSktDtnRmUQQTAI06RH32T8GEvUNQ2ZjMkgEzGhnfxA3tdY0uaqrjlfolioUVOFZEqV/MfGd/KRp1izQxRTtuUNal0SEtzmJ9525ab9Dpzh4LQsoKpGYmZjeOnWLNy/mu0aqUdmcq2VAa5pGXz1/XcWu0/hIsq0wuFEjzZglCSspBPtZp13j4+MneycQpQLWQ5VCSfEffEHz+FlRSsoSlJcBI5QoGefhtBkdegsmqKHESAdIkHmBz+ult9iyHIJ3u5ieQGpn5e4fpYN1KRlKwgKGh72/1pHlYQrTKlNrBV7KtfaHMjw0+VgFKXKc6CEq2GhMSeXLf52KAolaXkICey0He8hyHx+uhaqALTgTEggkk8/A/XOwy8wSCEqO8iZ25zYJzvIcCsw011nX6ixQo25INKpups5NGqhXe/zJGn/LbuG15UEBWYHUHWeW/kPnbinSV3bVoynuLbXoNvaH4i3KMywJB3g66H38rSKbktltmII3MEnQzt8jbMqUEJVEjoYiLYVkmVCJ8YERE/dbkgqOgGo73Ub/AJWSS2UBU8xpGuvPfrbSmkpJTJmYBI++3RK8xQpskHc89LczEFWhO08/ysklzlAJ0BIEeztblTWsKA12k2EzbnLmmCAVaQPvtpUT3ANehj6FnumsgFoSkab+dg/Owy092dbBAHUeFpAqBC0N7ezH7JXFKr79F6puNx3MvD2I6ylSn+VtxDTw/wCZxdvGcbi3p1+xnxOSxxNwa47oFXdebSJ/+mbWY/8AZ/K2PjrM9GT0IP8Ab+6uUDrTW6r0vVI3NucwG1tr2sGfPa3DhdC0XCE32tzEWxB2EW7AB2scC6bZBqGVbSjAhUfERYY6WAqP7on+UhQ9xsY20623KA/p2VeYc1mlt7crYJ520QCdvCbXwECywxFtDU23HIi2jPM2e1kitknb77BVCA4y4iRCkkRYUSdANbaUJGnjaYbcJr2SfTnMygkz3bbVJMxYKldQvtmEKSVMuqQoT7PMA+4/dYYyNOtoqV7Lka6WEY/iB1OhsGNz0t0FFJOQCSDEmAT0+hYMlgLlSjJvZDpbW4sISmVHQWLvVV1IafQ/XtJ7MFR74JSoDXQaz4WVKUXWy16zeVTTgKSSO1cASB5ExaLMR8TbgRjd3AhvKop3HmBVIbYIRnbGiw4QFOAAHNkSmf5ykA2zpcQFI3OSAeV/79FIjiuyhVX9Jhy8rt4vf21wJX0lO09dvqt4O1TDlI2t0lTZaW64lKZKM0EHMIIjQC1bsPcI8l5tY/xxjS6MOXLeLy00LLjwb/eBKiVFIAKyiDBUhJTmj2gDF4+L/EK5au42rmvPD9XeVI+UP01AxdqqenpRPcWXXWuxUtAhUZwiQUybUjxDwexbjYGvxNiynpO3fW8mvvJwUgLCRAAClAOEJ0OQEaEawTbDqMUdM0tdIGA7m2v23Qn07WPzBubyVgaK7vRhp6NhhuopXUNtJQlxdM6VLAAAJIqkiT/lHkNrZaGqTCuGaWlZpX8YXG44y2lta+wq+8oCCf7sfdbLckaB1/8AUv8A59la4v8A6BesLwPaanmbdpBjnpbToh3mJnlbtAAPzt9DO3XJN2WFRG9kyqqC2Xni66lLQJISqNAmdLKS5Inpysi1yS41VNE5QtKkExtKY/GxIxcoUhsod4b+mXwX4nX7Q4auDEN60V53ioN0tPeV3qaDqyJCM4zICjsASNdN9LTqxeFROV4gnlpaA8L+i/hC471uy+lOsVDt3PtVTbqWsqittQUk7nmkc7PX/SE3XY+VhGhpK5H7upi9VvVFK4whS1qGRLecDOISs5kynaCYMFyBw1CgHkHRSo29mAUIE2AfvLsl5Gmgop9okwBYGncIZnnvaOeOdJWXtwnxDdFDUOMu3my1RFbaoV2bzyEOAEaiUKUPI2GIxdTMhUgUWJ6SvQXaN6lqmkqyqXTPpcCVdCUzr4TZUbrG3Eg6687VI9HDgtWcKMcJvG66t1q77zpXaerpQtRaUQkrQqD/ABBSd/8AERzNrOitS1W+qkjVAWPfI/CzuiHJJshS84+00guLMJTvpNghedISCXonqg/lYlVuk0wE7qSPmLVy9KjGHGrCN4YUc4SX4xRodarnLwpnqFp9NUUFjsgStJKQJcHdInNrtYYiuph5vorQt1LDolp5CttjtYTMI018rMLCF+V1dhS48QX2wilra+hpn6poApDbriE5kgHUd5URZ2MP5kkg6EWgYraJxLdKMwRI+NsCjOhNmBjjE72FMI3/AItprqdvN66aKprk0TTnZrqS0lSg2lUGCcsTB32tBuG/TKevpxtlzg1jNhxUJKadYeKSTH8qbLhW3Kdry7YK2RV/NbM0gybJtHXLqWkO95IKQSFb2J1V6PNrqXe3LbVPMwmdAkKJ2nnZhGTsnMgG6Xs3Tbfe2FWu8C0K4B9Kbg5xLvZi48FcRqO8LwqZLNK7RP0y3YGYhPaoQCYBOUGYB0tLVNeSnQC4iDzA2sxjISEgKP5gedsJjnZEvfEdJdTjDTym0mozZO0dCAYAkCdzrYNGJ2nAFJplKB5pcCh7utmylSzBL+bqedsmNd5slUl+MVb3YJYdSYJlWWPvsoocSsSk2jZSQgV1tk8h77czHKwL9XT0ye0qX22kA6rcWEiTyk2iQkjClSf1tmfxsC3UMvJztuIWnqhQIPwt3mkSNpsyS6zdbbCup+NuJm2oJ5+dknWNHukTzI+dhM/ObF2v40nXvq++wgII3+NmCRQoV1A1tokg6TbknTytiTqQI31s6YLqdiZ+Ftheg59LBn5zbcgbCzFTBuhQrnGlszGelgkkbm25J062FayVkIFDcae+2ZzOlg+ZtgPjZJITNHP320tDbgyuISoHcETYMHnboK162YgIjSgTdt2LIUbvpyRseyEj5W5dui7Xk5F05KTyC1AfAGxgbaC2wR77QyN6I2Z3VNq9OFPDe+ypV8YLuWtKxCjVUDL+YdD2iTNmnU+ir6PtXnP+ivDjJXGb1e7WWCQNtWkpOk6WlIdbbBg2gYIzyT8R45qB709Bv0dryzKTgv1VZEZqevrEn4B6PlZl33+zb4HXqVOU15YkoSrkxeKIG+3aMuHmedrXZjpJkC2wrTQ+c2h3WLkFLjPtYlUDvj9kXw9fzC5OJeJqQFMJ9YFNVR5gNtE/HnZgX3+x5v1KZuHjN2ipJAqbiCEjXTVFUo6T/LyFvT1KhvFu0rGotPhW2KhfyXkBe/7I3j7TFa7rxvhCrSmcgWuvaWR7qZYB9/vswb7/AGY/pX3WhaafCtyXlGYJVT32w0I/+0Fsj4W9vgsH8rdJUN+VlZw5peE8l4AXz6DPpV3OV+t8F75qCJA9QfpqyYG8MOLmzQvf0afSFuQKVe/AzH1OlOpWvDdYUAdcwbKY9/nb6LVtsuj7VtCv8wBsXF13UTP7upgY5MpFkHvG6bhtK+Z57DGJLnTWMXzcN5XeoNHu1VI40ZChocwEc7I4SoNzty1t9Or1xXVUJKHqcqSd0B1YSfcDFmve/BLhNfxUb64fYfr83teuXWxUT/7RCrPxnDkkYgea+bMocEZtNNdCNOU20pRM9yATtNvoWvj0KvRdvxJFbwTwinNrNPc9PTn4tISfnZi3v+zK9EO9ypX+i+npFKMzTXjXNQfIP5flZd5tu1R4PmvCSCD3UxG2ttlLihCUmJ0jW3tTe/7JH0Y64f8AZzWI7ujb1a9zt/8AXNu2Y1+fscOGTqnF3BxDxTSAju+sLpaqD4jsmpG/OzGraNwUuCSvI9CDlzDQzrIjT6/C2dkiJKhGmo13FvTS+P2Nl8IBVcnF9xwnlVXA2B4SUVZJ/wCGzDvz9kVx2oCs3TjHCNcn+FLyK5pR/wCBhxM++yFXGU/AeqCqSj+ImT42BKSDFrhX/wDsvvSwuwRRYcuC8ttaa+m2gT/9o7I/LnaPr59Aj0uLiUF1fBi8qhJUB/qNbSVh+DLqiPeLFbPGfiCiYX9FXvUWu/8AsjcRqur0k7zuJTuVu+8M1TQST7S2nWXR8krtXC+/Rg9I3D6lG9eBOPWG0/7T+z9UtEf50oKfnaT/AEDW8TcOfS84fVN/3Fed1oq6167HPW6RxmfWGHGkg5gP4lJsGvyy0rwDyT07XMmabc17kKMggW4522VamLaHjbz4DVdKFnusIgzIFuBr4W2BrY7Eiu3EZ21IP8QIttpWdtCjzSLYk5hB3txTK+yUjTuLUny1kD4EW16A2JCrS6BCj3W3FtCDvpbYmNLaoCr3Wc5m2hlgcrbJ03tke+LOEyyCBIM25URtbek/ibYZ2FihRJ1SahIRW1CEpAzwvQb8rdq31GptzVAiubWD7aCkj527VKhMzaFksy5G4IG3Kx+6ruFW+X1tpyta7bq5CxFsdo+3TI1ccMJTv7/IWdvYeo0JbpU5lJRKdPaV1PvtXka06nkmfIWaDcpt3xQ3Ve7qziR5hm52VJbDbjvZpfeUYhZkSASEhM6qJkGBaMuLXD3hxdlwV9fT05w2mphp28aeudoHHgogqSMiVuPKMJ7mQ5tdbTS8GKTItbWcNkFAy5lZoiR46n4myPifCVFjK7Td+KqRD9A6tLhogshKinUBwpIzCYOX2dNZti1VFHIx0b2hznc7bfflbknZKWuBB0VSmlPXtThq7r0x5f13UjfaJXWMuPtKUlKyQy0ptxSFQ2YByLy90e1Brzih3CWIcbU9wVeNsVu1F/VyqZ2gqaPs1sNkpzuZE5glAyz3koACZCXIMXz4m33U1l10l1YBoFIYoX6dKKimYCmlpCi2tlkJICYSCCpRSjZIVMlNQ+N+M2g1fdVgm+Lpuy+H7qDFe3dRbdqWVtocYCHKhBkAdqhTeTKFDtNJFuIqaLu02Vri4deQPodlrts+LMdPJMuvYwNS11RTU/ol8SFNNOrQgqdr21FIJAlCachJjkCQNgTbLRX+4OMqu9dtVfIpDrTh15IWG/4cwKhBiJ0FssT2fBzkP/5uWfxz8v4C9jnxD0a26A0j522+gdqOptiRpJ3t7s46rmW7LgxyMWbGKgpdwXy2DGekqBv/AOrNnUpEg6TOwtCXEk8a2MUVFDhqruN3Dt4ZW+zqGFB9lCkhLneG5nMR0kTtYjHNb7xshPY5xsAq+cD+BGL+GuMbgv65MX3kKBuoZbq6NVQS08wSErSobHunSZiBGokW6vZtg3lQurbBXlcSkxqB3dPKxdjDNXR3C45QpbVXNMqUwlWiS4BKQT0kCzDwXjTifizEjd1Yu4d010U9Iha/XGq0LBWIGUIJkg7zOmWxjIzS7kMRv6KaaUxTb7J52b+M2/WMLwnvAu0yjPTtE62ctIwpLGRQ5Rp5WirEfFO6+Hle9hziFdte1dzhUaS82KdTzCmiZyOBPeSpO2gMgA6WUbgTome0gJ23Mwhp2mWlIgSNtu4q3dVUZ8TlCYhFK3OvPMo/daO3fSS4HXVS9uzi5yscAhqnp6J8uL/wjMgCfMjzs4OGtdeuLkvYtr7vdoU3m6XWaZ722mAAlsKn+LKkE+JNnkeL6pRxuAUiPkilbJH+0Ry/xC0I+kT6QaeCF44cp3eHL2Jqa+G6px19moCFUYZU1HdLas2btDzEZec6TNiC8KK5LndvS9KxqlpKYoU686oIQ2nMO8onQDW0Z4pc4WcRaqkqazG9wPCjbcbaSmtYcBzkT/Fp7ItAG+ykR1SrTvUHFDD+G+INzXveTFGtlN409GVBDTwUg6PJAkqTJjvQCNuYf11uq7FIVMxr1tGT+NcDYNuq6sB4cvOlrat9SKOkpqV9DikIJlbiwk9xITmOu5gDwki6AtVOhShEj32Y7papGxVV3Fd+FL7vDFLymbnpaSqevB0pUrLTJSpTioSCowgEwATpoLRThfiz6MFU8y3hvHFD2zykoaQ4y+gqUSAAM7Y1JI0m0t4hu5u/MN3xcamw569TVNNkVEKzhSYM6c+doywl6PuE7ir2rxew5QpeaUlSVdkhRBBBBkeNhSa7i6LFpzspqoT9mAgDLG4snOjMa9tWynCCfNtIsp0iUNtBAVoBFk1SAXbwTPtKPL/1abTZtZQfuoMwn6MWGsLYoubFDQp2qu6ahFShxlBCllMmNRsZg+FpaubGFLe991l3ULNUhN3rDLqqimWxmcgE5UrAJSAU96IM6EjW1UeF3B/HvDbGlwX1dGN73dudNY23VUD1UtbTjKyEqSRMHQzrMeYBtbspaNawsphZSoBUcpGnuk2ThlCduuyReLfCy4OKNysUN/UYfVSKU5TrzlJbUoQSIjcCLIfCzh/SYGwBW4QpFLS01UVgSSokgODNuTM942lhgZmQCSdLIaUpS/eLYG7xPxbT+VmGoNynvYgWULej5wovPAN6uXnUYmvq8EuU3YIarKtTjbYKgSQk8+6NfE2shTkBEEAWbGGqdtLDZSkeyPOzoQiBpp0Nq1iNCbqxodghiqzH4qKUq4Utg93t0KIGs2e5O8aWZnEtAXcZJ1hxJtNouDdRcU0sC4j/AHe47drildk/30CfZXGvxH3WTcMX1UDGia1T7qkLqnioZjBnMNttzaLqzGFdc3GsYMqZRS12H2Lyu1w/+kNPvJqEeeQsq9yrPHBztQqsadqW1NuevP6Hp2ysp96YjwsPZgBUhq64ViaR8OpmbGQZ0H62SLpUpTQk8hZVBgWThYpLlr2nJ5L5+QsLMaxYFsgLcGs5gY9wt2TPKbMBdJNvFGMjhypYYaoRUrdSVLlzJlHLkd9bD0+LqN65Dfbram0NtqU6gKzFJTumY1/UWaXEgH96056tHkep/OzIrMTMKwviJuiqgsUzL7TyQofZvNo1BHIxB8QRYLn8OpEbvdIVlsXEp87dwVM2HcYUWIu0SxSvsqaSlRz5dZ8jZaU8IBSPKbRZgt9VHVBLayQ418dRZ04wvx66cNVVayftQjKggxClGAfnNnpnGVlzuo1EbYn2CP1mOML0FSqkqr4aDqDlWlCVOZT0JSCAfDeynd18XZerPb3bXM1LYgEtqnKfEcvI2qDxIuviBfd2sNYAv83XUtqW86sNBRd07qJMwNyY128QXDwIxbiZVPc1fiIKbvJThorwSABmIWUEkDT+VWkCRpYTXvc3iHa9kQxNByDdWnW6lKcyiAneegttDza0haCCCORs0cd3dU4iwZel0UTpbeqadSWyFFPeGoBjkSAD1BItDvCfGLl3Vie3loOj1SrbUfZcSdCRyIPyUbWnxZWB/K9j5Ks05nFvNWQDiSqBbqQBJMWgyqxbed94pQunvWrYo+1S0hNO8pGZsHU6Eb6md9RaYUV6V06n8xCInvdLV43CVxDeSNJGYwCeaUA6Npt2FAiRHhaBeIPGupuC76zELL9Yi66SAhFFTh59+SACB4zO4AAk7TZV4QcaWsd0FBeLNWqpu+8wQy462EPNOAlOVYTpuCOfIgmwjKL2tpe10UQuAvfXeymYK26WwLAgmJsj35frNzXPVXq+kqRStlzKCAVRy9+gsyl8YKKmDZvCkYpw4cqM9aE5zEwJSJ0tOVzYQC/mmia6a4aNlJwUPL3W2FTZsYWxtQYnU8zSsqQplAWSXEqBBMbizjSqedmY8PF2qTmFhsUMF6aRbebxsDm00Pvsi1OMLhoqpyiqr4pWnmzC0rXGUxt0tImwuVEAk2CcIX4267TlrZJu2/bqvRRbobxpahYEkNPJUQPEAzY+XAEk9LR0cLhIgjdGQsEaG3Qc87FG3Q4mYOmltioSpWSdRaNk4dbdGguNfjbfaCJBsB2gSnMT520lwKEpMi0HAqV77IwHDoOVug6ORsWKwNSdLbCtgPvtGymLI12pnU6W32s7WLZo0m2ZoG5i0UrAoyVk24JB3sBnJnWTbO0nS0SErWQqjPT4WBcS05IU2lU9QDbCsnYC3Kj4izFo5qQNkWcu+hUcyqNgnr2YsA5dNCuCWimNgHFCPcDY6penP324Kj1sMxNI2RWvI5og5dFNlIbdfR4h0n75sCbqXMt3jUf7wQR/4bKSvPa3A9oePUWAaOF3wowleOaTV0Vcn+7rWj0zMT9yhbnsrzQdfVV+9SPwNlJY03iwWnvtDuER5KXHciYVXo1NG2SP5XvzAsQumpvdx2rVUXe00pTuYN9tJAiNwCD7PzstGedirY7OsWJjOJjyj9bJtI2IgtT8YuFiF0l2qjvUh8crgP5W2agpmaV8R0SD9xsPsJ5G28xHQWthh6oOZFjWsDRQdT5tK/K3IvGhKoVVIBA2On32NnXU21AIgj42bK4c09wghU07glD7avJYNuiZ2I+NuHKWncELZbV4lANgF3XdylKPqbQKtyEx91peJRJARe93006WH1BUJeSNBsCYtwKk1Ci3TQcuil7pT+ZsSv8Aw829ddSihedYdyKKFlxa0pMb5CqDz0sfo0JRTNJbEJyiABA26WXiLiEjlAuE4rkugUKDVOJJed9oq9qPwstKcSlkvKCiEAkhKSokeAGpPhZGpr9ZRTIRUZlOAQTHTY2Zt78U8WYXDlTeHDh28KMvFDT913gh4pRBILqHEtqSdDojPrAEkgWo1dXFRsMkxIA52JQhDJO6zRqn8+yoH1+oKm+5AQVd1tPjynqfdytEfFbGOJr/AHEYLwNSMlKSl2vqKxakN1TWaFU7aEguLSZBWpMDKCnMCrRuY746X1fFUjCzLFNcbrikuFovpXVvNgahKTATBIBJSqI2HJpXxj2uuK4n0U123I1U3k7kqF1iy6p4oSIU44rMpzQgDNISAQJ2tw2JY/NUPbHQg8M2u62p672stinwtzGZp7X6XQOJGL1RcdXduLcCVfaPBZq7wonGbxbJgAKNPmQy3DaUpSghRSAhIB7pNauINw8J8Mm7bwwtW3qugvFxLLpW0W3UvaOJcebXlSlBKoSlKEiFE6zaY73x/iRFEf3HeWHlVT6FNoRTXc0ptDkEpUVhsJyJMGSoa6QSYMNX9iKiqr27bGr9Vie+EEIStkqpbsIlJAKEJ7RWaNYDbhKT3VFOmTJO6STQG3RWZImxstcEqJ6ziHxOoKt+hpMNYmqGKdxTTTq+yWpxCTAUVZBJIEzAm2WnRdXg5S1KYwuA2SSjs7szoy8sqlZlKHQlRJ5k72y0Pa0X+x+/+VT4D+q9HK1ID5002AFuECdbD14+28zysEkaAje3v+65FYEnmLAP0TbypUmTY0lMxIt3Gm1mukiaaVKUZQnfSxZFzU7b5fQ0ArmQNbKuWPdbeWNxZiE40QCGsqYsmXzh6ivhos1bCHEq0IUkEGy0RbYRPKzGx0KVyNlGzPCTCtPV+ss3PSoWFTmSykH4gWeN3XU1RNpbbSEpSNNLK5QneLZk8LQbG1puAiF7njVJd93LRX5dNRdN40zVRTVKOzcacSFJUnoQbQzeXoxcPKl0rGH6VJJJ7qMv3WnuNY2i2lNpVrFk5ufmQosdl5KHsD8DsI4LrfXrpudhp+IDmXMuJ/mOvO0pU9Lkby5YseDCQZCBbYbA0jeyY3JzTvOc3UC8SeFPEC9cTVl9YW4o3/c9PVZVCkZdJabUEgHKkECDE+ZNmuzw89ICjVDXGa8lJ2lyhQ5/4jr8rWfXTocPeTYH1Fon2bJ3EJ0d+yk0sA1aku4qapRQMN1z6nn0NpStwpAK1AQVQOptHHEpji9SYlQvAlfc6bqqEIS81WUylLS5JClBSdxGXTqPG0wIYCNrcO0jb0ZgNPCxRI5vuoRY07hR5WYWq0YYqP3e22quaYK6cKkJLidUgnlqBZtcO8R8TMR3w5SYzwpd91sUqCpLzFSVKWskDKEEmExOpPIeNpoFIkJKQNCLF27saadLiUJBO8c7J0rzZIMYNVqnZKWwmZ05WijHPFCiwNit6467DWIKxValt1p+hou0aMjLlKiod7T5i0xJbCRBFk+8bjpq91LrzQUpBBEizGQtGgT5Gk6pMw2y6mmaUtBHdG/lZxIiwNPSpa7g0FjQSRzm0C6+6f6LShzizU4hKabuJx2ocQ022tJUtaglKdeZNnYYsh4uwzdmLLhqrhvilRUUlWkJcbWNDBBHzANptcBuolt9lWbi5hb+0NPRYtw0609emGyirplMrCs6Uk52iRyUhR06xZwYUvBurTQXi1HZPlpaOcTH17rSZgvhXhzBF3O3Xc13IZZdcU4oanMowJJOp0AHus0mvR2w5RYzTimmerkZaj1lNOH1BoLmQMo5Axp4RZnyMcwA30U2scHXBCk5m86e7KL1qqUoIGVOgkkkgAD42WKG8qeta7ZkrAnUKGvWzVxlharv7C1RdV2Xi9d1UooU1UsgFbakqB0kEco99g+GtwX/AHBcvqOI7+qb2qi4VqffQlBgwAkBIiBHnqbQc5ugCQaSLp7oIDqyD0sIYiB52DRHan/KPxsJrGny1sgbJlHvEVsGtoTqJSsH5G1abvYewpxvxVcN6OKduPiA0iupQZCUVbTCGX2v99AQeWxtYrjeMVU1zUt4YRuFF71bbxQthTgQA2RqZO50A352j29eHWIMb4GpLxvG7W7vxBQuJvCmbQ4Fll5JJCMw01T3Ty15xaMkQkmu46EW8wUeKUxxWA1BunnhOWXaRBWV5GuzzEbxGtl3iInPg2tVPsBCjH+dJtGnC/FGJ7/xCbvvDAd5XS3ToUXqipWAnMDAATAJJmegFpixDc7t8YavC7ElIXUU60IJ2C47vzi0aBhibleeaeueJHBzeijLBFO09UPIcAPdGh982bTVM3d2Jq9lgZQ3eJWlIIG+VRPz+fwRcP8AEy7bivF27b+fTdF70Us1FPW9xKj1B25Tv5SLK1y3xdGKMTFdBeVLUv1b4fcbpnA5lACQToTGiRv1tGcmGEwuGt1KJvElEoOllYW7yXmQlQ9oSbQZxCwyvC2PEV9Ex/qF/hanUpGiKhIkn/eE+ZJtOt0sqSykqSQSOlmbxkp2UYfp612Einq0qUTskEEHXkLXYiHtLDsQqjyWuDhyKaWA7sFXWpeCISyAkCI1/paVL3aW3h+tSgwoUzkeeU2YnCd5itpqlTS0ryPDVJB3SPyNpNqmQ9RONEGHEFJ99qFEwM1Kt1j87rKBLnuakv6nprtq2kONPISopIme7NiuA+Bt54Txg0u7sVVjNwU9Waxq7W2UBMk5spWQTGbpGgjxsewheLVI+2HVAu3e+umfSN0KBKdRuOtlurxXiF6/akXbWFijZDaWShCVdpIOcmZnUx7rQe0QGQOOt9kZhdNky7WT/wAdtF7Bd7J3PqjioPgmfwtA2LMCYf4i3PRUOI6RL7NO4mpRqQUqyFMgggjRRtYS+0+v3DXMAhXa0riSOspNq+X1hZeOcIUlzovK8LtWvsHA/RP9k6CncBQ5HW0q8foMcDazv3CjQn9V7SL6KRuBeB8H4Boai78M0iWn31BypXlVncAJCZUSdBJ08T1tMCFAjS0O8DOHDuBaasdqsQXxe79UUp7S8axT3ZoHJA2Enc76C0wtwIEWFFfLqbqUpu7QWXSzCSqBpaueL2K5yoxExQVPY1ZeqksO5QoocIORUHTQwddNLWLcEII8LQPithTOLb6Z2BdSsH/MgGxpm3pJD0sfyoU7rVTPuPwom9HzF/EWsYvJGPHQu+LjvINtVTdOGe0QUgiQmE/orW1wq6rd/ddS7TLyudgtSD0VlMWp1wu4mYsv7Gl64Oxfh2koK26EJq2HqZa+zeRngaL6778iItbS5a395XMzVEAdu0CoDkrmPKZtVpvDOQ4WvY25KxU6wgg3tcXUY8KL6ql3rRVT9W8760y424VrUqVQTJnnmTZOv3H2JWOIl7tJvaraZu6raShlt0oQWsoMFMwZk6n8NC/D4qo65uiUqDRXk9TGdP4yPuNknHbAo+Kd8NgkGtpaapGnROT/AKbWclu9M6EkeoKCTmdA7qNfQqx98Xv6hctbeaEhz1enceCSfaKUkx8rMThdjy/MQVqWr2qmltP0xeQlLSUwvunQjlBO/wCFnJQqF7YRZKiT63QpB/3m9fvtEXB+rLdddCFqMhC6dXmAoR8hYUv9Bsg+YflThH6pYeh/ClniTjheDropaxqkFS9VVKaZsKVlSkkE5jGpGnzFj2BsVuYquU3g/SIp3W3lMqQhZUNAkgz/AL1mNx+aWvBbFakEmjvFh0lPQkp+9Qsa4NvqFFeFMDol5Dm38wj/AKbKpbkbG4c7g/ZRhGYvB5WUp5tJJgWaD/EG6mrxqaBAqnV0zhbWUNjLmAkgSRMbWdWYlo+VoHraZpWI8QXW6ohDlQ8hcEpIDgmQeWhsKoJjpXys3bb97KcQDpmxu2P+FLl14zuutqE0peW24vRIdbKJPSdvnZwFYgkjYWptga5MUcOuKNXhC8MSXneV1V1CirovXX1OlohRBAKjpsqYiRl6Ta113166+5G3s5Lq2ilRj+ICCfiLAhdJxBFLbUAg+RRZGsDC5nI2QtBim4rxvB26qW+7verWZz0zdShTqY3lAMj4WUKioSykLV15m1P+ENcbs4q3U84YUt9ynVP+NC0ffFpW9J+sqk4GpW6d1xLTtXkdCVQFdxRTPUSLaXdCap1MDsqgmHAE3VTQKlCmi6NR52xp1LqSocjztSHBHF298J4GxHhlu8HGDUsioonkKOZp0KSFhJPs5kbbQU6b2kL0TsXXtX35f103tfFdWBVMh5hNRVLdSgIcIOXMSBIWnbpau6LLEJfOyKJLyZLcrqzi3ENpzKMCwYfaV3QseVos488W3uGuH2lXZSt1F5VoV2Ad/u2wClJWoSCYKhA0nrpatN1+krxfob7pUXlfgfRWoDjDdRdzaWXo3CSlIJGm4V4WC4ZCARujtOa5HJXqJ3iwcmY1+Nmxw1x3ScQsI0mJKdsNOOZmqhnNPZPJ9pM8xsR4KFnOZJ0982YG4upWI0K0TpYBaQKhtXXSw5sDUEwhZ/hVPu+hZiEhoULyg7W2OpFsgeVtDQWkm3Wz+FtGPxtsxbRjysklzy099sAG8aWw+0BbfLeLOAoE3QT6M6FIOx08bJd2LUujQD7TZLavAgxZYKQQfvsjXelbVZXU5QUgPB1JPRQn8LORqCkDcI3zsKxTJrWKilISSUZ0Trqk6WDI2jnbpisp6BZq6peRlCFZ1QTpHIDU68hqdhNq9ZE2SFzX7WSY4tddqhniRhbD143k7dV9XVR16ajI96u8hKikAZS7CtgkzqNdY52jOvwLetY0wrCGOr0u5EuEu1bKa9socVmJbS6QpCU6BOVaZAG8ZrTPWXRWXled84irxUUZqXQxT077Ce0ZaQhJQTvBJVmyzAJJUCRAZ2Kqegw/hWqqL5YcNG42pIVUBZ9ZUUmEFQEkqOm0axoLcuGxTh0j2hreS0XPdlGtyojvPC4xxfT12VN90JumgQH2001OA7VMKAGd5/MokLV2gGUoTAWQkQCqrvEa5r6fxRe9z4rqTddxpqVUFJV1N6rWt2hSv7HMlJU6lgZEqg5UGCRBM2t7dFNdd0XCqrvGqup29rzfFdXBx1CWqZRAgBH+AAJSk8xPIm1M/S4wPi7EGPX2MMvu3hddV6rXNGnqszBdUgNrWsiQpcok7kZwAABFhU9JTMcA0ht9z0VeWRwZnOqkWixXh2moqemp+M16sNNNIQhpm63ltoSAAEpUTKkgaA8xbLVcpMAY6apWWximlZyNpT2YVIRA9nQxptpbLVjglJf/AFP7f4Qu/SfKvoEvBMPTE62AQkAzEWjJXpNcMa5wFKr2ZkxDlIlUHxyLNjjHH7hY4QF37UNHnnu6o0+CDb03vtOfjCyjSTjXKVIoEamTboJ5RttZmMcZeGFSQG8YUqSqQA4063/4kiyizxI4ev8AsY2uQHoqtbSfmbSFVC7Z49VAwSjdp9E4x1tmnI2TKbFmE6whNJie6HyZIDda0o/JVlFqppXxLNS05PNCwfuNiCVjtio8N43C7CRz2tuBprbYBPl4W3GmvOz3B2UcpXIE6xbUbQPK3cC2FMiPDeyulYriATG1thM7i3Uaz1thERAsk65jprp8bZljXrboCOdsg8gPGySXOU7wbaIFhBPO2ojU2SS4CegtoDwsJGvnbUDysklxEn5b21APK3ZTrpbRGnKySXOWbajmBpbsiNbaAOmlkdklxHO248LbAJNsIFo3SWoi2iARFtkAWwxOhs4KSDLSCdtRbSmUkju2FjlbCOUWYpIEspIjLpbQp0o1Fh9+Vtc7MkDZAAQ9BH8H42F/G2iPtkx/Kfwt0Brz1sk5QD1Ml9JDmo8rF03e22kpSjQ6WUIEW1AsrJrpLRc9Oy8XktpCj0FjYYATE2NRO3lbRAA2swFtlIuJ3TJxXw0wrishd93FR1ixsp5lKiPIkfU2Awvwxw3hVR/c1z0tGknUMtJRPwGtn5CToQTbMoBmPjaDow7VFZI5osi7LJQkAJ2sk4uwtdeL7iqLivmlD1LUxnTJGxkajxFl4pk6aWzLB8D4WkPCLKGl7lMvAnD+5MCXebruKhRTsFwuqAklSjzJOpMAD3CztLYiCPfYfKBy3thSOQ3tBtwk52bkoP4j8EbwvG/nsW4HxG9cl5VA/wBZR2QdYfPUoMa9T74kkkrgnh5julvIO4sxGzWttnut09IGQT1UZJPkIFp4LSTopMjxtx6q0NQkWhM0zm7iiwzCEWCTW6E+qKZP8Scutq13hcXHG4apygu+5sP1dEyvKypanc5bB0JMgT8tedrUdkBpFi713svd5SATztKQufHw9LeaeKRrHZlBPCwcTqjFTCsS4fu+76BlKluONVClLWYISkJ23M67AdbT62DA3NizV2MsrzIQPdY4hJToLAiZwxZFkfxDdbIlO/haAOMd9U2EcWLrKu7rxdRXMIUFUtKp0ApEEqI0Gw8bWAKZG1k687npLzATUMpXG0ixi48NzLXuLIbG2kD77Kq1xYhwhe2I0Vt23fXqvh1j1QRQvJPZ580KMZRrzVEdetlcM0S6O56elM/Zog+Z1PzNuqfCN3MO9o3SoB8BZdYpEtJygaARanDE5kmd3TrdXJpmvZlCrJiK/KfAfEm+Lovl31RqufF40LzhhtzOdRPLvSP90+EgYgrWMQ4rZxGipQmKFNGptKgvOrOVAgj3iPH4z9jbhxhnHFMinxBdNPWdlPZqWnvInfKrcfGzQw5wAwPhi8mryu25kIfa9hS1rXk8RmJg2PVylxc6O4LhY7WKHTNaAA+xynTqnhhKlfpcM0FLUAhxthIUFbp8D5bWhfBxF14qdpFLCfVL8eaAHJPbEfibWJYpQ0yG4G0Wjp7gxci8cOYvcerlFx71j1YVKksdrM5igb6666WiXAU3Cd5fhJo/X4g8/wAo1xjpPXeHF8ICdWmkvjwyLSr8LNThhf1Ldb7q6qpbbbqWUKCnCEpkHqT4m0r3/cjV93FW3O7KU1lOthRGpGZJE2gB7gpxMoV9lQcQAWkJhKHLuQrbaDM/0s1TK3gsF9QSefP6JU8ZMjjyIVh7svmgvRtSaSupnlJElLbqVEDqQDaHcVA0nEq8Wtk1TTL4HIwkJJ+Is4OEeC8T4bdqa7FGIE3g+62GW0N0qWUITIJUYJJJgdAIOmuiRxiw1jI4go8R4Ru6krf9V9XeQ+8pEQokEQDO/wArMyVslNI19hcH/jdIxFszC3kU3cfVDC8f4IrGWzndbrGXCOY7JJGvSc1piwmvNc6G9SEkgSPGbQncrHE6tvBinvXB9EyAYNQasKDaecAJJMwNLTzcF3+pULbIT7KdfE8zbPjnfPUMcRYNbbe/O6tuiEUThe9zdVKeAw1xXVJyJor/AFK6AJD2Ye6Dad+P93/vDhpWPBoLVRvMVAA6Zwk/JRtFPF/B9/ucTqtu57irqpV4OtvsraYJaBKACSvYQpOsnTe0746uypvXh1fNGtuX3bucISnX7QIkAddRbojOzv7ZQd8t/QXWS2NwpTGR1Xn3ftYbuvWgoH0j1a8TUUi9IhyMyNtpCVj3i0w+i/WLu7iNR0jjgJqaR+mJ2JypChPn2YPvs06Lh5WcSlXpS3RSrefoGFVlK8EnK3UoUCElWwKgFp95PKx3g5UVl3cSsOOMsrU6qtbbcZSk50BRKVynlAKiegtWqWBocGciiwuNwXcwpY9LiiccauioB7qqd9M/ylK21fdaIb1fwtfPDvCF2VV41VLfNKmpcDrCEHIgPKRBKtJICYAk6eU2K9Jy53KzA9NeyEHLQVMOq/lbcTln/iy/G1QLzuK4b/w4u471oSa2kqUPUbwBKS2VpLjZ5xAJ6mTtZ5WtLYpfIi/Q/wAKlGSDIzzB+quT6NNRhZjCNTc2HReRVTPJeq3K0pzuuOJgKGQwBCIgAbe+0xe61a/RfvamVf183fTkBLtKhYAEAdm4Ux7s5FrJjQcrZcTrt1V6UAHRYSOf32AqhmZUNOXwsMTrbhYlBSIkgxYiDm1WNrK2kK6pBsII8D42BpyFMAARqR89LCwroRZwE91s21Bjb326A0g20Tl2tKyiSuYBVMW3sNLb3E21ykDezgXTFcxqYFkp0djfAg919nXzSfyNlcmdhZKvRLiKqifQUhIeKHJmcqknb3gWk4aKLSbowRAg2T2Ev1N8+tLATSUKkhkA+29rnUfBIISPHP0BsokHlbjKRyG8/Ow5Yw8C+ydr7LvGdEhTYVplLfTSR9C0J4wu2mW5QU79Q3UJ7cut0TErqKp5CcyG0iQIEZlE9wAakCbSvjS9MVLpWmLlwgK5KmQfWX69DDYVqCnQKV/CDtztFmOafElx3JeN6Ul3q9ZNGhVT2KF1JLpABbQoABDQWUnUAGFLOaABxWNYkyJggidlcfLofNaFMwubZ2qiS/KGpvrGBoMc381d131LZ9ToWVkBLTaAVl6pAISDmzFKMkCMyyBZIviguvEpv/1LBN03w4puoU9elJkcTd4SUpbCUalS1DM4ZDclQSP5rOPGF13jeDbNNft5LoLu7MdpSMinFStRITlSjOpEbqzJUSmQIJEGHMYUzhvN5q7OJCaO4qVCHEUdNUFpa3FJKjCSMzhSkI1JWSqAkTojh6yaorMzJTa3Mixv9v8A6V4MjA8IToprs4bMUzTD90XitxtCULUq87vSVKAgkjszHlJ8zbLNym9H+jrKZqrreJ1bTVD6EuOslxxRbWRKkyNDBJE+Fstl54v98+hUO7u+T8owmsfZcICjERqrbX7j+FlmivntIC3O8AJJ+ug+tbN2pCu1zoVKo1PLTwsNTKSAEkqSQDsTI57+/wCFvYHAOUmnVPRitQ6MyVGJ1P19bWOtvQkGTJgmPrzs0GKp1pRQZgSnQa6T+dlenr3EaFRKSPamD8vo2rubbZGCWezpVqIVTNEHmUja3aAw2oFDLaVcoSBr9fdYo1VJJAIzDaNtrDpey+zz3ObUifnYWoU7ApTp71raYZqevqmRMy08pPwIP1NlBjGmLKYhLGKb4T4CudEdf4rIGaUgKBMb89fzsIFKju5Y9qSdrSEjxsVHhsO4Top+I+Pqf2MXXvOvt1a1j5k2PU/GDiSyMreK6k9M7TS+n8yTNmSFEg6nfeZ/p+tuiSQCFGDMg8yenW0xUzN2cfVRMER3aPRSIzxz4jte1fLLw09uja/6Ug2UGfSEx63o43dLpEHv0yh80rFouJVIg65hEb23mUCM3MQCN4ixBXVLdnlCNHA74QphpvSMxUEkVFx3S5z7gcQfms2U2PSKvAEJqcJUyidJRXKTr0jIbQe0qFZjoJ5jb6+uVj4JTAKdY11+tbOcVq2bPSGG0zvhU3s+kRSE/wCsYTeRoD3K0K+9A+hY416QmG5+1uC9kx/KWlf9YtA6c2TMI0gAwNfgbdQRHIRr4fXSzjG6tu7vwonCKY8lYFnj3gtyO0ob4bkfxMNn7nDY6zxs4fOiV3jVs/56Nw/+EG1cBJiDOoI0ts94QR4b72KMeqh0QzgtPyurMt8W+HjsD+0aEE8nKd5B+aLHWuImBX9G8V3dP+J4I/8AFFqsBR1BXA315nmLYFZQf8R57WK3tDPzaEM4HEdnFWzaxdhN7RjE90r/AMta2fxsbavK7KkA095Uzs80PJI+RtUMx3cyhodlDy+Vs7MAQAhR2ggRYg7RP+JigcCbyerigpUJBB56W3lPIfG1O0uraMoWtvrCog2Ms3zfFPBp73rG/wCXK8tMHyBsQdohzZ+UP2EeT/wrdFJki2ssbDQ2qk3jHFzRQGsTXwIjatcjb/N9eVj7XEnHTMZMVV2n86grT/eB1sQdoYubCoHA5OTgrP5TMHW2ik7jytWxri9xBZ0OIe0H+KmZJ/8ABY61xux417VRQPf/AElKNf8AhIixW4/THcFCOCVA2IVh8p6WwDTa0Cs8fMYJGZ277ncA/wDUup/95Y4x6Qd8pI7fDNEsEEAoqFIn4hVjNxqkPP8ACGcIqhyU0uJPbNmZkED5W7KVbHpaF1ekM+p1n/5HNqgkri8iI0j/ALrrZRZ9IO6yQKvDFWgk69lUoWB8QmxBi9GfjQ3YZVN3apXiNbaA1tGzHH3CTpAdui+W5P8A3bKh0/7yx9njfgJ4AuVFcwSf46RR/wDDNitxKlds8IZoKkfAU+oi2Qf6WaDXGDh04Qn+0BQf8dG+n55LHGOJWAqgAIxVQCdB2iij/wAQFiCsp3bPHqhmlnG7D6JxgTuLYNwPwsks4ywe+qGcV3QszsK5v/4rHWr4uiogsXvROf5KhB+42IJozs4JcJ7dwUZG2otgTvbG1trEtuJUOqVA27CSBt8rIuB2KgWlcbxOtsIn87dZfD5W3AJ20s11GxQcTbevW3URrbeXpZJWXBAO41trKI1sIQR/W2o1328bLVTa0LkjlbI6i3RTbcGLRRRouI8LbCSNNyLdAco87YBzsk60ExuOW1ugBGmlsANuh1skwN1wUz5m2ZU9Ld76RbNedklsuABO9uQgTO9hIjS2tzZrJ1yUAjTnYJdM2r2kjXwsYgRobaMTaJATglAIpkI2AFtPU6HRC0CPHWxkjwtnKw8o2RLlJwuxgKzBsAjnFjDbSUaAaWHO1syjff8AG0crW7J85O6I1F3sPvJWpsFSdASNrCGnT2ZbIBB0jlYwBOgGk20oa6WiGAG6mHl26Rqa4KOjUv1ZhCAskmBGtiNLg+56C83Lzortp2ah0krWhsBSid5Is5YHObai0DE3opiRyTbxuqlvOgeu+tpkP076FNutuJCkqSRqCDuLQ7W+jNgYVxfpKWoaZJJ7BNQvswT0EyI5a6WnIwExAsGUCdQLJ7Mzct0mOyG9lG/DHg/hbhoHBcN3lC1jJ2jjilqCJkgEnQE6mNzvaQ9hFhCgDby2topgaHSzMjyCyd7y/dcGYtuARoLbgbC2xIG+tiAIaLUWhdb/AJXDHlt94NjPu+VizJy1rySPaAUPL+pNjOvOyGyc7rUSN9bZOu9tnTW2jyjz2s6ZYZ5W142yYNsnraTQouWaEWIXskqpFrTuhQcHuM2PconnYB9Hasrb2zpKfiLEy3CgTlIQRhUFPMSLckachNuKRZXSNFUzlAPmLdkEmAJ8rROyfmjlb2rmE6j1YgPtZktEicqtwdxtPUWiDGtNVVl206KW8lO0SaxmrdbZ7M+uBvcBS1ACeomTAkWlW7EOVtFUi8EAQA81TTIQobFRjVW2mw8YksDGLLLVU4cpzNkq0EyI+W/ytyNfhTquodODYGw213utCmmbG0sOvNRVju6E4iv25PW7iZ9RKFO1hL6G3oSUltpboClIaJCtG5Ku6CUpJUWTxFevNi6K697n4YYPRW3fSuKo6x55Lwpg02rs1hosCYCQQiYmJjUhzXzidiVUtwUK7zq0lSnmqNSUttLMqV2jqiEJ10iSrX2d7IqaavfadqcSvFKCoBNFQjMFZoHZqKhLkwRIyCCZGmYCdgVPw+NOcz7eVgkKt2bIwWCqSjj5d7aEt1fFHE5fSAHSze7oQVjfLodJ21Olss8qnhE05UurquO2D6J5S1FymRR3clLCp1QAgZAEnSE93TTS2W5rg4d/D/8Ayj3l6/z1T1eYUHFlTQKQqdQZ1t2w0AlJicumgH0bHq5tindAfdS0snKAo5ZPQdTbmmCFpBDyPNKwR7vr426dsheLhaXDDTqtttZRGUq015TznlH6+NhkII1CZI5kdBtHy+tRUNACEKhQgDKI5bfG3SWURKCBz5QAOXyn32fMeafRdsOrZhCgNwe7Ikz4an63scp34y50E6bxr9b2KJCSQG1BR035ba8uVttZoUEkpn2goROvl9fG0NSkldqqMyTOUaAySN+XusOHErSQVKGUCTvr9ac7IyXVg5pAB1E6jx2saS64mJJMwZj5f152ikErar7ySDpmnUx+e/L+uABI7yyRzER5WJofaJhJ19oGdrGEOEApHI+IPy91olSFkKNUnTNrI393lbsJGbvEnXQE9frnbhKkFWRSpJOutuoA2KSN9Oenj5fKzXS3XTcEhKVHTl9fX3WUmhrMnXcT9fQslpISuQIOhk9LKbIUUAAEc42/rYL3IzAhQOoAkb7gR02+vkISmII02idd+mxtwAAkETqRoPv+uttaD2UgEgASJ90WDdFXahGgIAAiR5W4zDUZRJB0Enl9fO2ylSBBOp11525WmJjKQkHTcz+VkHJrBYDmJAOg5qGk+62JSQhSAkE+HMG2kJJASVQDvpvbsZMw0zTuBaWZINCxKlRpA7pIO/1+ttgaKATMQBB05/X9bbjLJUdY0OWOmnytzMgkCN/hZsxSsF1KJzK5aGfmR9b25I/mA1122t0QZAlRJ130H1FtEmAZJBMz9ctrK6VliU6aiYMEA8o/rbYBUNADpzHL6i29IgQDy8vr7ra0UkhEQI25fX10srpZQszCQQZ06WxWaRm1zawZtmWMpKtQZJtikiTCfa05QLK6kGrFFM9IOk24KzBPdGmwkC3ZUCsAkQBsNR/T63sErVYTOm6tBr9aWldRLViElR0SASdQTbshMQSdNBPPS2IkLBBG0CTrbQSQMo8xH3zZXUcq6BSqZTuAIOh62wqCSTuROv3fn8bB5SATmk767eHjbM0kHLJ3GtlcqWVCBYJg6lPe+fP6/TEqnULO0z9fW9uQRIO8aax9dfoW5zEJ0ImAd+XSzJWCFzARlnXTnv8AU22EoUkhQBM7lPI2CzJTosHUaEk7aT9edsUojed40/P6/N7lItBQg7JKcyEAEHSBHXWxpm87wYEsXjUsj/1by0/cd7EEHXukzNu4TlKTpG3P6/WziRzdio8Np3CV28WYnp1DscS3sgdPX3o/8VjzPETHNPOTFd4nnCnQsj/imzbGpzATvtvr/S21AJGUp8ztaYqpW7OPqoOponaloTxZ4t8QWpBxGpR6Lp2DH/J+NjzXGnHjUZq6je1g9pSJE/8ADHjZhBZ1GoI3gW1miSspPU+E2IK+pGzz6oZoqc7sHopKa4642RAcobncJ5erOA//AITwPKxxnj7fyT9vcd3L/wAinEfeTaKUqjcmddYH1426zqQNTpzmJ/rYgxOrbs8qJw6mPwKYUekC+NHsKtkTEpriP/dmx6n4+3WrR/DtUjWPs30r+8C0I51d2ZHIjeT0torOsK0iTp93xtMYxVj4vwoeyqV3wqfWuOuFF6O3bezXU9m2Y/57HmeM+B3P7yorGZ/npyf/AAk2rulzU6AjlpGvS3ZdSpBHOSJ6+FiDHKpu9j9lE4NTHqrJM8WsAPaC/cp2IVTPDX/hix5riDgl7RGJaIHfvrKP/EBar3aHMCCDOxjf7rCIdIGYaggxztMdoKgbtCH7Cg5OKtM3jDCTh7mKbqM7f643+djbN83NUa097Ubv+R9B/G1UA8UjMeQ38Ldl4HURAnf8eXWxG9opObB6qBwFnJ5VtkKbWJQtKh/hM23lIExapCH0NglshMExACfr9bGmr7vOmEMXpVtRqMj6xHwNijtGOcf5UDgJ5P8AwrXR0FtAR0tWCmxtihkkJxHeXd1M1Kz+NlFniRjJtP2eIakEie+Qr/xA2mO0UR3YUM4FKNnBWOgzbIJ1mbV+p+LGNkGF34XP8zDX35bHW+MuLmwM7lG4T/M0PjpGlpjH6Y7gqJwaoGxCnQjS2iNBFoXa424hTo7Q3ev/AHFj7lfU2OtccK0D7e42FRuUOFI+c2K3HKR3NDOE1I5KWcvUW0RPMzytGLPG5lyO0w/5xUz/ANFjbXGi5lf3t0VYMx3FJV98WkMXoz8f7oZwyqb8KkAjTS2iJMmzHTxiworRxmvRPVpJ/wCqxpHFXBbioNe63/nYV+E2MMQpXbPCh3OoZuwp2Eac4to/UWbieI2C3YKL7QOWrLgj/lsOjHWEHTKcQ0Y6Z15fvi0hVwHZ49U3d5Ru0+iWSP6W0Rynaya3i3C1QYZxHdiyeSaxske6bG27yu5/RivpXOmV5J+42IJY3bEKBjeNwhjAi2trYClWqSFeRm2bH7jYgIPNRsRuizgy1rawfbSUx8TYyI5WKVi+zXTrzR9oEz9/3WNTNnGqYmyyR10trz52w2yZ52mBZQJutTrNs2E2yTrblci0gNVG9ltREwNR5W4JGwBtsmRpztogDcx52Khk3RCmkds0f4HVD3HX8bCmBztwRkrXdNFpSofdYRPZpUVrbKpjYkbG1eZz2x5ohmPS9lMAX1QlGspfE7KkHlvaMsV1uH8U1T9Ca9qoDCRT113heVaVAyc5BCikiIiARvIOkjBlpb6qjOpDqiCDnJJ126QLQnxi9G+7uJd5pxPhrFVRcN9pUFmraSpbShBSMpQpJQqYJjNMDQb283xTtTWUVSGVlOWMO/O/Qgj8rQZC0tzMNykl+gum7gKO7aJuhokLKGmGx3GUcthsAdTHxtEF+4ixtdV5uX41w0U9eDLgoEqTe4qqdpAJU6rsktoUnlmc7ydU94JylU23fw2xPgO54vjEVbjV5tta0gppqNU5MoRncKirvEwYHtTrlSmyFT3PfF9X+HXcMLud2jQ2F0nZ0ricsAlTjqUaoEQMqk90pISCqLZtT2tbi8nAo3ZWMF9Tv9lYipQwZ37lV3q8OYK9ae9evhwVPaK7bMaGc896YqDznnbLT07gThE46td5YdxG7VqUS+5+5anvuT3joobmeQ8hbLcycQk+f8q3wWdFBH7SN91PCzC6W82ZV+hRCZB0pXQNj1Vt42p9wmerH3qY+sPFJeQrL2qj/tIMz59LW3/aUOBOA8I00qld7Oq0J2SwP/i+dqqcHWypqgfcKSXX946rBgdfZ1529c7INDsNjzdT+6r9o3EVr7dB+ynBg1aSnsqmoSsHQJdIkTqLGU1F7wkKvOrmeTy4A6an8bD0rUkhPeBkgAagdT01H1yUUspCMypkDOQdQI5z9bW7gwxn4QuWbM/kUVRfGJGtr8rjJghSyoRr8o+8WHRiLFaQQL3dzDTvoSdtuVh/VW0d0IG+mms+X199hAwEk6SToBMAkeJ+tB7hmlhPwD0UxUzN2cVprFWKVGDXIWoEe20mT8o5WNN40xQ0hUrpVmQYLZ0PlPXz+63Ip4ICkknXUq1P0frnbXqiFRAjkIAMD+htHuFM7dgS79UDZxRxviJf7aIXS0SiNBAUP+rpOsdLG2eJt7ApR+6WN5hLhEn4GyP6ohw5VpB2khMdN45W36kjQGNDsdvl7vjZjhVI7dgUhiVSNnJyI4p1MlK7jBlO4fiOp1TYdPFunQftLkqJTz7RJgz7vocrNY0pSk6k6+Ex9bTYF6hbAkDluFbRH6WF7Eo3fCpDF6ofElq8/SMwlctYilvS5b6+0QHQppppxIExBlYM/XPR30PGLC9VStP+r3kgOoCk52U7ETOivK1UOLCRT4hpwkZZpUlJA55lDnqOlpXw/SB26KLMASqmbJnSZSOlq3/T1G9x0PqrIxyqY0bKZE8W8JjQOVgJM606uv4mLGWeKODXJi8XUkclUzgPxy+fw6Wif92JUAlCCDMzGp167WEF2AiASdYT3Znf9LRPZijOxPqnHaGp5gKXEcQ8HPwEX8wgnUhSSnUyOYsYRjfCjxTkxFQ6TMvJAHxOp3+jaGTdiUHIlESNp5bW4/dzcglIAUIOn42Cey0HJ5RG9o5ubQpyZxJh9xOZu+6JUDRQqEH7vxsbReF3KJyV7BJI2dST4c/D5WgIXQlZClogA8+Rg87JOIbjbN2jMhKe8IkjTQ76eVgP7LMtdsn4VhnaJ5OrPyrNJcbcTAWlQB3GutukwSkAwU6mOnl7rUTNzOoZxC0y842S52iileUwUDXQyPMW4oziK5GbnqF4ivZoq7R10Jr3QJUM2ozbDkDtFqR7OO5P/CsjHm82flXw7oKVE5Y5D7/G2ZSVnQCNByg/DwtQ+6cRcSW27xWrHGJEKfp/WWki9nzkBUqAJXA0jbw6WNU2OeLlHhWorE8Rb+VVracfC3atThQYVCQFzoJGnnaH/Tkx2cFMY9FzaVeiDIVl0gbGdPxtrJJGbXzHyNvPZv0kuN93qpEDiHUu/ZCUuUtOrOqSCTLcjXoeVnfeXpLcX2K1qhuzEKFpQ60h5bt3sqzZtx3UDl+O9gu7P1IOhCMMcp7agq7ZICYUox8PKLZlG6c0zp+Fqtscf+KTKJeXdLmm7lEtPjqErHSyW16VnFf94rof7PYaezVCkolh9spaBMrVDx5+7Q2G/AatvT1RI8bpT1Vt/AaxAIJIPX8rBN5ZnUk6zHW1c6v0mcXtU3af2XutYQMy4fcTmEbc4nTXW3OHfSxvW9aJVVWcPGachzIUN3sVcgebQ8R7rQOC1bfh/KmMXpHa5vwrH5SU7wIg68uluAFE66j2o52gCg9Lahr7zcov9H9elticzzde2sSOQlKZ+Njd1elphCurXrvqMI4hZcpwVKUkMLQqI2PaAn4WGcLqx8Cn7SpfnU7KkQE6dJGptoyQAJTsY5D32iVr0mMCLJbduq/mjMwaVpQ+Tp2/Gwa/Sp4QMPFirvC82SlWUlV2uKAOuncmfjaD6CpZuwqYrqZ2zwpdMqJSCRzkdYtpS1yZ0E6D7/rxtFdB6T/BO8nwyxix4Oq1SF3XVjlrqW4sof8AlBcGkDI9xCu5pQMw8lxv3d5I6iwzSVAFyw+iIKqA/GPVSHDgUQkanSPrfzt0ToCUkwdI62ZtLxk4UVlMKtniJh4spJClqr0JSI3kqgfXusoUfETh9XJD1Fjq4Hkq0Cm7yZXPl3rDMMo3afREEsR2cE4QtciVAd4zrEfWmvhbtBgyrQRvGvw87JTeJMOVaiilv67XiDJSirbUR8D9a2PNVDDolp9Czr7KgZ1/pYZjf0Uw5pO6MIUJjkByj5/E/O2AkaDSJ5zHOwYWFHMoqgHkeWn18LbzqSlRHvnX3wdrRylEuF2VBSgRruRH10toqC+6IgxB2A/KwfaFQEjz0PhOn1vbAtJMKEx3QFcvD5xZWUbC6FCiU59QowRPKLdQCSEg/pv9edggrMnvE67SLaUsAZSozEAcjvaJKmuwYICY1OoHwttJEzqIgSRM2DU5IOswdTER7vM2wuc8uhEifrxs6VkMleVZIUQRsR99ulOKyxlkRAnXnzjyFixWCISFaxuRB6W0FAR3Vd4bbkT9CyToyFKkwYMx1k6c+dug6QmZGus7/W1ipdSUwhwCOXIfUW2pxRJJ0O4gaxy++zWCSNKcIUcsmddtBytovpKjBjoQTOuv4WJl5RkgzsRJ6iwangYO2mYg/n7trSDVEmyP9uToTqfZ12130sEqpHsyRrIJHlNiaqlJ0JBCvjFgXXEpEkiZJiY08PfZw3VRLilNqpBVMgbETNjKKrUCeUab9bICKgSJJ3MTsff8LG0VEphIJJ1B8Pj7rMWJw5KwqjKjm55o5/hbaakwVZ9dBqoiPGyUupKUyVagxG2vT8LcqqFgSBqNJmPo/XiW4afNZLPrJjRZnWTGtsTVd3KIEDUn+vnZFFWUJ3X+m5+vDlbBWLygKJEHy1suGlnS6mqBmDHgJ+H11sIKyEgKOaTAmZ/TnZvpvA6lWyTuDpYRNcEggEkHQTp9fpaPCTh10uOVatSTqd4Jg2Lmr725lMbaT9RZLXWAolZ1Gm02LqrkhRVMfAg6aizhlki5ORutGxWMp8dvz/pZTo6sFtcFJnfys0KatzDTNodROtlGgqVnOkq0KfcRaBbZLdLpqydDG068tpsEp5gnVttUn+UHz+6yaag6EqV4gjQ2CXXJQM4OaRKoO4+hZ81gllBSsmoabOdpCUET7Jgz7udtP33VtKDSbyvELUe41TVTqVK2/lULI1JU1d6KPqaQloEzUESmP8P8x+X3Wcd03PTU+wzuq0W6rVa4Okn8LMJJHGzSmLY2i7gnDgdnEVbfTNfeV/3mplkLy0i61x1pRUIClhRMkSYjTztNaVZkg9QCLRth1k07YUBEQR10tIVAsqpW9ZKQU/AxPyt22FROjhAcbkrk8ReJJbhGDrbRVp525nTe2iQdd7a4aLLOJXRX15W0VTrtbmfG2ieVnAsok3XW1smIOmh2tzOsb+VsJ5mzlocLFMmRV3RjK5L2L1zXh69d7qlKbpnSntGZM5Ez7SZmNRAPQAErXY9vy43xTX1hvZIKnUdqgBJMaDIoEyU/xRvrMAviukobcSfYWknlodDbeYwRy3g7W4ebsxiFJK6TC6xzAfhd4mj6X2V1tVG4ASsv57KOF8fMI0rymaihqUAJIU4mHFZp0AToIy7nltrvZZf4/wDCu80UbbN8qYeeb7qC0SJy5ssolJVoRCSTNnU7hnDl407tRfVyUFQh320rpkFThG0kjxPxNilbwywFW3EXWMMUzKHKz1tbbRyDtu+kKJTEwHFGOsdLYsrMcjjk45ZIG3Bdb8bIzXQZ25QQmO5xF4fioXSoxdS17j7SnBSsKCh2YgKK1L7oyg7CI3M72MUmELqxAh69KW6WmrwdUGVoDhabyIMJBMFKgmVQcp1UYiZshYj4HcN27s7Wsqa+7RSNoWqqbqsqWw3uvvEhOkydI5QNLMa+cLYyuXDVy1XC2/7/AE0ja6pp1uoS409eXbrCWsichyjKlULzIIkuSmc1vKMQpe8zNe1uRx0NtB+FsZ3NYU4rxuTCtDeFVRHihTUBp3ltGkRWUiE0+VRHZhJZJSExEE6RbLEWeFHEBTKFJuTBNOCkEMvUNM843p7KnOz76hsVczrbLZ/cJvmP4S4jlVP9pNUpbuHA7BOUqrq1QBjk0z+dq28I6eWrt7MSDVg6bATBB9yp8vdawH7S99xDGAWCrRTt5LO2kJpgNDrz5HwtBPB1A7G68xzpU6lUE+z3x9Tb6R7JtLcNi+/7rP7Q61sn0H7KeKenSIUsSSo6DkdOtjrbYJC9DmICoET0tlM2pwDMgEkHNInT85mxxLYKc5B08OfXT+nhbuFygRfIrMogaROkR9fl8BEtNgxlJA32P1z+Fhy1kMqGmggE727CTIzJlKo5x0G5sk65CJMGTCgY3M68vra3RbkBAUZIzHymJ+vnbaUFaQsDQp1HP3WFSMqJ3BkwTt5zZDRIhFyjvQgRBInqT08bYWwO7Mge4T+n52M5MyjOm5nr9aWEbQQsyCDMGRyHM2kHFRyhFcpSc0kkecxufrzsAtrKMxIRzKuWn32PKbCiEqSCmZ328PDnYFxAyBKhA9k676nw8voaECE4KC+MrahiOiISdaQZknYnOrY7c/6c5cw22n9y0BKSqaVklQH+FM/1tFfGhJRiChJghVJ/ESNlqJJjfl8JtLeFU57luxYUrL6ozrm1IyD42gz3ipP9wJZbaISUlsTvH3/WlhlNKIyJza/ETH142xtGcAHQ/wAR2n4+7ysaaQYOg8OY+trFQwEWLQ7xJMnWIGn4HT77dFgwEwkbR4fp+djfZj2tI156T93LrztvsUrHsEDw+ulo3U7IoWUpJ7gzAyZEST9fdZMxAwTdxICNFpmY67n4ffZdDZXCkchtoeX52T8QJX+7nNUg5kED3jW0XHRTbuoZviA/eFIyAFVLzWYdEpRJPjsB7+Vhb2ok1d03YyhP2joQhJUdgpOp+HzNlKqpEO3teb24ap8gB5lSdfkPn4WTW+1rKq7lQezplMsmP5imTGmp0A+NqGbkrFlzeLTF33g72iE5fUuzSOW8DTz/AB5WLUNMheGkpyaBtQIPvH5Gyte9I1WX4xTqjKhBcWAYzZRIHjqQSPKyMmqcpbhbpaZMvvFSUgEEAZjJ6xFptfYprKBb3pHmK5Tysqe3ddcKdw0M+VKd5OieelpRormYo7rulpuCpTiHXFlMFaiJ931Fo6xE4WqhtDolP2kAkDXOSmeca9eZ87Sqwkqu26Vbn7E9f4Z002P4fCBsnA0TuqA1R0q6l0oyNpiTzPT5WRsN3emorK2ucT31wkEkbSZge762so1ua97yRdjTn2KCHH1Ae0BGnXpbq4WQ1X3gltEDtIUEnQROnhaRdchK1gha272QytISkqAMHmRGv3fGzWomE09xrZbQO1edUG4TrsNdeY39/us/H2dCHEgI1EnrtZGue7kVdT61ADFNDbKDBM/zH6+60HHVSaNLopdmHG7vo0oDaO1UQtxQT/F+m30DZKu66kJxLVKDY7wV3ADA202s+XWgDnI9rXaIjn91kFhnJiCoU6YzJUdTpGUbfP6FmcQLJgCUUvWjFJTKAZCVnRIVy8fAAeFmfUXE072knvkZySYzHX8/nZ53i4qodW842CkHKmfd18vqLFmbue9Vqatxak5Gzl19o/kPxsF7s5RWtyjVJFwYWpmKUVicgdfkkjUAdBOu/WzB4k0IbecEGcqenvEDXl89xaZLsSk3WxBJ0UN4jvfrtaMOKzYS444dEhgEmRroQJ58jItZAGSyCb3RTBN3ovLBbqlnMkLdSYSCSNT+nu52cd03MmiuVplLIzrnKnTNr0/LnvYjwkdQrDdWHmwsIqlSkEHTKkhPnr8xZ+XFTprX1VruVQbgITyST7unjy8bDIBUwSBdIl34KapmnHXG0l92C4qNhyA+P1AsYOHuzWClJSdwdR3hOvh8vyea2kkynNlnQxEdD+FuCyAdUcpiDKfeLTDG9FDO7qo9xFdl4Ju9Cma+rbhWYFD6kyNddCAN7GrmcxUzd1N2OJb3QpDeXMK51K9CRMhXn9buW/KUOXevKjREKk/l749/nYG6adJu5KSTCSrKOe8zPx+tLC4UZdqFMSvtoSiTN/cQWyezxtf4O4AvB7QjpKo6fKxhOPeKlMO7ja9lqTIBcqCsnror+ulj6aNKBJQoZRzgz0H3WTbwCQU09OkqcWSICeXTyGnwtF9PCfhHopiomGzj6oGt40ca2F+rXdjmpLuaO/TML0I3lTZ0++OQsoYp4u+krg/hg/xFXfrT1C1WNUYfqbqZCHXFSSlMIGgA1JO+mpmF7h1g7C1Vei3MXX01Q0zKO1VmHffXE5EkAgDqTy+QvpT4oocTcNqy4KKrpX6dllK2GaRRDFOlsQlOXROaNNtLUZKWHihjYx56K0yqmDC4vPqouub00+Nj1DSVFQbgdLxhal3eofxRPdcH3cjpaa7m4/cQ6ulpqiqoLmUt5tLpCWXEjUeKzHS1G7lX2V00iyUhbLyiNJKe94ef3WtzhRpp65aFUJI7IAGOmnP6/EU9BT5tGBWoK6e2ripCTx3xZlh24LscgScq3ESOe5Nkq/PSkvy4KhltfD9ipQ6gqzpvFTeUjSILZ/PXaxJujbKMyZlQjMNT8fePl7mvjS52nfVnSgfxJChGpOU/He1Y4bTk+6rPf57XDk6GPTGXH+s8M6hJ3IbvZKtCd9WR9T0saR6ZGHf/AJ7gS/mlRPdeZWNh1I18LRA7c7JQSGQMoBjw2Aj3jpvYBVxMKSW0tp7o1JOoPL36j4D3L2VTnl+VH2nUDmpuR6YeAlJHbYaxM1lmfsGD16PWNtel5woWtCapF9UxIMldATH/AAqMjpavb2HaYBLgZAVmgDLAEdAOcWKvYcplgpI12MEg+f3mzeyYPNP7UnHRWcR6VXBgpleJ6tnQBIcuupHjuEH4Hpbs+lBwOdjtOINKwSY+2pn2yNBvmQLVQqML07g6zCenWJJnx+jZo42wkGsNV1a0oyhtLglKYhKhMabxvEa++0m4LATuUxxidvIK9NL6QnBeoyqa4mXCkL0GasSjXc+15iy7S8YeGFWlKafiFhxcjSLzYk/80g/lbyjCQbt9sgtPkAQJOYfH+HytJdzYeYqqBiqzD7RlKylQkAESZ16CedmmwOJguHFPHjUh3aF6UMYzwvWpilxJdT0iU5K1pU9I71jib2onATT1jbo1EoWDO/IG3m8jB7Trax2bcKVuoazHL5WLVGEG0oIZSEagHQjWfDw/rztV9jt+b8I/tk/KvS1FQpau6k/zbG3K6gpCcxUDyMxH19b28z3LrvmnQUUF41dO4ASns6laSDsBIIj+lsums4puVCGbpxriRnZIKLzqEADzzac/CzDBbj3/AMKQxgbZPyvS4ViidVTqBBM/1sIKs5j3zoNzbznqsY8WsPIqUq4o4peebpi73r3dUEHKCAApWuubWOVvQv8AZ3XhT8WeCV4V3EBIvy97txA/Rqq6wZnuzNOw4hJOhgKWuPfajX0JoYeNe4urVPiLZn5Mtij5qEwUp0OsgdOmti5qiFHKdxsTv5bWss5wqwC6sFWHGgRp3XXBpHgqxKp4LcPHUn/sVxGsyiqdH/VbD76w7hXuICq/U1WlR9rSPM/W1lGmvBKVxmmQR3Ty52l1XAjA+cqbXejRIiE1UwPDMk2S724DXOKCoVdOIr4YqA2sslamVoC4MSOzBI20BGnjrYT62MorHC6jV683C6mnZK3HVaIQg5iT4R/TX4H6G4nqpSKm9FglGqaZJ+zH+b+Y+ER52Gum62LraCAkrd0Lji9VrPOSPHlZXZlXskkA9d7FAL9kYkN3RylaSAlISBAGg5eHh+ll27GSVhMjTWI2sl0jXf118J+tP0s4rsaOYCPr8OVtehpLuuVm1U9gnJdyQ20kZrPK63SqkE8o+YBsz6YykQIB3s5bidzNKTGwAHun8xbsKZuUWXLzuuSUqGB8banlOm9sVrvbU8totcVRbm2TG/W3JM79LamCRFkkuiRFtkwIm3OYDf3W0SOR2sRouoOKDqgVU6wD/DP425bV2gSQfaiwiwFAid50sVo1zTo1hSZBM7EGzSNuNEmmyUq+oKECjbVlSkArG0+FlC4FdvdlVTkAlBzDnuP0sgLJJJJ1O5ssYUdiqeZ3K2iY6kEfgbZbaFlPSmLfmfPqiGQl11H/ABGuqoqVJqQhD9Oy1nFKoEIUvcqXyUAIhJEAyTJCSmMLzvXE17KqLwF6OpZpHwXVKJyoQQo5UnRIVmLZyiSRrGk2mrHN2P19E7TuVK6OnAUlRa0dOh0kapBMajXe0d3Fd923vclbhp1TdGhNZTpT2RgqK1wkRtEiNAJ6zJt5RiBMMksnC8LXN3Gtr62C6mN+aJrk3W71xU42lxqVoUkKSrt3u8Dsf7s/fbLOV65eKSnVqoqinYpyollputShLaJ7qQk6pAECDtbLQ7/S/K3+fZQzu6/lUG/aVvKVeWBGEgZkNXkSOYzGmA+vC0QcIkZKe5pEAuiIGhAWB9eZtJH7Ri8k3jfOAXAsEO3bVVGUnRWdTUjwMDf87R/wUSU0tymA4hIlBI0iQdB7z7yT429I7Li2HQ38/wByqOPm9ZL/ADkp2aASNikpAO+qdY+jZQQlBg77FSYnTnYGmaTGpEATO342NMtpSoOJiRqe7rH1Nu0K5Zi67M5YkmAYB2nztgbymQM3Lp5/hYSFAaKEbQR7/fbbYUUTOidyTr4/cfrdlPZBoQgEbmTHiNNP6WES0kiQiSrQfXw/QW6y94AnbUbaH6Bt0EEJIBA0gac+k/W1m5p7LhSE54ClGFSRzjrtboJU4EqKozAiQNhpr8bCBoklKN9Y6+/4W6S0QrMCRGsxtvPO0m7qB8kGpvPBUIM6dB5xBPLf52LuAgArPdXvpPWx0tiCDlOgOmgGvv8ACxd9sEEpOkjnqfHxP1pYgKE7RQZxqQE3/QqOX/zQxzg51a+7r4Wl7CaALguskpBNIyCUjXRCefx+t4e45PKYxBQKUTrSGNZjvqncdPqdbTNg1Pa4ZupW00DJI13yDT7t9bQabuKd3uBLSQZKROWZAj4a/X42MBJI3WDI5azvrP1zsEhpSITm217p5yPzsZQjuyRmKRoDpE8vjH5WndIN6LBmTmkjWNfztsypMkH4cuk8v6+YFyiFKBSrcpgfKOv1pbZaK0yjRJEz9dPqLQzKeRcFtt2cqBr743EWTL/Qk3O4Z1BTBPPUH8N7KxCj9mEazESfLymCbJ+Ims91FRQUwtChBAHtb/daDz4bqQteyi90ttVV7uOQlKEImTzKNuvhYqpluguq7HFFKQt8PLOpjuk8t401t1eqKh6tr6OTlWQ+4QmYQhEhPxiw95ssv0V00IVlU/2YIgk5con4gx77Z4N1bLbJKo0OPXkqrqMx9aZcdyie6g+yPKBYvddChFG7eJbkraWhnT2UDUnwPP8ALmtXm63R3q4pxSAlqiCQBuCTAjnrG1ilKhCMLqTkDf2CiSeW/X6199pNOqidlX/GjRpn6YQFFRWVACSQSCPHcWk27Uo/svdlQ4AUtNNEZjrITt933+UZY9palm9Gal4q7J5aQwkz/dhIOY6fxEmz6w2upvDD7JGZNPQ07bZgxLmg5dPw8bO86aKLVI+H6FVPSl99RNRU/aKJBA6xB56j5W1c6CL5vBuSZykz0/rZQuxSnKKnehXebSYI8Jj77J5qEXbfNY45KR2IVrpJ7unhaDn2AUgCSu72ddqH27sphC1n7VSU6BM8/G3GG0LS1UtGe67IEb6Gxy46NSkLvCpb+2qSdDyG+lgsPqIfrG0ggBYJnzMHX6087QzG4KJawsj5aK+6ogAgaQTGo+o/OzdvbsKeqdQ2SpxyM5iMqQBp8RPw5WcNfVikpy4QS4rQCdvrrZqOrK3Vd2XHPdJ6WhLJyTxs5oJqkdrKhLSUwZIJCdBG+mll6spG6W7FtgiEtkeY+P177D3TdvqlPnWgdusBR5EeH3W6vQoNE8sEphByab9NPrn7nYLC5TO1KR7obP7oQlaZIKok+PT4j32jjiqwFtKKk589OSJ5mTy56H52kq6gP3bK1kgFYM777fO0XcWq0tVtBRM95VRmSokR3BqZ8ADr9G1kP8KCW3KT+EKXqluupW5Qy26HVkARqiABvqcvwFpQwgo9k/3CVSkGTBMg7cuQ26jbW0bcEXgp++KYO6lDK99J74jbTUm0l4UIVVVCTmAgEEkDYxH3fG0Bo4JyLhONLZCipSSeY1326/nbakhw6DWADzsOlGkAgAwIgQr3e/5W0sFMqCASZGu4672IXWQ7JLvBkqpnwRMpzCI0ixK4GvsXACkd8qM6GCBH3fA2W6hsKYWI1UCkTrB26a7WS6JtV2UxU6lQccOYJHtJ6DTn9eAA99ijMbosq3Q0kpaBUpZhISZPhGus6+VuWKBDRDrywt5emugG8gfP42NUtEsE1dSk9sU6g6ZQY+vo2HWhKjmKQogzod9R+vnr75NN9SmcCNAiLzZ3TGkakAacvo2bmM6P1nDN604JhdK4BG3sGB152dikQ2nMZCdo0PXp52SL3Z9YoqhkNkpW2pJ25jUx7+tjB10NVRuBIXcyykat1RV7MwCgfltztbjhs8iqwrdz8JKS0CAozlJE/cd9/K1RbkzBm8KZ0GW3EmOcDMAZPP3aT7rWo4K1SarBVGAUnswEiCemgM9NeVqE+hV+nN1IiEgJAUAQQSTOpEneOW39bI2LqcGibWmCpL0qy7TB2+htZaSTpBMTATMc+envslYoYcXdayNezWlUGZJ1n8dPO1cHW6snayZi2gV51AFIgrIIPL5crBqaSVShISrLMTPTY9NbGiVQViNeih8Pr8bAkKIEqBUvUEyDp4jz+fvtO6FlKKvNlXdciBAJ2B3nfwn6mxdynASI72ZRMxzjWden3WOrHIQpKRIMz8jz0+psC4DEEAzqSZUfv6zr158rSDkxBCSnWoKgITExlEyfDf7vush4hoVVWH70pkmC9RvBIAgHuqgT5x9HV0Og6JKSJEGBoT4AddLEFNNrXkUkZXDliNwRB/H84sVj0J4uquIAVd9W2D7K23B0AkiZ/wB4b/jaXsBn1jDdA8pBSpLYbHkFZZHwn32iembLSLwpCAn7I7nmlQPTwNpO4XuqXcLbaiAWX3G8pXBjQxHLVXzsSY+FDjGqejdMFNwptMxBAJkDbltPv0FsqaVJHdgGQoZun5ef42PUTC6hQQhuSNxMjnpZy0FwBoBdWiVghRBGsxvp5WomSytNjzJrXfhior3gqpQUN6QojU6A6/1jfay8i6mKJnsmGkJ3IjnJ308ffvZwBlKUZUaRHs7j6j3WK1DBSCoAFMSVAamPMeXwtHPcooZl2UW44owmrJKSQ/TwT0MFPlOgtbv9ktfpcuLiJhtxzv09Vdt4oSVa99DzazHL+7b+ItVbHNPlbpnUp0Wl1AMTB7p085+tbTV+yuvlV38cMV4acXlbvO4XyhEmFLYqWiOgJyrXrvali8fFw+QfdEpnZKhq9SlKzBKxztysbmNvlblBOTIRJSbdnYE9Lec5rhdBayLySY8LcOd5JB59bYruqPibcrJ6687VXuRwogxPdqqS+ahoAgFXaJI5pVr8NxYmw0SrmCBtFnrji7ypdNXBP8zSvvT+NmuhgJUCD7RCtetulwz9WJpSnfZHKNGylAizgu1CUictkiiaBIlMjTUdOdl6jQEpA20Bi3WUkVlg1Ut0rMCNJ8tbLlyOFLhRAgg/E6/gbIbO4kTtZTu1woqkkiNR8zH3G2ywWWU83Fk4s3L5xbQOk7W523tkzyG/S1iyAuzANtE6c7clXKPfbWbSQR02tMC6i46LeYz0i250gA624J10+RtudvqLEAtshreaNedijJyrebVoQskacjrY2ZA+VipBTVL6KQFaWdzdFEHVbVI11ix+4ngzezCp9olHxEWIK2gHytth0svNu6koWFAdYM2DK27SpDeyUsWs9omqYBGYgKA8x+dq135dj1ViE063VMtzmcWnVSUDfKB7SuSRzMWtDilphoLvR1xLbXq5zqJgBI1k/G1c78pqyrvh6vraL1elzzTU7iJW8mBC3kqEATJDfP8AiiChXA4g8xVRZzcNP2XTULg+nt0QPaYjGlFganXT/wCxUuhU6pSP4SVz3zEd7nvbLAOofcdW4uuqFKUokkrJJNstR9nu6D1KNlcqD/tFgaTGmEaBpay0xdT/AHQnUA1CgnUchlHysyuF1+MXTd1yPLoHnQzTNlWWO8ZPMxvHys6v2j7yFcVsOsqmUXGDIMb1Tw3jwP1oWRgVtCLgu7MpK1+rwVAbwpQ0A8vD3W3sHlMGFwuG9v7oGKxiXEJQVNdPxNuwIzLuq8UR3vYQfOO9ZQb4lXCtKZo7wEawGQdd/wCbxtGjJCtU6a5QdPrp8LH29dycpSFEnXz92/uNtH2vUeSoDDYTsFJKeIuG1EZk1if4ZNOdtddPrawox7hnNnFY+haeZZXEzt4/raNVEpkZuUkbe4/XW3TTn8JXqTMRI/TSyGMT9AkcNiHVSYMcYXWkAXmQo7Zm3E6c+Xn1sO3jDDIhRvZCTMGUKTp7x5fnaMkEBGZAhQ21Ik/ntpbBlKjISSNdYgjp7xtaYxiboE3s2MDcqV04pw2vX990yIOmZUfH65zYdWJMPLISL8oDP8PbDX3ToNrRGMqRGSQARp05aDe3C0oKspQEzoNDprO48/lYgxqQbtCGcNZyKmBV93MQUi9qEmJCvWExoeUmwLl6XcpJQmvpVyP++STHM/faHVBKgVkp03ISPGTp4jrZOqG0IGbKmBpqkSBsPf8ApY7cZdzahPwxp+JEvSEvClZxJdrjdQFAUiwSjvJJznQxv+k8rTrgOoZVhO53UPNyqgp9AQdChO9qecTKhulvWlhOUlgmAIJ1M/Ln/W0o4JQ1+4LtUlrL/qrZPjmSNz1384sV+KGJoky7oLaASeAHZWYbamCkkjeM31P62MpaWRGYmIJ3133tBbSyJKHFpIJEhagN9R9a2OsuViFQK6rBA0yvLEbfgRYftxp3Z+UZuFH5lNqUrACgjUGSSd9bcKQFhKUCCVd46yPH8LQ81WXoFAi96+I0ioWIO/M+djLd832lRIviviT7T50MzpPOyGNM5tKRwx4+JS6G3CAFEnmfHY2IYhZIu1yRGVSDB0nW0dtYhv8A3F9VszrLkgfEfXxt07ft9PMFt+931pJAhYSIM7xG82TsZiItlKQwuS97hE32A5UX5ULTK0NpQCBqAETHjr93xTqZtdVeFBXLJDbbiWGkjolBzGOWv11UC5VqU8fXCF1Byud0d6NNfjbhHrKAwhupBTTd5AKEwPqdz8LVe/tKOaN3VFr4o01l8upPeRTsdq4J10BgQOROvusRvBwUWEgy7MvU6kpOYaq1P6+6ymk1HbPVPaoLtQkocJQIUnb3crJN6Ui1sIYWpPZtJLaO54Dx6c7TbXNuhmjcBZQ7xUZ7J66lJJByiFCTpkG3U6c4G3lZ8YTpWlYNYDaEguskqjfNJnz208rR/wAYKqpbvOkaecZ7JDSShITBBGk76iJ+Nnnw9rKp7DVIhCmw0UrQAUmR3j+mnjaw+qaGhyC2nObKpDw04hV0UjhgwkT3toJ1+XzsDWpRVXy1Qg5QofaKJjNl5H652Bul6qoKRNIylooaGxmYOw315/EWEpaNVPVtVqVBS0KKzmJAVJ5fOwXVjDZGbSuGqcJROVKR3tk8tIgCOnvsjXQtDV716VqgDveIEz79Dv8AfY2bwqITmpW+8BJC4/CxBbalOVD7bZSuoA2cnTy8TFovrGHZJtM7mid61yqlztcwDY0SNNxGo+uXhYK7Uti8qclMyCSOR9qPmLDLux1xwLUgZZ7wB1jw6GxhxsCuZq2WMqGk5coAJPvJsFtQ1zrlT4JAsEuISMwgFI72ggz+ViN5qc9SqAncIUSE68j+thHL0bylAZeMa6QZ8NPrX4l3q5laFtFh0ZioSE/PQ9LWzUsI3QOA66Q2K4UlzmQEuFZSgH+Lx6xIsyMaXS4xd5r6pJdfcQ5EicgIGg+vus+W26cVIqKptamW0yygJkCOZ+uYs1uJV9XexdiVOdoCHMmrSiCMp0+AP67WnBOwmxKhJC7kEyOBr039eTedULpkkjlIWPd/EOfIWlXC5AveqYSpUAGQobd4T56RaFODla1R44qEuGErYdRojUnMlXlyO3M6bWmi5nWmL3cqDPZrCyFAEASZGmp5b+FiPka0i6i2IkJ6AKTA3MyUawR/S3OYgBOYka6kxA931HSwCK+kVC0uaSQRkIjwPxH0bcmto0JEOglIjQHX6+uVkZ29VEQO6IV5wNBRLkEnKJ1InfSyPdrvrN4rU+SVISSBG2oBP11saaNPVPqfqqlKUtGEJGhmN94PKxK7ghm+M7iClolSc61aGNdLAMoJRhGQFIGFsHVV/IXWVDTjVC2YK0IJ7VendSQfif0s9m8GUraRU1NMM5gtsKplAFPXYwPrlobuevw6zRU1QusoA22kIYYC2+8rbvQsGPPebNfGePGKtLt3XJUoGfR6oByk9W24J7um/uEWpcaSaTKDYKwI2sbcpv4weuamqF3fdTVO6AYdqENwkGdkaTp1+iyKtoOtKacSoiNiY57+X15KroQ6jKciSDoREDr9eNiL7SYIXCzqAAY+vO2vEWsFrrPeCTeyqOKcUWIL+oVpMocUlJCfZhyNdOh/K1h+AdSp3C6k9pmLTxgTMyVaef6+doHxcw1Q8R79SogBa3VeJkJVtvyPnAtL/o5VYcoK6mCtW3NRMwAE7aaaqI91gT66hWafQ2U5D2+6ZI7x05EffFid+JWq6X4Ue6kKBnQAEH362HaUEbKEAQOQI5gDblbi8kLVdz7eTMOzUkBPPQ6j77Vb9VbITEhQbUSCCkb69I5+/TpFtKme9OVEgac9dfo9LYMygM57vTbXrqZ5fK2nFQAoAZTIJjYzB9/62WYqGqDcSogbxJ0nlpt7jy8bF3CDIShSkxp4zr7/AOlhnDIClRMZRIJEDX387BrAGYrVPj7IJ16+X1ye9kxF0TdSCjNmMk/xaxA/IfebE1kpJVm0kyEmI8vl030sfWghBRlM6Du6H3aa+Fk5yCoqbJQREa6yNxy+Hl0ixGOUCxV3vOhFLi+9qKAn7eobTmJ5lUHbyP6WenBqgeq6SupUKUOzdSsmTpmTEf8AKLEca4ddYxtedcqEtpcbqEJIjtApCSr7zPzsucEHQxe16URUkKcaQ4AddEkif+bz2sSaUFhsoRM8YBUyXVd7NIj7BAznUqGm2u1lNORCIUQBvmgGfw6WLMBCTomQNBM+8DX6+VjTeZKgT7QEkAwAOURtr+NssvJWk0Zdl0AcoBSJBABA12/SxWraQpBJWJMhWpy7H8Z5/jY9nbyltXsjYlW+3vsA+vPLeaYEacvrTztNjkzhcKPcdtH1BLqMwyu97wKkq1G33WX/AEEL3GGvTAw4gryt3l6zRr5SHaRzLPh2iUgDqNhZPxe12l0PpTIIAUBmgykgTp4HkDZpcHr5GGPSM4dX2p1LSW7+uxTi9oR6ylKx55VQYsSZvFppGdQVXHhkafNe4TgCHjoIULdQcsg2yrEBKk/w6W0FZkZh1t5aTYkLpBq0FFnCM2k/G3Czp4m3bx72YHysC4tKUZlKypAkk8rVHHWysNSbf9Ga266hCRKmwHE+adfumzHbbBMQI3iNPdZ7XBii4cS1l4UNz1yav1HIHlISS33s2iVbK9kzEizYqqI0VY7Rx/dqKRP8o2+VuowAktLCg1ZLRYrVIyAEpywPvssUoAjY/hYlTtkiefjI8bH6cQZP6/X5W7inaucqDqj7UaAHaxymVkdBHIEj4WItgnU6x00mxxpQDiB0InS2gAVSKc8yJtgOu/jYGnWTTtkmTlAPiRp+Fu82k2sNF1XJXebbS3M+XwtqdJNskSdTFiAWUVu250gW0DztknXfSxGhQcV1uQZOti9QYfZPJWZJ+Gn42HzSIsWrzlbS5rCHEkkeOn42d2yiN12uY1ANuFE8rdr2iLBKj9LCIuFO9tU73w3WXPQ1C0BRQkakTBAyk/faNMQXWw9fKEO0rD6nVJQgPOFIBJ3ge1y+7ykW5HQ9cBaKSeycKYB6kH8bMfHtMUStsHTQkC3DdooA5wsbHa638IdmuwrhWL8IUyjTJuWgcS0cgWEQFAaT7FstH6KetCQBSlQjQyrX52y3M+xm/wC8fwr/AHZnX915kftFqlL3GW62CdG7gZASdv8AzioM/Dy3s1cFXrdVPcN3sKr6VCk0yM4U8kETrJE6fW+tnB+0Bq546tN51IyXDToVlOuVTjxI+B1tXJ1a26t4pUClCUyrUmAnkfceUdbdnhkAlw2Fu3hCq4i/JXSu81ZmnvK6VhJXelICdY9YTry/Hlzso095XcoS3XsKTtAcSY+B0P5m1VkrKSG8w6FJPOZPOBt91hA+4qftCkp2TG48+vLToLGND5quKu3JWtS6wCAFIlQiMwP9B+Ws2HadbKoBTpBOuny89otU3198O5mXVKyaAJJA322E2GF6VvtIraprN/K8oCYj4k7fpaPcD1TmrvrZWyQNsyMwG06/ERHW3RPfSSNCd55yBPv026/GqP7+vplGRF61k7oKX1yDrpM/P6I6MUX822rs74vAJjNHrKwJB6A9fx87IUThzS72DyVqM6T3lD3jofD32CXqBsVLmYHPQ7D3fpar6cZ4padStGILyTAlIFWoyfj5+8RGuopx7ixtMJxHeLgGhJfXI03k8pAP0ZXc39U3em9FZJxzQ6aCT3dJ2+WliNUoqbKW0knYJjbXl0HjaAP7fYxC0n+0NYRCRose6Ad9RHvNsc4i4vQslV/VCiNJUlJM890zGlptpXBRNQ0pV4tKP72ozMqDJ7qpIkL5+Pn+dpbwEQrDl2GFKmlakHQg5ROkfXlauV939XX4tp29Krt6hsZCopAAEzEAR46/dZy3XxKxPdFAxS0NSjsWWwltKmkqyoA6kbDTqeVizwOdG1oQ4pAxxKspTlJJBkEbkax+Gwjl7rH2YAKMvdOqRmmdPHztW9jjTjRvMf3hSLypEE0yRqRodNxzsKjjtjZmXlqoCEyO+wY8van4dbVO6SK02pYFZCAU9nlk6yYB2+/66WFUIJUmDodFJjLGsz00PxtXRHHvGTYSHmLqMjUqYWNOYHf1HUe7wsMj0hsWNjK7d90rIAEZXNRMT7fj9/jEe6SqXeWFWGQoIUkJSSTyKQJHTw+trDFwABMZgfCPfavzfpE4jcAcNz3Yon2gAsSf+Lp9dR//ACiL2ySrDtIoxMIeXrv4Gef0bD7tLfZOKlnVTwtaUAIKjvlgj6PU25LgUhJJVI7xM6fX6++DT6Rtdp2mGKdQkQU1SvgJR+PvsKn0iCSpbmGGxlISIqzqd49nw++bS7tKOSTp4zzU1F0qAIBUB3tOn46mxCqWSCEpmTEknnvvtytErfpF0+QH+y6wZ/8AS5IOw/gncx8bdOcfLpfht/D1WnOJBDiVQQR4Df3WkIJRyUOMzqkTjU+pm8KR5YzAtgmDrIKtvH56bWcnCt/PhOiJzjVYnb+NXnaM+ImN6PFq2amjo3EFpIQouK3Gs7acvv8AGy1gLiDd9yXQihrW3lLbUv2EAiM07kz03takY4wgc0Bj28UlTtROpQgtgTBmTv5aWPBQ9uD5QAOcjz8uvjaMabizhtJyPmqBO/2cx8DzIjrr5WUmOLeGFjv1bySZlKmTI09/3/pSLHdFazN6p/FacxCEKEkg+J6/L522H5Ep0iRI016WYSOLGEnUpzV7iQqRJaVpyMaaWEPFPCbkFN5Ebe02rpodvgfK0cjuibO3qnwp5OmZSSkaEE7W47RMEpIyzJiCT08LMxvifhRRE3qnfUqbWAPCfj99hDxFwrCc96sogTJKgTr8+vvssruie7eqd4cQSc2VI6zIUZ/rbha5OUyTsYO31+fLSzU/0hYVUlSf3ywMp1BBTMfXL9baXj/Cao/7bphEfxHmOfnHhys4DkszU4ahYKJkgREDfeeVmJxHR21wLhE5HUK05jXT4kedld3G+F1lShfVGTOxdA100jlpv/WGxjLEN01dz1TTF407q0EHKlxMjWNp87FizByE+xCjnho+lnHjICHAHUutAjYHISRt4WsPQuNyhJ7vOVbG1Z8D1JYx5QvO1CQ2XVQpaoE5CPhvHj77WJu+taUQA43ooQAsER4662tVRIcLdEGntlTnQ5pnWlWkEAnU6dfKdfEW6UmQc/dJiCInf+vysTafBR3VJWFbkqzGJ5df1sIV5+4UjMZISSZBOn5fCwMxR7BDZgDAkynT8o+vGwZCEg9xKsuswCNt/C3AczxIkAgAdNiR4crcrKoHdkxJEzI8Pz+hDOQU2UEoZSmZns0idJiBp9f1sCtDajqjUTpz32+o+duA8kyqQka6a7f0n4W5S4BJIEjqPqN/rS0muI1CcgELhxumBV9mdpVz005GxN9phMjMJkp7itD5/n42NuqXOdRmI7pJjQ/pPv52IVbpyaKHe89dNo+vdpYrZHIRYFAvE+mpqHGdcptpSHFobX3SRqpAGo+O362WOCdbUZLybYefaKVtkKbcKT3grQ/8E+4WKcYk9niFhax3XaZuROspWsae4ixDgpU9nfd5UqlFGdhCyADulUGf+I2vbw3VVotIp6o6+8sySL0q8hTmOZxXhr7/ABsfTeF4uoLZvN1SVe1KpnbSLItIoQCTBSnlMA7n4afDnNlFCiuCFpz5tTB+P3DyFqRerll03TpSezLiipJgz/KfuGse+3PqKW5Qe6YiZ6aDyibCtAwSQF5J1Pj1NuswBCUjPrzO+34j5WbiJZEVVQqcTCXzvr4Dw0+osEqiW0dHVBI1GbWPd462UFFQCAoBQG53I8dLcrAzEEwNpjNrHT42QkT5Aks0CiAAqFaScs/p1+hYSiuhIXKpUQSYB/A+Nji1JTKVAyQc2pAPv9/3W7pipOfKoZo06aTO3O0XSkJ2x3UUcWrvLF8B9KYL9CCCOUFSIHwHL9Gpwoq+yxmQCQmqYeSfEApVIkbaRp0jS0i8XGULN11KkZkkOskjYgKSf+oxaJ8CVDlNjG7H1riV5B3twoEdOZM8+vjY8RLoygvblkCsU25lgpUSAdkgwBrGsfUGx1oyBCDA2mfqbJTDxy6CEn4zHQDXysptuAqIWEjMZ0Op08/qLU72VyyHSsIkpEzqoETuOvvNizroCfZ2gGFcvKPD6i3bjyCsHRJ29qY/WxV5R7M5iND3jO/kQNOVlnsoloSHiFfb0NWyQe/TrMRJ2MfP66Qnfd4uUd5XPeVPUErpnCtKpGhBSUx4fmLTrXMocPYrEpV3DG4B029/1raAMUsqaoqdb+YervBKiZiYIJJnUfiogWuwOz+EqrM3Lqvfa6bzYvy4aG96ZQLV4UrVU2oHQpcQFA/A2Fp1ygojY2h3gLxRwvSei7w/xviu/qS7qJGHaKmefqHQAXWWw0tI5qVmbVoJOlo0xn6aGE6x9+7sK3i/RUmoNUunV2rgmJTyQN+qv8pt5lJSzOmc1jb2JC6OnAkaATb6qwuMcfXFhVBRUOmqrYJTSMEFw+KuSR4n3TaDca8TL0xJmarqkNUavYomSez8M5/2nvhO3d52hiv44YXqFKDV5KlxRUouBRUtU6lROpJgySZsg1fFi4qkdp+8RBIzTOg8J5/rYseGy3uWrZiNLAPeBKs56OmIlVeNrzutTuZNTd/bQf5m3EjT3LNpdxXR9neDdSgAJfRrA/iTpPwi1SvRp4i3NU8ZLku+nrkqcvFFRSxrJ+xUoD4pHytdHE1H6xdpdCZUwoLnwOhts4cw08wDgsnEXtkcS03TYp0QBEa766WNITA2JjeOtgGUmNSJiT52NIBMb6c58bdrBouXm1KGaPeGg+FjKCIjWbFQggwRIImw6IjqALXwqbjZOChcLjGm4Vrp11/GxgEga72IXWsFlSd/oj7osZqKlikZXUVTyGWkAqWtZypSOck6Wss91Vzuh5B1P3WyZMfGLErvvi6r0Cv3deDFTl3DTgJA8rCvV1JTKSioq2WlL9kLcCSfKbSBvqmNxojExtbYURubcTIzA6WxDrbkltSVwdcqpixAoG5QgIkfLzsBeCM1E9lE90ke7WwwMH8BbagFIKCB3tNbORfRMEXaUlxhC07KSDYNRBGwmwdAT6mlB1UglJ8wbdq02Ox1sMpwbpzYNd7RNZTKOikpUPmD+Fid90jK2jUVaQpFOMykKHtnkPrlNucIvdje4bUdHm1J167/AIWO3+3Vllxlqkbc1UVAu5Y6cukW5PtM0CAuIP2/4Wjh7iJALqO3cQ1/aryVlalOYwEvhKQJ5DLoPC2WFVQXlmMPJQJ9kPmE+A7tst5pxaXq78rq+DB0Xjt6d9QH/SBqEK2Zuuja1PIpKoj/AHrV+dRlrXg5kSZE6c51PUnfr5WnX04HQv0hL4gpJbo6FEEbf6sknbz++0Hpl111KgVZjJCANSCem34Tb1PCfDQQ/wDxCwcTN6yT6lY2tSVGRqARoRvyMcvHr87YoDdSlyRMcvH8fnYXskqUgnUgyNRrHIc+h8rbKBKUKRJUQcytCvkDa6SFVsuA6sGMqCcpMEkRJ+vEn3W5UVAlQIAXMwZBiD4jSwrjQQe1KgnNtJAn6gW6yntcx9pRIPlGs/D4RZgQmIRZK1JAJCgRJIUcsn3am2FAIhKVAHQEzJkcjtPOxwUxVBaSNJ0KYHXrrp9x6W0lghZMZyYEHkZ0jofxBsswTWRXMSU5Mqc2wI1PMj4R8Bbckp7NOYgA6H4R9c43saaZWoKzLKkKUE5MsyYO/Xx+hblDE5QpKpIAUCDKdNIB8h46WWYJWRRTnfUUglc666kzofv38ettFwk50JS2F65Rpm941A0ixkMFAyuwdu6pMwYgac4/XW3Cmm0ELDiRlB7u5MbR8+gtIOGyiWlFHu73ipPdUUyFeyRsB1Gm/vscbV9i2DrkTlPe08ZjfSefPxiwlz3BeeJ74u/D1wUIqryvCpTSUtOlQGd5ZASmSQBMxJIGnhaZ2vQz9IjKnPw+Da0gZpvOk1MAf97E/lYc9RDCAJHAfVEhgkk1YCVCZdKEw2DJE5ojXxP5xtYJeZa5LZMch8/Ly8bTw16FfpFPNKcVgVjKTAUL1o9CeUB3Tn9C2K9CL0j31AHAzKUmCVqvajEjkf7zpytXGIUvzj1R+51HyH0UFZ0hQWcqVJ001BPjHhpG3lrPJUOQzJUCU6mTrsfqNPC0/U/oH+ktUoUn+xdGlcSAb0pBqDpsvb8OlhFegj6S60JT/YmnGg0/elMZMc/tPDbUfOz+0aQacQeqXc6n5D6KAmaw0y1KU224E6QtJEgxt8v6WDcqFOKLuiQBO2kc48Plv1tOFV6FfpFUL6mXcEdotRnKiqaUEncR3o35gnWwafQ09It5SGG8BOdoo+0urZCBvBJKoH15WcVtKdQ8eqj3WcHVpUKh0lRCsoBGpSZ+Hl566+NtKqEg/wCGYWRroSOuh28vO01VXodekQy6pl7AL5KVAkoqmFpmBsQvpE6mxdfoh+kM0Y/0d1kbaPsxsdiF+PLws4q6b5x6pGnmHwlQ2y6ACjNJ1lXnMCOWg38B4WxFSkKSsoJCgQsq00k9fDTTx8LS4PRH9INtGdXDWrkzmBfYA9raAqwb/om8f0hShw3rRmyxDjPTX+PX+mtpiqpyffHqocCb5SogdcWlAEmJJgCMw6zy016fiPROuBpS0LJSdiDBkGYPjqLSo76JvpBPR2PDSvOg2daGwP8Ai+tfcNR+iP6RmY9vw0vGEnTK80ND1Gc9BaRq6fL749Uwpps3un0UXCoCRnCjEZVp0J0HhqNjH522mrWlsgFxOXuQFEx5dPaHnpaW/wDyTfSBUkp/0Y3jomP71oGQIjRX4fqGr0S+PpKh/o1vIlSZSSpoAe4K6Tt1sAVdOd3j1RO7zfKVE4qFElTqwCSMpKTsPCdLdIrFtNnvEKKdRO45e+NvM+FpSd9E3j+tAC+G145yNs7Yj/m5e/Qne2j6J/pC8uGV4qndRW0PCNFSZEbfPW0u80x+MeqjwJvlPoosNeIRKVBQTlCdZj3/AFM+Fuk16m2woAJUTqdZ023kdP62lRHol+kOMyTw4vDUnTtWRAMae34fRttHoj+kIUmeHFaNIhTrCuZnXPvy1/Oy7xTfOPVPwJj8JUTGqcKiVASdfHXnr110tyqsXlBUUnN3lZdSfr3be60wUvogekVVKLTfDerQcsqDlVTtgbjYrg6RYm96JvH8OhocOLyMzIDrcbbHvRrPXwsu9Uw0Lx6p+7T75T6KKfXnHVZjJ7+k90kdfDfn4205XJgJzglGgI0Khz193ytKi/RN9IVPcPDa8VFCoCwtBmdCR3o5DX3+YZ9E30hFZQeGl5gL9mVogR5K25bjraQq6T5x6hR7tP8AIfRQ4zULS+k5u9m2B56Tv7rCuVa/ZClCRqdI3IPly8rSqPRK9IcjKOGl4pymZzNiJJ/xfGwg9EX0hgkn/RtWmei0TtzGaTt99imtpL/1G+oUBTVA2YfRRSm8atB7RNStBBjRZSANfHwt3+/b3ZWEC8aoZdSA6RqBvvvtp4Wk1fokekIBH+jWvSMsHvI1A0/m3841t0fRJ9IVauzHDi8MwJUR2jYnx9rQ+z87OKuk+dvqEjT1Hyn0UcNYwxElv7K/7yGbUgVKwAdgfa6fR3sMnHWLUggYmvXKnQJFY5I16yR7uptITvolekOCVL4b14OnsutydDyzaa+Hwtx/5IvpEZj/APw2rIgwS41+KvGyFTRn42+oTd3qflPomEjH2L2niTiW98+bVKqxZkcxv52Ms8ScYoBR/aW8CdMoNQodQdeY5e/wmzzHokekQVZDw4rgNVBXaNxA2nveduv/ACRPSFSSP9H1WQZEesNCSdt1/Plz5WRqKP52+oTiCp+UpoDihjYDL/aWs7xMZoObXy89vH3iK4oY0nK7iKqVlGgUEq320KY6z1s8mPRA9I50Ky8OagQMxipagzznPrrv00sI/wCh36RlK6KdfDupJSAoFupZUCd9FBZE78/wtDj0fzt9Qn4FT8pUV37im/b7Uly+bwU+42ChtRbSCU9O6BOvUT8LB4exHeuGrxXX3a8EPOJUlX2YWMogzB05TItKb3oe+kS240g8PKllLyghKlVDYTqNicx+XjY8z6EnpFrcCP7FJlYE/wCvMpHQDVUWJ3ykAsXj1CYUlUTfIfRM+k40YtpUABVA6EjvBTG/iYVoNrH0ceMUJASaK7FEGP7lwaTy73h9a2XXvQr9I9Mf/IJE8z+8Kcgjr7evX6NiS/Q/9IlogKwCQnQAiupxv17/AI2HxqJ3xt9VPg1YPuH0QSOPt/JEOXNdpIJGUlyZjU+1Y4xx+rwuVYdpSEzJD6tDJ3H4+Gx2toeh76RmbshgY5hyTeFNA0HMOdB93U2T8XejJxxwPh+sxfibBC6O6buQlVQ/67TuhCSpKQYQszqRyMTaOajeQ1rhf6qWSpaLuafRLjHH15JLT1wU6lA65amB5+yfryEjDjrTrRK8NHltVDUdfZ+f32gpDqScoUZCdJJk6/1sMisWCA0U5iIJHj79dvusQ0jELvDlOjPHK7FhAcuWoTmTrlfCo59OQ/ppZSuzi5cr1WW03XVoJBKSFJ18teX1pE1+RUulXf1EglJBPj4eP4WV7jrlivbICiVEtgde78/zJsCWlAaSEWOc31Ur4/xjd9+0NMzTU1ShxpalhKgADKRpIO8gfQ0iukrBSYhYq+yJSxWdovL3tA5Ow8Of3aWXax5LjYhJWU+zmIBEadekWZ9epxFdUKIgFSSAkyIgRty0+Vo0guCE8ztQVYCnxtc0aLeSdDBR8Bryj4ab2UqfHdwZUpD7vd6smY2AiNbQ9TPj1ftE5hm/igxG8dI/DnBix/tlBKFgpIA0gkpBJ+ufI62pvaQUZshKls41uDIhSqogZjqELmdNzFuFYtuIqCPXBKgIJQrmPL9NLROmpKWgSVJMAeKvx5+Fki9cXXfdMtBYdeEdxBTod9Ty3PWyjifKbNCd8wbq5THV4iuQs9qLyQhIGq4IHI7kRv8Al1tA+PMTXS9X1t33YoPoeqSS4kygJzT3dBOs8ufibN2+cVXnfK1B15LTSlZuzbACfCepFkJwkmSZ1mbbNLRcLV+6zqiqzCzVKmAr341cQGKfB2GsSvP0eH6ZRpKOodSWqdtTpWoNpUCBK1knTnvtZ4VHDb0j2UFa3KF0azFOwZ08W+kfK0HXNiG+MM3i5W3FeD1G8oFGdpUEpkGPEaDeztp+PHFSmbDIxdUOJ59q22v/AMSfK0Kimnz3pwy3mNUemqKcMtOXX8inJX3Fx/pCfWKJokDVSael0Hnl0skOPcYKYAVFA2SZGrLIM6jlEGNNNbJr/GviHUp/1i/AuTJ/1dvU/DysQe4nYrqEdm7XpI6hpI6fDblppZmQVHxsYmfNBfwOcpd9HfG3ELD/AB84eXne1MGqRGJLvaqF5QPsnH0tq2P8qjb3KdYTUMuMLHdcSUH32+di7uI+IaO8qO8PW1BdLUIqElOhBSoEa+75+VvoiuO9Ke/blu++6Q5mbwpWqto8ihxAUD8CLY+LQujex5aB9FcoZQ4OaCT9UzUMqbWppYhTZKTpzBiw7YHjoI99j1/U3YXipYHddAXMc9j8wbEgNhyGnutqUzszQ5VphZ1l2lMkdPusKkaagadDYNPtDTT8bCjUTpBtoBU3C4SldLgJKBznX4R+NkXibct8X9hh2guJCXKntW1ltS8gWkKBIn3A+6ynd6gH0jxB+8fjY7etYLuu2prigrFOytzKNzlEx8rHZq2yrk2ddRTd11qwfii7qekrHl1D6GV1dOp3tQytRhQSdwkj+HbSQBNlHEmGb7vS+r2qat6poKdDanmK5JbWjRIypyqkx10G2h52TOGlddt5V1RinEd5sisdcK0pWSYJG/kBoB0FnLxFvdbyKO6KJ0BqqAdcczQnL/DJ6c/daLcoYTyRHE5gk2lvq8nsBVQLq0inqGm8xOobUpMp6xr8LJlzvV+Hb4u+9hea3ae8hPZFKQG8qsq0SB3gZB11BnXYB4C6rsRgqtua7K2nqX/V1uKLa0qUpwCZIHjHysxql1NXh673WlhS2Ktad9Q24jMD5EpsW1gChHc2Srik35e2ILypKGuS1UU4KqZK0lSIS2F5YBETr3ht42e3D2+n73uQesvKdcYUE5lGVKQQCCTz52adfUBnGN3Vyu4iupmHFA6DWUq+Ay2P8MnF0lfeF1qOjcoTP/q1lA+UWINDdQOye7cIdqGxpC8w94ti5UdNracBTeCyVaONgx5G21DXSyKYI1dLwprzpn9gl1M+RMGz4vCmCnV6e0i0diAd4jUG0lKWKmmYqQNHGwr4ibZWKQiSFwPRGhcWu0TGdoQHVjXRR5+Nss5F0iSonJubZbzI056Lb72V4K+mj3/SIxElBVPZUKYH/wCqtdbQ8UqcqHgW1KHakZ52n3dTG2utpu9L6hqLz9ILEdXdzJqqZfqWR5glaTlpmgdU+ItHQTdVI5UpqaRaX+1IjMBy13Ply1t3VC7JRxNt8I/YIdb4qqQ+ZTfNOvIVdmXIMEARoRoD4mCN+Vum0KUMqgYmSkIgnu6Hw3s42b+uWmSWkUhb7Q6gqSABy5zpztpNfhDR00jp00MmUmdNSoSefS1i7uir3HVN5NOtQSCFDIJJjbTczz11526TTKSQrs1qOgEE7xp5f1svJvDCi/sadp3vryJGZQiPMzz84B6ahorMNLUnKFoSvu7geMb7fITZru6Jhbe6RhTK7RBZUQANYTOu8nx25fp2lsOBbYBQghSVExB1PMzO55zrvZ3XFdFFftO7U0YORtZQVqWB3okiOehn8rKQwa0lKgnKIEAFcmQJjlGmthGUg2RRGXahR+pgEkZDm9jQDfw6a+7QG3JowBlghSN4BHI9OUH7+VpEZwSwk5XklKcxSR2qZBBI3jflz2EeBhGBLrUQlZcAk5T2qNTrtO58+gtHj2UxASoxUylWiGxKUEFMTJH6fHnIsG9TOskZ21QRA1ISTtM/dytK5wTcwSsOVDwC+jiMpJEztzn5ixd7h/cDo1vB5IMk6tkgbRtqNPhNk2qF05pnEaJF9Hp3Jx74al0pAViiggbjvPo01/W3uUmloHgla6RgnL7RbTP3W8VOGVzU9w+kPw2pGHnHUDEt2rK1lJOY1CBuPK3tMmqCW0x/CItQxSPjuY63JXMPcY2uaeqEN33aglQoKYf/AFQ/K3aqK7VaepsE7atpsA7ejKmxThiFzq5OkeVg1VC4y5pGkmd7ZxogFeFTfdDGiu4HSiY8fsxbZobuUO9Rsmf/AFY+elihvFFKStxCnJ0ATv524RXlYzGAN4mRZdyFr2T95J0uu13bQF7tU0rIUTybEnzNulUdINSw2I/wjewZqlHvAfG2i/Jjl1tNlKBpZMZ/NGW6Ojyx6ugQP5RvtYZNFRAEdg1GsDKLE26pI3V7tjYUVOmpGg2iyNN5JhPfmhzR0mqfV24Oh7oNgxd9Eoz2KPegfjbkVOYwUjrbn1sTEj4z8LIU9uSRm80KigokqJ7BsGI9gdbCt0tIP9mkTzyixUVAnp5CbdesmRoI5gmzmm8kuN5o0mnpEpILKRp/LblynpZksok7EpFi/rYgiQD+NuVVRmUmTt52j3XySE3mhPVmEq1QjQxqgeFuls0uiuxQSR/L87F1VSo9tXh4W4NSdif8Vpik8k3H80OaSjgD1VuR/hEWDVS0YBHq6IGh7osCKobAHa3HrRGvLfe0hS+SXeD1QppqQDL6syY/wWLVdNS9lpTteyf4BrbFVBgzJG2n19aWL1dQgNgmUwCJ8bQlo/Dspsqdd0LTtUxcWoMtzl0gC3KmKUttJLLRk7FINidNUIzokiVIn3zbhNUcyNZCSNZ6k/laiaMjkrPeB1Rw0lGWCo07R3klI5Hy1sXNBSBgqFM3Ko2SNPlYJVcAxCtyo87dPVI9VmRMA6ajey7nfkl3i3NGU0VFH/mrIEfyixdykpVLX/q7crIGqR8bYa0BvMCCI5mwCKxLj8rJGUEjl9fpYvc9ALKPeB1Q1VSUbaW/sG+kBA1t2qgoVJj1VnQDdsWLVtUFJSSNlHnrNuzWJUEqAExOlpso7HZQdU+aKu0NEapI9VbgJiEpAsVqqCkfUGkU7Mjc5IPKTYeqqwHisBMJTPgd9LF2XyUhawoKWrrMaHT5TaQpNU3eQEUqbsoWlJSmnYCYBICQDv5R9TzsiY3xHhPBlzLr75yUyHCllpTdOFLJUYlIjkDvECLK1+1rV3Ublc8VFDTS3lhOphIJ08Yn5+NoQ4ecV3Mb4pp75vAyUsetBouy2wlZAbaGsGJk76gnSNLMVAXnyUHVYbZPpGI7wxS8tGCOGN64gdY0RV1zq6eka/wJ7RRSo6knK2Umedg8S1XGbDF3euO8G2qtE5nDSVDTwbSQMwCMoV58trWCw1elB6m0xSMNiESCjQKOu/Unfzsevq8l07IU6ltKlwMugPv1m1luHQBuZyn3mRzg0BQNhDGOHcbUqwzdrdLXspHrNI8hCXEGNSUnvRPgR42PVV0UTy8qaJkRqqGx3dfdzstYmuS4W3nsUMXOzS1obzKrWBCnAnXKobEECPf8C13w9Tt1KSIdAXIiACJ395+hanHTtc4taE9RI6IAlEGLtpUVHZqp2vaBAygwLNT0grgoL34K43u8UjJAw9XLSnsk6KSwpaY6EFIPmB0s9KmvS24qmTJgZdAkDNmCs8xmmAU7xB2nWxHGVMi9sKXxdgSVeu0FSwZ6LaUn8TazFSBjwQFTkqszCF4h5UZoygAmDHx+vK3aCkyFLylJCZ2/pbpdMtsqVJkHv5tANSBPvHhbvsXHXClCBCk7JTqdfrbqbdXcLm7ELaELKu6tJCRvBhX1B+Njl1lTVYyvXuuJGZI0Eke75TNgGGlzmLYJBE5hOkgayfr7jbbaG4dQhKoVmkxAjYa76RMfKwJDcEIjBrdOqoUkJCFAZld4BSOh6jWdBGvPxs2r6UoV6Va+wI7s5tT79tyfnZ1VKE9iEtmUq0UCmZI0HPw38OtmriYgJaWlMjKqToZ1EHfTUqtSpCC+yszaNunDdLinKJhxMx2aO8lUkRp74GnusHeV60tCz6xVuZDHcj2lJ5QOc/gOmjTRiZdDQM0lOwntG5+0WJiSSBHPc7/CyLUvu1DxdfcLq1ays62tMosziXbKu6osLDdLd8YxrK9JZoppmtZP8Z8yPKzdKpOqjPjbR00Bi2jMa9NBa+yNsYs0Ko97n6uWxIgfdbSgZ1523Cuu56227BAM76xFiKFlysagpESBbREDS3RBCUK1gyAffbakZR9aWSVkGCelsOltwd7ZB2skloEzMW+gL0Q8TDGPoxcM7+zlS1YcpKRxUnVynR2C/wDmaNvn9O1vaj9lpiY396Jd13ctQKsPXzeN276wVioH/wDs/K2PjTM0LXdCtDDXWkLfJWgxLThdO3UBMFCsp8jZux7vfZ6V1P6zRusACVI08+VmbBB1EG1fDn3ZlVuqbZ110AZkEeXIGwiVEka6+f1424GU9QDbsCQRlPiTbXas8lDMOdm6FE6CSfdr+Flh9hqqYcp30BbTqShaeoIg2RRBOVZ7p0PlsbK9EtTlGwtftKaRm841+djRlVnjmmgxwgwXSyKRiuYBnRuvfA+SrKd74Dui+GqVpb1Wx6m32LamX1JOTSAeu1nGCYInztuYMDexg1vRQLnHmm5h3A1Bh2rcraetrH1LQW8r7gUACZ6eXwslvcKbr9eNTQ3pV01MpZcNIkIU2kkycpUkqSPAGBsIFnwd+VtiNgdtpNpBjbWso5nXum5iXA9DiGipWEVTtFU0IinfagkJIAKVA6KSYEjwBEHW3eEsHqw4XXaiu9bfcGUrDeQAeAk/fZwiAJmBbtKjvpaQaN02Y7IpWJy1NO71KkE+Yn8LdLSDuAD0i27wgNIcP8DiVfOD+NtrEAfGzEalK6BIg66Wf2H3RU3DTmZLctn3EgfKLMJXx/GzwwW6V3fUsGfs3Mw94/S1WqbmYQptKUSjU6Wyw5RJJtluU7i1Wsy+aKnqV0TLbTbriTJBKVQo6kgT79rYupf7ZSHUOJWhSgoFKe71GniTp5c7ErrRUPM06e0lSjHdIEHYDz8PlZw3dwt4kX7eTrd10HaPIQp05qpsbECSSdTJHnNulbAXOsAhGXKLkpKRXLJyw6cw7sjUHp9DWNfAVq8AiPsXE94SFIBJOkeOhP1vZyDgFxjQO9cyEiZM1rO//F4bmxpjgDxgUpSTddOTlJE1jJ13B38D9RYvc3n4UPvLPmTTTeBUlJLJjUElrUnQgGPKI8Oe1uvXErhCKdefY/ZSYIjpB+Pws5TwF4xpQS3crJCz/BVMTIn/ABCdLcI4KcY2kR+5BABTpUMczP8AN9TaJo5OicVLOqRmbzq20GlpnapCDKy220QFH+JRKfHLyHnYRF+X0FZU1l46gpGVTkDXbnHLygC0l4G4H41qKGpcxBhvtKhD47OKhsSMp7oKFa6xqfGzne4G30hSVUmFVNgCClNQCSZTrqvTTWdxpas+FzXWyq0x4cLhyg9q/b+efSUVd4EqTlBGeQR+oHT4myg1V4zWgGnTeeoBlLSwVancAHx8J1i0v0/BLETTqVKw3UNAKGoqEk6kyPa8APjY+nh3i6iWtNLh10BJlMZVctt/r52rSCQe7H+FYiyH3nqFm2+JTqClqhvtSBBMsLUZjQ6J93Tffe3NQviTQNhdTdd9ozGVLVSLlX/LpqPhNp2RceNqVK0OYXW5BmSyAdSdsvMQPjYk7duNy6Gqm461IcGfSnUlIEDaDpyA8rAa6e+seit8OntpIbqPeB9ZflRxzwCm8EVSHEYju1MvtFsgesNnmBvJ+t/Z9tw5QZiYOlvIzCN3V9Dx64fP3hQu0q1Ygu4gKbKCQKtI5nb9PG3rg2QUJGWFRO9lUsz5TayhTuyZgDddKUCRJJIHvm2jWtoEKGw+vvsE4HMhKUmBz6e/4WJL7RRJSox1ja0WQjZNJNbZGi4y4vOSNIk/XnbsOM5co166/dZPQh51RShJUqJIAtyXSFBK5B5z52L3cHRCFQQlULY3UOe9tlxoSQSn652S0vAjQg/hbC8JIzeR/GyFMEjVEpUFSxvqPCNhbBV0/JXh42SC6IJB5RpbkvnNGaROhn8LTFK0qPeiEtqqmSYMkjTe2GpZkEEn+tkUPySAr/NNsD+XvEyN43+jZd1AS70Ut+tMzEE7RbYqmTM6TvrZE7eUwTPPlpp1t0KgACSQPPlZxTCyXenJX9Zp5g8vvtpVVTzqPjZIL8GMxHnbXrIUrUieXl9Cy7q1LvTkrmqYzSNJtz6wxzSoRpZK7bTQK0H420qo2AmDG1n7sEu9OSsaljcoG3X87cmop5jKdZ5zZJU/zUrfXfS2lPnWDM89vCzimCgak8kql6mH8A8JNi1dUU5bIybggQbEvWFaAq0+Q+tLF66oCkJSN/Pwsn04spNqTdGG6uk7Rr7OPso+FuG6qm7ErCIlwDfTSyR61kQy6NB2SjbTbw9QJB2XJ1Gus2rmlaiiqclM1jJEZdZJ1OlhV1dMaUSn+GRrHQbWRS9nDywQYlI/H7rb7UmlCBBCkxE77WburVLvTilV+8KRLIT2YTmHUzpbVLW0xdCuzVKknUzIshPvqUqAe6BAVI0+NhkuparEQdkZddtv0FnbTC6Y1Tglm8axgNCG9AQLCJq2FITDWkdddtrIdc7LG+oMkGd55fXjYQ1IQ2FKVASnlB1+vv8AGxO7tCH3lyNVVZTrUlkszm8TPl8/qbDpepk5R2QypIhP4x9bWbztSMyVpJAKp9qSQDrz8fD32URUAnTWDBEnXr+Bs4p2pjUush72cpahCW1sIhYLahJ2Pj7vvtT2huS+Lr4o3VgG6rlvK7WzXKU89UUpYUulQohK050gqCjICkyIOh1M20q35Q2pKEnKdEqEz4b6frZCvbC9Bd9WxjCiTSsXVdjaW007Lcu1DrypfccUTOZICVCAPZJObQgUwMTSQFeobTu8RTpvaivdFzihuZNTkd7jy2qs0y8kAR22vZDQkkSoxAHSHeGuDb5XxCqnDTXZR05ePrDlHfFRebi2ZIAU8/MKjWABExrEmaqrE1Jh+6n61x1LyKZrN2TRAL6tMonb3k87RvVcT3aamqKujxzhK770Lkpu11lLzCWx/CEoU2oqMR2hJEzAgRbOa9rhYro44XXzNSDju7cV0HEKsu0O3k83TKytqpL1LJSwvvJdUye46M3dggkQdRNpWo1Jprup6NISSwy23sCAQmNuXW0TMcSr0xPeP72xHRXA4+hSWQihV2mQQIUFEmCJnyI90stMLcp01B9rKmRsdt4O36/CWG5JJH25Klj7XQsjzbn/AISReCyp9WQhPUAnWf6WFgrSG3CQFd0687Gl3VUVIeqaVJU2ynv94AzBMAc9EkwOlgygBgJOsJBEiNB+v1tbS4Wq59slwvNm8sCcHLvvu8LuqnrhFRTVbtO62qrSlSVIWpJBHaCCIiPC3bWC+CRICTcS0nvDLeBAPwc00+trNLjhwLvV7jDjN+kqHEt1F91j7Tfq61aOOFYggbDONf62YrnAnEAWA1UAhR0+yXMe4dCLVXQQk6zkFXmzSgf0AVOKeG3BVxtB7O6w257ATeSiFDYH2/EWOU/CfgzUFSUpoxlBKibzc/Az0087V/HAvEaRPraZ2MMOEBU7SUiPx194DvBbFDUjtmySDA7NzUjlMb/XOzGljdtUH+fdEFU9uppwrOK4c8ICnskNNqXlCQG7wcUmYEjzAFqc3iLxrb2qaAqW8tpbjaEN6gZZmAPBO/vsv1PCLFTCe0DjWTkTnEnp7NmZX0dVdN4u3fVAh6ncKFiTuD8bXsNpGQF2WTOT+Fn4jUumDbxZP7p9XDhe4a7ClYbxuS90X96wU072YCnDRA9pJ70iF+FuVYBu+FIKqxKgdsyT06J6fh0s38J4VxBiv1v9zPpHqSUrc7R0p0VIEfCysOGWMgExWsAiYAfVpzI2tZkeGOIMllTaC9oIZdSdgXhJwbxRW0lxPO4pTXvMkrcK2UMlaEyvLCVabxNpHc9D/hS072buIL7ZOmgrGCU+ctfWlq50XD3iOw+HrvvMtOgZQtutUg+QOnX52U2cJ8Z2wkM4orW8qcoAvZxOUCDG+m3ytmzRyudeOqsFq080TW2kprlT+PQ14WOphGKr/SIkS9TDX/2c/jpZgcePRkwhwy4fuYrw3iO8q2pYqmWXGKhbSkhC5E9xIMg5fjysyxh3j8ggJxjevcOYRfbndPX2tLB3lhHjtfV2que9MUV1fd7qg4unfvhTjRXMyUqVBPPa0YW1EUjXPqgWg6jTVFnkppYnMZSkOI0KiTIC0DmEhUWuxhD0DcD4rwtc2J0cTrxT+9rvp67sk0TXd7RtKymc+sTFqzI4GcRilSfUGACrUGpRqRtz8fnZ9XXdPpQXNd1Pdd1YwvVijpEBqnp270ORCEiAlImAkACBt87HxOZ07AKSoawjfbVVMMiZA5xq4C8HZTLWfs7bkRpR8RbyWraP3ahUq5ahyyY7+zrrQFOIx7VoQJGZdyk69P7wC0c1N+el/hygqr2/ttfzbFNTlx51F4gqS0kEqVvOgmSNbM930rPSHfc7V/ipfLqoABcUhcR0lOm3Le2dDT4zKLxVTXD6f8LQmqMIjNpaZzf59VNDn7PG/hAZx4XAoEibnWNBv/tOXO16/wBndwlvvgjgfFeBb2vYV7b96tXqwoUy2cpcaDaxCiZ/uU7W88sE8YPTexrdn77wfiW9K+iDymu2TRUhSFpAJT3m9BBGm2vna5/7PnGHpPXlxFv+6ePDla/dT1zFygcfp6ZsJqUPokS0kKJKVq3kd21eU4iHcOpqGOA3A979lMCgczPTQPaep2/dX0Eyd5s0L0p/V651uNCrMNeRs8Dpz+FkLEVPq1UAanuq/C1+hdlfbqqFS3M26RQkK84iI5WETA184jS2RsInn4fWttyDzIjebbjVllbAIOpI5WVaNwqZjmFKj3mR8jZK1EwD0Nj9CsdkpAPMHT4fhY8e6rP2R0RuAdrbBIEGfjbgGBoSIttKidBz6WsoS7EDczbAflbkdZ8NLbmdY162ko2QgO8iBbtJ1iRYESnQi3YOgPwtNMua5BcpXgPayGNOY1FuGFh6nadJMqSDYdRERBI6WKUcJpuyM/ZqUjfofytB26fkuliBFnFgd/JWVFPycbzecH9TZvOa9d+dlPCjpavlkkwFSk+MiwpRdqdqeZQqT52yxssgmbZbG7mUXiBfMtcjA7aibK0/3qClMcyofD63tZ3hc36xf1YhCkIHqyl94EQntE89eg+XlaugumruW/W7uvOkeo66iqEMvsutdm42sLAIUCMyTrz6aWsrwoYR+8a59TqJSwlGTKRoVSeWu3n+G3R6vVaq0jKkZdMppAl5CkKITlSqR4ae+PrUJIKHQsEpBVm2nQHr77HcjRkKfSJEknpPM9BbYZbGZfboBGkZuu8/XxtsgmyyQUSWlWYo7Mnp3oEfX0RbR70KKRBIkg6n3fXLxsdfYYOV3t0HQagmVEe7cQDrYFbSfZDiQIHsmAfL57/rZkjqlbDRSadyASUOpMxABj4fHxstKUlStOZlI57DSLJeHWULYeQFBQDiBGsDfpZXXTLSUkJOmo5SNJ8BztTl95aEHuIE6HOAZBJH/N0sGoLQoOIKCokknfpv99jCkZm0tBG0xGmup28/usGsJSoZkERqI122/wDDHxsFGGqKqzlPaTOxPPSDpHvB+teHRnhACQlMDKPMc/d91jAZUpvvpXqoJlU7aTYvV1LdHTO1VUshptBWoxpAE/jt99mNgNVMAnRQ/jNeT0hOE7cJzJvqjUop3A9aa+Wvvjwt6ctLSpIn3z99vLS9Gaq/fSD4bLW6Kd6uvyhT2rmoZ/1lsIEc430G8nc29L2KO+0o717g5tQex/W2FWuzPuFp0oLGkFONVe+ulVQJy9i5qrTWdOfwsWU1CVJMGBpZKFDfK+7+945/3c6WGFBewSqL0UDEQECwmEnkneAOaNMVzt1ZnWMvaKGUZkzAsjl9alKWtUqUdfPf8LFqu7b47WXL3cyFRgBAsSeu+8CIN8PgjnA87W27WsqjjqlX1kb5zvEDTlbPWcycyVE6TG9oM46Y9xDw0cuGoo8Q1KGq9x9t1HZoOfKEkHVM/wAR2NowX6SWLgsZb2UtMgqJCfdACeevuFkXBptZSa0kXureqqhmJzGAfo20qoE6Ekcp3+dqjtekdiEkhV8PAjT2Uyd9+7p52EHpE4i0y31UwqIhKJ+Q0mzZ/JPa+xVtPWhB36RI0+Fs9aSRJWZiNDGlqmH0hMQd0G/a47pgBMzoZOnnbn/T5f5Ol+VyRMESkH7tbNxQOSYMPVW2FWEyc2069NLb9bkwCmRpEWqV/p7xCdVX1XA8wVjTly8j8LCU/G3EdQSVX9XQTJUXfZ+G9mNQ0clIROPNWx9ZHXXTUW5VVECSRHhoZtU2q46XyVkN35eMCRIfIMjn4f1sGjjXfjhzfvu8SAQO8+rXXz+tLN3hvRLhHqrb+tDLpJBmYtwqqMiAdOfutWq6uKtdVFJevW8jO/8ArTiQJGmk2cCMcuVCErTe9fBEE+trnx5/XvtXfXtbyRmUbnbFTv6yVaJnoLaVUkCASOW0i0L1eKKlqlacN61ozpgkVi5knzsVRiJ51p1z98Xr3RoPXV9dt7C9qMteyMMNeeam81JzBOXUE6g2BrXz2QUOZ1kfXhaCXb8rKkBDWJLwaVkIbKqt1SSepAUNJj5CwN137eBeyvX9XVCk6k+sOhOgOuUqJGvj91huxmP5URuFP+ZTCuqUthoQZlSTO0TzsIalQpHgCqA5oY+utonZv+rKCo17pzLSgFVUvTnP6+Fi19YpfpXVoReD8KUSMtQuCB0B/HwsAYzETbKjHCpAL3UwNVLqg6QFEFRiIkfXW22X3vVUgNLmDMDSfD5Wgr+2zyVgmtq1CCTNQojXbn5/CxhnHDq25Qt5Qj2u1VqfKbSOLxj4VEYa/qpmaW84+ErScqQCox8vhNhHalxNa2pKFFJESR4nnaB7yxw42goXUOqJ3KXj+BiNvhZBXjZ4KPaVTpB1Ci+slPXn9aWePFmO+FDfQObzVl66rdLEgKI0Pd1+vr3l3K1x0NUqEmcokiQR8uUm1bnMc1IkGodAgCDUbTz38/DXzsUVjp4pKy+pUaH7YzHjrpzHw6WsiuaeSGaM9VZyrqlN9inUEGIAGpnffz+djhqnNglUyf4ZJPKBGvPlap7mOHgtPioEgLk85ETry+/yKHH4OZJU0O9qC7KcwjQ5fMaeHS021YOwUDSG2pVsbxqldmlHZqJBJyxOn19TaMPSJxTjnCnCisxZgJNcm8qNTLTqW1fYKpXe64X2j3FoTIVKhoY5G0LnHynm0eqqpShLkKcCgrNtpvE7QCJJOk2ScF8QF3txiw3ct+UZXdNRUVNNLi1BLlSqle7NfLKUODunNIORW4GW9Rs71K1hGhKE+8Au11lIXDT0m7qpqumwJxDvLtkhxNHdt91tKmkReCkJHdfaStaWnpAiFqSsQZSoFFrC0103FV3c/eVxpuVmsqCpbtSwsUzq1QdStEFS45yeWsW8++M/CKsww1+/ENVd/wCFryf1p6koTVXZUFagtKqpQygGdFOA5jEqBIKmRUcU77unA1ZePD7HS8PpuqsFEblvO+k1NWpvROdpl1lSijaQl6E5TCY1tOrwaKbNNSOA6tO/mtCmx2SjLYatpPQhXK4yYwwVwJuavx/WUl2VuIq1Xq9Gl51T9RVviCltS1KKuzBBJAMQdNYtYrBF93hiXBNz37fN0JuevvS72KqpoFOZjSuLQFFsqOuk8xO0i3kRgjF1XefECi4gcW37yxxdeG6+ncqEKr1BC2i5CcgdTAbDmUqTlSFARIkm3pNgb0sfR+xYlqkfvV3DTroPZi+qZLDKjvo8gqaSnoVqTNqFPQdzbrqTuVHEcUOIuBAs0bKV62rqKcpaoah5AWkoX2f8YJ1mNPr32KB1wU4TEkAjfb61+XSbGHsPUV50qX7rZpqhh5vtkLbKS2pEe0FTlKYjWY1FkdrD9K2pxtVGhKkKKFJUgAhWs7+/4GxXAhUWmyrnxXQzR8Rb57NKFhxTa0EkZUKU2gkx5n60s2A26j7ZbCcqlykFyCRzI0mPHaNLSHxpuVFJjlKadDbKXrvac/hSBlzJ5+CR8epsyGcqJyMl0+ylS/ZSNTt5C3F1pLJ3DzXdULeJTsd5LhgMoYzZVJcKYy66HqTv10+6LH2mXngKiqaUltBCUKU53SY/hAEmZ5bEWDZoXHSArNCtBmPdHn8fnZUTdL7pSy2pdQ5tDae6B4E6nWOUb2znSW5rTZH5ImKqgQpt31N1SGgdVuAIB0hWUCOnhFqN+kXTFvi/fj4IWKksv5+SiplEkeGabeiF13DX3dFYinpW3G5WHKhtDoSRyAUCkk77T5Wpb6bVHUjinQXvUKzLr7paK3A2lAUtDriSYAAGgTbb7N1F6wsvuFido6Y9zD+hC59C6n/e2OcSYVDvZt3lcrskqEDK4gAmd4Cztr94tUOHVUzeVHedNXUyl07yXyhLQyZwyhtSRprIC9uZGuhtUT0QHew9IC7LvIKhedNVsiDz7Eug7TugfI29J7rwq5UOJHYFSU+0IgRoTJ2+Nqnauc01YDfdoP8AZT7N0sdVR+MbEqPMJ4QbovWHb3u2jNZVViqjtGmwvs0FCU5RmAgyJmOehGlnpdeAsPVdO56xdVOiUkLqHmEFI1kBMe0dzz6xvL2p8POOSw02hLSo7jXsqM8zur39DtZdpsOUdMhJqFKWoaBpG4HjytxMuIk6grq2U8UIDbKPKbhjherMIuelcQFQ48ujbAGkTJM/j4WUqbhfhCmeCV4fuqoWUHKF3Y2RmJmAI1Gp5c9rSQzdzi8qWW1JCR3SqCoeXIfCyxQ4ddbkgZVnfWVK8z+dqJrppNiVCSaCIXcAo7oOFeCSpT14YQuxxajISqiaBPQkAaWXLv4V4LcgNYNuQKX7S13YxI8u7oPnaUrlwJWVagpTEN75z3RY9el64AwIAL3vBuqrNk0rHeUVchE2uwYdW1DeLK7IzqTb05n7LBqcZhzcOFuZ3QBRbV+jvhHFNxXncacHXW4LwpHaYlNE2kDOhQnNGhE7i3hzxG4dXxw2xpeeEb/aQ1V3dVO0TyErCgl1CilQmI3B+FvoO/fHE/HFP2OFbmbw3diyUioqgUuFP8yUDX/w+dvC70trkvXCPpA4uw7ftb65W3ffNQhx9ZBLoLkpcMndSSFak7629D7JRGlc6OMuc0jciwNul9Vy+LSvqG5pi3MOQNyPryV2P2ceMMVcTOCN68JcNcMrgcbuS80et3sWUoUtD4zhTqjmKnAUKAMeyEiNJN2uG/o7v8P6lzFl8YkeqrxCFgUzCA1SthW4Cd1eZPLa1Gf2K+NW7vvfijgV1SZfYoL1Z8m1utL/APwjdvTm8757VtTM6KFliUNFS1r5JSS7Sw5BCgr6p0Igi0bz80kDbaDYnerBfoXAAJRCx7rHJ1525MKGU7EG1+E5SCjPFwQmeOsgz42wTMjQGwlS0WnnGlE91ZE++wc5uZ/AW6JhuLrHcLGy6TqOZPjv9a2OUCu8U9Qfl/8A9WKSeW/LnYakUA8kAjptGkH9LGYbFVnahKUmYjY22FQJiPEWDzE6je28wBiLWggEXQmaZTvytuTGhmbBBRmY526Ss9ZtJOhAZ8bdpUOWgsCCTBB1Put2lR+OlphQQoXsZINiVIspq6xhX8LiVjyUkfkbG83LSxMyi95HsvMdOaVf/nWi8bFONkO7oJPxFh7pdLVc06P4FhW9gHNBofhbu7z/AKwD1NmcLhIbqVBqJtlk+nvBHq7WYiciZ+FstTUrFfNrWV1bXJRet51rtTVKQl156ocK1qypAEqJJMJGk9By0tYngjUprKSovBsaPU7CoIgiQTEe+LV2qGgLlSpBRm9WJUCYkZee/M+Um00ejVWqqcIOu5iXG1JpieYIJI6clC1ujFpECqN2KbC+5AheUwJmdD0+utuVPgAwkQFTM5RP19b2Khas2VCu8dBrB1+vvtwpRUCrkkhOXTveevWOlthZeVKDT2dpSQJKVBQGXSI18d48rAhwySnQq0EeHMfpYohamnFdwEK7p1Oug+YHw0tijlcJJUrNqeYIImNfIfnaOymnThSsy1D1KUAlwBaZPdGU6/DMT7rONbyCQgiVITAVO8FJ1+MWjm7KxNBWsPAEhtUkZdxz05GNvGz8SISEoXmSQIV/MO7B+61OYWKu05u2yMMvISsApygQRIB5R+I362AcLagcqSd1eB0PTTmbdylTaSBtuNoJIGg22tpRJUZUqFnb4Db37bWDcKyAuUaryqOhJJ18T87I2Lgs3E4gIGjyFwQNO9p8I+7nZYUnLlJXKTBGg20/GfzslYlyKud7MYSSg9Ce8PlqbQkPgKLH7wUMKqOx4/8AC6pOzV/UMgE/+lta+PP6iPUlpsxMEwCZHW3lZeYnjhw1hZEX5Qmcsf8Azps/n8fHT1bZQopAJFsaUXsrrTqUWUS0A2imzLJnPPs7frboBWVRB1Ow8rGVNDmN/C2nE9xUdOfSzxhQkKJ0ztE27F6MKdaA0SkwZshrQnOpSUQCo5QRqB4+NlRxBzKmd+n142LKbjQER4WtgWVRzrlVf9M656u9qHCTNO6tuKypzZACSOzSNbVsRgG+VFIcrKzSNCEkaeY6/WptfLiPhFjFt7YeoqmQGXKhxJH+VOnvANuKDgbd7jyHiwSknNISAfAQBZxTmQ3Tcfhiyo23w/vNcj1qtBOvdKNP+WfH32PIwJXJSCl+tOoCZCZ93d+tLXsXwHuju5GVSmAJTP8ATexlrgVdqDPqvxH4Wn3MqPegqIJwHeEBWesIBPtZdd9u6PxG9hhgO8gJCKzQiCVDUdfZ00+6192uB9zpVnTTQeo0sMOCVyDUUZgHTS0DQuUxVjmqG0GAK591IcTXgaD24jbbTw+tLLF58Na273TTUVNWrbCBqp32tj003td5rgvcqSFeqGec62Wn+EFyuHtnKRJlA3+dq76B+ZWI6tll51u4FvZHdXTVZnQ/aRI3108rHKHh7XLeSk0tVvAhwxH1+Fr5VPCO5ipR9QB3GwtjPCK6GzIowPCNjafs1xG6H31oOyq1hLg9QVYS7WM3hHMetKSZ6bfPe0iUvBPCqyls3ZW6gZj+8nzO8fxxawdz8OqKmPcpwnTTay1/YulzFXYjWIITFsqfCZS7RyvR4lGOSr89wiwfTtMIF2POyCoFV41BI8j2mnWwdTwswozROrauasStEJCfX6jIZ1JIK9d/qLTveuCahwJDLJjKU5kAfH6628z8cYZ/aNqxbfbWG2sYrulNc8KVSDShJRmUE5QUhRERB3587Uzg8p0LrfVWhiUe4F1ZxXDK6xUKdapX2kJbjWteMGN9VGyUOHFAp1akNqKgDqmqdkADX+L7vO1Y2cAftBK51v8AeNJj5wsHtRD9MgDSNI2JlQ05ac7KdRgn0z3r8uupYuDH10MNPI9Y9YvoOodGcSVZj7O8gRvYLsFfb+oFaZirR8CsJT4Fu9jLLJlpfdT27vd8fasUxJgpht7KGCtJzEkvLk6jUiYn65Wl+mulS6VXb06J9kefhYLENztIC1LYQgI0BI121PytmjDZA+4Kv99YWbKBmsFpddCVUIWCDI7VZg+czZbpuHt1qYBN1hWWIJWvlp18LSUzcbZeUUJQVpakwNvjZbp7nb9VQUtCAAkmOfjaE9DKdiox1cfRV8vrANMg5WrvSlQ1lWvyP32a39imC6lpd3MnOqAC0DAM/wBfoWshfVyoVVju6BJkH5mzZaw8isvgLQM6Ge9qoHcHl9a2NTUEttSgyVMbnbKMk8MbqLKC9dFKpSid2hr05bRbP9F91ggG6qYKTHeDYkyfHn9SbTe1cgdrkJUgJSlJy6aTyI87N3irXnA+B77xGyUdtRUi+zOSD2yhkbPuUR8PO15lJITa6Z08YbchV6xJWcN8LVC6KoqbtNY2ohbDbaFKCphXe29qAdZ5RaO784jYYbzO3ZdFIcpCSpNKlZSMxmAfE6SOXIG0H4mxA8isdqHHnFKSSnvKJKgrnr4qmPDaDYOnqRW0pccKpA0bIKshH8swTtMctDzgdLT4VGwAuJJXPT4i95s0WT3vjiPeb1S0G10rjTaloW2uUKAncKghIGndMb++xa4MT1lNja4L6CkMro71pS2oHulKXRmBnWSFDX/EnlNmiw/V0jpDThAI0UZJBkAgSNyY2nkecEdtpVMhT2dSHCgrStIyKROqCCBMSEEHQg8tBGxTMbBI1zRss6SV0gsSvQXGV4YGwhfd4UeKcU3BS0t5MN1FXd1dVM9sTBAWlgErIUIBASQYEHS1cr89HrhXiJmuxzhelCrsCnOzbu+uz0bikGFZA2SQSdShKkgA6AaS9MK8McO35h6nqWKSodVWzUVL1QhtLri1AZlOEqKnDMklYk69CLC02AcRYAVUN4Dfao1X19nWNICGGe6f78NQWe0RoElKEiFwSqBbpMS7LT1UxrovCHWJAR6XE2CIQSjNba6hG4LiwtgehvnDeJcO3y7WYkpy09R3fRJcRQtLOVDinCrVSRC8iUkqJEETJhVyuvW6w/dbNaXQw640VFpSFwkwVDMkLEmTCo6RuLXbu3hu5dzjiryzrUFFTpceU4tSlbrUtRJJOWdSdtY1mpPGTDLmGMf3zSIS6Gapw1rXbd4kLGYyfNIPKfG1TFcKdTUzZCNAbIImDnWCVeF/pBcUOE1b22DMT1NHTklL1F/eUTxO4cp3B2apEwSnN4ybXO4E+nhhziJiSgwnj+7kXdfl8VQYp6ykcz0j76zoFhfeaKlaAyqSRsLeatSvIwiFSRBUDtyEA78o91nFwh7ao4r4PaTmWVX7QAJG5PrCBvbl5IwRcozHkFeqnHG6qV7EN1V9StMKo3Gy3qCtSVmBMae2qzAYuXOlvtXWWGPZTlkz7iJVEa9J91pm4yXS9UouqrbpwsIddbzLXCAk5TmPX2fDW0dUNzrdWG0I7VU6JRJA10HMW82xl4jqnBem4I3iUbD/ADdFKG52QAmhbSt7LmlaSpXIa6QN+c+dly77odox6s4GHFE5g4pailCQNyI3205dOqvQ3HUttQ5UlKj3C21OWOijME/GzmufDi6hQWtpa0aEhLkfPbf62tzU9UG810LIRa6bzOHWnVhKn0vKMI7V3MAAdZSkCE/MxtaoX7RLB5uqowNiBqVtVbNdRlzIBJbU0rQHl3yfPpb0EpMOF9SEppwgJ2yDl4nfkLVj/aWYNSrhJhzEKlrcXd9+dgpaRo2l1hZiY6tJ26c7W+zeIEYpGL6G4/Cyu0LGPoHs57qkPox1f7u9JDh/UOr7tVejDEzH96C0Nv8ANb2WoriSWkNtAhAOYlSjAJ3EbTbxT4cOruXilgvEJSoC7r3oqlRAMFLT6Vmemk299Lrwo7VFMoXl5SJ0tr9uYJKmoh4QuSCPz/yuZwCvFFTva421v+E0KW4EKgoWrJoO5oTPLUaD32WrvwsXIbbbSE8zMk2kCnwlS0jIfvB5LLaBqVKAgW0nEFAyr1XCt1OXk+NO1AhtJ/zH8Lc9B2dcyzqx2QdN3H6Aaok+PSS3EAv58h90lXZgNcdpUIS02N1rMe+Bbt2+sK3C56ldFK5fdeNmqdOYA+JGgsqqwrfmISHMUXqtDMz6pTHIg/5jubL13XRc9xMdhd9G20kcwNTbp6PAxEA6JgjHzP1d9m7D7rEnxDP/AFnl56DQfc7n7JlvXBxDxn3b2vRFwXar/wCb0ozPKT4q2Hzsr4a4bYMwco1NBdqHKxft1T57R1XXvHbyECy9U3k2nTOJ8LI1Ze5JMKseWWgoTn/qSdXa+g2CCJamobw2+FvQaevM/dLNReLbSSlOg5AW8I/2m9ym5/S5xZUpSQ3eTdFeCARvnpmwT/xJV8Le1VZeiiCSvS3kn+1suZTfGTDWJUIHZ3jh9DKlcytl92f+VaLTwTGnVmJiNx3BspS0Jhp3PTX/AGW+Lk4a9Kr92lwJRiG4a6iAmASkIqBHX+5NvXqrvYTova3hB6IWJ3MJ+k1w4vhCgkOXyxQrI/kqJYJ+DlvbWprZTJWetsjt259PWxvbs5v7Fa3Z6AVMRvuCpDYcDrLbwiFpChbsn3WTsPVAqbmpXSZOTLPkY/CygRPS3RUcnFgY/qAgzNyPLTySDflNlqQ9ycHPwsnbQmbOC+WO1owoAyhU6dLIBkHTT3W6KlddiyahtnrBpqEgT1sIyrIsKUNiPv1+63Bk6A7cptitQcupIIHvFrgVN6UgsEQBbYPLadCbAoWlaEuCNRmAt2DPTzm1oFVl2DJ/W3QJjbQ2DzQJJ+dsMbREfK0wSUkNm1mBJ0tsKFgcwkazHhYRK4OvLnraQ3UShQfmOtita4GqmjcUDBcLStP5knf3gWHzToCd7Er5JTQKfjVhaHh/uqBPymzu2SG6PLAGsW6oD9tGYb/KwayFAKSeVuqEy/EjU9bPa6ZOHtgnTMNNLZbjtY0z/Xwtlo5U914CjDF51TzeGKVDS611RpQkudwrHdOp3E2lLgZgHEmBbtvS7sQNMtpfqUuMBt4OSQkydNv4dN7NCgvRq78XU9/PtvFCK1VQ4EEFWqiYGw/DW0nJ4tXD2edVBXBJ1MhBAGvRX1tZ4A1pzFBmzOGUBO/snDyVCvPrO0W0hAzQsaSJ2BNmsnilcqT9jQ16hoTojQ+Pe8LCOcVbneWEro64qjeGzz12V4/0td4zeqqmF/ROMIQE6iBB0A8NPLa2EAqKSJkgghQ1Ph+fhZqjiZcSgJu6vnLzCJI5D2vz2nlbj/Sdc0hSaCs7xgwhHPkdfL61tHjNPNNwXdE7MkEpAkgaCJ5bfXWz3uO8F1d1MBTiitr7NUSNdPwj52hb/SfcsEpu6uGwGUJKZ6b2WLl4zXNd6HBUXZeC0rWFApCNwddyLCme0jRWYGFh1UyNr7LveGmnlr9dLBqKe66VLOXnA8/wGnjaNFcecMoUpSLnvUhMn2UaaHX2uv10Bc4+YcUSpVyXnodT9mJG++bxtUuVcu1SgVlYKu6MiYSIAGk6eW3wslYsUHLldAaUkKLaRCZjvjU+8g+7pZhf6esOp7v7kvFUrAA7kHb/ABaR8fvsh4g9IXDNTQrpE3DeoUohRV9mUgAjcZtNh9A2hJctKnG5oIuka/n0N8aeHC0AJSq/aIZVT3pqmjqI93xPjb1gYTz8Zt450WPbvxxxmwC5QUdXTt0t/wBDKX8uZU1KehOkJj429jWMuUqtnSggC6sxkOJIXZA8bBuCUkT4jlFhZzDU8uti61GNRoREE2ePdQek9xJKjGvj+FgHEgjbflYw7lmBpvpG1gHCNSdevS1oKod1GnFvifcHCJVx4mxGwHqR6peowQ6hspUpoqBlZA2R5xNm9d/p4cHKJIDwS5IEA3ixz13BM9bS1eXD/BfEPs7sxvhW6r+oqUmpap7ypEVDSHQICwlYICsqlCY2JsJSejXwFa77fBnBDcq0/wCwabf3otaiY8i7TZAe9l7EKKnf2hHBptJKEUupgZr0QNJ30QbBp/aLcHogJu/XSDeit9dNGPC05U/o/cEWPY4QYKTGki4aX/4NbKlPwW4PsgdjwqwinyuKlH/RYhEt9XfhQvH0VdHP2j/CBoED90CP571dHkdKY2B//SQ8K1hRaRcRSBJJvZ+I6/8AmvutaFrhZwyb/uuHWGE9SLopxP8AyWOtcP8AAbOreCrhTH8t2sD/AKbMRKfi/CcGMclVBX7RnhslIXkw+kHc/vWo8Y19VjlZx8P/AE12eLl+jCPDzCt13xeop11RYavF5IDSN1Fa2EpGp2mTayzOD8JtK+zwxdKdI7tE0PuFkXEeEMOUuJMOX22qkudVK5VNKfZbaazBbJGUlSYiQNOoFo8OXcuUhKwaBqYycccd6lvtafgzd6klZSCb6b5Eg7gcxbpvGHH6deEN1thO83u2f+sTaQ7rvDBeH7vYuX+2NI8GQopdrL0bW8tSlFRKlFQkyr7rcv4vwBTuFD+MLjTG+e82Rr/x2nY295Qv/wCqZNLi7jyCFL4Z3WhM8qtC9f8A2wsrJxVxrfAIwPdSOpKkKAHT/wA4squ4/wCF7cJc4hYZQNfavdiYHjn8R8bFHuKHBptaVK4m4QTBMhV8Uxn/AJ/CwXRB27z+ERshHwoL+0fGwoQBhO6Ukci2g8v/ANatir341LaKv3DcbawJGdsR8qk2Bd4y8EEgpc4p4MSRv/2tTkj/AJvOxNfHLgCwpOfivg9J03vRgg/81h93b859Qp8U/KEkIxfxgqcT3lhdVHh9ipu6npqtwuAJQ42+XQnIe3JMFlc6CNPcBV0PF2tWS/V4XKCIID8Ry6mLN9rjv6PlBxlvm/HuI+GvU67Dd3MesNPBxtTjVTVnICkEZgHZj9LOR70p/RoZlw8T7k5SEtOqPyR9TYJpWOHiefVHFQ9uzB6JNGF+JACQqvwwCFSAKwiAP902AvLBXEqvIKL2ww0Jkk1qlSTz/uvL42UXfS79Ghru/wCkuhmD7FHUE6eTdi6vTF9GlOUJ4hNuFX/d3dVEdf8Au7B9mU97lx9UX2hPawaPRIqOH3FJDhJv7DQVGXuvk6dP7n8LKFFw84oBsoOJLhaB1MOFQA6asW096Z/o4NwP7Z1C9/YuqpP/AEWJu+mx6OiFjs8SXm4BAhN1Pa+RI8LI4bSu3J9UhX1LdgPRavXhpxCTUN0q8d3Il6pQtbaEoClLyiVR9lrAPum1frrvX0o3kmsu/gdip1twdol5D93ZXkkSFJzjY6EaDe063h6a3o3VgT67UXnWBsktpcucrCVbGMx00tN+EKmkvPCl13tQAopaykZqGUqTkKW1oCkSBtoQI5WYYZBfwE+qXf5j7w/CpWh/0vVLC08E8VMKCplVdc45eI5bWjT0hr69ISgwCpjibgW87humvq0NIfq667nQt5ILiW4pxnH92TOg7oHO3pe42lWuZSo130FqQftPb1FLhPBOH2n1j1mqraxeXcdmhtAO/wD61VpsoY43Ai6d1bI9pBXmpfleaipfo9VJBykEEco0iQdvOeY3OsPVqFt9lUIQS1o53u8Y7uvSZHWNTFi150jz7/qxfl9CU5QQSV6+0k7GdIjUiPKyVdC3Wb5WioQtCghasq1FMq3106Hn8RbSGipHVPNAh5KwpQJVBSQO0KVc/foROghR3GhxwsOMJNMlJSheckJBJVAjLGkbkxGknUmyQ68yKY5ohTeVKUxmiQZBJ5jLygag84VKdCHaft22kkvZVgxCpVqTqBMd1MAnaLEB1QyFdHgTiFN58Lborbrap6q8KakQy6pwCUqQMpkJPtd0HfprAssVd4XpUX3RLvOnYC3g8zlQgBJGRRA0HRJjrA15Wjf0XKxdZgN1ph5Lz1FeD9P2mYg5VK7RAkRMhwGfPraWb/rGbsoGUXheDC6x19DFKvLnzLPdEjQ5UiTPMa29ao5jNRRvJ5BUMoa8oq4xTUzLeSVvvqLhStSkjLmkb8tU8to6gCsnpY3O36/dWJkNKyPB6lXmlIJIC0ySY0g+Gm2lrTUIrXaVs1iaUuinCXXmimFLCf8AKmBuRoNOdoU9KO6BX4BcrUTnu91urQtAJIAVCh19lZ6T100p4vTiegkHlf01RGutILKlt6UjrKAtQCm80CE90adR7vL42mb0HMDPY39JPCVMFJDV1PKvh8qVEtsJKoAG8rKNLRXedCqppKgtKW6UDNoNIA0M89Ofh01tLvoD37T3H6UGFRVultNciqo0cgVuMLge8gfK3kktw0rVi94XXqxxKuxuvuimzoSQ3VJV3gVASkjbnrGm3zs1rnwglynQtCg3BBJVoOegSNTsLLPGPiLc/Dfh3e+PL9u+trbvuZLT77VIhJeKVOpQCkKUBoVgmSNAedq5UP7SXgLTJSo4Oxok5o7tJTHl/wDTjW3mfaHC6+qqeJSsJBC9EwTFKWlpOHM4B11ZGkwlffrlMzTqpRd0y6VpVnnllSNDz18rPShuFCkpabBCQZUY0nymLVPb/ah8BmVBDuEMbDkQKSmk+f2/SylS/tU/RrZCUqwjjxShqR6lSwP/AL+3MO7NYxKfFEbK3P2ggI8LwrfUVwNlSQlRjSTl3tDfp38Oq3EvozX8q76QvPXZUUlekIElKUOhK1dAAhaiT0BmwPDX9pF6K+KWKqorqnEFx+rIzobvG70y+RulPYrXB/zZQZ3tHPHz9qBwsvrAmI8D4Nw/frKr1u6oom6ldKypJzpKTmlzuggkTBInrbWwrs9NSyNlkvnB0AFz/gLAqcRkqTl+HmSV504YF13Nf91VF+NOLpaesCqhpqQSkqCjlUecAxuRmt9AuEMXf2uw3dN9YOut0UN50TFWzVVQy/ZuICkmJ1MEeFvnIvbFlPeLv+rsraV2gXJAiNY0HmfibeqPBz9rN6PODOEGDMJYnuTGH74uS4qK7a40l3MrZU8wyltakKLwJSSmRIB12t31TRyyObISR1sLn7dFgPkYW5Wi/wB/5dX/AP7Kt1boqL+rXK5Q1CFaNg+CRpZVQKGgaDdMyhtI0ASItRB79sX6MhScmH8en/8AZtP/APlFkiq/bDejg7OXDOPv/wB30v8A+UWpOZJS37pTku6kXPqULK6SwkeLdL6K/dVe6EJMKsjVd9Eg97a1DH/2u3o7uJ7uGMee+gpf/wAosnP/ALWn0fXdE4Yxz/8A4NL/APj7c/Ww41UknhOWhTxUrPeeFeuqveQTnsk1V7knRR91qNu/tXeASz//ACvjffnR03/4+xZz9qlwEUSRhbG2o50lN/8Aj7c/NgWNy/8Ahd+FsQzUDN5ArtVNepYMkxzt59ftWbjN4YfwPiBtAJpna+kUoH+YMrSP+RZjz6Wcq/2pvAY7YXxqf/sdNp/9/aEPSt9Nbg7x34fUuGrgw9iZmvpa9NU25WU7CEZOzWhQlLqjMqTy5G17s/geLUWJRVEsJDRufqFOuq6GWkfGyQXsqdYPvtzDmLbhxEyqHbqvGnq0naC26lY+63vO24qqbQtkKUHkhSMvMHa3gAXGoOUrCs8pECI8+u1vQ+6v2leC6PB9y3HWXfidytprspqas9VpmaZgvIaSlfeDxdcTmCjJWienK3U9ssFqMWEJp23Ivf72Wd2cxGKi4jZSBe269LsFFaLnNM8QHGXVJKc3eE96D0OuxsvH52qt6CXpE4W47XfjCmw7RVVG5cz9I+8y/TIa/vw4AUlK1Ff9zqVGflFqCTy57WVBTSUlOyGUWcBZFqpGTSufGbgoN9CXmltH+IEWajoKVqQT7Oh5WdiiYjptZuXs0WaxRSNDChHW27RHUhZlUNMyLFcmT8YtmbY6H3/hYMqG+Xa2T1M21AFmk3R2mUexSAT3ZR7gYsLPONTpFidOsHOkKJAVtzgpB++bGAox7VrLdQq50KFJO2sx0tgUDr99uArlI91thQnY2INEyECpgHXw526QSICTt99gSSYAMaW6SoDY677WkFBDBXXQddrc1CQ/TuMqAIWgp+ItyFDUdRtboq52kkuaR3taNlRkkoE+ca2M0J+3G8DlZPu+Ay5TwPsnVpA8Jn8bH6IgO6ybME3NOEAwNVjT+U2ywE/4494/O2WbVOvBLFj1RdVzVFVSOBL7CUqCikESVBJEDQTJ8JnY7POl4L8R32EKTjO6klSAsRSHTNy9n7vHaTZp4+addw1WNpRJWWyB/vp28fz8bXP4bcLsVcRlmlwpc79c5TMhx1LSUhKExuSSAJ1jXrFngYx9y9AqHvjtkVa0cEeJAUD/AGzurLvpSaAx5dOf52OM8B+IjmZs4yuwqA/9CO/w+ptYiuw7X3Jejl1XrTqZqadwtPIdTlUlQ3kdfMWn7g36OTeP7mXfTleKdpIyJzJzZiRsI5DT8bHfBCwZjsqzaiVxsvPRXAviUpIYRjCgccyapRQKJnTQaz8bBVfBLiRSns6nGFG0txEoCrtIPmJjQxHu+Fz+LnDF/h9fT13qqJLZA7RM7bxpr0+Nogryt1Z7RS3FqTlGYyoDkDrP372dtPG7UJd4kGhUCK4M4/UEhOPqGMuXS7xrvO/ntbocGcdOlLTnEOhSUKETdiSNNf5tt/qRaaydZKQJEaazpy+NgVCEgISZUNRMeOtp91jS7y++qjZPoy8THvs1cTbvPabkXYJI+Omn4eNu2vRj4gKzLd4p0kHci6ZEkbyVdJ1+dp/uCo9dulp7tSVtQ0oHckAR8RZWaaQlQCgCI70E6kae/e1F7A02WrGA5oKrafRZxmpUq4uU4MkkJuYaf8402+hZNvv0VMTUtGqre4rNumUpUn9yhJgq5faTufna0ak5kqDigNpBG8yNpsk4uJNxvvrKRlW1ISQZ+0SDHjpvYLxZpIKO1rSbWVJcH4aq8J8drjuCpvX15V34guuHUNBorCnG3CCkT1A3O3lb21p9WwSCR57728a6tzJ6TNG2lQM4hulRKdI1ZG46EgW9ladOVkJ13/G2fMSbFTjADiAuiBrCjO1i7o38NrGzH8p157WLPEkRPO0Y908mqT3x1PnYosD6HKx51IVqoSZsUcAzGSdTztdaqTt0g4jx+3w6p6e9nrpdrvWnxShDbiUFJKFLzEnl3PnZF/8AKhaCspwZUHKo6evJHOP5LJXHdSRhq7FBWSL0SJ56sPWhcrCFGUgjoTP1z+hbSpxdiz6hxDtFYJPpVZDCcBu6fzXkAZ9zdulelo4lHcwGD4KvOBtyhq1eC6ADCoyiD1n89futvPKiCZGx0kR8I6fCxiwKvxXBWEHpaXidRgGnTO03kr7+ysEr0tr7UTkwPd4CTBJvFZj/AO7tABUZKDBIMAADUe/3eNtFzLmKNdAJkb/X32YRhPxXFT9/5WeJCPssGXbmJ29ccP8A062jPjrxRxDx4uK6uHF43XQ3VRXheIW7UUzq1vJ7NpawAFaaqTB8LM5DqjCgkASIkCAI3tpt4C/bjWFjuVsQQebTg/H5+6zujBFikJHA3UeXn6N9yXS+lly/rwd7T2VBLYA5aSnoBYJv0fsMEBJve80iDoFNa6bexvaasbAB+mKz7aTlPUTIizadS6QlNMop111gTpYLqeIGwCK2olIvdR2jgBg1IPaXhei0lUd59AJJj/Dp/X3n6b0fcAuJUVPXooySo+tAAAeQ2+t7OpXrCNFE9yCYnnH6WU7sbqVqLocJZSTIBJ5dP0snQRgbJ2zyE7pns+j1w+ToBeUE6j1uOcRtrzEWFPo/cOwCVUlYRuQatR5eFn22+JHdAB0nbUTYwHhJSo5tYPXw/pYfCj6Iwlk6qKxwZwJS4zcpPUaxdIbiRU9gurWUl0VK2yrcH2SNJjawGI+F+AqGgNTRXIUlCtvWXTmHT2uWnjzs/wCpj+2NOAlQL2HqpGpOvZ1jB06aOfMWJYgbS/dlSkojKmdfDX3bfdZNiYWnRM6V9xqq+1F2YdocYpoauhSiiNKt3Kp1YCSJ55pnQWR10NNX095111uf6vQZ1JVmOZahEASdEgCyvxBCXLzu+7GVILl41CGSJM9mkyr7wIsnXShLFNjChIc+zUslO0pyHaNfD4+FgcME2Rw4gXuuL8TRMYQu69qenCXVuNhxRJMg7zr0+NlDF120jF001TdlKlBD7bi8p3RHekg7ADX39RZv19QHeEFI5lGdCWiJVlg9oNZ5acvEWGxxfjQwlS0/antH2mllKFDNlSBOnL9dLOWNASDyUvUNDSXpX0l5OUCWqBVShDKQTDqTzKehOvPa0gMekTxsurHtfguk4k3wxdF306WqSmYUhKWUpIQAFBIMACIn+GzPerW2rloVtRp2KgpIGUdwxrz57a/IWI0V309+8YMXFXaJdoH1BsNuCSlSlSTrBGgPu85rl7Y9URoLzYJ8Yy9IHjhQ1tyCk4uYkZZqKrsn0prSMwzJBG08ztraNePHEHFONa6hpMSYnvO+F3e0tTfrtWt/ss6xmCcxIAMD4DrZ/V+DLjvFFNU1dS9FKsKaIcgJjcQNP5flrFoa4x0dLcWJaxlpwllNGzUOBagpSlqJAA1E6GI6Zd7RiqGSPsCpuhfGLuCjW9WbmZYFZeTrocHeATEue1oJ13nXpBnqhXU+aquXUKDjbKGyEJMkJBVpvuAfu99hU3bX30+XnnA0ypQCUZ5AJPw2AJ8culuKBz1e/wBxpa0rIZCTlIOWNNCdPERtp0i1y6CAlStIQlTq3G0ZoIycuZ567mSTqCZ9rRQuFdL6g+yttE0jhCyCnKUqAKRJMGCFCdD4kwLJNQ/nRmU4MpOUzOkAAakwZjYfqQ7nr2qW9w0FJyvJLalZAOzcnTWJGo5awrrrZwUiLqyfow34xTLvi7XnlNKqa1DwUteYZVMp0JJ72bLI13OgAm0xYii8b1ZbpQrLRvNLSEoJBOcRkERAI3HkIM2gX0cK+hpb0v5h2mbfdaYZdZW23KQsqVMa92cqToNhB1Gtg2qijuG5U194IJqKlxBKFwIWVARMyImTOwSelvRsCcZ6JrCdAqsoyOzJTfevlbgRTILbTOeQFQVyUgqnfaN4jMN50j/jXR1tVgesoaynQl2qpXVEqCiJGx25k6CBAjbe0rUbl7UIFJeCkhltxZ7VB9mZEQBI/jnU6kbSRaD+OlVijGSqy67mqV0VPTJUkrRIUBGoEQBJJlWsDpbWrv06Z25JFrIAsXKoFI6s04qFZlFsqCgsghMAaEnlJnnv1Ih8ejXcldXekRgahuZKg+zf9NUJUknvMIV2ijOmyEqnaRaPaEKoXaijWQHWVqQogagpkE6eZ36aeE2ehzUXm16TmCn7vofW+0edZfIJORvsF5l+BSmTr0iZt49OMoK1Y9SAvUXiNw6Z4q4Fvvh69WmiRf8ASGj9ZDQd7ElQIWEEgKggGCeloJw3+yDuh8pdf4qVrrR1KlXO2lIEzElw/KbWww/VmjvelebTT5kOCO3MNEzAzdBrvy3i0tLwtf8AiBXaYsxKlFJ/6Dd0ttqH+JZ7yvkLcfiT6lsrRC426CwH1JO32W1E9jW+Kw8zcn7BULxx+zN4F0uGV4duHHF/Xli3OlLRoqZlxAE94KQfZ0nXtBqduVkPCX7GFm8UCsxbxdrLvS5B9XYuxtbgA2lRXA05Cbell20WHsOUwpboomKdCf5EiT4k87Y9fiQD3xapHX9yJdUTX/8AVu3qdf2UZJDMMsTPud/QaD8qgzv7GvhmyiaTjVilpyAMyqGnUNOUaWbOI/2OWDKS6KyspuOF+u1DFO4tptV1MhK1JSSkGF9dPfb0Nqb8PJXlZHr73C21IUuQQQdbZdX2pyG8Rsiw0U0lg7ZfOJiPD7mH3+xW4Xc5WknLlhQ/rPvsgkk762lHjldT12Y2vu63GC3+7r2raWTpOVwj/pn3j3RcbeiUcpmha87kLLqWCOQtCwEjrrbWpMzZwYXwBjbG63GsGYUva/nWBmeau2jcqXGkyAFLS2CUgkwCRBIMbWc7Po38f6gEs8E8dKCQVKIw/VQANyTk0HjYrp42GznAH6hDbC94uGkhRvHj8rZH1FnyngbxmVd5vUcKMX+pAhPrH7kqezzHQDNkixu7PR44730Cq6eDuMqpI1zN3JUER55LMamFupePUKXdpT8B9Co7n6m2ptKn/kq+kh//AGOxt/8AuV8f9NtK9Fj0jU78EMaj/wDYz/8A8Nm77Tf7jfUJu6VB+A+hUWTbJNpTT6LPpGKH/wDRTGI0nW6Xh96bIONOC/FLh1SUtdjvA173DT1y1t0zldTKaDq0pzFKZ3MWdlXTvdla8E/UJnUs7G5nMIH0KZxKOzSQCVGQokaDyta70W/Qww16QnDeoxxX8Qa66KilvV67V0jNCh1PcaaWFZisHUO7RytU4CTA10tfn9mji1LeHcb4UdcSOwq6O8Gkk6nOhaFn/wC7b+Ns7Hpp6aidLTmzhb91pYFDDU1jYpxcG6tD6Efow3T6N+NcQuXRjisvlrEF3NtOMP0iWkpW05mSoFKjJhaxEc7XEUAB0tAvDe+P/l3d7ZX3XS40T5tq08dQLT0sgm3N4ZVzVkZknN3XXQYlSRUcoZCLNsuTtp8bJd/U6i2h5P8ACcp8rKZE6T8bB1iQ9SLQRrEi21TuyuBWRKMzSE0gQRr8OQt0NQAk67W6W3lXlI3+G+otwQRzJ87bQN1jrthSU1CtNFtg/An/AOKxoEakjfexFJipaXOqgpGviJ/6bGgokyNxroLWI9UF267WsBClRMAxA18rETfjYb7Ruiqz3gkJyd6T4T4fUi3dQ3WLUg0tSlCdMySkEqPmdtJ1twwxegMvXmFgH+FoCR7/AH2sC3NQWv36Nhd1aZ00b8/H4+Nh03oolQRRvrCUhSTEBYMbTz12MWBTS3j2SgbyPakkpUGwQE6QI5nTfxtgpbxJSTeikpgZgECVEAbdOvTXytMZVDUJQpKpVQz2qqdbJkpyrGog7/jYxm55vOyQijvI6qvlRBER2YAmInrM62U0kaAnUDXxNmPknC4YHZ1dSkyO0SlwT7wfuFlCintZI21M2TVKCK9gn/aJWjptB/OynQwHdQNdpsyY7pYJcn+P/itlu8q/5j8D+Vsskrrwix0Ut3O64AmVvN5UgR3ivn035G12+DvGzFfCRblbhlVKU1bKG6hmoaztuAbEwQQRPI8/G1Hcfqi7MpTJNS2kykbZtvLzB5xvBtvgekbrr8u26n31N01ZUM0dQsESEKWATBHjt1Fo09g1xdsq1W7IA7pdODEWNr1xhieuxNfjra66ue7V7KnKgdIA5AD5a2lrhP6Rl84Au9y6aZNPUU7hBh0TCo3GotCOO2XbrxZet0VFUKhd1Vb1AhQBnIy4UjyGnzskCsW0vVR7u+5nra+A2WMdFRa/4gpE4k8R67Hl5PXneVSlx15ROZOgHwtHVSorzKKwrQ5v4swnX46726FQt0FOYBRkmDqBtH3/ADtyumdySUgxA8+WotJoDdAkdURX9mVBa0lJ1JmZ1+fPzsC4gJC+8mSDmnw5mfvsO8ypkKGcApOgkgzGu+362Kr7EgGPaVlgiYAPTfaetpqVku4OrS1eDlE4MyKgFQMmM41HxmPfZ8inKgtbYPdCiIEgaSPPW0UtPLYqEvtuFK0rCx5gjr5fK0rXZUoraFmub9h1tKiPZAJ3Hxtm1rcpzBalC7MC1a7BRILghMQCDHlHwsj4kZUbmqe0AUlKm1GdiM4PPx+FnI6krSY7oTJ1O2ug+unKyLiUpbuKtKkKOVvMAP8AMJ+Gvvtnvf4CtNjRmCp3eXZf+UpRuoSQRiC7EjSRopnaNDrPXTpb2UZ9kSdZt41XshbHpJ0a3m0JWu/LqJykAkHsTOn/ABafdNvZlhsqG8wdPC1N+wKdos931WKA3IA8bF3AJOXTrpY4ptIAJEE9dJsXdEbAd6zxqMlkQfT/ABGxNY1knfbWyg8kEa8uosVdGkHny+vfa4xUnbqLePCJwrRKRui8myD/APVOjl52gtSVESpwiecTawPGulNRhBsSO7XNK1H+FY5+doL9VOQ5IMGNOk/QtpU7hlWfUC7kSKHE+06SQZIHPQ6WyEhJK1bEqBj5WNCiKl5QrlIMbDmQfjp/Wwhu45gsZYVHTQ/Gxy8KuGEpOWnUHMesgb23lURmUo68pkedj3qLijlCisAGdJOnh7xZSGCsUuaowxehCgYUKNzaPLptZuIBumLcoSAAAoEOCR46HUaRHXS28sXpdBKlaXg0NY/ilMfOzhbwXiYmFYfvBASI7zKkjbTePGwNXh2su+puk19C5Tl28KbIF6EgOoCo672eQPa3MRomicx7soOqU8ZsNvOUzmZRAQuIMDkdf6WQH6SiYbJqHy2FzMqglQ5fX52kLHFFSU6KPMNcq9zue7zmdvrWzDqqKjrlBpzVKDMAxrEWBmzOurIZlbZI+e7O1ITUI7pglKzGpPTcfjZZux66lMFtypl9xXdbBIBP1Fk9Vx0CVplWm8lXQWV7pui6FINVCs7ZzIlU9I+vO0pDcJmA3Q/7uaUkuEKB5iSLdpu2kV3SkyQCRmyjx0sYbcaQgAqA0mI57zba3QFE5yEgHQnW1Y5lYBaEitXbTPcRcLUj8hiqprxpV5SQcpcpVEAnnAJtYo+jVw7rad6neqb5hYKFZ3kAHkdkTavS6lDGOcIVadOzq69uVA86dK//AHYteqnKF5VpU7KpC+6DPlP4WdhNyEQNadVX4ehbwGr69ivqrvvSrrKdyWXFVzqQ31PdTvFq6q4G4Iub01bw4Su3atzDF80dNUdh6wvOsKp3CZXIVPaI5Hp1t6FpLKKxFO264CpQMEgc9rU44vu09z/tEcEValhBvO5aJkAfxK7V9ESOcdfw0T9BdStqpFV6Gno5U92i42MDldCjRTLl41K0HWYADg5iy/Reh36NtRTobf4X3W8lXdX27j7q4GwSSuQOqZg6dLSUimSlbgNP2ozGSpBUd+tlKjWhitQ00ltDaVbjKB99nyhTK8oeJ1z3VcWLsSYeudopaui86mjaZjRLbT6kITEyISNz1+LYu6vbuPi/iu8jop+haqCDPeKmWzv/ALwPPU2lD0iOxTx/xndAZCnBej7hWhspzFZC9B5L3/O0NY8qv3VxJdfhpLddc1ACc2i4ZbAB96OfXxtgTvzZmq3C0MIcny/idx9r1dNMUzBUEg+O4O+p+pNq/cfMRF/ESqenq/tlsMioCTp3QrKPCM34crTSw6EpyoUSkkSY358vM8udq/Y7rbiuvEF4VlawuovJ+ocWULSFKPeMKHJIMCNJgDTe0MNitIXKxWyXYAmrQ3UpTX7wvmsU3StpDmVLkK1nQRt9QDNidxEVt6POMjszULCUJ1UQkqGnWdtfzsSve+7yvJXZ1CuzanMGhonpPjt+UWPYWYZX21TVEpYbGVRyk77bAzoDobbw1KyksVyUOiG15iqVKAVJEqVB3BIjXWJnxEAUNMyh1Km1r7SJXASoBO55kcyI59NbdVVQqtdW3TH7NRhKQgqBSNJA3946z5K1z3a52SHnyvtFpzNk5VH+Hr1kE6z5SDaQTXspU9G5+maxnedOtHZmooGnWyXEqSpSXCConY6q0HtR1nSxNUmkvq9WaN9C/VKZKgooBObQTsfPQ6E8zAip3CtxN3Yu7OnXToNTSko7QCCCpBBTH8UAxA6nu2tfgHLWAO1YDdMwTnzQSogGQJOqTJEa7jfe3oPZQ54cnmqdUbJ//vKlvC7qZ6vQhvM22pwkTBiCoHLI5eMAdNI1xBfdK8XqOnpEvOOLWArKYHidNDHUTIsttXbfl7FdTTlTFDXIb7FlDZSppABMgiFEkRoTpJ8ij46qbg4Y4bcvK8B2lW4nsaWnBSVrXlA7oMRoB3jEAa7ablTNI65Is0c1ARty76qkXEe628McQ7zoG3dA8KhfeMIUtIWU+4k/LeLWo/Z3cP8ACuLOId441r/WfX8IUjTt3ttuFDa1v9ohTi41VlSmAj2RnJM2pvi696+98U3hX3opHrD7pK8iRlkcgee0Ttpppa2H7NTGDN3cY7ywvU1CUpvu43ENIOmZ5haXAB45S6fcfK3kdeQ57y3a5WlT6EXXpWtJgoCimRGvI/XXp4Wk67cXisuqjqUOn7ZhDmvikG0aEZACJBkEHx008YsTwzfS0Xf6kt3WmqamnA27qHlpT8ki3l/bR0kMEckZ52XW4HStrHuY7lqpaN+5v9p75sWdvmBAXy62Z7F5LIkqgdbdprHnJKQVC3mMlZKRuumGFsZunE5eqlbqsUerVLzEqs3b+xJcOFKMV+K78orppyCQupeCCqOSU+0o+ABtENf6XOA6y+hhfh9QvX3eTiHC29VTTU2ZCFKjYrJISYBCZsSCkq6zVjTbqmLIotBuvO/0xbo/dHF3G9IJHZ349VJGXbtjnGu+zgtWVY70Hfr1tYr0l8Y3lj7G154xvqjpad+/GmansadGVIypCAIMqBhB5mTm8LV4fTleWidlHc297wMPbRMa/cALgcUt3hxCsl6CnGK9uEfEe+nLsulq8F3zcy6YIWCcqkOoWFRmE6BQ1nfa3qbw8xXjjiVgq578rXbvpGq9txLqC0HFMOtOFCoTAQc2ihKSRmiTpbxm9HV15PFy5KZh7s11Rep80SO80vlz1j329XPR4r7wuS7X7lqL0dqKRCzUNIWkZW1rAC8kCQDkTpMaeJtzHa+kbm47R4tLroOzt5Yw0C9id1NzOELnbfTW3s4/elSmMrlY52gR/lT7KfcLKq6plhIbYQlCQIASIFkdy9JTmK9vlYhUXmn2s5PlztwJc9xXVMpC7V6Vn7wzSUka87Enq/2hm/h6WQai+EJIBJ73dPn9ffYkq+0FwIBVOaDyOtpNieVbZAxqWHq9UnUkjpao37Raj/e3B65rwSlS1XdiGnKo0hDjTyDt4lNrF1N9CNUE+AE/G0Fel4j9/cBcTsgEuUSaeuRB0T2L7az5HLmtt4KHQVsT/MflVMWjbJRSNHReVbYAeSlQnvAGbWa9AG/1XZxbvS6VOEIvS5XUZR/EtDrah8s9q13m32N4VDfLtFERGxMjbTaLSn6Kt9m4+PeG6kKypqHX6YiND2jK0pHxKbep4lHxqORvUFeX4ZIYayN3mvU7DV6+oYruisU5AbrmCoyNElwA7+B/C1tVSBsR77UOcv1xpaVoPfTCkqTvprP0Putei6a5N7XPR17eqKumafTpyUkKHyNuLwhpYHNK7PGCHFrghdTO8W5UNNvObCEd3aOlgtSQdd9bbrVhOTerGexfcbKdAZFi0QSN4iLKl+tlLiHAJBTFkjOIiSZM22YTmYCsiZuVxWLARkUDP2iADPUx9xsaUDvrMWJvLzU7gTuEkiORAkfOw5UFd4bK1Enra0xVXLuNASB8LYJ3g624JCjIMQOm1sJOsWM0qKEBAOs+E23MjYx42DmCbZKgO8SOh6WkN0yHCjPSLdJOkA2AnQk26BA269LTCig6x0t9g+f9m8nNtoDofvstUSSHQqeetkWvldI6ge1lMeBFlW7XA8GnU7LAV5yLNzTFL47KBK1T/mtlg5XyHyNss6ivCPFLK3vUaZMKDl40yQo7jM4np5n7rWkpHWbrvWmeYqFuCkfQ7mCQmQkyCPhasV8oS5e9yNgI+1vqkB5SC6OWx5b9bWQCmO0zdoolMqmJkD7/AK5WalAc1wKDWNzANKdPEi9rtxBj2/7+w+4t2hvCsXVNKKCk98Z1Ag6yFFfwJs28xQjNBCZmD5fnYamepmVDMlBSVDvBOpj6H32AcUA+oMCRIKDoIT019/x+NyMcNoYOSoNjytsE48OUVG/XNCrcSlskZsx0I+/lt+trD8e+GXCfCGBbnvDCV8Nv3lUhJ7FL6Vl1BQSXYHswQB5ExsbVYZrw2orQrKoEd7WRtE8/CxuovesqEIQpwkbAE+/rpP4fFnMLnA32RGkAEEIjWFJWpCI906ifAfU2JLSqZIMHY7DQ/rYy48V5SlKes9Tty+uVgXYUpRgERsRqOUfW/usZMizhXn7qo5TI08fGz+wFePbUDtA6onsHMyEgwQFdNev32YqilQOaITsAr6+jZWwreKKC+W1OrCWXQWXCZ0Ctj7iB8LAqWcRhVmlfkkF1JqkrcGYwAdIB0kfW9kLFb7X7gr1OtlUMrJCUnSNP1s6G2iV95EmdQNgQfu+/TrYK9sM3Zet016qu6aisKWVuOZKtbAUjLqkwRp4jW3PyyBjTddLHCXOFlSPFLbbPpH0IzAkXvdJVuI1YOx+7wNvZil1bBIHnO9vIDiu7d49KKjF03O3dTIvO6AKZLzrwBlrMSpxRUVQPLUC3sDSp+yA89d7AzZmNPkhEFsjh5rtSYOUn5WLPIMd2QT1NjkaRqYsA8kkSYE87Ti3Q5EmPNFIzAeJsSVnBnUjf7rKTqVEEQdPr8rFFt9NLXWbKk7dR5xjH/wAjXiEgqTUNKAJ09qNPjaCVuKAJBBUI5ESRayWPcPV+Jbgdue7snbvLQW86so7qgTry0Fo5a4E44en7S7kpOgBfUYHlHnY8MgAsUJ1PJJq0XUasOvjtIy90DulBn2t7Yc6gFrg5f8IP1+VpSY9HzGc96tuxJI/71Z2/3bGEejpi05gu97tQCRMKcM/8utj8RvVN3Sb5VEzailSVJUBE94JEabfA2sXT0eK7wxQmloKaaJdIX4FIsoIyjvF+SkOZz7EezrZpN+jbf5JLt/0KZ1gNKMfd4WsZdzS6G76KkcKlqapm2yUqIEpSATYT421Dw6+yHJSvDbSCyhG88L40D5aXTVCW5knsZkEnQGCD8edmHxSpKukVh128KJ1hz1/RLqVAkB6n6jbU2ts25n1S0sjrnItG/HjBtDiSiuy8Kt6oaVdSXX2wgiCoKbVCpnmi2xUYiZqbu5b91m0eCiCp4zXX8lCPFNKWW6RYTnjPIgkgkTHytGaqvUpSiNRO43+ulrJYz4fXZfCKcVdTUoCe8cigJ0Om3ibNNHA7CHcdcvipaKpUEu1KUkbiYyzyNs2PIxt5CB9VozRPbqoZcUlSQ4CZGuuomRA+duEVTqUBJMAazGv1r99poqeDWGkLZWzXqcZz/bldQoymI0hO+1jl18FcM3nUep0dK6+s94Ht3ECJ11IAInz1NtBlI6RnEHu9VTEjS7IHC6gw1D/eUVJJnURofDW2hWPgZHFjLrBJiB5/C1jD6NtGqR+6WkAwBmr1H3xNhmvRwu1CMpuakWZ1mtWY133tAQs2LgjcF/VVmq6gi9sLOqUJavladejlDUjUcvZte+gqr0fZS7TM0wBQgpK1K1BSDMR4/K1YOIeCLjw3f+E6amoWmSjE9M2+lKyQZbebO/8Anj32ny7ajiDUJo13bQ3KqgU0gJU++4FhvL7WVOhOg0tQkZlkIBV1sZDBdK9Q9i92qC0+opQDKSS6dJ8B52qH6WNS7cXpl8EL7co0B+sRQUrrgkpyprlZo/49Z8OtrW1h4mIW23SLw602lJWo9g+dJ10I1PhOtof464WTfeLsIYnxvdty195Xa7/2LWMsvNmnUHUKJKFHecp1nnYEjAxhJd6osERc6w1U8VDvZVL7eUQVbEAzz5+djV2sMuTVVORKAuAcolR9ws0X7g4pOFC0YvuRKzq4Dd4USdNRKhpY5S0PEhthVLW44u5aVtlttLF1IC23FbLEOQYO4Omp02tF1Q5p0YT6f5RTCLe8FQf0uaNNL6TGJ6pFC6lD6adztFJAzBVOiQJ5Ei0C8YqepN8XDUUbai0bmZc7yohSHHUTm6kCBH9PQDjbw/YViCorsRUNFet6di0V1y6BtLiwG+6CNdojfpau67ipK647tcfu6kqE9g4wv1hsFZyvOGAVCYlWn6254y3neD1Wm2jORpB3ChO+7wqLqw3WXnRFPbtMFTMapCyAAfGJnTf5Wr7UC/l1r15XvW0pUtwrdccZQtZHMlRg+HI8tNBa7XGPDLTfCu8ks0NPTKFOw8XmmwVNthaFKOgkd2R4a9Dan99XBU1VQxSuU7nq7RzEBv8AvFbgRrp0G2scxOthzW5SRuqVe0scAUxqhVBeFS67TXYopAzFZWQCBuY9x08LH7gaQ5QrQpglSASQATBBiSD4yfjZRvlygu8pYapQl52EIbgIKiecEd3UDUkHWOWiHh6sSHgl2YKlSE6kSQTruToPhz2toWsVn8k46ah9Zd7R4BDaJAR3gNNdQdJ18PjFllYCiVaZQkKG6ZBTAPUgQRoTPhsUxiscK0DtUtkgFYCoVqRBkba7CBMKEnax5ASUoZMpaIGQa5SeWWBvCTtoQANgLEFlE3Tj4bXDdN74urn78qkMMMMNraUl/IrOVqBHMnRMayII91orgxhhGjSKKmPrbBCQvMSG1I1J1JkySN94IE2pLf8AeS7vvOhCbwq6dLzayXKdIcdCTAgBZGhKZ3j2tySbG2sSVq1oScT4gdDeV5X2jTGTvR/ATpr/ADJjXXXTdw3tAcKi4bGXPVDdTCY3cVf268dUCqJ1ugYbW+whxLVOkjtFd4htvr4DfztCmOMJXjeDzmLeKOMriw+04fsmaoOuFtG+QNgCVQrwPnpEc3Vf1PiKixJdt3XTd1319Fh929LrrKWtvNb6qhp1swntap5pRyKXqUJ1GhBAmE7zosUXrXrqb9avCsL4LKH3lFX2hQVIhexkiY5ifO0n9shW3Y+Pbz0Vp+H8FoLTundxMw7w9eZvC9sMX9eVbXUykEKcok01O4galSUFSlxEwSBOmmtkfgXxE/0XcV8MY3KVFu6rzYdfWCqfVpKHwE7HM2tW4nQeMtKvw5iK52FGpYcbQYzpQrMOe5GnXn18bJLSwk6xbBqp21bs4aB9FBoLF79NOs1KE1VK8HG30hxpaVEhaSNCPAg/pZAuy7qoXleqgclO1XKV2ijlQApCHDJOg1WdzaKPQk4kq4k+jth52rqA9XYezXBWKzEmWMvZE+bC2Z6kG0t1uBncS3sqprL6qGqBBT/qzaiO+EiTJ2+HO3n/AGsp2y0dnm1iux7NzGOpJbzCGvDHWEMPhLT9W5Xvo0DNImRMfzK0jxE2j7ilxJ4pHB1ZfeE7qcw9RUICnHw3mcWhSgkd9WoEqB7oB0tMFxYNwnh8A0V1sl0a9o4nOqfM7W3i3933pctXdNdStP01S0W3GVpCkqBG0Hxt5zEKencMjL/Vde4OmNivOq+nr7xa469fN6Vt51rpzFbq1KJ18dfjPjbrDGCr2ui86e+0kNGiUFgCRCVaHnyBtL153BRXPeT1PTUiGktuFIKURsdPh0/SyVXU6hdlU0CEq9XcAEazlJ0/rbtoA3J4RosmRmV2qq56TOC/7H3tdtI26p5NZdhdbKuudyR7ifnatlXlNQsp2nTvZvna5/paU6Lxw5ga/SkrU7SusuKE8uzIBPLVRtTSubKXSIjSD5jT8Lddgz81OLrkMZYGVBsnNwgvL908UsKV+bKG73pUqIEylTgSoR5E29acA1HqSllIIKgM0bbj9fhbxvu2tdoLxpaxpZSumeQ6gjkUqBHzt694Yr23WmH6cqyuN5wJ3BAItm9pY87W+a2Oy0li8dLKWk34exCo0MJiYsBUP1q2PWgw6pkAEqIMfHazOq6q+XrtfcuHsXK5KCtpt0ZkLIkhJE8wPKfhZHvG+8cUF4uKpahqqo/9vV1BZaUhsLSp7sk54WjsfZJH+1BPsxbhe6i9gu54lgnfVXkEyQRIEExuPfYj+9MiwpB1Cs0A8/H9bRD/AGl4h1VO4jtmmiw6puoqHHqYN07YeMqcIkocCMkgiIUo5QQAUh7FPENbfb017XZ2IabU4XKilKQpWUqCCFbDK4BOsLmTGlttJYIJnCma8azK+4BlgOKA8ACev1rZj8UaYX7w+xPci0hXrt0VTKQT/GplUfMj6i2sP3lfVTRLqL3eLy1vqNO8S2S4wQkpV9kcn8REjkATFh36kFDjTpSUqBzK3O3nqLGhZw3gjkUCd/EjLeoXk5e6Yqg5/wB62hcxEmNfPWdbLnDm9E4d4h4Xvxz+7pbypKhU6d1LozfcbEsXXau6b0eu10ELon36VUjctuqFkrO4E07wJGQlIM7EGf8Aqt6hYSR26heUaxzX6FeodTVBCnElSd9YInrqOXONbXi4L3qb34W4arcwWRQJYJmZ7Ils68/Zt543HiD99Yeu2+P4a2iYqTrM50BXLnJ+Pwtd30Ub6/evChmmKyTdt41NL7iQ6P8A8Lbh6QFkxaV29a7iQtcpiW4CdvjYIFIVoPjbpYUFEczYMq33HytsNWM5Fb3bD1MrmUCRFm2oiDuZnQ2dSxmSUnYgzZsPo7J9SI1m2lSvuMqz6pvxLgZTuBE7T7rapyoUzQVqoJynxI0M/C2gSnROkfX4Wxmci0ad1Zj394ffbQYs526GS57/AB52zMVAmdOnO3BBSMusb72TcU3hX3Xhu9rzupppyspKF9+mS6CUKdQhSkpUAQYJA5ix2tubBQJtqlfVXI6HnrbCZ1B28bUmwN6aPE3FeMrhw5XUOG6SmvW8qWjfebo3iptDrqUKUnM7EjNz6a6WDxn6ZnFPDuJ72uW66W43aOlq3WqV6pul9h15lKyErUguSkkDY9DttbSGFz5suipe0IcuZXeQogga/dbvMTqCNrRd6PXE29OK3DWlxXfTdI3eHrVRS1ApkFLYUhfdgEmO4pM6nWfK0mBeuadrVJGGJxY7cKyx4kaHN2KGIBBBOn4WM4bJVTNJIMtEtxG2Ux+FiYUNhPMGbD4c7tXVsSDldDg/3kj8QbD5pynWA3HtK9yv1tlszDnr4xbLOorwsvFXaYiw4JVlcvqkBT4Z59/9Da1eH+GeNMUYavbF9z4eU9c1xNlyvrFvtNtsjLmiVqGdUa5UAq1SI1AtVQFbuMMMJkEfvmnJzbEAq2/S1hRfF4GiN2+vVKaFbgeXTJeIZWsCErKNirlJE+O9npAcpshVRFxdGG6tpKx2uciYGVUEH6ixiorbsLY7FnQiZWSZMb/H8bJBUDrmUYMHkYOlhmUodKc1QEQogSnSIGvQ/pa6QqjTyQodbOZKlDQSlMcz9ffbSVlRMwDEJ6+7x+utjNRQ3Y3SreYvVLjgUkBIGqgQTIPh+J8rECiMoBCu0GwIJ30sg66T25U9qDhfjG98BVXEq7Ll9ZuKgdW1W1DLzalNQBOdrNnSnvA5ogAnYWZjry1AlZlJOxMg8pH1ysZo6pxumVSevPsUz5HbJbXKVBJkZkgwqInXaycpyFrzIBEEnWJ059LM24vdLQrvMZKiSDMmDvHn9b2CQ4CvNlBKZkjcnkfrxti1Zs06lRGXu6H8Im2kZcojvCD9T0tI6pWI1U7YWdF63LS3oO92rSUrlMnMNFfnqbLdddtSm5quW1rIYUCk7ER9eVmTwOvBNQqtuN4pzpAqWcx1UNlgf8p+Np2XcD711vuCQosKUJgBRAPTf9LcPis/dXuYV3+Exd5ia8Lzc4xsPp9J2k9Z7y03hdRKYnL/AHR1+Xu8pt7CUgHZJHKLeS3pH0iaf0r2xCoVW3OpRUQYJDUxA1Ezp4n3+tdKMzYG+nysSF+eCNw6LOqWZKiQeaEUCRziJ05WLupBkEbWNGCIIsXdAPKPfazGqkiIPJ3PP52LLBEEifPSx14abb8osUcQCSP62uA2Cpu3WqFkPXiw2oCFKj5Wcn7qSowgRz3slYdYadvqmS8FLBKoiAZymLPpFIxoMhKyYiLU5qjgvstWitw9U3EXTMCdRrvYYXdklWcCBFnA9dyWnMpSZiSI2tyaFKjog2Ea031VwFh1Cb4u9PenIqeZBsa7JP2aQFdxMaRt7zZV9RVICU2i3HDmK6fGdBddy3BUVdHVOhFRUpLiUsiEzqCEpjU67xG9iQV0mfLG3MT9lQxBrOHmUiIQkKBUpKI11KPwmzc4lJaqLgcCVJWpLLswoGO74WjnE+FMcLSpV0IrHFKgR2io35DMTtY1hK5MS0VNe7GIbqqGW/UwGnXUnvq70jUkE7W6uaiAoDVF4Bt7vNcxRYm+SubTGIgX35Jy32xmpaZZbnOAdvCwD+FL4vegut+6KV5Y7TKtbamk5AVnvqLg7yUxqkanNZdqGE1F20ZWAqUIOomO5ZzYZShOHWApWgcUkHwm2A3JVgRuO2q36uLLHqmHiHhxiFxeeidWoBKgA27oOYOWQfvtrBWGcTXDe5Vel2OIpFMhIqCtJ7+cQDBm0qJS0AVhC1eMH8LcXqibtT2QKSl1tUbfxCd7dGzEnx03dgBlXNtwqJtQJwTdNdLyytQXUBWXUd//APOtwy8DW5jVNhtAkjONPnYk/RVJrHUU93Z0CcpzkSN9Zt0xRrbqCp27UJAjve1+lqBcStUBQJxre7XFLIU2kCkxRdbydBpmqW0b/wD1nztO+GHH03Ddz6GkkCmSglRG4/py6WhDjDTmpxZeVIgKzt1t01MRoIraUz4/xWmvC6S7cdHTrCoZcdSvWIKHFJiPMfK2XT1YlqXRX1H9lp1VPkp2v+iVKt91xoF8tMgHKftNz8LRP6QLbBurDFc2FOdjeRBcSdEkgaT4wfhaXLxS0WW0qfcSdVQk/kLRpx1b7DAFNVFC3BTXmyvMqZEyB7rPi1+6PI5IeG/6hqkCrQczb6l/3qQQYnkN9LdsKQYCnFKVoBCDH3WApq0qoKN9DpPa0zSwnswqJSNjY41WVK4IQqOoSRra1GbsBVaQWcQo64y3RmvVpxPsv0yEqCzOska/PS1asH4fFdd6qAuNU4ZvKuaUsyUpAcJkgRrCveSbXJxvdibzTSvrpVnIjLtJkEHW0A4PwA5TX7iK565ioOS9XahpaUz3FpQobGNYjXnbzzEq4U9fM08l2VDGJaeJM1zCtXelz1VCqkYdaVRuBTTmVKXUZCChQXpqNIPWLecDSnqYrWoO9jT/ANy0lKlCOUQPADx13Jt6/wBVgKnWFNrYqlkpnRAmPEzuJt5U4twjWYZ4g35gdmlqnHqC9ai722A0pTi1B0pQEpA1JkQAJMjfSdns3iUdW57QdRZZ+P0xjDHBQ/iq76h8evu9x5HflAgpGndOs6dfPobN26socCSlI27xBEe8edrFYp9F3j7U3Sa1zgziwobaLsLup1Dkc4bKc51MwBPnFq6M07rFQ7T1QdYdaWULbWjIpKhoQRyMyCI/TqQ9rj4SuaLHN94WT4u9aUNJ7sh0FYSVZpSrQAjcmDoSNvdB1Va1TsIeqVko0Q7vGpKYgQdgDzOh9yJd9W6+wGkLLpXlAQO8YgbRvOnjqNZ1srU+R2mV2ri9ICpSUqjmJIHh4HveBBDtdDG64p7sujE9Yy/U09RVpbbDQyrCCoiSZ6SVSdASZM7gvuk4dYFBaqV3cEIBSvItySUyDrpsdevSRANm1hGlpW1ApaDebMstApgJEHUGe7toeg5mLSM1RKr6dxLVQ4ytSCEpSJlcER3dCJJ1/ITUe7VW42i109KzE2JmmBclTibEVRRJRl9UevqrdpMsD/YrcKCNAfZjwEgFrXlU1tSlVMzTodfXKVlxCSkEADSdxPvAMgawVpVCxUZnF0q3FK75Qaggie9B1mfZBET3T71+5bhYu/8A199hDLmUFAbKiEpTvqdZjXcaEbmZotj8eiuF4DdVD2O8Fk0tPdD9a7U11e+X3VqRo00kZin3lSZB5DTYi0PY7wevDlSHUCW1mCMuUDoRrqCQrXwtYzEGIk/vOqbuWlF43s6kNgAgopmwdSSJynfnpGuoi0eYk4XX256xeN7XoXHX0BbrSUjKnSDB9+mm0baC16N1tFTkZfVS3+zJ4mKuLiVfvDKsf/1XFFB61SIUY/1ymlUD/Myp2T/gTb0raqjTKypJAIzDSBPP8LeJXAPFiuHHHjB2JV1Abbu2/adFQudOwU4G3dehbUu3tXXNPdokMtrXlJCoExEW57tRFnonn6futrs3IG1jQfP9koKvEx3TJJnfn1s3sQ3oQ0pKjoNQZ36/jYRp0rdNNIzA+yDKp8vzs3sZpfoshcQtPaoIlSdd/wBRbzKCIOcAvSiWsGii3FzQF5uupJBc75HIA8/hZoOvMZ3GVuIyuJI1Me77/efGzb9K67r6vjC12uXGxW1LzVaha26VtayUlpUkhPSB4CfG1YF4Oxo+sJXhm+1KCVJCjROkHXxB3/LS3dUFG2SEOc6y5DEa10MxY1l1MfH5lq8eCOHa1PfN3XkunJSSYQQsa66juga8xalt7oW3UqDhJUFKSSdzBi1wqu67wHowXndl4UjtPU3Re2cNuJUlYBcTuk7e2dAOdqj4iaCax4hMDMFeyREjYj3D4m23g/gzMHIrAxe73NkPMBI6SAqSSdot6ncIbyF48PMKV/a5lVFz0ZUSdz2KZ+dvNDCmBMWYzZvFzC9zLrxdbBqKvItCS23B70KIJ2O08rX89GK/EVvBHCri1KUpll1hRV/D2by0/wDhSkRvFmx5odCCORVjs68smcDzCn27a5lBKHKgMBSZQ9knKQDv4QT8rM++8B4Vqyum/eVTUsuOPLU0lKUtJLq21OJAKSSCltLcaAIJAG0HVXgrtFIGXdQ3BB8/nrZNfrXHElOeRMdJ3Ma7D9PC3HiOxuF2xm0subnuOhw8p5TNU+92zaWk9uUkJSHHFxIGveeWdZ0IHIyjHCNzUzjLtNU1iTTsllAzpMBTDLJJBSQZQy3vIke6x9yqQYSVbQM0gn4/CyXV4iuikKzVXhTgMhIWC4BBUYEjoZgeJsUMdyQXStCUqNTV03bS3bTKcW1SMJZQXInIJAkJAGmmwiwb1YtQJCQOsjvEzHWbM7EnEu6bkHq6E1dbUtPFhTbKJIWUBUiY0gk2Srg4iKvi/XLlqLvcpklbgacK8wUps94EEaEDX32kIHWzWQHVLb5bqnPHy60XZxNxMw2Dl/ertQDyyvpDoH/MfjaPQqaMI17jkjwzD/8ANFpw9LC6fU+IVTWhGtfQ0lWVCd0lbROvLuJ+ItBrWYtupGoACjr4x+Nu+on56djvILzyubkqXjzV3uCN7qvLhRhx8rzKbpjTqE/904tsD4JFrw+hXfHbXXiq5Fr1YqaarSJ/nQpJI/8AZpt58+i7V+vcMCwXCDQXi80BMQClC9+Q75tdj0Oa/wBU4h3rdpOlfdK1gciW3UEH3BSrcvM0RVjh5rqYncSiafIK4qyTpHhtzsEogEgHTxFu1HQBR8NrckSJO4EG18bLOcgzptsLIV8NZX88ABQmy6RH8PlpZNvhouMByYKDNrVM7K9Vp25mpECtRtvbQJQ+sH+LKr8PwtsEjWI0+VuXTleSqVd5J1+H621mFZDguys+f3e62nEIdbU0sBSHElKgeYPKwZJMkE/CwiTm0J1262O02N1A6rzUwPdbuFcb41u67aNmoxJcNHWt3K280HIqWKlKXFNoUIU6KdL6k7kESNQmyxflK/x0uG9eIH7pormvqirspdqL4cNPeCFJW6622at5QQtnVYQghORRTlBCQXPxXr8H4L9ILElJia4Kena9YReVHeVKy+qr7Z7s3ZkPJSIKnYKQNUpEzJsTxFxi4ZY07BzFar5vI0zRSy3U0IeDaiolRBW6dFdySEySCSTpbr2vc7LIByC51zGtu0lSv6Bl9B7COJsOqOYUdezVtidMrreX4S1a0yTrBJtTf0T8c3JUcXr2w9h+7qWkuuvuou05TRpYeccbLZIcykgxmcjwtcVKhHdMDlH9bYWJDLUE23WrRG8IA5IYHmBFh7nWpN+Fue69TgjzSqP+qxUHcETpuR0t3TrLd60D2aE9oppXhmSY+aRairaeBWQSA2rToB+dst0Qf5h/w2y0rKC8LaOf7X4WbSqc15tqkmYTlJk++0+BtfZgI72YyVCPO0PcNbmpsR8Y8EXDVkhmrvQ5+zVBIS2smDvy15ee1rGY1wZeGCq5FIuoRV0r5Utl1RyqIEaKE77Dx08rBpaiNruETqVOopZHs4oGgTWKVJM5NJPLUHlI+vnbpCVKJQUkgaSOY15/UW6R2qYJG41/xfOetty9yCSD3Z11Gv4eNtJZpaQu10lY0pptTYCn2wtAS4DAOxMHQ848+tt/u+rU4hkkZlQQUid9uWp187cBb0ZgogDUgcunl+PjbpT78hXaGABqTJj6/K0EuS6coWmnVBSyttGsoSSR+E/lZWoMLU79PU1rd+XZFGwip9WeccbdfSokFDfdIK08xI0IidrJQeeyZS7qDBE6R0+t5t3mcy5+0He00HXz8J++zXKkwDdF/VXVuHK0pMKn2dNbdtXe+tU9ivvK8pPhPn/Wwmd0H+8gDunXYdT8/iLGqdx4K7ripI1jSInXp9G0XPsEVrASnVwvoL6oMY3VeNNd7zqUVCUuhJAzNqGVXxBP32vrS8Onl0hCqcErTAhMR7rU14QXY/VXwy4/fLlOG3EqVlGigCDB/L6HpTSdmqmaUgCCgEfC3I4pQjFanIXWsF0lLVy4bTjh/EV40emnhR7DvpetqRTqYQ47crgVObOe6JI5bK/4bemtOn7MJ6DkOVqM/tK6mjR6UGFWmwA8KC7M8AAk+tOkeegA+HS16qdMNAxvZ4IhBEIfl09Ez5DMeI7c6rsjTQzqOdgHQrmNzYyQAnSxd3RUgz52txoEmyJOpkculiqgORmeVjjwJBg/CxZWnKR0iLWhsqTt0tYLol1F/wBMAkkJzKPlFpPZuSmYcD7aPtB7JKiQPjZgcOnUt3+G51caWn7j+FpSsPu0cri526mZXMGVpSebtKlZlLBJ5xbf7sT/ADfKx+2rIUEPRR4z+qTlXUM2YLIs279o3Ke8EoB0WJnw+hZ7WZ+Mkreq0MIcLZKU99O41sanpY4X5mpnSueLORFDD5khf/MbA3ywwq7FNVL6EBXdClGBJ0H32GZSkaheb/Ny+VkDiI8lrC7z6icrbrStAN8w62JWXdA8A8kSlIErT5o3WUNPR0bNK64gdkhKVGf8Nhbpve6Ke620KveiZCXCSFupGnvsycUYuSjDf7wIkuNsn2huUjb42g/+1rt4VaqTs1JbWVQQqduX3Wy8MjeHZyVcr5gW5Fbqgvu5bzfNPQXtRVTqBnUht1KiBMSYGmpFkS/+JXD+kZqKB/F9zN1dK6hLzKqtCVIUlwBQIMajWRvpas3Cbig7c2Lr2W/RNNs09K4yt3OZUe0bUInwGv4c6t8bcVJr+JuLqZKHMwvm8WlgBJzD1hcRrpztYkxBzah0FtAL3VdlLeISL0lVxV4fpa7Y49w6lDmYoJrW06DnqbJTnHfhC2ShXFLDSldG61C5iZjL5W8h70oO2qCUERIJ0MfDmYmJ/OythdbtE620llspdJylwDTSO6T9a2aTEntFwAnZTNcbFXdvPHN1439IfEgufECLzuP+zrVVTqaI7EOsOsLVGkk906/mLTvcuOsPXbelfhyovkN1jV93gyGyy4oQp9a0gkJyjRQ3NqEcHa5268WXqUENn+z95pMd4KHq6laEHXUD4/G+PDSuFSziq8W1gqexC66DodFsskffbJoXSOry9thcFadU1vdAHa2Wrx48cOqSodbVfF7PFtICBTXHVuSfAhuOdo19IHjRhS9uF1Vdd33fipdY7UUz6FPXHUU7YTnT7S1pCRII3PO0+BbQcDvaLDmqvaB1syvSEu5F48DcTLb7RTrbTb4UqVDMh9CtZMAaeQt0c0b5InNcdLdFjxSCOQOaiOHeIt4MYdutit4XY6VUUtCyy8TdraE50oAJlTg5iy9SY+vSpQlTPC7FEFYB7RVG3kB0ky/pZ0sKdqKSnUFEoUyhXdJ1lIOuhsKGCtktFgqSSJCl7+6LRax4AAck6RpNyE78EIF53A3W3ldDtG84pYVT1Km1rbAMQShSkmQJ0POxS57suR3Gd+/6o0XXAw97MT9mEz4+zYvS38q7WPVpykoCgmR3dI/C0f4axg7RcTK9TrxKKi72gEk6ZkqXr12Atw+L4pTRVrYZGg5TdxIGv8utakop3xPkYbAjT9/7KblXFdJkmjb11Olq1VnAnAtwek7fnFZFxMuXrX09EaVSzKadwtOIdebSRotYbSkkawDEZlTNasdojQjpvaO+KOMW6FNPialby1LaFUxezJypBIUnMD7R0WEgc1e4zjxrDpZWx0zcpPMC1/JHoqGpZJeYXH81RPF9+12IcMX7T4NBqb8oO2p2WWnG0qTUR9mkl0pROqdyE672rB6TfoH3xxa4S1fEWg4WJu/izRULVVUuUlZToF7LRBeQ622tTa3CnNlUIUVBIKiNLWTu2+mMRXawzclTVqr1OGrZKHWwipZXBWwhSicqgqVpkEb6idJZurEdcu5KVF40LgrOySHEOuo1UBqQQTPLlbooXxRXcT/OibEGyEBoHlY/uvnBpVusVZaqEdmtpeVSH0/aJgxGXeQJERzMxNluirHXstE2mFKXBQlsJgkbzy3jXkRpOzn9KG/KW9/SQ4kXrTerIaqcT168tMhSWsweMmFEyoqCieWYqIgaBpYZZcrq1LbKApLac5BB1IGgE9Qk69NdptsB92By57JZ+VSNhu72WWUCqqEBoASlC/4oJUdCRqJ6ezI5w+aaqoqWlXVKUlDQQlkpJgulQBGg31JjSN/ez7mp1LyjtF5MwS6lRSEpRsN+U6g9DoNdT6a5V44gTdbCVrYokdstG/fBImY2Gux5DqZq3VkaJ60RqEIaebIbU0hpOQAJDZA7wmNgSfAkaaRY3iC8W0Xe4q8L2NDRDWurguFBKQO63J5qOmg0g6zIbLTl7NNLF2XJVXgtxa0MttMyAoFW6v4B8EiSQoEgWaF5Grvy/vVb+CMS3q2rPS4du5al01MJ9upcSCDB/gSSesa2ZrFJzwnBQXneeImVuYFo0YYw+2vK9fVa3nqahRkANiJBIPtEaZtxtZJxkL7ujDt40GErqQrtEOF28at1NQ/Ut6lSmiNBok+UGNdbO5N14+o0NM3veV9uZEBKLvuW7mkU7ImQgrdCW4BjmeZI11HdpXFJdvC9MP0tDQKKUupafVVuyBqXEspU2mYiSAJG82TrstZJnjuCqdds72/rGcl3Nnzc80zNvoF9GrE124qwzh7El8sMPJvzDlFeJ7RIWA6tlpaonbVareCuOruo7sxTX0920L1JRF0rp23TJCD0I3EzG/nb2B9FS/3mvRu4d3o24uWrgZYzeCMzceXd+QtVxYnujnsFyBdSoI884iJtc2VxqLDfDe7b1dvunuunTVOkklSipCSd8qCcqfcBaLPShq7lXh+53LuYp0OCrcQpTbYBCS2dJHkLN1eO3yDDx32n6+hZj8TcSO3xczTTjshmpC5JkbEfXu6284irp5rQuY0A9Au3iwcwSCd0hJHUqOLweDtO2tw7oIMiDueVkJ9BSolSArLoPET+Bsoh0u0pQQqW3VxI2mCJ95P1pYg+2VLStWqdoT49PrrbYiuBZHlN9UwMbUSarBPEW6gkSKduuAg6kIzT/wAlqE4jbUlxRJV3kpWJA6662v5i2+7iuWtv27r/AK/1YX7h91llJbWtTq05kFICQdgofRi1Br/RmJABzBGp111Hu6/K3QYOTc3XMY1Y5bKTvRKoLwvviJVYZu+vRRuXnQFGdwdxUOtkpV4QT9+1p79EaocHDy8rkWYVdN+1NLBABywhX3k6edqbYDxcrBOIBfIo/WkFhxhbXaZCQsRIVBggwoHqLWk9DDEH71XjlKW3GkuV7NehrNmy9r2siefsJExrFrmKxXgc76Krg8uWoa36qyt6VxobvfvAI7QMNqcKQTJhOYjnp+u/OKXOJd9OtuuKu5NI8232hBM7OJnfcFtYM7zNpWvSnNTdLjaQIfZUiPAgjTwPnZm4IuHB9+UqU30lvt3rpqiFB7sz2zdOS2TtOrZGXmRzNuQLmxNL3C9l0WJVZpI+JyCiitrcUX1dr1det+VgYYWioyJGQBC3uzUiQP4QJ/rZPvynXSVF6pbecW6aippnVrWZUlCEutz0Pd6fKz4xoxddC9fdz4feD121NO460M8hDiUhaoVzGk/U2Z98OGpvBxQImsdoX0idFdswW1QB8NuVr8JD2gqlHPxWB/VEm8Sl28HL+qAVBupoqpzTUgNlCwI0mT/WzjuasZXjynDTSm1OVq6pHad0hLrCTlMayT9GbNld1N3bc9JebLi1mtpFVOUgHIptbRKfHUKg2cD14C/LyexDc9PVlCU01QkFogqUhZQoaTOhB8uXKzyDTRGYTfVMT0xKMKvK461tvL6xRVVOpQBglpxLg/8AGr42rCx7SkxMoVa3vpZU9PV4auKtCwt1mvWyRm72R1pU6c9UJ18LVEpShNUguju5oPltbpMJdelaFzmMNy1RKsn6IV4lV34nukqJKHaaobRJ/iC0qP8AyptdT0ar2N28ZriClZUVSX6RwEiTmZVlHxSn428/vRUvVdDji8rvERW3avQjdSFoI18s1rfcL8UqpMZYWxEmnfpUNXpS1DjT7fZuNo7ZIUlQkxpM6++LY2KMLKvN9FtYY8Pow36r0tMjTTfabcBY2j3W7WkSRYMyZ10PysduyrOC0TCtJ1PhYCob7VpaOosKfvMGbcq10jcdbGjNihOF01lpyLj4zYs+uFsqJH95G/UH9LKN4tdnUGdcxneIsmV8ppysT3CFbHSDP4W2YzcArHlGUoYKiJM687Zmk/kbB5wdj4bW2FgiQDvqbHCCmhizg5wwxzexvzFuDaG8rwS2lnt3s2YoTsNCBpP1pZPb9Hrgg0YHDO4p379Pm+8mz/Jnmeu3O3WYEyCD0FjCaQCwcbKBjYTchNfDvCvhvhK8G73w1gi57srWkqSh+npkocSkiFAEa6jSzuziQTuTYFMyQDttboK5GY/CycXP1cUgwN2Q2aFQRqTuPrW3T6+xpjUwf9XUh4HwSoE/IGwSTr15AED8rDsNCqbdpiow4hSYHORFlyST2EKAISNddxbLItDfVKaKnLriCvskZiTBmBMi2WV1FeP3o+U5rfSMwIghRSmorHEk6exSPHTx0taf0gm+zVdcqUkqW7qkxO3S1VvR0ve67h9InCV7X0+lmnaF4ZlDUgKpVpCo6SqSbWo9Ie9aSrq7laoFesUzrLzwqmyC1qQIkHQwJ8iPK2Ixru/sNtLLZDmjD3i+t/8ACh9IX7KBBBkQN56/CbYEjMQolREA6fX38xbk+0CNUwTrt+mltLcWlTaEtjKYKl5hpA2321j426XMubcCdl2kFRJJUQCImYEa/f8AW8uzAnDDG/EapepMHXI5Xu0wSp1KXENlCTOpzqE7HbrZqNZipJJM5ZJI5gHTl0+tbTd6NvGC5eFWIKt+/qGofo6+mU248wpOdtQ9nuKgKBk89PGbVauaSOPNELn+XVeXOwXaE38V+jxxUwTci8QYkwuqiu9jV1xb7RjXbKlZOp+4b2jpTSm5JM6wdYMGfr+lrVcffSdwhxGwa5hu4aGuQ8taVhbyk5dCJkAnqfhaqzzrmVAbLamypWckwTtET7+fusGlqHygl4t0UoHOkLtNOS5SFkglSeUEjc/h9eNjdIUthJMAzIEjX8T52IHvJJkkRp9c9PvNt9u6xXU7Xao7BSV9oT7QgCPASJ38LWHG6vRgjdSPh2/F3U2FNqGZSTEbyIA+voej2E8Ts1OGrrqFrkuUbKyZ6oBt5QNYlpkP9mVkIKAoKC9zMbe75i19MKX5e6LhuarReTIu5FzNIcpewlxb2VGVwOZtAAFDLGs725DH5pKLLLEbHVdFh1MMQaYzyVF/2it9euemRdqUOEIapLmRy3DqlDfzOvU+63o9SkltMjcW8o/TfvFd4elqw+pKVDPdCM6lRslBj5/G3q5Sf3SDMyLFpZDNTxyO3IuhSx8GV0XQ2Q2w06WLuA6zNjPKEjaxdyZ9m12NAk2RR3TcTrr42Jr31A8NfxsddAiSqDYksiTvta23ZU3JawY/2GJKI5oCllOvOUkWl31jxtC2HXezvyi1g9smdfr6NpUD6dy58LU6ioMD7DmiRx8QXSqalI0zC2jUp/mFod4nUnGKuvZj/R/f130VDl+17UALnnMpM+H3WZ7GEvSVfWDU8S7uQJByoTBIkaSEdJ+Nsp2MyNcQGo/dNLqyJqQP4vnYnXO08turbbUpSwmSkEx52gqgwDx4NWp68OKbC2IP2Scw321y8rJ94cMOLTRbdreMlQ6jMEdmAsDMrTNoRsT9bgT8Ync33LJ20tzaysGp+kCNGWx/ui1esb3zUVN43pQu1jpaTUOBLRUcoAUY02sewvw8xtc1+sXve3E6trmGgUqpghWVWo17yjvEGQdzpZj40XWVN9XyKOhqHFduuDkVBM7gjlE2nSV76guEumnVWG0xis63NJGM8d3XdmGGbuvEuNBLTQUsxlToNTroNN7RZX3t6k8aynQVBIKgoRCkkcldI8Om9lnHmEl4muenpai6Hq1qqaTTVSFtqIXPdIOm28zZiX/g/GFEf3BdKaBmmQ203ShVSWWqdpDYRkInMDmIAM6xynS1R4jG0lhOqsS4aZrOCRWeIzVff980t1OJhmkfFWAUqOdCM0KAJymNYOugtE/F+voU8XsTUorKf1movSufQyVjP2ZeUVKjoPrY2dTmBcTNC9ayguBhip7Baq9+nvN1IcX2fe0CeaREK8OmpXHfCjGlTebNY5hJNTXN1Sah+obr6p5VQg6qTnU0BCiUkmd0iwH1Eb5y4m5VxtCWRBpNgowva96O7KZuoqKpLLfaNoJGuZSjAAjXf6HJfuPDF8OtuXulS0UqT2pKlFZypBKsqUyTok/E6WdNBwlxzfDjd3XZwifqKhDgdW6ahXsJBJnMkBIBiNOnnaTcK8D7+wjR3fjC8JpHqNDjjt2B5cBOZDYDpMZ9V5oAH5wmnDv049XdFXdTxxt4j3CybnDbD1et2nxGwAq77xuuqLKnEFDvZu0zgBKTqJkCCJtdXgOt1zB941TicwqaqmqTlOvfpGPDwtV66Kq9K69mVKcR6utS0qASJgpMDcmPrrazPoy1J/sS4AqAaK7HU5o1JpUjSP8AL+vQ2EFxnBeLbqpVEOgIBUpd1CR2jZEiRLh69ALN3jBUso4K4yLKkhtm6qh5ztZAyhMkydtAfhZx0bb9W8lIp3d9ZO3ysVxzcYvrhrii7aqnU2mruypSFJElEIMESNwdY2t1jj4T9FjN94XRPDb7l5YWuKspCpSaq66V3OlWhlsHcedlH93VrkypWvUkwfhZD4Xv0qcBYbz3klambqp21lSgCopQEydRExZ4IvO7SNK+nHPV5P3Taq2aMAXcEZ0br6BNLiJet8XPdrCsO3WzeN6mnSG6JyqDAcQlxCVrzkGMqVFW28DnaKa/EqKbF7TgPeQ0UKUDGmkDz1+dpVx6blvBNBfNLU0tY/RF+mbcacS4UBYSVpBB0koTPlaA8ZLCL0F4NABam1d5StQruAQBJjQyfC3lmPRsdizxvf8AwF22G3FCw9P8p3X5xOFz3O9ejjNQ+Gl99NOM69xsPeNN9LMy8uM9fetK/QU2EMRr7oIdFLKAdwcwmCIB948rIVRej5lSX8iErSciUkJEjaRrEjSIn4WI196vrYRDiUpUowhBgTpqdNSQBqZPwtnQ4a9sgkYbWVqWWQ+6bBP/AIVt4Hp8dsv3piJdy1NSs1BQ6RTMFxuFSFPDupWNwkpMjSRNrQU173RfDLFXdN8UlbTLK0h2lcDqCUyCMyZToQQddDahSHM9Q0673ZUMxUdIOwMnvbxETrazvo15XeG1OQSsUlbVNKTJAGYpXtsdFbW7ihmMjTG8LGr25iJb+S8P+MlSLz4vY2vFht4JqcR3m82lxICgFVLh7wGx1E9OtjfD9hosu16mo74SnUzGkQYjkd4k9NbC8W8L3oji7jOiapHVJaxHebUlIBEVbggHbX8bLWCqIXVRIS+UtFtQUCW90iJ170qjp0IjQgdTmGQALmADxCU5nKtu5qA1i1oKhK0hW4TuDrHXz1HhZLuCuF33YuuqWWy/WLcWolBQGxMgaRpAMT1POLI1/Xk9eTzVI04QiW1QYIMqiZ3/AJZnXqOvOIKpmkw+HJcSW2cspaAQTExmT4wec5pOthjdFJsjVxXXiHGFGq9FV1TS0Lsslqgbc7SobKl7qBykScvQDcjUWfOGsM44p2zdFwleGKN9QU52PedcIhOqUEob0P8AEpapPeVBtF3CK7qq91paN7Xsw00FJCKGrU2reYhOsakmNYCjytNN1YNuWqWad++Knt2SA4t++lFYVr1WFEzpGXqOoFgNQb3F067l4dXPSpZrL8qK283kAK7WtqFOpTGoGXNlT10EactBY7WjD1K+t4XK1U1a++z2DailsFOhzT4H/hjlYK7cBNNJC6LF1/IS6hRJavNahJJmBJEQUz4gzsAVZ24q+haQzTX/AFqnXXgS442hSkplXdlKR4GdfCdZUjfDspROs7UqsvpKYcxLT1d04kvRRXRvoXTN6x2apKwIJnUGOnct6N+jJTu0Pop4IQolJFwh+SJgFxbg+R+7laPuCeE8HYprsQ8PMZXK3iG7r/u9Dj6bxlwywsmUKJzNkdoohSSCCmZtZJjD90Ydwi3hfD9Cmhu27qD1KkpkEkNNJbISkE6mBzMzuddbUZf1IDGehVuL9OobIOoUcV5FdVUNa5WVDaqF1biUsvqS28S2pGV1I0WkSSEnZQB6WJ3zVLdplo2SCCNJjXx99gy8pRgDJOwUfl4bn5eFilYttynWrPIKQqCdQN/cfC3BMjs5eiufmam5cN3U92NV1LRh1XrFWurczuqclbk5ozGUiRokQByG9t3rdlFeKGU3gylxLL6KlsKUTldQQUKBG8HXXSw7S/tnQ3ErbSYAjYwJ5zqfrYvWr7QJypSYMjMdI3+vO15tyVTdYCyhL0kqd4XnhapabcUpw1LJCEZiYSDA66n4pFqkYnpFNVFQFtLQ4HlpWlehmOc68t/O3oNWZmMU4XrArKyiscZWSsZYWggEmY3199qZekLdyrp4l4qomwMiL0deBREK7TvSDrp350+Ok22cMltJw1zmLQWbxfNQwQQTr4Wsj6EN4djje/7tj+/utL2WNy26kR5982reqQpQGmpEWmn0RLyNDxmoqQEj94UVTTwEzJydoB/93barm5qZ48ljYe7JVMPmr7gzTJlQUFSI36Tr01tAV53NetNeFdRqq6amVSlLdGpx5I7YKccURG40Vt4Wn1umfUyUIacgHUpQTP1HX8rQ5ijBtU5eOJquku6tXeDDjVTTZW1ysBKHCECIVrmGnXe3HRWJ1XaVbBI0JpNV1x0d7Fb9/qqUIu9a1stNnvqLMKIJ5RqNvvsQbvzDz1Zh5dVdNcs0zCEJfdUlJCUuZUqWmNQDHhrrrbt7B1+rpKWMPXgHHqKkUVCkc0UUuNuJJywNCkkfG3N8XXiG8rquxScNXkzUN0gu97tadSQlaFtrzmYgGFAWttDQswNLdEPR33fd5oFNdd30TNNS3a/UBPZ51NkhaSEzP8Y6bGPCxOuvi8r6eZoWK55lirco1oDaoSnM2oECDp309eZspXVcWIbrxAhtCaVqmdeqKV9x99tKewU6F5gCoanOY+ob13XDfFA8k1C6EhpTCm0qr2IGVwnKZXvClHymzjLdTOYJv8QbscvDCd9XpeTT6bwu9ugrGw4TJBUlKiR1gn5+6ulYkMVzyUH2HVZSPA2tnxRauW9275ra+8qWlbFzvtt/620S4+kKyCAomDIjx6a2qhehBrFL0OcJVIMzoJ+c23cKdeMhYOKttICn7wJrU3fxbuwAqDb6n6cwRspteXz1g+61wUOuN/aBWVxEKkHUKGs7bjTx362ozg6vF044uC880JZrKVxRB5BSZ/G14iF5oiJ9lRI7wnr9fK1LGG2ka7yV/BX3ic3zXqDcF6ovq4LsvlBBRX0bFUkg7hxsK/Gxs7TGvKzD4DXsm+ODWEqsKBLd2opSQZ1ZJZ/93Z9FZid9dLQjN2hPJ7xWirkB+ttEpP8AEPIna2iZ1OnMGLYFCd9POxmoZSXe7RypcQNt/KyHUI7ZlTYGqgRt1s669pCqRUAT1FmusBILZMC2nTO8Nlm1Tdboow52tO2sgkrQJA1A0sMFAHVRsUpcyWygiezcWmOneMfhYcrAM5pOutrrdlSQoMjOTynW3Sl9IB6czYHNpJk/hbclIAkmBB+FnCS7B05TMwPhbpKh1FggqNQQB523PdM6DxM2I1MUYC5ESNNxY5d6z2gOokjlZNCifOx275L0pAI8TytJRRGrumtNU8WsmTtFZdtp0tlnWA0RJCjPOBbLNZJeKXDlSneKVzZs8JYrTqoCT2UT56/P32nK8alxRSlagQ37ObWNNuZ+vC0DcNcx4l3a+3KUilrIJGvsAQefX4fGbKxxa1ZlQM2kCdz49JJNo0oFlCdxCUbocbW/2D6Z7p15HedRvZeRTXWACppaxMQTJH1rZsXOHF1YQ0palAH2Rsn4672cKaWqKc2Rck90DWDp9fCzzuIcpU4BajDbF0iUpZQVQCN9Dv05eFszXXJUmmSBm012259NB8rFvVHSCANANSUyIHS2IoahSlZkEGTrEz89duVg5r7oxaOiMofoE94sCEkyZB26/UWDF5Ui2VBqnbQAZCQkRIjadOY91nPw9qcEXNelS9xBwnVX7d66VaGWKarVTqQ+fZVKSJGhB8510FmTTXah12pqKBisQhyoUpKHnw4ltkgZEDuzI70kmCI0EahbN+oW2+6TmWFwuqq9EtIU4G0IA0OgTpt+P6dYy4p4gdpzS9nWqpkPFTyHmnRmWpP8I0IiCNbSBfNCG7vc7ZKXEqRGVUqkbTHgT87RDjanaq7qVQ366oC7XnTTrGi0oDZKZ3OsHTnHhawZARYKGU2vZIlPi+svFppP+sLbbyOLcRGfss0cup8tjyNvTTB9/qewJdh76YuqmkH2ky0nffWD87eSeDayrXiO76Tt2mmmkn2gopeCQVJmJ0M/Lnz9OcJ1dS7hC7HA9lDlCx2qAmQfsk6AgCNfwty3aOzomA9Sun7NN/Uf9lSr0s60VHpMtOFXdVUXUIIn+FAPloPn5W9faNQDSAYPd59beO/pSOJHpCtuBSVRU3aIIlMZUa+MGLewbDhS0kJj2Qd+djU1m00Y8lUqjeqk+qPKUACSIsVUoKnvflbl2oUUkcuosVU64hJSlX6fUWssdqqj23QrokRPkBYi4VHz3t07WLSIUJI8N7AdoHBIBHWetrrDdUpBYo1dbmS9KRcah5G3mLSGKg8zaNKNeSupiFbPJOnnZUxNibGF2XimluDBjl5sqQFesB9KU5uYiZHvtlYi1zpGhvRaNDlEZJ6p31NR34nlbhl+VgZjaL7yxjxgCwKbhk2QU7qrET8M02BoMWcaXatpL/D1inaPtrcXlyn3qja2MaSYuzC3qtATRhtv7KW1vKSgjMdrJd5N1lRRGoaaUtDLralFMaALHKyRdF642fPY33h2iaaXr27dUJQOgSCcx+Fur8eWLvKEOEfbM6BW/wBom1acljsrlZgYXatR1L/aNgg9fvscuOhadSkqCSV5j46k2ZmIr3vq4LqevS7bpfvt5TjbbdE0Uo7KTClExmO45H3CbR1ScZuOd3VlWbo4G195sFYFPnrm2UpjQgZgCAT1J/C2fVtdkDgRa/UBGlBylrd1M9JhiidoqlKkJOSvqIPT7ZR/G1R6UJxyi+LwvwtJdTeFRQLQ0S2laW1pUhQ0VBnnaSLw4t+kPTUNS9h7hNSvu1L63KhirvFseqOk95sd9OYDQyOtmpwSwTjaiu2+Knifhpq7a9293Kmnp2KhDiFNuJSVElCl/wAQiCfjY+AU0T3yx3HEdbLqDte/4WHiz62FjXsJyjf+ybV716cM3XV0VPhxTlDWp7OoccUteQZcpgpSOXnZXwlg5ji5iW7rxxfdXq13hjsU+rvOtrWy2hRCozkgk8yNYtN9Lhi4b/q6S7r4uplyncWESUKGWTyOX6m0l3LgbC911blZd90MNONMt06kjNqEgK16+2belwQUdHTENjHFIPiXGvfiFVKOJMTGOSjvAvDC7LrpK5nBd1tXeisoEJbfqAtQP2hUoKUVHMYTHvsncVrsu5d5XDTVWHmKn1uqWXA62SFpU2yYGWZjoAdhadaFxhhqqSgobQ1XKQEto0CTlgADzsiXqhjtcN1ZS2VLUCFFrMrZA8+VqmGwtpJONKMzzzVqoa+WPhtcQFRu8KamuvFQu+gZaZY7dCuzSmISoAyOca7+NpF4EMUVVcVMa+9Kllmjw/RukMVq2QopcW1Kwg6gBOlov47XXfFNx4qnbuvtFIyxVsDsywnuNKS2VISqJAJBkeI6WlH0eGkm5qpxx1BULoeZzhOwavB1IHun5m1OZpFTdwtcrepzelsw3NlLdAnCa3pXeFWWzquK6pMjp7VsqsJ4Jvmgfu+gfrnELS6G2lV1VmUSgka9pBGmx0+JsiuM1AZTVrvdwoUY+ybI98AiBZy4R9TecDZvd5a+8hbTiT35SQIO552stc29i5voqnDqr3ymyZ/B7DGHqrBV33kq67uVVPNFFQ4872alFDigAdCDGo8Pufirmu6jpgtu7bndCEglAW2VHrr2ZJ5+O1mR6PwuHA3C6pbrqv1G7rqrK5bz9WvRH26lElR5SvTziyhx04iJuLAVNRYRq01V84yWm7rmNMrMVh2Ap5JHIBQg9VJsMRUkcHFkaCbeV1pRR1NVMI2E2JtfkEqXnc12oubtqO66WjWmpTHZJyyhQVI9lPOOXS0EcSmgy9TOSnIEqT2Z3Ptk68hoOn5Ta3hSswDwiu3Dtbez94Vl3UjIqap90uKcd7QZ0pnZAKiEjkkC0E8TH1OUaXjU5yXwUIygIGm3iZUfjbhu0UAjxKNzW5QWjRdLhjv+zezNms46pq1Abo6ZxReTAAcgrByJ13PT39LM+pxzhtKJReSld/8AkV3oHKQJ2NnQZU1Udq23k7spSJ128REffZvvXbdjIIRdtICSYKW0DWZ1kaj68LEpwAPEpSlxHhKkP0Z8aYGvPiarDeIjdtRQ31d7rSG7yZQWVvIKVoA7TQEgLgbmfK1tb3r8CcOblcdTW3Hhu6kK7VRSqnpGCoiMxUcqZgASSNhrbzsxM3R/uCuUzTtpCWs6YQBoFAx87QHiFls11KU0zchG4CR/EfEa6D4626XD8r4i0D7rn63M2UOJRHiS7RXzxExVe9LUest1d919Sl4rDhdS5ULWFlR3BzAyJ331tH2IA5R1ZpojOgGFwAQrMIJ1/lTHlZ+VFGpL3rDaCVAAajSRHyMRPgJsxMWKDeJFBKsy+waGgOgCBpGpI3119nkINtVrcoWc83KQ3hmrUhJAynRKZSDqCZBiOo6wOYAEmcEsM3Vj7jThHCl73QKyjcql1VbS1DYW06wyw46ttwCNFqSBtOuh1i0bOrX2wSXCpuMqe8SARodz3tQABuJMCZixPoG3Eu9uK1/YpcSSxclxinQQggJeqXkhJKjucjLo1AjYxFpDqhkqNOIfB9PAvi1WYXfp3H7hrgKy6aioTnTUUilSUHkpxCvs1bDuhRhKxMnYduvDdattHqt2LdVlQGQhHd8Ak7QD117u0Ws16RXCFzi5w+Xdt104cxFdLiq65CBlUt2BnpwojZ1ICde7nS2SSE2qbgLEVzVN20iL/KaepQEsIqHWygKAMJzEpEK12keybGifcoZGllI1BcFxUziBTXM0C0kEFSYSJA56k6jU7fCy5UNrQ0W6d0hIRIZUSYUNU6HTdI8N4sWo23Ujtm6gLQBIUkhUDKDA3nn120O9jVTUFhptyoqEBBIPeBOYyIAE9AOfwItYIuLJgbFGuHV/jD/EK5q5Cc3a1CaRSQJlD0NkSOYnN45d4tbtaJQUFO+ijprrHvO3hHzpVdwqKK9LvqlNKaFPVtrU84AhKQlwDSTzhI/C10pSrK4FTm1HMRH1tbNtqQrpOxUHvH1dbrZckpKkEHwO23hP42LuqzIUqICh7WojWx+/Gyi+bxaSAmKpzKZ27x26/lZJzAwcxCBokAkgc9umv4W4tzcryF38bs0Yd5JN7TI4QnMpGQjKdY035eO/3WAcCVNpIIABy5QfP6mwzhJeCciSO8IVqeev18drArCt15RrIA2+o8t7GZsgP3TZxhRIXc7LroMNVjS8wAB9qJ84J+t6n+krdzlHxCvRJVm7SnpnEZtCQltICtNDOVQ5bWuDitvPhyuAcEpbzgHXVJB1+Hjas/pYUA/tPQVoZJ9fuhKSQNMyVLI194+XS16gdaoBWVijc0BKrA8haXFhQykEgjpZ0cKn65jiJh/93VRp6l6uap23c5Rl7Q5NxqBCrNmqSUOEqEBRnT68LHsMXiu58R3Ve6NDR1rD6f8AcWFfhbqZBmjI8lycRyyA+avkcB8Q6snt8UU5CY0NQ4SSY3gfUWZuL8PX9c9Q5R3liZCFt0prEkqUO2E+wk81d0j3/GxNO+XEqaKWgEIUUrTOZwGDKuUg6CI5WinjI1R09XdlWu6KetUAsuFaiAlpMZo1EnvE8+fW3ER3z2XeSxNEeYKJ7nuK8r8vdm6Hr3dZcdbVJgqAytpVMTrmC/dHjYhTXPUP3iu7S7XOPs1RpnsrZKYlYSoKE6d0ctJHSzyp72vpgNXwtFDTtVDz9Aw602C4lSW1JTrvoUiNdbI9cnENQzVXjUX0+Qq7kV6G0EpEFYChAI1GuvOfO15oWcWhJDWFMTVdGPVqGrLy+zhLikpGUpJVuNO8I6ajxs4qXhHeDqUqqahxjvupV9olRDYSOzMZdTPtD87A1dEq4MS0dQ1XVDgbbo3UrWuc7anChcj3p15R7zNYbaW2IJgpmeu33a/GwZXFmyNDC2S91Cj/AAHZvWgTS3teS0rWpZcSyCAQYygTz05zyEaWqdfVO7T1CW3QQpsFozO6FFJ38vlb0cW20qcrpnmSnbnprpr91qFcX7pTc+Ob+ogILV61KgIjuLVnRp5K+uerg0znFzCsnGqdsbGuamcVlAp3UqBKR8CFE/ja91y3gm9LloL2GiKykaqCJ5KQlX4/1tQ/NNMkfyrPPqB+Vrn8IKxV5cNMP1K8qoouwOo/2alNj5JHxsfGGjhtd5oGCu8bmeS9DfRAvb94cIvUSpM3XelTTAA7JVld+9xXwtNizm1J6aC1YvQkvQKu/FdylWjb9LVgbRnStKjr/kTazZOoAn3b2oQG7Ar1QMrytEzpl+Vucxnn4k2w842+RtpUA6g7xa0Aqy2tRUjIdNLNytbLTziQefvNnCCOfLl0skXw2EL7QAz4Wt07rFVagXCbyDkqahE/xBfuIj/pNhe9src62De7l4FRgZ2jpt7J/wDzrCwQTvpOvW2iHLOy3WwVEHlB87YSCdNuttgADMIjz2tzIgGBtsTad1Aiy2CeRjwt1JKpBO+45W51gb9NLbEAkDlpvNpApIT+HnET9fXjY9duj0iRBkEediAgDNqNNTY5dqwXhJ36WnmTWS7mV/Mfj+tst3I5pPwtlnumsF4lcM3GnOI1AEADLSvwQNRKIIPXb63tM1aoq3A0VEjbNy/HS0BYbvNygxbTViS6FBtQBb3BJM7bDTUe60x/vt112nKlJyrTkUlZglXKD8eth0rxsh1DDZOfDlQxT1K3luQ2hKlFJ1JJgQP6bWcn9pLuTKU1QIkQAkjl9R5WZLVSHEKKHZSlI7wMneeX18LJTeMcNVN7G4W79o11qF5Q12kd4bpBmJB5WsSRNebkoUUjoxYKSP7RUQ0TUKhMykJJnpy+pFsOJbuQmEuvTvAT9x56j7rNBSiVZwuQDIG8Drpy1+psVfuulcS46C/nkkZHVAnw3sMwNGimKh19Urf6TLsDjyGk1LnZqUpUDKpIHICZMnN8+thrqxfdbLKnXlLQH5W2Csax3dNfC0SVdO/dle8s9tSvqqELJUSpKUKVBJB3EqEgHn8ClbVBq8Cp99aA3TqWWUA5RrmciNSIEg6e0Olq/CaDqrHFcVI9+cSsP1lSm53rurCqsR2aXM6YCjtPT2YHuHMWhXFN+Xo/iSouxlgU7iGFtLJIlYAGWY9rSfOT42K1d7NPYhReCnUClbzLKe1OZOwAk6pMgR4WPY0Zum/slZdVak1rTRcqsylZy2ACAnrHlyEkRYZYBqFIPJ0KalKw/QVNPedA0X00rAWVIQYCh3TMbDfUyBMyOfpdgqto3ME3MglfaKuumME6BRaSTz21+63nx6P9TUNcW7ho3nHTS1NS5TOtrkocQtKklJTEFJEA78t+d9cHMCmw1dTCNENUjKE/5UoEcvK3KdoJvdjXYdmodHSfZU59KJeXj8l/vKCa27svP/Zo938Q93vI9h6Mj1ZvqEj7rePHpPKqf9OzqUIzJS5dyiJ0ACEanl0+WtvYOhUPVWyI1QmBPha7F/po/osqoFqqT6owqCNI5TYBzQSdLDKJywJIsAvYySDy0sZirP3RV6NdhpYNLiCkSoA+zbp8xuY91iClALmSddLXojZUZBdDVThQyXBopIkR5WkGlrqenp6aodcCkuInmSqEyYtGN6OuGieS1mJKFZQnUzl5Dn5WUrqvCleoqO7b7rn6KspgShtxXZ9qlQIB130V7iLVa7KSLq9h4JBCetXilpK2k0jK051tpKS3lOVcwofCxV/FS+yWqldOcQpBUBCkhQB5/fZLNy3Q8ptZvioKkJaSgodEjs5g6DnOtuFque68110lPX1y3+8GUyoIBMzOkCeptS4cLtbG6vWc3pZLS67tLyqKdtKSkNJW5GgCyTG3MiyXfAJu1eckQttQI1iFgiyw1TU1KghDYT2hK3CTJKj1Nku/XqRu6aoh5uQmYKwCSCNBbnKt5mkuwaBalPZgsUE046WF5idHCNbaoHHHkqbaOy1b+dq6ekTjnGuEsROUVwYnq6KmdpmnOzZUEhMyM207gfU2jfhBxVx5U49aui9Ma3pUtLu+ocUlT6sodbQVBR2nbn5WjNgUtfC14cAE02IRwEtI1VnHa5aai+kZjCa9fLaW0H8bHw6Vsu6eyQR8bVh4c4mxFet63O/XX268X62pRUGqrFJ7Q5SBofbIED4WmzGWP2MAppPWLmrL3F6LUy2i7yFKS4mIB/zToBqSDarhOGvwzHImucDcHXbkoYlUNq8LeWN2t+4Up4VuBy96lhLzeRtSlKLhQdcsEhO31FpPpafsHKxEDVaDMa6oQPwtB/CXi5UXziClwreWCr0uSpZafqmU1jgzLbKADAA01AtJV74urrvvRh6iuVb7DzR7ZfaQltSVN5QdDuCr/hNvR3V0LpMmYE/j1XDilkYzOWlLlI6n1uqoUkh5yoDwCUCco7MHfnvZJdK313Ow2taV9g6EZkwEqCiAfA6WaFXxBrk3tX3bfLN33feqb1Fz0zqqpa2lLfp+3pyVISImEJMDcmDZKxNj/GN38PcI1tNclCbzevZFLUuJCl07QK1pJEkK9tI3smVcLn5A7UJjBIG5rKBOPl01DfFaqfBX2vZ06lkIkAgQSSdNYs4PRtdSfXaRxorKWb5bhaZHdvAqAjn7f9LJHFy9q+9sW3m5ejjIrWH105DTZSlQSs8iTr7/ANUbhhxBc4fCuvVq6m62L2vOlUyp3IPtC05MgHr03NsuWq48pudAVt08PBiFt1Z2vo6ZVO0aamoygCQU06QT/hM2N4ZaYbvNpZYaSudFBpIAPmNffaD1elqmlZWUYApiWXJ71URGXxyjfX4WLNembWF5akYBuxsurCmwqrUVDXfQa/Kyjpmh4kz7eSM6pcWFmXdSNw1obnvvCt/YexBdLV40D173jTPsKSFpWkOoIMcuoO4InQxZL4f8ArpwVxBRiqnvq8Lxuq7qdxq4rorSV/u1bipcKVkmRE5dAe8SSTraO8I8Yb0uKhbeorroS3f2IKhxxDgWexDic3dII00G/nZ0V3HrFN1ur9Ru67dSUgFtShrof4rAOO4bGQyYEub5KzHS4g1juA6zXDVTJj+rTWYPvNtLTKw2xmUW3kqUPtG9RHTXTwtV3G7VOKGDq32yMhUkFOoCfjP1vZ6o40Ymv27Ki7ayhutLVUytl0tMZVpCo2M7iB77R3j+oUigW8SsuEIKSkQE9/Ukxofuje2LjNdDilSyWC+gtqtHDaWWjp3Mk6psNqSSs5gc7KTqTqdDttH5/ErUJEE6CTII57zy3tujcZbqHRmbh5kqSQqTy5H38traeCQe8ZSUn2j4fp7rTjZZFDrjVIl9tKfuWuQoEDsHASE6zB6RHL4jxtAN7sk1lKpSYys65e8SokztqPx08IsVWILjDjOY6oIgqI7xEDz/AFtXu+kLTWMJUVo7iifDU7CR0jWN/cN3DOYWLiVrghFF0oUkNpSFBwkJEAzO2/h8hvrrGGOlrRi11lBStammSsAaEwdgdBvpHOPG0uZUrpgrPokkgJRG52mNdvkZtEeMW3TjKqWQ4AlDSQQDlCUpA5aJ2I35DlvsHTdZCbF6OJbZP2hZ7pAA1JG5AkxzOg8N51uv+z1uYMYAxbfq2Qh68L3p2ElUSW2WVZST/meVptNqZ1rZbU2pohpKt0hJJIG5VlkgQBp7zta/foe0hu3hLd7CgEKqGXH3FzqpZq3N9NwhTY66x0tAu0skWG11YBhYQ82SCiXEmDqNDr4zytVP0m+GrPDnisMT3ZSsvXPj0uVZp1mEtV6dahAIB9sqDyYTut0RATFplvB0LUTMmY3iB7Xuny36WaHpe4VdxNwWcvmmRnfwxVs3qlJQTDAOR/XfRCswjmkWJEOqC7dVZuE07bk3De9RdFSVpK6Z9P2RJg+yTkM/4eR5aS70VN5U7S3qxhLiwgFdTRZS6jaTlV5qO6p00A0sz7rW3e1GyatpLanUZgsIzNqVoIJ5HefETrsFemdvhl9NMilp3A0rKAvPlCZEEb6RBHPWPK6L2TaIKofoK+p9bpnK6pLRCg6XC4G1CAUuNASkTuop5yJMWurhu/rvxLcNFfV2OqVTPsjeApKhoUqHUEHTbmJFqlm82GElLFBSmqpEBZiRl05KOux5eNpn9HS+xel1X62spZeZrEKXThITlC0e2RuMxBH+5ram9mQ36q0w5m/Raxi0lnE94IU4QVrCogfxJBnw36c7IOYI0SsnUCCJ8tfrcdDZ0Y/T2eJVrGYdsy25A5wCPPl4ffZrrhJgKJ2ACTuDH9f6W4upGWdw813lG7NTMPkEnPRnJUnTNOVKtpMeHM/OwtfeNZe9WurvB4O1BSEKWlpLcwMo0SANhyHjYNxSFLKwQZTmnp9R7rBlCgtYUpMpJ1B0mdND+Vostuk7dEr2R211VzOUqK6dY3jXKYgfC1efSipE1eHcF3vlzJVTO05JBB0Q2R8cp67eOtkHEJdZW1MhcpPkRt06f0tAfHen9Y4MXHWnRVJeRZzAHMBDqY98C1qlOWZpVGubmgcFTKqSe0IPSBGs2DGmUjQ2N3o2WqtwHkpWk8pNiidd5gba87dkDcLiSLOXqZgWrRfGEbjvfKkmvu2nqNua2kn7z9a2bPEu72Ha253axoLaNQ4w4CO7C21AyI/w216O16m9uDGEKtRzKboU02sa9ktTfXX2fnZc4jXLXXtcLoutoO1lMtNQwhX8aknUE8pBNuGk/TnI6FehsPFp2kdAoJvairX6G9WqRbBu2mvhDlIECVAuGTqNCDnTGvw3tl1sCsuq6sySBVXVX0Cu9MlJKk6HWeeutnJR4CxtT0i7uacpBT1LbDruZZzJdayxAAOgPjysuU2AHmHmClxrsqavfqEADdl5JGSeRBPwB95zK0c1REDyb2UUXjdrzV23BeCax56aNbyw4rRYQtKy3PQSYHhadqZTb9Oy8mSy4hKwI5ESNNtrNC7+EjlIQivvp99torS22pEJQlaVJUnUnfMD7hZ63ZQKum7Ka7+0W6KZoNJW5BJCREnxgTpYUjw61kaCJzL3QBGubNmOYRJMe/Weg62pv6TV1+pcQ7xcQCG6xmlqxrz7Psjp5tm1zykOKJbGh1EmTzm1WvS4uxTWIrqvHISmsux1kDmC05m19zlr2Evy1FuoVHGGZqa/RVyT/drT4hX4fja23o4VvrXC1hs6+p1j7A00EqC//eH5dLVJb2WkH2k/jP4Wsv6KVcHsNX5dsyaatbfAnX7Rsgx/7O2zizb0xPQhYWDutUgdQVez0NLwNJxEva7VKlFbdSymNAVNuoI89FKNrhkCJBBnUfUWoj6Nt5fuvjVcUqKUVZfpVyrQ5mVgefeCffa9qtR1nmLYtG7MxbNa2z7rkkbTr4W1I/mHnbCevKbaKp1BB8elr4OqolYVayOXU2J3ijOxm0kGZsaOg0V+dg3EhbZTrtGtjxmzkKQZhZM68YRVUbs6Z1tzPVJP3pFhFKMT1nn8rc4hBapu1CR9g82uToAAoT8ptyXCdFKI1tpN1WY4WQwJ1BPWdbcZxOaee2/ysGFzuoaeFsSoxIjQT7rFQzqhCr/FAI+hboKGmo1sAVA6lQgn5W2les8wNT7rSCijCVCQTpr52ULuMvCSNfhFkpKjoYBGkb2PXYoh4lOkmzpJw9465FGedst0kqCQAQABoNfztlldNZeFt2uLcv5pTK4IbKjpmA7x5/Xy0khd4ssNttpWQ9HfVvsDoDytGVHUvM1ofadSCtoIiSTJUoZSI015b9edlOmeNRWINZV5G0rH2Sf4txlHnPS1OGTK+yPK3My6k2jr6JbdZW1dc3SMss9h2qnIT7OpknWPM2gfDt3qrMa0d00zrK4rUI7VAELSkznBAJ1ACtfnZ2OYgvHEeMF3XcDKKmmo6MIcpnmwWgSoZu6d+WsEjXTezfxdTVt3Xi5idvLRVhWCWko7MNKCP4QOQI5TuN7FkqA6QN6ITIMrMysxIQSpzTNuCPzg7ee1kqura3M9QUTjQeKc6Co65dR8otFGCOMVbfj7d24jdS28mSXUJAD7eypT/MBrp/LtZyXziWhorwarrqWqpWzqsZSQtOkjSZ1+467i1wzNIuqYiN9UnY14gLTVfutSG+1bb7+QEnLl5yJBnTTl1myNc+J6Jy7mBVQHIepS4rQhtfsE7SAREnXWzRxFUqvW+qq9A2XjUAx2h76UnyjbbrqLJ2d0NBzsysICSlREKzcwTtMz7o2tTdISbq01uiXroQxUVgNQouq7ZGUwDnSdNZ6aGenTeyxildwUNW0y48ilqEoUlKgmSdu8Fb7TodIJHMCzc7WjprnZrWQj1kLLayFGEEiN9xpr9GyFez66iriofZdVKUpWlRJAzTEDwjaRaObRSy6p8cEK5DPGHCyG4UF3glskCI7x2Hjrpp0629C8PN+rXbSU8AFDSW+6Omn15284eFFS5d/FDDAcKghV6UuZIVIyZ09OQIn3jxt6R3emClClAlKtYG5+63H9ov6jfou47MaxP+qpb6Uqn18a6ktsqWhpy7yswDlBSjWOWsDx8dLev1BPqTJ/9Wn36C3kd6TAW5xiv1ZnI0i7TIOhkIAnp3iPhy1t643WqbvpjG7SP/CLaUJ/7aP6LFqv9VJ9UOqSNyD0+NgFkDfWw6+u3vsA5CTMmJibHYqkm6KPE5SUzZOeJCjGkazPKyg+dCT9aWTniUqOb3C11ipO3QTrhQ2FlQ0Oxs8WWrvvIJRX0zTyBqlLiAoA2YV6LyUDqkaZUEaEb6/Gzeuzite1Cy1TrYpXltISC8oGVmBJIBAE+Atk4pFM97HRcls4U6MMeHqX6m57ru9SF0FC0yXQSrIInpZUuWpaep1oSrvIWUK5QbQ5WcWq2uhJp6VAAgKAOpmeulgUcWL8pyX6ZFKjMgIgNzoCYOusyTapNFPJGBzV8GMNsFNd7UBrLqrKbOUl5hxEjlKSJtVo4Pbu91utVedQsMLaecBEJCkKBI15aa2fo4uYweQWl1NN39IQ0n62szcTXhU1V2V3eUkOU7mZIG/dPTzsfDWvpY3sfzQJWZyHHku+MfCrEHFPFLd/4QraStusUYpHlIcQoB1ClHQ5t+8LRlhDgBi1jHlW3TPH1lmiqaVZW0W2/tG8hKVmUqIzgwPG0o4dxFeN00C7suyscpqUqkNNGE6gAkH3WUrprKt6822zVOlS1GVZzJlPz1tWjNVAzhtItyVl8EM13OCiGj4PYhw9fFLSP3i0XbgvIvVAiQtKwlcJHx+tLSPj55arvuZwCezvajVtzKo29/zshKvK9r6vm9GRTOl9KkpcyZipRSVJ133AHw0ssXtSX7eNzXfdzGH7yW+iop1rcLUJCUlKlGTB2PTpbncbZK6Rk0h1Gmi08P4LGOYxSBSqZp/SDuRmjBLdRddQgyddW0HSd9bSiH01dPWoomchd7GEDmQp1JP/ACWjVWH78XxlwriJikKqGmpnGHl9ohORSmgdZI0EGfMdbPbD96rp3GEKbLiKpx5sOZkgI7N11UHXmFC0KGbJAL9VmV8YfJp0UY49uTE7l7GtorsqKhTeKMN141T/AHaGcjitSPZjz87P3EN1XnW4Oorv9RUmopL6ZqchWB3PWnVTIO2UTYHHF93mwm7627rsadp6h9g1LinsuTIe7yPU2PVWOLpfVWpq6htAWhtKcn2moU500mFTaD8V4TiCfomiw5zg1zRf+BVx4533Q4fxpfDroLiKmudWgtgEkZp357je0S1eJXLpoH0N0CqhFdf6yAV5ezC6NhyTod491nfx8bqryvtxNyrccZbeKiVgCRA3H1vaPL5o36i5+wFOVuN3tQqUBEpzUWU6+bZtvYdM6ogEjhuq1VEIZDGOSb2JMYV1NT1BoWksuFKnEhcKElI2MdR05+GonBzEFXjd+8aS9lIS5QXS7XNOtpCe0eSpMSD/AAwdh4WCvnC9XeDislFVS02EfZvAJnXT2T9352C9HfCl8IxGhbjnqtPX0FQyVDvLKCYIhQiZSRPX4W0q0uZRPew2ICp0wDqprXbEqer6u6kuzBWE6ykzD1i82ah5RUT9opMEidBbeI+6pS0uyQZjYgdfhY7fl3preGN3sqcdSbuelDiCJCmlgDl+H6tzEN21KlLcXfdcsETBWmD8BbzaneZnEvOtyuykZw22aEo3BWOKd7k6J0kTOnz5fP3gY1vJ1y73W3E50ZQQG0gKQoTr1O/yFkXD93L9YQpdVXENnMiX1CDOugjoPrSx7GNSF0aUqUo5G1gBP8QJGubwk26CkiBKz55Dw7JisXvVqvMMGtdKEFSUjMkQlIICDpqJgzpYd696p7MguZTBk+6IB843/OxBwA3yQpaQkIJT706g/P424chuUjdMqlQ0jntt/X39XHC0hYgmdshHLxqYUpVQEwJJSqCTp7+XjytFWNKb1atafZbgLp0wmdSozJ2+fv01tJRgKPfMHRI3k6D6/rEf8QW15aEpJgJUkhM5dFQdSANo6bnxtfpmhrrBVKhxc3VJFMEIpmm21LkAAaxBPOY036xaK8XpbcxZWGVZVJbhSVFMSAd4EwI1E6AgRNpbaC2mmUqHdWghJExsNTtJ1+YtE+MwhGKKhIyolCJWI0ievgTvpqCTBtek2VJu6Qn2XO2Cex1EKgxMhOwlJkRHlJ9qNL9ei204zw+u9haCjJSONgT/ACs0qyI5d5S9DO3utQ9m6a687xZu2ha7SqqXfV2G0albqyUoGpGpKkADf8PSXhthFOEq2tw22nOi7r2q6EGCkqSGqhE/BlPht4Wz5KhrJWxk6lW2RF8bnDknu3lKMxKVSnbnO2u/L8h4vJFFSX7hp26a5rNT19Kad4RuFIjU78z8BrZnobAIyNJI1SZERp/QbeXQvK5HAq7KcK5JKSJ5Ax/UxbUh1WZICF58XfddZgvEd+YOqmlF26K0sFDumZqe4oAaGUxr5+NlleJ7rZayVzTiUBQlcCEyOfIa8vxOsm+lTge7bvxvdWMX2Cilv9tV3VjidCl5JKm1GOpJET+NoXqLrFCtNLfa1qQAEtPgZUkRPeG0iB8NbHFwmFk6ruqKGtYUKJxCg4ggrI74zHfSSdRy8LLvo6X5U3TxONzuvlabxYfoHpOuZCS6hRnUx2agPPxtFfqNTh50Xjd7yXqZRUVBObKRm/VXxEQLPHhverN3cWcP3wiE09bUtZikwJWOz+ROvny516gE2JVmAgXarFcT0lm86N5wlPa06kAdcqv/AM6zKeKs2ZRggRBO0HlI8BrtaQ+KDJW1dlQQDlU6gnXnlP4ffaPygwcq16HloCY+Ox+QtxuI+Gpcu3wo56RqKPaHRJzBWsA68p21On324CJ1UFRyjUkmNLGHUACCqVbkHefjr5/1tgbCSE95JMAg/Dn4H6Nq7XK4Qi6EQUiI1yzzPhHx+fvhvizSJc4JYkpkpK/3fexUkDU/+cJB+Sj9TaaQjvJ7qQSZM8vrX62jfG91GrwTxIuzRSktmqhWsS0F/enz0saJ9pAfNVqhmaMjyKoFiFtSK1xSgqVHMcw28PrwslAHefdZbxI1DyFlSdUaRzg/rZEgpIJ687dvEbsC4CQWcVfn0Nr0/eHBZimMKN3XlU04E/5Xf/ebeVp3q0BTi0lByqhRPnOuvl9C1YPQJvAv4YxXc+4pa6mqYB1HatqT/wC6tad5pQUkgBQKEkwNenwi3GYlaOqePP8Add9hX6lIw+SbrlMhJWooJSOZB1Ov19RYq6CO8Rv7RA2ssPMdmrKpKjBM6AHQn7+nn7yzjKkJyrJGx16dfL8LVA7RXS3qk1TRSSQYiBOUnx/GwLrZKQRqUiFJn3Rt1/Cx5bSoKAICTGUHw/LW3DrZUFDKESQAcsmf6n+vJZlAtSZ6uS4A5Ble400PMfXwtAfpa3IFYcuS8YUr1avcppSkzDzRny/ugPo2sT2aQoK1HMHcHmD4Wi30m7oVePC28qwA9pd9TTVaIO0OBJP/AArVa1QyhtSw+ao18OelePJUKYJQ6I3UCn4iOdp19EqtCcR35dhVHb0Tb4SZhRbXH3OG0G1QDNc6EiAh0keAnS0qejHW+pcWaej/APTaaop+uyc//Rbrq4ZqZ/0XF4e7JVM+qupgG8U3JjvDt7Z4FLedOtROpyh1M+Wk/fb0YXvChzNvNLsnEEFsAKBlPWdNQenj+tvR65q9N63LQXmmCmspWXxB3zpCp+duZoHXuF1GINtYo3JAIBI8baOUGSZg21oNTz5Dba2lanTxnztqBZRWiTEJi2oAgSdd7Yo6ae6bcEgezzHWxGqJSBiSnz0z6EjVbagPONLJLbwdbQ6lI7wCvlNnJejeZrMNdLNC7yPU22809kS2PJJI/C2lAbtWbM2zkcSsRqQR4W2VJTy36CwQWJ31md7aU4RBkE8tbWEA6IUuGdBrPPY24SogjLsI0sGXB/Fy6flbErJOunIeHnaQUCjGYSJ08bKF2Ll0EyD5WSg5qZ8rH7tX9qFE6zBnpFnSTlzeHzNssW7cnULV8Ytln0SXhtToPbdo0ohfZgRP8MqmPl7/ACsl3lWu094KDLigVEhIGggbxJ5DQfGbHlBDVQU5kqcKUicxJ/i0A8Bz+YmyHenapqyW2ySrUInTUbn63nebZLTd6su0YnJwCcbViy8nFKzFdJI7uYzmHx3g+fnZxcfblo7vu677yacf7esqXAtKl5kju6x02Gg+divo7Ubjd/X04R9iKZCSojmpUxrry+6yl6StTFDcCASmXXlKABmQE6jlztQe93tENG3/AAtBjWmhLioSoVLaq2HE9popBUYgDw8f00s9VXkWVpbWVpSQFpAOUK0Gh5kD8POzFpXXHXAylYUCCkyJA5j693msNorKgpyrygnUExJ8Ou/LXX3HZKyLXR+sfL6hUR33BqUkpTI2GvmfrWxVWoBb7yIzJkwBAPL4a/1sI25Rs0q2nHnO2S3lQotiJJ0Opk7CxFSlpUpS5kJCYBmTED89OQHuhdTDbIwe0RdrhS4stqdBykQNtY8do6jSxZtinS0CzVla0lIHcMHUGROxBnwiLH2altFw3i0pJDgdZyJHQEgxZz8EsM3NizEVbTX3SLep26PtUtpcUgZgtI1ymdj1IO9k52VtypRszuDQieCKRf8ApGwwUZVBN6UygpLk/wC0TOu4OpMHfXwt6W0yCH8gP8RBA5T4/X4WqtdfDnCNx1Ld4XRc/ZPMLS62ovLUc41ESo7ETaSKbHOKFVBm9VkKXJUEpGvPlHn525vFqV1c5pYbWXWYNUNoGua8XuoQ9KdxLPF++EFZyuNXaQiIChCQTOnWfdy5+uFzqP7tpTzLKJ5/wi3jF6Qt511dxMTU1tV6w4pdKlRJ1UAgQPPp529mLkcz3XRqECadskDxQLXWxmOBjTyCx5ZBLUSPHMo+rWRpr8rAr7o8Ok2FnSAJ8LAOGRqN9NbFjQZEUqFGCOY0+vjZLqVD8IspVE5SdxEWSqhQzd4kRpubXY9AqTt0n3oo+pPpAPsH7rQ+h0lITPLMNeVpRxHXpoqBS1ICpWlEGBuQLRNKgfsxGhO3142r1R1Cv4fsUdcccQcxMyIkjcHmPrewjLzimgAsnQRB0J5D7/hYFplxSA6dErEAzpp0svO3Cmjuakvc3jd7oqVFHqzTpXUNQN1pIgDfYncWqFwG61W2SahzKiCvNzBzbfptYO9Q47dtZlMDsHDrJjuWPCmQSomdR0Bjp+Nua1DblBVtoVlHYrBPuPT+thlymRoi1yqMLjVICVEk66izkuZIN7U6kdwJUXCCZ0G4n42bFwLbDUZs32bevIyD8LLDLrjJS+0ohSZgzqDtYLnIrBcI6w83QY4vp4QhJbaVPQZ3Nfus6arENHT3ezX07qFMpppWlImDlSPwFoxvVxSr6cedJUpymRmV4gkGfj99lJKj/ZJ4TCvVleMQJi3M4/8A0g4dVdw+IGQgqSq/Hl20artRU0j77hUpEpUEpyuMqSQZ15C0ccPsc4gdua86FVcrs6a9qppogSoJBB3InmfrSwmIFqULodTAipZKj7lD8vhZr8PVZP7RU6yqU3y8QCd5Sg/jbnWyvNM4XV10DGSDRPl2vqak/wCsvrUJmFK9+g5W7p3YYdIQQMyDodjrZNaWVAhASTJGxgHSxqnUBS1Ikx3DBGg70e/e2TLdw1WhDYHRRlxGUkXy6twEQpsoG4Pd+vnZl3kHm1VhfbLZ9aup5zTUSXUSnl0+PlLo4mlRvolIASWk7nQQd/62ZF9EG4Kp6A4ct3LKkqGsVKwN+ma3pmDEChYuPxJpNU8pRqmb+XeVQi5KtppbnZlYcUmDmCwD0Jifo2S+Da6qjxLQU1Y4e0acq2AAQYTnXtHKR87aIq26xaxTqWCykqlGmUKMHpsT+Z5FeF9YXMZMrS1l/wC0H2zrIzaz7vztp1MjX0kjPI/ss6KJzahjz1CnKq+0wFXtR3W6ipBG8aqj7jv0s27/AINMlSUpgpnMTvpsdPLpz87L6HO1wnfzAUD2dY7KSIiUn8x8fKzUvSpSu72FwlwqaQRJ2gDX4fC3m1K20h+q7Oc+FEbrdWiobT2mx25dDYXFwzXUah0pAMJUBoSJ2Hwsm0a0+uSkZioDbYGeYO4so40dyYfnLmUmSMoiBy8utupoG+KxWLMTkJTDcP8A25ToQhQJQ0mI2OQb/rbhxwZkyAqQBMxtrz3ttlxJxBdjgXJeYAhQkEAKTy/yi23EJQokmDPMyd+cW6dgsAsflqgCnXUgFWpJJjfrz3/CzSx6wHbqpQgFSjVlJA0kEKOvTX7x1s7VIVCdhAnVJn68bI2JKJFVRphCcqKyVDf2kxv4kp+FrcRs4IEgBCZ1e83S19LTtrjK2FEb6kdeoET5+M2iDHhdbxG8nLoUoVmgzlEpGgMjY+EdNZka/albmK1IAStLRyyQYAAG3vsweJNODfs/xFhCglOYKhRXrmAjkZAP6XpPdVNu6kf0PLsw1ePpD4OVid1pNBRmqvL/AFhxLaEVFOw48xmnUQ602YmDAE6xa/l2ss/2pvp5t9p5k4gbWlbawUkO5Z1iP/nJET5m3nH6P9MmuxZRvnuOUdHULCVGMwy5BvvPbTIHIWu1gXFfbqvChXlQqnqKKqSUp9kfYI2O393qPo85W0TjVNqw7YWt91r0k7QwxEbqTbwp+wyrbT3wqVFSiRPhOx3+t1jDzyl3cO8cyVLCiQBrM7ddRv42L37SuGhU4gpTACp9/LXx8PlYLDDx7BxsqIUl5Wk9QNduv1pNt6hkzBZlfHwym96QeEjjDhdezLDWaru1k19MOZLfeIka6p5c4i1U7sq3LzuxqqCkuO5EoWh0DvEAjyG3LbytetS21NkKKS2qJS4AQQRJBH1obUkxPhRzh7ja+MLk5EUb3a0bhRJXTOd9oH+Y5YB/xJMaW0gs4bJBXedBTuOU9XdymAFlLiuzKg31GmoH5xZHqaG8KWtpKy45cu9tZqWXkLBLK5BiCZyyJjw2s6y/SV+ZvtGfWhqhsHNJG8E7gz9aWal54aVRPKqWS5ShRAV2JhPgY8RpEWhKzOEWN2Q3Kt7f15t4jwLc1+tKT/rIaeBCtDmQZ+JFmaGFJSUhtWih4ieZ6cvv6W64VXw3fXB5u7lJKn7oqUsuyqIGeQRHgo/WljrrCCdjlgSkjUc/ONNrcLjX6dT9l32A+Ol06pPcazJyJGgjUmJ8umszt+NuAyUnWZMmeUjTfx91lJVIe/CSZOoA85+j08LYmmIJbVCcwJEk7z05fpbObItd0d0nFlQMhsAgbETzP17turQr7s9bvrFVzutApvG5wQAdT3FNny5fpZ+dkTIypzg6nWZ5e/WzbrGjT8QqMkAJrbrdaOb/AALBgctlT+liCSxugyRaLzRxQyWi0SFqWkrQSUwCfqfP42biSM2qrP7ijdq7sv29bvISBTXk+wU8wErUkbT05mzAJ/pbv6V2aIFebVTckpCtb6Ad4BOLMVXRrFTd7FT1gtu5Zj/62121s52wQAZBHKN9flbyg4f8QL44eV9Re1xXneNDVvs9gXaKoDKi2SCUklKtJSk7crPWp9JfiJUQHcT4nUiDAN+upJk/4Ep+VsXEcIlq6gysOi6DC8bio6YRPGoXozU0y85WUwqdQr66/Wtij7PY/arypB3JGUEzprbzWqOOWNKpZcevK83TO7t8Viv/AHosj1vEi+a8RVUtO9rIL7jz0f8AG4q1duAS83Ky/tLDyYvSupvS46VIU/fNDT6QS7VISB8TptZKq8dYBowUVeNbhYiUkLvJlO2n83X66eddXje713GxTUeH2Wb0SuXqpSGFtqTroGy1I5bqOtks40xBMtv07es9ykZT/wBNit7Pnm5Af2kA2YvRB/i3wqZyqd4hXDmA7wFe2qPckzZjcVuKXC2/sAX/AHBQ40oqmqrbsebp2WULWVu5CUJEJ3mN9Nd7UlGMsUBGRF9VCADPcVl193nYJeKcSL1Vf1frp/5wv87WIsDbG8Pzbfzoq0vaF0jCzLui16ILdYsFKhMKhQ12s8ODFci7+LOGanQJcr2mjJ/7zuHb/NZjvPPVDhdfdW4tW6lqJJ95spXFeK7ov26r3QrvUdQy+NIgoXP4W2pWZoizyWDC/LM1/mF6Mu02ZClJOsRJGgPu+t7Xg4LXsm+OFmHajtSotUYpD1lolvX/AIbU3NEh4Z0rCklIIUn+U6g/Py1taT0aaxT/AA4VRL3orwfbAGwCsrn3rVbiMPk/WLV3mJRfohyleQZVOswbaXA5HT7raKwDPQ8utuToOfibbwNlzxCwkc7cz3tVfrbZidJMTYMkphWliNKgUHUp7RoyIB982YiT2NZXU8Zcr+YCYEKAP3zZ/GeXdmzFvlBpcQvI2D7CFjT+UqBPzFrtOdbKpUt0uuSsT5bRbCsjUfxHTf6+jYuVQoiO9PP47212hKj3TrFr40CziTzRgLkxPObbCpkGR52Lhz2iTIB/O2docsCNefIWYJIyl3vROx5iOdjt3KQFhUgzpFkrPBJ02JMn8bHbvcJcmfHW0kk4gWo1Qj/hNssT7Y81GfP9LZZJLxJbShx1JGSEpRIPNZGumnnJ62bV89yoAUtJlSiDlMefnqZjpZ63E80KlxL1zIqiWkBvNKshCNB3dRvGvSzNvgMP3wrs8jas8DtDlAJM8xtrEx91spnvq04eBFKC+7wupLwu+scbK0pzAKgSkzm035n3CxW+sSXxfxR+9LzfqAyT2aHHVKCQd45chtZT/c4p7tdr3XkunIowkBSMoHL5a/1snYeo2K6vU3VoWUpSe6kgT4a6Ry958rWLNJzWQbuAy3QF3FCXD2iQVESkpgR5+74QdrLKVJUITUNpCEqJka+cxrZMokssXv2N4JBaT2mZJOhACiNt+Xy1sUefcIIWo5hy5DroP05b2kRdII6muLhLSFEpTKYGw3126A+dh/3felQhVSxTOLGhzhJJ0HMgfXU2IZEC71OhgjMYTz1nWOQEiB1087LeD6p9mrSz6yOzfaX3PaPdBIMHy+doW6KXJCUV2VIuS9XKqlXTuoDQQtwHWTqOh2nnZ+ejEioVxbui6C0H2b4cFFUtkqnIVicsHQyBy/KyDVsXtXUblFKEKbbShtZhISU9IE/HWBp4yZ6Il1Vdx8fsJ1FYtotpvBlSggkkAKkjY7wPz52pYo8x0cjgbWB1VzDh/wB1HpfUadVde8uBV2Xa+Gaq7rwoXCgKLTxKCUk6HUAxvZEPBmnrsU4cw3czvqb181RpA68VLShWWQTp4G1sMfuUeMMRm87vzqZDKG0lSCNt9PfaJeJiavA1fhTFtIgKdu29kvpaOgchCiUz4gEc97eN0ONVLsVFO2YvZcgG+hXqMppZ8NzGJrJSOmxXnp6aXDi8+FnHWqwdeNUxWv0hoXg4ylRQtLjKVCJ1G5BB3+/1pw+ofuahIE/6s1zn+EW8p/T54hL4m8fa3Fa7tTd6Q1RsBie1VlQ0BJJAAJ18reqWG1RcN3yFf+as79OzG9vXic0TSvMg1zXuDt+aVnVg6D5dLAqI8QbcrcMxynrYFaySRMWkxM8armpV3CkdJ1HSyPUqIVAEFPysovLTBnTUeBsl1RAzEDW12MKi/dIWIKf1qiIBAhxCpUdNCDaL2EoVGcQCN8vlr9fO0jYnrPV6VALaVBbyEAHkCRr9eFo9pUutOB0t5pGh3kf1tUrCGkXWhhoLmmyO05GVLKQQc4GYnXX6FjzFM+KlFKkoK3FJy/agATy6AfL3CxJgrWorICNgByj32N0TgaWQHgYJ0kc/K1FzxbdarWuulO/7jvbDV4uXZeHZOOJShYUy4HEFKkhSSCPP60NkqobfqmXUobVJbUI8YNjoq6hxtTDDKio7QkmPrWwoum9HmVLZpKrMpBBKWVHceAsDjR8yEcMdbVIuHrveQw0lTZbBZbk+Q/X7rOu5MP0d4vPU9ffjF2IbZK0uOtlYcVIhIA2Pn02Nke4MN32/TU7q6avP2CBm9XVqYH62Muqu273HEVF50bK2yUrS9WNoUDMEEKMgzPlYZka7QFTa1wbrokS9FNt3m2M8EsqAI0kBX6iw7UO4aqG1GT2LonmNCPOyRfuILjor5ZqHb+ursGad1LrjdcysI1QQDlUTJ1IToTBgGCbF/wDSRwvaw5WrqOI9yM1YaeU3SKdzLUrXKCAeZjlztj4xC+WAZATr0VuilZHIc7gE4b+dKrsuh4JWPtKUn/iGtm3ggOKvTE1OJBF55xoRllCfft+W9nVfxQrDVx1ly1CaxippqR9DoQUyhQBB1PL8bM6kvG9rtvm+zdxp1OOVKFLKyAIyR+FsKmpDJC5p/mqvzSDOCnmlt5AJyKKIkzuP6zY5RUL1RSVzil5S2ylwJGv+0SP+q0VY5xRxDwi6husfYQH0B1tLSWlHL0IOu0dbN65OOV+XI5XVmI311tK5SqaDLKmEqzykgwFTysX/AKenkjzssVU9rRRPyO0Tox9QuOXxTuLGqm0gSNTr/SzPxV/2XdV55cpKKKncCjpIRVoP/V8DtZKvrj1dN53k3U1WHqzsEo7PIKltCyudN0kRqevL3iO4ho8Y3BU11NRO0jNRd1ewGy6l1WZtxhU5gACNZ26a26mhopKenayQWssipqop5XFhusxTj5Vx3+miSFLF4MJpYBEQuCN+UT9CyTw3PZX88862EuG/DkSnUpSQkqMzp7XwmzjS3dKHGEFvtMzJV9otIM93WSFfdZIwq7SUWIrzcdbQyo16VIWupSmJbQZkogjx2kRrbS4DeE4W5KjxnGQa6XUv09Uhu7MRMOOIQfWUKTmIT7SBtOh/OzXdbcq7oYTTqbUvsEJKc6R3oEjfl+JstVV94GpmLxpqi9roSy+8pxIqLwRJzNhJgjLPPYWR0cScAXXddLRqxLcNMQgIQtNaSox0lcHQTNuJjw+UPJa0rqHVERaLuHqkekoa5FeCtATPtEyd99uUdJs4cS3S7V3ItlS0EKSOoPz23087N+o4rYHdWFnGVM62ZXmStS0wn2tUjoJ05DlYzenGG4Kijauiiv5au0CUJT2KkhR0PIagzGvjbbpaeZrgcpWbNLAGkZgmo7Shq65UELcQkoS4ESUp7RWk7jy8bBO01S8ciKOoXuYQhR5jw/H4WLXxiapu67luNVtUwh4uaNrUJggnQEfzaT4+Nm9XY0rEUxfCbxqkhJKW0Fbi1wNkJzySSdhbfaHbBZBcy2qdf7pvBaNLnqxlAMinVp4gjxn5b6WLVV2VyKZ9x+kXlYyrmBKOU685HMTr42IXthvjLd1DT3m/wYxE5TV1V6oyS/Rgl3IpUKHbkoASg95WUap/mTmJ0VxcXu3eeq+FrdBQMEKq6moxHdmZluJKg0l4rWe8YSkEkiBJ0tZjhmvchV3TRWsCojrHSvE1SkupPeIKdJAzRvv9bbWbeOkF2/G1KdCCKJnN0iVZTBBG86/fFlanqQb4vV4rlOdScs65dTE6gCBPXfbQFJxasm/EuIbIQKdhBOUd2UEjYHSfLzOlrsujFUjN3Jwej9CceN0RqG6cKp6oKU7IiAT3oB1OXpufOLM4arqejxHWsovFkrdoElBSXMqsgdVEkCCCkHWPZ+NXOGVQaXHdO8w+EF1l8pJCRP2e8mRsTtpoDrraacM3ktvH11tOOhYrqd5gAgjvKSoDWeYUI8uU2qGPO25RhLwnWVvb4xmVti6UUqcgbSguEnMRlTrH4xy+AmEapDi6hSQApCkkmIGoifr7osxTVIeTS1AM9pTMrnMBMoTp57n6NnJgysQF1TYRlJSDMHKTP6nbla/BE2P3VSnqHzHxJ9oeMZzIO0gwBvzPT8BavfpS3Kmnrrgxi1lQHELu2pUE8hLjRJHIZnRr/N8Z29ZSJCF6lWsj3/nFo74+3Z++eGd5paRLlEpFagp1goOoieh+uVpAG6qXe12sVawqmcVS1EaKRPfPP7t+eo0sni/sZYZa/wC2WG75u7ZS0GKhtG0xML2894mLOGhbpqgpadbHaglSeRI6ePnvYW87vfVTeriidfSvvJypzECOfMxMafqXHkpJ/cCcS3bUVN4XZRVIXR3xSds0UmSlxCk6EHYkQAD06zaXBdy1DOSRpmlQM7ePmP1tTHB9deuD8WC8qVxTK6KozOsQoJWhR1PwJ1O3vtKt9en7w8w/V1F2PYIxC7VsSHUqXStJzc4JcUfKRNuMx+iqaqdroG30Xa9nq+mpYHMndbVT2LuIGQJggxO493uHy5WaGO8QvYYqaZDFMFl1srClAAGCNPEwLQdV/tJbtaKv3bwukcu3vkJM+SGFzv1sy8W+n1eOIXm3f9H2HZaSpKS7UPumCRpJaTGw5WyYsFxIm5Zp9QtiXHcOAsJPwVNq+KN6dt2bNG0dAQDMDfwgmwNNi69r5xPc9QxR0xqEIfbbLtSmnRK49pxyEjQc+ZjzrbX+mtf9SkAYCwuSkHLnpnF5SZndQ+jZPc9M7GGdK2cF4RTCgdbsVt0/vBa2MFr3aFoH3VF2O0Pzk/ZNHjk4qoxziVwpQhx283nXG0LzpSpa5gHmNdCNNBaKVgpMEROsRZz44x/W49xBWYivWipqd6tKCtmjbDTKcqQnup1iconXcmzYdWVrJTMTp5W7KkidDC1j9wFxNZK2WZz2bXXEkn9LZJtsDwNukMur9htR8hNrVwFVAJ2QfKJtllWmwtiWuKfUsPXlUZtuypXFz8BZbpeDvFiuT2lHw0xQ8nqi6XyPjltEysG5CmIZHbNKZ5tlpFZ9HfjY8ATw3vlmf/SGex/8ZFlam9FTjfUkZsMUNOOtRflA1HuU9PysJ1XTt3ePUIjaKodsw+iiS2WnOl9Dviq8tCau8cKUebm7fbS4/wDZZj+NlFj0NMVqhVTxIwShEEns36x1WngKeD7jYRxKkG8gRhhlW7/xlV7sICezSeijr8LTze3oj3zRNJ/d2Pbnr3ioBSRSVaEBJHtZuzJJ20y9elku+fRxxDT09IKK9E1jwSUPpRRuIDcE5SFKjMDsIEiNQLMMSpXbPSOHVTN2KYLk9MbBVFhu7aGrwzez1UxSMsvLFQwkKcQhIUr251IJEj7rW89Abj1c/F9vGl03fdT1Cq6FUNRDr4cLgdDqSRAER2aRudxbzYZ9GnGjzq0LrqBhAPcLpczKHIwlJA5c+trZ+gDhtXAPiPfVXizFd1/u2/rp9WVkDoKX0OpU3Eo1MFYjxtkmnw+JxfEfEtdtTiEoEco8K9LiBBBMdPK3BEkFU2baOJODHBrfraY3lpwf9NuzxAwhl1vtpMSPZX/8NiAhQIKXzJGWNPG2lHUkiROulm8riHg3VJxBTyN9FA/MW0OIGDlwRf8AS6kcjr8rFaQoWS+ZOhI8bMzGgTT3vdtSRHbIdYn3JVP/ACmx88RcFqOUYho5iIkzt5WaHEDiTgZd30dQ3iu7ZZrW0yXgkAElB1O0E78rHhdZ4QJm3Yhy6mZzdd9ibaU8MsgQAJ56+P42Z6+JeAp7mMbnVzPZ1aF/cdbAL4r4GbGX9+hYEqPY0zzmkxplSZtqBwssstN09i6rKYIk6aibYHV7/GZtH7vGLCKE50G9HQRmHZ3dUQQPNAEe+xN/jXcwSVMYcxJVka5WqJIJBPLMsfdZZ29UshUmIdklWY66QB+djdC6rPvpO4M2h5fGrKErYwRfaUknV9bDWUzz7552CTxxr0OpbYw1StKkgiovZtKiY6JSo/XnZcRvVNlKnr1hX/dzbLQOfSEqp790UKVcx+8FaH/2Vss/EYnyFecWG73prsrnFVlQ23IahM75Up1kcgNbN7Eibscqn30XYhROZye2V3lFR1A6GR8eU2WLpuJi+u2XUuVCXG+zA7MBUS2CZ8dPmd7GXMNUhfX2y1uBKYcJA1010Gn142zmtANwjuJLbJnUrwVhZ1fZqbydqhISnSYOniYPwnyshXRU1LDjjlMklcphQBOn3/XlaWUYXu9V2Jo2WVntVLXlRmTMgCfl7p8LN7DGH6Zq9HkdittxKCCHASJCtToYO/hpaecBR4ZTFaQ/WV+iFF1ZIG4JVOnOehtst1DAW28nIuSlQOneG8+P66WkGluSiOK1JNK2odqVEqOitAoQM3htvvFiFdRUbVa4h6kZLnalJzJBJUegJ8fy52YypxGmW29V9kA6oRlkTrrPv5nfw+JzD5q/3nSpacASlckjURGun4+VnOlttlKUM01PCQCSGkASNYk6/lB62cFz3u9dylKPqq0ue0SClQMKlOngB578tYcbyRBEDuUSYvphwlo0z5LitSlgke6PADkd7SRwVxjdGC+JOG8X3o1XPUdNUmoeSzSlTmUaEAaA7zG/SyVdlfc9Y6C664lxWhBWU5vgZ5bG1g+B/EC+LsxDdVw4forhpS66hAqnLoZcdKSdu0UCrmee4tkYxUPFK9uW4IN9baLYwqlD52kOsbi31XpFwsuW7cb4RuvGFFUrapb1pkVLLbzOVxKFCQFCdD4WH4o8BGOIdzUd2ovlFCKWo9YU4aftMwCCIAzDmRzsNw2xOGblZVUpr62qKYcdRTHIs9QE90e6zyfxfUdiQ1h6vfCkkEJyII03OdQtw3Z+HAhA2SRtpOW5RMQqMQiq3AO2Pl/deGXppXF/ZnizUUyHy+lYp1BxKN9FAbmeU/Dbe3q5ht1IuG7gND6mx/8Ag028qfTwvBD/ABuqAzSGlSGaXKha0lSAlKhymNj7/O3qVh11P7gutYWNaGn08ezTzt3NEXyUUT37kKvW5WVcgG10vF47lQ0PKwC3SRvy26aWA7aJUVkE2CU+lIJBMza7GFSedLrbqlEEkiB87JtUuZB10iQRYR6qOoB+f5+6xB+qSoKhRB56xra9GqLymvjZ1SKBASVZy+0BB/xg/ODaPF8VqClSpn1/DzawezCTQUxUFD/MfM2fmL1F6gRlT7DzaxHgr5crVSqOG2Nqm9qx5i87oZZdqXFN9rWEHIVHTKlJ8PeLUq+kiqbcVXcOqZIL5AprPG1i7kKzYioGAhPtNUlKiI/yoNkCp9J6qF1m+6CqxjXUJLQRUUdxVaULLpR2QSoU6Ukq7VvKATOYRpZiNcF8Q1zeStxPcDJMSQ68RvB/2ImdrOO8eH/Em9sDVeA7x4yXPRUlaw005UXbcznbgoXmzJUXUgEnOSAAJcVAACMtCLDMPB8S0ZMQrLeEIWo9I3HdS4UU/D3iw8YnM5QuU6cpkSC4tGmh6bWTneNnG69FqbujhHfThADYFfiqjp1KUrTVCnySIIOk72I/6Dr/AHFKcqvSYxMhayVqNDc1PTHVOUmUKBEjf42Nsej5h+sR/wBuceOJ1eUjLJvdTaQOaQnNtoDHha2KLDI/hVc1mIO529Es4Yf9IrEWFf7QXXceC6dph5d3KTU4hdrSXm1BJlTDZb1Xoe/IyqBE5AuvLt53tfF4V15VdV2K7yfNU4yw+0tCFqkqCVQoECYkHUczM2sfdvArhlctyi56XHeN3qRTrtQpkXi0ErdchTi1/ZHOpRSkkmSYBNspeBnBK72g21SXupAGVKXbwCRHMJyISP6Wm2Wji/pMKG9lVN/VcoHo3nkUF5tpfcUvNTOZyUGVd8SMqfjp7usfXyy47er9Q4+lt4p7NYLhB5HknQ6+O3SbXd4f4C9HGmxXX0+MrvdaucXch1PaVdW4VPpdEA9monYmOXnZ+P3B6ANM+updwg1WOjvKKmrwcnTotUbco6+NrcTGyNz3A+qozF7HZSL2Rz0VsE4f4pej7cddf97pp74uxs0NQGHy802lMKZ33IbW3JmCQYkEG0k1Ho18PXX1O0+LLypnnoLq2nWyVmImFJIHlZiYT40eiZwsoHbt4e4Mq7ppKlztHWKG7ktocWB7RzOCTrv4mx2s9MLhAynPR4LvZ/SQVtsJnX/MTbn58BvMXRvAB+q0o8TmyAHN+Fxf3oRYIxRXiqvji1iJSEApaQ2ikSUg6xmLZnWTtZEqf2efCzs0pZ4jYodMqBUamjSQDMQBTGYnn0HvMv8ApuYNbVF3cO6lQCZBXWIT9yDZPe9OELBFBgBhvkC7WqUPkkWsMwt7G5eNb6BV3Tve7NlufOyUL4/Z58Kv3c0qhxZfoq0tgqcU/TkFQj+ENA6gkb2QMbei3dWDr+wnw+wPU19VR37QXww7X1ie1RTvrabKc/ZpASIb209k22z6auI6usRSruC5qJtaoLi0vOhOmg9tOvnG9ka8fSsxpfF+Plhu6kO4eD9dTON0xhZFO4lUhSiDovad7F7iGnMZnH7JMllB90LVL+zvxIxTNPI4o3SF5NUroXhlkaj2zrtrZMT6C18XZTXhdruH8N33WvU7zTd5OZp7ZeYh2VpCgUk7D+YiYSkWGvj0weJC8L0153XjGjF4uPllylTdjAI/xQUkxGszGsaGJZ1T6VfHerckY1daSdYboqdvMfcjwI+tbrIQzd5P2CBd7twPyjCPQh4vmmYo0YCwAk05KkOu1LRQoqHflJplHU6iSY1EmbLTHoX8cHDTn93cLaQMoS3lyv8AdSkJiOzZTGoJIECzIqeP/HSs7Qrx1iASIhtwoAPhlA6GyU9xT4xXiSKjGOJXAUzlNe8fCdDtp87HbwhyJQ8khO4CnK5fQ84yUlyLuivxtg9IXVLqlONUdQ4U5lBQSmcsgQB3sxICQfZTAFD6CD13qZqL14pUqDTlJATdsAkebvMeW9q+V+L8bvq/7QxJeCgBr214Lg6c8xjxsRbxrRUz03ziqgayJkJevRoHNGk5lTGg8dbJ72W9z8qTIjzcrcJ9E7A1WpuivLiQFoQFANs09OSAQJ0UVxsOX5W096IvCSmpyKziDWtrElFQxT07LqCdlBUGFDkY5C1dMN4lbTTIvK78V3C626jOgU19Urjzic0CGkuFyJ11T+dlml4juVbZQKhS+zJHNMwddSQCPLfflbn6yjq5ps8L8jei1qd8LWZZDc/ZTnjPg9w6xRdlNc2IuOuO6iloK1q8aZNFX0dO43UoCsqwpqmzpIzEyFTt0Fo+vTgRwHp2H6g4p4j3vUsIW4x69iuqca7USpMp7oV3tSCCDPjZpnHt3sNntjVJcAGvZiNJ9qFGPnYgriLQh3su1aWTplW8WyN+qDGmpHKLPFSVgILpnFT4lK0WDAql0CEsXrXNF9anUuHPpJUqdgBt77JOJqpFTiB2idVDblM2oK3IITOnTc+8bRBsp36WrmxtX05ZkOuqjMqcxMgCPu8OUmS2ceN1z96M1FG4e19RQ+dcoSQpYieum2sb8tOjlbdtlixkBylT0e37voeJFEbyu+mq30M1QaRVMJdQrMytSjkUCDEaeY2MWsXe98pTWUj7F03XTt09ShRRTU7bJJ1BEhO0aeHTSLU24ZYrql4mu2qpUst1zVQGlB/MpuSFAylIkjX5HQTFp/r7/wAT1jYp2a65UkfaZfVHEBX8MFUEjUg7fwaWqCIk3VgygCym648UhN30zDyuzNPTpQDnUUqygDSE9Bz+Vlq68e0NPVlDy3HEpbOYyNDIMTIJO2w+Vq502M8ZUSECruRFWB3CaN5KsoiR7WTSYnbysoXVjqrZce/fFxX22jIChKaBbpJBB2RmhUHrHnaXElZsVFscL9wrIP8AFK4kZkoS8SBKRkIjTp08fumyVf3Eq77wu6tu8Xc86h5pbZ+02RGnLUaRaETjWpeSr1bBOKXs47uWhQiZ0ABWtMeI5aDTkC7i3EeSWOGuJVAOEZnDRpCvH+/5yOWnSy7xKVMU8I2TPrHKuiUUUDSu1Z+ybCU5u0ImAQOcT+HMWHouIVdSU/7vvu7F0r7gKEPAKUiSJBmSY2jXTxto1lW3eKk19zuXa+9Ck0ry0qVB0IBQSD066crKrDRq6IU1Td7eUjKlK/aIOhkeGkxy99tJhLm6Kg8ZXWUbsLqaXE7zlfVmqdqGg4XE6COURuJHmOe4szMacGb9xziFD2GXKFtfZTUduvsUp2gzBnf9BaR8cfuG4nKOloe0L6AXEyvNkQDvqO7OnhoI3gH8DVT4vlhDbjSfWQCQqQkRIymNtz59elGokdELs3V2BjJSGv2UOU3olY+djtL9uBtRMZfWXFEDXUwjQaWU6X0PMSuEJrMW3a2SrKQ00tyI33jUWt61hWqWjtV1rKQoZpCSoRpB5HXTXXnPiZ/stTlcKeUdZlLStOeknr5fxG2OcTqid/wtgYZS72VTqf0OWQlKqvHbpGpPZXcNvCXPqD0soseiFhgNgOYivl1ZJkpYaRtyAk2tMjDTCE51POkhQUQEpkkxA5/QHSLcJuRCAQpDnfkxMFOwA2jX63tA19UfiUvZ9KNmqrTfonYZaWVGsvd5IPsqLYkf7v52UKT0fsNU/ZB3DNO4GNQVuLOYwPaP8Xlt8bWabuWlChNKoKToSXOe/wCP1rY03ctAkqBaHtaEpKpg6+fh+dl3uodu4pxRU7dgFCV24FwywUzgy4E6ADLc9OpQjlJaM/ebPq6KG5acBpDZpRySw0hoDyCEAnb5+cPj9z0rawOzAGWYCYKj+f1rYdm62ATm7MA84AMeX1vFqspkk3JVmJjGbJvsNXIdDWXs5rpFYuNOWg6fcfeaXdOHHgC/SV7p3lSlL000knXpy99loUVMSFtvZeUjlGkdPrW3WRhAKu1Qe9BzGAB4/XO1R0DirrZmgJDTh3CiYb/ckhQkjsEpgaiNQLGG7luZuOxuhttJgE5ANPLz+7lZQ7ahbQSKxlMmVBOUFXnHh0sGuqusArVeCE8zlTqIB/KfjZhSOdqUu8sCLqo6RlIZao0pTqlMNhURp1Ej68LBOUza4LTB/l9kR8fcLGFXrdyYArFLSToAFd0jTSdP6+8l3b9oUd3I8SgmZIBjkZOoOtjNondEJ1a3qiz91B0LUhpRUnYaajruR+FijtwuqKciQDvGYjofhpt1sOrEqKdXaN0rqwoc3SBttsQPMfGwasXvhGVqna1AgBRMGfCJ6WMKF3IIBrWdUUXhN5z2Xm2wZASSrcz+njbTWAXg+ioTWUh7NQISEqMkeM9bCOYsr4zAhsrICYRMyPnufxsFUYlvFxMh9Y1IEo0O3h4/fY7aF26A+sbZPz+2XElCswvO7VJTpoyohI02Mxy38ffbSuI/ElCEhN53eVgzm7AamBpPLytHSr/vInKu8HSknNCVQPl1/CwDl4vKTPrfaSRCQdxvvP18rXmQOG5VIzMOwUkDipxJaUkE3OEEz9oyTpM8+nu3sIrjfi9kBVUxdUHMZCwnnOkp2AP1Fovdq86lAJBInQmdvMzsBrrysTdUUq0akpGwMR0+psdsPVAdKOQUqVnH7EQaCWKOgDhAClSFCdNu7tr42a198Y8V3pdtVRFqk7VxIKc9OlbIWFDQpyDoedmWsvAKQGgkCJ70fHT62sEITmQAFDMNAZG+nysdkQaq75CUZVxEx/kaDDd0UyhGjdAgADfnr0/WxR7H3EhSxlvlhhajrkom4g+YsCoA6KUo5pEZjIJkfhYFRSk6uKWknuxp7v08PjYGgVY3ujBxVjpwfaYsrBmH+ybQmOswNP112sVqLyxHUFPbYsv/ACohWRqrLZUImO75/dbgF0FKQoBMmdNvGfP77chLqHFxmk7FKfa06c5185s4N0yDU7W5i65e18vKByku3g6sHTmCojb7zrbVJQUS3FOuUBVJzSZPIHmeo/pbS1uFEIKswMa7gzBiLd0rr6HA4VqMEbkHNy/Dpz99nUeaV1UtMklPqdNoY9q2WD7cjTt6Yf5iqffbLJSUBYTpKpPrLaLpqHytwFPZtKXICEyrSZEzp8jY1U3NiJ50qZuGvymFaUjpk7b5dedkJ2/b0Cm6egvyoQ0RAp6eoUWwEgbBKvLl46WDapMR3stTbVLetToR3ad5yYBGmWQTsCJ8bUuIAbkhHLdLWKcbVz4mYJdXd1akDSFslsak/wA3ujTbrNkeow/edO6qtp2qRhwuKWQutaSVAmCIJBBj5iJ6KFBws4g3o+kXbw9xJWKCVK+zut5YzR1y7jbSZjWdrOSl9HfjdVDNS8JcUw1lzZ7rcbnyzAT0/O0DUxDd49U4he4eFpTHqrhfdf8AW275oGnyElYVXMkGCCYyqJGw018bdVtzUtc0n1jEN1IegpW4ha1FQ3juIIO206T8ZFovRR9Iy80hFPwiv1CETlLyW2gnXYZ1pMf16Q57k9Cv0laqkFK5w9do0O95zt71pkpXERKQudvHkdp1G6tpm7yD1Ck2mmcbZCoKqcL3aw32T2LKNZjOvs23jIIBGnZwIE6cjHmCbtLctOMovqocy91RbpjuFTOsRz+trOL/AGenpC3mpta13KylOUlNTeZB/wAQSpCFxJJ8tOYss0n7M7ixUECuxphqkbUDIbW8sg+A7ODvvpqBpaHtKiaNZAiCiqTsxVIbqKBltCk11cQkDKCyk89/7wGZidPnZw3NxavfCt60943TVVyamnWFsudi1lCkkwqFZs3vEDpa2NN+zGvpsJVePE670LUZhmjWoZv94iRJPQ2TMRfs6blwxQqr774yKPfhDFPcwCl66JTmeUDtzSRYL8Uw+UZHOuDysjxUVcw5mCx+qhwenhx9pmSzR4wvVthIlXZerIyzAGgpxGp/PpZv4g9Mfjpf6CzeGNb7fQhSsgNeWpO0DsUo8eukbb2ma4PRQ4SXSnNfbF9X+oxnTU1XYtKUZIIS0kHmNCSB42Wbl9GTgvdJXWN4McfQ4uQ3V1LjsRuEdBrB5bedqjThEJvHCPQKz3fEX6ufv5qnbH9suOOM7ow9QUia6/L4rGaKkU7UOEurWopQlS3VGEjMdZ2mOQt7SXJhvFdLc9BQruJ4uU1Kyy4ovNgFSEAGJUOYtVvhlwr4WYQ4gXBjFOEbpuZu6rxYrF1qlrZRTpQoHtCoqCQkAH2hFrbv+klwIYPe4s4ZJBBhu8EOH4JJNtyidBXMuPCAsqsjmpH2dqSgxh3E5VBuohX/ANK3+CrYrCOKlIn1Sn22L+vyBFir/pR8CpAZx9T1OYSn1ajqXfeMjZ6WAX6T3CpwD1SsvurB1HY3DWxy1lTQHS1zgUzd3flUjJO7QNR5WAsVPCJoWxvq+o/9FsHCy/XR9re1I3m6NqXr8R4WSXPSZwiof6phXF7+4GW6w3P/ALRxNgV+k3duUKpuG+LFZpyqd9TQFQfF+0g+lZ8Q9UwhqH/CUp3hwSq7xZDNViZsSQruUZ3Go3c6gG1ZscYGvfBmNrzw/S+sXimmWhaakpS2FlaAsjeABmjmbT0/6S95rSk0XC6sUVSP9Zvhhr/wBdqocbuH2NuK/EO9eJLV3Ul0M3t6u09Qt3gt5SFMtJQlRWEICswQTsIg87VqmWleAA7VWaWGpiNy0p1Iu+/SEj/U2l7BK69lB3B5q8/rY41h29alAXUX7czM6FK7yQqDz9mfoWiZjhBjmmQlKr+eGQZUZaxeg/Lb4fFRa4L4mqaJyrfvlLrlOB2jfrC1fZye/udtARGkg9Tanan+ZXCKgn3VI5wuttwCpxVc6Ecsjrq9IP8AKj8bCfuW4WiXKjiHd7Ujf1d8GB5pHL62tF44NPPAl6tQhURISSofEHx+Fke/eEF7oeU5ddTX1lSEStptZSlwRqUDYK0iOc/GYfT7XUeHPvZTU2xw4aSpD/FBpa1CMrVMCJ8QpwWBdvnhPSJUl/GV5vKQdS1TMEGOUdrM9P6C1Z14dShYBaqMqVBCwXFdomdIIInQmOs6RYw1hdCnVKepMhnvZ1SUj+EHqJ6RGp8bSAi6KNpFLl+474aM1Ieu3EN4IQtvs3C/SElW2VGRtKlTuc22h11Fo9vPixciVrTSUd81SUHKFhumbS4BzAceSYPjrrrEGxWnuO7kKk0KAf4wo5ZPgTM6E6EcuekdruG7lhJpmG8hjMkwVpM8pGh5QdN9RFpBzNgFAxPKTjxKp3iS1hy9FOJBVDtVSIEDxQ6uN+li7vEyscTKML0YiCBUXupOUxpORhX1O1l5i56NLKltsU4UCQlWbKtI0kKH5aanewrVJRNlTdRT0JSD3ihsJIHOQN9ht8LPnaOSXCd1TUVxBxSs5qTDtxIJMgfvCpe93/m6RzB/G3CMe46qSDSsXAjLCu5d9S8oeEKdQeXSz0TQXeQFt9kpCBJOSU+Gk6aA+/3ACqpGgpCXVNO51EZ0yABGsGJn4fCz8RvRLhO6pkvYp4iVSM6bzo2FEQFMXFoUxvKqhfjuOVg6W9OKDJNSxiOrQp1tSF9nQ0SISqApMqbXuCefLY8n0UNtj7GrKm82Ud4kaK3zcvCZ326YtN3kOIDLqEoSJUEqUknfcanSN+k72WcHknEVuaYwRjqpSttWJL33yryro2gCf8SKcR7jYBVyYp0m9b/cBUZUq93U9Ng2pJPhFpCUllaA2h1LkwSYKUg7zIEQPvBsUUulpVQXSwUju65dI1PTmT8bLOeQUTE0c0yH8PXu+Slyvr+zMEB29q5QCifF73e7xsWTw+bMO1VPRvISSYqC4/mMHUdotUc9OelpAVeF3lkOvusLGXRSklJjTYyZkT5/G3CL8uLtBMoTsC4nuxGpJEjl79dtLPndyS4bBzTGRw1p1EH1G5BK9k3dT5iNdNUmDPjpPlYdrBaaAksIp2ERKizRtInXllSIj3naN7PE3vQq7zb5dBUDBToNDrJAkydPZHzkmu+Wm1JUgVaFH7Pvagxz3gQAddN+tnDnkJZWJCbw29TAJVfFfKoltuoPenkIIM6kRH52DXhRJKfWLwvFQX9p9pUq+ZnqOnMbQBZWXftRHbJSHz7MklKzroBMgnbXYR11sD++CSta6aoZW0Ncq8xSRz5wIB5nppzcZioEsCSncF0SnEreS4pYO+dR7x8NYI0Mc9ekW03hWhbC0JpAk94lJSSCkEEnczrzAIGtlJV5vOIWpCG3mykthOvtGQDI56fLwsXVV1jTZbZLK1ZSpKVNnQ6A7nYGOg231tINclmaNkwOJjBpryYrGGkDOhIzCEiBy+MmAdPLZIvqvURdFWhxWRbCmVKnTRYnfWe/8jPi7cYNO3wzSBCe0U2tSHFIQUwcoM7bkGI+in/6Nq+/rvy0VQhlxh9KgCNNRBHloOg02BsYtzNVfPlcmngfs7j4n3RWuUyX6f1hD5S4S2VpTm6+HUcx1tauj4i4RQM9XdTlKVgFJ9XQ4gzOwSoq2B3Gs6QbVWw4067fl3VTS1NVFAp1KylWVUFtXQydQIA5E+QerlVeLjy00ynV5CV5e3JWEiACQRO+kDSdRIJIrmLiI7ZuHsrP3XjnB1WhIZxDQU7RkZXkGmgiJMupHjqNJ18bOWlFDeQDtBX09Qkpkpb7N0AwRMp3G+418TtTF5+pUwXKt1LaHgMgeQEAhOg7pTBAHxB5wMum6ZapyJCnHSrs3GYCgYTqNATHdOnTnpAnUAJuHIza+27VdZN1A5JS2VA5VQlSZ1/wjl5jSdNLdKu9AOZypUkKVJCHACAVcp8lD3adLVAo7w4gtIU5dl9XwOzSM6E132aNDIyFW+3IiAIibLNPxB4m3WpNO9WULiFOe1VNZQrQj/ZpH825Mkk66pFoGiI5ogrmnkpl4jYeYqlU9VSvpFSiYzKSpQJgaeACR+PKzQa9YRlSpalPnuqQ2iQ3G+o03G8czvtZCo+KVbeakM112UruZc9uzXtJKQVCVFrMTHMQZ5cps5GKhupWVs5Ql6BKBqecCNOY/W2jTtyMylZ87g5+ZMa97lqXA5eNW12inHSvMUZlZUmBqRy2033625u6pXQv01UhnMqnUkkkGMydSNtOfWzrqKO8bzZe+0QaMKI7Rbm4EzEjUb2Zj6YfCCS4lKsgJOveJSojoTAn42BIy2hVmN3MKzWGsVXVV3QyFU7ilsogxkGh0CpmdgDz0B3spqxLSnNlpnFAnUKKYHnA00jYfHnCXDm+lCmNCXRkTISknvTPdnfl93xfPrpglTSBzCijuidDJ/p4TvaiaRm6vCsdbROtzFCASn93NgEc3BMAHUmBpJ/LbQsb/qHTnTTsJjXQkySNx4STy5WbqUuqzkJdgHVSRB93XlqND4cukKXklp+QCdMugnkep8Y5edmFKwJd5eUtuX/WlSiWkIIhIOhjXfU7mLF1XxeRVC6ooAJMBrSJ69dB8DZNQt9KZcQ2oKI2Okaajl9c97bUakgpLaTr3VJXJHl0P62mKdvRRNQ5GV3nWLzJU+ohRIgaKHhNgV1tSvKsOVZyynQnT7rBdmsKynMhJH8SRA6n77dlIH8KlJ2griZBmz8FvROJndUF6w+pJU6p9SojvqO/uNgCtbye1kFJAEFU6axzscLLKzk7IjWAUrIA6nU6b25XTEkykzoQT3h5SOf1rZxEByUeKSk5ZUqAqphZ/lVIJidtpP1vbaXHm0jsy4Qf5kyBt5QR5crGlsqSofaEzsVHRXn1+/SwC6dwgKSnTf2hOmm8aj8vO0xGFAyFABbgXmD6xlEzlj39I+X4cds85JCxrqDMeEEcxHzNjKYalYpgVezJ+/Qn657W5cUy6AFNrzSNMoB+Ea6eNlw+ibiFF3HCpJJQc0xOhM9frrblVW7JSpGaAU8oynwG3152NpbpJBzpSVCQI8ySdbcpaYSQpsz3dRl08NuW/n4WcMUeIgBeb6Up7enATvO8x4/X3W5XWBIhQVmJGkj468vlvYwpTYWEHs5BOp6chE+dglFOfIgIKpOZWWQT4wNPdaQYomRFV1UHMArKkDWNYnQx9eU2364gtlS+UaEgEmBrv9c7C5UhQDYQAAYypgADbQ+7624cQoFKkOazrJnpr+n5m0gwBRL7aoNT7LiSnKjTukBM6fU/py5X6stJiIImQ3O/u0+tbcvJcXK0rSUE5gc0Dr1j68rAO0j4UVpKErB0IPMg69Zm08qhxEIUsqCkpIBnSFEGdvyGnhzsEtDLqVBM6+ySomJ8J+XxtnY1bZWtVQkwCQpEj3dOR+fnYMpfy5VOI5zrpz1Gn1NnSJutBppMErSYEaD37b/1+IZQUAwsnKAMp03nTrYYiqOZBQ0FIESdpiOW/KwKwpCftWExMKTlMn468vlZhYJkCtxzMYdCiPEA7RMxvv4WCJyHs8+p0235zM9eU/GLGF1DSCEPUwCRoDsPOAI+EWCS8wrLKXVyiAUpMRHXbpp/Wz5gFEtKLOOEBZIWoZYIDJBPjvG3KwrK+8QUpgakwZ6g6672MpaSoAMB0IgElaD3p5z4WOU91rWnK0jInYBQiCT5T9RaOcAJ+ESi4LahmcYbzHVWnO2WWE3UnKPs6c6b5x+dss3ECfhFemDOCcMU7aW6a4ruZQkylKKRAAPkBY81c9G0kJS02ANgGwBbLZbysDmugzuWlUbZVAJSPC2l3a1spaiCLZbLFawHdFD3DYoBVOw0fYKvM20420kBRRO9stlpZQEZrid0A8hLeoSkgQdRbRcPZ5ghG8bWy2WgNCjt1tdJN8Xg5QUD1chtKi2kkJ2+doGxD65iWt9dvWvcWZKUoQkJSnyHutlssamF7kqyNkjN4epFvoQpxw5yATpznWwbt0UalZ1hZCQAlIV7I+vnNstlroUCuU3PQ1YVTPtl1CiEqDpzhW2457/Kwjty3RQVLtNRXZStBhZaBDKZOWNdt5M2y2WsROIGhQXtB3C6ZaOU9kUtgJBgJ01/rYaoazuoZmOzbSJA3mSfvtlssYOJ5oeUDkglICVBKiVSqN42BV/02FS029QNqKT3XlpTJkjQSfPS2WyxWqFkCmlRCtRokKHdH80fh87HKKjS6p1hSzl7FatBr3UlQ/8ADHvNstln5qKKu3ZTJUVkEgTI66n3cultUdOyxWJep0BK2pOsKCtNiOYiR77ZbLTCbmlG88NUdM+2thWRt9tLyUBPsBU92eYkf15kDhmjfLrSllJVELQkBSTMZgeotlss5UVGfE3BV1G4l42bKkVbDxp6pEAipCHuzCp/hXporXTQ5tCImcbp00yCadJLqdY00k7dNRMjrbLZbSpdWC6zanR+iIu3hTtU7MUeZNQ6lEKUk5SQVBRlOvsj36+FiSMQPs0jaktznqOxlRGYabzGuw0+62Wy10AKmXFGW78qX3mWlNt5ylHfKZ1UAdvqbEay/Lwp+wUFpyvgaJSEkBRPPYxlHK2WyxA0KDnFAu3/AHmahspqD2iVwFlKTptG22vPpYVmtvAVZoqmvdeStsOjXKBJiIHv+XSbZbLEDRZBLjddPdrTutvO1Dj32yGgCYgKSNZGvP5DxkOqU+h5omqdJdkI7x7iYBg9efTe2Wy0g0KNyiqAXqVpxMI7RAMCYB113125/pbpqrWhtmkqJcU64W86CG4iNSANfKbZbLPYJXKMIYaU6WinKSgjM3CfZ0H4W5ao1J0D6ozaAgHYkc9eXW2Wy0VFxKEVdrfbJSh1beUTKAAdCTE+87zvYuFR30rcIUsAZlSRKtRIAEeEWy2Weya5XCXKZDgacpyo1BDeYKAgDroZ38PnYneF4ponlqQ0tbhVBUtwGYT3dI1iOetstlpBMiDd8orVIUKFKFwO8VzG+2gj2jtZzXfdabyaCw4lkAxCWknqJncnTckna2WyzlRRm9sIquui/eD9e3UpXWIQEBgoIKp1nMQdxpHLygRm7BTkoafUC04jKYE7kfh87ZbLEbsou3UOfudqjxIwpDplFUlKyEiVy5BMGRyO4I185sXRcNcLVlR6oll5ChMkqTljLmgBKUn4kxPutlstUYd1ZIFlqqwFhy66pLCGKlRTupNa+gK15pC4P9etu2cLYdcZW45dTToBHdfJemT/AIyefSLZbLEuVEALlFw4SaUhJwtQBSnEolDLYGugJlBnfr12k2VH8PUN00yapimpBmyyEUyU848Rsek2y2WiN1MgLlla3m0qKjKBIkyJOm2x2G/TSLNN9S1XmrKoJ7VQTokaAkEbDkeke62WyxYt0OXqmnjTiJeVx3mMK0FIh1SkOPGoq3VuDRShAQkpj2eR2jpaPl8RXXXqti8rnadcpCUFdO6WkuAAH2VBcbxv162y2WHL7yND7icHDTiWbyv391MXMacBttXaGpzEkzEgIExB8dRqItY0pacAUtKoypV3VQZO/KPlbLZYbt1Jmy6FGlaA+lxaYbC45klAO/kQPdbWVxGqXlSO6CQDz8bZbLMnXSW1HtBn0B7o1EaeBH0Lbapg+o5XXEwspMkKBgnr5Wy2WSa5utuULjOVfrMhPRMHrvNiK3nUuDOsuELCDm1BM7m2Wy0gE91yupcStCFZVaADSBHl1sJSvl1bVOJCnAO+YMEgcotlsskgjd50TtNFQakKOYAAIIiVKHU8k/OyahxaiSFxBSmMoIPyn52y2WdJcuZ1uKSVDuiZj422ptJBJkkqMTEaAfr8bZbLJILHLuUlDjhfBAAhOTSJ23sVLUuKSCkajZO5m2WyzFMgkMdsla82UJ7sRJJBiZNtmmGVBSqAZ0InYn8vnbLZZX0SQakONFQS6QE6R74sNTtuuICy/MxlCkggDeOhtlstFPYI4i7FK/vXkrEbdnHQ8j42E/c1PGVxxZjQ5DEyJ5z0i2WyzOJT2C6YuGjqELWlx9OUjRTmaZ+Fjhw3SUrgQFlWROhjx5yTNstlhucUVrQtJudhTgADQUteSS0DBjewT9MhlGdpKE8kiDoQTrvbLZYYcbqQaEWSiUoeSopzlWmmgG2vXxsNT0frLvZLWkCSkdwHafytlss9ynaAumaCmeLjamkDTMTlmZIH5fCwzV20iSFKSqCf4SAdQDrIPW2WywwSpWF0IugpMxlmTO5O9stlstJPZf/Z" } ] } @@ -86,141 +68,72 @@ { "language": "markdown", "source": [ - "# Load a Detection Model" + "## Run inference on the detection model" ], "outputs": [] }, { "language": "typescript", "source": [ - "// Initialize OpenVINO core and load the detection model\nconst core = new ov.Core();\nconst detModel = await core.readModel(detModelXMLPath);\nconst detCompiledModel = await core.compileModel(detModel, 'AUTO');\nconst detInputLayer = detCompiledModel.input(0);\nconst detOutputLayer = detCompiledModel.output('boxes');\n" + "const detInferRequest = detCompiledModel.createInferRequest();\nconst detResult = await detInferRequest.inferAsync([tensor]);\n" ], "outputs": [] }, { "language": "markdown", "source": [ - "# Prepare Image for Inference" + "# Load Text Recognition Model" ], "outputs": [] }, { "language": "typescript", "source": [ - "const imageData = await getImageData(`${baseImagesDir}/intel_rnb.jpg`);\nconst inputImageMat = cv.matFromImageData(imageData);\nconst displayImageMat = inputImageMat.clone();\n\n// Resize the image to meet network input size\nconst [B, C, H, W] = detInputLayer.shape;\nconst resizedImage = new cv.Mat();\ncv.cvtColor(inputImageMat, inputImageMat, cv.COLOR_RGBA2RGB);\ncv.cvtColor(inputImageMat, inputImageMat, cv.COLOR_BGR2RGB);\ncv.resize(inputImageMat, resizedImage, new cv.Size(W, H));\n\n// Prepare input tensor\nconst inputImage = transform(resizedImage.data,\n { width: W, height: H },\n [0, 1, 2]);\nconst tensorData = new Float32Array(inputImage);\nconst tensor = new ov.Tensor(\n ov.element.f32,\n detInputLayer.shape,\n tensorData\n);\n" + "const recModelXMLPath = '../../assets/models/text-recognition-resnet-fc.xml';\n\n// Load the recognition model and prepare the inference request\nconst recModel = await core.readModel(recModelXMLPath);\nconst recModelCompiled = await core.compileModel(recModel, 'AUTO');\nconst recInferRequest = recModelCompiled.createInferRequest();\n" ], "outputs": [] }, { "language": "markdown", "source": [ - "## Define Post-Processing Functions" + "# Define Post-Processing Functions" ], "outputs": [] }, { "language": "typescript", "source": [ - "// Function to extract bounding boxes from the model output\nfunction extractBoundingBoxes(output) {\n console.log(`Output shape: ${output.getData()}`);\n const { data: boxes } = output;\n const foldingCoefficient = 5;\n const numberOfBoxes = boxes.length / foldingCoefficient;\n\n return setShape(boxes, [numberOfBoxes, foldingCoefficient]);\n}\n" + "async function performTextRecognition(model, inferenceRequest, img) {\n const inputLayerShape = model.input(0).shape;\n const outputLayer = model.output(0);\n\n const [,, inputHeight, inputWidth] = inputLayerShape;\n const resizedImg = img.resize(inputWidth, inputHeight);\n\n // Convert image to grayscale and create tensor\n const tensor = new ov.Tensor(\n ov.element.f32,\n inputLayerShape,\n new Float32Array(resizedImg.grayscale),\n );\n\n const result = await inferenceRequest.inferAsync([tensor]);\n const recognitionResults = extractRecognitionResults(result[outputLayer]);\n const annotation = parseAnnotations(recognitionResults);\n\n return annotation;\n}\n\n// Function to extract bounding boxes from the model output\nfunction extractBoundingBoxes(output) {\n const { data: boxes } = output;\n const foldingCoefficient = 5;\n const numberOfBoxes = boxes.length / foldingCoefficient;\n\n return setShape(boxes, [numberOfBoxes, foldingCoefficient]);\n}\n\n// Function to adjust bounding box coordinates by a given ratio\nfunction multiplyByRatio(ratioX, ratioY, box) {\n const scaleShape = (shape, idx) => {\n const position = idx % 2\n ? Math.max(shape * ratioY, 10)\n : shape * ratioX;\n\n return Math.floor(position);\n }\n\n return box.map(scaleShape);\n}\n\n// Function to extract recognition results from the model output\nfunction extractRecognitionResults(output) {\n const outputData = output.getData();\n const outputShape = output.getShape();\n const [, height, width] = outputShape;\n\n return setShape(outputData, [height, width]);\n}\n\n// Function to parse annotations from the recognition results\nfunction parseAnnotations(recognitionResults) {\n const letters = '~0123456789abcdefghijklmnopqrstuvwxyz';\n const annotation = [];\n\n for (const row of recognitionResults) {\n const letterIndex = argMax(row);\n const parsedLetter = letters[letterIndex];\n\n // Stop if end character is encountered\n if (parsedLetter === letters[0]) break;\n\n annotation.push(parsedLetter);\n }\n\n return annotation.join('');\n}\n\n// Takes original image and bounding boxes with annotations\n// and returns the image with annotations\nasync function putAnnotationsOnImage(img, boxesWithAnnotations, options) {\n const defaultOptions = { threshold: 0.3, confLabels: true };\n const { threshold, confLabels } = Object.assign(defaultOptions, options);\n\n let finalImage = img;\n\n for (const item of boxesWithAnnotations) {\n const { box, annotation } = item;\n const conf = box[box.length - 1];\n\n if (conf < threshold) continue;\n\n const [xMin, yMin, xMax, yMax] = box;\n const yOffset = 10;\n\n finalImage = finalImage.drawRect(\n xMin, yMin,\n xMax - xMin, yMax - yMin,\n { color: 'green', width: 3 },\n );\n finalImage = finalImage.drawText(\n annotation,\n xMin, yMin - yOffset,\n { font: '30px Arial' },\n );\n\n if (!confLabels) continue;\n\n finalImage = finalImage.drawText(\n conf.toFixed(2),\n xMin, yMax + 2 * yOffset,\n { font: '20px Arial' },\n );\n }\n\n return finalImage;\n}\n" ], "outputs": [] }, { "language": "markdown", "source": [ - "# Do Inference" + "### Do Inference and Show Detected Text Boxes and OCR Results for the Image\n" ], "outputs": [] }, { "language": "typescript", "source": [ - "// Create infer request\nconst detInferRequest = detCompiledModel.createInferRequest();\n\nconst detResult = await detInferRequest.inferAsync([tensor]);\nconst boundingBoxesArray = extractBoundingBoxes(detResult[detOutputLayer]);\n\n// Show original image\ndisplayArrayAsImage(\n displayImageMat.data,\n displayImageMat.cols,\n displayImageMat.rows,\n display\n);\n" + "// Calculate ratios\nconst [ratioX, ratioY] = [img.width / detInputWidth, img.height / detInputHeight];\nconst boundingBoxesArray = extractBoundingBoxes(detResult[detOutputLayer]);\n// Resize bounding boxes to the original image size\nconst boundingBoxesOriginalSizeArray = boundingBoxesArray.map(box =>\n [...multiplyByRatio(ratioX, ratioY, box), box[4]]);\n\n// Process each bounding box and run inference on the recognition model\nconst boxesWithAnnotations = [];\nfor (let i = 0; i < boundingBoxesOriginalSizeArray.length; i++) {\n const box = boundingBoxesOriginalSizeArray[i];\n const [xMin, yMin, xMax, yMax] = box;\n const croppedImg = img.crop(xMin, yMin, xMax - xMin, yMax - yMin);\n croppedImg.display(display);\n\n const annotation = await performTextRecognition(recModel, recInferRequest, croppedImg);\n\n boxesWithAnnotations.push({ box, annotation });\n\n console.log(`Box ${i}: [${box.join(',')}], Annotation: '${annotation}'`);\n}\n" ], "outputs": [ { "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAIFArIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1SXSkkG1ihGQehHQ59a429+D3h+9klkY3SNKxZjHcY5Jz/Epr0RvvfjSgVEcPSg/djYp1pyWrPJpvgbpjKVh1G/j/AN5kf+grPufgXMYysGtvj/btR/R69rFLVciFzM8Cn+CetpjytRtHx/fSRc/kpqhL8JfE0RYqLSX/AHJtv/oQFfRtLS9mh8x8yP8ADjxXGrE6cGPolzE38nqtJ4M8TQld+i6gRnkrAzD81zX1IUVuqg/hTDbQHrEn/fNL2Ycx8py6VqVqx+0Wd3F/11gZcfmKgG4ZBbnmvrMWsI6KV/3SRUU2mWlwCJYhIPR/m/nS9mw5j5W+fAzxTwW3AjJAFfS03hHQpx8+l2Z9zbRn/wBlqjJ8PfDshz/Z1uPopX/0Eip9mylJHzZEzYI3H7x6/Wp0ZiseM/d6173L8KfDjA+XZhDnOUmkH82NUZfg/pDABJLpAOBtuAR+sdSqbTLc00eJbjuY8kdOKYWbHPPocV7DN8Gockw6lcL7NGj/ANVrOn+Dd8ABDqStj/npBj+TGqsyLnmJPI5FKCSSNxrvpvhJr0Zys9k49MyD/wBkqhN8MvE0IbZaW8mf7twgH/jxFFmByO9gn3j+PanAuGAyfxrem8DeJIcZ0edsd49sn/oJNUpPD2tW5Jm0nUEAHVrZwP5UrDuZ4JYnp+VLuPHofSnNBNEAJUdGzjDLjrQFJYZ5xUlCgnOOfes+d3XU4scHA/rWiAc5x7Vm3fF3EQM57596mQ0aMUm6PrnnAp2/AznNRwHbFv754pxOAFweaUdimBftkY9qCxAz+tAIJPtTSV25I6+tUSOJbOc03cc8nHelbHTn3o25Ht069qkYmfu5yD2oyScnj60u05xyTTgpB49KVxgCRTWO5Dj09afg5HAye1KVODnnIzUsZl2TsplHPbI/OtLYjYbbgHms2Bis0q4/WteHJgTjgipe41sQ+SpxlTSGAf3m9OtXAoAzwKUqMcLmi4WKX2f+EMffNM+ynPOCO2RWhsB4oCZPFLmHYoCyLAEqhP5UNZnHCd+gNagj9BSFc5qeYfKYxtW7Ix9/Wo2tGByQ6jPcVulMDnjHrTGjOwnB6Ucwcpzo3K/r2qdYtyAnAPrSTrtkPHetSwVXtACoPJB4zQ2CRlPZAIWaJD34ArF1C12xl0hC8/wjFdfdJHgoFGO+KxL6P/RZMdAc/jW1KTTM5o5bawPANHmOP4m/OrxT3PPrUezII6/hXZzHPYq+Y/8AePFHmHvU5jGM4H5Unlgj7uPei6EQbzS7+MU8xjHIIpjIAuRmnoGopJ4JzzTd555609uY09gR+v8A9eo9poAXfil8w+lNwaTB9KYXJFcA8jinmYYwM49ahwfQ0lKwXJd4IxRuFRUUWAc/NNoopiHI5SRXBwQcg19uaPffb9EsLwHInt45fzUGviGvrv4X3pv/AIa6HKTkrAYj/wAAYr/SuLGLRM2o7nXlzSbzSUhrhOiyF3GjcaSkqh2FViHYZ9DT9x9ajHDj3Bp1ddPWJEh24+tG4+tJRViF3H1o3NTaKYh28+tQzMd8bZ74qWopwfLyOxzQxoXc3rSb29aTtSGkAu808OcVHTl6VLQ0O3tVS+lmjsriUT+XsQsCEzjAz+NWGOB0yT0FVLyyF3aTQzOxEileOAv0Hf8AGpavoPY46TxpbC2huJLdpZJNrBSqgpn+LO44+nB9ap6ze6fKJQ11cm7JCNGMny/baOAT6MQcVn3ekwXlhd2YmZLzT2MfmNgkqBz379f0riFvtkUNs06tPBJhcAFVABODkc4PvjNY1KV4tGTqNM6z7bpwHK3ue+JUH6eXRVJfFjBQHS4LAckAdfzorx/Y1OxtzRPdG+9Sjmkb71KK+wZ56HClpBxSikMWlpOtLQAtFFFABS0lLQAUtJS0hhS0lFAC0UlFAC0daSloAaUQ9VU/UU3yIv8Anmn4CpKM0ARNbRMMFePTJqnNoOlXBzLYW0me7xK38xWhmilZDuYM3gvQJs7tKsx9LdB/ICsy5+GHha5YM+mRhh0KvIv8mrsaM1PKg5mcE/wm8P8APlC4i9kmOP8Ax4GqM/we09jmC/uo+ONzK3/sor0uijkQ+ZnkkvwclCsI9WYk+tuP/i6ozfCHVhjy722YL03q4/kDXtNLS9mh87PBp/hb4hiDlBaSn/ZlIz/30BWfL8P/ABFABnTVZRyds8bfpur6JpCARyAfrUukhqoz5sfwrryKXOjXzDtshZv5Cqcuk39vgT2V1F674WX+Yr6aWGIoAY0/75FH2aH+4B9Dip9iUqh8ulcM249PejYdpwck9K+n5LG3l4ZC31YmqUvhvSJ/9bYWzH1aBG/mtQ6LGqiPlfaUu2XuRxWxaZNqgPQZ9s179N8PvDM7bn0m13eqx7f/AEHFVW+Gnh/btithGvoryZ/VjUujIpVInig+7nH6U4RbgF5+tevy/CzSWOY5bpcdB5q4/wDQDVRvhZEhJivpRngZQNj9RWbpTLVSJ5gIULHIGKcsKjr1Poa9Gl+GNyMeXfBsf34sfyY1Sm+HeqpuKyWzHtw//wATWbpz7FqUe5w/lgtxkbR1PemmI468n1FdbJ4E1tMYto3A6kSqP5kVTfwrrEWSdNnJ7BBu/lmo5Zdh3RzhhOSBj3OKYyN5fbn3rcl0TUYiA9heIAOS0LD+lUJbd4j86upHQEYNLUehyd8hVyCBnOTg1a06YrasOpDcUmooVDjHzNyah075oXHuK0SuiHuWXOWz3PrWbeoDDKD0A61pEHGe9Urpcq4xwVPvWkCJHOsPTnHvmoOTxjg1YJwABn+dQHBFdaOdjW4zSMBxnGfrS56kHnofemknjI4NMBDjpj3pj/dbFOOcd6aeQetUIQH/AEccfdfr9R/9al4OB1zSIMwSD0INIMkUwDC0YwcGjNHX+tABjNBUDijnOCOKPrQAmP8AOKQrTs+2fxpD7UCGkYptOI4pKYhK+mfgNffafh89uWybW8kQD0BAb+ZNfM1e7fs73vya7Yk94plH/fQP9K58Ur0zSk/ePcTRmg0leYdaFopBS1QCHgqfenU1/u/TmnV1UtiJC0UUVqSFJS0lAC01xlSPUU6kNOwEKcoPpSmmowO5RjKsQfanUgEpy96bS5xnHWpY0OAJOAOaazwhXDSKMe/Q1Onkqu+V05H8R4rAvNathqzad5roWXeFTjI756n3wB9cVlKryK493Y4DxmZYfE32/TpI0VodkrSKYwTyMFiAOmfeuJtNAxOupahf29naysfKUtjzeecd8e4HX1r1XxBq0ElosEttJNG2GSJISiJ6HLLtyOvXFeWXfh+91L95d3iR7mLCWc+X8o+p549K5Z13JWbsS4JPa52McPhJIkUMjAKACUbJ/wDHxRXMx2Nokao19bFlABO1+f0org9k/wCdl83kfQjfepRQ33qUV9azgQZqB3xubLAD0NTmqsoysi9MgimhM5rR/iJoOtXkVrbXM8c8pwiTxFdx9M8jNdYsrfxVyFl4Ksra4gnJV2iZXDBcHI5Fav8Aawl1k2UaSjyk3SM6FQSTwBnr0PI4qrCub4bIpjTbTgDOOtNQ/JWL4mjefw5eQRsVaZRHkHnDMAf0JpWHc2I71JRlGR1BwSjA4/KpxICK878H+HH0LVhLC7LDMhSSPPB4yD9cj9a7rzNsuz2zQ4gmXCwUZPSm+cn979Kikb93+IrivHGoa9YT6edFuFjDLIZUaJWD4246jjv09aXKO53gdW6MDTs1kafcyS6da3NwoSWSJGdRxgkDI/M1oq2RS5Q5iajNY+p3rWOm3l6kLTNBG8giU4L7QTgH8K5Sz+IbXLBToWoqx4wh3Y/QUcoJnoeaM1BHKXUHkcd6jedgXO7Cr7e2aVh3LeaXNctpXjfQ9ZuUt7DVI5Zn+7G0bIW78bgK6JJtw5FFguTZoqrcXiQFAxA3ZxlsU0XoI+4SPY5osFy5miq8dysjbQrD61MDmkMdmikprSKgy7BR6k4pAPJozTQ6sMggj1Bpc0ALmjNJRQAi9KdmmL3HvTu1AC5opKBQAtGaSigY7NGaaKKkB2aM0lFAC5oIB4IBptLmgaG+TEefLT/vmka3iYYK8fU0+ilZFXZRn0LS7nPn2FtJnr5kStn8xWe/gbw1Jn/iT2a567IVX/0ECt+ilyod2clP8M/DE2SLDYfVJZB/7NWZc/B7w/OSVlvIif7kw/qpr0HNGaXJEOZnkNx8A9MbP2fVr2Pjjfsf+i1j3P7PtwB/o+u5P+3a4H6Oa93BpQafKI+b7j4CeJEyYdQ0+QDpkyqf/QDWRc/BfxlADts7ab0KXKr/AOhYr6ozQDRqGh8gXHwy8Y2+d+g3D/8AXFkk/wDQSazbjwb4mtgTN4e1VAO5s5MfyxX2iQrdQD9aZ5EP/PJP++RRdhZHw01ldW4lWe2miO3pJGV7+9VsHFfdjWsLjDLkem44qjceG9Gus+fplpLnr5kCv/MGjmYcp8RYPegn2r7IuPhx4RuQfM0CwGf7luif+ggVk3HwY8E3GT/ZCRk/3JpV/wDZqOfyFynybRgnpX1BcfAXwlL/AKoXkP8AuXH/AMUGrJuf2etJYk22p30fpvKP/wCyrR7RBynzqBxRtHrXulx+zzOMm31sn/rpaj+kn9KyLn4B+Ios+TfWEo7BhIp/RSKPaIfKzyEgd6bjmvSbr4KeMof9Xa2s3+5chf8A0PFY9x8LfGtqcvoUzjP/ACykjk/9BY01JdxcrONr1T4CXnkeO57fOBcWTrj1IZW/oa4m58FeKLQnzvDuqKB3+yOR+YFb3wuF3pHxL0d7m3mgDyNCfMjK/eUqOvuRU1bODCCakj6rNJRmivKO0KUUlFUgHEZBHrQpyoPtQOaRD8pHoSK6KREh1FFFbkBRRRQAUhpaKoRABiVwB15pTSP/AK5T6jFKeaQXCpYIfMfcQMCohy6oOp6CtHb5UOEGSBx7moaQN2KVxFDcMftTKtspA2lsBmPr/LFYWvaTpcNnNIq/Yw/ytMkrRFs9RwCWPTjHNdO22PBIzjpxk5qte2EeoweVeIGhYg+Vng49cdfp0rmnTTTTV2wUtTztS06Yim1S6hQZBkUspwD91SCQePY4471xt61ld6sls9/fM9zKUaJ48FRxk4GcDj0H0Net61cvJbxw6bGQkbKA6LkEZwVXHt3OB2z6ec+JdRG26ewnghuWg2yrb4Z1IBXBYf7wIxjPNebOnyysnc6FrG5lyrp6Sui+C9YKqxAy0o4+gTiiuf8AsuunmJ7jy/4NzDOO2eaKfso9/wAWY83kfSzfeopWHzUCvqGcaEqhfc2dyPWNv5VoEVy2sf2+uoPHavbNZS4GHU7lBGDz+dNNLclps47w14XvdG1Ozube9m8oOokjL8MvQg/hXos4Xz4iRzg4/SmLZPHZsYwDKFyoPTPasnTtR1a/vhDe6WluiAnzFkzz6Y/z0qrruKzOoT/V/hVLURv0/jn5kP6ir0akJiufvNch0mZrXU4pVhOfLnRCylfQ45BFCYmjRt1CtGf89DSu+dQwOyD+ZrGbxj4fgj3LemRv4USNsn25FXNGlmvw17JE0QmbcqN1Vegz74FDYJM2W/1Y+o/nXLeLvFn/AAjU9kraW16k4csyvgx7dvbBznP6V013LHbWrTTOqRpgszHAArDvjo+ryRtJqFq3lghQJFPX8fakMsI0et2VlqcE0yxFRKkecBsj+IeorXgY7BmsJtS0/Tra3061lSSRiI40jYEgdycdBjNbdvkoCaAK19JbxaddS3jFbZI3aVsE4QZJ6c9K56y17wk7KLXUItzEBQVYZP4iuiu4Rc2NzbkZ8xHTB75yKw7DwpZWsyytaxBgQQdoNSyonURdOOlQNz5wPc/0FTxgBcZqEj5ph6n+lNCZydh4LtbHULa7GxZIHDgqOTiuit9QS4u5Io1cCI7WLoVyfbPbpzXn2ieH9S0fVbO4h1C4a28wB4mclSp4r0bA81DjnHWhgirr2h22t2qR3Me4oSUOcYJqroelJpmjS2SZCq8mOfXn+tdCvK1UAw8w/wBr+goA5jwnoUulXDStd3MoKbQskhIH+cV2ydKoWaAIvHar4FQWPzWVrhJswO24GtSszWRm0/EU0Jmdpl55TNESdrcj2NQWVy39qiQsxBds8/WsGTUJLfxX9gfiOS0WaFv9oMwYfltP51paezGVWcEHzW/9COP0pdB9TtI23Cn1XgOVqehgC9T9aWmjqfrS0gKV7qP2N0UR7yRk84xTkv0a0+0EEAAlhnOMVn6x/wAfCf7tZUl6p0+9Eb5CKwYZ6MBUt2nZ7FqN4XR01pqEd3uCoykAHnFWi3FYGnMY5MA9Vq/qFy0FhJIv3sYH1NODuhTSTJpNTtInKPOu4dQATj8qnhuIp03RSK49j0rzjWINSuYEGm3PkOCWY7c7vQVc8L392UtpLrInJ8uUevOP8DUptq5XKtjvy2BntQGBGR0rO1SF7vS54I2wzoQOcc1zWhagYZRu+XPySA9iP8/rVuNlchb2O2zzS1yj38tzqAKzSLHuAARsZFdIJQULZ496lO70KasTbqXNcjq3iRrWGS5VpBAnQRJuZqseH/Ea6pDDKrl4ZvulhhlPTBxU8w+VnT5ozVa5ult7aSZhkIM49ayz4gjTHmoqZ4GZMZ/SnJqO4RTlsbuaM1RsdSjvSyopBUZ6g1dBoTuNqw7NGabmqz6hbRyGN50Vh1BPSgRczS5qtDdQznEcqOfRWBqbPFG4EmaM1GGyKN4JxSC5JmjNNzgUA5HFJjH5o3UzNGaQyTdS7qjzRmgB+aSm5ozSAcaYQD1AP1ozSGiwxhijPPlr+VNMEZ7Y+hNSE0manlQ0yI2644LD/gVM8g9pX/HH+FTmk71Ps4voVzMgMcg6SL+K/wD16TbKP7h/MVOabS9lEfMyPMg/gH4NUMDzlpC0SqS2cbv8+lWqjHEp96FTS2DmuKGfun5Gjfj+Bvyp9Ga0sTcZ5i/7Q/4CaTzo8/fFSUUrMY3ep6MD+NLSFFPVQfwphgiJPyLz7U9RDLhtgVjnhhSb95wn4nsKiurQNbuI2ZWxwckgfhmpowBGoHTFGtw0Lttb+UN5+8euetWSQFLHPHoM1WS6UIA2S1Zlxrl7Zbml0tpY92FaCUNgf7QIBH4ZrKpUjBXkTyuT0Ndl/wCWjZHHTPSud13ULu6YWGnouBgyvISA690AHJHqR24zzVLVPE89xILNVS2YkHaWzIw9h2/I1nXGqSWtm4SK2V5mw5lO4tgdSTkn8a8ytipSaVPbudEKDSvIbeLMLSSK906TLZ8yaIrMPrsyFHAAA5xwPSuH1a10ay8iWzknMMpCtlcMG6gsDxjnjAHWuluNVuhF/o8toZGBACQqQD2OcYwPrXM3V3G9zuv2e9uRwCuUh/IDJz+B46HFYOTbLcUkc9Jq2rRSNGlpeuiEqrHaSQO+cUV1hksSTts/l7bYcj8Cc5/Oil7eP8pnys9qkHz0gp0v36QV9UcAYpjRhjzUgFLQBEEAGKYLdQ5YDmrGKMUANC4FV7i0juFKuoIPqKtUuKQGGug2aSb1gQHPUKK04YBGoAGBVjFGKSSRV2yC5to7m2eGVFdHGCpGQa5ibwXpjtn7Mg+gxXX0hANDVxJ2Ob0zwzZadL5kMCq/rjJ/Ot9EwMYqXaPSjFCVgepyWsaFqU9/LPZ6vdW6Pg+WrfKDjsKoLpPiSM8a7MR/tRA/zruygPUU3yl9KHfuNW7Fe1RxCokYs4ABbHX3rF1ldbjvwdOktxbuBuWRCSD3P8q6QLikaMN1qrvoTZGNJYuNPfywDKFygPTI6VQ0i81a8uil9ZxQIgyGV8kn0x6V1HljGKYIVVsgChyYWQiLha5/U9bj0zUWt5LS7kMgDK8UeV9MZz1rpAMVDNbJKwZhkii9h2RBZqwjUkdquimogXin4pXADWfqxAsyzEKARkk4ArQqrf2UV/ZyW06B43HINNMVjhtfsftaRXtqytcWmHQqc5x1H4g1csZRIIZV+62CK3NO0O102BoYIgqkkn3NZy+EbWPVReI0ow+8IGO3P0pNqw0nc3VmWGLe+ccCrUUyyLuXP41n6jYvdae8MMrQycFXXqCKbo1rc2tr5d1cvPJnJdgB+HFJtBY1R9406mj734UtAGPq4/exfQ1w0StY+LNQtpiWtNTUSJ6B1UKw/EYrtPEv2xLWOWythcSBsFCccVjT6Rc6lpEcssQivIyJUUHO1h2z9OKHG8tSoytE07D5WjGc4XGTVvVxnS5T6YP61h6Je3d1e+VLp00CoDudzxn6V0t3bm4sJouMshA+tKkrKzHVd3dGFpqKzsD6VRVBDfzKvAE2QPyNVbTWYrWdorlhb3MfyukvANWba4hvb/McqO7tuIQ5x0/wpS92PKwiry5jsYvmUA965XVrI2OsCSNf3NzksB2YV1cCkIM1m+IUUWSSHjbICfatY6qxD0dzO0uHzJQ2OF4rfuFIspcddh/lWToTLJHIVIOG7Vuuu6Jh6jFZU1Yuo7s5G3t0ukSJwCrDpUel+GZrDU1MV5ItmsnmLCqjHrjP1qXT5hG4z96JijD0PSrcl9cteP5T7YlwFwAc+tJrlvcpXlaxr6ou7Srgf9Mya5K/0u21e1ijukDKpDj2OMf1rsbkebZyr13If5Vx1zYnU9MjgEssJO074m2tx70VfhTCl8TRteGNMsdKieK0QK7HLnByfSukBrm/DOjtpiSM9zcXDvgZmkLYHtXSClHYJbinpmuK1BZC96sb7ZNzhWxnB7Gu0PSuSvlK6ldL/tA/mKqS/dsUH76Oe8J6hqkiTjUTm5tpsK4TbkY9v8816TLIfs7lDhtpI/KvNdE1m8utVuLG9tY4pYAHVkJwwz716HbSedaq/wDeXms4aS1Ln8JhaFcubiJ2dm3qQcnPP/6xUN1qt0ut3KiaRVikUBQ2BjHpTNJzHMsZ6xzMn61W1RfL8Q3I/wCeiI/6Y/pV2+NE9Ys7a4uPKtZZQM7ULY9eKydE1S4u5QJnUqybgAoHPFXoj5+mrn+OIfqK5zw/Jia2B9Cp/WlL4Uxx+Kx0OsamdPto3Cb2dwgBOAPepdMvjfWnmMgRg20gHPpWV4qUnSkkH/LOZTx+X9ak8PN+6mT0YH8//wBVE1ZJijrc381mtq0KzvGN7FDg4HGa0M/LXJSIDfXkJPBdge3WpnpByXQcdZJM6KDUYZHCbiGPQMMVdzXmemW13pHiCSxluppreSIPH5rFtv5/jXoMUpltA2fmK4J96mLd7SKaVrodFfW80zQpcRNKvVFcFh+FTO4UZrzfQJfJ8RW7HqWKnPuCK6HxrI40iMKSFaTDYPXg1t7P33Az5vd5jpt4K5oVgwryzTdfnsNIvbQSld67o2B5VsjOPqP5Vs+BL+aW7u4Zp5ZMoGUO5YDB7Z+oqHH3blX1sd2SAOabuB4zWB4o15tGslMSB55M7d3Qdsn864aDxlrcV3GJbjcJBlQ8ICt9MD+tS9ClqesUlUNG1RNW0yO6UbScq65+6w6j/PrV+gYlNI+cU6mv2PoaQDqKKKYBRRRQAlFHeloENYZBFV4STEPUcGrOKrRArLKuMfNkfjTAkpyp5iOnHTIzTaVZFiJdzhQDk1FSKcWmCdmczrFjbTTtDPDHKGw2xgDjtu59KwpdMmkVPsWozwjn5pFEowTnjdyB6YI/rXTyW7zT3N1Jvj3ttRHUZUADH884/Os2+SO006Rp1PlkEZfPzn0z71xWjK7asjVt2Ocmsv7Tu2ie4i+zxDcAifM6nuzZPU59B149eC1e3uG1C4gvH8i1DmOOR5yS0YPy5AywXgH0r0e3SG3sy8rwNcTN5kuWACH6e3QCvM/HumXt3rDraM0sD7JF2PldxGCT6nj9amFOCemhMm7XNmO+tUiRF12dVVQAqwMQB6A9xRXBR6VqKxqPtiLgAbc9Paio+rU/5yfavsfYUo+emAVhHxnpMp6zr9Y8/wAjUq+KtHPW4dfrC/8AhXte0h3MPZy7G1S4rLXxFpD4xfIM/wB5SP5ip11jTG6X9t+MgFHPHuLll2LtFQJf2Un3Lu3b6SKf61Mro33XU/Q1XMhWYuKWil7UBYSiloxQA2lxRS0AJRilooATFFLRQA3FFLRQAlFLiigBMUlLRQAlFLRSASiiimAm0UhUU6ikA3aKTYBT6KAGfx/hTqP4h9KWgBjIGHNMEQAxipqKAK4t1ViwAzUm3ipKTtQO5l32jWd9zcW8ch9WUGm2Wi2tif3ECRj/AGVArWoxSauNNoYq4FV7+xh1Cze3nTdG/UVbxRigRl6XpVvpcBht4wiZyfc1o44p+KMUkDZyuseGpZrxr3T7pradvvjbuVvwqPTdJ1FJ83t0sijoEj211u0dxSbF9KUlzblRlykAi/dlfbFcRLa+ILWRo4oLSSJThSSc4rv9tMaJW6im7tWCLSdzk9D/ALWfUUN1bRRQqCSVckn2xXXCmLCqnIFSgYqYqxTdw7Vx/iG5Sw1Eu8UzCRQcxoW6etdjioJrdJhhlB+tVfRoSWtzgLW7sp74SRRSm5Zdn+qYcZzz2/Ou4sozHaonoKEsIlbIQZq2se0YrOMWndmkpJqxwt3crpeu3ME58tZG82Jj0OaZdyLd6it0GA/deWQDnJzmuw1LR7TU4wtzCkmOhI5H0NZtn4V0+ynWWKAB16EknH51VSV7tdSYJdehpWCMlhEjfeCjIPauY0/9xqLJnGy5Yfhurs1TauKxm8O251Y3xaU5bfs3kLn1xSv7nKwXxXH+IY/M0O5Hoob8iDWfot0kDsXcBXUHJ4FdDdWwubOWA8CRCp/EVx7eG9WiOI9S+UDgGEGiclyoILVnZw3Ec4ISRGI7K2a5q+/d69MvZ1Vv0xVzQNOu7NpJLy5ErsNoCoFAFV/ENnffbY7qyijk+TYwdiO9CknB3DltJWKWqup1nSnUcsJFP5D/AOvXSWBzagelcvbLq0kyLNYxqP7/AJmcfpXW2sXlwqvoKxUnKafZGjjaLPO2/wBD8RnsI7on8N2a63xVF5ugyNjJjZW/XH9a5/xBp9ydfkEFvK5lYMpVeOnr9RXW6nC8+h3MZHzmE8D1xXZzL2ql6GCT5LHjt1J5NxDG33JdyH69R/I10ngqQw65GhP342T9M/0rPj0l9ZNwkKFmjXzEbHAcHpn35FS+HneHXbIqpLGQAqByM8H+tRNb2HF9zofHsZK2zdijD6YINc3O1ncaJpsTyulygc7kA4G4jv8AhXaeNLcyaSkwHEb/ADH0BGP54rzia1t7qxNvNH+9Rw0bdsZGRRJK0WNdUel+DXs1014LXzsowaRpcZYkdePpXS1w/gqdTeXMa9CgOPocf1rt6wi9DWQU1+VNOpD0IqibgDlQfalpqcoKdzQAUdqWjpTEJ3ooooASq7fLcj0Zf5VZqvOCJImGMbsH6GmxIfUKhnut54jjI2+7dz+HT86mpMVMlcaY7UYwVHpiuW1CFC0KswcbtwiTlnYDIA/me1dDqM94Y1W3sfNBX77ShRn07ntWBqaXVtaTzJEd/lgvtBfn0B7LnH6nmvOxNZJcsXZmsFpqc5cxNc6n5eoXKwwuP3cSnooHO5+3XOBj3NV7iKG8N55dhBcEhi08eCIsYAwO5PJ7dce9XdQgllVUuJTFBjmNdu8npgDJHvkHiua1BD9oZYdVEdogBEaPtJJGenU4GPXnp7eZUlOd1I1suhfSHSlRVaCYsAASZohn9KKpJ4USRFeTVpEdgCy5JwfSisLx/mFyPsPEjK3X9asx3Gep5qk+d2R1pyEe4r6B6gjUWQN0NShuKzVcqcfhVlJSO/HrUtFFnCE8ov5Uo2g8KAfpUav+NPDY6fzqRk6TyJysjr/usRUy6jeJ928uB/21b/GqeeKdk9setO7CyL6axqSdL24/GQmpU8QaqowLx/xVT/MVl5pe1Pnl3Fyx7GyvibVF6zq31jX/AAqZfFmojqIG+qH+hrB/xoyaftZ9yfZx7HSJ4vvMfNbwH6ZH9asL4ul/jskP0lI/pXKKec9qm6Ue3qLqHsYPodUvi1P4rJh9JM/0qVfFlr/FbTj6bT/WuRGcZpaf1mp3D6vA7FfFNgesdwPqo/xqRfEmmN1lkX6xn+lcVR1p/Wpk/VoHdDXtMb/l6A+qMP6VIur6c3S8h/FsVwOfejOKpYuXYX1aPc9DW/sm+7dwH6SCpFmif7sqN9GFecemT+dGPoaf1t9hfVV3PSuDS4rzUMVPBI/GpFuJ0+7PIPTDEVX1vyF9V8z0XFGK8+GoXq4xd3H/AH8P+NTLrGor0vJfxOf50/rcewvqz7nd4oxXELr+pr/y85+qL/hUq+JdRXq0TfVP8KpYqBP1aZ2WKK5JfFN8OsVuR/usP61IviycfetIz9HI/wAapYmmL6vM6cj5lpcVzB8WsWX/AEEH1/ff/Y1OviyH+O0kH+64P+FP6xT7kujPsdBRWIviqyY8wXA/BT/Wpl8S6cw5aVfrGf6VSrQ7i9lPsatFZy+INLJ/4+cfWNh/SpF1nTX6XkX4nH86ftIdxckuxdoquuo2LH5by3P/AG1H+NSrcQP92aM/RxT5l3DlY+jFAIPQg/Q0uKLisJRS4ooEJRS0YoASjFLRQNISilxRSKEoxTqSgAxS0UUAGKTFOooAbRilooATFNKA9afRSAYEAoZA3UU+ilYoh8lc5xTwuKdRilZBcieJWYEjkUuwbcVJSGiw7lZLVIydqgZ9KiTT4IpzNHEiu3Ugcmr1JS5UO7IJoEmhaKRQyMMMpGQRXNyeDNP84siOq/3A5xXVdqTFDV1YE7GJovh+z0YH7NHgnjJJJxWzTsUmKSVgbuJRS0VQhkf8Q9DT6YvErD1p9ABRRRQAUlFFNCYtQzjMRI7HNS0xhuUj1FOwhOvSkpIzmNfpS0hkkm46c+z74zt9q5zUkeSBAkpaISK7KuP3mPcmt+EGSJ/NH+0qen196yNQVVkPHI5rgq0HObkawkkrHP6pbi7vLXfbr5WC0mWAbjGATyQOvTr7DmsvV2mW2lng0jTxLEhMcjMG2bQcHbt9ulXri9XlLaMzSDJZYiAFPU5J4H8/aqoSRlZ7psDPEUXOc9jnr+lS8LC3NLVh7R3sjzseKYgAH1e93j7224bGfaitN9ABdi/iLT42JOUEcQCn044/KiuPlo/1/wAMV7xqMvzHjjNKq8D2qWUKjfMQp9+KRMEfeH4Gu1O5tawBcds08D2pwX0PPtQFHai4xVYrwf0qVG6ZFRjHY5oXOD29c0gLKv8ApT8gjqeKqhj1qQMRQBY68j60dOpqMMP608HjFIB3b1pcc00EZxml/KgBR1xmp1quODU65xUtlIdTuKTtSfQVJQppM+1LgikI+nFFwDr/APXoA4IxQB+VLxn1ouFgB/lR68UdOv8AKii4Bx1o+tL+dJQAAUdaWk6jigLC5pD796XHSkI9utAWA4oz9KM80h64/OmFgAzS0DrSY4oFYXg5ozikx15ozQOwuaAfek70Z4pALn0pcAj/AOtTc460E0wsO4AyB+VPWaVfuyuv0Yioh7UvGKLsVkWBf3aH5bqcD/rq3+NTLq+oJ0vJvxbP86o+9B47U+eXcThF9DTXXtTX/l6J+qL/AIVKviPUV6yRt9Yx/SsjNGfXFP2s+4vZw7G4vie/HWO3P/AD/jUi+Krkfet4T9CRXPg0ucVXtqncXsYdjpB4rb+KzH4S/wD1qlTxVCfvWrj6MDXLZPFGfen9Yqdxewg+h1y+J7M9Yp1/Af41MviLTz1aRfqn+FcYDS7gQaf1mYvq8Dt117TW/wCXjH1Rv8KlXVrBul3H+JxXB55pQ1P63PsL6rHud8NQsj0vIP8Av4KkW4gf7s0bfRhXn27HNLuprFvsL6qu56ICD0IP0pcV50GA6cfpUi3MqfdmkX6Mar655C+q+Z6DRXBpqV2vS6m4/wBs1MusXwHF0/480/rcewvqsu521Fccmu3463Gfqq/4VKPEV6OpjP1Wn9agL6vM6uiuYXxJcjrHEfwP+NSr4lk/it1P0OKpYmmL2EzocUVhL4lU9bb8n/8ArVIviOA9YJPwINH1in3J9jPsbFJWWPENmeqyj/gI/wAaeNcsCf8AWMPqpqvawfUXs5roaNJVIavYN0uB/wB8n/CnjU7JulzH+JxT9pHuHJLsWaKgF/Zv926hP0kFSCaJvuyIfowp8yFZjqKOvTmiqEMPEoPqMU+o5DgoffFSUAFJRRTEFFFIaYhTTaWkpiIk/iHoxp1J0lb3ANKMA5IzUybSutRixnD/AFrDvpLa+kePzFfaNssOcEH39q2toLlskMfeuW8Q+D4tZnF3a3j2t0DnzFBKntxgjB/OvHr46pTnapCyNVFWuiu0UMPyRRiKIHCoOiiucurvUILhrgaUWmU+UCLjeqjqTjAI9z06c9M9RFo93pdr+/upNSYAkcJGemMZOc8/z9gKqpbz3N7uNobdowMx4QjHqWA6fQjt61jPHqs+Wm7JFxhbVnFyWdh5jeZOd+Tuz5fXv/HRXXNpeiFiZbW8aQn5j9mfk9+9FcXtX3NOVHJ/GFiPD1hjOTdZwP8AcP8AjXm+gs7MnzNjcDjcfWvRPjGcaPpyc8zk/kv/ANevP/DwysLHHLf1r38vV6KuRjH+8Z1S7xjDuD7NTw8+B+9k/wC+jT0Xr3qYKMZP1r0uVdjhUmMFxdL/AMvEv4nNOF3eD/lu2fcCn7AOMUu3Bpckew+eXcFvrw/8tAT7qKkGo3ag8ofwpNn+c0nlg0vZQ7B7WfclGr3IHKRn8/8AGpF1qbIHkr+Bqt5YPB/lR5Yo9hT7D9tPuXhrj9Db9uzf/Wp415R1t3yPcVn7MCmtEP8AJqfq1PsP6xPuWpvF9lbShJoLnkZyqqR/OtGLxDaPGrbZhkZGVH+Nefa6Nl6mOPkGPzNdBaR7raLPdB/Ko+qU2y/rM0jphr1l6yf98U9dbsT/AMtWHsUP+Fc95IPAFO8n/wCtxR9Sph9bmdENWsW6XCj6jFPGpWbEYuovxYVzPkgcAUnkj86n6jHuNYyXY6tby2I4uIz77xUgliPSRf8AvoVyH2cE5Iqvd2w8jkAc1DwK7lrFvsdyCCOoNKO3tXk/2chbwKxHOTg47Ukf2q2W2Y3U65yWxK3fn1rP6m+5f1pdj1rjjtRjmvJYLvVQsxOoXgLJvUee3HJ96emp61HpzuNUujIVLZaQnHtzS+py7j+tR7Hq9GPWvGx4x8QRGMf2m7fL0KIcn8q0pvGWtrKscVyCAwDFolOfyFS8JMr6zA9S7c0Y+ua4JfFWrqPmMB9zGR/I1XXxzrPnmP7NZtlyB8rDA9T81J4WoNYmB6LTRXFSeM71Uz9kgOOT8xGaLTx3NPEXfTFTnGBPn/2Wl9Wqdh/WKb6na44pOc1yEXj2OW4aP+zZQq9WWUH+gqSDx5ZSytE1ldqy8kjaQf1qfYVOw/bQ7nWGk/SueXxnpxODDdL/AMAU/wDs1IfHGiK215JlwcZMJP8AKk6U10H7WD6nRdTQSeawIvGugTPtW9bcemYH/wAKl/4SvQhw2pwqf9oEfzFL2c+xXPHubXOeKO1ZqeIdGkj3rqdptHUmUAVLHq+myjdHqNqwPcTKf60uWXYfNHuXcn9aUVXF5ayHCXELewkBqVXVujA/Q1NmO6Hg0Umc8nNGcA0WGLnJozmm5zS5Hf8AWgBc8Z70tNzkf40E9qQxaBSE0ZoAcDg0pJxTM/Wkz7GgCTJyaN3FR7uODSk0ASE88Um7mo91NLf407CJt3/1qaXqIuKaxAosFydX5qQPVMPUgfiiwXLO/rQH75/Wq5fApC5pWC5a3+9AfjFVfMwO9HmHHNFguWw9L5nHPNUxL604S4o5R3LRkpnmc+9QGTjmmGUZosFy6JPfip45PlPSs5JM+tTROefTFKwFvzKaWX0B/CoN/uaQygDPX1ouFiwHA5AA+lI1y6naJZsnoqOwJ/I1Wjd5z8gwv989Pw9auwW6J7serHqaLt7CsluXdMW6ku1kluZiq5xGZCynPrmuoByBWHaLsGa2YjmNfbivSoRajqcNZ3kPpM0UldFjEXNGaSigQUtJRRa4GVJb31tcloJfNhYnCNjK+w9RUcuqXFs4S4te3LDI/ofbvWrL0U+hpc15ssFVhJujUa8nqjRTT3RiHxTZIxVo3Axyw5OfpVlvFWjzCILOVZhwNvtnHHetBrK1mRmnt4nDdQUGTUcmi6dJZ7ltEUGTzCF4+bkZ4+prmksSk+azt1KTjdWMo6vpu8oL1JWZSfLQ5475J44p8enw3YaZIVWZjtIB2jA6fXHPbvVS88M6WINzvLDsAJkD4Ax356VlXFjfW1havpFzdCMF1YOCrTbjhccccA85Hr714VWHNJNKx0XaRdmtrOKaSP8AtdItrFfLEiAJg9MbeMUVCuhakVGLfTU4+60SMR7E45PvRWXspdwuzz/4xOBZ6UvTMsn/AKCtcRoCfLBj/np/n+ddh8ZWITSFz/FMf/QK5Pw8Plt88gsD9Oa+wwC/cxMsX/EZ1qIOp65qUDnPr1oQZ6j61IBxmvTOEZg5NOCj0p23HWnY9RwaAEx/OjHGM89aUDIz7U4cCgBmOeKMdqfjJNOA5NFxWI8YphXA9PepSMnpxTSOOfpVCZyniIf6dFx/yz5H4muisx/osPGfkXn8BWB4jGL2L3j7/U10ViM2kByceWvf2pLcHsWQvGMc08qenNIBke/c1IopiGbevrS7e3FSY70bQe1AyPaB25qC7X9wenUVbxnkVDdg+QenUUmNHM3H35kXq7DP4ClnjEltAo6nAGfcVO8Ya5nbsqY/SoBmSSH0QqvHrWVywmVYpmyBjy8Co4kBsAMdjVi4jEl2i9gMn3xVYOUs1RR87ZA/OmmI5G4jZZSxwNzE4/u84A/St+O3WO3t1HUkEnHU1i3Z2uoPTn+fFdAozBbn/d/lSGaL4jjLnGAKq2cQeWWQjk1PJm4nEIPyjlj60tqu2aYAd+cU7gLJEu08DNUI1CWhUD5mY44rXZeOelVbeESPv/gThRSYIjhsxFEBgbjyTjvVeGAC/kOOueK1mXv61UVcXrk9x/SkwIp4/LjPy4PbNZr2obPr1+tacxLsWI46CmLC3lu5JGBx71Ldykita2KLGH43NWRrEWGb6CumhA+zr+P86wddGCT221fQkj02ITaU2eRlu1XYLcR2qrt5PQVFoLA2Mm4Zw54/AVrWqCRzIcEDoPSkMqxaaqKSQN7dTT/smDxxWoVHbOKbt9v/AK1OyFdmPdwyiEFZJBznhiKfbm8WBNt1cAgYz5rA/wA6vXKZgPHTmmwIPIA9M4qeVXHzMiW61IdL+6+nmt/jTxqmsIOL+cn3fP8AOpRGB2PFQy9lUZJ9qHCPYfPLuNk8R6+p2Rag+7PdFP8AMVNfa/4r0/w++qG4VohII9zwLhiew4q3pGn2b3BN7cLEijcc9WPoKd44vY73QpLaN0ZFUFVjPyoB2x0rJwjzWSLU5WvcwLf4ja+0UbN9lbd1JiP9DXUW/irU5I0Z47clgDwpH9a8qtjttozxlW/LmvRbFQ1pEePu1MqUL7Fxqy7myPFF5jm2hP0JFV7nxvcWrqDpquGGciYjH6VEIxjP61Q1G3DbDj1Gaj2MOxftZdy+vxCOPn0lwfacH/2WpB8Q7X/lpp10p9mU1zjW64+70phtVIIAHFHsIB7aZ1I+IOmkfNaXq4/2V/8AiqkXx/oxIDi5T6xf4GuOa0Tg7ec+lRtZoaPYRD28juh450HHN3IvoDA/+FO/4TXw+2M6mi/7yMP5ivPXslP8qztSsNthLID0APQdjTWGiH1iR6unizQXwRq1qM+sgH86tp4g0mQALqdmf+2y/wCNfPmP3HX7rfz/AP1VuW9orwo/qoODSlhorqCxL7HuC6jaSD5LqBvTEin+tSCeM/dcMPUGvExp6sp4HJ6mo308AfKMVH1ddyvrHke4h8nj60hfGM5rw4wToMRyyIe2HIogk1guFhv7xfpMw/rR9W8x/WPI9x8w+tL5nPWvFn1DWrQPnV75mCZ5uGOPzP1r2T4SSrr3hSaTUh9puYrtozJLy2NqkD9TWVWl7OPMXCspO1ibeOgpm/nj9a7g6HprHm1X8GP+NRP4c0xh/wAe5H0dv8a5vaI1ucekgPepklwf8K6M+F9PySDOv0f/AOtUE/heDyX8m6uFfB25KkZ/KpdRFJmG0x3BVyWPQDmporVnIaY9OiDp+PrToIFgXGMt3J6mrK81W5WxIijgYq3CvNV415q7CvIrelT1MZyLsIworTgbMVZydKu2rZUivQgrHDJliikNFaEBRRSUALRRmkqkJiPyh+lIDnHvSnkGmRn5B6ik0CJpXwAgOB3qe1O63kXuOapn9as2LfvGXuV6ViqSjCw76mPq8DOQ+AyKufLPQn1Pr9P/AK2MGae7nLy+cwVG+YnoBzx9c4rqNThaWJlLGNOQSvX/AOtWNaxRXFpLaNiMCRANvfJ4rwqvuuT5dE0dyfuopCe8KgjkEcHc3P6UVea21gsTGyKmflUSAADsMUUvaw7IV2eQ/GRiZ9JUdQsv/slc5oAwlt6bv61t/F2bzrrSDn70Lvj1yVrG8NjEdr3A6V7GC/gxMsV/EkdavHtj9KmAHH6imooxT1GDn+leicKFxxRjHvTufWgZx9KBjQBmnBR6daXHNGOKQxCBnqetLjOD607b2HWgLzTQhCM0w9s9DUuPpTGHpTJZyniMYvIjx/q/6mujsB/ocHT/AFa9PpXNeJm23sJ/6Z/1NdPpw3WFuf8Apkv8qS3B7Foenan4+uaaFI4z+VPA49cUwsHIzR1FOx1PB9KNuRx0pXHYTAbtUN0B9mP4VY9sfhUN2ubcnGOQf1pPYZgthZLkngAD+VRlRFbwE4GW3GlnDNLKnY/MeOwHSnTKrRW8fQtj8sVkaWK8YLTl2z86k49B2pkEQETS46qQvsKtTMI7hiSMLHio0AGnnjHymmhHH6iux075zmt2ED+z4GPQKOv0rC1RHW4V2ztYjaPbHX8a1rMvLZL1CRoB+NDEjatIike5vvvyaLcf6VMKmhJMSNzyBUO8Q3UhPHy5/lSbGLOxdxCnU/eIHakswQsg9GqS2jJBlcfM9JaH55R70rjJdueDVOfakjAck9fYVclk8tCf4jWexyx9TSkxpDVjMjgD+VW5IwkBHoKfBD5aZI+c0s+PKY9OOKEJlW3H+jDI9axdcXKnjOUrbg/1HJ9awdek2ywxryWyD9Ku+hNiHQAziVBwobcfy/8ArVvaeflbjnisTw02XuUz2U/zrcseZH60uoy6BznFKRk04Dj2/nQeOcc1VySvKuY2+maitV+U9OuatOMofU1BGDDHlgcnnHpUNlJBI20YHJPTFIsQU5Y5Y1IkZ5dx82Pyp5AJzjNNAyFhVLUY99hcL2KH+VaJHyjPSq1wu+J1xwQRVXEefWozanHUPnp7V6Lo7B9OhbjG3v2rzm2zsmRuxH9a7/w44fSo+nHFZSNYG0Bxz0qrfp+6BHUNzirQ/wAiq96pNufYg1BZmFecnp3ppUZ44OKk56/1pvOOvJpk2I2GevSmFOPXJqU/mBTT/nvTuBXZevb6VVu4t9lOg/ijbH5Veb6VEVBOMcHiqTJZwQ5gkHuD/n866PS/nsYWxggY/XFc8g2iZOny/wAq3dEbNmAeqsRjNOWwkagTI6Ch09PrzUsalzgDmr0Vrt5cc9ayuWkUIrJpW+YYX1q2IFjXCgCrm0AYH6VG64BPb1ouVaxganHiXp95P/rV6N8B7rNnrVqTyrxSgZ9QwP8AIVwGpphYyB1DD+VdR8D7gxeLdQtScCa1bA9Srr/QmssQr0mODtNHvZOcGg0g6Y9KWvIudYzvSHkGg8GkNQ2Uc3ewmO6ce+R+NRqvNampxZKSf8BNUAuCPfmuyjrFBJkkYq7CMCq0a1bjGBXfTics5Fhat2xw2KqLU8JxIK6EYMu5opKKskWikzRmmJhmlpKKYgzUa8Fh71JUfSQ+4oaEKaltW23Ke/FRGhW2uG9DmpktBk9+uRIvfrXD3MLPe7SSo7kdQPb1Nd7fKqgzEgLs5J9K4q5R5LlpJI9kefkQjlh6sD/L8/Q+XVdp27nZSd4Dc3X/ACz09Cn8JMRYkdue/wBaKYwYsSZHJJ9aKy9kyrM8i+Lv7vVdOjBO1YG4x/tHH8qy9EuVggtWMbMFQZx3rQ+MDA+IrJT2tv8A2dqy9MAFnBnBOzr+Jrqw8uWhFk11erI6hNahxkwzDv0H+NTDWbYgfJL/AN8//XrDXnp9KlH6YzW31iZl7GJuDV7U9RIP+AUv9qWmc72BH+yaxDx3pVPbNH1iQexibn9p2ZH+t59wRTxqFpx++A/AisIdMjrR37U/rEg9ijoBfWp/5eEH1NPN5anj7RF9N1c7wB0ppAzjGKf1l9ifYo6Q3MH/AD2i+u8U0zxEYEiH/gQrmjg88flULgDnj8qpYh9hOiu5F4slRb6Ahs/uzyOe9dZpbqdNtiGGTEv8hXm2suEuY+3y1vaaF+xwHGPkH8qp17K9iVSvodyF9P51IFPrXJqfQkfialUuD/rJPwY0vrK7FKh5nUgHrjn3pCMgY9ea5pZJs/6+X/vs1ILi4B/18v4tR9ZXYPYvudFg96iu1/cH2IrGW7uf+fiT86Gurhkw07EehxQ8RHsHsGRsuXu2I5AA/SoEBeaGQ/dBCqPoKny5LfPy3B4600bxtAbhORxWftUV7NjLiPzLph1Crk1FKfL00Kf4kIHPepxu3M+RuYYJx1qtPGSgU4wowOKaqoXs2c3ri7Xt8enX8K1bBAdLTAHK8/WsfxA7i4jDFdoUYGK09JkdrCMDG3BGMVbmrEqGtjZsyDbRn29abIA90sfQdz602BniiCKFwvrSpGUkWQHJBzz3qXURSgy5joB17VWtyFuZgT71J5rcZQc+9REEs7AYLe9J1ECgyKeUu2c8dqbCB56cf55pxhJOccd6eR+9V1XAUYxUqauPlLYHI7daimJ8p8eh6UpnGMbWpjSqQRtbnPatOdEcrKqy+Xa+jZwPesrUYCsPmONzkH8K1QF8zc4JUD5RiqGs3MSwAnPXHK+1OMkJxZleGW/0ycZPKf1/+vXQWX/HzIoJ/H61y/h6QR6s+ehUjp7g109uwW5LfwnPNNtIFE1OR/hSZ4xmmCVDyD+GKPMQDr09KOZC5WKx2g889KrwtvnO7qBT12u5Z3AC9BUUOFuskYXkZJqeYqxsWOnvdAuwKxDuB94+grVGnIBvZOeylD0p9vLarFGxki2gYVcjn681R1HVFkDRW7jn7zjj8B7VnzOTsXZJFPUGgRzHCEb+84HH4VlSLuUgg1YbDDsKhZeuea3jZGLPO9nl3t3GR0JA46c12XhZy2nnnO1v8a5K/URa7dj1JP8AI10nhCTMMq56H/D/ABqZFwOr71Fcgm2fnoM09TikmBMDjGeCOKg0MnnB9qQ9fYUdevSgn8qLiGkGmHvwSKcf/rUh75P9KAImHH+NRnjnP5VMRxjH5VAeuRxTTFY4yaLZqdxH0+ZwM1qeHYmkjlQZ4YH86i1K0K6rPIeFBDAeuQM1a8NNtubiPIyVB59v/wBdVKWgorU6aCJYx8o59anGAP61GuAakHB96wubLQXtUcigjrz3qXIxg9PrTGOeKaYmY+qL+5BGeG5/KrfwvuPsfxNshnCzb4z/AMCQ/wBQKh1Bc2zjv161n+H7j7F440W4yFAuoST7bwD+lOSvBonZo+qDw31pe1EnYijORXiHZ0Iz1pD0pW600nAznArNlIhuo/Mt3HcciskCtW1vba8lmjgkEnl43EDjnPQ9+lUHj8uVk9DXdhdrE1NBI16cVaSokFSpXpQRyTJl7VKhw1QipVPIrYyL1FNQ/IPpS5qkSLmikoqhC0UUU0Ji1G/31980/NMl4UH0IoYhTTTTj0pppDNFsSWsTEZwP/rVh3cCtdAFFYsQBuOK2rZt1lj+6cVlaomORXmYuN2deHfQQ6hZIdggiIXjOOv6UVjhJMD5M/nRXF9XX8xryI8L+LrhvFEC9harx/wJ6z9NnhSzhUyICEGcsKufFaT/AIq9RkjFqgOPQlq4piRI2DwAOa9GjG9GK8jOs7VZHcpNCQMzR/8AfYqZJoiOJFI+tcADjjP4U7cT3xjtVeyJ9oegBl45H509WGe1ee+a27KknHpTvPk6iR1z6MaXsg9oehD6Zpe4rz77VOowJpPYhjTxe3IU4nlx1++aPZsPaHf5Hemnt6muDGo3isCLmYen7w07+1L0Di6mI/3jR7Nhzo7Yn/IqJzlcAfhXH/2rfZH+lSdu9KdXvQeblz9QP8Kagxc6J9eP+kxeu3ofrXRaXzYwdT8g/lXF3N1JclWmfc44yRV2DWbu3hRI3G1RgAqDgVUotpIUXZncJUy+mOO3NcSviO/Gf3sZ4/uCnDxPqC/MTF+K/wD16z9my1NHa9sU4+o/UdK4seKb4Y3LB+Kn/GnjxbejgxQH2wf8aPZyHzo7IHBHH6U7P41x48XXRwfIhPr1/wAad/wls2ObWMn2Y1PJIOdHWkgDGaM5A6+tcp/wl8ne0Q/Rz/hTh4t6k2g44/1n/wBanySByR0+7I9aic+lc6PFy4/49D/38/8ArUp8Uwtw1tIM+4NPlkLmRV8RsVnjY88Vd0Ns6dF17/zNYer6kl+VaNCMDByas6Xq0VtbCOQMSCeg96tp8pKa5jrY2AGKlz3rCTXrUcNv/Kpl160PV2HsVrOzLujYyM8A0bvSsga7ZMB+8Iz6qaX+3LM/8tfzBpWYXRqlunIxRkdulZY1qzJ/1w/EGnf2vZ8ZmUfnRZhoaWRznA/rTSe1Z/8Aa1mQR568fhQdVszj/SE/OjULouOeKydYG6yPHRgastqVoc/v4/8AvqqGo3cMltIFlRiOwIqo3uSzF0ZtusLweQw/SuziI4HT61w2mPt1iJmYbdx5J9q7OKQHHI6+tXPcmGxfB7nNBHHPFRq3HBBz75pc547+lTcsdn+VJwM8A0mc0h6UrisOJX0FNIB7Um4delIDTTGIQmTx9ajZVHf24NSMT1/SoZG46002TY5HWkSLVJSAQSAePcVZ8NyNicKzLgjkHHr/AIVH4hGL1CehQfzNReG3xdzoTjKg/kf/AK9a/ZIW510csuR++kxjPJqUSysMeaxB681VjP8AKpwc9xnNZ3NACAcZORR5QHHSnL+eKM/jSuFiMxZH3qQxlT94496mOcDPNIev+TRcLFfyuPf6U6O3GeeakJA6/jTkyM880OQJHPa9Ftutw6tF/iKz9Ck26p7OrD+Rra19QTA5GRyv8v8AGue0tymqQMT3x19aqOqJa1OzBx0P5VKv0/Oq6tx7VODzzjmszQeDjp+NMZuOn60pYZHb8ajY8HP40XCxVuzuikX1Q1y1zMY57aVW5Q5B/KuslUE7T0PFcffLtiQtn5Wwf8/561pF30M5I+vYJlubOKdPuyorg+xGach4IrmvC2t2kfw+0fULy5jhiFpGjO7fxKNpHucg1h6j8RbKR3is5Wjj/wCehQ7j9PT+f0rxnCXM0kdkNUdlqGq29iMMd8vaNOv4+lcpqWszXmRI22M9Il6fj6/yrmJfEto5OJTycknOSfeqkmu278+bx3pqjLsdEeSPU7rwjd+Zq08JOQ8W78iP8a6O+jxMrjow/UV534N1eB/FFrEsgJlDJ/46T/SvT72PfBu7qc100Vyy1MKzTehQQdKkApiipBXoxOKQ5etSCo8U4VqZsuRHKfjUlQwH5CKe7rGpZ2CqOSTwBVrYgfR3qKK4hnz5Uiv/ALppzSohAZ1Unpk4pgPozSe/agMD0IP0NMQ6mSjMTfSnUHkY9aYhqnKAjuKaabF/qgO44pxqRl7TmyJE9QDUVzGpXc/IXkg96LBttyAf4gRUt0H2soQH1+bFcONXu3NaT1MVruTccPIBns2P6UU4xS5+8B7Bun6UV43NDzO/lifNnxQfd40cdlgjH9f61xzDErZwK6z4ltnxrc9OI4x/44K5UfMzA859K9uhpSj6HLW/iMASDQfqadtBIpcdBjr3PetTMbuPt0pDnOfWnlcc5xmjHzZ7mkAwHHr+PFGOOAak2Z+7/KgLz60XAjz0xgZ9aXtjnFPVSc85BOMYpAvTIPv7UXAjJ5OOtGecgAZ7etP24GD+RFIVA5yOO1O4rEbcc5HXHXpUoPyj2GOtLb2st7dQ2ttHvnlcIiA9WPQV06/DvxPgZ0zBHX98n/xVKU4x3Y4xb2OW3YHH5005J6V1q/DjxQyknTlx6+fH/wDFUH4a+KWP/IPUD1M8f+NR7WHcv2c+xyeec8AijP4jtXXp8L/Fbgj7BGD/ANd0/wAaU/DDxWQB9gT/AL/p/jR7an3D2c+xyKybCSQD9RTS5Jz0rqn+HHieJyp0/JPYOpx+tIPh34oYhRpp3HuZFx/On7SHcXJLscvu57fhQXH4d66d/h74nViraa3B7Op/rTD4A8TL/wAwyT/vpf8AGj2kO4ckuxzSt2z+NAcZBx165rov+EC8TAZOlSe/zL6/WkbwJ4kGf+JVJzjoV/xp88O4uWXY5tiQPb+dPjY7SQeK328CeJW+7pUv/fS/40+PwF4oyd2lTcejL/jT9pC24ckr7GBvxznjuKBIQO4xx1/z610f/CCeJSD/AMSmbp/eH+NNPgPxJk/8Sqbkccr/AI1HtIdx8kuxz289z9OKUSFV684rfbwJ4kI50qbP1H+NJ/wgnib/AKBMx98j/Gnzw7i5ZdjA83pwc4xilEpC56Gt8eA/E/I/suX/AL6X/GlHgLxNg/8AErk/FlP9aOeHcfLLsc9vOeaaZDjnHPJxXSp8P/E7kgaVIOOcuo/rUTeBPEm7H9lzf99D/Gjnh3Dkl2Of80sfx+lIZRwM8jv610B8CeJhx/ZUxweuR/jSf8IJ4mOP+JVNz05H+NP2lPuLkl2OZVyHHrmnGQ9Mmug/4QPxP0/sqYY9x/jTv+EB8TY/5BUn5j/Gn7Sn3FyT7HPCZxyHI/HFO+1TqQPNfj/ardPgLxN/0CpRx6j/ABo/4QLxMTj+y5c9eo/xp+0p9w5J9jFXULoLxczc+jn/ABpw1O9A/wCPufA7eYa2G8B+J++lS/gw/wAaT/hAfE+f+QVJ+Y/xo56fdByT7GQNVvVb/j7uM56GQ1IusXwGPtcvt85rT/4QLxPnH9ly/XI/xpf+EB8TD/mGSf8Afa/40c9Pug5Z9jN/tvUOn2uT8e9OOt3/AEN05x64P9K0l+H/AIpbONLf/vtf8ac3w+8UI206Y+R6Op/XNLmp90Pln2MC6vri5IM8hZhwCQOn4U20vJrOcyRNhiCDxnit9vh94nBUf2Y6hjgEuMfzqVfht4oLAfYBz/01Uf1p+0p9xezn2M2PxHeoP+WTY65X/wCvUw8UXg48uEn/AHT/AI1bb4ceKR/zDh9fNT/Goj8P/E69dN4/66p/jS5qb6j5anYaPFVwOsEP05/xqRfFUmebVOP9o0f8K+8UZx/Z/PtKn+NQ3/gvxBpllJe3enmO3iALv5iNgZA7Glem+o7TXQtL4qYfK1shPs//ANan/wDCTqRzafk//wBauTDds9qcJD2xmn7NE87OsXxNEcZgcZHZs1NDr8DSY8mQenIrjw5zz+VWbaU+cOvPH6VMoaFKR0Gq6hFdQoqo4YHOD9KwI5PLvUfHCyZOOe9W5GyvTPpms2UkSv8AUdKVMJM7BNSgx1YfhUyapbYA3N/3zXNo3yZGee9TbuAeMVm0UmdF/aVtgHf39DSG/t848zr7GueD4XuD/Oq09/FBxnc3oMUKLew3Kx0sl3b7c+aAPWuS1S9haaWKI7gz/eHTr2qlcX01wTlgqk52jpVU9a6YU7bmM532N/S7jXtVRLG0u2aK1Q+XE7DagLZOAfc1pPo/ilQSTE3/AABf8K5W3u57KdpLeRo2PGVPatFPFGsIu37a5H+0Af6Upwlf3bFQnC3vXLstr4kjPzRr9Qif4VXLa2n3ox+KrUDeJNTcfNcZ/wCACoW1q8cYMg/75FJRn1SByj0bOj8Jalqdp4x0aWZNsYvIlc4/hLAH9DX1Yyh1KnoRivjCHWLmOeOXeco4YY9jX2bbTrdWsNwnKyorr9CM1z14tNOxpSle6MwLglT1FPFS3SbZyex5qKt4O6uRLcUCnCkHWlrUzZPAe1Vtatp7rT2jtwDJuBwTjIzU8R+cVLPJ5MDyYztUnH0q1sR1OehgOn6hCqOxdgpkQtu2k9ce1T3llcT3Vwzs8SAFllGCOnAwar6NLFNK95dSr5hOQDV/V7gsI4I2+V/mLZ4x2pK1inuQJcyto8nzEbXUZ9jjiq9u0lpdQzeaWSX+HA4wcEe9afkRDSpYIZEdthJ2kEk1kuwksoSDkrIR+BGf6VRJYvvtE97OkcgV05QEZHAzitTSblri0+dizKep7is+V9uqQSdBIik/yP8ASptFJjmmhPbgfgcUxGoOGce+aDyaDxMfcUGgCSBtlwjdgwrVlTLH3FYtbhO9Eb1GawrxvFlRdmZTRfMfrRV0xjJ4orxeQ6faHyP8RufG97jP3Yxj/gC1zeCXbgn5utdT4/iabxpevEu9D5eGXkcIvpWLiGMvvQh93rXqUnanFeRNTWbKew4zjPtSgZ4P5Y9qurdQINoTGfcUCWy67G/w/Wq1JKYQnHXigIQQcE1cEtmflUNycDk00SWhIxkA8UaiK2w5G0/pSgZyOg71o2tvHdIzp0BxknvU/wDZygHp+JqeYqxjleTxz0pPL7dxW0umqD8wwM4+8KeNLhPBzjt8wo5h8pglQeg6DpSMhXqD/Sug/s2DByzc+4xTG0q2Yf61h+VCmHIyt4TbHjHQ845vY/8A0IV9WBI2wSi5x1xXy9otutr420NFYsPtkRycdd4r6eD4Ue1ZV1zNM0pOyY7yogf9Wn/fNKY4j/Av5CmtOpG3bz/eppc1j7M15x3lxZ/1a/lS+VEesa/lUfnBOSCfpTRLnmj2Yc4phj37gi5+lKY0/uj8qTfSbqagHMSCOPH3R+VOEcf91fyqIOPWnb6OQOYf5af3R+VN8qM/wj8qN+aTzKOQOYcIowfuj8qcET0H5VHvpd9HIHMPCIB90flQUT+6PypnmUhejkDmHbFB6D8qCqf3R+VMLn1NJvp+zFzD/Lj/ALg/KmlE/uj8qbvpN9HIHOO2J/dX8qjkRNv3V6elBemSOMelKVPQamORU3H5R0pCqbVG1fyqNHGR7ikD8is/Zl85J5abD8q/lTPKTZnYMn2pDL8nPrQz/u/ej2Yc5II48fcX8qYY0JPyjJ9qTzPlzTRJl+e1P2Yc4540AHyj8qUxRkfcX8qZI+QPrS+YCAaapicyNoo/MHyDp2FMeJGOAq/lTnk+bPHApitxk5yTT9mLnI3hjUgBVx9Kq6leWenWpknwgPyqQmT+FWLqQQxGRs4VSxx7Vy2k66dS1BJ5OoXft3cKD0FXGlcTqGsLyW+YjT9Jnu2XpJKxRF9hk8/gMU28fXbKDedDWQdT5bq2B9MZrsbOePylVFHTt3qW5mKJyACe1WqMbD523Y5HT9QtdSjO2IJMo+eNgAR+FTPbxscCNffjpVm9trcM12sCpLjJlTqcdjTIvmQOP4uazUE3Yc20QrCgfBVevpWf4rtY7jwrqsWxf+PSQgbR1Ckj+Vajy4YqPp265zn19qi1FBPp1zFjPmROv5qRVxp2Zm53R8sYGf8AClGO5xjilKEZ/XNLtLNwO3YV3HGABzwRx+tSwfLKp9xyKaqnrj86eAByADzmpY0aDkYwep55FUbknzgfb061oOBtGOh68Vn3vRSPT/P9azp7ly2LsBJiQj0HINNmnSJNznHp6ms8XpihVFUZHc1VdmdtzHcferVO71Icy1cag8o2x/Iv61Sz70UVqklsQ22FIaWlamIQ0lL2BoIxQAlFFFABX2D4Avf7Q8AaFcZyTZohPug2n9Vr4+r6f+CN79q+G8ER62txLD+u/wD9nrnxKvFM1ovU7y8TKBvQ4qlWpKm+Jl9RWZ3qaL0NJrUWlBpKWt0YjlOGzVllDoVYZVhgiq3selWIzmJCepUZqkQzOXw/YJ9xZV/3ZW/xqe40uC4WNS0i+WNoKsRxV2iqshXZStNMjtJDIskjEjGGOagbQofOLxzSIhOfLGCB9MjIrUpadkF2UrzTI7uKNQ7RvH9x19PQ+opbDT/se5mk8xzxnGKuUoNOwrkcgxIh+opSKJfug+hBpTSAbWvaNvs09uKyDWlpzZgdfQ5qJq6GibFFPxRXD7JGlz4cRzGqgMR9DQXbcQwIIJzwOKigDMiDPPtV2HRNUup2EMeWAzy4rsUbsnmsVxKf9rnpTllx/CRz3FXf+EV1wD/UAD/rov8AjUi+FdbJI8lOn/PRar2b7C513M4S5A+X/wAdpfMzwFOf92r3/CLa4Bxbrz6Ov+NIPDeuKP8Aj34/31/xpezYc6KqzOBsUuB1wq9fU8fhThcz5wHm9OCa3NM8NX7wubm1y4bj5x6exq+3hmcEFLMj2D/T3qHFp7Fp36nKrdXLOMPLyMd6mWS+I+UTfgp5rpE8N3SsCbV15/vj/GpRpF7GSEtW46dDUO/RFxt1ZzAGqsOI7kj/AHSf6UjnVYhl4bkZ6kxnn9K6wW1+gINoT/wH/Comh1DdhreQZ5+4QKlOXY0tDuY/hqS4fxdpAlDgi8iHzLj+MV9Ng8V86WEMkXjHR2ljZCbuLqMfxivoodBxzRNXsKDtcUmk8wCmnOOBUZyTSURORJlSc0uVxUADMeBk0bucGq5Bc5PlaXK/SoA1Juo5A5yxvWjzF9arbutG7mnyIXOWi60b1qru/wDr0bsfSjkDnLW9aN61V3cUu+jkDnZZ3rSF1qvu96Tfk0ciDnZZ3rmk3r6GoN31pC9HIHOyxvX0pN61XLe9BanyC5yxuX0qOV129Kj3n1qOV8gUOA1MkEibl4/hpodNucd6r78BW/2TQG/cn61HIh87J/MX0pTInl9O1Vt2dx/Cjd+7x6ijkQ+dlhpUCYxjNIkibuh5FVWYk+3rTgwWUfTFCgHOyzNIu3pThIpA44qpK3yU7fhc+gp8iJ52SPIpIXb1pwZBjjgdqps/II6Z9am30+RBzsfOUcAFRg8GvNoraeDxBb6bDBNCPNJZnTaSgPBGRzmvRJG4BAHHY1Unso4pEvowi28IwEUcuzH5iT6jg/hUy0RrS956l+eOYWojg34bhismw4/3u3865rRtOnOtSHZDGm75zHcPMSvuzd66h71LS3eQsGCLnav8R7ViPrRSN3j1CwinzxAyhlA+gIOfesU0dai9ypqkN5FrckWZmVDgGOfbhTyCV6HniugjISBEGPlUCueXWZr2fzrqO1LghcRHOPfNdCqkoG74FOjZtmeKTikV5Tlzjil6jB78VIYGfc6cqo55/wA+lJj5PwrblORM8Qm0vQ4ruaJ2td6SMrAyAEEEj1pV07QO32UjrxL/APXrP8S+GJm8T6myMQr3MjKNhPU5/rWSfDFznhuv+yahxj/MaqUv5Tqho+glRxBtPTE3X9alTQtCckAR8df3xrj/APhGLofxj/vk/wCFMbw5dr/EPyNLkX84+d/yHdnSNExgAE4xxKTXmk3myXLx5LFSQAPb/wDVVx9AvFGcrj8f8KzJY3gnaJ/vKcGtaNNRvZ3Mq03K3u2NW1sraXTpPNt7gXe/CNn5NvuOvrQdKjwRmQH6j/CqdhY3N95nkMP3YyctirH9i33H7xf++jVt2e5mtVsbumaDod7LHbsb4TMvJyoXIHOODW2fh/oythrm5U/9dF4/8drio9J1RX3RS7W9RIRU62GujG28kHGP9eRisZKTekzeEopawOw/4V5o7Di8uh/wJP8ACsjxR4LsdF0Y3lrdTSOrqpVypGD9B9Ky/sniQdL6fj/p5PH60k1h4iuYDBNdyywsclGuCRn6E0oqaabnoVJwcWlDU5zHy/jXqOn/AAu0++0+2uxq0w8+JZNojXjIBx1rhh4Z1TBHlrjP98VrQQeLreBIYb6dY0G1EE/AHoParrScl7k0jOjFRb543Omk+EluPuapMT/1xB/rUDfCOTkjUZAPU23/ANesR7nxvZwyTfb7oIiZZhN0A61mt458TM2W1i4Y+5B/pWMYYh/DNM1lPDreDR05+Etx/DqOc+tuf8a9Y+Emg3HhrSdQ0+abzVadZlOwrjK4PX/dFeNab4g8f6lb/aLG6nli3FdwjjxkduRXp3wo1DxbNrl5D4iMjW7W+YiyIMOGH90ehP5VEvbXtOaY/wB01eEWj1ys2dNkzDtmtKql2n3W/A1rSdmZTWhVxmnCiiupGItTxnKfiagqWI/KRVIhktFFANWSLRSUUxDqUU3pS0wElGY29cUincit6inmo4+I9voSKTADV3TGxK6+ozVM1PYttul9+KmWw0aeDmipNtFc/sx3Phi2X5ohkfeGB+Nd3og33koGANmefqK4vyHtrxYpUaOWNwrKy4IOe/pXcaEo8+ViRwoGMe9dNPciextFCo+8CDxgGmjhs9Oc1Lgc5YfjRtHJ3D866DAiIOcY/WjrjipWVeDuH+NMKj1H4UAWbPGw+zCrJIJ/lUFooKMM55FWShGOKzluax2Gd8/570hyDkYzTyMgDFNOAen+f8ipKIznGaRueOMDtT9pK8g9cc0yRxHGzuflAyaQzm9ROPGvh4cZ+0xk4/3xXuykEV4HOr3XjTQyTsaS5jG4/wAPzjFe4rHcAf67r/s1zVHdm8NEXTKxjMYxtPWmFeCKr+VOf+W36U/ypsH98fyqUDHrK0GSuNx45FVtxJJJ5NMkhn3czHGfSomik/57tWiM2WN/vRvyODXKeJ9UutGa0ZLlwshYMMDnGPb3rBPjG9z/AK4kd+n+FFxpHo5fk0F686Xxfc952z9B/hS/8Jddf893x9BSuB6Jvo3jHWvO/wDhLLnj/SJfTjFH/CU3P/PxKKOYLHonmYo8z6V53/wlN13uJfzpyeJLp/8Al5l+u7pRzofKz0LfSF689fxPPk4uJv8AvqmjxJcHn7RN+LGlzoOU9F38UheuHg1yR8bppv8Avsirg1MuARPL/wB/DUuqkUqbZ1u/PSkL1zEl6yxqfOk5H/PQ1GLtmVj58/H/AE0NT7dFexZ1W/nGKbI3y5rk2uXfhbqVTjjMjEfzpkF1Luw1zK5H+22PyzSeIXYaoPudKXJRfypd58tuvWueW6fGfMbk4yXNMub5kYgSt17Oaj6xHsV7BnSK5O7r1oVm8scGuT/tJgf9ZIf+BmnrqZK8Fj77jT+sLsL2LOnUsX5HA60rORKCAcVyU2pkDBZj9GqodSbPLt9Sx4oVddhOkzuJZDs7/hTTIW2oB27VxB1N/wC83/fdRnU2xncT/wACq/aoXszupHI2ipd59D+VeenU2yPr2NR/2r1GV6/3u/4U1UF7M9Cmf5QMH6VheLr7ULHw5Je6d5onjKqwU/KUbg7l6EDrzXMf2qWA2FMA8nOc1W07VTP4nsoLiPNuzunzE4LlGwfbB/ofprTXPJIl+71NnRvGkKSpp2py7hkRw3UqCMS4HRlBIVvxIPseK7JILeSB5bcWyyNks6HYxP1HU1474i0B7JftAWS6sJm+4+A8LZOcuePxPX19ct9cuINIll0zUTaCGTy/ss9yHcjpkKVPH0bjHSnUw8ZXlB/I1hinC0Zo9M8Q6hYeGLWbUnSGS9kO2MMxZpG7AknOK7PTbmS80m2uJ4BbyzRK7xE52EjOM1856bfvNrUWpa002pwWkqlwZTgjPGMjpnGRxXt2meO/DV+FRpms2PQXSBVP/AhkY+pFZQpciCtX9q/I6CSRkwsbMMjBx3qPcdmKe1pHNGGhVHRhuBXGCPXPTFVltEBIKAEHBBFUzJHF66Fj1y5wAckEexwKoYI+YqME8ZNbXiO2EerjaAoaJT2HqP6VlLgdF3dgT0FebU0kz06SvBCLtCdCDjpUqqzfM4IA4BJ/lSLEW9efXpU4gZsKCXPsOKybN0iPfGCp2MQvqePrivKfFyY8TXTdd+1s+vyivZoLWSH5wqBhzucBsfnxXmHxJjf/AISGGduTJAuTjGSCR/hXTg5/vLHNjIfu7h8OU8/V76zzgS2x7+hH+Nd//ZDrPHKkiEqwbAXjO0Aj+dec/D9tvjOCPr5yOv8A47u/pXt8FiXYfLkDrWeOly1B4OCnT1Maw08R72nij8x5C25RnAwBjmtODS7aRG3QoOOXZRj8PWtRLQn5VACnsvQ/41bSzRAN2Sf7orzZVjuUIx0MZNFtJDxAhGeWMYqdNEskbBtoHOOMwittYScBQRjpnrVmK0I9j+prL2smJyitzGi0OwyWksoST2Ma1ai0OwPSxtsnqTCv+Fb9tpbyHO3j16VLPPpul/6+UPL2jXk1rGjUkrt2RyzxEb2irswJPCVlfWc9v9hgPmxsnEYHUHvXyrq+kT6Pqs9lcgLJE5jYA5wQcGvsX7Rq2ppts4BZwHjfJwfwH/6q+UfHttNYeM9RtbiTzJYrhgXP8XPB/EV62AjyNpXaOKvJyV5Wueo/CDULzWfCdxotppNqRbzDzLjaASG5+Y85PB/SvUtH8JNpUjXs90zzYPyINqDPt3ryj9nLUhFc69p5I+ZYpl/Asp/mK92muNylaK0acKjb3JjVny8sditUc67om9uakpOtaxKZm0tK67XIPY03rXWjnYo6VLEecVHToz8wqkQyejNJmjNaEi5pabmlBoAWlBptKDTEOzUUZxJKvuD+YqTNR9Ln2Zf5H/69DAc1PgbbMp9DmmGli+/SYHQUVCko2Ln0orMZ8RySySgTSyM8hAZmc5Jx7/Suz8NOJInlH8SKa4x1/wBFGMZ2c/lXT+DZN+mMc8ghP8/nWlPcmex1O4+uKQt7cZ+lR5OcDrSE5GfTjHrXQYWJlbKkehz0pueuO9RglSeOvFB4buaQy/YyYdkx15H4f/rq6WHTuB1rFhk8qVG9D6VrjpgHI9fXpWcjSGw9WAPTAphwc4HvS8FR/Kg9ep5qSxB15qtf5+yEY/iB/WrBGMc8VXvMG1b04/nSexS3OZL7fGegt6XUf/oa174or5/m/wCRt0Pn/l5j7f7Yr6CUHFc8jRDD8uAFyfX0pecGnlaCODQhMiRow375Sy+gqqRycDjPFTkcmmEVoQ2cH8Rbd54tOVWI/ePnH0FcQNKnOMvJ+letaxYLf3NnG3RSxH5Cki8MxFg23j2FHJcXNY8qGlSn+OT8Mf4VKNLkA+9J7f5xXrJ8Lw8YU8e1PXwxEP4Kfsw5zyUaXJ1zJ+OP8Kd/ZcvpJ+f/ANavXV8NQA5CU7/hG7f+5S9kx+0PJYtKkZhnzfzqzNo0kTFY1kK47t1r1NfDkA/gq03h+3PzFB0qXSdylUR4u2mTDqkn51LFpMhYfI/5162+gQZP7uhdAhH8FP2LF7RHAWHh+OQAusv/AH2RW0nhuzOB5Unv++b/ABrsbfSI06LirX9nJn7v6VhKhK+5qqyOPbQLFFQeUzd+Zn/xpr6HZrExEEgI4x5r4/nXWz6axA2r2xkV4bqdl8UTqV0tquoG3ErbCNmMZOMcZrP6vLuX7ZHdHRYd5KoyjH/PRv8AGq/9jxliQDn2dv8AGuFXSviVKy+amqnb833kH+e9TvpvjtruB1ttVt0Vhv33O4Nz3z2qXhn3LVddjsU0yJcfL0PA3Hj9ajvNNUNjbkf7xrpEgJjO5RnpTbu3AzlQAKx9i7mvtFY5JdODNjy8/wDAjVpNJhKD9znHua3FthuOAMhatJbjyxx7UpUpAqiOOudKUcCIA+9Z/wDZq7gDEvJ/u121zbAyfhVFbQSXWRyF561UKUiXNXMIaLDtG6FCT/s0f2JD/wA8UyO+K6pbbdMMjAA4qlrkv9maRdXS43RxnBx/EeB+taKmxOSscbeSaXYuY2eHzAeUABOe/NY1zq9oMmKGPjjIQHFcpe3bCVmLEkccnrn/APXSI/mRknP064+ldsKCW5ySrNmrcaxKzrgoVBIIPB/PtTLW9dNWs58hTHOmD2wG5/n+orNVnjbg8etPC7AWyQ2Mgjgj0/pXRBKLTRk5NnsWoy6fp93Ml5eWqRyqGkhldd31C9efpXFXPhPR7tZdQtEzBk4WGXMZI64x/LIrTsdFtbmyR1R2MnzOzgBiT1zzk/jTk0q60ouNOZYzccSKMKvH8W37uR2wB1712VsDKcvaLS5UKy5eV6nLWtrZ6ZFc2t3a3DSXabWjhjBESngEnPUdcAc1y5lmgDwrJu2sVztIPHfkZ/OvUodHMJJlyTnLbmyST3JP0rzvxDZGy1m5QBtrnzF3c9ef6VnXoOEEyea7LGieK9X0GXdY3bxp0aLrG31Q8fpXpvhj4oWur30VlqUQhu532pLGcozH1zyuT9a8Pc4Qc/Wrugbn8R6aBkk3UXA/3hXE0UmfQHiaBGvbeRyOYyMepB/+vWOttkDJVU6DH+ea6bxDAzi3cLnDMMk8dv8ACsWK3LHAG4+grxsQ7TZ7WGV6aI4rdekYBbHcZNW4rcx/IdpPXJJwBViK2YLy+D02r0/Gr9vZlzkgkexrjlOx1qJSW0DHG4Mem5s/oK84+Len+Q+lXI5V1kjzj0IP9a9ijs9xAC4x6Vwnxk04Hw1ZXOSTFc7SR2DKf8BWmDq/v0YYtJ0mjyzwXJ5PjrR2Y8POq/8AfXy/1r6YjtRtAHTrya+X9HY23iHS7nB/dXEbn6Bga+u4LFnxwcVvmcXKceU4sLV9nB3M5LUHoTj2qzFY5wABitlLBI13SsFUeppBdxqdlnCZn/vdh+NckcI1rUdhyxTfwleHSzjLAKvqac1zZ2reXAhuZv7qDNTmxuLvm7mIX/nmnA/GrcNvBaptiQKK7aeGtrFW83v9xzSq3+J3MxrXU9R4mmFrB/cj5Y/jVmz0ew08744gZT1kflj+NW3mA71VkuPerlKlT13fmTzTkrLRFp5go4r5M+NFt9n+JWouB8sojlH4oP6g19QST+9fO/x5t8eKLG6A+WW0Ck+6s39CKeGxLnWsxypcsGyh8Er8WfxE8rOBdWskf8n/APZa+jZJ+etfJngC9Nh4+0SccZuFjP0b5f619SPJx1rnzRuNRNdUb4SPNE2VO5Vb1GaWoLR99rGfbFTV103eKZMlZ2KlymH3etQ+1XLhd0X0NU67IPQwmtQpynBzSUH2rQzZNmlpgOQDS1ZA6ikzRTAdnmlzTM04GmIdUch2vEffB/KnZqO44hLd1Ib8jQwJTSxfe60080sX36ALm7Heik3e9FFgPkH7FK7i0QKZSdmM8Z6Vv+GdKutMguIrkKAzgrhs9qzopxFqSXDBsCTcQOvWt0a9bbc+XLj8P8aI2WpMrmltPvSAc89KoDW4B92OU/l/jSnXIGPKS5/D/GtOZEcrLuBijvj+Rqh/bNuf+WUvT2/xpv8AbUGeI5OfYUcyDlZo4rVtpTJbpknK8GuX/tqD/nlL+GKs23iKCINuhlIJzxipk0XFWOmB280044OTWGfFFoCcQT4HsP8AGmnxTanrbzfp/jWZpob2c88cDiq9+c2rcEZwOB71kf8ACU2o4+zzHn2/xqrd+LLR4Sgt58nvxj+dJ7DTRVumA8VaIRwPtUfB7/OtfQiivmuPVI9T8U6QY45ECXUfDYyfnH+FfSq4xWUi0LTT0p3WmE0ITISOaaRT2xTDVmZh69rVtoBtbu6XdGztH94DBK57/SqUXxQ0OMfNg/8AbZa6KbSrDVsRahZwXUSfOqTRhwD64PfmnR+DvDi8jQtNHP8Az6p/hWkU+hLaOfb4saEBwE/Gcf4U0fF3Q8dIv+/5/wDia6tPCnh9emiaaP8At1T/AAqwnhzQ1+7o9gP+3VP8KfvdxaHFH4waIv8Az7/jO3/xFN/4XBo5ztFrj/ru3/xFd6uh6SOml2Q/7d0/wqRdK05elhaj6Qr/AIUe93DQ8+Pxe0oDOLQD/ru//wARV3SviQuv3n2LTLOC4uNhfYszD5R3yVAruF0+zU8WkA+ka1WvNPtUv7K4Hl25QuCyhVzlenIpWl3HzLsZI1PxE43LoURGcf8AHyKUah4k/wCgJAMf9PA/xrZglsLSFIPt0bBc4aScFjk55OfekbUNNQkNfWw+sy/40/mL5GWl/wCI+p0mAD/roD/7NVgX2vt/zD4B+IP/ALPVhtV0gcHU7ID3uF/xqNtb0IEE6tp4+twn+NS4+Y0/Ib9s1/A/0KAfgP8A4ug3GvFf+Pa2B9x/9nTW8ReHxkHWNOH/AG3T/Goj4m8NqRnWdPH/AG3X/GlyLuPm8iuNQ1t7+e02WivEiOd3AIbdjHzf7JpkkWtyH5nssf71U18UeGYvFFzcNqln5clnEu9WyCVeTjjv81Xm8ceFF5/te2/BWP8ASp5E92VztdCD7FqvGZLLr/z0/wDrUybTdVlxiayX/toT/wCy1M3j7wov/MVi/CN/8KafiF4UGP8AiZgn2hf/AAqfYw7le1l2Ko0nVw3/AB82eenDf/Y1NHpOr7cfarVfxz/7LSN8RfCw/wCX9z9IH/wqNviR4XB4upj9IGo9jBh7WYT6NqYcIdRtgzgkADJOOv8ADXHQT+L2BeLw/fMp5DBosMPUZrrJfiP4WkA8xppMdAbfOPzrqtPdJtOt5o+I5I1ZQRjAIyKXsY9A9rLqeXhvGxORoF8pz3lt6w/Ftz4mi0Yrq2nTWtvJIFDySwtlhyB8vPb9K9yIB7k15X8ap9mnaXbBj87ySHHsAP8A2Y01SSYOo2jw+5l3yPH1HSktJAVwwGV688+lMmjZn2bvnA49/pVe3LLdEMCDgnBOOa2MzTH3gcnr074/z/WpTtKDaBgHPA71WZl2HPpgAdf89KsIAybgB82D78/5AqhHp3hi787w9bSQqjzJGFYnsRx/SrMksz3cRlVctuXAHsf8KxPBMhk0dgrBmjlZM598j+ddDdSLDCoklUyMwVDjOT0/Svepy5qaZlazGFURRjl2OcEkcZrhfHduPOt7sA4bchzx7iu+i8xo13hN2zDMuOTj6CuX8bW/m6M0g6xMHBH6/oazxEOakxp6nmE8ZUZ6rn0rpvhnpjal4705cjbCxuGyeyjP88VgTRb43xlsc10fwsultviBp+84EgeMfUqa8CWxvHc+gdZhEtsmQOHB557Vn2+ngoCDj6/4Va8Q6vBo+iXGo3EUkkMAVmWMDdgsBxk+9cVF8YvDiAH7DqI57Rp/8VXjYuhVnO8Eevhq8IU7SZ20dhcebGqlPJ/iyDn8BWpFaggAdO9eeD42eHVOGstS/wC/af8AxVTp8cfCqgf6FqhP/XNP/iq4ng8Q94lyxcejPR47UZGD+lc18UNIkvPAV2Yk3NC6SjHYBsE/kTTdG+MPg++SRpGu7bYMgTRD5vYbSf1xWL4p+Nej3Oj3un2NtdKZoWjDlFI5GOea3oYSUGpPc5Z1nLToeLWXk295btcAmNZMuq+mc8H/AD1r7C0+/wDt9jbz2MLeVNGrrJJxwRkV8Vz36zN8qlTnNe/eHvjv4Z07wzplld2+ofabe1jhl8uFSpZVAJB3dOK9WdOTaZytq1j2D7CJG3XMhlPoen5VOPLiXCqAPavJm/aE8J44ttU/78p/8XVZ/wBoLws3/Lpqv/fpP/i6zacPghqTZvdnrz3AA61WkufevJG+Pvhgji01T/v0n/xdQN8ePDTdLTU/+/Sf/FVyVI4mX2WawjBbs9Ze496rPcV5U3x08Nn/AJc9S/79p/8AFUw/HHw4f+XPUv8Av2n/AMVXJLC4l/ZZ0RlSXU9ReUmvHfjnbebZaVcgfcaVCfrtI/kavH43+HP+fTUv+/af/FVy3jr4j6H4o0WO0tra9WZJd4MiKBjBHZj6itcJhq9OtGUo6Dq1KTptJnmun3Js9StLpfvQzK4/Ag19bAlwCuSGGRivkDIxxnOeK9lg+MlhHplrbvHemVIUSTy0VF3BQD/FuIznuK7sxw063LyIywdaMLpnuOm5FqUbqrEYzz61crgPhh4us/E8OpJaxvGbdkZlZAv3s9ME5+73rvqKUHCCjLcc2pSbQjDcpHrWe3BI9K0TVGddspxXTTMZjM0ZpuaWtzEkQ/KPbinVGh6j3p+ferRAtLmm5pc0wFzSim0A0xD80jjchU9xikzS5pgJG26JT7VJF9+oYvulfRiKlj+9SAu9u/5UUz8aKAPka/ZoLV3Q4dQDnHvitNPDmqMgP2+AZGf9X/8AWrP1VS1hKMdcfzFem6Pol5q52WcDSsi5YKBwKIpPcmba2OIHhrVcj/T4Mf8AXOpF8L6mcj7dDn/rnXZy2kltcGGZSrqdrBhgg12Hh7wgNVtTOZNijgZGc1bjFakKcjxo+GNVI2i+iLY6CI0knhvVU4e+jUkcZhr03X9FbSbpot3TuK5uXLHkkkjHPWhQTDnZyR8O6kcY1KLp/wA8qP8AhHdRbCnU4hg/88R/jXUU09OBzT5EHOzEHgzVm4/taLn/AKY0q+C9SOSdYjx/1w/+vXX2r+ZbK2eR8pqyqgHn8ayasbrVHEf8IRfk860n4W//ANeoLnwNdpEXbWA3QEfZsd/rXekZByar3/8Ax6OxxwV6fUVL2KSR5dp9m9h4vtLZ5vNMV3D8wXbnJB6V9Rp92vmiQ48exAH/AJe4Dx/wGvpdBhcVlIa3FpjVJ+FMakgZC1RmpWGajPWtUZsqXmqjSESZoWl3tswpAxwTn9Kq/wDCbDOPsD8H/nqP8Kr+KD/oEBzj9+P/AEFq5jODW0NjKb1OxHjnH/MOb8Zv/rUp8eEDjTvzm/8Asa4zd+lGeaqxHMzsv+E8l/6BqD/tsf8A4mmnx7cHpp8X/f4/4VyGe1JnGcUrBzM7D/hPLr+Gwhz/ANdD/hWF4n1u68UWlvpcsMUEUs2WdGJYYBPf3FZoY9cUgb/TLQ56Sf0NNoLsxpvB9vA4U3MrZ6HA/wAKaPClp/z2mH4r/hXUal99M9xxVFgeNpxUuESlORjDwrYjrLORnuw/wqVPCmmkHLTn1+etA7h17VPCHJzn5RQ4oFJmavhPTR/z2/7+U7/hFNM7pJ/38Nawb2p+78anlRXMzA/4R3Tk1Rk8uQx/ZQ+wyHGd5GaZeaJpsUO+O3xg/wB88/rWu/8AyFE6/NaOPykX/GorsBrdxjoKFFWByZx7w2sWqCN4wIthbBY8frVYxLKk8kJ+SPJBzyT/AIVY1bBnhiUjdK4X8B1qGABU1KPngnj2xU2KuJciNdMhmVcMSMmpr+FFtkaFAPmBOPTvVOV93hmM45AHfHenancj+zY1z8zKDgHnFFkFy3FEk80cpjCwlwFA/iH0rYXxdr8GsS2CarcLbRIBGikAKBx1x/nFZrSBbWIj/ZOR06VFHEt14n1HqGiY42ntk1N0ilqauo+K/EEUtrs1q8VWfDASdeRWH4o1a81KaJLq7muDEpI82QttyecZrYl063mCO7t8hyvPT/PFcz4hjS1v5FU/KI1Y5OSSf8/ypRmmxuLRhzrAqb5S2euB361UgbfKWOQoHA9OacIZLl9zHapPAz/n/OKSI7L1lJB+XHHatCSxJwCSQM+n+f8AP41LalPJdSB8h56dD0/rVd2yMk8dKS3lCXOMjDDBOOh/z/OgDt/BVyqG5iZipeQNknPVf1ziulu/31woTOI2BGB79q5HwfLGlxdqyhmCqykDjOT+XQV2KulraiSUfO5HB9c/5/KvYwr5qaRnLRk7NOWwowq579enP+fWsfxJHI+kyRuoDOhPOa6CMzRfJLjaCfmHb/PNcr4ne71EywwOY0QHkf5/Wt6ukGT1POI2OwNycZzntWr4OtpJfG2lRwA71ukcEd1Byf0BrGi/dM8Z+8pIP4V1Hw9eVfH2ltHH5mWKt7Dacn8BXz8tDdHver6QuuaRdaY0nli5j8vft3beeuO9clZ/ACBsFtYkZfU24A/nXoVpJ5dzGwCZB/j+7+NdEbG5uzm9uwI/+eUPAP1PU159Zz5lys6YtJankep/Bnw8lgbW21C6m1HIC+Uike+R2/Oqth+zqsw33utyRA/wJCCfzzXuEMdrZpsgjVAPQUrXI9azVX2espfJCb5tEjyFv2eNJUfJr18repiQ1RvP2e7GO2ldPEF0zqhKgwLgkDjvXsr3PvVaW4yCCawqY63wlRpye58UXlobR9pO7OR0xzVSt/xNA0OrXUJXHlTyJ+v/ANasCvXpy5opmE1Z2CirllpV/qRYWFnPdMvLLBGXKj1IHSr6+D/Ej/d0DUzjr/oj/wCFU5JbsSi30MSitUeGddMPnf2NqHldN/2Z8fnipIfCXiK5GYdD1Fx6rbP/AIUuePcOSXYxaK6D/hBvFP8A0L+pf+Azf4Uh8D+KB/zL+pf+A7f4Ue0h3Qezn2MCit8eCPFB/wCYBqH/AH4b/Cqmo+HNX0iOOTUdPuLVJCQhlQruIGcChVIN2TBwkldozeNo9e9eg+CPh1aeLNCfUJNSlt3SdoTGsQYcBTnOfevPa9d+DV/ix1WzJHyyRyqPqCD/ACFZYqUo03KG5thYxlUSkd58NvBUPg/VbwwahJcLdQgFXjCgFTkHr7mvSjXI6Pcf8TiEZ4OR+hrrjXHRqSmry3OqtTjCVo7CVXuk+UMO3FWKbIN0RFdEHZmEtUZ1L2pSMHFJXSc4qnDn3H+f51JUI/1in1yP8/lUmatEsUngn+VRfaRtyI5OuMY5pXDkjY4A7jHWkVZs/NLn6LVCE+1f9MpPyp4n64RjgZHvTQku0/vfm7HHajZLx++IHfA609BE0b71ztK9sGn596rCOX/nufyqcUAIvErj1ANTR/eqAnEy+4IqxF96gCzz7/nRS4NFAHyZqeBbMeOWGB75r1Lw94kvNBLSWhTDqA6uuQa8q1U/uOn8YHI969F0yMS3cELMQkjrG5HoTShs7kVHbUuXepTahqEt3cEGWRtzYGBXRaF4vn0qAwoEdD/e9a5XVFMGpXEDPvMMjRA+ynFV/MKn6Vro0ZJ9TZ1jWJNUnaWVgWY9RWM5zk5z6980by3GeaQocU0BEeM5IxTSMZ5HvT2XbnnkVGdv5mmMt6fJtmMZ5DfzrV2ZyR2rn1Yq4YHBBzXQQuJIVkHRgD6VjUXU2pO+gbT36VWvFP2V88gEH9ausMj0AqreYFnLweBn9axb0N0tTzabb/wncRH/AD9w/wA1r6YXpXzPOCvjuIsACbmHp/wH/wDXX0yozWbBbsDTD7VIRTGpoTImFRGpmFRtWiM2YPigf8S6IjtMP/QWrkyD3Ndh4kTfpg9pQf51ymzjitoPQynuRYI6n8KO3J96k8vJxTvJ5zxg1VyLEBHvS4PepfKJOOtTf2beHpaT89/LNFwtYqdxzRj/AEi35P8ArV/Xiro067720oHupFMktHikt/MjKFpUxn/eFDulcUWm7FjUVDMhyeh6VTaONR8zYz7962NTjRBH9D/Ssl40lOD0FTe7LtZFbMW77w/A1ZhaEpgv85PAqE20YI5/WrMFvARv5yORzQwQ/wAlSM80ohQ8f1xTwQBQW5PPFRqXoVVhRtcsEb7jpKhx9UNdp/whumSIys9xzwcsP8K40uF1fTnHaSQZP+4D/SvWEwcEbsnrxTQ7I4//AIVz4clmSR455JVPysZWGPyFcWfDOn2/xVm0VoibG4jRtm85OUPfr1FeyDaJQoY8mvNfEDLb/G3SnJx51tGv1O5hQxmyfh34XSD7Oun5iHVTM5H86uR/D7wq6ANpEDA9d5Zj+HP6VuBACfl3c9xmp4yFlCjAUH2osM+e9at4bXUr22gGFgmeNV9ArED9Kowyi28TahL3aJX+uVH+Nbvi3aPGepwbct57HIGM55/rXNao/ka6zfKBJbx9+vyj/CuWTvdFxVtTVa9LLt2Y9cVx/im73XxVX+Yqu/H44/nXTq2BweK4/VJLeC9meRS87OTgjJ69falRjrcuo9DPigJXzZ5CIwM4BqK1/eXDFeCxwB171FcXMsxwxwvXaOlTWKqdzPwo4PFdRgWZQG6HOevP1pkSKGBBOe+MHildzIxCdD0GM8VYt4TtDNncRkd/SgDoPB7IuqTqRjdErDkEEg//AF67NwlzcKjA+Wg5x3rz3Qz5OpYUoN6cZ+o6e/8AnivQdKxJy/CL1z3/APrV62B1jYzmbHnJLAjSADIBP+PSsO7uUbdGqBmJPOKtLDcT5dcrFIBtUDBUfzqtqb22i2Blk5kI2onGScdv8a6Zyb9BWVjyzWIBZa3PGD/FuPPQnnFegfCTSrO/1ue/k3+dYxholU4BLZBJ9cAdOnNeaX9xJcahNJMRvZucDivQfg3qCw+KJ7RmAFxbEKPVlIP8s14NXVuxtA9xI7ZxW7DqHmW8bg/eUH9Kw+lRWVyRB5ZP3HdPwDED+VeJmLcYpo78NBTbTOi+1Z70xrj3rNWY+tKJGPTmvFdSR2ewSLpnJ71E0mc81Tury2sYvMvLiOBPV2xn6DvXNy+PtOkuhaaZG1zOQcM/yJwCfr29qcadSeyC0UeM/EK3+z+JdVT0uWcDH97n+tcMetdp4y1CXVdWnvp40R7hVfagwOBj+n864xhhyPevqcNdU0meVX+Nnb/DHxDNoGu3RhhWU3FuUwexDA56/WvfNJvtQ1nSra4kMUayghgVyVZTg8dOevTvXzN4RZh4ltVVsF9yZ+qmvoLwlLLbQPA0zPGDuUEcAnrj8hXFmFNX5ludeE1Vjql0+AOJJi07jo0hzj6DoKnLqowoAHtVcz8ZzULzD1ry7tncqd9yw0uaiaXrz2qo9wB+NRG5G4DnrQos0UUiy0tedfF2Pz/DFtKASYrtc49CrD/Cu0e59q5Px+PtXg6/X+KMLIPbawP8s104a8akWZ10nTaPn8ffGfWu6+FV0YfEtxCTxNbMMepBB/xriJhtnce9b/ga5+zeMbF84DMyfmpFe5WXNTaPEovlqJnvtnP5Wo20hPSVc/TNeiGvJTdFSCOo5BFerwSie1jkHR0DD8RmvOw6tdHo4jWzFoNL2ptdKOZlORdrkYplT3QwwPtVfPFdMdjnkrMDxg+4p5/Wo2OUOPSn5z+NWiGLRSdTRVIQtLTaKYD80Cm0UxCSNjY3owzVmMfNVaXmJh3xViE7tpHfmkBb49T+dFJzRTEfJl8pbykHOZkGfxFd9GywXCMrE7GDZxjpXCXAzc2o4+a5T+ddtld3U8c0oapk1Fcv6xPFd6zeXNsS0UshdTjHXk/rmqWcD2pyMintjPXFNJ+c7fwNaLRWMkrIu2ccbTLvIA75rsvFOi6Np+kW0tlOGnfHyhgdwx972rgVlwcg4PrUj3DuACaGrsaIpMZIFREGnls44FNbkmqAYc561r6XNuhaM/wnIH1rJPPXt71YsZhFdKScKflP41M1dFwdmbpBIzVS+YfYpSRn5T0FXwvPSmz2UU9tMXheQhSTiQrkY6VySdkdijdnlt8AvjmLn/l4hz/47X0yn3a+b9daL/hYMfkwCBfOg+QMzf3c8k19IIPlqb3SFs2OIqNhUtMYU4ksgZcc1Ec1Ow4qMitEZsxvEP8AyC245Dqa5Inj3rt9UtJLyzaCLG9iMZOO9Yq+GNQbvCB/vGqiyXBvYw1ZuenHbHvRyeT29q318J33eSEf8CP+FPHhG95zPCPxP+FXdB7OXY50HBHP44rtUjvJdQCRp+6Me7/VnHTru9c9qzh4Oue9zEP+Amu0hUxQxIckhAMg+gqWlJicHbU5aayv9+0o4X/drJ1uN4zZGSNlbzejA/3lr0QHPQH86xPFGnR3kUEjs6mEMwx+B5/Kuidbmhy2MaeG5Z8yZy2uDasZxnrWEZOeBXb6jpMVwE3u4Hsazh4ZsuCZ3XPOGcD+lYqyWprKLOZJyM0gcgYrp38O2gKlZCVz82XPT8qkg8OWkz7ERmPX7xFaqm2r9DO6va5ym9uTxR5jdCeK7T/hDkP/ACxUfWU05fB8QGPIjP8A20NLlXcrlZw0j/6Tp5J6XBH5xv8A4V61E8zKCqpjAxkn0rg9W023s7zTlSNVIvUDAH2Yf1rr4X1NxEYo7YwlRguxzjHoKya1NUtCw7XrSZHlAdvvV5z47c2vxR8K3BQb3ESMe3EnP869Bk/tYEBDaKoGT8rVzfiexFzqWm3eoRW0s8Tf6NKqsNhyD0P4VLVluOMbs6122yOMd6fCqn53wBn061nNa6uSCL22B7/us/1qVItUCFJNQhIIwAsAyCe/Wk5tdCuXzPIvH0YTx7fuI2AYKckdcoK5HxCjfarRowdv2dTyfQsOteweJNKU3ryXUcc9xtGZTEATxxXGG1SW0gLRRuNpU7xz94+v1rk5veZsqeiOXuZWgsZZo8b1XK+ma45/tJlaWeRMk5ZioJ/OvUfENkB4dnCxoh2K25RyBkE15xc2rO6IVOxeenU10UUrGdVWZkuY5XZkhOOuc1NaqDCQV5HaprgxxYVUwzcAdM/4VVtJBu56Z7VqZF1It7ZbhR25qyeT7Yz6VAshyOQPXnmpR0C9F7Dt+H5UxF3R7WG41KVrhwqKoKkNg5ya7211CyjHlp+8XjOehFeW3UxiuIsSyIGB+ZBk4/H6fzqRbyQkf6XdtjDHkLjn2/xFdVHF+xjZIThzbnr8GpxmJhGoLqCFQdTzwK5fU7CWVmvdXvrW0U/dWTJwPTHrzWLBdLdxX0UUMMU0do00EkckxYsrDpudlPBPYVy80d3PMWuBLJu+UOxzzjI5pvMfaaNFulyrQ0dZtNMZZprS5mklQjlowikfTJNVvDGr/wBieI7DUMHbDMrMefudG4+hNZ0tndW6HepA7gHNVlODXLOSm72EtD6+VlcB0bKsMqQeoqpDC/n3HZBLnJ4HIB/rXPfDbWDrHgiyZ23S2ubWQ5/u4x/46VropNMN5cl3ndYRj5FPevJx8E6ep6GDlaeg6XU7K0wGcyuP4Yx/WsfW9Y1f+y5bizhNpFHyWxkkZx1P17V0lrp1laf6uFd394jJpb/yp7WSGRFZHXBUjINeRHki9Eeg7yPF7lri/ZmnmkmlbnLEmiy02a3uEuB8vlnP510k1rHbzsqoFAPYVXlT/R5B0Ow/yr0o2toYNanA+M9O/s+5gQEsHhyM/U1xEn3zjp9c16d48QTWOlXGMlkKk/l/jXmcow1d+Hd4Hn4hWmXvD83keIdPk6YuEB+hODX0RpT+WT79a+aYZDFPHIDgowYfga+jrKUMqMucEZrHGK6R0YF7nQi5+X9Ka7SFN+1ivrWZI87QObbaZQMqG5B9qrzXOoRTsUZXi/ikfapAyC23nkben19q8zkPTuaLzYqLz8HI9c1zn2zU3QjcqlWIdyyYQbup9DjH5niqzX2pkbkmh27QTudMZ4zjn2P51oqZPMdNNJh26YyaytbT7Vot9bkZ8y3dR9SpotJp3iLTtuJY7G45XjH3eO9PZ+CDjB6mqirMmTurHzzcD95n+8Aat6RP9k1uwuD91JkY/QNzUV/CYLhoj1jZkP4Mar5OEb04/wA/nXtbo8LaR7y74LDivVfDk/2jw9YyZz+6C/8AfPH9K8atrv7TZQT9pI1f8xmvUvAtz5/hxUz/AKqZ0/8AZv8A2avNp6SselUd4pnSk03vSnrSZroRzsjuBujPtVI1fPIxVFhtcitoPoYzQn8qRP8AVrnrjFHShehHof8A69aoyY4GjOaTpUF9LJBY3EsIUypEzIG6EgEjNUkIs9aK8t0z4jatfapaWskdnHHNMkbMI2yAzAEj5vek1H4iaxaahcQQpbNEkhCM8DKWXPBIzxW3sJXsZe1ja56mDS5rB8J61Nrugx3k4jE29kcIMDIPH6EVuZqGrOxad1dD+1Ps+UUf3ePyqLNPs+JJF9Gz+YqRmhx6n86KM0UxHyhMc31l1wblOPxr0C00a/vbC4vYLYtbWwzLIWUBeM9zyfYe1ef8tqdgP+nhev412X2iXyvK8xxETuKBvlJ9cetFPYmY8SAHnP4GnvJFj5V/E1Xz7mnKA2Mtjn0rQhC7hzk/QUA5J/SnvFCIyyzAtkcetRY6d80XBqxqRaJfXGjyarFb77SJisjqwJX6rnIHPWsxmJ69PQ1JG5EZTzGVG+8AeDj1HeoCeTxQgFz3poPOaCc59+nFA6etMDrLFvPtY5uuVGeO/erUsLi1k4JO08Vl+GZQ5lt2xn7657+tdYbVmt3PfaTz3rzK8uRtHq0I88UzxLxCrDx/Hv5Ilh49Pu19JR/dFfO/jCMJ8Rl68yQE5/4DX0QnKinF3imYzVpscaYwqSmNVozZCwqM1K1Rkc1oZsIl3ToPervkA9KgtFVrpA2T1/lWqI19OazlPlZvT2KQgp3k45zV1oQrYxSeUCelL2hpoU/KHPQ1Jt6DngVP5R7CsHUzeJqsMMFs8kTth3GQF6fgKcarvZK5lVStc2QBnkgflVLWQHs25BIU9D7Vi3tjqBB8gSEn3P8AjUlhbXccdwtzC6jy/lZh1613Sp/uue5xU6zdVQ5S7crmNDjrTWsZ7iG3aBGPOCQQMc9TnqB6Vbdd8EWeeB1+lXrIAWKfUiuVWnozrqRsjIu9HuicxkkYPRv6UabZXdrckzRERlcbye+a6AAYzgmknH7gY4wwP612Ks1DkONUI8/MZ4Y5OWz+P/16RW/e/fGB71E0b+awWLI7c0qxkPzEB79ayubnI+JG3aivA+S9hYfi4H9a6yyLCzhYKPuAc1y3iBN+pToM5EkD/wDkRP8A69dRZDdaRKc/KWB/AkVhCpebRtOFoJliRiV+bavbrXPeKwv2ewkGTtm6jtXRTAbVG4g9eKw/E426Mj4J2TKcmiv/AA2Kj8aNiQcq2eopVI7kk/SmpJmGJgfvIDjGe1SLIxxwfyrRbEPcxfEVvm5UjoyDOa4fT7TzYTHlU2zSKT2HNemalCJhGxQ8DFchp+lFLy9gkV+J2ZSB2IBryK1XlqyR6FJXhEzDYvPayRlFZTGcq2AGGOhzXialkyTu2r91QM19HvpanIKuePSvn+/sHstau9PVHLRTvEF2kkndgYFdODrKbaMsVC1mc3fRM37w8MOeO1U4MbgMD612d94J8SPbGQ6Ff4C7uYGB/LGa4tUKuyvuVgcEEYIrtumcdmtzViICjjrzjOetSmQIgZjx0b+VVYpCyYB3Zxx1qwmGjOSffjH+f/11RIiQw3sqs6vIANvBxn/Oa1o9I07KsYsDrgntVGwRAeBjOTtGOn+Fbax+ahAYqccAev4Vm2aJGnJe3ap9ne7u3ixjy2uXaP8A75Jx+lUJnkcFFUM54OQKsmJXyShJPOC/4/4flVy2tVi/eMoU44xngCsktTS+hzeqad+7SFpGeWRtzEj7oHOP1Fc1qmnmzfI+6fbH+e9drd3f+kSCBPOuGGPZB/Ssa80S4O+WabLMMsoHArVMzaOi+DGsm11270l2/d3kW+MH/nonP6qW/IV7gr7Dj8a+W/C1/wD2P4w027LYWG6UOf8AZJw36E19QSq24bQTjrXJjY3ps6cG7VETGaqd3P8AKRSq2W2d/Qdap6iGjxkEZHcV4sY3Z7GiMC/X/SGI781nMy5KkjBFUfHUM9xp8Bt1kdhICQgJ4wa4M6ffsebS5JwRnyzXqUqacbtnBWqOMrJHTeKlE3hOykHPlTFePTn/AArzC4BDnPXJFekyQS/8K/nhkRkeCfO1gQeo/wAa85u1xK3Hf0rpw+l0cmI1aZXHWvfdAm87RNPk3ZLW8eT/AMBFeG2Ol3morMbSAy+Su+TBAwPXmvX/AAXciTwnp5JJKqVOe2GI/kKWKV4l4R2kzr4ZVBwW25HDY6Vm3Ol2cmU813UliVAAUbiCR09Bj6VIZTuI471C0hIxnivPselzCW9tHaFirs24BRuxwMk/zY1W+wQIylXkBVdo5HdVX09FFSlx0z+NQSXcEed8qfLjPPrVWZLkieMiCCOJMlUUKM+n4U1pCe1Zt5rMNsNoEkjq20qo74zVe11c3F4YGjKDJ2nOckdafKyXNbHmnimAQ6/fKOnnlvwb5v61j5/dY9D/AJ/lXVeO4PL1p5Mf62JHz9Mr/QVyi52sPxr1KbvBHkVVabPU/DVwZvDlk2ckJtP4Ej+leq/Di43W+oW5PKujj8QR/QV454Jk83w+Vz/q5mX+R/rXqPw9l8vW7iL/AJ6QE/kR/ia4pK1RndF3po9KNNNKfejtWpixtVLhcPn1q3UNwuUz6VcHqRJXRUz0o6OffBoobhh15FdCMGLmkIDKQeQRg0lKOapCPD9MgNjq2qRRIr3ttHILYMufnVwCQD32BiKs3KN4ns7jUvJjt7qOXG57g7JQcswHmMcFeuBxg9OlX9dlsdO8aXqXdsiLvEsc0asZNzYb+8B3PT0FRXfiHSdR2G8NxMUXCh4t2Dnnq3fjtXoJt2ZxtLY6H4XXO7Tb616+XKsgH+8Mf+y13wrzTwJqdu/iW4traJI7eWDKYjCsSMdcfU16UDXLW+M3p/COp9uSLsjsyfyP/wBeo6VDi4hbPGSD+NZGhpZ9jRS/j+lFMR8oR/8AITsAD1mBrr8HbxznvXN6NbpeeKNKt3+68/OD7Gu01LTpdNmCFhJG2SrHg1MJpPlHODa5jPwR2pQCeMUoyP8APWj5vb0rYxsOMbqVBHLDIwaPKcsF7n0pMt1zxSlm4OfzpABiCscnIHoKsxWKskkguIf3ah9jEhmz2HHUVX3NjGaXJxnNA0M2EnoRz6UqxMT909aXLevtUiFs9TSbGkaGiRTxanbyrEzAOA2O4PBr1xNIYxcr1HYV5n4fhZ7pS05TBBOO9e4R4MakelcFel7adrnZCpKlDTqfM3xGsWtPiYuFKgtbnPXPT/P4V7og+XFeUfGR0HxA08D73lQ5/wC+2r1hB8tEY8seXsDfNqxaa1P7UxutaIlkTCmGpGphqzJlnToy94nHTJreW2RTuA+btzWRpDYvMeqmt6lyJu7HzNEPk5Oc0eSKmoo9lEOZkJg5zmqN1GUmA9a1KzdRBaQKDjgciqhBRd0JybIgrev6024VTAQ7AZ4yacoHrn61T1dtunsx6BgePrTqaxZUPiRJJEscSoxHygZ/KnQXEKW4BnjUA92FZd7fgWPm+oXv7Vyv29pZSmCFOe9YUU73NKstLHo0VzBM+2OaN2HJCsDVW61nTY1eNr23EiMAymQAgg81wug62bfUrgtGoVUKls9eQa4LxJfCXX9RTBz9olU9OfmNW6rU3EhQ9257adc00Lu/tG0APTMgFQHxRog4Or2ZPtID/KvnKeLc/FWLItGwUKMN0z/Sh1mgUEepzanDqXja+8i5E1p9kV0K/d3Kyk/yrrLbU7aG4ltWnxILmVcFSerEjtivI/D0pg1K4xx/okw9c/KTXrWjS711CUHlrtm/NVrnpNurdG80vZ6hN4o0uN2BnnbA42Wzn+lYfivxHZz+H5IYor4yl0YFrZ0HUdSRiuvyM5yd31rL8WQibwjfEZLBQ2TzyGBrskm4tM54uzuiK01eVbKBZNI1MukSq37kAZA9zVuPVZnAK6Re4z3MYx/49V9SXjQ9ioPFO25XG3I9zSSfcG0aWmjzrISSwNGxJyjkEj8iRUdvDbtql38i7jtbp7YqNLryU2dOM49KyLPUDHr0xZuHiXj6E15uIrwVRRa23N6dOTi2jqTaw8/IK4eTwxp1r4/u9ZFupuJUj2E9EO0hmA9TgD/9ZrqDqgrG1vUBEEu0GJANm7IwO4z+v501iaLklBWKp0pp+8RahdSXen3a2Pz3Ue5VVSAQ3YfNgentXB+NPhfPr3huTVI9HEXiGOIO5jkUeeR94MASCcZweucc13ENyt3Ai27yGYneuGGHU9VBPQ55FdDBeSG0jEsZ8zaMhmHWuuLitQqp7HxQhKyYYYIOCGHI/CrUchbEYHJPQDFXvG1ylx461uZNgV72U4QEDO7375z+OazrJTLKAoyAM/5/Kui+lzktrY27OJVUb2G30B79/wDPtWqjxpGXOAuAuP71ZtuhOOTjOGBxgCphKZr0QqCUjG4j3qCzUj3AKw4KhRjpipLuYCE+bN5UX/LWXPOB2H41RUzKp8q3eUkkKqr3579v881mzb7m82XOLy4BylnCSUT3cj+QpJDbLkU0t2hOnILKzBw11KMu59v8feq+o/aLexnjsoQcg7ppGDs477fyrREGpRhVmluTgYENrCoVfbLYFPZCQ0k1skUJxuCsZD+IUECh6AtTzXcd+7PzZzmvsHwbexX1hZ3U6qwuLOOXkZ+YqpP8zXyRqcKQ6jMsUbRxbsorelfSHga6ZfAuizAn5bVVz9Mj+lRX/htoKSvLlPSo7PSoblrhYUEh9TkD6DoKwPG0kBsrYxKgPmEEgdsVTOqN/erJ1q8NxaqCfuvmvHjVlL3bI9OOH5XzNmLK25AT6VUYYPIzU27dHjnhjUTDJBPT2roiVIx9Sj36VrUOP4BJ+mf6V5HeAhj15ANevX9zb20t3FcybPtNoyqME7iMjHH1ryK6GfriuvDnFiehu+A4pbnW5LSKQRtNFjJ6HkcGuu8Auf7Entz1gunT+RrzTS786be+fs3jaVK5x196734dXfnnVQAVBlWULnON2f8AAVpXj7rZGHl76R3M8vlQvLjO0E4H0zXPHWZ2ViYhGwGf1H9DXQTpvtmA/iUis3TbWxuYwJwNzQPyGx8wXj+XSvPukrs661TkVznpJLu5gaSa4k2KQ2BxwWwR+FQ3KGN7gBiW3ujEnqAAwrV1FYYmuoLZt0DoWUZ6EDNZtwd8xP8Az0aNh77lwa1jqjNSurkQvN0xuW5w8bn8sGrtvIp1hMAgmQuM8cMo4qiYBDbRzKSfMj349CCv/wBerjS/aZ2uYFkwAjD5epBwabKRk/EGPM9pIBjdG6k/Qg/1NcIvU/SvRvHaLJYWkmcsspXrzhlP+ArzlMCQZ6ZrsoP3EceIXvnb+AJswX8OejIwH5g/yFeoeDZ/J8U2mTgOGQ/ipx/IV4/4GnMWrTxdpIT+YI/+vXo+iXxj1SwugrIFnRirDBA3DINc1dWqXOmi707HuNJmlIptWiGHemONykU6g1SJZnkYNMY8r9ammXDn3qCXhM+nNdCOeQ7NGeabmlzVEmdf+HtJ1O5+0XtjFNMAF3tnOKgHhPw+p/5BNr+KZrYpc1XM+4rIoWmh6VYTLPaafbwyqCA6IAQO9aOaaKM0O7Cw/PNDHam7+6Q35GkFOVd6smeoIoA1Ovaiq0VynlJkjO0ZooEfN3hNPM8caSOcb5CPwRq7/wAVjBg6jJPSuA8I3ENr4306adgqDzcn/gBFd/4tnSSS1EZ3oys3mD7tcyT9qjpuvZM5oZ6ClxzzzSd/agkggAcdzmu25xsUc+uK0NL0XUNXkZLG3MrJywBAx+ZrPXOR9K6nwd4gg0O9ka5jdo5EILKRkHtwetZ1JNK6IldbFO+8Jaxptobm6tDHCv3mLL/LNYpUivQPFXjSx1fS2tbaOUMSDliMVwDMcDGCM81MJt7ji27iDPtUkfGKh6il3FZkXI2kHPrVs0RtWlyYFyDyRXtlheq9hAxPWNT+lfPa3iB8Z4xnOa9csbmcWds4lXyRbgGPbyWwMHOfrxXn4qThaUTsow9orHlHxcufM+KEAB4EduP1Jr2tPuivnz4lTGX4kI3B5gGSfpX0HH90VUHzQTZMlyyaHUw0+mmtESyJqjNSt9aiNaIzZa05tt9F9cfpXR765e0bF3F/vCug3D1rOc+VjSuT76N49a5rWo9cluU/s24ijix827r/ACrNWw8Vsfn1WEewH/1qweId9ivZnb7/AHqKVlyCQCScZxXJxaV4i8wtJrClP7oz/hUMui60u1pNcdhnGBnqal4iVthqB2JZMfdH5Vx2pXDPPcRs7bQ5wueOtS2Wk6hb3iTzatJKi9UwcH8zWVqJd7q68uNydx7GnTquV7lKFitqOqQw2CxS5XCrk9hWBLP5beYozjnPrVnVLA3tqiNA0iuoRwQee1ZN1p99Gfs0PlKgACZfaqADGPXrWlOsti5UebUrLq6y3lykJHyxt5nQ8gZ59K57xBLGPE18nmJ5jTyMFzzjcc1fOmXa/aJI7ZVfaTK6TMMnHPb09aZqmhX73CubIPKHDs6yuxcdxkr34/Koc05XNFStGxgz3CQxhmcKMgfUmrdtZTsrT5IjHPJz09MVoRaDqFwwii0RncHcW3noPqOK3bHw1c2EUN9L+7aMEtBuPTIHzevXNKUr6LchwSV2ylo9pIWS6XmGWB9pIw2GQ9q9Q8LljpkzkZ3uj8e6LXBW7zS3KkkbCTnj2ruvBj/8So+nlwkZ/wBwf4VWHvzakT+E3uABkfrVLxA6jwrqe0jCwMx3elXow0jAbWqPU7b7ToN/E6kB4XGR24rvexzLcjs2M2n2kiZIeBGyD7VN5Mh9aqaIyDRrLMoJWBQcnrgYrSE0X/PRP++hUKS7lNMztXnnt4ENrCs1xsGImfbkZAJz9DmuflvAmpqe4XBNdBqnkSiKdHSR49yBlIOM4z/IVyOonFwJB1weSe/FeHiknXZ6VH+EjRuda+z2zTFXbaeQnJrMm8RSTxvGtleHj72zj86qPO3UNgAjgDiopZ2KDkAE9BWMaLTujSUn0NfQxp6awrTXRtnc7sN8i5HPVux9sV3iXEFwqPDPHIhyN0Z3D8xxXkgOXUtxzzn/ADzXdeDsNoSd9kjqR9cH+tenSldWZzVVfU+V/EL+d4m1SVQ2HvJmAI5++etSaUo2tJjvgU7XrKYeJdTjCMQt5MvT/bNWtNj8iIBsLg56dv15ruvocXUvGQW8JckZ6jPpUFrL5VuZHUbnJJ4xiqt1M0zqinjg0XbrHZZ5GFxwvH5j/PNSUPtYLrUIjMZHSI/LtiBy4ye/T2rWs7LUEBgts2UTHnbyx7dBwPxJPqawNAhe4IHnTqozxFIQf0/z1rqINOt3O1p33L1LXJzn8/6VdiTQttIgQK9w0kzDndK5YD8M4FSyfZkct5CvIeV2A4HH+fypsOlhRmO9ugCD92Yn/ParDWskShUuZCxb7zAEgc8cChrQcXqcN4xs7tJLe6m5jYFB7Hr/AJ+le0+C0MXw60odP9F3fqTWP4bsLG+mvdMvoBdw3MQLCbn7p7HqOp5HpXbraQ2mmC0toxFBFF5caD+EAcVlLWNi46TTMSX97JFIZHBjYkBWIDcEYI7j2qO4cshHam7ifb61HIQUPPbNeWlqeu3oUrWFYVlRNx3SFzlieT169PpRPDHMFEqghWDjJ6EdDT1PzNjuOlMkOQMY/GtUZs5bxijfaLB1BJO9eBmvO72Mq75BDbiCDXsUmV1CwfOFEhU8+orzLxZCYNe1CMdBOW4755/rXRRlrY5K8dLnMV2/w1l26tdxf3oN2Pow/wAa4k9TXT+AZvK8UxJ/z1jdOntn+ldNVXgzmpO00eu/wDnINchNbzJNKhdEKcRlmHzZJP8AWuvCMVwAfwFcze6c5nvnSKQzKQ6YB56Hj9a86J6VRXRnrLbx3OWuS4ERJVR1O3moRc2rS2ReGUlFADsQOM4BIpW0+4Mcf+iy5aNDnyz6EEdPpSXEF1NbwH7JMriPym3IRggg5/nWisYWHx3NxMNsMcaokLNjGSOo4/GopbiW5dY1kZUcoRg9Mg/1FTwWtzBegAIqMzoxdgPlLZz196pw2s8TDcYuCpAMq9j9fc0aD1KerQmXTbqaUMJohHIN31wa4uQbZmA7McV6Hra29wtzJJKiL9nYD94vLDOO9eez/wCtJ9cGuqg9DlrrU1/DEnleJIOuGLL+YNekBiOc4I/nXlOny+Rq1pLnhZEJ/MV6rzn+R9azxC1TNcM/daPeLWcXNnBOOksauPxGalrI8Lz/AGjwvpz56QhP++fl/pWtmkthvcTNHFFGapEle4XgEVVcblI9a0JVBjNUDxxW0HoYzRGp3Ip9RTs1GmQuPQkfrTs1ojMd2zSk03PFHSmAtKDTc0vamgH5qSI81BmpYvvUxEUkEnmNjGMnFFaHHvRSA+XtIJbxDbZzwsnf2rq5nJwCenTNcjo2f7egYcDZJ/KupkJJ570QFJk1uQX2sO1WwkPcE1Rt8mTAJJ9quBHx0PtRJ6jhsPCw9AozS5h6hB1qPyz/AJFAiYk5FTcqw8NGOdvApPOQqcKB7Yq/pL6fb3EjanZPdQlCFVJChDdjxWUkIZnaNZAC5IDNkBew6fWpUtbA0K84UE4A/SsLXLsp5eJCgb5gytySO1bFxFiFtwBGOh5rnNSQSW5juCf3THYe4GOKu4rFVNQkmVR85UYJYddte56fdbtIg6j9wn4cCvnfTpHN9Cm5VVR3zhscivdbCR20yA7sZiXcMewrhxmsUduDWrPL/Hkm/wAeqc8F4f5Cvo+M/KPpXzZ43I/4TUHIPzw/ToK+kVOFGPSqhpBGc/jZKTxTCc0jOcVGWIHBq0zNoc1RHNK0hA5pmc1ojNkkBxcRn/aFbO+sOM4lT/eFT3t7fQzhLaxMykZ37gBmsKybaNqVrGk780itz1rBm1DW8/JpI6d5B/jTYr/XmkUNpiovck4x+tc/s5XNeZWOiLECq8wd4twBIVhnH1qtbz37fLcWsaqf4w/T8O9LcsfJwD/Evf3FRLR2Lir7EobIqW2iDAZxzWXd3E9rbtNFC1yxIAiXjb6n1rGj8ReIYZZPJ8PSzJn5MyhQPzrKona5UtrI6aOyjaJxgcSt/wChGvOkxqYuZbjaGErRELwCAQQe9bcuveJ0idrXRUZnYl1kmH7s9x1Gaz/Dem38cFy+rWiwzNcF0RHBBBAz0J71WFhFuS6vY5q7qRSa2KNxKLK3kjW1JikGHLEnHGOwqzYaeuv38Mt7Dsh27RsZgSoB5611KWVtdSRxTwqUJx0PH6Vu22mWkEjPFAqsqhSOe3P9a9mMacIaL3u55zdWctZaGLpmixQRyrYQrEJIgAz5PfJyc89Kh12GIz2iPbK+9zncOuQtdXEVVZAMALKRgDtxVWcLusXwMk9duT2rOjFQfM9WXNNqyZ5VKiQaj5UaqqbgdoHY1teF1jezj8yZ1WO0jOEkK5wSvOPpWD4ngnTxhIYrgRqsi/LtHAIGQDW94SUfZZCSM/Z2XOPSVhWcl7+p1Q+DQ6OIWRbmWTHf96/+NI9hYXELxRtKQQ2FMr5PH1qsVbaHM5wf7oq7YbGOPPYnkFSOvFWmu6ItPsZ3h+ytn0qGUwwmRlw5ZsE4JrWNvFHHkRW7YHTIz/Ksrwp9m0zw84kfyoYZJCzSHp82ev41P4n1cWujpHZOHur8iK32HOd3Vh+f6ily01G7RtFTnKyJ5reIWu5IUjIcfdGOOfYVyWsrtZDxjBGPzrqRYvpXhqC1kmaWSKNd8jtnLZ5x7c8e1cnrLZiDbs/NwMcf55rzMXG1ZNKx2UX+7avcz3xGjHcPXr0FZr6nagcSk8+h5q/1V8gY44FU2hiXpFH+AFOHmOV+hs+DNR0+bXza3XkvFcRMAsyjaWGCOv416JcS6dpFoxEltZ24OTgrGv58V4xehPscu1QAFyOPeuRu1HnR4QdPau2lZxsclS/Nch1ho7jW9QnRt4e5kcNnO4Fyc5/Gse7zHJs9R398/wCFa7x4bcBzWRfnF+cHJ2j+X+fyrdIxZVbmUY7dhxW54bsodV8VadZzQeZGXLyRuMqyqpYg/UisRid3XI6Dmuz+F1qZ/Ed3eEfJb220cdGduP0VqBGHq3h8eGPEktmylrST95A7jIdCenuR0P0z0NbtpBayEDZCW4G3A4/D/Pau68XaAdf0UxQrm9gPmW3Yk91z/tDj0yB6V55pd3A8EYucI4wodhjPpnjr/hVRYjbitbdCNkC5UdxVpwQuFPGPun17fypkYIG4Nkeo57VI77VBZhj371Yh+kXX2TWraQc5cIR7Nx/n6V6ORxjH1ry+HdHcQvgrtkBLHgDB/wD1V6dwcHPWsTU5VvkLDPTIpjHIJ/WprkYupgOPnOPzqvn8u1ec1Znqp3VyDOG7kY6Uw4Kj06Ypx+9jA79aac98VSIZR1CMG1UnosinNee+MoTHrVwM5yiEZ+g5/Q16RfDNjLz0GcVw3juL/T4pNv8ArLcD8QTWlJ++YV1eJwjAhj2q/obSLrdn5T7HaVVDZxjPH9aouMNzUtlMbe+t5x/yzkVh+BzXc9UcMdGet/2Xqcn3rtMf75rM1C0ubd2jluwCE8wcn5vYe/FdmjZBGF4HBHf61z/iFUSSBzAkh5zk9FHX+deatz1JRVrnPW9rLc3KwNOylgffoAf1zUKW7NOYt0pdX2Ngcd+c/hWmk9woWciJFdmjVlHIwCB/Kq0ouXWSVp25hEoUcd60RjYrLY3ckXyRybjjhiBxjn9aupoErAFmK8sDyDx27fnTZI/st/Gwkc4EbAk9QTg/0rqMAr+FTJ2KjFM5dvC6zwhJpSCSchf0rzy5Qo4DdR8v5cV7WQD0P6V5H4ggFvq93H6TufwPI/nW+Hk3dGGJgkk0ZucbDnp/jXrNtKJ7WGbtJGrfmAa8lz+7Hsa9O8PyGbQbNjg/u9v5Ej+lXiFomRhnq0ey/D+fzfDXl8fuZ3T88N/U11B5rhPhtPmHUIM9GR/zBB/kK7qso7Gs9wpM0UGrIAnIxVKQbWIq7Va4GDmtIMiZSHEjj3zTqRuJvqv8v/106tbmNgo70UUxBS96SjvTAd2qWH71Q1LCfmp3At596KX8KKBHy3oxB1yHHZG/lXTSHP51yNnMYtSRxuzg9K6X7SWZMkYIwQfWlBimi9ZuqSFieAOlXvtkQyA/6VlK+QcHjHWoBqFo9z9nFxGZQcbc9/SrcUyYto2/tcfZj9MUfbIgOC35Vmk85zUbQIQx+bPsxqeVD52WP7Zi3MAHOCSccEU+DUIVUliRu5Az+Fc7IjQzMfmR94PPIAz/APXqKR9sxLMQAhO0fmf8+9TyovmZt3Os20jiBopMyDAbI6/5FcvfXMzXzxKuwhSpz39KjknDXolJHljJxu5H+FTaisN1h4ZB5oXL5Jzj2qbDuZ6K0UiSxjcEXOQO/SvcdNkQ6TbDncYE/PArx3wo7L4ktI2LeW7lGB6EEEYx6V63p67LC3UdFjUD6Yrhxctkehg47s818bHHjINycSRY/If419JR/wCrX6V82+NS3/CXsAMgGI/oK+kYj+7X6CtI/AjCfxseaYaeelMNUiGRtSAjFK1Qk81rEyY5zhc9xWwkqoiMxyCPz4rCnY+UwGc4OMfSp4JUaKKK4kaOROgJxuz/APrqKtjWkakl8AVCKRkgYxjg1G18dpKHnqM9+ag+zQMQfPfIAAw3pSEwQZhRZZS38I5xWVos01RZMuZ3UAY2gn61Bcf6g59Qf1qyqIgOBjPJ+tQXTILaT5h09a5Kj5nobw0GKTsOfWiIlgQPU1xfi7U7+wvmjtruSNCgO1TjFYnh/XNRfWFgmv53UxMSCxxkDOaUsLKpFO4SqqOh3TSkPdDPSU/yFTbsq1cJpF7dT3Fs0lwzbpHDmSQjP+NdRqOqrpQj3QSXHnHaBDyQf/r1FCi6WJimxVpqdB2Rv2NqbiRAwwCSckenpW8ibTKPcfyFcroOvtcXsdnLYT2zqrOolPJGK3Li/kiuEaOAsjL8xzwDkY/r+Veu6sW7XPM5Glexbjb95JH/ABF93A+lVmyxtlBIO1sZHQ5rOk1aQXM0U6xRXAn8hGLkqSybk6D6Cq97qt9FounSJbxee04R2GSq8kfXqKFUi3a4crtc5HxTAw8RSNzuwpPFW/Bzf61CucLOOR6S5/rVfX55J9SnMxXzVYr8oxnBqrourHSRLMIRJ+/mQqWx12msZT5pHTCPLE7uWNSi7Ejx6hB+VS2SqLhTtUH12iuVPjzYpxpqfKe7+n4VGvxEk3kjTYRk8ZkOaFBXvcpzdrWNvRooLnTru2uYVmha4lRkIyCMiq+leFYdN1oXizyzW8SFbW3l58knrg/y+tYth4gmtYg0cMWLm7YkNn5cjPFX5fFN5Ax8uKH05BP9aj61RWkt0WoVUnyvRnTarJ5mmTgBThcnDAnqK4LUlXyvbcMZHHpWoPEd3dQPE8cAV1KttXBGaxtVciEtzu4xjtzXPiKsas04mtGEoQaZQBHPuoqNxxSxlQ7dPmXI5pG/THemkVcq3KlrWUf7J7VyFwv72MkdF7c812cgyjLnt61x1yCJVByODXTROasRlMjAGc9KwdTJGpMowSVXNdHjMec9PQVzmoA/2pIeeAPpwK6DAozkBeu3ivUPhNb7dG1G4K4aW4VRn0VTj9WNeZyDBGPlB7Y/wr174fR+T4bhU8F1LE+p3n+hFK4WOwU4Ze3Irz/xpoy6R4i+1worW2pZfYeiyj7w/HO7p3b2rvS27JrO8f2JvfCrTqMvZyLOAR/D0b9Dn8KqJLOBtdob/Rp3t3zyjjj8un5Voh5UUtIoJxy8WNw/A/jWbARPEu8AEjOcZBNWUadXCBEIU45zjFaCGu0csm9DK+3ndnOPqO31r1Czuo7yziniYlGX8QfQ1535yqPljTzEGfp+NdP4RufPtrsHCssgymMYyOv4/wBKzasaLVBqC7dQmGepz+lU846Gr+qjF+Tz8yg8VQPB/wAK86ekmepTd4Ihbqc9M9AadLNJcSF5W3PjBIUD27U0kE59s0mDk5I4pIGRTjdbyr1JQ/yrjfG0YksdLnxkFGXn6D/Cu2IDKR68VyPihN/ha1k7pNtz3/iFXDSSMqq91nmTj5qb6VJONsjfU1GK9E83qe+aZILjTbSfAzLCr/moqjrMStLbFxld5U56cg/4UeEZ/P8AC2nOeSIgn/fJI/pVrV7aSezbyRulQhlB7kV5j0kestYI5OeORobhUK+QlwDHjrz/APrFEC+Zb2/+3BJEefTmrsel36RmIGPY4VmyeQwxVtNKZWTBXakrOP8AdYdKrmRnys56aFlgs5A7N+7LHJ64IOK61CGRWH3SAazYtBMfEk7MFyACOACCCP1rUhiMFukeS2xcAt7Um7lRi0N/HPNeZ+M4PL1uYjo6o/6bf6V6djJ4/WuB8ewlb63lxxJCy/8AfJz/AFrSg7TMsQvcOKH3SK9F8Hyb/DyD/nnIy/1/rXnQ7j2rufAsu6wu4v7kgbH1H/1q6a69w5cO/fPWPh3L5et3ER6SQHH1DD/E16VXkvg6byPFVpzgPuQ8+qn+uK9ZNc1N6HTUWolFFJmtTIXNRTDKZqSmkZFUtyWZs3EkTe5H6UpNF38secfdYH9aTPrWyMWOpM0maAaoQufelzTM+9ANMRIDUsP36gBqWE/NxQBc59DRSjOKKAPlCEk3ilTzj6962zMqqBn5u5rDjdllDBhyuP1NTo2+Ub3woP3R39qzi7MqS0N2OWMrJI8gjVV27ieOlcjaRGTVYoUKn94BuHcDv/WtE3ct5qZhtlDokeCjD5ffiqd+kkM5uxiOXP3QMbTjtVOd3YlRsrnc9OTVeWSTLRxld2MjPpXP6b4hkuXEV0QGH8QH3h9PWrtxeRxzLJCS5Xrx1FacyM+Uh1LVSJPJIG4DnH0qrb3sZgTf1+ZNx9D0rOu3M91JNjcW/vdQKhy23OM46E9azbNEi3bhXlG47juGPerN8baKRVLBHAwCB+uapbkS2WRcb84PPSqk7F5PmZWPQEGlcLGr4alC+J7ADnMoH617JaDZBGvouOK8U0JzF4gsM5wZ0yM9sivbYh0HvXn4vdHp4L4WeYeNyx8VyYUkAxZ9uBX0fF/ql+lfOvjPJ8T3Z7KIv6f1r6KgP7lP90fyraPwI5p/Gx5php5pp4qkQyJuhqFutTNULcGtUZMaxwua0lWObAkRWHYEZrInOIWI7CqcOuzRKqlUYgD5j3rnrxk2nE6aDVnc6R7eGIgxxqpPXFT2zqyEA8g4Nc1Jr0kv8KD3po124T5kCDjHSolGTRrpY6ieLzLeRM43KR+lcD/Z4iYSGVztIY+mQa1/7fvmBBdOfRRWZeys9vLyRlDkfhVUU4JpkSV9RfEOh3OuagLmxkjkt/L2MQQfmBPv71haf4VvV1iQI37xY3Q5XA5GOD0PWt+0u5YITFE5SPP3V6VNBI7XCje2Seuahc8VZFuMZas5yPw/dWlzGjSrutptz+4ODxW3qrEw2p9J4z+tVDNNc3U67W3ggHGcnHFWZ47ia1hiW2mLh1Jbbxxg1yYlSbUmb0uVJpGwhVPGlqqdGgYfoK3twkSURrjdt4H1I/pWGbS4PinT7pUzEiFWOQMHbWraTlCgIyHLDORxhmP9aVKVomFVXZg6pbXZufMjhdyL20k7dAuCfwrXu4JZNMii8sh0uVfGe29jTdTuZVEMkUSsjMu8lsYx0qV9ThYyh2AyABjnuf8AGk69mEaLdmjivE1zHaarck8h5WIK/WudkvDBC4EZcSXR74xmNTWl4pDzXZFuSVDZ5rGuI2e227csJ4yfbMeP6V1UZOUbkTjyuxSvNQkRH8tQpxkZ57U7w9dvqTzJMQGigMisBjLAj9KbcWTyk4jfIGOG4/lTfCVjOL4EnYkkTLnqcfj9K2qXVNtGcPjSZ11zCkOlac6Z+aZWY57mi84JOaluYhJoEK5YeU3BHqDVK7hckk3Ep/EV40HzPU9FqyJrWQluPSm6lMxhYHkY7Doaq2kJ3jLy4HT5jUuoPmIZzwD07/jXXTiYyehkrcObjb5jYGcc+nantcO2RnBqE/8AH1yRjHFIeOPT1ruUUc3MxxmbnLY+hrn9RTZKrKOqdPetzv1+lY+rA4i+hHHTrWsFZkTd0VkwI1AJrn9QAOpS9cEDkHFdGuVVc9COK57UcDUH6Dgc1qzJFRlO77vP/wBb6f5969e8EKV0WFSMYjI/8dQ/zzXkqwSTTrFGMyO21QO7HgfzFe3aPYCwlltQMiKd4/TIww/9lFZOaUkjRRumzUGMdjxWmI0urBoZFykibW/EVmgegHpWnbHNun0xW0TFnjkUEmnX13YuDugk24buvY/lVk3sKriQEDPX0rd8caZFFq1vfMuI7keVIR2Ycg1zDweUQlwSR0Vug/GrEaELxyIfLIOR1781b8I3Lwa/5DNkSq0TfUfMD+hrn/Ke0bzYmDRnrjOOv/6609HnWHxJZzjhJHXOPfiomXE7PWhtuImPGUx+R/8Ar1mN1z3rZ1tcrA3oSP5Vj444Jrzq2k2enQ1pojb6c5puPXNPYcdeaMY9R9alM0GAdPyzXNa9GD4Tvkxnyp8gf8CH+NdPjkcCsTUoPM0nW4epA38/TP8ASqi9SJrQ8guxiVuvPPNQVavF+YHjpVXpXpR2PKe5678PJ/N8KonXypnX+Tf1rrZBliMcGuD+Fsu7T9Qg/uSo/wCYI/8AZa79lOR34rzq2k2epQ1popFAM8cVG3r+dWWXBxzTCpA5rO5rYgK4/wD1U1hwP1FSle3p2pGGQeMfhRcViDZ83Pr2rkPHltmxtZeTslKZA/vD/wCxrs8DOawfGlv53h6d/wCKJ0cfnj+RNaUpWmjKrG8GeRrw314rrPAcmL67iz96MNj1wf8A69co/wAszY7NXQeC5PL8SIn/AD0Rl/TP9K76usGedSdpo9P0qYW2sWU2fuTKT9MivaTXh20jp17V7Xbyie0hlHSRFb8xmuKkztqolpO9FBrcwEooNJTQineJmNx6g1XDblBx15q7OMrms6L/AFSjP3ePyraOxjJakgNLnFNzSE1ZI7NID6U3PrSg0xD81NCfmFV81LCfmoAvZopm73NFAHymg+bI64/TmoJpCkx2k89KlOFfsTgd/rVWfIk4HXtWC3Lexd8KkHUpz1zH6Z71d8VWyRQQygtvkc5BOR0qPwjGReXR/h2AZ/Gp/GT/ALqzHT5mJ/Ssm37axqkvZXOWiJWRSM9RmtQzbSAcgdR2zWSjFjtByOlWQHfGDgV0GBNI247u57jioz7cjtSgosZBY7gOCRURJBJPpikOw/kQNgnBbpTAqhflfJGO1Sq4FnMD13LgCr/huygv72VLhCyCPcACRzkUN2QJXZHpsZ/tyxxg4nQ5B9xXuKD58e9efwaRZW0gkgg2spDA7iefzrbTU7wv/rjyeuBXHXg6jVjvw81TTTOV8cHb4muRngrFx69K+i7c/uI/90V8yeLJpJdeDyPvJKA+/FfTNsc28R/2B/KtUrRSOeTvNslNNPAp1MNNEsjc1A5qZ+lV3PNaozZDOf3Tj2rmw3AFb95KI4ScZ5Arne/FRM1pEpJHNOViV601VJAPY1ba1EdrHP5sTbzjYrZdfqKzubogBwOufxpJ8tBL/un+VS7Bk0kgBhkA4+U0rjGWx6+nBq7bj/SUI4wc1RtSNvXPAqyrFcMDyO9S2NEisItWum6DCn9TWg92iQrIrAqE5A+lYU5Juix5JQZNTA/8S1vXYa4sV8NzWlHU3JdUijMAdGZskcHHVSKxNJ1O5a1njMhwk7hfXFOuzn7MR/fXP61Q0k4+2qc8XDf0rkUnyM0cUmazSs/32JHuaVG+RuO4qBTkHGKkQ/u3/D+dYSNYmFq5H2pifUY/KsubcDLuGDvhY/8Ajw4q/rWftXHTaKyrn/jzkbrxEcg/7Z/xr2cP/CR59Ze+yd1uDO4t3VScZyR71X8PF47+FHPKl1/DJo+cSk7SRtGeKj0STOqKQMfvWH41tNp02jKMXzpnVvzo8w7B3/rVK6/1YPHTrVsHdpt2uekh4/Cs+d8wIeDlRXjwWp6MtiKBiHXmnX/NvuOPeoIz+945qbUWxZdMkeld1JanNLYyT/x9oAD0A/SkJ5HehSPtsBz95e/4igjBrtRzjcfmaztUXdbR45PmY/nWiR0/wqreRh4hwMCTkfUf/qq47kszZWCTRqD0HSub1TIvmHsDn26Vt3Tk6iRwQvFY+sJ/pnvtBwM55z3rV7GaNv4fQ2k3jbTTdlRDHvl+cgAMillz+KivX4VX+0bpgysv2sEEHj5sf/F14p4UTzdSibo0cbHB78Y/9mr1LTL7cZozgFXjcYHT7o/pXHUpvnU7m9OStym5Km3BA5zzk1atGzAOeQTmmXSHySRgd6bZN8jDuGNdVJ3MaqsU/Flh/aHh+4VVzJCvmp+HP8q8+hkM1ur5BbGCG716wSCOcYPY15be2J0nVbm06CNt0bY6oeR9eP1FbGJUM0aEq8RXnk4ziqzxSpLG9vzCDvVgfun/AArQ3JLkZXzOw61nTWZjYuuUz129KUlcpOx6PdTC80i2uBj58N19RWbtIGMHrS6HcC58MLF1eB9rc+9SMoz7eleZidJnq4XWBCVyMCk24qcx9aAnbpmsUzdoh2nPSs2WHzLrUICOJbf+hFa+36ZqjIuzWovSSArz7H/69VclxPDr1du3qSMjkVRHWtjW4TDeXEfHyTMuPoSKx69WDvE8eatI9B+FcuNS1CD+/Er/AJHH/s1epFcqK+fNK1WfSZnmt5ZopGXbuifacenQ+grTfxlqb9bu9I/6+mH8gK5q2HlOd0ddDExhDlZ7U6HOcc+9Rsu3k4H6V4e/ia/c5aWZj/tXEh/9mqtJrE8v30Rv94s38zULCy7lvGR7HuLz26AFp4k/3nAqvJqemxjD39qvbmZR/WvF5NSiNoiJbKtwD80hCkH8Nv8AWoP7RuezIPpGo/pVLCeZDxnkezNr2jrjOp2ue/70H+VZOu63pF1o15bR38bySQsEVQTlscdq8u/tG7xgTuPocU0310etxL/32auOGSd7kSxbatYjnGJTwfXmtLw7KIvEli/Yyhfz4/rWUzM5yxJJ7k1PazG3u7ecHlHVvyNdEleNjli7STPamTINep+HJxceHrJs5xHs/wC+eP6V5p5YYZzkY6j0rvvBshbQzGf+WcrDH15/qa82k/esepWj7tzoPeg0ZpK6jkCk70tN6c00IRxlayR8ssq9MNn861/0rJuBsvWH95QfyzWsCJoTNGf1pmefejPPStTEfmjNMz1ozxSAkDc1LCRnNV881JEfmpgXePQflRUe73ooA+WgAWHTgD86o3HD8kdT2rUtWHmHdAH+UYzzjis242tdHGAc9+KwW5b2I4rmSAN5bkZAzz6VHc3k90R50rPt6AsTip/s+yBpGYNx25GKhtI1lmIcHAHQVem5OuwyHAPI57YqyDn+IDANQRhVudsn3RnI/Oo2Y9+tAEolzwDwOKf5UzgsqEj1xUOB5JO3r0q1p7ssgXdwynjr0pDFjhf7JcF0KMMYJrX8Fhj4ltoMblnPluvPTPaqkizSxNHwCAAD0xitzwDA9t4y05nK7RKucfWs67tTbNKPxo9Rm8MRQvteKWI4+63HFVf+EdWXULK1gPltcPs3NkgHFehaqY9QvvNiyV2gDIrndZD6ZNp96gy0U4YL68HivnqWJn7fkUro9uXJKj8KUjxv4jaPNofi+SxldZHTy2yo4IKg19EWh/0WL/cX+VfPvxT1Y614ylvDEIhtjXZ944Ar3+zP+hw9fuL/ACr3/so8WzTdywxpp/WkJppNNAxHPFVnPNTsarv3rRGTKl2m+L8QeawVAPX+Vbd7JsjHAOWArGQFTuxms6mhtR2JU6BR69alVG3hOMkjHzVEuSc9KkjO0/erNs3SZPdW01nOYpdpYAHKnI5GagcM6sADnBqTzGKlVU59hTvImZSVR847KajmRVmVrSJgigjHyitC2tI5nZZLhYQFyGYZz7VWtbO4aNCUl+6Odhp7GKJiGljUjghpAD+tK6Y0nYqzkCdef4Tz+NPX5rBwf7pqtdXdvHdKxuINqoQxEqnHT0P6Uz+2NIWxlLapbLJtYiMtkk9q58RFuOhpTkk9WXLps29s3PVP51S03JuL5f8Aptn6cCr91g2Fo8DCRHRGDYxwazY5pobq68rYSXBOfpXLCneLRrJ6mmAw7HFSxxM8cpzjC5x+Irn9TvdTsGAdlG4bgFCmqdt4muLZpXumMkZQrtUqDnj3qvqkmroz9vFOzL+qxE3SE9wKzL79xbz4xxGpz9HH+NQXPimGacM9tJtxjG8A5/Kla7TULN5EjaNWhlXG4NyCprupU3GCTOec4yk7Bfaqba8EfJ81Qn51X0f5bxmIw32ngDsOM/zq6BCCgxnK55I9vrVexZI72ckBT5oIJcDsPatuVcrM+Z3OjRwLe9UkD5wRnjqKoMDJbIFIJ2jjI61ae509EmRprcKzZG+Udxj2qsNY02C3jQ3VqnGARJz/ADrzVSlfRHa5xtuVo4pBNyMepq5eQGS0KkjkVTfXdPY5+3Iy9cgkinz+ILZ4lgjuCc4AG0jNdMIST2MZSjbcz2Tbb9iwGAQOnJpGR24COfoDUdxevDASsjqGz0J/z3qnLqLhN375xjgDJJ+gzXUrnPdGh5EpH+ok4/2DUbwyCNyyHC4P0qKez12GJJW0K7KSPsX5o/vYJ5+bjgdTjt6jMcdrre9mfSBFCv8ArHe8hyo9dobJ69BVqMiXKJzkjZv3GR16fjVHUxuuwScfu1z+tWUf/Sbhs8Zxj2qtfn/SwQONijp04rSWxmty54U41gR7gnyPkt/X8q7qzlWO+lUSrlohjGcHG4/0/SuC0V9msIytjKtj8q6eymI1mAE58xGXn1IP+NRa6K5rM9HuNRyvkhBjAGe/QUtg4JcjqCKyd4YRt6opzn2FXtOkGZBjHH4VrGKWxnKbluawbvXHeN7YJLZ3wwMgwucfiv8ANvzrrN/XB71i+KoftGgz4HMZEgI9qsg88nhWQjaSj+o71D9qvrJf36i4g7kffA/rVyIK+AR83UU6aJjHt8tmB5GBmgZr+GLyJ5Joo2zHPHuXHYjFdH5JIz+PNeZafLNp+oiVCVMb5ZOcEGugufipplpI8TafdtIv3gSijP5mvOxVOc5JxR6WEqwhFqTOu8ntis3VLtrKRAqZ3DOTXKyfGKJc+VpH/fdxj+Smsu/+Kct26n+zLT5QQNzs3/sorCOGrdjoliqPc6k63NuwqLTUv5rjULZljTeAwG5wg59SeK4iX4j3Lgf8S6y46ZQnH61CfiLfZBWwsBz/AM8T/jWn1aq+hk8VS7md4mO/V744AJmYkA5wSa588Ve1PVZNUvZbqaNEaTGVjG1RgY4H4VRY5NehTi4xSZ5tSScm0NopaUKT0BrQgbRVhLG7lx5dtM+f7qE1aTw9rMozHpV6w9RA3+FLmQ+V9jNoraXwlr7f8wq4X/fXb/OrCeBvED9bSJP9+5iX+bVLqQXUapzfQ52iurT4fawxAeWxjz/euVP8s1Ovw7vDy+qaaB/stI3/ALJS9tT7lexqdjjad2H1rrZ/AU8ajytRt5WzyPLcDH1xVe48IXSJH5cwkbGGAjIx6cnrS9tB9Q9jNdDpbb4hWEdjBG9pO0ixqrHeoyQBnvXo3ws8UweIF1OGKFojAY2wzZzncP6CvEF8G37MQZIlHbdnJ/SvQ/hVZnwtrl097eQeRcwbDjdwwYEdvrWHJSTvHc6FOq9JbHuVJ9aojWLA/wDLwB/wE/4Uv9q2OP8Aj4Ufgf8ACmIuUh/SqZ1ax5H2lP1o/tWxP/LylUhFuszUcJcwN/eDL/I1N/a9gePtUeazdV1jTzDEwvIcrIBy2Paqi9SJLQfuoLcVmnWdN7X1ufpIDTDrunjj7Rnv8qMf5Ct7mNjU3cUbjWO3iGyAyPPb6Qt/hUbeJIMEra3knssY/qaLoLG4G75qSJua5s+I8YK6fcgf7ZVcfrTR4mkDALaIp/25wD+gNF0Kx128+lFcl/wlj94Igf8Arqf/AImii6HY8Us7hIZT5jAfd4/CqV4ITI7CIHqfvHk1ZgtVudxcuCMdOe1SGzTedxJwOaySKexmo2dPY4IxuAwKqW7urMUHPrXQiyiNuEVT8xJwMiqdlaItww2kMB3+tO4rGSoaSbodxowy5DDB6Ee9bKW0f9okbAfmzz37+tQyxxrKwZF3bscjvS5h2MsM+3k9qltN/wBojweAavgBQAqJ/wB8irdvcNCT9wg9ex78UuYfKRrcqTja2Sey1t+HNQg07XbK+mErRI+5gqZOKrwywSN8zEE++M12PhrVZ4b23traO1TcwHmG3Utj6nmsMRN8jVjooQvJantuh20WpaZBfRuVjmQOqsuCAfWna34WXVrWOITiLY+4tsznj60/R73baqXEskmOWCcH8uK0m1Btvy2srcdsD+ZrzMJHC8qbWo6s6qqPU+U/iNa/YvEbqG3A7TkD619BWbD7HD/1zX+Qr5/+KEobxY+1Ng2pgEjIxmvfLRh9jgOf+WS/yFenTu6cWyallUZb3e9MLUzd70hYCtEZMGPWoHNOZ6hZ855rVGbKGpNiEYzncv8AOsY67GgK+ZaA9MeUmc/jWtqB3QjA6MD+tefvo9+9zKyy26qzkjdJ2z6AVlVpxluaUZuOx1H/AAkiwg5uolwOqxoP6VTfxq/2f7RG+oSxfLh47VwDnG3B2Ac5GPrWSvhy5lGJLu1X/gTf/E1dm0rVZ9Ik06XXbeOORQpeG3O7g5yDuHv+Z9sZRo0ups6tToOfxfqLthdM11vcxFB+pFQt4k1+ckQaLcnt+9vo0JJ9i1Rf8Izckkv4rvQTyfKt0T27GpF8J20g/wBI8Ra1Ljv9oIH061p7OiuhHtKrLNk3ie7077TDb6ciqxiIe7MnzA46qMdffsfbPGNNNcTSyu+0ytvKqykAnrg8120PhjSba18hNR1JoyxYr5y4JPJJ+XknAoTwzoES4CXBHQBpcflgCmpU18KE1OW7OSjZhDOAxJyhyce/oKx7hS1w7FgGxg5P/wBavU9K0vwsmozLfRstt5II3SOcsD7GtZrX4bI5drJZG6nKyn+ZrSKTVzKV07EvgfTbbW/BdpJczBLmEeW+xtwAHK/oRW2/g7TGclbyZGb7xVhz+YrJsPEfgzQ4Wi0yxkgjc5ZYoQAT68mpZPiDoijKWE7fUKP61ySwvvXTNlWlYS6+Gun3swefWrsgcKAEGPxxVV/hNo+0BdUvW998Y/8AZKe3xJsQf3WluRjvIB/SoW+Jef8AV6ao/wB6TP8ASrVBpW5iHJt3sTXHwm0fyFMd5deYBySy9fptqnqXgiHT7zT9M095XiuYp1aWQbgjFRjOBx0/SlX4j3UkoQ21vGCfvEM2P1FVZvHN/cXbbRAGtd0iEJ1+Ug9T70/ZW+0wUpdgT4SXSxqw1eDOOhibj9agHwxnhjmiNtZ3MrIwE5z949+Rn/8AX7CnXHxA1U6eksN9H5xbaYxCv59KzX8c+IpD/wAf7KP9mNRn9K1UbdSdR4+Gut+WkY03Sht5DM4wc9eNhqyvw68QHZ+60SPaAMYbjH0UVlv4q8QyZzqN3+Bx/Kq7a5rk33r68PHTzWqlyiszq7b4fa5HaGCS/wBPGXLkrG5xk5x2/XPb0FMi+F7RFWm1dBtx0h/+yrjpdQ1Bj+9upT/vSn+tRDUo0b9/eRLgdGnHX8TQ2uwKPmeijwJp8hEcuq5AzwqL/XNI3gHRUQ79TkB7OiIpHvn1ri7O8AjEsV5asGGRsuULEf7oOf0q0msGQY3E4/CuSpTqSleLsjog4pWZ1eo+H9LvbdILrxDqjxxSLKgiljQhxnByEyOvrWNP4X8Oojt9s1i4kUEr5t85Geo44zzWf/akSr828N9P/r1CdXj3Y3KfYtj+lONOp1kx3h2PPIgFuJRuJbPPuar3rh71o2+6UBz+FT3WLfVpl29WPU9aoaoJGuFZD83lBvTHWuuS0OZbnQeE2ii12LzYkkcK+0OoYH5SehrtLi4xLGywwIFccIgX+leZ6LfOb+B02iUPtO/JH5CuwluruRdqyWwPXHlkZ7df89KjlL5jqra+xDGrHG1AOvBx+FWYNUjSQhiSAOa4tNRvowN9uJO37tgcfnipoNTdWbz7e5AxwBEW/lnmi8kJKLO2bW7cZwG9uKr3WsxywSxeUzBlI69q5b+0mYHZYXzZ6YiA/mRTWv7rHy6VeHnqfLGf/HqOeQ+SJmSF4ziMHcvAwM5p8erSRp5VxEUc8BuSKPMcTnzIDCzciNiCf0qdV8yLY8Yx0APWtlqjJ7mIpdNQYyPvZ1zkVmaj4duNTvQ1oYgdvz7ztArb1P7NatGked45HOcD+lS6Y7fakAKjf69PpWc21saRSejOaTwHqTdbi1B9N5P9KsJ8PrsnD3sIOcfKpNejLYuRkyKM88DNSfYVzyx/BTXP7aZ0exgeeJ8PVwC+ot/wGH/69Tr4AtNozdXDH2VRXfCzUDJZuuegpBbAZ4PP6Uvaz7j9lDscCPAlopzvuGHocf0qaPwpaJtzaIdvqTz9fWu6FsmfuHI96kFtGM/L37jNHtJ9w9nA5aHTLRcf6Baj6W6E/wDoNa1vFAgwBs9kUL/IVq/Z0B6dvTrTlgX2rOV2XFJFNVt/785/7aGpDBasPmSVvqSateWmchsUYUdx171DizVSRVFpZjj7P/46BTxbQD7sAA+lTbowPvqPXGOaaXh6+aB9BS9mw50NMaKNqoAOgwM0woDjavt0qQzxD+MkfjxUbXUY4w3FUqbJdRDGg3ZIBzUbWpOMD9aebwIchGP/AAKmnUGxgKv51XsmT7REZsGPRgPahdKbeGDx8Hpg0pv5OvTPTimveSkfeP5VapMh1Ea/9o6qDnzYSB/snig6xqoAxLFn121im6l6GVsexphmYj7+fatFFmfMjb/tzVVI/wCPfH+0tKfEt8uN6wd++P6VgtJkn1qNjg9OlWokuRvyeKrraNqRZ7nr/SqFz4hvJ4JI8R7iOMoCuc+mKzDu5GMfjTemR1GapRIbHnV9Swu0W6H0EQqNtV1UnidVJ9IxTT75596YcDuSO1WQSfbtRPW9k/4CAKjea6fG69usDnCyYz+VJzwO1Jghj1z7DrQA0tJnJmuGPT5pWP8AWiOKMkkx5oJOOM5pUZgc5P8AjQIsFEBx5afnRSbv9pPxzRQM5CwjcbwIXbJ4wpPYc1I9vdMxK28uOvEZ/wAKqNdTZCx3Dhf7iOccfQ01Y7qckBJ3+iMf5VncqxdW3u1OTFIPquP51We0lRjIojU5JwZFGani0PU53HlaZeSHB+7Ax/pV1PCWvv8Ac0W++XGcwEfzpc8e4creyMp7VmfeJ4lfjOZV/oaWS3SVRuuoA3cgk5/IVsx+BfFEwwuiXQA6bgFx+ZFX7b4ceK3j2HTGjB5O6dAD+GaTqQ7jUJdjk3solGGvYz3OA3+FRskCcee57HCe9d0fhN4mmIJNsoGOHm/PoDVmP4M6y+PMv7OMexY/0pe2prqP2c+x52HjVRiSXjp8o/xq5b69PY3CSwvKHU5U7RjPrznNehp8F5xgy6tED/sxk/zqC7+EVvZQmS4109eES35Ptyx/lUuvSejZcadVao5n/haHiRF2pfThO+3YMf8AjlU7v4heIbobZL+5YAnH73b/AOggV09r4F0WAZuFubo9w77VP4KKs23gvQYMuLAsCfuyOW/Ks19XjtEvkrPdnmy/bvE2qW1tGglu55FjjLOfmJ4AJY9Oa+nbazvEtoYzbtuRFUncOoH1rgtF0PR9P1qzvhZQW4hmWQyklQgB69cY+teiN4x8Or11my/4DKD/ACrppuNRdjCopQeon2S7z/qef94f40GwvCPuJ+LVG3jbw9/DqSPnpsjdv5CmHxro5+5Jcyf7trJ/Va05YLqZXk+hKdLvG/55D/gR/wAKX+w7hus0Y/Amqx8Z2RHyWeoN9IMfzIpp8aRYymlX5z0LeWM/+PU7wXUOWb6E8vht5l2vdj14j/8Ar1w2p6ZPp2rT2yb5ghBD4AzkZ/rXXN4ymIHl6RIc/wB+4Vf5Zrz3xLpN/rut3GqiKO3WfarRCUtgqAAc4Gc4qJyg+pcIzXQ0BFccf6tT6GVR/WpFtJnGWuLdfYzA/wAq55fD+oIADctxwMSGpl8OXbxF2nDFeo3k8etZ+53NPf7G39iIPzXluB7Mx/kKd9mtl5bU4l/4A3+FYH/COs33pAD9OarXXh+YMTC8skmOVU4B+nvTvAVpHUBdLUENq6knsqf4mmtcaNGDuvpmI/uov/xVcMbTB6PgHByTkU9bIFiWTHrk9PSn7oanRXWqaUsm6K5lAIwd8fX2wBWPNrtuCQkdw4HGcIM/mwqNLaIH/Vj3zxSm1iONqjHcdxRdC5WQ/wBsq33bWcsP7zxj+TGmNrLkcWkf/A7gjH5KatrboFJCpnsc4I+tOWOMZDLFjvgYp3QcrM86teHlLW1Ht5rt/wCyCmjVNRfGxbUY54idj+rCtQRRdRtIHtTjGMgHa2T1FF0HKzLa+1NxkTRqfVbXt+LmmpPq6/Mt1ICQQcRRjg9RyDWtgAfK+RnHWgiL5htYADrjIouHKZONRcEG6uPQ4Ma/qEpptrz/AJ7XR56m4YfyIrYIUgAMD+gphKIeu3HTtRcXKjKa0nbgyS7f9qeQ8/8AfVRjSR951jYD+/ls/mTWyZY9uWKnjqRimi5t93oP9ocUXYWRlDRlP/LO269oVz/Knrpwi+6EUd9saj+QrS+0RnkNuGfT/P8ASozcAEEeYD05p3YWRUFmyDHny89g/Wmmx6bpJjnnlzVg3Tfexu/Q037R1JV1I9DnFGotCu2nRkgtkn6nrQLGMZATHXjHapzMxBIAZcYxUZkdVwu0nHAIp2YXRkayuydZFA5A5HFVrmU/6O4JwVKk/j/9etHUFNwseBkgkEgY7VD/AGPJdQ4jYKVYGqtoTfUz9MxbeILaQoGTeGw3GQK9Aj1eyHLwlM9PkBH6HNcBZqzXcLglXjJBIOD0NaZeUsdhY45xu5xUctylKx3kGp2MgG25iRfRhs/9CFXk8ubmORHGOi4avMWZimXIAbpuGOntj/P8gIT0GS2cFevb/wCtUul5lKr5HqAg6cDPQ8Y/lSmId3Iyex9//wBdebxy6moJinuOByBLwPwzVpNW1aAhWeIjPWRcZ/Ie/wDnil7MftUdNq9orlHRhvHqQTWau4YGSW6EAdKqx63JMQskSNk/fWVRjnrtzV1XDklcYb0raCsrGUnd3Mq4tnO6V1yS2c4ycCkhcxOjhclSPzrQeOWZG5Hl56k1mMPnA6gHHPvwalotM7qzvoZLZcqSVHt+dTm8TnCk/XFctpFyfL8vPHYd/atXzPUD644rL2aNfaM0Dejp5Q/OmfambkKoqkATnhseopQTjhv0o5EHOy0bqTJ4A96YbiXPL4/4DUALAcgHNKd3TA9iDT5ELnY8zOcgseaaZHODmT0puDnHIHuKXHsSPrRyoOZjd7EHO4n3NNyWGe31qTap4x+RpCnt/WjlDmITk9X59jQCwAxn8RUhUg9fz70woeuP1p2Fcbk5zuP5Um5jnmnj5ednPSkJVuoOfpRYVxhOR05pDIfTPb8KlAT1AzSBVHT09KdguM85gBuXikMmOuc1ISMgcZ9aYcZwMZ7nFFhXGF/rgUeYNuTTsDPGPwFIR0INOwriFlI6D8qQ7CP/AK1IwJ5yMdetMaNs5GAadhXHYU5H9aaQrA4/DJo2uCSWH1FNw2OSKADaB3FJjHfp2p3z8jC5FNPA5Ue4xQAwk565pvTjNPLqOGXikDKccMeOoFFxWIyevU8f3acp56CpAoP3d2PUjrUiQEjAGB70XHyjODyVGe9FWRAMdE/OilcOU9yXTbRFAS3hUDoBGKlW3RRwB+VFFeGdd2BjGfSgwjuTRRVJDuxhRV7ZpCAOcUUU7FJjWAHYUmflzgUUUi0V7iUxQtIADgdK5K78y8l8yaQn0AGAKKKqBZVFohcAk80jW6E5OeOgz0oorQQgt45Mqw3A8HdzmnNbQRSMkcSLtOPujtRRVxZLQqrxxgcelOdcsF9AKKKq5NhpGDjrz/8AX/pTgoaFT6MQKKKaEMCDn6Z6VLHGGLKTxtJ/IZ/pRRTAjaFQc0RoqyhlGCPXnNFFMRNNZxo6leFYBgMdM9qi+xIxYE49wORRRQBha1psP2M34yJFba4/v4bGfY/55rniFCD5R8wooraGxjPciaVVRf3eQzY5PTvnp7VELtljBA6vt560UVoZ3Hi5dmUYGcDnHrUUlzImw5GG9Bj/AD0ooqrCbGNdS7xhvmB6kCnLJL5nlvIzAjd6UUU7E3FbKMGLFvmA/OmuWDr87fN056CiinYQwfNGp6ZFKshAVG5LHGRxRRQA4KC23GDjqOOlCx4/iPWiikJimEbhgke4pme4J69zRRTAQFA20rndxnNRyzeWxIBJz1JoooAhFwJCD5YB9c/59avxQCZc5C/RRRRTEPnsDBF5rSBwZAMbcdfxpVh2cBjwRg0UU0JnNfZxHfIQejgHjrzXaR6NZyPsCsD9RjpnsBRRWaLB9LtYJAoVyR3EjDP4ZpVsbUqSYVb2f5v50UVQCC1slIH2OLJOOFH+FWGtI4Iw6pHz6JiiikMRSWA56VnsSZzjjJx0ooqokyM7UdXltrj7HGgYkFt8jE9z2GPSsc6uWeRZYFYpxlG25/A5ooqZblR2Lmjaz5155KwbOAc78/0rtMAgZz0B4NFFJjQvlgjcCRxn9KMEdGP40UUAKAfm5+lCpuPBI5+tFFAAYiuDu/SoixDcnPOOaKKAELkEA4NORtxVe570UUASTRlPm3559Pr/AIVACSTz6DpRRTADksRmggfrRRQAGEgE7vwxUe35j0/KiikA0LuBOcYpdnA5oooAaQVzg8CnICRnd9MiiigCUQ56sD+FL9nXuT+FFFJjFW1RwSCwx6nNS/Y0Q4znAoopNjSEFupb+HJOM7aayBRkYHpRRSuOwwDgMDjOacke9sEj06UUUAhViVsjA9elOWFByQfwoopDHGJMn5aKKKAP/9k=" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAAoAJ8DASIAAhEBAxEB/8QAHAAAAgIDAQEAAAAAAAAAAAAABwgABgEEBQID/8QAOBAAAQMDAwEFBgQEBwAAAAAAAQIDBAAFEQYSIQcTMUFRYQgUIoGhwRUjcZEyYqKxFhdSksLR8P/EABkBAAMBAQEAAAAAAAAAAAAAAAECAwQFAP/EACsRAAICAQIEBAYDAAAAAAAAAAECAAMRBBITITFBIlFhsQUycZGhwUJSgf/aAAwDAQACEQMRAD8AePU3UiRZr5ItcW3NupjbQtS3CkqURnjHhiuXpTr5pzULilQlxJcdtYbfdgTkSOyUc/xJA47jxnPB4NcTXbYb1ncNyQO0bZcH+3H2oM9OukjPT3WeodR2uQ0m33tKFCMhJKkubyonGMY/fknwrna5jRawDYwAQPOdbSILalJXOSQfSOs3IbdSFtrBSsBSSDwR51W7Z1E09cLv+Cx333HlOqZSvs/yytOcgK+WP/Cvvp9xSLLDZcUFFqO2gkHIOEgUFdOOKg6nmDdgwb44jxyB2mfuatYSNMbh1GPzI1qGu4R9fxDZeNd6bsFxRb7tdmmHVpC9uxSiE+Z2g4+fkfKu+iQhxoLbUFJUNySPEEUtPVJxaepy9yjtk2lpweHKVqT9qOGlLgqXpa2v78qMRtJV6hIB+oqlqBLQg6EA/eSr8VZfuCRO5FvUCVJXDTOjmQk4LQdTvyOSNuc1uOOhtG88+HFLt00mOMakjSH1bnkz5DKyrvKitaTn15oldYbrNt3T25TIEhxhxPZZW2rCgkuJChnv5Bx86lXWXuanuDiVfwVizzEICHO0QF4ODzXsr9f3oS9FrrIeauMN+U86MMut9osqCR8QOM93hRUSrcnNRU747KVntUhtKtpVz5VlLqVnAUDnyoW6g1lem9XzLTEfSwxHDaUp7JJKiU7iSTnzx8qqWkvaLsN+1G9puFdg7cohX28STDXHcwlWFEEjacHyzxzjxqOovXT2GtgeQB5eRlKqmtTeCMdIwBVjjvrG4E99acK4MzYjUtlR7N1AUPMenyoVN9eoX+YiNGIsLqmnJ3uHvhkAYc37M7Np4z/NmrqpdDYvQSZIV9h6wvqUPKvJUTwD61w9Xaqt2krBIv10U4I7GMhtO5SiTgJHqTx3gVVtA9atI69j3Jy3uSoirYz7w8JqUoPZc5WCFEEDHJ8MjzocNsAgdZ7cM4zCET4fvWDz3VRNE9ZNDa5vT+nrDdXZE1htTpQqM4gFCVAFQURjGVDxzzV2ffDTZX347vU0jKVbaesojAjImcfpx6VhRBzwPnVVHU7Q4nm1q1hY/fEqKSx7+0FhQOCnG7Oc8YqztupdSFtqBBAIxQIEfMEXUxnstYtubTh6Ckd3BIWftS/QOn970x1ua1RaZkxVnviZSp0cOLU0l3G4KI5ABJ447wMd9Mv1h0bqjULcSbpK5x4cpjchxT0ftdyDg4AyMeNDK2aG6xoltsTbnZFMk/mOe5rK8eOAFAZ+npWjXWAuGXB8OMGS0S+Da3Y5hk6fvLcsKUrJ/KdWhOfLOfvQvuKU27qFqSJkDdIZlJGPFaAT9aMml7QbbbWYqsqKE4JIxk+JP60Ket2j9YRb9F15oqAJ7gZMadCzguIB+FY88Z8OeBjPIp9Oos0rUMcEjl9RJWts1ItUd/wZyOsLzKtYaanMEYlRJDCjjnCcKAz8zRW6ZSO20dFRuyWi42e7wWSB+xFL5GvOpNSTYMSZoO+okMKUlkvs/lsKVgKO84wMfuPmKZPRNlXZ7CxCWdy8FazjGVHk49PCke823ICMYXEpwRVU3POTmBu2rNq1rdmP4REvi3AP5VKCvvRY6oRPfunl9ZSDkQnHE478oG4H+mhBruWiwdT7rDXDlLduimHo6WYy3Cs9mlJxtBHeDR3uURdx09JiutkGREW2pJ81III+tafk1pcdDgyJydKF7jMF3ROSkTUpKh+fASf1I2/9mjg2copU9D68sunBEXIu0BuVHbUy4w7IS2rjjBJ+VGfRHVa06luzNnjqirckJUpJZmpcPwpJPwgelYiVqudW8zLkF61ZfISv64HunURwHA94iNPJ45JBKftQ+6g6PhWnXGjtdw2ENOSX1QZCkjlZcGU5Pp8f70Q+rifd9X2KZjAkMusnj/SQf+VVvWjUrUttscKC62FW26sTV71dzaN27B55O7FNqGqp1S3WnAKY/UWgWPQUQZIaF3RUpTloLJwA24QPQYB/uTSu6vdVYeq1xf8A4VRLuZAzjIBcS4n6GmX0PvMNaiOFL4PnS9dereqB1JnuoGBKbjyAPM7NhP8AR/aj8HO/QOh/r7GDXjh6pG9feG7ri09N6X3RbWfyQ0+MHwDic/Sk3s1/maYuTzkNYQqS0/DWgq4cQ4ghSf2558QDjjl1JaBqfpW6jkm4WTIx37lM5H1pCdeRZqYaLhbmy67BktTsAA5bAw7nwP5al59K6FQL6RSP4t7iZrCFvPqIUfZlva4vVC1yCVBM5p6M4D35LauD5fE2KPHtMaoulm0CmNapTsdy4OLaccaVhXZhtRIB9Tj5Z86WHpXPZtuu7Bcm8BP4iw4VeaVKAV9CT+uaZv2lbaJugo7xGQxORuP8qkLR9xUKl42rIbvn2lHPDoBXtE+f0zcomlYmvYkiOYSJ5gyWinGzODuKh6Y7wMZpsfZO1nO1BpW5Wl15yVCtT6EwpJyU7Fg5aCu74doOPALx5UsrGvbxpfpvMtsW1uSYDFzXJuAbZQ4pLSmkhON3CU7m1HOM8eHiwHsx64vl1mps1wW41DetvvUeMthtCmTuRgHaO/ao8ZPd6VzCnCRT6nn+pvB3swHkIzC20uDBAIr5CA1vztFSpXRwDOeCR0n3bZSgd30rD0Zp1JQpIIIqVKGB0gyc5mj+BRN+8MJ5Oc4rdbjpbTtSAKlSkFSKcgShsZhzM1JNoiSZKJKmgXEcJVjkVte77UbKlShsCnIhLsRgwf3foxoW5yVy5GmLa4taitSjGQSSe85x31vaN6a6Y0ZJdlWOyRIbr6dq3G2UpUU9+MgZxkDj0FSpWZqlDbhn7ywtYrie+o3Tu1dQLaxDufboVGWVtLZeU0oEjnlJBI4HHoKHcH2eLdAkocavl87MKypsT1gK9D44+dSpS318X5iY9L7eghgslqatsRuK0nCW07QPIUOOr3SGZru8QrnBu4g7Ggw+DH7QqSFFSSMqGMZNSpTKzaevFZx2gIFzeMZl90/p+PZNOxbCwpxbMSOmOkunKikDHNL7pz2eb63rFci/uwjao/aNIQ3uWt9pSSnaoEAJ4Uc8ny9RKlMdRZWgrU8j1gWpHbeRzEqiPZr1lbdTItduVHRb2ZGY05xwlSWd2RlIGSoD5EimV6g6Ud1hom42FtxIkPtBTClcJDqSFJJ8hkc+lSpQq1VjWmw9RDZQipsHQxKpWmtYWCfPs8uxToypzaosmMqOpaHQrxSRlJ9CDx6ZxRH9nSNqVOvG3P8AD0+NEgNvR3VPRy0nYG8A5OOCsAAd5xnGOalSoamwvZg/WX06Ba8/5P/Z" } ] - } - ] - }, - { - "language": "markdown", - "source": [ - "# Load Text Recognition Model" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "// Loading the text recognition model\nconst recModel = await core.readModel(recModelXMLPath);\nconst recModelCompiled = await core.compileModel(recModel, 'AUTO');\nconst recInputLayer = recModelCompiled.input(0);\nconst recOutputLayer = recModelCompiled.output(0);\n" - ], - "outputs": [] - }, - { - "language": "markdown", - "source": [ - "# Define Post-Processing Functions" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "// Function to calculate the ratios for the image\nfunction calculateRatios(originalImage, resizedImage) {\n const realY = originalImage.rows;\n const realX = originalImage.cols;\n const resizedY = resizedImage.rows;\n const resizedX = resizedImage.cols;\n const ratioX = realX / resizedX;\n const ratioY = realY / resizedY;\n\n return { ratioX, ratioY };\n}\n\n// Function to convert the image to grayscale\nfunction convertToGrayscale(originalImage) {\n const grayscaleImage = new cv.Mat();\n cv.cvtColor(originalImage, grayscaleImage, cv.COLOR_BGR2GRAY);\n\n return grayscaleImage;\n}\n\n\n// Function to adjust bounding box coordinates by a given ratio\nfunction multiplyByRatio(ratioX, ratioY, box) {\n const scaleShape = (shape, idx) => idx % 2\n ? Math.max(shape * ratioY, 10)\n : shape * ratioX;\n\n return box.map(scaleShape);\n}\n\n\n// Function to resize and convert a crop to the recognition model input format\nfunction resizeAndConvertCropToModelInput(crop, netShape) {\n const [netWidth, netHeight] = netShape;\n\n // Resize the crop to the network's input shape\n const tempImg = new cv.Mat();\n cv.resize(crop, tempImg, new cv.Size(netWidth, netHeight));\n\n // Create the reshaped buffer\n const reshapedBuffer = new Uint8Array(netHeight * netWidth);\n let index = 0;\n\n for (let i = 0; i < netHeight; i++) {\n for (let j = 0; j < netWidth; j++) {\n reshapedBuffer[index++] = tempImg.ucharPtr(i, j)[0];\n }\n }\n\n // Clean up\n tempImg.delete();\n\n return reshapedBuffer;\n}\n\n// Function to extract recognition results from the model output\nfunction extractRecognitionResults(output) {\n const outputData = output.getData();\n const outputShape = output.getShape();\n const [batchSize, height, width] = outputShape;\n\n return setShape(outputData, [height, width]);\n}\n\n// Function to parse annotations from the recognition results\nfunction parseAnnotations(recognitionResults) {\n const letters = \"~0123456789abcdefghijklmnopqrstuvwxyz\";\n const annotation = [];\n\n for (const row of recognitionResults) {\n const letterIndex = argMax(row);\n const parsedLetter = letters[letterIndex];\n\n // Stop if end character is encountered\n if (parsedLetter === letters[0]) break;\n annotation.push(parsedLetter);\n }\n\n return annotation.join('');\n}\n\n// Function to crop the image based on the bounding box coordinates\nfunction cropImage(originalImage, xMin, yMin, xMax, yMax) {\n xMin = Math.max(0, xMin);\n yMin = Math.max(0, yMin);\n xMax = Math.min(originalImage.cols, xMax);\n yMax = Math.min(originalImage.rows, yMax);\n if (xMin >= xMax || yMin >= yMax) {\n throw new Error('Invalid crop coordinates');\n }\n const roi = originalImage.roi(\n new cv.Rect(xMin, yMin, xMax - xMin, yMax - yMin)\n );\n const cropped = new cv.Mat();\n roi.copyTo(cropped);\n roi.delete();\n\n return cropped;\n}\n\n// Function to log the bounding boxes with annotations\nfunction printSortedAnnotations(boxesWithAnnotations) {\n /* Sort the boxes with annotations based\n on their position in the input image */\n const sortedAnnotations = boxesWithAnnotations\n .sort((a, b) => {\n const [aXMin, aYMin] = a.box;\n const [bXMin, bYMin] = b.box;\n\n return (aYMin - bYMin) || (aXMin - bXMin);\n })\n .map(item => item.annotation);\n\n console.log('Sorted Annotations:', sortedAnnotations);\n}\n\n// Get Text size\nfunction getTextSize(text, fontFace, fontScale) {\n const canvas = createCanvas(200, 200);\n const ctx = canvas.getContext('2d');\n const adjustedFontScale = fontScale * 35;\n ctx.font = `${adjustedFontScale}px ${fontFace}`;\n const metrics = ctx.measureText(text);\n const width = metrics.width;\n const height =\n metrics.actualBoundingBoxAscent +\n metrics.actualBoundingBoxDescent;\n\n return { width, height };\n}\n\n/* The convertResultToImage function visualizes object detection\n results on an image by drawing bounding boxes around detected\n objects and optionally adding labels to them. */\nfunction convertResultToImage(\n bgrImage,\n resizedImage,\n boxesWithAnnotations,\n options,\n) {\n const defaultOptions = { threshold: 0.3, confLabels: true };\n const { threshold, confLabels } = Object.assign(defaultOptions, options);\n\n const colors = {\n red: [255, 0, 0, 255],\n green: [0, 255, 0, 255],\n white: [255, 255, 255, 255]\n };\n const [realY, realX] = [bgrImage.rows, bgrImage.cols];\n const [resizedY, resizedX] = [resizedImage.rows, resizedImage.cols];\n const [ratioX, ratioY] = [realX / resizedX, realY / resizedY];\n\n const rgbImage = new cv.Mat();\n cv.cvtColor(bgrImage, rgbImage, cv.COLOR_BGR2RGB);\n\n boxesWithAnnotations.forEach(({ box, annotation }) => {\n const conf = box[box.length - 1];\n\n if (conf < threshold) return;\n\n const [xMin, yMin, xMax, yMax] = multiplyByRatio(ratioX, ratioY, box);\n\n cv.rectangle(\n rgbImage,\n new cv.Point(xMin, yMin),\n new cv.Point(xMax, yMax),\n colors.green,\n 3\n );\n\n if (!confLabels) return;\n\n const text = `${annotation}`;\n const fontScale = 0.8;\n const thickness = 1;\n const { width: textW, height: textH } = getTextSize(text, 'Arial', fontScale);\n const imageCopy = rgbImage.clone();\n\n cv.rectangle(\n imageCopy,\n new cv.Point(xMin, yMin - textH - 10),\n new cv.Point(xMin + textW, yMin - 10),\n colors.white,\n cv.FILLED\n );\n cv.addWeighted(imageCopy, 0.4, rgbImage, 0.6, 0, rgbImage);\n cv.putText(\n rgbImage,\n text,\n new cv.Point(xMin, yMin - 10),\n cv.FONT_HERSHEY_SIMPLEX,\n fontScale,\n colors.red,\n thickness,\n cv.LINE_AA\n );\n\n imageCopy.delete();\n\n });\n\n return rgbImage;\n}\n" - ], - "outputs": [] - }, - { - "language": "markdown", - "source": [ - "# Async Inference Helper Function" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "async function inferAsyncProcess(\n tensor,\n recModelCompiled,\n recOutputLayer,\n i,\n annotations,\n) {\n // Create infer request\n const inferRequest = recModelCompiled.createInferRequest();\n\n // Define the completion callback function\n function completionCallback(outputTensor, i, annotations) {\n const recognitionResults = extractRecognitionResults(outputTensor);\n const annotation = parseAnnotations(recognitionResults);\n annotations.push(annotation);\n }\n\n // Start inference in asynchronous mode\n try {\n const result = await inferRequest.inferAsync([tensor]);\n completionCallback(result[recOutputLayer], i, annotations);\n }catch (error) {\n console.error('Error during inference:', error);\n }\n}\n" - ], - "outputs": [] - }, - { - "language": "markdown", - "source": [ - "### Do Inference and Show Detected Text Boxes and OCR Results for the Image\n" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "// Process each bounding box and run inference on the recognition model\nconst [batchSize, channels, height, width] = recInputLayer.shape;\n// Calculate ratios\nconst {\n ratioX,\n ratioY,\n} = calculateRatios(inputImageMat, resizedImage);\n\n// Convert image to grayscale\nconst grayscaleImage = convertToGrayscale(inputImageMat);\n\nconst annotations = [];\nconst croppedImages = [];\n\n\nfor (let i = 0; i < boundingBoxesArray.length; i++) {\n const crop = boundingBoxesArray[i];\n const [xMin, yMin, xMax, yMax] = multiplyByRatio(ratioX, ratioY, crop).map(Math.floor);\n const cropRect = new cv.Rect(xMin, yMin, xMax - xMin, yMax - yMin);\n const croppedImage = grayscaleImage.roi(cropRect);\n\n try {\n const preprocessedCrop = resizeAndConvertCropToModelInput(croppedImage, [width, height]);\n const tensorData = new Float32Array(preprocessedCrop);\n const tensor = new ov.Tensor(\n ov.element.f32,\n Int32Array.from(recInputLayer.shape),\n tensorData\n );\n\n await inferAsyncProcess(\n tensor,\n recModelCompiled,\n recOutputLayer,\n i,\n annotations\n );\n\n croppedImages.push(\n cropImage(inputImageMat, xMin, yMin, xMax, yMax)\n );\n } catch (error) {\n console.error('Error during preprocessing:', error);\n }\n\n croppedImage.delete();\n}\n\ngrayscaleImage.delete();\n\nconst boxesWithAnnotations = boundingBoxesArray.map((box, index) => ({\n box,\n annotation: annotations[index]\n}));\n\nconst resultImage = convertResultToImage(\n inputImageMat,\n resizedImage,\n boxesWithAnnotations,\n { threshold: 0.3, confLabels: true }\n);\n\ndisplayArrayAsImage(\n resultImage.data,\n resultImage.cols,\n resultImage.rows,\n display\n);\n\ncroppedImages.forEach((croppedImage) => {\n displayArrayAsImage(\n croppedImage.data,\n croppedImage.cols,\n croppedImage.rows,\n display\n );\n});\n" - ], - "outputs": [ + }, { "items": [ { "mime": "application/vnd.code.notebook.stdout", "value": [ - "Annotation for box 0: building", - "Cropped Image Size: 159 x 40", - "Annotation for box 1: noyce", - "Original Image Size: 690 x 517", - "Cropping Coordinates: (256, 50) to (377, 88)", - "Cropped Image Size: 121 x 38", - "Cropping Coordinates: (604, 205) to (653, 228)", - "Cropped Image Size: 49 x 23", - "Cropped Image Size: 26 x 32", - "Cropped Image Size: 31 x 23", - "Text: noyce, Width: 74.716796875, Height: 21", - "Text: 2200, Width: 62.2890625, Height: 19", - "Text: robert, Width: 73.14453125, Height: 20", + "Box 0: [391,59,550,99,0,0.518794059753418], Annotation: 'building'", "" ] } @@ -230,15 +143,18 @@ "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAIFArIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1SXSkkG1ihGQehHQ59a429+D3h+9klkY3SNKxZjHcY5Jz/Epr0RvvfjSgVEcPSg/djYp1pyWrPKx8C9HkJUahqKLjj50b/wBlFMm+BFkUKR6veY940r1uM7cn0BNYmla9qOu28F3p+nxJaNjfJcSkFjn5ggAOcdMnGT+dEuWLsa06NSpFzWy3bdt9vyPNZPgZtxs1F2x/eOP5IarP8GJ0JImd8/3ZlH81Fev3eqXT3sljpNtHcXEIBneZykcWRkAkAksfQDjvUunapLNdPYX8C29+ieZtVtySJnG5T9eoPI/Wl7t7DdCooc/ztfW3e29v+H2PEI/hZLIZhGLl2jco4W4iO1uv8jQ3wynjI3W2pn1IUH+SmvUtNuNTOqa5FptlBKov2Z5riQqmdiDaAAST39P6bum3yXrzW1zai3vrfHnQk7hg9GVu6nB9/XFTBpmuIw0qeq1Vk91dXS3XzPCm8CxwMfNGoJ/vqB/NajHhOyHBuLnP+8v+Fe2/2hf38jtotlbyWsbFPPuJmQSsDghAATgEYyevbjmpP7RtbjRLy+ltWWSzWTz7eQ5KOgyVz0I9D3BovFkPDVV0+V1dX7roeJf8IpZ8fvrn/vpf8KVfDFgZCq3FwWQDI3Lx+leuWEJ1sW93FollFp0gB3zhfNcEfeVQp49MnJ68VladoUs3i7VIZdP0/wAhPJ37ZHBVSDgqABknv0qG1pbqbRwkkp87ScVfdd0rf11Pn6JmwRuP3j1+tTozFY8Z+71r3uX4U+HGB8uzCHOcpNIP5saoy/B/SGACSXSAcDbcAj9Y6aptM5nNNHiW47mPJHTimFmxzz6HFewzfBqHJMOpXC+zRo/9VrOn+Dd8ABDqStj/AJ6QY/kxqrMi55iTyORSgkkjca76b4Sa9GcrPZOPTMg/9kqhN8MvE0IbZaW8mf7twgH/AI8RRZgcjvYJ94/j2pwLhgMn8a3pvA3iSHGdHnbHePbJ/wCgk1Sk8Pa1bkmbSdQQAdWtnA/lSsO5nglien5Uu48eh9Kc0E0QAlR0bOMMuOtAUlhnnFSUKCc4596z53ddTixwcD+taIBznHtWbd8XcRAznvn3qZDRoxSbo+uecCnb8DOc1HAdsW/vninE4AXB5pR2KYF+2Rj2oLEDP60Agk+1NJXbkjr61RI4ls5zTdxzycd6VsdOfejbke3Tr2qRiZ+7nIPajJJyePrS7TnHJNOCkHj0pXGAJFNY7kOPT1p+DkcDJ7UpU4OecjNSxmXZOymUc9sj860tiNhtuAeazYGKzSrj9a14cmBOOCKl7jWxD5KnGVNIYB/eb061cCgDPApSoxwuaLhYpfZ/4Qx980z7Kc84I7ZFaGwHigJk8UuYdigLIsASqE/lQ1mccJ36A1qCP0FIVzmp5h8pjG1bsjH39aja0YHJDqM9xW6UwOeMetMaM7CcHpRzBynOjcr+vap1i3ICcA+tJOu2Q8d61LBVe0AKg8kHjNDYJGU9kAhZokPfgCsXULXbGXSELz/CMV190keCgUY74rEvo/8ARZMdAc/jW1KTTM5o5bawPANHmOP4m/OrxT3PPrUezII6/hXZzHPYq+Y/948UeYe9TmMYzgflSeWCPu496LoRBvNLv4xTzGMcgimMgC5GaegaikngnPNN3nnnrT25jT2BH6//AF6j2mgBd+KXzD6U3BpMH0phckVwDyOKeZhjAzj1qHB9DSUrBcl3gjFG4VFRRYBz802iimIcjlJFcHBByDX25o999v0SwvAcie3jl/NQa+Ia+u/hfem/+Guhyk5KwGI/8AYr/SuLGLRM2o7nXlzSbzSUhrhOiyF3GjcaSkqh2FViHYZ9DT9x9ajHDj3Bp1ddPWJEh24+tG4+tJRViF3H1o3NTaKYh28+tQzMd8bZ74qWopwfLyOxzQxoXc3rSb29aTtSGkAu808OcVHTl6VLQ0O3tVS+lmjsriUT+XsQsCEzjAz+NWGOB0yT0FVLyyF3aTQzOxEileOAv0Hf8alq+g9jjpPGlsLaG4kt2lkk2sFKqCmf4s7jj6cH1qnrN7p8olDXVybskI0YyfL9to4BPoxBxWfd6TBeWF3ZiZkvNPYx+Y2CSoHPfv1/SuIW+2RQ2zTq08EmFwAVUAE4ORzg++M1jUpXi0ZOo0zrPtunAcre574lQfp5dFUl8WMFAdLgsByQB1/OivH9jU7G3NE90b71KOaRvvUor7Bnnoeoyrj/AGTWb4Qi8nwlpibNhEIypGOcmrstxDbIHnlSJScAuwAz+NRDVNP/AOf61/7/AC/41jKcIyvKSTKeKhCm6Umldp79r/5lCK7h0DVtRGpOILe8mE8Fy/3GOwBkJ7EbeM9QeOhp9tMus+I4L+0+extIJI/tAGBK7EZC+oG3r0z9DVttR01xte9tGHoZVP8AWnLqemqoVb21AHQCVf8AGs+entzqxq8xw9ua65mrXurbW2728zG0rWLLTNW1uDUZRaGS+Z43mBVHGxBwx4J9verumk6nrt1q0Kstk1stvDIRjzvmLFgOuOQAe/OOKttqOmSLte8tGHo0qn+tOGqacOBfWv8A3+X/ABpKdNaOasOpmWGldxaUmkn7ytpbb7u7ORsNL8OabbfYtcX7LewEhjJcSIsq5OHQ7sEEdhyD1ArZFraJ4N1VdPsZre3mglZBKW3y5TG7BJIzgYzz7CtN9R0uTG+8s2xyMyqcfrT/AO1dO/5/7X/v8v8AjSj7FaKS/A1q5zCo1Jzu73d5afJdPxsS6Onl6LYJt27beMbcYx8o4rJS5g0vxfqBvZVhW9ihMDvwrlcqVB/vcjj3rR/tXTv+f+1/7/L/AI0janpjY3XtocHIzKvH61TqU9LSWnmc0MfQUpuUlaXmu6f5ou0tUv7V07/n/tf+/wAv+NL/AGrp3/P/AGv/AH+X/Gr9vS/mX3nP9Yo/zr70XKKp/wBq6d/z/wBr/wB/l/xo/tXTv+f+1/7/AC/40e3pfzL7w+sUf5196LlHWqf9q6d/z/2v/f5f8aP7V07/AJ/7X/v8v+NHtqX8y+8PrFH+dfei0UQ9VU/UU3yIv+eafgKr/wBq6d/z/wBr/wB/l/xo/tXTv+f+1/7/AC/40e2pfzL7w+s0f5196J2tomGCvHpk1Tm0HSrg5lsLaTPd4lb+YqX+1dO/5/7X/v8AL/jR/aunf8/9r/3+X/Gl7al/MvvH9Zpfzr70Zk3gvQJs7tKsx9LdB/ICsy5+GHha5YM+mRhh0KvIv8mrpv7V07/n/tf+/wAv+NWldZEV0YMrAFWU5BFOLpz+Fplwrxn8Ekzg3+E3h/nyhcReyTHH/jwNU5fg5YOSYL+6j7Dcyt/7KK9JrH8VT3Nv4ekks5fKn86FVfsMyKOfUc0SjGKbsdNGMqtSNNPdpfeefyfBeXawj1nJPrb/AP2dULj4Pasm3y761cDpuVx/IGvQtX0ZdL0a61WO+vH1K3jMxuHmb5yOSCgO3aemMcCumD+ZFG+MblBxUqKbs0XVgoRU4Sum2traq34a/wDAPBp/hb4hiDlBaSn/AGZSM/8AfQFZ8vw/8RQAZ01WUcnbPG36bq+iaQgEcgH603SRiqjPmx/CuvIpc6NfMO2yFm/kKpy6Tf2+BPZXUXrvhZf5ivppYYigBjT/AL5FH2aH+4B9Dip9iUqh4UfCtjuOZrjj/aH+FH/CKWW0/vrnJ/2l/wDia9j1LWbGz1H+zvs89xetGskcEQyZASR3OABgkk8Co7J7O/vHsr3Svsl4sfmiOZY33JnGQRnv1qOWN7G3saqhztab/Lvbe3meIf8ACIWC3DfvrnJH95f8K0rfwvZC3RfOuMDP8Q5/SvUtSsdIjuza2ujfb7xRukjgVU8tT03NkAE9h1NJYaPpF+ssIs5LS6i/1ttIzB19CPmIKnsRxQ4JuwezqKHO1p/Wtt7eex5r/wAIzZY/1k/5j/Cj/hFbIgDzbj/vof4V6TY+HdM1Syju7V7hVbIKswBVgSCCNvUEEVTi0fT5NPuNQW9eK0id082RRg7TtJHIyMggetRyIOWd2rap2+fY4b/hFbDPLzf99D/CnL4WsR/y0nyf9of4V1DWFxHtmns7u3sieLiWIAKOxdQxZR6kjjvirGo6PLp0Ild45GkO2KKMMXlb0UY5P9OanlW5o6VSLUbavtr+Rx58MWRPEtwMD+8Of0rK1nSYtP8AJ8mSQmTdnfjtjHTHrXbLZ363MNvd6fJavMSIjI6lXPXGQcA98HrVDxF4e1OQQFLCdyu7/Vjdjp6VEo+7oglGUJJS/wA/xRwRhOSBj3OKYyN5fbn3rcl0TUYiA9heIAOS0LD+lUJbd4j86upHQEYNY6j0OTvkKuQQM5ycGrWnTFbVh1Ibik1FCocY+ZuTUOnfNC49xWiV0Q9yy5y2e59azb1AYZQegHWtIg4z3qldLlXGOCp960gRI54RtI4SNWdycKBySe3FSf2TqR+X+zrvB6/uW/wqxpRxrFgBn/Xx+/8AEK9N4IrrRzs8pOk6lz/xL7v/AL8t/hSNpGpcZ066z/1xb/CvVvoeaTJ79Kqwjyg6RqPT+zbv/vy3+FMfSNS2nGnXf/flv8K9ZNNyGUkHI+tMLaHk40jU/s6/8S674Y/8sG7j6e1L/ZGpHA/s67/78t/hXqy/6th6EGgc0xHlH9j6j/0Drv8A78t/hR/Y+pDrp13/AN+W/wAK9WJGQCRk0tA9jyf+x9SP/MOu/wDvy3+FB0bUhx/Z13/34b/CvWPwooEeTf2Pqf8A0Drv/vw3+FVHiaORo5FKupIKtwQfQ17JXk+sf8hq/wAf8/En/oRoAoEYptOI4pKYhK+mfgNffafh89uWybW8kQD0BAb+ZNfM1e7fs73vya7Yk94plH/fQP8ASufFK9M0pP3j3E0ZoNJXmHWhaKQUtUAh4Kn3p1Nf7v05p1dVLYiQtFFFakhSUtJQAtNcZUj1FOpDTsBCnKD6UppqMDuUYyrEH2p1IBKcvem0ucZx1qWNDgCTgDmms8IVw0ijHv0NTp5KrvldOR/EeKwLzWrYas2nea6Fl3hU4yO+ep98AfXFZSq8iuPd2OA8ZmWHxN9v06SNFaHZK0imME8jBYgDpn3ribTQMTrqWoX9vZ2srHylLY83nnHfHuB19a9V8QatBJaLBLbSTRthkiSEoiehyy7cjr1xXll34fvdS/eXd4ke5iwlnPl/KPqeePSuWddyVm7EuCT2udjHD4SSJFDIwCgAlGyf/HxRXMx2Nokao19bFlABO1+f0org9k/52XzeR9CN96lFDfepRX1rOBHGeL2Y6nChY7RCCBngEscn9B+VSt4SChib7hf+mP8A9lUPi/8A5C0X/XAf+hNXVSjKyL0yCK+ew+CoYrGV/bRvZq2r8+x8tQwNDF47E+3jezVtX59n5Hmej6p4f1q7itLfVZo55ThEmtCuT6Z3EZrqR4Ryeb7H/bL/AOyqKy8FWVtcQTkq7RMrhguDkcitX+1hLrJso0lHlJukZ0KgkngDPXoeRxXof2Lgf5Pxf+Z3/wBhZf8A8+/xl/mUx4NyP+P/AP8AIP8A9lTG8IYOBfZ9f3P/ANlXUofkrF8TRvP4cvII2KtMojyDzhmAP6E0v7FwP8n4v/Mf9hZf/wA+/wAZf5mXH4XjlBMeoo6g4JRA2Pyapx4OB/5iH/kH/wCyrH8H+HH0LVhLC7LDMhSSPPB4yD9cj9a7rzNsuz2zT/sXA/yfi/8AMFkWX/8APv8AGX+Zz58GBRk6hx/1x/8Asqb/AMIhH/0Ev/IH/wBlXTyN+7/EVxXjjUNesJ9POi3CxhlkMqNErB8bcdRx36etL+xcF/J+L/zH/YWX/wDPv8Zf5mgPBit01IH/ALY//ZUv/CFf9RD/AMg//ZVrafcyS6da3NwoSWSJGdRxgkDI/M1oq2RS/sXBfyfi/wDMP7Cy/wD59/jL/M5j/hCv+oh/5B/+yo/4Qn/qIf8AkH/7KtXU71rHTby9SFpmgjeQRKcF9oJwD+FcpZ/ENrlgp0LUVY8YQ7sfoKP7FwX8n4v/ADH/AGFl/wDz7/GX+Zrf8IT/ANRD/wAg/wD2VL/whP8A1EP/ACD/APZV0scpdQeRx3qN52Bc7sKvt7Zpf2Lgv5Pxf+Yf2Fl3/Pv8Zf5nO/8ACE/9RD/yD/8AZUv/AAhP/UQ/8g//AGVSaV430PWblLew1SOWZ/uxtGyFu/G4CuiSbcORR/Y2C/k/F/5h/YWXf8+/xl/mcx/whX/UQ/8AIP8A9lR/whX/AFEP/IP/ANlXR3F4kBQMQN2cZbFNF6CPuEj2OaP7GwX8n4v/ADH/AGDl3/Pv8Zf5nJ6n4X/s7T5bv7Z5nl4+XysZyQOufetjwe7No7hmJCzMFBPQYU8fiTT/ABDcrLoN0oVgfl6/761F4O/5BEv/AF3P/oK1x08NSw+ZRhSVk4+fd9zho4SjhM3jToKycL7vu+/odFmqOs2L6nphtY3VGMsb5YcYV1Y/yq7XO6h4sis72S3itjN5Z2sxfb8w6joa9XE4ijQhes7J6f1Y9+tj6WB5a1SXLrp119DZ1m1bUtGvbGNgjzwtGrN0BIxVlPlhjTP3VAzXJ/8ACa/9Q/8A8jf/AGNH/Ca/9Q//AMjf/Y1x/wBr4K9+f8H/AJHC+IsA4KHtNN9pf5eR1uaM1y0HjOJ50We0MUZOGcSbtvvjFdRXZh8XRxKbpSvb+up1YTHYfFpuhK9t91+Yi9KdmmL3HvTu1dB1mTCB/wAJ5dtgZ/s6IZx/00eluz/xWum/9ec//oUdam1RIZAo8wjaWxyR6Zo2qZBIVBdQQGI5APUZ/AVHJpbzudX1lc3Nb7PL+FjI0iWOy1nVrW6YR3E1ybiIvx5sZVQNp77cYOOn40sUiX3jET2mHitbVoZ5k5UuWBCZ7kYJPpn3q/e2FpqMQivbaK4RTkLIoYA06KKGwtBFbQiOGNSVjiXHvwBU8j26GjxEHeaT5mreW1r/APA6HNapdXGhajf6faAhtYw9ljok7EJJ+QIf860NZ05bHw5Y29tE8lvYzQPJGq7i0aMMnA6n+L8KW1huNV1eLVbuB7e3tkZbSGX75ZusjD+HjgDryelbe4+pqIwvf8DarifZumktVZy83svuX4tkEus6dFZpdG7iaGTAjMbbzIT0CgZLE+grC8S2wfVdIu57i5sbNFkjaaFgnkuwG3ccEAHBXPqR61rQ6PptveteQ2NvHckkmVYwGyevNXHVZY2jkAdGGGVhkEehFVKLkrMwpV6dCop079b3t1VtN9u/Xsc++l2MWoWST63qV7N5qyxW7TB8lTncQBwB6nA/OulkAOAQD9aoWOl2GmlzZWcNuX+8Y0C5q5knqc04xsTiK/tWrNtLvZfghvkxHny0/wC+aRreJhgrx9TT6KqyMLsoz6Fpdznz7C2kz18yJWz+YrPfwN4akz/xJ7Nc9dkKr/6CBW/RS5UO7OSn+GfhibJFhsPqksg/9mrMufg94fnJKy3kRP8AcmH9VNeg5ozS5IhzM8nT4E6ZBeQz2+qXi+W6uA+x+hz2C1Vn8MRW2rahZfbppJ4ZFCxRWrSMylFJbCngDOK9kU8isPQmi/4STxIFK+Z9oiLD+LHlDH4damad1bv+h14Xk5Krmrrl/wDbonn9p4GvNRgaayvbaRVYo6sHR0YdVYbeD7Gq0fgzV51lNvHDMIpGiYrKBhh1HzYr0yMqfHUv2ftYAXWOm7f+7z743fhR4X+VNWU/eGpzkjuMkEfoQaFJ3sKph4KDmuydvXueSnw1q7SzwrYySPCQsixkNtJGR0PoRWdJpV7HpslwljcC3CsfMETbRjg84x2r2fw/cQ3WveInhdZUFzGN6nIyI1BGfYgisq3Rf+FV3p2jPkXPOP8AbeolO6+TOmlQ9nNK7+Knp6q+q8uh5UYpIY2M0bxjb1ZSKpyzAx4G5SSMZGM817nq8cQuvDpuB/onn/Pk/L5nlny8/j+uKj8aWNgmgF5IIQxuIFBKAk5kXIHfpmnOUnF26CwtKnGtSck25P7tbdte78jxV1TzELAbu1DyqpIJ59BXsOs+HNIPiLQ4RYW6xzPMHVYwu7EeRnFVU8FaHP4tvrZrMCOO0hdQrsMFmcHv7Cm5tNpLr+hlGjFwjKo20o3tf+81ZdkeUpIHJAyCOxGDQ8gQ45J9AMmvTbnwHpJ8VW9lGJkiezklO2TnIdQOoPqap6d8PrO7utTgN3NFcW1wVKnaTsIBRsYHBH8jT9o9hPD0lepZ2sna/d23tt8uy8zz5JFkztPI6j0ryrV1H9tX/P8Ay8Sf+hGvopfABl8R3mnw34LwQRyMTEP4i3Bw3HQH8a4zV/gT4hk1C6uLe9sJFklZ1VxIDySeykU41LrUxxFGMJpQ2aT181c8aIHem45r0m6+CnjKH/V2trN/uXIX/wBDxWPcfC3xranL6FM4z/yykjk/9BY1aku5z8rONr1T4CXnkeO57fOBcWTrj1IZW/oa4m58FeKLQnzvDuqKB3+yOR+YFb3wuF3pHxL0d7m3mgDyNCfMjK/eUqOvuRU1bODCCakj6rNJRmivKO0KUUlFUgHEZBHrQpyoPtQOaRD8pHoSK6KREh1FFFbkBRRRQAUhpaKoRABiVwB15pTSP/rlPqMUp5pBcKlgh8x9xAwKiHLqg6noK0dvlQ4QZIHHuahpA3YpXEUNwx+1Mq2ykDaWwGY+v8sVha9pOlw2c0ir9jD/ACtMkrRFs9RwCWPTjHNdO22PBIzjpxk5qte2EeoweVeIGhYg+Vng49cdfp0rmnTTTTV2wUtTztS06Yim1S6hQZBkUspwD91SCQePY4471xt61ld6sls9/fM9zKUaJ48FRxk4GcDj0H0Net61cvJbxw6bGQkbKA6LkEZwVXHt3OB2z6ec+JdRG26ewnghuWg2yrb4Z1IBXBYf7wIxjPNebOnyysnc6FrG5lyrp6Sui+C9YKqxAy0o4+gTiiuf+y66eYnuPL/g3MM47Z5op+yj3/FmPN5H0s33qKVh81Ar6hnGjivF3/IWi/64D/0Jq6W+5s7kesbfyrm/F/8AyFov+uA/9Caresf2+uoPHavbNZS4GHU7lBGDz+deRlzSxmJv3X6ngZam8divWP6nHeGvC97o2p2dzb3s3lB1EkZfhl6EH8K9FnC+fESOcHH6UxbJ47NjGAZQuVB6Z7Vk6dqOrX98Ib3S0t0QE+YsmefTH+ele1ddz3LM6hP9X+FUtRG/T+OfmQ/qKvRqQmK5+81yHSZmtdTilWE58udELKV9DjkEUJiaNG3UK0Z/z0NK751DA7IP5msZvGPh+CPct6ZG/hRI2yfbkVc0aWa/DXskTRCZtyo3VV6DPvgUNgkzZb/Vj6j+dct4u8Wf8I1PZK2ltepOHLMr4Me3b2wc5z+ldNdyx21q00zqkaYLMxwAKw746Pq8kbSahat5YIUCRT1/H2pDLCNHrdlZanBNMsRUSpHnAbI/iHqK14GOwZrCbUtP062t9OtZUkkYiONI2BIHcnHQYzW3b5KAmgCtfSW8WnXUt4xW2SN2lbBOEGSenPSueste8JOyi11CLcxAUFWGT+IroruEXNjc25GfMR0we+cisOw8KWVrMsrWsQYEEHaDUsqJ1EXTjpUDc+cD3P8AQVPGAFxmoSPmmHqf6U0JnJ2Hgu1sdQtrsbFkgcOCo5OK6K31BLi7kijVwIjtYuhXJ9s9unNefaJ4f1LR9Vs7iHULhrbzAHiZyVKnivRsDzUOOcdaGCKuvaHba3apHcx7ihJQ5xgmquh6UmmaNLZJkKryY59ef610K8rVQDDzD/a/oKAOFtNCl0vS7uV7u5mBRVCySEgfOvb8K6rwf/yCZf8Aruf/AEFaj1ZQPD1wR6J/6EKk8Hj/AIlMv/Xc/wDoK14lT/kaR/wfqzxKv/I6h/g/VnRZrzm4UP4iuVYAgzycEZ7mvRa88k58SXP/AF3l/maM1SdSgn/Mv0FnSTq4ZP8AnX5o19PSzDtHLawPu6Fo1ODUNpDbNqQLW8TJvb5Sgx37VkSahJb+K/sD8RyWizQt/tBmDD8tp/OtLT2Yyqzgg+a3/oRx+len7Clb4V9x7P1ejf4F9yIfE8MMOpRLBEkSmEEhFCjOW9K72uD8UHOpQ/8AXAf+hNXd15eXpLF4hLuv1PGyqKjjcUl3j+oL1P1rP16eW20S5lhcpIAoDDqMsAf0NXx1P1rL8S/8i/df8A/9DFehjG1h6jW/K/yPUx8nHCVWnqoy/JnEW9hLcw+YjIBuK4JOeg9vero8O3RtjP51uEAJI3NkY/CpdJ/48D/11b+Qq1JeqdPvRG+QisGGejAV4mCyrDVaUJzT1Xc8PLsgweIwcK00+ZruZ9r4eu7stskgGAD8zH/CrX/CIah/z2tv++m/+JrX05jHJgHqtX9QuWgsJJF+9jA+prrhkmEavZ/edk+G8Ans/vOPk0GSJyj3toGHBAZzj8lqaLwtdzruiurNx7O3H/jtZ+sQalcwINNufIcEsx253egq54Xv7spbSXWROT5co9ecf4GpWS4W17P7xvhrAbWf3lg+ENQAyZrXH+83/wATQPCF+RkT2p/4G3/xNdNqkL3elzwRthnQgc45rmtC1Awyjd8ufkkB7Ef5/WreRYRK9n95C4cwN7Wf3if8IfqB/wCW1r/303/xNL/wh2of89rX/vpv/iauPfy3OoArNIse4ABGxkV0glBQtnj3qVkuEb0T+8b4awK6P7zjv+EP1D/nta/99N/8TS/8IdqH/Pa1/wC+m/8Aias6t4ka1hkuVaQQJ0ESbmarHh/xGuqQwyq5eGb7pYYZT0wcUv7GwnZ/eV/qzgez+8zv+EO1D/nta/8AfTf/ABNH/CHah/z2tf8Avpv/AImuvubpbe2kmYZCDOPWss+II0x5qKmeBmTGf0pyyXCR3T+8UeGsDLZP7zAuPCl9b20s7y2xWNC5AZs4Az6VqeC7iV4LqBnJjjKlFP8ADnOcflV+bUo73Sb9UUgrbueoOflNZfgr/l+/7Z/+zVywwtPDY+lGlezTv9zOOOApYDNaMKN0pKV9fJnXA1mXegadeXL3LxPHcuctPDI0bngDGQemAOPaoPE19PY6SGt32PJIIyw6gEE8eh4rk4ptYnjDx3s5U9CbrH82ruxWPjTq+xVNydr6HrV87nhMT7GhCUpWu+Xt8jv7Cyt9Nt/ItIxGhJZjnJZj1ZieSfc1TvdA0+/unuJVmSSQAS+TM0YkAGPmCkZ44rlIrfxDMcRXM7n0W8B/9mqX+zvFH/PS6/8AAof/ABVY/wBozat7CX3f8AyjxBi4zc1h6ib62f8AkdnZ2VrYeZ9kgjhD7dwQYBwAo49gAKRdPs1059PECi0dWVouxDEk/nk1xo0/xQekl1/4FD/4qj+z/FBOPMus/wDX0P8A4qj+0an/AD4l93/AJ/t/E3u8PU+59NunQ7iWGGe2e2mjSSBl2NGwyCPSsqPwtpKOrvDLOUx5f2iZ5BHgg/LuPHQdPSud/s7xT/z0uv8AwLH/AMVQNO8UnpJdf+BY/wDiqTzCb3oS+7/gFw4hxdNNQw9RX7J/5HbS28M1xBPJGGlgJMTHquRg/pSrBCl3JdqgE8iKjP3KjJA/U1xH9neKf+el1/4Fj/4qj+zvFP8Az0uv/Asf/FUf2jU/58S+4j+3a9rfVqn3P17HcGCE3i3ZjBnVDGH7hSQSPzArI8Q/2MHgm1Rpbd8FY7mIuhA7qXTsfQ1z39neKf8Anpdf+BY/+KoOm+KSOXusf9fQ/wDiqUsxqNW9hL7jSln9eE1J4app2un99mbHhmzgTUr6/sYGisZY444XcHdOQWLSHdyclup64rpSa4L+zvFP/PS6/wDAsf8AxVH9neKf+el1/wCBY/8AiqUcwqRVvYy+4K/EFetU53hanTo3srdjvDTCAeoB+tcN/Z3in/npdf8AgWP/AIqj+zvFP/PS6/8AAsf/ABVP+0an/PiX3GX9t1v+gWf3P/I7UxRnny1/KmmCM9sfQmuL/s7xR/z0uv8AwKH/AMVR/Z3ij/npdf8AgUP/AIql/aE/+fEvuH/blf8A6Ban3P8AyOyNuuOCw/4FTPIPaV/xx/hXIf2d4n/56XX/AIFD/wCKpP7O8T/37r/wKH/xVT9el/z4l9xX9u1/+gWp9z/yOvMcg6SL+K//AF6TbKP7h/MVxF2+uaY8TXNxcIWOUzPuBxjtk+o6121nM1xY287gBpI1cgdMkZrbC4iGIlKDg4tdzsy/N1i6k6Tg4SjbR+Y7Mg/gH4NUMDzlpC0SqS2cbv8APpVqoxxKfeu5U0tj1ea4oZ+6fkaN+P4G/Kn0ZrSxNxnmL/tD/gJpPOjz98VJRSsxjd6nowP40tIUU9VB/CmGCIk/IvPtT1EMuG2BWOeGFJv3nCfiewqK6tA1u4jZlbHBySB+GamjAEagdMUa3DQu21v5Q3n7x6561ZJAUsc8egzVZLpQgDZLVmXGuXtluaXS2lj3YVoJQ2B/tAgEfhmsqlSMFeRPK5PQ12X/AJaNkcdM9K53XdQu7phYaei4GDK8hIDr3QAckepHbjPNUtU8Tz3Egs1VLZiQdpbMjD2Hb8jWdcapJa2bhIrZXmbDmU7i2B1JOSfxrzK2KlJpU9u50QoNK8ht4swtJIr3TpMtnzJoisw+uzIUcAADnHA9K4fVrXRrLyJbOScwykK2VwwbqCwPGOeMAda6W41W6EX+jy2hkYEAJCpAPY5xjA+tczdXcb3O6/Z725HAK5SH8gMnP4HjocVg5NstxSRz0mratFI0aWl66ISqsdpJA75xRXWGSxJO2z+XtthyPwJzn86KXt4/ymfKz2qQfPSCnS/fpBX1RwHE+MP+QtF/1wH/AKE1dm0YY81z3iTRLu/uIrm1USEJ5bR5AI5Jzk/Ws7+zvFH9+6/8Ch/8VXz8atbCYqtL2UpKTVrL+u58xCtXwWMry9jKSm1Zpdr/AOZ2QQAYpgt1DlgOa5D+zfFH9+6/8Ch/8VR/Zvij+/df+BQ/+Kro/tSr/wA+J/d/wDq/tmt/0DT+5/5HahcCq9xaR3ClXUEH1Fcl/Zvij+/df+BQ/wDiqP7N8Uf37r/wKH/xVH9qVf8AnxP7v+AH9s1v+gaf3P8AyNxdBs0k3rAgOeoUVpwwCNQAMCuQ/s3xR/fuv/Aof/FUf2b4o/v3X/gUP/iqSzOov+XE/u/4A/7arP8A5hp/c/8AI7C5to7m2eGVFdHGCpGQa5ibwXpjtn7Mg+gxVb+zfFH9+6/8Ch/8VR/Zvij+/df+BQ/+KoeZ1X/y4n9zBZzWX/MNP7n/AJGnpnhmy06XzIYFV/XGT+db6JgYxXG/2Z4n/v3P/gUP/iqP7N8Uf37r/wACh/8AFULM6q/5cT+5/wCQPOaz/wCYaf3P/ItaxoWpT38s9nq91bo+D5at8oOOwqguk+JIzxrsxH+1ED/Opf7M8Tnq1z/4FD/4qk/svxN/euf/AAJH/wAVQ80rf8+J/d/wAWc1f+gWf3P/ACOttUcQqJGLOAAWx196xdZXW478HTpLcW7gblkQkg9z/Ks3+zfE4/juf/Aof/FUh0zxMerXJ/7eh/8AFU/7Vq/8+J/d/wAAX9sVf+gaf3P/ACN2Sxcae/lgGULlAemR0qhpF5q15dFL6zigRBkMr5JPpj0qj/Znib+9c/8AgUP/AIqkGleJQcg3AP8A18j/AOKoea1v+fEvuYf2xV/6Bp/c/wDI7NFwtc/qetx6ZqLW8lpdyGQBleKPK+mM561m/wBm+J/79z/4FD/4qmtpPiNzl/PYj1uQf/ZqP7Vq/wDPiX3f8AP7Yq/9A0/uf+RrawpHhy4JHZP/AEIU7wd/yCZf+u5/9BWsaTSPEUsZjkE7oeqtcAg/hurpvD+my6ZpvlTEGR3MjKP4cgDGe/SsaE6uIx6rOm4pRtqv67mOGnWxWZRxDpShFRtqrdX/AJmoa89Yqvia4LMFAnlyScD+KvQq831ERx65eC6tzJGZpMxt8uQScH9Qa0zecYSozlspK5eezjTnh6ktEppsl1+x+1pFe2rK1xaYdCpznHUfiDVyxlEghlX7rYIqpp+qWemQNDb6aApOT+97/lVBDZR6kL1baUEPvEfn/Ln8q3ec4Fq3P+D/AMjp/t7L7/xPwl/kanib/kIw/wDXAf8AoTV3EUyyLuXP4155qN6dVu43SAo20RhAdxJyfb3rsdGtbm1tfLurl55M5LsAPw4rlyyrGriK9SGqbVvxObJasK2KxNWm7xbjb8TVH3jWX4l/5F+6/wCAf+hitQfe/CsvxL/yL91/wD/0MV6WN/3ap/hf5Hq5j/udX/DL8mc5pA/4lw/67N/6CtY0StY+LNQtpiWtNTUSJ6B1UKw/EYrUtftieHjLZWwnkFywKE442rU8+kXOpaRHLLEIryMiVFBztYds/TisctjfC079iMmlbL6Xoadh8rRjOcLjJq3q4zpcp9MH9aw9Evbu6vfKl06aBUB3O54z9K6W7tzcWE0XGWQgfWu6krKzPSqu7ujC01FZ2B9Koqghv5lXgCbIH5GqtprMVrO0Vywt7mP5XSXgGrNtcQ3t/mOVHd23EIc46f4Upe7HlYRV5cx2MXzKAe9crq1kbHWBJGv7m5yWA7MK6uBSEGazfEKKLJJDxtkBPtWsdVYh6O5naXD5kobHC8Vv3CkWUuOuw/yrJ0JlkjkKkHDdq3XXdEw9RisqasXUd2cjb26XSJE4BVh0qPS/DM1hqamK8kWzWTzFhVRj1xn61Lp8wjcZ+9ExRh6HpVuS+uWvH8p9sS4C4AOfWk1y3uUrytY19UXdpVwP+mZNclf6XbavaxR3SBlUhx7HGP612NyPNs5V67kP8q465sTqemRwCWWEnad8TbW496KvwphS+Jo0NM0yx0vRL+K0QK7QuXODk/KcU/wV/wAv3/bP/wBmqPTNHbTNG1Fnubi4d4HGZpC2BtPSpPBX/L9/2z/9mrx6/wDyMKPo/wAmeBjf+Rxh/SX5MueMf+QRF/13H/oLVztuJG0thE22TawVsZwexrofGP8AyCIv+u4/9BasHTs/Zcf5700v+FGf+D9UPDv/AIXZf4P1Rn+E9Q1SRJxqJzc202FcJtyMe3+ea9JlkP2dyhw20kflXmuiazeXWq3Fje2scUsADqyE4YZ969DtpPOtVf8AvLzXqQ0lqfQT+EwtCuXNxE7Ozb1IOTnn/wDWKhutVul1u5UTSKsUigKGwMY9KZpOY5ljPWOZk/Wq2qL5fiG5H/PREf8ATH9Ku3xonrFnbXFx5VrLKBnahbHrxWTomqXF3KBM6lWTcAFA54q9EfP01c/xxD9RXOeH5MTWwPoVP60pfCmOPxWOh1jUzp9tG4TezuEAJwB71Lpl8b608xkCMG2kA59KyvFSk6Ukg/5ZzKePy/rUnh5v3UyejA/n/wDqomrJMUdbm/ms1tWhWd4xvYocHA4zWhn5a5KRAb68hJ4LsD261M9IOS6DjrJJnRQajDI4TcQx6Bhirua8z0y2u9I8QSWMt1NNbyRB4/NYtt/P8a9BilMtoGz8xXBPvUxbvaRTStdDor63mmaFLiJpV6orgsPwqZ3CjNeb6BL5PiK3Y9SxU59wRXQ+NZHGkRhSQrSYbB68GtvZ++4GfN7vMdNvBXNCsGFeWabr89hpF7aCUrvXdGwPKtkZx9R/KtnwJfzS3d3DNPLJlAyh3LAYPbP1FQ4+7cq+tjuyQBzTdwPGawPFGvNo1kpiQPPJnbu6Dtk/nXDQeMtbiu4xLcbhIMqHhAVvpgf1qXoUtTrvGn/Ll/20/wDZa3tM/wCQTZ/9cE/9BFcjrWppq2m6bdKNpPmB1/usNuR/n1rrtM/5BNn/ANcE/wDQRXk4f/kYVvRfkj5/B/8AI5xPpH8kWqaR84p1NfsfQ16p9COooopgFFFFACUUd6WgQ1hkEVXhJMQ9Rwas4qtECssq4x82R+NMCSnKnmI6cdMjNNpVkWIl3OFAOTUVIpxaYJ2ZzOsWNtNO0M8McobDbGAOO27n0rCl0yaRU+xajPCOfmkUSjBOeN3IHpgj+tdPJbvNPc3Um+Pe21EdRlQAMfzzj86zb5I7TTpGnU+WQRl8/OfTPvXFaMrtqyNW3Y5yay/tO7aJ7iL7PENwCJ8zqe7Nk9Tn0HXj14LV7e4bULiC8fyLUOY45HnJLRg/LkDLBeAfSvR7dIbezLyvA1xM3mS5YAIfp7dAK8z8e6Ze3esOtozSwPskXY+V3EYJPqeP1qYU4J6aEybtc2Y761SJEXXZ1VVACrAxAHoD3FFcFHpWorGo+2IuABtz09qKj6tT/nJ9q+x9hSj56YBWEfGekynrOv1jz/I1KvirRz1uHX6wv/hXte0h3MPZy7G1S4rLXxFpD4xfIM/3lI/mKnXWNMbpf234yAUc8e4uWXYu0VAl/ZSfcu7dvpIp/rUyujfddT9DVcyFZi4paKXtQFhKKWjFADaXFFLQAlGKWigBMUUtFADcUUtFACUUuKKAExSUtFACUUtFIBKgmsrW5cPPbQysBgM8YY49OanopSipK0lcmUIzVpK5T/srT/8Anxtv+/K/4UHStP8A+fG2/wC/S/4VcorP2NL+VfcZ/V6P8i+5FWPTrOGQSRWkCOOjLGAR+OKn2AU+irjCMVaKsaQhGCtFWGfx/hWX4l/5F+6/4B/6GK1v4h9KZPBFcwPDOgeNxhlPes8RTdWlKmuqa+9GWKpOtQnSW8k196OS8O6rYWWmyQ3c+xzKWA2MeMD0HtWqPEGjgYF1/wCQ3/wqL/hDtO/57XX/AH0v/wATR/wh2n/89rr/AL6X/wCJryKEcyo01TjGNl6/5nhYaGcYelGjCELLvf8AzHjXdGVtwuhn/rk/+FS/8JHpGP8Aj7/8hv8A4VX/AOEO0/8A57XX/fS//E0f8Idp/wDz2uv++l/+JrXnzT+WH4/5m/tc6/kh+P8AmVr678O3/M7xyH1aFj/Sm2V14fsT+4kjjH+zCw/pVv8A4Q7Tv+e11/30v/xNH/CHaf8A89rr/vpf/iaTlmj+zH8f8xqtna+xD8f8ywviPSAP+Pv/AMhv/hVa/wBX0PULR7ee4DI/UeW/+FL/AMIdp/8Az2uv++l/+Jo/4Q7T/wDntdf99L/8TT9pmn8sfx/zF7XOv5Ifj/mV9MvtB0uDybecImcn92/J/KtD/hI9I/5+/wDyG/8AhVf/AIQ7T/8Antdf99L/APE0f8Idp/8Az2uv++l/+Jo9pmn8sfx/zB1c6/kh+P8AmYOrLp1xeNe6dqrW07/fHlMVb8MU3TrhYpt17qiSKOgSBl/Piug/4Q7T/wDntdf99L/8TR/wh2n/APPa6/76X/4mlKeaS3jH8f8AMca+dR2hD8f8x41/SfLK/a+ox/q3/wAK419Q1CBykLWckSnCkh84rrv+EO0//ntdf99L/wDE0f8ACHWH/Pa5/wC+l/8AiabqZo1Zxh+P+YKtnSd1CH4/5nM2urXEgmS5WGJDC4BUsSxKkAdPetvwX/y/f9s//Zqt/wDCHWH/AD2uf++l/wDia1dN0y20uAxW6n5jlnblm+tZ0cPi54qFauklG+3mhUcNmFfHU8TilFKCe3mmvPuZfjH/AJBEX/Xcf+gtXNrcpYWVvI8UzCRScxoW6MetdJ4x/wCQRD/13H/oLVJoFuk/h613KD9//wBDNXD/AJGc1/c/VFUf+R5N/wBz9Ucta3dlPfCSKKU3LLs/1TDjOee3513FlGY7VE9BQlhErZCDNW1j2jFenGLTuz6OUk1Y4W7uV0vXbmCc+WsjebEx6HNMu5Fu9RW6DAfuvLIBzk5zXYalo9pqcYW5hSTHQkcj6Gs2z8K6fZTrLFAA69CSTj86qpK92upMEuvQ0rBGSwiRvvBRkHtXMaf+41FkzjZcsPw3V2aptXFYzeHbc6sb4tKctv2byFz64pX9zlYL4rj/ABDH5mh3I9FDfkQaz9Fukgdi7gK6g5PArobq2FzZywHgSIVP4iuPbw3q0RxHqXygcAwg0TkuVBBas7OG4jnBCSIxHZWzXNX37vXpl7Oqt+mKuaBp13ZtJJeXIldhtAVAoAqv4hs777bHdWUUcnybGDsR3oUk4O4ctpKxS1V1Os6U6jlhIp/If/XrpLA5tQPSuXtl1aSZFmsY1H9/zM4/SuttYvLhVfQVipOU0+yNHG0Wedt/ofiM9hHdE/huzXW+KovN0GRsZMbK364/rXP+INPuTr8ggt5XMrBlKrx09fqK63U4Xn0O5jI+cwngeuK7OZe1UvQwSfJY8dupPJuIY2+5LuQ/XqP5Guk8FSGHXI0J+/Gyfpn+lZ8ekvrJuEhQs0a+YjY4Dg9M+/IqXw87w67ZFVJYyAFQORng/wBaia3sOL7nQ+PYyVtm7FGH0wQa5udrO40TTYnldLlA53IBwNxHf8K7TxpbmTSUmA4jf5j6AjH88V5xNa291Ym3mj/eo4aNu2MjIoklaLGuqOi3Wi6ZbQWvnZRnaRpcZYkLzx9K9D0z/kE2f/XBP/QRXmEDKwbb0HFen6Z/yCbP/rgn/oIrxMN/v9b0X5I8PCf8jnE+kfyRapr8qadSHoRXrHv3AHKg+1LTU5QU7mgAo7UtHSmITvRRRQAlV2+W5Hoy/wAqs1XnBEkTDGN2D9DTYkPqFQz3W88Rxkbfdu5/Dp+dTUmKmSuNMdqMYKj0xXLahChaFWYON24RJyzsBkAfzPauh1Ge8Marb2Pmgr99pQoz6dz2rA1NLq2tJ5kiO/ywX2gvz6A9lzj9TzXnYmskuWLszWC01OcuYmudT8vULlYYXH7uJT0UDnc/brnAx7mq9xFDeG88uwguCQxaePBEWMAYHcnk9uuPeruoQSyqqXEpigxzGu3eT0wBkj3yDxXNagh+0MsOqiO0QAiNH2kkjPTqcDHrz09vMqSnO6ka2XQvpDpSoqtBMWAAJM0Qz+lFUk8KJIivJq0iOwBZck4PpRWF4/zC5H2HiRlbr+tWY7jPU81SfO7I605CPcV9A9QRqLIG6GpQ3FZquVOPwqykpHfj1qWiizhCeUX8qUbQeFAP0qNX/GnhsdP51IydJ5E5WR1/3WIqZdRvE+7eXA/7at/jVPPFOye2PWndhZF9NY1JOl7cfjITUqeINVUYF4/4qp/mKy80vanzy7i5Y9jZXxNqi9Z1b6xr/hUy+LNRHUQN9UP9DWD/AI0ZNP2s+5Ps49jpE8X3mPmt4D9Mj+tWF8XS/wAdkh+kpH9K5RTzntU3Sj29RdQ9jB9Dql8Wp/FZMPpJn+lSr4stf4racfTaf61yIzjNLT+s1O4fV4HYr4psD1juB9VH+NSL4k0xussi/WM/0riqOtP61Mn6tA7oa9pjf8vQH1Rh/SpF1fTm6XkP4tiuBz70ZxVLFy7C+rR7noa39k33buA/SQVIs0T/AHZUb6MK849Mn86MfQ0/rb7C+qruelcGlxXmoYqeCR+NSLcTp92eQemGIqvrfkL6r5nouKMV58NQvVxi7uP+/h/xqZdY1Fel5L+Jz/On9bj2F9Wfc7vFGK4hdf1Nf+XnP1Rf8KlXxLqK9Wib6p/hVLFQJ+rTOyxRXJL4pvh1ityP91h/WpF8WTj71pGfo5H+NUsTTF9XmdOR8y0uK5g+LWLL/oIPr++/+xqdfFkP8dpIP91wf8Kf1in3JdGfY6CisRfFVkx5guB+Cn+tTL4l05hy0q/WM/0qlWh3F7KfY1aKzl8QaWT/AMfOPrGw/pUi6zpr9LyL8Tj+dP2kO4uSXYu0VXXUbFj8t5bn/tqP8alW4gf7s0Z+jinzLuHKx9GKAQehB+hpcUXFYSilxRQISiloxQAlGKWigaQlFLiikUZXiDTZdT0zyoSPMRxIqn+LAIxnt1rmotI8SQRiOLz40HRUuQAPw3V3dJXn4nLaVep7Rtp7aM8nGZPQxVb20pSjK1tHb9GcP/Zvij+/df8AgUP/AIql/s3xR/fuv/Aof/FV3FFY/wBj0/8An5L7/wDgHN/q9R/5+z/8CX+Rw/8AZvij+/df+BQ/+KpP7N8Uf37r/wACh/8AFV3VFH9j0/8An5L7/wDgB/q9R/5+z/8AAl/kcL/Zvij+/df+BQ/+Ko/s3xR/fuv/AAKH/wAVXc0Uf2PT/wCfkvv/AOAH+r1H/n7P/wACX+Rw39m+KP791/4FD/4qkOmeJ+7XP/gUP/iq7qil/Y9P/n5L7/8AgB/q9R/5+z/8CX+Rw39meJx0a5/8Ch/8VSHTPE56tcn/ALeh/wDFV3VFL+x6X88vv/4A/wDV6j/z9n/4Ev8AI4T+yvEvXNx/4Ej/AOKpw03xOOj3P/gUP/iq7ijFH9j0v55ff/wA/wBXqP8Az9n/AOBL/I4U6V4lLZJuCfU3I/8AiqX+zPE2MbrnH/X0P/iq7mkNH9kU/wCeX3/8Af8Aq9R/5+z/APAl/kcIukeIk+6J1z6XA/8AiqYmia9HIZEikVz1ZZ1BP45rvqSl/ZFP+eX3/wDAD/V2j/z9n/4Ev8jhZNJ8RyxsknnujDBVrgEEfTdVP/hF9RUY+wKB/vJ/jXovakxR/ZFP/n5L7/8AgB/q7R/5+z/8CX+R57D4X1FSsaWiRIT13qAPfg13dtD9mtIYN27y41TdjGcDFT4pMV04XA08NJyi22+53YDKqOClKcG25dW7/ohKKWiu09IZH/EPQ0+mLxKw9afQAUUUUAFJRRTQmLUM4zESOxzUtMYblI9RTsITr0pKSM5jX6UtIZJJuOnPs++M7fauc1JHkgQJKWiEiuyrj95j3JrfhBkifzR/tKnp9fesjUFVZDxyOa4KtBzm5GsJJKxz+qW4u7y1326+VgtJlgG4xgE8kDr06+w5rL1dpltpZ4NI08SxITHIzBtm0HB27fbpV64vV5S2jM0gyWWIgBT1OSeB/P2qqEkZWe6bAzxFFznPY56/pUvCwtzS1Ye0d7I87HimIAB9Xvd4+9tuGxn2orTfQAXYv4i0+NiTlBHEAp9OOPyorj5aP9f8MV7xqMvzHjjNKq8D2qWUKjfMQp9+KRMEfeH4Gu1O5tawBcds08D2pwX0PPtQFHai4xVYrwf0qVG6ZFRjHY5oXOD29c0gLKv+lPyCOp4qqGPWpAxFAFjryPrR06moww/rTweMUgHdvWlxzTQRnGaX8qAFHXGanWq44NTrnFS2Uh1O4pO1J9BUlCmkz7UuCKQj6cUXAOv/ANegDgjFAH5UvGfWi4WAH+VHrxR06/yoouAcdaPrS/nSUAAFHWlpOo4oCwuaQ+/elx0pCPbrQFgOKM/SjPNIeuPzphYAM0tA60mOKBWF4OaM4pMdeaM0DsLmgH3pO9GeKQC59KXAI/8ArU3OOtBNMLDuAMgflT1mlX7srr9GIqIe1Lxii7FZFgX92h+W6nA/66t/jUy6vqCdLyb8Wz/OqPvQeO1Pnl3E4RfQ0117U1/5eifqi/4VKviPUV6yRt9Yx/SsjNGfXFP2s+4vZw7G4vie/HWO3P8AwA/41Iviq5H3reE/QkVz4NLnFV7ap3F7GHY6QeK2/isx+Ev/ANapU8VQn71q4+jA1y2TxRn3p/WKncXsIPodcviezPWKdfwH+NTL4i089WkX6p/hXGA0u4EGn9ZmL6vA7dde01v+XjH1Rv8ACpV1awbpdx/icVweeaUNT+tz7C+qx7nfDULI9LyD/v4KkW4gf7s0bfRhXn27HNLuprFvsL6qu56ICD0IP0pcV50GA6cfpUi3MqfdmkX6Mar655C+q+Z6DRXBpqV2vS6m4/2zUy6xfAcXT/jzT+tx7C+qy7nbUVxya7fjrcZ+qr/hUo8RXo6mM/Vaf1qAvq8zq6K5hfElyOscR/A/41KviWT+K3U/Q4qliaYvYTOhxRWEviVT1tvyf/61SL4jgPWCT8CDR9Yp9yfYz7GxSVljxDZnqso/4CP8aeNcsCf9Yw+qmq9rB9Rezmuho0lUhq9g3S4H/fJ/wp41Oybpcx/icU/aR7hyS7FmioBf2b/duoT9JBUgmib7siH6MKfMhWY6ijr05oqhDDxKD6jFPqOQ4KH3xUlABSUUUxBRRSGmIU02lpKYiJP4h6MadSdJW9wDSjAOSM1Mm0rrUYsZw/1rDvpLa+kePzFfaNssOcEH39q2toLlskMfeuW8Q+D4tZnF3a3j2t0DnzFBKntxgjB/OvHr46pTnapCyNVFWuiu0UMPyRRiKIHCoOiiucurvUILhrgaUWmU+UCLjeqjqTjAI9z06c9M9RFo93pdr+/upNSYAkcJGemMZOc8/wA/YCqqW89ze7jaG3aMDMeEIx6lgOn0I7etYzx6rPlpuyRcYW1ZxclnYeY3mTnfk7s+X17/AMdFdc2l6IWJltbxpCfmP2Z+T370Vxe1fc05Ucn8YWI8PWGM5N1nA/3D/jXm+gs7MnzNjcDjcfWvRPjGcaPpyc8zk/kv/wBevP8Aw8MrCxxy39a9/L1eirkYx/vGdUu8Yw7g+zU8PPgfvZP++jT0Xr3qYKMZP1r0uVdjhUmMFxdL/wAvEv4nNOF3eD/lu2fcCn7AOMUu3Bpckew+eXcFvrw/8tAT7qKkGo3ag8ofwpNn+c0nlg0vZQ7B7WfclGr3IHKRn8/8akXWpsgeSv4Gq3lg8H+VHlij2FPsP20+5eGuP0Nv27N/9anjXlHW3fI9xWfswKa0Q/yan6tT7D+sT7lqbxfZW0oSaC55GcqqkfzrRi8Q2jxq22YZGRlR/jXn2ujZepjj5Bj8zXQWke62iz3QfyqPqlNsv6zNI6Ya9Zesn/fFPXW7E/8ALVh7FD/hXPeSDwBTvJ/+txR9Sph9bmdENWsW6XCj6jFPGpWbEYuovxYVzPkgcAUnkj86n6jHuNYyXY6tby2I4uIz77xUgliPSRf++hXIfZwTkiq93bDyOQBzUPAruWsW+x3III6g0o7e1eT/AGchbwKxHOTg47Ukf2q2W2Y3U65yWxK3fn1rP6m+5f1pdj1rjjtRjmvJYLvVQsxOoXgLJvUee3HJ96emp61HpzuNUujIVLZaQnHtzS+py7j+tR7Hq9GPWvGx4x8QRGMf2m7fL0KIcn8q0pvGWtrKscVyCAwDFolOfyFS8JMr6zA9S7c0Y+ua4JfFWrqPmMB9zGR/I1XXxzrPnmP7NZtlyB8rDA9T81J4WoNYmB6LTRXFSeM71Uz9kgOOT8xGaLTx3NPEXfTFTnGBPn/2Wl9Wqdh/WKb6na44pOc1yEXj2OW4aP8As2UKvVllB/oKkg8eWUsrRNZXasvJI2kH9an2FTsP20O51hpP0rnl8Z6cTgw3S/8AAFP/ALNSHxxoitteSZcHGTCT/Kk6U10H7WD6nRdTQSeawIvGugTPtW9bcemYH/wqX/hK9CHDanCp/wBoEfzFL2c+xXPHubXOeKO1ZqeIdGkj3rqdptHUmUAVLHq+myjdHqNqwPcTKf60uWXYfNHuXcn9aUVXF5ayHCXELewkBqVXVujA/Q1NmO6Hg0Umc8nNGcA0WGLnJozmm5zS5Hf9aAFzxnvS03OR/jQT2pDFoFITRmgBwODSknFMz9aTPsaAJMnJo3cVHu44NKTQBITzxSbuaj3U0t/jTsIm3f8A1qaXqIuKaxAosFydX5qQPVMPUgfiiwXLO/rQH75/Wq5fApC5pWC5a3+9AfjFVfMwO9HmHHNFguWw9L5nHPNUxL604S4o5R3LRkpnmc+9QGTjmmGUZosFy6JPfip45PlPSs5JM+tTROefTFKwFvzKaWX0B/CoN/uaQygDPX1ouFiwHA5AA+lI1y6naJZsnoqOwJ/I1Wjd5z8gwv8AfPT8PWrsFuie7Hqx6mi7ewrJbl3TFupLtZJbmYqucRmQspz65rqAcgVh2i7BmtmI5jX24r0qEWo6nDWd5D6TNFJXRYxFzRmkooEFLSUUWuBlSW99bXJaCXzYWJwjYyvsPUVHLqlxbOEuLXtywyP6H271qy9FPoaXNebLBVYSbo1GvJ6o0U090Yh8U2SMVaNwMcsOTn6VZbxVo8wiCzlWYcDb7Zxx3rQaytZkZp7eJw3UFBk1HJounSWe5bRFBk8whePm5GePqa5pLEpPms7dSk43VjKOr6bvKC9SVmUny0OeO+SeOKfHp8N2GmSFVmY7SAdowOn1xz271UvPDOliDc7yw7ACZA+AMd+elZVxY31tYWr6Rc3QjBdWDgq0244XHHHAPOR6+9eFVhzSTSsdF2kXZrazimkj/tdItrFfLEiAJg9MbeMUVCuhakVGLfTU4+60SMR7E45PvRWXspdwuzz/AOMTgWelL0zLJ/6CtcRoCfLBj/np/n+ddh8ZWITSFz/FMf8A0CuT8PD5bfPILA/TmvsMAv3MTLF/xGdaiDqeualA5z69aEGeo+tSAcZr0zhGYOTTgo9Kdtx1p2PUcGgBMfzoxxjPPWlAyM+1OHAoAZjnijHan4yTTgOTRcViPGKYVwPT3qUjJ6cU0jjn6VQmcp4iH+nRcf8ALPkfia6KzH+iw8Z+RefwFYHiMYvYvePv9TXRWIzaQHJx5a9/aktwexZC8YxzTyp6c0gGR79zUiimIZt6+tLt7cVJjvRtB7UDI9oHbmoLtf3B6dRVvGeRUN2D5B6dRSY0czcffmRersM/gKWeMSW0CjqcAZ9xU7xhrmduypj9KgGZJIfRCq8etZXLCZVimbIGPLwKjiQGwAx2NWLiMSXaL2AyffFVg5SzVFHztkD86aYjkbiNllLHA3MTj+7zgD9K347dY7e3UdSQScdTWLdna6g9Of58V0CjMFuf93+VIZoviOMucYAqrZxB5ZZCOTU8mbicQg/KOWPrS2q7ZpgB35xTuAskS7TwM1QjUJaFQPmZjjitdl456VVt4RI+/wDgThRSYIjhsxFEBgbjyTjvVeGAC/kOOueK1mXv61UVcXrk9x/SkwIp4/LjPy4PbNZr2obPr1+tacxLsWI46CmLC3lu5JGBx71Ldykita2KLGH43NWRrEWGb6CumhA+zr+P86wddGCT221fQkj02ITaU2eRlu1XYLcR2qrt5PQVFoLA2Mm4Zw54/AVrWqCRzIcEDoPSkMqxaaqKSQN7dTT/ALJg8cVqFR2zim7fb/61OyFdmPdwyiEFZJBznhiKfbm8WBNt1cAgYz5rA/zq9cpmA8dOabAg8gD0zip5VcfMyJbrUh0v7r6ea3+NPGqawg4v5yfd8/zqURgdjxUMvZVGSfahwj2Hzy7jZPEevqdkWoPuz3RT/MVNfa/4r0/w++qG4VohII9zwLhiew4q3pGn2b3BN7cLEijcc9WPoKd44vY73QpLaN0ZFUFVjPyoB2x0rJwjzWSLU5WvcwLf4ja+0UbN9lbd1JiP9DXUW/irU5I0Z47clgDwpH9a8qtjttozxlW/LmvRbFQ1pEePu1MqUL7Fxqy7myPFF5jm2hP0JFV7nxvcWrqDpquGGciYjH6VEIxjP61Q1G3DbDj1Gaj2MOxftZdy+vxCOPn0lwfacH/2WpB8Q7X/AJaaddKfZlNc41uuPu9KYbVSCABxR7CAe2mdSPiDppHzWl6uP9lf/iqkXx/oxIDi5T6xf4GuOa0Tg7ec+lRtZoaPYRD28juh450HHN3IvoDA/wDhTv8AhNfD7YzqaL/vIw/mK89eyU/yrO1Kw22EsgPQA9B2NNYaIfWJHq6eLNBfBGrWoz6yAfzq2niDSZAAup2Z/wC2y/418+Y/cdfut/P/APVW5b2ivCj+qg4NKWGiuoLEvse4LqNpIPkuoG9MSKf61IJ4z91ww9Qa8TGnqyngcnqajfTwB8oxUfV13K+seR7iHyePrSF8YzmvDjBOgxHLIh7YciiCTWC4WG/vF+kzD+tH1bzH9Y8j3HzD60vmc9a8WfUNatA+dXvmYJnm4Y4/M/WvZPhJKuveFJpNSH2m5iu2jMkvLY2qQP1NZVaXs48xcKyk7WJt46Cmb+eP1ruDoemsebVfwY/41E/hzTGH/HuR9Hb/ABrm9ojW5x6SA96mSXB/wroz4X0/JIM6/R//AK1QT+F4PJfybq4V8HbkqRn8ql1EUmYbTHcFXJY9AOamitWchpj06IOn4+tOggWBcYy3cnqasrzVblbEiKOBircK81XjXmrsK8it6VPUxnIuwjCitOBsxVnJ0q7atlSK9CCscMmWKKQ0VoQFFFJQAtFGaSqQmI/KH6UgOce9KeQaZGfkHqKTQImlfACA4Hep7U7reRe45qmf1qzYt+8Ze5XpWKpKMLDvqY+rwM5D4DIq58s9CfU+v0/+tjBmnu5y8vnMFRvmJ6Ac8fXOK6jU4WliZSxjTkEr1/8ArVjWsUVxaS2jYjAkQDb3yeK8Kr7rk+XRNHcn7qKQnvCoI5BHB3Nz+lFXmttYLExsipn5VEgAA7DFFL2sOyFdnkPxkYmfSVHULL/7JXOaAMJbem7+tbfxdm8660g5+9C749claxvDYxHa9wOlexgv4MTLFfxJHWrx7Y/SpgBx+opqKMU9Rg5/pXonChccUYx707n1oGcfSgY0AZpwUenWlxzRjikMQgZ6nrS4zg+tO29h1oC800IQjNMPbPQ1Lj6Uxh6UyWcp4jGLyI8f6v8Aqa6OwH+hwdP9WvT6VzXiZtt7Cf8Apn/U10+nDdYW5/6ZL/KktwexaHp2p+PrmmhSOM/lTwOPXFMLByM0dRTsdTwfSjbkcdKVx2EwG7VDdAfZj+FWPbH4VDdrm3JxjkH9aT2GYLYWS5J4AA/lUZURW8BOBltxpZwzSyp2PzHjsB0p0yq0VvH0LY/LFZGlivGC05ds/OpOPQdqZBEBE0uOqkL7CrUzCO4YkjCx4qNABp54x8ppoRx+orsdO+c5rdhA/s+Bj0Cjr9KwtUR1uFds7WI2j2x1/GtazLy2S9QkaAfjQxI2rSIpHub778mi3H+lTCpoSTEjc8gVDvEN1ITx8uf5UmxizsXcQp1P3iB2pLMELIPRqktoyQZXHzPSWh+eUe9K4yXbng1Tn2pIwHJPX2FXJZPLQn+I1nscsfU0pMaQ1YzI4A/lVuSMJAR6CnwQ+WmSPnNLPjymPTjihCZVtx/owyPWsXXFyp4zlK24P9RyfWsHXpNssMa8lsg/SrvoTYh0AM4lQcKG3H8v/rVvaeflbjnisTw02XuUz2U/zrcseZH60uoy6BznFKRk04Dj2/nQeOcc1VySvKuY2+maitV+U9OuatOMofU1BGDDHlgcnnHpUNlJBI20YHJPTFIsQU5Y5Y1IkZ5dx82Pyp5AJzjNNAyFhVLUY99hcL2KH+VaJHyjPSq1wu+J1xwQRVXEefWozanHUPnp7V6Lo7B9OhbjG3v2rzm2zsmRuxH9a7/w44fSo+nHFZSNYG0Bxz0qrfp+6BHUNzirQ/yKr3qk259iDUFmYV5yenemlRnjg4qTnr/Wm8468mmTYjYZ69KYU49cmpT+YFNP+e9O4Fdl69vpVW7i32U6D+KNsflV5vpURUE4xweKpMlnBDmCQe4P+fzro9L+exhbGCBj9cVzyDaJk6fL/Kt3RGzZgHqrEYzTlsJGoEyOgodPT681LGpc4A5q9Fa7eXHPWsrlpFCKyaVvmGF9atiBY1woAq5tAGB+lRuuAT29aLlWsYGpx4l6feT/AOtXo3wHus2etWpPKvFKBn1DA/yFcBqaYWMgdQw/lXUfA+4MXi3ULUnAmtWwPUq6/wBCayxCvSY4O00e9k5waDSDpj0pa8i51jO9IeQaDwaQ1DZRzd7CY7px75H41Gq81qanFkpJ/wABNUAuCPfmuyjrFBJkkYq7CMCq0a1bjGBXfTics5Fhat2xw2KqLU8JxIK6EYMu5opKKskWikzRmmJhmlpKKYgzUa8Fh71JUfSQ+4oaEKaltW23Ke/FRGhW2uG9DmpktBk9+uRIvfrXD3MLPe7SSo7kdQPb1Nd7fKqgzEgLs5J9K4q5R5LlpJI9kefkQjlh6sD/AC/P0Pl1Xadu52UneA3N1/yz09Cn8JMRYkdue/1opjBixJkckn1orL2TKszyL4u/u9V06ME7VgbjH+0cfyrL0S5WCC1YxswVBnHetD4wMD4islPa2/8AZ2rL0wAWcGcE7Ov4murDy5aEWTXV6sjqE1qHGTDMO/Qf41MNZtiB8kv/AHz/APXrDXnp9KlH6YzW31iZl7GJuDV7U9RIP+AUv9qWmc72BH+yaxDx3pVPbNH1iQexibn9p2ZH+t59wRTxqFpx++A/AisIdMjrR37U/rEg9ijoBfWp/wCXhB9TTzeWp4+0RfTdXO8AdKaQM4xin9ZfYn2KOkNzB/z2i+u8U0zxEYEiH/gQrmjg88flULgDnj8qpYh9hOiu5F4slRb6Ahs/uzyOe9dZpbqdNtiGGTEv8hXm2suEuY+3y1vaaF+xwHGPkH8qp17K9iVSvodyF9P51IFPrXJqfQkfialUuD/rJPwY0vrK7FKh5nUgHrjn3pCMgY9ea5pZJs/6+X/vs1ILi4B/18v4tR9ZXYPYvudFg96iu1/cH2IrGW7uf+fiT86Gurhkw07EehxQ8RHsHsGRsuXu2I5AA/SoEBeaGQ/dBCqPoKny5LfPy3B4600bxtAbhORxWftUV7NjLiPzLph1Crk1FKfL00Kf4kIHPepxu3M+RuYYJx1qtPGSgU4wowOKaqoXs2c3ri7Xt8enX8K1bBAdLTAHK8/WsfxA7i4jDFdoUYGK09JkdrCMDG3BGMVbmrEqGtjZsyDbRn29abIA90sfQdz602BniiCKFwvrSpGUkWQHJBzz3qXURSgy5joB17VWtyFuZgT71J5rcZQc+9REEs7AYLe9J1ECgyKeUu2c8dqbCB56cf55pxhJOccd6eR+9V1XAUYxUqauPlLYHI7daimJ8p8eh6UpnGMbWpjSqQRtbnPatOdEcrKqy+Xa+jZwPesrUYCsPmONzkH8K1QF8zc4JUD5RiqGs3MSwAnPXHK+1OMkJxZleGW/0ycZPKf1/wDr10Fl/wAfMign8frXL+HpBHqz56FSOnuDXT27Bbkt/Cc8020gUTU5H+FJnjGaYJUPIP4Yo8xAOvT0o5kLlYrHaDzz0qvC2+c7uoFPXa7lncAL0FRQ4W6yRheRkmp5irGxY6e90C7ArEO4H3j6CtUacgG9k57KUPSn28tqsUbGSLaBhVyOfrzVHUdUWQNFbuOfvOOPwHtWfM5OxdkkU9QaBHMcIRv7zgcfhWVIu5SCDVhsMOwqFl655reNkYs872eXe3cZHQkDjpzXZeFnLaeec7W/xrkr9RFrt2PUk/yNdJ4QkzDKueh/w/xqZFwOr71Fcgm2fnoM09TikmBMDjGeCOKg0MnnB9qQ9fYUdevSgn8qLiGkGmHvwSKcf/rUh75P9KAImHH+NRnjnP5VMRxjH5VAeuRxTTFY4yaLZqdxH0+ZwM1qeHYmkjlQZ4YH86i1K0K6rPIeFBDAeuQM1a8NNtubiPIyVB59v/11UpaCitTpoIljHyjn1qcYA/rUa4BqQcH3rC5stBe1RyKCOvPepcjGD0+tMY54ppiZj6ov7kEZ4bn8qt/C+4+x/E2yGcLNvjP/AAJD/UCodQXNs479etZ/h+4+xeONFuMhQLqEk+28A/pTkrwaJ2aPqg8N9aXtRJ2IozkV4h2dCM9aQ9KVutNJwM5wKzZSIbqPzLdx3HIrJArVtb22vJZo4JBJ5eNxA45z0PfpVB4/LlZPQ13YXaxNTQSNenFWkqJBUqV6UEckyZe1SocNUIqVTyK2Mi9SvMbe1uJljaUxoWEadWwDwKYh+QfSp4P4vwqiU0nqZCWOu3Ea3T6x9nncB/sywK0SH+6c/MfQnI/CrVjqz3egnUDbN5qK+6GM5y6EghfqRxUGp3891cvo+lt/pRX/AEi4H3bVD3Pq5/hX8TgddSztIbCzhtLddsMShVHtURWun9M7K0v3adSKu2mkklp5276WvrpfrrkJY67cRrdPq/2eZwH+zLArRJ/snPzH0JyPwqe21xG8OPq1zGY/JRzKinPzISCAe/I4qPU7+a6uX0fS2/0or/pFwPu2qHufVyPur+J46weIrOKx8F3FrbJiC3jT5euUVgT9eAaV+W7j0X4mqiqrhCqknKStZJWj5276Wvrpfrq6K08QXMC3barHbzuA62ywBokz/CT94/UEVPBrsbaA+qXEZjMCuJ41OdroSGA9eQcVrqwZAV6EZFcTdAv4T8QTKD5Jv5JAMdVV13fnhqJXhquzFRUcS7TSXvRWiS0btb/h+xrx2niC5t1u21WO3ncB1tlgDRJ/sk/ePuQRVjT9QOp2Mdw0flygmOWPOdrqSGAPcZFaykMgK9CMiub0L5hqUi8RvfzFPoDg/qDTtyySuZSn7WlKTSVmrWSW99PP59jqGxJaxMRnA/8ArVh3cCtdAFFYsQBuOK2rZt1lj+6cVlaomORXDi43ZOHfQQ6hZIdggiIXjOOv6UVjhJMD5M/nRXF9XX8xryI8L+LrhvFEC9harx/wJ6z9NnhSzhUyICEGcsKufFaT/ir1GSMWqA49CWrimJEjYPAA5r0aMb0YryM6ztVkdyk0JAzNH/32KmSaIjiRSPrXAA44z+FO3E98Y7VXsifaHoAZeOR+dPVhntXnvmtuypJx6U7z5Ookdc+jGl7IPaHoQ+maXuK8++1TqMCaT2IY08XtyFOJ5cdfvmj2bD2h3+R3pp7eprgxqN4rAi5mHp+8NO/tS9A4upiP940ezYc6O2J/yKic5XAH4Vx/9q32R/pUnbvSnV70Hm5c/UD/AApqDFzon14/6TF67eh+tdFpfNjB1PyD+VcXc3UlyVaZ9zjjJFXYNZu7eFEjcbVGACoOBVSi2khRdmdwlTL6Y47c1xK+I78Z/exnj+4KcPE+oL8xMX4r/wDXrP2bLU0dr2xTj6j9R0rix4pvhjcsH4qf8aePFt6ODFAfbB/xo9nIfOjsgcEcfpTs/jXHjxddHB8iE+vX/Gnf8JbNjm1jJ9mNTySDnR1pIAxmjOQOvrXKf8JfJ3tEP0c/4U4eLepNoOOP9Z/9anySByR0+7I9aic+lc6PFy4/49D/AN/P/rUp8Uwtw1tIM+4NPlkLmRV8RsVnjY88Vd0Ns6dF17/zNYer6kl+VaNCMDByas6Xq0VtbCOQMSCeg96tp8pKa5jrY2AGKlz3rCTXrUcNv/Kpl160PV2HsVrOzLujYyM8A0bvSsga7ZMB+8Iz6qaX+3LM/wDLX8waVmF0apbpyMUZHbpWWNasyf8AXD8Qad/a9nxmZR+dFmGhpZHOcD+tNJ7Vn/2tZkEeevH4UHVbM4/0hPzo1C6LjnisnWBusjx0YGrLalaHP7+P/vqqGo3cMltIFlRiOwIqo3uSzF0ZtusLweQw/SuziI4HT61w2mPt1iJmYbdx5J9q7OKQHHI6+tXPcmGxfB7nNBHHPFRq3HBBz75pc547+lTcsdn+VJwM8A0mc0h6UrisOJX0FNIB7Um4delIDTTGIQmTx9ajZVHf24NSMT1/SoZG46002TY5HWkSLVJSAQSAePcVZ8NyNicKzLgjkHHr/hUfiEYvUJ6FB/M1F4bfF3OhOMqD+R/+vWv2SFuddHLLkfvpMYzyalEsrDHmsQevNVYz/Kpwc9xnNZ3NACAcZORR5QHHSnL+eKM/jSuFiMxZH3qQxlT94496mOcDPNIev+TRcLFfyuPf6U6O3GeeakJA6/jTkyM880OQJHPa9Ftutw6tF/iKz9Ck26p7OrD+Rra19QTA5GRyv8v8a57S3KapAxPfHX1qo6olrU7MHHQ/lUq/T86rq3HtU4PPOOazNB4OOn40xm46frSlhkdvxqNjwc/jRcLFW7O6KRfVDXLXMxjntpVblDkH8q6yVQTtPQ8Vx98u2JC2flbB/wA/561pF30M5I+vYJlubOKdPuyorg+xGach4IrmvC2t2kfw+0fULy5jhiFpGjO7fxKNpHucg1h6j8RbKR3is5Wjj/56FDuP09P5/SvGcJczSR2Q1R2Woarb2Iwx3y9o06/j6VympazNeZEjbYz0iXp+Pr/KuYl8S2jk4lPJySc5J96qSa7bvz5vHemqMux0R5I9TuvCN35mrTwk5DxbvyI/xro76PEyuOjD9RXnfg3V4H8UWsSyAmUMn/jpP9K9PvY98G7upzXTRXLLUwrNN6FBB0qQCmKKkFejE4pDl61IKjxThWpmy5Ecp+NJdfaWspo7SZYLh1wkrJuCH1x3/wA9aSA/IRWJq/iCWyvTbW8SEoBvaQE5JGeMGssTiaeHp89V6bHHisdSwUFWqvS/a+voO0/TNc0y0FvbX9gFyWZ2tXZpGPVmJfJJ9a1m/tN9KkhN3Al8wIW4SH5V99pJ5/GuY/4Sq+/55W//AHy3+NH/AAlV9/zyt/8Avlv8a86OcYOOib/E4qnF2EqS5pb3v8K/pmjp+ma5ploLe2v7ALku7NauzSMerMd+ST61twefJYtBqRhuGcFX2IVVge2CTXJ/8JVff88rf/vlv8aP+Eqvv+eVv/3y3+NOOc4KOzYq3FuFqu8nrvdRSf3o2E07V7eEWltrAW0HyozwbpkX0DZwfqRmtKK0tbXSfsEcAa3WMoVc53A9c+pPOfrXLf8ACV33/PK3/wC+W/xp8Piu581RPBCYs/NsBDY9smnHOMFe139zCXFeFqtKUmtf5ba93Y0o7DV4bcWtrrAW0AwjPBvmRf7obODx3IzWjbWsFlaRW1uhWONdoyck9ySe5J5NPi/1QHccU416ygo6o9ypiKlRWl+CSu/O25e05siRPUA1FcxqV3PyF5IPeiwbbcgH+IEVLdB9rKEB9fmxXHjV7txUnqYrXcm44eQDPZsf0opxilz94D2DdP0orxuaHmd/LE+bPig+7xo47LBGP6/1rjmGJWzgV1nxLbPjW56cRxj/AMcFcqPmZgec+le3Q0pR9DlrfxGAJBoP1NO2gkUuOgx17nvWpmN3H26UhznPrTyuOc4zRj5s9zSAYDj1/HijHHANSbM/d/lQF59aLgR56YwM+tL2xzinqpOecgnGMUgXpkH39qLgRk8nHWjPOQAM9vWn7cDB/IikKgc5HHancViNuOcjrjr0qUH5R7DHWlt7WW9uobW2j3zyuERAerHoK6dfh34nwM6Zgjr++T/4qlKcY7scYt7HLbsDj86ack9K61fhx4oZSTpy49fPj/8AiqD8NfFLH/kHqB6meP8AxqPaw7l+zn2OTzzngEUZ/Edq69Phf4rcEfYIwf8Arun+NKfhh4rIA+wJ/wB/0/xo9tT7h7OfY5FZNhJIB+oppck56V1T/DjxPE5U6fknsHU4/WkHw78UMQo007j3Mi4/nT9pDuLkl2OX3c9vwoLj8O9dO/w98TqxVtNbg9nU/wBaYfAHiZf+YZJ/30v+NHtIdw5Jdjmlbtn8aA4yDjr1zXRf8IF4mAydKk9/mX1+tI3gTxIM/wDEqk5x0K/40+eHcXLLsc2xIHt/OnxsdpIPFb7eBPErfd0qX/vpf8afH4C8UZO7SpuPRl/xp+0hbcOSV9jA345zx3FAkIHcY46/59a6P/hBPEpB/wCJTN0/vD/GmnwH4kyf+JVNyOOV/wAaj2kO4+SXY57ee5+nFKJCq9ecVvt4E8SEc6VNn6j/ABpP+EE8Tf8AQJmPvkf40+eHcXLLsYHm9ODnGMUolIXPQ1vjwH4n5H9ly/8AfS/40o8BeJsH/iVyfiyn+tHPDuPll2Oe3nPNNMhxzjnk4rpU+H/idyQNKkHHOXUf1qJvAniTdj+y5v8Avof40c8O4ckuxz/mlj+P0pDKOBnkd/WugPgTxMOP7KmOD1yP8aT/AIQTxMcf8SqbnpyP8aftKfcXJLscyrkOPXNOMh6ZNdB/wgfifp/ZUwx7j/Gnf8ID4mx/yCpPzH+NP2lPuLkn2OeEzjkOR+OKd9qnUgea/H+1W6fAXib/AKBUo49R/jR/wgXiYnH9ly569R/jT9pT7hyT7GKuoXQXi5m59HP+NOGp3oH/AB9z4HbzDWw3gPxP30qX8GH+NJ/wgPifP/IKk/Mf40c9Pug5J9jIGq3qt/x93Gc9DIakXWL4DH2uX2+c1p/8IF4nzj+y5frkf40v/CA+Jh/zDJP++1/xo56fdByz7Gb/AG3qHT7XJ+PenHW7/obpzj1wf6VpL8P/ABS2caW//fa/405vh94oRtp0x8j0dT+uaXNT7ofLPsYF1fXFyQZ5CzDgEgdPwptpeTWc5kibDEEHjPFb7fD7xOCo/sx1DHAJcY/nUq/DbxQWA+wDn/pqo/rT9pT7i9nPsZsfiO9Qf8smx1yv/wBeph4ovBx5cJP+6f8AGrbfDjxSP+YcPr5qf41Efh/4nXrpvH/XVP8AGlzU31Hy1Ow0eKrgdYIfpz/jUi+KpM82qcf7Ro/4V94ozj+z+faVP8ahv/BfiDTLKS9u9PMdvEAXfzEbAyB2NK9N9R2muhaXxUw+VrZCfZ//AK1P/wCEnUjm0/J//rVyYbtntThIe2M0/ZonnZ1i+JojjMDjI7NmpodfgaTHkyD05FceHOefyqzbSnzh154/SplDQpSOg1XUIrqFFVHDA5wfpWBHJ5d6j44WTJxz3q3I2V6Z9M1mykiV/qOlKmEmdgmpQY6sPwqZNUtsAbm/75rm0b5MjPPept3APGKzaKTOi/tK2wDv7+hpDf2+ceZ19jXPB8L3B/nVae/ig4zub0GKFFvYblY6WS7t9ufNAHrXJapewtNLFEdwZ/vDp17VSuL6a4JywVSc7R0qqetdMKdtzGc77G/pdxr2qoljaXbNFaofLidhtQFsnAPua0n0fxSoJJib/gC/4Vytvdz2U7SW8jRseMqe1aKeKNYRdv21yP8AaAP9KU4Sv7tioThb3rl2W18SRn5o1+oRP8KrltbT70Y/FVqBvEmpuPmuM/8AABULa1eOMGQf98ikoz6pA5R6NnR+EtS1O08Y6NLMm2MXkSucfwlgD+hr6sZQ6lT0IxXxhDrFzHPHLvOUcMMexr7Ntp1urWG4TlZUV1+hGa568WmnY0pSvdGYFwSp6inipbpNs5PY81FW8HdXIluKBThSDrS1qZsngPauR1tHk8QyxoMuzIFHTJKiusiPziuW1STyvFBkxnZJG2PoFrys4/hU/wDGv1PnuIf4VL/HH8mXYYDp+oQqjsXYKZELbtpPXHtU95ZXE91cM7PEgBZZRgjpwMGq+jSxTSveXUq+YTkA1f1e4LCOCNvlf5i2eMdq9dWsfQPcgS5lbR5PmI2uoz7HHFV7dpLS6hm80skv8OBxg4I960/IiGlSwQyI7bCTtIJJrJdhJZQkHJWQj8CM/wBKoksX32ie9nSOQK6coCMjgZxWLqs7zzws7FiIgMnuMn/Gt6V9uqQSdBIik/yP9K53UV2XIQ/wjb+RIryM8/3R+qPA4j/3F+qO8HDOPfNB5NB4mPuKDXsHvEkDbLhG7BhWrKmWPuKxa3Cd6I3qM1hXjeLKi7Mymi+Y/WirpjGTxRXi8h0+0Pkf4jc+N73GfuxjH/AFrm8Eu3BPzda6nx/E03jS9eJd6Hy8MvI4RfSsXEMZfehD7vWvUpO1OK8iams2U9hxnGfalAzwfyx7VdW6gQbQmM+4oEtl12N/h+tVqSUwhOOvFAQgg4Jq4JbM/Kobk4HJpoktCRjIB4o1EVthyNp/SlAzkdB3rRtbeO6RnToDjJPep/7OUA9PxNTzFWMcryeOelJ5fbuK2l01QfmGBnH3hTxpcJ4OcdvmFHMPlMEqD0HQdKRkK9Qf6V0H9mwYOWbn3GKY2lWzD/WsPyoUw5GVvCbY8Y6HnHN7H/6EK+rAkbYJRc464r5e0W3W18baGisWH2yI5OOu8V9PB8KPasq65mmaUnZMd5UQP+rT/vmlMcR/gX8hTWnUjbt5/vU0uax9ma847y4s/wCrX8qXyoj1jX8qj84JyQT9KaJc80ezDnFMMe/cEXP0pTGn90flSb6TdTUA5iQRx4+6Pypwjj/ur+VRBx607fRyBzD/AC0/uj8qb5UZ/hH5Ub80nmUcgcw4RRg/dH5U4InoPyqPfS76OQOYeEQD7o/Kgon90flTPMpC9HIHMO2KD0H5UFU/uj8qYXPqaTfT9mLmH+XH/cH5U0on90flTd9Jvo5A5x2xP7q/lUciJt+6vT0oL0yRxj0pSp6DUxyKm4/KOlIVTao2r+VRo4yPcUgfkVn7MvnJPLTYflX8qZ5SbM7Bk+1IZfk59aGf9370ezDnJBHHj7i/lTDGhJ+UZPtSeZ8uaaJMvz2p+zDnHPGgA+UflSmKMj7i/lTJHyB9aXzAQDTVMTmRtFH5g+QdOwpjxIxwFX8qc8nzZ44FMVuMnOSafsxc5G8MakAKuPpVXUryz061Mk+EB+VSEyfwqxdSCGIyNnCqWOPauW0nXTqWoJPJ1C79u7hQegq40ridQ1heS3zEafpM92y9JJWKIvsMnn8Bim3j67ZQbzoayDqfLdWwPpjNdjZzx+Uqoo6du9S3MxROQAT2q1RjYfO27HI6fqFrqUZ2xBJlHzxsACPwqZ7eNjgRr78dKs3ttbhmu1gVJcZMqdTjsaZF8yBx/FzWagm7Dm2iFYUD4Kr19Kz/ABXax3HhXVYti/8AHpIQNo6hSR/KtR5cMVH07dc5z6+1Raign065ixnzInX81Iq407Mzc7o+WMDP+FKMdzjHFKUIz+uaXaWbgduwruOMADngjj9alg+WVT7jkU1VPXH508ADkAHnNSxo0HIxg9TzyKo3JPnA+3p1rQcDaMdD14rPveiken+f61nT3LlsXYCTEhHoOQabNOkSbnOPT1NZ4vTFCqKoyO5qq7M7bmO4+9Wqd3qQ5lq41B5Rtj+Rf1qln3oorVJLYhtsKQ0tK1MQhpKXsDQRigBKKKKACvsHwBe/2h4A0K4zkmzRCfdBtP6rXx9X0/8ABG9+1fDeCI9bW4lh/Xf/AOz1z4lXima0Xqd5eJlA3ocVSrUlTfEy+orM71NF6Gk1qLSg0lLW6MRynDZrn9c0+8/tVrmGF5FfBUxruwQAOR+FdB7HpViM5iQnqVGa58XhI4qnySdrO+h52Y4GONpKnJtWd013ODSw1CP7ltdr9EYVYkXV5ggeG6+QbVxERx+Art6K5P7Lqf8AP+X3/wDBPO/sat/0Ez+9/wCZxMH9s27l4obkEjHMJP8AMVGsGqLnFtcAZJx5J/wruqWj+yqn/P8Al94f2PW/6CZ/e/8AM4mU6zP5fmQXB8sYXEGMD8BUa6bqV5cIr20oY8bnjKqB7nFd1Sg0pZO6itUrSa7NkTyJ1Fy1a85R7NkcgxIh+opSKJfug+hBpTXsn0A2te0bfZp7cVkGtLTmzA6+hzUTV0NE2KKfiiuH2SNLnw4jmNVAYj6Ggu24hgQQTngcVFAGZEGefarsOiapdTsIY8sBnlxXYo3ZPNYriU/7XPSnLLj+EjnuKu/8IrrgH+oAH/XRf8akXwrrZJHkp0/56LVezfYXOu5nCXIHy/8AjtL5meApz/u1e/4RbXAOLdefR1/xpB4b1xR/x78f76/40vZsOdFVZnA2KXA64VevqePwpwuZ84DzenBNbmmeGr94XNza5cNx849PY1fbwzOCClmR7B/p71Di09i079TlVurlnGHl5GO9TLJfEfKJvwU810ieG7pWBNq68/3x/jUo0i9jJCWrcdOhqHfoi426s5gDVWHEdyR/uk/0pHOqxDLw3Iz1JjPP6V1gtr9AQbQn/gP+FRNDqG7DW8gzz9wgVKcuxpaHcx/DUlw/i7SBKHBF5EPmXH8Yr6bB4r50sIZIvGOjtLGyE3cXUY/jFfRQ6DjmiavYUHa4pNJ5gFNOccCozkmkoiciTKk5pcrioAGY8DJo3c4NVyC5yfK0uV+lQBqTdRyBzljetHmL61W3daN3NPkQuctF1o3rVXd/9ejdj6Ucgc5a3rRvWqu7il30cgc7LO9aQutV93vSb8mjkQc7LO9c0m9fQ1Bu+tIXo5A52WN6+lJvWq5b3oLU+QXOWNy+lRyuu3pUe8+tRyvkChwGpkgkTcvH8NNDptzjvVffgK3+yaA37k/Wo5EPnZP5i+lKZE8vp2qtuzuP4Ubv3ePUUciHzssNKgTGMZpEkTd0PIqqzEn29acGCyj6YoUA52WZpF29KcJFIHHFVJW+Snb8Ln0FPkRPOyR5FJC7etODIMccDtVNn5BHTPrU2+nyIOdj5yjgAqMHg15tFbTweILfTYYJoR5pLM6bSUB4IyOc16JI3AIA47Gqk9lHFIl9GEW3hGAijl2Y/MSfUcH8KmWiNaXvPUvzxzC1EcG/DcMVk2HH+92/nXNaNp051qQ7IY03fOY7h5iV92bvXUPepaW7yFgwRc7V/iPasR9aKRu8eoWEU+eIGUMoH0BBz71imjrUXuVNUhvItbkizMyocAxz7cKeQSvQ88V0EZCQIgx8qgVzy6zNez+ddR2pcELiI5x75roVUlA3fAp0bNszxScUivKcuccUvUYPfipDAz7nTlVHPP8An0pMfJ+FbcpyJniE2l6HFdzRO1rvSRlYGQAggketKunaB2+ykdeJf/r1n+JfDEzeJ9TZGIV7mRlGwnqc/wBayT4Yuc8N1/2TUOMf5jVSl/KdUNH0EqOINp6Ym6/rUqaFoTkgCPjr++Ncf/wjF0P4x/3yf8KY3hy7X+IfkaXIv5x87/kO7OkaJjAAJxjiUmvNJvNkuXjyWKkgAe3/AOqrj6BeKM5XH4/4VmSxvBO0T/eU4Na0aajezuZVpuVvdsatrZW0unSebb3Au9+EbPybfcdfWg6VHgjMgP1H+FU7Cxub7zPIYfuxk5bFWP7FvuP3i/8AfRq27PczWq2N3TNB0O9ljt2N8JmXk5ULkDnHBrbPw/0ZWw1zcqf+ui8f+O1xUek6or7opdreokIqdbDXRjbeSDjH+vIxWMlJvSZvCUUtYHYf8K80dhxeXQ/4En+FZHijwXY6LoxvLW6mkdXVSrlSMH6D6Vl/ZPEg6X0/H/TyeP1pJrDxFcwGCa7llhY5KNcEjP0JpRU003PQqTg4tKGpzmPl/GvUdP8Ahdp99p9tdjVph58SybRGvGQDjrXDDwzqmCPLXGf74rWgg8XW8CQw306xoNqIJ+APQe1XWk5L3JpGdGKi3zxudNJ8JLcfc1SYn/riD/WoG+EcnJGoyAeptv8A69Yj3Pjezhkm+33QREyzCboB1rNbxz4mZstrFwx9yD/SsYwxD+GaZrKeHW8Gjpz8Jbj+HUc59bc/416x8JNBuPDWk6hp803mq06zKdhXGVwev+6K8a03xB4/1K3+0WN1PLFuK7hHHjI7civTvhRqHi2bXLyHxEZGt2t8xFkQYcMP7o9CfyqJe2vac0x/umrwi0euVmzpsmYds1pVUu0+634GtaTszKa0KuM04UUV1IxFqeM5T8TUFSxH5SKpEMloooBqyRaKSimIdSim9KWmAkozG3rikU7kVvUU81HHxHt9CRSYAau6Y2JXX1Gapmp7Ftt0vvxUy2GjTwc0VJtorn9mO58MWy/NEMj7wwPxru9EG+8lAwBszz9RXF+Q9teLFKjRyxuFZWXBBz39K7jQlHnysSOFAxj3rpp7kT2NooVH3gQeMA00cNnpzmpcDnLD8aNo5O4fnXQYERBzjH60dccVKyrwdw/xphUeo/CgCzZ42H2YVZJBP8qgtFBRhnPIqyUIxxWctzWOwzvn/PekOQcjGaeRkAYppwD0/wA/5FSURnOM0jc8cYHan7SV5B645pkjiONnc/KBk0hnN6iceNfDw4z9pjJx/vivdlIIrwOdXuvGmhknY0lzGNx/h+cYr3FY7gD/AF3X/ZrmqO7N4aIumVjGYxjaetMK8EVX8qc/8tv0p/lTYP74/lUoGPWVoMlcbjxyKrbiSSTyaZJDPu5mOM+lRNFJ/wA92rRGbLG/3o35HBrlPE+qXWjNaMly4WQsGGBzjHt71gnxje5/1xI79P8ACi40j0cvyaC9edL4vue87Z+g/wAKX/hLrr/nu+PoKVwPRN9G8Y6153/wllzx/pEvpxij/hKbn/n4lFHMFj0TzMUeZ9K87/4Sm673Ev505PEl0/8Ay8y/Xd0o50PlZ6FvpC9eev4nnycXE3/fVNHiS4PP2ib8WNLnQcp6Lv4pC9cPBrkj43TTf99kVcGplwCJ5f8Av4al1UilTbOt356UheuYkvWWNT50nI/56GoxdsysfPn4/wCmhqfbor2LOq384xTZG+XNcm1y78LdSqccZkYj+dMgupd2GuZXI/22x+WaTxC7DVB9zpS5KL+VLvPlt161zy3T4z5jcnGS5plzfMjECVuvZzUfWI9ivYM6RXJ3detCs3ljg1yf9pMD/rJD/wADNPXUyV4LH33Gn9YXYXsWdOpYvyOB1pWciUEA4rkptTIGCzH6NVQ6k2eXb6ljxQq67CdJncSyHZ3/AAppkLbUA7dq4g6m/wDeb/vuozqbYzuJ/wCBVftUL2Z3UjkbRUu8+h/KvPTqbZH17Go/7V6jK9f73f8ACmqgvZnoUz/KBg/SsLxdfahY+HJL3TvNE8ZVWCn5SjcHcvQgdea5j+1SwGwpgHk5zmq2naqZ/E9lBcR5t2d0+YnBco2D7YP9D9Naa55JEv3eps6N40hSVNO1OXcMiOG6lQRiXA6MoJCt+JB9jxXZJBbyQPLbi2WRslnQ7GJ+o6mvHfEWgPZL9oCyXVhM33HwHhbJzlzx+J6+vrlvrlxBpEsumaibQQyeX9lnuQ7kdMhSp4+jcY6U6mHjK8oP5GsMU4WjNHpniHULDwxazak6QyXsh2xhmLNI3YEk5xXZ6bcyXmk21xPALeWaJXeInOwkZxmvnPTb95tai1LWmm1OC0lUuDKcEZ4xkdM4yOK9u0zx34avwqNM1mx6C6QKp/4EMjH1IrKFLkQVq/tX5HQSSMmFjZhkYOO9R7jsxT2tI5ow0Ko6MNwK4wR656YqstogJBQAg4IIqmZI4vXQseuXOADkgj2OBVDBHzFRgnjJra8R2wj1cbQFDRKew9R/SspcDou7sCegrzamkmenSV4IRdoToQcdKlVWb5nBAHAJP8qRYi3rz69KnEDNhQS59hxWTZukR74wVOxiF9Tx9cV5T4uTHia6brv2tn1+UV7NBayQ/OFQMOdzgNj8+K8w+JMb/wDCQwztyZIFycYyQSP8K6cHP95Y5sZD93cPhynn6vfWecCW2Pf0I/xrv/7IdZ45UkQlWDYC8Z2gEfzrzn4ftt8ZwR9fOR1/8d3f0r2+CxLsPlyB1rPHS5ag8HBTp6mNYaeI97TxR+Y8hbcozgYAxzWnBpdtIjboUHHLsox+HrWoloT8qgBT2Xof8atpZogG7JP90V5sqx3KEY6GMmi2kh4gQjPLGMVOmiWSNg20DnHGYRW2sJOAoIx0z1qzFaEex/U1l7WTE5RW5jRaHYZLSWUJJ7GNatRaHYHpY22T1JhX/Ct+20t5Dnbx69Klnn03S/8AXyh5e0a8mtY0aklduyOWeIje0VdmBJ4Ssr6znt/sMB82Nk4jA6g96+VdX0ifR9VnsrkBZInMbAHOCDg19i/aNW1NNtnALOA8b5OD+A//AFV8o+Pbaaw8Z6ja3EnmSxXDAuf4ueD+Ir1sBHkbSu0cVeTkrytc9R+EGoXms+E7jRbTSbUi3mHmXG0AkNz8x5yeD+lepaP4SbSpGvZ7pnmwfkQbUGfbvXlH7OWpCK517TyR8yxTL+BZT/MV7tNcblK0Vo04VG3uTGrPl5Y7Fao513RN7c1JSda1iUzNpaV12uQexpvWutHOxR0qWI84qOnRn5hVIhk9GaTNGa0JFzS03NKDQAtKDTaUGmIdmooziSVfcH8xUmaj6XPsy/yP/wBehgOanwNtmU+hzTDSxffpMDoKKhSUbFz6UVmM+I5JZJQJpZGeQgMzOck49/pXZ+GnEkTyj+JFNcY6/wCijGM7Ofyrp/Bsm/TGOeQQn+fzrSnuTPY6ncfXFIW9uM/So8nOB1pCcjPpxj1roMLEytlSPQ56U3PXHeowSpPHXig8N3NIZfsZMOyY68j8P/11dLDp3A61iwyeVKjeh9K1x0wDkevr0rORpDYerAHpgUw4OcD3peCo/lQevU81JYg681Wv8/ZCMfxA/rVgjGOeKr3mDat6cfzpPYpbnMl9vjPQW9LqP/0Na98UV8/zf8jbofP/AC8x9v8AbFfQSg4rnkaIYflwAuT6+lLzg08rQRwaEJkSNGG/fKWX0FVSOTgcZ4qcjk0witCGzg/iLbvPFpyqxH7x84+griBpU5xl5P0r1rWLBb+5s426KWI/IUkXhmIsG28ewo5Li5rHlQ0qU/xyfhj/AAqUaXIB96T2/wA4r1k+F4eMKePanr4YiH8FP2Yc55KNLk65k/HH+FO/suX0k/P/AOtXrq+GoAchKd/wjdv/AHKXsmP2h5LFpUjMM+b+dWZtGkiYrGshXHdutepr4cgH8FWm8P25+YoOlS6TuUqiPF20yYdUk/OpYtJkLD5H/OvW30CDJ/d0LoEI/gp+xYvaI4Cw8PxyAF1l/wC+yK2k8N2ZwPKk9/3zf412NvpEadFxVr+zkz939KwlQlfc1VZHHtoFiioPKZu/Mz/4019Ds1iYiCQEcY818fzrrZ9NYgbV7YyK8N1Oy+KJ1K6W1XUDbiVthGzGMnGOM1n9Xl3L9sjujosO8lUZRj/no3+NV/7HjLEgHPs7f41wq6V8SpWXzU1U7fm+8g/z3qd9N8dtdwOttqtuisN++53Bue+e1S8M+5arrsdimmRLj5eh4G48frUd5pqhsbcj/eNdIkBMZ3KM9Kbd24GcqABWPsXc19orHJLpwZseXn/gRq0mkwlB+5zj3Nbi2w3HAGQtWktx5Y49qUqUgVRHHXOlKOBEAfes/wDs1dwBiXk/3a7a5tgZPwqitoJLrI5C89aqFKRLmrmENFh2jdChJ/2aP7Eh/wCeKZHfFdUttumGRgAcVS1yX+zNIurpcbo4zg4/iPA/WtFTYnJWONvJNLsXMbPD5gPKAAnPfmsa51e0GTFDHxxkIDiuUvbthKzFiSOOT1z/APrpEfzIyTn6dcfSu2FBLc5JVmzVuNYlZ1wUKgkEHg/n2plreumrWc+QpjnTB7YDc/z/AFFZqs8bcHj1p4XYC2SGxkEcEen9K6IJRaaMnJs9i1GXT9Pu5kvLy1SOVQ0kMrru+oXrz9K4q58J6PdrLqFomYMnCwy5jJHXGP5ZFadjotrc2SOqOxk+Z2cAMSeuecn8acmlXWlFxpzLGbjiRRhV4/i2/dyO2AOveuytgZTl7RaXKhWXLyvU5a1tbPTIrm1u7W4aS7Ta0cMYIiU8Ak56jrgDmuXMs0AeFZN21iudpB478jP516lDo5hJMuSc5bc2SSe5J+led+IbI2Ws3KANtc+Yu7nrz/Ss69BwgmTzXZY0TxXq+gy7rG7eNOjRdY2+qHj9K9N8MfFC11e+istSiEN3O+1JYzlGY+ueVyfrXh7nCDn61d0Dc/iPTQMkm6i4H+8K4mikz6A8TQI17byORzGRj1IP/wBesdbbIGSqp0GP88103iGBnFu4XOGYZJ47f4VixW5Y4A3H0FeNiHabPawyvTRHFbr0jALY7jJq3FbmP5DtJ65JOAKsRWzBeXwem1en41ft7MuckEj2Nccp2OtRKS2gY43Bj03Nn9BXnHxb0/yH0q5HKuskecehB/rXsUdnuIAXGPSuE+MmnA+GrK5ySYrnaSOwZT/gK0wdX9+jDFpOk0eWeC5PJ8daOzHh51X/AL6+X+tfTEdqNoA6deTXy/o7G28Q6Xc4P7q4jc/QMDX13BYs+ODit8zi5TjynFhavs4O5nJag9Cce1WYrHOAAMVspYJGu6Vgqj1NILuNTss4TM/97sPxrkjhGtajsOWKb+Erw6WcZYBV9TTmubO1by4ENzN/dQZqc2Nxd83cxC/8804H41bht4LVNsSBRXbTw1tYq3m9/uOaVW/xO5mNa6nqPE0wtYP7kfLH8as2ej2GnnfHEDKesj8sfxq28wHeqslx71cpUqeu78yeaclZaItPMFHFfJnxotvs/wAStRcD5ZRHKPxQf1Br6gkn96+d/jzb48UWN0B8stoFJ91Zv6EU8NiXOtZjlS5YNlD4JX4s/iJ5WcC6tZI/5P8A+y19GyT89a+TPAF6bDx9ok44zcLGfo3y/wBa+pHk461z5o3GomuqN8JHmibKncqt6jNLUFo++1jPtipq66bvFMmSs7FS5TD7vWofarlwu6L6GqddkHoYTWoU5Tg5pKD7VoZsmzS0wHIBpasgdRSZopgOzzS5pmacDTEOqOQ7XiPvg/lTs1HccQlu6kN+RoYEppYvvdaaeaWL79AFzdjvRSbveiiwHyD9ildxaIFMpOzGeM9K3/DOlXWmQXEVyFAZwVw2e1Z0U4i1JLhg2BJuIHXrW6NettufLlx+H+NEbLUmVzS2n3pAOeelUBrcA+7HKfy/xpTrkDHlJc/h/jWnMiOVl3AxR3x/I1Q/tm3P/LKXp7f403+2oM8Ryc+wo5kHKzRxWrbSmS3TJOV4Ncv/AG1B/wA8pfwxVm28RQRBt0MpBOeMVMmi4qx0wO3mmnHByawz4otATiCfA9h/jTT4ptT1t5v0/wAazNNDeznnjgcVXvzm1bgjOBwPesj/AISm1HH2eY8+3+NVbvxZaPCUFvPk9+MfzpPYaaKt0wHirRCOB9qj4Pf51r6EUV81x6pHqfinSDHHIgS6j4bGT84/wr6VXGKykWhaaelO60wmhCZCRzTSKe2KYaszMPXtattANrd3S7o2do/vAYJXPf6VSi+KGhxj5sH/ALbLXRTaVYatiLULOC6iT51SaMOAfXB7806Pwd4cXkaFpo5/59U/wrSKfQltHPt8WNCA4CfjOP8ACmj4u6HjpF/3/P8A8TXVp4U8Pr00TTR/26p/hVhPDmhr93R7Af8Abqn+FP3u4tDij8YNEX/n3/Gdv/iKb/wuDRznaLXH/Xdv/iK71dD0kdNLsh/27p/hUi6Vpy9LC1H0hX/Cj3u4aHnx+L2lAZxaAf8AXd//AIirulfEhdfvPsWmWcFxcbC+xZmHyjvkqBXcLp9mp4tIB9I1qteafapf2VwPLtyhcFlCrnK9ORStLuPmXYyRqfiJxuXQoiM4/wCPkUo1DxJ/0BIBj/p4H+NbMEthaQpB9ujYLnDSTgscnPJz70jahpqEhr62H1mX/Gn8xfIy0v8AxH1OkwAf9dAf/ZqsC+19v+YfAPxB/wDZ6sNqukDg6nZAe9wv+NRtrehAgnVtPH1uE/xqXHzGn5Dftmv4H+hQD8B/8XQbjXiv/HtbA+4/+zpreIvD4yDrGnD/ALbp/jUR8TeG1IzrOnj/ALbr/jS5F3HzeRXGoa29/PabLRXiRHO7gENuxj5v9k0ySLW5D8z2WP8AeqmvijwzF4oubhtUs/Lks4l3q2QSryccd/mq83jjwovP9r234Kx/pU8ie7K52uhB9i1XjMll1/56f/Wpk2m6rLjE1kv/AG0J/wDZambx94UX/mKxfhG/+FNPxC8KDH/EzBPtC/8AhU+xh3K9rLsVRpOrhv8Aj5s89OG/+xqaPSdX24+1Wq/jn/2Wkb4i+Fh/y/ufpA/+FRt8SPC4PF1MfpA1HsYMPazCfRtTDhDqNsGcEgAZJx1/hrjoJ/F7AvF4fvmU8hg0WGHqM11kvxH8LSAeY00mOgNvnH511Wnuk2nW80fEckasoIxgEZFL2Mege1l1PLw3jYnI0C+U57y29Yfi258TRaMV1bTprW3kkCh5JYWyw5A+Xnt+le5EA9ya8r+NU+zTtLtgx+d5JDj2AH/sxpqkkwdRtHh9zLvkePqOlJaSArhgMr1559KZNGzPs3fOBx7/AEqvblluiGBBwTgnHNbGZpj7wOT16d8f5/rUp2lBtAwDngd6rMy7Dn0wAOv+elWEAZNwA+bB9+f8gVQj07wxd+d4etpIVR5kjCsT2I4/pVmSWZ7uIyquW3LgD2P+FYngmQyaOwVgzRysmc++R/OuhupFhhUSSqZGYKhxnJ6fpXvU5c1NMytZjCqIoxy7HOCSOM1wvju3HnW92AcNuQ549xXfReY0a7wm7ZhmXHJx9BXL+NrfzdGaQdYmDgj9f0NZ4iHNSY09TzCeMqM9Vz6V03wz0xtS8d6cuRthY3DZPZRn+eKwJot8b4y2Oa6P4WXS23xA0/ecCQPGPqVNeBLY3jufQOswiW2TIHDg889qz7fTwUBBx9f8KteIdXg0fRLjUbiKSSGAKzLGBuwWA4yfeuKi+MXhxAD9h1Ec9o0/+KrxsXQqzneCPXw1eEKdpM7aOwuPNjVSnk/xZBz+ArUitQQAOnevPB8bPDqnDWWpf9+0/wDiqnT44+FVA/0LVCf+uaf/ABVcTweIe8S5YuPRno8dqMjB/Sua+KGkSXngK7MSbmhdJRjsA2CfyJpujfGHwffJI0jXdtsGQJoh83sNpP64rF8U/GvR7nR73T7G2ulM0LRhyikcjHPNb0MJKDUnucs6zlp0PFrLybe8t2uATGsmXVfTOeD/AJ619haff/b7G3nsYW8qaNXWSTjgjIr4rnv1mb5VKnOa9+8PfHfwzp3hnTLK7t9Q+029rHDL5cKlSyqASDu6cV6s6cm0zlbVrHsH2ESNuuZDKfQ9PyqceXEuFUAe1eTN+0J4Txxbap/35T/4uqz/ALQXhZv+XTVf+/Sf/F1m04fBDUmze7PXnuAB1qtJc+9eSN8ffDBHFpqn/fpP/i6gb48eGm6Wmp/9+k/+KrkqRxMvss1hGC3Z6y9x71We4rypvjp4bP8Ay56l/wB+0/8AiqYfjj4cP/LnqX/ftP8A4quSWFxL+yzojKkup6i8pNeO/HO282y0q5A+40qE/XaR/I1ePxv8Of8APpqX/ftP/iq5bx18R9D8UaLHaW1terMku8GRFAxgjsx9RWuEw1enWjKUdB1alJ02kzzXT7k2epWl0v3oZlcfgQa+tgS4BXJDDIxXyBkY4znPFeywfGSwj0y1t3jvTKkKJJ5aKi7goB/i3EZz3Fd2Y4adbl5EZYOtGF0z3HTci1KN1ViMZ59auVwHww8XWfieHUktY3jNuyMysgX72emCc/d7131FKDhBRluObUpNoRhuUj1rPbgkelaJqjOu2U4rppmMxmaM03NLW5iSIflHtxTqjQ9R70/PvVogWlzTc0uaYC5pRTaAaYh+aRxuQqe4xSZpc0wEjbdEp9qki+/UMX3SvoxFSx/epAXe3f8AKimfjRQB8jX7NBau6HDqAc498Vpp4c1RkB+3wDIz/q//AK1Z+qqWsJRjrj+Yr03R9EvNXOyzgaVkXLBQOBRFJ7kzbWxxA8Narkf6fBj/AK51IvhfUzkfboc/9c67OW0ktrgwzKVdTtYMMEGuw8PeEBqtqZzJsUcDIzmrcYrUhTkeNHwxqpG0X0RbHQRGkk8N6qnD30akjjMNem6/oraTdNFu6dxXNy5Y8kkkY560KCYc7OSPh3UjjGpRdP8AnlR/wjuothTqcQwf+eI/xrqKaenA5p8iDnZiDwZqzcf2tFz/ANMaVfBepHJOsR4/64f/AF66+1fzLZWzyPlNWVUA8/jWTVjdao4j/hCL8nnWk/C3/wDr1Bc+BrtIi7awG6Aj7Njv9a70jIOTVe//AOPR2OOCvT6ipexSSPLtPs3sPF9pbPN5piu4fmC7c5IPSvqNPu180SHHj2IA/wDL3AeP+A19LoMLispDW4tMapPwpjUkDIWqM1KwzUZ61qjNlS81UaQiTNC0u9tmFIGOCc/pVX/hNhnH2B+D/wA9R/hVfxQf9AgOcfvx/wCgtXMZwa2hsZTep2I8c4/5hzfjN/8AWpT48IHGnfnN/wDY1xm79KM81ViOZnZf8J5L/wBA1B/22P8A8TTT49uD00+L/v8AH/CuQz2pM4zilYOZnYf8J5dfw2EOf+uh/wAKwvE+t3Xii0t9LlhigilmyzoxLDAJ7+4rNDHrikDf6ZaHPST+hptBdmNN4Pt4HCm5lbPQ4H+FNHhS0/57TD8V/wAK6jUvvpnuOKosDxtOKlwiUpyMYeFbEdZZyM92H+FSp4U00g5ac+vz1oHcOvap4Q5Oc/KKHFApMzV8J6aP+e3/AH8p3/CKaZ3ST/v4a1g3tT9341PKiuZmB/wjunJqjJ5chj+yh9hkOM7yM0y80TTYod8dvjB/vnn9a13/AOQonX5rRx+Ui/41FdgNbuMdBQoqwOTOPeG1i1QRvGBFsLYLHj9arGJZUnkhPyR5IOeSf8KsatgzwxKRulcL+A61DAAqalHzwTx7YqbFXEuRGumQzKuGJGTU1/Ci2yNCgHzAnHp3qnK+7wzGccgDvjvTtTuR/Zsa5+ZlBwDziiyC5biiSeaOUxhYS4CgfxD6VsL4u1+DWJbBNVuFtokAjRSAFA464/zis1pAtrER/snI6dKijiW68T6j1DRMcbT2yam6RS1NXUfFfiCKW12a1eKrPhgJOvIrD8UatealNEl1dzXBiUkebIW25POM1sS6dbzBHd2+Q5Xnp/niuZ8Qxpa38iqflEascnJJP+f5UozTY3Fow51gVN8pbPXA79aqQNvlLHIUDgenNOEMly+5jtUngZ/z/nFJEdl6ykg/LjjtWhJYk4BJIGfT/P8An8altSnkupA+Q89Oh6f1qu7ZGSeOlJbyhLnGRhhgnHQ/5/nQB2/gq5VDcxMxUvIGyTnqv65xXS3f764UJnEbAjA9+1cj4PljS4u1ZQzBVZSBxnJ/LoK7FXS1tRJKPncjg+uf8/lXsYV81NIzloydmnLYUYVc9+vTn/PrWP4kjkfSZI3UBnQnnNdBGZovklxtBPzDt/nmuV8Tvd6iZYYHMaIDyP8AP61vV0gyep5xGx2BuTjOc9q1fB1tJL420qOAHet0jgjuoOT+gNY0X7pnjP3lJB/Cuo+Hryr4+0to4/MyxVvYbTk/gK+flobo971fSF1zSLrTGk8sXMfl79u7bz1x3rkrP4AQNgtrEjL6m3AH869CtJPLuY2ATIP8f3fxrojY3N2c3t2BH/zyh4B+p6mvPrOfMuVnTFpLU8j1P4M+HksDa22oXU2o5AXykUj3yO351VsP2dVmG+91uSIH+BIQT+ea9whjtbNNkEaoB6Cla5HrWaq+z1lL5ITfNokeQt+zxpKj5Nevlb1MSGqN5+z3Yx20rp4gumdUJUGBcEgcd69le596rS3GQQTWFTHW+EqNOT3Pii8tDaPtJ3ZyOmOaqVv+JoGh1a6hK48qeRP1/wDrVgV69OXNFMwmrOwUVcstKv8AUiwsLOe6ZeWWCMuVHqQOlX18H+JH+7oGpnHX/RH/AMKpyS3YlFvoYlFao8M66YfO/sbUPK6b/sz4/PFSQ+EvEVyMw6HqLj1W2f8Awpc8e4ckuxi0V0H/AAg3in/oX9S/8Bm/wpD4H8UD/mX9S/8AAdv8KPaQ7oPZz7GBRW+PBHig/wDMA1D/AL8N/hVTUfDmr6RHHJqOn3FqkhIQyoV3EDOBQqkG7Jg4SSu0ZvG0evevQfBHw6tPFmhPqEmpS27pO0JjWIMOApznPvXnteu/Bq/xY6rZkj5ZI5VH1BB/kKyxUpRpuUNzbCxjKolI7z4beCofB+q3hg1CS4W6hAKvGFAKnIPX3NelGuR0e4/4nEIzwcj9DXXGuOjUlNXludVanGErR2EqvdJ8oYduKsU2QboiK6IOzMJaozqXtSkYOKSuk5xVOHPuP8/zqSoR/rFPrkf5/KpM1aJYpPBP8qi+0jbkRydcYxzSuHJGxwB3GOtIqzZ+aXP0WqEJ9q/6ZSflTxP1wjHAyPemhJdp/e/N2OO1GyXj98QO+B1p6CJo33rnaV7YNPz71WEcv/Pc/lU4oAReJXHqAamj+9UBOJl9wRViL71AFnn3/OilwaKAPkzU8C2Y8csMD3zXqXh7xJeaCWktCmHUB1dcg15Vqp/cdP4wOR716LpkYlu4IWYhJHWNyPQmlDZ3IqO2pcu9Sm1DUJbu4IMsjbmwMCui0LxfPpUBhQI6H+961yuqKYNSuIGfeYZGiB9lOKr+YVP0rXRoyT6mzrGsSapO0srAsx6isZznJzn175o3luM80hQ4poCI8ZyRimkYzyPensu3PPIqM7fzNMZb0+TbMYzyG/nWrszkjtXPqxVwwOCDmughcSQrIOjAH0rGouptSd9A2nv0qteKfsr55AIP61dYZHoBVW8wLOXg8DP61i3obpanm023/hO4iP8An7h/mtfTC9K+Z5wV8dxFgATcw9P+A/8A66+mVGazYLdgaYfapCKY1NCZEwqI1MwqNq0RmzB8UD/iXREdph/6C1cmQe5rsPEib9MHtKD/ADrlNnHFbQehlPciwR1P4UduT71J5eTineTznjBqrkWICPelwe9S+UScdam/s28PS0n57+WaLhaxU7jmjH+kW/J/1q/rxV0add97aUD3UimSWjxSW/mRlC0qYz/vCh3SuKLTdixqKhmQ5PQ9KptHGo+ZsZ9+9bGpxogj+h/pWS8aSnB6Cpvdl2sitmLd94fgaswtCUwX+cngVCbaMEc/rVmC3gI385HI5oYIf5KkZ5pRCh4/ringgCgtyeeKjUvQqrCja5YI33HSVDj6oa7T/hDdMkRlZ7jng5Yf4VxpcLq+nOO0kgyf9wH+lesJg4I3ZPXimh2Rx/8Awrnw5LMkjxzySqflYysMfkK4s+GdPt/irNorRE2NxGjbN5ycoe/XqK9kG0ShQx5Nea+IGW3+NulOTjzraNfqdzChjNk/DvwukH2ddPzEOqmZyP51cj+H3hV0AbSIGB67yzH8Of0rcCAE/Lu57jNTxkLKFGAoPtRYZ8961bw2upXttAMLBM8ar6BWIH6VRhlFt4m1CXu0Sv8AXKj/ABrd8W7R4z1ODblvPY5AxnPP9a5rVH8jXWb5QJLePv1+Uf4Vyyd7ouKtqarXpZduzHriuP8AFN3uviqv8xVd+Pxx/OunVsDg8Vx+qSW8F7M8il52cnBGT16+1KjHW5dR6GfFASvmzyERgZwDUVr+8uGK8FjgDr3qK4uZZjhjheu0dKmsVU7mfhRweK6jAsygN0Oc9efrTIkUMCCc98YPFK7mRiE6HoMZ4qxbwnaGbO4jI7+lAHQeD2RdUnUjG6JWHIIJB/8Ar12bhLm4VGB8tBzjvXnuhnydSwpQb04z9R09/wDPFeg6ViTl+EXrnv8A/Wr1sDrGxnM2POSWBGkAGQCf8elYd3co26NUDMSecVaWG4ny65WKQDaoGCo/nVbU3ttFsDLJzIRtROMk47f410zk36CsrHlmsQCy1ueMH+LceehPOK9A+EmlWd/rc9/Jv86xjDRKpwCWyCT64A6dOa80v7iS41CaSYjezc4HFeg/BvUFh8UT2jMALi2IUerKQf5ZrwaurdjaB7iR2zit2HUPMt43B+8oP6Vh9KisrkiDyyfuO6fgGIH8q8TMW4xTR34aCm2mdF9qz3pjXHvWasx9aUSMenNeK6kjs9gkXTOT3qJpM55qndXltYxeZeXEcCertjP0Heubl8fadJdC00yNrmcg4Z/kTgE/Xt7U406k9kFoo8Z+IVv9n8S6qnpcs4GP73P9a4Y9a7TxlqEuq6tPfTxoj3Cq+1BgcDH9P51xjDDke9fU4a6ppM8qv8bO3+GPiGbQNdujDCspuLcpg9iGBz1+te+aTfahrOlW1xIYo1lBDArkqynB46c9enevmbwizDxLaqrYL7kz9VNfQXhKWW2geBpmeMHcoI4BPXH5CuLMKavzLc68JqrHVLp8AcSTFp3HRpDnH0HQVOXVRhQAPaq5n4zmoXmHrXl3bO5U77lhpc1E0vXntVR7gD8aiNyNwHPWhRZoopFlpa86+Lsfn+GLaUAkxXa5x6FWH+Fdo9z7Vyfj8favB1+v8UYWQe21gf5Zrpw141IszrpOm0fP4++M+td18Krow+JbiEnia2YY9SCD/jXETDbO4963/A1z9m8Y2L5wGZk/NSK9ysuam0eJRfLUTPfbOfytRtpCekq5+ma9ENeSm6KkEdRyCK9XglE9rHIOjoGH4jNedh1a6PRxGtmLQaXtTa6UczKci7XIxTKnuhhgfaq+eK6Y7HPJWYHjB9xTz+tRscocelPzn8atEMWik6miqQhaWm0UwH5oFNopiEkbGxvRhmrMY+aq0vMTDvirEJ3bSO/NIC3x6n86KTmimI+TL5S3lIOczIM/iK76NlguEZWJ2MGzjHSuEuBm5tRx81yn867bK7up45pQ1TJqK5f1ieK71m8ubYlopZC6nGOvJ/XNUs4HtTkZFPbGeuKaT852/ga0WisZJWRds442mXeQB3zXZeKdF0bT9ItpbKcNO+PlDA7hj73tXArLg5BwfWpHuHcAE0NXY0RSYyQKiINPLZxwKa3JNUAw5z1rX0ubdC0Z/hOQPrWSeevb3qxYzCK6Uk4U/KfxqZq6Lg7M3SCRmql8w+xSkjPynoKvheelNnsop7aYvC8hCknEhXIx0rkk7I7FG7PLb4BfHMXP/LxDn/x2vplPu183660X/CwY/JgEC+dB8gZm/u55Jr6QQfLU3ukLZscRUbCpaYwpxJZAy45qI5qdhxUZFaIzZjeIf+QW3HIdTXJE8e9dvqlpJeWbQRY3sRjJx3rFXwxqDd4QP941UWS4N7GGrNz047Y96OTye3tW+vhO+7yQj/gR/wAKePCN7zmeEfif8Ku6D2cuxzoOCOfxxXapHeS6gEjT90Y93+rOOnXd657VnDwdc97mIf8AATXaQqYoYkOSQgGQfQVLSkxODtqctNZX+/aUcL/u1k63G8ZsjJGyt5vRgf7y16IDnoD+dYnijTo7yKCR2dTCGYY/A8/lXROtzQ5bGNPDcs+ZM5bXBtWM4z1rCMnPArt9R0mK4Cb3cD2NZw8M2XBM7rnnDOB/SsVZLU1lFnMk5GaQOQMV07+HbQFSshK5+bLnp+VSQeHLSZ9iIzHr94itVTbV+hndXtc5Te3J4o8xuhPFdp/whyH/AJYqPrKacvg+IDHkRn/toaXKu5XKzhpH/wBJ08k9Lgj843/wr1qJ5mUFVTGBjJPpXB6tptvZ3mnKkaqReoGAPsw/rXXwvqbiIxR2xhKjBdjnGPQVk1qapaFh2vWkyPKA7ferznx25tfij4VuCg3uIkY9uJOf516DJ/awICG0VQMn5Wrm/E9iLnUtNu9QitpZ4m/0aVVYbDkHofwqWrLccY3Z1rttkcY70+FVPzvgDPp1rOa11ckEXtsD3/dZ/rUqRaoEKSahCQRgBYBkE9+tJza6FcvmeRePownj2/cRsAwU5I65QVyPiFG+1WjRg7fs6nk+hYda9g8SaUpvXkuo457jaMymIAnjiuMNqktpAWijcbSp3jn7x9frXJze8zZU9EcvcytBYyzR43quV9M1xz/aTK0s8iZJyzFQT+deo+IbIDw7OFjRDsVtyjkDIJrzi5tWd0Qqdi89OproopWM6qszJcxyuzJCcdc5qa1UGEgryO1TXBjiwqphm4A6Z/wqraSDdz0z2rUyLqRb2y3CjtzVk8n2xn0qBZDkcgevPNSjoF6L2Hb8PypiLuj2sNxqUrXDhUVQVIbBzk13trqFlGPLT94vGc9CK8tupjFcRYlkQMD8yDJx+P0/nUi3khI/0u7bGGPIXHPt/iK6qOL9jGyQnDm3PX4NTjMTCNQXUEKg6nngVy+p2EsrNe6vfWtop+6smTgemPXmsWC6W7ivoooYYpo7RpoJI5JixZWHTc7KeCewrl5o7ueYtcCWTd8odjnnGRzTeY+00aLdLlWho6zaaYyzTWlzNJKhHLRhFI+mSareGNX/ALE8R2GoYO2GZWY8/c6Nx9CazpbO6t0O9SB3AOarKcGuWclN3sJaH18rK4Do2VYZUg9RVSGF/PuOyCXOTwOQD/Wue+G2sHWPBFkztultc2shz/dxj/x0rXRSaYby5LvO6wjHyKe9eTj4J09T0MHK09B0up2VpgM5lcfwxj+tY+t6xq/9ly3FnCbSKPktjJIzjqfr2rpLXTrK0/1cK7v7xGTS3/lT2skMiKyOuCpGQa8iPJF6I9B3keL3LXF+zNPNJNK3OWJNFlps1vcJcD5fLOfzrpJrWO3nZVQKAewqvKn+jyDodh/lXpRtbQwa1OB8Z6d/Z9zAgJYPDkZ+priJPvnHT65r07x4gmsdKuMZLIVJ/L/GvM5Rhq78O7wPPxCtMveH5vI8Q6fJ0xcID9CcGvojSn8sn36180wyGKeOQHBRgw/A19HWUoZUZc4IzWOMV0jowL3OhFz8v6U12kKb9rFfWsyR52gc220ygZUNyD7VXmudQinYoyvF/FI+1SBkFtvPI29Pr7V5nIenc0XmxUXn4OR65rnPtmpuhG5VKsQ7lkwg3dT6HGPzPFVmvtTI3JNDt2gnc6YzxnHPsfzrRUyeY6aaTDt0xk1la2n2rRb63Iz5lu6j6lTRaTTvEWnbcSx2NxyvGPu8d6ez8EHGD1NVFWZMndWPnm4H7zP94A1b0if7JrdhcH7qTIx+gbmor+EwXDRHrGzIfwY1XycI3px/n869rdHhbSPeXfBYcV6r4cn+0eHrGTOf3QX/AL54/pXjVtd/abKCftJGr/mM16l4FufP8OKmf9VM6f8As3/s1ebT0lY9Ko7xTOlJpvelPWkzXQjnZHcDdGfaqRq+eRiqLDa5FbQfQxmhP5Uif6tc9cYo6UL0I9D/APXrVGTHA0ZzSdKgvpZILG4lhCmVImZA3QkAkZqkhFnrRXlumfEbVr7VLS1kjs445pkjZhG2QGYAkfN70mo/ETWLTULiCFLZokkIRngZSy54JGeK29hK9jL2sbXPUwaXNYPhPWptd0GO8nEYm3sjhBgZB4/QitzNQ1Z2LTurof2p9nyij+7x+VRZp9nxJIvo2fzFSM0OPU/nRRmimI+UJjm+suuDcpx+NegWmjX97YXF7BbFra2GZZCygLxnueT7D2rz/ltTsB/08L1/Guy+0S+V5XmOIidxQN8pPrj1op7EzHiQA85/A095IsfKv4mq+fc05QGxlsc+laEIXcOcn6CgHJP6U94oRGWWYFsjj1qLHTvmi4NWNSLRL640eTVYrffaRMVkdWBK/Vc5A561mMxPXp6GpI3IjKeYyo33gDwceo71ATyeKEAue9NB5zQTnPv04oHT1pgdZYt59rHN1yozx371alhcWsnBJ2nisvwzKHMtu2M/fXPf1rrDas1u577See9eZXlyNo9WhHnimeJeIVYeP49/JEsPHp92vpKP7or538YRhPiMvXmSAnP/AAGvohOVFOLvFMxmrTY40xhUlMarRmyFhUZqVqjI5rQzYRLunQe9XfIB6VBaKrXSBsnr/KtURr6c1nKfKzensUhBTvJxzmrrQhWxik8oE9KXtDTQp+UOehqTb0HPAqfyj2FYOpm8TVYYYLZ5InbDuMgL0/AU41XeyVzKqla5sgDPJA/KqWsgPZtyCQp6H2rFvbHUCD5AkJPuf8aksLa7jjuFuYXUeX8rMOvWu6VP91z3OKnWbqqHKXblcxocdaa1jPcQ27QIx5wSCBjnqc9QPSrbrvgizzwOv0q9ZACxT6kVyq09GddSNkZF3o90TmMkjB6N/SjTbK7tbkmaIiMrjeT3zXQADGcE0k4/cDHGGB/WuxVmochxqhHn5jPDHJy2fx/+vSK37374wPeomjfzWCxZHbmlWMh+YgPfrWVzc5HxI27UV4HyXsLD8XA/rXWWRYWcLBR9wDmuW8QJv1KdBnIkgf8A8iJ/9euoshutIlOflLA/gSKwhUvNo2nC0EyxIxK/NtXt1rnvFYX7PYSDJ2zdR2ropgNqjcQevFYficbdGR8E7JlOTRX/AIbFR+NGxIOVbPUUqkdySfpTUkzDEwP3kBxjPapFkY44P5VotiHuYviK3zcqR0ZBnNcPp9p5sJjyqbZpFJ7DmvTNShEwjYoeBiuQ0/Sil5ewSK/E7MpA7EA15Fary1ZI9CkrwiZhsXntZIyispjOVbADDHQ5rxNSyZJ3bV+6oGa+j30tTkFXPHpXz/f2D2WtXenqjlop3iC7SSTuwMCunB1lNtGWKhazObvomb94eGHPHaqcGNwGB9a7O+8E+JHtjIdCv8Bd3MDA/ljNcWqFXZX3KwOCCMEV23TOOzW5qxEBRx15xnPWpTIEQMx46N/KqsUhZMA7s4461YTDRnJPvxj/AD/+uqJESGG9lVnV5ABt4OM/5zWtHpGnZVjFgdcE9qo2CIDwMZydox0/wrbWPzUIDFTjgD1/Cs2zRI05L27VPs73d28WMeW1y7R/98k4/SqEzyOCiqGc8HIFWTEr5JQknnBf8f8AD8quW1qsX7xlCnHGM8AVklqaX0Ob1TTv3aQtIzyyNuYkfdA5x+ormtU082b5H3T7Y/z3rtbu7/0iQQJ51wwx7IP6VjXmiXB3yzTZZhllA4FapmbR0XwY1k2uu3eku37u8i3xg/8APROf1Ut+Qr3BX2HH418t+Fr/APsfxhpt2WwsN0oc/wCyThv0Jr6glVtw2gnHWuTGxvTZ04N2qImM1U7uf5SKVWy2zv6DrVPUQ0eMgjI7ivFjG7PY0RgX6/6QxHfms5mXJUkYIqj46hnuNPgNusjsJASEBPGDXBnT79jzaXJOCM+Wa9SlTTjds4K1RxlZI6bxUom8J2Ug58qYrx6c/wCFeYXAIc565Ir0mSCX/hX88MiMjwT52sCD1H+Nec3a4lbjv6V04fS6OTEatMrjrXvugTedomnybslrePJ/4CK8NsdLvNRWY2kBl8ld8mCBgevNev8Agu5EnhPTySSVUqc9sMR/IUsUrxLwjtJnXwyqDgttyOGx0rNudLs5Mp5rupLEqAAo3EEjp6DH0qQyncRx3qFpCRjPFefY9LmEt7aO0LFXZtwCjdjgZJ/mxqt9ggRlKvICq7RyO6qvp6KKlLjpn8agku4I875U+XGefWqsyXJE8ZEEEcSZKooUZ9PwprSE9qzbzWYbYbQJJHVtpVR3xmq9rq5uLwwNGUGTtOc5I60+Vkua2PNPFMAh1++UdPPLfg3zf1rHz+6x6H/P8q6rx3B5etPJj/WxI+fplf6CuUXO1h+NepTd4I8iqrTZ6n4auDN4csmzkhNp/Akf0r1X4cXG631C3J5V0cfiCP6CvHPBMnm+Hyuf9XMy/wAj/WvUfh7L5et3EX/PSAn8iP8AE1xSVqjO6LvTR6UaaaU+9HatTFjaqXC4fPrVuobhcpn0q4PUiSuipnpR0c++DRQ3DDryK6EYMXNIQGUg8gjBpKUc1SEeH6ZAbHVtUiiRXvbaOQWwZc/OrgEgHvsDEVZuUbxPZ3GpeTHb3UcuNz3B2Sg5ZgPMY4K9cDjB6dKv67LY6d40vUu7ZEXeJY5o1Yybmw394DuenoKiu/EOk6jsN4biYouFDxbsHPPVu/HavQTbszjaWx0Pwuud2m31r18uVZAP94Y/9lrvhXmngTU7d/EtxbW0SR28sGUxGFYkY64+pr0oGuWt8ZvT+EdT7ckXZHZk/kf/AK9R0qHFxC2eMkH8ayNDSz7Gil/H9KKYj5Qj/wCQnYAHrMDXX4O3jnPeub0a3S88UaVbv915+cH2NdpqWnS6bMELCSNslWPBqYTSfKOcG1zGfgjtSgE8YpRkf560fN7elbGNhxjdSoI5YZGDR5Tlgvc+lJluueKUs3Bz+dIAMQVjk5A9BVmKxVkkkFxD+7UPsYkM2ew46iq+5sYzS5OM5oGhmwk9COfSlWJifunrS5b19qkQtnqaTY0jQ0SKeLU7eVYmYBwGx3B4NeuJpDGLleo7CvM/D8LPdKWnKYIJx3r3CPBjUj0rgr0vbTtc7IVJUoadT5m+I1i1p8TFwpUFrc5656f5/CvdEHy4ryj4yOg+IGngfe8qHP8A321esIPlojHljy9gb5tWLTWp/amN1rREsiYUw1I1MNWZMs6dGXvE46ZNby2yKdwHzduayNIbF5j1U1vUuRN3Y+Zoh8nJzmjyRU1FHsohzMhMHOc1RuoykwHrWpWbqILSBQccDkVUIKLuhOTZEFb1/Wm3CqYCHYDPGTTlA9c/Wqertt09mPQMDx9adTWLKh8SJJIljiVGI+UDP5U6C4hS3AM8agHuwrLvb8Cx831C9/auV+3tLKUwQpz3rCine5pVlpY9GiuYJn2xzRuw5IVgaq3Ws6bGrxte24kRgGUyAEEHmuF0HWzb6lcFo1CqhUtnryDXBeJL4S6/qKYOftEqnpz8xq3Vam4kKHu3PbTrmmhd39o2gB6ZkAqA+KNEHB1ezJ9pAf5V85Txbn4qxZFo2ChRhumf6UOs0Cgj1ObU4dS8bX3kXImtPsiuhX7u5WUn+VdZbanbQ3Etq0+JBcyrgqT1YkdsV5H4elMGpXGOP9EmHrn5Sa9a0aXeuoSg8tds35qtc9Jt1bo3ml7PUJvFGlxuwM87YHGy2c/0rD8V+I7Ofw/JDFFfGUujAtbOg6jqSMV1+RnOTu+tZfiyETeEb4jJYKGyeeQwNdkk3FpnPF2d0RWmryrZQLJpGpl0iVW/cgDIHuatx6rM4BXSL3Ge5jGP/HqvqS8aHsVB4p23K425HuaST7g2jS00edZCSWBo2JOUcgkfkSKjt4bdtUu/kXcdrdPbFRpdeSmzpxnHpWRZ6gY9emLNw8S8fQmvNxFeCqKLW25vTpycW0dSbWHn5BXDyeGNOtfH93rIt1NxKkewnoh2kMwHqcAf/rNdQdUFY2t6gIgl2gxIBs3ZGB3Gf1/OmsTRckoKxVOlNP3iLULqS70+7Wx+e6j3KqqQCG7D5sD09q4Pxp8L59e8NyapHo4i8QxxB3McijzyPvBgCQTjOD1zjmu4huVu4EW3eQzE71www6nqoJ6HPIroYLyQ2kYljPmbRkMw611xcVqFVPY+KEJWTDDBBwQw5H4VajkLYjA5J6AYq942uUuPHWtzJsCveynCAgZ3e/fOfxzWdZKZZQFGQBn/AD+VdF9LnJbWxt2cSqo3sNvoD37/AOfatVHjSMucBcBcf3qzbdCccnGcMDjAFTCUzXohUEpGNxHvUFmpHuAVhwVCjHTFSXcwEJ82byov+WsuecDsPxqipmVT5Vu8pJIVVXvz37f55rNm33N5sucXlwDlLOEkonu5H8hSSG2XIppbtCdOQWVmDhrqUZdz7f4+9V9R+0W9jPHZQg5B3TSMHZx32/lWiINSjCrNLcnAwIbWFQq+2WwKeyEhpJrZIoTjcFYyH8QoIFD0Banmu4792fmznNfYPg29ivrCzup1VhcWccvIz8xVSf5mvkjU4Uh1GZYo2ji3ZRW9K+kPA10y+BdFmBPy2qrn6ZH9Kiv/AA20FJXlynpUdnpUNy1wsKCQ+pyB9B0FYHjaSA2VsYlQHzCCQO2KpnVG/vVk61eG4tVBP3XzXjxqyl7tkenHD8r5mzFlbcgJ9KqMMHkZqbdujxzwxqJhkgnp7V0RKkY+pR79K1qHH8Ak/TP9K8jvAQx68gGvXr+5t7aW7iuZNn2m0ZVGCdxGRjj615FdDP1xXXhzixPQ3fAcUtzrclpFII2mixk9DyODXXeAXP8AYk9uesF06fyNeaaXfnTb3z9m8bSpXOOvvXe/Dq7886qACoMqyhc5xuz/AICtK8fdbIw8vfSO5nl8qF5cZ2gnA+ma546zOysTEI2Az+o/oa6CdN9swH8SkUvh3Q9O1Sw8263CQoy/KwHIUY7e1cKR11qnIrnFySXdzA0k1xJsUhsDjgtgj8KhuUMb3ADEtvdGJPUABhXeXnhnT1FxDDLP5TqSBuHB6+lU5vDNlJKxMtxl2RvvL3XB7VsotmKqJq5xovN0xuW5w8bn8sGrtvIp1hMAgmQuM8cMo4rf/wCEQsIoVdZrk713feXjBB/u+1W38O2U85uI3uv4W6jscelDgxqaPPfiDHme0kAxujdSfoQf6muEXqfpXuni3wpYXtrbmSW4yjkcMucEfT2FcSngnS/N2me7yOT8y/8AxNdVHSCTOasnKbcSDwBNmC/hz0ZGA/MH+Qr1DwbP5Pim0ycBwyH8VOP5CuV8J+FbCy1CYRzXJ8yPBDMvYj2rt9K0q2t9Rs7pHnUpKr7WIGOeh4rCtB890b0pr2dj0ukzVgwL6mq7DDEds1VrE3uJ3pjjcpFOoNNCZnkYNMY8r9ammXDn3qCXhM+nNdCOeQ7NGeabmlzVEmdf+HtJ1O5+0XtjFNMAF3tnOKgHhPw+p/5BNr+KZrYpc1XM+4rIoWmh6VYTLPaafbwyqCA6IAQO9aOaaKM0O7Cw/PNDHam7+6Q35GkFOVd6smeoIoA1Ovaiq0VynlJkjO0ZooEfN3hNPM8caSOcb5CPwRq7/wAVjBg6jJPSuA8I3ENr4306adgqDzcn/gBFd/4tnSSS1EZ3oys3mD7tcyT9qjpuvZM5oZ6ClxzzzSd/agkggAcdzmu25xsUc+uK0NL0XUNXkZLG3MrJywBAx+ZrPXOR9K6nwd4gg0O9ka5jdo5EILKRkHtwetZ1JNK6IldbFO+8Jaxptobm6tDHCv3mLL/LNYpUivQPFXjSx1fS2tbaOUMSDliMVwDMcDGCM81MJt7ji27iDPtUkfGKh6il3FZkXI2kHPrVs0RtWlyYFyDyRXtlheq9hAxPWNT+lfPa3iB8Z4xnOa9csbmcWds4lXyRbgGPbyWwMHOfrxXn4qThaUTsow9orHlHxcufM+KEAB4EduP1Jr2tPuivnz4lTGX4kI3B5gGSfpX0HH90VUHzQTZMlyyaHUw0+mmtESyJqjNSt9aiNaIzZa05tt9F9cfpXR765e0bF3F/vCug3D1rOc+VjSuT76N49a5rWo9cluU/s24ijix827r/ACrNWw8Vsfn1WEewH/1qweId9ivZnb7/AHqKVlyCQCScZxXJxaV4i8wtJrClP7oz/hUMui60u1pNcdhnGBnqal4iVthqB2JZMfdH5Vx2pXDPPcRs7bQ5wueOtS2Wk6hb3iTzatJKi9UwcH8zWVqJd7q68uNydx7GnTquV7lKFitqOqQw2CxS5XCrk9hWBLP5beYozjnPrVnVLA3tqiNA0iuoRwQee1ZN1p99Gfs0PlKgACZfaqADGPXrWlOsti5UebUrLq6y3lykJHyxt5nQ8gZ59K57xBLGPE18nmJ5jTyMFzzjcc1fOmXa/aJI7ZVfaTK6TMMnHPb09aZqmhX73CubIPKHDs6yuxcdxkr34/Koc05XNFStGxgz3CQxhmcKMgfUmrdtZTsrT5IjHPJz09MVoRaDqFwwii0RncHcW3noPqOK3bHw1c2EUN9L+7aMEtBuPTIHzevXNKUr6LchwSV2ylo9pIWS6XmGWB9pIw2GQ9q9Q8LljpkzkZ3uj8e6LXBW7zS3KkkbCTnj2ruvBj/8So+nlwkZ/wBwf4VWHvzakT+E3uABkfrVLxA6jwrqe0jCwMx3elXow0jAbWqPU7b7ToN/E6kB4XGR24rvexzLcjs2M2n2kiZIeBGyD7VN5Mh9aqaIyDRrLMoJWBQcnrgYrSE0X/PRP++hUKS7lNMztXnnt4ENrCs1xsGImfbkZAJz9DmuflvAmpqe4XBNdBqnkSiKdHSR49yBlIOM4z/IVyOonFwJB1weSe/FeHiknXZ6VH+EjRuda+z2zTFXbaeQnJrMm8RSTxvGtleHj72zj86qPO3UNgAjgDiopZ2KDkAE9BWMaLTujSUn0NfQxp6awrTXRtnc7sN8i5HPVux9sV3iXEFwqPDPHIhyN0Z3D8xxXkgOXUtxzzn/ADzXdeDsNoSd9kjqR9cH+tenSldWZzVVfU+V/EL+d4m1SVQ2HvJmAI5++etSaUo2tJjvgU7XrKYeJdTjCMQt5MvT/bNWtNj8iIBsLg56dv15ruvocXUvGQW8JckZ6jPpUFrL5VuZHUbnJJ4xiqt1M0zqinjg0XbrHZZ5GFxwvH5j/PNSUPtYLrUIjMZHSI/LtiBy4ye/T2rWs7LUEBgts2UTHnbyx7dBwPxJPqawNAhe4IHnTqozxFIQf0/z1rqINOt3O1p33L1LXJzn8/6VdiTQttIgQK9w0kzDndK5YD8M4FSyfZkct5CvIeV2A4HH+fypsOlhRmO9ugCD92Yn/ParDWskShUuZCxb7zAEgc8cChrQcXqcN4xs7tJLe6m5jYFB7Hr/AJ+le0+C0MXw60odP9F3fqTWP4bsLG+mvdMvoBdw3MQLCbn7p7HqOp5HpXbraQ2mmC0toxFBFF5caD+EAcVlLWNi46TTMSX97JFIZHBjYkBWIDcEYI7j2qO4cshHam7ifb61HIQUPPbNeWlqeu3oUrWFYVlRNx3SFzlieT169PpRPDHMFEqghWDjJ6EdDT1PzNjuOlMkOQMY/GtUZs5bxijfaLB1BJO9eBmvO72Mq75BDbiCDXsUmV1CwfOFEhU8+orzLxZCYNe1CMdBOW4755/rXRRlrY5K8dLnMV2/w1l26tdxf3oN2Pow/wAa4k9TXT+AZvK8UxJ/z1jdOntn+ldNVXgzmpO00eu/wDnINVdJaSy064DDLC42RLkfOTjH06/pVoIxXAB/AVWtrUC2vCrBbiK6E8audu4BVJx+tefF9j1VGEpxU9rl0NdwyiR54pxty8SKV4xzgnr+OM+1JdX8cbWqKoErL8rynCqB3NQf2hHIqJFE5lKqCHG0L1ByemPpmmag8Un2e4ZY5PLQxyIcE9jkA1rdW91lQherFV4pb7WV9NNNt+vXuSxajPHJFE8sEyOCFKKAVPPbJ44p095dz3ptICFUqjbz0XOc8d+lQWt1Y/bVEC20agndJLtTAz2zg5pkNzbx6jI7Xdrt2JgmdMZBPHWknfqVUUIybcdVHrbe66Ly6fh3WczYmiuiHlCLKjAY4yAQR+NYZVxeP844x29zW5qeoaa+oPLJf2qRi1ddxnTG7IwOtc1Jq2nC6Lfb7VgyjlZlP9a1jsr9GZc2suRK8obWW9+i6aK5r2Nw0WpxpHJHGzZG+Too/rXRW17Il0lvLPDOZASkkXGCOcEZP51xMeraUl/DK95ZyISoK+arHr6Z5rr7W905rgNbx28KKOZXCoT7Adfx/nU1HLn1ZdKNL6v7sbq2r039d1bt1Xqd5DrGqatqd7p9h5ECW3ls1zIpbAdAwAXjJznuMD1zWtGsqRhZ5FklH3nVdoJ+nasfwgd4v7r5fKmePy3BBDhY1UkHvyCK23PzsevNEb2uznxMo3UIJJJL77K+vr0G5o4oozVo5ivcLwCKquNyketaEqgxmqB44raD0MZojU7kU+op2ajTIXHoSP1p2a0RmO7ZpSabnijpTAWlBpuaXtTQD81JEeagzUsX3qYiKSCTzGxjGTiitDj3opAfL2kEt4hts54WTv7V1czk4BPTpmuR0bP9vQMOBsk/lXUyEk896ICkya3IL7WHarYSHuCao2+TJgEk+1XAj46H2ok9Rw2HhYegUZpcw9Qg61H5Z/yKBExJyKm5Vh4aMc7eBSechU4UD2xV/SX0+3uJG1Oye6hKEKqSFCG7HispIQzO0ayAFyQGbIC9h0+tSpa2BoV5woJwB+lYWuXZTy8SFA3zBlbkkdq2LiLELbgCMdDzXOakgktzHcE/umOw9wMcVdxWKqahJMqj5yowSw67a9z0+63aRB1H7hPw4FfO+nSOb6FNyqqjvnDY5Fe62EjtpkB3YzEu4Y9hXDjNYo7cGtWeX+PJN/j1TngvD/IV9Hxn5R9K+bPG5H/Cag5B+eH6dBX0ipwox6VUNIIzn8bJSeKYTmkZzioyxA4NWmZtDmqI5pWkIHNMzmtEZskgOLiM/wC0K2d9YcZxKn+8Knvb2+hnCW1iZlIzv3ADNYVk20bUrWNJ35pFbnrWDNqGt5+TSR07yD/GmxX+vNIobTFRe5Jxj9a5/Zyua8ysdEWIFV5g7xbgCQrDOPrVa3nv2+W4tY1U/wAYfp+HeluWPk4B/iXv7iolo7FxV9iUNkVLbRBgM45rLu7ie1t2miha5YkARLxt9T61jR+IvEMMsnk+HpZkz8mZQoH51lUTtcqW1kdNHZRtE4wOJW/9CNedJjUxcy3G0MJWiIXgEAgg9625de8TpE7WuiozOxLrJMP3Z7jqM1n+G9Nv44Ll9WtFhma4LoiOCCCBnoT3qsLCLcl1exzV3Uik1sUbiUWVvJGtqTFIMOWJOOMdhVmw09dfv4Zb2HZDt2jYzAlQDz1rqUsra6kjinhUoTjoeP0rdttMtIJGeKBVZVCkc9uf617MY04Q0Xvdzzm6s5ay0MXTNFigjlWwhWISRABnye+Tk556VDrsMRntEe2V97nO4dcha6uIqqyAYAWUjAHbiqs4XdYvgZJ67cntWdGKg+Z6suabVkzyqVEg1Hyo1VU3A7QOxra8LrG9nH5kzqsdpGcJIVzglecfSsHxPBOnjCQxXAjVZF+XaOAQMgGt7wko+yyEkZ+zsucekrCs5L39Tqh8Gh0cQsi3MsmO/wC9f/GkewsLiF4o2lIIbCmV8nj61WKttDmc4P8AdFXbDYxx57E8gqR14q013RFp9jO8P2Vs+lQymGEyMuHLNgnBNaxt4o48iK3bA6ZGf5VleFPs2meHnEj+VDDJIWaQ9Pmz1/Gp/E+ri10dI7Jw91fkRW+w5zu6sPz/AFFLlpqN2jaKnOVkTzW8QtdyQpGQ4+6Mcc+wrktZXayHjGCMfnXUixfSvDUFrJM0skUa75HbOWzzj2549q5PWWzEG3Z+bgY4/wA815mLjasmlY7KL/dtXuZ74jRjuHr16Cs19TtQOJSefQ81f6q+QMccCqbQxL0ij/ACnDzHK/Q2fBmo6fNr5tbryXiuImAWZRtLDBHX8a9EuJdO0i0YiS2s7cHJwVjX8+K8YvQn2OXaoAC5HHvXI3ajzo8IOntXbSs42OSpfmuQ6w0dxreoTo28PcyOGzncC5Oc/jWPd5jk2eo7++f8K13jw24Dmsi/OL84OTtH8v8AP5VukYsqtzKMduw4rc8N2UOq+KtOs5oPMjLl5I3GVZVUsQfqRWIxO7rkdBzXZ/C61M/iO7vCPkt7baOOjO3H6K1AjD1bw+PDHiSWzZS1pJ+8gdxkOhPT3I6H6Z6Gt20gtZCBshLcDbgcfh/ntXdeLtAOv6KYoVzewHzLbsSe65/2hx6ZA9K880u7geCMXOEcYUOwxn0zx1/wqosRtxWtuhGyBcqO4q04IXCnjH3T69v5UyMEDcGyPUc9qkd9qgswx796sQ/SLr7JrVtIOcuEI9m4/wA/SvRyOMY+teXw7o7iF8FdsgJY8AYP/wCqvTuDg561ianKt8hYZ6ZFMY5BP61NcjF1MBx85x+dV8/l2rzmrM9VO6uQZw3cjHSmHBUenTFOP3sYHfrTTnviqRDKOoRg2qk9FkU5rz3xlCY9auBnOUQjP0HP6GvSL4ZsZeegziuG8dxf6fFJt/1luB+IJrSk/fMK6vE4RgQx7Vf0NpF1uz8p9jtKqhs4xnj+tUXGG5qWymNvfW84/wCWcisPwOa7nqjhjoz1v+y9Tk+9dpj/AHzUEvh/VpgTG+9QMg7gNx9OTXTo2QRheBwR3+tP1G7ksNIe5iVWaMLw3TGQD+ma8DF4ipR5VTSbk7amGdY+tg40o4eKcpysr7fhY42PwtrEkgWVSi4wW8xTj9e/9KjXwrq5YboH+9gnzE6c89fpWqvijUgiyPFaCN3ZFIDZyAcd/WmnxTqYiZ/KtOIRKBsb1xj71LmzD+WP4/5nD7XPv+fdP8f8zJPhbWvK4tW3ccGZPTnv605/DF1CVE+Yyd38QPHbp+tbE/im+huFTyrYoQjZ2t0Jwf4q6DVQDaqD/fH8jUfWcVTqwhWirS7E08xzKji6VDGQjao2tL30+b7nAN4XWeEJNKQSTkL+leeXKFHAbqPl/LivayAeh/SvI/EEAt9Xu4/Sdz+B5H869rDybuj6DEwSSaM3ONhz0/xr1m2lE9rDN2kjVvzANeS5/dj2NeneH5DNoNmxwf3e38iR/SrxC0TIwz1aPZfh/P5vhry+P3M7p+eG/qa6g81wnw2nzDqEGejI/wCYIP8AIV3VZR2NZ7hSZooNWQBORiqUg2sRV2q1wMHNaQZEykOJHHvmnUjcTfVf5f8A66dWtzGwUd6KKYgpe9JR3pgO7VLD96oalhPzU7gW8+9FL+FFAj5b0Yg65DjsjfyrppDn865GzmMWpI43ZweldL9pLMmSMEYIPrSgxTRes3VJCxPAHSr32yIZAf8ASspXyDg8Y61ANQtHufs4uIzKDjbnv6VbimTFtG39rj7Mfpij7ZEBwW/Ks0nnOajaBCGPzZ9mNTyofOyx/bMW5gA5wSTjginwahCqksSN3IGfwrnZEaGZj8yPvB55AGf/AK9RSPtmJZiAEJ2j8z/n3qeVF8zNu51m2kcQNFJmQYDZHX/Irl765ma+eJV2EKVOe/pUck4a9EpI8sZON3I/wqbUVhusPDIPNC5fJOce1TYdzPRWikSWMbgi5yB36V7jpsiHSbYc7jAn54FeO+FHZfElpGxby3cowPQggjGPSvW9PXZYW6josagfTFcOLlsj0MHHdnmvjY48ZBuTiSLH5D/GvpKP/Vr9K+bfGpb/AIS9gBkAxH9BX0jEf3a/QVpH4EYT+NjzTDTz0phqkQyNqQEYpWqEnmtYmTHOcLnuK2ElVERmOQR+fFYU7HymAznBxj6VPBKjRRRXEjRyJ0BON2f/ANdRVsa0jUkvgCoRSMkDGMcGo2vjtJQ89RnvzUH2aBiD575AAGG9KQmCDMKLLKW/hHOKytFmmqLJlzO6gDG0E/WoLj/UHPqD+tWVREBwMZ5P1qC6ZBbSfMOnrXJUfM9DeGgxSdhz60REsCB6muL8Xanf2F80dtdyRoUB2qcYrE8P65qL6wsE1/O6mJiQWOMgZzSlhZVIp3CVVR0O6aUh7oZ6Sn+QqbdlWrhNIvbqe4tmkuGbdI4cySEZ/wAa6jUdVXShHugkuPOO0CHkg/8A16ihRdLExTYq01Og7I37G1NxIgYYBJOSPT0reRNplHuP5CuV0HX2uL2OzlsJ7Z1VnUSnkjFblxfyRXCNHAWRl+Y54ByMf1/KvXdWLdrnmcjSvYtxt+8kj/iL7uB9KrNljbKCQdrYyOhzWdJq0guZop1iiuBP5CMXJUlk3J0H0FV73Vb6LRdOkS3i89pwjsMlV5I+vUUKpFu1w5Xa5yPimBh4ikbndhSeKt+Dm/1qFc4Wccj0lz/Wq+vzyT6lOZivmqxX5RjODVXRdWOkiWYQiT9/MhUtjrtNYynzSOmEeWJ3csalF2JHj1CD8qlslUXCnaoPrtFcqfHmxTjTU+U939PwqNfiJJvJGmwjJ4zIc0KCve5Tm7WsbejRQXOnXdtcwrNC1xKjIRkEZFV9K8Kw6brQvFnlmt4kK2tvLz5JPXB/l9axbDxBNaxBo4YsXN2xIbPy5GeKvy+KbyBj5cUPpyCf61H1qitJbotQqpPlejOm1WTzNMnACnC5OGBPUVwWpKvle24YyOPStQeI7u6geJ44ArqVbauCM1jaq5EJbndxjHbmufEVY1ZpxNaMJQg0ygCOfdRUbjiljKh26fMuRzSN+mO9NIq5VuVLWso/2T2rkLhf3sZI6L255rs5BlGXPb1rjrkESqDkcGumic1YjKZGAM56Vg6mSNSZRgkqua6PGY856egrnNQB/tSQ88AfTgV0GBRnIC9dvFeofCa326NqNwVw0twqjPoqnH6sa8zkGCMfKD2x/hXr3w+j8nw3Cp4LqWJ9TvP9CKVwsdgpwy9uRXn/AI00ZdI8Rfa4UVrbUsvsPRZR94fjnd07t7V3pbdk1neP7E3vhVp1GXs5FnAI/h6N+hz+FVElnA2u0N/o072755Rxx+XT8q0Q8qKWkUE45eLG4fgfxrNgIniXeACRnOMgmrKNOrhAiEKcc5xitBDXaOWTehlfbzuznH1Hb616hZ3Ud5ZxTxMSjL+IPoa8785VHyxp5iDP0/Gun8I3Pn212DhWWQZTGMZHX8f6Vm1Y0WqDUF26hMM9Tn9Kp5x0NX9VGL8nn5lB4qgeD/hXnT0kz1KbvBELdTnpnoDTpZpLiQvK258YJCge3amkgnPtmkwcnJHFJAyKcbreVepKH+Vcb42jEljpc+Mgoy8/Qf4V2xAZSPXiuR8UJv8AC1rJ3Sbbnv8AxCrhpJGVVe6zzJx81N9KknG2RvqajFeieb1PfNMkFxptpPgZlhV/zUVZ1NVOkyLIAVIUEH6isjwjP5/hbTnPJEQT/vkkf0ra1C2lu9MeCEoJGClS+ccEHnH0r5rMWoVKbeykebxHUjCWEqTdkppv00ODnjkaG4VCvkJcAx468/8A6xRAvmW9v/twSRHn05raTwzqccbRLPbbGCk8t94fhVlPDUyMn7yLakrOBz91h06V1fX8P/Ojt/tvLv8An8jk5oWWCzkDs37sscnrgg4r0TU8G2XPTeP5GsGHwjMmBJdK6jO0c4AIII/Wt/Uf+Pdf97+hrixGIp1sRS9m72b/AEPHxmNw+KzLCewmpWcr287GP+Oea8z8ZweXrcxHR1R/02/0r07GTx+tcD49hK31vLjiSFl/75Of617VB2mfWYhe4cUPukV6L4Pk3+HkH/PORl/r/WvOh3HtXc+BZd1hdxf3JA2PqP8A61dNde4cuHfvnrHw7l8vW7iI9JIDj6hh/ia9KryXwdN5Hiq05wH3IefVT/XFesmuam9DpqLUSiikzWpkLmophlM1JTSMiqW5LM2biSJvcj9KUmi7+WPOPusD+tJn1rZGLHUmaTNANUIXPvS5pmfegGmIkBqWH79QA1LCfm4oAuc+hopRnFFAHyhCSbxSp5x9e9bZmVVAz83c1hxuyyhgw5XH6mp0bfKN74UH7o7+1ZxdmVJaG7HLGVkkeQRqq7dxPHSuRtIjJqsUKFT+8A3DuB3/AK1om7lvNTMNsodEjwUYfL78VTv0khnN2MRy5+6Bjacdqpzu7EqNlc7npyaryySZaOMruxkZ9K5/TfEMly4iuiAw/iA+8Pp61duLyOOZZISXK9eOorTmRnykOpaqRJ5JA3Ac4+lVbe9jMCb+vzJuPoelZ125nupJsbi397qBUOW25xnHQnrWbZokW7cK8o3Hcdwx71ZvjbRSKpYI4GAQP1zVLciWyyLjfnB56VUnYvJ8zKx6Ag0rhY1fDUoXxPYAc5lA/WvZLQbII19FxxXimhOYvEFhnODOmRntkV7bEOg968/F7o9PBfCzzDxuWPiuTCkgGLPtwK+j4v8AVL9K+dfGeT4nuz2URf0/rX0VAf3Kf7o/lW0fgRzT+NjzTDTzTTxVIhkTdDULdamaoW4NaoyY1jhc1pKsc2BIisOwIzWROcQsR2FU4ddmiVVKoxAHzHvXPXjJtOJ00GrO50j28MRBjjVSeuKntnVkIB5Bwa5qTXpJf4UHvTRrtwnzIEHGOlRKMmjXSx1E8XmW8iZxuUj9K4H+zxEwkMrnaQx9Mg1r/wBv3zAgunPoorMvZWe3l5IyhyPwqqKcE0yJK+oviHQ7nXNQFzYyRyW/l7GIIPzAn396wtP8K3q6xIEb94sbocrgcjHB6HrW/aXcsEJiicpHn7q9KmgkdrhRvbJPXNQueKsi3GMtWc5H4furS5jRpV3W025/cHB4rb1ViYbU+k8Z/WqhmmubqddrbwQDjOTjirM8dxNawxLbTFw6ktt44wa5MSpNqTN6XKk0jYQqnjS1VOjQMP0Fb24SJKI1xu28D6kf0rDNpcHxTp90qZiRCrHIGDtrVtJyhQEZDlhnI4wzH+tKlK0TCqrswdUtrs3PmRwu5F7aSdugXBP4Vr3cEsmmRReWQ6XKvjPbexpup3MqiGSKJWRmXeS2MY6VK+pwsZQ7AZAAxz3P+NJ17MI0W7NHFeJrmO01W5J5DysQV+tc7JeGCFwIy4kuj3xjMamtLxSHmuyLckqGzzWNcRs9tt25YTxk+2Y8f0rqoyco3InHldileahIiP5ahTjIzz2p3h67fUnmSYgNFAZFYDGWBH6U24snlJxG+QMcNx/Km+ErGcXwJOxJImXPU4/H6VtUuqbaM4fGkzrrmFIdK050z80ysxz3NF5wSc1LcxCTQIVyw8puCPUGqV3C5JJuJT+IrxoPmep6LVkTWshLcelN1KZjCwPIx2HQ1VtITvGXlwOnzGpdQfMQzngHp3/GuunExk9DJW4c3G3zGwM459O1Pa4dsjODUJ/4+uSMY4pDxx6etdyijm5mOMzc5bH0Nc/qKbJVZR1Tp71ud+v0rH1YHEX0I46da1grMibuismBGoBNc/qAB1KXrggcg4ro1yqrnoRxXPajgag/QcDmtWZIqMp3fd5/+t9P8+9eveCFK6LCpGMRkf8AjqH+ea8lWCSadYoxmR22qB3Y8D+Yr27R7AWEstqBkRTvH6ZGGH/sorJzSkkaKN02agxjseK0xGl1YNDIuUkTa34is0D0A9K07Y5t0+mK2iYs8cigk06+u7Fwd0Em3Dd17H8qsm9hVcSAgZ6+lbvjjTIotWt75lxHcjypCOzDkGuYeDyiEuCSOit0H41YjQheORD5ZByOvfmrfhG5eDX/ACGbIlVom+o+YH9DXP8AlPaN5sTBoz1xnHX/APXWno86w+JLOccJI65x78VEy4nZ60NtxEx4ymPyP/16zG6571s62uVgb0JH8qx8ccE151bSbPToa00Rt9Oc03HrmnsOOvNGMeo+tSmaDAOn5ZrmtejB8J3yYz5U+QP+BD/GunxyOBWJqUHmaTrcPUgb+fpn+lVF6kTWh5BdjErdeeeagq1eL8wPHSqvSvSjseU9z134eT+b4VROvlTOv8m/rXXs7KSATjrXA/C2Xdp+oQf3JUf8wR/7LXfspyO/FeZiIRlJxkrneqFLEUVGrFSXZpNfiQGeYZ+bj6VGbqcfx/oKey4OOaYVIHNc/wBWofyL7kY/2TgP+fEP/AV/kBup/wDnp/46KglkklwXYsQOlPK9vTtSMMg8Y/CqjRpQd4xSfoaUsvwlGXPSpRi+6ik/wRBs+bn17VyHjy2zY2svJ2SlMgf3h/8AY12eBnNYPjS387w9O/8AFE6OPzx/ImuilK00bVY3gzyNeG+vFdZ4DkxfXcWfvRhseuD/APXrlH+WZsdmroPBcnl+JET/AJ6Iy/pn+ld9XWDPOpO00en6VMLbWLKbP3JlJ+mRXtJrw7aR069q9rt5RPaQyjpIit+YzXFSZ21US0neig1uYCUUGkpoRTvEzG49Qarhtyg4681dnGVzWdF/qlGfu8flW0djGS1JAaXOKbmkJqyR2aQH0pufWlBpiH5qaE/MKr5qWE/NQBezRTN3uaKAPlNB82R1x+nNQTSFJjtJ56VKcK/YnA7/AFqrPkScDr2rBblvYu+FSDqU565j9M96u+KrZIoIZQW3yOcgnI6VH4RjIvLo/wAOwDP41P4yf91Zjp8zE/pWTb9tY1SXsrnLRErIpGeozWoZtpAOQOo7ZrJRix2g5HSrIDvjBwK6DAmkbcd3c9xxUZ9uR2pQUWMgsdwHBIqIkgkn0xSHYfyIGwTgt0pgVQvyvkjHapVcCzmB67lwBV/w3ZQX97KlwhZBHuABI5yKG7IErsj02M/25Y4wcTocg+4r3FB8+PevP4NIsraQSQQbWUhgdxPP51tpqd4X/wBceT1wK468HUasd+HmqaaZyvjg7fE1yM8FYuPXpX0Xbn9xH/uivmTxZNJLrweR95JQH34r6Ztjm3iP+wP5VqlaKRzyd5tkppp4FOphpolkbmoHNTP0qu55rVGbIZz+6ce1c2G4ArfvJRHCTjPIFc734qJmtIlJI5pysSvWmqpIB7GrbWojtY5/NibecbFbLr9RWdzdEAOB1z+NJPloJf8AdP8AKpdgyaSQAwyAcfKaVxjLY9fTg1dtx/pKEcYOao2pG3rngVZViuGB5HepbGiRWEWrXTdBhT+prQe7RIVkVgVCcgfSsKck3RY8koMmpgf+Ja3rsNcWK+G5rSjqbkuqRRmAOjM2SODjqpFYmk6nctazxmQ4SdwvrinXZz9mI/vrn9aoaScfbVOeLhv6VyKT5GaOKTNZpWf77Ej3NKjfI3HcVApyDjFSIf3b/h/OsJGsTC1cj7UxPqMflWXNuBl3DB3wsf8Ax4cVf1rP2rjptFZVz/x5yN14iOQf9s/417OH/hI8+svfZO63BncW7qpOM5I96r+Hi8d/CjnlS6/hk0fOJSdpI2jPFR6JJnVFIGP3rD8a2m06bRlGL50zq350eYdg7/1qldf6sHjp1q2Du027XPSQ8fhWfO+YEPByorx4LU9GWxFAxDrzTr/m33HHvUEZ/e8c1NqLYsumSPSu6ktTmlsZJ/4+0AB6AfpSE8jvQpH22A5+8vf8RQRg12o5xuPzNZ2qLuto8cnzMfzrRI6f4VVvIw8Q4GBJyPqP/wBVXHclmbKwSaNQeg6VzeqZF8w9gc+3Stu6cnUSOCF4rH1hP9M99oOBnPOe9avYzRt/D6G0m8baabsqIY98vzkABkUsufxUV6/Cq/2jdMGVl+1ggg8fNj/4uvFPCiebqUTdGjjY4PfjH/s1epaZfbjNGcAq8bjA6fdH9K46lN86nc3pyVuU3JU24IHOecmrVo2YBzyCc0y6Q+SSMDvTbJvkYdwxrqpO5jVVin4ssP7Q8P3CquZIV81Pw5/lXn0Mhmt1fILYwQ3evWCQRzjB7GvLb2xOk6rc2nQRtujbHVDyPrx+orYxKhmjQlXiK88nGcVWeKVJY3t+YQd6sD90/wCFaG5JcjK+Z2HWs6azMbF1ymeu3pSkrlJ2PR7qYXmkW1wMfPhuvqKzdpAxg9aXQ7gXPhhYurwPtbn3qRlGfb0rzMTpM9XC6wISuRgUm3FTmPrQE7dM1imbtEO056Vmyw+ZdahARxLb/wBCK19v0zVGRdmtRekkBXn2P/16q5LieHXq7dvUkZHIqiOtbGtwmG8uI+PkmZcfQkVj16sHeJ481aR6D8K5calqEH9+JX/I4/8AZq9SK5UV8+aVqs+kzPNbyzRSMu3dE+049Oh9BWm/jLU363d6R/19MP5AVzVsPKc7o66GJjCHKz2p0Oc4596jZdvJwP0rw9/E1+5y0szH/auJD/7NVaTWJ5fvojf7xZv5moWFl3LeMj2PcXnt0ALTxJ/vOBVeTU9NjGHv7Ve3Myj+teLyalEbRES2VbgH5pCFIP4bf61B/aNz2ZB9I1H9KpYTzIeM8j2Zte0dcZ1O1z3/AHoP8qydd1vSLrRry2jv43kkhYIqgnLY47V5d/aN3jAncfQ4ppvro9biX/vs1ccMk73Ili21axHOMSng+vNaXh2UReJLF+xlC/nx/WspmZzliST3Jqe1mNvd284PKOrfka6JK8bHLF2kme1MmQa9T8OTi48PWTZziPZ/3zx/SvNPLDDOcjHUeld94NkLaGYz/wAs5WGPrz/U15tJ+9Y9StH3bnQe9BozSV1HIFJ3pab05poQjjK1kj5ZZV6YbP51r/pWTcDZesP7yg/lmtYETQmaM/rTM8+9GeelamI/NGaZnrRnikBIG5qWEjOar55qSI/NTAu8eg/Kio93vRQB8tAAsOnAH51RuOH5I6ntWpasPMO6AP8AKMZ5xxWbcbWujjAOe/FYLct7EcVzJAG8tyMgZ59Kjubye6I86Vn29AWJxU/2fZA0jMG47cjFQ2kayzEODgDoKvTcnXYZDgHkc9sVZBz/ABAYBqCMKtztk+6M5H51GzHv1oAlEueAeBxT/KmcFlQkeuKhwPJJ29elWtPdlkC7uGU8delIYscL/ZLguhRhjBNa/gsMfEttBjcs58t156Z7VUkWaWJo+AQAAemMVueAYHtvGWnM5XaJVzj61nXdqbZpR+NHqM3hiKF9rxSxHH3W44qr/wAI6suoWVrAfLa4fZubJAOK9C1Ux6hfebFkrtAGRXO6yH0ybT71BlopwwX14PFfPUsTP2/IpXR7cuSVH4UpHjfxG0ebQ/F8ljK6yOnltlRwQVBr6ItD/osX+4v8q+ffinqx1rxlLeGIRDbGuz7xwBXv9mf9Dh6/cX+Ve/8AZR4tmm7lhjTT+tITTSaaBiOeKrOeanY1XfvWiMmVLtN8X4g81gqAev8AKtu9k2RjgHLAVjICp3YzWdTQ2o7EqdAo9etSqjbwnGSRj5qiXJOelSRnafvVm2bpMnuraaznMUu0sADlTkcjNQOGdWABzg1J5jFSqqc+wp3kTMpKo+cdlNRzIqzK1pEwRQRj5RWhbWkczsslwsIC5DMM59qrWtncNGhKS/dHOw09jFExDSxqRwQ0gB/WldMaTsVZyBOvP8J5/Gnr81g4P901Wuru3julY3EG1UIYiVTjp6H9KZ/bGkLYyltUtlk2sRGWySe1c+Ii3HQ0pySerLl02be2bnqn86pabk3F8v8A02z9OBV+6wbC0eBhIjojBsY4NZsc00N1deVsJLgnP0rlhTvFo1k9TTAYdjipY4meOU5xhc4/EVz+p3up2DAOyjcNwChTVO28TXFs0r3TGSMoV2qVBzx71X1STV0Z+3inZl/VYibpCe4FZl9+4t58Y4jU5+jj/GoLnxTDNOGe2k24xjeAc/lStdpqFm8iRtGrQyrjcG5BU13UqbjBJnPOcZSdgvtVNteCPk+aoT86r6P8t4zEYb7TwB2HGf51dAhBQYzlc8ke31qvYskd7OSAp80EEuB2HtW3KuVmfM7nRo4FveqSB84Izx1FUGBktkCkE7RxkdatPc6eiTI01uFZsjfKO4x7VWGsabBbxobq1TjAIk5/nXmqlK+iO1zjbcrRxSCbkY9TVy8gMloVJHIqm+u6exz9uRl65BJFPn8QWzxLBHcE5wANpGa6YQknsYylG25nsm237FgMAgdOTSMjtwEc/QGo7i9eGAlZHUNnoT/nvVOXUXCbv3zjHAGST9BmupXOe6NDyJSP9RJx/sGo3hkEblkOFwfpUU9nrsMSStoV2UkfYvzR/ewTz83HA6nHb1GY47XW97M+kCKFf9Y73kOVHrtDZPXoKtRkS5ROckbN+4yOvT8ao6mN12CTj92uf1qyj/6TcNnjOMe1Vr8/6WCBxsUdOnFaS2M1uXPCnGsCPcE+R8lv6/lXdWcqx30qiVctEMYzg43H+n6VwWivs1hGVsZVsflXT2UxGswAnPmIy8+pB/xqLXRXNZno9xqOV8kIMYAz36ClsHBLkdQRWTvDCNvVFOc+wq9p0gzIMY4/CtYxS2M5TctzWDd647xvbBJbO+GBkGFzj8V/m3511m/rg96xfFUP2jQZ8DmMiQEe1WQeeTwrIRtJR/Ud6h+1X1kv79RcQdyPvgf1q5EFfAI+bqKdNExj2+WzA8jAzQM1/DF5E8k0UbZjnj3LjsRiuj8kkZ/HmvMtPlm0/URKhKmN8snOCDXQXPxU0y0keJtPu2kX7wJRRn8zXnYqnOck4o9LCVYQi1JnXeT2xWbql21lIgVM7hnJrlZPjFEufK0j/vu4x/JTWXf/ABTlu3U/2ZafKCBudm/9lFYRw1bsdEsVR7nUnW5t2FRaal/NcahbMsabwGA3OEHPqTxXES/Ee5cD/iXWXHTKE4/WoT8Rb7IK2FgOf+eJ/wAa0+rVX0MniqXczvEx36vfHABMzEgHOCTXPnir2p6rJql7LdTRojSYysY2qMDHA/CqLHJr0KcXGKTPNqSTk2htFLShSegNaEDaKsJY3cuPLtpnz/dQmrSeHtZlGY9KvWHqIG/wpcyHyvsZtFbS+Etfb/mFXC/767f51YTwN4gfraRJ/v3MS/zapdSC6jVOb6HO0V1afD7WGIDy2Mef71yp/lmp1+Hd4eX1TTQP9lpG/wDZKXtqfcr2NTscbTuw+tdbP4CnjUeVqNvK2eR5bgY+uKr3HhC6RI/LmEjYwwEZGPTk9aXtoPqHsZrodLbfEKwjsYI3tJ2kWNVY71GSAM969G+FnimDxAupwxQtEYDG2GbOc7h/QV4gvg2/ZiDJEo7bs5P6V6H8KrM+FtcunvbyDyLmDYcbuGDAjt9aw5KSd47nQp1XpLY9ypPrVEaxYH/l4A/4Cf8ACl/tWxx/x8KPwP8AhTEXKQ/pVM6tY8j7Sn60f2rYn/l5SqQi3WZqOEuYG/vBl/kam/tewPH2qPNZuq6xp5hiYXkOVkA5bHtVRepEloP3UFuKzTrOm9r63P0kBph13Txx9oz3+VGP8hW9zGxqbuKNxrHbxDZAZHnt9IW/wqNvEkGCVtbyT2WMf1NF0FjcDd81JE3Nc2fEeMFdPuQP9squP1po8TSBgFtEU/7c4B/QGi6FY67efSiuS/4Sx+8EQP8A11P/AMTRRdDseKWdwkMp8xgPu8fhVK8EJkdhED1P3jyaswWq3O4uXBGOnPapDZpvO4k4HNZJFPYzUbOnscEY3AYFVLd3VmKDn1roRZRG3CKp+Yk4GRVOytEW4YbSGA7/AFp3FYyVDSTdDuNGGXIYYPQj3rZS2j/tEjYD82ee/f1qGWONZWDIu7djkd6XMOxlhn28ntUtpv8AtEeDwDV8AKAFRP8AvkVbt7hoSfuEHr2PfilzD5SNblScbWyT2Wtvw5qEGna7ZX0wlaJH3MFTJxVeGWCRvmYgn3xmux8NarPDe29tbR2qbmA8w26lsfU81hiJvkasdFCF5LU9t0O2i1LTIL6NyscyB1VlwQD607W/Cy6taxxCcRbH3Ftmc8fWn6Pe7bVS4lkkxywTg/lxWk2oNt+W1lbjtgfzNeZhI4XlTa1HVnVVR6nyn8RrX7F4jdQ24HacgfWvoKzYfY4f+ua/yFfP/wAUJQ3ix9qbBtTAJGRjNe+WjD7HAc/8sl/kK9Ond04tk1LKoy3u96YWpm73pCwFaIyYMetQOacz1Cz5zzWqM2UNSbEIxnO5f51jHXY0BXzLQHpjykzn8a1tQO6EYHRgf1rz99Hv3uZWWW3VWckbpO2fQCsqtOMtzSjNx2Oo/wCEkWEHN1EuB1WNB/Sqb+NX+z/aI31CWL5cPHauAc424OwDnIx9ayV8OXMoxJd2q/8AAm/+Jq7NpWqz6RJp0uu28ccihS8Nud3BzkHcPf8AM+2Mo0aXU2dWp0HP4v1F2wuma63uYig/UioW8Sa/OSINFuT2/e30aEk+xaov+EZuSSX8V3oJ5PlW6J7djUi+E7aQf6R4i1qXHf7QQPp1rT2dFdCPaVWWbJvE93p32mG305FVjEQ92ZPmBx1UY6+/Y+2eMaaa4mlld9plbeVVlIBPXB5rtofDGk21r5CajqTRlixXzlwSeST8vJOBQnhnQIlwEuCOgDS4/LAFNSpr4UJqct2clGzCGcBiTlDk49/QVj3Clrh2LANjByf/AK1ep6VpfhZNRmW+jZbbyQRukc5YH2NazWvw2Ry7WSyN1OVlP8zWkUmrmUrp2JfA+m22t+C7SS5mCXMI8t9jbgAOV/Qitt/B2mM5K3kyM33irDn8xWTYeI/BmhwtFpljJBG5yyxQgAn15NSyfEHRFGUsJ2+oUf1rklhfeumbKtKwl18NdPvZg8+tXZA4UAIMfjiqr/CbR9oC6pet774x/wCyU9viTYg/utLcjHeQD+lQt8S8/wCr01R/vSZ/pVqg0rcxDk272Jrj4TaP5CmO8uvMA5JZev021T1LwRDp95p+mae8rxXMU6tLINwRioxnA46fpSr8R7qSUIba3jBP3iGbH6iqs3jm/uLttogDWu6RCE6/KQep96fsrfaYKUuwJ8JLpY1YavBnHQxNx+tQD4YzwxzRG2s7mVkYCc5+8e/Iz/8Ar9hTrj4gaqdPSWG+j84ttMYhX8+lZr+OfEUh/wCP9lH+zGoz+laqNupOo8fDXW/LSMabpQ28hmcYOevGw1ZX4deIDs/daJHtAGMNxj6KKy38VeIZM51G7/A4/lVdtc1yb719eHjp5rVS5RWZ1dt8PtcjtDBJf6eMuXJWNzjJzjt+ue3oKZF8L2iKtNq6DbjpD/8AZVx0uoagx/e3Up/3pT/WohqUaN+/vIlwOjTjr+JobXYFHzPRR4E0+QiOXVcgZ4VF/rmkbwDoqId+pyA9nREUj3z61xdneARiWK8tWDDI2XKFiP8AdBz+lWk1gyDG4nH4VyVKdSUrxdkdEHFKzOr1Hw/pd7bpBdeIdUeOKRZUEUsaEOM4OQmR19axp/C/h1EdvtmsXEiglfNvnIz1HHGeaz/7UiVfm3hvp/8AXqE6vHuxuU+xbH9KcadTrJjvDseeRALcSjcS2efc1XvXD3rRt90oDn8KnusW+rTLt6sep61Q1QSNcKyH5vKDemOtdclocy3Og8JtFFrsXmxJI4V9odQwPyk9DXaXFxiWNlhgQK44RAv9K8z0W+c38DptEofad+SPyFdhLdXci7VktgeuPLIz26/56VHKXzHVW19iGNWONqAdeDj8KswapGkhDEkAc1xaajfRgb7cSdv3bA4/PFTQam6s3n29yBjgCIt/LPNF5ISUWds2t24zgN7cVXutZjlgli8pmDKR17Vy39pMwOywvmz0xEB/Mimtf3WPl0q8PPU+WM/+PUc8h8kTMkLxnEYO5eBgZzT49WkjTyriIo54DckUeY4nPmQGFm5EbEE/pU6r5kWx4xjoAetbLVGT3MRS6agxkfezrnIrM1Hw7canehrQxA7fn3naBW3qf2a1aNI87xyOc4H9Kl0x2+1IAVG/16fSs5trY0ik9Gc0ngPUm63FqD6byf6VYT4fXZOHvYQc4+VSa9GWxcjJkUZ54Gak+wrnlj+Cmuf20zo9jA88T4ergF9Rb/gMP/16nXwBabRm6uGPsqiu+FmoGSzdc9BSC2Azwef0pe1n3H7KHY4EeBLRTnfcMPQ4/pU0fhS0Tbm0Q7fUnn6+td0LZM/cOR71ILaMZ+Xv3GaPaT7h7OBy0OmWi4/0C1H0t0J/9BrWt4oEGANnsihf5CtX7OgPTt6dacsC+1Zyuy4pIpqtv/fnP/bQ1IYLVh8ySt9STVry0zkNijCjuOveocWaqSKotLMcfZ//AB0Cni2gH3YAB9Km3RgffUeuMc00vD180D6Cl7NhzoaY0UbVQAdBgZphQHG1fbpUhniH8ZI/Hio2uoxxhuKpU2S6iGNBuyQDmo2tScYH6083gQ5CMf8AgVNOoNjAVfzqvZMn2iIzYMejAe1C6U28MHj4PTBpTfydemenFNe8lI+8fyq1SZDqI1/7R1UHPmwkD/ZPFB1jVQBiWLPrtrFN1L0MrY9jTDMxH38+1aKLM+ZG3/bmqqR/x74/2lpT4lvlxvWDv3x/SsFpMk+tRscHp0q1ElyN+TxVdbRtSLPc9f6VQufEN5PBJHiPcRxlAVzn0xWYd3Ixj8ab0yOozVKJDY86vqWF2i3Q+giFRtquqk8TqpPpGKaffPPvTDgdyR2qyCT7dqJ63sn/AAEAVG810+N17dYHOFkxn8qTngdqTBDHrn2HWgBpaTOTNcMenzSsf60RxRkkmPNBJxxnNKjMDnJ/xoEWCiA48tPzopN3+0n45ooGchYRuN4ELtk8YUnsOake3umYlbeXHXiM/wCFVGupshY7hwv9xHOOPoaasd1OSAk7/RGP8qzuVYurb3anJikH1XH86rPaSoxkURqck4MijNTxaHqc7jytMvJDg/dgY/0q6nhLX3+5ot98uM5gI/nS549w5W9kZT2rM+8TxK/Gcyr/AENLJbpKo3XUAbuQSc/kK2Y/AviiYYXRLoAdNwC4/Mir9t8OPFbx7DpjRg8ndOgB/DNJ1IdxqEuxyb2USjDXsZ7nAb/Co2SBOPPc9jhPeu6Pwm8TTEEm2UDHDzfn0BqzH8GdZfHmX9nGPYsf6UvbU11H7OfY87DxqoxJLx0+Uf41ct9ensbhJYXlDqcqdoxn15zmvQ0+C84wZdWiB/2Yyf51Bd/CK3soTJca6evCJb8n25Y/yqXXpPRsuNOqtUcz/wALQ8SIu1L6cJ327Bj/AMcqnd/ELxDdDbJf3LAE4/e7f/QQK6e18C6LAM3C3N0e4d9qn8FFWbbwXoMGXFgWBP3ZHLflWa+rx2iXyVnuzzZft3ibVLa2jQS3c8ixxlnPzE8AEsenNfTttZ3iW0MZt23IiqTuHUD61wWi6Ho+n61Z3wsoLcQzLIZSSoQA9euMfWvRG8Y+HV66zZf8BlB/lXTTcai7GFRSg9RPsl3n/U8/7w/xoNheEfcT8WqNvG3h7+HUkfPTZG7fyFMPjXRz9yS5k/3bWT+q1pywXUyvJ9CU6XeN/wA8h/wI/wCFL/Ydw3WaMfgTVY+M7Ij5LPUG+kGP5kU0+NIsZTSr856FvLGf/Hqd4LqHLN9CeXw28y7Xux68R/8A164bU9Mn07Vp7ZN8wQgh8AZyM/1rrm8ZTEDy9IkOf79wq/yzXnviXSb/AF3W7jVRFHbrPtVohKWwVAAOcDOcVE5QfUuEZroaAiuOP9Wp9DKo/rUi2kzjLXFuvsZgf5Vzy+H9QQAG5bjgYkNTL4cu3iLtOGK9RvJ49az9zuae/wBjb+xEH5ry3A9mY/yFO+zWy8tqcS/8Ab/CsD/hHWb70gB+nNVrrw/MGJheWSTHKqcA/T3p3gK0jqAulqCG1dST2VP8TTWuNGjB3X0zEf3UX/4quGNpg9HwDg5JyKetkCxLJj1yenpT90NTorrVNKWTdFcygEYO+Pr7YArHm123BISO4cDjOEGfzYVGltED/qx754pTaxHG1RjuO4ouhcrIf7ZVvu2s5Yf3njH8mNMbWXI4tI/+B3BGPyU1bW3QKSFTPY5wR9acscYyGWLHfAxTug5WZ51a8PKWtqPbzXb/ANkFNGqai+Ni2oxzxE7H9WFagii6jaQPanGMZAO1snqKLoOVmW19qbjImjU+q2vb8XNNSfV1+ZbqQEgg4ijHB6jkGtbAA+V8jOOtBEXzDawAHXGRRcOUycai4IN1cehwY1/UJTTbXn/Pa6PPU3DD+RFbBCkABgf0FMJRD1246dqLi5UZTWk7cGSXb/tTyHn/AL6qMaSPvOsbAf38tn8ya2TLHtyxU8dSMU0XNvu9B/tDii7CyMoaMp/5Z23XtCuf5U9dOEX3QijvtjUfyFaX2iM8htwz6f5/pUZuACCPMB6c07sLIqCzZBjz5eewfrTTY9N0kxzzy5qwbpvvY3foab9o6kq6kehzijUWhXbToyQWyT9T1oFjGMgJjrxjtU5mYgkAMuMYqMyOq4XaTjgEU7MLoyNZXZOsigcgcjiq1zKf9HcE4KlSfx/+vWjqCm4WPAyQSCQMdqh/seS6hxGwUqwNVbQm+pn6Zi28QW0hQMm8NhuMgV6BHq9kOXhKZ6fICP0Oa4CzVmu4XBKvGSCQcHoa0y8pY7Cxxzjdzio5blKVjvINTsZANtzEi+jDZ/6EKvJ5c3MciOMdFw1eYszFMuQA3TcMdPbH+f5AQnoMls4K9e3/ANapdLzKVXyPUBB04Geh4x/KlMQ7uRk9j7//AK683jl1NQTFPccDkCXgfhmrSatq0BCs8RGesi4z+Q9/88UvZj9qjptXtFco6MN49SCazV3DAyS3QgDpVWPW5JiFkiRsn76yqMc9duauq4ckrjDelbQVlYyk7u5lXFs53SuuSWznGTgUkLmJ0cLkqR+daDxyzI3I8vPUmsxh84HUA459+DUtFpndWd9DJbLlSSo9vzqc3ic4Un64rltIuT5fl547Dv7Vq+Z6gfXHFZezRr7Rmgb0dPKH50z7UzchVFUgCc8Nj1FKCccN+lHIg52WjdSZPAHvTDcS55fH/AagBYDkA5pTu6YHsQafIhc7HmZzkFjzTTI5wcyelNwc45A9xS49iR9aOVBzMbvYg53E+5puSwz2+tSbVPGPyNIU9v60cocxCcnq/PsaAWAGM/iKkKkHr+femFD1x+tOwrjcnOdx/Kk3Mc808fLzs56UhKt1Bz9KLCuMJyOnNIZD6Z7fhUoCeoGaQKo6enpTsFxnnMANy8Uhkx1zmpCRkDjPrTDjOBjPc4osK4wv9cCjzBtyadgZ4x+ApCOhBp2FcQspHQflSHYR/wDWpGBPORjr1pjRtnIwDTsK47CnI/rTSFYHH4ZNG1wSSw+opuGxyRQAbQO4pMY79O1O+fkYXIpp4HKj3GKAGEnPXNN6cZp5dRwy8UgZTjhjx1AouKxGT16nj+7TlPPQVIFB+7ux6kdakSAkYAwPei4+UZweSoz3oqyIBjon50UrhynuS6baIoCW8KgdAIxUq26KOAPyoorwzruwMYz6UGEdyaKKpId2MKKvbNIQBziiinYpMawA7Ckz8ucCiikWivcSmKFpAAcDpXJXfmXkvmTSE+gAwBRRVQLKotELgEnmka3QnJzx0GelFFaCEFvHJlWG4Hg7uc05raCKRkjiRdpx90dqKKuLJaFVeOMDj0pzrlgvoBRRVXJsNIwcdef/AK/9KcFDQqfRiBRRTQhgQc/TPSpY4wxZSeNpP5DP9KKKYEbQqDmiNFWUMowR685oopiJprONHUrwrAMBjpntUX2JGLAnHuByKKKAMLWtNh+xm/GRIrbXH9/DYz7H/PNc8QoQfKPmFFFbQ2MZ7kTSqqL+7yGbHJ6d89PaohdssYIHV9vPWiitDO48XLsyjAzgc49aikuZE2HIw3oMf56UUVVhNjGupd4w3zA9SBTlkl8zy3kZgRu9KKKdibitlGDFi3zAfnTXLB1+dvm6c9BRRTsIYPmjU9MilWQgKjcljjI4oooAcFBbbjBx1HHShY8fxHrRRSExTCNwwSPcUzPcE9e5oopgICgbaVzu4zmo5ZvLYkAk56k0UUAQi4EhB8sA+uf8+tX4oBMuchfoooopiHz2Bgi81pA4MgGNuOv40qw7OAx4IwaKKaEzmvs4jvkIPRwDx15rtI9Gs5H2BWB+ox0z2AoorNFg+l2sEgUK5I7iRhn8M0q2NqVJMKt7P8386KKoBBa2SkD7HFknHCj/AAqw1pHBGHVI+fRMUUUhiKSwHPSs9iTOccZOOlFFVEmRnajq8ttcfY40DEgtvkYnuewx6VjnVyzyLLArFOMo23P4HNFFTLcqOxc0bWfOvPJWDZwDnfn+ldpgEDOegPBoopMaF8sEbgSOM/pRgjox/GiigBQD83P0oVNx4JHP1oooADEVwd36VEWIbk55xzRRQAhcggHBpyNuKr3PeiigCSaMp82/PPp9f8KgBJJ59B0oopgByWIzQQP1oooADCQCd34YqPb8x6flRRSAaF3AnOMUuzgc0UUANIK5weBTkBIzu+mRRRQBKIc9WB/Cl+zr3J/Ciikxirao4JBYY9TmpfsaIcZzgUUUmxpCC3Ut/DknGdtNZAoyMD0oopXHYYBwGBxnNOSPe2CR6dKKKAQqxK2RgevSnLCg5IP4UUUhjjEmT8tFFFAH/9k=" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAAmAHgDASIAAhEBAxEB/8QAHAAAAgIDAQEAAAAAAAAAAAAAAAcFBgIECAMB/8QANhAAAQIFAwIFAgQEBwAAAAAAAQIDAAQFBhESITEHQRMiUXGBFGEWocHwCCMykSQlQmKy0eH/xAAYAQADAQEAAAAAAAAAAAAAAAACAwQBAP/EACURAAICAQQCAgIDAAAAAAAAAAECABEDBBIhMRNRBUEyYXGhwf/aAAwDAQACEQMRAD8A7evC6a5I1xUrTp4y7TaEpwEIVqJ3z5gfXHxG5VL6elbZTVmC19SShkpUNkuFQSdvnI9xEBe7Z/EbhxspCTz8fpCzp9+0+7rMVVKa55Wqu3JzTQ2LT7E6Gnkn2KT8YPBhDP4tUynqpYuPy6dWHdx/Wjc09WpR1yoeD4iFgJ0J05Tj0ye8TU9UZeQlVzc28lplpJWtR4AEL+0FFmYdaRsMJOAe28ZdU6g+xQGWWlkJfmEJXjuACcfkIPSkvh3HuL1KhMu0dT5M9bKXLzCW1UwpbcVpbL00ltaz6BOCM/bJi3W5edIuZpapFa0OtYLjLgwpOe+2xHsffEcl9Ruikp1TlzUFOutzkhLFuVcSo4bVnV/TnGCcZ9QB6DF76Jz89Iu0MTswVzBlRLTKlZytQRg5PfzJBha+QIMjHgmobKm4ovYE6AuK6KPa8impVqdTLsLcS0klJUVLVwkAbngn2BjKjXTRbgl3JmkTqX0NKCHPIpJScZ4UAfn39IrHUi3TeFmzlKaUQ/p8aXVnh1Jyn4yMfMKHp1ek1Sm3ZqYbd1BCmJtnBCg4jgY7HP8AyMVZMYRBk+r5/wAkyW7FPv6nRP4hoyZ0yDtWlETOQnwVPJCsngYJzk5G0SCnEhPIPaEJZ701M3Q1U30hx5LinlKUNtRznHpudvaGzW62KPQZmquI1eA0XNIONRA2HztEunfz3UfmxeEiTqpxprBddQjPGpWM+2Y9wsKSCk88GOSOrF53tSaI5cdv0qWq1SU4VuomtehDQSSrSlGCTwAMgAZ+WB0M6ki4JOlrLyW0VeVDxlC5q8B7RqIA7DZQ7Z2hS5C3NcXUccIQcHmrj3UoJ5OOYPGGcBUU+/rinKFQDNyK9Ly32mUr0g6QpQBO+3Gf7wobu/iKa6ezclK3PVJtJnkrWytFPDreEEAhRQnY+Yc8wzM3h2ggm/UHCpygmwKnSAVt/wCwRQOkvUZrqRQHa1KurdlkveCy6qVWxrwAVYCwCcE4yNu3IgjFbcLqc67TUir8CUV8OLG3gA8f7iI54takK6e9Ta9brjOmh3s+7V5IKOUInSf8Q3vxqGlXuTiH/wBZun91Xe5ITVqXW5RXJbUHtLIc8UHGM7jjfv3iOuPpK9WrSlpNmd/zimhD0nOvIGQ+gYClAdjuCB6/aKXRGyMWPBFfxE48rJjVVHIMlbUdCpnKVf1IGd/36x6dV0Zt2XfIOG5xvJPbORn84gek1rdR6bNTT9+ViSmiUpQy3Ky+gJxyoqwCc7YGNt4v96Wuq5rTn6Ih7wnZhr+U4f8AQ4N0q9sgRmiUY02MZ2rc5H3gSg2OthUvMskDVnJGPUY/SKtaCQzXmG0jSUTzqchPo6sbekVRy6b9sl9ynXLZNYMw0S2mcpjfitvJHB8vGef0HEXLpemfuKqJqJoFQp7DaisKnmw2tazydOSfU5PrCs7MmMYfR7uMxIGc5vY6j2kwHGSCcjHpCOv61zb99vVGT0ok6zLlxaPR9J3PyDn3zD5kpctNhJ5xCs6/T0tQqdTa3NocDTbq2VFttSj5k7bD2/fEXIQ6MnsST8XVvRnv05poWj61YGVqOCPscRaOobJNlVMNg5SzqGPsQf0iD6PzQqtrS1RZZdS26tami42UFSc7HB3/AHkbYi6XFTTVaFPU1OMzMutodtykgfnEujXxgAx2qcuxMU9sU+WqU0ZaYbS4hTRV5hsRt/3G/wBP+jtk2bcj1epFIlWZ5YVocKtS0BXITk+Udtu23EVCyLwlvHMsXA3UZHMvNyrmEOJIOCce4/SJmhNOv3ImozL5dden/qE6jnSjIITz2CQInetNaH8r/qVKGz0w6qXzqugmy5pwZBZcZc59HEwoblq9mUYSj141KQlEvLU0wqccCAo4BIBPfEOXqE2Zux6xpyCJVSwfuBmFLVLWt+7JeVauSny02y0fFQHmQsJUU4JGQcbRvySg48bHqzO+PYhnH6EZvSa5qBcFHX+GpqSmJWRKWdUrMB1AJGdORsD9vaCPTphI2tb9LRQLcRLSyEFTpYYZ8MEkjKsAAHkbwQGGtnBmZid/Mv6mUKG4Bz6xh9OjAGBg7wQRbIJiJVpBJSkc77RmWkkAYz2ggjBCPM0JykSsycONJPuI+SlIlZQ/yUJHbYQQQt1W7qGjtVXJFLWBkHiNSp06WqDH0002lxBPChkQQQVWtTgaa5hT6exJthlkBKU4AA4jcKAoYPBgggV44E1zZ5iw6gdDbEvWeNWqVLDU8capiXcU04rG3mKCNW22+YLB6T29ZrqlU1Li3HNi4+8t1ZAPGVEkD7DaCCEZSWcBjKMPC8S+VSkMVKkTVMfUoNzTC2lFOxAUkg4/vHPU50GrUnMLVIdUbhbSFEpQqYKk75OCMAbekEEDq8zqgUHiN0uNSxsS99IbCq1r1OZn6veE/WVLb8JpD+AhsZBJx3JwB7QQQQOAkoDAz8ORP//Z" } ] }, { "items": [ { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAoAJ8DASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1PUNYe0umiSIMFxkk4yaqWPi22umyux0BwzRSB9v1qHWlxqEpx1Cn9K5PS9DXStRu7mJgIbgA7AOQc5roprmitDOo+V7nqquGAIOQeQaoxaxayXf2YMxcsVBxxke9PsSRZwqSDtQDP4Vy0RMWqMf+edyR+tSlefKN/DzHVXOp2tpMI55gpIzjBP8AKrYZWUEHIPIritfY/wBsHPR4VP5Eiun06YyadA2efLAz+FSvhuNqzsWUuYnkMfmJvH8IYZqUnAzXF6XIU1OJ2OW81lOfXJFb+uzPFos7xsVbjkHnGRVPSKkJK8uU1QcjNLmuc8OTuTcRs7MPlI3HOOtdCDkZptWJWopcZxmgNk1gahqNxHfvFGwVVwMbRzxmsqw8YRXN61skwaZM7o3jKnjrz0qoxclcmUlF2Z2tJmooZlmiWRTwwzWE3idE1tbH7MxVpPL8zf3zjpj+tS9HYpK6ujojTc1Wvr2Oys3uJSdi9gMk+1Z+leIrLVVn8ovGYV3MJQBx69aLoLM2KSsrTvEGn6jdva28zPKoLYKEcDvn8a1GbC5ppit0DFJVD+2tO87yTfWvmZxs81c5/OrwYMMg5pgYOvLi9U+sY/nXFRaXNZ+J1u4XkNvcB/MQEkA9c13niKwu7uON7OVY3XIJZN2RXOxaXrfmBXkt9vc+Wc/zqaS0s+5VR638jqNFcmxAP8LECsO9Hlazdr0+dXH4iuk0+38i3VD2FYfiWwvFuUv7KPzTt2SR+o7GlJ2nzIcVeDiyt4hZTeWUi/xxsv8AWtzQn3aVGM/dyP1rjVuLu7kijewuQykhd68LnrzXcaTbG3sUjPJ6n60+XlixOV5I5s/uNXmHTZck/gTmuj1dPN0a6Uf88yfy5/pXNaw4ttdnj2OWmKsgVCc8AV1roZbNkYfeQgj8Kn/l3Yvapc57w6+Lsj+9H/hXVr0rz6x1OGykjYzRB1BBUsAa6bS9fivblYFCEsDgrID09qvVxTRm9JNMq6wNuqH/AGkVv6Vi6jp0cGoafqCKFLsY3I756f1rd8QDbe2z/wB5WH5f/rrLv3a6tLaKMjMUyuc+gzn+dKHM4NLuOfLGak+x0WkSE2hX+62K4vVj9m1+R+6T7v1Brr9Hz5LH1Ncv4riMerSsP4wrD8sf0pVtKiYUNaTR0/iAGTQpyOwDfqK80gupLC8cocFg0ZGfvAjkV6dj7ZoGP+ett/Na8m1eNxiWMbmjcSY9u/6ZqJaTfoarWC9Tb8IXRXxFbvziRWQ/kf6iuu8YXk1vopWF2UyEgleuMGuC0WQQaxZyjp5ynPsTzXe+LYfN0cH+7IM/iCKp6Q0Jirz1PMRYyrYR6ijJ5Ql8t1x09816F4E1GS60+eBmLxQuBFJ2wf4c+2P1riItTnstDmgSEvEsxeXaoJAKjHXoMiut8F6ncTT/AGeUkRtDvRCoBXkensa1vdsxtZI74qGpnkrnpRRWRqPCAUNGrDBFFFICv9jj3ZCj8qnWMKMCiiq5mxWSIpLVHkDkfMOhqQJgYooouxWMW58N2E8hdraIknJOwVLpui2umyM8ECIzDBIUA4ooquZ2Ja1H6xpMWq2yxybgVOQVYr/KsGPwikcgInuNueR5p5oopxdhSV9zprS2EEQQDAAxWP4g0OXU54pIphGANr/JnPOfWiij4nqNPlWhrWdqtrZR265KxoFG7riuQtvC9wdVL3Jj8hMqAMkuMYwfTrRRSUU3cfM0rIzj4RvYtQWGLaIVb5JSei/T1rudSszfaXNbgjey/KT/AHhyKKKcoK1hKbvc8umtLy0uJomgkQygo8ZQkNn0rQ8O3X9n37Xl7izs7cFJJrgiJFBwBlmwBliAKKKqKtG4pO8j/9k=" + "mime": "application/vnd.code.notebook.stdout", + "value": [ + "Box 1: [257,50,377,88,0,0.48566171526908875], Annotation: 'noyce'", + "" + ] } ] }, @@ -246,15 +162,18 @@ "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAmAHkDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1bV7ye3kjWF9nGTwOaWHU2OmvK5BkjU5yOvpUGuD95Gfaucj1SOeDVIYziS23RSL77dw/MEVolzQ8xN2n5HV6XqM10zrNt4AIwMVps4C5JwPWub0xsTgr0Zc1parKyaZKVOCRj8zSqJJhDVFafxLBE5CxZQHG9nCg1dsdWgvW2LlJMZ2t3+lcJf6FHrsaxSbsR5IwcYJ71L4fjl04W0Ej7ngk2Fj3AP8AgabSvYSva56BcXMVtbtPM+2NBliaitdRtbwsIJdzKMkYI/nTbqFbyylgY8OpBxXH6fcS2F4VfIeElHHqKzW9i3HS520l5bxSBJJ40YjOGYCpsjFcSkzXepLMwzlgwB7DtXXCXEBduwyauUeVIiL5mTGQL94gfU0oYMMg5rgdZ1C8kgluYI1km48tHJ2j8vapPCuuyXEcEkoETyNsliB4BzjgVXKK53fSjdVS9uGt7GaVfvKpI+tcnf8AiqTSo0kuppNjttBWINg4zzgUormHLQ7jNH51z3hzxANbErRsXjTHzGMrye3PWt7NNqwJ3MnWx/qifeuHltzpviU3yjFvqAEUvoJAMKfyyK7nxDp9xqFmqWtwYJVbO4DOR6VQTQjLo/2W6kMsmMmQjHzdQcfWoTajZFNK92M07CyoBxgYxWpqozpU3sAaw9G07V4b8m+niaJQQoRME/jXTywCa2kibo6lT+NOq+Z3FTXKc7pDKJ3U9SM1RnwupzAf89f6Cqc76jo8xjntJpVXhJoBnI9xU2mSS6jeBza3CDOWeZduTTiry5gk7LlO3g5QfSue1+x8u9ivI+A4KSe/pXRwLtQA9azfEbLFpZmYHEbBjgZNZMuLMvRoN8hc9jiujnTNpKo7oR+lYHhiZbm3kkQNs38EqRmulIymD3rSo7siCschZRiSeNWHBHf6VZt/DmnwasL0Qx+bu3AnsfUCqEVwtrfvbsds0LkbW43D1H4VJMxub15WJw20KpPTFVG8rCl7rZ0+oLu024Hfy2/lXGXUtrDAst5IiRcfNIcDJrtX/e2rj1Qj9K417WK7to4rhFeMhchlyOKKPxMKvwI2PDl9Z3AaGzkhcINx8tw2K6Ld7VgaDBYWAMNsI42c5IVMZ/Sug49acr3JjsP2g0mwUUViaCeWucgUuBiiimBFLAkg+YCo0tI4zkAUUU02JosquBTJY1kjKsMg0UUdQ6EUFukIwgwParGKKKGIxtY0Cx1XDXEQLgYDgkN+Yqjp3hu0sbjzI9zN0Bdi2Pzooq4bClqdEqAJj8K4q88Myi4dotUuo1JyFD8D2oopwSbCUmkWtD0ie21BZptQmnCg4Q9M+tcr/wANA+E/+gfrX/fmL/45RRRU0ZMNUf/Z" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAAYADEDASIAAhEBAxEB/8QAGwAAAwACAwAAAAAAAAAAAAAAAAUGAwQBBwj/xAAxEAACAQIFAgQEBQUAAAAAAAABAgMEBQAGERIhIjEHE0FhFSNSkRQWMlGyU3GBoeH/xAAYAQADAQEAAAAAAAAAAAAAAAABAgMABP/EABoRAAMBAQEBAAAAAAAAAAAAAAABAhESITH/2gAMAwEAAhEDEQA/APUd8z3nSozXdrbl9lZLRy9MrIrmNQCz9QO488KSAeBqO5q1z3O2UTeUijarWUQHUaLuOnVp/b017+2NHxJvVFYYlo7dTQrdr18nzUQBxGP1EnTnvtHufbDPK+VqWnyyaG9UsM61Oks0Uq7l400BB440H+cTlNW0mdNNOU2hTl/P98+LRUF/onWCoCtFMyIA6sSA6lCeNe4YAjUYy5o8Q7ta6+rprVbGnFEetEj3yMoALMAWHAB9OdAdNTxhVEozXmmnht1OkVutxCx7R0rGpB+5IGnsPbGzcoxQ+IhU67KsIxGnHWhjP8Rho1z9FrNKKLPcTZXa+/hl85GEJiB6S5I0IP08640sueJbXO9fBrlQCF2K7XCOg6tQpG7hlJBGoPcYkrKjNli+252Jaj2Te+sTnd/HHNTN5dLZLkoIaHzYCfXcrq6+n7bsU1/dE8O6Nze+DCn43R/1B9sGH7ExkL4i5IzNfcwW2/ZdlpN1LF5bLPKyNqG1BUhW98M7Fb8+slXTZhlpxFLTNHGUqTJ8w9jyi6D74MGIqF1pTt84T9st3iNYFeK22dEVzq2yeNwx079S4b5usGZK2K2ZgoaVZrhSxgVVOjBWbsQyE9JIYHg6AgnnjQmDGicTRqpvDFk/Ltxq3uc94t0tCtfFLGySBQSZDqxAVjieNozTThMt11jqGMFR5sc8abopCVK7w47Ag6kNyOe/cmDB58QN9Kv8m3H6v9/9wYMGKcoGn//Z" } ] }, { "items": [ { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAXADEDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDu7rVNQfUriK1wRB1QEA47nnr9K0RqrHSzOFBkB2+2fWotauY7VQkSKLm4+XcBzjv/AIVYsLFE0/y7hFcP8zKw4q3sJblWz1a4+0rHcxkK2CrEDkHuMU++1eeCeRIYSwj6gDJPqetVlH2/UkWJQsEXAx0AFSzDytcx2fB/MY/pSY0XV1QHTjcbBuB27R61FZ60Z7ryJY9pOMHBHXp16is62BOn3cR6x4b/AL5PP8qV22xWso6ruX8QQR/WkM6jmiq32pPX9KKQ7mPrOmXd1ewXNq0eUXBDtjvn0NT2kWpESJdMm0oQMPu5/IUUVV9CbalKGPU7TKxQqATzhwf51b1G2uZI4LpFDTIv7xQQM+47daKKGxIZpls8xneVNiyKQQcd+vQ1QYTxOLGZBvD7lIIw3GM//rooo6jOc/4Wd4J/6D3/AJKz/wDxFFFFK4H/2Q==" + "mime": "application/vnd.code.notebook.stdout", + "value": [ + "Box 2: [604,205,653,229,0,0.45074450969696045], Annotation: '2200'", + "" + ] } ] }, @@ -262,7 +181,18 @@ "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAgABoDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD03Ur25h1BhHMyqAMLnj8qtXOpsNN8+JgJDgeuDVDV0Jvmx121gWGsJqGg29yn+rn8tgP7pJAI/A8fhS6D6nZaVfyXUDNMwLhscDHGK0s1zujHBce9boPApgjH1FAb/P8AsiuK021OlXk+nFf9Hc+fBxwMn5h+B5rsNe0u9u7uOW0vpLdQNrKoBB561FqWiPcWitbttuo+Y3I/PNKzHdD9II3t+FbobgVzugWWoQeY19KjsSNuxcYFTT+LPDVtcSQT+I9JimiYpJHJexqyMDgggtwQe1N3ErH/2Q==" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAApAKMDASIAAhEBAxEB/8QAHQAAAQQDAQEAAAAAAAAAAAAABwMEBQYAAQgCCf/EADcQAAEDAwMCAwcDAwMFAAAAAAECAwQABREGEiEHMRNBUQgUIjJhcYEzkcEVNEJScrFiobLR8P/EABkBAAMBAQEAAAAAAAAAAAAAAAECAwAEBf/EACARAAICAgMBAAMAAAAAAAAAAAABAhEDIRITMUEEUWH/2gAMAwEAAhEDEQA/APoVNKVuIQR2ST6c8Vzf1C9sux9NepV36fXnpvqSQxanmm/6nAdQtDoWyhwqCFbcY8Qpxv7pNdGzQrxwOflP/NBm86U6Laj1vc0XHV9nVqCRJSiVb1zo5eQ5sSAjwid4JTs4IzyPWvRgtaOFvYW7Df496tEG/Wyap+BcY7UqOtWQVNuJCkE55HBGRU4xKLieTzQo1heLppGPYtL6Z0xMXb3X2Iip6VN+7w2G8HaQFb9ygnaPhCefmzgEh2V9TsZDijuJHrWnBUNGTRJyZymgENgFSueewqlxOs2g52q3dDwdeWCRqBhamnbaiUkvJcT8yMA8qGDlI5GDxVldcIdW5/p/gVx1pf2doKpMPWLYLd1YkpuQkIGHC/v8TcVd1Eq55oRxqvDPJ/TtSPcVOJ+NvCvMU7S+lScj71Xp8wQ/BWDgOPbPvwT/ABUgw/vZUoHORSSxr4UU3Qv/AFdk5Bbd4+g5/wC9Kt3SI6oJ8QpJ4G4YzQd9oO16rvPR+7QNFXi6Wu8OuQlMSrY6tuSgCS0pe1SPiAKAoHH+JNMPZwZ6kW7py7H6qXeZPnxZzyY0yd+quJtQUlajyrB3jJ9O+BQeJG7GkHpLqFDhWT9a8qfaRhK3UJJGQCrk1DWq5NT47UppWUuJCwfUEV5muFUxQUDwhOPyTSdVh7ScDzaz8LiT9lCvee3nXHMz2i+vEPWF5sDfSWx3CHBukqJDeakrbU6y26pKFKJWQFFISTx3zgeVdGdNdSX/AFFpSFc9SWFFluD4Wp2GiQHktfEduFjvkAH6Zx5UrjHxMpckra0XoKGfpWFXpzULPkuGQ0whxSRsWo4OMkFI/mgLrX2yOnHT3qBcunupbfq+PKtbrbT05qEh2Ire2hwKSrxN5ACwD8Gcg8Gj1MXtR0qVZ/Fa3c9zUFbL173HZktSUvsPtpdacH+SFDKT9iCK8an1TA0zYpV+ucpiLFiJCnXn1bW20lQG5R9Bml634Mppk/uHesCuO57VQNPdWNNarceY0zqKy3dyOne8iHKS4ptJPBUlJJGTxUk11AtRn/0pU23e9IcS26yJifEQo44KO4PI4+ooOLXqDyTLbu9M1lNhLTjkH9xWUvEYiJv642n/ABxxXHHUP2aLFr/qVqXVjqnWpc24qWpbalDcUhKBkdjwkcYrpPrBpfXF7jwZeh9Zy7FJilYdSy0lxD4VtxuB9NvH3NZoHR9ygWdoajuC59xXlyTIU2Gy64okqVtHA5PlXbDLCOm9nI8U3tCsNl+Loa2x5zin348OK064vJU4oBAKjnzJGeTVnshSWEqGO2eKDl/Y67wNYOWm1zLA/pp6SnZ7xHX4rccqG5OUjlQGcH7UarNDcZYQlR5xQeWErSdhWKa9RokOCU2k/GFFHfzKRiqTpuJ4VnbaUkhaG9pBHmBjH70t1FumrNGyhqXT1hVfYbiA3OtyHNjuBna62cHkZIKcHII9KHq/ac0PEcWqboPWTErOVMC2IJKvMg+IMn696rHIlG7JSxSk6oLGp5IBt7CVAKVKKvqQlBz/AOQqcg591J9BQh0JqPUvVC/L1LO01JsVqiN+72+NJ/WXuOVurGOCdqBjHGO5zkmZqOW4+0ZOE/vUeyL8ZXrlFKwZ9c9cav6edKJ+rNC2uFcL1EXDQxGmoUplaVvtocyEKSrIQpRGD3A79qq/RrX0r2ien11g9RtJKtcuHK90mxWHnG23gUhSHEYVuA75SVKHHOc4p1eut/Ra7xnNO6g1WI/hOpS+07FfQQtCv9RbxjcKRY609FdGWyQrS14aukqUdyIcRC1PSXcYSn5cJ7YyccZ79qa9bFpv4FjTzJgse6BYUGiUAjzxxT58Zmr+HOW0Hv8AVVQmiF3GVaY8q5NhuS8gOvJHYLUMqA+mTxU0+tKJxSedyE/8mlT/AEan9OeJ/tQdC7Jqu76euWkNRR7hbblIhvvptiC268h1SFrSpLmVJKgSDjJHlR00RqC2apsMK/2JD6IU1oOspfaLbgSe25J5FVBPSW2OXuXeZDEZxUmU7J7En41lWDkd+aIVpisQI6I7aNiUJCQB9Klu9ovrjpmp3FyaAPJZcHP+5NB7VfTrpjq3V10N1v1rVdpTqBIhF1lT7avDSEgo3bgSnaRkdiD6UYpoBuLBGf0nBj8prlLrV7Ntm6g9R79q1p1+PPkrZKnWlEbihhtAyOx4SKvXKiF8bD87dJNjl2nS1tsU0QGWEtiepSCylLaQlLXCiveeDykDAOCewtc+1xNQWZ63XBhuRHko2uNuJ3JUPqDx6VWdN+8J0faUz1qfebhxkPOK5KlpCQVH65zVxtuPCx6jmpS06KJasF+helGnunurblMsluZii4wwFhpsJB2OAjsP+o1U5PQvTNw6xzOoT0HdO9/blhfiKA8RCE7VYzjIKQe3lRuuQSi7MHABUw6PzlJ/imcFtJukk4wfFzn14FLkimkPjltk6wkhlAVgkDkmsp4hKQgZA7VlRtlhR1hLowoZrwiKhHyp8qcDt+K2r5R9hTkxm5b2XVBSkDPrS7TAQCAKXPes/wDVAI1kxG5DZQtOQfI1AStGWx5wuKiIJPnirR5H71pXb80rjGXqGjOUfGRECyR4KQhhoIA7ACpMM4SUilR3/FYO34rJKKpGk72yg3/pbpu8yXJMu0sOrcO5RU0Dk/tUdauj2lbTNEyHZorTqT86GUpV+4FFBz5fxSB7I+1SeGDdjxyySoaQoIYQEYAAqhdX+j9m6mtwHLi9KZet/ieE5HeLZwrbkEDv8o/+zRMT8xpN35arbqkTXts5iT7LrkJWbbrbVMfHYN3Ip/ijJ0x0hJ0fYGrNLu9xuSkuLX486QXXDk9txHYDsKth86cMdj+aSCknbk2Um4yVJJFK6raX1LqG0R0aS1NLsU6O8V+OwkK3pKSChQPlkg/cCkND6TusK0NI1NdXbpcVAl+U42lBcJPBwOBgYA+gq/v/AKP702jdh9qssk7pPRPhH2gKJsvWS3ayFtZvdrVplcrKULh5dTH3Z2AgD4sZG4nvz6CjPb43htAE84x2pB/+4H+6pNnuPt/FLHJKTfIPGMVpA/6sXzUelosK86d0m5f1hxTT0duQlkoQoZ35IOeQBjHnS2gZd2vttbvV1syrVIlErVEW6HC3zgZUAM5AB7eePKrPqD+z/JpOy/2zX+w0j/Il2ddDLElHkTCAdo4FZSqflFZWsU//2Q==" + } + ] + }, + { + "items": [ + { + "mime": "application/vnd.code.notebook.stdout", + "value": [ + "Box 3: [22,33,185,74,0,0.3334950804710388], Annotation: 'robert'", + "" + ] } ] }, @@ -270,7 +200,18 @@ "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAXAB8DASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDg538i7nWG3jVi5VGPOecH+lK0lzuNorrG4kKnYvHIBH8jVnUIo1S+yB5qvuQn3wf6VTlEseqQyy4LM0ZbHTJ4/wAa9WWjOFDLeCSaSFZJ5DviZhz3BxirGgttZ4yxIKK3J78g/wAqZ5P7y2TcyGO4dMg846gU/SEMOpyISTuDKue2D0/WlHcGaWq6fNNfAxbfLcZYnsQCP61ANNvJlzO0YYIFXBz908f1/OiitnFXIuSTaS8wk2y7CZfMVgOnGKjktYNKg+23NyypEdzPjP3gF7c9aKKmaUYuS6Djq0j/2Q==" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAAXACADASIAAhEBAxEB/8QAGgAAAQUBAAAAAAAAAAAAAAAAAAECBQYHA//EACsQAAEDAgUDBAEFAAAAAAAAAAECAwQFEQAGEiFBBzFREyIjYYEUJEJxkf/EABkBAAIDAQAAAAAAAAAAAAAAAAIGAAMEBf/EABoRAAIDAQEAAAAAAAAAAAAAAAECAAMREiL/2gAMAwEAAhEDEQA/AMPz5IZomb6lLiUaKXG0haZKyoq9fQHL2v4KvG+I92oV6lpVSHHYsORJ/SO/tmk6S25qQb2G9iUk/wCYuud6VTl16uNT20fPAQ4ypdhZzS42nTfsd7WxmmYG6sYVHrFQLa3VQkq+JJAKEup03PBspIPHm+Eys+RHq4csY+LS6q7NhR6nXJavVqT8VwpWRpUhNxbewCiPxfEl07Sqm5tcguPulLxltOJWq41tu7EePaobXthMxwkumpspeWyBWIspLjavchLyACQeO9746UOIqldTSt51bja5KmQF/wAVuNBSVgeSQRf65wbnpZWnlxNL6o5XqtVqECRSg171JYlFRtoQFhwKG+/YjnFXjZBzRMYaptXehiJGZkxWCFlStDibgnYdlaT/AEMGDGJLGVcE6VlKliTJCZ07lVGNMZcmIaXLixm1KQndDrJ3V93BHI5/Laf00ENSZkqsyJEhtxl8ukAFSmlKIPPcHSfoYMGIbGgilN3J/9k=" + } + ] + }, + { + "items": [ + { + "mime": "application/vnd.code.notebook.stdout", + "value": [ + "Box 4: [506,408,538,431,0,0.32059410214424133], Annotation: 'center'", + "" + ] } ] }, @@ -278,7 +219,18 @@ "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAqAKMDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD2d+QAfWuQ8R+OYfDesLYS6Zc3CtEsnmwsOMkjGD9PXvXXyZ4rnNUtdEudVP269gju9ijypJVB284ODz61kzVGtpWpw6rpkF/bM/kyruAcYYc4IPuCDWgr571gXrNo2kLDp9o8q/dBQjbHuPLHnOOewP4VrWbl4lJ5NCYNFpn2j1NZk3iHTbbUBYTajax3ZxiFnAbnp+daDHLV51d+EodU1W+upRl5Lh23d+pA5+gFDYJHo6Sk9RzUgbNZ8TNFYQl2JYIgYnueM1ajfcuaaYrDzMASMGlE6H1H1FZOtRzy6DqKW0kkc728gjeM4ZWwcEe+awfA8OtwW93Fq9zPcIGUwPOSW77hk8ntT5mFjtww9aCwHUgfjVWCZZVypBAJGR7U6U/Mv0Jo5hWLG4HoR+dLXnmr+K9dsNfurK30m2uLWNlEb7iGOVBOefUmum8P6leahZmW9sltJN5ARX3Ajjmq1FZG7mjNQSscIoJ5P9K5bXfHOneHdUWwvIb9naMSeZCgZMEkdSw54pcwWOwzxRms+w1CO/sobu3k8yCZA6MRg4NWZrgRQPK2AEUsSenAp3CxPRWLaeIbK9lEVteWs0hGdiSAnHrjNWH1WKKTy3eFZMZ2mQA4+lF0FjRopgfIB45opiIpO2K8/wDE/hSDXPEUtzJkOsaICCQcAZ/rXY65a3dzZAWV29tMrbg6DORg8GqWjWN2kbNfzmedjkuV259KzcWWmh+kWrWPh6G0di/kxsoJ9ATj9K0bTBQY9KwNYOv216Rpz2xtnA+WVCSD36V0FlG6wpuxuwM4o5WPmRPkeYw9hWZbptklDdfMb+ZqXVWu7ZVurSMTOgw8JON6+x7GsFvGdjG3+k6dfxS918oHP45GalxY0zoLt9tkozgkqKs2/KCuYs9WuPEF8hjspbayhywMow0jdOnbAzXVRLtQCqs0TdGfq93cWOh311aoj3EMMjxK4ypYAkA+2a5/wp4jvPEsd5Z6tYLbyogJ8osFdTx65H51oX3iPRopLiwvLvy3BKOpRv54xVeHX/DdgrzQXiMxGMKCWPsOKQzcsbdLJBbxcRxgKo9B2q3J99foazdIuZb23FzJGYzKdwU/wjsPyrRkIDr9DQI5HVfF2g6ZrVxZXlldG4jK7pVhBVsqCMHPoQK3tG1G11S0FzZBxESQA6bTx7VWutDiutQluXVTvI69eAB/StS1gS3QIowBV9BEs33o/Xd/Q1z+r6dpd7qWbu5hSfywBG7LuIycHBOfWugl6x/739DXE+LvC0Gu6us7llkWBUDKcHGWP9ago31/4lVnbWVrbSPEDt8wEbUHXJ5z9OK1UAmgKtghhgg1jaHbvaaDBayMXaFCmT6AnH6Vs25G0VSJZhW3h200/WkvIYVRmDKdq46j/wCtVDW/DNrquvrdzR7mWNF6kZAJ/wAa6y4wHiP+1/Q1Xxm6P0FNPUHsTxhhGo9qKnUfKKKokkK5FNEYHQU8UUhkbRBuopyoBT/SloAjZAwwapS6bDI2SgrQopptCtcpw2ccI+VQKtBcU6g9KGC0Mq+0e3vJC8kasT3IqlH4as45Q6wIGHfaM10I6GimpNC5UyvBAI1CgcCqOu6PFrFksEpYbW3qVbBBxj+ta/ekI4pDOAPglo2/dXt4n+7Liug0DS5dNikSW5uJyzZzM+4j6VtkD0pVptp9BK/cpatbT3OnvHbTvbzZBWReoxVDSLG7iiP265a4lz99lA49K3u1MUVNkVdnL6hDrcGpEWM8ItXI+V487fX610dvHtQZNPkAz0FPSnZCuzP1mae2svPt7c3EiMD5YYLn15qnpFzcaghnntDbNnaEZ9xx61tTgGM5FQW4AAwAKFFWuDlrYsjgUV8y/FjxT4h034mavaWOvapa20fk7IYLyREXMKE4UHA5JP40UBc//9k=" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAAgABoDASIAAhEBAxEB/8QAGAABAQEBAQAAAAAAAAAAAAAABwgEBgX/xAArEAABAgUDAwQBBQAAAAAAAAABAgMABAUGEQcSISIxURNBYXEUMpGhsdH/xAAVAQEBAAAAAAAAAAAAAAAAAAADAv/EABwRAAICAwEBAAAAAAAAAAAAAAABAhESITEDQf/aAAwDAQACEQMRAD8As68bqr1Pud5qRqbzDKEI2thXSfJ2x69bvqYbtUVamPoRNL2J5AV6atwzwe47xy2oDC1XDMKR+r0gRn3OP9/uC+1dR5S8dN6RcUooiWrP4b4Qe7K1rShbZHlKiUn5SYWqm18I24pooawrsn6/Tn3ao82t9t/aClAR0bQRwPkmOvC8gHI5+oJtNnFIMw3gY3A4hMSvKQee3mIjwRh7d8sly5FKUM5bTE4WJQ16f3FVLAeZxS5t1VXpIKcpTuVl1seClzCwB2Big9WNPbvuSuSlUtm852jtobDL7LLSFpX1Hq598ED37RhvjSmZrNEYmKFMpl65T8rlJhxOQCeFBQ9wR/OIXPz2m+hqE6VGrT1aQ+8BwSEnGfuEtL5CQMntBTpHaV8Uj8uYvSqSs084UpZEuzsCUjOc+Scj9vmE4yruThzj6g4Y10SV3w//2Q==" + } + ] + }, + { + "items": [ + { + "mime": "application/vnd.code.notebook.stdout", + "value": [ + "Box 5: [203,46,229,78,0,0.3125338554382324], Annotation: 'n'", + "" + ] } ] } @@ -287,28 +239,25 @@ { "language": "markdown", "source": [ - "### Print Annotations in Plain Text Format" + "## Display the OCR Results on the original image" ], "outputs": [] }, { "language": "typescript", "source": [ - "printSortedAnnotations(boxesWithAnnotations);\n" + "const annotatedImg = await putAnnotationsOnImage(\n img,\n boxesWithAnnotations,\n { threshold: 0.3, confLabels: false },\n);\nannotatedImg.display(display);\n" ], "outputs": [ { "items": [ { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "Sorted Annotations: [ 'robert', 'n', 'noyce', 'building', '2200', 'center' ]", - "" - ] + "mime": "image/jpeg", + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAIFArIDASIAAhEBAxEB/8QAHgAAAAYDAQEAAAAAAAAAAAAAAwQFBgcIAAECCQr/xABvEAABAwEGAgYGBQUJCQgOBgsBAgMRBAAFBhIhMQdBCBMiUWFxFDKBkaHwI7HB0eEJFUJS8RYXGDNicpOy0iRDgpKVoqPC4iUmNFNjZXODJzU2NzhHVFVWV3WFs8MZKERFSFhn09RGZnR2hJS08v/EABwBAAEFAQEBAAAAAAAAAAAAAAQBAgMFBgAHCP/EAEwRAAEDAgQDBAcEBQoFAwUBAAEAAgMEEQUSITETQVEGFGGRIjJCUnGBoRWisdEjksHS8BYzNERTYnKTo7IkQ2OC4QdU8Rclc4PC4v/aAAwDAQACEQMRAD8Au/X8P6atT1FQuncR1iXB2HEHsqChs53gcrV6xP8Ak2+B2J6mtrHVX/SvV7q33VUt75e2peckB1pyNRtta2z4+l/wrdoTpA2iw0GD0NI4mniDb9BZTuxGpnaBK8m3VUMv/wDJZ8Maa7qipY4i4nuqkp2luOv1dTSPNtNgEqUoltsBIEkkmBFo4/8Ao6MJ4noq5GBekzdF5sUCEu1SmqSlqPR0TopxTVYco0OpAnW3p1U0VLX0r1BX0rVTS1LamnmHmwtt1CgQpCkmQUkEgg6ETaqXDDhbhG+ukxxVw7U3HRJwzdf5vcFxs06G6J10tgtlbSQEqSmXCEEZSpQUQSkEDV2ankiZG0HOba6a2Jv8NFqezmF0GLUdbPWSOY6nYJPRAII4jGEWJBzHPpqB1KqXSfk88VYkdeYwFxswHiN2gg1LVO8+tTQOYJzhlDoTMEa8gbNnEHQk4rYYvJV312J8DKrHYSmnOIW6V5R5dipDROkaW9EeLWGcN9HTAmP+K/DS72rnvG9bvo7vRS0jDbVNTvdd1SX22wmEqHXFRGxKBpqZcvAbhRgyh4PXCb1w5d96V2IbsZvK96qup01D1Y/UNhxfWLWCVjtwJ5DzsO0TPm7sQM4GY72tewtz1t8vFGVGD4TDQHGmySGnc8RsFmh5eI2vkLtbAMLgBa+e/K115gK6FvSVpWl1X7hG6mTmQGL6u54KJM5iE1BJBBPLUGyPffCHiahxdyHh/e356uZXXXxT0d2qcXTNKTmQp0NBQSkgoIVMGRb0l6OFa9hji1xK4Fl1VVcOHalFfczb30ho2HSCphJOuQdY3A5Qo8zZDw3gC4cY9NDiZS3/AETFVdN3Xfd1Qq7nEAsPvejUwbLidlJTKzlOmbKd0iwct5xBLGLOzlpF+YDgdelxfZaXDcJjwd2LYdVSZoe6tma7LqQ6SBzHBpOj8ry217Aki5Gq8y2cOYlQpx7823oppJyrWujXCFSdCSI7vnSwSPSUy2uoPWdocySZ7j9tvTrpZ4aw/wAFRg/jXw+udi5L0ob8au+uTd6RTpraVxpxam3AmAdGSmd4X4CAek5e1z1/Gbh3wwvG7b4rrgrGjfd7Ul3Uz1XU1yB1nVtBtGZa0yysqSBsqT6sh81UYOIyUDM0tHgc23w536WVZRdim4uaSooJHcCdszjdoL2cAXkbYGziQW5NW3LgDaxK82i1eCEoLqFtjcqKOe/2WMoFWp1sspWsNp2CefLwI18fhb0O40YR4U4w4d3rdeDOA2MqC/g0HLudp+Hr1MS8kghClpYByKEggmNbD1/Ry4dv9G6u4g3xw2o7jxTT4Uqa6oZVQOUbrNY1TrJV1SVICe0nMApJGo0NmcSR2djbEtbmuL28R8Ug7L0sXdaicvZHLLwnMdl4gJALXN1s5pvroLEEa6FeZ9A1Xkrpsr+dLroWktqOsnw+Onh32Xbpp6hwNN1NO6kM00StJHb00jwE++1/uB/Qu4aY44QYexZflE47el8tKrXX2q6oYPaWoJTCVFBhIG6Dr7IZHSC6IWEuEWG8P31dN4Xo9U3vf9Lcr6na9DiCh1t1RUlAYSUq+jGuYjfs66BTNrBSmoADQWg3BNxey0WA4X2cg7TR4W975nsmcwhzW5HBoeL65r2IG4F7ggC1lSxxNUwtxx5h4IMJ1bieXlz+AsUceeyhLis0g5VZSDG2o93LmLegd6fk07uWVG7eIN9NJJkJdpaepj2hxqfdZoXr+TSxk2hKLpx8w9kAE1V0qbBAMwerdc8eRtfMilAs8a+C8lqn0rpSaW4Z/eIJ8wAPoqYKWorQS4ggGCDodjrHn38x7LdIU6pS0h5UCBzIIkEzz5fX42tLeP5O/jTSLKqe+cM1CRrlC65s95iaaB7zZsXj0GOkJdqHPRcM3RWZ5ILN80zafCA8pB5DcWcWO6IcEbqAzUVCaf8AjVAjXta5SeWu/L53GQ5VodSnrVEa+sNhH42lC8+irx/u3KHeFd7PhP6VGWqsaCdmFL5+Fm7WcHeLl0KWu9OGOMKVCExnduSqQk77kp+eVmFpTw66aiFrfUoEJgEgnKdu/wANB8Pd2HXAEAkZXBICTJGncDvIsM/dd6UKEt3jR1FK4pRSUvMls9oxpI+Y8LcoacU4nOAvLuDrroO7508rRkFSDdCpccJKQFQNVAAGI8jv4Wal61VSzi+iCCUqKEgnaBCtCfYNN9BvZ1oSsqLiUAx2QRGo9/j7TZn4gJbvyjdSkKChIUFbHOYj3Dbv8bDyi4UsZ1Tvoa1TtMPpAuHClPI6x3fhYQ1BSArMVCZ0O+hjf293xsVutRZoxUgEqzwnUCNI9/z32FWspShgIUes7gCfHbx+NmQizbKWQ66raqkeqFJAGpykg84j2ePK2OOrSnNvJAzAA677+zx87aSpClq7IOTWCDGvfy/A2DUpktlbqAc+wIjWdBr4a99piNFEhFuPlwqKzoBuAJ329/zzD61zNLjvVmSqPA9w8tfd4W26Ek9XK5UTmIJA5nSd9if226DOZJJnKOyBm0yzudxM/VaM7pwXBc/i86lIUSCkcx4jcxJPxtzmdUvMsFMDZREAd0d/nYXql5gkFSlbkjbb7xoLdtsOpUSkgjKBEADaZ1PdPhZhcNk5YhS0biY1OxA5HXccrBPqU9TrSgkSkwCoxrET7QDYyEOBScyUgqMZYVqCdRPlpvbtxhwtrDgKwpBVAJ9hEjXn8bQuNk8C4TIwzUvMLrUHMSotlQAiR2iQQPPQ89PKzx9FpnQl8MhAVKyCN08z46E+/wA7M+63VtV9cwls98BcQQe+Z2J8fboX9duZV3MHIcriBMg6nv7vn22gd6LrqVou2yTvzYy5kCmVAnlJ1OnKZ7vdbld1t65XnRMJ0UJM+fOfmLOFLSUJCiEJkAjWABHd7D9elulst5R1bWYkTMkCfGPdZOIkyJtm5/7yioUdBmBG33/brYA3E4SSstqBHZzp021j2b+XhZ1GmaUSkgjujcjv08vne2N0pWsZdTEzO2+o7vvJs0y5U4R3TXRhhTwC1MU61CAdSmR3nTTb5Nsew0vLCKMGFEwlfhufeB5We6KICMqRpI7oPMa24WwVBSYABiecRM/H7LRGZSCIKO3LgqDoikdV/KBEqEjST+z7Cj+Hn0qCnG6poBWuZBEGIA1+Ym0mrpykEqTkKDIzAaH79D8fCwL9E51CnAhRhJga6fVPP42aJSuEIKiVAeaegDNBKSIJGhmY8zt4DnsotUBfZQpYbSrkqd9dvPY++2r0Z6mpXKSQVmCDzJBnz0+HnZ6YTZp6q5G0usNqha21ApzHeQPjAsrn7FI1vJMeqwy22wt6pu+lcCZVCW0kmPA+33jabR1jG4i1Sqqqa6kshKhBaby6a9wiDI8fZafb/paIoVSNMJCQZWUggTPq6e37fCOMV0QFz1KW09lCwonftSPv8/K1lQyua4G6DqWCxsoQLTzZlKFiO4EW2KusRID7ok5iMx37/qs5HKck+uv6QnVQ5ySDp87HaxX0YLQUDtaEiUyIESdfZ7raMSg7qoLLc0imtrNFF5RywBOsW0Kt2e0AeeospqpEZA4UJMg6BI310/Dytx6IlSJUxlG5Ow32j519lnB7U2x6pNNSsjUCTzt16UcmQiQBA1+fH32MqpG8sqSpMCdNdNvn5FgHqdCGsyM2+s2cC0pPSWOOOwlxebtCASeVuPSne12tFbjvsO99JSUwUT2UqQPMKJ/1hYqGVEEyNBPOyixXG67FSUnMEiedu/TVkRlAmdjFgC2scvG2sqiYCTZbBJmKMtVLaVArR2ZGm9jKrybKerbSrLtmJgxPhZOLbgGqFbd1uYPdZMoK7MUd9JbUgI5jvGkydxbQeTqCRMjWYsT1i2a8hbsoXXQ1Qc0Kka91gpmLatlnDRIShaaodpqlqpaWUraWlaSNwQZBt9JfDfFJxdw7wviptYUm+Lmoq+Rz61hK/tt8143EW98+ghiZWLOiLw1vJbnWOU91Ku9Z7vRnnGAP8VsWzPaWO8ccnQkef/wrHDrFxBU/KqXI3sEah3cG3J3tpWtsmCrgNb0Wde4T61s69zUz7rcxztqCbTBPyhdMPuJfcQVaEJV9Y+yxjrnZ9Y2JpJTUIO2ZCh7iLDmZtoaM5ogh5QAUN1zp1KjbQfd/WtxMiLYZPOxQCiJXXXuDXNbOvd5K2sHztm3OzgLpN0L6Q6dMxsn3i84H6V4rOiyg68iLHPd7LEr1Ss0pW2RKFJXPkbc4aJW6Fdde9Hr25NQ8P04i2gZTIG9uFD22QjRLsuxUukRPnYdNS6UTPIcrFI0MjWwrAlAHssO5tlI3VDGpeiCr42QcV3hetHh+9Lwbvc0RpaV2oStFPnKAhJVqDObbkBOtld5fVpkIKlKMJSOZ+wfdZBxLhlvEVyV913tVPuIrmVtAIOVDBI0KUiMxBAPanUctrQuYZPRCeSGi6r7W9Ju4EXTd993hcj1dV1wZfSwthhK6UOSC7nDqgkgaltQQsbKIOlm7xMxPgi8E1qKjEd+Lv1biaR2jSFuGjOY9jqEw22pXrFDq0KCTJk7tXEPD658S4Yv3CbV6O0+IMFPuUfpTvVqUplKQVyQoAhZOaTqCMuuqbVrZxUqlpKDDz19MO3ndFblZCG0KaYShpasiitJC0pXqCVFIVETMWrqzDxLA5pNtD5qvdWyMcARcFTwMT4DSAlynxPnAhUXhTIE84SmkIHkCQO872yzcZ6QdS2yhuqpL3W8lIDikpSQVRqQUrAInuAHgNrZbzg4bW39X73/hWPFh6r00fH0unebdtgKEW4fJ66Invt2g6b/tt9IO3WRYLBaq6lNDRP1q2Xnk07SnS2y2VuLCQTlSkaqUdgBqTas3CvE15XNx94iYvvLhvj9i68XOXc1dry8L1iRKEZFFyUfRpBO50jW1lLwvm6blbQ7e960lChwlKFVL6WwojkCoibE044wV/wCl9yf5Qa/tWrKuETSRuzhpYb/Qjr0JWhwbtBT4PTVdLNFn7wwMJz5bAPa/TQ65mN+VxzuE/ixw9o+KXDu/MBVj4YTetNkaeKcwaeSoLaWRuQlaUkjmARaPeGWOsY8OcB3XgPH3C7Fr984epUXay9c93mupK9ppOVpaHkHKiUBMhzJB38JVGOMFEj/fhcgnl+cGv7VuhjnBX/phcn+UGf7VulijdMJ2Pyutbkbi9/py+KZRdpI4MPOFVTGyQZ+IBmLS19spII5OFg4EG9haxF1FnAvhxifDN9Y34z8Q7uNLf+MHzUJuqnIfcoaNuS2zKJzukZQQJ9RPMkCO8HXjxGp+k1jri5cnC7Fj+F7zoKOnep6y63aGreQllhBcYRUBCXFoW0ZRmBKFEiSADZgY5wTzxhcn+UGv7VtjHWCv/TC5P8oM/wBqwzqCLJGxklgx2blcuN9Sfmb6K6h7fnvFZU1MLHuqIxDa5DGRtLLNaAb6CNoac1xa5udVAPFG6MXdKDEGGMF03D/EOHsGXLeab1vq8b/o/Q11JQkpDLDROdUpW4M0R2p2HadnHfhhiuuxpgzjVw8u5u8r8wa8puousuJaVXUK5C0NqVCQtIU5AOhznmADKf7usE/+mFyD/wB4M/2rb/dzgmf+7C5P8oM/2rKaGCQPMj7ucQb6C2X1bfD587qJnbyamkpm0bGMhgbI0RklwcJQRLnJIJLwbG2WwAy2Iuo+xHxZxtiK4Km5eGnDLF1Liaua9HYqL5uv0Kku1xYgvuuuyhwN+tlb6zMQBsbC8Zai+rk4FXphOro8Q4svy9sP1V0Couu6HKlb9WulUnrHEMIhpKlneAkTFn4Mc4JGpxhcn+UGf7VuhjrBOv8AvwuPzF4Nf2rSPiztcHS3LhblYfLr4oCHtFQ000DoKdrWRvz+tdziLWBedQ0W0aANze5sRG/Rdvyqa4Y4fwJeuEcU3Pedx3WhFUb1uSoo2SrMRlbcdSAs6zA5WA6W+BMT464b3YcI3S9eldh/EFHfXoTBHWvtNJcQtKATqoB2Y3OUxJtJ/wC7nBWv+/G5P8oM/wBq2/3dYJE/78bk/wAoNf2rNNPE+k7o99xa19BtspW9rW0/aD+UFKxrXZzJlLswuSS4X0Njc+IHPmmvhTiBjLGuOG2rvwTedz4PprtWurqr9u1yjq3a4rAQ0yhagciU5ipRQQTABtI3LXnZC/d1gnb92Nyf5RZ/tW1+7rBR3xjccf8AtBr+1YmJzWD033Py+ipK/EKWqkBgY2NoAAANyfFxO5N97DkANEvGAdNLbPaHwsgnHWCf/TC4/wDKDP8Aatv93WCeeMbk/wAoNf2rScWP3gguNH7w80sLpqVzRynaX/OSDYI3Xdm/5vpxA3S2AfeLJn7usER/3YXIOU/nFn+1bP3d4JA/7sbj2/8AODP9q3cSPqF3Hj94eaPu3LdryOqdpyUfqlao37ps3rx4S8N73UV3ngq5Ksq3VUXey8T7VoNlH93WCYj92Vyf5QZ/tWz93OCh/wDvjchH/tBn+1ZpdD1CUVLB7Y80yby6MXBG8pDvDbDSJk/R3NSt+2UNpPM8+dmbfXQR6N99uoqKrh7RpcbgJWzVVjMd2iHwJ1PK00fu6wSD/wB2Fya/84M/2rKVDel3XrTiruyup6xjNlDtO6lxBI3GYEibM4cLzYWKlZVXNmuVZaj8nnwO7Yu5q+KAHUJp7zXlBjf6VLnzFoZ49dCWh4aXArFeC6DEV+UFHT1FReTi78o2hQtNpSQspUwlTsjNojXs+It6EgiJm0ddIszwIx4P+YKz/wCGbDVlO1tO9zPRIBIIty15ghX/AGbkjmxemiqoxJG97Wlri4AhxDd2Oa64vcWcNQL3FwfOqj6LGL8WYLpMXcOqK+cRUlRSIqK9xqkYaZp1lIUtlpS6gLqVI9U5GwAoFPrAgKuAOhjifizg+hxtgfGV0P0SlraLVbTVLLjTqdFIX1aXRI89QUm1/OjWwzTcBMCNMNhCVXJTOEDTtKTmUfMkk+20edCAJbwXjphAhtrHV4pQkGAkdVT7d1q2CN/EhEjriRpJ0GhAB0t8ed1sMWpMMkosTdR04idSTNa1wc8lzHOkaQ8OcRcZQQWhulwb7qpd69ArjpQIfXTN4er1kaBiucbKu4/TNtj4izUvDof8d7qQnrMAsOstkqWWL0ondBO6Q7OuvLlb1tmDbhaG1phxKVDbtC1m6haea86bVu5heO9RwC40U7SqpfCfFLre6TTXY89oBM9hJEE2Qbw4eY1ufI3fGEr+oSDKjU3c+yPEArSPDW3syzdt3KYSldBTEARHVJ+62/zLdREijSkncoUpH1EWgOGki4KlbWgbheJ6mAl5wvLgt7Aq8NTrr5Ww0zhZJQ6VOLkJ0Go92nLW1/8Apl8LcKXbfWC8c4audDWK78xVQ0DtbUPvutupCIbSptS8uUFCJygEgb6276RPRT4b4a4S39j65lXkrE1ztIrlV9VVKdTUkOJ6wLZP0YBBVCUpAGgiNLUssdQx0oDQeHqddxa+mm9uW3ivRaHs5hlVDh8rqxzTWEsaDEDleHBpuRJ6mY+v61teHvbzCLC6e+nmRqpaFZSTuZHL5+Fn/h8LcuemSpRCUZiDqkqM7bxt7rXy4hcJOAt29GtrjbXcKcOrvWrw/Q1jSG6EMJFVVIbSkfQlHZC3ZMcgbHOEXQm4X31wqw3e+IaRTt53xdrN4vupdeaU2X0BxKEhDmQBIUB6kyDNlfBLJIIohclube2h281WRYPHTUUmIYlIY42ymEZW53Oe0XdYFzBlaLXN9yABvajSQeqzlrQAAdmATyMefwjwsIih60JZJVr2io+QmRuNxztZjhn0X8IYp4q8QOE1+XjenpGE3kuUD7FU22HqVw6BwKZXKgFN6ggGTpoLEcLdFmjb434n4P3jf1RQ11HSovK5qj0dD3plEqM260DMnMAYIkpXpAtXlszsjmt0cS3fYi+n0VzJ2Ygp21jJZ7yQRtlaGsuJInhpDwS4EWD2lwykgXOtioDTdtGpwlSEAAEDz5kiwzN2soACkjrHDMpVJ9nePPvtYW7OhzX4mx3iO67hxE1UUuFlsUj9a/RZEu1y2+tW3CHF/wAWhTUmTquI0syMJcA8W8S8YYyua5aihdRg6sF3O5+vS289ncQSMraiRLKvWjcWhkbUAgFu5IGu9t/wXR9nqR7HzcchkUTJJSWC7eIWhjGgPOYuzDcssN7EEKMjRtuOnq8yQ0kAqUfW3Mgxy52DVQukR1pClyRmTHlpI+Fpqruh7xfu+spn7vuGldYQCHwm82WwrTQhK1JO/h3WifFFyVmHsZVuFsSti7XrsbT17C30arUlKgMyVEK0UDoeXstBKZoBeVttbeHmlw7s5DjVRw8NmMjGxmV9mem0NdlLcgcbuJy5QHWIcCSNbJDl3KK1JRkIAKlKykAeA75Enyixd2leNMQQ32pGi4J3Ebc4PvsZvV+6KHqn7mqwVJKetbQ4YWOY7p8tLK9Xdb9IlC326hsZQW0qTBVPMSPHeLNjmMhLRyUWN9m5sGp4Kt2bJLmAzsLHAttcFpJ5EEEEgg8iCFBeKaZxl9SVIRmzBSilUwI1J7j+FlzBt4rZuZ5sErWl45dIEEDXx23sHjOncaTUpLQ650ZlEEEwCNO6I+uyfgwl6hqGzMZkkAmY0M7+IG9rJjS5qyLjlfolioUVOFZEqV+sfGd/KRp3xZoYop23KGtS6JCW5zE+07ctN+46c4eC0LKCqRmJmY3ju74s3L+a6xqpR1ZyrZUBrmkZfPX8dxY2n9EiyGmFwokebMEoSVlIJ9bNOu8e/wAZO9k4hSgWshyqEk+I+uIPn7rKhUEhCR1gkciCDPPw2gyO/uFla97hw+rBzWJ7pF4IK6wUxbqXG1DKAok9lI7ha4fO2EtDr+kbfNJQYPPikU8sBb+hYXuBNiWjcjSxtcX1G+iaTvZzE8gNTPw9g/CwbqUjKVhAUND2t/nSPKwhWmVKbWCr1Va+sOZHhp8LAKUuU50EJVsNCYk8uW/xsYAqcrS8hAT1Wg7XkOQ9/wA9xaqALTgTEggkk8/A/POwy8wSCEqO8iZ25zYJztIcCsw011nX5i0oTbckGlU3U2cmjVQrtfzkjT/Nt3Da8qCArMDqDrPLfyHxsq3xQ4dprraTcN6P1pWyy/WZ2svUu9oKSNBpr4+Zmyte2G8OHBDGLLlN5oW5XeiFqqebWMoSokgoQNdBYd1WxuQkEZjYaW18eYV/F2cqpzUMjewmBmd1nh126XLS27Ta4BF9CbdbNAtsxBG5gk6GdvgbZlSghKokdxiIthWSZUInxgRET9VnTgC4MO4pvZu472F4oqKnOUOU7qEpSlKCe0FIJk5eRtJUTtpojK+9hqbdEFg+FTY3XRYfTECSQhrcxsC4mwF7Hc6DkmqUBU8xpGuvPfvtpTSUkpkzMAkfXY8KemRfJoaxxxujFT1broTmWhsLgqgbkDXxsJiGnuajvh9i4a52roUkdS86kpUsEAncDnOsCYs4SguDLHUX205c+vgoXUL2wPqC5tmuDSMwzXIJuG3uWi2rrWBIG5SXlAJ0BIEertblTWsKA12k2EzbnLmmCAVaQPrtpUT2ANe4x8i0t0DZALQlI0387B+dhlp7M62CAOo8LOBTCFob29mPySuKVX30Xqm43Hcy8PYjrKVKf1W3ENPD/OcXbxnG4t6dfkZ8TkscTcGuO6BV3Xm0if8Apm1mP6P4Wp8dZnoyehB/Z+1GUDrTW6r0vVI3NucwG1tr2sGfPa2HC0LRcITfa3MRbEHYRbsAHa04F0myDUMq2lGBCo94iwx0sBUfxRP6pCh7DYxtp32vKA/o7IeYc1mlt7crYJ520QCdvCbHgKCywxFtDU23HIi2jPM2W1lxWyTt9dgqhAcZcRIhSSIsKJOgGttKEjTxs8NuEl7JPpzmZQSZ7NtqkmYsFSuoX1zCFJKmXVIUJ9XmAfYfqsMZGnfZqdey5GulhGP0gdTobBjc91ugopJyASQYkwCe75FoZLAXKdGTeyHS2txYQlMqOgsXeqrqQ0+h+vaT1YKj2wSlQGug1nwsqUoutlr0m8qmnAUkkda4AkDyJi0WYj4m3AjG7uBDeVRTuPMCqQ2wQjO2NFhwgKcAAObIlM/rlIBtXS4gKRuckA8r/t6JxHFdlCqv0mHLyu3i9+7XAlfSU7T12+i3g7VMOUja3SVNlpbriUpkozQQcwgiNALVuw9wjyXm1j/HGNLow5ct4vLTQsuPBv8AOBKiVFIAKyiDBUhJTmj1gDF4+L/EK5au42rmvPD9XeVI+UP01AxdqqenpRPYWXXWupUtAhUZwiQUybUjxDwexbjYGvxNiynpOvfW8mvvJwUgLCRAAClAOEJ0OQEaEawTajqMUdM0tdIGA7m2vy3UT6drH5g3N4KwNFd3Rhp6NhhuopXUNtJQlxdM6VLAAAJIqkiT/NHkNrZaGqTCuGaWlZpX8YXG44y2lta+oq+0oCCf4sfVbLZI0Dr/ANJf/HyRXF/uBesLwPWanmbdpBjnpbToh3mJnlbtAAPxt9DO3WSbsqs9O6hp70u7h3dlWFFirxD1DuUwci0pSYPLQmypUdCTgKyHnDQX2ENAkxeKtgJ5ixPpvE/9jI//AMTo/wBS1g65JcaqmicoWlSCY2lMfbaigoKasxGpNQwOtktcX9lZXuNNWYlUmojDrZLXF/ZVHOHNx9BnihflFhq4avFdFed4qDdLT3iXWQ4siQjOnMgKOwBI1030tODPQg4DE5XbtvmeUXkr7rGsL9F/CFx3rdl9KdYqHbufaqm3UtZVFbagpJ3PNI52ev74Tddj5WEaGkrkfm6mL1W9UUrjCFLWoZEt5wM4hKzmTKdoJgxZ/YWGn/kN8gjPsXDb6QN8gmYjoLdH9aQfzdfev/OavusXf6EPR/bVkbuq+lkesTeigB8LWCp3CGZ572jnjnSVl7cJ8Q3RQ1DjLt5stURW2qFdW88hDgBGolClDyNoxgWHX/mW+QTzguG/2DfIKN6Loa9HSvSXaOkvKqaSrIpdNfXWBKu45QdfCbKjfQZ6PSwD+br8E/8AOavus0OjhwWrOFGOE3jddW61d950rtPV0oWotKISVoVB/SCk7/yiOZtZ0VqWq30UkaoCx7ZH2WV2A4dyhb5Bc3BcN/sGeQUKudBXo9NILi6C/Akb/wC6Sz9lgh0H+jkYJpL7APfeLn9m091bpNMBO6kj4i1culRjDjVhG8MKOcJL8Yo0OtVzl4Uz1C0+mqKCx1QJWklIEuDskTm12swYFh5/5LfIJ/2Lht9KdnkEsN9BXo7uiW6O+VeV6K0+FhP4B3R9/wDN19n/AN5q+60mYQvyursKXHiC+2EUtbX0NM/VNAFIbdcQnMkA6jtKiLOxh/MkkHQizDgVAP8Akt8guGDYYf6uz9UKB/4B3R+H/wB235/lNX3WwdA3o/E/9rr8/wApq+60lY4xO9hTCN/4tprqdvN66aKprk0TTnVrqS0lSg2lUGCcsTB32tBuG+mU9fTjbLnBrGbDioSU06w8UkmP1U277Cw8bxN8gnDBMNdtTt/VCdP8A3o+je7r8/ymr7rYOgb0fY/7XX3/AJTV91p3o65dS0h3tJBSCQrexOqvR5tdS715bap5mEzoEhRO087IMDoDtC3yCU4JhY3p2eQUJ/wDuj7/AObb7/ymr7rbPQN6Pn/m6+x/7zV91l/APSm4OcS72YuPBXEajvC8KmSzSu0T9Mt2BmIT1qEAmATlBmAdLS1TXkp0AuIg8wNrIcDoR/yW+QXDBcKP/Ib+qFAv8A7o+f8Am6+/8pq+62HoHdH3/wA3X3/lNX3Wm698R0l1OMNPKbSajNk6x0IBgCQJ3Otg0YnacAUmmUoHmlwKHs77J9iUH9i3yTvsPC/7BnkFAWJ+g7wFurDV7XpSUF8h+joah9oqvJRGdDalCRGokCyn0DTHR+pNP/vSt/rC0tYtvxiswhiBhLDqT+aqsyrLH8UrxtEfQOI/g/0g/wCdKz+sLAto4KTEoxAwNux17C3NqBZQ01Fi8QpmBt2PvYWvq1WLCu+zG4wYCxNxKwnVYPuLGVNh+kvKnepLxU7dXpqnmXExlR9K31ZGuuu+1nvMcrVtxz007nwbjq/MCtcNMQXvUXFUdQ+9RrSpKtB2ogkDXnY+vmpoYstU6zXac9fLVaVmOt7Ozx12cNe0+iS0PGYaj0SHA23Fxodd1JGBOG/ELAnDcYEpeJt31NTQU9PS3RXqw9lTSMtwCHGvSD1xUkETnTEzBslcDeCOLODT95073Eqkvq6b3r6i9KmkFw+jOGrdCAVh3r1wkBA7OX22jlPTobWMyeBWNSO8N/7Nt/w5e7gRjaP+i/2bVba3CmuY4PN2Cw9fT8/miJP/AFMM0VTC6dmWoOaT9CwZiNjcR3bbcZSLEk7kk2lzd9thXeffat2CumfdeLsd3HgOq4YYiuWqv2oDDLtcpKEp0PaggEjTlaxsE8/O1vS1kNY0ugdcDTn+1VtFX09e0vpnZgDY7jXfmsaPZInmR8bCZ+c2LtfppOvbV9dhAQRv77EBGFVc6ZWPsHMXvw+w+/iGjReNzYvu+8q+mK4XT0sE9aockwRrZ39JvidgGp6Pt8rYxXQrTiq7Hk3MQv8A4cULRmDffEj32nYnTytiTqQI31tWvoZHGY5x+kFvV20t72unw18lt6btVRU8eGsNM8mjcX/zoGclwfa3COUZgObvRuNzmFdLhue4OPvRDpOHWFr/AKSqvClw5dtG5kXpTXgwy04lpwxp22wk7wCbKvCbjnhTCHDa48I8T36vDOJsOUDV1VV211I6HnlMICEuMZUkPhaUhQKM0k2nU/GbbkDYWVtC+NzZWPGYNynTQgbaX0I+PNNqO1NNWwzUVTA4wPlMzQJAHsc4WcM/DIc1wtpkB0FjveBOjlgvElZj7H3HbE9y1NzjGVUlq6aCrQW6hFC2ey46k6oKwlvsnUZSdiLJvS6ob4wTeGD+kThKiD96YOrk0VazJHpNFUHJkURrGdRQI2L5MGLWFvBVYLuqjdoQasMuejhfql3Kcs+ExaHGLo4v8U3sKXRxNwXSYduy4qxm9L6cReLVQm96pgSyhptskoa67K6Qsz2Epg6mwdRSCKm7qy5cTcG3tXvcnYa6/DRXGEY/JWY79v1RjbDG0RuiLwLwiIx8NrScz7sGS4v6ZDjYah88GcI1uCuH13UF9KC78ry5et9O83K+pUXXj5BSso8Ei0OYQpl9G/jlj2uxXS1bOC+ID6L1or6Qwt2mpaoLcWth8oB6qS8uFK7JCU67xZbmbYD42LkomlsfDNjHsd+VjfrcLOUnaiaOWsNWzPHVi0jQcvth7S02OUtcBa4ItcW6Rjd3GCtx3jy57k4V0rF7YaaQ6/iG/nqZ4U7acv0LNM5KUuOqV60ZkpT47R7iu4q3g50m6jjXWXRW12DsYXULtveppKZVQbsqEhoJccbQCoNkMI7QBjMueU2PB526Cte+zZaJ0zQXu9IODgbaAjS1r7W315nVT0HaWHDZnNpae0L4nRPaXXc9rjmLi/KBnDrFpDAAGtBadbwzibjBdeKE0OH+A9Jd2JcQV9W025WG73HrvuymCwXnqhwZACEyEoC8xURpyMvOXRdzyAhxgkRqAtSQfYDYwNtBbYI9tpY4S0l0hvfwsB8tfnqqyurKaZjIqSLI1tzcuzPcTbdwa0WFhlAaLanUlNq9OFPDe+ypV8YLuWtKxCjVUDL+Ydx6xJmzTqeir0favOf3q8OMlcZvR7tZYJA21aSk6TpaUh322DBs8wRnkq7iPHNQPenQb6O15ZlJwX6KsiM1PX1iT7g9Hwsy77/Jt8Dr1KnKa8sSUJVyYvFEDfbrGXDzPO1rsx0kyBbYVpofObM7rFyCdxn2sSqB3x+SL4ev5hcnEvE1ICmE+kCmqo8wG2iffztDNZ0Ar0b4pDo7XPjBq8V0dGcRqr6yhLSAgykIU226oqAUtI0WD7tfWQKCQVHSNzaptfxFwxhjpw1V/wBfedObmqMIN0VTeLbgXT0mZ1BS46saIRnShBUdAXEzA1tXYoxjBFmNhnH7Vvv/AE+ZO+WvNPHncKaSwte5uy2h0J6Ag3OljsqVcafyf2PuFWCKrEuLqzCtTRofRTU9VczVSxUMrXOVTiVthBTIAMLJ1EA7hoVnQp4zcH7kRivH+GsMV+FnFJUK1l5qoXLoAbgOJS5lOmgTprNvRbpg42uTiPhC6eCvDq9aHEOJsU3rTZKagqEvinYQStTzqkSEJkJ1P6OY7JNnD0vsJV6+i3eFy3Sy5Vm427vcWECVFlhaErXHcEyo9wBPKwNQDkqY4jdoaCOdnWN7fKxWswUtbU4DV4i0MqJJ3MddoaXw5ow0vFhcZi9ocRqBa5DRbyVT0OekzgyvVe99cFKipu4DNUOD0SvabpwQpThDa15YSN40E2auKuCPFhV/O4qwxwaxNXXC6tLtHU0uHKh+gdSlISogobLakhQUDB3Bm3tpjri3gO9eBF5XrcF+Xbe1Xe+HX27voqd9Dj7r66ZXY6sSoZNSuR2AlRVEGyZ0QKnC6ujnhT0qouzrqSlqVVWdbedpJq3oK9ZSIjfwsY2Zrq4WIILDfpu39iy8lBV0vZOV0sbmuZVMDbjUehLe2lwM1vC46heIPEW4X6W5rneXhxF11btA4a5tqg9HyvBTfZWkAZVDtaHUa2IVAI4J0o2P56O/8xdvaTo2OYYruLnG+nqa+kWmpxMg0raavKH0/TzkAV2x77N5eFOEmIembe10X1d2GbwuBzBqHUtVbVPUUyqnr2hMOBSCuCrx1NhI5/8Ah4AeUh5+Llpa2kf9t4qWtIvRN5czFASdOZNyfG68ecL3HdV1YPrsdX7d7dctDopaKldkNqWSO2sDca7eB7wQrcL77ob5xtSmpuSioapCHlsOULXVJV2CFIWicp0MggTIjnp6kcZOj3wT4f8AH3D9Tjjh5cdNwpxPRGjdCKFtigoLyAVClpaSlLeaEHNp6yzPZJszukr0duiFw3we5iLg9hy6F4rS404hy57yfqU0dKtYQt51IfW2hCswbTmTqpwZdiQyrna+CqM5Of0gNdhbSw6Hn1RPZ6lnpsW7Px4XEDTvET3kMBzScQ8QufYnMwgAAkZQG2GuvlnSXgu7cSvuooaOoC6hbRRVMJeQAXNwFCJ038TZf4kIprg4hOKu256HqKYMOJpTTjqFHICQpCY0JJnvt6nVn5NDooXphFnGNLfF4ULqqdL63ae/0dWakt9YWz1qHBMmcsgxaJOG35Pfh70hsDNcTLw4lV1zXtUB8VNO25TONsNsrLaVqbWEqCYAJJXEnlY51TH3uMnYscD46tt+1Y+KKpb2arXAelHVQlpt6t2zk2uOZDbjY2F1SXFz9wXfcmG8SIwtdHptfTKcNKin6qnmElS1JSQVRMAE8zNkvEVDcmIMBsYxo7npLsrGK30SpRRpKWXAUgghHIiU/HU6WuJdHQBPF6+77wHcPFSkpXcBVRumnceom3fzkk5ocATUJIjqphOca7iNWtiDoPYyuG9XOAtBjrCVbelUj8+IrXXnWaVKAotlpZShag7LZMQRqNbAwTRxwxSNcc2cjc+rmIt0tZbDFWzV2KYjQSQtMfdGyWDG34ghicH3AzB2a+x2uNrqr2GLjua68H1uO78oEXgtDwpaKleJ6tTmkqWBuBJ0n9E+EHuHFdcOKsXMUV94WulDoQ64z6PTBLa+wZQ42ZSREkGJBTaeOKPQy428LOH9DhHFt3XUl2uvAGhrWa6KJS5HYceeCOqOqiCsJTA30MR/hTo3cWuHOLE37i24KNqhoKZbzz9De9HeCUJchpBWaV1zqwpSwAVQCdBNpKieN8FU+R9njMG62sLaWHQ/XVR4HSTQ4pgFPQQB1LIInSHI1wc8yESl7iD6TNAAT6IDbDXWDMTXgq8L2dV6DQ0oYUplKKSmQykpCjBISACdd/AWuV+SNxGq6uknedxKdyt33hmqaCSfWW06y6Pgldq6Yl6NPSFu2rfrqrgfjpNK84p1uoRcFU40pCjKVBaUFMEEHe0l9A1vE3DnpecPqm/7ivO60Vda9djnpdI4zPpDDjSQcwH6Sk2uqprHULmN91eNVMksuIvllJJLjqfivchRkEC3HO2yrUxbQ8bYMDVXAWeywiDMgW4GvhbYGtp2Liu3EZ21IP6QIttpWdtCjzSLYk5hB3txTK+iUjTsLUny1kD3EWt6A2JCGl0CFHstuLaEHfS2xMaWtQEPdZzmbaGWBytsnTe2R7YsoSLIIEgzblRG1t6T9pthnYWlCaTqk1CQitqEJSBnheg35W7VvqNTbmqBFc2sH10FJHxt2qVCZmzLLsy5G4IG3Kx+6ruFW+X1tpyta7bq5CxFsdY+3TI1ccMJTv7fIWdvUeg0JbpU5lJRKdPWV3n22Hka06nkkfIWaDcpt3xQ3Ve7qziR5hm52VJbDbjvVpfeUYhZkSASEhM6qJkGBaMuLXD3hxdlwV9fT05w2mphp28aeudoHHgogqSMiVuPKMJ7GQ5tdbTS8GKTItbWcNkFAy5lZoiR46n3myPifCVFjK7Td+KqRD9A6tLhogshKinUBwpIzCYOX1dNZtS1VFHIx0b2hznc7bfPlbklZKWuBB0VSmlPXtThq7r0x5f13UjfWJXWMuPtKUlKyQy0ptxSFQ2YByLy9ketBrzih3CWIcbU9wVeNsVu1F/VyqZ2gqaPq1sNkpzuZE5glAyz2koACZCXIMXz4m33U1l10l1YBoFIYoX6dKKimYCmlpCi2tlkJICYSCCpRSjZIVMlNQ+N+M2g1fdVgm+Lpuy+H7qDFe3dRbdqWVtocYCHKhBkAdahTeTKFDrNJFsRU0XdpsrXFw68gfI7K3bZ8WY6eCZdexgalrqimp+iXxIU006tCCp2vbUUgkCUJpyEmOQJA2BNstFf5g4yq7V21V8ikOtOHXkhYb/RzAqEGInQWy0n2fBzkP67lX8c+79AvY58Q9GtugNI+NtvoHWjvNsSNJO9vdnHVZluyq/03Y/7GX/9To/1bTtioKXcF8tgxnpKgb/8mbQX030/97LnOKEf6lnjxJPGtjFFRQ4aq7jdw7eGVvq6hhQfZQpIS52huZzEd0idrVlA5rcQqsxt6n+1UNIxzsSqgB7n+1V84H8CMX8NcY3Bf1yYvvIUDdQy3V0aqglp5gkJWlQ2PZOkzECNRIt1ezbBvKhdW2CvK4lJjUDs6eVi7GGaujuFxyhS2quaZUphKtElwCUgnukCzDwXjTifizEjd1Yu4d010U9Iha/TGq0LBWIGUIJkg7zOmW16ZGaXcrURv6KaaUxTb7J52b+M2/SMLwntAu0yjPd1idbOWkYUljIoco08rRViPindfDyvew5xCu2vau5wqNJebFOp5hTRM5HAntJUnbQGQAdLdG4E6JHtICdtzMIadplpSIEjbbsKt3VVGfE5QmIRStzrzzKP1Wjt3pJcDrqpevZxc5WOAQ1T09E+XF/yRmQBPmR52cHDWuvXFyXsW193u0KbzdLrNM967TAAS2FT+llSCfEmyyPF9V0cbgFIj5IpWyR/fEcv5QtCPSJ6QaeCF44cp3eHL2Jqa+G6px19moCFUYZU1HZLas2brDzEZec6TNiC8KK5LndvS9KxqlpKYoU686oIQ2nMO0onQDW0Z4pc4WcRaqkqazG9wPCjbcbaSmtYcBzkT+lp6oswG+ycR1SrTvUHFDD+G+INzXveTFGtlN409GVBDTwUg6PJAkqTJjtQCNuYf11uq6lIVMxr32jJ/GuBsG3VdWA8OXnS1tW+pFHSU1K+hxSEEytxYSewkJzHXcwB4SRdAWqnQpQiR7bId12qRsVVdxXfhS+7wxS8pm56WkqnrwdKVKy0yUqU4qEgqMIBMAE6aC0U4X4s9GCqeZbw3jih655SUNIcZfQVKJAAGdsakkaTaW8Q3c3fmG74uNTYc9OpqmmyKiFZwpMGdOfO0ZYS6PuE7ir2rxew5QpeaUlSVdUhRBBBBkeNopNdxdSxac7KaqE/RgIAyxuLJzozGvbVspwgnzbSLKdIlDbQQFaARZNUgF28Ez6yjy/5NNns2smP3UGYT6MWGsLYoubFDQp2qu6ahFShxlBCllMmNRsZg+FpaubGFLe991l3ULNUhN3rDLqqimWxmcgE5UrAJSAU9qIM6EjW1UeF3B/HvDbGlwX1dGN73dudNY23VUD1UtbTjKyEqSRMHQzrMeYBtbspaNawsphZSoBUcpGnsk25wyhK3XZIvFvhZcHFG5WKG/qMPqpFKcp15yktqUIJERuBFkPhZw/pMDYArcIUilpaaqKwJJUSQHBm3Jme0bSwwMzIBJOlkNKUpfvFsDd4n3tp+6yDUG5S3sQLKvnDHhPeeAbixFeVTia+rwS5cdUwhqsq1ONtgtkkhJ59ka+Js5+geQOANKDH/bSs/rC0hXiw2nAt9qSkf9qanlr/ABKrR50EUzwBpD/zpWa/4QtQzAjE4rm/oP8AxaqyexxiG3uP/FisWVWpThmVdJDjKATHpNOSBz7Y0tdUneNLUvwinP0kuM86w8yf88WWv1qKb/Gf9jkmLH/iqT/8n/8ADlOGBcR/m9x27XFK6p/toE+quNfePqsm4YvqoGNE1qn3VIXVPFQzGDOYbbbm0XVmMK65uNYwZUyilrsPsXldrh/8oafeTUI88hZV7FWeODnahVY07Utqbc9Of0Pd1ysp9qYjwtZ7MAKvBq64SPxve67pYcGFSd3f65taoGdB+Nql8YlFXSp4MEmf4z+ubWyBgWqaD+kVP+Mf7Wqkwv8ApVX/AIx/sauWvWcnkvn5CyVjRxbODb9daUULRdlUpKkmCCGlQQbKjZAW4NZzAx7BZHxyf95WIP8A2XV//CVaxk9R3wKt5j+jd8CqE8IuE148ROH9FjC8OKWLaepqqmoZWwxWqyIS2QEmSSSTr7rSlT9EujeuQ327xyxu2httSnUB/MUFO6Zza/iLEOi4D+8rdZ5GurRz5LH32k2sxMwrC+Im6KqCxTMvtPJCh9G82jUEcjEHxBFszhuG0cjITMy+ZoJNzuR8Vn8G7O4fV4RFUvju8gEm7tfqmFh3omUWIusDHG3HrKmkpUQt5Os+S7LZ6ESYkcdsbe13/atJ+C31UdUEtrJDjXv1FnTjC/Hrpw1VVrJ+lCMqCDEKUYB+M2sKbBqGVlyzX4n81YVHZnC4n2EX3nfmq2VnRVwtQVCqSq6R2MUuoOVaULU5lPcSkkA+G9lK7uhndl7M+kXb0hcYVDY0JbqJynxGbTyNkTiRdfEC+7tYawBf5uupbUt51YaCi7p2USZgbkxrt4guHgRi3Eyqe5q/EQU3eSnDRXgkADMQsoJIGn6qtIEjS0TcIpHN4hj0vbd35qQ9l8KByCPW3vO/NCL6ECUJzK48Y1Cd567T+tbEdCBC0haOPONSDzD3+1adsd3dU4iwZel0UTpbeqadSWyFFPaGoBjkSAD3gkWh3hPjFy7qxPXy0HR6JVtqPquJOhI5EH4KNi3YBQtYH5NL2OrtPqhm9ncMc4t4Wv8Aid+aTB0HwTA48Y2/pf8Aat1/AcAEnjxjYf8AW/7VnhVYtvO+8UoXT3rVsUfWpaQmneUjM2DqdCN9TO+otMKK9K6dT+YhET2u6w8eEUEriGs28T+alk7L4bGATFv4u/NVs/gQJ2/f4xv/AEv+1bodBwESOPGNv6X/AGrOriDxrqbgu+sxCy/WIuukgIRRU4effkgAgeMzuAAJO02VeEHGlrHdBQXizVqqbvvMEMuOthDzTgJTlWE6bgjnyIJtH9l0F7cPS9r3P5qUdlMNAvw/G2Z35pgjoOf/AKeMb/0v+1bX8B0c+PONv6b/AGrWNvy/WbmueqvV9JUilbLmUEAqjl7dBZlL4wUVMGzeFIxThw5UZ60JzmJgSkTpZ8uEUEIBezfxd+abF2Yw2UkNh2/vO/NQzf8A0LVXNcN5Xu1x0xqtdDRvVKUKegKKEFQBhXhZ69Bu9LxvXgNS1d6V9TWPm86xJdqHVOLIChAzKJNn9X42oMT4MxMxSsqQpm6KpZJWlQILSxuLRt0DT/8AV/pB/wA6Vn9YWFhp4Kevj7uLAtdfU9W9UIzDKfC8ZiZTty3Y++pOxZbclWMJCklJAUk6EbzZtXJwx4aYaqHavDnD3DN0v1DCqZ12huinYW4yogqbUpCASkkCUnQwO60ZdMTGmKMCcFqnEGDr7qbqvFF40rSamnICwhSjmGo52iagwL0kayipap7pXsUy6qnaqC04hct50BQSTESAbFVNaGT8IRF5aAbi2l/iR0Vs/tBVUVQ+gpI5HXALgwgAg3tcFwvsVbW4sG4Nwu89U4awnc10PVH8c5Q0DVOpz+cUJBV7bLSlJWktrTmSoQQQCCD32qPd3CDpQ3ooou/pXU1QoCSGklRA8QLHTwG6W4ST/Ce0H/IL+6zG1slrMp3W8Mv7yhmxzEJn8SallLupLCfMvVkLuwHgO6VVq7pwVcFEq8UKarDT3ay0alChCkuZUjOCNwZmxe7OGHDG5aSvoLn4c4YoKa9WRT1zNNc9O03VNfqOpSgBxOp0VI1tXdvgP0t3EkjpPHT/AJBf3WwcCelspZQOlBqP+QX91o+8P/8Aau+7+aJ/lTi+oME2tr+m3W21/T5cunJWDunhLwnuC8Wb3uPhhhK7q6mJLNTSXLTMutEgglK0oBGhI0OxNi7HBTgtTuoqabhFgpp5pQWhaLgpApCgZCgQ3III3FoIPAXpcJGY9KHT/oF/dbSeAvS4UJT0oJH/AEC/utGZnbd1P3fzU38sMcJLuHPc6H0xsL6ev4nzKtLeFBd170bl23td9NW0jwh2nqGkutrHcpKpBFkZHDnhw3cj+Gm8A4bTdFUpC37vF1MCmdUkhSStrJlUQQCJGhFq6HgJ0uRr/Cg/0C/utg4B9LoxHSg/0C/usrqqR2rqd33fzQ8PaLEoG5YqaVouDo5o1Gx9fccjurDp4W8LUXIrDA4bYVFzKqvTVXeLmphTGoy5euLWTL1mXs5omNJtl18K+Fdyt1rVy8M8KUCLxp1UdYmmuamaFTTqIzNOBKBnQYEpMgwNLV5/eD6XY0/hQ/6Bf3W3+8H0ugP/AAoTH/QL+6zOO7fux+7+an/lXjJaW8GexNz6bdT1Pp7qwlz8J+FWHLyYvrD3DLCd13hTElirorlpmHmiUlJKVoQFJkEgwdiRYrVcFuDVbVO1tZwlwW/UPuKddecuClUtxZMqUpRbkkmSSdbQJ+8J0uz/APihJ/6hf3Wz94TpdH/8UP8AoF/dZvGda3dj93808drsbD+IIp81rXztvbpfPtqVMfFbEmLLlvy5KRHDR7F+C69mpbv1ukpW6qop3Jb6hQZWodYj+MzAJJ0B3ACoo4lYbPGDCdPwg4U8I7xwlct63hTv31eldcabnYpqZpYWQ20oJW66SlMQmO8wZBL94Tpdf/mg/wBAv7raPAPpcc+lB/oF/dYeYzTFwdE6x/w7dAb7FXOFdvqvCGwyU+GP4sWrXZxlLgSWvczPZzm3GU6D0W3B1vZhi5rrpKZmlZoWQ2w2ltHYEgJAA18hbly6aFcEtFMbAOKEewG1ZzwF6XHPpPH+gX91tHgL0t//AMzv+hX91iDUSWt3d33fzWb+3K4m5o5L/Fn7yso5dFNlIbdfR4h0n65sCbqXMt3jUf4QQR/VtW88BelqP/xObf8AIL+63P7wvSzJj+E0de9hf3WhJzf1Z33fzUox6vH9Tk82fvKx66KuT/F1rR7szE/UoW56q80HX0VftUj7DamvE6l6SfBO88I1WIuO1VfNLft8tUKmWElEJzJKs2Yaggxa6+nts6mjiqXOYYy0ttvbn8CUZhuOPr5JIXxujdHa4db2hcbEomFV6NTRtkj9V77wLELpqb3cdq1VF3tNKU7mDfXSQIjcAg+r8bLRnnYq2OrrFiYziY8o/GxbaRsRBarfjFwsQuku1UdqkPjlcB+62zUFMzSviO5IP1Gw+wnkbbzEdwsWGHqocyLGtYGig6nzaV91uReNCVQqqQCBsdPrsbOuptqARBHvsmVw5pbhBCpp3BKH21eSwbdEzsR77cOUtO4IWy2rxKAbALuu7lKUfQ2gVbkJj6rO9JNJARe93006WH1BUJeSNBsCYtwKk1Ci3TQcuil7pT95sSv/AA829ddSihedYdyKKFlxa0pMb5CqDz0sfo0JRTNJbEJyiABA27rd6RcQuOUC4TiuS6BQoNU4kl531ir1o+yy0pxKWS8oKIQCSEpKiR4Aak+Fkamv1lFMhFRmU4BBMd2xszb34p4swuHKm8OHDt4UZeKGn7rvBDxSiCQXUOJbUk6HRGfWAJJAsDV1cVGwyTEgDnYlRCGSd1mjVP59lQPp9QVN9iAgq7LafHlPefZytEfFbGOJr/cRgvA1IyUpKXa+orFqQ3VNZoVTtoSC4tJkFakwMoKcwKtG5jvjpfV8VSMLMsU1xuuKS4Wi+ldW82BqEpMBMEgElKojYcmlfGPa64rifRTXbcjVTeTuSoXWLLqnihIhTjisynNCAM0hIBAna2GxLH5qh7Y6EHhm13W1PXe1lcU+FuYzNPa/S6BxIxeqLjq7txbgSr6x4LNXeFE4zeLZMABRp8yGW4bSlKUEKKQEJAPZJrVxBuHhPhk3beGFq29V0F4uJZdK2i26l7RxLjza8qUoJVCUpQkQonWbTHe+P8SIoj+Y7yw8qqfQptCKa7mlNocglKisNhORJgyVDXSCTBhq/sRUVVe3XY1fqsT3wghCVslVLdhEpIBQhPWKzRrAbcJSeyop0qZJ3SSaA26ImSJsbLXBKies4h8TqCrfoaTDWJqhincU006vqlqcQkwFFWQSSBMwJtlp0XV4OUtSmMLgNkko6u7M6MvLKpWZSh3EqJPMne2WZ9rRf2H4/mg+A/qvRytSA+dNNgBbhAnWw9ePpvM8rBJGgI3t7/usiqvdOAEfvYyP/wB6Ef6lrNP0TbypUmTaOOO3AW7OO913Rdl54hrrn/NFUqrbdpG0qUpSkxHa2jfS0YfwG/8A9O+Nv6X/AGrURFZTVkssUWdr8vtAbC3NZ4trqSummhhztflscwGwtzVl00qUoyhO+liyLmp23y+hoBXMga2rl/Aa/wD08Y2/pf8Aatn8Bo8+O+Nv6X/atKauv/8AbffaiBXYmP6p/qN/JWcQ1lTFky+cPUV8NFmrYQ4lWhCkgg2rt/AaP/r4xt/S/wC1bY6DM/8Aj4xv/S/7VkNZXnQ0332rhX4mNqT/AFG/kpaZ4SYVp6v0lm56VCwqcyWUg+8Czxu66mqJtLbaQlKRppauZ6DI/wDXxjb+l/2rZ/AZ3njvjf8Apf8AasxtRWtNxTffannEcUeNaX/Ub+Ssffdy0V+XTUXTeNM1UU1Sjq3GnEhSVJ7iDaGby6MXDypdKxh+lSSSeyjL9VmuOg1rH7/GNh/1v+1bX8Bgf+vfG39L/tW51TXv/q5H/eEjMQxNv9U++38lIWB+B2EcF1vp103Ow0/EBzLmXE/rHXnaUqelyN5csWrX/AX7uO2Nv6X/AGrZ/AZP/r3xt/S/7VlZVV7P6v8AfC59fibzfun32/knTxJ4U8QL1xNWX1hbijf9z09VlUKRl0lptQSAcqQQIMT5k2a7PDzpAUaoa4zXkpO0uUKHP6x1+FuT0GATrx2xqf8Arf8Aatz/AAF0zrx0xp/Sf7VkdWYm46Qfeb+Sc2vrwNaL/UarE3FTVKKBhuufU8+htKVuFIBWoCCqB3m0ccSmOL1JiVC8CV9zpuqoQhLzVZTKUtLkkKUFJ3EZdO8eNo/HQZj/AMe2NR5O/wC1bSugwlX/AI9canzc/wBq0oxDEW7U/wB9qiNZiJOtH/qNUt1mFqtGGKj83ttqrmmCunCpCS4nVIJ5agWbXDvEfEzEd8OUmM8KXfdbFKgqS8xUlSlrJAyhBJhMTqTyHjZkfwGOX7+uNY/6T/atwOguEqkccsZCeYc1/rWV2IYk616b74XCrxAf1T/Ub+SszTslLYTM6crRRjnihRYGxW9cddhrEFYqtS260/Q0XWNGRlylRUO1p8RZgjoNxvx2xqP+t/2rAvdBVKyFK434xWRzUsH/AFrcK+vH9W++1cazEibmk/1G/kppvNl5OAr7UtBH+5NVv/0KrRt0D4/eApO/86Vn9YWQP4DDbiFMu8ccZFCwUqSXJBB5EZtbTbwW4S3fwWwO1ge672qLyYaqXakP1CEoUS4QSITppYePvVRWsnmiyNa1w9YHcjp8EyBlbUYgyonhyNa1w9YOuSW9Pgn0oc4tTXBCmW+klxqdqHUNNtuMFS1qypT9INSTa5Zi1CMRYnOAePHFk4j4Y3/iO6sSPIpkikplZewpKwrMUwRIG3MWdiszKeSnlk9UPN/1XJMdlZTy000hs0P1Opt6LuikLi5hb90NPRYtw0609emGyirplMrCs6Uk52iRyUhR074s4MKXg3VpoLxajqny0tHOJj59loywb0g8M4Hu5y7Ll4AYwaZdWpxQ6knMowJJIM6AD2Wa1PxKwtR4wTium4OcQWymo9JTSgqDQXMgZQnaY08LSOx7DnNDc508D+SmHaHDQ64l+jv3VJnGurao+k7wdqns2RsOEwJPrkWtfQ3lT1rXXMlYE6hQ177ULxXi+/ePXGfAVTceCsS4XF3FdKaqppiS2pRKgsHLlERztcbhrcF/3BcvoOI7+qb2qi4VqffQlBgwAkBIiBHnqbD4XPHPLPJGbgu0391qZgcrKqWqniN2l4sdvYb1T3QQHVkHusk45j9xOIAD/wDddWf9EqysiOtP80fbZJxx/wBxWIP/AGXV7f8AQqtbSH0HfBXU38274FVT6LLYPA65lbTeVeD7FJNk272HsKcb8VXDejinbj4gNIrqUGQlFW0whl9r/DQEHlsbC8ARiqn6M91XjhG4UXvVt3zWoWwpwIhskSqTvsB7bSHevDrEGN8DUl43jdrd34goXE3hTNocCyy8kkhGYaap7J5a84tXYVEH0sJcdCwD4FCdnZjHg8AA1sD9E88Jyy7SIKyvI11eYjeI1su8RE58G1qp9QIUY/npNo04X4oxPf8AiE3feGA7yulunQovVFSsBOYGAAmASTM9wFpixDc7t8YavC7ElIXUU60IJ2C47Pxi1jQMMTcrzzVvXPEjg5vRRlginaeqHkOAHsjQ+2bNpqmbu7E1eywMobvErSkEDfKon4/H3IuH+Jl23FeLt238+m6L3opZqKet7CVHvB25Tv5SLK1y3xdGKMTFdBeVLUv1b4fcbpnA5lACQToTGiRv32bOTDCYXDW6dE3iSiUHSysLd5LzISoesJNoM4hYZXhbHiK+iY/uC/wtTqUjRFQkST/hCfMk2nW6WVJZSVJIJHdZm8ZKdlGH6etdhIp6tKlE7JBBB15CxsRD2lh2IQjyWuDhyKaWA7sFXWpeCISyAkCI1/ZaVL3aW3h+tSgwoUzkeeU2YnCd5itpqlTS0ryPDVJB3SPuNpNqmQ9RONEGHEFJ9tgKJgZqUXWPzusoEue5qS/qemu2raQ408hKikiZ7M2K4D4G3nhPGDS7uxVWM3BT1ZrGrtbZQEyTmylZBMZu6NBHjY9hC8WqR9sOqBdu99dM+kboUCU6jcd9lurxXiF6/akXbWFijZDaWShCVdZIOcmZnUx7LMe0QGQOOt9lMwumyZdrJ/47aL2C72TufRHFQfBM/ZaBsWYEw/xFueiocR0iX2adxNSjUgpVkKZBBBGijawl9p9PuGuYBCutpXEkd8pNq+X1hZeOcIUlzovK8LtWvqHA/RP9U6CncBQ5HWzq8foGOBtZ34hNoT+le0i+idmA8D4PwDw9xPd2GaRLT9RdtS5UryqzuANLCQVEnQSdPE99ifQPP/YApB/zpWf1hY1gPhw9gXAeK3qnEF83u/VXZUJ6y8axT3VoDSjCAdBJ3O+gsV6B/wD3gaTf/tpWf1han17/ABXN/Rd+LVSVpBxyGwt+jf8Aixb6eH/g/wBX/wC1KL+sbIDLNc7hVxigqeoqzRZWHcoUUOFkFCoOmhg66aWX+nhp0f6sf86UX9Y2JUbKmWGmdgaGjWD/ADqdBtIW3qKk9GMP3im4e63aKT/Az/8AtMDo+Yv4i1jF5Ix46F3xcd5Btqqbpwz1iCkESEwn8Fa2uFXVbv5rqXaZeVzqFqQe5WUxanXC7iZiy/saXrg7F+HaSgrboQmrYeplr6t5GeBovv335ERa2ly1v5yuZmqIA69oFQHJXMeUzYum9GchwtextyWkqdYQQb2uLqMeFF9VS71oqp+red9KZcbcK1qVKoJkzzzJsnX7j7ErHES92k3tVtM3dVtJQy26UILWUGCmYMydT9mhfh8VUdc3RKVBoryepjOn6ZH1GyTjtgUfFO+GwSDW0tNUjTuTk/1bE5Ld6Z0JI8wVCTmdA7qNfIqx98Xv6BctbeaEhz0enceCSfWKUkx8LMThdjy/MQVqWr2qmltP0xeQlLSUwvsnQjlBO/2WclCoXthFkqJPpdCkH/Cb1+u0RcH6st110IWoyELp1eYChHwFopf5hsg94fVPhH6UsPQ/RSzxJxwvB10UtY1SCpeqqlNM2FKypSSCcxjUjT4ix7A2K3MVXKbwfpEU7rbymVIQsqGgSQZ/wrMbj80teC2K1IJNHeLDpKe4kp+tQsa4NvqFFeFMDol5Dm36wj/Vt1S3I2Nw53B+SbCMxeDyspTzaSTAs0H+IN1NXjU0CBVOrpnC2sobGXMBJAkiY2s6sxLR8rQPW0zSsR4gut1RCHKh5C4JSQHBMg8tDaKoJjpXys3bb8bJ8QDpmxu2P5KXLrxnddbUJpS8ttxeiQ62USe6dvjZwFYgkjYWptga5MUcOuKNXhC8MSXneV1V1CirovTX1OlohRBAKjpsqYiRl7pta6769dfcjb2cl1bRSox+kBBPvFoIXScQRS21AIPgVLI1gYXM5GyFoMU3FeN4O3VS33d71azOembqUKdTG8oBke6yhUVCWUhau/mbU/4Q1xuzirdTzhhS33KdU/y0LR9cWlbpP1lUnA1K3TuuJadq8joSqArsKKZ7xItZd0JqnUwOyEEw4Am6qaBUoU0XRqPO2NOpdSVDkedqQ4I4u3vhPA2I8Mt3g4walkVFE8hRzNOhSQsJJ9XMjbaCnTe0hdE7F17V9+X9dN7XxXVgVTIeYTUVS3UoCHCDlzEgSFp27rDuiyxCXxspRJeTJbldWcW4htOZRgWDD7SuyFjytFnHni29w1w+0q7KVuovKtCuoDv8W2AUpK1CQTBUIGk9+lq03X0leL9DfdKi8r8D6K1AcYbqLubSy9G4SUpBI03CvC0LhkIBG6nac1yOSkrpw/8Aixj/ANKEf6lrOSZjX32pz0lcdUvEPCPCbElM2GnHMUJaqGZnqnk5MyZ5jYjwULXGMk6e2bVtKb1c5H938FQ4eCMWrQf+n/tK0TpYBaQKhtXfpYc2BqCYQs/oqn2fItYELQDQoXlB2tsd5FsgeVtDQWck3Wz9ltGPttsxbRjyty5c8tPbbABvGlsPrAW3y3iygJhN0E+jOhSDsdPGyXdi1Lo0A+s2S2rwIMWWCkEH67I13pW1WV1OUFIDwdST3KE/ZZSNQVwNwjfOwrFMmtYqKUhJJRnROuqTpYMjaOdumKynoFmrql5GUIVnVBOkcgNTryGp2E2HrImyQua/ay5ji112qGeJGFsPXjeTt1X1dVHXpqMj3o7yEqKQBlLsK2CTOo11jnaM6/At61jTCsIY6vS7kS4S7Vspr2yhxWYltLpCkJToE5VpkAbxmtM9ZdFZeV53ziKvFRRmpdDFPTvsJ6xlpCElBO8ElWbLMAklQJEBnYqp6DD+Faqovlhw0bjakhVQFn0lRSYQVASSo6bRrGgtlw2KcOke0NbyVi57so1uVEd54XGOL6euypvuhN00CA+2mmpwHaphQAzvP5lEhausAylCYCyEiAVVd4jXNfT+KL3ufFdSbruNNSqgpKupvVa1u0KV/Q5kpKnUsDIlUHKgwSIJm1vboprrui4VVd41V1O3teb4rq4OOoS1TKIEAI/kABKUnmJ5E2pn0uMD4uxBj19jDL7t4XXVei1zRp6rMwXVIDa1rIkKXKJO5GcAAARaKnpKZjgGkNvueiHlkcGZzqpFosV4dpqKnpqfjNerDTTSEIaZut5baEgABKVEypIGgPMWy1XKTAGOmqVlsYppWcjaU9WFSEQPV0MabaWywxwSkv8A0n8PyUXfpPdX0CXgmHpidbAISAZiLRkrpNcMa5wFKr2ZkxDlIlUHxyLNjjHH7hY4QF37UNHnnu6o09yDb03vtOfbCqjSTjXKVIoEamTboJ5RttZmMcZeGFSQG8YUqSqQA4063/WSLKLPEjh6/wCpja5Ae5Va2k/E2cKqF2zx5phglG7T5JxjvtmnI2TKbFmE6whNJie6HyZIDda0o/BVlFqppXxLNS05PNCwfqNpBKx2xTeG8bhdhI57W3A01tsAny8LbjTXnZbg7JuUrkCdYtqNoHlbuBbCmRHhvbrrrFcQCY2tsJncW6jWe+2EREC3JVzHdrp77ZljXvt0BHO2QeQHjbly5yneDbRAsIJ521Eam3LlwE9wtoDwsJGvnbUDyty5cRJ+G9tQDyt2U66W0Rpyty5c5ZtqOYGluyI1toA6aW47LlxHO248LbAJNsIFm3XLURbRAIi2yALYYnQ2UFcgy0gnbUW0plJI7NhY5WwjlFkK5AllJEZdLaFOlGosPvytrnZFwNkABD0EfofbZJxx/wBxV/8AjddX/wDCVZaI+mTH6p+y3FVSU9dTPUVY0HWKltTTiFbLQoEEHwIJs14zNITZWl7S3qFT7oh8ZuEmC+C1Ph7GeObsuu8E3hVOmmqFEKCFKGU6DnaZ09JDo9oSUp4pXJB5dYr+zY//AAXuj9/6q7k/o1f2ra/gvdH7/wBVdyf0av7VqWmp8TpYWwtMZDQB7XJZ2jpsZoqdlMwxENAHt8klo6RHR4adLyOJ9xBR7lq+6xwdJno/hMfvqXJ/SK/s2M/wXuj+duFVyaf8mr+1bR6L3R/A/wC9Vcn9Gr+1aYDFBtw/vIsuxs78L76ZeKuKfRXxYQu+sb4YrFjZTycxHkSmwGGOJ/Rdwqo/mbHeGqNJMkMgon3J1s+/4L3R+2/equU/9Wr+1bX8F7o//wDqruX2oV/asx0eJuNzw/vKRkmOtFhwvvouz0meADaQBxTuQR/yiv7NkrFvHbo2YwuOouG+OJNxv01TGdHWrGxkagd9l09F3o/k6cK7kH/Vq/tWz+C7wAkf9iu5Y/6NX9qzh9qtFhw/vJv/AN73/RffTMwJxc6M+A7uN13HxFuGnYU4XFAOLJUo8ySJJgAewWdv8JvgARH76dyf0iv7Nh/4L3R/58K7k/o1/wBq2HovdH/lwruT/EV/as0fag/s/vLnOxt39l99QlxExZwUvW/3sWYE463Vct5VI/ulEF1h895QQNfH2xJJIGC+J+BaK8Q/i7jjh+saQZS3TsdSCe9R1J8hAtOf8F3gBz4V3L/Rq/tW1/Bb4AAf9625f8RX9qzJhik/rmP7yfFLjcIs3hffSU30j+AYpVMK4pXL2k5dXFfdau9VxrXdFQ5RXVjLAtbQsrysKVUPZy2DoSdNfZGvO1nD0XeAA/8AFZcv9Gr+1bhfRa4Bq24X3MPJCv7VnvOKPZwzwyP+5dHJjbHZhwr/APeq9XBx+uerTeLGIcS4YoaVV3VaQpmtWtx1xTK0oQExGpVz+u0pdBAH94KkgH/tpW/1hZ5J6LPARKteGFzEfzFf2rP3CGDMMYDudOH8IXLT3Xd6VqdTTsAhAWr1jB5nS0FNR1IqWzz5bAEejfnbr8E+GmxGevZWVpZZrXNAbm5kHn8FCXTwH/1fqsztelGP842TMV31S4Rprjrau7rxdRXXJRKCqWlU6AUspBKiNBsPGyp08R/9X6r/APalF/WNpbw1c9JeeCcPJqGUri66WJH/ACSbSxuPfp2WvdjR9XJKQWx2V19mMP1eq43FiHCF7YjRW3bd9eq+HWPRBFC8k9XnzQoxlGvNUR399lcM0S6O56elM/Rog+Z1PxNuqfCN3MO9Y3SoB8BZdYpEtJygaARYiGJzJM7unW61M0zXsyhVkxFflPgPiTfF0Xy76I1XPi8aF5ww25nOonl2pH+CfCQMQVrGIcVs4jRUoTFCmjU2lQXnVnKgQR7RHj75+xtw4wzjimRT4gumnrOqnq1LT2kTvlVuPfZoYc4AYHwxeTV5XbcyEPteopa1ryeIzEwbT1cpcXOjuC4WO1io6ZrQAH2OU6dU8MJUr9LhmgpagEONsJCgrdPgfLa0L4OIuvFTtIpYT6JfjzQA5J64j7TaxLFKGmQ3A2i0dPcGLkXjhzF7j1couPekejCpUljrZnMUDfXXXSzS4Cm4TvD6Lmj9PxB4/VGuMdJ6bw4vhATq00l8eGRaVfZZqcML+pbrfdXVVLbbdSyhQU4QlMg95PibSvf9yNX3cVbc7spTWU62FEakZkkTaAHuCnEyhX1VBxABaQmEocu5CttoMz+yyVMreCwX1BJ58/gup4yZHHkQrD3ZfNBejak0ldTPKSJKW3UqIHeQDaHcVA0nEq8Wtk1TTL4HIwkJJ94s4OEeC8T4bdqa7FGIE3g+62GW0N0qWUITIJUYJJJgdwEHTXRI4xYaxkcQUeI8I3dSVv8Acvo7yH3lIiFEgiAZ3+FkZK2Smka+wuD/AON1xiLZmFvIpu4+qGF4/wAEVjLZzut1jLhHMdUkjXunNaYsJrzXOhvUhJIEjxm0J3KxxOrbwYp71wfRMgGDUGrCg2nnACSTMDS083Bd/oVC2yE+qnXxPM2r453z1DHEWDW23vzui3RCKJwve5uqlPAYa4rqk5E0V/qV3AJD2YeyDad+P93/AJw4aVjwaC1UbzFQAO7OEn4KNop4v4Pv9zidVt3PcVdVKvB1t9lbTBLQJQASV7CFJ1k6b2nfHV2VN68Or5o1ty+7dzhCU6/SBEgDv1FtEZ2d/bKDvlv5C6qWxuFKYyOq8+79rDd160FA+kejXiaikXpEORmRttISse0WmHov1i7u4jUdI44CamkfpidicqQoT59WD7bNOi4eVnEpV6Ut0Uq3n6BhVZSvBJyt1KFAhJVsCoBafaTysd4OVFZd3ErDjjLK1OqrW23GUpOdAUSlcp5QConuFhqlgaHBnIqWFxuC7mFLHS4onHGroqAeyqnfTP6pSttX1WiG9X8LXzw7whdlVeNVS3zSpqXA6whByIDykQSrSSAmAJOnlNiuk5c7lZgemvZCDloKmHVfqtuJyz/jZffaoF53FcN/4cXcd60JNbSVKHqN4AlJbK0lxs84gE95k7WWVrS2KXwIv0P8FOjJBkZ4g/FOviM9hdnCHDu5sOC8iqlxcy9VuVxTndccSkBQynKBCIgAbe23oB7LebONa6ndrsJsU0BP7oqQwBAGVZEeyYt6SjQcrZuhJNVPf+7+CraIAYvW2/6X+xYSOf12AqhmZUNOXusMTrbhYlBSIkgxa0V1m1WNrK2kK70g2EEeB8bA05CmAAI1I+OlhYV3EWUBLdbNtQY29tugNINtE5drOsmkrmAVTFt7DS29xNtcpA3soF0hXMamBZKdHU3wIPZfZ180n7jZXJnYWSr0S4iqon0FISHihyZnKpJ29oFnOGia0m6MEQINk9hL9TfPpSwE0lCpIZAPrva51HwSCEjxz9wNlEg8rcZSOQ3n42jljDwL7JWvsu8Z0SFNhWmUt92kj5FoTxhdtMtygp36huoT15dbomJXUVTyE5kNpEgQIzKJ7AA1IE2lfGl6YqXStMXLhAVyVMg+kv16GGwrUFOgUr9EHbnaLMc0+JLjuS8b0pLvV6SaNCqnqULqSXSAC2hQACGgspOoAMKWc0ADFY1iTImCCJ2Vx8Oh8VYUzC5tnaqJL8oam+sYGgxzfzV3XfUtn0OhZWQEtNoBWXqkAhIObMUoyQIzLIFki+KC68Sm/wD0LBN03w4puoU9elJkcTd4SUpbCUalS1DM4ZDclQSP1rOPGF13jeDbNNft5LoLu6sdZSMinFStRITlSjOpEbqzJUSmQIJEGHMYUzhvN5q7OJCaO4qVCHEUdNUFpa3FJKjCSMzhSkI1JWSqAkTojD1k1RWZmSm1uZFjf5f/AAjgyMD0QnRTXZw2YpmmH7ovFbjaEoWpV53ekqUBBJHVmPKT5m2WblN0f6Ospmqut4nVtNUPoS46yXHFFtZEqTI0MEkT4Wy1Xni/tz5FM7u73PqjCax9lwgKMRGqttfqP2WWaK+esgLc7QAkn57h862btSFdbnQqVRqeWnhYamUkAJJUkgHYmRz39vut7A4Byc06p6MVqHRmSoxOp+fnax1t6EgyZMEx8+dmgxVOtKKDMCU6DXSfvsr09e4jQqJSR60wfh8mw7m22UwSz1dKtRCqZog8ykbW7QGG1AoZbSrlCQNfn6rFGqpJIBGYbRttYdL2X1ee5zakT8bRahPsClOnvWtphmp6+qZEzLTyk+4g/M2UGMaYspiEsYpvhPgK50R3/pWQM0pAUCY356/fYQKVHZyx60k7WcJHjYpvDYdwnRT8R8fU/qYuvedfXq1rHxJsep+MHElkZW8V1J7s7TS+79ZJmzJCiQdTvvM/s/G3RJIBCjBmQeZPd32eKmZuzj5ppgiO7R5KRGeOfEdr1r5ZeGnr0bX+qkGygz0hMet6ON3S6RB7dMofFKxaLiVSIOuYRG9t5lAjNzEAjeItIK6pbs8qI0cDvZCmGm6RmKgkiouO6XOfYDiD8Vmymx0irwBCanCVMonSUVyk690ZDaD2lQrMdBPMbfPzysfBKYBTrGuvzrZTitWzZ64YbTO9lTez0iKQn+6MJvI0B7FaFfWgfIsca6QmG5+luC9kx+qWlf64tA6c2TMI0gAwNfcbdQRHIRr4fPdZRjdW3d30TThFMeSsCzx7wW5HWUN8NyP0mGz9ThsdZ42cPnRK7xq2f59G4f6oNq4CTEGdQRpbZ7Qgjw33tKMeqh0UZwWn5XVmW+LfDx2B+6NCCeTlO8g/FFjrXETAr+jeK7un+U8Ef1otVgKOoK4G+vM8xbArKD/KPPa0re0M/NoUZwOI7OKtm1i7Cb2jGJ7pX/NrWz9tjbV5XZUgGnvKmdnmh5JHwNqhmOzmUNDsoeXwtnVgCAEKO0ECLSDtE/2mJhwJvJ6uKClQkEHnpbeU8h77U7S6toyha2++FRBsZZvm+KeDT3vWN/q5Xlpg+QNpB2iHNn1Uf2EeT/ordFJki2ssbDQ2qk3jHFzRQGsTXwIjatcjb+d8+Vj7XEnHTMZMVV2n66grT/CB1tIO0MXNhTDgcnJwVn8pmDrbRSdx5WrY1xe4gs6HEPWD+VTMk/1LHWuN2PGvWqKB7/pKUa/4pEWlbj9MdwVEcEqBsQrD5T3WwDTa0Cs8fMYJGZ277ncA/wCRdT/8yxxjpB3ykjr8M0SwQQCioUifeFWmbjVIef0UZwiqHJTS4k9c2ZmQQPhbspVse60Lq6Qz6nWf95zaoJK4vIiNI/4rvsos9IO6yQKvDFWgk69VUoWB7wm0gxejPtqN2GVTd2qV4jW2gNbRsxx9wk6QHbovluT/AMWyod3/ABlj7PG/ATwBcqK5gk/p0ij/AFZtK3EqV2zwozQVI9gp9RFsg/ss0GuMHDpwhP7oCg/y6N9PxyWOMcSsBVAARiqgE6DrFFH9YC0grKd2zx5qM0s43YfJOMCdxbBuB9lklnGWD31Qziu6FmdhXN/2rHWr4uiogsXvROfzKhB+o2kE0Z2cF3Ce3cFGRtqLYE72xtbaxLbiVDvSoG3YSQNvhbi4HYphaVxvE62wifvt1l8PhbcAnbSyXTbFBxNt699uojW28vdbl1lwQDuNbayiNbCEEfttqNd9vG3ap7WhckcrZHeLdFNtwYs1SjRRf0iOEt48aeGz2B7rvWmu192sYqQ/UIUpADZJIhOus2iSk6O/SvoaZmhpOkulpinbS00hLC4ShIgAabAWtWByjztgHOwE+HQVEnFdcOtbRxG3wKqqvBqWsmNQ/MHWAu1zm6DbYjqVVocAulvH/hOf6Bf3W2OAPS3G3Se/0C/utaUA26HfaL7Jg95367vzQ47PUh9qT/Mf+8qsfvA9Lgn/AMJ3/QL+62v3gelt/wDmdH9Av7rWo30i2a87d9kQe879d35rv5PUnvSf5j/3lVgcAultP/hPf6Bf3W1+8D0tidOk6P6Bf3WtPEaW1ubd9kQe879d35rv5PUnvSf5j/3lVn94HpbEf+E6I/6Bf3W4V0felkr1ukykz306/utaqBGhtoxNmnCIOrv13fmlHZ6l9+T/ADH/ALyqsno+9LNPq9JpI8mF/dbS+j70s1iHOkwlQ8adf3WtWR4WzlZn2RT7Xd+s781J/J6l9+T/ADH/ALyqgOjt0rZzfwk0SOfo6/usKngB0tE6J6TQHkwv7rWoO1syjff7bIcJp27F36zvzS/yepTu+T/Mf+8qpO9HjpWOuhbnSTQtY2Uadcj4W7PR/wCllkKP4TCSk6EdQuD8LWoAnQDSbaUNdLNGEwdXfrO/NKOz1Id3yf5j/wB5VNY6N/SjpSoU/SLZbzGTlpliT7rFKbow9JShrV19Hx9o2Kl0krdbo1JWoneSE62t3A5zbUWQ4RB1d+s780/+TtL78n+a/wDeVUa3o7dKa8KN2ir+kY1U0zyS26y7TKUhaSNQQRBFmS50JeMTKsqOLN3ZZmE07gAPgIgWvEYCYgWDKBOoFuOFxWsHO/Wd+aQdnKMG+aT/ADH/ALypJcvQb4jsXjdLt4cSLqfpLrrmqtLHozm6VhSsvcTr7TNrrbCLCFAG3ltbRTA0OlpqWijpM2S9zvck7fFH0OGU+HF7oL3fa5Li4m22pJK4MxbcAjQW3A2FtiQN9bGAI9FqLQut/quGPLb6wbGfZ8LFmTlrXkkesAoeX7SbGdeduGyU7rUSN9bZOu9tnTW2jyjz2sqRYZ5W142yYNsnvs5oTXLNCLEL2SVUi1p3QoOD2GbHuUTzsA+jrWVt7Z0lPvFpMtwmE5SEEYVBTzEi3JGnITbikWV0jRVM5QD5i3ZBJgCfKzTsl5o5W9a5hOo9GID7WZLRInKrcHcbT3i0QY1pqqsu2nRS3kp2iTWM1brbPVn0wN7gKWoAT3iZMCRaVbsQ5W0VSLwQBADzVNMhChsVGNVbabDxiSwMYsstVThynM2SrQTIj4b/AAtka/CnVdQ6cGwNhtrvdWFNM2NpYdeairHd0JxFftyel3Ez6CUKdrCX0NvQkpLbS3QFKQ0SFaNyVdkEpSSosniK9ebF0V173Pwwweitu+lcVR1jzyXhTBptXVrDRYEwEghExMTGpDmvnE7EqpbgoV3nVpKlPNUakpbaWZUrrHVEITrpElWvq72RU01e+07U4leKUFQCaKhGYKzQOrUVCXJgiRkEEyNMwidgVPw+NOcz7eFguFW7NkYLBVJRx8u9tCW6vijicvpADpZvd0IKxvl0Ok7anS2WeVTwiacqXV1XHbB9E8pai5TIo7uSlhU6oAQMgCTpCezpppbLZrg4d/B//wAqe8vX+PNPV5hQcWVNApCp1BnW3bDQCUmJy6aAfJserm2Kd0B91LSycoCjlk9w7zbmmCFpBDyPNKwR7Pn3207ZC8XCsuGGnVbbayiMpVprynnPKPx8bDIQRqEyRzI7htHw+dRUNACEKhQgDKI5be+3SWURKCBz5QAOXwn22XMeaXRdsOrZhCgNwezIkz4an53scp34y50E6bxr872KJCSQG1BR035ba8uVttZoUEkpn1goROvl8++zNSuSu1VGZJnKNAZJG/L2WHDiVpIKlDKBJ31+dOdkZLqwc0gA6idR47WNJdcTEkmYMx8P287NXBK2q+0kg6Zp1Mffvy/bgASO0skcxEeViaH2iYSdfWBnaxhDhAKRyPiD8PZZpThZCjVJ0zayN/Z5W7CRm7RJ10BPf887cJUgqyKVJJ11t1AGxSRvpz08fL4WS67ddNwSEpUdOXz8/VZSaGsyddxPz8iyWkhK5Ag6GT3WU2QooAAI5xt+20L3KZgQoHeAJG+4Ed23z8BCUxBGm0Trv3bG3AACQROpGg+v577a0HqpAJAAkT7ItDdSrtQjQEAARI8rcZhqMokg6CTy+fjbZSpAgnU6687crTExlISDpuZ+63BySwWA5iQDoOahpPstiUkIUgJBPhzBtpCSQElUA76b27GTMNM07gWdmXBoWJUqNIHZJB3+fxtsDRQCZiAIOnP5/bbcZZKjrGhyx3afC3MyCQI391kzFdYLqUTmVy0M/Ej53tyR+sBrrttbogyBKiTrvoPmLaJMAySCZn55bW666yxKdNRMGCAeUfttsAqGgB05jl8xbekQIB5eXz9VtaKSQiIEbcvn57rdddlCzMJBBnTutis0jNrm1gzbMsZSVagyTbFJEmE+tpygW66cGrFFM90HSbcFZgnsjTYSBbsqBWASIA2Go/Z872CVqsJnTdWg1+dLOumlqxCSo6JAJOoJt2QmIJOmgnnpbESFggjaBJ1toJIGUeYj65t103KugUqmU7gCDoe+2FQSSdyJ1+r7/fYPKQCc0nfXbw8bZmkg5ZO41t1ynZUIFgmDqU9r48/n8MSqdQs7TPz8725BEg7xprHz3/ItzmIToRMA78u6yLrBC5gIyzrpz3+ZtsJQpJCgCZ3KeRsFmSnRYOo0JJ20n587YpRG87xp9/z963K4tBQg6pKcyEAEHSBHfrY0zed4MCWLxqWR/wAm8tP1HexBB17JMzbuE5Sk6Rtz+fxsokc3YpvDadwldvFmJ6dQ6nEt7IHd6e9H9ax5niJjmnnJiu8TzhToWR/jTZtjU5gJ3231/ZbagEjKU+Z2s8VUrdnHzTHU0TtS0J4s8W+ILUg4jUo9y6dgx/mfbY81xpx41Gauo3tYPWUiRP8Aix42YQWdRqCN4FtZokrKT3nwm0gr6kbPPmozRU53YPJSU1x1xsiA5Q3O4Ty9GcB/+J4HlY4zx9v5J+nuO7l/zFOI+sm0UpVG5M66wPnxt1nUganTnMT+20gxOrbs8ppw6mPsKYUdIF8aPYVbImJTXEf/ACzY9T8fbrVo/h2qRrH0b6V/WBaEc6uzMjkRvJ7raKzrCtIk6fV77PGMVY9r6Jn2VSu9lT61x1wovR27b2a7z1bZj/PseZ4z4Hc/jKisZn9enJ/qk2rulzU6AjlpGvdbsupUgjnJE9/haQY5VN3sfkmnBqY9VZJni1gB7QX7lOxCqZ4a/wCLFjzXEHBL2iMS0QO/bWUf1gLVe6w5gQQZ2Mb/AFWEQ6QMw1BBjnZ47QVA3aFH9hQcnFWmbxhhJw9jFN1Gdv7sb++xtm+bmqNae9qN3+Y+g/baqAeKRmPIb+Fuy8DqIgTv9vLvtI3tFJzYPNMOAs5PKtshTaxKFpUP5Jm28pAmLVIQ+hsEtkJgmIAT8/jY01fd50whi9KtqNRkfWI9xtKO0Y5x/VMOAnk/6K10dwtoCO61YKbG2KGSQnEd5dnUzUrP22UWeJGMm0/R4hqQSJ7ZCv6wNnjtFEd2FRnApRs4Kx0GbZBOszav1PxYxsgwu/C5/OYa+vLY63xlxc2Bnco3Cf1mh79I0s8Y/THcFNODVA2IU6EaW0RoItC7XG3EKdHaG71/4Cx9SvmbHWuOFaB9PcbCo3KHCkfGbStxykdzUZwmpHJSzl7xbRE8zPK0Ys8bmXI6zD/nFTP+pY21xouZX8bdFWDMdhSVfXFnDF6M+3+KjOGVTfZUgEaaW0RJk2Y6eMWFFaOM16J72kn/AFrGkcVcFuKg17rf89hX2TaYYhSu2eEzudQzdhTsI05xbR+Ys3E8RsFuwUX2gctWXBH+bYdGOsIOmU4hox3Z15frizhVwHZ480nd5Ru0+SWSP2W0Rynaya3i3C1QYZxHdiyeSaxskeybG27yu5/RivpXO7K8k/UbSCWN2xCYY3jcIYwItra2ApVqkhXkZtmx+o2kBB5ptiN0WcGWtbWD66SmPebGRHKxSsX1a6deaPpAmfr+qxqZso1SE2WSO/S2vPnbDbJnnZ4Fkwm61Os2zYTbJOtuVyLOA1Tb2W1ETA1HlbgkbAG2yZGnO2iANzHnaVRk3RCmkdc0f0HVD2HX7bCmBztwRkrXdNFpSofVYRPVpUVrbKpjYkbGw8zntjzRDMel7J4AvqhKNZS+J2VIPLe0ZYrrcP4pqn6E17VQGEinrrvC8q0qBk5yCFFJEREAjeQdJGDLS31VGdSHVEEHOSTrt3QLQnxi6N93cS7zTifDWKqi4b7SoLNW0lS2lCCkZShSShUwTGaYGg3t5vinamsoqkMrKcsYd+d+hBH1VgyFpbmYblJL9BdN3AUd20TdDRIWUNMNjsMo5bDYA6mPfaIL9xFja6rzcvxrhop68GXBQJUm9xVU7SASp1XVJbQpPLM52k6p7QTlKptu/htifAdzxfGIq3Grzba1pBTTUapyZQjO4VFXaJgwPWnXKlNkKnue+L6v8Ou4YXc7tGhsLpOrpXE5YBKnHUo1QIgZVJ7JSQkFUWranta3F5OBRuysYL6nf5IiKlDBnfuVXerw5gr0p706+HBU9YrrsxoZzz2pioPOedstPTuBOETjq13lh3EbtWpRL7n5lqe25PaOihuZ5DyFstmTiEnv/VF8FnRQR+UjfdTwswulvNmVfoUQmQdKV0DY96tvG1PuEz1Y+9TH0h4pLyFZetUf75BmfPutbf8AKUOBOA8I00qld7Oq0J2SwP7XxtVTg62VNUD7hSS6/vHesGB3+rrzt652QaHYbHm6n8UP2jcRWvt0H4KcGDVpKeqqahKwdAl0iROosZTUXvCQq86uZ5PLgDu1P22HpWpJCe0DJAA1A7z3aj55KKWUhGZUyBnIOoEc5+drbgwxn2Qss2Z/Ioqi+MSNbX5XGTBCllQjX4R9YsOjEWK0ggXu7mGnbQk7bcrD+ito7IQN9NNZ8vn67CBgJJ0knQCYBI8T86D2RmlhPsDyTxUzN2cVprFWKVGDXIWoEeu0mT8I5WNN40xQ0hUrpVmQYLZ0PlPf5/VbkU8EBSSTrqVan5Pzztr0RCogRyEAGB+w2b3CmduwLu/VA2cUcb4iX+2iF0tEojQQFD/W7p1jusbZ4m3sClH5pY3mEuESfcbI/oiHDlWkHaSEx3bxytv0JGgMaHY7fD2e+yHCqR27AnDEqkbOTkRxTqZKV3GDKdw/Ed51TYdPFunQfpLkqJTz6xJgz7PkcrNY0pSk6k6+Ex87TYF6hbAkDluFbRH4Wi+xKN3spwxeqHtJavPpGYSuWsRS3pct9fSIDoU0004kCYgysGfnno76HjFheqpWn/R7yQHUBSc7KdiJnRXlaqHFhIp8Q04SMs0qSkgc8yhz1HdaV8P0gduiizAEqpmyZ0mUjusN/J6je46HzRIxyqY0bKZE8W8JjQOVgJM606u/7TFjLPFHBrkxeLqSOSqZwH35fP3d1on/ADYlQCUIIMzManXv2sILsBEAk6wnszO/4WaezFGdifNKO0NTzAUuI4h4OfgIv5hBOpCklOpkcxYwjG+FHinJiKh0mZeSAPedTv8AJtDJuxKDkSiJG08trcfm5uQSkAKEHT7bQnstByeVI3tHNzaFOTOJMPuJzN33RKgaKFQg/V9tjaLwu5ROSvYJJGzqSfDn4fC0BC6ErIUtEAHnyMHnZJxDcbZu0ZkJT2hEkaaHfTytA/ssy12yfREM7RPJ1Z9VZpLjbiYC0qAO411t0mCUgGCnUx3eXstRM3M6hnELTLzjZLnWKKV5TBQNdDI8xbijOIrkZueoXiK9mirrHXQmvdAlQzajNsOQO0WCPZx3J/0RIx5vNn1V8OyClROWOQ+vxtmUlZ0AjQcoPu8LUPunEXEltu8VqxxiRCn6f0lpIvZ85AVKgCVwNI28O6xqmxzxco8K1FYniLfyqtbTj4W7VqcKDCoSAudBI087M/k5MdnBPGPRc2lXogyFZdIGxnT7baySRm18x8Dbz2b6SXG+71UiBxDqXfohKXKWnVnVJBJluRr3HlZ33l0luL7Fa1Q3ZiFC0odaQ8t272VZs247KBy+3e0Luz9SDoQphjlPbUFXbJATClGPd5RbMo3TmmdPstVtjj/xSZRLy7pc03colp8dQlY7rJbXSs4r/nFdD+57DT2aoUlEsPtlLQJlaoePP2aG0b8Bq29PNSR43Snqrb+A1iAQSQe/7rBN5ZnUk6zHfaudX0mcXtU3WfuXutYQMy4fcTmEbc4nTXW3OHeljet60Sqqs4eM05DmQobvYq5A82h4j2WYcFq2+z9U8YvSO1zfRWPykp3gRB15d1uAFE66j1o52gCg6W1DX3m5Rfvf16W2JzPN17axI5CUpn32N3V0tMIV1a9d9RhHELLlOCpSkhhaFRGx6wE+60ZwurHsJ/2lS++p2VIgJ07pGptoyQAJTsY5D22iVrpMYEWS27dV/NGZg0rSh8HTt9tg19KnhAw8WKu8LzZKVZSVXa4oA66diZ99mPoKlm7CniupnbPCl0yolIJHOR3xbSlrkzoJ0H1/PjaK6DpP8E7yfDLGLHg6rVIXddWOWupbiyh/CC4NIGR7iFdzSgZh5Ljfs7SR3i0ZpKgC5YfJSCqgPtjzUhw4FEJGp0j5387dE6AlJMHSO+zNpeMnCisphVs8RMPFlJIUtVehKRG8lUD59llCj4icPq5IeosdXA8lWgU3eTK58u1aMwyjdp8lIJYjs4Jwha5EqA7RnWI+dNfC3aDBlWgjeNfd52Sm8SYcq1FFLf12vEGSlFW2oj3H51seaqGHRLT6FnX1VAzr+y0Zjf0Tw5pO6MIUJjkByj4+8/G2AkaDSJ5zHOwYWFHMoqgHkeWnz7rbzqSlRHtnX2wdrNylSXC7KgpQI13Ij57raKgvsiIMQdgPusH1hUBI89D4Tp872wLSTChMdkBXLw+MW6ybYXQoUSnPqFGCJ5RbqASQkH8N/nzsEFZk9onXaRbSlgDKVGYgDkd7NJT12DBATGp1A91tpImdRECSJmwanJB1mDqYiPZ5m2Fznl0IkT8+NlXWQyV5VkhRBGxH126U4rLGWRECdefOPIWLFYIhIVrG5EHutoKAjsq7Q23In5FuSoyFKkwYMx3ydOfO3QdITMjXWd/naxUupKYQ4BHLkPmLbU4okk6HcQNY5fXZLBcjSnCFHLJnXbQcraL6SowY7iCZ11+yxMvKMkGdiJPeLBqeBg7aZiD9/s2s4NTSbI/15OhOp9XXbXfSwSqkerJGsgkeU2JqqUnQkEK98WBdcSkSSJkmJjTw9tlDdU0uKU2qkFUyBsRM2MoqtQJ5Rpv32QEVAkSTuYnY+33WNoqJTCQSTqD4e/2WQsShyVhVGVHNzzRz+y201Jgqz66DVREeNkpdSUpkq1BiNte77LcqqFgSBqNJmPk/PiU4aXNZLPpJjRZnWTGtsTVdnKIEDUn9vnZFFWUJ3X+G5+fDlbBWLygKJEHy1t3DXZ0upqgZgx4Cfd899hBWQkBRzSYEzP4c7N9N4HUq2SdwdLCJrgkEAkg6CdPn8LN4SUOulxyrVqSdTvBMGxc1fa3MpjbSfmLJa6wFErOo02mxdVckKKpj3EHTUWUMsuLk5G60bFYynx2+/wDZZTo6sFtcFJnfys0KatzDTNodROtlGgqVnOkq0KfYRZhbZdul01ZOhjadeW02CU8wTq22qT+qD5/VZNNQdCVK8QRobBLrkoGcHNIlUHcfIsuawXZQUrJqGmznaQlBE+qYM+znbT991bSg0m8rxC1HsNU1U6lStv1VCyNSVNXeij6GkJaBM1BEpj+T+sfh9VnHdNz01PsM7qtFuq1WuDpJ+yyCSRxs0pC2Nou4Jw4HZxFW30zX3lf95qZZC8tIutcdaUVCApYUTJEmI087TWlWZIPeARaNsOsmnbCgIiCO/S0hUCyqlb1kpBT7jE/C22wqJ0cIDjclZPEXiSW4Rg620VaeduZ03tokHXe1uGiyriV0V9/K2iqddrcz420TysoFk0m662tkxB00O1uZ1jfythPM2UtDhYpEyKu6MZXJexeua8PTrvdUpTdM6U9YzJnImfWTMxqIB7gAStdj2/LjfFNfWG9kgqdR1qAEkxoMigTJT+lG+swC+K6ShtxJ9RaSeWh0Nt5jBHLeDtbDzdmMQpJXSYXWOYD7LvSaPhfZGtqo3ACVl/HZRwvj5hGleUzUUNSgBJCnEw4rNOgCdBGXc8ttd7LL/H/hXeaKNtm+VMPPN9lBaJE5c2WUSkq0IhJJmzqdwzhy8ad2ovq5KCoQ766V0yCpwjaSR4n3mxSt4ZYCrbiLrGGKZlDlZ6Wtto5B13bSFEpiYDijHfHdallZjkccnHLJA24LrfTZTNdBnblBCY7nEXh+KhdKjF1LXuPtKcFKwoKHViAorUvsjKDsIjczvYxSYQurECHr0pbpaavB1QZWgOFpvIgwkEwUqCZVBynVRiJmyFiPgdw3buzraypr7tFI2haqpuqypbDe6+0SE6TJ0jlA0sxr5wtjK5cNXLVcLb/v9NI2uqadbqEuNPXl16wlrInIcoypVC8yCJLkpnNbyjEKXvMzXtbkcdDbQfRXGdzWFOK8bkwrQ3hVUR4oU1Aad5bRpEVlIhNPlUR1YSWSUhMRBOkWyxFnhRxAUyhSbkwTTgpBDL1DTPON6eqpzq+2obFXM62y1f3Cb3j9F3Ecqp/lJqlLdw4HYJylVdWqAMcmmfvtW3hHTy1dvViQasHTYCYIPsVPl7LWA/KXvuIYwCwVaKdvJZ20hNMBodefI+FoJ4OoHU3XmOdKnUqgn1e2PmbfSPZNpbhsXz/FV/aHWtk+A/BTxT06RCliSVHQcjp32OttgkL0OYgKgRPdbKZtTgGZAJIOaROn3zNjiWwU5yDp4c+/T9nhbcLKBF8isyiBpE6RHz93uES02DGUkDfY/PP3WHLWQyoaaCATvbsJMjMmUqjnHcNzbkq5CJMGTCgY3M68vna3RbkBAUZIzHymJ+fjbaUFaQsDQp1HP2WFSMqJ3BkwTt5zbhouIRco7UIEQSJ7ye7xthbA7MyB7BP4ffYzkzKM6bme/wCdLCNoIWZBBmDI5DmbODim5QiuUpOaSSPOY3Pz52AW1lGYkI5lXLT67HlNhRCVJBTM77eHhzsC4gZAlQgeqdd9T4eXyNJAonBQXxlbUMR0RCTrSDMk7E51bHbn+znLmG20/mWgJSVTSskqA/kpn9tor40JKMQUJMEKpP0iRstRJMb8vdNpbwqnPct2LClZfRGdc2pGQe+zGesU5/qBLLbRCSktid4+v50sMppRGRObX3iY+fG2NozgA6H9I7T7/Z5WNNIMHQeHMfO1pVGAixaHaJJk6xA0+w6fXbosGAmEjaPD8Pvsb6setpGvPSfq5d/O2+pSseoQPD57rNun2RQspST2BmBkyIkn5+qyZiBgm7iQEaLTMx37n3fXZdDZXCkchtoeX32T8QJX+bnNUg5kED2jWzXHRPbuoZviA/eFIyAFVLzWYdyUoknx2A9vKwt7USau6bsZQn6R0IQkqOwUnU+74mylVUiHb2vN7cNU+QA8ypOvwHx8LJrfW1lVdyoPV0ymWTH6xTJjTU6Ae+wGbkiLLm8WmLvvB3rEJy+hdWkct4Gnn9vKxahpkLw0lOTQNqBB9o+42Vr3pGqy/GKdUZUILiwDGbKJA8dSCR5WRk1TlLcLdLTJl94qSkAggDMZPfEWe19iksoFvekeYrlPKyp6911wp3DQz5Up3k6J56WlGiuZijuu6Wm4KlOIdcWUwVqIn2fMWjrEThaqG0OiU/SQCQNc5KZ5xr38z52lVhJVdt0q3P0J7/0Z002P2e5hslA0TuqA1R0q6l0oyNpiTzPd8LI2G7vTUVlbXOJ7a4SCSNpMwPZ87WUa3Ne95Iuxpz6FBDj6gPWAjTv7rdXCyGq+8EtogdZCgk6CJ08LOLrkLrWCFrbvZDK0hKSoAweZEa/V77NaiYTT3GtltA6151QbhOuw115jf2+yz8fZ0IcSAjUSe/ayNc93Iq6n0qAGKaG2UGCZ/WPz9VmOOqc0aXRS7MON3fRpQG0daohbign9L8NvkGyVd11ITiWqUGx2grsAGBtptZ8utAHOR62u0RHP6rILDOTEFQp0xmSo6nSMo2+PyLI4gWSAEopetGKSmUAyErOiQrl4+AA8LM+ouJp3rJPbIzkkxmOv3/GzzvFxVQ6t5xsFIOVM+zv8vmLFmbue9Fqatxak5Gzl19Y/cPttC92cqVrco1SRcGFqZilFYnIHX5JI1AHcJ1377MHiTQht5wQZyp7vaIGvL47i0yXYlJutiCToobxHa/Ha0YcVmwlxxw6JDAJMjXQgTz5GRYkAZLKE3uimCbvReWC3VLOZIW6kwkEkan8PZzs47puZNFcrTKWRnXOVOmbXu+7nvYjwkdQrDdWHmwsIqlSkEHTKkhPnr8RZ+XFTprX1VruVQbgITyST7O7x5eNoyAU8EgXSJd+CmqZpx1xtJfdguKjYcgPf8wLGDh7q1gpSUncHUdoTr4fD7nmtpJMpzZZ0MRHcfstwWQDqjlMQZT7RZ4Y3omZ3dVHuIrsvBN3oUzX1bcKzAofUmRrroQBvY1czmKmbupupxLe6FIby5hXOpXoSJkK8/ndy35Shy715UaIhUn7vbHt87A3TTpN3JSSYSVZRz3mZ9/zpaLhRl2oTxK+2hKJM39xBbJ6vG1/g7gC8HtCO6VR3fCxhOPeKlMOzja9lqTIBcqCsnv0V+3Sx9NGlAkoUMo5wZ7h9Vk28AkFNPTpKnFkiAnl3eQ091mvp4T7I8k8VEw2cfNA1vGjjWwv0a7sc1JdzR26ZhehG8qbOn1xyFlDFPF3pK4P4YP8AEVd+tPULVY1Rh+pupkIdcVJKUwgaADUk76amYXuHWDsLVV6LcxdfTVDTMo61WYdt9cTkSQCAO8nl8BelPiihxNw2rLgoqulfp2WUrYZpFEMU6WxCU5dE5o020sDJSw8UMbGPHRFMqpgwuLz5qLrm6afGx6hpKioNwOl4wtS7vUP0onsuD6uR0tNdzcfuIdXS01RVUFzKW82l0hLLiRqPFZjutRu5V9VdNIslIWy8ojSSnteHn9Vrc4UaaeuWhVCSOqABju05/P2xT0FPm0YEVBXT21cVISeO+LMsO3BdjkCTlW4iRz3JslX50pL8uCoZbXw/YqUOoKs6bxU3lI0iC2fv12sSbo2yjMmZUIzDU+/2j4exr40udp30Z0oH6SQoRqTlPv3sMcNpyfVRPf57XDk6GOmMuP7p4Z1CTuQ3eyVaE76sj5nusaR0yMO//bcCX80qJ7LzKxsO8jXwtEDtzslBIZAygGPDYCPaO7ewCriYUktpbT2RqSdQeXt1HuHs77Kpzy+qb9p1A5qbkdMPASkjrsNYmayzP0DB7+56xtrpecKFrQmqRfVMSDJXQEx/iqMjutXt7DtMAlwMgKzQBlgCO4DnFir2HKZYKSNdjBIPn9Zsn2TB4pftScdFZxHSq4MFMrxPVs6AJDl11I8dwg+491uz0oOBzsdZxBpWCTH01M+2RoN8yBaqFRhencHfMJ7u+JJnx+TZo42wkGsNV1a0oyhtLglKYhKhMabxvEa+2zm4LATuUhxidvIK9NL0hOC9RlU1xMuFIXoM1YlGu59bzFl2l4w8MKtKU0/ELDi5GkXmxJ/zpB+63lGEg3b65BafIAgScw9/6PlaS7mw8xVUDFVmH0jKVlKhIAIkzr3CedkmwOJguHFLHjUh3aF6UMYzwvWpilxJdT0iU5K1pU90dqxxN7UTgJp6xt0aiULBnfkDbzeRg9p1tY6tuFK3UNZjl8LFqjCDaUEMpCNQDoRrPh4ft52F+x2+99FP9sn3V6WoqFLV2Un9bY25XUFITmKgeRmI+fne3me5dd806CigvGrp3ACU9XUrSQdgJBEfstl01nFNyoQzdONcSM7JBRedQgAeebTn4WQYLcev9E4YwNsn1XpcKxROqp1Agmf22EFWcx7Z0G5t5z1WMeLWHkVKVcUcUvPN0xd7V7uqCDlBAAUrXXNrHK3oX+TuvCn4s8ErwruICRfl73biB+jVV1gzPdWadhxCSdDAUtce2wNfQmhh417i6Kp8RbM/JlsUfNQmClOh1kDu7tbFzVEKOU7jYnfy2tZZzhVgF1YKsONAjTsuuDSPBViVTwW4eOpP+4riNZlFU6P9a1H31h3CO4gKr9TVaVH1tI8z87WUaa8EpXGaZBHZPLnaXVcCMD5yptd6NEiITVTA8MyTZLvbgNc4oKhV04ivhioDayyVqZWgLgxI6sEjbQEaeOton1sZUrHC6jV683C6mnZK3HVaIQg5iT4R+zX3H6G4nqpSKm9FglGqaZJ+jH879Y+ER52Gum62LraCAkrd0Lji9VrPOSPHlZXZlXqkkA9+9pQC/ZTEhu6OUrSQEpCQIA0HLw8Pwsu3YySsJkaaxG1kuka7euvhPzp+FnFdjRzAR8/Zytb0NJd1yq2qnsE5LuSG2kjNZ5XW6VUgnlHxANmfTGUiBAO9nLcTuZpSY2AA9k/eLbCmblFll53XJKVDA99tTynTe2K13tqeW0WMQi3NsmN++3JM791tTBIi3Ll0SItsmBE25zAb+y2iRyO1pGi6Y4oOqBVTrAP6M/bbltXWBJB9aLCLAUCJ3nSxWjXNOjWFJkEzsQbJI240XNNkpV9QUIFG2rKlIBWNp8LKFwK6+7KqnIBKDmHPcfhZAWSSSTqdzZYwo7FU8zuVtEx3kEfYbVbaFlPSmLfmfHqpDIS66j/iNdVRUqTUhCH6dlrOKVQIQpe5UvkoARCSIBkmSElMYXneuJr2VUXgL0dSzSPguqUTlQghRypOiQrMWzlEkjWNJtNWObsfr6J2ncqV0dOApKi1o6dDpI1SCY1Gu9o7uK77tve5K3DTqm6NCaynSnqjBUVrhIjaJEaAT3zJt5RiBMMksnC9Frm7jW19bBamN+aJrk3W71xU42lxqVoUkKSrr3u0Dsf4s/XbLOV65eKSnVqoqinYpyollputShLaJ7KQk6pAECDtbLM7/S+63+PkmZ3dfqqDflK3lKvLAjCQMyGryJHMZjTAfPhaIOESMlPc0iAXREDQgLA+fM2kj8oxeSbxvnALgWCHbtqqjKTorOpqR4GBv99o/wCCiSmluUwHEJEoJGkSDoPafaSfG3pHZcWw6G/j+JQOPm9ZL/HJTs0AkbFJSAd9U6x8myghKDB32KkxOnOwNM0mNSIAmdvtsaZbSlQcTEjU9nWPmbbQrLMXXVnLEkwDAO0+dsDeUyBm5d3n9lhIUBooRtBHt9ttthRRM6J3JOvj9R+d0T9kGhCARuZMeI00/ZYRLSSJCJKtB8+78BbrL2gCdtRtofkG3QQQkgEDSBpz7p+drJzS2XCkJzwFKMKkjnHftboJU4EqKozAiQNhpr77CBoklKN9Y7/b7rdJaIVmBIjWY23nnZzd0w+CDU3ngqEGdO4ecQTy3+Ni7gIAKz2V76T32OlsQQcp0B00A19vhYu+2CCUnSRz1Pj4n50tIConaKDONSAm/wChUcv/AAQxzg51a+zv8LS9hNAFwXWSUgmkZBKRrohPP3/O8PccnlMYgoFKJ1pDGsx21TuO75nW0zYNT1uGbqVtNAySNd8g0+rfWzGm7ild6gS0kGSkTlmQI92vz9tjASSN1gyOWs76z887BIaUiE5tteyecj77GUI7MkZikaA6RPL3x91n3XBvRYMyc0kaxr99tmVJkg+7l3Ty/b5gXKIUoFKtymB8I7/nS2y0VplGiSJn57vmLMzJ+RcFtt2cqBr7Y3EWTL/Qk3O4Z1BTBPPUH7N7KxCj9GEazESfLymCbJ+Ims91FRQUwtChBAHrb/VZjz6N04WvZRe6W2qq93HISlCETJ5lG3f4WKqZboLquxxRSkLfDyzqY7JPLeNNbdXqioera+jk5VkPuEJmEIRIT74sPebLL9FdNCFZVP8AVgiCTlyifeDHttXg3RZbZJVGhx68lVdRmPpTLjuUT2UH1R5QLF7roUIo3bxLclbS0M6eqgak+B5/dzWrzdbo71cU4pAS1RBIA3BJgRz1jaxSlQhGF1JyBv6BRJPLfv8AnX22c06pp2Vf8aNGmfphAUVFZUAJJBII8dxaTbtSj9y92VDgBS000RmOshO31fX5Rlj2lqWb0ZqXirqnlpDCTP8AFhIOY6fpEmz6w2upvDD7JGZNPQ07bZgxLmg5d32eNledNE1qkfD9CqnpS++omoqfpFEggd8Qeeo+FtXOgi+bwbkmcpM937bKF2KU5RU70K7TaTBHhMfXZPNQi7b5rHHJSOpCtdJPZ08LMc+wCcASV3ezrtQ+3dlMIWs/SqSnQJnn424w2haWqloz2XZAjfQ2OXHRqUhd4VLf01SToeQ30sFh9RD9Y2kEALBM+Zg6/OnnZmY3BUlrCyPlor7KiACBpBMaj5j77N29uop6p1DZKnHIzmIypAGnvE+7lZw19WKSnLhBLitAJ2+e+zUdWVuq7MuOeyT3WZLJySxs5oJqkdrKhLSUwZIJCdBG+mll6spG6W7FtgiEtkeY9/z7bD3TdvolPnWgdesBR5EeH1W6vQoNE8sEphByab92nzz9isFhcpHalI90Nn80IStMkFUSfHu949to44qsBbSipOfPTkieZk8ueh+NpKuoD82ytZIBWDO++3xtF3FqtLVbQUTPaVUZkqJEdgamfAA6/JsSH+ioS25SfwhS9Ut11K3KGW3Q6sgCNUQAN9Tl9wtKGEFHqn+wSqUgyYJkHblyG3eNtbRtwReCn74pg7qUMr30ntiNtNSbSXhQhVVUJOYCAQSQNjEfV77MGjglIuE40tkKKlJJ5jXfbv8AvttSQ4dBrAB52HSjSAQAYEQIV7Pb8LaWCmVBAJMjXcd+9pC6yjsku8GSqmfBEynMIjSLErga+hcAKR2yozoYIEfV7jZbqGwphYjVQKROsHbu12sl0Tarspip1Kg44cwSPWT3DTn8+Age+xUzG6LKt0NJKWgVKWYSEmT4RrrOvlbligQ0Q68sLeXproBvIHx99jVLRLBNXUpPXFOoOmUGPn5Nh1oSo5ikKIM6HfUfj56+1zTfUpHAjQIi82d0xpGpAGnL5Nm5jOj9JwzetOCYXSuARt6hgd/OzsUiG05jITtGh7+7zskXuz6RRVDIbJSttSTtzGpj299pg66jVUbgSF3MspGrdUVerMAoH3bc7W44bPIqsK3c/CSktAgKM5SRP1HffytUW5MwZvCmdBltxJjnAzAGTz9mk+y1qOCtUmqwVRgFJ6sBIgnu0Bnu15WAn0KPpzdSIhICQFAEEEkzqRJ3jlt+2yNi6nBom1pgqS9Ksu0wdvkbWWkk6QTEwEzHPnp7bJWKGHF3WsjXq1pVBmSdZ+3TzsODrdEnayZi2gV51AFIgrIIPL4crBqaSVShISrLMTPdse7WxolUFYjXuUPd8/bYEhRAlQKl6gmQdPEefx9tn3UWUoq82VdlyIEAnYHed/CfmbF3KcBIjtZlEzHONZ17vqsdWOQhSUiQZn4Hnp8zYFwGIIBnUkyo/X3zr38+VnByQghJTrUFQEJiYyiZPhv9X1WQ8Q0KqrD96UyTBeo3gkAQD2VQJ84+Tq6HQdElJEiDA0J8AO/SxBTTa15FJGVw5YjcEQft++LSseoni6q4gBV31bYPqrbcHcBJEz/hDf7bS9gM+kYboHlIKVJbDY8grLI90+20T0zZaReFIQE/RHc80qB7vA2k7he6pdwttqIBZfcbylcGNDEctVfG0kx9FRxjVPRumCm4U2mYggEyBty2n26C2VNKkjswDIUM3d93n9tj1EwuoUEIbkjcTI56WctBcAaAXVolYIUQRrMb6eVgTJZFNjzJrXfhior3gqpQUN6QojU6A6/tjfay8i6mKJnqmGkJ3IjnJ308fbvZwBlKUZUaRHq7j5j2WK1DBSCoAFMSVAamPMeXus3PcqUMy7KLccUYTVklJIfp4J7jBT5ToLW7/JLX6XLi4iYbcc7dPVXbeKElWvbQ82sxy/i2/eLVWxzT5W6Z1KdFpdQDEweydPOfnW01fkrr5Vd/HDFeGnF5W7zuF8oRJhS2KlojuBOVa9d7BYvHxcPkHzUlM7JUNXqUpWYJWOduVjcxt8LcoJyZCJKTbs7Anut5zmuFoLWReSTHhbhztJIPPvtiuyo+Jtysnv152Fe5ThRBie7VUl81DQBAKusSRzSrX3bixNholXMEDaLPXHF3lS6auCf1mlfWn7bNdDASoEH1iFa99tLhn6WJpXTvsjlGjZSgRZwXahKROWyRRNAkSmRpqO7nZeo0BKQNtAYtrKSKyoaqW6VmBGk+Wtly5HClwogQQfedfsNkNncSJ2sp3a4UVSSRGo+Jj6ja5YLKqebiycWbl8YtoHSdrc7b2TsRYjuXCdyVeI8RV7VDdtAjraiocBKW0yBsASdSBA11tOSGgucdFHFE+d7YomlziQAALkk6AAcyeQSoYBtonTnaL0cfLoqbqGI7v4fY6rLiU2XhejVzjqVNf8YG1LDxTGshsiNbPPCGM8NY9uCnxPhG92Lxu6qH0bzYIgjdKknVKhzBAIsyGohmOVjrnf5dR1HiFYV2CYjh0XGqYS1oOUnkHe663qu0Pomx0OiW8xnui250gA62ju8ON2Fm8R3hhLDl039ie+LpOWvprmoS4KVXJLjrhQ0FaHTPOh7jY3hHjDhLFuIqjB3V3lc2I6VrrnLovelNNUlv9dAkocT4oUrv2s5tXAXZA4Xvb59L7X8N1z+z2KxQmofA4NDQ46ahptZxb6wabiziMpuNU+M0a87FGTlW82rQhZI05HW2Xtet23DdlVfN81zNHQ0TSnqh95eVDaEiSpROwgWjD+EJhJFGMUPYcxYxhh9KVIv5y6VJoygmA6RPWhsyO2WwNRrZ888MFhI4A7/Lr8PFD0GD4hibS+khc8AgXA9o7NHVx5NFyeQUoqka6xY/cTwZvZhU+sSj3iLJVJW0l40bFfd9U1U01S2l1l5pQUhxChKVJI0IIMizMvLjLh258VvYUua6r7xNfV2ht6tpLmpQ8aRJMp61xakNpUdwnNmjlZlRJGxl3G19vH4dUyiw2rrpXRU8ZcWi7v7oBtdxNg0XIFzYXIG6kzFrPWJqmARmICgPMffatd+XY9VYhNOt1TLc5nFp1UlA3ygesrkkczFp0ujiZgviLTXjetw1r7D905qW9aCvYVTVdA4mTleaVqnSYIkGDBMG0QX5TVlXfD1fW0Xo9Lnmmp3ESt5MCFvJUIAmSG+f6UQUKwdbMG1JDNc40/BaCOmnpGGmqWFr27gixGl/qNR1GqB6zEY0osDU66f+8qXQqdUpH6JK57ZiO1z3tlgHUPuOrcXXVClKUSSVkkm2WD+z3dB5lNyuVB/yiwNJjTCNA0tZaYup/shOoBqFBOo5DKPhZlcLr8Yum7rkeXQPOhmmbKssdoyeZjePhZ1flH3kK4rYdZVMouMGQY3qnhvHgfnQsjAraEXBd2ZSVr9HgqA3hShoB5eHstfYPKYMLhcN7ftUGKxiXEJQVNdPxNuwIzLuq8UR2vUQfOO1ZQb4lXCtKZo7wEawGQdd/wBbxtGjJCtU6a5QdPnu91j7eu5OUpCiTr5+zf2G1j9r1HggBhsJ2CklPEXDaiMyaxP6MmnO2uunztYUY9wzmzisfQtPMsriZ28fxtGqiUyM3KSNvYfnvt005+iV6kzESPw0twxifoFxw2IdVJgxxhdaQBeZCjtmbcTpz5effYdvGGGRCjeyEmYMoUnT2jy++0ZIICMyBChtqRJ+/bS2DKVGQkka6xBHd7RtZ4xiboEn2bGBuVK6cU4bXr+e6ZEHTMqPf885sOrEmHlkJF+UBn9Hrhr7J0G1ojGVIjJIAI07uWg3twtKCrKUBM6DQ6azuPP4WkGNSDdoUZw1nIqYFX3cxBSL2oSYkK9ITGh5SbAuXpdyklCa+lXI/wCOSTHM/XaHVBKgVkp03ISPGTp4jvsnVDaEDNlTA01SJA2Ht/C07cZdzaon4Y0+0iXSEvClZxJdrjdQFAUiwSjtJJznQxv+E8rTrgOoZVhO53UPNyqgp9AQdChO9qecTKhulvWlhOUlgmAIJ1M/Dn+20o4JQ1+YLtUlrL/crZPjmSNz37+cWlfihiaJMu6hbQCT0AdlZhtqYKSSN4zfM/jYylpZEZiYgnfXfe0FtLIkocWkgkSFqA31HzrY6y5WIVArqsEDTK8sRt9hFo/txp3Z9VM3Cj7ym1KVgBQRqDJJO+tuFICwlKBBKu0dZHj9loearL0CgRe9fEaRULEHfmfOxlu+b7SokXxXxJ9Z86GZ0nnbhjTObSuOGPHtKXQ24QAok8z47GxDELJF2uSIyqQYOk62jtrEN/7i+q2Z1lyQPePn326dv2+nmC2/e760kgQsJEGd4jebc7GYiLZSuGFyXvcIm+wHKi/KhaZWhtKAQNQAiY8dfq96dTNrqrwoK5ZIbbcSw0kdyUHMY5a/PeoFyrUp4+mELqDlc7I7Uaa++3CPSUBhDdSCmm7SAUJgfM7n3WF7+0qc0buqLXxRprL5dSe0inY61wTroDAgcidfZYjeDgosJBl2Zep1JScw1Vqfx9llNJqOueqetQXahJQ4SgQpO3s5WSb0pFrYQwtSeraSW0djwHj3c7PbXNuozRuAsod4qM9U9dSkkg5RChJ0yDbvOnOBt5WfGE6VpWDWA2hILrJKo3zSZ89tPK0f8YKqpbvOkaecZ6pDSShITBBGk76iJ99nnw9rKp7DVIhCmw0UrQAUmR2j+GnjYh9U0NDlC2nObKpDw04hV0UjhgwkT2toJ1+HxsDWpRVXy1Qg5QofSKJjNl5H552Bul6qoKRNIylooaGxmYOw315+8WEpaNVPVtVqVBS0KKzmJAVJ5fG0Lqxhspm0rhqnCUTlSkdrZPLSIAju9tka6FoavevStUAdrxAmfbod/rsbN4VEJzUrfaAkhcfZYgttSnKh9tspXUAbOTp5eJizX1jDsubTO5onetcqpc63MA2NEjTcRqPnl4WCu1LYvKnJTMgkjkfWj4iwy7sdccC1IGWe0AdY8O42MONgVzNWyxlQ0nLlABJ9pNoW1DXOuU/gkCwS4hIzCAUjtaCDP3WI3mpz0KoCdwhRITryP42EcvRvKUBl4xrpBnw0+dfeXermVoW0WHRmKhIT8dD3WLNSwjdQcB10hsVwpLnMgJcKylAP6Xj3xIsyMaXS4xd5r6pJdfcQ5EicgIGg+fqs+W26cVIqKptamW0yygJkCOZ+eYs1uJV9XexdiVOdYCHMmrSiCMp09wP47WfBOwmxKZJC7kEyOBr039eTedULpkkjlIWPZ+kOfIWlXC5AveqYSpUAGQobdoT56RaFODla1R44qEuGErYdRojUnMlXlyO3M6bWmi5nWmL3cqDPVrCyFAEASZGmp5b+FpHyNaRdNbESE9AFJgbmZKNYI/ZbnMQAnMSNdSYgez5jusAivpFQtLmkkEZCI8D7x8m3JraNCRDoJSI0B1+fnlbjO3qmiB3RCvOBoKJcgk5ROpE76WR7td9JvFanySpCSQI21AJ+e+xpo09U+p+qqUpS0YQkaGY33g8rEruCGb4zuIKWiVJzrVoY10tAZQSphGQFIGFsHVV/IXWVDTjVC2YK0IJ61enZSQfefws9m8GUraRU1NMM5gtsKplAFPfsYHzy0N3PX4dZoqaoXWUAbbSEMMBbfaVt2oWDHnvNmvjPHjFWl27rkqUDPo9UA5Se9tuCezpv7BFguNJNJlBsEQI2sbcpv4weuamqF3fdTVO6AYdqENwkGdkaTp3/ACWRVtB1pTTiVERsTHPfy+fJVdCHUZTkSQdCIgd/z42IvtJghcLOoABj587W8RawWuq94JN7Ko4pxRYgv6hWkyhxSUkJ9WHI107j91rD8A6lTuF1J6zMWnjAmZkq08/x87QPi5hqh4j36lRAC1uq8TISrbfkfOBaX+jlVhygrqYK1bc1EzAATtppqoj2Wgn11CJp9DZTkPX7JkjtHTkR9cWJ34larpfhR7KQoGdAAQfbrYdpQRsoQBA5AjmANuVuLyQtV3Pt5Mw6tSQE89DqPrsLfqiyExIUG1EggpG+vdHP26d0W0qZ7U5USBpz11+T3WwZlAZz2e7bXv1M8vhbTioAUAMpkExsZg+38bdmKZqg3EqIG8SdJ5abew8vGxdwgyEoUpMaeM6+39lhnDIClRMZRIJEDX287BrAGYrVPj6oJ17/AC+eS3skIuibqQUZsxkn9LWIH3D6zYmslJKs2kmQkxHl8O7fSx9aCEFGUzoOzofZpr4WTnIKipslBERrrI3HL3eXdFpGOTCxV3vOhFLi+9qKAn6eobTmJ5lUHbyP4WenBqgeq6SupUKUOrdSsmTpmTEf5osRxrh11jG151yoS2lxuoQkiOsCkJKvrM/Gy5wQdDF7XpRFSQpxpDgB10SSJ/zvPa0k0oLDZMiZ6YBUyXVd7NIj6BAznUqGm2u1lNORCIUQBvmgGfs7rFmAhJ0TIGgmfaBr8/CxpvMlQJ9YCSAYAHKI21+21WXkqyaMuy6AOUApEggAga7fhYrVtIUgkrEmQrU5dj9s8/tsezt5S2r1RsSrfb22AfXnlvNMCNOXzp52exyRwuFHuO2j6Al1GYZXe14FSVajb6rL/QQvcYa6YGHEFeVu8vSaNfKQ7SOZZ8OsSkAd42Fk/F7XWXQ+lMggBQGaDKSBOngeQNmlwevkYY6RnDq+1OpaS3f12KcXtCPSUpWPPKqDFpJm8WmkZ1BQ49GRp8V7hOAIeOghQt1ByyDbKsQEqT+jpbQVmRmHfby0mxIWkGrQUWcIzaT77cLOnibdvHtZgfKwLi0pRmUrKkCSTysI462RDUm3/RmtuuoQkSpsBxPmnX6psx22wTECN4jT2We1wYouHEtZeFDc9cmr9ByB5SEkt9rNolWyvVMxIs2KqiNFWO0cfxaikT+qNvhbUYASWlhUNWS0WK1SMgBKcsD67LFKAI2P2WJU7ZInn4yPGx+nEGT+Pz91txTtWcqDqj7UaAHaxymVkdBHIEj3WItgnU6x3aTY40oBxA7iJ0tYAFBFOeZE2RsY4SubHmGq/CWIWnHbuvJCUVCG3ChSkhQVAI1GoFlOnWTTtkmTlAPiRp9luysJSVKIAAkzyFpiwSNLXC4KbDUS0srZoHFr2kEEaEEG4IPUHZcKXSUFJKurp6anbkkwlDaEj3AAfC1fehvd1WLlxriinYVT3BiDElRU3O0UZQpkKIK0jkD2U/8AVnus4b5vCv4/1tVhLDdS9ScP6R0sX1fDKild8LSe3R0qh/epEOOjfVKeZtI9wXlhOjr6jh9h7qqd3DtHSldCywpCKZhwKDIBgJIIbVoCSI1ibBANqalkw0a24H94kcvAAH4/ALYF0uB4LVYa9pdPPw3SC1+FGx1xn6Pc5zRY+oND6T7DvDWDbiwnUXzVXNTKbdv+8XL0rlrXmLj60pSY7kwkQNhr3m0IcVKR3EnSs4Z3fhpE3hcFI/eF7PIH8TRkwlKz/KhaQP8AlR32k/iJxIew1UUmEsI3ci+sZXwkm77uCoQygaKqqlQ/i2Eczuo9lMk6EcKYcw5wbu9y/cW32uvxFim8aZi875dZUpdZWvKDbLSUpB6tpJIShPqpGpO5stTHHPanZYNaQ5x5CxzW+JO/QancXfgdRVYXnxepu+aaN8cTNS5+dhiLyN8jGkhvvPAa0ENcWsXpk19RV4cwfgNp9bdPizElLR1mUwVsJUJT5Z1IPmkWm69rkuyvupWHaikbN3VVK5QrYCOx1SkZckd2WRaEOmRQ1FNhzB+PW2FuU+E8SUtZWZRJQwpQ7X+OlCf8IWm6/L7u27rgcxM/VoF30jPpq3wrs9SElRXPdl52WG3fajidGfq2P0vdNxTN/JnCO6b5572/tM7OntZMludrKA+iRi+8KbgVfdLeiy+vBldXUrZUomWkNpeA11gKUsAcgBZR6GtKt/hPUYurFl688S3zWV9dUK1U6sLya+1JPmo2TuiThGvqeBV91V6NlleM6+uqmwpJENLbDIPfBUlZB5giyh0NqpTHCeowlWpLN54avisu+up1aKaXnz6ju7ShPek2q8PDyaTibZHW+Po2+6tn2zMAix/uds3eoc9ulpc3y4u/K9vBGsVVP7lOmvg2soEJTTY5wuaa8UDQOPsh8JcI2KgGWkg7gZu+0pYguth6+UIdpWH1OqShAecKQCTvA9bl9XlE1dTVGO+mlhVigSXaTAOHHaq8Fo1DbjxeCEE7Anr2VAHUgK7rTNj2mKJW2DpoSBbN4nBnkkym3pkX+Qv9bqpx5zTDhrXfzhpY836z8l//ANeS392y4Vi/CFMo0ybloHEtHIFhEBQGk+pbLR+inrQkAUpUI0Mq1+NstU/Yzf7Y/RZXuzOv4rzI/KLVKXuMt1sE6N3AyAk7f8IqDPu8t7NXBV63VT3Dd7Cq+lQpNMjOFPJBE6yROnzvrZwflAaueOrTedSMlw06FZTrlU48SPcdbVydWtureKVApQlMq1JgJ5H2HlHfbZ4ZAJcNhbt6IQuIvyV0rvFWZp7yulYSV3pSAnWPSE68vt5c7KNPeV3KEt17Ck7QHEmPcdD95tVZKykhvMO4pJ5zJ5wNvqsIH3FT9IUlOyY3Hn38tO4WmND4ocVduStal1gEAKRKhEZgf2D7tZsO062VQCnSCddPh57Rapvp74dzMuqVk0ASSBvtsJsML0rfWRW1TWb9V5QExHvJ2/Cze4HqlNXfWytkgbZkZgNp194iO+3RPbSSNCd55yBPt027/fVH8/X0yjIi9ayd0FL65B10mfj8kdGKL+bbV1d8XgExmj0lYEg9wPf9vnbhROHNd3sHkrUZ0ntKHtHcfD22CXqBsVLmYHPQ7D2fhar6cZ4padStGILyTAlIFWoyff5+0RGuopx7ixtMJxHeLgGhJfXI03k8pAPyZ7ub+qTvTeisk45odNBJ7Ok7fDSxGqUVNlLaSTsExtry7h42gD932MQtJ/dDWEQkaLHsgHfUR7TbHOIuL0LJVf1QojSVJSTPPdMxpZ7aVwTTUNKVeLSj+dqMzKgyeyqSJC+fj5/faW8BEKw5dhhSppWpB0IOUTpHz5Wrlfd/V1+LadvSq6+obGQqKQABMxAEeOv1Wct18SsT3RQMUtDUo6llsJbSppKsqAO8jYad55Wlngc6NrQo4pAxxKspTlJJBkEbkax9mwjl7LH2YAKMvZOqRmmdPHztW9jjTjRvMfzhSLypEE0yRqRodNxzsKjjtjZmXlqoCEyO2wY8vWn3d9hO6SIptSwKyEAp6vLJ1kwDt9fz3WFUIJUmDodFJjLGsz3aH32rojj3jJsJDzF1GRqVMLGnMDt6jvHs8LDI6Q2LGxldu+6VkACMrmomJ9fx+vxhvdJU7vLCrDIUEKSEpJJ5FIEju8PnawxcAATGYHwj22r830icRuAOG57sUT6wAWJP+N3fPeP/AAiL2ySrDtIoxMIeXrv4GefybR92lvslFSzqp4WtKAEFR3ywR8nvNuS4FISSVSO0TOnz+Ptg09I2u06zDFOoSIKapXuEo+322FT0iCSpbmGGxlISIqzqd49Xw+ubO7tKOS508Z5qai6VAEAqA7Wnd9upsQqlkghKZkxJJ5777crRK30i6fID+5dYM/8AlckHYfoTuY99unOPl0vw2/h6rTnEghxKoII8Bv7LOEEo5JnGZ1SJxqfUzeFI8sZgWwTB1kFW3j8dNrOThW/nwnRE5xqsTt+mrztGfETG9Hi1bNTR0biC0kIUXFbjWdtOX1+NlrAXEG77kuhFDWtvKW2pfqIBEZp3Jnu3sVIxxhA5qBj28UlTtROpQgtgTBmTv5aWPBQ9eD5QAOcjz8u/xtGNNxZw2k5HzVAnf6OY9x5kR36+VlJji3hhY7dW8kmZSpkyNPb9f4BFjuiKzN6p/FacxCEKEkg+J7/h8bbD8iU6RIkaa91mEjixhJ1Kc1e4kKkSWlacjGmlhDxTwm5BTeRG3rNq7tDt7j5Wbkd0SZ29U+FPJ0zKSUjQgna3HWJglJGWZMQSe7wszG+J+FFETeqd9SptYA8J9/12EPEXCsJz3qyiBMkqBOvx7/bbsruiW7eqd4cQSc2VI75kKM/ttwtcnKZJ2MHb5+/lpZqfvhYVUlSfzywMp1BBTMfPL8baXj/Cao/3bphEfpHmOfnHhysoDl2ZqcNQsFEyQIiBvvPKzE4jo664FwicjqFacxrp7yPOyu7jfC6ypQvqjJnYuga6aRy03/bDYxliG6au56ppi8ad1aCDlS4mRrG0+dpYswcon2IUc8NH0s48ZAQ4A6l1oEbA5CSNvC1h6FxuUJPZ5yrY2rPgepLGPKF52oSGy6qFLVAnIR7t48fbaxN31rSiAHG9FCAFgiPHXWxVUSHC3RQ09sqc6HNM60q0ggE6nTv8p18RbpSZBz9kmIIid/2/CxNp8FHZUlYVuSrMYnl3/jYQrz9gpGYyQkkyCdPu91oMxU9ghswBgSZTp90fPjYMhCQewlWXWYBG2/hbgOZ4kSAQAO7YkeHK3KyqB2ZMSRMyPD7/AJDM5BSZQShlKZmerSJ0mIGnz+2wK0NqOqNROnPfb5j424DyTKpCRrprt+yfdblLgEkgSO8fMb/OlnNcRqEpAIXDjdMCr6M7SrnppyNib7TCZGYTJT2FaHz+/wAbG3VLnOozEdkkxofwn287EKt05NFDteeum0fPs0tK2RyiLAoF4n01NQ4zrlNtKQ4tDa+ySNVIA1Hv2/GyxwTrajJeTbDz7RStshTbhSe0FaH/ABJ9gsU4xJ6vELC1jsu0zcidZStY09hFiHBSp6u+7ypVKKM7CFkAHdKoM/4xsdvDdCtFpFPVHX3lmSRelXkKcxzOK8Nfb42PpvC8XUFs3m6pKvWlUztpFkWkUIBJgpTymAdz7tPdzmyihRXBC0582pg+/wCoeQsEXoyy6bp0pPVlxRUkwZ/VP1DWPbbn0FLcoPZMRM92g8omwrQMEkBeSdT495t1mAISkZ9eZ32+0fCycRdkRVVCpxMJfO+vgPDT5iwSqJbR0dUEjUZtY9njrZQUVAICgFAbncjx0tysDMQTA2mM2sd3vtwkS5Aks0CiAAqFaScs/h3/ACLCUV0JC5VKiCTAP2HxscWpKZSoGSDm1IB9vt+q3dMVJz5VDNGndpM7c7NdKQlbHdRRxau8sXwH0pgv0III5QVIge4cvwanCir6rGZAJCaph5J8QClUiRtpGndGlpF4uMoWbrqVIzJIdZJGxAUk/wCsYtE+BKhymxjdj61xK8g7W4UCO7mTPPv8bTxEujKhe3LIFYptzLBSokA7JBgDWNY+YNjrRkCEGBtM/M2SmHjl0EJPvmO4DXysptuAqIWEjMZ0Op08/mLB3sjLIdKwiSkTOqgRO47/AGmxZ10BPq7QDCuXlHh8xbtx5BWDok7etMfjYq8o9WcxGh7RnfyIGnK3Z7JpaEh4hX19DVskHt06zESdjHx+e6E77vFyjvK57yp6gldM4VpVI0IKSmPD7xada5lDh6lYlKuwY3AOm3t+dbQBillTVFTrfzD0d4JUTMTBBJM6j7VECxsDs/olCzNy6r32um82L8uGhvemUC1eFK1VNqB0KXEBQPuNhadcoKI2Nod4C8UcL0nRd4f43xXf1Jd1EjDtFTPP1DoALrLYaWkc1KzNq0EnS0aYz6aGE6x9+7sK3i/RUmoNUunV1rgmJTyQN+9X8028ykpZnTOaxt7EhaOnAkaATb4qwuMcfXFhVBRUOmqrYJTSMEFw+KuSR4n2TaDca8TL0xJmarqkNUavUomSer8M5/vnthO3Z52hiv44YXqFKDV5KlxRUouBRUtU6lROpJgySZsg1fFi4qkdZ+cRBIzTOg8J5/jaWPDZb3LVcxGlgHrAlWc6OmIlVeNrzutTuZNTd/XQf1m3EjT2LNpdxXR9XeDdSgAJfRrA/STpPui1SujTxFuap4yXJd9PXJU5eKKiljWT9CpQHvSPha6OJqP0i7S6EyphQXPgdDa5w5hp5gHBVOIvbI4lpumxTogCI1310saQmBsTG8d9gGUmNSJiT52NIBMb6c58bbWDRZebUoZo9oaD3WMoIiNZsVCCDBEgibDoiO8AWPCDcbJwULhcY03CtdO/X7bJWN8JUuO8J3jhCvvS8rvp7zbDD1RdzyWn0ozAlKVKSoAKAKTIMpURY9dawWVJ3+SPqi0C8WukbxAwXxTf4bYL4f0V/ONUDdcVLfUheUplZOwgd9mVVTDTwXn9V2mgJvflpqgZsWGCPZWZiHNcMpAJObcWAB5jolCi6H+FLupWqK7+K3E+lpmU5WmWb9bQhA7gkMwB5WfHC3gtcXCqvve8bsxNiW+qq+kU7dQ9fVaipWlLPWZAlSW0kD6RW88oi0P0PSI6Sl5SLv6P9NUxqQ1VqVA9irCP8f8ApOUykoqOj0y0pfqhdUpJPlKrVUNRhsDhJHE4Ebeg/wCHRWGIf+qeJ4pA+lrJpXMf6wMR1sQ7WzAdwD8Qnnf/AEVcK3/i69saq4i8QKG8b5eU7UGhvdplKUzKWk/QlXVpEBKSTAAtzd/RTwvRXtdt71fEviLef5qrqe8Gqavvlt5hbrLgWjOgs6iUjYg9xFmp+/n0p4zDo4af9Mv77aRx36Ujklvo4pVB1y1Cz9tl4mF5s3Cdff1H7+Sk/wDq1jIiEIqJcoAaP0R0AFgL5L6DRWQvW67sv27am5r4oWayhrWlMv07yQpDqFCCkg7g2jSr6PWFRcJw4rE2LX8NMAdXcLt7FVEEJMpbJy9aWxpCC4U6DS0ejjn0qQdOjd/pl/fbMPdJTi4viVhrh5xB4TUuHf3SPdUhxx5ZcyagqSNjqIsVLiFBO4cVjrnS5Y4bnYm2xPI6Kkw/tzJhdo6WWSMOcPZcBm2DhcWDhewcLEcirFXdS0VFdtNR3dTN09KwyhtllpAShCAAAkAbAARHhZj35wgw5eWI6nGFzXrfOGr5r20tVtVc1UGfTEp9XrW1pW2pQGgVlzRzs9qAn0NKDqpBKT5g27Vpsdjra1khjlADxcD6fDorejxOroZHS08haXCzv7wJvZwOjhexsbi4B3WuC3D7C+A6e+Ka46d5yrvRxNTX19W8p+rrHNRmdcVqYnQaASYGtlu+6RlbRqKtIUinGZSFD1zyHzym3OEXupvcNqOjzak69+/2WO3+3Vllxlqkbc1UVAu5Y7uXdFsr2iijhpiGtsB0H5fFGQ1tRW1RnqXlz3bkm56b/DQdBoo7dxDX9avJWVqU5jAS+EpAnkMug8LZYVVBeWYw8lAn1Q+YT4Ds2y3m3FpervqtFwYOi8dunfUB/pA1CFbM3XRtankUlUR/hWr86jLWvBzIkyJ05zqe8nfv8rTr04HQvpCXxBSS3R0KII2/uZJO3n9doPTLrrqVAqzGSEAakE9232Tb1PCfRoIf8IVDiZvWSfErG1qSoyNQCNCN+Rjl49/xtigN1KXJExy8ft+NheqSpSCdSDI1Gschz7j5W2UCUoUiSog5laFfIGxpIQtlwHVgxlQTlJgkiJPz4k+y3KioEqBAC5mDIMQfEaWFcaCD1pUE5tpIE/MC3WU9bmPrKJB8o1n3e6LICEhCLJWpIBIUCJJCjlk+zU2woBEJSoA6AmZMjkdp52OCmKoLSRpOhTA7+/XT6j3W0lghZMZyYEHkZ0juP2g27MElkVzElOTKnNsCNTzI90e4W3JKerTmIAOh90fPON7GmmVqCsyypClBOTLMmDv3+PyLcoYnKFJVJACgQZTppAPkPHS3ZgusiinO2opBK5111JnQ/Xv499tFwk50JS2F65Rpm9o1A0ixkMFAyuwduypMwYgac4/HW3Cmm0ELDiRlB7O5MbR8e4WcHDZNLSij3Z7RUnsqKZCvVI2A7xpv7bHG1fQtg65E5T2tPGY30nnz8YsJc9wXnie+Lvw9cFCKq8rwqU0lLTpUBneWQEpkkATMSSBp4WmdroZ9IjKnPw+Da0gZpvOk1MAf8bE/daOeohhAEjgPipIYJJNWAlQmXShMNgyROaI18T98bWCXmWuS2THIfHy8vG08NdCvpFPNKcVgVjKTAUL1o9CeUB3Tn8i2K6EXSPfUAcDMpSYJWq9qMSOR/jO7lYcYhS++PNT9zqPcPkoKzpCgs5UqTppqCfGPDSNvLWeSochmSoEp1MnXY/MaeFp+p+gf0lqlCk/uLo0riQDelINQdNl7fZ3WEV0EekutCU/uJpxoNPzpTGTHP6Tw21Hxsv2jSDTiDzXdzqfcPkoCZrDTLUpTbbgTpC0kSDG3w/ZYNyoU4ou6JAE7aRzjw+G/facKroV9IqhfUy7gjrFqM5UVTSgk7iO1G/ME62DT0NOkW8pDDeAnOsUfWXVshA3gklUD58rKK2lOoePNN7rODq0qFQ6SohWUAjUpM+7y89dfG2lVCQf5MwsjXQkd+h28vO01VXQ66RDLqmXsAvkpUCSiqYWmYGxC+6J1Ni6+iH0hmjH73dZG2j7MbHYhfjy8LKKum98ea408w9kqG2XQAUZpOsq85gRy0G/gPC2IqUhSVlBIUCFlWmknv8NNPHwtLg6I/SDbRnVw1q5M5gX2APW2gKsG/wBE3j+kKUOG9aM2WIcZ7tf09f2a2eKqnJ9ceaZwJvdKiB1xaUASYkmAIzDvnlpr3faPROuBpS0LJSdiDBkGYPjqLSo70TekE9HU8NK86DZ1obA/yvnX2DUfRH6RmY9fw0vGEnTK80ND3jOe4Wcauny+uPNIKabN6p8lFwqAkZwoxGVadCdB4ajYx99tpq1pbIBcTl7EBRMeXd6w89LS3/BN6QKklP72N46Jj+NaBkCI0V9n4hq6JfH0lQ/e1vIlSZSSpoAewK7p277QCrpzu8eak7vN7pUTioUSVOrAJIykpOw8J0t0isW02e0Qop1E7jl7Y28z4WlJ3om8f1oAXw2vHORtnbEf53L26E720eif0heXDK8VTuoraHhGipMiNvjrZ3eaY+2PNN4E3unyUWGvEIlKgoJyhOsx7fmZ8LdJr1NthQASonU6zptvI7v22lRHRL6Q4zJPDi8NSdOtZEAxp6/h8m20dEfpCFJnhxWjSIU6wrmZ1z78tfvt3eKb3x5peBMfZKiY1ThUSoCTr4689e/XS3KqxeUFRSc3aVl1J+fZt7LTBS9EDpFVSi03w3q0HLKg5VU7YG42K4OkWJvdE3j+HQ0OHF5GZkB1uNtj2o1nv8Ld3qmGhePNL3affKfJRT6c46rMZPb0nskjv8N+fjbTlcmAnOCUaAjQqHPX2fC0qL6JvSFT2Dw2vFRQqAsLQZnQkdqOQ19vmGeib0hFZQeGl5gL9WVogR5K25bjvs4VdJ748wm92n9w+ShxmoWl9JzdrNsDz0nf2WFcq1+qFKEjU6RuQfLl5WlUdErpDkZRw0vFOUzOZsRJP8r32EHRF6QwST+9tWme5aJ25jNJ2+u0praS/wDON8wmCmqBsw+SilN41aD1ialaCDGiykAa+Phbv8+3uysIF41Qy6kB0jUDfffbTwtJq+iR0hAI/e1r0jLB7SNQNP1t/ONbdHok9IVaurHDi8MwJUR1jYnx9bQ+r8bKKuk99vmFxp6j3T5KOGsYYiS39Ff95DNqQKlYAOwPrd3yd7DJx1i1IIGJr1yp0CRWOSNe+SPZ3m0hO9ErpDglS+G9eDp6rrcnQ8s2mvh7rcfwRekRmP8A2NqyIMEuNfarxtwqaM+23zCTu9T7p8kwkY+xe08ScS3vnzapVWLMjmN/OxlniTjFAKP3S3gTplBqFDvB15jl7fCbPMdEjpEFWQ8OK4DVQV1jcQNp7Xnbr+CJ0hUkj976rIMiPSGhJO26/jy58rcaij99vmEogqfdKaA4oY2Ay/ulrO0TGaDm18vPbx9oiuKGNJyu4iqlZRoFBKt9tCmO+e+zyY6IHSOdCsvDmoEDMYqWoM85z66792lhH+h30jKV0U6+HdSSkBQLdSyoE76KCyJ35/ZZnHo/fb5hLwKn3Sorv3FN+32pLl83gp9xsFDai2kEp7uyBOveJ91g8PYjvXDV4rr7teCHnEqSr6MLGUQZg6cpkWlN7oe9IltxpB4eVLKXlBCVKqGwnUbE5j8PGx5noSdItbgR+4pMrAn+7mUjuA1VFpO+UgFi8eYSCkqib5D5Jn0nGjFtKgAKoHQkdoKY38TCtBtY+jjxihICTRXYogx/EuDSeXa8PnWy690K+kemP94SJ5n84U5BHf6+vf8AJsSX0P8ApEtEBWASE6AEV1ON+/t+No+NRO9tvmn8GrB9Q+SCRx9v5Ihy5rtJBIykuTMan1rHGOP1eFyrDtKQmZIfVoZO4+3w2O1tDoe9IzN1QwMcw5JvCmgaDmHO4fV3myfi7oycccD4frMX4mwQujum7kJVUP8AptO6EJKkpBhCzOpHIxNm5qN5DWuF/inZKlou5p8kuMcfXkktPXBTqUDrlqYHn6p+fISMOOtOtErw0eW1UNR3+r8frtBSHUk5QoyE6STJ1/bYZFYsEBopzEQSPH267fVaQ0jFF3hynRnjldiwgOXLUJzJ1yvhUc+7kP2aWUrs4uXK9VltN11aCQSkhSdfLXl86RNfkVLpV29RIJSQT4+Hj9lle465Yr2yAolRLYHf2fj95NoJaUBpIUsc5vqpXx/jG779oaZmmpqlDjS1LCVAAGUjSQd5A+RpFdJWCkxCxV9USlis6xeXtaBydh4c/q0su1jyXGxCSsp9XMQCI07+6LM+vU4iuqFEQCpJASZEQI25afCzaQXBCWZ2oKsBT42uaNFvJOhgo9w15R7tN7KVPju4MqUh93s97JmNgIjW0PUz49H6xOYZv0oMRvHdH2c4MWP9coJQsFJAGkElIJPzz5HWwb2kFTNkJUtnGtwZEKVVEDMdQhczpuYtwrFtxFQR6YJUBBKFcx5fhpaJ01JS0CSpJgDxV9vPwskXri677ploLDrwjsIKdDvqeW577dHE+U2aEr5g3VymOrxFchZ60XkhCQNVwQOR3Ijf7u+0D48xNdL1fW3fdig+h6pJLiTKAnNPZ0E6zy5+Js3b5xVed8rUHXktNKVm6tsAJ8J7yLIThJMkzrM2uaWi4Wr91XVFVmFmqVMBXvxq4gMU+DsNYlefo8P0yjSUdQ6ktU7anStQbSoECVrJOnPfazwqOG3SPZQVrcoXRrMU7BnTxb7o+FoOubEN8YZvFytuK8HqN5QKM7SoJTIMeI0G9nbT8eOKlM2GRi6ocTz61ttf9ZPlZlRTT5704ZbxGqnpqinDLTl1/ApyV9xcf6Qn0iiaJA1UmnpdB55dLJDj3GCmAFRQNkmRqyyDOo5RBjTTWya/xr4h1Kf7ovwLkyf7nb1Pu8rEHuJ2K6hHVu16SO8NJHd7tuWmlkZBUe2xiR80F/Qc5S70d8bcQsP8fOHl53tTBqkRiS72qheUD6Jx9Latj+qo29ynWE1DLjCx2XElB9tvnYu7iPiGjvKjvD0tQXS1CKhJToQUqBGvs+Plb6IrjvSnv25bvvukOZm8KVqraPIocQFA+4i1Pi0Lo3seWgfBGUMocHNBJ+KZqGVNrU0sQpslJ05gxYdsDx0Ee2x6/qbqLxUsDsugLmOex+INiQGw5DT2WtKZ2ZochphZ1l2lMkd31WFSNNQNO42DT6w00+2wo1E6QbWAQbhcJSulwElA5zr7o+21VeJN0XtfnS+q7uuVtLtUvDrSkoUvIFhISoifIWtHd6gH0jxB+sfbavF6Vgu/psVVcUFYp8Krcyjc5Wpj4WBxLVsI/wCo39qzeOaOpj/1WftT+u661YPxRd1PSVjy6h9DK6unU71oZWowoJO4SR+jtpIAmyjiTDN93pfV7VNW9U0FOhtTzFcktrRokZU5VSY79BtoedkzhpXXbeVdUYpxHebIrHXCtKVkmCRv5AaAdws5eIt7reRR3RROgNVQDrjmaE5f0ZPdz9lrNuUMJ5LQOJzBJtLfV5PYCqgXVpFPUNN5idQ2pSZT3xr7rJlzvV+Hb4u+9hea3ae8hPVFKQG8qsq0SB2gZB11BnXYB4C6rsRgqtua7K2nqX/R1uKLa0qUpwCZIHjHwsxql1NXh673WlhS2Ktad9Q24jMD5EptLawBUR3Nkq4pN+XtiC8qShrktVFOCqmStJUiEtheWARE69obeNofxje9Re/SK4OrqnVOrZW4gLUZUpMyJPPe01V9QGcY3dXK7CK6mYcUDoNZSr3DLaBsSNrp+k3wvpCf+D1TzInuStSR8ALVuMW4Df8AGz/cFQ9oP6I3/HH/ALwrftwh2obGkLzD2i2LlR02tpwFN4LJVo42DHkbbUNdLWxV0Eaul4U150z+wS6mfImDZ8XhTBTq9PWRaOxAO8RqDaSlLFTTMVIGjjYV7xNqrFIRJC4HopoXFrtExnaEB1Y10UefjbLORdIkqJybm2W8yNOeiu+9leCvTR7fSIxElBVPVUKYH/8AKtd9oeKVOVDwLalDrSM87T7O8xtrrabul9Q1F59ILEdXdzJqqZfoWR5glaTlpmgdU+ItHQTdVI5UpqaRaX+tIjMBy13Ply1tuqF2Sjibb2R+AUdb6VVIfEpvmnXkKurLkGCAI0I0B8TBG/K3TaFKGVQMTJSEQT2dD4b2cbN/XLTJLSKQt9YdQVJAA5c50520mvwho6aR06aGTKTOmpUJPPusRd3RD3HVN5NOtQSCFDIJJjbTczz11526TTKSQrq1qOgEE7xp5ftsvJvDCi/oadp3tryJGZQiPMzz84B7tQ0VmGlqTlC0JX2dwPGN9vgJsl3dEgtvdIwpldYgsqIAGsJnXeT47cvw7S2HAtsAoQQpKiYg6nmZnc8513s7riuiiv2ndqaMHI2soK1LA7USRHPQz91lIYNaSlQTlECACuTIExyjTW0RlINlKIy7UKP1MAkjIc3qaAb+Hdr7NAbcmjAGWCFI3gEcj3coP18rSIzglhJyvJKU5ikjrUyCCRvG/LnsI8DCMCXWohKy4BJynrUanXadz59ws3j2TxASoxUylWiGxKUEFMTJH4e/nIsG9TOskZ21QRA1ISTtM/VytK5wTcwSsOVDwC+5xGUkiZ25z8RYu9w/uB0a3g8kGSdWyQNo21Gnum3NqhdKaZxGiRej07k498NS6UgKxRQQNx2n0aa/jb3KTS0DwStdIwTl9Ytpn6reKnDK5qe4ekPw2pGHnHUDEt2rK1lJOY1CBuPK3tMmqCW0x+iIsBikfHcx1uSMw9xja5p6oQ3fdqCVCgph/wBUPut2qiu1WnobBO2rabAO3oypsU4Yhc6uTpHlYNVQuMuaRpJne1caIBHCpvuhjRXcDpRMeP0Yts0N3KHao2TP/Jj46WKG8UUpK3EKcnQBO/nbhFeVjMYA3iZFu7kLXsl7yTpddru2gL3WppWQonk2JPmbdKo6QalhsR/JG9gzVKPaA99tF+THLvs9lKBpZIZ/FGW6Ojyx6OgQP1RvtYZNFRAEdQ1GsDKLE26pI3V7NjYUVOmpGg2i3Gm8EgnvzQ5o6TVPo7cHQ9kGwYu+iUZ6lHtQPttyKnMYKR3259LExI98+63CntyXGbxQqKCiSonqGwYj1B32FbpaQf3tInnlFioqBPd5CbdekmRoI5gmymm8F3G8UaTT0iUkFlI0/Vty5T0syWUSdiUixf0sQRIB+23KqozKTJ287N7r4LhN4oT0ZhKtUI0MaoHhbpbNLorqUEkfq/GxdVUqPXV4eFuDUnYn+VZ4pPBJx/FDmko4A9FbkfyRFg1UtGAR6OiBoeyLAiqGwB2tx6URry33s4Uvgu7weqFNNSAZfRmTH8ixarpqXqtKdr1T+gNbYqoMGZI20+fnSxerqEBsEymARPjZktH6Oyeyp13QtO1TFxagy3OXSALcqYpS20kstGTsUg2J01QjOiSJUifbNuE1RzI1kJI1nvJ+6wJoyOSJ7wOqOGkoywVGnaO8kpHI+Wti5oKQMFQpm5VGyRp8LBKrgGIVuVHnbp6pHosyJgHTUb27ud+S7vFuaMpoqKP+CsgR+qLF3KSlUtf9ztysgapHvthrQG8wIIjmbAIrEuPyskZQSOXz+Fpe56AWTe8Dqhqqko20t/QN90BA1t2qgoVJj0VnQDdsWLVtUFJSSNlHnrNuzWJUEqAExOlnso7HZMdU+KKu0NEapI9FbgJiEpAsVqqCkfUGkU7Mjc5IPKTYeqqwHisBMJTPgd9LF2XyUhawoKWrvmNDp8Js4UmqTvICKVN2ULSkpTTsBMAkBIB38o+Z52RMb4jwngy5l1985KZDhSy0punClkqMSkRyB3iBFla/a1q7qNyueKihppbywnUwkE6eMT8fG0IcPOK7mN8U0983gZKWPSg0XZbYSsgNtDWDEyd9QTpGhMVAXnwTHVYbZPpGI7wxS8tGCOGN64gdY0RV1zq6eka/kJ6xRSo6knK2Umedg8S1XGbDF3emO8G2qtE5nDSVDTwbSQMwCMoV58trWCw1elB6G0xSMNiESCjQKOu/eTv52PX1eS6dkKdS2lS4GXQH26zYluHQBuZyf3mRzg0BQNhDGOHcbUqwzdrdLXspHpNI8hCXEGNSUntRPgR42PVV0UTy8qaJkRqqGx2dfZzstYmuS4W3nsUMXOzS1obzKrWBCnAnXKobEECPb7i13w9Tt1KSIdAXIiACJ39p+RYOOna5xa0JaiR0QBKIMXbSoqOrVTtesCBlBgWanSCuCgvfgrje7xSMkDD1ctKeqTopLClpjuIKQfMDus9KmvS24qmTJgZdAkDNmCs8xmmAU7xB2nWxHGVMi9sKXxdgSVem0FSwZ7ltKT9psTFSBjwQEHJVZmELxDyozRlABMGPf8+Vu0FJkKXlKSEzt+y3S6ZbZUqTIPbzaAakCfaPC3fUuOuFKECFJ2SnU6/O3ebau4WbsQtoQsq7K0kJG8GFfMH32OXWVNVjK9ey4kZkjQSR7PhM2AYaXOYtgkETmE6SBrJ+fqNttobh1CEqhWaTECNhrvpEx8LQSG4IUjBrdOqoUkJCFAZldoBSO4941nQRrz8bNq+lKFelWvqCOzObU+3bcn42dVShPUhLZlKtFApmSNBz8N/Dvs1cTEBLS0pkZVSdDOog76alVgqQgvsiZtG3ThulxTlEw4mY6tHaSqSI09sDT2WDvK9aWhZ9Iq3MhjsR6yk8oHOfsHdo00YmXQ0DNJTsJ6xufpFiYkkgRz3O/usi1L7tQ8XX3C6tWsrOtimUWZxLtkO6osLDdLd8YxrK9JZoppmtZP6Z8yPKzdKpOqjPjbR00Bi2jMa92gseyNsYs0IR73P1ctiRA+q2lAzrztuFd+577bdggGd9Yi0iZZcrGoKREgW0RA0t0QQlCtYMgH222pGUfOluXWQYJ7rYdLbg72yDtbly0CZmLfQF0Q8TDGPRi4Z39nKlqw5SUjipOrlOjqF/5zRt8/p2t7UfktMTG/uiXdd3LUCrD183jdu+sFYqB/8A7PwtT40zNC13Qqww11pC3wVoMS04XTt1ATBQrKfI2bsez22eldT+k0brAAlSNPPlZmwQdRBsPhz7syouqbZ110AZkEeXIGwiVEka6+fz424GU94Bt2BIIynxJtbtVeShmHOrdCidBJPs1+y1ceLNy8Wrg6Rb/EjBPDOoxNRLuVF375Wl5kFKxIMyPttYwQTlWeydD5bGyvRLU5RsLX6ymkZvONfjaKppBWMDMxaQQ4EW3HxVNieHjEI2tzlpa4OBFr3G24IVPWLx44Usik6NNewDybvapSPguyne2NOkZe7VK070dqxj0NrqW1M1ziTk0gHXXb67W2BMET523MGBvaIYZN/7h/k391A/ZlYf65J5M/dVSMP436SmHatdbS9H2tfUtBbh+rUpIBMnTTw91k5d+dIwVqqik4BV9NTqWXPREPhTSSTJCcwKkg9wMDYQNLXJO/K2xGwO20mzvsua1u8v8m/upv2bV3v3x/kz91VJxDjrpKYgpaRlXR3fpHqEQw+w+rMEkAFJBkKSYEgjkCIOtkXCmFeOOKuNuB8V4v4W11z0dy1UPvjVCUGSVHXTX67XREATMC3aVHfSyOwd8tuLO5wBBt6PI3GwUb8GlmytmqXuaCDYhtiQQRs2+4RSsTlqad3vKkE+Yn7LdLSDuAD3Rbd4QGkOH9BxKvjB+221iAPfa4I1KvroEiDrpZ/YfdFTcNOZkty2fYSB8Iswle/7bPDBbpXd9SwZ+jczD2j8LC1TczCE9pSiUanS2WHKJJNstlO4tRWZfNFT1K6Jltpt1xJkglKoUdSQJ9u1sXUv9cpDqHErQpQUClPZ7xp4k6eXOxK60VDzNOnrJUox2SBB2A8/D4WcN3cLeJF+3k63ddB1jyEKdOaqbGxAkknUyR5zbStgLnWAURlyi5KSkVyycsOnMOzI1B7vkaxr4CtXgER9C4ntCQpAJJ0jx0J+d7OQcAuMaB2rmQkTJmtZ3/xvDc2NMcAeMClKSbrpycpImsZOu4O/gfmLS9zefZUfeWe8mmm8CpKSWTGoJLWpOhAMeUR4c9rdemJXCEU68+x+ikwRHdB9/us5TwF4xpQS3crJCz+hVMTIn+UJ0twjgpxjaRH5kEAFOlQxzM/rfM2aaOTolFSzqkZm86ttBpaZ2qQgystttEBR/SUSnxy8h52ERfl9BWVNZeOoKRlU5A125xy8oAtJeBuB+NaihqXMQYb6yoQ+OriobEjKeyChWusanxs53uBt9IUlVJhVTYAgpTUAkmU66r001ncaWGfC5rrZUUx4cLhyg9q/b+efSUVd4EqTlBGeQR+IHd7zZQaq8ZrQDTpvPUAylpYKtTuAD4+E6xaX6fgliJp1KlYbqGgFDUVCSdSZHreAHvsfTw7xdRLWmlw66AkymMquW2/z8bDSCQerH9ERFkPrPULNt8SnUFLVDfakCCZYWozGh0T7O7ffe3NQviTQNhdTdd9ozGVLVSLlX+bpqPdNp2RceNqVK0OYXW5BmSyAdSdsvMQPfYk7duNy6Gqm461IcGfSnUlIEDaDpyA8rQNdPfWPRF8OntpIbqPeB9ZflRxzwCm8EVSHEYju1MvtFsgekNnmBvJ+d/Z9tw5QZiYOlvIzCN3V9Dx64fP3hQu0q1Ygu4gKbKCQKtI5nb8PG3rg2QUJGWFRO9uqWZ8ptZMp3ZMwBuulKBIkkkD2zbRrW0CFDYfP12CcDmQlKTA593t91iS+sUSUqMd8bWayEbJJJrbI0XGXF5yRpEn587dhxnLlGvfr9Vk9CHnVFKElSokgC3JdIUErkHnPnaXu4OiiFQQlULY3UOe9tlxoSQSn552S0vAjQg/ZbC8JIzeR+23CmC41RKVBUsb6jwjYWwVdPyV4eNkguiCQeUaW5L5zRmkToZ+yzxStKb3ohLaqpkmDJI03thqWZBBJ/bZFD8kgK/nTbA/l7RMjeN/k27uoC7vRS36UzMQTtFtiqZMzpO+tkTr5TBM8+WmnfboVAAEkgefKyimFl3enJX9Jp5g8vrtpVVTzqPfZIL8GMxHnbXpIUrUieXl8i3d1au705K5qmM0jSbc+kMc0qEaWSuu00CtB9ttKqNgJgxtZe7Bd3pyVjUsblA27/vtyainmMp1nnNklT/NSt9d9LaU+dYMzz28LKKYJhqTySqXqYfoDwk2LV1RTlsjJuCBBsS9IVoCrT4D50sXrqgKQlI38/C3PpxZObUm6MN1dJ1jX0cfRR7rcN1VN1JWERLgG+mlkj0rIhl0aDqlG2m3h6ASDsuTqNdZsOaVqlFU5KZrGSIy6yTqdLCrq6Y0olP6MjWO4bWRS9nDywQYlI+36rb60mlCBBCkxE77WTurU7vTilV+8KRLIT1YTmHeZ0tqlraYuhXVqlSTqZkWQn31KVAPZAgKkae+wyXUtViIOyMuu234CytphdIapwSzeNYwGhDegIFhE1bCkJhrSO/XbayHXOyxvqDJBneeXz42ENSENhSlQEp5Qdfn6/G0nd2hR95cjVVWU61JZLM5vEz5fH5mw6XqZOUdUMqSIT9sfO1m87UjMlaSQCqfWkkA68/Hw9tlEVAJ01gwRJ17/ALDZRTtSGpdZD3s5S1CEtrYRCwW1CTsfH2fXantDcl8XXxRurAN1XLeV2tmuUp56opSwpdKhRCVpzpBUFGQFJkQdDqZtpVvyhtSUJOU6JUJnw30/GyFe2F6C76tjGFEmlYuq7G0tpp2W5dqHXlS+44omcyQEqEAeqSc2hEUwMTSQEdQ2nd6RTpvaivdFzihuZNTkd7Dy2qs0y8kAR12vVDQkkSoxAHdDvDXBt8r4hVThprso6cvH0hyjviovNxbMkAKefmFRrAAiY1iTNVViakw/dT9a46l5FM1m6pogF9WmUTt7Sedo3quJ7tNTVFXR45wld96FyU3a6yl5hLY/RCUKbUVGI6wkiZgQItXNe1wsVo44XXzNSDju7cV0HEKsu0O3k83TKytqpL1LJSwvtJdUyew6M3ZggkQdRNpWo1Jprup6NISSwy23sCAQmNuXfaJmOJV6YnvH87YjorgcfQpLIRQq6zIIEKCiTBEz5EeyWWmFuU6ag+tlTI2O28Hb8fc7Dckkj7ckFj7XQsjzbn/wki8FlT6shCe8AnWf2WFgrSG3CQFdk687Gl3VUVIeqaVJU2ynt9oAzBMAc9EkwO6wZQAwEnWEgiRGg/H52tZcLVZ9slwvNm8sCcHLvvu8LuqnrhFRTVbtO62qrSlSVIWpJBHWCCIiPC3bWC+CRICTcS0ntDLeBAPuc00+drNLjhwLvV7jDjN+kqHEt1F91j7Tfo61aOOFYggbDONf22YrnAnEAWA1UAhR0+iXMewdxFhXQQk6zkFHNmlA/mAVOKeG3BVxtB6u6w256gTeSiFDYH1/EWOU/CfgzUFSUpoxlBKibzc+wz3aedq/jgXiNIn0tM7GGHCAqdpKRH26+0B3gtihqR1zZJBgdW5qRymN/nnZDSxu2qD/AB81IKp7dTThWcVw54QFPVIabUvKEgN3g4pMwJHmALU5vEXjW3tU0BUt5bS3G0Ib1AyzMAeCd/bZfqeEWKmE9YHGsnInOJPd6tmZX0dVdN4u3fVAh6ncKFiTuD77HYbSMgLssmcn6KvxGpdMG3iyftT6uHC9w12FKw3jcl7ov70gpp3swFOGiB6yT2pEL8LcqwDd8KQVViVA7Zknu7k932d1m/hPCuIMV+l/mZ9I9CSlbnWOlOipAj3WVhwyxkAmK1gETAD6tOZG1iZHhjiDJZBtBe0EMupOwLwk4N4oraS4nncUpr3mSVuFbKGStCZXlhKtN4m0judD/hS071buIL7ZOmgrGCU+ctfOlq50XD3iOw+HrvvMtOgZQtutUg+QOnf8bKbOE+M7YSGcUVreVOUAXs4nKBBjfTb4WrZo5XOvHVWCtaeaJrbSU1yp/HQ14WOphGKr/SIkS9TDX+jn7dLMDjx0ZMIcMuH7mK8N4jvKtqWKpllxioW0pIQuRPYSDIOX38rMsYd4/IICcY3r2DmEX252T3+tpYO8sI8dr6u1Vz3piiur7vdUHF0798KcaK5mSlSoJ57WbC2oika59UC0HUaaqWeSmlicxlKQ4jQqJMgLQOYSFRa7GEOgbgfFeFrmxOjideKfztd9PXdUmia7PWNpWUzn1iYtWZHAziMUqT6AwAVag1KNSNufj8bPq67p6UFzXdT3XdWML1Yo6RAap6du9DkQhIgJSJgJAAgbfG0+JzOnYBSVDWEb7aoTDImQOcauAvB2Uy1n5O25EaUfEW8lq2j82oVKuWocsmO/k660BTiMe1aECRmXcpOvd/GAWjmpvzpf4coKq9v3bX82xTU5cedReIKktJBKlbzoJkjWzPd6VnSHfc61/ipfLqoABcUhcR3SnTblvauhp8ZlF4qprh8P/CsJqjCIzaWmc3+Pipoc/J438IDOPC4FAkTc6xoN/wC+cudr1/k7uEt98EcD4rwLe17CvbfvVq9WFCmWzlLjQbWIUTP8Sna3nlgnjB03sa3Z+e8H4lvSvog8prrk0VIUhaQCU9pvQQRptr52uf8Ak+cYdJ68uIt/3Tx4crX7qeuYuUDj9PTNhNSh9EiWkhRJStW8js2HlOIh3DqahjgNwPW/BPAoHMz00D2nqdvxV9BMnebNC9Kf0eudbjQqzDXkbPA6c/dZCxFT6tVAGp7KvssfQuyvt1QFS3M26RQkK84iI5WETA184jS2RsInn4fOttyDzIjebXjVVlbAIOpI5WVaNwqZjmFKj2mR8DZK1EwD3Gx+hWOqUgHmDp7vstPHuhn7I6I3AO1tgkCDPvtwDA0JEW2lROg591iVEuxA3M2wH4W5HfPhpbczrGvfZybZCA7yIFu0nWJFgRKdCLdg6A+6z0i5rkFyleA9bIY05jUW4YWHqdp0kypINh1EREEjusUo4TTdUZ+jUpG/cfusx26XkuliBFnFgd/JWVFPycbzecH8TZvOa9+/OynhR0tXyySYCpSfGRaKUXalanmUKk+dssbLIJm2Wpu5lS8QL5lrkYHXUTZWn+NQUpjmVD3fO9rO8Lm/SL+rEIUhA9GUvtAiE9Ynnr3D4eVq6C6au5b9bu686R6jrqKoQy+y611bjawsAhQIzJOvPu0tZXhQwj841z6nUSlhKMmUjQqk8tdvP7Luj1ehqrSMqRl0ymkCXkKQohOVKpHhp7Y+dQkgodCwSkFWbadAe/22O5GjIU+kSJJPdPM9wtsMtjMvr0AjSM3fvPz77XIJsqkFElpVmKOrJ7u1Aj5+SLaPahRSIJEkHU+z55eNjr7DByu9eg6DUEyoj2biAdbAraT6ocSBA9UwD5fHf8bIuOqVsNFJp3IBJQ6kzEAGPd7/ABstKUlStOZlI57DSLJeHWULYeQFBQDiBGsDfusrrplpKSEnTUcpGk+A52Dl9ZWEHqIE6HOAZBJH+d3WDUFoUHEFBUSSTv3b/XYwpGZtLQRtMRprqdvP6rBrCUqGZBEaiNdtv6se+0KmGqKqzlPWTOxPPSDpHtB+deHRnhACQlMDKPMc/Z9VjAZUpvtpXqoJlU7aTYvV1LdHTO1VUshptBWoxpAE/bt9dkNgNU8AnRQ/jNeTpCcJ24TmTfVGpRTuB6U18NfbHhb05aWlSRPtn67eWl6M1V+9IPhstbop3q6/KFPWuahn+6WwgRzjfQbydzb0vYo77SjtXuDm1B6n8bUVa7M+4VnSgsaQU41V766VVAnL1LmqtNZ05+6xZTUJUkwYGlkoUN8r7P53jn/FzpYYUF7BKovRQMRAQLRMJPJK8Ac0aYrnbqzOsZesUMozJmBZHL61KUtapUo6+e/2WLVd23x1suXu5kKjACBYk9d94EQb4fBHOB52LbtayEcdUq+kjfOd4gacrZ6TmTmSonSY3tBnHTHuIeGjlw1FHiGpQ1XuPtuo6tBz5Qkg6pn9I7G0YL6SWLgsZb2UtMgqJCfZACeevsFuLg02snNaSL3VvVVQzE5jAPybaVUCdCSOU7/G1R2ukdiEkhV8PAjT1Uyd9+zp52EHSJxFplvqphURCUT8BpNkz+CW19iraelCDv3RI091s9KSRJWZiNDGlqmHpCYg7IN+1x3TACZnQydPO3P7/l/k6X5XJEwRKQfq1snFA5JAw9VbYVYTJzbTr3aW36XJgFMjSItUr9/vEJ1VfVcDzBWNOXLyPusJT8bcR1BJVf1dBMlRd9X3b2Q1DRyThE481bH0kd+umotyqqIEkiPDQzaptVx0vkrIbvy8YEiQ+QZHPw/bYNHGu/HDm/Pd4kAgdp9Wuvn86WTvDei7hHqrb+lDLpJBmYtwqqMiAdOfstWq6uKtdVFJevW8jO/91OJAkaaTZwIxy5UIStN718EQT6WufHn8+2w769reSmZRudsVO/pJVome4W0qpIEAkctpFoXq8UVLVK04b1rRnTBIrFzJPnYqjETzrTrn54vXsjQemr79t7RfajLXsphhrzzU3mpOYJy6gnUGwNa+eqChzOsj58LQS7flZUgIaxJeDSshDZVVuqST3kBQ0mPgLA3Xft4F7K9f1dUKTqT6Q6E6A65Soka+P1WjdjMfuqRuFP8AeUwrqlLYaEGZUkztE87CGpUKR4AqgOaGPnvtE7N/1ZQVGvdOZaUAqql6c5/HwsWvrFL9K6tCLwfhSiRlqFwQO4H7fC0AxmIm2VTHCpAL3UwNVLqg6QFEFRiIkfPfbbL73oqQGlzBmBpPh8LQV+7Z5KwTW1ahBJmoURrtz8/dYwzjh1bcoW8oR63Wq1PlNnHF4x7KaMNf1UzNLecfCVpOVIBUY+HumwjtS4mtbUlCikiJI8TztA95Y4cbQULqHVE7lLx+wxG3usgrxs8FHrKp0g6hRfWSnv5/OlljxZjvZUb6BzeasvXVbpYkBRGh7Ovz8+0u5WuOhqlQkzlEkSCPhyk2rc5jmpEg1DoEAQajaee/n4a+diisdPFJWX1KjQ/TGY8ddOY93dYkVzTyUZoz1VnKuqU31KdQQYgAamd9/P42OGqc2CVTJ/Rkk8oEa8+VqnuY4eC0+KgSAuTzkROvL6/Iocfg5klTQ7WoLspzCNDl8xp4d1ntqwdgmGkNtSrY3jVK6tKOrUSCTlidPn5m0YdInFOOcKcKKzFmAk1ybyo1MtOpbV9Aqld7LhfaPYWhMhUqGhjkbQucfKebR6KqlKEuQpwKCs22m8TtAIkk6TZJwXxAXe3GLDdy35Rld01FRU00uLUEuVKqV7q18spQ4Oyc0g5FbgZTqNnepWsI0JUT7wC7XWUhcNOk3dVNV02BOId5dckOJo7tvutpU0iLwUhI7L7SVrS09IEQtSViDKVAotYWmum4qu7n7yuNNys1lQVLdqWFimdWqDqVogqXHOTy1i3n3xn4RVmGGvz4hqrv/C15P609SUJqrsqCtQWlVUoZQDOinAcxiVAkFTIqOKd93TgasvHh9jpeH03VWCiNy3nfSamrU3onO0y6ypRRtIS9CcphMa2fV4NFNmmpHAdWnfxVhTY7JRlsNW0noQrlcZMYYK4E3NX4/rKS7K3EVar0ejS86p+oq3xBS2palFXVggkgGIOmsWsVgi+7wxLgm579vm6E3PX3pd7FVU0CnMxpXFoCi2VHXSeYnaRbyIwRi6rvPiBRcQOLb95Y4uvDdfTuVCFV6ghbRchOQOpgNhzKVJypCgIkSTb0mwN0sej9ixLVI/eruGnXQerF9UyWGVHfR5BU0lPcVqTNgKeg7m3XUncpuI4ocRcCBZo2Ur1tXUU5S1Q1DyAtJQvq/wBME6zGnz7bFA64KcJiSARvt86/Dumxh7D1FedKl+62aaoYeb65C2yktqRHrBU5SmI1mNRZHaw/StqcbVRoSpCihSVIAIVrO/t9xtK4EIFpsq58V0M0fEW+erShYcU2tBJGVClNoJMeZ+dLNgNuo+mWwnKpcpBcgkcyNJjx2jS0h8ablRSY5SmnQ2yl672nP0UgZcyefgke/vNmQzlRORkun1UqX6qRqdvIWxdaSydw8VuqFvEp2O8FwwGUMZsqkuFMZddD3k79+n1RY+0y88BUVTSktoIShSnOyTH6IAkzPLYiwbNC46QFZoVoMx7I8/f8bKibpfdKWW1LqHNobT2QPAnU6xyje1c6S3NWbI/BExVUCFNu+huqQ0DqtwBAOkKygR3eEWo30i6Yt8X78fBCxUll/PyUVMokjwzTb0Quu4a+7orEU9K243Kw5UNodCSOQCgUknfafK1Lem1R1I4p0F71Csy6+6WitwNpQFLQ64kmAABoE2u+zdResLL7hUnaOmPcw/oQuehdT/nbHOJMKh3q27yuV2SVCBlcQATO8BZ21+sWqHDqqZvKjvOmrqZS6d5L5QloZM4ZQ2pI01kBe3MjXQ2qJ0QHeo6QF2XeQVC86arZEHn1JdB2ndA+Bt6T3XhVyocSOoKkp9YRAjQmTt77Cdq5zTVgN92g/sT+zdLHVUfpjYlR5hPCDdF6Q7e920ZrKqsVUdY02F9WgoSnKMwEGRMxz0I0s9LrwFh6rp3PSLqp0SkhdQ8wgpGsgJj1jueffG8vanw845LDTaEtKjsNeqozzO6vb3Hay7TYco6ZCTUKUtQ0DSNwPHlbEy4iTqCtWynihAbZR5TcMcL1ZhFz0riAqHHl0bYA0iZJn7fCylTcL8IUzwSvD91VCyg5Qu7GyMxMwBGo1PLntaSGbucXlSy2pISOyVQVDy5D3WWKHDrrckDKs76ypXmfvsCa6aTYlMkmgiF3AKO6DhXgkqU9eGELscWoyEqomgT3EgDSy5d/CvBbkBrBtyBS/WWu7GJHl2dB8bSlcuBKyrUFKYhvfOeyLHr0vXAGBABe94N1VZsmlY7SirkImxsGHVtQ3iyuyM6k28uZ+SoanGYc3Dhbmd0AUW1fR3wjim4rzuNODrrcF4UjtMSmibSBnQoTmjQidxbw54jcOr44bY0vPCN/tIaq7uqnaJ5CVhQS6hRSoTEbg+630Hfnjifjin6nCtzN4buxZKRUVQKXCn9ZKBr/AFfO3hd0trkvXCPSBxdh2/a30ytu++ahDj6yCXQXJS4ZO6kkK1J31t6H2SiNK50cZc5pG5FgbdL6rL4tK+obmmLcw5A3I+PJXY/Jx4wxVxM4I3rwlw1wyuBxu5LzR6XexZShS0PjOFOqOYqcBQoAx6oSI0k3a4b9Hd/h/UuYsvjEj1VeIQsCmYQGqVsK3ATurzJ5bWoz+RXxq3d978UcCuqTL7FBerPk2t1pf/xG7enN53z1ramZ0ULdiUNFS1r5JSS7Sw5BRQV9U6EQRaN5+KSBttBsTvVgv0LgAEohY9ljk687cmFDKdiDY+E5SCpni4ITPHfIM+NsEzI0BsJUtFp5xpRPZWRPtsHObmfsFtEw3F1TuFjZdJ1HMnx3+dbHKBXaKe8H4f8A/Viknlvy52GpFAPJAI7to0g/haZhsUM7UJSkzEbG2wqBMR4iweYnUb23mAMRYoKAi6EzTKd+VtyY0MzYIKMzHO3SVnvmzkqEBnxt2lQ5aCwIJMEHU+y3aVH36WeExChexkg2JUiymrrGFfouJWPJSR9xsbzctLEzKL3keq8x3c0q/wBqzXjYpRsh3dBJ94sPdLparmnR+gsK3sA5oND7rd3ef7oB7zZHC4XDdSoNRNssn094I9HazETkTPutlg06xXza1ldW1yUXreda7U1SkJdeeqHCtasqQBKiSTCRpPcOWlrE8EalNZSVF4NjR6nYVBEESCYj2xau1Q0BcqVIKM3oxKgTEjLz35nyk2mjo1VqqnCDruYlxtSaYnmCCSO7koWLoxaRQVRuxTYX3IELymBMzoe7577cqfABhIgKmZyifn53sVC1ZsqFdo6DWDr8/XbhSioFXJJCcuna89e+O61wqvKlBp7O0pIElKgoDLpEa+O8eVgQ4ZJToVaCPDmPwsUQtTTiuwCFdk6nXQfED3aWxRyuEkqVm1PMEETGvkPvs3ZPTpwpWZah6lKAS4AtMnsjKdfdmJ9lnGt5BIQRKkJgKneCk6++LRzdlYmgrWHgCQ2qSMu456cjG3jZ+JEJCULzJIEK/WHZg/VYOYWKNpzdtkYZeQlYBTlAgiQDyj7Rv32AcLagcqSd1eB0PdpzNu5SptJA23G0EkDQbbW0okqMqVCzt7ht7dtrQ3CJAXKNV5VHQkk6+J+NkbFwWbicQEDR5C4IGna090fVzssKTlykrlJgjQbafbP32SsS5FXO9mMJJQe4ntD4amzJD6BUsfrBQwqo6nj/AMLqk7NX9QyAT/5W1r48/mI9SWmzEwTAJkd9vKy8xPHDhrCyIvyhM5Y/+1Nn7/f46erbKFFIBItTSi9ka06lFlEtANopsyyZzz6u3426AVlUQdTsPKxlTQ5jfwtpxPYVHdz7rLGEyQonTO0TbsXowp1oDRKTBmyGtCc6lJRAKjlBGoHj42VHEHMqZ37vnxsWU3GgIjwsWBZCOdcqr/TOuervahwkzTurbisqc2QAkjq0jW1bEYBvlRSHKys0jQhJGnmO/wCdTa+XEfCLGLb2w9RVMgMuVDiSP5qdPaAbcUHA273HkPFglJOaQkA+AgCyinMhuk4/DFlRtvh/ea5HpVaCdeyUaf5s+PtseRgSuSkFL9adQEyEz7Oz86WvYvgPdHZyMqlMASmf2b2MtcCrtQZ9F94+yz+5lN70FRBOA7wgKz1hAJ9bLrvt2R9o3sMMB3kBIRWaEQSoajv9XTT6rX3a4H3OlWdNNB7xpYYcErkGoozAOmlmGhcnirHNUNoMAVz7qQ4mvA0HrxG22nh86WWLz4a1t3ummoqatW2EDVTvrbHu03td5rgvcqSFeiGec62Wn+EFyuHrnKRJlA3+Nh30D8yIjq2WXnW7gW9kdldNVmdD9JEjfXTyscoeHtct5KTS1W8CHDEfP2WvlU8I7mKlH0AHcbC2M8IrobMijA8I2Nn/AGa4jdR99aDsqtYS4PUFWEu1jN4RzHpSkme7b472kSl4J4VWUtm7K3UDMfzk+Z3j9OLWDufh1RUx7FOE6abWWv3F0uYq6kaxBCYtVT4TKXaOR0eJRjkq/PcIsH07TCBdjzsgqBVeNQSPI9Zp32DqeFmFGaJ1bVzViVohIT6fUZDOpJBXrv8AMWne9cE1DgSGWTGUpzIA9/z328z8cYZ/KNqxbfbWG2sYrulNc8KVSDShJRmUE5QUhRERB3587BnB5ToXW+KKGJR7gXVnFcMrrFQp1qlfaQluNa14wY31UbJQ4cUCnVqQ2oqAOqap2QANf0vq87VjZwB+UErnW/zjSY+cLB60Q/TIA0jSNiZUNOWnOynUYJ6Z71+XXUsXBj66GGnkekekX0HUOjOJKsx9XeQI3tC7BX2/nAimYq0ewrCU+BbvYyyyZaX2U9e72fH1rFMSYKYbeyhgrScxJLy5Oo1ImJ+eVpfprpUulV19OifVHn4WCxDc7SAtS2EICNASNdtT8LVow2QPuCj++sLNlAzWC0uuhKqELBBkdaswfOZst03D261MAm6wrLEErXy07/C0lM3G2XlFCUFaWpMDb32W6e52/RUFLQgAJJjn42ZPQynYpsdXH0VfL6wDTIOVq70pUNZVr8D9dmt+4pgupaXdzJzqgAtAwDP7fkWshfVyoVVjs6BJkH4mzZaw8isvgLQM6Ge1qoHcHl862mpqCW2pUMlTG52yjJPDG6iygvXRSqUondoa93LaLZ+9fdYIBuqmCkx2g2JMnx5/Mm03tXIHa5CVICUpScumk8iPOzd4q15wPge+8RslHXUVIvqzkg9coZGz7FEe7zscykkJtdI6eMNuQq9YkrOG+FqhdFUVN2msbUQthttClBUwrtbetAOs8otHd+cRsMN5nbsuikOUhJUmlSspGYzAPidJHLkDaD8TYgeRWO1DjzilJJT2lElQVz18VTHhtBsHT1IraUuOFUgaNkFWQj9WYJ2mOWh5wNLT4VGwAuJJWenxF7zZosnvfHEe83qloNrpXGm1LQttcoUBO4VBCQNOyY39ti1wYnrKbG1wX0FIZXR3rSltQPZKUujMDOskKGv8pPKbNFh+rpHSGnCARooySDIBAkbkxtPI84I7bSqZCns6kOFBWlaRkUidUEECYkIIOhB5aCLimY2CRrmjZV0krpBYleguMrwwNhC+7wo8U4puClpbyYbqKu7q6qZ64mCAtLAJWQoQCAkgwIOlq5X50euFeIma7HOF6UKuwKc6tu767PRuKQYVkDZJBJ1KEqSADoBpL0wrwxw7fmHqepYpKh1VbNRUvVCG0uuLUBmU4SoqcMySViTr3EWFpsA4iwAqobwG+1Rqvr6OsaQEMM9k/wAeGoLPWI0CSlCRC4JVAtpMS7LT1UxrovRDrEgKelxNgiEEozW2uoRuC4sLYHob5w3iXDt8u1mJKctPUd30SXEULSzlQ4pwq1UkQvIlJKiRBEyYVcrr1usP3WzWl0MOuNFRaUhcJMFQzJCxJkwqO6Nxa7d28N3LuccVeWdagoqdLjynFqUrdalqJJJyzqTtrGs1J4yYZcwxj++aRCXQzVOGta67tEhYzGT5pB5T42ExXCnU1M2QjQGyhEwc6wSrwv6QXFDhNW9dgzE9TR05JS9RfxlE8TuHKdwdWqRMEpzeMm1zuBPTww5xExJQYTx/dyLuvy+KoMU9ZSOZ6R99Z0CwvtNFStAZVJI2FvNWpXkYRCpIgqB25CAd+Ueyzi4Q9dUcV8HtJzLKr9oAEjcn0hA3tl5IwRcqZjyCvVTjjdVK9iG6q+pWmFUbjZb1BWpKzAmNPXVZgMXLnS31rrLDHqpyyZ9hEqiNe6fZaZuMl0vVKLqq26cLCHXW8y1wgJOU5j3+r4a2jqhudbqw2hHWqnRKJIGug5i3m2MvEdU4L03BG8SjYf43RShudkAJoW0rey5pWkqVyGukDfnPnZcu+6HaMejOBhxROYOKWopQkDciN9tOXd3q9DcdS21DlSUqPYLbU5Y7lGYJ99nNc+HF1CgtbS1o0JCXI+O2/wA7WzU9UG81oWQi103mcOtOrCVPpeUYR1ruYAA6ylIEJ+Jja1QvyiWDzdVRgbEDUraq2a6jLmQCS2ppWgPLtk+fdb0EpMOF9SEppwgJ2yDl4nfkLVj/AClmDUq4SYcxCpa3F3ffnUKWkaNpdYWYmO9pO3dzsX2bxAjFIxfQ3H0VV2hYx9A9nPdUh6MdX+bukhw/qHV9mqvRhiZj+NBaG3863stRXEktIbaBCAcxKlGATuI2m3inw4dXcvFLBeISlQF3XvRVKiAYKWn0rM92k299Lrwo7VFMoXl5SJ0tb9uYJKmoh4QuSCPr/wCVmcArxRU72uNtb/RNCluBCoKFqyaDsaEzy1Gg9tlq78LFyG220hPMzJNpAp8JUtIyH7weSy2galSgIFtJxBQMq9FwrdTl5PjTrQIbSf5x+y2eg7OuZZ1Y7IOm7j8ANVJPj0ktxAL+PIfNJV2YDXHWVCEtNjdazHtgW7dvrCtwuehXRSuX3XjZqnTmAPiRoLKqsK35iEhzFF6rQzM+iUxyIP8AOO5svXddFz3Ex1F30bbSRzA1NtPR4GIgHRMEY95+rvk3YfNUk+IZ/wCeeXnoNB8zufkmW9cHEPGfZva9EXBdqv8A7PSjM8pPirYfGyvhrhtgzByjU0F2ocrF+vVPnrHVd/aO3kIFl6pvJtOmcT4WRqy9ySYVaeWWgoTn/nJOrtfIbBQiWpqG8Nvot6DTz5n5pZqLxbaSUp0HIC3hH+U3uU3P0ucWVKUkN3k3RXggEb56ZsE/4yVe63tVWXoogkr0t5J/lbLmU3xkw1iVCB1d44fQypXMrZfdn/NWiz8Exp1ZiYjcdwbJ0tCYadz01/yW+Lk4a6VX5tLgSjENw11EBMAlIRUCO/8AiTb16q72E6L2t4QdELE7mE+k1w4vhCgkOXyxQrI/UqJYJ9zlvbWprZTJWe+1R27c+nrY3t2c38Crbs9AKmI33BUhsOB1lt4RC0hQt2T7LJ2HqgVNzUrpMnJlnyMfZZQInutoqOTiwMf1AUMzcjy08kg35TZakPcnBz8LJ20JmzgvljraMKAMoVOndZAMg6aey2ipXXYqmobZ6waahIE99hGVZFhShsR9ev1W4MnQHblNsVqDl1JBA9osYEG9KQWCIAtsHltOhNgULStCXBGozAW7Bnu85sUChl2DJ/G3QJjbQ2DzQJJ+NsMbREfCzwSVyGzazAk6W2FCwOYSNZjwsIlcHXlz1s4bppQoPxHfYrWuBqpo3FAwXC0rT9ZJ39oFh806AnexK+SU0Cn41YWh4f4KgT8Jsrtlw3R5YA1i3VAfpozDf4WDWQoBSTyt1QmX4kanvstrpE4euCdMw00tluOtjTP8+62WblS3XgKMMXnVPN4YpUNLrXVGlCS52Csdk6ncTaUuBmAcSYFu29LuxA0y2l+pS4wG3g5JCTJ02/R03s0KC9GrvxdT38+28UIrVVDgQQVaqJgbD7NbScni1cPV51UFcEnUyEEAa9yvnaywBrTmKhmzOGUBO/qnDyVCvPvnaLaQgZoWNJE7AmzWTxSuVJ+hoa9Q0J0RofHteFhHOKtzvLCV0dcVRvDZ567K8f2WN4zeqFML+icYQgJ1ECDoB4aeW1sIBUUkTJBBChqfD7/CzVHEy4lATd1fOXmESRyHrfftPK3H751zSFJoKztGDCEc+R18vnWzeM080nBd0TsyQSkCSBoInlt899nvcd4Lq7qYCnFFbX0aoka6fZHxtC3759ywSm7q4bAZQkpnu3ssXLxmua70OCouy8FpWsKBSEbg67kWime0jREwMLDqpkbX1Xa8NNPLX57rBqKey6VLOXnA8/sGnjaNFcecMoUpSLnvUhMn1UaaHX1u/wCe4Fzj5hxRKlXJeeh1P0Ykb75vGwlyjLtUoFZWCrsjImEiABpOnlt7rJWLFBy5XQGlJCi2kQmY7Y1PtIPs7rML9/rDqez+ZLxVKwAOxB2/laR7/rsh4g6QuGamhXSJuG9QpRCir6MpABG4zabD5BsyS5aU+NzQRdI1/Pob408OFoASlV+0QyqntTVNHUR7PefG3rAwnn4zbxzose3fjjjNgFygo6unbpb/AKGUv5cypqU9xOkJj329jWMuUqtXSggC6JjIcSQuyB42DcEpInxHKLCzmGp5d9i61GNRoREE2WPdMek9xJKjGvj9lgHEgjbflYw7lmBpvpG1gHCNSde/usUEId1GnFvifcHCJVx4mxGwHqR6peowQ6hspUpoqBlZA2R5xNm9d/Tw4OUSQHglyQIBvFjnruCZ77S1eXD/AAXxD6u7Mb4Vuq/qKlJqWqe8qRFQ0h0CAsJWCArKpQmNibCUnRr4CtdtvgzghuVaf7g02/tRYqJjyLtNlA97L2IUVO/lCODTaSUIpdTAzXogaTvog2DT+UW4PRATd+ukG9Fb66aMeFpyp+j9wRY9ThBgpMaSLhpf7GtlSn4LcH2QOp4VYRT5XFSj/UtIRLfV30TLx9FXRz8o/wAIGgQPzQI/XvV0eR0pjYH/AOkh4VrCi0i4ikCSTez8R3/8F9lrQtcLOGTf8Vw6wwnvIuinE/5ljrXD/AbOreCrhTH6t2sD/VshEp9r6JQYxyVUFflGeGyUheTD6Qdz+dajxjX0WOVnHw/6a7PFy/RhHh5hW674vUU66osNXi8kBpG6itbCUjU7TJtZZnB+E2lfR4YulOkdmiaH1CyLiPCGHKXEmHL7bVSXOqlcqmlPsttNZgtkjKSpMRIGneBZvDl3Lk4SsGgamMnHHHepb62n4M3epJWUgm+m+RIO4HMW6bxhx+nXhDdbYTvN7tn/AFxNpDuu8MF4fu9i5f3Y0jwZCil2svRtby1KUVEqUVCTKvqty/i/AFO4UP4wuNMb57zZGv8Aj2fY29ZMv/dTJpcXceQQpfDO60JnlVoXr/TCysnFXGt8AjA91I7yVIUAO7/hFlV3H/C9uEucQsMoGvrXuxMDxz+I99ij3FDg02tKlcTcIJgmQq+KYz/n+FoXRB27z9FI2Qj2UF+6PjYUIAwndKSORbQeX/8ANWxV78altFX5huNtYEjO2I+FSbAu8ZeCCQUucU8GJI3/AN1qckf53nYmvjlwBYUnPxXwek6b3owQf860fd2++fMJ/FPuhJCMX8YKnE95YXVR4fYqbup6arcLgCUONvl0JyHryTBZXOgjT2AVdDxdrVkv1eFygiCA/Ecu8xZvtcd+j5QcZb5vx7iPhr0Ouw3dzHpDTwcbU41U1ZyApBGYB2Y/Czke6U/RoZlw8T7k5SEtOqPwR8zaE0rHD0nnzU4qHt2YPJJowvxIASFV+GAQqQBWEQB/gmwF5YK4lV5BRe2GGhMkmtUqSef8V5e+yi70u+jQ12f3y6GYPqUdQTp5N2Lq6YvRpTlCeITbhV/xd3VRHf8A8XaH7Mp73Lj5qX7QntYNHkkVHD7ikhwk39hoKjL2XydO7+J+yyhRcPOKAbKDiS4WgdTDhUAO7Vi2numf0cG4H7s6he/qXVUn/UsTd6bHR0QsdXiS83AIEJup7XyJHhbjhtK7cnzXCvqW7AeS1evDTiEmobpV47uRL1Sha20JQFKXlEqj6LWAfZNq/XXevSjeSay7+B2KnW3B1iXkP3dleSRIUnONjoRoN7TreHTW6N1YE+m1F51gbJLaXLnKwlWxjMdNLTfhCppLzwpdd7UAKKWspGahlKk5CltaApEgbaECOVkGGQX9AnzXd/mPrD6KlaH+l6pYWngniphQVMqrrnHLxHLa0adIa+ukJQYBUxxNwLedw3TX1aGkP1dddzoW8kFxLcU4zj+LJnQdkDnb0vcbSrXMpUa76C1IPynt6ilwngnD7T6x6TVVtYvLuOrQ2gHf/lVWeyhjjcCLpXVsj2kFeal+V5qKl+j1UkHKQQRyjSJB2855jc6w9WoW31VQhBLWjna7Rjs690yO+NTFi150jz7/AKMX5fQlOUEElevrJOxnSI1IjyslXQt1m+VoqELQoIWrKtRTKt9dO48/eLWQ0QR1TzQIeSsKUCVQUkDrClXP26EToIUdxoccLDjCTTJSUoXnJCQSVQIyxpG5MRpJ1JskOvMimOaIU3lSlMZokGQSeYy8oGoPOFSnQh2n69tpJL2VYMQqVak6gTHZTAJ2i0gOqjIV0eBOIU3nwtuitutqnqrwpqRDLqnAJSpAymQk+t2Qd+7WBZYq7wvSovuiXedOwFvB5nKhACSMiiBoO5Jjvga8rRv0XKxdZgN1ph5Lz1FeD9P1mYg5VK6xAkRMhwGfPvtLN/1jN2UDKLwvBhdY6+hilXlz5lnsiRocqRJnmNbetUcxmoo3k8ggMoa8oq4xTUzLeSVvvqLhStSkjLmkb8tU8to7wBWTpY3O36fdWJkNKyPB6lXmlIJIC0ySY0g+Gm2lrTUIrXaVs1iaUuinCXXmimFLCf5qYG5Gg052hTpR3QK/ALlaic93ut1aFoBJACoUO/1Vnunv00DxenE9BIPC/lqpGutILKlt6UjrKAtQCm80CE9kad49nl77TN0HMDPY36SeEqYKSGrqeVfD5UqJbYSVQAN5WUaWiu86FVTSVBaUt0oGbQaQBoZ56c/Du1tLvQHv2nuPpQYVFW6W01yKqjRyBW4wuB7SB8LeSS3DSrWL1hderHEq7G6+6KbOhJDdUlXaBUBKSNuesabfGzWufCCXKdC0KDcEElWg56BI1Owss8Y+Itz8N+Hd748v2762tu+5ktPvtUiEl4pU6lAKQpQGhWCZI0B52rlQ/lJeAtMlKjg7GiTmjs0lMeX/AE41t5n2hwuvqqniUrCQQvRMExSlpaThzOAddWRpMJX36ZTM06qUXdMulaVZ55ZUjQ89fKz0obhQpKWmwQkGVGNJ8pi1T2/yofAZlQQ7hDGw5ECkppPn9P3WUqX8qn0a2QlKsI48UoakehUsD/T2zDuzWMSn0ojZFz9oICPReFb6iuBsqSEqMaScu9ob6d/DqtxL0Zr+Vd9IXnrsqKSvSECSlKHQlau4AIWok9wM2B4a/lIuiviliqqK6pxBcfoyM6G7xu9MvkbpT1K1wf52UGd7Rzx8/KgcLL6wJiPA+DcP36yq9buqKJupXSsqSc6Sk5pc7IIJEwSJ77W2FdnpqWRssl84OgAufyCoKnEZKk5fZ5kledOGBddzX/dVRfjTi6WnrAqoaakEpKgo5VHnAMbkZrfQLhDF37rsN3TfWDrrdFDedExVs1VUMv0biApJidTBHhb5yL2xZT3i7/c7K2ldYFyQIjWNB5n3m3qjwc/KzdHnBnCDBmEsT3JjD88XJcVFdtcaS7mVsqeYZS2tSFF4EpJTIkA67W31TRyyObISR1sLn5dFQPkYW5Wi/wA/4ur/AP7lW6t0VF/VrlcoahCtGwfBI0sqoFDQNBumZQ2kaAJEWog9+WL6MhScmH8en/3bT/8A7RZIqvyw3RwdnLhnH3+T6X/9osE5klLfulOS7qRc+ZUWV0lhI8W6X0V+6q90ISYVZGq76JB7W1qGP/lduju4ns4Yx57aCl//AGiyc/8Alaej67onDGOf/wDBpf8A9fbP1sONVJJ4TlYU8VKz1nhXrqr3kE57JNVe5J0UfZajbv5V3gEs/wDcvjffnR03/wCvsWc/KpcBFEkYWxtqOdJTf/r7Z+bAsbl/5LvoriGagZvIFdqpr1LBkmOdvPr8qzcZvDD+B8QNoBNM7X0ilA/rBlaR/mLMefdZyr/Km8BjthfGp/8A7Om0/wBPaEOlb01uDvHfh9S4auDD2Jma+lr01TblZTsIRk6taFCUuqMypPLkbHdn8DxaixKKolhIaNz8Qn11XQy0j42SC9lTrB99uYcxbcOImVQ7dV409Wk7QW3UrH1W9523FVTaFshSg8kKRl5g7W8AC41BylYVnlIgRHn37W9D7q/KV4Lo8H3LcdZd+J3K2muympqz0WmZpmC8hpKV9oPF1xOYKMlaJ7uVtT2ywWoxYQmnbci9/nZV3ZzEYqLiNlIF7br0uwUVouc0zxAcZdUkpzdoT2oPcddjZePxtVboJdInC3Ha78YU2HaKqo3LmfpH3mX6ZDX8eHACkpWor/idSoz8ItQSeXPa3UFNJSU7IZRZwFlLVSMmlc+M3BQb6EvNLaP6QIs1HQUrUgn1dDys7FExHdtZuXs0WaxRSNDChHfa9ojqQqyqGmZFiuTJ98WzNsdD7fssGVDfLtbJ7zNrQBVpN0dplHqUgE9mUewGLCzzjU6RYnTrBzpCiQFbc4KQfrmxgKMetYluoQ50KFJO2sx3WwKB1+u3AVykey2woTsbSDRIhAqYB18OdukEiAk7fXYEkmADGlukqA2Ou+1nBMQwV36Dv2tzUJD9O4yoAhaCn3i3IUNR3ja3RVzs5cuaR3raNlRkkoE+ca2M0J+nG8DlZPu+Ay5TwPonVpA8Jn7bH6IgO6ybIEnNOEAwNVjT9U2ywE/y49o++2WTVKvBLFj1RdVzVFVSOBL7CUqCikESVBJEDQTJ8JnY7POl4L8R32EKTjO6klSAsRSHTNy9X6vHaTZp4+addw1WNpRJWWyB/hp28fv8bXP4bcLsVcRlmlwpc79c5TMhx1LSUhKExuSSAJ1jXviywMY+5eoKh747ZFWtHBHiQFA/uzurLvpSaAx5d3P77HGeA/ERzM2cZXYVAf8AkR393zNrEV2Ha+5L0cuq9adTNTTuFp5DqcqkqG8jv8xafuDfRybx/cy76crxTtJGROZObMSNhHIafbad8ELBmOyGbUSuNl56K4F8SlJDCMYUDjmTVKKBRM6aDWffYKr4JcSKU9XU4wo2luIlAVdpB8xMaGI9nuufxc4Yv8Pr6eu9VRJbIHWJnbeNNe732iCvK3VnrFLcWpOUZjKgOQOs/XvZW08btQu7xINCoEVwZx+oJCcfUMZcul3jXed/Pa3Q4M46dKWnOIdCkoUIm7Ekaa/rbb/Mi01k6yUgSI01nTl77AqEJAQkyoaiY8dbP7rGu7y++qjZPRl4mPfRq4m3ees3IuwSR79NPs8bdtdGPiArMt3inSQdyLpkSRvJV3Tr8bT/AHBUem3S091pK2oaUDuSAI94srNNISoBQBEdqCdSNPbvYF7A02VrGA5oKraeizjNSpVxcpwZJITcw0/zxpt8iybffRUxNS0aqt7is26ZSlSfzKEmCrl9JO5+NrRqTmSoOKA2kEbzI2myTi4k3G++spGVbUhJBn6RIMeOm9oXizSQVO1rSbWVJcH4aq8J8drjuCpvX05V34guuHUNBorCnG3CCkT3gbnbyt7a0+rYJBI8997eNdW5k6TNG2lQM4hulRKdI1ZG47iQLeytOnKyE67/AG2r5iTYp8YAcQF0QNYUZ2sXdG/htY2Y/VOvPaxZ4kiJ52bHulk1Se+O8+diiwPkcrHnUhWqhJmxRwDMZJ1POxrUE7dIOI8ft8OqenvZ66Xa70p8UoQ24lBSShS8xJ5dj42Rf4ULQVlODKg5VHT05I5x+pZK47qSMNXYoKyReiRPPVh60LlYQoykEdxM/PP5FrKnF2KvqHEO0Vgk9KrIYTgN3T9a8gDPsbt0rpaOJR2MBg+CrzgbcoatXgugAwqMog98/fr9Vt55UQTI2OkiPdHd7rTFgQ/FcFYQdLS8TqMA06Z2m8lfX1VgldLa+1E5MD3eAkwSbxWY/wBHaACoyUGCQYAAGo9vs8baLmXMUa6ATI3+frsgjCXiuKn7+FniQj6LBl25idvTHD/q62jPjrxRxDx4uK6uHF43XQ3VRXheIW7UUzq1vJ6tpawAFaaqTB8LM5DqjCgkASIkCAI3tpt4C/bjWFjsVsQQebTg+34+yyujBFiuEjgbqPLz6N9yXS+lly/rwd6z1VBLYA5aSnuAsE30fsMEBJve80iDoFNa6bepvaasbAB+mKz66TlPeJkRZtOpdISmmUU666wJ0tC6niBsApW1EpF7qO0cAMGpB6y8L0WkqjtPoBJMfydP2+0/TdH3ALiVFT16KMkqPpQAAHkNvnezqV6QjRRPYgmJ5x+FlO7G6lai6HCWUkyASeXd+FudBGBslbPITumez0euHydALygnUelxziNteYiwp6P3DsAlVJWEbkGrUeXhZ9tviR2QAdJ21E2MB4SUqObWD3+H7LR8KPophLJ1UVjgzgSlxm5Seg1i6Q3Eip6hdWspLoqVtlW4PqkaTG1gMR8L8BUNAamiuQpKFbekunMO71uWnjzs/wCpj92NOAlQL2HqpGpOvV1jB07tHPiLEsQNpfuypSURlTOvhr7NvqtzYmFp0SOlfcaqvtRdmHaHGKaGroUoojSrdyqdWAkieeaZ0FkddDTV9Pedddbn9z0GdSVZjmWoRAEnRIAsr8QQly87vuxlSC5eNQhkiTPVpMq+sCLJ10oSxTYwoSHPo1LJTtKch2jXw9/haDhgmynDiBe64vxNExhC7r2p6cJdW42HFEkyDvOvd77KGLrtpGLppqm7KVKCH23F5TuiO1JB2AGvt7xZv19QHeEFI5lGdCWiJVlg9YNZ5acvEWGxxfjQwlS0/WnrH2mllKFDNlSBOnL8dLKWNAXB5KXqGhpL0r6S8nKBLVAqpQhlIJh1J5lPcTrz2tIDHSJ42XVj2vwXScSb4Yui76dLVJTMKQlLKUkIACgkGABET+jZnvVrbVy0K2o06lQUkDKOwY1589tfgLEaK76e/eMGLirrEu0D6g2G3BJSpSpJ1gjQH2ecjl7Y9VI0F5sE+MZdIHjhQ1tyCk4uYkZZqKrqn0prSMwzJBG08ztraNePHEHFONa6hpMSYnvO+F3e0tTfptWt/qs6xmCcxIAMD3Dvs/q/Blx3iimqaupeilWFNEOQExuIGn6vw1i0NcY6OluLEtYy04SymjZqHAtQUpS1EgAaidDEd2XezYqhkj7Ap7oXxi7go1vVm5mWBWXk66HB2gExLnraCdd517oM96FdT5qq5dQoONsobIQkyQkFWm+4B+r22FTdtffT5eecDTKlAJRnkAk+7YAnxy6W4oHPR7/caWtKyGQk5SDljTQnTxEbad0WMuoQEqVpCEqdW42jNBGTlzPPXcySdQTPraKFwrpfQH2VtomkcIWQU5SlQBSJJgwQoTofEmBZJqH86MynBlJymZ0gADUmDMbD8SHc9e1S3uGgpOV5JbUrIB1bk6axI1HLWFd+tlBXEXVk+jDfjFMu+LteeU0qprUPBS15hlUynQkntZssjXc6ACbTFiKLxvVlulCstG80tISgkE5xGQREAjceQgzaBejhX0NLel/MO0zb7rTDLrK225SFlSpjXszlSdBsIOo1sG1UUdw3KmvvBBNRUuIJQuBCyoCJmREyZ2CT3W9GwJxnomsJ0CFlGR2ZKb718rcCKZBbaZzyAqCuSkFU77RvEZhvOkf8a6OtqsD1lDWU6Eu1VK6olQURI2O3MnQQIEbb2lajcvahApLwUkMtuLPWoPqzIiAJH6c6nUjaSLQfx0qsUYyVWXXc1Suip6ZKklaJCgI1AiAJJMq1gd1rau/R0ztySLWUAsXKoFI6s04qFZlFsqCgsghMAaEnlJnnv3kQ+OjXcldXdIjA1DcyVB9m/wCmqEqST2mEK6xRnTZCVTtItHtCFULtRRrIDrK1IUQNQUyCdPM792nhNnQ5qLza6TmCn7vofS+sedZfIJORvqF5l+BSmTr3RM28enGUFWsepAXqLxG4dM8VcC33w9erTRIv+kNH6SGg71JKgQsIJAVBAME91oJw3+SDuh8pdf4qVrrR1KlXO2lIEzElw/CbWww/VmjvelebTT5kOCOvMNEzAzdw135bxaWl4Wv/ABArrMWYlSik/wDIbulttQ/lLPaV8BbH4k+pbK0QuNugsB8STt8ldRPY1vpWHibk/IKheOPyZvAulwyvDtw44v68sW50paNFTMuIAntBSD6uk69YNTtysh4S/Iws3igVmLeLtZd6XIPo7F2NrcAG0qK4GnITb0su2iw9hymFLdFExToT+okSfEnnbHr8SAe2LCR1/ciXVE1/7rdvM6/gmySGYZYmfM7+Q0H1VBnfyNfDNlE0nGrFLTkAZlUNOoaco0s2cR/kcsGUl0VlZTccL9dqGKdxbTarqZCVqSklIML79Pbb0Nqb8PJXlZHr73C21IUuQQQdbVdX2pyG8RspYaKaSwdsvnExHh9zD7/UrcLucrSTlywoftn22QSSd9bSjxyup67MbX3dbjBb/N17VtLJ0nK4R/qz7R7IuNvRKOUzQtedyFV1LBHIWhYCR3621qTM2cGF8AY2xutxrBmFL2v51gZnmrto3KlxpMgBS0tglIJMAkQSDG1nOz0b+P8AUAlngnjpQSCpRGH6qABuScmg8bSunjYbOcAfiFG2F7xcNJCjePH4WyPmLPlPA3jMq7zeo4UYv9CBCfSPzJU9XmOgGbJFjd2dHjjvfQKrp4O4yqkjXM3clQRHnkshqYW6l48wnd2lPsHyKjufmbam0qfwVekh/wCo7G3+RXx/q20rosdI1O/BDGo/9zP/ANmyd9pv7RvmEndKg+wfIqLJtkm0pp6LPSMUP+8pjEaTrdLw+tNkHGnBfilw6pKWux3ga97hp65a26ZyuplNB1aU5ilM7mLKyrp3uyteCfiEjqWdjczmED4FM4lHVpIBKjIUSNB5Wtd0W+hhhrpCcN6jHFfxBrroqKW9XrtXSM0KHU9hppYVmKwdQ7tHK1TgJMDXS1+fyaOLUt4dxvhR1xI6iro7waSTqc6FoWf9G377V2PTT01E6WnNnC34qywKGGprGxTi4N1aHoR9GG6ejfjXELl0Y4rL5axBdzbTjD9IlpKVtOZkqBSoyYWsRHO1xFAAd1oF4b3x/v7u9sr7LpcaJ821aeOoFp6WQTbN4ZVzVkZknN3XWgxKkio5QyEWbZcnbT32S7+p1FtDyf0TlPlZTInSffYOsSHqRaCNYkWuqd2VwKqJRmaQmkCCNfdyFuhqAEnXa3S28q8pG/u31FuCCOZPna6BuqddsKSmoVpotsH3E/2rGgRqSN97EUmKlpc6qCka+In/AFbGgokyNxroLER6qF267WsBClRMAxA18rETfjYb6xuiqz2gkJydqT4T4fMi3dQ3WLUg0tSlCdMySkEqPmdtJ1twwxegMvXmFgH9FoCR7fbYgW5pi1+fRsLurTOmjfn4+/xsOm9FEqCKN9YSkKSYgLBjaeeuxiwKaW8eqUDeR60klKg2CAnSBHM6b+NsFLeJKSb0UlMDMAgSogDbu7+7Xys8ZUzUJQpKpVQz1qqdbJkpyrGog7/bYxm55vOyQijvI6qvlRBER1YAmInvmdbKaSNATqBr4myHwShcMDq6upSZHWJS4J9oP1CyhRT1skbambJqlBFewT/fErR3bQfvsp0MB3UDXabIkO6WCXJ/T/xrZbvKv9Y+4/dbLcuuvCLHRS3c7rgCZW83lSBHaK+fdvyNrt8HeNmK+Ei3K3DKqUpq2UN1DNQ1nbcA2JgggieR5+NqO4/VF2ZSmSaltJlI2zbeXmDzjeDbfA9I3XX5dt1PvqbpqyoZo6hYIkIUsAmCPHbvFm09g1xdshqt2QB3S6cGIsbXrjDE9dia/HW111c91r2VOVA7oA5AD4a2lrhP0jL5wBd7l00yaeop3CDDomFRuNRaEcdsu3Xiy9boqKoVC7qq3qBCgDORlwpHkNPjZIFYtpeqj2d9zPfY8BssY6IFr/aCkTiTxHrseXk9ed5VKXHXlE5k6Ae60dVKivMorCtDm/SzCdffrvboVC3QU5gFGSYOoG0fX8bcrpncklIMQPPlqLOaA3QLjqiK/oyoLWkpOpMzOvx5+dgXEBIX2kyQc0+HMz9dh3mVMhQzgFJ0EkGY132/GxVfUkAx6yssETAB7t9p77PTrJdwdWlq8HKJwZkVAKgZMZxqPfMe2z5FOVBa2weyFEQJA0keetopaeWxUJfbcKVpWFjzBHf5fC0rXZUoraFmub9R1tKiPVAJ3HvtW1rcpzBWlC7MC1a6hRILghMQCDHlHusj4kZUbmqesAUlKm1GdiM4PPx91nI6krSY7ITJ1O2ug+e7lZFxKUt3FWlSFHK3mAH84T7tfbave/0CrNjRmCp3eXVfwlKN1CSCMQXYkaSNFM7RodZ79O63soz6ok6zbxqvZC2OknRrebQla78uonKQCQepM6f42n1Tb2ZYbKhvMHTwsG/YFK0We74rFAbkAeNi7gEnLp36WOKbSACRBPfpNi7ojYDtWWNNksiD6f0jYmsayTvtrZQeSCNeXeLFXRpB58vn22MYgnbqLePCJwrRKRui8myD/wBU6OXnaC1JURKnCJ5xNrA8a6U1GEGxI7Nc0rUfyVjn52gv0U5DkgwY07p+Rayp3DKq+oF3IkUOJ9Z0kgyQOeh0tkJCSVq2JUDHwsaFEVLyhXKQY2HMg+/T9thDdxzBYywqO7Q++05eEOGEpOWnUHMe+QN7byqIzKUdeUyPOx70FxRyhRWADOknTw9ospDBWKXNUYYvQhQMKFG5tHl3bWTiAbpC3KEgAAKBDgkeOh1GkR36W3li9LoJUrS8GhrH6Upj42cLeC8TEwrD94ICRHaZUkbabx42Bq8O1l31N0mvoXKcu3hTZAvQkB1AVHfvZZA9rcxGiSJzHuyg6pTxmw285TOZlEBC4gwOR1/ZZAfpKJhsmofLYXMyqCVDl8/faQscUVJToo8w1yr3O57POZ2+dbMOqoqOuUGnNUoMwDGsRaDNmddEhmVtkj57s60hNQjsmCUrMak924+2yzdj11KYLblTL7iuy2CQCfmLJ6rjoErTKtN5Ku4WV7pui6FINVCs7ZzIlU90fPnZ0huEjAbof83NKSXCFA8xJFu03bSK7JSZIBIzZR46WMNuNIQAVAaTEc95ttboCic5CQDoTrYY5kQC0JFau2me4i4WpH5DFVTXjSrykg5S5SqIBPOATaxR6NXDutp3qd6pvmFgoVneQAeR2RNq9LqUMY5whVp06urr25UDzp0r/wDli16qcoXlWlTsqkL7IM+U/ZZWE3IUga06qvw6FvAavr2K+qu+9Kusp3JZcVXOpDfeeyneLV1VwNwRc3TVvDhK7dq3MMXzR01R1HpC86wqncJlchU9Yjke7vt6FpLKKxFO264CpQMEgc9rU44vu09z/lEcEValhBvO5aJkAfpK619ESOcd/wBmnP0F062qkVXQ06OVPdouNjA5XQo0Uy5eNStB1mAA4OYsv0XQ76NtRTobf4X3W8lXZX17j7q4GwSSuQO9MwdO60lIpkpW4DT9aMxkqQVHfvspUa0MVqGmktobSrcZQPrsuUJ5XlDxOue6rixdiTD1ztFLV0XnU0bTMaJbafUhCYmRCRue/wB7Yu6vbuPi/iu8jop+haqCDPaKmWzv/hA89TaUOkR1KeP+M7oDIU4L0fcK0NlOYrIXoPJe/wB9oax5VfmriS6/DSW665qAE5tFwy2AD7Uc+/xtQTvzZmouFoYQ5Pl/E7j7Xo6aYpmCoJB8dwd9T8ybV+4+YiL+IlU9PV/TLYZFQEnTshWUeEZvs5Wmlh0JTlQolJIkxvz5eZ5c7V+x3W3FdeILwrK1hdReT9Q4soWkKUe0YUOSQYEaTAGm9mYbFaQuRFbJdgCatDdSlNfnC+axTdK2kOZUuQrWdBG3zAM2J3ERW3o84yOrNQsJQnVRCSoad87a/fYle993leSurqFdW1OYNDRPdPjt90WPYWYZX11TVEpYbGVRyk77bAzoDobXw1KqksVyUOiG15iqVKAVJEqVB3BIjXWJnxEAUNMyh1Km1r6yJXASoBO55kcyI592tuqqoVWurbpj9GowlIQVApGkgb+0d8+Stc92udUh58r6xaczZOVR/R7++QTrPlINnBJeylTo3P0zWM7zp1o6s1FA062S4lSVKS4QVE7HVWg9aO+dLE1SaS+r1Zo30L9EpkqCigE5tBOx89DoTzMCKncK3E3di7q6ddOg1NKSjrAIIKkEFMfpQDEDvPZta/AOWsAdqwG6ZgnPmglRAMgSdUmSI13G+9vQeyhzw5PFB1Rsn/8AnKlvC7qZ6vQhvM22pwkTBiCoHLI5eMAd2ka4gvuleL1HT0iXnHFrAVlMDxOmhjvEyLLbV235exXU05UxQ1yG+pZQ2UqaQATIIhRJEaE6SfIo+Oqm4OGOG3LyvAdZVuJ6mlpwUla15QOyDEaAdoxAGu2l5UzSOuSLNHNMEbcu+qpFxHutvDHEO86Bt3QPCoX2jCFLSFlPsJPw3i1qPyd3D/CuLOId441r/SfT8IUjTt3ttuFDa1v9YhTi41VlSmAj1RnJM2pvi696+98U3hX3opHpD7pK8iRlkcgee0Ttpppa2H5NTGDN3cY7ywvU1CUpvu43ENIOmZ5haXAB45S6fYfK3kdeQ57y3a5VlT6EXXpWtJgoCimRGvI/Pf3eFpOu3F4rLqo6lDp+mYQ5r4pBtGhGQAiQZBB8dNPGLE8M30tF3+hLd1pqmppwNuyh5aU/BIt5f20dJDBHJGedlrcDpW1j3Mdy1UtG/c3989s2LO3zAgL5d9mexeSyJKoHfbtNY85JSCoW8xkrJSN1phhbGbpxOXqpW6rFHq1S8xKrN2/sSXDhSjFfiu/KK6acgkLqXggqjklPrKPgAbRDX9LnAdZfQwvw+oXr7vJxDhbeqppqbMhClRsVkkJMAhM2kgpKus1Y026pCyKLQbrzv6Yt0fmji7jekEjq78eqkjLt1xzjXfZwWrKsdqDv399rFdJfGN5Y+xteeMb6o6Wnfvxpmp6mnRlSMqQgCDKgYQeZk5vC1eH05XlonZR3Nve8DD20TGv3ACwOKW7w4hWS6CnGK9uEfEe+nLsulq8F3zcy6YIWCcqkOoWFRmE6BQ1nfa3qbw8xXjjiVgq578rXbvpGq9txLqC0HFMOtOFCoTAQc2ihKSRmiTpbxm6OrryeLlyUzD3Vrqi9T5okdppfLnrHtt6udHivvC5LtfuWovR2opELNQ0haRlbWsALyQJAOROkxp4m2Y7X0jc3HaPS0utB2dvLGGgXsTupuZwhc7b6a29nH70qUxlcrHOsCP5qfVT7BZVXVMsJDbCEoSBACRAsjuXpKcxXt8LEKi80+tnJ8udsCXPcVqmUhdq9Kz94ZpKSNediT1f6wzfo91kGovhCSASe12T5/P12JKvtBcCAVTmg8jrZzYnlFsgY1LD1eqTqSR3WqN+UWo/ztweua8EpUtV3YhpyqNIQ408g7eJTaxdTfQjVBPgBPvtBXS8R+fuAuJ2QCXKJNPXIg6J6l9tZ8jlzWu8FDoK2J/iPqhMWjbJRSNHReVbYAeSlQntAGbWa6AN/quzi3el0qcIRelyuoyj9JaHW1D4Z7VrvNvqbwqG+XWKIiNiZG2m0WlPoq32bj494bqQrKmodfpiI0PWMrSke8pt6niUfGo5G9QV5fhkhhrI3eK9TsNXr6Biu6KxTkBuuYKjI0SXADv4H7LW1VIGxHttQ5y/XGlpWg9tMKSpO+ms/I+q16Lprk3tc9HXt6oq6Zp9OnJSQofA2xeENLA5pWzxghxa4IXUzvFuVDTbzmwhHZ2jusFqSDrvra9aqJyb1Yz1L7jZToDIsWiCRvERZUv1spcQ4BIKYskZxESTJm1zCczAVUTNyuKxYCMigZ+kQAZ7zH1GxpQO+sxYm8vNTuBO4SSI5ECR8bDlQV2hsrUSe+xTEK5dxoCQPdbBO8HW3BIUZBiB3bWwk6xaZpTUICAdZ8JtuZGxjxsHME2yVAdokdx7rOG6RDhRnui3STpANgJ0JNugQNu/us8JqDrHS31D5/vbyc22gOh+uy1RJIdCp562Ra+V0jqB62Ux4EWVbtcDwadTssBXnIsnNIUvjqoErVP8AOtlg5XyHwNssqavCPFLK3vQaZMKDl40yQo7jM4nu8z9VrSUjrN13rTPMVC3BSPodzBITISZBHutWK+UJcve5GwEfS31SA8pBdHLY8t++1kApjrM3WKJTKpiZA+v55WSlAc1wKhrG5gGlOniRe124gx7f9/YfcW7Q3hWLqmlFBSe2M6gQdZCiv3E2beYoRmghMzB8vvsNTPUzKhmSgpKh2gnUx8j67AOKAfUGBIkFB0EJ7tfb7/eZGOG0MHJANjytsE48OUVG/XNCrcSlskZsx0I+vlt+NrD8e+GXCfCGBbnvDCV8Nv3lUhJ6lL6Vl1BQSXYHqwQB5ExsbVYZrw2orQrKoEdrWRtE8/CxuovesqEIQpwkbAE+3v0n7PejmFzgb7KRpABBCI1hSVqQiPZOonwHzNiS0qmSDB2Ow0P42MuPFeUpSnvnvO3L55WBdhSlGARGxGo5R87+y0yRFnCvP2VRymRp4+Nn9gK8euoHaB1RPUOZkJBggK7te/67MVRSoHNEJ2AV8/JsrYVvFFBfLanVhLLoLLhM6BWx9hA91oKlnEYUTSvySC6k1SVuDMYAOkA6SPneyFit9r8wV6nWyqGVkhKTpGn42dDbRK+0iTOoGwIP1fXp32CvbDN2XrdNeqrumorCllbjmSrWwFIy6pMEaeI1tn5ZAxputLHCXOFlSPFLbbPSPoRmBIve6SrcRqwdj9XgbezFLq2CQPOd7eQHFd27x0oqMXTc7d1Mi87oApkvOvAGWsxKnFFRVA8tQLewNKn6IDz13tBmzMafBREFsjh4rtSYOUn4WLPIMdmQT3mxyNI1MWAeSSJMCednxbqORJjzRSMwHibElZwZ1I3+qyk6lRBEHT5+6xRbfdpY1myCduo84xj/AHmvEJBUmoaUATp60ae+0ErcUASCCoRyIki1kse4er8S3A7c93ZOveWgt51ZR2VAnXloLRy1wJxw9P0l3JSdAC+owPKPO08MgAsVE6nkk1aLqNWHXx1kZeyB2Sgz629sOdQC1wcv8kH5+60pMdHzGc9qtuxJI/41Z2/wbGEdHTFpzBd73agEiYU4Z/zdbT8RvVJ3Sb3VEzailSVJUBE9oJEabe42sXT0eK7wxQmloKaaJdIX4FIsoIyjtF+SkOZz6kerrZpN9G2/ySXb/oUzrAaUY+rwtYy7ml0N30VI4VLU1TNtkpUQJSkAm0T421Dw6+yjkpXhtpBZQjeeF8aB8tLpqhLcyT1MyCToDBB9/OzD4pUlXSKw67eFE6w56fol1KgSA9T9421NrbNuZ9UtLI785Fo348YNocSUV2XhVvVDSrqS6+2EEQVBTaoVM80WuKjETNTd3Lfmq2jwUQVPGa6/goR4ppSy3SLCc8Z5EEkEiY+FozVV6lKURqJ3G/z3WsljPh9dl8IpxV1NSgJ7RyKAnQ6beJs00cDsIdh1y+KloqlQS7UpSRuJjLPI2rY8jG3kIHxVjNE9uqhlxSVJDgJka66iZED424RVOpQEkwBrMa/Ov12mip4NYaQtlbNepxnP9OV1CjKYjSE77WOXXwVwzedR6HR0rr6z2gevcQInXUgAifPU2sGUjpGcQer1QYkaXZA4XUGGof7SipJM6iND4a20Kx8DI4sZdYJMQPP3WsYejbRqkfmlpAMAZq9R9sTYZro4XahGU3NSLM6zWrMa772YIWbFwU3Bf1VZquoIvbCzqlCWr5WnXucoakajl6tr30FVej7KXaZmmAKEFJWpWoKQZiPH4WrBxDwRceG7/wAJ01NQtMlGJ6Zt9KVkgy282d/58e20+XbUcQahNGu7aG5VUCmkBKn33AsN5fWyp0J0GlgJGZZCAUa2Mhguleoexe7VBafQUoBlJJdOk+A87VD6WNS7cXTL4IX25RoD9YigpXXBJTlTXKzR/j6z4d9rW1h4mIW23SLw602lJWo9Q+dJ10I1PhOtof464WTfeLsIYnxvdty195Xa7/uLWMsvNmnUHUKJKFHecp1nnaCRgYwku81LBEXOsNVPFQ71VS+3lEFWxAM8+fnY1drDLk1VTkSgLgHKJUfYLNF+4OKThQtGL7kSs6uA3eFEnTUSoaWOUtDxIbYVS1uOLuWlbZbbSxdSAttxWyxDkGDuDpqdNrNdUOadGE+X5qUwi3rBUH6XNGml6TGJ6pFC6lD6adzrFJAzBVOiQJ5Ei0C8YqepN8XDUUbai0bmZc7SohSHHUTm7yBAj9noBxt4fsKxBUV2IqGivW9OpaK65dA2lxYDfZBGu0Rv3Wruu4qSuuO7XH7upKhPUOML9IbBWcrzhgFQmJVp+Ns8ZbzvB6qzbRnI0g7hQnfd4VF1YbrLzoinr2mCpmNUhZAAPjEzpv8AC1fagX8utevK962lKluFbrjjKFrI5kqMHw5HlpoLXa4x4Zab4V3klmhp6ZQp2Hi802CptsLQpR0EjsyPDXuNqf31cFTVVDFK5TuejtHMQG/4xW4Ea6dw21jmJtsOa3KSN0FXtLHAFMaoVQXhUuu012KKQMxWVkAgbmPYdPCx+4GkOUK0KYJUgEkAEwQYkg+Mn32Ub5coLvKWGqUJedhCG4CConnBHZ1A1JB1jloh4erEh4JdmCpUhOpEkE67k6D3c9rWFrFV/JOOmofSXeseAQ2iQEdoDTXUHSdfD3xZZWAolWmUJChumQUwD3kCCNCZ8NimMVjhWgdalskArAVCtSIMjbXYQJhQk7WPICSlDJlLRAyDXKTyywN4SdtCABsBaQWTTdOPhtcN03vi6ufvyqQwwww2tpSX8is5WoEcydExrIgj2WiuDGGEaNIoqY+lsEJC8xIbUjUnUmTJI33ggTakt/3ku77zoQm8KunS82slynSHHQkwIAWRoSmd49bckmxtrElataEnE+IHQ3leV9I0xk7UfoE6a/rJjXXXS9w3tAcKi4bGXPVRuphMbuKv7deOqBVE63QMNrfYQ4lqnSR1iu0Q233+A387QpjjCV43g85i3ijjK4sPtOH6JmqDrhbRvkDYAlUK8D56RHN1X9T4iosSXbd103dd9fRYfdvS66ylrbzW+qoadbMJ62qeaUcil6lCdRoQQJhO86LFF6166m/WrwrC+Cyh95RV9IUFSIXsZImOYnzs5/bIVt2Pj28dEU/D+C0Fp3Tu4mYd4evM3he2GL+vKtrqZSCFOUSaancQNSpKCpS4iYJAnTTWyPwL4ifvXcV8MY3KVFu6rzYdfWCqfRpKHwE7HM2tW4nQeMtKvw5iK52FGpYcbQYzpQrMOe5Gnfz7/GyS0sJOsWoaqdtW7OGgfBMaCxe/TTrNShNVSvBxt9IcaWlRIWkjQjwIP4WQLsu6qF5XqoHJTtVyldYo5UAKQhwyToNVnc2ijoScSVcSejth52rqA9XYezXBWKzEmWMvVE+bC2Z7yDaW63AzuJb2VU1l9VDVAgp/uZtRHbCRJk7e7nbz/tZTtlo7PNrFbHs3MY6klvMIa8MdYQw+EtP1ble+jQM0iZEx+srSPETaPuKXEnikcHVl94TupzD1FQgKcfDeZxaFKCR21agSoHsgHS0wXFg3CeHwDRXWyXRr1jic6p8ztbeLfzfely1d011K0/TVLRbcZWkKSoEbQfG3nMQp6dwyMv8AFa9wdMbFedV9PX3i1x16+b0rbzrXTmK3VqUTr46++fG3WGMFXtdF5099pIaNEoLAEiEq0PPkDaXrzuCiue8nqempENJbcKQUojY6e7u/CyVXU6hdlU0CEq9HcAEazlJ0/bbbQBuT0RoqmRmV2qq50mcF/uPva7aRt1Tyay7C62Vd+dyR7CfjatlXlNQsp2nTtZvja5/S0p0XjhzA1+lJWp2ldZcUJ5dWQCeWqjamlc2UukRGkHzGn2W12DPzU4ushjLAyoNk5uEF5fmnilhSvzZQ3e9KlRAmUqcCVCPIm3rTgGo9CUspBBUBmjbcfj7reN921rtBeNLWNLKV0zyHUEcilQI+NvXvDFe260w/TlWVxvOBO4IBFq3tLHna3xVx2WksXjpZS0m/D1IVGhhMTFgKh+tWx6UGHVMgAlRBj37WZ1XVXy9dr7lw9S5XJQVtNujMhZEkJInmB5T7rI9433jigvFxVLUNVVH/AH+rqCy0pDYWlT3VJzwtHU+qSP76CfVi2F7qL2C3PEsE76q8gmSCJAgmNx7bEfzpkWFIOoVmgHn4/jaIf3S8Q6qncR1zTRYdU3UVDj1MG6dsPGVOESUOBGSQREKUcoIAKQ9iniGtvr6a9rs6kNNqcLlRSlIUrKVBBCthlcAnWFzJjQttJYKEzhTNeNZlfcAywHFAeABPf862Y/FGmF+8PsT3ItIV6bdFUykE/pqZVHxI+YtrD95X1U0S6i93i8tb6jTvEtkuMEJKVfRHJ+kRI5AExYd+pBQ406UlKgcytzt56i00LOG8EcioJ38SMt6heTl7piqDn/GtoXMRJjXz1nWy5w5vROHeIeF78c/i6W8qSoVOnZS6M31GxLF12rum9HrtdBC6J9+lVI3LbqhZKzuBNO8CRkJSDOxBn/Wt6hYSR26heUaxzX6FeodTVBCnElSd9YInv1HLnGtrxcF71N78LcNVuYLIoEsEzM9US2defq288bjxB+esPXbfH6NbRMVJ1mc6Arlzk+/3Wu70Ub6/OvChmmKyTdt41NL7CQ6P/i2w9ICyYtK29a7iQtcpiW4CdvfYIFIVoPfbpYUFEczYMq33HwtcNVM5Fb3bD1MrmUCRFm2oiDuZnQ2dSxmSUnYgzZsPo6p9SI1m1lSvuMqr6pvtLgZTuBE7T7LapyoUzQVqoJynxI0M+62gSnROkfP2Wxmci0adlZj29ofXawYq526GS57fHnbMxUCZ07uduCCkZdY33sm4pvCvuvDd7XndTTTlZSUL79Ml0EoU6hClJSoAgwSBzFp2tubBMJtqlfVXI6HnrbCZ1B28bUmwN00eJuK8ZXDhyuocN0lNet5UtG+83RvFTaHXUoUpOZ2JGbn3a6WDxn0zOKeHcT3tct10txu0dLVutUr1TdL7DrzKVkJWpBclJIGx7jttayGFz5suiC+0IcuZXeQogga/VbvMTqCNrRd0euJt6cVuGtLiu+m6Ru8PSqilqBTIKWwpC+zAJMdhSZ1Os+VpMC9c07WEkYYnFjtwiWPEjQ5uxQxAIIJ0+yxnDZKqZpJBloluI2ymPssTChsJ5gzYfDnZq6tiQcrocH+EkfaDaPmlKdYDcesr2K/G2WzMOevjFssqavCy8VdZiLDglWVy+qQFPhnn2/sNrV4f4Z40xRhq9sX3Ph5T1zXE2XK+sW+022yMuaJWoZ1RrlQCrVIjUC1VAVu4wwwmQR+eacnNsQCrb8LWFF8XgaI3b6dUpoVuB5dMl4hlawISso2KuUkT472WkBymyiqiLi6MN1bSVjrc5EwMqoIPzFjFRW3YWx1LOhEyskyY39/22SCoHXMowYPIwdLDMpQ6U5qgIhRAlOkQNe4/hY0hCNPJCh1s5kqUNBKUxzPz9dtJWVEzAMQnv9nj899jNRQ3Y3SreYvVLjgUkBIGqgQTIPh9p8rECiMoBCusGwIJ30twddc9uVPag4X4xvfAVVxKuy5fSbioHVtVtQy82pTUATnazZ0p7QOaIAJ2FmY68tQJWZSTsTIPKR88rGaOqcbplUnpz7FM+R1yW1ylQSZGZIMKiJ12snKcha8yARBJ1idOfdZG3F7rtCu8xkqJIMyYO8efzvYJDgK82UEpmSNyeR+fG2LVmzTqVEZezofsibaRlyiO0IPzPdZx1XWI1U7YWdF63LS3oO11rSUrlMnMNFffqbLdddtSm5quW1rIYUCk7ER8+VmTwOvBNQqtuN4pzpAqWcx1UNlgf5p99p2XcD711vuCQosKUJgBRAPdv+FsPis/dXuYVv8ACYu8xNeF5ucY2H09J2k9J7S03hdRKYnL/FHX4ezym3sJSAdUkcot5LdI+kTT9K9sQqFVtzqUVEGCQ1MQNRM6eJ9vrXSjM2Bvp8LSQvzwRuHRV1SzJUSDxQigSOcROnKxd1IMgjaxowRBFi7oB5R7bExoSREHk7nn8bFlgiCRPnpY68NNt+UWKOIBJH7bGA2CDdutULIevFhtQEKVHws5PzUlRhAjnvZKw6w07fVMl4KWCVREAzlMWfSKRjQZCVkxEWDmqOC+ytaK3D1TcRdMwJ1Gu9hhd2SVZwIEWcD13JacylJmJIja3JoUqOiDaI1pvqjAWHUJvi709qcip5kGxrqk/RpAV2ExpG3tNlX0FUgJTaLccOYrp8Z0F13LcFRV0dU6EVFSkuJSyITOoISmNTrvEb2kgrpM+WNuYn5IDEGs4eZSIhCQoFSkojXUo+ybNziUlqouBwJUlaksuzCgY7PhaOcT4UxwtKlXQiscUqBHWKjfkMxO1jWErkxLRU17sYhuqoZb9DAaddSe2rtSNSQTtbVzUQFAaovANvV5rMUWJvkrm0xiIF9+Sct9sZqWmWW5zgHbwsA/hS+L3oLrfuileWOsyrW2ppOQFZ7ai4O0lMapGpzWXahhNRdtGVgKlCDqJjsWc2GUoTh1gKVoHFJB8JtQNyVYEbjtqr+riyx6ph4h4cYhcXnonVqASoANu6DmDlkH67awVhnE1w3uVXpdjiKRTISKgrSe3nEAwZtKiUtAFYQtXjB+y3F6om7U9UCkpdbVG36Qne2jZiT46buwAyrNtwqJtQJwTdNdLyytQXUBWXUdv/atwy8DW5jVNhtAkjONPjYk/RVJrHUU93Z0CcpzkSN9Zt0xRrbqCp27UJAjtet+FgC4lWoCgTjW91uKWQptIFJii63k6DTNUto3/wCs+Np3ww4+m4bufQ0kgUyUEqI3H7OXdaEOMNOanFl5UiArO3W3TUxGgitpTPj+laa8LpLtx0dOsKhlx1K9YgocUmI8x8LVdPViWpdFfUfsVnVU+Sna/wCCVKt91xoF8tMgHKfpNz7rRP0gW2DdWGK5sKc6m8iC4k6JJA0nxg+60uXilostpU+4k6qhJ+4WjTjq31GAKaqKFuCmvNleZUyJkD2WXFr90eRyUeG/0hqkCrQczb6l/wAakEGJ5DfS3bCkGApxSlaAQgx9VgKatKqCjfQ6T1tM0sJ6sKiUjY2ONVlSuCEKjvCSNbFRm7AUNILOIUdcZbozXq04n1X6ZCVBZnWSNfjpatWD8Piuu9VAXGqcM3lXNKWZKUgOEyQI1hXtJNrk43uxN5ppX10qzkRl2kyCDraAcH4Acpr9xFc9cxUHJertQ0tKZ7C0oUNjGsRrzt55iVcKevmaeS2VDGJaeJM1zCtXelz1VCqkYdaVRuBTTmVKXUZCChQXpqNIPfFvOBpT1MVrUHepp/4lpKVKEcogeAHjruTb1/qsBU6wptbFUslM6IEx4mdxNvKnFuEazDPEG/MDs0tU49QXrUXe2wGlKcWoOlKAlIGpMiABJkb6Tc9m8Sjq3PaDqLKvx+mMYY4KH8VXfUPj093sPI7coEFI07J1nTv8+42bt1ZQ4ElKRt2iCI9o87WKxT0XePtTdJrXODOLChtouwu6nUORzhspznUzAE+cWrozTusVDtPVB1h1pZQttaMikqGhBHIzIIj8NSHtcfRKzRY5vrCyfF3rShpPZkOgrCSrNKVaAEbkwdCRt7IOqrWqdhD1SslGiHd41JTECDsAeZ0PsRLvq3X2A0hZdK8oCB2jEDaN508dRrOtlanyO0yutcXpAVKSlUcxJA8PA9rwIkO11GN1xT3ZdGJ6xl+pp6irS22GhlWEFREkz3SVSdASZM7gvuk4dYFBaqV3cEIBSvItySUyDrpsde/ukQDZtYRpaVtQKWg3mzLLQKYCRB1Bns7aHuHMxaRmqJVfTuJaqHGVqQQlKRMrgiOzoRJOv3CRHu1RcbRa6elZibEzTAuSpxNiKookoy+iPX1Vu0mWB/eVuFBGgPqx4CQC1ryqa2pSqmZp0OvrlKy4hJSCABpO4n2gGQNYK0qhYqMzi6VbildsoNQQRPag6zPqgiJ7J9q/ctwsXf8A3e+whlzKCgNlRCUp31OsxruNCNzMgtj9PRGF4DdVD2O8Fk0tPdD9a7U11e+X3VqRo00kZin2lSZB5DTYi0PY7wevDlSHUCW1mCMuUDuI11BIVr4WsZiDESfznVN3LSi8b2dSGwAQUUzYOpJE5Tvz0jXURaPMScLr7c9IvG9r0Ljr6At1pKRlTpBg+3TTaNtBY6N1tEHIy+qlv8mTxMVcXEq/eGVY/wD3Liig9KpEKMf3ZTSqB/OZU7J/kJt6VtVRplZUkgEZhpAnn9lvErgHixXDjjxg7Eq6gNt3bftOioXOnUKcDbuvcW1Lt7V1zT3WJDLa15SQqBMRFs92oiz0Tz8PxV12bkDaxoPj+CUFXiY7Jkkzvz77N7EN6ENKSo6DUGd+/wC2wjTpW6aaRmB9UGVT5ffZvYzS/RZC4haetQRKk67/AIi3mUEQc4BelEtYNFFuLmgLzddSSC52yOQB5+6zQdeYzuMrcRlcSRqY9n1+0+Nm30rruvq+MLXa5cbFbUvNVqFrbpW1rJSWlSSE90DwE+NqwLwdjR9YSvDN9qUEqSFGidIOviDv92lt1QUbZIQ5zrLIYjWuhmLGsupj4/MtXjwRw7Wp7Zu68l05KSTCCFjXXUdkDXmLUtvdC26lQcJKgpSSTuYMWuFV3XeA6MF53ZeFI7T1N0XtnDbiVJWAXE7pO3rnQDnao+ImgmseITAzBXqkRI2I9g95td4P6GZg5FUGL3e5sh5gJHSQFSSTtFvU7hDeQvHh5hSv63MqouejKiTuepTPxt5oYUwJizGbN4uYXuZdeLrYNRV5FoSW24PahRBOx2nla/nRivxFbwRwq4tSlKZZdYUVfo9W8tP9VKRG8WTHmh0II5FEdnXlkzgeYU+3bXMoJQ5UBgKTKHsk5SAd/CCfhZn33gPCtWV035yqallxx5amkpSlpJdW2pxIBSSQUtpbjQBBIA2g6q8FdYpAy7qG4IPn8dbJr9a44kpzyJjuncxrsPw8LY8R2NwtsZtLLm57jocPKeUzVPvdc2lpPXlJCUhxxcSBr2nlnWdCByMoxwjc1M4y7TVNYk07JZQM6TAUwyySQUkGUMt7yJHssfcqkGElW0DNIJ9/usl1eIropCs1V4U4DISFguAQVGBI7jMDxNpQx3JQulaEpUamrpu2lu2mU4tqkYSyguROQSBISANNNhFg3qxagSEgd8jtEzHfNmdiTiXdNyD0dCautqWniwptlEkLKAqRMaQSbJVwcRFXxfrly1F3uUyStwNOFeYKU2e0CCNCBr7bOEDrZrKB1S2+W6pzx8utF2cTcTMNg5fzq7UA8sr6Q6B/nH32j0KmjCNew5I8Mw/2RacOlhdPofEKprQjWvoaSrKhO6StonXl2E+8Wg1rMW3UjUABR18Y+22+on56djvALzyubkqXjxV3uCN7qvLhRhx8rzKbpjTqE/8AFOLbA9yRa8PQrvjrrrxVci16sVNNVpE/roUkkf0abefPRdq/TuGBYLhBoLxeaAmIBShe/Ids2ux0Oa/0TiHet2k6V90rWByJbdQQfYFKtl5miKscPFamJ3EomnwCuKsk6R4bc7BKIBIB08RbtR0AUfDa3JEiTuBBseNlXOQZ022FkK+Gsr+eAAoTZdIj9Hy0sm3w0XGA5MFBmxVM7K9DTtzNSIFajbe2gSh9YP6WVX2fZbYJGsRp8LcunK8lUq7STr7vxtbMKqHBdlZ8/q9ltOIQ62ppYCkOJKVA8weVgySZIJ91hEnNoTrt32nabG6YdV5qYHut3CuN8a3ddtGzUYkuGjrW7lbeaDkVLFSlLim0KEKdFOl9SdyCJGoTZYvylf46XDevED800VzX1RV2Uu1F8OGnvBCkrddbbNW8oIWzqsIQQnIopyghILn4r1+D8F9ILElJia4Kena9IReVHeVKy+qr657q3ZkPJSIKnYKQNUpEzJsTxFxi4ZY06hzFar5vI0zRSy3U0IeDaiolRBW6dFdiSEySCSTpbXte52WQDkFnXMa27SVK/QMvoPYRxNh1RzCjr2atsTpldby+6WrWmSdYJNqb9E/HNyVHF69sPYfu6lpLrr7qLtOU0aWHnHGy2SHMpIMZnI8LXFSoR2TA5R+21FiQy1BNt1a0RvCAOSGB5gRYe51qTfhbnsvU4I80qj/WsVB3BE6bkd1u6dZbvWgezQnrFNK8MyTHxSLAotPArIJAbVp3AffbLdEH9Yf4tss6yYvC2jn91+Fm0qnNebapJmE5SZPttPgbX1YCO1mMlQjztD3DW5qbEfGPBFw1ZIZq70Ofq1QSEtrJg78teXntaxmNcGXhgquRSLqEVdK+VLZdUcqiBGihO+w8dPK0NLURtdwidSn1FLI9nFA0CaxSpJnJpJ5ag8pHz8bdISpRKCkkDSRzGvP5i3SOtTBI3Gv8r4z323L3IJIPZnXUa/Z42slWlpC7XSVjSmm1NgKfbC0BLgMA7EwdDzjz77b/ADfVqcQySMyoIKRO+3LU6+duAt6MwUQBqQOXd5fb426U+/IV1hgAakyY+fusxdyXTlC006oKWVto1lCSSPsn7rK1Bhanfp6mtbvy7Io2EVPozzjjbr6VEgob7JBWnmJGhETtZKDz2TKXdQYInSO753m3eZzLn6wdrTQd/n4T9dkuU5gG6L+iurcOVpSYVPq6a27au99ap6lfaV5SfCfP9thM7oP8ZAHZOuw7z8feLGqdx4K7LipI1jSInXu+TZrn2ClawEp1cL6C+qDGN1XjTXe86lFQlLoSQMzahlV7wT9dr60vDp5dIQqnBK0wITEey1NeEF2P1V8MuP3y5ThtxKlZRooAgwfu+R6U0nVqpmlIAgoBHutkcUoRitTkLrWC0lLVy4bTjh+0V40dNPCj2Hel62pFOphDjtyuBU5s57Ikjlsr/Ft6a06fownuHIcrUZ/KV1NGjpQYVabADwoLszwACT6U6R56AD3d1r1U6YaBjeywRCCIQ+7p5JHyGY8R251XZGmhnUc7AOhXMbmxkgBOli7uipBnzsXGoJNkSdTI5d1iqgORmeVjjwJBg+6xZWnKR3RFihsgnbpawXRLqL/pgEkhOZR8otJ7NyUzDgfbR9IPVJUSB77MDh06lu/w3OrjS0/UfstKVo+7RyuLnbp5lcwZWlJ5u0qVmUsEnnFt/mxP63wsftq3Cgh6JvGf1Scq6hmzBZFm3ftG5T3glAOixM+HyLPazPxklb1WhhDhbJSntp3GtpqeljhfmakdK54s5EUMPmSF/wCcbA3ywwq7FNVL6EBXZClGBJ0H12GZSkaheb+dy+FkDiI8lrC7z6icrbrStAN8w77SVl3QPAPJSUpAlafFG6yhp6OjZpXXEDqkJSoz/JsLdN73RT3W2hV70TIS4SQt1I09tmTijFyUYb/OBElxtk+sNykbe+0H/utdvCrVSdWpLayqCFTty+q1Xhkbw7OSjK+YFuRW6oL7uW83zT0F7UVU6gZ1IbdSogTEmBpqRZEv/iVw/pGaigfxfczdXSuoS8yqrQlSFJcAUCDGo1kb6WrNwm4oO3Ni69lv0TTbNPSuMrdzmVHrG1CJ8Br9nOrfG3FSa/ibi6mShzML5vFpYAScw9IXEa6c7ESYg5tQ6C2gF7odlLeISL0lVxV4fpa6449w6lDmYoJrW06DnqbJTnHfhC2ShXFLDSldzdahcxMxl8reQ96UHXVBKCIkE6GPdzMTE/fZWwut2idbaSy2Uuk5S4BppHZJ+dbJJiT2i4ASspmuNiru3njm68b9IfEgufECLzuP9zrVVTqaI6kOsOsLVGkk9k6/eLTvcuOsPXbelfhyovkN1jV93gyGyy4oQp9a0gkJyjRQ3NqEcHa5268WXqUENn9z95pMdoKHo6laEHXUD3+++PDSuFSziq8W1gqexC66DodFsskfXapoXSOry9thcFWdU1vdAHa2Wrx48cOqSodbVfF7PFtICBTXHVuSfAhuOdo16QPGjCl7cLqq67vu/FS6x2opn0KeuOop2wnOn1lrSEiQRuedp8C2g4HesWHNVesDrZldIS7kXjwNxMtvrFOttNvhSpUMyH0K1kwBp5C2jmjfJE5rjpboqeKQRyBzURw7xFvBjDt1sVvC7HSqiloWWXibtbQnOlABMqcHMWXqTH16VKEqZ4XYogrAPWKo28gOkmX9LOlhTtRSU6golCmUK7JOspB10NhQwVslosFSSRIUvf2RZrWPAADlzpGk3ITvwQgXncDdbeV0O0bzilhVPUqbWtsAxBKFKSZAnQ87FLnuy5HcZ37/AHI0XXAw96sT9GEz4+rYvS38q7WPRpykoCgmR2dI+y0f4axg7RcTK9TrxKKi72gEk6ZkqXr37AWw+L4pTRVrYZGg5TdxIGv8XVtSUU74nyMNgRp+P7FNyriukyTRt66nS1aqzgTgW4Ok7fnFZFxMuXrX09EaVSzKadwtOIdebSRotYbSkkawDEZlTNasdojQju3tHfFHGLdCmnxNSt5altCqYvZk5UgkKTmB9Y6LCQOavYXx41h0srY6ZuUnmBa/gp6KhqWSXmFx/GqJ4vv2uxDhi/afBoNTflB11Oyy042lSaiPo0kulKJ1TuQnXe1YOk30D744tcJaviLQcLE3fxZoqFqqqXKSsp0C9logvIdbbWptbhTmyqEKKgkFRGlrJ3bfTGIrtYZuSpq1V6nDVslDrYRUsrgrYQpROVQVK0yCN9ROks3ViOuXclKi8aFwVnVJDiHXUaqA1IIJnlytooXxRXcT/HRJiDZCA0DwsfxXzg0q3WKstVCOrW0vKpD6fpEwYjLvIEiI5mYmy3RVjr2WibTClLgoS2EwSN55bxryI0nZz9KG/KW9+khxIvWm9GQ1U4nr15aZCktZg8ZMKJlRUFE8sxURA0DSwyy5XVqW2UBSW05yCDqQNAJ7wk692u02uA+7A5Z7JZ+VSNhu72WWUCqqEBoASlC/0oJUdCRqJ7vVkc4fNNVUVLSrqlKShoISyUkwXSoAjQb6kxpG/tZ9zU6l5R1i8mYJdSopCUo2G/KdQe46DXU+muVeOIE3Wwla2KJHXLRv2wSJmNhrseQ7zIt0SNE9aI1CENPNkNqaQ0nIAEhsgdoTGwJPgSNNIsbxBeLaLvcVeF7GhohrXVwXCglIHZbk81HTQaQdZkNlpy9mmli7LkqrwW4taGW2mZAUCrdX6A9yRJIUCQLNC8jV35f3ot/BGJb1bVnpcO3ctS6amE+vUuJBBg/oJJPfGtkaxOc8JwUF53niJlbmBaNGGMPtryvX1Wt56moUZADYiQSD6xGmbcbWScZC+7ow7eNBhK6kK6xDhdvGrdTUP1LepUpojQaJPlBjXWzuTdePqNDTN73lfbmRASi77lu5pFOyJkIK3QluAY5nmSNdR3aVxSXbwvTD9LQ0CilLqWn1VbsgalxLKVNpmIkgCRvNuddlrLmencFU6653r/SM5LubPm55pmbfQL0asTXbirDOHsSXyww8m/MOUV4nrEhYDq2WlqidtVqt4K46u6juzFNfT3bQvUlEXSunbdMkIPcRuJmN/O3sD0VL/ea6N3Du9G3Fy1cDLGbwRmbjy7PwFhcWJ7o57BcgXTqCPPOIibXNlcaiw3w3u29Xb7p7rp01TpJJUoqQknfKgnKn2AWizpQ1dyrw/c7l3MU6HBVuIUptsAhJbOkjyFm6vHb5Bh477T8/Isx+JuJHb4uZppx2QzUhckyNiPn2d9vOIq6ea0LmNAPQLbxYOYJBO6QkjqVHF4PB2nbW4d0EGRB3PKyE+gpUSpAVl0HiJ+w2UQ6XaUoIVLbq4kbTBE+0n50sQfbKlpWrVO0J8e7577XEVwLKeU31TAxtRJqsE8RbqCRIp264CDqQjNP+ZahOI21JcUSVdpKViQO/XW1/MW33cVy1t+3df9f6ML9w+6yyktrWp1acyCkBIOwUPkxag1/ozEgA5gjU666j2d/wtoMHJubrMY1Y5bKTuiVQXhffESqwzd9eijcvOgKM7g7CodbJSrwgn69rT30RqhwcPLyuRZhV037U0sEAHLCFfWTp52ptgPFysE4gF8ij9KQWHGFtdZkJCxEhUGCDCge8WtJ0MMQfnVeOUpbcaS5Xs16Gs2bL1vWyJ5+okTGsWMxWK8DnfBC4PLlqGt+KsrelcaG737wCOsDDanCkEyYTmI56fjvzilziXfTrbriruTSPNt9YQTOziZ33BbWDO8zaVr0pzU3S42kCH2VIjwII08D52ZWDrowVedAHMRLZaWu6apRcL4ah5tg9WSTHNsjLzIG5Nsg5zYWl7hstLX1D4GgsBJ6AXJ+CiutrcUX1dr1det+VgYYWioyJGQBC3urUiQP0QJ/bZPvynXSVF6pbecW6aippnVrWZUlCEutz3Hs93ws8cXvXEwu+bqw3Ul67Kxla2QrNCXQAqASBIJH1Wa19Ol68HVqIBq3aKoAJgEOsFtf3bcrGwSsmbdpTZqKupWNdVwvjLtszXNv8LgXRJvEpdvBy/qgFQbqaKqc01IDZQsCNJk/ts47mrGV48pw00ptTlauqR1nZIS6wk5TGsk/JmzZN1ooLuYq2FOrdqKbrltlPqOJW0rINO0ZSrUfdZeqr0ava8ncRXa3UIbSKZ4dYjLK0LKFCQY2I+7lZHyRlpLXCw31RP2RiUUrIZIHh7zZoLXAuPRotcnbQJjdMSjCryuOtbby+kUVVTqUAYJacS4P66vfasLHrKTEyhVrg9I67xjnD1x1GHUmvfYrnGlJByKDa2jnJzRoClAnytU+8LivXDl6N0V/3e7SuGDlWPWSTEgiQRvtNtHg9TFJA2MOBd0uL2Wf7QYFidFI6pqKaRkYOUucxwAd7pJFgfDdWE6IV4lV34nukqJKHaaobRJ/SC0qP+am11OjVexu3jNcQUrKiqS/SOAkSczKso96U++1IujZQVGHaapvCtZbQa1YKVNuJcUtotylKshJGuuU6g8rWa4YYqNJjXDV/ttO06aa9qV15txASptPWpzJMEj1TqQSPG1DWVEc1Y/IdjbfwC1EvZqvwjC6eoqI3ZZWZ75XANu5zQ0ki1/Rv8CF6WmRppvtNuAsbR7LMW8OOvCWgqqmmdxgy6KJZaqX6amfqKdhY9YOPtoU0iOeZQizhqsX4ap8LP43VfVO9cTFIuvXW0569vqEJKlLT1clQAB0SCdO+xsc0bgbOGniqSowjEKcMMsD25zZt2uGYnYC41J6BK5MK0nU+FgKhvrWlo7xaPrk6RPBjEd1Xpfl146pTQ3N1Irn6mnfpktl3MG0jrUJKyrq1QEydNrKmAeLvDvim5eLWBMRC9VXX1Yq4pn2Q31mbLBcQnNORXqztruJkhqoXOAa8EnbUa/BOquzuL0kcktRSyMbHYPLmOAaTawcSPRvcWva9xbcIdaci4982LPrhbKiR/GRv3g/hZRvFrq6gzrmM7xFkyvlNOViewQrY6QZ+y1/GbgFZCUZShgqIkzrztmaT9xsHnB2PhtbYWCJAO+ptOFCmhizg5wwxzexvzFuDaG8rwS2lnr3s2YoTsNCBpPzpZPb6PXBBowOGdxTv26fN9ZNn+TPM9+3O3WYEyCD3C0wmkAsHGyYY2E3ITXw7wr4b4SvBu98NYIue7K1pKkofp6ZKHEpIhQBGuo0s7s4kE7k2BTMkA7bW6CuRmPstzi5+riuDA3ZDZoVBGpO4+dbdPr6mmNTB/udSHgfBKgT8AbBJOvfyAIH3WHYaFU27TFRhxCkwOciLdyXJ7CFAEJGuu4tlkWhvqlNFTl1xBX1SMxJgzAmRbLddNXj90fKc1vSMwIghRSmorHEk6epSPHTx0tafpBN9Wq65UpJUt3VJidu61Vujpe913D0icJXtfT6WadoXhmUNSAqlWkKjulUk2tR0h71pKuruVqgV6RTOsvPCqbILWpAiQdDAnyI8rUjGu7+w20srkOaMPeL63/JQ+kL9VAggyIG89/um2BIzEKJURAOnz9fMW5PrAjVME67fhpbS3FpU2hLYymCpeYaQNt9tY99tLmWbcCdl2kFRJJUQCImYEa/X87y7MCcMMb8Rql6kwdcjle7TBKnUpcQ2UJM6nOoTsdu+zUazFSSSZyySRzAOnLu+dbTd0beMFy8KsQVb9/UNQ/R19Mptx5hSc7ah6vYVAUDJ56eM2Fq5pI480Quf4uh5c7BdoTfxX0eOKmCbkXiDEmF1UV3sauuLfaMa7ZUrJ1P1De0dKaU3JJnWDrBgz8/starj70ncIcRsGuYbuGhrkPLWlYW8pOXQiZAJ7z7rVWedcyoDZbU2VKzkmCdoifbz9loaWofKCXi3ROgc6Qu005LlIWSCVJ5QSNz9nz42N0hS2EkwDMgSNftPnYge0kmSRGnzz0+s2317rFdTtdajqFJX1hPrCAI8BInfwsQ43R0YI3Uj4dvxd1NhTahmUkxG8iAPn5Ho9hPE7NThq66ha5LlGysme9ANvKBrEtMh/qyshBQFBQXuZjb2fEWvphS/L3RcNzVaLyZF3IuZpDlL1EuLeyoyuBzNoAAoZY1ne2Qx+aSiyyxGx1Wiw6mGINMZ5Ki/5RW+vTOmRdqUOEIapLmRy3DqlDfzOvefZb0epSS2mRuLeUfTfvFd4dLVh9SUqGe6EZ1KjZKDHx99vVyk/ikGZkWlpZDNTxyO3IuopY+DK6LobIbYad1i7gOszYzyhI2sXcmfVsbGoJNkUd03E66+Nia99QPDX7bHXQIkqg2JLIk77WLbsg3JawY/1GJKI5oCllOvOUkWl30jxtC2HXervyi1g9cmdfn5NpUD6dy57rB1FQYH2HNSRx8QXSqalI0zC2jUp/WFod4nUnGKuvZj97+/rvoqHL9L1oAXPOZSZ8Pqsz2MJdJV9YNTxLu5AkHKhMEiRpIR3T77VTsZka4gNU/dNLqyJqQP0vjYnXO08turbbUpSwmSkEx52gqgwDx4NWp68OKbC2IP0Scw321y8rJ94cMOLTRbdreMlQ6jMEdWAsDMrTNoRsT87iJ+MTub6lkraW5tZWDU/SBGjLY/wRavWN75qKm8b0oXax0tJqHAloqOUAKMabWPYX4eY2ua/WL3vbidW1zDQKVUwQrKrUa9pR3iDIO50sx8aLrKm+r5FHQ1DiuvXByKgmdwRyibPpK99QXCXTTqiG0xis63NJGM8d3XdmGGbuvEuNBLTQUsxlToNTroNN7RZX3t6E8aynQVBIKgoRCkkcld0eHdvZZx5hJeJrnp6Wouh6taqmk01UhbaiFz2SDptvM2Yl/4PxhRH8wXSmgZpkNtN0oVUllqnaQ2EZCJzA5iADOscp0Ko8RjaSwnVES4aZrOCRWeIzVff980t1OJhmkfFWAUqOdCM0KAJymNYOugtE/F+voU8XsTUorKf0movSufQyVjP1ZeUVKjuHzsbOpzAuJmhetZQXAwxU9QtVe/T3m6kOL6vtaBPNIiFeHdqVx3woxpU3mzWOYSTU1zdUmofqG6+qeVUIOqk51NAQolJJndItA+ojfOXE3KMbQlkQaTYKML2vejuymbqKiqSy31jaCRrmUowAI13+RyX7jwxfDrbl7pUtFKk9aSpRWcqQSrKlMk6JPvOlnTQcJcc3w43d12cIn6ioQ4HVumoV6iQSZzJASAYjTu87SbhXgff2EaO78YXhNI9RoccduwPLgJzIbAdJjPqvNAA+9k04d+jj1d0Q7qeONvEe4WTc4bYer1u0+I2AFXfeN11RZU4god6t2mcAJSdRMgQRNrq8B1uuYPvGqcTmFTVU1Scp17dIx4eFqvXRVXpXXsypTiPR1qWlQCRMFJgbkx899rM9GWpP7iXAFQDRXY6nNGpNKkaR/N/HumwguM4LxbdCVRDoCAVKXZQkdY2RIkS4e/uAs3eMFSyjgrjIsqSG2bqqHnOtkDKEyTJ20B91nHRtv1byUind31k7fCxXHNxi+uGuKLtqqdTaau7KlIUkSUQgwRI3B1ja2scfRPwVM31hdE8NvuXlha4qykKlJqrrpXc6VaGWwdx52UfzdWuTKla95Jg+6yHwvfpU4Cw3nvJK1M3VTtrKlAFRSgJk6iJizwRed2kaV9OOeryfqmwrZowBdwUzo3X0CaXES9b4ue7WFYdutm8b1NOkN0TlUGA4hLiErXnIMZUqKtt4HO0U1+JUU2L2nAe0hooUoGNNIHnr8bSrj03LeCaC+aWppax+iL9M2404lwoCwkrSCDpJQmfK0B4yWEXoLwaAC1Nq7SlahXYAgCTGhk+FvLMejY7Fnje/5BbbDbihYen5p3X5xOFz3O9ejjNQ+Gl9tNOM69xsPaNN9LMy8uM9fetK/QU2EMRr7IIdFLKAdwcwmCIB9o8rIVRej5lSX8iErSciUkJEjaRrEjSIn3WI196vrYRDiUpUowhBgTpqdNSQBqZPutXQ4a9sgkYbWRUssh9U2Cf/CtvA9Pjtl+9MRLuWpqVmoKHSKZguNwqQp4dlKxuElJkaSJtaCmve6L4ZYq7pvikraZZWkO0rgdQSmQRmTKdCCDrobUKQ5nqGnXezKhmKjpB2Bk9reIidbWd6NeV3htTkErFJW1TSkyQBmKV7bHRW1txQzGRpjeFTV7cxEt/BeH/GSpF58XsbXiw28E1OI7zebS4kBQCqlw9oDY6ie7vsb4fsNFl2vU1HbCU6mY0iDEcjvEnu1sLxbwveiOLuM6JqkdUlrEd5tSUgERVuCAdtftstYKohdVEhL5S0W1BQJb3SInXtSqO7uIjQganMMgAWYAPEJTmcq27moDWLWgqErSFbhO4Osd/nqPCyXcFcLvuxddUstl+sW4tRKCgNiZA0jSAYnvPOLI1/Xk9eTzVI04QiW1QYIMqiZ3/Vmde8d/OIKpmkw+HJcSW2cspaAQTExmT4wec5pOtoxupSbI1cV14hxhRqvRVdU0tC7LJaoG3OsqGype6gcpEnL3Abkaiz5w1hnHFO2bouErwxRvqCnOp7TrhEJ1SglDeh/SUtUntKg2i7hFd1Ve60tG9r2YaaCkhFDVqbVvMQnWNSTGsBR5Wmm6sG3LVLNO/fFT17JAcW/fSisK171hRM6Rl7x3gEBqhvcXTruXh1c9KlmsvyorbzeQArra2oU6lMagZc2VPfoI05aCx2tGHqV9bwuVqpq19tnqG1FLYKdDmnwP+LHKwV24CaaSF0WLr+Ql1CiS1ea1CSTMCSIgpnxBnYAqztxV9C0hmmv+tU668CXHG0KUlMq7MpSPAzr4TrPSN9HZOidZ2pVZekphzEtPV3TiS9FFdG+hdM3rHVqkrAgmdQY7uxb0b6MlO7Q9FPBCFEpIuEPyRMAuLcHwP1crR9wTwng7FNdiHh5jK5W8Q3df93ocfTeMuGWFkyhROZsjrFEKSQQUzNrJMYfujDuEW8L4foU0N23dQehUlMgkhppLZCUgnUwOZmdzrrYGX9JAYz0KLi/R1DZB1CjivIrqqhrXKyobVQurcSll9SW3iW1IyupGi0iSQk7KAPdYnfNUt2mWjZIII0mNfH22DLylGAMk7BR+Hhufh4WKVi23Kdas8gpCoJ1A39h8LYJkdnL0Vz8zU3Lhu6nuxqupaMOq9Iq11bmd1TkrcnNGYykSNEiAOQ3tu9bsorxQym8GUuJZfRUthSicrqCChQI3g666WHaX9M6G4lbaTAEbGBPOdT87F61fWBOVKTBkZjpG/wA+djm3JQbrAWUJdJKneF54WqWm3FKcNSyQhGYmEgwO/U+9ItUjE9IpqoqAtpaHA8tK0r0MxznXlv529BqzMxinC9YFZWUVjjKyVjLC0EAkzG+vttTLpC3cq6eJeKqJsDIi9HXgURCus7Ug66dudPfpNrnDJbScNZzFoLN4vioYIIJ18LWR6EN4dTje/wC7Y/j7rS9ljctupEefbNq3qkKUBpqRFpp6Il5Gh4zUVICR+cKKpp4CZk5OsA/0drqubmpnjwVNh7slUw+KvuDNMmVBQVIjfunXu1tXm/btvOjvl26XC0kNkt06lLEKBWtQJj1fXAg91rDN0z6mShDTkA6lKCZ+Y7/utDuI8H1ar3xFXUtDXLrKd1FS0EtKJdT1YUQgRBMlR057m2FlYZIy1vh+IuvXOzNXTUOLRT1Lsos8B3uucxzWu/7XEG/K1+SaCKu4qa9Sai/lVSEXepxTLbR7S+q1IJ021G21iy7yum+F3W/6A+h2npY614hJU2FqG3LbQzz77dqwhewRTq/c5eSs9NTKBNGvtdlxtU6QmQRI3Ea7g26rqW/qqnuurOFbzacapjQPtrYUgJUkoObXYaqA8rTTtE0bhGNbb7fLX/4UnZqVnZ3EKWTEZmmISF2VrhIGnLYSEsJAsSNAc3o3ttcRm970vNAp7tp6I01NRPO5EslSmwcwKgVcs42ieVkp+8rwvE091U15Ppp6h6nU2UKIASoOCBtGqR7z3WWLtw/f913t9A7RtJcL9M76Q6hKShbmcxKhGqjB7jtpoh01zXk3eHWvKu5hptTKmkGtYypCFkhJ7W5BUfaTZ0bW8XiXJsDuLdNLWF0yvqJG4a/DssDeJK0gMkLySA4F+d0r2sbrY5rF1xyaSEDiVQt1+EL3q3Os9JutFJVtCdDL2VRIHMCfjaL8frXUcPsK1r7hcfaeqmUuK9bIlZAE+SE+60y43uq56m7r3evi+qKlT+bKhCAKptRU6AooBCSSRJiBz9tocxhXYUrsGXZcVNiyncq7r65+PRX4fKiVBKTkgEkxJgT3WMw27YqcZTdrtfROgykG+nUhWHaOZlVieOvNRGWSwgM/TR2cRLE5ob6epDWuOmo23Iu9ujg+3Rt1iiDkVenVCI/Tbyj4kWne+ah5i7XVoVlc7KSQdQdJI8fiJ8bVW4WYwum5D+Z7yqF0y6iuoa2neCCpJW2tBUhUaickTECTO1rUKpamsaearRlD38WJBgctRuZ117rRYlFI2WZtiC4mx5WyjW/x06oTCa2gLcIxCWRjoqdkYkZmBfcVEhc3J63qkOuQGkaXvor0cKsSYyuDhhhq5bh4AXq/dguqmUh1u+LsQiqDjQUp0pU8D9ISVHMAe1qLZ0acH8Q8D1GMrrxRhRy4cP1l6m8LgpF1tO/1CHFOZ2gGXFhISkM9wmY52I8CMbcWG+EmHqRHDZu/mWKT0WgvGlvmnYbW00pTaA8lwhaCnJlJSFzE+FpAwPcWNsMXLf2I8VVrl/Yivd5y8PzbTVR9Gp8qIao6YukJSNIKzlBUqTprZaePMY3el6I6AAabbC/yUeM15ZBXQNEGWd7bOa9z3PIfmDtZXCPS+YvaDZxbYXNot4N3Bdl59JLi7fFdTpfeuyspBTJX2koW4lyXANs4CCkK3AWoD1jM5XZhDDdzYjvjFV2UCKe8r9SwivcSow6GQoNkp2kBagSNTpOwtCnDC6uLGF+KmNsXXtwjvBqgxpW0S0LF73eo0TbecLW4A8SqA5MJBPZMa2sIFCd9POxGHMHD9JtjmcdQRuTrr4Kt7c1cv2gODOHRuhhZ6EjXizIo8zTkcdpGk2PMXHVJd7tHKlxA238rIdQjrmVNgaqBG3fZ117SFUioAnvFmusBILZMC2jpnejZeZ1Tdboow51tO2sgkrQJA1A0sMFAHVRsUpcyWygiercWmO7tGPssOVgGc0nXWxrdkEhQZGcnlOtulL7oB7uZsDm0kyfstuSkASTAg+6yhcuwdOUzMD3W6Sod4sEFRqCAPO257JnQeJm0jUhRgLkRI03Fjl3rPWA6iSOVk0KJ87HbvkvSkAjxPKzk1Eau6a01TxayZOsVl22nS2WdYDREkKM84Fsslly8UuHKlO8UrmzZ4SxWnVQEnqonz1+PttOV41LiilK1Ahv1c2sabcz8+FoG4a5jxLu19uUpFLWQSNfUAg8+/wB3vmyscWtWZUDNpAnc+PdJJs2lAsmTuISjdDja3+ofTPZOvI7zqN7LyKa6wAVNLWJiCZI+dbNi5w4urCGlLUoA+qNk+/XezhTS1RTmyLknsgawdPn3WWdxDk6nALUYbYukSlLKCqARvod+7l4WzNdclSaZIGbTXbbn3aD4WLeiOkEAaAakpkQO62IoahSlZkEGTrEz8dduVoc191MWjojKH6BPaLAhJMmQdu/5iwYvKkWyoNU7aADISEiJEbTpzHss5+HtTgi5r0qXuIOE6q/bvXSrQyxTVaqdSHz6qpSRI0IPnOugsyaa7UOu1NRQMViEOVClJQ8+HEtskDIgdmZHakkwRGgjWFs36Qtt81zmWFwuqq9EtIU4G0IA0OgTpt9v4d8ZcU8QO05perrVUyHip5DzTozLUn9EaERBGtpAvmhDd3udclLiVIjKqVSNpjwJ+NohxtTtVd1Kob9dUBdrzpp1jRaUBslM7nWDpzjwsQZARYJmU2vZIlPi+svFppP90LbbyOLcRGfqs0cu8+Wx5G3ppg+/1PYEuw9tMXVTSD6yZaTvvrB+NvJPBtZVrxHd9J17TTTST6wUUvBIKkzE6Gfhz5+nOE6updwhdjgeyhyhY61ATIP0SdAQBGv2Wy3aOzomA9StP2ab+kf8lSrpZ1oqOky04VdlVRdQgif0UA+Wg+Plb19o1ANIBg9nn328d+lI4kdIVtwKSqKm7RBEpjKjXxgxb2DYcKWkhMeqDvztNTWbTRjwQlUb1UnxR5SgASRFiqlBU9r7rcu1Cikjl3ixVTriElKVfh8xYljtUI9t0K6JET5AWIuFR897dO1i0iFCSPDewHWBwSAR3z32NYboKQWKNXW5kvSkXGoeRt5i0hioPM2jSjXkrqYhWzyTp52VMTYmxhdl4ppbgwY5ebKkBXpAfSlObmImR7bVWItc6Rob0VjQ5RGSeqd9TUduJ5W4ZflYGY2i+8sY8YAsCm4ZNkFO6qxE+7NNgaDFnGl2raS/w9Yp2j663F5cp9qo2tTGkmLswt5qwE0Ybb9iltbykoIzHayXeTdZUURqGmlLQy62pRTGgCxyskXReuNnz1N94domml69e3VCUDuCQTmPut1fjyxd5Qhwj6ZnQK3+kTYacljsrkTAwu1ajqX+sbBB7/rscuOhadSkqCSV5j46k2ZmIr3vq4LqevS7bpfvt5TjbbdE0Uo6qTClExmO45H2CbR1ScZuOd3VlWbo4G195sFYFPnrm2UpjQgZgCAT3k/Zavq2uyBwItfqAppQcpa3dTPSYYonaKpSpCTkr6iD3fTKP22qPShOOUXxeF+FpLqbwqKBaGiW0rS2tKkKGioM87SReHFvpD01DUvYe4TUr7tS+tyoYq7xbHojpPabHbTmA0Mjvs1OCWCcbUV23xU8T8NNXbXu3u5U09OxUIcQptxKSokoUv8ASEQT77T4BTRPfLHccR1suoO17/RUeLPrYWNewnKN/wBibV716cM3XV0VPhxTlDWp6uoccUteQZcpgpSOXnZXwlg5ji5iW7rxxfdXo13hjqU+jvOtrWy2hRCozkgk8yNYtN9Lhi4b/q6S7r4uplyncWESUKGWTyOX5m0l3LgbC911blZd90MNONMt06kjNqEgK17/AFzb0uCCjo6YhsY4pB9JY178QqpRxJiYxyUd4F4YXZddJXM4Lutq70VlAhLb9QFqB+kKlBSio5jCY9tk7itdl3LvK4aaqw8xU+l1Sy4HWyQtKm2TAyzMdwB2Fp1oXGGGqpKChtDVcpAS2jQJOWAAPOyJeqGOtw3VlLZUtQIUWsytkDz5WEw2FtJJxpRmeeaKqGvlj4bXEBUbvCmprrxULvoGWmWOvQrq0piEqAMjnGu/jaReBDFFVXFTGvvSpZZo8P0bpDFatkKKXFtSsIOoATpaL+O113xTceKp27r7RSMsVbA6ssJ7DSktlSEqiQCQZHiO60o9HhpJuaqccdQVC6Hmc4TsGrwdSB7J+JsHM0ipu4WuVfU5vS2YbmylugThNb0rvCrLZ1XFdUmR3etbKrCeCb5oH7voH65xC0uhtpVdVZlEoJGvWQRpsdPebIrjNQGU1a73cKFGPomyPbAIgWcuEfQ3nA2b3eWvtIW04k9uUkCDuediWubexc3yQnDqr3ymyZ/B7DGHqrBV33kq67uVVPNFFQ4871alFDigAdCDGo8Pqfirmu6jpgtu7bndCEglAW2VHv16sk8/HazI6PwuHA3C6pbrqv0G7rqrK5bz9WvRH06lElR5SvTziyhx04iJuLAVNRYRq01V84yWm7rmNMrMVh2Ap5JHIBQg96k2jEVJHBxZGgm3hdWUUdTVTCNhNibX5BKl53NdqLm66juulo1pqUx1ScsoUFSPVTzjl3WgjiU0GXqZyU5AlSerO59cnXkNB3fdNreFKzAPCK7cO1t7P3hWXdSMipqn3S4px3rBnSmdkAqISOSQLQTxMfU5RpeNTnJfBQjKAgabeJlR99sN2igEeJRua3KC0aLS4Y7/AIN7M2azjqmrUBujpnFF5MAByCsHInXc93t7rM+pxzhtKJReSldv9RXagcpAnY2dBlTVR1rbeTsylInXbxER9dm+9dt2MghF20gJJgpbQNZnWRqPnwtJTgAeknSlxHolSH0Z8aYGvPiarDeIjdtRQ31d7rSG7yZQWVvIKVoA6zQEgLgbmfK1tb3r8CcOblcdTW3Hhu6kK61RSqnpGCoiMxUcqZgASSNhrbzsxM3R/mCuUzTtpCWs6YQBoFAx8bQHiFls11KU0zchG4CR+kfEa6D3620uH5XxFoHzWfrczZQ4lEeJLtFfPETFV70tR6S3V33X1KXisOF1LlQtYWVHcHMDInffW0fYgDlHVmmiM6AYXABCswgnX9VMeVn5UUakvekNoJUABqNJEfAxE+AmzExYoN4kUEqzL6hoaA6AIGkakjfXX1eQg2tWtyhVzzcpDeGatSEkDKdEplIOoJkGI7x3wOYAEmcEsM3Vj7jThHCl73QKyjcql1VbS1DYW06wyw46ttwCNFqSBtOuh1i0bOrX1wSXCpuMqe0SARodz2tQABuJMCZixPQNuJd7cVr+xS4kli5LjFOghBAS9UvJCSVHc5GXRqBGxiLOHVRkqNOIfB9PAvi1WYXfp3H7hrgKy6aioTnTUUilSUHkpxCvo1bDshRhKxMnYduvDdattHot2LdVlQGQhHZ8Ak7QD369naLWa6RXCFzi5w+Xdt104cxFdLiq65CBlUt2BnpwojZ1ICdeznS2SSE2qbgLEVzVN20iL/KaepQEsIqHWygKAMJzEpEK12keqbTRPuVGRpZSNQXBcVM4gU1zNAtJBBUmEiQOepOo1O3usuVDa0NFundISESGVEmFDVOh03SPDeLFqNt1I65uoC0ASFJIVAygwN559+2h3saqagsNNuVFQgIJB7QJzGRAAnuA5+4ixBFxZIDYo1w6v8Yf4hXNXITm62oTSKSBMoehsiRzE5vHLvFrdrRKCgp30UdNdY9p28I+NKruFRRXpd9UppTQp6ttannAEJSEuAaSecJH2WulKVZXAqc2o5iI+drVttSEaTsVB7x9HW62XJKSpBB8Dtt4T9ti7qsyFKiAoetqI1sfvxsovm8WkgJiqcymdu0du/7rJOYGDmIQNEgEkDnt3a/ZbFubleQt/G7NGHeCTesyOEJzKRkIynWNN+Xjv9VgHAlTaSCAAcuUHz+ZsM4SXgnIkjtCFannr8+/awKwrdeUayANvmPLe0zNlA/dNnGFEhdzsuugw1WNLzAAH1onzgn53qf0lbuco+IV6JKs3WU9M4jNoSEtpAVpoZyqHLa1wcVt58OVwDglLecA66pIOvu8bVn6WFAP3T0FaGSfT7oSkkDTMlSyNfaPh3WOoHWqAVVYo3NASqwPIWlxYUMpBII7rOjhU/XMcRMP/m6qNPUvVzVO27nKMvWHJuNQIVZs1SShwlQgKM6fPhY9hi8V3PiO6r3RoaOtYfT/AICwr7LamQZoyPBZOI5ZAfFXyOA+IdWT1+KKchMaGocJJMbwPmLMa/7svRDz9KcWNvKp2i7mbKj1igf4sH9bs/GbWQp3y4lTRS0AhCilaZzOAwZVykHQRHK1eL1dbpHHqgXa3VEVzxcKnFJytJjNEESe1PstgHySMlayO1zffwVxj9ZWUk9NSUAaXS59X5rDKAfZTXoLlr7wrW6avvlykbUhQU6qVJTlSIO/6WY+WXxsTauWpU+GX6irkPlpa4GUo7QStJk6aJ5aZhvFnO29X9Q3X1NNQt0lVUVFGyUJkhYQoJJmf0gI75smVFHe3o9ReCq+E/m5Fe22lsJgFQCk98jXWxIFZ/d+qrS3tH/0P9RIv5jvdyj+iDnWnqxlW8gADKSrcadoR7R42cV18MnL0R1qLwICVuoWUupVlRlHVmAnUz6w++3V4Uirovumb9MeeYyUj3biHELcKFyI5ynbb4mQ8DtNlq8mxKUpqCAB3cvdaJ01RFI1kltb7X5JKWrxWLEoKOvEZbLn1ZmuMrb89FHb/Adm9aBNLe15LStallxLIIBBjKBPPTnPIRpap19U7tPUJbdBCmwWjM7oUUnfy+FvRxbbSpyumeZKduemumv1WoVxfulNz45v6iAgtXrUqAiOwtWdGnkr556HBpnOLmFW2NU7Y2Nc1M4rKBTupUCUj3EKJ+217rlvBN6XLQXsNEVlI1UETyUhKvt/bah+aaZI/VWefeB91rn8IKxV5cNMP1K8qoouoOo/valNj4JHvtPjDRw2u8VBgrvTczwXob0QL2/OHCL0EqTN13pU0wAOyVZXfrcV7rTYs5tSe7QWrF0JL0CrvxXcpVo2/S1YG0Z0rSo6/wAxNrNk6gCfZvYCA3YEdUDK8rRM6ZfhbnMZ5+JNsPONvgbaVAOoO8WKAQy2tRUjIdNLNytbLTziQeftNnCCOfLl3WSL4bCF9YAZ8LF07rFC1AuE3kHJU1CJ/SC/YRH+qbC9rZW51sG92LwKjAztHTb1T/tWFggnfSde+1iHKuy3WwVEHlB87YSCdNu+2wABmER57W5kQDA22Js+6YRZbBPIx4W6klUgnfccrc6wN+7S2xAJA5abzZwK5Cfo84ifn58bHrt0ekSIMgjzsQEAZtRpqbHLtWC8JO/dZ+ZJZLuZX6x9/wCNst3I5pPutlluksF4lcM3GnOI1AEADLSvwQNRKIIPft872matUVbgaKiRtm5fbpaAsN3m5QYtpqxJdCg2oAt7gkmdthpqPZaY/wA9uuu05UpOVacikrMEq5Qff32jpXjZR1DDZOfDlQxT1K3luQ2hKlFJ1JJgQP2bWcn7pLuTKU1QIkQAkjl8x5WZLVSHEKKHZSlI7QMneeXz7rJTeMcNVN7G4W79o11qF5Q11kdobpBmJB5WIkia83JUUUjoxYKSP3RUQ0TUKhMykJJnu5fMi2HEt3ITCXXp3gJ+o89R9VmgpRKs4XIBkDeB36ctfmbFX7rpXEuOgv55JGR1QJ8N7RmBo0TxUOvqlb98y7A48hpNS51alKVAyqSByAmTJzfHvsNdWL7rZZU68paA/K2wVjWOzpr4WiSrp37sr3lnrqV9VQhZKiVJShSoJIO4lQkA8/cUraoNXgVPvrQG6dSyygHKNczkRqRAkHT1h3WH4TQdURxXFSPfnErD9ZUpud67qwqrEdWlzOmAo7T3erA9g5i0K4pvy9H8SVF2MsCncQwtpZJErAAyzHraT5yfGxWrvZp7EKLwU6gUreZZT1pzJ2AEnVJkCPCx7GjN039krLqrUmtaaLlVmUrOWwAQE98eXISRFoywDUJweToU1KVh+gqae86BovppWAsqQgwFDsmY2G+pkCZkc/S7BVbRuYJuZBK+sVddMYJ0Ci0knntr9VvPjo/1NQ1xbuGjecdNLU1LlM62uShxC0qSUlMQUkQDvy3531wcwKbDV1MI0Q1SMoT/ADUoEcvK2U7QTerGth2ah0dJ8lTnpRLy8fkv9pQTW3dl5/3tHs/SHs9pHsPRkejN94SPqt48dJ5VT+/s6lCMyUuXcoidAAhGp5d3w1t7B0Kh6K2RGqEwJ8LGxf0aP4KqqBaqk+KMKgjSOU2Ac0EnSwyicsCSLAL2Mkg8tLTMQz90VejXYaWDS4gpEqAPq26fMbmPZYgpQC5knXSx0RsgZBdDVThQyXBopIkR5WkGlrqenp6aodcCkuInmSqEyYtGN6OuGieS1mJKFZQnUzl5Dn5WUrqvCleoqO7b7rn6KspgShtxXV9alQIB130V7CLC12UkXR2HgkEJ61eKWkraTSMrTnW2kpLeU5VzCh7rFX8VL6paqV05xCkFQEKSFAHn9dks3LdDym1m+KgqQlpKCh0SOrmDoOc624Wq57rzXXSU9fXLf7QZTKggEzM6QJ7zYLhwu1sbo6zm9LJaXXdZeVRTtpSUhpK3I0AWSY25kWS74BN2rzkiFtqBGsQsEWWGqampUEIbCesJW4SZJUe82S79epG7pqiHm5CZgrAJII0Fs5VvM0l2DQK0p7MFigmnHSwvMTo4RrbVA448lTbR2WrfztXTpE45xrhLETlFcGJ6uipnaZpzq2VBITMjNtO4HzNo34QcVceVOPWrovTGt6VLS7vqHFJU+rKHW0FQUdp25+VmzYFLXwteHABJNiEcBLSNVZx2uWmovpGYwmvXy2ltB+2x8OlbLunqkEe+1YeHOJsRXretzv119uvF+tqUVBqqxSesOUgaH1yBA91psxlj9jAKaT0i5qy9xei1Mtou8hSkuJiAf506Aakg2FwnDX4ZjkTXOBuDrtyTMSqG1eFvLG7W/EKU8K3A5e9Swl5vI2pSlFwoOuWCQnb5i0n0tP1DlYiBqtBmNdUIH2Wg/hLxcqL5xBS4VvLBV6XJUstP1TKaxwZltlABgAaagWkq98XV133ow9RXKt9h5o9cvrIS2pKm8oOh3BV/im3o7q6F0mTMCfp5rDilkYzOWlLlI6n0uqoUkh5yoDwCUCco6sHfnvZJdK313Ow2taV9Q6EZkwEqCiAfA6WaFXxBrk3tX3bfLN33feqb1Fz0zqqpa2lLfp+vpyVISImEJMDcmDZKxNj/ABjd/D3CNbTXJQm83r2RS1LiQpdO0CtaSRJCvXSN7cyrhc/IHahIYJA3NZQJx8umob4rVT4K+t6unUshEgECCSTprFnB0bXUn02kcaKylm+W4WmR2bwKgI5+v+yyRxcvavvbFt5uXo4yK1h9dOQ02UpUErPIk6+38UbhhxBc4fCuvVq6m62L2vOlUyp3IPpC05MgHv7tzarlquPKbnQFXdPDwYhbdWdr6OmVTtGmpqMoAkFNOkE/yTNjeGWmG7zaWWGkrnRQaSAD5jX22g9XS1TSsrKMAUxLLk9qqIjL45Rvr7rFmumbWF5akYBuxsurCmwqrUVDXfQa/C3R0zQ8SZ9vBTOqXFhZl3UjcNaG577wrf2HsQXS1eNA9e940z7CkhaVpDqCDHLvB3BE6GLJfD/gFdOCuIKMVU99XheN1XdTuNXFdFaSv82rcVLhSskyInLoD2iSSdbR3hHjDelxULb1FddCW7+xBUOOIcCz1IcTm7JBGmg387Oiu49Yput1foN3XbqSkAtqUNdD+laA47hsZDJgS5vgiY6XEGsdwHWa4aqZMf1aazB95tpaZWG2Myi28lSh9I3qI7tdPC1XcbtU4oYOrfXIyFSQU6gJ98/O9nqjjRia/bsqLtrKG60tVTK2XS0xlWkKjYzuIHttHeP6hSKBbxKy4QgpKRAT29STGh+qN7UuM10OKVLJYL6C2qscNpZaOncyTqmw2pJKzmBzspOpOp0O20ff7ytQkQToJMgjnvPLe26NxluodGZuHmSpJCpPLkfby2tp4JB7RlJSfWPh+Hss+NllKHXGqRL7aU/ctchQIHUOAkJ1mD3RHL3jxtAN7sk1lKpSYys65e0SokztqPt08IsVWILjDjOY6oIgqI7REDz/ABtXu+kLTWMJUVo7CifDU7CR3RrG/sF7hnMKlxK1wQii6UKSG0pCg4SEiAZnbfw+A311jDHS1oxa6ygpWtTTJWANCYOwOg30jnHjaXMqV0wVn0SSQEojc7TGu3wM2iPGLbpxlVLIcAShpIIByhKUgctE7Eb8hy3uDpuqhNi9HEtsn6Qs9kgAakjcgSY5nQeG863X/J63MGMAYtv1bIQ9eF707CSqJLbLKspJ/nPK02m1M61stqbU0Q0lW6QkkkDcqyyQIA09p2tfvoe0hu3hLd7CgEKqGXH3FzqpZq3N9NwhTY79Y7rMLtLLiw2urAMLCHmyQUS4kwdRodfGeVqp9Jvhqzw54rDE92UrL1z49LlWadZhLVenWoQCAfXKg8mE7rdEQExaZbwdC1EzJmN4get7J8t+6zQ6XuFXcTcFnL5pkZ38MVbN6pSUEwwDkf130QrMI5pFpIh1ULt1Vm4TTtuTcN71F0VJWkrpn0/REmD6pOQz/J5HlpLvRU3lTtLerGEuLCAV1NFlLqNpOVXmo7qnTQDSzPutbd7UbJq2ktqdRmCwjM2pWggnkd58ROuwV6Z2+GX00yKWncDSsoC8+UJkQRvpEEc9Y8jReyTRBVD9BX1PpdM5XVJaIUHS4XA2oQClxoCUid1FPORJi11cN39d+Jbhor6ux1SqZ9kbwFJUNClQ7wQdNuYkWqWbzYYSUsUFKaqkQFmJGXTko67Hl42mfo6X2L0uq/W1lLLzNYhS6cJCcoWj1yNxmII/wNbBvZkN+qKYczfgtYxaSzie8EKcIK1hUQP0kgz4b93OyDmCNErJ1AgifLX53HcbOjH6erxKtYzDrmW3IHOAR58vD67NdcJMBROwASdwY/b+y2LqRlncPFbyjdmpmHwCTnozkqTpmnKlW0mPDmfjYWvvGsverXV3g8HagpCFLS0luYGUaJAGw5DxsG4pCllYIMpzT3fMeywZQoLWFKTKSdQdJnTQ/dZrLbrnboleyOuuquZylRXTrG8a5TED3Wrz0oqRNXh3Bd75cyVUztOSQQdENke/Ke/bx1sg4hLrK2pkLlJ8iNu7u/ZaA+O9P6RwYuOtOiqS8izmAOYCHUx7YFiqU5ZmlA1zc0DgqZVST1hB7oEazYMaZSNDY3ejZaq3AeSlaTyk2KJ13mBtrztsgbhYkizl6mYFq0XxhG473ypJr7tp6jbmtpJ+s/OtogYoqd+pU7WNhbP53ep3AodnKttcyPZZ8dHa9Te3BjCFWo5lN0KabWNeqWpvv19X42YlRelFTXXfVApwivF5ek0wUglCikkEEjaQTbzyrc2CrbnNgCUd2grqejxDDamqeGMtLqTYasFvqkC9qKtfob1apFsG7aa+EOUgQJUC4ZOo0IOdMa+7e2XWwKy6rqzJIFVdVfQK7UyUkqTodZ56621SvVtLSOXc1eLHo9Qhhx0qQqQ63BEabad/Ky1TP4fp3mCKxXV01fUVCEhgyWXUkFHgQT9ftKNfTj2x5qQdp8EJ/pcf6wTPvG7XmrtuC8E1jz00a3lhxWiwhaVlue4SYHhaUMBZHW7wXBLaqmYidIPKzOu668OU2VNbiGtqG2ysNNmnhKErSpKk6k6HMD7BZ58NAluhrg0StCXgEqWBJATufGLCTVEc88fDde1/wVaMUocRx2hFHM15HFvlINvQG9k4yNc2bMcwiSY9us9w77U36TV1+hcQ7xcQCG6xmlqxrz6vqjp5tm1zykOKJbGh1EmTzm1WulxdimsRXVeOQlNZdjrIHMFpzNr7HLXeEvy1FuoWtxhmamv0Vck/xa0+IV9n22tt0cK30rhaw2dfQ6x9gaaCVBf/AMw/DutUlvZaQfWT9s/Zay/RSrg9hq/LtmTTVrb4E6/SNkGP6O1zizb0xPQhUWDutUgdQVezoaXgaTiJe12qVKK26llMaAqbdQR56KUbXDIESCDOo+YtRHo23l+a+NVxSopRVl+lXKtDmZWB59oJ9tr2q1HfPMWpaN2Ziua1tn3XJI2nXwtqR+sPO2E9/KbaKp1BB8e6x4OqBKwq1kcu82J3ijOxm0kGZsaOg0V99g3EhbZTrtGtp4zZyikGYWTOvGEVVG7Omdbcz3pJ+tIsIpRie+efwtziEFqm60JH0Dza5OgAChPwm3JcJ0UojW1k3VVjhZDAnUE98624zic089t/hYMLndQ08LYlRiRGgn2WlUZ1QhV/KgEfIt0FDTUa2AKgdSoQT8LbSvWeYGp9lnBNRhKhIJ0187KF3GXhJGvuiyUlR0MAjSN7HrsUQ8SnSTZVycPaOuRRnnbLdJKgkAEAAaDX77ZbrpLLwtu1xbl/NKZXBDZUdMwHaPP5+GkkLvFlhtttKyHo7at9gdAeVoyo6l5mtD7TqQVtBESSZKlDKRGmvLfv52U6Z41FYg1lXkbSsfRJ/S3GUec91g4ZMr7KeVuZl1JtHX0S26ytq65ukZZZ6jrVOQn1dTJOseZtA+HbvVWY1o7ppnWVxWoR1qAIWlJnOCATqAFa/GzscxBeOI8YLuu4GUVNNR0YQ5TPNgtAlQzdk78tYJGum9m/i6mrbuvFzE7eWirCsEtJR1YaUEfogcgRyncb2lkqA6QN6KJkGVmZWYkIJU5pm3BH3wdvPayVXVtbmeoKJxoPFOdBUdcuo+EWijBHGKtvx9u7cRupbeTJLqEgB9vZUp/WA10/V2s5L5xLQ0V4NV11LVUrZ1WMpIWnSRpM6/UddxYwzNIugxEb6pOxrxAWmq/NakN9a2328gJOXLzkSDOmnLvmyNc+J6Jy7mBVQHIepS4rQhtfqE7SAREnXWzRxFUqvW+qq9A2XjUAx1h7aUnyjbbv1Fk7O6Gg51ZWEBJSoiFZuYJ2mZ9kbWDdISboprdEvXQhiorAahRdV1yMpgHOk6az3aGe7u3ssYpXcFDVtMuPIpahKFJSoJknbtBW+06HSCRzAs3Oto6a52a1kI9JCy2shRhBIjfcaa/JshXs+uoq4qH2XVSlKVpUSQM0xA8I2kWbm0TsuqfHBCuQzxhwshuFBd4JbJAiO0dh466ad3fb0Lw836NdtJTwAUNJb7I7tPnzt5w8KKly7+KGGA4VBCr0pcyQqRkzp7uQIn2jxt6R3emClClAlKtYG5+q2P7Rfzjfgtx2Y1if8VS3pSqfXxrqS2ypaGnLvKzAOUFKNY5awPHx0t6/UE+hMn/k0+3QW8jukwFucYr9WZyNIu0yDoZCAJ7u0R7uWtvXG61Td9MY3aR/VFrKE/8ADR/BUtV/SpPih1SRuQe732AWQN9bDr79vbYByEmZMTE2nYhJN0UeJykpmyc8SFGNI1meVlB86En50snPEpUc3sFjWIJ26CdcKGwsqGh2Nniy1d95BKK+maeQNUpcQFAGzCvReSgdUjTKgjQjfX32b12cVr2oWWqdbFK8tpCQXlAyswJJAIAnwFqnFIpnvY6LkrnCnRhjw9S/U3Pdd3qQugoWmS6CVZBE91lS5alp6nWhKu0hZQrlBtDlZxara6EmnpUACAoA6mZ79LAo4sX5Tkv0yKVGZARAbnQEwddZkmwk0U8kYHNHgxhtgprvagNZdVZTZykvMOIkcpSRNqtHB7d3ut1qrzqFhhbTzgIhIUhQJGvLTWz9HFzGDyC0uppu3pCGk/O1mbia8Kmquyu7SkhynczJA37J7vO0+GtfSxvY/moJWZyHHku+MfCrEHFPFLd/4QraStusUYpHlIcQoB1ClHQ5t+0LRlhDgBi1jHlW3TPH0lmiqaVZW0W2/pG8hKVmUqIzgwPG0o4dxFeN00C7suyscpqUqkNNGE6gAkH2WUrprKt6822zVOlS1GVZzJlPx1sNGaqBnDaRbkiXwQzXc4KIaPg9iHD18UtI/eLRduC8i9UCJC0rCVwke/50tI+Pnlqu+5nAJ6u9qNW3Mqjb2/GyEq8r2vq+b0ZFM6X0qSlzJmKlFJUnXfcAe7Syxe1Jft43Nd93MYfvJb6KinWtwtQkJSUqUZMHY93dbO42yV0jJpDqNNFZ4fwWMcxikClUzT9IO5GaMEt1F11CDJ11bQdJ31tKIfTV09aiiZyF3qYQOZCnUk/5lo1Vh+/F8ZcK4iYpCqhpqZxh5fWITkUpoHWSNBBnzHfZ7YfvVdO4whTZcRVOPNhzMkBHVuuqg68woWZQzZIBfqqyvjD5NOijHHtyYncvY1tFdlRUKbxRhuvGqf4tDORxWpHqx5+dn7iG6rzrcHUV3+gqTUUl9M1OQrA7HpTqpkHbKJsDji+7zYTd9bd12NO09Q+walxT2XJkPZ5HvNj1Vji6X1VqauobQFobSnJ9JqFOd2kwqbMfivCcQT8EkWHOcGuaL/wFXHjnfdDh/Gl8OuguIqa51aC2ASRmnfnuN7RLV4lcumgfQ3QKqEV1/rIBXl6sLo2HJOh3j2Wd/HxuqvK+3E3Ktxxlt4qJWAJEDcfO9o8vmjfqLn6gU5W43e1CpQESnNRZTr5tm19h0zqiASOG6GqohDIYxyTexJjCupqeoNC0llwpU4kLhQkpGxjvHdz8NRODmIKvG7940l7KQlygul2uadbSE9Y8lSYkH9GDsPCwV84Xq7wcVkoqqWmwj6N4BM66eqfq++wXR3wpfCMRoW456LT19BUMlQ7SygmCIUImUkT3+61lWlzKJ72GxAQdMA6qa12xKnq+rupLswVhOspMw9IvNmoeUVE/SKTBInQW3iPsqUtLskGY2IHf7rHb8u9Nbwxu9lTjqTdz0ocQRIU0sAcvs/FuYhu2pUpbi77rlgiYK0wfcLebU7zM4l51uVspGcNtmhKNwVjinexOidJEzp8eXx9oGNbydcu91txOdGUEBtICkKE6953+Asi4fu5fpCFLqq4hs5kS+oQZ10Edw+dLHsY1IXRpSpSjkbWAE/pAka5vCTbQUkQJVfPIeHZMVi96tV5hg1rpQgqSkZkiEpBAQdNRMGdLDvXvVPZkFzKYMn2RAPnG/wB9iDgBvkhS0hIQSn2p1B+Pvtw5DcpG6ZVKhpHPbb9vt1ccLSFSCZ2yEcvGphSlVATAklKoJOnt5ePK0VY0pvRq1p9luAunTCZ1KjMnb4+3TW0lGAo9swdEjeToPn9sR/xBbXloSkmAlSSEzl0VB1IA2ju3PjY+maGusEJUOLm6pIpghFM022pcgADWIJ5zGm/fForxeltzFlYZVlUluFJUUxIB3gTAjUToCBE2ltoLaaZSodlaCEkTGw1O0nX4i0T4zCEYoqEjKiUIlYjSJ7/AnfTUEmDY6TZBN3SE+y51wT1OohUGJkJ2EpMiI8pPrRpfrottOM8PrvYWgoyUjjYE/qs0qyI5dpS9DO3stQ9m6a687xZu2ha6yqqXfR2G0albqyUoGpGpKkADf7PSXhthFOEq2tw22nOi7r2q6EGCkqSGqhE+5lPht4Wr5KhrJWxk6lFsiL43OHJPdvKUZiUqlO3Odtd+X3DxeSKKkv3DTt01zWanr6U07wjcKRGp35n3DWzPQ2ARkaSRqkyIjT9g28u4vK5HAq7KcK5JKSJ5Ax+0xa0h1VZICF58XfddZgvEd+YOqmlF26K0sFDumZqewoAaGUxr5+NlleJ7rZayVzTiUBQlcCEyOfIa8vtOsm9KnA923fje6sYvsFFLf7arurHE6FLySVNqMd5JET9toXqLrFCtNLfa1qQAEtPgZUkRPaG0iB7tbTi4SCydV3VFDWsKFE4hQcQQVkdsZjvpJOo5eFl3o6X5U3TxONzuvlabxYfoHpOuZCS6hRnUx1agPPxtFfoNTh50Xjd7yXqZRUVBObKRm/FXvEQLPHhverN3cWcP3wiE09bUtZikwJWOr+BOvny5j1AJsSiYCBdqsVxPSWbzo3nCU9bTqQB35Vf7VmU8VZsyjBAiCdoPKR4DXa0h8UGStq7KggHKp1BOvPKfs+u0flBg5Vr0PLQEx79j8BbG4j6NS5bfCjnpGoo9odEnMFawDrynbU6fXbgInVQVHKNSSY0sYdQAIKpVuQd59+vn+22BsJIT2kkwCD7ufgfk2Ha5GEIuhEFIiNcs8z4R7/j7Yb4s0iXOCWJKZKSv833sVJA1P/CEg/BR+ZtNIR2k9lIJMmeXzr87Rvje6jV4J4kXZopSWzVQrWJaC/rT56WmifaQHxQ1QzNGR4FUCxC2pFa4pQVKjmOYbeHz4WSgDvPsst4kah5CypOqNI5wfxsiQUkE9/O23iN2BYCQWcVfnobXp+cOCzFMYUbuvKppwJ/mu/8AzNvK021l0XWt1ybrpyFwonqUzrJmSJtWroE3gX8MYrufcUtdTVMA6jrW1J/+Va07zSgpJACgUJJga93ui2LxIBlU9p6/it3hsbKmkYXgGw5pqruW7EFZ/NtPlHMsjU6+Hz8LFXLsoAMxoKaD60NDT4WcDzHVqyqSowTOgB0J+vu8/aWcZUhOVZI2Ovd3+X2WDGWyJNHB7g8gkVV1UCSSKKnEafxINufRGWUfQMoRp20oAAOkawO/7LKS2lQUAQEmMoPh92tuHWyoKGUIkgA5ZM/tP7eS+iNkjaaKM3a0A/AJM9HJcAcgyvcaaHmPn3WgPpa3IFYcuS8YUr0avcppSkzDzRny/igPk2sT1aQoK1HMHcHmD4Wi3pN3Qq8eFt5VgB6y76mmq0QdocCSf8VarF0MobUsPihq+HPSvHgqFMEodEbqBT7xHO069EqtCcR35dhVHX0Tb4SZhRbXH1OG0G1QDNc6EiAh0keAnS0qdGOt9C4s09H/AOW01RT9+yc/+pbXVwzUz/gsXh7slUz4q6mAbxTcmO8O3tngUt5061E6nKHUz5aT9dvRhe8KHM280uqcQQWwAoGU986ag93j+NvR65q9N63LQXmmCmspWXxB3zpCp+NszQOvcLUYg21ijckAgEjxto5QZJmDbWg1PPkNtraVqdPGfO1oFVFaJMQmLagCBJ13tijpp7JtwSB6vMd9pGppSBiSnz0z6EjVbagPONLJLbwdbQ6lI7QCvhNnJejeZrMNdLNC7yPQ22809US2PJJI+y1lAbtVbM2zkcSsRqQR4W2VJTy37hYILE76zO9tKcIgyCeWtiFAdEKXDOg1nnsbcJUQRl2EaWDLg/S5d33WxKyTrpyHh52cEwoxmEidPGyhdi5dBMg+VkoOamfKx+7V/ShROswZ7osq5OXN4fE2yxbrydQtXvi2WXRcvDanQeu6xpRC+rAif0ZVMfD2+Vku8q12nvBQZcUCokJA0EDeJPIaD3zY8oIaqCnMlThSkTmJP6WgHgOfxE2Q7061NWS22SVahE6ajc/O87zapabvRLtGJycAnG1YsvJxSsxXSSOzmM5h794Pn52cXH25aO77uu+8mnH+vrKlwLSpeZI7Osd2w0HxsV6O1G43f19OEfQimQkqI5qVMa68vqspdJWpihuBAJTLrylAAzICdRy52Ae932iGjb/wrBjWmhLioSoVLaq2HE9ZopBUYgDw8fw0s9VXkWVpbWVpSQFpAOUK0Gh5kD7POzFpXXHXAylYUCCkyJA5j59nmsNorKgpyrygnUExJ8O/flrr7DclVFro/WPl9QqI7bg1KSUpkbDXzPzrYqrUAt9pEZkyYAgHl7tf22Ebco2aVbTjznXJbyoUWxEk6HUydhYipS0qUpcyEhMAzJiB9+nID2MunhtkYPWIu1wpcWW1Og5SIG2seO0d40sWbYp0tAs1ZWtJSB2DB1BkTsQZ8Iix9mpbRcN4tKSQ4HWciR3AkGLOfglhm5sWYiraa+6Rb1O3R9altLikDMFpGuUzse8g725zsrblOjZncGhE8EUi/wB8bDBRlUE3pTKCkuT/AHxM67g6kwd9fC3pbTIIfyA/pEEDlPj8/Zaq118OcI3HUt3hdFz9U8wtLrai8tRzjURKjsRNpIpsc4oVUGb1WQpclQSka8+UefnbN4tSurnNLDay1mDVDaBrmvF7qEOlO4lni/fCCs5XGrtIREBQhIJnTvn2cufrhc6j+baU8yyief6It4xdIW866u4mJqa2q9IcUulSok6qAQIHn3edvZi5HM910ahAmnbJA8UCxrYzHAxp5BU8sglqJHjmUfVrI01+FgV9keHdNhZ0gCfCwDhkajfTW0sahkRSoUYI5jT599kupUPsiylUTlJ3ERZKqFDN2iRGm5sbHoEE7dJ96KPoT6QD6h+q0PodJSEzyzDXlaUcR16aKgUtSAqVpRBgbkC0TSoH6MRoTt8+Nh6o6hH4fsUdcccQcxMyIkjcHmPnewjLzimgAsnQRB0J5D6/dYFplxSA6dErEAzpp3WXnbhTR3NSXubxu90VKij0Zp0rqGoG60kQBvsTuLCFwG6tW2SahzKiCvNzBzbfhtYO9Q47dtZlMDqHDrJjsWPCmQSomdR3Ax3fbbmtQ25QVbaFZR1KwT7D3fttGXJ5GiLXKowuNUgJUSTrqLOS5kg3tTqR2AlRcIJnQbiffZsXAtsNRmzfRt68jIPussMuuMlL7SiFJmDOoO1oXOUrBcI6w83QY4vp4QhJbaVPcM7mv1WdNViGjp7vZr6d1CmU00rSkTBypH2C0Y3q4pV9OPOkqU5TIzK8QSDPv+uyklR/ck8JhXoyvGIExbM4/wDzQcOqNw+IGQgqSq/Hl20artRU0j77hUpEpUEpyuMqSQZ15C0ccPsc4gdua86FVcrq6a9qppogSoJBB3InmfnSwmIFqULodTAipZKj7FD7vdZr8PVZP3RU6yqU3y8QCd5Sg/bbOtleaZwujXQMZINE+Xa+pqT/AHS+tQmYUr26Dlbundhh0hBAzIOh2Otk1pZUCEBJMkbGAdLGqdQFLUiTHYMEaDtR7d7VMt3DVWENgdFGXEZSRfLq3ARCmygbg9n5+NmXeQebVWF9stn0q6nnNNRJdRKeXd7/ACl0cTSo30SkAJLSdzoIO/7bMi+iDcFU9AcOW7llSVDWKlYG/dmt6ZgxAoWLH4k0mqeUo1TN/LvKoRclW00tzqysOKTBzBYB7iYn5Nkvg2uqo8S0FNWOHrGnKtgAEGE517RykfG2iKtusWsU6lgspKpRplCjB7tifvPIrwvrC5jJlaWsv+6D7Z1kZtZ9n32s6mRr6SRngfwVdFE5tQx56hTlVfSYCr2o7LdRUgjeNVR9R37rNu/4NMlSUpgpnMTvpsdPLu5+dl9DnW4Tv5gKB6usdlJERKT949/lZqXpUpXd7C4S4VNIIk7QBr7vdbzalbaQ/FbOc+iiN1urRUNp6zY7cu42FxcM11GodKQDCVAaEidh7rJtGtPpkpGYqA22BnmDuLKONHcmH5y5lJkjKIgcvLvtqaBvpWKpZichKYbh/wB3KdCEKBKGkxGxyDf8bcOODMmQFSAJmNtee9tsuJOILscC5LzAEKEggBSeX80W24hKFEkwZ5mTvzi2nYLAKn5aoAp11IBVqSSY37+e/wBlmlj1gO3VShAKlGrKSBpIIUde7X6x32dqkKhOwgTqkz8+NkbElEiqo0whOVFZKhv6yY38SU+6xcRs4KCQAhM6vebpa+lp21xlbCiN9SO/vAifPxm0QY8LreI3k5dClCs0GcolI0BkbHwju1mRr9qVuYrUgBK0tHLJBgAAbe2zB4k04N+z+kWEKCU5gqFFeuYCORkA/gdJ6qDbupH6Hl2YavHpD4OVid1pNBRmqvL+6HEtoRUU7DjzGadRDrTZiYMATrFr+Xayz+6m+nm32nmTiBtaVtrBSQ7lnWI/+0kRPmbecfR/pk12LKN89hyjo6hYSoxmGXIN9566ZA5C12sC4r69V4UK8qFU9RRVSSlPqj6BGx2/i9R8nOVtE41TasO2FrfNW9JO0MMRG6k28KfqMq209sKlRUokT4Tsd/ndYw88pd3DtHMlSwokAazO3fqN/Gxe/aVw0KnEFKYAVPt5a+Ph8LBYYePUONlRCkvK0nvA127/AJ0m19QyZgqyvj4ZTe6QeEjjDhdezLDWaru1k19MOZLfaIka6p5c4i1U7sq3LzuxqqCkuO5EoWh0DtEAjyG3LbytetS21NkKKS2qJS4AQQRJBHzobUkxPhRzh7ja+MLk5EUb3W0bhRJXTOdtoH9Y5YB/lJMaWsgq4bJBXedBTuOU9XdymAFlLiurKg33jTUD74sj1NDeFLW0lZccuXe2s1LLyFgllcgxBM5ZEx4bWdZfpK/M31jPpQ1Q2DmkjeCdwZ+dLNS88NKonlVLJcpQogK6kwnwMeI0iLMlZnCljdkNyre39ebeI8C3NfrSk/3SGngQrQ5kGfeRZmhhSUlIbVooeInme7l9fdbrhVfDd9cHm7uUkqfuipSy7KogZ5BEeCj86WOusIJ2OWBKSNRz8402thca/R1PyW+wH06XTqk9xrMnIkaCNSYny7tZnb7bcBkpOsyZM8pGm/j7LKSqQ9uEkydQB5z8nu8LYmmIJbVCcwJEk7z3cvwtXNkVu6O6TiyoGQ2AQNiJ5n59m3e0K+7PS76xVc7rQKbxucEAHU9hTZ8uX4WfnVEyMqc4Op1meXt1s26xo0/EKjJACa263Wjm/kLBgctlT+FpBJY3UMkWi80cUMlotEhalpK0ElMAn5nz99m4kjNqqz+4o3au7L9vW7yEgU15PsFPMBK1JG093M2YBP7Lb+ldmiBXm1U3JKQrW9AO8AnFmKro1ipu9ip74LbuWY/62121s52wQAZBHKN9fhbyg4f8QL44eV9Re1xXneNDVvs9QXaKoDKi2SCUklKtJSk7crPWp6S/ESogO4nxOpEGAb9dSTJ/kJT8LUuI4RLV1BlYdFoMLxuKjphE8ahejNTTLzlZTCp1Cvnv+dbFH2ep+lXlSDuSMoJnTW3mtUccsaVSy49eV5umd3b4rFf/ADRZHreJF814iqpad7WQX3Hno/x3FWHbgEvNyJf2lh5MXpXU3pcdKkKfvmhp9IJdqkJA9502slVeOsA0YKKvGtwsRKSF3kynbT9bv+e7zrq8b3eu42Kajw+yzeiVy9VKQwttSddA2WpHLdR1slnGmIJlt+nb1nsUjKf9W0rez55uUD+0gGzF6IP8W+FTOVTvEK4cwHaAr21R7EmbMbitxS4W39gC/wC4KHGlFU1VbdjzdOyyhayt3IShIhO8xvprvakoxligIyIvqoQAZ7Csuvs87BLxTiReqr+r9dP+EL++xEWBtjeH5tv46IaXtC6RhZl3Ra9EFusWClQmFQoa7WeHBiuRd/FnDNToEuV7TRk/8Z2Dt/Osx3nnqhwuvurcWrdS1Ek+02UrivFd0X7dV7oV2qOoZfGkQULn7LXUrM0RZ4Khhflma/xC9GXabMhSknWIkjQH2fO9rwcFr2TfHCzDtR1pUWqMUh75aJb1/wAW1NzRIeGdKwpJSCFJ/VOoPx8tbWk6NNYp/hwqiXvRXg+2ANgFZXPrWq2Iw+T9MWreYlF+hDlK8gyqdZg20uByOn1W0VgGe48u+3J0HPxNr4GyzxCwkc7cz2tVfjbZidJMTYMkphWlpGlMKDqU9Y0ZEA+2bMRJ6msrqeMuV/MBMCFAH65s/jPLszZi3yg0uIXkbB9hCxp+qVAn4ixtOdbISpbpdclYny2i2FZGo/SOm/z8mxcqhREdqefv3trrCVHsnWLHjQKuJPNGAuTE85tsKmQZHnYuHPWJMgH77Z1hywI158hZAuRlLvaidjzEc7HbuUgLCpBnSLJWeCTpsSZP22O3e4S5M+OtnLk4gWo1Qj/FNssT6481GfP8LZbly8SW0ocdSRkhKUSDzWRrpp5ye+zavnsVAClpMqUQcpjz89TMd1nrcTzQqXEvXMiqJaQG80qyEI0HZ1G8a91mbfAYfvhXV5G1Z4HWHKASZ5jbWJj6rVTPXRTh6CKUF93hdSXhd9Y42VpTmAVAlJnNpvzPsFit9Ykvi/ij86Xm/UBknq0OOqUEg7xy5Dayn+ZxT3a7XuvJdORRhICkZQOXw1/bZOw9RsV1epurQspSk9lJAnw10jl7T5WIs0nNZQ3cBlugLuKEuHrEgqIlJTAjz9nug7WWUqSoQmobSEJUTI185jWyZRJZYvfqbwSC0nrMySdCAFEbb8vhrYo8+4QQtRzDlyHfoPw5b2cRdcEdTXFwlpCiUplMDYb67dwPnYf833pUIVUsUzixoc4SSdBzIHz3mxDIgXep0MEZjCees6xyAkQO/Tzst4Pqn2atLPpI6t9pfY9Y9kEgwfL42ZbonckJRXZUi5L1cqqVdO6gNBC3AdZOo7jtPOz86MSKhXFu6LoLQfZvhwUVS2SqchWJywdDIHL7rINWxe1dRuUUoQpttKG1mEhJT3QJ9+sDTxkzoiXVV3Hx+wnUVi2i2m8GVKCCSQAqSNjvA+/nYLFHmOjkcDawOqMw4f8AFR6X1GnVXXvLgVdl2vhmqu68KFwoCi08SglJOh1AMb2RDwZp67FOHMN3M76G9fNUaQOvFS0oVlkE6eBtbDH7lHjDEZvO786mQyhtJUgjbfT22iXiYmrwNX4UxbSICnbtvZL6WjoHIQolM+IBHPe3jdDjVS7FRTtmL2XIBvoV6jKaWfDcxiayUjpsV56dNLhxefCzjrVYOvGqYrX6Q0LwcZSooWlxlKhE6jcgg7/X604fUPzNQkCf7ma5z+iLeU/T54hL4m8fa3Fa7tTd6Q1RsBietVlQ0BJJAAJ18reqWG1RcN3yFf8ABWd+7qxvb14nNE0rzINc17g7fmlZ1YOg+HdYFRHiDblbhmOU99gVrJJExZzEjxqualXYKR3TqO6yPUqIVAEFPwsovLTBnTUeBsl1RAzEDWxsYQL90hYgp/SqIgECHEKlR00INovYShUZxAI3y+Wvz8bSNies9HpUAtpUFvIQAeQJGvz4Wj2lS604HS3mkaHeR+2wlYQ0i6sMNBc02R2nIypZSCDnAzE66/IseYpnxUopUlBW4pOX6UACeXcB8PYLEmCtaisgI2AHKPbY3ROBpZAeBgnSRz8rAueLbq1a110p3/cd7YavFy7Lw6pxxKULCmXA4gpUkKSQR5/OhslVDb9Uy6lDapLahHjBsdFXUONqYYZUVHaEkx862FF03o8ypbNJVZlIIJSyo7jwFoONHzIU4Y62qRcPXe8hhpKmy2Cy3J8h+P1WddyYfo7xeep6+/GLsQ2yVpcdbKw4qRCQBsfPu2Nke4MN32/TU7q6avP0CBm9HVqYH42Muqu273HEVF50bK2yUrS9WNoUDMEEKMgzPlaMyNdoCnta4N10SJeim27zbGeCWVAEaSAr8RYdqHcNVDajJ6l0TzGhHnZIv3EFx0V8s1Dt/XV1DNO6l1xuuZWEaoIByqJk6kJ0JgwDBNi/75HC9rDlauo4j3IzVhp5TdIp3MtStcoIB5mOXO1PjEL5YBkBOvRF0UrI5DncAnDfzpVdl0PBKx9JSk/4w1s28EBxV6YmpxIIvPONCMsoT7dvu3s6r+KFYauOsuWoTWMVNNSPodCCmUKAIOp5fbZnUl43tdt832buNOpxypQpZWQBGSPstRU1IZIXNP8AGqPmkGcFPNLbyATkUURJncftmxyioXqikrnFLyltlLgSNf74kf61oqxzijiHhF1DdY+wgPoDraWktKOXuIOu0d9m9cnHK/LkcrqzEb662lcpVNBllTCVZ5SQYCp5Wl/k9PJHnZYoT7WiifkdonRj6hccvincWNVNpAkanX9lmfir/cu6rzy5SUUVO4FHSQirQf8AW9x2slX1x6um87ybqarD1Z1CUdXkFS2hZXOm6SI1Pfy9ojuIaPGNwVNdTUTtIzUXdXsBsupdVmbcYVOYAAjWdu7W2poaKSnp2skFrKoqaqKeVxYbrMU4+Vcd/pokhSxeDCaWARELgjflE/Isk8Nz1V/PPOthLhvw5Ep1KUkJKjM6et7ps40t3ShxhBb6zMyVfSLSDPZ1khX1WSMKu0lFiK83HW0MqNelSFrqUpiW0GZKII8dpEa2suA3hOFuSB4zjINdLqX6eqQ3dmImHHEIPpKFJzEJ9ZA2nQ/fZrutuVd0MJp1NqX1CElOdI7UCRvy+02Wqq+8DUzF401Re10JZfeU4kVF4Ik5mwkwRlnnsLI6OJOALruulo1YluGmIQEIWmtJUY7pXB0EzbEx4fKHktaVqHVERaLuHmkekoa5FeCtATPrEyd99uUd02cOJbpdq7kWypaCFJHeD8dt9POzfqOK2B3VhZxlTOtmV5krUtMJ9bVI7hOnIcrGb04w3BUUbV0UV/LV1gShKepUkKOh5DUGY18bXdLTzNcDlKrZpYA0jME1HaUNXXKghbiElCXAiSlPWK0nceXjYJ2mqXjkRR1C9zCEKPMeH2+6xa+MTVN3XctxqtqmEPFzRtahMEE6Aj9bSfHxs3q7GlYimL4TeNUkJJS2grcWuBshOeSSTsLX7Q7YKoLmW1Tr/NN4LRpc9WMoBkU6tPEEeM/DfSxaquyuRTPuP0i8rGVcwJRynXnI5idfGxC9sN8Zbuoae83+DGInKauqvRGSX6MEu5FKhQ68lACUHtKyjVP6ycxOiuLi917z1XwtboKBghVXU1GI7szMtxJUGkvFaz2jCUgkkQJOliY4Zr3IQ7porWBUR1jpXiapSXUntEFOkgZo33+dtrNvHSC7fjalOhBFEzm7olWUwQRvOv1xZWp6kG+L1eK5TnUnLOuXUxOoAgT377aApOLVk34lxDZCBTsIJyjsygkbA6T5eZ0sbLoxCRm7k4Oj9CceN0RqG6cKp6oKU7IiAT2oB1OXu3PnFmcNV1PR4jrWUXiyVu0CSgpLmVWQOqiSBBBSDrHq++rnDKoNLjuneYfCC6y+UkhIn6PeTI2J200B11tNOGbyW3j662nHQsV1O8wAQR2lJUBrPMKEeXKbCGPO25Uwl4TrK3t8YzK2xdKKVOQNpQXCTmIyp1j7Y5e4TCNUhxdQpIAUhSSTEDURPz9UWYpqkPJpagGespmVzmAmUJ089z8mzkwZWIC6psIykpBmDlJn8TtysfBE2P1UFPUPmPpJ9oeMZzIO0gwBvzPd9gtXvpS3Kmnrrgxi1lQHELu2pUE8hLjRJHIZnRr+t7529JSJCF6lWsj2/fFo74+3Z+eeGd5paRLlEpFagp1goOoie4/PIpQDdVLva7WKtYVTOKpaiNFIntnn9W/PUaWTxf2MsMtf7ssN3zd2yloMVDaNpiYXt57xMWcNC3TVBS062OtBKk8iR3ePnvYW87vfVTejiidfSvtJypzECOfMxMafiVHgnJ/cCcS3bUVN4XZRVIXR3xSdc0UmSlxCk6EHYkQAD3d82lwXctQzkkaZpUDO3j5j8bUxwfXXrg/FgvKlcUyuiqMzrEKCVoUdT7idTt7bSrfXT94eYfq6i7HsEYhdq2JDqVLpWk5ucEuKPlIm2Mx+iqaqdroG30W17PV9NSwOZO62qnsXcQMgTBBidx7PYPhys0Md4hewxU0yGKYLLrZWFKAAMEaeJgWg6r/KS3a0Vfm3hdI5dffISZ8kMLnfvsy8W9Pq8cQvNu/vfYdlpKkpLtQ+6YJGklpMbDlapiwXEiblmnxCuJcdw4Cwk+hU2r4o3p13Vs0bR0BAMwN/CCbA02Lr2vnE9z1DFHTGoQh9tsu1KadErj1nHISNBz5mPOttf01r/qUgDAWFyUg5c9M4vKTM7qHybJ7nTOxhnStnBeEUwoHW7Fbd38YLFjBa92haB80C7HaH3yfkmjxycVUY5xK4UoQ47ebzrjaF50pUtcwDzGuhGmgtFKwUmCInWIs58cY/rce4grMRXrRU1O9WlBWzRthplOVIT2U6xOUTruTZsOrK1kpmJ08rbKkidDC1j9wFiayVsszns2uuJJP4WyTbYHgbdIZdX6jaj5CbFXAQoBOyD5RNssq02FsS1xT6Fh68qjNt1VK4ufcLLdLwd4sVyeso+GmKHk96LpfI9+WzTKwbkJ4hkds0pnm2WkVno78bHgCeG98sz/5Qz1P9ciytTdFTjfUkZsMUNOO+ovygaj2Ken4WidV07d3jzCkbRVDtmHyUSWy050vQ74qvLQmrvHClHm5u320uP6LMftsosdDTFaoVU8SMEoRBJ6t+sdVp4Cng+w2iOJUg3kCmGGVbv+WVXuwgJ6tJ7lHX3Wnm9uiPfNE0n83Y9ueveKgFJFJVoQEketm6sknbTL391ku+ejjiGnp6QUV6JrHgkofSijcQG4JykKVGYHYQJEagWQYlSu2euOHVTN2KYLk6Y2CqLDd20NXhm9nqpikZZeWKhhIU4hCQpXrzqQSJH1Wt50BuPVz8X28aXTd91PUKroVQ1EOvhwuB0OpJEARHVpG53FvNhno040edWhddQMIB7BdLmZQ5GEpIHLn32tn0AcNq4B8R76q8WYruv8239dPoysgdBS+h1Km4lGpgrEeNqk0+HxOL4j6St21OISgRyj0V6XECCCY7vK3BEkFU2baOJODHBrfraY3lpwf6tuzxAwhl1vtpMSPVX/ZtICEwgpfMkZY08baUdSSJE66WbyuIeDdUnEFPI30UD8RbQ4gYOXBF/wBLqRyOvwtK0hMsl8ydCR42ZmNAmnve7akiOuQ6xPsSqf8ANNj54i4LUcoxDRzERJnbys0OIHEnAy7vo6hvFd2yzWtpkvBIAJKDqdoJ35WnhdZ4UEzbsQ5dTM5u/fYm2lPDLIEACeevj9tmeviXgKexjG51cz1dWhf1HWwC+K+Bmxl/PoWBKj1NM85pMaZUmbWgcLKrLTdPYuqymCJOmom2B1e/vmbR+7xiwihOdBvR0EZh1d3VEEDzQBHtsTf413MElTGHMSVZGuVqiSCQTyzLH1W7O3quyFSYh2SVZjrpAH32N0Lqs++k7gzaHl8asoStjBF9pSSdX1sNZTPPtnnYJPHGvQ6ltjDVK0qSCKi9m0qJjuSlR+fO3cRvVJlKnr0hX/FzbLQOekJVT27ooUq5j84K0P8ARWyy8RiXIV5xYbvemuyucVWVDbchqEzvlSnWRyA1s3sSJuxyqffRdiFE5nJ65XaUVHUDuMj38pssXTcTF9dcupcqEuN9WB1YColsEz46fE72MuYapC+vrlrcCUw4SBrproNPnxtXNaAbhTuJLbJnUrwVhZ1fVqbydahISnSYOniYPunyshXRU1LDjjlMklcphQBOn1/PlaWUYXu9V2Jo2WVnrVLXlRmTMgCfh7J8LN7DGH6Zq9HkdSttxKCCHASJCtToYO/hpZ+cBN4ZTFaQ/WV+iFF1ZIG4JVOnOe422W6hgLbeTkXJSoHTtDefH8dLSDS3JRHFakmlbUOtKiVHRWgUIGbw233ixCuoqNqtcQ9SMlzrSk5kgkqPcCfH7udkMqURpltvVfVAOqEZZE66z7eZ38Pecw+av850qWnAEpXJI1ERrp9vlZzpbbZSlDNNTwkAkhpAEjWJOv3Qe+zgue93ruUpR9FWlz1iQUqBhUp08APPflqzjeCkEQO5RJi+mHCWjTPkuK1KWCR7I8AOR3tJHBXGN0YL4k4bxfejVc9R01Sah5LNKVOZRoQBoDvMb91kq7K+56x0F11xLitCCspze4zy2NrB8D+IF8XZiG6rhw/RXDSl11CBVOXQy46Uk7dYoFXM89xaoxioeKV7ctwQb620VxhVKHztIdY3FvivSLhZct243wjdeMKKpW1S3rTIqWW3mcriUKEgKE6HwsPxR4CMcQ7mo7tRfKKEUtR6Qpw0/WZgEEQBmHMjnYbhticM3KyqpTX1tUUw46imORZ7wE9keyzyfxfUdSQ1h6vfCkkEJyII03OdQthuz8OBCBskjbSctypMQqMQiq3AO2Ph+1eGXTSuL9zPFmopkPl9KxTqDiUb6KA3M8p9229vVzDbqRcN3AaH0Nj/AOGm3lT08LwQ/wAbqgM0hpUhmlyoWtJUgJSocpjY+3zt6lYddT+YLrWFjWhp9PHq087bmiL5KKJ79yEPW5WVcgG10vF47lQ0PKwC3SRvy27tLAddEqKyCbBKfSkEgmZsbGEE86XW3VKIJJED42TapcyDrpEgiwj1UdQD8fv9liD9UlQVCiDz1jWx0aBeU18bOqRQICSrOX2gIP8ALB+MG0eL4rUFKlTPp+Hm1g9WEmgpioKH84+Zs/MXqL1AjKn1Hm1iPBXw5WqlUcNsbVN7VjzF53Qyy7UuKb62sIOQqOmVKT4e0WCr6SKptxUbh1TJBfIFNZ42sXchWbEVAwEJ9ZqkpURH81BsgVPSeqhdZvugqsY11CS0EVFHcVWlCy6UdUEqFOlJKutbygEzmEaWYjXBfENc3krcT3AyTEkOvEbwf7yJnazjvHh/xJvbA1XgO8eMlz0VJWsNNOVF23M514KF5syVF1IBJzkgACXFQAAjKBFhmHg+krGTEKy3ohC1HSNx3UuFFPw94sPGJzOULlOnKZEguLRpoe7ayc7xs43XotTd0cI76cIAbAr8VUdOpSlaaoU+SRBB0nexH946/wBxSnKrpMYmQtZK1GhuanpjqnKTKFAiRv77G2Oj5h+sR/u5x44nV5SMsm91NpA5pCc22gMeFixRYZH7KHNZiDudvJLOGH+kViLCv7oLruPBdO0w8u7lJqcQu1pLzagkyphst6r0PbkZVAicgXXl2872vi8K68quq6ld5PmqcZYfaWhC1SVBKoUCBMSDqOZmbWPu3gVwyuW5Rc9LjvG71Ip12oUyLxaCVuuQpxa/ojnUopSSTJMAm2UvAzgld7QbapL3UgDKlLt4BIjmE5EJH7LPbLRxfzTCo3sqpv51ygejeeRQXm2l9xS81M5nJQZV2xIyp9+ns74+vllx29X6hx9LbxT1awXCDyPJOh18du6bXd4f4C6ONNiuvp8ZXe61c4u5Dqesq6twqfS6IB6tROxMcvOz8fuDoA0z66l3CDVY6O0oqavBydO5ao25R3+Ni4mNkbnuB8UDMXsdlIvZHOitgnD/ABS6Ptx11/3umnvi7GzQ1AYfLzTaUwpnfchtbcmYJBiQQbSTUdGvh66+p2nxZeVM89BdW062SsxEwpJA8rMTCfGjomcLKB27eHuDKu6aSpc6x1ihu5LaHFgesczgk67+JsdrOmFwgZTno8F3s/pIK22Ezr/OJtn58BvMXRvAB+Kso8TmyAHN9Fxf3QiwRiivFVfHFrESkIBS0htFIkpB1jMWzOsnayJU/k8+FnVpSzxGxQ6ZUCo1NGkgGYgCmMxPPuHtMv8ATcwa2qLu4d1KgEyCusQn6kGye904QsEUGAGG+QLtapQ+CRYhmFvY3LxrfAId073uzZbnxslC+PyefCr83NKocWX6KtLYKnFP05BUI/RDQOoJG9kDG3RburB1/YT4fYHqa+qo79oL4Ydr6xPWop31tNlOfq0gJEN7aeqbbZ6auI6usRSruC5qJtaoLi0vOhOmg9dOvnG9ka8elZjS+L8fLDd1Idw8H66mcbpjCyKdxKpClEHRe072l7iGnMZnH5LmSyg+qFql/J34kYpmnkcUbpC8mqV0LwyyNR65121smJ6C18XZTXhdruH8N33WvU7zTd5OZp65eYh2VpCgUk7D9YiYSkWGvjpg8SF4XprzuvGNGLxcfLLlKm7GAR/KgpJiNZmNY0MSzqnpV8d6tyRjV1pJ1huip28x9iPAj51NZCGbvJ+QUF3u3A+qMI6EPF80zFGjAWAEmnJUh12paKFFQ7cpNMo6nUSTGokzZaY6F/HBw05/N3C2kDKEt5cr/ZSkJiOrZTGoJIECzIqeP/HSs6wrx1iASIhtwoAPhlA7jZKe4p8YrxJFRjHErgKZymvePhOh20+Np28IciVHkkJ3AU5XL0POMlJci7or8bYPSF1S6pTjVHUOFOZQUEpnLIEAdrMSAkH1UwBQ9BB671M1F68UqVBpykgJu2ASPN3mPLe1fK/F+N31f7oYkvBQA1668FwdOeYx42It41oqZ6b5xVQNZEyEvXo0DmjScypjQeOtue9lvU+qcyI83K3CeidgarU3RXlxIC0ICgG2aenJAIE6KK42HL7rae6IvCSmpyKziDWtrElFQxT07LqCdlBUGFDkY5C1dMN4lbTTIvK78V3C626jOgU19Urjzic0CGkuFyJ11T99lml4juVbZQKhS+rJHNMwddSQCPLfflbP1lHVzTZ4X5G9FbU74WsyyG5+SnPGfB7h1ii7Ka5sRcdcd1FLQVrV40yaKvo6dxupQFZVhTVNnSRmJkKnbuFo+vTgRwHp2H6g4p4j3vUsIW4x6diuqca60SpMp7IV2tSCCDPjZpnHt3sNnrjVJcAGvViNJ9aFGPjYgriLQh3qutaWTplW8WyN+9BjTUjlFlipKwEF0zin8SlaLBgVS6BCWL1rmi+tTqXDn0kqVOwA29tknE1UipxA7ROqhtymbUFbkEJnTu3PtG0QbKd+lq5sbV9OWZDrqozKnMTIAj6vDlJktnHjdc/ejNRRuHrfQUPnXKEkKWInv021jflpo5W3bZUsZAcpU6Pb930PEiiN5XfTVb6GaoNIqmEuoVmZWpRyKBBiNPMbGLWLve+UprKR9i6brp26epQoopqdtkk6giQnaNPDu0i1NuGWK6peJrtqqVLLdc1UBpQfzKbkhQMpSJI1+B0Exaf6+/8AE9Y2KdmuuVJH0mX0RxAV+jBVBI1IO36GlhBESbogygCym648UhN30zDyurNPTpQDnUUqygDSE9w5/Cy1dePaGnqyh5bjiUtnMZGhkGJkEnbYfC1c6bGeMqJCBV3IirA7BNG8lWURI9bJpMTt5WULqx1VsuPfni4r7bRkBQlNAt0kgg7IzQqD3x52dxJWbFNbHC/cKyD/ABSuJGZKEvEgSkZCI07u7x+qbJV/cSrvvC7q27xdzzqHmltn6TZEactRpFoRONal5KvRsE4pezjs5aFCJnQAFa0x4jloNOQLuLcR5JY4a4lUA4RmcNGkK8f4/nI5ad1u7xKU8U8I2TPrHKuiUUUDSutZ+ibCU5usImAQOcT9nMWHouIVdSU/5vvu7F0r7gKEPAKUiSJBmSY2jXTxto1lW3eKk19zuXa+9Ck0ry0qVB0IBQSD3d+nKyqw0auiFNU3e3lIypSv1iDoZHhpMcvbayYS5uiAeMrrKN2F1NLid5yvqzVO1DQcLidBHKI3EjzHPcWZmNODN+45xCh7DLlC2vqpqOvX1KU7QZgzv+AtI+OPzDcTlHS0PWF9ALiZXmyIB31HZnTw0EbwD+Bqp8XywhtxpPpIBIVISIkZTG258+/uBqJHRC7N0bAxkpDX7KHKbolY+djrL9uBtRMZfSXFEDXUwjQaWU6XoeYlcITWYtu1slWUhppbkRvvGotb1rCtUtHWrrWUhQzSElQjSDyOumuvOfEz+5anK4U8o6zKWlac9JPf5fpG1OcTqid/orgYZS72VTqfocshKVVeO3SNSequ4beEufMHusosdELDAbAcxFfLqyTJSw0jbkBJtaZGGmEJzqedJCgogJTJJiBz+QO6LcJuRCAQpDnbkxMFOwA2jX53sw19UfaTvs+lGzVVpvonYZaWVGsvd5IPqqLYkf4P32UKTo/Yap+qDuGadwMagrcWcxgesf0vLb32s03ctKFCaVQUnQkuc9/t+dbGm7loElQLQ9bQlJVMHXz8Pvt3e6h27ilFFTt2AUJXbgXDLBTODLgToAMtz06lCOUloz9Zs+rooblpwGkNmlHJLDSGgPIIQCdvj5w+PzPStrA6sAZZgJgqP3/Oth2brYBObqwDzgAx5fO8WFlMkm5KJiYxmyb7DVyHQ1l7Oa6RWLjTloO76j7TS7pw48AX6SvdO8qUpemmkk693L22WhRUxIW29l5SOUaR3fOtusjCAVdag9qDmMADx+edhHQOKNbM0BIacO4UTDf5kkKEkdQlMDURqBYw3ctzNx1N0NtpMAnIBp5ef1crKHXULaCRWMpkyoJygq848O6wa6q6wCtV4ITzOVOogH7p99kFI52pXd5YEXVR0jKQy1RpSnVKYbCojTvEj58LBOUza4LTB/V9UR7/AGCxhV63cmAKxS0k6ABXZI00nT9vtJd2/aFHZyPEoJmSAY5GTqDraZtE7oonVreqLP3UHQtSGlFSdhpqO/cj7LFHbhdUU5EgHeMxHcfdpt32HViVFOrrG6V1YUObpA222IHmPfYNWL3wjK1TtagQAomDPhE91phQu5BQGtZ1RReE3nPVebbBkBJKtzP4eNtNYBeD6KhNZSHq1AhISoyR4z32EcxZXxmBDZWQEwiZkfHc/bYKoxLeLiZD6xqQJRodvDx+u07aF26gfWNsn5+7LiShWYXndqkp00ZUQkabGY5b+PttpXEfiShCQm87vKwZzdQNTA0nl5WjpV/3kTlXeDpSTmhKoHw7/ssA5eLykz6X1kkQkHcb7z8/CxzIHDcoIzMOwUkDipxJaUkE3OEEz9IyTpM8+72b2EVxvxeyAqqYuqDmMhYTznSU7AH5i0Xu1edSgEgkToTO3mZ2A115WJuqKVaNSUjYGI7vmbTth6qB0o5BSpWcfsRBoJYo6AOEAKVIUJ027O2vjZrX3xjxXel21VEWqTrXEgpz06VshYUNCnIO487MtZeAUgNBIET2o9+nztYIQnMgAKGYaAyN9PhadkQah3yEoyriJj/I0GG7oplCNG6BAAG/PXu/GxR7H3EhSxlvlhhajrkom4g+YsCoA6KUo5pEZjIJkfZYFRSk6uKWknsxp7Pw8PeQNAhje6MHFWOnB9JiysGYf3ptCY75gafjrtYrUXliOoKeuxZf+VEKyNVZbKhEx2fP6rcAugpSFAJkzpt4z5/XbkJdQ4uM0nYpT62ndznXzmyg3SINTtbmLrl7Xy8oHKS7eDqwdOYKiNvrOttUlBRLcU65QFUnNJk8geZ7x+y2lrcKIQVZgY13BmDEW7pXX0OBwrUYI3IObl9ndz9tlTeaV1UtMklPodNoY9a2WD68jTr6YfziqfbbLcnKAsJ0lUn0ltF01D5W4Cnq2lLkBCZVpMiZ0+BsaqbmxE86VM3DX5TCtKR0ydt8uvOyE7ft6BTdPQX5UIaIgU9PUKLYCQNglXly8dLBtUmI72WptqlvWp0I7NO85MAjTLIJ2BE+NguIAbkhTlulrFONq58TMEuru6tSBpC2S2NSf1vZGm3fNkeow/edO6qtp2qRhwuKWQutaSVAmCIJBBj4iJ7lCg4WcQb0fSLt4e4krFBKlfR3W8sZo78u420mY1nazkpejvxuqhmpeEuKYay5s91uNz5ZgJ7vvsw1MQ3ePNKIXuHotKY9VcL7r/pbd80DT5CSsKrmSDBBMZVEjYaa+Nuq25qWuaT6RiG6kPQUrcQtaiobx2EEHbadJ98i0XRR6Rl5pCKfhFfqEInKXkttBOuwzrSY/b3Q57k6FfSVqqQUrnD12jQ72nOvvWmSlcREpC528eR2nWN1bTN3kHmE5tNM42yFQVU4Xu1hvqnsWUazGdfVtvGQQCNOrgQJ05GPME3aW5acZRfVQ5l7Ki3THcKmdYjn87WcX+T06Qt5qbWtdyspTlJTU3mQf5QSpCFxJJ8tOYss0n5M7ixUECuxphqkbUDIbW8sg+A6uDvvpqBpZn2lRNGsgUgoqk7MVSG6igZbQpNdXEJAygspPPf+MBmYnT42cNzcWr3wretPeN01Vcmpp1hbLnUtZQpJMKhWbN7RA7rWxpvyY19NhKrx4nXehajMM0a1DN/hESJJ7jZMxF+TpuXDFCqvvvjIo9uEMU9zAKXrolOZ5QO3NJFoX4ph8oyOdcHlZTxUVcw5mCx+KhwdPDj7TMlmjxherbCRKuq9GRlmANBTiNT9/dZv4g6Y/HS/0Fm8Ma32+hClZAa8tSdoHUpR49+kbb2ma4OihwkulOa+2L6v9RjOmpqupaUoyQQlpIPMaEkDxss3L0ZOC90ldY3gxx9Di5DdXUuOxG4R3DWDy287CNOEQm8cI8gie74i/Vz9/FU7Y/dlxxxndGHqCkTXX5fFYzRUinahwl1a1FKEqW6owkZjrO0xyFvaS5MN4rpbnoKFdxPFympWWXFF5sAqQgAxKhzFqt8MuFfCzCHEC4MYpwjdNzN3VeLFYutUtbKKdKFA9YVFQSEgA+sItbd/pJcCGD2uLOGSQQYbvBDh9ySTa8onQVzLj0QFVVkc1I+ztSUGMO4nKoN1EK/6Vv7FWxWEcVKRPolPtsX9fgCLFX+lHwKkBnH1PU5hKfRqOpd9oyNnusAvpPcKnAPRKy+6sHUdTcNbHLWVNAd1jOBTN3d9UEZJ3aBqPKwFip4RNC2N9X1H/Utg4WX66Ppb2pG83c2pevvHhZJc6TOEVD+5MK4vf3Ay3WG5/pHE2BX0m7tyhVNw3xYrNOVTvoaAqD4v2cH0rPaHmkENQ/2SlO8OCVXeLIZqsTNiSFdijO41G7neAbVmxxga98GY2vPD9L6ReKaZaFpqSlLYWVoCyN4AGaOZtPT/AEl7zWlJouF1Yoqkf3TfDDX9QLtVDjdw+xtxX4h3rxJau6kuhm9vR2nqFu8FvKQplpKEqKwhAVmCCdhEHnYaplpXgAO1RNLDUxG5aU6kXffpCR/cbS9gldeyg7g81efzscaw7etSgLqL9uZmdCld5IVB5+rPyLRMxwgxzTISlV/PDIMqMtYvQfdt7veotcF8TVNE5Vv3yl1ynA6xv0havo5Pb3O2gIjSQe82DtT+8jCKgn1VI5wuttwCpxVc6Ecsjrq9IP6qPtsJ+ZbhaJcqOId3tSN/R3wYHmkcvna0Xjg088CXq1CFREhJKh7wfH3WR794QXuh5Tl11NfWVIRK2m1lKXBGpQNgrSI5z73h9PtdN4c+9lNTbHDhpKkP8UGlrUIytUwInxCnBYF2+eE9IlSX8ZXm8pB1LVMwQY5R1sz3fsFqzrw6lCwC1UZUqCFguK6xM6QQROhMd86RYw1hdCnVKepMhntZ1SUj9EHvE90RqfGzgIuibaRS5fuO+GjNSHrtxDeCELb6twv0hJVtlRkbSpU7nNtoddRaPbz4sXIla00lHfNUlByhYbpm0uAcwHHkmD4666xBsVp7ju5CpNCgH9MKOWT4EzOhOhHLnpHa7hu5YSaZhvIYzJMFaTPKRoeUHTfURZwczYBMMTyk48Sqd4ktYcvRTiQVQ7VUiBA8UOrjfusXd4mVjiZRhejEQQKi91JymNJyMK+Z2svMXPRpZUttinCgSEqzZVpGkhQ+7TU72FapKJsqbqKehKQe0UNhJA5yBvsNvdZc7RyXcJ3VNRXEHFKzmpMO3EgkyB+cKl72f8HSOYP224Rj3HVSQaVi4EZYV2LvqXlDwhTqDy7rPRNBd5AW31SkIEk5JT4aTpoD7fYAKqkaCkJdU07nURnTIAEawYmfd7rLxG9F3Cd1TJexTxEqkZ03nRsKIgKYuLQpjeVVC/HccrB0t6cUGSaljEdWhTrakL6uhokQlUBSZU2vcE8+Wx5PoobbH0NWVN5so7RI0Vvm5eEzvt3YtN3kOIDLqEoSJUEqUknfcanSN+6d7dnB5JRFbmmMEY6qUrbViS998q8q6NoAn+UinEew2AVcmKdJvW/3AVGVKvd1PdsG1JJ8ItISksrQG0OpcmCTBSkHeZAiB9YNiil0tKqC6WCkdnXLpGp7uZPvt2c8gmmJo5pkP4evd8lLlfX9WYIDt7VygFE+L3s9njYsnh82YdqqejeQkkxUFx/MYOo6xao56c9LSAq8LvLIdfdYWMuilJKTGmxkzInz99uEX5cXWCZQnYFxPZiNSSJHL267aWXO7ku4bBzTGRw1p1EH0G5BK9k3dT5iNdNUmDPjpPlYdrBaaAksIp2ERKizRtInXllSIj2naN7PE3vQq7Tb5dBUDBToNDrJAkydPVHxkmu+Wm1JUgVaFH6Ptagxz3gQAddN++yhzyF2ViQm8NvUwCVXxXyqJbbqD2p5CCDOpER99g14USSn0i8LxUF/SfSVKviZ7x3cxtAFlZd+1EdclIfPqySUrOugEyCdtdhHfrYH88Ela101Qytoa5V5ikjnzgQDzPdpzUZimEsCSncF0SnEreS4pYO+dR7R8NYI0Mc9e6LabwrQthaE0gSe0SkpJBSCCTuZ15gEDWykq83nELUhDbzZSWwnX1jIBkc9Ph4WLqq6xpstsllaspUlKmzodAdzsDHcNt9bODXLszRsmBxMYNNeTFYw0gZ0JGYQkQOXvkwDp5bJF9V6iLoq0OKyLYUypU6aLE76z2/gZ8XbjBp2+GaQIT1im1qQ4pCCmDlBnbcgxHyU/wDe2r7+u/LRVCGXGH0qAI01EEeWg7hpsDaYtzNQ+fK5NPA/V3HxPuitcpkv0/pCHylwlsrSnN3+HeOY77Wro+IuEUDPV3U5SlYBSfR0OIMzsEqKtgdxrOkG1VsONOu35d1U0tTVRQKdSspVlVBbV3GTqBAHInyD1cqrxceWmmU6vISvL15KwkQASCJ30gaTqJBJA5i4inbNw9lZ+68c4Oq0JDOIaCnaMjK8g00ERJl1I8dRpOvjZy0oobyAdoK+nqElMlLfVugGCJlO433GvidqYvP1KmC5VupbQ8BkDyAgEJ0HZKYIA94POBl03TLVORIU46VdW4zAUDCdRoCY7J07uekROoATcOUza+27VdZN1A5JS2VA5VQlSZ1/kjl5jSdNLdKu9AOZypUkKVJCHACAVcp8lD2ad1qgUd4cQWkKcuy+r4HVpGdCa76NGhkZCrfbkRAERNlmn4g8TbrUmnerKFxCnPWqmsoVoR/e0j9bcmSSddUizDREc1IK5p5KZeI2HmKpVPVUr6RUomMykqUCYGngAkfbys0GvSEZUqWpT57KkNokNxvqNNxvHM77WQqPilW3mpDNddlK7mXPXs17SSkFQlRazExzEGeXKbORiobqVlbOUJegSgannAjTmPxtY07cjMpVfO4OfmTGve5alwOXjVtdYpx0rzFGZWVJgakcttN9++3N3VK6F+mqkM5lU6kkkgxmTqRtpz77Ouoo7xvNl76RBowojrFubgTMSNRvZmPph8IJLiUqyAk69olKiO4mBPvtBIy2hRMbuYVmsNYquqruhkKp3FLZRBjINDoFTM7AHnoDvZTViWlObLTOKBOoUUwPOBppGw9/OEuHN9KFMaEujImQlJPamezO/L6ve+fTTBKmkDmFFHZE6GT+zwnewJpGbo4VjraJ1uYoQCU/m5sAjm4JgA6kwNJP3baFjf8AUOnOmnYTGuhJkkbjwknlys3UpdVnIS7AOqkiD7O/lqND4cukKXklp+QCdMugnke8+McvOyClYF3eXlLbl/1pUolpCCISDoY131O5ixdV8XkVQuqKACTAa0ie/v0HuNk1C30plxDagojY6RpqOXzz3ttRqSCktpOvZUlckeXcfxs8U7eiaahyMrvOsXmSp9RCiRA0UPCbArraleVYcqzllOhOn1WC6tYVlOZCSP0kiB3n67dlIH6KlJ2griZBmy8FvRKJndUF6Q+pJU6p9SojtqO/sNgCtbyetkFJAEFU6axzscLLKzk6ojWAUrIA7zqdN7crpiSZSZ0IJ7Q8pHP51soiA5JvFJScsqVAVUws/qqkExO20n53ttLjzaR1ZcIP6yZA28oI8uVjS2VJUPpCZ2Kjorz7/r0sAuncICkp039YTppvGo+7zs8RhMMhQAW4F5g+sZRM5Y9vdHw+zjrnnJIWNdQZjwgjmI+JsZTDUrFMCr1ZP16E/PPa3LimXQApteaRplAPujXTxt3D6JOIUXccKkklBzTE6Ez3/PfblVW7JSpGaAU8oynwG3z52NpbpJBzpSVCQI8ySdbcpaYSQpsz2dRl08NuW/n4WUMTeIgBeb6Up6+nATvO8x4/P1W5XWBIhQVmJGkj368vhvYwpTYWEHq5BOp7uQifOwSinPkQEFUnMrLIJ8YGnss4MTTIiq6qDmAVlSBrGsToY+fKbb9MQWypfKNCQCTA13+edhcqQoBsIAAMZUwABtofZ87cOIUClSHNZ1kz3a/h95s4MATS+2qDU+y4kpyo07JATOnzP4cuV+jLSYiCJkNzv7NPnW3LyXFytK0lBOYHNA7++PnysA7SPhRWkoSsHQg8yDr3zNn5UziIQpZUFJSQDOkKIM7fcNPDnYJaGXUqCZ19UlRMT4T8PfbOpq2ytaqhJgEhSJHs7uR+PnYMpfy5VOI5zrpz1GnzNlXE3Wg00mCVpMCNB7dt/wBvvDKCgGFk5QBlOm86d9hiKo5kFDQUgRJ2mI5b8rArCkJ+lYTEwpOUyffry+FkFgkQK3HMxh0KI8QDtEzG+/hYInIerz6nTbfnMz38p98WMLqGkEIepgEjQHYecAR7osEl5hWWUurlEApSYiO/bu0/bZcwCaWlFnHCAskLUMsEBkgnx3jblYVlfaIKUwNSYM94Ouu9jKWkqADAdCIBJWg9qec+FjlPda1pytIyJ2AUIgk+U/MWbnACXhEouC2oZnGG8x1VpztllhN1Jyj6OnOm+cffbLJxAl4RXpgzgnDFO2lumuK7mUJMpSikQAD5AWPNXPRtJCUtNgDYBsAWy2W8rA5rQZ3LSqNsqgEpHhbS7ta2UtRBFstlpWsB3Uoe4bFAKp2Gj6hV5m2nG2kgKKJ3tlss7KApmuJ3QDyEt6hKSBB1FtFw9XmCEbxtbLZZg0Knbra6Sb4vBygoHq5DaVFtJITt8bQNiH0zEtb6bete4syUpQhISlPkPZbLZaamF7kokbJGbw9SLfQhTjhzkAnTnOtg3boo1KzrCyEgBKQr1R8/GbZbLGhMK5Tc9DVhVM+2XUKISoOnOFbbjnv8LCO3LdFBUu01FdlK0GFloEMpk5Y123kzbLZYiJxA0Khe0HcLplo5T1RS2AkGAnTX9thqhrO6hmY6ttIkDeZJ+u2Wy0wcTzUeUDkglICVBKiVSqN42BV/q2FS029QNqKT2XlpTJkjQSfPS2Wy0rUyyBTSohWo0SFDsj9aPs+NjlFRpdU6wpZy9StWg17KSof1Y9ptlssvNNRV27KZKisgkCZHfqfZy7rao6dlisS9ToCVtSdYUFabEcxEj22y2WeEnNKN54ao6Z9tbCsjb7aXkoCfUCp7M8xI/bzIHDNG+XWlLKSqIWhICkmYzA94tlsspTVGfE3BV1G4l42bKkVbDxp6pEAipCHurCp/RXporXTQ5tCImcbp00yCadJLqdY00k7d2omR32y2WsqXVguq2p0foiLt4U7VOzFHmTUOpRClJOUkFQUZTr6o9uvhYkjED7NI2pLc56jqZURmGm8xrsNPqtlssaAEGXFGW78qX3mWlNt5ylHbKZ1UAdvmbEay/Lwp+oUFpyvgaJSEkBRPPYxlHK2Wy0gaExzigXb/ALzNQ2U1B6xK4CylJ02jbbXn3WFZrbwFWaKpr3XkrbDo1ygSYiB7fh3TbLZaQNFlCXG66e62ndbedqHHvpkNAExAUkayNefwHjIdUp9DzRNU6S7IR2j2EwDB7+fdvbLZZwaE25RVAL1K04mEdYgGBMA66767c/wt01VrQ2zSVEuKdcLedBDcRGpAGvlNstllsF1yjCGGlOlopykoIzNwn1dB9luWqNSdA+qM2gIB2JHPXl32y2WamuJQirtb65KUOrbyiZQADoSYn2ned7FwqO2lbhClgDMqSJVqJAAjwi2Wyy2SXK4S5TIcDTlOVGoIbzBQEAd+hnfw+Nid4XimieWpDS1uFUFS3AZhPZ0jWI562y2WcEiIN3yitUhQoUoXA7RXMb7aCPWO1nNd91pvJoLDiWQDEJaSe8TO5Om5JO1stllKajN7YRVddF+cH69upSusQgIDBQQVTrOYg7jSOXlAjN2CnJQ0+oFpxGUwJ3I+z42y2Wkbsmu3UOfmdqjxIwpDplFUlKyEiVy5BMGRyO4I185sXRcNcLVlR6Ill5ChMkqTljLmgBKUn3kxPstlssIw7okgWWqrAWHLrqksIYqVFO6k1r6ArXmkLg/t77ds4Ww64ytxy6mnQCOy+S9Mn+WTz7otlstJcpoAXKLhwk0pCThagClOJRKGWwNdATKDO/f37SbKj+HqG6aZNUxTUgzZZCKZKeceI2PdNstlmjdPIC5ZWt5tKioygSJMiTptsdhv3aRZpvqWq81ZVBPWqCdEjQEgjYcj3R7LZbLSxbqOXqmnjTiJeVx3mMK0FIh1SkOPGoq3VuDRShAQkpj1eR2jutHy+Irrr1WxeVztOuUhKCundLSXAAD6qguN437++2Wy0cvrKaH1E4OGnEs3lfv5qYuY04DbausNTmJJmJAQJiD46jURaxpS04ApaVRlSrsqgyd+UfC2Wy0bt05my6FGlaA+lxaYbC45klAO/kQPZbWVxGqXlSOyCQDz8bZbLIlXSW1HrBn0B7I1EaeBHyLbapg+o5XXEwspMkKBgnv8rZbLckubrblC4zlX6TIT3Jg9+82IredS4M6y4QsIObUEzubZbLOAS3XK6lxK0IVlVoANIEeXfYSlfLq2qcSFOAdswYJA5RbLZblwRu86J2mioNSFHMAAEERKlDvPJPxsmocWokhcQUpjKCD8J+NstllXLlzOtxSSodkTMe+21NpIJMklRiYjQD8ffbLZblwWOXcpKHHC+CABCcmkTtvYqWpcUkFI1GydzNstlkKRBIY65K15soT2YiSSDEybbNMMqClUAzoROxP3fG2Wy3X0XINSHGioJdICdI9sWGp23XEBZfmYyhSQQBvHcbZbLNS2COIuxSv415KxG3Vx3HkfGwn5mp4yuOLMaHIYmRPOe6LZbLI4lLYLpi4aOoQtaXH05SNFOZpn3WOHDdJSuBAWVZE6GPHnJM2y2Wjc4qVrQtJudhTgADQUteSS0DBjewT9MhlGdpKE8kiDoQTrvbLZaMON04NCLJRKUPJUU5yrTTQDbXv8bDU9H6S71S1pAkpHYB2n7rZbLLcpWgLpmgpni42ppA0zE5ZmSB93usM1dtIkhSkqgn9EgHUA6yD32y2WjBKdYXQi6CkzGWZM7k72y2Wyzktl/9k=" } ] } ] } ] -} \ No newline at end of file +} diff --git a/samples/js/node/notebooks/pose-estimation.nnb b/samples/js/node/notebooks/pose-estimation.nnb index 6ca358ed2b0cf4..16774b4514ab11 100644 --- a/samples/js/node/notebooks/pose-estimation.nnb +++ b/samples/js/node/notebooks/pose-estimation.nnb @@ -17,32 +17,17 @@ { "language": "javascript", "source": [ - "const path = require('node:path');\nconst { cv } = require('opencv-wasm');\nconst { display } = require('node-kernel');\nconst tf = require('@tensorflow/tfjs-node');\nconst {\n getImageData,\n displayArrayAsImage,\n arrayToImageData,\n getImageBuffer,\n transform,\n downloadFile,\n} = require('../helpers.js');\n\nconst { addon: ov } = require('openvino-node');\n" - ], - "outputs": [] - }, - { - "language": "markdown", - "source": [ - "## Download the Model" - ], - "outputs": [] - }, - { - "language": "javascript", - "source": [ - "const baseArtifactsDir = '../../assets/models';\n\nconst modelName = 'human-pose-estimation-0001';\nconst modelXMLName = `${modelName}.xml`;\nconst modelBINName = `${modelName}.bin`;\n\nconst modelXMLPath = baseArtifactsDir + '/' + modelXMLName;\n\nconst baseURL = `https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/3/${modelName}/FP16-INT8/`;\n\nawait downloadFile(baseURL + modelXMLName, modelXMLName, baseArtifactsDir);\nawait downloadFile(baseURL + modelBINName, modelBINName, baseArtifactsDir);\n\nconst imgUrl = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/intel_rnb.jpg';\nconst imageFilename = path.parse(imgUrl).base;\n\nawait downloadFile(imgUrl, imageFilename, '../../assets/images');\n\nconst imagePath = `../../assets/images/${imageFilename}`\n" + "const path = require('node:path');\nconst { display } = require('node-kernel');\nconst tf = require('@tensorflow/tfjs-node');\nconst { addon: ov } = require('openvino-node');\n\nconst { transform } = require('../helpers.js');\nconst Image = require('../image');\n" ], "outputs": [ { "items": [ { - "mime": "application/vnd.code.notebook.stdout", + "mime": "application/vnd.code.notebook.stderr", "value": [ - "Proxy agent configured using: 'http://proxy-dmz.intel.com:911'", - "Proxy agent configured using: 'http://proxy-dmz.intel.com:911'", - "Proxy agent configured using: 'http://proxy-dmz.intel.com:911'", - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/images/intel_rnb.jpg'", + "2024-11-04 17:09:35.860051: I tensorflow/core/util/util.cc:169] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.", + "2024-11-04 17:09:35.872537: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX_VNNI FMA", + "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.", "" ] } @@ -60,7 +45,7 @@ { "language": "typescript", "source": [ - "const core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'CPU', { PERFORMANCE_HINT: 'LATENCY' });\n\nconst inputLayer = compiledModel.inputs[0];\nconst outputLayers = compiledModel.outputs;\n\nconst [height, width] = inputLayer.shape.slice(2);\n\nconst heatmapsOutputKey = 'Mconv7_stage2_L2';\n\nconst THRESHOLD = 0.3;\nconst COLOR = [0, 255, 0, 255];" + "const modelXMLPath = '../../assets/models/human-pose-estimation-0001.xml';\n\nconst core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'AUTO', { PERFORMANCE_HINT: 'LATENCY' });\n\nconst inputLayer = compiledModel.inputs[0];\nconst outputLayers = compiledModel.outputs;\n\nconst [height, width] = inputLayer.shape.slice(2);\n\nconst heatmapsOutputKey = 'Mconv7_stage2_L2';\n\nconst THRESHOLD = 0.3;\nconst COLOR = 'rgb(0,255,0)';\n" ], "outputs": [] }, @@ -74,9 +59,18 @@ { "language": "typescript", "source": [ - "const imgData = await getImageData(imagePath);\n\nconst originalImage = cv.matFromImageData(imgData);\nconst { cols: originalWidth, rows: originalHeight } = originalImage;\n\nconst image = new cv.Mat();\ncv.cvtColor(originalImage, image, cv.COLOR_RGBA2RGB);\ncv.cvtColor(image, image, cv.COLOR_BGR2RGB);\ncv.resize(image, image, new cv.Size(width, height), cv.INTER_AREA);\n\n// NHWC to NCHW\nconst inputImage = transform(image.data, { width, height }, [0, 1, 2]);" + "const imagePath = '../../assets/images/intel_rnb.jpg';\nconst img = await Image.load(imagePath);\nimg.display(display);\n\n// Resize the image to meet network input size\nconst [inputHeight, inputWidth] = inputLayer.shape.slice(2);\nconst resizedImg = img.resize(inputWidth, inputHeight);\n\n// Prepare input tensor\nconst inputImageTransformedData = transform(\n resizedImg.rgb,\n { width: inputWidth, height: inputHeight },\n [0, 1, 2],\n);\nconst tensorData = new Float32Array(inputImageTransformedData);\nconst tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, tensorData);\n" ], - "outputs": [] + "outputs": [ + { + "items": [ + { + "mime": "image/jpeg", + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAIFArIDASIAAhEBAxEB/8QAHgAAAAYDAQEAAAAAAAAAAAAAAwQFBgcIAAECCQr/xABgEAABAwEGAwQHBQUEBwMHBg8BAgMRBAAFBhIhMQdBURMiYXEIFDKBkaHwI7HB0eEJFUJS8RYzYnIXJEOCkqLCJVOyGDRUY5PD0iZEc4OEo7PT1OIZdIWVJzY3V5S08v/EABwBAAEFAQEBAAAAAAAAAAAAAAMAAQIEBQYHCP/EAD4RAAEDAgQDBQcDAwMEAgMAAAEAAgMEEQUSITETQVEGFCJhkRUyQlJxgaEjsfAzwdE0U+EWJENicvGSouL/2gAMAwEAAhEDEQA/ALv1/D+mrU9hULp3EdolwdxxB7qgobOdQOVq9Yn/AGbfA7E9TW1jqr/pXq91b7qqW98vfUvOSA605Go22tbZ8fa/71u0J0gbRatBg9DSOJp4g2/QWR3YjUztAleTbqqJXn+yo4ePsli6cd4tpARlBdepaka8o7NuzTvr9lJei6ZdNc3GCqCDEesXAgnQ9U1U/LmbejyQRHxt2ArQCTY5pY0MTOXlje37Lvi/TBBu3HmHqkI5VFNWs5tD/I0sdBvtZr1/7PH0hrvcccYbw9X5hH+r3mWjpz/1hDfKLevIHMmeeot2LDNG081LjnmF4w1HoU+kvSNvLcwEl5ZJhLF9Xc6c07kJqCdieXOyRW+jP6Ql2LbNTwjxgtKVStbF0vPp2M95oLHIaz8be2a6dh0/asNrHigGxddyXO4cy7spSevZAG0O522KlxwvDauwBxAuB5X77wriKhGgJrbreZynX+dI8Nv0smJNQjMhyo74ChOpMg7wfjrb3dTcV2IENU6mo/7txSPuIsRvLA+Gb3SUXndrdWgjVNQO1HwXItE0juqQnC8PB61CApRTzJjWYnYeVjSVVIcbUgqUG0zEcx8iBPj8rex15ejpwbvYZavh1hpfPMq5KNR+JamzarfQ44D1qys4CudJUCO4w40Pgy4gWAaN6Mydo3XjxQvOhKmlPqH2rqTmBIKsx8OvP4dbKdO9UKZpEthUBgd6J3Agx0j7xb1Ir/2fnAV9KvUcKIpVKWVZmLzrkak6+264B8LNmu/ZwcL3UoRR11/UobASgNXw0pIgR/HSE7f4rAjpJI3XIVh9RG9tgV5udqsOPKIWpIIQQB7vdv8AdYBx57KEuKzSDlVlIMbaj4cuYt6BXn+zQusqUq68fXyyCdEuUlPUwPMONTy5WaN6/s0sZNoSi6cfMPZABNVdKmwQDMHs3XPHkbH4bhyVbMCqYKWorQS4ggGCDodjrHn15j3W6Qp1SlpDyoEDmQRIJnny+/xtaW8f2d/GmkWVU984ZqEjXKF1zZ6mJpoHxNmxePoMekJdqHPVcM3RWZ5ILN80zafCA8pB5DcWYsd0TgjdQGaioTT/AN6oEa97XKTy135fW4yHKtDqU9qojX2hsI/W0oXn6KvH+7cod4V3s+E/xUZaqxoJ2YUvn4WbtZwd4uXQpa704Y4wpUITGd25KpCTvuSn65WgWlTDrpqIWt9SgQmASCcp26+Gg+Xw7DrgCASMrgkBJkjToDvIsM/dd6UKEt3jR1FK4pRSUvMls94xpI+o8LcoacU4nOAvLuDrroOn1p5WGQUQboVLjhJSAqBqoAAxHkd/CzUvWqqWcX0QQSlRQkE7QIVoT7hpvoN7OtCVlRcSgGO6CI1Hx8febM/EBLd+UbqUhQUJCgrY5zEfAbdfG1eUXCLGdU76GtU7TD7QLhwpTyOsdP0sIagpAVmKhM6HfQxv7+nzsVutRZoxUgEqzwnUCNI+P11sKtZSlDAQo9p0AJ8dvH52hCLNsiyHXVbVUj2QpIA1OUkHnEe7x5Wxx1aU5t5IGYAHXff3ePnbSVIUtXdByawQY168v0Ng1KZLZW6gHPsCI1nQa+GvWxiNEJCLcfLhUVnQDcATvt8frmH2rmaXHezMlUeB6Dy1+HhbboST2crlROYgkDmdJ32J/rboM5kkmco7oGbTLO53Ez91hndSC4Ln93nUpCiQUjmPEbmJJ+duczql5lgpgbKIgDpHXzsL2S8wSCpStyRtt+Y0Fu22HUqJSQRlAiABtM6npPhaBcNlJYhS0biY1OxA5HXccrBPqU9TrSgkSkwCoxrET7wDYyEOBScyUgqMZYVqCdRPlpvbtxhwtrDgKwpBVAJ9xEjXn87BcbKYFwmRhmpeYXWoOYlRbKgBEjvEggeeh56eVnj6rTOhL4ZCAqVkEbp5nx0J+PnZn3W6tqvrmEtnrAXEEHrM7E+Pv0L+u3Mq7mDkOVxAmQdT16fXvsB3hddFaLtsk792MuZAplQJ5SdTpymenwtyu629crzomE6KEmfPnP1FnClpKEhRCEyARrAAjp7j9+lulst5R2bWYkTMkCfGPhZuImyJtm5/9iioUdBmBG35/jrYA3E4SSstqBHdzp021j3b+XhZ1GmaUSkgjpG5HXTy+t7Y3Slaxl1MTM7b6jp+ZNomXKpCO6a6MMKeAWpinWoQDqUyOp002+jbHsNLywijBhRMJX4bn4geVnuiiAjKkaSOkHmNbcLYKgpMAAxPOImfn+FhGZEEQUduXBUHRFI6r/ECJUJGkn+n4FH8PPpUFON1TQCtcyCIMQBr9RNpNXTlIJUnIUGRmA0P56H5+FgX6JzsFOBCjCTA10+6efztESlIQgqJUB5p6AM0EpIgkaGZjzO3gOeyi1QF9lClhtKuSp31289j8bavRnsalcpJBWYIPMkGfPT5ednphNmnqrkbS6w2qFrbUCnMd5A+cCzufsUzW8kx6rDLbbC3qm76VwJlUJbSSY8D7/iNptHWMbiLVKqqprqSyEqEFpvLpr0EQZHj7rT7f9LRFCqRphISDKykECZ9nT3/AI+EcYrogLnqUtp7qFhRO/ekfn5+VtKhlc1wN1TqWCxsoQLTzZlKFiOgItsVdYiQH3RJzEZjv1+6zkcpyT7a/tCdVDnJIOn1sdrFfVgtBQO9oSJTIgRJ193wt0YlB3WQWW5pFNbWaKLyjlgCdYtoVbs94A89RZTVSIyBwoSZB0CRvrp+nlbj1RKkSpjKNydhvtH1r7rSD2qNj1SaalZGoEnnbr1o5MhEgCBr9ePxsZVSN5ZUlSYE6a6bfX0LAPU6ENZkZt9ZtIFpTeJY447CXF5u8IBJ5W49ad73e0VuOth3vtKSmCie6lSB5hRP/ULFQyogmRoJ52cWKRuuxUlJzBInnbv11ZEZQJnYxYAtrHLxtrKomAk2ewTZijLVS2lQK0d2RpvYyq8myns20qy7ZiYMT4WTi24BqhW3S3MHpZsoKWYo76y2pARzHUaTJ3FtB5OoJEyNZixPWLZryFllCV0NUHNCpGvSwUzFtWy0homJQtNUO01S1UtLKVtLStJG4IMg2+kvhvik4u4d4XxU2sKTfFzUVfI59qwlf42+a8biLe+foIYmViz0ReGt5Lc7RynupV3rPT1Z5xgD/hbFuZ7Sx3jjk6Ej1/8ApaOHWLiCp+VUuRvYI1Du4NuTvbStbcmCtgNb0Wdu4T7Vs7dzUz8LcxztqCbGCnlC6YfcS+4gq0ISr7x+FjHbOz7RsTSSmoQdsyFD4EWHMzboaM5ogq8oAKG7Z06lRtoPu/zW4mRFsMnna0AhErrt3Brmtnbu8lbWD52zbnaQF026F9YdOmY2T7xecD9K8VnRZQdeRFjnw91iV6pWaUrbIlCkrnyNk4aJ26Fddu9Ht25NQ8P44i2gZTIG9uFD32YjRPsuxUukRPnYdNS6UTPIcrFI0MjWwrAlAHutXc2yI3VDGpeiCr52QcV3hetHh+9Lwbvc0RpaV2oStFPnKAhJVqDObbkBOtld5fZpkIKlKMJSOZ/AflZBxLhlvEVyV913tVPuIrmVtAIOVDBI0KUiMxBAPenUctrBcwyeEKZIaLqvtb6TdwIum777vC5Hq6rrgy+lhbDCV0ockF3OHVBJA1LaghY2UQdLN3iZifBF4JrUVGI78Xfq3E0jtGkLcNGcx7nYJhttSvaKHVoUEmTJ3auIeH1z4lwxfuE2r0dp8QYKfco/WnezUpTKUgrkhQBCyc0nUEZddU2rWzipVLSUGHnr6YdvO6K3KyENoU0wlDS1ZFFaSFpSvUEqKQqImYtnVmHiWBzSbaH1We6tkY4Ai4KngYnwGkBLlPifOBCovCmQJ5wlNIQPIEgdTvbLNxn0g6ltlDdVSXut5KQHFJSkgqjUgpWARPQAeA2tlvODhtbf3f8A9v8AhaPFh6r00fH2unU27bAUItw+T20RPW3aDpv/AFt9IO3XIsFghkjrHvt3AAtwklO9u0kWHZSXYEH9bbAjbblbkEKI193S3YPKySWwBO3vt0NtjbkRztsGySXUbWwTG58ranwi29Js1k4XQJne2wSeY00twPf1t2Dv94tEqS3MA6W34C3MmSAf6WyYmySXQI2Nt8teduZERMe+2TO+1kkujAOmltnvD5W4Jts+NmskuF01K5o5TtL/AMyQbBG67s3/AHfTiBulsA/EWMSAOnKbZmAHusrBPcom7ct2vI7J2nJR/KVqjfpNm9ePCXhve6iu88FXJVlW6qi72XifetBs7c2kZjbJjnItEsb0ThxHNRbeXoxcEbykO8NsNImT9nc1K375Q2k8zz52Zt9egj6N99uoqKrh7RpcbgJWzVVjMdNEPgTqeVrBzr52wq/mG1hGCM7hSEzxsVVyo/Z58Du+LuavigB1Cae815QY3+1S59RZtXr+zdwS8srujGl/0hCSEh19l8A6awGm522kWuQCImbZPLoLR7tH0UuO/mVQyv8A2a14pacRRcTXVqXr3rmQE+UipnkOVm1eX7OXiYktmgxdcjyWdU+sMVLc6RPcQ5yn5W9GQZETbJBM/jaBo4ypCpeF5fXr6BXHSgQ+umbw9XrI0DFc42VdD9s22PmLNS8PQ/473UhPaYBYdZbJUssXpRO6Cd0h2ddeXK3rbMG3C0NrTDiUqG3eFhOoWnmiNq3cwvHeo4BcaKdpVUvhPil1vdJprsee0Ame4kiCbIN4cPMa3PkbvjCV/UJBlRqbufZHiAVpHhrb2ZZu27lMJSugpiAIjsk/lbf7luoiRRpSTuUKUj7iLAOGki4KK2tA3C8T1MBLzheXBb2BV4anXXythpnCySh0qcXIToNR8NOWtvaerwtcVcCippVug6Qt1agfcSbNy8OCXC+9ZN4YMuV8k7vXVSOke9bRPzsB2GSciisrWdF4gFhdPfTzI1UtCspJ3Mjl9fKz/wAPhblz0yVKISjMQdUlRnbeNvhb1QvP0O/R5vZwvVfDO4e0IIzt0QYMHfVkoj3WRX/Qf4Glo093YeRRNme6xVVYVr0Kn1D5WBJhsxGiKysi5rziSD2WctaAADuwCeRjz+UeFhEUPahLJKte8VHyEyNxuOdr+13oD8MnVZ6C879Yyg5Ui8GsvwVTqPzshu+gPd1Mpxd3Yxr0FQyp7Snaeypjb2m7U34fUDYKwyrg6qlibto1OEqQgAAgefMkWGZu1lAAUkdo4ZlKpPu6jz62tzXegrfzYQKDGTbwSc3293hsTB17jy+fhys3rx9DTiRSl1ynrrlfWYCAE1Qnz+wI687U5KOpbqWlWWTwHZyrQaNtx09nmSGkgFSj7W5kGOXOwaqF0iO1IUuSMyY8tJHytPFb6J3F2myhvD1HUJTqpabwYQSfJakn5ToLIFVwB4qUJccdwBey1AQgUyA+B1nsyoHrvy91q5gnbu0+iIJIzsQohcu5RWpKMhABUpWUgDwHWRJ8osXdpXjTEEN96RouCdxG3OD8bSZX8LcdUK0t1WCsR0qUpCnFO3c+kdYkpj4aWalfc9TQKAqmalkpEJQtGVRPM6jx3iwjnG4RBl5KCsU0zjL6kqQjNmClFKpgRqT0P6WXMG3itm5nmwStaXjl0gQQNfHbeweM6dxpNSktDtnRmUQQTAI06RH32T8GEvUNQ2ZjMkgEzGhnfxA3tdY0uaqrjlfolioUVOFZEqV/MfGd/KRp1izQxRTtuUNal0SEtzmJ9525ab9Dpzh4LQsoKpGYmZjeOnWLNy/mu0aqUdmcq2VAa5pGXz1/XcWu0/hIsq0wuFEjzZglCSspBPtZp13j4+MneycQpQLWQ5VCSfEffEHz+FlRSsoSlJcBI5QoGefhtBkdegsmqKHESAdIkHmBz+ult9iyHIJ3u5ieQGpn5e4fpYN1KRlKwgKGh72/1pHlYQrTKlNrBV7KtfaHMjw0+VgFKXKc6CEq2GhMSeXLf52KAolaXkICey0He8hyHx+uhaqALTgTEggkk8/A/XOwy8wSCEqO8iZ25zYJzvIcCsw011nX6ixQo25INKpups5NGqhXe/zJGn/LbuG15UEBWYHUHWeW/kPnbinSV3bVoynuLbXoNvaH4i3KMywJB3g66H38rSKbktltmII3MEnQzt8jbMqUEJVEjoYiLYVkmVCJ8YERE/dbkgqOgGo73Ub/AJWSS2UBU8xpGuvPfrbSmkpJTJmYBI++3RK8xQpskHc89LczEFWhO08/ysklzlAJ0BIEeztblTWsKA12k2EzbnLmmCAVaQPvtpUT3ANehj6FnumsgFoSkab+dg/Owy092dbBAHUeFpAqBC0N7ezH7JXFKr79F6puNx3MvD2I6ylSn+VtxDTw/wCZxdvGcbi3p1+xnxOSxxNwa47oFXdebSJ/+mbWY/8AZ/K2PjrM9GT0IP8Ab+6uUDrTW6r0vVI3NucwG1tr2sGfPa3DhdC0XCE32tzEWxB2EW7AB2scC6bZBqGVbSjAhUfERYY6WAqP7on+UhQ9xsY20623KA/p2VeYc1mlt7crYJ520QCdvCbXwECywxFtDU23HIi2jPM2e1kitknb77BVCA4y4iRCkkRYUSdANbaUJGnjaYbcJr2SfTnMygkz3bbVJMxYKldQvtmEKSVMuqQoT7PMA+4/dYYyNOtoqV7Lka6WEY/iB1OhsGNz0t0FFJOQCSDEmAT0+hYMlgLlSjJvZDpbW4sISmVHQWLvVV1IafQ/XtJ7MFR74JSoDXQaz4WVKUXWy16zeVTTgKSSO1cASB5ExaLMR8TbgRjd3AhvKop3HmBVIbYIRnbGiw4QFOAAHNkSmf5ykA2zpcQFI3OSAeV/79FIjiuyhVX9Jhy8rt4vf21wJX0lO09dvqt4O1TDlI2t0lTZaW64lKZKM0EHMIIjQC1bsPcI8l5tY/xxjS6MOXLeLy00LLjwb/eBKiVFIAKyiDBUhJTmj2gDF4+L/EK5au42rmvPD9XeVI+UP01AxdqqenpRPcWXXWuxUtAhUZwiQUybUjxDwexbjYGvxNiynpO3fW8mvvJwUgLCRAAClAOEJ0OQEaEawTbDqMUdM0tdIGA7m2v23Qn07WPzBubyVgaK7vRhp6NhhuopXUNtJQlxdM6VLAAAJIqkiT/lHkNrZaGqTCuGaWlZpX8YXG44y2lta+wq+8oCCf7sfdbLckaB1/8AUv8A59la4v8A6BesLwPaanmbdpBjnpbToh3mJnlbtAAPzt9DO3XJN2WFRG9kyqqC2Xni66lLQJISqNAmdLKS5Inpysi1yS41VNE5QtKkExtKY/GxIxcoUhsod4b+mXwX4nX7Q4auDEN60V53ioN0tPeV3qaDqyJCM4zICjsASNdN9LTqxeFROV4gnlpaA8L+i/hC471uy+lOsVDt3PtVTbqWsqittQUk7nmkc7PX/SE3XY+VhGhpK5H7upi9VvVFK4whS1qGRLecDOISs5kynaCYMFyBw1CgHkHRSo29mAUIE2AfvLsl5Gmgop9okwBYGncIZnnvaOeOdJWXtwnxDdFDUOMu3my1RFbaoV2bzyEOAEaiUKUPI2GIxdTMhUgUWJ6SvQXaN6lqmkqyqXTPpcCVdCUzr4TZUbrG3Eg6687VI9HDgtWcKMcJvG66t1q77zpXaerpQtRaUQkrQqD/ABBSd/8AERzNrOitS1W+qkjVAWPfI/CzuiHJJshS84+00guLMJTvpNghedISCXonqg/lYlVuk0wE7qSPmLVy9KjGHGrCN4YUc4SX4xRodarnLwpnqFp9NUUFjsgStJKQJcHdInNrtYYiuph5vorQt1LDolp5CttjtYTMI018rMLCF+V1dhS48QX2wilra+hpn6poApDbriE5kgHUd5URZ2MP5kkg6EWgYraJxLdKMwRI+NsCjOhNmBjjE72FMI3/AItprqdvN66aKprk0TTnZrqS0lSg2lUGCcsTB32tBuG/TKevpxtlzg1jNhxUJKadYeKSTH8qbLhW3Kdry7YK2RV/NbM0gybJtHXLqWkO95IKQSFb2J1V6PNrqXe3LbVPMwmdAkKJ2nnZhGTsnMgG6Xs3Tbfe2FWu8C0K4B9Kbg5xLvZi48FcRqO8LwqZLNK7RP0y3YGYhPaoQCYBOUGYB0tLVNeSnQC4iDzA2sxjISEgKP5gedsJjnZEvfEdJdTjDTym0mozZO0dCAYAkCdzrYNGJ2nAFJplKB5pcCh7utmylSzBL+bqedsmNd5slUl+MVb3YJYdSYJlWWPvsoocSsSk2jZSQgV1tk8h77czHKwL9XT0ye0qX22kA6rcWEiTyk2iQkjClSf1tmfxsC3UMvJztuIWnqhQIPwt3mkSNpsyS6zdbbCup+NuJm2oJ5+dknWNHukTzI+dhM/ObF2v40nXvq++wgII3+NmCRQoV1A1tokg6TbknTytiTqQI31s6YLqdiZ+Ftheg59LBn5zbcgbCzFTBuhQrnGlszGelgkkbm25J062FayVkIFDcae+2ZzOlg+ZtgPjZJITNHP320tDbgyuISoHcETYMHnboK162YgIjSgTdt2LIUbvpyRseyEj5W5dui7Xk5F05KTyC1AfAGxgbaC2wR77QyN6I2Z3VNq9OFPDe+ypV8YLuWtKxCjVUDL+YdD2iTNmnU+ir6PtXnP+ivDjJXGb1e7WWCQNtWkpOk6WlIdbbBg2gYIzyT8R45qB709Bv0dryzKTgv1VZEZqevrEn4B6PlZl33+zb4HXqVOU15YkoSrkxeKIG+3aMuHmedrXZjpJkC2wrTQ+c2h3WLkFLjPtYlUDvj9kXw9fzC5OJeJqQFMJ9YFNVR5gNtE/HnZgX3+x5v1KZuHjN2ipJAqbiCEjXTVFUo6T/LyFvT1KhvFu0rGotPhW2KhfyXkBe/7I3j7TFa7rxvhCrSmcgWuvaWR7qZYB9/vswb7/AGY/pX3WhaafCtyXlGYJVT32w0I/+0Fsj4W9vgsH8rdJUN+VlZw5peE8l4AXz6DPpV3OV+t8F75qCJA9QfpqyYG8MOLmzQvf0afSFuQKVe/AzH1OlOpWvDdYUAdcwbKY9/nb6LVtsuj7VtCv8wBsXF13UTP7upgY5MpFkHvG6bhtK+Z57DGJLnTWMXzcN5XeoNHu1VI40ZChocwEc7I4SoNzty1t9Or1xXVUJKHqcqSd0B1YSfcDFmve/BLhNfxUb64fYfr83teuXWxUT/7RCrPxnDkkYgea+bMocEZtNNdCNOU20pRM9yATtNvoWvj0KvRdvxJFbwTwinNrNPc9PTn4tISfnZi3v+zK9EO9ypX+i+npFKMzTXjXNQfIP5flZd5tu1R4PmvCSCD3UxG2ttlLihCUmJ0jW3tTe/7JH0Y64f8AZzWI7ujb1a9zt/8AXNu2Y1+fscOGTqnF3BxDxTSAju+sLpaqD4jsmpG/OzGraNwUuCSvI9CDlzDQzrIjT6/C2dkiJKhGmo13FvTS+P2Nl8IBVcnF9xwnlVXA2B4SUVZJ/wCGzDvz9kVx2oCs3TjHCNcn+FLyK5pR/wCBhxM++yFXGU/AeqCqSj+ImT42BKSDFrhX/wDsvvSwuwRRYcuC8ttaa+m2gT/9o7I/LnaPr59Aj0uLiUF1fBi8qhJUB/qNbSVh+DLqiPeLFbPGfiCiYX9FXvUWu/8AsjcRqur0k7zuJTuVu+8M1TQST7S2nWXR8krtXC+/Rg9I3D6lG9eBOPWG0/7T+z9UtEf50oKfnaT/AEDW8TcOfS84fVN/3Fed1oq6167HPW6RxmfWGHGkg5gP4lJsGvyy0rwDyT07XMmabc17kKMggW4522VamLaHjbz4DVdKFnusIgzIFuBr4W2BrY7Eiu3EZ21IP8QIttpWdtCjzSLYk5hB3txTK+yUjTuLUny1kD4EW16A2JCrS6BCj3W3FtCDvpbYmNLaoCr3Wc5m2hlgcrbJ03tke+LOEyyCBIM25URtbek/ibYZ2FihRJ1SahIRW1CEpAzwvQb8rdq31GptzVAiubWD7aCkj527VKhMzaFksy5G4IG3Kx+6ruFW+X1tpyta7bq5CxFsdo+3TI1ccMJTv7/IWdvYeo0JbpU5lJRKdPaV1PvtXka06nkmfIWaDcpt3xQ3Ve7qziR5hm52VJbDbjvZpfeUYhZkSASEhM6qJkGBaMuLXD3hxdlwV9fT05w2mphp28aeudoHHgogqSMiVuPKMJ7mQ5tdbTS8GKTItbWcNkFAy5lZoiR46n4myPifCVFjK7Td+KqRD9A6tLhogshKinUBwpIzCYOX2dNZti1VFHIx0b2hznc7bfflbknZKWuBB0VSmlPXtThq7r0x5f13UjfaJXWMuPtKUlKyQy0ptxSFQ2YByLy90e1Brzih3CWIcbU9wVeNsVu1F/VyqZ2gqaPs1sNkpzuZE5glAyz3koACZCXIMXz4m33U1l10l1YBoFIYoX6dKKimYCmlpCi2tlkJICYSCCpRSjZIVMlNQ+N+M2g1fdVgm+Lpuy+H7qDFe3dRbdqWVtocYCHKhBkAdqhTeTKFDtNJFuIqaLu02Vri4deQPodlrts+LMdPJMuvYwNS11RTU/ol8SFNNOrQgqdr21FIJAlCachJjkCQNgTbLRX+4OMqu9dtVfIpDrTh15IWG/4cwKhBiJ0FssT2fBzkP/5uWfxz8v4C9jnxD0a26A0j522+gdqOptiRpJ3t7s46rmW7LgxyMWbGKgpdwXy2DGekqBv/AOrNnUpEg6TOwtCXEk8a2MUVFDhqruN3Dt4ZW+zqGFB9lCkhLneG5nMR0kTtYjHNb7xshPY5xsAq+cD+BGL+GuMbgv65MX3kKBuoZbq6NVQS08wSErSobHunSZiBGokW6vZtg3lQurbBXlcSkxqB3dPKxdjDNXR3C45QpbVXNMqUwlWiS4BKQT0kCzDwXjTifizEjd1Yu4d010U9Iha/XGq0LBWIGUIJkg7zOmWxjIzS7kMRv6KaaUxTb7J52b+M2/WMLwnvAu0yjPTtE62ctIwpLGRQ5Rp5WirEfFO6+Hle9hziFdte1dzhUaS82KdTzCmiZyOBPeSpO2gMgA6WUbgTome0gJ23Mwhp2mWlIgSNtu4q3dVUZ8TlCYhFK3OvPMo/daO3fSS4HXVS9uzi5yscAhqnp6J8uL/wjMgCfMjzs4OGtdeuLkvYtr7vdoU3m6XWaZ722mAAlsKn+LKkE+JNnkeL6pRxuAUiPkilbJH+0Ry/xC0I+kT6QaeCF44cp3eHL2Jqa+G6px19moCFUYZU1HdLas2btDzEZec6TNiC8KK5LndvS9KxqlpKYoU686oIQ2nMO8onQDW0Z4pc4WcRaqkqazG9wPCjbcbaSmtYcBzkT/Fp7ItAG+ykR1SrTvUHFDD+G+INzXveTFGtlN409GVBDTwUg6PJAkqTJjvQCNuYf11uq7FIVMxr1tGT+NcDYNuq6sB4cvOlrat9SKOkpqV9DikIJlbiwk9xITmOu5gDwki6AtVOhShEj32Y7papGxVV3Fd+FL7vDFLymbnpaSqevB0pUrLTJSpTioSCowgEwATpoLRThfiz6MFU8y3hvHFD2zykoaQ4y+gqUSAAM7Y1JI0m0t4hu5u/MN3xcamw569TVNNkVEKzhSYM6c+doywl6PuE7ir2rxew5QpeaUlSVdkhRBBBBkeNhSa7i6LFpzspqoT9mAgDLG4snOjMa9tWynCCfNtIsp0iUNtBAVoBFk1SAXbwTPtKPL/1abTZtZQfuoMwn6MWGsLYoubFDQp2qu6ahFShxlBCllMmNRsZg+FpaubGFLe991l3ULNUhN3rDLqqimWxmcgE5UrAJSAU96IM6EjW1UeF3B/HvDbGlwX1dGN73dudNY23VUD1UtbTjKyEqSRMHQzrMeYBtbspaNawsphZSoBUcpGnuk2ThlCduuyReLfCy4OKNysUN/UYfVSKU5TrzlJbUoQSIjcCLIfCzh/SYGwBW4QpFLS01UVgSSokgODNuTM942lhgZmQCSdLIaUpS/eLYG7xPxbT+VmGoNynvYgWULej5wovPAN6uXnUYmvq8EuU3YIarKtTjbYKgSQk8+6NfE2shTkBEEAWbGGqdtLDZSkeyPOzoQiBpp0Nq1iNCbqxodghiqzH4qKUq4Utg93t0KIGs2e5O8aWZnEtAXcZJ1hxJtNouDdRcU0sC4j/AHe47drildk/30CfZXGvxH3WTcMX1UDGia1T7qkLqnioZjBnMNttzaLqzGFdc3GsYMqZRS12H2Lyu1w/+kNPvJqEeeQsq9yrPHBztQqsadqW1NuevP6Hp2ysp96YjwsPZgBUhq64ViaR8OpmbGQZ0H62SLpUpTQk8hZVBgWThYpLlr2nJ5L5+QsLMaxYFsgLcGs5gY9wt2TPKbMBdJNvFGMjhypYYaoRUrdSVLlzJlHLkd9bD0+LqN65Dfbram0NtqU6gKzFJTumY1/UWaXEgH96056tHkep/OzIrMTMKwviJuiqgsUzL7TyQofZvNo1BHIxB8QRYLn8OpEbvdIVlsXEp87dwVM2HcYUWIu0SxSvsqaSlRz5dZ8jZaU8IBSPKbRZgt9VHVBLayQ418dRZ04wvx66cNVVayftQjKggxClGAfnNnpnGVlzuo1EbYn2CP1mOML0FSqkqr4aDqDlWlCVOZT0JSCAfDeynd18XZerPb3bXM1LYgEtqnKfEcvI2qDxIuviBfd2sNYAv83XUtqW86sNBRd07qJMwNyY128QXDwIxbiZVPc1fiIKbvJThorwSABmIWUEkDT+VWkCRpYTXvc3iHa9kQxNByDdWnW6lKcyiAneegttDza0haCCCORs0cd3dU4iwZel0UTpbeqadSWyFFPeGoBjkSAD1BItDvCfGLl3Vie3loOj1SrbUfZcSdCRyIPyUbWnxZWB/K9j5Ks05nFvNWQDiSqBbqQBJMWgyqxbed94pQunvWrYo+1S0hNO8pGZsHU6Eb6md9RaYUV6V06n8xCInvdLV43CVxDeSNJGYwCeaUA6Npt2FAiRHhaBeIPGupuC76zELL9Yi66SAhFFTh59+SACB4zO4AAk7TZV4QcaWsd0FBeLNWqpu+8wQy462EPNOAlOVYTpuCOfIgmwjKL2tpe10UQuAvfXeymYK26WwLAgmJsj35frNzXPVXq+kqRStlzKCAVRy9+gsyl8YKKmDZvCkYpw4cqM9aE5zEwJSJ0tOVzYQC/mmia6a4aNlJwUPL3W2FTZsYWxtQYnU8zSsqQplAWSXEqBBMbizjSqedmY8PF2qTmFhsUMF6aRbebxsDm00Pvsi1OMLhoqpyiqr4pWnmzC0rXGUxt0tImwuVEAk2CcIX4267TlrZJu2/bqvRRbobxpahYEkNPJUQPEAzY+XAEk9LR0cLhIgjdGQsEaG3Qc87FG3Q4mYOmltioSpWSdRaNk4dbdGguNfjbfaCJBsB2gSnMT520lwKEpMi0HAqV77IwHDoOVug6ORsWKwNSdLbCtgPvtGymLI12pnU6W32s7WLZo0m2ZoG5i0UrAoyVk24JB3sBnJnWTbO0nS0SErWQqjPT4WBcS05IU2lU9QDbCsnYC3Kj4izFo5qQNkWcu+hUcyqNgnr2YsA5dNCuCWimNgHFCPcDY6penP324Kj1sMxNI2RWvI5og5dFNlIbdfR4h0n75sCbqXMt3jUf7wQR/4bKSvPa3A9oePUWAaOF3wowleOaTV0Vcn+7rWj0zMT9yhbnsrzQdfVV+9SPwNlJY03iwWnvtDuER5KXHciYVXo1NG2SP5XvzAsQumpvdx2rVUXe00pTuYN9tJAiNwCD7PzstGedirY7OsWJjOJjyj9bJtI2IgtT8YuFiF0l2qjvUh8crgP5W2agpmaV8R0SD9xsPsJ5G28xHQWthh6oOZFjWsDRQdT5tK/K3IvGhKoVVIBA2On32NnXU21AIgj42bK4c09wghU07glD7avJYNuiZ2I+NuHKWncELZbV4lANgF3XdylKPqbQKtyEx91peJRJARe93006WH1BUJeSNBsCYtwKk1Ci3TQcuil7pT+ZsSv8Aw829ddSihedYdyKKFlxa0pMb5CqDz0sfo0JRTNJbEJyiABA26WXiLiEjlAuE4rkugUKDVOJJed9oq9qPwstKcSlkvKCiEAkhKSokeAGpPhZGpr9ZRTIRUZlOAQTHTY2Zt78U8WYXDlTeHDh28KMvFDT913gh4pRBILqHEtqSdDojPrAEkgWo1dXFRsMkxIA52JQhDJO6zRqn8+yoH1+oKm+5AQVd1tPjynqfdytEfFbGOJr/AHEYLwNSMlKSl2vqKxakN1TWaFU7aEguLSZBWpMDKCnMCrRuY746X1fFUjCzLFNcbrikuFovpXVvNgahKTATBIBJSqI2HJpXxj2uuK4n0U123I1U3k7kqF1iy6p4oSIU44rMpzQgDNISAQJ2tw2JY/NUPbHQg8M2u62p672stinwtzGZp7X6XQOJGL1RcdXduLcCVfaPBZq7wonGbxbJgAKNPmQy3DaUpSghRSAhIB7pNauINw8J8Mm7bwwtW3qugvFxLLpW0W3UvaOJcebXlSlBKoSlKEiFE6zaY73x/iRFEf3HeWHlVT6FNoRTXc0ptDkEpUVhsJyJMGSoa6QSYMNX9iKiqr27bGr9Vie+EEIStkqpbsIlJAKEJ7RWaNYDbhKT3VFOmTJO6STQG3RWZImxstcEqJ6ziHxOoKt+hpMNYmqGKdxTTTq+yWpxCTAUVZBJIEzAm2WnRdXg5S1KYwuA2SSjs7szoy8sqlZlKHQlRJ5k72y0Pa0X+x+/+VT4D+q9HK1ID5002AFuECdbD14+28zysEkaAje3v+65FYEnmLAP0TbypUmTY0lMxIt3Gm1mukiaaVKUZQnfSxZFzU7b5fQ0ArmQNbKuWPdbeWNxZiE40QCGsqYsmXzh6ivhos1bCHEq0IUkEGy0RbYRPKzGx0KVyNlGzPCTCtPV+ss3PSoWFTmSykH4gWeN3XU1RNpbbSEpSNNLK5QneLZk8LQbG1puAiF7njVJd93LRX5dNRdN40zVRTVKOzcacSFJUnoQbQzeXoxcPKl0rGH6VJJJ7qMv3WnuNY2i2lNpVrFk5ufmQosdl5KHsD8DsI4LrfXrpudhp+IDmXMuJ/mOvO0pU9Lkby5YseDCQZCBbYbA0jeyY3JzTvOc3UC8SeFPEC9cTVl9YW4o3/c9PVZVCkZdJabUEgHKkECDE+ZNmuzw89ICjVDXGa8lJ2lyhQ5/4jr8rWfXTocPeTYH1Fon2bJ3EJ0d+yk0sA1aku4qapRQMN1z6nn0NpStwpAK1AQVQOptHHEpji9SYlQvAlfc6bqqEIS81WUylLS5JClBSdxGXTqPG0wIYCNrcO0jb0ZgNPCxRI5vuoRY07hR5WYWq0YYqP3e22quaYK6cKkJLidUgnlqBZtcO8R8TMR3w5SYzwpd91sUqCpLzFSVKWskDKEEmExOpPIeNpoFIkJKQNCLF27saadLiUJBO8c7J0rzZIMYNVqnZKWwmZ05WijHPFCiwNit6467DWIKxValt1p+hou0aMjLlKiod7T5i0xJbCRBFk+8bjpq91LrzQUpBBEizGQtGgT5Gk6pMw2y6mmaUtBHdG/lZxIiwNPSpa7g0FjQSRzm0C6+6f6LShzizU4hKabuJx2ocQ022tJUtaglKdeZNnYYsh4uwzdmLLhqrhvilRUUlWkJcbWNDBBHzANptcBuolt9lWbi5hb+0NPRYtw0609emGyirplMrCs6Uk52iRyUhR06xZwYUvBurTQXi1HZPlpaOcTH17rSZgvhXhzBF3O3Xc13IZZdcU4oanMowJJOp0AHus0mvR2w5RYzTimmerkZaj1lNOH1BoLmQMo5Axp4RZnyMcwA30U2scHXBCk5m86e7KL1qqUoIGVOgkkkgAD42WKG8qeta7ZkrAnUKGvWzVxlharv7C1RdV2Xi9d1UooU1UsgFbakqB0kEco99g+GtwX/AHBcvqOI7+qb2qi4VqffQlBgwAkBIiBHnqbQc5ugCQaSLp7oIDqyD0sIYiB52DRHan/KPxsJrGny1sgbJlHvEVsGtoTqJSsH5G1abvYewpxvxVcN6OKduPiA0iupQZCUVbTCGX2v99AQeWxtYrjeMVU1zUt4YRuFF71bbxQthTgQA2RqZO50A352j29eHWIMb4GpLxvG7W7vxBQuJvCmbQ4Fll5JJCMw01T3Ty15xaMkQkmu46EW8wUeKUxxWA1BunnhOWXaRBWV5GuzzEbxGtl3iInPg2tVPsBCjH+dJtGnC/FGJ7/xCbvvDAd5XS3ToUXqipWAnMDAATAJJmegFpixDc7t8YavC7ElIXUU60IJ2C47vzi0aBhibleeaeueJHBzeijLBFO09UPIcAPdGh982bTVM3d2Jq9lgZQ3eJWlIIG+VRPz+fwRcP8AEy7bivF27b+fTdF70Us1FPW9xKj1B25Tv5SLK1y3xdGKMTFdBeVLUv1b4fcbpnA5lACQToTGiRv1tGcmGEwuGt1KJvElEoOllYW7yXmQlQ9oSbQZxCwyvC2PEV9Ex/qF/hanUpGiKhIkn/eE+ZJtOt0sqSykqSQSOlmbxkp2UYfp612Einq0qUTskEEHXkLXYiHtLDsQqjyWuDhyKaWA7sFXWpeCISyAkCI1/paVL3aW3h+tSgwoUzkeeU2YnCd5itpqlTS0ryPDVJB3SPyNpNqmQ9RONEGHEFJ99qFEwM1Kt1j87rKBLnuakv6nprtq2kONPISopIme7NiuA+Bt54Txg0u7sVVjNwU9Waxq7W2UBMk5spWQTGbpGgjxsewheLVI+2HVAu3e+umfSN0KBKdRuOtlurxXiF6/akXbWFijZDaWShCVdpIOcmZnUx7rQe0QGQOOt9kZhdNky7WT/wAdtF7Bd7J3PqjioPgmfwtA2LMCYf4i3PRUOI6RL7NO4mpRqQUqyFMgggjRRtYS+0+v3DXMAhXa0riSOspNq+X1hZeOcIUlzovK8LtWvsHA/RP9k6CncBQ5HW0q8foMcDazv3CjQn9V7SL6KRuBeB8H4Boai78M0iWn31BypXlVncAJCZUSdBJ08T1tMCFAjS0O8DOHDuBaasdqsQXxe79UUp7S8axT3ZoHJA2Enc76C0wtwIEWFFfLqbqUpu7QWXSzCSqBpaueL2K5yoxExQVPY1ZeqksO5QoocIORUHTQwddNLWLcEII8LQPithTOLb6Z2BdSsH/MgGxpm3pJD0sfyoU7rVTPuPwom9HzF/EWsYvJGPHQu+LjvINtVTdOGe0QUgiQmE/orW1wq6rd/ddS7TLyudgtSD0VlMWp1wu4mYsv7Gl64Oxfh2koK26EJq2HqZa+zeRngaL6778iItbS5a395XMzVEAdu0CoDkrmPKZtVpvDOQ4WvY25KxU6wgg3tcXUY8KL6ql3rRVT9W8760y424VrUqVQTJnnmTZOv3H2JWOIl7tJvaraZu6raShlt0oQWsoMFMwZk6n8NC/D4qo65uiUqDRXk9TGdP4yPuNknHbAo+Kd8NgkGtpaapGnROT/AKbWclu9M6EkeoKCTmdA7qNfQqx98Xv6hctbeaEhz1enceCSfaKUkx8rMThdjy/MQVqWr2qmltP0xeQlLSUwvunQjlBO/wCFnJQqF7YRZKiT63QpB/3m9fvtEXB+rLdddCFqMhC6dXmAoR8hYUv9Bsg+YflThH6pYeh/ClniTjheDropaxqkFS9VVKaZsKVlSkkE5jGpGnzFj2BsVuYquU3g/SIp3W3lMqQhZUNAkgz/AL1mNx+aWvBbFakEmjvFh0lPQkp+9Qsa4NvqFFeFMDol5Dm38wj/AKbKpbkbG4c7g/ZRhGYvB5WUp5tJJgWaD/EG6mrxqaBAqnV0zhbWUNjLmAkgSRMbWdWYlo+VoHraZpWI8QXW6ohDlQ8hcEpIDgmQeWhsKoJjpXys3bb97KcQDpmxu2P+FLl14zuutqE0peW24vRIdbKJPSdvnZwFYgkjYWptga5MUcOuKNXhC8MSXneV1V1CirovXX1OlohRBAKjpsqYiRl6Ta113166+5G3s5Lq2ilRj+ICCfiLAhdJxBFLbUAg+RRZGsDC5nI2QtBim4rxvB26qW+7verWZz0zdShTqY3lAMj4WUKioSykLV15m1P+ENcbs4q3U84YUt9ynVP+NC0ffFpW9J+sqk4GpW6d1xLTtXkdCVQFdxRTPUSLaXdCap1MDsqgmHAE3VTQKlCmi6NR52xp1LqSocjztSHBHF298J4GxHhlu8HGDUsioonkKOZp0KSFhJPs5kbbQU6b2kL0TsXXtX35f103tfFdWBVMh5hNRVLdSgIcIOXMSBIWnbpau6LLEJfOyKJLyZLcrqzi3ENpzKMCwYfaV3QseVos488W3uGuH2lXZSt1F5VoV2Ad/u2wClJWoSCYKhA0nrpatN1+krxfob7pUXlfgfRWoDjDdRdzaWXo3CSlIJGm4V4WC4ZCARujtOa5HJXqJ3iwcmY1+Nmxw1x3ScQsI0mJKdsNOOZmqhnNPZPJ9pM8xsR4KFnOZJ0982YG4upWI0K0TpYBaQKhtXXSw5sDUEwhZ/hVPu+hZiEhoULyg7W2OpFsgeVtDQWkm3Wz+FtGPxtsxbRjysklzy099sAG8aWw+0BbfLeLOAoE3QT6M6FIOx08bJd2LUujQD7TZLavAgxZYKQQfvsjXelbVZXU5QUgPB1JPRQn8LORqCkDcI3zsKxTJrWKilISSUZ0Trqk6WDI2jnbpisp6BZq6peRlCFZ1QTpHIDU68hqdhNq9ZE2SFzX7WSY4tddqhniRhbD143k7dV9XVR16ajI96u8hKikAZS7CtgkzqNdY52jOvwLetY0wrCGOr0u5EuEu1bKa9socVmJbS6QpCU6BOVaZAG8ZrTPWXRWXled84irxUUZqXQxT077Ce0ZaQhJQTvBJVmyzAJJUCRAZ2Kqegw/hWqqL5YcNG42pIVUBZ9ZUUmEFQEkqOm0axoLcuGxTh0j2hreS0XPdlGtyojvPC4xxfT12VN90JumgQH2001OA7VMKAGd5/MokLV2gGUoTAWQkQCqrvEa5r6fxRe9z4rqTddxpqVUFJV1N6rWt2hSv7HMlJU6lgZEqg5UGCRBM2t7dFNdd0XCqrvGqup29rzfFdXBx1CWqZRAgBH+AAJSk8xPIm1M/S4wPi7EGPX2MMvu3hddV6rXNGnqszBdUgNrWsiQpcok7kZwAABFhU9JTMcA0ht9z0VeWRwZnOqkWixXh2moqemp+M16sNNNIQhpm63ltoSAAEpUTKkgaA8xbLVcpMAY6apWWximlZyNpT2YVIRA9nQxptpbLVjglJf/AFP7f4Qu/SfKvoEvBMPTE62AQkAzEWjJXpNcMa5wFKr2ZkxDlIlUHxyLNjjHH7hY4QF37UNHnnu6o0+CDb03vtOfjCyjSTjXKVIoEamTboJ5RttZmMcZeGFSQG8YUqSqQA4063/4kiyizxI4ev8AsY2uQHoqtbSfmbSFVC7Z49VAwSjdp9E4x1tmnI2TKbFmE6whNJie6HyZIDda0o/JVlFqppXxLNS05PNCwfuNiCVjtio8N43C7CRz2tuBprbYBPl4W3GmvOz3B2UcpXIE6xbUbQPK3cC2FMiPDeyulYriATG1thM7i3Uaz1thERAsk65jprp8bZljXrboCOdsg8gPGySXOU7wbaIFhBPO2ojU2SS4CegtoDwsJGvnbUDysklxEn5b21APK3ZTrpbRGnKySXOWbajmBpbsiNbaAOmlkdklxHO248LbAJNsIFo3SWoi2iARFtkAWwxOhs4KSDLSCdtRbSmUkju2FjlbCOUWYpIEspIjLpbQp0o1Fh9+Vtc7MkDZAAQ9BH8H42F/G2iPtkx/Kfwt0Brz1sk5QD1Ml9JDmo8rF03e22kpSjQ6WUIEW1AsrJrpLRc9Oy8XktpCj0FjYYATE2NRO3lbRAA2swFtlIuJ3TJxXw0wrishd93FR1ixsp5lKiPIkfU2Awvwxw3hVR/c1z0tGknUMtJRPwGtn5CToQTbMoBmPjaDow7VFZI5osi7LJQkAJ2sk4uwtdeL7iqLivmlD1LUxnTJGxkajxFl4pk6aWzLB8D4WkPCLKGl7lMvAnD+5MCXebruKhRTsFwuqAklSjzJOpMAD3CztLYiCPfYfKBy3thSOQ3tBtwk52bkoP4j8EbwvG/nsW4HxG9cl5VA/wBZR2QdYfPUoMa9T74kkkrgnh5julvIO4sxGzWttnut09IGQT1UZJPkIFp4LSTopMjxtx6q0NQkWhM0zm7iiwzCEWCTW6E+qKZP8Scutq13hcXHG4apygu+5sP1dEyvKypanc5bB0JMgT8tedrUdkBpFi713svd5SATztKQufHw9LeaeKRrHZlBPCwcTqjFTCsS4fu+76BlKluONVClLWYISkJ23M67AdbT62DA3NizV2MsrzIQPdY4hJToLAiZwxZFkfxDdbIlO/haAOMd9U2EcWLrKu7rxdRXMIUFUtKp0ApEEqI0Gw8bWAKZG1k687npLzATUMpXG0ixi48NzLXuLIbG2kD77Kq1xYhwhe2I0Vt23fXqvh1j1QRQvJPZ580KMZRrzVEdetlcM0S6O56elM/Zog+Z1PzNuqfCN3MO9o3SoB8BZdYpEtJygaARanDE5kmd3TrdXJpmvZlCrJiK/KfAfEm+Lovl31RqufF40LzhhtzOdRPLvSP90+EgYgrWMQ4rZxGipQmKFNGptKgvOrOVAgj3iPH4z9jbhxhnHFMinxBdNPWdlPZqWnvInfKrcfGzQw5wAwPhi8mryu25kIfa9hS1rXk8RmJg2PVylxc6O4LhY7WKHTNaAA+xynTqnhhKlfpcM0FLUAhxthIUFbp8D5bWhfBxF14qdpFLCfVL8eaAHJPbEfibWJYpQ0yG4G0Wjp7gxci8cOYvcerlFx71j1YVKksdrM5igb6666WiXAU3Cd5fhJo/X4g8/wAo1xjpPXeHF8ICdWmkvjwyLSr8LNThhf1Ldb7q6qpbbbqWUKCnCEpkHqT4m0r3/cjV93FW3O7KU1lOthRGpGZJE2gB7gpxMoV9lQcQAWkJhKHLuQrbaDM/0s1TK3gsF9QSefP6JU8ZMjjyIVh7svmgvRtSaSupnlJElLbqVEDqQDaHcVA0nEq8Wtk1TTL4HIwkJJ+Is4OEeC8T4bdqa7FGIE3g+62GW0N0qWUITIJUYJJJgdAIOmuiRxiw1jI4go8R4Ru6krf9V9XeQ+8pEQokEQDO/wArMyVslNI19hcH/jdIxFszC3kU3cfVDC8f4IrGWzndbrGXCOY7JJGvSc1piwmvNc6G9SEkgSPGbQncrHE6tvBinvXB9EyAYNQasKDaecAJJMwNLTzcF3+pULbIT7KdfE8zbPjnfPUMcRYNbbe/O6tuiEUThe9zdVKeAw1xXVJyJor/AFK6AJD2Ye6Dad+P93/vDhpWPBoLVRvMVAA6Zwk/JRtFPF/B9/ucTqtu57irqpV4OtvsraYJaBKACSvYQpOsnTe0746uypvXh1fNGtuX3bucISnX7QIkAddRbojOzv7ZQd8t/QXWS2NwpTGR1Xn3ftYbuvWgoH0j1a8TUUi9IhyMyNtpCVj3i0w+i/WLu7iNR0jjgJqaR+mJ2JypChPn2YPvs06Lh5WcSlXpS3RSrefoGFVlK8EnK3UoUCElWwKgFp95PKx3g5UVl3cSsOOMsrU6qtbbcZSk50BRKVynlAKiegtWqWBocGciiwuNwXcwpY9LiiccauioB7qqd9M/ylK21fdaIb1fwtfPDvCF2VV41VLfNKmpcDrCEHIgPKRBKtJICYAk6eU2K9Jy53KzA9NeyEHLQVMOq/lbcTln/iy/G1QLzuK4b/w4u471oSa2kqUPUbwBKS2VpLjZ5xAJ6mTtZ5WtLYpfIi/Q/wAKlGSDIzzB+quT6NNRhZjCNTc2HReRVTPJeq3K0pzuuOJgKGQwBCIgAbe+0xe61a/RfvamVf183fTkBLtKhYAEAdm4Ux7s5FrJjQcrZcTrt1V6UAHRYSOf32AqhmZUNOXwsMTrbhYlBSIkgxYiDm1WNrK2kK6pBsII8D42BpyFMAARqR89LCwroRZwE91s21Bjb326A0g20Tl2tKyiSuYBVMW3sNLb3E21ykDezgXTFcxqYFkp0djfAg919nXzSfyNlcmdhZKvRLiKqifQUhIeKHJmcqknb3gWk4aKLSbowRAg2T2Ev1N8+tLATSUKkhkA+29rnUfBIISPHP0BsokHlbjKRyG8/Ow5Yw8C+ydr7LvGdEhTYVplLfTSR9C0J4wu2mW5QU79Q3UJ7cut0TErqKp5CcyG0iQIEZlE9wAakCbSvjS9MVLpWmLlwgK5KmQfWX69DDYVqCnQKV/CDtztFmOafElx3JeN6Ul3q9ZNGhVT2KF1JLpABbQoABDQWUnUAGFLOaABxWNYkyJggidlcfLofNaFMwubZ2qiS/KGpvrGBoMc381d131LZ9ToWVkBLTaAVl6pAISDmzFKMkCMyyBZIviguvEpv/1LBN03w4puoU9elJkcTd4SUpbCUalS1DM4ZDclQSP5rOPGF13jeDbNNft5LoLu7MdpSMinFStRITlSjOpEbqzJUSmQIJEGHMYUzhvN5q7OJCaO4qVCHEUdNUFpa3FJKjCSMzhSkI1JWSqAkTojh6yaorMzJTa3Mixv9v8A6V4MjA8IToprs4bMUzTD90XitxtCULUq87vSVKAgkjszHlJ8zbLNym9H+jrKZqrreJ1bTVD6EuOslxxRbWRKkyNDBJE+Fstl54v98+hUO7u+T8owmsfZcICjERqrbX7j+FlmivntIC3O8AJJ+ug+tbN2pCu1zoVKo1PLTwsNTKSAEkqSQDsTI57+/wCFvYHAOUmnVPRitQ6MyVGJ1P19bWOtvQkGTJgmPrzs0GKp1pRQZgSnQa6T+dlenr3EaFRKSPamD8vo2rubbZGCWezpVqIVTNEHmUja3aAw2oFDLaVcoSBr9fdYo1VJJAIzDaNtrDpey+zz3ObUifnYWoU7ApTp71raYZqevqmRMy08pPwIP1NlBjGmLKYhLGKb4T4CudEdf4rIGaUgKBMb89fzsIFKju5Y9qSdrSEjxsVHhsO4Top+I+Pqf2MXXvOvt1a1j5k2PU/GDiSyMreK6k9M7TS+n8yTNmSFEg6nfeZ/p+tuiSQCFGDMg8yenW0xUzN2cfVRMER3aPRSIzxz4jte1fLLw09uja/6Ug2UGfSEx63o43dLpEHv0yh80rFouJVIg65hEb23mUCM3MQCN4ixBXVLdnlCNHA74QphpvSMxUEkVFx3S5z7gcQfms2U2PSKvAEJqcJUyidJRXKTr0jIbQe0qFZjoJ5jb6+uVj4JTAKdY11+tbOcVq2bPSGG0zvhU3s+kRSE/wCsYTeRoD3K0K+9A+hY416QmG5+1uC9kx/KWlf9YtA6c2TMI0gAwNfgbdQRHIRr4fXSzjG6tu7vwonCKY8lYFnj3gtyO0ob4bkfxMNn7nDY6zxs4fOiV3jVs/56Nw/+EG1cBJiDOoI0ts94QR4b72KMeqh0QzgtPyurMt8W+HjsD+0aEE8nKd5B+aLHWuImBX9G8V3dP+J4I/8AFFqsBR1BXA315nmLYFZQf8R57WK3tDPzaEM4HEdnFWzaxdhN7RjE90r/AMta2fxsbavK7KkA095Uzs80PJI+RtUMx3cyhodlDy+Vs7MAQAhR2ggRYg7RP+JigcCbyerigpUJBB56W3lPIfG1O0uraMoWtvrCog2Ms3zfFPBp73rG/wCXK8tMHyBsQdohzZ+UP2EeT/wrdFJki2ssbDQ2qk3jHFzRQGsTXwIjatcjb/N9eVj7XEnHTMZMVV2n86grT/eB1sQdoYubCoHA5OTgrP5TMHW2ik7jytWxri9xBZ0OIe0H+KmZJ/8ABY61xux417VRQPf/AElKNf8AhIixW4/THcFCOCVA2IVh8p6WwDTa0Cs8fMYJGZ277ncA/wDUup/95Y4x6Qd8pI7fDNEsEEAoqFIn4hVjNxqkPP8ACGcIqhyU0uJPbNmZkED5W7KVbHpaF1ekM+p1n/5HNqgkri8iI0j/ALrrZRZ9IO6yQKvDFWgk69lUoWB8QmxBi9GfjQ3YZVN3apXiNbaA1tGzHH3CTpAdui+W5P8A3bKh0/7yx9njfgJ4AuVFcwSf46RR/wDDNitxKlds8IZoKkfAU+oi2Qf6WaDXGDh04Qn+0BQf8dG+n55LHGOJWAqgAIxVQCdB2iij/wAQFiCsp3bPHqhmlnG7D6JxgTuLYNwPwsks4ywe+qGcV3QszsK5v/4rHWr4uiogsXvROf5KhB+42IJozs4JcJ7dwUZG2otgTvbG1trEtuJUOqVA27CSBt8rIuB2KgWlcbxOtsIn87dZfD5W3AJ20s11GxQcTbevW3URrbeXpZJWXBAO41trKI1sIQR/W2o1328bLVTa0LkjlbI6i3RTbcGLRRRouI8LbCSNNyLdAco87YBzsk60ExuOW1ugBGmlsANuh1skwN1wUz5m2ZU9Ld76RbNedklsuABO9uQgTO9hIjS2tzZrJ1yUAjTnYJdM2r2kjXwsYgRobaMTaJATglAIpkI2AFtPU6HRC0CPHWxkjwtnKw8o2RLlJwuxgKzBsAjnFjDbSUaAaWHO1syjff8AG0crW7J85O6I1F3sPvJWpsFSdASNrCGnT2ZbIBB0jlYwBOgGk20oa6WiGAG6mHl26Rqa4KOjUv1ZhCAskmBGtiNLg+56C83Lzortp2ah0krWhsBSid5Is5YHObai0DE3opiRyTbxuqlvOgeu+tpkP076FNutuJCkqSRqCDuLQ7W+jNgYVxfpKWoaZJJ7BNQvswT0EyI5a6WnIwExAsGUCdQLJ7Mzct0mOyG9lG/DHg/hbhoHBcN3lC1jJ2jjilqCJkgEnQE6mNzvaQ9hFhCgDby2topgaHSzMjyCyd7y/dcGYtuARoLbgbC2xIG+tiAIaLUWhdb/AJXDHlt94NjPu+VizJy1rySPaAUPL+pNjOvOyGyc7rUSN9bZOu9tnTW2jyjz2s6ZYZ5W142yYNsnraTQouWaEWIXskqpFrTuhQcHuM2PconnYB9Hasrb2zpKfiLEy3CgTlIQRhUFPMSLckachNuKRZXSNFUzlAPmLdkEmAJ8rROyfmjlb2rmE6j1YgPtZktEicqtwdxtPUWiDGtNVVl206KW8lO0SaxmrdbZ7M+uBvcBS1ACeomTAkWlW7EOVtFUi8EAQA81TTIQobFRjVW2mw8YksDGLLLVU4cpzNkq0EyI+W/ytyNfhTquodODYGw213utCmmbG0sOvNRVju6E4iv25PW7iZ9RKFO1hL6G3oSUltpboClIaJCtG5Ku6CUpJUWTxFevNi6K697n4YYPRW3fSuKo6x55Lwpg02rs1hosCYCQQiYmJjUhzXzidiVUtwUK7zq0lSnmqNSUttLMqV2jqiEJ10iSrX2d7IqaavfadqcSvFKCoBNFQjMFZoHZqKhLkwRIyCCZGmYCdgVPw+NOcz7eVgkKt2bIwWCqSjj5d7aEt1fFHE5fSAHSze7oQVjfLodJ21Olss8qnhE05UurquO2D6J5S1FymRR3clLCp1QAgZAEnSE93TTS2W5rg4d/D/8Ayj3l6/z1T1eYUHFlTQKQqdQZ1t2w0AlJicumgH0bHq5tindAfdS0snKAo5ZPQdTbmmCFpBDyPNKwR7vr426dsheLhaXDDTqtttZRGUq015TznlH6+NhkII1CZI5kdBtHy+tRUNACEKhQgDKI5bfG3SWURKCBz5QAOXyn32fMeafRdsOrZhCgNwe7Ikz4an63scp34y50E6bxr9b2KJCSQG1BR035ba8uVttZoUEkpn2goROvl9fG0NSkldqqMyTOUaAySN+XusOHErSQVKGUCTvr9ac7IyXVg5pAB1E6jx2saS64mJJMwZj5f152ikErar7ySDpmnUx+e/L+uABI7yyRzER5WJofaJhJ19oGdrGEOEApHI+IPy91olSFkKNUnTNrI393lbsJGbvEnXQE9frnbhKkFWRSpJOutuoA2KSN9Oenj5fKzXS3XTcEhKVHTl9fX3WUmhrMnXcT9fQslpISuQIOhk9LKbIUUAAEc42/rYL3IzAhQOoAkb7gR02+vkISmII02idd+mxtwAAkETqRoPv+uttaD2UgEgASJ90WDdFXahGgIAAiR5W4zDUZRJB0Enl9fO2ylSBBOp11525WmJjKQkHTcz+VkHJrBYDmJAOg5qGk+62JSQhSAkE+HMG2kJJASVQDvpvbsZMw0zTuBaWZINCxKlRpA7pIO/1+ttgaKATMQBB05/X9bbjLJUdY0OWOmnytzMgkCN/hZsxSsF1KJzK5aGfmR9b25I/mA1122t0QZAlRJ130H1FtEmAZJBMz9ctrK6VliU6aiYMEA8o/rbYBUNADpzHL6i29IgQDy8vr7ra0UkhEQI25fX10srpZQszCQQZ06WxWaRm1zawZtmWMpKtQZJtikiTCfa05QLK6kGrFFM9IOk24KzBPdGmwkC3ZUCsAkQBsNR/T63sErVYTOm6tBr9aWldRLViElR0SASdQTbshMQSdNBPPS2IkLBBG0CTrbQSQMo8xH3zZXUcq6BSqZTuAIOh62wqCSTuROv3fn8bB5SATmk767eHjbM0kHLJ3GtlcqWVCBYJg6lPe+fP6/TEqnULO0z9fW9uQRIO8aax9dfoW5zEJ0ImAd+XSzJWCFzARlnXTnv8AU22EoUkhQBM7lPI2CzJTosHUaEk7aT9edsUojed40/P6/N7lItBQg7JKcyEAEHSBHXWxpm87wYEsXjUsj/1by0/cd7EEHXukzNu4TlKTpG3P6/WziRzdio8Np3CV28WYnp1DscS3sgdPX3o/8VjzPETHNPOTFd4nnCnQsj/imzbGpzATvtvr/S21AJGUp8ztaYqpW7OPqoOponaloTxZ4t8QWpBxGpR6Lp2DH/J+NjzXGnHjUZq6je1g9pSJE/8ADHjZhBZ1GoI3gW1miSspPU+E2IK+pGzz6oZoqc7sHopKa4642RAcobncJ5erOA//AITwPKxxnj7fyT9vcd3L/wAinEfeTaKUqjcmddYH1426zqQNTpzmJ/rYgxOrbs8qJw6mPwKYUekC+NHsKtkTEpriP/dmx6n4+3WrR/DtUjWPs30r+8C0I51d2ZHIjeT0torOsK0iTp93xtMYxVj4vwoeyqV3wqfWuOuFF6O3bezXU9m2Y/57HmeM+B3P7yorGZ/npyf/AAk2rulzU6AjlpGvS3ZdSpBHOSJ6+FiDHKpu9j9lE4NTHqrJM8WsAPaC/cp2IVTPDX/hix5riDgl7RGJaIHfvrKP/EBar3aHMCCDOxjf7rCIdIGYaggxztMdoKgbtCH7Cg5OKtM3jDCTh7mKbqM7f643+djbN83NUa097Ubv+R9B/G1UA8UjMeQ38Ldl4HURAnf8eXWxG9opObB6qBwFnJ5VtkKbWJQtKh/hM23lIExapCH0NglshMExACfr9bGmr7vOmEMXpVtRqMj6xHwNijtGOcf5UDgJ5P8AwrXR0FtAR0tWCmxtihkkJxHeXd1M1Kz+NlFniRjJtP2eIakEie+Qr/xA2mO0UR3YUM4FKNnBWOgzbIJ1mbV+p+LGNkGF34XP8zDX35bHW+MuLmwM7lG4T/M0PjpGlpjH6Y7gqJwaoGxCnQjS2iNBFoXa424hTo7Q3ev/AHFj7lfU2OtccK0D7e42FRuUOFI+c2K3HKR3NDOE1I5KWcvUW0RPMzytGLPG5lyO0w/5xUz/ANFjbXGi5lf3t0VYMx3FJV98WkMXoz8f7oZwyqb8KkAjTS2iJMmzHTxiworRxmvRPVpJ/wCqxpHFXBbioNe63/nYV+E2MMQpXbPCh3OoZuwp2Eac4to/UWbieI2C3YKL7QOWrLgj/lsOjHWEHTKcQ0Y6Z15fvi0hVwHZ49U3d5Ru0+iWSP6W0Rynaya3i3C1QYZxHdiyeSaxske6bG27yu5/RivpXOmV5J+42IJY3bEKBjeNwhjAi2trYClWqSFeRm2bH7jYgIPNRsRuizgy1rawfbSUx8TYyI5WKVi+zXTrzR9oEz9/3WNTNnGqYmyyR10trz52w2yZ52mBZQJutTrNs2E2yTrblci0gNVG9ltREwNR5W4JGwBtsmRpztogDcx52Khk3RCmkds0f4HVD3HX8bCmBztwRkrXdNFpSofdYRPZpUVrbKpjYkbG1eZz2x5ohmPS9lMAX1QlGspfE7KkHlvaMsV1uH8U1T9Ca9qoDCRT113heVaVAyc5BCikiIiARvIOkjBlpb6qjOpDqiCDnJJ126QLQnxi9G+7uJd5pxPhrFVRcN9pUFmraSpbShBSMpQpJQqYJjNMDQb283xTtTWUVSGVlOWMO/O/Qgj8rQZC0tzMNykl+gum7gKO7aJuhokLKGmGx3GUcthsAdTHxtEF+4ixtdV5uX41w0U9eDLgoEqTe4qqdpAJU6rsktoUnlmc7ydU94JylU23fw2xPgO54vjEVbjV5tta0gppqNU5MoRncKirvEwYHtTrlSmyFT3PfF9X+HXcMLud2jQ2F0nZ0ricsAlTjqUaoEQMqk90pISCqLZtT2tbi8nAo3ZWMF9Tv9lYipQwZ37lV3q8OYK9ae9evhwVPaK7bMaGc896YqDznnbLT07gThE46td5YdxG7VqUS+5+5anvuT3joobmeQ8hbLcycQk+f8q3wWdFBH7SN91PCzC6W82ZV+hRCZB0pXQNj1Vt42p9wmerH3qY+sPFJeQrL2qj/tIMz59LW3/aUOBOA8I00qld7Oq0J2SwP/i+dqqcHWypqgfcKSXX946rBgdfZ1529c7INDsNjzdT+6r9o3EVr7dB+ynBg1aSnsqmoSsHQJdIkTqLGU1F7wkKvOrmeTy4A6an8bD0rUkhPeBkgAagdT01H1yUUspCMypkDOQdQI5z9bW7gwxn4QuWbM/kUVRfGJGtr8rjJghSyoRr8o+8WHRiLFaQQL3dzDTvoSdtuVh/VW0d0IG+mms+X199hAwEk6SToBMAkeJ+tB7hmlhPwD0UxUzN2cVprFWKVGDXIWoEe20mT8o5WNN40xQ0hUrpVmQYLZ0PlPXz+63Ip4ICkknXUq1P0frnbXqiFRAjkIAMD+htHuFM7dgS79UDZxRxviJf7aIXS0SiNBAUP+rpOsdLG2eJt7ApR+6WN5hLhEn4GyP6ohw5VpB2khMdN45W36kjQGNDsdvl7vjZjhVI7dgUhiVSNnJyI4p1MlK7jBlO4fiOp1TYdPFunQftLkqJTz7RJgz7vocrNY0pSk6k6+Ex9bTYF6hbAkDluFbRH6WF7Eo3fCpDF6ofElq8/SMwlctYilvS5b6+0QHQppppxIExBlYM/XPR30PGLC9VStP+r3kgOoCk52U7ETOivK1UOLCRT4hpwkZZpUlJA55lDnqOlpXw/SB26KLMASqmbJnSZSOlq3/T1G9x0PqrIxyqY0bKZE8W8JjQOVgJM606uv4mLGWeKODXJi8XUkclUzgPxy+fw6Wif92JUAlCCDMzGp167WEF2AiASdYT3Znf9LRPZijOxPqnHaGp5gKXEcQ8HPwEX8wgnUhSSnUyOYsYRjfCjxTkxFQ6TMvJAHxOp3+jaGTdiUHIlESNp5bW4/dzcglIAUIOn42Cey0HJ5RG9o5ubQpyZxJh9xOZu+6JUDRQqEH7vxsbReF3KJyV7BJI2dST4c/D5WgIXQlZClogA8+Rg87JOIbjbN2jMhKe8IkjTQ76eVgP7LMtdsn4VhnaJ5OrPyrNJcbcTAWlQB3GutukwSkAwU6mOnl7rUTNzOoZxC0y842S52iileUwUDXQyPMW4oziK5GbnqF4ivZoq7R10Jr3QJUM2ozbDkDtFqR7OO5P/CsjHm82flXw7oKVE5Y5D7/G2ZSVnQCNByg/DwtQ+6cRcSW27xWrHGJEKfp/WWki9nzkBUqAJXA0jbw6WNU2OeLlHhWorE8Rb+VVracfC3atThQYVCQFzoJGnnaH/Tkx2cFMY9FzaVeiDIVl0gbGdPxtrJJGbXzHyNvPZv0kuN93qpEDiHUu/ZCUuUtOrOqSCTLcjXoeVnfeXpLcX2K1qhuzEKFpQ60h5bt3sqzZtx3UDl+O9gu7P1IOhCMMcp7agq7ZICYUox8PKLZlG6c0zp+Fqtscf+KTKJeXdLmm7lEtPjqErHSyW16VnFf94rof7PYaezVCkolh9spaBMrVDx5+7Q2G/AatvT1RI8bpT1Vt/AaxAIJIPX8rBN5ZnUk6zHW1c6v0mcXtU3af2XutYQMy4fcTmEbc4nTXW3OHfSxvW9aJVVWcPGachzIUN3sVcgebQ8R7rQOC1bfh/KmMXpHa5vwrH5SU7wIg68uluAFE66j2o52gCg9Lahr7zcov9H9elticzzde2sSOQlKZ+Njd1elphCurXrvqMI4hZcpwVKUkMLQqI2PaAn4WGcLqx8Cn7SpfnU7KkQE6dJGptoyQAJTsY5D32iVr0mMCLJbduq/mjMwaVpQ+Tp2/Gwa/Sp4QMPFirvC82SlWUlV2uKAOuncmfjaD6CpZuwqYrqZ2zwpdMqJSCRzkdYtpS1yZ0E6D7/rxtFdB6T/BO8nwyxix4Oq1SF3XVjlrqW4sof8AlBcGkDI9xCu5pQMw8lxv3d5I6iwzSVAFyw+iIKqA/GPVSHDgUQkanSPrfzt0ToCUkwdI62ZtLxk4UVlMKtniJh4spJClqr0JSI3kqgfXusoUfETh9XJD1Fjq4Hkq0Cm7yZXPl3rDMMo3afREEsR2cE4QtciVAd4zrEfWmvhbtBgyrQRvGvw87JTeJMOVaiilv67XiDJSirbUR8D9a2PNVDDolp9Czr7KgZ1/pYZjf0Uw5pO6MIUJjkByj5/E/O2AkaDSJ5zHOwYWFHMoqgHkeWn18LbzqSlRHvnX3wdrRylEuF2VBSgRruRH10toqC+6IgxB2A/KwfaFQEjz0PhOn1vbAtJMKEx3QFcvD5xZWUbC6FCiU59QowRPKLdQCSEg/pv9edggrMnvE67SLaUsAZSozEAcjvaJKmuwYICY1OoHwttJEzqIgSRM2DU5IOswdTER7vM2wuc8uhEifrxs6VkMleVZIUQRsR99ulOKyxlkRAnXnzjyFixWCISFaxuRB6W0FAR3Vd4bbkT9CyToyFKkwYMx1k6c+dug6QmZGus7/W1ipdSUwhwCOXIfUW2pxRJJ0O4gaxy++zWCSNKcIUcsmddtBytovpKjBjoQTOuv4WJl5RkgzsRJ6iwangYO2mYg/n7trSDVEmyP9uToTqfZ12130sEqpHsyRrIJHlNiaqlJ0JBCvjFgXXEpEkiZJiY08PfZw3VRLilNqpBVMgbETNjKKrUCeUab9bICKgSJJ3MTsff8LG0VEphIJJ1B8Pj7rMWJw5KwqjKjm55o5/hbaakwVZ9dBqoiPGyUupKUyVagxG2vT8LcqqFgSBqNJmPo/XiW4afNZLPrJjRZnWTGtsTVd3KIEDUn+vnZFFWUJ3X+m5+vDlbBWLygKJEHy1suGlnS6mqBmDHgJ+H11sIKyEgKOaTAmZ/TnZvpvA6lWyTuDpYRNcEggEkHQTp9fpaPCTh10uOVatSTqd4Jg2Lmr725lMbaT9RZLXWAolZ1Gm02LqrkhRVMfAg6aizhlki5ORutGxWMp8dvz/pZTo6sFtcFJnfys0KatzDTNodROtlGgqVnOkq0KfcRaBbZLdLpqydDG068tpsEp5gnVttUn+UHz+6yaag6EqV4gjQ2CXXJQM4OaRKoO4+hZ81gllBSsmoabOdpCUET7Jgz7udtP33VtKDSbyvELUe41TVTqVK2/lULI1JU1d6KPqaQloEzUESmP8P8x+X3Wcd03PTU+wzuq0W6rVa4Okn8LMJJHGzSmLY2i7gnDgdnEVbfTNfeV/3mplkLy0i61x1pRUIClhRMkSYjTztNaVZkg9QCLRth1k07YUBEQR10tIVAsqpW9ZKQU/AxPyt22FROjhAcbkrk8ReJJbhGDrbRVp525nTe2iQdd7a4aLLOJXRX15W0VTrtbmfG2ieVnAsok3XW1smIOmh2tzOsb+VsJ5mzlocLFMmRV3RjK5L2L1zXh69d7qlKbpnSntGZM5Ez7SZmNRAPQAErXY9vy43xTX1hvZIKnUdqgBJMaDIoEyU/xRvrMAviukobcSfYWknlodDbeYwRy3g7W4ebsxiFJK6TC6xzAfhd4mj6X2V1tVG4ASsv57KOF8fMI0rymaihqUAJIU4mHFZp0AToIy7nltrvZZf4/wDCu80UbbN8qYeeb7qC0SJy5ssolJVoRCSTNnU7hnDl407tRfVyUFQh320rpkFThG0kjxPxNilbwywFW3EXWMMUzKHKz1tbbRyDtu+kKJTEwHFGOsdLYsrMcjjk45ZIG3Bdb8bIzXQZ25QQmO5xF4fioXSoxdS17j7SnBSsKCh2YgKK1L7oyg7CI3M72MUmELqxAh69KW6WmrwdUGVoDhabyIMJBMFKgmVQcp1UYiZshYj4HcN27s7Wsqa+7RSNoWqqbqsqWw3uvvEhOkydI5QNLMa+cLYyuXDVy1XC2/7/AE0ja6pp1uoS409eXbrCWsichyjKlULzIIkuSmc1vKMQpe8zNe1uRx0NtB+FsZ3NYU4rxuTCtDeFVRHihTUBp3ltGkRWUiE0+VRHZhJZJSExEE6RbLEWeFHEBTKFJuTBNOCkEMvUNM843p7KnOz76hsVczrbLZ/cJvmP4S4jlVP9pNUpbuHA7BOUqrq1QBjk0z+dq28I6eWrt7MSDVg6bATBB9yp8vdawH7S99xDGAWCrRTt5LO2kJpgNDrz5HwtBPB1A7G68xzpU6lUE+z3x9Tb6R7JtLcNi+/7rP7Q61sn0H7KeKenSIUsSSo6DkdOtjrbYJC9DmICoET0tlM2pwDMgEkHNInT85mxxLYKc5B08OfXT+nhbuFygRfIrMogaROkR9fl8BEtNgxlJA32P1z+Fhy1kMqGmggE727CTIzJlKo5x0G5sk65CJMGTCgY3M68vra3RbkBAUZIzHymJ+vnbaUFaQsDQp1HP3WFSMqJ3BkwTt5zZDRIhFyjvQgRBInqT08bYWwO7Mge4T+n52M5MyjOm5nr9aWEbQQsyCDMGRyHM2kHFRyhFcpSc0kkecxufrzsAtrKMxIRzKuWn32PKbCiEqSCmZ328PDnYFxAyBKhA9k676nw8voaECE4KC+MrahiOiISdaQZknYnOrY7c/6c5cw22n9y0BKSqaVklQH+FM/1tFfGhJRiChJghVJ/ESNlqJJjfl8JtLeFU57luxYUrL6ozrm1IyD42gz3ipP9wJZbaISUlsTvH3/WlhlNKIyJza/ETH142xtGcAHQ/wAR2n4+7ysaaQYOg8OY+trFQwEWLQ7xJMnWIGn4HT77dFgwEwkbR4fp+djfZj2tI156T93LrztvsUrHsEDw+ulo3U7IoWUpJ7gzAyZEST9fdZMxAwTdxICNFpmY67n4ffZdDZXCkchtoeX52T8QJX+7nNUg5kED3jW0XHRTbuoZviA/eFIyAFVLzWYdEpRJPjsB7+Vhb2ok1d03YyhP2joQhJUdgpOp+HzNlKqpEO3teb24ap8gB5lSdfkPn4WTW+1rKq7lQezplMsmP5imTGmp0A+NqGbkrFlzeLTF33g72iE5fUuzSOW8DTz/AB5WLUNMheGkpyaBtQIPvH5Gyte9I1WX4xTqjKhBcWAYzZRIHjqQSPKyMmqcpbhbpaZMvvFSUgEEAZjJ6xFptfYprKBb3pHmK5Tysqe3ddcKdw0M+VKd5OieelpRormYo7rulpuCpTiHXFlMFaiJ931Fo6xE4WqhtDolP2kAkDXOSmeca9eZ87Sqwkqu26Vbn7E9f4Z002P4fCBsnA0TuqA1R0q6l0oyNpiTzPT5WRsN3emorK2ucT31wkEkbSZge762so1ua97yRdjTn2KCHH1Ae0BGnXpbq4WQ1X3gltEDtIUEnQROnhaRdchK1gha272QytISkqAMHmRGv3fGzWomE09xrZbQO1edUG4TrsNdeY39/us/H2dCHEgI1EnrtZGue7kVdT61ADFNDbKDBM/zH6+60HHVSaNLopdmHG7vo0oDaO1UQtxQT/F+m30DZKu66kJxLVKDY7wV3ADA202s+XWgDnI9rXaIjn91kFhnJiCoU6YzJUdTpGUbfP6FmcQLJgCUUvWjFJTKAZCVnRIVy8fAAeFmfUXE072knvkZySYzHX8/nZ53i4qodW842CkHKmfd18vqLFmbue9Vqatxak5Gzl19o/kPxsF7s5RWtyjVJFwYWpmKUVicgdfkkjUAdBOu/WzB4k0IbecEGcqenvEDXl89xaZLsSk3WxBJ0UN4jvfrtaMOKzYS444dEhgEmRroQJ58jItZAGSyCb3RTBN3ovLBbqlnMkLdSYSCSNT+nu52cd03MmiuVplLIzrnKnTNr0/LnvYjwkdQrDdWHmwsIqlSkEHTKkhPnr8xZ+XFTprX1VruVQbgITyST7unjy8bDIBUwSBdIl34KapmnHXG0l92C4qNhyA+P1AsYOHuzWClJSdwdR3hOvh8vyea2kkynNlnQxEdD+FuCyAdUcpiDKfeLTDG9FDO7qo9xFdl4Ju9Cma+rbhWYFD6kyNddCAN7GrmcxUzd1N2OJb3QpDeXMK51K9CRMhXn9buW/KUOXevKjREKk/l749/nYG6adJu5KSTCSrKOe8zPx+tLC4UZdqFMSvtoSiTN/cQWyezxtf4O4AvB7QjpKo6fKxhOPeKlMO7ja9lqTIBcqCsnror+ulj6aNKBJQoZRzgz0H3WTbwCQU09OkqcWSICeXTyGnwtF9PCfhHopiomGzj6oGt40ca2F+rXdjmpLuaO/TML0I3lTZ0++OQsoYp4u+krg/hg/xFXfrT1C1WNUYfqbqZCHXFSSlMIGgA1JO+mpmF7h1g7C1Vei3MXX01Q0zKO1VmHffXE5EkAgDqTy+QvpT4oocTcNqy4KKrpX6dllK2GaRRDFOlsQlOXROaNNtLUZKWHihjYx56K0yqmDC4vPqouub00+Nj1DSVFQbgdLxhal3eofxRPdcH3cjpaa7m4/cQ6ulpqiqoLmUt5tLpCWXEjUeKzHS1G7lX2V00iyUhbLyiNJKe94ef3WtzhRpp65aFUJI7IAGOmnP6/EU9BT5tGBWoK6e2ripCTx3xZlh24LscgScq3ESOe5Nkq/PSkvy4KhltfD9ipQ6gqzpvFTeUjSILZ/PXaxJujbKMyZlQjMNT8fePl7mvjS52nfVnSgfxJChGpOU/He1Y4bTk+6rPf57XDk6GPTGXH+s8M6hJ3IbvZKtCd9WR9T0saR6ZGHf/AJ7gS/mlRPdeZWNh1I18LRA7c7JQSGQMoBjw2Aj3jpvYBVxMKSW0tp7o1JOoPL36j4D3L2VTnl+VH2nUDmpuR6YeAlJHbYaxM1lmfsGD16PWNtel5woWtCapF9UxIMldATH/AAqMjpavb2HaYBLgZAVmgDLAEdAOcWKvYcplgpI12MEg+f3mzeyYPNP7UnHRWcR6VXBgpleJ6tnQBIcuupHjuEH4Hpbs+lBwOdjtOINKwSY+2pn2yNBvmQLVQqML07g6zCenWJJnx+jZo42wkGsNV1a0oyhtLglKYhKhMabxvEa++0m4LATuUxxidvIK9NL6QnBeoyqa4mXCkL0GasSjXc+15iy7S8YeGFWlKafiFhxcjSLzYk/80g/lbyjCQbt9sgtPkAQJOYfH+HytJdzYeYqqBiqzD7RlKylQkAESZ16CedmmwOJguHFPHjUh3aF6UMYzwvWpilxJdT0iU5K1pU9I71jib2onATT1jbo1EoWDO/IG3m8jB7Trax2bcKVuoazHL5WLVGEG0oIZSEagHQjWfDw/rztV9jt+b8I/tk/KvS1FQpau6k/zbG3K6gpCcxUDyMxH19b28z3LrvmnQUUF41dO4ASns6laSDsBIIj+lsums4puVCGbpxriRnZIKLzqEADzzac/CzDBbj3/AMKQxgbZPyvS4ViidVTqBBM/1sIKs5j3zoNzbznqsY8WsPIqUq4o4peebpi73r3dUEHKCAApWuubWOVvQv8AZ3XhT8WeCV4V3EBIvy97txA/Rqq6wZnuzNOw4hJOhgKWuPfajX0JoYeNe4urVPiLZn5Mtij5qEwUp0OsgdOmti5qiFHKdxsTv5bWss5wqwC6sFWHGgRp3XXBpHgqxKp4LcPHUn/sVxGsyiqdH/VbD76w7hXuICq/U1WlR9rSPM/W1lGmvBKVxmmQR3Ty52l1XAjA+cqbXejRIiE1UwPDMk2S724DXOKCoVdOIr4YqA2sslamVoC4MSOzBI20BGnjrYT62MorHC6jV683C6mnZK3HVaIQg5iT4R/TX4H6G4nqpSKm9FglGqaZJ+zH+b+Y+ER52Gum62LraCAkrd0Lji9VrPOSPHlZXZlXskkA9d7FAL9kYkN3RylaSAlISBAGg5eHh+ll27GSVhMjTWI2sl0jXf118J+tP0s4rsaOYCPr8OVtehpLuuVm1U9gnJdyQ20kZrPK63SqkE8o+YBsz6YykQIB3s5bidzNKTGwAHun8xbsKZuUWXLzuuSUqGB8banlOm9sVrvbU8totcVRbm2TG/W3JM79LamCRFkkuiRFtkwIm3OYDf3W0SOR2sRouoOKDqgVU6wD/DP425bV2gSQfaiwiwFAid50sVo1zTo1hSZBM7EGzSNuNEmmyUq+oKECjbVlSkArG0+FlC4FdvdlVTkAlBzDnuP0sgLJJJJ1O5ssYUdiqeZ3K2iY6kEfgbZbaFlPSmLfmfPqiGQl11H/ABGuqoqVJqQhD9Oy1nFKoEIUvcqXyUAIhJEAyTJCSmMLzvXE17KqLwF6OpZpHwXVKJyoQQo5UnRIVmLZyiSRrGk2mrHN2P19E7TuVK6OnAUlRa0dOh0kapBMajXe0d3Fd923vclbhp1TdGhNZTpT2RgqK1wkRtEiNAJ6zJt5RiBMMksnC8LXN3Gtr62C6mN+aJrk3W71xU42lxqVoUkKSrt3u8Dsf7s/fbLOV65eKSnVqoqinYpyollputShLaJ7qQk6pAECDtbLQ7/S/K3+fZQzu6/lUG/aVvKVeWBGEgZkNXkSOYzGmA+vC0QcIkZKe5pEAuiIGhAWB9eZtJH7Ri8k3jfOAXAsEO3bVVGUnRWdTUjwMDf87R/wUSU0tymA4hIlBI0iQdB7z7yT429I7Li2HQ38/wByqOPm9ZL/ADkp2aASNikpAO+qdY+jZQQlBg77FSYnTnYGmaTGpEATO342NMtpSoOJiRqe7rH1Nu0K5Zi67M5YkmAYB2nztgbymQM3Lp5/hYSFAaKEbQR7/fbbYUUTOidyTr4/cfrdlPZBoQgEbmTHiNNP6WES0kiQiSrQfXw/QW6y94AnbUbaH6Bt0EEJIBA0gac+k/W1m5p7LhSE54ClGFSRzjrtboJU4EqKozAiQNhpr8bCBoklKN9Y6+/4W6S0QrMCRGsxtvPO0m7qB8kGpvPBUIM6dB5xBPLf52LuAgArPdXvpPWx0tiCDlOgOmgGvv8ACxd9sEEpOkjnqfHxP1pYgKE7RQZxqQE3/QqOX/zQxzg51a+7r4Wl7CaALguskpBNIyCUjXRCefx+t4e45PKYxBQKUTrSGNZjvqncdPqdbTNg1Pa4ZupW00DJI13yDT7t9bQabuKd3uBLSQZKROWZAj4a/X42MBJI3WDI5azvrP1zsEhpSITm217p5yPzsZQjuyRmKRoDpE8vjH5WndIN6LBmTmkjWNfztsypMkH4cuk8v6+YFyiFKBSrcpgfKOv1pbZaK0yjRJEz9dPqLQzKeRcFtt2cqBr743EWTL/Qk3O4Z1BTBPPUH8N7KxCj9mEazESfLymCbJ+Ims91FRQUwtChBAHtb/daDz4bqQteyi90ttVV7uOQlKEImTzKNuvhYqpluguq7HFFKQt8PLOpjuk8t401t1eqKh6tr6OTlWQ+4QmYQhEhPxiw95ssv0V00IVlU/2YIgk5con4gx77Z4N1bLbJKo0OPXkqrqMx9aZcdyie6g+yPKBYvddChFG7eJbkraWhnT2UDUnwPP8ALmtXm63R3q4pxSAlqiCQBuCTAjnrG1ilKhCMLqTkDf2CiSeW/X6199pNOqidlX/GjRpn6YQFFRWVACSQSCPHcWk27Uo/svdlQ4AUtNNEZjrITt933+UZY9palm9Gal4q7J5aQwkz/dhIOY6fxEmz6w2upvDD7JGZNPQ07bZgxLmg5dPw8bO86aKLVI+H6FVPSl99RNRU/aKJBA6xB56j5W1c6CL5vBuSZykz0/rZQuxSnKKnehXebSYI8Jj77J5qEXbfNY45KR2IVrpJ7unhaDn2AUgCSu72ddqH27sphC1n7VSU6BM8/G3GG0LS1UtGe67IEb6Gxy46NSkLvCpb+2qSdDyG+lgsPqIfrG0ggBYJnzMHX6087QzG4KJawsj5aK+6ogAgaQTGo+o/OzdvbsKeqdQ2SpxyM5iMqQBp8RPw5WcNfVikpy4QS4rQCdvrrZqOrK3Vd2XHPdJ6WhLJyTxs5oJqkdrKhLSUwZIJCdBG+mll6spG6W7FtgiEtkeY+P177D3TdvqlPnWgdusBR5EeH3W6vQoNE8sEphByab9NPrn7nYLC5TO1KR7obP7oQlaZIKok+PT4j32jjiqwFtKKk589OSJ5mTy56H52kq6gP3bK1kgFYM777fO0XcWq0tVtBRM95VRmSokR3BqZ8ADr9G1kP8KCW3KT+EKXqluupW5Qy26HVkARqiABvqcvwFpQwgo9k/3CVSkGTBMg7cuQ26jbW0bcEXgp++KYO6lDK99J74jbTUm0l4UIVVVCTmAgEEkDYxH3fG0Bo4JyLhONLZCipSSeY1326/nbakhw6DWADzsOlGkAgAwIgQr3e/5W0sFMqCASZGu4672IXWQ7JLvBkqpnwRMpzCI0ixK4GvsXACkd8qM6GCBH3fA2W6hsKYWI1UCkTrB26a7WS6JtV2UxU6lQccOYJHtJ6DTn9eAA99ijMbosq3Q0kpaBUpZhISZPhGus6+VuWKBDRDrywt5emugG8gfP42NUtEsE1dSk9sU6g6ZQY+vo2HWhKjmKQogzod9R+vnr75NN9SmcCNAiLzZ3TGkakAacvo2bmM6P1nDN604JhdK4BG3sGB152dikQ2nMZCdo0PXp52SL3Z9YoqhkNkpW2pJ25jUx7+tjB10NVRuBIXcyykat1RV7MwCgfltztbjhs8iqwrdz8JKS0CAozlJE/cd9/K1RbkzBm8KZ0GW3EmOcDMAZPP3aT7rWo4K1SarBVGAUnswEiCemgM9NeVqE+hV+nN1IiEgJAUAQQSTOpEneOW39bI2LqcGibWmCpL0qy7TB2+htZaSTpBMTATMc+envslYoYcXdayNezWlUGZJ1n8dPO1cHW6snayZi2gV51AFIgrIIPL5crBqaSVShISrLMTPTY9NbGiVQViNeih8Pr8bAkKIEqBUvUEyDp4jz+fvtO6FlKKvNlXdciBAJ2B3nfwn6mxdynASI72ZRMxzjWden3WOrHIQpKRIMz8jz0+psC4DEEAzqSZUfv6zr158rSDkxBCSnWoKgITExlEyfDf7vush4hoVVWH70pkmC9RvBIAgHuqgT5x9HV0Og6JKSJEGBoT4AddLEFNNrXkUkZXDliNwRB/H84sVj0J4uquIAVd9W2D7K23B0AkiZ/wB4b/jaXsBn1jDdA8pBSpLYbHkFZZHwn32iembLSLwpCAn7I7nmlQPTwNpO4XuqXcLbaiAWX3G8pXBjQxHLVXzsSY+FDjGqejdMFNwptMxBAJkDbltPv0FsqaVJHdgGQoZun5ef42PUTC6hQQhuSNxMjnpZy0FwBoBdWiVghRBGsxvp5WomSytNjzJrXfhior3gqpQUN6QojU6A6/1jfay8i6mKJnsmGkJ3IjnJ308ffvZwBlKUZUaRHs7j6j3WK1DBSCoAFMSVAamPMeXwtHPcooZl2UW44owmrJKSQ/TwT0MFPlOgtbv9ktfpcuLiJhtxzv09Vdt4oSVa99DzazHL+7b+ItVbHNPlbpnUp0Wl1AMTB7p085+tbTV+yuvlV38cMV4acXlbvO4XyhEmFLYqWiOgJyrXrvali8fFw+QfdEpnZKhq9SlKzBKxztysbmNvlblBOTIRJSbdnYE9Lec5rhdBayLySY8LcOd5JB59bYruqPibcrJ6687VXuRwogxPdqqS+ahoAgFXaJI5pVr8NxYmw0SrmCBtFnrji7ypdNXBP8zSvvT+NmuhgJUCD7RCtetulwz9WJpSnfZHKNGylAizgu1CUictkiiaBIlMjTUdOdl6jQEpA20Bi3WUkVlg1Ut0rMCNJ8tbLlyOFLhRAgg/E6/gbIbO4kTtZTu1woqkkiNR8zH3G2ywWWU83Fk4s3L5xbQOk7W523tkzyG/S1iyAuzANtE6c7clXKPfbWbSQR02tMC6i46LeYz0i250gA624J10+RtudvqLEAtshreaNedijJyrebVoQskacjrY2ZA+VipBTVL6KQFaWdzdFEHVbVI11ix+4ngzezCp9olHxEWIK2gHytth0svNu6koWFAdYM2DK27SpDeyUsWs9omqYBGYgKA8x+dq135dj1ViE063VMtzmcWnVSUDfKB7SuSRzMWtDilphoLvR1xLbXq5zqJgBI1k/G1c78pqyrvh6vraL1elzzTU7iJW8mBC3kqEATJDfP8AiiChXA4g8xVRZzcNP2XTULg+nt0QPaYjGlFganXT/wCxUuhU6pSP4SVz3zEd7nvbLAOofcdW4uuqFKUokkrJJNstR9nu6D1KNlcqD/tFgaTGmEaBpay0xdT/AHQnUA1CgnUchlHysyuF1+MXTd1yPLoHnQzTNlWWO8ZPMxvHys6v2j7yFcVsOsqmUXGDIMb1Tw3jwP1oWRgVtCLgu7MpK1+rwVAbwpQ0A8vD3W3sHlMGFwuG9v7oGKxiXEJQVNdPxNuwIzLuq8UR3vYQfOO9ZQb4lXCtKZo7wEawGQdd/wCbxtGjJCtU6a5QdPrp8LH29dycpSFEnXz92/uNtH2vUeSoDDYTsFJKeIuG1EZk1if4ZNOdtddPrawox7hnNnFY+haeZZXEzt4/raNVEpkZuUkbe4/XW3TTn8JXqTMRI/TSyGMT9AkcNiHVSYMcYXWkAXmQo7Zm3E6c+Xn1sO3jDDIhRvZCTMGUKTp7x5fnaMkEBGZAhQ21Ik/ntpbBlKjISSNdYgjp7xtaYxiboE3s2MDcqV04pw2vX990yIOmZUfH65zYdWJMPLISL8oDP8PbDX3ToNrRGMqRGSQARp05aDe3C0oKspQEzoNDprO48/lYgxqQbtCGcNZyKmBV93MQUi9qEmJCvWExoeUmwLl6XcpJQmvpVyP++STHM/faHVBKgVkp03ISPGTp4jrZOqG0IGbKmBpqkSBsPf8ApY7cZdzahPwxp+JEvSEvClZxJdrjdQFAUiwSjvJJznQxv+k8rTrgOoZVhO53UPNyqgp9AQdChO9qecTKhulvWlhOUlgmAIJ1M/Ln/W0o4JQ1+4LtUlrL/qrZPjmSNz1384sV+KGJoky7oLaASeAHZWYbamCkkjeM31P62MpaWRGYmIJ3133tBbSyJKHFpIJEhagN9R9a2OsuViFQK6rBA0yvLEbfgRYftxp3Z+UZuFH5lNqUrACgjUGSSd9bcKQFhKUCCVd46yPH8LQ81WXoFAi96+I0ioWIO/M+djLd832lRIviviT7T50MzpPOyGNM5tKRwx4+JS6G3CAFEnmfHY2IYhZIu1yRGVSDB0nW0dtYhv8A3F9VszrLkgfEfXxt07ft9PMFt+931pJAhYSIM7xG82TsZiItlKQwuS97hE32A5UX5ULTK0NpQCBqAETHjr93xTqZtdVeFBXLJDbbiWGkjolBzGOWv11UC5VqU8fXCF1Byud0d6NNfjbhHrKAwhupBTTd5AKEwPqdz8LVe/tKOaN3VFr4o01l8upPeRTsdq4J10BgQOROvusRvBwUWEgy7MvU6kpOYaq1P6+6ymk1HbPVPaoLtQkocJQIUnb3crJN6Ui1sIYWpPZtJLaO54Dx6c7TbXNuhmjcBZQ7xUZ7J66lJJByiFCTpkG3U6c4G3lZ8YTpWlYNYDaEguskqjfNJnz208rR/wAYKqpbvOkaecZ7JDSShITBBGk76iJ+Nnnw9rKp7DVIhCmw0UrQAUmR3j+mnjaw+qaGhyC2nObKpDw04hV0UjhgwkT3toJ1+XzsDWpRVXy1Qg5QofaKJjNl5H652Bul6qoKRNIylooaGxmYOw315/EWEpaNVPVtVqVBS0KKzmJAVJ5fOwXVjDZGbSuGqcJROVKR3tk8tIgCOnvsjXQtDV716VqgDveIEz79Dv8AfY2bwqITmpW+8BJC4/CxBbalOVD7bZSuoA2cnTy8TFovrGHZJtM7mid61yqlztcwDY0SNNxGo+uXhYK7Uti8qclMyCSOR9qPmLDLux1xwLUgZZ7wB1jw6GxhxsCuZq2WMqGk5coAJPvJsFtQ1zrlT4JAsEuISMwgFI72ggz+ViN5qc9SqAncIUSE68j+thHL0bylAZeMa6QZ8NPrX4l3q5laFtFh0ZioSE/PQ9LWzUsI3QOA66Q2K4UlzmQEuFZSgH+Lx6xIsyMaXS4xd5r6pJdfcQ5EicgIGg+vus+W26cVIqKptamW0yygJkCOZ+uYs1uJV9XexdiVOdoCHMmrSiCMp0+AP67WnBOwmxKhJC7kEyOBr039eTedULpkkjlIWPd/EOfIWlXC5AveqYSpUAGQobd4T56RaFODla1R44qEuGErYdRojUnMlXlyO3M6bWmi5nWmL3cqDPZrCyFAEASZGmp5b+FiPka0i6i2IkJ6AKTA3MyUawR/S3OYgBOYka6kxA931HSwCK+kVC0uaSQRkIjwPxH0bcmto0JEOglIjQHX6+uVkZ29VEQO6IV5wNBRLkEnKJ1InfSyPdrvrN4rU+SVISSBG2oBP11saaNPVPqfqqlKUtGEJGhmN94PKxK7ghm+M7iClolSc61aGNdLAMoJRhGQFIGFsHVV/IXWVDTjVC2YK0IJ7VendSQfif0s9m8GUraRU1NMM5gtsKplAFPXYwPrlobuevw6zRU1QusoA22kIYYC2+8rbvQsGPPebNfGePGKtLt3XJUoGfR6oByk9W24J7um/uEWpcaSaTKDYKwI2sbcpv4weuamqF3fdTVO6AYdqENwkGdkaTp1+iyKtoOtKacSoiNiY57+X15KroQ6jKciSDoREDr9eNiL7SYIXCzqAAY+vO2vEWsFrrPeCTeyqOKcUWIL+oVpMocUlJCfZhyNdOh/K1h+AdSp3C6k9pmLTxgTMyVaef6+doHxcw1Q8R79SogBa3VeJkJVtvyPnAtL/o5VYcoK6mCtW3NRMwAE7aaaqI91gT66hWafQ2U5D2+6ZI7x05EffFid+JWq6X4Ue6kKBnQAEH362HaUEbKEAQOQI5gDblbi8kLVdz7eTMOzUkBPPQ6j77Vb9VbITEhQbUSCCkb69I5+/TpFtKme9OVEgac9dfo9LYMygM57vTbXrqZ5fK2nFQAoAZTIJjYzB9/62WYqGqDcSogbxJ0nlpt7jy8bF3CDIShSkxp4zr7/AOlhnDIClRMZRIJEDX387BrAGYrVPj7IJ16+X1ye9kxF0TdSCjNmMk/xaxA/IfebE1kpJVm0kyEmI8vl030sfWghBRlM6Du6H3aa+Fk5yCoqbJQREa6yNxy+Hl0ixGOUCxV3vOhFLi+9qKAn7eobTmJ5lUHbyP6WenBqgeq6SupUKUOzdSsmTpmTEf8AKLEca4ddYxtedcqEtpcbqEJIjtApCSr7zPzsucEHQxe16URUkKcaQ4AddEkif+bz2sSaUFhsoRM8YBUyXVd7NIj7BAznUqGm2u1lNORCIUQBvmgGfw6WLMBCTomQNBM+8DX6+VjTeZKgT7QEkAwAOURtr+NssvJWk0Zdl0AcoBSJBABA12/SxWraQpBJWJMhWpy7H8Z5/jY9nbyltXsjYlW+3vsA+vPLeaYEacvrTztNjkzhcKPcdtH1BLqMwyu97wKkq1G33WX/AEEL3GGvTAw4gryt3l6zRr5SHaRzLPh2iUgDqNhZPxe12l0PpTIIAUBmgykgTp4HkDZpcHr5GGPSM4dX2p1LSW7+uxTi9oR6ylKx55VQYsSZvFppGdQVXHhkafNe4TgCHjoIULdQcsg2yrEBKk/w6W0FZkZh1t5aTYkLpBq0FFnCM2k/G3Czp4m3bx72YHysC4tKUZlKypAkk8rVHHWysNSbf9Ga266hCRKmwHE+adfumzHbbBMQI3iNPdZ7XBii4cS1l4UNz1yav1HIHlISS33s2iVbK9kzEizYqqI0VY7Rx/dqKRP8o2+VuowAktLCg1ZLRYrVIyAEpywPvssUoAjY/hYlTtkiefjI8bH6cQZP6/X5W7inaucqDqj7UaAHaxymVkdBHIEj4WItgnU6x00mxxpQDiB0InS2gAVSKc8yJtgOu/jYGnWTTtkmTlAPiRp+Fu82k2sNF1XJXebbS3M+XwtqdJNskSdTFiAWUVu250gW0DztknXfSxGhQcV1uQZOti9QYfZPJWZJ+Gn42HzSIsWrzlbS5rCHEkkeOn42d2yiN12uY1ANuFE8rdr2iLBKj9LCIuFO9tU73w3WXPQ1C0BRQkakTBAyk/faNMQXWw9fKEO0rD6nVJQgPOFIBJ3ge1y+7ykW5HQ9cBaKSeycKYB6kH8bMfHtMUStsHTQkC3DdooA5wsbHa638IdmuwrhWL8IUyjTJuWgcS0cgWEQFAaT7FstH6KetCQBSlQjQyrX52y3M+xm/wC8fwr/AHZnX915kftFqlL3GW62CdG7gZASdv8AzioM/Dy3s1cFXrdVPcN3sKr6VCk0yM4U8kETrJE6fW+tnB+0Bq546tN51IyXDToVlOuVTjxI+B1tXJ1a26t4pUClCUyrUmAnkfceUdbdnhkAlw2Fu3hCq4i/JXSu81ZmnvK6VhJXelICdY9YTry/Hlzso095XcoS3XsKTtAcSY+B0P5m1VkrKSG8w6FJPOZPOBt91hA+4qftCkp2TG48+vLToLGND5quKu3JWtS6wCAFIlQiMwP9B+Ws2HadbKoBTpBOuny89otU3198O5mXVKyaAJJA322E2GF6VvtIraprN/K8oCYj4k7fpaPcD1TmrvrZWyQNsyMwG06/ERHW3RPfSSNCd55yBPv026/GqP7+vplGRF61k7oKX1yDrpM/P6I6MUX822rs74vAJjNHrKwJB6A9fx87IUThzS72DyVqM6T3lD3jofD32CXqBsVLmYHPQ7D3fpar6cZ4padStGILyTAlIFWoyfj5+8RGuopx7ixtMJxHeLgGhJfXI03k8pAP0ZXc39U3em9FZJxzQ6aCT3dJ2+WliNUoqbKW0knYJjbXl0HjaAP7fYxC0n+0NYRCRose6Ad9RHvNsc4i4vQslV/VCiNJUlJM890zGlptpXBRNQ0pV4tKP72ozMqDJ7qpIkL5+Pn+dpbwEQrDl2GFKmlakHQg5ROkfXlauV939XX4tp29Krt6hsZCopAAEzEAR46/dZy3XxKxPdFAxS0NSjsWWwltKmkqyoA6kbDTqeVizwOdG1oQ4pAxxKspTlJJBkEbkax+Gwjl7rH2YAKMvdOqRmmdPHztW9jjTjRvMf3hSLypEE0yRqRodNxzsKjjtjZmXlqoCEyO+wY8van4dbVO6SK02pYFZCAU9nlk6yYB2+/66WFUIJUmDodFJjLGsz00PxtXRHHvGTYSHmLqMjUqYWNOYHf1HUe7wsMj0hsWNjK7d90rIAEZXNRMT7fj9/jEe6SqXeWFWGQoIUkJSSTyKQJHTw+trDFwABMZgfCPfavzfpE4jcAcNz3Yon2gAsSf+Lp9dR//ACiL2ySrDtIoxMIeXrv4Gef0bD7tLfZOKlnVTwtaUAIKjvlgj6PU25LgUhJJVI7xM6fX6++DT6Rtdp2mGKdQkQU1SvgJR+PvsKn0iCSpbmGGxlISIqzqd49nw++bS7tKOSTp4zzU1F0qAIBUB3tOn46mxCqWSCEpmTEknnvvtytErfpF0+QH+y6wZ/8AS5IOw/gncx8bdOcfLpfht/D1WnOJBDiVQQR4Df3WkIJRyUOMzqkTjU+pm8KR5YzAtgmDrIKtvH56bWcnCt/PhOiJzjVYnb+NXnaM+ImN6PFq2amjo3EFpIQouK3Gs7acvv8AGy1gLiDd9yXQihrW3lLbUv2EAiM07kz03takY4wgc0Bj28UlTtROpQgtgTBmTv5aWPBQ9uD5QAOcjz8uvjaMabizhtJyPmqBO/2cx8DzIjrr5WUmOLeGFjv1bySZlKmTI09/3/pSLHdFazN6p/FacxCEKEkg+J6/L522H5Ep0iRI016WYSOLGEnUpzV7iQqRJaVpyMaaWEPFPCbkFN5Ebe02rpodvgfK0cjuibO3qnwp5OmZSSkaEE7W47RMEpIyzJiCT08LMxvifhRRE3qnfUqbWAPCfj99hDxFwrCc96sogTJKgTr8+vvssruie7eqd4cQSc2VI6zIUZ/rbha5OUyTsYO31+fLSzU/0hYVUlSf3ywMp1BBTMfXL9baXj/Cao/7bphEfxHmOfnHhys4DkszU4ahYKJkgREDfeeVmJxHR21wLhE5HUK05jXT4kedld3G+F1lShfVGTOxdA100jlpv/WGxjLEN01dz1TTF407q0EHKlxMjWNp87FizByE+xCjnho+lnHjICHAHUutAjYHISRt4WsPQuNyhJ7vOVbG1Z8D1JYx5QvO1CQ2XVQpaoE5CPhvHj77WJu+taUQA43ooQAsER4662tVRIcLdEGntlTnQ5pnWlWkEAnU6dfKdfEW6UmQc/dJiCInf+vysTafBR3VJWFbkqzGJ5df1sIV5+4UjMZISSZBOn5fCwMxR7BDZgDAkynT8o+vGwZCEg9xKsuswCNt/C3AczxIkAgAdNiR4crcrKoHdkxJEzI8Pz+hDOQU2UEoZSmZns0idJiBp9f1sCtDajqjUTpz32+o+duA8kyqQka6a7f0n4W5S4BJIEjqPqN/rS0muI1CcgELhxumBV9mdpVz005GxN9phMjMJkp7itD5/n42NuqXOdRmI7pJjQ/pPv52IVbpyaKHe89dNo+vdpYrZHIRYFAvE+mpqHGdcptpSHFobX3SRqpAGo+O362WOCdbUZLybYefaKVtkKbcKT3grQ/8E+4WKcYk9niFhax3XaZuROspWsae4ixDgpU9nfd5UqlFGdhCyADulUGf+I2vbw3VVotIp6o6+8sySL0q8hTmOZxXhr7/ABsfTeF4uoLZvN1SVe1KpnbSLItIoQCTBSnlMA7n4afDnNlFCiuCFpz5tTB+P3DyFqRerll03TpSezLiipJgz/KfuGse+3PqKW5Qe6YiZ6aDyibCtAwSQF5J1Pj1NuswBCUjPrzO+34j5WbiJZEVVQqcTCXzvr4Dw0+osEqiW0dHVBI1GbWPd462UFFQCAoBQG53I8dLcrAzEEwNpjNrHT42QkT5Aks0CiAAqFaScs/p1+hYSiuhIXKpUQSYB/A+Nji1JTKVAyQc2pAPv9/3W7pipOfKoZo06aTO3O0XSkJ2x3UUcWrvLF8B9KYL9CCCOUFSIHwHL9Gpwoq+yxmQCQmqYeSfEApVIkbaRp0jS0i8XGULN11KkZkkOskjYgKSf+oxaJ8CVDlNjG7H1riV5B3twoEdOZM8+vjY8RLoygvblkCsU25lgpUSAdkgwBrGsfUGx1oyBCDA2mfqbJTDxy6CEn4zHQDXysptuAqIWEjMZ0Op08/qLU72VyyHSsIkpEzqoETuOvvNizroCfZ2gGFcvKPD6i3bjyCsHRJ29qY/WxV5R7M5iND3jO/kQNOVlnsoloSHiFfb0NWyQe/TrMRJ2MfP66Qnfd4uUd5XPeVPUErpnCtKpGhBSUx4fmLTrXMocPYrEpV3DG4B029/1raAMUsqaoqdb+YervBKiZiYIJJnUfiogWuwOz+EqrM3Lqvfa6bzYvy4aG96ZQLV4UrVU2oHQpcQFA/A2Fp1ygojY2h3gLxRwvSei7w/xviu/qS7qJGHaKmefqHQAXWWw0tI5qVmbVoJOlo0xn6aGE6x9+7sK3i/RUmoNUunV2rgmJTyQN+qv8pt5lJSzOmc1jb2JC6OnAkaATb6qwuMcfXFhVBRUOmqrYJTSMEFw+KuSR4n3TaDca8TL0xJmarqkNUavYomSez8M5/2nvhO3d52hiv44YXqFKDV5KlxRUouBRUtU6lROpJgySZsg1fFi4qkdp+8RBIzTOg8J5/rYseGy3uWrZiNLAPeBKs56OmIlVeNrzutTuZNTd/bQf5m3EjT3LNpdxXR9neDdSgAJfRrA/iTpPwi1SvRp4i3NU8ZLku+nrkqcvFFRSxrJ+xUoD4pHytdHE1H6xdpdCZUwoLnwOhts4cw08wDgsnEXtkcS03TYp0QBEa766WNITA2JjeOtgGUmNSJiT52NIBMb6c58bdrBouXm1KGaPeGg+FjKCIjWbFQggwRIImw6IjqALXwqbjZOChcLjGm4Vrp11/GxgEga72IXWsFlSd/oj7osZqKlikZXUVTyGWkAqWtZypSOck6Wss91Vzuh5B1P3WyZMfGLErvvi6r0Cv3deDFTl3DTgJA8rCvV1JTKSioq2WlL9kLcCSfKbSBvqmNxojExtbYURubcTIzA6WxDrbkltSVwdcqpixAoG5QgIkfLzsBeCM1E9lE90ke7WwwMH8BbagFIKCB3tNbORfRMEXaUlxhC07KSDYNRBGwmwdAT6mlB1UglJ8wbdq02Ox1sMpwbpzYNd7RNZTKOikpUPmD+Fid90jK2jUVaQpFOMykKHtnkPrlNucIvdje4bUdHm1J167/AIWO3+3Vllxlqkbc1UVAu5Y6cukW5PtM0CAuIP2/4Wjh7iJALqO3cQ1/aryVlalOYwEvhKQJ5DLoPC2WFVQXlmMPJQJ9kPmE+A7tst5pxaXq78rq+DB0Xjt6d9QH/SBqEK2Zuuja1PIpKoj/AHrV+dRlrXg5kSZE6c51PUnfr5WnX04HQv0hL4gpJbo6FEEbf6sknbz++0Hpl111KgVZjJCANSCem34Tb1PCfDQQ/wDxCwcTN6yT6lY2tSVGRqARoRvyMcvHr87YoDdSlyRMcvH8fnYXskqUgnUgyNRrHIc+h8rbKBKUKRJUQcytCvkDa6SFVsuA6sGMqCcpMEkRJ+vEn3W5UVAlQIAXMwZBiD4jSwrjQQe1KgnNtJAn6gW6yntcx9pRIPlGs/D4RZgQmIRZK1JAJCgRJIUcsn3am2FAIhKVAHQEzJkcjtPOxwUxVBaSNJ0KYHXrrp9x6W0lghZMZyYEHkZ0jofxBsswTWRXMSU5Mqc2wI1PMj4R8Bbckp7NOYgA6H4R9c43saaZWoKzLKkKUE5MsyYO/Xx+hblDE5QpKpIAUCDKdNIB8h46WWYJWRRTnfUUglc666kzofv38ettFwk50JS2F65Rpm941A0ixkMFAyuwdu6pMwYgac4/XW3Cmm0ELDiRlB7u5MbR8+gtIOGyiWlFHu73ipPdUUyFeyRsB1Gm/vscbV9i2DrkTlPe08ZjfSefPxiwlz3BeeJ74u/D1wUIqryvCpTSUtOlQGd5ZASmSQBMxJIGnhaZ2vQz9IjKnPw+Da0gZpvOk1MAf97E/lYc9RDCAJHAfVEhgkk1YCVCZdKEw2DJE5ojXxP5xtYJeZa5LZMch8/Ly8bTw16FfpFPNKcVgVjKTAUL1o9CeUB3Tn9C2K9CL0j31AHAzKUmCVqvajEjkf7zpytXGIUvzj1R+51HyH0UFZ0hQWcqVJ001BPjHhpG3lrPJUOQzJUCU6mTrsfqNPC0/U/oH+ktUoUn+xdGlcSAb0pBqDpsvb8OlhFegj6S60JT/YmnGg0/elMZMc/tPDbUfOz+0aQacQeqXc6n5D6KAmaw0y1KU224E6QtJEgxt8v6WDcqFOKLuiQBO2kc48Plv1tOFV6FfpFUL6mXcEdotRnKiqaUEncR3o35gnWwafQ09It5SGG8BOdoo+0urZCBvBJKoH15WcVtKdQ8eqj3WcHVpUKh0lRCsoBGpSZ+Hl566+NtKqEg/wCGYWRroSOuh28vO01VXodekQy6pl7AL5KVAkoqmFpmBsQvpE6mxdfoh+kM0Y/0d1kbaPsxsdiF+PLws4q6b5x6pGnmHwlQ2y6ACjNJ1lXnMCOWg38B4WxFSkKSsoJCgQsq00k9fDTTx8LS4PRH9INtGdXDWrkzmBfYA9raAqwb/om8f0hShw3rRmyxDjPTX+PX+mtpiqpyffHqocCb5SogdcWlAEmJJgCMw6zy016fiPROuBpS0LJSdiDBkGYPjqLSo76JvpBPR2PDSvOg2daGwP8Ai+tfcNR+iP6RmY9vw0vGEnTK80ND1Gc9BaRq6fL749Uwpps3un0UXCoCRnCjEZVp0J0HhqNjH522mrWlsgFxOXuQFEx5dPaHnpaW/wDyTfSBUkp/0Y3jomP71oGQIjRX4fqGr0S+PpKh/o1vIlSZSSpoAe4K6Tt1sAVdOd3j1RO7zfKVE4qFElTqwCSMpKTsPCdLdIrFtNnvEKKdRO45e+NvM+FpSd9E3j+tAC+G145yNs7Yj/m5e/Qne2j6J/pC8uGV4qndRW0PCNFSZEbfPW0u80x+MeqjwJvlPoosNeIRKVBQTlCdZj3/AFM+Fuk16m2woAJUTqdZ023kdP62lRHol+kOMyTw4vDUnTtWRAMae34fRttHoj+kIUmeHFaNIhTrCuZnXPvy1/Oy7xTfOPVPwJj8JUTGqcKiVASdfHXnr110tyqsXlBUUnN3lZdSfr3be60wUvogekVVKLTfDerQcsqDlVTtgbjYrg6RYm96JvH8OhocOLyMzIDrcbbHvRrPXwsu9Uw0Lx6p+7T75T6KKfXnHVZjJ7+k90kdfDfn4205XJgJzglGgI0Khz193ytKi/RN9IVPcPDa8VFCoCwtBmdCR3o5DX3+YZ9E30hFZQeGl5gL9mVogR5K25bjraQq6T5x6hR7tP8AIfRQ4zULS+k5u9m2B56Tv7rCuVa/ZClCRqdI3IPly8rSqPRK9IcjKOGl4pymZzNiJJ/xfGwg9EX0hgkn/RtWmei0TtzGaTt99imtpL/1G+oUBTVA2YfRRSm8atB7RNStBBjRZSANfHwt3+/b3ZWEC8aoZdSA6RqBvvvtp4Wk1fokekIBH+jWvSMsHvI1A0/m3841t0fRJ9IVauzHDi8MwJUR2jYnx9rQ+z87OKuk+dvqEjT1Hyn0UcNYwxElv7K/7yGbUgVKwAdgfa6fR3sMnHWLUggYmvXKnQJFY5I16yR7uptITvolekOCVL4b14OnsutydDyzaa+Hwtx/5IvpEZj/APw2rIgwS41+KvGyFTRn42+oTd3qflPomEjH2L2niTiW98+bVKqxZkcxv52Ms8ScYoBR/aW8CdMoNQodQdeY5e/wmzzHokekQVZDw4rgNVBXaNxA2nveduv/ACRPSFSSP9H1WQZEesNCSdt1/Plz5WRqKP52+oTiCp+UpoDihjYDL/aWs7xMZoObXy89vH3iK4oY0nK7iKqVlGgUEq320KY6z1s8mPRA9I50Ky8OagQMxipagzznPrrv00sI/wCh36RlK6KdfDupJSAoFupZUCd9FBZE78/wtDj0fzt9Qn4FT8pUV37im/b7Uly+bwU+42ChtRbSCU9O6BOvUT8LB4exHeuGrxXX3a8EPOJUlX2YWMogzB05TItKb3oe+kS240g8PKllLyghKlVDYTqNicx+XjY8z6EnpFrcCP7FJlYE/wCvMpHQDVUWJ3ykAsXj1CYUlUTfIfRM+k40YtpUABVA6EjvBTG/iYVoNrH0ceMUJASaK7FEGP7lwaTy73h9a2XXvQr9I9Mf/IJE8z+8Kcgjr7evX6NiS/Q/9IlogKwCQnQAiupxv17/AI2HxqJ3xt9VPg1YPuH0QSOPt/JEOXNdpIJGUlyZjU+1Y4xx+rwuVYdpSEzJD6tDJ3H4+Gx2toeh76RmbshgY5hyTeFNA0HMOdB93U2T8XejJxxwPh+sxfibBC6O6buQlVQ/67TuhCSpKQYQszqRyMTaOajeQ1rhf6qWSpaLuafRLjHH15JLT1wU6lA65amB5+yfryEjDjrTrRK8NHltVDUdfZ+f32gpDqScoUZCdJJk6/1sMisWCA0U5iIJHj79dvusQ0jELvDlOjPHK7FhAcuWoTmTrlfCo59OQ/ppZSuzi5cr1WW03XVoJBKSFJ18teX1pE1+RUulXf1EglJBPj4eP4WV7jrlivbICiVEtgde78/zJsCWlAaSEWOc31Ur4/xjd9+0NMzTU1ShxpalhKgADKRpIO8gfQ0iukrBSYhYq+yJSxWdovL3tA5Ow8Of3aWXax5LjYhJWU+zmIBEadekWZ9epxFdUKIgFSSAkyIgRty0+Vo0guCE8ztQVYCnxtc0aLeSdDBR8Bryj4ab2UqfHdwZUpD7vd6smY2AiNbQ9TPj1ftE5hm/igxG8dI/DnBix/tlBKFgpIA0gkpBJ+ufI62pvaQUZshKls41uDIhSqogZjqELmdNzFuFYtuIqCPXBKgIJQrmPL9NLROmpKWgSVJMAeKvx5+Fki9cXXfdMtBYdeEdxBTod9Ty3PWyjifKbNCd8wbq5THV4iuQs9qLyQhIGq4IHI7kRv8Al1tA+PMTXS9X1t33YoPoeqSS4kygJzT3dBOs8ufibN2+cVXnfK1B15LTSlZuzbACfCepFkJwkmSZ1mbbNLRcLV+6zqiqzCzVKmAr341cQGKfB2GsSvP0eH6ZRpKOodSWqdtTpWoNpUCBK1knTnvtZ4VHDb0j2UFa3KF0azFOwZ08W+kfK0HXNiG+MM3i5W3FeD1G8oFGdpUEpkGPEaDeztp+PHFSmbDIxdUOJ59q22v/AMSfK0Kimnz3pwy3mNUemqKcMtOXX8inJX3Fx/pCfWKJokDVSael0Hnl0skOPcYKYAVFA2SZGrLIM6jlEGNNNbJr/GviHUp/1i/AuTJ/1dvU/DysQe4nYrqEdm7XpI6hpI6fDblppZmQVHxsYmfNBfwOcpd9HfG3ELD/AB84eXne1MGqRGJLvaqF5QPsnH0tq2P8qjb3KdYTUMuMLHdcSUH32+di7uI+IaO8qO8PW1BdLUIqElOhBSoEa+75+VvoiuO9Ke/blu++6Q5mbwpWqto8ihxAUD8CLY+LQujex5aB9FcoZQ4OaCT9UzUMqbWppYhTZKTpzBiw7YHjoI99j1/U3YXipYHddAXMc9j8wbEgNhyGnutqUzszQ5VphZ1l2lMkdPusKkaagadDYNPtDTT8bCjUTpBtoBU3C4SldLgJKBznX4R+NkXibct8X9hh2guJCXKntW1ltS8gWkKBIn3A+6ynd6gH0jxB+8fjY7etYLuu2prigrFOytzKNzlEx8rHZq2yrk2ddRTd11qwfii7qekrHl1D6GV1dOp3tQytRhQSdwkj+HbSQBNlHEmGb7vS+r2qat6poKdDanmK5JbWjRIypyqkx10G2h52TOGlddt5V1RinEd5sisdcK0pWSYJG/kBoB0FnLxFvdbyKO6KJ0BqqAdcczQnL/DJ6c/daLcoYTyRHE5gk2lvq8nsBVQLq0inqGm8xOobUpMp6xr8LJlzvV+Hb4u+9hea3ae8hPZFKQG8qsq0SB3gZB11BnXYB4C6rsRgqtua7K2nqX/V1uKLa0qUpwCZIHjHysxql1NXh673WlhS2Ktad9Q24jMD5EpsW1gChHc2Srik35e2ILypKGuS1UU4KqZK0lSIS2F5YBETr3ht42e3D2+n73uQesvKdcYUE5lGVKQQCCTz52adfUBnGN3Vyu4iupmHFA6DWUq+Ay2P8MnF0lfeF1qOjcoTP/q1lA+UWINDdQOye7cIdqGxpC8w94ti5UdNracBTeCyVaONgx5G21DXSyKYI1dLwprzpn9gl1M+RMGz4vCmCnV6e0i0diAd4jUG0lKWKmmYqQNHGwr4ibZWKQiSFwPRGhcWu0TGdoQHVjXRR5+Nss5F0iSonJubZbzI056Lb72V4K+mj3/SIxElBVPZUKYH/wCqtdbQ8UqcqHgW1KHakZ52n3dTG2utpu9L6hqLz9ILEdXdzJqqZfqWR5glaTlpmgdU+ItHQTdVI5UpqaRaX+1IjMBy13Ply1t3VC7JRxNt8I/YIdb4qqQ+ZTfNOvIVdmXIMEARoRoD4mCN+Vum0KUMqgYmSkIgnu6Hw3s42b+uWmSWkUhb7Q6gqSABy5zpztpNfhDR00jp00MmUmdNSoSefS1i7uir3HVN5NOtQSCFDIJJjbTczz11526TTKSQrs1qOgEE7xp5f1svJvDCi/sadp3vryJGZQiPMzz84B6ahorMNLUnKFoSvu7geMb7fITZru6Jhbe6RhTK7RBZUQANYTOu8nx25fp2lsOBbYBQghSVExB1PMzO55zrvZ3XFdFFftO7U0YORtZQVqWB3okiOehn8rKQwa0lKgnKIEAFcmQJjlGmthGUg2RRGXahR+pgEkZDm9jQDfw6a+7QG3JowBlghSN4BHI9OUH7+VpEZwSwk5XklKcxSR2qZBBI3jflz2EeBhGBLrUQlZcAk5T2qNTrtO58+gtHj2UxASoxUylWiGxKUEFMTJH6fHnIsG9TOskZ21QRA1ISTtM/dytK5wTcwSsOVDwC+jiMpJEztzn5ixd7h/cDo1vB5IMk6tkgbRtqNPhNk2qF05pnEaJF9Hp3Jx74al0pAViiggbjvPo01/W3uUmloHgla6RgnL7RbTP3W8VOGVzU9w+kPw2pGHnHUDEt2rK1lJOY1CBuPK3tMmqCW0x/CItQxSPjuY63JXMPcY2uaeqEN33aglQoKYf/AFQ/K3aqK7VaepsE7atpsA7ejKmxThiFzq5OkeVg1VC4y5pGkmd7ZxogFeFTfdDGiu4HSiY8fsxbZobuUO9Rsmf/AFY+elihvFFKStxCnJ0ATv524RXlYzGAN4mRZdyFr2T95J0uu13bQF7tU0rIUTybEnzNulUdINSw2I/wjewZqlHvAfG2i/Jjl1tNlKBpZMZ/NGW6Ojyx6ugQP5RvtYZNFRAEdg1GsDKLE26pI3V7tjYUVOmpGg2iyNN5JhPfmhzR0mqfV24Oh7oNgxd9Eoz2KPegfjbkVOYwUjrbn1sTEj4z8LIU9uSRm80KigokqJ7BsGI9gdbCt0tIP9mkTzyixUVAnp5CbdesmRoI5gmzmm8kuN5o0mnpEpILKRp/LblynpZksok7EpFi/rYgiQD+NuVVRmUmTt52j3XySE3mhPVmEq1QjQxqgeFuls0uiuxQSR/L87F1VSo9tXh4W4NSdif8Vpik8k3H80OaSjgD1VuR/hEWDVS0YBHq6IGh7osCKobAHa3HrRGvLfe0hS+SXeD1QppqQDL6syY/wWLVdNS9lpTteyf4BrbFVBgzJG2n19aWL1dQgNgmUwCJ8bQlo/Dspsqdd0LTtUxcWoMtzl0gC3KmKUttJLLRk7FINidNUIzokiVIn3zbhNUcyNZCSNZ6k/laiaMjkrPeB1Rw0lGWCo07R3klI5Hy1sXNBSBgqFM3Ko2SNPlYJVcAxCtyo87dPVI9VmRMA6ajey7nfkl3i3NGU0VFH/mrIEfyixdykpVLX/q7crIGqR8bYa0BvMCCI5mwCKxLj8rJGUEjl9fpYvc9ALKPeB1Q1VSUbaW/sG+kBA1t2qgoVJj1VnQDdsWLVtUFJSSNlHnrNuzWJUEqAExOlpso7HZQdU+aKu0NEapI9VbgJiEpAsVqqCkfUGkU7Mjc5IPKTYeqqwHisBMJTPgd9LF2XyUhawoKWrrMaHT5TaQpNU3eQEUqbsoWlJSmnYCYBICQDv5R9TzsiY3xHhPBlzLr75yUyHCllpTdOFLJUYlIjkDvECLK1+1rV3Ublc8VFDTS3lhOphIJ08Yn5+NoQ4ecV3Mb4pp75vAyUsetBouy2wlZAbaGsGJk76gnSNLMVAXnyUHVYbZPpGI7wxS8tGCOGN64gdY0RV1zq6eka/wJ7RRSo6knK2Umedg8S1XGbDF3euO8G2qtE5nDSVDTwbSQMwCMoV58trWCw1elB6m0xSMNiESCjQKOu/Unfzsevq8l07IU6ltKlwMugPv1m1luHQBuZyn3mRzg0BQNhDGOHcbUqwzdrdLXspHrNI8hCXEGNSUnvRPgR42PVV0UTy8qaJkRqqGx3dfdzstYmuS4W3nsUMXOzS1obzKrWBCnAnXKobEECPf8C13w9Tt1KSIdAXIiACJ395+hanHTtc4taE9RI6IAlEGLtpUVHZqp2vaBAygwLNT0grgoL34K43u8UjJAw9XLSnsk6KSwpaY6EFIPmB0s9KmvS24qmTJgZdAkDNmCs8xmmAU7xB2nWxHGVMi9sKXxdgSVeu0FSwZ6LaUn8TazFSBjwQFTkqszCF4h5UZoygAmDHx+vK3aCkyFLylJCZ2/pbpdMtsqVJkHv5tANSBPvHhbvsXHXClCBCk7JTqdfrbqbdXcLm7ELaELKu6tJCRvBhX1B+Njl1lTVYyvXuuJGZI0Eke75TNgGGlzmLYJBE5hOkgayfr7jbbaG4dQhKoVmkxAjYa76RMfKwJDcEIjBrdOqoUkJCFAZld4BSOh6jWdBGvPxs2r6UoV6Va+wI7s5tT79tyfnZ1VKE9iEtmUq0UCmZI0HPw38OtmriYgJaWlMjKqToZ1EHfTUqtSpCC+yszaNunDdLinKJhxMx2aO8lUkRp74GnusHeV60tCz6xVuZDHcj2lJ5QOc/gOmjTRiZdDQM0lOwntG5+0WJiSSBHPc7/CyLUvu1DxdfcLq1ays62tMosziXbKu6osLDdLd8YxrK9JZoppmtZP8Z8yPKzdKpOqjPjbR00Bi2jMa9NBa+yNsYs0Ko97n6uWxIgfdbSgZ1523Cuu56227BAM76xFiKFlysagpESBbREDS3RBCUK1gyAffbakZR9aWSVkGCelsOltwd7ZB2skloEzMW+gL0Q8TDGPoxcM7+zlS1YcpKRxUnVynR2C/wDmaNvn9O1vaj9lpiY396Jd13ctQKsPXzeN276wVioH/wDs/K2PjTM0LXdCtDDXWkLfJWgxLThdO3UBMFCsp8jZux7vfZ6V1P6zRusACVI08+VmbBB1EG1fDn3ZlVuqbZ110AZkEeXIGwiVEka6+f1424GU9QDbsCQRlPiTbXas8lDMOdm6FE6CSfdr+Flh9hqqYcp30BbTqShaeoIg2RRBOVZ7p0PlsbK9EtTlGwtftKaRm841+djRlVnjmmgxwgwXSyKRiuYBnRuvfA+SrKd74Dui+GqVpb1Wx6m32LamX1JOTSAeu1nGCYInztuYMDexg1vRQLnHmm5h3A1Bh2rcraetrH1LQW8r7gUACZ6eXwslvcKbr9eNTQ3pV01MpZcNIkIU2kkycpUkqSPAGBsIFnwd+VtiNgdtpNpBjbWso5nXum5iXA9DiGipWEVTtFU0IinfagkJIAKVA6KSYEjwBEHW3eEsHqw4XXaiu9bfcGUrDeQAeAk/fZwiAJmBbtKjvpaQaN02Y7IpWJy1NO71KkE+Yn8LdLSDuAD0i27wgNIcP8DiVfOD+NtrEAfGzEalK6BIg66Wf2H3RU3DTmZLctn3EgfKLMJXx/GzwwW6V3fUsGfs3Mw94/S1WqbmYQptKUSjU6Wyw5RJJtluU7i1Wsy+aKnqV0TLbTbriTJBKVQo6kgT79rYupf7ZSHUOJWhSgoFKe71GniTp5c7ErrRUPM06e0lSjHdIEHYDz8PlZw3dwt4kX7eTrd10HaPIQp05qpsbECSSdTJHnNulbAXOsAhGXKLkpKRXLJyw6cw7sjUHp9DWNfAVq8AiPsXE94SFIBJOkeOhP1vZyDgFxjQO9cyEiZM1rO//F4bmxpjgDxgUpSTddOTlJE1jJ13B38D9RYvc3n4UPvLPmTTTeBUlJLJjUElrUnQgGPKI8Oe1uvXErhCKdefY/ZSYIjpB+Pws5TwF4xpQS3crJCz/BVMTIn/ABCdLcI4KcY2kR+5BABTpUMczP8AN9TaJo5OicVLOqRmbzq20GlpnapCDKy220QFH+JRKfHLyHnYRF+X0FZU1l46gpGVTkDXbnHLygC0l4G4H41qKGpcxBhvtKhD47OKhsSMp7oKFa6xqfGzne4G30hSVUmFVNgCClNQCSZTrqvTTWdxpas+FzXWyq0x4cLhyg9q/b+efSUVd4EqTlBGeQR+oHT4myg1V4zWgGnTeeoBlLSwVancAHx8J1i0v0/BLETTqVKw3UNAKGoqEk6kyPa8APjY+nh3i6iWtNLh10BJlMZVctt/r52rSCQe7H+FYiyH3nqFm2+JTqClqhvtSBBMsLUZjQ6J93Tffe3NQviTQNhdTdd9ozGVLVSLlX/LpqPhNp2RceNqVK0OYXW5BmSyAdSdsvMQPjYk7duNy6Gqm461IcGfSnUlIEDaDpyA8rAa6e+seit8OntpIbqPeB9ZflRxzwCm8EVSHEYju1MvtFsgesNnmBvJ+t/Z9tw5QZiYOlvIzCN3V9Dx64fP3hQu0q1Ygu4gKbKCQKtI5nb9PG3rg2QUJGWFRO9lUsz5TayhTuyZgDddKUCRJJIHvm2jWtoEKGw+vvsE4HMhKUmBz6e/4WJL7RRJSox1ja0WQjZNJNbZGi4y4vOSNIk/XnbsOM5co166/dZPQh51RShJUqJIAtyXSFBK5B5z52L3cHRCFQQlULY3UOe9tlxoSQSn652S0vAjQg/hbC8JIzeR/GyFMEjVEpUFSxvqPCNhbBV0/JXh42SC6IJB5RpbkvnNGaROhn8LTFK0qPeiEtqqmSYMkjTe2GpZkEEn+tkUPySAr/NNsD+XvEyN43+jZd1AS70Ut+tMzEE7RbYqmTM6TvrZE7eUwTPPlpp1t0KgACSQPPlZxTCyXenJX9Zp5g8vvtpVVTzqPjZIL8GMxHnbXrIUrUieXl9Cy7q1LvTkrmqYzSNJtz6wxzSoRpZK7bTQK0H420qo2AmDG1n7sEu9OSsaljcoG3X87cmop5jKdZ5zZJU/zUrfXfS2lPnWDM89vCzimCgak8kql6mH8A8JNi1dUU5bIybggQbEvWFaAq0+Q+tLF66oCkJSN/Pwsn04spNqTdGG6uk7Rr7OPso+FuG6qm7ErCIlwDfTSyR61kQy6NB2SjbTbw9QJB2XJ1Gus2rmlaiiqclM1jJEZdZJ1OlhV1dMaUSn+GRrHQbWRS9nDywQYlI/H7rb7UmlCBBCkxE77WburVLvTilV+8KRLIT2YTmHUzpbVLW0xdCuzVKknUzIshPvqUqAe6BAVI0+NhkuparEQdkZddtv0FnbTC6Y1Tglm8axgNCG9AQLCJq2FITDWkdddtrIdc7LG+oMkGd55fXjYQ1IQ2FKVASnlB1+vv8AGxO7tCH3lyNVVZTrUlkszm8TPl8/qbDpepk5R2QypIhP4x9bWbztSMyVpJAKp9qSQDrz8fD32URUAnTWDBEnXr+Bs4p2pjUush72cpahCW1sIhYLahJ2Pj7vvtT2huS+Lr4o3VgG6rlvK7WzXKU89UUpYUulQohK050gqCjICkyIOh1M20q35Q2pKEnKdEqEz4b6frZCvbC9Bd9WxjCiTSsXVdjaW007Lcu1DrypfccUTOZICVCAPZJObQgUwMTSQFeobTu8RTpvaivdFzihuZNTkd7jy2qs0y8kAR22vZDQkkSoxAHSHeGuDb5XxCqnDTXZR05ePrDlHfFRebi2ZIAU8/MKjWABExrEmaqrE1Jh+6n61x1LyKZrN2TRAL6tMonb3k87RvVcT3aamqKujxzhK770Lkpu11lLzCWx/CEoU2oqMR2hJEzAgRbOa9rhYro44XXzNSDju7cV0HEKsu0O3k83TKytqpL1LJSwvvJdUye46M3dggkQdRNpWo1Jprup6NISSwy23sCAQmNuXW0TMcSr0xPeP72xHRXA4+hSWQihV2mQQIUFEmCJnyI90stMLcp01B9rKmRsdt4O36/CWG5JJH25Klj7XQsjzbn/AISReCyp9WQhPUAnWf6WFgrSG3CQFd0687Gl3VUVIeqaVJU2ynv94AzBMAc9EkwOlgygBgJOsJBEiNB+v1tbS4Wq59slwvNm8sCcHLvvu8LuqnrhFRTVbtO62qrSlSVIWpJBHaCCIiPC3bWC+CRICTcS0nvDLeBAPwc00+trNLjhwLvV7jDjN+kqHEt1F91j7Tfq61aOOFYggbDONf62YrnAnEAWA1UAhR0+yXMe4dCLVXQQk6zkFXmzSgf0AVOKeG3BVxtB7O6w257ATeSiFDYH2/EWOU/CfgzUFSUpoxlBKibzc/Az0087V/HAvEaRPraZ2MMOEBU7SUiPx194DvBbFDUjtmySDA7NzUjlMb/XOzGljdtUH+fdEFU9uppwrOK4c8ICnskNNqXlCQG7wcUmYEjzAFqc3iLxrb2qaAqW8tpbjaEN6gZZmAPBO/vsv1PCLFTCe0DjWTkTnEnp7NmZX0dVdN4u3fVAh6ncKFiTuD8bXsNpGQF2WTOT+Fn4jUumDbxZP7p9XDhe4a7ClYbxuS90X96wU072YCnDRA9pJ70iF+FuVYBu+FIKqxKgdsyT06J6fh0s38J4VxBiv1v9zPpHqSUrc7R0p0VIEfCysOGWMgExWsAiYAfVpzI2tZkeGOIMllTaC9oIZdSdgXhJwbxRW0lxPO4pTXvMkrcK2UMlaEyvLCVabxNpHc9D/hS072buIL7ZOmgrGCU+ctfWlq50XD3iOw+HrvvMtOgZQtutUg+QOnX52U2cJ8Z2wkM4orW8qcoAvZxOUCDG+m3ytmzRyudeOqsFq080TW2kprlT+PQ14WOphGKr/SIkS9TDX/2c/jpZgcePRkwhwy4fuYrw3iO8q2pYqmWXGKhbSkhC5E9xIMg5fjysyxh3j8ggJxjevcOYRfbndPX2tLB3lhHjtfV2que9MUV1fd7qg4unfvhTjRXMyUqVBPPa0YW1EUjXPqgWg6jTVFnkppYnMZSkOI0KiTIC0DmEhUWuxhD0DcD4rwtc2J0cTrxT+9rvp67sk0TXd7RtKymc+sTFqzI4GcRilSfUGACrUGpRqRtz8fnZ9XXdPpQXNd1Pdd1YwvVijpEBqnp270ORCEiAlImAkACBt87HxOZ07AKSoawjfbVVMMiZA5xq4C8HZTLWfs7bkRpR8RbyWraP3ahUq5ahyyY7+zrrQFOIx7VoQJGZdyk69P7wC0c1N+el/hygqr2/ttfzbFNTlx51F4gqS0kEqVvOgmSNbM930rPSHfc7V/ipfLqoABcUhcR0lOm3Le2dDT4zKLxVTXD6f8LQmqMIjNpaZzf59VNDn7PG/hAZx4XAoEibnWNBv/tOXO16/wBndwlvvgjgfFeBb2vYV7b96tXqwoUy2cpcaDaxCiZ/uU7W88sE8YPTexrdn77wfiW9K+iDymu2TRUhSFpAJT3m9BBGm2vna5/7PnGHpPXlxFv+6ePDla/dT1zFygcfp6ZsJqUPokS0kKJKVq3kd21eU4iHcOpqGOA3A979lMCgczPTQPaep2/dX0Eyd5s0L0p/V651uNCrMNeRs8Dpz+FkLEVPq1UAanuq/C1+hdlfbqqFS3M26RQkK84iI5WETA184jS2RsInn4fWttyDzIjebbjVllbAIOpI5WVaNwqZjmFKj3mR8jZK1EwD0Nj9CsdkpAPMHT4fhY8e6rP2R0RuAdrbBIEGfjbgGBoSIttKidBz6WsoS7EDczbAflbkdZ8NLbmdY162ko2QgO8iBbtJ1iRYESnQi3YOgPwtNMua5BcpXgPayGNOY1FuGFh6nadJMqSDYdRERBI6WKUcJpuyM/ZqUjfofytB26fkuliBFnFgd/JWVFPycbzecH9TZvOa9d+dlPCjpavlkkwFSk+MiwpRdqdqeZQqT52yxssgmbZbG7mUXiBfMtcjA7aibK0/3qClMcyofD63tZ3hc36xf1YhCkIHqyl94EQntE89eg+XlaugumruW/W7uvOkeo66iqEMvsutdm42sLAIUCMyTrz6aWsrwoYR+8a59TqJSwlGTKRoVSeWu3n+G3R6vVaq0jKkZdMppAl5CkKITlSqR4ae+PrUJIKHQsEpBVm2nQHr77HcjRkKfSJEknpPM9BbYZbGZfboBGkZuu8/XxtsgmyyQUSWlWYo7Mnp3oEfX0RbR70KKRBIkg6n3fXLxsdfYYOV3t0HQagmVEe7cQDrYFbSfZDiQIHsmAfL57/rZkjqlbDRSadyASUOpMxABj4fHxstKUlStOZlI57DSLJeHWULYeQFBQDiBGsDfpZXXTLSUkJOmo5SNJ8BztTl95aEHuIE6HOAZBJH/N0sGoLQoOIKCokknfpv99jCkZm0tBG0xGmup28/usGsJSoZkERqI122/wDDHxsFGGqKqzlPaTOxPPSDpHvB+teHRnhACQlMDKPMc/d91jAZUpvvpXqoJlU7aTYvV1LdHTO1VUshptBWoxpAE/jt99mNgNVMAnRQ/jNeT0hOE7cJzJvqjUop3A9aa+Wvvjwt6ctLSpIn3z99vLS9Gaq/fSD4bLW6Kd6uvyhT2rmoZ/1lsIEc430G8nc29L2KO+0o717g5tQex/W2FWuzPuFp0oLGkFONVe+ulVQJy9i5qrTWdOfwsWU1CVJMGBpZKFDfK+7+945/3c6WGFBewSqL0UDEQECwmEnkneAOaNMVzt1ZnWMvaKGUZkzAsjl9alKWtUqUdfPf8LFqu7b47WXL3cyFRgBAsSeu+8CIN8PgjnA87W27WsqjjqlX1kb5zvEDTlbPWcycyVE6TG9oM46Y9xDw0cuGoo8Q1KGq9x9t1HZoOfKEkHVM/wAR2NowX6SWLgsZb2UtMgqJCfdACeevuFkXBptZSa0kXureqqhmJzGAfo20qoE6Ekcp3+dqjtekdiEkhV8PAjT2Uyd9+7p52EHpE4i0y31UwqIhKJ+Q0mzZ/JPa+xVtPWhB36RI0+Fs9aSRJWZiNDGlqmH0hMQd0G/a47pgBMzoZOnnbn/T5f5Ol+VyRMESkH7tbNxQOSYMPVW2FWEyc2069NLb9bkwCmRpEWqV/p7xCdVX1XA8wVjTly8j8LCU/G3EdQSVX9XQTJUXfZ+G9mNQ0clIROPNWx9ZHXXTUW5VVECSRHhoZtU2q46XyVkN35eMCRIfIMjn4f1sGjjXfjhzfvu8SAQO8+rXXz+tLN3hvRLhHqrb+tDLpJBmYtwqqMiAdOfutWq6uKtdVFJevW8jO/8ArTiQJGmk2cCMcuVCErTe9fBEE+trnx5/XvtXfXtbyRmUbnbFTv6yVaJnoLaVUkCASOW0i0L1eKKlqlacN61ozpgkVi5knzsVRiJ51p1z98Xr3RoPXV9dt7C9qMteyMMNeeam81JzBOXUE6g2BrXz2QUOZ1kfXhaCXb8rKkBDWJLwaVkIbKqt1SSepAUNJj5CwN137eBeyvX9XVCk6k+sOhOgOuUqJGvj91huxmP5URuFP+ZTCuqUthoQZlSTO0TzsIalQpHgCqA5oY+utonZv+rKCo17pzLSgFVUvTnP6+Fi19YpfpXVoReD8KUSMtQuCB0B/HwsAYzETbKjHCpAL3UwNVLqg6QFEFRiIkfXW22X3vVUgNLmDMDSfD5Wgr+2zyVgmtq1CCTNQojXbn5/CxhnHDq25Qt5Qj2u1VqfKbSOLxj4VEYa/qpmaW84+ErScqQCox8vhNhHalxNa2pKFFJESR4nnaB7yxw42goXUOqJ3KXj+BiNvhZBXjZ4KPaVTpB1Ci+slPXn9aWePFmO+FDfQObzVl66rdLEgKI0Pd1+vr3l3K1x0NUqEmcokiQR8uUm1bnMc1IkGodAgCDUbTz38/DXzsUVjp4pKy+pUaH7YzHjrpzHw6WsiuaeSGaM9VZyrqlN9inUEGIAGpnffz+djhqnNglUyf4ZJPKBGvPlap7mOHgtPioEgLk85ETry+/yKHH4OZJU0O9qC7KcwjQ5fMaeHS021YOwUDSG2pVsbxqldmlHZqJBJyxOn19TaMPSJxTjnCnCisxZgJNcm8qNTLTqW1fYKpXe64X2j3FoTIVKhoY5G0LnHynm0eqqpShLkKcCgrNtpvE7QCJJOk2ScF8QF3txiw3ct+UZXdNRUVNNLi1BLlSqle7NfLKUODunNIORW4GW9Rs71K1hGhKE+8Au11lIXDT0m7qpqumwJxDvLtkhxNHdt91tKmkReCkJHdfaStaWnpAiFqSsQZSoFFrC0103FV3c/eVxpuVmsqCpbtSwsUzq1QdStEFS45yeWsW8++M/CKsww1+/ENVd/wCFryf1p6koTVXZUFagtKqpQygGdFOA5jEqBIKmRUcU77unA1ZePD7HS8PpuqsFEblvO+k1NWpvROdpl1lSijaQl6E5TCY1tOrwaKbNNSOA6tO/mtCmx2SjLYatpPQhXK4yYwwVwJuavx/WUl2VuIq1Xq9Gl51T9RVviCltS1KKuzBBJAMQdNYtYrBF93hiXBNz37fN0JuevvS72KqpoFOZjSuLQFFsqOuk8xO0i3kRgjF1XefECi4gcW37yxxdeG6+ncqEKr1BC2i5CcgdTAbDmUqTlSFARIkm3pNgb0sfR+xYlqkfvV3DTroPZi+qZLDKjvo8gqaSnoVqTNqFPQdzbrqTuVHEcUOIuBAs0bKV62rqKcpaoah5AWkoX2f8YJ1mNPr32KB1wU4TEkAjfb61+XSbGHsPUV50qX7rZpqhh5vtkLbKS2pEe0FTlKYjWY1FkdrD9K2pxtVGhKkKKFJUgAhWs7+/4GxXAhUWmyrnxXQzR8Rb57NKFhxTa0EkZUKU2gkx5n60s2A26j7ZbCcqlykFyCRzI0mPHaNLSHxpuVFJjlKadDbKXrvac/hSBlzJ5+CR8epsyGcqJyMl0+ylS/ZSNTt5C3F1pLJ3DzXdULeJTsd5LhgMoYzZVJcKYy66HqTv10+6LH2mXngKiqaUltBCUKU53SY/hAEmZ5bEWDZoXHSArNCtBmPdHn8fnZUTdL7pSy2pdQ5tDae6B4E6nWOUb2znSW5rTZH5ImKqgQpt31N1SGgdVuAIB0hWUCOnhFqN+kXTFvi/fj4IWKksv5+SiplEkeGabeiF13DX3dFYinpW3G5WHKhtDoSRyAUCkk77T5Wpb6bVHUjinQXvUKzLr7paK3A2lAUtDriSYAAGgTbb7N1F6wsvuFido6Y9zD+hC59C6n/e2OcSYVDvZt3lcrskqEDK4gAmd4Cztr94tUOHVUzeVHedNXUyl07yXyhLQyZwyhtSRprIC9uZGuhtUT0QHew9IC7LvIKhedNVsiDz7Eug7TugfI29J7rwq5UOJHYFSU+0IgRoTJ2+Nqnauc01YDfdoP8AZT7N0sdVR+MbEqPMJ4QbovWHb3u2jNZVViqjtGmwvs0FCU5RmAgyJmOehGlnpdeAsPVdO56xdVOiUkLqHmEFI1kBMe0dzz6xvL2p8POOSw02hLSo7jXsqM8zur39DtZdpsOUdMhJqFKWoaBpG4HjytxMuIk6grq2U8UIDbKPKbhjherMIuelcQFQ48ujbAGkTJM/j4WUqbhfhCmeCV4fuqoWUHKF3Y2RmJmAI1Gp5c9rSQzdzi8qWW1JCR3SqCoeXIfCyxQ4ddbkgZVnfWVK8z+dqJrppNiVCSaCIXcAo7oOFeCSpT14YQuxxajISqiaBPQkAaWXLv4V4LcgNYNuQKX7S13YxI8u7oPnaUrlwJWVagpTEN75z3RY9el64AwIAL3vBuqrNk0rHeUVchE2uwYdW1DeLK7IzqTb05n7LBqcZhzcOFuZ3QBRbV+jvhHFNxXncacHXW4LwpHaYlNE2kDOhQnNGhE7i3hzxG4dXxw2xpeeEb/aQ1V3dVO0TyErCgl1CilQmI3B+FvoO/fHE/HFP2OFbmbw3diyUioqgUuFP8yUDX/w+dvC70trkvXCPpA4uw7ftb65W3ffNQhx9ZBLoLkpcMndSSFak7629D7JRGlc6OMuc0jciwNul9Vy+LSvqG5pi3MOQNyPryV2P2ceMMVcTOCN68JcNcMrgcbuS80et3sWUoUtD4zhTqjmKnAUKAMeyEiNJN2uG/o7v8P6lzFl8YkeqrxCFgUzCA1SthW4Cd1eZPLa1Gf2K+NW7vvfijgV1SZfYoL1Z8m1utL/APwjdvTm8757VtTM6KFliUNFS1r5JSS7Sw5BCgr6p0Igi0bz80kDbaDYnerBfoXAAJRCx7rHJ1525MKGU7EG1+E5SCjPFwQmeOsgz42wTMjQGwlS0WnnGlE91ZE++wc5uZ/AW6JhuLrHcLGy6TqOZPjv9a2OUCu8U9Qfl/8A9WKSeW/LnYakUA8kAjptGkH9LGYbFVnahKUmYjY22FQJiPEWDzE6je28wBiLWggEXQmaZTvytuTGhmbBBRmY526Ss9ZtJOhAZ8bdpUOWgsCCTBB1Put2lR+OlphQQoXsZINiVIspq6xhX8LiVjyUkfkbG83LSxMyi95HsvMdOaVf/nWi8bFONkO7oJPxFh7pdLVc06P4FhW9gHNBofhbu7z/AKwD1NmcLhIbqVBqJtlk+nvBHq7WYiciZ+FstTUrFfNrWV1bXJRet51rtTVKQl156ocK1qypAEqJJMJGk9By0tYngjUprKSovBsaPU7CoIgiQTEe+LV2qGgLlSpBRm9WJUCYkZee/M+Um00ejVWqqcIOu5iXG1JpieYIJI6clC1ujFpECqN2KbC+5AheUwJmdD0+utuVPgAwkQFTM5RP19b2Khas2VCu8dBrB1+vvtwpRUCrkkhOXTveevWOlthZeVKDT2dpSQJKVBQGXSI18d48rAhwySnQq0EeHMfpYohamnFdwEK7p1Oug+YHw0tijlcJJUrNqeYIImNfIfnaOymnThSsy1D1KUAlwBaZPdGU6/DMT7rONbyCQgiVITAVO8FJ1+MWjm7KxNBWsPAEhtUkZdxz05GNvGz8SISEoXmSQIV/MO7B+61OYWKu05u2yMMvISsApygQRIB5R+I362AcLagcqSd1eB0PTTmbdylTaSBtuNoJIGg22tpRJUZUqFnb4Db37bWDcKyAuUaryqOhJJ18T87I2Lgs3E4gIGjyFwQNO9p8I+7nZYUnLlJXKTBGg20/GfzslYlyKud7MYSSg9Ce8PlqbQkPgKLH7wUMKqOx4/8AC6pOzV/UMgE/+lta+PP6iPUlpsxMEwCZHW3lZeYnjhw1hZEX5Qmcsf8Azps/n8fHT1bZQopAJFsaUXsrrTqUWUS0A2imzLJnPPs7frboBWVRB1Ow8rGVNDmN/C2nE9xUdOfSzxhQkKJ0ztE27F6MKdaA0SkwZshrQnOpSUQCo5QRqB4+NlRxBzKmd+n142LKbjQER4WtgWVRzrlVf9M656u9qHCTNO6tuKypzZACSOzSNbVsRgG+VFIcrKzSNCEkaeY6/WptfLiPhFjFt7YeoqmQGXKhxJH+VOnvANuKDgbd7jyHiwSknNISAfAQBZxTmQ3Tcfhiyo23w/vNcj1qtBOvdKNP+WfH32PIwJXJSCl+tOoCZCZ93d+tLXsXwHuju5GVSmAJTP8ATexlrgVdqDPqvxH4Wn3MqPegqIJwHeEBWesIBPtZdd9u6PxG9hhgO8gJCKzQiCVDUdfZ00+6192uB9zpVnTTQeo0sMOCVyDUUZgHTS0DQuUxVjmqG0GAK591IcTXgaD24jbbTw+tLLF58Na273TTUVNWrbCBqp32tj003td5rgvcqSFeqGec62Wn+EFyuHtnKRJlA3+dq76B+ZWI6tll51u4FvZHdXTVZnQ/aRI3108rHKHh7XLeSk0tVvAhwxH1+Fr5VPCO5ipR9QB3GwtjPCK6GzIowPCNjafs1xG6H31oOyq1hLg9QVYS7WM3hHMetKSZ6bfPe0iUvBPCqyls3ZW6gZj+8nzO8fxxawdz8OqKmPcpwnTTay1/YulzFXYjWIITFsqfCZS7RyvR4lGOSr89wiwfTtMIF2POyCoFV41BI8j2mnWwdTwswozROrauasStEJCfX6jIZ1JIK9d/qLTveuCahwJDLJjKU5kAfH6628z8cYZ/aNqxbfbWG2sYrulNc8KVSDShJRmUE5QUhRERB3587Uzg8p0LrfVWhiUe4F1ZxXDK6xUKdapX2kJbjWteMGN9VGyUOHFAp1akNqKgDqmqdkADX+L7vO1Y2cAftBK51v8AeNJj5wsHtRD9MgDSNI2JlQ05ac7KdRgn0z3r8uupYuDH10MNPI9Y9YvoOodGcSVZj7O8gRvYLsFfb+oFaZirR8CsJT4Fu9jLLJlpfdT27vd8fasUxJgpht7KGCtJzEkvLk6jUiYn65Wl+mulS6VXb06J9kefhYLENztIC1LYQgI0BI121PytmjDZA+4Kv99YWbKBmsFpddCVUIWCDI7VZg+czZbpuHt1qYBN1hWWIJWvlp18LSUzcbZeUUJQVpakwNvjZbp7nb9VQUtCAAkmOfjaE9DKdiox1cfRV8vrANMg5WrvSlQ1lWvyP32a39imC6lpd3MnOqAC0DAM/wBfoWshfVyoVVju6BJkH5mzZaw8isvgLQM6Ge9qoHcHl9a2NTUEttSgyVMbnbKMk8MbqLKC9dFKpSid2hr05bRbP9F91ggG6qYKTHeDYkyfHn9SbTe1cgdrkJUgJSlJy6aTyI87N3irXnA+B77xGyUdtRUi+zOSD2yhkbPuUR8PO15lJITa6Z08YbchV6xJWcN8LVC6KoqbtNY2ohbDbaFKCphXe29qAdZ5RaO784jYYbzO3ZdFIcpCSpNKlZSMxmAfE6SOXIG0H4mxA8isdqHHnFKSSnvKJKgrnr4qmPDaDYOnqRW0pccKpA0bIKshH8swTtMctDzgdLT4VGwAuJJXPT4i95s0WT3vjiPeb1S0G10rjTaloW2uUKAncKghIGndMb++xa4MT1lNja4L6CkMro71pS2oHulKXRmBnWSFDX/EnlNmiw/V0jpDThAI0UZJBkAgSNyY2nkecEdtpVMhT2dSHCgrStIyKROqCCBMSEEHQg8tBGxTMbBI1zRss6SV0gsSvQXGV4YGwhfd4UeKcU3BS0t5MN1FXd1dVM9sTBAWlgErIUIBASQYEHS1cr89HrhXiJmuxzhelCrsCnOzbu+uz0bikGFZA2SQSdShKkgA6AaS9MK8McO35h6nqWKSodVWzUVL1QhtLri1AZlOEqKnDMklYk69CLC02AcRYAVUN4Dfao1X19nWNICGGe6f78NQWe0RoElKEiFwSqBbpMS7LT1UxrovCHWJAR6XE2CIQSjNba6hG4LiwtgehvnDeJcO3y7WYkpy09R3fRJcRQtLOVDinCrVSRC8iUkqJEETJhVyuvW6w/dbNaXQw640VFpSFwkwVDMkLEmTCo6RuLXbu3hu5dzjiryzrUFFTpceU4tSlbrUtRJJOWdSdtY1mpPGTDLmGMf3zSIS6Gapw1rXbd4kLGYyfNIPKfG1TFcKdTUzZCNAbIImDnWCVeF/pBcUOE1b22DMT1NHTklL1F/eUTxO4cp3B2apEwSnN4ybXO4E+nhhziJiSgwnj+7kXdfl8VQYp6ykcz0j76zoFhfeaKlaAyqSRsLeatSvIwiFSRBUDtyEA78o91nFwh7ao4r4PaTmWVX7QAJG5PrCBvbl5IwRcozHkFeqnHG6qV7EN1V9StMKo3Gy3qCtSVmBMae2qzAYuXOlvtXWWGPZTlkz7iJVEa9J91pm4yXS9UouqrbpwsIddbzLXCAk5TmPX2fDW0dUNzrdWG0I7VU6JRJA10HMW82xl4jqnBem4I3iUbD/ADdFKG52QAmhbSt7LmlaSpXIa6QN+c+dly77odox6s4GHFE5g4pailCQNyI3205dOqvQ3HUttQ5UlKj3C21OWOijME/GzmufDi6hQWtpa0aEhLkfPbf62tzU9UG810LIRa6bzOHWnVhKn0vKMI7V3MAAdZSkCE/MxtaoX7RLB5uqowNiBqVtVbNdRlzIBJbU0rQHl3yfPpb0EpMOF9SEppwgJ2yDl4nfkLVj/aWYNSrhJhzEKlrcXd9+dgpaRo2l1hZiY6tJ26c7W+zeIEYpGL6G4/Cyu0LGPoHs57qkPox1f7u9JDh/UOr7tVejDEzH96C0Nv8ANb2WoriSWkNtAhAOYlSjAJ3EbTbxT4cOruXilgvEJSoC7r3oqlRAMFLT6Vmemk299Lrwo7VFMoXl5SJ0tr9uYJKmoh4QuSCPz/yuZwCvFFTva421v+E0KW4EKgoWrJoO5oTPLUaD32WrvwsXIbbbSE8zMk2kCnwlS0jIfvB5LLaBqVKAgW0nEFAyr1XCt1OXk+NO1AhtJ/zH8Lc9B2dcyzqx2QdN3H6Aaok+PSS3EAv58h90lXZgNcdpUIS02N1rMe+Bbt2+sK3C56ldFK5fdeNmqdOYA+JGgsqqwrfmISHMUXqtDMz6pTHIg/5jubL13XRc9xMdhd9G20kcwNTbp6PAxEA6JgjHzP1d9m7D7rEnxDP/AFnl56DQfc7n7JlvXBxDxn3b2vRFwXar/wCb0ozPKT4q2Hzsr4a4bYMwco1NBdqHKxft1T57R1XXvHbyECy9U3k2nTOJ8LI1Ze5JMKseWWgoTn/qSdXa+g2CCJamobw2+FvQaevM/dLNReLbSSlOg5AW8I/2m9ym5/S5xZUpSQ3eTdFeCARvnpmwT/xJV8Le1VZeiiCSvS3kn+1suZTfGTDWJUIHZ3jh9DKlcytl92f+VaLTwTGnVmJiNx3BspS0Jhp3PTX/AGW+Lk4a9Kr92lwJRiG4a6iAmASkIqBHX+5NvXqrvYTova3hB6IWJ3MJ+k1w4vhCgkOXyxQrI/kqJYJ+DlvbWprZTJWetsjt259PWxvbs5v7Fa3Z6AVMRvuCpDYcDrLbwiFpChbsn3WTsPVAqbmpXSZOTLPkY/CygRPS3RUcnFgY/qAgzNyPLTySDflNlqQ9ycHPwsnbQmbOC+WO1owoAyhU6dLIBkHTT3W6KlddiyahtnrBpqEgT1sIyrIsKUNiPv1+63Bk6A7cptitQcupIIHvFrgVN6UgsEQBbYPLadCbAoWlaEuCNRmAt2DPTzm1oFVl2DJ/W3QJjbQ2DzQJJ+dsMbREfK0wSUkNm1mBJ0tsKFgcwkazHhYRK4OvLnraQ3UShQfmOtita4GqmjcUDBcLStP5knf3gWHzToCd7Er5JTQKfjVhaHh/uqBPymzu2SG6PLAGsW6oD9tGYb/KwayFAKSeVuqEy/EjU9bPa6ZOHtgnTMNNLZbjtY0z/Xwtlo5U914CjDF51TzeGKVDS611RpQkudwrHdOp3E2lLgZgHEmBbtvS7sQNMtpfqUuMBt4OSQkydNv4dN7NCgvRq78XU9/PtvFCK1VQ4EEFWqiYGw/DW0nJ4tXD2edVBXBJ1MhBAGvRX1tZ4A1pzFBmzOGUBO/snDyVCvPrO0W0hAzQsaSJ2BNmsnilcqT9jQ16hoTojQ+Pe8LCOcVbneWEro64qjeGzz12V4/0td4zeqqmF/ROMIQE6iBB0A8NPLa2EAqKSJkgghQ1Ph+fhZqjiZcSgJu6vnLzCJI5D2vz2nlbj/Sdc0hSaCs7xgwhHPkdfL61tHjNPNNwXdE7MkEpAkgaCJ5bfXWz3uO8F1d1MBTiitr7NUSNdPwj52hb/SfcsEpu6uGwGUJKZ6b2WLl4zXNd6HBUXZeC0rWFApCNwddyLCme0jRWYGFh1UyNr7LveGmnlr9dLBqKe66VLOXnA8/wGnjaNFcecMoUpSLnvUhMn2UaaHX2uv10Bc4+YcUSpVyXnodT9mJG++bxtUuVcu1SgVlYKu6MiYSIAGk6eW3wslYsUHLldAaUkKLaRCZjvjU+8g+7pZhf6esOp7v7kvFUrAA7kHb/ABaR8fvsh4g9IXDNTQrpE3DeoUohRV9mUgAjcZtNh9A2hJctKnG5oIuka/n0N8aeHC0AJSq/aIZVT3pqmjqI93xPjb1gYTz8Zt450WPbvxxxmwC5QUdXTt0t/wBDKX8uZU1KehOkJj429jWMuUqtnSggC6sxkOJIXZA8bBuCUkT4jlFhZzDU8uti61GNRoREE2ePdQek9xJKjGvj+FgHEgjbflYw7lmBpvpG1gHCNSdevS1oKod1GnFvifcHCJVx4mxGwHqR6peowQ6hspUpoqBlZA2R5xNm9d/p4cHKJIDwS5IEA3ixz13BM9bS1eXD/BfEPs7sxvhW6r+oqUmpap7ypEVDSHQICwlYICsqlCY2JsJSejXwFa77fBnBDcq0/wCwabf3otaiY8i7TZAe9l7EKKnf2hHBptJKEUupgZr0QNJ30QbBp/aLcHogJu/XSDeit9dNGPC05U/o/cEWPY4QYKTGki4aX/4NbKlPwW4PsgdjwqwinyuKlH/RYhEt9XfhQvH0VdHP2j/CBoED90CP571dHkdKY2B//SQ8K1hRaRcRSBJJvZ+I6/8AmvutaFrhZwyb/uuHWGE9SLopxP8AyWOtcP8AAbOreCrhTH8t2sD/AKbMRKfi/CcGMclVBX7RnhslIXkw+kHc/vWo8Y19VjlZx8P/AE12eLl+jCPDzCt13xeop11RYavF5IDSN1Fa2EpGp2mTayzOD8JtK+zwxdKdI7tE0PuFkXEeEMOUuJMOX22qkudVK5VNKfZbaazBbJGUlSYiQNOoFo8OXcuUhKwaBqYycccd6lvtafgzd6klZSCb6b5Eg7gcxbpvGHH6deEN1thO83u2f+sTaQ7rvDBeH7vYuX+2NI8GQopdrL0bW8tSlFRKlFQkyr7rcv4vwBTuFD+MLjTG+e82Rr/x2nY295Qv/wCqZNLi7jyCFL4Z3WhM8qtC9f8A2wsrJxVxrfAIwPdSOpKkKAHT/wA4squ4/wCF7cJc4hYZQNfavdiYHjn8R8bFHuKHBptaVK4m4QTBMhV8Uxn/AJ/CwXRB27z+ERshHwoL+0fGwoQBhO6Ukci2g8v/ANatir341LaKv3DcbawJGdsR8qk2Bd4y8EEgpc4p4MSRv/2tTkj/AJvOxNfHLgCwpOfivg9J03vRgg/81h93b859Qp8U/KEkIxfxgqcT3lhdVHh9ipu6npqtwuAJQ42+XQnIe3JMFlc6CNPcBV0PF2tWS/V4XKCIID8Ry6mLN9rjv6PlBxlvm/HuI+GvU67Dd3MesNPBxtTjVTVnICkEZgHZj9LOR70p/RoZlw8T7k5SEtOqPyR9TYJpWOHiefVHFQ9uzB6JNGF+JACQqvwwCFSAKwiAP902AvLBXEqvIKL2ww0Jkk1qlSTz/uvL42UXfS79Ghru/wCkuhmD7FHUE6eTdi6vTF9GlOUJ4hNuFX/d3dVEdf8Au7B9mU97lx9UX2hPawaPRIqOH3FJDhJv7DQVGXuvk6dP7n8LKFFw84oBsoOJLhaB1MOFQA6asW096Z/o4NwP7Z1C9/YuqpP/AEWJu+mx6OiFjs8SXm4BAhN1Pa+RI8LI4bSu3J9UhX1LdgPRavXhpxCTUN0q8d3Il6pQtbaEoClLyiVR9lrAPum1frrvX0o3kmsu/gdip1twdol5D93ZXkkSFJzjY6EaDe063h6a3o3VgT67UXnWBsktpcucrCVbGMx00tN+EKmkvPCl13tQAopaykZqGUqTkKW1oCkSBtoQI5WYYZBfwE+qXf5j7w/CpWh/0vVLC08E8VMKCplVdc45eI5bWjT0hr69ISgwCpjibgW87humvq0NIfq667nQt5ILiW4pxnH92TOg7oHO3pe42lWuZSo130FqQftPb1FLhPBOH2n1j1mqraxeXcdmhtAO/wD61VpsoY43Ai6d1bI9pBXmpfleaipfo9VJBykEEco0iQdvOeY3OsPVqFt9lUIQS1o53u8Y7uvSZHWNTFi150jz7/qxfl9CU5QQSV6+0k7GdIjUiPKyVdC3Wb5WioQtCghasq1FMq3106Hn8RbSGipHVPNAh5KwpQJVBSQO0KVc/foROghR3GhxwsOMJNMlJSheckJBJVAjLGkbkxGknUmyQ68yKY5ohTeVKUxmiQZBJ5jLygag84VKdCHaft22kkvZVgxCpVqTqBMd1MAnaLEB1QyFdHgTiFN58Lborbrap6q8KakQy6pwCUqQMpkJPtd0HfprAssVd4XpUX3RLvOnYC3g8zlQgBJGRRA0HRJjrA15Wjf0XKxdZgN1ph5Lz1FeD9P2mYg5VK7RAkRMhwGfPraWb/rGbsoGUXheDC6x19DFKvLnzLPdEjQ5UiTPMa29ao5jNRRvJ5BUMoa8oq4xTUzLeSVvvqLhStSkjLmkb8tU8to6gCsnpY3O36/dWJkNKyPB6lXmlIJIC0ySY0g+Gm2lrTUIrXaVs1iaUuinCXXmimFLCf8AKmBuRoNOdoU9KO6BX4BcrUTnu91urQtAJIAVCh19lZ6T100p4vTiegkHlf01RGutILKlt6UjrKAtQCm80CE90adR7vL42mb0HMDPY39JPCVMFJDV1PKvh8qVEtsJKoAG8rKNLRXedCqppKgtKW6UDNoNIA0M89Ofh01tLvoD37T3H6UGFRVultNciqo0cgVuMLge8gfK3kktw0rVi94XXqxxKuxuvuimzoSQ3VJV3gVASkjbnrGm3zs1rnwglynQtCg3BBJVoOegSNTsLLPGPiLc/Dfh3e+PL9u+trbvuZLT77VIhJeKVOpQCkKUBoVgmSNAedq5UP7SXgLTJSo4Oxok5o7tJTHl/wDTjW3mfaHC6+qqeJSsJBC9EwTFKWlpOHM4B11ZGkwlffrlMzTqpRd0y6VpVnnllSNDz18rPShuFCkpabBCQZUY0nymLVPb/ah8BmVBDuEMbDkQKSmk+f2/SylS/tU/RrZCUqwjjxShqR6lSwP/AL+3MO7NYxKfFEbK3P2ggI8LwrfUVwNlSQlRjSTl3tDfp38Oq3EvozX8q76QvPXZUUlekIElKUOhK1dAAhaiT0BmwPDX9pF6K+KWKqorqnEFx+rIzobvG70y+RulPYrXB/zZQZ3tHPHz9qBwsvrAmI8D4Nw/frKr1u6oom6ldKypJzpKTmlzuggkTBInrbWwrs9NSyNlkvnB0AFz/gLAqcRkqTl+HmSV504YF13Nf91VF+NOLpaesCqhpqQSkqCjlUecAxuRmt9AuEMXf2uw3dN9YOut0UN50TFWzVVQy/ZuICkmJ1MEeFvnIvbFlPeLv+rsraV2gXJAiNY0HmfibeqPBz9rN6PODOEGDMJYnuTGH74uS4qK7a40l3MrZU8wyltakKLwJSSmRIB12t31TRyyObISR1sLn7dFgPkYW5Wi/wB/5dX/AP7Kt1boqL+rXK5Q1CFaNg+CRpZVQKGgaDdMyhtI0ASItRB79sX6MhScmH8en/8AZtP/APlFkiq/bDejg7OXDOPv/wB30v8A+UWpOZJS37pTku6kXPqULK6SwkeLdL6K/dVe6EJMKsjVd9Eg97a1DH/2u3o7uJ7uGMee+gpf/wAosnP/ALWn0fXdE4Yxz/8A4NL/APj7c/Ww41UknhOWhTxUrPeeFeuqveQTnsk1V7knRR91qNu/tXeASz//ACvjffnR03/4+xZz9qlwEUSRhbG2o50lN/8Aj7c/NgWNy/8Ahd+FsQzUDN5ArtVNepYMkxzt59ftWbjN4YfwPiBtAJpna+kUoH+YMrSP+RZjz6Wcq/2pvAY7YXxqf/sdNp/9/aEPSt9Nbg7x34fUuGrgw9iZmvpa9NU25WU7CEZOzWhQlLqjMqTy5G17s/geLUWJRVEsJDRufqFOuq6GWkfGyQXsqdYPvtzDmLbhxEyqHbqvGnq0naC26lY+63vO24qqbQtkKUHkhSMvMHa3gAXGoOUrCs8pECI8+u1vQ+6v2leC6PB9y3HWXfidytprspqas9VpmaZgvIaSlfeDxdcTmCjJWienK3U9ssFqMWEJp23Ivf72Wd2cxGKi4jZSBe269LsFFaLnNM8QHGXVJKc3eE96D0OuxsvH52qt6CXpE4W47XfjCmw7RVVG5cz9I+8y/TIa/vw4AUlK1Ff9zqVGflFqCTy57WVBTSUlOyGUWcBZFqpGTSufGbgoN9CXmltH+IEWajoKVqQT7Oh5WdiiYjptZuXs0WaxRSNDChHW27RHUhZlUNMyLFcmT8YtmbY6H3/hYMqG+Xa2T1M21AFmk3R2mUexSAT3ZR7gYsLPONTpFidOsHOkKJAVtzgpB++bGAox7VrLdQq50KFJO2sx0tgUDr99uArlI91thQnY2INEyECpgHXw526QSICTt99gSSYAMaW6SoDY677WkFBDBXXQddrc1CQ/TuMqAIWgp+ItyFDUdRtboq52kkuaR3taNlRkkoE+ca2M0J+3G8DlZPu+Ay5TwPsnVpA8Jn8bH6IgO6ybME3NOEAwNVjT+U2ywE/4494/O2WbVOvBLFj1RdVzVFVSOBL7CUqCikESVBJEDQTJ8JnY7POl4L8R32EKTjO6klSAsRSHTNy9n7vHaTZp4+addw1WNpRJWWyB/vp28fz8bXP4bcLsVcRlmlwpc79c5TMhx1LSUhKExuSSAJ1jXrFngYx9y9AqHvjtkVa0cEeJAUD/AGzurLvpSaAx5dOf52OM8B+IjmZs4yuwqA/9CO/w+ptYiuw7X3Jejl1XrTqZqadwtPIdTlUlQ3kdfMWn7g36OTeP7mXfTleKdpIyJzJzZiRsI5DT8bHfBCwZjsqzaiVxsvPRXAviUpIYRjCgccyapRQKJnTQaz8bBVfBLiRSns6nGFG0txEoCrtIPmJjQxHu+Fz+LnDF/h9fT13qqJLZA7RM7bxpr0+Nogryt1Z7RS3FqTlGYyoDkDrP372dtPG7UJd4kGhUCK4M4/UEhOPqGMuXS7xrvO/ntbocGcdOlLTnEOhSUKETdiSNNf5tt/qRaaydZKQJEaazpy+NgVCEgISZUNRMeOtp91jS7y++qjZPoy8THvs1cTbvPabkXYJI+Omn4eNu2vRj4gKzLd4p0kHci6ZEkbyVdJ1+dp/uCo9dulp7tSVtQ0oHckAR8RZWaaQlQCgCI70E6kae/e1F7A02WrGA5oKrafRZxmpUq4uU4MkkJuYaf8402+hZNvv0VMTUtGqre4rNumUpUn9yhJgq5faTufna0ak5kqDigNpBG8yNpsk4uJNxvvrKRlW1ISQZ+0SDHjpvYLxZpIKO1rSbWVJcH4aq8J8drjuCpvX15V34guuHUNBorCnG3CCkT1A3O3lb21p9WwSCR57728a6tzJ6TNG2lQM4hulRKdI1ZG46EgW9ladOVkJ13/G2fMSbFTjADiAuiBrCjO1i7o38NrGzH8p157WLPEkRPO0Y908mqT3x1PnYosD6HKx51IVqoSZsUcAzGSdTztdaqTt0g4jx+3w6p6e9nrpdrvWnxShDbiUFJKFLzEnl3PnZF/8AKhaCspwZUHKo6evJHOP5LJXHdSRhq7FBWSL0SJ56sPWhcrCFGUgjoTP1z+hbSpxdiz6hxDtFYJPpVZDCcBu6fzXkAZ9zdulelo4lHcwGD4KvOBtyhq1eC6ADCoyiD1n89futvPKiCZGx0kR8I6fCxiwKvxXBWEHpaXidRgGnTO03kr7+ysEr0tr7UTkwPd4CTBJvFZj/AO7tABUZKDBIMAADUe/3eNtFzLmKNdAJkb/X32YRhPxXFT9/5WeJCPssGXbmJ29ccP8A062jPjrxRxDx4uK6uHF43XQ3VRXheIW7UUzq1vJ7NpawAFaaqTB8LM5DqjCgkASIkCAI3tpt4C/bjWFjuVsQQebTg/H5+6zujBFikJHA3UeXn6N9yXS+lly/rwd7T2VBLYA5aSnoBYJv0fsMEBJve80iDoFNa6bexvaasbAB+mKz7aTlPUTIizadS6QlNMop111gTpYLqeIGwCK2olIvdR2jgBg1IPaXhei0lUd59AJJj/Dp/X3n6b0fcAuJUVPXooySo+tAAAeQ2+t7OpXrCNFE9yCYnnH6WU7sbqVqLocJZSTIBJ5dP0snQRgbJ2zyE7pns+j1w+ToBeUE6j1uOcRtrzEWFPo/cOwCVUlYRuQatR5eFn22+JHdAB0nbUTYwHhJSo5tYPXw/pYfCj6Iwlk6qKxwZwJS4zcpPUaxdIbiRU9gurWUl0VK2yrcH2SNJjawGI+F+AqGgNTRXIUlCtvWXTmHT2uWnjzs/wCpj+2NOAlQL2HqpGpOvZ1jB06aOfMWJYgbS/dlSkojKmdfDX3bfdZNiYWnRM6V9xqq+1F2YdocYpoauhSiiNKt3Kp1YCSJ55pnQWR10NNX095111uf6vQZ1JVmOZahEASdEgCyvxBCXLzu+7GVILl41CGSJM9mkyr7wIsnXShLFNjChIc+zUslO0pyHaNfD4+FgcME2Rw4gXuuL8TRMYQu69qenCXVuNhxRJMg7zr0+NlDF120jF001TdlKlBD7bi8p3RHekg7ADX39RZv19QHeEFI5lGdCWiJVlg9oNZ5acvEWGxxfjQwlS0/antH2mllKFDNlSBOnL9dLOWNASDyUvUNDSXpX0l5OUCWqBVShDKQTDqTzKehOvPa0gMekTxsurHtfguk4k3wxdF306WqSmYUhKWUpIQAFBIMACIn+GzPerW2rloVtRp2KgpIGUdwxrz57a/IWI0V309+8YMXFXaJdoH1BsNuCSlSlSTrBGgPu85rl7Y9URoLzYJ8Yy9IHjhQ1tyCk4uYkZZqKrsn0prSMwzJBG08ztraNePHEHFONa6hpMSYnvO+F3e0tTfrtWt/ss6xmCcxIAMD4DrZ/V+DLjvFFNU1dS9FKsKaIcgJjcQNP5flrFoa4x0dLcWJaxlpwllNGzUOBagpSlqJAA1E6GI6Zd7RiqGSPsCpuhfGLuCjW9WbmZYFZeTrocHeATEue1oJ13nXpBnqhXU+aquXUKDjbKGyEJMkJBVpvuAfu99hU3bX30+XnnA0ypQCUZ5AJPw2AJ8culuKBz1e/wBxpa0rIZCTlIOWNNCdPERtp0i1y6CAlStIQlTq3G0ZoIycuZ567mSTqCZ9rRQuFdL6g+yttE0jhCyCnKUqAKRJMGCFCdD4kwLJNQ/nRmU4MpOUzOkAAakwZjYfqQ7nr2qW9w0FJyvJLalZAOzcnTWJGo5awrrrZwUiLqyfow34xTLvi7XnlNKqa1DwUteYZVMp0JJ72bLI13OgAm0xYii8b1ZbpQrLRvNLSEoJBOcRkERAI3HkIM2gX0cK+hpb0v5h2mbfdaYZdZW23KQsqVMa92cqToNhB1Gtg2qijuG5U194IJqKlxBKFwIWVARMyImTOwSelvRsCcZ6JrCdAqsoyOzJTfevlbgRTILbTOeQFQVyUgqnfaN4jMN50j/jXR1tVgesoaynQl2qpXVEqCiJGx25k6CBAjbe0rUbl7UIFJeCkhltxZ7VB9mZEQBI/jnU6kbSRaD+OlVijGSqy67mqV0VPTJUkrRIUBGoEQBJJlWsDpbWrv06Z25JFrIAsXKoFI6s04qFZlFsqCgsghMAaEnlJnnv1Ih8ejXcldXekRgahuZKg+zf9NUJUknvMIV2ijOmyEqnaRaPaEKoXaijWQHWVqQogagpkE6eZ36aeE2ehzUXm16TmCn7vofW+0edZfIJORvsF5l+BSmTr0iZt49OMoK1Y9SAvUXiNw6Z4q4Fvvh69WmiRf8ASGj9ZDQd7ElQIWEEgKggGCeloJw3+yDuh8pdf4qVrrR1KlXO2lIEzElw/KbWww/VmjvelebTT5kOCO3MNEzAzdBrvy3i0tLwtf8AiBXaYsxKlFJ/6Dd0ttqH+JZ7yvkLcfiT6lsrRC426CwH1JO32W1E9jW+Kw8zcn7BULxx+zN4F0uGV4duHHF/Xli3OlLRoqZlxAE94KQfZ0nXtBqduVkPCX7GFm8UCsxbxdrLvS5B9XYuxtbgA2lRXA05Cbell20WHsOUwpboomKdCf5EiT4k87Y9fiQD3xapHX9yJdUTX/8AVu3qdf2UZJDMMsTPud/QaD8qgzv7GvhmyiaTjVilpyAMyqGnUNOUaWbOI/2OWDKS6KyspuOF+u1DFO4tptV1MhK1JSSkGF9dPfb0Nqb8PJXlZHr73C21IUuQQQdbZdX2pyG8Rsiw0U0lg7ZfOJiPD7mH3+xW4Xc5WknLlhQ/rPvsgkk762lHjldT12Y2vu63GC3+7r2raWTpOVwj/pn3j3RcbeiUcpmha87kLLqWCOQtCwEjrrbWpMzZwYXwBjbG63GsGYUva/nWBmeau2jcqXGkyAFLS2CUgkwCRBIMbWc7Po38f6gEs8E8dKCQVKIw/VQANyTk0HjYrp42GznAH6hDbC94uGkhRvHj8rZH1FnyngbxmVd5vUcKMX+pAhPrH7kqezzHQDNkixu7PR44730Cq6eDuMqpI1zN3JUER55LMamFupePUKXdpT8B9Co7n6m2ptKn/kq+kh//AGOxt/8AuV8f9NtK9Fj0jU78EMaj/wDYz/8A8Nm77Tf7jfUJu6VB+A+hUWTbJNpTT6LPpGKH/wDRTGI0nW6Xh96bIONOC/FLh1SUtdjvA173DT1y1t0zldTKaDq0pzFKZ3MWdlXTvdla8E/UJnUs7G5nMIH0KZxKOzSQCVGQokaDyta70W/Qww16QnDeoxxX8Qa66KilvV67V0jNCh1PcaaWFZisHUO7RytU4CTA10tfn9mji1LeHcb4UdcSOwq6O8Gkk6nOhaFn/wC7b+Ns7Hpp6aidLTmzhb91pYFDDU1jYpxcG6tD6Efow3T6N+NcQuXRjisvlrEF3NtOMP0iWkpW05mSoFKjJhaxEc7XEUAB0tAvDe+P/l3d7ZX3XS40T5tq08dQLT0sgm3N4ZVzVkZknN3XXQYlSRUcoZCLNsuTtp8bJd/U6i2h5P8ACcp8rKZE6T8bB1iQ9SLQRrEi21TuyuBWRKMzSE0gQRr8OQt0NQAk67W6W3lXlI3+G+otwQRzJ87bQN1jrthSU1CtNFtg/An/AOKxoEakjfexFJipaXOqgpGviJ/6bGgokyNxroLWI9UF267WsBClRMAxA18rETfjYb7Ruiqz3gkJyd6T4T4fUi3dQ3WLUg0tSlCdMySkEqPmdtJ1twwxegMvXmFgH+FoCR7/AH2sC3NQWv36Nhd1aZ00b8/H4+Nh03oolQRRvrCUhSTEBYMbTz12MWBTS3j2SgbyPakkpUGwQE6QI5nTfxtgpbxJSTeikpgZgECVEAbdOvTXytMZVDUJQpKpVQz2qqdbJkpyrGog7/jYxm55vOyQijvI6qvlRBER2YAmInrM62U0kaAnUDXxNmPknC4YHZ1dSkyO0SlwT7wfuFlCintZI21M2TVKCK9gn/aJWjptB/OynQwHdQNdpsyY7pYJcn+P/itlu8q/5j8D+Vsskrrwix0Ut3O64AmVvN5UgR3ivn035G12+DvGzFfCRblbhlVKU1bKG6hmoaztuAbEwQQRPI8/G1Hcfqi7MpTJNS2kykbZtvLzB5xvBtvgekbrr8u26n31N01ZUM0dQsESEKWATBHjt1Fo09g1xdsq1W7IA7pdODEWNr1xhieuxNfjra66ue7V7KnKgdIA5AD5a2lrhP6Rl84Au9y6aZNPUU7hBh0TCo3GotCOO2XbrxZet0VFUKhd1Vb1AhQBnIy4UjyGnzskCsW0vVR7u+5nra+A2WMdFRa/4gpE4k8R67Hl5PXneVSlx15ROZOgHwtHVSorzKKwrQ5v4swnX46726FQt0FOYBRkmDqBtH3/ADtyumdySUgxA8+WotJoDdAkdURX9mVBa0lJ1JmZ1+fPzsC4gJC+8mSDmnw5mfvsO8ypkKGcApOgkgzGu+362Kr7EgGPaVlgiYAPTfaetpqVku4OrS1eDlE4MyKgFQMmM41HxmPfZ8inKgtbYPdCiIEgaSPPW0UtPLYqEvtuFK0rCx5gjr5fK0rXZUoraFmub9h1tKiPZAJ3Hxtm1rcpzBalC7MC1a7BRILghMQCDHlHwsj4kZUbmqe0AUlKm1GdiM4PPx+FnI6krSY7oTJ1O2ug+unKyLiUpbuKtKkKOVvMAP8AMJ+Gvvtnvf4CtNjRmCp3eXZf+UpRuoSQRiC7EjSRopnaNDrPXTpb2UZ9kSdZt41XshbHpJ0a3m0JWu/LqJykAkHsTOn/ABafdNvZlhsqG8wdPC1N+wKdos931WKA3IA8bF3AJOXTrpY4ptIAJEE9dJsXdEbAd6zxqMlkQfT/ABGxNY1knfbWyg8kEa8uosVdGkHny+vfa4xUnbqLePCJwrRKRui8myD/APVOjl52gtSVESpwiecTawPGulNRhBsSO7XNK1H+FY5+doL9VOQ5IMGNOk/QtpU7hlWfUC7kSKHE+06SQZIHPQ6WyEhJK1bEqBj5WNCiKl5QrlIMbDmQfjp/Wwhu45gsZYVHTQ/Gxy8KuGEpOWnUHMesgb23lURmUo68pkedj3qLijlCisAGdJOnh7xZSGCsUuaowxehCgYUKNzaPLptZuIBumLcoSAAAoEOCR46HUaRHXS28sXpdBKlaXg0NY/ilMfOzhbwXiYmFYfvBASI7zKkjbTePGwNXh2su+puk19C5Tl28KbIF6EgOoCo672eQPa3MRomicx7soOqU8ZsNvOUzmZRAQuIMDkdf6WQH6SiYbJqHy2FzMqglQ5fX52kLHFFSU6KPMNcq9zue7zmdvrWzDqqKjrlBpzVKDMAxrEWBmzOurIZlbZI+e7O1ITUI7pglKzGpPTcfjZZux66lMFtypl9xXdbBIBP1Fk9Vx0CVplWm8lXQWV7pui6FINVCs7ZzIlU9I+vO0pDcJmA3Q/7uaUkuEKB5iSLdpu2kV3SkyQCRmyjx0sYbcaQgAqA0mI57zba3QFE5yEgHQnW1Y5lYBaEitXbTPcRcLUj8hiqprxpV5SQcpcpVEAnnAJtYo+jVw7rad6neqb5hYKFZ3kAHkdkTavS6lDGOcIVadOzq69uVA86dK//AHYteqnKF5VpU7KpC+6DPlP4WdhNyEQNadVX4ehbwGr69ivqrvvSrrKdyWXFVzqQ31PdTvFq6q4G4Iub01bw4Su3atzDF80dNUdh6wvOsKp3CZXIVPaI5Hp1t6FpLKKxFO264CpQMEgc9rU44vu09z/tEcEValhBvO5aJkAfxK7V9ESOcdfw0T9BdStqpFV6Gno5U92i42MDldCjRTLl41K0HWYADg5iy/Reh36NtRTobf4X3W8lXdX27j7q4GwSSuQOqZg6dLSUimSlbgNP2ozGSpBUd+tlKjWhitQ00ltDaVbjKB99nyhTK8oeJ1z3VcWLsSYeudopaui86mjaZjRLbT6kITEyISNz1+LYu6vbuPi/iu8jop+haqCDPeKmWzv/ALwPPU2lD0iOxTx/xndAZCnBej7hWhspzFZC9B5L3/O0NY8qv3VxJdfhpLddc1ACc2i4ZbAB96OfXxtgTvzZmq3C0MIcny/idx9r1dNMUzBUEg+O4O+p+pNq/cfMRF/ESqenq/tlsMioCTp3QrKPCM34crTSw6EpyoUSkkSY358vM8udq/Y7rbiuvEF4VlawuovJ+ocWULSFKPeMKHJIMCNJgDTe0MNitIXKxWyXYAmrQ3UpTX7wvmsU3StpDmVLkK1nQRt9QDNidxEVt6POMjszULCUJ1UQkqGnWdtfzsSve+7yvJXZ1CuzanMGhonpPjt+UWPYWYZX21TVEpYbGVRyk77bAzoDobbw1KyksVyUOiG15iqVKAVJEqVB3BIjXWJnxEAUNMyh1Km1r7SJXASoBO55kcyI59NbdVVQqtdW3TH7NRhKQgqBSNJA3946z5K1z3a52SHnyvtFpzNk5VH+Hr1kE6z5SDaQTXspU9G5+maxnedOtHZmooGnWyXEqSpSXCConY6q0HtR1nSxNUmkvq9WaN9C/VKZKgooBObQTsfPQ6E8zAip3CtxN3Yu7OnXToNTSko7QCCCpBBTH8UAxA6nu2tfgHLWAO1YDdMwTnzQSogGQJOqTJEa7jfe3oPZQ54cnmqdUbJ//vKlvC7qZ6vQhvM22pwkTBiCoHLI5eMAdNI1xBfdK8XqOnpEvOOLWArKYHidNDHUTIsttXbfl7FdTTlTFDXIb7FlDZSppABMgiFEkRoTpJ8ij46qbg4Y4bcvK8B2lW4nsaWnBSVrXlA7oMRoB3jEAa7ablTNI65Is0c1ARty76qkXEe628McQ7zoG3dA8KhfeMIUtIWU+4k/LeLWo/Z3cP8ACuLOId441r/WfX8IUjTt3ttuFDa1v9ohTi41VlSmAj2RnJM2pvi696+98U3hX3opHrD7pK8iRlkcgee0Ttpppa2H7NTGDN3cY7ywvU1CUpvu43ENIOmZ5haXAB45S6fcfK3kdeQ57y3a5WlT6EXXpWtJgoCimRGvI/XXp4Wk67cXisuqjqUOn7ZhDmvikG0aEZACJBkEHx008YsTwzfS0Xf6kt3WmqamnA27qHlpT8ki3l/bR0kMEckZ52XW4HStrHuY7lqpaN+5v9p75sWdvmBAXy62Z7F5LIkqgdbdprHnJKQVC3mMlZKRuumGFsZunE5eqlbqsUerVLzEqs3b+xJcOFKMV+K78orppyCQupeCCqOSU+0o+ABtENf6XOA6y+hhfh9QvX3eTiHC29VTTU2ZCFKjYrJISYBCZsSCkq6zVjTbqmLIotBuvO/0xbo/dHF3G9IJHZ349VJGXbtjnGu+zgtWVY70Hfr1tYr0l8Y3lj7G154xvqjpad+/GmansadGVIypCAIMqBhB5mTm8LV4fTleWidlHc297wMPbRMa/cALgcUt3hxCsl6CnGK9uEfEe+nLsulq8F3zcy6YIWCcqkOoWFRmE6BQ1nfa3qbw8xXjjiVgq578rXbvpGq9txLqC0HFMOtOFCoTAQc2ihKSRmiTpbxm9HV15PFy5KZh7s11Rep80SO80vlz1j329XPR4r7wuS7X7lqL0dqKRCzUNIWkZW1rAC8kCQDkTpMaeJtzHa+kbm47R4tLroOzt5Yw0C9id1NzOELnbfTW3s4/elSmMrlY52gR/lT7KfcLKq6plhIbYQlCQIASIFkdy9JTmK9vlYhUXmn2s5PlztwJc9xXVMpC7V6Vn7wzSUka87Enq/2hm/h6WQai+EJIBJ73dPn9ffYkq+0FwIBVOaDyOtpNieVbZAxqWHq9UnUkjpao37Raj/e3B65rwSlS1XdiGnKo0hDjTyDt4lNrF1N9CNUE+AE/G0Fel4j9/cBcTsgEuUSaeuRB0T2L7az5HLmtt4KHQVsT/MflVMWjbJRSNHReVbYAeSlQnvAGbWa9AG/1XZxbvS6VOEIvS5XUZR/EtDrah8s9q13m32N4VDfLtFERGxMjbTaLSn6Kt9m4+PeG6kKypqHX6YiND2jK0pHxKbep4lHxqORvUFeX4ZIYayN3mvU7DV6+oYruisU5AbrmCoyNElwA7+B/C1tVSBsR77UOcv1xpaVoPfTCkqTvprP0Putei6a5N7XPR17eqKumafTpyUkKHyNuLwhpYHNK7PGCHFrghdTO8W5UNNvObCEd3aOlgtSQdd9bbrVhOTerGexfcbKdAZFi0QSN4iLKl+tlLiHAJBTFkjOIiSZM22YTmYCsiZuVxWLARkUDP2iADPUx9xsaUDvrMWJvLzU7gTuEkiORAkfOw5UFd4bK1Enra0xVXLuNASB8LYJ3g624JCjIMQOm1sJOsWM0qKEBAOs+E23MjYx42DmCbZKgO8SOh6WkN0yHCjPSLdJOkA2AnQk26BA269LTCig6x0t9g+f9m8nNtoDofvstUSSHQqeetkWvldI6ge1lMeBFlW7XA8GnU7LAV5yLNzTFL47KBK1T/mtlg5XyHyNss6ivCPFLK3vUaZMKDl40yQo7jM4np5n7rWkpHWbrvWmeYqFuCkfQ7mCQmQkyCPhasV8oS5e9yNgI+1vqkB5SC6OWx5b9bWQCmO0zdoolMqmJkD7/AK5WalAc1wKDWNzANKdPEi9rtxBj2/7+w+4t2hvCsXVNKKCk98Z1Ag6yFFfwJs28xQjNBCZmD5fnYamepmVDMlBSVDvBOpj6H32AcUA+oMCRIKDoIT019/x+NyMcNoYOSoNjytsE48OUVG/XNCrcSlskZsx0I+/lt+trD8e+GXCfCGBbnvDCV8Nv3lUhJ7FL6Vl1BQSXYHswQB5ExsbVYZrw2orQrKoEd7WRtE8/CxuovesqEIQpwkbAE+/rpP4fFnMLnA32RGkAEEIjWFJWpCI906ifAfU2JLSqZIMHY7DQ/rYy48V5SlKes9Tty+uVgXYUpRgERsRqOUfW/usZMizhXn7qo5TI08fGz+wFePbUDtA6onsHMyEgwQFdNev32YqilQOaITsAr6+jZWwreKKC+W1OrCWXQWXCZ0Ctj7iB8LAqWcRhVmlfkkF1JqkrcGYwAdIB0kfW9kLFb7X7gr1OtlUMrJCUnSNP1s6G2iV95EmdQNgQfu+/TrYK9sM3Zet016qu6aisKWVuOZKtbAUjLqkwRp4jW3PyyBjTddLHCXOFlSPFLbbPpH0IzAkXvdJVuI1YOx+7wNvZil1bBIHnO9vIDiu7d49KKjF03O3dTIvO6AKZLzrwBlrMSpxRUVQPLUC3sDSp+yA89d7AzZmNPkhEFsjh5rtSYOUn5WLPIMd2QT1NjkaRqYsA8kkSYE87Ti3Q5EmPNFIzAeJsSVnBnUjf7rKTqVEEQdPr8rFFt9NLXWbKk7dR5xjH/wAjXiEgqTUNKAJ09qNPjaCVuKAJBBUI5ESRayWPcPV+Jbgdue7snbvLQW86so7qgTry0Fo5a4E44en7S7kpOgBfUYHlHnY8MgAsUJ1PJJq0XUasOvjtIy90DulBn2t7Yc6gFrg5f8IP1+VpSY9HzGc96tuxJI/71Z2/3bGEejpi05gu97tQCRMKcM/8utj8RvVN3Sb5VEzailSVJUBE94JEabfA2sXT0eK7wxQmloKaaJdIX4FIsoIyjvF+SkOZz7EezrZpN+jbf5JLt/0KZ1gNKMfd4WsZdzS6G76KkcKlqapm2yUqIEpSATYT421Dw6+yHJSvDbSCyhG88L40D5aXTVCW5knsZkEnQGCD8edmHxSpKukVh128KJ1hz1/RLqVAkB6n6jbU2ts25n1S0sjrnItG/HjBtDiSiuy8Kt6oaVdSXX2wgiCoKbVCpnmi2xUYiZqbu5b91m0eCiCp4zXX8lCPFNKWW6RYTnjPIgkgkTHytGaqvUpSiNRO43+ulrJYz4fXZfCKcVdTUoCe8cigJ0Om3ibNNHA7CHcdcvipaKpUEu1KUkbiYyzyNs2PIxt5CB9VozRPbqoZcUlSQ4CZGuuomRA+duEVTqUBJMAazGv1r99poqeDWGkLZWzXqcZz/bldQoymI0hO+1jl18FcM3nUep0dK6+s94Ht3ECJ11IAInz1NtBlI6RnEHu9VTEjS7IHC6gw1D/eUVJJnURofDW2hWPgZHFjLrBJiB5/C1jD6NtGqR+6WkAwBmr1H3xNhmvRwu1CMpuakWZ1mtWY133tAQs2LgjcF/VVmq6gi9sLOqUJavladejlDUjUcvZte+gqr0fZS7TM0wBQgpK1K1BSDMR4/K1YOIeCLjw3f+E6amoWmSjE9M2+lKyQZbebO/8Anj32ny7ajiDUJo13bQ3KqgU0gJU++4FhvL7WVOhOg0tQkZlkIBV1sZDBdK9Q9i92qC0+opQDKSS6dJ8B52qH6WNS7cXpl8EL7co0B+sRQUrrgkpyprlZo/49Z8OtrW1h4mIW23SLw602lJWo9g+dJ10I1PhOtof464WTfeLsIYnxvdty195Xa7/2LWMsvNmnUHUKJKFHecp1nnYEjAxhJd6osERc6w1U8VDvZVL7eUQVbEAzz5+djV2sMuTVVORKAuAcolR9ws0X7g4pOFC0YvuRKzq4Dd4USdNRKhpY5S0PEhthVLW44u5aVtlttLF1IC23FbLEOQYO4Omp02tF1Q5p0YT6f5RTCLe8FQf0uaNNL6TGJ6pFC6lD6adztFJAzBVOiQJ5Ei0C8YqepN8XDUUbai0bmZc7yohSHHUTm6kCBH9PQDjbw/YViCorsRUNFet6di0V1y6BtLiwG+6CNdojfpau67ipK647tcfu6kqE9g4wv1hsFZyvOGAVCYlWn6254y3neD1Wm2jORpB3ChO+7wqLqw3WXnRFPbtMFTMapCyAAfGJnTf5Wr7UC/l1r15XvW0pUtwrdccZQtZHMlRg+HI8tNBa7XGPDLTfCu8ks0NPTKFOw8XmmwVNthaFKOgkd2R4a9Dan99XBU1VQxSuU7nq7RzEBv8AvFbgRrp0G2scxOthzW5SRuqVe0scAUxqhVBeFS67TXYopAzFZWQCBuY9x08LH7gaQ5QrQpglSASQATBBiSD4yfjZRvlygu8pYapQl52EIbgIKiecEd3UDUkHWOWiHh6sSHgl2YKlSE6kSQTruToPhz2toWsVn8k46ah9Zd7R4BDaJAR3gNNdQdJ18PjFllYCiVaZQkKG6ZBTAPUgQRoTPhsUxiscK0DtUtkgFYCoVqRBkba7CBMKEnax5ASUoZMpaIGQa5SeWWBvCTtoQANgLEFlE3Tj4bXDdN74urn78qkMMMMNraUl/IrOVqBHMnRMayII91orgxhhGjSKKmPrbBCQvMSG1I1J1JkySN94IE2pLf8AeS7vvOhCbwq6dLzayXKdIcdCTAgBZGhKZ3j2tySbG2sSVq1oScT4gdDeV5X2jTGTvR/ATpr/ADJjXXXTdw3tAcKi4bGXPVDdTCY3cVf268dUCqJ1ugYbW+whxLVOkjtFd4htvr4DfztCmOMJXjeDzmLeKOMriw+04fsmaoOuFtG+QNgCVQrwPnpEc3Vf1PiKixJdt3XTd1319Fh929LrrKWtvNb6qhp1swntap5pRyKXqUJ1GhBAmE7zosUXrXrqb9avCsL4LKH3lFX2hQVIhexkiY5ifO0n9shW3Y+Pbz0Vp+H8FoLTundxMw7w9eZvC9sMX9eVbXUykEKcok01O4galSUFSlxEwSBOmmtkfgXxE/0XcV8MY3KVFu6rzYdfWCqfVpKHwE7HM2tW4nQeMtKvw5iK52FGpYcbQYzpQrMOe5GnXn18bJLSwk6xbBqp21bs4aB9FBoLF79NOs1KE1VK8HG30hxpaVEhaSNCPAg/pZAuy7qoXleqgclO1XKV2ijlQApCHDJOg1WdzaKPQk4kq4k+jth52rqA9XYezXBWKzEmWMvZE+bC2Z6kG0t1uBncS3sqprL6qGqBBT/qzaiO+EiTJ2+HO3n/AGsp2y0dnm1iux7NzGOpJbzCGvDHWEMPhLT9W5Xvo0DNImRMfzK0jxE2j7ilxJ4pHB1ZfeE7qcw9RUICnHw3mcWhSgkd9WoEqB7oB0tMFxYNwnh8A0V1sl0a9o4nOqfM7W3i3933pctXdNdStP01S0W3GVpCkqBG0Hxt5zEKencMjL/Vde4OmNivOq+nr7xa469fN6Vt51rpzFbq1KJ18dfjPjbrDGCr2ui86e+0kNGiUFgCRCVaHnyBtL153BRXPeT1PTUiGktuFIKURsdPh0/SyVXU6hdlU0CEq9XcAEazlJ0/rbtoA3J4RosmRmV2qq56TOC/7H3tdtI26p5NZdhdbKuudyR7ifnatlXlNQsp2nTvZvna5/paU6Lxw5ga/SkrU7SusuKE8uzIBPLVRtTSubKXSIjSD5jT8Lddgz81OLrkMZYGVBsnNwgvL908UsKV+bKG73pUqIEylTgSoR5E29acA1HqSllIIKgM0bbj9fhbxvu2tdoLxpaxpZSumeQ6gjkUqBHzt694Yr23WmH6cqyuN5wJ3BAItm9pY87W+a2Oy0li8dLKWk34exCo0MJiYsBUP1q2PWgw6pkAEqIMfHazOq6q+XrtfcuHsXK5KCtpt0ZkLIkhJE8wPKfhZHvG+8cUF4uKpahqqo/9vV1BZaUhsLSp7sk54WjsfZJH+1BPsxbhe6i9gu54lgnfVXkEyQRIEExuPfYj+9MiwpB1Cs0A8/H9bRD/AGl4h1VO4jtmmiw6puoqHHqYN07YeMqcIkocCMkgiIUo5QQAUh7FPENbfb017XZ2IabU4XKilKQpWUqCCFbDK4BOsLmTGlttJYIJnCma8azK+4BlgOKA8ACev1rZj8UaYX7w+xPci0hXrt0VTKQT/GplUfMj6i2sP3lfVTRLqL3eLy1vqNO8S2S4wQkpV9kcn8REjkATFh36kFDjTpSUqBzK3O3nqLGhZw3gjkUCd/EjLeoXk5e6Yqg5/wB62hcxEmNfPWdbLnDm9E4d4h4Xvxz+7pbypKhU6d1LozfcbEsXXau6b0eu10ELon36VUjctuqFkrO4E07wJGQlIM7EGf8Aqt6hYSR26heUaxzX6FeodTVBCnElSd9YInrqOXONbXi4L3qb34W4arcwWRQJYJmZ7Ils68/Zt543HiD99Yeu2+P4a2iYqTrM50BXLnJ+Pwtd30Ub6/evChmmKyTdt41NL7iQ6P8A8Lbh6QFkxaV29a7iQtcpiW4CdvjYIFIVoPjbpYUFEczYMq33HytsNWM5Fb3bD1MrmUCRFm2oiDuZnQ2dSxmSUnYgzZsPo7J9SI1m2lSvuMqz6pvxLgZTuBE7T7rapyoUzQVqoJynxI0M/C2gSnROkfX4Wxmci0ad1Zj394ffbQYs526GS57/AB52zMVAmdOnO3BBSMusb72TcU3hX3Xhu9rzupppyspKF9+mS6CUKdQhSkpUAQYJA5ix2tubBQJtqlfVXI6HnrbCZ1B28bUmwN6aPE3FeMrhw5XUOG6SmvW8qWjfebo3iptDrqUKUnM7EjNz6a6WDxn6ZnFPDuJ72uW66W43aOlq3WqV6pul9h15lKyErUguSkkDY9DttbSGFz5suipe0IcuZXeQogga/dbvMTqCNrRd6PXE29OK3DWlxXfTdI3eHrVRS1ApkFLYUhfdgEmO4pM6nWfK0mBeuadrVJGGJxY7cKyx4kaHN2KGIBBBOn4WM4bJVTNJIMtEtxG2Ux+FiYUNhPMGbD4c7tXVsSDldDg/3kj8QbD5pynWA3HtK9yv1tlszDnr4xbLOorwsvFXaYiw4JVlcvqkBT4Z59/9Da1eH+GeNMUYavbF9z4eU9c1xNlyvrFvtNtsjLmiVqGdUa5UAq1SI1AtVQFbuMMMJkEfvmnJzbEAq2/S1hRfF4GiN2+vVKaFbgeXTJeIZWsCErKNirlJE+O9npAcpshVRFxdGG6tpKx2uciYGVUEH6ixiorbsLY7FnQiZWSZMb/H8bJBUDrmUYMHkYOlhmUodKc1QEQogSnSIGvQ/pa6QqjTyQodbOZKlDQSlMcz9ffbSVlRMwDEJ6+7x+utjNRQ3Y3SreYvVLjgUkBIGqgQTIPh+J8rECiMoBCu0GwIJ30sg66T25U9qDhfjG98BVXEq7Ll9ZuKgdW1W1DLzalNQBOdrNnSnvA5ogAnYWZjry1AlZlJOxMg8pH1ysZo6pxumVSevPsUz5HbJbXKVBJkZkgwqInXaycpyFrzIBEEnWJ059LM24vdLQrvMZKiSDMmDvHn9b2CQ4CvNlBKZkjcnkfrxti1Zs06lRGXu6H8Im2kZcojvCD9T0tI6pWI1U7YWdF63LS3oO92rSUrlMnMNFfnqbLdddtSm5quW1rIYUCk7ER9eVmTwOvBNQqtuN4pzpAqWcx1UNlgf8p+Np2XcD711vuCQosKUJgBRAPTf9LcPis/dXuYV3+Exd5ia8Lzc4xsPp9J2k9Z7y03hdRKYnL/AHR1+Xu8pt7CUgHZJHKLeS3pH0iaf0r2xCoVW3OpRUQYJDUxA1Ezp4n3+tdKMzYG+nysSF+eCNw6LOqWZKiQeaEUCRziJ05WLupBkEbWNGCIIsXdAPKPfazGqkiIPJ3PP52LLBEEifPSx14abb8osUcQCSP62uA2Cpu3WqFkPXiw2oCFKj5Wcn7qSowgRz3slYdYadvqmS8FLBKoiAZymLPpFIxoMhKyYiLU5qjgvstWitw9U3EXTMCdRrvYYXdklWcCBFnA9dyWnMpSZiSI2tyaFKjog2Ea031VwFh1Cb4u9PenIqeZBsa7JP2aQFdxMaRt7zZV9RVICU2i3HDmK6fGdBddy3BUVdHVOhFRUpLiUsiEzqCEpjU67xG9iQV0mfLG3MT9lQxBrOHmUiIQkKBUpKI11KPwmzc4lJaqLgcCVJWpLLswoGO74WjnE+FMcLSpV0IrHFKgR2io35DMTtY1hK5MS0VNe7GIbqqGW/UwGnXUnvq70jUkE7W6uaiAoDVF4Bt7vNcxRYm+SubTGIgX35Jy32xmpaZZbnOAdvCwD+FL4vegut+6KV5Y7TKtbamk5AVnvqLg7yUxqkanNZdqGE1F20ZWAqUIOomO5ZzYZShOHWApWgcUkHwm2A3JVgRuO2q36uLLHqmHiHhxiFxeeidWoBKgA27oOYOWQfvtrBWGcTXDe5Vel2OIpFMhIqCtJ7+cQDBm0qJS0AVhC1eMH8LcXqibtT2QKSl1tUbfxCd7dGzEnx03dgBlXNtwqJtQJwTdNdLyytQXUBWXUd//APOtwy8DW5jVNhtAkjONPnYk/RVJrHUU93Z0CcpzkSN9Zt0xRrbqCp27UJAjve1+lqBcStUBQJxre7XFLIU2kCkxRdbydBpmqW0b/wD1nztO+GHH03Ddz6GkkCmSglRG4/py6WhDjDTmpxZeVIgKzt1t01MRoIraUz4/xWmvC6S7cdHTrCoZcdSvWIKHFJiPMfK2XT1YlqXRX1H9lp1VPkp2v+iVKt91xoF8tMgHKftNz8LRP6QLbBurDFc2FOdjeRBcSdEkgaT4wfhaXLxS0WW0qfcSdVQk/kLRpx1b7DAFNVFC3BTXmyvMqZEyB7rPi1+6PI5IeG/6hqkCrQczb6l/3qQQYnkN9LdsKQYCnFKVoBCDH3WApq0qoKN9DpPa0zSwnswqJSNjY41WVK4IQqOoSRra1GbsBVaQWcQo64y3RmvVpxPsv0yEqCzOska/PS1asH4fFdd6qAuNU4ZvKuaUsyUpAcJkgRrCveSbXJxvdibzTSvrpVnIjLtJkEHW0A4PwA5TX7iK565ioOS9XahpaUz3FpQobGNYjXnbzzEq4U9fM08l2VDGJaeJM1zCtXelz1VCqkYdaVRuBTTmVKXUZCChQXpqNIPWLecDSnqYrWoO9jT/ANy0lKlCOUQPADx13Jt6/wBVgKnWFNrYqlkpnRAmPEzuJt5U4twjWYZ4g35gdmlqnHqC9ai722A0pTi1B0pQEpA1JkQAJMjfSdns3iUdW57QdRZZ+P0xjDHBQ/iq76h8evu9x5HflAgpGndOs6dfPobN26socCSlI27xBEe8edrFYp9F3j7U3Sa1zgziwobaLsLup1Dkc4bKc51MwBPnFq6M07rFQ7T1QdYdaWULbWjIpKhoQRyMyCI/TqQ9rj4SuaLHN94WT4u9aUNJ7sh0FYSVZpSrQAjcmDoSNvdB1Va1TsIeqVko0Q7vGpKYgQdgDzOh9yJd9W6+wGkLLpXlAQO8YgbRvOnjqNZ1srU+R2mV2ri9ICpSUqjmJIHh4HveBBDtdDG64p7sujE9Yy/U09RVpbbDQyrCCoiSZ6SVSdASZM7gvuk4dYFBaqV3cEIBSvItySUyDrpsdevSRANm1hGlpW1ApaDebMstApgJEHUGe7toeg5mLSM1RKr6dxLVQ4ytSCEpSJlcER3dCJJ1/ITUe7VW42i109KzE2JmmBclTibEVRRJRl9UevqrdpMsD/YrcKCNAfZjwEgFrXlU1tSlVMzTodfXKVlxCSkEADSdxPvAMgawVpVCxUZnF0q3FK75Qaggie9B1mfZBET3T71+5bhYu/8A199hDLmUFAbKiEpTvqdZjXcaEbmZotj8eiuF4DdVD2O8Fk0tPdD9a7U11e+X3VqRo00kZin3lSZB5DTYi0PY7wevDlSHUCW1mCMuUDoRrqCQrXwtYzEGIk/vOqbuWlF43s6kNgAgopmwdSSJynfnpGuoi0eYk4XX256xeN7XoXHX0BbrSUjKnSDB9+mm0baC16N1tFTkZfVS3+zJ4mKuLiVfvDKsf/1XFFB61SIUY/1ymlUD/Myp2T/gTb0raqjTKypJAIzDSBPP8LeJXAPFiuHHHjB2JV1Abbu2/adFQudOwU4G3dehbUu3tXXNPdokMtrXlJCoExEW57tRFnonn6futrs3IG1jQfP9koKvEx3TJJnfn1s3sQ3oQ0pKjoNQZ36/jYRp0rdNNIzA+yDKp8vzs3sZpfoshcQtPaoIlSdd/wBRbzKCIOcAvSiWsGii3FzQF5uupJBc75HIA8/hZoOvMZ3GVuIyuJI1Me77/efGzb9K67r6vjC12uXGxW1LzVaha26VtayUlpUkhPSB4CfG1YF4Oxo+sJXhm+1KCVJCjROkHXxB3/LS3dUFG2SEOc6y5DEa10MxY1l1MfH5lq8eCOHa1PfN3XkunJSSYQQsa66juga8xalt7oW3UqDhJUFKSSdzBi1wqu67wHowXndl4UjtPU3Re2cNuJUlYBcTuk7e2dAOdqj4iaCax4hMDMFeyREjYj3D4m23g/gzMHIrAxe73NkPMBI6SAqSSdot6ncIbyF48PMKV/a5lVFz0ZUSdz2KZ+dvNDCmBMWYzZvFzC9zLrxdbBqKvItCS23B70KIJ2O08rX89GK/EVvBHCri1KUpll1hRV/D2by0/wDhSkRvFmx5odCCORVjs68smcDzCn27a5lBKHKgMBSZQ9knKQDv4QT8rM++8B4Vqyum/eVTUsuOPLU0lKUtJLq21OJAKSSCltLcaAIJAG0HVXgrtFIGXdQ3BB8/nrZNfrXHElOeRMdJ3Ma7D9PC3HiOxuF2xm0subnuOhw8p5TNU+92zaWk9uUkJSHHFxIGveeWdZ0IHIyjHCNzUzjLtNU1iTTsllAzpMBTDLJJBSQZQy3vIke6x9yqQYSVbQM0gn4/CyXV4iuikKzVXhTgMhIWC4BBUYEjoZgeJsUMdyQXStCUqNTV03bS3bTKcW1SMJZQXInIJAkJAGmmwiwb1YtQJCQOsjvEzHWbM7EnEu6bkHq6E1dbUtPFhTbKJIWUBUiY0gk2Srg4iKvi/XLlqLvcpklbgacK8wUps94EEaEDX32kIHWzWQHVLb5bqnPHy60XZxNxMw2Dl/ertQDyyvpDoH/MfjaPQqaMI17jkjwzD/8ANFpw9LC6fU+IVTWhGtfQ0lWVCd0lbROvLuJ+ItBrWYtupGoACjr4x+Nu+on56djvILzyubkqXjzV3uCN7qvLhRhx8rzKbpjTqE/904tsD4JFrw+hXfHbXXiq5Fr1YqaarSJ/nQpJI/8AZpt58+i7V+vcMCwXCDQXi80BMQClC9+Q75tdj0Oa/wBU4h3rdpOlfdK1gciW3UEH3BSrcvM0RVjh5rqYncSiafIK4qyTpHhtzsEogEgHTxFu1HQBR8NrckSJO4EG18bLOcgzptsLIV8NZX88ABQmy6RH8PlpZNvhouMByYKDNrVM7K9Vp25mpECtRtvbQJQ+sH+LKr8PwtsEjWI0+VuXTleSqVd5J1+H621mFZDguys+f3e62nEIdbU0sBSHElKgeYPKwZJMkE/CwiTm0J1262O02N1A6rzUwPdbuFcb41u67aNmoxJcNHWt3K280HIqWKlKXFNoUIU6KdL6k7kESNQmyxflK/x0uG9eIH7pormvqirspdqL4cNPeCFJW6622at5QQtnVYQghORRTlBCQXPxXr8H4L9ILElJia4Kena9YReVHeVKy+qr7Z7s3ZkPJSIKnYKQNUpEzJsTxFxi4ZY07BzFar5vI0zRSy3U0IeDaiolRBW6dFdySEySCSTpbr2vc7LIByC51zGtu0lSv6Bl9B7COJsOqOYUdezVtidMrreX4S1a0yTrBJtTf0T8c3JUcXr2w9h+7qWkuuvuou05TRpYeccbLZIcykgxmcjwtcVKhHdMDlH9bYWJDLUE23WrRG8IA5IYHmBFh7nWpN+Fue69TgjzSqP+qxUHcETpuR0t3TrLd60D2aE9oppXhmSY+aRairaeBWQSA2rToB+dst0Qf5h/w2y0rKC8LaOf7X4WbSqc15tqkmYTlJk++0+BtfZgI72YyVCPO0PcNbmpsR8Y8EXDVkhmrvQ5+zVBIS2smDvy15ee1rGY1wZeGCq5FIuoRV0r5Utl1RyqIEaKE77Dx08rBpaiNruETqVOopZHs4oGgTWKVJM5NJPLUHlI+vnbpCVKJQUkgaSOY15/UW6R2qYJG41/xfOetty9yCSD3Z11Gv4eNtJZpaQu10lY0pptTYCn2wtAS4DAOxMHQ848+tt/u+rU4hkkZlQQUid9uWp187cBb0ZgogDUgcunl+PjbpT78hXaGABqTJj6/K0EuS6coWmnVBSyttGsoSSR+E/lZWoMLU79PU1rd+XZFGwip9WeccbdfSokFDfdIK08xI0IidrJQeeyZS7qDBE6R0+t5t3mcy5+0He00HXz8J++zXKkwDdF/VXVuHK0pMKn2dNbdtXe+tU9ivvK8pPhPn/Wwmd0H+8gDunXYdT8/iLGqdx4K7ripI1jSInXp9G0XPsEVrASnVwvoL6oMY3VeNNd7zqUVCUuhJAzNqGVXxBP32vrS8Onl0hCqcErTAhMR7rU14QXY/VXwy4/fLlOG3EqVlGigCDB/L6HpTSdmqmaUgCCgEfC3I4pQjFanIXWsF0lLVy4bTjh/EV40emnhR7DvpetqRTqYQ47crgVObOe6JI5bK/4bemtOn7MJ6DkOVqM/tK6mjR6UGFWmwA8KC7M8AAk+tOkeegA+HS16qdMNAxvZ4IhBEIfl09Ez5DMeI7c6rsjTQzqOdgHQrmNzYyQAnSxd3RUgz52txoEmyJOpkculiqgORmeVjjwJBg/CxZWnKR0iLWhsqTt0tYLol1F/wBMAkkJzKPlFpPZuSmYcD7aPtB7JKiQPjZgcOnUt3+G51caWn7j+FpSsPu0cri526mZXMGVpSebtKlZlLBJ5xbf7sT/ADfKx+2rIUEPRR4z+qTlXUM2YLIs279o3Ke8EoB0WJnw+hZ7WZ+Mkreq0MIcLZKU99O41sanpY4X5mpnSueLORFDD5khf/MbA3ywwq7FNVL6EBXdClGBJ0H32GZSkaheb/Ny+VkDiI8lrC7z6icrbrStAN8w62JWXdA8A8kSlIErT5o3WUNPR0bNK64gdkhKVGf8Nhbpve6Ke620KveiZCXCSFupGnvsycUYuSjDf7wIkuNsn2huUjb42g/+1rt4VaqTs1JbWVQQqduX3Wy8MjeHZyVcr5gW5Fbqgvu5bzfNPQXtRVTqBnUht1KiBMSYGmpFkS/+JXD+kZqKB/F9zN1dK6hLzKqtCVIUlwBQIMajWRvpas3Cbig7c2Lr2W/RNNs09K4yt3OZUe0bUInwGv4c6t8bcVJr+JuLqZKHMwvm8WlgBJzD1hcRrpztYkxBzah0FtAL3VdlLeISL0lVxV4fpa7Y49w6lDmYoJrW06DnqbJTnHfhC2ShXFLDSldG61C5iZjL5W8h70oO2qCUERIJ0MfDmYmJ/OythdbtE620llspdJylwDTSO6T9a2aTEntFwAnZTNcbFXdvPHN1439IfEgufECLzuP+zrVVTqaI7EOsOsLVGkk906/mLTvcuOsPXbelfhyovkN1jV93gyGyy4oQp9a0gkJyjRQ3NqEcHa5268WXqUENn+z95pMd4KHq6laEHXUD4/G+PDSuFSziq8W1gqexC66DodFsskffbJoXSOry9thcFadU1vdAHa2Wrx48cOqSodbVfF7PFtICBTXHVuSfAhuOdo19IHjRhS9uF1Vdd33fipdY7UUz6FPXHUU7YTnT7S1pCRII3PO0+BbQcDvaLDmqvaB1syvSEu5F48DcTLb7RTrbTb4UqVDMh9CtZMAaeQt0c0b5InNcdLdFjxSCOQOaiOHeIt4MYdutit4XY6VUUtCyy8TdraE50oAJlTg5iy9SY+vSpQlTPC7FEFYB7RVG3kB0ky/pZ0sKdqKSnUFEoUyhXdJ1lIOuhsKGCtktFgqSSJCl7+6LRax4AAck6RpNyE78EIF53A3W3ldDtG84pYVT1Km1rbAMQShSkmQJ0POxS57suR3Gd+/6o0XXAw97MT9mEz4+zYvS38q7WPVpykoCgmR3dI/C0f4axg7RcTK9TrxKKi72gEk6ZkqXr12Atw+L4pTRVrYZGg5TdxIGv8utakop3xPkYbAjT9/7KblXFdJkmjb11Olq1VnAnAtwek7fnFZFxMuXrX09EaVSzKadwtOIdebSRotYbSkkawDEZlTNasdojQjpvaO+KOMW6FNPialby1LaFUxezJypBIUnMD7R0WEgc1e4zjxrDpZWx0zcpPMC1/JHoqGpZJeYXH81RPF9+12IcMX7T4NBqb8oO2p2WWnG0qTUR9mkl0pROqdyE672rB6TfoH3xxa4S1fEWg4WJu/izRULVVUuUlZToF7LRBeQ622tTa3CnNlUIUVBIKiNLWTu2+mMRXawzclTVqr1OGrZKHWwipZXBWwhSicqgqVpkEb6idJZurEdcu5KVF40LgrOySHEOuo1UBqQQTPLlbooXxRXcT/OibEGyEBoHlY/uvnBpVusVZaqEdmtpeVSH0/aJgxGXeQJERzMxNluirHXstE2mFKXBQlsJgkbzy3jXkRpOzn9KG/KW9/SQ4kXrTerIaqcT168tMhSWsweMmFEyoqCieWYqIgaBpYZZcrq1LbKApLac5BB1IGgE9Qk69NdptsB92By57JZ+VSNhu72WWUCqqEBoASlC/4oJUdCRqJ6ezI5w+aaqoqWlXVKUlDQQlkpJgulQBGg31JjSN/ez7mp1LyjtF5MwS6lRSEpRsN+U6g9DoNdT6a5V44gTdbCVrYokdstG/fBImY2Gux5DqZq3VkaJ60RqEIaebIbU0hpOQAJDZA7wmNgSfAkaaRY3iC8W0Xe4q8L2NDRDWurguFBKQO63J5qOmg0g6zIbLTl7NNLF2XJVXgtxa0MttMyAoFW6v4B8EiSQoEgWaF5Grvy/vVb+CMS3q2rPS4du5al01MJ9upcSCDB/gSSesa2ZrFJzwnBQXneeImVuYFo0YYw+2vK9fVa3nqahRkANiJBIPtEaZtxtZJxkL7ujDt40GErqQrtEOF28at1NQ/Ut6lSmiNBok+UGNdbO5N14+o0NM3veV9uZEBKLvuW7mkU7ImQgrdCW4BjmeZI11HdpXFJdvC9MP0tDQKKUupafVVuyBqXEspU2mYiSAJG82TrstZJnjuCqdds72/rGcl3Nnzc80zNvoF9GrE124qwzh7El8sMPJvzDlFeJ7RIWA6tlpaonbVareCuOruo7sxTX0920L1JRF0rp23TJCD0I3EzG/nb2B9FS/3mvRu4d3o24uWrgZYzeCMzceXd+QtVxYnujnsFyBdSoI884iJtc2VxqLDfDe7b1dvunuunTVOkklSipCSd8qCcqfcBaLPShq7lXh+53LuYp0OCrcQpTbYBCS2dJHkLN1eO3yDDx32n6+hZj8TcSO3xczTTjshmpC5JkbEfXu6284irp5rQuY0A9Au3iwcwSCd0hJHUqOLweDtO2tw7oIMiDueVkJ9BSolSArLoPET+Bsoh0u0pQQqW3VxI2mCJ95P1pYg+2VLStWqdoT49PrrbYiuBZHlN9UwMbUSarBPEW6gkSKduuAg6kIzT/wAlqE4jbUlxRJV3kpWJA6662v5i2+7iuWtv27r/AK/1YX7h91llJbWtTq05kFICQdgofRi1Br/RmJABzBGp111Hu6/K3QYOTc3XMY1Y5bKTvRKoLwvviJVYZu+vRRuXnQFGdwdxUOtkpV4QT9+1p79EaocHDy8rkWYVdN+1NLBABywhX3k6edqbYDxcrBOIBfIo/WkFhxhbXaZCQsRIVBggwoHqLWk9DDEH71XjlKW3GkuV7NehrNmy9r2siefsJExrFrmKxXgc76Krg8uWoa36qyt6VxobvfvAI7QMNqcKQTJhOYjnp+u/OKXOJd9OtuuKu5NI8232hBM7OJnfcFtYM7zNpWvSnNTdLjaQIfZUiPAgjTwPnZm4IuHB9+UqU30lvt3rpqiFB7sz2zdOS2TtOrZGXmRzNuQLmxNL3C9l0WJVZpI+JyCiitrcUX1dr1det+VgYYWioyJGQBC3uzUiQP4QJ/rZPvynXSVF6pbecW6aippnVrWZUlCEutz0Pd6fKz4xoxddC9fdz4feD121NO460M8hDiUhaoVzGk/U2Z98OGpvBxQImsdoX0idFdswW1QB8NuVr8JD2gqlHPxWB/VEm8Sl28HL+qAVBupoqpzTUgNlCwI0mT/WzjuasZXjynDTSm1OVq6pHad0hLrCTlMayT9GbNld1N3bc9JebLi1mtpFVOUgHIptbRKfHUKg2cD14C/LyexDc9PVlCU01QkFogqUhZQoaTOhB8uXKzyDTRGYTfVMT0xKMKvK461tvL6xRVVOpQBglpxLg/8AGr42rCx7SkxMoVa3vpZU9PV4auKtCwt1mvWyRm72R1pU6c9UJ18LVEpShNUguju5oPltbpMJdelaFzmMNy1RKsn6IV4lV34nukqJKHaaobRJ/iC0qP8AyptdT0ar2N28ZriClZUVSX6RwEiTmZVlHxSn428/vRUvVdDji8rvERW3avQjdSFoI18s1rfcL8UqpMZYWxEmnfpUNXpS1DjT7fZuNo7ZIUlQkxpM6++LY2KMLKvN9FtYY8Pow36r0tMjTTfabcBY2j3W7WkSRYMyZ10PysduyrOC0TCtJ1PhYCob7VpaOosKfvMGbcq10jcdbGjNihOF01lpyLj4zYs+uFsqJH95G/UH9LKN4tdnUGdcxneIsmV8ppysT3CFbHSDP4W2YzcArHlGUoYKiJM687Zmk/kbB5wdj4bW2FgiQDvqbHCCmhizg5wwxzexvzFuDaG8rwS2lnt3s2YoTsNCBpP1pZPb9Hrgg0YHDO4p379Pm+8mz/Jnmeu3O3WYEyCD0FjCaQCwcbKBjYTchNfDvCvhvhK8G73w1gi57srWkqSh+npkocSkiFAEa6jSzuziQTuTYFMyQDttboK5GY/CycXP1cUgwN2Q2aFQRqTuPrW3T6+xpjUwf9XUh4HwSoE/IGwSTr15AED8rDsNCqbdpiow4hSYHORFlyST2EKAISNddxbLItDfVKaKnLriCvskZiTBmBMi2WV1FeP3o+U5rfSMwIghRSmorHEk6exSPHTx0taf0gm+zVdcqUkqW7qkxO3S1VvR0ve67h9InCV7X0+lmnaF4ZlDUgKpVpCo6SqSbWo9Ie9aSrq7laoFesUzrLzwqmyC1qQIkHQwJ8iPK2Ixru/sNtLLZDmjD3i+t/8ACh9IX7KBBBkQN56/CbYEjMQolREA6fX38xbk+0CNUwTrt+mltLcWlTaEtjKYKl5hpA2321j426XMubcCdl2kFRJJUQCImYEa/f8AW8uzAnDDG/EapepMHXI5Xu0wSp1KXENlCTOpzqE7HbrZqNZipJJM5ZJI5gHTl0+tbTd6NvGC5eFWIKt+/qGofo6+mU248wpOdtQ9nuKgKBk89PGbVauaSOPNELn+XVeXOwXaE38V+jxxUwTci8QYkwuqiu9jV1xb7RjXbKlZOp+4b2jpTSm5JM6wdYMGfr+lrVcffSdwhxGwa5hu4aGuQ8taVhbyk5dCJkAnqfhaqzzrmVAbLamypWckwTtET7+fusGlqHygl4t0UoHOkLtNOS5SFkglSeUEjc/h9eNjdIUthJMAzIEjX8T52IHvJJkkRp9c9PvNt9u6xXU7Xao7BSV9oT7QgCPASJ38LWHG6vRgjdSPh2/F3U2FNqGZSTEbyIA+voej2E8Ts1OGrrqFrkuUbKyZ6oBt5QNYlpkP9mVkIKAoKC9zMbe75i19MKX5e6LhuarReTIu5FzNIcpewlxb2VGVwOZtAAFDLGs725DH5pKLLLEbHVdFh1MMQaYzyVF/2it9euemRdqUOEIapLmRy3DqlDfzOvU+63o9SkltMjcW8o/TfvFd4elqw+pKVDPdCM6lRslBj5/G3q5Sf3SDMyLFpZDNTxyO3IuhSx8GV0XQ2Q2w06WLuA6zNjPKEjaxdyZ9m12NAk2RR3TcTrr42Jr31A8NfxsddAiSqDYksiTvta23ZU3JawY/2GJKI5oCllOvOUkWl31jxtC2HXezvyi1g9smdfr6NpUD6dy58LU6ioMD7DmiRx8QXSqalI0zC2jUp/mFod4nUnGKuvZj/R/f130VDl+17UALnnMpM+H3WZ7GEvSVfWDU8S7uQJByoTBIkaSEdJ+Nsp2MyNcQGo/dNLqyJqQP4vnYnXO08turbbUpSwmSkEx52gqgwDx4NWp68OKbC2IP2Scw321y8rJ94cMOLTRbdreMlQ6jMEdmAsDMrTNoRsT9bgT8Ync33LJ20tzaysGp+kCNGWx/ui1esb3zUVN43pQu1jpaTUOBLRUcoAUY02sewvw8xtc1+sXve3E6trmGgUqpghWVWo17yjvEGQdzpZj40XWVN9XyKOhqHFduuDkVBM7gjlE2nSV76guEumnVWG0xis63NJGM8d3XdmGGbuvEuNBLTQUsxlToNTroNN7RZX3t6k8aynQVBIKgoRCkkcldI8Om9lnHmEl4muenpai6Hq1qqaTTVSFtqIXPdIOm28zZiX/g/GFEf3BdKaBmmQ203ShVSWWqdpDYRkInMDmIAM6xynS1R4jG0lhOqsS4aZrOCRWeIzVff980t1OJhmkfFWAUqOdCM0KAJymNYOugtE/F+voU8XsTUorKf1movSufQyVjP2ZeUVKjoPrY2dTmBcTNC9ayguBhip7Baq9+nvN1IcX2fe0CeaREK8OmpXHfCjGlTebNY5hJNTXN1Sah+obr6p5VQg6qTnU0BCiUkmd0iwH1Eb5y4m5VxtCWRBpNgowva96O7KZuoqKpLLfaNoJGuZSjAAjXf6HJfuPDF8OtuXulS0UqT2pKlFZypBKsqUyTok/E6WdNBwlxzfDjd3XZwifqKhDgdW6ahXsJBJnMkBIBiNOnnaTcK8D7+wjR3fjC8JpHqNDjjt2B5cBOZDYDpMZ9V5oAH5wmnDv049XdFXdTxxt4j3CybnDbD1et2nxGwAq77xuuqLKnEFDvZu0zgBKTqJkCCJtdXgOt1zB941TicwqaqmqTlOvfpGPDwtV66Kq9K69mVKcR6utS0qASJgpMDcmPrrazPoy1J/sS4AqAaK7HU5o1JpUjSP8AL+vQ2EFxnBeLbqpVEOgIBUpd1CR2jZEiRLh69ALN3jBUso4K4yLKkhtm6qh5ztZAyhMkydtAfhZx0bb9W8lIp3d9ZO3ysVxzcYvrhrii7aqnU2mruypSFJElEIMESNwdY2t1jj4T9FjN94XRPDb7l5YWuKspCpSaq66V3OlWhlsHcedlH93VrkypWvUkwfhZD4Xv0qcBYbz3klambqp21lSgCopQEydRExZ4IvO7SNK+nHPV5P3Taq2aMAXcEZ0br6BNLiJet8XPdrCsO3WzeN6mnSG6JyqDAcQlxCVrzkGMqVFW28DnaKa/EqKbF7TgPeQ0UKUDGmkDz1+dpVx6blvBNBfNLU0tY/RF+mbcacS4UBYSVpBB0koTPlaA8ZLCL0F4NABam1d5StQruAQBJjQyfC3lmPRsdizxvf8AwF22G3FCw9P8p3X5xOFz3O9ejjNQ+Gl99NOM69xsPeNN9LMy8uM9fetK/QU2EMRr7oIdFLKAdwcwmCIB948rIVRej5lSX8iErSciUkJEjaRrEjSIn4WI196vrYRDiUpUowhBgTpqdNSQBqZPwtnQ4a9sgkYbWVqWWQ+6bBP/AIVt4Hp8dsv3piJdy1NSs1BQ6RTMFxuFSFPDupWNwkpMjSRNrQU173RfDLFXdN8UlbTLK0h2lcDqCUyCMyZToQQddDahSHM9Q0673ZUMxUdIOwMnvbxETrazvo15XeG1OQSsUlbVNKTJAGYpXtsdFbW7ihmMjTG8LGr25iJb+S8P+MlSLz4vY2vFht4JqcR3m82lxICgFVLh7wGx1E9OtjfD9hosu16mo74SnUzGkQYjkd4k9NbC8W8L3oji7jOiapHVJaxHebUlIBEVbggHbX8bLWCqIXVRIS+UtFtQUCW90iJ170qjp0IjQgdTmGQALmADxCU5nKtu5qA1i1oKhK0hW4TuDrHXz1HhZLuCuF33YuuqWWy/WLcWolBQGxMgaRpAMT1POLI1/Xk9eTzVI04QiW1QYIMqiZ3/AJZnXqOvOIKpmkw+HJcSW2cspaAQTExmT4wec5pOthjdFJsjVxXXiHGFGq9FV1TS0Lsslqgbc7SobKl7qBykScvQDcjUWfOGsM44p2zdFwleGKN9QU52PedcIhOqUEob0P8AEpapPeVBtF3CK7qq91paN7Xsw00FJCKGrU2reYhOsakmNYCjytNN1YNuWqWad++Knt2SA4t++lFYVr1WFEzpGXqOoFgNQb3F067l4dXPSpZrL8qK283kAK7WtqFOpTGoGXNlT10EactBY7WjD1K+t4XK1U1a++z2DailsFOhzT4H/hjlYK7cBNNJC6LF1/IS6hRJavNahJJmBJEQUz4gzsAVZ24q+haQzTX/AFqnXXgS442hSkplXdlKR4GdfCdZUjfDspROs7UqsvpKYcxLT1d04kvRRXRvoXTN6x2apKwIJnUGOnct6N+jJTu0Pop4IQolJFwh+SJgFxbg+R+7laPuCeE8HYprsQ8PMZXK3iG7r/u9Dj6bxlwywsmUKJzNkdoohSSCCmZtZJjD90Ydwi3hfD9Cmhu27qD1KkpkEkNNJbISkE6mBzMzuddbUZf1IDGehVuL9OobIOoUcV5FdVUNa5WVDaqF1biUsvqS28S2pGV1I0WkSSEnZQB6WJ3zVLdplo2SCCNJjXx99gy8pRgDJOwUfl4bn5eFilYttynWrPIKQqCdQN/cfC3BMjs5eiufmam5cN3U92NV1LRh1XrFWurczuqclbk5ozGUiRokQByG9t3rdlFeKGU3gylxLL6KlsKUTldQQUKBG8HXXSw7S/tnQ3ErbSYAjYwJ5zqfrYvWr7QJypSYMjMdI3+vO15tyVTdYCyhL0kqd4XnhapabcUpw1LJCEZiYSDA66n4pFqkYnpFNVFQFtLQ4HlpWlehmOc68t/O3oNWZmMU4XrArKyiscZWSsZYWggEmY3199qZekLdyrp4l4qomwMiL0deBREK7TvSDrp350+Ok22cMltJw1zmLQWbxfNQwQQTr4Wsj6EN4djje/7tj+/utL2WNy26kR5982reqQpQGmpEWmn0RLyNDxmoqQEj94UVTTwEzJydoB/93barm5qZ48ljYe7JVMPmr7gzTJlQUFSI36Tr01tAV53NetNeFdRqq6amVSlLdGpx5I7YKccURG40Vt4Wn1umfUyUIacgHUpQTP1HX8rQ5ijBtU5eOJquku6tXeDDjVTTZW1ysBKHCECIVrmGnXe3HRWJ1XaVbBI0JpNV1x0d7Fb9/qqUIu9a1stNnvqLMKIJ5RqNvvsQbvzDz1Zh5dVdNcs0zCEJfdUlJCUuZUqWmNQDHhrrrbt7B1+rpKWMPXgHHqKkUVCkc0UUuNuJJywNCkkfG3N8XXiG8rquxScNXkzUN0gu97tadSQlaFtrzmYgGFAWttDQswNLdEPR33fd5oFNdd30TNNS3a/UBPZ51NkhaSEzP8Y6bGPCxOuvi8r6eZoWK55lirco1oDaoSnM2oECDp309eZspXVcWIbrxAhtCaVqmdeqKV9x99tKewU6F5gCoanOY+ob13XDfFA8k1C6EhpTCm0qr2IGVwnKZXvClHymzjLdTOYJv8QbscvDCd9XpeTT6bwu9ugrGw4TJBUlKiR1gn5+6ulYkMVzyUH2HVZSPA2tnxRauW9275ra+8qWlbFzvtt/620S4+kKyCAomDIjx6a2qhehBrFL0OcJVIMzoJ+c23cKdeMhYOKttICn7wJrU3fxbuwAqDb6n6cwRspteXz1g+61wUOuN/aBWVxEKkHUKGs7bjTx362ozg6vF044uC880JZrKVxRB5BSZ/G14iF5oiJ9lRI7wnr9fK1LGG2ka7yV/BX3ic3zXqDcF6ovq4LsvlBBRX0bFUkg7hxsK/Gxs7TGvKzD4DXsm+ODWEqsKBLd2opSQZ1ZJZ/93Z9FZid9dLQjN2hPJ7xWirkB+ttEpP8AEPIna2iZ1OnMGLYFCd9POxmoZSXe7RypcQNt/KyHUI7ZlTYGqgRt1s669pCqRUAT1FmusBILZMC2nTO8Nlm1Tdboow52tO2sgkrQJA1A0sMFAHVRsUpcyWygiezcWmOneMfhYcrAM5pOutrrdlSQoMjOTynW3Sl9IB6czYHNpJk/hbclIAkmBB+FnCS7B05TMwPhbpKh1FggqNQQB523PdM6DxM2I1MUYC5ESNNxY5d6z2gOokjlZNCifOx275L0pAI8TytJRRGrumtNU8WsmTtFZdtp0tlnWA0RJCjPOBbLNZJeKXDlSneKVzZs8JYrTqoCT2UT56/P32nK8alxRSlagQ37ObWNNuZ+vC0DcNcx4l3a+3KUilrIJGvsAQefX4fGbKxxa1ZlQM2kCdz49JJNo0oFlCdxCUbocbW/2D6Z7p15HedRvZeRTXWACppaxMQTJH1rZsXOHF1YQ0palAH2Rsn4672cKaWqKc2Rck90DWDp9fCzzuIcpU4BajDbF0iUpZQVQCN9Dv05eFszXXJUmmSBm012259NB8rFvVHSCANANSUyIHS2IoahSlZkEGTrEz89duVg5r7oxaOiMofoE94sCEkyZB26/UWDF5Ui2VBqnbQAZCQkRIjadOY91nPw9qcEXNelS9xBwnVX7d66VaGWKarVTqQ+fZVKSJGhB8510FmTTXah12pqKBisQhyoUpKHnw4ltkgZEDuzI70kmCI0EahbN+oW2+6TmWFwuqq9EtIU4G0IA0OgTpt+P6dYy4p4gdpzS9nWqpkPFTyHmnRmWpP8I0IiCNbSBfNCG7vc7ZKXEqRGVUqkbTHgT87RDjanaq7qVQ366oC7XnTTrGi0oDZKZ3OsHTnHhawZARYKGU2vZIlPi+svFppP+sLbbyOLcRGfss0cup8tjyNvTTB9/qewJdh76YuqmkH2ky0nffWD87eSeDayrXiO76Tt2mmmkn2gopeCQVJmJ0M/Lnz9OcJ1dS7hC7HA9lDlCx2qAmQfsk6AgCNfwty3aOzomA9Sun7NN/Uf9lSr0s60VHpMtOFXdVUXUIIn+FAPloPn5W9faNQDSAYPd59beO/pSOJHpCtuBSVRU3aIIlMZUa+MGLewbDhS0kJj2Qd+djU1m00Y8lUqjeqk+qPKUACSIsVUoKnvflbl2oUUkcuosVU64hJSlX6fUWssdqqj23QrokRPkBYi4VHz3t07WLSIUJI8N7AdoHBIBHWetrrDdUpBYo1dbmS9KRcah5G3mLSGKg8zaNKNeSupiFbPJOnnZUxNibGF2XimluDBjl5sqQFesB9KU5uYiZHvtlYi1zpGhvRaNDlEZJ6p31NR34nlbhl+VgZjaL7yxjxgCwKbhk2QU7qrET8M02BoMWcaXatpL/D1inaPtrcXlyn3qja2MaSYuzC3qtATRhtv7KW1vKSgjMdrJd5N1lRRGoaaUtDLralFMaALHKyRdF642fPY33h2iaaXr27dUJQOgSCcx+Fur8eWLvKEOEfbM6BW/wBom1acljsrlZgYXatR1L/aNgg9fvscuOhadSkqCSV5j46k2ZmIr3vq4LqevS7bpfvt5TjbbdE0Uo7KTClExmO45H3CbR1ScZuOd3VlWbo4G195sFYFPnrm2UpjQgZgCAT1J/C2fVtdkDgRa/UBGlBylrd1M9JhiidoqlKkJOSvqIPT7ZR/G1R6UJxyi+LwvwtJdTeFRQLQ0S2laW1pUhQ0VBnnaSLw4t+kPTUNS9h7hNSvu1L63KhirvFseqOk95sd9OYDQyOtmpwSwTjaiu2+Knifhpq7a9293Kmnp2KhDiFNuJSVElCl/wAQiCfjY+AU0T3yx3HEdbLqDte/4WHiz62FjXsJyjf+ybV716cM3XV0VPhxTlDWp7OoccUteQZcpgpSOXnZXwlg5ji5iW7rxxfdXq13hjsU+rvOtrWy2hRCozkgk8yNYtN9Lhi4b/q6S7r4uplyncWESUKGWTyOX6m0l3LgbC911blZd90MNONMt06kjNqEgK16+2belwQUdHTENjHFIPiXGvfiFVKOJMTGOSjvAvDC7LrpK5nBd1tXeisoEJbfqAtQP2hUoKUVHMYTHvsncVrsu5d5XDTVWHmKn1uqWXA62SFpU2yYGWZjoAdhadaFxhhqqSgobQ1XKQEto0CTlgADzsiXqhjtcN1ZS2VLUCFFrMrZA8+VqmGwtpJONKMzzzVqoa+WPhtcQFRu8KamuvFQu+gZaZY7dCuzSmISoAyOca7+NpF4EMUVVcVMa+9Kllmjw/RukMVq2QopcW1Kwg6gBOlov47XXfFNx4qnbuvtFIyxVsDsywnuNKS2VISqJAJBkeI6WlH0eGkm5qpxx1BULoeZzhOwavB1IHun5m1OZpFTdwtcrepzelsw3NlLdAnCa3pXeFWWzquK6pMjp7VsqsJ4Jvmgfu+gfrnELS6G2lV1VmUSgka9pBGmx0+JsiuM1AZTVrvdwoUY+ybI98AiBZy4R9TecDZvd5a+8hbTiT35SQIO552stc29i5voqnDqr3ymyZ/B7DGHqrBV33kq67uVVPNFFQ4872alFDigAdCDGo8Pufirmu6jpgtu7bndCEglAW2VHrr2ZJ5+O1mR6PwuHA3C6pbrqv1G7rqrK5bz9WvRH26lElR5SvTziyhx04iJuLAVNRYRq01V84yWm7rmNMrMVh2Ap5JHIBQg9VJsMRUkcHFkaCbeV1pRR1NVMI2E2JtfkEqXnc12oubtqO66WjWmpTHZJyyhQVI9lPOOXS0EcSmgy9TOSnIEqT2Z3Ptk68hoOn5Ta3hSswDwiu3Dtbez94Vl3UjIqap90uKcd7QZ0pnZAKiEjkkC0E8TH1OUaXjU5yXwUIygIGm3iZUfjbhu0UAjxKNzW5QWjRdLhjv+zezNms46pq1Abo6ZxReTAAcgrByJ13PT39LM+pxzhtKJReSld/8AkV3oHKQJ2NnQZU1Udq23k7spSJ128REffZvvXbdjIIRdtICSYKW0DWZ1kaj68LEpwAPEpSlxHhKkP0Z8aYGvPiarDeIjdtRQ31d7rSG7yZQWVvIKVoA7TQEgLgbmfK1tb3r8CcOblcdTW3Hhu6kK7VRSqnpGCoiMxUcqZgASSNhrbzsxM3R/uCuUzTtpCWs6YQBoFAx87QHiFls11KU0zchG4CR/EfEa6D4626XD8r4i0D7rn63M2UOJRHiS7RXzxExVe9LUest1d919Sl4rDhdS5ULWFlR3BzAyJ331tH2IA5R1ZpojOgGFwAQrMIJ1/lTHlZ+VFGpL3rDaCVAAajSRHyMRPgJsxMWKDeJFBKsy+waGgOgCBpGpI3119nkINtVrcoWc83KQ3hmrUhJAynRKZSDqCZBiOo6wOYAEmcEsM3Vj7jThHCl73QKyjcql1VbS1DYW06wyw46ttwCNFqSBtOuh1i0bOrX2wSXCpuMqe8SARodz3tQABuJMCZixPoG3Eu9uK1/YpcSSxclxinQQggJeqXkhJKjucjLo1AjYxFpDqhkqNOIfB9PAvi1WYXfp3H7hrgKy6aioTnTUUilSUHkpxCvs1bDuhRhKxMnYduvDdattHqt2LdVlQGQhHd8Ak7QD117u0Ws16RXCFzi5w+Xdt104cxFdLiq65CBlUt2BnpwojZ1ICde7nS2SSE2qbgLEVzVN20iL/KaepQEsIqHWygKAMJzEpEK12keybGifcoZGllI1BcFxUziBTXM0C0kEFSYSJA56k6jU7fCy5UNrQ0W6d0hIRIZUSYUNU6HTdI8N4sWo23Ujtm6gLQBIUkhUDKDA3nn120O9jVTUFhptyoqEBBIPeBOYyIAE9AOfwItYIuLJgbFGuHV/jD/EK5q5Cc3a1CaRSQJlD0NkSOYnN45d4tbtaJQUFO+ijprrHvO3hHzpVdwqKK9LvqlNKaFPVtrU84AhKQlwDSTzhI/C10pSrK4FTm1HMRH1tbNtqQrpOxUHvH1dbrZckpKkEHwO23hP42LuqzIUqICh7WojWx+/Gyi+bxaSAmKpzKZ27x26/lZJzAwcxCBokAkgc9umv4W4tzcryF38bs0Yd5JN7TI4QnMpGQjKdY035eO/3WAcCVNpIIABy5QfP6mwzhJeCciSO8IVqeev18drArCt15RrIA2+o8t7GZsgP3TZxhRIXc7LroMNVjS8wAB9qJ84J+t6n+krdzlHxCvRJVm7SnpnEZtCQltICtNDOVQ5bWuDitvPhyuAcEpbzgHXVJB1+Hjas/pYUA/tPQVoZJ9fuhKSQNMyVLI194+XS16gdaoBWVijc0BKrA8haXFhQykEgjpZ0cKn65jiJh/93VRp6l6uap23c5Rl7Q5NxqBCrNmqSUOEqEBRnT68LHsMXiu58R3Ve6NDR1rD6f8AcWFfhbqZBmjI8lycRyyA+avkcB8Q6snt8UU5CY0NQ4SSY3gfUWZuL8PX9c9Q5R3liZCFt0prEkqUO2E+wk81d0j3/GxNO+XEqaKWgEIUUrTOZwGDKuUg6CI5WinjI1R09XdlWu6KetUAsuFaiAlpMZo1EnvE8+fW3ER3z2XeSxNEeYKJ7nuK8r8vdm6Hr3dZcdbVJgqAytpVMTrmC/dHjYhTXPUP3iu7S7XOPs1RpnsrZKYlYSoKE6d0ctJHSzyp72vpgNXwtFDTtVDz9Aw602C4lSW1JTrvoUiNdbI9cnENQzVXjUX0+Qq7kV6G0EpEFYChAI1GuvOfO15oWcWhJDWFMTVdGPVqGrLy+zhLikpGUpJVuNO8I6ajxs4qXhHeDqUqqahxjvupV9olRDYSOzMZdTPtD87A1dEq4MS0dQ1XVDgbbo3UrWuc7anChcj3p15R7zNYbaW2IJgpmeu33a/GwZXFmyNDC2S91Cj/AAHZvWgTS3teS0rWpZcSyCAQYygTz05zyEaWqdfVO7T1CW3QQpsFozO6FFJ38vlb0cW20qcrpnmSnbnprpr91qFcX7pTc+Ob+ogILV61KgIjuLVnRp5K+uerg0znFzCsnGqdsbGuamcVlAp3UqBKR8CFE/ja91y3gm9LloL2GiKykaqCJ5KQlX4/1tQ/NNMkfyrPPqB+Vrn8IKxV5cNMP1K8qoouwOo/2alNj5JHxsfGGjhtd5oGCu8bmeS9DfRAvb94cIvUSpM3XelTTAA7JVld+9xXwtNizm1J6aC1YvQkvQKu/FdylWjb9LVgbRnStKjr/kTazZOoAn3b2oQG7Ar1QMrytEzpl+Vucxnn4k2w842+RtpUA6g7xa0Aqy2tRUjIdNLNytbLTziQefvNnCCOfLl0skXw2EL7QAz4Wt07rFVagXCbyDkqahE/xBfuIj/pNhe9src62De7l4FRgZ2jpt7J/wDzrCwQTvpOvW2iHLOy3WwVEHlB87YSCdNuttgADMIjz2tzIgGBtsTad1Aiy2CeRjwt1JKpBO+45W51gb9NLbEAkDlpvNpApIT+HnET9fXjY9duj0iRBkEediAgDNqNNTY5dqwXhJ36WnmTWS7mV/Mfj+tst3I5pPwtlnumsF4lcM3GnOI1AEADLSvwQNRKIIPXb63tM1aoq3A0VEjbNy/HS0BYbvNygxbTViS6FBtQBb3BJM7bDTUe60x/vt112nKlJyrTkUlZglXKD8eth0rxsh1DDZOfDlQxT1K3luQ2hKlFJ1JJgQP6bWcn9pLuTKU1QIkQAkjl9R5WZLVSHEKKHZSlI7wMneeX18LJTeMcNVN7G4W79o11qF5Q12kd4bpBmJB5WsSRNebkoUUjoxYKSP7RUQ0TUKhMykJJnpy+pFsOJbuQmEuvTvAT9x56j7rNBSiVZwuQDIG8Drpy1+psVfuulcS46C/nkkZHVAnw3sMwNGimKh19Urf6TLsDjyGk1LnZqUpUDKpIHICZMnN8+thrqxfdbLKnXlLQH5W2Csax3dNfC0SVdO/dle8s9tSvqqELJUSpKUKVBJB3EqEgHn8ClbVBq8Cp99aA3TqWWUA5RrmciNSIEg6e0Olq/CaDqrHFcVI9+cSsP1lSm53rurCqsR2aXM6YCjtPT2YHuHMWhXFN+Xo/iSouxlgU7iGFtLJIlYAGWY9rSfOT42K1d7NPYhReCnUClbzLKe1OZOwAk6pMgR4WPY0Zum/slZdVak1rTRcqsylZy2ACAnrHlyEkRYZYBqFIPJ0KalKw/QVNPedA0X00rAWVIQYCh3TMbDfUyBMyOfpdgqto3ME3MglfaKuumME6BRaSTz21+63nx6P9TUNcW7ho3nHTS1NS5TOtrkocQtKklJTEFJEA78t+d9cHMCmw1dTCNENUjKE/5UoEcvK3KdoJvdjXYdmodHSfZU59KJeXj8l/vKCa27svP/Zo938Q93vI9h6Mj1ZvqEj7rePHpPKqf9OzqUIzJS5dyiJ0ACEanl0+WtvYOhUPVWyI1QmBPha7F/po/osqoFqqT6owqCNI5TYBzQSdLDKJywJIsAvYySDy0sZirP3RV6NdhpYNLiCkSoA+zbp8xuY91iClALmSddLXojZUZBdDVThQyXBopIkR5WkGlrqenp6aodcCkuInmSqEyYtGN6OuGieS1mJKFZQnUzl5Dn5WUrqvCleoqO7b7rn6KspgShtxXZ9qlQIB130V7iLVa7KSLq9h4JBCetXilpK2k0jK051tpKS3lOVcwofCxV/FS+yWqldOcQpBUBCkhQB5/fZLNy3Q8ptZvioKkJaSgodEjs5g6DnOtuFque68110lPX1y3+8GUyoIBMzOkCeptS4cLtbG6vWc3pZLS67tLyqKdtKSkNJW5GgCyTG3MiyXfAJu1eckQttQI1iFgiyw1TU1KghDYT2hK3CTJKj1Nku/XqRu6aoh5uQmYKwCSCNBbnKt5mkuwaBalPZgsUE046WF5idHCNbaoHHHkqbaOy1b+dq6ekTjnGuEsROUVwYnq6KmdpmnOzZUEhMyM207gfU2jfhBxVx5U49aui9Ma3pUtLu+ocUlT6sodbQVBR2nbn5WjNgUtfC14cAE02IRwEtI1VnHa5aai+kZjCa9fLaW0H8bHw6Vsu6eyQR8bVh4c4mxFet63O/XX268X62pRUGqrFJ7Q5SBofbIED4WmzGWP2MAppPWLmrL3F6LUy2i7yFKS4mIB/zToBqSDarhOGvwzHImucDcHXbkoYlUNq8LeWN2t+4Up4VuBy96lhLzeRtSlKLhQdcsEhO31FpPpafsHKxEDVaDMa6oQPwtB/CXi5UXziClwreWCr0uSpZafqmU1jgzLbKADAA01AtJV74urrvvRh6iuVb7DzR7ZfaQltSVN5QdDuCr/hNvR3V0LpMmYE/j1XDilkYzOWlLlI6n1uqoUkh5yoDwCUCco7MHfnvZJdK313Ow2taV9g6EZkwEqCiAfA6WaFXxBrk3tX3bfLN33feqb1Fz0zqqpa2lLfp+3pyVISImEJMDcmDZKxNj/GN38PcI1tNclCbzevZFLUuJCl07QK1pJEkK9tI3smVcLn5A7UJjBIG5rKBOPl01DfFaqfBX2vZ06lkIkAgQSSdNYs4PRtdSfXaRxorKWb5bhaZHdvAqAjn7f9LJHFy9q+9sW3m5ejjIrWH105DTZSlQSs8iTr7/ANUbhhxBc4fCuvVq6m62L2vOlUyp3IPtC05MgHr03NsuWq48pudAVt08PBiFt1Z2vo6ZVO0aamoygCQU06QT/hM2N4ZaYbvNpZYaSudFBpIAPmNffaD1elqmlZWUYApiWXJ71URGXxyjfX4WLNembWF5akYBuxsurCmwqrUVDXfQa/Kyjpmh4kz7eSM6pcWFmXdSNw1obnvvCt/YexBdLV40D173jTPsKSFpWkOoIMcuoO4InQxZL4f8ArpwVxBRiqnvq8Lxuq7qdxq4rorSV/u1bipcKVkmRE5dAe8SSTraO8I8Yb0uKhbeorroS3f2IKhxxDgWexDic3dII00G/nZ0V3HrFN1ur9Ru67dSUgFtShrof4rAOO4bGQyYEub5KzHS4g1juA6zXDVTJj+rTWYPvNtLTKw2xmUW3kqUPtG9RHTXTwtV3G7VOKGDq32yMhUkFOoCfjP1vZ6o40Ymv27Ki7ayhutLVUytl0tMZVpCo2M7iB77R3j+oUigW8SsuEIKSkQE9/Ukxofuje2LjNdDilSyWC+gtqtHDaWWjp3Mk6psNqSSs5gc7KTqTqdDttH5/ErUJEE6CTII57zy3tujcZbqHRmbh5kqSQqTy5H38traeCQe8ZSUn2j4fp7rTjZZFDrjVIl9tKfuWuQoEDsHASE6zB6RHL4jxtAN7sk1lKpSYys65e8SokztqPx08IsVWILjDjOY6oIgqI7xEDz/AFtXu+kLTWMJUVo7iifDU7CR0jWN/cN3DOYWLiVrghFF0oUkNpSFBwkJEAzO2/h8hvrrGGOlrRi11lBStammSsAaEwdgdBvpHOPG0uZUrpgrPokkgJRG52mNdvkZtEeMW3TjKqWQ4AlDSQQDlCUpA5aJ2I35DlvsHTdZCbF6OJbZP2hZ7pAA1JG5AkxzOg8N51uv+z1uYMYAxbfq2Qh68L3p2ElUSW2WVZST/meVptNqZ1rZbU2pohpKt0hJJIG5VlkgQBp7zta/foe0hu3hLd7CgEKqGXH3FzqpZq3N9NwhTY66x0tAu0skWG11YBhYQ82SCiXEmDqNDr4zytVP0m+GrPDnisMT3ZSsvXPj0uVZp1mEtV6dahAIB9sqDyYTut0RATFplvB0LUTMmY3iB7Xuny36WaHpe4VdxNwWcvmmRnfwxVs3qlJQTDAOR/XfRCswjmkWJEOqC7dVZuE07bk3De9RdFSVpK6Z9P2RJg+yTkM/4eR5aS70VN5U7S3qxhLiwgFdTRZS6jaTlV5qO6p00A0sz7rW3e1GyatpLanUZgsIzNqVoIJ5HefETrsFemdvhl9NMilp3A0rKAvPlCZEEb6RBHPWPK6L2TaIKofoK+p9bpnK6pLRCg6XC4G1CAUuNASkTuop5yJMWurhu/rvxLcNFfV2OqVTPsjeApKhoUqHUEHTbmJFqlm82GElLFBSmqpEBZiRl05KOux5eNpn9HS+xel1X62spZeZrEKXThITlC0e2RuMxBH+5ram9mQ36q0w5m/Raxi0lnE94IU4QVrCogfxJBnw36c7IOYI0SsnUCCJ8tfrcdDZ0Y/T2eJVrGYdsy25A5wCPPl4ffZrrhJgKJ2ACTuDH9f6W4upGWdw813lG7NTMPkEnPRnJUnTNOVKtpMeHM/OwtfeNZe9WurvB4O1BSEKWlpLcwMo0SANhyHjYNxSFLKwQZTmnp9R7rBlCgtYUpMpJ1B0mdND+Vostuk7dEr2R211VzOUqK6dY3jXKYgfC1efSipE1eHcF3vlzJVTO05JBB0Q2R8cp67eOtkHEJdZW1MhcpPkRt06f0tAfHen9Y4MXHWnRVJeRZzAHMBDqY98C1qlOWZpVGubmgcFTKqSe0IPSBGs2DGmUjQ2N3o2WqtwHkpWk8pNiidd5gba87dkDcLiSLOXqZgWrRfGEbjvfKkmvu2nqNua2kn7z9a2bPEu72Ha253axoLaNQ4w4CO7C21AyI/w216O16m9uDGEKtRzKboU02sa9ktTfXX2fnZc4jXLXXtcLoutoO1lMtNQwhX8aknUE8pBNuGk/TnI6FehsPFp2kdAoJvairX6G9WqRbBu2mvhDlIECVAuGTqNCDnTGvw3tl1sCsuq6sySBVXVX0Cu9MlJKk6HWeeutnJR4CxtT0i7uacpBT1LbDruZZzJdayxAAOgPjysuU2AHmHmClxrsqavfqEADdl5JGSeRBPwB95zK0c1REDyb2UUXjdrzV23BeCax56aNbyw4rRYQtKy3PQSYHhadqZTb9Oy8mSy4hKwI5ESNNtrNC7+EjlIQivvp99torS22pEJQlaVJUnUnfMD7hZ63ZQKum7Ka7+0W6KZoNJW5BJCREnxgTpYUjw61kaCJzL3QBGubNmOYRJMe/Weg62pv6TV1+pcQ7xcQCG6xmlqxrz7Psjp5tm1zykOKJbGh1EmTzm1WvS4uxTWIrqvHISmsux1kDmC05m19zlr2Evy1FuoVHGGZqa/RVyT/drT4hX4fja23o4VvrXC1hs6+p1j7A00EqC//eH5dLVJb2WkH2k/jP4Wsv6KVcHsNX5dsyaatbfAnX7Rsgx/7O2zizb0xPQhYWDutUgdQVez0NLwNJxEva7VKlFbdSymNAVNuoI89FKNrhkCJBBnUfUWoj6Nt5fuvjVcUqKUVZfpVyrQ5mVgefeCffa9qtR1nmLYtG7MxbNa2z7rkkbTr4W1I/mHnbCevKbaKp1BB8elr4OqolYVayOXU2J3ijOxm0kGZsaOg0V+dg3EhbZTrtGtjxmzkKQZhZM68YRVUbs6Z1tzPVJP3pFhFKMT1nn8rc4hBapu1CR9g82uToAAoT8ptyXCdFKI1tpN1WY4WQwJ1BPWdbcZxOaee2/ysGFzuoaeFsSoxIjQT7rFQzqhCr/FAI+hboKGmo1sAVA6lQgn5W2les8wNT7rSCijCVCQTpr52ULuMvCSNfhFkpKjoYBGkb2PXYoh4lOkmzpJw9465FGedst0kqCQAQABoNfztlldNZeFt2uLcv5pTK4IbKjpmA7x5/Xy0khd4ssNttpWQ9HfVvsDoDytGVHUvM1ofadSCtoIiSTJUoZSI015b9edlOmeNRWINZV5G0rH2Sf4txlHnPS1OGTK+yPK3My6k2jr6JbdZW1dc3SMss9h2qnIT7OpknWPM2gfDt3qrMa0d00zrK4rUI7VAELSkznBAJ1ACtfnZ2OYgvHEeMF3XcDKKmmo6MIcpnmwWgSoZu6d+WsEjXTezfxdTVt3Xi5idvLRVhWCWko7MNKCP4QOQI5TuN7FkqA6QN6ITIMrMysxIQSpzTNuCPzg7ee1kqura3M9QUTjQeKc6Co65dR8otFGCOMVbfj7d24jdS28mSXUJAD7eypT/MBrp/LtZyXziWhorwarrqWqpWzqsZSQtOkjSZ1+467i1wzNIuqYiN9UnY14gLTVfutSG+1bb7+QEnLl5yJBnTTl1myNc+J6Jy7mBVQHIepS4rQhtfsE7SAREnXWzRxFUqvW+qq9A2XjUAx2h76UnyjbbrqLJ2d0NBzsysICSlREKzcwTtMz7o2tTdISbq01uiXroQxUVgNQouq7ZGUwDnSdNZ6aGenTeyxildwUNW0y48ilqEoUlKgmSdu8Fb7TodIJHMCzc7WjprnZrWQj1kLLayFGEEiN9xpr9GyFez66iriofZdVKUpWlRJAzTEDwjaRaObRSy6p8cEK5DPGHCyG4UF3glskCI7x2Hjrpp0629C8PN+rXbSU8AFDSW+6Omn15284eFFS5d/FDDAcKghV6UuZIVIyZ09OQIn3jxt6R3emClClAlKtYG5+63H9ov6jfou47MaxP+qpb6Uqn18a6ktsqWhpy7yswDlBSjWOWsDx8dLev1BPqTJ/9Wn36C3kd6TAW5xiv1ZnI0i7TIOhkIAnp3iPhy1t643WqbvpjG7SP/CLaUJ/7aP6LFqv9VJ9UOqSNyD0+NgFkDfWw6+u3vsA5CTMmJibHYqkm6KPE5SUzZOeJCjGkazPKyg+dCT9aWTniUqOb3C11ipO3QTrhQ2FlQ0Oxs8WWrvvIJRX0zTyBqlLiAoA2YV6LyUDqkaZUEaEb6/Gzeuzite1Cy1TrYpXltISC8oGVmBJIBAE+Atk4pFM97HRcls4U6MMeHqX6m57ru9SF0FC0yXQSrIInpZUuWpaep1oSrvIWUK5QbQ5WcWq2uhJp6VAAgKAOpmeulgUcWL8pyX6ZFKjMgIgNzoCYOusyTapNFPJGBzV8GMNsFNd7UBrLqrKbOUl5hxEjlKSJtVo4Pbu91utVedQsMLaecBEJCkKBI15aa2fo4uYweQWl1NN39IQ0n62szcTXhU1V2V3eUkOU7mZIG/dPTzsfDWvpY3sfzQJWZyHHku+MfCrEHFPFLd/4QraStusUYpHlIcQoB1ClHQ5t+8LRlhDgBi1jHlW3TPH1lmiqaVZW0W2/tG8hKVmUqIzgwPG0o4dxFeN00C7suyscpqUqkNNGE6gAkH3WUrprKt6822zVOlS1GVZzJlPz1tWjNVAzhtItyVl8EM13OCiGj4PYhw9fFLSP3i0XbgvIvVAiQtKwlcJHx+tLSPj55arvuZwCezvajVtzKo29/zshKvK9r6vm9GRTOl9KkpcyZipRSVJ133AHw0ssXtSX7eNzXfdzGH7yW+iop1rcLUJCUlKlGTB2PTpbncbZK6Rk0h1Gmi08P4LGOYxSBSqZp/SDuRmjBLdRddQgyddW0HSd9bSiH01dPWoomchd7GEDmQp1JP/ACWjVWH78XxlwriJikKqGmpnGHl9ohORSmgdZI0EGfMdbPbD96rp3GEKbLiKpx5sOZkgI7N11UHXmFC0KGbJAL9VmV8YfJp0UY49uTE7l7GtorsqKhTeKMN141T/AHaGcjitSPZjz87P3EN1XnW4Oorv9RUmopL6ZqchWB3PWnVTIO2UTYHHF93mwm7627rsadp6h9g1LinsuTIe7yPU2PVWOLpfVWpq6htAWhtKcn2moU500mFTaD8V4TiCfomiw5zg1zRf+BVx4533Q4fxpfDroLiKmudWgtgEkZp357je0S1eJXLpoH0N0CqhFdf6yAV5ezC6NhyTod491nfx8bqryvtxNyrccZbeKiVgCRA3H1vaPL5o36i5+wFOVuN3tQqUBEpzUWU6+bZtvYdM6ogEjhuq1VEIZDGOSb2JMYV1NT1BoWksuFKnEhcKElI2MdR05+GonBzEFXjd+8aS9lIS5QXS7XNOtpCe0eSpMSD/AAwdh4WCvnC9XeDislFVS02EfZvAJnXT2T9352C9HfCl8IxGhbjnqtPX0FQyVDvLKCYIhQiZSRPX4W0q0uZRPew2ICp0wDqprXbEqer6u6kuzBWE6ykzD1i82ah5RUT9opMEidBbeI+6pS0uyQZjYgdfhY7fl3preGN3sqcdSbuelDiCJCmlgDl+H6tzEN21KlLcXfdcsETBWmD8BbzaneZnEvOtyuykZw22aEo3BWOKd7k6J0kTOnz5fP3gY1vJ1y73W3E50ZQQG0gKQoTr1O/yFkXD93L9YQpdVXENnMiX1CDOugjoPrSx7GNSF0aUqUo5G1gBP8QJGubwk26CkiBKz55Dw7JisXvVqvMMGtdKEFSUjMkQlIICDpqJgzpYd696p7MguZTBk+6IB843/OxBwA3yQpaQkIJT706g/P424chuUjdMqlQ0jntt/X39XHC0hYgmdshHLxqYUpVQEwJJSqCTp7+XjytFWNKb1atafZbgLp0wmdSozJ2+fv01tJRgKPfMHRI3k6D6/rEf8QW15aEpJgJUkhM5dFQdSANo6bnxtfpmhrrBVKhxc3VJFMEIpmm21LkAAaxBPOY036xaK8XpbcxZWGVZVJbhSVFMSAd4EwI1E6AgRNpbaC2mmUqHdWghJExsNTtJ1+YtE+MwhGKKhIyolCJWI0ievgTvpqCTBtek2VJu6Qn2XO2Cex1EKgxMhOwlJkRHlJ9qNL9ei204zw+u9haCjJSONgT/ACs0qyI5d5S9DO3utQ9m6a687xZu2ha7SqqXfV2G0albqyUoGpGpKkADf8PSXhthFOEq2tw22nOi7r2q6EGCkqSGqhE/BlPht4Wz5KhrJWxk6lW2RF8bnDknu3lKMxKVSnbnO2u/L8h4vJFFSX7hp26a5rNT19Kad4RuFIjU78z8BrZnobAIyNJI1SZERp/QbeXQvK5HAq7KcK5JKSJ5Ax/UxbUh1WZICF58XfddZgvEd+YOqmlF26K0sFDumZqe4oAaGUxr5+NlleJ7rZayVzTiUBQlcCEyOfIa8vxOsm+lTge7bvxvdWMX2Cilv9tV3VjidCl5JKm1GOpJET+NoXqLrFCtNLfa1qQAEtPgZUkRPeG0iB8NbHFwmFk6ruqKGtYUKJxCg4ggrI74zHfSSdRy8LLvo6X5U3TxONzuvlabxYfoHpOuZCS6hRnUx2agPPxtFfqNTh50Xjd7yXqZRUVBObKRm/VXxEQLPHhverN3cWcP3wiE09bUtZikwJWOz+ROvny516gE2JVmAgXarFcT0lm86N5wlPa06kAdcqv/AM6zKeKs2ZRggRBO0HlI8BrtaQ+KDJW1dlQQDlU6gnXnlP4ffaPygwcq16HloCY+Ox+QtxuI+Gpcu3wo56RqKPaHRJzBWsA68p21On324CJ1UFRyjUkmNLGHUACCqVbkHefjr5/1tgbCSE95JMAg/Dn4H6Nq7XK4Qi6EQUiI1yzzPhHx+fvhvizSJc4JYkpkpK/3fexUkDU/+cJB+Sj9TaaQjvJ7qQSZM8vrX62jfG91GrwTxIuzRSktmqhWsS0F/enz0saJ9pAfNVqhmaMjyKoFiFtSK1xSgqVHMcw28PrwslAHefdZbxI1DyFlSdUaRzg/rZEgpIJ687dvEbsC4CQWcVfn0Nr0/eHBZimMKN3XlU04E/5Xf/ebeVp3q0BTi0lByqhRPnOuvl9C1YPQJvAv4YxXc+4pa6mqYB1HatqT/wC6tad5pQUkgBQKEkwNenwi3GYlaOqePP8Add9hX6lIw+SbrlMhJWooJSOZB1Ov19RYq6CO8Rv7RA2ssPMdmrKpKjBM6AHQn7+nn7yzjKkJyrJGx16dfL8LVA7RXS3qk1TRSSQYiBOUnx/GwLrZKQRqUiFJn3Rt1/Cx5bSoKAICTGUHw/LW3DrZUFDKESQAcsmf6n+vJZlAtSZ6uS4A5Ble400PMfXwtAfpa3IFYcuS8YUr1avcppSkzDzRny/ugPo2sT2aQoK1HMHcHmD4Wi30m7oVePC28qwA9pd9TTVaIO0OBJP/AArVa1QyhtSw+ao18OelePJUKYJQ6I3UCn4iOdp19EqtCcR35dhVHb0Tb4SZhRbXH3OG0G1QDNc6EiAh0keAnS0qejHW+pcWaej/APTaaop+uyc//Rbrq4ZqZ/0XF4e7JVM+qupgG8U3JjvDt7Z4FLedOtROpyh1M+Wk/fb0YXvChzNvNLsnEEFsAKBlPWdNQenj+tvR65q9N63LQXmmCmspWXxB3zpCp+duZoHXuF1GINtYo3JAIBI8baOUGSZg21oNTz5Dba2lanTxnztqBZRWiTEJi2oAgSdd7Yo6ae6bcEgezzHWxGqJSBiSnz0z6EjVbagPONLJLbwdbQ6lI7wCvlNnJejeZrMNdLNC7yPU22809kS2PJJI/C2lAbtWbM2zkcSsRqQR4W2VJTy36CwQWJ31md7aU4RBkE8tbWEA6IUuGdBrPPY24SogjLsI0sGXB/Fy6flbErJOunIeHnaQUCjGYSJ08bKF2Ll0EyD5WSg5qZ8rH7tX9qFE6zBnpFnSTlzeHzNssW7cnULV8Ytln0SXhtToPbdo0ohfZgRP8MqmPl7/ACsl3lWu094KDLigVEhIGggbxJ5DQfGbHlBDVQU5kqcKUicxJ/i0A8Bz+YmyHenapqyW2ySrUInTUbn63nebZLTd6su0YnJwCcbViy8nFKzFdJI7uYzmHx3g+fnZxcfblo7vu677yacf7esqXAtKl5kju6x02Gg+divo7Ubjd/X04R9iKZCSojmpUxrry+6yl6StTFDcCASmXXlKABmQE6jlztQe93tENG3/AAtBjWmhLioSoVLaq2HE9popBUYgDw8f00s9VXkWVpbWVpSQFpAOUK0Gh5kD8POzFpXXHXAylYUCCkyJA5j693msNorKgpyrygnUExJ8Ou/LXX3HZKyLXR+sfL6hUR33BqUkpTI2GvmfrWxVWoBb7yIzJkwBAPL4a/1sI25Rs0q2nHnO2S3lQotiJJ0Opk7CxFSlpUpS5kJCYBmTED89OQHuhdTDbIwe0RdrhS4stqdBykQNtY8do6jSxZtinS0CzVla0lIHcMHUGROxBnwiLH2altFw3i0pJDgdZyJHQEgxZz8EsM3NizEVbTX3SLep26PtUtpcUgZgtI1ymdj1IO9k52VtypRszuDQieCKRf8ApGwwUZVBN6UygpLk/wC0TOu4OpMHfXwt6W0yCH8gP8RBA5T4/X4WqtdfDnCNx1Ld4XRc/ZPMLS62ovLUc41ESo7ETaSKbHOKFVBm9VkKXJUEpGvPlHn525vFqV1c5pYbWXWYNUNoGua8XuoQ9KdxLPF++EFZyuNXaQiIChCQTOnWfdy5+uFzqP7tpTzLKJ5/wi3jF6Qt511dxMTU1tV6w4pdKlRJ1UAgQPPp529mLkcz3XRqECadskDxQLXWxmOBjTyCx5ZBLUSPHMo+rWRpr8rAr7o8Ok2FnSAJ8LAOGRqN9NbFjQZEUqFGCOY0+vjZLqVD8IspVE5SdxEWSqhQzd4kRpubXY9AqTt0n3oo+pPpAPsH7rQ+h0lITPLMNeVpRxHXpoqBS1ICpWlEGBuQLRNKgfsxGhO3142r1R1Cv4fsUdcccQcxMyIkjcHmPrewjLzimgAsnQRB0J5D7/hYFplxSA6dErEAzpp0svO3Cmjuakvc3jd7oqVFHqzTpXUNQN1pIgDfYncWqFwG61W2SahzKiCvNzBzbfptYO9Q47dtZlMDsHDrJjuWPCmQSomdR0Bjp+Nua1DblBVtoVlHYrBPuPT+thlymRoi1yqMLjVICVEk66izkuZIN7U6kdwJUXCCZ0G4n42bFwLbDUZs32bevIyD8LLDLrjJS+0ohSZgzqDtYLnIrBcI6w83QY4vp4QhJbaVPQZ3Nfus6arENHT3ezX07qFMpppWlImDlSPwFoxvVxSr6cedJUpymRmV4gkGfj99lJKj/ZJ4TCvVleMQJi3M4/8A0g4dVdw+IGQgqSq/Hl20artRU0j77hUpEpUEpyuMqSQZ15C0ccPsc4gdua86FVcrs6a9qppogSoJBB3InmfrSwmIFqULodTAipZKj7lD8vhZr8PVZP7RU6yqU3y8QCd5Sg/jbnWyvNM4XV10DGSDRPl2vqak/wCsvrUJmFK9+g5W7p3YYdIQQMyDodjrZNaWVAhASTJGxgHSxqnUBS1Ikx3DBGg70e/e2TLdw1WhDYHRRlxGUkXy6twEQpsoG4Pd+vnZl3kHm1VhfbLZ9aup5zTUSXUSnl0+PlLo4mlRvolIASWk7nQQd/62ZF9EG4Kp6A4ct3LKkqGsVKwN+ma3pmDEChYuPxJpNU8pRqmb+XeVQi5KtppbnZlYcUmDmCwD0Jifo2S+Da6qjxLQU1Y4e0acq2AAQYTnXtHKR87aIq26xaxTqWCykqlGmUKMHpsT+Z5FeF9YXMZMrS1l/wC0H2zrIzaz7vztp1MjX0kjPI/ss6KJzahjz1CnKq+0wFXtR3W6ipBG8aqj7jv0s27/AINMlSUpgpnMTvpsdPLpz87L6HO1wnfzAUD2dY7KSIiUn8x8fKzUvSpSu72FwlwqaQRJ2gDX4fC3m1K20h+q7Oc+FEbrdWiobT2mx25dDYXFwzXUah0pAMJUBoSJ2Hwsm0a0+uSkZioDbYGeYO4so40dyYfnLmUmSMoiBy8utupoG+KxWLMTkJTDcP8A25ToQhQJQ0mI2OQb/rbhxwZkyAqQBMxtrz3ttlxJxBdjgXJeYAhQkEAKTy/yi23EJQokmDPMyd+cW6dgsAsflqgCnXUgFWpJJjfrz3/CzSx6wHbqpQgFSjVlJA0kEKOvTX7x1s7VIVCdhAnVJn68bI2JKJFVRphCcqKyVDf2kxv4kp+FrcRs4IEgBCZ1e83S19LTtrjK2FEb6kdeoET5+M2iDHhdbxG8nLoUoVmgzlEpGgMjY+EdNZka/albmK1IAStLRyyQYAAG3vsweJNODfs/xFhCglOYKhRXrmAjkZAP6XpPdVNu6kf0PLsw1ePpD4OVid1pNBRmqvL/AFhxLaEVFOw48xmnUQ602YmDAE6xa/l2ss/2pvp5t9p5k4gbWlbawUkO5Z1iP/nJET5m3nH6P9MmuxZRvnuOUdHULCVGMwy5BvvPbTIHIWu1gXFfbqvChXlQqnqKKqSUp9kfYI2O393qPo85W0TjVNqw7YWt91r0k7QwxEbqTbwp+wyrbT3wqVFSiRPhOx3+t1jDzyl3cO8cyVLCiQBrM7ddRv42L37SuGhU4gpTACp9/LXx8PlYLDDx7BxsqIUl5Wk9QNduv1pNt6hkzBZlfHwym96QeEjjDhdezLDWaru1k19MOZLfeIka6p5c4i1U7sq3LzuxqqCkuO5EoWh0DvEAjyG3LbytetS21NkKKS2qJS4AQQRJBH1obUkxPhRzh7ja+MLk5EUb3a0bhRJXTOd9oH+Y5YB/xJMaW0gs4bJBXedBTuOU9XdymAFlLiuzKg31GmoH5xZHqaG8KWtpKy45cu9tZqWXkLBLK5BiCZyyJjw2s6y/SV+ZvtGfWhqhsHNJG8E7gz9aWal54aVRPKqWS5ShRAV2JhPgY8RpEWhKzOEWN2Q3Kt7f15t4jwLc1+tKT/rIaeBCtDmQZ+JFmaGFJSUhtWih4ieZ6cvv6W64VXw3fXB5u7lJKn7oqUsuyqIGeQRHgo/WljrrCCdjlgSkjUc/ONNrcLjX6dT9l32A+Ol06pPcazJyJGgjUmJ8umszt+NuAyUnWZMmeUjTfx91lJVIe/CSZOoA85+j08LYmmIJbVCcwJEk7z05fpbObItd0d0nFlQMhsAgbETzP17turQr7s9bvrFVzutApvG5wQAdT3FNny5fpZ+dkTIypzg6nWZ5e/WzbrGjT8QqMkAJrbrdaOb/AALBgctlT+liCSxugyRaLzRxQyWi0SFqWkrQSUwCfqfP42biSM2qrP7ijdq7sv29bvISBTXk+wU8wErUkbT05mzAJ/pbv6V2aIFebVTckpCtb6Ad4BOLMVXRrFTd7FT1gtu5Zj/62121s52wQAZBHKN9flbyg4f8QL44eV9Re1xXneNDVvs9gXaKoDKi2SCUklKtJSk7crPWp9JfiJUQHcT4nUiDAN+upJk/4Ep+VsXEcIlq6gysOi6DC8bio6YRPGoXozU0y85WUwqdQr66/Wtij7PY/arypB3JGUEzprbzWqOOWNKpZcevK83TO7t8Viv/AHosj1vEi+a8RVUtO9rIL7jz0f8AG4q1duAS83Ky/tLDyYvSupvS46VIU/fNDT6QS7VISB8TptZKq8dYBowUVeNbhYiUkLvJlO2n83X66eddXje713GxTUeH2Wb0SuXqpSGFtqTroGy1I5bqOtks40xBMtv07es9ykZT/wBNit7Pnm5Af2kA2YvRB/i3wqZyqd4hXDmA7wFe2qPckzZjcVuKXC2/sAX/AHBQ40oqmqrbsebp2WULWVu5CUJEJ3mN9Nd7UlGMsUBGRF9VCADPcVl193nYJeKcSL1Vf1frp/5wv87WIsDbG8Pzbfzoq0vaF0jCzLui16ILdYsFKhMKhQ12s8ODFci7+LOGanQJcr2mjJ/7zuHb/NZjvPPVDhdfdW4tW6lqJJ95spXFeK7ov26r3QrvUdQy+NIgoXP4W2pWZoizyWDC/LM1/mF6Mu02ZClJOsRJGgPu+t7Xg4LXsm+OFmHajtSotUYpD1lolvX/AIbU3NEh4Z0rCklIIUn+U6g/Py1taT0aaxT/AA4VRL3orwfbAGwCsrn3rVbiMPk/WLV3mJRfohyleQZVOswbaXA5HT7raKwDPQ8utuToOfibbwNlzxCwkc7cz3tVfrbZidJMTYMkphWliNKgUHUp7RoyIB982YiT2NZXU8Zcr+YCYEKAP3zZ/GeXdmzFvlBpcQvI2D7CFjT+UqBPzFrtOdbKpUt0uuSsT5bRbCsjUfxHTf6+jYuVQoiO9PP47212hKj3TrFr40CziTzRgLkxPObbCpkGR52Lhz2iTIB/O2docsCNefIWYJIyl3vROx5iOdjt3KQFhUgzpFkrPBJ02JMn8bHbvcJcmfHW0kk4gWo1Qj/hNssT7Y81GfP9LZZJLxJbShx1JGSEpRIPNZGumnnJ62bV89yoAUtJlSiDlMefnqZjpZ63E80KlxL1zIqiWkBvNKshCNB3dRvGvSzNvgMP3wrs8jas8DtDlAJM8xtrEx91spnvq04eBFKC+7wupLwu+scbK0pzAKgSkzm035n3CxW+sSXxfxR+9LzfqAyT2aHHVKCQd45chtZT/c4p7tdr3XkunIowkBSMoHL5a/1snYeo2K6vU3VoWUpSe6kgT4a6Ry958rWLNJzWQbuAy3QF3FCXD2iQVESkpgR5+74QdrLKVJUITUNpCEqJka+cxrZMokssXv2N4JBaT2mZJOhACiNt+Xy1sUefcIIWo5hy5DroP05b2kRdII6muLhLSFEpTKYGw3126A+dh/3felQhVSxTOLGhzhJJ0HMgfXU2IZEC71OhgjMYTz1nWOQEiB1087LeD6p9mrSz6yOzfaX3PaPdBIMHy+doW6KXJCUV2VIuS9XKqlXTuoDQQtwHWTqOh2nnZ+ejEioVxbui6C0H2b4cFFUtkqnIVicsHQyBy/KyDVsXtXUblFKEKbbShtZhISU9IE/HWBp4yZ6Il1Vdx8fsJ1FYtotpvBlSggkkAKkjY7wPz52pYo8x0cjgbWB1VzDh/wB1HpfUadVde8uBV2Xa+Gaq7rwoXCgKLTxKCUk6HUAxvZEPBmnrsU4cw3czvqb181RpA68VLShWWQTp4G1sMfuUeMMRm87vzqZDKG0lSCNt9PfaJeJiavA1fhTFtIgKdu29kvpaOgchCiUz4gEc97eN0ONVLsVFO2YvZcgG+hXqMppZ8NzGJrJSOmxXnp6aXDi8+FnHWqwdeNUxWv0hoXg4ylRQtLjKVCJ1G5BB3+/1pw+ofuahIE/6s1zn+EW8p/T54hL4m8fa3Fa7tTd6Q1RsBie1VlQ0BJJAAJ18reqWG1RcN3yFf+as79OzG9vXic0TSvMg1zXuDt+aVnVg6D5dLAqI8QbcrcMxynrYFaySRMWkxM8armpV3CkdJ1HSyPUqIVAEFPysovLTBnTUeBsl1RAzEDW12MKi/dIWIKf1qiIBAhxCpUdNCDaL2EoVGcQCN8vlr9fO0jYnrPV6VALaVBbyEAHkCRr9eFo9pUutOB0t5pGh3kf1tUrCGkXWhhoLmmyO05GVLKQQc4GYnXX6FjzFM+KlFKkoK3FJy/agATy6AfL3CxJgrWorICNgByj32N0TgaWQHgYJ0kc/K1FzxbdarWuulO/7jvbDV4uXZeHZOOJShYUy4HEFKkhSSCPP60NkqobfqmXUobVJbUI8YNjoq6hxtTDDKio7QkmPrWwoum9HmVLZpKrMpBBKWVHceAsDjR8yEcMdbVIuHrveQw0lTZbBZbk+Q/X7rOu5MP0d4vPU9ffjF2IbZK0uOtlYcVIhIA2Pn02Nke4MN32/TU7q6avP2CBm9XVqYH62Muqu273HEVF50bK2yUrS9WNoUDMEEKMgzPlYZka7QFTa1wbrokS9FNt3m2M8EsqAI0kBX6iw7UO4aqG1GT2LonmNCPOyRfuILjor5ZqHb+ursGad1LrjdcysI1QQDlUTJ1IToTBgGCbF/wDSRwvaw5WrqOI9yM1YaeU3SKdzLUrXKCAeZjlztj4xC+WAZATr0VuilZHIc7gE4b+dKrsuh4JWPtKUn/iGtm3ggOKvTE1OJBF55xoRllCfft+W9nVfxQrDVx1ly1CaxippqR9DoQUyhQBB1PL8bM6kvG9rtvm+zdxp1OOVKFLKyAIyR+FsKmpDJC5p/mqvzSDOCnmlt5AJyKKIkzuP6zY5RUL1RSVzil5S2ylwJGv+0SP+q0VY5xRxDwi6husfYQH0B1tLSWlHL0IOu0dbN65OOV+XI5XVmI311tK5SqaDLKmEqzykgwFTysX/AKenkjzssVU9rRRPyO0Tox9QuOXxTuLGqm0gSNTr/SzPxV/2XdV55cpKKKncCjpIRVoP/V8DtZKvrj1dN53k3U1WHqzsEo7PIKltCyudN0kRqevL3iO4ho8Y3BU11NRO0jNRd1ewGy6l1WZtxhU5gACNZ26a26mhopKenayQWssipqop5XFhusxTj5Vx3+miSFLF4MJpYBEQuCN+UT9CyTw3PZX88862EuG/DkSnUpSQkqMzp7XwmzjS3dKHGEFvtMzJV9otIM93WSFfdZIwq7SUWIrzcdbQyo16VIWupSmJbQZkogjx2kRrbS4DeE4W5KjxnGQa6XUv09Uhu7MRMOOIQfWUKTmIT7SBtOh/OzXdbcq7oYTTqbUvsEJKc6R3oEjfl+JstVV94GpmLxpqi9roSy+8pxIqLwRJzNhJgjLPPYWR0cScAXXddLRqxLcNMQgIQtNaSox0lcHQTNuJjw+UPJa0rqHVERaLuHqkekoa5FeCtATPtEyd99uUdJs4cS3S7V3ItlS0EKSOoPz23087N+o4rYHdWFnGVM62ZXmStS0wn2tUjoJ05DlYzenGG4Kijauiiv5au0CUJT2KkhR0PIagzGvjbbpaeZrgcpWbNLAGkZgmo7Shq65UELcQkoS4ESUp7RWk7jy8bBO01S8ciKOoXuYQhR5jw/H4WLXxiapu67luNVtUwh4uaNrUJggnQEfzaT4+Nm9XY0rEUxfCbxqkhJKW0Fbi1wNkJzySSdhbfaHbBZBcy2qdf7pvBaNLnqxlAMinVp4gjxn5b6WLVV2VyKZ9x+kXlYyrmBKOU685HMTr42IXthvjLd1DT3m/wYxE5TV1V6oyS/Rgl3IpUKHbkoASg95WUap/mTmJ0VxcXu3eeq+FrdBQMEKq6moxHdmZluJKg0l4rWe8YSkEkiBJ0tZjhmvchV3TRWsCojrHSvE1SkupPeIKdJAzRvv9bbWbeOkF2/G1KdCCKJnN0iVZTBBG86/fFlanqQb4vV4rlOdScs65dTE6gCBPXfbQFJxasm/EuIbIQKdhBOUd2UEjYHSfLzOlrsujFUjN3Jwej9CceN0RqG6cKp6oKU7IiAT3oB1OXpufOLM4arqejxHWsovFkrdoElBSXMqsgdVEkCCCkHWPZ+NXOGVQaXHdO8w+EF1l8pJCRP2e8mRsTtpoDrraacM3ktvH11tOOhYrqd5gAgjvKSoDWeYUI8uU2qGPO25RhLwnWVvb4xmVti6UUqcgbSguEnMRlTrH4xy+AmEapDi6hSQApCkkmIGoifr7osxTVIeTS1AM9pTMrnMBMoTp57n6NnJgysQF1TYRlJSDMHKTP6nbla/BE2P3VSnqHzHxJ9oeMZzIO0gwBvzPT8BavfpS3Kmnrrgxi1lQHELu2pUE8hLjRJHIZnRr/N8Z29ZSJCF6lWsj3/nFo74+3Z++eGd5paRLlEpFagp1goOoieh+uVpAG6qXe12sVawqmcVS1EaKRPfPP7t+eo0sni/sZYZa/wC2WG75u7ZS0GKhtG0xML2894mLOGhbpqgpadbHaglSeRI6ePnvYW87vfVTeriidfSvvJypzECOfMxMafqXHkpJ/cCcS3bUVN4XZRVIXR3xSds0UmSlxCk6EHYkQAD06zaXBdy1DOSRpmlQM7ePmP1tTHB9deuD8WC8qVxTK6KozOsQoJWhR1PwJ1O3vtKt9en7w8w/V1F2PYIxC7VsSHUqXStJzc4JcUfKRNuMx+iqaqdroG30Xa9nq+mpYHMndbVT2LuIGQJggxO493uHy5WaGO8QvYYqaZDFMFl1srClAAGCNPEwLQdV/tJbtaKv3bwukcu3vkJM+SGFzv1sy8W+n1eOIXm3f9H2HZaSpKS7UPumCRpJaTGw5WyYsFxIm5Zp9QtiXHcOAsJPwVNq+KN6dt2bNG0dAQDMDfwgmwNNi69r5xPc9QxR0xqEIfbbLtSmnRK49pxyEjQc+ZjzrbX+mtf9SkAYCwuSkHLnpnF5SZndQ+jZPc9M7GGdK2cF4RTCgdbsVt0/vBa2MFr3aFoH3VF2O0Pzk/ZNHjk4qoxziVwpQhx283nXG0LzpSpa5gHmNdCNNBaKVgpMEROsRZz44x/W49xBWYivWipqd6tKCtmjbDTKcqQnup1iconXcmzYdWVrJTMTp5W7KkidDC1j9wFxNZK2WZz2bXXEkn9LZJtsDwNukMur9htR8hNrVwFVAJ2QfKJtllWmwtiWuKfUsPXlUZtuypXFz8BZbpeDvFiuT2lHw0xQ8nqi6XyPjltEysG5CmIZHbNKZ5tlpFZ9HfjY8ATw3vlmf/SGex/8ZFlam9FTjfUkZsMUNOOtRflA1HuU9PysJ1XTt3ePUIjaKodsw+iiS2WnOl9Dviq8tCau8cKUebm7fbS4/wDZZj+NlFj0NMVqhVTxIwShEEns36x1WngKeD7jYRxKkG8gRhhlW7/xlV7sICezSeijr8LTze3oj3zRNJ/d2Pbnr3ioBSRSVaEBJHtZuzJJ20y9elku+fRxxDT09IKK9E1jwSUPpRRuIDcE5SFKjMDsIEiNQLMMSpXbPSOHVTN2KYLk9MbBVFhu7aGrwzez1UxSMsvLFQwkKcQhIUr251IJEj7rW89Abj1c/F9vGl03fdT1Cq6FUNRDr4cLgdDqSRAER2aRudxbzYZ9GnGjzq0LrqBhAPcLpczKHIwlJA5c+trZ+gDhtXAPiPfVXizFd1/u2/rp9WVkDoKX0OpU3Eo1MFYjxtkmnw+JxfEfEtdtTiEoEco8K9LiBBBMdPK3BEkFU2baOJODHBrfraY3lpwf9NuzxAwhl1vtpMSPZX/8NiAhQIKXzJGWNPG2lHUkiROulm8riHg3VJxBTyN9FA/MW0OIGDlwRf8AS6kcjr8rFaQoWS+ZOhI8bMzGgTT3vdtSRHbIdYn3JVP/ACmx88RcFqOUYho5iIkzt5WaHEDiTgZd30dQ3iu7ZZrW0yXgkAElB1O0E78rHhdZ4QJm3Yhy6mZzdd9ibaU8MsgQAJ56+P42Z6+JeAp7mMbnVzPZ1aF/cdbAL4r4GbGX9+hYEqPY0zzmkxplSZtqBwssstN09i6rKYIk6aibYHV7/GZtH7vGLCKE50G9HQRmHZ3dUQQPNAEe+xN/jXcwSVMYcxJVka5WqJIJBPLMsfdZZ29UshUmIdklWY66QB+djdC6rPvpO4M2h5fGrKErYwRfaUknV9bDWUzz7552CTxxr0OpbYw1StKkgiovZtKiY6JSo/XnZcRvVNlKnr1hX/dzbLQOfSEqp790UKVcx+8FaH/2Vss/EYnyFecWG73prsrnFVlQ23IahM75Up1kcgNbN7Eibscqn30XYhROZye2V3lFR1A6GR8eU2WLpuJi+u2XUuVCXG+zA7MBUS2CZ8dPmd7GXMNUhfX2y1uBKYcJA1010Gn142zmtANwjuJLbJnUrwVhZ1fZqbydqhISnSYOniYPwnyshXRU1LDjjlMklcphQBOn3/XlaWUYXu9V2Jo2WVntVLXlRmTMgCfl7p8LN7DGH6Zq9HkdittxKCCHASJCtToYO/hpaecBR4ZTFaQ/WV+iFF1ZIG4JVOnOehtst1DAW28nIuSlQOneG8+P66WkGluSiOK1JNK2odqVEqOitAoQM3htvvFiFdRUbVa4h6kZLnalJzJBJUegJ8fy52YypxGmW29V9kA6oRlkTrrPv5nfw+JzD5q/3nSpacASlckjURGun4+VnOlttlKUM01PCQCSGkASNYk6/lB62cFz3u9dylKPqq0ue0SClQMKlOngB578tYcbyRBEDuUSYvphwlo0z5LitSlgke6PADkd7SRwVxjdGC+JOG8X3o1XPUdNUmoeSzSlTmUaEAaA7zG/SyVdlfc9Y6C664lxWhBWU5vgZ5bG1g+B/EC+LsxDdVw4forhpS66hAqnLoZcdKSdu0UCrmee4tkYxUPFK9uW4IN9baLYwqlD52kOsbi31XpFwsuW7cb4RuvGFFUrapb1pkVLLbzOVxKFCQFCdD4WH4o8BGOIdzUd2ovlFCKWo9YU4aftMwCCIAzDmRzsNw2xOGblZVUpr62qKYcdRTHIs9QE90e6zyfxfUdiQ1h6vfCkkEJyII03OdQtw3Z+HAhA2SRtpOW5RMQqMQiq3AO2Pl/deGXppXF/ZnizUUyHy+lYp1BxKN9FAbmeU/Dbe3q5ht1IuG7gND6mx/8Ag028qfTwvBD/ABuqAzSGlSGaXKha0lSAlKhymNj7/O3qVh11P7gutYWNaGn08ezTzt3NEXyUUT37kKvW5WVcgG10vF47lQ0PKwC3SRvy26aWA7aJUVkE2CU+lIJBMza7GFSedLrbqlEEkiB87JtUuZB10iQRYR6qOoB+f5+6xB+qSoKhRB56xra9GqLymvjZ1SKBASVZy+0BB/xg/ODaPF8VqClSpn1/DzawezCTQUxUFD/MfM2fmL1F6gRlT7DzaxHgr5crVSqOG2Nqm9qx5i87oZZdqXFN9rWEHIVHTKlJ8PeLUq+kiqbcVXcOqZIL5AprPG1i7kKzYioGAhPtNUlKiI/yoNkCp9J6qF1m+6CqxjXUJLQRUUdxVaULLpR2QSoU6Ukq7VvKATOYRpZiNcF8Q1zeStxPcDJMSQ68RvB/2ImdrOO8eH/Em9sDVeA7x4yXPRUlaw005UXbcznbgoXmzJUXUgEnOSAAJcVAACMtCLDMPB8S0ZMQrLeEIWo9I3HdS4UU/D3iw8YnM5QuU6cpkSC4tGmh6bWTneNnG69FqbujhHfThADYFfiqjp1KUrTVCnySIIOk72I/6Dr/AHFKcqvSYxMhayVqNDc1PTHVOUmUKBEjf42Nsej5h+sR/wBuceOJ1eUjLJvdTaQOaQnNtoDHha2KLDI/hVc1mIO529Es4Yf9IrEWFf7QXXceC6dph5d3KTU4hdrSXm1BJlTDZb1Xoe/IyqBE5AuvLt53tfF4V15VdV2K7yfNU4yw+0tCFqkqCVQoECYkHUczM2sfdvArhlctyi56XHeN3qRTrtQpkXi0ErdchTi1/ZHOpRSkkmSYBNspeBnBK72g21SXupAGVKXbwCRHMJyISP6Wm2Wji/pMKG9lVN/VcoHo3nkUF5tpfcUvNTOZyUGVd8SMqfjp7usfXyy47er9Q4+lt4p7NYLhB5HknQ6+O3SbXd4f4C9HGmxXX0+MrvdaucXch1PaVdW4VPpdEA9monYmOXnZ+P3B6ANM+updwg1WOjvKKmrwcnTotUbco6+NrcTGyNz3A+qozF7HZSL2Rz0VsE4f4pej7cddf97pp74uxs0NQGHy802lMKZ33IbW3JmCQYkEG0k1Ho18PXX1O0+LLypnnoLq2nWyVmImFJIHlZiYT40eiZwsoHbt4e4Mq7ppKlztHWKG7ktocWB7RzOCTrv4mx2s9MLhAynPR4LvZ/SQVtsJnX/MTbn58BvMXRvAB+q0o8TmyAHN+Fxf3oRYIxRXiqvji1iJSEApaQ2ikSUg6xmLZnWTtZEqf2efCzs0pZ4jYodMqBUamjSQDMQBTGYnn0HvMv8ApuYNbVF3cO6lQCZBXWIT9yDZPe9OELBFBgBhvkC7WqUPkkWsMwt7G5eNb6BV3Tve7NlufOyUL4/Z58Kv3c0qhxZfoq0tgqcU/TkFQj+ENA6gkb2QMbei3dWDr+wnw+wPU19VR37QXww7X1ie1RTvrabKc/ZpASIb209k22z6auI6usRSruC5qJtaoLi0vOhOmg9tOvnG9ka8fSsxpfF+Plhu6kO4eD9dTON0xhZFO4lUhSiDovad7F7iGnMZnH7JMllB90LVL+zvxIxTNPI4o3SF5NUroXhlkaj2zrtrZMT6C18XZTXhdruH8N33WvU7zTd5OZp7ZeYh2VpCgUk7D+YiYSkWGvj0weJC8L0153XjGjF4uPllylTdjAI/xQUkxGszGsaGJZ1T6VfHerckY1daSdYboqdvMfcjwI+tbrIQzd5P2CBd7twPyjCPQh4vmmYo0YCwAk05KkOu1LRQoqHflJplHU6iSY1EmbLTHoX8cHDTn93cLaQMoS3lyv8AdSkJiOzZTGoJIECzIqeP/HSs7Qrx1iASIhtwoAPhlA6GyU9xT4xXiSKjGOJXAUzlNe8fCdDtp87HbwhyJQ8khO4CnK5fQ84yUlyLuivxtg9IXVLqlONUdQ4U5lBQSmcsgQB3sxICQfZTAFD6CD13qZqL14pUqDTlJATdsAkebvMeW9q+V+L8bvq/7QxJeCgBr214Lg6c8xjxsRbxrRUz03ziqgayJkJevRoHNGk5lTGg8dbJ72W9z8qTIjzcrcJ9E7A1WpuivLiQFoQFANs09OSAQJ0UVxsOX5W096IvCSmpyKziDWtrElFQxT07LqCdlBUGFDkY5C1dMN4lbTTIvK78V3C626jOgU19Urjzic0CGkuFyJ11T+dlml4juVbZQKhS+zJHNMwddSQCPLfflbn6yjq5ps8L8jei1qd8LWZZDc/ZTnjPg9w6xRdlNc2IuOuO6iloK1q8aZNFX0dO43UoCsqwpqmzpIzEyFTt0Fo+vTgRwHp2H6g4p4j3vUsIW4x69iuqca7USpMp7oV3tSCCDPjZpnHt3sNntjVJcAGvZiNJ9qFGPnYgriLQh3su1aWTplW8WyN+qDGmpHKLPFSVgILpnFT4lK0WDAql0CEsXrXNF9anUuHPpJUqdgBt77JOJqpFTiB2idVDblM2oK3IITOnTc+8bRBsp36WrmxtX05ZkOuqjMqcxMgCPu8OUmS2ceN1z96M1FG4e19RQ+dcoSQpYieum2sb8tOjlbdtlixkBylT0e37voeJFEbyu+mq30M1QaRVMJdQrMytSjkUCDEaeY2MWsXe98pTWUj7F03XTt09ShRRTU7bJJ1BEhO0aeHTSLU24ZYrql4mu2qpUst1zVQGlB/MpuSFAylIkjX5HQTFp/r7/wAT1jYp2a65UkfaZfVHEBX8MFUEjUg7fwaWqCIk3VgygCym648UhN30zDyuzNPTpQDnUUqygDSE9Bz+Vlq68e0NPVlDy3HEpbOYyNDIMTIJO2w+Vq502M8ZUSECruRFWB3CaN5KsoiR7WTSYnbysoXVjqrZce/fFxX22jIChKaBbpJBB2RmhUHrHnaXElZsVFscL9wrIP8AFK4kZkoS8SBKRkIjTp08fumyVf3Eq77wu6tu8Xc86h5pbZ+02RGnLUaRaETjWpeSr1bBOKXs47uWhQiZ0ABWtMeI5aDTkC7i3EeSWOGuJVAOEZnDRpCvH+/5yOWnSy7xKVMU8I2TPrHKuiUUUDSu1Z+ybCU5u0ImAQOcT+HMWHouIVdSU/7vvu7F0r7gKEPAKUiSJBmSY2jXTxto1lW3eKk19zuXa+9Ck0ry0qVB0IBQSD066crKrDRq6IU1Td7eUjKlK/aIOhkeGkxy99tJhLm6Kg8ZXWUbsLqaXE7zlfVmqdqGg4XE6COURuJHmOe4szMacGb9xziFD2GXKFtfZTUduvsUp2gzBnf9BaR8cfuG4nKOloe0L6AXEyvNkQDvqO7OnhoI3gH8DVT4vlhDbjSfWQCQqQkRIymNtz59elGokdELs3V2BjJSGv2UOU3olY+djtL9uBtRMZfWXFEDXUwjQaWU6X0PMSuEJrMW3a2SrKQ00tyI33jUWt61hWqWjtV1rKQoZpCSoRpB5HXTXXnPiZ/stTlcKeUdZlLStOeknr5fxG2OcTqid/wtgYZS72VTqf0OWQlKqvHbpGpPZXcNvCXPqD0soseiFhgNgOYivl1ZJkpYaRtyAk2tMjDTCE51POkhQUQEpkkxA5/QHSLcJuRCAQpDnfkxMFOwA2jX63tA19UfiUvZ9KNmqrTfonYZaWVGsvd5IPsqLYkf7v52UKT0fsNU/ZB3DNO4GNQVuLOYwPaP8Xlt8bWabuWlChNKoKToSXOe/wCP1rY03ctAkqBaHtaEpKpg6+fh+dl3uodu4pxRU7dgFCV24FwywUzgy4E6ADLc9OpQjlJaM/ebPq6KG5acBpDZpRySw0hoDyCEAnb5+cPj9z0rawOzAGWYCYKj+f1rYdm62ATm7MA84AMeX1vFqspkk3JVmJjGbJvsNXIdDWXs5rpFYuNOWg6fcfeaXdOHHgC/SV7p3lSlL000knXpy99loUVMSFtvZeUjlGkdPrW3WRhAKu1Qe9BzGAB4/XO1R0DirrZmgJDTh3CiYb/ckhQkjsEpgaiNQLGG7luZuOxuhttJgE5ANPLz+7lZQ7ahbQSKxlMmVBOUFXnHh0sGuqusArVeCE8zlTqIB/KfjZhSOdqUu8sCLqo6RlIZao0pTqlMNhURp1Ej68LBOUza4LTB/l9kR8fcLGFXrdyYArFLSToAFd0jTSdP6+8l3b9oUd3I8SgmZIBjkZOoOtjNondEJ1a3qiz91B0LUhpRUnYaajruR+FijtwuqKciQDvGYjofhpt1sOrEqKdXaN0rqwoc3SBttsQPMfGwasXvhGVqna1AgBRMGfCJ6WMKF3IIBrWdUUXhN5z2Xm2wZASSrcz+njbTWAXg+ioTWUh7NQISEqMkeM9bCOYsr4zAhsrICYRMyPnufxsFUYlvFxMh9Y1IEo0O3h4/fY7aF26A+sbZPz+2XElCswvO7VJTpoyohI02Mxy38ffbSuI/ElCEhN53eVgzm7AamBpPLytHSr/vInKu8HSknNCVQPl1/CwDl4vKTPrfaSRCQdxvvP18rXmQOG5VIzMOwUkDipxJaUkE3OEEz9oyTpM8+nu3sIrjfi9kBVUxdUHMZCwnnOkp2AP1Fovdq86lAJBInQmdvMzsBrrysTdUUq0akpGwMR0+psdsPVAdKOQUqVnH7EQaCWKOgDhAClSFCdNu7tr42a198Y8V3pdtVRFqk7VxIKc9OlbIWFDQpyDoedmWsvAKQGgkCJ70fHT62sEITmQAFDMNAZG+nysdkQaq75CUZVxEx/kaDDd0UyhGjdAgADfnr0/WxR7H3EhSxlvlhhajrkom4g+YsCoA6KUo5pEZjIJkfhYFRSk6uKWknuxp7v08PjYGgVY3ujBxVjpwfaYsrBmH+ybQmOswNP112sVqLyxHUFPbYsv/ACohWRqrLZUImO75/dbgF0FKQoBMmdNvGfP77chLqHFxmk7FKfa06c5185s4N0yDU7W5i65e18vKByku3g6sHTmCojb7zrbVJQUS3FOuUBVJzSZPIHmeo/pbS1uFEIKswMa7gzBiLd0rr6HA4VqMEbkHNy/Dpz99nUeaV1UtMklPqdNoY9q2WD7cjTt6Yf5iqffbLJSUBYTpKpPrLaLpqHytwFPZtKXICEyrSZEzp8jY1U3NiJ50qZuGvymFaUjpk7b5dedkJ2/b0Cm6egvyoQ0RAp6eoUWwEgbBKvLl46WDapMR3stTbVLetToR3ad5yYBGmWQTsCJ8bUuIAbkhHLdLWKcbVz4mYJdXd1akDSFslsak/wA3ujTbrNkeow/edO6qtp2qRhwuKWQutaSVAmCIJBBj5iJ6KFBws4g3o+kXbw9xJWKCVK+zut5YzR1y7jbSZjWdrOSl9HfjdVDNS8JcUw1lzZ7rcbnyzAT0/O0DUxDd49U4he4eFpTHqrhfdf8AW275oGnyElYVXMkGCCYyqJGw018bdVtzUtc0n1jEN1IegpW4ha1FQ3juIIO206T8ZFovRR9Iy80hFPwiv1CETlLyW2gnXYZ1pMf16Q57k9Cv0laqkFK5w9do0O95zt71pkpXERKQudvHkdp1G6tpm7yD1Ck2mmcbZCoKqcL3aw32T2LKNZjOvs23jIIBGnZwIE6cjHmCbtLctOMovqocy91RbpjuFTOsRz+trOL/AGenpC3mpta13KylOUlNTeZB/wAQSpCFxJJ8tOYss0n7M7ixUECuxphqkbUDIbW8sg+A7ODvvpqBpaHtKiaNZAiCiqTsxVIbqKBltCk11cQkDKCyk89/7wGZidPnZw3NxavfCt60943TVVyamnWFsudi1lCkkwqFZs3vEDpa2NN+zGvpsJVePE670LUZhmjWoZv94iRJPQ2TMRfs6blwxQqr774yKPfhDFPcwCl66JTmeUDtzSRYL8Uw+UZHOuDysjxUVcw5mCx+qhwenhx9pmSzR4wvVthIlXZerIyzAGgpxGp/PpZv4g9Mfjpf6CzeGNb7fQhSsgNeWpO0DsUo8eukbb2ma4PRQ4SXSnNfbF9X+oxnTU1XYtKUZIIS0kHmNCSB42Wbl9GTgvdJXWN4McfQ4uQ3V1LjsRuEdBrB5bedqjThEJvHCPQKz3fEX6ufv5qnbH9suOOM7ow9QUia6/L4rGaKkU7UOEurWopQlS3VGEjMdZ2mOQt7SXJhvFdLc9BQruJ4uU1Kyy4ovNgFSEAGJUOYtVvhlwr4WYQ4gXBjFOEbpuZu6rxYrF1qlrZRTpQoHtCoqCQkAH2hFrbv+klwIYPe4s4ZJBBhu8EOH4JJNtyidBXMuPCAsqsjmpH2dqSgxh3E5VBuohX/ANK3+CrYrCOKlIn1Sn22L+vyBFir/pR8CpAZx9T1OYSn1ajqXfeMjZ6WAX6T3CpwD1SsvurB1HY3DWxy1lTQHS1zgUzd3flUjJO7QNR5WAsVPCJoWxvq+o/9FsHCy/XR9re1I3m6NqXr8R4WSXPSZwiof6phXF7+4GW6w3P/ALRxNgV+k3duUKpuG+LFZpyqd9TQFQfF+0g+lZ8Q9UwhqH/CUp3hwSq7xZDNViZsSQruUZ3Go3c6gG1ZscYGvfBmNrzw/S+sXimmWhaakpS2FlaAsjeABmjmbT0/6S95rSk0XC6sUVSP9Zvhhr/wBdqocbuH2NuK/EO9eJLV3Ul0M3t6u09Qt3gt5SFMtJQlRWEICswQTsIg87VqmWleAA7VWaWGpiNy0p1Iu+/SEj/U2l7BK69lB3B5q8/rY41h29alAXUX7czM6FK7yQqDz9mfoWiZjhBjmmQlKr+eGQZUZaxeg/Lb4fFRa4L4mqaJyrfvlLrlOB2jfrC1fZye/udtARGkg9Tanan+ZXCKgn3VI5wuttwCpxVc6Ecsjrq9IP8AKj8bCfuW4WiXKjiHd7Ujf1d8GB5pHL62tF44NPPAl6tQhURISSofEHx+Fke/eEF7oeU5ddTX1lSEStptZSlwRqUDYK0iOc/GYfT7XUeHPvZTU2xw4aSpD/FBpa1CMrVMCJ8QpwWBdvnhPSJUl/GV5vKQdS1TMEGOUdrM9P6C1Z14dShYBaqMqVBCwXFdomdIIInQmOs6RYw1hdCnVKepMhnvZ1SUj+EHqJ6RGp8bSAi6KNpFLl+474aM1Ieu3EN4IQtvs3C/SElW2VGRtKlTuc22h11Fo9vPixciVrTSUd81SUHKFhumbS4BzAceSYPjrrrEGxWnuO7kKk0KAf4wo5ZPgTM6E6EcuekdruG7lhJpmG8hjMkwVpM8pGh5QdN9RFpBzNgFAxPKTjxKp3iS1hy9FOJBVDtVSIEDxQ6uN+li7vEyscTKML0YiCBUXupOUxpORhX1O1l5i56NLKltsU4UCQlWbKtI0kKH5aanewrVJRNlTdRT0JSD3ihsJIHOQN9ht8LPnaOSXCd1TUVxBxSs5qTDtxIJMgfvCpe93/m6RzB/G3CMe46qSDSsXAjLCu5d9S8oeEKdQeXSz0TQXeQFt9kpCBJOSU+Gk6aA+/3ACqpGgpCXVNO51EZ0yABGsGJn4fCz8RvRLhO6pkvYp4iVSM6bzo2FEQFMXFoUxvKqhfjuOVg6W9OKDJNSxiOrQp1tSF9nQ0SISqApMqbXuCefLY8n0UNtj7GrKm82Ud4kaK3zcvCZ326YtN3kOIDLqEoSJUEqUknfcanSN+k72WcHknEVuaYwRjqpSttWJL33yryro2gCf8SKcR7jYBVyYp0m9b/cBUZUq93U9Ng2pJPhFpCUllaA2h1LkwSYKUg7zIEQPvBsUUulpVQXSwUju65dI1PTmT8bLOeQUTE0c0yH8PXu+Slyvr+zMEB29q5QCifF73e7xsWTw+bMO1VPRvISSYqC4/mMHUdotUc9OelpAVeF3lkOvusLGXRSklJjTYyZkT5/G3CL8uLtBMoTsC4nuxGpJEjl79dtLPndyS4bBzTGRw1p1EH1G5BK9k3dT5iNdNUmDPjpPlYdrBaaAksIp2ERKizRtInXllSIj3naN7PE3vQq7zb5dBUDBToNDrJAkydPZHzkmu+Wm1JUgVaFH7Pvagxz3gQAddN+tnDnkJZWJCbw29TAJVfFfKoltuoPenkIIM6kRH52DXhRJKfWLwvFQX9p9pUq+ZnqOnMbQBZWXftRHbJSHz7MklKzroBMgnbXYR11sD++CSta6aoZW0Ncq8xSRz5wIB5nppzcZioEsCSncF0SnEreS4pYO+dR7x8NYI0Mc9ekW03hWhbC0JpAk94lJSSCkEEnczrzAIGtlJV5vOIWpCG3mykthOvtGQDI56fLwsXVV1jTZbZLK1ZSpKVNnQ6A7nYGOg231tINclmaNkwOJjBpryYrGGkDOhIzCEiBy+MmAdPLZIvqvURdFWhxWRbCmVKnTRYnfWe/8jPi7cYNO3wzSBCe0U2tSHFIQUwcoM7bkGI+in/6Nq+/rvy0VQhlxh9KgCNNRBHloOg02BsYtzNVfPlcmngfs7j4n3RWuUyX6f1hD5S4S2VpTm6+HUcx1tauj4i4RQM9XdTlKVgFJ9XQ4gzOwSoq2B3Gs6QbVWw4067fl3VTS1NVFAp1KylWVUFtXQydQIA5E+QerlVeLjy00ynV5CV5e3JWEiACQRO+kDSdRIJIrmLiI7ZuHsrP3XjnB1WhIZxDQU7RkZXkGmgiJMupHjqNJ18bOWlFDeQDtBX09Qkpkpb7N0AwRMp3G+418TtTF5+pUwXKt1LaHgMgeQEAhOg7pTBAHxB5wMum6ZapyJCnHSrs3GYCgYTqNATHdOnTnpAnUAJuHIza+27VdZN1A5JS2VA5VQlSZ1/wjl5jSdNLdKu9AOZypUkKVJCHACAVcp8lD3adLVAo7w4gtIU5dl9XwOzSM6E132aNDIyFW+3IiAIibLNPxB4m3WpNO9WULiFOe1VNZQrQj/ZpH825Mkk66pFoGiI5ogrmnkpl4jYeYqlU9VSvpFSiYzKSpQJgaeACR+PKzQa9YRlSpalPnuqQ2iQ3G+o03G8czvtZCo+KVbeakM112UruZc9uzXtJKQVCVFrMTHMQZ5cps5GKhupWVs5Ql6BKBqecCNOY/W2jTtyMylZ87g5+ZMa97lqXA5eNW12inHSvMUZlZUmBqRy2033625u6pXQv01UhnMqnUkkkGMydSNtOfWzrqKO8bzZe+0QaMKI7Rbm4EzEjUb2Zj6YfCCS4lKsgJOveJSojoTAn42BIy2hVmN3MKzWGsVXVV3QyFU7ilsogxkGh0CpmdgDz0B3spqxLSnNlpnFAnUKKYHnA00jYfHnCXDm+lCmNCXRkTISknvTPdnfl93xfPrpglTSBzCijuidDJ/p4TvaiaRm6vCsdbROtzFCASn93NgEc3BMAHUmBpJ/LbQsb/qHTnTTsJjXQkySNx4STy5WbqUuqzkJdgHVSRB93XlqND4cukKXklp+QCdMugnkep8Y5edmFKwJd5eUtuX/WlSiWkIIhIOhjXfU7mLF1XxeRVC6ooAJMBrSJ69dB8DZNQt9KZcQ2oKI2Okaajl9c97bUakgpLaTr3VJXJHl0P62mKdvRRNQ5GV3nWLzJU+ohRIgaKHhNgV1tSvKsOVZyynQnT7rBdmsKynMhJH8SRA6n77dlIH8KlJ2griZBmz8FvROJndUF6w+pJU6p9SojvqO/uNgCtbye1kFJAEFU6axzscLLKzk7IjWAUrIA6nU6b25XTEkykzoQT3h5SOf1rZxEByUeKSk5ZUqAqphZ/lVIJidtpP1vbaXHm0jsy4Qf5kyBt5QR5crGlsqSofaEzsVHRXn1+/SwC6dwgKSnTf2hOmm8aj8vO0xGFAyFABbgXmD6xlEzlj39I+X4cds85JCxrqDMeEEcxHzNjKYalYpgVezJ+/Qn657W5cUy6AFNrzSNMoB+Ea6eNlw+ibiFF3HCpJJQc0xOhM9frrblVW7JSpGaAU8oynwG3152NpbpJBzpSVCQI8ySdbcpaYSQpsz3dRl08NuW/n4WcMUeIgBeb6Up7enATvO8x4/X3W5XWBIhQVmJGkj468vlvYwpTYWEHs5BOp6chE+dglFOfIgIKpOZWWQT4wNPdaQYomRFV1UHMArKkDWNYnQx9eU2364gtlS+UaEgEmBrv9c7C5UhQDYQAAYypgADbQ+7624cQoFKkOazrJnpr+n5m0gwBRL7aoNT7LiSnKjTukBM6fU/py5X6stJiIImQ3O/u0+tbcvJcXK0rSUE5gc0Dr1j68rAO0j4UVpKErB0IPMg69Zm08qhxEIUsqCkpIBnSFEGdvyGnhzsEtDLqVBM6+ySomJ8J+XxtnY1bZWtVQkwCQpEj3dOR+fnYMpfy5VOI5zrpz1Gn1NnSJutBppMErSYEaD37b/1+IZQUAwsnKAMp03nTrYYiqOZBQ0FIESdpiOW/KwKwpCftWExMKTlMn468vlZhYJkCtxzMYdCiPEA7RMxvv4WCJyHs8+p0235zM9eU/GLGF1DSCEPUwCRoDsPOAI+EWCS8wrLKXVyiAUpMRHXbpp/Wz5gFEtKLOOEBZIWoZYIDJBPjvG3KwrK+8QUpgakwZ6g6672MpaSoAMB0IgElaD3p5z4WOU91rWnK0jInYBQiCT5T9RaOcAJ+ESi4LahmcYbzHVWnO2WWE3UnKPs6c6b5x+dss3ECfhFemDOCcMU7aW6a4ruZQkylKKRAAPkBY81c9G0kJS02ANgGwBbLZbysDmugzuWlUbZVAJSPC2l3a1spaiCLZbLFawHdFD3DYoBVOw0fYKvM20420kBRRO9stlpZQEZrid0A8hLeoSkgQdRbRcPZ5ghG8bWy2WgNCjt1tdJN8Xg5QUD1chtKi2kkJ2+doGxD65iWt9dvWvcWZKUoQkJSnyHutlssamF7kqyNkjN4epFvoQpxw5yATpznWwbt0UalZ1hZCQAlIV7I+vnNstlroUCuU3PQ1YVTPtl1CiEqDpzhW2457/Kwjty3RQVLtNRXZStBhZaBDKZOWNdt5M2y2WsROIGhQXtB3C6ZaOU9kUtgJBgJ01/rYaoazuoZmOzbSJA3mSfvtlssYOJ5oeUDkglICVBKiVSqN42BV/02FS029QNqKT3XlpTJkjQSfPS2WyxWqFkCmlRCtRokKHdH80fh87HKKjS6p1hSzl7FatBr3UlQ/8ADHvNstln5qKKu3ZTJUVkEgTI66n3cultUdOyxWJep0BK2pOsKCtNiOYiR77ZbLTCbmlG88NUdM+2thWRt9tLyUBPsBU92eYkf15kDhmjfLrSllJVELQkBSTMZgeotlss5UVGfE3BV1G4l42bKkVbDxp6pEAipCHuzCp/hXporXTQ5tCImcbp00yCadJLqdY00k7dNRMjrbLZbSpdWC6zanR+iIu3hTtU7MUeZNQ6lEKUk5SQVBRlOvsj36+FiSMQPs0jaktznqOxlRGYabzGuw0+62Wy10AKmXFGW78qX3mWlNt5ylHfKZ1UAdvqbEay/Lwp+wUFpyvgaJSEkBRPPYxlHK2WyxA0KDnFAu3/AHmahspqD2iVwFlKTptG22vPpYVmtvAVZoqmvdeStsOjXKBJiIHv+XSbZbLEDRZBLjddPdrTutvO1Dj32yGgCYgKSNZGvP5DxkOqU+h5omqdJdkI7x7iYBg9efTe2Wy0g0KNyiqAXqVpxMI7RAMCYB113125/pbpqrWhtmkqJcU64W86CG4iNSANfKbZbLPYJXKMIYaU6WinKSgjM3CfZ0H4W5ao1J0D6ozaAgHYkc9eXW2Wy0VFxKEVdrfbJSh1beUTKAAdCTE+87zvYuFR30rcIUsAZlSRKtRIAEeEWy2Weya5XCXKZDgacpyo1BDeYKAgDroZ38PnYneF4ponlqQ0tbhVBUtwGYT3dI1iOetstlpBMiDd8orVIUKFKFwO8VzG+2gj2jtZzXfdabyaCw4lkAxCWknqJncnTckna2WyzlRRm9sIquui/eD9e3UpXWIQEBgoIKp1nMQdxpHLygRm7BTkoafUC04jKYE7kfh87ZbLEbsou3UOfudqjxIwpDplFUlKyEiVy5BMGRyO4I185sXRcNcLVlR6oll5ChMkqTljLmgBKUn4kxPutlstUYd1ZIFlqqwFhy66pLCGKlRTupNa+gK15pC4P9etu2cLYdcZW45dTToBHdfJemT/AIyefSLZbLEuVEALlFw4SaUhJwtQBSnEolDLYGugJlBnfr12k2VH8PUN00yapimpBmyyEUyU848Rsek2y2WiN1MgLlla3m0qKjKBIkyJOm2x2G/TSLNN9S1XmrKoJ7VQTokaAkEbDkeke62WyxYt0OXqmnjTiJeVx3mMK0FIh1SkOPGoq3VuDRShAQkpj2eR2jpaPl8RXXXqti8rnadcpCUFdO6WkuAAH2VBcbxv162y2WHL7yND7icHDTiWbyv391MXMacBttXaGpzEkzEgIExB8dRqItY0pacAUtKoypV3VQZO/KPlbLZYbt1Jmy6FGlaA+lxaYbC45klAO/kQPdbWVxGqXlSO6CQDz8bZbLMnXSW1HtBn0B7o1EaeBH0Lbapg+o5XXEwspMkKBgnr5Wy2WSa5utuULjOVfrMhPRMHrvNiK3nUuDOsuELCDm1BM7m2Wy0gE91yupcStCFZVaADSBHl1sJSvl1bVOJCnAO+YMEgcotlsskgjd50TtNFQakKOYAAIIiVKHU8k/OyahxaiSFxBSmMoIPyn52y2WdJcuZ1uKSVDuiZj422ptJBJkkqMTEaAfr8bZbLJILHLuUlDjhfBAAhOTSJ23sVLUuKSCkajZO5m2WyzFMgkMdsla82UJ7sRJJBiZNtmmGVBSqAZ0InYn8vnbLZZX0SQakONFQS6QE6R74sNTtuuICy/MxlCkggDeOhtlstFPYI4i7FK/vXkrEbdnHQ8j42E/c1PGVxxZjQ5DEyJ5z0i2WyzOJT2C6YuGjqELWlx9OUjRTmaZ+Fjhw3SUrgQFlWROhjx5yTNstlhucUVrQtJudhTgADQUteSS0DBjewT9MhlGdpKE8kiDoQTrvbLZYYcbqQaEWSiUoeSopzlWmmgG2vXxsNT0frLvZLWkCSkdwHafytlss9ynaAumaCmeLjamkDTMTlmZIH5fCwzV20iSFKSqCf4SAdQDrIPW2WywwSpWF0IugpMxlmTO5O9stlstJPZf/Z" + } + ] + } + ] }, { "language": "markdown", @@ -88,7 +82,7 @@ { "language": "typescript", "source": [ - "\n// Utility function to get the coordinates of the maximum value in a tensor\nfunction getCoords(tensor) {\n const { values, indices } = tf.topk(tensor.flatten(), 1);\n const [maxVal] = values.dataSync();\n const [maxIndex] = indices.dataSync();\n const x = maxIndex % tensor.shape[1];\n const y = Math.floor(maxIndex / tensor.shape[1]);\n\n return { x, y, confidence: maxVal };\n}\n\n// Draw keypoints on the input image\nfunction drawKeypoints(image, keypoints) {\n keypoints.forEach(keypoint => {\n if (keypoint.confidence > THRESHOLD) {\n cv.circle(image,\n new cv.Point(keypoint.x, keypoint.y),\n 2,\n COLOR,\n 3,\n );\n }\n });\n}\n\n// Draw skeleton (lines between keypoints) on the input image\nfunction drawSkeleton(image, keypoints) {\n const skeleton = [\n [1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9], [9, 10],\n [1, 11], [11, 12], [12, 13], [1, 0], [0, 14], [14, 16], [0, 15], [15, 17],\n ];\n\n skeleton.forEach(([start, end]) => {\n const startPoint = keypoints[start];\n const endPoint = keypoints[end];\n\n if (startPoint.confidence > THRESHOLD\n && endPoint.confidence > THRESHOLD) {\n cv.line(image,\n new cv.Point(startPoint.x, startPoint.y),\n new cv.Point(endPoint.x, endPoint.y),\n COLOR,\n 2,\n );\n }\n });\n}\n\nfunction toTFTensor(ovTensor) {\n return tf.tensor(ovTensor.data, ovTensor.getShape());\n}" + "// Utility function to get the coordinates of the maximum value in a tensor\nfunction getCoords(tensor) {\n const { values, indices } = tf.topk(tensor.flatten(), 1);\n const [maxVal] = values.dataSync();\n const [maxIndex] = indices.dataSync();\n const x = maxIndex % tensor.shape[1];\n const y = Math.floor(maxIndex / tensor.shape[1]);\n\n return { x, y, confidence: maxVal };\n}\n\n// Draw keypoints on the input image\nfunction drawKeypoints(image, keypoints) {\n let modifiedImage = image;\n\n keypoints.forEach(keypoint => {\n if (keypoint.confidence < THRESHOLD) return;\n\n modifiedImage = modifiedImage.drawCircle(\n keypoint.x,\n keypoint.y,\n 4,\n { color: COLOR, width: 3 },\n );\n });\n\n return modifiedImage;\n}\n\n// Draw skeleton (lines between keypoints) on the input image\nfunction drawSkeleton(image, keypoints) {\n const skeleton = [\n [1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9], [9, 10],\n [1, 11], [11, 12], [12, 13], [1, 0], [0, 14], [14, 16], [0, 15], [15, 17],\n ];\n\n let modifiedImage = image;\n skeleton.forEach(([start, end]) => {\n const startPoint = keypoints[start];\n const endPoint = keypoints[end];\n\n if (startPoint.confidence < THRESHOLD\n || endPoint.confidence < THRESHOLD) return;\n\n modifiedImage = modifiedImage.drawLine(\n startPoint.x,\n startPoint.y,\n endPoint.x,\n endPoint.y,\n { color: COLOR, width: 3 },\n );\n });\n\n return modifiedImage;\n}\n\nfunction toTFTensor(ovTensor) {\n return tf.tensor(ovTensor.data, ovTensor.getShape());\n}" ], "outputs": [] }, @@ -102,7 +96,7 @@ { "language": "typescript", "source": [ - "const tensorData = new Float32Array(inputImage);\nconst tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, tensorData);\n\nconst inferRequest = compiledModel.createInferRequest();\ninferRequest.setInputTensor(tensor);\nconst outputs = inferRequest.infer();\n\nconst heatmaps = outputs[heatmapsOutputKey];" + "const inferRequest = compiledModel.createInferRequest();\nconst outputs = await inferRequest.inferAsync([tensor]);\n\nconst heatmaps = outputs[heatmapsOutputKey];" ], "outputs": [] }, @@ -116,18 +110,30 @@ { "language": "typescript", "source": [ - "try {\n const outputHeatmapShape = heatmaps.getShape();\n\n // Define constants\n const numKeypoints = 18;\n const heatmapWidth = outputHeatmapShape[3];\n const heatmapHeight = outputHeatmapShape[2];\n\n // Extract keypoints from heatmaps\n let keypoints = [];\n const heatmapsTFTensors = toTFTensor(heatmaps);\n const xCoef = originalWidth / heatmapWidth;\n const yCoef = originalHeight / heatmapHeight;\n\n for (let i = 0; i < numKeypoints; i++) {\n const heatmap = heatmapsTFTensors.slice(\n [0, i, 0, 0],\n [1, 1, heatmapHeight, heatmapWidth]).squeeze();\n\n const { x, y, confidence } = getCoords(heatmap);\n\n keypoints.push({\n x: x * xCoef,\n y: y * yCoef,\n confidence,\n });\n }\n\n drawKeypoints(originalImage, keypoints);\n drawSkeleton(originalImage, keypoints);\n\n displayArrayAsImage(originalImage.data, originalWidth, originalHeight, display);\n} catch(e) {\n console.log(e);\n}" + "try {\n const [N, C, heatmapHeight, heatmapWidth] = heatmaps.getShape();\n\n // Define constants\n const numKeypoints = 18;\n\n // Extract keypoints from heatmaps\n let keypoints = [];\n const heatmapsTFTensors = toTFTensor(heatmaps);\n const xCoef = img.width / heatmapWidth;\n const yCoef = img.height / heatmapHeight;\n\n for (let i = 0; i < numKeypoints; i++) {\n const heatmap = heatmapsTFTensors.slice(\n [0, i, 0, 0],\n [1, 1, heatmapHeight, heatmapWidth]).squeeze();\n\n const { x, y, confidence } = getCoords(heatmap);\n\n keypoints.push({\n x: x * xCoef,\n y: y * yCoef,\n confidence,\n });\n }\n\n let imgWithKeypoints = drawKeypoints(img, keypoints);\n imgWithKeypoints = drawSkeleton(imgWithKeypoints, keypoints);\n\n imgWithKeypoints.display(display);\n} catch(e) {\n console.log(e);\n}" ], "outputs": [ + { + "items": [ + { + "mime": "application/vnd.code.notebook.stderr", + "value": [ + "(node:3778163) [DEP0051] DeprecationWarning: The `util.isNullOrUndefined` API is deprecated. Please use `arg === null || arg === undefined` instead.", + "(Use `node --trace-deprecation ...` to show where the warning was created)", + "" + ] + } + ] + }, { "items": [ { "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAIFArIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1SXSkkG1ihGQehHQ59a429+D3h+9klkY3SNKxZjHcY5Jz/Epr0RvvfjSgVEcPSg/djYp1pyWrPJpvgbpjKVh1G/j/AN5kf+grPufgXMYysGtvj/btR/R69rFLVciFzM8Cn+CetpjytRtHx/fSRc/kpqhL8JfE0RYqLSX/AHJtv/oQFfRtLS9mh8x8yP8ADjxXGrE6cGPolzE38nqtJ4M8TQld+i6gRnkrAzD81zX1IUVuqg/hTDbQHrEn/fNL2Ycx8py6VqVqx+0Wd3F/11gZcfmKgG4ZBbnmvrMWsI6KV/3SRUU2mWlwCJYhIPR/m/nS9mw5j5W+fAzxTwW3AjJAFfS03hHQpx8+l2Z9zbRn/wBlqjJ8PfDshz/Z1uPopX/0Eip9mylJHzZEzYI3H7x6/Wp0ZiseM/d6173L8KfDjA+XZhDnOUmkH82NUZfg/pDABJLpAOBtuAR+sdSqbTLc00eJbjuY8kdOKYWbHPPocV7DN8Gockw6lcL7NGj/ANVrOn+Dd8ABDqStj/npBj+TGqsyLnmJPI5FKCSSNxrvpvhJr0Zys9k49MyD/wBkqhN8MvE0IbZaW8mf7twgH/jxFFmByO9gn3j+PanAuGAyfxrem8DeJIcZ0edsd49sn/oJNUpPD2tW5Jm0nUEAHVrZwP5UrDuZ4JYnp+VLuPHofSnNBNEAJUdGzjDLjrQFJYZ5xUlCgnOOfes+d3XU4scHA/rWiAc5x7Vm3fF3EQM57596mQ0aMUm6PrnnAp2/AznNRwHbFv754pxOAFweaUdimBftkY9qCxAz+tAIJPtTSV25I6+tUSOJbOc03cc8nHelbHTn3o25Ht069qkYmfu5yD2oyScnj60u05xyTTgpB49KVxgCRTWO5Dj09afg5HAye1KVODnnIzUsZl2TsplHPbI/OtLYjYbbgHms2Bis0q4/WteHJgTjgipe41sQ+SpxlTSGAf3m9OtXAoAzwKUqMcLmi4WKX2f+EMffNM+ynPOCO2RWhsB4oCZPFLmHYoCyLAEqhP5UNZnHCd+gNagj9BSFc5qeYfKYxtW7Ix9/Wo2tGByQ6jPcVulMDnjHrTGjOwnB6Ucwcpzo3K/r2qdYtyAnAPrSTrtkPHetSwVXtACoPJB4zQ2CRlPZAIWaJD34ArF1C12xl0hC8/wjFdfdJHgoFGO+KxL6P/RZMdAc/jW1KTTM5o5bawPANHmOP4m/OrxT3PPrUezII6/hXZzHPYq+Y/8AePFHmHvU5jGM4H5Unlgj7uPei6EQbzS7+MU8xjHIIpjIAuRmnoGopJ4JzzTd555609uY09gR+v8A9eo9poAXfil8w+lNwaTB9KYXJFcA8jinmYYwM49ahwfQ0lKwXJd4IxRuFRUUWAc/NNoopiHI5SRXBwQcg19uaPffb9EsLwHInt45fzUGviGvrv4X3pv/AIa6HKTkrAYj/wAAYr/SuLGLRM2o7nXlzSbzSUhrhOiyF3GjcaSkqh2FViHYZ9DT9x9ajHDj3Bp1ddPWJEh24+tG4+tJRViF3H1o3NTaKYh28+tQzMd8bZ74qWopwfLyOxzQxoXc3rSb29aTtSGkAu808OcVHTl6VLQ0O3tVS+lmjsriUT+XsQsCEzjAz+NWGOB0yT0FVLyyF3aTQzOxEileOAv0Hf8AGpavoPY46TxpbC2huJLdpZJNrBSqgpn+LO44+nB9ap6ze6fKJQ11cm7JCNGMny/baOAT6MQcVn3ekwXlhd2YmZLzT2MfmNgkqBz379f0riFvtkUNs06tPBJhcAFVABODkc4PvjNY1KV4tGTqNM6z7bpwHK3ue+JUH6eXRVJfFjBQHS4LAckAdfzorx/Y1OxtzRPdG+9Sjmkb71KK+wZ56HClpBxSikMWlpOtLQAtFFFABS0lLQAUtJS0hhS0lFAC0UlFAC0daSloAaUQ9VU/UU3yIv8Anmn4CpKM0ARNbRMMFePTJqnNoOlXBzLYW0me7xK38xWhmilZDuYM3gvQJs7tKsx9LdB/ICsy5+GHha5YM+mRhh0KvIv8mrsaM1PKg5mcE/wm8P8APlC4i9kmOP8Ax4GqM/we09jmC/uo+ONzK3/sor0uijkQ+ZnkkvwclCsI9WYk+tuP/i6ozfCHVhjy722YL03q4/kDXtNLS9mh87PBp/hb4hiDlBaSn/ZlIz/30BWfL8P/ABFABnTVZRyds8bfpur6JpCARyAfrUukhqoz5sfwrryKXOjXzDtshZv5Cqcuk39vgT2V1F674WX+Yr6aWGIoAY0/75FH2aH+4B9Dip9iUqh8ulcM249PejYdpwck9K+n5LG3l4ZC31YmqUvhvSJ/9bYWzH1aBG/mtQ6LGqiPlfaUu2XuRxWxaZNqgPQZ9s179N8PvDM7bn0m13eqx7f/AEHFVW+Gnh/btithGvoryZ/VjUujIpVInig+7nH6U4RbgF5+tevy/CzSWOY5bpcdB5q4/wDQDVRvhZEhJivpRngZQNj9RWbpTLVSJ5gIULHIGKcsKjr1Poa9Gl+GNyMeXfBsf34sfyY1Sm+HeqpuKyWzHtw//wATWbpz7FqUe5w/lgtxkbR1PemmI468n1FdbJ4E1tMYto3A6kSqP5kVTfwrrEWSdNnJ7BBu/lmo5Zdh3RzhhOSBj3OKYyN5fbn3rcl0TUYiA9heIAOS0LD+lUJbd4j86upHQEYNLUehyd8hVyCBnOTg1a06YrasOpDcUmooVDjHzNyah075oXHuK0SuiHuWXOWz3PrWbeoDDKD0A61pEHGe9Urpcq4xwVPvWkCJHOsPTnHvmoOTxjg1YJwABn+dQHBFdaOdjW4zSMBxnGfrS56kHnofemknjI4NMBDjpj3pj/dbFOOcd6aeQetUIQH/AEccfdfr9R/9al4OB1zSIMwSD0INIMkUwDC0YwcGjNHX+tABjNBUDijnOCOKPrQAmP8AOKQrTs+2fxpD7UCGkYptOI4pKYhK+mfgNffafh89uWybW8kQD0BAb+ZNfM1e7fs73vya7Yk94plH/fQP9K58Ur0zSk/ePcTRmg0leYdaFopBS1QCHgqfenU1/u/TmnV1UtiJC0UUVqSFJS0lAC01xlSPUU6kNOwEKcoPpSmmowO5RjKsQfanUgEpy96bS5xnHWpY0OAJOAOaazwhXDSKMe/Q1Onkqu+V05H8R4rAvNathqzad5roWXeFTjI756n3wB9cVlKryK493Y4DxmZYfE32/TpI0VodkrSKYwTyMFiAOmfeuJtNAxOupahf29naysfKUtjzeecd8e4HX1r1XxBq0ElosEttJNG2GSJISiJ6HLLtyOvXFeWXfh+91L95d3iR7mLCWc+X8o+p549K5Z13JWbsS4JPa52McPhJIkUMjAKACUbJ/wDHxRXMx2Nokao19bFlABO1+f0org9k/wCdl83kfQjfepRQ33qUV9azgQZqB3xubLAD0NTmqsoysi9MgimhM5rR/iJoOtXkVrbXM8c8pwiTxFdx9M8jNdYsrfxVyFl4Ksra4gnJV2iZXDBcHI5Fav8Aawl1k2UaSjyk3SM6FQSTwBnr0PI4qrCub4bIpjTbTgDOOtNQ/JWL4mjefw5eQRsVaZRHkHnDMAf0JpWHc2I71JRlGR1BwSjA4/KpxICK878H+HH0LVhLC7LDMhSSPPB4yD9cj9a7rzNsuz2zQ4gmXCwUZPSm+cn979Kikb93+IrivHGoa9YT6edFuFjDLIZUaJWD4246jjv09aXKO53gdW6MDTs1kafcyS6da3NwoSWSJGdRxgkDI/M1oq2RS5Q5iajNY+p3rWOm3l6kLTNBG8giU4L7QTgH8K5Sz+IbXLBToWoqx4wh3Y/QUcoJnoeaM1BHKXUHkcd6jedgXO7Cr7e2aVh3LeaXNctpXjfQ9ZuUt7DVI5Zn+7G0bIW78bgK6JJtw5FFguTZoqrcXiQFAxA3ZxlsU0XoI+4SPY5osFy5miq8dysjbQrD61MDmkMdmikprSKgy7BR6k4pAPJozTQ6sMggj1Bpc0ALmjNJRQAi9KdmmL3HvTu1AC5opKBQAtGaSigY7NGaaKKkB2aM0lFAC5oIB4IBptLmgaG+TEefLT/vmka3iYYK8fU0+ilZFXZRn0LS7nPn2FtJnr5kStn8xWe/gbw1Jn/iT2a567IVX/0ECt+ilyod2clP8M/DE2SLDYfVJZB/7NWZc/B7w/OSVlvIif7kw/qpr0HNGaXJEOZnkNx8A9MbP2fVr2Pjjfsf+i1j3P7PtwB/o+u5P+3a4H6Oa93BpQafKI+b7j4CeJEyYdQ0+QDpkyqf/QDWRc/BfxlADts7ab0KXKr/AOhYr6ozQDRqGh8gXHwy8Y2+d+g3D/8AXFkk/wDQSazbjwb4mtgTN4e1VAO5s5MfyxX2iQrdQD9aZ5EP/PJP++RRdhZHw01ldW4lWe2miO3pJGV7+9VsHFfdjWsLjDLkem44qjceG9Gus+fplpLnr5kCv/MGjmYcp8RYPegn2r7IuPhx4RuQfM0CwGf7luif+ggVk3HwY8E3GT/ZCRk/3JpV/wDZqOfyFynybRgnpX1BcfAXwlL/AKoXkP8AuXH/AMUGrJuf2etJYk22p30fpvKP/wCyrR7RBynzqBxRtHrXulx+zzOMm31sn/rpaj+kn9KyLn4B+Ios+TfWEo7BhIp/RSKPaIfKzyEgd6bjmvSbr4KeMof9Xa2s3+5chf8A0PFY9x8LfGtqcvoUzjP/ACykjk/9BY01JdxcrONr1T4CXnkeO57fOBcWTrj1IZW/oa4m58FeKLQnzvDuqKB3+yOR+YFb3wuF3pHxL0d7m3mgDyNCfMjK/eUqOvuRU1bODCCakj6rNJRmivKO0KUUlFUgHEZBHrQpyoPtQOaRD8pHoSK6KREh1FFFbkBRRRQAUhpaKoRABiVwB15pTSP/AK5T6jFKeaQXCpYIfMfcQMCohy6oOp6CtHb5UOEGSBx7moaQN2KVxFDcMftTKtspA2lsBmPr/LFYWvaTpcNnNIq/Yw/ytMkrRFs9RwCWPTjHNdO22PBIzjpxk5qte2EeoweVeIGhYg+Vng49cdfp0rmnTTTTV2wUtTztS06Yim1S6hQZBkUspwD91SCQePY4471xt61ld6sls9/fM9zKUaJ48FRxk4GcDj0H0Net61cvJbxw6bGQkbKA6LkEZwVXHt3OB2z6ec+JdRG26ewnghuWg2yrb4Z1IBXBYf7wIxjPNebOnyysnc6FrG5lyrp6Sui+C9YKqxAy0o4+gTiiuf8AsuunmJ7jy/4NzDOO2eaKfso9/wAWY83kfSzfeopWHzUCvqGcaEqhfc2dyPWNv5VoEVy2sf2+uoPHavbNZS4GHU7lBGDz+dNNLclps47w14XvdG1Ozube9m8oOokjL8MvQg/hXos4Xz4iRzg4/SmLZPHZsYwDKFyoPTPasnTtR1a/vhDe6WluiAnzFkzz6Y/z0qrruKzOoT/V/hVLURv0/jn5kP6ir0akJiufvNch0mZrXU4pVhOfLnRCylfQ45BFCYmjRt1CtGf89DSu+dQwOyD+ZrGbxj4fgj3LemRv4USNsn25FXNGlmvw17JE0QmbcqN1Vegz74FDYJM2W/1Y+o/nXLeLvFn/AAjU9kraW16k4csyvgx7dvbBznP6V013LHbWrTTOqRpgszHAArDvjo+ryRtJqFq3lghQJFPX8fakMsI0et2VlqcE0yxFRKkecBsj+IeorXgY7BmsJtS0/Tra3061lSSRiI40jYEgdycdBjNbdvkoCaAK19JbxaddS3jFbZI3aVsE4QZJ6c9K56y17wk7KLXUItzEBQVYZP4iuiu4Rc2NzbkZ8xHTB75yKw7DwpZWsyytaxBgQQdoNSyonURdOOlQNz5wPc/0FTxgBcZqEj5ph6n+lNCZydh4LtbHULa7GxZIHDgqOTiuit9QS4u5Io1cCI7WLoVyfbPbpzXn2ieH9S0fVbO4h1C4a28wB4mclSp4r0bA81DjnHWhgirr2h22t2qR3Me4oSUOcYJqroelJpmjS2SZCq8mOfXn+tdCvK1UAw8w/wBr+goA5jwnoUulXDStd3MoKbQskhIH+cV2ydKoWaAIvHar4FQWPzWVrhJswO24GtSszWRm0/EU0Jmdpl55TNESdrcj2NQWVy39qiQsxBds8/WsGTUJLfxX9gfiOS0WaFv9oMwYfltP51paezGVWcEHzW/9COP0pdB9TtI23Cn1XgOVqehgC9T9aWmjqfrS0gKV7qP2N0UR7yRk84xTkv0a0+0EEAAlhnOMVn6x/wAfCf7tZUl6p0+9Eb5CKwYZ6MBUt2nZ7FqN4XR01pqEd3uCoykAHnFWi3FYGnMY5MA9Vq/qFy0FhJIv3sYH1NODuhTSTJpNTtInKPOu4dQATj8qnhuIp03RSK49j0rzjWINSuYEGm3PkOCWY7c7vQVc8L392UtpLrInJ8uUevOP8DUptq5XKtjvy2BntQGBGR0rO1SF7vS54I2wzoQOcc1zWhagYZRu+XPySA9iP8/rVuNlchb2O2zzS1yj38tzqAKzSLHuAARsZFdIJQULZ496lO70KasTbqXNcjq3iRrWGS5VpBAnQRJuZqseH/Ea6pDDKrl4ZvulhhlPTBxU8w+VnT5ozVa5ult7aSZhkIM49ayz4gjTHmoqZ4GZMZ/SnJqO4RTlsbuaM1RsdSjvSyopBUZ6g1dBoTuNqw7NGabmqz6hbRyGN50Vh1BPSgRczS5qtDdQznEcqOfRWBqbPFG4EmaM1GGyKN4JxSC5JmjNNzgUA5HFJjH5o3UzNGaQyTdS7qjzRmgB+aSm5ozSAcaYQD1AP1ozSGiwxhijPPlr+VNMEZ7Y+hNSE0manlQ0yI2644LD/gVM8g9pX/HH+FTmk71Ps4voVzMgMcg6SL+K/wD16TbKP7h/MVOabS9lEfMyPMg/gH4NUMDzlpC0SqS2cbv8+lWqjHEp96FTS2DmuKGfun5Gjfj+Bvyp9Ga0sTcZ5i/7Q/4CaTzo8/fFSUUrMY3ep6MD+NLSFFPVQfwphgiJPyLz7U9RDLhtgVjnhhSb95wn4nsKiurQNbuI2ZWxwckgfhmpowBGoHTFGtw0Lttb+UN5+8euetWSQFLHPHoM1WS6UIA2S1Zlxrl7Zbml0tpY92FaCUNgf7QIBH4ZrKpUjBXkTyuT0Ndl/wCWjZHHTPSud13ULu6YWGnouBgyvISA690AHJHqR24zzVLVPE89xILNVS2YkHaWzIw9h2/I1nXGqSWtm4SK2V5mw5lO4tgdSTkn8a8ytipSaVPbudEKDSvIbeLMLSSK906TLZ8yaIrMPrsyFHAAA5xwPSuH1a10ay8iWzknMMpCtlcMG6gsDxjnjAHWuluNVuhF/o8toZGBACQqQD2OcYwPrXM3V3G9zuv2e9uRwCuUh/IDJz+B46HFYOTbLcUkc9Jq2rRSNGlpeuiEqrHaSQO+cUV1hksSTts/l7bYcj8Cc5/Oil7eP8pnys9qkHz0gp0v36QV9UcAYpjRhjzUgFLQBEEAGKYLdQ5YDmrGKMUANC4FV7i0juFKuoIPqKtUuKQGGug2aSb1gQHPUKK04YBGoAGBVjFGKSSRV2yC5to7m2eGVFdHGCpGQa5ibwXpjtn7Mg+gxXX0hANDVxJ2Ob0zwzZadL5kMCq/rjJ/Ot9EwMYqXaPSjFCVgepyWsaFqU9/LPZ6vdW6Pg+WrfKDjsKoLpPiSM8a7MR/tRA/zruygPUU3yl9KHfuNW7Fe1RxCokYs4ABbHX3rF1ldbjvwdOktxbuBuWRCSD3P8q6QLikaMN1qrvoTZGNJYuNPfywDKFygPTI6VQ0i81a8uil9ZxQIgyGV8kn0x6V1HljGKYIVVsgChyYWQiLha5/U9bj0zUWt5LS7kMgDK8UeV9MZz1rpAMVDNbJKwZhkii9h2RBZqwjUkdquimogXin4pXADWfqxAsyzEKARkk4ArQqrf2UV/ZyW06B43HINNMVjhtfsftaRXtqytcWmHQqc5x1H4g1csZRIIZV+62CK3NO0O102BoYIgqkkn3NZy+EbWPVReI0ow+8IGO3P0pNqw0nc3VmWGLe+ccCrUUyyLuXP41n6jYvdae8MMrQycFXXqCKbo1rc2tr5d1cvPJnJdgB+HFJtBY1R9406mj734UtAGPq4/exfQ1w0StY+LNQtpiWtNTUSJ6B1UKw/EYrtPEv2xLWOWythcSBsFCccVjT6Rc6lpEcssQivIyJUUHO1h2z9OKHG8tSoytE07D5WjGc4XGTVvVxnS5T6YP61h6Je3d1e+VLp00CoDudzxn6V0t3bm4sJouMshA+tKkrKzHVd3dGFpqKzsD6VRVBDfzKvAE2QPyNVbTWYrWdorlhb3MfyukvANWba4hvb/McqO7tuIQ5x0/wpS92PKwiry5jsYvmUA965XVrI2OsCSNf3NzksB2YV1cCkIM1m+IUUWSSHjbICfatY6qxD0dzO0uHzJQ2OF4rfuFIspcddh/lWToTLJHIVIOG7Vuuu6Jh6jFZU1Yuo7s5G3t0ukSJwCrDpUel+GZrDU1MV5ItmsnmLCqjHrjP1qXT5hG4z96JijD0PSrcl9cteP5T7YlwFwAc+tJrlvcpXlaxr6ou7Srgf9Mya5K/0u21e1ijukDKpDj2OMf1rsbkebZyr13If5Vx1zYnU9MjgEssJO074m2tx70VfhTCl8TRteGNMsdKieK0QK7HLnByfSukBrm/DOjtpiSM9zcXDvgZmkLYHtXSClHYJbinpmuK1BZC96sb7ZNzhWxnB7Gu0PSuSvlK6ldL/tA/mKqS/dsUH76Oe8J6hqkiTjUTm5tpsK4TbkY9v8816TLIfs7lDhtpI/KvNdE1m8utVuLG9tY4pYAHVkJwwz716HbSedaq/wDeXms4aS1Ln8JhaFcubiJ2dm3qQcnPP/6xUN1qt0ut3KiaRVikUBQ2BjHpTNJzHMsZ6xzMn61W1RfL8Q3I/wCeiI/6Y/pV2+NE9Ys7a4uPKtZZQM7ULY9eKydE1S4u5QJnUqybgAoHPFXoj5+mrn+OIfqK5zw/Jia2B9Cp/WlL4Uxx+Kx0OsamdPto3Cb2dwgBOAPepdMvjfWnmMgRg20gHPpWV4qUnSkkH/LOZTx+X9ak8PN+6mT0YH8//wBVE1ZJijrc381mtq0KzvGN7FDg4HGa0M/LXJSIDfXkJPBdge3WpnpByXQcdZJM6KDUYZHCbiGPQMMVdzXmemW13pHiCSxluppreSIPH5rFtv5/jXoMUpltA2fmK4J96mLd7SKaVrodFfW80zQpcRNKvVFcFh+FTO4UZrzfQJfJ8RW7HqWKnPuCK6HxrI40iMKSFaTDYPXg1t7P33Az5vd5jpt4K5oVgwryzTdfnsNIvbQSld67o2B5VsjOPqP5Vs+BL+aW7u4Zp5ZMoGUO5YDB7Z+oqHH3blX1sd2SAOabuB4zWB4o15tGslMSB55M7d3Qdsn864aDxlrcV3GJbjcJBlQ8ICt9MD+tS9ClqesUlUNG1RNW0yO6UbScq65+6w6j/PrV+gYlNI+cU6mv2PoaQDqKKKYBRRRQAlFHeloENYZBFV4STEPUcGrOKrRArLKuMfNkfjTAkpyp5iOnHTIzTaVZFiJdzhQDk1FSKcWmCdmczrFjbTTtDPDHKGw2xgDjtu59KwpdMmkVPsWozwjn5pFEowTnjdyB6YI/rXTyW7zT3N1Jvj3ttRHUZUADH884/Os2+SO006Rp1PlkEZfPzn0z71xWjK7asjVt2Ocmsv7Tu2ie4i+zxDcAifM6nuzZPU59B149eC1e3uG1C4gvH8i1DmOOR5yS0YPy5AywXgH0r0e3SG3sy8rwNcTN5kuWACH6e3QCvM/HumXt3rDraM0sD7JF2PldxGCT6nj9amFOCemhMm7XNmO+tUiRF12dVVQAqwMQB6A9xRXBR6VqKxqPtiLgAbc9Paio+rU/5yfavsfYUo+emAVhHxnpMp6zr9Y8/wAjUq+KtHPW4dfrC/8AhXte0h3MPZy7G1S4rLXxFpD4xfIM/wB5SP5ip11jTG6X9t+MgFHPHuLll2LtFQJf2Un3Lu3b6SKf61Mro33XU/Q1XMhWYuKWil7UBYSiloxQA2lxRS0AJRilooATFFLRQA3FFLRQAlFLiigBMUlLRQAlFLRSASiiimAm0UhUU6ikA3aKTYBT6KAGfx/hTqP4h9KWgBjIGHNMEQAxipqKAK4t1ViwAzUm3ipKTtQO5l32jWd9zcW8ch9WUGm2Wi2tif3ECRj/AGVArWoxSauNNoYq4FV7+xh1Cze3nTdG/UVbxRigRl6XpVvpcBht4wiZyfc1o44p+KMUkDZyuseGpZrxr3T7pradvvjbuVvwqPTdJ1FJ83t0sijoEj211u0dxSbF9KUlzblRlykAi/dlfbFcRLa+ILWRo4oLSSJThSSc4rv9tMaJW6im7tWCLSdzk9D/ALWfUUN1bRRQqCSVckn2xXXCmLCqnIFSgYqYqxTdw7Vx/iG5Sw1Eu8UzCRQcxoW6etdjioJrdJhhlB+tVfRoSWtzgLW7sp74SRRSm5Zdn+qYcZzz2/Ou4sozHaonoKEsIlbIQZq2se0YrOMWndmkpJqxwt3crpeu3ME58tZG82Jj0OaZdyLd6it0GA/deWQDnJzmuw1LR7TU4wtzCkmOhI5H0NZtn4V0+ynWWKAB16EknH51VSV7tdSYJdehpWCMlhEjfeCjIPauY0/9xqLJnGy5Yfhurs1TauKxm8O251Y3xaU5bfs3kLn1xSv7nKwXxXH+IY/M0O5Hoob8iDWfot0kDsXcBXUHJ4FdDdWwubOWA8CRCp/EVx7eG9WiOI9S+UDgGEGiclyoILVnZw3Ec4ISRGI7K2a5q+/d69MvZ1Vv0xVzQNOu7NpJLy5ErsNoCoFAFV/ENnffbY7qyijk+TYwdiO9CknB3DltJWKWqup1nSnUcsJFP5D/AOvXSWBzagelcvbLq0kyLNYxqP7/AJmcfpXW2sXlwqvoKxUnKafZGjjaLPO2/wBD8RnsI7on8N2a63xVF5ugyNjJjZW/XH9a5/xBp9ydfkEFvK5lYMpVeOnr9RXW6nC8+h3MZHzmE8D1xXZzL2ql6GCT5LHjt1J5NxDG33JdyH69R/I10ngqQw65GhP342T9M/0rPj0l9ZNwkKFmjXzEbHAcHpn35FS+HneHXbIqpLGQAqByM8H+tRNb2HF9zofHsZK2zdijD6YINc3O1ncaJpsTyulygc7kA4G4jv8AhXaeNLcyaSkwHEb/ADH0BGP54rzia1t7qxNvNH+9Rw0bdsZGRRJK0WNdUel+DXs1014LXzsowaRpcZYkdePpXS1w/gqdTeXMa9CgOPocf1rt6wi9DWQU1+VNOpD0IqibgDlQfalpqcoKdzQAUdqWjpTEJ3ooooASq7fLcj0Zf5VZqvOCJImGMbsH6GmxIfUKhnut54jjI2+7dz+HT86mpMVMlcaY7UYwVHpiuW1CFC0KswcbtwiTlnYDIA/me1dDqM94Y1W3sfNBX77ShRn07ntWBqaXVtaTzJEd/lgvtBfn0B7LnH6nmvOxNZJcsXZmsFpqc5cxNc6n5eoXKwwuP3cSnooHO5+3XOBj3NV7iKG8N55dhBcEhi08eCIsYAwO5PJ7dce9XdQgllVUuJTFBjmNdu8npgDJHvkHiua1BD9oZYdVEdogBEaPtJJGenU4GPXnp7eZUlOd1I1suhfSHSlRVaCYsAASZohn9KKpJ4USRFeTVpEdgCy5JwfSisLx/mFyPsPEjK3X9asx3Gep5qk+d2R1pyEe4r6B6gjUWQN0NShuKzVcqcfhVlJSO/HrUtFFnCE8ov5Uo2g8KAfpUav+NPDY6fzqRk6TyJysjr/usRUy6jeJ928uB/21b/GqeeKdk9setO7CyL6axqSdL24/GQmpU8QaqowLx/xVT/MVl5pe1Pnl3Fyx7GyvibVF6zq31jX/AAqZfFmojqIG+qH+hrB/xoyaftZ9yfZx7HSJ4vvMfNbwH6ZH9asL4ul/jskP0lI/pXKKec9qm6Ue3qLqHsYPodUvi1P4rJh9JM/0qVfFlr/FbTj6bT/WuRGcZpaf1mp3D6vA7FfFNgesdwPqo/xqRfEmmN1lkX6xn+lcVR1p/Wpk/VoHdDXtMb/l6A+qMP6VIur6c3S8h/FsVwOfejOKpYuXYX1aPc9DW/sm+7dwH6SCpFmif7sqN9GFecemT+dGPoaf1t9hfVV3PSuDS4rzUMVPBI/GpFuJ0+7PIPTDEVX1vyF9V8z0XFGK8+GoXq4xd3H/AH8P+NTLrGor0vJfxOf50/rcewvqz7nd4oxXELr+pr/y85+qL/hUq+JdRXq0TfVP8KpYqBP1aZ2WKK5JfFN8OsVuR/usP61IviycfetIz9HI/wAapYmmL6vM6cj5lpcVzB8WsWX/AEEH1/ff/Y1OviyH+O0kH+64P+FP6xT7kujPsdBRWIviqyY8wXA/BT/Wpl8S6cw5aVfrGf6VSrQ7i9lPsatFZy+INLJ/4+cfWNh/SpF1nTX6XkX4nH86ftIdxckuxdoquuo2LH5by3P/AG1H+NSrcQP92aM/RxT5l3DlY+jFAIPQg/Q0uKLisJRS4ooEJRS0YoASjFLRQNISilxRSKEoxTqSgAxS0UUAGKTFOooAbRilooATFNKA9afRSAYEAoZA3UU+ilYoh8lc5xTwuKdRilZBcieJWYEjkUuwbcVJSGiw7lZLVIydqgZ9KiTT4IpzNHEiu3Ugcmr1JS5UO7IJoEmhaKRQyMMMpGQRXNyeDNP84siOq/3A5xXVdqTFDV1YE7GJovh+z0YH7NHgnjJJJxWzTsUmKSVgbuJRS0VQhkf8Q9DT6YvErD1p9ABRRRQAUlFFNCYtQzjMRI7HNS0xhuUj1FOwhOvSkpIzmNfpS0hkkm46c+z74zt9q5zUkeSBAkpaISK7KuP3mPcmt+EGSJ/NH+0qen196yNQVVkPHI5rgq0HObkawkkrHP6pbi7vLXfbr5WC0mWAbjGATyQOvTr7DmsvV2mW2lng0jTxLEhMcjMG2bQcHbt9ulXri9XlLaMzSDJZYiAFPU5J4H8/aqoSRlZ7psDPEUXOc9jnr+lS8LC3NLVh7R3sjzseKYgAH1e93j7224bGfaitN9ABdi/iLT42JOUEcQCn044/KiuPlo/1/wAMV7xqMvzHjjNKq8D2qWUKjfMQp9+KRMEfeH4Gu1O5tawBcds08D2pwX0PPtQFHai4xVYrwf0qVG6ZFRjHY5oXOD29c0gLKv8ApT8gjqeKqhj1qQMRQBY68j60dOpqMMP608HjFIB3b1pcc00EZxml/KgBR1xmp1quODU65xUtlIdTuKTtSfQVJQppM+1LgikI+nFFwDr/APXoA4IxQB+VLxn1ouFgB/lR68UdOv8AKii4Bx1o+tL+dJQAAUdaWk6jigLC5pD796XHSkI9utAWA4oz9KM80h64/OmFgAzS0DrSY4oFYXg5ozikx15ozQOwuaAfek70Z4pALn0pcAj/AOtTc460E0wsO4AyB+VPWaVfuyuv0Yioh7UvGKLsVkWBf3aH5bqcD/rq3+NTLq+oJ0vJvxbP86o+9B47U+eXcThF9DTXXtTX/l6J+qL/AIVKviPUV6yRt9Yx/SsjNGfXFP2s+4vZw7G4vie/HWO3P/AD/jUi+Krkfet4T9CRXPg0ucVXtqncXsYdjpB4rb+KzH4S/wD1qlTxVCfvWrj6MDXLZPFGfen9Yqdxewg+h1y+J7M9Yp1/Af41MviLTz1aRfqn+FcYDS7gQaf1mYvq8Dt117TW/wCXjH1Rv8KlXVrBul3H+JxXB55pQ1P63PsL6rHud8NQsj0vIP8Av4KkW4gf7s0bfRhXn27HNLuprFvsL6qu56ICD0IP0pcV50GA6cfpUi3MqfdmkX6Mar655C+q+Z6DRXBpqV2vS6m4/wBs1MusXwHF0/480/rcewvqsu521Fccmu3463Gfqq/4VKPEV6OpjP1Wn9agL6vM6uiuYXxJcjrHEfwP+NSr4lk/it1P0OKpYmmL2EzocUVhL4lU9bb8n/8ArVIviOA9YJPwINH1in3J9jPsbFJWWPENmeqyj/gI/wAaeNcsCf8AWMPqpqvawfUXs5roaNJVIavYN0uB/wB8n/CnjU7JulzH+JxT9pHuHJLsWaKgF/Zv926hP0kFSCaJvuyIfowp8yFZjqKOvTmiqEMPEoPqMU+o5DgoffFSUAFJRRTEFFFIaYhTTaWkpiIk/iHoxp1J0lb3ANKMA5IzUybSutRixnD/AFrDvpLa+kePzFfaNssOcEH39q2toLlskMfeuW8Q+D4tZnF3a3j2t0DnzFBKntxgjB/OvHr46pTnapCyNVFWuiu0UMPyRRiKIHCoOiiucurvUILhrgaUWmU+UCLjeqjqTjAI9z06c9M9RFo93pdr+/upNSYAkcJGemMZOc8/z9gKqpbz3N7uNobdowMx4QjHqWA6fQjt61jPHqs+Wm7JFxhbVnFyWdh5jeZOd+Tuz5fXv/HRXXNpeiFiZbW8aQn5j9mfk9+9FcXtX3NOVHJ/GFiPD1hjOTdZwP8AcP8AjXm+gs7MnzNjcDjcfWvRPjGcaPpyc8zk/kv/ANevP/DwysLHHLf1r38vV6KuRjH+8Z1S7xjDuD7NTw8+B+9k/wC+jT0Xr3qYKMZP1r0uVdjhUmMFxdL/AMvEv4nNOF3eD/lu2fcCn7AOMUu3Bpckew+eXcFvrw/8tAT7qKkGo3ag8ofwpNn+c0nlg0vZQ7B7WfclGr3IHKRn8/8AGpF1qbIHkr+Bqt5YPB/lR5Yo9hT7D9tPuXhrj9Db9uzf/Wp415R1t3yPcVn7MCmtEP8AJqfq1PsP6xPuWpvF9lbShJoLnkZyqqR/OtGLxDaPGrbZhkZGVH+Nefa6Nl6mOPkGPzNdBaR7raLPdB/Ko+qU2y/rM0jphr1l6yf98U9dbsT/AMtWHsUP+Fc95IPAFO8n/wCtxR9Sph9bmdENWsW6XCj6jFPGpWbEYuovxYVzPkgcAUnkj86n6jHuNYyXY6tby2I4uIz77xUgliPSRf8AvoVyH2cE5Iqvd2w8jkAc1DwK7lrFvsdyCCOoNKO3tXk/2chbwKxHOTg47Ukf2q2W2Y3U65yWxK3fn1rP6m+5f1pdj1rjjtRjmvJYLvVQsxOoXgLJvUee3HJ96emp61HpzuNUujIVLZaQnHtzS+py7j+tR7Hq9GPWvGx4x8QRGMf2m7fL0KIcn8q0pvGWtrKscVyCAwDFolOfyFS8JMr6zA9S7c0Y+ua4JfFWrqPmMB9zGR/I1XXxzrPnmP7NZtlyB8rDA9T81J4WoNYmB6LTRXFSeM71Uz9kgOOT8xGaLTx3NPEXfTFTnGBPn/2Wl9Wqdh/WKb6na44pOc1yEXj2OW4aP+zZQq9WWUH+gqSDx5ZSytE1ldqy8kjaQf1qfYVOw/bQ7nWGk/SueXxnpxODDdL/AMAU/wDs1IfHGiK215JlwcZMJP8AKk6U10H7WD6nRdTQSeawIvGugTPtW9bcemYH/wAKl/4SvQhw2pwqf9oEfzFL2c+xXPHubXOeKO1ZqeIdGkj3rqdptHUmUAVLHq+myjdHqNqwPcTKf60uWXYfNHuXcn9aUVXF5ayHCXELewkBqVXVujA/Q1NmO6Hg0Umc8nNGcA0WGLnJozmm5zS5Hf8AWgBc8Z70tNzkf40E9qQxaBSE0ZoAcDg0pJxTM/Wkz7GgCTJyaN3FR7uODSk0ASE88Um7mo91NLf407CJt3/1qaXqIuKaxAosFydX5qQPVMPUgfiiwXLO/rQH75/Wq5fApC5pWC5a3+9AfjFVfMwO9HmHHNFguWw9L5nHPNUxL604S4o5R3LRkpnmc+9QGTjmmGUZosFy6JPfip45PlPSs5JM+tTROefTFKwFvzKaWX0B/CoN/uaQygDPX1ouFiwHA5AA+lI1y6naJZsnoqOwJ/I1Wjd5z8gwv989Pw9auwW6J7serHqaLt7CsluXdMW6ku1kluZiq5xGZCynPrmuoByBWHaLsGa2YjmNfbivSoRajqcNZ3kPpM0UldFjEXNGaSigQUtJRRa4GVJb31tcloJfNhYnCNjK+w9RUcuqXFs4S4te3LDI/ofbvWrL0U+hpc15ssFVhJujUa8nqjRTT3RiHxTZIxVo3Axyw5OfpVlvFWjzCILOVZhwNvtnHHetBrK1mRmnt4nDdQUGTUcmi6dJZ7ltEUGTzCF4+bkZ4+prmksSk+azt1KTjdWMo6vpu8oL1JWZSfLQ5475J44p8enw3YaZIVWZjtIB2jA6fXHPbvVS88M6WINzvLDsAJkD4Ax356VlXFjfW1havpFzdCMF1YOCrTbjhccccA85Hr714VWHNJNKx0XaRdmtrOKaSP8AtdItrFfLEiAJg9MbeMUVCuhakVGLfTU4+60SMR7E45PvRWXspdwuzz/4xOBZ6UvTMsn/AKCtcRoCfLBj/np/n+ddh8ZWITSFz/FMf/QK5Pw8Plt88gsD9Oa+wwC/cxMsX/EZ1qIOp65qUDnPr1oQZ6j61IBxmvTOEZg5NOCj0p23HWnY9RwaAEx/OjHGM89aUDIz7U4cCgBmOeKMdqfjJNOA5NFxWI8YphXA9PepSMnpxTSOOfpVCZyniIf6dFx/yz5H4muisx/osPGfkXn8BWB4jGL2L3j7/U10ViM2kByceWvf2pLcHsWQvGMc08qenNIBke/c1IopiGbevrS7e3FSY70bQe1AyPaB25qC7X9wenUVbxnkVDdg+QenUUmNHM3H35kXq7DP4ClnjEltAo6nAGfcVO8Ya5nbsqY/SoBmSSH0QqvHrWVywmVYpmyBjy8Co4kBsAMdjVi4jEl2i9gMn3xVYOUs1RR87ZA/OmmI5G4jZZSxwNzE4/u84A/St+O3WO3t1HUkEnHU1i3Z2uoPTn+fFdAozBbn/d/lSGaL4jjLnGAKq2cQeWWQjk1PJm4nEIPyjlj60tqu2aYAd+cU7gLJEu08DNUI1CWhUD5mY44rXZeOelVbeESPv/gThRSYIjhsxFEBgbjyTjvVeGAC/kOOueK1mXv61UVcXrk9x/SkwIp4/LjPy4PbNZr2obPr1+tacxLsWI46CmLC3lu5JGBx71Ldykita2KLGH43NWRrEWGb6CumhA+zr+P86wddGCT221fQkj02ITaU2eRlu1XYLcR2qrt5PQVFoLA2Mm4Zw54/AVrWqCRzIcEDoPSkMqxaaqKSQN7dTT/smDxxWoVHbOKbt9v/AK1OyFdmPdwyiEFZJBznhiKfbm8WBNt1cAgYz5rA/wA6vXKZgPHTmmwIPIA9M4qeVXHzMiW61IdL+6+nmt/jTxqmsIOL+cn3fP8AOpRGB2PFQy9lUZJ9qHCPYfPLuNk8R6+p2Rag+7PdFP8AMVNfa/4r0/w++qG4VohII9zwLhiew4q3pGn2b3BN7cLEijcc9WPoKd44vY73QpLaN0ZFUFVjPyoB2x0rJwjzWSLU5WvcwLf4ja+0UbN9lbd1JiP9DXUW/irU5I0Z47clgDwpH9a8qtjttozxlW/LmvRbFQ1pEePu1MqUL7Fxqy7myPFF5jm2hP0JFV7nxvcWrqDpquGGciYjH6VEIxjP61Q1G3DbDj1Gaj2MOxftZdy+vxCOPn0lwfacH/2WpB8Q7X/lpp10p9mU1zjW64+70phtVIIAHFHsIB7aZ1I+IOmkfNaXq4/2V/8AiqkXx/oxIDi5T6xf4GuOa0Tg7ec+lRtZoaPYRD28juh450HHN3IvoDA/+FO/4TXw+2M6mi/7yMP5ivPXslP8qztSsNthLID0APQdjTWGiH1iR6unizQXwRq1qM+sgH86tp4g0mQALqdmf+2y/wCNfPmP3HX7rfz/AP1VuW9orwo/qoODSlhorqCxL7HuC6jaSD5LqBvTEin+tSCeM/dcMPUGvExp6sp4HJ6mo308AfKMVH1ddyvrHke4h8nj60hfGM5rw4wToMRyyIe2HIogk1guFhv7xfpMw/rR9W8x/WPI9x8w+tL5nPWvFn1DWrQPnV75mCZ5uGOPzP1r2T4SSrr3hSaTUh9puYrtozJLy2NqkD9TWVWl7OPMXCspO1ibeOgpm/nj9a7g6HprHm1X8GP+NRP4c0xh/wAe5H0dv8a5vaI1ucekgPepklwf8K6M+F9PySDOv0f/AOtUE/heDyX8m6uFfB25KkZ/KpdRFJmG0x3BVyWPQDmporVnIaY9OiDp+PrToIFgXGMt3J6mrK81W5WxIijgYq3CvNV415q7CvIrelT1MZyLsIworTgbMVZydKu2rZUivQgrHDJliikNFaEBRRSUALRRmkqkJiPyh+lIDnHvSnkGmRn5B6ik0CJpXwAgOB3qe1O63kXuOapn9as2LfvGXuV6ViqSjCw76mPq8DOQ+AyKufLPQn1Pr9P/AK2MGae7nLy+cwVG+YnoBzx9c4rqNThaWJlLGNOQSvX/AOtWNaxRXFpLaNiMCRANvfJ4rwqvuuT5dE0dyfuopCe8KgjkEcHc3P6UVea21gsTGyKmflUSAADsMUUvaw7IV2eQ/GRiZ9JUdQsv/slc5oAwlt6bv61t/F2bzrrSDn70Lvj1yVrG8NjEdr3A6V7GC/gxMsV/EkdavHtj9KmAHH6imooxT1GDn+leicKFxxRjHvTufWgZx9KBjQBmnBR6daXHNGOKQxCBnqetLjOD607b2HWgLzTQhCM0w9s9DUuPpTGHpTJZyniMYvIjx/q/6mujsB/ocHT/AFa9PpXNeJm23sJ/6Z/1NdPpw3WFuf8Apkv8qS3B7Foenan4+uaaFI4z+VPA49cUwsHIzR1FOx1PB9KNuRx0pXHYTAbtUN0B9mP4VY9sfhUN2ubcnGOQf1pPYZgthZLkngAD+VRlRFbwE4GW3GlnDNLKnY/MeOwHSnTKrRW8fQtj8sVkaWK8YLTl2z86k49B2pkEQETS46qQvsKtTMI7hiSMLHio0AGnnjHymmhHH6iux075zmt2ED+z4GPQKOv0rC1RHW4V2ztYjaPbHX8a1rMvLZL1CRoB+NDEjatIike5vvvyaLcf6VMKmhJMSNzyBUO8Q3UhPHy5/lSbGLOxdxCnU/eIHakswQsg9GqS2jJBlcfM9JaH55R70rjJdueDVOfakjAck9fYVclk8tCf4jWexyx9TSkxpDVjMjgD+VW5IwkBHoKfBD5aZI+c0s+PKY9OOKEJlW3H+jDI9axdcXKnjOUrbg/1HJ9awdek2ywxryWyD9Ku+hNiHQAziVBwobcfy/8ArVvaeflbjnisTw02XuUz2U/zrcseZH60uoy6BznFKRk04Dj2/nQeOcc1VySvKuY2+maitV+U9OuatOMofU1BGDDHlgcnnHpUNlJBI20YHJPTFIsQU5Y5Y1IkZ5dx82Pyp5AJzjNNAyFhVLUY99hcL2KH+VaJHyjPSq1wu+J1xwQRVXEefWozanHUPnp7V6Lo7B9OhbjG3v2rzm2zsmRuxH9a7/w44fSo+nHFZSNYG0Bxz0qrfp+6BHUNzirQ/wAiq96pNufYg1BZmFecnp3ppUZ44OKk56/1pvOOvJpk2I2GevSmFOPXJqU/mBTT/nvTuBXZevb6VVu4t9lOg/ijbH5Veb6VEVBOMcHiqTJZwQ5gkHuD/n866PS/nsYWxggY/XFc8g2iZOny/wAq3dEbNmAeqsRjNOWwkagTI6Ch09PrzUsalzgDmr0Vrt5cc9ayuWkUIrJpW+YYX1q2IFjXCgCrm0AYH6VG64BPb1ouVaxganHiXp95P/rV6N8B7rNnrVqTyrxSgZ9QwP8AIVwGpphYyB1DD+VdR8D7gxeLdQtScCa1bA9Srr/QmssQr0mODtNHvZOcGg0g6Y9KWvIudYzvSHkGg8GkNQ2Uc3ewmO6ce+R+NRqvNampxZKSf8BNUAuCPfmuyjrFBJkkYq7CMCq0a1bjGBXfTics5Fhat2xw2KqLU8JxIK6EYMu5opKKskWikzRmmJhmlpKKYgzUa8Fh71JUfSQ+4oaEKaltW23Ke/FRGhW2uG9DmpktBk9+uRIvfrXD3MLPe7SSo7kdQPb1Nd7fKqgzEgLs5J9K4q5R5LlpJI9kefkQjlh6sD/L8/Q+XVdp27nZSd4Dc3X/ACz09Cn8JMRYkdue/wBaKYwYsSZHJJ9aKy9kyrM8i+Lv7vVdOjBO1YG4x/tHH8qy9EuVggtWMbMFQZx3rQ+MDA+IrJT2tv8A2dqy9MAFnBnBOzr+Jrqw8uWhFk11erI6hNahxkwzDv0H+NTDWbYgfJL/AN8//XrDXnp9KlH6YzW31iZl7GJuDV7U9RIP+AUv9qWmc72BH+yaxDx3pVPbNH1iQexibn9p2ZH+t59wRTxqFpx++A/AisIdMjrR37U/rEg9ijoBfWp/5eEH1NPN5anj7RF9N1c7wB0ppAzjGKf1l9ifYo6Q3MH/AD2i+u8U0zxEYEiH/gQrmjg88flULgDnj8qpYh9hOiu5F4slRb6Ahs/uzyOe9dZpbqdNtiGGTEv8hXm2suEuY+3y1vaaF+xwHGPkH8qp17K9iVSvodyF9P51IFPrXJqfQkfialUuD/rJPwY0vrK7FKh5nUgHrjn3pCMgY9ea5pZJs/6+X/vs1ILi4B/18v4tR9ZXYPYvudFg96iu1/cH2IrGW7uf+fiT86Gurhkw07EehxQ8RHsHsGRsuXu2I5AA/SoEBeaGQ/dBCqPoKny5LfPy3B4600bxtAbhORxWftUV7NjLiPzLph1Crk1FKfL00Kf4kIHPepxu3M+RuYYJx1qtPGSgU4wowOKaqoXs2c3ri7Xt8enX8K1bBAdLTAHK8/WsfxA7i4jDFdoUYGK09JkdrCMDG3BGMVbmrEqGtjZsyDbRn29abIA90sfQdz602BniiCKFwvrSpGUkWQHJBzz3qXURSgy5joB17VWtyFuZgT71J5rcZQc+9REEs7AYLe9J1ECgyKeUu2c8dqbCB56cf55pxhJOccd6eR+9V1XAUYxUqauPlLYHI7daimJ8p8eh6UpnGMbWpjSqQRtbnPatOdEcrKqy+Xa+jZwPesrUYCsPmONzkH8K1QF8zc4JUD5RiqGs3MSwAnPXHK+1OMkJxZleGW/0ycZPKf1/+vXQWX/HzIoJ/H61y/h6QR6s+ehUjp7g109uwW5LfwnPNNtIFE1OR/hSZ4xmmCVDyD+GKPMQDr09KOZC5WKx2g889KrwtvnO7qBT12u5Z3AC9BUUOFuskYXkZJqeYqxsWOnvdAuwKxDuB94+grVGnIBvZOeylD0p9vLarFGxki2gYVcjn681R1HVFkDRW7jn7zjj8B7VnzOTsXZJFPUGgRzHCEb+84HH4VlSLuUgg1YbDDsKhZeuea3jZGLPO9nl3t3GR0JA46c12XhZy2nnnO1v8a5K/URa7dj1JP8AI10nhCTMMq56H/D/ABqZFwOr71Fcgm2fnoM09TikmBMDjGeCOKg0MnnB9qQ9fYUdevSgn8qLiGkGmHvwSKcf/rUh75P9KAImHH+NRnjnP5VMRxjH5VAeuRxTTFY4yaLZqdxH0+ZwM1qeHYmkjlQZ4YH86i1K0K6rPIeFBDAeuQM1a8NNtubiPIyVB59v/wBdVKWgorU6aCJYx8o59anGAP61GuAakHB96wubLQXtUcigjrz3qXIxg9PrTGOeKaYmY+qL+5BGeG5/KrfwvuPsfxNshnCzb4z/AMCQ/wBQKh1Bc2zjv161n+H7j7F440W4yFAuoST7bwD+lOSvBonZo+qDw31pe1EnYijORXiHZ0Iz1pD0pW600nAznArNlIhuo/Mt3HcciskCtW1vba8lmjgkEnl43EDjnPQ9+lUHj8uVk9DXdhdrE1NBI16cVaSokFSpXpQRyTJl7VKhw1QipVPIrYyL1FNQ/IPpS5qkSLmikoqhC0UUU0Ji1G/31980/NMl4UH0IoYhTTTTj0pppDNFsSWsTEZwP/rVh3cCtdAFFYsQBuOK2rZt1lj+6cVlaomORXmYuN2deHfQQ6hZIdggiIXjOOv6UVjhJMD5M/nRXF9XX8xryI8L+LrhvFEC9harx/wJ6z9NnhSzhUyICEGcsKufFaT/AIq9RkjFqgOPQlq4piRI2DwAOa9GjG9GK8jOs7VZHcpNCQMzR/8AfYqZJoiOJFI+tcADjjP4U7cT3xjtVeyJ9oegBl45H509WGe1ee+a27KknHpTvPk6iR1z6MaXsg9oehD6Zpe4rz77VOowJpPYhjTxe3IU4nlx1++aPZsPaHf5Hemnt6muDGo3isCLmYen7w07+1L0Di6mI/3jR7Nhzo7Yn/IqJzlcAfhXH/2rfZH+lSdu9KdXvQeblz9QP8Kagxc6J9eP+kxeu3ofrXRaXzYwdT8g/lXF3N1JclWmfc44yRV2DWbu3hRI3G1RgAqDgVUotpIUXZncJUy+mOO3NcSviO/Gf3sZ4/uCnDxPqC/MTF+K/wD16z9my1NHa9sU4+o/UdK4seKb4Y3LB+Kn/GnjxbejgxQH2wf8aPZyHzo7IHBHH6U7P41x48XXRwfIhPr1/wAad/wls2ObWMn2Y1PJIOdHWkgDGaM5A6+tcp/wl8ne0Q/Rz/hTh4t6k2g44/1n/wBanySByR0+7I9aic+lc6PFy4/49D/38/8ArUp8Uwtw1tIM+4NPlkLmRV8RsVnjY88Vd0Ns6dF17/zNYer6kl+VaNCMDByas6Xq0VtbCOQMSCeg96tp8pKa5jrY2AGKlz3rCTXrUcNv/Kpl160PV2HsVrOzLujYyM8A0bvSsga7ZMB+8Iz6qaX+3LM/8tfzBpWYXRqlunIxRkdulZY1qzJ/1w/EGnf2vZ8ZmUfnRZhoaWRznA/rTSe1Z/8Aa1mQR568fhQdVszj/SE/OjULouOeKydYG6yPHRgastqVoc/v4/8AvqqGo3cMltIFlRiOwIqo3uSzF0ZtusLweQw/SuziI4HT61w2mPt1iJmYbdx5J9q7OKQHHI6+tXPcmGxfB7nNBHHPFRq3HBBz75pc547+lTcsdn+VJwM8A0mc0h6UrisOJX0FNIB7Um4delIDTTGIQmTx9ajZVHf24NSMT1/SoZG46002TY5HWkSLVJSAQSAePcVZ8NyNicKzLgjkHHr/AIVH4hGL1CehQfzNReG3xdzoTjKg/kf/AK9a/ZIW510csuR++kxjPJqUSysMeaxB681VjP8AKpwc9xnNZ3NACAcZORR5QHHSnL+eKM/jSuFiMxZH3qQxlT94496mOcDPNIev+TRcLFfyuPf6U6O3GeeakJA6/jTkyM880OQJHPa9Ftutw6tF/iKz9Ck26p7OrD+Rra19QTA5GRyv8v8AGue0tymqQMT3x19aqOqJa1OzBx0P5VKv0/Oq6tx7VODzzjmszQeDjp+NMZuOn60pYZHb8ajY8HP40XCxVuzuikX1Q1y1zMY57aVW5Q5B/KuslUE7T0PFcffLtiQtn5Wwf8/561pF30M5I+vYJlubOKdPuyorg+xGach4IrmvC2t2kfw+0fULy5jhiFpGjO7fxKNpHucg1h6j8RbKR3is5Wjj/wCehQ7j9PT+f0rxnCXM0kdkNUdlqGq29iMMd8vaNOv4+lcpqWszXmRI22M9Il6fj6/yrmJfEto5OJTycknOSfeqkmu278+bx3pqjLsdEeSPU7rwjd+Zq08JOQ8W78iP8a6O+jxMrjow/UV534N1eB/FFrEsgJlDJ/46T/SvT72PfBu7qc100Vyy1MKzTehQQdKkApiipBXoxOKQ5etSCo8U4VqZsuRHKfjUlQwH5CKe7rGpZ2CqOSTwBVrYgfR3qKK4hnz5Uiv/ALppzSohAZ1Unpk4pgPozSe/agMD0IP0NMQ6mSjMTfSnUHkY9aYhqnKAjuKaabF/qgO44pxqRl7TmyJE9QDUVzGpXc/IXkg96LBttyAf4gRUt0H2soQH1+bFcONXu3NaT1MVruTccPIBns2P6UU4xS5+8B7Bun6UV43NDzO/lifNnxQfd40cdlgjH9f61xzDErZwK6z4ltnxrc9OI4x/44K5UfMzA859K9uhpSj6HLW/iMASDQfqadtBIpcdBjr3PetTMbuPt0pDnOfWnlcc5xmjHzZ7mkAwHHr+PFGOOAak2Z+7/KgLz60XAjz0xgZ9aXtjnFPVSc85BOMYpAvTIPv7UXAjJ5OOtGecgAZ7etP24GD+RFIVA5yOO1O4rEbcc5HXHXpUoPyj2GOtLb2st7dQ2ttHvnlcIiA9WPQV06/DvxPgZ0zBHX98n/xVKU4x3Y4xb2OW3YHH5005J6V1q/DjxQyknTlx6+fH/wDFUH4a+KWP/IPUD1M8f+NR7WHcv2c+xyeec8AijP4jtXXp8L/Fbgj7BGD/ANd0/wAaU/DDxWQB9gT/AL/p/jR7an3D2c+xyKybCSQD9RTS5Jz0rqn+HHieJyp0/JPYOpx+tIPh34oYhRpp3HuZFx/On7SHcXJLscvu57fhQXH4d66d/h74nViraa3B7Op/rTD4A8TL/wAwyT/vpf8AGj2kO4ckuxzSt2z+NAcZBx165rov+EC8TAZOlSe/zL6/WkbwJ4kGf+JVJzjoV/xp88O4uWXY5tiQPb+dPjY7SQeK328CeJW+7pUv/fS/40+PwF4oyd2lTcejL/jT9pC24ckr7GBvxznjuKBIQO4xx1/z610f/CCeJSD/AMSmbp/eH+NNPgPxJk/8Sqbkccr/AI1HtIdx8kuxz289z9OKUSFV684rfbwJ4kI50qbP1H+NJ/wgnib/AKBMx98j/Gnzw7i5ZdjA83pwc4xilEpC56Gt8eA/E/I/suX/AL6X/GlHgLxNg/8AErk/FlP9aOeHcfLLsc9vOeaaZDjnHPJxXSp8P/E7kgaVIOOcuo/rUTeBPEm7H9lzf99D/Gjnh3Dkl2Of80sfx+lIZRwM8jv610B8CeJhx/ZUxweuR/jSf8IJ4mOP+JVNz05H+NP2lPuLkl2OZVyHHrmnGQ9Mmug/4QPxP0/sqYY9x/jTv+EB8TY/5BUn5j/Gn7Sn3FyT7HPCZxyHI/HFO+1TqQPNfj/ardPgLxN/0CpRx6j/ABo/4QLxMTj+y5c9eo/xp+0p9w5J9jFXULoLxczc+jn/ABpw1O9A/wCPufA7eYa2G8B+J++lS/gw/wAaT/hAfE+f+QVJ+Y/xo56fdByT7GQNVvVb/j7uM56GQ1IusXwGPtcvt85rT/4QLxPnH9ly/XI/xpf+EB8TD/mGSf8Afa/40c9Pug5Z9jN/tvUOn2uT8e9OOt3/AEN05x64P9K0l+H/AIpbONLf/vtf8ac3w+8UI206Y+R6Op/XNLmp90Pln2MC6vri5IM8hZhwCQOn4U20vJrOcyRNhiCDxnit9/h/4mQrnTWQMcZLgD+dSp8N/E7EYsAcjr5q/wCNHtae10RZuXL1M2PxHeoP+WTY65X/AOvUw8UXg48uEn/dP+NWW+Hfifdt/s9dw6jzk6fnTD8P/E69dN4/66p/jSU6T2aHFylrHUaPFVwOsEP05/xqRfFUmebVOP8AaNH/AAr7xRnH9n8+0qf41Df+C/EGmWUl7d6eY7eIAu/mI2BkDsaL031LtNdC0viph8rWyE+z/wD1qf8A8JOpHNp+T/8A1q5MN2z2pwkPbGafs0Tzs6xfE0RxmBxkdmzU0OvwNJjyZB6ciuPDnPP5VZtpT5w688fpUyhoUpHQarqEV1Ciqjhgc4P0rAjk8u9R8cLJk4571bkbK9M+mazZSRK/1HSlTCTOwTUoMdWH4VMmqW2ANzf981zaN8mRnnvU27gHjFZtFJnRf2lbYB39/Q0hv7fOPM6+xrng+F7g/wA6rT38UHGdzegxQot7DcrHSyXdvtz5oA9a5LVL2FppYojuDP8AeHTr2qlcX01wTlgqk52jpVU9a6YU7bmM532N/S7jXtVRLG0u2aK1Q+XE7DagLZOAfc1pPo/ilQSTE3/AF/wrlbe7nsp2kt5GjY8ZU9q0U8Uawi7ftrkf7QB/pSnCV/dsVCcLe9cuy2viSM/NGv1CJ/hVctrafejH4qtQN4k1Nx81xn/gAqFtavHGDIP++RSUZ9Ugco9Gzo/CWpanaeMdGlmTbGLyJXOP4SwB/Q19WModSp6EYr4wh1i5jnjl3nKOGGPY19m2063VrDcJysqK6/QjNc9eLTTsaUpXujMC4JU9RTxUt0m2cnseaireDurkS3FApwpB1pa1M2TwHtVbWrae609o7cAybgcE4yM1PEfnFSzyeTA8mM7VJx9KtbEdTnoYDp+oQqjsXYKZELbtpPXHtU95ZXE91cM7PEgBZZRgjpwMGq+jSxTSveXUq+YTkA1f1e4LCOCNvlf5i2eMdqStYp7kCXMraPJ8xG11GfY44qvbtJaXUM3mlkl/hwOMHBHvWn5EQ0qWCGRHbYSdpBJNZLsJLKEg5KyEfgRn+lUSWL77RPezpHIFdOUBGRwM4rU0m5a4tPnYsynqe4rPlfbqkEnQSIpP8j/SptFJjmmhPbgfgcUxGoOGce+aDyaDxMfcUGgCSBtlwjdgwrVlTLH3FYtbhO9Eb1GawrxvFlRdmZTRfMfrRV0xjJ4orxeQ6faHyP8AEbnxve4z92MY/wCALXN4JduCfm611Pj+JpvGl68S70Pl4ZeRwi+lYuIYy+9CH3etepSdqcV5E1NZsp7DjOM+1KBng/lj2q6t1Ag2hMZ9xQJbLrsb/D9arUkphCcdeKAhBBwTVwS2Z+VQ3JwOTTRJaEjGQDxRqIrbDkbT+lKBnI6DvWja28d0jOnQHGSe9T/2coB6fianmKsY5Xk8c9KTy+3cVtLpqg/MMDOPvCnjS4Twc47fMKOYfKYJUHoOg6UjIV6g/wBK6D+zYMHLNz7jFMbSrZh/rWH5UKYcjK3hNseMdDzjm9j/APQhX1YEjbBKLnHXFfL2i262vjbQ0Viw+2RHJx13ivp4PhR7VlXXM0zSk7JjvKiB/wBWn/fNKY4j/Av5CmtOpG3bz/eppc1j7M15x3lxZ/1a/lS+VEesa/lUfnBOSCfpTRLnmj2Yc4phj37gi5+lKY0/uj8qTfSbqagHMSCOPH3R+VOEcf8AdX8qiDj1p2+jkDmH+Wn90flTfKjP8I/Kjfmk8yjkDmHCKMH7o/KnBE9B+VR76XfRyBzDwiAfdH5UFE/uj8qZ5lIXo5A5h2xQeg/Kgqn90flTC59TSb6fsxcw/wAuP+4PyppRP7o/Km76TfRyBzjtif3V/Ko5ETb91enpQXpkjjHpSlT0GpjkVNx+UdKQqm1RtX8qjRxke4pA/IrP2ZfOSeWmw/Kv5Uzyk2Z2DJ9qQy/Jz60M/wC796PZhzkgjjx9xfyphjQk/KMn2pPM+XNNEmX57U/ZhzjnjQAfKPypTFGR9xfypkj5A+tL5gIBpqmJzI2ij8wfIOnYUx4kY4Cr+VOeT5s8cCmK3GTnJNP2YucjeGNSAFXH0qrqV5Z6damSfCA/KpCZP4VYupBDEZGzhVLHHtXL6fq9xqEiXzoQrKSnPC+gHrj+dTUcKUXObsjnxWNpYWk6tZ2S/qy8y9dXUV0VeLzdoyMPnH4A/jSy/wBrWNqrppokt5CCJmdcKDjnGM1pC1SDQLqV23zzBCT/AHRuHH+NNEr6wba23NHbW0ShvdgMZ+vYewJ9q8hc0KznJe/JaLte619Ers+LjWr0swlXlH99Vj7kb7XbSvr0Su/06ZtkgubsqJV3J8zjIJ/KtR7eNjgRr78dKs3drbITdJbpHIF/1iDGQO30pkXzIHH8XNelhMJGhHlTu+p9RleWQyyjyKTberfn6dPzIVhQPgqvX0rP8V2sdx4V1WLYv/HpIQNo6hSR/KtR5cMVH07dc5z6+1Raign065ixnzInX81IrsjTsz0XO6PljAz/AIUox3OMcUpQjP65pdpZuB27Cu44wAOeCOP1qWD5ZVPuORTVU9cfnTwAOQAec1LGjQcjGD1PPIqjck+cD7enWtBwNox0PXis+96KR6f5/rWdPcuWxdgJMSEeg5Bps06RJuc49PU1ni9MUKoqjI7mqrsztuY7j71ap3epDmWrjUHlG2P5F/WqWfeiitUktiG2wpDS0rUxCGkpewNBGKAEooooAK+wfAF7/aHgDQrjOSbNEJ90G0/qtfH1fT/wRvftXw3giPW1uJYf13/+z1z4lXima0Xqd5eJlA3ocVSrUlTfEy+orM71NF6Gk1qLSg0lLW6MRynDZqyyh0KsMqwwRVb2PSrEZzEhPUqM1SIZnL4fsE+4sq/7srf41PcaXBcLGpaRfLG0FWI4q7RVWQrspWmmR2khkWSRiRjDHNQNoUPnF45pEQnPljBA+mRkVqUtOyC7KV5pkd3FGodo3j+46+nofUUthp/2PczSeY54zjFXKUGnYVyOQYkQ/UUpFEv3QfQg0ppANrXtG32ae3FZBrS05swOvoc1E1dDRNiin4orh9kjS58OI5jVQGI+hoLtuIYEEE54HFRQBmRBnn2q7DomqXU7CGPLAZ5cV2KN2TzWK4lP+1z0pyy4/hI57irv/CK64B/qAB/10X/GpF8K62SR5KdP+ei1Xs32FzruZwlyB8v/AI7S+ZngKc/7tXv+EW1wDi3Xn0df8aQeG9cUf8e/H++v+NL2bDnRVWZwNilwOuFXr6nj8KcLmfOA83pwTW5pnhq/eFzc2uXDcfOPT2NX28MzggpZkewf6e9Q4tPYtO/U5Vbq5Zxh5eRjvUyyXxHyib8FPNdInhu6VgTauvP98f41KNIvYyQlq3HToah36IuNurOYA1VhxHckf7pP9KRzqsQy8NyM9SYzz+ldYLa/QEG0J/4D/hUTQ6huw1vIM8/cIFSnLsaWh3Mfw1JcP4u0gShwReRD5lx/GK+mweK+dLCGSLxjo7SxshN3F1GP4xX0UOg45omr2FB2uKTSeYBTTnHAqM5JpKInIkypOaXK4qABmPAyaN3ODVcgucnytLlfpUAak3Ucgc5Y3rR5i+tVt3WjdzT5ELnLRdaN61V3f/Xo3Y+lHIHOWt60b1qru4pd9HIHOyzvWkLrVfd70m/Jo5EHOyzvXNJvX0NQbvrSF6OQOdljevpSb1quW96C1PkFzljcvpUcrrt6VHvPrUcr5AocBqZIJE3Lx/DTQ6bc471X34Ct/smgN+5P1qORD52T+YvpSmRPL6dqrbs7j+FG793j1FHIh87LDSoExjGaRJE3dDyKqsxJ9vWnBgso+mKFAOdlmaRdvSnCRSBxxVSVvkp2/C59BT5ETzskeRSQu3rTgyDHHA7VTZ+QR0z61Nvp8iDnY64Kuu3GMgjiuSEqJeQ2AhlSRnKLG0ZTao43c/w+mOv546mRuAQBx2NVJ7KOKRL6MItvCMBFHLsx+Yk+o4P4Vx4rCU6zi59DkxOW0MfOEqzfuO+j/r/Mfq1s4tUdWYxp94Hp6Z+tZemWk0+shh5YiCgMytliPT0B5rb1GVf7MlQOOi9O/IrKGrm3tsR6jZRzrgCGRMjA46Ag5461yRpQ+uufXl/WxFPB01ncqqWvInu922vyW2xW1SG8i1uSLMzKhwDHPtwp5BK9DzxXQRkJAiDHyqBXPLrM17P511HalwQuIjnHvmuhVSUDd8CvQo2bZ7WKTikV5Tlzjil6jB78VIYGfc6cqo55/wA+lJj5PwrblORM8Qm0vQ4ruaJ2td6SMrAyAEEEj1pV07QO32UjrxL/APXrP8S+GJm8T6myMQr3MjKNhPU5/rWSfDFznhuv+yahxj/MaqUv5Tqho+glRxBtPTE3X9alTQtCckAR8df3xrj/APhGLofxj/vk/wCFMbw5dr/EPyNLkX84+d/yHdnSNExgAE4xxKTXmk3myXLx5LFSQAPb/wDVVx9AvFGcrj8f8KzJY3gnaJ/vKcGtaNNRvZ3Mq03K3u2NW1sraXTpPNt7gXe/CNn5NvuOvrQdKjwRmQH6j/CqdhY3N95nkMP3YyctirH9i33H7xf++jVt2e5mtVsbumaDod7LHbsb4TMvJyoXIHOODW2fh/oythrm5U/9dF4/8drio9J1RX3RS7W9RIRU62GujG28kHGP9eRisZKTekzeEopawOw/4V5o7Di8uh/wJP8ACsjxR4LsdF0Y3lrdTSOrqpVypGD9B9Ky/sniQdL6fj/p5PH60k1h4iuYDBNdyywsclGuCRn6E0oqaabnoVJwcWlDU5zHy/jXqOn/AAu0++0+2uxq0w8+JZNojXjIBx1rhh4Z1TBHlrjP98VrQQeLreBIYb6dY0G1EE/AHoParrScl7k0jOjFRb543Omk+EluPuapMT/1xB/rUDfCOTkjUZAPU23/ANesR7nxvZwyTfb7oIiZZhN0A61mt458TM2W1i4Y+5B/pWMYYh/DNM1lPDreDR05+Etx/DqOc+tuf8a9Y+Emg3HhrSdQ0+abzVadZlOwrjK4PX/dFeNab4g8f6lb/aLG6nli3FdwjjxkduRXp3wo1DxbNrl5D4iMjW7W+YiyIMOGH90ehP5VEvbXtOaY/wB01eEWj1ys2dNkzDtmtKql2n3W/A1rSdmZTWhVxmnCiiupGItTxnKfiagqWI/KRVIhktFFANWSLRSUUxDqUU3pS0wElGY29cUincit6inmo4+I9voSKTADV3TGxK6+ozVM1PYttul9+KmWw0aeDmipNtFc/sx3Phi2X5ohkfeGB+Nd3og33koGANmefqK4vyHtrxYpUaOWNwrKy4IOe/pXcaEo8+ViRwoGMe9dNPciextFCo+8CDxgGmjhs9Oc1Lgc5YfjRtHJ3D866DAiIOcY/WjrjipWVeDuH+NMKj1H4UAWbPGw+zCrJIJ/lUFooKMM55FWShGOKzluax2Gd8/570hyDkYzTyMgDFNOAen+f8ipKIznGaRueOMDtT9pK8g9cc0yRxHGzuflAyaQzm9ROPGvh4cZ+0xk4/3xXuykEV4HOr3XjTQyTsaS5jG4/wAPzjFe4rHcAf67r/s1zVHdm8NEXTKxjMYxtPWmFeCKr+VOf+W36U/ypsH98fyqUDHrK0GSuNx45FVtxJJJ5NMkhn3czHGfSomik/57tWiM2WN/vRvyODXKeJ9UutGa0ZLlwshYMMDnGPb3rBPjG9z/AK4kd+n+FFxpHo5fk0F686Xxfc952z9B/hS/8Jddf893x9BSuB6Jvo3jHWvO/wDhLLnj/SJfTjFH/CU3P/PxKKOYLHonmYo8z6V53/wlN13uJfzpyeJLp/8Al5l+u7pRzofKz0LfSF689fxPPk4uJv8AvqmjxJcHn7RN+LGlzoOU9F38UheuHg1yR8bppv8Avsirg1MuARPL/wB/DUuqkUqbZ1u/PSkL1zEl6yxqfOk5H/PQ1GLtmVj58/H/AE0NT7dFexZ1W/nGKbI3y5rk2uXfhbqVTjjMjEfzpkF1Luw1zK5H+22PyzSeIXYaoPudKXJRfypd58tuvWueW6fGfMbk4yXNMub5kYgSt17Oaj6xHsV7BnSK5O7r1oVm8scGuT/tJgf9ZIf+BmnrqZK8Fj77jT+sLsL2LOnUsX5HA60rORKCAcVyU2pkDBZj9GqodSbPLt9Sx4oVddhOkzuJZDs7/hTTIW2oB27VxB1N/wC83/fdRnU2xncT/wACq/aoXszupHI2ipd59D+VcBLeTRCNnRlWT5lPqKg/tXqMr1/vd/wojVUtURFKSvF3R6FM/wAoGD9KwvF19qFj4ckvdO80TxlVYKflKNwdy9CB15rmP7VLAbCmAeTnOaradqpn8T2UFxHm3Z3T5icFyjYPtg/0P02prnkkD93qbOjeKzcOukXbvLIfkguGj2edtGTuUE7WwD3II9DxXXwW0DWm+AW6TEfM6nYxOO5HU155ZaFNp/iazuo3eaykZyBIuXgfa+QW9D79f583rWt3dvJqklhqYt2gvJI2t5bgMzfMRuVSucewbj0rjlRUsbLk/lX5s4KVd080fP8A8+1/6VI9A8Q6hYeGLWbUnSGS9kO2MMxZpG7AknOK7PTbmS80m2uJ4BbyzRK7xE52EjOM1856bfvNrUWpa002pwWkqlwZTgjPGMjpnGRxXuOj+NfD+rOkEZlgmcErHLD97jPBXIx9cVvGmqSuz06+IU/eeiRuySMmFjZhkYOO9R7jsxUj2aSrmNFZSM8dPrmqq2iAkFACDggiqM09Lo4vXQseuXOADkgj2OBVDBHzFRgnjJra8R2wj1cbQFDRKew9R/SspcDou7sCegrzqmkmenSV4IRdoToQcdKlVWb5nBAHAJP8qRYi3rz69KnEDNhQS59hxWTZukR74wVOxiF9Tx9cV5T4uTHia6brv2tn1+UV7NBayQ/OFQMOdzgNj8+K8w+JMb/8JDDO3JkgXJxjJBI/wrpwc/3ljmxkP3dw+HKefq99Z5wJbY9/Qj/Gu/8A7IdZ45UkQlWDYC8Z2gEfzrzn4ftt8ZwR9fOR1/8AHd39K9vgsS7D5cgdazx0uWoPBwU6epjWGniPe08UfmPIW3KM4GAMc1pwaXbSI26FBxy7KMfh61qJaE/KoAU9l6H/ABq2lmiAbsk/3RXmyrHcoRjoYyaLaSHiBCM8sYxU6aJZI2DbQOccZhFbawk4CgjHTPWrMVoR7H9TWXtZMTlFbmNFodhktJZQknsY1q1FodgeljbZPUmFf8K37bS3kOdvHr0qWefTdL/18oeXtGvJrWNGpJXbsjlniI3tFXZgSeErK+s57f7DAfNjZOIwOoPevlXV9In0fVZ7K5AWSJzGwBzgg4NfYv2jVtTTbZwCzgPG+Tg/gP8A9VfKPj22msPGeo2txJ5ksVwwLn+Lng/iK9bAR5G0rtHFXk5K8rXPUfhBqF5rPhO40W00m1It5h5lxtAJDc/Mecng/pXqWj+Em0qRr2e6Z5sH5EG1Bn2715R+zlqQiude08kfMsUy/gWU/wAxXu01xuUrRWjThUbe5Mas+XljsVqjnXdE3tzUlJ1rWJTM2lpXXa5B7Gm9a60c7FHSpYjzio6dGfmFUiGT0ZpM0ZrQkXNLTc0oNAC0oNNpQaYh2aijOJJV9wfzFSZqPpc+zL/I/wD16GA5qfA22ZT6HNMNLF9+kwOgoqFJRsXPpRWYz4jklklAmlkZ5CAzM5yTj3+ldn4acSRPKP4kU1xjr/ooxjOzn8q6fwbJv0xjnkEJ/n860p7kz2Op3H1xSFvbjP0qPJzgdaQnIz6cY9a6DCxMrZUj0OelNz1x3qMEqTx14oPDdzSGX7GTDsmOvI/D/wDXV0sOncDrWLDJ5UqN6H0rXHTAOR6+vSs5GkNh6sAemBTDg5wPel4Kj+VB69TzUliDrzVa/wA/ZCMfxA/rVgjGOeKr3mDat6cfzpPYpbnMl9vjPQW9LqP/ANDWvfFFfP8AN/yNuh8/8vMfb/bFfQSg4rnkaIYflwAuT6+lLzg08rQRwaEJkSNGG/fKWX0FVSOTgcZ4qcjk0witCGzg/iLbvPFpyqxH7x84+griBpU5xl5P0r1rWLBb+5s426KWI/IUkXhmIsG28ewo5Li5rHlQ0qU/xyfhj/CpRpcgH3pPb/OK9ZPheHjCnj2p6+GIh/BT9mHOeSjS5OuZPxx/hTv7Ll9JPz/+tXrq+GoAchKd/wAI3b/3KXsmP2h5LFpUjMM+b+dWZtGkiYrGshXHdutepr4cgH8FWm8P25+YoOlS6TuUqiPF20yYdUk/OpYtJkLD5H/OvW30CDJ/d0LoEI/gp+xYvaI4Cw8PxyAF1l/77IraTw3ZnA8qT3/fN/jXY2+kRp0XFWv7OTP3f0rCVCV9zVVkce2gWKKg8pm78zP/AI019Ds1iYiCQEcY818fzrrZ9NYgbV7YyK8N1Oy+KJ1K6W1XUDbiVthGzGMnGOM1n9Xl3L9sjujosO8lUZRj/no3+NV/7HjLEgHPs7f41wq6V8SpWXzU1U7fm+8g/wA96nfTfHbXcDrbarborDfvudwbnvntUvDPuWq67HYppkS4+XoeBuPH61HeaaobG3I/3jXSJATGdyjPSm3duBnKgAVj7F3NfaKxyS6cGbHl5/4EatJpMJQfuc49zW4tsNxwBkLVpLceWOPalKlIFURx1zpSjgRAH3rP/s1dwBiXk/3a7a5tgZPwqitoJLrI5C89aqFKRLmrmENFh2jdChJ/2adHo8KSKxt422nOCvWuoW23TDIwAOKbcReUhYYJ+lOpHlg3LZIxxFWnCjOU9knf0sc5rGrafY7IbtY5XI3rG4HTkZ5/Kuel8U6QmSujwPjgkBf/AImuY8Y6l5/iO8YM+2JvJVWPTaMHHtuyf171kI/mRknP064+lZ4LKqPsIud7tX3a39GfL5flOHjhYc6fM1d+9Jb67JpHaS+NbHIA0G2fBx80gGB/3xSweL9P+1wkeHrWN/MAWUMuVOeo+T3/AFrilZ424PHrTwuwFskNjII4I9P6V3RyzDJ7P/wKX+Z1PKsL2f8A4FL/ADPb3MMN41sskPnFfMZA678dMleuOnNY+p22nmWe4n0azuXXG6SSNSzYA4ztJzjH4VB4asoIU+0xK26RCXdwASTg8nOSeOpqbVYr793c2chWSVBDMC2EGMkMAcjIJbn3+lcNXLqSzP2ai1FpLd7tNrW99Unpc8aOFw0cz9nKL5bJfFL4nqtb32VrXMP+2dHs0+zy+GIo5ZceZFDAjKFz/EcDPrjBqPQvGGlPrSWa6LDYNKxhWaEDlsjAI2AgE/rjI6kXYdHMJJlyTnLbmyST3JP0rz3XbaTTNenMTSRtv86Ns8g9c59civSxeR0FSdk1fzl+rPVq5PhakHBJq6/ml+rOu8R+MdS0TxbLHprm3EKqsuDuWclQ3zofl4BwOM+/QDpfDHxQtdXvorLUohDdzvtSWM5RmPrnlcn61wPxFRJ7nTNYhRUhv7UMAQA5IwctjvtZR1PT2Fc1oG5/EemgZJN1FwP94Vz4BJ4WFui/Fb/idOUyX1Oml0VvmtH+J9AeJoEa9t5HI5jIx6kH/wCvWOttkDJVU6DH+ea6bxDAzi3cLnDMMk8dv8KxYrcscAbj6CuPEO02fVYZXpojit16RgFsdxk1bitzH8h2k9cknAFWIrZgvL4PTavT8av29mXOSCR7GuOU7HWolJbQMcbgx6bmz+grzj4t6f5D6VcjlXWSPOPQg/1r2KOz3EALjHpXCfGTTgfDVlc5JMVztJHYMp/wFaYOr+/Rhi0nSaPLPBcnk+OtHZjw86r/AN9fL/WvpiO1G0AdOvJr5f0djbeIdLucH91cRufoGBr67gsWfHBxW+Zxcpx5TiwtX2cHczktQehOParMVjnAAGK2UsEjXdKwVR6mkF3Gp2WcJmf+92H41yRwjWtR2HLFN/CV4dLOMsAq+ppzXNnat5cCG5m/uoM1ObG4u+buYhf+eacD8atw28Fqm2JAortp4a2sVbze/wBxzSq3+J3MxrXU9R4mmFrB/cj5Y/jVmz0ew08744gZT1kflj+NW3mA71VkuPerlKlT13fmTzTkrLRFp5go4r5M+NFt9n+JWouB8sojlH4oP6g19QST+9fO/wAebfHiixugPlltApPurN/QinhsS51rMcqXLBsofBK/Fn8RPKzgXVrJH/J//Za+jZJ+etfJngC9Nh4+0SccZuFjP0b5f619SPJx1rnzRuNRNdUb4SPNE2VO5Vb1GaWoLR99rGfbFTV103eKZMlZ2KlymH3etQ+1XLhd0X0NU67IPQwmtQpynBzSUH2rQzZNmlpgOQDS1ZA6ikzRTAdnmlzTM04GmIdUch2vEffB/KnZqO44hLd1Ib8jQwJTSxfe60080sX36ALm7Heik3e9FFgPkH7FK7i0QKZSdmM8Z6Vv+GdKutMguIrkKAzgrhs9qzopxFqSXDBsCTcQOvWt0a9bbc+XLj8P8aI2WpMrmltPvSAc89KoDW4B92OU/l/jSnXIGPKS5/D/ABrTmRHKy7gYo74/kaof2zbn/llL09v8ab/bUGeI5OfYUcyDlZo4rVtpTJbpknK8GuX/ALag/wCeUv4YqzbeIoIg26GUgnPGKmTRcVY6YHbzTTjg5NYZ8UWgJxBPgew/xpp8U2p62836f41maaG9nPPHA4qvfnNq3BGcDge9ZH/CU2o4+zzHn2/xqrd+LLR4Sgt58nvxj+dJ7DTRVumA8VaIRwPtUfB7/OtfQiivmuPVI9T8U6QY45ECXUfDYyfnH+FfSq4xWUi0LTT0p3WmE0ITISOaaRT2xTDVmZh69rVtoBtbu6XdGztH94DBK57/AEqlF8UNDjHzYP8A22Wuim0qw1bEWoWcF1Enzqk0YcA+uD35p0fg7w4vI0LTRz/z6p/hWkU+hLaOfb4saEBwE/Gcf4U0fF3Q8dIv+/5/+Jrq08KeH16aJpo/7dU/wqwnhzQ1+7o9gP8At1T/AAp+93FocUfjBoi/8+/4zt/8RTf+FwaOc7Ra4/67t/8AEV3q6HpI6aXZD/t3T/CpF0rTl6WFqPpCv+FHvdw0PPj8XtKAzi0A/wCu7/8AxFXdK+JC6/efYtMs4Li42F9izMPlHfJUCu4XT7NTxaQD6RrVa80+1S/srgeXblC4LKFXOV6cilaXcfMuxkjU/ETjcuhREZx/x8ilGoeJP+gJAMf9PA/xrZglsLSFIPt0bBc4aScFjk55OfekbUNNQkNfWw+sy/40/mL5GWl/4j6nSYAP+ugP/s1WBfa+3/MPgH4g/wDs9WG1XSBwdTsgPe4X/Go21vQgQTq2nj63Cf41Lj5jT8hv2zX8D/QoB+A/+LoNxrxX/j2tgfcf/Z01vEXh8ZB1jTh/23T/ABqI+JvDakZ1nTx/23X/ABpci7j5vIrjUNbe/ntNlorxIjndwCG3Yx83+yaZJFrch+Z7LH+9VNfFHhmLxRc3DapZ+XJZxLvVsglXk447/NV5vHHhRef7XtvwVj/Sp5E92VztdCD7FqvGZLLr/wA9P/rUybTdVlxiayX/ALaE/wDstTN4+8KL/wAxWL8I3/wpp+IXhQY/4mYJ9oX/AMKn2MO5XtZdiqNJ1cN/x82eenDf/Y1NHpOr7cfarVfxz/7LSN8RfCw/5f3P0gf/AAqNviR4XB4upj9IGo9jBh7WYT6NqYcIdRtgzgkADJOOv8NcdBP4vYF4vD98ynkMGiww9Rmusl+I/haQDzGmkx0Bt84/Ouq090m063mj4jkjVlBGMAjIpexj0D2sup5eG8bE5GgXynPeW3pWutetYZ7jXLCa2t4o2kUPJEzORycbPb1I6j8PVSAe5NeYfFbUPJ02aBHmUkJCCp4BJ3Hv0KjBrjx1NeyVNbyaX37/AIXPNzWrKeH9j/O1H73r+Fzwy9uHnnl8xmdmYlmYkknufemWkgK4YDK9eefSmTRsz7N3zgce/wBKr25ZbohgQcE4JxzXprQ7bGmPvA5PXp3x/n+tSnaUG0DAOeB3qszLsOfTAA6/56VYQBk3AD5sH35/yBVAepeFLnz/AA3DLHhpFjCk46sB3/StJHkntpPOUb1OQB0x/nNYXglpP+EVwd3+vcRseNwyOQe/fn2Nb6SpC6JPKhaQ7FJP3vb9K5nF1aOIxS+KE01/25GP5q6Pm6sHJV68d4zTX/bqX5q6KZVEUY5djnBJHGa4Xx3bjzre7AOG3Ic8e4r0DY6kq+0lRguABn/CuV8bW/m6M0g6xMHBH6/oa+hq8tWhzR1TV0e9CaklKOzMS/T+0/hdbzcNLplzsd5OTsJwFQ88fPHxx932FUfhnpjal4705cjbCxuGyeyjP88Vp+DU/tLTtc0TO9rm3LwiTmNWAI3fXJQ5A/h9hVD4WXS23xA0/ecCQPGPqVNfK4b3J1aXZ3+Utfzuc2X+5VrUe0rr0lr+dz6B1mES2yZA4cHnntWfb6eCgIOPr/hVrxDq8Gj6JcajcRSSQwBWZYwN2CwHGT71xUXxi8OIAfsOojntGn/xVceLoVZzvBH1OGrwhTtJnbR2Fx5saqU8n+LIOfwFakVqCAB07154PjZ4dU4ay1L/AL9p/wDFVOnxx8KqB/oWqE/9c0/+KrieDxD3iXLFx6M9HjtRkYP6VzXxQ0iS88BXZiTc0LpKMdgGwT+RNN0b4w+D75JGka7ttgyBNEPm9htJ/XFYvin416Pc6Pe6fY210pmhaMOUUjkY55rehhJQak9zlnWctOh4tZeTb3lu1wCY1ky6r6Zzwf8APWvsLT7/AO32NvPYwt5U0ausknHBGRXxXPfrM3yqVOc1794e+O/hnTvDOmWV3b6h9pt7WOGXy4VKllUAkHd04r1Z05NpnK2rWPYPsIkbdcyGU+h6flU48uJcKoA9q8mb9oTwnji21T/vyn/xdVn/AGgvCzf8umq/9+k/+LrNpw+CGpNm92evPcADrVaS5968kb4++GCOLTVP+/Sf/F1A3x48NN0tNT/79J/8VXJUjiZfZZrCMFuz1l7j3qs9xXlTfHTw2f8Alz1L/v2n/wAVTD8cfDh/5c9S/wC/af8AxVcksLiX9lnRGVJdT1F5Sa8d+Odt5tlpVyB9xpUJ+u0j+Rq8fjf4c/59NS/79p/8VXLeOviPofijRY7S2tr1Zkl3gyIoGMEdmPqK1wmGr060ZSjoOrUpOm0mea6fcmz1K0ul+9DMrj8CDX1sCXAK5IYZGK+QMjHGc54r2WD4yWEemWtu8d6ZUhRJPLRUXcFAP8W4jOe4ruzHDTrcvIjLB1owume46bkWpRuqsRjPPrVyuA+GHi6z8Tw6klrG8Zt2RmVkC/ez0wTn7veu+opQcIKMtxzalJtCMNyketZ7cEj0rRNUZ12ynFdNMxmMzRmm5pa3MSRD8o9uKdUaHqPen596tEC0uabmlzTAXNKKbQDTEPzSONyFT3GKTNLmmAkbbolPtUkX36hi+6V9GIqWP71IC727/lRTPxooA+Rr9mgtXdDh1AOce+K008OaoyA/b4BkZ/1f/wBas/VVLWEox1x/MV6bo+iXmrnZZwNKyLlgoHAoik9yZtrY4geGtVyP9Pgx/wBc6kXwvqZyPt0Of+uddnLaSW1wYZlKup2sGGCDXYeHvCA1W1M5k2KOBkZzVuMVqQpyPGj4Y1UjaL6ItjoIjSSeG9VTh76NSRxmGvTdf0VtJumi3dO4rm5cseSSSMc9aFBMOdnJHw7qRxjUoun/ADyo/wCEd1FsKdTiGD/zxH+NdRTT04HNPkQc7MQeDNWbj+1ouf8ApjSr4L1I5J1iPH/XD/69dfav5lsrZ5HymrKqAefxrJqxutUcR/whF+TzrSfhb/8A16gufA12kRdtYDdAR9mx3+td6RkHJqvf/wDHo7HHBXp9RUvYpJHl2n2b2Hi+0tnm80xXcPzBduckHpX1Gn3a+aJDjx7EAf8Al7gPH/Aa+l0GFxWUhrcWmNUn4UxqSBkLVGalYZqM9a1RmypeaqNIRJmhaXe2zCkDHBOf0qr/AMJsM4+wPwf+eo/wqv4oP+gQHOP34/8AQWrmM4NbQ2MpvU7EeOcf8w5vxm/+tSnx4QONO/Ob/wCxrjN36UZ5qrEczOy/4TyX/oGoP+2x/wDiaafHtwemnxf9/j/hXIZ7UmcZxSsHMzsP+E8uv4bCHP8A10P+FYXifW7rxRaW+lywxQRSzZZ0YlhgE9/cVmhj1xSBv9MtDnpJ/Q02guzGm8H28DhTcytnocD/AApo8KWn/PaYfiv+FdRqX30z3HFUWB42nFS4RKU5GMPCtiOss5Ge7D/CpU8KaaQctOfX560DuHXtU8IcnOflFDigUmZq+E9NH/Pb/v5Tv+EU0zukn/fw1rBvan7vxqeVFczMD/hHdOTVGTy5DH9lD7DIcZ3kZpl5ommxQ747fGD/AHzz+ta7/wDIUTr81o4/KRf8aiuwGt3GOgoUVYHJnHvDaxaoI3jAi2FsFjx+tVjEsqTyQn5I8kHPJP8AhVjVsGeGJSN0rhfwHWoYAFTUo+eCePbFTYq4lyI10yGZVwxIyamv4UW2RoUA+YE49O9U5X3eGYzjkAd8d6dqdyP7NjXPzMoOAecUWQXLcUSTzRymMLCXAUD+IfSthfF2vwaxLYJqtwttEgEaKQAoHHXH+cVmtIFtYiP9k5HTpUUcS3XifUeoaJjjae2TU3SKWpq6j4r8QRS2uzWrxVZ8MBJ15FQeP76aQ2NpPNJJOsXmyszEhzwoPPU8N19ferdrpUF3e2u47tkgYAnggckY/Cue8bTbtfuCWYokajDHpjjA9uv51wzkqmLhFfZTf6L9Tza8XPHU4fypy+/3V+py06wKm+Utnrgd+tVIG3yljkKBwPTmnCGS5fcx2qTwM/5/zikiOy9ZSQflxx2r0T0SxJwCSQM+n+f8/jUtqU8l1IHyHnp0PT+tV3bIyTx0pLeUJc4yMMME46H/AD/OgD1nQX8jwnpiSkxvIzsA55ILMc89QQR+YqS6Jkuk8vP7p1YcZ5z2FN3x29hpNvKo+0xWakoV+ZOFHI6jkH/vk+lSK6WtqJJR87kcH1z/AJ/Kt8kip4Rt/alJ/e2eTlqUqDm/tSk/vkzQvvO85GiPyFc+meg/w/Oue8SRyPpMkbqAzoTzmuoUOtovmN8yMQx/Mf1rjfE73eomWGBzGiA8j/P61rlsnHCOjLV024fdt+FgyxtUfZPeDcfu2/Cxyngu9Nn4msZC0u2RzC4Q9dwwAeem4g/h61dstKktfizDZwIiFNQWVFiGFEZO8DtjCnGK5S3d7WY7HKSxtkMpwVI7givVo1aT4keGtYtIFMOoQcjGHGFJLNjjIVlHU/dx6V4lX93jIv8AmTXzWq/C5rL93mEJdJxa+cdV+DZ6Xq+kLrmkXWmNJ5YuY/L37d23nrjvXJWfwAgbBbWJGX1NuAP516FaSeXcxsAmQf4/u/jXRGxubs5vbsCP/nlDwD9T1NTWc+ZcrPbi0lqeR6n8GfDyWBtbbULqbUcgL5SKR75Hb86q2H7OqzDfe63JED/AkIJ/PNe4Qx2tmmyCNUA9BStcj1rNVfZ6yl8kJvm0SPIW/Z40lR8mvXyt6mJDVG8/Z7sY7aV08QXTOqEqDAuCQOO9eyvc+9VpbjIIJrCpjrfCVGnJ7nxReWhtH2k7s5HTHNVK3/E0DQ6tdQlceVPIn6//AFqwK9enLmimYTVnYKKuWWlX+pFhYWc90y8ssEZcqPUgdKvr4P8AEj/d0DUzjr/oj/4VTkluxKLfQxKK1R4Z10w+d/Y2oeV03/Znx+eKkh8JeIrkZh0PUXHqts/+FLnj3Dkl2MWiug/4QbxT/wBC/qX/AIDN/hSHwP4oH/Mv6l/4Dt/hR7SHdB7OfYwKK3x4I8UH/mAah/34b/Cqmo+HNX0iOOTUdPuLVJCQhlQruIGcChVIN2TBwkldozeNo9e9eg+CPh1aeLNCfUJNSlt3SdoTGsQYcBTnOfevPa9d+DV/ix1WzJHyyRyqPqCD/IVlipSjTcobm2FjGVRKR3nw28FQ+D9VvDBqElwt1CAVeMKAVOQevua9KNcjo9x/xOIRng5H6GuuNcdGpKavLc6q1OMJWjsJVe6T5Qw7cVYpsg3REV0QdmYS1RnUvalIwcUldJziqcOfcf5/nUlQj/WKfXI/z+VSZq0SxSeCf5VF9pG3Ijk64xjmlcOSNjgDuMdaRVmz80ufotUIT7V/0yk/KnifrhGOBke9NCS7T+9+bscdqNkvH74gd8DrT0ETRvvXO0r2wafn3qsI5f8AnufyqcUAIvErj1ANTR/eqAnEy+4IqxF96gCzz7/nRS4NFAHyZqeBbMeOWGB75r1Lw94kvNBLSWhTDqA6uuQa8q1U/uOn8YHI969F0yMS3cELMQkjrG5HoTShs7kVHbUuXepTahqEt3cEGWRtzYGBXRaF4vn0qAwoEdD/AHvWuV1RTBqVxAz7zDI0QPspxVfzCp+la6NGSfU2dY1iTVJ2llYFmPUVjOc5Oc+vfNG8txnmkKHFNARHjOSMU0jGeR709l2555FRnb+ZpjLenybZjGeQ3861dmckdq59WKuGBwQc10ELiSFZB0YA+lY1F1NqTvoG09+lVrxT9lfPIBB/WrrDI9AKq3mBZy8HgZ/WsW9DdLU82m2/8J3ER/z9w/zWvphelfM84K+O4iwAJuYen/Af/wBdfTKjNZsFuwNMPtUhFMamhMiYVEamYVG1aIzZg+KB/wAS6IjtMP8A0Fq5Mg9zXYeJE36YPaUH+dcps44raD0Mp7kWCOp/CjtyfepPLycU7yec8YNVcixAR70uD3qXyiTjrU39m3h6Wk/PfyzRcLWKncc0Y/0i35P+tX9eKujTrvvbSge6kUyS0eKS38yMoWlTGf8AeFDulcUWm7FjUVDMhyeh6VTaONR8zYz7962NTjRBH9D/AErJeNJTg9BU3uy7WRWzFu+8PwNWYWhKYL/OTwKhNtGCOf1qzBbwEb+cjkc0MEP8lSM80ohQ8f1xTwQBQW5PPFRqXoVVhRtcsEb7jpKhx9UNdp/whumSIys9xzwcsP8ACuNLhdX05x2kkGT/ALgP9K9YTBwRuyevFNDsjj/+Fc+HJZkkeOeSVT8rGVhj8hXFnwzp9v8AFWbRWiJsbiNG2bzk5Q9+vUV7INolChjya818QMtv8bdKcnHnW0a/U7mFDGbJ+HfhdIPs66fmIdVMzkfzq5H8PvCroA2kQMD13lmP4c/pW4EAJ+Xdz3GanjIWUKMBQfaiwz571q3htdSvbaAYWCZ41X0CsQP0qjDKLbxNqEvdolf65Uf41u+Ldo8Z6nBty3nscgYznn+tc1qj+RrrN8oElvH36/KP8K5ZO90XFW1O08PXdvN5kYQrcAZOe6+3pz/n0858c39vPrUv2NmCsBvJP3jzyPTr/npXb+Fz/wATGQA8eSf5ivNtUkt4L2Z5FLzs5OCMnr19q87B0UsbUnd7L8f+GPGoYeMczr1Lu9o9e9/8tOxnxQEr5s8hEYGcA1Fa/vLhivBY4A696iuLmWY4Y4XrtHSprFVO5n4UcHivaPVLMoDdDnPXn60lvAZJkSJXeVyAFQbic9h79sUjuZGIToegxnitjw1bO+u6adrs32hGAAycBgSfy5P0qKkuSDl2RnVn7OnKfZNnealNEfGzxhwWFjGCAR8pDscH3wwP41O4S5uFRgfLQc471zWqzL/wn18Y3jO5VXOc8hVBH14IrpNKxJy/CL1z3/8ArV3ZFHlwdOPkjhy+HJg6S/ur8Vc3reUTxgNwXQZ+uK568uFLPEI8uSR0PFXrYXP2hJ04ibBCBeg7/mKg1+a10mOW5cZeQ/IgOSSevWnd0swmnoqkb/OOj/Br7jOC9ljZR6TV/nHR/g19x5PqkMdjr8sb7mjDh3CttODyQDg4/I/Q17j4XlsNXs7CXS9hgtwDGH5aHaMYPOcgHHXkHrg5rwG/uJLjUJpJiN7NzgcV6D8G9QWHxRPaMwAuLYhR6spB/lmvIx+DjiJxk21yu+hvisvjip05Sk1yO+n9fie4kds4rdh1DzLeNwfvKD+lYfSorK5Ig8sn7jun4BiB/KuDMW4xTR7+GgptpnRfas96Y1x71mrMfWlEjHpzXiupI7PYJF0zk96iaTOeap3V5bWMXmXlxHAnq7Yz9B3rm5fH2nSXQtNMja5nIOGf5E4BP17e1ONOpPZBaKPGfiFb/Z/Euqp6XLOBj+9z/WuGPWu08ZahLqurT308aI9wqvtQYHAx/T+dcYww5HvX1OGuqaTPKr/Gzt/hj4hm0DXbowwrKbi3KYPYhgc9frXvmk32oazpVtcSGKNZQQwK5KspweOnPXp3r5m8Isw8S2qq2C+5M/VTX0F4SlltoHgaZnjB3KCOAT1x+QrizCmr8y3OvCaqx1S6fAHEkxadx0aQ5x9B0FTl1UYUAD2quZ+M5qF5h615d2zuVO+5YaXNRNL157VUe4A/GojcjcBz1oUWaKKRZaWvOvi7H5/hi2lAJMV2ucehVh/hXaPc+1cn4/H2rwdfr/FGFkHttYH+Wa6cNeNSLM66TptHz+PvjPrXdfCq6MPiW4hJ4mtmGPUgg/41xEw2zuPet/wNc/ZvGNi+cBmZPzUivcrLmptHiUXy1Ez32zn8rUbaQnpKufpmvRDXkpuipBHUcgivV4JRPaxyDo6Bh+IzXnYdWuj0cRrZi0Gl7U2ulHMynIu1yMUyp7oYYH2qvniumOxzyVmB4wfcU8/rUbHKHHpT85/GrRDFopOpoqkIWlptFMB+aBTaKYhJGxsb0YZqzGPmqtLzEw74qxCd20jvzSAt8ep/Oik5opiPky+Ut5SDnMyDP4iu+jZYLhGVidjBs4x0rhLgZubUcfNcp/Ou2yu7qeOaUNUyaiuX9Yniu9ZvLm2JaKWQupxjryf1zVLOB7U5GRT2xnrimk/Odv4GtForGSVkXbOONpl3kAd812XinRdG0/SLaWynDTvj5QwO4Y+97VwKy4OQcH1qR7h3ABNDV2NEUmMkCoiDTy2ccCmtyTVAMOc9a19Lm3QtGf4TkD61knnr296sWMwiulJOFPyn8amaui4OzN0gkZqpfMPsUpIz8p6Cr4XnpTZ7KKe2mLwvIQpJxIVyMdK5JOyOxRuzy2+AXxzFz/y8Q5/8dr6ZT7tfN+utF/wsGPyYBAvnQfIGZv7ueSa+kEHy1N7pC2bHEVGwqWmMKcSWQMuOaiOanYcVGRWiM2Y3iH/kFtxyHU1yRPHvXb6paSXlm0EWN7EYycd6xV8Mag3eED/eNVFkuDexhqzc9OO2Pejk8nt7Vvr4Tvu8kI/4Ef8ACnjwje85nhH4n/Crug9nLsc6Dgjn8cV2qR3kuoBI0/dGPd/qzjp13eue1Zw8HXPe5iH/AAE12kKmKGJDkkIBkH0FS0pMTg7anLTWV/v2lHC/7tZOtxvGbIyRsreb0YH+8teiA56A/nWJ4o06O8igkdnUwhmGPwPP5V0Trc0OWxjTw3LPmTOW1wbVjOM9awjJzwK7fUdJiuAm93A9jWcPDNlwTO655wzgf0rFWS1NZRZzJORmkDkDFdO/h20BUrISufmy56flUkHhy0mfYiMx6/eIrVU21foZ3V7XOU3tyeKPMboTxXaf8Ich/wCWKj6ymnL4PiAx5EZ/7aGlyruVys4aR/8ASdPJPS4I/ON/8K9aieZlBVUxgYyT6Vwerabb2d5pypGqkXqBgD7MP6118L6m4iMUdsYSowXY5xj0FZNamqWhYdr1pMjygO33q858dubX4o+FbgoN7iJGPbiTn+degyf2sCAhtFUDJ+Vq5vxPYi51LTbvUIraWeJv9GlVWGw5B6H8Klqy3HGN2da7bZHGO9PhVT874Az6dazmtdXJBF7bA9/3Wf61KkWqBCkmoQkEYAWAZBPfrSc2uhXL5nkXj6MJ49v3EbAMFOSOuUFcj4hRvtVo0YO37Op5PoWHWvYPEmlKb15LqOOe42jMpiAJ44rjDapLaQFoo3G0qd45+8fX61yc3vM2VPRFfwt/yEHPbyT0+q15c/2kytLPImScsxUE/nXt9hbrC4IjRSY+oH0rxq5tWd0Qqdi89OprLCL/AGmr6R/U8iCtmFf0h/7cZLmOV2ZITjrnNTWqgwkFeR2qa4McWFVMM3AHTP8AhVW0kG7npntXpHcXUi3tluFHbmuo8Gxu/im1ZVLKgdmIHQbCAT7Z4/GuZWQ5HIHrzzXY+AMx6tdTNlLeO1be54RfmXqeg4B/L2rkzCXLhKj8mvv0PPzSbjgqr8mvv0MxYLa88V6nLJKvk+c7I6v1y57jqMfhzXY2uoWUY8tP3i8Zz0Iry26mMVxFiWRAwPzIMnH4/T+dSLeSEj/S7tsYY8hcc+3+Ir1MNifq9NQS2OuFFKKjfY9fg1OMxMI1BdQQqDqeeBWX4htY73SrO+1GeCxdMq/mAsDnoOxzwTj3NcjBdLdxX0UUMMU0do00EkckxYsrDpudlPBPYVY8Pm51bwv4g0e6E0khj+0wEAl3YAHGeQeVTjryfw4cwx3M6da2sJfg9H+d/kcmZUvZKniF9iSv6PR/nf5GBrNppjLNNaXM0kqEctGEUj6ZJqt4Y1f+xPEdhqGDthmVmPP3OjcfQms6WzurdDvUgdwDmqynBreclN3segtD6+VlcB0bKsMqQeoqpDC/n3HZBLnJ4HIB/rXPfDbWDrHgiyZ23S2ubWQ5/u4x/wCOla6KTTDeXJd53WEY+RT3rycfBOnqehg5WnoOl1OytMBnMrj+GMf1rH1vWNX/ALLluLOE2kUfJbGSRnHU/XtXSWunWVp/q4V3f3iMmlv/ACp7WSGRFZHXBUjINeRHki9Eeg7yPF7lri/ZmnmkmlbnLEmiy02a3uEuB8vlnP510k1rHbzsqoFAPYVXlT/R5B0Ow/yr0o2toYNanA+M9O/s+5gQEsHhyM/U1xEn3zjp9c16d48QTWOlXGMlkKk/l/jXmcow1d+Hd4Hn4hWmXvD83keIdPk6YuEB+hODX0RpT+WT79a+aYZDFPHIDgowYfga+jrKUMqMucEZrHGK6R0YF7nQi5+X9Ka7SFN+1ivrWZI87QObbaZQMqG5B9qrzXOoRTsUZXi/ikfapAyC23nkben19q8zkPTuaLzYqLz8HI9c1zn2zU3QjcqlWIdyyYQbup9DjH5niqzX2pkbkmh27QTudMZ4zjn2P51oqZPMdNNJh26YyaytbT7Vot9bkZ8y3dR9SpotJp3iLTtuJY7G45XjH3eO9PZ+CDjB6mqirMmTurHzzcD95n+8Aat6RP8AZNbsLg/dSZGP0Dc1FfwmC4aI9Y2ZD+DGq+ThG9OP8/nXtbo8LaR7y74LDivVfDk/2jw9YyZz+6C/988f0rxq2u/tNlBP2kjV/wAxmvUvAtz5/hxUz/qpnT/2b/2avNp6SselUd4pnSk03vSnrSZroRzsjuBujPtVI1fPIxVFhtcitoPoYzQn8qRP9WueuMUdKF6Eeh/+vWqMmOBozmk6VBfSyQWNxLCFMqRMyBuhIBIzVJCLPWivLdM+I2rX2qWlrJHZxxzTJGzCNsgMwBI+b3pNR+ImsWmoXEEKWzRJIQjPAyllzwSM8Vt7CV7GXtY2uepg0uawfCetTa7oMd5OIxNvZHCDAyDx+hFbmahqzsWndXQ/tT7PlFH93j8qizT7PiSRfRs/mKkZocep/OijNFMR8oTHN9ZdcG5Tj8a9AtNGv72wuL2C2LW1sMyyFlAXjPc8n2HtXn/LanYD/p4Xr+Ndl9ol8ryvMcRE7igb5SfXHrRT2JmPEgB5z+Bp7yRY+VfxNV8+5pygNjLY59K0IQu4c5P0FAOSf0p7xQiMsswLZHHrUWOnfNFwasakWiX1xo8mqxW++0iYrI6sCV+q5yBz1rMZievT0NSRuRGU8xlRvvAHg49R3qAnk8UIBc96aDzmgnOffpxQOnrTA6yxbz7WObrlRnjv3q1LC4tZOCTtPFZfhmUOZbdsZ++ue/rXWG1Zrdz32k8968yvLkbR6tCPPFM8S8Qqw8fx7+SJYePT7tfSUf3RXzv4wjCfEZevMkBOf+A19EJyopxd4pmM1abHGmMKkpjVaM2QsKjNStUZHNaGbCJd06D3q75APSoLRVa6QNk9f5VqiNfTms5T5Wb09ikIKd5OOc1daEK2MUnlAnpS9oaaFPyhz0NSbeg54FT+UewrB1M3iarDDBbPJE7YdxkBen4CnGq72SuZVUrXNkAZ5IH5VS1kB7NuQSFPQ+1Yt7Y6gQfIEhJ9z/jUlhbXccdwtzC6jy/lZh1613Sp/uue5xU6zdVQ5S7crmNDjrTWsZ7iG3aBGPOCQQMc9TnqB6Vbdd8EWeeB1+lXrIAWKfUiuVWnozrqRsjIu9HuicxkkYPRv6UabZXdrckzRERlcbye+a6AAYzgmknH7gY4wwP612Ks1DkONUI8/MZ4Y5OWz+P/ANekVv3v3xge9RNG/msFiyO3NKsZD8xAe/Wsrm5yPiRt2orwPkvYWH4uB/Wussiws4WCj7gHNct4gTfqU6DORJA//kRP/r11FkN1pEpz8pYH8CRWEKl5tG04WgmWJGJX5tq9utc94rC/Z7CQZO2bqO1dFMBtUbiD14rD8TjboyPgnZMpyaK/8Nio/GjYkHKtnqKVSO5JP0pqSZhiYH7yA4xntUiyMccH8q0WxD3MXxFb5uVI6MgzmuH0+082Ex5VNs0ik9hzXpmpQiYRsUPAxXIafpRS8vYJFfidmUgdiAa8itV5askehSV4RM5YXSMfLxjsK8h1iayk1WeXTonjtAfkTHBPcqMfKCeg/l0Hv76WpyCrnj0r5/v7B7LWrvT1Ry0U7xBdpJJ3YGBSwFOHt51VJ3fT+v6R5M8rjQxc8UpNufS+i/rp21Obvomb94eGHPHaqcGNwGB9a7O+8E+JHtjIdCv8Bd3MDA/ljNcWqFXZX3KwOCCMEV690zWzW5qxEBRx15xnPWux8MMIfDPiCec7IWtwm9uFLbXG0H15XjryPWuJikLJgHdnHHWuy0+SMfDTUld8F7tVAJwSf3ZwM9eAT+BrjzDWkod5RX4o8zNFzUYw/mlFf+TI5RIYb2VWdXkAG3g4z/nNa0ekadlWMWB1wT2qjYIgPAxnJ2jHT/CttY/NQgMVOOAPX8K6mz1UjTkvbtU+zvd3bxYx5bXLtH/3yTj9K0vDF5ax3rxzbluZBtRyflPT5fz6Z/TvjmJXyShJPOC/4/4flWjp1ultcxTyAJtZW4ycAHmuLFUFWoyh3XQ58woLEYWdJ31XQzfHFpaT6kIreJkmc5mOMA45yB75GfXr654PVNPNm+R90+2P8969I8UvJb65MI4fMnmRSnoq4xn25B/KuTvNEuDvlmmyzDLKBwKvANfVqduyMcuS+p0mnf3V1v0Oi+DGsm11270l2/d3kW+MH/nonP6qW/IV7gr7Dj8a+W/C1/8A2P4w027LYWG6UOf9knDfoTX1BKrbhtBOOtLGxvTZ6+DdqiJjNVO7n+UilVsts7+g61T1ENHjIIyO4rxYxuz2NEYF+v8ApDEd+azmZclSRgiqPjqGe40+A26yOwkBIQE8YNcGdPv2PNpck4Iz5Zr1KVNON2zgrVHGVkjpvFSibwnZSDnypivHpz/hXmFwCHOeuSK9Jkgl/wCFfzwyIyPBPnawIPUf415zdriVuO/pXTh9Lo5MRq0yuOte+6BN52iafJuyWt48n/gIrw2x0u81FZjaQGXyV3yYIGB6816/4LuRJ4T08kklVKnPbDEfyFLFK8S8I7SZ18Mqg4LbcjhsdKzbnS7OTKea7qSxKgAKNxBI6egx9KkMp3Ecd6haQkYzxXn2PS5hLe2jtCxV2bcAo3Y4GSf5sarfYIEZSryAqu0cjuqr6eiipS46Z/GoJLuCPO+VPlxnn1qrMlyRPGRBBHEmSqKFGfT8Ka0hPas281mG2G0CSR1baVUd8Zqva6ubi8MDRlBk7TnOSOtPlZLmtjzTxTAIdfvlHTzy34N839ax8/useh/z/Kuq8dweXrTyY/1sSPn6ZX+grlFztYfjXqU3eCPIqq02ep+GrgzeHLJs5ITafwJH9K9V+HFxut9QtyeVdHH4gj+grxzwTJ5vh8rn/VzMv8j/AFr1H4ey+XrdxF/z0gJ/Ij/E1xSVqjO6LvTR6UaaaU+9HatTFjaqXC4fPrVuobhcpn0q4PUiSuipnpR0c++DRQ3DDryK6EYMXNIQGUg8gjBpKUc1SEeH6ZAbHVtUiiRXvbaOQWwZc/OrgEgHvsDEVZuUbxPZ3GpeTHb3UcuNz3B2Sg5ZgPMY4K9cDjB6dKv67LY6d40vUu7ZEXeJY5o1Yybmw394DuenoKiu/EOk6jsN4biYouFDxbsHPPVu/HavQTbszjaWx0Pwuud2m31r18uVZAP94Y/9lrvhXmngTU7d/EtxbW0SR28sGUxGFYkY64+pr0oGuWt8ZvT+EdT7ckXZHZk/kf8A69R0qHFxC2eMkH8ayNDSz7Gil/H9KKYj5Qj/AOQnYAHrMDXX4O3jnPeub0a3S88UaVbv915+cH2NdpqWnS6bMELCSNslWPBqYTSfKOcG1zGfgjtSgE8YpRkf560fN7elbGNhxjdSoI5YZGDR5Tlgvc+lJluueKUs3Bz+dIAMQVjk5A9BVmKxVkkkFxD+7UPsYkM2ew46iq+5sYzS5OM5oGhmwk9COfSlWJifunrS5b19qkQtnqaTY0jQ0SKeLU7eVYmYBwGx3B4NeuJpDGLleo7CvM/D8LPdKWnKYIJx3r3CPBjUj0rgr0vbTtc7IVJUoadT5m+I1i1p8TFwpUFrc5656f5/CvdEHy4ryj4yOg+IGngfe8qHP/fbV6wg+WiMeWPL2Bvm1YtNan9qY3WtESyJhTDUjUw1Zkyzp0Ze8Tjpk1vLbIp3AfN25rI0hsXmPVTW9S5E3dj5miHycnOaPJFTUUeyiHMyEwc5zVG6jKTAetalZuogtIFBxwORVQgou6E5NkQVvX9abcKpgIdgM8ZNOUD1z9ap6u23T2Y9AwPH1p1NYsqHxIkkiWOJUYj5QM/lToLiFLcAzxqAe7Csu9vwLHzfUL39q5X7e0spTBCnPesKKd7mlWWlj0aK5gmfbHNG7DkhWBqrdazpsavG17biRGAZTIAQQea4XQdbNvqVwWjUKqFS2evINcF4kvhLr+opg5+0SqenPzGrdVqbiQoe7c9tOuaaF3f2jaAHpmQCoD4o0QcHV7Mn2kB/lXzlPFufirFkWjYKFGG6Z/pQ6zQKCPU5tTh1LxtfeRcia0+yK6Ffu7lZSf5V1ltqdtDcS2rT4kFzKuCpPViR2xXkfh6UwalcY4/0SYeuflJr1rRpd66hKDy12zfmq1z0m3VujeaXs9Qm8UaXG7AzztgcbLZz/SsPxX4js5/D8kMUV8ZS6MC1s6DqOpIxXX5Gc5O761l+LIRN4RviMlgobJ55DA12STcWmc8XZ3RFaavKtlAsmkamXSJVb9yAMge5q3HqszgFdIvcZ7mMY/8AHqvqS8aHsVB4p23K425HuaST7g2jS00edZCSWBo2JOUcgkfkSKjt4bdtUu/kXcdrdPbFRpdeSmzpxnHpWRZ6gY9emLNw8S8fQmvNxFeCqKLW25vTpycW0dSbWHn5BXDyeGNOtfH93rIt1NxKkewnoh2kMwHqcAf/AKzXUHVBWNreoCIJdoMSAbN2Rgdxn9fzprE0XJKCsVTpTT94i1C6ku9Pu1sfnuo9yqqkAhuw+bA9PauD8afC+fXvDcmqR6OIvEMcQdzHIo88j7wYAkE4zg9c45ruIblbuBFt3kMxO9cMMOp6qCehzyK6GC8kNpGJYz5m0ZDMOtdcXFahVT2PiqBlS4UTIxVWG9c7WxnkDrg9exr1fUbvQW8EQutqxsZBstoVXayyc9+dpBDZJznn72cHz7xtcpceOtbmTYFe9lOEBAzu9++c/jmuhlXzPhlo4A6XTHH0MtceY0lUlRbbXvLZ+Tf6HzebYdVKlC7a99LR+Tf36b+pkWcSqo3sNvoD37/59q1UeNIy5wFwFx/erNt0JxycZwwOMAVMJTNeiFQSkY3Ee9eie4ake4BWHBUKMdMVJdzAQnzZvKi/5ay55wOw/GqKmZVPlW7ykkhVVe/Pft/nms2bfc3my5xeXAOUs4SSie7kfyFJIbZ13iQ3F9Dp2oWk0dtDcwBnkZcuowCoH13Htx+NcrqP2i3sZ47KEHIO6aRg7OO+38q63UIrt/DOmSy+cs8RMTRWirtI5xkN6bRznHJ4rGZCQ0k1skUJxuCsZD+IUECuLAe7Q5P5W19zZ5mUaYb2f8rlH7pP9DzXcd+7PzZzmvsHwbexX1hZ3U6qwuLOOXkZ+YqpP8zXyRqcKQ6jMsUbRxbsorelfSHga6ZfAuizAn5bVVz9Mj+ldVf+G2j06SvLlPSo7PSoblrhYUEh9TkD6DoKwPG0kBsrYxKgPmEEgdsVTOqN/erJ1q8NxaqCfuvmvHjVlL3bI9OOH5XzNmLK25AT6VUYYPIzU27dHjnhjUTDJBPT2roiVIx9Sj36VrUOP4BJ+mf6V5HeAhj15ANevX9zb20t3FcybPtNoyqME7iMjHH1ryK6GfriuvDnFiehu+A4pbnW5LSKQRtNFjJ6HkcGuu8Auf7Entz1gunT+RrzTS786be+fs3jaVK5x196734dXfnnVQAVBlWULnON2f8AAVpXj7rZGHl76R3M8vlQvLjO0E4H0zXPHWZ2ViYhGwGf1H9DXQTpvtmA/iUis3TbWxuYwJwNzQPyGx8wXj+XSvPukrs661TkVznpJLu5gaSa4k2KQ2BxwWwR+FQ3KGN7gBiW3ujEnqAAwrV1FYYmuoLZt0DoWUZ6EDNZtwd8xP8Az0aNh77lwa1jqjNSurkQvN0xuW5w8bn8sGrtvIp1hMAgmQuM8cMo4qiYBDbRzKSfMj349CCv/wBerjS/aZ2uYFkwAjD5epBwabKRk/EGPM9pIBjdG6k/Qg/1NcIvU/SvRvHaLJYWkmcsspXrzhlP+ArzlMCQZ6ZrsoP3EceIXvnb+AJswX8OejIwH5g/yFeoeDZ/J8U2mTgOGQ/ipx/IV4/4GnMWrTxdpIT+YI/+vXo+iXxj1SwugrIFnRirDBA3DINc1dWqXOmi707HuNJmlIptWiGHemONykU6g1SJZnkYNMY8r9ammXDn3qCXhM+nNdCOeQ7NGeabmlzVEmdf+HtJ1O5+0XtjFNMAF3tnOKgHhPw+p/5BNr+KZrYpc1XM+4rIoWmh6VYTLPaafbwyqCA6IAQO9aOaaKM0O7Cw/PNDHam7+6Q35GkFOVd6smeoIoA1Ovaiq0VynlJkjO0ZooEfN3hNPM8caSOcb5CPwRq7/wAVjBg6jJPSuA8I3ENr4306adgqDzcn/gBFd/4tnSSS1EZ3oys3mD7tcyT9qjpuvZM5oZ6ClxzzzSd/agkggAcdzmu25xsUc+uK0NL0XUNXkZLG3MrJywBAx+ZrPXOR9K6nwd4gg0O9ka5jdo5EILKRkHtwetZ1JNK6IldbFO+8Jaxptobm6tDHCv3mLL/LNYpUivQPFXjSx1fS2tbaOUMSDliMVwDMcDGCM81MJt7ji27iDPtUkfGKh6il3FZkXI2kHPrVs0RtWlyYFyDyRXtlheq9hAxPWNT+lfPa3iB8Z4xnOa9csbmcWds4lXyRbgGPbyWwMHOfrxXn4qThaUTsow9orHlHxcufM+KEAB4EduP1Jr2tPuivnz4lTGX4kI3B5gGSfpX0HH90VUHzQTZMlyyaHUw0+mmtESyJqjNSt9aiNaIzZa05tt9F9cfpXR765e0bF3F/vCug3D1rOc+VjSuT76N49a5rWo9cluU/s24ijix827r/ACrNWw8Vsfn1WEewH/1qweId9ivZnb7/AHqKVlyCQCScZxXJxaV4i8wtJrClP7oz/hUMui60u1pNcdhnGBnqal4iVthqB2JZMfdH5Vx2pXDPPcRs7bQ5wueOtS2Wk6hb3iTzatJKi9UwcH8zWVqJd7q68uNydx7GnTquV7lKFitqOqQw2CxS5XCrk9hWBLP5beYozjnPrVnVLA3tqiNA0iuoRwQee1ZN1p99Gfs0PlKgACZfaqADGPXrWlOsti5UebUrLq6y3lykJHyxt5nQ8gZ59K57xBLGPE18nmJ5jTyMFzzjcc1fOmXa/aJI7ZVfaTK6TMMnHPb09aZqmhX73CubIPKHDs6yuxcdxkr34/Koc05XNFStGxgz3CQxhmcKMgfUmrdtZTsrT5IjHPJz09MVoRaDqFwwii0RncHcW3noPqOK3bHw1c2EUN9L+7aMEtBuPTIHzevXNKUr6LchwSV2ylo9pIWS6XmGWB9pIw2GQ9q9Q8LljpkzkZ3uj8e6LXBW7zS3KkkbCTnj2ruvBj/8So+nlwkZ/wBwf4VWHvzakT+E3uABkfrVLxA6jwrqe0jCwMx3elXow0jAbWqPU7b7ToN/E6kB4XGR24rvexzLcjs2M2n2kiZIeBGyD7VN5Mh9aqaIyDRrLMoJWBQcnrgYrSE0X/PRP++hUKS7lNMztXnnt4ENrCs1xsGImfbkZAJz9DmuflvAmpqe4XBNdBqnkSiKdHSR49yBlIOM4z/IVyOonFwJB1weSe/FeHiknXZ6VH+EjRuda+z2zTFXbaeQnJrMm8RSTxvGtleHj72zj86qPO3UNgAjgDiopZ2KDkAE9BWMaLTujSUn0NfQxp6awrTXRtnc7sN8i5HPVux9sV3iXEFwqPDPHIhyN0Z3D8xxXkgOXUtxzzn/ADzXdeDsNoSd9kjqR9cH+tenSldWZzVVfU+V/EL+d4m1SVQ2HvJmAI5++etdmhB+GGkHHH2lsf8AfUlclr1lMPEupxhGIW8mXp/tmuzVPJ+G+kqxA23Df+hSUYx60v8AEvyZ4GY/xKH+NflIxDILeEuSM9Rn0qC1l8q3MjqNzkk8YxVW6maZ1RTxwaLt1jss8jC44Xj8x/nmuw9QfawXWoRGYyOkR+XbEDlxk9+ntWtZ2WoIDBbZsomPO3lj26DgfiSfU1gaBC9wQPOnVRniKQg/p/nrXUQadbudrTvuXqWuTnP5/wBKuxJ09rYo3hfybtmnEEu9DIeR0646/eP5/Ss2T7MjlvIV5DyuwHA4/wA/lWnotksen38CXMz+dHlQZSxU/NyvoeR+OKpNayRKFS5kLFvvMASBzxwK4cKuWpWh2lf70medl75K1en2lf8A8CSf53+Zw3jGzu0kt7qbmNgUHsev+fpXtPgtDF8OtKHT/Rd36k1j+G7Cxvpr3TL6AXcNzECwm5+6ex6jqeR6V262kNppgtLaMRQRReXGg/hAHFdMtY2PVjpNMxJf3skUhkcGNiQFYgNwRgjuPao7hyyEdqbuJ9vrUchBQ89s15aWp67ehStYVhWVE3HdIXOWJ5PXr0+lE8McwUSqCFYOMnoR0NPU/M2O46UyQ5Axj8a1RmzlvGKN9osHUEk714Ga87vYyrvkENuIINexSZXULB84USFTz6ivMvFkJg17UIx0E5bjvnn+tdFGWtjkrx0ucxXb/DWXbq13F/eg3Y+jD/GuJPU10/gGbyvFMSf89Y3Tp7Z/pXTVV4M5qTtNHrv8A5yDXITW8yTSoXRCnEZZh82ST/WuvCMVwAfwFcze6c5nvnSKQzKQ6YB56Hj9a86J6VRXRnrLbx3OWuS4ERJVR1O3moRc2rS2ReGUlFADsQOM4BIpW0+4Mcf+iy5aNDnyz6EEdPpSXEF1NbwH7JMriPym3IRggg5/nWisYWHx3NxMNsMcaokLNjGSOo4/GopbiW5dY1kZUcoRg9Mg/wBRU8FrcwXoACKjM6MXYD5S2c9feqcNrPEw3GLgqQDKvY/X3NGg9Snq0Jl026mlDCaIRyDd9cGuLkG2ZgOzHFeh62tvcLcySSoi/Z2A/eLywzjvXns/+tJ9cGuqg9DlrrU1/DEnleJIOuGLL+YNekBiOc4I/nXlOny+Rq1pLnhZEJ/MV6rzn+R9azxC1TNcM/daPeLWcXNnBOOksauPxGalrI8Lz/aPC+nPnpCE/wC+fl/pWtmkthvcTNHFFGapEle4XgEVVcblI9a0JVBjNUDxxW0HoYzRGp3Ip9RTs1GmQuPQkfrTs1ojMd2zSk03PFHSmAtKDTc0vamgH5qSI81BmpYvvUxEUkEnmNjGMnFFaHHvRSA+XtIJbxDbZzwsnf2rq5nJwCenTNcjo2f7egYcDZJ/KupkJJ570QFJk1uQX2sO1WwkPcE1Rt8mTAJJ9quBHx0PtRJ6jhsPCw9AozS5h6hB1qPyz/kUCJiTkVNyrDw0Y528Ck85CpwoHtir+kvp9vcSNqdk91CUIVUkKEN2PFZSQhmdo1kALkgM2QF7Dp9alS1sDQrzhQTgD9Kwtcuynl4kKBvmDK3JI7VsXEWIW3AEY6Hmuc1JBJbmO4J/dMdh7gY4q7isVU1CSZVHzlRglh1217np91u0iDqP3CfhwK+d9Okc30KblVVHfOGxyK91sJHbTIDuxmJdwx7CuHGaxR24Nas8v8eSb/HqnPBeH+Qr6PjPyj6V82eNyP8AhNQcg/PD9Ogr6RU4UY9KqGkEZz+NkpPFMJzSM5xUZYgcGrTM2hzVEc0rSEDmmZzWiM2SQHFxGf8AaFbO+sOM4lT/AHhU97e30M4S2sTMpGd+4AZrCsm2jalaxpO/NIrc9awZtQ1vPyaSOneQf402K/15pFDaYqL3JOMfrXP7OVzXmVjoixAqvMHeLcASFYZx9arW89+3y3FrGqn+MP0/DvS3LHycA/xL39xUS0di4q+xKGyKltogwGcc1l3dxPa27TRQtcsSAIl42+p9axo/EXiGGWTyfD0syZ+TMoUD86yqJ2uVLayOmjso2icYHErf+hGvOkxqYuZbjaGErRELwCAQQe9bcuveJ0idrXRUZnYl1kmH7s9x1Gaz/Dem38cFy+rWiwzNcF0RHBBBAz0J71WFhFuS6vY5q7qRSa2KNxKLK3kjW1JikGHLEnHGOwqzYaeuv38Mt7Dsh27RsZgSoB5611KWVtdSRxTwqUJx0PH6Vu22mWkEjPFAqsqhSOe3P9a9mMacIaL3u55zdWctZaGLpmixQRyrYQrEJIgAz5PfJyc89Kh12GIz2iPbK+9zncOuQtdXEVVZAMALKRgDtxVWcLusXwMk9duT2rOjFQfM9WXNNqyZ5VKiQaj5UaqqbgdoHY1teF1jezj8yZ1WO0jOEkK5wSvOPpWD4ngnTxhIYrgRqsi/LtHAIGQDW94SUfZZCSM/Z2XOPSVhWcl7+p1Q+DQ6OIWRbmWTHf8Aev8A40j2FhcQvFG0pBDYUyvk8fWqxVtocznB/uirthsY489ieQVI68Vaa7oi0+xneH7K2fSoZTDCZGXDlmwTgmtY28UceRFbtgdMjP8AKsrwp9m0zw84kfyoYZJCzSHp82ev41P4n1cWujpHZOHur8iK32HOd3Vh+f6ily01G7RtFTnKyJ5reIWu5IUjIcfdGOOfYVyWsrtZDxjBGPzrqRYvpXhqC1kmaWSKNd8jtnLZ5x7c8e1cnrLZiDbs/NwMcf55rzMXG1ZNKx2UX+7avcz3xGjHcPXr0FZr6nagcSk8+h5q/wBVfIGOOBVNoYl6RR/gBTh5jlfobPgzUdPm182t15LxXETALMo2lhgjr+NeiXEunaRaMRJbWduDk4Kxr+fFeMXoT7HLtUABcjj3rkbtR50eEHT2rtpWcbHJUvzXIdYaO41vUJ0beHuZHDZzuBcnOfxrR1jKfDzTV/6eiMH/ALaVmvHhtwHNa2u8eAdNwf8Al57fSSsMYvepf4l+TPHzL46H+NflI4huZRjt2HFbnhuyh1XxVp1nNB5kZcvJG4yrKqliD9SKxGJ3dcjoOa7P4XWpn8R3d4R8lvbbRx0Z24/RWruPRMPVvD48MeJJbNlLWkn7yB3GQ6E9Pcjofpnoa3bSC1kIGyEtwNuBx+H+e1d14u0A6/opihXN7AfMtuxJ7rn/AGhx6ZA9K880u7geCMXOEcYUOwxn0zx1/wAKqLEdVocNvBekRxhGMZUED6H+lRyo0e5Acbf4Cc4I6fyqTTTsvYmzuXOOOeop19mK6k3sMFs9+/P+Fckfdxsl/NFP7m1+p5lP3Mxmv5op/c2v1QzSLr7JrVtIOcuEI9m4/wA/SvRyOMY+teXw7o7iF8FdsgJY8AYP/wCqvTuDg5610HrnKt8hYZ6ZFMY5BP61NcjF1MBx85x+dV8/l2rzmrM9VO6uQZw3cjHSmHBUenTFOP3sYHfrTTnviqRDKOoRg2qk9FkU5rz3xlCY9auBnOUQjP0HP6GvSL4ZsZeegziuG8dxf6fFJt/1luB+IJrSk/fMK6vE4RgQx7Vf0NpF1uz8p9jtKqhs4xnj+tUXGG5qWymNvfW84/5ZyKw/A5rueqOGOjPW/wCy9Tk+9dpj/fNZmoWlzbu0ct2AQnmDk/N7D34rs0bIIwvA4I7/AFrn/EKokkDmBJDznJ6KOv8AOvNW56koq1znre1lublYGnZSwPv0AP65qFLdmnMW6Uur7GwOO/OfwrTSe4ULOREiuzRqyjkYBA/lVaUXLrJK07cwiUKOO9aIxsVlsbuSL5I5NxxwxA4xz+tXU0CVgCzFeWB5B47dvzpskf2W/jYSOcCNgSeoJwf6V1GAV/Cpk7FRimcu3hdZ4Qk0pBJOQv6V55coUcBuo+X8uK9rIB6H9K8j8QQC31e7j9J3P4HkfzrfDybujDEwSSaM3ONhz0/xr1m2lE9rDN2kjVvzANeS5/dj2NeneH5DNoNmxwf3e38iR/SrxC0TIwz1aPZfh/P5vhry+P3M7p+eG/qa6g81wnw2nzDqEGejI/5gg/yFd1WUdjWe4UmaKDVkATkYqlINrEVdqtcDBzWkGRMpDiRx75p1I3E31X+X/wCunVrcxsFHeiimIKXvSUd6YDu1Sw/eqGpYT81O4FvPvRS/hRQI+W9GIOuQ47I38q6aQ5/OuRs5jFqSON2cHpXS/aSzJkjBGCD60oMU0XrN1SQsTwB0q99siGQH/SspXyDg8Y61ANQtHufs4uIzKDjbnv6VbimTFtG39rj7Mfpij7ZEBwW/Ks0nnOajaBCGPzZ9mNTyofOyx/bMW5gA5wSTjginwahCqksSN3IGfwrnZEaGZj8yPvB55AGf/r1FI+2YlmIAQnaPzP8An3qeVF8zNu51m2kcQNFJmQYDZHX/ACK5e+uZmvniVdhClTnv6VHJOGvRKSPLGTjdyP8ACptRWG6w8Mg80Ll8k5x7VNh3M9FaKRJYxuCLnIHfpXuOmyIdJthzuMCfngV474Udl8SWkbFvLdyjA9CCCMY9K9b09dlhbqOixqB9MVw4uWyPQwcd2ea+NjjxkG5OJIsfkP8AGvpKP/Vr9K+bfGpb/hL2AGQDEf0FfSMR/dr9BWkfgRhP42PNMNPPSmGqRDI2pARilaoSea1iZMc5wue4rYSVURGY5BH58VhTsfKYDOcHGPpU8EqNFFFcSNHInQE43Z//AF1FWxrSNSS+AKhFIyQMYxwaja+O0lDz1Ge/NQfZoGIPnvkAAYb0pCYIMwosspb+Ec4rK0WaaosmXM7qAMbQT9aguP8AUHPqD+tWVREBwMZ5P1qC6ZBbSfMOnrXJUfM9DeGgxSdhz60REsCB6muL8Xanf2F80dtdyRoUB2qcYrE8P65qL6wsE1/O6mJiQWOMgZzSlhZVIp3CVVR0O6aUh7oZ6Sn+QqbdlWrhNIvbqe4tmkuGbdI4cySEZ/xrqNR1VdKEe6CS4847QIeSD/8AXqKFF0sTFNirTU6DsjfsbU3EiBhgEk5I9PSt5E2mUe4/kK5XQdfa4vY7OWwntnVWdRKeSMVuXF/JFcI0cBZGX5jngHIx/X8q9d1Yt2ueZyNK9i3G37ySP+Ivu4H0qs2WNsoJB2tjI6HNZ0mrSC5minWKK4E/kIxclSWTcnQfQVXvdVvotF06RLeLz2nCOwyVXkj69RQqkW7XDldrnI+KYGHiKRud2FJ4q34Ob/WoVzhZxyPSXP8AWq+vzyT6lOZivmqxX5RjODVXRdWOkiWYQiT9/MhUtjrtNYynzSOmEeWJ3csalF2JHj1CD8qlslUXCnaoPrtFcqfHmxTjTU+U939PwqNfiJJvJGmwjJ4zIc0KCve5Tm7WsbejRQXOnXdtcwrNC1xKjIRkEZFV9K8Kw6brQvFnlmt4kK2tvLz5JPXB/l9axbDxBNaxBo4YsXN2xIbPy5GeKvy+KbyBj5cUPpyCf61H1qitJbotQqpPlejOm1WTzNMnACnC5OGBPUVwWpKvle24YyOPStQeI7u6geJ44ArqVbauCM1jaq5EJbndxjHbmufEVY1ZpxNaMJQg0ygCOfdRUbjiljKh26fMuRzSN+mO9NIq5VuVLWso/wBk9q5C4X97GSOi9uea7OQZRlz29a465BEqg5HBrponNWIymRgDOelX/EWR4D04cE/a/T/rpVXGY856egq54gyfAmne116enmVljPipf4l+TPFzH4qH+NflI4KcgL128V6h8Jrfbo2o3BXDS3CqM+iqcfqxrzOQYIx8oPbH+FevfD6PyfDcKngupYn1O8/0IrsuenY7BThl7civP/GmjLpHiL7XCitball9h6LKPvD8c7undvau9Lbsms7x/Ym98KtOoy9nIs4BH8PRv0Ofwqoks4TT3jiuI3hkeIq4ZomOA3Ocen5VuaruhvGcKSGQNuQDcvbv9K5+AieJd4AJGc4yCa3tUMpjsZQqPI8fzk5weh/qa5a3u4ulJdVJfk/0PMxHuY6jJfaUo/k1+TMh2jlk3oZX287s5x9R2+teoWd1HeWcU8TEoy/iD6GvO/OVR8saeYgz9Pxrp/CNz59tdg4VlkGUxjGR1/H+ldTVj1lqg1BduoTDPU5/SqecdDV/VRi/J5+ZQeKoHg/4V509JM9Sm7wRC3U56Z6A06WaS4kLytufGCQoHt2ppIJz7ZpMHJyRxSQMinG63lXqSh/lXG+NoxJY6XPjIKMvP0H+FdsQGUj14rkfFCb/AAtayd0m257/AMQq4aSRlVXus8ycfNTfSpJxtkb6moxXonm9T3zTJBcabaT4GZYVf81FUdZiVpbYuMrvKnPTkH/CjwjP5/hbTnPJEQT/AL5JH9KtavbST2beSN0qEMoPcivMekj1lrBHJzxyNDcKhXyEuAY8def/ANYogXzLe3/24JIjz6c1dj0u/SMxAx7HCs2TyGGKtppTKyYK7UlZx/usOlVzIz5Wc9NCywWcgdm/dljk9cEHFdahDIrD7pANZsWgmPiSdmC5ABHABBBH61qQxGC3SPJbYuAW9qTdyoxaG/jnmvM/GcHl63MR0dUf9Nv9K9Oxk8frXA+PYSt9by44khZf++Tn+taUHaZliF7hxQ+6RXovg+Tf4eQf885GX+v9a86Hce1dz4Fl3WF3F/ckDY+o/wDrV0117hy4d++esfDuXy9buIj0kgOPqGH+Jr0qvJfB03keKrTnAfch59VP9cV6ya5qb0OmotRKKKTNamQuaimGUzUlNIyKpbkszZuJIm9yP0pSaLv5Y84+6wP60mfWtkYsdSZpM0A1Qhc+9LmmZ96AaYiQGpYfv1ADUsJ+bigC5z6GilGcUUAfKEJJvFKnnH171tmZVUDPzdzWHG7LKGDDlcfqanRt8o3vhQfujv7VnF2ZUlobscsZWSR5BGqrt3E8dK5G0iMmqxQoVP7wDcO4Hf8ArWibuW81Mw2yh0SPBRh8vvxVO/SSGc3YxHLn7oGNpx2qnO7sSo2VzuenJqvLJJlo4yu7GRn0rn9N8QyXLiK6IDD+ID7w+nrV24vI45lkhJcr146itOZGfKQ6lqpEnkkDcBzj6VVt72MwJv6/Mm4+h6VnXbme6kmxuLf3uoFQ5bbnGcdCetZtmiRbtwryjcdx3DHvVm+NtFIqlgjgYBA/XNUtyJbLIuN+cHnpVSdi8nzMrHoCDSuFjV8NShfE9gBzmUD9a9ktBsgjX0XHFeKaE5i8QWGc4M6ZGe2RXtsQ6D3rz8Xuj08F8LPMPG5Y+K5MKSAYs+3Ar6Pi/wBUv0r518Z5Pie7PZRF/T+tfRUB/cp/uj+VbR+BHNP42PNMNPNNPFUiGRN0NQt1qZqhbg1qjJjWOFzWkqxzYEiKw7AjNZE5xCxHYVTh12aJVUqjEAfMe9c9eMm04nTQas7nSPbwxEGONVJ64qe2dWQgHkHBrmpNekl/hQe9NGu3CfMgQcY6VEoyaNdLHUTxeZbyJnG5SP0rgf7PETCQyudpDH0yDWv/AG/fMCC6c+iisy9lZ7eXkjKHI/CqopwTTIkr6i+IdDudc1AXNjJHJb+XsYgg/MCff3rC0/wrerrEgRv3ixuhyuByMcHoetb9pdywQmKJykefur0qaCR2uFG9sk9c1C54qyLcYy1Zzkfh+6tLmNGlXdbTbn9wcHitvVWJhtT6Txn9aqGaa5up12tvBAOM5OOKszx3E1rDEttMXDqS23jjBrkxKk2pM3pcqTSNhCqeNLVU6NAw/QVvbhIkojXG7bwPqR/SsM2lwfFOn3SpmJEKscgYO2tW0nKFARkOWGcjjDMf60qUrRMKquzB1S2uzc+ZHC7kXtpJ26BcE/hWvdwSyaZFF5ZDpcq+M9t7Gm6ncyqIZIolZGZd5LYxjpUr6nCxlDsBkADHPc/40nXswjRbs0cV4muY7TVbknkPKxBX61zsl4YIXAjLiS6PfGMxqa0vFIea7ItySobPNY1xGz223blhPGT7Zjx/SuqjJyjciceV2KV5qEiI/lqFOMjPPaneHrt9SeZJiA0UBkVgMZYEfpTbiyeUnEb5Axw3H8qb4SsZxfAk7EkiZc9Tj8fpW1S6ptozh8aTOuuYUh0rTnTPzTKzHPc0XnBJzUtzEJNAhXLDym4I9QapXcLkkm4lP4ivGg+Z6notWRNayEtx6U3UpmMLA8jHYdDVW0hO8ZeXA6fMal1B8xDOeAenf8a66cTGT0MlbhzcbfMbAzjn07U9rh2yM4NQn/j65IxjikPHHp613KKObmY4zNzlsfQ1z+opslVlHVOnvW536/SsfVgcRfQjjp1rWCsyJu6K8QTCKWYLxkgcj3/ya6XWrbRn8JKktwyWq/NBIhyxfnoO5OTkcd+mMjnFyqrnoRxVnXhjwLYdv9KPPp/rK4sxpuUqTUmve6ej/wAjwc2oupUoNSa99LT0bv66W+bOKZTu+7z/APW+n+fevXvBCldFhUjGIyP/AB1D/PNeSrBJNOsUYzI7bVA7seB/MV7do9gLCWW1AyIp3j9MjDD/ANlFdrmlJI91RumzUGMdjxWmI0urBoZFykibW/EVmgegHpWnbHNun0xW0TFnjsEDadqF1Yyg/uJdpyOdvYgd+K6q6+w/2OJZJCYEUbZOpHb+fGP5VF440yKLVre+ZcR3I8qQjsw5BrO0hfnm026O6KdPlOO4HbsDjn8BXn5lSk4xrRk1yO+nbr+B42cUJShHERk17N3aXbr80vluQwvHIh8sg5HXvzVvwjcvBr/kM2RKrRN9R8wP6GsGS3l0+ZivIViGH0P/AOutHR51h8SWc44SR1zj34rvk7pNHtwaa0Oz1obbiJjxlMfkf/r1mN1z3rZ1tcrA3oSP5Vj444Jrz62k2enQ1pojb6c5puPXNPYcdeaMY9R9alM0GAdPyzXNa9GD4Tvkxnyp8gf8CH+NdPjkcCsTUoPM0nW4epA38/TP9KqL1ImtDyC7GJW68881BVq8X5geOlVelelHY8p7nrvw8n83wqidfKmdf5N/WutkGWIxwa4P4Wy7tP1CD+5Kj/mCP/Za79lOR34rzq2k2epQ1popFAM8cVG3r+dWWXBxzTCpA5rO5rYgK4//AFU1hwP1FSle3p2pGGQeMfhRcViDZ83Pr2rkPHltmxtZeTslKZA/vD/7GuzwM5rB8aW/neHp3/iidHH54/kTWlKVpoyqxvBnka8N9eK6zwHJi+u4s/ejDY9cH/69co/yzNjs1dB4Lk8vxIif89EZf0z/AErvq6wZ51J2mj0/SphbaxZTZ+5MpP0yK9pNeHbSOnXtXtdvKJ7SGUdJEVvzGa4qTO2qiWk70UGtzASig0lNCKd4mY3HqDVcNuUHHXmrs4yuazov9Uoz93j8q2jsYyWpIDS5xTc0hNWSOzSA+lNz60oNMQ/NTQn5hVfNSwn5qAL2aKZu9zRQB8poPmyOuP05qCaQpMdpPPSpThX7E4Hf61VnyJOB17Vgty3sXfCpB1Kc9cx+me9XfFVskUEMoLb5HOQTkdKj8IxkXl0f4dgGfxqfxk/7qzHT5mJ/Ssm37axqkvZXOWiJWRSM9RmtQzbSAcgdR2zWSjFjtByOlWQHfGDgV0GBNI247u57jioz7cjtSgosZBY7gOCRURJBJPpikOw/kQNgnBbpTAqhflfJGO1Sq4FnMD13LgCr/huygv72VLhCyCPcACRzkUN2QJXZHpsZ/tyxxg4nQ5B9xXuKD58e9efwaRZW0gkgg2spDA7iefzrbTU7wv8A648nrgVx14Oo1Y78PNU00zlfHB2+JrkZ4KxcevSvou3P7iP/AHRXzJ4smkl14PI+8koD78V9M2xzbxH/AGB/KtUrRSOeTvNslNNPAp1MNNEsjc1A5qZ+lV3PNaozZDOf3Tj2rmw3AFb95KI4ScZ5Arne/FRM1pEpJHNOViV601VJAPY1ba1EdrHP5sTbzjYrZdfqKzubogBwOufxpJ8tBL/un+VS7Bk0kgBhkA4+U0rjGWx6+nBq7bj/AElCOMHNUbUjb1zwKsqxXDA8jvUtjRIrCLVrpugwp/U1oPdokKyKwKhOQPpWFOSboseSUGTUwP8AxLW9dhrixXw3NaUdTcl1SKMwB0ZmyRwcdVIrE0nU7lrWeMyHCTuF9cU67OfsxH99c/rVDSTj7apzxcN/SuRSfIzRxSZrNKz/AH2JHuaVG+RuO4qBTkHGKkQ/u3/D+dYSNYmFq5H2pifUY/KsubcDLuGDvhY/+PDir+tZ+1cdNorKuf8AjzkbrxEcg/7Z/wAa9nD/AMJHn1l77J3W4M7i3dVJxnJHvVfw8Xjv4Uc8qXX8Mmj5xKTtJG0Z4qPRJM6opAx+9YfjW02nTaMoxfOmdW/OjzDsHf8ArVK6/wBWDx061bB3abdrnpIePwrPnfMCHg5UV48FqejLYigYh15p1/zb7jj3qCM/veOam1FsWXTJHpXdSWpzS2Mk/wDH2gAPQD9KQnkd6FI+2wHP3l7/AIigjBrtRzjcfmaztUXdbR45PmY/nWiR0/wqreRh4hwMCTkfUf8A6quO5LM2Vgk0ag9B0qbxDn/hBbD2uz2/66VQunJ1EjgheK0PEi/8ULYDB/4+s8H/AK6Vz43el/iX5M8jMPjof41+Uil8PobSbxtppuyohj3y/OQAGRSy5/FRXr8Kr/aN0wZWX7WCCDx82P8A4uvFPCiebqUTdGjjY4PfjH/s1epaZfbjNGcAq8bjA6fdH9KdSm+dTue1TkrcpuSptwQOc85NWrRswDnkE5pl0h8kkYHem2TfIw7hjXVSdzGqrFPxZYf2h4fuFVcyQr5qfhz/ACrgLSdmSOdcM64OG7kf/qr1YkEc4wexry29sTpOq3Np0Ebbo2x1Q8j68fqK1aTVmYOKkmmTayIYblJRHiKdd2/HGe/HUdj+NYrxSpLG9vzCDvVgfun/AAroI9t9pEiAKZrdtwA9D15P4/kK56azMbF1ymeu3pXHg2/ZulLeDt8uj+6xxZZNxpujJ603y/Lo/mrHo91MLzSLa4GPnw3X1FZu0gYwetLodwLnwwsXV4H2tz71IyjPt6VhidJn0uF1gQlcjApNuKnMfWgJ26ZrFM3aIdpz0rNlh8y61CAjiW3/AKEVr7fpmqMi7Nai9JICvPsf/r1VyXE8OvV27epIyORVEda2NbhMN5cR8fJMy4+hIrHr1YO8Tx5q0j0H4Vy41LUIP78Sv+Rx/wCzV6kVyor580rVZ9Jmea3lmikZdu6J9px6dD6CtN/GWpv1u70j/r6YfyArmrYeU53R10MTGEOVntToc5xz71Gy7eTgfpXh7+Jr9zlpZmP+1cSH/wBmqtJrE8v30Rv94s38zULCy7lvGR7HuLz26AFp4k/3nAqvJqemxjD39qvbmZR/WvF5NSiNoiJbKtwD80hCkH8Nv9ag/tG57Mg+kaj+lUsJ5kPGeR7M2vaOuM6na57/AL0H+VZOu63pF1o15bR38bySQsEVQTlscdq8u/tG7xgTuPocU0310etxL/32auOGSd7kSxbatYjnGJTwfXmtLw7KIvEli/Yyhfz4/rWUzM5yxJJ7k1PazG3u7ecHlHVvyNdEleNjli7STPamTINep+HJxceHrJs5xHs/754/pXmnlhhnORjqPSu+8GyFtDMZ/wCWcrDH15/qa82k/esepWj7tzoPeg0ZpK6jkCk70tN6c00IRxlayR8ssq9MNn861/0rJuBsvWH95QfyzWsCJoTNGf1pmefejPPStTEfmjNMz1ozxSAkDc1LCRnNV881JEfmpgXePQflRUe73ooA+WgAWHTgD86o3HD8kdT2rUtWHmHdAH+UYzzjis242tdHGAc9+KwW5b2I4rmSAN5bkZAzz6VHc3k90R50rPt6AsTip/s+yBpGYNx25GKhtI1lmIcHAHQVem5OuwyHAPI57YqyDn+IDANQRhVudsn3RnI/Oo2Y9+tAEolzwDwOKf5UzgsqEj1xUOB5JO3r0q1p7ssgXdwynjr0pDFjhf7JcF0KMMYJrX8Fhj4ltoMblnPluvPTPaqkizSxNHwCAAD0xitzwDA9t4y05nK7RKucfWs67tTbNKPxo9Rm8MRQvteKWI4+63HFVf8AhHVl1CytYD5bXD7NzZIBxXoWqmPUL7zYsldoAyK53WQ+mTafeoMtFOGC+vB4r56liZ+35FK6PblySo/ClI8b+I2jzaH4vksZXWR08tsqOCCoNfRFof8ARYv9xf5V8+/FPVjrXjKW8MQiG2Ndn3jgCvf7M/6HD1+4v8q9/wCyjxbNN3LDGmn9aQmmk00DEc8VWc81Oxqu/etEZMqXab4vxB5rBUA9f5Vt3smyMcA5YCsZAVO7GazqaG1HYlToFHr1qVUbeE4ySMfNUS5Jz0qSM7T96s2zdJk91bTWc5il2lgAcqcjkZqBwzqwAOcGpPMYqVVTn2FO8iZlJVHzjspqOZFWZWtImCKCMfKK0La0jmdlkuFhAXIZhnPtVa1s7ho0JSX7o52GnsYomIaWNSOCGkAP60rpjSdirOQJ15/hPP409fmsHB/umq11d28d0rG4g2qhDESqcdPQ/pTP7Y0hbGUtqlssm1iIy2ST2rnxEW46GlOST1Zcumzb2zc9U/nVLTcm4vl/6bZ+nAq/dYNhaPAwkR0Rg2McGs2OaaG6uvK2ElwTn6Vywp3i0ayeppgMOxxUscTPHKc4wucfiK5/U73U7BgHZRuG4BQpqnbeJri2aV7pjJGUK7VKg5496r6pJq6M/bxTsy/qsRN0hPcCsy+/cW8+McRqc/Rx/jUFz4phmnDPbSbcYxvAOfypWu01CzeRI2jVoZVxuDcgqa7qVNxgkznnOMpOwX2qm2vBHyfNUJ+dV9H+W8ZiMN9p4A7DjP8AOroEIKDGcrnkj2+tV7FkjvZyQFPmgglwOw9q25Vysz5nc6NHAt71SQPnBGeOoqgwMlsgUgnaOMjrVp7nT0SZGmtwrNkb5R3GPaqw1jTYLeNDdWqcYBEnP8681UpX0R2ucbblaOKQTcjHqauXkBktCpI5FU313T2OftyMvXIJIp8/iC2eJYI7gnOABtIzXTCEk9jGUo23M9k22/YsBgEDpyaRkduAjn6A1HcXrwwErI6hs9Cf896py6i4Td++cY4AySfoM11K5z3RoeRKR/qJOP8AYNRvDII3LIcLg/Sop7PXYYklbQrspI+xfmj+9gnn5uOB1OO3qMxx2ut72Z9IEUK/6x3vIcqPXaGyevQVajIlyic5I2b9xkden41s68N3gXT8nH+kf/F1go/+k3DZ4zjHtW/rh/4obTSBx9o/o9c+N3pf4l+TPIzD46H+NflIw/CnGsCPcE+R8lv6/lXdWcqx30qiVctEMYzg43H+n6VwWivs1hGVsZVsflXT2UxGswAnPmIy8+pB/wAa6bXR6vNZno9xqOV8kIMYAz36ClsHBLkdQRWTvDCNvVFOc+wq9p0gzIMY4/CtYxS2M5TctzWDd647xvbBJbO+GBkGFzj8V/m3511m/rg96xfFUP2jQZ8DmMiQEe1WQcdpLxRaqm5VCzKYmBGQ2f8A64FZ1499pN3NDMv2i2ViASRvC9j6HipYgr4BHzdRWrrUX2m1t7tIyxkXbJtHRh+vqPwFcU/3WKjLpNW+a1X3q559T9zjYz6VFyv1Wq+9XXyRJ4YvInkmijbMc8e5cdiMV0fkkjP4815lp8s2n6iJUJUxvlk5wQa6C5+KmmWkjxNp920i/eBKKM/mazxVOc5JxR9HhKsIRakzrvJ7YrN1S7aykQKmdwzk1ysnxiiXPlaR/wB93GP5Kay7/wCKct26n+zLT5QQNzs3/sorCOGrdjoliqPc6k63NuwqLTUv5rjULZljTeAwG5wg59SeK4iX4j3Lgf8AEusuOmUJx+tQn4i32QVsLAc/88T/AI1p9WqvoZPFUu5neJjv1e+OACZmJAOcEmufPFXtT1WTVL2W6mjRGkxlYxtUYGOB+FUWOTXoU4uMUmebUknJtDaKWlCk9Aa0IG0VYSxu5ceXbTPn+6hNWk8PazKMx6VesPUQN/hS5kPlfYzaK2l8Ja+3/MKuF/312/zqwngbxA/W0iT/AH7mJf5tUupBdRqnN9DnaK6tPh9rDEB5bGPP965U/wAs1Ovw7vDy+qaaB/stI3/slL21PuV7Gp2ONp3YfWutn8BTxqPK1G3lbPI8twMfXFV7jwhdIkflzCRsYYCMjHpyetL20H1D2M10OltviFYR2MEb2k7SLGqsd6jJAGe9ejfCzxTB4gXU4YoWiMBjbDNnOdw/oK8QXwbfsxBkiUdt2cn9K9D+FVmfC2uXT3t5B5FzBsON3DBgR2+tYclJO8dzoU6r0lse5Un1qiNYsD/y8Af8BP8AhS/2rY4/4+FH4H/CmIuUh/SqZ1ax5H2lP1o/tWxP/LylUhFuszUcJcwN/eDL/I1N/a9gePtUeazdV1jTzDEwvIcrIBy2Paqi9SJLQfuoLcVmnWdN7X1ufpIDTDrunjj7Rnv8qMf5Ct7mNjU3cUbjWO3iGyAyPPb6Qt/hUbeJIMEra3knssY/qaLoLG4G75qSJua5s+I8YK6fcgf7ZVcfrTR4mkDALaIp/wBucA/oDRdCsddvPpRXJf8ACWP3giB/66n/AOJoouh2PFLO4SGU+YwH3ePwqleCEyOwiB6n7x5NWYLVbncXLgjHTntUhs03ncScDmskinsZqNnT2OCMbgMCqlu7qzFBz610IsojbhFU/MScDIqnZWiLcMNpDAd/rTuKxkqGkm6HcaMMuQwwehHvWyltH/aJGwH5s89+/rUMscaysGRd27HI70uYdjLDPt5PapbTf9ojweAavgBQAqJ/3yKt29w0JP3CD17HvxS5h8pGtypONrZJ7LW34c1CDTtdsr6YStEj7mCpk4qvDLBI3zMQT74zXY+GtVnhvbe2to7VNzAeYbdS2PqeawxE3yNWOihC8lqe26HbRalpkF9G5WOZA6qy4IB9adrfhZdWtY4hOItj7i2zOePrT9HvdtqpcSySY5YJwfy4rSbUG2/Laytx2wP5mvMwkcLyptajqzqqo9T5T+I1r9i8RuobcDtOQPrX0FZsPscP/XNf5Cvn/wCKEobxY+1Ng2pgEjIxmvfLRh9jgOf+WS/yFenTu6cWyallUZb3e9MLUzd70hYCtEZMGPWoHNOZ6hZ855rVGbKGpNiEYzncv86xjrsaAr5loD0x5SZz+Na2oHdCMDowP615++j373MrLLbqrOSN0nbPoBWVWnGW5pRm47HUf8JIsIObqJcDqsaD+lU38av9n+0RvqEsXy4eO1cA5xtwdgHORj61kr4cuZRiS7tV/wCBN/8AE1dm0rVZ9Ik06XXbeOORQpeG3O7g5yDuHv8AmfbGUaNLqbOrU6Dn8X6i7YXTNdb3MRQfqRULeJNfnJEGi3J7fvb6NCSfYtUX/CM3JJL+K70E8nyrdE9uxqRfCdtIP9I8Ra1Ljv8AaCB9OtaezoroR7SqyzZN4nu9O+0w2+nIqsYiHuzJ8wOOqjHX37H2zxjTTXE0srvtMrbyqspAJ64PNdtD4Y0m2tfITUdSaMsWK+cuCTySfl5JwKE8M6BEuAlwR0AaXH5YApqVNfChNTluzko2YQzgMScocnHv6Cse4UtcOxYBsYOT/wDWr1PStL8LJqMy30bLbeSCN0jnLA+xrWa1+GyOXayWRupysp/ma0ik1cyldOxL4H0221vwXaSXMwS5hHlvsbcAByv6EVtv4O0xnJW8mRm+8VYc/mKybDxH4M0OFotMsZII3OWWKEAE+vJqWT4g6IoylhO31Cj+tcksL710zZVpWEuvhrp97MHn1q7IHCgBBj8cVVf4TaPtAXVL1vffGP8A2Snt8SbEH91pbkY7yAf0qFviXn/V6ao/3pM/0q1QaVuYhybd7E1x8JtH8hTHeXXmAcksvX6bap6l4Ih0+80/TNPeV4rmKdWlkG4IxUYzgcdP0pV+I91JKENtbxgn7xDNj9RVWbxzf3F220QBrXdIhCdflIPU+9P2VvtMFKXYE+El0sasNXgzjoYm4/WoB8MZ4Y5ojbWdzKyMBOc/ePfkZ/8A1+wp1x8QNVOnpLDfR+cW2mMQr+fSs1/HPiKQ/wDH+yj/AGY1Gf0rVRt1J1Hj4a635aRjTdKG3kMzjBz142GrK/DrxAdn7rRI9oAxhuMfRRWW/irxDJnOo3f4HH8qrtrmuTfevrw8dPNaqXKKzOrtvh9rkdoYJL/Txly5KxucZOcdv1z29BTIvhe0RVptXQbcdIf/ALKuOl1DUGP726lP+9Kf61ENSjRv395EuB0acdfxNDa7Ao+Z6KPAmnyERy6rkDPCov8AXNI3gHRUQ79TkB7OiIpHvn1ri7O8AjEsV5asGGRsuULEf7oOf0q0msGQY3E4/CuSpTqSleLsjog4pWZ1eo+H9LvbdILrxDqjxxSLKgiljQhxnByEyOvrWNP4X8Oojt9s1i4kUEr5t85Geo44zzWf/akSr828N9P/AK9QnV492Nyn2LY/pTjTqdZMd4djzyIBbiUbiWzz7mui1ra3gPTwTw1x/wDHK5y6xb6tMu3qx6nrW54g8x/h5prIfmFyT1x/z0qcdvS/xL8meHmHx0P8a/KRU8JtFFrsXmxJI4V9odQwPyk9DXaXFxiWNlhgQK44RAv9K8z0W+c38DptEofad+SPyFdhLdXci7VktgeuPLIz26/56V1cp63MdVbX2IY1Y42oB14OPwqzBqkaSEMSQBzXFpqN9GBvtxJ2/dsDj88VNBqbqzefb3IGOAIi38s80XkhJRZ2za3bjOA3txVe61mOWCWLymYMpHXtXLf2kzA7LC+bPTEQH8yKa1/dY+XSrw89T5Yz/wCPUc8h8kTMkLxnEYO5eBgZzWzot1LfWV3YyxbbhU8yL/aPTr2/h/OsnzHE58yAws3IjYgn9K0NOnMFzDIyKoVsc9cHg/zqMVTlUovl+Jar1Wq/yPOzCjKpQkofEtV6rVffsc2pdNQYyPvZ1zkVmaj4duNTvQ1oYgdvz7ztArp/E0Fvp2oCONCpb94oB4APoOwzkfhUGmO32pACo3+vT6U4VlUpqpDqrnXhasMRTjUjtJJnNJ4D1Jutxag+m8n+lWE+H12Th72EHOPlUmvRlsXIyZFGeeBmpPsK55Y/gprP20zt9jA88T4ergF9Rb/gMP8A9ep18AWm0Zurhj7KorvhZqBks3XPQUgtgM8Hn9KXtZ9x+yh2OBHgS0U533DD0OP6VNH4UtE25tEO31J5+vrXdC2TP3Dke9SC2jGfl79xmj2k+4ezgctDplouP9AtR9LdCf8A0Gta3igQYA2eyKF/kK1fs6A9O3p1pywL7VnK7Likimq2/wDfnP8A20NSGC1YfMkrfUk1a8tM5DYowo7jr3qHFmqkiqLSzHH2f/x0Cni2gH3YAB9Km3RgffUeuMc00vD180D6Cl7NhzoaY0UbVQAdBgZphQHG1fbpUhniH8ZI/Hio2uoxxhuKpU2S6iGNBuyQDmo2tScYH6083gQ5CMf+BU06g2MBV/Oq9kyfaIjNgx6MB7ULpTbwwePg9MGlN/J16Z6cU17yUj7x/KrVJkOojX/tHVQc+bCQP9k8UHWNVAGJYs+u2sU3UvQytj2NMMzEffz7Voosz5kbf9uaqpH/AB74/wBpaU+Jb5cb1g798f0rBaTJPrUbHB6dKtRJcjfk8VXW0bUiz3PX+lULnxDeTwSR4j3EcZQFc59MVmHdyMY/Gm9MjqM1SiQ2POr6lhdot0PoIhUbarqpPE6qT6Rimn3zz70w4Hckdqsgk+3aiet7J/wEAVG810+N17dYHOFkxn8qTngdqTBDHrn2HWgBpaTOTNcMenzSsf60RxRkkmPNBJxxnNKjMDnJ/wAaBFgogOPLT86KTd/tJ+OaKBnIWEbjeBC7ZPGFJ7DmpHt7pmJW3lx14jP+FVGupshY7hwv9xHOOPoaasd1OSAk7/RGP8qzuVYurb3anJikH1XH86rPaSoxkURqck4MijNTxaHqc7jytMvJDg/dgY/0q6nhLX3+5ot98uM5gI/nS549w5W9kZT2rM+8TxK/Gcyr/Q0slukqjddQBu5BJz+QrZj8C+KJhhdEugB03ALj8yKv23w48VvHsOmNGDyd06AH8M0nUh3GoS7HJvZRKMNexnucBv8ACo2SBOPPc9jhPeu6Pwm8TTEEm2UDHDzfn0BqzH8GdZfHmX9nGPYsf6UvbU11H7OfY87DxqoxJLx0+Uf41ct9ensbhJYXlDqcqdoxn15zmvQ0+C84wZdWiB/2Yyf51Bd/CK3soTJca6evCJb8n25Y/wAql16T0bLjTqrVHM/8LQ8SIu1L6cJ327Bj/wAcqnd/ELxDdDbJf3LAE4/e7f8A0ECuntfAuiwDNwtzdHuHfap/BRVm28F6DBlxYFgT92Ry35Vmvq8dol8lZ7s82X7d4m1S2to0Et3PIscZZz8xPABLHpzX07bWd4ltDGbdtyIqk7h1A+tcFouh6Pp+tWd8LKC3EMyyGUkqEAPXrjH1r0RvGPh1eus2X/AZQf5V003GouxhUUoPUT7Jd5/1PP8AvD/Gg2F4R9xPxao28beHv4dSR89Nkbt/IUw+NdHP3JLmT/dtZP6rWnLBdTK8n0JTpd43/PIf8CP+FL/Ydw3WaMfgTVY+M7Ij5LPUG+kGP5kU0+NIsZTSr856FvLGf/Hqd4LqHLN9CeXw28y7Xux68R//AF64bU9Mn07Vp7ZN8wQgh8AZyM/1rrm8ZTEDy9IkOf79wq/yzXnviXSb/XdbuNVEUdus+1WiEpbBUAA5wM5xUTlB9S4RmuhoCK44/wBWp9DKo/rUi2kzjLXFuvsZgf5Vzy+H9QQAG5bjgYkNTL4cu3iLtOGK9RvJ49az9zuae/2Nv7EQfmvLcD2Zj/IU77NbLy2pxL/wBv8ACsD/AIR1m+9IAfpzVa68PzBiYXlkkxyqnAP096d4CtI6gLpaghtXUk9lT/E01rjRowd19MxH91F/+KrhjaYPR8A4OScinrZAsSyY9cnp6U/dDU6K61TSlk3RXMoBGDvj6+2AKx5tdtwSEjuHA4zhBn82FRpbRA/6se+eKU2sRxtUY7juKLoXKyH+2Vb7trOWH954x/JjTG1lyOLSP/gdwRj8lNW1t0CkhUz2OcEfWnLHGMhlix3wMU7oOVmedWvDylraj2812/8AZBTRqmovjYtqMc8ROx/VhWoIouo2kD2pxjGQDtbJ6ii6DlZltfam4yJo1Pqtr2/FzTUn1dfmW6kBIIOIoxweo5BrWwAPlfIzjrQRF8w2sAB1xkUXDlMnGouCDdXHocGNf1CU0215/wA9ro89TcMP5EVsEKQAGB/QUwlEPXbjp2ouLlRlNaTtwZJdv+1PIef++qjGkj7zrGwH9/LZ/Mmtkyx7csVPHUjFNFzb7vQf7Q4ouwsjKGjKf+Wdt17Qrn+VPXThF90Io77Y1H8hWl9ojPIbcM+n+f6VGbgAgjzAenNO7CyKgs2QY8+XnsH6002PTdJMc88uasG6b72N36Gm/aOpKupHoc4o1FoV206MkFsk/U9aBYxjICY68Y7VOZmIJADLjGKjMjquF2k44BFOzC6MrVkVbuJ2yqHAZlAJA74Gee/Ga7jUotE/4Q6MG6IslAa3lU7mL89uMkktkcd+mOON1BTcLHgZIJBIGO1RnSZru2EaSYCPuCk8AkYP8h+QrlxWEeIcGptcrueZjsFLEzpyU3Hld9P63/4Ohm6Zi28QW0hQMm8NhuMgV6BHq9kOXhKZ6fICP0Oa4CzVmu4XBKvGSCQcHoa0y8pY7CxxzjdziunluempWO8g1OxkA23MSL6MNn/oQq8nlzcxyI4x0XDV5izMUy5ADdNwx09sf5/kBCegyWzgr17f/WqXS8ylV8j1AQdOBnoeMfypTEO7kZPY+/8A+uvN45dTUExT3HA5Al4H4Zq0mratAQrPERnrIuM/kPf/ADxS9mP2qOm1e0Vyjow3j1IJrOTIZQzNknDbVzt/pVSPW5JiFkiRsn76yqMc9duauq4ckrjDelbRVlYynqzb1a30240JGnfMcSjyZQNzg+3qTjkfyxkcdC5idHC5KkfnWi6zTxsu7MIJxlvzxWWw+cDqAcc+/BrhwmEeGi4uTd3f+v1OHLMC8FCUedyu769P66/kd1Z30MlsuVJKj2/OpzeJzhSfriuW0i5Pl+XnjsO/tWr5nqB9ccVv7NHre0ZoG9HTyh+dM+1M3IVRVIAnPDY9RSgnHDfpRyIOdlo3UmTwB70w3EueXx/wGoAWA5AOaU7umB7EGnyIXOx5mc5BY800yOcHMnpTcHOOQPcUuPYkfWjlQczG72IOdxPuabksM9vrUm1Txj8jSFPb+tHKHMQnJ6vz7GgFgBjP4ipCpB6/n3phQ9cfrTsK43JzncfypNzHPNPHy87OelISrdQc/SiwrjCcjpzSGQ+me34VKAnqBmkCqOnp6U7BcZ5zADcvFIZMdc5qQkZA4z60w4zgYz3OKLCuML/XAo8wbcmnYGeMfgKQjoQadhXELKR0H5Uh2Ef/AFqRgTzkY69aY0bZyMA07CuOwpyP600hWBx+GTRtcEksPqKbhsckUAG0DuKTGO/TtTvn5GFyKaeByo9xigBhJz1zTenGaeXUcMvFIGU44Y8dQKLisRk9ep4/u05Tz0FSBQfu7sepHWpEgJGAMD3ouPlGcHkqM96KsiAY6J+dFK4cp7kum2iKAlvCoHQCMVKtuijgD8qKK8M67sDGM+lBhHcmiiqSHdjCir2zSEAc4oop2KTGsAOwpM/LnAoopFor3EpihaQAHA6VyV35l5L5k0hPoAMAUUVUCyqLRC4BJ5pGt0Jyc8dBnpRRWghBbxyZVhuB4O7nNOa2gikZI4kXacfdHaiiriyWhVXjjA49Kc65YL6AUUVVybDSMHHXn/6/9KcFDQqfRiBRRTQhgQc/TPSpY4wxZSeNpP5DP9KKKYEbQqDmiNFWUMowR685oopiJprONHUrwrAMBjpntUX2JGLAnHuByKKKAMLWtNh+xm/GRIrbXH9/DYz7H/PNc8QoQfKPmFFFbQ2MZ7kTSqqL+7yGbHJ6d89PaohdssYIHV9vPWiitDO48XLsyjAzgc49aikuZE2HIw3oMf56UUVVhNjGupd4w3zA9SBTlkl8zy3kZgRu9KKKdibitlGDFi3zAfnTXLB1+dvm6c9BRRTsIYPmjU9MilWQgKjcljjI4oooAcFBbbjBx1HHShY8fxHrRRSExTCNwwSPcUzPcE9e5oopgICgbaVzu4zmo5ZvLYkAk56k0UUAQi4EhB8sA+uf8+tX4oBMuchfoooopiHz2Bgi81pA4MgGNuOv40qw7OAx4IwaKKaEzmvs4jvkIPRwDx15rtI9Gs5H2BWB+ox0z2AoorNFg+l2sEgUK5I7iRhn8M0q2NqVJMKt7P8AN/OiiqAQWtkpA+xxZJxwo/wqw1pHBGHVI+fRMUUUhiKSwHPSs9iTOccZOOlFFVEmRnajq8ttcfY40DEgtvkYnuewx6VjnVyzyLLArFOMo23P4HNFFTLcqOxc0bWfOvPJWDZwDnfn+ldpgEDOegPBoopMaF8sEbgSOM/pRgjox/GiigBQD83P0oVNx4JHP1oooADEVwd36VEWIbk55xzRRQAhcggHBpyNuKr3PeiigCSaMp82/PPp9f8ACoASSefQdKKKYAcliM0ED9aKKAAwkAnd+GKj2/Men5UUUgGhdwJzjFLs4HNFFADSCucHgU5ASM7vpkUUUASiHPVgfwpfs69yfwoopMYq2qOCQWGPU5qX7GiHGc4FFFJsaQgt1Lfw5JxnbTWQKMjA9KKKVx2GAcBgcZzTkj3tgkenSiigEKsStkYHr0pywoOSD+FFFIY4xJk/LRRRQB//2Q==" + "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAIFArIDASIAAhEBAxEB/8QAHgAAAAYDAQEAAAAAAAAAAAAAAwQFBgcIAAECCQr/xABhEAABAwEGAwQHBQUEBwMIAhMBAgMRBAAFBhIhMQdBURMiYXEIFDKBkaHwI7HB0eEJFUJS8RYzYnIXJEOCkqLCJVOyGDRUY5Ojw9ImRHODhLPidIWV09QZJyg2N1eUtPL/xAAcAQABBQEBAQAAAAAAAAAAAAADAAECBAUGBwj/xABIEQABAwIEBAMFBAYIBAYDAAABAAIDBBEFEiExBhNBURRhkSJCcYGhFTJSsQcjM8HD8CQ0U3JzdNHhFiVDojViY4KS8VSD4v/aAAwDAQACEQMRAD8Au/X8P6atT2FQuncR2iXB3HEHuqChs51A5Wr1if8AZt8DsT1NbWOqv+ler3Vvuqpb3y99S85IDrTkajba1tnx9r/vW7QnSBtFq0GD0NI4mniDb9hZHdiNTO0CV5Nu6olef7Kjh4+yWLpx3i2kBGUF16lqRryjs27NO+v2Ul6Lpl01zcYKoIMR6xcCCdD1TVT8uZt6PJBEfG3YCtAJNjmljQxM5eWN7fsu+L9MEG7ceYeqQjlUU1azm0P8jSx0G+1mvX/s8fSGu9xxxhvD1fmEf6veZaOnP/WEN8ot68gcyZ56i3YsM0bT1UueeoXjDUehT6S9I28tzASXlkmEsX1dzpzTuQmoJ2J5c7JFb6M/pCXYts1PCPGC0pVK1sXS8+nYz3mgschrPxt7Zrp2HT9qw2seKAbF13Jc7hzLuylJ69kAbQ8HbYqXPC8Nq7AHEC4HlfvvCuIqEaAmtut5nKdf50jw2/SyYk1CMyHKjvgKE6kyDvB+Otvd1NxXYgQ1Tqaj/u3FI+4ixG8sD4ZvdJRed2t1aCNU1A7UfBci0TSO7pCcLw8HrUIClFPMmNZidh5WNJVUhxtSCpQbTMRzHyIE+Pyt7HXl6OnBu9hlq+HWGl88yrko1H4lqbNqt9DjgPWrKzgK50lQI7jDjQ+DLiBYBo3ozJ2jdePFC86EqaU+ofaupOYEgqzHw68/h1sp071QpmkS2FQGB3oncCDHSPvFvUiv/Z+cBX0q9RwoilUpZVmYvOuRqTr7brgHws2a79nBwvdShFHXX9ShsBKA1fDSkiBH8dITt/isCOkkjdchWH1Eb22BXm52qw48ohakghBAHu92/wB1gHHnsoS4rNIOVWUgxtqPhy5i3oFef7NC6ypSrrx9fLIJ0S5SU9TA8w41PLlZo3r+zSxk2hKLpx8w9kAE1V0qbBAMwezdc8eRsfluHRVswKpgpaitBLiCAYIOh2OsefXmPdbpCnVKWkPKgQOZBEgmefL7/G1pbx/Z38aaRZVT3zhmoSNcoXXNnqYmmgfE2bF4+gx6Ql2oc9VwzdFZnkgs3zTNp8IDykHkNxZix3ZOCN1AZqKhNP8A3qgRr3tcpPLXfl9bjIcq0OpT2qiNfaGwj9bShefoq8f7tyh3hXez4T/FRlqrGgnZhS+fhZu1nB3i5dClrvThjjClQhMZ3bkqkJO+5KfrlaBaVMOumoha31KBCYBIJynbr4aD5fDsOuAIBIyuCQEmSNOgO8iwz913pQoS3eNHUUrilFJS8yWz3jGkj6jwtyhpxTic4C8u4Ouug6fWnlYZBRBuhUuOElICoGqgADEeR38LNS9aqpZxfRBBKVFCQTtAhWhPuGm+g3s60JWVFxKAY7oIjUfHx95sz8QEt35RupSFBQkKCtjnMR8Bt18bV5RcIsZ1Tvoa1TtMPtAuHClPI6x0/SwhqCkBWYqEzod9DG/v6fOxW61FmjFSASrPCdQI0j4/XWwq1lKUMBCj2nQAnx28fnaEIs2yLIddVtVSPZCkgDU5SQecR7vHlbHHVpTm3kgZgAdd9/d4+dtJUhS1d0HJrBBjXry/Q2DUpktlbqAc+wIjWdBr4a9bGI0QkItx8uFRWdANwBO+3x+uYfauZpcd7MyVR4HoPLX4eFtuhJPZyuVE5iCQOZ0nfYn+tugzmSSZyjugZtMs7ncTP3WGd1ILguf3edSkKJBSOY8RuYkn525zOqXmWCmBsoiAOkdfOwvZLzBIKlK3JG235jQW7bYdSolJBGUCIAG0zqek+FoFw2UliFLRuJjU7EDkddxysE+pT1OtKCRKTAKjGsRPvANjIQ4FJzJSCoxlhWoJ1E+Wm9u3GHC2sOArCkFUAn3ESNefzsFxspgXCZGGal5hdag5iVFsqAESO8SCB56Hnp5WePqtM6EvhkICpWQRunmfHQn4+dmfdbq2q+uYS2esBcQQeszsT4+/Qv67cyruYOQ5XECZB1PXp9e+wHey66K0XbZJ37sZcyBTKgTyk6nTlM9Phbld1t65XnRMJ0UJM+fOfqLOFLSUJCiEJkAjWABHT3H79LdLZbyjs2sxImZIE+MfCzcxNkTbNz/7FFQo6DMCNvz/AB1sAbicJJWW1Aju506bax7t/Lws6jTNKJSQR0jcjrp5fW9sbpStYy6mJmdt9R0/Mm0TLlUhHdNdGGFPALUxTrUIB1KZHU6abfRtj2Gl5YRRgwomEr8Nz8QPKz3RRARlSNJHSDzGtuFsFQUmAAYnnETPz/CwjMiCIKO3LgqDoikdV/iBEqEjST/T8Cj+Hn0qCnG6poBWuZBEGIA1+om0mrpykEqTkKDIzAaH89D8/CwL9E52CnAhRhJga6fdPP52iJSkIQVEqA809AGaCUkQSNDMx5nbwHPZRaoC+yhSw2lXJU767eex+NtXoz2NSuUkgrMEHmSDPnp8vOz0wmzT1VyNpdYbVC1tqBTmO8gfOBZ3P2KZreiY9Vhltthb1Td9K4EyqEtpJMeB9/xG02jrGNxFqlVVU11JZCVCC03l016CIMjx91p9v+loihVI0wkJBlZSCBM+zp7/AMfCOMV0QFz1KW091CwonfvSPz8/K2lQyua4G6p1LBY2UIFp5sylCxHQEW2KusRID7ok5iMx36/dZyOU5J9tf2hOqhzkkHT62O1ivqwWgoHe0JEpkQIk6+74W6MSg7rILLdUimtrNFF5RywBOsW0Kt2e8AeeospqpEZA4UJMg6BI310/Tytx6olSJUxlG5Ow32j6191pB7VGx7pNNSsjUCTzt160cmQiQBA1+vH42MqpG8sqSpMCdNdNvr6FgHqdCGsyM2+s2kC0pvaWOOOwlxebvCASeVuPWne93tFbjrYd77SkpgonupUgeYUT/wBQsVDKiCZGgnnZxYpG67FSUnMEiedu/XVkRlAmdjFgC2scvG2sqiYCTZ7BNmKMtVLaVArR3ZGm9jKrybKezbSrLtmJgxPhZOLbgGqFbdLcwelmygpZijvrLakBHMdRpMncW0Hk6gkTI1mLE9YtmvIWWUJXQ1Qc0Kka9LBTMW1bLSGiYlC01Q7TVLVS0spW0tK0kbggyDb6S+G+KTi7h3hfFTawpN8XNRV8jn2rCV/jb5rxuIt75+ghiZWLPRF4a3ktztHKe6lXes9PVnnGAP8AhbFuZ4ljvHHJ2JHr/wDS0cOsXEFT8qpcjewRqHdwbcne2la25MFbAa3ss7dwn2rZ27mpn4W5jnbUE2MFPKF0w+4l9xBVoQlX3j8LGO2dn2jYmklNQg7ZkKHwIsOZm3Q0ZzRBV5QAUN2zp1KjbQfd/mtxMiLYZPO1oBCJXXbuDXNbO3d5K2sHztm3O0gLpt0L6w6dMxsn3i84H6V4rOiyg68iLHPh7rEr1Ss0pW2RKFJXPkbJw0Tt0K67d6Pbtyah4fxxFtAymQN7cKHvsxGifZdipdIifOw6al0omeQ5WKRoZGthWBKAPdau5tkRuqGNS9EFXzsg4rvC9aPD96Xg3e5ojS0rtQlaKfOUBCSrUGc23ICdbK7y+zTIQVKUYSkcz+A/KyDiXDLeIrkr7rvaqfcRXMraAQcqGCRoUpEZiCAe9Oo5bWC5hk9kKZIaLqvtb6TdwIum777vC5Hq6rrgy+lhbDCV0ockF3OHVBJA1LaghY2UQdLN3iZifBF4JrUVGI78Xfq3E0jtGkLcNGcx7nYJhttSvaKHVoUEmTJ3auIeH1z4lwxfuE2r0dp8QYKfco/WnezUpTKUgrkhQBCyc0nUEZddU2rWzipVLSUGHnr6YdvO6K3KyENoU0wlDS1ZFFaSFpSvUEqKQqImYtnVmHiWBzSbaH1We6tkY4Ai4KngYnwGkBLlPifOBCovCmQJ5wlNIQPIEgdTvbLNxn0g6ltlDdVSXut5KQHFJSkgqjUgpWARPQAeA2tlvODhtbf7v/d/stHmw916aPj7XTqbdtgKEW4fJ7aInrbtB03/AK2+kHbrkWCwQyR1j327gAW4SSne3aSLDspLsCD+ttgRttytyCFEa+7pbsHlZJLYAnb326G2xtyI522DZJLqNrYJjc+VtT4Rbek2aycLoEzvbYJPMaaW4Hv627B3+8WiVJbmAdLb8BbmTJAP9LZMTZJLoEbG2+WvO3MiImPfbJnfaySXRgHTS2z3h8rcE22fGzWSXC6alc0cp2l/5kg2CN13Zv8Au+nEDdLYB+IsYkAdOU2zMAPdZWCe5RN25bteR2TtOSj+UrVG/SbN68eEvDe91Fd54KuSrKt1VF3svE+9aDZ25tIzG2THORaJY3snDiOqi28vRi4I3lId4bYaRMn7O5qVv3yhtJ5nnzszb69BH0b77dRUVXD2jS43AStmqrGY6aIfAnU8rWDnXzthV/MNrCMEZ3CkJnjYqrlR+zz4Hd8Xc1fFADqE095rygxv9qlz6izavX9m7gl5ZXdGNL/pCEkJDr7L4B01gNNzttItcgERM2yeXQWj4aPspc9/Uqhlf+zWvFLTiKLia6tS9e9cyAnykVM8hys2ry/Zy8TEls0GLrkeSzqn1hipbnSJ7iHOU/K3oyDIibZIJn8bQNHGVIVLwvL69fQK46UCH10zeHq9ZGgYrnGyroftm2x8xZqXh6H/AB3upCe0wCw6y2SpZYvSid0E7pDs668uVvW2YNuFobWmHEpUNu8LCdQtPVEbVu6heO9RwC40U7SqpfCfFLre6TTXY89oBM9xJEE2Qbw4eY1ufI3fGEr+oSDKjU3c+yPEArSPDW3syzdt3KYSldBTEARHZJ/K2/3LdREijSkncoUpH3EWAcNJFwUVtaBuF4nqYCXnC8uC3sCrw1OuvlbDTOFklDpU4uQnQaj4actbe09Xha4q4FFTSrdB0hbq1A+4k2bl4cEuF96ybwwZcr5J3euqkdI962ifnYDsMk6FFZWs7LxALC6e+nmRqpaFZSTuZHL6+Vn/AIfC3LnpkqUQlGYg6pKjO28bfC3qhefod+jzezheq+Gdw9oQRnbogwYO+rJRHusiv+g/wNLRp7uw8iibM91iqqwrXoVPqHysCTDZiNEVlZF1XnEkHss5a0AAHdgE8jHn8o8LCIoe1CWSVa94qPkJkbjcc7X9rvQH4ZOqz0F536xlBypF4NZfgqnUfnZDd9Ae7qZTi7uxjXoKhlT2lO09lTG3tN2pvw+oGwVhlXB3VLE3bRqcJUhAABA8+ZIsMzdrKAApI7RwzKVSfd1Hn1tbmu9BW/mwgUGMm3gk5vt7vDYmDr3Hl8/DlZvXj6GnEilLrlPXXK+swEAJqhPn9gR152pyUdS3UtKssngOzlWg0bbjp7PMkNJAKlH2tzIMcudg1ULpEdqQpckZkx5aSPlaeK30TuLtNlDeHqOoSnVS03gwgk+S1JPynQWQKrgDxUoS447gC9lqAhApkB8DrPZlQPXfl7rVzBO3dp9EQSRnYhRC5dyitSUZCACpSspAHgOsiT5RYu7SvGmIIb70jRcE7iNucH42kyv4W46oVpbqsFYjpUpSFOKdu59I6xJTHw0s1K+56mgUBVM1LJSIShaMqieZ1HjvFhHONwiDL0UFYppnGX1JUhGbMFKKVTAjUnof0suYNvFbNzPNgla0vHLpAgga+O29g8Z07jSalJaHbOjMoggmARp0iPvsn4MJeoahszGZJAJmNDO/iBva6xpc1VXHK/RLFQoqcKyJUr+Y+M7+UjTrFmhiinbcoa1LokJbnMT7zty036HTnDwWhZQVSMxMzG8dOsWbl/Ndo1Uo7M5VsqA1zSMvnr+u4tdp/ZIsq0wuFEjzZglCSspBPtZp13j4+MneycQpQLWQ5VCSfEffEHz+FlRSsoSlJcBI5QoGefhtBkdegsmqKHESAdIkHmBz+ult9iyHIJ3u5ieQGpn5e4fpYN1KRlKwgKGh72/1pHlYQrTKlNrBV7KtfaHMjw0+VgFKXKc6CEq2GhMSeXLf52KAolaXkICey0He8hyHx+uhaqALTgTEggkk8/A/XOwy8wSCEqO8iZ25zYJzvIcCsw011nX6ixQo26INKpups5NGqhXe/wAyRp/y27hteVBAVmB1B1nlv5D524p0ld21aMp7i216Db2h+ItyjMsCQd4Ouh9/K0im6LZbZiCNzBJ0M7fI2zKlBCVRI6GIi2FZJlQifGBERP3W5IKjoBqO91G/5WSS2UBU8xpGuvPfrbSmkpJTJmYBI++3RK8xQpskHc89LczEFWhO08/ysklzlAJ0BIEeztblTWsKA12k2EzbnLmmCAVaQPvtpUT3ANehj6FnumsgFoSkab+dg/Owy092dbBAHUeFpAqBC0N7ezH7JXFKr79F6puNx3MvD2I6ylSn+VtxDTw/5nF28ZxuLenX7GfE5LHE3BrjugVd15tIn/7M2sx/7P5Wx8dZnoyexB/d+9XKB1prd16Xqkbm3OYDa217WDPntbhwuhaLhCb7W5iLYg7CLdgA7WOBdNsg1DKtpRgQqPiIsMdLAVH90T/KQoe42MbadbblAf1dlXmHVZpbe3K2CedtEAnbwm18BAssMRbQ1NtxyItozzNntZIrZJ2++wVQgOMuIkQpJEWFEnQDW2lCRp42mG3Ca9kn05zMoJM9221STMWCpXUL7ZhCklTLqkKE+zzAPuP3WGMjTraKley5GulhGP4gdTobBjc9LdBRSTkAkgxJgE9PoWDJYC5Uoyb2Q6W1uLCEplR0Fi71VdSGn0P17SezBUe+CUqA10Gs+FlSlF1stes3lU04CkkjtXAEgeRMWizEfE24EY3dwIbyqKdx5gVSG2CEZ2xosOEBTgABzZEpn+cpANs6XEBSNzkgHpf9/ZSI5rsoVV/SYcvK7eL39tcCV9JTtPXb6reDtUw5SNrdJU2WluuJSmSjNBBzCCI0AtW7D3CPJebWP8cY0ujDly3i8tNCy48G/wB4EqJUUgArKIMFSElOaPaAMXj4v8Qrlq7jaua88P1d5Uj5Q/TUDF2qp6elE9xZdda7FS0CFRnCJBTJtSPEPB7FuNga/E2LKek7d9bya+8nBSAsJEAAKUA4QnQ5ARoRrBNsOoxR0zS10gYDuba/LdCfTtY/MG5vJWBoru9GGno2GG6ildQ20lCXF0zpUsAAAkiqSJP+UeQ2tloapMK4ZpaVmlfxhcbjjLaW1r7Cr7ygIJ/ux91styRoHX/rL/5+Stc3/wAgXrC8D2mp5m3aQY56W06Id5iZ5W7QAD87fQzt1yTdlhURvZMqqgtl54uupS0CSEqjQJnSykuSJ6crItckuNVTROULSpBMbSmPxsSMXKFIbKHeG/pl8F+J1+0OGrgxDetFed4qDdLT3ld6mg6siQjOMyAo7AEjXTfS06sXhUTleIJ5aWgPC/ov4QuO9bsvpTrFQ7dz7VU26lrKorbUFJO55pHOz1/0hN12PlYRoaSuR+7qYvVb1RSuMIUtahkS3nAziErOZMp2gmDBcgcNQoB5B0UqNvZgFCBNgH7y7JeRpoKKfaJMAWBp3CGZ572jnjnSVl7cJ8Q3RQ1DjLt5stURW2qFdm88hDgBGolClDyNhiMXUzIVIFFiekr0F2jepappKsql0z6XAlXQlM6+E2VG6xtxIOuvO1SPRw4LVnCjHCbxuurdau+86V2nq6ULUWlEJK0Kg/xBSd/8RHM2s6K1LVb6qSNUBY98j8LO6IdEmyFLzj7TSC4swlO+k2CF50hIJeieqD+ViVW6TTATupI+YtXL0qMYcasI3hhRzhJfjFGh1qucvCmeoWn01RQWOyBK0kpAlwd0ic2u1hiK6mHm+itC3UsOiWnkK22O1hMwjTXyswsIX5XV2FLjxBfbCKWtr6GmfqmgCkNuuITmSAdR3lRFnYw/mSSDoRaBitonEt0ozBEj42wKM6E2YGOMTvYUwjf+Laa6nbzeumiqa5NE052a6ktJUoNpVBgnLEwd9rQbhv0ynr6cbZc4NYzYcVCSmnWHikkx/Kmy5Vtyna8u2CtkVfzWzNIMmybR1y6lpDveSCkEhW9idVejza6l3ty21TzMJnQJCidp52YRk7JzIBul7N0233thVrvAtCuAfSm4OcS72YuPBXEajvC8KmSzSu0T9Mt2BmIT2qEAmATlBmAdLS1TXkp0AuIg8wNrMYyEhICj+YHnbCY52RL3xHSXU4w08ptJqM2TtHQgGAJAnc62DRidpwBSaZSgeaXAoe7rZspUswS/m6nnbJjXebJVJfjFW92CWHUmCZVlj77KKHErEpNo2UkIFdbZPIe+3MxysC/V09MntKl9tpAOq3FhIk8pNokJIwpUn9bZn8bAt1DLyc7biFp6oUCD8Ld5pEjabMkus3W2wrqfjbiZtqCefnZJ1jR7pE8yPnYTPzmxdr+NJ176vvsICCN/jZgkUKFdQNbaJIOk25J08rYk6kCN9bOmC6nYmfhbYXoOfSwZ+c23IGwsxUwboUK5xpbMxnpYJJG5tuSdOthWslZCBQ3GnvtmczpYPmbYD42SSEzRz99tLQ24MriEqB3BE2DB526CtetmICI0oE3bdiyFG76ckbHshI+VuXbou15ORdOSk8gtQHwBsYG2gtsEe+0MjeyNmd3TavThTw3vsqVfGC7lrSsQo1VAy/mHQ9okzZp1Poq+j7V5z/orw4yVxm9Xu1lgkDbVpKTpOlpSHW2wYNoGCM9E/MeOqge9PQb9Ha8syk4L9VWRGanr6xJ+Aej5WZd9/s2+B16lTlNeWJKEq5MXiiBvt2jLh5nna12Y6SZAtsK00PnNoeFi6BS5z7WJVA74/ZF8PX8wuTiXiakBTCfWBTVUeYDbRPx52YF9/seb9Smbh4zdoqSQKm4ghI101RVKOk/y8hb09SobxbtKxqLT5VtioX8l5AXv+yN4+0xWu68b4Qq0pnIFrr2lke6mWAff77MG+/2Y/pX3WhaafCtyXlGYJVT32w0I/wDugtkfC3t8Fg/lbpKhvysrOHVL2T0XgBfPoM+lXc5X63wXvmoIkD1B+mrJgbww4ubNC9/Rp9IW5ApV78DMfU6U6la8N1hQB1zBspj3+dvotW2y6PtW0K/zAGxcXXdRM/u6mBjkykWQe8bpuW0r5nnsMYkudNYxfNw3ld6g0e7VUjjRkKGhzARzsjhKg3O3LW306vXFdVQkoepypJ3QHVhJ9wMWa978EuE1/FRvrh9h+vze165dbFRP/tEKs/OcOiRiB6r5syhwRm0010I05TbSlEz3IBO02+ha+PQq9F2/EkVvBPCKc2s09z09Ofi0hJ+dmLe/7Mr0Q73Klf6L6ekUozNNeNc1B8g/l+Vl4m27VHk+a8JIIPdTEba22UuKEJSYnSNbe1N7/skfRjrh/wBnNYju6NvVr3O3/wBubdsxr8/Y4cMnVOLuDiHimkBHd9YXS1UHxHZNSN+dmNW0bgpckleR6EHLmGhnWRGn1+Fs7JESVCNNRruLeml8fsbL4QCq5OL7jhPKquBsDwkoqyT/AMNmHfn7IrjtQFZunGOEa5P8KXkVzSj/AMDDiZ99kKuMp+Q9UFUlH8RMnxsCUkGLXCv/APZfelhdgiiw5cF5ba019NtAn/7o7I/LnaPr59Aj0uLiUF1fBi8qhJUB/qNbSVh+DLqiPeLFbPGfeCiYX9lXvUWu/wDsjcRqur0k7zuJTuVu+8M1TQST7S2nWXR8krtXC+/Rg9I3D6lG9eBOPWG0/wC0/s/VLRH+dKCn52k/0DW8TcOfS84fVN/3Fed1oq6167HPW6RxmfWGHGkg5gP4lJsGvyy0rwD0T07XMmabdV7kKMggW4522VamLaHjbz4DVdKFnusIgzIFuBr4W2BrY7Eiu3EZ21IP8QIttpWdtCjzSLYk5hB3txTK+yUjTuLUny1kD4EW16A2JCrS6BCj3W3FtCDvpbYmNLaoCr3Wc5m2hlgcrbJ03tke+LOEyyCBIM25URtbek/ibYZ2FihRJ1SahIRW1CEpAzwvQb8rdq31GptzVAiubWD7aCkj527VKhMzaFksy5G4IG3Kx+6ruFW+X1tpyta7bq5CxFsdo+3TI1ccMJTv7/IWdvYeo0JbpU5lJRKdPaV1PvtXka06nomfIWaDcpt3xQ3Ve7qziR5hm52VJbDbjvZpfeUYhZkSASEhM6qJkGBaMuLXD3hxdlwV9fT05w2mphp28aeudoHHgogqSMiVuPKMJ7mQ5tdbTS8GKTItbWcNkFAy5lZoiR46n4myPifCVFjK7Td+KqRD9A6tLhogshKinUBwpIzCYOX2dNZti1VFHIx0b2hzndbbfPpbonZKWuBB0VSmlPXtThq7r0x5f13UjfaJXWMuPtKUlKyQy0ptxSFQ2YByLy90e1Brzih3CWIcbU9wVeNsVu1F/VyqZ2gqaPs1sNkpzuZE5glAyz3koACZCXIMXz4m33U1l10l1YBoFIYoX6dKKimYCmlpCi2tlkJICYSCCpRSjZIVMlNQ+N+M2g1fdVgm+Lpuy+H7qDFe3dRbdqWVtocYCHKhBkAdqhTeTKFDtNJFuIqaLw02Vri4d+gPodlrts+LMdPJMuvYwNS11RTU/ol8SFNNOrQgqdr21FIJAlCachJjkCQNgTbLRX+4OMqu9dtVfIpDrTh15IWG/wCHMCoQYidBbLE+z4Osh/8Am5Z/PP4foF7HPiHo1t0BpHztt9A7UdTbEjSTvb3Zx1XMt2XBjkYs2MVBS7gvlsGM9JUDf/1Zs6lIkHSZ2FoS4knjWxiioocNVdxu4dvDK32dQwoPsoUkJc7w3M5iOkidrEY5rfvGyE9jnGwCr5wP4EYv4a4xuC/rkxfeQoG6hluro1VBLTzBIStKhse6dJmIEaiRbq9m2DeVC6tsFeVxKTGoHd08rF2MM1dHcLjlCltVc0ypTCVaJLgEpBPSQLMPBeNOJ+LMSN3Vi7h3TXRT0iFr9carQsFYgZQgmSDvM6ZbGMjNLuQxG/spppTFNvsnnZv4zb9YwvCe8C7TKM9O0TrZy0jCksZFDlGnlaKsR8U7r4eV72HOIV217V3OFRpLzYp1PMKaJnI4E95Kk7aAyADpZRuBOiZ7SAnbczCGnaZaUiBI227ird1VRnxOUJiEUrc688yj91o7d9JLgddVL27OLnKxwCGqenony4v/AAjMgCfMjzs4OGtdeuLkvYtr7vdoU3m6XWaZ722mAAlsKn+LKkE+JNnkeL6pRxuAUiPkilbJH+0Ry/xC0I+kT6QaeCF44cp3eHL2Jqa+G6px19moCFUYZU1HdLas2btDzEZec6TNiC8KK5LndvS9KxqlpKYoU686oIQ2nMO8onQDW0Z4pc4WcRaqkqazG9wPCjbcbaSmtYcBzkT/ABaeyLQBvspEd0q071BxQw/hviDc173kxRrZTeNPRlQQ08FIOjyQJKkyY70AjbmH9dbquxSFTMa9bRk/jXA2DbqurAeHLzpa2rfUijpKalfQ4pCCZW4sJPcSE5jruYA8JIugLVToUoRI99mO6WqRsVVdxXfhS+7wxS8pm56WkqnrwdKVKy0yUqU4qEgqMIBMAE6aC0U4X4s+jBVPMt4bxxQ9s8pKGkOMvoKlEgADO2NSSNJtLeIbubvzDd8XGpsOevU1TTZFRCs4UmDOnPnaMsJej7hO4q9q8XsOUKXmlJUlXZIUQQQQZHjYUmu4uixadbKaqE/ZgIAyxuLJzozGvbVspwgnzbSLKdIlDbQQFaARZNUgF28Ez7Sjy/8AVptNm1lB+6gzCfoxYawtii5sUNCnaq7pqEVKHGUEKWUyY1GxmD4Wlq5sYUt733WXdQs1SE3esMuqqKZbGZyATlSsAlIBT3ogzoSNbVR4XcH8e8NsaXBfV0Y3vd2501jbdVQPVS1tOMrISpJEwdDOsx5gG1uylo1rCymFlKgFRykae6TZOGUJ267JF4t8LLg4o3KxQ39Rh9VIpTlOvOUltShBIiNwIsh8LOH9JgbAFbhCkUtLTVRWBJKiSA4M25Mz3jaWGBmZAJJ0shpSlL94tgbvE/FtP5WYag3Ke9iBZQt6PnCi88A3q5edRia+rwS5Tdghqsq1ONtgqBJCTz7o18TayFOQEQQBZsYap20sNlKR7I87OhCIGmnQ2rWI0JurGh2CGKrMfiopSrhS2D3e3QogazZ7k7xpZmcS0BdxknWHEm02i4N1FxTSwLiP93uO3a4pXZP99An2Vxr8R91k3DF9VAxomtU+6pC6p4qGYwZzDbbc2i6sxhXXNxrGDKmUUtdh9i8rtcP/AKQ0+8moR55Cyr3Ks8cHO1Cqxp2pbU2568/oenbKyn3piPCw9mAFSGrrhWJpHw6mZsZBnQfrZIulSlNCTyFlUGBZOFikuWvacnkvn5CwsxrFgWyAtwazmBj3C3ZM8pswF0k28UYyOHKlhhqhFSt1JUuXMmUcuR31sPT4uo3rkN9utqbQ22pTqArMUlO6ZjX9RZpcSAf3rTnq0eR6n87MisxMwrC+Im6KqCxTMvtPJCh9m82jUEcjEHxBFgufy6kRu+6QrLYuZT527gqZsO4wosRdolilfZU0lKjny6z5Gy0p4QCkeU2izBb6qOqCW1khxr46izpxhfj104aqq1k/ahGVBBiFKMA/ObPTOMrLndRqI2xPsEfrMcYXoKlVJVXw0HUHKtKEqcynoSkEA+G9lO7r4uy9We3u2uZqWxAJbVOU+I5eRtUHiRdfEC+7tYawBf5uupbUt51YaCi7p3USZgbkxrt4guHgRi3Eyqe5q/EQU3eSnDRXgkADMQsoJIGn8qtIEjSwmve5vMO17IhiaDkG6tOt1KU5lEBO89BbaHm1pC0EEEcjZo47u6pxFgy9LoonS29U06ktkKKe8NQDHIkAHqCRaHeE+MXLurE9vLQdHqlW2o+y4k6EjkQfko2tPiysD+l7HyVZpzOLeqsgHElUC3UgCSYtBlVi2877xShdPetWxR9qlpCad5SMzYOp0I31M76i0wor0rp1P5iERPe6WrxuEriG9EaSMxgE9UoB0bTbsKBEiPC0C8QeNdTcF31mIWX6xF10kBCKKnDz78kAEDxmdwABJ2myrwg40tY7oKC8WatVTd95ghlx1sIeacBKcqwnTcEc+RBNhGUXtbS9roohcBe+u9lMwVt0tgWBBMTZHvy/WbmueqvV9JUilbLmUEAqjl79BZlL4wUVMGzeFIxThw5UZ60JzmJgSkTpacrmwgF/VNE101w0bKTgoeXutsKmzYwtjagxOp5mlZUhTKAskuJUCCY3FnGlU87Mx4eLtUnMLDYoYL00i283jYHNpoffZFqcYXDRVTlFVXxStPNmFpWuMpjbpaRNhcqIBJsE4Qvxt12nLWyTdt+3Veii3Q3jS1CwJIaeSogeIBmx8uAJJ6Wjo4XCRBG6MhYI0Nug552KNuhxMwdNLbFQlSsk6i0bJw626NBca/G2+0ESDYDtAlOYnztpLgUJSZFoOBUr32RgOHQcrdB0cjYsVgak6W2FbAffaNlMWRrtTOp0tvtZ2sWzRpNszQNzFopWBRkrJtwSDvYDOTOsm2dpOlokJWshVGenwsC4lpyQptKp6gG2FZOwFuVHxFmLR1UgbIs5d9Co5lUbBPXsxYBy6aFcEtFMbAOKEe4Gx1S9OfvtwVHrYZiaRsiteR1RBy6KbKQ26+jxDpP3zYE3UuZbvGo/3ggj/wANlJXntbge0PHqLANHC73UYSvHVJq6KuT/AHda0emZifuULc9leaDr6qv3qR+BspLGm8WC099oeAiPRS57kTCq9Gpo2yR/K9+YFiF01N7uO1aqi72mlKdzBvtpIERuAQfZ+dlozzsVbHZ1ixMZxMeUfrZNpGxEFqfnFwsQuku1Ud6kPjlcB/K2zUFMzSviOiQfuNh9hPI23mI6C1sMPdBzIsa1gaKDqfNpX5W5F40JVCqpAIGx0++xs66m2oBEEfGzZXDqnuEEKmncEofbV5LBt0TOxHxtw5S07ghbLavEoBsAu67uUpR9TaBVuQmPutL2lEkBF73fTTpYfUFQl5I0GwJi3AqTUKLdNBy6KXulP5mxK/8ADzb111KKF51h3IooWXFrSkxvkKoPPSx+jQlFM0lsQnKIAEDbpZe0XEJHKBcJxXJdAoUGqcSS877RV7UfhZaU4lLJeUFEIBJCUlRI8ANSfCyNTX6yimQiozKcAgmOmxszb34p4swuHKm8OHDt4UZeKGn7rvBDxSiCQXUOJbUk6HRGfWAJJAtRq6uKjYZJiQB1sShCGSd1mjVP59lQPr9QVN9yAgq7rafHlPU+7laI+K2McTX+4jBeBqRkpSUu19RWLUhuqazQqnbQkFxaTIK1JgZQU5gVaNzHfHS+r4qkYWZYprjdcUlwtF9K6t5sDUJSYCYJAJKVRGw5NK+Me11xXE+imu25Gqm8nclQusWXVPFCRCnHFZlOaEAZpCQCBO1uGxLH5qh7Y6EHlm13W1Pfe1lsU+FuYzNPa/a6BxIxeqLjq7txbgSr7R4LNXeFE4zeLZMABRp8yGW4bSlKUEKKQEJAPdJrVxBuHhPhk3beGFq29V0F4uJZdK2i26l7RxLjza8qUoJVCUpQkQonWbTHe+P8SIoj+47yw8qqfQptCKa7mlNocglKisNhORJgyVDXSCTBhq/sRUVVe3bY1fqsT3wghCVslVLdhEpIBQhPaKzRrAbcJSe6op0yZJ3SSaA27KzJE2NlrglRPWcQ+J1BVv0NJhrE1QxTuKaadX2S1OISYCirIJJAmYE2y06Lq8HKWpTGFwGySUdndmdGXllUrMpQ6EqJPMne2Wh9rRf2H5/6qnyH916OVqQHzppsALcIE62Hrx9t5nlYJI0BG9vf91yKwJPMWAfom3lSpMmxpKZiRbuNNrNdJE00qUoyhO+liyLmp23y+hoBXMga2Vcse628sbizEJxogENZUxZMvnD1FfDRZq2EOJVoQpIINloi2wieVmNjoUrkbKNmeEmFaer9ZZuelQsKnMllIPxAs8buupqibS22kJSkaaWVyhO8WzJ4Wg2NrTcBEL3PGqS77uWivy6ai6bxpmqimqUdm404kKSpPQg2hm8vRi4eVLpWMP0qSST3UZfutPcaxtFtKbSrWLJzc/UhRY7L0UPYH4HYRwXW+vXTc7DT8QHMuZcT/MdedpSp6XI3lyxY8GEgyEC2w2BpG9kxuTqnec5uoF4k8KeIF64mrL6wtxRv+56eqyqFIy6S02oJAOVIIEGJ8ybNdnh56QFGqGuM15KTtLlChz/xHX5Ws+unQ4e8mwPqLRPs2TuYTo78lJpYBq1JdxU1SigYbrn1PPobSlbhSAVqAgqgdTaOOJTHF6kxKheBK+503VUIQl5qsplKWlySFKCk7iMunUeNpgQwEbW4dpG3ozAaeFiiRzfuoRY07hR5WYWq0YYqP3e22quaYK6cKkJLidUgnlqBZtcO8R8TMR3w5SYzwpd91sUqCpLzFSVKWskDKEEmExOpPIeNpoFIkJKQNCLF27saadLiUJBO8c7J0rzZIMYNVqnZKWwmZ05WijHPFCiwNit6467DWIKxValt1p+hou0aMjLlKiod7T5i0xJbCRBFk+8bjpq91LrzQUpBBEizGQtGgT5Gk6pMw2y6mmaUtBHdG/lZxIiwNPSpa7g0FjQSRzm0C6+6f4LShzizU4hKabuJx2ocQ022tJUtaglKdeZNnYYsh4uwzdmLLhqrhvilRUUlWkJcbWNDBBHzANptcBuolt9lWbi5hb+0NPRYtw0609emGyirplMrCs6Uk52iRyUhR06xZwYUvBurTQXi1HZPlpaOcTH17rSZgvhXhzBF3O3Xc13IZZdcU4oanMowJJOp0AHus0mvR2w5RYzTimmerkZaj1lNOH1BoLmQMo5Axp4RZnyMcwA30U2scHXBCk5m86e7KL1qqUoIGVOgkkkgAD42WKG8qeta7ZkrAnUKGvWzVxlharv7C1RdV2Xi9d1UooU1UsgFbakqB0kEco99g+GtwX/cFy+o4jv6pvaqLhWp99CUGDACQEiIEeeptBzm6AJBpIunuggOrIPSwhiIHnYNEdqf8o/GwmsafLWyBsmUe8RWwa2hOolKwfkbVpu9h7CnG/FVw3o4p24+IDSK6lBkJRVtMIZfa/30BB5bG1iuN4xVTXNS3hhG4UXvVtvFC2FOBADZGpk7nQDfnaPb14dYgxvgakvG8btbu/EFC4m8KZtDgWWXkkkIzDTVPdPLXnFoyRCSa7joRbzBR4pTHFYDUG6eeE5ZdpEFZXka7PMRvEa2XeIic+Da1U+wEKMf50m0acL8UYnv/EJu+8MB3ldLdOhReqKlYCcwMABMAkmZ6AWmLENzu3xhq8LsSUhdRTrQgnYLju/OLRoGGJuV56p654kcHN7KMsEU7T1Q8hwA90aH3zZtNUzd3Ymr2WBlDd4laUggb5VE/P5/BFw/xMu24rxdu2/n03Re9FLNRT1vcSo9QduU7+Uiytct8XRijExXQXlS1L9W+H3G6ZwOZQAkE6Exokb9bRnJhhMLhrdSibzJRKDpZWFu8l5kJUPaEm0GcQsMrwtjxFfRMf6hf4Wp1KRoioSJJ/3hPmSbTrdLKkspKkkEjpZm8ZKdlGH6etdhIp6tKlE7JBBB15C12Ih7Sw7EKo8lrg4dCmlgO7BV1qXgiEsgJAiNf6WlS92lt4frUoMKFM5HnlNmJwneYraapU0tK8jw1SQd0j8jaTapkPUTjRBhxBSffahRMDNSrdY/O6ygS57mpL+p6a7atpDjTyEqKSJnuzYrgPgbeeE8YNLu7FVYzcFPVmsau1tlATJObKVkExm6RoI8bHsIXi1SPth1QLt3vrpn0jdCgSnUbjrZbq8V4hev2pF21hYo2Q2lkoQlXaSDnJmZ1Me60HtEBkDjrfZGYXTZMu1k/wDHbRewXeydz6o4qD4Jn8LQNizAmH+Itz0VDiOkS+zTuJqUakFKshTIIII0UbWEvtPr9w1zAIV2tK4kjrKTavl9YWXjnCFJc6LyvC7Vr7BwP0T/AGToKdwFDkdbSrx+oY4G1nfmFGhP617SL6KRuBeB8H4Boai78M0iWn31BypXlVncAJCZUSdBJ08T1tMCFAjS0O8DOHDuBaasdqsQXxe79UUp7S8axT3ZoHJA2Enc76C0wtwIEWFFfLqbqUpu7QWXSzCSqBpaueL2K5yoxExQVPY1ZeqksO5QoocIORUHTQwddNLWLcEII8LQPithTOLb6Z2BdSsH/MgGxpm3pJD2sfqoU7rVTPmPoom9HzF/EWsYvJGPHQu+LjvINtVTdOGe0QUgiQmE/orW1wq6rd/ddS7TLyudgtSD0VlMWp1wu4mYsv7Gl64Oxfh2koK26EJq2HqZa+zeRngaL6778iItbS5a395XMzVEAdu0CoDkrmPKZtVpvZnIcLXsbdFYqdYQQb2uLqMeFF9VS71oqp+red9aZcbcK1qVKoJkzzzJsnX7j7ErHES92k3tVtM3dVtJQy26UILWUGCmYMydT+Ghfh8VUdc3RKVBoryepjOn8ZH3GyTjtgUfFO+GwSDW0tNUjTonJ/02s5LeKZ2JI9QUEnM6B3ca+hVj74vf1C5a280JDnq9O48Ek+0UpJj5WYnC7Hl+YgrUtXtVNLafpi8hKWkphfdOhHKCd/ws5KFQvbCLJUSfW6FIP+83r99oi4P1ZbrroQtRkIXTq8wFCPkLCl/YNkH4h9VOEfrSw9j9FLPEnHC8HXRS1jVIKl6qqU0zYUrKlJIJzGNSNPmLHsDYrcxVcpvB+kRTutvKZUhCyoaBJBn/AHrMbj80teC2K1IJNHeLDpKehJT96hY1wbfUKK8KYHRLyHNv5hH/AE2VS3I2Nw63B+SjCMxeD0spTzaSTAs0H+IN1NXjU0CBVOrpnC2sobGXMBJAkiY2s6sxLR8rQPW0zSsR4gut1RCHKh5C4JSQHBMg8tDYVQTHSvlZu2352U4gHTNjdsf9FLl14zuutqE0peW24vRIdbKJPSdvnZwFYgkjYWptga5MUcOuKNXhC8MSXneV1V1CirovXX1OlohRBAKjpsqYiRl6Ta113166+5G3s5Lq2ilRj+ICCfiLAhdJzBFLbUAg+RRZGsDC5nQ2QtBim4rxvB26qW+7verWZz0zdShTqY3lAMj4WUKioSykLV15m1P+ENcbs4q3U84YUt9ynVP+NC0ffFpW9J+sqk4GpW6d1xLTtXkdCVQFdxRTPUSLaXhCap1MDsqgmHIE3dTQKlCmi6NR52xp1LqSocjztSHBHF298J4GxHhlu8HGDUsioonkKOZp0KSFhJPs5kbbQU6b2kL0TsXXtX35f103tfFdWBVMh5hNRVLdSgIcIOXMSBIWnbpau6LLEJfOyKJLyZLdLqzi3ENpzKMCwYfaV3QseVos488W3uGuH2lXZSt1F5VoV2Ad/u2wClJWoSCYKhA0nrpatN1+krxfob7pUXlfgfRWoDjDdRdzaWXo3CSlIJGm4V4WC4ZCARujtOa5HRXqJ3iwcmY1+Nmxw1x3ScQsI0mJKdsNOOZmqhnNPZPJ9pM8xsR4KFnOZJ0982YG4upWI0K0TpYBaQKhtXXSw5sDUEwhZ/hVPu+hZiEhoULyg7W2OpFsgeVtDQWkm3Wz+FtGPxtsxbRjysklzy099sAG8aWw+0BbfLeLOAoE3QT6M6FIOx08bJd2LUujQD7TZLavAgxZYKQQfvsjXelbVZXU5QUgPB1JPRQn8LORqCkDcI3zsKxTJrWKilISSUZ0Trqk6WDI2jnbpisp6BZq6peRlCFZ1QTpHIDU68hqdhNq9ZE2SFzX7WSY4tddqhniRhbD143k7dV9XVR16ajI96u8hKikAZS7CtgkzqNdY52jOvwLetY0wrCGOr0u5EuEu1bKa9socVmJbS6QpCU6BOVaZAG8ZrTPWXRWXled84irxUUZqXQxT077Ce0ZaQhJQTvBJVmyzAJJUCRAZ2Kqegw/hWqqL5YcNG42pIVUBZ9ZUUmEFQEkqOm0axoLcuGxTh0j2hrei0XPdlGtyojvPC4xxfT12VN90JumgQH2001OA7VMKAGd5/MokLV2gGUoTAWQkQCqrvEa5r6fxRe9z4rqTddxpqVUFJV1N6rWt2hSv7HMlJU6lgZEqg5UGCRBM2t7dFNdd0XCqrvGqup29rzfFdXBx1CWqZRAgBH+AAJSk8xPIm1M/S4wPi7EGPX2MMvu3hddV6rXNGnqszBdUgNrWsiQpcok7kZwAABFhU9JTMcA0ht9z2VeWRwZnOqkWixXh2moqemp+M16sNNNIQhpm63ltoSAAEpUTKkgaA8xbLVcpMAY6apWWximlZyNpT2YVIRA9nQxptpbLVjglJf+s/l/oheOk/CvoEvBMPTE62AQkAzEWjJXpNcMa5wFKr2ZkxDlIlUHxyLNjjHH7hY4QF37UNHnnu6o0+CDb03xtOffCyjSTjXKVIoEamTboJ5RttZmMcZeGFSQG8YUqSqQA4063/4kiyizxI4ev+xja5Aeiq1tJ+ZtIVULtnj1UDBKN2n0TjHW2acjZMpsWYTrCE0mJ7ofJkgN1rSj8lWUWqmlfEs1LTk80LB+42IJWO2Kjy3jcLsJHPa24GmttgE+Xhbcaa87PcHZRylcgTrFtRtA8rdwLYUyI8N7K6ViuIBMbW2EzuLdRrPW2ERECyTrmOmunxtmWNetugI52yDyA8bJJc5TvBtogWEE87aiNTZJLgJ6C2gPCwka+dtQPKySXESflvbUA8rdlOultEacrJJc5ZtqOYGluyI1toA6aWR2SXEc7bjwtsAk2wgWjdJaiLaIBEW2QBbDE6GzgpIMtIJ21FtKZSSO7YWOVsI5RZikgSykiMultCnSjUWH35W1zsyQNkABD0EfwfjYX8baI+2TH8p/C3QGvPWyTlAPUyX0kOajysXTd7baSlKNDpZQgRbUCysmuktFz07LxeS2kKPQWNhgBMTY1E7eVtEADazAW2Ui4ndMnFfDTCuKyF33cVHWLGynmUqI8iR9TYDC/DHDeFVH9zXPS0aSdQy0lE/Aa2fkJOhBNsygGY+NoOjDtUVkjmiyLsslCQAnayTi7C114vuKouK+aUPUtTGdMkbGRqPEWXimTppbMsHwPhaQ9kWUNL3KZeBOH9yYEu83XcVCinYLhdUBJKlHmSdSYAHuFnaWxEEe+w+UDlvbCkchvaDbhJzs3RQfxH4I3heN/PYtwPiN65LyqB/rKOyDrD56lBjXqffEkklcE8PMd0t5B3FmI2a1ts91unpAyCeqjJJ8hAtPBaSdFJkeNuPVWhqEi0Jmmc3cUWGYQiwSa3Qn1RTJ/iTl1tWu8Li443DVOUF33Nh+romV5WVLU7nLYOhJkCflrztajsgNIsXeu9l7vKQCedpSFz4+XpbzTxSNY7MoJ4WDidUYqYViXD933fQMpUtxxqoUpazBCUhO25nXYDrafWwYG5sWauxlleZCB7rHEJKdBYETOWLIsj+YbrZEp38LQBxjvqmwjixdZV3deLqK5hCgqlpVOgFIglRGg2HjawBTI2snXnc9JeYCahlK42kWMXHluZa9xZDY20gffZVWuLEOEL2xGitu2769V8OseqCKF5J7PPmhRjKNeaojr1srhmiXR3PT0pn7NEHzOp+Zt1T4Ru5h3tG6VAPgLLrFIlpOUDQCLU4YnMkzu7d7q5NM17MoVZMRX5T4D4k3xdF8u+qNVz4vGhecMNuZzqJ5d6R/unwkDEFaxiHFbOI0VKExQpo1NpUF51ZyoEEe8R4/GfsbcOMM44pkU+ILpp6zsp7NS095E75Vbj42aGHOAGB8MXk1eV23MhD7XsKWta8niMxMGx6uUuLnR3BcLHaxQ6ZrQAH2OU6d08MJUr9LhmgpagEONsJCgrdPgfLa0L4OIuvFTtIpYT6pfjzQA5J7Yj8TaxLFKGmQ3A2i0dPcGLkXjhzF7j1couPeserCpUljtZnMUDfXXXS0S4Cm5TvL6JNH6/mDz+qNcY6T13hxfCAnVppL48Mi0q/CzU4YX9S3W+6uqqW226llCgpwhKZB6k+JtK9/3I1fdxVtzuylNZTrYURqRmSRNoAe4KcTKFfZUHEAFpCYShy7kK22gzP9LNUyt5LBfUEnr1+CVPGTI49CFYe7L5oL0bUmkrqZ5SRJS26lRA6kA2h3FQNJxKvFrZNU0y+ByMJCSfiLODhHgvE+G3amuxRiBN4PuthltDdKllCEyCVGCSSYHQCDprokcYsNYyOIKPEeEbupK3/VfV3kPvKREKJBEAzv8rMyVslNI19hcH/bdIxFszC3oU3cfVDC8f4IrGWzndbrGXCOY7JJGvSc1piwmvNc6G9SEkgSPGbQncrHE6tvBinvXB9EyAYNQasKDaecAJJMwNLTzcF3+pULbIT7KdfE8zbPjnfPUMcRYNbbe/W6tuiEUThe9zdVKeAw1xXVJyJor/UroAkPZh7oNp34/wB3/vDhpWPBoLVRvMVAA6Zwk/JRtFPF/B9/ucTqtu57irqpV4OtvsraYJaBKACSvYQpOsnTe0746uypvXh1fNGtuX3bucISnX7QIkAddRbojOzx7ZQd8t/QXWS2NwpTGR3Xn3ftYbuvWgoH0j1a8TUUi9IhyMyNtpCVj3i0w+i/WLu7iNR0jjgJqaR+mJ2JypChPn2YPvs06Lh5WcSlXpS3RSrefoGFVlK8EnK3UoUCElWwKgFp95PKx3g5UVl3cSsOOMsrU6qtbbcZSk50BRKVynlAKiegtWqWBocGdCiwuNwXdQpY9LiiccauioB7qqd9M/ylK21fdaIb1fwtfPDvCF2VV41VLfNKmpcDrCEHIgPKRBKtJICYAk6eU2K9Jy53KzA9NeyEHLQVMOq/lbcTln/iy/G1QLzuK4b/AMOLuO9aEmtpKlD1G8ASktlaS42ecQCepk7WeVrS2KXyIv2P8lSjJBkZ5g/FXJ9GmowsxhGpubDovIqpnkvVblaU53XHEwFDIYAhEQANvfaYvdatfov3tTKv6+bvpyAl2lQsACAOzcKY92ci1kxoOVsuJ126q9KADosJHP77AVQzMqGnL4WGJ1twsSgpESQYsRBzarG1lbSFdUg2EEeB8bA05CmAAI1I+elhYV0Is4Ce62bagxt77dAaQbaJy7WlZRJXMAqmLb2Glt7iba5SBvZwLpiuY1MCyU6OxvgQe6+zr5pP5GyuTOwslXolxFVRPoKQkPFDkzOVSTt7wLScNFFpN0YIgQbJ7CX6m+fWlgJpKFSQyAfbe1zqPgkEJHjn6A2USDytxlI5DefnYcsYeBfZO19l3jOiQpsK0ylvppI+haE8YXbTLcoKd+obqE9uXW6JiV1FU8hOZDaRIECMyie4ANSBNpXxpemKl0rTFy4QFclTIPrL9ehhsK1BToFK/hB252izHNPiS47kvG9KS71esmjQqp7FC6kl0gAtoUAAhoLKTqADClnNAA4rGsSZEwQROyuPl2PmtCmYXNs7VRJflDU31jA0GOb+au676ls+p0LKyAlptAKy9UgEJBzZilGSBGZZAskXxQXXiU3/AOpYJum+HFN1Cnr0pMjibvCSlLYSjUqWoZnDIbkqCR/NZx4wuu8bwbZpr9vJdBd3ZjtKRkU4qVqJCcqUZ1IjdWZKiUyBBIgw5jCmcN5vNXZxITR3FSoQ4ijpqgtLW4pJUYSRmcKUhGpKyVQEidEcPWTVFZmZKbW6kWN/l/8ASvBkYHshOimuzhsxTNMP3ReK3G0JQtSrzu9JUoCCSOzMeUnzNss3Kb0f6Ospmqut4nVtNUPoS46yXHFFtZEqTI0MEkT4Wy2Xni/tz6FQ8O78H1RhNY+y4QFGIjVW2v3H8LLNFfPaQFud4AST9dB9a2btSFdrnQqVRqeWnhYamUkAJJUkgHYmRz39/wALewOAcpNOqejFah0ZkqMTqfr62sdbehIMmTBMfXnZoMVTrSigzAlOg10n87K9PXuI0KiUke1MH5fRtXc22yMEs9nSrUQqmaIPMpG1u0BhtQKGW0q5QkDX6+6xRqqSSARmG0bbWHS9l9nnuc2pE/OwtQp2BSnT3rW0wzU9fVMiZlp5SfgQfqbKDGNMWUxCWMU3wnwFc6I6/wAVkDNKQFAmN+ev52EClR3cse1JO1pCR42Kjy2HcJ0U/EfH1P7GLr3nX26tax8ybHqfjBxJZGVvFdSemdppfT+ZJmzJCiQdTvvM/wBP1t0SSAQowZkHmT062mKmZuzj6qJgiO7R6KRGeOfEdr2r5ZeGnt0bX/SkGygz6QmPW9HG7pdIg9+mUPmlYtFxKpEHXMIje28ygRm5iARvEWIK6pbs8oRo4He6FMNN6RmKgkiouO6XOfcDiD81mymx6RV4AhNThKmUTpKK5SdekZDaD2lQrMdBPMbfX1ysfBKYBTrGuv1rZzitWzZ6Qw2md7qm9n0iKQn/AFjCbyNAe5WhX3oH0LHGvSEw3P2twXsmP5S0r/rFoHTmyZhGkAGBr8DbqCI5CNfD66WcY3Vt3d9FE4RTHorAs8e8FuR2lDfDcj+Jhs/c4bHWeNnD50Su8atn/PRuH/wg2rgJMQZ1BGltnvCCPDfexRj1UOyGcFp+l1Zlvi3w8dgf2jQgnk5TvIPzRY61xEwK/o3iu7p/xPBH/ii1WAo6grgb68zzFsCsoP8AiPPaxW8Qz9WhDOBxHZxVs2sXYTe0YxPdK/8ALWtn8bG2ryuypANPeVM7PNDySPkbVDMd3MoaHZQ8vlbOzAEAIUdoIEWIOIn+8xQOBN6PVxQUqEgg89LbynkPjanaXVtGULW31hUQbGWb5ving0971jf8uV5aYPkDYg4iHVn1Q/sI9H/RW6KTJFtZY2GhtVJvGOLmigNYmvgRG1a5G3+b68rH2uJOOmYyYqrtP51BWn+8DrYg4hi6sKgcDk6OCs/lMwdbaKTuPK1bGuL3EFnQ4h7Qf4qZkn/wWOtcbseNe1UUD3/2SlGv/CRFitx+mO4KEcEqBsQrD5T0tgGm1oFZ4+YwSMzt33O4B/6l1P8A8Sxxj0g75SR2+GaJYIIBRUKRPxCrGbjVIev0QzhFUOimlxJ7ZszMggfK3ZSrY9LQur0hn1Os/wD0ObVBJXF5ERpH/ddbKLPpB3WSBV4Yq0EnXsqlCwPiE2IMXoz76G7DKpu7VK8RrbQGto2Y4+4SdIDt0Xy3J/7tlQ6f95Y+zxvwE8AXKiuYJP8AHSKP/hmxW4lSu2eEM0FSPcKfURbIP9LNBrjBw6cIT/aAoP8Ajo30/PJY4xxKwFUABGKqAToO0UUf+ICxBWU7tnj1QzSzjdh9E4wJ3FsG4H4WSWcZYPfVDOK7oWZ2Fc3/APNY61fF0VEFi96Jz/JUIP3GxBNGdnBLlPbuCjI21FsCd7Y2ttYltxKh1SoG3YSQNvlZFwOxUC0rjeJ1thE/nbrL4fK24BO2lmuo2KDibb1626iNbby9LJKy4IB3GttZRGthCCP621Gu+3jZaqbWhckcrZHUW6KbbgxaKKNFxHhbYSRpuRboDlHnbAOdknWgmNxy2t0AI00tgBt0OtkmBuuCmfM2zKnpbvfSLZrzsktlwAJ3tyECZ3sJEaW1ubNZOuSgEac7BLpm1e0ka+FjECNDbRibRICcEoBFMhGwAtp6nQ6IWgR462MkeFs5WHlGyJcpOF2MBWYNgEc4sYbaSjQDSw52tmUb7/jaOVrdk+cndEai72H3krU2CpOgJG1hDTp7MtkAg6RysYAnQDSbaUNdLRDADdTDy7dI1NcFHRqX6swhAWSTAjWxGlwfc9Bebl50V207NQ6SVrQ2ApRO8kWcsDnNtRaBib2UxI5Jt43VS3nQPXfW0yH6d9Cm3W3EhSVJI1BB3FodrfRmwMK4v0lLUNMkk9gmoX2YJ6CZEctdLTkYCYgWDKBOoFk9mZuW6THZDeyjfhjwfwtw0DguG7yhaxk7RxxS1BEyQCToCdTG53tIewiwhQBt5bW0UwNDpZmR5BZO95fuuDMW3AI0FtwNhbYkDfWxAENFqLQut/yuGPLb7wbGfd8rFmTlrXkke0AoeX9SbGdedkNk53Wokb62ydd7bOmttHlHntZ0ywzytrxtkwbZPW0mhRcs0IsQvZJVSLWndCg4PcZse5RPOwD6O1ZW3tnSU/EWJluFAnKQgjCoKeYkW5I05CbcUiyukaKpnKAfMW7IJMAT5Widk/VHK3tXMJ1HqxAfazJaJE5Vbg7jaeotEGNaaqrLtp0Ut5Kdok1jNW62z2Z9cDe4ClqAE9RMmBItKt2IcraKpF4IAgB5qmmQhQ2KjGqttNh4xJYGMWWWqpw5TmbJVoJkR8t/lbka/CnVdQ6cGwNhtrvdaFNM2NpYdeqirHd0JxFftyet3Ez6iUKdrCX0NvQkpLbS3QFKQ0SFaNyVd0EpSSosniK9ebF0V173Pwwweitu+lcVR1jzyXhTBptXZrDRYEwEghExMTGpDmvnE7EqpbgoV3nVpKlPNUakpbaWZUrtHVEITrpElWvs72RU01e+07U4leKUFQCaKhGYKzQOzUVCXJgiRkEEyNMwE7Aqfl86c5n28rBIVbs2RgsFUlHHy720Jbq+KOJy+kAOlm93QgrG+XQ6TtqdLZZ5VPCJpypdXVcdsH0TylqLlMiju5KWFTqgBAyAJOkJ7ummlstzXJw7+T//ACj3l7/z6p6vMKDiypoFIVOoM627YaASkxOXTQD6Nj1c2xTugPupaWTlAUcsnoOptzTBC0gh5HmlYI9318bdO2QvFwtLlhp1W22sojKVaa8p5zyj9fGwyEEahMkcyOg2j5fWoqGgBCFQoQBlEctvjbpLKIlBA58oAHL5T77PmPVPou2HVswhQG4PdkSZ8NT9b2OU78Zc6CdN41+t7FEhJIDago6b8tteXK22s0KCSUz7QUInXy+vjaGpSSu1VGZJnKNAZJG/L3WHDiVpIKlDKBJ31+tOdkZLqwc0gA6idR47WNJdcTEkmYMx8v687RSCVtV95JB0zTqY/Pfl/XAAkd5ZI5iI8rE0PtEwk6+0DO1jCHCAUjkfEH5e60SpCyFGqTpm1kb+7yt2EjN3iTroCev1ztwlSCrIpUknXW3UAbFJG+nPTx8vlZrpbrpuCQlKjpy+vr7rKTQ1mTruJ+voWS0kJXIEHQyellNkKKAACOcbf1sF7kZgQoHUASN9wI6bfXyEJTEEabROu/TY24AASCJ1I0H3/XW2tB7KQCQAJE+6LBuirtQjQEAARI8rcZhqMokg6CTy+vnbZSpAgnU6687crTExlISDpuZ/KyDk1gsBzEgHQc1DSfdbEpIQpASCfDmDbSEkgJKoB303t2MmYaZp3AtLMkGhYlSo0gd0kHf6/W2wNFAJmIAg6c/r+ttxlkqOsaHLHTT5W5mQSBG/ws2YpWC6lE5lctDPzI+t7ckfzAa67bW6IMgSok676D6i2iTAMkgmZ+uW1ldKyxKdNRMGCAeUf1tsAqGgB05jl9RbekQIB5eX191taKSQiIEbcvr66WV0soWZhIIM6dLYrNIza5tYM2zLGUlWoMk2xSRJhPtacoFldSDViimekHSbcFZgnujTYSBbsqBWASIA2Go/p9b2CVqsJnTdWg1+tLSuolqxCSo6JAJOoJt2QmIJOmgnnpbESFggjaBJ1toJIGUeYj75srqOVdApVMp3AEHQ9bYVBJJ3InX7vz+Ng8pAJzSd9dvDxtmaSDlk7jWyuVLKhAsEwdSnvfPn9fpiVTqFnaZ+vre3IIkHeNNY+uv0Lc5iE6ETAO/LpZkrBC5gIyzrpz3+ptsJQpJCgCZ3KeRsFmSnRYOo0JJ20n687YpRG87xp+f1+b3KRaChB2SU5kIAIOkCOutjTN53gwJYvGpZH/q3lp+472IIOvdJmbdwnKUnSNuf1+tnEjm7FR5bTuErt4sxPTqHY4lvZA6evvR/4rHmeImOaecmK7xPOFOhZH/FNm2NTmAnfbfX+ltqASMpT5na0xVSt2cfVQdTRO1LQnizxb4gtSDiNSj0XTsGP+T8bHmuNOPGozV1G9rB7SkSJ/4Y8bMILOo1BG8C2s0SVlJ6nwmxBX1I2efVDNFTndg9FJTXHXGyIDlDc7hPL1ZwH/754HlY4zx9v5J+3uO7l/5FOI+8m0UpVG5M66wPrxt1nUganTnMT/WxBidW3Z5UTh1MfcUwo9IF8aPYVbImJTXEf/DNj1Px9utWj+HapGsfZvpX94FoRzq7syORG8npbRWdYVpEnT7vjaYxirHvfRQ+yqV3uqfWuOuFF6O3bezXU9m2Y/57HmeM+B3P7yorGZ/npyf/AAk2rulzU6AjlpGvS3ZdSpBHOSJ6+FiDHKpu9j8lE4NTHurJM8WsAPaC/cp2IVTPDX/hix5riDgl7RGJaIHfvrKP/EBar3aHMCCDOxjf7rCIdIGYaggxztMcQVA3aEP7Cg6OKtM3jDCTh7mKbqM7f643+djbN83NUa097Ubv+R9B/G1UA8UjMeQ38Ldl4HURAnf8eXWxG8RSdWD1UDgLOjyrbIU2sShaVD/CZtvKQJi1SEPobBLZCYJiAE/X62NNX3edMIYvSrajUZH1iPgbFHEY6x/VQOAno/6K10dBbQEdLVgpsbYoZJCcR3l3dTNSs/jZRZ4kYybT9niGpBInvkK/8QNpjiKI7sKGcClGzgrHQZtkE6zNq/U/FjGyDC78Ln+Zhr78tjrfGXFzYGdyjcJ/maHx0jS0xj9MdwVE4NUDYhToRpbRGgi0LtcbcQp0dobvX/uLH3K+psda44VoH29xsKjcocKR85sVuOUjuqGcJqR0Us5eotoieZnlaMWeNzLkdph/zipn/osba40XMr+9uirBmO4pKvvi0hi9Gff/ADQzhlU33VIBGmltESZNmOnjFhRWjjNeierST/1WNI4q4LcVBr3W/wDOwr8JsYYhSu2eFDwdQzdhTsI05xbR+os3E8RsFuwUX2gctWXBH/LYdGOsIOmU4hox0zry/fFpCrgOzx6pvDyjdp9Eskf0tojlO1k1vFuFqgwziO7Fk8k1jZI902Nt3ldz+jFfSudMryT9xsQSxu2IUDG8bhDGBFtbWwFKtUkK8jNs2P3GxAQeqjYjdFnBlrW1g+2kpj4mxkRysUrF9munXmj7QJn7/usambONUxNlkjrpbXnzthtkzztMCygTdanWbZsJtknW3K5FpAaqN7LaiJgajytwSNgDbZMjTnbRAG5jzsVDJuiFNI7Zo/wOqHuOv42FMDnbgjJWu6aLSlQ+6wiezSorW2VTGxI2Nq8zntjzRDMe17KYAvqhKNZS+J2VIPLe0ZYrrcP4pqn6E17VQGEinrrvC8q0qBk5yCFFJEREAjeQdJGDLS31VGdSHVEEHOSTrt0gWhPjF6N93cS7zTifDWKqi4b7SoLNW0lS2lCCkZShSShUwTGaYGg3t5vinFNZRVIZWU5Yw79b9iCPqtBkLS3Mw3KSX6C6buAo7tom6GiQsoaYbHcZRy2GwB1MfG0QX7iLG11Xm5fjXDRT14MuCgSpN7iqp2kAlTquyS2hSeWZzvJ1T3gnKVTbd/DbE+A7ni+MRVuNXm21rSCmmo1TkyhGdwqKu8TBge1OuVKbIVPc98X1f4ddwwu53aNDYXSdnSuJywCVOOpRqgRAyqT3SkhIKotm1PFrcXk5FG7Kxgvqd/krEVKGDO/cqu9XhzBXrT3r18OCp7RXbZjQznnvTFQec87ZaencCcInHVrvLDuI3atSiX3P3LU99ye8dFDczyHkLZbmTiEn4/qrfJZ2UEftI33U8LMLpbzZlX6FEJkHSldA2PVW3jan3CZ6sfepj6w8Ul5CsvaqP+0gzPn0tbf9pQ4E4DwjTSqV3s6rQnZLA/8Am+dqqcHWypqgfcKSXX946rBgdfZ1529c4QaHYbHm7n81X4jcRWvt2H5KcGDVpKeyqahKwdAl0iROosZTUXvCQq86uZ5PLgDpqfxsPStSSE94GSABqB1PTUfXJRSykIzKmQM5B1AjnP1tbuDDGfdC5Zsz+hRVF8Yka2vyuMmCFLKhGvyj7xYdGIsVpBAvd3MNO+hJ225WH9VbR3Qgb6aaz5fX32EDASTpJOgEwCR4n60HuGaWE+4PRTFTM3ZxWmsVYpUYNchagR7bSZPyjlY03jTFDSFSulWZBgtnQ+U9fP7rcinggKSSddSrU/R+udteqIVECOQgAwP6G0fAUzt2BLx1QNnFHG+Il/tohdLRKI0EBQ/6uk6x0sbZ4m3sClH7pY3mEuESfgbI/qiHDlWkHaSEx03jlbfqSNAY0Ox2+Xu+NmOFUjt2BSGJVI2cnIjinUyUruMGU7h+I6nVNh08W6dB+0uSolPPtEmDPu+hys1jSlKTqTr4TH1tNgXqFsCQOW4VtEfpYX2JRu91SGL1Q95LV5+kZhK5axFLely319ogOhTTTTiQJiDKwZ+uejvoeMWF6qlaf9XvJAdQFJzsp2ImdFeVqocWEinxDThIyzSpKSBzzKHPUdLSvh+kDt0UWYAlVM2TOkykdLVv+HqN7jofVWRjlUxo2UyJ4t4TGgcrASZ1p1dfxMWMs8UcGuTF4upI5KpnAfjl8/h0tE/7sSoBKEEGZmNTr12sILsBEAk6wnuzO/6WieGKM7E+qccQ1PUBS4jiHg5+Ai/mEE6kKSU6mRzFjCMb4UeKcmIqHSZl5IA+J1O/0bQybsSg5EoiRtPLa3H7ubkEpAChB0/GwTwtB0eURvEc3VoU5M4kw+4nM3fdEqBooVCD9342NovC7lE5K9gkkbOpJ8Ofh8rQELoSshS0QAefIwedknENxtm7RmQlPeESRpod9PKwH8LMtdsn0VhnETydWfVWaS424mAtKgDuNdbdJglIBgp1MdPL3WombmdQziFpl5xslztFFK8pgoGuhkeYtxRnEVyM3PULxFezRV2jroTXugSoZtRm2HIHaLUjw47o/wCisjHm9WfVXw7oKVE5Y5D7/G2ZSVnQCNByg/DwtQ+6cRcSW27xWrHGJEKfp/WWki9nzkBUqAJXA0jbw6WNU2OeLlHhWorE8Rb+VVracfC3atThQYVCQFzoJGnnaH/Dkx2cFMY9F1aVeiDIVl0gbGdPxtrJJGbXzHyNvPZv0kuN93qpEDiHUu/ZCUuUtOrOqSCTLcjXoeVnfeXpLcX2K1qhuzEKFpQ60h5bt3sqzZtx3UDl+O9gu4fqQdCEYY5T21BV2yQEwpRj4eUWzKN05pnT8LVbY4/8UmUS8u6XNN3KJafHUJWOlktr0rOK/wC8V0P9nsNPZqhSUSw+2UtAmVqh48/dobDfgNW3t6okeN0p7q2/gNYgEEkHr+Vgm8szqSdZjraudX6TOL2qbtP7L3WsIGZcPuJzCNucTprrbnDvpY3retEqqrOHjNOQ5kKG72KuQPNoeI91oHBatvu/VTGL0jtc30Vj8pKd4EQdeXS3ACiddR7Uc7QBQeltQ195uUX+j+vS2xOZ5uvbWJHISlM/Gxu6vS0whXVr131GEcQsuU4KlKSGFoVEbHtAT8LDOF1Y9xT+0qX8anZUiAnTpI1NtGSABKdjHIe+0StekxgRZLbt1X80ZmDStKHydO342DX6VPCBh4sVd4XmyUqykqu1xQB107kz8bQfQVLN2FTFdTO2eFLplRKQSOcjrFtKWuTOgnQff9eNoroPSf4J3k+GWMWPB1WqQu66sctdS3FlD/yguDSBke4hXc0oGYeS437u8kdRYZpKgC5YfREFVAffHqpDhwKISNTpH1v526J0BKSYOkdbM2l4ycKKymFWzxEw8WUkhS1V6EpEbyVQPr3WUKPiJw+rkh6ix1cDyVaBTd5Mrny71hmGUbtPoiCWI7OCcIWuRKgO8Z1iPrTXwt2gwZVoI3jX4edkpvEmHKtRRS39drxBkpRVtqI+B+tbHmqhh0S0+hZ19lQM6/0sMxv7KYc0ndGEKExyA5R8/ifnbASNBpE85jnYMLCjmUVQDyPLT6+Ft51JSoj3zr74O1o5SiXC7KgpQI13Ij66W0VBfdEQYg7AflYPtCoCR56HwnT63tgWkmFCY7oCuXh84srKNhdChRKc+oUYInlFuoBJCQf03+vOwQVmT3iddpFtKWAMpUZiAOR3tElTXYMEBManUD4W2kiZ1EQJImbBqckHWYOpiI93mbYXOeXQiRP142dKyGSvKskKII2I++3SnFZYyyIgTrz5x5CxYrBEJCtY3Ig9LaCgI7qu8NtyJ+hZJ0ZClSYMGY6ydOfO3QdITMjXWd/raxUupKYQ4BHLkPqLbU4okk6HcQNY5ffZrBJGlOEKOWTOu2g5W0X0lRgx0IJnXX8LEy8oyQZ2Ik9RYNTwMHbTMQfz921pBqiTZH+3J0J1Ps67a76WCVUj2ZI1kEjymxNVSk6EghXxiwLriUiSRMkxMaeHvs4bqolxSm1UgqmQNiJmxlFVqBPKNN+tkBFQJEk7mJ2Pv+FjaKiUwkEk6g+Hx91mLE4clYVRlRzc80c/wttNSYKs+ug1URHjZKXUlKZKtQYjbXp+FuVVCwJA1GkzH0frxLctPmsln1kxoszrJjW2Jqu7lECBqT/XzsiirKE7r/Tc/XhytgrF5QFEiD5a2XLSzpdTVAzBjwE/D662EFZCQFHNJgTM/pzs303gdSrZJ3B0sImuCQQCSDoJ0+v0tHlJw66XHKtWpJ1O8EwbFzV97cymNtJ+oslrrAUSs6jTabF1VyQoqmPgQdNRZwyyRcnI3WjYrGU+O35/0sp0dWC2uCkzv5WaFNW5hpm0OonWyjQVKznSVaFPuItAtslul01ZOhjadeW02CU8wTq22qT/ACg+f3WTTUHQlSvEEaGwS65KBnBzSJVB3H0LPmsEsoKVk1DTZztISgifZMGfdztp++6tpQaTeV4haj3GqaqdSpW38qhZGpKmrvRR9TSEtAmagiUx/h/mPy+6zjum56an2Gd1Wi3VarXB0k/hZhJI42aUxbG0XcE4cDs4irb6Zr7yv+81MsheWkXWuOtKKhAUsKJkiTEaedprSrMkHqARaNsOsmnbCgIiCOulpCoFlVK3rJSCn4GJ+Vu2wqJ0cIDjclcniLxJLcIwdbaKtPO3M6b20SDrvbXDRZZxK6K+vK2iqddrcz420Tys4FlEm662tkxB00O1uZ1jfythPM2ctDhYpkyKu6MZXJexeua8PXrvdUpTdM6U9ozJnImfaTMxqIB6AAla7Ht+XG+Ka+sN7JBU6jtUAJJjQZFAmSn+KN9ZgF8V0lDbiT7C0k8tDobbzGCOW8Ha3DzcMYhSSukwuscwH3Xe00fC+yutqo3ACVl/PZRwvj5hGleUzUUNSgBJCnEw4rNOgCdBGXc8ttd7LL/H/hXeaKNtm+VMPPN91BaJE5c2WUSkq0IhJJmzqdwzhy8ad2ovq5KCoQ77aV0yCpwjaSR4n4mxSt4ZYCrbiLrGGKZlDlZ62tto5B23fSFEpiYDijHWOlsWVmORxyc8skDbgut9NkZroM7coITHc4i8PxULpUYupa9x9pTgpWFBQ7MQFFal90ZQdhEbmd7GKTCF1YgQ9elLdLTV4OqDK0BwtN5EGEgmClQTKoOU6qMRM2QsR8DuG7d2drWVNfdopG0LVVN1WVLYb3X3iQnSZOkcoGlmNfOFsZXLhq5arhbf9/ppG11TTrdQlxp68u3WEtZE5DlGVKoXmQRJclM5reUYhS+Jma9rcjjobaD6LYzuawpxXjcmFaG8KqiPFCmoDTvLaNIispEJp8qiOzCSySkJiIJ0i2WIs8KOICmUKTcmCacFIIZeoaZ5xvT2VOdn31DYq5nW2Wz/AAE34j9EuY5VT/aTVKW7hwOwTlKq6tUAY5NM/natvCOnlq7ezEg1YOmwEwQfcqfL3WsB+0vfcQxgFgq0U7eSztpCaYDQ68+R8LQTwdQOxuvMc6VOpVBPs98fU2+keE2luGxfP81n8Q61snwH5KeKenSIUsSSo6DkdOtjrbYJC9DmICoET0tlM2pwDMgEkHNInT85mxxLYKc5B08OfXT+nhbuFygRfIrMogaROkR9fl8BEtNgxlJA32P1z+Fhy1kMqGmggE727CTIzJlKo5x0G5sk65CJMGTCgY3M68vra3RbkBAUZIzHymJ+vnbaUFaQsDQp1HP3WFSMqJ3BkwTt5zZDRIhFyjvQgRBInqT08bYWwO7Mge4T+n52M5MyjOm5nr9aWEbQQsyCDMGRyHM2kHFRyhFcpSc0kkecxufrzsAtrKMxIRzKuWn32PKbCiEqSCmZ328PDnYFxAyBKhA9k676nw8voaECE4KC+MrahiOiISdaQZknYnOrY7c/6c5cw22n9y0BKSqaVklQH+FM/wBbRXxoSUYgoSYIVSfxEjZaiSY35fCbS3hVOe5bsWFKy+qM65tSMg+NoM+8VJ/3AlltohJSWxO8ff8AWlhlNKIyJza/ETH142xtGcAHQ/xHafj7vKxppBg6Dw5j62sVDARYtDvEkydYgafgdPvt0WDATCRtHh+n52N9mPa0jXnpP3cuvO2+xSsewQPD66WjdTsihZSknuDMDJkRJP191kzEDBN3EgI0WmZjrufh99l0NlcKRyG2h5fnZPxAlf7uc1SDmQQPeNbRcdFNu6hm+ID94UjIAVUvNZh0SlEk+OwHv5WFvaiTV3TdjKE/aOhCElR2Ck6n4fM2UqqkQ7e15vbhqnyAHmVJ1+Q+fhZNb7WsqruVB7OmUyyY/mKZMaanQD42oZuisWXN4tMXfeDvaITl9S7NI5bwNPP8eVi1DTIXhpKcmgbUCD7x+RsrXvSNVl+MU6oyoQXFgGM2USB46kEjysjJqnKW4W6WmTL7xUlIBBAGYyesRabX2KaygW96R5iuU8rKnt3XXCncNDPlSneTonnpaUaK5mKO67pabgqU4h1xZTBWoifd9RaOsROFqobQ6JT9pAJA1zkpnnGvXmfO0qsJKrtulW5+xPX+GdNNj+HwgbJwNE7qgNUdKupdKMjaYk8z0+VkbDd3pqKytrnE99cJBJG0mYHu+trKNbmve8kXY059ighx9QHtARp16W6uFkNV94JbRA7SFBJ0ETp4WkXXIStYIWtu9kMrSEpKgDB5kRr93xs1qJhNPca2W0DtXnVBuE67DXXmN/f7rPx9nQhxICNRJ67WRrnu5FXU+tQAxTQ2ygwTP8x+vutBx1UmjS6KXZhxu76NKA2jtVELcUE/xfpt9A2SruupCcS1Sg2O8FdwAwNtNrPl1oA5yPa12iI5/dZBYZyYgqFOmMyVHU6RlG3z+hZnECyYAlFL1oxSUygGQlZ0SFcvHwAHhZn1FxNO9pJ75GckmMx1/P52ed4uKqHVvONgpBypn3dfL6ixZm7nvVamrcWpORs5dfaP5D8bBe7OUVrco1SRcGFqZilFYnIHX5JI1AHQTrv1sweJNCG3nBBnKnp7xA15fPcWmS7EpN1sQSdFDeI7367WjDis2EuOOHRIYBJka6ECefIyLWQBksgm90UwTd6LywW6pZzJC3UmEgkjU/p7udnHdNzJorlaZSyM65yp0za9Py572I8JHUKw3Vh5sLCKpUpBB0ypIT56/MWflxU6a19Va7lUG4CE8kk+7p48vGwyAVMEgXSJd+CmqZpx1xtJfdguKjYcgPj9QLGDh7s1gpSUncHUd4Tr4fL8nmtpJMpzZZ0MRHQ/hbgsgHVHKYgyn3i0wxvZQzu7qPcRXZeCbvQpmvq24VmBQ+pMjXXQgDexq5nMVM3dTdjiW90KQ3lzCudSvQkTIV5/W7lvylDl3ryo0RCpP5e+Pf52BumnSbuSkkwkqyjnvMz8frSwuVGXahTEr7aEokzf3EFsns8bX+DuALwe0I6SqOnysYTj3ipTDu42vZakyAXKgrJ66K/rpY+mjSgSUKGUc4M9B91k28AkFNPTpKnFkiAnl08hp8LRfTwn3R6KYqJhs4+qBreNHGthfq13Y5qS7mjv0zC9CN5U2dPvjkLKGKeLvpK4P4YP8RV3609QtVjVGH6m6mQh1xUkpTCBoANSTvpqZhe4dYOwtVXotzF19NUNMyjtVZh331xORJAIA6k8vkL6U+KKHE3DasuCiq6V+nZZSthmkUQxTpbEJTl0TmjTbS1GSlh5oY2MeeitMqpgwuLz6qLrm9NPjY9Q0lRUG4HS8YWpd3qH8UT3XB93I6Wmu5uP3EOrpaaoqqC5lLebS6QllxI1Hisx0tRu5V9ldNIslIWy8ojSSnveHn91rc4UaaeuWhVCSOyABjppz+vxFPQU+bRgVqCuntq4qQk8d8WZYduC7HIEnKtxEjnuTZKvz0pL8uCoZbXw/YqUOoKs6bxU3lI0iC2fz12sSbo2yjMmZUIzDU/H3j5e5r40udp31Z0oH8SQoRqTlPx3tWOG05P3VZ8fPa4cnQx6Yy4/1nhnUJO5Dd7JVoTvqyPqeljSPTIw7/8AXcCX80qJ7rzKxsOpGvhaIHbnZKCQyBlAMeGwEe8dN7AKuJhSS2ltPdGpJ1B5e/UfAe5fZVOen1UftOoHVTcj0w8BKSO2w1iZrLM/YMHr0esba9LzhQtaE1SL6piQZK6AmP8AhUZHS1e3sO0wCXAyArNAGWAI6Ac4sVew5TLBSRrsYJB8/vNm+yYPNP8Aak47KziPSq4MFMrxPVs6AJDl11I8dwg/A9Ldn0oOBzsdpxBpWCTH21M+2RoN8yBaqFRhencHWYT06xJM+P0bNHG2Eg1hqurWlGUNpcEpTEJUJjTeN4jX32k3BYCdymOMTt6BXppfSE4L1GVTXEy4UhegzViUa7n2vMWXaXjDwwq0pTT8QsOLkaRebEn/AJpB/K3lGEg3b7ZBafIAgScw+P8AD5Wku5sPMVVAxVZh9oylZSoSACJM69BPOzTYHEwXDinjxqQ7tC9KGMZ4XrUxS4kup6RKcla0qekd6xxN7UTgJp6xt0aiULBnfkDbzeRg9p1tY7NuFK3UNZjl8rFqjCDaUEMpCNQDoRrPh4f152q/Y7fxfRH+2T+FelqKhS1d1J/m2NuV1BSE5ioHkZiPr63t5nuXXfNOgooLxq6dwAlPZ1K0kHYCQRH9LZdNZxTcqEM3TjXEjOyQUXnUIAHnm05+FmGC3H3/AKKQxgbZPqvS4ViidVTqBBM/1sIKs5j3zoNzbznqsY8WsPIqUq4o4peebpi73r3dUEHKCAApWuubWOVvQv8AZ3XhT8WeCV4V3EBIvy97txA/Rqq6wZnuzNOw4hJOhgKWuPfajX0JoYede4urVPiLZn5Mtij5qEwUp0OsgdOmti5qiFHKdxsTv5bWss5wqwC6sFWHGgRp3XXBpHgqxKp4LcPHUn/sVxGsyiqdH/VbD8aw7hXuYCq/U1WlR9rSPM/W1lGmvBKVxmmQR3Ty52l1XAjA+cqbXejRIiE1UwPDMk2S724DXOKCoVdOIr4YqA2sslamVoC4MSOzBI20BGnjrYT62MorHC6jV683C6mnZK3HVaIQg5iT4R/TX4H6G4nqpSKm9FglGqaZJ+zH+b+Y+ER52Gum62LraCAkrd0Lji9VrPOSPHlZXZlXskkA9d7FAL9kYkN3RylaSAlISBAGg5eHh+ll27GSVhMjTWI2sl0jXf118J+tP0s4rsaOYCPr8OVtehpLuuVm1U9gnJdyQ20kZrPK63SqkE8o+YBsz6YykQIB3s5bidzNKTGwAHun8xbsKZuUWXLzuuSUqGB8banlOm9sVrvbU8totcVRbm2TG/W3JM79LamCRFkkuiRFtkwIm3OYDf3W0SOR2sRouoOKDqgVU6wD/DP425bV2gSQfaiwiwFAid50sVo1zTo1hSZBM7EGzSNuNEmmyUq+oKECjbVlSkArG0+FlC4FdvdlVTkAlBzDnuP0sgLJJJJ1O5ssYUdiqeZ3K2iY6kEfgbZbaFlPSmLfqfPuiGQl11H/ABGuqoqVJqQhD9Oy1nFKoEIUvcqXyUAIhJEAyTJCSmMLzvXE17KqLwF6OpZpHwXVKJyoQQo5UnRIVmLZyiSRrGk2mrHN2P19E7TuVK6OnAUlRa0dOh0kapBMajXe0d3Fd923vclbhp1TdGhNZTpT2RgqK1wkRtEiNAJ6zJt5RiBMMksnK9lrm7jW19bBdTG/NE1ybrd64qcbS41K0KSFJV273eB2P92fvtlnK9cvFJTq1UVRTsU5USy03WpQltE91ISdUgCBB2tloePpfwt/n5KGd3f6qg37St5SrywIwkDMhq8iRzGY0wH14WiDhEjJT3NIgF0RA0ICwPrzNpI/aMXkm8b5wC4Fgh27aqoyk6KzqakeBgb/AJ2j/gokppblMBxCRKCRpEg6D3n3knxt6RwuLYdDfz/Mqjj5vWS/z0U7NAJGxSUgHfVOsfRsoISgwd9ipMTpzsDTNJjUiAJnb8bGmW0pUHExI1Pd1j6m3aFcsxddmcsSTAMA7T52wN5TIGbl08/wsJCgNFCNoI9/vttsKKJnRO5J18fuP1uynsg0IQCNzJjxGmn9LCJaSRIRJVoPr4foLdZe8ATtqNtD9A26CCEkAgaQNOfSfrazdU9lwpCc8BSjCpI5x12t0EqcCVFUZgRIGw01+NhA0SSlG+sdff8AC3SWiFZgSI1mNt552k3dQPkg1N54KhBnToPOIJ5b/OxdwEAFZ7q99J62OlsQQcp0B00A19/hYu+2CCUnSRz1Pj4n60sQFCdooM41ICb/AKFRy/8AmhjnBzq193XwtL2E0AXBdZJSCaRkEpGuiE8/j9bw9xyeUxiCgUonWkMazHfVO46fU62mbBqe1wzdStpoGSRrvkGn3b62g03cU7vuBLSQZKROWZAj4a/X42MBJI3WDI5azvrP1zsEhpSITm217p5yPzsZQjuyRmKRoDpE8vjH5WndIN7LBmTmkjWNfztsypMkH4cuk8v6+YFyiFKBSrcpgfKOv1pbZaK0yjRJEz9dPqLQzKeRcFtt2cqBr743EWTL/Qk3O4Z1BTBPPUH8N7KxCj9mEazESfLymCbJ+Ims91FRQUwtChBAHtb/AHWg8+zdSFr2UXultqqvdxyEpQhEyeZRt18LFVMt0F1XY4opSFvh5Z1Md0nlvGmtur1RUPVtfRycqyH3CEzCEIkJ+MWHvNll+iumhCsqn+zBEEnLlE/EGPfbPBurZbZJVGhx68lVdRmPrTLjuUT3UH2R5QLF7roUIo3bxLclbS0M6eygak+B5/lzWrzdbo71cU4pAS1RBIA3BJgRz1jaxSlQhGF1JyBv7BRJPLfr9a++0mnVROyr/jRo0z9MICiorKgBJIJBHjuLSbdqUf2XuyocAKWmmiMx1kJ2+77/ACjLHtLUs3ozUvFXZPLSGEmf7sJBzHT+Ik2fWG11N4YfZIzJp6GnbbMGJc0HLp+HjZ3nTRRapHw/Qqp6UvvqJqKn7RRIIHWIPPUfK2rnQRfN4NyTOUmen9bKF2KU5RU70K7zaTBHhMffZPNQi7b5rHHJSOxCtdJPd08LQc+wCkASV3ezrtQ+3dlMIWs/aqSnQJnn424w2haWqloz3XZAjfQ2OXHRqUhd4VLf21SToeQ30sFh9RD9Y2kEALBM+Zg6/WnnaGY3BRLWFkfLRX3VEAEDSCY1H1H52bt7dhT1TqGyVOORnMRlSANPiJ+HKzhr6sUlOXCCXFaATt9dbNR1ZW6ruy457pPS0JZOieNnVBNUjtZUJaSmDJBIToI300svVlI3S3YtsEQlsjzHx+vfYe6bt9Up860Dt1gKPIjw+63V6FBonlglMIOTTfpp9c/c7BYXKZ2pSPdDZ/dCErTJBVEnx6fEe+0ccVWAtpRUnPnpyRPMyeXPQ/O0lXUB+7ZWskArBnffb52i7i1WlqtoKJnvKqMyVEiO4NTPgAdfo2sh/soJbcpP4QpeqW66lblDLbodWQBGqIAG+py/AWlDCCj2T/cJVKQZMEyDty5DbqNtbRtwReCn74pg7qUMr30nviNtNSbSXhQhVVUJOYCAQSQNjEfd8bQGjgnIuE40tkKKlJJ5jXfbr+dtqSHDoNYAPOw6UaQCADAiBCvd7/lbSwUyoIBJka7jrvYhdZDsku8GSqmfBEynMIjSLErga+xcAKR3yozoYIEfd8DZbqGwphYjVQKROsHbprtZLom1XZTFTqVBxw5gke0noNOf14AD32KMxuiyrdDSSloFSlmEhJk+Ea6zr5W5YoENEOvLC3l6a6AbyB8/jY1S0SwTV1KT2xTqDplBj6+jYdaEqOYpCiDOh31H6+evvk031KZwI0CIvNndMaRqQBpy+jZuYzo/WcM3rTgmF0rgEbewYHXnZ2KRDacxkJ2jQ9ennZIvdn1iiqGQ2SlbaknbmNTHv62MHXQ1VG4EhdzLKRq3VFXszAKB+W3O1uOGzyKrCt3PwkpLQICjOUkT9x338rVFuTMGbwpnQZbcSY5wMwBk8/dpPutajgrVJqsFUYBSezASIJ6aAz015WoT6FX6c3UiISAkBQBBBJM6kSd45bf1sjYupwaJtaYKkvSrLtMHb6G1lpJOkExMBMxz56e+yVihhxd1rI17NaVQZknWfx087VwdbqydrJmLaBXnUAUiCsgg8vlysGppJVKEhKssxM9Nj01saJVBWI16KHw+vxsCQogSoFS9QTIOniPP5++07oWUoq82Vd1yIEAnYHed/CfqbF3KcBIjvZlEzHONZ16fdY6schCkpEgzPyPPT6mwLgMQQDOpJlR+/rOvXnytIOTEEJKdagqAhMTGUTJ8N/u+6yHiGhVVYfvSmSYL1G8EgCAe6qBPnH0dXQ6DokpIkQYGhPgB10sQU02teRSRlcOWI3BEH8fzixWPQni6q4gBV31bYPsrbcHQCSJn/eG/42l7AZ9Yw3QPKQUqS2Gx5BWWR8J99onpmy0i8KQgJ+yO55pUD08DaTuF7ql3C22ogFl9xvKVwY0MRy1V87EmPsocY1T0bpgpuFNpmIIBMgbctp9+gtlTSpI7sAyFDN0/Lz/Gx6iYXUKCENyRuJkc9LOWguANALq0SsEKII1mN9PK1EyWVpseZNa78MVFe8FVKChvSFEanQHX+sb7WXkXUxRM9kw0hO5Ec5O+nj797OAMpSjKjSI9ncfUe6xWoYKQVAApiSoDUx5jy+Fo57lFDMuyi3HFGE1ZJSSH6eCehgp8p0Frd/slr9LlxcRMNuOd+nqrtvFCSrXvoebWY5f3bfxFqrY5p8rdM6lOi0uoBiYPdOnnP1raav2V18qu/jhivDTi8rd53C+UIkwpbFS0R0BOVa9d7UsXj5uHyD5olM7JUNXqUpWYJWOduVjcxt8rcoJyZCJKTbs7AnpbznNcLoLWReSTHhbhzvJIPPrbFd1R8TblZPXXnaq9yOFEGJ7tVSXzUNAEAq7RJHNKtfhuLE2GiVcwQNos9ccXeVLpq4J/maV96fxs10MBKgQfaIVr1t0uGfrYmlKd9kco0bKUCLOC7UJSJy2SKJoEiUyNNR052XqNASkDbQGLdZSRWWDVS3SswI0ny1suXI4UuFECCD8Tr+Bshs7iRO1lO7XCiqSSI1HzMfcbbLBZZTzcWTizcvnFtA6Ttbnbe2TPIb9LWLIC7MA20TpztyVco99tZtJBHTa0wLqLjot5jPSLbnSADrbgnXT5G252+osQC2yGt5o152KMnKt5tWhCyRpyOtjZkD5WKkFNUvopAVpZ3N0UQdVtUjXWLH7ieDN7MKn2iUfERYgraAfK22HSy827qShYUB1gzYMrbtKkN7JSxaz2iapgEZiAoDzH52rXfl2PVWITTrdUy3OZxadVJQN8oHtK5JHMxa0OKWmGgu9HXEtternOomAEjWT8bVzvymrKu+Hq+tovV6XPNNTuIlbyYELeSoQBMkN8/wCKIKFcDiDzFVFnVw0/JdNQuD6e3ZA9piMaUWBqddP/ALFS6FTqlI/hJXPfMR3ue9ssA6h9x1bi66oUpSiSSskk2y1H7Pd2HqUbK5UH/aLA0mNMI0DS1lpi6n+6E6gGoUE6jkMo+VmVwuvxi6buuR5dA86GaZsqyx3jJ5mN4+VnV+0feQrith1lUyi4wZBjeqeG8eB+tCyMCtoRcF3ZlJWv1eCoDeFKGgHl4e629g8pgwuFw3t+9AxWMS4hKCprp+Jt2BGZd1XiiO97CD5x3rKDfEq4VpTNHeAjWAyDrv8AzeNo0ZIVqnTXKDp9dPhY+3ruTlKQok6+fu39xto/a9R5KgMNhOwUkp4i4bURmTWJ/hk052110+trCjHuGc2cVj6Fp5llcTO3j+to1USmRm5SRt7j9dbdNOfwlepMxEj9NLIYxP2CRw2Id1JgxxhdaQBeZCjtmbcTpz5efWw7eMMMiFG9kJMwZQpOnvHl+doyQQEZkCFDbUiT+e2lsGUqMhJI11iCOnvG1pjGJuwTfZsYG5UrpxThtev77pkQdMyo+P1zmw6sSYeWQkX5QGf4e2GvunQbWiMZUiMkgAjTpy0G9uFpQVZSgJnQaHTWdx5/KxBjUg3aEM4azoVMCr7uYgpF7UJMSFesJjQ8pNgXL0u5SShNfSrkf98kmOZ++0OqCVArJTpuQkeMnTxHWydUNoQM2VMDTVIkDYe/9LHbjLurUJ+GNPvIl6Ql4UrOJLtcbqAoCkWCUd5JOc6GN/0nladcB1DKsJ3O6h5uVUFPoCDoUJ3tTziZUN0t60sJyksEwBBOpn5c/wCtpRwShr9wXapLWX/VWyfHMkbnrv5xYr8UMTRJl3QW0Ak9gHZWYbamCkkjeM31P62MpaWRGYmIJ3133tBbSyJKHFpIJEhagN9R9a2OsuViFQK6rBA0yvLEbfgRYf2407s+qM3Cj+JTalKwAoI1BkknfW3CkBYSlAglXeOsjx/C0PNVl6BQIveviNIqFiDvzPnYy3fN9pUSL4r4k+0+dDM6TzshjTOrSkcMePeUuhtwgBRJ5nx2NiGIWSLtckRlUgwdJ1tHbWIb/wBxfVbM6y5IHxH18bdO37fTzBbfvd9aSQIWEiDO8RvNk7GYiLZSkMLkve4RN9gOVF+VC0ytDaUAgagBEx46/d8U6mbXVXhQVyyQ224lhpI6JQcxjlr9dVAuValPH1whdQcrndHejTX424R6ygMIbqQU03eQChMD6nc/C1Xx7Sjmjd3Ra+KNNZfLqT3kU7HauCddAYEDkTr7rEbwcFFhIMuzL1OpKTmGqtT+vusppNR2z1T2qC7UJKHCUCFJ293KyTelItbCGFqT2bSS2jueA8enO021zboZo3AWUO8VGeyeupSSQcohQk6ZBt1OnOBt5WfGE6VpWDWA2hILrJKo3zSZ89tPK0f8YKqpbvOkaecZ7JDSShITBBGk76iJ+Nnnw9rKp7DVIhCmw0UrQAUmR3j+mnjaw+qaGhyC2nObKpDw04hV0UjhgwkT3toJ1+XzsDWpRVXy1Qg5QofaKJjNl5H652Bul6qoKRNIylooaGxmYOw315/EWEpaNVPVtVqVBS0KKzmJAVJ5fOwXVjDZGbSuGqcJROVKR3tk8tIgCOnvsjXQtDV716VqgDveIEz79Dv99jZvCohOalb7wEkLj8LEFtqU5UPttlK6gDZydPLxMWi+sYdkm0zuqJ3rXKqXO1zANjRI03Eaj65eFgrtS2LypyUzIJI5H2o+YsMu7HXHAtSBlnvAHWPDobGHGwK5mrZYyoaTlygAk+8mwW1DXOuVPkkCwS4hIzCAUjvaCDP5WI3mpz1KoCdwhRITryP62EcvRvKUBl4xrpBnw0+tfiXermVoW0WHRmKhIT89D0tbNSwjdA5DrpDYrhSXOZAS4VlKAf4vHrEizIxpdLjF3mvqkl19xDkSJyAgaD6+6z5bbpxUioqm1qZbTLKAmQI5n65izW4lX1d7F2JU52gIcyatKIIynT4A/rtacE7CbEqEkLugTI4GvTf15N51QumSSOUhY938Q58haVcLkC96phKlQAZCht3hPnpFoU4OVrVHjioS4YSth1GiNScyVeXI7czptaaLmdaYvdyoM9msLIUAQBJkaanlv4WI+RrSLqLYiQnoApMDczJRrBH9Lc5iAE5iRrqTED3fUdLAIr6RULS5pJBGQiPA/EfRtya2jQkQ6CUiNAdfr65WRnb3URA7shXnA0FEuQSconUid9LI92u+s3itT5JUhJIEbagE/XWxpo09U+p+qqUpS0YQkaGY33g8rEruCGb4zuIKWiVJzrVoY10sAyglGEZAUgYWwdVX8hdZUNONULZgrQgntV6d1JB+J/Sz2bwZStpFTU0wzmC2wqmUAU9djA+uWhu56/DrNFTVC6ygDbaQhhgLb7ytu9CwY895s18Z48Yq0u3dclSgZ9HqgHKT1bbgnu6b+4RalzpJpMoNgrAjaxtym/jB65qaoXd91NU7oBh2oQ3CQZ2RpOnX6LIq2g60ppxKiI2Jjnv5fXkquhDqMpyJIOhEQOv142IvtJghcLOoABj687a8RawWus94JN7Ko4pxRYgv6hWkyhxSUkJ9mHI106H8rWH4B1KncLqT2mYtPGBMzJVp5/r52gfFzDVDxHv1KiAFrdV4mQlW2/I+cC0v+jlVhygrqYK1bc1EzAATtppqoj3WBPrqFZp9DZTkPb7pkjvHTkR98WJ34larpfhR7qQoGdAAQffrYdpQRsoQBA5AjmANuVuLyQtV3Pt5Mw7NSQE89DqPvtVv3VshMSFBtRIIKRvr0jn79OkW0qZ705USBpz11+j0tgzKAznu9Nteupnl8racVACgBlMgmNjMH3/rZZioaoNxKiBvEnSeWm3uPLxsXcIMhKFKTGnjOvv/AKWGcMgKVExlEgkQNffzsGsAZitU+PsgnXr5fXJ72TEXRN1IKM2YyT/FrED8h95sTWSklWbSTISYjy+XTfSx9aCEFGUzoO7ofdpr4WTnIKipslBERrrI3HL4eXSLEY5QLFXe86EUuL72ooCft6htOYnmVQdvI/pZ6cGqB6rpK6lQpQ7N1KyZOmZMR/yixHGuHXWMbXnXKhLaXG6hCSI7QKQkq+8z87LnBB0MXtelEVJCnGkOAHXRJIn/AJvPaxJpQWGyhEz2wCpkuq72aRH2CBnOpUNNtdrKaciEQogDfNAM/h0sWYCEnRMgaCZ94Gv18rGm8yVAn2gJIBgAcojbX8bZZeStJoy7LoA5QCkSCACBrt+litW0hSCSsSZCtTl2P4zz/Gx7O3lLavZGxKt9vfYB9eeW80wI05fWnnabHJnC4Ue47aPqCXUZhld73gVJVqNvusv+ghe4w16YGHEFeVu8vWaNfKQ7SOZZ8O0SkAdRsLJ+L2u0uh9KZBACgM0GUkCdPA8gbNLg9fIwx6RnDq+1OpaS3f12KcXtCPWUpWPPKqDFiTN5tNIzuCq49mRp817hOAIeOghQt1ByyDbKsQEqT/DpbQVmRmHW3lpNiQukGrQUWcIzaT8bcLOnibdvHvZgfKwLi0pRmUrKkCSTytUcdbKw1Jt/0ZrbrqEJEqbAcT5p1+6bMdtsExAjeI091ntcGKLhxLWXhQ3PXJq/UcgeUhJLfezaJVsr2TMSLNiqojRVjtHH92opE/yjb5W6jACS0sKDVktFitUjIASnLA++yxSgCNj+FiVO2SJ5+MjxsfpxBk/r9flbuKdq5yoOqPtRoAdrHKZWR0EcgSPhYi2CdTrHTSbHGlAOIHQidLaABVIpzzIm2A67+NgadZNO2SZOUA+JGn4W7zaTaw0XVcld5ttLcz5fC2p0k2yRJ1MWIBZRW7bnSBbQPO2Sdd9LEaFBxXW5Bk62L1Bh9k8lZkn4afjYfNIixavOVtLmsIcSSR46fjZ3bKI3Xa5jUA24UTyt2vaIsEqP0sIi4U721TvfDdZc9DULQFFCRqRMEDKT99o0xBdbD18oQ7SsPqdUlCA84UgEneB7XL7vKRbkdD1wFopJ7JwpgHqQfxsx8e0xRK2wdNCQLcNxFAHOFjY7XW/hDs12FcKxfhCmUaZNy0DiWjkCwiAoDSfYtlo/RT1oSAKUqEaGVa/O2W5n7Gb/AGx+iv8Ahmd/zXmR+0WqUvcZbrYJ0buBkBJ2/wDOKgz8PLezVwVet1U9w3ewqvpUKTTIzhTyQROskTp9b62cH7QGrnjq03nUjJcNOhWU65VOPEj4HW1cnVrbq3ilQKUJTKtSYCeR9x5R1t2eGQCXDYW7eyFVxF+Suld5qzNPeV0rCSu9KQE6x6wnXl+PLnZRp7yu5QluvYUnaA4kx8DofzNqrJWUkN5h0KSecyecDb7rCB9xU/aFJTsmNx59eWnQWMaHzVcVduital1gEAKRKhEZgf6D8tZsO062VQCnSCddPl57Rapvr74dzMuqVk0ASSBvtsJsML0rfaRW1TWb+V5QExHxJ2/S0fAHunNXfWytkgbZkZgNp1+IiOtuie+kkaE7zzkCffpt1+NUf39fTKMiL1rJ3QUvrkHXSZ+f0R0Yov5ttXZ3xeATGaPWVgSD0B6/j52QonDql4sHorUZ0nvKHvHQ+HvsEvUDYqXMwOeh2Hu/S1X04zxS06laMQXkmBKQKtRk/Hz94iNdRTj3FjaYTiO8XANCS+uRpvJ5SAfoyvBv7pvFN7KyTjmh00Enu6Tt8tLEapRU2UtpJOwTG2vLoPG0Af2+xiFpP9oawiEjRY90A76iPebY5xFxehZKr+qFEaSpKSZ57pmNLTbSuCiahpSrxaUf3tRmZUGT3VSRIXz8fP8AO0t4CIVhy7DClTStSDoQconSPrytXK+7+rr8W07elV29Q2MhUUgACZiAI8dfus5br4lYnuigYpaGpR2LLYS2lTSVZUAdSNhp1PKxZ4HOja0IcUgY4lWUpykkgyCNyNY/DYRy91j7MAFGXunVIzTOnj52rexxpxo3mP7wpF5UiCaZI1I0Om452FRx2xszLy1UBCZHfYMeXtT8OtqnhJFabUsCshAKezyydZMA7ff9dLCqEEqTB0OikxljWZ6aH42rojj3jJsJDzF1GRqVMLGnMDv6jqPd4WGR6Q2LGxldu+6VkACMrmomJ9vx+/xiPhJVLxLCrDIUEKSEpJJ5FIEjp4fW1hi4AAmMwPhHvtX5v0icRuAOG57sUT7QAWJP/F0+uo//AJRF7ZJVh2kUYmEPL138DPP6Nh+GlvsnFSzup4WtKAEFR3ywR9HqbclwKQkkqkd4mdPr9ffBp9I2u07TDFOoSIKapXwEo/H32FT6RBJUtzDDYykJEVZ1O8ez4ffNpeGlHRJ08Z6qai6VAEAqA72nT8dTYhVLJBCUzJiSTz3325WiVv0i6fID/ZdYM/8ApckHYfwTuY+NunOPl0vw2/h6rTnEghxKoII8Bv7rSEEo6KHOZ3SJxqfUzeFI8sZgWwTB1kFW3j89NrOThW/nwnRE5xqsTt/GrztGfETG9Hi1bNTR0biC0kIUXFbjWdtOX3+NlrAXEG77kuhFDWtvKW2pfsIBEZp3Jnpva1IxxhA6oDHt5pKnaidShBbAmDMnfy0seCh7cHygAc5Hn5dfG0Y03FnDaTkfNUCd/s5j4HmRHXXyspMcW8MLHfq3kkzKVMmRp7/v/SkWO7K1mb3T+K05iEIUJJB8T1+XztsPyJTpEiRpr0swkcWMJOpTmr3EhUiS0rTkY00sIeKeE3IKbyI29ptXTQ7fA+Vo5Hdk2dvdPhTydMyklI0IJ2tx2iYJSRlmTEEnp4WZjfE/CiiJvVO+pU2sAeE/H77CHiLhWE571ZRAmSVAnX59ffZZXdk92907w4gk5sqR1mQoz/W3C1ycpknYwdvr8+Wlmp/pCwqpKk/vlgZTqCCmY+uX620vH+E1R/23TCI/iPMc/OPDlZwHJZmpw1CwUTJAiIG+88rMTiOjtrgXCJyOoVpzGunxI87K7uN8LrKlC+qMmdi6BrppHLTf+sNjGWIbpq7nqmmLxp3VoIOVLiZGsbT52LFmDkJ9iFHPDR9LOPGQEOAOpdaBGwOQkjbwtYehcblCT3ecq2Nqz4HqSxjyhedqEhsuqhS1QJyEfDePH32sTd9a0ogBxvRQgBYIjx11taqiQ4W7INPbKnOhzTOtKtIIBOp06+U6+It0pMg5+6TEERO/9flYm0+CjuqSsK3JVmMTy6/rYQrz9wpGYyQkkyCdPy+FgZij2CGzAGBJlOn5R9eNgyEJB7iVZdZgEbb+FuA5niRIBAA6bEjw5W5WVQO7JiSJmR4fn9CGcgpsoJQylMzPZpE6TEDT6/rYFaG1HVGonTnvt9R87cB5JlUhI1012/pPwtylwCSQJHUfUb/WlpNcRqE5AIXDjdMCr7M7SrnppyNib7TCZGYTJT3FaHz/AD8bG3VLnOozEd0kxof0n387EKt05NFDveeum0fXu0sVsjkIsCgXifTU1DjOuU20pDi0Nr7pI1UgDUfHb9bLHBOtqMl5NsPPtFK2yFNuFJ7wVof+CfcLFOMSezxCwtY7rtM3InWUrWNPcRYhwUqezvu8qVSijOwhZAB3SqDP/EbXt4bqq0WkU9UdfeWZJF6VeQpzHM4rw19/jY+m8LxdQWzebqkq9qVTO2kWRaRQgEmClPKYB3Pw0+HObKKFFcELTnzamD8fuHkLUi9XLLpunSk9mXFFSTBn+U/cNY99ufUUtyg90xEz00HlE2FaBgkgLyTqfHqbdZgCEpGfXmd9vxHys3MSyIqqhU4mEvnfXwHhp9RYJVEto6OqCRqM2se7x1soKKgEBQCgNzuR46W5WBmIJgbTGbWOnxshInyBJZoFEABUK0k5Z/Tr9CwlFdCQuVSogkwD+B8bHFqSmUqBkg5tSAff7/ut3TFSc+VQzRp00mdudoulITtjuoo4tXeWL4D6UwX6EEEcoKkQPgOX6NThRV9ljMgEhNUw8k+IBSqRI20jTpGlpF4uMoWbrqVIzJIdZJGxAUk/9Ri0T4EqHKbGN2PrXEryDvbhQI6cyZ59fGx4iXRlBe3LIFYptzLBSokA7JBgDWNY+oNjrRkCEGBtM/U2SmHjl0EJPxmOgGvlZTbcBUQsJGYzodTp5/UWp3srlkOlYRJSJnVQIncdfebFnXQE+ztAMK5eUeH1Fu3HkFYOiTt7Ux+tiryj2ZzEaHvGd/IgacrLPZRLQkPEK+3oatkg9+nWYiTsY+f10hO+7xco7yue8qeoJXTOFaVSNCCkpjw/MWnWuZQ4exWJSruGNwDpt7/rW0AYpZU1RU638w9XeCVEzEwQSTOo/FRAtdgdn9kqrM3Lqvfa6bzYvy4aG96ZQLV4UrVU2oHQpcQFA/A2Fp1ygojY2h3gLxRwvSei7w/xviu/qS7qJGHaKmefqHQAXWWw0tI5qVmbVoJOlo0xn6aGE6x9+7sK3i/RUmoNUunV2rgmJTyQN+qv8pt5lJSzOmc1jb2JC6OnAkaATb4qwuMcfXFhVBRUOmqrYJTSMEFw+KuSR4n3TaDca8TL0xJmarqkNUavYomSez8M5/2nvhO3d52hiv44YXqFKDV5KlxRUouBRUtU6lROpJgySZsg1fFi4qkdp+8RBIzTOg8J5/rYseGy3uWrZiNLAPvAlWc9HTESqvG153Wp3Mmpu/toP8zbiRp7lm0u4ro+zvBupQAEvo1gfxJ0n4RapXo08RbmqeMlyXfT1yVOXiiopY1k/YqUB8Uj5Wujiaj9Yu0uhMqYUFz4HQ22cOYaeYBwWTiL2yOJabpsU6IAiNd9dLGkJgbExvHWwDKTGpExJ87GkAmN9Oc+Nu1g0XLzalDNHvDQfCxlBERrNioQQYIkETYdER1AFr4VNxsnBQuFxjTcK1066/jYwCQNd7ELrWCypO/0R90WM1FSxSMrqKp5DLSAVLWs5UpHOSdLWWfdVc7oeQdT91smTHxixK774uq9Ar93XgxU5dw04CQPKwr1dSUykoqKtlpS/ZC3Aknym0gb6pjcaIxMbW2FEbm3EyMwOlsQ625JbUlcHXKqYsQKBuUICJHy87AXgjNRPZRPdJHu1sMDB/AW2oBSCggd7TWzkX0TBF2lJcYQtOykg2DUQRsJsHQE+ppQdVIJSfMG3atNjsdbDKcG6c2DXe0TWUyjopKVD5g/hYnfdIyto1FWkKRTjMpCh7Z5D65TbnCL3Y3uG1HR5tSdeu/4WO3+3Vllxlqkbc1UVAu5Y6cukW5PiZoEBcQfl/stHD3ESAXUdu4hr+1XkrK1KcxgJfCUgTyGXQeFssKqgvLMYeSgT7IfMJ8B3bZbzTm0vd31XV8mDsvHb076gP8ApA1CFbM3XRtankUlUR/vWr86jLWvBzIkyJ05zqepO/XytOvpwOhfpCXxBSS3R0KII2/1ZJO3n99oPTLrrqVAqzGSEAakE9Nvwm3qeE+zQQ/3QsHEzesk+JWNrUlRkagEaEb8jHLx6/O2KA3UpckTHLx/H52F7JKlIJ1IMjUaxyHPofK2ygSlCkSVEHMrQr5A2ukhVbLgOrBjKgnKTBJESfrxJ91uVFQJUCAFzMGQYg+I0sK40EHtSoJzbSQJ+oFusp7XMfaUSD5RrPw+EWYEJiEWStSQCQoESSFHLJ92pthQCISlQB0BMyZHI7TzscFMVQWkjSdCmB1666fceltJYIWTGcmBB5GdI6H8QbLME1kVzElOTKnNsCNTzI+EfAW3JKezTmIAOh+EfXON7GmmVqCsyypClBOTLMmDv18foW5QxOUKSqSAFAgynTSAfIeOllmCVkUU531FIJXOuupM6H79/HrbRcJOdCUtheuUaZveNQNIsZDBQMrsHbuqTMGIGnOP11twpptBCw4kZQe7uTG0fPoLSDhsolpRR7u94qT3VFMhXskbAdRpv77HG1fYtg65E5T3tPGY30nnz8YsJc9wXnie+Lvw9cFCKq8rwqU0lLTpUBneWQEpkkATMSSBp4Wmdr0M/SIypz8Pg2tIGabzpNTAH/exP5WHPUQwgCRwHxRIYJJNWAlQmXShMNgyROaI18T+cbWCXmWuS2THIfPy8vG08NehX6RTzSnFYFYykwFC9aPQnlAd05/QtivQi9I99QBwMylJglar2oxI5H+86crVxiFL+MeqP4Oo/AfRQVnSFBZypUnTTUE+MeGkbeWs8lQ5DMlQJTqZOux+o08LT9T+gf6S1ShSf7F0aVxIBvSkGoOmy9vw6WEV6CPpLrQlP9iacaDT96Uxkxz+08NtR87P9o0g05g9UvB1P4D6KAmaw0y1KU224E6QtJEgxt8v6WDcqFOKLuiQBO2kc48Plv1tOFV6FfpFUL6mXcEdotRnKiqaUEncR3o35gnWwafQ09It5SGG8BOdoo+0urZCBvBJKoH15WcVtKdQ8eqj4WcHVpUKh0lRCsoBGpSZ+Hl566+NtKqEg/4ZhZGuhI66Hby87TVVeh16RDLqmXsAvkpUCSiqYWmYGxC+kTqbF1+iH6QzRj/R3WRto+zGx2IX48vCzirpvxj1SNPMPdKhtl0AFGaTrKvOYEctBv4DwtiKlIUlZQSFAhZVppJ6+Gmnj4WlweiP6QbaM6uGtXJnMC+wB7W0BVg3/RN4/pClDhvWjNliHGemv8ev9NbTFVTk/fHqocib8JUQOuLSgCTEkwBGYdZ5aa9PxHonXA0paFkpOxBgyDMHx1FpUd9E30gno7HhpXnQbOtDYH/F9a+4aj9Ef0jMx7fhpeMJOmV5oaHqM56C0jV0+X749Uwpps33T6KLhUBIzhRiMq06E6Dw1Gxj87bTVrS2QC4nL3ICiY8untDz0tLf/km+kCpJT/oxvHRMf3rQMgRGivw/UNXol8fSVD/RreRKkyklTQA9wV0nbrYAq6c7vHqieHm/CVE4qFElTqwCSMpKTsPCdLdIrFtNnvEKKdRO45e+NvM+FpSd9E3j+tAC+G145yNs7Yj/AJuXv0J3to+if6QvLhleKp3UVtDwjRUmRG3z1tLxNMffHqo8ib8J9FFhrxCJSoKCcoTrMe/6mfC3Sa9TbYUAEqJ1Os6bbyOn9bSoj0S/SHGZJ4cXhqTp2rIgGNPb8Po22j0R/SEKTPDitGkQp1hXMzrn35a/nZeIpvxj1T8iY+6VExqnColQEnXx1569ddLcqrF5QVFJzd5WXUn6923utMFL6IHpFVSi03w3q0HLKg5VU7YG42K4OkWJveibx/DoaHDi8jMyA63G2x70az18LLxVMNC8eqfw0++U+iin15x1WYye/pPdJHXw35+NtOVyYCc4JRoCNCoc9fd8rSov0TfSFT3Dw2vFRQqAsLQZnQkd6OQ19/mGfRN9IRWUHhpeYC/ZlaIEeStuW462kKuk/GPUKPhp/wAB9FDjNQtL6Tm72bYHnpO/usK5Vr9kKUJGp0jcg+XLytKo9Er0hyMo4aXinKZnM2Ikn/F8bCD0RfSGCSf9G1aZ6LRO3MZpO332Ka2kv+0b6hQFNUDZh9FFKbxq0HtE1K0EGNFlIA18fC3f79vdlYQLxqhl1IDpGoG+++2nhaTV+iR6QgEf6Na9Iywe8jUDT+bfzjW3R9En0hVq7McOLwzAlRHaNifH2tD7Pzs4q6T8bfUJGnqPwn0UcNYwxElv7K/7yGbUgVKwAdgfa6fR3sMnHWLUggYmvXKnQJFY5I16yR7uptITvolekOCVL4b14OnsutydDyzaa+Hwtx/5IvpEZj/+7asiDBLjX4q8bIVNGffb6hN4ep/CfRMJGPsXtPEnEt7582qVVizI5jfzsZZ4k4xQCj+0t4E6ZQahQ6g68xy9/hNnmPRI9IgqyHhxXAaqCu0biBtPe87df+SJ6QqSR/o+qyDIj1hoSTtuv58ufKyNRR/jb6hOIKn8JTQHFDGwGX+0tZ3iYzQc2vl57ePvEVxQxpOV3EVUrKNAoJVvtoUx1nrZ5MeiB6RzoVl4c1AgZjFS1BnnOfXXfppYR/0O/SMpXRTr4d1JKQFAt1LKgTvooLInfn+Foc+j/G31CfkVP4Sorv3FN+32pLl83gp9xsFDai2kEp6d0Cdeon4WDw9iO9cNXiuvu14IecSpKvswsZRBmDpymRaU3vQ99IltxpB4eVLKXlBCVKqGwnUbE5j8vGx5n0JPSLW4Ef2KTKwJ/wBeZSOgGqosTxlIBYvHqEwpKom+Q+iZ9JxoxbSoACqB0JHeCmN/EwrQbWPo48YoSAk0V2KIMf3Lg0nl3vD61suvehX6R6Y/+gSJ5n94U5BHX29ev0bEl+h/6RLRAVgEhOgBFdTjfr3/ABsPnUTvfb6qfJqwfuH0QSOPt/JEOXNdpIJGUlyZjU+1Y4xx+rwuVYdpSEzJD6tDJ3H4+Gx2toeh76RmbshgY5hyTeFNA0HMOdB93U2T8XejJxxwPh+sxfibBC6O6buQlVQ/67TuhCSpKQYQszqRyMTaOajeQ1rhf4qWSpaLuafRLjHH15JLT1wU6lA65amB5+yfryEjDjrTrRK8NHltVDUdfZ+f32gpDqScoUZCdJJk6/1sMisWCA0U5iIJHj79dvusQ0jELxDlOjPHK7FhAcuWoTmTrlfCo59OQ/ppZSuzi5cr1WW03XVoJBKSFJ18teX1pE1+RUulXf1EglJBPj4eP4WV7jrlivbICiVEtgde78/zJsCWlAaSEWOc31Ur4/xjd9+0NMzTU1ShxpalhKgADKRpIO8gfQ0iukrBSYhYq+yJSxWdovL3tA5Ow8Of3aWXax5LjYhJWU+zmIBEadekWZ9epxFdUKIgFSSAkyIgRty0+Vo0guCE8ztQVYCnxtc0aLeSdDBR8Bryj4ab2UqfHdwZUpD7vd6smY2AiNbQ9TPj1ftE5hm/igxG8dI/DnBix/tlBKFgpIA0gkpBJ+ufI62pvaQUZshKls41uDIhSqogZjqELmdNzFuFYtuIqCPXBKgIJQrmPL9NLROmpKWgSVJMAeKvx5+Fki9cXXfdMtBYdeEdxBTod9Ty3PWyjifKbNCd8wbq5THV4iuQs9qLyQhIGq4IHI7kRv8Al1tA+PMTXS9X1t33YoPoeqSS4kygJzT3dBOs8ufibN2+cVXnfK1B15LTSlZuzbACfCepFkJwkmSZ1mbbNLRcrV+6zqiqzCzVKmAr341cQGKfB2GsSvP0eH6ZRpKOodSWqdtTpWoNpUCBK1knTnvtZ4VHDb0j2UFa3KF0azFOwZ08W+kfK0HXNiG+MM3i5W3FeD1G8oFGdpUEpkGPEaDeztp+PHFSmbDIxdUOJ59q22v/AMSfK0Kimnz3pwy3mNUemqKcMtOXX8inJX3Fx/pCfWKJokDVSael0Hnl0skOPcYKYAVFA2SZGrLIM6jlEGNNNbJr/GviHUp/1i/AuTJ/1dvU/DysQe4nYrqEdm7XpI6hpI6fDblppZmQVHvsYmfNBf2HOUu+jvjbiFh/j5w8vO9qYNUiMSXe1ULygfZOPpbVsf5VG3uU6wmoZcYWO64koPvt87F3cR8Q0d5Ud4etqC6WoRUJKdCClQI193z8rfRFcd6U9+3Ld990hzM3hStVbR5FDiAoH4EWx8WhdG9jy0D4K5Qyhwc0En4pmoZU2tTSxCmyUnTmDFh2wPHQR77Hr+puwvFSwO66AuY57H5g2JAbDkNPdbUpnZmhyrTCzrLtKZI6fdYVI01A06GwafaGmn42FGonSDbQCpuFwlK6XASUDnOvwj8bIvE25b4v7DDtBcSEuVPatrLal5AtIUCRPuB91lO71APpHiD94/Gx29awXddtTXFBWKdlbmUbnKJj5WOzVtlXJs66im7rrVg/FF3U9JWPLqH0Mrq6dTvahlajCgk7hJH8O2kgCbKOJMM33el9XtU1b1TQU6G1PMVyS2tGiRlTlVJjroNtDzsmcNK67byrqjFOI7zZFY64VpSskwSN/IDQDoLOXiLe63kUd0UToDVUA645mhOX+GT05+60W5QwnoiOJzBJtLfV5PYCqgXVpFPUNN5idQ2pSZT1jX4WTLner8O3xd97C81u095CeyKUgN5VZVokDvAyDrqDOuwDwF1XYjBVbc12VtPUv+rrcUW1pUpTgEyQPGPlZjVLqavD13utLClsVa076htxGYHyJTYtrAFCO5slXFJvy9sQXlSUNclqopwVUyVpKkQlsLywCInXvDbxs9uHt9P3vcg9ZeU64woJzKMqUggEEnnzs06+oDOMburldxFdTMOKB0GspV8Blsf4ZOLpK+8LrUdG5Qmf/VrKB8osQaG6gdk924Q7UNjSF5h7xbFyo6bW04Cm8Fkq0cbBjyNtqGulkUwRq6XhTXnTP7BLqZ8iYNnxeFMFOr09pFo7EA7xGoNpKUsVNMxUgaONhXxE2ysUhEkLgeyNC4tdomM7QgOrGuijz8bZZyLpElROTc2y3mRpz2W34srwV9NHv+kRiJKCqeyoUwP/AMVa62h4pU5UPAtqUO1IzztPu6mNtdbTd6X1DUXn6QWI6u7mTVUy/UsjzBK0nLTNA6p8RaOgm6qRypTU0i0v9qRGYDlrufLlrbuqF2Sjibb3R+QQ632qqQ+ZTfNOvIVdmXIMEARoRoD4mCN+Vum0KUMqgYmSkIgnu6Hw3s42b+uWmSWkUhb7Q6gqSABy5zpztpNfhDR00jp00MmUmdNSoSefS1i7uyr3HdN5NOtQSCFDIJJjbTczz11526TTKSQrs1qOgEE7xp5f1svJvDCi/sadp3vryJGZQiPMzz84B6ahorMNLUnKFoSvu7geMb7fITZru7Jhbe6RhTK7RBZUQANYTOu8nx25fp2lsOBbYBQghSVExB1PMzO55zrvZ3XFdFFftO7U0YORtZQVqWB3okiOehn8rKQwa0lKgnKIEAFcmQJjlGmthGUg2RRGXahR+pgEkZDm9jQDfw6a+7QG3JowBlghSN4BHI9OUH7+VpEZwSwk5XklKcxSR2qZBBI3jflz2EeBhGBLrUQlZcAk5T2qNTrtO58+gtHn2UxASoxUylWiGxKUEFMTJH6fHnIsG9TOskZ21QRA1ISTtM/dytK5wTcwSsOVDwC+jiMpJEztzn5ixd7h/cDo1vB5IMk6tkgbRtqNPhNk2qF05pnEaJF9Hp3Jx74al0pAViiggbjvPo01/W3uUmloHgla6RgnL7RbTP3W8VOGVzU9w+kPw2pGHnHUDEt2rK1lJOY1CBuPK3tMmqCW0x/CItQxSPnuY63RXMPcY2uae6EN33aglQoKYf8A2oflbtVFdqtPU2CdtW02AdvRlTYpwxC51cnSPKwaqhcZc0jSTO9s40QCvCpvuhjRXcDpRMeP2Yts0N3KHeo2TP8A6sfPSxQ3iilJW4hTk6AJ387cIrysZjAG8TIsvBC17J/Ek6XXa7toC92qaVkKJ5NiT5m3SqOkGpYbEf4RvYM1Sj3gPjbRfkxy62mylA0smM/mjLdHR5Y9XQIH8o32sMmiogCOwajWBlFibdUkbq92xsKKnTUjQbRZGm8kwnv1Q5o6TVPq7cHQ90GwYu+iUZ7FHvQPxtyKnMYKR1tz62JiR8Z+FkKe3RIzeaFRQUSVE9g2DEewOthW6WkH+zSJ55RYqKgT08hNuvWTI0EcwTZzTeSXO80aTT0iUkFlI0/lty5T0syWUSdiUixf1sQRIB/G3KqozKTJ287R8L5JCbzQnqzCVaoRoY1QPC3S2aXRXYoJI/l+di6qpUe2rw8LcGpOxP8AitMUnkm5/mhzSUcAeqtyP8IiwaqWjAI9XRA0PdFgRVDYA7W49aI15b72kKXyS8Qe6FNNSAZfVmTH+CxarpqXstKdr2T/AADW2KqDBmSNtPr60sXq6hAbBMpgET42hLR+zspsqdd0LTtUxcWoMtzl0gC3KmKUttJLLRk7FINidNUIzokiVIn3zbhNUcyNZCSNZ6k/laiaMjorPiB3Rw0lGWCo07R3klI5Hy1sXNBSBgqFM3Ko2SNPlYJVcAxCtyo87dPVI9VmRMA6ajey8Hfol4i3VGU0VFH/AJqyBH8osXcpKVS1/wCrtysgapHxthrQG8wIIjmbAIrEuPyskZQSOX1+li+D0Aso+IHdDVVJRtpb+wb6QEDW3aqChUmPVWdAN2xYtW1QUlJI2Uees27NYlQSoATE6WmyjsdlB1T5oq7Q0Rqkj1VuAmISkCxWqoKR9QaRTsyNzkg8pNh6qrAeKwEwlM+B30sXZfJSFrCgpausxodPlNpCk1TeJARSpuyhaUlKadgJgEgJAO/lH1POyJjfEeE8GXMuvvnJTIcKWWlN04UslRiUiOQO8QIsr33Vt0FKqrdJIQhThCdzAJ08d/ebRbcGI79rcR01/YmDSVjMoU6XipNPPsN9IGpJ1k69IBPPRUUrYqqVrC7a5tf+e+ywcW4vwfA5oqfEalkb5PuhxAvrb5C+lzYX6pcRiO8MUvLRgjhjeuIHWNEVdc6unpGv8Ce0UUqOpJytlJnnYPEtVxmwxd3rjvBtqrROZw0lQ08G0kDMAjKFefLa1gsNXpQeptMUjDYhEgo0Cjrv1J387Hr6vJdOyFOpbSpcDLoD79Ztstw6ANzOW94mRzg0BQNhDGOHcbUqwzdrdLXspHrNI8hCXEGNSUnvRPgR42PVV0UTy8qaJkRqqGx3dfdzstYmuS4W3nsUMXOzS1obzKrWBCnAnXKobEECPf8AAtd8PU7dSkiHQFyIgAid/efoWpx07XOLWhPUSOiAJRBi7aVFR2aqdr2gQMoMCzU9IK4KC9+CuN7vFIyQMPVy0p7JOiksKWmOhBSD5gdLPSpr0tuKpkyYGXQJAzZgrPMZpgFO8Qdp1sRxlTIvbCl8XYElXrtBUsGei2lJ/E2sxUgY8EBU5KrMwheIeVGaMoAJgx8fryt2gpMhS8pSQmdv6W6XTLbKlSZB7+bQDUgT7x4W77Fx1wpQgQpOyU6nX626m3V3C5uxC2hCyrurSQkbwYV9QfjY5dZU1WMr17riRmSNBJHu+UzYBhpc5i2CQROYTpIGsn6+4222huHUISqFZpMQI2Gu+kTHysCQ3BCIwa3TqqFJCQhQGZXeAUjoeo1nQRrz8bNq+lKFelWvsCO7ObU+/bcn52dVShPYhLZlKtFApmSNBz8N/DrZq4mICWlpTIyqk6GdRB301KrUqQgvsrM2jbpw3S4pyiYcTMdmjvJVJEae+Bp7rB3letLQs+sVbmQx3I9pSeUDnP4Dpo00YmXQ0DNJTsJ7RuftFiYkkgRz3O/wsi1L7tQ8XX3C6tWsrOtrTKLM4l2yruqLCw3S3fGMayvSWaKaZrWT/GfMjys3SqTqoz420dNAYtozGvTQWvsjbGLNCqPe5+rlsSIH3W0oGdedtwrruettuwQDO+sRYihZcrGoKREgW0RA0t0QQlCtYMgH322pGUfWlklZBgnpbDpbcHe2QdrJJaBMzFvoC9EPEwxj6MXDO/s5UtWHKSkcVJ1cp0dgv/maNvn9O1vaj9lpiY396Jd13ctQKsPXzeN276wVioH/APs/K2PjTM0LXditDDXWkLfJWgxLThdO3UBMFCsp8jZux7vfZ6V1P6zRusACVI08+VmbBB1EG1fDn3ZlVuqbZ110AZkEeXIGwiVEka6+f1424GU9QDbsCQRlPiTbXas8lDMOdm6FE6CSfdr+Flh9hqqYcp30BbTqShaeoIg2RRBOVZ7p0PlsbK9EtTlGwtftKaRm841+djRlVnjqmgxwgwXSyKRiuYBnRuvfA+SrKd74Dui+GqVpb1Wx6m32LamX1JOTSAeu1nGCYInztuYMDexg1vZQLnHqm5h3A1Bh2rcraetrH1LQW8r7gUACZ6eXwslvcKbr9eNTQ3pV01MpZcNIkIU2kkycpUkqSPAGBsIFnwd+VtiNgdtpNpBjbWso5nXum5iXA9DiGipWEVTtFU0IinfagkJIAKVA6KSYEjwBEHW3eEsHqw4XXaiu9bfcGUrDeQAeAk/fZwiAJmBbtKjvpaQaN02Y7IpWJy1NO71KkE+Yn8LdLSDuAD0i27wgNIcP8DiVfOD+NtrEAfGzEalK6BIg66Wf2H3RU3DTmZLctn3EgfKLMJXx/GzwwW6V3fUsGfs3Mw94/S1WqbmYQptKUSjU6Wyw5RJJtluU8C1Wsy+aKnqV0TLbTbriTJBKVQo6kgT79rYupf7ZSHUOJWhSgoFKe71GniTp5c7ErrRUPM06e0lSjHdIEHYDz8PlZw3dwt4kX7eTrd10HaPIQp05qpsbECSSdTJHnNulbAXOsAhGXKLkpKRXLJyw6cw7sjUHp9DWNfAVq8AiPsXE94SFIBJOkeOhP1vZyDgFxjQO9cyEiZM1rO//ABeG5saY4A8YFKUk3XTk5SRNYyddwd/A/UWL4N591D8Sz8SaabwKkpJZMagktak6EAx5RHhz2t164lcIRTrz7H7KTBEdIPx+FnKeAvGNKCW7lZIWf4KpiZE/4hOluEcFOMbSI/cggAp0qGOZn+b6m0TRydk4qWd0jM3nVtoNLTO1SEGVlttogKP8SiU+OXkPOwiL8voKyprLx1BSMqnIGu3OOXlAFpLwNwPxrUUNS5iDDfaVCHx2cVDYkZT3QUK11jU+NnO9wNvpCkqpMKqbAEFKagEkynXVemms7jS1Z8LmutlVpjw4XDlB7V+388+koq7wJUnKCM8gj9QOnxNlBqrxmtANOm89QDKWlgq1O4APj4TrFpfp+CWImnUqVhuoaAUNRUJJ1Jke14AfGx9PDvF1EtaaXDroCTKYyq5bb/XztWkEg+7H9FYiyH7z1CzbfEp1BS1Q32pAgmWFqMxodE+7pvvvbmoXxJoGwupuu+0ZjKlqpFyr/l01Hwm07IuPG1KlaHMLrcgzJZAOpO2XmIHxsSdu3G5dDVTcdakODPpTqSkCBtB05AeVgNdPfWPRW+XT20kN1HvA+svyo454BTeCKpDiMR3amX2i2QPWGzzA3k/W/s+24coMxMHS3kZhG7q+h49cPn7woXaVasQXcQFNlBIFWkczt+njb1wbIKEjLConeyqWZ8ptZQp3ZMwBuulKBIkkkD3zbRrW0CFDYfX32CcDmQlKTA59Pf8ACxJfaKJKVGOsbWiyEbJpJrbI0XGXF5yRpEn687dhxnLlGvXX7rJ6EPOqKUJKlRJAFuS6QoJXIPOfOxfDg6IQqCEqhbG6hz3tsuNCSCU/XOyWl4EaEH8LYXhJGbyP42QpgkaolKgqWN9R4RsLYKun5K8PGyQXRBIPKNLcl85ozSJ0M/haYpWlR8UQltVUyTBkkab2w1LMggk/1sih+SQFf5ptgfy94mRvG/0bLwoCXiilv1pmYgnaLbFUyZnSd9bInbymCZ58tNOtuhUAASSB58rOKYWS8U5K/rNPMHl99tKqqedR8bJBfgxmI87a9ZClakTy8voWXhWpeKclc1TGaRpNufWGOaVCNLJXbaaBWg/G2lVGwEwY2s/hgl4pyVjUsblA26/nbk1FPMZTrPObJKn+alb676W0p86wZnnt4WcUwUDUnolUvUw/gHhJsWrqinLZGTcECDYl6wrQFWnyH1pYvXVAUhKRv5+Fk+nFlJtSbow3V0naNfZx9lHwtw3VU3YlYREuAb6aWSPWsiGXRoOyUbabeHqBIOy5Oo11m1c0rUUVTkpmsZIjLrJOp0sKurpjSiU/wyNY6DayKXs4eWCDEpH4/dbfak0oQIIUmInfazeFapeKcUqv3hSJZCezCcw6mdLapa2mLoV2apUk6mZFkJ99SlQD3QICpGnxsMl1LVYiDsjLrtt+gs7aYXTGqcEs3jWMBoQ3oCBYRNWwpCYa0jrrttZDrnZY31BkgzvPL68bCGpCGwpSoCU8oOv19/jYnh2hD8S5Gqqsp1qSyWZzeJny+f1Nh0vUyco7IZUkQn8Y+trN52pGZK0kgFU+1JIB15+Ph77KIqATprBgiTr1/A2cU7UxqXWRbGbP7zuhdPStpS6QISCdYIMa/wCW0KXfhJFBfqFUyltVlU+sPpc9t5SicqTOvdmfCPE2mu8KxtlgVDnZhDUkhfSNt9P1swEv3FTYjVelLnRSMKltrJ/eZoC1E7giVLHUjxt5rxrR07a2nzVHL536t4uNWDMb69Lm3a5B3AXiv6SMOoKzGaEVFd4fxREEozNGaEB7rm+wzHLc+zmc07tCku8qC+Gbjbu66PWSlaQ2+61VGmWUhIB+217IGCSRKtIA6Q/w1wbfK+IVU4aa7KOnLx9Yco74qLzcWzJACnn5hUawAImNYky/XY6urDaE07jpqHMmbsmBo5I0KjsBp567WY1XxHvCioHLybxXhu57wecJZu6ro81OW0gSEhKm1FW0rJME7AaW6SHFKB83gGygyD3b66fv+q95ouIsEkxM4HTVUZqWC3LzAu0Got1IGpG4GpTdx3duK6DiFWXaHbyebplZW1Ul6lkpYX3kuqZPcdGbuwQSIOom0rUak013U9GkJJYZbb2BAITG3LraJmOJV6YnvH97YjorgcfQpLIRQq7TIIEKCiTBEz5Ee6WWmFuU6ag+1lTI2O28Hb9fhpYbkkkfbotXH2uhZHm3P+ySLwWVPqyEJ6gE6z/SwsFaQ24SArunXnY0u6qipD1TSpKm2U9/vAGYJgDnokmB0sGUAMBJ1hIIkRoP1+traXK1XPtkuF5s3lgTg5d993hd1U9cIqKardp3W1VaUqSpC1JII7QQRER4W7awXwSJASbiWk94ZbwIB+Dmmn1tZpccOBd6vcYcZv0lQ4luovusfab9XWrRxwrEEDYZxr/WzFc4E4gCwGqgEKOn2S5j3DoRaq6CEnWcgq82aUD9gCpxTw24KuNoPZ3WG3PYCbyUQobA+34ixyn4T8GagqSlNGMoJUTebn4Gemnnav44F4jSJ9bTOxhhwgKnaSkR+OvvAd4LYoakds2SQYHZuakcpjf652Y0sbtqg/z80QVT26mnCs4rhzwgKeyQ02peUJAbvBxSZgSPMAWpzeIvGtvapoCpby2luNoQ3qBlmYA8E7++y/U8IsVMJ7QONZOROcSens2ZlfR1V03i7d9UCHqdwoWJO4Pxtew2kZAXZZM5P0WfiNS6YNvFk/en1cOF7hrsKVhvG5L3Rf3rBTTvZgKcNED2knvSIX4W5VgG74UgqrEqB2zJPTonp+HSzfwnhXEGK/W/3M+kepJStztHSnRUgR8LKw4ZYyATFawCJgB9WnMja1mR4Y4gyWVNoL2ghl1J2BeEnBvFFbSXE87ilNe8yStwrZQyVoTK8sJVpvE2kdz0P+FLTvZu4gvtk6aCsYJT5y19aWrnRcPeI7D4eu+8y06BlC261SD5A6dfnZTZwnxnbCQziitbypygC9nE5QIMb6bfK2bNHK5146qwWrTzRNbaSmuVP49DXhY6mEYqv9IiRL1MNf8A2c/jpZgcePRkwhwy4fuYrw3iO8q2pYqmWXGKhbSkhC5E9xIMg5fjysyxh3j8ggJxjevcOYRfbndPX2tLB3lhHjtfV2que9MUV1fd7qg4unfvhTjRXMyUqVBPPa0YW1EUjXPqgWg6jTVFnkppYnMZSkOI0KiTIC0DmEhUWuxhD0DcD4rwtc2J0cTrxT+9rvp67sk0TXd7RtKymc+sTFqzI4GcRilSfUGACrUGpRqRtz8fnZ9XXdPpQXNd1Pdd1YwvVijpEBqnp270ORCEiAlImAkACBt87HxOZ07AKSoawjfbVVMMiZA5xq4C8HZTLWfs7bkRpR8RbyWraP3ahUq5ahyyY7+zrrQFOIx7VoQJGZdyk69P7wC0c1N+el/hygqr2/ttfzbFNTlx51F4gqS0kEqVvOgmSNbM930rPSHfc7V/ipfLqoABcUhcR0lOm3Le2dDT4zKLxVTXD4f7LQmqMIjNpaZzf5+Kmhz9njfwgM48LgUCRNzrGg3/ANpy52vX+zu4S33wRwPivAt7XsK9t+9Wr1YUKZbOUuNBtYhRM/3KdreeWCeMHpvY1uz994PxLelfRB5TXbJoqQpC0gEp7zeggjTbXztc/wDZ84w9J68uIt/3Tx4crX7qeuYuUDj9PTNhNSh9EiWkhRJStW8ju2rynEQ7l1NQxwG4H3vyUwKBzM9NA9p7nb81fQTJ3mzQvSn9XrnW40Ksw15GzwOnP4WQsRU+rVQBqe6r8LX6F2V9u6oVLczbpFCQrziIjlYRMDXziNLZGwiefh9a23IPMiN5tuNWWVsAg6kjlZVo3CpmOYUqPeZHyNkrUTAPQ2P0Kx2SkA8wdPh+Fjx7qs/ZHRG4B2tsEgQZ+NuAYGhIi20qJ0HPpayhLsQNzNsB+VuR1nw0tuZ1jXraSjZCA7yIFu0nWJFgRKdCLdg6A/C00y5rkFyleA9rIY05jUW4YWHqdp0kypINh1EREEjpYpRwmm7Iz9mpSN+h/K0Hbp+i6WIEWcWB38lZUU/JxvN5wf1Nm85r1352U8KOlq+WSTAVKT4yLClF2p2p5lCpPnbLGyyCZtlsbwZReYF8y1yMDtqJsrT/AHqClMcyofD63tZ3hc36xf1YhCkIHqyl94EQntE89eg+XlaugumruW/W7uvOkeo66iqEMvsutdm42sLAIUCMyTrz6aWsrwoYR+8a59TqJSwlGTKRoVSeWu3n+G3R6vVaq0jKkZdMppAl5CkKITlSqR4ae+PrUJIKHQsEpBVm2nQHr77HcjRkKfSJEknpPM9BbYZbGZfboBGkZuu8/XxtsgmyyQUSWlWYo7Mnp3oEfX0RbR70KKRBIkg6n3fXLxsdfYYOV3t0HQagmVEe7cQDrYFbSfZDiQIHsmAfL57/AK2ZI6pWw0UmncgElDqTMQAY+Hx8bLSlJUrTmZSOew0iyXh1lC2HkBQUA4gRrA36WV10y0lJCTpqOUjSfAc7U5fvLQg+4gToc4BkEkf83SwagtCg4goKiSSd+m/32MKRmbS0EbTEaa6nbz+6wawlKhmQRGojXbb/AMMfGwUYaoqrOU9pM7E89IOke8H614dGeEAJCUwMo8xz933WMBlSm++leqgmVTtpNi9XUt0dM7VVSyGm0FajGkAT+O332Y2A1UwCdFD+M15PSE4TtwnMm+qNSincD1pr5a++PC3py0tKkiffP328tL0Zqr99IPhstbop3q6/KFPauahn/WWwgRzjfQbydzb0vYo77SjvXuDm1B7H9bYVa7M+4WnSgsaQU41V766VVAnL2LmqtNZ05/CxZTUJUkwYGlkoUN8r7v73jn/dzpYYUF7BKovRQMRAQLCYSeid4A6o0xXO3VmdYy9ooZRmTMCyOX1qUpa1SpR189/wsWq7tvjtZcvdzIVGAECxJ677wIg3w+COcDztbbtayqOOqVfWRvnO8QNOVs9ZzJzJUTpMb2gzjpj3EPDRy4aijxDUoar3H23Udmg58oSQdUz/ABHY2jBfpJYuCxlvZS0yCokJ90AJ56+4WRcGm1lJrSRe6t6qqGYnMYB+jbSqgToSRynf52qO16R2ISSFXw8CNPZTJ337unnYQekTiLTLfVTCoiEon5DSbNn8k9r7FW09aEHfpEjT4Wz1pJElZmI0MaWqYfSExB3Qb9rjumAEzOhk6eduf9Pl/k6X5XJEwRKQfu1s3NA6Jgw91bYVYTJzbTr00tv1uTAKZGkRapX+nvEJ1VfVcDzBWNOXLyPwsJT8bcR1BJVf1dBMlRd9n4b2Y1DR0UhE49VbH1kdddNRblVUQJJEeGhm1TarjpfJWQ3fl4wJEh8gyOfh/WwaONd+OHN++7xIBA7z6tdfP60s3iG9kuUe6tv60MukkGZi3CqoyIB05+61arq4q11UUl69byM7/wCtOJAkaaTZwIxy5UIStN718EQT62ufHn9e+1d9e1vRGZRudsVO/rJVomegtpVSQIBI5bSLQvV4oqWqVpw3rWjOmCRWLmSfOxVGInnWnXP3xevdGg9dX123sL7UZa9kYYa89VN5qTmCcuoJ1BsDWvnsgoczrI+vC0Eu35WVICGsSXg0rIQ2VVbqkk9SAoaTHyFgbrv28C9lev6uqFJ1J9YdCdAdcpUSNfH7rDdjMf4URuFP/EphXVKWw0IMypJnaJ52ENSoUjwBVAc0MfXW0Ts3/VlBUa905lpQCqqXpzn9fCxa+sUv0rq0IvB+FKJGWoXBA6A/j4WAMZiJtlRjhUgF7qYGql1QdICiCoxESPrrbbL73qqQGlzBmBpPh8rQV/bZ5KwTW1ahBJmoURrtz8/hYwzjh1bcoW8oR7Xaq1PlNpHF4x7qiMNf3UzNLecfCVpOVIBUY+XwmwjtS4mtbUlCikiJI8TztA95Y4cbQULqHVE7lLx/AxG3wsgrxs8FHtKp0g6hRfWSnrz+tLPHizHe6hvoHN6qy9dVuliQFEaHu6/X17y7la46GqVCTOUSRII+XKTatzmOakSDUOgQBBqNp57+fhr52BYxhXVryKelLtQ84QhCEOkqVPLfzH9Jsc4gwDMRYIElMImF8jgANSToABuSeysvV1Sm+xTqCDEADUzvv5/Oxw1TmwSqZP8ADJJ5QI158rVyvCmxfQUnr7tBmbR33UtO5loSJnQHXlO/XyaJx+DmSVNDvaguynMI0OXzGnh0sGixqkxBpfSPDwNDYg29Fl4XiGHY5G6XDahkrWmxLHB1j2Nj/wDe6tPiNTtVdyqZtCyuZCddgZ+vIc5syMQXtdd0XTUX1Vlymp6Cn7WrJTKUpSnVWnu0jnaDWMbVN4uM0t2Ip3lrdDYKDnLh07sTBO0Ajc6TZwYfwrj13Gt2XliK76L9x0qKhbzS6sqcW8plSG15UjRSFrzJOaQUBUyEhPGcYYVhVbO2pxKpEQLQCLAucGkn2NQQ7UjqDcAheSfpU4bwGpnbW4tXCElgBYWhz3tYXOHL9oOa72nNvZzTcAhSHhrFFLiehZv6hYrW2FuKQ0K5oIcWltWUHKFKGU5dIJ0s87joHL0XeF5U66amASpLeapLCgo6nIUkEGOn83wqF6Ut0YhvClu2sdoEXpgtARTpo2UBhyjqkrWhTi3nJQmUkBOdOXQCUkgrhio4p33dOBqy8eH2Ol4fTdVYKI3Led9JqatTeic7TLrKlFG0hL0JymExrYuAfo6kDhirpeWMzniA3JZe4ALySSQ0+1oCTuTZQ4B4ClwTEocaxN72MMhmbBbMWOcHNbmkc4uJDXe2LC7gMxOVXK4yYwwVwJuavx/WUl2VuIq1Xq9Gl51T9RVviCltS1KKuzBBJAMQdNYtYrBF93hiXBNz37fN0JuevvS72KqpoFOZjSuLQFFsqOuk8xO0i3kRgjF1XefECi4gcW37yxxdeG6+ncqEKr1BC2i5CcgdTAbDmUqTlSFARIkm3p/wy4+cMOJVyvXhcd03xRN0xyqRWUCUpWoiYQtClIUNQdwYIMW7N0dPgkBnqnhreribDyGv8le38Q8SUrIjV1jxFCzq4gDXzPU9An9W1dRTlLVDUPIC0lC+z/jBOsxp9e+xQOuCnCYkgEb7fWvy6TbinobnxJTLcuVpp0IGZxsoylGmpVOgEDysSaw/StqcbVRoSpCihSVIAIVrO/v+BseGeKrjE0Dg5jtiDcH5hVKGtp6+BtTSSB8btQ5pBB+BCrnxXQzR8Rb57NKFhxTa0EkZUKU2gkx5n60s2A26j7ZbCcqlykFyCRzI0mPHaNLSHxpuVFJjlKadDbKXrvac/hSBlzJ5+CR8epsyGcqJyMl0+ylS/ZSNTt5C3I1pLJ3DzXodC3mU7HeS4YDKGM2VSXCmMuuh6k79dPuix9pl54CoqmlJbQQlClOd0mP4QBJmeWxFg2aFx0gKzQrQZj3R5/H52VE3S+6UstqXUObQ2nugeBOp1jlG9s50luq02R+SJiqoEKbd9TdUhoHVbgCAdIVlAjp4RajfpF0xb4v34+CFipLL+fkoqZRJHhmm3ohddw193RWIp6VtxuVhyobQ6EkcgFApJO+0+VqW+m1R1I4p0F71Csy6+6WitwNpQFLQ64kmAABoE22+G6i9YWX3CxOI6Y+DD+xC59C6n/e2OcSYVDvZt3lcrskqEDK4gAmd4Cztr94tUOHVUzeVHedNXUyl07yXyhLQyZwyhtSRprIC9uZGuhtUT0QHew9IC7LvIKhedNVsiDz7Eug7TugfI29J7rwq5UOJHYFSU+0IgRoTJ2+NqnFc5pqwG+7Qf3KfDdLHVUftjYlR5hPCDdF6w7e920ZrKqsVUdo02F9mgoSnKMwEGRMxz0I0s9LrwFh6rp3PWLqp0SkhdQ8wgpGsgJj2juefWN5e1Ph5xyWGm0JaVHca9lRnmd1e/odrLtNhyjpkJNQpS1DQNI3A8eVuJlxEnUFdWynihAbZR5TcMcL1ZhFz0riAqHHl0bYA0iZJn8fCylTcL8IUzwSvD91VCyg5Qu7GyMxMwBGo1PLntaSGbucXlSy2pISO6VQVDy5D4WWKHDrrckDKs76ypXmfztRNdNJsSoSTQRC7gFHdBwrwSVKevDCF2OLUZCVUTQJ6EgDSy5d/CvBbkBrBtyBS/aWu7GJHl3dB87SlcuBKyrUFKYhvfOe6LHr0vXAGBABe94N1VZsmlY7yirkIm12DDq2obzZXZGdybenU/JYNTjMOblwtzO7AKLav0d8I4puK87jTg663BeFI7TEpom0gZ0KE5o0IncW8OeI3Dq+OG2NLzwjf7SGqu7qp2ieQlYUEuoUUqExG4Pwt9B3744n44p+xwrczeG7sWSkVFUClwp/mSga/+Hzt4Xeltcl64R9IHF2Hb9rfXK2775qEOPrIJdBclLhk7qSQrUnfW3ofCURpXOjjLnNI3IsDbtfVcvi0r6huaYtzDoDcj49Fdj9nHjDFXEzgjevCXDXDK4HG7kvNHrd7FlKFLQ+M4U6o5ipwFCgDHshIjSTdrhv6O7/D+pcxZfGJHqq8QhYFMwgNUrYVuAndXmTy2tRn9ivjVu7734o4FdUmX2KC9WfJtbrS/wD743b05vO+e1bUzOihZYlDRUta+SUku0sOgQoK+qdCIItG9fNJA22g2J3qwX6FwACUQse6xydeduTChlOxBtfhOUgozxcEJnjrIM+NsEzI0BsJUtFp5xpRPdWRPvsHObmfwFuiYbi6x3Cxsuk6jmT47/WtjlArvFPUH5f/APViknlvy52GpFAPJAI6bRpB/SxmGxVZ2oSlJmI2NthUCYjxFg8xOo3tvMAYi1oIBF0JmmU78rbkxoZmwQUZmOdukrPWbSToQGfG3aVDloLAgkwQdT7rdpUfjpaYUEKF7GSDYlSLKausYV/C4lY8lJH5GxvNy0sTMoveR7LzHTmlX/4VovGxTjZDu6CT8RYe6XS1XNOj+BYVvYBzQaH4W7u8/wCsA9TZnC4SG6lQaibZZPp7wR6u1mInImfhbLU1KxXza1ldW1yUXreda7U1SkJdeeqHCtasqQBKiSTCRpPQctLWJ4I1KaykqLwbGj1OwqCIIkExHvi1dqhoC5UqQUZvViVAmJGXnvzPlJtNHo1VqqnCDruYlxtSaYnmCCSOnJQtboxaRAqjdimwvuQIXlMCZnQ9PrrblT4AMJEBUzOUT9fW9ioWrNlQrvHQawdfr77cKUVAq5JITl073nr1jpbYWXlSg09naUkCSlQUBl0iNfHePKwIcMkp0KtBHhzH6WKIWppxXcBCu6dTroPmB8NLYo5XCSVKzanmCCJjXyH52jspp04UrMtQ9SlAJcAWmT3RlOvwzE+6zjW8gkIIlSEwFTvBSdfjFo5uysTQVrDwBIbVJGXcc9ORjbxs/EiEhKF5kkCFfzDuwfutTmFirtObtsjDLyErAKcoEESAeUfiN+tgHC2oHKkndXgdD005m3cpU2kgbbjaCSBoNtraUSVGVKhZ2+A29+21g3CsgLlGq8qjoSSdfE/OyNi4LNxOICBo8hcEDTvafCPu52WFJy5SVykwRoNtPxn87JWJcirnezGEkoPQnvD5am0JD7BRY/vBQwqo7Hj/AMLqk7NX9QyAT/6W1r48/qI9SWmzEwTAJkdbeVl5ieOHDWFkRflCZyx/9abP5/Hx09W2UKKQCRbGlF7K606lFlEtANopsyyZzz7O3626AVlUQdTsPKxlTQ5jfwtpxPcVHTn0s8YUJCidM7RNuxejCnWgNEpMGbIa0JzqUlEAqOUEagePjZUcQcypnfp9eNiym40BEeFrYFlUc65VX/TOuervahwkzTurbisqc2QAkjs0jW1bEYBvlRSHKys0jQhJGnmOv1qbXy4j4RYxbe2HqKpkBlyocSR/lTp7wDbig4G3e48h4sEpJzSEgHwEAWcU5kN03P5YsqNt8P7zXI9arQTr3SjT/lnx99jyMCVyUgpfrTqAmQmfd3frS17F8B7o7uRlUpgCUz/TexlrgVdqDPqvxH4Wn4MqPigqIJwHeEBWesIBPtZdd9u6PxG9hhgO8gJCKzQiCVDUdfZ00+6192uB9zpVnTTQeo0sMOCVyDUUZgHTS0DQuUxVjqqG0GAK591IcTXgaD24jbbTw+tLLF58Na273TTUVNWrbCBqp32tj003td5rgvcqSFeqGec62Wn+EFyuHtnKRJlA3+dq76B+ZWI6tll51u4FvZHdXTVZnQ/aRI3108rHKHh7XLeSk0tVvAhwxH1+Fr5VPCO5ipR9QB3GwtjPCK6GzIowPCNjaf2a4jdD8a0HZVawlweoKsJdrGbwjmPWlJM9NvnvaRKXgnhVZS2bsrdQMx/eT5neP44tYO5+HVFTHuU4TpptZa/sXS5irsRrEEJi2VPhMpdo5Xo8SjHRV+e4RYPp2mEC7HnZBUCq8agkeR7TTrYOp4WYUZonVtXNWJWiEhPr9RkM6kkFeu/1Fp3vXBNQ4EhlkxlKcyAPj9dbeZ+OMM/tG1YtvtrDbWMV3SmueFKpBpQkozKCcoKQoiIg78+dqZweU6F1virQxKPcC6s4rhldYqFOtUr7SEtxrWvGDG+qjZKHDigU6tSG1FQB1TVOyABr/F93nasbOAP2glc63+8aTHzhYPaiH6ZAGkaRsTKhpy052U6jBPpnvX5ddSxcGProYaeR6x6xfQdQ6M4kqzH2d5AjewXYK+37QK0zFWj3FYSnwLd7GWWTLS+6nt3e74+1YpiTBTDb2UMFaTmJJeXJ1GpExP1ytL9NdKl0qu3p0T7I8/CwWIbnaQFqWwhARoCRrtqflbNGGyB9wVf8aws2UDNYLS66EqoQsEGR2qzB85my3TcPbrUwCbrCssQStfLTr4Wkpm42y8ooSgrS1JgbfGy3T3O36qgpaEABJMc/G0J6GU7FRjq4+yr5fWAaZBytXelKhrKtfkfvs1v7FMF1LS7uZOdUAFoGAZ/r9C1kL6uVCqsd3QJMg/M2bLWHkVl8BaBnQz3tVA7g8vrWxqagltqUGSpjc7ZRknhjdRZQXropVKUTu0NenLaLKmHMDXfcV7NXg3dtO240FJS4lsBXeBHu3jTrubS21cgdrkJUgJSlJy6aTyI87FsR0X7sonXm8pWSlrMlEKClkICh5TPu87NiVCJaKZk7iGFrrkdrarC4nZRVmB1cFYS2J0Tw4jcNym5HmB307phV1ThvBFDVX3e9axStuEuP1D7gCnFnZInedgn4Wg2v468J6XM6zwwoKrKcqlIbamJ1/gnrv8pm0R+lNjVddxXvRtDyizdDbd3tpnQwkFfv7Rxem+kxraNKepFbSlxwqkDRsgqyEfyzBO0xy0POBn8J8A00tEyuxF7nSSta6wc5mUEXA9lwvofgNgBrfwXgn9H1NJh7cTxeR8k1Q1jjZ74w0Wu1vsPF7A210FgA0a3tRhv0kuGzuKrup6rAtFcyO2WybyU8kmkBBGbRsQkmAe8ICp11tYAV13UV2P30pbCaXKuqW8yrtErbAkLkDvSkDr0BMTbzXYfq6R0hpwgEaKMkgyAQJG5MbTyPODcnh1iynwpw14Z0t5JSaG/lu3Y+VAhKO1Ky2YjYrCE6xAWbYn6QOC6WiEFTQ5yXHKWlznEhrXPNi4kg2BAGovaw3vyX6UeA6SjFPV4fnJcSxzC9zyQ1r5Dlc8uINmkAai9rAG95NxjffD/CT9TdOLr7w5SCrbTUVVLV1bPbOAiATTyVrmAJCSD42jQYW4HV1yVGJbm4TYWvG7mlOZX6RphbbpSYIQW0kT/h0I+E49wTuCldqbtapXXu0fW89UOhvO7mg5nFFRU6eqlCTrvBFkymwDiLACqhvAb7VGq+vs6xpAQwz3T/AH4ags9ojQJKUJELglUC3szf0YYXilPFieHuk5MjWuH66U3DgCDq+97L1PCuF+Gq+kjmYZXNe0OafETjQi405lkiqx1waww0u4K30f8A1WqvNDfrtFQ3ey80hkrgF1QCZMEKCUpUTIjcGz04T454acTsLV+CsOXHRXL6rVPU9VcweLC0tZjNQ2nIlSpOsFIIPtQYlIu3hu5dzjiryzrUFFTpceU4tSlbrUtRJJOWdSdtY1mqPFa5q3AvE28XrvW8yr1kXnSLc1IUSFyT1zIB5THO2VxT+imidh4dTPfG8EEEySPsRtcOcR8xYjv0Odjf6M8MxSm5NHLJHKCHNcZJHgEbey55HzFiO+4NoOL3HG+/R7obgwjgLGL7t+0tS85Upey1Kf3eouFmnqUuSlfdWgDMCqG50JBL74E+nhhziJiSgwnj+7kXdfl8VQYp6ykcz0j76zoFhfeaKlaAyqSRsLVf9NCmYvWswVxRu5sCkxRcrajEEZ0ZViTvJQ8keSLQvwh7ao4r4PaTmWVX7QAJG5PrCBvbI4Kp2NwKJ9yXOzF1/wAeY5h5WdcLb/RkwQ8OQOBJc8vc+/4y85xpoAHAgBeqnHG6qV7EN1V9StMKo3Gy3qCtSVmBMae2qzAYuXOlvtXWWGPZTlkz7iJVEa9J91pm4yXS9UouqrbpwsIddbzLXCAk5TmPX2fDW0dUNzrdWG0I7VU6JRJA10HMWxMZeI6pwX0RgjeZRsP87opQ3OyAE0LaVvZc0rSVK5DXSBvznzsuXfdDtGPVnAw4onMHFLUUoSBuRG+2nLp1V6G46ltqHKkpUe4W2pyx0UZgn42c1z4cXUKC1tLWjQkJcj57b/W1uanqg3quhZCLXTeZw606sJU+l5RhHau5gADrKUgQn5mNrVC/aJYPN1VGBsQNStqrZrqMuZAJLamlaA8u+T59LeglJhwvqQlNOEBO2QcvE78hasf7SzBqVcJMOYhUtbi7vvzsFLSNG0usLMTHVpO3Tna3w3iBGKRi+huPosriFjH0D2dd1SH0Y6v93ekhw/qHV92qvRhiZj+9BaG3+a3stRXEktIbaBCAcxKlGATuI2m3inw4dXcvFLBeISlQF3XvRVKiAYKWn0rM9NJt76XXhR2qKZQvLykTpbX45gkqaiHlC5II+v8AuuZwCvFFTva421v9E0KW4EKgoWrJoO5oTPLUaD32WrvwsXIbbbSE8zMk2kCnwlS0jIfvB5LLaBqVKAgW0nEFAyr1XCt1OXk+NO1AhtJ/zH8Lc9Bw65lnVjsg7buPwA1RJ8ekluIBfz6D5pKuzAa47SoQlpsbrWY98C3bt9YVuFz1K6KVy+68bNU6cwB8SNBZVVhW/MQkOYovVaGZn1SmORB/zHc2Xruui57iY7C76NtpI5gam3T0eBiIB0TBGPxP1d8m7D5rEnxDP+2eXnsNB8zufkmW9cHEPGfdva9EXBdqv/q9KMzyk+Kth87K+GuG2DMHKNTQXahysX7dU+e0dV17x28hAsvVN5Np0zifCyNWXuSTCrHlloKE5/2knd2voNggiWpqG8tvst7DT16n5pZqLxbaSUp0HIC3hH+03uU3P6XOLKlKSG7yborwQCN89M2Cf+JKvhb2qrL0UQSV6W8k/wBrZcym+MmGsSoQOzvHD6GVK5lbL7s/8q0WngmNOrMTEbjuDZSloTDTuemv+y3xcnDXpVfu0uBKMQ3DXUQEwCUhFQI6/wBybevVXewnRe1vCD0QsTuYT9JrhxfCFBIcvlihWR/JUSwT8HLe2tTWymSs9bZHHbn09bG9uzm/kVrcPQCpiN9wVIbDgdZbeEQtIULdk+6ydh6oFTc1K6TJyZZ8jH4WUCJ6W6Kjk5sDH9wEGZuR5aeiQb8pstSHuTg5+Fk7aEzZwXyx2tGFAGUKnTpZAMg6ae63RUrrsWTUNs9YNNQkCethGVZFhShsR9+v3W4MnQHblNsVqDl1JBA94tcCpvSkFgiALbB5bToTYFC0rQlwRqMwFuwZ6ec2tAqsuwZP626BMbaGweaBJPzthjaIj5WmCSkhs2swJOlthQsDmEjWY8LCJXB15c9bSG6iUKD8x1sVrXA1U0bigYLhaVp/Mk7+8Cw+adATvYlfJKaBT8asLQ8P91QJ+U2d2yQ3R5YA1i3VAftozDf5WDWQoBSTyt1QmX4kanrZ7XTJw9sE6ZhppbLcdrGmf6+FstHKnuvAUYYvOqebwxSoaXWuqNKElzuFY7p1O4m0pcDMA4kwLdt6XdiBpltL9SlxgNvBySEmTpt/DpvZoUF6NXfi6nv59t4oRWqqHAggq1UTA2H4a2k5PFq4ezzqoK4JOpkIIA16K+trPAGtOYoM2ZwygJ39k4eSoV59Z2i2kIGaFjSROwJs1k8UrlSfsaGvUNCdEaHx73hYRzirc7ywldHXFUbw2eeuyvH+lrvOb3VUwv7JxhCAnUQIOgHhp5bWwgFRSRMkEEKGp8Pz8LNUcTLiUBN3V85eYRJHIe1+e08rcf6TrmkKTQVneMGEI58jr5fWto85p6puS7snZkglIEkDQRPLb662e9x3gurupgKcUVtfZqiRrp+EfO0Lf6T7lglN3Vw2AyhJTPTeyxcvGa5rvQ4Ki7LwWlawoFIRuDruRYUz2kaKzAwsOqmRtfZd7w008tfrpYNRT3XSpZy84Hn+A08bRorjzhlClKRc96kJk+yjTQ6+11+ugLnHzDiiVKuS89DqfsxI33zeNqlyrl2qUCsrBV3RkTCRAA0nTy2+FkrFig5croDSkhRbSITMd8an3kH3dLML/T1h1Pd/cl4qlYAHcg7f4tI+P32Q8QekLhmpoV0ibhvUKUQoq+zKQARuM2mw+gbQkuWlTjc0EXSNfz6G+NPDhaAEpVftEMqp701TR1Ee74nxt6wMJ5+M28c6LHt3444zYBcoKOrp26W/6GUv5cypqU9CdITHxt7GsZcpVbOlBAF1ZjIcSQuyB42DcEpInxHKLCzmGp5dbF1qMajQiIJs8e6g9J7iSVGNfH8LAOJBG2/Kxh3LMDTfSNrAOEak69elrQVQ7qNOLfE+4OESrjxNiNgPUj1S9Rgh1DZSpTRUDKyBsjzibN67/Tw4OUSQHglyQIBvFjnruCZ62lq8uH+C+IfZ3ZjfCt1X9RUpNS1T3lSIqGkOgQFhKwQFZVKExsTYSk9GvgK132+DOCG5Vp/2DTb+9FrUTHkXabID3svYhRU7+0I4NNpJQil1MDNeiBpO+iDYNP7Rbg9EBN366Qb0Vvrpox4WnKn9H7gix7HCDBSY0kXDS/8Aya2VKfgtwfZA7HhVhFPlcVKP+ixCJb6u+ihePsq6OftH+EDQIH7oEfz3q6PI6UxsD/8AtIeFawotIuIpAkk3s/Edf/Nfda0LXCzhk3/dcOsMJ6kXRTif+Sx1rh/gNnVvBVwpj+W7WB/02YiU+99E4MY6KqCv2jPDZKQvJh9IO5/etR4xr6rHKzj4f+muzxcv0YR4eYVuu+L1FOuqLDV4vJAaRuorWwlI1O0ybWWZwfhNpX2eGLpTpHdomh9wsi4jwhhylxJhy+21UlzqpXKppT7LbTWYLZIykqTESBp1AtHly7lykJWDQNTGTjjjvUt9rT8GbvUkrKQTfTfIkHcDmLdN4w4/TrwhutsJ3m92z/1ibSHdd4YLw/d7Fy/2xpHgyFFLtZeja3lqUoqJUoqEmVfdbl/F+AKdwofxhcaY3z3myNf+O07G33lC/wD5UyaXF3HkEKXwzutCZ5VaF6/+2FlZOKuNb4BGB7qR1JUhQA6f+cWVXcf8L24S5xCwyga+1e7EwPHP4j42KPcUODTa0qVxNwgmCZCr4pjP/P4WC6IO3efoiNkI91Bf2j42FCAMJ3SkjkW0Hl/+NWxV78altFX7huNtYEjO2I+VSbAu8ZeCCQUucU8GJI3/AO1qckf83nYmvjlwBYUnPxXwek6b3owQf+aw/Dt/GfUKfNP4QkhGL+MFTie8sLqo8PsVN3U9NVuFwBKHG3y6E5D25JgsrnQRp7gKuh4u1qyX6vC5QRBAfiOXUxZvtcd/R8oOMt8349xHw16nXYbu5j1hp4ONqcaqas5AUgjMA7MfpZyPelP6NDMuHifcnKQlp1R+SPqbBNKxw9p59UcVD27MHok0YX4kAJCq/DAIVIArCIA/3TYC8sFcSq8govbDDQmSTWqVJPP+68vjZRd9Lv0aGu7/AKS6GYPsUdQTp5N2Lq9MX0aU5QniE24Vf93d1UR1/wC7sH7Mp73Lj6ov2hPawaPRIqOH3FJDhJv7DQVGXuvk6dP7n8LKFFw84oBsoOJLhaB1MOFQA6asW096Z/o4NwP7Z1C9/YuqpP8A0WJu+mx6OiFjs8SXm4BAhN1Pa+RI8LI4bSu3J9UhX1LdgPRavXhpxCTUN0q8d3Il6pQtbaEoClLyiVR9lrAPum1frrvX0o3kmsu/gdip1twdol5D93ZXkkSFJzjY6EaDe063h6a3o3VgT67UXnWBsktpcucrCVbGMx00tN+EKmkvPCl13tQAopaykZqGUqTkKW1oCkSBtoQI5WYYZBf2CfVLx8x+8PoqVof9L1SwtPBPFTCgqZVXXOOXiOW1hmbx440jrtVxewdeOHrop2Hqltyrrbve7VaAJSBTDMISSSTA0G86XecbSrXMpUa76C1RfTyxem5cEXrQU9QoKFAmhTG4cqVQv/3WvutzXFtK1mGmmiJzzOZEP/e4A/8AbmK4r9IWIzTYG7D2GzqlzIR/+xwDv+zMV5eY3xHUYkxFetbVSpddVOvugz7S1EmIkHWfGRuOZHD1ahbfZVCEEtaOd7vGO7r0mR1jUxYtedI8+/6sX5fQlOUEElevtJOxnSI1IjyslXQt1m+VoqELQoIWrKtRTKt9dOh5/EW7eNghaGMFgNB8F0scTYmCNgsALAeQTzQIeSsKUCVQUkDtClXP36EToIUdxpPnEpQ/8nHh09TkQiqW4goAnMA7BTHjrPSTvavDrzIpjmiFN5UpTGaJBkEnmMvKBqDziwvEIh/0auHTqEZy7UKWOozdodNvKBvtbmuIz/TcN/xv4ci4riwf8wwr/MfwpFYDhrjN7GvDS5cY0aGKu9FUiGatTgHdfQIcJCTp3hmidAoa87F6u8L0qL7ol3nTsBbweZyoQAkjIogaDokx1ga8rMn0Rr39bwFUXYupDqaevqGc4VqCT2iYIAmUrmff5SRiKqaumiaavGuaVWuvoYpVKRmKlnROYaGEjWZ2t6FwDUOwp9Rw1KdIrSw+cMhJsP8ADkzM+Bas/hcfY1dU8PO2jPMi/wAJ5JsP7j7t+GVFHGKamZbySt99RcKVqUkZc0jflqnltHUAVk9LG52/X7qxMhpWR4PUq80pBJAWmSTGkHw020taahFa7StmsTSl0U4S680UwpYT/lTA3I0GnO0KelHdAr8AuVqJz3e63VoWgEkAKhQ6+ys9J66adxi9OJ6CQeV/TVd011pBZR7ienOOfQyu+tIS7V4FvbsXFDU9gpRSE6cgioa/9n4GzV9BzAz2N/STwlTBSQ1dTyr4fKlRLbCSqABvKyjSz29GJpvF2GuI/Cd57tP37dCnqbNshwJU3nBO5lbJ/wBwcrNT0C75aw96UeF2bwUtk1aauhAIiHVsLASR5j4xb55wK9JVV+Hfhkzj+7KM30dmXN8HWosSxLCTsybmN/uzAP0+Dw8L1W4lXY3X3RTZ0JIbqkq7wKgJSRtz1jTb52a1z4QS5ToWhQbggkq0HPQJGp2FlnjHxFufhvw7vfHl+3fW1t33Mlp99qkQkvFKnUoBSFKA0KwTJGgPO1cqH9pLwFpkpUcHY0Sc0d2kpjy/+zjW3PcQ4XX1VTzKVhIIXvuCYpS0tJy5nAOurI0mEr79cpmadVKLumXStKs88sqRoeevlZ6UNwoUlLTYISDKjGk+Uxap7f7UPgMyoIdwhjYciBSU0nz+36WUqX9qn6NbISlWEceKUNSPUqWB/wC/tzDuGsYlPtRGytz8QQEey8K31FcDZUkJUY0k5d7Q36d/DqtxL6M1/Ku+kLz12VFJXpCBJSlDoStXQAIWok9AZsDw1/aReiviliqqK6pxBcfqyM6G7xu9MvkbpT2K1wf82UGd7Rzx8/agcLL6wJiPA+DcP36yq9buqKJupXSsqSc6Sk5pc7oIJEwSJ621sK4empZGyyXzg6AC5/0CwKnEZKk5fd6kledOGBddzX/dVRfjTi6WnrAqoaakEpKgo5VHnAMbkZrfQLhDF39rsN3TfWDrrdFDedExVs1VUMv2biApJidTBHhb5yL2xZT3i7/q7K2ldoFyQIjWNB5n4m3qjwc/azejzgzhBgzCWJ7kxh++LkuKiu2uNJdzK2VPMMpbWpCi8CUkpkSAddrd9U0csjmyEkd7C5+XZYD5GFuVov8AP+bq/wD/AGVbq3RUX9WuVyhqEK0bB8EjSyqgUNA0G6ZlDaRoAkRaiD37Yv0ZCk5MP49P/wCTaf8A/WLJFV+2G9HB2cuGcff/AJvpf/1i1JzJKW/hKcl3ci59ShZXSWEjxbtfRX7qr3QhJhVkarvokHvbWoY/+129HdxPdwxjz30FL/8ArFk5/wDa0+j67onDGOf/APBpf/09ufrYcaqSTynLQp4qVn3nhXrqr3kE57JNVe5J0Ufdajbv7V3gEs//AMr43350dN/+nsWc/apcBFEkYWxtqOdJTf8A6e3PzYFjcv8A0XfRbEM1AzeQK7VTXqWDJMc7efX7Vm4zeGH8D4gbQCaZ2vpFKB/mDK0j/kWY8+lnKv8Aam8BjthfGp/+46bT/wB/aEPSt9Nbg7x34fUuGrgw9iZmvpa9NU25WU7CEZOzWhQlLqjMqTy5G17h/A8WosSiqJYSGjc/EKddV0MtI+NkgvZU6wffbmHMW3DiJlUO3VeNPVpO0Ft1Kx91vedtxVU2hbIUoPJCkZeYO1vAAuNQcpWFZ5SIER59dreh91ftK8F0eD7luOsu/E7lbTXZTU1Z6rTM0zBeQ0lK+8Hi64nMFGStE9OVup4ywWoxYQmnbci9/nZZ3DmIxUXMbKQL23XpdgorRc5pniA4y6pJTm7wnvQeh12Nl4/O1VvQS9InC3Ha78YU2HaKqo3LmfpH3mX6ZDX9+HACkpWor/udSoz8otQSeXPayoKaSkp2QyizgLItVIyaVz4zcFBvoS80to/xAizUdBStSCfZ0PKzsUTEdNrNy9mizWKKRoYUI623aI6kLMqhpmRYrkyfjFszbHQ+/wDCwZUN8u1snqZtqALNJujtMo9ikAnuyj3AxYWecanSLE6dYOdIUSArbnBSD982MBRj2rWW6hVzoUKSdtZjpbAoHX77cBXKR7rbChOxsQaJkIFTAOvhzt0gkQEnb77AkkwAY0t0lQGx132tIKCGCuug67W5qEh+ncZUAQtBT8RbkKGo6ja3RVztJJc0jva0bKjJJQJ841sZoT9uN4HKyfd8BlyngfZOrSB4TP42P0RAd1k2YJuqcIBgarGn8ptlgJ/xx7x+dss2qdeCWLHqi6rmqKqkcCX2EpUFFIIkqCSIGgmT4TOx2edLwX4jvsIUnGd1JKkBYikOmbl7P3eO0mzTx8067hqsbSiSstkD/fTt4/n42ufw24XYq4jLNLhS5365ymZDjqWkpCUJjckkATrGvWLPAxj7l6BUPfHbIq1o4I8SAoH+2d1Zd9KTQGPLpz/OxxngPxEczNnGV2FQH/oR3+H1NrEV2Ha+5L0cuq9adTNTTuFp5DqcqkqG8jr5i0/cG/Rybx/cy76crxTtJGROZObMSNhHIafjY74IWDMdlWbUSuNl56K4F8SlJDCMYUDjmTVKKBRM6aDWfjYKr4JcSKU9nU4wo2luIlAVdpB8xMaGI93wufxc4Yv8Pr6eu9VRJbIHaJnbeNNenxtEFeVurPaKW4tScozGVAcgdZ+/eztp43ahLxEg0KgRXBnH6gkJx9Qxly6XeNd5389rdDgzjp0pac4h0KShQibsSRpr/Ntv9SLTWTrJSBIjTWdOXxsCoQkBCTKhqJjx1tPwsaXiX31UbJ9GXiY99mribd57Tci7BJHx00/Dxt216MfEBWZbvFOkg7kXTIkjeSrpOvztP9wVHrt0tPdqStqGlA7kgCPiLKzTSEqAUARHegnUjT372ovYGmy1YwHNBVbT6LOM1KlXFynBkkhNzDT/AJxpt9CybffoqYmpaNVW9xWbdMpSpP7lCTBVy+0nc/O1o1JzJUHFAbSCN5kbTZJxcSbjffWUjKtqQkgz9okGPHTewXizSQUdrWk2sqS4Pw1V4T47XHcFTevryrvxBdcOoaDRWFONuEFInqBudvK3trT6tgkEjz33t411bmT0maNtKgZxDdKiU6RqyNx0JAt7K06crITrv+Ns+Yk2KnGAHEBdEDWFGdrF3Rv4bWNmP5Trz2sWeJIiedox7p5NUnvjqfOxRYH0OVjzqQrVQkzYo4BmMk6nna61UnbpBxHj9vh1T097PXS7XetPilCG3EoKSUKXmJPLufOyL/5ULQVlODKg5VHT15I5x/JZK47qSMNXYoKyReiRPPVh60LlYQoykEdCZ+uf0LaVOLsWfUOIdorBJ9KrIYTgN3T+a8gDPubt0r0tHEo7mAwfBV5wNuUNWrwXQAYVGUQes/nr91t55UQTI2OkiPhHT4WMWBV+a4Kwg9LS8TqMA06Z2m8lff2VgleltfaicmB7vASYJN4rMf8Au7QAVGSgwSDAAA1Hv93jbRcy5ijXQCZG/wBffZhGE/NcVP3/AJWeJCPssGXbmJ29ccP/AE62jPjrxRxDx4uK6uHF43XQ3VRXheIW7UUzq1vJ7NpawAFaaqTB8LM5DqjCgkASIkCAI3tpt4C/bjWFjuVsQQebTg/H5+6zujBFikJHA3UeXn6N9yXS+lly/rwd7T2VBLYA5aSnoBYJv0fsMEBJve80iDoFNa6bexvaasbAB+mKz7aTlPUTIizadS6QlNMop111gTpYLqeIGwCK2olIvdR2jgBg1IPaXhei0lUd59AJJj/Dp/X3n6b0fcAuJUVPXooySo+tAAAeQ2+t7OpXrCNFE9yCYnnH6WU7sbqVqLocJZSTIBJ5dP0snQRgbJ2zyE7pns+j1w+ToBeUE6j1uOcRtrzEWFPo/cOwCVUlYRuQatR5eFn22+JHdAB0nbUTYwHhJSo5tYPXw/pYfKj7Iwlk7qKxwZwJS4zcpPUaxdIbiRU9gurWUl0VK2yrcH2SNJjawGI+F+AqGgNTRXIUlCtvWXTmHT2uWnjzs/6mP7Y04CVAvYeqkak69nWMHTpo58xYliBtL92VKSiMqZ18Nfdt91k2JhadEzpX3Gqr7UXZh2hximhq6FKKI0q3cqnVgJInnmmdBZHXQ01fT3nXXW5/q9BnUlWY5lqEQBJ0SALK/EEJcvO77sZUguXjUIZIkz2aTKvvAiyddKEsU2MKEhz7NSyU7SnIdo18Pj4WBywTZHDiBe64vxNExhC7r2p6cJdW42HFEkyDvOvT42UMXXbSMXTTVN2UqUEPtuLyndEd6SDsANff1Fm/X1Ad4QUjmUZ0JaIlWWD2g1nlpy8RYbHF+NDCVLT9qe0faaWUoUM2VIE6cv10s5Y0BIPJS9Q0NJelfSXk5QJaoFVKEMpBMOpPMp6E689rSAx6RPGy6se1+C6TiTfDF0XfTpapKZhSEpZSkhAAUEgwAIif4bM96tbauWhW1GnYqCkgZR3DGvPntr8hYjRXfT37xgxcVdol2gfUGw24JKVKVJOsEaA+7zmuXtj1RGgvNgnxjL0geOFDW3IKTi5iRlmoquyfSmtIzDMkEbTzO2tivpYYovZ/DeGMPXxedTX3jUoN5Vz1S6XXVKSkIRmKjJHfcAnbKBZZufhXS4rrburFtVDtNRVLa8/aBKEpBlQ5TsNB4crRf6WlfU/6SagVaVopqS6qfsgsR2gzL9iYkSpYPLbnpbjqutp8V4ipKON4PJD5HC40Ngxo+IzE/VcBitRDifFdDhjJGu8OJJntBBs6wjYCOhGdzrHW2uygW9WbmZYFZeTrocHeATEue1oJ13nXpBnqhXU+aquXUKDjbKGyEJMkJBVpvuAfu99hU3bX30+XnnA0ypQCUZ5AJPw2AJ8culuKBz1e/wBxpa0rIZCTlIOWNNCdPERtp0i3d3XeAJUrSEJU6txtGaCMnLmeeu5kk6gmfa0sTjVDTXov8Ou0CC2h52RKSkf3vM6adZHhrANcKh/OjMpwZScpmdIAA1JgzGw/U2Ex9WN0vo0cL0mAHqp5nORGVf2sHaR3hGgnW3K8RH+m4df+2/hyLiuLB/zDCv8AMfwpEc9F6/2qF+96FdQplx+ubqELUvMAC0nck97Nkka7nQATawOP7uN7Juu/6JRUhDyErCElZCwsQQBPMETvoItWX0cK+hpb0v5h2mbfdaYZdZW23KQsqVMa92cqToNhB1Gtp8HJQu5zdNes9tWS+lDh9lzQkSDM6AxuINuwxzNQUlJxLELmkcRJbcwPIEg88hyyDtlPdVONGOwuSn4iiGtO6z7dYX2D/jl0eO2UpOfevlbgRTILbTOeQFQVyUgqnfaN4jMN50j/AI10dbVYHrKGsp0JdqqV1RKgoiRsduZOggQI23tK1G5e1CBSXgpHYIcWe2Qr2ZkbAafxzqdxtJFoP46VWKMZKrLruapXRU9MlSStEhQEagRAEkmVawOlvVat7TSF7PazDS3W666N7ZLOadDrdQZ6MmJ14d4u4brXXVhqsqF3Y+CZgPJyJBM6DtFIV576nRy4cwFeFyenDduG7oYU061i5m8mcsx6qtQqYB6JbKgfAEWg3D7tddN7JaoMxvCkqU9l2aCpfaoV3SANzJ89NPC/jGCcVvek7w24zXRhyLtqaFFLfgecS0uiUpCwFKQohWYJdCdjBbgxpb5vxaeHCOIop5nBrZ43MNyB7THBzd+4c4Dz0XLV9ZT4JxbTVNRI1jKmJ0ZJIAzRuDmXv3DnNHnYdlaziNw6Z4q4Fvvh69WmiRf9IaP1kNB3sSVAhYQSAqCAYJ6WgnDf7IO6Hyl1/ipWutHUqVc7aUgTMSXD8ptbDD9WaO96V5tNPmQ4I7cw0TMDN0Gu/LeLS0vC1/4gV2mLMSpRSf8AoN3S22of4lnvK+Qs+JPqWytELjbsLAfEk7fJesxPY1vtWHmbk/IKheOP2ZvAulwyvDtw44v68sW50paNFTMuIAnvBSD7Ok69oNTtysh4S/Yws3igVmLeLtZd6XIPq7F2NrcAG0qK4GnITb0su2iw9hymFLdFExToT/IkSfEnnbHr8SAe+LVI6/wRLqia/wD5W7ep1/JRkkMwyxM+Z39BoPqqDO/sa+GbKJpONWKWnIAzKoadQ05RpZs4j/Y5YMpLorKym44X67UMU7i2m1XUyErUlJKQYX1099vQ2pvw8leVkevvcLbUhS5BBB1tl1fFOQ3iNkWGimksHbL5xMR4fcw+/wBitwu5ytJOXLCh/WffZBJJ31tKPHK6nrsxtfd1uMFv93XtW0snScrhH/TPvHui429Eo5TNC153IWXUsEchaFgJHXW2tSZmzgwvgDG2N1uNYMwpe1/OsDM81dtG5UuNJkAKWlsEpBJgEiCQY2s52fRv4/1AJZ4J46UEgqURh+qgAbknJoPGxXTxsNnOAPxCG2F7xcNJCjePH5WyPqLPlPA3jMq7zeo4UYv9SBCfWP3JU9nmOgGbJFjd2ejxx3voFV08HcZVSRrmbuSoIjzyWY1MLdS8eoUvDSn3D6FR3P1NtTaVP/JV9JD/APsdjb/8yvj/AKbaV6LHpGp34IY1H/5Gf/8Als3jab+0b6hN4SoPuH0KiybZJtKafRZ9IxQ//opjEaTrdLw+9NkHGnBfilw6pKWux3ga97hp65a26ZyuplNB1aU5ilM7mLOyrp3uyteCfiEzqWdjczmED4FM4lHZpIBKjIUSNB5Wtd6LfoYYa9IThvUY4r+INddFRS3q9dq6RmhQ6nuNNLCsxWDqHdo5WqcBJga6Wvz+zRxalvDuN8KOuJHYVdHeDSSdTnQtCz/7tv42zsemnpqJ0tObOFvzWlgUMNTWNinFwbq0PoR+jDdPo341xC5dGOKy+WsQXc204w/SJaSlbTmZKgUqMmFrERztcRQAHS0C8N74/wDp3d7ZX3XS40T5tq08dQLT0sgm3N4ZVzVkZknN3XXQYlSRUcoZCLNsuTtp8bJd/U6i2h5P8JynyspkTpPxsHWJD1ItBGsSLbVO7K4FZEozNITSBBGvw5C3Q1ACTrtbpbeVeUjf4b6i3BBHMnzttA3WOu2FJTUK00W2D8Cf/msaBGpI33sRSYqWlzqoKRr4if8ApsaCiTI3GugtYj1QXbrtawEKVEwDEDXysRN+NhvtG6KrPeCQnJ3pPhPh9SLd1DdYtSDS1KUJ0zJKQSo+Z20nW3DDF6Ay9eYWAf4WgJHv99rAt1UFr9+jYXdWmdNG/Px+PjYdN6KJUEUb6wlIUkxAWDG089djFgU0t49koG8j2pJKVBsEBOkCOZ038bYKW8SUk3opKYGYBAlRAG3Tr018rTGVQ1CUKSqVUM9qqnWyZKcqxqIO/wCNjGbnm87JCKO8jqq+VEERHZgCYieszrZTSRoCdQNfE2Y+ScLhgdnV1KTI7RKXBPvB+4WUKKe1kjbUzZNUoIr2Cf8AaJWjptB/OynQwHdQNdpsyY7pYJcn+P8A4rZbvKv+Y/A/lbLJK68IsdFLdzuuAJlbzeVIEd4r59N+Rtdvg7xsxXwkW5W4ZVSlNWyhuoZqGs7bgGxMEEETyPPxtR3H6ouzKUyTUtpMpG2bby8wecbwbb4HpG66/Ltup99TdNWVDNHULBEhClgEwR47dRaNPYNcXbKtVuyAO7XTgxFja9cYYnrsTX462uurnu1eypyoHSAOQA+Wtpa4T+kZfOALvcummTT1FO4QYdEwqNxqLQjjtl268WXrdFRVCoXdVW9QIUAZyMuFI8hp87JArFtL1Ue7vuZ62vgNljHZUWv94KROJPEeux5eT153lUpcdeUTmToB8LR1UqK8yisK0Ob+LMJ1+Ou9uhULdBTmAUZJg6gbR9/ztyumdySUgxA8+WotJoDdAkdURX9mVBa0lJ1JmZ1+fPzsC4gJC+8mSDmnw5mfvsO8ypkKGcApOgkgzGu+362Kr7EgGPaVlgiYAPTfaetpqVku4OrS1eDlE4MyKgFQMmM41HxmPfZ8inKgtbYPdCiIEgaSPPW0UtPLYqEvtuFK0rCx5gjr5fK0rXZUoraFmub9h1tKiPZAJ3Hxtm1rcpzBalC7MC1a7BRILghMQCDHlHwsj4kZUbmqe0AUlKm1GdiM4PPx+FnI6krSY7oTJ1O2ug+unKyLiUpbuKtKkKOVvMAP8wn4a++2e9/sFabGjMFTu8uy/wDKUo3UJIIxBdiRpI0UztGh1nrp0t7KM+yJOs28ar2Qtj0k6NbzaErXfl1E5SASD2JnT/i0+6bezLDZUN5g6eFqb9gU7RZ7visUBuQB42LuAScunXSxxTaQASIJ66TYu6I2A71njUZLIg+n+I2JrGsk77a2UHkgjXl1Firo0g8+X177XGKk7dRbx4ROFaJSN0Xk2Qf/ALU6OXnaC1JURKnCJ5xNrA8a6U1GEGxI7tc0rUf4Vjn52gv1U5DkgwY06T9C2lTuGVZ9QLuRIocT7TpJBkgc9DpbISEkrVsSoGPlY0KIqXlCuUgxsOZB+On9bCG7jmCxlhUdND8bHLwq4YSk5adQcx6yBvbeVRGZSjrymR52PeouKOUKKwAZ0k6eHvFlIYKxS5qjDF6EKBhQo3No8um1m5gG6YtyhIAACgQ4JHjodRpEddLbyxel0EqVpeDQ1j+KUx87OFvBeJiYVh+8EBIjvMqSNtN48bA1eHay76m6TX0LlOXbwpsgXoSA6gKjrvZ5A9rcxGiaJzHuyg6pTxmw285TOZlEBC4gwOR1/pZAfpKJhsmofLYXMyqCVDl9fnaQscUVJToo8w1yr3O57vOZ2+tbMOqoqOuUGnNUoMwDGsRYGbM66shmVtkj57s7UhNQjumCUrMak9Nx+Nlm7HrqUwW3KmX3Fd1sEgE/UWT1XHQJWmVabyVdBZXum6LoUg1UKztnMiVT0j687SkNwmYDdD/u5pSS4QoHmJIt2m7aRXdKTJAJGbKPHSxhtxpCACoDSYjnvNtrdAUTnISAdCdbVjmVgFoSK1dtM9xFwtSPyGKqmvGlXlJBylylUQCecAm1ij6NXDutp3qd6pvmFgoVneQAeR2RNq9LqUMY5whVp07Orr25UDzp0r/+GLXqpyheVaVOyqQvugz5T+FnYTchEDWnVV+HoW8Bq+vYr6q770q6yncllxVc6kN9T3U7xauquBuCLm9NW8OErt2rcwxfNHTVHYesLzrCqdwmVyFT2iOR6dbehaSyisRTtuuAqUDBIHPa1OOL7tPc/wC0RwRVqWEG87lomQB/ErtX0RI5x1/DRP0F1K2qkVXoaejlT3aLjYwOV0KNFMuXjUrQdZgAODmLL9F6Hfo21FOht/hfdbyVd1fbuPurgbBJK5A6pmDp0tJSKZKVuA0/ajMZKkFR362UqNaGK1DTSW0NpVuMoH32fKFMryh4nXPdVxYuxJh652ilq6LzqaNpmNEttPqQhMTIhI3PX4ti7q9u4+L+K7yOin6FqoIM94qZbO/+8Dz1NpQ9IjsU8f8AGd0BkKcF6PuFaGynMVkL0Hkvf87Q1jyq/dXEl1+Gkt11zUAJzaLhlsAH3o59fG2BO/NmarcLQwhytFgC9v33cNzLul1tCKVbiLwZ2UkwqDHQnX3+Bit/p0Yqu1D2HcKUte27fNK28/XFsAKSyvL2SVEbGUqVHLf+IWVrrvSrux5FRd1Y6ytKknOgkTzA09/xsk+lxQ3HVXXhrjKzdgeN6Mi7agpbnK8kKUgEnQHRxMmf7sW8zw7AG4RxRT1cj7skMmXSxDyHGzj1uHOttsAdl4hNwY3hnjelxOSXNFO6bLoARI4Pfle7XMCHvynQnK0G9tax0N1KU1+8L5rFN0raQ5lS5CtZ0EbfUAzYncRFbejzjI7M1CwlCdVEJKhp1nbX87Er3vu8ryV2dQrs2pzBoaJ6T47flFj2FmGV9tU1RKWGxlUcpO+2wM6A6G3s41K9iSxXJQ6IbXmKpUoBUkSpUHcEiNdYmfERP3Ehr/8Ahd4XhJV3alxRICVQIeJ30PTnMxGtq+VVQqtdW3TH7NRhKQgqBSNJA3946z5WRxxTKd9GHholwqDiXlqBMKJJ7Tr5yfvG45biL+u4d/jfw5FxPFn/AIhhP+Y/hSpsejc/TNYzvOnWjszUUDTrZLiVJUpLhBUTsdVaD2o6zpZOlvBK8S0TzWdLV3EnMgHvaQdjtqdDoSd9BFROFbibuxd2dOunQamlJR2gEEFSCCmP4oBiB1Pdta/AOWsAdqwG6ZgnPmglRAMgSdUmSI13G+9vZuHYIq6ifSTtzMdcOB2IIsQfiCukxGKOoidDKLtcCCD1BFiPmFJ16vUNRTMVjpR6tUBDgWU7eMgSDEfAdLRNiC+6V4vUdPSJeccWsBWUwPE6aGOomRZ/3WzU4lwvU0LiFUqHMrtFkBSUtwITyInXx79mHjqpuDhjhty8rwHaVbg7GlpgUla15QO6DEaAd4xAGu2mJwXWVFLSzcP1Zu+hdy7nd0R1hd82ez8Wlcfwa90NNLglS68tK7J8YyLxO+bfZPm0qrWGHcO8OPSbu2oxG+hN2U15JqKhxZ+zYW62ShRn+FDi0q/3Z5Wvfg2nxjeV+3qcRlh272XO0w69d7xUa1t5xavtgDENt5EpATqYVmOtvLjF171974pvCvvRSPWH3SV5EjLI5A89onbTTS1t/wBm3xAqWOLVVgq8r2ecpbxuR5NHTuOqyIfZWhz7NJMAlHaTA/h8hbyzjPhJvEFaapjw32Sw5m5rAk+03UWdqdTcbG1wFHivgEcU1TKnmNachjOZmezTf2me03K8ZiLm42NrtC9HaNl9mgp6aoc+2QyhCyf54En4/daVbtxeKy6qOpQ6ftmEOa+KQbRoRkAIkGQQfHTTxixPDN9LRd/qS3daapqacDbuoeWlPySLYHGDX0lLFyidNPlbr6L27hzD45W+HGoaABfU6aaqWjfub/ae+bFnb5gQF8utmexeSyJKoHW3aax5ySkFQt5pJWSkbrqxhbGbpxOXqpW6rFHq1S8xKrN2/sSXDhSjFfiu/KK6acgkLqXggqjklPtKPgAbRDX+lzgOsvoYX4fUL193k4hwtvVU01NmQhSo2KySEmAQmbEgpKus1Y027piyKLQbrzv9MW6P3RxdxvSCR2d+PVSRl27Y5xrvs4LVlWO9B369bWK9JfGN5Y+xteeMb6o6Wnfvxpmp7GnRlSMqQgCDKgYQeZk5vC1eH05XlonZR3Nve8DD20TGv3AC4HFLeIcQrJegpxivbhHxHvpy7LpavBd83MumCFgnKpDqFhUZhOgUNZ32t6m8PMV444lYKue/K1276RqvbcS6gtBxTDrThQqEwEHNooSkkZok6W8ZvR1deTxcuSmYe7NdUXqfNEjvNL5c9Y99vVz0eK+8Lku1+5ai9HaikQs1DSFpGVtawAvJAkA5E6TGnibcxxfSNzc9o9rS66Dh28sYaBexO6m5nCFztvprb2cfvSpTGVysc7QI/wAqfZT7hZVXVMsJDbCEoSBACRAsjuXpKcxXt8rEKi80+1nJ8uduBLnuK6plIXavSs/eGaSkjXnYk9X+0M38PSyDUXwhJAJPe7p8/r77ElX2guBAKpzQeR1tJsTyrbIGNSw9Xqk6kkdLVG/aLUf724PXNeCUqWq7sQ05VGkIcaeQdvEptYupvoRqgnwAn42gr0vEfv7gLidkAlyiTT1yIOiexfbWfI5c1tvBQ6Ctif5j6qpi0bZKKRo7LyrbADyUqE94AzazXoA3+q7OLd6XSpwhF6XK6jKP4lodbUPlntWu82+xvCob5dooiI2JkbabRaU/RVvs3Hx7w3UhWVNQ6/TERoe0ZWlI+JTb1PEo+dRyN7gry/DJDDWRu816nYavX1DFd0VinIDdcwVGRokuAHfwP4WtqqQNiPfahzl+uNLStB76YUlSd9NZ+h91r0XTXJva56Ovb1RV0zT6dOSkhQ+RtxeENLA5pXZ4wQ4tcELqZ3i3Khpt5zYQju7R0sFqSDrvrbdasJyb1Yz2L7jZToDIsWiCRvERZUv1spcQ4BIKYskZxESTJm2zCczAVkTNyuKxYCMigZ+0QAZ6mPuNjSgd9ZixN5eancCdwkkRyIEj52HKgrvDZWok9bWmKq5dxoCQPhbBO8HW3BIUZBiB02thJ1ixmlRQgIB1nwm25kbGPGwcwTbJUB3iR0PS0humQ4UZ6RbpJ0gGwE6Em3QIG3XpaYUUHWOlvsHz/s3k5ttAdD99lqiSQ6FTz1si18rpHUD2spjwIsq3a4Hg06nZYCvORZuqYpfHZQJWqf8ANbLByvkPkbZZ1FeEeKWVveo0yYUHLxpkhR3GZxPTzP3WtJSOs3XetM8xULcFI+h3MEhMhJkEfC1Yr5Qly97kbAR9rfVIDykF0ctjy362sgFMdpm7RRKZVMTIH3/XKzUoDmuBQaxuYBpTp4kXtduIMe3/AH9h9xbtDeFYuqaUUFJ74zqBB1kKK/gTZt5ihGaCEzMHy/Ow1M9TMqGZKCkqHeCdTH0PvsA4oB9QYEiQUHQQnpr7/j8bkY5bQwdFQbHlbYJx4coqN+uaFW4lLZIzZjoR9/Lb9bWH498MuE+EMC3PeGEr4bfvKpCT2KX0rLqCgkuwPZggDyJjY2qwzXhtRWhWVQI72sjaJ5+FjdRe9ZUIQhThI2AJ9/XSfw+LOYXOBvsiNIAIIRGsKStSER7p1E+A+psSWlUyQYOx2Gh/Wxlx4rylKU9Z6nbl9crAuwpSjAIjYjUco+t/dYyZFnCvP3VRymRp4+Nn9gK8e2oHaB1RPYOZkJBggK6a9fvsxVFKgc0QnYBX19GythW8UUF8tqdWEsugsuEzoFbH3ED4WBUs5jCrNK/JILqTVJW4MxgA6QDpI+t7IWK32v3BXqdbKoZWSEpOkafrZ0NtEr7yJM6gbAg/d9+nWwV7YZuy9bpr1Vd01FYUsrccyVa2ApGXVJgjTxGtuflkDGm66WOEucLKkeKW22fSPoRmBIve6SrcRqwdj93gbezFLq2CQPOd7eQHFd27x6UVGLpudu6mRed0AUyXnXgDLWYlTiioqgeWoFvYGlT9kB5672BmzMafJCILZHDzXakwcpPysWeQY7sgnqbHI0jUxYB5JIkwJ52nFuhyJMeaKRmA8TYkrODOpG/3WUnUqIIg6fX5WKLb6aWus2VJ26jzjGP/AKGvEJBUmoaUATp7UafG0ErcUASCCoRyIki1kse4er8S3A7c93ZO3eWgt51ZR3VAnXloLRy1wJxw9P2l3JSdAC+owPKPOx4ZABYoTqeSTVouo1YdfHaRl7oHdKDPtb2w51ALXBy/4Qfr8rSkx6PmM571bdiSR/3qzt/u2MI9HTFpzBd73agEiYU4Z/5dbH5je6bwk34VEzailSVJUBE94JEabfA2sXT0eK7wxQmloKaaJdIX4FIsoIyjvF+SkOZz7EezrZpN+jbf5JLt/wBCmdYDSjH3eFrGXc0uhu+ipHCpamqZtslKiBKUgE2E+NtQ8OvshyUrw20gsoRvPC+NA+Wl01QluZJ7GZBJ0Bgg/HnZh8UqSrpFYddvCidYc9f0S6lQJAep+o21NrbNuZ9UtLI65yLRvx4wbQ4korsvCreqGlXUl19sIIgqCm1QqZ5otsVGImam8OW/NZtHgogqec11/JQjxTSllukWE54zyIJIJEx8rRmqr1KUojUTuN/rpayWM+H12XwinFXU1KAnvHIoCdDpt4mzTRwOwh3HXL4qWiqVBLtSlJG4mMs8jbNjyMbeQgfFaM0T26qGXFJUkOAmRrrqJkQPnbhFU6lASTAGsxr9a/faaKng1hpC2Vs16nGc/wBuV1CjKYjSE77WOXXwVwzedR6nR0rr6z3ge3cQInXUgAifPU20GUjpGcwfd7qmJGl2QOF1BhqH+8oqSTOojQ+GttCsfAyOLGXWCTEDz+FrGH0baNUj90tIBgDNXqPvibDNejhdqEZTc1IszrNasxrvvaAhZsXBG5L+6rNV1BF7YWdUoS1fK069HKGpGo5eza99BVXo+yl2mZpgChBSVqVqCkGYjx+VqwcQ8EXHhu/8J01NQtMlGJ6Zt9KVkgy282d/88e+0+XbUcQahNGu7aG5VUCmkBKn33AsN5fayp0J0GlqEjMshAKutjIYLpXqHsXu1QWn1FKAZSSXTpPgPO1Q/SxqXbi9MvghfblGgP1iKCldcElOVNcrNH/HrPh1ta2sPExC226ReHWm0pK1HsHzpOuhGp8J1tD/AB1wsm+8XYQxPje7blr7yu13/sWsZZebNOoOoUSUKO85TrPOwJGBjCS71RYIi51hqp4qHeyqX28ogq2IBnnz87GrtYZcmqqciUBcA5RKj7hZov3BxScKFoxfciVnVwG7wok6aiVDSxyloeJDbCqWtxxdy0rbLbaWLqQFtuK2WIcgwdwdNTptaLqhzTown0/1RTCLfeCoP6XNGml9JjE9UihdSh9NO52ikgZgqnRIE8iRaBeMVPUm+LhqKNtRaNzMud5UQpDjqJzdSBAj+noBxt4fsKxBUV2IqGivW9OxaK65dA2lxYDfdBGu0Rv0tXddxUldcd2uP3dSVCewcYX6w2Cs5XnDAKhMSrT9bc8ZbzvB7rTbRnI0g7hQnfd4VF1YbrLzoint2mCpmNUhZAAPjEzpv8rG+DC744o8H8bcLMQVLdRe9Is3vdKlJC1Ek5ikD/7ImCdD9udtLSbxjwy03wrvJLNDT0yhTsPF5psFTbYWhSjoJHdkeGvQ2gPg5fN6YK4uYerWmn00TrpoKpCGz9qh7u6gakJUUqAHNIG5E0sZoHVmEyPp/wBrGRIz+8z2h62I+a4f9ImGzVmEvNP+2htLH/fjOYeoBb8CoCqTQ11U85T3WuEjMVqWQNNzB8jp4WP3A0hyhWhTBKkAkgAmCDEkHxk/G07ekPwSxLhLEt+4muHB9RUYfqHzUIqKUJUhoO99eZKdW0oXpmVAiNelfsPViQ8EuzBUqQnUiSCddydB8Oe1uhwnFKXGKVlXSvDg4A6EGxIvY22I6jdFwXGqPH6JldRPDmuAJsQS0kA5TbYi+oOqcdNQ+su9o8AhtEgI7wGmuoOk6+Hxi1i+JmVXo4cPFSCkPuEa5QO67HOYjTQ6+Vq9MVjhWgdqlskArAVCtSIMjbXYQJhQk7WsDxHS0fRs4ctnKlJqVhI8YdiI8AdtOW1sriP+u4b/AI38ORc9xXriGFf5j+FIov4bXDdN74urn78qkMMMMNraUl/IrOVqBHMnRMayII91orgxhhGjSKKmPrbBCQvMSG1I1J1JkySN94IE2pLf95Lu+86EJvCrp0vNrJcp0hx0JMCAFkaEpnePa3JJsbaxJWrWhJxPiB0N5XlfaNMZO9H8BOmv8yY11109Iw3iA4VFy2Mue67F1MJjdxXoFcOOrqWjsWG0ZmSpCWUQXYzwhA57SBv52i/jhgGj/tJUYqxXju57ouh9KfVRXocWUwBnSlCYCjJJEEGCOcWhm6r+p8RUWJLtu66buu+vosPu3pddZS1t5rfVUNOtmE9rVPNKORS9ShOo0IIEvG78P396Rfo5LuGuoqitxjgyrKKF15xKF1bak9z7VUI1QYOupZTJEzbh+IOK2UeNQY+5lmW5M1zYFrjeN57ZH3BPZxXGcUUreGMTpeIc2WJ36mY9A1xvG830Aa/Qk9HqEOJmHeHrzN4Xthi/ryra6mUghTlEmmp3EDUqSgqUuImCQJ001sj8C+In+i7ivhjG5Sot3VebDr6wVT6tJQ+AnY5m1q3E6DxlFxdw24g8Pnl3fiu4q27HSkKLa9UqT/MFJlChuJBPMdbNZpYSdYtsS1kOIgTw2yu6tNwfgV2lNNFNG2aF4c06ggggjyI0K9+mnWalCaqleDjb6Q40tKiQtJGhHgQf0sgXZd1ULyvVQOSnarlK7RRyoAUhDhknQarO5tFHoScSVcSfR2w87V1AersPZrgrFZiTLGXsifNhbM9SDaW63AzuJb2VU1l9VDVAgp/1ZtRHfCRJk7fDnbg+LKdstHZ5tYrt+G5jHUkt6hDXhjrCGHwlp+rcr30aBmkTImP5laR4ibR9xS4k8Ujg6svvCd1OYeoqEBTj4bzOLQpQSO+rUCVA90A6WmC4sG4Tw+AaK62S6Ne0cTnVPmdrbxb+770uWrumupWn6apaLbjK0hSVAjaD4285iFPTuGRl/iuvcHTGxXnVfT194tcdevm9K286105it1alE6+Ovxnxt1hjBV7XRedPfaSGjRKCwBIhKtDz5A2l687gornvJ6npqRDSW3CkFKI2Onw6fpZKrqdQuyqaBCVeruACNZyk6f1t20Abk9kaLJkZldqquekzgv8Asfe120jbqnk1l2F1sq653JHuJ+dq2VeU1CynadO9m+drn+lpTovHDmBr9KStTtK6y4oTy7MgE8tVG1NK5spdIiNIPmNPwt12DPzU4uuQxlgZUGyc3CC8v3TxSwpX5sobvelSogTKVOBKhHkTb1pwDUepKWUggqAzRtuP1+FvG+7a12gvGlrGllK6Z5DqCORSoEfO3r3hivbdaYfpyrK43nAncEAi2bxLHna3zWxwtJYvHaylpN+HsQqNDCYmLAVD9atj1oMOqZABKiDHx2szquqvl67X3Lh7FyuSgrabdGZCyJISRPMDyn4WR7xvvHFBeLiqWoaqqP8A29XUFlpSGwtKnuyTnhaOx9kkf7UE+zFuF8KL2C7nmWCd9VeQTJBEgQTG499iP70yLCkHUKzQDz8f1tEP9peIdVTuI7ZposOqbqKhx6mDdO2HjKnCJKHAjJIIiFKOUEAFIexTxDW329Ne12diGm1OFyopSkKVlKgghWwyuATrC5kxpbbSWCCZwpmvGsyvuAZYDigPAAnr9a2Y/FGmF+8PsT3ItIV67dFUykE/xqZVHzI+otrD95X1U0S6i93i8tb6jTvEtkuMEJKVfZHJ/ERI5AExYd+pBQ406UlKgcytzt56ixoWct4I6FAnfzIy3uF5OXumKoOf962hcxEmNfPWdbLnDm9E4d4h4Xvxz+7pbypKhU6d1LozfcbEsXXau6b0eu10ELon36VUjctuqFkrO4E07wJGQlIM7EGf+q3qFhJHbuF5RrHNfsV6h1NUEKcSVJ31gieuo5c41teLgvepvfhbhqtzBZFAlgmZnsiWzrz9m3njceIP31h67b4/hraJipOsznQFcucn4/C13fRRvr968KGaYrJN23jU0vuJDo/++24ekBZMWldvWu5kLXKYluAnb42CBSFaD426WFBRHM2DKt9x8rbDVjORW92w9TK5lAkRZtqIg7mZ0NnUsZklJ2IM2bD6OyfUiNZtpUr7jKs+qb7y4GU7gRO0+62qcqFM0FaqCcp8SNDPwtoEp0TpH1+FsZnItGndWY9/eH320GLOduhkue/x52zMVAmdOnO3BBSMusb72TcU3hX3Xhu9rzupppyspKF9+mS6CUKdQhSkpUAQYJA5ix2tubBQJtqlfVXI6HnrbCZ1B28bUmwN6aPE3FeMrhw5XUOG6SmvW8qWjfebo3iptDrqUKUnM7EjNz6a6WDxn6ZnFPDuJ72uW66W43aOlq3WqV6pul9h15lKyErUguSkkDY9DttbSGFz5suipfaEOXMrvIUQQNfut3mJ1BG1ou9HribenFbhrS4rvpukbvD1qopagUyClsKQvuwCTHcUmdTrPlaTAvXNO1qkjDE4sduFZY8SNDm7FDEAggnT8LGcNkqpmkkGWiW4jbKY/CxMKGwnmDNh8Od2rq2JByuhwf7yR+INh9U5TrAbj2le5X62y2Zhz18YtlnUV4WXirtMRYcEqyuX1SAp8M8+/wDobWrw/wAM8aYow1e2L7nw8p65ribLlfWLfabbZGXNErUM6o1yoBVqkRqBaqgK3cYYYTII/fNOTm2IBVt+lrCi+LwNEbt9eqU0K3A8umS8QytYEJWUbFXKSJ8d7PSA5TZCqiLi6MN1bSVjtc5EwMqoIP1FjFRW3YWx2LOhEyskyY3+P42SCoHXMowYPIwdLDMpQ6U5qgIhRAlOkQNeh/S10hVGnohQ62cyVKGglKY5n6++2krKiZgGIT193j9dbGaihuxulW8xeqXHApICQNVAgmQfD8T5WIFEZQCFdoNgQTvpZB10ntyp7UHC/GN74CquJV2XL6zcVA6tqtqGXm1KagCc7WbOlPeBzRABOwszHXlqBKzKSdiZB5SPrlYzR1TjdMqk9efYpnyO2S2uUqCTIzJBhUROu1k5TkLXmQCIJOsTpz6WZtxe6WhXeYyVEkGZMHePP63sEhwFebKCUzJG5PI/XjbFqzZp1KiMvd0P4RNtIy5RHeEH6npaR1SsRqp2ws6L1uWlvQd7tWkpXKZOYaK/PU2W667alNzVctrWQwoFJ2Ij68rMngdeCahVbcbxTnSBUs5jqobLA/5T8bTsu4H3rrfcEhRYUoTACiAem/6W4fFZ/Cvcwrv8Ji8TE14Xm5xjYfT6TtJ6z3lpvC6iUxOX+6Ovy93lNvYSkA7JI5RbyW9I+kTT+le2IVCq251KKiDBIamIGomdPE+/1rpRmbA30+ViQvzwRuHZZ1SzJUSDzQigSOcROnKxd1IMgjaxowRBFi7oB5R77WY1UkRB5O55/OxZYIgkT56WOvDTbflFijiASR/W1wGwVN261Qsh68WG1AQpUfKzk/dSVGECOe9krDrDTt9UyXgpYJVEQDOUxZ9IpGNBkJWTERanNUcl9lq0VuXqm4i6ZgTqNd7DC7skqzgQIs4HruS05lKTMSRG1uTQpUdEGwjWm+quAsOoTfF3p705FTzINjXZJ+zSAruJjSNvebKvqKpASm0W44cxXT4zoLruW4Kiro6p0IqKlJcSlkQmdQQlManXeI3sSCukz5Y25ifkqGINZy8ykRCEhQKlJRGupR+E2bnEpLVRcDgSpK1JZdmFAx3fC0c4nwpjhaVKuhFY4pUCO0VG/IZidrGsJXJiWipr3YxDdVQy36mA066k99Xekakgna3VzUQFAaovANvu9VzFFib5K5tMYiBffonLfbGalplluc4B28LAP4Uvi96C637opXljtMq1tqaTkBWe+ouDvJTGqRqc1l2oYTUXbRlYCpQg6iY7lnNhlKE4dYClaBxSQfCbYDclWBG47arfq4sseqYeIeHGIXF56J1agEqADbug5g5ZB++2sFYZxNcN7lV6XY4ikUyEioK0nv5xAMGbSolLQBWELV4wfwtxeqJu1PZApKXW1Rt/EJ3t0bMSfHTeGAGVc23Com1AnBN010vLK1BdQFZdR3//AMK3DLwNbmNU2G0CSM40+diT9FUmsdRT3dnQJynORI31m3TFGtuoKnbtQkCO97X6WoFxK1QFAnGt7tcUshTaQKTFF1vJ0GmapbRv/wDbPnad8MOPpuG7n0NJIFMlBKiNx/Tl0tCHGGnNTiy8qRAVnbrbpqYjQRW0pnx/itNeF0l246OnWFQy46lesQUOKTEeY+VsunqxLUuivqP3LTqqfJTtf8EqVb7rjQL5aZAOU/abn4Wif0gW2DdWGK5sKc7G8iC4k6JJA0nxg/C0uXilostpU+4k6qhJ/IWjTjq32GAKaqKFuCmvNleZUyJkD3WfFr+EeR0Q8N/rDVIFWg5m31L/AL1IIMTyG+lu2FIMBTilK0AhBj7rAU1aVUFG+h0ntaZpYT2YVEpGxscarKlcEIVHUJI1tajN2AqtILOIUdcZbozXq04n2X6ZCVBZnWSNfnpatWD8Piuu9VAXGqcM3lXNKWZKUgOEyQI1hXvJNrk43uxN5ppX10qzkRl2kyCDraAcH4Acpr9xFc9cxUHJertQ0tKZ7i0oUNjGsRrzt55iVcKevmaei7KhjEtPEma5hWrvS56qhVSMOtKo3AppzKlLqMhBQoL01GkHrFqS+jzW3XdXFS63r+eQxSIDrNIXf7tDi0ENmSIGwSCf4j1NvTqqwFTrCm1sVSyUzogTHiZ3E28qcW4RrMM8Qb8wOzS1Tj1BetRd7bAaUpxag6UoCUgakyIAEmRvpNvDpIsepKmgLi3O0tuNxcEXXO8bYKMTw6Shc4tErXMuNxmFrq07V78SKTFFZevEekui5sE3NdlW3V1Rq0uN3hKkZXFNk9yEpVoQPbUnWRbzdp10i7xeVSM9nTqdUprNMoQVd0FQ8Dr5WspjXgD6T984bSxefDfHtVd9K12zdNU0lSQmB/AysE6ToAmekxasbNO6xUO09UHWHWllC21oyKSoaEEcjMgiP0scKcNDh8yvL2kvDRZgLWgNvrq5xLjmNzew0A0C8x4M4Kk4T50kpbeQNFmNLW2Zm1s5ziXOLiSb2GgAACfF3rShpPdkOgrCSrNKVaAEbkwdCRt7on/iXWN03o0cOn6pXc9ZWhzQwQe1GwM/jpaul2VFRWNopWM1Q48pKENJBWpR0gJA1M6ba6jnra0PEzB+JmvRrwVSVlyVyHrpWXrxb7JXaU7ZC9VjdO4mRAkzG4nxPPFFW4aHuAPO6n/03j8yB8SFDi+ohhxHCWyOAPiOpA/6Ug/MgfEgdVW+nuy6MT1jL9TT1FWltsNDKsIKiJJnpJVJ0BJkzuC+6Th1gUFqpXdwQgFK8i3JJTIOumx169JEA2bWEaWlbUCloN5syy0CmAkQdQZ7u2h6DmYtIzVEqvp3EtVDjK1IISlImVwRHd0IknX8hPSvdqvRY2i109KzE2JmmBclTibEVRRJRl9UevqrdpMsD/YrcKCNAfZjwEgGYuD94uVeDqGludym9bp651N7JX7QBSvKsJ0JE9lGolIPlaDlULFRmcXSrcUrvlBqCCJ70HWZ9kERPdPvd+Bru/s/e9JexSKZfat6NLVl7IKGYEneQJ3GhG5meL4rwI47h7oIzlc32hpcEgEWI+enY2PRcn+kTho8V4E+kY/K5hztuLgkNcLEXG99OzgDrayK+lVdTdfg3DuELzvJFdfZvOoqA9k+0TQguEBW/IsJMnvZFHUg2pvjvB68OVIdQJbWYIy5QOhGuoJCtfC1wOPdW5dHESuauq71V163hSsONgzlYagJKieUlKtdtOcRaCMScLr7c9YvG9r0Ljr6At1pKRlTpBg+/TTaNtBa3wPA2mwSAtdmzjOdLAF/tEAdACbKj+jvD20nDNK9j83MHMOlgDIc5aANAGk2+V+tlLf7MniYq4uJV+8Mqx//AFXFFB61SIUY/wBcppVA/wAzKnZP+BNvStqqNMrKkkAjMNIE8/wt4lcA8WK4cceMHYlXUBtu7b9p0VC507BTgbd16FtS7e1dc092iQy2teUkKgTERaxxRFnonn4fmvVOG5A2saD5/klBV4mO6ZJM78+tm9iG9CGlJUdBqDO/X8bCNOlbpppGYH2QZVPl+dm9jNL9FkLiFp7VBEqTrv8AqLeZQRBzgF6US1g0UW4uaAvN11JILnfI5AHn8LNB15jO4ytxGVxJGpj3ff7z42bfpXXdfV8YWu1y42K2pearULW3StrWSktKkkJ6QPAT42rAvB2NH1hK8M32pQSpIUaJ0g6+IO/5aW7qgo2yQhznWXIYjWuhmLGsupj4/MtXjwRw7Wp75u68l05KSTCCFjXXUd0DXmLUtvdC26lQcJKgpSSTuYMWuFV3XeA9GC87svCkdp6m6L2zhtxKkrALid0nb2zoBztUfETQTWPEJgZgr2SIkbEe4fE228H9jMwdCsDF7vc2Q9QEjpICpJJ2i3qdwhvIXjw8wpX9rmVUXPRlRJ3PYpn5280MKYExZjNm8XML3MuvF1sGoq8i0JLbcHvQognY7Tytfz0Yr8RW8EcKuLUpSmWXWFFX8PZvLT/4UpEbxZseaHQgjoVY4deWTOB6hT7dtcyglDlQGApMoeyTlIB38IJ+VmffeA8K1ZXTfvKpqWXHHlqaSlKWkl1banEgFJJBS2luNAEEgDaDqrwV2ikDLuobgg+fz1smv1rjiSnPImOk7mNdh+nhbjxHY3C7YzaWXNz3HQ4eU8pmqfe7ZtLSe3KSEpDji4kDXvPLOs6EDkZRjhG5qZxl2mqaxJp2SygZ0mAphlkkgpIMoZb3kSPdY+5VIMJKtoGaQT8fhZLq8RXRSFZqrwpwGQkLBcAgqMCR0MwPE2KGO6ILpWhKVGpq6btpbtplOLapGEsoLkTkEgSEgDTTYRYN6sWoEhIHWR3iZjrNmdiTiXdNyD1dCautqWniwptlEkLKAqRMaQSbJVwcRFXxfrly1F3uUyStwNOFeYKU2e8CCNCBr77SEDrZrIDqlt8t1Tnj5daLs4m4mYbBy/vV2oB5ZX0h0D/mPxtHoVNGEa9xyR4Zh/8Agi04elhdPqfEKprQjWvoaSrKhO6StonXl3E/EWg1rMW3UjUABR18Y/G3fUT89Ox3kF55XNyVLx5q73BG91Xlwow4+V5lN0xp1Cf+6cW2B8Ei14fQrvjtrrxVci16sVNNVpE/zoUkkf8As028+fRdq/XuGBYLhBoLxeaAmIBShe/Id82ux6HNf6pxDvW7SdK+6VrA5EtuoIPuClW5eZoirHDzXUxO5lE0+QVxVknSPDbnYJRAJAOniLdqOgCj4bW5IkSdwINr42Wc5BnTbYWQr4ayv54AChNl0iP4fLSybfDRcYDkwUGbWqZ2V6rTtzNSIFajbe2gSh9YP8WVX4fhbYJGsRp8rcunK8lUq7yTr8P1trMKyHBdlZ8/u91tOIQ62ppYCkOJKVA8weVgySZIJ+FhEnNoTrt1sdpsbqB1Xmpge63cK43xrd120bNRiS4aOtbuVt5oORUsVKUuKbQoQp0U6X1J3IIkahNli/KV/jpcN68QP3TRXNfVFXZS7UXw4ae8EKSt11ts1byghbOqwhBCciinKCEgufivX4PwX6QWJKTE1wU9O16wi8qO8qVl9VX2z3ZuzIeSkQVOwUgapSJmTYniLjFwyxp2DmK1XzeRpmillupoQ8G1FRKiCt06K7kkJkkEknS3Xte52WQDoFzrmNbdpKlf0DL6D2EcTYdUcwo69mrbE6ZXW8vwlq1pknWCTam/on45uSo4vXth7D93UtJddfdRdpymjSw8442WyQ5lJBjM5Hha4qVCO6YHKP62wsSGWoJtutWiN4QB0QwPMCLD3OtSb8Lc916nBHmlUf8AVYqDuCJ03I6W7p1lu9aB7NCe0U0rwzJMfNItRVtPArIJAbVp0A/O2W6IP8w/4bZaVlBeFtHP9r8LNpVOa821STMJykyffafA2vswEd7MZKhHnaHuGtzU2I+MeCLhqyQzV3oc/ZqgkJbWTB35a8vPa1jMa4MvDBVcikXUIq6V8qWy6o5VECNFCd9h46eVg0tRG13KJ1KnUUsj2c0DQJrFKkmcmknlqDykfXzt0hKlEoKSQNJHMa8/qLdI7VMEjca/4vnPW25e5BJB7s66jX8PG2ks0tIXa6SsaU02psBT7YWgJcBgHYmDoecefW2/3fVqcQySMyoIKRO+3LU6+duAt6MwUQBqQOXTy/Hxt0p9+QrtDAA1Jkx9flaCXRdOULTTqgpZW2jWUJJI/CfysrUGFqd+nqa1u/LsijYRU+rPOONuvpUSChvukFaeYkaERO1koPPZMpd1BgidI6fW827zOZc/aDvaaDr5+E/fZrlSYBui/qrq3DlaUmFT7Omtu2rvfWqexX3leUnwnz/rYTO6D/eQB3TrsOp+fxFjVO48Fd1xUkaxpETr0+jaLn2CK1gJTq4X0F9UGMbqvGmu951KKhKXQkgZm1DKr4gn77X1peHTy6QhVOCVpgQmI91qa8ILsfqr4ZcfvlynDbiVKyjRQBBg/l9D0ppOzVTNKQBBQCPhbkcUoRitTkLrWC6Slq5cNpxy/eK8aPTTwo9h30vW1Ip1MIcduVwKnNnPdEkctlf8NvTWnT9mE9ByHK1Gf2ldTRo9KDCrTYAeFBdmeAASfWnSPPQAfDpa9VOmGgY3s8EQgiEP4dPRM+QzHmO3Oq7I00M6jnYB0K5jc2MkAJ0sXd0VIM+drcaBJsiTqZHLpYqoDkZnlY48CQYPwsWVpykdIi1obKk7dLWC6JdRf9MAkkJzKPlFpPZuSmYcD7aPtB7JKiQPjZgcOnUt3+G51caWn7j+FpSsPw0cri526mZXMGVpSebtKlZlLBJ5xbf7sT/N8rH7ashQQ9lHnP7pOVdQzZgsizbv2jcp7wSgHRYmfD6FntZn4ySt6rQwhwtkpT307jWxqeljhfmamdK54s5EUMPmSF/8xsDfLDCrsU1UvoQFd0KUYEnQffYZlKRqF5v83L5WQOIjyWsLvPqJytutK0A3zDrYlZd0DwD0RKUgStPmjdZQ09HRs0rriB2SEpUZ/wANhbpve6Ke620KveiZCXCSFupGnvsycUYuSjDf7wIkuNsn2huUjb42g/8Ata7eFWqk7NSW1lUEKnbl91svDI3h2clXK+YFuRW6oL7uW83zT0F7UVU6gZ1IbdSogTEmBpqRZEv/AIlcP6RmooH8X3M3V0rqEvMqq0JUhSXAFAgxqNZG+lqzcJuKDtzYuvZb9E02zT0rjK3c5lR7RtQifAa/hzq3xtxUmv4m4upkoczC+bxaWAEnMPWFxGunO1iTEHNqHQW0AvdV2Ut4hIvSVXFXh+lrtjj3DqUOZigmtbToOepslOcd+ELZKFcUsNKV0brULmJmMvlbyHvSg7aoJQREgnQx8OZiYn87K2F1u0TrbSWWyl0nKXANNI7pP1rZpMSe0XACdlM1xsVd288c3Xjf0h8SC58QIvO4/wCzrVVTqaI7EOsOsLVGkk906/mLTvcuOsPXbelfhyovkN1jV93gyGyy4oQp9a0gkJyjRQ3NqEcHa5268WXqUENn+z95pMd4KHq6laEHXUD4/G+PDSuFSziq8W1gqexC66DodFsskffbJoXSOry9thcFadU1vhAHa2Wrx48cOqSodbVfF7PFtICBTXHVuSfAhuOdo19IHjRhS9uF1Vdd33fipdY7UUz6FPXHUU7YTnT7S1pCRII3PO0+BbQcDvaLDmqvaB1syvSEu5F48DcTLb7RTrbTb4UqVDMh9CtZMAaeQt0c0b5InNcdLdljxSCOQOaiOHeIt4MYdutit4XY6VUUtCyy8TdraE50oAJlTg5iy9SY+vSpQlTPC7FEFYB7RVG3kB0ky/pZ0sKdqKSnUFEoUyhXdJ1lIOuhsKGCtktFgqSSJCl7+6LRax4AAck6RpNyE78EIF53A3W3ldDtG84pYVT1Km1rbAMQShSkmQJ0POxS57suR3Gd+/6o0XXAw97MT9mEz4+zYvS38q7WPVpykoCgmR3dI/C0f4axg7RcTK9TrxKKi72gEk6ZkqXr12Atw+L4pTRVrYZGg5TdxIGv83WtSUU74nyMNgRp+f7lNyriukyTRt66nS1aqzgTgW4PSdvzisi4mXL1r6eiNKpZlNO4WnEOvNpI0WsNpSSNYBiMypmtWO0RoR03tHfFHGLdCmnxNSt5altCqYvZk5UgkKTmB9o6LCQOavcZx41h0srY6ZuUnqBa/kj0VDUskvMLj+dUTxfftdiHDF+0+DQam/KDtqdllpxtKk1EfZpJdKUTqnchOu9qwek36B98cWuEtXxFoOFibv4s0VC1VVLlJWU6Bey0QXkOttrU2twpzZVCFFQSCojS1k7tvpjEV2sM3JU1aq9Thq2Sh1sIqWVwVsIUonKoKlaZBG+onSWbqxHXLuSlReNC4KzskhxDrqNVAakEEzy5W6KF8UV3E/z2TYg2QgNA8rH814F+jRW3LdnG/D5xUGmGm33WkGsCU9nUFtSWZB1CgsgDoog6Wve21iSiZYqcSXrQm7mF167yLqAEKpipZaBJMQE5OQgAgzahHpQ35S3v6SHEi9ab1ZDVTievXlpkKS1mDxkwomVFQUTyzFREDQTBjWsva8/Ro4ahVdUumodX2wW6o9sGw4UpVJkgZRE7AdBbguOuHvtavo52yZRKRGbtvYZXuzDXQ2J+Yaei+c/0mcHOxnGaF4lDec7km7c2WzZHZmm+hsT8w030TCoKK7BXVK2HUt3eXlqZbSv2UyoiY09nyPdJ6w76aqoqWlXVKUlDQQlkpJgulQBGg31JjSN/ez7mp1LyjtF5MwS6lRSEpRsN+U6g9DoNdT6a5V44gTdbCVrYokdstG/fBImY2Gux5DqZ9KaMrQ1e5RsEbQwdE9aI1CENPNkNqaQ0nIAEhsgd4TGwJPgSNNIsbxBeKG7vWq8L2NDRJ71dXBcKSlIHdbk6So6aDSDrMhstOXs00sXZclVeC3FrQy20zICgVbq/gHwSJJCgSBZoXkau/L+9Vv4IxLeras9Lh27lqXTUwn26lxIIMH+BJJ6xrZBnREc4KeuOK7/xCjC+KMMXlRXTdd7XU27UXs61mqAhQzNoQORIdOpGmblaD8ZC+7ow7eNBhK6kK7RDhdvGrdTUP1LepUpojQaJPlBjXW094wocRvcH8H1L4vNFdQqNHUUtzMMwdFAJIdIQlCeySJ220INo5dpXFJdvC9MP0tDQKKUupafVVuyBqXEspU2mYiSAJG825bhAmHC20/WJ8jD/AO17gPpZcP8Ao5GTAxRO3gkli/8AhI4D/tsqdds72/rGcl3Nnzc80zNvoF9GrE124qwzh7El8sMPJvzDlFeJ7RIWA6tlpaonbVareCuOruo7sxTX0920L1JRF0rp23TJCD0I3EzG/nb2B9FS/wB5r0buHd6NuLlq4GWM3gjM3Hl3fkLb+LE+Ec9guQLruqCPPOIibXNlcaiw3w3u29Xb7p7rp01TpJJUoqQknfKgnKn3AWiz0oau5V4fudy7mKdDgq3EKU22AQktnSR5CzdXjt8gw8d9p+voWY/E3Ejt8XM0047IZqQuSZGxH17utvOIq6ea0LmNAPYLt4sHMEgndISR3Kji8Hg7TtrcO6CDIg7nlZCfQUqJUgKy6DxE/gbKIdLtKUEKlt1cSNpgifeT9aWIPtlS0rVqnaE+PT6622IrgWR5TfVMDG1EmqwTxFuoJEinbrgIOpCM0/8AJahOI21JcUSVd5KViQOuutr+Ytvu4rlrb9u6/wCv9WF+4fdZZSW1rU6tOZBSAkHYKH0YtQa/0ZiQAcwRqdddR7uvyt0GDk3N1zGNWOWyk70SqC8L74iVWGbvr0Ubl50BRncHcVDrZKVeEE/ftae/RGqHBw8vK5FmFXTftTSwQAcsIV95Onnam2A8XKwTiAXyKP1pBYcYW12mQkLESFQYIMKB6i1pPQwxB+9V45SltxpLlezXoazZsva9rInn7CRMaxa5isV4HO+Cq4PLlqGt+KsrelcaG737wCO0DDanCkEyYTmI56frvzilziXfTrbriruTSPNt9oQTOziZ33BbWDO8zaVr0pzU3S42kCH2VIjwII08D52ZuCLhwfflKlN9Jb7d66aohQe7M9s3Tktk7Tq2Rl5kczbkC5sTS9wvZdFiVWaSPmdAoora3FF9Xa9XXrflYGGFoqMiRkAQt7s1IkD+ECf62T78p10lReqW3nFumoqaZ1a1mVJQhLrc9D3enys+MaMXXQvX3c+H3g9dtTTuOtDPIQ4lIWqFcxpP1NmffDhqbwcUCJrHaF9InRXbMFtUAfDbla/CQ9oKpRz81gf3RJvEpdvBy/qgFQbqaKqc01IDZQsCNJk/1s47mrGV48pw00ptTlauqR2ndIS6wk5TGsk/RmzZXdTd23PSXmy4tZraRVTlIByKbW0Snx1CoNnA9eAvy8nsQ3PT1ZQlNNUJBaIKlIWUKGkzoQfLlys8g00RmE31TE9MSjCryuOtbby+sUVVTqUAYJacS4P/ABq+Nqwse0pMTKFWt76WVPT1eGrirQsLdZr1skZu9kdaVOnPVCdfC1RKUoTVILo7uaD5bW6TCXXpWhc5jDctUSrJ+iFeJVd+J7pKiSh2mqG0Sf4gtKj/AMqbXU9Gq9jdvGa4gpWVFUl+kcBIk5mVZR8Up+NvP70VL1XQ44vK7xEVt2r0I3UhaCNfLNa33C/FKqTGWFsRJp36VDV6UtQ40+32bjaO2SFJUJMaTOvvi2NijCyrzfBbWGPD6MN+K9LTI0032m3AWNo91u1pEkWDMmddD8rHbsqzgtEwrSdT4WAqG+1aWjqLCn7zBm3KtdI3HWxozYoThdNZaci4+M2LPrhbKiR/eRv1B/SyjeLXZ1BnXMZ3iLJlfKacrE9whWx0gz+FtmM3AKx5RlKGCoiTOvO2ZpP5GwecHY+G1thYIkA76mxwgpoYs4OcMMc3sb8xbg2hvK8EtpZ7d7NmKE7DQgaT9aWT2/R64INGBwzuKd+/T5vvJs/yZ5nrtzt1mBMgg9BYwmkAsHGygY2E3ITXw7wr4b4SvBu98NYIue7K1pKkofp6ZKHEpIhQBGuo0s7s4kE7k2BTMkA7bW6CuRmPwsnFz9XFIMDdkNmhUEak7j61t0+vsaY1MH/V1IeB8EqBPyBsEk69eQBA/Kw7DQqm3aYqMOIUmBzkRZdEk9hCgCEjXXcWyyLQ31Smipy64gr7JGYkwZgTItlldRXj96PlOa30jMCIIUUpqKxxJOnsUjx08dLWn9IJvs1XXKlJKlu6pMTt0tVb0dL3uu4fSJwle19PpZp2heGZQ1ICqVaQqOkqkm1qPSHvWkq6u5WqBXrFM6y88KpsgtakCJB0MCfIjytiMa7x7DbSy2Q5ow94vrf/AEUPpC/ZQIIMiBvPX4TbAkZiFEqIgHT6+/mLcn2gRqmCddv00tpbi0qbQlsZTBUvMNIG2+2sfG3S5lzbgTsu0gqJJKiAREzAjX7/AK3l2YE4YY34jVL1Jg65HK92mCVOpS4hsoSZ1OdQnY7dbNRrMVJJJnLJJHMA6cun1rabvRt4wXLwqxBVv39Q1D9HX0ym3HmFJztqHs9xUBQMnnp4zarVzSRx5ohc/wA3VeXOwXaE38V+jxxUwTci8QYkwuqiu9jV1xb7RjXbKlZOp+4b2jpTSm5JM6wdYMGfr+lrVcffSdwhxGwa5hu4aGuQ8taVhbyk5dCJkAnqfhaqzzrmVAbLamypWckwTtET7+fusGlqHygl4t2UoHOkLtNOi5SFkglSeUEjc/h9eNjdIUthJMAzIEjX8T52IHvJJkkRp9c9PvNt9u6xXU7Xao7BSV9oT7QgCPASJ38LWHG6vRgjdSPh2/F3U2FNqGZSTEbyIA+voej2E8Ts1OGrrqFrkuUbKyZ6oBt5QNYlpkP9mVkIKAoKC9zMbe75i19MKX5e6LhuarReTIu5FzNIcpewlxb2VGVwOZtAAFDLGs725DH5pKLLLEbHVdFh1MMQaYz0VF/2it9euemRdqUOEIapLmRy3DqlDfzOvU+63o9SkltMjcW8o/TfvFd4elqw+pKVDPdCM6lRslBj5/G3q5Sf3SDMyLFpZDNTxyO3IuhSx8mV0XY2Q2w06WLuA6zNjPKEjaxdyZ9m12NAk2RR3TcTrr42Jr31A8NfxsddAiSqDYksiTvta23ZU3JawY/2GJKI5oCllOvOUkWl31jxtC2HXezvyi1g9smdfr6NpUD6dy58LU6ioMD7DqiRx8wXSqalI0zC2jUp/mFod4nUnGKuvZj/AEf39d9FQ5fte1AC55zKTPh91mexhL0lX1g1PEu7kCQcqEwSJGkhHSfjbKdjMjXEBqP4TS6siakD+L52J1ztPLbq221KUsJkpBMedoKoMA8eDVqevDimwtiD9knMN9tcvKyfeHDDi00W3a3jJUOozBHZgLAzK0zaEbE/W4E/GJ3N+5ZO2lubWVg1P0gRoy2P90Wr1je+aipvG9KF2sdLSahwJaKjlACjGm1j2F+HmNrmv1i9724nVtcw0ClVMEKyq1GveUd4gyDudLMfGi6ypvq+RR0NQ4rt1wcioJncEcom06SvfUFwl007qw2mMVnW6pIxnju67swwzd14lxoJaaClmMqdBqddBpvaLK+9vUnjWU6CoJBUFCIUkjkrpHh03ss48wkvE1z09LUXQ9WtVTSaaqQttRC57pB023mbMS/8H4woj+4LpTQM0yG2m6UKqSy1TtIbCMhE5gcxABnWOU6WqPEY2ksJ1ViXDTNZwSKzxGar7/vmlupxMM0j4qwClRzoRmhQBOUxrB10Fon4v19Cni9ialFZT+s1F6Vz6GSsZ+zLyipUdB9bGzqcwLiZoXrWUFwMMVPYLVXv095upDi+z72gTzSIhXh01K474UY0qbzZrHMJJqa5uqTUP1DdfVPKqEHVSc6mgIUSkkzukWA+ojfOXE3KuNoSyINJsFGF7XvR3ZTN1FRVJZb7RtBI1zKUYAEa7/Q5L9x4Yvh1ty90qWilSe1JUorOVIJVlSmSdEn4nSzpoOEuOb4cbu67OET9RUIcDq3TUK9hIJM5kgJAMRp087SbhXgff2EaO78YXhNI9RoccduwPLgJzIbAdJjPqvNAA/OE04d+rj1d2Vd1PHG3mPcLJucNsPV63afEbACrvvG66osqcQUO9m7TOAEpOomQIIm11eA63XMH3jVOJzCpqqapOU69+kY8PC1Xroqr0rr2ZUpxHq61LSoBImCkwNyY+utrM+jLUn+xLgCoBorsdTmjUmlSNI/y/r0NhBcZwXi26qVRDoCAVKXdQkdo2RIkS4evQCzd4wVLKOCuMiypIbZuqoec7WQMoTJMnbQH4WcdG2/VvJSKd3fWTt8rFcc3GL64a4ou2qp1Npq7sqUhSRJRCDBEjcHWNrdY4+yfgsZv3hdE8NvuXlha4qykKlJqrrpXc6VaGWwdx52Uf3dWuTKla9STB+FkPhe/SpwFhvPeSVqZuqnbWVKAKilATJ1ETFngi87tI0r6cc9Xk/dNqrZowBdwRnRuvoE0uIl63xc92sKw7dbN43qadIbonKoMBxCXEJWvOQYypUVbbwOdopr8SopsXtOA95DRQpQMaaQPPX52lXHpuW8E0F80tTS1j9EX6ZtxpxLhQFhJWkEHSShM+VoDxksIvQXg0AFqbV3lK1Cu4BAEmNDJ8LeWY9Gx2LPG9/8AQLtsNuKFh7f6p3X5xOFz3O9ejjNQ+Gl99NOM69xsPeNN9LMy8uM9fetK/QU2EMRr7oIdFLKAdwcwmCIB948rIVRej5lSX8iErSciUkJEjaRrEjSIn4WI196vrYRDiUpUowhBgTpqdNSQBqZPwtnQ4a9sgkYbWVqWWQ/dNgn/AMK28D0+O2X70xEu5ampWagodIpmC43CpCnh3UrG4SUmRpIm1oKa97ovhlirum+KStpllaQ7SuB1BKZBGZMp0IIOuhtQpDmeoadd7sqGYqOkHYGT3t4iJ1tZ30a8rvDanIJWKStqmlJkgDMUr22OitrdxQzGRpjeFjV7cxEt/JeH/GSpF58XsbXiw28E1OI7zebS4kBQCqlw94DY6ienW1ir5LavRh4ZvFEBLrhSJM7u843jrE+FoF4t4XvRHF3GdE1SOqS1iO82pKQCIq3BAO2v42sVfDCbv9G3hu06tLfq7ijqBsC5PI6xrp0PlZcQuHicOA/tv4b15DxYP+Z4Vf8A/I/hSqNXKtu5qA1i1oKhK0hW4TuDrHXz1HhZLuCuF33YuuqWWy/WLcWolBQGxMgaRpAMT1POLI1/Xk9eTzVI04QiW1QYIMqiZ3/lmdeo684gqmaTD4clxJbZyyloBBMTGZPjB5zmk626UbrvCbI1cV14hxhRqvRVdU0tC7LJaoG3O0qGype6gcpEnL0A3I1FnzhrDOOKds3RcJXhijfUFOdj3nXCITqlBKG9D/EpapPeVBtF3CK7qq91paN7Xsw00FJCKGrU2reYhOsakmNYCjytNN1YNuWqWad++Knt2SA4t++lFYVr1WFEzpGXqOoFgNQb3F1NOGsL0i+Dzlz33UvXkLvrRVZ6xztFTKVAncADMdBp5bWZ1aMPUr63hcrVTVr77PYNqKWwU6HNPgf+GOVnpwzwwzT4VxJcdLfl4upvGlK0OmtU4ppcLBU2STl3RtzBmzcduKvoWkM01/1qnXXgS442hSkplXdlKR4GdfCdZ5TAm8mtxGk7Sh/ykY0/mCuK4Qd4XFMVoT0mEg+EsbHfmHfO6rL6SmHMS09XdOJL0UV0b6F0zesdmqSsCCZ1Bjp3LejfoyU7tD6KeCEKJSRcIfkiYBcW4Pkfu5Wj7gnhPB2Ka7EPDzGVyt4hu6/7vQ4+m8ZcMsLJlCiczZHaKIUkggpmbWSYw/dGHcIt4Xw/Qpobtu6g9SpKZBJDTSWyEpBOpgczM7nXW27L+sgMZ7FehRfq6hsg7hRxXkV1VQ1rlZUNqoXVuJSy+pLbxLakZXUjRaRJISdlAHpYnfNUt2mWjZIII0mNfH32DLylGAMk7BR+Xhufl4WKVi23Kdas8gpCoJ1A39x8LcEyOzl6K5+Zqblw3dT3Y1XUtGHVesVa6tzO6pyVuTmjMZSJGiRAHIb23et2UV4oZTeDKXEsvoqWwpROV1BBQoEbwdddLDtL+2dDcSttJgCNjAnnOp+ti9avtAnKlJgyMx0jf687Xm3JVN1gLKEvSSp3heeFqlptxSnDUskIRmJhIMDrqfikWqRiekU1UVAW0tDgeWlaV6GY5zry387eg1ZmYxThesCsrKKxxlZKxlhaCASZjfX32pl6Qt3KuniXiqibAyIvR14FEQrtO9IOunfnT46TbZwyW0nLXOYtBZvN81DBBBOvhayPoQ3h2ON7/u2P7+60vZY3LbqRHn3zat6pClAaakRaafREvI0PGaipASP3hRVNPATMnJ2gH/u7bVc3NTPHksbD3ZKph81fcGaZMqCgqRG/SdemtoCvO5r1prwrqNVXTUyqUpbo1OPJHbBTjiiI3GitvC0+t0z6mShDTkA6lKCZ+o6/laHMUYNqnLxxNV0l3Vq7wYcaqabK2uVgJQ4QgRCtcw067246KxOq7SrYJGhNJquuOjvYrfv9VShF3rWtlps99RZhRBPKNRt99iDd+YeerMPLqrprlmmYQhL7qkpISlzKlS0xqAY8NddbdvYOv1dJSxh68A49RUiioUjmiilxtxJOWBoUkj425vi68Q3ldV2KThq8mahukF3vdrTqSErQtteczEAwoC1toaFmBpboh6O+77vNAprru+iZpqW7X6gJ7POpskLSQmZ/jHTYx4WJ118XlfTzNCxXPMsVblGtAbVCU5m1AgQdO+nrzNlK6rixDdeIENoTStUzr1RSvuPvtpT2CnQvMAVDU5zH1Deu64b4oHkmoXQkNKYU2lVexAyuE5TK94Uo+U2cZbqZzBN/iDdjl4YTvq9LyafTeF3t0FY2HCZIKkpUSOsE/P3V0rEhiueSg+w6rKR4G1s+KLVy3u3fNbX3lS0rYud9tv8A1tolx9IVkEBRMGRHj01tVC9CDWKXoc4SqQZnQT85tu4U68ZCwcVbaQFP3gTWpu/i3dgBUG31P05gjZTa8vnrB91rgodcb+0CsriIVIOoUNZ23Gnjv1tRnB1eLpxxcF55oSzWUriiDyCkz+NrxELzRET7KiR3hPX6+VqWMNtI13kr+CvvE5vmvUG4L1RfVwXZfKCCivo2KpJB3DjYV+NjZ2mNeVmHwGvZN8cGsJVYUCW7tRSkgzqySz/8Oz6KzE766WhGbtCeT7xWirkB+ttEpP8AEPIna2iZ1OnMGLYFCd9POxmoZSXe7RypcQNt/KyHUI7ZlTYGqgRt1s669pCqRUAT1FmusBILZMC2nTO9myzaput0UYc7WnbWQSVoEgagaWGCgDqo2KUuZLZQRPZuLTHTvGPwsOVgGc0nXW11uypIUGRnJ5TrbpS+kA9OZsDm0kyfwtuSkASTAg/CzhJdg6cpmYHwt0lQ6iwQVGoIA87bnumdB4mbEamKMBciJGm4scu9Z7QHUSRysmhRPnY7d8l6UgEeJ5WkoojV3TWmqeLWTJ2isu206WyzrAaIkhRnnAtlmskvFLhypTvFK5s2eEsVp1UBJ7KJ89fn77TleNS4opStQIb9nNrGm3M/XhaBuGuY8S7tfblKRS1kEjX2AIPPr8PjNlY4tasyoGbSBO58ekkm0aUCyhO4hKN0ONrf7B9M9068jvOo3svIprrABU0tYmIJkj61s2LnDi6sIaUtSgD7I2T8dd7OFNLVFObIuSe6BrB0+vhZ53EOUqcAtRhti6RKUsoKoBG+h36cvC2ZrrkqTTJAzaa7bc+mg+Vi3qjpBAGgGpKZEDpbEUNQpSsyCDJ1iZ+eu3Kwc190YtHZGUP0Ce8WBCSZMg7dfqLBi8qRbKg1TtoAMhISIkRtOnMe6zn4e1OCLmvSpe4g4Tqr9u9dKtDLFNVqp1IfPsqlJEjQg+c66CzJprtQ67U1FAxWIQ5UKUlDz4cS2yQMiB3Zkd6STBEaCNQtm/WFtvmk5lhcLqqvRLSFOBtCANDoE6bfj+nWMuKeIHac0vZ1qqZDxU8h5p0ZlqT/AAjQiII1tIF80Ibu9ztkpcSpEZVSqRtMeBPztEONqdqrupVDfrqgLtedNOsaLSgNkpnc6wdOceFrBkBFgoZTa9kiU+L6y8Wmk/6wttvI4txEZ+yzRy6ny2PI29NMH3+p7Al2Hvpi6qaQfaTLSd99YPzt5J4NrKteI7vpO3aaaaSfaCil4JBUmYnQz8ufP05wnV1LuELscD2UOULHaoCZB+yToCAI1/C3LcR2dEwHuV0/DTf1j/kqVelnWio9Jlpwq7qqi6hBE/woB8tB8/K3r7RqAaQDB7vPrbx39KRxI9IVtwKSqKm7RBEpjKjXxgxb2DYcKWkhMeyDvzsams2mjHkqlUb1UnxR5SgASRFiqlBU978rcu1Cikjl1FiqnXEJKUq/T6i1ljtVUe26FdEiJ8gLEXCo+e9unaxaRChJHhvYDtA4JAI6z1tdYbqlILFGrrcyXpSLjUPI28xaQxUHmbRpRryV1MQrZ5J087KmJsTYwuy8U0twYMcvNlSAr1gPpSnNzETI99srEWudI0N7LRocojJPdO+pqO/E8rcMvysDMbRfeWMeMAWBTcMmyCndVYifhmmwNBizjS7VtJf4esU7R9tbi8uU+9UbWxjSTF2YW9VoCaMNt+5S2t5SUEZjtZLvJusqKI1DTSloZdbUopjQBY5WSLovXGz57G+8O0TTS9e3bqhKB0CQTmPwt1fjyxd5Qhwj7ZnQK3+0TatOSx2VyswMLtWo6l/tGwQev32OXHQtOpSVBJK8x8dSbMzEV731cF1PXpdt0v328pxttuiaKUdlJhSiYzHccj7hNo6pOM3HO7qyrN0cDa+82CsCnz1zbKUxoQMwBAJ6k/hbPq2uyBwItfuAjSg5S1u6mekwxRO0VSlSEnJX1EHp9so/jao9KE45RfF4X4WkupvCooFoaJbStLa0qQoaKgzztJF4cW/SHpqGpew9wmpX3al9blQxV3i2PVHSe82O+nMBoZHWzU4JYJxtRXbfFTxPw01dte7e7lTT07FQhxCm3EpKiShS/wCIRBPxsfAKaJ75Y7jmOtl1B2vf6LDxZ9bCxr2E5Rv+5Nq969OGbrq6Knw4pyhrU9nUOOKWvIMuUwUpHLzsr4SwcxxcxLd144vur1a7wx2KfV3nW1rZbQohUZyQSeZGsWm+lwxcN/1dJd18XUy5TuLCJKFDLJ5HL9TaS7lwNhe66tysu+6GGnGmW6dSRm1CQFa9fbNvS4IKOjpiGxjmkH2lxr34hVSjmTExjoo7wLwwuy66SuZwXdbV3orKBCW36gLUD9oVKClFRzGEx77J3Fa7LuXeVw01Vh5ip9bqllwOtkhaVNsmBlmY6AHYWnWhcYYaqkoKG0NVykBLaNAk5YAA87Il6oY7XDdWUtlS1AhRazK2QPPlaphsLaSTnSjM89VaqGvlj5bXEBUbvCmprrxULvoGWmWO3Qrs0piEqAMjnGu/jaReBDFFVXFTGvvSpZZo8P0bpDFatkKKXFtSsIOoATpaL+O113xTceKp27r7RSMsVbA7MsJ7jSktlSEqiQCQZHiOlpR9HhpJuaqccdQVC6Hmc4TsGrwdSB7p+ZtTmaRU3cLXK3qc3pbMNzZS3QJwmt6V3hVls6riuqTI6e1bKrCeCb5oH7voH65xC0uhtpVdVZlEoJGvaQRpsdPibIrjNQGU1a73cKFGPsmyPfAIgWcuEfU3nA2b3eWvvIW04k9+UkCDuedrLXNvYub6Kpy6q98psmfwewxh6qwVd95Kuu7lVTzRRUOPO9mpRQ4oAHQgxqPD7n4q5ruo6YLbu253QhIJQFtlR669mSefjtZkej8LhwNwuqW66r9Ru66qyuW8/Vr0R9upRJUeUr084socdOIibiwFTUWEatNVfOMlpu65jTKzFYdgKeSRyAUIPVSbDEVJHBzZGgm3ldaUUdTVTCNhNibX6BKl53NdqLm7ajuulo1pqUx2ScsoUFSPZTzjl0tBHEpoMvUzkpyBKk9mdz7ZOvIaDp+U2t4UrMA8Irtw7W3s/eFZd1IyKmqfdLinHe0GdKZ2QCohI5JAtBPEx9TlGl41Ocl8FCMoCBpt4mVH424biKAR4lG5rcoLRoulwx39DezNms46pq1Abo6ZxReTAAcgrByJ13PT39LM+pxzhtKJReSld/8AkV3oHKQJ2NnQZU1Udq23k7spSJ128REffZvvXbdjIIRdtICSYKW0DWZ1kaj68LEpwAPaUpS4j2SpD9GfGmBrz4mqw3iI3bUUN9Xe60hu8mUFlbyClaAO00BIC4G5nytbW96/AnDm5XHU1tx4bupCu1UUqp6RgqIjMVHKmYAEkjYa287MTN0f7grlM07aQlrOmEAaBQMfO0B4hZbNdSlNM3IRuAkfxHxGug+Otulw/K+ItA+a5+tzNlDiUR4ku0V88RMVXvS1HrLdXfdfUpeKw4XUuVC1hZUdwcwMid99bOzjCF03o94DaGhNW6ghZgQUu6E6+HwszqijUl71htBKgANRpIj5GInwE2e/G0lHAjAISZIq1jQEz3HNOZj47e8ZmPty1eHj/wBb+G9eb8YG+I4T/mP4Mqri8M1akJIGU6JTKQdQTIMR1HWBzAAkzglhm6sfcacI4Uve6BWUblUuqraWobC2nWGWHHVtuARotSQNp10OsWjZ1a+2CS4VNxlT3iQCNDue9qAANxJgTMWJ9A24l3txWv7FLiSWLkuMU6CEEBL1S8kJJUdzkZdGoEbGIt1Q7rsyVGnEPg+ngXxarMLv07j9w1wFZdNRUJzpqKRSpKDyU4hX2ath3QowlYmTsO3XhutW2j1W7FuqyoDIQju+ASdoB6693aLWa9IrhC5xc4fLu266cOYiulxVdchAyqW7Az04URs6kBOvdzpbJJCbVNwFiK5qm7aRF/lNPUoCWEVDrZQFAGE5iUiFa7SPZNjRPuUMjSynDhZd903Xfq6ahu8M9tSFtRCYBHdMa6n2Trty5WK3hTOUq3aVh0hLWYBlRJhSScuh03SPDeNbD4KLjF/0TyHw42peQEEEQpAAE+ZPXbQ72M4tPqN81XbPoS2Xs8KBMkwQAPKPhyi3OQDw/E08Y2lhY75sc5p+jguHpD4TjKojG00Eb/nG9zD9HNRbh1f4w/xCuauQnN2tQmkUkCZQ9DZEjmJzeOXeLW7WiUFBTvoo6a6x7zt4R86VXcKiivS76pTSmhT1ba1POAISkJcA0k84SPwtdKUqyuBU5tRzER9bW2bakL0QnYqD3j6ut1suSUlSCD4Hbbwn8bF3VZkKVEBQ9rURrY/fjZRfN4tJATFU5lM7d47dfysk5gYOYhA0SASQOe3TX8LcW5uV5C7+N2aMO8km9pkcITmUjIRlOsab8vHf7rAOBKm0kEAA5coPn9TYZwkvBORJHeEK1PPX6+O1gVhW68o1kAbfUeW9jM2QH7ps4wokLudl10GGqxpeYAA+1E+cE/W9T/SVu5yj4hXokqzdpT0ziM2hIS2kBWmhnKoctrXBxW3nw5XAOCUt5wDrqkg6/DxtWf0sKAf2noK0Mk+v3QlJIGmZKlka+8fLpa9QOtUArKxRuaAlVgeQtLiwoZSCQR0s6OFT9cxxEw/+7qo09S9XNU7bucoy9ocm41AhVmzVJKHCVCAozp9eFj2GLxXc+I7qvdGho61h9P8AuLCvwt1MgzRkeS5OI5ZAfNXyOA+IdWT2+KKchMaGocJJMbwPqLM3F+Hr+ueoco7yxMhC26U1iSVKHbCfYSeau6R7/jYmnfLiVNFLQCEKKVpnM4DBlXKQdBEcrRTxkao6eruyrXdFPWqAWXCtRAS0mM0aiT3iefPrbiI757LvJYmiPMFE9z3FeV+XuzdD17usuOtqkwVAZW0qmJ1zBfujxsQprnqH7xXdpdrnH2ao0z2VslMSsJUFCdO6OWkjpZ5U97X0wGr4Wihp2qh5+gYdabBcSpLakp130KRGutkeuTiGoZqrxqL6fIVdyK9DaCUiCsBQgEajXXnPna80LOLQkhrCmJqujHq1DVl5fZwlxSUjKUkq3GneEdNR42cVLwjvB1KVVNQ4x33Uq+0SohsJHZmMupn2h+dgauiVcGJaOoarqhwNt0bqVrXOdtThQuR7068o95msNtLbEEwUzPXb7tfjYMrizZGhhbJe6hR/gOzetAmlva8lpWtSy4lkEAgxlAnnpznkI0tU6+qd2nqEtughTYLRmd0KKTv5fK3o4ttpU5XTPMlO3PTXTX7rUK4v3Sm58c39RAQWr1qVARHcWrOjTyV9c9XBpnOLmFZONU7Y2Nc1M4rKBTupUCUj4EKJ/G17rlvBN6XLQXsNEVlI1UETyUhKvx/rah+aaZI/lWefUD8rXP4QViry4aYfqV5VRRdgdR/s1KbHySPjY+MNHLa7zQMFd7bmeS9DfRAvb94cIvUSpM3XelTTAA7JVld+9xXwtNizm1J6aC1YvQkvQKu/FdylWjb9LVgbRnStKjr/AJE2s2TqAJ929qEBuwK9UDK8rRM6ZflbnMZ5+JNsPONvkbaVAOoO8WtAKstrUVIyHTSzcrWy084kHn7zZwgjny5dLJF8NhC+0AM+FrdO6xVWoFwm8g5KmoRP8QX7iI/6TYXvbK3Otg3u5eBUYGdo6beyf/wrCwQTvpOvW2iHLOy3WwVEHlB87YSCdNuttgADMIjz2tzIgGBtsTad1Aiy2CeRjwt1JKpBO+45W51gb9NLbEAkDlpvNpApIT+HnET9fXjY9duj0iRBkEediAgDNqNNTY5dqwXhJ36WnmTWS7mV/Mfj+tst3I5pPwtlnumsF4lcM3GnOI1AEADLSvwQNRKIIPXb63tM1aoq3A0VEjbNy/HS0BYbvNygxbTViS6FBtQBb3BJM7bDTUe60x/vt112nKlJyrTkUlZglXKD8eth0rxsh1DDZOfDlQxT1K3luQ2hKlFJ1JJgQP6bWcn9pLuTKU1QIkQAkjl9R5WZLVSHEKKHZSlI7wMneeX18LJTeMcNVN7G4W79o11qF5Q12kd4bpBmJB5WsSRNebkoUUjoxYKSP7RUQ0TUKhMykJJnpy+pFsOJbuQmEuvTvAT9x56j7rNBSiVZwuQDIG8Drpy1+psVfuulcS46C/nkkZHVAnw3sMwNGimKh19Urf6TLsDjyGk1LnZqUpUDKpIHICZMnN8+thrqxfdbLKnXlLQH5W2Csax3dNfC0SVdO/dle8s9tSvqqELJUSpKUKVBJB3EqEgHn8ClbVBq8Cp99aA3TqWWUA5RrmciNSIEg6e0Olq/KaDqrHNcVI9+cSsP1lSm53rurCqsR2aXM6YCjtPT2YHuHMWhXFN+Xo/iSouxlgU7iGFtLJIlYAGWY9rSfOT42K1d7NPYhReCnUClbzLKe1OZOwAk6pMgR4WPY0Zum/slZdVak1rTRcqsylZy2ACAnrHlyEkRYZYBqFIPJ0KalKw/QVNPedA0X00rAWVIQYCh3TMbDfUyBMyOfpdgqto3ME3MglfaKuumME6BRaSTz21+63nx6P8AU1DXFu4aN5x00tTUuUzra5KHELSpJSUxBSRAO/LfnfXBzApsNXUwjRDVIyhP+VKBHLytynEE33Y12HDUOjpPkqc+lEvLx+S/3lBNbd2Xn/s0e7+Ie73kew9GR6s31CR91vHj0nlVP+nZ1KEZkpcu5RE6ABCNTy6fLW3sHQqHqrZEaoTAnwtdi/q0fwWVUC1VJ8UYVBGkcpsA5oJOlhlE5YEkWAXsZJB5aWMxVn7oq9Guw0sGlxBSJUAfZt0+Y3Me6xBSgFzJOulr0RsqMguhqpwoZLg0UkSI8rSDS11PT09NUOuBSXETzJVCZMWjG9HXDRPJazElCsoTqZy8hz8rKV1XhSvUVHdt91z9FWUwJQ24rs+1SoEA676K9xFqtdlJF1ew8EghPWrxS0lbSaRlac620lJbynKuYUPhYq/ipfZLVSunOIUgqAhSQoA8/vslm5boeU2s3xUFSEtJQUOiR2cwdBznW3C1XPdea66Snr65b/eDKZUEAmZnSBPU2pcuF2tjdXrOb2slpdd2l5VFO2lJSGkrcjQBZJjbmRZLvgE3avOSIW2oEaxCwRZYapqalQQhsJ7QlbhJklR6myXfr1I3dNUQ83ITMFYBJBGgtzlW8zSXYNAtSnswWKCacdLC8xOjhGttUDjjyVNtHZat/O1dPSJxzjXCWInKK4MT1dFTO0zTnZsqCQmZGbadwPqbRvwg4q48qcetXRemNb0qWl3fUOKSp9WUOtoKgo7Ttz8rRmwKWvha8OACabEI4CWkaqzjtctNRfSMxhNevltLaD+Nj4dK2XdPZII+Nqw8OcTYivW9bnfrr7deL9bUoqDVVik9ocpA0PtkCB8LTZjLH7GAU0nrFzVl7i9FqZbRd5ClJcTEA/5p0A1JBtVwnDX4ZjkTXOBuDrt0UMSqG1eFvLG7W/MKU8K3A5e9Swl5vI2pSlFwoOuWCQnb6i0n0tP2DlYiBqtBmNdUIH4Wg/hLxcqL5xBS4VvLBV6XJUstP1TKaxwZltlABgAaagWkq98XV133ow9RXKt9h5o9svtIS2pKm8oOh3BV/wAJt6O6uhdJkzAn6eq4cUsjGZy0pcpHU+t1VCkkPOVAeASgTlHZg7897JLpW+u52G1rSvsHQjMmAlQUQD4HSzQq+INcm9q+7b5Zu+771TeouemdVVLW0pb9P29OSpCREwhJgbkwbJWJsf4xu/h7hGtprkoTeb17IpalxIUunaBWtJIkhXtpG9kyrhc/IHahMYJA3NZQJx8umob4rVT4K+17OnUshEgECCSTprFnB6NrqT67SONFZSzfLcLTI7t4FQEc/b/pZI4uXtX3ti283L0cZFaw+unIabKUqCVnkSdff+qNww4gucPhXXq1dTdbF7XnSqZU7kH2hacmQD16bm2XLVc+U3OgK26eHkxC26s7X0dMqnaNNTUZQBIKadIJ/wAJmxvDLTDd5tLLDSVzooNJAB8xr77Qer0tU0rKyjAFMSy5PeqiIy+OUb6/CxZr0zawvLUjAN2Nl1YU2FVaioa76DX5WUdM0PEmfbyRnVLiwsy7qRuGtDc994Vv7D2ILpavGgeve8aZ9hSQtK0h1BBjl1B3BE6GLJfD/gFdOCuIKMVU99XheN1XdTuNXFdFaSv92rcVLhSskyInLoD3iSSdbR3hHjDelxULb1FddCW7+xBUOOIcCz2IcTm7pBGmg387Oiu49Yput1fqN3XbqSkAtqUNdD/FYBx3DYyGTAlzfJWY6XEGsdyHWa4aqZMf1aazB95tpaZWG2Myi28lSh9o3qI6a6eFqu43apxQwdW+2RkKkgp1AT8Z+t7PVHGjE1+3ZUXbWUN1paqmVsulpjKtIVGxncQPfaO8f1CkUC3iVlwhBSUiAnv6kmND90b2xcZrocUqWSwX0FtVo4bSy0dO5kndNhtSSVnMDnZSdSdTodto/P4lahIgnQSZBHPeeW9t0bjLdQ6MzcPMlSSFSeXI+/ltbTwSD3jKSk+0fD9PdacbLIodcapEvtpT9y1yFAgdg4CQnWYPSI5fEeNoBvdkmspVKTGVnXL3iVEmdtR+OnhFiqxBcYcZzHVBEFRHeIgef62r3fSFprGEqK0dxRPhqdhI6RrG/uG7hnULFxK1wQii6UKSG0pCg4SEiAZnbfw+Q311cnHouo4GYEbGValV6phIg91zly8PGOVkjIF0wVn0SSQEoiZO0xrt8jNnFx3uy8hwQwXNDVJ9TrFKePYqHZJyrgq/kB0AMjcawbZvEbmtrMPzH/rfw3ry7i97W4jhAJ3qP4Ug/ePVVavRxLbJ+0LPdIAGpI3IEmOZ0HhvOt1/2etzBjAGLb9WyEPXhe9OwkqiS2yyrKSf8zytNptTOtbLam1NENJVukJJJA3KsskCANPedrX79D2kN28JbvYUAhVQy4+4udVLNW5vpuEKbHXWOlunLtLLuiw2urAMLCHmyQUS4kwdRodfGeVqp+k3w1Z4c8Vhie7KVl658elyrNOswlqvTrUIBAPtlQeTCd1uiICYtMt4OhaiZkzG8QPa90+W/SzQ9L3CruJuCzl80yM7+GKtm9UpKCYYByP676IVmEc0ixIh3QXbqr+FHqSkvBiouW9am6nkPtuO0j2jau8FaJJyaj+XkZ00mT+ICnqG/HKpFIstu0yFmop8qnkAGDCVDoCdzyEWim61t3tRsmraS2p1GYLCMzalaCCeR3nxE67CU8csXu9d+H7zoqFuomlCKlSkryI0QQdNt1ROvLnI5/FHtpcfoJybB4ljPzDXgerT8yuExt7KLijDKhxsJGzRE/ENe0erDbzKj6ofoK+p9bpnK6pLRCg6XC4G1CAUuNASkTuop5yJMWurhu/rvxLcNFfV2OqVTPsjeApKhoUqHUEHTbmJFqlm82GElLFBSmqpEBZiRl05KOux5eNpn9HS+xel1X62spZeZrEKXThITlC0e2RuMxBH+5rboXsyG/deiMOZvwWsYtJZxPeCFOEFawqIH8SQZ8N+nOyDmCNErJ1AgifLX63HQ2dGP09niVaxmHbMtuQOcAjz5eH32a64SYCidgAk7gx/X+luLqRlncPNd5RuzUzD5BJz0ZyVJ0zTlSraTHhzPzsLX3jWXvVrq7weDtQUhClpaS3MDKNEgDYch42DcUhSysEGU5p6fUe6wZQoLWFKTKSdQdJnTQ/laLLbpO3RK9kdtdVczlKiunWN41ymIHwtXn0oqRNXh3Bd75cyVUztOSQQdENkfHKeu3jrZBxCXWVtTIXKT5EbdOn9LQHx3p/WODFx1p0VSXkWcwBzAQ6mPfAtapTlmaVRrm5oHBUyqkntCD0gRrNgxplI0Njd6NlqrcB5KVpPKTYonXeYG2vO3ZA3C4kizl6mYFq0XxhG473ypJr7tp6jbmtpJ+8/WtmzxLu9h2tud2saC2jUOMOAjuwttQMiP8NtejtepvbgxhCrUcym6FNNrGvZLU3119n52XOI1y117XC6LraDtZTLTUMIV/GpJ1BPKQTbhpP1c5HYr0Nh5tO0jsFBN7UVa/Q3q1SLYN2018IcpAgSoFwydRoQc6Y1+G9sutgVl1XVmSQKq6q+gV3pkpJUnQ6zz11s5KPAWNqekXdzTlIKepbYddzLOZLrWWIAB0B8eVlymwA8w8wUuNdlTV79QgAbsvJIyTyIJ+APvOZWjqqIgeTeyii8bteau24LwTWPPTRreWHFaLCFpWW56CTA8LTtTKbfp2XkyWXEJWBHIiRpttZoXfwkcpCEV99PvttFaW21IhKErSpKk6k75gfcLPW7KBV03ZTXf2i3RTNBpK3IJISIk+MCdLCkeHWsjQROZe6AI1zZsxzCJJj36z0HW1N/Sauv1LiHeLiAQ3WM0tWNefZ9kdPNs2ueUhxRLY0Ookyec2q16XF2KaxFdV45CU1l2OsgcwWnM2vuctewl+Wot3Co4wzNTX7KuSf7tafEK/D8bW29HCt9a4WsNnX1OsfYGmglQX/8Q/LpapLey0g+0n8Z/C1l/RSrg9hq/LtmTTVrb4E6/aNkGP8A2dtnFm3piexCwsHdapA7gq9noaXgaTiJe12qVKK26llMaAqbdQR56KUbXDIESCDOo+otRH0bby/dfGq4pUUoqy/SrlWhzMrA8+8E++17VajrPMWxaN2Zi2a1tn3XJI2nXwtqR/MPO2E9eU20VTqCD49LXwdVRKwq1kcupsTvFGdjNpIMzY0dBor87BuJC2ynXaNbHjNnIUgzCyZ14wiqo3Z0zrbmeqSfvSLCKUYnrPP5W5xCC1TdqEj7B5tcnQABQn5TbkuE6KURrbSbqsxwshgTqCes624zic089t/lYMLndQ08LYlRiRGgn3WKhnVCFX+KAR9C3QUNNRrYAqB1KhBPyttK9Z5gan3WkFFGEqEgnTXzsoXcZeEka/CLJSVHQwCNI3seuxRDxKdJNnSTh7x1yKM87ZbpJUEgAgADQa/nbLK6ay8LbtcW5fzSmVwQ2VHTMB3jz+vlpJC7xZYbbbSsh6O+rfYHQHlaMqOpeZrQ+06kFbQREkmSpQykRpry3687KdM8aisQayryNpWPsk/xbjKPOelqcMmV9keVuZl1JtHX0S26ytq65ukZZZ7DtVOQn2dTJOseZtA+HbvVWY1o7ppnWVxWoR2qAIWlJnOCATqAFa/OzscxBeOI8YLuu4GUVNNR0YQ5TPNgtAlQzd078tYJGum9m/i6mrbuvFzE7eWirCsEtJR2YaUEfwgcgRyncb2LJUB0gb2QmQZWZlZiQglTmmbcEfnB289rJVdW1uZ6gonGg8U50FR1y6j5RaKMEcYq2/H27txG6lt5MkuoSAH29lSn+YDXT+XazkvnEtDRXg1XXUtVStnVYykhadJGkzr9x13FrhmaRdUxEb6pOxrxAWmq/dakN9q2338gJOXLzkSDOmnLrNka58T0Tl3MCqgOQ9SlxWhDa/YJ2kAiJOutmjiKpVet9VV6BsvGoBjtD30pPlG23XUWTs7oaDnZlYQElKiIVm5gnaZn3Rtam6Qk3VprdEvXQhiorAahRdV2yMpgHOk6az00M9Om9ljFK7goatplx5FLUJQpKVBMk7d4K32nQ6QSOYFm52tHTXOzWshHrIWW1kKMIJEb7jTX6NkK9n11FXFQ+y6qUpStKiSBmmIHhG0i0c2ill1T44IVyGeMOFkNwoLvBLZIER3jsPHXTTp1t6F4eb9Wu2kp4AKGkt90dNPrzt5w8KKly7+KGGA4VBCr0pcyQqRkzp6cgRPvHjb0ju9MFKFKBKVawNz91uP4i/aN+C7jhjWJ/wAVS30pVPr411JbZUtDTl3lZgHKClGsctYHj46W9fqCfUmT/wCrT79BbyO9JgLc4xX6szkaRdpkHQyEAT07xHw5a29cbrVN30xjdpH/AIRbShP9Gj+Cxar+tSfFDqkjcg9PjYBZA31sOvrt77AOQkzJiYmx2KpJuijxOUlM2TniQoxpGszysoPnQk/Wlk54lKjm9wtdYqTt0E64UNhZUNDsbPFlq77yCUV9M08gapS4gKANmFei8lA6pGmVBGhG+vxs3rs4rXtQstU62KV5bSEgvKBlZgSSAQBPgLZOKRTPex0XRbOFOjDHh6l+pue67vUhdBQtMl0EqyCJ6WVLlqWnqdaEq7yFlCuUG0OVnFqtroSaelQAICgDqZnrpYFHFi/Kcl+mRSozICIDc6AmDrrMk2qTRTyRgdVfBjDbBTXe1Aay6qymzlJeYcRI5SkibVaOD27vdbrVXnULDC2nnARCQpCgSNeWmtn6OLmMHkFpdTTd/SENJ+trM3E14VNVdld3lJDlO5mSBv3T087Hw1r6WN7H9UCVmchx6LvjHwqxBxTxS3f+EK2krbrFGKR5SHEKAdQpR0ObfvC0ZYQ4AYtYx5Vt0zx9ZZoqmlWVtFtv7RvISlZlKiM4MDxtKOHcRXjdNAu7LsrHKalKpDTRhOoAJB91lK6ayrevNts1TpUtRlWcyZT89bVozVQM5bSLdFZfBDNdzgoho+D2IcPXxS0j94tF24LyL1QIkLSsJXCR8frS0j4+eWq77mcAns72o1bcyqNvf87ISryva+r5vRkUzpfSpKXMmYqUUlSdd9wB8NLLF7Ul+3jc133cxh+8lvoqKda3C1CQlJSpRkwdj06W53G2SukZNIdRpotPD+SxjmMUgUqmaf0g7kZowS3UXXUIMnXVtB0nfW0oh9NXT1qKJnIXexhA5kKdST/yWjVWH78XxlwriJikKqGmpnGHl9ohORSmgdZI0EGfMdbPbD96rp3GEKbLiKpx5sOZkgI7N11UHXmFC0KGbJAL91mV8YfJp2UY49uTE7l7GtorsqKhTeKMN141T/doZyOK1I9mPPzs/cQ3Vedbg6iu/wBRUmopL6ZqchWB3PWnVTIO2UTYHHF93mwm7627rsadp6h9g1LinsuTIe7yPU2PVWOLpfVWpq6htAWhtKcn2moU500mFTaD8V5TiCfgmiw5zg1zRf8AkKuPHO+6HD+NL4ddBcRU1zq0FsAkjNO/Pcb2iWrxK5dNA+hugVUIrr/WQCvL2YXRsOSdDvHus7+PjdVeV9uJuVbjjLbxUSsASIG4+t7R5fNG/UXP2Apytxu9qFSgIlOaiynXzbNt7DpnVEAkcN1WqohDIYx0TexJjCupqeoNC0llwpU4kLhQkpGxjqOnPw1E4OYgq8bv3jSXspCXKC6Xa5p1tIT2jyVJiQf4YOw8LBXzhervBxWSiqpabCPs3gEzrp7J+787BejvhS+EYjQtxz1Wnr6CoZKh3llBMEQoRMpInr8LaVaXMonvYbEBU6YB1U1rtiVPV9XdSXZgrCdZSZh6xebNQ8oqJ+0UmCROgtvEfdUpaXZIMxsQOvwsdvy701vDG72VOOpN3PShxBEhTSwBy/D9W5iG7alSluLvuuWCJgrTB+At5tTvMziXnW5XZSM5bbNCUbgrHFO9ydE6SJnT58vn7wMa3k65d7rbic6MoIDaQFIUJ16nf5CyLh+7l+sIUuqriGzmRL6hBnXQR0H1pY9jGpC6NKVKUcjawAn+IEjXN4SbdBSRAlZ88h5dkxWL3q1XmGDWulCCpKRmSISkEBB01EwZ0sO9e9U9mQXMpgyfdEA+cb/nYg4Ab5IUtISEEp96dQfn8bcOQ3KRumVSoaRz22/r7+rjhaQsQTO2Qjl41MKUqoCYEkpVBJ09/Lx5WirGlN6tWtPstwF06YTOpUZk7fP36a2kowFHvmDokbydB9f1iP8AiC2vLQlJMBKkkJnLoqDqQBtHTc+Nr9M0NdYKpUOLm6pT4SpuoYxuVq8VDsgs9mHB3S9kOTlvmiIMTFp6qHb5cSy1fdLR/utfrovMuqBQKcZskyAIjLO+kzasTBcZbZ3AUk5SJgGBBO0mfvHhZy8c7/vscEcKg3xVI/eFUtqrX2xl5tIXCVk+0NB7WmgJ2t57x5w8/E8RpJWvA5h5eoJy6PdmbqNd/mGrwD9J/CEmOYzQSskA5xEOoJyWbI/M2xGtidPxNYb6Kt19MUwvmqRdrRVSpfWWM3tFAnINUme7G+0n2o0vj6LbTjPD672FoKMlI42BP8rNKsiOXeUvQzt7rUPZumuvO8WbtoWu0qql31dhtGpW6slKBqRqSpAA3/D0l4bYRThKtrcNtpzou69quhBgpKkhqoRPwZT4beFu+dM2GRkROp/cve4YHck21sBqd0928pRmJSqU7c52135fkPF5IoqS/cNO3TXNZqevpTTvCNwpEanfmfgNbM9DYBGRpJGqTIiNP6Dby6F5XI4FXZThXJJSRPIGP6mLa0OqzpAQqH4CucYQ4j1uCr6TCbqvE06kvaAo/wBkqBoZ7mvn42nxt6uezCpp0NNN1DiHM5BC2YVlUP8Al+dml6VGB7tu7G91YyfZKKW/21XdWOJ0KXkkqaUSOpJET+Nm7gC974drH8EYsvB6spbwpFU9M44NT3DInrlzc+VvM/0jcNz4hH9qRuBETfaBBJAubub0vYkkH8IINwvE/wBLfCM+LQ/bULgWwss5pBJDQSXPZrbMASSDb7rSDcJLU/dVXVViLndaXTrccCHE6nKpRgiJJ2Hy52VfR0vypunicbndfK03iw/QPSdcyEl1CjOpjs1AefjaMrwuS8MGXm+WVhbbLq23UAKCe6uNj5q+IiBZ1cN71Zu7izh++EQmnralrMUmBKx2fyJ18+XP0RuUwRljswsLHuLb/Ne14Y6PwzBG7O0tFnb3FtDfz3ViuJ6SzedG84SntadSAOuVX/4VmU8VZsyjBAiCdoPKR4DXa0h8UGStq7KggHKp1BOvPKfw++0flBg5Vr0PLQEx8dj8hblMR9mpcvQMKOekaij2h0ScwVrAOvKdtTp99uAidVBUco1JJjSxh1AAgqlW5B3n46+f9bYGwkhPeSTAIPw5+B+jau1yuEIuhEFIiNcs8z4R8fn74b4s0iXOCWJKZKSv933sVJA1P/nCQfko/U2mkI7ye6kEmTPL61+to3xvdRq8E8SLs0UpLZqoVrEtBf3p89LGifaQHzVaoZmjI8iqBYhbUitcUoKlRzHMNvD68LJQB3n3WW8SNQ8hZUnVGkc4P62RIKSCevO3bxG7AuAkFnFX59Da9P3hwWYpjCjd15VNOBP+V3/4m3lad6tAU4tJQcqoUT5zrr5fQtWD0CbwL+GMV3PuKWupqmAdR2rak/8AwrWneaUFJIAUChJMDXp8ItxmJWjqnjz/ADXfYV+spGHyTdcpkJK1FBKRzIOp1+vqLFXQR3iN/aIG1lh5js1ZVJUYJnQA6E/f08/eWcZUhOVZI2OvTr5fhaoHaK6W90mqaKSSDEQJyk+P42BdbJSCNSkQpM+6Nuv4WPLaVBQBASYyg+H5a24dbKgoZQiSADlkz/U/15LMoFqTPVyXAHIMr3Gmh5j6+FoD9LW5ArDlyXjClerV7lNKUmYeaM+X90B9G1iezSFBWo5g7g8wfC0W+k3dCrx4W3lWAHtLvqaarRB2hwJJ/wCFarWqGUNqWHzVGvhz0rx5KhTBKHRG6gU/ERztOvolVoTiO/LsKo7eibfCTMKLa4+5w2g2qAZrnQkQEOkjwE6WlT0Y631LizT0f/ptNUU/XZOf/ot11cM1M/4Li8PdkqmfFXUwDeKbkx3h29s8ClvOnWonU5Q6mfLSfvt6ML3hQ5m3ml2TiCC2AFAynrOmoPTx/W3o9c1em9bloLzTBTWUrL4g750hU/O3M0Dr3C6jEG2sUbkgEAkeNtHKDJMwba0Gp58httbStTp4z521AsorRJiExbUAQJOu9sUdNPdNuCQPZ5jrYjVEpAxJT56Z9CRqttQHnGlklt4OtodSkd4BXymzkvRvM1mGulmhd5HqbbeaeyJbHkkkfhbSgN2rNmbZyOJWI1II8LbKkp5b9BYILE76zO9tKcIgyCeWtrCAdEKXDOg1nnsbcJUQRl2EaWDLg/i5dPytiVknXTkPDztIKBRjMJE6eNlC7Fy6CZB8rJQc1M+Vj92r+1CidZgz0izpJy5vD5m2WLduTqFq+MWyz6JLw2p0Htu0aUQvswIn+GVTHy9/lZLvKtdp7wUGXFAqJCQNBA3iTyGg+M2PKCGqgpzJU4UpE5iT/FoB4Dn8xNkO9O1TVkttklWoROmo3P1vO82yWm71ZdoxOTgE42rFl5OKVmK6SR3cxnMPjvB8/Ozi4+3LR3fd133k04/29ZUuBaVLzJHd1jpsNB87FfR2o3G7+vpwj7EUyElRHNSpjXXl91lL0lamKG4EAlMuvKUADMgJ1HLnag97vtENG3+y0GNaaEuKhKhUtqrYcT2mikFRiAPDx/TSz1VeRZWltZWlJAWkA5QrQaHmQPw87MWldcdcDKVhQIKTIkDmPr3eaw2isqCnKvKCdQTEnw678tdfcdkrItdH6x8vqFRHfcGpSSlMjYa+Z+tbFVagFvvIjMmTAEA8vhr/AFsI25Rs0q2nHnO2S3lQotiJJ0Opk7CxFSlpUpS5kJCYBmTED89OQHuhdTDbIwe0RdrhS4stqdBykQNtY8do6jSxZtinS0CzVla0lIHcMHUGROxBnwiLH2altFw3i0pJDgdZyJHQEgxZz8EsM3NizEVbTX3SLep26PtUtpcUgZgtI1ymdj1IO9k52VtypRszuDQieCKRf+kbDBRlUE3pTKCkuT/tEzruDqTB318LeltMgh/ID/EQQOU+P1+FqrXXw5wjcdS3eF0XP2TzC0utqLy1HONREqOxE2kimxzihVQZvVZClyVBKRrz5R5+dubxaldXOaWG1l1mDVDaBrmvF7qEPSncSzxfvhBWcrjV2kIiAoQkEzp1n3cufrhc6j+7aU8yyief8It4xekLeddXcTE1NbVesOKXSpUSdVAIEDz6edvZi5HM910ahAmnbJA8UC11sZjgY09AseWQS1Ejx1KPq1kaa/KwK+6PDpNhZ0gCfCwDhkajfTWxY0GRFKhRgjmNPr42S6lQ/CLKVROUncRFkqoUM3eJEabm12PQKk7dJ96KPqT6QD7B+60PodJSEzyzDXlaUcR16aKgUtSAqVpRBgbkC0TSoH7MRoTt9eNq9UdQr+H7FHXHHEHMTMiJI3B5j63sIy84poALJ0EQdCeQ+/4WBaZcUgOnRKxAM6adLLztwpo7mpL3N43e6KlRR6s06V1DUDdaSIA32J3FqhcButVtkmocyogrzcwc236bWDvUOO3bWZTA7Bw6yY7ljwpkEqJnUdAY6fjbmtQ25QVbaFZR2KwT7j0/rYZcpkaItcqjC41SAlRJOuos5LmSDe1OpHcCVFwgmdBuJ+NmxcC2w1GbN9m3ryMg/Cywy64yUvtKIUmYM6g7WC5yKwXCOsPN0GOL6eEISW2lT0GdzX7rOmqxDR093s19O6hTKaaVpSJg5Uj8BaMb1cUq+nHnSVKcpkZleIJBn4/fZSSo/wBknhMK9WV4xAmLczj/AOyDh3V3D4gZCCpKr8eXbRqu1FTSPvuFSkSlQSnK4ypJBnXkLRxw+xziB25rzoVVyuzpr2qmmiBKgkEHcieZ+tLCYgWpQuh1MCKlkqPuUPy+Fmvw9Vk/tFTrKpTfLxAJ3lKD+NudbK80zhdXXQMZINE+Xa+pqT/rL61CZhSvfoOVu6d2GHSEEDMg6HY62TWllQIQEkyRsYB0sap1AUtSJMdwwRoO9Hv3tky3cNVoQ2B0UZcRlJF8urcBEKbKBuD3fr52Zd5B5tVYX2y2fWrqec01El1Ep5dPj5S6OJpUb6JSAElpO50EHf8ArZkX0QbgqnoDhy3csqSoaxUrA36ZremYMQKFi4/Emk1TylGqZv5d5VCLkq2mludmVhxSYOYLAPQmJ+jZL4NrqqPEtBTVjh7RpyrYABBhOde0cpHztoirbrFrFOpYLKSqUaZQowemxP5nkV4X1hcxkytLWX/tB9s6yM2s+787adTI19JIzyP5LOiic2oY89wpyqvtMBV7Ud1uoqQRvGqo+479LNu/4NMlSUpgpnMTvpsdPLpz87L6HO1wnfzAUD2dY7KSIiUn8x8fKzUvSpSu72FwlwqaQRJ2gDX4fC3m1K20h+K7Oc+yiN1urRUNp7TY7cuhsLi4ZrqNQ6UgGEqA0JE7D4WTaNafXJSMxUBtsDPMHcWUcaO5MPzlzKTJGURA5eXW3U0DfasVizE5CUw3D/25ToQhQJQ0mI2OQb/rbhxwZkyAqQBMxtrz3ttlxJxBdjgXJeYAhQkEAKTy/wAottxCUKJJgzzMnfnFunYLALH6aoAp11IBVqSSY3689/ws0sesB26qUIBUo1ZSQNJBCjr01+8dbO1SFQnYQJ1SZ+vGyNiSiRVUaYQnKislQ39pMb+JKfha3EbOCBIAQmdXvN0tfS07a4ythRG+pHXqBE+fjNjvHkrTwPwSEyQmvcBOUgwA4OWos2b9qVuYrUgBK0tHLJBgAAbe+zn9IBrPwOwV3TJrlkBKoOqXOY/pbF4j/rWH/wCN/DevO+K//EsI/wAx/BlSD6Hl2YavH0h8HKxO60mgozVXl/rDiW0IqKdhx5jNOoh1psxMGAJ1i1/LtZZ/tTfTzb7TzJxA2tK21gpIdyzrEf8A1kiJ8zbzj9H+mTXYso3z3HKOjqFhKjGYZcg33ntpkDkLXawLivt1XhQryoVT1FFVJKU+yPsEbHb+71H0S1tE41TasO2FrfNen0k7QwxEbqTbwp+wyrbT3wqVFSiRPhOx3+t1jDzyl3cO8cyVLCiQBrM7ddRv42L37SuGhU4gpTACp9/LXx8PlYLDDx7BxsqIUl5Wk9QNduv1pNt6hkzBZlfHyym96QeEjjDhdezLDWaru1k19MOZLfeIka6p5c4i1WsO3vVOM0l70zgXUsZCUuQMykz8No08trXkUttTZCiktqiUuAEEESQR9aG1JMT4Uc4e42vjC5ORFG92tG4USV0znfaB/mOWAf8AEkxpa/JG2VhjkF2kWI7grKkiZPG6KQXa4EEdwdCErcTxd9JXUd9rpD6re7QUt7syrIoAA5o27pHvm0Z1NDeFLW0lZccuXe2s1LLyFgllcgxBM5ZEx4bWlu73W8W4GrLuT2a626HO2bSlWfTU6E7g98fC0SXnhpVE8qpZLlKFEBXYmE+BjxGkRbleF3PZRvwuU3fTOMfmW7xn5sI9CuS4CnfS0cmDzm8lI8xa7ln3onfOMtHyKt7f15t4jwLc1+tKT/rIaeBCtDmQZ+JFmaGFJSUhtWih4ieZ6cvv6W64VXw3fXB5u7lJKn7oqUsuyqIGeQRHgo/WljrrCCdjlgSkjUc/ONNrZ+Nfq6n5L3TAfbpdO6T3GsyciRoI1JifLprM7fjbgMlJ1mTJnlI038fdZSVSHvwkmTqAPOfo9PC2JpiCW1QnMCRJO89OX6WzmyLXdHdJxZUDIbAIGxE8z9e7bq0K+7PW76xVc7rQKbxucEAHU9xTZ8uX6WfnZEyMqc4Op1meXv1s26xo0/EKjJACa263Wjm/wLBgctlT+liCSxugyRaLzRxQyWi0SFqWkrQSUwCfqfP42biSM2qrP7ijdq7sv29bvISBTXk+wU8wErUkbT05mzAJ/pbv6V2aIFebVTckpCtb6Ad4BOLMVXRrFTd7FT1gtu5Zj/7ba7a2c7YIAMgjlG+vyt5QcP8AiBfHDyvqL2uK87xoat9nsC7RVAZUWyQSkkpVpKUnblZ61PpL8RKiA7ifE6kQYBv11JMn/AlPyti4jhEtXUGVh0XQYXjcVHTCJ41C9GammXnKymFTqFfXX61sUfZ7H7VeVIO5IygmdNbea1RxyxpVLLj15Xm6Z3dvisV/8UWR63iRfNeIqqWne1kF9x56P+NxVq7cAl6uVl/EsPRi9K6m9LjpUhT980NPpBLtUhIHxOm1kqrx1gGjBRV41uFiJSQu8mU7afzdfrp511eN7vXcbFNR4fZZvRK5eqlIYW2pOugbLUjluo62SzjTEEy2/Tt6z3KRlP8A02K3h89XID+JANmL0Qf4t8KmcqneIVw5gO8BXtqj3JM2Y3Fbilwtv7AF/wBwUONKKpqq27Hm6dllC1lbuQlCRCd5jfTXe1JRjLFARkRfVQgAz3FZdfd52CXinEi9VX9X66f+cL/O1iLA2xvD82389lWl4hdIwsy7oteiC3WLBSoTCoUNdrPDgxXIu/izhmp0CXK9poyf+87h2/zWY7zz1Q4XX3VuLVupaiSfebKVxXiu6L9uq90K71HUMvjSIKFz+FtqVmaIs8lgwvyzNf5hejLtNmQpSTrESRoD7vre14OC17JvjhZh2o7UqLVGKQ9ZaJb1/wCG1NzRIeGdKwpJSCFJ/lOoPz8tbWk9GmsU/wAOFUS96K8H2wBsArK5961W4jD5P1xau8xKL9SHKV5BlU6zBtpcDkdPutorAM9Dy625Og5+JtvA2XPELCRztzPe1V+ttmJ0kxNgySmFaWI0qBQdSntGjIgH3zZiJPY1ldTxlyv5gJgQoA/fNn8Z5d2bMW+UGlxC8jYPsIWNP5SoE/MWu051sqlS3S65KxPltFsKyNR/EdN/r6Ni5VCiI708/jvbXaEqPdOsWvjQLOJPVGAuTE85tsKmQZHnYuHPaJMgH87Z2hywI158hZgkjKXe9E7HmI52O3cpAWFSDOkWSs8EnTYkyfxsdu9wlyZ8dbSSTiBajVCP+E2yxPtjzUZ8/wBLZZJLxJbShx1JGSEpRIPNZGumnnJ62bV89yoAUtJlSiDlMefnqZjpZ63E80KlxL1zIqiWkBvNKshCNB3dRvGvSzNvgMP3wrs8jas8DtDlAJM8xtrEx91spn31acPYRSgvu8LqS8LvrHGytKcwCoEpM5tN+Z9wsVvrEl8X8UfvS836gMk9mhx1SgkHeOXIbWU/3OKe7Xa915LpyKMJAUjKBy+Wv9bJ2HqNiur1N1aFlKUnupIE+GukcvefK1izSc1kG7gMt0BdxQlw9okFREpKYEefu+EHayylSVCE1DaQhKiZGvnMa2TKJLLF79jeCQWk9pmSToQAojbfl8tbFHn3CCFqOYcuQ66D9OW9pEXSCOpri4S0hRKUymBsN9dugPnYf933pUIVUsUzixoc4SSdBzIH11NiGRAu9ToYIzGE89Z1jkBIgddPOy3g+qfZq0s+sjs32l9z2j3QSDB8vnaFuyl0QlFdlSLkvVyqpV07qA0ELcB1k6jodp52fnoxIqFcW7ougtB9m+HBRVLZKpyFYnLB0Mgcvysg1bF7V1G5RShCm20obWYSElPSBPx1gaeMmeiJdVXcfH7CdRWLaLabwZUoIJJACpI2O8D8+dqWKPMdHI4G1gdVcw4f0qPS+o07q695cCrsu18M1V3XhQuFAUWniUEpJ0OoBjeyIeDNPXYpw5hu5nfU3r5qjSB14qWlCssgnTwNrYY/co8YYjN53fnUyGUNpKkEbb6e+0S8TE1eBq/CmLaRAU7dt7JfS0dA5CFEpnxAI5728bocaqXYqKdsxey5AN9CvUZTSz4bmMTWSkdtivPT00uHF58LOOtVg68apitfpDQvBxlKihaXGUqETqNyCDv9/rTh9Q/c1CQJ/wBWa5z/AAi3lP6fPEJfE3j7W4rXdqbvSGqNgMT2qsqGgJJIABOvlb1Sw2qLhu+Qr/zVnfp2Y3t68TmiaV5kGua9wdv1Ss6sHQfLpYFRHiDblbhmOU9bArWSSJi0mJnjVc1Ku4UjpOo6WR6lRCoAgp+VlF5aYM6ajwNkuqIGYga2uxhUX7pCxBT+tURAIEOIVKjpoQbRewlCoziARvl8tfr52kbE9Z6vSoBbSoLeQgA8gSNfrwtHtKl1pwOlvNI0O8j+tqlYQ0i60MNBc02R2nIypZSCDnAzE66/QseYpnxUopUlBW4pOX7UACeXQD5e4WJMFa1FZARsAOUe+xuicDSyA8DBOkjn5WoueLbrVa110p3/AHHe2Grxcuy8OyccSlCwplwOIKVJCkkEef1obJVQ2/VMupQ2qS2oR4wbHRV1DjamGGVFR2hJMfWthRdN6PMqWzSVWZSCCUsqO48BYHOj6kI4Y62qRcPXe8hhpKmy2Cy3J8h+v3WddyYfo7xeep6+/GLsQ2yVpcdbKw4qRCQBsfPpsbI9wYbvt+mp3V01efsEDN6urUwP1sZdVdt3uOIqLzo2VtkpWl6sbQoGYIIUZBmfKwzI12gKm1rg3XRIl6KbbvNsZ4JZUARpICv1Fh2odw1UNqMnsXRPMaEedki/cQXHRXyzUO39dXYM07qXXG65lYRqggHKomTqQnQmDAME2L/6SOF7WHK1dRxHuRmrDTym6RTuZala5QQDzMcudsfGIXywDICdeyt0UrI5DncAnDfzpVdl0PBKx9pSk/8AENbNvBAcVemJqcSCLzzjQjLKE+/b8t7Oq/ihWGrjrLlqE1jFTTUj6HQgplCgCDqeX42Z1JeN7XbfN9m7jTqccqUKWVkARkj8LYVNSGSFzT/OqvzSDOCnmlt5AJyKKIkzuP6zY5RUL1RSVzil5S2ylwJGv+0SP+q0VY5xRxDwi6husfYQH0B1tLSWlHL0IOu0dbN65OOV+XI5XVmI311tK5SqaDLKmEqzykgwFTysX/h6eSPOyxVT7WiifkdonRj6hccvincWNVNpAkanX+lmfir/ALLuq88uUlFFTuBR0kIq0H/q+B2slX1x6um87ybqarD1Z2CUdnkFS2hZXOm6SI1PXl7xHcQ0eMbgqa6monaRmou6vYDZdS6rM24wqcwABGs7dNbdTQ0UlPTtZILWWRU1UU8riw3WYpx8q47/AE0SQpYvBhNLAIiFwRvyifoWSeG57K/nnnWwlw34ciU6lKSElRmdPa+E2caW7pQ4wgt9pmZKvtFpBnu6yQr7rJGFXaSixFebjraGVGvSpC11KUxLaDMlEEeO0iNbaXIbynC3RUec4yDXS6l+nqkN3ZiJhxxCD6yhScxCfaQNp0P52a7rblXdDCadTal9ghJTnSO9Akb8vxNlqqvvA1MxeNNUXtdCWX3lOJFReCJOZsJMEZZ57CyOjiTgC67rpaNWJbhpiEBCFprSVGOkrg6CZtxMeHyh5LWldQ6oiLRdw9Uj0lDXIrwVoCZ9omTvvtyjpNnDiW6Xau5FsqWghSR1B+e2+nnZv1HFbA7qws4ypnWzK8yVqWmE+1qkdBOnIcrGb04w3BUUbV0UV/LV2gShKexUkKOh5DUGY18bbdLTzNcDlKzZpYA0jME1HaUNXXKghbiElCXAiSlPaK0nceXjYJ2mqXjkRR1C9zCEKPMeH4/Cxa+MTVN3XctxqtqmEPFzRtahMEE6Aj+bSfHxs3q7GlYimL4TeNUkJJS2grcWuBshOeSSTsLb7Q7YLILmW1Tr/dN4LRpc9WMoBkU6tPEEeM/LfSxaquyuRTPuP0i8rGVcwJRynXnI5idfGxC9sN8Zbuoae83+DGInKauqvVGSX6MEu5FKhQ7clACUHvKyjVP8ycxOiuLi927z1XwtboKBghVXU1GI7szMtxJUGkvFaz3jCUgkkQJOlrMcM17kKu6aK1gVEdY6V4mqUl1J7xBTpIGaN9/rbaz/AONyFOcDsEZnAiKkkmBHsLjcHnHl42jKnqQb4vV4rlOdScs65dTE6gCBPXfbQGT+My44H4EWlJCDUkEgbS2uDEHnHMedsbiTSpw//G/hvXnfFRviWEf5j+DKo69H6E48bojUN04VT1QUp2REAnvQDqcvTc+cWZw1XU9HiOtZReLJW7QJKCkuZVZA6qJIEEFIOsez8aucMqg0uO6d5h8ILrL5SSEifs95MjYnbTQHXW004ZvJbePrracdCxXU7zABBHeUlQGs8woR5cpttmPO25XfiXlOsre3xjMrbF0opU5A2lBcJOYjKnWPxjl8BMI1SHF1CkgBSFJJMQNRE/X3RZimqQ8mlqAZ7SmZXOYCZQnTz3P0bOTBlYgLqmwjKSkGYOUmf1O3K1+CJsf3VSnqHzH2k+0PGM5kHaQYA35np+AtXv0pblTT11wYxayoDiF3bUqCeQlxokjkMzo1/m+M7espEhC9SrWR7/zi0d8fbs/fPDO80tIlyiUitQU6wUHURPQ/XK0gDdV+4d3lSXRjKnJhpF4INI8nXK4VRBjacyRr0J8rNXF9Ri3h9iC8LuvGmTe9zIeJaUkxUtU6tUbmFwnTroYkiLGbuSy442lScr6FZ0EaKj9N53s++JtGrEOHbrxIxRrfW8gMVKUIkpUAdTGsA5x7xbk6r/lePw1Q0ZUt5bu2dt3MJ8yMzfRcPXf8l4ogrBpHVt5Tu3MZd8ZPmW52+iH4E4lu2oqbwuyiqQujvik7ZopMlLiFJ0IOxIgAHp1m0uC7lqGckjTNKgZ28fMframOD669cH4sF5UrimV0VRmdYhQStCjqfgTqdvfaVb69P3h5h+rqLsewRiF2rYkOpUulaTm5wS4o+UibBx+iqaqdroG30Xt/D1fTUsDmTutqp7F3EDIEwQYnce73D5crNDHeIXsMVNMhimCy62VhSgADBGniYFoOq/2kt2tFX7t4XSOXb3yEmfJDC5362ZeLfT6vHELzbv8Ao+w7LSVJSXah90wSNJLSY2HK2TFguJE3LNPiFsS47hwFhJ9CptXxRvTtuzZo2joCAZgb+EE2BpsXXtfOJ7nqGKOmNQhD7bZdqU06JXHtOOQkaDnzMedba/01r/qUgDAWFyUg5c9M4vKTM7qH0bJ7npnYwzpWzgvCKYUDrditun94LWxgte7QtA+aoux2h/GT8k0eOTiqjHOJXClCHHbzedcbQvOlKlrmAeY10I00FopWCkwRE6xFnPjjH9bj3EFZiK9aKmp3q0oK2aNsNMpypCe6nWJyiddybNh1ZWslMxOnlbsqSJ0MLWP3AXE1krZZnPZtdcSSf0tkm2wPA26Qy6v2G1HyE2tXAVUAnZB8om2WVabC2Ja4p9Sw9eVRm27KlcXPwFlul4O8WK5PaUfDTFDyeqLpfI+OW0TKwbkKYhkds0pnm2WkVn0d+NjwBPDe+WZ/9IZ7H/xkWVqb0VON9SRmwxQ0461F+UDUe5T0/KwnVdO3d49QiNoqh2zD6KJLZac6X0O+Kry0Jq7xwpR5ubt9tLj/ANlmP42UWPQ0xWqFVPEjBKEQSezfrHVaeAp4PuNhHEqQbyBGGGVbv+mVXuwgJ7NJ6KOvwtPN7eiPfNE0n93Y9ueveKgFJFJVoQEke1m7MknbTL16WS759HHENPT0gor0TWPBJQ+lFG4gNwTlIUqMwOwgSI1AswxKlds9I4dVM3YpguT0xsFUWG7toavDN7PVTFIyy8sVDCQpxCEhSvbnUgkSPutbz0BuPVz8X28aXTd91PUKroVQ1EOvhwuB0OpJEARHZpG53FvNhn0acaPOrQuuoGEA9wulzMocjCUkDlz62tn6AOG1cA+I99VeLMV3X+7b+un1ZWQOgpfQ6lTcSjUwViPG2SafD4nF8R9pa7anEJQI5R7K9LiBBBMdPK3BEkFU2baOJODHBrfraY3lpwf9NuzxAwhl1vtpMSPZX/8ALYgIUCCl8yRljTxtpR1JIkTrpZvK4h4N1ScQU8jfRQPzFtDiBg5cEX/S6kcjr8rFaQoWS+ZOhI8bMzGgTT3vdtSRHbIdYn3JVP8Aymx88RcFqOUYho5iIkzt5WaHEDiTgZd30dQ3iu7ZZrW0yXgkAElB1O0E78rHhdZ4QJm3Yhy6mZzdd9ibaU8MsgQAJ56+P42Z6+JeAp7mMbnVzPZ1aF/cdbAL4r4GbGX9+hYEqPY0zzmkxplSZtqBwssstN09i6rKYIk6aibYHV7/ABmbR+7xiwihOdBvR0EZh2d3VEEDzQBHvsTf413MElTGHMSVZGuVqiSCQTyzLH3WWdvdLIVJiHZJVmOukAfnY3Quqz76TuDNoeXxqyhK2MEX2lJJ1fWw1lM8++edgk8ca9DqW2MNUrSpIIqL2bSomOiUqP152XMb3TZSp69YV/3c2y0Dn0hKqe/dFClXMfvBWh/9lbLPzGJ8hXnFhu96a7K5xVZUNtyGoTO+VKdZHIDWzexIm7HKp99F2IUTmcntld5RUdQOhkfHlNli6biYvrtl1LlQlxvswOzAVEtgmfHT5nexlzDVIX19stbgSmHCQNdNdBp9eNs5rQDcI7iS2yZ1K8FYWdX2am8naoSEp0mDp4mD8J8rIV0VNSw445TJJXKYUATp9/15WllGF7vVdiaNllZ7VS15UZkzIAn5e6fCzewxh+mavR5HYrbcSgghwEiQrU6GDv4aWnnAUeWUxWkP1lfohRdWSBuCVTpznobbLdQwFtvJyLkpUDp3hvPj+ulpBpbkojitSTStqHalRKjorQKEDN4bb7xYhXUVG1WuIepGS52pScyQSVHoCfH8udmMqcRpltvVfZAOqEZZE66z7+Z38Picw+av950qWnAEpXJI1ERrp+PlZzpbbZSlDNNTwkAkhpAEjWJOv5QetnBc97vXcpSj6qtLntEgpUDCpTp4Aee/LWHO8kQRA7lEmL6YcJaNM+S4rUpYJHujwA5He0kcFcY3RgviThvF96NVz1HTVJqHks0pU5lGhAGgO8xv0slXZX3PWOguuuJcVoQVlOb4GeWxtYPgfxAvi7MQ3VcOH6K4aUuuoQKpy6GXHSknbtFAq5nnuLZGMVDxSvbluCDfW2i2MKpQ+dpDrG4t8V6RcLLlu3G+EbrxhRVK2qW9aZFSy28zlcShQkBQnQ+Fh+KPARjiHc1HdqL5RQilqPWFOGn7TMAgiAMw5kc7DcNsThm5WVVKa+tqimHHUUxyLPUBPdHus8n8X1HYkNYer3wpJBCciCNNznULcNw/DgQgbJI20nTcomIVGIRVbgHbHy/evDL00ri/szxZqKZD5fSsU6g4lG+igNzPKfhtvb1cw26kXDdwGh9TY/8AvabeVPp4Xgh/jdUBmkNKkM0uVC1pKkBKVDlMbH3+dvUrDrqf3Bdawsa0NPp49mnnbuaIvkoonv3IVetysq5ANrpeLx3KhoeVgFukjflt00sB20SorIJsEp9KQSCZm12MKk86XW3VKIJJED52TapcyDrpEgiwj1UdQD8/z91iD9UlQVCiDz1jW16NUXlNfGzqkUCAkqzl9oCD/jB+cG0eL4rUFKlTPr+Hm1g9mEmgpioKH+Y+Zs/MXqL1AjKn2Hm1iPBXy5WqlUcNsbVN7VjzF53Qyy7UuKb7WsIOQqOmVKT4e8WpV9JFU25qu4dUyQXyBTWeNrF3IVmxFQMBCfaapKVER/lQbIFT6T1ULrN90FVjGuoSWgioo7iq0oWXSjsglQp0pJV2reUAmcwjSzEa4L4hrm8lbie4GSYkh14jeD/sRM7Wcd48P+JN7YGq8B3jxkueipK1hppyou25nO3BQvNmSoupAJOckAAS4qAAEZaEWGYeD7S0ZMQrLeyELUekbjupcKKfh7xYeMTmcoXKdOUyJBcWjTQ9NrJzvGzjdei1N3RwjvpwgBsCvxVR06lKVpqhT5JEEHSd7Ef9B1/uKU5VekxiZC1krUaG5qemOqcpMoUCJG/xsbY9HzD9Yj/tzjxxOrykZZN7qbSBzSE5ttAY8LWxRYZH7qrmsxB3W3olnDD/AKRWIsK/2guu48F07TDy7uUmpxC7WkvNqCTKmGy3qvQ9+RlUCJyBdeXbzva+Lwrryq6rsV3k+apxlh9paELVJUEqhQIExIOo5mZtY+7eBXDK5blFz0uO8bvUinXahTIvFoJW65CnFr+yOdSilJJMkwCbZS8DOCV3tBtqkvdSAMqUu3gEiOYTkQkf0tNstHF+yYUN7Kqb9q5QPRvPIoLzbS+4peamczkoMq74kZU/HT3dY+vllx29X6hx9LbxT2awXCDyPJOh18duk2u7w/wF6ONNiuvp8ZXe61c4u5Dqe0q6twqfS6IB7NROxMcvOz8fuD0AaZ9dS7hBqsdHeUVNXg5OnRao25R18bW4mNkbnuB8VRmL2OykXsjnorYJw/xS9H2466/73TT3xdjZoagMPl5ptKYUzvuQ2tuTMEgxIINpJqPRr4euvqdp8WXlTPPQXVtOtkrMRMKSQPKzEwnxo9EzhZQO3bw9wZV3TSVLnaOsUN3JbQ4sD2jmcEnXfxNjtZ6YXCBlOejwXez+kgrbYTOv+Ym3Pz4DeYujeAD8VpR4nNkAOb6Li/vQiwRiivFVfHFrESkIBS0htFIkpB1jMWzOsnayJU/s8+FnZpSzxGxQ6ZUCo1NGkgGYgCmMxPPoPeZf9NzBraou7h3UqATIK6xCfuQbJ73pwhYIoMAMN8gXa1Sh8ki1hmFvY3LzrfAKu6d73Zstz52ShfH7PPhV+7mlUOLL9FWlsFTin6cgqEfwhoHUEjeyBjb0W7qwdf2E+H2B6mvqqO/aC+GHa+sT2qKd9bTZTn7NICRDe2nsm22fTVxHV1iKVdwXNRNrVBcWl50J00Htp1843sjXj6VmNL4vx8sN3Uh3DwfrqZxumMLIp3EqkKUQdF7TvYvgQ05jM4/JJksoP3QtUv7O/EjFM08jijdIXk1SuheGWRqPbOu2tkxPoLXxdlNeF2u4fw3fda9TvNN3k5mntl5iHZWkKBSTsP5iJhKRYa+PTB4kLwvTXndeMaMXi4+WXKVN2MAj/FBSTEazMaxoYlnVPpV8d6tyRjV1pJ1huip28x9yPAj61ushDN3k/IIF3u3A+qMI9CHi+aZijRgLACTTkqQ67UtFCiod+UmmUdTqJJjUSZstMehfxwcNOf3dwtpAyhLeXK/3UpCYjs2UxqCSBAsyKnj/AMdKztCvHWIBIiG3CgA+GUDobJT3FPjFeJIqMY4lcBTOU17x8J0O2nzsdvKHQlDySE7gKcrl9DzjJSXIu6K/G2D0hdUuqU41R1DhTmUFBKZyyBAHezEgJB9lMAUPoIPXepmovXilSoNOUkBN2wCR5u8x5b2r5X4vxu+r/tDEl4KAGvbXguDpzzGPGxFvGtFTPTfOKqBrImQl69Ggc0aTmVMaDx1snvZb7n1UmRHq5W4T6J2BqtTdFeXEgLQgKAbZp6ckAgToorjYcvytp70ReElNTkVnEGtbWJKKhinp2XUE7KCoMKHIxyFq6YbxK2mmReV34ruF1t1GdApr6pXHnE5oENJcLkTrqn87LNLxHcq2ygVCl9mSOaZg66kgEeW+/K3P1lHVzTZ4X5G9lrU74WsyyG5+SnPGfB7h1ii7Ka5sRcdcd1FLQVrV40yaKvo6dxupQFZVhTVNnSRmJkKnboLR9enAjgPTsP1BxTxHvepYQtxj17FdU412olSZT3QrvakEEGfGzTOPbvYbPbGqS4ANezEaT7UKMfOxBXEWhDvZdq0snTKt4tkb9UGNNSOUWeKkrAQXTOKnzKVosGBVLoEJYvWuaL61OpcOfSSpU7ADb32lfjIpp/gVgZp1UJeqF6xqDkcPx35HytEt+lq5sbV9OWZDrqozKnMTIAj7vDlJkyRx3ZvKo9HrAb9KlaXWqlTpJOXL9m4R5Ty+PKwuJR/SsPH/AK38N68s4qcBiWE/5j+FKkX0e37voeJFEbyu+mq30M1QaRVMJdQrMytSjkUCDEaeY2MWsXe98pTWUj7F03XTt09ShRRTU7bJJ1BEhO0aeHTSLU24ZYrql4mu2qpUst1zVQGlB/MpuSFAylIkjX5HQTFp/r7/AMT1jYp2a65UkfaZfVHEBX8MFUEjUg7fwaW3RESbr0EygCym648UhN30zDyuzNPTpQDnUUqygDSE9Bz+Vlq68e0NPVlDy3HEpbOYyNDIMTIJO2w+Vq502M8ZUSECruRFWB3CaN5KsoiR7WTSYnbysoXVjqrZce/fFxX22jIChKaBbpJBB2RmhUHrHnaXMlZsVFscL9wrIP8AFK4kZkoS8SBKRkIjTp08fumyVf3Eq77wu6tu8Xc86h5pbZ+02RGnLUaRaETjWpeSr1bBOKXs47uWhQiZ0ABWtMeI5aDTkC7i3EeSWOGuJVAOEZnDRpCvH+/5yOWnSy8RKVMU8I2TPrHKuiUUUDSu1Z+ybCU5u0ImAQOcT+HMWkXhXiS8sU3HfOCb1ojS3gmnVU0iiSULOgkKmQAS2YnYnpZgGsq27xUmvudy7X3oUmleWlSoOhAKCQenXTlZ2YLvd+570oa5ylS2hpwIIJ7xbXorTaQD8h0NqfEVBJiWFvbB+1bZ7P77Dmb6kW+BXE8Z4VLieFSx0v7ZlpIz2kjOZtviRb4EqIWvXKHFVR+8qhVRUPN5lq9mPAdQCD4jnuLMvGnBm/cc4hQ9hlyhbX2U1Hbr7FKdoMwZ3/QWnTj1cVJhjEAvGku6p7OsUqsS+kFTTZUZWM0QglWwnpG9kHA1U+L5YQ240n1kAkKkJESMpjbc+fXoqLFGYjRR11N74B+BI1B8xsV0HDuJ03ENBDXRfckaD8CRqD5g6FQ5TeiVj52O0v24G1Exl9ZcUQNdTCNBpZTpfQ8xK4QmsxbdrZKspDTS3IjfeNRa3rWFapaO1XWspChmkJKhGkHkddNdec+Jn+y1OVwp5R1mUtK056Sevl/EbVzidUTv9F1gwyl3sqnU/ocshKVVeO3SNSeyu4beEufUHpZRY9ELDAbAcxFfLqyTJSw0jbkBJtaZGGmEJzqedJCgogJTJJiBz+gOkW4TciEAhSHO/JiYKdgBtGv1vaBr6o+8pfZ9KNmqrTfonYZaWVGsvd5IPsqLYkf7v52UKT0fsNU/ZB3DNO4GNQVuLOYwPaP8Xlt8bWabuWlChNKoKToSXOe/4/WtjTdy0CSoFoe1oSkqmDr5+H52Xi6h27inFFTt2AUJXbgXDLBTODLgToAMtz06lCOUloz95s+rooblpwGkNmlHJLDSGgPIIQCdvn5w+P3PStrA7MAZZgJgqP5/Wth2brYBObswDzgAx5fW8WqymSTclWYmMZsm+w1ch0NZezmukVi405aDp9x95pd04ceAL9JXuneVKUvTTSSdenL32WhRUxIW29l5SOUaR0+tbdZGEAq7VB70HMYAHj9c7VHQOKutmaAkNOHcKJhv9ySFCSOwSmBqI1AsYbuW5m47G6G20mATkA08vP7uVlDtqFtBIrGUyZUE5QVeceHSwa6q6wCtV4ITzOVOogH8p+NmFI52pS8SwIuqjpGUhlqjSlOqUw2FRGnUSPrwsE5TNrgtMH+X2RHx9wsYVet3JgCsUtJOgAV3SNNJ0/r7yXdv2hR3cjxKCZkgGORk6g62M2id2QnVre6LP3UHQtSGlFSdhpqOu5H4WKO3C6opyJAO8ZiOh+Gm3Ww6sSop1do3SurChzdIG22xA8x8bBqxe+EZWqdrUCAFEwZ8InpYwoXdAgGtZ3RReE3nPZebbBkBJKtzP6eNtNYBeD6KhNZSHs1AhISoyR4z1sI5iyvjMCGysgJhEzI+e5/GwVRiW8XEyH1jUgSjQ7eHj99jtoXboD6xtk/P7ZcSUKzC87tUlOmjKiEjTYzHLfx99tK4j8SUISE3nd5WDObsBqYGk8vK0dKv+8icq7wdKSc0JVA+XX8LAOXi8pM+t9pJEJB3G+8/XyteZA4blUjMw7BSQOKnElpSQTc4QTP2jJOkzz6e7ewiuN+L2QFVTF1QcxkLCec6SnYA/UWi92rzqUAkEidCZ28zOwGuvKxN1RSrRqSkbAxHT6mx2w90B0o6BSpWcfsRBoJYo6AOEAKVIUJ027u2vjZrX3xjxXel21VEWqTtXEgpz06VshYUNCnIOh52Zay8ApAaCQInvR8dPrawQhOZAAUMw0Bkb6fKx2RBqrvkJRlXETH+RoMN3RTKEaN0CAAN+evT9bFHsfcSFLGW+WGFqOuSibiD5iwKgDopSjmkRmMgmR+FgVFKTq4paSe7Gnu/Tw+NgaBVje6MHFWOnB9piysGYf7JtCY6zA0/XXaxWovLEdQU9tiy/wDKiFZGqstlQiY7vn91uAXQUpCgEyZ028Z8/vtyEuocXGaTsUp9rTpznXzmzg3TINTtbmLrl7Xy8oHKS7eDqwdOYKiNvvOttUlBRLcU65QFUnNJk8geZ6j+ltLW4UQgqzAxruDMGIt3SuvocDhWowRuQc3L8OnP32dR6pXVS0ySU+p02hj2rZYPtyNO3ph/mKp99sslJQFhOkqk+stoumofK3AU9m0pcgITKtJkTOnyNjVTc2InnSpm4a/KYVpSOmTtvl152Qnb9vQKbp6C/KhDRECnp6hRbASBsEq8uXjpYNqkxHey1NtUt61OhHdp3nJgEaZZBOwInxtS5gBuSEct0tYpxtXPiZgl1d3VqQNIWyWxqT/N7o026zZHqMP3nTuqradqkYcLilkLrWklQJgiCQQY+YieihQcLOIN6PpF28PcSViglSvs7reWM0dcu420mY1nazkpfR343VQzUvCXFMNZc2e63G58swE9PztA1MQ3ePVOIXuHstKY9VcL7r/rbd80DT5CSsKrmSDBBMZVEjYaa+Nuq25qWuaT6xiG6kPQUrcQtaiobx3EEHbadJ+Mi0Xoo+kZeaQin4RX6hCJyl5LbQTrsM60mP69Ic9yehX6StVSClc4eu0aHe852960yUriIlIXO3jyO06jdW0zd5B6hSbTTONshUFVOF7tYb7J7FlGsxnX2bbxkEAjTs4ECdORjzBN2luWnGUX1UOZe6ot0x3CpnWI5/W1nF/s9PSFvNTa1ruVlKcpKam8yD/iCVIQuJJPlpzFlmk/ZncWKggV2NMNUjagZDa3lkHwHZwd99NQNLQ+0qJo1kCIKKpOzFUhuooGW0KTXVxCQMoLKTz3/vAZmJ0+dnDc3Fq98K3rT3jdNVXJqadYWy52LWUKSTCoVmze8QOlrY037Ma+mwlV48TrvQtRmGaNahm/3iJEk9DZMxF+zpuXDFCqvvvjIo9+EMU9zAKXrolOZ5QO3NJFgvxTD5Rkc64PSyPFRVzDmYLH4qHB6eHH2mZLNHjC9W2EiVdl6sjLMAaCnEan8+lm/iD0x+Ol/oLN4Y1vt9CFKyA15ak7QOxSjx66RtvaZrg9FDhJdKc19sX1f6jGdNTVdi0pRkghLSQeY0JIHjZZuX0ZOC90ldY3gxx9Di5DdXUuOxG4R0GsHlt52qNOEQm8cI9ArPh8Rfq5+/mqdsf2y444zujD1BSJrr8visZoqRTtQ4S6tailCVLdUYSMx1naY5C3tJcmG8V0tz0FCu4ni5TUrLLii82AVIQAYlQ5i1W+GXCvhZhDiBcGMU4Rum5m7qvFisXWqWtlFOlCge0KioJCQAfaEWtu/wCklwIYPe4s4ZJBBhu8EOH4JJNtyidBXMuPZAWVWRzUj7O1JQYw7icqg3UQr/7K3+CrYrCOKlIn1Sn22L+vyBFir/pR8CpAZx9T1OYSn1ajqXfeMjZ6WAX6T3CpwD1SsvurB1HY3DWxy1lTQHS1zkUzd3fVUjJO7QNR5WAsVPCJoWxvq+o/9FsHCy/XR9re1I3m6NqXr8R4WSXPSZwiof6phXF7+4GW6w3P/tHE2BX6Td25Qqm4b4sVmnKp31NAVB8X7SD6VnvD1TCGof7pSneHBKrvFkM1WJmxJCu5RncajdzqAbVmxxga98GY2vPD9L6xeKaZaFpqSlLYWVoCyN4AGaOZtPT/AKS95rSk0XC6sUVSP9Zvhhr/AMAXaqHG7h9jbivxDvXiS1d1JdDN7ertPULd4LeUhTLSUJUVhCArMEE7CIPO1aplpXgAO1VmlhqYjctKdSLvv0hI/wBTaXsErr2UHcHmrz+tjjWHb1qUBdRftzMzoUrvJCoPP2Z+haJmOEGOaZCUqv54ZBlRlrF6D8tvh8VFrgviaponKt++UuuU4HaN+sLV9nJ7+520BEaSD1Nqdqf8SuEVBP3VI5wuttwCpxVc6Ecsjrq9IP8AKj8bCfuW4WiXKjiHd7Ujf1d8GB5pHL62tF44NPPAl6tQhURISSofEHx+Fke/eEF7oeU5ddTX1lSEStptZSlwRqUDYK0iOc/GYfT7XUeXPvZTU2xw4aSpD/FBpa1CMrVMCJ8QpwWBdvnhPSJUl/GV5vKQdS1TMEGOUdrM9P6C1Z14dShYBaqMqVBCwXFdomdIIInQmOs6RYw1hdCnVKepMhnvZ1SUj+EHqJ6RGp8bSAi7KNpFLl+474aM1Ieu3EN4IQtvs3C/SElW2VGRtKlTuc22h11Fo9vPixciVrTSUd81SUHKFhumbS4BzAceSYPjrrrEGxWnuO7kKk0KAf4wo5ZPgTM6E6EcuekdruG7lhJpmG8hjMkwVpM8pGh5QdN9RFpBzNgFAxPKTjxKp3iS1hy9FOJBVDtVSIEDxQ6uN+li7vEyscTKML0YiCBUXupOUxpORhX1O1l5i56NLKltsU4UCQlWbKtI0kKH5aanewrVJRNlTdRT0JSD3ihsJIHOQN9ht8LPnaOiXKd3TUVxBxSs5qTDtxIJMgfvCpe93/m6RzB/G3CMe46qSDSsXAjLCu5d9S8oeEKdQeXSz0TQXeQFt9kpCBJOSU+Gk6aA+/3ACqpGgpCXVNO51EZ0yABGsGJn4fCz8xvZLlO7pkvYp4iVSM6bzo2FEQFMXFoUxvKqhfjuOVg6W9OKDJNSxiOrQp1tSF9nQ0SISqApMqbXuCefLY8n0UNtj7GrKm82Ud4kaK3zcvCZ326YtN3kOIDLqEoSJUEqUknfcanSN+k72WcHonEVuqYwRjqpSttWJL33yryro2gCf8SKcR7jYBVyYp0m9b/cBUZUq93U9Ng2pJPhFpCUllaA2h1LkwSYKUg7zIEQPvBsUUulpVQXSwUju65dI1PTmT8bLOegUTE0dUyH8PXu+Slyvr+zMEB29q5QCifF73e7xsWTw+bMO1VPRvISSYqC4/mMHUdotUc9OelpAVeF3lkOvusLGXRSklJjTYyZkT5/G3CL8uLtBMoTsC4nuxGpJEjl79dtLPnd0S5bB1TGRw1p1EH1G5BK9k3dT5iNdNUmDPjpPlYdrBaaAksIp2ERKizRtInXllSIj3naN7PE3vQq7zb5dBUDBToNDrJAkydPZHzkmu+Wm1JUgVaFH7Pvagxz3gQAddN+tnDnkJZWJCbw29TAJVfFfKoltuoPenkIIM6kRH52DXhRJKfWLwvFQX9p9pUq+ZnqOnMbQBZWXftRHbJSHz7MklKzroBMgnbXYR11sD++CSta6aoZW0Ncq8xSRz5wIB5nppzcZioEsCSncF0SnEreS4pYO+dR7x8NYI0Mc9ekW03hWhbC0JpAk94lJSSCkEEnczrzAIGtlJV5vOIWpCG3mykthOvtGQDI56fLwsXVV1jTZbZLK1ZSpKVNnQ6A7nYGOg231tINclmaNl1hDDtxXzxpw5T37Tsro3s6g2oANurbaUpCFA6EFaZy89iIMWn9jEl5YqvlzA1/YBr0XcU1qLxqqqmy0fZIVDXZrnvFQKY05KUIi1WcTuV1Wu76uhdcRVUj0h5iW1NrSEqSpJA0Vy0gz42dV6Yu4yY2ww/cZxqttpZQy+EsNNrdbIMoUtKQog7EyJEzzFuF4r4Rq8eq2VELm2DcozFwLDcnO3LoSQdjbVrdV4/x5wLW8UYlHV072WDA0ZnPBidcnmMDRZxIOxy6tab22h3DLNFhvjBRP07Yq7vbry42XSUF1pClQZ592NxzHW1o6PiLhFAz1d1OUpWAUn1dDiDM7BKirYHcazpBtVbDjTrt+XdVNLU1UUCnUrKVZVQW1dDJ1AgDkT5B6uVV4uPLTTKdXkJXl7clYSIAJBE76QNJ1EgkjthBmAaTey9cikMLA29/irP3XjnB1WhIZxDQU7RkZXkGmgiJMupHjqNJ18bOWlFDeQDtBX09Qkpkpb7N0AwRMp3G+418TtTF5+pUwXKt1LaHgMgeQEAhOg7pTBAHxB5wMum6ZapyJCnHSrs3GYCgYTqNATHdOnTnpEHUAJuHKy2vtu1XWTdQOSUtlQOVUJUmdf8ACOXmNJ00t0q70A5nKlSQpUkIcAIBVynyUPdp0tUCjvDiC0hTl2X1fA7NIzoTXfZo0MjIVb7ciIAiJss0/EHibdak071ZQuIU57VU1lCtCP8AZpH825Mkk66pFoGiI6ogrmnopl4jYeYqlU9VSvpFSiYzKSpQJgaeACR+PKyHhOmbqcQ3dR3kvMHXkodQlPdTB2kaakRtzNmfR8Uq281IZrrspXcy57dmvaSUgqEqLWYmOYgzy5TZy09Sh9wuMEJDsQpvQnnAjmJFrL4HS0r6drspc0gEbi4tcfDdYuKwurYJYY3FjntIDhu0kWBHmN1MVTfN51t+U+GV4YfXRrdearHHWJplU+VWQhR0JPcEeKhHMV0vcUdx4srEXQ32lLQV7jbB1IyoWYEx00nWz5expj+vu9+kavvJRAdn2/dS4UjQwvLJ2Osz42it9MPhJJcSlWQEnXvEhRHQmBPxtw3CnCtRw/JI6fKA5oFmFxDiCSXnMBY62AF7DquA/RzwTWcJyyyzljWua1uWNz3BzgXEyOzgWcb2AF7DS5VmsNYquqruhkKp3FLZRBjINDoFTM7AHnoDvZTViWlObLTOKBOoUUwPOBppGw+POEuHN9KFMaEujImQlJPeme7O/L7vi+fXTBKmkDmFFHdE6GT/AE8J3t1hpGbr2AVjraJ1uYoQCU/u5sAjm4JgA6kwNJP5baFjf9Q6c6adhMa6EmSRuPCSeXKzdSl1WchLsA6qSIPu68tRofDl0hS8ktPyATpl0E8j1PjHLzswpWBLxLylty/60qUS0hBEJB0Ma76ncxYuq+LyKoXVFABJgNaRPXroPgbJqFvpTLiG1BRGx0jTUcvrnvbajUkFJbSde6pK5I8uh/W0xTt7KJqHIyu86xeZKn1EKJEDRQ8JsCutqV5VhyrOWU6E6fdYLs1hWU5kJI/iSIHU/fbspA/hUpO0FcTIM2fkt7JxM7ugvWH1JKnVPqVEd9R39xsAVreT2sgpIAgqnTWOdjhZZWcnZEawClZAHU6nTe3K6YkmUmdCCe8PKRz+tbOIgOijzSUnLKlQFVMLP8qpBMTttJ+t7bS482kdmXCD/MmQNvKCPLlY0tlSVD7Qmdio6K8+v36WAXTuEBSU6b+0J003jUfl52mIwoGQoALcC8wfWMomcse/pHy/DjtnnJIWNdQZjwgjmI+ZsZTDUrFMCr2ZP36E/XPa3LimXQApteaRplAPwjXTxsuX2Tcwou44VJJKDmmJ0Jnr9dbcqq3ZKVIzQCnlGU+A2+vOxtLdJIOdKSoSBHmSTrblLTCSFNme7qMunhty38/CzhijzEALzfSlPb04Cd53mPH6+63K6wJEKCsxI0kfHXl8t7GFKbCwg9nIJ1PTkInzsEopz5EBBVJzKyyCfGBp7rSDFEyIquqg5gFZUgaxrE6GPrym2/XEFsqXyjQkAkwNd/rnYXKkKAbCAADGVMAAbaH3fW3DiFApUhzWdZM9Nf0/M2kGAKJfbVBqfZcSU5Uad0gJnT6n9OXK/VlpMRBEyG5392n1rbl5Li5WlaSgnMDmgdesfXlYB2kfCitJQlYOhB5kHXrM2nlUOYhCllQUlJAM6Qogzt+Q08OdgloZdSoJnX2SVExPhPy+Ns7GrbK1qqEmASFIke7pyPz87BlL+XKpxHOddOeo0+ps6RN1oNNJglaTAjQe/bf+vxDKCgGFk5QBlOm86dbDEVRzIKGgpAiTtMRy35WBWFIT9qwmJhScpk/HXl8rMLBMgVuOZjDoUR4gHaJmN9/CwROQ9nn1Om2/OZnryn4xYwuoaQQh6mASNAdh5wBHwiwSXmFZZS6uUQClJiI67dNP62fMAolpRZxwgLJC1DLBAZIJ8d425WFZX3iClMDUmDPUHXXexlLSVABgOhEAkrQe9POfCxynuta05WkZE7AKEQSfKfqLRzgBPyiUXBbUMzjDeY6q052yywm6k5R9nTnTfOPztlm5gT8or0wZwThinbS3TXFdzKEmUpRSIAB8gLHmrno2khKWmwBsA2ALZbLeVgdV0Gdy0qjbKoBKR4W0u7WtlLUQRbLZYrWA7ooe4bFAKp2Gj7BV5m2nG2kgKKJ3tlstLKAjNcTugHkJb1CUkCDqLaLh7PMEI3ja2Wy0BoUdutrpJvi8HKCgerkNpUW0khO3ztA2IfXMS1vrt617izJSlCEhKU+Q91stljUwvclWRskZvD1It9CFOOHOQCdOc62DduijUrOsLISAEpCvZH185tlstdCgVym56GrCqZ9suoUQlQdOcK23HPf5WEduW6KCpdpqK7KVoMLLQIZTJyxrtvJm2Wy1iJxA0KC9oO4XTLRynsilsBIMBOmv9bDVDWd1DMx2baRIG8yT99stljBxPVDygdEEpASoJUSqVRvGwKv+mwqWm3qBtRSe68tKZMkaCT56Wy2WK1QsgU0qIVqNEhQ7o/mj8PnY5RUaXVOsKWcvYrVoNe6kqH/hj3m2Wyz9VFFXbspkqKyCQJkddT7uXS2qOnZYrEvU6AlbUnWFBWmxHMRI99stlphN1SjeeGqOmfbWwrI2+2l5KAn2Aqe7PMSP68yBwzRvl1pSykqiFoSApJmMwPUWy2WcqKjPibgq6jcS8bNlSKth409UiARUhD3ZhU/wr00Vrpoc2hETON06aZBNOkl1OsaaSdumomR1tlstpUurBdZtTo/REXbwp2qdmKPMmodSiFKScpIKgoynX2R79fCxJGIH2aRtSW5z1HYyojMNN5jXYafdbLZa6AFTLijLd+VL7zLSm285SjvlM6qAO31NiNZfl4U/YKC05XwNEpCSAonnsYyjlbLZYgaFBzigXb/vM1DZTUHtErgLKUnTaNttefSwrNbeAqzRVNe68lbYdGuUCTEQPf8ALpNstliBosglxuunu1p3W3nahx77ZDQBMQFJGsjXn8h4yHVKfQ80TVOkuyEd49xMAwevPpvbLZaQaFG5RVAL1K04mEdogGBMA66767c/0t01VrQ2zSVEuKdcLedBDcRGpAGvlNstlnsErlGEMNKdLRTlJQRmbhPs6D8LctUak6B9UZtAQDsSOevLrbLZaKi4lCKu1vtkpQ6tvKJlAAOhJifed53sXCo76VuEKWAMypIlWokACPCLZbLPZNcrhLlMhwNOU5UaghvMFAQB10M7+HzsTvC8U0Ty1IaWtwqgqW4DMJ7ukaxHPW2Wy0gmRBu+UVqkKFClC4HeK5jfbQR7R2s5rvutN5NBYcSyAYhLST1EzuTpuSTtbLZZyooze2EVXXRfvB+vbqUrrEICAwUEFU6zmIO40jl5QIzdgpyUNPqBacRlMCdyPw+dstliN2UXbqHP3O1R4kYUh0yiqSlZCRK5cgmDI5HcEa+c2LouGuFqyo9USy8hQmSVJyxlzQAlKT8SYn3Wy2WqMO6skCy1VYCw5ddUlhDFSop3UmtfQFa80hcH+vW3bOFsOuMrccupp0AjuvkvTJ/xk8+kWy2WJcqIAXKLhwk0pCThagClOJRKGWwNdATKDO/XrtJsqP4eobppk1TFNSDNlkIpkp5x4jY9JtlstEbqZAXLK1vNpUVGUCRJkSdNtjsN+mkWab6lqvNWVQT2qgnRI0BII2HI9I91stlixbocvdNPGnES8rjvMYVoKRDqlIceNRVurcGilCAhJTHs8jtHS0fL4iuuvVbF5XO065SEoK6d0tJcAAPsqC43jfr1tlssOX7yND9xODhpxLN5X7+6mLmNOA22rtDU5iSZiQECYg+Oo1EWsaUtOAKWlUZUq7qoMnflHytlssN26kzZdCjStAfS4tMNhccySgHfyIHutrK4jVLypHdBIB5+NstlmTrpLaj2gz6A90aiNPAj6FttUwfUcrriYWUmSFAwT18rZbLJNc3W3KFxnKv1mQnomD13mxFbzqXBnWXCFhBzagmdzbLZaQCe65XUuJWhCsqtABpAjy62EpXy6tqnEhTgHfMGCQOUWy2WSQRu86J2mioNSFHMAAEERKlDqeSfnZNQ4tRJC4gpTGUEH5T87ZbLOkuXM63FJKh3RMx8bbU2kgkySVGJiNAP1+NstlkkFjl3KShxwvggAQnJpE7b2KlqXFJBSNRsnczbLZZimQSGO2StebKE92IkkgxMm2zTDKgpVAM6ETsT+Xztlssr6JINSHGioJdICdI98WGp23XEBZfmYyhSQQBvHQ2y2WinsEcRdilf3ryViNuzjoeR8bCfuanjK44sxochiZE856RbLZZnEp7BdMXDR1CFrS4+nKRopzNM/Cxw4bpKVwICyrInQx485Jm2Wyw3OKK1oWk3OwpwABoKWvJJaBgxvYJ+mQyjO0lCeSRB0IJ13tlssMON1INCLJRKUPJUU5yrTTQDbXr42Gp6P1l3slrSBJSO4DtP5Wy2We5TtAXTNBTPFxtTSBpmJyzMkD8vhYZq7aRJClJVBP8ACQDqAdZB62y2WGCVKwuhF0FJmMsyZ3J3tlstlpJ7L//Z" } ] } ] } ] -} \ No newline at end of file +} diff --git a/samples/js/node/notebooks/question-answering.nnb b/samples/js/node/notebooks/question-answering.nnb index 3b53ca163f0826..c1bf61063b5e15 100644 --- a/samples/js/node/notebooks/question-answering.nnb +++ b/samples/js/node/notebooks/question-answering.nnb @@ -17,62 +17,10 @@ { "language": "javascript", "source": [ - "const {\n exp,\n sum,\n tril,\n triu,\n argMax,\n reshape,\n getShape,\n downloadFile,\n extractValues,\n matrixMultiplication,\n} = require('../helpers.js');\nconst tokens = require('./tokens_bert.js');\n\nconst { addon: ov } = require('openvino-node'); \n" + "const {\n exp,\n sum,\n tril,\n triu,\n argMax,\n reshape,\n getShape,\n downloadFile,\n extractValues,\n matrixMultiplication,\n} = require('../helpers.js');\nconst Tokenizer = require('./tokenizer_bert.js');\n\nconst { addon: ov } = require('openvino-node');\n" ], "outputs": [] }, - { - "language": "markdown", - "source": [ - "## Download the Model" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "const baseArtifactsDir = '../../assets/models';\n\nconst modelName = 'bert-small-uncased-whole-word-masking-squad-int8-0002';\nconst modelXMLName = `${modelName}.xml`;\nconst modelBINName = `${modelName}.bin`;\n\nconst modelXMLPath = baseArtifactsDir + '/' + modelXMLName;\n\nconst baseURL = 'https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/bert-small-uncased-whole-word-masking-squad-int8-0002/FP16-INT8/';\n\nawait downloadFile(baseURL + modelXMLName, modelXMLName, baseArtifactsDir);\nawait downloadFile(baseURL + modelBINName, modelBINName, baseArtifactsDir);\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/models/bert-small-uncased-whole-word-masking-squad-int8-0002.bin'", - "" - ] - } - ] - } - ] - }, - { - "language": "markdown", - "source": [ - "## Download the Vocab" - ], - "outputs": [] - }, - { - "language": "typescript", - "source": [ - "const baseImagesDir = '../../assets/text';\nconst imgUrl = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/text/bert-uncased/vocab.txt';\n\nawait downloadFile(imgUrl, 'vocab.txt', baseImagesDir);\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "File successfully stored at '/home/nvishnya/Code/wasm-openvino/samples/js/assets/text/vocab.txt'", - "" - ] - } - ] - } - ] - }, { "language": "markdown", "source": [ @@ -83,36 +31,10 @@ { "language": "typescript", "source": [ - "const core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\n\nconst _ppp = new ov.preprocess.PrePostProcessor(model);\n_ppp.input(0).tensor().setElementType(ov.element.f32);\n_ppp.input(1).tensor().setElementType(ov.element.f32);\n_ppp.input(2).tensor().setElementType(ov.element.f32);\n_ppp.input(3).tensor().setElementType(ov.element.f32);\n_ppp.build();\n\nconst compiledModel = await core.compileModel(model, 'CPU');\n\nconst inputs = compiledModel.inputs;\nconst outputs = compiledModel.outputs;\n\nconst inputSize = compiledModel.input(0).shape[1];\n" + "const modelXMLPath =\n '../../assets/models/bert-small-uncased-whole-word-masking-squad-0001.xml',\n\nconst core = new ov.Core();\nconst model = await core.readModel(modelXMLPath);\n\nconst _ppp = new ov.preprocess.PrePostProcessor(model);\n_ppp.input(0).tensor().setElementType(ov.element.f32);\n_ppp.input(1).tensor().setElementType(ov.element.f32);\n_ppp.input(2).tensor().setElementType(ov.element.f32);\n_ppp.build();\n\nconst compiledModel = await core.compileModel(model, 'AUTO');\n\nconst { inputs, outputs } = compiledModel;\nconst inputSize = compiledModel.input(0).shape[1];\n\nconsole.log('Input size:', inputSize);\n" ], "outputs": [] }, - { - "language": "javascript", - "source": [ - "console.log('=== Model Inputs:');\ninputs.forEach(i => console.log(`${i}`));\nconsole.log('=== Model Outputs:');\noutputs.forEach(o => console.log(`${o}`));\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "=== Model Inputs:", - "input_ids", - "attention_mask", - "token_type_ids", - "position_ids", - "=== Model Outputs:", - "output_s", - "output_e", - "" - ] - } - ] - } - ] - }, { "language": "markdown", "source": [ @@ -123,14 +45,14 @@ { "language": "javascript", "source": [ - "// The path to the vocabulary file.\nconst vocabFilePath = \"../../assets/text/vocab.txt\";\n\n// Create a dictionary with words and their indices.\nconst vocab = await tokens.loadVocabFile(vocabFilePath);\n\n// Define special tokens.\nconst clsToken = vocab[\"[CLS]\"];\nconst padToken = vocab[\"[PAD]\"];\nconst sepToken = vocab[\"[SEP]\"];\n\n// A function to load text from given urls.\nfunction loadContext(sources) {\n const input_urls = [];\n const paragraphs = [];\n \n for (source of sources) {\n paragraphs.push(source);\n\n // Produce one big context string.\n return paragraphs.join('\\n');\n }\n}\n" + "// Initialize BERT tokenizer\nconst tokenizer = await Tokenizer.load('../../assets/vocab/vocab.txt');\n" ], "outputs": [] }, { "language": "markdown", "source": [ - "## Preprocessing\n\nThe input size in this case is 384 tokens long. The main input (`input_ids`) to used BERT model consists of two parts: question tokens and context tokens separated by some special tokens. \n\nIf `question + context` are shorter than 384 tokens, padding tokens are added. If `question + context` is longer than 384 tokens, the context must be split into parts and the question with different parts of context must be fed to the network many times. \n\nUse overlapping, so neighbor parts of the context are overlapped by half size of the context part (if the context part equals 300 tokens, neighbor context parts overlap with 150 tokens). You also need to provide the following sequences of integer values: \n\n- `attention_mask` - a sequence of integer values representing the mask of valid values in the input. \n- `token_type_ids` - a sequence of integer values representing the segmentation of `input_ids` into question and context. \n- `position_ids` - a sequence of integer values from 0 to 383 representing the position index for each input token. \n\nFor more information, refer to the **Input** section of [BERT model documentation](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/intel/bert-small-uncased-whole-word-masking-squad-int8-0002#input)." + "## Preprocessing\n\nThe input size in this case is 384 tokens long. The main input (`input_ids`) to used BERT model consists of two parts: question tokens and context tokens separated by some special tokens. \n\nIf `question + context` are shorter than 384 tokens, padding tokens are added. If `question + context` is longer than 384 tokens, the context must be split into parts and the question with different parts of context must be fed to the network many times. \n\nUse overlapping, so neighbor parts of the context are overlapped by half size of the context part (if the context part equals 300 tokens, neighbor context parts overlap with 150 tokens). You also need to provide the following sequences of integer values: \n\n- `attention_mask` - a sequence of integer values representing the mask of valid values in the input. \n- `token_type_ids` - a sequence of integer values representing the segmentation of `input_ids` into question and context.\n\nFor more information, refer to the **Input** section of [BERT model documentation](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/intel/bert-small-uncased-whole-word-masking-squad-int8-0002#input)." ], "outputs": [] }, @@ -158,14 +80,14 @@ { "language": "javascript", "source": [ - "// A function to add padding.\nfunction pad({ inputIds, attentionMask, tokenTypeIds }) {\n // How many padding tokens.\n const diffInputSize = inputSize - inputIds.length;\n\n if (diffInputSize > 0) {\n // Add padding to all the inputs.\n inputIds = inputIds.concat(Array(diffInputSize).fill(padToken));\n attentionMask = attentionMask.concat(Array(diffInputSize).fill(0));\n tokenTypeIds = tokenTypeIds.concat(Array(diffInputSize).fill(0));\n }\n\n return [inputIds, attentionMask, tokenTypeIds, diffInputSize];\n}\n" + "// A function to add padding.\nfunction pad({ inputIds, attentionMask, tokenTypeIds }) {\n // How many padding tokens.\n const diffInputSize = inputSize - inputIds.length;\n\n if (diffInputSize > 0) {\n // Add padding to all the inputs.\n inputIds = inputIds.concat(Array(diffInputSize).fill(tokenizer.padToken));\n attentionMask = attentionMask.concat(Array(diffInputSize).fill(0));\n tokenTypeIds = tokenTypeIds.concat(Array(diffInputSize).fill(0));\n }\n\n return [inputIds, attentionMask, tokenTypeIds, diffInputSize];\n}\n" ], "outputs": [] }, { "language": "javascript", "source": [ - "// A generator of a sequence of inputs.\nfunction* prepareInput(questionTokens, contextTokens) {\n // A length of question in tokens.\n const questionLen = questionTokens.length;\n // The context part size.\n const contextLen = inputSize - questionLen - 3;\n\n if (contextLen < 16)\n throw new Error('Question is too long in comparison to input size. No space for context');\n\n const inputLayerNames = inputs.map(i => i.toString());\n\n // Take parts of the context with overlapping by 0.5.\n const max = Math.max(1, contextTokens.length - contextLen);\n\n for (let start = 0; start < max; start += parseInt(contextLen / 2)) {\n // A part of the context.\n const partContextTokens = contextTokens.slice(start, start + contextLen);\n // The input: a question and the context separated by special tokens.\n let inputIds = [clsToken, ...questionTokens, sepToken, ...partContextTokens, sepToken];\n // 1 for any index if there is no padding token, 0 otherwise.\n let attentionMask = Array(inputIds.length).fill(1);\n // 0 for question tokens, 1 for context part.\n let tokenTypeIds = [...Array(questionLen + 2).fill(0), ...Array(partContextTokens.length + 1).fill(1)];\n\n let padNumber = 0;\n\n // Add padding at the end.\n [inputIds, attentionMask, tokenTypeIds, padNumber] = pad({ inputIds, attentionMask, tokenTypeIds });\n\n // Create an input to feed the model.\n const inputDict = {\n 'input_ids': new Float32Array(inputIds),\n 'attention_mask': new Float32Array(attentionMask),\n 'token_type_ids': new Float32Array(tokenTypeIds),\n };\n\n // Some models require additional position_ids.\n if (inputLayerNames.includes('position_ids')) {\n positionIds = inputIds.map((_, index) => index);\n inputDict['position_ids'] = new Float32Array(positionIds);\n }\n\n yield [inputDict, padNumber, start];\n }\n}\n" + "// A generator of a sequence of inputs.\nfunction* prepareInput(questionTokens, contextTokens) {\n // A length of question in tokens.\n const questionLen = questionTokens.length;\n // The context part size.\n const contextLen = inputSize - questionLen - 3;\n\n if (contextLen < 16)\n throw new Error('Question is too long in comparison to input size. No space for context');\n\n const inputLayerNames = inputs.map(i => i.toString());\n\n // Take parts of the context with overlapping by 0.5.\n const max = Math.max(1, contextTokens.length - contextLen);\n\n for (let start = 0; start < max; start += parseInt(contextLen / 2)) {\n // A part of the context.\n const partContextTokens = contextTokens.slice(start, start + contextLen);\n // The input: a question and the context separated by special tokens.\n let inputIds = [\n tokenizer.clsToken,\n ...questionTokens,\n tokenizer.sepToken,\n ...partContextTokens,\n tokenizer.sepToken,\n ];\n // 1 for any index if there is no padding token, 0 otherwise.\n let attentionMask = Array(inputIds.length).fill(1);\n // 0 for question tokens, 1 for context part.\n let tokenTypeIds = [...Array(questionLen + 2).fill(0), ...Array(partContextTokens.length + 1).fill(1)];\n\n let padNumber = 0;\n\n // Add padding at the end.\n [inputIds, attentionMask, tokenTypeIds, padNumber] = pad({ inputIds, attentionMask, tokenTypeIds });\n\n // Create an input to feed the model.\n const inputDict = {\n 'input_ids': new Float32Array(inputIds),\n 'attention_mask': new Float32Array(attentionMask),\n 'token_type_ids': new Float32Array(tokenTypeIds),\n };\n\n // Some models require additional position_ids.\n if (inputLayerNames.includes('position_ids')) {\n positionIds = inputIds.map((_, index) => index);\n inputDict['position_ids'] = new Float32Array(positionIds);\n }\n\n yield [inputDict, padNumber, start];\n }\n}\n" ], "outputs": [] }, @@ -186,7 +108,7 @@ { "language": "javascript", "source": [ - "function getBestAnswer(question, context) {\n // Convert the context string to tokens.\n const [contextTokens, contextTokensStartEnd] = tokens.textToTokens(context.toLowerCase(), vocab);\n // Convert the question string to tokens.\n const [questionTokens] = tokens.textToTokens(question.toLowerCase(), vocab);\n\n const results = [];\n // Iterate through different parts of the context.\n for ([networkInput, padding, startIdx] of prepareInput(questionTokens, contextTokens)) {\n // Get output layers.\n const outputStartKey = compiledModel.output('output_s');\n const outputEndKey = compiledModel.output('output_e');\n\n // OpenVINO inference.\n const inferRequest = compiledModel.createInferRequest();\n\n const transformedInput = {\n 'input_ids': new ov.Tensor(ov.element.f32, [1, 384], networkInput['input_ids']),\n 'attention_mask': new ov.Tensor(ov.element.f32, [1, 384], networkInput['attention_mask']),\n 'token_type_ids': new ov.Tensor(ov.element.f32, [1, 384], networkInput['token_type_ids']),\n 'position_ids': new ov.Tensor(ov.element.f32, [1, 384], networkInput['position_ids']),\n }\n\n inferRequest.infer(transformedInput);\n\n const resultStart = inferRequest.getTensor(outputStartKey).data;\n const resultEnd = inferRequest.getTensor(outputEndKey).data;\n\n // Postprocess the result, getting the score and context range for the answer.\n const scoreStartEnd = postprocess(resultStart,\n resultEnd,\n questionTokens,\n contextTokensStartEnd,\n padding,\n startIdx);\n results.push(scoreStartEnd);\n }\n\n // Find the highest score.\n const scores = results.map(r => r[0]);\n const maxIndex = scores.indexOf(Math.max(scores));\n\n const answer = results[maxIndex];\n // Return the part of the context, which is already an answer.\n return [context.slice(answer[1], answer[2]), answer[0]];\n}\n" + "function getBestAnswer(question, context) {\n // Convert the context string to tokens.\n const [contextTokens, contextTokensStartEnd] = tokenizer.tokenize(context.toLowerCase());\n // Convert the question string to tokens.\n const [questionTokens] = tokenizer.tokenize(question.toLowerCase());\n\n // Get output layers.\n const outputStartKey = compiledModel.output('output_s');\n const outputEndKey = compiledModel.output('output_e');\n\n const inferRequest = compiledModel.createInferRequest();\n\n const results = [];\n const preparedInput = prepareInput(questionTokens, contextTokens);\n\n // Iterate through different parts of the context.\n for ([networkInput, padding, startIdx] of preparedInput) {\n // OpenVINO inference\n inferRequest.infer({\n 'input_ids': new ov.Tensor(ov.element.f32, [1, inputSize], networkInput['input_ids']),\n 'attention_mask': new ov.Tensor(ov.element.f32, [1, inputSize], networkInput['attention_mask']),\n 'token_type_ids': new ov.Tensor(ov.element.f32, [1, inputSize], networkInput['token_type_ids']),\n });\n\n const resultStartData = inferRequest.getTensor(outputStartKey).data;\n const resultEndData = inferRequest.getTensor(outputEndKey).data;\n\n // Postprocess the result, getting the score and context range for the answer.\n const scoreStartEnd = postprocess(resultStartData,\n resultEndData,\n questionTokens,\n contextTokensStartEnd,\n padding,\n startIdx);\n\n results.push(scoreStartEnd);\n }\n\n // Find the highest score.\n const scores = results.map(r => r[0]);\n const maxIndex = argMax(scores);\n\n const answer = results[maxIndex];\n // Return the part of the context, which is already an answer.\n return [context.slice(answer[1], answer[2]), answer[0]];\n}\n" ], "outputs": [] }, @@ -200,23 +122,9 @@ { "language": "javascript", "source": [ - "function runQuestionAnswering(sources, exampleQuestion) {\n console.log(`Context: ${sources}`);\n const context = loadContext(sources);\n\n if (!context.length)\n return console.log('Error: Empty context or outside paragraphs');\n\n if (exampleQuestion) {\n const startTime = process.hrtime.bigint();\n const [answer, score] = getBestAnswer(exampleQuestion, context);\n const execTime = Number(process.hrtime.bigint() - startTime) / 1e9;\n\n console.log(`Question: ${exampleQuestion}`);\n console.log(`Answer: ${answer}`);\n console.log(`Score: ${score}`);\n console.log(`Time: ${execTime}s`);\n }\n}\n\nconst sources = [\"Computational complexity theory is a branch of the theory of computation in theoretical computer \" +\n \"science that focuses on classifying computational problems according to their inherent difficulty, \" +\n \"and relating those classes to each other. A computational problem is understood to be a task that \" +\n \"is in principle amenable to being solved by a computer, which is equivalent to stating that the \" +\n \"problem may be solved by mechanical application of mathematical steps, such as an algorithm.\"]\n\nrunQuestionAnswering(sources, 'What is the term for a task that generally lends itself to being solved by a computer?');\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "Context: Computational complexity theory is a branch of the theory of computation in theoretical computer science that focuses on classifying computational problems according to their inherent difficulty, and relating those classes to each other. A computational problem is understood to be a task that is in principle amenable to being solved by a computer, which is equivalent to stating that the problem may be solved by mechanical application of mathematical steps, such as an algorithm.", - "Score: 0.5286847737759395", - "Time: 0.045750747s", - "" - ] - } - ] - } - ] + "function runQuestionAnswering(sources, exampleQuestion) {\n const context = sources.join('\\n');\n\n if (!context.length)\n return console.log('Error: Empty context or outside paragraphs');\n\n if (exampleQuestion) {\n const startTime = process.hrtime.bigint();\n const [answer, score] = getBestAnswer(exampleQuestion, context);\n const execTime = Number(process.hrtime.bigint() - startTime) / 1e9;\n\n console.log(`Question: ${exampleQuestion}`);\n console.log(`Answer: ${answer}`);\n console.log(`Score: ${score}`);\n console.log(`Time: ${execTime}s`);\n }\n}\n\nconst context = [\n 'Computational complexity theory is a branch of the theory of computation in ' +\n 'theoretical computer science that focuses on classifying computational ' +\n 'problems according to their inherent difficulty and relating those classes ' +\n 'to each other. A computational problem is understood to be a task that is in ' +\n 'principle amenable to being solved by a computer, which is equivalent to ' +\n 'stating that the problem may be solved by mechanical application of ' +\n 'mathematical steps, such as an algorithm.',\n];\n\nconst question = 'What is the term for a task that generally lends itself to being solved by a computer?';\n\ntry {\n runQuestionAnswering(context, question);\n} catch (error) {\n console.error(error);\n}\n" + ], + "outputs": [] } ] -} +} \ No newline at end of file diff --git a/samples/js/node/notebooks/tokens_bert.js b/samples/js/node/notebooks/tokenizer_bert.js similarity index 79% rename from samples/js/node/notebooks/tokens_bert.js rename to samples/js/node/notebooks/tokenizer_bert.js index 10dc250abe8e51..b23bcbfc8519d2 100644 --- a/samples/js/node/notebooks/tokens_bert.js +++ b/samples/js/node/notebooks/tokenizer_bert.js @@ -1,10 +1,39 @@ const fs = require('node:fs/promises'); -exports.cleanWord = cleanWord; -exports.encodeByVoc = encodeByVoc; -exports.textToTokens = textToTokens; -exports.splitToWords = splitToWords; -exports.loadVocabFile = loadVocabFile; +class Tokenizer { + constructor(vocab, original) { + this.vocab = vocab; + this.original = original; + } + + get clsToken() { + return this.vocab["[CLS]"]; + } + + get padToken() { + return this.vocab["[PAD]"]; + } + + get sepToken() { + return this.vocab["[SEP]"]; + } + + tokenize(text) { + return textToTokens(text, this.vocab); + } + + detokenize(tokens) { + return tokens.map(t => this.original[t]).join(' '); + } + + static async load(path) { + const { vocab, original } = await loadVocabFile(path); + + return new Tokenizer(vocab, original); + } +} + +module.exports = Tokenizer; // Load vocabulary file for encoding async function loadVocabFile(vocabFileName) { @@ -17,7 +46,7 @@ async function loadVocabFile(vocabFileName) { vocab[token] = index; }); - return vocab; + return { vocab, original: lines }; } // Remove mark and control chars diff --git a/samples/js/node/notebooks/vision-background-removal.nnb b/samples/js/node/notebooks/vision-background-removal.nnb index 92e4da28d1b4e9..fea5ba9fc5dd68 100644 --- a/samples/js/node/notebooks/vision-background-removal.nnb +++ b/samples/js/node/notebooks/vision-background-removal.nnb @@ -10,38 +10,23 @@ { "language": "typescript", "source": [ - "const { cv } = require(\"opencv-wasm\");\nconst fs = require(\"fs\");\nconst { addon: ov } = require(\"openvino-node\");\nconst { display } = require(\"node-kernel\");\nconst {\n downloadFile,\n getImageData,\n transform,\n setShape,\n displayArrayAsImage,\n} = require(\"../helpers\");\n" + "const { addon: ov } = require('openvino-node');\nconst { display } = require('node-kernel');\n\nconst { transform } = require('../helpers');\nconst Image = require('../image');\n" ], "outputs": [] }, { "language": "markdown", "source": [ - "## Download Images and Model" + "## Load and Compile Unet Model" ], "outputs": [] }, { "language": "typescript", "source": [ - "const baseArtifactsDir = \"../../assets/models\";\nconst baseImagesDir = \"../../assets/images\";\n\nconst modelXMLName = \"unet_ir_model.xml\";\nconst modelBINName = \"unet_ir_model.bin\";\nconst modelXMLPath = `${baseArtifactsDir}/${modelXMLName}`;\nconst modelBinPath = `${baseArtifactsDir}/${modelBINName}`;\n\nconsole.log(`Model XML path: ${modelXMLPath}`);\nconsole.log(`Model BIN path: ${modelBinPath}`);\n\nconst foregroundImgUrl =\n \"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_hollywood.jpg\";\nconst backgroundImgUrl =\n \"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/wall.jpg\";\nconst modelBaseURL =\n \"https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/vision-background-removal/\";\n\nawait downloadFile(foregroundImgUrl, \"coco_hollywood.jpg\", baseImagesDir);\nawait downloadFile(backgroundImgUrl, \"wall.jpg\", baseImagesDir);\n\nawait downloadFile(modelBaseURL + modelXMLName, modelXMLName, baseArtifactsDir);\nawait downloadFile(modelBaseURL + modelBINName, modelBINName, baseArtifactsDir);\n" + "const modelXMLPath = '../../assets/models/unet_ir_model.xml';\n\nconst core = new ov.Core();\n\n// Read and compile model\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, 'AUTO');\nconst inputLayer = compiledModel.input(0);\nconst outputLayer = compiledModel.output(0);\n\nconsole.log(`inputLayer: ${inputLayer}`);\nconsole.log(`outputLayer: ${outputLayer}`);\n" ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "Model XML path: ../assets/models/unet_ir_model.xml", - "Model BIN path: ../assets/models/unet_ir_model.bin", - "File successfully stored at '/home/prakash/OpenSource-Repos/ocr-node-sample-clone/ocr-node-sample/assets/images/coco_hollywood.jpg'", - "File successfully stored at '/home/prakash/OpenSource-Repos/ocr-node-sample-clone/ocr-node-sample/assets/images/wall.jpg'", - "" - ] - } - ] - } - ] + "outputs": [] }, { "language": "markdown", @@ -53,37 +38,10 @@ { "language": "typescript", "source": [ - "function normalizeImage(imageData, width, height) {\n // Mean and scale values\n const inputMean = [123.675, 116.28, 103.53];\n const inputScale = [58.395, 57.12, 57.375];\n\n const normalizedData = new Float32Array(imageData.length);\n const channels = 3;\n\n for (let i = 0; i < height; i++) {\n for (let j = 0; j < width; j++) {\n for (let c = 0; c < channels; c++) {\n const index = i * width * channels + j * channels + c;\n normalizedData[index] =\n (imageData[index] - inputMean[c]) / inputScale[c];\n }\n }\n }\n\n return normalizedData;\n}\n\nfunction removeBackground(mask, image) {\n // Iterate over the mask and set all background pixels to white\n for (let i = 0; i < mask.rows; i++) {\n for (let j = 0; j < mask.cols; j++) {\n if (mask.ucharPtr(i, j)[0] === 0) {\n image.ucharPtr(i, j)[0] = 255;\n image.ucharPtr(i, j)[1] = 255;\n image.ucharPtr(i, j)[2] = 255;\n }\n }\n }\n}\n\nfunction removeForeground(mask, image) {\n // Iterate over the mask and set all foreground pixels to black\n for (let i = 0; i < mask.rows; i++) {\n for (let j = 0; j < mask.cols; j++) {\n if (mask.ucharPtr(i, j)[0] === 1) {\n image.ucharPtr(i, j)[0] = 0;\n image.ucharPtr(i, j)[1] = 0;\n image.ucharPtr(i, j)[2] = 0;\n } else {\n image.ucharPtr(i, j)[0] = image.ucharPtr(i, j)[0];\n image.ucharPtr(i, j)[1] = image.ucharPtr(i, j)[1];\n image.ucharPtr(i, j)[2] = image.ucharPtr(i, j)[2];\n }\n }\n }\n}\n\nfunction combineImages(mask, fgImage, bgImage, newImage) {\n // Iterate over the mask and combine the foreground and background images\n for (let i = 0; i < mask.rows; i++) {\n for (let j = 0; j < mask.cols; j++) {\n if (mask.ucharPtr(i, j)[0] === 1) {\n newImage.ucharPtr(i, j)[0] = fgImage.ucharPtr(i, j)[0];\n newImage.ucharPtr(i, j)[1] = fgImage.ucharPtr(i, j)[1];\n newImage.ucharPtr(i, j)[2] = fgImage.ucharPtr(i, j)[2];\n } else {\n newImage.ucharPtr(i, j)[0] = bgImage.ucharPtr(i, j)[0];\n newImage.ucharPtr(i, j)[1] = bgImage.ucharPtr(i, j)[1];\n newImage.ucharPtr(i, j)[2] = bgImage.ucharPtr(i, j)[2];\n }\n }\n }\n}\n" - ], - "outputs": [] - }, - { - "language": "markdown", - "source": [ - "## Load and Compile Unet Model" + "// Details about this normalization:\n// https://docs.openvino.ai/2024/notebooks/vision-background-removal-with-output.html#load-and-pre-process-input-image\nfunction normalizeImage(imageData, width, height) {\n // Mean and scale values\n const inputMean = [123.675, 116.28, 103.53];\n const inputScale = [58.395, 57.12, 57.375];\n\n const normalizedData = new Float32Array(imageData.length);\n const channels = 3;\n\n for (let i = 0; i < height; i++) {\n for (let j = 0; j < width; j++) {\n for (let c = 0; c < channels; c++) {\n const index = i * width * channels + j * channels + c;\n\n normalizedData[index] =\n (imageData[index] - inputMean[c]) / inputScale[c];\n }\n }\n }\n\n return normalizedData;\n}" ], "outputs": [] }, - { - "language": "typescript", - "source": [ - "const core = new ov.Core();\n\n// Read and compile model\nconst model = await core.readModel(modelXMLPath);\nconst compiledModel = await core.compileModel(model, \"CPU\");\nconst inputLayer = compiledModel.input(0);\nconst outputLayer = compiledModel.output(0);\n\nconsole.log(`inputLayer: ${inputLayer}`);\nconsole.log(`outputLayer: ${outputLayer}`);\n" - ], - "outputs": [ - { - "items": [ - { - "mime": "application/vnd.code.notebook.stdout", - "value": [ - "inputLayer: x", - "outputLayer: test_0", - "" - ] - } - ] - } - ] - }, { "language": "markdown", "source": [ @@ -94,7 +52,7 @@ { "language": "typescript", "source": [ - "// Get Image data from the foreground image\n\nconst fgrImageData = await getImageData(`${baseImagesDir}/coco_hollywood.jpg`);\nconst inputImageMat = cv.matFromImageData(fgrImageData);\nconst originalImageDisplayMat = inputImageMat.clone();\n\n// Convert the image shape to a shape and a data type expected by the network\nconst [B, C, H, W] = inputLayer.shape;\nconst resizedImage = new cv.Mat();\n\ncv.cvtColor(inputImageMat, inputImageMat, cv.COLOR_BGR2RGB);\ncv.resize(inputImageMat, resizedImage, new cv.Size(W, H));\n\nlet inputImage = transform(\n resizedImage.data,\n { width: W, height: H },\n [0, 1, 2]\n);\n\ninputImage = normalizeImage(inputImage, W, H);\n\nconst tensorData = new Float32Array(inputImage);\nconst tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, tensorData);\n\n" + "const foregroundImagePath = '../../assets/images/coco_hollywood.jpg';\n\n// Load foreground image\nconst originalImg = await Image.load(foregroundImagePath);\n\n// Resize image to a shape expected by the network\nconst [modelInputHeight, modelInputWidth] = inputLayer.shape.slice(2);\nconst resized = await originalImg.resize(modelInputWidth, modelInputHeight);\n\n// Create a tensor from the normalized input image\nconst transformed = transform(\n resized.rgb,\n {\n width: modelInputWidth,\n height: modelInputHeight\n },\n [0, 1, 2]\n);\nconst normalizedInputImage = normalizeImage(\n transformed,\n modelInputWidth,\n modelInputHeight,\n);\nconst tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, normalizedInputImage);\n" ], "outputs": [] }, @@ -108,26 +66,9 @@ { "language": "typescript", "source": [ - "// Do inference\nconst inferRequest = compiledModel.createInferRequest();\nconst inferResult = await inferRequest.inferAsync([tensor]);\n\nconst { data } = inferResult[outputLayer];\nconst reshapedResult = setShape(data, [512, 512]);\n\n// Create a Mat from the reshaped result\nconst reshapedMat = cv.matFromArray(512, 512, cv.CV_32F, reshapedResult.flat());\n\n// Get the height and width of the original image\nconst height = inputImageMat.rows;\nconst width = inputImageMat.cols;\n\n// Resize the inference result to the original image size\nconst resizedResult = new cv.Mat();\ncv.resize(\n reshapedMat,\n resizedResult,\n new cv.Size(width, height),\n 0,\n 0,\n cv.INTER_LINEAR\n);\n\n// Convert the resized result to uint8\nresizedResult.convertTo(resizedResult, cv.CV_8U);\n\n// Create a Mat to store the background removed result\nconst bgRemovedResult = originalImageDisplayMat.clone();\n\nremoveBackground(resizedResult, bgRemovedResult);\n\ndisplayArrayAsImage(\n originalImageDisplayMat.data,\n originalImageDisplayMat.cols,\n originalImageDisplayMat.rows,\n display\n);\ndisplayArrayAsImage(\n bgRemovedResult.data,\n bgRemovedResult.cols,\n bgRemovedResult.rows,\n display\n);\n" + "const inferRequest = compiledModel.createInferRequest();\n const inferResult = await inferRequest.inferAsync([tensor]);\n const { data: resultData } = inferResult[outputLayer];\n\n // Normalize the result data from grayscale to RGB\n const rgbData = [];\n for (let i = 0; i < resultData.length; i += 1) {\n const value = resultData[i] * 255;\n\n rgbData.push(value, value, value, 255);\n }\n\n // Create image based on result data\n const [outputHeight, outputWidth] = outputLayer.shape.slice(2);\n const maskImg = await Image.fromArray(rgbData, outputWidth, outputHeight);\n\n // Resize the result mask to the original image size and save it\n const { width, height } = originalImg;\n const resizedMaskImg = await maskImg.resize(originalImg.width, originalImg.height);\n resizedMaskImg.display(display);\n\n // Remove the foreground from the original image\n const removedBgImg = Image.mask(originalImg, resizedMaskImg);\n removedBgImg.display(display);\n" ], - "outputs": [ - { - "items": [ - { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAKXA+IDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwC3iloxSgV7h5wmKKWjFABRS0YoAKKWigBMUtGKWgBKKWjFAhMUtLRQAlGKWigAoopcUAJRilooASilxS0AJRS0YoASilxRQAUUYpcUAJRS0UAJiilxRigBMUuKKWkIbS4paMUAJijFLijFACUUuKKAEpaXFFACUUtFACYopaKAEopaU0AJRS0UAJiiloxQAmKKXFGKAExRilxRigBKMUuKMUAJRS0YpAJijFLiimAmKMU6ikA2inUlABSUtLQA2inUlACYopaMUAJijFLijFACYopaKAEoxS4ooASilopgNop2KKQDcUYpcUUwExRilooATFFLiigBKTFOooGNxRS4oxQISilxRQAlJinUlAxMUYpcUUAJiilpKAEoxS0UAJRS0lABSU6kxTASilNFADaKdRQA3FFLRigBtFLRQAlJTqKAG0UuKKAEpKXFFADaKWigBKTFLRigBKKWigBaWiloGJRilxRQAYopaKACilxRigBMUtLRigBKMUtLigBKMUuKMUCExRinYooGJRilxRQISilxS4pAJRilooATFGKXFGKAEpaWigBMUYpcUYoATFFLiigBMUYpaWgQmKKWigBKMUuKMUAJiilxRigBMUU6igBtLS0YoASjFLijFACUUuKMUAGKKKWkAlFLRQAmKKWigBMUUtFACUYpaMUAJijFLRigBMUU6igBtFOooASijFFABRS0UAJikp1FADcUYpaXFADaMUtLQA3FGKWigBMUUtFACYopaKAExRS0YoGNpaKMUCEoxTqSgBKMUuKKAExRS0YoASilo70AJikxTsUlACUYpaKYCUUtGKBjaKdSYoASkxTsUUANxRTqTFACUlOxRQA2jFOIpKAExRS0UAJSU6igBtFLiigBMUlOoxQA2ilxRimA2ilooAbRTsUmKAEpMU7FJQAlJTqSgBMUUtFAC0tFLQMSlpaKAExS0UtACYoxS4pcUAJiilpaQCUYpcUYoASilpaBCUUuKKAExRilxRigBMUtLRQAmKMUuKWgBuKWlooASjFLijFACYoxS0tACUUtFIBMUUtFACYoxS0tADaMUuKKBCUtLRQAlGKWigBKKXFGKAExRS0UAJS4pcUUAJRiloxQAlFLRQAmKMUtGKAEopcUUAJiilxS4oAAMg0mKCxXAx1palPUp7ITFFLRimSJRilxRQAmKKXFFACUUtFACUUtGKAEopaKAG0U7FGKYDaMU6koASinUlACUUtGKAEoxS0YoASilxRQA3FFOpKAEoxS0UAJRS4ooASkxTsUYoAbRTqSgBKMUuKMUAJRS4ooASkp1JQAmKKXFGKBiUUtFACYpKdSUAJijFLiigBuKMU6imA2ilooAbijFOoxQA3FJTsUYoAbRS0YoAbRS0YoASkxTqSmAlJTqTFACYpKdSYoASilooAdRilooGFFGKXFIAoopaAExS0tFACYpaKWgBKKXFKBQIbS0tFACUYpcUYoASlpaKAEoxS4opAJiloooAKMUtGKAExRiloxQAlLRS0AJRilooASjFLRigQmKKXFFACUtLRQA3FLilxRQAlFLRigBKMUtFACUtLRQAmKMUtGKQCYoxS0UAJRS4ooASjFOxRigBuKKdRQAmKKWjFAD027TnrUZ61MreYpBQAL0NRVEN2XLZCUUtFWQJijFLRQAlGKXFFACUUtGKAEopaKAEopaKAEopaKAG0U6igBKSnUYoAbiilxS4oAbijFLRQAmKKWjFACUUuKKAG4oxS4paAG0YpaKAExRS0UwEopaMUANxRilooASilxRigBKSnUYoAbRilxRjmgBKKXFGKBjaKU0UAJiilxRQAlJTqTFACYopaKAEIopaSgBKMUtFACUmKXFGKYCYpKdSYoASkp2KTFACUUuKTFACUlOpKAEopcUUAOopaKBhRS4ooAMUUtFABiilxRQAmKWlxRigBKKWloASjFLRSASiloxQIKKWjFACUtFFABRS0YoASjFLRigBKMU7FFACYoxS4oxQAmKMUuKKBCYoxTsUUgExRilxRigBMUUuKMUAJijFLiigBMUU7FFACYopcUUAJijFLiigBKMUuKWgBtFLiloATFFLRQAlFLRigBKMUuKMUgExRilooGSx42NURHNQyXBglUBchuDU5OTkCojuy5LRDcUUuKMVZmJiilxRigBMUYpcUYoASjFLijFACYoxS0YpgJiilxRSASilopgJRS0UgEopaMUAJSU6imA3FGKdikpAJijFLiimAlGKWigBKMUuKKAEpMU6koASilxRQAlFLijFACUUtFADaKdRigBuKKXFFACYpKdRQA2ilxRigBKTFOxRimA00U7FJigYlJinUUANoxS4ooASkp1JigBKKWigBKSnUmKAEopcUUANxSYp1FADaTFOxRimA2kxTqKAG0UuKKAHYpaKWgYlLilxRikAlLRiloATFFLiigAopaMUCExS0tFACUYpaXFADcUtLRQAmKKWigBMUtLQBQAlGKXFGKAExRTqKQCUUuKKBCYopcUYoATFLiloxQAmKKWigBMUUuKMUAJiilxRigAxRS0UhiUUtGKAExRilxRigBMUUuKXFMQ2inUUhiUUtGKBCUUtGKAEopcUYoASjFLRQAmKKXFLQMqTI0kq7V4B61axip44yyttBPFREYJrKD1ZpPZDMUYp1JWpkJiinYpMUAJijFOxRigBuKMUtFACUYp2KKAG4oxS0YoATFGKWjFACYopaMUAJijFLiigBKMUtGKAEoxS0YoAbRS4ooATFGKXFGKAExRS4ooAbilxS0UANoxS0YoASilooASilopgNop2KSgBKKXFFACUUtJQAlFLRQAlGKXFGKAG0YpxpMUAJRiloxQA2iloxQMSkp1FMBtGKdSUAJSU6kxQAlFLRQA3FFLiigBuKMUuKKAG4oIpcYoxQA3FFOopgc0vipWB+TkdqI/ErSSLldozzXNzxKqghTk88UkOEbk5zXmfWJ73O50Yo719Ys4o0d5AN3vVq2uobpN8Lhh7V51M6EYc8dqt6bqUlmpWJ9q+9axxXdGboX2PQMUtcuuvXAj3gBwBVvS/EKXQbzx5bZ4zW8a8JdTOVGSN2lqOK4im+44NS1qmmZNWEpcUYpaBCYoxS0UAGKKXFFACUUtGKADFFLRQAmKKWjFACYpaWigBKMUtGKAExRilxRQAUYpaKAExRS4opAJilxRijFACYoxS0uKAExRilxRigBKMUtGKAExRS4oxQAlLS0UAJiiloxQAlGKXFGKBCUUuKMUhiYopaWgBuKWjFFAgxSUtLQMSijFOVGbO0ZxQBZtb5LWUr5ZZSMdOlV5iGlYgYBOcVXmniFqxVj5+7kfjUqksik9SKxppczZvVTUUFFLRWxziUUtFADaWlooASkp1FACUUtJQAlLRRQAlGKdSUAFJinYooAbijFLiigBMUUuKMUAJRS0YoAbRinUUwG4opcUYoATFGKXFFACUYpcUYoAbRTqKAG0Yp2KTFACUUuKMUAJSYp2KKAG0YpaMUAJRS0lACYoxS4oxQAlFLijFADaMUuKMUwEoxS0UAJSU6kxQAlFLijFAxuKMUtFACUUtFAhuKKXFFAxuKKWjFMBuKKdSUANopaDQA2inUUAedPM7wK23II5qq5K9GFbUTwWimF0BHY1FNDZSDPT6V4akeq0Y8cD3D7d3NXLe0ZAWZN4HXFTmaztEIjjJJ/iq9Z3cMMJyPlbpmm5BGKKAnWNwFBAPVTSzxhjmJwvcipbkRTqTCwL56VS8vZkuDkURGy7bX81iAY2yTWvZ+I5WUiVASOmK5wOGGBT7ZS84w2MetaRqyhsZygpbneaffC9i3kbT6GrgrhJLy6tnxG2F9RVqDWLlVJMwzXVDFK3vHPLD66HZUVx8Hie6BIeMEA9RUs3iSfcNsfymtPrMDP2Ejq8ilrgn1u9SUuH4J6VpW/ih2Ta6fNQsTBjdCR1RZRwWAqnLqltDN5buAaxftZmO8s289BVLUbV5gJc4I61jLGK9kjWOFdrs6hdTtWziQGpo7uGVcrIMfWuMt4Ci5Eu7dQIp9xKOy7Tkc9aSxmuqB4Xsd0OaAK5m38QTptSWEgDgtW5a6hBc/dcZ+tdUasZbM5pU5R3LdGKQuijJYAfWlVlcZVgR7VpcmwUUuKWgQlFLRigBMUYpcUUAJRilooATFGKdRigBKKXFFIBMUYpaMUAJS0UUAFFFLQAlFLRigBKKXFFACYoxS4ooASinYooAbRTqKAG4paWigBKKXFGKBCVFO0qxExEhqmpCMqwHXFTLbUuHxIyYypikMwO7PGK04Rm3RuOlRQ6czWTyyA8n5RmpYYjHEoYEGsoPU6atnEfiilxRW5xiYopcUYoASilxRigBKKWigBKKWigBMUUuKKAG0uKWigBtFOooAbRTqSgBKMUuKMUAJiilooASilooASilpKAEopcUUAJijFLRQAmKKXFFACYopaKBjcUYpaMUCEoxS4oxTAbRinUlACUYpcUUANoxS4oxQAmKKWigBKTFOooAbiilxRQMSkxTqTFACYopcUUANoxTsUYoAbiilxRTAbRTqTFAhKTFOpKBiYpMU6koEJSU7FJimMSilxRQBwgjjuULMrbuxrLl3RMV5Aq9a6mRGYioyBxUE94JGGUGVrwlc9V2IraOS5JCoSPcVYSby22OmQvbFW9MuAA7PhQeAKt/Z4ixBC4fvSchpaFeG0SVPMU7WPTmq9zb3W4bVDr3p1xKLSQIC2R+tW1uo5rUlGKsOtF2tRtJ6EDafFFAJGP7wjpUEcIlJzhcVdRYZ4VLsSR71QeFC77JeAemaUZXBoZLIAdgbIFReZzyKa6YOetMwc8GtEzJ3uW0OyJiw+U0glDqMkACrlhZG6jKzZCDpmoZbaCO9EB+6ehpcybLs7EBCyjC1YESQwg5w3pVxNICSKyv8AL1pbjSpJJw4YFR2pOS2HykllcCaMKVAde5plzqTQ742jyCOtNXTXSQP5u3BzgVLdyABVMOSP4qmyuVqZCXDqflyB1wa2bWVruM/uyBjis2SJriQGMAHOMVuWsX2W3UEjp1pztYIkCuHiaOdQoHGazmmeBiIGPXqDWrPDHPGyk8HuKzYLHy5z+93R+9EHbUUkmFxe3ZRUeVlyOxostWubd9qyFh71Dcxs9ztU7h0FQ/Z3SbDLhvWt41HvcycF2OpbxIsUaZUsx61eOuW4jRieWrlTZyGEOvzGqzWl5I2NuRWixMjN0IncyataxrkuOmaW01W1vCRG4z71whS4iYhwcj1pbe4dG/dD589jVrFO+qIeHR6RRisvR79ri2Anwrr15q3NfQoh2uCw6CupVItXuc7pyTtYllmjhxvcDPqagfUrVJNjSjP1rC1NJbkrLuJbP3QaiTTGmDNIpJI4Oa5JY2K2OmOFfU6cXUDAESDB6c06SeKJC7OMD3rlrizlMKKGKbT2NK9pPcWhTzmJHvS+uq2w/quu5srr1k0mwPmrv2uEKDvGDXFQaW9tIXkb5R61ofZ5LjaY3OB6Gk8bYawqZtahq8NnDuzknoKqWWstcwNI2BjPFVJNOF0hjflwKn07SntE2uQVPrWUsbK1y44aJnv4smF35apld2K3U1iGWMKrASkdKrS6JZzN5hAVwc5FV5dPtIbhZHmAIGOtCxsgeGibi3kShVkkXeR61Y3LgEsAD3rCFhDNJ5okzkdc1e+zLLbeUZjj1BprHtboTwi6M0FZW6MD9KXFZVtp8ttOGjuiy91Y1rbXAyVz7iuuniqc+pzTw849AooJwMkGjcPWt1JPZmLi1uFFIHUmncetCdwasJijGKR3WNSSas2tvLeWrToo2L6nk0nJLcai3sV8UUtFUSJiilxRigBKMUuKMUAISFGT0FCXlswPDBgDk4rYisGitkl8tXbOcZqrfRPM0hFuF2jnbjmuWpVu7I6IU7ashsop5rRtmSvJAJpjxSiMMw4HBqeCRrKDMYKgjkNSAs9sW3FsnJpQbvcqS90qUYpaK6zlExRilxRQISjFLRQAlFLiigBKKXFFACUYpaMUAJRS4ooATHNFLRQAlFLRQAlJTqKAG4opaMUAJijFLijFACYpKdijFADaKdRQA3FFLiimAlFLijFADcUU6igY3FFLijFACUUuKKBCUmKWigBMUUuKKAEpMU6koASjFLRQAhFJTqKAG0YpcUYpjEopTwOTTGcBcg0roYuKMVALoCQIVIz3qwOaEwasJRS4opiG4opaKAEpMU6kxQAlJTsUmKAEpMU6jFMBuKTFOpKAExRS4ooA8ptcefuALetW5IhKhcRlceorPsb54rkEp8pPPFdE3kXO0I/J5wK8SV0z1I2aMyJmHHRRVia5B2gE5WpLixK8I/bpWbLmMEMOaSSYaodPqDyTKXQYH61aEjTEBV25HaslJUdvnPTpV+Dfw8b5A9+lKatsJMVZZ7a42SA4/pTLh8y7YwcVakaSWMFlHPGcVHDAfObJzjtWPOk7lsRU4DFskdq0LeK0OJXGB7+tNMcKxAlMH1qSOSB18s42nrThO40XCPNQmF8Rj0qL7NbJ/r2y56E9aliRYLdvLPFULqaJ1DMCX7VUXfYbNKS4jjhAT5iKlt5jJEWcbfSucaSROmSD71J9smWPZu4NU4iUja83G4vjHaqs14+SdgKAdabbOJrb96Dn3qS2kjSNopB34BoGY8twygsDtzWppt09zaODliO9PutLS7UFPlxTNPtRHuSJ+QeRVNpolJpkUT3kaSEjgdM1Sgum8wu547gVstaSsXi34LCsltKmhk2FhzQmgaY37f8A6RlR0ORVqOd7uR96k54GO1ZjK9pIQwBrasbgRRLIsfX71D8hJ3HwRXcaNGT8uOCetS2SSlSWc5U4xVn7dbFTubt0rNk1AC4HlHaAefep1K0Gai1yJc+UxX1FVEk2E4BVjXQy3C+SrOASR6Vm3NnGxE+/aD2qlITRVS5uIm4lPPvUfnyrISXPXPWnvDEzDbJlvSiWAry6ke9Vclocb2cMWEjVYt9YuIzgyZ+tZJlCZGc1VFwEYkn8KlxuF7G81/NcTEebjPAra0hpo1ZXYMD3riI7gFtykhs10OmXTyMoQtnH4VE46WKi9Torq3FzE0Z4zWfptnc2N2MPuhPHJqCbVZYJ/LkRiAOWAq7HdBrcTDgdeazs0i7ps13jSJy/c81XlufSmWly99bvMV4Xge9ULjUUhDEpgj1qIxbdi2yOa8uPtRjUEBhwcV3dv4S0288PRy3EQeUx53nrnFcNb6jHdD+EEV67pSLJocCE/KYwD+Vbw00MZHH+HvDEN3ZSHzXG0lR+FZ8NmyarLp0J3uHxk16Lp2nQ6dC6QH5WJPXNcdp0G3xzPJ/eY/yFEooIyZdTwXISJWu2D46DpVe7sbjS2VJMvGxwGA6V0Ou3s1o9t5TYDOAak1vD6QWbGcDmlKEWCk0c3Pbz2sHnSJ+665qos8FwPkIJ9BXUawP+JCRjPy1zHhiwj/tlpCMgr0NRZp2TKvdXaGGJc5pSmASCa3dTt4RqkSBQqt1xTNT0cRND9nJAkOD7VcZ1Y35WS4we6OcJMpwvUetLBcXVuGhR2ER6gGtHUNAubFPMjdpM9cDpWSPtKtufOD0BFW8VUXxEqjDoaEYaQ4GPxpJD5bEHH4VR+1lWAYgVP5iOmSa0WPl1RDwkejJvMFG4VHG8cgwDg+lObavANarMI9UZ/U30Y/I9aMjtUaMUY9CCKiuLnYVwvHtVLHwbtYTwkrbmt9tjSwKtJIr5wGHQVGk0SxPHJfM1wxG3J71lSyySweXGwIzkgVk+TILwPOjCP+8vY1LqQk9GWqcktTr7uzkaFo7mcfMuQV4qDSXUaZIm9SQSMk9a59tSnmdoy7SAZCluuKv6QsBkDTsRnoM96uK0B7F2inOFDtg/LnimhlboQTXZzI4uUTFFOopkjaKXFFACUYp1JQAmKMUtGKAEoxS4oxQAmKMUuKMUAJRS4ooATFGKWjFACYpKdiigBMUmKdRimA3FFLS0ANxRilooASilxRigBtGKWjFACYopaMUAJRS0UCEpKWigYlFLijFACUYpaKBDcUU6kxQAlJinUUwG4opaMUDEopcU13WNcscCgApGYKMmqR1a2Z3jV8so7VHJqCJZPJIcHBxms3UiluUoMilummkdVfbjtVSS9AdVMnK+lc2NZ3XEmDgEnmm2135d4zH5s8ivLni29EdKp2R2Us7tbxygfLkCtROUB9RXP2epC7s3idcMoyBV6LVoIraMyNg9K9GnVi481zCSZp0YqpFqMMzYjO41bBBFbKSexFgpKUnvSEgDJIAp3EFNZgvU0nnxE4DCq14kkjLtIVR3pOVloNIt0Vlz6itouzO5vWktdVaeYJ5Zx61HtY3sVyPc1MUUF1HU01pUUZLACtbkWFxRimxzJLnYwbFPxRcBuKKXFFAHlUdpIV3KuQatWUj21wrMvHoavWbRvGkcbfNjpUsmm+e24uAfavEcr6M9RRtqi1dXEUsSgRYY9xWTqUHlwZyMmtmzszCm2Rt3oaTULNXiOBnjis00maNXRwBys2CD161q20kcKbQTz1pbi3CSlcAkdQKntoIkQSyLkjtWlWqlG7MYxsydXaQoiZ2jmgQSrcFskKTyaignMuo5I2xjoK0JpW8z5UymOtcnMzTcS4ilZAgA244NVraCQuI+nNaFm4ltTvOWB4qfyBERMxwMU41LR0Ha+pUvbhbREhD5Ydazr+9UxIVTBHWpr3TXnJnSTd3NYskUzuy4OFropJNEtsvNc74lIrYsxbLCGmUZxmubTd8qDmtGOF3zhu3Q1biCZenvVdgsK4APFaUNlFIEmYZcCodNtI/KBYAt3zWsoX7qkDFZtlJEOTGD6elVFuYonxtAYnk1LI6rId5PWsm9QmfMRJFCVxt2NKaVfORw2DWNqN663qsD0460FZ29eO9VxYTXExBJPPX0q0ktWS23sak1mlyImZW+YdqsWls9tHJG65XtU1uJIrUJ99lFPgMxWR5+BjiocmVYw5FHmMynHPSpX0+S4gFwi59qp3F0v2lscc10WkzhrQkjgVTbSuSkmZUMs/yxyqRt7GpdSJNupByPQUtwGv710i+Rl6dqt/ZZobVYmj3k/eJo5uo7GRYxC7lADbSBWyikL5Ug3D1NRWVtFBOSVAY1oYVsnK8VM5X2CMbI5y+01ElaQOMHtXPzsyseM47ium1ewuJ/mhYY9K517eZW2sp61rSemrM5oiiYkkgdK6fQJ28vbt5J9Kx7XTJZHyAQDxXV2MdvpsK5ILetKq1aw6cX1Lr7I0LSqD9RWPrd3thSOJtinrj0qzPqccrcDcoPNIUsdYQquQV61hHR3Zq9dEW9LuvJtI1RsqRzS6hYrfR53BT7VTvUbTrAeUPlXuaNJvHuIyW5x3ot9pBfoy5p+kRRqV3AHHJrux4q02y0dLb7QvnKm0KDzmuMkDeSzL2FcbdidpzIpyd1Om227kzske8+F7mW50nzJXLtzya5LTtUX/hYc8D/ACqCQCe54rN0XUdVsLMIk48th0ZelYutX0lrcCdeZXOS3fNUpJuwmmlc9g1nTpdSa2aB1AjkDHPORTtd+XSChIDcCvN/C3jTUSzRzS+Yg6bhyK2L7UZtRu4mkmbYpDbBwCaJTUdAjFvU6nW1c6AQiszbRwoyawvChY6m6urBgv8AEMVoJ4nVIgkluWCjFSWviHT5p1by/KYcEkYqeaLadx2aVirrdwF8QwR552k10sO25gic84rPltdK1K5W6LqZQMBgauWEawW8iI+4KTg5q0tbkslV1neWPg44qtPpkMjQZjXCnniqWh3Dy6jdq2eGreLDBPpVWuK9jkp/D0GpaxKqfJFGADt9aW68GR+Uxtrhw47E5q/oF0Lm/wBSx/BLt/QUujyTHXNQV5S0YYbVJ+7xU8keo+ZnFWunXg1CWDYzSJwQKNTXU9PG97GQxjqy84rv7FE/tS8cAbgR/KnWt7HqU11bPEMRNtOe/FSqcd2U5voeewSm4jEgyMjoati1lCZMTkHviq2r3EWn63JZxRnGeAOgr0SJoYNLSWRBtC5PFZqndspysjgIoFgkdgpy3UGpgQseSuR3FdZqVna32mNcxALhdwYVwwvo5IyY3yQORUTpuLKjJMs7LaQZCgE1F9mSPAjJHOeayYJJmuXdz+56jFWH120dTH5m1umaalUj8LE+V7mqyTtEdpD/AEp+jlYpJ2ugBheA1YWl6gbYysJDIm716V0BEWoQZ4yRwa3+tVI6S1MvYRa0ES4ikPyup9s1J1rlZdF1Gzvd8UheItnrW7DNNGAJVrthjab3OSWFn0LtFRC4XPzAj3qUMrDIIrqjVhPZmEqco7oKKWirIEoxS0YoASloooATFLijFGKAExRiloxTATFFLiigBKKWigYlFLRQAlFLRQIbiilxR2oGJiimrKrDqB7GkeeKP7zqPxouFh9FJvXbncMU0Sxs20OC3oDRcLD6SlxVDU9RTT4d7DtSbSV2CTbsi5JIsSF2OAK5q/8AE4jlMcAzjvWNeeJLidHHSM9KxstIdynArza+MvpTOylh7ayOpXxVKpAZM+9X4vFNqEzN8rVw248bmHFQ3Dbm5P0rGGLqLd3NJUIPY9DXxTp5IBkxmrtpq1tefccV5HO7KVxnn0q7a3E9uQ0bnNbxxsuqMnhl0PXhg9DTSyg4yM1wOn+J54JAs77l7+1Xv7fBuXkDZUjpXTHF02rmTw8k7HYKyt0OaWuVg1bcpkSTAPbNa9jqYkhLykcd61jWjLREypSRp4orGfxDAZQkfzc80469Cs20jC+tP2sO4vZT7GtRiqUWqwSOFzjPSrwIIyDVqSexDi1uJisXXWICDeVU9a2XkRMbmAzXP6/E5iyHGPSs6rtB2Lpr3kYcDfY7xivzK56mtTV4hJpLOASxXgCqkdmrWpJbEgFW7IzXcawEAKBg5rzZ1FTi+bqdag5OyOJs0QKxZWzU/nCJNzDGO+a6i90ZbGN5mwyH0Fc/e2sMluWVTivL9pzM1cXFBY33ySFGIY8AVPJLgJuYnuRWHBMltJlclc8itaS4ilRXJA9BWnNKLstiEk9zUh1P7IQwHJqxY6/KJnLsCp6VgO5EY7g1CzCLnBxWsMVUT3F7GLR2UniGKaIgnbjvXNTeJLiSRo95CA8GsmS6GSO1Up5ONw4rd4upII0Yo3LnxBKJEMROF61OniO6kIDsdtcr9p6cVKt5yAeKmVWq+pahFHY22qxyjDj5vetNL0QqrAAfSuGglJlT5uD3raa72RrGnzEdSa1pVpJasmcF0NufXyYz8pVu1Y114gkltpEJOfUdqZdjzGRV3cjnFZcgAPl7CCTSqVql7XIjBPobmna5JDBHFHkj+JjXa2lwJ7VZSRyK81it3tp4/vCJjzxXTm9WxtBCpLbhkV04eu0nzGU6V3ZHU719RRXDHWpgcBjRW31yIvYMzo0mtyJFUj8K29OmjlizJkOTVBNRUxNC6g9gcVWeeXO2Pp2rzpNvc7lpsdNJJGqg7wMdqguLgPAxHpWDb3TfaAJssB2rcW2LIG5VG6CptbcpO5y0WWvnZ8n1zU11IUUIBhD3rRuNJXzPOjc+4qoLf/SAjglKxqNSmuxCi0SWEcUtvvYjelJcXhWRYSvyn0qxPEqAiNdoIqtEIxqMSyHJxzmsnJbFk2nttuNuCF61evrhXhO3otOQR24dzgg8fSmtBCIHO7KmiDT3DoQafcedESCM9MVnXAuEaUCMEEkDFXrOKOOQFfuk1o4X7SRsyO1awajK0RW0OS+zXKL5vlla2NJtlceZJnOatamkzL5aJle2KXSI5IQy3CBR1Ga6HUurCS1LiIEuSiHjFSNvReDg1my30sd6fLjyCcZrVEiSoMryRzUllVt5Y7lDA96qhZBKylcg961Y444wV7GlkSMY2HFNOwrGFMs8BI2krVX+0HCbMbDnrXTssLLywJ9KqPYwPN+9QbKOa4nF9ChZ6kY1IY7jitEbruz3BsbqmjsLRGyiAjpUk/l2tt8iYx2qW0UlZanFajaTWtwSykoT1rb029gt7VAwJLdakubmOZlDJkd8inwraowbysq3tVyldWZKjZ3RqQtatiVQob1qvqOpy2kf+q3KeNwpJ7RHg2xhlJ6YqrLcyJavbzKGIHBrNItszX1BpJdynk9qY091CxJ3AHnmqKtsu14wSelaur3qtbRx+XtcY5xWrVmkQndXIo9QdZBls54xV+5SIGFsDB6muY83dKu0100ds1zaRJv9M0pLlCLuWJ7UxW++AjBFYdzdsE2HOc8mti9ulW3+zxHLIOa5ieUh8nO70op6hN2NbS3D3Cgjj3rpI7eNNzxIFJ5OO9chp81xc3EarGMKecV094LsxKYCAR1FTNWdgjsN1INc2rQ889qz9LkeC4FsI229zitm3WKS33SyZkA5FWLeOIYYjH4VPNZWK5eoy5ytpIV7CuLF2Y7ndnOGyRXePEksbqOhHeuXbQVjuvOZiFLcCik1rcJxbtY6OyvRPaqwGBjpVW90+PVYwu4qQetWIYVjjAU8YqZApzsI+tRezuit9GQaR4fGlh5Wl3bvWtZGxyRTYZMpskIPpSGLBOD1rNybepokktCf7Sm7aaaWQtuBH0qo8RAYnioEmTn5x+dPl7E3L4lZXJRsH2NXrXVbu2UqHOD1zWQgU/NnFSkGSPKtketGqC1zodN1hLCZ5JsfPzn3rSs9eikS5Mp25OU9xXnl/p91c7fLnYY7ZrTSZ7SzUyjdtHOKtVJJEOCZ0fha4htr28WR8NcSeYufcf8A1q37PTYrO+ubwTE+dgkE8DFebQava3EhG7Zt79MVbfUnmtXdb+URAYI3f1q41f5kQ4djtNHu4rrVNS8pw21wpwe+BS6PazQajfvIpCyPlT6jFcFotwNJZ72zmBVhuZM8Mea3k8fPdQyR29m32hVyQxGK1Uk9iGmtzG8QXEEeu3bHBlUjA/Cu51EsfCkhXIYwnGPpXkURkv8AVXvb+UebIdxUdK9NtvF+mpZRJcKyqflyVJBP+RUxsmypXaRcsgw8IJkHd5Hf1xXmWlItvMfPIDt/Ca9Ev/EtpJamGzG8OMbh0WuEv9BMkvnRzNlumD0qKkk9CoRe5oERmM7EBB7CuY1TR40m8wOUDnpXRWNq9rF5cku89s9anntYriErKM1ip8r0NHG61OctLS4sIiwxJG1bdpdNGijIUnoKpX7G3hEccg+XsaoS6gs8SkjbKnTFXZz1EnynZLcKY/nZR9apxahFcTvCcBl6e9Yc8Mt/ZJskKPjnmls4msrfM3zH+93qPZqw29Tfmb90wABOOKo2csrqfMyjA9KqW+qRx3BikJwehNXTc28svlA4J6Gi0o7C0ZbVpFbht3tVlZQeDwazll8slZGwB0arKSKVBPPuK6KWMqU/i1RlUw0J7aFw9DimRliuWpFlGOT0p0UiyrlelerSrwqfCzz6lGUNx9JS0e1bGIlLRiloAbijFOxRQA3FGKdikoGJijFLijFAhKMUuKKBiUhwBUVzcx2sReQ4ArldT8WoY2S1HPrWc6sYL3mXCnKWxvXWrQ26vk8rXOyeNY43ZQu70rkp7+e5LEyEk9RVRYcNljlj2rhni5P4Tqjh0tzem1+8uboOj7F64FRXOqXMzAvIePQ1m+VIp5XA9aUgEDnpXM6tRvc3UIrobA1+7WNYy3y+tWNJ1G4bUw5YkHrk1gMhcDmr+nzJA4LNitKdSTkrsmcI22PTY5UaMMGzxXIeJpnnfYJBtFJL4jigt/KiG4461yt3eySu7OST6V0VsTGS5UYUqLTuMeQICpwcUxLtcEdKzkmkcthc89KRzKCHKYHpXByq51F+S4QOOarvcAvkn8KqPMpPPBqREQkNmnygaEBDpuIHtTvNw3GKrB9vyjGKbI+1tyng0rDHBnklJPAq2mCBhjnvVBPNklCxjdnsK2rXSppFDt8vsaTdgRBBO0eVyeTWiupyRw+WO9PNmsUZBUH3qi8WWypAqFUfRjUbl1ZliZZAOtSTXi5yRwRWc8hVQDzVSW4kdtuCvpRGcrWKsbUd786nJwprcttdKqVJxxxXHJIq4DNyatbi7KQTj1pOpUWiZLiuqNm71eaU7Rng8GqN5qFxPgyMdoqnNOy4UH8arG5LAq7dKHUqNWbCy7F6LUJA+dxrTs9SZXWMMF3d65sHfyMjFWPOVAGHUd6ymm1ZlJ21O4WQSQGGQ7lbvWZdaYrRPHGcZFZUGtNHHtPNWoNTJAJbIJrFRkitGY0nh27eQqAAnrWjaaIYosSgELWwJ96/K3WneaE4c8EVpzSa1J9mkYZsWLnB+QGsvVgLdcB811E0caRFi2M1yPiGHaNyZK06esrMmSsjGa4+YZNNe43HZ1BqiwYZbPFNWRl3EGu1RRjc0di7cg8+lJ95h6n0qtbrLO4RASx7VZkt7y1J3wtntxRYpO5aiHlNuLcdhVuG7JmUE8VlqLjAaWJwPXFPWRw4CJ1PWocRpno1mIZLVW2gnHWmvp0Bl80R5P0qDSmK28Yc8YrVRCZARIAlcvNJvc1SSM6W7tocxypz24qN2jkCsqFvw6VryWdpJMGdQaUCOMsioAn0q+aSVrjsc+dNgYlueeaK3fKg9qKnml3CyOVS0ikI2ygSdxTUiaKcb+RmnCylhl56Z61c+RdozuYVvUqLoZJXGJCPPG1BnrWw80vk7QuSOlZ8E8TSYZcNmr7yApuUjjqKy9pdWZaGFlaAbzhj2rHN5suimARuwKmnupD8iL8zHArOEa29wxk3Mw5xWTloD2N6URyQHBBfqKwLgldXh3DaWGKsJMWPnI/A7VUmIvJhMSd0XJp8ztqhM05FeIGItuXrSW0jOzKfu+9U1uW8xSzfK1Sm5Pm+WinmqgubSJLZo2sXl3DHHy461YuZ0huUJPDDtVKeV41QqrEEYPtU/wBme7MKgbSByaVPmlK0ivI1VQSIrCsPxBK1uUZHIz2rdgRoYhH1x3rP1SwW8ADLmuxPXUbWhmaRceY481wciuhSPcmYxx71TstOtbZQdo3DtT7m78oqFYBSfuis6k0nZBFWWpLJx17daiIV0Pzc0iOxXcRkGplCsAQlNNsZSeGberIeKnJlXg85pZ7uOHCsuCelOhkRmBboelX0ETQQsY8g4NUNUmcrsA4B61oE8kK1QNAXUpIM0k+4Paxg295FHLtmXI9a3rQ2t5/qiPl7Vj39gPMG0YHc1YsII4lIQkOauVrXJjdM6hbmOGIkxB8DjArjb26aW5lYAqM9K1UuLoP5Qwwz1p9zp0E7gEFXI5NRFKLuVJuSMiGGznxKw/eird3ZxXtqNpAYdKifRwjEQzjcOxqtFI8SyrK/K8DFaPXVErTcqw+HrqSXPAX1rp7XTvstkytJ82OprL07UpFkw+TEO9bLSxXcGQSox1qZuT0ZUVFHOxEC5IJ3NnBNbFvodvKm+RAWNc5Mj2185yQpOVNaVrq1xC685X3qmnbQlNX1N220uKAkogUnuKxtWe4spyqyEqecVbj18+cfM+72FYGr6hJdXhYH5e1TCMnLUqclbQ1NEtZbyRpDKVI7VuqpjcpuDEelYGgXG1njkcKWHBFXrZI7a9d3uy2f4SamafMEXoa6bSwUnGaWSEYIx09aiLJvRgw9jTri6AjZy4OBUWZelhjuvKsQvHNNggzbsqSYBPDZqjFe21xNjb8zcYq1dxSJZ4hJXHenYSZLJZS+fGy3DYXqM9a047hNrKw4AxmuY+13YQKrAuOuatJeymwyFzMe1Eqd0NTNSXLhk3YBHFcbI09teyRsxJ3cYNWLjUL63JMnAPT2qlEhvb5W8whj0q4R5dyJyT2Oo0eeeZdk0Z2+prdgREG0dPSsVJxYW6iRs4qpLrbrLmMfJ61m48z0Kvbc6icIiF/Ss1LyK5JRW9sGsxNdDDDDcD1rOa/jjvvNhHyDqKybUNx3Nm60mOWOQRjDkZ4rCjuDbrJaykgYxitQ+IoDgKCCe9PuGs5oDP5YLFetXCom7ClFPVGJHMbcbY5GCjoM1fttRcSgr8rMME+tc/NLskJwRUlpcBpkDPxmunl0uRdbGtfQXf2kFIyR2Irc0yGdrZEulBRTkA1YR1eyWRACQOKzIk1WS6VmIEWegrHmbViralzW2lihVbeQID1FJoV1cMhEzbgvAo1HTzfbRvKkVYtLb7LCFGDjvU6ctiupZWyb7X9oErEHkrVpnHHFEEwVMcbvSpCN6EkAMOcVkykjG1mwa6tzJAuXHNcqHMEoE0ZB7g1satrk8FyYYsrjrnvVaKCbWoCXUKw6HHWuiF0tdjKWr0HR3zMAkZ2qehqyZkERjlmJZulYwt57CYpMpAHT0NPnlR3V1PQdKvQV2ak1h58asp+Yc5pLuREWMMSpH8VUBq0iEBeBiql3qJmdVx2rPnK0N6e6WSFV34THXNWLC7a3tiufM7j6VykUxlkKMSF7Vp6PLI1y3PyDjmkmmg6nSWt810xAGMdRV4TrAmXYLWFJfxwybIQAx6mmXcLXSCSSQlQOgNNJxd1oNtNWZtjWIiMqN6jqR2qza6lb3eQjcjtXLaTNFGzxRnK9wa0IraGK9SaOTYO4zwa7Y4uUdJHLLDRlqjowR60FlB5IzWJdawkLHCnj0qpNrYAV88mt44uDWuhhLDSR0+RjOeKQOrDIIrjbjxFLJGVDbeOoqtY69NCp3NuB4p/XKd7E/V5WOxn1G3gieRnGF6iqFv4msJ85fbg45rkrqfzmYu5+bnGaxZjIkmUXOT2rGeMaeiNo4ZNanra3ULRiQONp6c0/zoz0YV5nDqk8aqHckAdKmXXLpXDAnb6VTxq7E/Vj0jIxnNMkmSJSxYcVxM3iiR7Tbkq1Zn/CQzNIFZiRTljY20FHDO+pp69qZu7zyBJtjHUCuYvF8qTavIx1qe6kEsxcN8zU2SIGIFjljXDKq56s7IwUdEZvyqMoTk063heSXeSSBVy3hidxG3yjPWtCC3S2LqSCpqObQdilLxDzzVFizuFxgVsTRxlwFYYqhNHtYhSM0IGRquxRg5p+5dnIOadAqnCt1q2tuuwt1Woa1FykChGj5HNMSz8yTAGSasvsEfUYpsM3kuXJ47UldDSKwsGiuCfLwD1qCZCrEBfzrZF+GI3AGq9yiSSB8YBq1LUbRy1zbyK28oeTUEvnDAUHFdZPFEUwMEn1qD7CC6kbdvetPaonlMOyinuCFG41p/YpA6xmNiO9dPplpbwxbjGPrTL93jf92o+aolUtsVyoitLGztESRRlu+amuLsD7vAqi0U2wnkmlWGRceaDg1i3fVhYleRmiyT8tUZDl8oeKt3TBY/LzxWYQVU7TQi1ZIcxOfelnjM0ahcKaYm/qOaWadYQrOPwqkrMehSmt5Vbljk1JDeyWyGNgT6VsRQwXCo+8ZPamXGnRiXcOlUpJ6Mlx7GLI80uCDjdTo4iFG5smtf8As9ZCEA+lV7rS3j4V8U+ZE8tirvkQYxkGmMW2hWB5qaOCSBTv+b3qTK7M4/Os21cRUR9hxkkVYiuVXgHp1puUAJxkmmCJBkjjNGgI07S8ZZQQ3y5rovNWWIPgEVxcMZ8xVDEAnrXW2PlpbbQ2WxU7MtBLc27lVZuBUMljDfWsicYPANRSWRkJOcE1k3UtzanylkZVJotfVA/Mpz+GJIpyFbcnUkUn/COeeVSFeT1NdXaQ/wChea0uSRyDVNtTjhkEdsuT0NVCcpPUlwQ2y8PwWASRyDIB1p8+17gLIFI9cVEbuafcOgHXNMd/OgyjfPnmtxJErRxM5WQqI+2ayJ7VVv1WIDYT1qy6yTKFfIA71eSziWJGYnFTJ6BYuWdlKsRO/PFWoizIUQnIp2nOirtzwRipnjWFm8vv3rl5b6l9CuPODe4qwkkjcFDVcbzn5stV62uRFFiQfNRbUCHMY7UVKbu1z9w0U+VBc566mYyEMfkzUVtkTKNylCfxqylsJbeQOpYZ4aqlpakynORjpWnIuXQzd7k4jIvpGzhccVZBkAbbg8c0Iu2IqwyQepqN5vLbYh2lupNck21KxSEEDm4gfp9ah1UBWccbyOorYiVHWPcwyBVS8tDNISANq55q9W0kNmHbsLa3LlSSaekDpavcblxJ2rZisIpLMeYQwPpWO0ZnmezgyqqfWtne2qJsU7YpNOI8nI5rYhUxuJCoIUYFYPlz212JgAuxtuPWttbyO4O1WAAHes23C1hXRqQI08Lebgd6iaR0ICk7V7+tRxzeRHvkYGMirW4XMSSQJuC9fekve0LtciSa5w3lEkn1qa3e6UkzkH2qe1tJpnaVsIMcAVYeMoNrLyOc10LRBZlVozK+VByKoy25aZkIINag1DyCA0YGT19ahvJY2n89cY4zUOTWw7KxLYIIY/KI3HuTUOowzIwlt5AT3Wqk+qqkmY8AYqe2lZUa5lbIboprSEnLUV1sQuWnZQ6ZepEjdZAjEA1ELqFZi55+lPeZW/0gAkCtRE+8pcBdwqaaVbdhIWB9qqWmoQTuysnzD1qO8ZJ33BsAcYNCXcGxtxqKu/yqKpJeFZTmormGRGDJznvUcylFXcpB/vetWkSbtndwyNjo1XZJfMOBgHHWuYt1eU/ugcgdqsWv2troJ85Hcmp5R3JLsukrAkhvUVhSSvFI2WJyec12k+mpNBtz+8AzmucuNLaVzEcBvWqhJCkmXNFniuFNuqAk0+/L205gZiM9AKl0nQ/sDLLJLh+2Kv3VhHdy+eWy44qZNc3kUk7HK6rdxsY0XqopLFmuCFIJ+ldQug2PlmWWMMx7mr9rpNrEF8uIAmj2sUrB7Nt3MWCzheIiSLay9Ce9Yd/atDMz7cqPSu4ubVdrBG6VhmGaKVt0W5CcZpwqdRTp9DlrKVxdgjIxV9klmuMoTuJrpzptvEqyPCoOOoFZ9zb7JMWsZaRj19KpzUtUTycqsNMVxYW4aVi5PTFMin80MrI5J6cVftjJBMq3RBBHU9qvGLdbF4VQgegrNzsaKNzm4xLZTGd4yf7tbuh38tzbS/a1ABJ2g1jy37mRoyoLA4wa1tPS1gKG4lwx7dAKJ2a1FHR6DbtT8zJGfrUVu/lw8qQ3pWu8lrKWVJFNNe3g2hicMBS5tCuXUxNSs57i08xW3e1Yunrc216JBCxX3rqGuljm8qKNmU9SRxWdfzXFtvZ1AU9OKpTaViHFbjr/AFMCzKFMuR1rBXUowrBjk+lUZr64knKk9epqlJEkkwCyZY1jy3d2DZt/anIzGuR7Uu9lXJbGetVYJRbgK3U0s04k+Q9KThqFyeWeFSADljVq31HygEcEg/lWStmsuSSQexqCWd4T5bg8d6PZdgudIJIbmFkZBz0NUYbN/t6ovCMcZqrpN0z38YJ/dg8rXaT2MTKk0CHf14raEnFNMTV9TZtIoo7NI42zgc04OY/l2kGq2nyqxKOQrjtSajfLa8DLNWNnc1voNvJJzgRDnPJqaJpfL+brWFHqk9yzQoAHY8e1aO6W3sirOd47mqcbbk3NWJehJ+b61aVw4+dhntXIwanI8mxnO71zViLUpUufLY59KlwKUkamoQ2buBOq5PeqtxDDFEgik2beRiqF9dzXJZZYiqg8NVIsxwWlyB0Bp6pBuzVv5RJaguFk461zMq7Rg5G48VPcXbAGMMaogSPnG5wPTtSbeyJaA5UkglsVDJPtk3Ec4pJbiO1OZpVQH+8cVSn17T0JDsJOOiDOaajJ9CdEaMN2u055NXbW8W2jyHwxPPt9T2/GuQXVZJpP3MSxJ2Lckf5+lWxby3RV5ZiwwMAnI+gUccc/55raNCxLmdEdVt5pWJkII5JAJ/z+GaX/AISAW6FQ7EEcIV/mfz4xn8eKx1tC9qpiG0nLZU9Tj6+3p3zzT4rNI4Q9wMJGQc8gj1BYZ9/b69Rtyoi7EuPE11CzfZ1UE5BJUZPvwKpQ+NtQhmAnSKePPK42kfTHSs7Vpy8hCxGNAeA2M/UgdD/ntWG8pRufxp6Bc9W0/wAS22rxGE/upAOFIySfbnnnjt1pbmF5QRDKsiDrt4I/A+9eY2V8beZW3cE9Qa7xR/aGnPNC5aQpuYY5Y9fz9/8AGocIvcfMyd0dEOc8cc/59jVeOQctngdqsw6lF9mK3E6um0A7jyAQOf5ev9KzVmtwSEnVt3Qf5/zxUSpdilIutdLItXYbdprUug7Vz7QsgBVs55Faun6hLBCUJ6+tYTXKO5ILVwFZx35Bq7NJbxgcDpSROLhctxiobizMqEIai7bKKN75bfPEeO9VIZEjuELfMDWn/ZL+QCrHnqKqnQ5BOGLHaORW0bWswFnC5LICO9QZk3A5P0q+IGVSxOSOMVGI42Vsn5u1SnbQAjRiOFz3zSzFygyansCc+UcnPWrF9Zwxw5V8HsKT3H0MvsAW59ab5B6k5NQSLscbTVuBGkjJzgVV7CSEhiCyE5zVtWfytoA2mqJ3E/JzjrT4ZmXCM2TSfcaY65hKFSMkntTGUlwjccVsQAyqP3fTuamSyj83zHALdqlSuFjIht2zkLVswM67pFwKdcF4pHI7dqy5r2dvvEhKV3fQd0hLhfnOG+lQGd4xtzn3qKSdcEl+fSoHuV28U2nYhm7b6o0cIVjn0rQtglx+/eQfQ1yUM/mHBPFbOnybvl2kipaa3KjqbUcql9uAeaW8nRp1twoyRxSWNhOsvmnHl9hV0RQNKW2DzR3qeZI0UTAn026DsSNw9azbmJolJz9RWxd64Yrt4iMKOCa569vlluCFOVPNVBSbCXLYlt7gKp6EHvViG0S4l3NIG9q5ttQYs0Ma4yeta2mpKkYndzjPStZRa1M07mzFbKkoIGAKshBJISxyoqss7FMgdaDdNDj5c57VCV0Vctb/AChuXGRWZcXLyuSxzzU80++EuBj2qpEyuuZPlzRy23JeuxEZiG+Y5FRmRWBycCmXCrvO1s0za2wDIp6WFcXkHI6VKrBlwetRRxkZBPNSZG7GOlJq4IWLzMkDpWvpV44kKMufesxZyDwg2jvU6XW1g64XPUVLRSsa092EbAPOajmjjulDMMsOaou4kbcCcd6nWVkUEdKaTKsXLJC6ujNgYwBSQabHGzsGBb1qib5kbjioG1CRfuMeTzUq6E2jSOnkucS8HrViy0+KFWLHI96p2t8DGA/U1fgXzG5b5DVczFZE5MDtsCDHTOKtpDD5O1wABVNkWBC4HApI7nzScn8KV2Fi1FawRtlH/Ckug3y7TxUAmCOMrgVphBLb5UcH1pBYz4IGJDg96uNNbBgHxuFRyMLaPywOT3qiAjSbiDnNFrAafm2/9wflRVXC0UWGUkSdbUmOQYznBFVzLtQuD+87in3GsQmPykUbiOcVDEgFuWPO6nGErXIdnsHmySjIOBT4zHLcIkiFgTjNQykRws8Z+g9aiSdlKs5w56D0rlnq7itY050FrK6x5OFyMVHNJIul7kJZ261Te6d84OWzzmp4LoxpsKbsnNKD5X6j0GWaXgtWRTgDpmq1sjxPLKzZfpWneXxeFvJTyyFxj1rnprx4I1JRsk1vzuSshSRbkkglQRuMynpVGOA2+oiGTOCM0ktvcS+XNEh4OcmtiUKYkmZAZtuKVS0VYVr7jpWQ23l/wj9a0bO/hS1AQdsECse23TxMHI8wc7aZbM4VyUIbuKzhGSKTsdFZ3ztlYxkZ6ntVyVkuQzvLt2jp61z2nXflo4B+YnpUrfaJFIzhT1rWzZVxt1chhgcnopqkxneFgXwatw8z+WV3FeQ1SyxKiSbx1q6UnblJasZcduY9pkbJ61PI+YsGRtvpVX94s+GOF7Vp2FosshMmcjoDWqffQmz6FZUG3O8Ae9W7YsrCLepQnmkltrea48rOXz0WnNpnlSCIbkY9CavXuCTQX/lx3K+UAOOSKp3M4ZfL6H1q3cabPHGXZwzLUSD7S6LJHggVaaB3Es50ijKzjfxxU015BPAqBBjPJqGaz28pz6iquzy2xjj0o31Hqiz9rSyuk8rBGORW8LmE24mK7Cec1iR2UFyGkdwpA4FLPM62qxBtw6ZqWtRq50ME32iQbMYI6k1UvLdBJvfKkHqKwYrya3kUiQ8dga1YbyTUYnjZSoAyGNJpphdM0VS2u4o3WU5Q9M1bKQxFiDjIrmbO5FqXSRsHsa37MwpbhpJfNDjOTUSTRpFj5IxLb5EnuMUlsLyaRJM7Y17etJ/aVrEGVcALWPP4gneYpb8JmpSk9BtpdTWnvUM8kKA+bU4WNoAjttJGSTWHdq6QC4Vv3rdajV7xrYtPnaR1rRQuiHPU6FWFzG0UJDhepqtBdww3DR7QWQfNWVo1w9q0iRt971pkzsZmk3KrscVKg1oVz31JJbo3V0crtRm71pLcC0RY1Thu5rJRVWYeZ8wAzxVW5vJZbgLuIUHgVbS2M03uaWo6da/aoZg21s7m561Q1C1eWVWjkG08U+6dbizAZiJB71lGZ8Ebzx71PM0D1JS7wSFQ2CO4NXF1Z0iO+TJxisd1JXO/k96iuEKwfeyaFdsWxuw6q6EHIYZ5rTmurTUrRo2wpx1NcrZneqpjHrVyVQqhVODUO40yo9rFE7jgj1qKVbWOzZUi/eD+LvUN2ZQ3AJxTreJ5IWMp61STetxGaCZZF/eH6VbB2nGM4Gc0stj5boYlLeppDBOGO4bVx371bd2hWFW83Dg4APSrpgiuApcZxWG8EqyK6fcB5rZSZVjHUEilLXYqLXUtQ2dvEfMi4YVqWmr3EMYjABGe9YkU4BOCasLODy1Ll7lXRt3t3JIROvyMPTvVEam7TN5zdqoS3MsiiNDnB5zWhb6PJPhjk5HJpa9Q9CtDqHk3JdVHB4NXrrVJbmI5/SoX0N97gHFRNALZSpJYim/ImzK8Uv7wtzuq9bXDi7DYyRWQGdpSVHGa2LGdIiSy5OOtTKVtxxRrXF1vA3gfSsTU5kSb5Tt46VLLcljubGD0qhdmIAyTMqjH3nPH5dT+FVDV2Q35lNrkO24sFUdWJ4FY2ta8PINrZSOiZy5z948+lZ2q6n5sjxw8RZOOOtYj56nvXRCCRhKVyU3cjn58GnwEOwHeqgBYgdzU1qD9rjUdd1aEnW6XYqiiV9pbsAD8noT6/TB9MZzXRCO3SHZK4j+TH3znJPAI6egOB/SsGbU7fT4IoSWcqMsAerYHBP8Ah0+tY02rS3LtMzYbjaB0GPpQB3EeSrujBi6/dAAXOe5GOPftgVWleGSIySTvhDne5yMcd+ffr6DkZxXH2l/fyyERPkZ3MSpI5wP/ANQ6VcbUvMcLdsuFztfy8gHuSOc/r06UmCH3Wo23mMY7dGUjCKvJbHH9Of8A69YV3ezXLFtsSog+4kYAAz6kZP8A9f04q/eQuWFxiOVOzKQOPwwR+NRosFwhWSGcFj2dQPx+XJ/PtUlEcNnHcxebFtBAyVBziug8O6nBBIqyNhlG0D+97f5/pWefskMUYhstpVgWZXwzL3XPPJHQ4PNQQRr9oWYKVw3Y9PpTA7bUdKtr8GWFwHmG/J/i9x+f4/jXGarpF9pTeY8b+UGK7xyucev0zXSJrUcYCYeR8YG35eeO5609dQnkR1kjVkfrGVDA/UHP9KLhYz9M1Brq1jZ1wwGCT/FWkrBsg5qCO2soLrdFZQIxOOJWGfwzgVZaS0jKg+anc4wwH54rGcG3dFp2Rfs45uNoLCtJ0dgAq49aXTNQ08QlYrmF2xx8wBP4GrBuJVYgxjnnpXNJWepejFRWKBNuAO9NvIWAHlnPHNWGvVMW0gA0Q7pASB8p9aa7jaOeuiyIc8MetZ/PrW9qNk7ocY31lR2LGQBxwaaegrF7TECyhjjpRq8eDletTwWzQEAHIqwYVn+Y447GpvqM53yFRg78g1ZRo40OOc07U2TcLdEwAck1WHlSOqq+1VocmIkFuxt3AG1j3FMt9PmMisSdq8kmr3mDYMHipWnLERxnjvS5yrIs27Heqqvy+oqaeCRTyrZPTFPsIwqHIJwOvvT01dBcLGwHXHNT6F2M1rOZ8k5yfWqZ01pY2jmyvPUV1pmj8ws4AXHFVVaJ3IIG09KabWomjz+80rZIyxktjiqUdvJDlXU5969Jm023EDEqNx71i3tnE+cYyBWsa10RKnY5yzgjBDMMGuh0/wAuOULgHNYskbmXA6D0rT0udYbgGbnFTPVChozpZrtLSBUTJY9qZBGJYmnkOw+tSW8SahcB1Q4A796u3OkNNFsBwp6iue6Wh0Wb1PP9faBZCsZyx6kVzTrIHAwa7rVfD7xNwvy1imGKAlJY+TwGrqhUVtDGUHfUzbPRXlPnHjnvXWQWsTWHlkAECrGkWCXEah0Kxr39au3ptbVGCgEjtWU6jk7FxhZGLbeWshQ9qWYxM+Tjjisi9vitwREmCT2qXzgIw7kgmrtbUzbWxfukiW3OzGRWSMSgYPPpUd1dlUOG4IqjbTS7tw5WqUW1chs0HiVBk1Eq/NycCkW8xORIAc1Vu7hQSYzg01FvQTZeQp52Cfxq5BCjylG+ZW6H0rnYbg5ZmfmtbTLzy5FkYZQ9aHFxCLNO50owjCOSCKyLjNvIFY8V2Vrcx3CE4GMVQ1LR47uNpIVG+pi9dTRrsc4LzChUbJNSrcuFILcUtvozW0xaYcValgjUcJitG1shWZQM/Gd1SptkTJPzelRz2hU7o+R6UQo2cHhjSVidTRjyI92M1owNJ5QfO1RWSJmEe30rorNIrrTUB+Vqya1KRVlvJCuMZUVn/biJcjrVyaMrK0ZOVPAxUTWKEIFH1NJAbNl/pVsCw+atESFYAidRUNk0NvFHGFzkcmrDuin5BQlqUVNs0ikMvzClby1jAOMjrTxMwdielZ0+WkJ5xmmIu7k9qKo7x70UAYf2WNOZW57YqYTyW8IbYfLJwCapAySygLyWrXaX90lsU3IvWqd7e8ZpIgE0ZhMjNjHaohNDOnmYIepLlYxC2IgC3SqTuEkjLACPviufk7DTL9jaCaOSTPU4+lW9ohkEaxZx/FWbC6QmQwytsYZAqSK6dtiHdlj1NZuD1YNoffySOypFgMOpqe3gilCedhm9KqXEQWTCSHcevNNtrW63t84C/wAPvSgrrcEzUeRfIYgBVU4wKp3MmRCQCq9TmnQwCKF1ll3Sk5C5qG6nNzMEK4CjBFVGKctBtlKeT7PqAaCJmZh17Vot9pe3xhVz1NRq8SgbQCV6cU5g8kbyCT5V6rVNvoC2LVraKtr5jj5lHatBYmnlgRG2qR8xNZ9lfhYGXAbjinHUHFshcYPTinTblK7HoS3SeRfGKPBXruFI10iv5bDJrMlmfzNysct0pHWUncfmfrXSopLTqS22aUQWZ2maLKp0FX4reOazkuPM2Sdl9Kr2KzZVioCMBnNNvrR5Z8oW298cCm2pKzLSshNKmtrW8LsA8pPBPNaupzNJF5zFRjpg1zl08XCouCvcUxbh3+WR2IHQE07a3DmsrE/mSvkGU7j60+GcxYLYLDvWeZyJORwKswvDLGXcnirexBNPc7ZDJnr1qETJMGbjJrNNx5k7IeF7ZpsMreYV/h9RUaod0X9gJOXI/GleOWNEIy69zUJk2jnk1oWFzHGp835lPQGnzu4aFa3S3WdTJznrW080MRCqAFI7U7TotOlk8yRFBPTNUr6BI7pyj/J2FDmgtZBe+RLbMyrhh0qpDcOIdgzgUhnhQBS2TUQkO5gCNpojNdRWLAVXXLHLelRkxo2c4xVGQyISQSSaiFw+3DDJpTk+jCxus4lCksSBWkLqK6gWAfdA5rn7C5DShZchDWi80URxF9096nnk0PYdNFHGCYzjFZ5eNpOclqmFx5jlB3qpPZTwuWPINEZyb1FuTNqkdqSAu4mqE1yWYyqMFqRolZiTw3vVdyinaWwaG7sbViZLo7SHJyaejxSHYevrVaO2eRtyHinx27Cf371TihK5bFj5kgCuCuM0hsmaUJ0X1pIRJFIwBOKtF2YDnpU3aHYhMckCkhPlHcCqX2h/Ow4PtXRWkc11CYodvPXNZt1pUkE2Z2XPtRGa6jcdLldJA2WdfzqOWQAADgMaleHMbEN04FZ0lvdvKhVSVB6VcXcm5rW0bytsTHNMvbC5DAfeHtV+ys5dqOvBxyO9aaxxnCSNt9azT1LVjkPKEbBHp7ozxlEXp3q9qFg/2gyKQUB4NVyzR4U9atSsTYpmKVV6Y96sw2kskRfB6VbVRLGMjpVmS6+yxCNVHSh1LhYpWWmzswbacHvXUQy3EUIRFHyjk1kx6oyQgIoq/azM9uzM3J7VLbY9OhVutRkyxJwfaqdxIxRH6k9a0F2lwGjGCaukWqou9FDGou0wOZRV3/dPNV55jHLsz17Vs3wKtmLG0dxWUIBJMWdcnsatWe4+hALtIIZLicExRckE9T2H+e2a5O9vrnUZ2aWTrwE5x9MVo69ekhYkZjDHliFbGWzj8OB+XSqF1bw2unoXlCzEZZV6L7D/AD611Qiooxk7mZLEzcrlj25684/nVE8mpPOcFgGPPB9cU8I2do49fWrIEtojJLkfwDcfzqW3dEvRKAdiZbk9eKjHyZAPJ9O9AB3BOnr/AJ9aAHSyNI5YkkmgDOAccdKQ4DYGceuOam2/KD3piNjTIzJbMpb5VyTuJxn07Y/DP0p8kDyKYjHl/wCFWQ/MfUYBGPUdeR0qpbT+Wgj+VSxGDgFs98HgfqKstdgFnaVWjOAcMzDr/dwAfbr1/JDMsvcWcpXeykAjGOBnqMVpWl6txKBcRx5c4HbaPc+/T/8AVTnEepbRk7h0Tj5eevHHPHqaljs7fTys0zhDnoHGSMe2ev8AOgC/LpgCNIOEwOT6gc4/z6+mTWjtxAdzAE57c4HvU41tpv8AUphR1brtAxwPTjjj3q39sjKqXTa5U52HGB36/j+WPYooit5IRmRgN+MKSBz+nbp9TV1gYtztGABwMNz+nPP+eKy21W2t5S9vAm4cB2OfpjqPb8qsretcBdxV2OeG4OT1zjnP09D07oZl3l1PNfbRlAMbQpOce9XLaLzopJ5pEkSFlTaHB27s4LDrztPP06ZGUuUZwt4gbynfaZUAGXABwuO+MfnVW9Z7mKMRlIWSPYgbA3nIJy2BxkkjPC9M85LQG0moaTbSLNfuRbEgGOJQHxjOQOPy47896oP4ojhut+mSOsP91zkH8P61y9xYapLI3mWsxCZy4XKAA4zu6Yz3ziqhBibZkEjqR603qhXPWtP1GPV7H7TDgMp2yLn7p/wrSsrh1JQnOK4fwYtxFp9xK5IjnkXZnvtByf1/SulEzROSDXBP3ZtI6IK8bmrcysIy6jJqS2SN4UZ1G7rWUt3iP5hkVqW0yTxrhCOO1J66jS1ILx1ticNkms5bqRMjd1qzqiLEm9jWEJjK5ycYqLNvQT3JbgvLMXY4qrDC7yNtOQDyaWS6RUKOee1RR3SQREI/zNzVq9rEuxqqzRQuDz6Cm2tyDdR+YNq5/OqC3b7gSd39a3bSxs5ofNumCvjIGamVNxWpcfeOnkCNpo+zDBIrljaXUcjGeAgA53VtWuoRRERMw2r0Oa0buaB7AkuCW6VnFuOhrKKaucqJp3bO87R2NbdvJGbQMcFqyn8mNCC3zHsKZbTPCxA6H1rd6mK0Zfu76RYWRATxVTT7drp/3pJyOeKsQRNjzZCME9DWtaWrPloFU5FQ9EUtWcpqVqLNx5XIJxU+jWEz3DF4yRjjitS6geOVY7uAbd2Q1bWm7BGwUj2pTl7ug4x1H6dBLBEd6hQOlGoaj9nt9wOc1NdSCG2Y+ZziuUuZ/NXaWPBzWcI82pcpWJvt9xPLtkXMbHrVa+srYT+YxBxyPrV20ZZFCryFqhrIWWP9ycYPNaONtjPm0LFvqUJQIGCsKLu3a+cCJOW4JrmA4ikyc7q7bQJXmgH7rA9TUSXLqVBuWjMy18KNHdGSY70xmuT8QSLDeOqsAAcbRXr0kLC3LFwBjmvIPEkNs+tyeVJuGfmOe9XSk5SuxVYpR0MyFjKMMTSPkECM4HQ1oJEiBAmDQzwMxXADVvz6nKYbGYTZGSB3qyNjYJ6jrmrjbTJt2gA1BcWaMd6tuI6gGr57iKYdDIQMYNatopAVQMqajhsbdhvbC7e1acBiEfy49qUnc0hT6ss28z2/Ck4qxHqUyAmqO8j5j0zTXmG47TxWfKbaI0Zb4zpytQH5uD0ql57DGBxUnnquCW607W0Juicx+WRnnNRZiMnC8ipRKGlG7BAFNklUEnAFDJZCXMl2ihcL3rceTy7cR9B2xWKnEisozzmtJ51mQcDik9dBxEPK+ZvJx61ZhBdPvYzVbaioh6g9RVp5YwFKYHFLlGzXtGEcI3DJApksjtKCBgVnR6ikAxI2RRNq8LRZB5oSJNB5kWPLGs6XVYUVgoyaybnU3nG1ehrOkd1JP407Bc2DqLE5wKKwfPeijlJubVpGizZ3AkdBWjabnmclMqKx0VGkk2SDKHHFX7W4mR9i5KnqaxnNvYF2LNwoYsSowKzp4IxFu5xVyR1RzGxPz8k1A5j8gwb9vf61hzNWG13M+OBmHmtJiJewq1Fcw7lwfm7ZpzQb4VSMggdagKwQQxSbCz55xVq0r6kofM4M6hDvYnk5pXkeMMzHc69PaoIZohOf4SeVFPniWIvKSxZhlh2pqy0YgguEaZpG3ZPVjU3mTS7pYCu3ODmstZQZOOEI6VpwQtDZZB+VuTWijG92JMjuoZRJE4IAPU00zowMILbG+8RVe9uSGiTkoPQU6G8SQNFswxHFQ432GpdiawlAkMPr0J71qOjuyqQMCsaGy2tvLnevStGG4+Vi7HrgVMtNEWmr6kE0brMSx6dAK1bBoBkuctjNYr3GLxgfmHatKNAY2ZeGZafO4qzKVuhsWEqyRO0zDys8CsvU9eAZrO3XaAfve1UUadPlGcDsKSKFHkZpB8xHU10U5R6icm9hytuZST16024lXzAAaZ5DKhdicVTmhl3ggkg9K2TVyCWSR3APYdasQsGQRqeWqrscrt/iotoZlugT09abVwHT2bJIQpy1OWCSMDAzWiqIgJY7mNULq8dDhRxU+8NjZd5YDaasxxFkGWxVZLsTJtAG+po1I5ZuazlJok0rSO2BBmkYYPTNR6lPA74tmOKqM+3qaQQA/vN2ajn6s0jqrEKxYILZIqbdtHA4ppVs9cAU7cEx82c0OVwTsSMVwGPftRGkO7LcioWBdeopMBM8nNCKcrluUwgZTqahG6XAB6VXYsF9QKSCWUZbpzwK0TdjNs1o2WIjCA+pqSW4kKkFeO1Z8dy67iVJBNS/a/VeKlp7gQyWztJuPeoW0tpXwT83rWjBIJye1XBtVQRjdVajSMyHTbi3AXOfapJIfs+4spBPetOWQxQbxjd3zWfczm4VSSMUNXLM9pQCNrc96RnJkXDYHU1TuM+adhp6bmjyRntS5STTt7yWBi0bYqKa+ad2LsSR6mq8QdV5/I0YMpLAAEdqlRQakyYkHJwO9X4nURhkxn0rLiVhIQ3HFXrSVEGCc5NWk0C0NKyujBOS6g5qW6m8/cwUDNU4yrzZ3Ac1YuVJG2MZOM8USdnoNGewfaVY1QnZQw3DmpbiRxOM53dKjmQsAWoa6g3dE0Uw24Uc9qLlPNjCgnzDVSLdvyp4FTxsTIeefekkLoWtPhAt2WUgEHvWhG8cSY8wEVnC1LLkZJPXmqly5jkSJASzdqVriRqz3qlAo65qpcXRkIQsQVplraszZdvwqWe2iYZLfOOlF+g7kSXBGdzEg9qq3N+0VtOyD59p247GnC3kZmywNVL+2eGyZ2xydoLdM1pBJuwmzjdWmy4jHUcHGMYx6/karz39zI7kysQ46Z9adfgbtxzvbHfoKpGuwwLVpZTXhZ0IAGfmPc4zx/P2FJLKIyYl+90ZverWlPezXcUNnCZpAM7cA8A9z2Hv71DqemyaZciCWRHfGSV6frSur2ArpJtzjBbtUirhfc9aijGDk1ZhVWOWIH1BpiJI4SAZSCdo644FRNPmXKjODnkVLKwlQooOOgP+c/zqI20nl7wpI7ELxSAs7xKud21sYJOe/Xv/AJzTijsP3j5UYAyOfp7f5+tZisUfOeR0qwsjso3Fj6D/AA9BQMvRMsQVEXYG6yPzn6Dp/Orz6daTHe+oGZ8jPT0xj/J9MdayHaJVzLKS3ZF6fj61CLti4KKqYHAHH+f8/SgZ0uy2sxHIsfyjkEt93vnjGfy/pVY3Sv8ALuBGc8k5P4d/x+lUYb+aUeW0hUHpuOQfTihZFtjudMt/BhuM0hmjNbRIivvBcfw7ckcnuP5+mfap9nk2qq0D7WTL7hjqAR78cnPH5Vjf2leGSQrPh2wSec8d+nt+ntVdr252YNwADyOc49+Oh4FArm8WWNnV8q7JgS9cgHP164z+GeorOvZPl3AjK8DDVkGVgNqscnlvc0+SZpMgnIoaC4153I2bzt9BTEGGByMikZT1wcGrWm2hu9Qhh6Atlj6KOTQ9ELdnc2UTW+lWsLN8yoM+2ecfrV6CRghJORnGapQObmdgPuipGfypfIGSCfyrldrnQrpGrHFE8W4tz6VuF4rSwjVP9Y1YA810ComMfxV2unaak9pE9xtYha46jaehvFXOT1W4QW7CZckjrXEzTTTT7IiQoPWvYtQ8OWF3Fhm57AGuQuvCrWF+rxEGIcmtaNSKInBnJTWMuFAySR1qEWzwybHBPvXa3UcXygbQayzFG7NvAIHSumE79DFxMRI3LEpnjpWvb+awVWJ461JbxRLOFwMVpywIhyCNpXt2qZy6FQRnsQrEhs1NHcMQoLnGemazZ22NtRs88k0sLM0mA2fap5Ey+do3JI4yoL4x1JqylrDdqpik5WueN/tkaFs/StTRJ/8ASsDgEc1MouI00zZltsxCJX+bFaujQPaW2Glyaxr4PDKjKckmlS6nhJfdkelZSTa3KTSNzUIvPgbJDY5rn9JupotRkib7hOAKsQahI8rb8hcc1Tt4nOqbkbg859KajaLuHNd6GvrspgtVbPB7Vkxq91GsqR8Y596TxFvDxLvLAnoKkguhY2y7yFyOAaqKShcmT11L1jp0jfPFhEIwQaz9Rs1tSQTkse1Qf21cvuWI4XPap4b8KpkuBv2+tTdsSsUjoYnuopXbbGOo9a7e0S3trVFXGMVxzask0vHyr6VuLqsQsgyLuIHWoqalwaRqXvmm2bZyCOlecXvhXUJppZiqKpJOc1sXPiqfzxEnKg811FswurFW5yw701zUtR6TPHb3zrKTycEMvU1A85dM4/eY6iun8R6Pdy3xcRblPQKKwG0+6jJAt5Bg4+7XTFpq5zShZmY9y2MkncO1W7a9G0gLyRzW1p3g++v3DyReUh7sOauz+DPsLrmTcCee1Pnhew/ZStc5je0wKqpz3Oa0rO0nS0MjkYB9a17bRbZbrEh2p60y/lhjVoIeinH1olUT0RXK0rsgEytaMMCsxHdXJIPParcZG8BTx/EKGt9z8Hg96EydXqNUvJhVGSatw2QBDTodo6VJZRJazbmGR2rQklWVCO57UnLsWo21ZTlhAxJGowB0rPD+Y7hhj0rTuLebygIyeayxbTjO/kg01FNainuT2UZk3RuduO9PbfECAfxqKR1tyqrklhzSNP5nytwBS6grDjdMihM5qQyEKMGqwA69amA3HhSO1DFqyOR2Z+uc0wnC7as/Z+N3fvVa8jaGHz8HHShK5LTGNIsYz3qCeeQ4Xbx2NOjVnXcw4IzUiAEZbtVWsK5WxJRV0SJjpRS5l2AvtHFFL56cL396vw3CM20dD6VBDC8kUf3do55p81rHaTC4WXg/w1xyjG9pMaG3jhJSmPmK/LUEUf2lFJOJB1FS3ZR5VkyScc1BFcmGeQBMjHFQ1p7om9RryS27bc8H0qu4cjIJKH1NaAaKZC6p8+P4qozLMyYCjPcCrptWF0JIZLVHUyR73P3faprqRREZJBlScYrOsow10vnEgLVq/QMpWI5j6im4JSGloRJ5LptWMhuxqWWSQ2+wkgCqsDfZjhwxLdPar0MkMzgztsQDrnrT62EjJeSR51jQfpTfniYswOQeDUkEqW+tO+d0eDirLMJUYlQWPIrWStsJLqMF6GcFSRxg1dh/0mFivCr196qRRxSICy7SDzU8LxQSsASVI6Vi9ykPe0LbXiGcck1Lb3Mgl+YEg8YrPl1KSH5I14PGKlimfy1c5BHOaHF9QV0y/JHJGwliyV71IJEmO1htY96htmu3TORg9qcqPscyLjHQ0+VoZMMqpRzlB3qm7ZbPp2q7bSK6NjGcdDVaWMzKCihT3qoTa0Y+hXDbsnGDUqMwjyTTI4yAQT0qVApO0DJ71rz6DQqOSME02S3+0gKnXvSvJGCVHGPWkt7jad61Sm2PlQ1LQQvlhhu1ROWWQktx6VenJnKlR71GLdXYM3QdRRa7JcCp9oLLsxj0NPtHdpAjH5QamlSJW3Lg47VDHIocsEwRTcVbQErGyYYgDwDkViXbBbghVPy1fhm8xO4PpUcqNIWCp+NQotblOzK0TFwe2KQsXJGRjpS+S6RuM5akg06aC2NxITg9BSdl1IY4syAL1B71KuET3NRFlCgYOT61Tnu/JcAtye1VGNwNMykjAGR3p7IHiyB1qvbTjaC33WrSMcc0YRDgAZzRKajoCM+NzG+BxipmumMqlD93rSeQfPLcEYptvGTclnQgelZykhl27uEaMFmOCMcVRCxopJbr0FWZFTB29M9DVWRV4yKqE00O5lys7uRtIHqKu2xIj2gZ460FRuKjpSBuvbFU5CvYjkeRHKnNSRsdm4k1NAiTPvJ4p80SZwD8pqUykna4xR8hKnLEVDHG6nk1JJiJG2HoKpfaTxzzmi7JbL8bsCfmPNaNreGI4kbOe9Y5kby8qKVXkXbvHWjlbGmX70RG5LxtkN2qncSmMAk59BUckhE/I4x0oP7wlWOOOCapeYXCO4MgOwY/CrPlYIb1rPt2EGRuGc81qxYlAycinJWehS13NLR4jIr85NXY7JGuCzwjI6HFULJjAMxPhgelasN1NK2WUAetZu4bGdqFjIzlkGzH61lG2vZmUbcDua6i6lKMGbBBpFuIkjOQvI/KmkTZGRb2cYG04B781n+LHjt9JiTHBk5PsFPNaEyLJdMYXw3fmuP8cahIq2lpvy43O306D+taU4tzJk7I4+8YvcsTwe/1q74e0tdX1mG2kbbCPnlP+yOo/HgfjVlPCeptgyiOLnkO2SPriut8PaLFpkTYYPIeZJMYz7D2redRJaEwptvU6WODTdNtJDZwQwoF3OI0C5x6nqe9eQ6ve/b9TmnwAu4hQPSuz8SasbezkjibAkIXI7f5wfyrgNuBUUIvdlVmtkKuM88j2pxbA469vaod2OlGSxrcwJllOOefar8TNcoFAJAzkgbR9PQcc/h9KywrM21QSfYVsWTrBaSKZASRkkKOMYI56n+QznnqrAoXEKLgxZbnGcYB+g/X/OBC0pUFV79T60sr/vXAPy54PPSkKoMfMWbGT6D/ADxSGRhWY5qdInAyuG9dvJqI53LkZU9h3rqtIttPl09zK8ajOZFbrUTk4lRjc5hj2I6fpSGV9u0nOfXmt3VLObTdk8sCtaTuwi3ZDAD19PbPpVaOGyuiMBoi3HzdM/WhSK5L7Mx+tG3v2rZl0GVSdvanReD9eurY3VrpV7Pbjkyx27sn5gYqxcjMTIHuaQsT3re8PW2lR6uF8QrNHbbSF2ISN3+1j5sdemecds1uX/giwlLXenami2jMAihfM+uGzUSqRi7SBU21c4cZfaignn8zXYaJo7QxBpV/eyAEjuo9Kuad4Wt7R1lXdPMnzb34C49B9fX0rZjiWNkcnae+a56tZPSJpGm92VZbZ4AiRI2T1IFaNhYho8kDzFOcHvWnaCS4HyopVerCs+eeeLUnigQ/dySRwK5byk7G9kkWpriFUKkbGFW4Na2RBFcbAPWufuYLtVLzL87dD2rMCTxMfMkxnpg1sqXNoZ87TPVE1C3+yoUXdI2OKqatHI9v5oYhccis3wnDIcOXEvHA9K6+SPETCSMYx0Nc7XJLQ2vdHmlzYidlLMwx6GqjxfZYW2jcM1v6iY2nkWNSNp5AqmQGjQbNwY8+1dMKltTGUUc4gm8zzuQKstKwTKueeoJrpzaW0VuUdV5FYkmlo0bPGc88Cm6ikTy2MXUY5NgdFIBHUVmwX0sLhAMk967zTbaKSEQ3KBh71TvdMsYZn8uJRzxSjWS0aHydUzBhtXZTc53MO1XdLu5Eu/mXaDVmFFQsmQAac0MKTADAGKbldCtY3zdpcWwbHzKKqSXH+jsR94U22tt6ZikwB2NOWIvK0ci7fTHesoruXcXTbpZbadpuGHTNTaVdpNIw2456mlWzXBQKcEVHbxCJ9uMEdMVTaYlcsasdrISAfSuYvpJriTBJ46V1EwjkBWRvmA4rl9QLrKxQ5pW1sTMfYMYHAk5yav30QKLtPBrBtnYsfMJzmtG5uXMaqvIFaOPYSehVnjKvtQkn2rZ0yyuryyaLds4rJhkYMSqZPfir9rfXFu4OcDPSsZXuOMtdSew0D7PfmS6YFRXTW2oxR5jH3B0xWWJo72HJcgnrVg6WkcIlhmGMc80qkr7msfItLeQzzBVAJJxz2rRudPgNr5iqhI5rizcfZ7o/NznrVyDV5wkkZc7SeDWFmCqdGdFa3QP7ogDAqPUrJGjEpG4jnFc/LfGPDRHce5qxLrLlQwYYxypqo3TuVzrqUNQuQUKCEJg4zWTPACOFznuKs6nfK5BbjNVbW/QSbQPl9TWqbSuYznfQZBaRNuPIbOMVbSx3Nt6D1qGZo2lLRN7mrR1IQwhWAyR1pNyMyjMkkVx5ecgd6uLH5KiUEHI6GiSWOSNJSRWdLf4crzjNNtsZqRuzyLnjmr0lqnDYGcViLMZ1HJB7Vfjn2Q7Xcs1PmaQ+YrXdurN9wAisyWHzG2qMVrSzQiP72WqhlhKCF4PWtKbb3DcfFbLFFnOTUc0mwjHAoMuX9s9aqzTbmI681so3C9i2khddoPJ706VJTAInIaMGq0Ei/LxippnyCAxpWexaae43aowo4A4okt0MJCnDUEK8YyeaeeEAAyaLEtoxjb3m446UVtBZSOlFO5NiZg7RhI3wvTrStAWlClyxUdz1qm/nJgbCeecVKZzkvgg9K85Qb1ITuWJYTC4LN97qKgc4nJTt696W6kM8a7G+b371I+EgjYpliMGhKyGyKG6V2dWG1jximvL9mlBxz3qB2QkcbecE0+cokQUtuyPvVpGKuJEwgFwxuBgd8U2P9+rhWChe9WdLt2EDvIcZGBnvVe3KiZ1YBeaTdm/Ib0QgWJQZpWzxhRinW9oLyJ5JOf7oFOe6gklWF4wyg4zUd3qC2YCW6hR60R5r7DutzGEDtdyAggJwKtQyP5mwdcdTV20LSwuzlCzHOarOm2c/L0HWtnJPQLIcJQsRQj5s0yGQPPuPUVGUJjZmBwOlS6dEWVyV4Pepe9xJj/LR5CSB1pXRhuAPBpQUaTarfMKmjjYnk8VrdOwx32zyVRAfmxViK7klXymTtmqjWLSSeYDwD0q5FYzrIJN2RjoKzkocvmOKYW7fPkJtI61YwZDtVR+FFuIY2LPksTzmnyj5S8J/KsXJXsFhDbxRxjePmByasWyxKHkCgEjiq6h3QZI96iuJY8hVYjA6U7vYdmirqrGI4jQHPUiqlsHji+cdTVtGDN8xzg1ajCzHaVAxXXHSNgSuQxT+WQAM8Up8yYEqAFNSIFMhRgMCnzlQoVOKGuxfQpOixn5icVVM6LIwGcdq0GKldrrUcttDuXanSnF9yHqLaTROd2/BHatHeAofgKe9Z8VmrS4xya0f7NmkgyFPlr3NSy1KyKTyqzs0a/jUouPPCxykAVDOBBb4HUnkVnCcmTcRwK53G+pk3qaN3ZIF8xZPlXt61VRIZSSYxu7ZpVuFkYBjj2p7/J82KuEtPeC5LcRKLYBQA2KjsmkKbHYqD3pGkaUACoyJ45FYj5OlTJ30C5oIscblQ+T1pHuioKqvb71UoXAkJlHOeKmaYOSmNq+tZ8oCrIrkjNSbS8XABI71myN5b/Srtvcq0RVCR61WsdhqVitMShBI5bpUOWJI6etWrhUIGTyOc1QMju5CcqTW0HzK4IvxbII92cg1IjpNyBjFEcKi12xqC3U5qJpAi7SQrjnipvfY2UtLISVWHBHDcVQltiGOR9KtNcgIVPLnvVcvl8iXnHANaQTMmSwB44ycE+1SmV5VA2H6+lQwvJkDBbPpWjGgMDgk7sdBTbs9QRmssgZnJBHap7LMuTjOOtVSs3XoucDPetqw06WCIOzfM4+7U1Goq7BbkAskuTiFQZO9W4tMmjh3KvzDgip3B0+JmUAM3SkXU5bXBf8AeM44ArF1ZPYblqUfNktJSkvWtPTp3uFZlJ2CoLpo7uDcYv3hqCC9NgPLUY3dRVqfMgubU8iR2+6Xr/CKxnuHJJPAJp95qYliWPbkjoazpLsA46k8Uasdy6vJ8xOv1rm7nSLq58Tm+uSpgQqY1zndgcDHpmt62dRbv1OTUE84V1xkkValKOwmk9xzrK5OQSCcmoHkmVGRCeauQ3IGcN94d6g1JJ0t1aKGSXjJ8oZI/CiMujKucZrcsv2jY/Q/eGOPr/L9KyC5VsONw/Krup3bS3TKyupHUMuD+IrP3bhg9RXbFaHNLctCS0bpb/8AfTHj/P4daiZVUgmPbnpg8VFwOvT1FXrUs+U2+YPeqEVpp3cKhCJGOiooA/H1P1yaBM8MZWNyN+Cx9QDkA+2ecfT0qw0CgnA47iq0kLA7hz7UARj5myx6nknmrSqXcyRrmMkDr93jv/jVQ5BwRg+9T2k/kXCOeQDz0NAGpbaO13KkTOqCRgFl8wBEOR97j29uoOQKnubO90qKL7XEvJJSWEhhgY646fj+VQSX8alZYkeCTHVUKqfqOeff61rafr10wRBGbhBkHykySBzjBHtnvxntUts1hy9SGfU31bT0tJ5I5FUgmQj5o/cAYzjJp99Y6cttbf2YXDoCkgfJaQ56+3+FXn1bRZ9Q+3XFrEk5k83cijOTznB4/Sq0mpaDEHMFugbO5coMZHT8P8nnmpRryx7mjo3iKHw1fRCe0s9WDRBlDHPlMOSO4OPUj6V7N4H8b2/i21nkVfJkt5ArRFgSFI4P0PI/Cvna7ljuLdlhKBFYNsUYDde3Tj+p9TV7wx4juvC1x51tb+YHcM4JKll7j3HXHvTautBxnZ2ex7v4r+Hul+JJGnULa3h6yquVkPqw9f8AaHP1ryLVfCmo+GrsR3UbRh87JY2+WQD0I/kefavctD1y11vRo7u0k3xyLkHuvqCOxzxiotSitr+1eyvYllhcg4PYjuD2P+PvTjLuaSp3PDzql3DalQwJX+LHOPSsy41GSS4UvIce1dDrWkSaPqj275eBuY5CPvr/AI+tZ9t4Yupgr7SQx9OlZzUI+8YuM07G94YvLqV0ihjzCeT6muivNNET/axwWOChpND0hNOty7yFXC8CrUMhvrhYVfdjue1cEmnJtGy21M+7toLp4ozlQByKov4Ke6uN8UpC56Guzj0W3Qln5bHWpIY0hG0SihVXHYHBPcyND8PNpMxdpj06Vs3FxHLGYt4BIplwrGJnDEAd65pLmUaiCzZjB7mou5O7HpE0V0UFJGI3Fu9VW0ZIvl389cVJqPiyK1iMcKb3HGBWXZ+IXklLXCcn9K0tK1yfduM1CwkmuEEbnao+Y0lvZNHE2BwPWtnTpI7lHJKnJ4qpqbvCGWEgDvSUr6CaW5mQoyyln4xWfeSAXLM4+U0lzdSxxF2f8KjNyLmNRsOe9aqKJuiu8qGT7uB2qrMZGlUoflzyafdsqS7e1MjSQgEA7c1S0Mnds3Y5VitAe/tVlLvO0kYOOCabpkEM0RjfknpmrUthJCAGTcB0qedbGiWlwjvncbdn402ZljBkJ5NOGI4hhfrUrQQT25aQHbQG5lot3JOHMeUPQ1S1G0mLsVXk9q662ESW6BT8i+tYusarbJN5cMeW7ml1Bx0Oft7NiSGGMVchsXkkG87UpsU5dm/h71d82GSI/P8AvF7ClObWxDJAkNrEQqgg1RusI+7sR0pJNQVEwRkjpVS4d5yrjhTURve7JuWkvFjIGcA1Ze6dYCxlZV7DNZWU3BZBwBSqDckjedg6CnJJu47sjmm+YsG561PFeKYdo61UuY0jiOPvVDZSjz8uOB2quVNE3NWSUxxgjvSo+UOep6VHcSeYw7DFQuWGAp6VFh3ILwM8yxjmiL9wNsqgCrUanzNxUZ7VHfMssRBXDCrvfQRCVCMXV8p161WmuDI2O1NJxF3A71LapDKxVuh71fLYBPPbaqrnBpixyNLk9K1CtukQTA3L0NU5XVuTgGlfsMnileRhGq4I71JKWjGcknvUNvciKA5UZB4NSJcB3O8dahrUaICC2MZ9c1Y2yCPkcUxvK83kkKvNPN0JF2D7varuwvYpyLIV2oTnPWpJtOdYA4cNxzU4uY1Ox1GD0NWLe4tXV0due1W6khGLDvQ4INWdkr84wK0Y4EcnkdeKsfZztAUcetXz3KRkEBWUGpmzgbKnuLQyLuXqtQBfLHJwaFIGiTz8dqKZmP1FFFwuPhEyMCx4I71YmkWOEDYGPfjrVdHe4Kk5wvOKlW3luSJEPfhT7VxSWupnYjihJkw0ZC4yKvTXlvDGFlhOzbxx3pIZzGwjdN0mcEY6VRvWcXJVhlSM4NY8nNKzFqOWCzuFBB255psVgDMyOy+WPes7zWjZgAODkA1XS9mScszMN5456VtCEtkxxa6nQzosAjSJmPoKqNJ5rCEqFbuwpBeM9sFfBbsfSqjSkKUAO7P3qtJ9RyLaWyOuwk7lOcis27DSOUA/OrlpPNBIXyHHcU7Utjae11tCN6U4txlqHQzNNldfNQklc9fSr0UxR9kxyrdGqjaXCRaW2Dl3OT61M86zJEwXpwRTmndsbSsXL0CMKqAFGHUUsUqxAIp+8MGo45QwCP0NRiJhchQ4welTF2VmBcsrItcmTA+UVaELZIQZJNRWshhnffksBjg9aiTVNk7qyFeeMU43ZWhfWNwwibHHJqzDIUZtzc44FZlqZJpGeRyqnkE96uN5j4KAYA+9WNW6YDpWRjsEeGNUzcNCzIpy2fu+lWV3lxKWU7ODVFX23jEbXdjTik9AvY1rPy2dmcYO3iqjwvMzFFBA7irtvCfs7PKMZ7UsDxWYL4yG9ahT5XoO5kt+77URXBjyx6ntWw0UNyjNtHTIxWK0TGU714X9a66dRSQWJ/Lcp5y9zzT1XDZ3ZJp8b5hCdM9qlEabM9DWiaH0KrW8ssvI465qfy9kZJHPSmz3XkKQG5I4NVPtzyR8847Vnzu9ibl0u0YV1GWPStdb4x2qwuQDjJrHS6DRjMfzdqpXErhiXYqaqUn0C9i1dz+ZI54x2rFYSpIWKHb1Jq1FMrqd+eKYbluVIBUnpWaujN6kcOyW4yemOlWZjI6gKp2UsPkoxcAAkdKel2T8jDkdKmT1ugRXjZohls4Bq75jTR/Iu4Cq8zZABAwevtVuHy44htfBxilJ6DIGiZh5p+XA4FKFIi3yN1NMu7gAcHgVRe6Mg25O31oimwuWZYTLLuRs5q0oEEQiIwxqpbAxRCRs1K8rFlZu/Sh32ALg/wAPVsdKr2wiV8scH0qyVDSZU9utVHt5DKzkd+1VF2QXsTy3QjiIz19KqKxYgjDE9qlkhwVDHK96mdLe3jDKeSM4q4NJlKVmZl3DcK28HHpmqquXGZSFcHitb7fDeQCAsM54qe38MzSsJQMqfWuhtJajau9CxpDK1vtfAOeCa1mgjijL4yzdxVa20oWcbJN9QfSmw3WJyhwUBxXDN+9oAgaCcCJkGQflOKvRi5Vj5m0KB8pzVe9ntUeHjaCeTiq+q3y/ZGWFjmpknLQTZNeSEgGSRdpOOvSmlAyLJA64UdD3rItrK4u4lX5sHnOa6KzsoYrfAQtIowc0SXLpcSK9rdCUYMTZzyakeydphIYvkqwm23kG3aFJ5B9auGcOCpHHakppbFJmdJawsACNpFY13ZlZCYyGxWk9yZLxo41zVW5bZLtOPetKbkmNIgj3pD8ybcnr2qlcuYz5pIIzirVwsj2md/CHIrMlSSXy1AO09a6Iu+oNly2xhGByc9K3oJHYsVXgCubkie2KSryo4xU+sa5dWmlYt2MLsPvhQePqQcGplBzaURqSSuzD8VyXUkpFzfxtg/LbJnA/nn8awHiS6VpLa3ZPmC7Q275jk4HHTg46njvxUUpmnLysGYZ+Zz6/WpHla6+6saMzAlUwgz7DoOp/Ou6EeVWOeTu7jXs7mO285o8xrwxBBKem4dRntn/Gn2kxjYMAM9BVq1huGnluJ3YeXHLmdX+ZHVDtBPucLz1zgezI2TUXJhto4blUZ32kLG4AyTg8Kfpx0wBVMRM6lhvwCSPb+YxUapk5xn8RkVVE3HKuo4zxuzj6/wCNaNtIwt8y4EeeQSwH8j/gPxFSMjayE424KtgnkdKypYXhcqw5HpXR3EmxC0ZiSML/AAhfn/MDP4cj0FZN0yyLnA6cbFx+f6UJgygrYPQH61Iso2kbV56nHPWoaM4qhGn5ZvWy14nuZG+Yn1/r+NaFtaQWqho8G5XDIwJLDpjj/PX3rngaf5h2bQ2BUhex11xr9yihXlgV1O99gG9j/te/1zj86wNQuDLIZTKrYJUHfnI6j/H/ADiqMuJJmYH7zEikaGRRkxuB7ihJIpyctzp/Bvja98K6hlS0tjI2ZoM/+PL7/wA+n091sPF+lazpv2mG5jY7c4JwR7etfNNrZXd3KI7a3eRmOAAv9a73QfCS28IluZH+0sDuRWwo6cEd+tZVZKOtzoo1Wvdauj2W6i024sZWvESS3IyVbk+2Pf0xWFovk2UEkMsgcjoa52JXsoC8js5/h3uTj6Zqey1mOLKOgOeprjqTc9DeU1ck8RaowQLDwc9R6Uzw7OFuVPmEY5JrM1CZLi/IxiIjjFTWMT+Yoiwig8k1NtLGe7uehDUI2yF6Y5Y1SiESzPMDvPcZrIS+nMHlqqkKcFqqzaoLaRogcE9az5CuaxtNrau7QSx4UdMVzWrzqJwkGRk81au5A8COgAb9awrppMmQMCwPSrprUTYxRuZgeoqKWTK9cEelW4bqHY0hXDgYxWQ1w0pcsNpz0re9zOTLEOpzWikq546c1aTVpL8Eg4PvWa1u0wCouM96sWOjXKy9flJxmlaG4knuK6CWQq56dKsWrLGpEmK2r3S7e1tUJI8zFYVwpXcV5o51si1Ebci3cBsc96bBcKGEePlrKmeVXKjk1JE0khHGCKWo04nR2z7SCpwuetX73WTHEqI6saxImAhCvwad9kjk+YHFZuzepDfYfJq7vLg4C1pw6rAtuEI3Zrmri1ljfKjIqzCGWNT5ZppK5KZuSXPmINp2rWLcQB5WO7PvVgTblCr+NCQlnzkKPek5a3G7FZLYIobJYGmzIsS71PJ4PtUt5DLHtaNuAcnFUGkLswJ470tXqZSZXaREnznNXgrPESB+FZxaPzQnb1rXSaHyQIuW71UkCKUrEoFcYPrSW0u1yM8DvVuZY51w3FUpIVhwYzuBPNF01YbIbtvNkznC0/T4fO3ugyVp9wirFlVzxzUNnMYFYBsZ7VXTQRoOypHgr8xpPLZkBxUXmF489TUylpMHO0dxWdmBWnkePAH504yRkAsQSetSz24YbA+c9apW8INw0UnboTWkUmO5FP8AvGZI+mKqRzNESOprTFmN7gOARWbcgRyOvp3rRWehJNHdbmO4mnsqy49BVKGQMhOPm7VdsvNMwLR/L3zRypDvcl+zs6DYePSk8wxtg9aJbgwzlR0HpS2s0U02SKl9xloRsFDMuSaUrE6Aov1FSEJKvD4UVDxDEWRs1ncBlzCsg3KcVHBb4fcx6VLBIZJVyBgmrN9B5agx9xV83RgPTCBX4xV43iCIkrxjisuE7EVZBnHWpvNR43OMgdMUlIaYkd0zMTtxk0TmOVsMQrAcVHDdCMYdR7UPF9pmDIMetHUlsqmNs9KKueUo4J5+lFXcXMTLJuy/AZhwoFRJPN5rDO3HC4pLZi92xyAq9M1QubzMzhfvButcii27E3uW4buWOVnkAdu5qnd3bvcb85JHJpHfEXXOetViyfMG9OBW0Yq9yk2Ak3Nuc9KtXEUNwkTR4U561REOIS5Jx71Np5DzCM5PtTtroJdi1caZLCQFkDqwzkVAnmb9mMY710GEARPLy/QU9NOWMuX2lRzz1rL2vLuXY59YpQx2Pz3qGVvtET285K7eg9a6BIrJ7rBbbgZz2qtdabbPN5qMZN3QA1brJpXE0YPywyIFUbSMVO0DrIu05Vv0rQXTIZi0abo3zxuq22lskIQygMO570OrFjjF7sw0wLtQx5z0q1IY4Z1Zh0qveabPFcrKTlc9RTrxy6pGRgiqVnsJ6CrcEyswY7SalYbplcKSD1p8FvGY19e4FW2MYIEYLcdu1KTS0DdETBmcRliEHIq5auz27xg5K1TuGdZIsqQtT20qpHLxgnpmsJK6GiKSUR7liB3MfmqW309IJROzEE8gVUtWb+0CX+6fWtxh50QUjA7GqleySKSuWPOWQJCrjPWqdy8b3CRyfLTo4UjukPJbGKkuoIyUZuDmsbpOzCw8SrBH8sfX171XjkQ7mI9sGpmQvgFs7eRVfJy52ZOeK1hawNiSzojKAARntQs6YYA8+hqZVjaPcY8PVKYRlnA4PqKtWe2gm+wskYmUu65CjtVRUiBBOVGatWspSBtzZWqrxtJuMfTtVpu+pLLwZcZi5GOKq3i+bgsvIHJqKS5EEKIQQwply7SWxMb54oV7i5irGZHDBRwKqSzsh2981b0+YMjqTjjrVW4jRJc53E1qrXsKVmtAjklkkUDnHer8ZzJ7461Rs2cK5H0qT7SqkKPxpSWoloaCKxGHGR3NRtKElwucCmi5wF5pZNrA4HJ5zU2KFmcSIMd6rvJtj2BRx3qv5rRyjac1Xnu2MxAqoxZLkXjcsYsE8CnJcmRsZ9qyllYAhu9Swz+W2MdapwFfU2rebJ8s/nV2UZXEQ3EjkCs+xkjZtzCtQzRxbXXAJrCSsaIoOjyFfkIweat/2bHIhY53MMYpLq5UkNEfmotr1nGMZbtU3e6ErXMKTS5ob9I4kJUHOa9A02RbeCNHfPAz7VlQTAyHzE5xjNSOuzqeTWk6jkjRabF7WVPlMiOMP0PpXMSRNChMSFm7kVsyyCZFiPpxzVNAVcpn5e5qIsT1Maa5nkCgjKj1q7b/AOkRmYKCV42+tLe2cKxySQygkdiapo4jhUxylB1Ye9aOzWhB0VlDMttuPyM3b0FMe/FkCu/cScVXTVWNsNp3EDnNY97qDsm04B65rGMHJ6jbOjSye4PmPLgHlVqyrqibC25h1rl9O1aczKZHyo4xW3JmO6WVVzvHTNTKDTswTG28ge8clAADwarazKqsqeWRnnPrWwVgdBvQpJ7Vk6pDO0qkJvUd6ulJOZorlKdCbNZUkB3cCPNV9PuU2OZjtK8bT3qG4iZJSwyAwwPY0yy0m4muU3MSmeTXbZWswb10NDb/AGhgk7VXtWfrem3F95MacKvXP/6q3La0NjKY8bhnNaa7JWC7Rk1EW1LQTV0cxHY2tppitPaq6phFZjxn1A9f157jgY2p6datm8eYRwfd+VfmJ7j2/XHTtXb61ZpNpMkBk8h0PmI2QBkAjBz2wSPbr2rg76Q/8ImmC3N2YmywI+VF6Y7dPWt4ybMpKzsY0kkacRwo0YOQJBzj3Iwa2vDd/Gt3IsJstOujE3l3Eu9lb1QglhyM4OM5A55rGALEduOpqtKuxyK13ViTq9T0iRBJcWjgxAF9uFJHqfy/zxWGbmT5ArICDn5QqEe4wOBWcCw6E0DIPpSjGyswNCfU5i+CQdp+UBuB68iqUkjSNubrTMH/ACakkhMchj3K5UZO3ke/5flxVARUUpBzSYNABmlpVxuGcke1bHhvy/7ZtfN8tUVwzO3GMc9T0pN2VwMhWKMCOoNeneGCb/TEkd1lZTgNkcexGBirL6Np1zOk8dtENvIZBwfy4IrTs7KCyR/JjEe85IUYrCpO6NIpxJmjQ7AqBSh42jHNJMgifzAcZHNK9wqyBOhHWquoXf7vdg4HpXO43NblW7keeMqGOM8VRJkgTEnOO9KZ/PAER5NFzIwgClT7moS6CvctW8iSIxdgABkVbtJZDG21d6Y7VgvIRCZMHZjFW9KvJBINsmAeoNU4uxSep0thOXshtXYc8is+8YtcF2jw3ajzhLcAJlQDT7sjzkfIO0Vmir6lS5mkynzEH0zVRpB5qrIMMTwc0+R3uHJC4CmmSW0wkEkiADtVqJDd2LdJxujOfUCs2VY5G3byCDyBVmSVwQU5BPSqLuqynHU/eqkmJs0IZQkPmDJUdq19Jv0NyqknZ6e9YdtIjQFUBGDzUkL4m8xDgjpUyity1Kx0HiG7kLIAMAVjs2YyucE1JeTPeMGZvujkVXWMyZFJNWKXvMzpInLsyN0qrDNctcABDtB610EMEY6gZpHQKMqoFNSM5J3GRt5hAYHNWowVIUc4qO3BAZmHHrV1BDsAX7x71DYrjJD5se0daiSYxYUjJHanoggfcWzk1SublFm+Q5YnFSlclslkzKzMi7KSK4BR1kPPapWjdLcMTgY71mO6+bt656mqtcTLkVwksD7pCCO1RSJG9uQnamS/ZgERDtz1NRTTJbD5TuHShLXQRVuh5mxoF4Uc07Ti3ms2ePeoI7oGdtgOwip7ZipMgHGelaPYEXhuklBGAAeaeYQpbcwKmrQERiEsiheOgqlNKHdgqkKayQCNAnl4L/eqjcKA4AGNvpVqWZFQIR06Gqf2gRo7lck8VcUwL+lrC6FnfBHrVqQRO4KEADrisSFwAHU1eiAlcBXIz1pSjqBPIrTowg+9WLK1zBMRLGeTwRW8UktxiPA96jMYmGXYEDnmiMrA9Sqto6wLM743dqq36oSNoHStLzIpTslbCrVOaykkJ8v7ueDVReuoivpVvFLIfMIAWtcyRxfIADmsVo2tgRg781oWxbCmRevrTnvcaZlXZH2jqducGr8Cwxx4jH3u9T31miR79mc+lQqqiDPQ9hQ3dB1FSNsFQxOO1QmQplKljlCrvCk46mqN1dRyy4TvQkDLaSDy+ThgeMVJ9tY4Dnd6CkhRBDkDc2OlZwkbz2yuMHjNPluJsutJJMxHRafFcNAhTtUEDMzlD1PNTC2YYzyaTSBDyGmKnb9Kuk3CRqI156dKqxeZFMM/dq1bSs5bB4BpXaDcXy7s8mMZoq8Lg4++KKrnCxz4ugh+XhAMYPeoCfOn3qmABk1FdMARhOO1FvcusbAd+KjltqiGxXnJLKBgHvS27RC4SOXuOTVRpCJP3h6dKsnDIsgIJNXZIaZNcMRuhU5Gcg1FbRmOUur/ALwCiRAAXMgB7CohK8M4Yj5Djn1oS7FdToreaSC3+0zENjgCqs+py+fy37t6c12t/b7CNmBwPWqEihDGH6VzqKcm2W3bYnKSXXmKPuAdRW1a21ulpAfMIcc9ax7SWWYNDCuAeprU8rMaRdJAM5oqLRIcddRlzPGLgyjO5e3rS2l79v3iY7cdBULyKodZEC7RwfWsu3k2ySMThR2ojBNC5mmXJJGt7llLb1PFZ90jCYknOO1FxehiOAMVJGpunGGCkjvWqTjqyG7k1ltFu9z/AAqcYrRtp4Qm7gGQ8is9VHl/ZInBOcmprWJ7aYNcJ8oHB7GplDmRV7bE2qs3no8S5QDAqK2R5VeRwRxUFzPLOsZBwuScCr9qyyWwQdB196zcXGIluR2i+YQSuQnpV+G5JyxOEBxg0y1PkpI8aht3Wo8pNIFyFUc4qN2WnZGhhQ4kQ8+tOlmR1KnBPas+SeSMhAQF7GozIVmCq43Dkmh0rl8ysXynmIfLfawHSqvnvbjYfvdOaiVm3Ftx3E8Vfl8u4jTzMCQdxTjHlM20NVgCctliOlVHhLNhuMnJNPM0UJbB5HSm/bFPJ61ok0Tcj8kHKrnb0qIq0alQ2MVE1zJ52EOMmrZhMkTBmyx71Wq3EZr+W98iSvla0g9s/wC4RcN0zVF7LzJQWbGO9TCGJSCsvzUStoMWeNLdGSNRv9axLlSrEtzWncLIHzuyO9YF1PI07JyBnvWtJEyLL3AjhCp3qu8vzKe5PNOiiyAxNVpW+dh6VskiGavyhQS2c1KJTtzkccVlwkumR2q7bwtNjkjHWs3EabYx2ZpTtXmqbxMshz1rdvLaOFUETbpMfNUVtAhHmydO4NJTS1HymOWYkj0FXLOzEwyxPWtJYLRyTHFzu5zVo7Y+IQoz1olU6IfKkQpYtCu1eR39qgmnaNdhFX4LsJE7HvxWdLbyXbOyHp0qE7vUb8iOCc7wGyRV23uFjnAB6mstg8BCbTuB9KuSWNzBEty6gA/nTaRKub8cqsxLN06VG90WznlTWWs7vCcH5lHaoTPJGd7HC4o9nYu5dUyyylkJXB6ZqvfTvBhQ5BP3sVUOqFkG0ENnHFPZhI4Mg+Vh1quW24rjFtrie4AUkhhnFb8VlawwAXEZMgH3R61Np2lskaSrKuDzg0ahcZnVUAVl7msJTcnZDIlHmR7liCKOCDVeayttjGUrsAySKI7smZixLJ6Duax9UvSt0E2MsbH5gacYybsJtWJ7GzAVpMfu92U9xWhbaoodo2BZgflFVG1KDyFijAGBxVfTl868Z96qfWqcbpuQk+x0dpqRdm8xOh4rWlaCa1KBhvcc47Vi2oggkZ3cN7VNFeQxSsq/xciue2t4mkZWIrvQTFEDvLe+KjjWS2iVFU4z96ttJDJGpY5WpZYoJYACAMc12K7Wpo7MzmT5d3cjrU1vbrENxYE9SaewVyg6KasYSEHBwMVaEedeMvEn22VrC22+VEcPIM5Y+g9q5dr0tpEVjtPyXDzbsjHzKox0/wBn1rQ17T7iPVZXa1MMczkxg8jHuf51ikMjFWGCO1dcUrI5pXuPE7KR/Sl3CQ88mmsmFBqPoaYicLx2pccDIqESMO/508Teq/lSsBNbxRPc7ZUdowMlUYKfzIP8q0pk07aDCkls2MEpFvJ/FpP1AHWs+2IaR3yPu4GTyeRUkhYAfeOOm45/OkBXu9rTkq7MMdWTafyyagIFT3CeU5VgQw6qTyPr7+1RBGbkAn6CmIaBVhJGhikZGKsQACDgg5B/pUPGaXCyTqpcIpIBZug9zS3BHbeD9fubmU2Vy6vgZVzncf6H+f1rt5HKopB5rg9I8JahYXyXLX0MIXvCxYsPTkYxXYO7FByWx3rKdr6G6vbUmnHmneCA2MVnXbFLYxk5p6ykzdenWnXEQltzkgbqixRhvPDA0flfe706S6eUY/hqvfWPlOGDZPaq0U0nn7MexqXEm5qBy0AjwcdcUsWBKm0he3FQmdoTk9AKpSXBDrIG6mos2Vc7FjHGqfMORikMS4Kkkk85rItLtZPLB+Zga2QrBmI5BFY2tow5jPuZLe3wVf5geRVC+v5XRmVyVA6elX7uzSV+VGT0Irn9SFzbz/ZV5Vh2FbU7XC5Np9ySu6RxtbjHpV1Fsd4LDd65rnpIprZVI6VbhRhhjJuJHIq5JboalboaaqqmXysBOoHpTI5Auwk8k0+zgkuIXEa5zxUkenskeJlKsvANZN9wSuPlmAiIj5bvTLS5YEByDTfKIbZn5e5pjokcwEfTHJqIpFvuXXfMmVHFIXJX1qBnZGUKDircTI3VcYq2rEXuNDySR7WBCmiB3jBBGcHjNOFwQ20ruGeKWZ/3ZdgAAO1Z3JbIZHkbKls5qAxbVDk/MvaoDdmQnbwueDU0MgSYbvmA5571RLLcU8swUSL8i9jULQNJ57x7QB/DUN3dPyy/Kp6AVUjluEBfJAb9aIpiGFX2AscEdahuJyVG3kDrTldm3E9O9SeUBb5UHHertYCGFR5YYDk9q0YLWQBXbhBziqSONq7MHHUVdW7Yx7c9e1JtjRclngkByOgqNZYTbYAyfWsvczSODznpT4hMHCbTxU8oXLRRJkZecimSaeXtyEOD71px2SmESbwCRzVaWFkUr5hJPShS7BYzxYyrENvUVagdNm0/KQOtL9sNsVTG5scioZ5kyCiYLdabuxX7FlpjlQzfLTLiQqMpwpqGQxyhFRvnFW5bRo4VLnKkUthEMflPHuPAp5nWN1+Y7fSoCsa2zZbB7CmQNvBAGcetFrhceXiuZGJypzxRNLhlBbhaWZQiBsANiqtviedUbqTVJX1GXDcFgFDZX3qFcu7AHPai5hkgLfLlexFQ2ryAjIGc0ktALZcRQGNkH1rKsVQXLsybhnita4t3nhYow3Y6Vn6WkwlIdQMVUXoxsubdkpdThWFQz2wlcFGPuav3MAlRQq4OetJDcRCbyJIxhe471KlYVjIGY7vAb7oq9HeZcdKhudkd60m3CHpWczETllbGD0q+XmFsbU8jXGdgIx1p1upRN6Nk96ihu1iRSwwCOadbzb5XAHyHsKhjTJ9pPO8UUpt8nODRTuiro5/fv78ZwBSRMdzjHetGOyRVAK5b1pkcKCVlC9KqTTMuUY8PnMBgcLmqkL4k2HgBq3LaPAYuMccVni2RZ5DjOTQthtMr3AEjkqPlzVa5kk+RT91elak0EaFQD17VTvtpQKBgiqjvYLWJ45StvkHnFRyOpRWYkt/Ko4FLRZz8qikjuYhuV0yB3pcqBss2N1LBKBGCVbgmt+SQ/Z1mLDnisu2eBNOeRsAk/LxTTfRtpwjJ+ZelYTi5WNIWS1LeqASW8LRE7z1rHmR/MVFH1rVSTFspkIKleKzzIssmVJytOm+XSwpNFS6UA/dNS28Nw6eepwFFad9bxnT0cAB881VgSRWRJMiI+laRlzInl1J7SIgiYEZ7j1q40jywkOCepBNVUjaC4YxtlewrRhKXFu3mHGBwKynJpl8tjniJmlBVuM9KtC+dcRZx71oLYM6KyDDVkajBNBMsm3gHGKcWpuzJs1qamnXRJeBTkn1NW4lDzBZEw2eawrMStdKUXaepNdLCheTewxxUySTGtUMuZLYBlODnpSpHCQjbOfaoJLVGZiDlqkhm8mPaV5HQmpt2BNkc6mRmI4x2qGHzWlAHanw+a8zsRkGnMs8bjKYyeMVe2hO5BeiTzIyF+XPJpGmRG2EAj1rSuED2mCuWrmJo51kZFzxzVRtITVicyJ9oLbuB0FWI78mUKOo4qjaQq26aUnIHQ02OXF0FUDGatxFc3hEzIWfAzzWdMQhO05OatzXKso2nHFZrMRLk8rjk1nFX3LdiRbgjDAFjVeaJJGMjx8+gqSGZMsAafgLjc3B5rS3YDKlPlLgAjNU1AeUA85NX7+QMu5BkdKpLEwKmIFmzkit47Gb3La2VxGMp909KuW5YSCNxtB6mrNiJ7gE7QMDoasT2uVWTgFetZc+tmaJFeUosoCknPrRKSGXbyDVmNIZ5AjYyR1qRbNY5sEl8dKzckDTKwR1yynjHSoVdnmCg4JPWrSnbv4JOaqeU+0yBScGmJlqa0K4RG96WCN4rgA7sDuKmsbhXQBwOvU1qFwITsQDHespSa0GtdShHBH5cnmDc5bIJ7VZnQzwbZegHy4ps8illBABHWoDNtQDfgA9KizbuVcz5EEcm1STng4qGWANAWMvT+GtJo4HBkyd56isq4RvNOwE11xdyWVYI1in559jW1bvE5C7AQfWsWZTGwBHJHJojuZA6sh+UcY9aqUeZEp2OourgwqoVigxxg1UmuFndBIeR1OayrjUHmYBugFVmuGMqoAcnislSDmOgG0xYjIyDkYrmddu5nuQJcAjjGK0pFk08JNHJuDdRUltYWmos890Tu9KcWoPmYPXQybC1a9gyjHzPQVpjQJ7e2E32gh+pWpbeyNnen7MB5TevapmnYTt5snK9BRKbb02BJCWbSyQkzqdi8VoPCjQj7OckDqaYEViqROv7zls9qtpAqLt3AEelY82ppAmheRbeNDyB1NWPPBjCZzmoQAkWOue1NEDSPuTiumNzRlw8KpznFOzuY9+OlQSS/ZYS0hHTPNchqvi+e2kZbdYST053cfgeK0jBslySOwuAZIHRI0kcLny2I+cZwVPbkEjv1rzvxBa2qalHa2SmGGXDGNlJaM8jHPI+nv6YrofAEd9r+q3V3eXbmJAoZiee52r2HIB/L1ql49kNv49CQxhWgWL5WyckqH5787v1rqSSio9TLn3utDnb7RbzT4xJIm6FuFbHP4jtWYluXZQxALdAetekrNFd2HnXbKdxLMW+UR4PCgex6nufbFMXSLe4i8z7KqjOVZ1wT746j8cVUoNGfOmzgZbDytq7tzMMjHpSiyVbWWZm5XgZPfP/AOs1pTjOsvaRZBDtGMdAB7fr+AqG6sZI7fyAQwODn0IAz/n3FZXKWpiA4qTf5cmY2zg/Kf61O+m3CKzhdyDJJB7VAIGY4HJ9Bk1QNW3NLSZ7C2DT3QV5g37tWBIHHXA98Vp3XiiOW2liSMnfGyDCYGSCM5zXMmPaxDZBHUEYpNvOAOahxTd2S0rj3neQIpxhRhR6Vs6bpyXGmyhxhpB8rH17Ee3/ANesZonhIEiMpIz8wxkVtQ3TwadC5P7s5U8YxwcY/X8q0ik3YG9DqvC2onUNGS3kb9/bHymHfb/CfyyP+A10kiIqbV+9jpXkEmoSJezXFm724dtwCNj866PRPEd1cOI7h5HI7qoJxWUoa3RrGXRnYpCsIJ6u3U0swEkBKNyB0qBLkSxE4P1PWpoyh4B5xWRoc9qDttBYEE8fSqDKYJFk3feroLlUV/3oyGPHFZVzEJrkB1wi+lJvoKysV57hWjXIJOKp20ck5IVSQK15VTylxGMdjU1vttYiAoyRmlF2JG6chgUFxyDzW4JJ5MAEBcZzUVlai4y8uFJHSpXkRJfKU8gVlNpsTeuhYhjVEZmYH0zWdcwh/wB4cbieM1YtWDyvGxyfT0ptzbSSFjGxwvSpUS4uxjS2Ql5lOBngVHFp/l3GdxK1rzQP9lywAx1qtPGfJVo2JU9RT5mvdNNHqzStJEsoFjUDLck0l1eiVNm3kH061SZwIlZ+w6VItxDhQnf1qWkJpXuhTEr4YjBPWoJkO3aoAPY1b2huQ1NeACMkNk9RS2FqUXEoKqa04YAIl3cZ61DGh3qzYrQV1IwQDRJtoaXcoSQkuwUgAdDVC4MikIG3BjzzWvOytGYxw5rLYRwShZX96SZErDGi2p8qZB6mo2kCwj5fmqVrpHYgN8lEqq0HmLg54q15i3M+SQyLjNS5aQoi9OlVXUKDg85p/nCNVA+9VEl1rSOO3k+b5j0qC0eUgxkAjGKYLv8A0hFxkGtGzt8hixCgmp23BFMQLbvwMluarzMY5Nx4wcVp38EcTB1JIFZV7H5se6I896cHd3DYd5uJVeIbqv2cpYySyABsdK52J3hywbvWtZu05DZ/CqlHQlMvQXburgnGDxVd5rhZDI6naOBUsaL9qO4gegqQyNPJ5BUbajS4J2IbazabNwZAzE/dNJJBI7Hjp2q3Ghi3BRgL3pslxsiY5wSKOZjRlgNDN90itGCYvatLM+cdB6VnSzu0eOD70kG+RNu7A71VrgWm/wBIznAAGaiik8mXKD8KbvUHy1JyODikVX3fdOQeOKVhFwySTEs8Z+UcCohbYiNxu2kHpV2KchVDrt470yYpIhRDlT1FJOxSJFMhtwvD7hWW5xGSRhlNaIYxxjGQBWXeCUlpgpCU4gyS3vd8oAJAPWrEwjRw6uc9xmsiwfM+045NaNwcNtVcsapxsxXuixNcs0a7JOPSmW7gXGPvH3qs4KspZcA08iNblCj80rAh2sBHiDL1HXFZkNqZot4Y7s1uXCILR2LBs1nWQdELMB5eelNS93QHuPkTKLGwOauxwGAK6DPHIFOCSZUDBVhxx0qeKX+zkLSJ5qt+lQ3cnnRXN8+T8h/Kin/aEbnyDzz0FFKwucS3IKAjnAqBiiXpLD5KkVhHGMD5x1FR5WR+mDmteW+poidpPtChY+CKZGoy4bAYcVIqrbsSOd36VFEu5nznOelHkA0xqFJbkism7KzyYzgjsK3LmPyExjlqy51RTlV+aiO4MqWshiDDGR0xVqaCNowUQA9TmmWQDTsGXNX72AR2pkBwfanJ+9YRBNDH/Yu1/vg/KBWQNgICMSe4NTTXZEQHbFUo2LXAcd6tKzFJm5alngEbdQeKmjtg10vO31xVI7wg8pstV6xUjLzMQQetTZMtFq5ZFtGGdwB+73qsXNw0SRjAA5q6EjM2U+bNTW1j5dyXK8EdqzaUFqaNESW/loJFwQOuaq/2gokb5QM0ahdNCZETO3tisEykt7mphDm1Zk5WOwsrxARuYYI/KpJooLp933iORXOQbhAspJOTiuk02GNEDu+cisqkVDUIvmMoypaXBZ1+8cCrYvyZkQcKBV64isZ3AZAWFUr2S2R/LCjOOCKFJS6DehakYGDeq4z3pVtGubcOXqG2fbCIywI9DVkymG3ZsgDHSoba0Q9Bhtm8xVD4QDnFU7p7jzMpllXgCo7fVgqsrDcc9auNdI0QCAKW6k1dpLcWglq0otXNxwexqnM3lxeYgDds1dmlHliHr6moUETKsTDC5ojvcGZEhIiIYbd3NZX2hYbgvjPHFdNfac10wEZwBVeTQYChec7QOmO9bqpFLUlxbKsCmW3EobJbtT7mCSG32HG5xnipJ0NqkQj5UcYqK7uAMHneB3qLu+grJGZCXSTYwIY+taPlt5fKHp1qpLeQSSREcuOtbEckdzEEU4bFayel2EbFWG2haL5sDjpVeKOOKUhQMmrMNpcNcsgXtxVSSzcXgAY9fmpJ9Lgyz5zwEBTjd3FaiWks9qSWGDVFQvcbsdT6VZjuG6J90cAVnK/QpMq2NlMLppHJCocVrSNtUmIbmJ6UrSMAoI2hutU43kincqpOOlS25O4xZJRCwSRQrNTYJS8kkQ249ar3EnnuN/3ievpV2DT1jUTRk7gPmzTlZLUWtxILINLnfgDnFaQViDkjAFUBMTIFAwT1qWdWiBUS8kVm03uFylMSblc/xHFNnwshjyPrVOeeaNkdjkKeKht7oyyycZJNbKHUi5v28EbhWBzxzUVyihS5GBUCSPGuM7Tiq1zqBGU7YpRi7lXCaCOaD7w3HpWK0EtvlsfKprUtZ4pGJPJFSSyxOQrgYPSt02hPUyIpoyN8g5Bqe3kT7WjgjJOMGku4VBUEAKT2qN9Pu4lFxFCzRjkGqdmiTpW0oNF98E9c1XaFrQKHYHcelY7andpEvJGe1LPPcbY3Z92TzWCpy6sq6N2aMyW2UDBiOCKyoLOS7RppGIZWwPerA1SXykQbdgHPvSf2wGiAhUZHUAUkpLYeg6O0kVg7SHd9a27VMqpbr3NZUOp+ZCcxAMO9bNg/2iNenSqUW9zaNlsWZVUTI2eBTlcB93IzUhjD4B4xTCAJAW+76VskFzG8SXZgtSWAAPRm7+w964Q2bEmWUYLc4zzjv+P+NdvrVr9p1RGlx9lt1EshIyOvAP1Nc7qLh7hmAO5pCcHnuevrXRDRGMtWZ9ndXukXQutPuGilxj5eM/0//VS3Wq3V/wCIX1W6tvNkYguuQucAAc/QCk4A56Ef5/z7UwsBjPOaq9ibI39I1jTE1NmlVUaXDASqMxPk5wT2IxznOfrXSSXbMQ7lEQfd+cHI9a80mVZeG+nvTB9rtQxhmdUxkhWwCPpV899yXHsXjID4qd1cKDcPhiCR37V0UmhICJJLyZj1YKFUE+3XFcTFdSR3aT7Vd1bdg9Cfwrof+EtLpsmtWXj7yODg/QioFZmhqeniCGdkBK7CM+nHeuV0+RY7pizAAoRk/hXQf8JPaTxOrl03KVKsvqPbNcvbvFHKGmUsoXgA45qbdzR1G7N9AumVrqQqcj1qzpOReFkK+cEPlhuhP/6s1RkffIzcc+nSm7irZUkEHgim0Tua2qOwggjmKtcZJYg5wOnTt/8AWqg1wxtBDuJXdkDPTr/jVfJNJTWgWFAp8bvE4dGKsOhFMpc80xnYeHdXnlkWGV95PA+bmuuZGZQVwvtXm+guov1V1DI3Xk/0r0eJAoDAkLjPNYVVY0g77izoGhVBgkdzVQ2xKnHJ9aWe4eRioHAqJJJdmwHkmuaWpbswaANkEcDsKrNb3G/zFX5Bxya0TBJGiNnLHtUdzveMRrnPtQmKwiX0awDOSw9DVKCRmleQ5254qneM9tMF3YbHSkgld4+4ye1CgTbU6Oy2KJJhycVLBLI5JI2rnpWdbu8aDYcKetWlkZH5ztapZaQ8ktMwL5X0qeztAd5IAHYGse+vfsk3mY356KtVpPEdwqbyVX/Z9qapylqipS6MsXsclvcFGOVY8ULE+FKLup32k6nHHMkeRitKziKx5cYok+UT1KtqWBKstT4UNyeB2qwY1HKLznmmtCpIL96x5ru4EPmB/kAwKgeZo3wh5HrUkji24TBye9SxeTdLnbhx1p3ByM5rkq+9wSe9TzRxXca5TDEcEUy6hM4cBQAvpVa0uC7mINtK8c07dUZPcrPB9ncbzkHpSXcuxFjjzjviid91wVdskHOabdyeY2YxjjH1rRDvoUZWCxhs/MaZEstwzDOSOgptzGwVcnJFP09p0nMioWHfFW9iOogd4phvUgit3TgblSfM2n+7VG+cTCNgu1jwTViKNrWNX3AMaiWqKWhqyWxMDLJjGOKxY4N5YK2BUz6q8gMRbjpmli8qNN2Cw7ipScUF7mRc2ssW44yOxpLCZzLt3bcVtvNFMuGX5TVaWwgeRZbbgjr2q+e6sybdSEsY5ckk571YW4IOUH41FOgVueSB0pqSKEyfl4qdwJJLyR325pHdpEdSeQOKrqwZwQDipJiqkFDyetOwIYY9vJbt0p1uh+zO+/AzwKttBC1oGY/Mao70iiZAcii5RBbTbLvceee9b/2lVAlAXLdq5NXJuBg4Oa0jO20AHNOUbslM15p3kYfKOelEckVsPnXLmoN263V0+8o5qs05VsspaoUSrkrXzb+PuZ6VeYNqdoYogEUjrisy2KyzHcmFNX1udhEUY6dMU2rbB6mR9kNneYfOB3qws4M29h0/Wp9VWSePKLhh1qgqlEOTnirXvK4tieS8SU7WPHaoFyZMAZJPBNVLZg11hvu5xW/c6YEijaJtxbGCDQ/ddiW9CzZ2DSRkTkBT3pX0pEdTuOw9PQ1LELhbQLIAYx6Go3ea7AijO4L0NY63IcmyYCDy2jVlEg+lV3gHQyEjuDVZo5Y7orMrK3971qvcXUq3Kxsfl9cVSQkrvU0PJ9JiB6UUgVioO1ORRTHYrPMm8EDrVabcZQUOM1M0PkuxIOc4xToraScsQMAVd0lc2ZCpkXPzZA71atXzJgcnrmkMAjRoyR9aWFSVCpjPrUOd9hE1zLHnc5OewqkyLLGz9SOtTXa5kVDgkD71TSxx/YGKgBiOTST6gYcLsk7dQDVwu90PKGNqjnNUVmRN2/twK1ZYEh0Tzl4kfn8K3a9640Yc9sXJ54XsKhBgWTg47Y96m+0xLDycOKijSKfdtwWNXa+5O7H2935U/IJXPWtnzZLgDy1+Q+lYMcLlv909K6qxUC1WMKMnvWc/d1RSTRLbSC3blM8U23vpp75w/CAcVaOyMKGIPrmoZLm2klMUKgMRjNczlza2FK5FKIJT5QALsetJ/YMER82YjHpU0elOjK4fnrmmXguHbyUk3cVPM72ix201LELaYtu0SAFvSq93OsEOI1Iz0FV/sJtChdcNnJNW/sqTyIzscdQKbtuxIr2TTSYZlIb3qhd212ZnlAOFNdQslvkRghStOlUMAFUYPWkqtnsO1zjY9SkDGRwfl7U+41gyqNrHB4xWnqcdvEhDRYJ9K5eWARyb1B25rogoz1sJ3RvqGNorRgEn0q5bQTTW+JARisqC9jjCndgAdK1DqyNZfKcHsajVaFIWZ1gYgkk44qEXGZBt5NUb2dmCvnOe9OtAXXzCeBRy6XZnzam7aTNKdz8DpTbudzlOqg8UlqxePYE4Hems4+0Hjgc1DSbLK0+VA4yxOTntU2LScbvLyQMGlkQTTqqnBaoki8oSAHoetPoK5mS6Ixut0IwCc49K0rOzELH58uO1XFm8uABeWYdaSMiBPMYbs9eKHKTVg0FkheGIOD857io0haWYLtAGOtH2zcuHHB6VdgVnkVkT5QvNTstRrUztscUjxlflPUkd6WztyM7fXIq5dWplYFRgZ5qaGVYE8oICwHFF9NBbCCUSKY5U2kCojEzJmPlu9LczM0YO0bj6VnpfSRybWyuOopKL6FXJHtVlYIDhhyavAPDAyPgYHBqsJUceaikFqJ7sllD8rjFNpsTZCb5DkRqC+aZc3fmRsMfNiiO2CTRuCAHbmqtzCw1Aov3ScZ7VairksqXFwZcKozxVezEkdwxKnNadxbwW+QG+eiyuoViKyAbicA1pzaaE8qvqMkZ5oS+cOPWsqV2VmJySeK17oRRrhSQx5rNYM5LJtq6exTsRxGRFIVcHuauWtuZ5AXHy9uaiEo2AAZ/vGpLfcswXew7im2CNk6bDIF3EDBqeVZJEFrbv9R2qhJcmPaGbcO9IbvyW8xSyhu9YuDepWhQvNOvJbnoMD0qRYzbQKs0Qya1YLtJk/dZye5qObc5dWALY4qlJ7MWhzMkrJdELwh7VatHjgulwOG65qePTUaUmXh+oyaoXoWG4UITkVpo9CH3NmSWAS7c4U96u6fdLHwCRg1y8tw0zJuOAK6TRXifG5c5Hen8K1NISua7ahtbyicE96hfUCz+UvPvVO7iZrzeeAo4+lShFlRnVguBTvoaDNSnzGzDDZwvzLkA8/p398e/HLzozPIwILdFIHbpk/wDfOffNdTPZwG3T96rNsLtuI255z169Rkc8DHWqL2yzo8axBkiOW6b2zj0Psef8cDeKsjF7nJs59M5OAP0x/KmOwO7J4xnn04/rVq4sJolDMN6lvmKjp0/+KGPeqJRsITnB7+/t+YpiFRtzYJ68Ee9XoY1mVo8Z2oWwOu0f/W4qkFKlSRwOCe1b2lxm3KSAH93JvcHuoOSPy/PIpAZGoaUbTODkKFLHtzj8+T+n41Hp+mPe3kce1tm3zHwOdo6498c1tzBriKSZmzNcspHHCj5jnj1yv45rQtVjs73MKMyxoPlJAOCAPY9+3T3p3Cxk674Wazie5tTujQHegyehOWX2x6+jc9K5csRxXpNvetE48xPMijLHle/fjrjA6ccY9DWPd6Bp929y8DG3SNtqbfm3En9eSPTHHY8F+4HGdaK3r/RooYx5RIIOCSeO/wDk1jvbSKSNp45PFFwIaKKKYBRRRQB0nhiO2l80TICyEEH2P/6q68X4aXy8ZT3rz/Q7gwaii5+WX5D+PSupWMreKhLbQeawqrUuJtBVdDKrcDjFSBFjiDsOe1VopVCMEGFziluiyxrk8YrBI0Lnnb1Ug1WkhmVywYZbpVU3DJEFYYC0wX371SHzmk4tBcp6lt27pB8w6ms+G8CMI8cE8Vau3kvLj7OsbOzH5QoySfQVPpdjLLfJpj2bLOzhcMuGBPtW0fh2FFXZt2FuDGkj5YGtO4thsP7sgY4JFdrb/C7ULW2jeK+ikbblo3QgA+x5rC1ixu7a5NnOiqyjkA1hOEou7RpFxexx0mlRzzxhnKK7YZ/QV0p+HenatAkFrclWUcODk1jSPE8pjm4UHGRVnSbm50i8aaC8ZkPRSc4o9/ozSDjqmiebwpN4XC28svmRnkN0qNyhTCjmrlzfXesyESM8zAZwO1ZxiaNyGyCPWs5rWzepEo9thVDeXkdajdzjJprysGwvQVCzMiHcDg1CjqTsPuSoClQCT2qoLtRNtwF7GmukskiksQops9kX/eJwAMnPersjOQ6a/VlKKMAd6zpYcMJEyN1TW+yZivGR1q1HaLIuN/IPApp8pN0zONrLOd64+Uc571aNqEtQW64qaWMWjhTyD2FLOphjXfkq/QCqUtRnPPIN7K3TPFXbSeTT1zhSHqG+tGRDMQQAe9VppiURT1xxWjV0QnqalzIJ0yAPWmG4+0QAHhl4NUIZ2LotXVji8zrjPWpaKuVnTBOxc4NXY0mFv5iDcp6itGGyt2QAOBkcmqxf7HIYVJdTU819gsZsk6s4A4Ap8U5jXG48moZLeQSs4U4J6GqrM4lIY9KtJPQVzdiWEx7pOSe5p989pLECqhdoxxWYsplUJuwD0NRSkhhBuye5qeVg2TgeVtXjDdKSW3Ifcr545qO4lBjVe696kHOGJyMc0xWK0Vz85R34HFVbidUlZEORS3MKiQsnc8U2Wz2IsrE81okhFVH2ygk1oICY8hveqUcHmSkgEjtWlIg+yoMAFetOVgsaWmyq6bHI+Yc1pGygMRCsC2OKwbFwGJxkYq212wlQJ92sZRd9C0aDtbpZlQgEg4JFYiXDLPuU8g1oXCB4wyA5btWfLYSwuhYja1EBSNZZxJFknJPWsjUQUfCng1bt94bA5pl3DgqWGaqOjBbEumRxGwdJIxk87qQSyR5UOWRelUZLxreRogcKwp8U6rhh84HUUmnuS9UaVvqMhjwN209Q1XbeOe3ikm3gZ5A9axDfQm7DMNsJGMdKsPeReaoNwTD060OPQz8jVhvMwtcTxh3IwuR0qjOXubcvPCF2ngipVFmzhBK2MZXtToIJAsonU+V/ASamyuIoKke0fvXHHTcaKcySB2HmR9e4oq+ULnR3OlebhyQCTUi2ohhZSuM96VJmaTc7fKvekmuTJE2cc8CuJ870Z2aGNdQAORHz9aqxrN5pAXCqOa1fMieB12fvF4FQpII1bcvOOcGumF3ozNozXZpDt2kmrJANmfNyuB0qGCURzK8gyP4RVyRfOlzIRt649Kq1nYSMKW1H2QzcEZ6U+7upLm1jhHG1cUXWfOZQf3ecgVf0/TUktTNL36Ct5OyuK/Q5K7haGDkbie/pVW1meFiQetdHqix+Z9nRDzxWYmiyvL5anGelXGWmoW1IkumDgk/WuqtLpWhQKRjHNY8mjJb2peZW8zoAKXRVkNyEKsF6EHrUz5ZK5cXZm3eLJcuhiJx3xUkllHFEsjZVx3rShsxECYuvoaWWF52WN9uDXnuor6FtLcRLlPKVC+cjrTVSKN92ctUbWLwsETlBzmo5LyCMgMCT7VFk9ibloSrcPiVfYZodfITavJqh9vjYk4AVelTJcrJ8xPGO9VysRVlR5ZQQ2O5qY37xqIxz71mT3R85jnAJxUu+BAhDFietXy9xXNCVBeQEMwBHSsO9spCQmOPWrV1exxy4Vj9KoTatOzmILgetaU1JbAyOTTxDGHZ8gdqlgured/K6RgUSRSXVniLLMetLp+meVETNHyvU1pdWuxJCXBV1KIOFp0EoER5xg9KuWNsl3csgO1QOtPa1tra5XA3e1K6tYnl1uS208kTDB4IqWc5+VfvN3qKUZQkEAHtVYuV2tuyTUJBdmjBZSs6tvG4VFdK1uXyc561NaybRudyWxVO8bdnc3yk9alJ3G9gjmfYGBFW2kJhVTjHc1l+dGsQQdfWpIZjtdTyKqxKLqvGrqDhqsNfmNMJxWQZVVgM05WaRztBNJwTDmNaO+ZiqDkmrLOqnkfN3Nc8ssyvnkFTVhriZ4mbd0o5NQ5maauocsuGA4xTJreKV/M2gMRzmq8MckESS5yG5NXIY1vchmKY7ipasUik0vIVei9RVC9uMrgdFNat3YmByY+eK5+7favXkmtaaTBo0Ibr/AEbaeW6ioEuJwdxTduPBrLtrl9z46DitW2uAkSiU4IOVzTcbEXK11MfOkMxwdvAqlZXO+Py24YNwalv7hbi97Hjms+FlWc46ZrSMdBN6nQS2M91CSo3YGc1UWwuEPzIwXHHNbFnc/wDEtcqcECqa3szpsJyc9KiLexppYbbbUHlyINrdfar0aRxy79nygcE1EIcqJCMeoolmUgjPCCjcNCndOkl3uJwM9KuMybVyAVAzWPO4dsk4+lCXJA8sk/jV20JuaDXab0VMLk8Yq/IAiCRpAa51ruCP5iCWpYtQ2nLMWQ9jQ43EmX9akVYRKjnzDwMGufWV5eXOT61t/YX1JA6N16DNXk8PlI0UoMjrS54xWo2mzAt4wxG/NdfYC3jhjxwT1qhNaW1pIU+8zDp71bt9PuY1ErkbQMgGs5zUioqxZ1TLQHy8cjk1k218kSOrjJAOa0FuhcKUZMY6k1nalZww6TPc4+YAgH1zWtNaWZrd7lG21Np/llA2knPY47/TqcccDNdHZSwiya4kWMJwzb2xlj0wuc+hz2OR0HHB2sgEi5bIJwc9P/1V2ztE+l2q+csgnYLIqsCcAYOcHtuP4jqcc9hiUI4o7lEimdtzKWdQM5YLn17/AC/Tms+50tNzBQFDMVHPfJ/Xr+v1qVHLsryYwzbc5BGCRnA9ACvSryJJJcxxwECUyg8ScAZwPyYkCpGY0mnvAsTTKFUqpIYZAynf368e3rV6Jx9nkQqI3lTdG/A2jLYZs9uB+XpWvdLFqkTNkRJ5u0EEgrGPlz7gEfXrWVeLNkJDkTn5uFABGORjr6fXn8QDHt7soY2kUo8BJ2kcZDZ6dzzjHoB6VrxykxZll3xzEhipOXHBbnjGcMfx9q50qZdQiiKt12Hvx69PQ/kBXUR20kFqryAjdEcj06AfoBgfngUNCGyPJKEc5zOf3iYAAJHQDr0BBHrSxeSkKqXBLgAjHGSD19uB+IB4rOurwSTRuxCxgtgJ3GQe/wBf0HWqsN2zXMbSNtVcg4P04H6EfQcUhmksJuk80plCowTnLHBzx+Bz6AfSo5dJd7XK7SwJZ2GTt57Yzn+PIHcA8jFaemE/updzsfmO/bkJuOD37/MvfgnucjTjso3SIOUx5uPLDDAQDkEehdiO3Le9AHnGp6dLYy4dHA6ZYdTznGOo4rPr02802G8t5Irn5FbZGrZyx2kDcM+uOSPQ8V51eWslncNDIOV7iqEQUuKSnrVJCZJbgmaPb97cMfWvRXkFzGHRQpAG76153CpNzEo7sMfnXfWa7nyM7QeawrF0x8EMgUtg7B2pXkZ9quCAK047drhQsfyoOoo1GzMUccicr0PFY3VzZmQ4Eg2A5qs8KRFWGM1pQ7ZXO2Pjpk1bbSUv4QUxlTzVaCsdr8N9OsrPQbrXJo0kvHZkjyOUUcYHpmrfhzTLbUvHYv5R+9jBY4GBnsKd4b0xbS3s7J3zG4ywB716Ha2unWCDyFiRu5GMmu6PLGnaxz6uWppzTLDHkniuT8TaRb6pdW7su2Rep6bh6GtW8m3pv8wEIc4zWVrPiTQo7MNdTHzQPl2gkg/hWNRLltc0pp817HlPi3TbXTrxIY4vLdycnPBrmZ5JrSEs6uFxwSOK9HuNb0HV57c3UGZ42+QsOa5z4ja7ZxaX9kt44y0nAx2rhhPXlsdbirc1zmND8UPbXDkglWG3itaS+N1+/wAEA9q8+t3ZHVF6n0rr4JdkSAtlQMmnUpRUufqZe0k42ZeJdV3k8N0pkl8uzy2xVdb1GYh+AOlY9/O6SkqDg9KhQuyb9TXNxuBYNlRxioZLxlJVmOCOKybSZpQVJI55q2TG0qoThQOtNwsS31I4JgkxZQc5rZSRItsrMN5HSsyWNY3GwZHaoZZiFViec0mrkGszxyShnO4+1PjIkBPXB+UHtWbazAAuWxirFo5aV9zDnvSa0Bu+xdvIkubNg+MjoK5DUgIZ02HPHSunuZQgCK2cjpWLHp8dzct57FWJ4FXS03JepQtJNp3n71aMMZkb5jjJ6017YWcrcBsdKZ9pCRgtwT2q3rqgTsaqkLn5ztQVJDNDHbu7gFz0NZEd15kG0DgnmnSyAR4B4FZ8pSZekuFuIiCAPcVgTqUnKkk1djnBTA9elV7hhudsdquKs7A9hm87AvcUpSVHWaQkCqiSkEmrDTS3EPz8DtV2ZOhOCsyuAck9KmYk2nk/x1m27tGxK1fZyib/AOOpaswRWZnij8txyKdcXhltVjIHFVrmZ5JNzd6aqPK6oO9Ul1YF6xDbCeM0/erblc4PapIrNIQAZMnHQGs67JSXgEVK1ehWxZR2hkG08VfhXFsxYfOx4NZUUo2qW5wauT6gBt2jpQ1cC5BdOqFG5INNuXaeINu5U1QileaNyvDE5zViyLZKuc1PLbUQttdPHMMdTxzWnaRi81GKGYkK5xkVky2kv2tccKxzkdq2dOxDdQs+SEbk09LoNkaOveGbS0sZ5Vjbcq5EmTXnUU8wztBwe9e066Vl8KTyN90p3FeQ2drPGBuA2sODXRNJIyhdlz92tgDIMselR2MJuWdlHyJ1B71IbSXyNuN3piprRo7ZsBWGBhgeKwHJajpLhfJDRfeU9BV6e6vb+GGGLcqkc+9c9c3SefIsXAz0Fb+m3L5i39QByO1ElZXEJ/ZF4efLP/fRorUad9x/e9/Wio55diSUXYKFQw47VnS3oLZYldpxgVg22rIAFZsE96bcaosZ2r8/rVqnZnQ53N9bpD0JBJ60yVJY7kqHJ3CsuyuIbkKQ3zg/drbkliF6iv3UYpbSsNaxbIIVfz9rJk9q0WIihJYjdWZ9q8qSTn5icAH0qa0MlwzSOMxoM0JNyEjPnXfJwOScYrbtpmt4grr8mOKxpJQZw+QozUy6g858pRuNXU12JiJfOxlaWOIEj1qppsN5cXpuc7FHY1sRxuYWLgHjj2qK4/0exLxyfvPQUk9LFpFh7yFpER8MRVuGK1SQ3IUBj1rjYrnOWckPnvXaeCfL1bWDaSpuwmRkds1E6TeiBuyFl1GNZAqnFZ811KJQ4bCk4rV8daedNvk2IFix1HrXMNcJKiDJwtYOjyuwJ3RtTXxSEKTkGsp7sMXkZPlWonvo1g+Ugt71iS3kjO4dsKewqqdIGzSiuIJpTvcqPTNTJdp5m1XJArmzPtbj1q0lymOBj3rd0xGjLKC7Z6VWa4aNfaqu5mfcGz6U2QvswTxmhQJbsWXkMxBGSw6VsWVisoEk4IbtWFbTm3kVzHkV0kfiGBYh5iqO1TPmWkRrzLaSRWzIkaAZNXblPM2RwY5HzVzFxqaSTB4xkZ6VrWmopFbb2BDk81nyNK4OQ24im0/c44U+lRxNu2k4LGm6tqCyxbVfIxmseynMtzvMmOOBVxi2rsiT7G5JJEqFHHzetUg6kn1B71Vmuj5oXOTmkbdI/BxTURcxom7+XA4NQXcxKKCaorc5ZkPYVEl3uBy3TpTUAuaMRQw7m+9mpg4XLCsyFjKDz0py3RXjg4ocQRbkO6RSD17Vp2Qy7EEDArHgY+csnB5rRSOdDJOpAQjkGk0jSELli4nQWsnGWz1FQJEzWMku4BfSqck5W2f1JqNbwmDYMgHqKEiJO5q218DEEbJ9BT5b1ICQGwevFYSzMJBg4xVa8vzv2Hg+tHs7slM6O41NpICVfnGMViTHeBk5bqagSaPy0y/J7U25YowK9CKqMOUbbIoZfLkbPc1auLkSw5Bwyjis12xz609HLR1py9SQEjKxPciiMnOadtaQEqOntTlXMYHfvTuVys14J2WxKjqT0qrdXBhcNGcEDkVPEpNiWjAZu4zWOWZndJM5HrURWpXkbdrqkbACVs7uoqdrd5NxUnYeefSuftljFyu45UVtXWoCOIbGycYwKbjroO5HPAohLRtuK9qotNFj5/vVDHeSRMzHox5FVJWDSFlzgmqURXJZcFs9qWLa2VyRmoByeTVmIJsIYfjVMhbnTWF1DBEiqcbRzzWnPqOI1nicH1WuEklKy/KzEVJHevGynJI9K53RvqUpWOlWY29z9plXcGOQD2q/Hqhv3EJO1D6VgjU2vl8r5VUDvRaXCx3mY+SDzUunpruNS1Oxlsbcohj57cUt5py3Wl3FqP4kwPapbCRXtkaTCr1JqW9k8sBlPykdRTi3Hc6VG6PI2RoLhonG1kYhgexrrtH1OIWptjneSCrEcqAc545B64A7muTu5zcXs1wesrs/5nNSQSvHgLnGemetdqehzHTOsRnL8pFtwwyG3rhjx68ccdOM85qwgEzFSiB2RQxXjLYI298/ezzk4jPc1zkd8GnDSOmc/OQM7uc9O4zjqQTWpDfniRUSOOHc6pnJ68ISMAscleBwOmMCgo3XVdiR+ZhDGcoWx/E2Tjt9e/pSxQFIWLtltyyEktxtBG3Hrkk/kPUVm207RlTJJubI5OdzADGB9RuzzxgjqObVpqaIGDoJPLBYPydxz1OeSOGX8RnqcAGJqskFheyLHGRLlhlhySf4vY9fzA7VXl1OSdCgO1CScKcDGOn0xxVDVZXl1GR5DmQsc8+9RxI0vyxhiSOoGcc0gHTOQmQfc1BFvkkwT1Oat/YpZoGcAgLjAP8AEDgAj15yPb88T2NlIA021gq/8tOmD6/Udvw6cZAOo0NY/sSAFjiVT8pIBJ6j8MEk+i/gbzzLBbAGQNJJHh3Cnb0B69O+e/07VmxQ28ChgzK0QAJJweQBgDHoMH0BPWkn1GNofKEoKhSTnGGbJ5XHGMZI9AD68gCSTK0gkcuUDlB83QggjBHbAGeeoPpzzuowrKW6si/KjEdskZwPpk4q21wizuG3GFztdehIyQGz69/Tk/QwSxzxt5JAcEEYx7ZBHXH+H6sDBkgaI+opmcVumI3dhIm1TKoGMDDduSO/px7E44zhSIySMrDDKcEVSZLRa035tStwe8grvObfbHGOW7155bSGG4jlHVHVvyOa9Z0rTGvpsQAyTMfkSuaurs0pk2mRtDOA8mVYflXQiziazMcuCByD61teEPD1ta3EkmrxJ57f6sN0ArvvsGnyQ4EMZTHYVgoNm9rHgF7ZfZ7oKBhH5GKsWPnWcnyrlCeRXXeMdFtW1CIae6KUBZ0Brmo3X7QigZ5xSbaWguVpm5pr3I1awnD4gJKlK6K98IQ3+pNqUeo3Vu0ihSiP8px3wc81laVAtxrNrbxnKwqXb2NegR6aJUV2bao7Cu3Dvmp+8Z1/dn7pyV7owtrSQLrF3GwT75fIP1zXkeof2hJc5ubzftPC9Afwr6B1aG0msGtVkihRjtLsRyf8a8q8U+H0069Q7klUjIYccUV6ceTmJhUlexxmoSm3sWnDkSKMgjsa48S3eqXXmTM0hX1ro/EDqbZljbv0qDSLXda/u4zk9TisaFNuN0huaTs2UVeVJljFrtHdsVNcXkiPsU8d66FIMQMG2g479q5e7g3Ssd2cHtVzpyXxITknsyVb3aAGOTUV9etNsHYVEYTkcU82q7ct+FQoq4tRLSd45flwQaszzk42nnvWd5bQHO+kWYqvrzTcdbhc1luTGhJbPHFV5JTMQN3FUZJy0R9z1qRIfKgWVWzn3qeQNy6jbH25ODTZ7p4ZgY2wKrpI8sm1uoHWq0r7pQgOSOtPkFtsacF3K04c/Mc1oXcq4EuQGFYMbtEWA64p7XDPCN7cipcNQ2J7q7MjK+e9Q3DCQqV9Krl9yYoUsGHcVSjYVkWbWUghAMYPNXVlR2dcdqzDMItzDqaSKUryW69aTiCLMbAO4zgjpTrgpLajYct3qtIDGdxPBpiX6xJIhXO7oaOV7jXYhBIyewqwH3rgHgDoKq7w0XH41LDIq8HpVNC5S9axHytxXjNW73agUDGGFV4rmOO3b5hz2qjf3yylNuRtFRytyK8hJiGYKO3FSgPGy7VwwqlBNm4DMPpV+4nPlh1+9irceg0uo5nczK5b8KmvYw0kZPQ1nRXDAHcOSeKvzLIUTnPFS1ZgymwIdwvIFSRqiwF3OSO1NXCeYrD5z0qu7Hpn600iGX4bxcOEG2ptKnD3mW+7WOsgVsYq7A3lyjHANEkJHQ3vlhwY35xVVb4CZVXBINRIrStyflI61WeEQSAgknPWski9z0a/u1l8JEnGCvNeYy3RMnlI2VHSuguGuZdPhiEpEbdVzWVJpsIbcJNrDtW05J2JjBpFdb6ZHARWAA7ip21Az2zQmMbj1YVRlmdSwGDjjioFuGQkgdaFFESYnlhZVJfAJ61oW9wYLnZ5haL1FZx3PGelRKj5wCc96pq61JR1fnWh585/++qK5Iu4ON5496Kz9n5iIgpHFLyACOtBzmnCJ2AIVj9K2N7GhpcaxyiSQkHjpXRXYja9iIfaSowTWDYW1wo8x1O08c1s6nGBJAXyGCjGaxnui4p8ruGoWpk+a2O5wOa1tFieLQbhW5k71UaCb+0o4YQTujycCt63Q22iXGVw4yCDVJW0FbqcPfHJCg4OaS1ujDLjgHHJFU9VnHnEd+2KpwTsvXkn1qmrmfU6+3u2mjES9T3rSiso0gPmjJIrltKvI4ZQZFJ/CuuuBf21pHc3NpJFbyf6t2UgNWMlY6ILQw7nTI3uFI7HOK6fwVLBpuutcy/LHs25/Gueu7sBSY8Bz1zVjRFluJ1/eFSW596nmkncnl5nY6vx5eWurQosbnIbNcT/AGSY4M7+tdF4jtZvLUxjKrjOKxnndbXDZAx3pSk2wcOVHN30ZifA6Cs12JHNal4XcE9RmsqRxnBraCMmRE5p8eeR+dRE05ZCAQO9aiuKXIbgnFSeeBgY4qv3oKtRZAa6ajC0DROmCRwayfMJJBbNNIyOetMIxQopDcrllJ3TGDW20/l6YuTl2rng3y4qQzOyhSxwOgpSjcll+WVmh68471Wsrjy7lT2pjS5jKk1Gic5Gc0JKwJXNG4mzN5i/dBpFumcM4bGO1UwWwFbpT0QOxUGlZDUCdLlEgcnmRqScqgQjjIqnIpRiM5qRyZQoY8AcU7By6kkdy6ZAyfpTBOd3OcU1dy8IuT7Uwo+c7TRZD5TQjujGUIPer9xfyMUzJ+764rFgV5229MetLI5Ryh5AqHBNmidkbTXQls5GUfjVRbhXUAjBFVFuitsYAMAnrUiyxxuF4II5o5TK2tidJ/3u4jKiq0+Lq5IQgD3qzdT262/7vG6s1Rlg2cZ60RXUrlSdh7kLKFB6d6neYeUoPWoHhVQSDSDnAbpVbg4aiz/vMMp/Cp7Eb8I3TNQqgwcDNT20xglxtyaHtYpJItMfscrxEZVu+KrO/lMSOQaL25eRgAKpSSO55pKINpbF22vDC5YscVBO4dmkUn5jVcBvQ04kkYI6VVidGWFdYgCvU9c1JHMHDBh+NUlyx5zU8Y5OTjIosToRySDJA6UwNkYqRrZgeoOaYqEE8U9BNWAdetWI33e1V9vNWbUKGJbpiixK3JmjBQHvTGjBHI6VI77iABxTd/GMdKdiyqpKvwcVt6BGjXn7wZBrFkG1tw71s6KiyzqN2B61nUWg4K7Oud9oECdDUd7dyLpk6KcMkTEH04rOczQagqLvdD1ajU7rEMsOzbujYZ9eKyWtjZu2xxLYDLjjCimDJbAUs3t2pZTh/fFTWQDzICpK55G/aT9K61sYE9lptzdODEgx3LYwPw+nr/8Ar0ZLu3tIxFFJmQKFLJg8+uPUdvqTyck09WuZIWNorSxIoAKb/Tseev8AL0FYm4g5BNAzq47uFiYz8oU7uDnt69zgtk9/yNaIkjhs5TGQzyIDkNwynOe3UHIx7+vB4UTyA53Vbh1F1JUsQrDB7igY+7fdcFgOtbujQIscxZARuHQ8Y64P/fJ/M1zspBkViSMtkkfzrqbS6svtbC0ikjhhiAEbvvEw3dcg/e5Leg2t0xSAseX+6Id0jQcEYyZOuB37MT7e/AM8LGO3luJdqKGCqcAHjPcjjp7nuTjik88JOqbsMDuyMjAyACCf4u+exOOO/M63q/nbLW3bMUQwGxg/QY9sD8KYF/U/EEYjMUYzkbcAnp3J56kD/wDWea59tRkMxfuTn8apnJOScnvRjiiwi6L89CDjp+H+RV+11OM9dyOQRuB/X/H2OKxAvNWIQAw3DI+tAzpbcx3BTy5kiOOUwfmGeRnHp6nHT3xzmoKUvJFbJOepzz+ddDY24SxmlEhYMvKwsCwH94ruHAwRyCOenORzl3kyZJyfUrtJH0HFNMTIVHy/jX094BtIrf7VduoEoUKgPYYz/n6V8xJ2+tfSnwpvRq2jC5kcNIV2sM9xwaymryRUHozhvG/i3Ujcz2+8xskx2+WSCOa7GPxdJ4U0iztLuVrmWaLczFskH/8AXXJ+OtNjOu3sygcMT09q8yvdZu7p9s0zOE+VcnoBWfsnfU2VZWO90u4uL7X7i9FzJJ5pJdd2cAngV08UcEbpKByDkg15r4O1RrfUT1KsuGFd/FdxyOFZSQ3Ws6is7DU9U2a0HiS18MXVxqNwmQ64CjvXMa78bdSuQ1vpkZtkbjJOT+eKg8duraGVVMFCDn2ryxFLSjA5z3rqoS9yxhWtKdzrb7VdV1dInvNRmmCHKIW4U+oHr71vw6lqN3ao15cGQqu3cTya4+KTzI8D5SOorp9HgIsEkd8qexqcQ24kxWuhUmtxOkjjO0HnNXLS8S3RQq5XHQCk1CSQLshi4PXFUbAsshRuDnvW2BlZNGdePUdqGrAs0YjZSw61kO6pgZrZ1aFTbZYYYHg1iSQq/fkVeJvzWYUbWJY5FXJLZOOKid2l4B6VAVIGAeaZ5rqMkVy2NrrqWmiBAOcmongZRtx1oil+TceK0oHQxMGxkjjNFxXTMuRF+zncMYpqzEwrEelTyO6kxuowTVSQHdkcGmK6JPmjbeDQixp+8HLGkU4gbd1FQo7bwf4aAuLJIzPnOKbuJOM1FPuYkr0zTY9xHQ0WFLU0LMB5GD9AKjWURuxPQ9KSJJQDgHnjpUMkUyvho3+pWlYfQV2Jk6/LUiKCR6VA6snUc1JBcbQFK5OadhxsT3rsWVV4XFUSpB55qzI7O3I5pI7d532rge5pLQqxChwcHvT2cCtaPw9cCMyNImAM8VjzKUYr6GhNPYUk47iAs59qZIcHnrSCQgYBpwG9TnrVEpmhZeSwCCPc5HWrd9EohUAgMR0FZsEwgUbPvdzV6zljy7yfMx9aza1uaraxn8DG7ORVq3vN8y7zgDjFQXOJZMrgc9KZ5QKk9/WqtcnYuXQDTmQH5R0NUlyQ7A5zSG4dofKY8CkRlVcL+NCVhXTGFinBHNTRy5wSeaSeRSmByahRSxApkNWeh0lnMHh8rHJrVitI54QGHTrWRpkTll2r0Fa9w7RKoHyk9a5Zb6Gi2FvilvZJjJK9MVzdz5rSBmJw/pXQTQs8O+WTjHFZv2V5nAjwwArVOyEzIlAT5V61VIfOQOKvz28kbO7DGOMVSacBCuK0iYMdGrFgV5xT7qGaBxI5CluwqqkzL901JPJLclWkb7owKYER2560U3Z70UwPRYPDOl2yZlYOwHOasR22k2yHy1QE84Ncm82puQ7Fqme6kYDPUDmuVxb6neqkVsjR1C9gaMJEoUA9QKh1JBL5Tu/G3issyDBBHB61e1EbltgCcECrStYhvmuzq7WeO0vrR4wGMkeCfemahd/LOrkbieRTJ5ooLy3EMJCxx7jxWZq86yoJlPzSHkVqo9TNzdkmctc2n2i4dg3U8UselOozuGas+UQxGOavWUIkByuPrScgXKa/gjwzFqev2cc2HVX3uvqBXs3jxLV/DgtJFUMCuwe4rzTwXexaDqUmoSozRqmDtGcVsXmujxTfNcISIE4QGuWo25XN4Lqc3L4dtZxvORx2q/o2kQWIAzuYngmtNbLbwHyPSnfZ/Lw27pzUuVzSyIL+W3jDLLIowOhNcHrWoKzNEmNp6YFdLcWtrc6hJcTsTg4xniq2oro4ThBv9hVRaTuTJXVjzud5sYJIFUiCc5ro75VLsI0G2s17cvztArqizllAzaSrJtm7CpoLMsDuq7ohRZRwaXBxk9KtyWjhjtHAqMRsOCDRcqxWIoYcVY8pjxik8ls9KdxcpXGQKKstFgcjmm+X6Ci4uUjQZPIqxvAAwMVIluXHyjn0pfscm09BUtotKxA5xyOaSKby3LAVZFk2wkuOKgKbcjg0aBqROQ5J9aVU5GTxR+FHzY6GmK5dtHEe4Ej6miWddoVSCc81SwxPQ0/yyDnBx9KViuYewUHcr7T7VCSeSTmnMrE8DinR200oOxSQKZNyIEscdqmaI7MgjNRtDInVSKequV5zQK+uoiRup2kDnuaV4mjbB5PtS4ZiMnp0qWJZEcSFcgetK47oqu7EjIPFGcpVyQNK2RFjPtSC0ds/Ifpii6DcrpIyjg0gdg2ehqf7DMDgRvn2BqY6bcFd3lP9NpovEh3KjMWGSaYr4PNX10u7Zflt5SP92pV0mbYN1vL/AN8GjmihWbK0Do7hSvHrU7WRdnKLwO9bOk6ZEziOeCQZPUqa6xNBsGUKgYCuedeMWXGD7nmUUfO3bzUyWmTnHANdve+FxuP2cVTHhq6jiPyHcapYiD6jcVc5i+twoXy1PTkiqcStnmusTQNSRioh3D3qKbwxqJIKW+D1NX7eHchvscyY8McipoVQSdMjFbDeFtWkc/uCPxp8HhXUFlw8L/XNU61NLcFoYrjAU479ajfg11jeDruVNgXA96mh8AyOB5spB9qh4mmuoXOOKBl+tWLCRbacM2dtdzb+BrdFxK7MaePA1tvyXOPSs5Yum9BD9NuLa9t8KAGx3qW40m2mtZN+1nEbbSe3FWrLQYdPYmME59TV97BZcDaOeCPWuP2iUvdLTsjwqY/MPoKtWDlLiPleo5OOPfBFV7mNo5GjbqhKn8DipLIE3CbWIbPGFLHPtXrrYSDUCpuXHHBxx0FUSMVZuVIlYFtxzycYzVY00MSilpKYEqtuQrx7Vp6dO4aA52Bd4ZhwApGDx9Cf096yFJBBHBFbEDLJpDPhVKOFwM9P8SeePbFJgXr3VJGspThluLj5pZCSCcknAz+I4x3x783x2qxdPKx/ecZxjk8YGAPyqsOepoAcMUvFNHFLkk5PNIB3oM1PFt3DjNVwfepof9YvzEHPWgaOklMMGlkqy5YjjIAU44bHY+/tyeMVzc/Qggg+h9a3NRuv9FWD5w4GWG459cYPbnPQda55yc8UxMRfuivSvhhr0+mC7jjchQQ+O3Ix/SvNM10ngu6jg1+OGU/u7lTF+J5X9QB+NRPSLYQdnqdlq+qTX+oTeZgiTca8wliaS+kRRzuNekS2jS6mIkBJ5rlotKca4QUJ+c1mqvc2lFdB3hO1db5iyk47V26o8Ts4Pyk8+1XdL0K1it1kETB2HJxWmunwKuPLYiuOdfmexHN0Oav4Vu0KSfOh7GuL8QafFYXSeUm0MO1esm0tyeYzkVx3ju1i8q2ZI9rZ60Uqr5rCTuzDhsYLuwt2hwJ8/MPWuqtrS1hgQPlWjHTFZ+kWluk1tNswGjzj3rrxHHJED5QIIpzrNOyNqmkVI5O91S3gykKbyevFZCXUEmolJP3YcZDHjBrvvsFvnP2dc/Sud8WaRAbA3cabXi5471vhsVaoro5ajckZOoqTasqzCYjpiuf/AHpPMbD8K6zR0triBWbG8j8K1pPD7XABBUA9MCuzHT5LS7mVNvVI4NocpnHNQmFgNpXIrvV8JnJ3sTUw8JQEDO4muD6wja1+hwUGniaLduC4PTNSS2jQx53g+mK7+PwtbquAh/Onf8IlaN95Caj2+ocp5eyzscnkU0W5I3se/SvVE8JWK8mPIqT/AIRbTl6wD8qr60uwch5S0O/KpnpSxwPEoyM16t/wjtgowIF/KnroFirbhAMY9KPrXkVyHlsYcfLsHzetW3sLgQebmLA9Aa9An0KzklVhEAB2xSto8QUqsK49MUPELsNaHnRimEQw68c8Kf8AGoSLm5fGM7fRa9BTRZAxCxRgfSqw0C8ilZ4mT5j0xQq6C5wz6blNzkq3uKryW8UaDAO4d67+bw5f3DbmZfwqu/gy4m4klA+gqlXXULnBjjoOfem+cw6cGu6/4V+d/wA07EU1vAIDY89sVft4dwd+5xZvbkLtErY+tRLbyTqSEY/SvQY/A0ar/rM49QKcnhGZCdk4APtS9vDoPTqzzc2bhskY9jSxwM7Ywa9G/wCELjLAyTMT3q2vhCyix8mTQ8REEkeZeRtODyKdlgQqqSPpXqsXhLSwAxj59Kt/8I/pijAhX8qX1mJSS7nj5hI+YqVJ9akO3aqgdK9Xk8N6bIf9Uv5U3/hHdOTpEtJ4mPYHZHkDW8jucLgGpItPuXYBYmb6CvYU0OxwD5CZFWYtPtoyMQqPwo+srsJRj1Z4+mgahIwVbdsmtix8IXzyDzECjua9RW3hQ52gfhUuIVxk1LxDZajTOUi8PPDbiNMbh1NUNR8NX7qHiYHbyQe9d3mPqpphMRPL1kp2dymoHEW2mma1ZLtSCnWsm5a30/c9o2XHbNektFbyqwGOaxLzw3Dcg7AEz1xjmrVVX1Iko2sjyu+vpbuVnwVB7VnnI616U/ggdPNGDUQ8BRNnfNXQq8Ec/IedABTmnsfQ5zXoqeAbXJLTnFWY/A2nKCGbNDxEA5LnmHFFen/8ITpXpRS+sRDkEbRJniC7wDUJ8OPtA3Ln1roxbnplqRrZhggsa4ueSF7xzv8Awi4JBLfWrS6NDO6q3SLgVtiKT0NQwRn7TJnIp88n1Ki3Z3GX+LTcWjDF48A+lclb232zUEhPQnNdR4guxIixxYyBg1i6BFnVSGJ3Ktd7fLTbFLVGkPDtuOuadF4egVs5OPStwREgZzR5WPWvO5pdxKNjKuLRbfTbmOI43KeazPCEbfYpc54fH611ceIgR5ec+pqOOySMsYowgY5IFae0fJys0Ta1QCUIeal82MqQRkH0phg45GaVIyvG2s7j9pIqXGn203QFc9apnw9Yl9zZJ962xGxPCihomBHApqTH7SRgN4b09mJYEmlHhnTt6ny847VuMm09BQqNuzwKOeXcXOzIbQNMJ/49lH4U0+HdNIx5AGfatzyxjqM0qxKR94Uc0u4cz7nOyeGrEgqoxmqy+ELMDJYk11RhTP3qcsKY+9Rzy7hdnKx+EbAOWky3tV6HQtMhAUWyn3IreEEXdqUJCOtDnJ7sabXU52Xw3pkjbjbj8qiHhjS88Q105EGOTxUamLccL+NClJ6XE79znl8L6Yp/1WDQPCenMcmMn2rooxGy7nGDUyvDjilzS7i5fM5Y+ENLP/LKk/4QvTGYHya6v9xj5gaA8GcDOKOeXcOVHLHwXpuRtgFTp4UsE4+zLiuidolHGc0imLuxFHPJ9Q5InPf8Ilp27/j3XJqRfC1h0Nstb48jP3zQTFxhyaOZ9x8qMEeGNNU/8eqGpk0GyCkR2qD8K2dkB58ykDRqcBsUXfcLJGK3hzT8fNaoT9KaPDdgBj7On5Vunyz/AB0whFOAc0XfcOVGND4csATi2jFWV0GxC4MEf5Voq0a8mpPMXrt4ou+4KKRmf2HZKBiGP8ql/se0GCIox+FXzMrdE6UhYEfdpfMqyKP9m20fIRD+FKLOAHiJM/SrigBTkU5Y4zyS1FhcpUW1QDAjUfhUy20IXmJCfpUnkIx4Z6cLQAglmxTCxCIYB1gTP0oEMQJzGB9KsCzDHKkmkazYngtS0DlIGiQdBSHyxjK1N9hIOC5pDaYHJzSsLlIgy54UUxiWPGBUptkDc5qVbSEAlmpaC5SuqvjkrSZYHBAq4lpAx4bIpfsluzH5jRYfIVTnHQUzafQfnV5rOAHqSPrSraRe/wCdOyDkKQidhkKPzpxglY8YH41eESKQpOPxpzRw5zv6e9FkP2aKH2V+5FUtRmj06ISTPyfuqO9brJAqFieMeteV65qz3OpyM7/IrFU9AAa6cJh1VnrsiZxjFHA6kCuo3St1Ez/zNRRMQ2QwHuatawo/tKZh0kw4P4f45qlHw+CDg+hxmvRas7EDZGLMSTz6jvUdOkOWJplMoKKKKACr9jcxwK+6MvJ/AGxsBPGWBHzY6jkDOPpVCpoZdqspG4dcH1/z+NAEbszsWY5J5JpKVm3MSAAD2HagYNABR3oxijpSAcDirVoV81d/IzyAKqDmrlmA0y5UYz37+1JjLGpCSKYqV2qQMdP6cflWYePpV7UXd53Mh3HoGJ5xWeaZIoqSKR45FkRsOp3AjsRUfaheKGB7pZ2xmisdehwbW8tw+R/C38Q/Agj8KyNJhEniDewHLMcn61a+FOonUPCWoaVcNuFjMJIwTyqSZyB7bgT/AMCq/pFrENdnyPlTOK4KmjkjqWqubnAGBIAPam7hnHmfpVryIFPIpyrb5AIriuTylP5GzyB+Fc34usVubFCMsynjArsituuflFaWhafFfXpPlKyqO4rSinKaSC3LqeVG1MOi2DoCXUbSAK6iyjX7FGGyGxzkV2un6NEuvzwmFcRndjHTNUdcso7TUmVlA3DKgVtVpOMebzDm5lymCwXaADWR4kjV/D14B/cNb7JHnBzWL4oD/wBiyQWcDzTzfIqKMnNZUruasTKKSPOdBlZlQbjXqNtKgtowOuK4Ow8K63pKRvfaZPDESB5uNy/mOK9Ggt4o4UXZ822vTzGS5Iowox1YxJ/9kU/zcMCFqRRD1KdKUMgPC9fWvIudHKMLllyABTPPI42VOShPYYpDJHkcA5ouHKu5WZ3K8CnRu56rVhjGvPGKBOuOApouHKiB1Zz0/IUgDBtpHH0qYTMTyqij7QMkDbmi7HyogaLJGAT+FAjYj7pqVrhgOCvNM+2MoIJWnditEjKODkKfypCkvJUEfhVhbkMu7cM0C6Xu4/KldjtEgjWfBHelFvMQck5qf7Wg4BzTGussMEj8KNRWiQi1nxy5xR9lkI5Y1P8AaAcliQKj+1Ak7cmjULRGfZWHBP60fZCDkuAKesrE5ZTintIrADbzRYLRIDbqSMtQYVV/vZp5ZSccCgSLuwcZFMVkMZPm4phjAbcSasO53YBGDUec9SMUBYiJRemaYwVj0NWC6N8oApvAHy4qgsQMGUfKDihS5GQDxVnggfMBTkwVNAWKTPNnlTilCyMM7TVpnBbaOcUm7BIBIpXDlKuJcYwaPJl+8Fq0XYEYFIzsMdKeorIrGGUDhcE0bJemas7xkgsKheQDPzijULIjNvKG60ghfJzxUnmgAEt+tHnRn+LntQLQgMDlsA0z7M+4Et+FT+aqty1BkRuj4xT1FoJ5R96KUOMf60UUahoL5zOPuCpUDsMYAqslxkAbDmpC7BcjjNSK5LiTPbAqpErfaZTkDiplWQ+tQBZfPdR1xQtyktChPpP2q5SQPg5zj1qvo9ts125Ax8vFaF4k0l7Etm+TGuW9M1Do12EN5HMoE7PyfSvSUHUjyhK0UbawSs4VOSTwBUr2M6ybHIjbrzXPjxG2n3TQucyI2VPqKLrxKby7SS5ZkDcDIwKFg6a+JmDkzZeGRCRu3Y9KVYJw24N8pFR6fceYhPbsfWtAMK5K0VTk4pGkI3VyqLaU9WoFsTyWNWw+Ccc5pCcDjPNYXNOREItyBwxzTxZgjLMTUo3ADinbXI4ouPkRXNgvJ3mkFkp/jqyI5GIyMU4wvuGOlLmQciIPsKcfNyOtKbNM8Gp/s7nnvUi2pI64pcw+RFL7IoJweaQWbdc8Voi2YdDmlFscfeFLmH7NFEWuB604QgdVzV3yADyacFjA60uZlKCM9oAwwUpBbAL9ytAtEOQaY08I7mjmYckSkLZT/DSpbqp+5Vk3Fuvc5pjXkI4AouxcsRnlgfw1G8QzwuKlOoRAYCZpv28FciKnditEZ5QPVaPsikdM0G8Y9EFILyQZwFFGovdFFooHSnfZ4xgk4AqP7U4OGcVG1yWfO7IFPUPdLXlwnjHP0pPs6N0Aqv8Aav4jj8qRrxgQQRRqHulvyUX7wHNI1rGcEVV+1Ozc8+lSLNLkcce9GoaEy20YXBpxhUADBqu00qnOBTxNI2KWoaE6w9tvFBjCsPl61Es5GQ0mKbPMSAFkI96V2PQslWDAhBj0peWz8oFRQt8nzSEmmtMFkOWGPrRdi0Jh5gBximvLL5eCuRUH2yIEgNn8aBdRnjJNF5BdEkU7gdNtI88ndsVG0sJIBOBTHeI8A5p6iuiQynOWbJpPtBU9MmkjCdsfjT98KH5iM0xXAz7h93mkeTj7o/Go3mhB+8PwqM3UXTdk0IV0S+btxyOfSlMhI4FVTexKNpGWFOF2jjAOCKBcy7lkTMOvHtT1ldhjBqlvRh/rMGpElAIBmosO6LBc8g5NMDk5XGKUPCcjzuaTFuSMzGiwwYFhhj8v1rx7xJFJp2vXFvKMxM26PjqDXsJazBILNmsHxRoFnrenHYdt0gzG/v6V04ar7KeuxMo3R5HqNm0dtHcB96k4I7r6f1rMSQxy5Bxz2rUmSWEy2c6lHHDK36GspwQ2CORwa9SVm7oyQswO7J288/KQR+lQ1djgkuI9qrjAzkt2689h+OKrSR+W7LkHBxxSLI6KKKACnp0fB5x+lMqxZIkl3Gkr+WjHBbHTPFAFenou7uPxprAqxB4IOKdGSHGPpQA7YaAvOKuPASm9VyB1I7D39KrjAOSSKkBiRF3x/OtWzt9jhmIxtBBU+/rnFU1ulQDaoPrmtAsGsz5a7twyFPIY+wzww9RyM+jChDMu62+aQFAx/dGP07VWNSyj94SAQPQ9RUVMkD1pVNNzk04UAel/A++hh8fDT7lVaHUbWWDDHgMo8wH/AMcI/wCBV6lc6Tb2Guz20UikOcqxPQHsa+a9PvZNPv7e8hYrJBIsikHHIOa91EwkVJRlldQwOex5FYVpqHS9x38zXvnNjd/ZmVXBGQ69qh+2DGAn41mtyc5x+NMe7SIncwGPevPai2P2qRpeeTKiyHapIBPoK7zTdS0PRdNLC7jdyMn5gWJ/CvMUvlkTO4EGneeNuVAx0rSlNU3dITqJnoum+J9LD3N/NJskkIATvgdOKwfFGrwX9/HJatkBcE1yhnIY5H0oedi5VWHSrniHOPK0TzpHTaHJpj+YdUlKkH5QCRn8q37KbwrDcC5E67l+6rE8fhXmhuCJA34VK85HPalTq8myG6iZ6rdeLNJBSNWDxHqcVxOs3Vteak0to+yMjkDiuc+1SDkduxqNr9i+AM59KVSs5qzD2qRrtk8CUmgHb/Hk/WskXb4OFO6ovtk7cLHk96xug9suxsyOrDAbmkjCAHlifWsaSaVAC2Rz2py3rCAsAevSnoHtl2NqTYV+8fzpiNGDu7Dtmsdr6aQ/6oqMcmnwvJgnBAPShtC9t5Gt5iMCeahM0cbjPJrPkldE4bJzVGR5ZHO7PHSjcXtn2OhNxAJADwDU/wBptRj5Q2K5ORpmXAhdmHQ5qxYTs8mJo3SjQFWfY6D7Vb87VwPSo5ruMkFAKz1QmQ5+7mntwScdKSaH7WRO12VYEDJp5vy6LlQDVLIbgtg1E7Pvwh+XpQDqMvPdg8M3I7CiO6U5GeRVKTy0O0SDf3pm0Mw54HXFMj2kjVF8xUhsU1roDqeapRtEoXfu609kQs2QdvalcftJE32xQrHrQbzaAdvJqnJtWMhR1PFPj2+WA45ovYXtJFj7e2QRgU77RK6ZUDINV/KJH7tePenoQPu8HvS5h88h8l06xFivSoftUjFSCQGFWBbo3BbJPO2pQsaw7dgJ7e1LmDmb6lRrl0woOc0ov5FcAdDU6RquT5QOfWo1Q7yQq4WjmFeXcZJeyKAMd+tI0twME5Iqx5AmjzgDJ702KMqxDnK9sUvaDvLuVGkuX+7kE9qfiVUBZyTmrESHz2LJgD0NSAxbiuevPNNTYrO17lQlhyysxPQU1wxXLZUkdKskyBg2AaaSzOpIwB1NHtBamfHvwVZmzngmrScdV3VYkt1Uby2QenFQSOyMqBMEjqaOZisxksWDuyaRkXZnJxTpBI4UqMkH5qkKhlJCbTRzhZkQjiwPmoqQLOBwq/lRRzisTrIu3oCaXzwP4RVYOgGSpAFG6OQfKDVcqN+ZF0Xu0dBUFxfxxBpAfmb5RVWRJBjYOPWo5bczKPMIABpxsndh7Wxs6Ja/uZZ2+83JJrh7pL1/Ec32X/Ulss1dYss62/lrL8ntVJYooi6huX611LEKOqCVRNWJNJtLd2a5uogz4wCRUmv2VvqOmGOOMK6HcpHqKrhZETYsxC9uKYsErcm4bFYzqc0+e4lUSVi9pVygtosJwBhs9jWsbmMY2jIrChtUhQqkh55pwgbbkStSqSUxqpY3BcRDkGl+2x7ckiueCMjHMh21MI43XAkasuVDVZdjc+2pj1py3qE4xWAsZB/1jcU4rKmDuODRyor2yOgW9w3AzSf2gS2DxWA8sqkAPwaaDI7bt5osg9sjov7SjB+9StqkeBg5+lc4YTv4JNKqOmeD9aLRD2x0A1LA+9j60n9qYPy81gPbGT5i7Z9qVbYnAy1FkP2xuNqmT1AqF74jjeKzVtQshyTmg22cgE5pWQnVNEXTEcSrULs7P/rR7YqmtmF65z9acY2RfkU5zRYn2hcWdYwA7DdTvMA+bOQaoNEykOxzU6KWjOPyosL2ha3Y+YY5oMm/5RgVTUMScueKegBPzEn0o5Q9oTMgX+Ln60GWJcMfyqBrUtJuJPtzUpiwoHGfeiwvaMPtsBcgxH2OKaZjsOxQBStHxuwDSBSydQfpRYXOxBcERgmPJ9KabsynBjGPapzCAqktjjpQUGScjkUWDmkQefISAAVxQZrjdjJP4U9ZhGcFCeeuKV7ho3BK4H0o0FdkLTSuuOcg05LiRVwQakWaJk3BTk8/WmmfKkmP6UWDmIxKCcnOaGkkduAdvenCaFDllqT7VAX9AelOyFcrN5zEYkII64pVO0/O5arSy7yQEHNLLAEOWAGeaLBcr7NxBXOKTG3dzUylc/KwwKhfk8EEnih2QXI3JdAFODU0Q8uPcx5qMxgOdx5XpStnapYHFK6C4obJ6nHWmu+QMA596nzDw2eR2qESRiTLHPqKrQVyPJCDPc05YmbhaDNDJIArDFRFhHJ8hP4mi4XJDGUkOeWqQRgcswTimSSwiMMwO6mo6yoCEb2JphcEnRGKlgfrQ9xkfKO/FRSqiSr+7+b1qUzxRgcZx1pbD5mKw5DDg1M0mQpJxVU6hCyuqAlh3Iqb5JUDFuPQUw5mSlioPII9aXcoIUsDkcVAfK3fMxAUUglQSH0HQmi4XZj+IfDEGtoZI3WO5QcMB19jXl+p2FxY3LRXKbZAcE9j717eskMqfKmCTyRXO+MdNtr7R7iRISJ4ULq3Tpyc/hW9Gu4vlew1c8n851tvL7Z4/rUFW3AECn5B7AEn8f8APf8AAVSMdK9E0QlJS0lACilVirhh1ByKbTlG5gM4oAmu02zZBJyoPPXOOf1zUArfu9PEmnpL0dYwT+Arn6lSuJGxpiJMGSVJCmMkx8lffHp+lVri38kjcCM9A3BP4dRSWTHdho1ZB1JRTj6kjir97FG0W+JnZSMtvbgcevfmmMxyDnpgfWtLTojc28sQc5AJVQPveo/l/nrngZb5R+Jq/pUjw3QfBC45IH3ff86AKdwT3wTkkmq54GKtXkpeVt6jOe3+elVKAClzSUUwHV69oU11N4fsZBICrQqoz7cf0ryCvZPC4SHwxpyFct5W7n3JP9a5MX8KsRK3UtiORciSQHjOKcunC4QsXIzzg1I1xGGBKAuOlC3JdgcAE9BXB7xFoj4bSKFfwqCWfymGITjv7VcjDMw3dPWmXaNnZwXJ7dqFF2Ah3IQG5DHsaYZVViQpJPFSxxOpxIucd6aAYZWwMgc8UcoXIiheRRJEQCakMDEFVVgO2TU0spMXmBMkdM0yKS4kkBJRfanysLjRC+zYE3OKckDxr86D3FTjzQgy4U7sZpjyOcbjuJ7iq5QKpdFY/NgdOKkWPbzHgqw5NRvAgcEPknkikDBbUxry2fypWFckkhBQt5gIHSolKSMp/g/uihIZRFHHtJyTj3qYWbBkDcd+KTSDUmWONWGTx1waR2iUZ7etQCykExIlYh+MHtVk26L8mOgo0GRiOGOEsHBc9qZJMqKN0YAAznFWXt7cYZBhgtRtaxScBs5HzbqWgalVLxi7MIwSRxxUUuoqgYFQrg88VqxW6ONyryvAqGTTVe4IZQdwzyKVxWZShuiVLbcZoSdAWDEn1q8LLMbYUADoKjktYIUIHLdwfWlzBZkMewleuTViO2LynI2jHSoUWd3XKKq54qWT7R5hwwwvQinzoaRD9lWG4cyoDnpUBjIkJUc+gq15crENKc96Gk8tyFi+ZhzUufYLCrAxbJPB6CpWhIJyfxNR25LE7mIPbPap8M52kbg3U0rtlJKxF5KhBnB7g0/ySQpfAYmo5ECqQCSF6c0wNcFQ2cAd6G2LQmaQrKEUYx29aifapyxwc9BT1y8p35JXHNRTKZGKoOR1pa2BkqlcK4kwAetLJIquTkk+lNS3VfL7p1bPeiaDM+0qwTqCKEmIc11vBwOcdBSwJuPLcDqKjWDynZwucY4JqeOEq5YE4PJFHKxk0giRPvdD0BquZUDEE4HanNbl3Bjyc9arS20iHOP4uQarlsFxwkG8BGbGOTSod0hy3HY+tLbmJn+ddozjmo5YXRjwMZ+XntRYVx0Tv5jrMcen0p0k3lqY1OcD8qh80P8AMq8heTVSdp3g3D5uMYHegV2XPtG9NuelOkmQDJZcjvWQFuBhQCAR0NPNq8sJZwd3TiqUUHMzUdx5I2yKS3IpQJdh+YDAz9ayZoXEIi8wqV9atRRHbH+9ICjnNDikF7lvdIOCvP1oquQM/wDHwfzopaDsx6MXHC4z609XZRg4GKjBAO3pkZ+lAdiSox0qhExZ9u4MPpTSm5gHyc1VNw0LEFxx0GKkDyTqCWKgc8UriuW/LEabNvFQPED8wHFVGkdrnAkZwBUrmQxg7iMdqeoyVpAcdPTFLvRU+9xVczrlF25Y08u0knEeQKPURajVHXG7B7U8Mis2DnFVS0wIKgY7+1PKgHJ43U1YYOUwdwz3NOtjAykggVG9q0h+9xUbxiDABUCkBbWRBwSPY0C5VmEfUiqDb5YwFcLzTkiIcbmJajUC3IYt3Xr2pHdYlJA/GoHWJHUsDk1N9qt2HlsfYUhirL5g4PNSLKWOCuQO9QSEq/7sDHelDkqQoz6mnYLltboFiPLyvSkjmkDEKoA96oPOYuQCT2AqSKRpmBYFQetAJltp8biw5pPtbbsYHNQTxlQFRgQeKh2lUwp3GgGy0boljkjIpGuZCOOPrTILRJGBZ8P6GoLuCVZWy2VHYUxakxaViNx4NOkuRb4O761VSNpdqefirv2e3J2ud5pBqRRX8Zct2PSla/G7AIz2qR7aBuI1UkDioVt0cEnGRxikn3HZkU+qyRkdScZ4pU1CWeMEZ3H1pY7ZtjF8AA9xUsQiwqsRwcg4p3VgsJbXEyZEufamzPdNMTEMKemO9aG6Hbg44HehZUYAKcH0pahZGe4vip7t6UKLzGZnCr2Aqy1y+TGqNu61Tlju523I20D+FhTSYDjdzK4jVt2fappZLhlxtyai2sAoGC69an8yUIWOD607BYqmS4EfzjHPQVIJbg8qmQO9PdUkIbdtH86he8hT9yuSxO0gUWYiVLOSRt8jcntmm3O212lI2kbPSnmOQtmNzwMdakUMq/OwyOhqba6jIY55ZRnaUb0PapwlxKB5kpx2AoJ4JUrjvmlLMRtQgEj1oAiMUm7Ebg5qRLNlJB+bPoaprdJDMVCMecFq0o5w6NIVIIHFO2mrFdEa7N7Rs4DgelRSpPKu1ZeKTzwkLPxvJ7inS3g+zDCnOOwqH5FXVgELHA8wACqEzSq7YVnBbnipI2EECyEszk5waYZnlkxgj/Z9KtJktj0SFWwBtbHr3qQ2QmTdLOVxx8prOSIPL94r1ye1TriKPb5jH5snNDTFcsyaciIP9Jdl7CpY4kKBBK4HvUUe6RQiyEg9DjpUhuI7Jf3jbmzQ0O6Lf2ZNoJbd65qq725lwRz6003hlJW3xtPXPrUXlJDjeS0hOcCkojbJ0Csx2oFUnmrDC3hRWJ5z0FVnhRFDmQgEcg1Ebi2RjGz5HUNnpTsK5pGWB1PyjGOeKR0tpI1UrletUI7tJGKq6svQGrbrmEOTt9eeKVktx37EyyRRKDEgyPWud8YeJGg0SRGRPOuQYUHtj5j+A/UitSOWEwuqy/MTXEfECFo4tOJcMP3gAH/Aa1oKMppDUm9DjwoaIxqQzA8D+nvTbi3EUSsMjoBnnPGf5Gn2RCyq7EgBhkgZ+tMuzJvOc44IB9MDH5Zr1TQqHn2ptKTk0lMYUqttYMOoOaSigDpba8E+kPvYArGVP04/xrnH+8asW8xWGWP1H+f5VWPNTFWbEh8T7Gzj9K07ef7UyxOAxH3VPT8fU/Xj61kjrVyzuGtplkQfMvI9z2/ofwpjNC6tFjnZVILdBz0Ocdep571LZBLdWadD5bLhiuBwQD+XH+e93SbcX0haXIjUEbxzgbTk8+4z+eBnFN1BDEZI9irlSApGeD1+pGRz/s5yBSH0Oau1CzvtJKk8E1BViSMZKjpniq9MQUUUUwFHWvdUtYoBFbqwCRRhB9AMV4jZRedfW8X9+RV/MivcgyXPPlnd6muLFvZGc2VXIHyAAj19KkS2OFbrjt61JClut0BLwOhA71PJJbwruQ4TOCT61x8yRmVnmQuoG4KOpHap4Ut5opMOSwbIJNRw3ESqdyDb1JPeplZQnnRxrsY8gVPtEVYhmZ2DNCQy9CO9RR8BiMhmHQ1adYmVlXClj261V8+FH8rdlh+dPnQmOljBUhWbgciktttnKM4+YfxHpTHm81VMRJBOPekt7aeacGVdwJwPalzh1Jih8p180EE5GTTkyAFVgGI+bNJJZbSDESRg5U1JHZuih93y9dpo52Mi8yND5YxycFqb5aoSEAIJqZbR5XkHBHVfar1tZbHVWKttHB9Km7YWKaTO5RQh4P6VMYf4lBBzknPSpJUWQRrA/wAyn5sCmmaNGEeWDMucnpRqGg3yZQS4wR2xUZM6AhUDKf4j2NK93hAoPGaWaUrDhmI78UhkPktuLHcCBnAp6RCYhipDD070yWcoFDyH5hgMO1VmupbVkSOT7xOXNFhXSNFm8g7cksRkY7U7zwIhIW+71PrWYCZ42EkxXLfMe5qOG5jB8vJZCdoDGgfMahuofMjCH8e1Vrh43kYscN2NRTOkcgjAXeOmOmKXGVAkUHFHKHMyd5CWTAB47dKfNdpDCSVUyEfMBVB5DbRlVfBONo9KgLSEfviDmq2E2WmuWYb1ON35ClF2kpCshDd2FMhhVIVZMtnnaakU4X92nLdV9qAVySCRTGHwGwcGrMayIkbFSPM5AasISGKEgbhlzgHvV1biZIQruWYqcbuiincFqXWLo7qQjFuABQpErDJwg4I96r277JIy5BDL19Kj2AM4jOQOevWi4WZota5iZJlwSM7l9KmS2VGjeMjBHz5rOW4ma2lIfLqMBT3qGFrvJYkjfxtb1pXQWZtJFGH8zIKnjaelCyQiVmYYIOAKpNG7tsJ+XGAR2NAsJAAJZCd33eeaY+UvsLNz5qkZ4496sx+RIm0RqXQZB9ay0thFabywYBsbO+afGxIXZjgYOTyaaHYsSylkdRtUk8HtVG5MY3qzZYAYPrTblkaJVBG7Jbr0pfNXCGRFD44B70mwsUZolH7zcTg8AU55tw2HczYzyOlXkmtRKCyBc8mmTXkGWKRqd+FpWQcqXUoi6tkmCmNs98dKEk3GRljx3VauzLB5ZKRqpGBk01CsWWcKdw4C0WCyHQI7sFdV2nketMeFl3rgDPINRG4lAZVCqg5Y55p0Vy8m5XG4kfL609B6CBN1tmeMb88n2oIdiixxDbnj6UNNIhGF3qx5A7Ukc8oYPtZOSMDnIo0FeJMY1Bx5aflRVZtUjDEEgHPTFFFkF4iLMmNxG49OlS741wNo3E9RWRC0q7gXGDUqEhMAkjrnPSnZkXNEpCSzyBeKhe4iBZVTk9BWVPcMVIRWf6UsbxNGWKyK49TTcWK5oi6WJMiMKc+lMfUYzJsbA45qsJYpLRxuIYd2HeoIbJsmZn3t2pqPcLsvmeNIyzKAc8HFMhvw0jAYbHpTPs0s0YEjZAHTFR2lvHFO/lxspHHNUkg1LFzcXSGMwRFt33qsJKzEB/lOOBVWS5khYqGLBu/pUcUzhgThjnFHKFzS2g/M8pB+tVXhjlZm8zOPekluI0Uqy5J5NV5Gh8tGjxuzyKVguWFaJUVEfkdqnSeKNgG3Me5qnAHkkyEG09Gqyi/Z2LTsNp6UNXBCySwScoSPrUK7S3ypu5yTSNCk8hMZzgdqkAYALGmHAwc+tLlASWUxxkqpJNLFOV4PGRTCZNihwffApRsLAE4BHWqSAsWiE7nY7gTxxTyFViXbjsBWelxcWoAwPLyQCamOFjMsrbsnoKXKMejCZ2GSvbB61Y8oxLgA5qukiM4CjGec0PNIFJVt3PSjkVwuOMztJwhUL3p5LuuRnrzmmW7TFt2znvup8skjDBCqO1PlC4gt23Ehdu3p704eYq/dwT1qCOWTnfJk5wAKXz3csVGFQY570WAeqsSVVtueMg0wWzRtuWfnPbvVaO/2hQFyrHk4p0dyrhgqMAT1o5QuWZNUVJfLZCVUckDvUyXIKqzBNh6etZrzYURGPLMcg0iwGRd7vsGeFPrSaQrs0jJG0rc84+7R9riCkBDuHcVlStcg5CL8oxmrVjG21iCGOOcmmFy8LlmyRgLjqahOoIDgyKc0xo5ZJNuRg9QKoNo0j3RdmITGAtTddR6mncyRNEPJf94e4NQohfLvIQvQjNNtNNeNdsmVXOd2eaui1jERVfmUHOTRfUCGJTJ8uDgHjNOJty7nyv3h4HFOlQxZYPggZI9BSLKibJEYSD26ih3AhSV1dYsNx1bFOkkKhiSQvbNOvb8QDCRZLdDVZZWlmQSKGyKWoh0JaQYPIPOAauLATgklf6VPbwxg8AKQM1MhjIGTjnvUtlJEHkRwNtHO7mnPOEhJ257cCpWeJSvIJPTNQyzR9FVmBODjvSsxlR5ZZlDKFQ56GrdnHwTIwbj0p6LAwGcL6DvU5lt4ovuc+tNxYloVJLMSOMHaRzmhLWIEszH/ABpy3iOwCnBPrR9oU7vnUr057VNpIeg5TaBPLULk80eXFht0QI7GkRbZkLDAK9xSmRDCdsn3TzmhRbC6KxmaNSi25Ge1EMZlBEkSr7GmvORKXLD0FUG1GVbh2MbuOgAFPlYm0XmRoiQsaMCflp5+0KcrErEjGfSqcHmTqX3OhJ71pW6SorFnyAO9PUW5QuN0SN58oII+X2qr9mtJYFILMT6VpSTQSO8eOV6qRTdq2+w+VtLcnA6UXYFe3tIAw2xlGB5q+yqqkEEpjvQGEsykYCHj60+7t2cZSXjoR6UNXC5XEcZjwExjkcVxPjwoba0CEkCRv5V3kaxpGpLklRgMOhrkviHEqaLZsAQRcYP4qf8ACtcPFKomUtzgoVHkjjJznA/GpLsF4xIcEsOffv8AyP6VBb9G47duaszDzImbBVS2Ao5A/wA/rXqGxjkYJFJUzRkjPeoaoQuKSnU2gByNtP14NNoooAcoqaNdzAAZ5qKMZrT02DfOPl3bQTx/n6UgOm0tzaW6zHBQLnaRkMMEc/jt/Osm+meWQMzeZJty244z2/w/zzW+6Qmyh2EMYyGkGMBsjcefwFYGpEqSwGOMAkc0hmJK3zkDkA1WblifepZG3MxqGmhBRRRTA0dBj83xBpseM7rqMH/voV7qtsixOAcMeDXiPhZd3irS+MhblHI9gcn9BXt7z2zzmFTyv3gT37VwYz4kRIoyWISfDS5DAkEVXSwMcBaaRjE75A9a12KuAi4yAeKQfZ5rZQzfMrfdHpXDykXRSEUSoiMhZc4H0qcLHHA56fN8q98VbW3jaUbeA3GCfSobj7Hbz5ByRyQT+lLlC5EY0eKP5gHpUsLaCbzJlHmFclhRFd2gu1yuI/Wpg/myvhQw6Ae1FgREix8HKjJyQKuxbSqMEI38VB9ie4dBgAqOTnrU0ZggBE0gAAyMdBQo9xla5hZFcxksFxjHYUSKWZFLgEdqmnvooofKjZGcjn1rn7qea4kMkZy6jGwHrinyoTZsyTfY43yclumKhh1F7iBBHGcg5PvWZY3LXFopmQglz949BWu7/ZbKNlCYLZyp5FHKK4sYkKNcBNm5sDPBpfKDLkgcnv1NCyn7u7e74A/2RTLqdYYcZyVbauOtXZBcdb2ts4BLfIOntUciwxpsYEg/cc9CKz42nZJnRl37j8oPSiJ7lrULNklT8uRwanQdy2bdAfm3DuO9TtbosKs6qSDnB7VnxPPFE4myzMfl287auLcxG1EZkIlGMjHUUaAMdEmVH2KGY9B1pGs4lAMaAlhyMdKicLCTJFwG6L3Bq9FNGIlkZSHI59qFYCKCzVZFYQ4KqckdM1MLYxusTqckbjxTHlKsrhi3PyqO9JNfeb5hJbccfN3FVoBG5R97bAw3YOF5FJ+62YCEt/eI4qBphLZHY+CGwcdTSQFiuVbec7cVLkgV2ThISHO/Cr87HofoKW3nhSJpW656eoqtdW77pHUZUjG3vSGOXao2MNuM+gqebsPUlbyFPztg54HXApoMbKzE7vmwB7CnEq8pQhVZhkZ9qYsUruvlINm05OehpBexZeKOZcowA7rR9nRXCQnnHINNgLK/luq5AIJ9fepVh+SOSZx8wyMcfSmO4ySfZDs8ja+QAfWoZbgJG6u5jLEH1qUyJPAHLcw/eqrMoklCoAc/3qLi5mSJeOxjTdz0X1NNuJ7ueFHwwaPIOOtRRwyKollZXZG2rt7Vb3yRQlypJlwMCi7Fe+4yCWaOxdGUs7Px65pkSNIrZlKt1AalFvM0rOkg2owZj3FJPGxufLjdSAfvDvmn0EEETQzKZtoSRsHnOKbcW8TzN+/LKpJBX+VNjR2l8hmYZzknoDVz+zmW3WMOrAZ3H0P1p2ApYIYyRRgo3HXpT5YhxIhGRgke1NgjJsXDnKnJCKcZ96jAmDxgnKFfnLGkFy0zosTxqN2/BDDt9akmnCPGfkx935aph1MUWCSgciQEY6e9WrqBpFzDbBrdFDnHUmnqFyvdTRmFRP8AIOgx1NKzx7GkB4GFOB0NI1q7mOTOCwIBYcYqwiqtm4Vtzcbm96BFWO6e1DCMiRVPJPpTWuT5rlJQMchQepNNuEEc5jUkq3PA4NVZkKiJjDgIxyAO1MTZZMhY7vJHPNFPVUZQQrAEZAzRS1HdjHQbAQy5PIqW0tsRn7RJwx7U8ukUJk2j5eAtZs63dxKAjBVJx16VerQyw9qqzOY3wB0xUe1VCiRcnsTVqGzWJVEjs+OtWXt7e5O1R8oHNKMrDsY2yJpdyEBQfmWrcW1CwDA45HPanDSoFR8uwDccGq5sl8ttpYEDkk9a0umTaxoK0LM2JQCRwKa/7nCtyX71lxjaybioC9geasSlZyMOc9sHpRZBcmktBOpQOUHUU230/wAlt7uX3djVK51NrR0T7yAcnHelj1Q3OFzsI7mlZhdGiLXziX2D5e5qvJBa+fkvnHUDoKtwRSiEr52eM/WqNwJY+I7fIPcCkpLYGi2yRRAAsVHUYqG4iE0wBZigHGag+0XDNHvtjnvnpVmKeQ/uxAXbPWmgCO1EKFkY4AzimRXKG4c7vT8KT7RKzsGQBQcFfSrcVpHjzCq0SfYLEZl/eAg7h0/GomtGK5EmG9O1aD20CIrgHOc4FRPK5YosPzYzUczHYqG3YoyTZkIHbtT0gmZFUR4A9acfOyMDy/72e9DSSxpje2QM4ouwsTyRRoiMwPHBqrlVUlBnByasWl002UmRgQOMjrVh8l1QQjDcZxQpdx2KTSs0YA5zyMdqBGeV2vuC8Gp5JjFtXytu04PFSE73Em4hW6Yp84WM9Yp2lCsAE9auQRqrMgG4dyallZYV2OhZs5J9qqtJcbwIoSIj39afMK1iWSJHjZ/LULnjFV54nZN68bemO9WRp10EWEvgsd556VJBEDGyzNhuigGhMLMpXF1BBHGXjJIGRtHNVXJv9rxZZd3I/u1rSQ+apWEIxUfNmmx24ggkSMKsjjcxx3quYLXKC2+5fL3NgnlqtWdqlug3Nk+tQxyqCq7+/wCdPxKtxzgr2pXEkWGhWOX75GOc1WN5ullYZMecKaleHzVmaWTawwAtNS3gChJiwXrxSbHZki3RyVdu2R9KcbhY2TA+/wA7apXE0vmpHFDuQdz3FRyJOrx7sFsgqBxgelPQDUdtxZ5I8g/d96hnyqKyIqkjoKPMlmQwJGwJPBPrU5t3jljM53ditDaHYyZvtVxsEcO7/aPFEFpeLITKORzhewrZnLBFjWIqwPygd6mihAU4O2RhgknpUuXRCsRQttT5uuKJrdpAp/ixkYPapTbqEC5+Y5yfUU8lYW2M2C6/Liot5lMz3iRSvUntUsZJHCMp7ZFSJEN5Q87Tng8iknuljY7ScDtjrVJIke8ZYq4wTjHFU2MhSUOGDkYHcUNfOVZovTgY605NRjWNDMBncMgVSa6DM1I7jcBICBngjrUyWc7DYgxGWwdxqe5uUF0Sil1I3fL2plpfi4jlRciTIwCefrSuKxKtoYiSz7QzYKg4GKsQ28Zd4x8yEZLVA0olmeJkXOMLk9xT0mZJljCMdqckUKVgtqNkiEMYcANtYHOO1LO9qT5sIYh8Bs9jVhruGOGTdGSCMhT1zVJLqO5m2NGY0XG1AOvvRdASTOrhRF8hGOvSq9xdTQtvbhD096t3nCrEsYLKeRjkUz7P9pZI5IWwvTnihNILMz3nG/zQGIPRhV+OeOWGd2f5tn3vf0qVrIRrkEAxYUoB1qNbVZoCI1CEkscjGfpRdIVmNRwyo7kK+3AB4waa5MofDBHPBOalFldSYDRg7eQfSopdHumuDOWCxgcAdC1HMh8rC38uLELycDnI7n3rC8asJvDssbYLwypICPrt/wDZq6ePS9rKHXcHGcnsao+K9LtpPCuoyxKWljhDEjtgg/0qqc1zoai1qeQxAlTg81bBPkkDP97HYnH/ANeqURI5JIA/z/hWgqKYW65AwcDjqcn8gK9U3RTRNzHoQD3/ACqrcoFfgYq7u2M3JBJ5/CqMpyelCBkVBpaSmSJRSmkpjLlpFvxnvW1p6GNztXcT8oAGe3P6GqNkik9wMYA65rasYxHdo7LwCHOO2M/4ipKRdu7rZEUIDOQrDAxg5578jnqfQYxznmrybMLDGTn1/X+Vb2otI0ssbqd0Zwrtyep46+o/WuWvCd2D68D0FAiox4amU4/d+pptNCCiiimB03gSzN34mjIx+5jaTnp/d/8AZq9Be1eG9mkadmZTnGP0rlfhrboZtRut6CSJURQx7MST/wCgiu7exk85sybjL3HceleZinepYzkrkcLNJbvIwKSDkgGp4ZQiRyIp3Y785pAVS9EDKVVMhmx1461IpWOZfIKkA/mK5LEpIWZ5LhWQxEMq7t2cbap3NnILiMzKdhUDdn9auXDy+a6tgDOTu43ChI58Md4dD9wN6elMfKUZYHWUlcO2dqIPSnfamjYiAMSp5XHJq0YY4ozI0jBwc49/ShbeMMQgdSwyT0waB8o1r6S3mTerRqV3D2FUHmmmQYbKF8hc9av3DLcrmT/liuDk84FNVo4pImSMAE5PHak2JxMv7PNdXDtIvlhOOOCaYlndQq7RsWkXkYPTNbN35SykIS3Gc44qFjJFahgNrAHIb0ptk8qKyQssYjkBRiMnvinLBOuQ7loQMjFK11tXzirfvAPl65FXHBazKBgrueT1GKQ1YjtQIpHfz8lxw2O1RyQiKYkPlBk7sZyTT4rZIQuQXYEqeeAasRRiG8aIIHkOAV6jmgCJ5EidBbxglgN7etWLdo/O8s7iOWAx0qNIBNdzSK4jjV/udhV5IDK6yIQcLgqPSi9mFyoYxHMXj4B5JPNVkKRXBB3YcenpWjc2RtXD5PJzjsBUa5dXUZGAX5759KTmhlV0V4g4yxbjGMVZEAaLbgAMe/NK5Vp44mlIhVRk47+9STy77QQqoVkU89j6UcyC5TmDPGqRyKfJGTjrinC2T7KJRJ90Zyff1qtI0pkeZAGYYTavcd6sz2/lW0ggZmhkbaW/u8elNSEmVS2VaKGD5s5diKDujtxI0f3jlGzjNTwtJ5DRyKPNHHA5+tNjiIgMch+Unofr1FK4k2iSUM7RRR/xqCze9NmldojvXBjbDr6ip7aQeXwfmHRvRR1qnCkl4J5hG5OTyTjIH/1qLj1ZWRonheQ/wtwRzxVu3DzyIsTZ3ruwBwD71GYY1ZT80aiMnYw6/wCNWdNYLEPKI5TJPQk0XFazILszQMN4+c8KVGasSxm4iCooAxj6GmRyiRnIk3BW+QHt61fhYzP5Eagydc9Bk007j1K9vY24O2RSsjqCuOcmkQW8ZkeRGaROAMetOdDb3JMsm1QNrcHKGnsy2sqOTu3sdzHoaq1ugjPlMVsWUnhsMoPUn3q7AyJi3mfEci7kkA/i9KJLWOSRgoLAnJK9QfQGq5glnQxhcqjfLvPJOO1JPUEmWvs3kwM5AClOFH8bd6jkWCO3Vi4hZcMBjknsDUlu7TaZGk8ZVlyy9+nekklWW7F0+HRQAAVxk96bGYtxdt53+ry7ZXI+7n1qFtRmWSaRtzR4+ZAcY962LuzBtjLCx8ndwMcg+9QpY+dKE2L5McZb5up+tQ9ybMgaeFUUwAshj3gZ5FSi3Lxq7Fjay7T5qjJUn2qC1kgyY4Uw+5sIw+6O34VMbi5RHS1gaaOIhjzj5s8haI+YDlspIl8rcWEm5tzdDzV0hIYl2uVZiSy5xxVNpVs3R2mkCyNgIwyBn3pjo1+v7htpR9r7/r0p3DUs3lw11ujOfLGznGFwBxj1qayS2+xnyyzSkgsv17VHJZ3Ubt5vGEwSnKbajtLkiPybePa27f5hHBHp+FNt31HYmljBZfNRE2xllAPSqqFjb3CzIA6kbR/ept3LIJpZHcAlfmIGev8AjUcEc8szyFJFZX2KOvHbNDQisYbfccwTk55INFW/NuuwbFFLULMhMkUwKkkYPT1pFmt4x5bOAAcg980fZdkjEOx3AD2pzWUfyjaG9zXRzoY538yIBHG0dTmi3GYyqMGzyTVS4WVJhCkB8vuc0QR3uWXytkbjANJtWBXuXZlAgDOCMHkZ61H5cZh2b2XceMVWCXU8xikTC9Mk8UhtJQcecVZc49qlJDe5BdQR2rEYL5PJqub1Y5FVYj8vcDNW4oitsXuGZcnHPer9tdW/mCJYgcLySO1Ve2wrXM4BLuULHEQRyd44NXm03nCRrk88Cr0dzAo83aCvPIHpT01GCcIsPcnpUNy7DSRVihuYz8xBGOKdLvJKxrjI6+9WPtsRX5cdDj1yKha5kcoSoVepIqdRixRuoYXK4UDAxSfY4o5N0bEHOetFxepAsZlYAEnJNUzeQTJKyzHIx161SixNonYxQl3ZMnOfrVMX8sjCFIG3scg9gKt2qwsmCQ56nNL5kDyF2YIUU4YGjXsGhbt9xQbwA2Oakl/dRhipJPoKz49StnQ/vMHpn1q3DcA27Krg7uR3xUtMpNW0G3Ll7dm4Vh0yOaptJI6rIU56VbZFUq27JI5B7UyJhCjo7b93Iz2oTexLuRIwaUyFSGAzinSTy5iABI6gCiO7hvE+X5MdSR0qezSOWR1STcympejBFO6ubl3KRxZXr71FHczNIirGwYdiOK0JbK53Ha6qFPWkWG53qxKq27I45xTuNxI3llD7tgY8AjFPllm3RsB8hBPHY1caQeW4G0se4FKtwjRCKTGO5Wkrsp2RQiaZ4T5pYNuyoFNG4NucEMh6VLNeQRyInKbTnd2IqVbuK5mTjgNjJHWqvYSRBZ2kn2h5EfarckUCG6fzGLAqM4PSrYKCVioHXGabl2LxnnIyQKXO+g+UorpkaSCU5LOO56GpZE8uJDtG4ja3vVmQeYyhVcMuA2egp4t4pHdBzIg45o52HL2M5XNyuNnznpj2qS4YvP5cK5BUZHvV9baKK3bAYSZB9sVVNvAboMJghwScdzU8zC3cgh8zPlmPEinGe2KvSWu6VGlAJ7Y9fSqlqXmmcx7sKSCT/EK0Ig86hCQu0blPpVKQh8UsEjMiR4kQcD3qrKhMBaUN5it60iwOJXmL9XAJXqTS4b+0VXzcq5wSfTFPmuAklzsbfInAXOB1rOmvpriRnhjMakd60Wi8yWZWfcR8qn1pZYvs7RRxohPH4GpbDl6lU3juyqY2UKoXcRxzSahK8t5GVG3aMAmrsxhhTkllB3EY4NRhEEW5/uuQ2Qc4qWx8t9GZsP2qGCa4AYgttye9Wo4y4jVlDMV+Zs1ejczQHysFT19M1XglaNmiVBsB5brg+maCuVJEsNhai7VlyF25Ge1RfYYJJ2LgZYce9LG4nmLx4wBnGalnuIUzvjIKLgFTxVK5BDPZJCoiA2MVwSp71E1vAI4phGmYztPYmrAaWRQUjO9weW6Z7U23snwVkfOxd2AeR65odwJms41ljuCoJHIxyeackCrIXwGz9/2p9vGZZRtZfI+6OeSajV/s7GN23K+SS38qpIV0MaCGOElVPmEkH3BqwsSpGjpCgdRgZ6kU4SQeWtxuOM4J6jHSq1/eRAwBGBRflz/U0WC9i4LaJ1AUnzGO5jjvUYJKDClShw4x2NZcF3PLcymJ87AFAB/WrEF87ExFSjO4z34oVhcxqBYvnXjd0JHpSC3gjjZss5JPX+GoZ7lUmd44/LGPmJ549arx3LXVrKEc4Zd3ufYVLaK5jRtpTwE3YK45FU70zGLCrhYxufPQHtUljdLNHCnCsgyfcVIG89roOTuK/d/lRcV2xgE2UnkzlemBxjFV76Nrzw5qsYUBpYJAMj73yHFTx6lceYgkhLAcDcOMdM0rXESwgoSCvVcHkE9KuL1TFqfP8Z+brgVsW6jyHOARtK4zzkg1Qvbf7Hq91bZ/1Mzp9cEit2ww9nMgG+NVLE+hP/6x+IH0r2FqjZGHdAozjAG3jj/P+fxrNc1oXofzCSflPQGs5qaGxuaKSiqELQOtFFAGzpxO7oTzz/hXUWSocvJgjaMqTz+Xfv79O+M8rp52yduh612EHyWyyFeJF+Vd2M4+YD65x+f5SUjJ1RGcuwbaFJGAemTx/SuWuW+faGyBXQahJHsATOSAwz79R+WPxFc5JzIcevGKBMY3YU2lflzSVQgopKciF2CqCSfSgDp/A17Dbay8FxMI47iPapbpvBBXP6j8a9YkjCK8mSJIzu49MV4cNPkTHmbUOAwDHkgjI4FXILjUrZSIbq5RMcgMdv5dK4sRhvaS5kyXBs9geUPGsjtlnPl5H5U42C+dHFHvOentjvXJ+G/FqrZfYNTjBO/ek4X+L39P5V29pPHJG85cNIwURkNlfrxXBUpzg7MjltuUTBNJebJwGcP1J6rUshkhYBslUj61duZt+/AiICgb8ZZcHJxULTG5t1VfmickGTvjFSBmoLkR5YAhsuC2fyq2xuRAoILswGSPfpSLHLCWhyJyEG1gc59eKvtJBLp9vEWc3fLbgcKFB4x68UJMRk3Ikt7uC3GJASTJz+lE6SSSvMuFi42KGzxRNpbyyPLC0xRMsRjq3bJ9+aWWBontpFYoqx4MYH3jnrUvUWooBmaO3ug6Nncigc/jTHgkvJ5bYvnYcYBxn2q9I0jS+cBiZRhyV5KiieJTP5kceEIDnjBx3z9aGmUrdSg2mH7IkaYZ1yxycY54p0NlIjEvLtDJ0Pc1fult45EIDxpKA3zH26VI6GKMSOoaEqDk9Sx44puIijaQOEUKCSPmJIzk9qmF3HHNu2rGz/fJHOPSnrI0LjZKzMy7djLyo9aotcWz3KTmFnMQKuvXn1A9aSVh6WNefy44pLlYfkdchfXHU1Sa4RJnmiLFHTagXgBsdajvNXSO03OxkMo2Km3lR0qOyhXYImfEUUYZSDkM5HA/LNNxuK5L5s8qCO8uSFOOntyagjleTUEBlbazbgxGRt7ZqWO3dkMbpIYjmRd3qfSoreG7mtBHuP8Ao8eCccgEnAPtSUbAWHZGfEfLE4cHgnmnKAbREdCNpLfMeRnsaZbQSKQgykkiKGycnPtSSpdXjsJGMaCQ/LjBYbeKq2lwQkErGV1hVGdMcZ70kcM4eVw6NGGy/P3cH0pbb5opyqqI1IVjt5znGfpT7xW8yBAhV3GeRjd6H6HFHS4CmUfamuVlOyOPG0epPI+lVvllMb3BMfzFYz04ParMUao0kkiFVYhQO3rzTpEjurho51cMDkFRxz0NO1xFSCLEvkMxzuO5h3TrwatxwFooZVRxEcv7kD2+lWmtIXtrjy3Hmtwoz90AdafaWqvAhSbJC7CpP3B6/TNHKMp37rKbG4AYIwEeAOMA4NRpaLbXDpb/ADFWCBGzg55zmrMsi+VPCY1Zon554Xnt60WjiVgJmWPdl9wznPTmi2oEMdtaxJ5+4NIQSMD7ntQt35UonDFPkwQ3QHNWroQJZRohDHdsbb2x15qo86G52xwDay7QvZWHem9GDZJF9pmkVXiR/OOS4PX3+nNTXiHz0V1UKF2qo4HHeq1xd+XIiBsREhgAeUY/eH0pbmZ7rSru1hIcxqrGQdVBJGPegLiui2i+bCWO0fvNo4DEfrUf2gS3ccgUKFJOB3OME4p8s3mWn2IFY4EDNJIzYLemKz1vUCW5xuC7l8wjAxznn16Uth3LeoaohlW384bAoXK9hnn+dVpG32tvBDOXV3Ow4x3xWbFEl3d7y4Ql1bJJOVByBnt0rXusPHsVAGj+dJFOB1zwe9Jy1JV2XLqMLaxROHhlRT5mGzkjAz+VUxuMRaJgqZJ3c5cdeaklRhbKZIw0sjBmIfLKD2qMNiaTcr+UkRODxyD/ADpuzZSNRYIiVuDCC2DIojAAPHf2qpHeKlwrmMBCuEjU8ZHSqkMktkxdyWaZdqjB2jPYVXfLQiaRkH71gxU/MMdh/jTuD0Zcd3F6iyx+ZAWBweik9jVW5uWi3MihVUs3X34H1pkk9zACFUssybipIyAaktLeT7WPNicwuRgsOoxnGfzpbuwmiYXrwOolQsGQMyqfvZ6A+9V4LwyeUskZWB8s0adQe5qmYhGzyoGMMbA7CNxGR6d+KDFJDOFMnlpKhCCUcjPqe1VsTdsku5W+x3RR9xT5V2tztxjkdcZqVJ7q38tsiRmUNtR87sVUZHt78xMqyE/u1KchiRwc/wCelX9PtY0tpjIkzgkbjGMFMdxnkf8A1qBpNss/2jdj+Jl9gvSio9sh6Xa/jIKKOYZmR6jDKmSW2K20joasI4kVAFbA4Az61lrpzlBMZBGw46dakgE5HmM564APrVNEXNNrgDIA4BwTnpRNeRxx5LFh1BHpUbMLe0G5lVmIBDc7jUAsZbq288HDYyBj86El1KuyvcakZHU27gL/ABZ9angutzMJGLEkkgDkVDDpv2csChZWbaQRzip1D2ricwsx+5jrtHTmr0J1Kl1e3EoA8rIU5UnuKzDPdfbCjEojAhiO3eujkXzgihV2/wCrBPYetJPBBBGVZEd+QrEd6amkFmZtjdzTQGKAhiFIEY7n1rSnj3TiO0Voo1RSWxjnvRp8kNvCJRAgkwQ2OtWI7/7SHBTZ0Q+hI9aTqN7FK1iCPTYxE5MzCbIO4GpUshgnz2WPOMHqabM/+jMRwSxBwO/aq6TK6t5k7bi+1lPb2FTe4aDL2wmeEq8qnJG36VTOiTs5WKdDgZOTWzNY+SolibexYJhjnApZLbZKVXO/HzUrtdQKEYl0/IkIdMc0i3NrdERlC27ghauyRNJHsmX59pAwOoqKHR1JLKREkeDnPNAehHIbERnEDIQwwD2PTmrNne2kN/8AZwh5HDEcE+1FwkJlcQgKpGd55JOeasGe0jEaiIGUqDvx371LsUiS6kQOFIOH+6QOvtVURPI/7xfoKtNIJLd5OCyHgH0qvJdrb2zO+S55+gqLX2GyzFaL5OF2kZ6jt7Vagt7eIh92CcYwK56O7ea2eQHajNtUA981oxzzxfLMwBX5Tjp7GmqYJpmhMyuzKr5IbJ9DS/aY5mG9CCOARWPFPPMsuMoQ+C2OtKGYxzxxNsWMhT3J9cU/Zj5jWM8DOoCN5bEj8e1VY9ux0kj+cdQOelRWzXfkCFk3CRfkbpx60S28qCRYmMe5cE57iq5UhXuWXaJZP3yLlRzxmiS6kLbliQbSBx6etZTxXDwoyyF5jwR6Cr0MEbr5aSb3ijLyMe2KmyewczI57sPJFGWCAk/MBVtXZIxkh+mX71kzW5+1jewdFIZdvXntU10/lzSHPlFE6g9TT5A5jRM8sbJkbgzY47iq4uhHq7Iqsm75gTTYZIpoizO55GHI6U17aRo1A+bYeWHJ21MohdssyX3nSMqHnd8x7Yqs5jEwWL5Rjr61CbRiyeTKcMuSfX2qwLZIdhjJHQHPPPpWbhILXQpaVIdsTDIGeKI5pCGCsQcHI96YIzDKQD83cMc4zTprZoUyThs4+X396v2emovQilaTcBC7AjBP1qzFMzxqDHtZfnLHpUTRSi13xJmTurDtV6GDNlHKw+Uqd4Yj5SO1VyxtoNXKLxSszu02xgcjHepI2Kxq/J5+8RTpIpXkC4Hkhs5FSTW8XlqokZSCTuzx9KNARDDGDcNG5Yhh900s0McTLCp27T8y5q3Zxl0DuwRwD+8P8YFZwn+1XEhbbsDdW4zRokDYkwjhtrgrI0YYBuD1NMa4tWt0iidyScsQcc4qPVFWbegbAXHHUE02IW81u6vbCOQIMbe59aE7oXUtWkbmNEHBi6gd6lt3ZhIrhcMDkMKq28rWys53uUBO0DmnxlCHMYcSum47u1JsElY0IRIBGjyqFJzkc4xU0sMSySPG5aXcuCDxtNZ9uBHaSASsHOCrdcVFZ3EaXAd4pAsgwrZwM+4pcxadlqjTs2jN2zMcMe3v9KhuY/MO0OchtwPbrSWsKNPKpY/aApIOfvetQxea8PkKuJjJtB3dqdyAeKOOzePzCwUEkAe9JHa2+pWflIHKx4L+rVait1NzeQtkAsAGXpnHNVLUpZXscX3WZygUZ5Y+9CExIrUWbzmNW2k4wTyPrS2kaQly7HLFFV37Cr8kqw3kaA7hkksw++QKpzxXEjLNEv7rhmXOco3X8RSY9C15sPmywSttYgDg9T6Vdms/KiSKLa3l7dp7nPrWBZ3gS4topUBCnzS7DPC8DNX4L4XUpaMmWQjzPm6YHQD3o0QJpl+3t4Vt53QEhG2vx0J/nVW7lECGWNwrkdO4AFNtr0yQOm8oRIZJMj5evQ1Rup0TUkjuYfMEb5DN05/mKAua1pIr2y3cUhkQYEuTyo9KpreKklw6FyvmDc45Ge1VUCS3FusUmxJMiSNFwoweBxVqaWQBoreIeWsgdgo64PX36CgVzyTxcgj8W6gR0eQScHpuAP8AWrOiDzoWhILDcCOT3/lnAHrzxUHjCML4muNvRlU59TtGf1zT9AmSGWQlSXAyo7duPf0x6fQ17NN3gjWOxW1OEBWIyGVirHtnOT/P/Oaw3HJ+tdPqSsEuIiAArbsNnJJ7fTrz71zkwww4/OrRTIOlJTjSVYhKKKVfvD60gNqwQPJg8Ke+O2e3+e1de5K2pKxgZjyFY4xyeffjj/gXpXL6FGGvUMgJRSHbnoBXSJdF7Rppt20szADI6dMH2P5fjzBaOZ1M4LsATgAbhnjGMfSsUsTJuPXrWpq7DzyVbrng9vb/APV6Vk/dQ+/FMhkdFFFUAVr6YqRRGVyBu6E+ncfkayKtkMW2j7gwD6DApMaNpbqONGBCMx5J2A5+n5VFBeLuZAjMp52AZH+NUktZAoLDgjPB6D3FWrGaCF2jnjWRCeOeVP8AjUGiZqWt0lrcRzJGqkEHy5Bwfzq83ia50+YXscuXk4Ks5bIye31+v1rHitWkm/cEFP7pYA03V2BXayqzKAu9VAzj3/i7c+9Fk1qRI9G0PVo9cinubdPLPlnegPHHf2+n9Oa6PTkBtIUjHS3ErknPpmvDtD1ebRNQE8e7Yw2yJnG5T1FeyWt5bT6OLiAL9l8sOXAxxjp/jXnYikqcrrYwasOswIprr5NuSdrHr061FKslvctHGjvGeEJPK/8A1+9WYZ5ZFN4UEkk+Y1UNtXOOv6frWXHqIkitiWYuy7VPTax/wxXK3ZE3NPzhGpKzOYwC7DacEr1BpY5ZbiCQPFmRRlNx98j9KpPcbNPdFlbLuS+BnCt1z75FT3t4tnfSx3BDW5jRXii7YH3gfx5oTuFyy7NBl5ZS2V2umOCSM5BqNd/lrEEYt5ZMh3YyPQ+nFZy33nE24iIjCnLZzkdPwIFWLTUJPtjRqQ0BdZHJ79sfQ0X1DmHJPK9tHDKh2QEojHBNUkvpp41jSJmVV2tuPAbJ6fhzV2WOIXbOsUg3vnyyencgVHGtortFBcKIH3DDjayk8Zx39KBBYi4uUV5JtoUFd5GMgdBmnvHGtkzRSYudodc9Wyef0FMmnV5IxDHyV2SoOnoT9ajQIY1nPWM8oDnbjtSuFxuovBKEnRmRYyD5eMkvjkVZSS3mkjgVniUZZgO2B/n86rB47m0muTIjx26rK6OvB3Egj6jiqpuIriWaWIOnm/MOOGA7D0p+YmzXs7sKUjIDHy9jCRuOoPy/hTmla111N0m8XGSUTrsA5U/jWPZslxdxtFcRJtlAZpVIwMYwD35pYy8OoGSVVV4nbO7OVJ4BB9Ohp3dho2r26ge4hCoN8bAsT1c4z/Wmu8bwT3FzKY2TJxnlRjoKwLppJJElWZTK8bFeu4YOM+9WrVJL1IoLjdmRiArcFjg8+wwOlPULmhDuF7IZi7RDarsMZOe2KffXUgMdyU3OCVDdiAcAD+tZMZlvZpLWzXAZiVxwcKME9fyqJb0xbwxysjYlRuxz19s0LsFzZMiliN4aNgpK/wC2c8Y9qlmvoyI5IlKl48sdvTHUD3zWY1xbHT1Do22XdJbSA4IcdQfwyPyptrE9xIk007DdGu3OcEe57U1uF+hp2rkwBHZRHJKEaQ9fm6DPp1pyyC3jlgRfN8wny41POM4Bz+v0qpIYXltyn7uFuXjDZG8dD+op8WoGKSWb5VVFMSEjnaBjP5jFMosZkMVtc+dHIdrbkYc7hkdO/rSoWljjMjKkbg4YDGGP9D/Ss7R7g3KkImQFYBJD91wCQQfc4496lknL2drJLvRuFYAcIwzwfz/WjbUVyV3mspDHIiOjyYG0ZAP9PxpZoZLdzKih0YEnbyRjqcelKJdlrIPLYSSFhnZzlRlsfTNJGs4mkkywc7duxCvDAZIHbvkUmroNBdSEZ01n2MUlG2ORR0buf1qO2bMRt4mZJChkfZ14GMc9f/r1MUaSKG0hkd4Hwx3DHO3nA9Ce/tTd2dKmLxjzY1WIkL1XOf06UWHs7lC6tHuYSsbEhG+UfhUciTyxFJwAcCNIy20HoM/yq9p95DFPLbTLGFyAd+TgAHk/Uk/lT7lGnMvIKnLKMg8Kcn9KAtpcoQpHa2AjufMSffhXhGVOOoJp0DNJdC2dpPKXDKoPyht3TnsatT3KTfZn+ZUkHG0c4znd9c8VHAomspWinUOf9YXPB3cBf0FJjTQl2zyXU1x0iJdndRwn4UszTLpgkeTbvVdxTOHI/rUVztX9xdEgAbyq9NhGTg/57Uy9mjvfOELAWzxB9mD8rY44zzQJsZDPN9pjgfdyN6qSME5wMVLCYre9QMVWT5i6Fu+D+dZ7tJEsLM6lWj2EMOnfj65q1viuNPe3uFQyI+ImBBH49z9eaIiTLrXqhpDHCpgZCFAXPplj+Zqst1MiwmGaSMuu5ItxbHphfzGPeozbG1Dyhi0YOTC3KgEA7T/j9K1F+wpcfaYbUqmSSWO4AnB4J6Z9e1V1KTbGnTrOJmElxJI6viWNQVP3cgk/8B6deay7n7ZdXONkfkqo/djoFz39K2LwRXl7I6yhuN7RbcBAAclsdwP51mWkUbapeQZlRZ4PMgJPUBuATnkZ/PFD1B9iP7NKIJCJQZERZ4cHJA6kE/TJqMtdWyh9+2SaXrvPzqBnPB7Zq8sFxE0kTKwEceDkcsnrj2/LFZ7yPdz2wgikW1EbDei5bGB+mf5UJiNELoePntLtm7kE4JorL/sdX+dpZQzckBARn67qKXKPXsF3LviQMMK7ZO09MdjTVZ5ESFY8YG/J5AOakiUzxsGiIiLHGR1OeSfSpkaKIROgypbbnOASKttmavcLmJZ4kcnc6OGAHarSST2ACukbAKcEHJz9KhnTbDI+VR5GXbz97nqKr6g9xbShmJMgGHVR09Dj1o1HsTzvKoZi4YuOgbpTrdnjhbyZS7OMMnXdSLDDDZyNHIS0g3KXHPPr+NZ1pFcmZQsgVN+ARwMdWxSd2wNKN3n+TyN3l5JC8+1QT24kbDPgN8wYA8DFZ82pmx1LzUmYxsSGwcYHYZp010ZFV2lKxNkDA5cU+VobJJLO4ikKQSrsUAncf1poDrIscgYBwSGA6nt+FPMrXEsYgQCNQDjON3vmp4F+3QSzSsN+MRhScKB70rMVh0bMwuBcBllBDbF6YAqvIYp4luBEwcH5ievJ6/hUoiYTK7St5pQorDkGm3Ie0it5pVJi5XcP0NTZjeo+eObAzMojIzGc85qyryjT5JA2+QAIzd6rSTp5BilC7kThuuS3SrkMsdsz26kF3IZdwzmlZgkinHNdADzBnA2gnsfWrqi4W2eWQKyZCnb29OaUlI3kQAtGeWUjvVXz7h7cw+Uwi3ZIUdqF5laIkgcNcRZjClck7uSc1O9uHJYpgBNyOx96r2TE3tvIR8uRjf160TGSZ3DuxYyEELwuM8UcrHGSS1EeJYI3LYYPnYc9CewqtGYC7h337BtAz3xmpBP9otljaPaEkKrJjP41Bb2pE4mjcKUJCyEcN2NVZInmJ7bbEfObYU+Zo4yM4I6/jUzOXtGmP3C2WY042apKbeMKVbkyL/CDj+tSxW5hYxF1ETdQRwcD1o5kCI1u1nEsU6AADfujGMHtmoUkRHIztzgyEdqvxWcSROPMYs6kADue1UZ4bcKrvIGmUAZHTg9DS50GvU07a6GYYpGUY5ibpx71DdXsZjVgRy+3Ge5PJqOWxiMK3JfmUllXrgVFHYx5uMBgrKGdT1H+FU3cdxBaszmeMkOPlPNJa27xSuRKWZs+YAetWbe1H+lYdiq7SN3AxT0hRpTIEORngHsOoqL2GiJ1jFsHjUHy+Qe4Pamz2wmMzFTLHIPvY55H9KtIkFwTFGMR5BHOMnt+VW4omCGJ2PqGH6ijnG7GPNA8cKjLFgSGx344zTobK4csEcpvXHXrxzitQRF5VjkjIB+YsCBgU02rbpWlJUIc5qHJj5bmb9nkgto7dHGVzzU0QE8JSFm25DHHbFaLwRSOTKzMQvaiKMeWzWiBVKZw55NGpTjbqEsMM1srNgMV3FhxgiqkLrsKPKPmHJZvypDcOJFjlTjHGzv3prWsEkwDgqRyNuCSe1LmbILkcTycuR5e0KGBxilaP5Wjk2bU5JB4x0qsJruIsrFMJnrzuPpSFnJ+Zc5QYRepyaWoXJw9usfUquSOejUbwbdWMSjDYOPT1qLVTE8SBmVQWAUYx17U2JP9E5KgqNhPv0xTswuSRzmQMuQVDkL2wKmjtIAC0hGVO9QF6nsKyrdwhCM3zMQBxnir9g8jSsyMzEEqQ3AHpVKPclMGto5Y55nAWUHOzHHPtUctlbSxF0Y8LsKDqPQ0j28kzbQ2HZi0gHZfSp7dXjkkQrvTgrx2/wAKVh3s9h1vbRWe6OSPO8D73PTrzSNJBe6iCWVQyAjPGO1S6gslwYHtwPNyQyk8AYx+Xeq0Nksd7Gs8ahmVdpDZUggkj+VXtoxNsjTT4p7uWNZ2aNDkqBxzxSXlkkUzRu+AcqgHTaOKcyNawSzQMzMYUkAHQFT0P61BJPNd2RuHRsbmdCem3GCuanRCcrkdsZrR4zvUNnBjbuP8KvRSh7tbpfljJAYH1x61mpdbZBPJEjBh5fzDJUjoP1qcF4FSDeCJOCqjOSP8c/pS5iUy/KJEuCS5SMgyYXo2QP8A69ULyLfCWG7zcBlJ/n9c08l45DHINyxg4Kcnof1qK/DRJEv3NwDOv91ew+tHONifa5V8mRkyrJhty5Cbj1+tW7tHtLU/Zw0gKZHy9PXmmzwSWiZkO4M3IIwAo56Vea9SWdMoq5ZRtU8KvY00w1MWC1IcXCKQFUIGzn36VpYCLGYRgwHajL3bOTmnhY2vzb+UC6ugypxjPU+9Pjiitrp473EWCAMNgsexpoEV0klkWSIwkThiuduNw7sfXGCKtTWkItPOIE90ybAx5AGc/wBKkuJHSWKaQqNrtk5B+Xn+tNVmWziLDywwLPkZ5FUkMpSWD2t4iQKUidlQEn+dadvpxihgkUblWUr17DJ/I8VDJrNvMzMSHWJ9rlcYBGM7fXrViZiAt1aSI3yFJEZj06Zx2NCSBHkHxCCjxRmNFQGBOFHHVqzdJIS5Vm+53GM7vb6nFbPxBDvrNlI+MtahcDjkM3+NZGkDdPgICQATnHTv19v0zXrUXeCNY7Ghry/6WZM7hNGW6/7RGPfj/PSuUvBiXOMZ7eldNqkwngikjOVC7CfbCnn8QfrzXNXzZm6DOOfrWhT2KwpDS0lUSJTkGXUe9NqSEZmT60DOr0OHcxIYDIA3McAHkDJ/H9PfNXriJrO1aMEKGUhcnJOTnHt+Pr+UuiQKsSDndJIsWQdp5BwMnjHB/EdhTdakMSGMZLjJbC4JBAPPHXOe3BGako424w75AxwBj0qtJwoq1IBtLAd+melU5DkgUIkZRRRVAS2yeZcxpkDLDrWusEaTOr8rncqgdc/n7cVmWR2zbv7ozn0q0zedI5DFf6+1RLcuJHcskbfKRntg5qoJCHz19qmeNj0GR9OPrVZsZ4polmzaX68LnAxypNXXlNynMzCTHyng5P4YOPx/CuaDspBBIxWnYX5U7HOVPUHvRawXuMuLfyXI2nI65x1/DFdv4P1SGTQZ9MnLExyByu7rHkHp6bs5+ormNQjikiWXbsUjcMoF9sDBOcevtWbZ3ktheRXMJG6Nwyg9D9R6HpWdan7SFjOSuj2S0S4Q2+6URocqqkZC4OOT6EVUfTTDA6xOcmX93t/5ZjHOPz/KrunTxalBFeIztvQMsecAEjP5jgVekgt/sEzLcBTGQEB65Y4HPoMH9K8WzTsZNFFNO8oCCNgsuB8+cqVxgEg9veq11dSmfe6q0RGwMVzv4xgVr6dHE7wxSys8mGQY7lQC4z7gcVWS7t5I45b5yZCwWEdCXLbfzyRVWuFirJCIoYUkLI02X4+8oOAAaAXswpnBSO6O+M5zuXOPw5qLVYna88uN5UdXYEDofnwv5inGxa4W2geFi6gsE/A4/PFJrUVgs73zXMgkMmGdQGbp8vGPT/8AXUs0LIoSaJWlcK0AVsluMnd/OlMEFrBbmzhaOOORMrIc73J5we/f8qbcSuJhdLhzE7DaerjqPx7UWQ3poEG5WlikmO9JA8gHJGAM/nkVbvY4OJWZFlLeWyR/ezjPzfQd6qXK+frM8sCMIpkDEt/Cu0YB/HNJcoUuP3U2WZSAgHcgcE/ypjE1DZbnUIreJmWWXBVMg7Qo4/PNEFi7oJCqxGB/3cz85BH3WHbjpV+6NxE8QjRVEgVZD6MQvJ9uat2qJb29xayOzwoE756YA565NOw0k2UtCtbe+nzM58hVUmTG0E4JyPxwDVW3tTq+tT3RO22DbQS3seAO5GK0UiltPtFurHMiA7SozgtnP5flmrEU/kRqJ40dZg53Rj5hwN3I9cfpRohW6GXa2sTSSrJAoDIY4xjBBzxg9cc1bntzZQS3vL5uRGqocnAPHHXngZogvI/tke6L5hMF6dB2wPp/Omm+a6t57SJxGscuzCjkndkEntwBTT0AiWyazuba8t5ZBIrFo2ZAAp7qfY5/KqTFHtpt0AkZm88nryP4RxzzmhdTY7S6yCRSxYyNuVgDgN9BmoJdSaCdVtVUHI2lvuhs8gduw6+tA2WYikhjsGbejHJKp/qDzhsjkA5waZDZzKRGGMPcr/eJJzk+gHGaj+3iOGMxkxbz0U53HkjnuMitSO8luZr6WF0afLHyh1IJ5x6nP+NIS1KFkZZPOS0PnFSZNr8kqDzj+dQXImjtWkchQLkh5F5K59PzPFS2l4ltL5lmn71OIgFzle/5kVFJOxuGkii/d3DKssJG/wCbJB/HpR6C0SJIt1usczb14TzQpyWBPVffHbvV+3eDVLlop2dJWgZnVSQrN1Gewb+eKginVbF7bzoormRDsWQg+WP7qtnqf0yaz4plFxOPL/dbSn3sHcMAj3+9T2HsbCxTrHJc+YPMjkKhuQGdlOSB6YH8qbaysJRDEXDGQx+Zj73HIIPI71STUM2zEcKJfKQs/wA4xgnPr2HpUkGozQvHfRiKSWKQTMZTjfjDHpzzz+VOwI1ZjLLbvFEcMiiNuMeaPT2qvbqrw3Vrcs8cgDLDuHVdpyfwI7io7PUImt5WupPKimmcMIyG+Y/Nvz6HIq3emSAFJJllRTsQg4G1uQPrnp+NPYpx0uY9gknm3olyZ2XzFbHAVece+c5oubuOG1DoTFNtB2s3XOensRmrNyksEaHCeaR93bnZx2x1J9OelZS3fn2zg22+5gUHMgzgdcYOQOTU2uS9NCSKK5RzdnyzAyeXGhbP3uPw7flUv/Hrf2shkAEihtp+6SfUDjrUdtEDpwN6fIucNc2xY7DIxBwMZ6MRkHgfKfxfbzCQG1ltVSaXY8e8/KgJJ54yO3Sk00FjQNraXENxBLdRh9m1FckMABjGfXp9axrOwRQYnZQiAZ553Z/ya0r+Z7uRxPbw2ktqrbCh5kjySQOxwOQao2bo0twJvNQhQwlA+UjOASfqMelN66Ca1I7r91OrwOjRoCCSvI9Bn061IJENmnnRKZoiCgbGSC3CEdiOx+tEE4h0x3cOtwl1GSE6kBW7fVs/gKHLzqwdIwVIL3IkzucnIJ6/gfWlYRP9rw7xyQh1kDFHkJyV6fmM/pUn2iX7DHDKwjW5jVkAOc4BIAHqcDA+oqhMqGVJSGhiXMZ5JLyFCcc9zj2poJFhGTv+zAfOHHI5+Urnv7H060arcZceaeWeZA4LBVicqpVuMDB9+DzVuSGKC1kdpwWOTbgc7E7p+Z+lZKfaLqZBkEMkfm3AONwGBn1zjGeDyauvpupm2jhaF7XYZVAkB6DJGOO4xz7+4pq40mSQXjW9xIssvmGK2IyOxxgDPoQcj29wafYtHb6fG0hQxnJZXHY85+mMf98msezsb37dL8xEiKS29/ft7D+tJFNO1uqhJCjJ5eccAZ9SMZxgAH09qewrs3BZTTDzfsO7f82Q/Bzz60VnCa3QBftt0u3jC22QPod1FLmZWhShWZGeB3wwUvgHkHvT7tImt2RcqVAaNQe5rMlubqW889YjmThWxnj1q3dSyxWkcEjYdOA3XKnpj6V0XMy/alJLt1lZiydFOSqgDNSu4muxEzvl1xkA7mOeAfwqhaXsrXLwhldVh+cuB07fWnzXLi7DwbI1Q55GdvHTHtzUuRdtC2txKw2ADdjG3GCCp6/zqx9nR0iwQJJtqIx4PPJx71i2k81vK08rBgvzF8EnHPH41r2xkVoioVpAfMXIOQx6cnsKlzsCQxtJgMzTOnmPFzsA4OO9RJHBFK0JwPLfzFXsqnsK6GUPbR7XcR8l3kxnJqpHNaTqPL/eSEbgI1zn1qHNluPQPs9t9jmRFjO3bsLcblPUAVAI47UyoEVVjTITHQYq19vtRbiMRCMqdmCcnPrVO8UBI4pSJJNp83noOuM0JtkvQinmnSSOeCJGEShcYwefWia3kkgWDiSBXL4bjDYz+VLBfxTEGFQsjcHcM7sEdvWopDcyWk4RlIchypGeh5wfStNifQha1aCF5pHGZPn2Dnp0qa4RjHHJG25lHy+3aqd3cJMotx85Xnah+6oqrDqTJBskVjKr7PMPQDnp6+lFhM6m1eOSDcpO50+96GkjuGY7SoLojB1TqDjp78VjWU0oljb5AMbiOmT0BH+e1Ntrm4t5biRGwgZuScnPUH64qHuUmbkJ8/ZO0ihF3GPj5sLjk/nUhSG0+Ri2XIdt3Y45NZ9nqn2oH7NH5URGJA3JY5PPtzU8l7EHVJPnDKFJkOTnsc/lUOQ+hBC0YtZ4zGz5GVUk9+eBRb2O65iMsjKiocIB1Y/WnLOiTJlWjZSAAM4x3Oa0bhjHbSGKQEh92W53Htj0qWwVrGbHaTTXTCd/LIYZRTj6fXtUrlzI22JwFJXEp5DdPypup/a1h80HcAdrEY57ZA60RSXbIBIjkBzubGeR3pNXDYkWSYLbbpArxg/IowOnrWbN/pG9cE+c/O0dD/nmtO4gSRVmc/OykoAeWPqKihhkhMciYOxuOcg8+v501oJ6kMZb7VDDISdqNyOmB/8Arq+tzHJboUjIByGXqTg9T61LcWkeEwSpYnBUZJGe9MsNPaOKWJm3sQVYnPXHrVJhZ3KxugfNjGADtyD/ABVc82FrdF2ADOSB16VTgtDARd/dUgoEbkhjx+XNWZYw83lhcSFQpAbkHvUtjVy/Elubfe/zPjcCp4Bx3/SqLXSRM8eWLOMIB0+lQQbrVhDGxZJfuSjsOTnHqKsNpNxdzxTQxvNED8kmdoLAc8d6Vws2Mkv54LMkKyOVO75PukH7pq8syzWxMblmlwCOvQdqxLyWfUTtllZkwACrdACMn3z05qRb02dlBM0EnnAsjptPAPQn0707ju7F+2uJAo3hkkycN/dPv+lNgW6MrvMN0xJASPoR6/59ajhMpuoUkjUxOMPsI59wf6VNNIlrLblnB3NwzcFRzwaa1Vx201KzRNbndI7jqBgc57daRUM3lYyHAxv9/b8a03Et0ibVjMbcEucYJGeDUZecafFHLKQpITOwAqc9/wAqGkgsV1tpRB5kbHZtbeGOTn6Use6NoHlym5uQnVRjNWkmRZUZmymQHMY79tvbPsaHuI4ZZblwrttIGcHr0pXG7FWa1jYu9yT84DhW6Ejp9M0OQXuliQLJgD5hwQAOnvWl9niuLS2NxuTI2q+cjHv+NRCJY3BV4g7fMwk4AA/l0/Slcbj1MyG1bfG7YWVhnYevp1p0t6torRSqEjLbNx65ByMValeWOQeZEZdoZ8x8+xP0705bIlY0nTfMCQ2TlSpI/Oi5nYs2qx3MTOxjRkfkqc5U8U2W6iSN4dwEqhhheq/X/Cmx3Ef2uWFYQskOQwLnaAejDpkZyKn+whJXeNg8Izg9z6+/Wncpozdr+ZHDCzSoCcMw2ng9TUk0vmK8MYVZZGyBkYwCOR6d6WC4t7m3W4hST7QyFcHOGJPahPKUJGI33FMiYc/UfWp3FYnQW0qIr5QBgAQcjAPGcUl5psZiaCMHAQ/LnAfuSPx/lRZokCMYn2qiFnjyAAAcFjn6inzTeQUKTI6liD8p4Hpx9RVW0DTqZktmlkkZRJJA7ozAHO3A5J9scUsluI0jeLJAOF+bGQc5OPpVi9YLbysJNse1NwyTuOPu/qT+FVRdRSR29o0u5U2rvzn19ulSFkWrCG2trSNn3PuDEbgRlcYH61VuleVHEjARBeSPvHHP9BUxYNJMkRL+SuFUHjr0z+f5U03UV5dlI412kYbccADOMgn8aTDQl1KOSeSCRFbytibiSME+n5A1C1gHcXDzu45TI6EYOK07m3huLhoTOywxRgqh/iPTGf61BKPs9m6K4Cu4yBkkYGM/nVNXHZXII7iR4URiomBIdzxnaeOexxTZog90FEwEgzIqkhiwHPNRWqxRRAtcCBTIWcnnJ55ANVL+S5tLoPtfyzwo25ABHXA5XOOlUibaFiSeOcGBUAWQ7lIxuU4J6exBqvZa48YQagoYNmKPI9c5yPz5qna2dzeah57PCiRRggY2kg8EfkM1b1OKWKSeSIhre2I2b1ALAjk+3BNbJka7lzSdIjTTL+5dzuuZWEEJb5VJGc59f5VWieTyLi5O5lJ23KZ5HTbt/XnvU/nuLG3tpECzxwxbkXPXueP51Da20yRTrGSVcbODxwcjPfvRdXG2cR8QElW+s2k2/cdUKnnaCMZ9+axNJLecShOTwMfz/Cuo+JMBWPSLopgyCUF8feIKgj04IIrmtDG+72nvx1xx/X/OeBXo0PgRtDYlulaOe5ikcCTBYEHqQckDtgDNc5cnMmTwe9dfr8caPbzxnZu3xsueBnKkfr/OuNmOW561qi2R0UUVQhKmtP8Aj6j+tQ1b02PztSt4x1dwv50Ad5aW/wDo8UY6wlG46sw5/wDZsDsMjPArO19x9pdlYHoCQO44z9cjOD6+9b+n7pFVZF3CUmcsMFkB4H44xx1/DmuY1f8AcyyIOp5KkY45GMd8dM+xqUUzAueGxjtxjpVNuXarEhy+SRzzUCqQ34GmiWR0UtFMCSJS27FaMamOBlXknk1mRuUPFXWuAYgpG0gdqhlJleQtv/iqJsmpGbJ5PNNyOnFNCZFilBKnINKRj6U2qEa+n6gqQyQuXBYYADEL9eCOf8fwqjNIrTEqCAT3JP8AOq469cVNDFJdXKRRjc7sFA96TEz1XwfeRw6Lp8Uyl23b1C8Y5PU/QVs20ky+dG8pkhLIgO3AU84/Gq2k2EMOkzM4CxRwxiB3GedpH1xlf1FdBbzxWmjS2zRsUuU82YYUgAJuz6g8DrzwcV4k9ZtmVr6mVqSJb61apaXRUQR+ZsY9ySzZHYlcGoHWa9vZJHTL+d5sBwOcAlePqf0qS9llura4kktD59ywhSUqFCbgRnA5yAM/gadZ3cdjc2+6FgdyNcbl2gA9cd84zj6j0NJWE0Onu3dfPe0y0uCzYyUPU/SoBBL/AGqlwt00ksOTJGPlMSgEjr1A/rVywiS81ieFo5BJO3lMAflDEZT+Qz9TUFtJcXeobWgETRoySA5wUGM5OcknAAHv71KD1B32xyEMdskpIjHsxIyfoM097iJY45I02xgFXiK8qwJBP6GorhMQbjH5jvJ0T+8Rzx6cdO9X7oGQQ6fjbLDiIyFcEkZzkevX86fQehX1AJ9meW2YLAkm3AGcqMEAn/gRFULgskIyGjaOQqzKc53N6+w4/lXSrpn2C3h8yaO4Eq7vIwQFU4wx75xk1iRiDfbQSu7tI7BiAOGBPUY59B05xTYNO5oxCe6JmVJNkMeOU/uLxn8D+lQXE0cAmd52iE0SuAvTJBxgev8ASlglurWOaLzTJGvyl8kRsHUEZxnr7VIbKOKylBBd9oKyTDG7a2Mckjpg9qClFsrXM8kdw0jOpPlofMGMKpHb8OKlS3uP3zrtkxjbHnBGT19/TP8AhWVeXAksgkiBvNjJiVDgMxPT1yMHpU+66ifzBOzxFMb1BIPA+Tnpnk/lT6k9SrNJIkphuCxKYAdeCcZGMjvx+hNaMyW8NsFtppjcMBvd1ym0Hhg3U9eh5wDVOeT99HdK8MrgEYk/gKnJbjr3xj1FS2+pyXmlvcwbYrlHMpVo92FIB5zyRn/0L8mkJW6mQzpmTIG6WFgz5+Ugrxj8TTIrWW7slkaTzmt0AdAOQgI/xx69K0rezurAJNeKoEbPbyidxtdS5GcE9MEg47Yqe+s45YTfQmNYwQFSKQMzAAAsMckcjB9qWq2Dl0GWOlxX9mvluieQyND13EYPy+hyOfX5fTNQ3RitHurpAgnjVn+Vs5IOABgkHjP5dKTTp1jvmEsTrcGQBZA3D88kY6jpzVpbJZ2vGjuPMRpd6jcUUsCc7jg8Bcjgg5I9aNxpXRhzuqz+SwaJJSVzGepCjnOfpU1tFcmxm8kldgBkcenQd+vK0l1pkMIgQMRLK2xgfuqcchcdfTPvjtTL65lkvGjSF4pYdohB+U5AB5wOcljj6cdqpbaE6bMt3Mcb2qTM5QRyKcLj938oJ6ev9cdazriRlS1ljl3Fi3lpH8xwGXBJx7n8q6F4YnaVboKYcomFl2kktnrjnHf0BH1FCMF7mYRpsjjJ2tyrR7VzjJGDwPqcVKY7GdHNKl0EWM+WPnTjOSec57DrVmPSpdQmu3gbEEQV7ho03BIiM5AJBPGeB74zVS4nkW1guSPLSXdhQM4IJyOeozjn61NFqDuLkWkhglAEzKfmGQe+e2MdPT86QlbqWoFmtlQBoY2jCyq6DAI2nt378fWrc0st6scMpMlsyHcpY4CsueAB6fkRjvSyW9rJp1rd+WslmQUnKsdyAj7xIHyjLAjGQOnesy0ke01COUXDIIfmjTPTlc5PpgMcHoam422tC1Lfyz7jEymOFCUyMO+Ohx3xyD9KvabLa+YoFsss0kSxiM52yFsAsOR1XB69jUGp6e0zJLbvOVRWcRAjfFltxBP8QySc+gPPSpILW3jitZYnFsBJ++uFkLbeeu0dhuPTHQc5oYJO9wvls2uJHitQ628e3zS5+6rY45I6dB6HrTLdLZzFLaNOQww4KcBz26dRkf0pL4QyKyxRSRQlNwTksVJyWYjjPByOvB7Yp8FndpdeVDL9pheKSQFWZVCgrtPPRunUE4X1NGvQPiZHame2WOdmkcxYkAcg71TGQPUY6+1TabPbX0l0btvLUKoWVYyYvUJjoDljx7Y61bvdKSJcLqds7C3kjDuxVAyryAuOh6Z9TxWJY63HHLerELi1troOWt4pDsKnIAOTyAD/ADqkrBs9R2mWr3Guy201xAoiVCnmkpu5CrjPXk/kM1c1O2jsb+3hYRRLHvikER+WRBI/zZ9geOPTpisu4mFvdNMhyJYCssO0HcV3ZI7ccEZ5GfWn+RGdEjCrj9y2845YkADOOgGM9PcnNDsLTZGiq3k1msU8RkC3C3AdkbrsHJ/PAP4dafFO9/bTRyywtcCQpI8TblVm/jOM5wx6DpyOTVNbya41OdWAdAiFMqQqkqGABweSCCpOM/jUCSG7t/Mia4aKVwWj2DcFxlTuI+XOD9fxApNMq+hcv1TSY/mu0uTFJiSNAcybhz9AcjP0Ptl9v9peSF5bwzXOzI80s67mwSuCcjACgd6yLuW1t75ZIQpKMC5kIl3tg5JzyOoHXqO+eJ5YxDeRRiD7HFIplxjp2I49cg/QkUaW0Fe2qNa1xbSy3kwitIIiQXOSueeCTjJyBkDn9KoxytcXLWjKs2xW8jcdq4wx+6Op5J/L3rSi1pZ7SNpbaKTTSdrwyp1VQeR25JHOcjHQ9Sltc2MMiIIWPmnamY1GwYPDZ4yeP19sj02GYgt7aQB3guC7csUVME+3PSitl9D1d3Z47RPLYkr8qnjtz3oqedk8o0aYpliUqRFt4A/h/wAaj/sW1KqHRmRiR5eM9+Kf/aDW4Bcr5PynIGASeQB7VetdVhu0ikUo2xA3GAR1Bz9MUXZSSMkeH7RSX8lU+ZWOD29PxqV9MCKk0c5jHzRnYAG2njBqW7u2+0B0G7Dn92PTI5/wp+o3nmYiEGEzhW3ZJPGM470m2Oy3CGxj0+18iLNxjAy4zge1XFV/viJijYO0YyDn+VUYb02V4sbFTIpyx5JAx0P0OBVjcZSHheQzM+QwHG3sMHnNJsE0xqyx3c8o87EmMdPlz3BHrWZa2t3pepsWQRqBlXU9fcD0qw9vKruzTHe7l/MXtyM/oKnlMDuYSd2Od2SMcdvWrUrIUtQMcY0yUTJFNIZMogwCAcc59KLTyzJMWiDh+BuPIPT9Kj2Rz27rnkD5hzlsHI/HpTpFEdvHEo3hlyWRtxDZ/wAg/SlzNBuMXTFa5kWFGlJUOGV9pVsc4z17/nVZLSea5TbGIoyo8wjPUD+ZrYRZHiVSirMY9uQeOQf8KLe3ZZwZJnkYkK0SqPnGOvsQKbkxpaGPHprNf+TZRMkUh6Yzkge/bNWxpVtcTys4aMSKXZVAOCc8D05rQm+R5ZYBiKNAwDEZA/un375FNjWQxhpGAYsGARsgHPf8/wBKlyYJFOQRR6hvCLlc4AXO3A/x/nUX2VJUmljxlpAQvtjJ/EVrQSCK4kQR/vixGSM7lHOR61TEotgxYBZnkbaei+mR/ntSuw5RrW0Itgyjy9wKg45yMZOKihtHctFIuGDAIcckYrReeM27Js4UH92/TsOPfp+dAZHCugfe5zuyBtxn9c0miuVFeCApalYkMyM23B5JIHJx+NMFi0MQKkKpzIfMByOR8p9s4/CrEWoy2ryTPGowwZWUdQeCf0q7LdfaJjIY1MahQQTknI7j8qYKyKFwX+zbmUkEBCFA4Pv7UR3GNQbzWGyWPsec844/Cppbrzp1tlVEhYB0O35jnnFNjs7aR45JwTImXRFfnIPXn27UluO2pTupJXhijK8cYJH3cdD/APWqSGxu12yxj93AfmVh/reOcY/Cpr2xt91u8N0Gd5GcBx8uByvP86lnmmS1imkZS6gqSoIGDjkD07UWsTbuR6bK0d4/2g5icgrgn5R1x+Qqw94ipI3zTxE5Vc8hfWoY2MtuqRKGK9ZNnHJ5z+FEEUSSt8wZsBIArAZyec/rS1DZWGPILhlhKJEIxuK7jknqDmoZUjlvJF3sn3drqcbcgZ/Sqcsrki2sI/NeUrtIBBbOMgeuOlaMhgacqpC7ZVPODgFOadmK6ZBHmO8RLdiIw+RIOuc4/wAafJNPFMj8nY5MaluM5HQVNBGdPiliZQs8hOx2ONoBycfU8UNH5kEUojLZYE7ecE8g0WY9irbwb5op2TawLbwzfeHp9avSmN7JvlJb7pLdz6+xwBUsMQ85pjtWIrtBAHU9cfzzVa9nEdxOqSnPllIyRgFiPT16DH1osF7bEVhZjyZ0yEWNSWYc9+anmsbqWCNHETI0isSvOFH8h1qrIs1mjSRBnLqPMXaG75JwfSraSSNFE8QKHI3lgfmPcjPQHFC2HzdDRmuk06K4dpSXVR+72nBP8OO3Y9KzJ7jyw4kbelwokTAOAe/4003LEtIZiAnCq3G4Dv7U63hjvWDyu8UcAY7lX1PHHf8Axpbic76D0aPYxIYBmUqW5+bGKS4hAtjIZMSyg5U9jyRge+Ka3keWgldmihbGE+Useck/Qfyp8xJdS8SFezKTxt7/AJUWJuU5J5ftbKole3hCu7AHDNwv4dP0rSiR5ljQbVYsZMv1VRkLn6/0rMtZsSyIshO8q20tzjJwSP1p8Ltaia6nlR5pJDtD/wASgeh9f60JCUi9ayLdSyDzRhvl3qMFlB9B68U/Ihlkkj3lgu5lVuAB0H55rKtpLeK8Ee5CvJEin5eT909+oPHsKmdJERrZzGfPiD7lYjKkcc+gwKofM7FuSSG9tFu3TzQrCDKtyMndj+Z/CpmmY2sCqBEUY7X/ALyqep/MisSwd0uGiV1ZZMklc4xnHGeTwOKlW6drIxBowYLjYSM8AkkD6ZOM+1NC5i/BHdRNDPbMixlyFXOCo53Hn8qT7QJQPJiEYkZHMgYnPPPHbis8X0kccNv84m3SfMFHzgtnGc+xpyyqFhjVyrsmMPkfPyTnHQAYP0pbDUi1c2yXd60QLhI13sNpzgk7SaaInS5mmlcMqgiReCQCRuf8vpUVtBm2clz5kbJJuJ5YEcg+w4/CqqyGS2il+0TRyuSkjggbufmHuMdKOoKxobI0EbEnyGKhi/DNngnA9hj8arW9qby9luS0bKfliVcKpHcH8KYJJ/Of93mBY1EfOS5AHftkYqvFdTqZ3RFMKkg5AAVu4yPSjqJsu2hNtItvEp2MR8uCNmBz/OpwIBEq7VMQGSByzc8j261TMksj+ZLIIeqlcZbrnnnjjnpVm2tXnkvGmlSKGHKQ7kP7zdxtJ7GpBX2LDW65gmdUmSJzlCeRjtg9RnBz7UsN0LlS7R/ufNIO4cYI5APqOKoIA96Gnx59yNjFs8YI3DHfpj8DWn9rLzJF5CrDHGW8xBlipJ5PuMUFMbNb28dskUMaujcsSBnZjGATzzmkuUiFxG4kciIYQIfnPy9AfoP1qxPcWsUisgWIqwCnAwRxz7VXNnbz2i6gdSjjuWbLRkZIAORxnv6j2qrC9CrYRu9ksyr5XmyNDJ5i5IPAH4daLexRluJ4i8kzJgD7xGOoJ9vpTpp00yGR0F0TLy4Qg7wT1APGeBVS0n+xz77fzFhExbCLgDo2WHqDn8qVybltvKRxPFiUrtBDdNqgcfXPNNt7qNZ5SkrxB1JUvztG75gf8Khlv0eyPmAiMStI6MMAjJ5B7c449q4jU/F63TskcJVjkO+eeuQRWtKnOp8Ib7F34jXMkun6fE8AUJO5R1GAylVH9CfxrldEkjiui0n3MYYc9Dxn8zU3iDXp9VsLS3mRR5LFlI9SAPy46e9Z9guQzkqAM/hx/wDXr1aMXGPLI2gmlqb2u7biwiniQhzJlgR06H8+Qf19a4u4BWZlPY4rv7GAalp6K/yAEE4z2B/H8fp6CuL1qJodXuFZQpLbsAYHIzwK1W5bKFFJS0xCVoaIjPrFuq5BLYyO2Ris+ui8HW4m1kyFWYRRlsAZye1DBHXzyw/bAikiNegU8qMDIHqB+X6VyGrXRkupmZt+WPzHqfeuylWK2jMvmOXdWyzgBWOW7HqOMdun1x5/ekGTIORjn1/H3qeg2UyctUiR9SewIpkYDSYPcHH1xVpQFjB9RzQQygwAA9fSmtVl06kDOO4qqetNDQlLmkpaYxd3vinBjnjn2poxjoD+PNPVC33AxbHYd6QASVbleR2IqMnJz61IJ2HZT9VBpjuZGLEKCf7qgD8hTASuv8B6ZLcX9xqKxb47RMAnoHYED+RrlrS1mvbqK2t42kmlcIiKMlmJwAK9q8PabP4ds4NPjCxvExeVmTlpjwQTjptGMGuXFVOWFurJlbYu2VsFtZbeWRmN0zeREpAYRrnJOegJB69RipFijuoJriZzEYSvoA/B6j05XI5IxSTWdm0y3yKVmfcsjL/DjB4B4GN/Ax0/OpoIIbWaea6nJDxgqi/KqDIzjvnDDJNeUDa2K0UUc6FIIy1sigGGQlXYkc5Pfg8c9uadcw3VvcyrZym4dokWacrgRhc8D8D1zzg+tKkc10r+V/o8BkVEuZV3g5PJbHGPmPI6Cp4LVTHd2sLs9wsQjeVeDjkheR1Bz165pE30sVrRozDPFbOpMEx6KPmP3uCeQdxzxjgYollLRyyI2+ZkdVwcbZOT19STnt90Vbt7Yi0W6mhIm6NIxBDZIAIA7Db1+vXFVre28m8ljnhTyo0ZxON3BGTn13bcnHPQUxWuWSJWnluFQLBh1dRzltw6/gCf/rVXv5rjTITNKVS4LoGWQ7s7h8pBGehH+eaW3W7nlhXcpWGRQcRkpIMgdO5KlvpgZxVKaBmuFuJV/cuoVBg7uuOTzzkgY78+5ovYk07dp72KO5lhA3RmVZEQ7RlSeD3B3Y71Vlkgtby5BJMpV5AeoLcjA9MgDn3NNjsvLcwNdyRW6MxZFcsAh6Lt7Dnp9PSrMkUdxFHPONrfMy7UySvHAPAOd2c9OPQUNhuUWkke3dZXkbcsdtsYD7gXapwvGQBj8KfNeTS2/lBHJA3JIillJUjpxnOAeevB9aSKzjkRzLnz3YylPmAVj91SR16j057irkUqJDA8dvIWkjVQ7PtVAzcqU9QcjJz3oGnJnJ6w0sUKzYCwiQMk0b7grMScf99Z6Yz0xV6G4lltfsEKjzWxLHubg7eg9Qew9q1ZDbaZpVvZGzFzCHIl3EpjjPzHpgMcr7ntVOS13X8EUdtEqtNuinljy4YhzsZeDggckDrg+mW2O3Uosvk3kLSq8uFBLx8fJ8oOOuCen1FWViMV7FPGHtQE8lvtDgLMw3Ej8QB9TW2+ii4hgKXNuLqN45QrghZw6rlVbpx0we+frUF3aJb6cdPwGZnVrdxhhwBncR7np/tfWhuwuWyZk2ls93BEJ/30MhO1wfmUEkHCnsOvqOa0oFhEI4aeM5Dxy5DMo28t6HLHp71FCZ7a9ulvGPzN8yOeFPU9MA5wfyGKPLjOoTBSz+Q28I57EI3Htuz+IqeZjTsXrixtYrqS5VBbQJDk+Wu5ULfKzk9ePTI6+5NZEkixvK0cnmRxzMEAO9ZTu++noP8APrV2/uLN3VVQzxxowUDJQPjLE7fXjjB4JOOapxW9pFZLcIhKvIItqH+M5OATjIyw45PP4mr3Dd6Er2geWF0y5iBAbPXkctu9s9O3pWi0Q1G2Au3V4opgzTtt3rv5cgkY4I3YP+FZ8toot5EiN2XlKzKjuCWySeccDH9Tz0rQ02yeytZopY5du9ZoJgdvTIHPTBI6fl3wXG1roYlnYRW1xLcWubu2STmZSpCpz/ED3yoyBgnOfSn3KqJt6XSHcQsyqSfvArgj/ZHP4e/GrdWi3FgmZFwikSJEwXO8Z3HjG0sRnPY8dTVa+sD5hNouSWYCOJmcpkkjIPGARnqePSncLPlsjKvtMM9j9odmeOP5kRv4B8zNnsOVI4/vU7R9KuUMLtDFuYAtHhh5g3EcjOMgnpWul/EsQsJBKXDENtAbcmcg56jgMO5GastfmeB3O2NoyAAq5zx0yT6/oM9+FzMnS5iX8MujQRWyQRSxxnEu8GQMA2QhPZRnt6nniodHsJFs/tLRcYYtG4DKQTwR+AP48/TTu4LO2uppVsYvLf5n2kswLL5mWKnGznHOenAHaGzgiQSxRx/aEWb9y7ucKmACrY9QVx7+vOW2hW1FillmsovLliOXKOuGQHIIKqfwxwcDHtipo7Wye0nMxBtZIiWVGAY5OARkEAjOemfmz1qcwxxW6FHYMmFVHUBHyMHn1Hy9exPFVBPLDfyXCqpST+B3G0NuAbB7Asx/MdRU7MYyERzTtGrBDMgVixOY2HJyM9G6dyN3tVgWlpJqccMxjs4Y0kbzUKkscbsNngHOfwYdcYqtdsYpFlhjTyQilXY4LHPUg4J/h9eBjNJqLW5kilNuqecgQIWLptAwD1OeT9flbuDiosSdiSPTYDPbz2k6y7kDTi4UfKPlAwvQgbs4wOmTwKpyaUITLbBkdmOH8zDkfeBC9+/6ZrTspyVcJG7yjc4kkwDglgTs42jhhjvkY9qt3eb7iC2kMmElMjhCfkjXG4qvRjjHPByenOA9QuiM6V9q86YMwQXCM8zkEQjYFBCjCk54zgZHbioreF55SLQtDLAzGNGO4YPPGT0xkEHqCevIqy1u8tv/AKCzOVGRJcKQJBnOARkcq2c9qqzTXBvJp96JdmVXZVlbC/KN2ME9CScnPO3rmlqK+pcivFS2t7VJWjeNVIiUnYo5IPrgZIHtj1qnqejzXiT3aPNFLJ944PlrtO0AY9yOp6Hj3sSzM15IyGJncKBEh2oN+PlC5+U5yOnGMcVe0WE30E9leu6ooiYOpIDLnnP+0AOnHPOOMUK4730Mqz02DSb6J2Z2SSEgBOGLBgCOejfXjnPGObV6HvL8G4luYEldWy8e5VI5+U45HGeMjnqaiv5Zo50iklWZzI0illJLjaMkjsSMZ7444otZ7RIrlZWnuBIGCRgH92WxhwOhwQq4Bxkg445Svew0+gxPKEsk0M0pSRlZZA235GwQwGcg539B2HpU81mr2sEU7ytJMxVmyAu/cu1+TyCpx3PHaodIa3SKEXFlG23evk5YI2GYHp03dD06LxnFXbjxDcNpi28vmRTbg8M8bBdoAz5TFflJIycHk4Oc5zTtqCWjGYuYf3S6bDIqfKHMhBYDv93vRVKZ75p5CkpKliQd4GRn3Ofz5opWRJUltxNLNndNFtU89WVRgED15xVn7CkEJS1BEYHmKFPJ4zyO+M/pV9XjWJjwyouBmMZzzz75/wAav2zHy4y8MUU0xbcAeDwMYPb7uaq1y+VW3MlA7QGVo8M+eoOeKzzFcedHMsSCIjcCGzk4J457kAV0U3zxzxC5WNmj8z51OMYywHvjJGeCR2qBrZbOwDSMjHDp8ozs25JI+uBSsS0UIw3nm7ePyj5QAQ9gDnPPXtSBLqNlaeYB2YgKckEHJGMY5/wqeS7jW3jm2KwCBvnAIJxx7dB/KkW2F7D9puSYxGPNco3uORnqOo/KkhaASsi+SV2eWVfzV+UcZ4A/DmnLYM0MsauVmRuHZic9Sc+nA/Ws03+LoXEa4RQV2E4JXuT75OMVZudRKyxqihhIAXQLjf22fXA/M0xqS6mvYR7Y4kkZTuHzP0wQTgfnUJ8qxRnt/ldWGRj7/wA3Oc+2fwqlHePJp8lxM3lxIzNE+MM6g9x6Anj159KkdZGjisWgiKTRN+8f7zDK4OOoJ3D8faizuO9ti/d3HkzKYthj4P3Pu569OwP6Glf57qONvlmcCNJCchjxxu6DP8jVKfTZT9ntUmkZBt2SFyXGSeSR2wAAPapUZoHURxRvD5bDcxJyVPJJHBHp7iizGn3Ji4Mcyyx5cTjcAxb5eAP1oiS7/tK2jt9/lFlTCkZ6EgZPTkCqNvcXMpbzLVmjGCJR1wOfm9Rjn8BWvON3lzhfMzM+XyFAGMdBz7fhSsCfUlaJoYfPuPKU7zGiK2WUen4VgXBa5YRTvko2SAOgAPTHc/1qbfcRXIMmXVCw2xpkIOCvGfTBND3MMVxutSgViVcsnPy5GSPqSfyp2CTT2JpW+zaeUEwaYSiNnB9/07HNWbVLoQJMywYkc7FYgkgHBGPf39DULXMFvEQlqjCYurzONxcAY4Hp7+/FU5Eu0uLWPHlHzi+xTgKqkgdemc/kKfKgvY0JUikDxSxogMwSZHBAVeqn6cUxLx5YwZCYY95VgqcjBGDz04NSIGVbgyxSBSDCzhdwBBBXdj9Pxqpd2yxoyQ3EpjkR8l1AUnGCDz6ZPrn16UtAbYyO7kuX+xx3CllkLI4BUEEccHpjp1qa3a4ufMEjyLKvykKBhRjnPoScYz60sNsYbeOUiKQsqq8m8YLYO0eo44yRTZI5o7DDhxK8uI2YckoeFJ/A/gVosK+uoiiSaaNnAFtgfMoyAfl5/EnBFW5mInlE5XEW0bHBA29QAfrTLC9tk0g2rCROXSZs5aQkHJ/n9KIhFc28AuA0hTcrNDg7VHueScN0NJ+Q9CJdQCyz+W5dA4G0cYOOO9RwzJJuSWEyCYBIxkl1JODtI7gknH0pv9mFLkQwnzDKGTOApAHJJPsdv506O0QaWzrCzOS7xluue+COnT+XrUk6kI820ns4AjoIHKozjDAEn9c/lVOKbzdQCnCeacFgcgMOvJ9PStmOW7u7eZp2UrbLt8uRvmkJ64HXIC+/b8WnT4LZAzKAyMzYZdu04wc/iAQeOhp3DlIZ9TmvJWSULlA6NJwRkHGN2cDPzH6mqdnqciwygSbhHGUIbqWBxx68EY/GrMscUmJ1QSLOR5BZdu3DAYx0xjt36VTt57EKZo9OaNvMDSTNMSjMMLwmOAS2f/rUITTN2UzLYXKyTRNIqHARsnrjgjjIHH4CopRvnkeECQ4AnQr1B7+hIwffim/Z2vCq34kSCbdJJcxrtfpjyzzgqflH1qW4K2+oPNZqZLUKQ+RtLEdPcEd/Xgih66lqNyu8y21yPNlkOZCmWGBjcMgH6Gk/tGeQX9upcQMNit/usuSPxI/KnXt0kz5mCsBhgUGRjkgY79On4d6kuNLnFnbFExBKw8xY2zszg4bjrnjH1pLYlp30KK3T3b3EckYbZuZiwI3NgnAHfjFWI3jnsbogtGyhJEG7k8gkDB9RU9tFfTy20dswQwbSoxnKheTjpzz+XNRwaQrQvE9zPDINwDeUCueoxz+BHbPHSi2orOxnSmV54n8k/vB5MwDDGTjn6Zbr61faZRZwwvvMayKC2Qc5XgEg5/vCrSadLE7tLFlVbYQGwrHBO4evUj+lQw6c0SyyxqWt5MMkbHlW3HgdyMNgN3HHamgUWRKomshqCYWXfsBHACqcAkH1zio5zHc+XarInmrHuVSThhnPUe/vVi5sP9AisFUiKVG+VR8zeh9Tg4P41NZWtu9r5l8srXWMBnYZ2gbd2R1/+t2ougscyhlhuZWlt3AXKiNhgnnIIP1wfoTWilw6pbRmNm8hmUq3RUK/MAM84K/qatvaQPZW8U0jKjJI28N8ycnB+mOg/wAiC2tsXkb3RjYlmO+NuBnK59PejmFaxZvLaeOF43ZCEj3SKARuAUk4YdTyuV+lULcXP+kotvIw3oX4x5a7SDwcHHTpxVx5JLNp7eV2d1EYLOMnBJDY9OOMj1zTpjtt4XeXaqk72VMtxyCWzyO3OOBR0K3Zhy2skFy8lvau91By8qEnKqctkH7wAB5GehrVaCd7vyVYxT7WQhl5zgnJ/Dgn0IrWgsbNY2nRCXYpHIPlk+993bnnPPAXnnnIqpFcFNsa5kS2TzEye5wG2g88f0/ChtsbiiK1uFW8hjuIoVeLLM245dgCBnPOcADGKqXdu6acQiMjq4cYHzMR90AY7jH9auzOHj3CNGt3lMkORja4yMtgctjkHpj8apJdHUY74SuYJYQ4jwQQW2g4JPTgEjknIPpSVxMtqIFsFaSVs+cZJPJZdvlsu7IyQAcA8cdPpTpIGSG2a0ixJMgbfdcLvGSxyOfurkfhXLxDzWitrq5EdsoxGSTtdc9BnnGG710Npe29ra3VnJbb2BRoYpFZsEHcAOcexPoDVMFJMu3D2dzdQG0yUId3ZSSu4EYwDzx05Peg5WCaJHKtG+FdcFlcAkAE8dM9elZkk8VvttoEEVndbw0uDuUbsBW7fdbJ6denFW9UuEguIY1gjmWQ+bLCOGC8qWB7cH6jP4hIOYrxGa0EMnkveTJK+I4WO75sA/KD/nk1rXMTJKm4M3nqylCudmB6jvnrn3rMsIwLu2uJAYyC8jRY+8SGAwTnAJUkZPQ578Wry/t991YNFJayqCyuZASpxzuYD0A465J6nihgth+prcwWM1ulrmYSh8ufurnt7fIR/wDrqnBvmU+X5nlMCFYKCAxOBjPfp7VPfXWLfZYNG7TSFljb5hyN2OemQuPw7ZqBYprhIfNjdXkJGYzmUrgYHA+boepycr3NTcN2TXMssAV5UUbI33rgqQwI3ADp0B9O3c0tpPDeW1zCgEqRx4JyTjkgfMPUkcD1qW7hlngNreR+YqMYpJAQCCd3y7u5/ukgEgDPAOa+l2yW0M1vKsMxZVW3ljYKVK8nOAOMDtkc+pp2TK5bMpahpralY3NvZSOiOQdxXCYIyMjjDYPXvz0PXySdJLW5kikJVkYqevUHFew6xPeXE6Q21wDNIkkcpjB2KeCCcjO7BPUc4PPFc3ceDpPElut3BcwW97IxHkznAkcMquc9EGTu59cV24SpyaPYIrU4AtuTOT1q9p6OUcowXII46/55FVLq2ezHlyFGO7gocgjAwQe4OeKu6ZG0pRFY7ienp7/yr0U76mqNWC/Nt8hysWenXHf/ACKx/EU8N5Lb3UW7cybJMjqw7/r+lT6itxby7TyTlSAO/Of5/wAqy2y1rJE3LKdy+2Oo/Ln8KBlEUpoKkAEggHpx1pKoQV3XhfTzaacZJlYNcbZGXIBKfwjn1zn3/OuMs40lu4klz5ZbLY9Bya72yvGuJnZQBj+AAALx0AGOxx+XrwmNE2pTP++c9djJjLA9A3Xp3xz17elcFMxZmOcjOetdjdSAJcysWGI2C7QCBjOBnp1IB/znjZQoLAAjB70gZGn+vUdjViQnySp5PrVQMUlRh1FWDJngUEMjchVAzzUB554P0qSQ85AOOx9ajPJ4/KmihpGKKcRwKbTAXpUyP5YO5Dk9GVsf/r5xUaBidqruLcAY5z7UmSBjPB5xQAsrtJIWZyxPJY9TTKKvaRZf2jq1raE7RLIAxzjA78/Sk3ZXYHWeDNKubSSHW+ImiO+DcuenV/w5x17mu9XVZrnWHuL9yZEDDKICspGCM88A+3cmq0dilnFZwRyR+WIyYw6kMrEBipHtuXvyPQ4NOcxXrTtGwJuZV8xif9VGGLE89TkoMf414lWq5ybZjK99TY0yeW6AuFDyxywAISQgDFTjr7sV9yPpToI5LiJUkkh35GJGUEuSzKVDHnaANuOAeGrNtbm6vL+GGOIbmVvNYvjorDJJORjG0A8ZPqc1Dqs8086iztyrrbBkMJC7mbli+T94oQB0AI9eahWGpLqbc6x2uhm3gnZYyABGV3DauA2WPck/pxTLZWnMbzEhZcHIBOFXcR057rzz0I+lK5uEubKVAiFLsFTDlkDjCsrBlx02g8AgnA9zMLxo0is5d6xNDGiOrHMoUEyLxxwQOvpgehW421c07+6jt/lTAzu3RpwEbuPfnn1+b1xUUrRNbBiZGgMTt5ZJJy2R14xzx7ZrNSZb3VYp5ljeORHZWcDc2CuMg+23HPGfbNOuQk1oNPTLukhRnA4RQc9ckcjBznkZ9aVxXe4/T5jFJEs7n532KsSkpGc8HB5O3jPTOOvWpzZ2t5NKojJQbITsHMhONo7DgkHGARg8CqEbyx3QuRczhZF+c+UqurbD1B74J+nA74p2trHZ2z36eY0hQuIIhwzEcFvTvjHXcKfkF0WrpxP8wgaRjgBR1UswI3NjAByc+5z2oWxkcSIZWOyQIomP7zbkDrwPXHYgcdqpz31rbyNcX1rcRqHWOSSNmxCemfl+9yue/OO+Kda6sJtQihkaNnR0dZMlh1wWHPTnnv7+hsDkiaTTLJL9B50klrkiBDMqBjjA3DHIAII5H3hms65vmXVZFNvLHAihQZVJXdjGWOOvTK/pQ6T6TJNCBLMzLvKqfn2dTtYAFgCfbjNQzXZuIZ5JWBkjKOFXOG44GV6BsLg+47Gle4m1YueTPFbQpMzNbCJgssrgJLkqApQHoAR16eg5FRpfvHaSwwNlQQ0QEpLYyQCvT5cfkB7VFHLHd3sCpHtdwiSwhdoR+7kL/sBSf949xgUbq8WGK3vlgeSNVy3P3cjt7gE9sY/CnZiub2mQC+hnubq7SOWOU+WyqwLkcYAP3Oh4ODlc96kme0Nqs2p306iOQpEUPmYP3SCCdzfNnHPpiuThuYraCXZCWaUyMgGFJzu2jjnHDDHftW3509zpVxHIpJFwHIKAOoIVw5x23Mwz1GQOOKq2uxUZR6osXU0FvpNyC7xLIyKUmUbgwXlg2B7nHHUehxHa21wZN0aK8McTAvsLhVwT94/dzxgc8+vGB9NhiVppJHNu7K8kJbzQgwQc4zgbmAxnHLDPODsXuuW95o95JAm2CO1QKoONnmFoycA4xyx9B5QJ68LlL5FZtuxiLcOI7eDo6vhvlbBYjGCRyThQmD6flIytp0UMAljeJnZQcbmZnwSGXGB0wfo2OgpUuFmmclt8KhHkikA+98pJB54LA46Y4Paql1p7NY+RFIonGSls+5XCrzuAAOCeACccnjPFJGbatoaNxMLs2bxCBQrYLbyWcAHhgc4HfHXr6CmKwtU8lS5e3dVfGU80F9ofgZGCR1z15HWq0RunRWm3Q2+8zCR1AyAcFOenXOeh47MTUU0cMNpFMYvNjuD5crEbWCbztAJ4G5SeScfMORjJLXDm6mtNPcC6XbKDGsbCHY2VQ5B3Y6Y6Yz1zjHaqjXMTzD7VNP515JyUkIVotxXBJHBJCt9M9c0RX0F43kuoh2JGZmeMkZX7xXByAx3cYwSBgfdymo2qW1usq20ryqyJHcZGFUBYxkg4xweD1zxjnLQczsL9oNlBDC9tHLLxLGzEqg2gkA45OSDwM1EL60ubuZlid7c7jmGIr8u3J4BOFXJ7dFPeoZ7mE3McQ3vGFwAyqJEwwORjjPzgn2JpscSJe3OnXDB5LvcwkAykbkllzkZ6g8Y5DfgKURX1sWrqOdIXuLKSO4T7OQQgO9cDqcgBgOFBU5AwMHpTZb+K002zgmOQ8QKSmMEZUYCsBxgsCDyeCeABzLcNd2+oXH2W2ljjWIOzqWOWIG4jjHJDAjnPT0C1ZJo7nUme7igkgvNzpMXPJYEfMDnGWx8w4OEyRgU0lsxXF8+SdhPE8ciOqPEr4QCUchgTwQemc9c5xwarFrhdNlk+0ebDLCgkkOMwv+7yrA84YZ5xjGO5qH7StheTW7qxt5rZx5I+UYUEgjAByDtbAIzs7irVtKIUaK3BEZiXDsdzZYEcNnkjcBg9MnpyKNEhbogsvKlQQTsTeBVVJCARlip4yQBn5yeV474pmoZht4re6ljaYxKx3HdtIJJU9Nu1oyf91yee55NtLfNEgaERoTJGo3JIVHBAwApGScfd649pCkN2DNeI7W1xKLdt4wYxsUZUDryxPHt7ihWF5Fb7U0TreQXIkeRPNBUkeYhAVhgknK4U5PUK2ecZdbXkVyb++3sHRUjSQgnaVUFmbg8ENg9MFQfaq2pWt5baeZ7fCfZWbegjwxXuwHJyuX3c8fN2FLpiSLp7xyiCMTN5Z8yQiLdg84AyCfmfr2BAHFaJqwJa2Ny21RYI18+3FzJLuVsEpIJcDa+QCpH3hx1w2QrE1nARSy3NwFJkMflxt1IfCpgqeOrZBOMEc1mS+al4mWZUhIdd+I3dQyAttI6nIwOfX3q7LbTPoMupO7GSKYvEJRgMuSCCD03FM47F89aHG7DUekT3NpNL8rpP+8jZgB82QPwA+UY6HHBPFWSJVR9PjvZTtVpYpA4DgAcox7keowGDkkjPENlqWorIkGFZ3jbdHK+4FQB8p5wBkY54+YDtikfzJLyQsipdyATxr8oIc7iq7hwQenUHbJ0GDU+Qytd6nvght4Y2jRLz/j5KltobsQc4+/kkdwO5zUEssmnxNPHKSsKBZFC5BAckd8dVx2PXrWlcqdP1eGWdnWzG17kBRnjDDHIBO9SuQccdutZCw3EtqJI5pWtpUzIrsCGyWA475LEgkZPpxWita4nuaMA8yJoFg3SJvmMTMWCIfmUErjjdx06AHjg1Hrd2be3vYlbZ5xjWSOOQlTIME9AAWG4jnJGSAT1NQC70tYLqORhK0jN5W7JVTlFDEdMrgbR6N6YEotopbOadliuGnQZnjIOPm3MBuHynGc7eysSSM1PLZ3Y+Z2sdLayanJZwPF4fV42jUozX0qlgRwSAcD6CisSO/voYkij1OCJEUKsckUpZQOgJCYyPaijnQjVgjaWKJ5JIwis8LCRSw+UMQcDsBx6jsPW5BeQNbwupkDqDG4/LHPoWIz6e9FFR0KvYqWMhgEEshId5isyp1VMlQoPfAGfxPrVq7nWCO4jNvGRBAWTA6EAHH0II/KiikxrYoxcbIrjyxH9nLlWTK5BXgAdjmi8uoJZ0jhTZBKoDoBjIUgYHpyB/Oiilch6Ectn5RkghQFmjRlLN1JBJPf0GM496pxTNcyySx4Igkjk/1YHzHOPwyB+VFFC2KsWby2lutJmkODBHJHvyeRGZtu0j/D1q7PqVtc6lCTarHEIWjj2AAq5OVORyeQMk85bPtRRVfZFdllprxLeNk3bj5ZdQQFKnDYP4/lWfe7bYx28TYE4IcgdVBIJ9s9PpRRU9R9C+yMS22bY0kauVA+9u5we2Bu+vNLBqFs1p9hVQkjKIvNTdwf4ep68kZ9uaKKXUu9rELQCS4YGdv3kZjLjjlR1HHXgdfQ+tSTaXNPEYkkAJ2yI2ASCzEE8+5P4CiihBZEqaRcTXdsYgWUHy1VmGScfN7cnn8amnkjlj37PLZ4NjMOSDwp/AnNFFV0BqyIJrufTLIFo0WIuEmwATnb1PrnIJ6cjv1qMuz6cZVP7pwshU9EYrkZ55HaiioZPUfBbxNMyNIJVWRQQ4IB+U5/l/KpZrkwXU6SbnMcau3TDZBwc9mydvGfu/Q0UUuhdkQNFCr+csBhjZRwZC2MnqvTBIbBzU8L3EcZt1wjyu2WJz95BgNz06/nRRTvqOUUkVI7yKDS72aaMSJGoRwRhwQTja3bLEZ9ada3MTiJZZXKvGswKjIjyMkc+hwDwc4oooexlF6k6iRrhJljAkk2jDc4LdOc98qD7CqMFw0kMiTK0aXETiPod+w5YHHQelFFJLQYtwsZkltWtyYbYKyZbIXg5OPXJbn0Ud6kt0tPsRg+zCMhlKlACuQehB9F7+woopXDqaV7YrBpEl9DMH83aFQk8LsLZAxgd/zqh9vS8MduFzcKy5ZvQcYPrxn9KKKuSHNWasRQFUglihgDyl1WJnYkAFgp7992fw+laZlkMRKIjAuJmUcAkNjAz0xuGOv9aKKXQaehTaWZ5RdRMYYm2ypEWIES4K8Y74HT/Zq9Zzl9r4BkZhFtPzbweec4+6Oh75PoMlFKwky0szxkBrhJUO9fLeM/KVBAKkd8/TpVOP7JcBmi8yJo3SNQ5LFQW3bc55XkkDH4juUVRV9LBfs0JS5yiTyxDAjLBTz+YBDEdqgun+0zjch+zoSuVkwWBDegHbd+QooqXuS9HYdqTkPMbFYkLQrLGQCucLxj06+3f1qib6NYp41j8vyS3AH3WIySOfrx+XoCiktbhJlkQNdQSz3KlLhyHjZeiBuFbqcjoMY9TR9mlF3ckwwfZ4wGlRl+6OSCADjjLHAHJooqthrYlWOa+8uURqxhj3qnAAfI2n6gN+tKjxGAFAI925XZRkkhhu65Prx0PqKKKnoBmz2qQ2Mb2+77O7MgRjxvz0Pc5AXvgc1Zm03T9K1G5isXlnjjiEkiTAbZFXgsM5PqNpx17g8FFNbCto2Z1tpD3lnKGnWQLIWVgu3awcEqcdQQSf0zVu2MEaSqoacyXW5SePkY/Lz1wCTkHrxx1ooquhCdixDYFLYvMF2BmmAPIIwuV49Tmq1jZsby4u5XLSugR++MAEjPGclhzjv+FFFSWkMv7mKzDShzNIgE3zZGFztTtwQRjGeMdT1qG2nsNQQSTwGG7mfbHIGPPIPOO/XDcHrntRRSS6k31Lst1FZ2kquLiWYTfZ42LlVUk7TlVI5B9M55yehotYZLdYY5IQ0W9MyI20jkAsB9Oec9ec80UVUkrXLSIrzLQSQSyu3k7muCqgkqCF78HnaPzOO1RyXVzYzy2DD93NtVCgUFdzD1/x7Hg8UUUthEh0uWa8mu7Yurw4afzHGXbdtUjHQbQR1yCPQ4qWdYr2RY755UCzv8q4YCVMBlIOQU5Hbnd044KKd2U9LHBfEJo31q0jjUAeQXOGJyWdsnn1OT+NN0/TDBHEyAbgqsSRwcjI/mDntjtRRXsUF+7RojUhijbzDOqlAB5ZPzbRk8gEdgPbp9K47xCsSXLGBQqdAQMZBAPP50UVqiindIW0TT58YG+aHPqV2t/KQVnUUU0SixacTFvRTj+VdlploYLN2uUGJc7R97O3H9SPyoooY0RahHt01x5h3gZ78jsf8n+tctJ8vHeiilcGVpfv1Lbkgu/XYhP58fzOfwoop9BCHj1GaYWAG3H40UUyUMZSvWlCE444JxRRQMv2a41+2SMYBuU2AehYY/pVaUKsK4ypJOR1/wA//XooqeoDvsuNO+1s2N03louOuBlj+GV/M+lbfg8Wkd7cXF1HcybYtkawOFIduhJJ6AA8d6KKiprBiex17awbpbNZJB+7KsrKhITr5hUEjgeWpHf5QK2tPtma5mhY5kRkM+G5KjJYcghs59u9FFeTUSVrGSepKrXEN5dRXJMQuLdU8xCCPNVgobHcYO7oPzqK5uzZTG2mgjSYBpGCfemX7zAtyOx7d/yKKzaBlOK4EwuYbiJINrFzOrdAFOcBRx8uD0zkDg5Iq9Pe77hrlJtxktZDtBIKEBWxgjoTsxgn7hzjNFFBLk2VrNpZdPjnjj3OliHLb8ADhGbB9CyDA69e1a2hr8splTfPjLNkE/e3A5wOwUHjkg+xoooaKRDqazQsLuJt/wDy1IwABErEH3/i245+4p6VXE801v5l0zRQXWVHkSMPLUruDN06dBgEhRj5gAGKKSeo7JjrYSWDXNhcyNcSRF97lFJZG46HK5GCR257VBLpkenGS2NyIpwvzuEZgVw2zHzZXhT07t7UUVTJaL4vo1uY7dd8xwzOswAKDacDIOM7hzjrnNYcbW8rmbe8cs8ZiaPHDkoSuQOOo6+x9aKKEkOasi9pJjMk90Yd8iQFGkLZwuMFgD1bA6E4IYg8EiodVuontJbiCNYI3kMcaxrhAdpZsdwADt9eeMAZooqo62F0sJbafZ3EMEM9s8dxxH/o7hVR+Wyvc5Tkg8cHnLcEc0ljelTKzTxlYH2uQkiNgHgjPBLfTjrRRRLcfVIjneF43dgsqI7RtGVK7Pm3BeDjhc9M4J+hrQgguk0TVLSAxQRL5csJidsHDZdSGB45II6HPA60UVF7Ml7mSqtBezQrbKwKviYsNy5QkfUcH07E81qT6igmtbcRTx6gSI5y7hgQfmQ8HqQzZx6DpRRVWTF0Ift6iO4hMQk+yIJlRzuUljtzyDyDyPTP1y/TXXUI7V4nXy2xFtmQtsbBMZxnBHAB9i3TjBRTatsBWsElme9RLiSH7PuSZm5aEDk4K4z9xsdSCvUjBNmIvBBZy5KxiFYrvc52AbVaNyuDkrkcDPKHnHUoqnpIZm2OoJLe2luzk3E5EUeRkAONmzPU4J4J7Aegq1bStOPtTyOIpX2Rg9S3OFBzwu4uxB9cjmiiqaSSJT1J/NR7h7s+eluZdkzqQfmcBgAM5xubPbGDgHvXk0+YGS8cIiLIm187Tu3gbmKgE5z26beB0ooqWNamlHPbTWORZxPLPJJsmfkoAikj83K4xjjsKylvrhJpUlkaTypJ08xm5BA3KTxkjPmNgdMnbtbBoopxSsU9B8t0ZfnlDKQgZ4toBVlJycg4xv3fn6U15JNN0NnikuBKjJLGxIBUiMszAg8HaoYfT1JooqYK8rCXckuzIdTktopWYLHJuaElWj24JKljyVIPPBPNQL9nluUs5ETb9pZUklyATt5U7OgKZXp34wRklFCVnoHU1/J8y3mgmyjqyvLskIViT1242ggvHg443Me5FZzteXMNxDG43wzDMeAU8zIj6N1GSpzRRR0uMp3W57y08hXma3hYqFIjXaFyzEdOdrZUDjHGeM3NIgfzJACqwXKmJJEAEg2ht3JyRuV+ccc+1FFEnoEVqXGullWQMvlm4k+1K+wHDb1VzwcknPsfcioNPgF9NbqiMwmQ25jZzucuzRod2eAHw3Un5R1IoooguolqylaWa3MEplZzEsyibd1mBwSAM43Aow5wDuzxiponkZLdPtErx3MzRxmc53FcqORyOQQfXB5GckoqnuUt0Ij3SRqryszqAGPkqcn67hn8h9KKKKqyK5Uf/9k=" - } - ] - }, - { - "items": [ - { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAKXA+IDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD3+iiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAorI1PxV4f0UzLqWtWFrJCu94pLhRIBjP3M7icdABk9q47U/jn4I0/y/Iu7rUC2ci1tz8uMdfM29c9s9KAPSKhuru2sbZ7m8uIreBMb5ZnCKuTgZJ4HJFfOmtfHLxHrXm2+mQQaTavwHTMk4G3B+c8DnJyFBGBz68hcPe6vcm71C9mupnAJeeQuT2A69gP89aAPoa7+LvhK3ZVhubm767jBbsAmMdd+3OfbPSue1X4328ayppmlsW48uS5fA7Z3KvTv/F6d+K8fS1L2oMQ2k5bIPJOPr7enfOTT4rNY4Q9wMLGQc8gj1BYZ9/b69QAdfq/xv8TzR+XZx2dqwbPmRQlmPGMYcsMc+nbr1rP0/wCPfiyxHl3aWF+C+S8sJVwOOBsKj9M81wOrTl5CFiMaKeA2M/iB0P8AntWFJKUb+dAH074e+OGnaoBHfabLBKEGfJcNvfj7qnGBnPcnp9a6y0+I3hu6id2u5ICpxskhYsff5c9+PXj6Z+QLK+NvMrbjgnqDXdp/xMNPeaFy0hTcy45Y9fz68/407CufVFre2t9EZbO5huIwxUvDIHAI6jI71PXy/Ya49liZL54nVcLKkhV0BAycjkfr+uK9N8HfF2yvSLHxDdQQ3B2iK6VSqSdBhxyFbPOeFxn7oAyhnqdFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRWfrWu6Z4d05r/VrxLW2DBd7Aklj2AGSTwTgDoCe1AE2o6jZ6Tp89/f3CW9rAu6SVzgKP6nsB1JOBXzR8Rvi9qXiGWaz0q5mstLPy+WjbWlXBB3kcnIJyudvTrjJyfiX8SLrxnq8iwGWDSosLBblzhsZxIy5xvO49Og45xk+dsD1PegRKbuRz8+DT4WDsB3qoAWIHrU1qD9rjA67qBnWaXYqqiV9pbsAD8noT6/TB9Oua6MRW8cIWVxH8mPvnOSeAR09AcD+lYMup2+nwRQnc5UZYA9WwOCfb26fWsefVpbl2mZsNxtA4Ax9KAO3jyVd0IYuv3QAFznuRjj37YFVZXheMySTvhDne5yMcd+ffr6DkZxXIWt/fyyFYnyM7mJUkc4H/6h0q42peY4W7ZcLna/l5APckc5+vPTpQhMdd6jbeYxjgRlIwiryWxx/Tn/AOvWFd3stwxbbEqqOESMAAZ9SMn/AOv6cVfvIXZhPiOVOzKQOPwwR+NRosE6FXhnBJ7OoH4/Lk/nVCI4bOO5i82LaCBkqDmug8O6nBBIqyNh1G0D+97f5/pWcfskMUYhs9pVssyvhmXuueeSOhweahhjX7QswUrhux6fSgDttR0u2v1MsLgNMN+T/F7j8/x/GuM1XSL7Sm8x438oMV3jlc46Z+ma6OPWY4wEw8j4wNvHPHc9aeuoTyI6yRqyP1jKhgfqDn+lOwXPZPgh4xOveFzpN3ITeaeB5ZdsmSE9MZOTtPBwAACg716nXyZpky6LrEOoaZDHa3cL5SSOVgQe/GduCDyMcjI9a9O0z41TWxji1ewWdcKGmtzsYD+JiDwx9htGc/guUaZ7LRXL6X8RfCerBRBrVtG7JuKXB8ojpxlsDPPQE966ipGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQBheL/FVl4O8PTateq0gUhIol4MkhBIXPYcEk9gD16H5O8T+LNa8Yak91ql4zgsTFACwjiBx8qJ2+6AepOOSetdZ8YPGbeI/ETQW8ofTNPLRweWR87ZwzbuepXg/3QCMZNcHdW8NrYJvlCzEZZV6L7D/AD60xGZLCzcrlienPXnH86pHk0/zXG4BiM8H1xTwjZ2gY9fWkAlvGXlyP4BuP51LbuiXqygHYuW5PXimD5cgHk+lAU7gnT1oAdLI0shYkkmgc4zj2oOA2B09cc1Jt+UHvSGbOmIXtWUt8q5J3E4z6dsfhn6U+WB5FMRjyw+4rIfmPqMAjHqOvI6VUtrjy0EfyqWIwcAtnvg8D9RVhrofM7Sq0ZwDhmYdf7uAD7dev5MRll7izlK72VgCMY4GeoxWlaXi3EoFxHHlzgdto9z79P8A9VPcR6mFGTuHRBj5eevHHPHqakis7fTys0zhDnoHGSMe2eufzpgXptMAjaQcJgcn1A5x/n19MmtHbiA7mAJznjnA96sDW2m4gTCjgt12gY4Hpxxx71b+2RlVLptcqc7DjA79fx/LHsWIht5IQTIwG/GFJA5/Tt0/GrrAxbnaMADgYbn9Oef88VlnVba3lL28CbhwHY5+mOo9vyq0t61wF3FWY54bg5PXOOc/T0PTuwMq8up5r7aMoBjaFJzj3q7axCaKSeaRJEhZU2hwdu7OCw687Tz9OmRlt1GzBbxA3lO+0yoAMuADhcd8Y/Oqt4zXMUYjKQskexA2BvOQTlsDjJJGeF6Z5yQEbaajpNvIs1+5FsSAY4lAfGM5A4/Ljvz3qTS/iTfeHb4SaBdyx2m7LW0x3RPyCcoeASFA3DBx3FcLcafqkjt5lrMQmcuFygAOM7umM984qmR5TbMgkdcetK5TZ9m/D/xtb+OfD321ESK7gfyrqFWztbGQwHXae2fQjnGa6yvEP2btOuYtC1zVJGH2e6uI4Yl5zmNWLHpjB8wD/gJr2+pYkFFFFAwooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACue8da3P4d8E6rqlqubmGILEePld2CK3IIOCwOCOcYroa4X4wXcNv8ADbUIpZFWS5eKKEMDhnDh8e3yox59PwoA+TtVmy4jGMrxxjGMev5Gq89/cSO5MrEOOmfWlvlG7cSd7Y79BVM0AWrWymvCzoQAM/Me5xnj+fsKSWTyyY1+90ZvetDw/barqurW2n6RZvd3chASJVBz9SeFUdySAOTUXiDQL3w3q0umaiuy7iOJE9D/AFB6g9wc0CM9JNucY3dqlVcL7nrUUYwaswqrHLED6g0hj44SAZWBO0dccCo2nzJlecc8ipZWEqFEBx0B/wA5/nUJtpPL3hSR2IXimIs7xKud21sYJOe/Xv8A5zTijsP3j5UYAyOfp7f5+tZqsUbPQjpU4kdlG4sfQf4egoAvwssIEaLsDdZH5/IdP51dfTrSY731AzPkZ6emMf5PpjrWQ7RKMyykt2Ren4+tRC6YuCiqmBwBx/n/AD9KYHSbba0Ecix/KOQS33e+eMZ/L+lVmulf5dwIznknJ/D/AB+lUYb+aVfLaQoD03HIPpxQsi2xLOmW/gw3GaYjQmtYUjVy4Lj+Hbkjk9x/P0z7VYCeTaqpgfayZfcMdQCPfjk54/Ksb+0bwySFZ8OcEnJzx36e36e1Vze3OzBuAAeRznHvx0PAouBvFljZw+VdkwJcZyAc/Xrj9M9RWbeyfLuBHy8DDVlGQgYVjk8t7mnSSs5IJyKVwaGtM5XbvO30FNQYYHjIpGXvg4NbXhHw7P4p8V6botudrXMoDvx8kY5dsEjOFBOO+Md6QH1n8LtNm0n4Z6DazsrSG387K9AJGMgH4BwPwrrqZFFHBCkMMaxxRqFREGFUDgAAdBT6CgooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAryL4/X5g8O6VZkL5c1y8rEg5GxMD8PnP6V67Xz/wDtJarEtzoGmxyf6QiTTyJg8KxVUPpyVf8AL3oBnhF4xe4Yng9/rXR/Drwl/wAJr40stIkaVLQ7pbmSJMlI1GT9MnC5PQsOD0PT6R8AvGmq2KXU/wBg07eQRDeTMJMEA5IRWx1xgkHIOQK92+Hfw00z4fWMohl+26lOSJr549hKZ4RVydq9CeTk8noAADf0jw/4f8I6fMNMsLPTbZV3zSKAuVXJy7nkgZPLHgV8c+Mtc/4STxhqmrBpWinuXaISn5ljzhAeT0XA4OOMDivpX41+LT4a8DyWkDbbzVS1shx92PH7xuQR0IXsfnyOlfJ+3AoYgXrzyKeWwOOvb2qHdijJY0gJll4559qvxM1ygUAkDOSBtH09Bxz+H0rLCszbVBJ9hWvZOsFpIvmAkjJIUcYwRz1P8hnPPVQZQuIUXBjy3OM4wD9B/n+ghaUqNq9+p9aWV/3rAH5c8HnpSFUGPmLMRk+g/wA8UxEYVm5qdI3UZXDeu3rURzlSRlT0A710uk21jLYO0zxqM5kU9aqMbkzbSOdJ7H/9VIZX2kE5z681tanZTacEuJIVa0ndhEGyGAHr6e2fSqscVncsAA0Rbj5umfrV8l9mCZl9aNvftWxJoUwzt7U+Lwjrt1bG5tdLvZ7ccmWO3dk/MDFTyMrUxMge9G4mug8MWmi/27HF4nkuLaxII8yOEyBW7FgCGK/7vNesWvwA0fxBCNQ0LxpFLYyY2+XaCUKcA4z5oweRweR3pctlcL62PCBliqgE8/ma+rPgt8P28J6A2q6jC8esaigLxSxgNbxAkqnqCeGYcc7QQCvM3gr4KeHPCVzBqFw0mq6pCweOecbY4mBOGSMHGeRyxbBUEYr0qkAUUUUhhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABXlWofCafW/jHL4r1aWyn0QeW8dmd7O7pEqgMMBQu4FupzgAjBOPVaKACiiqmqXcthpN5eQWsl3LbwPKlvF96ZlUkIvB5JGBx3oA+Yvjzqc938SLi1kCiKzt4oEwOqlRJkn13OfTgCvLi5V8ONwrY8T6ndah4i1Ce9t2t7qW4kkniaMoUdmJZdp5GDxg81ibtw56imyUWhJanpB+bHj/AD+HWomCqQTHjPTB4qLgdenrV21LSZTb5g96QyvNO7hUIRIx0VFAH4+p+uTQJnhjKxuRvwWPqAcgH2zzj6elTtAoJwOO4qtJCwO4c+1ADPvNljnJ5J5q0ql3MiLmMkDryvHf/GqhyDgjB96mtZ/InRz0B56GgDUg0hruVImdUDsAsvmAInI6/l7dQcgVLc2l3pcUX2uJeSSksJDDjHXHT8fyqKS+jUrJGjwSY6qpVT9Rzz7/AFrTsdbumCIIzcIMg+UmSQOcYI9s9+M9q1hy9RNkM2pvqtgtpPJHIqkEuR8ye4AxnqaffWWmi2tf7MLh0BSUPktIc9fb/Crj6po8+ofbri2iScyebuRRnJ5zg8fpVaTUNDiDGC3TdncuUGMjp+H+TzzWvLHuStDT0fxDF4avYhNaWerBogyhjnyiOSO4OPUj6V7H4H8bweLbWdwnky28gVoiwJCkcH6dR+FfPVzNHcQMkRQIrBtijAbr26cf1PqaveG/EF14Yn8+1t94dwzjJUsvce46496cZ2dpbFeh7r4p+H2l+JJGnXba3h+9Kq5WQ+rD1/2hz9a8qu/D+r+D9Tjk3zWcxz5N1bSld3GDh1weh6cHB6V7PoeuWutaMl1ayb43XPuvqCPXPGKj1GO3v7V7K9iWWFyDg9iO4PY+/v71cqVylLuUPh38TNQ1DV10fxBMk7XLf6PdbUj2tj7jAAAg4wMDOSBznj12vlXUrG50DWgqSyKFYSW86na3HIII6MP0r6Z0DUW1bw9p2oSGIy3FukknlfdDkDcBycYbIxnjFctSDixu3Q0aKKKzEFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRXnHxn8X3nhbwjHHpl19m1C+kMayBTuSMD52VsYVslBzz8xxyMgA8R+Mclw/jzVlm1mK7RZhsijAUKNowhC5yV+6Se6nvxXBPEl0rSW1uyfMF2ht3zHJwOOnBx1PHfiopfNmLSkMRn5nPr9akeVrv7qxoWYZVMIMj26DqfzobuJKwj2Vylt55jzGvDEEEp6bh1Ge2f8AGltZjGwYAZ6VbtYrhp5Lid2Hlxy5nV/mR1Q7QT7nC89c4HsxGTUXzDbRw3KozvtIWNwBycHhT9OOmAKBsldSw34BJHt/MYqNUyc4z+IyKrCbjlWUcdt2cfX/ABrQtpGFuTKQI88glgP5H/D8xTJI2shONuCrYJ5HSsuWF4XKsOR6V0NxJsQtGYkjC/wBTv8AzAz+HI9BWXdMrrnA6cbFx+f6UMLlBW2nsfqKkEo2kbV56nHPWoaBSKNLyzetlrtPcyN8xPr/AF/GtC3tILZQyYNyPmRlJLDpjj/PX3rn80/edm0HAppknV3Gu3KKFeWBXU732Ab2P+17/XOPzrCv7hpXMhlVsEqDvzkdR/jVKXEkrsD95iRTWikUZKMB7im5OW4zpPCHjO98LahuQtLZSNmaDP8A48vof59Pp7fY+MNK1jTvtEVxGSFzgnBHtXz5oug6v4gv1s9H064vJ2x8sUeQoJAyx6KuSOSQBnk19BeBvgNZWFtDfeJ5pbi8dMvZQyFIo8g/KzKcuRkHggZH8Q5relX5VaWwmn0NJrSz1K3eOS2N0rgnyVQuxxz8oHOfpXoHhfSjonhmw05i5eKLL7yCQ7EswyOMAkge3rV6006ysAws7O3tg+N3kxKmcdM4FWairV59kNIKKKKxGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABXB/En4b/8LDTTVOrf2eLIyE4t/N379v8AtLjG33613lFAHz74v+F+meCfCsN9JqCXbq6wbJYSu5iWOUAJwQo7+jHceFryrU9PtWJu2mEcH3flX5ie49v1x07V9S/FWIz/AA01iMSiIlYiHPQYlQ8+x6Zr5UvnP/CLJgnm7MRBYEfKi9MduR60yG7MyHkjTiOFGjByBIOfxIwa2fDt/Gt1IsP2LT7oxN5dxJvZW9UIJYcjODjOQPWsYAkjqOOc1XkXa5FIrdWOp1HSXRZLi0cGIZfbhSR6n8v88ViG4k+QKyAg5+UKhHuMDgVngkdM0Dj2oFGLSszQm1KYvgkHaflAbp68iqUkjO249aZg+351JJCY5DHuVioydvI9/wAvy4oKIqKUg0mDQAtFKuNwzkj2rq/h1Jp0PjrR31W3t5bEXC+d577FX0ZieMKSGx0IGDwTQJuxyqsVYEdQa+xvhNry+IPhxpcvmRtNaJ9jmEakBTHwoOe5TYTjjJ7dBrP4F8JSSxSnwzpIeJiylbONeqlTnA+YYJ4OR0PUDGzZ2Vpp1qlrY2sNrbJnZDBGERckk4A4GSSfxoAnooooGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAHzJ8a/iNNrmqz+GrOPyrDTrhllZh88sy5Un2UZIHr1PYDyh7wvpMVltPyTvNuyMfMqj0/2fWus+Juh6jp3xC1k3lk1ut3eyzwE8rJGzEhgffv6cg9DXFHcjFXXBFILaDxOykZ7elBYSHnk010woNR9DQFibb6U7HTIqISMO+frThN6r+VBNia3jie52yo7RgZKowU/mQf5VpTJpwUGJJLZsYJSLeT+LSfqAOtZ1sQ0jvuGduBk8nkVJIWGPvHHTcc0wILra05KuzDHVk2n8smoCBU9wnlOVYEMOqk8j6+/tUQRmyQCfoKQDQKnSRoo5GVirEAAg8g5B/pUXGaXAknVS4RSQCzdB7mgNz6B+BXxIv8AUroeFNYmEwSItYzvnzPl5MZPQjaCQTjG3HORj3ivnv4WfB7XrDX9P8SahqEdnaRYmiS0m3SXAIBAPGBGwPPcgEYGcj6EplBRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUVWv7+00uwmvr6dILaFd0kjngD/Htjua+fPFfx71v7XLFocdraW4kzC7IJJWUcfNklRnrgDjpk9SAeofFK7s28Pf2cUjnv3YSxQF8EKMqxI6YILJz13H0r5j8Q21tHqKWlopihlwxjZSWjPIxzyPp7+mK6jwLLqnifW73UdT1CebhBJLK5Z26kKCc4xtH0AFZnjyQw+PlSGMK0CxfK2TyVD89+d360KW6exXRI52+0W8sIhLImYGyA2OfxHasxICzKCQC3QHqa9OWSG807zrtlO4lmLfKI8HhQPY9T3PtioRo9vPF5n2VVGcqzrgn3x1H44qOdNhKLR59LYeVtXduZhkfSlFmq2ssrNyvAye+f8A9ZrSnH/E5e0iyCHaMDsB9P1/AVDd2MkVv5IIYHBz6EAZ/wA/SrM7mMDin7zHJlGzg/Kf61M+nTqhcLuQZyQe1QiFmOAQT6DJoatuUaOkz2NsGnulV5g37tWBIHHXA98Vp3XieOW2liWMnfGyDCYGSCM5zmuaKBSQc5HYjFJt5wBzU2RDgm7se07ybVOMKMAelbOl6alzp0wYENIPlY9j2I9v/r1jNE8JAkRlJGfmGMity2uWttOikJ+Q5U8YxwcY/X8qG7IuKTdj6G+BHi8654ROi3cu6+0kiNQzfM0B+4eTk7eV9AAter18KprV3Y6tJf6TdT2EpbcjwSlGGRg8jnufzNe+/Cr4zXev39t4f8RRrJeTMwj1BSkYY9VR04GTyAV6naMZyaoD22iiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigDyz45a9Lpnhi106PCpfOzSuSOVj2nbjHUkqcg/w+hOPmw2bEmSUYLc4zzjv/n3r2v40W5u/HNoZo2S1tbFJJGcZV/nfGMZ4yccjt9M+U6i4e4ZgDuaQnB57nr60wM+yu73SLoXWn3DRS4x8pxn+n/6qdc6tc3viJ9Wu7bzZGILrkLnAAHP0AFN+UDnoR/n/PtTGYDGec9v8+tKyC9jotI1rS11NmlVY2mwwWVRmJ8nOCexGOc5z9a6SW6ZiHYoiDlfnByPWvMZVSThvp70xTd2u4wTuiYyQrYBH0qXDsVz33L7SA+K3dXCg3D4Ygkd+1dFNoSDDyXkzHqwUKoJ9uuK4iK6kjvEuCqu6tu56E/hXRf8JaZE2T2rLx95HBwfoR/WlqRZ9C5qmmi3jnZAWTYR9OO9cvYOEuWLMAChGT+FdD/wk9pPE6uXj3KVKsvqPbNcxbyRRShplLKF4AOOa0dRuz7B6jbkqbmQqcj1q1pWRdlkK+aEPlhuhP8A+rNUZH3yM3HPoOKbuKtlSQQeCKncGjW1R2EEMcxVrjJLEHOB9O3/ANaqLXL/AGIQbiVLZAz06/41WyTzSUWBKw4DIp0bvE4dGIYdCKYKXPNMo+gfgr8StZ1PXY/D2tXst9HPE5t5JcNIjrlzlj8xUjd1zghQMDNe+V8efCOW5T4l6KLVwkjT4LFGf5MHeMAHqpYZxgdSQASPsOgQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB4V8aLhD4ojWNo3ZbOONxgNsYtIQG9OGDY78eua8dnVmkkYEFuikDt0BP/fOffNd/4vvrfxHrt3qoKoJ/mVWwAEVdqg57lQu7BI64xmube2WZXjWIMkRy3Te2ceh9jz/jgNgcm8hHbPOAP0x/Ko3YEtk8Y7+nH9at3FhNGoZhvUthto6dP/ihj3qiyNtQnOD39/b8xSEKjZOCevBHvV2GNZg0eM7ULADrtH/1uKphSrKccDgntW7paG3KSAH93JvcHuoOSPy/PIpiMm/0s2nQ5ChSx7c4/Pk/p+NR6fpr3t3HHtbbt8x8DnaOuPfHNbUwa4iknZszXDKRxwo+Y549cr+Oa0LVY7O9zCrMsaD5SQDggewP5dPekkO5k674XNpG9zandGgO9Bk9Ccsvtj19G56VzBJHFekW940TgyJ5kUbMeU79+OuMDpxxj0rIu9B0+7e5eFjbpG21NvzbiT+vJHpjjseAL9zjOtFbt9o0UMYMRIIOCSeO/wDk1kPbyKSCp45PFAXIaKKKBhSikpRQB6/+z3Y21744lmlDCaxtnniK45LYjIPHTDnpjnFfT9fHvwd1s6J8TdIZnmEN3IbORYz97zBtTIyMgOUP4Z619hUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAVS1jUBpOiX+pGPzRaW0k5j3bd2xS2M9s461drh/jAwX4Va4Tn7kXT/rslAHzXa6mZ/llA2nOexx3+nU444Ga6OylhFk1xIsYThm3tjLHphc59DnscjoOOCtZAJF3NkE4Oen/6q7Z2ifSrVfOWQTsFkVWBOAMHOD23H8R1OOQZnxxR3KJFM7bmBZ1AzlgufXv8v05qhc6WmWCgAMxUfXJ/Xr+v1qRHLsryYwzbc5BGCRnA9ACvSryJI9xHFAQJTKDgScAZwPyYkCqJMaTT5IFiaZQqlVJ3DIGU7+/Xj29avxOPs8iFRG8qbo34G0ZbDNntwPwHpWtcrFqkTtkRJ5u0EEgrGPlz7gEfXrWXeLNkJDkTn5sBQARjkY/L68/igMaC7KGNpFKPASdpHGQc9O55xj0A9K145SYsyy745SQxUnLjgtzxjOGP4+1c8UMuoRRFW67D349enofyArp47aS3tleQEbojkenQD9AMD88CkDQ2R5JQjnOZz+8TAABI6AdegII9aWLyUhVS4JbAIxxkg9fbgfiAeKzru9EkyOxCxgtgJ3GQe/1/Qdaqw3bNcxtI21VyDg/TgfoR9BxTEaSwm5TzSmUKjBOcscHPH4HPoB9Kjl0l3tcrtLAlnYZO3ntjOf48gdwDyMVqaYSfKlLOx+Y79uQm7g9+/wAy9+Ce5yNKOyjZIg5THm48sEYCAcgj0LsR25b3oA851PTpbGXDo4HTLDqec4x1HFUK9LvNOhvLeSK5yivsjVs5Y7SBuGfpyR3zxXnl5avZ3DQyDlfSkUV8UoFFSJSbKSHQAmVMfe3DH1r73r4KiUm4jUd2H86+9aES9wooopgFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABWT4m0G38T+GtQ0W5O2O7hKB8E7G6q2ARnDAHGecYrWooA+EGVoZzG4wyEhgexrrtH1OIWptjneSCrEcqAc545B64A7muSupjcXks56yuzn8TmpIJXjIC5xnpnrR1EnodM6xNO0nKRbfmGQxdcMePXjjjpxnnNWUAmYqUQOyKGK8ZbBG3vn72ecnEZ7mucjvg04aR0yT85Azu5z07jOOpBNakN+eJFRI44dzhM5PXhCRgEnJXgcDoBgUwNyRF2JH5mEMZyhbH8TZOO317+lOigKQsXbJ3LISS3G0EbceuST+Q9RWbbXDRFWkk3MSOTncwAxgfUbs88YI6jm1Z6miBg6CTywWD8ncc9Tnkjhl/EZ6nABiarLBYX0ixxkS5YZYckn+L2PX8wO1VpdTknQoDtRiThTgYx0+mOKoarK8uoyPIcyFjnn3qOKNpTtjDEkcEDOOaBDpnITIPuagi3yScnqc1bFlLNbvIAQFxgH+IHABHrzke354nsbKQBptrBV/5adMH1+o7fh04yAdToaxixQBmO2VT8pIBJ6j8MEk+i/gb0kywWygyBpJI8O4U7egPXp3z3+nasyKC3gUMGZWiABJODyAMAY9Bg+gJ60T6jG0PlCUFQpJzjDNk8rjjGMkegB9eWMbJMrSLI5cpvKD5uhBBGCO2AM+4Ppzz2oQpKW6si/KrEY4yRnH4ZOKttcIs7htxhc7XXoSMkBs+vf05P0MEsc8beSQHBBGMfiCOuP8P1QGFJAYj6j1phOOnStzyjd2EibFMqgYwMN25I7+nHsTjjOFKrJIVIwynBFS0UmXdJ8qTV7RJ3dIjModkQMwGeSBkZPtkfUV9218EW8hhnjlHVHVvyNfdmlX39p6PZah5fl/areObZu3bdyhsZ79etNbE9S3RRRQMKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigD4MOAV7YUU0ZLYClj7dqJThx9KmswJJkBXKk8jftJ+lIlbE9lptzdODEgx3LYwPw+g7//AK9KS7t7SMRRSZkAA3Lg8+uPUdvqTyck0tWuZIWNorSxIoAKb/Tseev8vQVi7iDkE0xnVJdwtmM/KFO7g57evc4LZPf8jWkJI4rKUxkM8iA5DcMpznt1ByMe/rweFE0gOd1WodQkUlSxCsMH0p3AfdvuuCwHXpW7o0CKk5ZARuHQnGDzg/8AfJ/M1zsuDIrEkAtkkdfrXUWl1ZC7YWkUkcMMQAjd94mG7rkH73Jb0G1umKYiz5X7oh5EjQcEYyZOvHfs5Pt78AzwsY7eS4l2ooYKpwAeM9yOOnue5OOKTzwk4QNhgd2RkYGQAQT/ABd89iccd+Z1rV/O2Wtu2YohgNjB+gx7YH4Uhl/U/ECCMxRjORtwCencnnqQP/1nmsBtQkMxk7k5/GqZyTk8nuaMUgsXBfnoQcYx+H+RV+11OPvuRyCNwP6/4+xxWKF5qeIAMNwyPrTA6W3MdwyeXMkRxymD8wzyM49PU46e+Od1BSl3IpySD1bPP510FjbhLGaUSFgy8rCwLAf3iu4cDBHII56c5HO3ZJkyTk+pXaSPoOKTGiEfd/Gvufw0NvhXSBjGLKEY9PkFfDKdB9a+xvhRrp8QfDnS7iSYS3EKm2mwMbWQ4A/752/nQtiep2lFFFBQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAHwTL94fQVZsWKXEfK9Rycce+CKr3KNHI0bdUJU/gcVJZZNwm1iGzxhSxzQQgvypuXHHBxx0FUjVi5UiVgW3HPJxjNVzQUhKKWkoGSK2UK8e1amnzuGgOdgXeGYcAKRg8fQn9PeshSQQRwRWvCyyaSz4VSjheM9P8SeePbFAi/e6pI1jKcMtxcfNLISQTyTgZ/EcY749+b+lWLp5Sf3nGcY5PAAwB+VVhz1NACjFLxSDijOTk80CH+2ali27hxmoAfepof9YvzEHPWmgOkkMMOlEqy5YjjIAU44bGeD7+3J4xXNz9CCCD6H1rc1C5/0RYPnDgZb5jn1xg9uc9B1rn3PNJjEX7or6H/Ztu5ZNP8AENmZGMUUsEipngFw4JA99g/IV875r0v4G60uk/Eu1hkeJItQiktGeVsYJG9QDnqWRVA77sdcUkI+sKKKKZQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFYninxLaeGdIe6nkXz3BW3ixkyPjjjI4HGT/AFIrbr5h8Z+J5de8TXV67MIN5jt0J+5GpwOMnBPU44yTSk7IqMeZ2PLdSUpqN0rdRM/8zUUTENkMB7mrmtKBqkzDpJhx+XP65qhHw+CDg+hxmkjNqzsMkbcxJPJ7imU6Q5Y0yqKCiiigAq9Y3McCvujLyfwBsbATxlgR82Oo5Azj6VRqaKXarKRuHXB9f8/jQBG7M7FmOSeSaSlZtzEgAA9h2oGDQAlL3oxik6UCHg4qzaFfOXfyM8gCqg5q5ZqGmXKjGe/emhMsakJIpipXaCBjp/Tj8qzTx9Ku6i7vO5kO49AxOTiqBpDFFTW1zPaXMVzbyvFPC4kjkRiGRgcggjoQRmof4aFOKBH3L4W1yPxJ4V0zWYyh+126yOEBAV8YdefRgR+Fa9eM/s7+I3vvDF9oM7gtpsokgBIB8qQkkYxk4cMckn74HGBXs1DKQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFfJHxC0mXwz441DTpItlo8hmtcbiDE5yoBPXH3T7g8mvreuO+IvgK28d6GtuZPI1C23PZzknarHGVYf3TgZPUYB9imrgnZ3R8manZNHbxXIfepO0juvp/WspJDHLkHHPat7VNPvtJvbrR9Tt3truFtksUnY9QQe46EEcEdKwHBDYIwRwaUQlvdBMDvydvPPykEfpUVXY4JLhNqrjAzkt2689h+OKrSR+W7LkHBxxVAR0UUUAFPTo+Dzj9KZViyRJLuNJX8tGOC2OmeKAK9PRd3cfjTWBViDwQcU6MkOMUALsNAHOKuPASm9VyB1I7D39KrjAOSSKCRiRF3wOPrWrZW+xwzEY2ggqff1ziqa3SoBhc+ua0dwayPlru3DIU8hj7DPDD1HIz6MKbAyrrb5pAUDHHyjH6dqrmpZf8AWEgED0PUVFSGB60oNNzzSigD1T4A6qbH4kx2eMrqFrLBj0KjzAfyQj8a+qa+EdF1SbRNbsNVtwrTWdwk6K2cMVYHBxzg4wa+6LS6hvrOC7tn8yCeNZY3wRuVhkHn2NAImooooGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAHE/EX4c2HjzTAcrbavbqfs13jt12P6oT+IPI7g/J+v6JqGhapPYanbtBdwtskUj8iD3BHcfXvX3NXnXxi8HWfiPwXfaj5SLqWm27TxTZwSifMyHAORgNgevcAnKsS+6Pk7zmW22ds8f1qvVtwBbqfkHTgAk/j/nv+AqkY6U0UJSUtJQAopVYq4YdQcim05RuYDNAE12m2bIJOVB565xz+uariuovtCk/4Ry11RmQJKzwhc/MGjVCxPtiRcc9j+PL0EqVzX0xEmDJKkhQDJMfJX3x6fpVa4t/JYbgRnoG4J/DqKLJju+aNWTuSinH1JHFXr2KNot8TOykZbe2QOPXvzQMxyDnpx9a09OhNzbyxBjkAlVA+/wCo/l/nrnAZPyj8TWhpMjw3YfBC45IH3ff8/wBaAKU+e+Cckk1AeBirN5IXmbcoznt/npVWgApaSigY6vs74XahJqfwx8P3EmzctqIflzjEZMY/HCjPvmvjAV9j/CK2ktPhVoEcsbIzQvJhgQSHkZgfoQwP40C6nbUUUUDCiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigArzf41eLV8N+CJbKIg3urBrWMHtHj943THQhfq4PavSK8E/aVHy+GT6fav8A2jSYnseE7Q0RjUhmB4A/l70k9uIolYZA4AzznjP8jT7IhZVdiQAwyQM/WmXZkLkHOOCAfTAx9cZpgioefam0pOTSUDClVtrBh2OaSigDpLa7E+kvvIBVCp+n+TXOv941Yt5isMkfqP8AP8qrHmgiKs2PifY2f6Vp28/2plicbiPuqemeOvqfrx9ayR1q3Z3DW0yyL95eR7nt/Q/hQUaF1aLHOyggv0HPQ5x16nnvUtkEt1Zp0by2XDEYHBAP5cf573dJtxfSlpciNQRvHOBtOTz7jP8AIZxTb9DEZIwir8pAUjPB6/UjI5/2c5Ap9BHN3ahZ22klSeCetQVPIgyVHTPFQUhoKKKKBijrX3npthDpWlWenW5YwWkCQRlzk7VUKM++BXwlZxedewRf35FX8zX3vQLqFFFFAwooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAK8M/aSglbT/D04jcwpLOjOFO0MwQgE+pCtj6H0r3OvIv2iuPh/p//AGFY/wD0VNQJ7HzfCB5I4yc5wPxqS7BeMSHBLAZx37/yP6VDb9G47duasTL5kRbBVS2Ao5A/z+uKYjIIwTSVKyEjPeoqRQuKSnU2gByNtP1602iigBVqaNSzAAZ5qNBk1pabBvnHy7tozx/n6UCOm0uQ2luspwUC52kZDDBHP47fzrJvpnlkDM3mSbctuOM/54/zzW+6Qmyh2EN5ZDSDGA2RuP8AIVgakxUlgMcYBI5qgMSVhvI6gGqx+8alkbczGoqkEFFFFAzS0CB7nxFpkEaM7yXUShVGSSWHAFfdlfEvw+jMnxF8NqoyRqVu35SA/wBK+2qBdQooooGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABXlP7QcXmfDiJucx6hEw/wC+XH9a9WrhvjBY/bvhZrarB50kMaTJhNxTa6lmHphd2T6ZoFLY+R4s7TjrVsE+SQM/3sds4/8Ar1SjJAyWIArQWNTC3XIGDgcdTk/kBTQimibmPQgHv+VVbhAj8DFXdxQtyQSefwqjKck8UMER0GlpKQxKKU0lAy5axb8Z71s6ehjc7V3E/KMDPbn9DVGyRc9DjGAOua2bGMR3aOy8A7zjtjP+IqkSXru6CRFCAz4VhgYwc89+Rz1PoMY5zzN5KDCRjJz6/r/Kt/UWkMssbqd0Zwrtyep46+o/WuWuyd20/gPQUAVSeGplOP3fqabUjQUUUUDPR/gZbmb4r6ZIP+WMc8h/79Mv/s1fW9fOX7NulQ3Gt63qr7TLaQRwxqVzjzCxLA9jiPH/AAI19G0CQUUUUDCiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigArE8ZxNP4G8QRIMs+m3Kge5iYVt0UAz4LjPzdcCti3UeQ5wCNpXGeckGodc00aN4n1TSxJ5gs7uWAOVxvCOVzjPGQM9TWpYYks5kxvjVSxOOhP/1yPxA+lNErYw7oFGcYA28cf5/z+NZrmtC8D+YST8p7HtWe1DBDc0GkopFC0d6KKANjTid3c88/4V1FkqHLyYI2jKk8/l37+/TvjPK6e2JO3Q9a6+DK2yyFeJF+Vd2M4+YD65x/npaJMnVI2fewbaFJGAemTx/SuYuG+faGyB/n8q39QkQRgLnJAYZ9+o/LH4iudfmQ49eMVLAY3YU2lfljSUigopKcql2CqCSfSgD0v4GeIbfQfiCqXlzBb2t7bvA8s7hFUj515JABJXHPrjqa+sq+DxYSJjzCqHAYKx5IIyOB61t6P4j8S6BsGlazqNrEkglEaTN5Rbjkp91ugzkcinZk3sfatFeIeAfjeZUTT/F+RL8xGpRx8HuA6IOO43D2yOpr2m1u7a+tkubS4iuIJBlJYnDq3bgjg0hp3JqKKKBhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB8cfFKxfTfihr8LlSXujP8p4xIBIPx+aqeiAzQtCQWG4Ecnv/LOAPXnit342KR8WtY4xuWAj/vylc9oMyQyyEqS4GVHbtx7+mPT6GmiEVtThAViM7lYqx7Zzk/z/AM5rDYcn610+pKwW4jIACtuw2cknt9OvPvXOSjDDr1702NEOMUlOJzRUljaKKUfeFAjZsE3SYPyqe+O2e3+e1dc5K2pKxgZjztY4xyeffjj/AIF6VzGhxhr1DICUUh256AV0cd0XtWmmLbSzMoGR06YPsfy/Hm0Sc1qZwXYAnAA3DPGMY+lYxOX3Hr1rT1cr55Kt1zwe3t/+r0rK6IffipYEdFFFIoK1dNCRRNK+Bu6E/qPyNZVWiGLFR90YB9BgU0Jmut0kaNkIzHknYDn6flUcF4u5kWNmU87AAR/jVNLVwmWHB54PQe4qzZSwRM0c8SyITxzyp/xrRMg1bS6W1uI5kjVSCD5cg4P51v6Z4/1XwteHUdPuRunwJYZW3rIAxOCvHfPI5GThua5aG1Z5v3BBTrtLAGjV2BTayqzKAu9VC5x7j73bn3qJDR9XeAvHlj480qW7tYXt54H2zQM27bnO0hgOc4PHBBB7YJ6yvi3wP4uu/BPiaDVbdDIgyk0BcqJYz1Bx+Y9CAcHFfZGmalZ6xptvqOn3CXFpcIHilTow/HkHsQeQeDUlJlqiiigYUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB8y/tE/8AJQrA/wDUKj/9Gy157pJC3IZvudxjO72+pxXo37RCH/hPtObs2mIB+Esn+NedaQN0+AgLAAnOOnfr7fpmhEmhrykXZkJ3edGW6/7RGPfj/PSuUux+9zjGe3pXTarMJ4IpIzlQuwn2wp5/EH681zd62Zugzjn61TArCkJzS0lQUJTkGXUe9NqSIZmUe9MDqtEh3OSGAyANzHAB5Ayfx/T3zV64iaztWjBChlIUk5Jyc49vx9fyl0SBViQHO6SRYsg7TyDgZ6Y4P4jtTdakaKMxjJcZLYXBIYA88dc57cEZxVknG3GHfIGOAMelVn4Ue9WpFGCwHfpnpVSQ8gVAIZRRRQUS26eZcRpkDLDrWosKJK6vyudyqB1z+ftxWdZnbNu9BnPpVhm82RyGK/19quOxLGXBRG4Iz2wc1WDkNnr7VKyE9BkfTj61XPXik2CNe0vgML0AHIJq88v2lOZmEmPlPByfwwcfj+Fc0GZTkE8Vp2F8VbY5yp6g96L3C1hlxb+S5G05HXOOv4Yr6J/Z+8UxXnh2fw1PMxvLF3ngQoAPs7EZwR1w7NnP98Yz28Gv44pIll27FI3DKhfbAwTnHr7VU0fV77QdWttT064aG5tpBJG6nv6EdwQSCO4JFQHmfdNFUdG1a117RbLVbJiba7hWVMkZUEfdOCQGHQjPBBFXqZQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB83/tFpjxhpLjvYY6+kjf415lpJbziUJyeBj+f4V6v+0dauutaFeFAI3t5Ig/clWBI/DcPzryvQxvu9p78dccf1/zngUIhEt0rRTXMTuBJgsCD1IOSB2wBmucuTmTJ4Peuv1+ONHt7iM7N2+NlzwM5Uj9f51xspy3PWmxoZRRRSKEqa1/4+o/rUNW9Nj87UreMdXcL+dAHd2lt/o8UY6wlG46sw59P9rA7DIzwKztecfaHYMD0BIHccZ+uRnB9fet/T90iqsi7hKTOWHLIDwD9cY46/hzXMav+5lkQdTyVIxxyMY7+mfY1TEYF0MPjHbseKqHljU8hy+SRzzUKqQ34GpYkR0UtFBQ+Nc5xV9FZIWVeSeTWejlDxVtp8xhT8pA7VcWSyBywbnNRnJp7NnqeabkdOKlgiPFKCVORSkY57U2kUa+n6gqQyQuXBYYADEL9eCOf8fwqjPIrSkqCAT3JP86rjr1xVi1tri/vobW2ieW4mcRxxoMlmJwAAKTEz67+DWf+FTaFnrtm/wDRz13dVNK06HR9HstMt2doLO3jt4zIQWKooUZwBzgVbpgtEFFFFAwooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKK8p8RfHjQNJvJrPS7OfVJYZdjSrII4GGOSj/MWwePugHkg4xkAwP2kj/onhwY58y4OfwjrxnRJI4rotJymMMPY8Z/M11HxP+IsnjwacDpq2MdpvKqJvMYs2MktgcYUcY9eT242wXIZyVAGfw4/+vQiepva7tnsIp4kIcyZYEdOh/PkH9fWuMnBWZlPY4rvrGAajp6K/wAgBBOM9gfx/H6egrjNaiaHVrhWUKS27AGByM8CmxrcoUUlLSGJWhoqtJq9uq5BLYyO2eKz66HwdAJtZMhVmEUZbaozk9qEDOumkhF4FBIjXoFPKjaMgeoHr0/SuS1a6Ml1MzHflj83c+9djIsNtGZfNcu6tlnACsct2PUcY7dPrjz++IMuQe3Pr+PvTZPQp5y1SJHnJPYEUyJQ0mD3Bx9cVbUBYwfUc1LEUGAAB7+lNarEidSBnHcVWNMpCUuaSloGLn3xShjnjn2pBjHQH8eacqbvuBi2Ow70CFJKtyvI7EVGTk59akE7L2U/VQaY7mRixCgn+6AB+QoGJXsH7PvhePVvF1zrVwEaHSY1MaN1M0mQpwRggBXPUEHaa8ms7Se+u4bW2ieWeZxHGiDJZicAAdyTX2p4F8KQeDPCFjo0QRpo133MqgfvZm5ds4GRngZGdqqO1At9Do6KKKBhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAVNUtJb/SL2zguntJriB4o7iPO6JmUgOMEHIJz1HTrXxHq2nX2hatcabqMMltd277JI3ByPf3BHII4III4r7nrz74n/AAwh8f21tcW91HZ6naI6xyPECsykZCOR8wAYDB5xub5TmkI+TmbemST1q9p6OUcowXII46/55FT+JvDGqeENUl0rVoRHcI2VKklZE7Opxyp5x9CCAQQItMjaUoik5J6env8AyFMRqw3zW3yHKxZzjgkd/wDIrH8Q3EN5Lb3UW7cybJMjqw7/AK/pU+orcW8u08k5UgDvzn+f8qy2Ja1kibllO5fbHUflz+FMCkKU0FSACQQD0460lIoK7nwxp5tNPMkysGuNsjLkAlP4Rz65z7/nXG2caS3cSy58stlseg5Nd3ZXjXErsoAx/AAAB7ADHY4/l14aEyfUpn/fOf7jJjJB6BuvTvjnr29K4KZyzsc5BOetdjdSDZcSsWGI2C7QCBjOBnp1IB/znjZQoLAAjB70MQyP/XKOxqw5PlFTz7+tU1YpKrDsasmTPAqWBE5CqB3qE888H6VJIecgHHr61GeTx+VMaGkYopxHAptAxelTI/lg7kOT0ZWx/wDr5xUaBidqruLcAY5z7UmSBjPB5xQAsrtI5ZnLE8lj1NMorT8PaS+veItP0qNlV7u4SEMxwBuOMng0A9Ee0fAPwDM91H4yvWaKGEvHZRbSDKxUq0hJH3QGKjHU56Y5+hKq6bYQ6VpVnp1vu8i0gSCPccnaqhRn3wKtUCSCiiigYUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB8+ftH6fEureH79S/nTwzQMCRtCoykY465kbP0FeeafphgjiZANwVWJI4ORkfzBz2x2oooEakMUbGQzqpQAGMn5ioyeQCOwHt0+lcd4gWJbpjAoVOgIGMggHn86KKpiRUuUJ0TT58YXfLDn1K7W/9nFZ1FFSNE9rxKW9AcfjxXY6baGCzdrlBiXO0feztx/Uj8qKKaBjNQQLprjzD5gGR15HY/5P9a5WT5RjvRRQxFeT71S25ILv12IT+fH9QaKKTH0EPHqCaaWAG3H40UUkMYykHmgITjjgnFFFMC/ZrjXrZYxgG5TaB6Fhiq0gCwrjIJJyOv8An/69FFInqL9mxp32pmxul8tFx1wMsfwyv5n0r1P9nvSrTUPHtxc3UKymys2lhDdFkLqA3uQC3X1z1AoophufUdFFFBQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAf//Z" - } - ] - } - ] + "outputs": [] }, { "language": "markdown", @@ -139,26 +80,9 @@ { "language": "typescript", "source": [ - "// Get the background image data\nconst bgrImageData = await getImageData(`${baseImagesDir}/wall.jpg`);\nconst bgrImageMat = cv.matFromImageData(bgrImageData);\n\n// Resize the background image to the original image size\nconst resizedBgrImageMat = new cv.Mat();\ncv.resize(bgrImageMat, resizedBgrImageMat, new cv.Size(width, height));\n\n// Remove the foreground from the background image by\n// setting all foreground pixels to white\nremoveForeground(resizedResult, resizedBgrImageMat);\n\ndisplayArrayAsImage(\n resizedBgrImageMat.data,\n resizedBgrImageMat.cols,\n resizedBgrImageMat.rows,\n display\n);\n\n// create a new Mat to store the final image\nconst newImage = new cv.Mat(\n resizedBgrImageMat.rows,\n resizedBgrImageMat.cols,\n cv.CV_8UC3\n);\n\n// combine the foreground and background images to get the final image\ncombineImages(resizedResult, bgRemovedResult, resizedBgrImageMat, newImage);\n\ndisplayArrayAsImage(newImage.data, newImage.cols, newImage.rows, display);\n" + "const backgroundImagePath = '../../assets/images/wall.jpg';\n\n// Load the background image\nconst bgrImage = await Image.load(backgroundImagePath);\n\n// Resize the background image to the same size as the original image\nconst resizedBgrImage = bgrImage.resize(width, height);\n\n// Remove object from the background image\nconst removedFgImg = Image.mask(resizedBgrImage, resizedMaskImg.invert());\nremovedFgImg.display(display);\n\n// Combine the background and foreground images\nconst resultImg = Image.merge(removedBgImg, removedFgImg);\nresultImg.display(display);\n" ], - "outputs": [ - { - "items": [ - { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAKXA+IDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD3WlpKKxNBfwoFJS0CDvS0lJQA6k70oooAAaKKTpQMWiiigAooopiCiiigAooopgLRSUUgFooopgLRSUUAOzRSUUCFopKWgApaSigBaKBRQAUlLRRcAxSUtJTuAuaM0lFADs0UmcUA0ALRRRSAKKWkpgFLmkozQIWikpaYgooooGFFFFABS0lFAC0UlLmgAooooAKMUUUAGKKKKAEopaMUwEooxRQAUUUUAFFFFABRRRQAUUUUALmikooAWgUlFADqKbS9qAFoxSUZoELRSUtIApKWimAUUUlAC0UUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAVKOtJRWRoOozSUtAC0lFFAgpc0lFMBaKSigBaWkozQAuaKQUUALRRmigYUdqKO1AgooooAKXNJRQAtFJRmmAtFGaKAYtFJRQIWlpKKAFooooAWikooAWjFJS0AJiiloouAlFLRTASlzSUUALR3pKM0AOpKKKAFpKWkoAWikooEOoptLmgBaKTPFFMBaKKKACiiigApaSigBaKSjNAC0UUUAFFFFABSYpaKAEopaKYCUUuKSgAooooAKKKKACiiigAooooAKKKKAFozSUUALRSUUALRRmigBaKSigQtFJmigBaKKKACikooAWiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAKVLmkorE0FzS5ptLTAdRSCigQtFJ3paACiiimAUUUd6AClFJRQAtFGaKAFzRSUUgFNFJmlpgBoooxQAUUUUAFFFFAC0UlFMBaKKKBC0UlFAC0tNpaAFopKXNABmiiigBaKSigBaSiloASilpKYBQKKKAFopKWgAooooAO1FFFABRRRQAuaM0lFMQtLmm0ZoAdRTaXNAC0UlFAC0ZoooAKXNJRQAuaKSigBaKTNLmgAoozRQAUUUUAFJS0UAJRilopgJRS0mKACiiigAooooAKKKKACiiigAooooAKM0UUALmikooAWikooAXNFJS0AFLSZooAWikooAWikooELRSUUALRRSUALRRRQAUUUUAFFFFABRRRQAUUlLQAUUlFAC0UUlAC0UlFAC0UnFFAFKijiisTQXijNJRQAuaXNJS0AFLSUUxDqSkBpaAFopDRQAtJRRTELRRSUAKDS02lzQMWjNJmlFABmlBpKSgB3FFJmjNAC0UlFAC0UlFMBaKKKAFopKKAFooooEFLSUUALRSUtAC0UmaWgAooooAKKKKAFpKDRQAc0UUU7gFFGKKACiiloAKKKKYgoopKQxaKKKYBRRRQAUZoooAXNFJRQIWlptGaAHUUlFAC0UlLQAUZoooAKKKKAClpKKAFopM0ZoAWijNFABRRRQAUUUUAFJilooASilopgJRS0UAJRRiigAooooAKKKKACiiigAooooAKKKKACiiigBc0maKKADNLmkooAXNFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQBSpaSisSxaWm0vNABmiiloAWiko70AKKKTtRTAWjOKSloEL1o6UlFAC5paSimAtFJRQAtFFFAgooooAKWkozQMKKKKAFopKKAFoozRTEFLSUUDFooozQAUtJRQAtFJRQAtFFFAhc0UlFAC5oBpKKYC0UUUALRSUUALRSUtIAooopiCiiigYZozSUtAgpaTNGaYC0UlLQAUUUUAFFFFAwooooEFFFFAwooooAM0uaSigQtLmm0UALS02igB1FJmigBaKTNFAC0UlLQAUZoooAXNGaSigBaKSigBaKSigBaKSloAKKKKACiiigBMUtFFACUUtFMBKKKXFACUUuKKAEopaTFABRRRQAUUtJQAUUUUAFFFLQAlFFLigBKKKKACilxSUAFFLijFACUUuKKAEopaKAKNFJRWJYtLmkooAWjNJRQA6jvSUUwFzRRRQAtHekpaACikNLQIKKKMUAGaXNJRmmAtFIKXNIBc0UlFMBaSjNLQIKTNLSUDFzRmkopgLRSUtIAzS0lFMBaKSjNAhaXNJRQAtFJRQO4tFJS0AFFFFAhaSlzRQAUZoop3AM0ZpMUUALmlpKKAFozSUZoAWjNJmloEFFFFAC0lFFAC0UlFAC0ZpKWgQZopKKYDqKSigYtFJmigBaKTNLQAUUUUAFFFFABRRSE0ALRRRQMKKKKADNGaKKAFzRmkooELmlptFADqKbmlzQAtFIDRmgBaKTNLQAUUUUAFGaKKADNGaKKAFzSGiigApRSUUALRSUUALRSZozQAtFJmjNAC0UmaM0ALRikzRmgBaKTNGaAFxRSUZoAWikozQAtFJmigBaKSigBaKSigBc0UlFAFCjNAOaKyLFopKKAFpaSjNAC5opM0uaAFBopKBzTAWgUlKKQC0ZpKKAFpaQEUtMQUmKWigBOlLmiigAozRRTAKWkozQAtFJmigApaSimAtFFGRSAKM0UUwFzRSUUALRSZpaAFpM0UUALRQKKBBS0lFAC0UmaWgAozRRQAUUUUAFFFFMAopaSgAooooAWikooAXNLmkzSUCHUUmaKAFopKM0wFoozSUgFopKWgAooopgLRSUUCEdtiM2M4BOKEYlQSMZ7Ukv+pf/dNEZzGp9hSGPzRmkzRTAdTT3opD3oAfRTc0tAC0nejNGaAFpKM0tABRRRQAUUUnFAC0UUUAFFFFABRRRTGFFGaKBBmlzSUUgFzRmkopgLmjNJRQAtFJRQA7NFNozSAWlpuaM0AOopuaM0AOopuaM0AOopuaM0AOopuaM0AOpKTNFADqM03migB2aSkooAWjNJRQAuaM0lFAC5opKKAKGaX2pM0d6yLFzRmkooAdk0U2loAWikooAdmim5paAHZopM0UAKKWkzijOaAFo6UmaXNAgFLmkoFAxetGaSigB3Wik6UZpiCijtRQAtFBpKAFozSUZpgFLSUUALRSZpaACiiigBaKSimAtLmm5paAFzRSUUCHZoptFAWHUUmaM0ALRSZpaAClqtfXcVhYXF5MHMcEbStsGWwoycD6Cqfh/VjrWkpfFVXzGbCrzgBiB+OMUAatFAx/kUZHXNFgCijI9aM0AFFUL/V7TTrqzt7hijXbmONscBgOAfTNX6ACiiimAUUUUgClxSUZoAKKKKLgFLSUUAFLSUU7iFzRUEl5bQkiW4iQjqGcA9M/1rk7/wAWySo0UMYiB4LhyT+B7VLmkFjs6K4az8U3VvGUKpJly2WzwMdB7VuQa/HfW04RzaNGqnzWUOOvTH6UlNMDbk5jcf7JoiI8pMegrFu/EdrarCgfzt6gsw4wP8fb6Vlf27qCr5cIT5ydr8E4Hpz6etNzQG9qusrpbxK0Jk8zJzuxjGPY+tZy+LVKsTaYx0/e/wD1qwdQ1KfUxH5qjMYIzjGfw/CqRfYrLye1S22ykkdMfF0hU4tUVvXzCf6Uh8WTHGLaMdmyxNcrvOcbevvUqCWQ4CAE+9CuFkdN/wAJbNwBbR5AyTk0p8Vy5x9mj4x1Y1zggmzgLxjsak+zzf3e/YinZhodAviuZnx9njC5x941Z0zxDLe3kNu8KDzCcspPHBPT8K5cW8pIyv61d0hZLbU4JpNgRGO4s+ABggn9aLMWh3WeKonV7UXHkFm35I4UnkVWPiKw2tl3UZxkrz9cVgW6D+2YxA7hZCXRUYEgj+9kY9abl2EkdsDkAjoaKr24uwT9oeMjsEGMVYqkIWjNJRTAWjNJRQA6ikooAWikozQMWikzRmgBaKM0UAFJS0UAJS0UUwCiiikAUUUUAFFFFMAooopAFFFFABRRRQAUUUUwCiiikAUUUUAFFJS5oAKKM0lAC0UlLmgAoozRRcDPozSUuaxLFopM0UALRRRmmAUUZooAXNFJ70ZoAWlptLQAuaKSigB2aM0lGaAHZopuaXNAC5pRTc0ueaAHUlGaBQAUtJRmmIWik5oyKAFzRSZooAWikzRxTAWjNJS5oAKKSlzQAUtJmjNAxaKSigQ6ikophYWikooAWikpc460CFqOe4it4WmmkWOJerseBXGeMPE2raRqtjbWtoEtncSNccP5qjG5MfwcnBPX0xXJXviC71hpXvbkoq8xqM4xyCAB/X8T0pb7A9DX8X+M5ruKbT9NVxbvGQ8gXLSLzuAHYY6/j05FYmjavNZQLG7Si0L7I3RdhX16cHqD+PHFZt5dyXMnl2vSMBTMeDxgg5HXHb6DvzVdry6gnZrvfMGXBY/MDjp1FXFWE9T0CPVbsLuhvrh0I4cTcfqOD7e1LFrupFlQ3bjPGDJnAzjNcXbXDpcm4sLnPHEeeTjGME9uBwe3rT7zW9XnjSaJJ1mjkAkiDn5hg5G373JPv2qr+RNjt/7Y1AMQLuSXB7Z4GOvB98VG2u6ghDG7yeePMwD+vtXNx3V1cIqzMyzJw6iTJTjIB6ZOCPz71M4d5dxlk2D0bB/EE4xV8pNy5rF7fPDDM0jGSN/MjfzN5VgCQByfQdvr0roPDXjmDUFSDUWSC4PAkHCN9f7p/Tkd+K5C6smuU2yMWAGQdxXHbHWsqW3lglHmmTYhyJUXJGeuV/TNS4saZ7tke1LXio8V6xZ6c0FnebQxXy+cqmCCNp/hHGMHjB6c16rod5dz6ZaNqTW4vZohMY4FICqcepPr/hWdi7mpRRmjNIYUUmaKAFopKM0ALRSZrB8Qa8dOVYrZ0MzfeOMlB/LNJtIDYubuG0RWncIGO0E9zWLB4qtpZmWRDEmSFYnPHuK4651S7u1xJPLIuc4ZiRmqjyOQcis3N9BXL+ran9svnfBxnA57VntNg/KMGmrtCk8sT1odEI649sVnuF0L5pyD/KtK2C+R5jcZPGeKxlVvM2gkntW4xEeyPBwqgcKTVwRWhHI/71Qpix/ESVH86ltmeO6EhnRgv3V3A9v8aX7QcYCufwP9aaZiVOUYD+8SAB+tXZDvYkjiURDdnI4x+lNnBK8JgZz071agXEOeOcAd6jnJ2/T2rbl0M+bUzd2H5q5btmRcNjg/yrOd9sgHGatW8oEgJYAYOT+FQmUzSjOUJOSef51IPug8VnR3yDdnoOnvVo3EaxAtjFUpJkMnc8DAxz2ppd1Bwxzg4Occ1GssbqCGB57U/wCU5+lPcEyCS8cAJKuR/tDPWrAukmlSQFxKMhSAM84xjpgjH61Rlboc800qDhlOKycexaZ0emandxX628s6TRn753g447nt9K6rNcBptxDbXSG8hUqDwWUNj+v5V2keo2kkYf7RCATjJkGM0R0BlykzTUkSRAyOrqehU5FOzWghaKSigBaKSigBaKTNGaBC0UUUDClpKKAFopKM0ALRSZozQAtFJS5oAKM0lGaAFopKXNABRSUUALmikzS5oAKKTNGaAFopM0ZoAWikzRmgBaKSigBaKSigQtFJRSAWikozQAtFJmigdjP5ozRRWRYtFJRTGLmlpKM0CFo70lFAC0tJmigBaKSigQtLTaWgYtFJRQAtGaKM0ALRSUuaAFBopM80ZoAdRnmm5pc0wHZpKM0lAhcUuabmigBaKSkpgOopKKAFopM0UAOzRSUZoAWlpuaKYDqKSigBaOe1JS5oERtcwpPHA00azSAlIyw3MB1IHU1zviLxRFpnn2qShLpFR43GHDHdgow6g4HT37VkfELUvsVzaBNRkhfyzugizvZS3B9MEg5/3a8323N5IZXDKkikNJJ8zHnt3HSkk5bA3Y1tY8QNfSB5YwypLJJBCOdrMct9Mntz36ZrPZZbiQtdEordIx1x7+1SQW0cJby1AbuxA3fn2qfy0QbiQwz0U/5/WtYwIchUiIQIiBEHcDg/40SKEXy3AdO+R/Sia4lSweW3gikkDKu2RRg5Jznn2otpnlS4a4t4Yljm2IUTb8uOM8/X8qonUoSWTqd9nKQeuwnr9OxqSDUmhZEu4yGRgQ7DOD9D1/DNW5UgyGEuGI9v/iqhIdgVmQPH2LDP8uR+tKw7llJElnLI2IzKZJFjYjc3HXJPfHGfTgVq2MrT2wMjRtckkMqso2nPbJHQHqMng96537MkT7rScxEZOx87ckYJyO/1x0qMasbCY/bYmkORtxIVHQ5LY7dOMdutNOwrXO0eJkXoGGOMY4H9RQ0eQFKblPqMfyrIh8baY0Kb1CMRlgZQdv6Usnj+wVwNqMgweVJ9fQew/Or5hcrJptJifLxlomfnaBkN+nP4VNFc6vau4AeRX2+b5ecuFJIUjrjkjHOQx9aypfiHat88UUbE8Omxs/nmmTfEGxaM7bVQ2e6Hn9al2fQaUjvLLx5AiTnUYrgS78r5cZYKDwFwMkYx7966nSr9dT02G8QALICRgnsSO4B7eleK2PjFtSuY7SGxE00vyfKm3OfX5un19K9d0yf7BosMU95DPLGpbcAEBUscAD2HHrxzWUkkrlq/U2aM1zs3ieIoDCpVupDYx+lVrzxawVRaxjPO4uufpjn+dZc6KOrLAdSBWLqXiO2siY0HmyAkHBxtP5c1yV3rN/K7FrqQhx0VsDHpgVmSSu+4s5JJycmpc30Ea83iO9aZ5EnIJGBjtz2rHmuHmlZ5XZ3PUtyahyPUGoi5zgcntUDRN5mGA/SofPwSc9aAcH5mGPaniSMdAufoKBN2EMmzsee9NYuAGCn3pXmJwMYFIJXPUUaCv1LGnYmvowwAx8xOew7/AJ1feXfM7B2GOMBc/wBKz9L3fbJH5wsZA9skf4VZE4wSZefdkH8q0jsWibdnHzyfipH9KXdjn95jPXJP6f8A1qgE0TD5Jgw7nzcYPpxT4W8yVRkkZ7SlvzFUgehrp8tsuTk5NUJ7lj8q9jyasXVwLe3TIOQBmsSa5Z2Yrxu6nvWs5JaGSGTSr53XmojKVbrwewprOBjAHvmmrg8EDn865ZO5poTpKCc5596nE25SN3TpzxVF0K4KjIHXPNAYDqCp9qSbQaGhHOysMcBea0P7SDbflHIyeehrDMuw4/I0qyZHFUptbC5TTebcVIHT3p8UuBgnIrOWTAxzTjKykdwTVe0Yamn9owSCMrSjDkmMlT71neZubjPNWVlCxgE01K+4bGpY6pc6bJ+7c8jBV+VJ9ccV02j64168kd35MMgxtX7pJPbBPWuKW4QrtbBHvSKQSSpJH93PWn6DPUaM1w2neI7qyfZMXmizysjEsPof8iuwsr2O+tVnjDKGz8rdeDiqUr6CsWc0ZoopiFpM0UUxi0UlFAC0UlFAC0ZpKKAFopM0ZoAWl4ptLmgBaKTNGaAFoptLQAUUmaM0AOopuaKVwHUlJRTuAtGaSilcLC5ozSUUBYWjNJRQCFzRmkooAXNJmiigAooooAKKKKLgUKKbS1kaC0ZpM0A0AOzRSUZpiFzRSZpc0gFozxSZo7UwFpabmlzQIWikFGaAFzS5puaKBjqWm0UCHUUlGaAFpabRmgY6ikzRmgQuaWm0uaACikozTAWlptLQAtH1pKKAFpKKKYC0UlLQAUtJRQAtFJS0wFpD1X6/0NLVLVruew06S6t4EneLBKPJsG3PJzg9KQip4k0GDW9MmX7PG14sZ8iXaN4IOQoY9ASMenNeQp5gjyctx1C43f4V7rDJ5sKS4I3ANg15B4nEVj4m1GGGPcnmBzjHBdQxwPTJP6VUXaViZbGT5mFx5JHpnNMYuqMdozj0qNpQ21mJT2JH+FSF1dcbA3HBxmtSBsTSMD8xz+NPfDZ/ekfUE1XkmjQLn5fYcZpPMjZsfL/3yAR+NMCZSy4+bIz13f0qQ5DDaGBxySCf51BuGR8hyfoTU0YyCcbe9Ah8m0RbpFIYdWBxn9Kx9YiXyEMedznlFGcdeScAZ+ma0bxikKjcnX/P8QrA1tG/tO5AbICIBlicDA/xqZTUNbFRMvyAtyDnDdDkgCrf2cuMb4z/ANtF/wAaZcIwuZQcZ8wn144x+lAh+YncB9/HHt1/CiWJ5UrJaq5RHFHDGXG4Lzk9T/KknRNvykMR6Zp6qArAtg7en4ipntZJFbyo5HO4jCqT3FU60nPktoHmdT4GshbRyX7oNz/LExJOB0P9RXWy3LAY5x1yO9cjoV62naVbhrOR/MkZWYEjYM9Tx71s3mpCDy441Es8n3YtxBI7nODXPVjJy0BMutJnkUwSlQRnr2rNuLq9t0LS2UKMBu8s3Pz/AIAqM1ctQ13Yfa4sOvO3YGYE+mQPwqFSn2DmSHl88ZqJzkcH6nFRmV1JVwVYdVP/ANemCbOcZJqHvYNRsjMo5ORnrQhO0EDr2pWk+YNjI9qYbkZ+YD64qStSV+cbtuQaRcqoyQDnJqs021yeSKa8xOCp69qLisyyxkzxg+mKaWdCFyD+lUxcOzBc9ulTvveDcVII9aNws1ua2llhb3c5HGAMgdSM5/pQGwPvTfnJ/hUdiPK0ZXBZXkYnO3PfHp6CpA5x/rpf+/X/ANjWyVkihkSqgf5pBuYtxvHX8Ks25XzAVEjH+8Wbj88Zqv5hz/rpf+/X/wBjVq2LPMoYuwJ+8w2/pxVRWopPQNekCzRKB/Dn+dZIfeOCfpV/W3zfKM/dQfzNZjyEAdvepq6yZmtiY5545HrUYPz8rk1ElyWye2cZpvmY+bn8axbRST6lwEY6de1ACkbex/SqYuDu6Y+lSeeepz9afMOzRZwAu0nIFMIwAOp65FIjAjuM08DA4559adk9guRCRgfap1mzjGTmmBt2V2jr0o2bckHaDSt2He5NjIypNPWU8K/6Cq6kp0/MVKJkOB0PvQA4EDqTzVq3lC53enFUvNXkbQT6Gnqdy8DGD0oi9QNhGSQAE5z61Is11bqVgnkC/wB1XYfjwRzWTDcEMPm6dRV6OfseR61spKW4tjct/FV+qBXjgfYBuZ8hj29ev4V1ltdxXUCSRup3qDtDA446V50wVwMkH6HmpbS7m0+XzoCrY7EkfoDTs1sG56PmiuTsvFF1LPDHNHb7GcKzAkEDuetdSrq6hkYMp6EHINCYWH0UmaWquAUtJmkzQIdRSUUALRTc0ZouOwtLTc0uaLhYKXNNzRmi4DqKbRmi4C0UmaM0hi0UmaKAFopKKAFopKKAFopKKAFopKKAFoopKAFpaSigBaKSigAzRRRQBnUZpKO9Zljs0U2lzQAtLTaM0AO/GlptLmgQtApuaWgBTS0maBTAWikzRQAuaXNNpRQAtFIKKAHdKM+9NzRQA6ikzRmgBwozTaKAHZopM0UxC0UmaKAFzS02jmgB+aSkpaACjNGaSmA7NFJmjNAC0tNzQOtAGbr+sNo2nCeK2a5ndxHFEpxuY+pxgAAEkn0rzWTxd4tdz/pwiUqDxabQDnGPmTOc8cE13HjVd2mWv7suBcjOFLY+VuSARxXECEsgEkbHjH3XPfoOTiiCu2S3YBq3iMvHLJf3vmId2D5iIT2yAMEeueKxby81OUXcF5ql5IqQmVg0suCBzjDEZyPbFbTpISD5RxuGS0Eh/Prn/PrXMa5OLC+ugFDCeDyyAGXbkY/iGatqwk7k1jJcPBE5u/MZhsYNIQQpBB6oc/TJ61ZjtVgQrDIojZhhPOkTb1zwIsY6dKXRYJH0uGVQ6sykA7CeMnvn29KXWrm406zSeONp13hSG3KMGraJv0KVxpt60hMdwAqhRhSxVzxk5K5HfsfQcU0eYhdw8gXGN0qsM8em3+tbaRH7Ks0jOuQCoYE59vve49KzNUQRCVsYKqflIGR/48aLWC421LXCkee0ag5Lgtj9V/zkZpqRTSSgyNKEY4I3Dgex2807RYWurR3EJkUOQzBDxwPQ+/1q9ezXdnpst1b20gMajHyuAe3PNPzF5FFrZ3jRIrmUknPRdqj0A28n3/SmWUepG9ML5lUr1UHb0xnhRWhpxlubYOFZXkOGClhjnHr7Vo2PnpexOxkZAMY2Sfz9afLqF9Dn/OWeNFIcBvvsjc4z2yp9e/5ipP7LS/mMk0jCRgoGJIssRjg/KPTtnJ7VDobMdSCxeYzpGWAVWJ69flOa6PEhcj9+rEt8+ZvT+7/D/T8KXLfcL2MUaRAEk82VWcfNuDW+w5POcgEc49an0/TYDI0awWkrISNxSCYMDnqVZTyePcetainanDyk7RuZ2lHPqM5Ht74/KxYeXPK24gOpUqTIw9RwMYqox1FJ6FK+tGgsxLGgiuYArK6IVEajuQGweFHHvnFaFnB59sl0ZSzuEkYqAxBx0AwvH6+9MnEr2V+WzhQwj2gEkeWO477t3WixiefRrdFkKkeUdx/2HUkdv7pFa8upnzaGbbRlda1C3ErJEoV1iDMigsOSFGQOo6enpVbxIIpNNWRSz7JBjfIJMdRxkdQT1z/KtBIpYvEl0xUJDJAMDOORgdPzpmoaYZNFNtHvU+YWBIHdifUDHJo5dAvqaEYt1uxGEzKQzdwSMgHOeO4rN8PTiDQkdiGYTiMArtxucAcgju2c+1bKxobhd2PN2Hbg5O3IyfXHT9Kr2toujaYzOxMCyfN5eWA3Mq9PqR/hQ1qJPQqaiMXeWUglM8tnue9VPOXGMHnrxVvxEwinh2lCMEfKc9P/ANdYyMcFuM+9ebW0mzpirxLyyptPGfX5sYqG4jOchcg91PSoxLHtBYDj04pxmjKEAt83PWo6Bs9CBd0jFVJAHUmpvLj2bB37k803gqOSAOelKAM7ifpQkNsiiYIx6Ag4yc5pZX3FQpJyec02dFIyoOe+KfpkLTajD1IVtxJ9uf6UktbFJ9ToJ0WGCG3OAUAPLYPA/wDr1CWTu8Kj/afNFzcn7XJ8kRxgZaUD9KZ9rIyAtsD/ANdx/hXQwQjMhOBLbjPrg/1q5pTrLIjrE0eTkcjHX6/0FUTd5OPMtR/22z/StHSkCbdo4x0H3fw9qcFqTPYz9YkH9rTLyQNv/oIP9apSAlOGOO9TapJnVJ+4LAAfQCqO4oxyfl9qzqfEyEiwuxTjAHt2pXwRyADjiqm7J+Vhg0qyEDBOfXmsixfMZGyxC+9TRzhhg8+vFMR0Yj5flHTI6VL5aqQdg/KmkDa6khCsDg4NODiNVyc+tMBw2ATj0oJA4YHnvVWJuTkg89+vTrTQ4cFTkH0qJXQEAMfbqak+X0yf50bjQ/cwGDxS4DrxjdTFJJI/h96MbeVDYpWHcVVK8nrQHKOTmkDh/Y0jAjJDAipfkNPuWBKp64PuKkSby+AxIPrVMg4Gw4qUA4wSM0JsC5HP8wCnjv3q4sqN3znpWMSy/LyM96micoMk5/pVxm0DRrsivywGcdR1FS215dWE6Swt9w9hwfrjGaoQzluC3brmrQlPUD61tpIVzo9P8Uu06i/8hImBIeMNkenGTXQ297bXYJt50kxyQDyPqOtedMiycqArU+2v7vTrgvCRCXG3OMgj8c+lKzQ7npOaXNczY+Koimy/BWQfxoMqfyP8q3ba7hu4fNgkDpnGRxg+4PShSCxY3UZpuaM0wHZopuaWgBc0ZpKKAHZpM0lFADs0maSigBc0UlFAC5opKKAFozSUUALRmkzRQAuaKSlFAAKWgUUALRSFgO/60wzRggbqYElFUmlct94/QGkM74xuNK4F2lrPWRh0Y04zMf4qLgXaKo+Y/wDeNFFwI6KSjNZFjs8UZpuaKYDqKSigB1GaSimIWlzSUUALmikozQA7NFIKM0ALRmkooAdRSUCgBaKTNHNAh1FNzRuFADqM03INLmgB2aM03NLQAvFFNpaAFzxS03NGaAHZozTc0ZoAdmjNMpadwHZozSUUAO7UU2lFMDC8WoH023ygYi4GPlVjna3TcQM1wChI8fKque4Qrjvj5XA7V2PjbV9Ogso7SS6gNyJlYwblLAbW5IPQciuFaa1nk3C4VGRSF2NHgc54ww/X1PrVU+pEy/kttJTI3Eq4ic8j0w+R2rIvtMgaSWT7Gq/KMAxY7fj/ADq/PdWccmI5Idr5A+WLbj/vrj8a5XxO7XCRpYz7SrfOBMg4P+6xGOP1rS9tiLXOi0mGQabE8aq6KCTm13DgnqSwzjpTb/ULHTViluvkQthC1qMjv2PQ5/SqXh66t4LBR5vlTKzkMwhHfg8kEn61HqUWnXxEU8uEU7s/u1LH65Apt6hobEfkyJGyBQsiAhhAgBH3skbsnio7uw+1NGjJMp9Ft4wCDjH8XP4mqtrqVlDEio6sETaA6KO2Oc81R1q6a40wxWNy6TK4bjYntxgDPX+tEmC1Nq30tbXKJbmQEjPnWykg8Dj5sVHeTW9jD9rmAREK5dbbJB6Do30qKw1CP7DAlxLGZERA/mpHjO3nk8nvzz+tQX91Z3kDQyykwsAAAkfrnjbx196aYMltpY7uMSxktbs3Cta/KxHH976j/CrUsBaMKiyRvjIYW+GHXOPnx69qyrK80+0tPITeqqDz5Ck9c8kj61PeX1vJYXEcNyqsVZV2+WnPYjv1H8qcmhJEtppYhvA5EjsAQVe3TB44Jw3r/Wr7oCjN5S5A5ZYGHB4Of3nv/wDrzWDol35dmVuZSJVPzFzG2fl6/NzycjAFaD6jaMvRN+OcJGDn8qL2AtqrAF0h2Kep2vgn2xJ71oaUGa8bKyBtnGN59OeXNZLXtq3Rxl9wyUhzg+px/X1q/oU1hHdMxuLdQsfO14gOxxj8OxqotXJa0NSMpcXl1aEMTAqlyQCG3g9uvbBpJpUGh3E1mN5it3eNlwyllB6A++eM02fUtLhvnmEq+e6KrBm3ZA9Bg89e9Z9jqccVmtpbx3E+2RjmO3PIZj6/XtV3Isaj+SL6CIRgSSRuysq4BA29QT/tCqskd1YWOoScF1eaWHzmIBGMjvjHbt1qJ11G5uUljtUgIRliaRuFU4yNoIPYflU32DU5Yv3syRgH5Xjhzn3y2P5d6OYLE8tuk11b3iSxCSKNhiPOW3bf4lyB92ot9r5V1BfzrgzM6NI4V9mQRhg2QAfb+lMi0MXUAlknuJQQMM0hCnr/AHQAPzq7H4ftbd1ZbSNSSBllGTx2Y5P50hnP67d21/LE9tNJIELZLJxzj+IAZ6d6yXBRSAeD3Brq/EUMUcEYC4k355BB5B9evTtXLhCZCMFmHOa4a6983py0K/llhgHAp4j7cr71Z2YwXwOehpGQyKcIOBxjvWPKU5jVbCgKR7470wsVJJ5HXioJFeJgrRkE9PehZWJ2tkY9sUAl1HtKOxIzWpoAX7RIzHO2M8DsTWPt34zwM9K2NCG83ZUHIj+UDgnrVQWoXFeZXYttAJOSQiH+bUhk5yJlA/3Yh/U1MlpnpHKv/Ah/jUn2N8cK2P8Arof6CtLFXKxnYA4nQe26EVt6W2YlOCOOcnOTxWS1rhSXIx/tXDAfyq1YXEcMUwVogwXOBjJH8z1q4aMmeqMi8dLieWQN8rOSCPrVd/l4AyB0JquJGTCEAYFP84cjg1jJpu5nqiSSQIMbcjHaoGmAyAgAx6U10kDfIN3tUQSVsMUbH0rNmisSpcHcAGIAPPtV6GVgctgL3z1NZIyJsYxg8irIkY/xUIckmahZWG44z14yKDllxkMO9U1diuwkkdAakRtm0FgXFaJmb0JI0KAnFSq4Ix6GoxKmCSCc9qAu1coe/ei3YLkwdWJGSW7EUucY6/WoyVxkgDPpQWIUMpJx780WGTbhvOf/ANdNEqhtpU4yaRZlYZ709iDgEZBpWuO9heCvDfrUfII5OKYyMjZQrt9M9aXLo3Q47g9qgpMk3dj+poEyqxUZ+ppgG4Ekce1NZA3OcGkMtRz7+c4HtUwuiT97p09KzWEigAHpSCQkgMcDrRzMOU347kOMdDmpmm2qQ+CB2rFhlwvDEnHYVOZH4J71rGo7E2NEbH5Tr/dNXNP1S606XMTnaODG5O0/gD+tYkc5XpwR3Jp4uPMJ65NHOmtR6noFn4mtZwBOphfHJzlenrWpbXkF2heCQMAcHHavKvMbbnPFWrW/mgIaOZl/3HI/lRzsD1HNGa4mbxLcyQQpDIyOqAO+QS57mi01y5hglBlZyw+Xdzj8avmC52+aXNcrZa5PG5kuJSYsfccc/wDATjmn3Hi5FOIbYn3kbH6f/XqopvoS5JHT5ozXBT+O7uJ9pitF7gEknHr1qGL4gX8oDQ2UEy9SyqyqB67iefwBNVyMOdHoeaK4+Tx1FDGd9oJJAo+WOXqfbIzj3IH0rFu/iZdQybI9Lj56ZLMSfoMUckg5kelZpM153D471+4TdFoMZGeWmYwge/JJI/CtuLxbItuGurONZe4hmLL+ZUH9KORhzI6rNGa46Xxlch1KWVuIs8lpyW/LaP5mpU8Zbm2mzH1MuAf0NHIw5kdZmjNclL42RMrHYmRwBwJeF+vHFU5PF97JnBhhH+yu79T1/KjkYc6O5zSZrzuTW9Wk3EXUu0fxZ2/ouM1TOqXTnDXtwWP3Vy2T75BJ/Sq9kyedHqNRyXCR8dWHavNvtGs+UfMuHEGM4MrKPUE5JP8AKsS++IFjp0TCfxAHm6iO2BlJ9srgA/UilyD5j143TF8jgelPe7ATpg+vYV84X3xQvjIWsxeOhHBu5yB/3wh/9mrnbrx74hvRtfVZYYzkbYCI8f8AAh8x/Emp5bdSrn03f63pungtfalaW/f99OqH9TWdZeNvDN/O0Ftrdm0q/wALSbM/QtgH8M18vrc/aN0k0pkfOWZzktUti6f2sqKeCDxScQufWsVxFMm+GRJU6bkYMPzFP3V4P4O1a70vXZHjlJtjEQyqQMHI65PNblx8VL+11WVIbe2vLVGCBS21j6kN068dDT9nKzYlNXseuE0Bq4Of4o6Qnh641CNSLqJVItJXCl8sB8rDIPc+uAeK4a5+M1/dKypZpACOMSkAfkMn8xUqL3Kue7ZPvRXz4fieSc/2ahz3Mrc0VXKxXPoLNFJRWJoLmikzRmgBfrS5pM0lMB2aBSCjNAh1LkU3NGaAHUU2lzTAXNApjOBTCxPelcCfNJu9ahBozRcCUv6Um8+tMopASbz60m4+tNBooEOzmikzRmgB2aAfem5ozTAkDU4H3qHNOpgSZpN1MzilzQA7NLmmZxS5oAdmgU3NLQA7NGabmlzTAdRTc0uaAHCgc03NKDTA5Px/pEGo6VFcu2yaBtqNxjB7HJHoMc15a2h7PkDw7ySMsACPUffr2HxiR/wjsgLKoMijJfb69PevOo4yXIM6kYOA0oGR04OOaqmt0ZzZkR6BG5Z2eA8dQwxkc/3uT2ph8MyGTmKAegyOQf8Agf8AWugKlgxEgbG4FvMQkHHQ+ncfnSGKQwgDenIIKtGWOMjPJI/zxWtibmGfDO4ZjiUqBlgpGOvf959OlH/COIhJZU+8SAGA69x82TW2isrhJSXkPygbF3A9uQCPTp6gdSKc8uwndKqB/mypB3e5I659efqaaiDZix+HopUZQQxK7QA3PHXgMfWnjw7bMGeQRHGejPj26kY/OtA8s8fm5J5AboMjrz/TJqV5JHWNVuJS/cRyHGMfyPrjvg9aHFEpmM3h2IMGVrXH97zCRnvzup0fhhXQZaINtwBnHB9PmrTWaQqxW83LgfKLjlR+PJzTQ7qQvnEq+cKZccj/APXTsFzNbwxH5jMJIMYw2TyMfRv61ZGjTsjM80ZEm0ndvIZu3PrjpitJHkMays8/JHy+ZkfyHPfvgYPpU5naLmRsFRjcDz75zyevvQ0FznX8MHBlUwyxSSZ3L5jZx7gYxz6+tJJ4Xk25QYXO5SUfg/lXQjyyH25Y5KnIycdR0z3yOevpxVfYJEMsQIH3nATAHPfH8v8AIFEHIwT4VuInVo3QGNiysoOAfXNX9A0Rv7QZZlTYAZNu0/eyBnkgD86sTysFPCx/PnmMsD+G4VP4cbOoKHVWcodxKFc856ZziqjDUTlodLDo9nb5dAFX/ZUIT+IBH6irUEdtGG3x7s9GzyePXJ/QfypwaNcFFy2cnC44+q89u4qjq2pwaXpcl7d8pEQMDlmJ4AA49f8AIq3oSaM9yIo0ZIQScDJIzyOMYx+uR7VKB5ckjQgK2cFhwD9ex4rCudVj/wCEck1CGQMkcJuFCk8/LkfTtVWHxLLfeDLjVVUR3PkSjCgkK4B5H6Hn1qdmx7o3p5N1vcM7KB82Wz82PrnFPSRJ1jmi2vEyKUZSCCOcY7HtXFWN/d6h8PL4zuTcJDKjtu5YBc/j8pHvVvwPcmXw2kbMrmCVohnnA+9z/wB9YprXQGjQ8RFhY45Cq4IC8Lnp06g81yUsp25U8g112vB5NLuB3CBuuTgMOh7iuJBXdlySD61yYhe8aU9h++SQnPzfQ1Y3COMjLBj1zisu51vTrDzElvoIzGMshcbhxn7vWsO48c6VEco0s+7/AJ5x42/Xdj9KxUWU1c69nSTCufpxVe6aCGJppZ0iReWaRgqj8TwOTXmuoeONTu1eO2VLSNuMp8z4xgjcfzyACOOa5yeea5maaeV5ZW+88jFmPbkmtFTvuKz6Hplx450OGJUVprgN3hiwVPvux+npVAfE1rJLuOw04P5qhUkuJCAB3yq/iPve/tXn1FWoJDSsdXc/EHWp4QkSWluwOS8cRYn2+csMfh2otfiDq8ERSaCxu2LZDzQYI9vkKj9O9cpRTsh3PRrH4l2rMBf6RGnycyQoJCz8fwkrgdT1P49a27b4h6BNYzh7iW1bYVWOSBssccEbdwx9TXj1FFgPY47yK5jL288Uyg43RsGGcdMjv0pEkCuMnn8sV5BFNJBIJIZHjkHRkYgj8RXa6D4rt7hxDqzrFICuyYAhW/3vQ989OvTHOMqbWwXsjsdzIeRg+lSpcsuOcevFPSVWPzHd6HGaa0UUmBkowznBz+lQkRzLqiZmhlXLABqQpHHEx2kHIxms5X8qd1J6HqBU5uCuAkh9QaTKtbYsCRNw3EBscYpQNzgux9MiqUTkL93I71b83IwCQzdAOTQgkSN8nAOc96eHwBjGR271AsjBwMDB9adweQR16iqBeZcBwBgrj2pytjjGfes0yyK4V8Z+tWg7YHT65p3E0yUlVbOefWpEcZ5IPtmqckv3Q/J7mnICxUKc56VN9R20L25D3Bz2oYgJkcCpIdMvZFz5flju0hwAKe1vHbozz3kIQDllyePqcVShJ9BJ2M/7SwOepzUqy+auPun6ZrPv9Y8PWhz/AGtGT3VUJz+K7qpL4t0QBzBdNKyI742MuQqlsDI9v89aXsZdS+ZHQlc8EZ9aZ5D5G1C3tjNcS/xJcnEOmAKOMtKM/wDoNNTx/dT70SDyJCrHeGDcAE+gP45qlRXVhzNHfxWs24K22MHsxGfwHWrsVjK0X7zzFPckgA/jXk8Xj7XA5KX6xAk/diQHHpnGf1qtfeNfEkpIOrzqP9khR+grSNOCJbk2exSWllbwme4uwqd2d1VfzIrCvPGnhvTv3a3K3LjgrbwmTH0ZsD9a8YuL+6u5C91cTTv/AHpHLH9aiEvJ5OKr3Vsgs+rPSdR+JsxhZNOt/KHIDyydR/ujGD+NccPE+uz3Rnk1S8Zj/wBNmH8qxl5OSPpVy2Co4PIobuO1j0fQ/GsEFup1a+SV8fcigbcPYnhfyB+taTfE3SEwLZZI2B4d4yT9e/8AKvI4X2g7iMn1oeVQwAwSfSlGKWwM9Uj8dR38phTUgMqzElCmAASTnaOgBPr6ZqjJ4x0lpSZry+lYfxBMBvxyTj06fQdK86tSwmYfdzG4yTgcqR1psDAkKQf8avmYrI9Mj8W+HFUeWwiIOQ0lsX/nnH1HNTT+L7C7jCpqbO6gnCJIQAOT94cV5mEGOcjJyQRVvRY9x1FugjtSc+nzoP60KTFyo9Di17QIwDPqNw4Iy0aB8fnitOPxx4eQLHDJPEQfvCLoPTPPFeNXspV1VTwRzg0ls5AYtnB96OYOU9qm8a+HTiQam7EHaEWGTJJ7AEYrBk+J9koxHpc7p23TCMkfgDivO4/3t1DGAzjzQzBOuBycfhVAsQ464ocmHKj03/hZlsuSmjSncMEPd8f+g1DN8UpoSFt9JtQo/vuzH8+K86Zzkbc5pjse45Peldj5UddqvxG1++k/czrbQgD93GgOT9WBNUh468S7QBqbAe0SA/ntzXOu+wYYEZGRnv8A5xTBKNgUGk2x2R1Z8e+J4tudRD+zQRt/NasW/wARNaiMhaRwH5wk8q8/Tdj9K49pgQBnmkeZdqjIBHX1NAWRs3Ws6hrrs99dzSqONrMTj255/OqckSpGSAQfWoYr6JBypB+lJPdRSqoUnGfTmgAErfKTxj+tMkC7QV/H3qAyhj1/SpAU9z+BpDDJXgMcVYtJxDdCUVDwwyqMR6hTT1ilVdwglYeyHFFhHbadcotpc3RI8xFBjI7E9KwVvmkfAjKsSSSp5rOS7ktleNlwJFGAef5fX9asRXisURYZGY8YVSSc1pfSxKWpYv53axQeZlXwMdxjPFZqTiPKknp2rq7bwD4n1qGE22mM0DnMVw80QVlPc4b/ABqGL4d6rFK6XstrAAOCZ1XJz7/5+lT8WxWyMD7bjpI+O3B/xorpl8ABlDLqVqVIyCLmHn/x+ijlYuZH02KM03NGa5jYd0ozTd3pQGoAfnNGcU3dRuoAfRmmbqN1AD80Zpm40hb3ouIkLADmmFs0wnmjNADqKbQDSAfRmmilzzTAdRSUUAOopKWgQZoopaAClpKO9NALRmkpaYBmlzSUUALS0xmVELucKoyT6U37RH5ccilmWQgKVUnr046igCbNGcVkQeIrK5vPs0Xms28rnYcfX2H1rWo8xJ3FzS5ptLmgY6im5pc0wHZpc02jNMDD8X4OhgEkAzKCQSOx9K87m/dYbzEGUIY7guOf93jtXoPi+UppCIoYyNKNu1WPQHn5enUc15/JI8chYlkJBIw7KRz645rSluzOZP55ZfnlizzyWjz9MFaQKMgHYFz0KxcE/hUa3chLL55DZ6+ZI3H/AAEYH9KPMaXGbrIyMnfJ/hg1qQh1wwY7lb5UUKMOOncccdTj6fkfOvEus3dxdSRpK6QjgKpxke/+FegSH5AGeIgf3WcZ55HtXnGtQ4uJOAMGom9Bx3KOl67d6fdRFppHgBG6NmJGPb0P0r06O5jnQtu3ggEyM/UfyPUenU14+4+avTfD0rrpdkcFyIk2k5wOPQf54pwd0VNGndysI8uyyBvmPORnvj0Gc1WhlDEAgqfpwKfeOwGzLOpHQ9/qCetZsYffuYEgcg8DB/OmmRY6UyoZHdigwpHYK2OvT3YfgTx6+deJ/EeoG/eCCd4YxxlDtZueuetdSt1KU2rMdo+bG8jn+XYVwfiTnVd2MZX1z3pTvbQcFqa/hbxJd/bVsrtzMk3Cu/LA+hPcf59a7hnYIUBU5JLEqD+X4GvK9DXOtWfOP3q8+nP0P8q9OUFF3NKeF5JJOf04q6crrUmpGzIJFQkgk84yAcHBrQ0Bo/7QiQnnBGTz27kj+tUbuc79nm7hjOAxbH51f8NhUvINmMlm6bhj5T3A29q27GZ17Dox2A8dcD8sk/oa5D4k86BajdjdfoeM/wDPKT1rrVVWLODhvcY/UZH54rkPiEmdHtVxgfbEPznGPkf8KU17oQeoy1dn+GNxGTuARwe/8RP4VFoIA8Baum3CqsxGccnyx/8AW60lsQPhxNtJGVfjqB8zf5/KpfDkb3HgjVYo8mR1nVV6ZJiGOvHWo6v0L6fMj0Bl/wCES1uOMKsZilOF5H+rq94CydMlIYkrct2HHyrUHhqyuJND1O0f5PO3xhnYHG5QvUH/APXVmxbT/BmjSXWq3awRNMMllydxx8u0ZLcDPGe57GiOlmEtbo2vEF3FY+Hb+6uJYo4o4XJdgMBscYyMZzgAdzx3xXz94h8aahrTGGEi1sgNvlxIqNIMEEuVA6g8qPl6cE8l/jfxnc+LtWaRfNh0+PCwW7OcHGcOwzjedx6dBgZOMnlqym7suEbIKKKKksKKKKACiiigAooooAKKKKACiiigD0DwZq32ix+xu37+34Uk9Yz079jx0xjbXSO0hZScfLxkDOK8ft7iW1uEngcpKhyrDtXUWHji4iwl5brKPlHmRnawHckdCfbispQd7oXmdvhJ5CVYhj1yODUjBI2wcdPSsOz8RaRcSJsvFQldzLL+72+2Txn6Gtie8tIJVEk0YdsBVZvmYn261HKxNk6yq8fysAR2xSZYtlWII56dPWua1HxKsDBYYw7A5zuwD/jWFqWr6oLua2lnkjdJGRkQ4wQcEcdelNQY0elW9vc3IYxwMUHV3IVVPuxwB+dXUg0m1QvqOuWMAHVYnDt9OSM/8BzXlbWOsXEFpGtneT/usoiRM2BubsBxnFRNoetMQDpt0Pmx80ZUD8T0rRQSA9C1Txr4Ts2VbC2u7+aPpISUVvqTj9F/GuV1H4iavckraCKwQ/8APuuD+J71j3Hh/UYceakCDv8A6TE2PqAxIqynhGaREc6vpeGA2jznJ5z1AXjp39vUVdn0QaEtr4o1fySzz+bg/ekUMf16/jVyTxRrHnhor+e3GBlLZ/JU9OyYHetCLwLLp+gtqs93ZXlojDzoklK9WCqFP3ufoMDkZpYLPSRNuOlwPFtyVLOTwOxLH+VNQZLkjK1LVtUbSrS7l1G8l+0zTL887sAFEfqf9o1ixpeXW54oZJWbjKIST+Vd/Feb41iW3s1SJCsSJbphM4yemc/KOfakn1W/Wzn/ANKkULGxXZhcccngVXKJSONuvCmt/wBoXNvbaZdyrFKyLJ5ZwygnDfQjvVnTfCmrQ3bNfWEkEJglQvKQoUtGyqSSegYireoazqP9oTobiZIjI37tXYDBJ7Z54qhqF3PKuHkZlAyF/wABRyIfMyx/wgepxYMl1YohYgkynjHrx71Pb+E/scjTXN/ZSo0brtiYu3zIRkD1BPHuKc13FDbqIfusuY8DBKdvXHH1qs+oSyKvzZG3AJ9etPlRPNIqx6DYO4J1CUhWG5RBhsd++P8APStAaN4fYASTaiG2DBDJjJJGeR+mazRdfP8AKmwFQpORneO/06/nUvmyRwskqggEBT9PT86VkNORZGm+H7W0MjwahcSM/wAis6BCB6svIP4VV/4lcQyNFU8Y+e7c8888Y6VYtRC1v+/uBGznOGcDPpwaqHy4y4aVGAOAc8e3JqSgjvRb/c0uwIByFeIOfzbJNdNpmt6I+mmKTw3pxuG+UyrGimLcfvYKlmA5OFI449CObt4zdyARK0uOdqLk4/CtA+H7xdNvdVns5LWGIBYhJGRvJBLFeOcKGzxxnPHUCV2DLV1rkBvropo+jsHmY5lskc46Yzj8elV1v44YGul0vS2cEJuayTA6+mBn8O1ZkRimBZGyDyfUZ9asTs6+GnAJ4u0GAOPuN/8AWrVpGeppweIUifJ0rT4Gxw8MCRnpnBwp7454/qH/ANv6aApHh3TmfIJZyT+WAP8APrXHiYZBPOKeJDIzAZwalFWOxg1Twxc3DW8/h6OIEFg0U78BQT3Yfz/Oq8U+miSR7bTFSJlKgNJuJBIOGyMEDHT9TisHTNi3c2cki1nOSM8+U9P06eRrbEh+bOQCuP09M1ULX1Jnc1o00VkG/RYnbJJPmbQec/3fSmS2vh1zIP7ImXJyrC9bjOeMbcen5VGrOBIGO7nK80zfmL7pBJ6GrsuxF5E9laaVYxm8FszypKECNIc7W35OcjoB19SKs21xp0ERVNGtWAOcuu/8PmBP61QiQG2mkIXcJI1GT0GJP8/hUIceWdz7c8AkZxTjCIOTL8Gs6fYKd2i6bMT1MturHGegGMDrwcZ9zipo/FUExIttC0qDHdbXDD/vkgH8q5+eH96yu2eN3A7fjViOJLe2KoVz1I7jtinyK4czsJqEd9eXLX628LRSAjGzhSijOB2GCPzqi80btGvkxDbwD82Ov1raOr3NppwgjjhEEwkjywwc4Xd+m38q51sc+lZSWppHYkecyztK6RoNxO2Ncfz7U0JHK2WfKnsODUeR+FQnIyy4xnHWoZSHSSxZwqDI7nmpbW7ijf8Aep8noADVPAJOTjvk96cZYjHGoiCsudzbs7uako3xfWBKmK1hHHJOcn8alg1aCO2YTQGZ8/KwYKAPpjmueRkI5z+HU+1KXzkhsD+7T5hcp0lnqMM8qq6onpjNdE82lQRJ5pcTlWPkhSxbAB69gcg5rz60YGUDd8xPGSAPzNdzcTWC6As80aRXLSBbaQAnbk5I45xtzxzyKcJESjqOlezDjNmyucMo8zII9+AakgubSRkja0TzC2AQeg7dQaxeUumhSRnjDb1xnrxzg9KuxeUwaTco2AqOvpx+tWmJrU9e0nxBFb+CJLhTmSxtiCjqAeB8nA7EYrxu41jULxX+0X1wyO24xmVtmevC5wPwrWm1u3j8GX0QTNzJGIzIy4yPMXjrxxg/n71xTXLFjhs47jpURSitC9Wjc/t/V1+UazegDj/j4f8AxornfMH9wfkaKd0HKfXJbFNzTc0ZrhOgfmjNMpaAH5ozTc0ZoAdmgnimZxSZoAcWz0pKKKYC0CkpaQC0UlOFAg6UopMUtMBRSimkhQSxAA6k04fSgQufelFFI27Y2zG/HGemfemAjSIm0MyqWOFycZPoKdXI69BrN1ewRMYIYQrEPHKdpPYEcE4xmtDS9WYTixnczzLgtIsb7QD0Oec8+4pxV0Rz2dmjepaqXV6ts1urBszSBAAOcdz9P8agutcsLXaHuIwHfYGyCB6k8jApFXRpUuK5ey8VQwWO6/Zmm8tpuFx8gLc46gDb79RW3pmqW2q2MV1A+VkBOM+nB/ImnYFJMbrcs0GlSSwlgyEEhe46ehx169sVn3PiWEac13GrAIcN82DvBGVAI54OSfSuO1nxpdia705Ps89uHeMs8ZIYDoTnoc849R6Vx93f3N3bhGld4923buyBx1A/CpV2roxnVs7I9VXxvYC9nV5wYSieSFXB3H73Pt/Q+2eVi8WJawTRRLdbmwquXC9CCOAMY46DGfWuXSKPYIxOSSN27HA49Tjrn3qI299Axm81I2TLB425yMdD6+lW42dzn9u27XOoXXmudcvL4xykTLtYCUoRwu07h6Hse3pmt/QvErX2rwLJ9phaVgiW8bKyttGCT3UDGcZ9a8+sdREUbGVGZ5MqoDMo249VI44/x44qaPVLrSpTNaQj7TKMrJkkqBySf89iCaFZWLVR31Pd9wzjPPXFV4b6G4maFH/eKMsp4I5x0/CvP9L8T/YtKMhuori9u1BQqTJJngAY3N79cYOPlxW9Y6g1l4j1GK+uj5YiiZWnwm35ckDBweT1wP05Op08yOrorNg1m1uAzQv5iBtqsvR/p61Ol6jHaSu7HQNQVuW2dUUsxCqBkknAFYFz4w0+2vDblXcqCWK8/wCf/r1k+J/FkEEEtnAWeUf6zaOmO2fWvNZ9U8/LNEdwYng/e59KSu35GVSpy6I7nWfEMetQ4U/LGwIUDOPrisdpDLv2lsgY6ue+ePQVhWc5WX5W2KQeDzzx9DVuS+NpKWkk+U5B3McfXit4R5URGbmtTQmjxyTjOT94nHHr3pqsPdj6kn86hh1SG5QLE8ciDGQwl59cckD/AD1xUrS4jAkMbAIACplABz6c5/z6Vd2XYq6hey21lPNGQzopIDNgfj61yF0j6lcIJJIIHkGXaRiqKevJ5Nb2vyKfscjywQRJMxkJLZOBxlccD/H2NaPhDRUTS5dSjFjdTTTuUM5IZACNhAIJ4IYnkZz14FXWVKGH51dz00/r7zSEVu3Y8suIzFO8ZIJViCRyDg16N4fmP9gWpUHIQDgv2z6cdqZ4i8I2i6q95bNB5QCtJbuzD5sfMTtHcgngjr+FdDosVudMTbBHBHtICQ8AEMRyx5PTvt9mzWNFSmr2sTUkkYC3kN0zFJ18ztGmHkJ+h6/hmmmG5B81JY448/K5QZLDqD8wx1rXumLazPHI2IBAwVDH07/xEgjPru+tU7qRYraBDLlGJIDOExnHTg+3pWvJZXZl7RFUfupQjE5wDllPI4IOO3WuQ8Sqw1BCx5wecYzzXT3Vyy3jZlcgYA3ZHGBjgdPwqZJnl/d+YxyQcZc/yPSplG6Li+pxOisV1izIHPmr/OvTUYeWcEk46BiO361TSAKV2oWO3klWHOfc9fpU65BJaTAJ5whx+WaunGyJm02DqGXai8jHY/jxjmrnhvcb2IbWI3NyqN/dNU28sx8uucfLldxPPbmp9HYjV4wcqQ/Ycn9a1SM29DtU+8TuO38v/rVkeINMGsWSwRF2JlV8KR9OAcD+LtWqpwhJVhjnPUfTP9DVS/mlFs8lqAX8pioOOuODjn/CnJaERepTtNOjsfD40+WPzlJYNkHJySe44/EVZ022gt7JYodmyQkkLyMjg9OM8YOPSm6VLLd6VYySRSyStboZAigEMQPbA+hxU+laWLHT1tZZkKxb2DHqdzM2OvGM9Qe1RbUu+hi6z4ptPD2jXGp3MMzqkzW0MI48yRWYBcgYC/KTz0HqcCvBNc8Qal4hvWudQuGf5i0cIJ8uEHHCLngcD3OOcmtr4g+IxrviGSC2kD6fZO6W54O8k5d9wJyCw4I4wBxnOeSrCUr6G8Y21CiiipKCiiigAooooAKKKKACiiigAooooAKKKKACiiigAp8crxMGRsEc0yigDqtCj029hM179olki6wqwVScjHPJx19P8eq/4SWSK6le1sraB3kaSSVYgXdmJ3Els9Tk/jXJ+EbSeUXMi7EgbCtI7dxzjHU9e1dN/ZCrcSBb6GRsNJiFC+AF3EclfQ+vStI2tcylvYz9T1q8ufMV7mU7zh/mPI/lWNc6ndXZ3XF1LK3TdI5Y/ma6e407Sfs7vIbksTglZFGCATgDafTvmqLW2nRWMTjTkbzQSJHkcvwxHrt/h/u03cFYwHuPlAZzj0qP7UAFCuwI6811Uc8EaZOmaaVHQvbKzfj6/lUNzrFmn7s6Xpg24GRZoPx6VNyjWtvtF14HlgikLQLdxv8AOQoGByASeetR2VrcyHc0S+UVIybmIHO09fn45/TNV7jXmm8OPap9mjj3qBHCgXHOT0HsKqeYzaRDIGOWmcYz6Bcfzq0yLGmYbiCXzM2yLjkm6jIP5Mae6TNNJb3d1bW8XlvG+wu5+ZCOy4OM9M1lTQiaxDZO0DORTNSuX/tW9BBA+0Pwe3zHimxLUj1SGaXUCYGEivIwRtu0EckZyc5657cdTTJbRjDEZJlLM4UhVPC/3hnr7Dv7VM7SvCJFG5VyWJ6gHj+uKdcOWtQ2QQCCDUpXG3Ys6XZWEsFompXc8U8bBFSCJWHYZJZx2Hp6/jVlOlRTqsIu3VSSS8iAtzx8oQ44zxk8nrxzWgmMl9AGYKPNTJ7Abhk/lVVWYyxrwWDAEetFhmtG2kW0iu+kyTkDcBcXTEZ4/uBPTpW3B4l0+bSWtk0TTreQho2kigGWRkK4yctnkHO7qBXPzsBHKR8zYO3iqtrBILQ3DZCF9gGD1AB/rRKKWoRbYsaWtvITBApAY439R17jFbmjeKdR0JJVtJU2yEAh1yvGfy69vQViwWlxOG8uCWUB9uY0Lc+nHerz6D4geEtFoWpFMZyLR+fpxUXKsbL/ABH8RDJF2oU8cJ29OtEfjnUp4hDPcRywnKtE0CbdpGCBxxwSPxrmT4d8QJ8jaNqIfptNpJn+VPtvDmvidPP0bVIw2cFrOTt9RRcLG1NcWDSS/wDErslidsjCmMk45+6R78VZH9k/ZhbjSfk3+Yym5fG7AGd3UdxjpVFoJUleO4jkicfK0cibWB9x2NTBlIwGGfatjIi1GDw1CIo20+ZDKMkxysSvzEcZPP406fQ9Dgn2j7dtCqd3nxjqAcYKVnanCHltpHCkiA9R1PmPUmuXWdZu1jLmPzjtL/eIHAye5wPzqepZoxaJpcWXhu3WNkeNvOt/MOGBBOVZex9O1UBoUdu8ax6ujA9C1swAHbvnr7VWbUJDbsu/KEYAJ6VCl8yrk/MRgCmrE6muvh64OQms6bvdTsV2YDjv0z3FQJod4VZ0nsZsHpHMyZ/F1UfrS2DteNJNI4VYBnaq5LZOP8/hV9kjBwjcY78UJgyvZ6O9zZy2ck9vbvJKj7vNWTIAbj5c/wB6pB4YSO6AGp2OQc5ZnGcc/wBzFDP5f+rAJz1PeqkuqSwTEOm7cOhOOe38qd7IVi8nhJZJfMOu2G+Viix5bbkdADj9MelS3Hge+t9jW7wXDgcKs6jjHJ+bAxWGNTkmvY5IwEMZ+VlJO0+oPrVuXxJqExlDX1w4YANulY7vY880Xdx2F1fQNVGj2MEdhcSzRTzM6wr5pwwTbkpn+6a52bRtXQEtpF+ijjLWsgx+lbx1a4jgXbcSpwfl3HHXOcZwD706LxBebWSW9mYEdCxwB+dS1cadjlPIuFIDQyBm6Lt5/LrWzp3hXUbuCWT+zLuVWjPk7Ux82e+ccdeldHZeIL+8iud97cjbFuBSQjB3oO3sx46c1DPrl5BZzO1/MRKdh3kEH25/GkoXHzHISeE/EEblX0m7Bz1WFmB+hANV30DWFwG0u9/78N/hXRjxDfopAdAoJALQoxx9Sp/nTx4p1FXVgYCACBmCMZ/8d60nBD5zkJrS5tGAuIZIiRnBUjFM2sygqp54z1zXdy+KtTuYvKldCvUHByPXvWcb53idZYbdueWaEMfrnFHs0HOc7bWszzxpsZS7YBfgfmeK9K1DR7i68MQQ2cCXEkY3EKjEtnJYADPXCgVxyIEkFym1JEPDxoF5/wCAjH6Vu6Zrd5ev5FzeSOWz8zEbu3Q9qIQtK5MprcpjT9Td5FXTr9DLx5TW8gbI7dKnt9G8Qy7hHpdysbnLmYeUBjnnfj1FdFp+oixikAuZHCfKqzHcFzjoDxn6jjJx1q5LrYjRQJzEeeVVeT65x/KtY0Xtch1l2MqDwNrF/pt1DJDDEzRExF50HzgggYDZwRkZx/jXnGyY52wS/KMt8h4r0y98WXVpp0sG5tszbDgKDzznOD6dqzU8W3pYlribccDaHIx27VM4KOhcJtq9jg98mP8AUy/980V2knjjX1ldVZSoYgZVun50VnyruXdn0TmjNIDRmuE6BaWm5pN1MB+cUhamE0dqAHDmlJptAoAdS03NLmgBaB0pKUUCHZqlqOr2mlLGbpmXzCQu1Cc4FTXd5BY25nuZBHECAWPbNct4l1zR73SQouFaTcdhKD5TjHO4EjIPGB+Iob7EykkbWuaxHY6HLcpMI5GPloWXOGPbH0zXOad47EGnyNer5ku4LEF6Ee59Bg+prnZNXvGsP7Psw6RuhM0SJjKjvuxyD68enpWO0WoSBYXilgjVTIquPY/NjH15/Cq2uc06km7o6u58fXU4kQRxorHcgB+ZQD0PY8f5Fb+h+KZr3UPLvERQ0QKiJ9wX5iMuRwCeP09TXmQ025WGE7GKSHjJ+8CPT8vfnvWtDeQ2sMYOnot3CwYTgsCMDAHXGOPT14BOatWM1UnF3kz0LVtc1G1ubhbO2SaO327/AJWPBwM56ZBPT/A1UufEkpsp5GaKNpbVHjXcwwWzz2wPc4/SuJXX5rR3kid1WUhmSKUqBzz1zyR6569Koa9q7axPaRXO2MQIqRtbRYwD0GCc4+vT9ahxbVi/bp3ZLP4k1C5t47dpWdE3KCeF+lR/2ncRSxOvmxzqoYFG98DA5rn57e8tM7ydgfZsyecjPUdsVbIUxpMjyCRAQySfMGPcjjjoOP5VotDB663N+58U6ldTs1zcNjC42OQAwGFOOmeT6c1BJq0szSXZmO3JBDMxJzwTxzjBxz29awUuZYkO3ftVTjn8f5gflUkdxPFEjTNJEpBA2naPQ/gf5/jTS7Cak92bUl+LuzCA42LtAXoOpJP6fnTrTWLqzikslYpE8PlJtYgqM5OOR3Bz61kxRsFTzAEBw4YSA8AZPA5z09MfjxcuIbSSIBHb7WANqhFxg4x16f8A18cVTjclVOXRMynLSXDLJIGy2M5wcn2x1Oc1ZtowRGdjBh8rmTjJ7/hUb20VpM7NJuMjt8h6cf8A6/T1qa4uDDEiOwyFAyeQw9s//rqYx5Spz50lEfP50rO0bbFPQY4GOxP69/wpd8kUESyeYjNkkuOcjBH9earx6nMuGtgUUgq5BweoOQAcAcDjpVmNpmRUlUuqDOSNqkZ65BB9fwJp2IcWtwa1SRo32ZKk4KNuxznOPxoRQyhQQVHG48E5ODyR/nNIkKPbLIzjK/deSQDpkdh146e1VJLhorgbvpy24c9z1z17ik9HYai3qP8At8lrcyyLO58xzvjdjiQnPzHpn/6/1ps1++oTtI7uWJ3ORls9gOOgqK5mj2PH5a5cZAjHHXrz9D0qDTrC81CURWVq8sh5wP4fqc4H4mptdnTFXWp2WjeIdPs4ox5LNdRgsSCoHA6DJ6Y59c1rr4tEnlLFC6sxOSe+cHI7k59R/jWDaeDIrKPdrGprnGfItySdv+02OB26fjWut1HaRJa6XZpbxNwHR2LkeoY5z07Bh7itVBy3KScVZMyF0K7lneW6kS3jL8yXBIyP55+uKkOmaLblkQzX9xGAW3MEVOOpAGQPc8e9abtM0k8rs7LyhcsOhHOCTx7/AD/8B6VWeCNV+UY2vkZQAqP73QYPvtU8/erRU0kGidzJlu3g3FDHAu3BWMBN3GeCMZP0Y59KxHLmYu4ch8nLMScH8FJH51tapJamZzKVdz9cn+vb1NYlyXm4jiwMdB0/IcVDRrFk0DLG7NHtwB8xyR/TPcdauRXoknZY0G9VByrZOOB0xz26Virp1xcSKoQ7icKqjkmuz0bz9FtQiaJIofHmTfa4Szn1KHoOemTj68lx1BjtLnjN0IWOpfbE+cAxyptzxnsQOcZJHX3qS7020s9TsXjs4laZmLrGpIOD3OcHoeMYOON1XdWRW8QaSWVzuSYzOCcH5OuMH/a9OMcng1DqcTXGtaXI4IbnacrtAPqGU5/Aj6VqopWfmZObeiLWpZGj3DbjG3lkxKqhRGNvIHp+lM8PuzaNa+Uqbju6/Mc7j05PPXPfnpWtdwJd2s8TIw8xCp+QcAjH4j2J+jCsoXJ8O6LgI0JUhRPsGELHIJAxkAk9jnH3q12bbMr6FG7gY6yZwpAZAgZgoLHjnA5OO+Afes3VdMj1GON5bmXzlJO5WznnnnByc9fT+WdN4zu76Bo5JFEikq0iEg/w4Hf+nf14qS61cNaxxpKylCfmOMN6ZwO3OPrWbnFqxHOr6lW7laC42TD5lPMnUY9Pr3pthfbrmRZt4hU5U7Rzz3JP+fwqteSi7mZiu0kjkdCcdaqOnDRFpCp67R+XSsObXQuPL0Oq0/UYr6VYo9qkIxbO0YI7DJ57/mK0HDsOG6HOflz+hNcdbG4gkw58s/w7hgKP/r/0roVtEhhllfU4GcL8qtuOWz0+76Z/StYS01Hzbpjry/jsoFcbWZpNjKOoP07npV/R5xcanBJEM+XMFkCjKgAiuVW1TUrjcLpY3AH+s4BPsfWotOnjm1SC3jZk3zKnXkKW6e31pqo+Yhvqj3NUJG7iMdCznJ/L0+tQl7Zg8kHmXcwGBtHyde3Rf5059KjQNLdzvPj5grHgZGBjsCOvAqxc39paLDHLNDamUqscYPztk9B3/M4rZsZUig1KQMZmFuCTx/EBx6+v0rH8Z3ieHfCWo39uXku44VVHPQM7BQ3Oc43A4I5xiuj+028km5P3obG3LcN+A/xri/iremHwDepOFhF3JFFCgXBdg6vg45+6pOT6fSlLRDjq0fPdFFFcZ2BRViwsLrU76GysoWmuZm2pGvUn+g7kngDmi/sLrTL6ayvYWhuIW2vG3UH+oPUEcEHIoAr0UUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFS28JuLhIgcbjgn0Hc0AdnpTraaLZAsoyhZscclifxOCta1pHc2srSzQlFaFwvnYjDbkKjBfAPWqf9pyLHDDARDGY/LKRjaCuOhA47VTkQC+lWPEahFYBTxyoP9a1sZXLbRXU5kRjCqkg7mkzg/wDAckn86vHT8afBEmo2SFD8zMJSvLHoRH6EfjmshJl8hgH+YMDTrmSa6sRDalpJDjCx8sefQU5Kwo6miNHgJ2z67aBOuIo5GJ/AqP51Rn0m1OCNTikPfbAe3uTWM1rerJmUOCBtwRz9MVWaK5IJVXOPaoui7M6awsbaGCQ3NxHcwFwBBtZecEg7gwxwD0qZNSs7eIwrptuIkJZQS7kk4z95zg8Dp6VkeHrObVLe5gSaOJo2WT99KEXaAwOMnBOSvHXBOO9aB0K+WT53syFbkLfQH/2ei/YVjbvI9OFxLHHZ2/ko5UBXk2sM+z4I4qnePZSSzzzafayzzszuxkmHzE5J4k/zn8BNdaTdiWVxLaFPMJX/AE2Js5Jxzu9u/wDWqg0y/e6WKSOLG7aQk6OevZQxP6VWnUVmWYrq0sbeeO3srUxuSpdw0hK5zjDMRwQCOOwqhfyWlzGxFlbRnAAaNShUjvwRk/XPWmGOUuYRE7Ny2EjLcZx26c0xLK6uykcFpdS70Yr5cLEsQDgDjnkfoarSwtR6uiiU7IcSMXOYUbHXhcqcDk8D29BjO8m3VBKHk3KclC2QfqDnNW5LO8ijBmgniDDdiWB1GCeDyO5qGXS78SOiafduykZ227nr06DnqKTsNXLlskESWtxCHE2xixcggEll4GOhUDg56k962zqt9dW6wz3LTRpyqSgOqn2B6daxrXTL+YQIloweZflBdVz1POW+Xj1x0HrXRWnhPXCpZ7ERqT957iIAf+PfypxcepLUjFl8V+IbaQwW2sXkMKYRI1kwqKOAAO1VpvEWvSnD67f7sZ5unGf1q9c+Ddaa7nhgFheiLkyW19FsGf8AfZT7dKZ/wgnimRCE05MDksbuH+e+odrlq5jSa3rLEg6xfttOAftTnn861PCPibVbbxTZb7u4uEllWGSOWUvlWOOMk4IzkH1pf+EA8UFt39nIxPf7ZCf/AGerem+A/EFrqlpeSwwxJDIsrILqMuQCCQAG/wD1ZqWrrQZf1vVIJvFV59osYHjjuXjlCjY5CsR94EEnjvUUc+hzOAtrfQqc4KzBz+TZ/nWFBJJcM5OUnO6QllKkqRuJGeeRz+NWYhsUHIGfetW9TOxqXOm6TdxwB725Vo1KDMIPylywz83vTb7wu2savdTWN/ZRpK+Y47l2RsnORnbjqCBk9xSQkBxuP/16c0vLenQqRQ0Fzn9V8NatpEHn3MEJtsf62G5jdW5I4G7J+6e3GOazorO+nXEFheSYwT5cDtwyqV6A9Qyn8RXQ6tdx7IlC4ZoIs7T1JQEnj61l282VaJ0DqDjawBFTYu5atLDUbLSpbqWzkjtpgIzLMCgz1GM/eyR2oWdisZjO/wCTccHOc9KfJeWvkqJtOtWwON0KnHPTp0qpHc2ttMbqK0hE0YyqNGpjPGMFTwRQronRkx1CKOPa8saNjBDHHPpVWcrcRhxJGQM7ih3ValiH9ptKtvaQyGQuU+zROiH+6FKlcDGOmO9dFPrNwJry2ksbCFTGtvLDFaqER1G0lffg8nOM4GKbvsNJI5O3spxpl3cRlVjTazZGSckAD2Pze3GarxQeTIfNdfXA5z6V11rfpZ2D2502wniKgOJ4Sxcqcgk59RzjHGRxk1F9t8PzRN9t8NQefghWtLmWFVwPl+Xcc88Hn3wTU2aHoznrtCrRbirMyD5Vzx7fWqbhuo5BroL260a6vhLHpU0ECr8sUF2wJY4yWeRZMgc4AA6+3OjaWfhB4D5w8QruICAGAkckMScAEcdsU7iSOf0p1AuFdiB5eVHZjuXr+Gah1Ur9mRAynL/Lk46V3Gl6H4Sm0yfzNXuILychV86Bv3AB5+VCQ2QMctjkcA1ch+HHhu73PP4rDKU+QRiOMg++Wb16cfWjmVg5Xc8u5YDOA2OV9KcJFBZNo2DqSCea9Fj+FlmInKeLLd1U9fs3HTg8Ocinx/DSwGA/iu2U9Rm3OD+BajmVg5WeaiWQnKthc9kNNkmcMAsrcnuo/KvT5fh1paxFZfF9mjbh8/2cAAe4MlQD4Z6JMGB8dad9BAue5z/raXMPlPLpL2cFlKowPTg8frTjDcSqt0FGI8A7ONpr0SX4Y6AsoH/CdWZzjOyBeO3XzTW9pXw30W4Rbey8WRyXRG7alqHYr67Vfp05JI57VPNrqNrseWQX13HHIQ+VIwytzkfjT/t17t2NHJnhseiMAQf1/WvWn+EUEsW/+31Xeej2JjLDv96TNUZ/hrpVtL+/8UFIyoUs8aOMAYAPz9sYHXpitFfoZtLqeZ3cjuiEvkKRzkZBIzVJJ1Zw+ecdq6vxP4O/sPw9BdDWor+XzEjeKOMjy1wxySWPTG3gAYxXIgMqhW3ZA4zRO+7KiklodRFbLJCjkJllBOW/+vRU1nf262UAae2BEaggquRxRXlupK5ofQQoyKYWoyTVmwpalpBSjk9KYBUL3aR3HklXLBd5bb8oGccn19qjl1O2iaQB1cxjMgVl/dj1OSMCsPWl0+8vIZtR+zpBFiWGQyA+YoAJP93GTjHfBoS11MpzsvdLdj4otLu0ecwXMWzd8hhYk4zk5AxjIIz0yMZrdryyXxtaafcGytbXz4IW8wRGUlUUtyOQR+XIPfqK2IfGFkY5G0+3uYm5llWKAyx5xuI46EgE8beRk1rKnpdIwpYqMvdbuztnu7dLhbdp4lmYZWMuNxHrjrU1cRoIu7jUje3E73VuzhbcTw7ztYZDqwGEJXbnpnuOAK7eolHlsb06nOrgKjN1Et0tuxxIwyvv7CmX1s13Yz26zvA0iFRJH95c9xXiPia+1bSLuTR9Ula7ljZGt5t4AWMZ7Dnn0J45+tQn71hzk4q6R7D4l0y31HSWNxIUFuTKp3hRkA4ycGvE73Vx9o5TEMTKJAsn+sAP1zzz9M/SrFv8Q9W8qe1nkhltCAvkumV24xt/3a5c3TyAgAndkA5+Y5qlFqTfQwm1PodjZ6/HLGkVi0G8IsEUcsRd3xwRnB6/07V2EO2C0Tfpq7CywpI8Z3SA85VnbhcFedvJYDFcX4BvdH0e4urzVIWa9jwLaRZGBHBBHynHfrWn4k8XQ6klpHZNPHaBmYrOQSrkjDA5JwMZHfkjpjGko6pLqLmjCLbdyxqBSDdJLBcRy2+N25sEOQSVHAIXuPxx1Nc9eXN5dNNdSRja2SFUBeCSQT61Hc65PLbhp/NeU7Y2kY/LgdB+IHSo7e/ujtWK4kihEv8AGW2bvU9ugo5JXsYe2hJc2yK8MkkxKnOV6jk5/Kn216ZJDb7iEbJVnYAAe5OcD/Cr1uI0D3ceJoV6hoii+nGOcj8M+9RxWzXcqTNMPNmB4DYzzwMfhn06YrX2bWpzPExeliS4lQJGJLRZQrp5jo5I2jrgg+mOelQyRnzysVzHIdm8gkA4OR0/AfmKnjKiJJpmZgj+WWBAHqM55PQ1Z1KW2eRUtn+ZeSITLtxyMkOxx+GRz+dculzFVtbIybiG7jjMhiYleflHXvkY6/Wr6ShoYo3lWWQ5B+VcAkDr3z15J7/jVbUZGEUYRs/KA52gkHtgkfUcYqgm9YiBI+c525IDde2cenaoTSuje7qRRrW29RJ5koibb05+YgDhSfrnj1FEepGWQISAu4FlVcZH4VQivJcl/l+U9CCQ3tSoVcHahjU/8tDnaM+tF+xm4PXmNGciWAgyBSCMqXJ4yew9j+XaofMeN5CzjkYVs8H6Z6mq6SNCCr7juwGGeAOvFSPJE3lururxnKvsHBHrjr260/MI3S5XsQ3ThvMcIUkAVmIHABGeew/yKdbzSTygSiPZ0IbC447f4VYUQxlCjE/KRljnt04pC67gFgOQMLu5A/Lngf55oNlVVrWLUZAaYRbpN5ZQAdoIPTp0A68fpiq6wPqE6WlnDvcvt2RgFhgdfYc9enNa2j+GZb2RpJfMitsj5ipYvxnGD0we5NdXbG20VRbachDEktmMjOD65y2Ce5wPUU1C+5tTg92Zdj4MtrURT6vKpcL8lvGpOT6cDJOewrQl1GI2/wBm0uOKK3IwvlKMHHXp1/Dp3YVGUe7vJZrh2m5VghYFQpxgHjBHHUjHopPNTPDEkrb9zSEqOo4bAxnOSSOwI+ir1raMDa6jsQvGssQQj75YhcgkkHGRgHJ9wG/3h1pT5C/vt67HG3AP3sDpxnd36lsegqW4uEEuGbCN1z3I9T3xnvuP0qq2yRmbcyMD8yNkFug5zk49Bk49BT0QasguZAfmUviQ4GxhwMZ6hv0z/wABqFor24XaY2jT+E9P8P0x+NaMenxk5UFmz/y04bP17/0q4scavkct7t75zkDk+5pORVjnofD6PJym4k9DgE1ZXRY4zuEaEZOBnJ/Q1utOmAAkJ4wQyDgY+lNd1k+ZkXcOuR0/H+lQMxpdNuY9v9nxW5uCflM5woHrxzT7cajFuttXW0LeUXBtw+RhlAzkY7+uf1xFcrqE9xK8GsPbRZ/dp5COFGOc7jkn34rQSK5ks7QXDB51V1MhA5+ePOVy2DwBw3156VBXYpOyJb+2km8Q2k4Unyo2G4dBnOO38j+B5zO1uZ7y3uImVkAOMN25xgjt9PzPSpbYLJcecpBBUg9wRkZ4/Af56cbreqXemw2kFjI8U8FstvNDn725R8y9iRtPatXaKMHI6+0ad7y9V2ZgJwIgMcL5adAOnzFvQ1W1S509WjsL0RJJPyizj5dw788Z5rlv+FiOkVq7W43Fts0SMVjZOOc/ez+PHPWrurFPFHhaZ0nRUVfOCKoARlyPmJ5JwSM5FJyTWmpKkmcB4jsItP1GP7JcQTwthg1uc4Pp/L169aorcGTYY13PgRnJxgDk02VnglOwAqOpXqOMVSgY+c5aQfIO9cnNqWoqSv2L6yOhRXfO0naAf1qybhbcq0YDKxBcZ96zI5klAVypdfu+gqKecgGMMHYHufu/ShMn2d2bD3oefzJI12nB2Kc7ailugSAcru5XnknrWcJttuCMh8fmPeiOOaZfNYoDGRhRnIHtxincaprdlv7Q1wGZcKFHJ7Z/yKW2U2oNxKNrdUODg/8A1qaqBdqAL1AbPX8ac14gUhoVPVevGO1K4eSR0w+JWoQyagkMgeOW4LRSOMMidAPyArntS1iXVHgWS5dmgBVXfnjJx+PP6VlR2ymbcJRsA3Nn+VReU/2ktHyrc9OlDnJrUpQjfQ7fS/GmoaVo4to5/wDVZ3kDHBACD34XPHr7VR+Inih9Z8OaJb/IDIzzTANnDL8q/TIZj+Irmd93MRbIJHBYbY1BOT04Hrz+tReI7C705rOG9gkt5XiMgikBDBScAkdskHjrweMYzanJqxUKdpJmHW54Q0AeJfEtrpsjyxwNl5pI1yUQDP0GThcnoWHXpWhD8O9ce3tJbj7NZtdyiKGK4chySAQSFBx15BwR3ArpPAV9pfhPUtQiuYxczSOkUU+zBID4IwRkdQce3PbDjHXU0nUSWhf199C8N6pbDRrdYba0O2eJCQZiGD4ZzksOSOSccADAry/XNTfWdcvtRkaRjcTM6+YclVz8q/QDAA7AVueMNWi1DUbq5tsiC4mJjBUA4wMkgcelcpSlK+gUo/aYUUUVJqFFFFABRRRQAUUUUAFFFFABRRRQAUU5ApcB2KqepAzir8emxSgMl1lT32dP1ppXC5nV0ukaUYYGnn3oxBZgEDFVxnpke2ef5Utho8UVnNdtDLMF+VJsfIr5B47Z259fWtu1uVEUiIrNIYjt+UjnHvx+fp3q4xM5SIBHbxz70WQxgfKWbk898e2OlaUV3tRTHDahQQu828bNjgfeIz+Oe9QWltELaVridonHMaLCJAx9Cd67f1pJYo1nItpmktwWx5sexs5OOAWBHT0q0vIhmqmozBj5cjoSvJViO2O3sSKr33iHUtOjP2S+nikIyGEh4JOCf8+tFqbYRlLiKdyc7jHIq4HGOqnvn8xT7200t41kMV4Y0GGX7QhLHPY+XgfkaqWqskTHR3Zjf8Jp4jcHdrN6T/13b/GpP+Ey1vyyr6veA5DbvtDgjrnv9PyqQQ6AqtnTb+RiRtzdoABgZ6J9fzqCWHSCF/0OWNRnJE+SemOSMfp3rLlkauSIIL+VhmPHJyWx1I71t6pJFa6pfRRYEUc8ioB6BiBVawTSktPkg8yfcfmdjgLjqQCOc9OOx69n3Btby4e4lt0iDsCwiZ+meT8zNyR+HtVWZN0Vp77dbBVPLDoDzVjULrz7ya72cO5fb9Tmo9lpCFwGMalv9Zgk9AMkAZ6HPTOe1Nma3uHyUKkgZWJiBwAON248kZ5z14wKdmK6Y2OVTGHJ5PWo7acFijcqAQMjtnpU3k2SuYVN1GBjLO6t9cjaMZ9M8e9SfZdNWVPIurtUbPmb4kcj0x8y/rT+QfMYqxnPlqAB0wMUyRE8hZT95naMcDooU4H/AH3SrFF5AmW4nNwVI8vylKjBfjO8ZzhOfRz/AHebNxb2r6ZBDFeytKJWkObYIFDKAR/rGycqv5n0xRfyGkZTuqE7VXGM8AVrWV/JJoSQZYIszv14PCgD8MfrVez0FbyMhtWtLVyoZkmEp2jOOoQjqQPxz2rasvCzXGnw20evaQGiaSSTdM6Z+7gDKf7JqeYLHNySq+WkIJPPNV2mXACs6jrw1dFH4IlvB5sniPQIN+diG7JP6Lx/P2qKT4eakD+61vQJT2C32P5qKmTKSMOLVby2JSG7uFHoJDj8qv2GvaiZRt1G4DjnJkbipv8AhX3iBOVewaNskPHexkN+O6kn+H3iXTIzdvFbvB/EY7hT2ye/H1OKSYNF261yS/SSR0tvNdWQz+UBIFK7cZGOg45z6dOKHbR797C1tTc288jpFK5ZXQEtjcoABPB5Bx7Z78zIJrOaW1nQpNGxR1PY9MU/SYwdWhy+CMsnXqASBx3Jq3sJLU6YojT2qW95bSbnCbpN8GDzz864x/wLJz0q0un3lxdvb2kcdzKqBv8ARp45MjPbax59utZDr5sahSBj9Kz7jzoJyiyHnqDzVKL7kOVuhL4ntrm01UQTxujRwxIcrj51jXcPwPFZMLyA/KD1yTWr/beoWsrTQ3DrIwIL55OeT+frTraRpTLM4RpJovKkLxqwZOOMEY7A5xnIBznmpaZd0Z8yvtDs3Dc8VChZp1RSSzOMZ9c10upJDe6VZg2dtbDzXiL26FGcIqYzzyfmwT1PGc4qnaWuleTHHJZzFlDHzYblkc5IKnkEDHPQc5HcZM2bQ7og1O+V9Xu5YkKo07MBkcKSccirmZU1G9SfAmWdw427Pm3HPy9uc8VTm0mGSd5LctFH5o2IcNiPPv8AxAY6nnBya1ZYILzWry8lmNrDcztInmAsBuyTuK5PXHQH73saq7uS9hzPGAOmDWczr8x7Val07UXs2mhiia3ViBJ9ojGQCMnDMDjleo71lbbrzzB9mmZ1PKohYj8BTbQJMViOnHr1rSuFcCAZCgRKV78Ek5PvzWbsnEyq0Usb43YZCMDAbP0wQc+hFdLd6Lq91pWnXUGm3UqvbhGaJNxJDsF4GTjbt5IpX1HYzo5RGuQc+vFVL/VZ1tiseQCcMQOgqQWd/bqqXFlcx7vu74WXP0yKy759sQLL/FzilcaRVN67OQXPI6gUv2ho+RK+PrUOPMcbPm7dOKUwPu+b5c+tZ6lDlu5Vbhmz70pnuCMkswzxTRA245dcY4pm6RMgEccZoAeuoToR8xBz1r0Pwxr0cfgzxA87xCeOBDEzEKysSRuB4OckYxzz715i255AuMknA+tX52UeHnXeB/paAYXG7Cvk++3PP+8KE9UwaOxm8VXNzbljODleoGCfx9ayE1jbAsBkf92Op7855z/nisu3jje1O8HcMFRnOeOvNOKkRcfLjGC3WrUmQ4o6LRvEJt9TjuJIFuguS8TkYYEEemOM5/CunHjCzs0ddM8O6baeYu1sRqQ3HfAGR7V51YyJ9oc7TwvfrWjHKrv8rFtpx09qdk3di1Wx0v8Abtn38O6Bnv8A6CtFcuWXcfnYUVXLELs9/vr9bERlopJN7bcRjJFYJ8RpFfzwNqCR7HJWKZYwzDONobeBg54zz0ro7qF57Z40laJm4DocEfSvMda8N69ZakzQ3Je0MxMYMpLquPvEnnHXrkj34rgpqLdmGJnUhrHVHolrqybd9+0VoJZSlusrhGcYB5B6NnIx7VqySRQJvmkSNP7zsAPzNeP6t4jS8W3s2kZzbYjlkjQksdvI54BBz2wR6DOc+PXEsra3R97XMUpxPnnlslSMjA+9z+laqlcwePs+VK56de6xoGm/aL0XhR5HCNJHl1Y+gyCvvXNeK9T0fUr9bCS2SeC2j8+FYHKbpM4ZTj8OnvzmudvvFfmTwx2+m29rFFtZo4QIwwODlQAMHvzk8DtnOamp3KP9nQMiSSlxmMbsMQCCepyB0zVRorS5zVMbK7grWNXxDGmkJaS2wiSCdQbi0eIkqffLFtwxzyv8PvWNa+IbjS7kTaaY1O3DkKVHP3sr07/hgYrNa2aKVonn+dydu87ck9x198dKW3WOCOSWJ5EdcMh3bio6c4A74qo+6jOpLXnR2ukazAHlkug0ioDGMqHB4O3Cgjjg/wDfI9BW34b8aQRalcWN5dM6zTb4pZZPljVlBCZPpzya87ivbdhbRi12FZVlkkIPQjBA56YGcEcEn3J0rnT47xhcnU4IWt2ChsDJXruJyPYZ561cqabuyaWJdFqKPVZNcj1Wa4sNNuljfyWK3akMEYEA8f54IPcV5xrc9vbXwn16ximmkT97NbYdZASRvT5uuMduufTmpaakLd/tNj5kYE2yKXGUQg9y3I6+vc9c1DqMtxqGqzySSRZYFm2tlQScnA6DPP8AjWToJSTRt/aDa13MzXrG2acT28bLb5C4ZSuexPsOCfzqitlE8zyW5URIC2zeWKqO7HA9Rg4wa09annv5IXnlDkp1XIGO3bjP5kCshdTksHZIgFcxmJ2AGSM5zn1BxzS5Yx0CFWtUVluOtnDq6BCx5AC9a2rPRVurC8Zgpnt0wYo5lJyOQe4x9DknNYtg7GNhHKyxhgT83Q+uPX3rYuALOOG7+0vLdb94YksoBIB49cnpkVVOPUdefK+VblW00mRhG10ssabt0kS/eCDv0x/n3qwXS0mA3O9lIwkVCPvYyAR9Mt6Uybdcv9tnlWOGKQpH5Z2jcP7u7OVPXkcZ6dqozzzM+4EzQhMSOpBII74HQYH+ela3iuhyS9pO+peFy9iZII55nRyTIAwCsDzyMdcd+3NT26RzXZjhT7RFGwOVXDYY+mOeT36ZrBlcq1vNF5ik4fa+RgD0JArVsrsCVImhV1ZT5hbJIIOQB+QHI6mnGRNWlJR1L88sUDyPbndFlVCE7wxzyTngY7Zz+RqK5uPPmjlVAkJIBZQxUHHPUZ6/X8ahFw8RkP2Nt3lFAh+U49SPbH9Kl0+5L2CxEEBiC2FHzqD2Hcjn8qUrPQmHNH3mh9z5kcMrMFARQVIDDeM8MAwA257YHp1rLmSWRWMRw38SrnsB2xn/AOvmtW5XyYrxYXhurMFJGkA3FM4wcdvT64qq0sa28TONjBwUI6J2+g6Z6+lZSijppVGtUjNhDNGqFzjPzLk4X049a0EikhikkIClOibSSR1+mMf571PhJori4wqwRtuby/lVyexC8D8vWqs97IiLNCVZdxATnHGDgAj39al6bmznKbsiwi7o45I0lj54VgDu+mP0xVq2aR0KmHYvJYEOvTr7eorLjuoolcGQ4dAP3g4BwOpHPX/Pap5LqKeyRHnBKcqMkgH2547U7kyhJ7hcqyXPyZQdABnDc8AHp6d+1dbofhgzKl7emVV4KwlWPOP4u34dqu+GPC1xbRpdXPliXCsquC+wjvjIwfett/PuJJI1ld0H3u6t+H8h36ngc6xjY6adK2siJQJ32oCYtuR5Yzu9Pz7Dv14XBaWCFoXIfAwAHBOeD0ye54OCfoq4xU9umwbgWJJLdc5zyTnvk9z1x/dAFUbx0ikZwT3556nk8+p49z3IHFapWNm7kzzN57FFwwztGMnPfv1457+rDpVOW7csUihEhThuCVB9OBgj2GB7mq5mlulRJkMaBs/Kc7vTPqB6DAHpVny1iBYBSc9EBxn0xUtgo9yEQzSkyuz/AJYwf8KsrbjION2OCxGefw+lOUFlwSi56k4/lTtyIyjzAwBzuU4z+lTcqwu1QpyrZ4A64/lQdjE7gVHvyaGmQr8jFTnnLhs/pUXnwKMtIeD0VR+eaQyR2Cr+7aQY4GSOKiura4ubEyGWaNW4EijnjspII9exrMvbsLBcSRz/ADrG5jDDPzYOP1xXJ2etf2rf2tlq15dfZl3mf7TcN5QIBK/KTtUAgDAH88VEpa2Kir6nY2NhZvPb/ZNSllkRw00ZuwS3zDhlJYAdsgD8K07VtyWRXlXiYqQT3KkYBAPT24/Ws+2v/C6xw7YowFkAinW1ZUD5BG1sAZzjvWhFE8VpaIzjdEoBwNoxjg4HTrn/ADmumktDGqyZGkijWP5QA/PzEYXd/wDq+pry7x81x/wkRdMxgxqcHPIBPJ7Z+nHNd3q11dxW63tqPNWKQF4253KQRn8M59BiuZ1jxLpmqW9st3CHzesUcR4IgDAbSfUg5x9KKqurGClrscvDdWzS3W9XZ5EAhUKCd+Rnjp0zVL+1ZIrSW03OsbNllJx0zj/0I/nVO8kCTyeQGISQnBORjPBFR+aHB3gc81yOTNY01uSCYlQAcEcE9hUKozuzHrwxbHX2zUWTt2qcr0pxuvLAAUfQika8ttiaONogxZV3NxkHkDvUM0Thg4K/hTsggvI2d3TH5mmlvLQlMcnn0FISvctxbVXlAzA8k4oLqocbxgn5geh71Racc4YnPp0qYFZ4cEqGUE5x6c/0pi5H1LSTvPPI2Aeclj2qtJLuO0qSD19Kje6+UhemPxJqW1cRqCOZAN2c8UCceXUrYeOVccA8DPQ1b/eLhtwwi4NSjY37z7zK2QTyPpimgiWQpnYW7gfypCc7iQzywyRyLIVMbbgc/d61cGi6z4gvZ/EPkC9AlXdHtLM+FxkLjBAwuR/tLwc1IsNtax4WMyT5BLFgU7Hpj61b07WrrSYruSKcxS/K5CDKthxhTg8Dlv0rSNtmyfaLeIWOoXEF2l3fyyyX8LOLe3uQwClUJBOfcAY/Oqmr6xFf6pLqVvaGMzQkypjIWY9WGOmev5+9bWmeN9VtTLqM9hFO0pCLcNGQMAH5cj1479qwdUtobezbU7JgYLglTAygtE3p+a8HuPxFXLbQuKV9UcxqEoluuI1QKoHy5575/XtVSnSEtIxYYYkkj0NNrM6EFFFFAwooooAKKKKACiiigAooooAKKKKAClVmRgyMVYdCDgikooA6my8UXl/M8OpTmaSViyytgYPUg9uT+v6bOnu0kO4FSIndCSeSZIzj8vLJrz5WKsGUkMDkEdRXaJfGaxgYt/rEDOV9cc/rWkNTOSsbUWi308beasUDZI2z3MUTZHX5XYHj6U+DT5JkBSW12qcfvLuGM/kzgke9Zd9cTu8yEnEbnOOgJ4P8hUaB5YnkGckfyFapMzbNd0MMmxlVn25BSVHXqOpUketWm064lspQZbONsBh5l3Gox65LY9sdazIdoLRhlOV5OOmetU9Xm8q2aMkk7sfhnp/KnLRXEtXYtrol1cO2dV0iJOgJv4+PyJqYeGJFRSNa0WQ4PW+HHueP5Zrjd8jLgKfXrRFK65ADZPcmseZ3NeVHXR6bqMULwgWTQ7g5nhvoynHGDlhj72fXr6HDoNNnm8yLMKmNsNvuI0+uNzDd06Ln9RWLp1wUikLHjAHPbr3rXuJngdUxjgg5GDkfyrWKuZvQqXFpfTExx20rOieYVVdxC/3uO3I59xVcE/aEBziRQYzj7wzjj8ePrV+a8lbYA+c9/wA/8TV6/wBan/tm4ntppYv3jsu1yGUlsnnrk9/WhpsLoxrlXtLq5tXHzRSFW+oJFRIylWUnGf51pG5W5PmToHkJ3Fn5JPXn1p88sZuA6QworlXOyMABsHIA7DJPAwOnHAw7MLopqjCMY7HJB4pJJsKoPAHvWldyAvGzwRmWSQMzrldwxyu1SFA+gGPpUKiwLHz4p3GMfJKqjPHqp9z9fpiiz7CuivaieSK4dCohCgPnqckEY/EU1J33omCyZ5rWsv7Dhibzxf7NoB8udDznryg9+PfrV6Kw8NyLHOLjUY1ZiFfykYAjkjAIJPI/OknbdA0nsclHcsHYcDB5OKjlvJjMBg7Oma6CDRtAkmkWXX2typ2KG05nzj3Dn6fh2q4vhDSJxlPF9mT/ANNIWQfTk1MmUkcqZN3O8j1qC6mkMDhJWGVJxjrgZ/TrXXxeC4pyBb+ItIJyfleVk/8AZeat2Xw8gu7hY7zX9PEbHbm0mLS/hlQOT19s0rjsZ1jc6RdWFpFqWlrdIkRzOhZZucsDnIyBnoegzVW20/Rmu4pUbULPypFYHek+7BB6YXH45+lVmto4pri1WQyRQrMI5AMbxGjFT+JUUlpMjLmQhcijl6MLtGyNOsw8n2fU4ZeB5b3EUsAJyPVSvtywHJPaqWoaBqfmzzR2ck4R9rG1YThSOoJjJAPHQ4NQYIY/vMjORTomZm8wFi+CA+SCB6D8zWiTT3IbT3OflkQhDv65AB7mtKyYhVB6AVradq17p0KwW15MkKqUELv5ke0nJGxsrjk9qt2upW9vYC2/sbR5BhFEj2Sb8L0yRjP1PPvUPmuUrWMjUZHFtpxw20+YyE9CN+Mj8QR+FAYKAqjHHTFbs99Y6vJBFqNmIooFCRCxOwQqTk4Q5BP4joOvOebuQ1vNK0ZaZd2EzwxHbjmmvMHrsTw3G+VxjBBqGebJCFwu0ngc4pltFN5qtscOR9zaST7cDrT57G5QR3MlvIUkXcMDjHv1x0PXB4PpTutxWZZ0ny0gkkkUFnPp6VaYozjCqB14Ap2oQTaWkFpcWzRTGLcwJY5y7dM9Og4HHfuapwOVUlgQc0lZ6hqi5HCqKVjG0dMDj/PSrMMkysrebIHj+4Q7ZXqeDnjnP51TS52tz6Usl2pXIYCnYDqIfGGr6boKTxXjPL9pCASjd8m1ifrzt5NU5fiXIUb+0tC0u8YqctJF8w9OW3fl3rnLm4WWwtoQCQJWZjj5TwAB/P8ASsC9QhiitxkbgTwayaSNEegRfEbTZEHmeDdKfgLuwmcAYHWM9velk8a+Gpcs3hKxVsYxtUgf+OivOhGABj8hQe4/EVKSKPQY/E3gtXEknhpQ3okSsP1Yfypl34g8ETSID4ehYOoJMeU2Z7HHp7Z/GuBPIIAzTflB54pisd5L/wAIAXCixUqOQRLPjH4qP5Vrapq/gzXNDGlK8iyg5gdUdhE2MZGVz+A4OK8ul3MuQc0umzlHkD4wMY+UHP8AnijZpgdW/hPw/aP5UmsSTSLGJBtjby2yMhcr0JH5Z5wRitXTtJ8LjzJrix2x8bY2vGcAg88dfrn8BXHNfPknEnzLtZhjAAIPTFSG7RWZfn59T0/L8adkJ3Oh1jTdF0uyW703Ll5VGNpG0EHplifSsJ5Fjl67dx5GOKv6VexQ3KyXdul5BGpzBL90kggflnP4V1kPi/RSgS88K6cyhdg8uKMAJkHGCnTPOOKbl2RKXc4bzovVfzFFewQ6H4dlgjkbQNKRnUMVbTosrkdDx1oqfaILI2dZ1ZtHNpK8O61kl8uWXP8Aq89CfapdQW0uNIuboBZVMDMHjIywAJwD3rK1a/0XW9HjX7YrGdSYIg2DI23gY/H27V5gnjC+06yGim8Ef2WYmOaME5XsOB05riUW7we5tUmoq+5QU2j3TvJK8Mb5eGdU2jfgZXg9OenbjpmsK5vZJDOhugCj4ZRzuPTIIFdTdW8xkN1Kqi1dgDMqDMueCcH2HTpms6NLq922F3cXckWfJTzH3iML0IB6emAeK7Em0rHj80U22JBeSXen/aolDToQkigcOMdxj0z0x0pov47yVpWb7LHGu4jzASoA6JkYye2azxYvDLcW/mrHJFuOD3YcHn8KoyTyjEYU5QlTn+JTQ5scKEG9DSe6g1PUCwQhH5V5PlZmHoef8P5UwyRiNYvO2yAkFzjn8M4x9OvHFUYnA2ygRBc42Sds9sfnVlIxI7ykgMBhV6g+xzmpLlBL0LMV5cXTlVRMpzu/vEDHJzgDHWpJ2updO+0RPbyJENoXzQWXnuAee/X1+lL9kgmjR7aGSUeWPMD5GZOc7cce3Pb3qT+zbea4jS3cwQTFSY/NDFXxySvU9wOOv44rW1jG1NSH/wBstLp4t5kjWELlpF3Es+B1/wA9u1W9K10263Pn6iscpB3RmMFZRg9GAPzducdeM1yUsdxHLLBLuSQHDqB2Hr7VctZ0htzHJGGRm/1iAB+eoDenHfI70lNtms8LHluupvLPDND5zzRo4kJEfUqOcE4HI/wHtVO8tFurfz0bzJfur16AcADnOeeh69sCreg/Z1mmzdyQ3QYG3kiU/PknIOPbH61YmtiWlFugV0YhiTxgHBPv9cUTjpcwp1PZzsjLtrtTIsUkKZx5f7mPDc8fwjk5Ixx+VaCXKBZY5fNlDsScxlWAIHOPX9OODmqKwwoqXiyCWRZCsiMRx6DHX8f8K1WviJmlg3Kki7G3L8qgjGG49uDx1xzRHTcdWak/dRnahawRRxWsCCRQhdpozww9OuARg/mM4qpbw48gW21EYODJkAMMEEZPT73OT0qxI8cyNFJD5Tcl2x15HVfzqnDAVuxDs2JGjsgzgMSVB5GD/wDXqn8VyofDbqaDQxJLbhcTToNuC24MMcYHpjoeB/OpoX0uN42LFJpwUkSNWCK2TjI64IAA/M46VTKSwuEMXkofm+71H/6u/WkgiiublYzdRROTlFn+YNzjjj1p63MuVSVm2X9QubVbWJAolRVZS8Z2MDx94YOf0z7USbZY4dLs4/NeP5gA6AsxwT0JyeccHsPpVZo4obh7KOTLMuFV+hcZBUgDrnpn+tQq1olsHt0Mjsi4RWORj7wPv3z7Ur31ZCgkrK5etbBkV9zq07jy/szbhIrZzgggdgec8H0pFMJje3e38pvJXcSwOGB+8fTPToaZpqXF5AJFnZCshWMOzBixB2kMD0DA8D057VKl0GlaCa4WaXdtxHIBhhkdvvAg9apK4S5lddipdyEW8a27+UjjDc4D/wB3PGAMgn8vSmQ3gdxBNCzMfvMpyceowO3+TV6bTILi3kkLqoikBaANhh6N05BzjjH8qx0t447mRwTIABjaQCDx2Oee1ZTTUtTroOnKNluLcRytKchcNxgnkfhniu08BeHXuphqEzxSWkLfuUmXcrP/AHgB1x6569uK5vQNLk17Uvs3yxxL80lwxwVU4wMHjJ5wPr6V7HEbfTbJIrUKyRKEVBzn0BJ9+/506cbe8dkYvqPv7qeCHYl1GzODjapG0evBPTqfYH6VWto5LWyDBWZmHHGeep+UcEk9u59FFFvE11Kbmb5ZAcDnpkZx7HhT/wB8+9LNKiGSR2VViBOccKmB/iBjvwvrXRFWQ2yKa6EEbSSsBgZ3bgSQemCepOPxIzwoFYPnz3B8xpGIxkL2Tn/9XNPklN/deZuKrvyik8D1JOOp/wA9KlWGHcpzKGHYMeTUtjSEVGKr+8dx2B7fSp4i0bKVwCO5NTRKgyNgcc/MQCfbjNDRs0isV3KpGcIFyPTjpUlkjXEhXZNO+D1CnimmGMk7Wfnp0qOMNtGJW5HIBqfgSM4jYNzwO1IBj2zEMyowI6E8EfgKrG0WWQA88cA5U5+nWpZyZGyBKM4GBk8fQCqJtryRf+PduOhcdPzpAiaXSFAO141TbghgT1ry/QryRddsvtKqYEfLqi84AycdeePxr0K9tNUSEpBiIE5Owgfyrho/DrWeo/avNdipZsY46H0+tR9pMtPRnpS+MfDCf6M97IhPU4kP6gEdqt3M8U+nrc2RWa1YYVxggj7v8xz/ACrFsrjS7PTlllghUBRHL5oDZYduevqM1bOoafLaCKwe2ILcLCQQMnnvxyf1rppb2MKq0uaNuPlhfHykdPU4zXJ+O/Clu1gNUsxFamI/OqjAbJ4wB0OfzJ/Guq0sboiQQxjUhc9CR3rlbnx1ptzpdzYarbskxyjxEHHsc/5Na1kmuXqc9NtO6PNIblo2deMsPnz1Pt7VatZbBkEF9bjZJgtcRn50444zg9uPapdb0zTraV5bXUTLGygxKUIZiSM59OM/pWDOSsgAclO27riuF3i9TqSUtUSzxoQRAcgHGTxmqu1lX5uvYVOJQI+oHqMc1B5m5ix6+lQaK5JHuBy2doGakyWTAAwOuagDgsOTkd6flgdxOaAY+ONVl3Pt2E/dqVp9u4kBs/KNvp3/AJ1WBDvyeO4NIx4IC496BWvuWV8ppMMq884ABxTQ0qsUUZQ02NsR4Vfm6k+tRNcMH9B6UCs7lpZjFlFGT/WlidjcAuhLY4GeKpNKQ+5eoPercMMzBZCygdeT/n0oJastTRjbzEHzCM84QHOTV7Sr3TYVdtR05r5iMNmcoqD+9gDk9uT+FYVwr71aKTcemB2pIvOO5SpIA5A9KcW0ZKF1dM9KEuhaX4SjuEkAuSjNDFKTKFZmDcA8ZGAM8dPWvNZbm5mkIllLguZCCeAx6nHqalVLi6kjhkL/AN1UAy30xUeqRT6ZCjGC5heU/u5JUZcgddvQdx+dXOXMa06bW5l6ikkd9Isrq7jGSpzjgcfh09sVVoopG6CiiigAooooAKKKKACiiigAooooAKKKKACinxRSTOEjQsx7Ct6z0SzW3WS8maSU4PkoSoXrwTg57dMfjTSuJuxz6I0jBUUsx6ADJrsdBijgRfMg8xokZgpG7B2luR0IB/DGetbjW+g7IobazuY3bCsY5I13ZJ6/u/eqtp/Z9usiyW/mlh98yMCp9Rg4/MHpWsYGcppi3GuTtA8ISArkj/j3TngAk8deBz14pZkWC5igKKQ8SOTsAyWRWIx+NX47TTooIZY7edlfJIeZff0QH0PWi5u7OW4BbTY87Aq4lkyoAAGPm7Bcc+v0xeu6RGhWN6bSzlSMfKwAUdlOQc4/Aj8ad/wkd2unPDPBbXKjlRNAkhUnHzDcD7dPSq51LTJYnV7SVQ2eI5AMc8YyD0qd4dFitJJPPvoWlG5FaNHG3dgAtuU9vSh7aoa3Kh8V6gWm/wCJdpWSCBt06Bdvpj5OfxzUEfifUVcNJa6SRnJH9mwD+SUQafDMwLX8CRnPIZc44xwWHPtntUkeiRSOzteoF2gqSY8k9xjzPrg9+OmeMtCyxZ6n/aguPtNnabiQV8qER7RnPG3Hp3zU4t4Ly62Mkhkmk4JkYhdzD15P48++aZpGhXTzzpE0Kk/cWaVIy2PTLYzz0zWk+h6np13CLiFEbehH7+Ns/MBwA2SAeDx2rWLRnJO+hlXFjDOqiKQ253bmwNwOewHUY55yeo44ybNzZafcXMksKyWcbNkxr+/2/TJXv6/nTbiKS3kcSo0Zjba28YwfTn8ajZxtY7hwM5zVWTJu0RTaW0dy62t2lxHgFXwUyMDqD06+/Q/jFLFcCURi3fap27lGVPOM56e9WvKlhfLoyOowwYEEEdQR2NSRXe6JQ/IPX607DuR2pkitJzJFKGKMqtsJG7IOCe31PqPWsf7VJ5UztgEEAe3+c1rDUWWz8pmIV1BZAeDyp/min/gI9Krx3doqPthA3DDMqjB5HY9D05HNJJibRlm4IUxuSSw/I57/AMq6GGYJ4Y09dnJknIwOpO0D+VZ4a2nmKvFuDDhs4IP4duv51u2N3o8MYstR0uSW3ByjxzMGTPYDOMHnt3HpSd0ytGcwsgIcu2GDncVOefr3qczomEXrjGa6G5PgQgRppuqSN3la5CnPfjODn1x+VJA/guBCzadqjDbj/j4XDH145qbvqh2Xc5/duJIY5x17VLDdPEwIcnZ+lak+p+FfIcQaHeCUMdjm74I7ZGD+Qx9aoaNbaXq18IJrx7T5SSSvDAcnnPynGfXtTUhNGhEumiGbzY7qOVo2WNoSrZLIytu3HIHIPGf4umRilHoH2i9lTTdQRoo497SXq+VnkKAMbv7w6471VjmWUPtYlVPHNWLW8Nv9oIOVkiMZP/Alb+aik46jT01LFto2p3LmGCKCVtoJH2mJScjJxuYZx3AziomhurAvDd2M8Mi9RJGVOMnnntx19qZJNhSQwYFsVoW+s6hawLFa6jdwRDokU7Ko/AHFXyyWxPMjGSTDAcFifWrgQsmcGpWurHz/ADtQsFuiJTNJIsrRSPkDIJXgjgk8ZySc8mti4stHi1Oe2W8vEhSVk8zakg4PUYI46+tZ3aZejRh7iuGXg96S3YLNcuyEqIuWH8ILAZ/XH41pXekrA8i2+pWF0IwM4lEbHPoGwpx7Mfz4qKPS9Vit7xk0m4mSaHywY13/APLRGyNuc/d7fzwDTkiUncwy4F225j5IBLYGSRj9a1ILl0uLhEnOHYq+2TO7B4yR9761FotrE2o2iXAZWaZQUKkNgtj7vX9KoCOa3uQVDFQeCP8AChO4PY3bnX9QtVDm4EuM7RPGsgTdnOAwIB5xnrVe08QaSI0g1DQoJgoIWSCd4W5OSWKn5vb0HFZV3K+5ZHVghbHTrVSWRWIYrjIquRNC5mjtCnhsmJ4odTEJfLH7QmQmByFKnJzngsBjH4S3Gi6E7xuus3MCSJ5ipPaLKwBJABKFeePQ9RXGwXcpiTdnaOMYrY1CUx3MKkk5hjIOexUH+tTyj5jpE8FWeq6bcWth4gt5J4XWRfMhaNFVhjluTyQO3B4rF1D4YeKo5nSGG2vtjgFobhFwSAQD5mw5wR271X81RjOMetObUbm0gLWdzLBITw0blf5VLpy7jVRbA/w68VoSh0nDjnAuoTxz0w/PTtS2vw18V3TsTpUkKqCcysoyR265/pVG08QazYSySwajcRs5yxRz83GOfX/9XoKtx+L9diO86pdv0+/KX/Rs1Diy7lefwJ4qt423aLe4HBCR7/y25z+FZsvhfXrf/XaJqiD1aylA/Va6P/hYPiFcMt+3X/nkn/xNWU+JniPHzaiMf9cYx/7LS5WO6OVTQNZkQBNIvyGHygW75b6cVWbSr2yuGhu7S4t52PypPC0Zb6ButdpN8RtYlCtPPGxH8XlDOPwx+ldBa+NCtnZz20EKSztIJiIeONrKwJz2bkHrxVKDehLmkedLo+oNaLcJYXJtyeJhE2xsejYwanXw7rM2Hj0TUJY2XKmO3dsjqCMDmvRB4supC5HlqD97ES4P5j/OKgl1+7c7jeTRnGP3blARnphcCr9kyPao4G3VovNjlBjmU7WiZCpUj1yOtK+SrYOBjFaniG8kuZoJJZnfO45dix7etY7zqqBk5zyeama5XYtO6ue0Q6tHHDGjowZVAPyjrj60VnrayxqEjlZkUYUmReR2oqIwjYd0eGm4ns74FJp1MZ/dMw2suPbJx+dWre9jZpxfR+YZvnVwwGH+nTHPNbV0LG9mt1s9NhhDxDzgwZNr5OdhLcnHuc1VuIdMnhK2UM0/yna4hXII7HBz9Tx9KhJo4JYhSXK19xDcC3t9/wBmYFoyN7nnJU9j6E9K0LG6tb6yNr5kpvrqQZMxwpJ4HOT0yecZP1rn+UYpJGSi5/djufrikaSGKBjHuLhsAHpj19QaqMnaz2FKkpqy1L17o97pUxEpYySjBDjp/wACB/w/CsfZLKDIAzqgy46hR7mr39qX88TWr+ZLuYnMmWfJ6c9egNJcaVeRQW+0xstwMoiMSQR2I9RUvuaxcor32hfszBo0kt5lAUSPjuD0I68fnUsNsFnaSAS+RgDEufvNnHzAY7E9s4NQxvdAQPK8kSKpiD7Dyvce/Of/AK1Ty3cdxEsMSssiDDgJkNjoxxx+Y49TVWTMpOb90s6SIIBdpczSwTOA0QXO2QcjGPXpg8dDzRBLpgkmglijZxE2C/yDIBIAJPX8iSaqwW7x3McktuGUASYV+XOOM9eSTk/jWnHEgkWc2ywOVIj8mTYwPGCDnJP596uzsc9RxUrye/ZnNyytJKjOSzdz1OB0z+VW0ldVDIRuK4HzDgHt+VNt1i+1st4rRsSdxVBuB9Mcd+1XLa333VxGEWVIsMzK+Cg/vdORWKTO6copbElnEdpmd0WFMGQkk9T2Hrz7VPJqD20f2ZZVKqu0NFtJIYZAyQenGfyrN1FZBekrOJY2Abcpz1GeRng+1LFKFjVGhLRnO4qwyce3tnrVX0sckoJ+9ua2jTRafbTXKStLPcAh4wgIzzjByc/Tj+tT/aZJdQeO/JQyDMTrhAWPO5l7evHfsRxWIl3LbTzQRW5jSQEFXwrr17kcdsjv0qxbPKiQvbN90/NuIAHI6E478da0i0ZVaT3fUtblitd6YLDJViDgj2yOnB9OhrEby2uIfJ8xNuTjPO4kdOOOMetdSbOS/t5ZIZI4Y42EjqOVWQntxkg89eBjr0qpDotrHFLM8oMgJjjQAAFmOMrn3A54wM9aXK7hSqwgve3Zn/ZpxJdKLht/3HZ2PTdyT3xx0x/9eK5smM3nboi28MGVuCAcdBz/AJ7Vfu9PeDT4plkJmbIYKCSoGOp/H9KiLJa2zSKjMVB2kHjnORUNNbm0aqexWuN8bLJOTJNJ8oYA59e/Xt+dW0ubjSpEhtLlCpbahEnBbABJBABB9xRpcY1i3eFZoVnLqiJIpVsnPIIzkDGPxFVJoLFLZ4JpppLpGBKbeMdcDuR+I+lGoOSvyDJILyB7oncFdgJSFwC2cjJ6A59/bvU8FhNCsN4WKyOxMZ9NoyW/SoGuRAiwxyulvMBuVwVAIIwxXJ9OufWuit3RLKOCVh59mwj82Ah8gscY6ZHJ9e3FVGOmhNWcopGG001znmIMT5hkxjgnkYHYHHGO9WtJ0uTUrowqBuflmB+4B1JGef05NUrxIoHMciOrIfu9Bn246Ywelei+ArB9N0sXIaBJbwB2eXkqvYAZ/E/X2qILmep204KycdjoNO0+PRLBFtVjiUDceh3ADqSep4H6dsCn3P226uA8ownAjCngHPXHfGOn8R46CrYaWVsPNFIAMho03Y9Px7j3AB6065QurY2YVt3ynaBx2P049l56muiK1NZbFdQryddqhsHknPX/ABbnv857isfVczaiUWRmjXBePKlGYZxnjPGSOuM54rRvrgQR78qGcnCk9Txnj6Y+mFHY1liNZcfLGoQZJj9/qabEidfLKq7CKNhn5Y4wR+tSfKU5UMT6ooH5ioUR8gjOW7EVcFoVIc3MSdlyd3/1v/11m2i0hi+W0ZPlYbOBjAH5UrEHHyj2IxSmYksGnYqo6jAqKLEb5BBZucsucfnQBMHbheB74PFIUJQ/vJCe5XpSNckHI4HTgYqGa6kkJHnMdoz97/PrQCGhizHCMyLwMlRk+/zZqRbpo9qMIEHoHyR9azWEx2jAwzcFjgfmajg+2TzMsLmP5ecnhRx/XFIZpXFwssZIww9V3LWFeusiyfLIeP7pwORVi5W/jGHniZVPPXJrLy8iSZZWcDA2sQeo/Kktxkmnafp19rSR3kGNy7EkdTt3jnHUDJAPBz29eOxXRdGiZdot4p0BCOAq7R9FwCeK4B3MkhjY7VdQN3uc85/P6VNpFkl/qgTVtT1Bn3YijidUR/UZBPPsAD71a0egtzu4lFvhAw2rgZBznnGf6/jXkXjDQp4fEDNAyyi5JkXgIOT0HNet2ejQQxSR21zdsznKxzyiQD2HcD6k15p4x8Ma0damltoXuoGG8NEfu9sEdjxV1WpRT6mEIuM/I5C8sLy3WKSZG/eZxlgSSODwCTj69e1UDlyCTzWvqWja1o4Et/ZyxLn7/Vcnp8wyP1rPWWJYxgbieSCO9cj3OhN2K0gZWw2c05YTEyu+MHnB71YDAR/MCSx6GorguT0HHFIabehdEULsWKKeOOwoFlHGNzy/uz2HWq1rvMZRh05GRUhYom0/MBzkUzFqSdkx9xbQ4/dEpj171RdSi5JP0NWlcO5JxtHQUrpGVIJxnt6Ui4tx0Y22f91kj6Y7mpLiyLFWhIYEZIPUVFF8mUVuh4q2rbkOUZn6EY5FMUm07oqC2MUvOGx2HNNnmkVyDWjDaSPKMb2JIyka7mNaH2GZJGNvp8UB4BN3JH5n/j+APwAoSbKSb1Zzlqs88uI1LEda6DStIeaVWmdVtyGy7HGTjHyj+LHB7DtkEjOvZ6WY7RZbgRs24/Kksbjj1Kk9vWrM88DeVGgSVlLEdk24HY9vlHXA9jWsYdwduhEsFxbh2spFtYS+fPkYhn4xyygs3I7DAPpWTqWkprNxEs+pk3AyF2RGQMTjuSD0HpXR3ro1ozMTMWwAqg88469T9OOneqT6cwQSR6dJagHhyJAPf7x/lV2WxKb3OP1nw2uk2QnN4HfeE2bMbs5Oc9vp7HntWBXoHiUJ/YV2bdX2YTzHkIyTvHT07cDPTvjNef1nJWZrF3QUUUVJQUUUUAFFFFABRRRQAUUVZ09oV1CA3CK8W7DBjgDPGT7Dr+FAFau98NAX+nQMbiGPyQUdX3DkdMYB7Ee3WqmmW0N3fyQ3Wm20DIQyI8YTgg9SeOnPP+Fb8dkthE4V7QDBxHHIpA47bcgZ+vU1vCNtbmM5X0sQXemWMwR7m+kjcDCpFBv+Xnk5Yc5B49Oc9qdZ6Bps0ULLqcocEEr9kBBJHIz5nQY+p9BU81ijwFYb22eZlBxtkG3nkfc9fwqxpFnJDpiify4ZxK3yOcnZgYOVBHXdTtFsV2kVYrBYLmOaW4Mao25cQb8sOQPvDjr3/wAahaw01pnH9pTgArg/Y1GRnBP+t6+g7+oq7qDImUJilxk/u2fKn/gSjtn8xWbFE7OkpVghbgMuMgAnPOAehH1GO4zdkTqazfYsLbRXhMaKg894SmcKBwgZvzzWexhs9Rin82GdEdWxllxhs85X2965zWry7tr5XjYohO4DP4cjt06GuhEU15ZNcQW0ssYi27okLgEk4zjpycfl60rp+7cdmtRLLRICsyf2jZtIQQilmXcPXc6gZ9s07U9EuzpMDyLCyRJtLpdRN1J2g7WOOe596cbG5t9Ntrqa0ukVWYMxibA4QDJx65pqXcax5ctg8kj0zjii11uK9nsZ7eC9cSNAPsTs43Ki3kWf/QuD7HB9Aeall8DeLbaEyy6JdIgXcSF3ZHTtnmq8l/MieWjHYfX61p6Xq17H5kn2l1VUxwTypOD+HIrHk8zXmM220m9guzHdxzwShgpiYFWxg/w9euK0ZpmR2CnAB59TWgusMJ1nuFS5dCCpmLMVIJIIIII5Oev1qKG9054JoZrIed1WRJiu0cDBDbs9+eK2heJlKzMXzwWZNo78hAcVKHWHUiYVUtHIWjynOQcg+v51dS1tygd42Vgckq33huHbHHy7vXnHrxVNtvnUkJEZW2Ak/KMjP4Drzird92iU+xbvNVnvbwtcTb5RjdKoGSRjqcc9Mc54qFoopI2RfMQlQPvjBPr90nn07En2w7+y3+0rGgE0oTeyW7CXCjqfkJ9P85qqsisFYMNp6YNFosLyQSWjpJGEXemPmY4GDk8DnnjHPH09XQ+Hr1ovtSQtLbGYwq68729l+9jkc47iopriXyvlHzZ6VLI5i0yG4675XyPXhf8AP/6qTjrowv3B9LurZhcPazpBu2l3iZVU9hkjr7Ul2XSGZw5JUEjPsM1PpOu3llKxt7h41ZcNg/Kw9CDwRyeo71dknEGqQwQ2lnKbsRx+RNGixnci8k4yuSRkqR1pOTihpJnKwtlFLJyOmTzT2lO8nNdRA1qLr7FL4StIZmZ+JLmfDYYghSrEYByOOu2rUrxi2lgbwvp9vI0e1JHlkyhGcHpk549M9zWN2zWyOKacMOtUrgqWjBHJcceo5rtr/WbI2MkH/CIWkczIUSSOUttb+8MHPr39PoeUaQX2y1QeUiMHWNkBYnABO7qM4pajOtOl6TNYzNbNLaXITcqH95G2FGcnqCW3HsAMdT1gTRy7LFZX0chkKgLNE0RLZAHALL3PJYfgOak06fMV85wpW3IXJx1kRf5E1Wjlw7YJxkVaXYlvuMXTrmJXE0BDK5GVIdTgc4ZcjAweenvUEmY1dCGV1JypGCPz/GtX7bJGpSKRlyRna2OR0NWdPvVu9QA1OP7bHGs0oSU5DFY5HwT1AJycZxnnFaKTsZ2RzsCtdyJAjiOR5AqyOcBCTgE/SrKy4tiwI3ckg+3auisLjwwuqW+oLY3trJA6yeXFcB49ynIOGG7rjjd2qtb+GVvGVbG/s5kkd/LjnlMcuBnllxjkeh7/AFpKVndlON9jCu5Et70wRtJFEhwu8YPHHPvS+aHQtIFJBBz15FRa1Ex1u7IbzN0z4+Yc/MfT+lRRo8cZXkt1xTuK1jorTxdqscKwG/keFSD5c4Eq8dsMDgfSq+pXaag8ss8UaS3EnmeZEm052gY9MHGTxySTmseONwM7TnqealSZ2SOMr8yvk8cYppITbNeO08O3kURupdUACNmJJYnBY42lXKA/3sgr3GCO9G40CyeHfZaj82yRzBNEylMY2qHXcHY5OOFHHbOKzJD/AKaqScIORjgc1p2QxDsBGCOR3FQ9NivUsWPhTV7ywSW1tlnhBKlvORdp9CCQR65IxU0/hnWbYHzNIvDtIXdHEZATj1XP51QkuWEgAJ46nHH0q5H4m1KC4ldL64DMQWbzSc4xjPPsKdpNCvFFJjuBUj5lOCPQ1BNkW2CM85+ldbFr82oMGvrW1uSE2v5kSgvgk53YyD8zZI9vfNj7J4VvoZLi/W8sDnPlW5zG3svysQR7/wD6kpNIdk3oec/vGB2nOfXtTvKnUDJTnoCa7qTw74McFofE80ROSEeDJH/oP/16r3nhDw8qfJ4ztPMPQvEuP0kJFRddUXZ9DiWicDa7qP8AdbNSJbKxJWQD6jNdP/wgtlPtEHi7SndjwGDLn+dEvw61WBybXUbG46dJNpP4EVIPY50xFDjqT26ZqcTiSe3jt/MIiRhNFJKdqNkqw4A6gKcfme1dHY/D29vJtmoajaQruA/dShifb24+v0rVufhdDYzRPa+IoIpZf9Z9oQMrH25H5c01ITjc5eG+AzlxtAxhT09uSasLeoI/vcHnmtafwNZW0x+2eIbPzCcDy7fGOe3z9BVC10OwXVrqO8vpDZxnbC9qF3Pyeu4Y6emfxrbnb2RnypEP+hXClri3eeIfwpLsP4Nz/I1PDZ+GFFuZrC/YhCJVE+VLZODwQTxgduefYs1KHTNOuBbabJPLAqjLzbcseemO3Sqgn3DI6VMo33Gm1sdPHq2gLGqm21IYAGBMpA/EjNFcobpASPm/Kip5IhqVrkpNMVa6VWWMkIQQFIz94Ed+Twa1dPmjtLeO5IjSKSNsGKAsuRxzkEjqBxjkjOaxX1GSWHiEyAf69yTt5+g45PFTo9zCkEcrIsfy7gQXVUbhjnPB4U/hS32PGcHa0tB1tLDdS3kd0mHldWh+8H5J6BTt9OCO/HNNit4rW5UJF54GGlEhDZDHgqNoK4weSefanTxK8cjLKyMHwEPOTj17cU62EkcqNDCJPPDx+aR0AAJz+GfyzUpJaMpTbWmxW/sgQKJp7mHaylgUJ37sgEY9RkfUYqW8sUe1cpKfs4QO4LAE8jkdfm/zzTxCohgvpMSQwP5cuUI67QQT9CDn3qpcNbG7dIpWltnOIV2qfrlv4ef50NpDi5zabexPNaRBYMyPJ5OdiyHAC59c9+Dxgc+9X5L2GG3ktzbsZJZQEJ+6SeuTxzn17VStrpprGZriMGO1IjUqQd+7oG55A28H09aSzuY5o2DTpHKZCjAttKjaWHHXHGD+FUpLVGU6cpaS1sXhb3NlM4kmXejBkEXOPXOSCf5U2ba9mEcbELfuXPyOM9Qex6ng/n2qv5n2wvFbjbdLnfIH/wBbjAJGefQ45qUQ75pFvZRbSp8yboPPEh2lT0OODjg5wQc+lPmdiPZvm1diTXY7cSrBMkHnsSVn2hXZe24+vT6flWUllKbiKQ7408tgk0KseB/eAyR7gf3hVu6tpNsE13cSP8oVJCpULgcZz6Y7E/WoQn2m8W2m+TzdzAuigNnAOOB1KjHuPc1Ds2bUZOK3uVVtnRpWm+SHJKs4G5m6fUevNWredRcRpaqpKxkELkliBz9etTyRyXVizlxJGjFWDKdgyByB2PUZ9qpTefZ2VvdW9g0EbnaLhmwSwzwo7dOtNXT0Luqj1f8AkLPbSSlURgrPIF2fxk4449OPxq1EsFrPDbyS7QcghwVAz3Yjtk9M/jWYNQvrfVZJbhlMyu25c5Oe+D17VPeEzsl1cTFWmA+Re68HnOc9B19Ki/VFyg1aLehp5cBre0lzIFPmeU27zB7H0AGfaq5juI2zKREwj3p5n3Zcfw8Hg4xgH07VWsNam0i7drZE6/cUZVgO1Xb6/uLyUS3cKSQ8swjXAGewBPIBNVvszL2bi9iOW4nFtFI9wSTGFIzuIxU32q3gaKGedfJcCQop6d+COnX+f0qxDp0NzDbJJJETgsUSZTtP0459u+OM9Ky3u7MxLZ3CsCjMUPQZ7rkZznAzjHP503J3szNRT/4BQ1G0jivydPuBIiOXV1OcAc9D6VqaLFZ3MjzXU0qXcjONzRjyznoc9u46YOfrSpoEVzZPqNrIIk5DK8gI24J69MEggepB6VQtraGWV2Rd8HBJDD5fXgkZ+malK2ljpdRTg1fY0brSZ4oEg2AyRFmESZL4ycjdjGABn1579BmGc2yMQjAMdxwQcA+3P8q27dL0XmyzvPNLIZlEnmIN4xkjbzkKOfYGszU7cpDDdieITSjYIF4C9RwQeoOPzpOPVDo1dVGVncp2putd1m0sJJy0cswXcegDEbj+Q/SveVs7C3tmMdujN2EiE8/U5rzDwrqSX+pRXE0EX2qxRmefYBkbduWIBYnv+HYZr0O11Fpgjm6tHQk4ZOSfoRn+laKNonfTmtrWLUSgTMI4kbA5xx34XHbkdfUChlyR0wqAyO65GOCSRzwOD/3yvQVDYx3NxJLLbxDJUxuzN5aH5jkE4Jz9Af8ADR+xSOrw3N3EhfBkWJRnjnq+cjJz90VV7Ivc5ht+pXhnggcqh2RYBZgAep689z7k1bTSrtUjdrZ/LPJO3aT7c10a6PZQoN9uJABnM7FwPpuJA/DFTDywNsaRAegfFQ2Ukc7Fpl66sShUng5OePwpG0+aKRXDIpUZG4MeR9R61vzXCwYRmtoy3ABBbOf+BVzmo2lzrUkltaGIlFO+VvkSLoRnk9TwOv8AOhJsbaQ2WJB+4a4QOefuZYj6njFNWOBT8twzsW2jcB6Z7e1Ml0ixEkNxfXd1cOreWywYiUNychjklSc44HFMmj0e3t0eLSIjGXActNKzkYJyDuAB/CrUCOYdIVEm1fnKDJK1BLISpOcljyDwRj+VNuooY5P9BswkyjKr5rkyjdyGLE+x4x6UsciyRiRVUgrkAZ9CfWplGw4yuUZEYsfNZlCqcZyRn0/E1SeGQRq20ndyMD8P6VpPNNyQ4AbgjaKqSyuZFy7qAvRABzjr+dZmiKjLfncqhwijpJkD/PNQwRDEpl2l9p4UH1H+f88zG4lEUp+0kkkcP8xPXkf571HAk5k8tZn3TKBjZnfyGA9skDkU1uJlaUK7BRgHI5B+YckfSiGSKdWt8bGL43bSrfXn+eeKHQ+WTwQ3I49z3qOBlFwJUVsCQkHdnH0/wrS2pKLM2va5JcR2UepadbTq20NIrLISP7xKlcn2x1rsNLttWNx5l9rKXRZcMghwEPoHGP1FcPdyLcTtG7MkeQWC3KOpGQMeWBkD3J4rVsPEt5piPCsaXcCgEK5/eKv+y38Q9Aee2aUW1qEkmrEvxBT7DYwSySuE8wItsYt0L555JHsOCfoO9eazXVtqVzFDcRW1kFY5eOER8H1x6fSvTJfH7ahIbAaQ81q3EkM0Pnsw9PLyo/U9qxb3S/Dl44C6N4hijUlwghUBVPJ25OcccEk1M5OTFGCR59MttHvaC4kkxgfMmP61F5uegB4rtZdM0GKMpB4a1J2ONr3dwUGOxO0YI/EdOtT6YkOmYddA0eSVuQ07NN9OC5HfqPzrPluaWRwqbhJyAAeoqZY3uHZI1bpyq84rubu4vDLlrDw9AztlI1s/zJO38OtZ15q1/OyQNrEflBtvkWiuq56Y27Rn8aORisYFjpF1fbhaxCTacF2YIg+rMQo/E1ZOh26MRfazZxFfvLGHlZT9VXafwat2OCGFhvtz85AVpZDuPGSCBjvz0zUltI0d+qPMkUIJKiGIZXPtxk8Dv2qlAXmYi2WnGRItNS+vZsANPKqqg99i7iAPUv8AhV9NLis4y8s4fGN6IMA+oz1P6VrHVYrm1MQ1C/nJ4Alk2jP+7lv51lyRHcyu6xrvHC9T+Pbg/pVKKQPVkwSW5tEaKb7NaNw8aA5OO5TgMfqQapz2dpcXsvkJdyyM5O1ZlYZ9NoTP61qQxy2zrDGghUfMWm9fxBPoeB3pbm7YTyLd67cSLu4W3jeUAenzslN6ghLbT1j0YpcQ3VlI02RE8bKXUDnBbGBzxgHmhQkVsyAxwxDr3LHj8+ncj29KjRIvJd7G6Zy8m1mnhWN0wP4cMxwc/pTYwoKqpEkueDIAVBzwcfX1zVJXE2WDdhLeRYwttHt2o2755OR1PX+Q/WqXn2ZmCW080shbasRhwWb8HNXDDvjlK2r6pek7sMHIc5AO1VIY+uT+Qqgxv4JVE/huGzUsPnltWQrzjILc0XadgSVrnM+J9Wku72SxACw28hU4x8zDIJGO3XGP/wBWBV/WYJYNWufMjZBJI0ibh95SxwRVCspNt6msUraBRRRUjCiiigAooooAKKKKACiiigDf8N3h8yWzJO6RdyMOo2gkj8s89sV10a/YZZTJCTuiKMrDBG4FQQO/JrjdF0m4kniuy/lop3DDfMf/AK3NdbCxguGUsOe5PBrop3aszCdr3RNYWl4sjupjZcZb96AcBiOASCefT69Oa03MsZzKAFOF++pOT6jP1+lUoDNLBKsaPIxCk7FLYB5HT2yfpTBNbmHc80QdCBsLDdyCf6ChMVrlC5luLm5cRxys7ksEVSzY6n36ZNX57S4itIcwy7VUA74yMNjcRz3HPFRzMrRhoyMDv17VGk7pdR4OCDwR1454/GrV9xOxyWpws+sTIDkOwwMevSuwnmaIu4HKpgH05HI/z3rLuNZ1G6v41e5YRlwuBjdjoRu6+tWtsbKAMrkFQdxyAT759KmGjY5PYlg1q4QeUbh/LIxs3cY9MfjWi175N28Yt7Q7GK5e2jY8ZA6r/nr1qpLp0ULxwso3xErLhjl2HfJzjt0H8+FmSMzvIJDh8HzGQDJ78Angew/AcZp+aJ9GLM9l5qPLp1qoD7nKb138g9AwA6EcAfePtizb2enrahWluIsnk7RINvX7uV5yF6nHXgYFUpbR7iPAkh3gZXdIFB6/3sc9MAZ61V8q9iV98Uu0dWCnAPXFZ2Vy7sutpkjRealzbkscBGYqSRyeSAvf1qNdI1VYpfItJJZC6/6grMAMHj5SQc57ehpyyw/YYhlRIJWLAD5sELjPtkNj8aS2dZJ4IXIxJKqfN0IJAOfzrVRdtyG1cqPNc2sv72GRCGZcONvpkYPcHFW7aaO5W8B5aODfGM/xb0H8iR+NMa/Y8ySyNjIGXIxuGDznuOKYzeakIVRCqs7gIq8lgRzke5x6dsVV5E2iYN34jntL544UjdUOMtk8/wD1jW1aXKahZfaEgVmZGcq3ONoOefov5VymqTW01wzwwJEQTkKWIJz15JP61veFcPp74uY4pyXjihcFfM3LjhsbeScYyKw5/e1NXH3dC6s0E7qk1uwCkeYYThnAPvkA844XsPfMz21pfKiy3U9uAxEaeQrBBk9W3AtgYyQufQVnbrm0ndZIXjlRsEMuCParV7OrwxyMcMy8Dt71sknsQ7p6iPpJs7rYt1FJkhQVyC2VVwQrANggjsK1L9o7fULO78vzEgWJhyQGKop6/UAcVhTmWG4dJkAkiIi5XBGwBecd+OfetFdWubazlh8+RYWUhlVztOcg5HQ//XqLNoq6TMXUdYnubkzwNJEWbJO45wfft/8AXqst7epcsPtcrFDgFXPP+NW4rexnkVRDJFHGCS0TDJ59D15I/A1tweG9KvJEZbuS3t12mUsok/LGCeSvbuTkAVg07miaHWXiMWumlprUzTSN+6lkcheBzwuM/iSKtR+L7RY1MsJEv8QRQQPwOarX/gto7WO90e/tL+N8ZgVwk6MTgJ5bHLcnjGSfSuaRZLO4KTxYZThg3BHt7daak0DSOyktrCaJNt9NbNvzIGg3oQW/2Tk4XnGOSvUZ4lHhy8ubcyaXPZ33zfcin8uQD/aWQLj6ZNUMZBzxzwamgkCkHdjHcVty9UzJS6NFS8sNQ03c95aT24J2hpUIUn0B6H8CagsbgRzSPMWVPJlGQO5jYAdPUj/61bGpzTDUprdnZoon2CNmJUheBkHg9B1qL7RYXCRR3FjCm2QF3twYiyc5UqPl79QoPAp2kkK6uc41w4lHDbTjgHrVqG+eGI8biOct2rYfQdMmnH9nasmGYhY7tPLKjBPLjKn06Dmornw7fwwvm0kdQhYyQ4mQfVkJA/E0KVnqNxuV7i+mNujSP5wh5VJMMMA56Gliv4pUWGS1tiv3ywjCMCcAcptzwB1zzz1JrGaSVEYgHITDZ/nTvOJs4IVx/rM9MHJAzz+AolYFexp5t4r3YWcxAgMoHzEY55zjJ57d6ltkLSQxhI8ylVMu8bVzjlvQDPXHY1nXQe51iRAH2oFU/goH9K2YVWJQigfdwG9hTjG4pOxkS6dc3sck1vGZTbBBN5S7lUv93p75HscA4NVo/OtSVCmKQZSVW65/pWmbplvACdqMSMepBz/MA/UVeXWb3yZI7iV51clSk6iZecZ4cEDpRySWwc0WYMcbtucOWePJOTwajlvPMRCqfMMk454rsGvNAuSkd7pTWiLEMPp8gRnI4OUIK49+DWO1jZLMqwM/2YoMvMoDBsnIwucjp+OeKFLugt2Zntc3CwxyRH5m9qi1DW7lYY41KpIwy5AFbl74a1S7tLSXSLae/h8s820TP5Z3HhgBkH69c8Zrkb+K5h2w3Ebq6k71ZcEH+dZuXY0iu5XN5L5rSEknGOeaa8vmYAyD1JJ4pCuR0/Sk24GSKxuzSyHpcyRrt3HGc9avwajfQRSSQXDJ5g52nHFZvBx7e1SQ5bCHoe1NMTRsweJdWMsTNfT4QELl93B9jWrDqZvomW7nlldjgeY244Hrnt/9eubSNWYjnC9MdPzq3HGFYnzcv1OOP8itoXMpm1Leg3ck+5mmkJ8xiclye5PXNQS3b7Cu5gSBkBv8+lZuZHw4jfrjgHB+hqRY5FJ3Bw4wSpUk4/EVtczJYbpdj787t2ee1OgnWdGZegJyTVvTreOe3dpI43VzwuOMDvkc/rVh7WzghmcRiPdKWUIGIRT2GW5xxgH86xle5orMo/aI/wC5/OitWLQEMSF9btFcqNymNwQfTBGfzorH2sSuUwIdURbdrZY0ZGQZLHDL0Gfcd8VJI0qI0P2gkxny9pU5IHQc9vasexTJdHZIZsboyx4OD0Gfp/SmLc3Ml6xEreY2FLMeg6cmp5mcEsOm3Y6G2VDYXL3hYhQF3rnIGcjPPA64xULXEdtHCIZHMzsXKlmJRs4wOcbT16ZwayJWu4PNt2nDIy7XCvkZyDz+I7+lW0vlljmjks42EkYCAfejcDgjPOOORT576maouL7pj9Qe6kgZMHczbigyeR6j88delUmlnWE2sgA3DeXTjI/p6dqt2t80zCRolmlhdchwCGA4AK555PT/AANQX95c6i27yAAilR5Y4A569fT9Kl7GsI8vutCwG2ljc3E5SQgJgEDKjGDz1PH6VRnXzJx5JyuSSfSo0DzMittRh/Eenp/jWo8IspwskgaZRlHVc57g+h4NKzepo7Qe4mVe5hYqWjU4I7Bfw/yaupNiVbU/Z5VkYRq7McIMjnI9M981WlaOXT7eSORUkziSNIxwB/ETnJzx9eegFTWuiXlwt2ZIXWSJN4QuIyuexB9h61aujCcI2vJm6dUmn0t9PltkMJ+ZnD4ZDnhsY6Enp65rBlt0jsFE2W5JWQMd3+7zx74H156VJb2+o6NqEccjEME3+XDMuSMA4wchh6j29qtHUZpxe2V9DGxh+QGONYyp98cHp0IyeelN2kYpcj93Yy7cXG6QwfaXikjDMVPGB2f05Hf9anMM7xxD7QTISSimQvwT93Hbk9h1NX4v3ciCKBfK8sjAODI/Q8dD+PXtTkSOFLk38EEiTDYilgrRnB/i2nHc/h7U1CyCVfXRHP3kFxPK7BEBU4KhcCkkRY1RFkVuNrL/AHa35NFtU0+JFQvdKT5zJKW3nr+GB6deuawbqGCGUrCzuAuSMYx9f1qHFxOilVhU91PYRpJUmJWUgbSCwJ5/xrRjubudQAFVYMM0gYr8/bgd/wDA9qzJ02W4IMgjbhQ3H51oOklxGlxbyKCq7ZERtoOD2B5PahaIckrbGjZRRsglaUee+3ymLbQWz0Lf56VLeC2WO3trm38ueDJk2fc3DkEMD15PT047Vg2dy8EXlM222nkDPtAJGG6/h7V0MMd7rNtdTI/mJ5imaRmG5OvzcZIyOOmOK0ck1ock6bjO/QiktrW+u7qeNWYlkIeYgOnA+Y4+UA9+DjrU0kdnaWcCF58SRkts/hf+RGTnjnBpbW1t7pIprc3M18qEyxOQzOQTkAkcjGOx6VWgn8rVk+y/Z5VwsYRtzqCOmW+U5HHTH0Pc02FZzbV9i8C8kTxSSXLMFwpwVJ2jdgpyDnC9u2Tzisq5mBtFtS8Rsbk7xFGNuxweSVySD+hB4ro9J8LavqF6bq4kSyt2IJVQ7l+c4AbIIPXLegrrtN8K6Xp7tILd7mdjlp7g7mY+vOcGk9TqoYaVr3MDwPo8ltPcefb7LSUfNcMuzd7DJByOcH3P4d7HZWMrAxksi8ARgMMYxyDkflSJaqZATsP+8parzwRIB5skq4HB8r+WTTbud1OnyK1yWEWccZVoTg9sHH4AGpHvrYgoEwgHdWH/ANeqRMKqG+13IJ6ZjA/UvWVea5bWjNFHcNOR95VXGPr2/ImlqzTRG48mnRRGRVjO0Z+/J/jWJqeuIbcNbgxR9SVY5I79Saw31iKQ75J2BZigiLMQeDjHvx+tW00caZY2moa0cTkjy7BiMAZ+/Jg/XA49/SrUO5DkMNvqWq2ZnhVLWwPMd1cSbQxPTA5Y/Nu7c9aS/MEentZWM5kFvIZLmdhzM4A+b2XHA9APc0mt3zXtvMXVQu2NRg8KynbwPTAY+mDUHhq3uILK61WUBI0iIgLc+bJ6gdwvr2IHvVok2otNkk0uJLp/s8zbX8l0Jc4AA44xnaep79DUF7/Y9pAEeORwV+YSykFSVJx8mAfTmoLaW91S8UxK0jlVLyEkBBz8zN+GOevStK/sNMZJLiZWunwvyglE3DPcHJB49OnvimSZ07abcXcMBt2g85RtbzXbbnHGGJ/iAqqqC1Z4JImXClsHPGc4IPfkE/Q1fRNIubgZsgAieYJFmfK4+b5ctgdfSq92k5uGjLB8hRGwPysuAARn3B/GpmtBxauUJXjEa5BPJwBxVea4iVzE0AjVQQQTlsgc847n+dWHiDFRIpBUEZBAyeTVV7ZzG0jSIRnHzE9epP8An1rE20KYkRUeQQRAMcBpcnH0A/Cs+6mZ9nmRuMc5ZMLx069q0nsZZghjDsDwWCnGahm002q75pmZAPubcf1pAQzJJHDsaJ0YYJUgZ2EZ9B69R71AzM2ZHmmk+XBdjk9ec5zkVMJ0YBAvyx5JyecHGPwB7f7VFynktGm3KuwU4PtWydydiu9yZLmSPdaldu3AgCy9P7wXk/jzSbvskXlHKoQWZ2TcoJ49Pl+vJqw0yeX5fK8fK27hvpx174J6EVBFJIZMFye2V6jpStpoHqQWItcuYZ7WS64IEO9dqgHJYsqjOcdCatOst7utJYIRabvM3pIAQQP72fc8e/1pYPNhf7RIbqRWxtM9oUiyT8pDBuTweowRn0p5lCz8RxCRMOf3jYHfoGqUhtlG/jjhijWWyjKgbQTccngdQGzj3pYJA0ap5VoISoBUjJUDHfBI6duadqciTRRmRbcyNuLY8w+mOfektzHBFHsWHzSo4EYIGfqKdmK6JBMsqw+dLC+M4JQllBA6kjJ/E9ap20en294881xFOQr+XFGmCWKkL6dCQfw71N5jxgFpLZcHOBApx0yOR9KmOqwLa3EQubjzWmDB0hAGzByp+bgEkflQ0NMjkWKa5UzJKHRSfLlPl7M+vTn/AD7UxY40uoyZ7UBAcELvYcHvj1PXPapJ2FxLn7FOqGNfmkY45Gc5wByKZbReSykwWyEgsm/bIPTkHP600hXJ11eN8G51PVbm1hALwCMAbegG4yHAyQOnfsaZBHuuPNYLAA3G3Jbr2/wJFNtbaa8muVttTgXyInmk8lHX5F5bA2jPrj/CnRMfNVlAQbxh37e+OvX0HahIGx4EUxJSJnPQGRsAMPTGMfQk0qzOt6rXlxpcQVwxxbIXxycbkQ/TOe9VI5l83dH5lwoXbgJjnngc81Pbwo19Ck9pp0Adl3yXk7RkAnkkGQdueB9KcloKIx3uJbOZpL3+0JQNqSAyHyxg8fMB6moVO2PDtyP4YzznjgkjFWp1jlu3RIoIoTJ+7jtpUkAUnu6lstj1/wDrU2NnhhRI0S3VztaR1+Zjg555PYggccc1URSFvZ0+wtFeXr20Tk7oIY95YDoSu4Djn7xzXPpbWazD7LLPcMHBA8gIAMjqQ5A7/wCNat88NlYPM0Al2Id0sgJC5IGcDgdQMHNchc+Jb51aC2cQ2u7KxhRj8R0zWc3aRcNUa/iu7hOnpauyvceZkKrfdA78ceorjqc8jyuXkdnY9WY5JptZylzO5cVZWCiiipKCiiigAooooAKKKKACiiigD0XwL4kvRYvYC+nUwEPGglIGMYBxnt0+mBXQXWr6o8Mrvf3OWU7v3p5GDx16e3SvHY5ZIZBJE7I46MpwR+Nd54Z8aS3N1DY38FoZWyI7n7PEDnsCMY+hHfHXrW0JpaGU4Pc6R9X1ExNm9uS2CdxlbOfXr60ses6pFCgj1K8RABhVncYx6c029nRME2dnlmJz9lQbRxwMDGOfT8fSKORPsxla1t3WN1jKlpRuyG9JBj7taLuZkYuLl7k3T3Er3DHDSs5LNjjk9+Bj6cVKzKLOe4ktLaURoXIKFBjco4KFefm69f0xILrTySrac8QHQ29w2c++8Nx+WPfFOSS3urSax854FkVgrSQhlHKHlg2f4Oymh+gLfc5JJNPWRJmt183gnl9w55x82M/UY9q0DbpLHE6OfnAcl02hD1weST+X4U2bwxcSXmbW6srtMDHkzBS3yg/KsgVm64wAeQR2p88M1qEjuY5IHKj5ZVKHPTv9DSjYcrluSC6lld0SS4DSEb4lZyxJ64xu/EgVVnYxlA2cglTuPIPBx+VWIIiyGYtjaQuO5znn9K1ItTv4EULeTlV6K7llx/unI/Srd7aEKyepjxzcDAIIqIxSxXciOEYFdxOwZw4zjOM/dIH0rYS/t5WJu7G3ndsbmQGFup6bMAHn0PSqjixubwvKs9qhAGY283HHHBxnsOo6Vmrl6FRjFExby1UMOcIDnknuPc/hx0qBovOdJoDFbuv3RGHPIPXJYnPFW7uxZWLW1zBdxgbjsLK2OP4WAJOTjC5qW50i6sHLS2ssUSBEaTaSm8qCRu6Zyc4z3qroWpQNoGCAOqEt87PxGvPBzycdP4ajAKXiWTsPMfhQpznOQMf4dauJLtjl6tlGClRkgkEA/h1qvdz7bSQrIAyLnB5xgU7voxadTlrvQ9TM0jJYztGSSpEZ6ZrodLcW9jDZSx7WCHerjn15rl2vrwyFvtM/zHJxIRk10x09pTAPM2sUQO7sWO7aNxPGcZBxgHr7VlTdpNms1dK4jvvvYyJni+UBTncoA4A2njGPpViWC28pInDyXCq2X3hVYnocBTjjtz9fStf6fPZ3YRlOAEKyFWUOGB2kbgOCASPXFW05VGOPmXK/Qcc/iDWqSZm20WLsx6hdz3ILMZpXmkWbaG5IJ+bO1iSTz8vTOOaoXFjO9i8sSP5G3II5AxzgnoD7ZzV4riIY5J7Coopri3lZred4Z5BsEsTFWGfcYNNxaWgk1fUrWKpFoV9KciQSwRkH0O9jj8Y1qC2vCkZilbCISUPOecfyxXRXGpW86zrdxReVcgbzGoRty7sNuAx/Ec5BJ6ZFRReGdJ1CRBBrIgLsFUXkLKCT33JvAHu2Ky1W6NNGc9BezJeiZWYNuyDk8HOa6W+1O48UwwDV5GuPs/CEkBx0/ixk9P1NNPgjW2t4ZrO3jvbdjvWa0nSYE9SBtOc+2Kr2zJATG21WBCkehxTglJkybijRlt7O7VltHmt5VQt5cpEiMQMn5hgr09G57iqc9nexGJYBFctMP3QglBLdOMfeHUcED9DSGb/SI1BwfmBK8ZGKQOIgHjLDaw+79a0cLaIlSW7LGtuTrV3IqkFpWOCuCOe47GswBpATyG/lWnJfJ5gE6xzllK/vQW47YIII/A1UheBnk3DyS0m1DyyhSTwerccDjP07002lqS0mysoeGNsninwXLeY7ZYMQASOCPxHPfmrR0u61DzEsU+1NGu9kh+ZtpJAIThu3pwCM4yKoLa3NtKzXEMsfzFPnUjJUAMPqD1HbNHMrD5WWpWFzD5b4CgYB2LkdcZbG4n5j354znAFQSWEUt4jIFhh25cZLfOB1UY4B44JPfmnp0OTk9QalX0J7mlyphzMlufDtzZvNqMF1Bf2wYb5IZfmQsQMsrgNyW4wD746VHfQzWJjhlililUfMsiFG+uGANOaNJU2uisPRhkVqya7eLKLeWUXtskSL5d2omj5UN0bofpg8daFzRemo7xlqca/mLOu4kMDkZ/U1pKsjXsjBxsUYA9T3reDeGb07prCe0dWGWsrgkEHr8km4D8D3pNQ0jTFs55dM1yOZEG8RTwtG5H90HBDN9AKcZWeqE432Zz94hMgKEenFFtclUZByQen41IsUiRSedBJCFYHfIpCsGztIJA4IUkHvg46GmxIBmRcFHHBFP2iuJwZsaVqN3p0v2m0uGhl6AjB/MHg/jVnV/GV/cy28twbaSaJcBpIFdu397OOn8OKyY3Qxg54rN1STfNsxyVBJqJJPUuLexuR+NtWiaRopYFM3+sCWsY3n1Py1Xv8AX01OIpPpullmXDSJaIJPruxnP4/lXNBXT1A7ZpQMkZbkc1Ct2KbZ1mmazpFlEsJ8OaRMgXnzI8uT67nLHJ9sVrLrvhO5Yfa/CkUIAx/o7bc/gu2uAzgZ706K4aMN6470ckewuaR2Qfwjc3ggGhG2Q5PmzXM7KPfark/rW7f2HhTStVjivtF37bdGiktndSQxbO4bsHGB1Oee9ecQzTSSxgSbcEYbAyv/ANau5+JcguL60cMx2wKSdw2n5mwRj8famopzt5Bd2uEviLwrDCY18PQykEsWaCPd/wChEHt1NUV8T+H/ADA0Phizk3cFXhiK9eTyhIPbjiuRZkXB5JdcjPSq3nyJPhImZj0VF/z3qpJLoC1Ohkv7e51Caa3tILKI4AghG1R3OAKrXV4GhdiAQvI44NZUdw0U0iSja5wcZyBWzpcNvqN7FbXTqtuGLSZbAKgbjk9hwRmk5dQtqU2UTMZX2MzncSy5JJ9TRUCXEmxdsgC44Ht+dFRysOaRh7Q1ynlSk5Oct/COpOKtLiVmmHlxhB8yr1Y1FDcxRwSq1tG5eXOR94r6Z7etFpcW8GpRSXMW6ADDKSeAfT6cVild2Mp3aem34jtplUyxxgKSNx3DOO+B3p8gjWVQ0i+7HJH14qC7uYG1C4awUrbvj5WHTjn9aQM8cTxvsxnPJ6H2xU2Eo6JsuzLZB4FtWleFowWZhghsnI6DIz/k1nNJGpcbgW3ZD85I96unfMCWZmHTIHXtVG9hjDfISGJ4G3g/Sm00VCzdi350U0KlECBR+8OOfw/z/wDWns7mSOSGO2dss+9TtVjuHbn8azE3CMj5lABJx+tOML/u2hkYrENxdRgg57j8RRezuHs4u6Z28Mqazq11HOUjaRC5lZSXhPGAoPuRnPbODUN6s1pJ5b3b7poQCwYgyDtkZ9c4rCAJhjYSs1xIglkaX5cEE4wfTABz+FWdZa61CLT5buRRb4CLKoJYqeufbjI+pq1LSxwOh7yV9C0Yo38k3XnMlsnmERvtkyOByegP07dqqiS2jika0LfvDnY7bnGc9SOuD3AqC+u545GiuSkrybQ1yEO4r6cHB7H8BzUKwShX8iRWRQQz+oz0zST6ov2fu2bNNbqZ4LeDK7A2zLLh1HXAOfXPbP8AKrtzLJNuimwr27LkucZJH8Q+mATxmqUNvPstFSdXEwOyL7xDHBJ9iaZc28lxdSu+955HxIzkgZGADnpVanO4xvuW/wC0dl3F9hk8pEYLtd/3Z4PGcjAGcdB1OK5/UYpbW8e1cFJQSGZnznODnr0Iwc+9X2t18xbGV1t5GBKs6na2eAGP8PfnHerln4F17Xp/Pj0+SCAbQJ7tzGCAOMBhuYehAPAqZXdjtw0EtUYkiOT5ksrSRsCpJ7n2BrUisopLKExyGOOVSWBIchsgdCVySBnC5rudO+HNoiLHqmqPLgFjFbgBQccneRn8NvP1ruNI0rS9LANnHBC7Lnecs+Pdm5B5PGAaajobSpyk0eaw+ENW1HRbK2tLSZoIxkyXUezYcndyfmA5yMZq/pXwwuRP9qu9Zt7dfulIhuIHHRm246dcGvTZN0iBmnRgeAfMDf1pEt/MH3n+ioc07a3HHDxje+tzD07wboFtKJYjJdTqwbzpbg5yO+F2j9K2E0nToD+4tLaLJyRCuxWPqwUgE+55qO8ihtIme4iuFjHWQxOF/wC+iuP1rlr/AMUaNaA/Zy9y56bQQPxJ/pmi9zWNOMdlY7OaOCKP9/DBGh7t8g/MmqTz6em4eYGKjpE9eb3Xiq9unKW+IQxwqxdT+NWrbUFsrDdJOgOMYPXjPFWordg21sdRdeIbOEZWOXAPJdxnH4CuUvfGF7KGWOUQoegjGD+fWlstD1bWoI7mG2KWBba93MQkSqT975sbh9M1tWHhLw1BNGLu+uL+baWP2fCwnHbJG78RT5ewXtuccmt3jl/MkkkXh0JcnB5z/IVuaJ4b8Ra4srQwPbRGMES3KFFbkHgkZ7ZyB2rsTq+k6e0FtaaHYxxgkr5i+ZtPGcE9CcfpWTrXinUr3U/sPnGK1GRtj+UNimk2JySIre2tvCOoSqsovb/y9ou5ABFAxOQUQZJ4wdxP0HNVNV1KXV3Ek7HzJIsAg9SD/wDWxWQ1wralL9om+RZPmZj27ZrrdMsoNMsReygx3cYc2gYfdB6MQehznHtz6GtHFKKZnd3Hx21nbWET6hH5k8ce6aBzlN2SuCCMk/NyCccHOaqTXU+rQRxWw/eSOLeONflUKRhVAHAHT261kXr3F9qshxNM0qhyI1Lk9icD2Ga6LTrZtEsj5pAu3wrYIPlLjJGeRu9T04I5odrXBXJ7i9W2jg0+yCrBHKseQuDIemT3yc5q1cachEi6g7pG7BhFFy/APBP8PX3PPY1VsVe1s5718fa7mMvCv/PNccMPdv5fWmwPPd2C3N2ywRCTIkYE7wMfdA6/oPekNkEsumabxb6bEXdShMjvJgDpgMxxxnnrSPNC+noYCyqJQQSfmTg5BPfHFT319Yw2wnXT4pW3fK0rMevcgEA8ZqqstqYwEUIsg8xo933W5+7nt8rDvSktBJ6mU4bazq3Q7cfXNVp1VUUfMH53ZOM+laT+RIpG0LwTuJNVikbsziN3YLk/LwB0z+ZFYWNzN865RyI5H8qLkgHbkZ/qTUEt2HVvNO3+7xmtCQGNXG1VHRssBnv61VntdtujlV2yAshyDuAJB6e4NLoMz7U28dwTOpeJ0Kg5wyZ43Ad8VZuImcosjAMm3n+8B0/DHNRTWkDSqIUZWJGC3aiynja3dLggTwj92WH+uTOShOfvDOQfqPSqixSQq26uzcgowwV9+xzjg/8A16h8vLkZjyODknj2IHNXYZUdDJbkLnOcgdPcfz9OtUo7eV7y7VEbeyqTs7HaOmCdwrTYgW3i8uV2iaMDJVjDKJDg9tucnnB59KDLHAofMO8DBDQqc/pUNvbzDG+GyYiPmVcI6+g5xuPHvU89yVlEMkt5C7lioaLcCwwcFiRg456GkvMplC6mWWe3kM9qVK5kLWyk9RwAV9OlPSSFQiC4zCEwWWAA47cZ49T9ahupwZ4/NvrkRFd3+rDlenABcfzFWhIECot5M0YwSzrg9P7u4gH8e1KLB7CwXJS5V7e8kTJ+WRI8OoHbg+nvV2TXI1tZrVdU1CW5knU+c68qgDApneTyWGfp0qil3FdSGSC7nYRORlk2kkY9/wCtV2v7sTOkmoTLE7ZMTgOr4OepOR0HaqdrCTLqiQu4WGbaW/5adD+n+c1FJAwu42eKJDtJCl/mGBnlc5Gcd6dNaCaT7UlnfvFGpZljOSQMchtmOvt6VDbSrdBLu2t3QMhG2dwWHBBxwMnPpTv0FYVtXls2leKfSo3CsjKlmgchh8ykiPnIJBycdakihZ+VV2ZR/GcY757D/JqS3WXe7LPBaM5LKBGFc8dCyqTz9ceuKJFg3q80s8uRltxCEnPHJznt/nmpjoxy2IooXaTZO8KRbD8seHIJGO3H45zSpp1o6SBdON+yICodpMgkgcCMrxjHXPQ+vFbS7K4ijuI5IGhVJiMXbfMqnkHBG5h3yorQiYmJlLFS2TjblT+uabSaErohhhFvF5n2VYBIclFbHGPclvzrL1GOI3el+W00kvnYGOM52j7vOeeOvOa3vswmQ/I8wH3gSTjHX7oB/OqF0su6KMXKwIHXdCjY8xQfReD36kUr2H5mJ40uJreGO1/d7J2LHYysABg44yOpHQ8Yri66Txsgj1qJBHIoFuuC5B3ctyMdu3Pp+Nc3WM3eRrBWQUUUVJQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB1/hbXpLnUIrDVZbm4tsM0YV1DB+D8zEElSFx69MY79hI+nm0aNZ7pM4bCwrIC4/4GvGCf/r15hoayPrlmkcgiZpAC5QsFB68Dnpn/AOt1r0M6fdpKRDJbXAwcMkgUH8Hwa1hLTUymtS1FFCZI2gvVZJNyF5onjCjBBJADYH0J6j8LSaLqnl+bBbLdR5IDW0yP0GThc7vTt3FZlvbahFahpLC6WPJIYxNtPfrjHeql7eqVTLqyhyDGx4RsDJ9sgL+XtWvN2ZnbXYnkhls5mt51KSxna6HqD6VatNX1OwQx21xIkHOE4Kc9Rg8d+frSadq88zhGjju44o8slyglCqO4J5H4Grcl1pl2qRSQNYxoxYGAtJyQASVdv9kdCOh69mnfSwnoUodUgkkd7rT7aVTgERhoOMgnHlFQCcHnHetSNtEuPNZmvrUmTEaLtmRU445Ctnr1J/GsWe0gjmSOG8jnjmPEkcb4QbiPmyowe/8AWrEtlcWsKyXFtKkZOFkZfkP+63RvwJpaD1HzWAIMkd7aFOD88nlnJ7HdgZHPcjjrUMuj6mOtjOyHBDIpdSMZ+8uQePemyxfuTOrcKyrj65/wNIl1JbbZYJXjdc4ZCQQD1xiiz7hoUpi6ttPT9KsWd9dWkWLe5lhydxMbleQeOR+FbC6/JcFJNUsbXUhIpJaWIRynkj/WLg9uvNNu4NBuIGa0mvLWUNkRzhZA43fwlQNuB65+veqT7omy6My47l1nlnysksqssnnKJN4bG7O7PPHXqMnHWoJrKwuIhHi4hYptLLICGYD0I6Hjvxjoc1pnRLhzi0lt735SwW3lBk+nlnEhOOeFrG1GJ44irI6TxndsYEMCD0IPQ0nysa5kYa6UInZrlJ/s6jjysFs/l/StFi63JQB9qkL+Hb9Ksx6mjCRlX5lACh+QR0x9eKtWVxZSRtHcWrsCP9ZFLscdeOQQevUgnjGaSjy6oblzaMR4wMoBj5VDDPPQd/qKj3IrOWgUlshWX5Cuc9AOOp7g9MDFSXnli8eS3WTypXGxcFyOOpPqT+pp0YfDCSNkOFbDKRwRwaFZ6A3bUa0a5VI3OTnKyqcqMnbyuQcgcnAAPtzRbaVf3MCXMNlcTRucK0SGTkHvtzjpnB5x9akCt9rJXgFFGffLf4irlveXVlcGa3lkjl/vo5Un646/Q02pdGGnU5e6LrcGJiQyMQw7g/0rWt5+ACeR0rcTxT9rPlazZW+qQYIb7QgSYem2VACO3UE471Zh0vw5fK81nfyaaVUsIdQwytgdpVOFH+9k1F2nqirJrQw4NWudPLm1neJpiA7Rsyt8uccgjI5PWrh1SN7E2U1tBJb5JVSgDIeeVZcMDyepI5xjHFVNV0a80+ZXnhAiddySIQyspPDZHQemarIxWNS4+U/mKtJPUhtrQu/Z7KUK8DmKVV4WQ5ViFAxuHIJOT0x0HHWqiadcWoE/lsYHbbvyGQkgtgMMjPB46jBzio4T8xyM5PFSyM2wBw4R2BDEcZGefcjj8/eq5X3FdFe6XaocgnAOPzqK1DvKCqg8b3yccfTv1FXGicElfLeEksy7ArNx0GPlAz7d+/AqeGyhurmWO1kZHSJBFHOgjeVicEAE44AzgFjggDPWnzW3Fy32K73cltKskUrJKvzKynBB6cH8TV638Q3cdsYrpo7u23bzHejzAD6gnleg6EVlTIWkVJCAxRZNu4EgHpnHf27EVga/duqrax5x95z/ACH9fypScbXHFO9jtHbR52DQedYsx5RwZYlAHr97k/XrUcdjLJC8sKCeFeGeI7scZJK43Ae5AHB9K88tNYvLPaqvujHGxq7Cwu1uoYbmMlT1GDgqfr9azhJS0Rc423LwHzAq2UI7HrRqkXk3rRx5BTCMo4JIABP58/jUVzNcTzrLJceYeSQFG70w7Yyx4BGScZHrWlNFpuobZFu7u0ujksbrE0bEnu6AMPX/AFZHvWl2nqibLZGFFC25gznHQ89+1S+aI8LGx6E7u4PrVjWND1LREcyGKaF8MJ4XDoQenI5HtkDNU8n7GoRTvbjrkY+orSEk9iJRa3JrbU5s7xM0bMpUlGKkj046ium/tizu7OOW/wBDtZ42byWngPkyZ4PIUYzwf16ZriI7eRI3ZiFkUg7Seo71sOuNFsmWWUPJLJvj5C8BMEds8kflWU4prYqDszcew8M3F00drqc2nBpDsS6j3ptx2YHI5z96uabQ764nnlhktpYkYqm24UM4BxnacNjp1HfvVHWjL/aBLBlISNct3wij8uOv41nZaWcyeaFOQBhsAVk9EarU6RvCHiWRA0WlOxK5GHQn8s5qo/hPxPAD52i3mSN3yRFsD8M8UyDT7sMPszBm/wBiQbh+HWrsR1zSTvSe4Rz8pUZPyn36Y9qm7fUehmz6RqNqx+0afeQjH/LSBh+uKqtleGwv1rfi1jVoXWT7ZK8kUgkXzXdyGwRkbsjvz0zWwPHOsp5SzWzSICCyxxod5z3BXv0OPwxRdi0ONs7kiTEcYlIPRVJP6V3GuThjpttMDG0Glwkh+AOWB59RnB+lNf4kX9rLKU0eK3jYggSKwKttx1GO4rm9WvtR8SPFPfFJVXIUrEq8Enrjn86uEmpczJko2sQXdvDGGkEymPPGx8lh7VDZWlxJIsUlu0BfDpLcDy124JB3HA57etaFu06r5UCSpGECNg4yB6c1kXbNGFJQ7CT8rAjJ/wAa0k76kx7Hd6HH4cl0GSx1uKPzg7bJgCzAEfwlOmD2zg5/KppNlp2na/IiXNjeWhjwv2xHTPGGzxjgFjyDxnvisW3O23jUDACj+VSCfZwrEfQ1Hs7oanbQy7swx3k6LPDMqyMBLkNv5+9nvnrmir5MhYkSADP92itrrsPnOQlR5djpGI2C9Q33vepWKHC4cqRlztBKn2NQG73oSzEvxwvGachlufLLbY1J27uwPPXFefa4W01FhgkleUQo7gAt8oyQAev096H8yRlWbIVOPSrv2WSxLqbgyNtzxnBH4EGla7uEsYS0MfmREjcCwZlPqCMH8/601Yhyu7rUjE8FxaCIbxsGM8Z9veoomie4jiKhvLGAoHJY9T71BcXkksZVYBHEWJVQM8n3qa0tYZFiWVXjI5Z1bLH2wOnc9/pRuNx5Y6kCuzGQJKyJkkYBwx9KuxxSx7Y5G8o4B+YEnGOgHp0qdtOgmvxBY3Q+ziMSfOTv7fL0HPPp2qO4uJLG4dmhG9sgK4GVyeo24/T/AOtS1RKnGT5VuTxta29s7SKJnV2RfN+b5OMYB46+nTFSMbS40yN4PMeVMr5b5Kx5JPB6YPp71m4kC/fDSDLHBG0AHkk/j+NdFo3hXVtThM2l6fcskqYLyMIkJP8AEGf73f7uafQmdNy16mNBPFeTC1v5yiINsbAcIR6gdadbO0EctvE2UOcNjP4+1egaZ8FL+5kEl/qEYLD7trC0g/76OAPyrt9M+Gdnpibk06WZ8cEsgP1BBBH500u5UqN9Oh4tZ6PqM9sk1orBS3BbCqfxPFdhb+Gda1HUVurjCbwA8dvbs6sMYOSwC+/GefpXrNtp9zbFvJ0Z4+xdAoJ/LrT3hviQDYygAZIaPNUnYaw8L3krs5bT9CtNMKTiJUvFVU+0TRs8uAMfeAOPfbjNaBZH377jcB1YxTAH6fJzVq6SONwZ7V0J7lWAH/jtZ876fHgFohuGRi55x+VO7NlFJWQ9bm1jYA3cIP8AtLL/AFShrqFlLpc27duGwfyIFUp7iKJdypP5fByjqc/mMGqEjSXABWdooudxBDMPxxj9KaTYNpGtdagLK2NxK0flDuhD/wAvwrKj183gL264TdtHmdz9B/jUF/ZQzSGDzNyLCGnO7k/OoGfxasz7HeWz+XYYm/uR4LNk+g71pGK6mbk3sXtUvZoLSaOeXcxAIUHAwfUD6GlXSIHvLy3S3JUTPHFGepwcAc9+K1LXwLdSH+0fEl4lnacObeLmWTnpnovXtnr261qah4qsLCa5l0vT7aG5cF3uBGpkYkknJ+vrmna+wm7bs5/TfAs4lS9uE/s61PA8w5lJI/hHQfiePSr2qa5pPhu4hOm6VbyXJUI1zKPMfaMLx2B45xTNC1i81p5pbq5Z280Ehn+6CD0HbkD2/WuZ8X27Wt9GG5BJwf5VL0aTKTOg1TXb/V9Q+yT3JaBArbMAAHt069axLhnsdSWaByFIOF7ZIIqJTNFNbXW2Vxd267AiFizLkMABycAqfxrah01otUtJtaK20CES/Z8qznB4D54AOOg3HHYZzWysrGWtx2sQKmmxXMeciRXIPUE9v5is+5tL7VbuGXT7ZpZFAR9vAGQMEk8Cuivb/RZEbbpVtLEy4KmFSGwO3Gc/TFXCdlokUBMcW0bUUbQB6ADpWa90vczo9GsNAQXLQx3OpqwMk7HcsZPZFPGRwN2M9a5y61aQzu7FpGQnCk9cE8ZrTvLoQ3Twyh3WQBRjkg9R+HNT6XoUNreLd6wi7mBaO0fBIb/bAOPfHvz0INruS/MIEOmW3nK486YfIducw5yPpnhvpirWnWj6nOzTuwtFQF5M9W5G0epx+WOaZevJfXoldyi/M0j7d21Rg9M/h+IrSjlhS3VLVdtuM7E69epPqT60N6XEtxu4XGrSTygJaRJnb2OeFQe3H5CoLh7i/uGSWULEsLAu5wsYyMf/AFgKJI5jbWmmgAXBVTIST98qMg+mAOfTBqrqEslzcR2NjG3lbjnnlztILEnp/IUJBe42W80+Nls/sMlwE53Suy5I4+6p6c9Dn6024lgNwXECxsqBXRScFW6Ec8YbjHuKs+Ra28wD/wCkXG0twxCDPGOBk+ucj/GK6BkYMkLBtpQlCAu3GSceuWx9Ppy20KzKc0xn82Z0JySM4wN3Xr+ZqizOEAM6IrjoG+br39K07hRG/lsomiIDAFCje2fmI/8AHR9KY1y9vGcQSNGeisUKjjjkLn9a5mjoTMzyg6xlyVjHJZEJY8/zqubWckNsZyCMdTx19KuIbq5nAtS0JkOCQzlV/MkY9aQ2F65Be2XzRyHMEaEn1DYzj8akq5mSCVyYwsSyu/3mYAgc5+Un6du1V20+4w4lfOT0KEZ/EjH611sFheTRFbhBLnjlmbn6802fTZ2DKyqhJ64OfzI/pRYLnFvcSQ3EarF5jnCsiFdz46dCfm9/f89GL7PPA8qIhMm0fMcc+hP8Lc9OnfpWu3hGG5fdJMsbHq+N36AirD+FkEscqTyfaCMSyhtgnX/aBDZ7c+1VFvqS7GFLbLCyeadoOPlnU5K+zjkjjtgVn3cAOpWMkUJLBmP7mUMAMenJB+prtrfw/atGwivJ40zynmB0b2I24/PPSifQtOiVZZbZhuzhxuG78Bx+QFXewjko7WZZHwbmLGMhIyfxPIpRbSgnabhWJ++qnOfcf/Xrr7XTLNGLLbyy5Od72rNgc/7NaRt4LU7SqQOcFVkWOInp0BIpXaA8s022uZobyCAysn2luiklunXFbUWh6l5KiO3uiSPumAqM89816B9qkWMCV3Xrg+WzD80BpRLNIuY2kcAc+Wq/n85XFF3YDgT4X1SUMr6fMSVKgtOiY496dpHgrULW1RJbWFWG4fvLncuCT12gHPPUV3ix3TcnLDoVkIU/pvpTb3WV8lraDj5t8ZkJPsVKcdO1F2ByY8FyllaSW1R1ORs3vj8DVk+D8f629OG6rHGI8+/cfpXSPDOpxNcTEnnMIjI/JskVIbVWGXLTDHTzGjYe/wAopJ2A5mx8A6bZs5E9zJuYuyM6+nsua2U0DRrQK72tsu3oZT/8Ueatmz2tuUK3IK+ezSFfpuY1JDaJC2YikTNxmONV/pQxkCw6a7GRIIHf+9FCJD+YBNO3yKNsUNyw9Bxj/vrFWyBtw8hYjvuP/svFDNIVURusZ6nMe7+RFKwXPBPi5KJPF0AJXelkiugYEodznBwTg4IP0Irgq2PFOtnxF4mv9V27Vnk+QFcHYoCrkZPO1RnnrmsesmarYKKKKBhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAHR+CbA3+vELndDC0i4OMnIXn/vqvRE0W/B3RsIn7YkwfwrzTwhevZeKbFllMYlfyWOCRh+BuAIJAJB/DODjFe0mwncNJHcG4QdfJ2/+gkFx+IqokSMPOs2uA+rXMe3ooY8fgatyX7X8Xl6hYQagmAJJTB84H1HT9Ks/Yme2kQSN5bHcyNMcH8Cf5VSk06Ip8si7P7qqxP6jH61dybC2/h3R57i4fRr2WC5aMqbe4GBz1xwDj25rPuvDGuWuQ9l5yj/AJaQSKy/zyPxFXf7Mt2OGaV0/u7Aufxycfka17S+itYltYMuyqMJM4kZPbIC/lTT7CaODZZIGAlR4nPRWGDj1rU07UZ7Rv3c8qHa+CjH+6T2/rW7dhLtibqO0lXBXaUwFJ+h69OvpWK+l2cYZixEgPy7ZDtC4IIxg8nPXPHpT5nsKwjauboEXNvbTAn5iYgjNjplkwT19aQwaZNatMftMBV1RgpWQfNuIIBwf4fXv+FVYtOZ2Cpe2q56iZmTHJ77SPerEun31vp1wZbWVx5kTxvCDJGy7X3NlcjHK8nHX600xMlGnxTWdsLW9t5JlBQwu3lEAsSMFsAn5j0P0zVK5tbmzm2XMEkMmOFkUqT9KoG4Hl7gwwR2NWNO1S8sovIguXFqefIPKEnvtPHr+dXe2iJt1Zau1EcFk3Klrctkdz5jj+lI2ovNbxw3QS5hj4CzKGwOMgHqBxjgjvV19S03UEjS+szD5SFY2siE6knBDZyNxJ4x1qC60RjA82myNqFsmC7xRkPH/vJyR9Rke9JvuNeRnNodg0C3FrJ5G9ipgmYsm7IJKsBkAAjhv++jUy6PPBa/aGhcjdtV0+dCMEnDDIPT1qO5ONItjG4LLNMMZ74j6/57U20ubhFXE0sQJ3YVyATgjkdCcHH4kURTtoEmr6jlZkAx37VYgvltyGmhWeMgp5brkDIOCP7vPJK4PvVmCfTry4VNQSS2DAgT24BGd3Vo8dAP7pH0NTXnh65WNpbGa21C3T53aCVSUQdSyk5BA7DNVpsydd0QxR6fcEYee1YknzCRJGOThSMBgBwM5Y9eDU02kXiQmZI/tEA6zW7eYg+pH3f+BYqgsfyqBkHFS2t5PZ3Hm28skcyj5XjYqR+I7fXIolHsNPuNsIgb23JQSL5qkgjIxnp+VIqCNSEZl7cVeOpRmbzbixhlm2jM8TCCVm65O3KHrjJUk4zweKsQaEbm1MmmXiXcmWxZylY7gAHjA3EPxgnac+gqL63ZVuxh/IqmJol8s/eCqNp57qQQencVPM0c6SOYCqojPutY1XBO1V3KflK7icnIPz4GAAKfNZ3UMrRXNtJDIvJWRCjY9cHBp6xSraXLx7NmwLLu64LAjH4gVdk9UTdrcywUZilsXwdzbZkEbYDcY5IYkYPykn27lk0UwlmhliaOVX27JAQRjGRj86leNSvzAMp42kZH5VPZSW6Oxu7dbtCmzbIxBReMbSuMEYAGcgDIxg4qrSXmK6ZBCVjVUyMjNPkVSwJwQRtwa0rPRNP1S4WKz1KW2mKfIl3GHViAS3zrjHAJ5XtjNUprK7gRXubV4kVBJuJBXaWKg5UkckHHPPBGQQS1NPRi5XuivMN0Uq8qXTazgAsBz0JzjqaydW1GxvYIUbTLeK5iQK8kS7fN/wBo4OAfUAdcmtefY0bBmyp4471yt+YxKfLXAXjFZzSuXDYztQFqQogiZNvUscbq3fDLE2MgwQokIXJz2Fc3c/MM11Ohxm102FSisHcSSEZ8wKSv3cnaSFBxnHLVjHSehpJXjZmxdvNDcBbhWCtFGY+P4SikY/En8aiWaN2Zdw3DgKOpqIi51K9AN5FJLsCRxyYhZVUKiL82FJIA6Ej3qn9lktZomnjmidl3/vEK5HqM9R710RqK1jGUNbmgghW8DcxXKYKuhKsvccj86vtcGW8iuL1IrtFyGUqI2Ye7qNx69/as0gG7lZWEkYkOxx0wDxyO2MVOhLK2evp3q+RS1J5nEvf2fpN7p8cn2/7HdKVEkV4B5YJOMiVf4enBGe54BIjAmuPD9j5cSzQieZlkhdXbbtXO5QdygbCcnHGKqCYxyADjsSR0q1HJ5ZzFujZ0ZS8ZKMQeoJHXPvxUypytoxqcVujntYkaXUJGEbRREgpGf4BgYHvxjnvWbLtZQTkBTk4r0X7dY6g0cWvafb3Nuh3Pc20SwXA/hHKYVhkrxt4pU03wHe2v9m7dYtDOfMF5cJG3kMAQFLLxt/T3HUc81JaNG8WnqmcDBq0tpIjRbTtHAYfqa108cagQY9kZRuobJ/r/ACq3efDDW5JGfRXi1KAcrIksSs474Acj8ie30rlL/StS0efZqVhc2rA4AmiZA2PTI56j86z5rF8qN1fGFyk25oYxETgoinJ/E/QV0Fn4hs7lIhIiAMSCZeQo9+D7+1eczTqZEOAV7qOM1Is2843YGOA3Yf1q1NpkuKZ7RaT208COYklRvumE9ccduf0pTZ6VO21keMqOok5H0/XtXklhfNZusyEkEYdQSBjPQ+tdQnjG1tz5a2vnISC7knP4fT/63HWtFNdTNwfQ7I6NYgjZNI6L1V2DLj3GRxTE0axKv5gWXthIlUH8iM/niuXPjC3M4RLeQx7c7/Nz+hH9avJ4msmUPPITEOhL4I6fWtE4shxkYVyiw3U8SLtCSMuCOeDjmqzBg+cnbj8qfc38WoXE0giaF3bgxEAL74IwSeO47+taVvob38Mj6ddpcMDxbzIIZiOAMZYo3JPRvTucAlUjew1B7mbvoq0+ha0rsraNfZBwcQEj8wMH8KKOaIuSXY4BNyvv2bcHPWnz3EknO/LSEHAboBVcrgHOAQegpqHDHaob2IrgN7IupcxxM5J3k5AC8VYtblcOj8+YOCc8ew/lVKO2STHzKrHtzmr8dncTFLdUlaYjCRqhJ69MdfSmjOUYsgd/3g85CiNjAjGBxxn9KWVk3swD4HRlAz7Z/wD111tn8PNf1ONPNijtVAPzXMm0n0+Xlh+KjpXYaN8OtMsFP9p3Et+c58tIxEn0JB3H65H0qkmGvQ880DRtU1Ig2MEzZbZvI2oD2+c8A+2c+ld3YfCqScq+rX0zkncwiYDB/wB5ue3938a72za1soRDa20FvCAABEmD+PHP41aF8rj93NFGQf41/wAaqyQKmr83Uz9K8G+HdHkWW10xJZl5E053kH1H90/QiuhaRicokSP6hMn8yCaz1nlL4E8Tt1BIH+fyphnux1eNh/s5z+lMs0jcXDDEhLD1I3VGXDfdVSemWjxVJpBs8ySaRCOd25Rj80qnNqVvIq+RdXE7MMjY0ZQ/js/lTuBrtZxyIxMcL+pKAVl3Y06zQvNHbr7BBz+VYk/iW4F+bFpvMXyQ7Yj2mNiSMHrjp685HTpWbd6jFbwMjyKzkkc8807MTdi/L4jhYhbK1VEQ88BAR9etc54imuJ9NnlM7O+ElUf7SuMkemF3fgT61QuNXV4XQDgkjI7Vt+HPCes+KUdpFe3sGhdTNKm0NuQgFcjnHB4496pWDXcl/wCEmt5oI12MXAAZFHTpyP1p0cGr+JGNrpuntDBLjMr54HfnoOv6V0mm+G9D8PW5/tO7F9cAciFAqAjr1PNM1rxfJHp5t7FFtrMrtwoyTgcAn3GPypvUi6RYsNC0DwtayJeTi/u5UCSxqQw+8D/MDr6dKz5/HI0ye2i0q1gtrQPuZI13FwDzkn2rkF1OWdjulzkgqv5VSbEtywABxj3x6Vdra7k3bOr1PWpb+2W5leRmkAJBbJHt+FcrdXTE4JJO4jA9K05o1tvKtGk8x3jGFRckbxkADucMBjua0bHwDf3EhutYmTS7XbuZPvTMPZR93Pqef9k0XS0QkurIPCNxHDrE1uVbLwjaqjJZlPAAHXqeBXUa/othqJjfU737EisDjbufBIGMduSBzyPSltza6fbJYaakkFuOQoc73J/idhySccjOPbgVj+JpSmizsqqHRd4x/Dt5/pUy13KTNl9TstEsIrHSVkWGMt+9Z8uxOOScZHQenQVxGvXTXFyrEDao4A/P/GtiWJXCYYbWQOT/ADqsPDF5qAE8khtYs/IZI8+aOckAkcAHIPAPQE54Yl3HaDKLu0mFyeIdojz0Oc5/HGK6GCaRrcW/lyPdJGFeNFyQw4Oew57nAp2kR6bo0QhtrfzrljkyXGGKn8AB2B4/M4qd9UntrSVj5aIR5ZCqBsy3bH1olqFyK20+Cy+1X0+6W+KloUA+WBsDgnu3GMjp29Tg3Wo7r+EyMAM8t7ZFXry/MMauxA5xyevFS2WhtblNR1FVDDmCArn/AIEw/kOvr0wWu7EzRktXj0SVM4aYbyO4x0H5Z/OqdjKLa0SYlPMAPkqwz0OC359Pz7Vea7UWss07gJnBJ98f41mN5l1JGgThABEq+mSMf59KcdbikiK0klis5pzjdI5gDZ555Y4+mB+NTSNHZxIFVvNdQ7yMfuqSDjH0wSfwqxN5TK0ECqUtXLuTyJGABPp9B6dKLTT21yyluZLiW2GS3nxrnaQQc/pjpik5XGo2Fmls9KG6dRJdyrvji3qAg7F8/oPzrAWS5cytLdnLMDsXaf5YA6VX1zT9V1DWJTFPc7cBVeHADjHXJUEnnv6Y7ZrJh0W5tWMX2nUw2STIskPP1JXNSmirHQLIobZ5gLe7KP6mrZheG2WRinzMejZI4B5GOP8APpWYuhNt3SajNKDjdumdTn/gKr/OrS6BpcCL58cjzY3I0skjDZ0H3nIP5enpymxpDhf26uqNKmSdvLY59+eKlGr2a3DWY8k3MblXCP5pB6bdqFiCCDS/2RpxwW0y2GOjfZU/qtaSgm1ULJKQn3s7QgJ6YA6UhorrK+fls5ixGQyWcg/VlUfrUrXF4iI8lhd8kgSzyxoD04wCx4+lSxYYBxCzsONyt/TFTGZypRpoUPVAQDg8epA6UtQKwuZXGG8kA9hKzn9VT+tJJ57F5PMXywTtZomJC54y2/Hf071ZbUPJBje5kZgcMBlcfpiozfqQVjNwxPUnAz+VAFcM+75pWfPGFVB+u3P61MNphZZPtG/+H/S5F/Ndwz+VWFkldQEhn3dCrM2D71AbgxoQNsbD+IRh/wCZoAgk02Ey4kskdx/EyeZn8ef51oW0LozEwth+X6KWPTqfb2qpFqF2ZQzSCSMDBAQA49eB1qzHfTCMkRu3qSTigCYQSA4CJ+ZP/wBaphFKF42of9lRn881VTUZCCGRQf7xbj9KT7fyRJ8ynrhM4+h60rDLyRfJtkYyZORkjI/Lr+NIzQKCGVMD+9j+tZQEbSFt87KTwCP/AK9SeZAgx5byH3cf0p2Av+da7SuV2+m3ila6iYgBmY9sVm/bLfYVMAHGcqfmFNS8UT4USHsA2Cent/hRYLmuHGOARnuelU7i8aLBaWFB15ySfwpQZQ3K4XvvIqvIbREJjgid2JG5l25B68hTSAeL9GAIRpWAwGWI8fmagv8AWLiw0m9vjZuy29vJMAXC52qTjp7U+KUQqBHE2SMlA2QD2CkjJH1ya5v4jyXD+ANUEsTIMRYyP+mqdaTGlc8BooorI2CiiigAooooAKKKKACiiigAooooAKKKKACiiigAr6S+y7eGST+X86+ba+oXkJj3BCQf72AfyNVEiRmmJRyEB+p//XUIjwTnhe20Vck85nIQQgYGCWI+oxt6/jj3qNop1j/fHaM43hMA+wPIP4UxFaSyEqfu5w5PBQ/I3+fxNZM1qqsNgQMhzkjJX/CtxlBxg8juabLMzoscs7MifdTPApiOekTH33GfUnGagkhLKWBG3p1ranktoW+eREHTLcVDHH9tBa0hknRTtLRqSoPpnpmgDB+zNKwV3jiU8bnJ/oDUxt77Ti0thdzS7RmR7b7q/Xk/+PAfSr01tywZGVgxUoSMj64/Kq7xQRAFi6MRncEBGf8AvofnQmFiL+1orxlTVtJtrx8583PkyED1Yf4Vn3trpMVtJPZ6gUckbLedSSBnkblBB7EE+nvkWcxrK0ssRkkbnLykgn1IH+NRy3J3Fore3iOc4VN+P++y1VcmxQeG5tUR7qB4g5+UsOGHsfwog1BrW5kSNivmx7C4OCuGVuD6/L+pqxJqVybVrXEBgb76Laxrv+pC5P161lPbqSdrPDlgcKNy8euTkd+mfpT5nYOU6P8AthdQhW21aE3EcY+SeMhZgcYyWIO/gAc+nWoJtJHyNpdyt8jNgRx/69eP4o+v4rkYGc1hrcSQXOwjzIlGTKgOAM9TkZHUDnFXVdJGt5IZSGWUMrxsQysAT1HI6VUX2Ja7isDkBuCCRgj9Klt7mW2mWaCWSKVPuvGxUj8RW02tJextb63B9p3ABLqNFEyfNk/Nj5vofQ+uRQu9I+zrJPZXcV9bK20bFKyrkjrGeT1H3c++K0UnsyeXsXFvoNReCOW3jjnZ8SXEICls5xlOFJyRkjbwDnNOv9BvLa3+2wNFeWpyPOtW3gcZ+YYyvAOeOPWqGlg/a5EdSpWGbII5BEbf1FEF1PZzJNbTNFKhGGVsH6fT2qXvoUnpqRXColuNndQxDDBViMkH6HIplph08sqHHbdW02pwas+7WY3a5bav26A7XAyeXTG18AjsDhQAaZc+HLy1zcaf/wATK0PSW1XcfxQEsPwyB60ua24WvsMXxBeQWbxThb2Ftv7m6Zn29vkbO5DjjIP4VbtpdIurO5AvZdOaVAoivF3xZ3Ag+aoGB9VzyTWBqQkjhAeN0cSDcrqVI56EHBFNLlYB244ppJ6oTbW5bvdLvrBfMnhxDhWEysGjYHODuHHODVcIO45qW0u7mxZZbK6mtpMhi8L7ckHPI6N36g9TVs3+mXSSxXtsLe8kz5NzZAgFsAKrRFiuOpJXHTt3vmkibRZRUFEedXKtECQF4J3DaR9MMaS11a6snZ7a4kiLYDqD8rgZwGB4I5PUHqfWta50V4tGvrq0uYb+NWjCPFGQ4GMtuQ5KYHPORgHmuaBWSPIx+XWjmUh8rRsjUdInN5darZyZdFEcdntjRWHVip4Gc8449hXNXa+Gd7galcSlieRGQBzx1X86m1D5NPlIODtrj3wSRWE9DWKua17baFhPsupSOxYAho2x9egroYVC8bQFGAPbFcJgAgivU7rQLmJZJ7WRL60VA5uLfnCkZ+dPvIcAkgjjB5wM06MkpaiqRbWhkyiKT76KcnoRkU+3u7m1eNUkSe1RlLWtwDJG4XHy88gYAHHamSxExgryMfnUO0qBu6dcc8V0SipbmKk0dLdXGm+I9TtvIhTQZ2xHIHUPaucE53jBQ8BeRzkemTU1rw3rGit5l/YyiFnO2eE+bHwM9RyBgE/MB0NUobWXUJYbW3ZUnuHWGNn+6GYgDPB4yfSiy1jVtIS3jtNVvlFt/qy0pZQcEcI2VHBPGO9Z2lB2i9C7xkrtFZH892bzBIpGAFPSnzkm1zFuDqcDPatGTWzqeZdR02xuJ2YF7qFTbXDAE5JdPlZjnuuOPxpZLOzktopdP1FxI5Aaz1ALGye/nAiNvyXj8qpVf5kJ0+zMqa7KWcbbs7gI5PfkN/NajlvIbf7pyw/z1q5rdjfWOlPBPpN3GTMkj3BUNGFG8DBXIwS55zj5R1NcxNx1PNQ53L5LWNuHW9mx1JWUEHKsVYc9mHPYflXSWvxIuLuKK01S0tNSgjBAS+iV9xJGOcdRjGcdznNeehwfu5ziljcdD1rN2e5Sutj06G1+H2spKJNGbTrqTHMczMijuV5AB/4CR7VlX3wi1h7OO90qWC+gddyxq4WRM9iOh/A9e1c5a3i7kX/ZOMetbEPiLUdI8uTT72SNy+0LnKnPUEdMVPsuw1U1szkNR0nUtEuPs+oWstvLyuJVIB+h7/hVZ22qQcqQPukYzXsNn461O3i8hmgnQsZBHcR71BJJOOQepP0rlPEWmWes38FzZ2VtYQrEElhteN7g8sBjC544/GlytD5kziYWYcZxu606SdpMRNnbu+6fWtm50G500STQL58EbAMxXBx+vP406JZNPu3DboftUO3HHzKWBwQQf7vbHIFFnsF1uVldVaMY3Fs85qysizxkPgjzA4VucgA549jiieHYd/VhyMVLbRt9i8zDNG8kkY4GFKhCcd+d4z9BVkdDQXW9aRAo16/QAYCi6fA9utFZZZVYjK8cck0VryrsRzSMK00bUtSf/QLWWbPHyIcDPvXX6b8I/ElyA11CtmvVjOwH0IPQ/nXrYLgYCxqNu0BEEakf7qYX8wafFlYypRDnoRnP14rmVM2ucTp/wt02wnNxdXZnkXnHnIsf/juT+tdvpmjWltB5domnQ55Ki4UE/XjmpYrSSNS5BRGH+7n8jUTARAqYwB1+8ePwzVJJaCLq2TLgG4sol2/Mz3SAA+nBzSCygYA/2lpRJ/u3fX9Ky2nKnaOc9MVUmkLH52Ube7Hj9aAOgFjCfu6jpn+8bkGnDSLqZc27W1zn+GGden41yrTWzI+ArOvJKfKB+NUH1eSCJoop9mOVZQN2e3zCmoXFzI2p9Qht72WymjdLqFtrxlc7T7npjGDx6ioZNRZChSOMA9yM4rmfFepTnUNG1pyd15ZFJmHRpY3Zc/8AfIT9Kr2uvLKYo5cZY5Z89KLdOqGdOs819N5ck5KscEE4Hr0p19PDpds6quTGpYbRjNcquq3l3cSWuk2M01zKAoeJGIGc55/h6jniuvs/hxr+shH8Qaktnaj5vLhIZzkdCTwMfjR6CscPq0plmj1SDeRIpRiOPKK7CCT77yB9DVvR/CWt+IJAyq0ELf8ALSUHcfcAexzyQD2Jr0sSeFvDWmNp9jCLryW83LtvUv0yc9+B0HHasm78dXARjYrHbopA2xxjI/E5/TFWoN+hPOkSWnhDwz4UhWbVh9vu0GfLkwVU/wC73/Gq2pePbzVSbayjS3gztOM9O1cVq+r3eo3Cz3UjynlNxPIwf/r1oaR5MLK77SQDu9Cc/wD1qcUuom2JdRXsl4yO7+Wg4LH2qG81Ca30eexZVJZ0bJHIxnBH5mtLUdQiCNsGQeAPU9j/AFxVaz0G+8QTSGzWEID++mlcqkeCO4789PrTd2iUc3A3lyqzn5QOuOn+cV0mk+F73UZFubmVLPT5VV1aTG+TBPKJnJB45OB7mukSDwv4TgErRjVr8cGeZMqh7bVPAx7c+9Zmu+Jnulhu4mI+0LukI6bwSMfXAX86qKbE5Lob0+safoiL/ZtpBE6bY/OCKZWxgfMxHt2rHn1+aW5d97FJBgknrmuPlvpJCyyEndhue9S26XeoXMUduGaWQhUVerH0xVpRRNpPc7eDcHD9zSRaV/a7Nb3E3k2sheMzMQOoIIXJ5P6DvVq1iMdti7hBujGdkCHcpk2nAZhweewJ989KqHUGuH3l8kjAOAOB0GOgHsOKztdjbsXI/selQqlrB5skeAJ7jDsMegxtX8Bn3NUrzUZFtp7mWRnYvncxyTxSBjKGXPPQEc1Wh0qXXYGEM3k2y8vMykgYHKgZGW9s8d/dbME7opzXbNNFdwPhixMoLBQEXbgnJwBlzVrSX1HVLWZYoFntxIwNyxCxE7dwGSQDyAOPXtWxZ6bZ6YsZSMSSo29bmRF3jjHB/h7dMHjrUs+rzS3c0MjkkYCknqNo/wAarrZFPYjg0+LT5lur2SKa5z+5jHKR8ZB6YLfyx361Bf3buglxuIycZxUd3cg7N74wQcgdulTReVYYublVeVgHgicZ46B+fpx6dfSla+gr9Sa4g/4lvlzAh+GKdwecA/4fT0qrEv2WweRm+YjC4z9Of89jU8Usl7AWcEFtxYD65z+tIzm7tSh2IQNi5OAuOmT6e9VsS9SvFD5NpqExyRLORGo75PJ+nKj8ateINT0+y0y30OOR45ABK7xr93PODweefb9aryXcVtDLeJultbdRghMnbuHJA/M/ia5txBqN9Pcz3czeYQ24SFj7D5g5xjjr27dKhopPWxet5tNXBmvLnaT1z0H0qEzAp+5Y7yON68A/1pr2lgkeESaclc/NcvGAfTCFc/8A1qcqRqNvlptPZsvj8WJNIosxOHQq0bs56Edvw7/Srm2eOGOSbcWLFAjnARVA7ZyASeB7VTiP7kQLuMa/dQNhV/DoKsQiKONjJHuORhRIMfjjv0oYDo5iWLMLdPwLfnxU8UheVY/OTMjKg8uMjOTxxx35qASqh3LHGntjdn/vrNAuSjZTMeerRnaPyFIZYnWaN33s5CnBz2+tQFmPVR7/ADCpLlfKvJYy4k2n7/1AP9aQDbj5CT7ikMkKmZpJXZGkxvbAJPv0GPf8DQGEYxuk/wC+cURhmYgGMEHoZU/lmpZbZ4GMbmIMACQM9xkdqAEW5eP5l3gjo2eanLvcI0v7sgsQSF6nv/OokRMZLM5PZQF/U5/lUwdIInRIsqxDEyvnaR6YxQII5HUfLLg/3R2/CpFuJS2csW7nH9KjXdNwu18/3Ywf5UpWThAHOOCNwH8zQMcltI2SkL7V9F4Htn8akFvNwNoB9AM/ypkNlMlwLiMor+pO7I9CMcipH1C33FY2dVHbbS9AJlhZkKSHdGwwVZcgj8cfnVcabbKxyZME9C/+FAvEzuEg2H++2P0Ap8OoxAnJOewC5H86AJUtbWMDFujf7wz/ADqTzIgMBVT2XH8hVW+igmK3lvOpZ+JIHJOD6p6D1HQdqhR3g4yEdjwGAx+BNOyAsTCGUlpCysAMOv3vZT7fy/Qs+1JGMRIB7lev8zUyQSNgyb2PcEjB/KnvbQELnap/uDv/AFpXQ7ECS3cxyCwX2XP9ao61p6a5pF5pN2TtuYyocpyjdVbAI6MFOM84weK0yixgGNo9uOoXoffnj25qdZxwjPk5x1/pSYHyVRRRWJuFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFADohGZUErMsZYb2RdxA7kDIyfbIr6bYMq7trKoIG7bxk9smvmKvpjS9Qa90yz1BCYDcQpNsDZ27lBxnjpnrVImQNnbgKfqaajSxtlXKE9dp6/WpjHtQnzDu6kuBtP4DGP5e1QFiVMjREqpwWiBkXPpkD+YFMghWFY1Ybd7MSSSxxz6AEYqGWNZOCgwOgH+efxqeScq2Ft5n7fwr+jEGoSZZG+7CidwZTuH4BSP1oAq+QiH5IkX3C4pWD5B6EdKlEUrEh7hdvTCw849clj79qV9PMse4SvIAOkcmWX6gKp/HGKYFQJGZHkaFWd/vMcgn8jRNDaSskcEjJcM2BFK4G7jPDYA/A4+pqJ9LtifmR5F6/vJncfkxNJHaW8AKxW8Uef7igfyoAzLryUlkgZZWmjOGRUJx+Qqo8XmKR5Dgjp5gAz/P8AlW+0A28YxVdoAvt64pDMH7LIDuYIiD0Jc/0pRaW7nBlZHJAG8Hb+gz+H61ozQgHOc1VkjJUKASfSgZRe2a0ufMjJ3xMQskfIJ6H2IIPT0ODWi1/p+oJ5mpWrxXnIFzaRplvTchx9OOcZ6VVRpoCNobaT0Me5W+owQR060skfmzMHVY9w+8hAUHHXHP5DFNMTQ7UtPu7BBKQ09tx/pCxsFB9Gz91vY+ves1Z2BLRkh0OQw6g9eDVllubWVvIukKHgh0BRwOcMhGCM9jU6aTb3KE2d0sdyQAsMqiNGJ64JY+/ftWvP0M+W2pdi8QQXsxl1K1ErsfluYzsmCkYwxGFfjuR68U650nanm2NzFeQFS5KMN6AcncvUY74z68VhS2tzZOsV1C0UmONw+97g9xU+n6hd6bdx3dpK0c0RyrrgkfnS06DsWVPykdKdAzQSpPA7RSodyPG20g9Oo/KtZLzRtU06E3y/Yb8fJ59vHlJMY5dPU56j6knpVG7sJLBIZDJDNDPnypYZN6tjr9PoeaqLTdmJq2xPqGtrrKW8OuW7XRjYf6VC+yVV9x91+x598daqSaFNPA76RdQ6pGiK5jhyLiMH+9F1H4ZqpITnoAehqJGkWeKWKR45Y2zHIjFWU+xH8uhq3HsSpdxPN2lonykiffRhhlPuKjDlpwASDtOcHtx/9auouNdFytva6/bnUoDFHIZWwk8bMoY7WGOOQMcdPoBJB4X0fVbaWbQL+Vp41ObO6jBkznjDYXGenfP8pUmt0DSb0ObhuZ7S4SeGR45ouUdG2sv0IrSbULfWLiGDU7fMrsEW5tYwszMzZ5UcOST19zxmsueOSC58maJ4pAcMkilWH1B6VJDG7zjyVLMgL8Y6KCzdeOgNVJJoUbpljVPCsl7o01zo9/DeRBQWRiIpE5x8yseOnr61x6eFtUj2SPbxhcnKu/T3OOfyro0kyCpLeW4GQCR/Kkls5jA8drfz7HI4Dkkd+/Ss5U2+pUai2Ocg8PvBqkazyRPEMt8pIJx2wRn0/wAa6i2u7mxkd4Li4jaQFHMcm3ep7GsSGy1KHVlaQy3ShGAKgkqMZORW3Gg28gdM4oppK6Y5tt3NJtSt7+RTqNpDIMkvNaKLeds9SSBsc/7yj69aF8MtqSyy6Ndpf7Fy1vIvkzoO/wApyrY9Vbntmsp125KHGD0psTSIPOPOCVyHIbsePyHPqBWlrfCyL3+JFzS7W8i1rSxJb3VupvIP3k0LIAPMU5BI5HHWs6eILuUD866mx8Za7a2whiu0mjCBEiul+UADA+ZcMegyWJzWQtlFdozR31vBNtDPDev5eTjnZJ91uc4ztP8AOkm1rIdlb3TFQ9P0qdJCARnI70++sbnTXSHULd7V5BlPMHyv/ut0YdOQT1quMg49qE10BpmkmrXUdtLF5nmwSRmN7eUlo3GCMEZ4PPBHIrnLu2jnus25EQcktEC0ixcnjdjJHT8+prQPIIU49qdFaRR2XngEM87R5/3VQ/8AswqZJNlRdkY1ppV9qEkyWVrLcNCA0giXdtB+n+RUG0q3PX6VuNKbaVZUyJUOY5VJV4z/ALLDkH3FBMV5NFJd75Arl5SNoebJycvjJPXk561PI0NTTMeIgtjnr1q7dPiKJh13dffFW5rG1EwFmJvK2/8ALwy5UhefmGM9DjIXqBg1nzOs7pFCjSuW2qgXJz9B1qk1yu5NveHR6i+8u3OOAKuHU5yyYbgDAx69ay5rWe0umguIZIZB1WRSpx9DSLkDAbkHA96zUi3E6u11Z1/1bkMy4+tLZsLa7N5bvi4JJDyokoUZJIAYHAJJPGOprmradkmBLH5eRjvWg9yVba2NwHGPWrST0ZOqOoivPD9/bldT0NrecIEWbT7jAfHG4oSFU/gc96pabZaNLYXFp/bAtLkztJH9ugVFQEKvMikg5wvA9CcDmsNZZJjycY5OOKqtP/pRLSMiAEMVGceg/PFDgkroam76nUDwF4wkUOul2rK3IYXcWCPX79FYMep+KY41SLUb4xqAExKw47d6KyvPuP3T15tQjTlyGGeMCnDVY9gZLqDP90ZDfqBVU+FnYv5GrXKrnJAmkX+TCn3WjO9oC2pXcvQfKQD+fXj3NPQdyZ7tGJZ59vfc+T+tQvcRsMi7UjGSApzj6HmuZ1AarpM32gTQvak+Wyht0ke4EhsYAGcHkenPbOdLrlqbQqjNu27ixP3iapJdxcx0V/qiRHZC67iDl2OcEe3T865+TVj5zmS4LuP51iz39xqDbYQ5dj1U1ch0CJYS9zIzseTzjH45ovbYLPqOl8QB24Ylceves4ahc38rJEMZ6n2rptM8GS6koW2tSyk43vlV/PBz/wDrrutN8F+HfCccUuqSPczHnaq8Aj2GMgZHJo1vdhoZsngu88SeA9NsA7W11byyFd8JJ2MQDnkYPGRnrx6nF7RvhVoOkK02uXb3LDDfO/lL+QP8yRVm/wDiFBbxNaafbxw/3OxUfTGPX6e9crPrcs8ryXTu5I+ZSdxo5bycu4cx6JPquk6Fpk8miwW4iXndCBjcf8ivO/EHi7Vb+ACW6fY3UL8ox+FQ/wBpo+k3Ajdl8yeKMoVGCCHP4dKwLh2kZkzwpOAT0rRJIyd2x32yTHVjxg1DCJLhNsRI+YAHPBJ7VqWS27q0DxlMYJb1BHNJo9/aW0k8OfnJJUKNzOQRgBR1NVdhsZNzDPEod4/3ZZlDZ43AKSPw3L+dXrWK6vr2GxsoXmlfaQqLwMrkknsOOprt9J8KWk+hK2tx3NqPtb3CRNhXYMiAhuu3lOnUD9NhdTtNJtRb6TAlpbjnEY5J9STyT25qL9CzLsvBcOm2X27xLcJK6DItomITJ6bmxnH4Dp3rL1XxPJIFs7Jo4Yl4WOEbUUegA/nWpqV3MdM1AK+d8BcknJwvJ/8AHQ35153blo7h1x8+c49KuKVzN3Zp20M92GkYl05B565H/wBf9KoahbNbLbqeELsvHQE8gH8mP51oW0UiNGoHf7oG4sewA9a6+28IR3VkJtd8yKF5Ekjt0YiRioP3sfdyGIx159ejk+okjlND8OX/AIgQNBHEtvCxEt1I2EjwOQe54/8Arkda7JLKw0WJrTTB5lxkpcXUgHzr0wmOin2/XrTdS8Qw29qtnaqsFvEQqQxDaq84xx165rAS+mlSXIA3CplcpamlK7+arBVh8ojaIzwAO9ZNt5kaxxlSzABcDqT0/nVnSrS+ubn7PAPNkOScnhV9Sewrakey8MZdJBc3pztkYAbST0Udvr1x6d3AmaJLPRTZD7RqzrFtP/HuGBb1+bB4Ht1+lRXOpGZkWIhYhwAoAGPTjgD2rl9R1a6upy87jLHJ28A8Vf02C4vWjigjLZ6noFXuSewFNoUTeuGEcSF3Crg7ieOPf/Pesy8069mullVRDEy7fOuZBGqnucHkjjsD1H1rSlmt9OjaGGVrq6SOTyp3Ufu22HaVHrnHP9DiuKn1Ga5V/tLySSDABZiTj8alK2pW51D2djaMjzXK3VwBhdvEakHsOrH3PHtUIupdVdRMzNsJIJ689efyrEtoGaSzfPyvGXfPZgW/oF610+m2wtGW4A2Sj/VrIPlyf4iP5fjViY6aJbKAwszeYWBYdNuRgL/j7/SqkoMWn3Em7GcgAEc8fN/h+dSTM8wfMoWV1IDt25PNLBbTX17JFDDJ5JP7tWwDj37e9ZXutSraleS/ttM8Ktp00Lm5vDsBlVghjGMnI64z0Fc5FGsGEWSOQA/fUHn8Dj+QrT8S7xquxLdIGRFSMSSAkDAPGevXk5/lVNLC9nAZ4p5Dn5SCXz+WcUr3dyi7GkT2zrHIqt1zOm0n2B34H5VX8loeHCHPPysGz+IzUsel30ZTzbbyInJAeUqDx6DOakaKGNiHvR8vA2xbiT+dPQBse1sZdYx03MGAz+VbTaUbfTo7ieUlZMnai4z/AMCNYJSFpFYeY+Dn59uD+GP61qtrEjWKWjQw+WowCc5/PNDTsLQpyNETtjDKe25t36gCpLcRvKqluT0G3cfypouSqgJHbr6EQqT+ZGadLd3MsPlvJI0fpjikPQ1DbgIJJTMoc8M58sNj64zULSxx9Tb4z3yc/lkVGkv/ABKljKB2DEDLABFIznPsc/nVI28LDDTNyOTGuR+uKTWoy79tjMhHmeV7xQg/lkinXGoJcBF2MXRSN7YXd9R7fWqsaWyDHlyOR3Dhf0wat215ZrcwxS2MXlscE4dyO2euP0prsBEglfc/mxlcfdJUAdu3NWo7e9jiUiZ19HSEgf8AfQH86SSS4t5ZIUKJtO3EUQjGCP8AZAqHe4OW3Y74BOaQGlcZuLaD7VNl1TazBg2cHjIB9/0qBUhiiVVlUkfxKmP5mqX2gtz5ZUercCpbtXt0tX3KY7hW2sBkZHUZ9e/IFDA0Ip1C/fZ8dmfH6CmIlnDGHeFHJP8Aq1Y8D86zBKxzyOacshzkPz7UAaMltKhMnlokDHMZHPy5PHPOcf1qJLtLdcmCZyOQBz+mP5Zqs97bxxZlwMH77yYA/OmokdwWMJDAclkO5R9SOKQGiNQlm5ijjXtlnGfyJz+lWYrW8mYgXSksPkEY5b2HA59OayYrVi4IYDd93vWhDbTQgsjCT1weKNA1IDNBGxSSS5cqfm5A59x/9epVXTwVchCT1Ly9PYrn+lSXbx3siy3QWOYfel3A+Z9e/Ax7/pSSafa3A+RNvHUKR/OgYqz2IO2NYmJ7Imc/lSJrNobgWyS77g9IVHzn6L1NONhb+WEkhjkA6Bk3celPS7lhMaxLciNTkYBUL19cfSkB8p0UUVibhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAV9D+Gos+GdIbcx/wBDh6uRj5B2FfPFe4+CNct7nwTZvcT4mtQYHdVC/dPyKRwD8pTkc8jJOMU0TI67cUHBRSOpRQufyquW3EAZyTmqsOqtKg32NwJCAdiKGBBGeG+7jn1FK816/wDqLIhj186UKP8Ax3dVEk+GDfcyM/SosLuO4jPYUrxXbg+ZPboeQMRM3/swqsbCaXcH1K5XGcbY0wP/AB3P60CJmKthSAQelRNcwwShTcJHIOgLYP4VM2iwCFnwb1M5IkZ5SuAeSjEhe/TipIoViiURoiR9gowKAKEl/aPFloZZHJO2WGJuT7kKQ316+9RrHM5IdVTGcEtwfyya0WiUnOeab5S7ux/Ci4Gd5H3g7FT2KLuz6dSKhNuuORJn/fGP5f1rVdc8dB7VVaFs/eoGUUjETKTArAE5zySP+Bbl/Tt+NRm3e4n2oC528OVVDn0wDz7AVdl8lPvyIPqcVAZoDwrE+6qSPzApgZUoXeVeQ7h1B4oNvuUHqCOD61p3CwzsrXFt5wHTLle2OSCDx+NRGG2gBMLFFJ5XGQPxyM/kKAMxrMHFRvYqc4Of61qOylcAkn2GKrsCMjcR+VAWGG4meI218Dc25GACfnT3Vv6Hj6daz5dN+VWtJPNBByjgI4OcYxk57dOvar546k/iSRV7R763sZ3N1bJLC3PCAsp/HqD6UXEc9cQeVa2vXLxl8HqMuw/pTrS8udOlLwvhHA8xGUMkvs6ng/z9K7WXSNK8QR+fDcvGyLsLKAdvcZU49fxz171ymoaRNaI7GSMqq7hk43Kc4PPvxj14ppsCWM6RexD99Jp15ySJsNbN6YbO5B/vZ9M96rXmlX+mXCQ3lq8JZvlLDhsHna3Q9R0PcVlxkHAdgFPBOM4FWLjXr1ZrlIp0a3nYs6GNdjkdG2HhW9xgjjB4rRSsQ43L+tIsWpbI23RrDDggf9M1qgjy28iywuyMGypU4IrSjubLWMfbWi024jjWNLiFCbdwowN6ZyhwAMrlfYVU1DTb7TWjNzCRFKMxSoQ8co9UccH+fritIzVrMhxe5v2GtabqTuuv2cc7yqqC6RdsiAZwcqM9z09gQcU8+FhM17caXctc20YkjjjC7Zmbyg4A7N98Agc9eK5SJirlSMitKa4ksbaxNuzRNIrTs8Z2sx3lcE9wPKBweM0nDrEFPuZoyGKMCpU4II5B9DSjKuCrEEHsa6aLVdL8SSLBrsQtLpuBqUBwTxx5i9D9T+YqjrXhu90QecQtxZMQEu4TlDnGM+h+v4ZoU9bMHDS6K9nql1ZXUdzayiO5jyEkK5xkYOR9CfzrXS98P6xY3MuqJ/Zusu23z7ZJZISccPsGQASDnv3z3rmsggY7elXZ9q6PbgAZM8r7vX5Yx+m0/nTnFOwQbVy5feGdRg0438Xk31gE3/a7KTzE29zj72B3447mqMe1tFgG4MxuZQQDngLH/XIqKyvLmxleawu57Sd+skT4yccFl6Nj3B9q6WbxJY6/EYfEFqIL0/d1O0h37cdAy/ex7ZI+lZvmjuX7r2OWcFT2BHagSsoJIzkYrXvPD15Hai/sWj1Oy52zWh3Dg4OV6qfzrIGHOevr7VpGomZuNiaG6k8uKLfugSUSi1c7oWI6ZQ8Y57VvbPC+rlvtNt/Ylzgnz7c5hJ9CvAA79B9fXmmBQ46j3p9tIG1CySRd8RnQyJjduUHLcfQGiUI/EOMnsSX3h7UtNsheyQPLasoYSR4LBSSAWQZK9M55Xkc54quZY30aC2DMJUuJXkiYFWQkIBkdshaj8+R0QLKyIqlVCfdClgxG37uCQCeKmsLWO71tZL/URBbScErEdqHbjdtB5JIGf16YrN8y1epd4vYzpELcYpIIyqsNuMHA5rqLzwldGKSfTJoNQt1GT9mk3SIMDJKdeu7pnpnjpWHPZ3FnIVuAY2B+6wwQKfOmTyshHyKWOeTT47iW0vIru2laK4jOY5ozhl7cH+lMLgOsOCN3YjpQyHbggD1GarR6C1Rs3vjK4uraaPWLK1v4ZWBkURBJW+jjhTk9h3PrmqUmhaHdiSS01C6sHIykN5AZ1bJ4+eMFlAHXKnnHbkV4xEcebG7pzuCDnocfQZxz2+tSK/Jz37Go9mmy/aNIq/8ACM6r5RvLOGLU7cYLS6fKJtpwDhlHzqeehUVnxjzwzI3sfrW4yQSESMi+YvAkAG4D0z1x14rRk1Swns3GuW0l5I7bvtcRC3HJHylj/Dkk8eufUFckltqCnGRyq7owGPRR61UmR5VLxtjLheM55I6Htz3roLbTrO6ght49QkjvZMri6QCENnIzIOgwMcjrjsciObQb6x0lb27gMMfnJvQsN6jPDYz9044Pfr0o5ujHbqicRnHaimiSAgHcD75orosjnseoSIs0gLsSRnjJxU2nvHBAYlI2JnvnGTnmuTm1m4tSplYb3BJBXr2H4daXRdevZ7iez06ye+kAaRhEMhf949Bz61grJG1mbOtQLeQ3MSuBvEXmNnGB58WT+Wa4q40yyEwQr827p3Pt0613ujWCiDUJfFbQKl0F8u34YbUYPjHO7JUdMjpyc1Ztr7Q9PaSfStKjhuCeZ5yC2PwJOPx/Cly3Y0+Xc5/R/Bms3EamOyitoSOJJmwT9FXJ/PFdFBo+i6DqPk3l09zfEb/LZsbB2wABjvg9TzUcvi77aDA0z7fRAEBx7jmsVZIp9d0u8hVSCJbaQnk8Hepz35c/lVcrQKXMzevvFPlhkg3xQrkbYfkHPfPXv1ziuG1PXUmEgjZjIWzubOT+v862PEcTw6cZljCncAQcjn6VwJdWG5l+7yTmpvcEaOn3CiRjP8wPTPar9xOnmgAlAwwZByDkf/Xrn45ijh4wCFweehpkV208gihiknuBhVSIEkn/AB7U7obRu2oeWG7QIkyLCJnV2K9GAByD23Zx+FU7eZ3uUEcE8szHEcUal2kPoAOTXdeFPBsgsryfxFaSQW9xEqxRcCZvmDevy/dHB5+lb1veaXokLwaNZQ2m7hz95z/vOeW/PHbHenexLOTsfCUkcgl17VItNWX/AJdI2DzEZP3j91D+Z68DpXTWv9jaDDHLotnEtxIHVpH3M5IxyWPPOQcAge1clq1+PNWeUm5ck4aQcdfatPR71b/S7kEqJYZI2BAxwQ2R+gpWur9gvY0zdS3SuZZGkZx8zt2+mO3sKzbkmVUOT04GKtLN8gC4FRRWs17NHa2kLSynkKn8z6D3pAEEbXMb28pBEsUkfpwykf1rmtD0G+1hob2CMeRKgbzywEakgE5Oeoz0GTn6HHfWlhb6bcC4mkW6vowV8tDmGMkYIJ/iOCenA96p3moBLcMXVLaAeWiKuERfRVHAH0q1vclmhb6ZYeG4VliKXOo7f9ey52567R249OeeT6515rU11ITLIN+TwDwPYVn3d/JeLaum4q0SNk8ZAymf/HKkWxO8BVLbjwAM5ahhY5dna7uPJDYZ32/T0rrNM0Rr+KO4SQRWmTvnYZyeeF/vH9B3IrS0fw3baZqb3+ogPc8+XbLykee7+rc9Ogz37X9Uv5Jw7uRhV4A4AHtRugehkatqcGioLCxHlRlQ4Y8tIPViOpzkY6cdOhrlpLoXM7TOCT0GefWoZ7g3l4C4b7pCt24Pb3+b+VXNI0qe/lchCtsnMsuOEH+PoK3itDJkll4futZmikjdI7fLLLM/3YwuD+JO7gdT+eNy61C00vTm0/Tt22NsvK/LyH3I/HjoM/jWfqWtRw2n2DTl8q3Qkj5iSx7k/wCewrJsSbuYQMSC+AKVrjuy9p9y93fR5/iIXI6jJ60mkeG7q98yeYC3h+6Wnym73GRyPcDFbENlF4fbe4DXUo+RSARH1/M8fQY71QvHurlzLJIzbJVYFmznkZzSdmCdmasP2CwMTQxtPJBxmUAKTnOcZ57Y+mcVAZHb53Yktk5/Wq80g3sAcDJzmnQ3EK7JpCSUOI4ipBJH8R+mOPz9KgqwNMs0z20buJw+2UYG3ABIA7k9z9B6Vu3mtR+HdISKO336k6H7wIC+xP8AQc5/OqsF5Dpey7vYS0twzGIbCSi9N2Mc8/ofeud1lpdS1Ce4OViLYjBbjbgdu3NQ9itjJurq8u7l5ryZ5JCe54H0HYUqB9+4tjtk9aBamMnb5bD1GanhtnkwQefdcGknbYotSXn2yGKCcFUgBEbIM5+oJxUcUMCuSxeQnjAGP15p9xYT2UwinaIEjPyOSQPfjGfoTTVUdOTTvfUSVifzbcH93CzNjo77v5AV0Uem2r6ELtrfyZf7zhse2ATjGSOvrXNCTyBuD+X75xUT6rC7FWvVcjnAfcaT2Gi1IZQQVbn/AGQB/Kmq85J+Ybu+cN+p6UyGYyPtVJAMZ3Ouwfrg/lUsh2yqoZJEPVlLcfgQM0AJFb6jcTCG30+eWVwSF82MZHXPLfzpyH5F3bQ5GSA2RToGltLwTiIPb8ja6ld4I74P6c0ivEgZfs8RBYkJyAo9FwRx+dKwaEyFFXLng443AH8utO+2WNo3mz3CRAds7T+ZqIyM0YURQ7e37lWP5kZ/WmI0tspMRMa45EZ2/oDRYfoXIdXgmcJZ7CD8wckMD+PQfnUhvrnaDvcj/ZwP5mr1lKdU0KazuCSbMGeDd1JAPyj35Ix/te1ZJzG+zdGrYztLdKNmBI8ry43RqT6hQD+lI7F7OSBk+8yvkNyrKcggY+oPrUIPmPtkfag5BQEnP6Yq5YJbzS+XI7tnoQNv9TQFjMaG4LENdTOh6AhVx+KqKhNrvABLEZ53sXJ/Ou2uNMtIrWB4Y42ZmKvvLZXHY4OPpx+VUXYRTEJZ2qn+GRIQOPcgf1NK4zBsRcRXC3EWSqHgx8kevbIrrBeLqWnsZo8TKpB38rKP7pAz+f51Ql1mUP5bsJccZ3lh+GaqzXl1LwrSRg9PLGP/AK9G+4rl1zO1ukogbfj7qKrlfzIx+NRPHd70QyggkHlwpH/AQD/OqEYmlY+ZJLIynuNxFSbXRiFB3Dg7l24/OgC3c6DJe4M17GB2AR8/nvx+lLJ4cjh0+a4jurmZoDvNvGwG6PAzjjPHXrwO9JbPciQeZOyj+7gEY+taMWqpDMGViCvdkOP5UejGVLK303Z5kARx90/vTIPpySKv+agIjBUHGQn9cVSa4s0ndrW3jTzDuIQbUHqakS5hk4SSIsOwbPH6UrBc+UqKKKxNwooooAKKKKACiiigAooooAKKKKACiiigAooooAK9Q+Ec8xg1aDzpPJVonEYJKhiHBIHqQo59h6V5fXXfDbUf7P8AGdurGMR3SNbsX9xuXHuWVR+NNCex7csiBiADkVJvyOBx707dKxwECehY4/lk0mzAJMpY+gTI/PP9KozIpFYt0FGFH3z+GaeYo3GFR19GJ3Eflj/Peo54YY0U3CwvExCCRlA+bngqclf1HvTAa1xBEMeYifVsVXaRQwZPMIJxlELr+laMVlHEuECKDz8igCnxwbVIYk88H2oAy3klZ9ot8YONxbCn6YyfzAp/lvuHykr328H8+f5VqbADwtBQE0WFcyprN5VKfOFJyCz/ADAemVx/KqkmjqxxgP673ZgfwJrdbAOCfxqMYX7zBm9QKrlC5gx2MNrshMbRx7iS0ecg+gBO0/Tj61cn02OSMvZ3IuCuMx7CknTqF53fgTV6ZVZTkAehNZYZd21GMso7QqXIP0GTSasNMoMCT0H1qrIrAkba2PsMlxN58sJiJJDSF1jLZ7sp5Y59Rn3FVbm2eO48uGa3lQjIcsVP5YP8zSGZZD9O1RFnXnuO4q81tcno0QHpsJP6kU37OVbLySHH8PygH24Gf1oC5msHIyeDTMM2RtYnritMvsLFbePaRjDpu/nmmyPFPFscvCcYAzuQ9uMDI+mD9aAuUba8uLCdZoTtZT35B+oreTXdJvAf7Sh8uQ4OdokUEDqMDIPfkHr1rEudNuLdVd12q33SGDq30YZBqg6sg5U00Jlq603RzPCdMvYnfcHa1kWRBJz9xHIIGRx8x49TXKlZldEmhkilduEZcE84OPXkEfga2mHAO0cHIPoasXGoC7tFttTt1u1DZE5OJ1Hs/f8AHP8AKm/IDCDFXJyVcc4PetfTdcuLKNreGVRDK372CYB4pD/usCAfcD8+lLeeE7qZPt2kTvd2brlfMb5/Q84A4PHOD7VzkkLu7KwcOpwV6EYPpSTFY7qfSdN1ERTWFwNNup18yOxu5Q6SDJGUkByBwcBhn9KzNRMipp1vNG8ckNqVYMODmaVgQRweGHI7gjtWMtz5tuiTEOFUKM8gADitaz8Q+Yi2Gqwtd2CbjC4fE9vnk7GP3gTztbPtiqUrC5blJztbp+Nbfh7X9Qsb+GzhkEtvcyLC0E3zxtuOOh+v+Oap3miym3N3YSfbrMDLyQrlouOki9U+p496p6U7HXNPO7B+0xYJHQ7xzTlJNMSVmdfcw+HNem8uzZNE1IMVMMvFvIw64I4TngY6/wB2svWraaxsbCxubeWG4hEqyeZGVDHzWxtJ4YbdvIyOayLr9/NK8iDDuzEY45PpXQ6L4uvLKwex1SIavp/BSK5YF0UDGFY9x2zjHrV8slblFzRb1MHy1RAcfjTTIrNg4GOtMjulnupmBeGAy4ggljKuUONuCeDjJz9KeyIzBkYNnOQeDVXvsS0luWrXVbnS5jcWdxJBIV2l4zyQevXg/jXQ2d5pnipJ21K2h067iCF9QhXy4yp3BdylsAkgD1OOvYckY+uTj0HpViB2h0bU8AMJZbVTkA4A84/zUVMo3sNM0r3QNQsbU3skQkss5E0bq4VexfaWC/mR71l2cUVxfjaQdsM0mQeoWJz1weOP/wBXUFlq1/p88dxZXcsEqHI8tyFP1HQj61v/APCQ+HdTQC+0NbW8clZrzTxsKqykbvlwSegIIPBPXpSkpLzHFxZyO/5uAAPSnh88Y69a07rw3cozSaUX1OzUkebAu5wR13IPmHXrjH06DJJKNhwVYdVIwR+FUpJkuLRPHM8Z+RmU9MqSDj61aF/O9y1xNKJmcBWEyqcDIyVYqQrYBGSrfeJxms8cj2pwOOpzmk1cabRrQ2+kX0TvPcS6XfOBsWVd9uWx0Ei5KrnP3unqetVrzSb+wciWISRjkSwnehGcZJxwD2yBmnaTBDdara29zn7NJKolHYpnLfpmqVlKbZ/MijWPAwF2hlI9w2QfXJ5B5HPNSk1sVdNXZo6TCpsNYmfHy2qlSfeWPofpn86zg6g//qqWNo5b7deReZbkhgsIRHDep4+ce25eeetR3bWy3kNvakTSzFzsjOSg6qp4BLdew5HHGKadm7iautADB1JBPPB4qrqThbZNysQ0qKQPrViKQSAEbuRnaykHH0NQ36qVgVjz5wOPoCa0T0I6iKxHIp66hc28UlvBcSRxSAiSNWwrgjByOh4NMVgz/wAjTJVC5yAw9qrRi2Zsw+NtZghjhQwbY1CjNtH0Ax6UVh59FbH1oqPZRK5mejWei+HdKuC+rXkuvXa/dTaYoFx04zlhnPU4/wBnvT9U8Qtc/uo0aygKqq21pIY0VQPukLgEcnsB/OuYlmlkcuZCioMjnHtVPzy1wrHkDjNSootts2lvF88kABhx8oGfrSS3RgViGJ9/asz7TiUktjPHQ1K0yEE5yGFVciw1bn90x5yCeav+HXaQSIJCHguYZhnB+Q7kI/EstZCMZF2ZYjkkfStvwXplzfatM0MbtG0LqWRd20/wE9vvAHn0pvccLGzrln9oDKu6WVweATheO+D7Dsf0FcnbeGNQnuPKRJJJGf5IYUBLeucngcdehr07+yrazti+samqkZb7LZKC5Gehb/AcetUbnxclrZ/ZNIhNjACWZx99/wASM596wjF30KbSOetPhhMiR3XiHVYtMh+VjbQgvMR6HsD9A1dtptz4X8NWzQ6HYxodpLzyH5pPxPzNzg46DPBFcPNqbTv5kQJZjj5iSaZKzefC78kDOQudp7H88VbgkrkqbbsddqGuG7y8kjjjkN1HtxXET6yk2qIiqfLzjP8A9arMMt1emSJlO4LlR0A7Vmf2RdyagCkJYIRuPAA568kUtmUiTVElET4cMikMAF6Dp1/H0rW8I2Go39hqLafaRzOssXLHBAGSQv8AtHj8M1s2HhoiEXN4DaWpT5pGA3zE9Nq/j1PHpmtJ9at7O0Gn2EP2aL/pmfv/AFbufpVwi3exEpqO46HSbaz+bVptrn5vs0RDN9GYcDnPAyfpUerX8EFs0Nnm3hZQfLgGwtx1kYcsee571jy3zzS+Vv6fyp0MI3SlmDFkBCtyFzxn/PpTnT5VciNTmdhbe5keFAvyKSAB0wKztUEp0x4uCVfdxxWjbxiIHeylRzn0x61q2fh671CVpZ1+y2KjLzSjBb/dH9Tx9azeppYwtAt57uz0+JlZ2ZJEhQdThySR/wB9jnpxXaw29tocYaSaKXUgvybclYsnHHq3ucf44+pazb6FGtrpaIjbdvmhRvYZzy317dqxTqkqSwyhywLKGP16iteRy1JdS2x043RtlnLEnOWOTWN4mlmWCJISf3pIKgdcVduHkcRpGC0pOAAMkn0FWMW1ukc18UlnV9qRKMqOCeTjnp24+tTEUjndE8Ni6s2n1GSW3t95MJQjzG5GdoIxgkLkn0xznjQ1S+gjsm0u0PkwHIAU52ehJ6k5Gck5p+o6i8yyqsjNLgHJ7Z4H8ulYK6cftRg815AR5Yc8n0z9a0Wu5LZjfeuWOeD+lddFZReGYTLMgkv3X92rD/Ug9CfRj6DpTLXT4tBh866TffuAE3rxESRg4PRvT0+vSjPdPduTIxYA5JJ/rSb00GiZDJcSLLIxO0dM+p//AF/nTzG13FJboQGKk7mPC+me+M44AJp0ULvBauqiKNlkzI/cAptwO/8AF+nrTLi5tLXDptjjKZZy2S2ByT6ng8Um+4npqSJaJE+LmYzzIchEG1Nw6ADkuPrjnqKsadLp1pMk2oXML3mflia4RSAP4iDkgnrz/Osg6mbhFls7mCOZlG1ZDkjI5P5Hp/Trl/2dLK5P2mFmJy/z8k+p4zUvQqElLZnQ6hPeXF1NqV3Ym5kQBXnt5UdETOVUFmU4BY9uucZzWPJqqyHK2XA/56Sf4A/zpLuJobJYVuoyfODMi7gWAU4znjGTx+PtVQIxHYVnboakhnkk/hWNiOcHI+lXLS8NtIkigF1IILcrn6VTSHJB3c+wq/p+l/bZJSZCkUMTSyNjkKoycDvT2AS7u7jULs3F1O0jnHQBQAOMcYphQNgEEgepzSJGq9WOCalZA2Ogx6c0BoMESKpPlIMnqFrWAnfw08sUkaLFOPOyAGZSAFGe4yTx+PscxEZGyZgw9NgwKtMuAAbtD3KDdx+QxRYVymxbZnBI/wBgZx+VCtIAWXeRjHy/Kf8Ax7FTsVUgpnjqWH+cU0XUTkgSh2B5VPmP6UwJbe2d5lG4GMnkqCSPwIxn8au3OnxaffPbzyyXLhVIEa+UBkAjkliwx/sj61mpL52Tbwytt/h2Fcc471pas2o6tqsdxDbwoBAsbPNMybyM/NtC5yRjqO1JgQ+akchZI442HBIBbP4MSP0pXvJpPlLnb6Dj+VOj0O+Zg0txEgz9yJSxP4tj+VOGknesMl9iRR3baT7jAFTcZDGTI3EbOdp5cEY96mXTZTGJmjIiYcSDofxq5a20SrsMy3FwDlGceYYz2wx5H51k3NvLa6rNDIWJc+cpmbkI3IBP5j8KANqzs7SH5pbiAsOzleP1qyuo2QIiAaXn7sce7Hvg1gwPYiR/ODg57EEAf1rWXUtOjQDcNoHQxEn+VFhmsl0dk0fkM8cqYO/5TnsRg8EVSlWYRhPI83+9vXd+XzLVK51mzwgjXc56/IOfbmol19p38iGxkzjOd+wAdvmOAPzosInmhv15jSNUI6eSnH5g4/Os+W4nik2iaQD0SUhc/ReKtTazOxEbRRLkZJWUOF/FTyfpmqMy272zkS75CMgFMqD75Oadmg0Nmzu/7XtzZX08uAwaOSOX5kYHI4OQc+4we9TWyRRPLDOMzwviRZEAPscAng9RyfrkVzQlK4aIICOclSfp3GK0Ztcuri5t3NpE9wi7FYliX474IB7n2otcEzVlvxFuEaF3HVFQ/wA6p+Xe3QeY2kjRt1Ls2xf5Y/Gsi5mnaUvMYw2c8ybth9gSQp/KpbYG9usXM878feyXI/nRYLm1b6d5Y3yvboOuI8OfxIyR+VY3inxNp3hbTJ5nZTdyK4tUW3YiSQAdd235QSCSOg9TitW50/TrtRC0Dxns8btG35givnHxBqzazrM93vlaHOyBZWJZIx90HJPOOTz1JqJOxcVczKKKKyNQooooAKKKKACiiigAooooAKKKKACiiigAooooAKfFLJBKksTtHIjBkdDgqRyCD2NMooA+m/DmqR634dsNSARmniBk2KQoccOADzgMCPw71p8dlFeZfCbX3l0e60iXLmzPmwhBuby2JLAKBkgNyTz98Djiu+W/aUukVrcM6Y3K6eWcHPPz4yOO1aq1jJ6Mu5544pA0qMWjdo2xw6MQR+VVSL5+iW8ef7zlj+WB/OkFvdGT97efL6RQ7T+ZLfyp6CLxK3JySltPu4kjUtGw4+8mcqfcfUiqM2q2kUpS5Y27DODKNocDurHg05rCN2JeS4Yf3fOZR/47jP41OVAZmJRQ4IYBQoOfYcUXQWKj3pJTyrO9kV8bWFtIFPvuICge+cVWu7jV0wIdFlLMOC9zAB+OJD6e1aLSJnmQkAYxngVBJKP4VOPXNFwsZlvba5M+bi5s7RSP4U805+nAx2zuP0q8lqWLidbuQKduUnzu98IqsB+ePXvSF5MZEhHfikBk4O/PP40cwWJSLRIdtsluiHIDBFcnJ5yxyx/OopbhpgFcu3oCx4qO5JncSSSOJsABwRnjOMg8HqevPuKqNI2xjchSqDJlXjH/AAHJI/DI9TRcLFoxDBJAwfaoTGRxniljeORAyzb1YZBU5BqNphHHuZGX2ZuR/SkMjYkuy85HUkcGq8keAc4pjavaPKUDqX/uo4dj+C5NNkuZH4SzuS3TDJs/9CxQA1o8jg5+vSoJIcjoM/Sr9pbXVzLsJtogADgyhmOewHAJ/HvUbGG3dYruKXzsZKlwin3HB4/GgCgJpbb91HK6pJwyA8N7MOhqK7MEyZEAhfPLo5KY/wB09/owHtVqZowwZbOEHH8TO36FiP0pI7u4twTA625PUwoqH8wAaLjsZbaHfSJG9nFJeJIMloQSEPcMMDH1xj3rPks7wAFoniBbbmTCn8ic1rzyz3LkTyyvhi3zOep7/Wqsq/IwGARQBVtpr3TLmU212EBGCyFsNkduB+dad1qmm6jbWw1CCaO+jzi8gcd8csMHj14JOPes0Mdg3ZJ+lQyspHzITjjOKAIbzw9f2UEl0gF1p4H/AB82w3qOf4u6/lxWSsm4hlOCK6LSdZm0S9M1uolicbZI35Dqeoro9V8Gad4gshqvh3bazMMtauw2MSP4SCQDz0zj6UgOEtNSvLC7iurK5kt54jlZIyM+49x7Hiuq086V4tsrtry2j0u9tyha+hz5D7jtHmJnCZP8S9zk+h4/U7C70q8eC7tZrc5JCvzxnseh9Mir+mTrD4d1MbzukntwAD1GJDnHpwP0o3YF+5gksmNtOhSQqHXII3KRwwz2P/1uoqFSpXBGfSmW+pzfZPsplWS32nZFMu9YyeNyd0P0PPcVdis49RnmGledMsWC0DgCdOMn5Ry4H94Dv0FbKempk49iqFVwwfLLjAFNkJdIbUEMEJZXYlXTPJww6jvzTs5XK8j1Bpv2b7QJHB5jXzPy/wD11T2J6nT3fhbfZw3uhXD6vbudrqFCzRkHn5erYxzjn26mucd41tLnEgLtLEpUN02hyePx/nUmhX2oWurQGwlVJcPId5O3bGpkP44UgfX3pdW1G41S4tpL5I7mOAAGMMYnchSoJcdTyCemcY4pK687D09CqrB1znk9KYqAZY5Jbk1tzaDDc2LX2iX6TxxKDNb3REM8PHJbOFI685/OsOC4V8nPzKdrKeoPuKuMk0Q4tFhWaOZZFeRHT7rxyFHH0Yciuth17S9eiWHxJaRmfAVb6FDu4B5bHI/DI68AVyZIIGQKQjGcUpQUhxk0dJdeBLh4hcaHew30DZKozBXI9AejHr6VyU6SW9wYZo2ilUkMjjDL9R1rorLUbiz8NGOGeWIyag5JRyMhY4/6n9Ktw+KZp/8AkMWdpqkSj5RLEFkH0bp0rO0krml0zA05PO1O3h3mMu+AwUNjgnoev0qoo4ANdfaaLpVxex32h6u1rdFyI7G7jDAMyHgEHJGNwzyPr357UNHv9KWM3dtKiyAbGZSFbIzwT1PtQpag1oUzyMcA0yRVbBZVfB4JHP4HqKTdk8dKcG4wa03M9iaKdmlj85IbqNQg8u5Tc21CCFWUfOowNvUjB6dKtCTwpdTWFnqLXul3XLzTbRMmTn5VYN8q/VCcHB6ZrOZc9OtNMhHysoKnqDUuknsWp23Ok1bwJfadF9rs54b+12GQvGwVguM5weox6En2rmZYnSRo5UZHU4ZGGGH1B5FW7HxBeaLYvbQSXUto6uj2pkLLtKnOBj5cDuPTnNdPeeLIdWlkbVNDsr2DaCmxiJ0Pcbzxj3GPpSTnHRg1F7HG7U9DRW+bTwi7FhceJIwTnYIY22+2cHOPWinzhyFQFZVaNCMY+YbuQe3FVmVI22q4PsOamEEkki7FfzM4ATr+XrXWaF4C1TUpYXvLeW0tmzukk2qwI6Aqec59qtpLVkLU5AwuA2A24jj39q09I0K/1XcLW0knI5Z+iL/vMcKPoTXcRaFoOj3Ec8Nu2qTZ5lu3AiQf7KAfOfrx3q7feJXfyhENixMCvzenoowAPb880rNj5kjEtvDGjaDcwS6veS3s4+f7NbpsiX6seX9sbR61oan4wEkRhsohaoX3My9W/oM9e596wfGN2k/iV5I94eeGKR1HXJQY/TArnftDFgApB/OpWquwv2Ni71CSV3bzGJJyzMxJJPck9fxrPCmSQOXYMBxRGrs5O0DPQtyf/wBfFPLLkY5bGDiqsIRZAmd/B9c9adCGvXLE7AvzZLAYHGTk98ZNRqqscgDcfU8V03hvw881ykt/F5emujhnZirOeQAoHPBwc+1HQFuU7ONry/aytIZTLkr8keMc+px79Txx0rtrWxtfDWly3t60F5cqyqIyvyxk84xzk98nHsPVlxqdlpcbJpsEcSgBS38bY9T1P+elcfqOuG7c26tiMMDgHg8EY/MilGPM0hylZOxa1LWbnVLhriVzyc7QcCsxVJkZi2XXoPanxN8mDz6U94+Mk4xzmutK2xyb7jVh3OG55GelaFsk9zc/YrazlneRNxaMAfdZQMk/7xOPQGpdI0O9vis99JHZWbEeXPKcGUf7K55+vA+tbs1/BpafY9Ikfy/45CPmdu5JPbgcDFY1HfQ1guV3YkFpa6EGkvcXV/wxjUkRRYHGT/Ef8+9VPEWvzyqVJULtDRhRgKCoOAPXDcn61nX91PPmWVySThixyT/kVna40rRafKx/dm0UbR0JEkg/9BCVEUkXJuRlXczSlHOScVp6ajahdQWSRb/MPzk4IVeMtz0AA71Q0/TrrVLmO2tY2kkY8KOw9Sew967tEt9B006dbqHlU5nmxy7cZA9FyPxxmtW7E20HyvBYRmG2eR8jaZn+9J9fRenHfqcmsO9mMkqsRghhjB6cjNEt0XZSCWGTyT2otoZb69WCJd7NuHHHVSM/hUpWE7tk8/71hGEPD4GO/Tip1H2TLbCLkcD/AKZ8c49/ft255Ek2o2+gX0ce1ZLjftnl67M8Mq+mOeepPtxVa1l8+1inIwZUVsemR0qEy2irqiSyWMxJZpPLbbzzux8v64qGKK3itW1C8BNucmKIc+eev/fIPHv0q3dMBCbiRVNujBdpOPMbrj6Y6/gO+Rhz6gdUuI9vybMfd4CgdAB0AFJscUTRlNavLm8mZtyYRcMR2PH06VLe+HZ/sYuXWJLZFBHnF+vGCu11wegGcg56VY0uzstJ0MXOrSPFAoDGMqQ8jEHAA4/yD0Ark9S1i51JpFaSb7KSNsLMSMDHJ98ioZdr77EkFtNA+HaeQnHDE4PvjHH4VLum37FCoSScLgH/ABrKhhijUskaKCcn5QK1tNvGtYnKPiSVQqjkEKDyQOnXA59TSbbHGMY7Ia6vEV81DzyOM5FSLIrHhWA/3D/hT2KyNvLM5POW60EKAME8e3WhDH+ZsC4jkcnrtA4/MitK3vIodJvVBkW6kUKiHGCNwOGODgcdRn6Hoc5Hxg+2ck05dzn5FyPYZpgKmSSHfZkZAXnLfTjj3/SpQAQNpIb3FNZgkY3yKin+8wXP50pkRYfMDgqSR8gLnP0XJpAOiUg5ceZnqpOB+nP607UvsCXVrbLC/wBoaDzXbznI6kYAJx/PtjpTrSSAsJHYkYJ+6QP15/So9Smnv7+CXzlMcMbRqGHJG7K/kP50mCRCY4BMHWztyTjcxX09BjFW2upWBaPdHGTjaowP0qm8TupxJg9iF6UvlzSBRNeXMu08BnAA9htApXHYne6nlIaSRn292JOKZ55iBfftHc5qD7NDuxh2yf8Alo7MP1Na/wDZFpfeG7ic6fbGa32/P5ALbdw3cgccbqLtIDKGvRF9gumkb+4jFifwFP8A7beQqY7K5+XnbtZM+/zmnIAilQoCk5x7+tAAYnIAPbFO7EWU1i+kUyLZxQy+s0gJ6+xbH5Ut9fTX5sWJWFoEdJdnz79xyMZGFwR6Ecniq+3qTimn731oAkbInLLLOIyD+7LAf+PKqtn8fwqZr0vH5Rt7fy8YIMKsT9WbLfrVfuDnA/nSJIm8BiSpODtAJH60DLo1W9SHyY7mWOHtGjkAfgOKdDcS3sDafNKHV2DRmQbmVgcgKSflz0znvVIT2pn8oNuYDOzcA3Az0rV05rAxl3XLA5CyDaP16/WnqIzZopYJnglheOVGIYScHNRFnHbp1NXdSiRr43NvNE26NQ8e5vvgnJHUcgr/AN8n1qizTFf9WmfQvgD9KLiHwXjx7ljiiLHjMigkfnwKVria6k2C3csMZJUAH2A4qDy5C/zPAAp42KXB/E7cflVqaSGRMCFVAH99gf0IougsX9NuU8oNKIyEbHCgEfkP1zRdasc+XY+VHLzlZx989iGDfzz+FVYNMsb6FlhMi3AGQJT5it7c9uOvJGarJmNXiEIgYMQ8aptwwPPApMaNmLUXMxJSHI/idNjE9CeTj9fzrwDxNpZ0jxDeWoi8uHzC8ABLDyycrgnrgcE+oNeziMyJkZ4OCfesbxH4X/4SO0WNXCXMWTA54XJxkN7HA57foZmroqMrM8boqe8s7iwvJbS6iMU8TbXQ9j/Ue44NQVibBRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAdl8LrmaDx7ZxRPhLiOWOUYB3KELY9uVU8ele9/wCieev2mFHYA7TnDD3U9j0/LmvlqwvJNP1G2vYQrS28qyoHGQSpBGcduK+k7a9W8hhuofnt5o1kjcAjKsMg4PTrVxZEkabBdryROJYVGdxIDL7MP6gYPt0qv5x28DH15pYpWhkE0T7XXo3f8aikjjZnaCQLcv8AMLeQhUk9QmB8p74PBqvQkPMY9W/Ko8pljuBPfvUUdxNJGSllOhBAKSqIzk+zEVBdDUCi+THahic/vJGO38AuD+BqRl0EEcjn2OeabwpweP8ACs2G3vjIDdah8pzlLeML/wCPNu/kKd9gtGG6VZJzkgGeZnB+qk7f0oAfcX1pA4ElxED/AHQ43fgO9NEskiiSK2mdcdWxHx9GIz+FWwVW2McEcFu3RXjjCqPZlUcjp05A6ZqvHO7KizwtC7AkA8hgDjIP4dDg9OKYDHF43EYt0yOGYsxB9MDr+dMa3eQSGS9nV8jYbYLHt+u4MT+YqeTKk7sgeopobrnA9hQIqx2ESXa3jvJLcj78khGZD6sAACfw5wM5q04WRNrJFjpgRqPzwOaaeO3NMLfL7fSgYrMxYgsSR6mo3Uke9B6ZB4pN3yDIPpQIicBgcKM+9PE4lgW1v4jc26vuCM+GU+obqP8ADjionkCDJPTrx0prdMqdwIyPcUDJrjS4pFeTTJFkhj5dJpERgM9BnAwBwc46HBascncSRhvXB5q4rFfmUkY7ir9vqVlKBFqVtG3AAuFGJB9T3ot2AwigUYxwB0qGSLIGBjPf1reuvD7vGs9jOl7bnOGjI3dcYx3PHb8qy59A1RozJ5BiAPWWVIyPwZhRcDGkg+U/MPpmqskZAwhO316VvQ6HCiyNqGrQRYOQqfvSR9F6VFJb6ZHKyrdXE9vgFdqBGz3HOaFcDnZIzuyVxx+dS28stjKJ7aSWKUD78bY4966m0Xw1GgFwGZiDlgZTg/TA7fWobyTRTC409mhlz8rPbhgeuckknHT0x6Hs0mIv6bqFt4m8OyW2o6TaTyGUpG0Z8uX1yh2Ebs54XqQcjkk8bq/h2fQoGlS4E9ozjeEVlKMAfvDBXjJGQT36VbuIJ/s4lnVApJAZJVdRxxkA5X8QKTT9S1DR5hLayuiHmSLPySj0YdDS0t5j1OXE7KV2jjqfarMBWO6NxGpS4+9vRypT0II5Brp7bS/Dutm68yQ6ZcbQUlciOEnGcEHgdOgbJ6461iatpN7pEnnXKeZCSEFzCN0b8ZGH7nk+/H5PYNGXrfV7W/BGqRslyzcX1uuWYn/nrHwH/wB5dreu7pVlrDUrUebb2jXttcRMFubVTNGAR328qR0wwH0rlUlDAyI2NrjHtWxoepXmnTyS6feSWxHDhP4un4f/AK6ak+hLRY0Ni2sRq6Abra6B56H7PIf6U1MMnIGexre0S+0i71kXGsJJBdSbkW4tlwh3AqxdTkA4J5GBWZcaXcW7ztCpurONsC8txvi/EjO0+xrSMrXuZtXWhTIEihTkEDAOcEZ64PatVb+wurFbK/08SSfKv26PAuFx33cbh65yT9ay2A2Zxx15qMsA3NW4qRKbR0GoeFJbTToL21vY762lOEK8Sc8jK/1FYZDIWVgQy8EMCDUsc86xuomdEYfMVPOe2PQ+9dRc6/o/iCW2bWYpIY4IWSbYo378fKQwG7GM8epHBxUXlHTcqyZjz4Xwrpb5/wBZPck8f3Si/wBKyxJ8uK7SXw5puq6BZ2Xh3VVuXtTI6RzyDMhkIYjPG0+xA/CuKurW4065a1vIngmXqkgwR7/T3pRloNoUuSOOvaugtfGGoRxQ2t2lve2qNl1uohIe445APBP3gT71zYPBzSh85PHFW0pbkptbG9NpmkaxNLPp99Fpk7uSlvdELFjjoyjC9T07KPl5JGbqGi3+lASXcIMRJX7RAS8JOcY3YGD7HB9qpeYFIGBn0PStPRb+9bWNLtrfEqCTyvIZdymN2XeCv93Az7daTi46ormT3MpyAcZ5qNs4wR+NXLOwsSYEur57d2YI8rAGOMAHDEAc7iQOoxjJJzkW9T8P3unS48y3uV8n7Rm3csRGW27yuAQN3GeR70lOzswcL7FPT0Xdc5GcWsmBs3ZyMehx168fWoFJjJA6DrVzS7+OxF5PtDGS0eEDPUuVX+ufwrOYj33evtWkdbsiWyJRdPjv+dFQbov+emKKsnU95sLXQdACtplkHm73U2WbnrjccjPtj6VSvNauJpnV5ZDuHZuB7YHSsibVCzAO6opGOetYN5rDLJshBLtxnpWRWrNbULwljtyWzg5NU0LyLuUYweh9arwsD87sDx0PerlqiNG7qFI9zVbCsQ+LYpjNp15Ky7bi0CAL6qxX+QX8zXObdpD8nHtXXa/G1x4e02R8KInmQY64+U8n8ao6N4T1XVtrrAbe2/inuPlUfQdSf0PrUrd+ozLSQPGzP8uSD7VsaV4S1LVW85R5FqOPtEy4z/ug4z+g967bSfDejaVE0yJ/aF1EhbLsuAQOcLnj8cVna34tRrRJFcorDcuGzu/Lt7fzpiY+HR9D8OxGVgl1OBkvcYKof9lcevrzWHqHiK/vpn8mZlhzn5m5Y9jjp+ArJe8lvSWnZhGeQue/uKjeZVwo4DcDjFDdhpPqQ3eoTNIW8zk8ewrLzceaJIl3eUDM4I42INxP6Dr6471Kp3TqcDk9CcV2HhTQ9uj39/qgdYr6M28canBdDgkjPQcYB/8ArURb3KdrEGn2Fxeyi3tYzJIOuOAoPcnoBXS2un6RZPH9stzezq+0hwQin0VWwG7ct6dqYt4IYEs7GMW9uuQVjzz7k9Sfc01l4yByOQa0lO5io2M2+1l7q7ab5TI6jnqQp5259B6UnnrmPDY45FZEsSwybIduAzDgcAAnirEcsqRKjIzyEhVVVyzH2H9Ki5aRJqkha3bj8cVPpnh6+8QaVpuCI7YTztNOx+6n7rGPxDgCte10F1hSfXAYYiR5dop/eyH0P90cZ9cZ6YNTXmpNFbCPEcUSDEcEQwiD0x3Pv/KnurIV7bjXksfD9g9jpe8kgeZcMf3kxHqew9gAKyYLk3KySO7M46ljnGax727kmd8ZJ61raHDPf7LaO1m2eUd87JiNW3E8t9COPrjPZpWWpLdx1rZS3t9tXAUDLueFQep/z9KvXVzBb27WtiSVPEkzdZPw7DPYfqeanknt0sZrO0YmONlZ5SMGQ8jn25GBWPOhQcN17Y6Cs5Sbdi4pGfPEMkjI3HNXdMZX01AXZYISY3KrycMQFX3469gM+gMKxyXBkA4iUfPKRkJ6fU+g9s8AEhZbsi2jt4fLWBS2NgIByeSSepOBk8Z9BTvYLFbXrpbxtsDgRwERhE6ICoOP89cknml0TTA4llmYpFCu8qF+aU44UHHA9ap20DarrMtrGClsdrzTuPlBUENjJ7gL/ga7DWdUSO0+w6ezDy1wWh4z6jHU/wCPrU3stCjzfUtXn1nWpheNMgidlji8plVPXGQByepOCePQUsVtFMAVkkAx8pKgZ/WpLiEC6diuzcSAzDGf0puGK7BJgD05qF5mgCxkXnB2jqd2OKvuWkhjiXYsUS7V/vHuSfXknnjjtVaKFwuMk/hU6wyHqpX0yRzTAsRNZxrtkjlduhdHCZ/AhsfT9aVlhdvkM0YxjIcE59fu1EIpOMIxB5yamiTjjOO+VNFxDJbNbho1LPGB1Zcgt7k/4Yp8+jR2nks2+SOUFgZZGckZwPvE4zhuPapY7VpXLbWCjHO04z9a1/ENmltd29opEYhgXeGYb9x5JPTrkflUsZg+UkDbYUSMYx8igVMdjAYU4H3j1JFKLnTbV9tzdwqTwA0i8/kTU8t9pUUeE82XP8UUTv8AyHNAFVtu4lQQOwJyaTBIzjOOwNJLqsBO230y7cjpujMe7/vrFQNrV1NI0C6VHAVAJE03rnnC59D37UDLSt22nJNG8bSWbgj1qs895IApuIkQ9Ujgx/7NzTgI5AGbkgY4wP50ASiRVxy2PY5NdV4evLZbeS2V0MdzA6MHITrxjnrwD+A9K5aO4VdqbU2joGGfwp63DECRcbAdodeAD6CnpbURmWmqRyqscKNPIFG5bdHkIPfoCetasVvqUylo9PlVR3mYRY/76Of0pJLguNoaTOAGYvn+lR4DKAASFOcZ70hl5LNWUi4v4IH7KiNL/LA/WntoclzMscGpvEh6O9kw3H65wPpz+NUBls8cjoDUsE86DklCPujd0Oe386asxMgjsY4L28tZ7iW5MT7BvkwB8oPGwKc89yaVrW38sI0EbgHgyKHI/Fsmr2q3Nn/b8LWhjK3UAYpKNuxgR26HO7sP4fbi61jE0YeaaMDu0QJ/+sKWtguYytsj8pflj67RwM/SnbifvEn61NcwxRoHEZWJhgO/Ab8aILQzRLInzRH7pRS+fpjNK9gIc9+vakO7d1yKsb7aGUo8OoEggYSzkbH1O3APt1pZLmz3CNbeXdnjzT5efr1P4YB9qd7hYoTSJDCWckDgcAn9KnWMYxt68HIpzSzykJBDbRn+EqGlOfx21bRdaKbS20cYkVFj/LJNIY63s5InjkUAhSGwDirOq6ROq/2tFzDOP3oLj5XyBkex6Y7bR68VIJbuTdH+8uHJHLkkj8sDH4VfhstRj05rRJ9tu/3o3cvn8Wyf1qvmSY8cF2zAJExz0OP/AK1SPBdwWzvNA/lrzu2k98YBxzz2rSj0eaFCsd0sKH7wjyufqBTJNGjgjO64h+bqz9T+JNFwOS8T+H7LxFpo8mB49QjH7mYjZ/wB9xGV+nTr6g+Q3lnPYXktpdRmOeJtrqSDg/UcEehHBr3z7GvKrIpfsi5Yn8siuW8Z+HEvtJnunjEV3axNIjtkblXllPHPGce/pk1M431LhK2jPJaKKKxNQooooAKKKKACirD2ciafDekr5csskSjnOUCE57Y+cfrVegAooooAKKKKACiiigAooooAK9/8DyyX/gXTJJHIfyjEdvGFRii498KK8Ar374exvH4D0xXRlbEjYIIODIxB+hBBpomRtrplsrEyebJnqs0zyL/3yxI/SnxWtvbAiCGKBT2iQKD+AqckscVEzEDjnnBx296ZIuwJn0/OmMd3HUelRbHQkRSOSeSH5z+PapIzKR+8Rd/tTAbsUKARjHIAHSm5aRTuDDnkVKC3O5R9M0hI54oAQKd3JH1xSGNDt3gOB0Vi20/XBB/WmsxBoZ0jh3SyKqLxljgfnTAaRdy3B8qyMkQDFvKIJRQBzt3FmHXJAz0GD1ppARiAuHU4KsOhHUEdjUEeq2V00qWrm7dBllt0MuB77AcCnEy3twrGRrdwmxXbaRgdmXhuOeevPtiiwiQuTkEAVXLBc5OfSoB9sYv58SRrkgNG+8nj0OMVImB8rb2IIPGFyM9xz+eaADeQMEHHr6VXYFMNLOqxjnA4B+uamuYop1ZfK2g9CJXBA/Bv6VUXSbOLcy2sbMRy0g8wn8WyRQBA+pWe7EMwmYdogXK845I4HJxzSm8vfMXy7L7wyC8owfr1xVtbfyo9qfJznKcYPrxSLLaRl9yiNiPldCTHn/aHOP8AgOPp6HoMoSQahMm57iGAnqiJvx/wI4/lTDpy4/fT3M/OcM4X/wBBANab58pJWRkSQZUsPlbjPB6GoxjHyjPoeopARJPPDbmGEiOLOSqfLk+px1NVXZx1q0dvQketRtEOx4p3Aothuo+tVpFCjAHXrWk8eD1zVZg27oppDKTPtOChx64qtIPM/wD1VpTQiTABANRiDb1Uc0AU4Zp7WTfDI8bDupII/EVJLcx3Xn+cXE0rBmnJMhznJJBPXHpj9TU0tsByVYGoxBGQRtJoQGZwXZJcjBIDgYUj156VLDd6hpqSCxnaNGPzgchgOxz1Ht0PerrWEjKTHDI2BnAXqP60kck01lcpNplxI6zHdexxkmP1VlIAPf8AiB574p3EYr29rdSvNM0sE0jFi4QMuT1zg5A5PQH2FSSWbacm7cptZACsqtvDOMBunKgnJGR0+ua7Lw9ouhG5D312LwScqLhDbhRyOAfvnPGASPrXS3Wg6fb3ERt0ijSfCSQqAq3A7BhxuPoOvB96Em+gro8rBYkYPP1zkVe07VLvSrkzWkhRnXbIuAVkX0YHgijXtNbRNQuntoHFiGVkjB8zyweuSD8oBzjP881UsXS8YLFkyNwqdycZrW/RkNGnOljfzqNP22crLmRLmQiHdgH5Hwcc5GG+oIHTOvbO6069eG8geGXjAZeGHqp6MPcZFP3DPB61o6drMmmwtZTQreaZIPmtJTkA+qHqp+hFPVbC0e5lRMTGD6GnWzOJp1kVSgcY9xitJ9IF3dldEWWYPCk4tZWUSoGHQcjfjHYZ5HHespYLuC+uUuYHiZWEZDoVw6jJHPfBHFDmmJRtc0Id0dwtxC8kcy/cKuQB+Xt/Ouwh8a2MmnpY67ZPcIFwxkVZVc+3Qr+FcVGxDspyCPSpGclee/am4J7iUrHQTeHNB1e2E3hu6+z3B5+y3MrEMB6FhuH1PH865O5hm067e3uojFMvDRycH8Pb3q3HJJHPHMrbZE5Vx94fjXU2HiSxltvsniO0hvYScLIYQdmfTHK/Ufl3qbOK0KunucI6gnIzg+van6Z5javZRpIYSbqHa4XOMuMcHOTXcXvge3vrb7Z4XuvtEbHm1kcZBzyFc4z9DzXDyLdafqSr5ckV5DPGdjLhkYMCOCODxnkfpT5k4is0yRsFQwb5vUVoWOt6nYNA9vdOph4RX+dVHIIAPQEE9MH34FZ3J5yAe4HSkViFJrWyluRe2x0tzrmi6vFKuraVFHezbgt7ZxYKyM5Yuyknpxzy2MjkGszVfDN9pkT3STW97Z5Ki4tn3hcddw/h/kPXmswnPI6+tXNOv7rTbtLm1meOVOhViAevUdD1PX1qJU3H4SlUT+IxwlwwysDEHkEMOaK6NdevFUKbbSpCBgvJpcLM3uTjk+9FT7/Yr3e5rm0huFZ3GM5OfSq/9nbWyhJx3J6ipkjuBIFdgoGTyR6f/WrU0/Q7/UpxFbRGXBw79FX6n/J9Ka7i9DMRF+4w3Fec+1dRpHhy+1S3Ty1NvAOTNKOOfQdW/l71t2WgaV4eMc+qTi5u15CkZVT64A4+rUzUPFNxKSIWESg/KF6gfWhtvYNFuWBZaJoqxwfLd3EOT5k6h9rHrwBwfY449aoa1qkU6K0w3RxjID9AcY6dOlYUl67yEbvlzkA1aaPzkJPzArgZqr2VyGmUIdUmhujLlhIrHbjoQO2K5jUNsckdvJJuW2Zoi2euDj8uK3J44o3RY2PygfMDWFf23l6jdAj5ZCkmDyDlQSfxobuUlYQ3JygXO3GcipEhluLiK3hjaaZzhI4wSSfQVo+GPDdxq8yzyK0GnRnEtyTtB46KcHJ+g/8Ar93Zx6HoMM9tp0Jt2mRk+1O5MmSMAjJJ25wcZH40mU2Y2neFYNKZbnU3juLxeRarzFGf9o/xkenT69atXly8zMzksz9SaFmFxEH3AkjPB7VG6bmz7UIhu4+GMiMEr9DUkjBYiTgUMywxZZgFUc0+LRvtkLXusCWGwXlLUnY82CD83dV5HHBOOcAcjYWMCxsrnV9Zlh06DzQjAyTHiNMgdW6flz7V06SWPh8SfY2FxesuGuD0X1CDsPfv+lUrjW7mUmGNvKgUbUhj4RR0AGPas2aTjPU1rCnd2ZlOpZaFi+1svdrJLGxn8hSshbOfmIIA7fdBPrWPcX5mjiP3ZZHYN6gZwKt3Nt5kEVypkLIxjKgfLyMjP5NUuieHnvXN1cnyYVbmRjjnngZ6n27dT2y3ZNoF7yQzT9Hk1K8JZvLiUfvZWHCj/H2q5qOrQwo+l6fI628SlOvLHvnHUn9OnFXL++g8tLXT8LbqcqVP3/8Aaz3z61kvDGsyhUADckgd6i9y9iK3aTyJwwO5tpbjsrKx/QGpUC3K+Y0i29qHEZnYjBY9FUd2OenbqfdXkjhtpby5ZUskDK2fvTnH3F/qe1VbqWfVNOWaWJYsENbwpwIlByoH6Ek8k9ahvUpbajbvU41uHs7cNFaAkpEvGGGclj1LHjJ7dBWvoGjPqlo0880cFtC20EjG7jnH04/GuOjuI3kup5csWupPmjXoAx6fp9Ku20uszrEwmWC03mSFVRl49sjnoB3+poVrag3bZXO8tdP0rT4ktrZwlvnc0shHI9iR16fgfpWFNcWZmlVJHYBiMqjEkf8AAR1rInndLR457qdkwQFEuOD2x/n9ay/7SnnBSJmCgYByCT+OKh8tyo3a1NO5j06ZyjsyYXOSu0/iGwc1UiTTSoRLSdpem93QE89vT/PNU1OCckE+9XbGFp7lQGVDtOGZgig9sntzS6l20LTafbJB58whh+baIzcsWyOucRnnp0/Oqpe2jz5GfXLxk/qHH8vwqtqMzza1MjFJDHtX5GGxSQMgHHXpk8ngZzUZDxAZdcnuCD/I0Oy2BeZoJe8ENCoOPvRnBz9G3VFDczQzbhO7rk/JJtwR2zgA/wAqoQyyXUpSHaW7s8iqi/VvX2GasXcSQOVimWVQPvgYz+dAFpb6UMW83JxjOagu8anctPdolxIxG55VyTjGOvpgVXUbBuY5yewzipTcKozkY9aLjHXEclxn/SZkY9drd/fPWqf2O9iKlLwvg5YOoA/DHP8AKtAMuPvDngZ6/hTmU4AJUDPGTilZ9AIrdJFbMpRu5IzwKf5MDamw81zN9lhMi7CoQktxg9TyDnoc/iXQ3G2GWKFkLMpAbBPb16Us91c2uuvcWxjuIWhSM+fKzMu0k4VuBj5vQDuByaqwr6lw6U7zNMimC3Cgbpm29O+OeTUa28CH97dBsdfJiOD+ZFZ15FJqWoJLcv8AuhIC3luVk2DqMjj3H863LjVEwq21nAqA9GhByPz+n/16mwyutxZxNzFK3HGZNv6AZ/Wqz3JIC7yFznaDx+tJdIt2+SDFkjmHAIHccgj9Ks/8I5ZPotzqCHUJ3hRiUkm+RSBk5KqBwCDyRnHvijZXEUGuIt+1cuSfuqwz+pFW1sL4qWSCTB5+f5B+ZpFc20DW0GEjxgqnAb/ex1+pqM3Mk+0MzP7E5Ap2B3JyhjG15Yg/PG7P8sinwi3ZE80qGU5JRC+76kkVWaNZAPnIKtwY8DPt3qRU2/KPr1zTA0L26tdRsFsryyS5gQs0TGRojGWbJKhW4PPXr1HQ4qrHsgjSJTIQh73UvI9CN2D+Oc1EqcZP86UyDggAijQRds7uCwd2tbSzgeQ5ZorcKW+pGM1dOuzRtC8sBbcCCwYj8geP1rFDuTgDjtxU8UX2h1V2IbPyknpRoLY6y2dJytyjsMjblWyMfSuW1u0OnaxPsEkkM5Eyb49p54IHPI4HP+1XSWETW8CoQCM5DAY/OsTxjJqJs4pPIeS3tZfMMkIBxHtwdwznjOcnjAz2JE6Ior6X9u3SPbpgH5WBfJ+uM/0rTNq88QNyZwxG4qBkDvjpzXNR7ANyM3HQq3H6U5rqYKWldkQdSX6fWqsI6eysbaOR2E74B4z8uD9e9aG6J02mSMn0LCuItv8ASy32dvP2nB8tc88Z/mPz960bfSr6Zgoj8tcZ3ykAD6gZP6UtANe4imikLOJfI6jy5MKPyGf0qoZrIOfO81GOCVbDH6Hg0h0XUYGia3nspRn5t0kigfgUqVtO/tBpYjcwRSxMFbyo3kOcA4ywQjgg9MUXQWKd5dwxhfskjKM4YABT+mK4nxzrT2WjNArnzrzMa55IT+I9MHggdvvZ7V6N/YOmQRk3FxcPtHJMqop+o25/8erzD4uWFtYjRjbJMBJ5xLSyFsj93jAJ46+gqZS00KitTzOiiisTYKKKKACiiigAooooAKKKKACiiigAooooAKKKKACvqGy02O30yzstMUCa2hWJkc4E6qoAIxwH4GeADzn2+Xq+o47Z1VpJ7jc56JbqcD6O2D/47VRREhqNkcgqw4ZXBBB9CDTXAwdwBGc89jVe8glvwrXN3cLMo2lodqkr6EkHOPXrVB9CgcgS3F5KoGwh5+o564A9feq0JNISxF9gdWf0ByfyoaKdcyKtwQTgIwEYJHpuAP61LFLLBCIklYIOioqoB+CgD9Kg43MSp3HkmgCvczX8KZjsld8ZANymO3cE+vp2NVI49eut7G6sbVtvyxhDJnp3OMd/atHIHGBzSbx90ED6UahcbeWwuI4/MjkjBH8EzxlvckN/Liqo0mwikSUWcLuBjfIu9uv95sk1aN8lqhMqxyRk/OCvzAeq47/z96S4uolmSOOVzHMR5Lsv3uvBxnB/n1FFgTH7gwCscY6e1RkfOMbcd80yeJynyuQ2etNSMq3zEN70tAHXALxlUYIxHDYzioVLIgV2BPrjrVgEbiSVx04pGBDfIy7fpQMrHrgcn29Kd5m0kMME9qkm2Roc4OOSTUIkQgNjaD7UxCsQ+QeeOlRtGAMBAB6etTHYex474xTWVQAzMQKQyEgrC8Q/1TjDJ2I+ntUDqyMotfnJHzQuwH4qxP6H8znFTmeFyQrB2B5VDk5+gpMNjdFbyNnjnC4/Mg0XAqQXUF0CY3+ZfvI3DL+FK+0HDMAT6mry2/nAfbkDKgxGquSy564PGPpgio49Ms0ZnaOSTvtlkJ/ligCmUHsKoT3engtFLcQE9Cu4H9K1JrC2cqDbINvIHJH6k0kcMcP+qjRCB/AoH8qAKUaSToWgt5nXqHYBF/NiBTZ7XUYLaGaS3ihSf/Vl5d2fX7oxnnpkGrsoEg2mVlOe2P8AClE7xRsgk3Bl2uMcOORgjoev+GKAMN7e5JINyqn+6kQH8yaXyYQ0bOhlKD5hI7FWPqVGB+FbkVhZ3CFYLlLSXdxFdbniPPZx8yjr1z9ayYke4ikkiiaSGN9hmQHZnsMkAgn0NHqMV9SulQxQyC2jOcpbIIlP12gZ/GqMjOX3li7Y6k5q00R9PzqNo1B5HX0p3EQfadpyiKjlSGK5GRjB6e2atQaxeRxPGs5CP94SRpJn/vsH0H1qtJFk5UDPtTCABgg9fSndisdDZavNcQiCSeKIKcky/Om08EAEMVB+XgccdgKq3/g6zkY3FlfQRoq4GGDIxHXJUcZGOQOvbvWQRwAG24ORjr+Yp0N9LZzebA8kUmMbkYrn646ijfcNjOuibY+UYmKoxKOVwWUj/wDV39aSXeVI+8cHBXkGu/i1fRtVszbXvAkXY6zjex5yMP14PSsjXdCNgE1C12XFkUIeLAWRQR1OeG+owfbvVq6IumzkLm8ne3tGkYqyRxqGzyAFAGPy61fsNakkvIbq+jXVVjTyPJu3JJQnqGzkMOcH/wCtVY2brpMMYiYOmRtYYYYJ3cexB/WqMXlfeKkt0yKi+lmXbsdqdIsdX82fQJzmP72nXDDzo/8AdOSJB6d/qawpo5raUwzQTQuCeJImTp1xke9ZsU0kMqFHZWGCrocMPoa6ePxheXemy2GqRLqMRX928mfMiYDhsgjd+nU9c1cZNbEuKZjqz4ZiSKXzACofLHJ46g5FTW9vdXMxhgtmuQVBU2+JWPGSNq5PH9O3QRNFtfcBjHy89ODjmtYyTMnFo0NDuLiw1E3VvcyIyI7rH1V9iM21u5B2+ufSn+JfE83iTTVtr/SoZ5l5S6gYpJHgHCnJ+YHPrUWngm8AxwLe4YkenkuP61kt/eDED0pOKbY1JpI1hpKXtiLvSZXmVU3SwT4SeL146MPcflWWAVXDBlbuGBBH4UtvLLBKs0MrRSr0dTgitRdT0uc7tYt2UsCDdwNhwSepXBBpq8V3J0ZkAAcinbigGTXQt4Qubi3Nxo9zFqSAEsiYSVf+Ak8/gfwrnZI5FYpIrI6HDK6kFT6EHkU1JMHFoTfH3YUUeUp5wtFVcR7Jo/hKO5UT6pIpQthbeGTJYgjgt0+oGR710d5ew6ZZG009Y4pMcRpxsHuex9q5PT9cnvJ5tOiCMrwuI1BwyyBCc56KBgdDnrVezvYpUcxYOOVwOCD396ycddTS9lZC6lqRefEnzMfl3EmsmW7CKS5JJ6Cq+rtJ5oYdjkgdcVnOJLx1G3btOQO5qb3Go6XJZbuW4JCrtHcDnH41v2ksnkeXyW2ZUZyKyIEeST7PZwG4mOQ4XJIwRn5cZ6H8OK7HSvDUkS+dqMzW0OBiLcGkbvyeg/DORTW1gdjko9O1HVb54LW0aVt3O3hU/wB4np0rr5/CWl2V1FeX00t3OEQG3YgRblUD0zt46d/0p+oeJYNHtvs1nAlsvJCqQxb3PHX/ABrj73xRJexTYQq0v3nySfr7n61UU7CbvsbOu+NJIy1vajCAbcJwo7YGMcVyU+p3F1PvdiSPXms+WTcwBOc96sxhMs6jIRTnPT604iasdtpk/m2ETHuoBxjrj2p01w8dwkcMEkzOcDYCSOw4A5PtU2jaXdzrbpDbOLea3R1ZvuoxRScnPTcfr6A4re86HRLV7awlMlzIf31wRjPsPT269aJbiM17QaLZPq2poJruPaVtlYMluGIUM3ZmyR7DtnGayby+luXeR5GbnqTzT9XNxPpl4kbEu0TAZ53HGeaoxsJrdZVOcrux9aqPUiQsGSzfzqUgHK/hVe3zvOM5JxtAyT7CtdYIdPLG8RZLzgrbt92Pvl/U+i/ieoFac1jNxuOtoVt9Of7XuVZirxIpwzYzznsvJGep5x61nXd/NcjZv+VRhUUYRR6Adv8AOcmi7u5JZHdnJdjuLMck1QX935hZgF5Yk9hUN3d2aRVlZGja+VMHMYAVHdVUdFXOQPwGKVlWVXkeQw2kJ/eyDqT/AHV9+n0/IGvAjrdTqm8ROI2LoucZjHC54J4/DvV6a6S20+G2uQFjciNUPIXIPX1J7k9Sah+RdupjTRSeIr9dhWLToBsSJOhA5x/j9a0sS3BMUKGSZvTnb7n8j+VQ3V3Bo1iI7UFiV2JCigk7up6j/wDWRVNLqWGzJkkVQyYZ24GOcggdevfNTawbjG0tDqH7sz5KZNwrKoJHGOOfXOf51rfYlt7dnRN8wUYLHJ9AMmuJh1e5s1e3gmWSMEsshQg/Mc85PufarQ1m9kCsZNuByEJwffr1+mOlJ6rcaVtkaVxYaoVJcKVbooIOT6U6PQrkg7jEoznDZrNfU7+RCpuGUnowGSP6VWkM9xkXbtKg5GJWH14GAO9S0u5ombjWdtAwSeRN3Y7hxikEltAxliuHDxAsWXqTjgLxgnn9KxPs9vziCMjOcEZzxirCyseORjoRximrJ3JfM9jIg1KHbLNKjqWdpDwSeT+fcCr0NyZxGY7O4IboWTYD+LYq0u/++QPQHrUqKSwHHJ9eKkogAm3BCsMR6ZkkyoP/AAHP6U6Ky1O6kVY4bU5OM+Y5A9zhKsLbmdiFALA92AH61bFu9hLGxnwjkqXhcDHynuQeOOcZ4zVKzeopXtoZl7Y3lrMsQvbYvsBkxCxC5HGCSM/kO3rxDBaMpJluXfPZRtA/mf1ojuvtV7dyONrecUwcFhgADJHU4HXvVlvuYxmp2ZS1Wo0QRHO5dwPZmLD8jxTgkacpGgPqqgGnQLG8mJZfLTGc4z+HFNkCiRkXJjHRz0P4dqTYJIl3Ajk84oBqMIEYkFsZ+XDdqN20EhRk+gpgShzuwUIAH3uMfSnGTK7AxHtmofMbGAoPuaC6pkk9RRcLEyncyjfjB7Gt3TUkm0rV7KBt63FtyM4weVAz77vrxXLidHG4OoUdecVu+HTbzzTI87SRyrsYRLuUjnIJwQOcY/nT3Vgempkq7SoHC4BHApYmkWPLMgx1OcUltDfxJsGmSuE+Te80SDjgHDMCc4z071ejh1OQAxjTo1PVmldiv1AQD/x6jTcRVSbcpKqxIHHynBzU6swwXXBpLlLhmKLqIbjloINnP/AmbNRJYW7HbcS6hcA9VacIP/Iaqf1p3ESNOBy7BVP94iq8mpWkIIadFbGQHO3P0z1/Cuj0t4AEtFsITCB95kV2BBzkswLHn1NSXCS6ddy3H2iGzhnjVvN8jLb+hUEFQcYz0/iH1pNtArHOx3kzwmSO0uMbgFdozGGz2XdjNXczow22gkk6lWvIUx7HBbn/ABqhc3c80sn2y4ZwzFo13EAp/D1xk8du5Na2lw6bZwrdEveTEbgfKIRTjsCMD609bBoXrZfEbjDR6daoMBcyPK2Md+FFaUNxdWENwupXcM0E6GNo1tsJz3IJbPGRzx/Ksue81CdUeKLykJwGMqjJPQYJ61RmsL5lae6LKw5Cbwd34Cl6sZVg0vw9YXUktrHcsS+4RtJ+7HP9309qvu1rcN9o/sqzklQYR5UVmXHoCP5VmwMI5wJIyVXn3rdt7I3sZkM0g3KRheB+X+c0NJIWtx1u+pTuA0iovLYUCnNe265RWAkPMjBtuSO3ToadFo+xm3XDbf7qgKcenFR3ekWsVs0quIjGCxZiSMY79aV0MvW97FPJ5e4bgoJXOcA+tYXiFpdO1JNaEUjWqBVvArH5xnCsR04z27U7TLO+W98+dkET9Zd2MjHHbnoOP5V0sUmy4jVo1mhJxLHJyjDtkHg4OD0pvyEvMwV12K/nEFjGZHYAklvlAPfjrXn/AMX455I9JuCWkhDSoXC/KrEIQM+pwfyNd5d2Fvp/iea0lXbbTASWyp8qqO6nnjsccfe9K5P4rRSReEbJVDLbi9UKrnLD5Hx+HXvUy1RUfiPHaKKKxNgooooAKKKKACiiigAooooAKKKKACiiigAooooAfFFJPMkMMbSSuwVEQZLE8AADqa+qCWYfNjNfNXhcZ8W6KP8Ap+g/9GLX0uwUcAn6CrgZzKjoVYmq8wmVCYY1Zv8AabFaDxlhgVVeNh/FgiqZKKsck+WEojxjgqefy/8Ar03z/nwUcbvunHBqcHcBlTn0b/61IUyfug4/SkMrTQrMPnLj/dOKdGiIoVQv5VMyEnBJH0prlMhRyx6DufpQAm0H0xVSezh2NDtUxyZLR44J9eOh61O0wjZQ0cu5zhRtPzduD0696WCTzi4AVChwyu+SfoVyP1H0oAzg8+nw75WmuLQdWKZkix64+8PfrV22uI7m1WePIRhlSwxxVuaFGtmWOV1kI+V1UfKfoc5rnR4PsmjP2ieedySTubamT6KOBRfuBsrHGsYVQqr1AUYGaicFCAwYZ+7kEA/jS6fpFnpTE2KGF2XazCRiSPxPtVllLZLOSeuTSuMrhGZ2Rk2nGQWPB/EZrOk+2oWMVoh5J3eZk9cdAP8APetXYM8A/U00n2OD04PNMRU0+K4Mqzzywtlf9UIcfjyc+tXt8yHKTGLBHEahc+x9aQTyDYJVLxxklYyxGOeuR/XI9qZBBLPIcZkO4kBFOQOwPuPY+nToAYjhux4zkcd6D1GVB9T0pwAZQVYn0xTGyBgjvSAOSueMehqPKoeM/nT2UEdqi8tc4zz9aAIyuTwO9RmLJzVkA/hTe5BBoApsnfFGwqQSKsHAzwahLEEkniiwDHjWTBXAPt3q1puo3WmmRI41kgcYkikGVYf5/wAmq+do4OR7inefGU4UDHXmmBbbTrDWiz2UyWl2QcWLrhWPYI+eM+n6Cufu7KezmMVxFJDIOqSjB/8Ar9e1aE0tu8Z3Ovpg8/nTB4lgEJsmEV9Gx/1JRpCD0yu05B/yMUWAySo9KaUx2/GtG60u9lPn2DxQWrAEJetiX/vnJOPYjPuahl00pvUasjnHylLbnPoQSAPqCaQyg0fA+Xj2phjGCX6fStJtPh89WE1y0XGUd1GT35UDimNp9pjc0Ckr0zIzZ49yaLisZimON9xkXg5255rutAv5poJWmURwBMYlXG7PAGMdOPxrnYZzDHJFFviic/MqZRW/AUx5hHkI8uDyVGcU02JxuXfEXhnT7uGI6en2e62/wFiJOnKpgn15GOnQ9a4h4pbWVo7uMpKT8r9mHSujkupiyt5wcDkJKNy5yDnB+g/KrJnt9bn+z6lBFE7AKsikLke/GM//AKvQU3ZhaxyhhcFWO0qw4IYEf/W/GmxxqZGJJAXk4OOBz+tdk3gxLeIXGnXKuAMsk5DI3HGD269/XtXNSRyWuoCF08ifB+V+Ezjsen/16VmMZb3N1pd20iZimBw25c5xxgj/APVXV3epaNqWlWU+pQG1uLneouIYcsSgAOcfeHIwDnjIGOtcTcPLcXDvJjcxLfKAByc8Y4rUnkf/AIR/SImwMLM/vkykfyUVS3E0bOlaJdzXc7ad5epQS2sqK8EqqRuXjKsQVPbB98ZrnLmCS1uHtpo2jmjOGVhgqfcUltutbhLm3leOdDkSI5Vh+VbS+I4LqMW/iKzN7DnC3cY23MXuD0cZ7Ed+OlWpNambSZgr8pxniiRd6YAOM1vXHha6eGS80eWHU7AZO6A4kQf7UR+YH2GfwrBTBQsGDDqOetaRkpGbi0WrDUbnT7lLi3kaORe4xz7H1rsbTxPpPiDbZ+ILVd2wIt2OGU/73UeuDkepFcEWIzhaTedxBBHTpRKKYRk0ein4e28jF4ddXym5TNoxO3tyDg/hRXnu9uxP50VHLLuXzLsdhomqGPXrK5YfdmTJ28kZAIP4Vo6SptoprVyPNgkaKRQ2cMpwR+YrnLNHkuo8cAsFBPrXoWr6ah8UzfZoZ5PtarKDEjFSQApxgY/hBJ/2quWiXzH0Odkj8w+Y2c9lrS0Pwxeao4kA8i1HDSsOvsvr/nvxXQQ6PZWpEuo4l6EWsf8A7O2ee3A446kVDe6zcTOYcGK3XhI4CFCjsMccfnUKLYnK2hpull4e0e8exiDz2qb5WflicAnGenA4A7+lcbqPiKeb7sjckH5jzg8/hWrpt4H1mO1QNsmDB8jvjP49q87W68yziIPLIqnirT5XYm3NqT6vdtNN5u8sMY9h9KzUk28Z7806RSVUAd60dD8LX2vzYgASBWw8pzj6cVNy0rGXGHkf5eR2OK7XQPCMlzAt3qe6308gMF6PN349F9/y9R0Np4e0jwttllVLy6HMYYYSM+pGTkj8vypt5qs965aRz1yOflX6U15CbLl9q3mILaBNltGAscSjAAHSs9WL5BGMUxT82ck9iaUvgHBOKozJflC+o7g96yrK0MkzWFqm54sg9gqD+Jj2GO5rRt4JruTy1by4RkyTt92MDt7t6D/Jj1bxHBFA1ppqIkDYZpEXDOR3Y8ZPHX/9YYWuXozYaJpt/Jbt9o1GGEyGbj5FyA2wHpgE89fpXMC6nurkyyyFmcdye1JbSrIWEoYrMrRSgEjcrDaRSw2xiysjBnQ7WI6Eg0JDYq5V+Tnn8SamijgeR2usLZp/x8OegU9vcnoB70+2gMjvIwkEEf33QZJ/2V9WP6dT705byLVJLPTrS2kgtbdxIyODlcepPVj3JyTUNjSOhzCkaGGFYo0UJFGvRVHT8axdbie5sSFOCJEYN6EMD17dDVi8v44ImlkbCKDxWRLq1zqBWS3gdYI12IsigEncSSRjPX346c9aVuoXKN1p9zqF+9xFdbFLkhQmcdOnpwKzruxHmGM7pfLGc787V79e/wDjXSWyTfvJpWJVFJbCkk/Qd/wrIu7uCWC5uVEsboCvlE7uc/xDjHOMghh19KL6icbLQzYoiS0gC7DjB35wMdPQcDtU4ULgqwI9fSq0G9Y1V3LAfdz1xUwZN3DBeeBnJqDZKxaUoRzktTsbV2+tVWu7eHPmTxqf9psU9byFwxRt+ODs5IoDYsZQffIUetNE8OflcHt1qk9vBJGA+n3Em5gVkkhOevZmHI49x+dKbF4n8wacBuOQssuGHvsY/wCe1KwNo0dwOQMCnL24PHfHFVlN0Ysn7PGxJAQuWIx34GPXv27VOCwUAuwcdV2YB/OnYTZYiYqQVGT9KnSRjLCyrumViVJ6D5TyeR3wO/U8VlCO7aQk3YVf7qx/1z/SpYoiHLSSPKRzh8ADtjjGetNNDa0KcMUlvf36ySBv9JdeBjkHB6jPUHrVlZQWCBhuPQA81Eul2aAYjfAJP+tfHPtnFXIooIMlYowSOqjB7dx1/GpQ3foRFWB2GNiw6gjFDtdKq+Xa7wc5zKoIx7E1PkbuOacPU4oEkxkKSPOoYW8aYBLSO5Gfoq5rQMNlZlC8Xn748llO0bu/BJPHr3/A1URdz/u1LMfStG4cLpTROu2aGRSMr94MD/LZ+oo0G7maxh2hRG7MOfvcnjpxge9TecDHsMMSncW4zx04znkD+p9TUKkkZAHPOaR9wHXHrkUbAOjlFr/qI4o2ZvvJGobP+9jNPlvLib/XTSybegZiQPpVZVBHLH8O9OARVwOe/wB7mj0AnEhH3ckntTfMZn2lwCPXtUSKoGF9OlXLeG38tWnuDg5ygTnP157e1O7AjHyswBDY7jpVm2nt05ZZN/c8Y/IimRQiZ22s8UajO8gt/If4VCIyF3EN6Z6f56UJiNGHVGF8JhEDgYXB6j/GtK6kOpaO7RwlDFJvHyk/ewOo7k4AGOSa56OMhgF5+grqtOsj9j8i4nKJIccJhgSe3XnPQ4ND2sLqcZcJ54ML7trc/KeR71c0eaKJiLuSR4F/1e0/fx61asrfTvLna7mEfkStC2+TbtIOME5wfxFQGz0CR/8AQp5nkByRaxvKv4iMUr2HYkN1GjN5OoXq7eg3YAHpgU8XgaEM8pZlIwNxzn61at/Dqtbl1WJmYfL9oZ1IPTkAnH5ZpsGjalBC4iWyWZiV3+cxUL6geXnPtkfWnzA0Q5sUO7zZTMOcEcfrzVuLV4H2+V5rEHDbFyPpnjHWmHw5Isoea93gDgRwbSD/ALxY5/IVKuk6bLDsY3MkmfmeSXa+f+AgD9M/WhMTJZtVYQ5iifPQ9DgfhmqVxcSSMjGQW6qeHuJBtzzyRxzx05rej0+O1Kv5Fs0D58hxFyFBxhmYkluOue/FX/N+XbngdB6UnLsNIx5tIuSqPcCZmABU7dufwJB/WpES4tmKxWs1wPul2lQKfcZOf0rQ83dn5aUEAelK7HZGddWc19aqXAtrqIloJEO8qSMckjGMcYx/IVwPxWtZrfwXb/aZzLKdRTBB+Xb5cnsOf8816hzXI/Ey0+1eAdRxb+dLF5cqYTcUw67mHphd2T6Z7ZpO7Q1ufO9FFFZGoUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAdT8Odv/Ceabv3bf3uSucj90/I+nWvelm8qVYLiRN7rujdW+WQeoz0PsfQ9eteR/B/S4rnWb7UZCC1nEqIhXnc5PzA9sBSPcMfx9iYRSxhX2NGG3BWwVB9hWsNjKe5C777gbJ1K4+4gJb68VE9oZnGFuFBP3mmZQOO4Bz+lXtpYBcnZjjBp5bC4IzTEYy2VxGzbXQEjAdpHf9Dj+dQCy1JpSJNSRYN2B5Fvsfb2OWLAY+hrdcAjI4qp5u4uFVjtOOVxk1OozDj02G6uZEvLmeae2bZJB55VQ2OGwuG56gZxU66bZWMsl5FbsJOTuyzsMjBA5J6VoSGaaELIsaupypHzlPxwDj1HQ4FUVvGhvpbe8ZFYOAhVTiQf3lPofTqOQaNwC3hW9iS5lYmJ1BSE9AP9odz7dB79auKq4CgDaOw4ApREY2JQAo5ycdjTiFz1pAAXtQVI78elJk9qAGPViT70DArnmo3kA+UuoJ4GT1qUg4PIqu9pAZkmaPfImAGJPHOaBCLHKrkMFb3HFP2ALjIA9zimNdxhijJMDnAxExB+hApfOIChFLBjjJHQe4JoAa8LEDAH51EztE+7lXXlWXqD7VZDDklc++KZL0zjj1PamA37ZBqF+FmuUt7kIQfLQDzj2Lj19wPr0pbmJre58h1xIFDfKQQQe49qz5bmzhkVpLmASZwqj5mz0xgc0yTUYpOI7S8umwRhYXQc+jHAHTrmh+YF7YGHCkf0puzHbJqhNHraMPIMJicbh9okCvH/ALLbQQT9P0pDbao7fNqkaIO0dvlvzLY/Slddx2NDOwYwPpVOe7ghJ8yQIQMkswAA9+ajXRYGYNPcXlyQdw82cgfkuBT00qxiztsbbg7gxjDEH2J5FO6EJEZruMSwxvJEx4kVSUOOOGPH61K2nyAP5lzbwkcDL56+yg03YBkIAuOwGP5VH8roQGViDzg9KBlrTPD63credq0cowSIY/3bE+24H9M1Ru7SG2ufsd1Yz28oGf3kxbcD0IK4H5VLnDg4YY5AHGK1LbWRJCLW/jS7thnllzIp/vBu59+vvRYDCGmWIOfIRs92G7+dOjRDHi3ChT/dFba6IskMt1YXn2iIYIjl+V0GOh7E59QPrWK2WIZW2nrtC4/MetKwCLGVGS2eOnFKoUkbl49TzSkjOGODjJwOlBRSuAze3NDAJI4t3Hb8KhZQRnJHoaGwG+9k1GcjpQMa8Z7Emoym0Yxmp1VnXI7UjxSE8UAVWTcMMo5NQtbRhCAoI9COKueUOd7gUhVQOGXHvQBnI91YzLPaSNEyngKcD6HFbkOoafrFp9n1KC2M59zG2f7wxnJ+mPpWROeAqKWJ/iJwKqTwLnI4YHhgeQfUU0InvfDF5EhbTp1vYFywTDKQehwO/QVkeZP5cVvcqUmt1Y+Wy42gsW/HqTn3rorbxbqFpbJBMvnIilfNXIYjBALAHDEcdfT8a3rvRdJ8V2DXEKGFwcrcKCr8+oPUH0/WmL1PPN2TlT15qN3bdVjUtPu9Eu/Jvx50SfL5kL4PTjqMj6Ee1U1lDgsmWTJ5/wDrdqpSsJo0tN1i/wBLlMtlctC5wWZQCTjp1rofN8N+JYVN9cppGrBcvcRKEhmbgZKnAJ9cYPXOQM1xkbZbHUU/zCrHB4/z/ifzpuzEro1NY0S90OSMXSq0M3+puIm3RyfQ+vseazkYZOODn0q5pOs3Gly74UhkjOd1vcDdGwOQRyflBzzjrW0ugWniCym1Hw/mOVDtl02R9zI3fY2eR6BsE8/SqUrfEQ432Oe259KKryXDRSvHJBKjoxVlZGBBHUEY60VpdEcjPc9E8LWHhtPtl1Obu8jTbsUqERsHIX3I+tWbzxEQjLEnkxnk4wS/9T+Nc0dWSZjtYAhck+1Upr6J4nKS/IBgMO59qfJ3E5voLPr/AJlxJHIWVf4ccZ+p/P8AyapW2qCS+bdxlcbs5zWTd3u67K/wMPxxnv8AiKW0mjb5YkYOv8VXdCS6nS6LeSHVrKYIzuJxFnHQZx9etcvPumneNjskjldGJzsUhjwCeK6Hw080eq20SgFxOCAOpGRk4rvvI0nRGkeKOBbhwd6JEodz1+dgM9T3/Tmspxd00aJ2RwuieDze28V/qL/ZrEElzIpDSegVevPPP0wDmt7UNUSGzXTtPsxbaaq4IDYL+zfX09+Saj1TV57oBSwCryAONv0x9cfn6nPPy3BwqhsgnjPTP9apRM5Tvojbe9+36da3MhJwTFwD25Ht0YflUJIx02gdagscLpNuRHjF5JHngDLIpx/47n8KtStFGoRmA3naPVj6UpfEyrDA4YHLjC9cHp9ac48qIXN45t7XsuP3sn0H8I9zyew71pppL6TbyahdFSzMqxI2MJnJBPHzMPToDk81w+v6jcX94yPK0gB6senNK9wtqaWp6/HcpFaWsYhtFPEa55PqSep96yNzS3LRjlT90DFNtYlcorElQOW74q55fleWEBeVjhFXkknsB6mm2UlqTRPHApDnBPCkmtDSNBuE05JtQV7KxT5MyfLJIBwAqnkZA6kdORVyTT9P0KFL7U0E+rOCYIdwKQHHDHnkjrnnnp61yl3r2o6qFtpryWW3Thd7cn3Pqfc8+9RzblWudTdazF9otrGz2pEAYVjjJCgHoT6nO0k+xou7lNoaMAMyjkDFcdpJxrNrnJVHDYHXFdJJBJLa/wCrlFrHnzLtVynfGDwCODk9O1GlrikrHN6kt3qF9IscbGIAKrbxg5HXHfJ9PSrq2uohFgW7jhRcDKMCT+h7/wAu+ao32u/ZZDFZ3RdVUxuhiKAnPTAJHHrn8BWfHrt8u5UKhSOhQHFLmXclU+p1kFutsnmzahdScdHYYH4AVXnudNRJdrxSGX7/AApJ9s4ziuWaee4bMsrt7buPyqRBjp+lLmNOS+5qPcWf3YtNs0UEHiIHOPrx+nc+tMmvHuWBkWH5eFxEox+QqomMjJYLnkgZ4p2ccEVJViyJ5m+YyybsEbgx4z170m8+WELsVB4BPFRAnOAOfrTipzmi4DlQvKq5X5iBy2P1qxesst/IY8eWmETtwKfYLFHIbqeSIxxqzCN1ySQO2PTqM9xVK3LR26q5y2Ouc02CJDjBwRkdu5pUbIII696jY56UqsRxUjJcgDk4x2o3ZPFNKhlJKHNKI5OAVYegxQBLuUDLdKTcu7GKY0UvUgge5qRLZmjL71UKMklqEBI0EkcSSPC6o/3XKkBvoe9MJHODxVc3MG9F+0BtxwpVWbJ9OAetQpfK77TBcpxnLwOBj64x+tAGnayKlwpLELwCc4rY1e4jksreOM8oW34Jwc42+2eufqKwobySCQE6ZOBjnzXVAfphif0ovnkvnin81oZlZyyIcjBXCgnvyM8jjJ69KelhEqrI/EYXIGTxkAVEz734waECkMt1Ndyr/CsUoiX6kYOT78VDPCCR5FxeRR7iQpmUkf8AAtuccetAE6gtnlVCj/IqHzIVwTLHg9DuHP8An+tPgSC3IItYpJcY8ybMh/8AHjgH3GD71L/oplMn9m2bg4yjoWTI7hScZ/xoC5TlvoLVYzJLzLny1AJ3Y9K0bEX1y8kjaTqDQxArnydvzf3ckcdfrV+LWxFhBZxRIMAeV8uB0xx0qfXPOhs4763JbYgm+98hAYcE5yBg9fehXW4mxf7I1ORCxXThuIIXzZCU+uF2n8Klfw7e3CFZr21gTrthtmYj8Wf+laltcNIu7YSAcZGMH9asvIykbecsBjaT/wDqpNPuPQzbXQre1dfnupmUMTKZduSVx0XGMZ4xj8adHp1ok6t9llLKdytLM8mD6gFjitQByMlgPYDFBQb9xbp2/wA/jRcDOZLNblz5Vutw5GW2DccDA569B+lOMks5bZdIUHBXg4q1LaW7szSQo5bglhniq406ySN44raOIP8Ae8tAufyoVuwir50Clt94XjHG3oSfpkfyqaC9t1by1mLAZP0Huar3OiQCNmjdkbHAYiqw0KWSPKyDgnGG6+4I6VV11EaNzqMSqSCTnlSV4NZU16s5ZmcxqRhSOoPtTpf7Sto3jdBKhGDu+Y/nWY0avGXdiDnovUfpihNdAOjur+GHQJVuEeVQokjli5UN3BH+AqtoV9LcNtVxJCVyGJBI7j6iqFhcYtjGZ5AgbATy8ktng56jn8Kt+HbO3tnnt402yWjGLBcPhSAVwRx04PHUGhjR0W7GMjPvS9iOopCD60wbg5yevtUDF3AYCnOBg55NZHil8+E9a4PNhP8A+i2rSdlWJmlby0HLOXx+uay5PEOnxSkNcRIgJBkkkC5x/dHJP6U0mwPmeirF9BHa6hc28M4niilZElC7RIASA2O2etV6xNgooooAKKKKACiiigAooooAKKKKACiiigAooooA674batHpHjCJ55oIbeeGSGWWZtoUY3DkkAHcijn1r3oHPzbVx2zXyvWhp2u6rpO0afqNzboJPM8uOUhC3HJXoeg6jmqUrEyjc+nCSwpC244I4x6V5T4V+K2Atp4j/wBoi+RfxAZFH1GR7cdTXp9rcQ31utxazxzQtnbLE4dTzg4I4PINWmmZtWLBKgdqglYRsFLZLcgBTwKn2lgB0qGeFZ4jG27afQ4oAiOPUE/Sms5jHzgPFkFkYZU+5ppjSGBUZsLGoGS2OnrVe51Kxsh/pF3DEB3dwMClZsZfWGKfc1gxKqNxgd90g/3fUfr+dV1ZJOFPQ8isuPxBp8I822aeVsFla2hdj9cgYouEvNXIv7MtZXLDDx3CtEH5J3YAPJBHGMdaNANUDB5oJHQd6zol1h1IeWwC5IEil3LAEjOMKO2everNuY2jPmbp5P8Anp5pA/74ULj6EtSGTbO5PAHU0iq06EwJJOo6+Shf9Rx/+qrCXjQsJAIY2HIZIVQj8QM96Rr9pWO+WRi3O4kn9aAA2M0aAuY1GcfNICehPRcntj60yCzeRxM8sQgw2EVgJSR0+96nPb09abzuxngD0pGQSIyNkhhg09AIGt1WdWna9iY/dgnby84/3QCR7gke9V7nTrOaUym1jd85/elnA+gJIq1Da21wjw3bzQzRyZhuwS2FIzgjODznr6/jUt5A9gsk0j+ZaL1nCEBck4Dfhjnp7ihiK8EEMK4hhjiXH8CBc/lTyg/vHHvTmUEbkxjrkd6YQWU7B82OM8jNKyGNx+XcUhiGeOKlZkjQM+R9BQ0e4fLn8BigCH7vHHWmMCeAM/hVo2z46gHHGafHb7VG4qzeoHFOwrmdJESCpTI6EEdaih05YifKjEYPVRwK2nQbecZFRhQFAGeBjmjlFcxpg0eQVYkdgKaMD7wwT69q1Z4Nw6c9jVGVFWTBBzQ0NMhhuZ7ecPDvVxzuU9vf29jxRqcceoQs4iW2uQwYyQx4EnT74B9u1POMkq5qrNeW0fMs8akHoz0DGzytYW8Ml5EY4ZSFjlbADEjIyCcqcDPfvzwaFZJQdpB9xzUVvqNlNeqIoUvZRkiMweaMd+MHirtjoUbwzXLXKWBkk/c27KGiU9+jHaCR68c5BpAVYoXYne24fTFMZPmKbfl65PpTbj7bA2B5MhXIOxxhvTHOOvvn2qnN9vkB+e1UEZCjccfyp2GaLSKANoAwOwqubhh25/Oq8MTRKPOuJZj1O3Cf0P8An0qCazWYndvwRjiRh+PBxQInmv4UH7yVE+pxmqz3LupaC2uZUAyXSJto/HpVm0ht0t2ga4exyu0TQR7jIMj5ZACCV4z3JxUNxPa5kiNzGzDpJErEceuRn8aBlWSe9YALbeWCPvSuP5DNUWad2Iku4Uz2RCx/Wp57KZFLO24dnzmqm3Djk59KVx2GSKnkSIzvIXYHcTgjAPAx2Of0FPh13UrG2FtbXMkMa5A2tyBxxn8KR9rcBc+57VWeIjOWHB70Xa2Cy6mtaeIhK5XUIUfLEmRVAJz6r91vyH1q1deDoZbUXmm3qBpQeCAI3yenONv881y5UE1PZ3X2SQSIilgR95QQfqDwaFLuJxfQryQXenXzWt5C0cqE57gj1BHBHuKs71KnaRXZWdzpfiqwW3vNOto5UOIRHIUPv05A5PqOuQK57WvCWp6bdTzWcJubTzG8sQncQp5GQCT0/wA+rJ9TLZudo/PHWl05jFPNKrTROD8ssTFCwJzgkdRkf5wKpC4PmMkiMjD+E8c1qNtj0iyuFgjUtNNE0gYEvt2NyO2N/wCVO97ILDpLWOeR5ZLq4LuSzZkPU8+tFQ+ev9+it+Z9zK8u50kUztKY0BMQ4JbPTr1/CnXlwn2NRuCkHIU9Ov69qLS2ZpmjJVAF3Fm46dvrVeSB3kEcUYPb69ea0b0IWpWYiQg4YjACnGOOvH61teHtMutVuFit1Xrkk8BV9T6DjrW94e8BXB/07V3e2gUblhX/AFhH/soIGM9eeMGugutVs7PTRYaXb+Rbp1OSDn+p9z+lQn2KbSHWlvp3hxHa0H2rUiuPPc/JHkdEH9ep9ayrjUN5YnALHJxxk1mNqBa4Xcxz1AqB28wk4JHUe9aKNjKUmxtzdF5wOdo6k9KJiAiEchehxk/h6ml2I67cA4JOD61o6PoN5qz+aJBb2aHEl04+Uey56n9Pem3bcSVwsLO51HQbqzso83S3Ec8OS2wHkNuYAgfKT371tNHY2HlvGqTaptwbkjIjHcJn+YpNQ1Gz021XTNJJWD/lpL/FK3qT36D+Q4rHin+QI7l2PIHrWUpXdzZRE1SWWXS78Rs7PtSX73JZZEJPPfG7r61gTWzG8cHJP97sa6KSGOS1vEljBzbSgheCcISPxyKz4raa/ntYtGh8+aWPK/3UXpkk9KSY30Klv5iyLDHCXmkbCKOp/KtefVF8LT+VCYJ9Wb5ZmkUgWwYAgKD1yDnP07VuW0dh4VsZJI51u9RZcSXOMqnsoPbP19+mK4bV7MareJfwiUTuhV2diwfaRzz3w6dfSk5PoPYgvL+51F5HlkaSZzyxOSen+cVXtFdbgwhVaViQATjmtbS9N8qQyudxXgnBwK2rDREn1AQ2sM00rFDLNEOIkbk8n+LsPxPblWsx83QTwp4emublpdn7peHmZT857qvt2zVbxXf2tvbGG3lESsWVDHKG3AcZAH3R3zznrWn451s6Zof9naZE9rE5e3IO0Db0PB5zweffPWvJlmYyFiBLkEEnJ5x1z6jr+FQ3cLEwjAA8vLAHv2NSBD1xz71JbifyUlVpAVHBBOR6fSpIozu+Yg+zHFBQinK4xjA6+tSoe1K1tMCoHlcgnPnof5HijypUETAwnexBUS7imO5x9RigCQYAzinrhh2Hc81C8VwWBWSIAnkFSePzqUqxQDzG3Yx8uMfkc0ASDaCeeaUna2MnrioZEYHdCqIS2QpB2ipGU7RmGN3HQe/40BqO/tBbaznUytGHYIJGIXg9RnqcgYx05NVhqUEn+qeSYY5aGJpB+ag1cG4AKDgAdB0FPKngk0MNiqk7yA+VbSvzgBwIyf8AvsjH1NOhnuW4e2hibPKvIzFR74GM/jU6rg559qcSAQADk+1KzHcHecxgRSwxtnvCScf99Ujo02Qbu4RuMFQn49VNKcgH1pCGwAxA9SO9O4iK6sonjVXubqQuWUNJPj9EC9vX0/Coo9PsSVYRLKVPDSM0nP8AwImnXbF1VPMGRKG8s5xgDH59akjmVSAd3PsTTkrExle7TLDFi+4sc9aA5VgcnOcg570EjijcVTGQcUixzsWy7t77mPWmqyPlVILYBI7gfSmCO3kbc6I/YbucH1+tOhijiQLGoVPQU9xE8aKXAdginHzEZqSQWqnbHJIzdMMgA/nUD5yBHtzjkk9Pw70qocDPzEd8UAPwoYgciglTgfwjkc01VwCFA2+3SjAwaAQnOSAP1ragEN9p6W2poz2gyAqkBm49Txx/Xn1GZBbvM6xplmY9q6T+yIF0vy5Fb5WSaQA8nbyRnjHHGaLoTuVbOWKHQFePK+YnDNgnPPUjP86vaPeiS22sfmU7Tx196rQ2trc2N1DbeYqxXMwVXkLlcOwHP0A4rMhFxZzny7mJZDwIzIpJP0zRfuB14YDPJJ7UoclSWwACehrKt49TaJZLrcUYBgg2ox+m4g/jUpa9KBYbNuehmmUfntJx/nip0C5fdumTwenFBlVFJbt3rmtmvXkjiQW1vG23hrptsYOMZxHnuPWov+Efv7wEpqMKKSP3iWzlhyem5sHgDqO/tmnoBr3Os2IG1m38YwF61VTW5JJMJbsU9EBbiqsHgyLcDearqFwccqriJT/3yM/rWtH4f0lAQ1jHOWIJa5zOePQyFiKXMuiCxnTa0LgeSDEpPADyLzn8ahj0y6cNmOQYAJ+QgH6E8H8K6FI0t8RQxrHGBkJGNoH4Cpg5XGRznindgYlnpV3FIpMUgUZ++64/Q5/SrU+l304hk/tEW1zFH5e6OEOrqOVBzg8ZbuevXir0t4iAnOdvVR1qkuvWjXBhLsGJAAZcdfTvQlJhdEyW16YlSbUizDq8MCpn8G3Uh01ZXzLd3rgcFfOKA/8AfG2rIu42kKDHA+bJ6Uz7XAOsiqvYsRzRZhcrTeHdIupFkuLNZ2Tp57tJ/wChE5/GrH9l2McDLa2ltbuRhXjt0yp/Kny3cMCZZjjrx0plrepdRPLGcIpIOaXJ1aC585+NLSWy8aaxFOys7XTy5UAAhzvHA6cMOO1YVdZ8S23/ABB1Ns5yIT/5CSuTrJ7my2CiiikMKKKKACiiigAooooAKKKKACiiigAooooAKKKKACtbQPEmqeGrxrjTbjZvwJY2G5JADnDD8xkYIycEZrJooA+iPCXjC38X2UzwqLO6gz5qS5ZEzkqdwxkHB7cEfQnWurLVBsVtQiikYkGNIuQuSM8k+npXzt4c1248N63BqUC+ZsyskRYqJEPUEj8x1wQDg4r6KbxDZXOgQXunyeedu9Ej5V1HXBONpAzlcZyAO1awbexlJWZC/hu0mtpBeS3k8rOCjtdEAL3G1QBz/KktND0+wlWazsYopUAHmjJfOc5ySe4HStKC5ju7RJEZGVxu+U5FIY3JyjlR6dR+tJx7gmVZLqS3jLKMKMjg4A/+tVd3L/NEDv7kHI/+vVl4pAx3Kp78HmjgDIHOe1ICtJEZo1/ehccELlaSJgjCNEOPUnOaW8ZzGPLkjjYjH7z+lLFue2PmupZgeQMDH0NAyQ+YTgIGB7jjFAUhQOBn2pUJ2jO4+5p4UEfdyPrQAwEBeoOOM5owG64p/k5XoeeuOKRI97bTgHP8qBCY6DNTWs6+TNBPETDKNrqjYOPYik8pUOGYZ9DTmjLxKUhnZHICtGjYPPHIHuKdgIptImtrcz6az3UIG6SJ5U3DPpyAPocdePSqkEqSbgYiijn0Iz6jrmtS0Nzb35Do9pEV/eSOoO70G0Hnk/zqfUbLTJ5pJIBJHIzDc0D43cnJ+ZSM89xnpzTsK5nIscQwDx1G45p3mx43FhjpnPFSzadpnkFEjvi3Jybt1Az7LjNVks7C3k3Lp0TuCCHky7AgjByT14H5U7CuMlvbaNTm4jGeMbsn24pokLRmUQ3Owc7jC6gj1GQMj6VeF6BuxHCGZtxPkoTnuckHnioWubqXapv7hiCTtZ+CT7VQXKMuq+WAFtLpuuSLdzjjOScY6VG8+ott8qyOCeGMqgfXk5xx1x6eoq/IXkUo+XiVs7ZORnrkDmqvlTQxvKXluLdRvkkZvmUk9cHnHXPpkcUrgILObzcX2qIqN92KOMPznoDkHNF/ZWcqiKJ9QVuN0haMr/LNLFGTly6lj0YdMVK4yhH6imIpA21ovliyjl9WnZmJ/IgVltp1g0zSmzQFjkjkjrnvz+tbD2zuCGIx2wOlVjEEYq/WoaRSZASUf5ECDaVGxQvB6jio5Mkksck96n27TjjHtUTbQDxk0ttiirKVUZxVGe62ttAHPQM2D+AxWjIwzjnd7VGQGALYz1GRyKLgRKpKAtjOOgqLzVa6EO1tx55FWCctzkimBwpI54PpSAZIEU5IyR3HNRNNI1tLbxXEkUchBcIeGweMjoamcBl5B9qi8mNTkDB74oGUIl+wwT7pYQpBYLJlcgY+UHGO/A47+1VYh9pG94cKejK39DWvLHFJEySDPHUdqqTQIQpEaB1PEwBDn68/N/n8QDLlUICASc9PTFU3BLHIPFa8kttKmGIikHGDxu96qy2jAAqpA9exqWikyjtGM8fSgxqwJx0qVoWJxt744NRGPB9vY1NirjrdpIpMRleeBvxgZ9z0rpdN8Z/YrWK1mhRokz/qsqy5OT161y7xbVzuzz+NRFc8Y49j3qoycSXFM7n+ydG8bzNLFcR2koBZtsOJCPUjdg9un41keIvD81jFb2sEUUmw43RPw7HJ3FT0JGB1P3RgmsrTtSutNuVntmUOnqOvsfatlNa/tFvKuFiE75OTkqTnoc5PqeP5VpGzd3uZtNHLGC4VirQkEcEFTkUV6LFH4gMKGKefy9o2/uIm47ckZP1NFLXuF4ljSvD+o6xcmSBlisQ3N1ICFBJ5C/3j/wDqJFdNpz6FoQY2VrLe3adLqVV5b2B+6Pbr6+tGs608zfut8FtGo2JnDHGOTjgDjoPSuT1DUHWL9zjAHRRjHtXVZvc5+a2iOh17Xby61OyQsDaS2nmbk4BkDYcHnnGB19a57U7rznjSFpAgz07/AKUK0k9tYyEHZG8yEk93VCB/441EcByrEYx0FWlbQmW9yKG1LzBS52YySRj+tWZMxKRuBxxyamhWaSb7PbW7T3LnConJ/wA+9dJb6ZDoEaXN8I7jVMbki6pB7/7TA/570OaQlFsq6boltBDHfa3ujicZitQCHl9z3Cj/AD2zNeaobwLlVSGPiKGPhEx/X3rKvr+S6kkd2Lyv95zyc/57VTtXwxR2ITtWTldmqjoLcTGWcsSD/KmeZ5QXadu3GMUhDGQtjOT0z09qu6Vo7auXuJ5Db6ZCSZrhhjdgchfU478gVBoX/D6PqUjSBV+zxKxuZHbCgYIxkd/8+lNku9O8OaWdO0kEQN8skrnLy/U9hyePc1X1nxJbxxLpOlR/ZbCMAqrD5pD1yx6/55rlBNNfszMc844HFG4i3eX8t9IFVgsZOCc/nWlpskAs3jlGZDP+7G3+Eqc/T7o9+lUGtBDGgxwWz75qDUHKtFawT7Ll5QWO8/uxgjkDn+I//XqkS2rHSSYha2UTKqXAZXl2l1iIztBA5JJU8duO5AMevaxcaHpn2G3WSASgy7rkMkjgk/3R83IADHA5/Kt4W1K1j8ubW7kWthYtmBDuG+T1IGTkcfj9DXLeLPEM/iLWJLhpFECnEeBgY4HfnoBSneOwR1VzJW8unJL3U55zgyE4P51H85OSzH8aVFyMqpbv7YqQIdx3AE7uAOcj1qNXuaDwgVGXq+cZBzU6bUUclnPb0FN2cAZFP27fmYhcdWJ6UAXLOz8+5i8nzJcnLBkChMDJ5GeOOv6dqkkmSe4kmWNV3HA2qFzjua6KKK00Tww0wkEl7cYERClCM9+QemM9OdvTmuZZkYFUVVX0QdPam9xIccrgEgZpwPHNRjAApeQwJyPrSGStgjril+Uck4pp4ycDjpzTRE7EMdzKP7tFxkwUA7hgmpMkDnpUCsdxO7gdOn86kWQbTuxx1oQgDhhwevT1NLvCg7gcgZ4BP8qVDhjgZGaXzOTztA75oAZ5gUhT8vs2c1Io8w4yR61EkqxvvYx7Qc/O+4H86kbURcQsn2iJYj8oEe1V9B09yKdmBSkVriPTxE4VmV2mLEDHzHAxjkAd+vNXI1CA7iCc8YNVby3uLC8MOHmRY1CCD58A46Y7VYVGQfN5j4JIOFH06nv9P602TGSaLEal8/JuJ4H1pZrV4S24KQDtJz3po1m/t5CINLslIACM0rbuepJC9fxqvcXGr3DhpHs1OOQsTTE/i7D3qLruVqPbAZQF555z/wDWq3Dp95Ku/wAhgmM7pMIMfU1TuLW5WWxa7vr3FwrsIEVYU2gkZ+TGQSO3GK0IbaAPviigEwAG9lG7GemepppBcclgzlljuLaV0++kUyuw/Bc1N/ZciFSJCSRlUCsGPPoRx+P9c1dvLyU2oV51L90XjPP0qpZy2gcfallbJxu3HC8eg5ppCbLbXJlhaDEshjBdw0ojBH5Nx9BTIlRVEiWW1iu4CSNmxwMHkrke+BUV5YvAZHhR44wchiwzjPpUULyvA0b3G2NeQrnqfXFLcFY07Oa5uJtgupFUcZhtggz9CXxU91b3McBlluZnBBUiVgo57YUDNJp8q2di0s9yu1zuVUGD/niqd1ffaJA9u8zSLnBfGMU7MRXTT4rl5Hjgs0BxkyglWOMZwcjPHXFbKacIIFY3DhAORCwjQj3xWO8l2H3SSMDjndjP5GpkvHgTBklbI42YXB+vNFgFub4JKY4YGV8j/WOxx9RV/wDta4jcb4VwcdGBJ/DrWBJIHdmPyAg8nn/JqtudZQ0LsGHOAMfyp37jsXdcmvG0W9NtsWKCQTjbcIrnhXBCdSoXaOOc7uozWxH4gjl8poFzE4BDE84NV4zNdrGZb2WCaRDGWSHzIxnO0sBnoT1x6Zrl9KMtpLPZyQzKbWQqGuI9pKkkr8p6cY4H/wBejmu7MSjbY7kXhmRztlOP7pwMfUVbhl8yLeoOCPlzyDXKR3Vy9x8khYuehPWukspibdI5UMZAwDuyOP5cUNaAiItcQXMk74aLbhxjHToR+vH9eKmE4l3EhwinklTtYeoPQj6VMgyDhick++KZtiSFY/LCxBThAnH+ef1qQEkkQRtJMj+WvJxGW4HPQD+VVGRQrssZI4kQNGQEHOCAcZPsehqbTkul803E5YE4RduML+POc56mrDqRIssmxtnIznK/SgDJEKi6SOaZp3jXcSzFGGT1YAY4B9f/AK1sWEMtsi3MQlkAyMHB44HXHODVK+aRJoyrskLZQBWwAxPy/KOT+B/KpPIPyy3U0aIoMZ4wjgkZypyR0A5J4+tMClcRvaALIZJt5KjzGUDHUe2eOn60llqclhLJHNBsTOSAeRx29a1pNPshHI7YSI/MxY4X/wDVWUyaHBFJM9wJI422yeQC+z/eC5Kj3OB707vsKx5l8V7pLzxNZzRhgpsV+8MH/WSVwldn8S7ixuPEVq2nljCtmoJKsvO9z39iK4ysJbm8dgoooqSgooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACvZPhJrVveaRNoNy2+e1Zp4UeMFfLJHQ+zk5z/AHhjPbxurFhf3WmX0N7ZTvBcwtuSReoP9R2IPBHBpp2E1c+mr660KxkDWsgsncDzoDC4RWzjcCRtA5A7Z471cUL94MTn8q5c2i+K9Pi1GO9ia2uE3RI0CsVB/hYhjyDwcdx7Vq2Fo9lE0ct7Nc54G4bQv0xz+tb2jbQ5+Z31RqMwBJppXeeFzkZ4FV98ZIO0bgCoPfH1qXzZHJLOxz6sTSsVccEbe0YiKsv9/wCQfmcCqUpaU4WByFYZwdvPtkjP4VY2s3KnjvigqwA9PWlZBqMW3mMeYYogO6vLtIPcY/wz7EmoxFd+Vu32qNkhgGZynXrkDnPBHNSl9px37GqtpDi+kEH3nG54yT+8PqvX5gO3GQOORTSQXLC20nngveyzQk9I4lix145Lexz7Ed+LDCJI9kVso+XG93Zmyep64z+FQCV0c8fL0PtUwcg4LdeQaAGMrYXCqoVdowo6ehPU0rSyOPnlZ8dAxJ+lPxnBzkU1lQc4P4UXEN2k5IHymm7iMY6U0fONy5A9SKUHn19aBil+TUU7OYz5bAP15Gf61OVyOF4qMr/s/jRcRjXsN9LOrb38scARkr+PHNI6alB8sZluN/GRtOz67iK2cA8A8ikztOcA+oouwIF88shdlXj5lFS+cYGDoz7gcgjqKZK6EHoB71FGyAbNxRgejEZNMCK+mFvHDLFFiVm2yRqcKfQqMdc9eefxp0EyzoHQhlIBBB65q0E43q21v7wODWXN9l01J5/tEguJMu0ZQbHOR8xORhuR9cfkb7DLxzjjNQyBmY5RSvt1qGzvBc2izBkIP93+voaV7gkHAx+NK6CxHLbsCSpDVSljZkO04Jqw8hI5cj6VHkFcnqO4qX5FEJh+RRuJP86ieM7TkjFWWI2jB/SmEr3XOB6UhlcLjBAzx3NRyuu8LwGI4A71MU+csMg9+TUUkRJByVPTOBSAjZN5yc8EEAHFMOUJKnrzyc1J5b4GWyfpio2Ts3fnmgCAfNITyCB7gGk2ZB5NT7VUKuc+2akS2nkIEcEzNgkBY2JOPTjmi4yhIv7h49qOG7MuQD2I7j8CKoK+oRRspSOZI1LEM2GwOp/Kte5hu7ZkEthdqz/dLwlAfxbFU3jubg7XgjiXofNft+ANAEctrH/d2MBhlbOQff0qo1oqsGEsZXPPNajWunOUe7kuLh9o3LGfLAx/CDk5GPan20mlWS5h0lHfk7p5Wcf985AP40WC5jyfZFKqzqXPQKuTVbfbIzDyJiy5BUJmtmS4/eFoI1gycjy+Mfj1/DNQzTTySK8rmQqAql23ED0BPQUNCuOTRRc2KztJa2oIJ8t3Hm8f7PT8yOtSWdpo2nJ50l/dSTMmR5UO0rkfdOW6+uD+PWqWXOcsSPQ9qhk+pwadkFjeGpWoAEepXyIPuqUU4Hp0P8zRXO7H9D+VFVzrsLlR3E92bhc8njIOOKy5l8xsDJbuD+X41dj5iC9WX0NMW2Zpu2Dzj8a7TivYvRr9m0VVfHOoQqC/H/LKbp+X86v6Zot7r1y8Fq3k26cS3ZXKqfQDIyfoeO/vpaf4ct7rQA+rym2tTcpNGQ3zvtBxjHPO49OeOo5xa1HxJBaaeLSyh8i2jXaiDrjtnHSspS1dv60Nrdy0smneFLd4dNQS3PSW4mPzOfQYxx16Yx+dc/r9/bz6xaXCvvS8s8gfwhkYhgMYxjIH4Vzk2oXN/PI6yERlcEenNSXbIbLQXbK+TJcQEf3i2Gz+n61n5miXckw5YhE2rSvLbQToks0aOwyAxxwKZLcS4SK1haeaRgscajlvx/z1rT+xaT4fK3GtObvU2O7yF5WHHIAzxkZ6n8PSkkDYtvFYxQpfavI0dmTmO3VWMlyeSBgdF474z6gdWa94lmvFNpbR+RbDlYhGoA6YBAzyMfT+dZ+ra1e6jolvqK2UQkivDEGALfI445PcFAM/SsNUmlyzyhCp+Y9cnvT02DXqIbe4nuizHqfXpWzbW6WybiwGMEknoTVOAyIxDOGH4DBqe0hudYuvscI8tI0Msk3AIA4wmSAWzwPf6HDQn2Nm1sJ73zJlh3QwEea+SMH+6OMbvXPAHJ4rT1OwsLTS1v7u5W2hj++pBfdxnZGpOGY4wW7dT6izod7caMzaZbRKlpDGJWebPyk9uANxJyePz6VjavdXN68jyqEhLFzEhKqPfGc+/XrSl5Csjg9b1ZNYv/Mgtxa2sa7IYl6hQTgt2LHvis9LePcZNmfQkZ5/GulmutGgUhoirsMH95jP61nz3+lzSr5lw7dQvm3BbA/Ems7MvUoruHQFT0NKHjALAMxx25//AFVopNonliCE27yd2jiyR75Aq7H9ieRTI0rIvZkYD8+h/CnqMq6dptzcosjRKoY8KzdvU4zxXT6fprxZS9htUdxvT7OSxCrzn5lHOSPXqPXNZ8t3AsHl2z+USuUwOM+5HTv05qy+p262bia7kvbpl2NvGxOOm3aemSeue/XjFW6iZR1vUm1G6Fv8wtbcBUUsCCSATkYHrx9T1zk5gAVSsYGBximSW9wqjyZYZCckiTdheSeAO5zU6qQn3YI2P32RDlj9ST/KobGthh3Zx360A7XDsEZRzhqf5MRRhKZpcnA2yBMcg4+UZ/WtPR9OsWvIEvLOSZXkCBnnJxnJ+71PHqe1UkDZi3eoWcI2ho4xnklsmqx122DLADI7PwqhSd3oAB1rufElxpWjKLa00ayMknzbUjQKgPPPuR6EdcdiK5qXV7siTcYoPMHKQIseeh5x15A7npUq40zMj1OWTJg0+dmBxmXEag5/2iM/TFO+1apKAzNp8AOQVyzMv14AP51E96VbZJHtGcKoYEt/gKtLaW5Ky+Sm/g5IyRRqMWO1mYqZruZhkErGqxhgOx6nnnoafJpcFwh80zshOSjzNj6YzjFWQ23kkZ+tKsgK5GPrQIrpptkqBRaQfQIMYp8xk85ngjRA2S+MLknPPA5qRt/lEoAW7U2KJ0TBOT1JoCxKkkrBQZMYxnaMZ/nQ3VnMjMfTI5qNpGVEKYw3XIII/ClnSNyu5QXVT36/Ud6GwGJMSpdsAcDKg/5/GnRyB503yCNGGFBB7nqe/T2/OoSUOFmDBj2yeKUXUUjFVJLLjj0+tAG5dtu0jS1McXmAPgq2WCjHJz0zu6An8KggkVXTzTKFJ+cR4zjtgk+vtVsaLbxaZp1406I81uf4j8x4OOTgEc5x6GqWBx09qp7koklkQuREreX/AAl/vD8uKniicNtF4kS4zncBnj61ADlSOBn1OKRI2dwoAwTy3pQAbyJDIJ5Hc8nI4/PNN+csSwye5J5qWe2aI4DLgfxZFVQPm60XGWoUMjhEBZmOAo710NrpDQfOXEkgU/IE3DNc1G7fMVAZcfMCM8V0j+ILZLTFuJC+3AJXvjue9Du1oI0rWBBBlgfnPIdAuD6YxSTxwN8paNGI4BC/1Fc0msXMzDezHYNxOMZPOeg4H+NE2qhiNqrGR1JUHP1o5UxF24tWk3QoImbj94EwEz/tAj+VZd5YzWAMhmQEgr8vOfbkfypkmqyKjyNdqsZIzjCgH8Kz5dVsGB8y8Lgfd8tdxJ9B60ctho6nw7ORCrQu32hTkHdtC8HgnjHBJzUXiCxniu4Zp7gyT3RKESszMm3J29xxz3HLDjgmsiPWLa0lWS2truRjgGJYzyM9Tnr+PHHFXdVvpdW0ybYIre7Xy5rd3YMQy4wB2Xgc4x2B60W0uF0Zu51APQ+grTsrq/kjEUEH3TuEix5wfcnioYrnW7gLtl0WGQjLAh5Cox3OMZ696fGzTALN4kPysBItnsUY6/KNhOce5pq/9IC6l3qkkrRvG6sOyqAAOxoe8aGMC9u7aIYIDyTEM35EZqGew8P3ccYnEtwyKYz5txIVA7MAuAT7cfl0px2mhaf532XSkeSUDLTnftH+zuyR+eeAeoFPXqidCdPFOmQRqG1XzJMkCKGMsOB0yoPb3pz+LA7u0FtO4RMqWiKFjgn7p+YjA6gHHU4FT6ZqKxXGxbS2UFCg8uJUJX+724PvWvei0TS3lkgWNFQF1UY2gEE4x6YBGPQUr67f1+A7aaHEf8JRrAuGQWFspbnzEyWIHbGByBxj29KLK28Va/KEOoLb2quDI4jA4BBAwMZ6dM9+exqZP7EtfEVpDG0txGXzvlIZQSrHHHHYf56dNc63bwp9ns44ymMApwB+GKqd9o6ERT3kUJNF0WyjiM0ct9IBzLeyNMXP+6x2/pWdbXSJGYIo0tiGLGJVCA88HaMDpgZx2p73Ek0rvM4OTwBnNQO1lI6M8ih1O1DvA5PWpSsWeefEYlvEMBJzm1X/ANDeuQrsviLaTRarZ3LBvIlgKRsxGSVY546/xDnHNcbWEtzaOwUUUVJQUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRT4opJ5UiiRpJHYKiIMlieAAO5oA99+G4/4oDTCP+mv/AKNeuo2c9OtVbS2TTdOtbKFmaK2jSFGY8kKABn3wKsh2bICmtlojB7ieWoJ4+uKeuB0/PFJngCoy+cj3wQKAJFb5yQQRjikkkUA9z3xUKsM5Rc4OOeDTwQw3Alc9c0DEcBgOx7c1CbfJGeQM4B561YYIRkso/GowY92E7jr2NCET2sLXW5Y1VZo0y0agYkA7gevr6+3FJ833GGCP0qJozkDIDA5HsacilbVGeV5G3ESM+MhiTtHAA55wPamA9XYEq2MClMigZNR8HAGSQOtPKDcGC9R3oAaJichYjwcZyKgaWbzQggwrAndnpz7VYbPTPAqEhucdehz3oASOaUsVkXtxj/HNSod33R/9aogGPU/TB603f823IB7jOaAJSeePyqM7s9CfpSGVVB3HI9qh3ghlDY7cd6VwsOklROWCgg8Bj3qBPJRtyqBk9AAP6ZpzojRgyYz6jrURAZxtIVRz83f6UuYdiZrocgDH41myu09zsiJQ45dSAc88H1qS8uIUUqxUr7kf41jNdWkUoLX1tCOvlHnH609R2NY+UGiiwkNyNqeZGAscnTJkH8PGTkDn0qCO68x5EZCjqehI59f/ANVZ8utQTstvBDLcO+FVUiJ3EnAxgHqeg96aGiv18iDTLu2kDjddBSAFOMhhnkDgjgHgjvU3WwzW6PjAOelOAxztJPoayPP11olSK1tywJG5shfqP8ioG03xDclluNXiiXPS3TrQ7LcRulovLLlk/DpWdPq9jArGeeKIg9Cw5/Ko5fDFjOIzObmZ1QBzJOW3N3PsOelWYdH0+1A8iytowO/lgn8zz/n60r+QzLk8Q2om2xJNKSMjZGxyDjHanpqGoXEsccOjTqZACHnYRqB6nNbaxRRKWVVVR6ACjYuORkEdSKLsDBVdVdm3p8yn/VxsuD9GOBn9B71cFrFZ6l/pl5LOjRZSFlAUE44LDqR6e4q75xC+WiEduT1qxZ30unvJJGscqsNssTjIZaNQY3+2mtFxaWNrbNwS8anJ+uTz+NU5tX1G4JVrqTa3UKdo/SppIZLm522dmzxn5nAb/VjrjHJOPXp7DvWubSW2ILgAMMq3Zh6g96L20QFSd3lzvO7PXPeoipA5P51MxPAYcDvimEHseKd7jKsiZ7Y+lQuhx368Vd2dxTXUkEflSuIokMaTkcEY/DrVhgV4x/8AWqMgj3+tADcrkFf1ocbw2QPypGUkk8ZpjE7eO3pTCwnlf7v6UUb2/vn86KQaHTQkFmKkMA20KPvHPYDqTXQ22k+Qy3F+iB1UPFZnnOTwZSDgDI+7+ueK2TZaJ4XBktokN4wxulYu/wDwIk8f7v54HFcxfak927oHESOcl8dSa6nNvY5lFItazrtzMASUeReCcbVT6DJrkdSuTJatMkhyeWyg/GrQZZcQyurlVJPHB98His2/doHSBOYieAwA4qHoWhLGcYEcfzMQCcdTXTHR59c0qxht2VHt73zCWBwEKYboOvTrgH1FZ+haK06l42ihtoot013MAFT/ABP+PvWpc6pDabLOyJ+zKmfMA+eXI5LH69qq3Rjb7EdxqNr4WjksNKMk184ImvJOWyc/LHxwB6//AFqx7LR5rstNfXTK0jZCuMk/lWlYWkl5qAvLhRtXmME5K59PbpW26RAE71Q9dzDpUu4lpuVZbG2i8Najb4KpGEmyScgiQdPz9+vrWBa2YgAlDsA3PI6+9dFeiaXRtVgR/wB4bKQocZ5XDDA69QK57RrS8vtHaPVZY7aRR5PlFyJM84LfK2BxyO/tmhLULla+Se4mSK3EqQld0k6KxD4ySFI+nXPXvWha3w0aBvssPlW7qCEcf6wZJ+bJPGCeMjjGQTTNYvTHDbWkoVbOMYMUbY+UZ6gqD8x9u5965rUdQmvpCzPj0AHAocuVaDSubJ8SXiyOY5EDv95zGGbjGOuegAA9Og4rMM8sjAuwnHY3CLJ+W4H1/wAmqVvGGI3sxb6VpLb+SrEk5xkZwKjmZpaxKNX1ARCNLyeNfRHKD8hioTLPLtMssj7QANzk4A6DmouM9RUqjcQowM8cnApb7isJtfdgPkHqCP8A69O2cck/X1pFOAeakDDABx+VFgERRg53fQmn4BJPIoz3NAYHNAEnmblVeMD2qBoIncMwJIOfvEfyp455pdpJwBz+dAy1aSRRbVSIrIWABUYA59K2FSaL7XfHY1lbx4LNMEw2Mnae7DgAHjqOc1kabbQ3UrxSvhwpCKX25Y8L9MkgAmtnxVZx6No62YWMy3c2WLOGcZ+Y8pkAHngn6Zqk7K5DXQ4+12T4LTYDZLNJO24Zxx+AwOo6VHd2aecF+0QTsy9EVmIAPXI4+nBq0AAQGwOP07003tukwiLkHGQMcfn0qdyyOPSYAVZ8uRz8wBbPuauYAIC42gc8dPpTGvLdVDmVVX1J4qJL+2mbEMiOemM4oHYtDHUelLjIyMn29ajRCMsSee2acSPlLMFJ460CJBu2/Ln86cp4Gf0qJGLBhhhjt61KxViWUNszwCcn86BkcsrKxUKWGM/KOf1pI7mKXkZB6AHqKa7rg/MRg4JPtVaS9s4HMbyJHISTjcMnH+elIC49uJI1AL5Ax15pCxgKIkZHOCFUH86ms7OZrZpi5jT5cfaiInbPopwzD3AP1q3MtpDBLZ3UCzStGSlzbzgrE2DgZ9eOQARyKdn0FcbcaiZU02LbDk2pO8ICzcp1c8nkHv8ATGDlgLDc4UnH05qrO0yPam0eDyYYjA3mp2LbsDA5PA5Yk5z9KpvY3Dom7UpFO4vmJVXg9j1xjBx9fpTk/eYlsbwkE4SJICJMZyvOf/r1JFaTvKFDbSeAhYD/AOvn/OKwF06AwmOa5v5V6lZJzjOevHNXtOA0lJV09ntxKR5hSRiWxnqSc9z+dFwsWJNqXMdq7TJOylynkPnaBknGPQVDJdwBSVt9SkYZ2bbbAb3O7BAPr+lTSXr4Pmz9OpdsVELiNx/rVK+xoswFtprq4ljitdFuXds7S91HHuODwBzknoOnJFXZNJ1GIK1zNY6YmFDxSs8jKCVBI2lQSBk7Rnpg4zkZ66jYGOX/AElTtXaTnJ+g9ap6xqjz3Km4kS1doVA8wgZBJYHnnPzd+cAVVu4r6lm+0tHt9n9v3Ew3ZbbZJGCvoGDbs474/CqX/CP6e00habUHhJzGrTAEDPG5tvzcccYp39s6bGI4Y7lZS2FUIC2fyqOXVJhzBpt/Mh6MsRxUuy/4cpF1NL0mExtHp6Foidskjs559cnH6VcjneEL9nIgGMfulCE/XHWs4vqbbRFpjcjJMk6Lj269akgj1Ga6SOcWltBn5pNzSFeM8AAZ7UroRaZ5HYhmY59agjHkS+Wzud2SM9h3qGK01JWAa6tkiPaNCT+uBViTQZNRlk33GYAh5MXIbHGcds4/D06076XFfoJFPayxSQsFcLLyz9eO2M4x7d6lknjg2q8qxKBgAkKB+FQQ+ELUXqC1vZYpWjJyBhcBipPOSOg4w3XrXSaf4F0W3aGeZLi4nQh/9Jm3gHOeVHyke2KTl5DSRzR1KxiVmNzHgds5JPt6/hUKa9bSu6wQXd06KWKwwkkAdSfQe9d4PDOg2cGV0uzG0ZDNCGP5mrNn9ktk8uDy48cFUG38/WhOT2QaHn9rqGpzSb08O6oydVKIw/8AHiuP0revLjW77SJrZNAu4DJGQJJLmLg/iQf0rRubq8TUCVvIthOUiXliPQj8DzWrA009sjzqA/dR2P1p3lbp+P8AmLQ8z1Pwxf6dFZX6GK3miK7zNKiID0wOcn/A/jWvaeFNcAfzL6ztEIO11DTuc9jnaBzkgjn65zXRPqfmu0UZhQxHgv6DrioPOe/b57rbD1CqMlvwFHLLuFzKg+H6yr/pmvXs6Z6W8axD17ls1ei8A6BbMZJ576X186625/FAtalm4iTZskLA/KWGP8muB1n4n2VtPNBZQyXbLIQZA4RCMdjyT6dAPepcV1b+8abexl/FTRtL0ddIXTbZYPN84yYd2LY2YJLE+przmt7xL4nm8SPbeZAIUtw20btxJbGecD0H61g1nK19DWN7ahRRRSGFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAV3Xwu0Eapr8moOU8rTwrhWGcu2dpwRjA2k9QQcYriIopJ5kiijaSR2CoiDJYngADua+jfDWiJ4b8OWtgiB5FXfMyYPmSHljnAyOwzzgAdqqKuyZOyNGZnPlGN0kcMGKKRuI9smrHzKxADAHuDQuMqduD6d6OW5yOe461oZDN3ZSfrUKTCSWRTFKrIfvMnB+h71YI3DIPfHBpqxEZJUA/XrQBA0Ks28blY8Z20rPtZVWGV891X+eas+wX8KQYHPf60AIVHlkbRk84//AFVX8u5DAAxYAG4YIz9P8mrBIxknn0B6VF9pjaXYsyuf7iHLfpQA5kBHXn0NRNCnmKZVWVM528Aj6ZB/X+VQf23ZSErAJZ2GRiGJnwffA4qv/aN7cKY4dJuA/TfJ+6Vfc7sH8gaNt2BZt75VvmspmKtt3w7myWXj+LAyRnngfrxcMbDjeR/dPpWL/ZMjWG2a6f7WDvjcOxCvk9s+hIPHSqklr4iuWzNqFlbR5yqpG7n8SdtNyihpM6BZginzCoYHBGc1Vm1K3ix+8wSOo6VlNoLSEtd6pczHvsjVPrjgkD8alj0HTYeTEZT6uxP6UuZdgsTNrVuHIM68AYAOTVN/EEcshW2huLpsZKwJvwMZJNWltbWIEw2kWT1Kxj+tPYxyxlWZTGcAp0A/Wpu+wzPe/wBTcReVo07hx0MiKPoeeKijfWvOijfT1tY5pViLRyedIpJ4O0cHOcYB/Lvs7yFyDwe47Ux7mJYt0zK0akHeoJIP4dOaNQMI2GqSzywf275cqNtkjWDhOOQ27Dgg8dPoTSx+HZp4nW91W8lcj5VhlCqCQMZ4+vGD9e5feW8dg0cls5dCwL5YMAWIxuJOQOvI/I5NaMUjSW48uKSOTarbJAV4PQ89RwcHpTb7MLkGm6HpNigka1WaZcqZbnLk5GMYJxjBPb1z7W5Io0khlgQW/koUiW1ZowqkkkLg9CSTUcElytz5VzyGB8tgBhsc849vaonuT/aCRs21V6oQctnGD9KmyGXV1G8lkDO0q7RgM0pPHpUMM7y5LAJjj602ObzYyAjhgTwykZ96XYJGGYzu6gmmIlePL7myR/d7U13SIcrtHXhSR+lJhk9WpWj3FWYEFeQQaQCKyyoGwcHoSMUxl7HmkX5GIZ3bJyC2OKkOVYcA0ARYJGOwqM9RnirDHPPT2qAOjH17UAM6HjrVeaGGdysg+fGcg4Iq06Ky42/KRg5qv5ap0UAAUDFyyDeGbcOjKcEfiKrrsjSW5KyXW7kkPyc9S394gjtg8nJOMGwo+U7STnnmq8iPG5lhAyT8y4+9TA0n0Kd7H7ba/wClW395VZW4yCNpAPBHvxWK6bQegFa+k63LZONjK8XR4JCQDzyR6fUU6/stOu/OuNKuPInCtLLa3Gdp7kqx4H0z6fSkkMwOOO9NKnGVP4VMuJoVZWGcAkf1HsetROMHr+VFwGNypBH51E6dPTsakKk9QCR0oIyOelAEJUEEA8ewpjx568ipvKzyMYqME4IB/GgCHyv9kUVNuHcNRSA1tV1lpWxjj35FNjBmjJcMFIyR6VkTT7sLzvUckdM1Oki26wo772b5QoPzZ4xgDr/9cV0rsYO1iW8lSFgEchozu+U43e3Patyw0BngXVdYDW9jj5IGBEkvfAHbp9as2Om2nh6yOr6tCZ74/NDbsD+75yCx7Hocnp0HvRutfkn1F7me+Zg4VTHjhQQCAB2HP44NO3QRNqWsu9ukS28VvpUQBht1Uc5/vc+vOO55PIAFW0sn1G6hmm2qhH+rXgEDHH+fWsXy59U1BmZwY1IAx049vauxto49PgQP80hOOPU/5FLceyuTYKOFA2J/DxjPvmh32MuFLkMM8dM9P6VUuL2H7R5e4hwyjhOSx6KPU/59a1YrcI3nXM0QZZcCFZASowSob1JxzxgZHpmmlZE7leSD7JPDf3YCMSPKIRmaEEZ3EcDPTj8KZdyw2unzSqYWaONfLk8jYZSf4jwSWJJGD6+ladzrMaQwfarNrdY2KFgzNJ2wAAOeuMnH04rBuZHuLpbxGZIwP3cbt168njrz+ApPYfU5/U11CVGadp2MpJLOSeOy8+n+e+efZBF5arJli4Hzc8fWuo1V3ELSSsOByqvu/wAK56bZIoVyIlUhgenPXrWT3NESpII0BQbWHVsc0m9n65+pPWo/OGSAd2TnODt/A4qVNxIVFDE9NvzE0xirG7HPAH1qTb9M1Hubo2V+tKZNsZGR7nFADlBHHSpNwxgrz71UWaMN80yKR1DMKke6t4gu+4hQHpvcCgCzuBPFJvXOKSO3kMJdYbmdevmRQPIpz7opFSJp1zcSqILaWTgH5gI8fhIVP5CldANVg49fQ03EoOBsPXsce1aLaDqccW94YIlXqZpQP/QQ1Vkgu3lKZt2UcFkJP6lR/KlzAbWhRW96YxFHILqIFp3QZBj+bBxnHUgc+9ZnifUIbjXzDBFNAkCAFZCMZPcADgYAqzp0v9ks7mQb3YMSuH2kZxgY65J568np2z7uzjmvbjULiW7Jnk8xyFIDk++CeaqT2sC7lA7pEkjba3OQQBkioLmzjZmeSUnfknO3JPqcg1u6Xa2trLHa6tpjkXDHy7iZyBxyQwJwuB+f14rcim8N2YkggFrJkfOsMJlBPXBwD6dCahp9BnCWsVkCIwXu35xGqbyM+gUVpw6RqHD2mlmJeeZV8s/k2M/nXSy+ILdVaG3NwoH3Y1t2iH1y2B/X86xJtQlkIY2bycDAa5Axx3GKai2DaRTe01JT8z28ZPGGkUgf98bjTRa3juWkvLZNoLAqrsF/DjNXpZWkeLyrW3RD9/dMzED1xx78Z/Gmp54ZkZrbyZFw4VGJORyMlunXnGfYUWYXRkvDeCd4Jr9o23EKFtwhIHchs9v6VrWUv9nohRY5JFHMtwPN3e+xsoPwUfnzVH/hH4JZPNle4kcDGRJt/lU0Oi2tsc+QZG9ZWL/zpoVyzaajPpryvZXj2sko/ePFIUZvqRzjmmXGrS3bF7i9luGUdXkLnBP496etpHtKfZoNp5/1S/4VKyuM5bOevP40+lgM4X8ckhVVnZh1xC/H6VKruWAW1nOT1ZQv8yPSrLBgeT+vWoo1PmF2IVB05o1AfHaapcYFppFxcljjETIT0z03VDHHfO6u8UMSEZKySfP69gR/k12tnJbaV4fFxI6LNLCTFFPASCzkruViMEgY4B7nI6VzKlUiWOMnYigZLEk4HrQ00wRW+x3Hnu32qJY8/KnkFjj67hz+FX9PNnZWzxyRtdmRTuMhXHPp8pI/A1WVt5xkk+tOXzEBAIwRg8UeoDJtO052yluIVxjahzn3+bJ/p9KrNommOuw2hZcYw0r4P4Zxn8Ku4JXpgn3pQ3GGpWTAgt9NtLUiSC0t42XDK2wFgc9QTznmp2l8xVjc71TO1W6DPWm7iOnSl3DrwfeiytYB/nuI9oyB2IJ49hSK2CSck+uaaSvTv2pec4JG76015AO3ZOVYqo6jjmmOWK56/SjHHIrQtrW4vbTybdTsOfMLIAo9Pmzz+lAbFDOMdMitOwvfKVlklCIx+YKmWb8QM1UlsGgiLPJGWDbdqvk8d6fFaGREcPEp3ZGZMH6YzQmGhowWn224jdQ4YozMAAAAZHIyPX5QMnJOPQCte4j8q3HmTLG2PmfPCjHPWq7QxaQFumlXzJIF3L5nAI3HoeeSxPHr07nB1TWZNQSKAArGpDP0+c49PTv+ApiRZuNUlMDQQ3MoQY2ykhSPpjoPrmqun6hNaTsWJdVH33BOfpVm301Y4xJdhyuCW2DhMDPJ/pWbIVDOEY9flz2oUgsa8V+k92xuUiO8AfMM8fl/Ord5fG2DIm1sABUQg5/LpWDDPIm7ZIQD3Aq7bR3EjrKFMiD/AG8Z+tPQRNb27XFsrvAkcYGd7cFfqOD1/pUsWY5g0MTTqCMu0oLLx2HQfh1zya0YrXBEj7A3GEUnimyxWcyM0aRFFY7yg4z7471IWK2oxwrp91BDqItpJYnQSqdzoSCAwOc5Gc1843NrPZXMlvcxtHNGcMrdRX0DJFYsu392gVjhlU5x78D+vbmuS8QeEl8QbZEuEt5ogwWQoWVh2DEcgZ784yeDSnG5UJWPJqKvatpN3ouoyWN4gEichlOVdezKe4P/AOvB4qjWJsFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUVPY2j39/bWcboj3EqxK0hwoLEAEn05oA9G+FXhaSa7TxHPI0cULPHbxhceYSpVmJP8IyRx3B6YwfXHKgHLYH0qtpUWmadpttZwN9qW3iWJWactkKABwmPSrVxJa3dsI3tIkXsfLYMfxPNbKLRi3caZosgs49smiDzLiTFsrSA5JZVJHbv0HUfmKjilsbeVRF5MUyrgcBWwOfr75/GrHnPIRli2ehZiadhGfdahNbT+T9gvHk7COIlTz/e+7+tQfbdYb7uj4GON86A/jya12TzMjeyEeg6/nVeBohvWGQuQ5D5k3bT6e30o1Axby416VxFZW1vIrL85Lsmw9jkgZ/AYHHWprfTNZklja81OGKNcb47aHJYem5un5VtNDaXKxi8kaF4yPIniUllyfmU4PIPoQcc45qtaTySoQ0c0Ui8FZkKn9cZB65x/Wh3Ei8DGIUj8iBtowWaFSxPqSRUDXUoj8oMyovRRkAfQdBTdzl8cjB69jQYiecEmkO5EXZlwzsccYyajC8EF2/PpTyjL1pBjFSURCNg24yswzwHA4/ICng9Qec04gZ700qT6UADk7ef1ppJzyOKGQ9PSmnK8cGkA0gUhRCF8xVOOeRnFHO8NjJpoO7sQc0AHkwnnykBPP3QKZJGr5BUA8AsvBHvmpCOcYoKhTnHTtQBDMhMSpJmWPODxz/n6CqD2iC4DQyLHPuP70KxZhz8rjcARkjkDI9a05CSpx1NU45Ibo/Z5v3dynIXPzDHGQe9CAXUtN1aMWryxtbpIR+9ikDKw74I6cetJcRRmUyKgEjDCt2HORwePxq/Y6tLp4kguHa5sHBV4vLJJbn7o7Hp7fTrVO6vbSO5CxmVIXHyNMMZ9vr7HBppMLjIVfyUPmkk8kt/niptuH4JwR0rDk1R8yrZ20kiBvmZSB37CrVlqsN65RQ8bLyVdcfyosM1CdvbNVbwPJD+5Y5yOh7d6lIDnucehoIVBgKAOuBQBXtGeMFHY5X26ipjICcCM/1NHmbcnGAOtNWVWXcpbB9BSEJIkgxtGAfXnFMKyBiW27T3xUwkbPPI+tNYF/lPzZ9cUAV/M3NtQBiBnGaQ5C7iCD3HpSw2oglZ1LgH15qVgAehP0pjKnmrtywKjjlsc0rcgEcn0xUk0aMpDKpHow4qoZegKnjggDA//VSASW2inIfZhx0fHIP9aIfMRQJsiRGyHAx9CKkWQ4wWHvgVDPcqoHmSKqngZYCiwF7UtZu7/T0guLaC4aIAJMwIkA7gkHOPpjp3GQc++ghVEminJj27nDZXZyB1YAkc+nrwKge9jFq9wqtJCjbC0alsH04702yv7qSRTBaOkZ/im+VD37fSnp1Aao3fwMPc0m3PGKs6jbySXKGznWGHb84Ve+c4HoOcfhUUttCd4SeYo42kuQSOnQY46frRYCu+E5LAd+TVZ7+0Q5knj4OMBgea2c2TWMcD6dZu6jDTeWQze5AIHb0qtLOFt4YkjBih3eXGqDCFuTj6n+dIZEIywBFjfkHkEW7EH9KKnXXdSRQiiRVUYABHAopWfcV2YtqwW6iaZSzM2EUdz2/WvQ4odJ8FwxSagm/VHUMNqb/K4x8ueM4zz0/QUUV1wSZjLexmeK7pxf28hXFve2sZJGC3Jbk569a5p/DszXACyhUIBOWycUUVm1r8xxVjpLK2js7VCjbUXjI71p22nvqc6RSSvCsmN2zl2Gfug5wM9M9s0UVdid2O1qXTdIMTwQQfaSzKHMeCCpGUUhc8H+LjPbFZmteJJ7PTEt4Yv3wieckncV7E5Pvnjn2x3KKuyUbkXu7HGxeI3lvo4JJGW1lG4IUVckk5X5eOpOWxk89Mmum/t2COEqykN/CXYnPqSQM0UUviV2aWtaxm6pqsU0azMkRXbujVy2BzgEDB5z9Kx57LVrq/jWAxXjOm8IrEBF5/vY9z3NFFc1R8r0HeyuasWgayUJe0VADglp1AyOvTNPi8N6pKZHd7OOIHnEjswB79P8+lFFEfeVxKbZIvh54kZJtQaUowV2SPbtOenX264P0q/Y6TbLaGZZ33NHtY8jORgjI55Gc9u3OaKK1hFO5TFmstJtoiYbBb+YEBlZ3RNoJGTuY+3AGD+GKrLdanbWwh03TbKxQcYjuP5/u+TRRUzgk9ATKv2rVpW3XVzaFeuBvY/wAlFXYhrN0Eis9UVJGYgqlqq9v7xYn9e1FFDuluUMvNB12CALca1cy+Y/IaQsAPTmq0ejTbgwvbtF3FZF84AN/3yo9/zoorNN73E2x0FrLbWjAXtxNHgjdLISfT/wDVxU96lpFLHGlrC1xGSZHZFbD98fKMdBxzg5we9FFaL4bg4opzPPO2ZGAGdw4HX14/zzUbBzjL/nRRQtWA/wAp9pZpDgd6ktYVuJJEWdF2Ebtylj1x7elFFNK4yX7MUcc4UH6Z9+9XEtI0OQSc85NFFC1EKuxJDnO0HgHnNNd0/vHB7Yoop2AhLgHK5x060xjluv0ooqethjGyTzginRKAGCjLdh60UUIGbmr6vc32m2Vi8aR20cayRkJ8zj5kXLbj6NkYHJrFxtU7Sc0UVKlzaiTHJx07DrTyTtBBooqhjN5yOmPpT+qk4FFFCQDR03EhcdPehPn5HTPFFFSkMcVKMRwSKcQ3HIoopoT3EYHAyOaBjp04/OiihDDaw7DGeua6fTtNT7Irs4YsDyFwQD2BoopdBMx9evrqeW3WaQGN498cYjUCMHjAPU9O+KZpUVg9rJczs0iRdVGcD6+tFFXJaNkrY09ctrxLVjDLEliCuY1BDccY9MZArnQeAB9OaKKgvoOUkvgHnsK6fTNPEEKySxKJM5BByRRRRcll4oZDlsrhsjnP/wCqoIts0js00qrGdoG7IPuQRRRVLURcW3jIwo+QjtVd9M3o6fvVVjxyoUY9h9BRRWbdmM8m+LWix6beaZdrK7vcJJGwbGAEKkfj85/SvOKKKh7m0dgooopDCiiigAooooAKKKKACiiigAooooAKKKKACiiigArufhTY2174wZrmBJTb2zTxb+iuGQBseoyceh56gUUU47ilse4TRLNE0cg3KRyM02HcilXffg8Erg/j2P6UUVsjEkznIxkHg1G8VvIhjkgjZT1VkBBoop2ESRRpEfLiiVFH91QAKry28Vv5lzBFFHKeZGC43geuO/vRRS6gTOY5LfbtGzHTFUL24uYgl7I8s8qHZKHwFEQ5DZznI57c8+1FFDdgZYt7j7QrFk2sjlDnnPTkfmKmBPTGaKKVwGyqHHQ8VBuGcd6KKTKTF3Zximl+dtFFSNiLyenenHBJ4oopiRGdqngdPamsvzBj0oooYxANp4zTMAsFPJ60UUAIFAJxz7VUuLWG/jKTxBkzjBAPINFFIAjaWHMM7h0AyjgYOPQj1+lVdRVJ4njKozKpwSoJHIJKkj5TgDBHPFFFCBFW6tpLS/Gn6lCj7kMkNxkEyKDwSMcH1HtVDVLdLeSSdBKQrAhi+Gwfpj3oopoaK2narZ2+pSTyxFGfncn8XrkDA6V1N5dLBHGQpbf90D/PFFFMGNDBG3AkE9vXP/16lEIx82CSKKKkSI8EHtjrSNlBw3X1oooGVmvFWQozhSMkgA/TNUf7ft2n8mEzTys4RUUAZJ6DJxRRWjSUXIaHA6k8EpitII3jbZ5byc/oMY5PfsfxfZW2pNeRtfG0W1/iigB3H8SOPyoorDmbSJua2+yhcrHZREMMEy5fP4E4/Ss+W3tSZf8ARLcCRt7ARjBP0xRRVWVihm6KFCsKKg/uquB+lQu2Dz1oop2AiYg85I+lNJI7ceuaKKBDDz68+9MOQCM8d/WiigZkzeJNIgnkiku9siMVYeW5wQcHtRRRU3HY/9k=" - } - ] - }, - { - "items": [ - { - "mime": "image/jpeg", - "value": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAKXA+IDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD3WlpKKxNBfwoFJS0CDvS0lJQA6k70oooAAaKKTpQMWiiigAooopiCiiigAooopgLRSUUgFooopgLRSUUAOzRSUUCFopKWgApaSigBaKBRQAUlLRRcAxSUtJTuAuaM0lFADs0UmcUA0ALRRRSAKKWkpgFLmkozQIWikpaYgooooGFFFFABS0lFAC0UlLmgAooooAKMUUUAGKKKKAEopaMUwEooxRQAUUUUAFFFFABRRRQAUUUUALmikooAWgUlFADqKbS9qAFoxSUZoELRSUtIApKWimAUUUlAC0UUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAVKOtJRWRoOozSUtAC0lFFAgpc0lFMBaKSigBaWkozQAuaKQUUALRRmigYUdqKO1AgooooAKXNJRQAtFJRmmAtFGaKAYtFJRQIWlpKKAFooooAWikooAWjFJS0AJiiloouAlFLRTASlzSUUALR3pKM0AOpKKKAFpKWkoAWikooEOoptLmgBaKTPFFMBaKKKACiiigApaSigBaKSjNAC0UUUAFFFFABSYpaKAEopaKYCUUuKSgAooooAKKKKACiiigAooooAKKKKAFozSUUALRSUUALRRmigBaKSigQtFJmigBaKKKACikooAWiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAKVLmkorE0FzS5ptLTAdRSCigQtFJ3paACiiimAUUUd6AClFJRQAtFGaKAFzRSUUgFNFJmlpgBoooxQAUUUUAFFFFAC0UlFMBaKKKBC0UlFAC0tNpaAFopKXNABmiiigBaKSigBaSiloASilpKYBQKKKAFopKWgAooooAO1FFFABRRRQAuaM0lFMQtLmm0ZoAdRTaXNAC0UlFAC0ZoooAKXNJRQAuaKSigBaKTNLmgAoozRQAUUUUAFJS0UAJRilopgJRS0mKACiiigAooooAKKKKACiiigAooooAKM0UUALmikooAWikooAXNFJS0AFLSZooAWikooAWikooELRSUUALRRSUALRRRQAUUUUAFFFFABRRRQAUUlLQAUUlFAC0UUlAC0UlFAC0UnFFAFKijiisTQXijNJRQAuaXNJS0AFLSUUxDqSkBpaAFopDRQAtJRRTELRRSUAKDS02lzQMWjNJmlFABmlBpKSgB3FFJmjNAC0UlFAC0UlFMBaKKKAFopKKAFooooEFLSUUALRSUtAC0UmaWgAooooAKKKKAFpKDRQAc0UUU7gFFGKKACiiloAKKKKYgoopKQxaKKKYBRRRQAUZoooAXNFJRQIWlptGaAHUUlFAC0UlLQAUZoooAKKKKAClpKKAFopM0ZoAWijNFABRRRQAUUUUAFJilooASilopgJRS0UAJRRiigAooooAKKKKACiiigAooooAKKKKACiiigBc0maKKADNLmkooAXNFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQAtFJRQBSpaSisSxaWm0vNABmiiloAWiko70AKKKTtRTAWjOKSloEL1o6UlFAC5paSimAtFJRQAtFFFAgooooAKWkozQMKKKKAFopKKAFoozRTEFLSUUDFooozQAUtJRQAtFJRQAtFFFAhc0UlFAC5oBpKKYC0UUUALRSUUALRSUtIAooopiCiiigYZozSUtAgpaTNGaYC0UlLQAUUUUAFFFFAwooooEFFFFAwooooAM0uaSigQtLmm0UALS02igB1FJmigBaKTNFAC0UlLQAUZoooAXNGaSigBaKSigBaKSigBaKSloAKKKKACiiigBMUtFFACUUtFMBKKKXFACUUuKKAEopaTFABRRRQAUUtJQAUUUUAFFFLQAlFFLigBKKKKACilxSUAFFLijFACUUuKKAEopaKAKNFJRWJYtLmkooAWjNJRQA6jvSUUwFzRRRQAtHekpaACikNLQIKKKMUAGaXNJRmmAtFIKXNIBc0UlFMBaSjNLQIKTNLSUDFzRmkopgLRSUtIAzS0lFMBaKSjNAhaXNJRQAtFJRQO4tFJS0AFFFFAhaSlzRQAUZoop3AM0ZpMUUALmlpKKAFozSUZoAWjNJmloEFFFFAC0lFFAC0UlFAC0ZpKWgQZopKKYDqKSigYtFJmigBaKTNLQAUUUUAFFFFABRRSE0ALRRRQMKKKKADNGaKKAFzRmkooELmlptFADqKbmlzQAtFIDRmgBaKTNLQAUUUUAFGaKKADNGaKKAFzSGiigApRSUUALRSUUALRSZozQAtFJmjNAC0UmaM0ALRikzRmgBaKTNGaAFxRSUZoAWikozQAtFJmigBaKSigBaKSigBc0UlFAFCjNAOaKyLFopKKAFpaSjNAC5opM0uaAFBopKBzTAWgUlKKQC0ZpKKAFpaQEUtMQUmKWigBOlLmiigAozRRTAKWkozQAtFJmigApaSimAtFFGRSAKM0UUwFzRSUUALRSZpaAFpM0UUALRQKKBBS0lFAC0UmaWgAozRRQAUUUUAFFFFMAopaSgAooooAWikooAXNLmkzSUCHUUmaKAFopKM0wFoozSUgFopKWgAooopgLRSUUCEdtiM2M4BOKEYlQSMZ7Ukv+pf/dNEZzGp9hSGPzRmkzRTAdTT3opD3oAfRTc0tAC0nejNGaAFpKM0tABRRRQAUUUnFAC0UUUAFFFFABRRRTGFFGaKBBmlzSUUgFzRmkopgLmjNJRQAtFJRQA7NFNozSAWlpuaM0AOopuaM0AOopuaM0AOopuaM0AOopuaM0AOpKTNFADqM03migB2aSkooAWjNJRQAuaM0lFAC5opKKAKGaX2pM0d6yLFzRmkooAdk0U2loAWikooAdmim5paAHZopM0UAKKWkzijOaAFo6UmaXNAgFLmkoFAxetGaSigB3Wik6UZpiCijtRQAtFBpKAFozSUZpgFLSUUALRSZpaACiiigBaKSimAtLmm5paAFzRSUUCHZoptFAWHUUmaM0ALRSZpaAClqtfXcVhYXF5MHMcEbStsGWwoycD6Cqfh/VjrWkpfFVXzGbCrzgBiB+OMUAatFAx/kUZHXNFgCijI9aM0AFFUL/V7TTrqzt7hijXbmONscBgOAfTNX6ACiiimAUUUUgClxSUZoAKKKKLgFLSUUAFLSUU7iFzRUEl5bQkiW4iQjqGcA9M/1rk7/wAWySo0UMYiB4LhyT+B7VLmkFjs6K4az8U3VvGUKpJly2WzwMdB7VuQa/HfW04RzaNGqnzWUOOvTH6UlNMDbk5jcf7JoiI8pMegrFu/EdrarCgfzt6gsw4wP8fb6Vlf27qCr5cIT5ydr8E4Hpz6etNzQG9qusrpbxK0Jk8zJzuxjGPY+tZy+LVKsTaYx0/e/wD1qwdQ1KfUxH5qjMYIzjGfw/CqRfYrLye1S22ykkdMfF0hU4tUVvXzCf6Uh8WTHGLaMdmyxNcrvOcbevvUqCWQ4CAE+9CuFkdN/wAJbNwBbR5AyTk0p8Vy5x9mj4x1Y1zggmzgLxjsak+zzf3e/YinZhodAviuZnx9njC5x941Z0zxDLe3kNu8KDzCcspPHBPT8K5cW8pIyv61d0hZLbU4JpNgRGO4s+ABggn9aLMWh3WeKonV7UXHkFm35I4UnkVWPiKw2tl3UZxkrz9cVgW6D+2YxA7hZCXRUYEgj+9kY9abl2EkdsDkAjoaKr24uwT9oeMjsEGMVYqkIWjNJRTAWjNJRQA6ikooAWikozQMWikzRmgBaKM0UAFJS0UAJS0UUwCiiikAUUUUAFFFFMAooopAFFFFABRRRQAUUUUwCiiikAUUUUAFFJS5oAKKM0lAC0UlLmgAoozRRcDPozSUuaxLFopM0UALRRRmmAUUZooAXNFJ70ZoAWlptLQAuaKSigB2aM0lGaAHZopuaXNAC5pRTc0ueaAHUlGaBQAUtJRmmIWik5oyKAFzRSZooAWikzRxTAWjNJS5oAKKSlzQAUtJmjNAxaKSigQ6ikophYWikooAWikpc460CFqOe4it4WmmkWOJerseBXGeMPE2raRqtjbWtoEtncSNccP5qjG5MfwcnBPX0xXJXviC71hpXvbkoq8xqM4xyCAB/X8T0pb7A9DX8X+M5ruKbT9NVxbvGQ8gXLSLzuAHYY6/j05FYmjavNZQLG7Si0L7I3RdhX16cHqD+PHFZt5dyXMnl2vSMBTMeDxgg5HXHb6DvzVdry6gnZrvfMGXBY/MDjp1FXFWE9T0CPVbsLuhvrh0I4cTcfqOD7e1LFrupFlQ3bjPGDJnAzjNcXbXDpcm4sLnPHEeeTjGME9uBwe3rT7zW9XnjSaJJ1mjkAkiDn5hg5G373JPv2qr+RNjt/7Y1AMQLuSXB7Z4GOvB98VG2u6ghDG7yeePMwD+vtXNx3V1cIqzMyzJw6iTJTjIB6ZOCPz71M4d5dxlk2D0bB/EE4xV8pNy5rF7fPDDM0jGSN/MjfzN5VgCQByfQdvr0roPDXjmDUFSDUWSC4PAkHCN9f7p/Tkd+K5C6smuU2yMWAGQdxXHbHWsqW3lglHmmTYhyJUXJGeuV/TNS4saZ7tke1LXio8V6xZ6c0FnebQxXy+cqmCCNp/hHGMHjB6c16rod5dz6ZaNqTW4vZohMY4FICqcepPr/hWdi7mpRRmjNIYUUmaKAFopKM0ALRSZrB8Qa8dOVYrZ0MzfeOMlB/LNJtIDYubuG0RWncIGO0E9zWLB4qtpZmWRDEmSFYnPHuK4651S7u1xJPLIuc4ZiRmqjyOQcis3N9BXL+ran9svnfBxnA57VntNg/KMGmrtCk8sT1odEI649sVnuF0L5pyD/KtK2C+R5jcZPGeKxlVvM2gkntW4xEeyPBwqgcKTVwRWhHI/71Qpix/ESVH86ltmeO6EhnRgv3V3A9v8aX7QcYCufwP9aaZiVOUYD+8SAB+tXZDvYkjiURDdnI4x+lNnBK8JgZz071agXEOeOcAd6jnJ2/T2rbl0M+bUzd2H5q5btmRcNjg/yrOd9sgHGatW8oEgJYAYOT+FQmUzSjOUJOSef51IPug8VnR3yDdnoOnvVo3EaxAtjFUpJkMnc8DAxz2ppd1Bwxzg4Occ1GssbqCGB57U/wCU5+lPcEyCS8cAJKuR/tDPWrAukmlSQFxKMhSAM84xjpgjH61Rlboc800qDhlOKycexaZ0emandxX628s6TRn753g447nt9K6rNcBptxDbXSG8hUqDwWUNj+v5V2keo2kkYf7RCATjJkGM0R0BlykzTUkSRAyOrqehU5FOzWghaKSigBaKSigBaKTNGaBC0UUUDClpKKAFopKM0ALRSZozQAtFJS5oAKM0lGaAFopKXNABRSUUALmikzS5oAKKTNGaAFopM0ZoAWikzRmgBaKSigBaKSigQtFJRSAWikozQAtFJmigdjP5ozRRWRYtFJRTGLmlpKM0CFo70lFAC0tJmigBaKSigQtLTaWgYtFJRQAtGaKM0ALRSUuaAFBopM80ZoAdRnmm5pc0wHZpKM0lAhcUuabmigBaKSkpgOopKKAFopM0UAOzRSUZoAWlpuaKYDqKSigBaOe1JS5oERtcwpPHA00azSAlIyw3MB1IHU1zviLxRFpnn2qShLpFR43GHDHdgow6g4HT37VkfELUvsVzaBNRkhfyzugizvZS3B9MEg5/3a8323N5IZXDKkikNJJ8zHnt3HSkk5bA3Y1tY8QNfSB5YwypLJJBCOdrMct9Mntz36ZrPZZbiQtdEordIx1x7+1SQW0cJby1AbuxA3fn2qfy0QbiQwz0U/5/WtYwIchUiIQIiBEHcDg/40SKEXy3AdO+R/Sia4lSweW3gikkDKu2RRg5Jznn2otpnlS4a4t4Yljm2IUTb8uOM8/X8qonUoSWTqd9nKQeuwnr9OxqSDUmhZEu4yGRgQ7DOD9D1/DNW5UgyGEuGI9v/iqhIdgVmQPH2LDP8uR+tKw7llJElnLI2IzKZJFjYjc3HXJPfHGfTgVq2MrT2wMjRtckkMqso2nPbJHQHqMng96537MkT7rScxEZOx87ckYJyO/1x0qMasbCY/bYmkORtxIVHQ5LY7dOMdutNOwrXO0eJkXoGGOMY4H9RQ0eQFKblPqMfyrIh8baY0Kb1CMRlgZQdv6Usnj+wVwNqMgweVJ9fQew/Or5hcrJptJifLxlomfnaBkN+nP4VNFc6vau4AeRX2+b5ecuFJIUjrjkjHOQx9aypfiHat88UUbE8Omxs/nmmTfEGxaM7bVQ2e6Hn9al2fQaUjvLLx5AiTnUYrgS78r5cZYKDwFwMkYx7966nSr9dT02G8QALICRgnsSO4B7eleK2PjFtSuY7SGxE00vyfKm3OfX5un19K9d0yf7BosMU95DPLGpbcAEBUscAD2HHrxzWUkkrlq/U2aM1zs3ieIoDCpVupDYx+lVrzxawVRaxjPO4uufpjn+dZc6KOrLAdSBWLqXiO2siY0HmyAkHBxtP5c1yV3rN/K7FrqQhx0VsDHpgVmSSu+4s5JJycmpc30Ea83iO9aZ5EnIJGBjtz2rHmuHmlZ5XZ3PUtyahyPUGoi5zgcntUDRN5mGA/SofPwSc9aAcH5mGPaniSMdAufoKBN2EMmzsee9NYuAGCn3pXmJwMYFIJXPUUaCv1LGnYmvowwAx8xOew7/AJ1feXfM7B2GOMBc/wBKz9L3fbJH5wsZA9skf4VZE4wSZefdkH8q0jsWibdnHzyfipH9KXdjn95jPXJP6f8A1qgE0TD5Jgw7nzcYPpxT4W8yVRkkZ7SlvzFUgehrp8tsuTk5NUJ7lj8q9jyasXVwLe3TIOQBmsSa5Z2Yrxu6nvWs5JaGSGTSr53XmojKVbrwewprOBjAHvmmrg8EDn865ZO5poTpKCc5596nE25SN3TpzxVF0K4KjIHXPNAYDqCp9qSbQaGhHOysMcBea0P7SDbflHIyeehrDMuw4/I0qyZHFUptbC5TTebcVIHT3p8UuBgnIrOWTAxzTjKykdwTVe0Yamn9owSCMrSjDkmMlT71neZubjPNWVlCxgE01K+4bGpY6pc6bJ+7c8jBV+VJ9ccV02j64168kd35MMgxtX7pJPbBPWuKW4QrtbBHvSKQSSpJH93PWn6DPUaM1w2neI7qyfZMXmizysjEsPof8iuwsr2O+tVnjDKGz8rdeDiqUr6CsWc0ZoopiFpM0UUxi0UlFAC0UlFAC0ZpKKAFopM0ZoAWl4ptLmgBaKTNGaAFoptLQAUUmaM0AOopuaKVwHUlJRTuAtGaSilcLC5ozSUUBYWjNJRQCFzRmkooAXNJmiigAooooAKKKKLgUKKbS1kaC0ZpM0A0AOzRSUZpiFzRSZpc0gFozxSZo7UwFpabmlzQIWikFGaAFzS5puaKBjqWm0UCHUUlGaAFpabRmgY6ikzRmgQuaWm0uaACikozTAWlptLQAtH1pKKAFpKKKYC0UlLQAUtJRQAtFJS0wFpD1X6/0NLVLVruew06S6t4EneLBKPJsG3PJzg9KQip4k0GDW9MmX7PG14sZ8iXaN4IOQoY9ASMenNeQp5gjyctx1C43f4V7rDJ5sKS4I3ANg15B4nEVj4m1GGGPcnmBzjHBdQxwPTJP6VUXaViZbGT5mFx5JHpnNMYuqMdozj0qNpQ21mJT2JH+FSF1dcbA3HBxmtSBsTSMD8xz+NPfDZ/ekfUE1XkmjQLn5fYcZpPMjZsfL/3yAR+NMCZSy4+bIz13f0qQ5DDaGBxySCf51BuGR8hyfoTU0YyCcbe9Ah8m0RbpFIYdWBxn9Kx9YiXyEMedznlFGcdeScAZ+ma0bxikKjcnX/P8QrA1tG/tO5AbICIBlicDA/xqZTUNbFRMvyAtyDnDdDkgCrf2cuMb4z/ANtF/wAaZcIwuZQcZ8wn144x+lAh+YncB9/HHt1/CiWJ5UrJaq5RHFHDGXG4Lzk9T/KknRNvykMR6Zp6qArAtg7en4ipntZJFbyo5HO4jCqT3FU60nPktoHmdT4GshbRyX7oNz/LExJOB0P9RXWy3LAY5x1yO9cjoV62naVbhrOR/MkZWYEjYM9Tx71s3mpCDy441Es8n3YtxBI7nODXPVjJy0BMutJnkUwSlQRnr2rNuLq9t0LS2UKMBu8s3Pz/AIAqM1ctQ13Yfa4sOvO3YGYE+mQPwqFSn2DmSHl88ZqJzkcH6nFRmV1JVwVYdVP/ANemCbOcZJqHvYNRsjMo5ORnrQhO0EDr2pWk+YNjI9qYbkZ+YD64qStSV+cbtuQaRcqoyQDnJqs021yeSKa8xOCp69qLisyyxkzxg+mKaWdCFyD+lUxcOzBc9ulTvveDcVII9aNws1ua2llhb3c5HGAMgdSM5/pQGwPvTfnJ/hUdiPK0ZXBZXkYnO3PfHp6CpA5x/rpf+/X/ANjWyVkihkSqgf5pBuYtxvHX8Ks25XzAVEjH+8Wbj88Zqv5hz/rpf+/X/wBjVq2LPMoYuwJ+8w2/pxVRWopPQNekCzRKB/Dn+dZIfeOCfpV/W3zfKM/dQfzNZjyEAdvepq6yZmtiY5545HrUYPz8rk1ElyWye2cZpvmY+bn8axbRST6lwEY6de1ACkbex/SqYuDu6Y+lSeeepz9afMOzRZwAu0nIFMIwAOp65FIjAjuM08DA4559adk9guRCRgfap1mzjGTmmBt2V2jr0o2bckHaDSt2He5NjIypNPWU8K/6Cq6kp0/MVKJkOB0PvQA4EDqTzVq3lC53enFUvNXkbQT6Gnqdy8DGD0oi9QNhGSQAE5z61Is11bqVgnkC/wB1XYfjwRzWTDcEMPm6dRV6OfseR61spKW4tjct/FV+qBXjgfYBuZ8hj29ev4V1ltdxXUCSRup3qDtDA446V50wVwMkH6HmpbS7m0+XzoCrY7EkfoDTs1sG56PmiuTsvFF1LPDHNHb7GcKzAkEDuetdSrq6hkYMp6EHINCYWH0UmaWquAUtJmkzQIdRSUUALRTc0ZouOwtLTc0uaLhYKXNNzRmi4DqKbRmi4C0UmaM0hi0UmaKAFopKKAFopKKAFopKKAFopKKAFoopKAFpaSigBaKSigAzRRRQBnUZpKO9Zljs0U2lzQAtLTaM0AO/GlptLmgQtApuaWgBTS0maBTAWikzRQAuaXNNpRQAtFIKKAHdKM+9NzRQA6ikzRmgBwozTaKAHZopM0UxC0UmaKAFzS02jmgB+aSkpaACjNGaSmA7NFJmjNAC0tNzQOtAGbr+sNo2nCeK2a5ndxHFEpxuY+pxgAAEkn0rzWTxd4tdz/pwiUqDxabQDnGPmTOc8cE13HjVd2mWv7suBcjOFLY+VuSARxXECEsgEkbHjH3XPfoOTiiCu2S3YBq3iMvHLJf3vmId2D5iIT2yAMEeueKxby81OUXcF5ql5IqQmVg0suCBzjDEZyPbFbTpISD5RxuGS0Eh/Prn/PrXMa5OLC+ugFDCeDyyAGXbkY/iGatqwk7k1jJcPBE5u/MZhsYNIQQpBB6oc/TJ61ZjtVgQrDIojZhhPOkTb1zwIsY6dKXRYJH0uGVQ6sykA7CeMnvn29KXWrm406zSeONp13hSG3KMGraJv0KVxpt60hMdwAqhRhSxVzxk5K5HfsfQcU0eYhdw8gXGN0qsM8em3+tbaRH7Ks0jOuQCoYE59vve49KzNUQRCVsYKqflIGR/48aLWC421LXCkee0ag5Lgtj9V/zkZpqRTSSgyNKEY4I3Dgex2807RYWurR3EJkUOQzBDxwPQ+/1q9ezXdnpst1b20gMajHyuAe3PNPzF5FFrZ3jRIrmUknPRdqj0A28n3/SmWUepG9ML5lUr1UHb0xnhRWhpxlubYOFZXkOGClhjnHr7Vo2PnpexOxkZAMY2Sfz9afLqF9Dn/OWeNFIcBvvsjc4z2yp9e/5ipP7LS/mMk0jCRgoGJIssRjg/KPTtnJ7VDobMdSCxeYzpGWAVWJ69flOa6PEhcj9+rEt8+ZvT+7/D/T8KXLfcL2MUaRAEk82VWcfNuDW+w5POcgEc49an0/TYDI0awWkrISNxSCYMDnqVZTyePcetainanDyk7RuZ2lHPqM5Ht74/KxYeXPK24gOpUqTIw9RwMYqox1FJ6FK+tGgsxLGgiuYArK6IVEajuQGweFHHvnFaFnB59sl0ZSzuEkYqAxBx0AwvH6+9MnEr2V+WzhQwj2gEkeWO477t3WixiefRrdFkKkeUdx/2HUkdv7pFa8upnzaGbbRlda1C3ErJEoV1iDMigsOSFGQOo6enpVbxIIpNNWRSz7JBjfIJMdRxkdQT1z/KtBIpYvEl0xUJDJAMDOORgdPzpmoaYZNFNtHvU+YWBIHdifUDHJo5dAvqaEYt1uxGEzKQzdwSMgHOeO4rN8PTiDQkdiGYTiMArtxucAcgju2c+1bKxobhd2PN2Hbg5O3IyfXHT9Kr2toujaYzOxMCyfN5eWA3Mq9PqR/hQ1qJPQqaiMXeWUglM8tnue9VPOXGMHnrxVvxEwinh2lCMEfKc9P/ANdYyMcFuM+9ebW0mzpirxLyyptPGfX5sYqG4jOchcg91PSoxLHtBYDj04pxmjKEAt83PWo6Bs9CBd0jFVJAHUmpvLj2bB37k803gqOSAOelKAM7ifpQkNsiiYIx6Ag4yc5pZX3FQpJyec02dFIyoOe+KfpkLTajD1IVtxJ9uf6UktbFJ9ToJ0WGCG3OAUAPLYPA/wDr1CWTu8Kj/afNFzcn7XJ8kRxgZaUD9KZ9rIyAtsD/ANdx/hXQwQjMhOBLbjPrg/1q5pTrLIjrE0eTkcjHX6/0FUTd5OPMtR/22z/StHSkCbdo4x0H3fw9qcFqTPYz9YkH9rTLyQNv/oIP9apSAlOGOO9TapJnVJ+4LAAfQCqO4oxyfl9qzqfEyEiwuxTjAHt2pXwRyADjiqm7J+Vhg0qyEDBOfXmsixfMZGyxC+9TRzhhg8+vFMR0Yj5flHTI6VL5aqQdg/KmkDa6khCsDg4NODiNVyc+tMBw2ATj0oJA4YHnvVWJuTkg89+vTrTQ4cFTkH0qJXQEAMfbqak+X0yf50bjQ/cwGDxS4DrxjdTFJJI/h96MbeVDYpWHcVVK8nrQHKOTmkDh/Y0jAjJDAipfkNPuWBKp64PuKkSby+AxIPrVMg4Gw4qUA4wSM0JsC5HP8wCnjv3q4sqN3znpWMSy/LyM96micoMk5/pVxm0DRrsivywGcdR1FS215dWE6Swt9w9hwfrjGaoQzluC3brmrQlPUD61tpIVzo9P8Uu06i/8hImBIeMNkenGTXQ297bXYJt50kxyQDyPqOtedMiycqArU+2v7vTrgvCRCXG3OMgj8c+lKzQ7npOaXNczY+Koimy/BWQfxoMqfyP8q3ba7hu4fNgkDpnGRxg+4PShSCxY3UZpuaM0wHZopuaWgBc0ZpKKAHZpM0lFADs0maSigBc0UlFAC5opKKAFozSUUALRmkzRQAuaKSlFAAKWgUUALRSFgO/60wzRggbqYElFUmlct94/QGkM74xuNK4F2lrPWRh0Y04zMf4qLgXaKo+Y/wDeNFFwI6KSjNZFjs8UZpuaKYDqKSigB1GaSimIWlzSUUALmikozQA7NFIKM0ALRmkooAdRSUCgBaKTNHNAh1FNzRuFADqM03INLmgB2aM03NLQAvFFNpaAFzxS03NGaAHZozTc0ZoAdmjNMpadwHZozSUUAO7UU2lFMDC8WoH023ygYi4GPlVjna3TcQM1wChI8fKque4Qrjvj5XA7V2PjbV9Ogso7SS6gNyJlYwblLAbW5IPQciuFaa1nk3C4VGRSF2NHgc54ww/X1PrVU+pEy/kttJTI3Eq4ic8j0w+R2rIvtMgaSWT7Gq/KMAxY7fj/ADq/PdWccmI5Idr5A+WLbj/vrj8a5XxO7XCRpYz7SrfOBMg4P+6xGOP1rS9tiLXOi0mGQabE8aq6KCTm13DgnqSwzjpTb/ULHTViluvkQthC1qMjv2PQ5/SqXh66t4LBR5vlTKzkMwhHfg8kEn61HqUWnXxEU8uEU7s/u1LH65Apt6hobEfkyJGyBQsiAhhAgBH3skbsnio7uw+1NGjJMp9Ft4wCDjH8XP4mqtrqVlDEio6sETaA6KO2Oc81R1q6a40wxWNy6TK4bjYntxgDPX+tEmC1Nq30tbXKJbmQEjPnWykg8Dj5sVHeTW9jD9rmAREK5dbbJB6Do30qKw1CP7DAlxLGZERA/mpHjO3nk8nvzz+tQX91Z3kDQyykwsAAAkfrnjbx196aYMltpY7uMSxktbs3Cta/KxHH976j/CrUsBaMKiyRvjIYW+GHXOPnx69qyrK80+0tPITeqqDz5Ck9c8kj61PeX1vJYXEcNyqsVZV2+WnPYjv1H8qcmhJEtppYhvA5EjsAQVe3TB44Jw3r/Wr7oCjN5S5A5ZYGHB4Of3nv/wDrzWDol35dmVuZSJVPzFzG2fl6/NzycjAFaD6jaMvRN+OcJGDn8qL2AtqrAF0h2Kep2vgn2xJ71oaUGa8bKyBtnGN59OeXNZLXtq3Rxl9wyUhzg+px/X1q/oU1hHdMxuLdQsfO14gOxxj8OxqotXJa0NSMpcXl1aEMTAqlyQCG3g9uvbBpJpUGh3E1mN5it3eNlwyllB6A++eM02fUtLhvnmEq+e6KrBm3ZA9Bg89e9Z9jqccVmtpbx3E+2RjmO3PIZj6/XtV3Isaj+SL6CIRgSSRuysq4BA29QT/tCqskd1YWOoScF1eaWHzmIBGMjvjHbt1qJ11G5uUljtUgIRliaRuFU4yNoIPYflU32DU5Yv3syRgH5Xjhzn3y2P5d6OYLE8tuk11b3iSxCSKNhiPOW3bf4lyB92ot9r5V1BfzrgzM6NI4V9mQRhg2QAfb+lMi0MXUAlknuJQQMM0hCnr/AHQAPzq7H4ftbd1ZbSNSSBllGTx2Y5P50hnP67d21/LE9tNJIELZLJxzj+IAZ6d6yXBRSAeD3Brq/EUMUcEYC4k355BB5B9evTtXLhCZCMFmHOa4a6983py0K/llhgHAp4j7cr71Z2YwXwOehpGQyKcIOBxjvWPKU5jVbCgKR7470wsVJJ5HXioJFeJgrRkE9PehZWJ2tkY9sUAl1HtKOxIzWpoAX7RIzHO2M8DsTWPt34zwM9K2NCG83ZUHIj+UDgnrVQWoXFeZXYttAJOSQiH+bUhk5yJlA/3Yh/U1MlpnpHKv/Ah/jUn2N8cK2P8Arof6CtLFXKxnYA4nQe26EVt6W2YlOCOOcnOTxWS1rhSXIx/tXDAfyq1YXEcMUwVogwXOBjJH8z1q4aMmeqMi8dLieWQN8rOSCPrVd/l4AyB0JquJGTCEAYFP84cjg1jJpu5nqiSSQIMbcjHaoGmAyAgAx6U10kDfIN3tUQSVsMUbH0rNmisSpcHcAGIAPPtV6GVgctgL3z1NZIyJsYxg8irIkY/xUIckmahZWG44z14yKDllxkMO9U1diuwkkdAakRtm0FgXFaJmb0JI0KAnFSq4Ix6GoxKmCSCc9qAu1coe/ei3YLkwdWJGSW7EUucY6/WoyVxkgDPpQWIUMpJx780WGTbhvOf/ANdNEqhtpU4yaRZlYZ709iDgEZBpWuO9heCvDfrUfII5OKYyMjZQrt9M9aXLo3Q47g9qgpMk3dj+poEyqxUZ+ppgG4Ekce1NZA3OcGkMtRz7+c4HtUwuiT97p09KzWEigAHpSCQkgMcDrRzMOU347kOMdDmpmm2qQ+CB2rFhlwvDEnHYVOZH4J71rGo7E2NEbH5Tr/dNXNP1S606XMTnaODG5O0/gD+tYkc5XpwR3Jp4uPMJ65NHOmtR6noFn4mtZwBOphfHJzlenrWpbXkF2heCQMAcHHavKvMbbnPFWrW/mgIaOZl/3HI/lRzsD1HNGa4mbxLcyQQpDIyOqAO+QS57mi01y5hglBlZyw+Xdzj8avmC52+aXNcrZa5PG5kuJSYsfccc/wDATjmn3Hi5FOIbYn3kbH6f/XqopvoS5JHT5ozXBT+O7uJ9pitF7gEknHr1qGL4gX8oDQ2UEy9SyqyqB67iefwBNVyMOdHoeaK4+Tx1FDGd9oJJAo+WOXqfbIzj3IH0rFu/iZdQybI9Lj56ZLMSfoMUckg5kelZpM153D471+4TdFoMZGeWmYwge/JJI/CtuLxbItuGurONZe4hmLL+ZUH9KORhzI6rNGa46Xxlch1KWVuIs8lpyW/LaP5mpU8Zbm2mzH1MuAf0NHIw5kdZmjNclL42RMrHYmRwBwJeF+vHFU5PF97JnBhhH+yu79T1/KjkYc6O5zSZrzuTW9Wk3EXUu0fxZ2/ouM1TOqXTnDXtwWP3Vy2T75BJ/Sq9kyedHqNRyXCR8dWHavNvtGs+UfMuHEGM4MrKPUE5JP8AKsS++IFjp0TCfxAHm6iO2BlJ9srgA/UilyD5j143TF8jgelPe7ATpg+vYV84X3xQvjIWsxeOhHBu5yB/3wh/9mrnbrx74hvRtfVZYYzkbYCI8f8AAh8x/Emp5bdSrn03f63pungtfalaW/f99OqH9TWdZeNvDN/O0Ftrdm0q/wALSbM/QtgH8M18vrc/aN0k0pkfOWZzktUti6f2sqKeCDxScQufWsVxFMm+GRJU6bkYMPzFP3V4P4O1a70vXZHjlJtjEQyqQMHI65PNblx8VL+11WVIbe2vLVGCBS21j6kN068dDT9nKzYlNXseuE0Bq4Of4o6Qnh641CNSLqJVItJXCl8sB8rDIPc+uAeK4a5+M1/dKypZpACOMSkAfkMn8xUqL3Kue7ZPvRXz4fieSc/2ahz3Mrc0VXKxXPoLNFJRWJoLmikzRmgBfrS5pM0lMB2aBSCjNAh1LkU3NGaAHUU2lzTAXNApjOBTCxPelcCfNJu9ahBozRcCUv6Um8+tMopASbz60m4+tNBooEOzmikzRmgB2aAfem5ozTAkDU4H3qHNOpgSZpN1MzilzQA7NLmmZxS5oAdmgU3NLQA7NGabmlzTAdRTc0uaAHCgc03NKDTA5Px/pEGo6VFcu2yaBtqNxjB7HJHoMc15a2h7PkDw7ySMsACPUffr2HxiR/wjsgLKoMijJfb69PevOo4yXIM6kYOA0oGR04OOaqmt0ZzZkR6BG5Z2eA8dQwxkc/3uT2ph8MyGTmKAegyOQf8Agf8AWugKlgxEgbG4FvMQkHHQ+ncfnSGKQwgDenIIKtGWOMjPJI/zxWtibmGfDO4ZjiUqBlgpGOvf959OlH/COIhJZU+8SAGA69x82TW2isrhJSXkPygbF3A9uQCPTp6gdSKc8uwndKqB/mypB3e5I659efqaaiDZix+HopUZQQxK7QA3PHXgMfWnjw7bMGeQRHGejPj26kY/OtA8s8fm5J5AboMjrz/TJqV5JHWNVuJS/cRyHGMfyPrjvg9aHFEpmM3h2IMGVrXH97zCRnvzup0fhhXQZaINtwBnHB9PmrTWaQqxW83LgfKLjlR+PJzTQ7qQvnEq+cKZccj/APXTsFzNbwxH5jMJIMYw2TyMfRv61ZGjTsjM80ZEm0ndvIZu3PrjpitJHkMays8/JHy+ZkfyHPfvgYPpU5naLmRsFRjcDz75zyevvQ0FznX8MHBlUwyxSSZ3L5jZx7gYxz6+tJJ4Xk25QYXO5SUfg/lXQjyyH25Y5KnIycdR0z3yOevpxVfYJEMsQIH3nATAHPfH8v8AIFEHIwT4VuInVo3QGNiysoOAfXNX9A0Rv7QZZlTYAZNu0/eyBnkgD86sTysFPCx/PnmMsD+G4VP4cbOoKHVWcodxKFc856ZziqjDUTlodLDo9nb5dAFX/ZUIT+IBH6irUEdtGG3x7s9GzyePXJ/QfypwaNcFFy2cnC44+q89u4qjq2pwaXpcl7d8pEQMDlmJ4AA49f8AIq3oSaM9yIo0ZIQScDJIzyOMYx+uR7VKB5ckjQgK2cFhwD9ex4rCudVj/wCEck1CGQMkcJuFCk8/LkfTtVWHxLLfeDLjVVUR3PkSjCgkK4B5H6Hn1qdmx7o3p5N1vcM7KB82Wz82PrnFPSRJ1jmi2vEyKUZSCCOcY7HtXFWN/d6h8PL4zuTcJDKjtu5YBc/j8pHvVvwPcmXw2kbMrmCVohnnA+9z/wB9YprXQGjQ8RFhY45Cq4IC8Lnp06g81yUsp25U8g112vB5NLuB3CBuuTgMOh7iuJBXdlySD61yYhe8aU9h++SQnPzfQ1Y3COMjLBj1ziqFxqVrZKRJcxxgDozAfpWNceLtNhJxK0pPaNf8axUX2Kep07OkmFc/TioZhGqM5lCgcnccD864mTxZe3TFYI0t4zxuPzN0/L9KYYp71xNcXDyMQD87Z59MDj1/zzWip33FZrY6z+2NPKmIyFgOSVQ8e/Tn8KSDxRDp0VwscZkaZdoDkgAepx+PH9eK55LUvagxDaTlsg8k4+vt6d85NPis1jhD3AwsZBzyCPUFhn39vr1FqCQ0rE194vulBS3t4V5+8VLH9c/yrPt/HV5A4W5tLO5Xdklo8Nj0GMD9KytWnLyELEY0U8BsZ/EDof8APasKSUo386dkO563pXiew1LINvDFLj7rRBix9uR3rcW/ga0uEjlHEeAqqRknIHHb8a8Ssr428ytuOCeoNd2n/Ew095oXLSFNzLjlj1/Prz/jTUQvc2HdwO3sahSQK4yefyxVeHU4hbkTzqybQMlvmAIHP8vX+lQ2t/ZvIqzTruzlSM4/H0P6VhKk1sF7I1dzIeRg+lSpcsuOcevFPSVWPzHd6HGaa0UUmBkowznBz+lQkRzLqiZmhlXLABqQpHHEx2kHIxms5X8qd1J6HqBU5uCuAkh9QaTKtbYsCRNw3EBscYpQNzgux9MiqUTkL93I71b83IwCQzdAOTQgkSN8nAOc96eHwBjGR271AsjBwMDB9adweQR16iqBeZcBwBgrj2pytjjGfes0yyK4V8Z+tWg7YHT65p3E0yUlVbOefWpEcZ5IPtmqckv3Q/J7mnICxUKc56VN9R20L25D3Bz2oYgJkcCpIdMvZFz5flju0hwAKe1vHbozz3kIQDllyePqcVShJ9BJ2M/7SwOepzUqy+auPun6ZrPv9Y8PWhz/AGtGT3VUJz+K7qpL4t0QBzBdNKyI742MuQqlsDI9v89aXsZdS+ZHQlc8EZ9aZ5D5G1C3tjNcS/xJcnEOmAKOMtKM/wDoNNTx/dT70SDyJCrHeGDcAE+gP45qlRXVhzNHfxWs24K22MHsxGfwHWrsVjK0X7zzFPckgA/jXk8Xj7XA5KX6xAk/diQHHpnGf1qtfeNfEkpIOrzqP9khR+grSNOCJbk2exSWllbwme4uwqd2d1VfzIrCvPGnhvTv3a3K3LjgrbwmTH0ZsD9a8YuL+6u5C91cTTv/AHpHLH9aiEvJ5OKr3Vsgs+rPSdR+JsxhZNOt/KHIDyydR/ujGD+NccPE+uz3Rnk1S8Zj/wBNmH8qxl5OSPpVy2Co4PIobuO1j0fQ/GsEFup1a+SV8fcigbcPYnhfyB+taTfE3SEwLZZI2B4d4yT9e/8AKvI4X2g7iMn1oeVQwAwSfSlGKWwM9Uj8dR38phTUgMqzElCmAASTnaOgBPr6ZqjJ4x0lpSZry+lYfxBMBvxyTj06fQdK86tSwmYfdzG4yTgcqR1psDAkKQf8avmYrI9Mj8W+HFUeWwiIOQ0lsX/nnH1HNTT+L7C7jCpqbO6gnCJIQAOT94cV5mEGOcjJyQRVvRY9x1FugjtSc+nzoP60KTFyo9Di17QIwDPqNw4Iy0aB8fnitOPxx4eQLHDJPEQfvCLoPTPPFeNXspV1VTwRzg0ls5AYtnB96OYOU9qm8a+HTiQam7EHaEWGTJJ7AEYrBk+J9koxHpc7p23TCMkfgDivO4/3t1DGAzjzQzBOuBycfhVAsQ464ocmHKj03/hZlsuSmjSncMEPd8f+g1DN8UpoSFt9JtQo/vuzH8+K86Zzkbc5pjse45Peldj5UddqvxG1++k/czrbQgD93GgOT9WBNUh468S7QBqbAe0SA/ntzXOu+wYYEZGRnv8A5xTBKNgUGk2x2R1Z8e+J4tudRD+zQRt/NasW/wARNaiMhaRwH5wk8q8/Tdj9K49pgQBnmkeZdqjIBHX1NAWRs3Ws6hrrs99dzSqONrMTj255/OqckSpGSAQfWoYr6JBypB+lJPdRSqoUnGfTmgAErfKTxj+tMkC7QV/H3qAyhj1/SpAU9z+BpDDJXgMcVYtJxDdCUVDwwyqMR6hTT1ilVdwglYeyHFFhHbadcotpc3RI8xFBjI7E9KwVvmkfAjKsSSSp5rOS7ktleNlwJFGAef5fX9asRXisURYZGY8YVSSc1pfSxKWpYv53axQeZlXwMdxjPFZqTiPKknp2rq7bwD4n1qGE22mM0DnMVw80QVlPc4b/ABqGL4d6rFK6XstrAAOCZ1XJz7/5+lT8WxWyMD7bjpI+O3B/xorpl8ABlDLqVqVIyCLmHn/x+ijlYuZH02KM03NGa5jYd0ozTd3pQGoAfnNGcU3dRuoAfRmmbqN1AD80Zpm40hb3ouIkLADmmFs0wnmjNADqKbQDSAfRmmilzzTAdRSUUAOopKWgQZoopaAClpKO9NALRmkpaYBmlzSUUALS0xmVELucKoyT6U37RH5ccilmWQgKVUnr046igCbNGcVkQeIrK5vPs0Xms28rnYcfX2H1rWo8xJ3FzS5ptLmgY6im5pc0wHZpc02jNMDD8X4OhgEkAzKCQSOx9K87m/dYbzEGUIY7guOf93jtXoPi+UppCIoYyNKNu1WPQHn5enUc15/JI8chYlkJBIw7KRz645rSluzOZP55ZfnlizzyWjz9MFaQKMgHYFz0KxcE/hUa3chLL55DZ6+ZI3H/AAEYH9KPMaXGbrIyMnfJ/hg1qQh1wwY7lb5UUKMOOncccdTj6fkfOvEus3dxdSRpK6QjgKpxke/+FegSH5AGeIgf3WcZ55HtXnGtQ4uJOAMGom9Bx3KOl67d6fdRFppHgBG6NmJGPb0P0r06O5jnQtu3ggEyM/UfyPUenU14+4+avTfD0rrpdkcFyIk2k5wOPQf54pwd0VNGndysI8uyyBvmPORnvj0Gc1WhlDEAgqfpwKfeOwGzLOpHQ9/qCetZsYffuYEgcg8DB/OmmRY6UyoZHdigwpHYK2OvT3YfgTx6+deJ/EeoG/eCCd4YxxlDtZueuetdSt1KU2rMdo+bG8jn+XYVwfiTnVd2MZX1z3pTvbQcFqa/hbxJd/bVsrtzMk3Cu/LA+hPcf59a7hnYIUBU5JLEqD+X4GvK9DXOtWfOP3q8+nP0P8q9OUFF3NKeF5JJOf04q6crrUmpGzIJFQkgk84yAcHBrQ0Bo/7QiQnnBGTz27kj+tUbuc79nm7hjOAxbH51f8NhUvINmMlm6bhj5T3A29q27GZ17Dox2A8dcD8sk/oa5D4k86BajdjdfoeM/wDPKT1rrVVWLODhvcY/UZH54rkPiEmdHtVxgfbEPznGPkf8KU17oQeoy1dn+GNxGTuARwe/8RP4VFoIA8Baum3CqsxGccnyx/8AW60lsQPhxNtJGVfjqB8zf5/KpfDkb3HgjVYo8mR1nVV6ZJiGOvHWo6v0L6fMj0Bl/wCES1uOMKsZilOF5H+rq94CydMlIYkrct2HHyrUHhqyuJND1O0f5PO3xhnYHG5QvUH/APXVzTI7PwnpMk+o3KxI0oJ3LkhsD5do+9wM8Z7+hojpZhLW5ta5OltoN9NJJEiLC53NjaGxx1GM5xwev414j4m8Yy3kS21oywoq4ZokVDJ1HO0D1xgce3emeOPGs/iTUXWIvHYx/LFGWPOM/MR03cn6Dj3rimB6nvWc3dlwjYlN3I5+fBp8LB2A71UALED1qa1B+1xgdd1QWdZpdiqqJX2luwAPyehPr9MH065roxFbxwhZXEfyY++c5J4BHT0BwP6Vgy6nb6fBFCdzlRlgD1bA4J9vbp9ax59WluXaZmw3G0DgDH0oA7ePJV3Qhi6/dAAXOe5GOPftgVVleF4zJJO+EOd7nIxx359+voORnFcha39/LIVifIzuYlSRzgf/AKh0q42peY4W7ZcLna/l5APckc5+vPTpQhMdd6jbeYxjgRlIwiryWxx/Tn/69YV3ey3DFtsSqo4RIwABn1Iyf/r+nFX7yF2YT4jlTsykDj8MEfjUaLBOhV4ZwSezqB+Py5P51QiOGzjuYvNi2ggZKg5roPDupwQSKsjYdRtA/ve3+f6VnH7JDFGIbPaVbLMr4Zl7rnnkjocHmoYY1+0LMFK4bsen0oA7bUdLtr9TLC4DTDfk/wAXuPz/AB/GuM1XSL7Sm8x438oMV3jlc46Z+ma6OPWY4wEw8j4wNvHPHc9aeuoTyI6yRqyP1jKhgfqDn+lOwXJfDeqG9sUDgiZBt3H+JfX8/wCla7tIWUnHy8ZAziufhgtLa8EkFnBG+7GRKwz+Gdv6Vp/2pDFgMHBIyWHIH8qxlTd7oVy5hJ5CVYhj1yODUjBI2wcdPSqcNzbO6mKeNw3GAwB/I81YnvLSCVRJNGHbAVWb5mJ9utZOLG2TrKrx/KwBHbFJli2VYgjnp09a5rUfEqwMFhjDsDnO7AP+NYWpavqgu5raWeSN0kZGRDjBBwRx16U1BjR6Vb29zchjHAxQdXchVU+7HAH51dSDSbVC+o65YwAdVicO305Iz/wHNeVtY6xcQWka2d5P+6yiJEzYG5uwHGcVE2h60xAOm3Q+bHzRlQPxPStFBID0LVPGvhOzZVsLa7v5o+khJRW+pOP0X8a5XUfiJq9yStoIrBD/AM+64P4nvWPceH9Rhx5qQIO/+kxNj6gMSKsp4RmkRHOr6XhgNo85yec9QF46d/b1FXZ9EGhLa+KNX8ks8/m4P3pFDH9ev41ck8Uax54aK/ntxgZS2fyVPTsmB3rQi8Cy6foLarPd2V5aIw86JJSvVgqhT97n6DA5GaWCz0kTbjpcDxbclSzk8DsSx/lTUGS5IytS1bVG0q0u5dRvJftM0y/PO7ABRH6n/aNYsaXl1ueKGSVm4yiEk/lXfxXm+NYlt7NUiQrEiW6YTOMnpnPyjn2pJ9Vv1s5/9KkULGxXZhcccngVXKJSONuvCmt/2hc29tpl3KsUrIsnlnDKCcN9CO9WdN8KatDds19YSQQmCVC8pChS0bKpJJ6BiKt6hrOo/wBoTobiZIjI37tXYDBJ7Z54qhqF3PKuHkZlAyF/wFHIh8zLH/CB6nFgyXViiFiCTKeMevHvU9v4T+xyNNc39lKjRuu2Ji7fMhGQPUE8e4pzXcUNuoh+6y5jwMEp29ccfWqz6hLIq/NkbcAn160+VE80irHoNg7gnUJSFYblEGGx374/z0rQGjeH2AEk2ohtgwQyYySRnkfpms0XXz/KmwFQpORneO/06/nUvmyRwskqggEBT9PT86VkNORZGm+H7W0MjwahcSM/yKzoEIHqy8g/hVX/AIlcQyNFU8Y+e7c8888Y6VYtRC1v+/uBGznOGcDPpwaqHy4y4aVGAOAc8e3JqSgjvRb/AHNLsCAchXiDn82yTXTaZreiPppik8N6cbhvlMqxopi3H72CpZgOThSOOPQjm7eM3cgEStLjnai5OPwrQPh+8XTb3VZ7OS1hiAWISRkbyQSxXjnChs8cZzx1Aldgy1da5Ab66KaPo7B5mOZbJHOOmM4/HpVdb+OGBrpdL0tnBCbmskwOvpgZ/DtWZEYpgWRsg8n1GfWrE7OvhpwCeLtBgDj7jf8A1q1aRnqacHiFInydK0+BscPDAkZ6ZwcKe+OeP6h/9v6aApHh3TmfIJZyT+WAP8+tceJhkE84p4kMjMBnBqUVY7GDVPDFzcNbz+Ho4gQWDRTvwFBPdh/P86rxT6aJJHttMVImUqA0m4kEg4bIwQMdP1OKwdM2LdzZySLWc5Izz5T0/Tp5GtsSH5s5AK4/T0zVQtfUmdzWjTRWQb9Fidskk+ZtB5z/AHfSmS2vh1zIP7ImXJyrC9bjOeMbcen5VGrOBIGO7nK80zfmL7pBJ6GrsuxF5E9laaVYxm8FszypKECNIc7W35OcjoB19SKs21xp0ERVNGtWAOcuu/8AD5gT+tUIkBtppCF3CSNRk9BiT/P4VCHHlnc+3PAJGcU4wiDky/BrOn2CndoumzE9TLbqxxnoBjA68HGfc4qaPxVBMSLbQtKgx3W1ww/75IB/Kufnh/esrtnjdwO341YjiS3tiqFc9SO47Yp8iuHM7CahHfXly1+tvC0UgIxs4Uoozgdhgj86ovNG7Rr5MQ28A/Njr9a2jq9zaacII44RBMJI8sMHOF3fpt/KudbHPpWUlqaR2JHnMs7SukaDcTtjXH8+1NCRytlnyp7Dg1HkfhUJyMsuMZx1qGUh0ksWcKgyO55qW1u4o3/ep8noADVPAJOTjvk96cZYjHGoiCsudzbs7uako3xfWBKmK1hHHJOcn8alg1aCO2YTQGZ8/KwYKAPpjmueRkI5z+HU+1KXzkhsD+7T5hcp0lnqMM8qq6onpjNdE82lQRJ5pcTlWPkhSxbAB69gcg5rz60YGUDd8xPGSAPzNdzcTWC6As80aRXLSBbaQAnbk5I45xtzxzyKcJESjqOlezDjNmyucMo8zII9+AakgubSRkja0TzC2AQeg7dQaxeUumhSRnjDb1xnrxzg9KuxeUwaTco2AqOvpx+tWmJrU9e0nxBFb+CJLhTmSxtiCjqAeB8nA7EYrxu41jULxX+0X1wyO24xmVtmevC5wPwrWm1u3j8GX0QTNzJGIzIy4yPMXjrxxg/n71xTXLFjhs47jpURSitC9Wjc/t/V1+UazegDj/j4f/Giud8wf3B+Rop3Qcp9clsU3NNzRmuE6B+aM0yloAfmjNNzRmgB2aCeKZnFJmgBxbPSkoopgLQKSlpALRSU4UCDpSikxS0wFFKKaSFBLEADqTTh9KBC596UUUjbtjbMb8cZ6Z96YCNIibQzKpY4XJxk+gp1cjr0Gs3V7BExghhCsQ8cp2k9gRwTjGa0NL1ZhOLGdzPMuC0ixvtAPQ55zz7inFXRHPZ2aN6lqpdXq2zW6sGzNIEAA5x3P0/xqC61ywtdoe4jAd9gbIIHqTyMCkVdGlS4rl7LxVDBY7r9maby2m4XHyAtzjqANvv1FbemapbarYxXUD5WQE4z6cH8iadgUkxutyzQaVJLCWDIQSF7jp6HHXr2xWfc+JYRpzXcasAhw3zYO8EZUAjng5J9K47WfGl2JrvTk+zz24d4yzxkhgOhOehzzj1HpXH3d/c3duEaV3j3bdu7IHHUD8KlXaujGdWzsj1VfG9gL2dXnBhKJ5IVcHcfvc+39D7Z5WLxYlrBNFEt1ubCq5cL0II4AxjjoMZ9a5dIo9gjE5JI3bscDj1OOufeojb30DGbzUjZMsHjbnIx0Pr6VbjZ3Of27btc6hdea51y8vjHKRMu1gJShHC7TuHoex7ema39C8StfavAsn2mFpWCJbxsrK20YJPdQMZxn1rz6x1ERRsZUZnkyqgMyjbj1Ujjj/Hjipo9UutKlM1pCPtMoysmSSoHJJ/z2IJoVlYtVHfU933DOM89cVXhvobiZoUf94oyyngjnHT8K8/0vxP9i0oyG6iuL27UFCpMkmeABjc3v1xg4+XFb1jqDWXiPUYr66PliKJlafCbflyQMHB5PXA/Tk6nTzI6uis2DWbW4DNC/mIG2qy9H+nrU6XqMdpK7sdA1BW5bZ1RSzEKoGSScAVgXPjDT7a8NuVdyoJYrz/n/wCvWT4n8WQQQS2cBZ5R/rNo6Y7Z9a81n1Tz8s0R3BieD97n0pK7fkZVKnLojudZ8Qx61DhT8sbAhQM4+uKx2kMu/aWyBjq57549BWFZzlZflbYpB4PPPH0NW5L42kpaST5TkHcxx9eK3hHlREZua1NCaPHJOM5P3iccevemqw92PqSfzqGHVIblAsTxyIMZDCXn1xyQP89cVK0uIwJDGwCAAqZQAc+nOf8APpV3ZdirqF7LbWU80ZDOikgM2B+PrXIXSPqVwgkkggeQZdpGKop68nk1va/Ip+xyPLBBEkzGQktk4HGVxwP8fY1o+ENFRNLl1KMWN1NNO5QzkhkAI2EAgnghieRnPXgVdZUoYfnV3PTT+vvNIRW7djyy4jMU7xkglWIJHIODXo3h+Y/2BalQchAOC/bPpx2pniLwjaLqr3ls0HlAK0lu7MPmx8xO0dyCeCOv4V0OixW50xNsEcEe0gJDwAQxHLHk9O+32bNY0VKavaxNSSRgLeQ3TMUnXzO0aYeQn6Hr+GaaYbkHzUljjjz8rlBksOoPzDHWte6YtrM8cjYgEDBUMfTv/ESCM+u761TupFitoEMuUYkgM4TGcdOD7ela8lldmXtEVR+6lCMTnAOWU8jgg47da5DxKrDUELHnB5xjPNdPdXLLeNmVyBgDdkcYGOB0/CpkmeX935jHJBxlz/I9KmUbouL6nE6KxXWLMgc+av8AOvTUYeWcEk46BiO361TSAKV2oWO3klWHOfc9fpU65BJaTAJ5whx+WaunGyJm02DqGXai8jHY/jxjmrnhvcb2IbWI3NyqN/dNU28sx8uucfLldxPPbmp9HYjV4wcqQ/Ycn9a1SM29DtU+8TuO38v/AK1ZHiDTBrFksERdiZVfCkfTgHA/i7VqqcISVYY5z1H0z/Q1Uv5pRbPJagF/KYqDjrjg45/wpyWhEXqU7TTo7Hw+NPlj85SWDZBycknuOPxFWdNtoLeyWKHZskJJC8jI4PTjPGDj0pulSy3elWMkkUskrW6GQIoBDED2wPocVPpWlix09bWWZCsW9gx6nczNjrxjPUHtUW1LvoZd/wCIbbR9LutRuI5THDI8EcfA3urMMAjoOCfYfTFeKa1ruo+ILtpbycldxMcIJ2R57KvboB36d+tbvj/XRf6k0FvIWsbVnEZDDLMWO4k+5zg+mCOprmrq3htbBN8oWYjLKvRfYf59axk76GsV1MyWFm5XLE9OevOP51SPJp/muNwDEZ4PrinhGztAx6+tQWJbxl5cj+Abj+dS27ol6soB2LluT14pg+XIB5PpQFO4J09aAHSyNLIWJJJoHOM49qDgNgdPXHNSbflB70hmzpiF7VlLfKuSdxOM+nbH4Z+lPlgeRTEY8sPuKyH5j6jAIx6jryOlVLa48tBH8qliMHALZ74PA/UVYa6HzO0qtGcA4ZmHX+7gA+3Xr+TEZZe4s5Su9lYAjGOBnqMVpWl4txKBcRx5c4HbaPc+/T/9VPcR6mFGTuHRBj5eevHHPHqakis7fTys0zhDnoHGSMe2eufzpgXptMAjaQcJgcn1A5x/n19MmtHbiA7mAJznjnA96sDW2m4gTCjgt12gY4Hpxxx71b+2RlVLptcqc7DjA79fx/LHsWIht5IQTIwG/GFJA5/Tt0/GrrAxbnaMADgYbn9Oef8APFZZ1W2t5S9vAm4cB2OfpjqPb8qtLetcBdxVmOeG4OT1zjnP09D07sDKvLqea+2jKAY2hSc496u2sQmiknmkSRIWVNocHbuzgsOvO08/TpkZbdRswW8QN5TvtMqADLgA4XHfGPzqreM1zFGIykLJHsQNgbzkE5bA4ySRnhemeckBG2mo6TbyLNfuRbEgGOJQHxjOQOPy47896w9T1LTry4DWsbBF4VmOGFYtxp+qSO3mWsxCZy4XKAA4zu6Yz3ziqZHlNsyCR1x60rlNnWabZ6XMpnuXubgLnESkIucjCk8noc8Yrov+ElkiupXtbK2gd5GkklWIF3ZidxJbPU5P41zHha0upYppDhLd/wCOQ4GQew6nn0Hauh/shVuJAt9DI2GkxChfAC7iOSvofXpVxS3Mm2zP1PWry58xXuZTvOH+Y8j+VY1zqd1dndcXUsrdN0jlj+Zrp7jTtJ+zu8huSxOCVkUYIBOANp9O+aotbadFYxONORvNBIkeRy/DEeu3+H+7Q7grGA9x8oDOcelR/agAoV2BHXmuqjngjTJ0zTSo6F7ZWb8fX8qhudYs0/dnS9MG3AyLNB+PSpuUa1t9ouvA8sEUhaBbuN/nIUDA5AJPPWo7K1uZDuaJfKKkZNzEDnaevz8c/pmq9xrzTeHHtU+zRx71AjhQLjnJ6D2FVPMZtIhkDHLTOMZ9AuP51aZFjTMNxBL5mbZFxyTdRkH8mNPdJmmkt7u6treLy3jfYXc/MhHZcHGemaypoRNYhsnaBnIpmpXL/wBq3oIIH2h+D2+Y8U2JakeqQzS6gTAwkV5GCNt2gjkjOTnPXPbjqaZLaMYYjJMpZnCkKp4X+8M9fYd/apnaV4RIo3KuSxPUA8f1xTrhy1qGyCAQQalK427FnS7KwlgtE1K7ninjYIqQRKw7DJLOOw9PX8asp0qKdVhF26qSSXkQFuePlCHHGeMnk9eOa0ExkvoAzBR5qZPYDcMn8qqqzGWNeCwYAj1osM1o20i2kV30mScgbgLi6YjPH9wJ6dK24PEunzaS1smiadbyENG0kUAyyMhXGTls8g53dQK5+dgI5SPmbB28VVtYJBaG4bIQvsAweoAP9aJRS1CLbFjS1t5CYIFIDHG/qOvcYrc0bxTqOhJKtpKm2QgEOuV4z+XXt6CsWC0uJw3lwSygPtzGhbn0471efQfEDwlotC1IpjORaPz9OKi5VjZf4j+Ihki7UKeOE7enWiPxzqU8QhnuI5YTlWiaBNu0jBA444JH41zJ8O+IE+RtG1EP02m0kz/Kn23hzXxOnn6NqkYbOC1nJ2+oouFjamuLBpJf+JXZLE7ZGFMZJxz90j34qyP7J+zC3Gk/Jv8AMZTcvjdgDO7qO4x0qi0EqSvHcRyROPlaORNrA+47GpgykYDDPtWxkRajB4ahEUbafMhlGSY5WJX5iOMnn8adPoehwT7R9u2hVO7z4x1AOMFKztThDy20jhSRAeo6nzHqTXLrOs3axlzH5x2l/vEDgZPc4H51PUs0YtE0uLLw3brGyPG3nW/mHDAgnKsvY+naqA0KO3eNY9XRgeha2YADt3z19qrNqEht2XflCMAE9KhS+ZVyfmIwBTVidTXXw9cHITWdN3up2K7MBx36Z7ioE0O8Ks6T2M2D0jmZM/i6qP1pbB2vGkmkcKsAztVclsnH+fwq+yRg4RuMd+KEwZXs9He5s5bOSe3t3klR93mrJkANx8uf71SDwwkd0ANTscg5yzOM45/uYoZ/L/1YBOep71Ul1SWCYh03bh0Jxz2/lTvZCsXk8JLJL5h12w3ysUWPLbcjoAcfpj0qW48D31vsa3eC4cDhVnUcY5PzYGKwxqck17HJGAhjPyspJ2n1B9aty+JNQmMoa+uHDABt0rHd7Hnmi7uOwur6Bqo0exgjsLiWaKeZnWFfNOGCbclM/wB01zs2jaugJbSL9FHGWtZBj9K3jq1xHAu24lTg/LuOOuc4zgH3p0XiC82skt7MwI6FjgD86lq407HKeRcKQGhkDN0Xbz+XWtnTvCuo3cEsn9mXcqtGfJ2pj5s984469K6Oy8QX95Fc7725G2LcCkhGDvQdvZjx05qGfXLyCzmdr+YiU7DvIIPtz+NJQuPmOQk8J+II3KvpN2DnqsLMD9CAarvoGsLgNpd7/wB+G/wrox4hv0UgOgUEgFoUY4+pU/zp48U6irqwMBABAzBGM/8AjvWk4IfOchNaXNowFxDJESM4KkYpm1mUFVPPGeua7uXxVqdzF5UroV6g4OR696zjfO8TrLDbtzyzQhj9c4o9mg5znba1meeNNjKXbAL8D8zxXpWoaPcXXhiCGzgS4kjG4hUYls5LAAZ64UCuORAkguU2pIh4eNAvP/ARj9K3dM1u8vX8i5vJHLZ+ZiN3boe1EIWlcmU1uUxp+pu8irp1+hl48preQNkdulT2+jeIZdwj0u5WNzlzMPKAxzzvx6iui0/URYxSAXMjhPlVZjuC5x0B4z9Rxk461cl1sRooE5iPPKqvJ9c4/lWsaL2uQ6y7GVB4G1i/026hkhhiZoiYi86D5wQQMBs4IyM4/wAa842THO2CX5RlvkPFemXviy6tNOlg3NtmbYcBQeec5wfTtWani29LEtcTbjgbQ5GO3apnBR0LhNtXscHvkx/qZf8Avmiu0k8ca+srqrKVDEDKt0/Ois+Vdy7s+ic0ZpAaM1wnQLS03NJupgPzikLUwmjtQA4c0pNNoFADqWm5pc0ALQOlJSigQ7NUtR1e00pYzdMy+YSF2oTnAqa7vILG3M9zII4gQCx7ZrlvEuuaPe6SFFwrSbjsJQfKcY53AkZB4wPxFDfYmUkja1zWI7HQ5blJhHIx8tCy5wx7Y+ma5zTvHYg0+Rr1fMl3BYgvQj3PoMH1Nc7Jq941h/Z9mHSN0JmiRMZUd92OQfXj09Kx2i1CQLC8UsEaqZFVx7H5sY+vP4VW1zmnUk3dHV3Pj66nEiCONFY7kAPzKAeh7Hj/ACK39D8UzXuoeXeIihogVET7gvzEZcjgE8fp6mvMhptysMJ2MUkPGT94Een5e/PetaG8htYYwdPRbuFgwnBYEYGAOuMcenrwCc1asZqpOLvJnoWra5qNrc3C2dsk0dvt3/Kx4OBnPTIJ6f4Gqlz4klNlPIzRRtLao8a7mGC2ee2B7nH6VxK6/NaO8kTuqykMyRSlQOeeueSPXPXpVDXtXbWJ7SK52xiBFSNraLGAegwTnH16frUOLasX7dO7JZ/EmoXNvHbtKzom5QTwv0qP+07iKWJ182OdVDAo3vgYHNc/Pb3lpneTsD7NmTzkZ6jtirZCmNJkeQSICGST5gx7kccdBx/KtFoYPXW5v3PinUrqdmubhsYXGxyAGAwpx0zyfTmoJNWlmaS7Mx25IIZmJOeCeOcYOOe3rWClzLEh279qqcc/j/MD8qkjuJ4okaZpIlIIG07R6H8D/P8AGml2E1J7s2pL8XdmEBxsXaAvQdSSf0/OnWmsXVnFJZKxSJ4fKTaxBUZyccjuDn1rJijYKnmAIDhwwkB4AyeBznp6Y/Hi5cQ2kkQCO32sAbVCLjBxjr0/+vjiqcbkqpy6JmU5aS4ZZJA2WxnODk+2OpzmrNtGCIzsYMPlcycZPf8ACo3torSZ2aTcZHb5D04//X6etTXFwYYkR2GQoGTyGHtn/wDXUxjylTnzpKI+fzpWdo22KegxwMdif17/AIUu+SKCJZPMRmySXHORgj+vNV49TmXDWwKKQVcg4PUHIAOAOBx0qzG0zIqSqXVBnJG1SM9cgg+v4E07EOLW4NapI0b7MlScFG3Y5znH40IoZQoIKjjceCcnB5I/zmkSFHtlkZxlfuvJIB0yOw68dPaqklw0VwN305bcOe56569xSejsNRb1H/b5LW5lkWdz5jnfG7HEhOfmPTP/ANf602a/fUJ2kd3LE7nIy2ewHHQVFczR7Hj8tcuMgRjjr15+h6VBp1heahKIrK1eWQ84H8P1OcD8TU2uzpirrU7LRvEOn2cUY8lmuowWJBUDgdBk9Mc+ua118WiTylihdWYnJPfODkdyc+o/xrBtPBkVlHu1jU1zjPkW5JO3/abHA7dPxrXW6jtIktdLs0t4m4Do7FyPUMc56dgw9xWqg5blJOKsmZC6FdyzvLdSJbxl+ZLgkZH88/XFSHTNFtyyIZr+4jALbmCKnHUgDIHuePetN2maSeV2dl5QuWHQjnBJ49/n/wCA9KrPBGq/KMbXyMoAVH97oMH32qefvVoqaSDRO5ky3bwbihjgXbgrGAm7jPBGMn6Mc+lYjlzMXcOQ+TlmJOD+Ckj862tUktTM5lKu5+uT/Xt6msS5LzcRxYGOg6fkOKho1iyaBljdmj24A+Y5I/pnuOtXIr0STssaDeqg5VsnHA6Y57dKxV064uJFUIdxOFVRyTXZ6N5+i2oRNEkUPjzJvtcJZz6lD0HPTJx9eS46gx2lzxm6ELHUvtifOAY5U254z2IHOMkjr71Jd6baWep2Lx2cStMzF1jUkHB7nOD0PGMHHG6rurIreINJLK53JMZnBOD8nXGD/tenGOTwah1OJrjWtLkcENztOV2gH1DKc/gR9K1UUrPzMnNvRFrUsjR7htxjbyyYlVQojG3kD0/SmeH3ZtGtfKVNx3dfmOdx6cnnrnvz0rWu4Eu7WeJkYeYhU/IOARj8R7E/RhWULk+HdFwEaEqQon2DCFjkEgYyASexzj71a7NtmV9CjdwMdZM4UgMgQMwUFjxzgcnHfAPvWbqumR6jHG8tzL5ykncrZzzzzg5Oevp/LOm8Z3d9A0ckiiRSVaRCQf4cDv8A07+vFSXWrhrWONJWUoT8xxhvTOB25x9azc4tWI51fUq3crQXGyYfMp5k6jHp9e9NsL7dcyLNvEKnKnaOee5J/wA/hVa8lF3MzFdpJHI6E461UdOGiLSFT12j8ulYc2uhceXodVp+oxX0qxR7VIRi2dowR2GTz3/MVoOHYcN0Oc/Ln9Ca462NxBJhz5Z/h3DAUf8A1/6V0K2iQwyyvqcDOF+VW3HLZ6fd9M/pWsJaaj5t0x15fx2UCuNrM0mxlHUH6dz0q/o84uNTgkiGfLmCyBRlQARXKrapqVxuF0sbgD/WcAn2PrUWnTxzapBbxsyb5lTryFLdPb601UfMQ31R7mqEjdxGOhZzk/l6fWoS9sweSDzLuYDA2j5Ovbov86c+lRoGlu53nx8wVjwMjAx2BHXgVYub+0tFhjlmhtTKVWOMH52yeg7/AJnFbNjKkUGpSBjMwtwSeP4gOPX1+lZ3ieaPRfDl9dwF5LpIQEY9AzEAHJz0yDjHbFbn2m3kk3J+9DY25bhvwH+Ncx4/unj8IXYlCQi4KRxLjbuIZW5/AHk+nelLYa1aPAtVmy4jGMrxxjGMev5Gq89/cSO5MrEOOmfWlvlG7cSd7Y79BVM1xnYWrWymvCzoQAM/Me5xnj+fsKSWTyyY1+90ZveruirqF7qEFrp1s1xcHgIFBzz3PYe56cmotZ0e50O/ayuyPPX74HY/1Hv3p2driur2KSSbc4xu7VKq4X3PWooxg1ZhVWOWIH1BqRj44SAZWBO0dccCo2nzJlecc8ipZWEqFEBx0B/zn+dQm2k8veFJHYheKYizvEq53bWxgk579e/+c04o7D94+VGAMjn6e3+frWarFGz0I6VOJHZRuLH0H+HoKAL8LLCBGi7A3WR+fyHT+dXX060mO99QMz5GenpjH+T6Y61kO0SjMspLdkXp+PrUQumLgoqpgcAcf5/z9KYHSbba0Ecix/KOQS33e+eMZ/L+lVmulf5dwIznknJ/D/H6VRhv5pV8tpCgPTccg+nFCyLbEs6Zb+DDcZpiNCa1hSNXLguP4duSOT3H8/TPtVgJ5NqqmB9rJl9wx1AI9+OTnj8qxv7RvDJIVnw5wScnPHfp7fp7VXN7c7MG4AB5HOce/HQ8Ci4G8WWNnD5V2TAlxnIBz9euP0z1FZt7J8u4EfLwMNWUZCBhWOTy3uadJKzkgnIpXBoa0zldu87fQU1BhgeMikZe+Dg1PZQeddxIc7c5b6d6W4M7O0dbPTbVCwyE+ft82c4/IitC0jubWVpZoSitC4XzsRhtyFRgvgHrVR9VmIjSNvLV0KsE+UMuOQ2Oue9UpEAvpVjxGoRWAU8cqD/WtbEXLbRXU5kRjCqkg7mkzg/8BySfzq8dPxp8ESajZIUPzMwlK8sehEfoR+OayEmXyGAf5gwNOuZJrqxENqWkkOMLHyx59BTkrCjqaI0eAnbPrtoE64ijkYn8Co/nVGfSbU4I1OKQ99sB7e5NYzWt6smZQ4IG3BHP0xVZorkglVc49qi6LszprCxtoYJDc3EdzAXAEG1l5wSDuDDHAPSpk1Kzt4jCum24iQllBLuSTjP3nODwOnpWR4es5tUt7mBJo4mjZZP30oRdoDA4ycE5K8dcE471oHQr5ZPnezIVuQt9Af8A2ei/YVjbvI9OFxLHHZ2/ko5UBXk2sM+z4I4qnePZSSzzzafayzzszuxkmHzE5J4k/wA5/ATXWk3YllcS2hTzCV/02Js5Jxzu9u/9aqDTL97pYpI4sbtpCTo569lDE/pVadRWZZiurSxt547eytTG5Kl3DSErnOMMxHBAI47CqF/JaXMbEWVtGcABo1KFSO/BGT9c9aYY5S5hETs3LYSMtxnHbpzTEsrq7KRwWl1LvRivlwsSxAOAOOeR+hqtLC1Hq6KJTshxIxc5hRsdeFypwOTwPb0GM7ybdUEoeTcpyULZB+oOc1bks7yKMGaCeIMN2JYHUYJ4PI7moZdLvxI6Jp927KRnbbuevToOeopOw1cuWyQRJa3EIcTbGLFyCASWXgY6FQODnqT3rbOq311brDPctNGnKpKA6qfYHp1rGtdMv5hAiWjB5l+UF1XPU85b5ePXHQetdFaeE9cKlnsRGpP3nuIgB/49/KnFx6ktSMWXxX4htpDBbaxeQwphEjWTCoo4AA7VWm8Ra9KcPrt/uxnm6cZ/Wr1z4N1prueGAWF6IuTJbX0WwZ/32U+3Smf8IJ4pkQhNOTA5LG7h/nvqHa5auY0mt6yxIOsX7bTgH7U55/OtTwj4m1W28U2W+7uLhJZVhkjllL5VjjjJOCM5B9aX/hAPFBbd/ZyMT3+2Qn/2erem+A/EFrqlpeSwwxJDIsrILqMuQCCQAG//AFZqWrrQZf1vVIJvFV59osYHjjuXjlCjY5CsR94EEnjvUUc+hzOAtrfQqc4KzBz+TZ/nWFBJJcM5OUnO6QllKkqRuJGeeRz+NWYhsUHIGfetW9TOxqXOm6TdxwB725Vo1KDMIPylywz83vTb7wu2savdTWN/ZRpK+Y47l2RsnORnbjqCBk9xSQkBxuP/ANenNLy3p0KkUNBc5/VfDWraRB59zBCbbH+thuY3VuSOBuyfuntxjms6Kzvp1xBYXkmME+XA7cMqlegPUMp/EV0OrXceyJQuGaCLO09SUBJ4+tZdvNlWidA6g42sARU2LuWrSw1Gy0qW6ls5I7aYCMyzAoM9RjP3skdqFnYrGYzv+TccHOc9KfJeWvkqJtOtWwON0KnHPTp0qpHc2ttMbqK0hE0YyqNGpjPGMFTwRQronRkx1CKOPa8saNjBDHHPpVWcrcRhxJGQM7ih3ValiH9ptKtvaQyGQuU+zROiH+6FKlcDGOmO9dFPrNwJry2ksbCFTGtvLDFaqER1G0lffg8nOM4GKbvsNJI5O3spxpl3cRlVjTazZGSckAD2Pze3GarxQeTIfNdfXA5z6V11rfpZ2D2502wniKgOJ4Sxcqcgk59RzjHGRxk1F9t8PzRN9t8NQefghWtLmWFVwPl+Xcc88Hn3wTU2aHoznrtCrRbirMyD5Vzx7fWqbhuo5BroL260a6vhLHpU0ECr8sUF2wJY4yWeRZMgc4AA6+3OjaWfhB4D5w8QruICAGAkckMScAEcdsU7iSOf0p1AuFdiB5eVHZjuXr+Gah1Ur9mRAynL/Lk46V3Gl6H4Sm0yfzNXuILychV86Bv3AB5+VCQ2QMctjkcA1ch+HHhu73PP4rDKU+QRiOMg++Wb16cfWjmVg5Xc8u5YDOA2OV9KcJFBZNo2DqSCea9Fj+FlmInKeLLd1U9fs3HTg8Ocinx/DSwGA/iu2U9Rm3OD+BajmVg5WeaiWQnKthc9kNNkmcMAsrcnuo/KvT5fh1paxFZfF9mjbh8/2cAAe4MlQD4Z6JMGB8dad9BAue5z/raXMPlPLpL2cFlKowPTg8frTjDcSqt0FGI8A7ONpr0SX4Y6AsoH/CdWZzjOyBeO3XzTW9pXw30W4Rbey8WRyXRG7alqHYr67Vfp05JI57VPNrqNrseWQX13HHIQ+VIwytzkfjT/ALde7djRyZ4bHojAEH9f1r1p/hFBLFv/ALfVd56PYmMsO/3pM1Rn+GulW0v7/wAUFIyoUs8aOMAYAPz9sYHXpitFfoZtLqeZ3cjuiEvkKRzkZBIzVJJ1Zw+ecdq6vxP4O/sPw9BdDWor+XzEjeKOMjy1wxySWPTG3gAYxXIgMqhW3ZA4zRO+7KiklodRFbLJCjkJllBOW/8Ar0VNZ39utlAGntgRGoIKrkcUV5bqSuaH0EKMimFqMk1ZsKWpaQUo5PSmAVC92kdx5JVywXeW2/KBnHJ9fao5dTtomkAdXMYzIFZf3Y9TkjArD1pdPvLyGbUfs6QRYlhkMgPmKACT/dxk4x3waEtdTKc7L3S3Y+KLS7tHnMFzFs3fIYWJOM5OQMYyCM9MjGa3a8sl8bWmn3BsrW18+CFvMERlJVFLcjkEflyD36itiHxhZGORtPt7mJuZZVigMsecbiOOhIBPG3kZNayp6XSMKWKjL3W7s7Z7u3S4W3aeJZmGVjLjcR6461NXEaCLu41I3txO91bs4W3E8O87WGQ6sBhCV256Z7jgCu3qJR5bG9Opzq4CozdRLdLbscSMMr7+wpl9bNd2M9us7wNIhUSR/eXPcV4j4mvtW0i7k0fVJWu5Y2RrebeAFjGew559CeOfrUJ+9Yc5OKukew+JdMt9R0ljcSFBbkyqd4UZAOMnBrxO91cfaOUxDEyiQLJ/rAD9c88/TP0qxb/EPVvKntZ5IZbQgL5LplduMbf92uXN08gIAJ3ZAOfmOapRak30MJtT6HY2evxyxpFYtBvCLBFHLEXd8cEZwev9O1dhDtgtE36auwssKSPGd0gPOVZ24XBXnbyWAxXF+Ab3R9HuLq81SFmvY8C2kWRgRwQR8px361p+JPF0OpJaR2TTx2gZmKzkEq5IwwOScDGR35I6YxpKOqS6i5owi23csagUg3SSwXEctvjdubBDkElRwCF7j8cdTXPXlzeXTTXUkY2tkhVAXgkkE+tR3OuTy24afzXlO2NpGPy4HQfiB0qO3v7o7ViuJIoRL/GW2bvU9ugo5JXsYe2hJc2yK8MkkxKnOV6jk5/Kn216ZJDb7iEbJVnYAAe5OcD/AAq9biNA93HiaFeoaIovpxjnI/DPvUcVs13KkzTDzZgeA2M88DH4Z9OmK19m1qczxMXpYkuJUCRiS0WUK6eY6OSNo64IPpjnpUMkZ88rFcxyHZvIJAODkdPwH5ip4yoiSaZmYI/llgQB6jOeT0NWdSltnkVLZ/mXkiEy7ccjJDscfhkc/nXLpcxVbWyMm4hu44zIYmJXn5R175GOv1q+koaGKN5VlkOQflXAJA6989eSe/41W1GRhFGEbPygOdoJB7YJH1HGKoJvWIgSPnOduSA3XtnHp2qE0ro3u6kUa1tvUSeZKIm29OfmIA4Un6549RRHqRlkCEgLuBZVXGR+FUIryXJf5flPQgkN7UqFXB2oY1P/AC0Odoz60X7Gbg9eY0ZyJYCDIFIIypcnjJ7D2P5dqh8x43kLOORhWzwfpnqarpI0IKvuO7AYZ4A68VI8kTeW6u6vGcq+wcEeuOvbrT8wjdLlexDdOG8xwhSQBWYgcAEZ57D/ACKdbzSTygSiPZ0IbC447f4VYUQxlCjE/KRljnt04pC67gFgOQMLu5A/Lngf55oNlVVrWLUZAaYRbpN5ZQAdoIPTp0A68fpiq6wPqE6WlnDvcvt2RgFhgdfYc9enNa2j+GZb2RpJfMitsj5ipYvxnGD0we5NdXbG20VRbachDEktmMjOD65y2Ce5wPUU1C+5tTg92Zdj4MtrURT6vKpcL8lvGpOT6cDJOewrQl1GI2/2bS44orcjC+UowcdenX8OndhUZR7u8lmuHablWCFgVCnGAeMEcdSMeik81M8MSStv3NISo6jhsDGc5JI7Aj6KvWtowNrqOxC8ayxBCPvliFyCSQcZGAcn3Ab/AHh1pT5C/vt67HG3AP3sDpxnd36lsegqW4uEEuGbCN1z3I9T3xnvuP0qq2yRmbcyMD8yNkFug5zk49Bk49BT0QasguZAfmUviQ4GxhwMZ6hv0z/wGoWivbhdpjaNP4T0/wAP0x+NaMenxk5UFmz/AMtOGz9e/wDSrixxq+Ry3u3vnOQOT7mk5FWOeh8Po8nKbiT0OATVldFjjO4RoRk4Gcn9DW606YACQnjBDIOBj6U13WT5mRdw65HT8f6VAzGl025j2/2fFbm4J+UznCgevHNPtxqMW621dbQt5RcG3D5GGUDORjv65/XEVyuoT3Erwaw9tFn92nkI4UY5zuOSffitBIrmSztBcMHnVXUyEDn5485XLYPAHDfXnpUFdik7Ilv7aSbxDaThSfKjYbh0Gc47fyP4HnM7W5nvLe4iZWQA4w3bnGCO30/M9Kltgslx5ykEFSD3BGRnj8B/npxut6pd6bDaQWMjxTwWy280OfvblHzL2JG09q1doowcjr7Rp3vL1XZmAnAiAxwvlp0A6fMW9DVbVLnT1aOwvREkk/KLOPl3DvzxnmuW/wCFiOkVq7W43Fts0SMVjZOOc/ez+PHPWrurFPFHhaZ0nRUVfOCKoARlyPmJ5JwSM5FJyTWmpKkmcB4jsItP1GP7JcQTwthg1uc4Pp/L169aorcGTYY13PgRnJxgDk02VnglOwAqOpXqOMVSgY+c5aQfIO9cnNqWoqSv2L6yOhRXfO0naAf1qybhbcq0YDKxBcZ96zI5klAVypdfu+gqKecgGMMHYHufu/ShMn2d2bD3oefzJI12nB2Kc7ailugSAcru5XnknrWcJttuCMh8fmPeiOOaZfNYoDGRhRnIHtxincaprdlv7Q1wGZcKFHJ7Z/yKW2U2oNxKNrdUODg//WpqoF2oAvUBs9fxpzXiBSGhU9V68Y7Urh5JHTD4lahDJqCQyB45bgtFI4wyJ0A/ICue1LWJdUeBZLl2aAFVd+eMnH48/pWVHbKZtwlGwDc2f5VF5T/aS0fKtz06UOcmtSlCN9Dt9L8aahpWji2jn/1Wd5AxwQAg9+Fzx6+1WfFniZdX8J6eXwJGffMobIGAwBx26Z/EVwW+7mItkEjgsNsagnJ6cD15/Wna7b3+kQ21tdwPbyTJ5oikyG28gEr2B5688HtjNqcmrFRp2kmYl4xe4Yng9/rWv4O0AeJfEttp8jOlvzJO6DJCL29snC57Z79K0I/hzrhitJLprW0e7mEUcU0h3kkA5IUEDr3OR3rovAWoaZ4U1DUIbmNbmWR0jim2YJw+CMEZHBBx7euMOMdfeNJ1EloaPiKbRdC1i2/se2SGC1IWeCFdpmIYPhjzn05z2GK818R6tJrev3l/ISfMkO0E5wvQD8sD8K2vE+qR32pXeoW2Vt5Jj5eVAOcdSB0PH6d65PbgUpSuKlHqwXrzyKeWwOOvb2qHdijJY1BqTLLxzz7VfiZrlAoBIGckDaPp6Djn8PpWWFZm2qCT7CteydYLSRfMBJGSQo4xgjnqf5DOeeqgyhcQouDHlucZxgH6D/P9BC0pUbV79T60sr/vWAPy54PPSkKoMfMWYjJ9B/nimIjCs3NTpG6jK4b129aiOcqSMqegHeul0m2sZbB2meNRnMinrVRjcmbaRzpPY/8A6qQyvtIJzn15ra1Oym04JcSQq1pO7CINkMAPX09s+lVY4rO5YABoi3HzdM/Wr5L7MEzL60be/atiTQphnb2p8XhHXbq2Nza6Xez245Msdu7J+YGKnkZWpiZA96NxNa2lRWNveOdXt5XRYz5ca8K7/wC0w5AxnoOuM4GTT7zTLZ7ktbFoIW+6pPmD0OG7ilyuwr62McZYqoBPP5mul0zSzDAZZi6Nt3NtUMQMZxjI9s80ljpMUVnNd+TLKFAVJiPlV8g8ds4zxz/Wtq1uVEUiIrNIYjt+UjnHvx+fp3qoxJlIgEdvHPvRZDGB8pZuTz3x7Y6VpRXe1FMcNqFBC7zbxs2OB94jP4571BaW0QtpWuJ2iccxosIkDH0J3rt/WklijWci2maS3BbHmx7Gzk44BYEdPSrS8iGaqajMGPlyOhK8lWI7Y7exIqvfeIdS06M/ZL6eKQjIYSHgk4J/z60WpthGUuIp3JzuMcirgcY6qe+fzFPvbTS3jWQxXhjQYZftCEsc9j5eB+RqpaqyRMdHdmN/wmniNwd2s3pP/Xdv8ak/4TLW/LKvq94DkNu+0OCOue/0/KpBDoCq2dNv5GJG3N2gAGBnon1/OoJYdIIX/Q5Y1GckT5J6Y5Ix+nesuWRq5Iggv5WGY8cnJbHUjvW3qkkVrql9FFgRRzyKgHoGIFVrBNKS0+SDzJ9x+Z2OAuOpAI5z047Hr2fcG1vLh7iW3SIOwLCJn6Z5PzM3JH4e1VZk3RWnvt1sFU8sOgPNWNQuvPvJrvZw7l9v1Oaj2WkIXAYxqW/1mCT0AyQBnoc9M57U2Zre4fJQqSBlYmIHAA43bjyRnnPXjAp2YrpjY5VMYcnk9ajtpwWKNyoBAyO2elTeTZK5hU3UYGMs7q31yNoxn0zx71J9l01ZU8i6u1Rs+ZviRyPTHzL+tP5B8xirGc+WoAHTAxTJETyFlP3mdoxwOihTgf8AfdKsUXkCZbic3BUjy/KUqMF+M7xnOE59HP8Ad5s3FvavpkEMV7K0olaQ5tggUMoBH+sbJyq/mfTFF/IaRlO6oTtVcYzwBWtZX8kmhJBlgizO/Xg8KAPwx+tV7PQVvIyG1a0tXKhmSYSnaM46hCOpA/HPatqy8LNcafDbR69pAaJpJJN0zpn7uAMp/smp5gsc3JKr5aQgk881XaZcAKzqOvDV0UfgiW8HmyeI9Ag352Ibsk/ovH8/aopPh5qQP7rW9AlPYLfY/moqZMpIw4tVvLYlIbu4UegkOPyq/Ya9qJlG3UbgOOcmRuKm/wCFfeIE5V7Bo2yQ8d7GQ347qSf4feJdMjN28Vu8H8RjuFPbJ78fU4pJg0XbrXJL9JJHS2811ZDP5QEgUrtxkY6DjnPp04odtHv3sLW1NzbzyOkUrlldAS2NygAE8HkHHtnvzMgms5pbWdCk0bFHU9j0xT9JjB1aHL4IyydeoBIHHcmrewktTpiiNPapb3ltJucJuk3wYPPPzrjH/AsnPSrS6feXF29vaRx3MqoG/wBGnjkyM9trHn261kOvmxqFIGP0rPuPOgnKLIeeoPNUovuQ5W6Evie2ubTVRBPG6NHDEhyuPnWNdw/A8VkwvID8oPXJNav9t6haytNDcOsjAgvnk55P5+tOtpGlMszhGkmi8qQvGrBk44wRjsDnGcgHOealpl3RnzK+0OzcNzxUKFmnVFJLM4xn1zXS6kkN7pVmDZ21sPNeIvboUZwipjPPJ+bBPU8Zziqdpa6V5McclnMWUMfNhuWRzkgqeQQMc9BzkdxkzZtDuiDU75X1e7liQqjTswGRwpJxyKuZlTUb1J8CZZ3Djbs+bcc/L25zxVObSYZJ3kty0UfmjYhw2I8+/wDEBjqecHJrVlggvNavLyWY2sNzO0ieYCwG7JO4rk9cdAfvexqru5L2HM8YA6YNZzOvzHtVqXTtRezaaGKJrdWIEn2iMZAIycMwOOV6jvWVtuvPMH2aZnU8qiFiPwFNtAkxWI6cevWtK4VwIBkKBEpXvwSTk+/NZuycTKrRSxvjdhkIwMBs/TBBz6EV0t3our3WladdQabdSq9uEZok3EkOwXgZONu3kilfUdjOjlEa5Bz68VUv9VnW2Kx5AJwxA6CpBZ39uqpcWVzHu+7vhZc/TIrLvn2xAsv8XOKVxpFU3rs5Bc8jqBS/aGj5Er4+tQ48xxs+bt04pTA+75vlz61nqUOW7lVuGbPvSme4IySzDPFNEDbjl1xjimbpEyARxxmgB66hOhHzEHPWvQ/DGvRx+DPEDzvEJ44EMTMQrKxJG4Hg5yRjHPPvXmLbnkC4yScD61fnZR4edd4H+loBhcbsK+T77c8/7woT1TBo7GbxVc3NuWM4OV6gYJ/H1rITWNsCwGR/3Y6nvznnP+eKy7eON7U7wdwwVGc546804qRFx8uMYLdatSZDijotG8Qm31OO4kgW6C5LxORhgQR6Y4zn8K6ceMLOzR10zw7ptp5i7WxGpDcd8AZHtXnVjIn2hztPC9+taMcqu/ysW2nHT2p2Td2LVbHS/wBu2ffw7oGe/wDoK0Vy5Zdx+dhRVcsQuz3++v1sRGWikk3ttxGMkVgnxGkV/PA2oJHsclYpljDMM42ht4GDnjPPSujuoXntnjSVombgOhwR9K8x1rw3r1lqTNDcl7QzExgykuq4+8SecdeuSPfiuCmot2YYmdSGsdUeiWurJt337RWgllKW6yuEZxgHkHo2cjHtWrJJFAm+aRI0/vOwA/M14/q3iNLxbezaRnNtiOWSNCSx28jngEHPbBHoM5z49cSytrdH3tcxSnE+eeWyVIyMD73P6VqqVzB4+z5Urnp17rGgab9ovReFHkcI0keXVj6DIK+9c14r1PR9Sv1sJLZJ4LaPz4VgcpukzhlOPw6e/Oa52+8V+ZPDHb6bb2sUW1mjhAjDA4OVAAwe/OTwO2c5qanco/2dAyJJKXGYxuwxAIJ6nIHTNVGitLnNUxsruCtY1fEMaaQlpLbCJIJ1BuLR4iSp98sW3DHPK/w+9Y1r4huNLuRNppjU7cOQpUc/eyvTv+GBis1rZopWief53J27ztyT3HX3x0pbdY4I5JYnkR1wyHduKjpzgDviqj7qM6ktedHa6RrMAeWS6DSKgMYyocHg7cKCOOD/AN8j0FbfhvxpBFqVxY3l0zrNNvillk+WNWUEJk+nPJrzuK9t2FtGLXYVlWWSQg9CMEDnpgZwRwSfcnSudPjvGFydTgha3YKGwMleu4nI9hnnrVyppu7JpYl0Woo9Vk1yPVZriw026WN/JYrdqQwRgQDx/ngg9xXnGtz29tfCfXrGKaaRP3s1th1kBJG9Pm64x2659OalpqQt3+02PmRgTbIpcZRCD3Lcjr69z1zUOoy3GoarPJJJFlgWba2VBJycDoM8/wCNZOglJNG39oNrXczNesbZpxPbxstvkLhlK57E+w4J/OqK2UTzPJblREgLbN5Yqo7scD1GDjBrT1qee/kheeUOSnVcgY7duM/mQKyF1OSwdkiAVzGYnYAZIznOfUHHNLljHQIVa1RWW462cOroELHkAL1ras9FW6sLxmCme3TBijmUnI5B7jH0OSc1i2DsY2EcrLGGBPzdD649feti4As44bv7S8t1v3hiSygEgHj1yemRVU49R158r5VuVbTSZGEbXSyxpu3SRL94IO/TH+ferBdLSYDc72UjCRUI+9jIBH0y3pTJt1y/22eVY4YpCkflnaNw/u7s5U9eRxnp2qjPPMz7gTNCExI6kEgjvgdBgf56VreK6HJL2k76l4XL2JkgjnmdHJMgDAKwPPIx1x37c1PbpHNdmOFPtEUbA5VcNhj6Y55PfpmsGVyrW80XmKTh9r5GAPQkCtWyuwJUiaFXVlPmFskgg5AH5AcjqacZE1aUlHUvzyxQPI9ud0WVUITvDHPJOeBjtnP5Gorm48+aOVUCQkgFlDFQcc9Rnr9fxqEXDxGQ/Y23eUUCH5Tj1I9sf0qXT7kvYLEQQGILYUfOoPYdyOfypSs9CYc0feaH3PmRwyswUBFBUgMN4zwwDADbntgenWsuZJZFYxHDfxKuewHbGf8A6+a1blfJivFheG6swUkaQDcUzjBx29PriqrSxrbxM42MHBQjonb6Dpnr6VlKKOmlUa1SM2EM0aoXOM/MuThfTj1rQSKSGKSQgKU6JtJJHX6Yx/nvU+EmiuLjCrBG25vL+VXJ7ELwPy9aqz3siIs0JVl3EBOccYOACPf1qXpubOcpuyLCLujjkjSWPnhWAO76Y/TFWrZpHQqYdi8lgQ69Ovt6isuO6iiVwZDh0A/eDgHA6kc9f89qnkuop7JEecEpyoySAfbnjtTuTKEnuFyrJc/JlB0AGcNzwAenp37V1uh+GDMqXt6ZVXgrCVY84/i7fh2q74Y8LXFtGl1c+WJcKyq4L7CO+MjB96238+4kkjWV3Qfe7q34fyHfqeBzrGNjpp0rayIlAnfagJi25HljO70/PsO/XhcFpYIWhch8DAAcE54PTJ7ng4J+irjFT26bBuBYkkt1znPJOe+T3PXH90AVRvHSKRnBPfnnqeTz6nj3PcgcVqlY2buTPM3nsUXDDO0Yyc9+/Xjnv6sOlU5btyxSKESFOG4JUH04GCPYYHuarmaW6VEmQxoGz8pzu9M+oHoMAelWfLWIFgFJz0QHGfTFS2Cj3IRDNKTK7P8AljB/wqytuMg43Y4LEZ5/D6U5QWXBKLnqTj+VO3IjKPMDAHO5TjP6VNyrC7VCnKtngDrj+VB2MTuBUe/JoaZCvyMVOecuGz+lRefAoy0h4PRVH55pDJHYKv7tpBjgZI4qK6tri5sTIZZo1bgSKOeOykgj17Gsy9uwsFxJHP8AOsbmMMM/Ng4/XFcnZ61/at/a2WrXl19mXeZ/tNw3lAgEr8pO1QCAMAfzxUSlrYqKvqdjY2Fm89v9k1KWWRHDTRm7BLfMOGUlgB2yAPwrTtW3JZFeVeJipBPcqRgEA9Pbj9az7a/8LrHDtijAWQCKdbVlQPkEbWwBnOO9aEUTxWlojON0SgHA2jGODgdOuf8AOa6aS0MarJkaSKNY/lAD8/MRhd3/AOr6mvLvHzXH/CRF0zGDGpwc8gE8ntn6cc13erXV3Fbre2o81YpAXjbncpBGfwzn0GK5nWPEumapb2y3cIfN6xRxHgiAMBtJ9SDnH0oqq6sYKWuxy8N1bNLdb1dnkQCFQoJ35GeOnTNUv7VkitJbTc6xs2WUnHTOP/Qj+dU7yQJPJ5AYhJCcE5GM8EVH5ocHeBzzXI5M1jTW5IJiVABwRwT2FQqjO7MevDFsdfbNRZO3apyvSnG68sABR9CKRry22Jo42iDFlXc3GQeQO9QzROGDgr+FOyCC8jZ3dMfmaaW8tCUxyefQUhK9y3FtVeUDMDyTiguqhxvGCfmB6HvVFpxzhic+nSpgVnhwSoZQTnHpz/SmLkfUtJO888jYB5yWPaq0ku47SpIPX0qN7r5SF6Y/EmpbVxGoI5kA3ZzxQJx5dSth45VxwDwM9DVv94uG3DCLg1KNjfvPvMrZBPI+mKaCJZCmdhbuB/KkJzuJDPLDJHIshUxtuBz93rV8aVrmv3sviHy/tojkUbApZmwOoXGCFOM/7y8HNKsNtax4WMyT5BLFgU7Hpj61b07WrrSYruSKcxS/K5CDKthxhTg8Dlv0rSNtmyfaJ6xCx1C4gu0u7+WWS/hZxb29yGAUqhIJz7gDH51U1fWIr/VJdSt7QxmaEmVMZCzHqwx0z1/P3ra0zxvqtqZdRnsIp2lIRbhoyBgA/LkevHftWHqNrbwW39pWjj7PcttaEqC8LZ5Hp1BweMj8RVvbQuKV9UYOoTFljAQKhXkDoff27fkKzy5V8ONwq1e3O+XYAwUDA3DB/KqW7cOeoqGbxWhaElqekH5seP8AP4daiYKpBMeM9MHiouB16etXbUtJlNvmD3pDK807uFQhEjHRUUAfj6n65NAmeGMrG5G/BY+oByAfbPOPp6VO0CgnA47iq0kLA7hz7UAM+82WOcnknmrSqXcyIuYyQOvK8d/8aqHIOCMH3qa1n8idHPQHnoaANSDSGu5UiZ1QOwCy+YAicjr+Xt1ByBUtzaXelxRfa4l5JKSwkMOMdcdPx/KopL6NSskaPBJjqqlVP1HPPv8AWtOx1u6YIgjNwgyD5SZJA5xgj2z34z2rWHL1E2Qzam+q2C2k8kciqQS5HzJ7gDGepp99ZaaLa1/swuHQFJQ+S0hz19v8KuPqmjz6h9uuLaJJzJ5u5FGcnnODx+lVpNQ0OIMYLdN2dy5QYyOn4f5PPNa8se5K0NPR/EMXhq9iE1pZ6sGiDKGOfKI5I7g49SPpXsfgfxvB4ttZ3CeTLbyBWiLAkKRwfp1H4V89XM0dxAyRFAisG2KMBuvbpx/U+pq94b8QXXhifz7W33h3DOMlSy9x7jrj3pxnZ2lsV6Huvin4faX4kkaddtreH70qrlZD6sPX/aHP1ryPVvCV74euRFeQ7Fk+5IhykmPQ+3oefavbtD1y11rRkurWTfG65919QR654xUeox29/avZXsSywuQcHsR3B7H39/erlSuUpdzxW4vrm4SQ3MzSuSW3sfU5P61Fp7tJDuBUiJ3QknkmSM4/Lyya0NX0uTR9Ue3cloW5ikI++v8AiO9ZNzM8SiJ9wOPm4wTWLg1oyJ2TujTi0W+njbzVigbJG2e5iibI6/K7A8fSnwafJMgKS2u1Tj95dwxn8mcEj3rLvrid3mQk4jc5x0BPB/kKjQPLE8gzkj+QppMybNd0MMmxlVn25BSVHXqOpUketWm064lspQZbONsBh5l3Gox65LY9sdazIdoLRhlOV5OOmetU9Xm8q2aMkk7sfhnp/KnLRXEtXYtrol1cO2dV0iJOgJv4+PyJqYeGJFRSNa0WQ4PW+HHueP5Zrjd8jLgKfXrRFK65ADZPcmseZ3NeVHXR6bqMULwgWTQ7g5nhvoynHGDlhj72fXr6HDoNNnm8yLMKmNsNvuI0+uNzDd06Ln9RWLp1wUikLHjAHPbr3rXuJngdUxjgg5GDkfyrWKuZvQqXFpfTExx20rOieYVVdxC/3uO3I59xVcE/aEBziRQYzj7wzjj8ePrV+a8lbYA+c9/z/wATV6/1qf8Atm4ntppYv3jsu1yGUlsnnrk9/WhpsLoxrlXtLq5tXHzRSFW+oJFRIylWUnGf51pG5W5PmToHkJ3Fn5JPXn1p88sZuA6QworlXOyMABsHIA7DJPAwOnHAw7MLopqjCMY7HJB4pJJsKoPAHvWldyAvGzwRmWSQMzrldwxyu1SFA+gGPpUKiwLHz4p3GMfJKqjPHqp9z9fpiiz7CuivaieSK4dCohCgPnqckEY/EU1J33omCyZ5rWsv7Dhibzxf7NoB8udDznryg9+PfrV6Kw8NyLHOLjUY1ZiFfykYAjkjAIJPI/OknbdA0nsclHcsHYcDB5OKjlvJjMBg7Oma6CDRtAkmkWXX2typ2KG05nzj3Dn6fh2q4vhDSJxlPF9mT/00hZB9OTUyZSRypk3c7yPWoLqaQwOElYZUnGOuBn9OtdfF4LinIFv4i0gnJ+V5WT/2Xmrdl8PILu4WO81/TxGx25tJi0v4ZUDk9fbNK47GdY3OkXVhaRalpa3SJEczoWWbnLA5yMgZ6HoM1VttP0ZruKVG1Cz8qRWB3pPuwQemFx+OfpVZraOKa4tVkMkUKzCOQDG8RoxU/iVFJaTIy5kIXIo5ejC7RsjTrMPJ9n1OGXgeW9xFLACcj1Ur7csByT2qlqGgan5s80dnJOEfaxtWE4UjqCYyQDx0ODUGCGP7zIzkU6JmZvMBYvggPkggeg/M1ok09yG09zn5ZEIQ7+uQAe5rSsmIVQegFa2nate6dCsFteTJCqlBC7+ZHtJyRsbK45PardrqVvb2Atv7G0eQYRRI9km/C9MkYz9Tz71D5rlK1jI1GRxbaccNtPmMhPQjfjI/EEfhQGCgKoxx0xW7PfWOryQRajZiKKBQkQsTsEKk5OEOQT+I6Drznm7kNbzStGWmXdhM8MR245przB67E8NxvlcYwQahnmyQhcLtJ4HOKZbRTearbHDkfc2kk+3A60+exuUEdzJbyFJF3DA4x79cdD1weD6U7rcVmWdJ8tIJJJFBZz6elWmKM4wqgdeAKdqEE2lpBaXFs0Uxi3MCWOcu3TPToOBx37mqcDlVJYEHNJWeoaouRwqilYxtHTA4/wA9KswyTKyt5sgeP7hDtlep4OeOc/nVNLna3PpSyXalchgKdgOoh8YavpugpPFeM8v2kIBKN3ybWJ+vO3k1Tl+JchRv7S0LS7xipy0kXzD05bd+XeucubhZbC2hAJAlZmOPlPAAH8/0rAvUIYorcZG4E8GsmkjRHoEXxG02RB5ng3Sn4C7sJnAGB1jPb3pZPGvhqXLN4SsVbGMbVIH/AI6K86EYAGPyFB7j8RUpIo9Bj8TeC1cSSeGlDeiRKw/Vh/KmXfiDwRNIgPh6Fg6gkx5TZnscentn8a4E8ggDNN+UHnimKx3kv/CAFwosVKjkESz4x+Kj+Va2qav4M1zQxpSvIsoOYHVHYRNjGRlc/gODivLpdzLkHNLps5R5A+MDGPlBz/nijZpgdW/hPw/aP5UmsSTSLGJBtjby2yMhcr0JH5Z5wRitXTtJ8LjzJrix2x8bY2vGcAg88dfrn8BXHNfPknEnzLtZhjAAIPTFSG7RWZfn59T0/L8adkJ3Oh1jTdF0uyW703Ll5VGNpG0EHplifSsJ5Fjl67dx5GOKv6VexQ3KyXdul5BGpzBL90kggflnP4V1kPi/RSgS88K6cyhdg8uKMAJkHGCnTPOOKbl2RKXc4bzovVfzFFewQ6H4dlgjkbQNKRnUMVbTosrkdDx1oqfaILI2dZ1ZtHNpK8O61kl8uWXP+rz0J9ql1BbS40i5ugFlUwMweMjLAAnAPesrVr/Rdb0eNftisZ1JgiDYMjbeBj8fbtXmCeML7TrIaKbwR/ZZiY5owTlew4HTmuJRbvB7m1Sair7lBTaPdO8krwxvl4Z1TaN+BleD056duOmawrm9kkM6G6AKPhlHO49MggV1N1bzGQ3UqqLV2AMyoMy54JwfYdOmazo0ur3bYXdxdyRZ8lPMfeIwvQgHp6YB4rsSbSsePzRTbYkF5Jd6f9qiUNOhCSKBw4x3GPTPTHSmi/jvJWlZvssca7iPMBKgDomRjJ7ZrPFi8Mtxb+asckW44PdhwefwqjJPKMRhTlCVOf4lNDmxwoQb0NJ7qDU9QLBCEflXk+VmYeh5/wAP5UwyRiNYvO2yAkFzjn8M4x9OvHFUYnA2ygRBc42Sds9sfnVlIxI7ykgMBhV6g+xzmpLlBL0LMV5cXTlVRMpzu/vEDHJzgDHWpJ2updO+0RPbyJENoXzQWXnuAee/X1+lL9kgmjR7aGSUeWPMD5GZOc7cce3Pb3qT+zbea4jS3cwQTFSY/NDFXxySvU9wOOv44rW1jG1NSH/2y0uni3mSNYQuWkXcSz4HX/PbtVvStdNutz5+orHKQd0ZjBWUYPRgD83bnHXjNclLHcRyywS7kkBw6gdh6+1XLWdIbcxyRhkZv9YgAfnqA3px3yO9JTbZrPCx5brqbyzwzQ+c80aOJCRH1KjnBOByP8B7VTvLRbq389G8yX7q9egHAA5znnoevbAq3oP2dZps3ckN0GBt5IlPz5JyDj2x+tWJrYlpRboFdGIYk8YBwT7/AFxROOlzCnU9nOyMu2u1MixSQpnHl/uY8Nzx/COTkjHH5VoJcoFljl82UOxJzGVYAgc49f044OaorDCipeLIJZFkKyIxHHoMdfx/wrVa+ImaWDcqSLsbcvyqCMYbj24PHXHNEdNx1ZqT91GdqFrBFHFawIJFCF2mjPDD064BGD+YziqlvDjyBbbURg4MmQAwwQRk9Pvc5PSrEjxzI0UkPlNyXbHXkdV/OqcMBW7EOzYkaOyDOAxJUHkYP/16p/FcqHw26mg0MSS24XE06DbgtuDDHGB6Y6HgfzqaF9LjeNixSacFJEjVgitk4yOuCAAPzOOlUyksLhDF5KH5vu9R/wDq79aSCKK5uVjN1FE5OUWf5g3OOOPWnrcy5VJWbZf1C5tVtYkCiVFVlLxnYwPH3hg5/TPtRJtljh0uzj814/mADoCzHBPQnJ5xwew+lVmjihuHso5Msy4VX6FxkFSAOuemf61CrWiWwe3QyOyLhFY5GPvA+/fPtSvfVkKCSsrl61sGRX3OrTuPL+zNuEitnOCCB2B5zwfSkUwmN7d7fym8ldxLA4YH7x9M9OhpmmpcXkAkWdkKyFYw7MGLEHaQwPQMDwPTntUqXQaVoJrhZpd23EcgGGGR2+8CD1qkrhLmV12Kl3IRbxrbv5SOMNzgP/dzxgDIJ/L0pkN4HcQTQszH7zKcnHqMDt/k1em0yC4t5JC6qIpAWgDYYejdOQc44x/KsdLeOO5kcEyAAY2kAg8djnntWU01LU66DpyjZbi3EcrSnIXDcYJ5H4Z4rtPAXh17qYahM8UlpC37lJl3Kz/3gB1x6569uK5vQNLk17Uvs3yxxL80lwxwVU4wMHjJ5wPr6V7HEbfTbJIrUKyRKEVBzn0BJ9+/506cbe8dkYvqPv7qeCHYl1GzODjapG0evBPTqfYH6VWto5LWyDBWZmHHGeep+UcEk9u59FFFvE11Kbmb5ZAcDnpkZx7HhT/3z70s0qIZJHZVWIE5xwqYH+IGO/C+tdEVZDbIproQRtJKwGBnduBJB6YJ6k4/EjPCgVg+fPcHzGkYjGQvZOf/ANXNPklN/deZuKrvyik8D1JOOp/z0qVYYdynMoYdgx5NS2NIRUYqv7x3HYHt9KniLRspXAI7k1NEqDI2Bxz8xAJ9uM0NGzSKxXcqkZwgXI9OOlSWSNcSFdk074PUKeKaYYyTtZ+enSo4w20YlbkcgGp+BIziNg3PA7UgGPbMQzKjAjoTwR+AqsbRZZADzxwDlTn6dalnJkbIEozgYGTx9AKom2vJF/49246Fx0/OkCJpdIUA7XjVNuCGBPWvL9CvJF12y+0qpgR8uqLzgDJx154/GvQr201RISkGIgTk7CB/KuGj8OtZ6j9q812KlmxjjofT61H2ky09GelL4x8MJ/oz3siE9TiQ/qAR2q3czxT6etzZFZrVhhXGCCPu/wAxz/KsWyuNLs9OWWWCFQFEcvmgNlh256+ozVs6hp8toIrB7YgtwsJBAyee/HJ/WumlvYwqrS5o24+WF8fKR09TjNcn478KW7WA1SzEVqYj86qMBsnjAHQ5/Mn8a6rSxuiJBDGNSFz0JHeuVufHWm3Ol3NhqtuyTHKPEQcexz/k1rWSa5epz0207o80huWjZ14yw+fPU+3tVq1lsGQQX1uNkmC1xGfnTjjjOD249ql1vTNOtpXltdRMsbKDEpQhmJIzn04z+lYM5KyAByU7buuK4XeL1OpJS1RLPGhBEByAcZPGaq7WVfm69hU4lAj6geoxzUHmbmLHr6VBorkke4HLZ2gZqTJZMADA65qAOCw5OR3p+WB3E5oBj441WXc+3YT92pWn27iQGz8o2+nf+dVgQ78njuDSMeCAuPegVr7llfKaTDKvPOAAcU0NKrFFGUNNjbEeFX5upPrUTXDB/QelArO5aWYxZRRk/wBaWJ2NwC6EtjgZ4qk0pD7l6g96twwzMFkLKB15P+fSglqy1NGNvMQfMIzzhAc5NXtKvdNhV21HTmvmIw2ZyioP72AOT25P4VhXCvvVopNx6YHaki847lKkgDkD0pxbRkoXV0z0oS6FpfhKO4SQC5KM0MUpMoVmYNwDxkYAzx09a81lubmaQiWUuC5kIJ4DHqcepqVUuLqSOGQv/dVAMt9MVLc2s+nQmd7e5iJwY5ZYyoJ9sjBrScuY1hBpakOtC6S2hiuL2PCqu22VicfX3+vTp7VlvEl0rSW1uyfMF2ht3zHJwOOnBx1PHfiopfNmLSkMRn5nPr9akeVrv7qxoWYZVMIMj26DqfzqW7m0VZWEeyuUtvPMeY14YgglPTcOoz2z/jS2sxjYMAM9Kt2sVw08lxO7Dy45czq/zI6odoJ9zheeucD2YjJqL5hto4blUZ32kLG4A5ODwp+nHTAFA2SupYb8Akj2/mMVGqZOcZ/EZFVhNxyrKOO27OPr/jWhbSMLcmUgR55BLAfyP+H5imSRtZCcbcFWwTyOlZcsLwuVYcj0robiTYhaMxJGF/gCnf8AmBn8OR6Csu6ZXXOB042Lj8/0oYXKCttPY/UVIJRtI2rz1OOetQ0CkUaXlm9bLXae5kb5ifX+v41oW9pBbKGTBuR8yMpJYdMcf56+9c/mn7zs2g4FNMk6u4125RQrywK6ne+wDex/2vf65x+dYV/cNK5kMqtglQd+cjqP8apS4kldgfvMSKa0UijJRgPcU3Jy3GdJ4Q8Z3vhbUNyFpbKRszQZ/wDHl9D/AD6fT2+x8YaVrGnfaIriMkLnBOCPavm+KCeVv3UTN2zjj866XT7C0tLcSSTO9y2MqjFQPY8YNdFKs0rS2Jfke7Xv9kvps0uoeU1rjLCQjj0wex9MeteOQywzanPOY/PCbvLJUtkKpwSp4xwOvv71rXUumXkcEBTUGYfKD9oTnJ9Nh9vyFZ1p/Z9usiyW/mlh98yMCp9Rg4/MHpVTlz7Izcr7i3GuTtA8ISArkj/j3TngAk8deBz14pZkWC5igKKQ8SOTsAyWRWIx+NX47TTooIZY7edlfJIeZff0QH0PWi5u7OW4BbTY87Aq4lkyoAAGPm7Bcc+v0xGu6QtCsb02lnKkY+VgAo7Kcg5x+BH407/hI7tdOeGeC2uVHKiaBJCpOPmG4H26elVzqWmSxOr2kqhs8RyAY54xkHpU7w6LFaSSeffQtKNyK0aONu7ABbcp7elD21Q1uVD4r1AtN/xLtKyQQNunQLt9MfJz+Oagj8T6irhpLXSSM5I/s2AfySiDT4ZmBa/gSM55DLnHGOCw59s9qkj0SKR2dr1Au0FSTHknuMeZ9cHvx0zxloWWLPU/7UFx9ps7TcSCvlQiPaM54249O+anFvBeXWxkkMk0nBMjELuYevJ/Hn3zTNI0K6eedImhUn7izSpGWx6ZbGeema0n0PU9Ou4RcQojb0I/fxtn5gOAGyQDweO1axaM5J30Mq4sYZ1URSG3O7c2BuBz2A6jHPOT1HHGTZubLT7i5klhWSzjZsmNf3+36ZK9/X86bcRSW8jiVGjMbbW3jGD6c/jUbONrHcOBnOaqyZN2iKbS2juXW1u0uI8Aq+CmRgdQenX36H8YpYrgSiMW77VO3coyp5xnPT3q15UsL5dGR1GGDAggjqCOxqSK73RKH5B6/WnYdyO1MkVpOZIpQxRlVthI3ZBwT2+p9R61j/apPKmdsAggD2/zmtYaiy2flMxCuoLIDweVP80U/wDAR6VXju7RUfbCBuGGZVGDyOx6HpyOaSTE2jLNwQpjcklh+Rz3/lXQwzBPDGnrs5Mk5GB1J2gfyrPDW08xV4twYcNnBB/Dt1/Ot2xu9HhjFlqOlyS24OUeOZgyZ7AZxg89u49KTumVozmFkBDl2wwc7ipzz9e9TmdEwi9cYzXQ3J8CECNNN1SRu8rXIU578Zwc+uPypIH8FwIWbTtUYbcf8fC4Y+vHNTd9UOy7nP7txJDHOOvapYbp4mBDk7P0rUn1Pwr5DiDQ7wShjsc3fBHbIwfyGPrVDRrbS9WvhBNePafKSSV4YDk85+U4z69qakJo0Il00QzebHdRytGyxtCVbJZGVt245A5B4z/F0yMUo9A+0XsqabqCNFHHvaS9Xys8hQBjd/eHXHeqscyyh9rEqp45qxa3ht/tBByskRjJ/wCBK381FJx1GnpqWLbRtTuXMMEUErbQSPtMSk5GTjcwzjuBnFRNDdWBeG7sZ4ZF6iSMqcZPPPbjr7UySbCkhgwLYrQt9Z1C1gWK11G7giHRIp2VR+AOKvlktieZGMkmGA4LE+tXAhZM4NStdWPn+dqFgt0RKZpJFlaKR8gZBK8EcEnjOSTnk1sXFlo8Wpz2y3l4kKSsnmbUkHB6jBHHX1rO7TL0aMPcVwy8HvSW7BZrl2QlRFyw/hBYDP64/GtK70lYHkW31KwuhGBnEojY59A2FOPZj+fFRR6XqsVveMmk3EyTQ+WDGu//AJaI2Rtzn7vb+eAackSk7mGXAu23MfJAJbAySMfrWpBculxcIk5w7FX2yZ3YPGSPvfWotFtYm1G0S4DKzTKChUhsFsfd6/pVARzW9yCoYqDwR/hQncHsbtzr+oWqhzcCXGdonjWQJuznAYEA84z1qvaeINJEaQahoUEwUELJBO8LcnJLFT83t6Disq7lfcsjqwQtjp1qpLIrEMVxkVXImhczR2hTw2TE8UOpiEvlj9oTITA5ClTk5zwWAxj8JbjRdCd43XWbmBJE8xUntFlYAkgAlCvPHoeorjYLuUxJuztHGMVsahKY7mFSScwxkHPYqD/Wp5R8x0ieCrPVdNuLWw8QW8k8LrIvmQtGiqwxy3J5IHbg8Vi6h8MPFUczpDDbX2xwC0Nwi4JAIB8zYc4I7d6r+aoxnGPWnNqNzaQFrO5lgkJ4aNyv8ql05dxqotgf4deK0JQ6ThxzgXUJ456Yfnp2pbX4a+K7p2J0qSFVBOZWUZI7dc/0qjaeINZsJZJYNRuI2c5Yo5+bjHPr/wDq9BVuPxfrsR3nVLt+n35S/wCjZqHFl3K8/gTxVbxtu0W9wOCEj3/ltzn8KzZfC+vW/wDrtE1RB6tZSgfqtdH/AMLB8Qrhlv26/wDPJP8A4mrKfEzxHj5tRGP+uMY/9lpcrHdHKpoGsyIAmkX5DD5QLd8t9OKrNpV7ZXDQ3dpcW87H5UnhaMt9A3Wu0m+I2sShWnnjYj+LyhnH4Y/SugtfGhWzs57aCFJZ2kExEPHG1lYE57NyD14qlBvQlzSPOl0fUGtFuEsLk25PEwibY2PRsYNTr4d1mbDx6JqEsbLlTHbu2R1BGBzXog8WXUhcjy1B+9iJcH8x/nFQS6/dudxvJozjH7tygIz0wuBV+yZHtUcDbq0XmxygxzKdrRMhUqR65HWlfJVsHAxitTxDeSXM0EkszvnccuxY9vWsd51VAyc55PNTNcrsWndXPaIdWjjhjR0YMqgH5R1x9aKz1tZY1CRysyKMKTIvI7UVEYRsO6PDTcT2d8Ck06mM/umYbWXHtk4/OrVvexs04vo/MM3zq4YDD/Tpjnmtq6FjezW62emwwh4h5wYMm18nOwluTj3Oaq3EOmTwlbKGaf5TtcQrkEdjg5+p4+lQk0cEsQpLla+4huBb2+/7MwLRkb3POSp7H0J6VoWN1a31kbXzJTfXUgyZjhSTwOcnpk84yfrXP8oxSSMlFz+7Hc/XFI0kMUDGPcXDYAPTHr6g1UZO1nsKVJTVlqXr3R73SpiJSxklGCHHT/gQP+H4Vj7JZQZAGdUGXHUKPc1e/tS/nia1fzJdzE5kyz5PTnr0BpLjSryKC32mNluBlERiSCOxHqKl9zWLlFe+0L9mYNGklvMoCiR8dwehHXj86lhtgs7SQCXyMAYlz95s4+YDHYntnBqGN7oCB5XkiRVMQfYeV7j35z/9ap5buO4iWGJWWRBhwEyGx0Y44/McepqrJmUnN+6WdJEEAu0uZpYJnAaILnbIORjHr0weOh5ogl0wSTQSxRs4ibBf5BkAkAEnr+RJNVYLd47mOSW3DKAJMK/LnHGevJJyfxrTjiQSLObZYHKkR+TJsYHjBBzkn8+9XZ2Oeo4qV5Pfszm5ZWklRnJZu56nA6Z/KraSuqhkI3FcD5hwD2/Km26xfa2W8Vo2JO4qg3A+mOO/arltb77q4jCLKkWGZlfBQf3unIrFJndOUUtiSziO0zO6LCmDISSep7D159qnk1B7aP7MsqlVXaGi2kkMMgZIPTjP5Vm6isgvSVnEsbANuU56jPIzwfalilCxqjQlozncVYZOPb2z1qr6WOSUE/e3NbRpotPtprlJWlnuAQ8YQEZ5xg5Ofpx/Wp/tMkuoPHfkoZBmJ1wgLHncy9vXjv2I4rES7ltp5oIrcxpICCr4V169yOO2R36VYtnlRIXtm+6fm3EADkdCcd+OtaRaMqtJ7vqWtyxWu9MFhkqxBwR7ZHTg+nQ1iN5bXEPk+Ym3JxnncSOnHHGPWupNnJf28skMkcMcbCR1HKrIT24yQeevAx16VUh0W1jilmeUGQExxoAACzHGVz7gc8YGetLldwpVYQXvbsz/ALNOJLpRcNv+47Ox6buSe+OOmP8A68VzZMZvO3RFt4YMrcEA46Dn/Par93p7wafFMshMzZDBQSVAx1P4/pURZLW2aRUZioO0g8c5yKhprc2jVT2K1xvjZZJyZJpPlDAHPr369vzq2lzcaVIkNpcoVLbUIk4LYAJIIAIPuKNLjGsW7wrNCs5dURJFKtk55BGcgYx+IqpNBYpbPBNNNJdIwJTbxjrgdyPxH0o1ByV+QZJBeQPdE7grsBKQuAWzkZPQHPv7d6ngsJoVhvCxWR2JjPptGS36VA1yIEWGOV0t5gNyuCoBBGGK5Pp1z610Vu6JZRwSsPPs2EfmwEPkFjjHTI5Pr24qox00JqzlFIw2mmuc8xBifMMmMcE8jA7A44x3q1pOlyaldGFQNz8swP3AOpIzz+nJqleJFA5jkR1ZD93oM+3HTGD0r0XwFYPpuli5DQJLeAOzy8lV7ADP4n6+1RBcz1O2nBWTjsdBp2nx6JYItqscSgbj0O4AdST1PA/TtgU+5+23VwHlGE4EYU8A56474x0/iPHQVbDSyth5opABkNGm7Hp+Pce4APWnXKF1bGzCtu+U7QOOx+nHsvPU10RWprLYrqFeTrtUNg8k56/4tz3+c9xWPquZtRKLIzRrgvHlSjMM4zxnjJHXGc8Vo31wII9+VDOThSep4zx9MfTCjsayxGsuPljUIMkx+/1NNiROvllVdhFGwz8scYI/WpPlKcqGJ9UUD8xUKI+QRnLdiKuC0KkObmJOy5O7/wCt/wDrrNtFpDF8toyfKw2cDGAPypWIOPlHsRilMxJYNOxVR1GBUUWI3yCCzc5Zc4/OgCYO3C8D3weKQoSh/eSE9yvSka5IORwOnAxUM11JISPOY7Rn73+fWgENDFmOEZkXgZKjJ9/mzUi3TR7UYQIPQPkj61msJjtGBhm4LHA/M1HB9snmZYXMfy85PCjj+uKQzSuLhZYyRhh6ruWsK9dZFk+WQ8f3TgcirFyt/GMPPEyqeeuTWXl5Ekyys4GBtYg9R+VJbjJNO0/Tr7WkjvIMbl2JI6nbvHOOoGSAeDnt68diui6NEy7RbxToCEcBV2j6LgE8VwDuZJDGx2q6gbvc55z+f0qbSLJL/VAmranqDPuxFHE6oj+oyCefYAH3q1o9BbndxKLfCBhtXAyDnPOM/wBfxryLxhoU8PiBmgZZRckyLwEHJ6DmvW7PRoIYpI7a5u2ZzlY55RIB7DuB9Sa808Y+GNaOtTS20L3UDDeGiP3e2COx4q6rUop9TCEXGfkcheWF5brFJMjfvM4ywJJHB4BJx9evaqBy5BJ5rX1LRta0cCW/s5Ylz9/quT0+YZH61nrLEsYwNxPJBHeuR7nQm7FaQMrYbOacsJiZXfGDzg96sBgI/mBJY9DUVwXJ6DjikNNvQuiKF2LFFPHHYUCyjjG55f3Z7DrVa13mMow6cjIqQsUTafmA5yKZi1JOyY+4tocfuiUx696oupRckn6GrSuHck42joKV0jKkE4z29KRcW46MbbP+6yR9MdzUlxZFirQkMCMkHqKii+TKK3Q8VbVtyHKMz9CMcimKTad0VBbGKXnDY7DmmzzSK5BrRhtJHlGN7EkZSNdzGtD7DMkjG30+KA8Am7kj8z/x/AH4AUJNlJN6s5y1WeeXEaliOtdBpWkPNKrTOq25DZdjjJxj5R/Fjg9h2yCRnXs9LMdostwI2bcflSWNxx6lSe3rVmeeBvKjQJKyliOybcDse3yjrgexrWMO4O3QiWC4tw7WUi2sJfPnyMQz8Y5ZQWbkdhgH0rNvtOTVpUjn1ItOTx5cfmbvbJIPT2rfvXRrRmYmYtgBVB55x16n6cdO9Un05ggkj06S1APDkSAe/wB4/wAqtpbEpvcqHTrPTrArcRxylCFTPYeuPXr6n1J6Vh6np9qxN20wjg+78q/MT3Ht+uOnaumvYFntZ0tnaJdvzyykckHP4A8DAzgc84zXJ3zn/hFkwTzdmIgsCPlRemO3I9aiSsUpMyHkjTiOFGjByBIOfxIwa2fDt/Gt1IsP2LT7oxN5dxJvZW9UIJYcjODjOQPWsYAkjqOOc1XkXa5FQabqx1Oo6S6LJcWjgxDL7cKSPU/l/nisQ3EnyBWQEHPyhUI9xgcCs8Ejpmgce1AoxaVmaE2pTF8Eg7T8oDdPXkVSkkZ23HrTMH2/OpJITHIY9ysVGTt5Hv8Al+XFBRFRSkGkwaAFopVxuGcke1amh7RqMRYIoXLbycEEDIOe3OKFuJuyMxWKsCOoNegaGov7SCU3UKhVIfeT1H0B5x64HFUtLUXWoyx3VlBBImCgdQnBUjknjp/npW4tmLKN/wB5a85wiSK2OP8AZyOee/U1vCKWtzGbv0K93pljMEe5vpI3AwqRQb/l55OWHOQePTnPanWegabNFCy6nKHBBK/ZAQSRyM+Z0GPqfQVPNYo8BWG9tnmZQcbZBt55H3PX8KsaRZyQ6Yon8uGcSt8jnJ2YGDlQR13U7RbC7SKsVgsFzHNLcGNUbcuIN+WHIH3hx17/AONQtYaa0zj+0pwAVwfsajIzgn/W9fQd/UVd1BkTKExS4yf3bPlT/wACUds/mKzYonZ0lKsELcBlxkAE55wD0I+ox3GbsidTWb7FhbaK8JjRUHnvCUzhQOEDN+eaz2MNnqMU/mwzojq2MsuMNnnK+3vXOa1eXdtfK8bFEJ3AZ/DkdunQ10IimvLJriC2lljEW3dEhcAknGcdOTj8vWldP3bjs1qJZaJAVmT+0bNpCCEUsy7h67nUDPtmnanol2dJgeRYWSJNpdLqJupO0Haxxz3PvTjY3NvpttdTWl0iqzBmMTYHCAZOPXNNS7jWPLlsHkkemccUWutxXs9jPbwXriRoB9idnG5UW8iz/wChcH2OD6A81LL4G8W20Jll0S6RAu4kLuyOnbPNV5L+ZE8tGOw+v1rT0vVr2PzJPtLqqpjgnlScH8ORWPJ5mvMZttpN7BdmO7jnglDBTEwKtjB/h69cVozTMjsFOADz6mtBdYYTrPcKly6EFTMWYqQSQQQQRyc9frUUN7pzwTQzWQ87qsiTFdo4GCG3Z788VtC8TKVmYvngsybR35CA4qUOsOpEwqpaOQtHlOcg5B9fzq6lrblA7xsrA5JVvvDcO2OPl3evOPXiqbbfOpISIytsBJ+UZGfwHXnFW77tEp9i3earPe3ha4m3yjG6VQMkjHU456Y5zxULRRSRsi+YhKgffGCfX7pPPp2JPth39lv9pWNAJpQm9kt2EuFHU/IT6f5zVVZFYKwYbT0waLRYXkgktHSSMIu9MfMxwMHJ4HPPGOePp6uh8PXrRfakhaW2MxhV153t7L97HI5x3FRTXEvlfKPmz0qWRzFpkNx13yvkevC/5/8A1UnHXRhfuD6XdWzC4e1nSDdtLvEyqp7DJHX2pLsukMzhySoJGfYZqfSddvLKVjb3Dxqy4bB+Vh6EHgjk9R3q7JOINUhghtLOU3Yjj8iaNFjO5F5JxlckjJUjrScnFDSTOVhbKKWTkdMnmntKd5Oa6iBrUXX2KXwlaQzMz8SXM+GwxBClWIwDkcddtWpXjFtLA3hfT7eRo9qSPLJlCM4PTJzx6Z7msbtmtkcU04YdapXBUtGCOS449RzXbX+s2RsZIP8AhELSOZkKJJHKW2t/eGDn17+n0PKNIL7ZaoPKRGDrGyAsTgAnd1GcUtRnWnS9JmsZmtmltLkJuVD+8jbCjOT1BLbj2AGOp6wJo5dlisr6OQyFQFmiaIlsgDgFl7nksPwHNSadPmK+c4UrbkLk46yIv8iarRy4dsE4yKtLsS33GLp1zEriaAhlcjKkOpwOcMuRgYPPT3qCTMauhDK6k5UjBH5/jWr9tkjUpFIy5IztbHI6GrOn3q3eoAanH9tjjWaUJKchiscj4J6gE5OM4zzitFJ2M7I52BWu5EgRxHI8gVZHOAhJwCfpVlZcWxYEbuSQfbtXRWFx4YXVLfUFsb21kgdZPLiuA8e5TkHDDd1xxu7VWt/DK3jKtjf2cySO/lxzymOXAzyy4xyPQ9/rSUrO7Kcb7GFdyJb3pgjaSKJDhd4weOOfel80OhaQKSCDnryKi1qJjrd2Q3mbpnx8w5+Y+n9KijR44yvJbrincVrHRWni7VY4VgN/I8KkHy5wJV47YYHA+lV9Su01B5ZZ4o0luJPM8yJNpztAx6YOMnjkknNY8cbgZ2nPU81KkzskcZX5lfJ44xTSQm2a8dp4dvIojdS6oAEbMSSxOCxxtKuUB/vZBXuMEd6NxoFk8O+y1H5tkjmCaJlKYxtUOu4OxyccKOO2cVmSH/TVSThByMcDmtOyGIdgIwRyO4qHpsV6lix8KaveWCS2tss8IJUt5yLtPoQSCPXJGKmn8M6zbA+ZpF4dpC7o4jICceq5/OqElywkABPHU44+lXI/E2pQXErpfXAZiCzeaTnGMZ59hTtJoV4opMdwKkfMpwR6GoJsi2wRnnP0rrYtfm1Bg19a2tyQm1/MiUF8EnO7GQfmbJHt75sfZPCt9DJcX63lgc58q3OY29l+ViCPf/8AUlJpDsm9Dzn94wO05z69qd5U6gZKc9ATXdSeHfBjgtD4nmiJyQjwZI/9B/8Ar1XvPCHh5U+TxnaeYeheJcfpISKi66ouz6HEtE4G13Uf7rZqRLZWJKyAfUZrp/8AhBbKfaIPF2lO7HgMGXP86Jfh1qsDk2uo2Nx06SbSfwIqQexzpiKHHUnt0zU4nEk9vHb+YREjCaKSU7UbJVhwB1AU4/M9q6Ox+Ht7eTbNQ1G0hXcB+6lDE+3tx9fpWrc/C6Gxmie18RQRSy/6z7QgZWPtyPy5pqQnG5y8N8BnLjaBjCnp7ck1YW9QR/e4PPNa0/gaytpj9s8Q2fmE4Hl2+Mc9vn6CqFrodgurXUd5fSGzjO2F7ULufk9dwx09M/jW3O3sjPlSIf8AQrhS1xbvPEP4Ul2H8G5/kanhs/DCi3M1hfsQhEqifKlsnB4IJ4wO3PPsWalDpmnXAttNknlgVRl5tuWPPTHbpVQT7hkdKmUb7jTa2Onj1bQFjVTbakMADAmUgfiRmiuUN0gJHzflRU8kQ1K1yUmmKtdKrLGSEIICkZ+8CO/J4NaunzR2lvHckRpFJG2DFAWXI45yCR1A4xyRnNYr6jJLDxCZAP8AXuSdvP0HHJ4qdHuYUgjlZFj+XcCC6qjcMc54PCn8KW+x4zg7WloOtpYbqW8jukw8rq0P3g/JPQKdvpwR345psVvFa3KhIvPAw0okIbIY8FRtBXGDyTz7U6eJXjkZZWRg+Ah5ycevbinWwkjlRoYRJ54ePzSOgABOfwz+WalJLRlKba02K39kCBRNPcw7WUsChO/dkAjHqMj6jFS3lij2rlJT9nCB3BYAnkcjr83+eaeIVEMF9JiSGB/LlyhHXaCCfoQc+9VLhrY3bpFK0ts5xCu1T9ct/Dz/ADobSHFzm029iea0iCwZkeTyc7FkOAFz6578HjA596vyXsMNvJbm3YySygIT90k9cnjnPr2qlbXTTWMzXEYMdqRGpUg793QNzyBt4Pp60lncxzRsGnSOUyFGBbaVG0sOOuOMH8KpSWqMp05S0lrYvC3ubKZxJMu9GDIIuceuckE/yps217MI42IW/cufkcZ6g9j1PB/PtVfzPtheK3G26XO+QP8A63GASM8+hxzUoh3zSLeyi2lT5k3QeeJDtKnoccHHBzgg59KfM7EezfNq7Emux24lWCZIPPYkrPtCuy9tx9en0/KspLKU3EUh3xp5bBJoVY8D+8Bkj3A/vCrd1bSbYJru4kf5QqSFSoXA4zn0x2J+tQhPtN4ttN8nm7mBdFAbOAccDqVGPce5qHZs2oycVvcqrbOjStN8kOSVZwNzN0+o9eatW86i4jS1VSVjIIXJLEDn69ankjkurFnLiSNGKsGU7BkDkDseoz7VSm8+zsre6t7BoI3O0XDNglhnhR26daaunoXdVHq/8hZ7aSUqiMFZ5Auz+MnHHHpx+NWolgtZ4beSXaDkEOCoGe7Edsnpn8azBqF9b6rJLcMpmV23LnJz3wevap7wmdkuriYq0wHyL3Xg85znoOvpUX6ouUGrRb0NPLgNb2kuZAp8zym3eYPY+gAz7VXMdxG2ZSImEe9PM+7Lj+Hg8HGMA+naq1hrU2kXbtbInX7ijKsB2q7fX9xeSiW7hSSHlmEa4Az2AJ5AJqt9mZezcXsRy3E4topHuCSYwpGdxGKm+1W8DRQzzr5LgSFFPTvwR06/z+lWIdOhuYbZJJIicFiiTKdp+nHPt3xxnpWW93ZmJbO4VgUZih6DPdcjOc4GcY5/Om5O9mZqKf8AwChqNpHFfk6fcCREcurqc4A56H0rU0WKzuZHmuppUu5GcbmjHlnPQ57dx0wc/WlTQIrmyfUbWQRJyGV5ARtwT16YJBA9SD0qhbW0Msrsi74OCSGHy+vBIz9M1KVtLHS6inBq+xo3WkzxQJBsBkiLMIkyXxk5G7GMADPrz36DMM5tkYhGAY7jgg4B9uf5Vt26XovNlneeaWQzKJPMQbxjJG3nIUc+wNZmp25SGG7E8QmlGwQLwF6jgg9QcfnSceqHRq6qMrO5TtTda7rNpYSTlo5Zgu49AGI3H8h+le8rZ2FvbMY7dGbsJEJ5+pzXmHhXUkv9SiuJoIvtVijM8+wDI27csQCxPf8ADsM16Ha6i0wRzdWjoScMnJP0Iz/StFG0TvpzW1rFqJQJmEcSNgc4478LjtyOvqBQy5I6YVAZHdcjHBJI54HB/wC+V6CobGO5uJJZbeIZKmN2ZvLQ/McgnBOfoD/ho/YpHV4bm7iQvgyLEozxz1fORk5+6Kq9kXucw2/UrwzwQOVQ7IsAswAPU9ee59yatppV2qRu1s/lnknbtJ9ua6NdHsoUG+3EgAzmdi4H03Egfhiph5YG2NIgPQPiobKSOdi0y9dWJQqTwcnPH4UjafNFIrhkUqMjcGPI+o9a35rhYMIzW0ZbgAgtnP8AwKuc1G0udakktrQxEop3yt8iRdCM8nqeB1/nQk2NtIbLEg/cNcIHPP3MsR9TximrHAp+W4Z2LbRuA9M9vamS6RYiSG4vru6uHVvLZYMRKG5OQxySpOccDimTR6Pb26PFpERjLgOWmlZyME5B3AA/hVqBHMOkKiTavzlBklaglkJUnOSx5B4Ix/Km3UUMcn+g2YSZRlV81yZRu5DFifY8Y9KWORZIxIqqQVyAM+hPrUyjYcZXKMiMWPmsyhVOM5Iz6fiapPDII1baTu5GB+H9K0nmm5IcANwRtFVJZXMi5d1AXogA5x1/OszRFRlvzuVQ4RR0kyB/nmoYIhiUy7S+08KD6j/P+eZjcSiKU/aSSSOH+YnryP8APeo4EnMnlrM+6ZQMbM7+QwHtkgcimtxMrShXYKMA5HIPzDkj6UQyRTq1vjYxfG7aVb68/wA88UOh8snghuRx7nvUcDKLgSorYEhIO7OPp/hWltSUWZte1yS4jso9S062nVtoaRWWQkf3iVK5PtjrXYaXbasbjzL7WUuiy4ZBDgIfQOMfqK4e7kW4naN2ZI8gsFuUdSMgY8sDIHuTxWrYeJbzTEeFY0u4FAIVz+8Vf9lv4h6A89s0otrUJJNWJfiCn2GxglklcJ5gRbYxboXzzySPYcE/Qd681murbUrmKG4itrIKxy8cIj4Prj0+lemS+P21CQ2A0h5rVuJIZofPZh6eXlR+p7Vi3ul+HLxwF0bxDFGpLhBCoCqeTtyc444JJqZycmKMEjz6ZbaPe0FxJJjA+ZMf1qLzc9ADxXay6ZoMUZSDw1qTscbXu7goMdidowR+I6dan0xIdMw66Bo8krchp2ab6cFyO/UfnWfLc0sjhU3CTkAA9RUyxvcOyRq3TlV5xXc3dxeGXLWHh6BnbKRrZ/mSdv4dazrzVr+dkgbWI/KDbfItFdVz0xt2jP40cjFYwLHSLq+3C1iEm04LswRB9WYhR+JqydDt0Yi+1mziK/eWMPKyn6qu0/g1bscEMLDfbn5yArSyHceMkEDHfnpmpLaRo79UeZIoQSVEMQyufbjJ4HftVKAvMxFstOMiRaal9ezYAaeVVVB77F3EAepf8KvppcVnGXlnD4xvRBgH1Gep/StY6rFc2piGoX85PAEsm0Z/3ct/OsuSI7mV3WNd44Xqfx7cH9KpRSB6smCS3NojRTfZrRuHjQHJx3KcBj9SDVOeztLi9l8hLuWRnJ2rMrDPptCZ/WtSGOW2dYY0EKj5i03r+IJ9DwO9Lc3bCeRbvXbiRd3C28bygD0+dkpvUEJbaesejFLiG6spGmyInjZS6gc4LYwOeMA80KEitmQGOGIde5Y8fn07ke3pUaJF5LvY3TOXk2s08KxumB/DhmODn9KbGFBVVIklzwZACoOeDj6+uapK4mywbsJbyLGFto9u1G3fPJyOp6/yH61S8+zMwS2nmlkLbViMOCzfg5q4Yd8cpW1fVL0ndhg5DnIB2qpDH1yfyFUGN/BKon8Nw2alh88tqyFecZBbmi7TsCStcxvE2rs0sthEV2xtiRl9R2HbFYD3hfSYrLafknebdkY+ZVHp/s+tX9asbhdSkkMBjjncmInoR65/nWOdyMVdcEVjJvmdzWKXKh4nZSM9vSgsJDzyaa6YUGo+hqR2JtvpTsdMiohIw75+tOE3qv5UE2JreOJ7nbKjtGBkqjBT+ZB/lWlMmnBQYkktmxglIt5P4tJ+oA61nWxDSO+4Z24GTyeRUkhYY+8cdNxzTAgutrTkq7MMdWTafyyagIFT3CeU5VgQw6qTyPr7+1RBGbJAJ+gpANAqdJGijkZWKsQACDyDkH+lRcZpcCSdVLhFJALN0HuaA3Oh8P3kty1zE5/esobeOCcZyT+Gcn2zzya6aNfsMspkhJ3RFGVhgjcCoIHfk1zOnaJc2d+X+0x/JlW8pjzzyOQMiuhhYwXDKWHPcng10wu1ZmU7J3RNYWl4sjupjZcZb96AcBiOASCefT69Oa03MsZzKAFOF++pOT6jP1+lUoDNLBKsaPIxCk7FLYB5HT2yfpTBNbmHc80QdCBsLDdyCf6ChMVrlC5luLm5cRxys7ksEVSzY6n36ZNX57S4itIcwy7VUA74yMNjcRz3HPFRzMrRhoyMDv17VGk7pdR4OCDwR1454/GrV9xOxyWpws+sTIDkOwwMevSuwnmaIu4HKpgH05HI/wA96y7jWdRur+NXuWEZcLgY3Y6EbuvrVrbGygDK5BUHccgE++fSpho2OT2JYNauEHlG4fyyMbN3GPTH41ote+TdvGLe0OxiuXto2PGQOq/569aqS6dFC8cLKN8RKy4Y5dh3yc47dB/PhZkjM7yCQ4fB8xkAye/AJ4HsPwHGafmifRizPZeajy6daqA+5ym9d/IPQMAOhHAH3j7Ys29np62oVpbiLJ5O0SDb1+7lechepx14GBVKW0e4jwJId4GV3SBQev8Aexz0wBnrVXyr2JX3xS7R1YKcA9cVnZXLuy62mSNF5qXNuSxwEZipJHJ5IC9/Wo10jVVil8i0klkLr/qCswAwePlJBznt6GnLLD9hiGVEglYsAPmwQuM+2Q2PxpLZ1knghcjEkqp83QgkA5/OtVF23IbVyo81zay/vYZEIZlw42+mRg9wcVbtpo7lbwHlo4N8Yz/FvQfyJH40xr9jzJLI2MgZcjG4YPOe44pjN5qQhVEKqzuAiryWBHOR7nHp2xVXkTaJg3fiOe0vnjhSN1Q4y2Tz/wDWNbVpcpqFl9oSBWZkZyrc42g55+i/lXKapNbTXDPDAkRBOQpYgnPXkk/rW94Vw+nvi5jinJeOKFwV8zcuOGxt5JxjIrDn97U1cfd0LqzQTuqTW7AKR5hhOGcA++QDzjhew98zPbWl8qLLdT24DERp5CsEGT1bcC2BjJC59BWduubSd1kheOVGwQy4I9qtXs6vDHIxwzLwO3vWySexDunqI+kmzuti3UUmSFBXILZVXBCsA2CCOwrUv2jt9Qs7vy/MSBYmHJAYqinr9QBxWFOZYbh0mQCSIiLlcEbAF5x345960V1a5trOWHz5FhZSGVXO05yDkdD/APXqLNoq6TMXUdYnubkzwNJEWbJO45wfft/9eqy3t6lyw+1ysUOAVc8/41bit7GeRVEMkUcYJLRMMnn0PXkj8DW3B4b0q8kRlu5Le3XaZSyiT8sYJ5K9u5OQBWDTuaJodZeIxa6aWmtTNNI37qWRyF4HPC4z+JIq1H4vtFjUywkS/wAQRQQPwOarX/gto7WO90e/tL+N8ZgVwk6MTgJ5bHLcnjGSfSuaRZLO4KTxYZThg3BHt7daak0DSOyktrCaJNt9NbNvzIGg3oQW/wBk5OF5xjkr1GeJR4cvLm3Mmlz2d9833Ip/LkA/2lkC4+mTVDGQc8c8GpoJApB3Yx3FbcvVMyUujRUvLDUNN3PeWk9uCdoaVCFJ9Aeh/AmoLG4Ec0jzFlTyZRkDuY2AHT1I/wDrVsanNMNSmt2dmiifYI2YlSF4GQeD0HWovtFhcJFHcWMKbZAXe3BiLJzlSo+Xv1Cg8CnaSQrq5zjXDiUcNtOOAetWob54YjxuI5y3ath9B0yacf2dqyYZiFju08sqME8uMqfToOaiufDt/DC+bSR1CFjJDiZB9WQkD8TQpWeo3G5XuL6Y26NI/nCHlUkwwwDnoaWK/ilRYZLW2K/fLCMIwJwBym3PAHXPPPUmsZpJURiAchMNn+dO84mzghXH+sz0wckDPP4CiVgV7Gnm3ivdhZzECAygfMRjnnOMnnt3qW2QtJDGEjzKVUy7xtXOOW9AM9cdjWddB7nWJEAfagVT+Cgf0rZhVYlCKB93Ab2FOMbik7GRLp1zexyTW8ZlNsEE3lLuVS/3envkexwDg1Wj861JUKYpBlJVbrn+laZumW8AJ2oxIx6kHP8AMA/UVeXWb3yZI7iV51clSk6iZecZ4cEDpRySWwc0WYMcbtucOWePJOTwajlvPMRCqfMMk454rsGvNAuSkd7pTWiLEMPp8gRnI4OUIK49+DWO1jZLMqwM/wBmKDLzKAwbJyMLnI6fjnihS7oLdmZ7XNwsMckR+ZvaotQ1u5WGONSqSMMuQBW5e+GtUu7S0l0i2nv4fLPNtEz+Wdx4YAZB+vXPGa5G/iuYdsNxG6upO9WXBB/nWbl2NIruVzeS+a0hJJxjnmmvL5mAMg9SSeKQrkdP0pNuBkisbs0sh6XMka7dxxnPWr8Go30EUkkFwyeYOdpxxWbwce3tUkOWwh6HtTTE0bMHiXVjLEzX0+EBC5fdwfY1qw6mb6Jlu55ZXY4HmNuOB657f/Xrm0jVmI5wvTHT86txxhWJ83L9Tjj/ACK2hcymbUt6DdyT7maaQnzGJyXJ7k9c1BLdvsK7mBIGQG/z6Vm5kfDiN+uOAcH6GpFjkUncHDjBKlSTj8RW1zMlhul2Pvzu3Z57U6CdZ0Zl6AnJNW9Ot457d2kjjdXPC44wO+Rz+tWHtbOCGZxGI90pZQgYhFPYZbnHGAfzrGV7misyj9oj/ufzorVi0BDEhfW7RXKjcpjcEH0wRn86Kx9rErlMCHVEW3a2WNGRkGSxwy9Bn3HfFSSNKiND9oJMZ8vaVOSB0HPb2rHsUyXR2SGbG6MseDg9Bn6f0pi3NzJesRK3mNhSzHoOnJqeZnBLDpt2OhtlQ2Fy94WIUBd65yBnIzzwOuMVC1xHbRwiGRzM7FypZiUbOMDnG09emcGsiVruDzbdpwyMu1wr5Gcg8/iO/pVtL5ZY5o5LONhJGAgH3o3A4IzzjjkU+e+pmqLi+6Y/UHupIGTB3M24oMnkeo/PHXpVJpZ1hNrIANw3l04yP6enardrfNMwkaJZpYXXIcAhgOACueeT0/wNQX95c6i27yAAilR5Y4A569fT9Kl7GsI8vutCwG2ljc3E5SQgJgEDKjGDz1PH6VRnXzJx5JyuSSfSo0DzMittRh/Eenp/jWo8IspwskgaZRlHVc57g+h4NKzepo7Qe4mVe5hYqWjU4I7Bfw/yaupNiVbU/Z5VkYRq7McIMjnI9M981WlaOXT7eSORUkziSNIxwB/ETnJzx9eegFTWuiXlwt2ZIXWSJN4QuIyuexB9h61aujCcI2vJm6dUmn0t9PltkMJ+ZnD4ZDnhsY6Enp65rBlt0jsFE2W5JWQMd3+7zx74H156VJb2+o6NqEccjEME3+XDMuSMA4wchh6j29qtHUZpxe2V9DGxh+QGONYyp98cHp0IyeelN2kYpcj93Yy7cXG6QwfaXikjDMVPGB2f05Hf9anMM7xxD7QTISSimQvwT93Hbk9h1NX4v3ciCKBfK8sjAODI/Q8dD+PXtTkSOFLk38EEiTDYilgrRnB/i2nHc/h7U1CyCVfXRHP3kFxPK7BEBU4KhcCkkRY1RFkVuNrL/drfk0W1TT4kVC90pPnMkpbeev4YHp165rBuoYIZSsLO4C5IxjH1/WocXE6KVWFT3U9hGklSYlZSBtILAnn/ABrRjubudQAFVYMM0gYr8/bgd/8AA9qzJ02W4IMgjbhQ3H51oOklxGlxbyKCq7ZERtoOD2B5PahaIckrbGjZRRsglaUee+3ymLbQWz0Lf56VLeC2WO3trm38ueDJk2fc3DkEMD15PT047Vg2dy8EXlM222nkDPtAJGG6/h7V0MMd7rNtdTI/mJ5imaRmG5OvzcZIyOOmOK0ck1ock6bjO/QiktrW+u7qeNWYlkIeYgOnA+Y4+UA9+DjrU0kdnaWcCF58SRkts/hf+RGTnjnBpbW1t7pIprc3M18qEyxOQzOQTkAkcjGOx6VWgn8rVk+y/Z5VwsYRtzqCOmW+U5HHTH0Pc02FZzbV9i8C8kTxSSXLMFwpwVJ2jdgpyDnC9u2Tzisq5mBtFtS8Rsbk7xFGNuxweSVySD+hB4ro9J8LavqF6bq4kSyt2IJVQ7l+c4AbIIPXLegrrtN8K6Xp7tILd7mdjlp7g7mY+vOcGk9TqoYaVr3MDwPo8ltPcefb7LSUfNcMuzd7DJByOcH3P4d7HZWMrAxksi8ARgMMYxyDkflSJaqZATsP+8parzwRIB5skq4HB8r+WTTbud1OnyK1yWEWccZVoTg9sHH4AGpHvrYgoEwgHdWH/wBeqRMKqG+13IJ6ZjA/UvWVea5bWjNFHcNOR95VXGPr2/ImlqzTRG48mnRRGRVjO0Z+/J/jWJqeuIbcNbgxR9SVY5I79Saw31iKQ75J2BZigiLMQeDjHvx+tW00caZY2moa0cTkjy7BiMAZ+/Jg/XA49/SrUO5DkMNvqWq2ZnhVLWwPMd1cSbQxPTA5Y/Nu7c9aS/MEentZWM5kFvIZLmdhzM4A+b2XHA9APc0mt3zXtvMXVQu2NRg8KynbwPTAY+mDUHhq3uILK61WUBI0iIgLc+bJ6gdwvr2IHvVok2otNkk0uJLp/s8zbX8l0Jc4AA44xnaep79DUF7/AGPaQBHjkcFfmEspBUlScfJgH05qC2lvdUvFMStI5VS8hJAQc/Mzfhjnr0rSv7DTGSS4mVrp8L8oJRNwz3ByQePTp74pkmdO2m3F3DAbdoPOUbW81225xxhif4gKqqgtWeCSJlwpbBzxnOCD35BP0NX0TSLm4GbIAInmCRZnyuPm+XLYHX0qvdpObhoywfIURsD8rLgAEZ9wfxqZrQcWrlCV4xGuQTycAcVXmuIlcxNAI1UEEE5bIHPOO5/nVh4gxUSKQVBGQQMnk1Ve2cxtI0iEZx8xPXqT/n1rE20KYkRUeQQRAMcBpcnH0A/Cs+6mZ9nmRuMc5ZMLx069q0nsZZghjDsDwWCnGahm002q75pmZAPubcf1pAQzJJHDsaJ0YYJUgZ2EZ9B69R71AzM2ZHmmk+XBdjk9ec5zkVMJ0YBAvyx5JyecHGPwB7f7VFynktGm3KuwU4PtWydydiu9yZLmSPdaldu3AgCy9P7wXk/jzSbvskXlHKoQWZ2TcoJ49Pl+vJqw0yeX5fK8fK27hvpx174J6EVBFJIZMFye2V6jpStpoHqQWItcuYZ7WS64IEO9dqgHJYsqjOcdCatOst7utJYIRabvM3pIAQQP72fc8e/1pYPNhf7RIbqRWxtM9oUiyT8pDBuTweowRn0p5lCz8RxCRMOf3jYHfoGqUhtlG/jjhijWWyjKgbQTccngdQGzj3pYJA0ap5VoISoBUjJUDHfBI6duadqciTRRmRbcyNuLY8w+mOfektzHBFHsWHzSo4EYIGfqKdmK6JBMsqw+dLC+M4JQllBA6kjJ/E9ap20en294881xFOQr+XFGmCWKkL6dCQfw71N5jxgFpLZcHOBApx0yOR9KmOqwLa3EQubjzWmDB0hAGzByp+bgEkflQ0NMjkWKa5UzJKHRSfLlPl7M+vTn/PtTFjjS6jJntQEBwQu9hwe+PU9c9qknYXEufsU6oY1+aRjjkZznAHIpltF5LKTBbISCyb9sg9OQc/rTSFcnXV43wbnU9VubWEAvAIwBt6AbjIcDJA6d+xpkEe6481gsADcbcluvb/AkU21tprya5W21OBfIieaTyUdfkXlsDaM+uP8ACnRMfNVlAQbxh37e+OvX0HahIGx4EUxJSJnPQGRsAMPTGMfQk0qzOt6rXlxpcQVwxxbIXxycbkQ/TOe9VI5l83dH5lwoXbgJjnngc81Pbwo19Ck9pp0Adl3yXk7RkAnkkGQdueB9KcloKIx3uJbOZpL3+0JQNqSAyHyxg8fMB6moVO2PDtyP4YzznjgkjFWp1jlu3RIoIoTJ+7jtpUkAUnu6lstj1/8ArU2NnhhRI0S3VztaR1+Zjg555PYggccc1URSFvZ0+wtFeXr20Tk7oIY95YDoSu4Djn7xzXPpbWazD7LLPcMHBA8gIAMjqQ5A7/41sXCLFbEpbee4UgySZIUkgZwOAOQOcisG912SyQ28RR485EQP7vrnOFIrOej1Lhqjq7SOW48wIEkaNWDRnBEg4G3PTkEj8vWuN8Q21tHqKWlopihlwxjZSWjPIxzyPp7+mK6L4dwXes6ldXNxcFYEChscdSTtUdByAf8A9dUvHkhh8fKkMYVoFi+VsnkqH5787v1rN1Lt3WhsklFROdvtFvLCISyJmBsgNjn8R2rMSAsygkAt0B6mvTlkhvNO867ZTuJZi3yiPB4UD2PU9z7YqEaPbzxeZ9lVRnKs64J98dR+OKy502OUWjz6Ww8rau7czDI+lKLNVtZZWbleBk98/wD6zWlOP+Jy9pFkEO0YHYD6fr+AqG7sZIrfyQQwODn0IAz/AJ+lWZ3MYHFP3mOTKNnB+U/1qZ9OnVC4Xcgzkg9qhELMcAgn0GTQ1bco0dJnsbYNPdKrzBv3asCQOOuB74rTuvE8cttLEsZO+NkGEwMkEZznNc0UCkg5yOxGKTbzgDmpsiHBN3Y9p3k2qcYUYA9K2dL01LnTpgwIaQfKx7HsR7f/AF6xmieEgSIykjPzDGRW5bXLW2nRSE/IcqeMY4OMfr+VDdkXFJux1ng3xBfHSzbrezo1uwHlrIQAQMK2Bx0GP+Amte61fVHhld7+5yynd+9PIweOvT26V5TNqDrfT3FmzwLI24BWx+ddv4Y8Wy30qWl3bWrTDO2cW0Y7cZ4/l7VvCatZmU4Pc3X1fUTE2b25LYJ3GVs59evrSx6zqkUKCPUrxEAGFWdxjHpzTb2dEwTZ2eWYnP2VBtHHAwMY59Px9Io5E+zGVrW3dY3WMqWlG7Ib0kGPu1qu5mRi4uXuTdPcSvcMcNKzks2OOT34GPpxUrMos57iS0tpRGhcgoUGNyjgoV5+br1/TEgutPJKtpzxAdDb3DZz77w3H5Y98U5JLe6tJrHzngWRWCtJCGUcoeWDZ/g7KaH6At9zkkk09ZEma3XzeCeX3DnnHzYz9Rj2rQNukscTo5+cByXTaEPXB5JP5fhTZvDFxJeZtbqyu0wMeTMFLfKD8qyBWbrjAB5BHanzwzWoSO5jkgcqPllUoc9O/wBDSjYcrluSC6lld0SS4DSEb4lZyxJ64xu/EgVVnYxlA2cglTuPIPBx+VWIIiyGYtjaQuO5znn9K1ItTv4EULeTlV6K7llx/unI/Srd7aEKyepjxzcDAIIqIxSxXciOEYFdxOwZw4zjOM/dIH0rYS/t5WJu7G3ndsbmQGFup6bMAHn0PSqjixubwvKs9qhAGY283HHHBxnsOo6Vmrl6FRjFExby1UMOcIDnknuPc/hx0qBovOdJoDFbuv3RGHPIPXJYnPFW7uxZWLW1zBdxgbjsLK2OP4WAJOTjC5qW50i6sHLS2ssUSBEaTaSm8qCRu6Zyc4z3qroWpQNoGCAOqEt87PxGvPBzycdP4ajAKXiWTsPMfhQpznOQMf4dauJLtjl6tlGClRkgkEA/h1qvdz7bSQrIAyLnB5xgU7voxadTlrvQ9TM0jJYztGSSpEZ6ZrodLcW9jDZSx7WCHerjn15rl2vrwyFvtM/zHJxIRk10x09pTAPM2sUQO7sWO7aNxPGcZBxgHr7VlTdpNms1dK4jvvvYyJni+UBTncoA4A2njGPpViWC28pInDyXCq2X3hVYnocBTjjtz9fStf6fPZ3YRlOAEKyFWUOGB2kbgOCASPXFW05VGOPmXK/Qcc/iDWqSZm20WLsx6hdz3ILMZpXmkWbaG5IJ+bO1iSTz8vTOOaoXFjO9i8sSP5G3II5AxzgnoD7ZzV4riIY5J7Coopri3lZred4Z5BsEsTFWGfcYNNxaWgk1fUrWKpFoV9KciQSwRkH0O9jj8Y1qC2vCkZilbCISUPOecfyxXRXGpW86zrdxReVcgbzGoRty7sNuAx/Ec5BJ6ZFRReGdJ1CRBBrIgLsFUXkLKCT33JvAHu2Ky1W6NNGc9BezJeiZWYNuyDk8HOa6W+1O48UwwDV5GuPs/CEkBx0/ixk9P1NNPgjW2t4ZrO3jvbdjvWa0nSYE9SBtOc+2Kr2zJATG21WBCkehxTglJkybijRlt7O7VltHmt5VQt5cpEiMQMn5hgr09G57iqc9nexGJYBFctMP3QglBLdOMfeHUcED9DSGb/SI1BwfmBK8ZGKQOIgHjLDaw+79a0cLaIlSW7LGtuTrV3IqkFpWOCuCOe47GswBpATyG/lWnJfJ5gE6xzllK/vQW47YIII/A1UheBnk3DyS0m1DyyhSTwerccDjP07002lqS0mysoeGNsninwXLeY7ZYMQASOCPxHPfmrR0u61DzEsU+1NGu9kh+ZtpJAIThu3pwCM4yKoLa3NtKzXEMsfzFPnUjJUAMPqD1HbNHMrD5WWpWFzD5b4CgYB2LkdcZbG4n5j354znAFQSWEUt4jIFhh25cZLfOB1UY4B44JPfmnp0OTk9QalX0J7mlyphzMlufDtzZvNqMF1Bf2wYb5IZfmQsQMsrgNyW4wD746VHfQzWJjhlililUfMsiFG+uGANOaNJU2uisPRhkVqya7eLKLeWUXtskSL5d2omj5UN0bofpg8daFzRemo7xlqca/mLOu4kMDkZ/U1pKsjXsjBxsUYA9T3reDeGb07prCe0dWGWsrgkEHr8km4D8D3pNQ0jTFs55dM1yOZEG8RTwtG5H90HBDN9AKcZWeqE432Zz94hMgKEenFFtclUZByQen41IsUiRSedBJCFYHfIpCsGztIJA4IUkHvg46GmxIBmRcFHHBFP2iuJwZsaVqN3p0v2m0uGhl6AjB/MHg/jVnV/GV/cy28twbaSaJcBpIFdu397OOn8OKyY3Qxg54rN1STfNsxyVBJqJJPUuLexuR+NtWiaRopYFM3+sCWsY3n1Py1Xv9fTU4ik+m6WWZcNIlogk+u7Gc/j+Vc0FdPUDtmlAyRluRzUK3YptnWaZrOkWUSwnw5pEyBefMjy5Prucscn2xWsuu+E7lh9r8KRQgDH+jttz+C7a4DOBnvTorhow3rjvRyR7C5pHZB/CNzeCAaEbZDk+bNczso99quT+tbt/YeFNK1WOK+0Xftt0aKS2d1JDFs7huwcYHU55715xDNNJLGBJtwRhsDK/wD1q7n4lyC4vrRwzHbApJ3DafmbBGPx9qainO3kF3a4S+IvCsMJjXw9DKQSxZoI93/oRB7dTVFfE/h/zA0Phizk3cFXhiK9eTyhIPbjiuRZkXB5JdcjPSq3nyJPhImZj0VF/wA96qSS6AtToZL+3udQmmt7SCyiOAIIRtUdzgCq11eBoXYgELyOODWVHcNFNIko2ucHGcgVs6XDb6jexW106rbhi0mWwCoG45PYcEZpOXULalNlEzGV9jM53EsuSSfU0VAlxJsXbIAuOB7fnRUcrDmkYe0Ncp5UpOTnLfwjqTirS4lZph5cYQfMq9WNRQ3MUcEqtbRuXlzkfeK+me3rRaXFvBqUUlzFugAwykngH0+nFYpXdjKd2npt+I7aZVMscYCkjcdwzjvgd6fII1lUNIvuxyR9eKgu7mBtQuGsFK274+Vh045/WkDPHE8b7MZzyeh9sVNhKOibLsy2QeBbVpXhaMFmYYIbJyOgyM/5NZzSRqXG4Ft2Q/OSPerp3zAlmZh0yB17VRvYYw3yEhieBt4P0ptNFQs3Yt+dFNCpRAgUfvDjn8P8/wD1p7O5kjkhjtnbLPvU7VY7h25/GsxNwjI+ZQAScfrTjC/7toZGKxDcXUYIOe4/EUXs7h7OLumdvDKms6tdRzlI2kQuZWUl4TxgKD7kZz2zg1DerNaSeW92+6aEAsGIMg7ZGfXOKwgCYY2ErNcSIJZGl+XBBOMH0wAc/hVnWWutQi0+W7kUW+AiyqCWKnrn24yPqatS0scDoe8lfQtGKN/JN15zJbJ5hEb7ZMjgcnoD9O3aqokto4pGtC37w52O25xnPUjrg9wKgvrueORorkpK8m0NchDuK+nBwex/Ac1CsEoV/IkVkUEM/qM9M0k+qL9n7tmzTW6meC3gyuwNsyy4dR1wDn1z2z/KrtzLJNuimwr27LkucZJH8Q+mATxmqUNvPstFSdXEwOyL7xDHBJ9iaZc28lxdSu+955HxIzkgZGADnpVanO4xvuW/7R2XcX2GTykRgu13/dng8ZyMAZx0HU4rn9Riltbx7VwUlBIZmfOc4OevQjBz71fa3XzFsZXW3kYEqzqdrZ4AY/w9+cd6uWfgXXten8+PT5IIBtAnu3MYIA4wGG5h6EA8Cpld2O3DQS1RiSI5PmSytJGwKknufYGtSKyiksoTHIY45VJYEhyGyB0JXJIGcLmu5074c2iIseqao8uAWMVuAFBxyd5Gfw28/Wu40jStL0sA2ccELsud5yz492bkHk8YBpqOhtKnKTR5rD4Q1bUdFsra0tJmgjGTJdR7Nhyd3J+YDnIxmr+lfDC5E/2q71m3t1+6UiG4gcdGbbjp1wa9Nk3SIGadGB4B8wN/WkS38wfef6KhzTtrcccPGN763MPTvBugW0oliMl1OrBvOluDnI74XaP0rYTSdOgP7i0tosnJEK7FY+rBSAT7nmo7yKG0iZ7iK4WMdZDE4X/vorj9a5a/8UaNaA/Zy9y56bQQPxJ/pmi9zWNOMdlY7OaOCKP9/DBGh7t8g/MmqTz6em4eYGKjpE9eb3Xiq9unKW+IQxwqxdT+NWrbUFsrDdJOgOMYPXjPFWordg21sdRdeIbOEZWOXAPJdxnH4CuUvfGF7KGWOUQoegjGD+fWlstD1bWoI7mG2KWBba93MQkSqT975sbh9M1tWHhLw1BNGLu+uL+baWP2fCwnHbJG78RT5ewXtuccmt3jl/MkkkXh0JcnB5z/ACFbmieG/EWuLK0MD20RjBEtyhRW5B4JGe2cgdq7E6vpOntBbWmh2McYJK+YvmbTxnBPQnH6Vk614p1K91P7D5xitRkbY/lDYppNickiK3trbwjqEqrKL2/8vaLuQARQMTkFEGSeMHcT9BzVTVdSl1dxJOx8ySLAIPUg/wD1sVkNcK2pS/aJvkWT5mY9u2a63TLKDTLEXsoMd3GHNoGH3QejEHoc5x7c+hrRxSimZ3dx8dtZ21hE+oR+ZPHHumgc5TdkrggjJPzcgnHBzmqk11Pq0EcVsP3kji3jjX5VCkYVQBwB09utZF69xfarIcTTNKociNS5PYnA9hmui062bRLI+aQLt8K2CD5S4yRnkbvU9OCOaHa1wVye4vVto4NPsgqwRyrHkLgyHpk98nOatXGnIRIuoO6RuwYRRcvwDwT/AA9fc89jVWxV7WznvXx9ruYy8K/881xww92/l9abA893YLc3bLBEJMiRgTvAx90Dr+g96Q2QSy6ZpvFvpsRd1KEyO8mAOmAzHHGeetI80L6ehgLKolBBJ+ZODkE98cVPfX1jDbCddPilbd8rSsx69yAQDxmqqy2pjARQiyDzGj3fdbn7ue3ysO9KS0EnqZThtrOrdDtx9c1WnVVRR8wfndk4z6VpP5EikbQvBO4k1WKRuzOI3dguT8vAHTP5kVhY3M3zrlHIjkfyouSAduRn+pNQS3YdW807f7vGa0JAY1cbVUdGywGe/rVWe1226OVXbICyHIO4AkHp7g0ugzPtTbx3BM6l4nQqDnDJnjcB3xVm4iZyiyMAybef7wHT8Mc1FNaQNKohRlYkYLdqLKeNrd0uCBPCP3ZYf65M5KE5+8M5B+o9KqLFJCrbq7NyCjDBX37HOOD/APXqHy8uRmPI4OSePYgc1dhlR0MluQuc5yB09x/P061Sjt5XvLtURt7KpOzsdo6YJ3CtNiBbeLy5XaJowMlWMMokOD225yecHn0oMscCh8w7wMENCpz+lQ29vMMb4bJiI+ZVwjr6DnG48e9Tz3JWUQyS3kLuWKhotwLDBwWJGDjnoaS8ymULqZZZ7eQz2pUrmQtbKT1HABX06U9JIVCILjMITBZYADjtxnj1P1qG6nBnj82+uREV3f6sOV6cAFx/MVaEgQKi3kzRjBLOuD0/u7iAfx7UosHsLBclLlXt7yRMn5ZEjw6gduD6e9XZNcjW1mtV1TUJbmSdT5zryqAMCmd5PJYZ+nSqKXcV1IZILudhE5GWTaSRj3/rVdr+7EzpJqEyxO2TE4Dq+DnqTkdB2qnawky6okLuFhm2lv8Alp0P6f5zUUkDC7jZ4okO0kKX+YYGeVzkZx3p01oJpPtSWd+8UalmWM5JAxyG2Y6+3pUNtKt0Eu7a3dAyEbZ3BYcEHHAyc+lO/QVhW1eWzaV4p9KjcKyMqWaByGHzKSI+cgkHJx1qSKFn5VXZlH8ZxjvnsP8AJqS3WXe7LPBaM5LKBGFc8dCyqTz9ceuKJFg3q80s8uRltxCEnPHJznt/nmpjoxy2IooXaTZO8KRbD8seHIJGO3H45zSpp1o6SBdON+yICodpMgkgcCMrxjHXPQ+vFbS7K4ijuI5IGhVJiMXbfMqnkHBG5h3yorQiYmJlLFS2TjblT+uabSaErohhhFvF5n2VYBIclFbHGPclvzrL1GOI3el+W00kvnYGOM52j7vOeeOvOa3vswmQ/I8wH3gSTjHX7oB/OqF0su6KMXKwIHXdCjY8xQfReD36kUr2H5kOtytaRSFmiC5JAVgQD1wCD1+n6Vx5s2JMkowW5xnnHf8Az712ep2CyXsJdWFrAgmlL8g4OAOB0Jx61g6i4e4ZgDuaQnB57nr61lN3ZrDRGfZXd7pF0LrT7hopcY+U4z/T/wDVTrnVrm98RPq13bebIxBdchc4AA5+gApvygc9CP8AP+famMwGM857f59aiyKvY6LSNa0tdTZpVWNpsMFlUZifJzgnsRjnOc/WuklumYh2KIg5X5wcj1rzGVUk4b6e9MU3druME7omMkK2AR9Klw7Fc99y+0gPit3VwoNw+GIJHftXRTaEgw8l5Mx6sFCqCfbriuIiupI7xLgqrurbuehP4V0X/CWmRNk9qy8feRwcH6Ef1pakWfQuappot452QFk2EfTjvXL2DhLlizAAoRk/hXQ/8JPaTxOrl49ylSrL6j2zXMW8kUUoaZSyheADjmtHUbs+weo25Km5kKnI9ataVkXZZCvmhD5YboT/APqzVGR98jNxz6Dim7irZUkEHgip3Bo1tUdhBDHMVa4ySxBzgfTt/wDWqi1y/wBiEG4lS2QM9Ov+NVsk80lFgSsOAyKdG7xOHRiGHQimClzzTKO58M60b2WO11SSaaJdzIEkVW3cZySpJBxjt7V0kj6ebRo1nukzhsLCsgLj/ga8YJ/+vXnnhr59Wij3KnmcbypbbwecA81250+7SUiGS2uBg4ZJAoP4Pg1rCWmpjOOpaiihMkbQXqskm5C80TxhRggkgBsD6E9R+FpNF1Ty/Ngtluo8kBraZH6DJwud3p27isy3ttQitQ0lhdLHkkMYm2nv1xjvVS9vVKpl1ZQ5BjY8I2Bk+2QF/L2rXm7MztrsTyQy2czW86lJYztdD1B9KtWmr6nYIY7a4kSDnCcFOeoweO/P1pNO1eeZwjRx3ccUeWS5QShVHcE8j8DVuS60y7VIpIGsY0YsDAWk5IAJKu3+yOhHQ9ezTvpYT0KUOqQSSO91p9tKpwCIw0HGQTjyioBODzjvWpG2iXHmszX1qTJiNF2zIqccchWz16k/jWLPaQRzJHDeRzxzHiSON8INxHzZUYPf+tWJbK4tYVkuLaVIycLIy/If91ujfgTS0HqPmsAQZI720KcH55PLOT2O7AyOe5HHWoZdH1MdbGdkOCGRS6kYz95cg8e9Nli/cmdW4VlXH1z/AIGkS6kttssErxuucMhIIB64xRZ9w0KUxdW2np+lWLO+urSLFvcyw5O4mNyvIPHI/CthdfkuCkmqWNrqQkUktLEI5TyR/rFwe3Xmm3cGg3EDNaTXlrKGyI5wsgcbv4SoG3A9c/XvVJ90TZdGZcdy6zyz5WSWVWWTzlEm8Njdndnnjr1GTjrUE1lYXEQjxcQsU2llkBDMB6EdDx34x0Oa0zolw5xaS2978pYLbygyfTyziQnHPC1jajE8cRVkdJ4zu2MCGBB6EHoaT5WNcyMNdKETs1yk/wBnUceVgtn8v6VosXW5KAPtUhfw7fpVmPU0YSMq/MoAUPyCOmPrxVqyuLKSNo7i1dgR/rIpdjjrxyCD16kE8YzSUeXVDcubRiPGBlAMfKoYZ56Dv9RUe5FZy0CktkKy/IVznoBx1PcHpgYqS88sXjyW6yeVK42LguRx1J9Sf1NOjD4YSRshwrYZSOCODQrPQG7ajWjXKpG5yc5WVTlRk7eVyDkDk4AB9uaLbSr+5gS5hsriaNzhWiQycg99ucdM4POPrUgVvtZK8Aooz75b/EVct7y6srgzW8skcv8AfRypP1x1+hptS6MNOpy90XW4MTEhkYhh3B/pWtbz8AE8jpW4nin7WfK1myt9UgwQ32hAkw9NsqAEduoJx3qzDpfhy+V5rO/k00qpYQ6hhlbA7SqcKP8Aeyai7T1RVk1oYcGrXOnlzazvE0xAdo2ZW+XOOQRkcnrVw6pG9ibKa2gkt8kqpQBkPPKsuGB5PUkc4xjiqmq6NeafMrzwgROu5JEIZWUnhsjoPTNVkYrGpcfKfzFWknqQ21oXfs9lKFeBzFKq8LIcqxCgY3DkEnJ6Y6DjrVRNOuLUCfy2MDtt35DISQWwGGRng8dRg5xUcJ+Y5GcnipZGbYA4cI7AhiOMjPPuRx+fvVcr7iuivdLtUOQTgHH51Fah3lBVQeN75OOPp36irjROCSvlvCSWZdgVm46DHygZ9u/fgVPDZQ3VzLHayMjpEgijnQRvKxOCACccAZwCxwQBnrT5rbi5b7Fd7uS2lWSKVklX5lZTgg9OD+Jq9b+IbuO2MV00d3bbt5jvR5gB9QTyvQdCKypkLSKkhAYosm3cCQD0zjv7diKwNfu3VVtY84+85/kP6/lSk42uOKd7HaO2jzsGg86xZjyjgyxKAPX73J+vWo47GWSF5YUE8K8M8R3Y4ySVxuA9yAOD6V55aaxeWe1VfdGONjV2FhdrdQw3MZKnqMHBU/X61nCSloi5xtuXgPmBVsoR2PWjVIvJvWjjyCmEZRwSQACfz5/Gorma4nnWWS48w8kgKN3ph2xljwCMk4yPWtKaLTdQ2yLd3dpdHJY3WJo2JPd0AYev+rI960u09UTZbIwooW3MGc46Hnv2qXzRHhY2PQnd3B9asaxoepaIjmQxTQvhhPC4dCD05HI9sgZqnk/Y1CKd7cdcjH1FaQknsRKLW5NbanNneJmjZlKkoxUkenHUV039sWd3Zxy3+h2s8bN5LTwHyZM8HkKMZ4P69M1xEdvIkbsxCyKQdpPUd62HXGi2TLLKHklk3x8heAmCO2eSPyrKcU1sVB2ZuPYeGbi6aO11ObTg0h2JdR70247MDkc5+9XNNod9cTzywyW0sSMVTbcKGcA4ztOGx06jv3qjrRl/tAlgykJGuW74RR+XHX8azstLOZPNCnIAw2AKyeiNVqdI3hDxLIgaLSnYlcjDoT+Wc1Ufwn4ngB87RbzJG75Ii2B+GeKZBp92GH2Zgzf7Eg3D8OtXYjrmknek9wjn5SoyflPv0x7VN2+o9DNn0jUbVj9o0+8hGP8AlpAw/XFVWyvDYX61vxaxq0LrJ9sleSKQSL5ru5DYIyN2R356ZrYHjnWU8pZrZpEBBZY40O857gr36HH4YouxaHG2dyRJiOMSkHoqkn9K7jXJwx022mBjaDS4SQ/AHLA8+ozg/Smv8SL+1llKaPFbxsQQJFYFW246jHcVzerX2o+JHinvikqrkKViVeCT1xz+dXCTUuZkyUbWILu3hjDSCZTHnjY+Sw9qhsrS4kkWKS3aAvh0luB5a7cEg7jgc9vWtC3adV8qBJUjCBGwcZA9OayLtmjCkodhJ+VgRk/41pJ31Jj2O70OPw5LoMljrcUfnB22TAFmAI/hKdMHtnBz+VTSbLTtO1+REubG8tDHhftiOmeMNnjHALHkHjPfFYtudtvGoGAFH8qkE+zhWI+hqPZ3Q1O2hl3ZhjvJ0WeGZVkYCXIbfz97PfPXNFXyZCxIkAGf7tFbXXYfOchKjy7HSMRsF6hvve9SsUOFw5UjLnaCVPsagN3vQlmJfjheM05DLc+WW2xqTt3dgeeuK8+1wtpqLDBJK8ohR3ABb5RkgA9fp70P5kjKs2QqcelXfssliXU3BkbbnjOCPwINK13cJYwloY/MiJG4FgzKfUEYP5/1pqxDld3WpGJ4Li0EQ3jYMZ4z7e9RRNE9xHEVDeWMBQOSx6n3qC4vJJYyqwCOIsSqgZ5PvU1pawyLEsqvGRyzq2WPtgdO57/Sjcbjyx1IFdmMgSVkTJIwDhj6Vdjilj2xyN5RwD8wJOMdAPTpU7adBNfiCxuh9nEYk+cnf2+XoOefTtUdxcSWNw7NCN7ZAVwMrk9Rtx+n/wBalqiVOMnyrcnja1t7Z2kUTOrsi+b83ycYwDx19OmKkY2lxpkbweY8qZXy3yVjySeD0wfT3rNxIF++GkGWOCNoAPJJ/H8a6LRvCuranCZtL0+5ZJUwXkYRISf4gz/e7/dzT6EzpuWvUxoJ4ryYWt/OURBtjYDhCPUDrTrZ2gjlt4myhzhsZ/H2r0DTPgpf3Mgkv9QjBYfdtYWkH/fRwB+Vdvpnwzs9MTcmnSzPjglkB+oIII/Oml3KlRvp0PFrPR9RntkmtFYKW4LYVT+J4rsLfwzrWo6it1cYTeAHjt7dnVhjByWAX34zz9K9ZttPubYt5OjPH2LoFBP5dae8N8SAbGUADJDR5qk7DWHhe8ldnLafoVpphScRKl4qqn2iaNnlwBj7wBx77cZrQLI+/fcbgOrGKYA/T5OatXSRxuDPauhPcqwA/wDHaz530+PALRDcMjFzzj8qd2bKKSsh63NrGwBu4Qf9pZf6pQ11Cyl0ubdu3DYP5ECqU9xFEu5Un8vg5R1OfzGDVCRpLgArO0UXO4ghmH44x+lNJsG0jWutQFlbG4laPyh3Qh/5fhWVHr5vAXt1wm7aPM7n6D/GoL+yhmkMHmbkWENOd3J+dQM/i1Zn2O8tn8uwxN/cjwWbJ9B3rSMV1M3JvYvapezQWk0c8u5iAQoOBg+oH0NKukQPeXluluSomeOKM9Tg4A578VqWvgW6kP8AaPiS8SztOHNvFzLJz0z0Xr2z17da1NQ8VWFhNcy6Xp9tDcuC73AjUyMSSTk/X1zTtfYTdt2c/pvgWcSpe3Cf2dangeYcykkfwjoPxPHpV7VNc0nw3cQnTdKt5LkqEa5lHmPtGF47A8c4pmhaxea080t1cs7eaCQz/dBB6DtyB7frXM+L7drW+jDcgk4P8ql6NJlJnQaprt/q+ofZJ7ktAgVtmAAD26detYlwz2OpLNA5CkHC9skEVEpmimtrrbK4u7ddgRCxZlyGAA5OAVP41tQ6a0WqWk2tFbaBCJfs+VZzg8B88AHHQbjjsM5rZWVjLW47WIFTTYrmPORIrkHqCe38xWfc2l9qt3DLp9s0sigI+3gDIGCSeBXRXt/osiNt0q2liZcFTCpDYHbjOfpirhOy0SKAmOLaNqKNoA9AB0rNe6XuZ0ejWGgILloY7nU1YGSdjuWMnsinjI4G7Getc5datIZ3di0jIThSeuCeM1p3l0Ibp4ZQ7rIAoxyQeo/Dmp9L0KG1vFu9YRdzAtHaPgkN/tgHHvj356EG13JfmECHTLbzlcedMPkO3OYc5H0zw30xVrTrR9TnZp3YWioC8mercjaPU4/LHNMvXkvr0Su5RfmaR9u7aowemfw/EVpRywpbqlqu23GdidevUn1J9aG9LiW43cLjVpJ5QEtIkzt7HPCoPbj8hUFw9xf3DJLKFiWFgXc4WMZGP/rAUSRzG2tNNAAuCqmQkn75UZB9MAc+mDVXUJZLm4jsbGNvK3HPPLnaQWJPT+QoSC9xst5p8bLZ/YZLgJzuldlyRx91T056HP1ptxLAbguIFjZUCuik4Kt0I54w3GPcVZ8i1t5gH/0i42luGIQZ4xwMn1zkf4xXQMjBkhYNtKEoQF24yTj1y2Pp9OW2hWZTmmM/mzOhOSRnGBu69fzNUWZwgBnRFcdA3zde/pWncKI38tlE0RAYAoUb2z8xH/jo+lMa5e3jOIJGjPRWKFRxxyFz+tczR0JmZ5QdYy5KxjksiEsef51XNrOSG2M5BGOp46+lXEN1czgWpaEyHBIZyq/mSMetIbC9cgvbL5o5DmCNCT6hsZx+NSVczJBK5MYWJZXf7zMAQOc/KT9O3aq7afcYcSvnJ6FCM/iRj9a62CwvJoitwglzxyzNz9eabPps7BlZVQk9cHP5kf0osFzi3uJIbiNVi8xzhWRCu58dOhPze/v+ejF9nngeVEQmTaPmOOfQn+FuenTv0rXbwjDcvukmWNj1fG79ARVh/CyCWOVJ5PtBGJZQ2wTr/tAhs9ufaqi31JdjCltlhZPNO0HHyzqclfZxyRx2wKz7uAHUrGSKElgzH9zKGAGPTkg/U121v4ftWjYRXk8aZ5TzA6N7EbcfnnpRPoWnRKsstsw3Zw43Dd+A4/ICrvYRyUdrMsj4NzFjGQkZP4nkUotpQTtNwrE/fVTnPuP/AK9dfa6ZZoxZbeWXJzve1ZsDn/ZrSNvBanaVSBzgqsixxE9OgJFK7QHlmm21zNDeQQGVk+0t0Ukt064rai0PUvJUR290SR90wFRnnvmvQPtUixgSu69cHy2YfmgNKJZpFzG0jgDny1X8/nK4ou7AcCfC+qShlfT5iSpUFp0THHvTtI8Faha2qJLawqw3D95c7lwSeu0A556iu8WO6bk5YdCshCn9N9Kbe6yvktbQcfNvjMhJ9ipTjp2ouwOTHguUsrSS2qOpyNm98fgasnwfj/W3pw3VY4xHn37j9K6R4Z1OJriYk85hEZH5NkipDaqwy5aYY6eY0bD3+UUk7AczY+AdNs2cie5k3MXZGdfT2XNbKaBo1oFd7W2Xb0Mp/wDijzVs2e1tyhW5BXz2aQr9NzGpIbRIWzEUiZuMxxqv9KGMgWHTXYyJBA7/AN6KESH8wCadvkUbYoblh6DjH/fWKtkDbh5CxHfcf/ZeKGaQqojdYz1OY938iKVgueZeOZmGojcqbvJRSpw20ksRnBPrnHfj1zXnk6s0kjAgt0UgdugJ/wC+c++a7fxBLBq9/Nfb1zIN2GwAFAwAQe+Nu4AkYGB1rDe2WZXjWIMkRy3Te2ceh9jz/jgZvc1WxybyEds84A/TH8qjdgS2Txjv6cf1q3cWE0ahmG9S2G2jp0/+KGPeqLI21Cc4Pf39vzFIBUbJwT14I96uwxrMGjxnahYAddo/+txVMKVZTjgcE9q3dLQ25SQA/u5N7g91ByR+X55FMRk3+lm06HIUKWPbnH58n9PxqPT9Ne9u449rbdvmPgc7R1x745ramDXEUk7Nma4ZSOOFHzHPHrlfxzWharHZ3uYVZljQfKSAcED2B/Lp70kh3MnXfC5tI3ubU7o0B3oMnoTll9sevo3PSuYJI4r0i3vGicGRPMijZjynfvx1xgdOOMelZF3oOn3b3LwsbdI22pt+bcSf15I9McdjwBfucZ1ordvtGihjBiJBBwSTx3/yayHt5FJBU8cnigLkNFFFAwpRSUooA67wPZrfXM6KgM0S71OccHj/AD9a7hNFvwd0bCJ+2JMH8K868GXz2PiW12yiNZ28liylh83AyAQSM4/+v0r2M2E7hpI7g3CDr5O3/wBBILj8RVRIktTDzrNrgPq1zHt6KGPH4Grcl+1/F5eoWEGoJgCSUwfOB9R0/SrP2JntpEEjeWx3MjTHB/An+VUpNOiKfLIuz+6qsT+ox+tXcmwtv4d0ee4uH0a9lguWjKm3uBgc9ccA49uaz7rwxrlrkPZeco/5aQSKy/zyPxFXf7Mt2OGaV0/u7Aufxycfka17S+itYltYMuyqMJM4kZPbIC/lTT7CaODZZIGAlR4nPRWGDj1rU07UZ7Rv3c8qHa+CjH+6T2/rW7dhLtibqO0lXBXaUwFJ+h69OvpWK+l2cYZixEgPy7ZDtC4IIxg8nPXPHpT5nsKwjauboEXNvbTAn5iYgjNjplkwT19aQwaZNatMftMBV1RgpWQfNuIIBwf4fXv+FVYtOZ2Cpe2q56iZmTHJ77SPerEun31vp1wZbWVx5kTxvCDJGy7X3NlcjHK8nHX600xMlGnxTWdsLW9t5JlBQwu3lEAsSMFsAn5j0P0zVK5tbmzm2XMEkMmOFkUqT9KoG4Hl7gwwR2NWNO1S8sovIguXFqefIPKEnvtPHr+dXe2iJt1Zau1EcFk3Klrctkdz5jj+lI2ovNbxw3QS5hj4CzKGwOMgHqBxjgjvV19S03UEjS+szD5SFY2siE6knBDZyNxJ4x1qC60RjA82myNqFsmC7xRkPH/vJyR9Rke9JvuNeRnNodg0C3FrJ5G9ipgmYsm7IJKsBkAAjhv++jUy6PPBa/aGhcjdtV0+dCMEnDDIPT1qO5ONItjG4LLNMMZ74j6/57U20ubhFXE0sQJ3YVyATgjkdCcHH4kURTtoEmr6jlZkAx37VYgvltyGmhWeMgp5brkDIOCP7vPJK4PvVmCfTry4VNQSS2DAgT24BGd3Vo8dAP7pH0NTXnh65WNpbGa21C3T53aCVSUQdSyk5BA7DNVpsydd0QxR6fcEYee1YknzCRJGOThSMBgBwM5Y9eDU02kXiQmZI/tEA6zW7eYg+pH3f+BYqgsfyqBkHFS2t5PZ3Hm28skcyj5XjYqR+I7fXIolHsNPuNsIgb23JQSL5qkgjIxnp+VIqCNSEZl7cVeOpRmbzbixhlm2jM8TCCVm65O3KHrjJUk4zweKsQaEbm1MmmXiXcmWxZylY7gAHjA3EPxgnac+gqL63ZVuxh/IqmJol8s/eCqNp57qQQencVPM0c6SOYCqojPutY1XBO1V3KflK7icnIPz4GAAKfNZ3UMrRXNtJDIvJWRCjY9cHBp6xSraXLx7NmwLLu64LAjH4gVdk9UTdrcywUZilsXwdzbZkEbYDcY5IYkYPykn27lk0UwlmhliaOVX27JAQRjGRj86leNSvzAMp42kZH5VPZSW6Oxu7dbtCmzbIxBReMbSuMEYAGcgDIxg4qrSXmK6ZBCVjVUyMjNPkVSwJwQRtwa0rPRNP1S4WKz1KW2mKfIl3GHViAS3zrjHAJ5XtjNUprK7gRXubV4kVBJuJBXaWKg5UkckHHPPBGQQS1NPRi5XuivMN0Uq8qXTazgAsBz0JzjqaydW1GxvYIUbTLeK5iQK8kS7fN/2jg4B9QB1ya159jRsGbKnjjvXK35jEp8tcBeMVnNK5cNjO1AWpCiCJk29Sxxurd8MsTYyDBCiQhcnPYVzdz8wzXU6HGbXTYVKKwdxJIRnzApK/dydpIUHGcctWMdJ6GkleNmbF280NwFuFYK0UZj4/hKKRj8SfxqJZo3Zl3DcOAo6moiLnUr0A3kUkuwJHHJiFlVQqIvzYUkgDoSPeqf2WS1miaeOaJ2Xf+8Qrkeoz1HvXRGorWMZQ1uaCCFbwNzFcpgq6Eqy9xyPzq+1wZbyK4vUiu0XIZSojZh7uo3Hr39qzSAbuVlYSRiQ7HHTAPHI7YxU6EsrZ6+ner5FLUnmcS9/Z+k3unxyfb/sd0pUSRXgHlgk4yJV/h6cEZ7ngEiMCa48P2PlxLNCJ5mWSF1dtu1c7lB3KBsJyccYqoJjHIAOOxJHSrUcnlnMW6NnRlLxkoxB6gkdc+/FTKnK2jGpxW6Oe1iRpdQkYRtFESCkZ/gGBge/GOe9Zsu1lBOQFOTivRft1jqDRxa9p9vc26Hc9zbRLBcD+EcphWGSvG3ilTTfAd7a/wBm7dYtDOfMF5cJG3kMAQFLLxt/T3HUc81JaNG8WnqmcDBq0tpIjRbTtHAYfqa108cagQY9kZRuobJ/r/Krd58MNbkkZ9FeLUoBysiSxKzjvgByPyJ7fSuUv9K1LR59mpWFzasDgCaJkDY9MjnqPzrPmsXyo3V8YXKTbmhjEROCiKcn8T9BXQWfiGzuUiEiIAxIJl5Cj34Pv7V5zNOpkQ4BXuo4zUizbzjdgY4Ddh/WrU2mS4pntFpPbTwI5iSVG+6YT1xx25/SlNnpU7bWR4yo6iTkfT9e1eSWF81m6zISQRh1BIGM9D611CeMbW3Plra+chILuSc/h9P/AK3HWtFNdTNwfQ7I6NYgjZNI6L1V2DLj3GRxTE0axKv5gWXthIlUH8iM/niuXPjC3M4RLeQx7c7/ADc/oR/WryeJrJlDzyExDoS+COn1rROLIcZGFcosN1PEi7QkjLgjng45qswYPnJ24/Kn3N/FqFxNIImhd24MRAC++CMEnjuO/rWlb6G9/DI+nXaXDA8W8yCGYjgDGWKNyT0b07nAJVI3sNQe5m76KtPoWtK7K2jX2QcHEBI/MDB/CijmiLkl2OATcr79m3Bz1p89xJJzvy0hBwG6AVXK4BzgEHoKahwx2qG9iK4DeyLqXMcTOSd5OQAvFWLW5XDo/PmDgnPHsP5VSjtkkx8yqx7c5q/HZ3ExS3VJWmIwkaoSevTHX0pozlGLIHf94POQojYwIxgccZ/SllZN7MA+B0ZQM+2f/wBddbZ/DzX9TjTzYo7VQD81zJtJ9Pl5Yfio6V2GjfDrTLBT/adxLfnOfLSMRJ9CQdx+uR9KpJhr0PPNA0bVNSINjBM2W2byNqA9vnPAPtnPpXd2HwqknKvq19M5J3MImAwf95ue3938a72za1soRDa20FvCAABEmD+PHP41aF8rj93NFGQf41/xqrJAqavzdTP0rwb4d0eRZbXTElmXkTTneQfUf3T9CK6FpGJyiRI/qEyfzIJrPWeUvgTxO3UEgf5/KmGe7HV42H+znP6UyzSNxcMMSEsPUjdUZcN91VJ6ZaPFUmkGzzJJpEI53blGPzSqc2pW8ir5F1cTswyNjRlD+Oz+VO4Gu1nHIjExwv6koBWXdjTrNC80duvsEHP5ViT+JbgX5sWm8xfJDtiPaY2JIweuOnrzkdOlZt3qMVvAyPIrOSRzzzTsxN2L8viOFiFsrVURDzwEBH161zniKa4n02eUzs74SVR/tK4yR6YXd+BPrVC41dXhdAOCSMjtW34c8J6z4pR2kV7ewaF1M0qbQ25CAVyOccHjj3qlYNdyX/hJreaCNdjFwAGRR06cj9adHBq/iRja6bp7QwS4zK+eB356Dr+ldJpvhvQ/D1uf7TuxfXAHIhQKgI69TzTNa8XyR6ebexRbazK7cKMk4HAJ9xj8qb1IukWLDQtA8LWsiXk4v7uVAksakMPvA/zA6+nSs+fxyNMntotKtYLa0D7mSNdxcA85J9q5BdTlnY7pc5IKr+VUmxLcsAAcY98elXa2u5N2zq9T1qW/tluZXkZpACQWyR7fhXK3V0xOCSTuIwPStOaNbbyrRpPMd4xhUXJG8ZAA7nDAY7mtGx8A39xIbrWJk0u127mT70zD2Ufdz6nn/ZNF0tEJLqyDwjcRw6xNblWy8I2qoyWZTwAB16ngV1Gv6LYaiY31O9+xIrA427nwSBjHbkgc8j0pbc2un2yWGmpJBbjkKHO9yf4nYcknHIzj24FY/iaUpos7Kqh0XeMfw7ef6VMtdykzZfU7LRLCKx0lZFhjLfvWfLsTjknGR0Hp0FcRr101xcqxA2qOAPz/AMa2JYlcJhhtZA5P86rDwxeagBPJIbWLPyGSPPmjnJAJHAByDwD0BOeGJdx2gyi7tJhcniHaI89DnOfxxiuhgmka3Fv5cj3SRhXjRckMODnsOe5wKdpEem6NEIba3865Y5Mlxhip/AAdgePzOKnfVJ7a0lY+WiEeWQqgbMt2x9aJahcittPgsvtV9PulvipaFAPlgbA4J7txjI6dvU4N1qO6/hMjADPLe2RV68vzDGrsQOccnrxUtlobW5TUdRVQw5ggK5/4Ew/kOvr0wWu7EzRktXj0SVM4aYbyO4x0H5Z/OqdjKLa0SYlPMAPkqwz0OC359Pz7Vea7UWss07gJnBJ98f41mN5l1JGgThABEq+mSMf59KcdbikiK0klis5pzjdI5gDZ555Y4+mB+NTSNHZxIFVvNdQ7yMfuqSDjH0wSfwqxN5TK0ECqUtXLuTyJGABPp9B6dKLTT21yyluZLiW2GS3nxrnaQQc/pjpik5XGo2Fmls9KG6dRJdyrvji3qAg7F8/oPzrAWS5cytLdnLMDsXaf5YA6VX1zT9V1DWJTFPc7cBVeHADjHXJUEnnv6Y7ZrJh0W5tWMX2nUw2STIskPP1JXNSmirHQLIobZ5gLe7KP6mrZheG2WRinzMejZI4B5GOP8+lZi6E23dJqM0oON26Z1Of+Aqv86tLoGlwIvnxyPNjcjSySMNnQfecg/l6enKbGkOF/bq6o0qZJ28tjn354qUavZrcNZjyTcxuVcI/mkHpt2oWIIINL/ZGnHBbTLYY6N9lT+q1pKCbVQskpCfeztCAnpgDpSGiusr5+WzmLEZDJZyD9WVR+tStcXiIjyWF3ySBLPLGgPTjALHj6VLFhgHELOw43K39MVMZnKlGmhQ9UBAODx6kDpS1ArC5lcYbyQD2ErOf1VP60knnsXk8xfLBO1miYkLnjLb8d/TvVltQ8kGN7mRmBwwGVx+mKjN+pBWM3DE9ScDP5UAVwz7vmlZ88YVUH67c/rUw2mFlk+0b/AOH/AEuRfzXcM/lVhZJXUBIZ93QqzNg+9QG4MaEDbGw/iEYf+ZoAgk02Ey4kskdx/EyeZn8ef51oW0LozEwth+X6KWPTqfb2qpFqF2ZQzSCSMDBAQA49eB1qzHfTCMkRu3qSTigCYQSA4CJ+ZP8A9aphFKF42of9lRn881VTUZCCGRQf7xbj9KT7fyRJ8ynrhM4+h60rDLyRfJtkYyZORkjI/Lr+NIzQKCGVMD+9j+tZQEbSFt87KTwCP/r1J5kCDHlvIfdx/SnYC/51rtK5Xb6beKVrqJiAGZj2xWb9st9hUwAcZyp+YU1LxRPhRIewDYJ6e3+FFgua4cY4BGe56VTuLxosFpYUHXnJJ/ClBlDcrhe+8iq8htEQmOCJ3YkbmXbkHryFNIB4v0YAhGlYDAZYjx+ZqO61Se00+5u2tHKQwvLy4X7qk+ntRFKIVAjibJGSgbIB7BSRkj65NYfjqaf/AIQnUzNE6DYmOP8AbXrQO1zyy11Mz/LKBtOc9jjv9OpxxwM10dlLCLJriRYwnDNvbGWPTC5z6HPY5HQccFayASLubIJwc9P/ANVds7RPpVqvnLIJ2CyKrAnAGDnB7bj+I6nHOJuZ8cUdyiRTO25gWdQM5YLn17/L9OaoXOlplgoADMVH1yf16/r9akRy7K8mMM23OQRgkZwPQAr0q8iSPcRxQECUyg4EnAGcD8mJAqiTGk0+SBYmmUKpVSdwyBlO/v149vWr8Tj7PIhURvKm6N+BtGWwzZ7cD8B6VrXKxapE7ZESebtBBIKxj5c+4BH161l3izZCQ5E5+bAUAEY5GPy+vP4oDGguyhjaRSjwEnaRxkHPTuecY9APSteOUmLMsu+OUkMVJy44Lc8Yzhj+PtXPFDLqEURVuuw9+PXp6H8gK6eO2kt7ZXkBG6I5Hp0A/QDA/PApA0NkeSUI5zmc/vEwAASOgHXoCCPWli8lIVUuCWwCMcZIPX24H4gHis67vRJMjsQsYLYCdxkHv9f0HWqsN2zXMbSNtVcg4P04H6EfQcUxGksJuU80plCowTnLHBzx+Bz6AfSo5dJd7XK7SwJZ2GTt57Yzn+PIHcA8jFammEnypSzsfmO/bkJu4Pfv8y9+Ce5yNKOyjZIg5THm48sEYCAcgj0LsR25b3oA851PTpbGXDo4HTLDqec4x1HFUK9LvNOhvLeSK5yivsjVs5Y7SBuGfpyR3zxXnl5avZ3DQyDlfSkUV8UoFFSJSbKSHQAmVMfe3DH1r6K+y7eGST+X86+d4lJuI1Hdh/Ovpd5CY9wQkH+9gH8jVQM57maYlHIQH6n/APXUIjwTnhe20Vck85nIQQgYGCWI+oxt6/jj3qNop1j/AHx2jON4TAPsDyD+FUSVpLISp+7nDk8FD8jf5/E1kzWqqw2BAyHOSMlf8K3GUHGDyO5psszOixyzsyJ91M8CmI56RMffcZ9ScZqCSEspYEbenWtqeS2hb55EQdMtxUMcf20FrSGSdFO0tGpKg+memaAMH7M0rBXeOJTxucn+gNTG3vtOLS2F3NLtGZHtvur9eT/48B9KvTW3LBkZWDFShIyPrj8qrvFBEAWLoxGdwQEZ/wC+h+dCYWIv7WivGVNW0m2vHznzc+TIQPVh/hWfe2ukxW0k9nqBRyRst51JIGeRuUEHsQT6e+RZzGsrSyxGSRucvKSCfUgf41HLcncWit7eI5zhU34/77LVVybFB4bm1RHuoHiDn5Sw4Yex/CiDUGtbmRI2K+bHsLg4K4ZW4Pr8v6mrEmpXJtWtcQGBvvotrGu/6kLk/XrWU9upJ2s8OWBwo3Lx65OR36Z+lPmdg5To/wC2F1CFbbVoTcRxj5J4yFmBxjJYg7+ABz6dagm0kfI2l3K3yM2BHH/r14/ij6/iuRgZzWGtxJBc7CPMiUZMqA4Az1ORkdQOcVdV0ka3khlIZZQyvGxDKwBPUcjpVRfYlruKwOQG4IJGCP0qW3uZbaZZoJZIpU+68bFSPxFbTa0l7G1vrcH2ncAEuo0UTJ82T82Pm+h9D65FC70j7Osk9ldxX1srbRsUrKuSOsZ5PUfdz74rRSezJ5excW+g1F4I5beOOdnxJcQgKWznGU4UnJGSNvAOc06/0G8trf7bA0V5anI861beBxn5hjK8A5449aoaWD9rkR1KlYZsgjkERt/UUQXU9nMk1tM0UqEYZWwfp9Pape+hSempFcKiW42d1DEMMFWIyQfocimWmHTyyocdt1bTanBqz7tZjdrltq/boDtcDJ5dMbXwCOwOFABplz4cvLXNxp//ABMrQ9JbVdx/FASw/DIHrS5rbha+wxfEF5BZvFOFvYW2/ubpmfb2+Rs7kOOMg/hVu2l0i6s7kC9l05pUCiK8XfFncCD5qgYH1XPJNYGpCSOEB43RxINyupUjnoQcEU0uVgHbjimknqhNtblu90u+sF8yeHEOFYTKwaNgc4O4cc4NVwg7jmpbS7ubFllsrqa2kyGLwvtyQc8jo3fqD1NWzf6ZdJLFe2wt7yTPk3NkCAWwAqtEWK46klcdO3e+aSJtFlFQUR51cq0QJAXgncNpH0wxpLXVrqydntriSItgOoPyuBnAYHgjk9Qep9a1rnRXi0a+urS5hv41aMI8UZDgYy25Dkpgc85GAea5oFZI8jH5daOZSHytGyNR0ic3l1qtnJl0URx2e2NFYdWKngZzzjj2Fc1dr4Z3uBqVxKWJ5EZAHPHVfzqbUPk0+Ug4O2uPfBJFYT0NYq5rXttoWE+y6lI7FgCGjbH16CuhhULxtAUYA9sVwmACCK9TutAuYlkntZEvrRUDm4t+cKRn50+8hwCSCOMHnAzToySlqKpFtaGTKIpPvopyehGRT7e7ubV41SRJ7VGUta3AMkbhcfLzyBgAcdqZLETGCvIx+dQ7SoG7p1xzxXRKKluYqTR0t1cab4j1O28iFNBnbEcgdQ9q5wTneMFDwF5HOR6ZNTWvDesaK3mX9jKIWc7Z4T5sfAz1HIGAT8wHQ1ShtZdQlhtbdlSe4dYY2f7oZiAM8HjJ9KLLWNW0hLeO01W+UW3+rLSllBwRwjZUcE8Y71naUHaL0LvGSu0Vkfz3ZvMEikYAU9KfOSbXMW4OpwM9q0ZNbOp5l1HTbG4nZgXuoVNtcMATkl0+VmOe644/Glks7OS2il0/UXEjkBrPUAsbJ7+cCI2/JePyqlV/mQnT7MyprspZxtuzuAjk9+Q381qOW8ht/unLD/PWrmt2N9Y6U8E+k3cZMySPcFQ0YUbwMFcjBLnnOPlHU1zE3HU81DncvktY24db2bHUlZQQcqxVhz2Yc9h+VdJa/Ei4u4orTVLS01KCMEBL6JX3EkY5x1GMZx3Oc156HB+7nOKWNx0PWs3Z7lK62PTobX4faykok0ZtOupMcxzMyKO5XkAH/gJHtWVffCLWHs473SpYL6B13LGrhZEz2I6H8D17VzlreLuRf9k4x61sQ+ItR0jy5NPvZI3L7Qucqc9QR0xU+y7DVTWzOQ1HSdS0S4+z6hay28vK4lUgH6Hv+FVnbapBypA+6RjNew2fjrU7eLyGaCdCxkEdxHvUEkk45B6k/SuU8RaZZ6zfwXNnZW1hCsQSWG143uDywGMLnjj8aXK0PmTOJhZhxnG7rTpJ2kxE2du77p9a2bnQbnTRJNAvnwRsAzFcHH68/jTolk0+7cNuh+1Q7ccfMpYHBBB/u9scgUWewXW5WV1VoxjcWzzmrKyLPGQ+CPMDhW5yADnj2OKJ4dh39WHIxUttG32LzMM0bySRjgYUqEJx353jP0FWR0NBdb1pECjXr9ABgKLp8D260VlllViMrxxyTRWvKuxHNIwrTRtS1J/9AtZZs8fIhwM+9dfpvwj8SXIDXUK2a9WM7AfQg9D+detguBgLGo27QEQRqR/uphfzBp8WVjKlEOehGc/XiuZUza5xOn/C3TbCc3F1dmeRececix/+O5P612+maNaW0Hl2iadDnkqLhQT9eOalitJI1LkFEYf7ufyNRMBECpjAHX7x4/DNUkloIurZMuAbiyiXb8zPdIAD6cHNILKBgD/aWlEn+7d9f0rLacqdo5z0xVSaQsfnZRt7seP1oA6AWMJ+7qOmf7xuQacNIuplzbtbXOf4YZ16fjXKtNbMj4Cs68kp8oH41QfV5IImiin2Y5VlA3Z7fMKahcXMjan1CG3vZbKaN0uoW2vGVztPuemMYPHqKhk1FkKFI4wD3IziuZ8V6lOdQ0bWnJ3XlkUmYdGljdlz/wB8hP0qva68spijlxljlnz0ot06oZ06zzX03lyTkqxwQTgevSnX08Ol2zqq5MalhtGM1yq6reXdxJa6TYzTXMoCh4kYgZznn+HqOeK6+z+HGv6yEfxBqS2dqPm8uEhnOR0JPAx+NHoKxw+rSmWaPVIN5EilGI48orsIJPvvIH0NW9H8Ja34gkDKrQQt/wAtJQdx9wB7HPJAPYmvSxJ4W8NaY2n2MIuvJbzcu29S/TJz34HQcdqybvx1cBGNisduikDbHGMj8Tn9MVag36E86RJaeEPDPhSFZtWH2+7QZ8uTBVT/ALvf8aral49vNVJtrKNLeDO04z07VxWr6vd6jcLPdSPKeU3E8jB/+vWhpHkwsrvtJAO70Jz/APWpxS6ibYl1FeyXjI7v5aDgsfaobzUJrfR57FlUlnRskcjGcEfma0tR1CII2wZB4A9T2P8AXFVrPQb7xBNIbNYQgP76aVyqR4I7jvz0+tN3aJRzcDeXKrOflA646f5xXSaT4XvdRkW5uZUs9PlVXVpMb5ME8omckHjk4Hua6RIPC/hOAStGNWvxwZ5kyqHttU8DHtz71ma74me6WG7iYj7Qu6QjpvBIx9cBfzqopsTkuhvT6xp+iIv9m2kETptj84IplbGB8zEe3asefX5pbl33sUkGCSeua4+W+kkLLISd2G571Lbpd6hcxR24ZpZCFRV6sfTFWlFE2k9zt4NwcP3NJFpX9rs1vcTeTayF4zMxA6gghcnk/oO9WrWIx22LuEG6MZ2QIdymTacBmHB57An3z0qodQa4feXySMA4A4HQY6Aew4rO12Nuxcj+x6VCqWsHmyR4AnuMOwx6DG1fwGfc1SvNRkW2nuZZGdi+dzHJPFIGMoZc89ARzVaHSpddgYQzeTbLy8zKSBgcqBkZb2zx391swTuinNds00V3A+GLEygsFARduCcnAGXNWtJfUdUtZligWe3EjA3LELETt3AZJAPIA49e1bFnptnpixlIxJKjb1uZEXeOMcH+Ht0weOtSz6vNLdzQyOSRgKSeo2j/ABqutkU9iODT4tPmW6vZIprnP7mMcpHxkHpgt/LHfrUF/du6CXG4jJxnFR3dyDs3vjBByB26VNF5Vhi5uVV5WAeCJxnjoH5+nHp19KVr6Cv1JriD/iW+XMCH4Yp3B5wD/h9PSqsS/ZbB5Gb5iMLjP05/z2NTxSyXsBZwQW3FgPrnP60jObu1KHYhA2Lk4C46ZPp71WxL1K8UPk2moTHJEs5Eajvk8n6cqPxq14g1PT7LTLfQ45HjkAErvGv3c84PB559v1qvJdxW0Mt4m6W1t1GCEydu4ckD8z+Jrm3EGo309zPdzN5hDbhIWPsPmDnGOOvbt0qGik9bF63m01cGa8udpPXPQfSoTMCn7ljvI43rwD/WmvaWCR4RJpyVz81y8YB9MIVz/wDWpypGo2+Wm09my+PxYk0iizE4dCrRuznoR2/Dv9KubZ44Y5JtxYsUCOcBFUDtnIBJ4HtVOI/uRAu4xr91A2FX8OgqxCIo42Mke45GFEgx+OO/ShgOjmJYswt0/At+fFTxSF5Vj85MyMqDy4yM5PHHHfmoBKqHcscae2N2f++s0C5KNlMx56tGdo/IUhlidZo3fezkKcHPb61AWY9VHv8AMKkuV8q8ljLiTafv/UA/1pANuPkJPuKQyQqZmkldkaTG9sAk+/QY9/wNAYRjG6T/AL5xRGGZiAYwQehlT+WalltngYxuYgwAJAz3GR2oARbl4/mXeCOjZ5qcu9wjS/uyCxBIXqe/86iRExkszk9lAX9Tn+VTB0gidEiyrEMTK+dpHpjFAgjkdR8suD/dHb8KkW4lLZyxbucf0qNd03C7Xz/djB/lSlZOEAc44I3AfzNAxyW0jZKQvtX0Xge2fxqQW83A2gH0Az/KmQ2UyXAuIyiv6k7sj0IxyKkfULfcVjZ1UdttL0AmWFmQpId0bDBVlyCPxx+dVxptsrHJkwT0L/4UC8TO4SDYf77Y/QCnw6jECck57ALkfzoAlS1tYwMW6N/vDP8AOpPMiAwFVPZcfyFVb6KCYreW86ln4kgck4PqnoPUdB2qFHeDjIR2PAYDH4E07ICxMIZSWkLKwAw6/e9lPt/L9Cz7UkYxEgHuV6/zNTJBI2DJvY9wSMH8qe9tAQudqn+4O/8AWldDsQJLdzHILBfZc/1qpq1kusaXd6XdZ23MZQMUGVbqDgHsQD156GtAosYBjaPbjqF6H3549uanWccIz5Ocdf6UgPlplaGcxuMMhIYHsa67R9TiFqbY53kgqxHKgHOeOQeuAO5rkrqY3F5LOesrs5/E5qSCV4yAucZ6Z61j1Nk9DpnWJp2k5SLb8wyGLrhjx68ccdOM85qygEzFSiB2RQxXjLYI298/ezzk4jPc1zkd8GnDSOmSfnIGd3OencZx1IJrUhvzxIqJHHDucJnJ68ISMAk5K8DgdAMCmBuSIuxI/MwhjOULY/ibJx2+vf0p0UBSFi7ZO5ZCSW42gjbj1ySfyHqKzba4aIq0km5iRyc7mAGMD6jdnnjBHUc2rPU0QMHQSeWCwfk7jnqc8kcMv4jPU4AMTVZYLC+kWOMiXLDLDkk/xex6/mB2qtLqck6FAdqMScKcDGOn0xxVDVZXl1GR5DmQsc8+9RxRtKdsYYkjggZxzQIdM5CZB9zUEW+STk9TmrYspZrd5ACAuMA/xA4AI9ecj2/PE9jZSANNtYKv/LTpg+v1Hb8OnGQDqdDWMWKAMx2yqflJAJPUfhgkn0X8DekmWC2UGQNJJHh3Cnb0B69O+e/07VmRQW8ChgzK0QAJJweQBgDHoMH0BPWifUY2h8oSgqFJOcYZsnlccYxkj0APryxjZJlaRZHLlN5QfN0IIIwR2wBn3B9Oee1CFJS3VkX5VYjHGSM4/DJxVtrhFncNuMLna69CRkgNn17+nJ+hgljnjbySA4IIxj8QR1x/h+qAwpIDEfUetMJx06VueUbuwkTYplUDGBhu3JHf049iccZwpVZJCpGGU4IqWiky3pmx9UtlkJCGVQxC5IGew719GsGVd21lUEDdt4ye2TXzVbyGGeOUdUdW/I19K2d00ttBdRuYjIivgHPBGcZqo7GctyJs7cBT9TTUaWNsq5QnrtPX61MY9qE+Yd3UlwNp/AYx/L2qAsSpkaIlVOC0QMi59MgfzAqiSFYVjVht3sxJJLHHPoARioZY1k4KDA6Af55/Gp5JyrYW3mft/Cv6MQahJlkb7sKJ3BlO4fgFI/WgCr5CIfkiRfcLilYPkHoR0qURSsSHuF29MLDzj1yWPv2pX08yx7hK8gA6RyZZfqAqn8cYpgVAkZkeRoVZ3+8xyCfyNE0NpKyRwSMlwzYEUrgbuM8NgD8Dj6mon0u2J+ZHkXr+8mdx+TE0kdpbwArFbxR5/uKB/KgDMuvJSWSBllaaM4ZFQnH5CqjxeYpHkOCOnmADP8/5VvtANvGMVXaAL7euKQzB+yyA7mCIg9CXP9KUWlu5wZWRyQBvB2/oM/h+taM0IBznNVZIyVCgEn0oGUXtmtLnzIyd8TELJHyCeh9iCD09Dg1otf6fqCeZqVq8V5yBc2kaZb03IcfTjnGelVUaaAjaG2k9DHuVvqMEEdOtLJH5szB1WPcPvIQFBx1xz+QxTTE0O1LT7uwQSkNPbcf6QsbBQfRs/db2Pr3rNWdgS0ZIdDkMOoPXg1ZZbm1lbyLpCh4IdAUcDnDIRgjPY1Omk29yhNndLHckALDKojRieuCWPv37Vrz9DPltqXYvEEF7MZdStRK7H5bmM7JgpGMMRhX47kevFOudJ2p5tjcxXkBUuSjDegHJ3L1GO+M+vFYUtrc2TrFdQtFJjjcPve4PcVPp+oXem3cd3aStHNEcq64JH50tOg7FlT8pHSnQM0EqTwO0UqHcjxttIPTqPyrWS80bVNOhN8v2G/Hyefbx5STGOXT1Oeo+pJ6VRu7CSwSGQyQzQz58qWGTerY6/T6Hmqi03ZiatsT6hra6ylvDrlu10Y2H+lQvslVfcfdfseffHWqkmhTTwO+kXUOqRoiuY4ci4jB/vRdR+GaqSE56AHoaiRpFnilikeOWNsxyIxVlPsR/Loatx7EqXcTzdpaJ8pIn30YYZT7iow5acAEg7TnB7cf/AFq6i410XK29rr9udSgMUchlbCTxsyhjtYY45Axx0+gEkHhfR9VtpZtAv5WnjU5s7qMGTOeMNhcZ6d8/ylSa3QNJvQ5uG5ntLhJ4ZHjmi5R0bay/QitJtQt9YuIYNTt8yuwRbm1jCzMzNnlRw5JPX3PGay545ILnyZonikBwySKVYfUHpUkMbvOPJUsyAvxjooLN146A1UkmhRumWNU8KyXujTXOj38N5EFBZGIikTnHzKx46evrXHp4W1SPZI9vGFycq79Pc45/KujSTIKkt5bgZAJH8qSWzmMDx2t/PscjgOSR379KzlTb6lRqLY5yDw+8GqRrPJE8Qy3ykgnHbBGfT/Guotru5sZHeC4uI2kBRzHJt3qexrEhstSh1ZWkMt0oRgCoJKjGTkVtxoNvIHTOKKaSumObbdzSbUre/kU6jaQyDJLzWii3nbPUkgbHP+8o+vWhfDLakssujXaX+xctbyL5M6Dv8pyrY9Vbntmsp125KHGD0psTSIPOPOCVyHIbsePyHPqBWlrfCyL3+JFzS7W8i1rSxJb3VupvIP3k0LIAPMU5BI5HHWs6eILuUD866mx8Za7a2whiu0mjCBEiul+UADA+ZcMegyWJzWQtlFdozR31vBNtDPDev5eTjnZJ91uc4ztP86SbWsh2VvdMVD0/Sp0kIBGcjvT76xudNdIdQt3tXkGU8wfK/wDut0YdOQT1quMg49qE10BpmkmrXUdtLF5nmwSRmN7eUlo3GCMEZ4PPBHIrnLu2jnus25EQcktEC0ixcnjdjJHT8+prQPIIU49qdFaRR2XngEM87R5/3VQ/+zCpkk2VF2RjWmlX2oSTJZWstw0IDSCJd20H6f5FQbSrc9fpW40ptpVlTIlQ5jlUlXjP+yw5B9xQTFeTRSXe+QK5eUjaHmycnL4yT15OetTyNDU0zHiILY569au3T4iiYdd3X3xVuaxtRMBZibytv/Lwy5UhefmGM9DjIXqBg1nzOs7pFCjSuW2qgXJz9B1qk1yu5NveHR6i+8u3OOAKuHU5yyYbgDAx69ay5rWe0umguIZIZB1WRSpx9DSLkDAbkHA96zUi3E6u11Z1/wBW5DMuPrS2bC2uzeW74uCSQ8qJKFGSSAGBwCSTxjqa5q2nZJgSx+XkY71oPclW2tjcBxj1q0k9GTqjqIrzw/f25XU9Da3nCBFm0+4wHxxuKEhVP4HPeqWm2WjS2Fxaf2wLS5M7SR/boFRUBCrzIpIOcLwPQnA5rDWWSY8nGOTjiqrT/wClEtIyIAQxUZx6D88UOCSuhqbvqdQPAXjCRQ66XasrchhdxYI9fv0Vgx6n4pjjVItRvjGoATErDjt3orK8+4/dPXm1CNOXIYZ4wKcNVj2BkuoM/wB0ZDfqBVU+FnYv5GrXKrnJAmkX+TCn3WjO9oC2pXcvQfKQD+fXj3NPQdyZ7tGJZ59vfc+T+tQvcRsMi7UjGSApzj6HmuZ1AarpM32gTQvak+Wyht0ke4EhsYAGcHkenPbOdLrlqbQqjNu27ixP3iapJdxcx0V/qiRHZC67iDl2OcEe3T865+TVj5zmS4LuP51iz39xqDbYQ5dj1U1ch0CJYS9zIzseTzjH45ovbYLPqOl8QB24Ylceves4ahc38rJEMZ6n2rptM8GS6koW2tSyk43vlV/PBz/+uu603wX4d8JxxS6pI9zMedqrwCPYYyBkcmjW92GhmyeC7zxJ4D02wDtbXVvLIV3wknYxAOeRg8ZGevHqcXtG+FWg6QrTa5dvcsMN87+Uv5A/zJFWb/4hQW8TWmn28cP9zsVH0xj1+nvXKz63LPK8l07uSPmUncaOW8nLuHMeiT6rpOhaZPJosFuIl53QgY3H/IrzvxB4u1W/gAlun2N1C/KMfhUP9po+k3Ajdl8yeKMoVGCCHP4dKwLh2kZkzwpOAT0rRJIyd2x32yTHVjxg1DCJLhNsRI+YAHPBJ7VqWS27q0DxlMYJb1BHNJo9/aW0k8OfnJJUKNzOQRgBR1NVdhsZNzDPEod4/wB2WZQ2eNwCkj8Ny/nV61iur69hsbKF5pX2kKi8DK5JJ7Djqa7fSfClpPoStrcdzaj7W9wkTYV2DIgIbrt5Tp1A/TYXU7TSbUW+kwJaW45xGOSfUk8k9uai/Qsy7LwXDptl9u8S3CSugyLaJiEyem5sZx+A6d6y9V8TySBbOyaOGJeFjhG1FHoAP51qaldzHTNQCvnfAXJJycLyf/HQ35153blo7h1x8+c49KuKVzN3Zp20M92GkYl05B565H/1/wBKoahbNbLbqeELsvHQE8gH8mP51oW0UiNGoHf7oG4sewA9a6+28IR3VkJtd8yKF5Ekjt0YiRioP3sfdyGIx159ejk+okjlND8OX/iBA0EcS28LES3UjYSPA5B7nj/65HWuySysNFia00weZcZKXF1IB869MJjop9v1603UvEMNvarZ2qrBbxEKkMQ2qvOMcdeuawEvppUlyANwqZXKWppSu/mqwVYfKI2iM8ADvWTbeZGscZUswAXA6k9P51Z0q0vrm5+zwDzZDknJ4VfUnsK2pHsvDGXSQXN6c7ZGAG0k9FHb69cendwJmiSz0U2Q+0as6xbT/wAe4YFvX5sHge3X6VFc6kZmRYiFiHACgAY9OOAPauX1HVrq6nLzuMscnbwDxV/TYLi9aOKCMtnqegVe5J7AU2hRN64YRxIXcKuDuJ449/8APesy8069mullVRDEy7fOuZBGqnucHkjjsD1H1rSlmt9OjaGGVrq6SOTyp3Ufu22HaVHrnHP9DiuKn1Ga5V/tLySSDABZiTj8alK2pW51D2djaMjzXK3VwBhdvEakHsOrH3PHtUIupdVdRMzNsJIJ689efyrEtoGaSzfPyvGXfPZgW/oF610+m2wtGW4A2Sj/AFayD5cn+Ij+X41YmOmiWygMLM3mFgWHTbkYC/4+/wBKqSgxafcSbsZyAARzx83+H51JMzzB8yhZXUgO3bk80sFtNfXskUMMnkk/u1bAOPft71le61KtqV5L+20zwq2nTQubm8OwGVWCGMYycjrjPQVzkUawYRZI5AD99QefwOP5CtPxLvGq7Et0gZEVIxJICQMA8Z69eTn+VU0sL2cBninkOflIJfP5ZxSvd3KLsaRPbOsciq3XM6bSfYHfgflVfyWh4cIc8/KwbP4jNSx6XfRlPNtvIickB5SoPHoM5qRooY2Ie9Hy8DbFuJP509AGx7Wxl1jHTcwYDP5VtNpRt9OjuJ5SVkydqLjP/AjWCUhaRWHmPg5+fbg/hj+taraxI1ilo0MPlqMAnOfzzQ07C0KcjRE7Ywyntubd+oAqS3Ebyqpbk9Bt3H8qaLkqoCR26+hEKk/mRmnS3dzLD5bySNH6Y4pD0NQ24CCSUzKHPDOfLDY+uM1C0scfU2+M98nP5ZFRpL/xKljKB2DEDLABFIznPsc/nVI28LDDTNyOTGuR+uKTWoy79tjMhHmeV7xQg/lkinXGoJcBF2MXRSN7YXd9R7fWqsaWyDHlyOR3Dhf0wat215ZrcwxS2MXlscE4dyO2euP0prsBEglfc/mxlcfdJUAdu3NWo7e9jiUiZ19HSEgf99AfzpJJLi3lkhQom07cRRCMYI/2QKh3uDlt2O+ATmkBpXGbi2g+1TZdU2swYNnB4yAff9KgVIYolVZVJH8Spj+Zql9oLc+WVHq3AqW7V7dLV9ymO4VtrAZGR1GfXvyBQwNCKdQv32fHZnx+gpiJZwxh3hRyT/q1Y8D86zBKxzyOacshzkPz7UAaMltKhMnlokDHMZHPy5PHPOcf1qJLtLdcmCZyOQBz+mP5Zqs97bxxZlwMH77yYA/OmokdwWMJDAclkO5R9SOKQGiNQlm5ijjXtlnGfyJz+lWYrW8mYgXSksPkEY5b2HA59OayYrVi4IYDd93vWhDbTQgsjCT1weKNA1IDNBGxSSS5cqfm5A59x/8AXqVV08FXIQk9S8vT2K5/pUl28d7Ist0FjmH3pdwPmfXvwMe/6Ukmn2twPkTbx1CkfzoGKs9iDtjWJieyJnP5UiazaG4Fsku+4PSFR85+i9TTjYW/lhJIY5AOgZN3HpT0u5YTGsS3IjU5GAVC9fXH0pAfL5wCvbCimjJbAUsfbtRKcOPpU1mBJMgK5Unkb9pP0rA1WxPZabc3TgxIMdy2MD8PoO//AOvSku7e0jEUUmZAANy4PPrj1Hb6k8nJNLVrmSFjaK0sSKACm/07Hnr/AC9BWLuIOQTTGdUl3C2Yz8oU7uDnt69zgtk9/wAjWkJI4rKUxkM8iA5DcMpznt1ByMe/rweFE0gOd1WodQkUlSxCsMH0p3AfdvuuCwHXpW7o0CKk5ZARuHQnGDzg/wDfJ/M1zsuDIrEkAtkkdfrXUWl1ZC7YWkUkcMMQAjd94mG7rkH73Jb0G1umKYiz5X7oh5EjQcEYyZOvHfs5Pt78AzwsY7eS4l2ooYKpwAeM9yOOnue5OOKTzwk4QNhgd2RkYGQAQT/F3z2Jxx35nWtX87Za27ZiiGA2MH6DHtgfhSGX9T8QIIzFGM5G3AJ6dyeepA//AFnmsBtQkMxk7k5/GqZyTk8nuaMUgsXBfnoQcYx+H+RV+11OPvuRyCNwP6/4+xxWKF5qeIAMNwyPrTA6W3MdwyeXMkRxymD8wzyM49PU46e+Od1BSl3IpySD1bPP510FjbhLGaUSFgy8rCwLAf3iu4cDBHII56c5HO3ZJkyTk+pXaSPoOKTGiEfd/Gvo+wg/0K25biNODIfl49BXzinQfWvePDWvW994Us7u5lCTImx3THLKSACO/bnryOT0qomb3Oi3FBwUUjqUULn8qrltxAGck5qrDqrSoN9jcCQgHYihgQRnhvu459RSvNev/qLIhj186UKP/Hd1MCfDBvuZGfpUWF3HcRnsKV4rtwfMnt0PIGImb/2YVWNhNLuD6lcrjONsaYH/AI7n9aBEzFWwpAIPSomuYYJQpuEjkHQFsH8KmbRYBCz4N6mckSM8pXAPJRiQvfpxUkUKxRKI0RI+wUYFAFCS/tHiy0MsjknbLDE3J9yFIb69feo1jmckOqpjOCW4P5ZNaLRKTnPNN8pd3Y/hRcDO8j7wdip7FF3Z9OpFQm3XHIkz/vjH8v61quueOg9qqtC2fvUDKKRiJlJgVgCc55JH/Aty/p2/GozbvcT7UBc7eHKqhz6YB59gKuy+Sn35EH1OKgM0B4VifdVJH5gUwMqULvKvIdw6g8UG33KD1BHB9a07hYZ2Vri284DplyvbHJBB4/GojDbQAmFiik8rjIH45GfyFAGY1mDio3sVOcHP9a1HZSuAST7DFV2BGRuI/KgLDDcTPEba+BubcjABPzp7q39Dx9OtZ8um/KrWknmgg5RwEcHOMYyc9unXtV88dSfxJIq9o99b2M7m6tklhbnhAWU/j1B9KLiOeuIPKtbXrl4y+D1GXYf0p1peXOnSl4XwjgeYjKGSX2dTwf5+ldrLpGleII/PhuXjZF2FlAO3uMqcev456965TUNImtEdjJGVVdwycblOcHn34x68U02BLGdIvYh++k0685JE2Gtm9MNncg/3s+me9VrzSr/TLhIby1eEs3ylhw2Dztboeo6HuKy4yDgOwCngnGcCrFxr16s1ykU6NbzsWdDGuxyOjbDwre4wRxg8VopWIcbl/WkWLUtkbbo1hhwQP+ma1QR5beRZYXZGDZUqcEVpR3NlrGPtrRabcRxrGlxChNu4UYG9M5Q4AGVyvsKqahpt9prRm5hIilGYpUIeOUeqOOD/AD9cVpGatZkOL3N+w1rTdSd11+zjneVVQXSLtkQDODlRnuensCDinnwsJmvbjS7lrm2jEkccYXbMzeUHAHZvvgEDnrxXKRMVcqRkVpTXEljbWJt2aJpFadnjO1mO8rgnuB5QODxmk4dYgp9zNGQxRgVKnBBHIPoaUZVwVYgg9jXTRarpfiSRYNdiFpdNwNSgOCeOPMXofqfzFUda8N3uiDziFuLJiAl3Ccoc4xn0P1/DNCnrZg4aXRXs9UurK6jubWUR3MeQkhXOMjByPoT+da6Xvh/WLG5l1RP7N1l22+fbJLJCTjh9gyACQc9++e9c1kEDHb0q7PtXR7cADJnlfd6/LGP02n86c4p2CDauXL7wzqMGnG/i8m+sAm/7XZSeYm3ucfewO/HHc1Rj2tosA3BmNzKCAc8BY/65FRWV5c2MrzWF3PaTv1kifGTjgsvRse4PtXSzeJLHX4jD4gtRBen7up2kO/bjoGX72PbJH0rN80dy/dexyzgqewI7UCVlBJGcjFa954evI7UX9i0ep2XO2a0O4cHByvVT+dZAw5z19fatI1EzNxsTQ3UnlxRb90CSiUWrndCxHTKHjHPat7Z4X1ct9ptv7EucE+fbnMJPoV4AHfoPr680wKHHUe9PtpA2oWSSLviM6GRMbtyg5bj6A0ShH4hxk9iS+8PalptkL2SB5bVlDCSPBYKSQCyDJXpnPK8jnPFVzLG+jQWwZhKlxK8kTAqyEhAMjtkLUfnyOiBZWRFUqoT7oUsGI2/dwSATxU1hax3etrJf6iILaTglYjtQ7cbtoPJJAz+vTFZvmWr1LvF7GdIhbjFJBGVVhtxg4HNdReeEroxST6ZNBqFuoyfs0m6RBgZJTr13dM9M8dKw57O4s5CtwDGwP3WGCBT50yeVkI+RSxzyafHcS2l5Fd20rRXEZzHNGcMvbg/0phcB1hwRu7EdKGQ7cEAeozVaPQWqNm98ZXF1bTR6xZWt/DKwMiiIJK30ccKcnsO59c1Sk0LQ7sSSWmoXVg5GUhvIDOrZPHzxgsoA65U847civGIjjzY3dOdwQc9Dj6DOOe31qRX5Oe/Y1Hs02X7RpFX/AIRnVfKN5ZwxanbjBaXT5RNtOAcMo+dTz0Kis+MeeGZG9j9a3GSCQiRkXzF4EgA3AemeuOvFaMmqWE9m41y2kvJHbd9riIW45I+Usf4cknj1z6grkkttQU4yOVXdGAx6KPWqkyPKpeNsZcLxnPJHQ9ue9dBbadZ3UENvHqEkd7JlcXSAQhs5GZB0GBjkdcdjkRzaDfWOkre3cBhj85N6FhvUZ4bGfunHB79elHN0Y7dUTiM47UU0SQEA7gffNFdFkc9j1CRFmkBdiSM8ZOKm0944IDEpGxM984yc81yc2s3FqVMrDe4JIK9ew/DrS6Lr17PcT2enWT30gDSMIhkL/vHoOfWsFZI2szZ1qBbyG5iVwN4i8xs4wPPiyfyzXFXGmWQmCFfm3dO59unWu90awUQahL4raBUugvl2/DDajB8Y53ZKjpkdOTmrNtfaHp7ST6VpUcNwTzPOQWx+BJx+P4UuW7Gny7nP6P4M1m4jUx2UVtCRxJM2Cfoq5P54rooNH0XQdR8m8unub4jf5bNjYO2AAMd8Hqeajl8XfbQYGmfb6IAgOPcc1irJFPrul3kKqQRLbSE8ng71Oe/Ln8qrlaBS5mb194p8sMkG+KFcjbD8g57569+ucVw2p66kwkEbMZC2dzZyf1/nWx4jieHTjMsYU7gCDkc/SuBLqw3Mv3eSc1N7gjR0+4USMZ/mB6Z7VfuJ080AEoGGDIOQcj/69c/HMUcPGAQuDz0NMiu2nkEUMUk9wMKqRAkk/wCPandDaN21Dyw3aBEmRYRM6uxXowAOQe27OPwqnbzO9ygjgnlmY4jijUu0h9ABya7rwp4NkFleT+IrSSC3uIlWKLgTN8wb1+X7o4PP0ret7zS9EheDRrKG03cOfvOf95zy3547Y7072JZydj4SkjkEuvapFpqy/wDLpGweYjJ+8fuofzPXgdK6a1/sbQYY5dFs4luJA6tI+5nJGOSx55yDgED2rktWvx5qzyk3LknDSDjr7Vp6Perf6XcglRLDJGwIGOCGyP0FK11fsF7GmbqW6VzLI0jOPmdu30x29hWbckyqhyenAxVpZvkAXAqKK1mvZo7W0haWU8hU/mfQe9IAgja5je3lIIlikj9OGUj+tc1oeg32sNDewRjyJUDeeWAjUkAnJz1Gegyc/Q4760sLfTbgXE0i3V9GCvlocwxkjBBP8RwT04HvVO81AJbhi6pbQDy0RVwiL6Ko4A+lWt7ks0LfTLDw3CssRS51Hb/r2XO3PXaO3HpzzyfXOvNamupCZZBvyeAeB7Cs+7v5LxbV03FWiRsnjIGUz/45Ui2J3gKpbceABnLUMLHLs7Xdx5IbDO+36eldZpmiNfxR3CSCK0yd87DOTzwv94/oO5FaWj+G7bTNTe/1EB7nny7ZeUjz3f1bnp0Ge/a/ql/JOHdyMKvAHAA9qN0D0MjVtTg0VBYWI8qMqHDHlpB6sR1OcjHTjp0NctJdC5naZwSegzz61DPcG8vAXDfdIVu3B7e/zfyq5pGlT38rkIVtk5llxwg/x9BW8VoZMksvD91rM0UkbpHb5ZZZn+7GFwfxJ3cDqfzxuXWoWml6c2n6du2xtl5X5eQ+5H48dBn8az9S1qOG0+wacvlW6EkfMSWPcn/PYVk2JN3MIGJBfAFK1x3Ze0+5e7vo8/xELkdRk9aTSPDd1e+ZPMBbw/dLT5Td7jI5HuBitiGyi8PtvcBrqUfIpAIj6/mePoMd6oXj3Vy5lkkZtkqsCzZzyM5pOzBOzNWH7BYGJoY2nkg4zKAFJznOM89sfTOKgMjt87sSWyc/rVeaQb2AOBk5zTobiFdk0hJKHEcRUgkj+I/THH5+lQVYGmWaZ7aN3E4fbKMDbgAkAdye5+g9K3bzWo/DukJFHb79SdD94EBfYn+g5z+dVYLyHS9l3ewlpbhmMQ2ElF6bsY55/Q+9c7rLS6lqE9wcrEWxGC3G3A7duah7FbGTdXV5d3LzXkzySE9zwPoOwpUD79xbHbJ60C1MZO3y2HqM1PDbPJgg8+64NJO2xRakvPtkMUE4KpACI2QZz9QTio4oYFcli8hPGAMfrzT7iwnsphFO0QJGfkckge/GM/Qmmqo6cmne+okrE/m24P7uFmbHR33fyAroo9NtX0IXbW/ky/3nDY9sAnGMkdfWuaEnkDcH8v3zion1WF2Kteq5HOA+40nsNFqQyggq3P8AsgD+VNV5yT8w3d84b9T0pkMxkfaqSAYzuddg/XB/KpZDtlVQySIerKW4/AgZoASK31G4mENvp88srgkL5sYyOueW/nTkPyLu2hyMkBsinQNLaXgnEQe35G11K7wR3wf05pFeJAy/Z4iCxITkBR6Lgjj86Vg0JkKKuXPBxxuAP5dad9ssbRvNnuEiA7Z2n8zURkZowoih29v3KsfzIz+tMRpbZSYiY1xyIzt/QGiw/QuQ6vBM4Sz2EH5g5IYH8eg/OpDfXO0He5H+zgfzNXrKU6poU1ncEk2YM8G7qSAflHvyRj/a9qyTmN9m6NWxnaW6UbMCR5XlxujUn1CgH9KR2L2ckDJ95lfIblWU5BAx9QfWoQfMfbI+1ByCgJOf0xVywS3ml8uR3bPQgbf6mgLGY0NwWIa6mdD0BCrj8VUVCbXeACWIzzvYuT+ddtcaZaRWsDwxxszMVfeWyuOxwcfTj8qouwimISztVP8ADIkIHHuQP6mlcZg2IuIrhbiLJVDwY+SPXtkV1gvF1LT2M0eJlUg7+VlH90gZ/P8AOqEusyh/LdhLjjO8sPwzVWa8upeFaSMHp5Yx/wDXo33FcuuZ2t0lEDb8fdRVcr+ZGPxqJ47veiGUEEg8uFI/4CAf51QjE0rHzJJZGU9xuIqTa6MQoO4cHcu3H50AW7nQZL3BmvYwOwCPn89+P0pZPDkcOnzXEd1czNAd5t42A3R4GccZ469eB3pLZ7kSDzJ2Uf3cAjH1rRi1VIZgysQV7shx/Kj0YypZW+m7PMgCOPun96ZB9OSRV/zUBEYKg4yE/riqTXFmk7ta28aeYdxCDag9TUiXMMnCSRFh2DZ4/SlYLny3L94fQVZsWKXEfK9Rycce+CKr3KNHI0bdUJU/gcVJZZNwm1iGzxhSxzWJogvypuXHHBxx0FUjVi5UiVgW3HPJxjNVzQUhKKWkoGSK2UK8e1amnzuGgOdgXeGYcAKRg8fQn9PeshSQQRwRWvCyyaSz4VSjheM9P8SeePbFAi/e6pI1jKcMtxcfNLISQTyTgZ/EcY749+b+lWLp5Sf3nGcY5PAAwB+VVhz1NACjFLxSDijOTk80CH+2ali27hxmoAfepof9YvzEHPWmgOkkMMOlEqy5YjjIAU44bGeD7+3J4xXNz9CCCD6H1rc1C5/0RYPnDgZb5jn1xg9uc9B1rn3PNJjEX7or1T4VXcrWGpWvmP5cciSBAcjLAg8f8BFeVZrsfhtqS6f4uiRygS7RoCW7E8rj3JUD8aI7ks9nWRAxAByKk35HA496dulY4CBPQscfyyaTZgEmUsfQJkfnn+lWSRSKxboKMKPvn8M08xRuMKjr6MTuI/LH+e9RzwwxopuFheJiEEjKB83PBU5K/qPemA1riCIY8xE+rYqu0ihgyeYQTjKIXX9K0YrKOJcIEUHn5FAFPjg2qQxJ54PtQBlvJKz7Rb4wcbi2FP0xk/mBT/LfcPlJXvt4P58/yrU2AHhaCgJosK5lTWbyqU+cKTkFn+YD0yuP5VUk0dWOMB/Xe7MD+BNbrYBwT+NRjC/eYM3qBVcoXMGOxhtdkJjaOPcSWjzkH0AJ2n6cfWrk+mxyRl7O5FwVxmPYUk6dQvO78CavTKrKcgD0JrLDLu2oxllHaFS5B+gyaTVhplBgSeg+tVZFYEjbWx9hkuJvPlhMRJIaQusZbPdlPLHPqM+4qrc2zx3HlwzW8qEZDlip/LB/maQzLIfp2qIs689x3FXmtrk9GiA9NhJ/Uim/ZyrZeSQ4/h+UA+3Az+tAXM1g5GTwaZhmyNrE9cVpl9hYrbx7SMYdN38802R4p4tjl4TjAGdyHtxgZH0wfrQFyjbXlxYTrNCdrKe/IP1Fbya7pN4D/aUPlyHBztEiggdRgZB78g9etYlzptxbqruu1W+6QwdW+jDINUHVkHKmmhMtXWm6OZ4Tpl7E77g7WsiyIJOfuI5BAyOPmPHqa5UrMrok0MkUrtwjLgnnBx68gj8DW0w4B2jg5B9DVi41AXdottqdut2obInJxOo9n7/jn+VN+QGEGKuTkq45we9a+m65cWUbW8MqiGVv3sEwDxSH/dYEA+4H59KW88J3UyfbtIne7s3XK+Y3z+h5wBweOcH2rnJIXd2Vg4dTgr0IwfSkmKx3U+k6bqIimsLgabdTr5kdjdyh0kGSMpIDkDg4DDP6VmaiZFTTreaN45IbUqwYcHM0rAgjg8MOR3BHasZbnzbdEmIcKoUZ5AAHFa1n4h8xFsNVha7sE3GFw+J7fPJ2MfvAnna2fbFUpWFy3KTna3T8a2/D2v6hY38NnDIJbe5kWFoJvnjbccdD9f8AHNU7zRZTbm7sJPt1mBl5IVy0XHSReqfU8e9U9Kdjrmnndg/aYsEjod45pykmmJKzOvuYfDmvTeXZsmiakGKmGXi3kYdcEcJzwMdf7tZetW01jY2Fjc28sNxCJVk8yMqGPmtjaTww27eRkc1kXX7+aV5EGHdmIxxyfSuh0XxdeWVg9jqkQ1fT+CkVywLooGMKx7jtnGPWr5ZK3KLmi3qYPlqiA4/GmmRWbBwMdaZHdLPdTMC8MBlxBBLGVcocbcE8HGTn6U9kRmDIwbOcg8GqvfYlpLctWuq3OlzG4s7iSCQrtLxnkg9evB/Guhs7zTPFSTtqVtDp13EEL6hCvlxlTuC7lLYBJAHqcdew5Ix9cnHoPSrEDtDo2p4AYSy2qnIBwB5x/moqZRvYaZpXugahY2pvZIhJZZyJo3Vwq9i+0sF/Mj3rLs4ori/G0g7YZpMg9QsTnrg8cf8A6uoLLVr/AE+eO4sruWCVDkeW5Cn6joR9a3/+Eh8O6mgF9oa2t45KzXmnjYVVlI3fLgk9AQQeCevSlJSXmOLizkd/zcAAelPD54x161p3Xhu5Rmk0ovqdmpI82BdzgjruQfMOvXGPp0GSSUbDgqw6qRgj8KpSTJcWieOZ4z8jMp6ZUkHH1q0L+d7lriaUTM4CsJlU4GRkqxUhWwCMlW+8TjNZ45HtTgcdTnNJq402jWht9Ivonee4l0u+cDYsq77ctjoJFyVXOfvdPU9arXmk39g5EsQkjHIlhO9CM4yTjgHtkDNO0mCG61W1t7nP2aSVRKOxTOW/TNUrKU2z+ZFGseBgLtDKR7hsg+uTyDyOealJrYq6auzR0mFTYaxM+PltVKk+8sfQ/TP51nB1B/8A1VLG0ct9uvIvMtyQwWEIjhvU8fOPbcvPPWo7trZbyG3tSJpZi52RnJQdVU8AluvYcjjjFNOzdxNXWgBg6kgnng8VV1Jwtsm5WIaVFIH1qxFIJACN3IztZSDj6Gob9VKwKx584HH0BNaJ6EdRFYjkU9dQubeKS3guJI4pARJGrYVwRg5HQ8GmKwZ/5GmSqFzkBh7VWjFszZh8bazBDHChg2xqFGbaPoBj0orDz6K2PrRUeyiVzM9Gs9F8O6VcF9WvJdeu1+6m0xQLjpxnLDOepx/s96fqniFrn91GjWUBVVW2tJDGiqB90hcAjk9gP51zEs0sjlzIUVBkc49qp+eWuFY8gcZqVFFttm0t4vnkgAMOPlAz9aSW6MCsQxPv7VmfacSklsZ46GpWmQgnOQwqrkWGrc/umPOQTzV/w67SCRBIQ8FzDMM4PyHchH4llrIRjIuzLEckj6Vt+C9Mub7Vpmhjdo2hdSyLu2n+Ant94A8+lN7jhY2dcs/tAZV3SyuDwCcLx3wfYdj+grk7bwxqE9x5SJJJIz/JDCgJb1zk8Djr0Nenf2VbWdsX1jU1UjLfZbJQXIz0Lf4Dj1qjc+LktbP7JpEJsYASzOPvv+JGc+9YRi76FNpHPWnwwmRI7rxDqsWmQ/KxtoQXmI9D2B+gau20258L+GrZodDsY0O0l55D80n4n5m5wcdBngiuHm1Np38yIEsxx8xJNMlZvPhd+SBnIXO09j+eKtwSVyVNt2Ou1DXDd5eSRxxyG6j24riJ9ZSbVERVPl5xn/61WYZbq9MkTKdwXKjoB2rM/si7k1AFISwQjceABz15IpbMpEmqJKInw4ZFIYAL0HTr+PpWt4RsNRv7DUW0+0jmdZYuWOCAMkhf9o8fhmtmw8NEQi5vAbS1KfNIwG+YnptX8ep49M1pPrVvZ2g0+wh+zRf9Mz9/6t3P0q4RbvYiU1HcdDpNtZ/Nq021z832aIhm+jMOBzngZP0qPVr+CC2aGzzbwsoPlwDYW46yMOWPPc96x5b55pfK39P5U6GEbpSzBiyAhW5C54z/AJ9Kc6fKrkRqczsLb3MjwoF+RSQAOmBWdqglOmPFwSr7uOK0beMRA72UqOc+mPWtWz8PXeoStLOv2WxUZeaUYLf7o/qePrWb1NLGFoFvPd2enxMrOzJIkKDqcOSSP++xz04rtYbe20OMNJNFLqQX5NuSsWTjj1b3OP8AHH1LWbfQo1tdLREbbt80KN7DOeW+vbtWKdUlSWGUOWBZQx+vUVryOWpLqW2OnG6Nss5Yk5yxyaxvE0sywRJCT+9JBUDrirtw8jiNIwWlJwABkk+gqxi2t0jmviks6vtSJRlRwTycc9O3H1qYikc7onhsXVm0+oyS29vvJhKEeY3IztBGMEhck+mOc8aGqX0Edk2l2h8mA5ACnOz0JPUnIzknNP1HUXmWVVkZpcA5PbPA/l0rBXTj9qMHmvICPLDnk+mfrWi13JbMb71yxzwf0rrorKLwzCZZkEl+6/u1Yf6kHoT6MfQdKZa6fFoMPnXSb79wAm9eIiSMHB6N6en16UZ7p7tyZGLAHJJP9aTemg0TIZLiRZZGJ2jpn1P/AOv86eY2u4pLdCAxUncx4X0z3xnHABNOihd4LV1URRssmZH7gFNuB3/i/T1plxc2lrh02xxlMs5bJbA5J9TweKTfcT01JEtEifFzMZ5kOQiDam4dAByXH1xz1FWNOl060mSbULmF7zPyxNcIpAH8RByQT15/nWQdTNwiy2dzBHMyjashyRkcn8j0/p1y/wCzpZXJ+0wsxOX+fkn1PGal6FQkpbM6HUJ7y4uptSu7E3MiAK89vKjoiZyqgsynALHt1zjOax5NVWQ5Wy4H/PST/AH+dJdxNDZLCt1GT5wZkXcCwCnGc8YyePx9qqBGI7Cs7dDUkM8kn8KxsRzg5H0q5aXhtpEkUAupBBblc/SqaQ5IO7n2FX9P0v7bJKTIUihiaWRschVGTgd6ewCXd3cahdm4up2kc46AKABxjjFMKBsAgkD1OaRI1XqxwTUrIGx0GPTmgNBgiRVJ8pBk9QtawE7+GnlikjRYpx52QAzKQAoz3GSePx9jmIjI2TMGHpsGBVplwADdoe5QbuPyGKLCuU2LbM4JH+wM4/KhWkALLvIxj5flP/j2KnYqpBTPHUsP84pouonJAlDsDyqfMf0pgS29s7zKNwMZPJUEkfgRjP41dudPi0++e3nlkuXCqQI18oDIBHJLFhj/AGR9azUl87Jt4ZW2/wAOwrjnHetLVm1HVtVjuIbeFAIFjZ5pmTeRn5toXOSMdR2pMCHzUjkLJHHGw4JALZ/BiR+lK95NJ8pc7fQcfyp0eh3zMGluIkGfuRKWJ/Fsfypw0k71hkvsSKO7bSfcYAqbjIYyZG4jZztPLgjHvUy6bKYxM0ZETDiQdD+NXLW2iVdhmW4uAcozjzDGe2GPI/Osm5t5bXVZoZCxLnzlMzchG5AJ/MfhQBtWdnaQ/NLcQFh2crx+tWV1GyBEQDS8/djj3Y98GsGB7ESP5wcHPYggD+tay6lp0aAbhtA6GIk/yosM1kujsmj8hnjlTB3/ACnPYjB4IqlKswjCeR5v97eu78vmWqVzrNnhBGu5z1+Qc+3NRLr7Tv5ENjJnGc79gA7fMcAfnRYRPNDfrzGkaoR08lOPzBx+dZ8txPFJtE0gHokpC5+i8Vam1mdiI2iiXIySsocL+Knk/TNUZlt3tnIl3yEZAKZUH3yc07NBobNnd/2vbmyvp5cBg0ckcvzIwORwcg59xg96mtkiieWGcZnhfEiyIAfY4BPB6jk/XIrmhKVw0QQEc5Kk/TuMVoza5dXFzbubSJ7hF2KxLEvx3wQD3PtRa4JmrLfiLcI0LuOqKh/nVPy726DzG0kaNupdm2L/ACx+NZFzNO0peYxhs55k3bD7AkhT+VS2wN7dYuZ534+9kuR/OiwXNq307yxvle3QdcR4c/iRkj8qrajqVjpUMjOfnYN5ZSBsOwxxlgvqOmaludP067UQtA8Z7PG7Rt+YIry7V9SMl+4EjtDGxSLzGLEKD3yaznLlRrThzux59qSlNRulbqJn/maiiYhshgPc1c1pQNUmYdJMOPy5/XNUI+HwQcH0OM1ihtWdhkjbmJJ5PcUynSHLGmVRQUUUUAFXrG5jgV90ZeT+ANjYCeMsCPmx1HIGcfSqNTRS7VZSNw64Pr/n8aAI3ZnYsxyTyTSUrNuYkAAHsO1AwaAEpe9GMUnSgQ8HFWbQr5y7+RnkAVUHNXLNQ0y5UYz3700JljUhJFMVK7QQMdP6cflWaePpV3UXd53Mh3HoGJycVQNIYoqWCaSCZJonZJY2Do6nBUjkEVF/DQpxQI+ndE1CLV9EstQQKRcRK5A6A45H4HI/Cr3HZRXm/wAKNaebQLrTpBI/2KTegRNxEb5OMAZPzBjnnrXcrftKXSK1uGdMbldPLODnn58ZHHatUkZsu5544pA0qMWjdo2xw6MQR+VVSL5+iW8ef7zlj+WB/OkFvdGT97efL6RQ7T+ZLfyp6AXiVuTklLafdxJGpaNhx95M5U+4+pFUZtVtIpSlyxt2GcGUbQ4HdWPBpzWEbsS8lww/u+cyj/x3GfxqcqAzMSihwQwChQc+w4ougsVHvSSnlWd7Ir42sLaQKffcQFA984qtd3GrpgQ6LKWYcF7mAD8cSH09q0WkTPMhIAxjPAqCSUfwqceuaLhYzLe21yZ83FzZ2ikfwp5pz9OBjtncfpV5LUsXE63cgU7cpPnd74RVYD88evekLyYyJCO/FIDJwd+efxo5gsSkWiQ7bZLdEOQGCK5OTzljlj+dRS3DTAK5dvQFjxUdyTO4kkkcTYADgjPGcZB4PU9efcVUaRtjG5ClUGTKvGP+A5JH4ZHqaLhYtGIYJIGD7VCYyOM8UsbxyIGWberDIKnINRtMI49zIy+zNyP6UhkbEl2XnI6kjg1XkjwDnFMbV7R5SgdS/wDdRw7H8FyabJcyPwlnclumGTZ/6FigBrR5HBz9elQSQ5HQZ+lX7S2urmXYTbRAAHBlDMc9gOAT+Peo2MNu6xXcUvnYyVLhFPuODx+NAFATS237qOV1SThkB4b2YdDUV2YJkyIBC+eXRyUx/unv9GA9qtTNGGDLZwg4/iZ2/QsR+lJHd3FuCYHW3J6mFFQ/mADRcdjLbQ76RI3s4pLxJBktCCQh7hhgY+uMe9Z8lneAAtE8QLbcyYU/kTmteeWe5cieWV8MW+Zz1Pf61VlX5GAwCKAKttNe6Zcym2uwgIwWQthsjtwPzrTutU03Uba2GoQTR30ecXkDjvjlhg8evBJx71mhjsG7JP0qGVlI+ZCccZxQBDeeHr+ygkukAutPA/4+bYb1HP8AF3X8uKyVk3EMpwRXRaTrM2iXpmt1EsTjbJG/IdT1FdHqvgzTvEFkNV8O7bWZhlrV2GxiR/CQSAeemcfSkBwlpqV5YXcV1ZXMlvPEcrJGRn3HuPY8V1WnnSvFtldteW0el3tuULX0OfIfcdo8xM4TJ/iXucn0PH6nYXelXjwXdrNbnJIV+eM9j0PpkVf0ydYfDupjed0k9uAAeoxIc49OB+lG7Av3MElkxtp0KSFQ65BG5SOGGex/+t1FQqVK4Iz6Uy31Ob7J9lMqyW+07Ipl3rGTxuTuh+h57irsVnHqM8w0rzpliwWgcATpxk/KOXA/vAd+grZT01MnHsVQquGD5ZcYApshLpDaghghLK7Eq6Z5OGHUd+adnK5XkeoNN+zfaBI4PMa+Z+X/AOuqexPU6e78Lb7OG90K4fV7dztdQoWaMg8/L1bGOcc+3U1zjvGtpc4kBdpYlKhum0OTx+P86k0K+1C11aA2EqpLh5DvJ27Y1Mh/HCkD6+9Lq2o3GqXFtJfJHcxwAAxhjE7kKVBLjqeQT0zjHFJXXnYenoVVYOuc8npTFQDLHJLcmtubQYbmxa+0S/SeOJQZre6Ihnh45LZwpHXnP51hwXCvk5+ZTtZT1B9xVxkmiHFosKzRzLIryI6fdeOQo4+jDkV1sOvaXr0Sw+JLSMz4CrfQod3APLY5H4ZHXgCuTJBAyBSEYzilKCkOMmjpLrwJcPELjQ72G+gbJVGYK5HoD0Y9fSuSnSS3uDDNG0UqkhkcYZfqOtdFZajcWfhoxwzyxGTUHJKORkLHH/U/pVuHxTNP/wAhiztNUiUfKJYgsg+jdOlZ2klc0umYGnJ52p28O8xl3wGChscE9D1+lVFHABrr7TRdKuL2O+0PV2tbouRHY3cYYBmQ8Ag5IxuGeR9e/Paho9/pSxm7tpUWQDYzKQrZGeCep9qFLUGtCmeRjgGmSKrYLKr4PBI5/A9RSbsnjpTg3GDWm5nsTRTs0sfnJDdRqEHl3Kbm2oQQqyj51GBt6kYPTpVoSeFLqaws9Ra90u65eabaJkyc/KrBvlX6oTg4PTNZzLnp1ppkI+VlBU9Qal0k9i1O250mreBL7TovtdnPDf2uwyF42CsFxnOD1GPQk+1czLE6SNHKjI6nDIwww+oPIq3Y+ILzRbF7aCS6ltHV0e1MhZdpU5wMfLgdx6c5rp7zxZDq0sjapodlewbQU2MROh7jeeMe4x9KSc46MGovY43anoaK3zaeEXYsLjxJGCc7BDG232zg5x60U+cOQqArKrRoRjHzDdyD24qsypG21XB9hzUwgkkkXYr+ZnACdfy9a6zQvAWqalLC95by2ls2d0km1WBHQFTznPtVtJashanIGFwGwG3Ece/tWnpGhX+q7ha2kk5HLP0Rf95jhR9Ca7iLQtB0e4jnht21SbPMt24ESD/ZQD5z9eO9Xb7xK7+UIhsWJgV+b09FGAB7fnmlZsfMkYlt4Y0bQbmCXV7yW9nHz/ZrdNkS/Vjy/tjaPWtDU/GAkiMNlELVC+5mXq39Bnr3PvWD4xu0n8SvJHvDzwxSOo65KDH6YFc79oYsAFIP51K1V2F+xsXeoSSu7eYxJOWZmJJJ7knr+NZ4UySBy7BgOKI1dnJ2gZ6FuT/+vinllyMctjBxVWEIsgTO/g+uetOhDXrlidgX5slgMDjJye+Mmo1VWOQBuPqeK6bw34eea5SW/i8vTXRwzsxVnPIAUDng4OfajoC3KdnG15ftZWkMplyV+SPGOfU49+p446V21rY2vhrS5b29aC8uVZVEZX5YyecY5ye+Tj2Hqy41Oy0uNk02COJQApb+Nsep6n/PSuP1HXDdubdWxGGBwDweCMfmRSjHmaQ5SsnYtalrNzqlw1xK55OdoOBWYqkyMxbLr0HtT4m+TB59Ke8fGScY5zXWlbY5N9xqw7nDc8jPStC2Se5ufsVtZyzvIm4tGAPusoGSf94nHoDUukaHe3xWe+kjsrNiPLnlODKP9lc8/XgfWt2a/g0tPsekSP5f8chHzO3ckntwOBisajvoawXK7sSC0tdCDSXuLq/4YxqSIosDjJ/iP+feqniLX55VKkqF2howowFBUHAHrhuT9azr+6nnzLK5JJwxY5J/yKztcaVotPlY/uzaKNo6EiSQf+ghKiKSLk3IyruZpSjnJOK09NRtQuoLJIt/mH5ycEKvGW56AAd6oafp11qlzHbWsbSSMeFHYepPYe9d2iW+g6adOt1DyqczzY5duMgei5H44zWrdibaD5XgsIzDbPI+RtMz/ek+vovTjv1OTWHezGSVWIwQwxg9ORmiW6LspBLDJ5J7UW0Mt9erBEu9m3DjjqpGfwqUrCd2yef96wjCHh8DHfpxU6j7JlthFyOB/wBM+Oce/v27c8iSbUbfQL6OPaslxv2zy9dmeGVfTHPPUn24qtay+faxTkYMqK2PTI6VCZbRV1RJZLGYks0nltt553Y+X9cVDFFbxWraheAm3OTFEOfPPX/vkHj36VbumAhNxIqm3Rgu0nHmN1x9MdfwHfIw59QOqXEe35NmPu8BQOgA6ACk2OKJoymtXlzeTM25MIuGI7Hj6dKlvfDs/wBjFy6xJbIoI84v14wV2uuD0AzkHPSrGl2dlpOhi51aR4oFAYxlSHkYg4AHH+QegFcnqWsXOpNIrSTfZSRthZiRgY5PvkVDLtffYkgtpoHw7TyE44YnB98Y4/Cpd02/YoVCSThcA/41lQwxRqWSNFBOT8oFa2m3jWsTlHxJKoVRyCFB5IHTrgc+ppNtjjGMdkNdXiK+ah55HGcipFkVjwrAf7h/wp7FZG3lmcnnLdaCFAGCePbrQhj/ADNgXEcjk9doHH5kVpW95FDpN6oMi3UihUQ4wRuBwxwcDjqM/Q9DnI+MH2zkmnLuc/IuR7DNMBUySQ77MjIC85b6cce/6VKACBtJDe4prMEjG+RUU/3mC5/OlMiLD5gcFSSPkBc5+i5NIB0SkHLjzM9VJwP05/Wnal9gS6tbZYX+0NB5rt5zkdSMAE4/n2x0p1pJAWEjsSME/dIH68/pUepTT39/BL5ymOGNo1DDkjdlfyH86TBIhMcAmDrZ25JxuYr6egxirbXUrAtHujjJxtUYH6VTeJ3U4kwexC9KXy5pAomvLmXaeAzgAew2gUrjsTvdTykNJIz7e7EnFM88xAvv2juc1B9mh3Yw7ZP/AC0dmH6mtf8Asi0vvDdxOdPtjNb7fn8gFtu4buQOON1F2kBlDXoi+wXTSN/cRixP4Cn/ANtvIVMdlc/Lzt2smff5zTkARSoUBSc49/WgAMTkAHtindiLKaxfSKZFs4oZfWaQE9fYtj8qW+vpr82LErC0COkuz59+45GMjC4I9COTxVfb1JxTT9760ASNkTlllnEZB/dlgP8Ax5VVs/j+FTNel4/KNvb+XjBBhVifqzZb9ar9wc4H86RJE3gMSVJwdoBI/WgZdGq3qQ+THcyxw9o0cgD8BxTobiW9gbT5pQ6uwaMyDcysDkBST8uemc96pCe1M/lBtzAZ2bgG4GelaunNYGMu65YHIWQbR+vX609RGbNFLBM8EsLxyoxDCTg5qIs47dOpq7qUSNfG5t5om3RqHj3N98E5I6jkFf8Avk+tUWaYr/q0z6F8AfpRcQ+C8ePcscURY8ZkUEj8+BStcTXUmwW7lhjJKgA+wHFQeXIX+Z4AFPGxS4P4nbj8qtTSQyJgQqoA/vsD+hFF0Fi/ptynlBpRGQjY4UAj8h+uaLrVjny7Hyo5ecrOPvnsQwb+efwqrBpljfQssJkW4AyBKfMVvbntx15IzVZMxq8QhEDBiHjVNuGB54FJjRsxai5mJKQ5H8TpsYnoTycfr+deQeJUfT/EFzA6YgkYyRAHI2ntnvjpXo4jMiZGeDgn3rN1rw9/btr5JIWZctDIeAp9z6VFSPMi4S5Xc8v1OyaO3iuQ+9SdpHdfT+tZSSGOXIOOe1bN3DPayT6fdIY5VO10Jzz2IPcd8isVwQ2CMEcGsF2NZWvdBMDvydvPPykEfpUVXY4JLhNqrjAzkt2689h+OKrSR+W7LkHBxxVAR0UUUAFPTo+Dzj9KZViyRJLuNJX8tGOC2OmeKAK9PRd3cfjTWBViDwQcU6MkOMUALsNAHOKuPASm9VyB1I7D39KrjAOSSKCRiRF3wOPrWrZW+xwzEY2ggqff1ziqa3SoBhc+ua0dwayPlru3DIU8hj7DPDD1HIz6MKbAyrrb5pAUDHHyjH6dqrmpZf8AWEgED0PUVFSGB60oNNzzSigDvvhLqAtfF7WrLuF7bvEvP3WXD5/JWH417b/onnr9phR2AO05ww91PY9Py5r5f0y+k03UrW+i5e3lWUA8A4OcHHY9K+kYrnz1SZAGikUMjDuCMg1cWRI0GC7XkicSwqM7iQGX2Yf1AwfbpVfzjt4GPrzSxStDIJon2uvRu/41FJHGzO0EgW5f5hbyEKknqEwPlPfB4NV6CDzGPVvyqPKZY7gT371FHcTSRkpZToQQCkqiM5PsxFQXQ1Aovkx2oYnP7yRjt/ALg/gakZdBBHI59jnmm8KcHj/Cs2G3vjIDdah8pzlLeML/AOPNu/kKd9gtGG6VZJzkgGeZnB+qk7f0oAfcX1pA4ElxED/dDjd+A700SySKJIraZ1x1bEfH0YjP4VbBVbYxwRwW7dFeOMKo9mVRyOnTkDpmq8c7sqLPC0LsCQDyGAOMg/h0OD04pgMcXjcRi3TI4ZizEH0wOv50xrd5BIZL2dXyNhtgse367gxP5ip5MqTuyB6imhuucD2FAirHYRJdreO8ktyPvySEZkPqwAAJ/DnAzmrThZE2skWOmBGo/PA5pp47c0wt8vt9KBiszFiCxJHqajdSR70HpkHik3fIMg+lAiJwGBwoz708TiWBbW/iNzbq+4Iz4ZT6huo/w44qJ5AgyT068dKa3TKncCMj3FAya40uKRXk0yRZIY+XSaREYDPQZwMAcHOOhwWrHJ3EkYb1weauKxX5lJGO4q/b6lZSgRalbRtwALhRiQfU96LdgMIoFGMcAdKhkiyBgYz39a3rrw+7xrPYzpe25zhoyN3XGMdzx2/KsufQNUaMyeQYgD1llSMj8GYUXAxpIPlPzD6ZqrJGQMITt9elb0Ohwosjahq0EWDkKn70kfRelRSW+mRysq3VxPb4BXagRs9xzmhXA52SM7slccfnUtvLLYyie2klilA+/G2OPeuptF8NRoBcBmYg5YGU4P0wO31qG8k0UwuNPZoZc/Kz24YHrnJJJx09Meh7NJiL+m6hbeJvDsltqOk2k8hlKRtGfLl9codhG7OeF6kHI5JPG6v4dn0KBpUuBPaM43hFZSjAH7wwV4yRkE9+lW7iCf7OJZ1QKSQGSVXUccZAOV/ECk0/UtQ0eYS2sroh5kiz8ko9GHQ0tLeY9TlxOyldo46n2qzAVjujcRqUuPvb0cqU9CCOQa6e20vw7rZuvMkOmXG0FJXIjhJxnBB4HToGyeuOtYmraTe6RJ51ynmQkhBcwjdG/GRh+55Pvx+T2DRl631e1vwRqkbJcs3F9brlmJ/56x8B/wDeXa3ru6VZaw1K1Hm29o17bXETBbm1UzRgEd9vKkdMMB9K5VJQwMiNja4x7VsaHqV5p08kun3klsRw4T+Lp+H/AOumpPoS0WNDYtrEaugG62ugeeh+zyH+lNTDJyBnsa3tEvtIu9ZFxrCSQXUm5FuLZcIdwKsXU5AOCeRgVmXGl3Fu87QqbqzjbAvLcb4vxIztPsa0jK17mbV1oUyBIoU5BAwDnBGeuD2rVW/sLqxWyv8ATxJJ8q/bo8C4XHfdxuHrnJP1rLYDZnHHXmoywDc1bipEptHQah4UltNOgvbW9jvraU4QrxJzyMr/AFFYZDIWVgQy8EMCDUsc86xuomdEYfMVPOe2PQ+9dRc6/o/iCW2bWYpIY4IWSbYo378fKQwG7GM8epHBxUXlHTcqyZjz4Xwrpb5/1k9yTx/dKL/SssSfLiu0l8OabqugWdl4d1Vbl7UyOkc8gzIZCGIzxtPsQPwrirq1uNOuWtbyJ4Jl6pIMEe/096UZaDaFLkjjr2roLXxhqEcUNrdpb3tqjZdbqISHuOOQDwT94E+9c2Dwc0ofOTxxVtKW5KbWxvTaZpGsTSz6ffRaZO7kpb3RCxY46MowvU9Oyj5eSRm6hot/pQEl3CDESV+0QEvCTnGN2Bg+xwfaqXmBSBgZ9D0rT0W/vW1jS7a3xKgk8ryGXcpjdl3gr/dwM+3Wk4uOqK5k9zKcgHGeajbOMEfjVyzsLEmBLq+e3dmCPKwBjjABwxAHO4kDqMYySc5FvU/D97p0uPMt7lfJ+0Zt3LERltu8rgEDdxnke9JTs7MHC+xT09F3XORnFrJgbN2cjHocdevH1qBSYyQOg61c0u/jsReT7QxktHhAz1LlV/rn8KzmI993r7VpHW7IlsiUXT47/nRUG6L/AJ6YoqydT3mwtdB0AK2mWQebvdTZZueuNxyM+2PpVK81q4mmdXlkO4dm4HtgdKyJtULMA7qikY561g3msMsmyEEu3GelZFas1tQvCWO3JbODk1TQvIu5RjB6H1qvCwPzuwPHQ96uWqI0buoUj3NVsKxD4timM2nXkrLtuLQIAvqrFf5BfzNc5t2kPyce1ddr8bXHh7TZHwoieZBjrj5Tyfxqjo3hPVdW2usBt7b+Ke4+VR9B1J/Q+tSt36jMtJA8bM/y5IPtWxpXhLUtVbzlHkWo4+0TLjP+6DjP6D3rttJ8N6NpUTTIn9oXUSFsuy4BA5wuePxxWdrfi1GtEkVyisNy4bO78u3t/OmJj4dH0Pw7EZWCXU4GS9xgqh/2Vx6+vNYeoeIr++mfyZmWHOfmblj2OOn4Csl7yW9JadmEZ5C57+4qN5lXCjgNwOMUN2Gk+pDd6hM0hbzOTx7CsvNx5okiXd5QMzgjjYg3E/oOvrjvUqndOpwOT0JxXYeFND26Pf3+qB1ivozbxxqcF0OCSM9BxgH/AOtRFvcp2sQafYXF7KLe1jMkg644Cg9yegFdLa6fpFk8f2y3N7Or7SHBCKfRVbAbty3p2pi3ghgSzsYxb265BWPPPuT1J9zTWXjIHI5BrSU7mKjYzb7WXurtpvlMjqOepCnnbn0HpSeeuY8NjjkVkSxLDJsh24DMOBwACeKsRyypEqMjPISFVVXLMfYf0qLlpEmqSFrduPxxU+meHr7xBpWm4IjthPO007H7qfusY/EOAK17XQXWFJ9cBhiJHl2in97IfQ/3Rxn1xnpg1Neak0VsI8RxRIMRwRDCIPTHc+/8qe6shXtuNeSx8P2D2Ol7ySB5lwx/eTEep7D2AArJguTcrJI7szjqWOcZrHvbuSZ3xknrWtocM9/sto7WbZ5R3zsmI1bcTy30I4+uM9mlZakt3HWtlLe321cBQMu54VB6n/P0q9dXMFvbta2JJU8STN1k/DsM9h+p5qeSe3Sxms7RiY42VnlIwZDyOfbkYFY86FBw3XtjoKzlJt2LikZ88QySMjcc1d0xlfTUBdlghJjcqvJwxAVffjr2Az6AwrHJcGQDiJR88pGQnp9T6D2zwASFluyLaO3h8tYFLY2AgHJ5JJ6k4GTxn0FO9gsVteulvG2wOBHARGETogKg4/z1ySeaXRNMDiWWZikUK7yoX5pTjhQccD1qnbQNqusy2sYKWx2vNO4+UFQQ2MnuAv8Aga7DWdUSO0+w6ezDy1wWh4z6jHU/4+tTey0KPN9S1efWdamF40yCJ2WOLymVU9cZAHJ6k4J49BSxW0UwBWSQDHykqBn9akuIQLp2K7NxIDMMZ/Sm4YrsEmAPTmoXmaALGRecHaOp3Y4q+5aSGOJdixRLtX+8e5J9eSeeOO1VooXC4yT+FTrDIeqlfTJHNMCxE1nGu2SOV26F0cJn8CGx9P1pWWF2+QzRjGMhwTn1+7UQik4wjEHnJqaJOOM475U0XEMls1uGjUs8YHVlyC3uT/hinz6NHaeSzb5I5QWBlkZyRnA+8TjOG49qljtWlcttYKMc7TjP1rX8Q2aW13b2ikRiGBd4Zhv3Hkk9OuR+VSxmD5SQNthRIxjHyKBUx2MBhTgfePUkUoudNtX23N3CpPADSLz+RNTy32lRR4TzZc/xRRO/8hzQBVbbuJUEDsCcmkwSM4zjsDSS6rATtt9Mu3I6bozHu/76xUDa1dTSNAulRwFQCRNN655wufQ9+1Ay0rdtpyTRvG0lm4I9arPPeSAKbiJEPVI4Mf8As3NOAjkAZuSBjjA/nQBKJFXHLY9jk11Xh68tlt5LZXQx3MDowchOvGOevAP4D0rlo7hV2ptTaOgYZ/CnrcMQJFxsB2h14APoKeltRGZaapHKqxwo08gUblt0eQg9+gJ61qxW+pTKWj0+VVHeZhFj/vo5/SkkuC42hpM4AZi+f6VHgMoABIU5xnvSGXks1ZSLi/ggfsqI0v8ALA/WntoclzMscGpvEh6O9kw3H65wPpz+NUBls8cjoDUsE86DklCPujd0Oe386asxMgjsY4L28tZ7iW5MT7BvkwB8oPGwKc89yaVrW38sI0EbgHgyKHI/Fsmr2q3Nn/b8LWhjK3UAYpKNuxgR26HO7sP4fbi61jE0YeaaMDu0QJ/+sKWtguYytsj8pflj67RwM/SnbifvEn61NcwxRoHEZWJhgO/Ab8aILQzRLInzRH7pRS+fpjNK9gIc9+vakO7d1yKsb7aGUo8OoEggYSzkbH1O3APt1pZLmz3CNbeXdnjzT5efr1P4YB9qd7hYoTSJDCWckDgcAn9KnWMYxt68HIpzSzykJBDbRn+EqGlOfx21bRdaKbS20cYkVFj/ACyTSGOt7OSJ45FAIUhsA4qzqukTqv8Aa0XMM4/eguPlfIGR7HpjttHrxUglu5N0f7y4ckcuSSPywMfhV+Gy1GPTmtEn227/AHo3cvn8Wyf1qvmSY8cF2zAJExz0OP8A61SPBdwWzvNA/lrzu2k98YBxzz2rSj0eaFCsd0sKH7wjyufqBTJNGjgjO64h+bqz9T+JNFwOX8Q6JZa9p4CQSJex/wCpmxsI/wBlt5GR/L+flOradc6deSQXSbZUO1iDkH0IPcV7f9jXlVkUv2RcsT+WRXO+LNEF5otzJJGI54ImljZsgkLyR09j+NROF9UOLaPJ/OZbbZ2zx/Wq9W3AFup+QdOACT+P+e/4CqRjpWSNxKSlpKAFFKrFXDDqDkU2nKNzAZoAmu02zZBJyoPPXOOf1zVcV0l7pDNosF6SBuBTHfKKpP8A6EP1rm6CYyua+mIkwZJUkKAZJj5K++PT9KrXFv5LDcCM9A3BP4dRRZMd3zRqydyUU4+pI4q9exRtFviZ2UjLb2yBx69+aBmOQc9OPrWnp0JubeWIMcgEqoH3/Ufy/wA9c4DJ+Ufia0NJkeG7D4IXHJA+77/n+tAFKfPfBOSSagPAxVm8kLzNuUZz2/z0qrQAUtJRQMdX0F4Qc3vgzTJDKxZoAhIPTb8ox+VfPgr6C8DQmHwRpaFSD5RfH+8xb+tNESNVdMtlYmTzZM9VmmeRf++WJH6U+K1t7YEQQxQKe0SBQfwFTkljiomYgcc84OO3vTELsCZ9PzpjHdx1HpUWx0JEUjknkh+c/j2qSMykfvEXf7UwG7FCgEYxyAB0puWkU7gw55FSgtzuUfTNISOeKAECndyR9cUhjQ7d4DgdFYttP1wQf1prMQaGdI4d0siqi8ZY4H50wGkXctwfKsjJEAxbyiCUUAc7dxZh1yQM9Bg9aaQEYgLh1OCrDoR1BHY1BHqtldNKlq5u3QZZbdDLge+wHApxMt7cKxka3cJsV22kYHZl4bjnnrz7YosIkLk5BAFVywXOTn0qAfbGL+fEka5IDRvvJ49DjFSJgfK29iCDxhcjPcc/nmgA3kDBBx6+lV2BTDSzqsY5wOAfrmprmKKdWXytoPQiVwQPwb+lVF0mzi3MtrGzEctIPMJ/FskUAQPqVnuxDMJmHaIFyvOOSOBycc0pvL3zF8uy+8MgvKMH69cVbW38qPanyc5ynGD68Uiy2kZfcojYj5XQkx5/2hzj/gOPp6HoMoSQahMm57iGAnqiJvx/wI4/lTDpy4/fT3M/OcM4X/0EA1pvnyklZGRJBlSw+VuM8HoajGMfKM+h6ikBEk88NuYYSI4s5Kp8uT6nHU1VdnHWrR29CR61G0Q7HincCi2G6j61WkUKMAdetaTx4PXNVmDbuimkMpM+04KHHriq0g8z/wDVWlNCJMAEA1GINvVRzQBThmntZN8MjxsO6kgj8RUktzHdef5xcTSsGackyHOckkE9cemP1NTS2wHJVgajEEZBG0mhAZnBdklyMEgOBhSPXnpUsN3qGmpILGdo0Y/OByGA7HPUe3Q96utYSMpMcMjYGcBeo/rSRyTTWVyk2mXEjrMd17HGSY/VWUgA9/4gee+KdxGK9va3UrzTNLBNIxYuEDLk9c4OQOT0B9hUklm2nJu3KbWQArKrbwzjAbpyoJyRkdPrmuy8PaLoRuQ99di8EnKi4Q24UcjgH75zxgEj610t1oOn29xEbdIo0nwkkKgKtwOwYcbj6DrwfehJvoK6PKwWJGDz9c5FXtO1S70q5M1pIUZ12yLgFZF9GB4Io17TW0TULp7aBxYhlZIwfM8sHrkg/KAc4z/PNVLF0vGCxZMjcKncnGa1v0ZDRpzpY386jT9tnKy5kS5kIh3YB+R8HHORhvqCB0zr2zutOvXhvIHhl4wGXhh6qejD3GRT9wzwetaOnazJpsLWU0K3mmSD5rSU5APqh6qfoRT1WwtHuZUTExg+hp1sziadZFUoHGPcYrSfSBd3ZXRFlmDwpOLWVlEqBh0HI34x2GeRx3rKWC7gvrlLmB4mVhGQ6FcOoyRz3wRxQ5piUbXNCHdHcLcQvJHMv3CrkAfl7fzrsIfGtjJp6WOu2T3CBcMZFWVXPt0K/hXFRsQ7Kcgj0qRnJXnv2puCe4lKx0E3hzQdXthN4buvs9wefstzKxDAehYbh9Tx/OuTuYZtOu3t7qIxTLw0cnB/D296txySRzxzK22ROVcfeH411Nh4ksZbb7J4jtIb2EnCyGEHZn0xyv1H5d6mzitCrp7nCOoJyM4Pr2p+meY2r2UaSGEm6h2uFzjLjHBzk13F74Ht762+2eF7r7RGx5tZHGQc8hXOM/Q81w8i3Wn6kq+XJFeQzxnYy4ZGDAjgjg8Z5H6U+ZOIrNMkbBUMG+b1FaFjrep2DQPb3TqYeEV/nVRyCAD0BBPTB9+BWdyecgHuB0pFYhSa1spbkXtsdLc65ourxSrq2lRR3s24Le2cWCsjOWLspJ6cc8tjI5BrM1XwzfaZE90k1ve2eSouLZ94XHXcP4f5D15rMJzyOvrVzTr+6027S5tZnjlToVYgHr1HQ9T19aiVNx+EpVE/iMcJcMMrAxB5BDDmiujXXrxVCm20qQgYLyaXCzN7k45PvRU+/wBivd7mubSG4VncYzk59Kr/ANnbWyhJx3J6ipkjuBIFdgoGTyR6f/WrU0/Q7/UpxFbRGXBw79FX6n/J9Ka7i9DMRF+4w3Fec+1dRpHhy+1S3Ty1NvAOTNKOOfQdW/l71t2WgaV4eMc+qTi5u15CkZVT64A4+rUzUPFNxKSIWESg/KF6gfWhtvYNFuWBZaJoqxwfLd3EOT5k6h9rHrwBwfY449aoa1qkU6K0w3RxjID9AcY6dOlYUl67yEbvlzkA1aaPzkJPzArgZqr2VyGmUIdUmhujLlhIrHbjoQO2K5jUNsckdvJJuW2Zoi2euDj8uK3J44o3RY2PygfMDWFf23l6jdAj5ZCkmDyDlQSfxobuUlYQ3JygXO3GcipEhluLiK3hjaaZzhI4wSSfQVo+GPDdxq8yzyK0GnRnEtyTtB46KcHJ+g/+v3dnHoegwz22nQm3aZGT7U7kyZIwCMknbnBxkfjSZTZjad4Vg0pludTeO4vF5FqvMUZ/2j/GR6dPr1q1eXLzMzOSzP1JoWYXEQfcCSM8HtUbpubPtQiG7j4YyIwSv0NSSMFiJOBQzLDFlmAVRzT4tG+2Qte6wJYbBeUtSdjzYIPzd1XkccE45wByNhYwLGyudX1mWHToPNCMDJMeI0yB1bp+XPtXTpJY+HxJ9jYXF6y4a4PRfUIOw9+/6VSuNbuZSYY28qBRtSGPhFHQAY9qzZpOM9TWsKd3ZmU6lloWL7Wy92sksbGfyFKyFs5+YggDt90E+tY9xfmaOI/dlkdg3qBnAq3c23mQRXKmQsjGMqB8vIyM/k1S6J4ee9c3VyfJhVuZGOOeeBnqfbt1PbLdk2gXvJDNP0eTUrwlm8uJR+9lYcKP8farmo6tDCj6Xp8jrbxKU68se+cdSf06cVcv76Dy0tdPwtupypU/f/2s98+tZLwxrMoVAA3JIHeovcvYit2k8icMDubaW47Kysf0BqVAtyvmNItvahxGZ2IwWPRVHdjnp26n3V5I4baW8uWVLJAytn705x9xf6ntVW6ln1TTlmliWLBDW8KcCJQcqB+hJPJPWob1KW2o271ONbh7O3DRWgJKRLxhhnJY9Sx4ye3QVr6Boz6paNPPNHBbQttBIxu45x9OPxrjo7iN5LqeXLFrqT5o16AMen6fSrttLrM6xMJlgtN5khVUZePbI56Ad/qaFa2oN22VzvLXT9K0+JLa2cJb53NLIRyPYkden4H6VhTXFmZpVSR2AYjKoxJH/AR1rInndLR457qdkwQFEuOD2x/n9ay/7SnnBSJmCgYByCT+OKh8tyo3a1NO5j06ZyjsyYXOSu0/iGwc1UiTTSoRLSdpem93QE89vT/PNU1OCckE+9XbGFp7lQGVDtOGZgig9sntzS6l20LTafbJB58whh+baIzcsWyOucRnnp0/Oqpe2jz5GfXLxk/qHH8vwqtqMzza1MjFJDHtX5GGxSQMgHHXpk8ngZzUZDxAZdcnuCD/ACNDstgXmaCXvBDQqDj70Zwc/Rt1RQ3M0M24Tu65PySbcEds4AP8qoQyyXUpSHaW7s8iqi/VvX2GasXcSQOVimWVQPvgYz+dAFpb6UMW83JxjOagu8anctPdolxIxG55VyTjGOvpgVXUbBuY5yewzipTcKozkY9aLjHXEclxn/SZkY9drd/fPWqf2O9iKlLwvg5YOoA/DHP8q0Ay4+8OeBnr+FOZTgAlQM8ZOKVn0Ait0kVsylG7kjPAp/kwNqbDzXM32WEyLsKhCS3GD1PIOehz+JdDcbYZYoWQsykBsE9vXpSz3Vza669xbGO4haFIz58rMy7SThW4GPm9AO4HJqrCvqXDpTvM0yKYLcKBumbb07455NRrbwIf3t0Gx18mI4P5kVnXkUmpagkty/7oSAt5blZNg6jI49x/Oty41RMKttZwKgPRoQcj8/p/9epsMrrcWcTcxStxxmTb+gGf1qs9ySAu8hc52g8frSXSLdvkgxZI5hwCB3HII/SrP/COWT6Lc6gh1Cd4UYlJJvkUgZOSqgcAg8kZx74o2VxFBriLftXLkn7qsM/qRVtbC+Klkgkwefn+QfmaRXNtA1tBhI8YKpwG/wB7HX6mozcyT7QzM/sTkCnYHcnKGMbXliD88bs/yyKfCLdkTzSoZTklEL7vqSRVZo1kA+cgq3BjwM+3epFTb8o+vXNMDQvbq11GwWyvLJLmBCzRMZGiMZZskqFbg89evUdDiqseyCNIlMhCHvdS8j0I3YP45zUSpxk/zpTIOCACKNBF2zu4LB3a1tLOB5Dlmitwpb6kYzV067NG0LywFtwILBiPyB4/WsUO5OAOO3FTxRfaHVXYhs/KSelGgtjrLZ0nK3KOwyNuVbIx9K5bW7Q6drE+wSSQzkTJvj2nnggc8jgc/wC1XSWETW8CoQCM5DAY/OsTxjJqJs4pPIeS3tZfMMkIBxHtwdwznjOcnjAz2JE6Ior6X9u3SPbpgH5WBfJ+uM/0rTNq88QNyZwxG4qBkDvjpzXNR7ANyM3HQq3H6U5rqYKWldkQdSX6fWqsI6eysbaOR2E74B4z8uD9e9aG6J02mSMn0LCuItv9LLfZ28/acHy1zzxn+Y/P3rRt9KvpmCiPy1xnfKQAPqBk/pS0A17iKaKQs4l8jqPLkwo/IZ/Sqhmsg587zUY4JVsMfoeDSHRdRgaJreeylGfm3SSKB+BSpW07+0GliNzBFLEwVvKjeQ5wDjLBCOCD0xRdBYp3l3DGF+ySMozhgAFP6YrkfF+smx0iRCd010DEisc8EfMx+gP5kV339g6ZBGTcXFw+0ckyqin6jbn/AMerzX4r6fbWKaS1skwD+cSZZC2R8mMAnjv2BqZytHQaSPPNoaIxqQzA8Afy96Se3EUSsMgcAZ5zxn+Rp9kQsquxIAYZIGfrTLsyFyDnHBAPpgY+uM1ibIqHn2ptKTk0lAwpVbawYdjmkooA6S2uxPpL7yAVQqfp/k1zr/eNWLeYrDJH6j/P8qrHmgiKs2PifY2f6Vp28/2plicbiPuqemeOvqfrx9ayR1q3Z3DW0yyL95eR7nt/Q/hQUaF1aLHOyggv0HPQ5x16nnvUtkEt1Zp0by2XDEYHBAP5cf573dJtxfSlpciNQRvHOBtOTz7jP8hnFNv0MRkjCKvykBSM8Hr9SMjn/ZzkCn0Ec3dqFnbaSVJ4J61BU8iDJUdM8VBSGgooooGKOtfUNtYRx2Nra6ZGqzQRiNo2OBMqgAEdg/A9M85Pp8yWcXnXsEX9+RV/M19Nx2zqrST3G5z0S3U4H0dsH/x2qiZy3Go2RyCrDhlcEEH0INNcDB3AEZzz2NV7yCW/Ctc3dwsyjaWh2qSvoSQc49etUH0KByBLcXkqgbCHn6jnrgD196rQRpCWIvsDqz+gOT+VDRTrmRVuCCcBGAjBI9NwB/WpYpZYIREkrBB0VFVAPwUAfpUHG5iVO48k0AV7ma/hTMdkrvjIBuUx27gn19OxqpHHr11vY3Vjatt+WMIZM9O5xjv7Vo5A4wOaTePuggfSjULjby2FxHH5kckYI/gmeMt7khv5cVVGk2EUiSizhdwMb5F3t1/vNkmrRvktUJlWOSMn5wV+YD1XHf8An70lxdRLMkccrmOYjyXZfvdeDjOD/PqKLAmP3BgFY4x09qjI+cY2475pk8TlPlchs9aakZVvmIb3paAOuAXjKowRiOGxnFQqWRArsCfXHWrAI3EkrjpxSMCG+Rl2/SgZWPXA5Pt6U7zNpIYYJ7VJNsjQ5wcckmoRIhAbG0H2piFYh8g88dKjaMAYCAD09amOw9jx3ximsqgBmYgUhkJBWF4h/qnGGTsR9PaoHVkZRa/OSPmhdgPxVif0P5nOKnM8LkhWDsDyqHJz9BSYbG6K3kbPHOFx+ZBouBUguoLoExv8y/eRuGX8KV9oOGYAn1NXlt/OA+3IGVBiNVcllz1weMfTBFRx6ZZozO0cknfbLIT/ACxQBTKD2FUJ7vTwWiluICehXcD+lak1hbOVBtkG3kDkj9SaSOGOH/VRohA/gUD+VAFKNJJ0LQW8zr1DsAi/mxAps9rqMFtDNJbxQpP/AKsvLuz6/dGM89Mg1dlAkG0yspz2x/hSid4o2QSbgy7XGOHHIwR0PX/DFAGG9vckkG5VT/dSID+ZNL5MIaNnQylB8wkdirH1KjA/CtyKws7hCsFylpLu4iutzxHns4+ZR165+tZMSPcRSSRRNJDG+wzIDsz2GSAQT6Gj1GK+pXSoYoZBbRnOUtkESn67QM/jVGRnL7yxdsdSc1aaI+n51G0ag8jr6U7iIPtO05RFRypDFcjIxg9PbNWoNYvI4njWchH+8JI0kz/32D6D61WkiycqBn2phAAwQevpTuxWOhstXmuIRBJPFEFOSZfnTaeCACGKg/LwOOOwFVb/AMHWcjG4sr6CNFXAwwZGI65KjjIxyB17d6yCOAA23ByMdfzFOhvpbObzYHkikxjcjFc/XHUUb7hsZ10TbHyjExVGJRyuCykf/q7+tJLvKkfeODgryDXfxavo2q2Ztr3gSLsdZxvY85GH68HpWRruhGwCaha7LiyKEPFgLIoI6nPDfUYPt3q1dEXTZyFzeTvb2jSMVZI41DZ5ACgDH5dav2GtSSXkN1fRrqqxp5Hk3bkkoT1DZyGHOD/9aqxs3XSYYxEwdMjawwwwTu49iD+tUYvK+8VJbpkVF9LMu3Y7U6RY6v5s+gTnMf3tOuGHnR/7pyRIPTv9TWFNHNbSmGaCaFwTxJEydOuMj3rNimkhlQo7KwwVdDhh9DXTx+MLy702Ww1SJdRiK/u3kz5kTAcNkEbv06nrmrjJrYlxTMdWfDMSRS+YAVD5Y5PHUHIqa3t7q5mMMFs1yCoKm3xKx4yRtXJ4/p26CJotr7gMY+Xnpwcc1rGSZk4tGhodxcWGom6t7mRGRHdY+qvsRm2t3IO31z6U/wAS+J5vEmmrbX+lQzzLyl1AxSSPAOFOT8wOfWotPBN4BjgW9wxI9PJcf1rJb+8GIHpScU2xqTSRrDSUvbEXekyvMqpulgnwk8Xrx0Ye4/KssAquGDK3cMCCPwpbeWWCVZoZWilXo6nBFai6npc53axbspYEG7gbDgk9SuCDTV4ruTozIAA5FO3FAMmuhbwhc3FubjR7mLUkAJZEwkq/8BJ5/A/hXOyRyKxSRWR0OGV1IKn0IPIpqSYOLQm+Puwoo8pTzhaKq4j2TR/CUdyon1SRShbC28MmSxBHBbp9QMj3ro7y9h0yyNpp6xxSY4jTjYPc9j7Vyen65PeTzadEEZXhcRqDhlkCE5z0UDA6HPWq9nexSo5iwccrgcEHv71k466ml7KyF1LUi8+JPmY/LuJNZMt2EUlyST0FV9XaTzQw7HJA64rOcSXjqNu3acgdzU3uNR0uSy3ctwSFXaO4HOPxrftJZPI8vktsyozkVkQI8kn2ezgNxMchwuSRgjPy4z0P4cV2OleGpIl87UZmtocDEW4NI3fk9B+GcimtrA7HJR6dqOq3zwWto0rbudvCp/vE9OldfP4S0uyuory+mlu5wiA27ECLcqgemdvHTv8ApT9Q8SwaPbfZrOBLZeSFUhi3ueOv+Ncfe+KJL2KbCFWl+8+ST9fc/WqinYTd9jZ13xpJGWt7UYQDbhOFHbAxjiuSn1O4up97sSR681nyybmAJznvVmMJlnUZCKc56fWnETVjttMn82wiY91AOMdce1OmuHjuEjhgkmZzgbASR2HAHJ9qm0bS7udbdIbZxbzW6OrN91GKKTk56bj9fQHFb3nQ6JavbWEpkuZD++uCMZ9h6e3XrRLcRmvaDRbJ9W1NBNdx7StsrBktwxChm7M2SPYds4zWTeX0ty7yPIzc9Seafq5uJ9MvEjYl2iYDPO44zzVGNhNbrKpzld2PrVR6kSFgyWb+dSkA5X8Kr2+d5xnJONoGSfYVrrBDp5Y3iLJecFbdvux98v6n0X8T1ArTmsZuNx1tCtvpz/a9yrMVeJFOGbGec9l5Iz1POPWs67v5rkbN/wAqjCoowij0A7f5zk0Xd3JLI7s5LsdxZjkmqC/u/MLMAvLEnsKhu7uzSKsrI0bXypg5jACo7qqjoq5yB+AxSsqyq8jyGG0hP72QdSf7q+/T6fkDXgR1up1TeInEbF0XOMxjhc8E8fh3q9NdJbafDbXICxuRGqHkLkHr6k9yepNQ/Iu3UxpopPEV+uwrFp0A2JEnQgc4/wAfrWliW4JihQyTN6c7fc/kfyqG6u4NGsRHagsSuxIUUEnd1PUf/rIqml1LDZkySKoZMM7cDHOQQOvXvmptYNxjaWh1D92Z8lMm4VlUEjjHHPrnP861vsS29uzom+YKMFjk+gGTXEw6vc2avbwTLJGCWWQoQfmOecn3PtVoazeyBWMm3A5CE4Pv16/THSk9VuNK2yNK4sNUKkuFKt0UEHJ9KdHoVyQdxiUZzhs1mvqd/IhU3DKT0YDJH9KrSGe4yLt2lQcjErD68DAHepaXc0TNxrO2gYJPIm7sdw4xSCS2gYyxXDh4gWLL1JxwF4wTz+lYn2e35xBGRnOCM54xVhZWPHIx0I4xTVk7kvmexkQalDtlmlR1LO0h4JPJ/PuBV6G5M4jMdncEN0LJsB/FsVaXf/fIHoD1qVFJYDjk+vFSUQATbghWGI9MySZUH/gOf0p0Vlqd1Iqxw2pycZ8xyB7nCVYW3M7EKAWB7sAP1q2Ld7CWNjPhHJUvC4GPlPcg8cc4zxmqVm9RSvbQzL2xvLWZYhe2xfYDJiFiFyOMEkZ/IdvXiGC0ZSTLcu+eyjaB/M/rRHdfar27kcbW84pg4LDAAGSOpwOverLfcxjNTsylqtRogiOdy7gezMWH5HinBI05SNAfVVANOgWN5MSy+WmM5xn8OKbIFEjIuTGOjnofw7UmwSRLuBHJ5xQDUYQIxILYz8uG7UbtoJCjJ9BTAlDndgoQAPvcY+lOMmV2BiPbNQ+Y2MBQfc0F1TJJ6ii4WJlO5lG/GD2NbumpJNpWr2UDb1uLbkZxg8qBn33fXiuXE6ONwdQo684rd8Om3nmmR52kjlXYwiXcpHOQTggc4x/OnurA9NTJV2lQOFwCOBSxNIseWZBjqc4pLaG/iTYNMlcJ8m95okHHAOGYE5xnp3q9HDqcgBjGnRqerNK7FfqAgH/j1Gm4iqk25SVViQOPlODmp1ZhguuDSXKXDMUXUQ3HLQQbOf8AgTNmoksLdjtuJdQuAeqtOEH/AJDVT+tO4iRpwOXYKp/vEVXk1K0hBDTorYyA525+mev4V0elvAAlothCYQPvMiuwIOclmBY8+pqS4SXTruW4+0Q2cM8at5vkZbf0KggqDjGen8Q+tJtoFY52O8meEyR2lxjcArtGYw2ey7sZq7mdGG20EknUq15CmPY4Lc/41QubueaWT7ZcM4Zi0a7iAU/h64yeO3cmtbS4dNs4VuiXvJiNwPlEIpx2BGB9aetg0L1sviNxho9OtUGAuZHlbGO/CitKG4urCG4XUruGaCdDG0a22E57kEtnjI54/lWXPeahOqPFF5SE4DGVRknoME9aozWF8ytPdFlYchN4O78BS9WMqwaX4esLqSW1juWJfcI2k/djn+76e1X3a1uG+0f2VZySoMI8qKzLj0BH8qzYGEc4EkZKrz71u29kb2MyGaQblIwvA/L/ADmhpJC1uOt31KdwGkVF5bCgU5r23XKKwEh5kYNtyR26dDTotH2M264bb/dUBTj04qO70i1itmlVxEYwWLMSRjHfrSuhl63vYp5PL3DcFBK5zgH1rC8QtLp2pJrQika1QKt4FY/OM4ViOnGe3anaZZ3y3vnzsgifrLuxkY47c9Bx/Kulik2XEatGs0JOJY5OUYdsg8HBwelN+Ql5mCuuxX84gsYzI7AEkt8oB78da4f4spK9nptwWaSISOm4D5VJA4z6nB/Ku0u7C30/xPNaSrttpgJLZU+VVHdTzx2OOPvelct8UYpIfClkqhltxeKFVzlh8j4/r3qZaxGlqeWQgeSOMnOcD8akuwXjEhwSwGcd+/8AI/pUNv0bjt25qxMvmRFsFVLYCjkD/P64rI1MgjBNJUrISM96ipFC4pKdTaAHI20/XrTaKKAFWpo1LMABnmo0GTWlpsG+cfLu2jPH+fpQI6bS5DaW6ynBQLnaRkMMEc/jt/Osm+meWQMzeZJty244z/nj/PNb7pCbKHYQ3lkNIMYDZG4/yFYGpMVJYDHGASOaoDElYbyOoBqsfvGpZG3MxqKpBBRRRQM0NCiM2v6dEASWuoxgf7wr6aJZh82M183+EYzL4w0dV/5/Im/Jgf6V9JMFHAJ+gq4GU9yo6FWJqvMJlQmGNWb/AGmxWg8ZYYFVXjYfxYIqmJFWOSfLCUR4xwVPP5f/AF6b5/z4KON33Tjg1ODuAypz6N/9akKZP3QcfpSGVpoVmHzlx/unFOjREUKoX8qmZCTgkj6U1ymQo5Y9B3P0oATaD6YqpPZw7Gh2qY5Mlo8cE+vHQ9anaYRsoaOXc5wo2n5u3B6de9LBJ5xcAKhQ4ZXfJP0K5H6j6UAZwefT4d8rTXFoOrFMyRY9cfeHv1q7bXEdzarPHkIwypYY4q3NCjWzLHK6yEfK6qPlP0Oc1zo8H2TRn7RPPO5JJ3NtTJ9FHAov3A2VjjWMKoVV6gKMDNROChAYMM/dyCAfxpdP0iz0pibFDC7LtZhIxJH4n2qyylslnJPXJpXGVwjM7IybTjILHg/iM1nSfbULGK0Q8k7vMyeuOgH+e9auwZ4B+pppPscHpweaYipp8VwZVnnlhbK/6oQ4/Hk59avb5kOUmMWCOI1C59j60gnkGwSqXjjJKxliMc9cj+uR7UyCCWeQ4zIdxICKcgdgfcex9OnQAxHDdjxnI470HqMqD6npTgAygqxPpimNkDBHekAclc8Y9DUeVQ8Z/OnsoI7VF5a5xnn60ARlcngd6jMWTmrIB/Cm9yCDQBTZO+KNhUgkVYOBng1CWIJJPFFgGPGsmCuAfbvVrTdRutNMiRxrJA4xJFIMqw/z/k1XztHByPcU7z4ynCgY680wLbadYa0WeymS0uyDixdcKx7BHzxn0/QVz93ZT2cxiuIpIZB1SUYP/wBfr2rQmlt3jO519MHn86YPEsAhNkwivo2P+pKNIQemV2nIP+RiiwGSVHpTSmO341o3Wl3sp8+weKC1YAhL1sS/985Jx7EZ9zUMumlN6jVkc4+Upbc59CCQB9QTSGUGj4Hy8e1MMYwS/T6VpNp8PnqwmuWi4yjuoye/KgcUxtPtMbmgUlemZGbPHuTRcVjMUxxvuMi8HO3PNd1oF/NNBK0yiOAJjEq43Z4Axjpx+Nc7DOYY5Iot8UTn5lTKK34CmPMI8hHlweSozimmxONy74i8M6fdwxHT0+z3W3+AsRJ05VME+vIx06HrXEPFLaytHdxlJSflfsw6V0cl1MWVvODgchJRuXOQc4P0H5VZM9vrc/2fUoIonYBVkUhcj34xn/8AV6Cm7MLWOUMLgqx2lWHBDAj/AOt+NNjjUyMSSAvJwccDn9a7JvBiW8QuNOuVcAZZJyGRuOMHt17+vauakjktdQELp5E+D8r8JnHY9P8A69KzGMt7m60u7aRMxTA4bcuc44wR/wDqrq7vUtG1LSrKfUoDa3FzvUXEMOWJQAHOPvDkYBzxkDHWuJuHluLh3kxuYlvlAA5OeMcVqTyP/wAI/pETYGFmf3yZSP5KKpbiaNnStEu5rudtO8vUoJbWVFeCVVI3LxlWIKntg++M1zlzBJa3D200bRzRnDKwwVPuKS23WtwlzbyvHOhyJEcqw/KtpfEcF1GLfxFZm9hzhbuMbbmL3B6OM9iO/HSrUmtTNpMwV+U4zxRIu9MAHGa3rjwtdPDJeaPLDqdgMndAcSIP9qI/MD7DP4VgpgoWDBh1HPWtIyUjNxaLVhqNzp9ylxbyNHIvcY59j612Np4n0nxBts/EFqu7YEW7HDKf97qPXByPUiuCLEZwtJvO4ggjp0olFMIyaPRT8PbeRi8Our5Tcpm0Ynb25Bwfworz3e3Yn86Kjll3L5l2Ow0TVDHr1lcsPuzJk7eSMgEH8K0dJU20U1q5HmwSNFIobOGU4I/MVzlmjyXUeOAWCgn1r0LV9NQ+KZvs0M8n2tVlBiRipIAU4wMfwgk/7VXLRL5j6HOyR+YfMbOey1paH4YvNUcSAeRajhpWHX2X1/z34roIdHsrUiXUcS9CLWP/ANnbPPbgccdSKhvdZuJnMODFbrwkcBChR2GOOPzqFFsTlbQ03Sy8PaPePYxB57VN8rPyxOATjPTgcAd/SuN1HxFPN92RuSD8x5wefwrV028D6zHaoG2TBg+R3xn8e1edrdeZZxEHlkVTxVp8rsTbm1J9Xu2mm83eWGMew+lZqSbeM9+adIpKqAO9aOh+Fr7X5sQAJArYeU5x9OKm5aVjLjDyP8vI7HFdroHhGS5gW71Pdb6eQGC9Hm78ei+/5eo6G08PaR4W2yyql5dDmMMMJGfUjJyR+X5U281We9ctI565HPyr9Ka8hNly+1bzEFtAmy2jAWOJRgADpWerF8gjGKYp+bOSexNKXwDgnFUZkvyhfUdwe9ZVlaGSZrC1Tc8WQewVB/Ex7DHc1o28E13J5at5cIyZJ2+7GB2929B/kx6t4jgiga001ESBsM0iLhnI7seMnjr/APrDC1y9GbDRNNv5LdvtGowwmQzcfIuQG2A9MAnnr9K5gXU91cmWWQszjuT2pLaVZCwlDFZlaKUAkblYbSKWG2MWVkYM6HaxHQkGhIbFXKvyc8/iTU0UcDyO11hbNP8Aj4c9Ap7e5PQD3p9tAZHeRhIII/vugyT/ALK+rH9Op96ct5Fqklnp1pbSQWtu4kZHByuPUnqx7k5JqGxpHQ5hSNDDCsUaKEijXoqjp+NYutxPc2JCnBEiMG9CGB69uhqxeX8cETSyNhFB4rIl1a51ArJbwOsEa7EWRQCTuJJIxnr78dOetK3ULlG60+51C/e4iutilyQoTOOnT04FZ13YjzDGd0vljOd+dq9+vf8AxrpLZJv3k0rEqikthSSfoO/4VkXd3BLBc3KiWN0BXyid3Of4hxjnGQQw6+lF9RONloZsURJaQBdhxg784GOnoOB2qcKFwVYEevpVaDesaq7lgPu564qYMm7hgvPAzk1BslYtKUI5yWp2Nq7fWqrXdvDnzJ41P+02Ket5C4Yo2/HB2ckUBsWMoPvkKPWmieHPyuD261Se3gkjAfT7iTcwKySQnPXszDkce4/OlNi8T+YNOA3HIWWXDD32Mf8APalYG0aO4HIGBTl7cHjvjiqym6MWT9njYkgIXLEY78DHr37dqnBYKAXYOOq7MA/nTsJssRMVIKjJ+lTpIxlhZV3TKxKk9B8p5PI74HfqeKyhHdtISbsKv91Y/wCuf6VLFEQ5aSR5SOcPgAdscYz1ppobWhThikt7+/WSQN/pLrwMcg4PUZ6g9asrKCwQMNx6AHmol0uzQDEb4BJ/1r459s4q5FFBBkrFGCR1UYPbuOv41KG79CIqwOwxsWHUEYodrpVXy7XeDnOZVBGPYmp8jdxzTh6nFAkmMhSR51DC3jTAJaR3Iz9FXNaBhsrMoXi8/fHksp2jd34JJ49e/wCBqoi7n/dqWY+laNw4XSmidds0MikZX7wYH+Wz9RRoN3M1jDtCiN2Yc/e5PHTjA96m84GPYYYlO4txnjpxnPIH9T6moVJIyAOec0j7gOuPXIo2AdHKLX/URxRszfeSNQ2f97GafLeXE3+umlk29AzEgfSqyqCOWP4d6cAirgc9/vc0egE4kI+7kk9qb5jM+0uAR69qiRVAwvp0q5bw2/lq09wcHOUCc5+vPb2p3YEY+VmAIbHcdKs209unLLJv7njH5EUyKETO21nijUZ3kFv5D/CoRGQu4hvTPT/PShMRow6owvhMIgcDC4PUf41pXUh1LR3aOEoYpN4+Un72B1HcnAAxyTXPRxkMAvP0FdVp1kfsfkXE5RJDjhMMCT26856HBoe1hdTjLhPPBhfdtbn5TyPermjzRRMRdySPAv8Aq9p+/j1q1ZW+neXO13MI/IlaFt8m3aQcYJzg/iKgNnoEj/6FPM8gOSLWN5V/ERilew7Ehuo0ZvJ1C9Xb0G7AA9MCni8DQhnlLMpGBuOc/WrVv4dVrcuqxMzD5ftDOpB6cgE4/LNNg0bUoIXES2SzMSu/zmKhfUDy859sj60+YGiHNih3ebKZhzgjj9eatxavA+3yvNYg4bYuR9M8Y60w+HJFlDzXu8AcCODaQf8AeLHP5CpV0nTZYdjG5kkz8zyS7Xz/AMBAH6Z+tCYmSzaqwhzFE+eh6HA/DNUri4kkZGMgt1U8PcSDbnnkjjnjpzW9Hp8dqVfyLZoHz5DiLkKDjDMxJLcdc9+Kv+b8u3PA6D0pOXYaRjzaRclUe4EzMACp27c/gSD+tSIlxbMVitZrgfdLtKgU+4yc/pWh5u7Py0oIA9KV2OyM66s5r61UuBbXURLQSId5UkY5JGMY4xj+Qri/iVYTR+B2a5mMs0d5GylT8uMEeg55PSvRua5j4g2v2nwLqiiLe6RiRcDJXawJP5A/hmlK9gPn6LO0461bBPkkDP8Aex2zj/69UoyQMliAK0FjUwt1yBg4HHU5P5AVmjQpom5j0IB7/lVW4QI/AxV3cULckEnn8KoynJPFDBEdBpaSkMSilNJQMuWsW/Ge9bOnoY3O1dxPyjAz25/Q1RskXPQ4xgDrmtmxjEd2jsvAO847Yz/iKpEl67ugkRQgM+FYYGMHPPfkc9T6DGOc8zeSgwkYyc+v6/yrf1FpDLLG6ndGcK7cnqeOvqP1rlrsndtP4D0FAFUnhqZTj936mm1I0FFFFAzsPhnbibxrbSNu2wRySEr1Hylc/huz+Fe5LN5UqwXEib3XdG6t8sg9RnofY+h69a8q+D1jFNqGp3r8vAkaKOf4ixP/AKDXrbCKWMK+xow24K2CoPsK0hsYy+Ihd99wNk6lcfcQEt9eKie0MzjC3Cgn7zTMoHHcA5/Sr20sAuTsxxg08thcEZqgMZbK4jZtroCRgO0jv+hx/OoBZak0pEmpIsG7A8i32Pt7HLFgMfQ1uuARkcVU83cXCqx2nHK4yanUZhx6bDdXMiXlzPNPbNskg88qobHDYXDc9QM4qddNsrGWS8it2EnJ3ZZ2GRggck9K0JDNNCFkWNXU5Uj5yn44Bx6jocCqK3jQ30tveMisHAQqpxIP7yn0Pp1HING4BbwrexJcysTE6gpCegH+0O59ug9+tXFVcBQBtHYcAUoiMbEoAUc5OOxpxC560gAL2oKkd+PSkye1ADHqxJ96BgVzzUbyAfKXUE8DJ61KQcHkVXe0gMyTNHvkTADEnjnNAhFjlVyGCt7jin7AFxkAe5xTGu4wxRkmBzgYiYg/QgUvnEBQilgxxkjoPcE0ANeFiBgD86iZ2ifdyrryrL1B9qshhySuffFMl6Zxx6ntTAb9sg1C/CzXKW9yEIPloB5x7Fx6+4H16UtzE1vc+Q64kChvlIIIPce1Z8tzZwyK0lzAJM4VR8zZ6YwOaZJqMUnEdpeXTYIwsLoOfRjgDp1zQ/MC9sDDhSP6U3Zjtk1Qmj1tGHkGExONw+0SBXj/ANltoIJ+n6UhttUdvm1SNEHaO3y35lsfpSuu47GhnYMYH0qnPdwQk+ZIEIGSWYAAe/NRrosDMGnuLy5IO4ebOQPyXAp6aVYxZ22NtwdwYxhiD7E8indCEiM13GJYY3kiY8SKpKHHHDHj9albT5AH8y5t4SOBl89fZQabsAyEAXHYDH8qj+V0IDKxB5welAy1pnh9buVvO1aOUYJEMf7tifbcD+mao3dpDbXP2O6sZ7eUDP7yYtuB6EFcD8qlzhwcMMcgDjFaltrIkhFrfxpd2wzyy5kU/wB4N3Pv196LAYQ0yxBz5CNnuw3fzp0aIY8W4UKf7orbXRFkhlurC8+0RDBEcvyugx0PYnPqB9axWyxDK209doXH5j1pWARYyoyWzx04pVCkjcvHqeaUkZwxwcZOB0oKKVwGb25oYBJHFu47fhULKCM5I9DQ2A33smozkdKBjXjPYk1GU2jGM1Oqs65HakeKQnigCqybhhlHJqFraMIQFBHoRxVzyhzvcCkKqBwy496AM5HurGZZ7SRomU8BTgfQ4rch1DT9YtPs+pQWxnPuY2z/AHhjOT9MfSsic8BUUsT/ABE4FVJ4FzkcMDwwPIPqKaET3vhi8iQtp063sC5YJhlIPQ4HfoKyPMn8uK3uVKTW6sfLZcbQWLfj1Jz710Vt4t1C0tkgmXzkRSvmrkMRggFgDhiOOvp+Nb13ouk+K7BriFDC4OVuFBV+fUHqD6frTF6nnm7Jyp681G7tuqxqWn3eiXfk3486JPl8yF8Hpx1GR9CPaqayhwWTLJk8/wD1u1UpWE0aWm6xf6XKZbK5aFzgsygEnHTrXQ+b4b8Swqb65TSNWC5e4iUJDM3AyVOAT64weucgZrjI2y2Oop/mFWODx/n/ABP503ZiV0amsaJe6HJGLpVaGb/U3ETbo5PofX2PNZyMMnHBz6Vc0nWbjS5d8KQyRnO63uBujYHII5Pyg55x1raXQLTxBZTaj4fzHKh2y6bI+5kbvsbPI9A2CefpVKVviIcb7HPbc+lFV5LhopXjkglR0YqysjAgjqCMdaK0uiORnueieFrDw2n2y6nN3eRpt2KVCI2DkL7kfWrN54iIRliTyYzycYJf+p/GuaOrJMx2sAQuSfaqU19E8TlJfkAwGHc+1Pk7ic30Fn1/zLiSOQsq/wAOOM/U/n/k1SttUEl827jK43Zzmsm7vd12V/gYfjjPf8RS2k0bfLEjB1/iq7oSXU6XRbyQ6tZTBGdxOIs46DOPr1rl59007xsdkkcroxOdikMeATxXQ+Gnmj1W2iUAuJwQB1IyMnFd95Gk6I0jxRwLcODvRIlDuevzsBnqe/6c1lOLumjROyOF0Tweb23iv9Rf7NYgkuZFIaT0Cr1555+mAc1vahqiQ2a6dp9mLbTVXBAbBf2b6+nvyTUeqavPdAKWAVeQBxt+mPrj8/U55+W4OFUNkE8Z6Z/rVKJnKd9Ebb3v2/TrW5kJOCYuAe3I9ujD8qhJGOm0DrUFjhdJtyI8YvJI88AZZFOP/Hc/hVqVoo1CMwG87R6sfSlL4mVYYHDA5cYXrg9PrTnHlRC5vHNva9lx+9k+g/hHueT2HetNNJfSbeTULoqWZlWJGxhM5IJ4+Zh6dAcnmuH1/Ubi/vGR5WkAPVj05pXuFtTS1PX47lIrS1jENop4jXPJ9ST1PvWRuaW5aMcqfugYptrErlFYkqBy3fFXPL8rywgLyscIq8kk9gPU02yktSaJ44FIc4J4Uk1oaRoNwmnJNqCvZWKfJmT5ZJAOAFU8jIHUjpyKuSafp+hQpfamgn1ZwTBDuBSA44Y88kdc889PWuUu9e1HVQttNeSy26cLvbk+59T7nn3qObcq1zqbrWYvtFtY2e1IgDCscZIUA9CfU52kn2NF3cptDRgBmUcgYrjtJONZtc5Ko4bA64rpJIJJbX/Vyi1jz5l2q5TvjB4BHByenajS1xSVjm9SW71C+kWONjEAFVt4wcjrjvk+npV1bXUQiwLdxwouBlGBJ/Q9/wCXfNUb7Xfsshis7ouqqY3QxFATnpgEjj1z+ArPj12+XcqFQpHQoDilzLuSqfU6yC3W2TzZtQupOOjsMD8AKrz3Omoku14pDL9/hST7ZxnFcs089w2ZZXb23cflUiDHT9KXMacl9zUe4s/uxabZooIPEQOcfXj9O59aZNePcsDIsPy8LiJRj8hVRMZGSwXPJAzxTs44IqSrFkTzN8xlk3YI3Bjxnr3pN58sIXYqDwCeKiBOcAc/WnFTnNFwHKheVVyvzEDlsfrVi9ZZb+Qx48tMInbgU+wWKOQ3U8kRjjVmEbrkkgdsenUZ7iqVuWjt1Vzlsdc5psESHGDgjI7dzSo2QQR171Gxz0pVYjipGS5AHJxjtRuyeKaVDKSUOaURycAqw9BigCXcoGW6Um5d2MUxopepBA9zUiWzNGX3qoUZJLUICRoJI4kkeF1R/uuVIDfQ96YSOcHiq5uYN6L9oDbjhSqs2T6cA9ahS+V32mC5TjOXgcDH1xj9aANO1kVLhSWIXgE5xWxq9xHJZW8cZ5QtvwTg5xt9s9c/UVhQ3kkEgJ0ycDHPmuqA/TDE/pRfPJfPFP5rQzKzlkQ5GCuFBPfkZ5HGT16U9LCJVWR+IwuQMnjIAqJn3vxg0IFIZbqa7lX+FYpREv1Iwcn34qGeEEjyLi8ij3EhTMpI/wCBbc449aAJ1BbPKqFH+RUPmQrgmWPB6HcOf8/1p8CQW5BFrFJLjHmTZkP/AI8cA+4wfepf9FMpk/s2zcHGUdCyZHcKTjP+NAXKct9BarGZJeZc+WoBO7HpWjYi+uXkkbSdQaGIFc+Tt+b+7kjjr9avxa2IsILOKJBgDyvlwOmOOlT6550NnHfW5LbEE33vkIDDgnOQMHr70K63E2L/AGRqciFiunDcQQvmyEp9cLtP4VK/h29uEKzXtrAnXbDbMxH4s/8AStS2uGkXdsJAOMjGD+tWXkZSNvOWAxtJ/wD1Umn3HoZtroVvauvz3UzKGJlMu3JK46LjGM8Yx+NOj060SdW+yyllO5WlmeTB9QCxxWoA5GSwHsBigoN+4t07f5/Gi4GcyWa3LnyrdbhyMtsG44GBz16D9KcZJZy2y6QoOCvBxVqW0t3ZmkhRy3BLDPFVxp1kkbxxW0cQf73loFz+VCt2EVfOgUtvvC8Y429CT9Mj+VTQXtureWsxYDJ+g9zVe50SARs0bsjY4DEVWGhSyR5WQcE4w3X3BHSquuojRudRiVSQSc8qSvBrKmvVnLMzmNSMKR1B9qdL/aVtG8boJUIwd3zH86zGjV4y7sQc9F6j9MUJroB0d1fww6BKtwjyqFEkcsXKhu4I/wABVbQr6W4bariSErkMSCR3H1FULC4xbGMzyBA2Anl5JbPBz1HP4Vb8O2dvbPPbxptktGMWC4fCkArgjjpweOoNDGjot2MZGfel7EdRSEH1pg3Bzk9faoGLuAwFOcDBzyazPEKmfw5qsSKS0lnKg+pQirzsqxM0reWg5Zy+P1zWXJ4h0+KUhriJEBIMkkgXOP7o5J/SmotoGfOUZ+brgVsW6jyHOARtK4zzkg1QvIYrbVrqGCUSwxzMsb4wHUHg+2RzW7YYks5kxvjVSxOOhP8A9cj8QPpWSLWxh3QKM4wBt44/z/n8azXNaF4H8wkn5T2Pas9qGNDc0GkopFC0d6KKANjTid3c88/4V1FkqHLyYI2jKk8/l37+/TvjPK6e2JO3Q9a6+DK2yyFeJF+Vd2M4+YD65x/npaJMnVI2fewbaFJGAemTx/SuYuG+faGyB/n8q39QkQRgLnJAYZ9+o/LH4iudfmQ49eMVLAY3YU2lfljSUigopKcql2CqCSfSgDtfhfq8OmeKTHcTRxQ3ULRlpDhdw+Yc/gR+Ne4g5+bauO2a+ZhYSJjzCqHAYKx5IIyOB61p2Gq61pe0WWo3kKK28IJCUz7r0PTv1q43RnKN3dH0USWFIW3HBHGPSvOvC3xFMka2mu5EvOLtE4PsygcfUf8A169AhkS5iWaGRZI2GVdCGBHt2qkyCYlQO1QSsI2ClsluQAp4FT7SwA6VDPCs8RjbdtPocUDIjj1BP0prOYx84DxZBZGGVPuaaY0hgVGbCxqBktjp61XudSsbIf6RdwxAd3cDApWbGX1hin3NYMSqjcYHfdIP931H6/nVdWSThT0PIrLj8QafCPNtmnlbBZWtoXY/XIGKLhLzVyL+zLWVyww8dwrRB+Sd2ADyQRxjHWjQDVAweaCR0Hes6JdYdSHlsAuSBIpdywBIzjCjtnr3qzbmNoz5m6eT/np5pA/74ULj6EtSGTbO5PAHU0iq06EwJJOo6+Shf9Rx/wDqqwl40LCQCGNhyGSFUI/EDPeka/aVjvlkYtzuJJ/WgANjNGgLmNRnHzSAnoT0XJ7Y+tMgs3kcTPLEIMNhFYCUkdPvepz29PWm87sZ4A9KRkEiMjZIYYNPQCBrdVnVp2vYmP3YJ28vOP8AdAJHuCR71XudOs5pTKbWN3zn96WcD6AkirUNrbXCPDdvNDNHJmG7BLYUjOCM4POevr+NS3kD2CyTSP5lovWcIQFyTgN+GOenuKGIrwQQwriGGOJcfwIFz+VPKD+8ce9OZQRuTGOuR3phBZTsHzY4zyM0rIY3H5dxSGIZ44qVmSNAz5H0FDR7h8ufwGKAIfu8cdaYwJ4Az+FWjbPjqAccZp8dvtUbirN6gcU7CuZ0kRIKlMjoQR1qKHTliJ8qMRg9VHAradBt5xkVGFAUAZ4GOaOUVzGmDR5BViR2ApowPvDBPr2rVng3Dpz2NUZUVZMEHNDQ0yGG5nt5w8O9XHO5T29/b2PFGpxx6hCziJba5DBjJDHgSdPvgH27U84ySrmqs15bR8yzxqQejPQMbPK1hbwyXkRjhlIWOVsAMSMjIJypwM9+/PBoVklB2kH3HNRW+o2U16oihS9lGSIzB5ox34weKu2OhRvDNctcpYGST9zbsoaJT36MdoJHrxzkGkBVihdid7bh9MUxk+Ypt+Xrk+lNuPtsDYHkyFcg7HGG9Mc46++faqc32+QH57VQRkKNxx/KnYZotIoA2gDA7Cq5uGHbn86rwxNEo864lmPU7cJ/Q/59Kgms1mJ3b8EY4kYfjwcUCJ5r+FB+8lRPqcZqs9y7qWgtrmVAMl0ibaPx6VZtIbdLdoGuHscrtE0Ee4yDI+WQAgleM9ycVDcT2uZIjcxsw6SRKxHHrkZ/GgZVknvWAC23lgj70rj+QzVFmndiJLuFM9kQsf1qeeymRSztuHZ85qptw45OfSlcdhkip5EiM7yF2B3E4IwDwMdjn9BT4dd1KxthbW1zJDGuQNrcgccZ/Ckfa3AXPue1VniIzlhwe9F2tgsuprWniISuV1CFHyxJkVQCc+q/db8h9atXXg6GW1F5pt6gaUHggCN8npzjb/PNcuVBNT2d19kkEiIpYEfeUEH6g8GhS7icX0K8kF3p181reQtHKhOe4I9QRwR7irO9Sp2kV2Vnc6X4qsFt7zTraOVDiERyFD79OQOT6jrkCue1rwlqem3U81nCbm08xvLEJ3EKeRkAk9P8+rJ9TLZudo/PHWl05jFPNKrTROD8ssTFCwJzgkdRkf5wKpC4PmMkiMjD+E8c1qNtj0iyuFgjUtNNE0gYEvt2NyO2N/5U73sgsOktY55Hlkurgu5LNmQ9Tz60VD56/wB+it+Z9zK8u50kUztKY0BMQ4JbPTr1/CnXlwn2NRuCkHIU9Ov69qLS2ZpmjJVAF3Fm46dvrVeSB3kEcUYPb69ea0b0IWpWYiQg4YjACnGOOvH61teHtMutVuFit1Xrkk8BV9T6DjrW94e8BXB/07V3e2gUblhX/WEf+yggYz154wa6C61Wzs9NFhpdv5FunU5IOf6n3P6VCfYptIdaW+neHEdrQfatSK489z8keR0Qf16n1rKuNQ3licAscnHGTWY2oFrhdzHPUCoHbzCTgkdR71oo2MpSbG3N0XnA52jqT0omICIRyF6HGT+HqaXYjrtwDgk4PrWjo+g3mrP5okFvZocSXTj5R7Lnqf096bdtxJXCws7nUdBurOyjzdLcRzw5LbAeQ25gCB8pPfvW00djYeW8apNqm3BuSMiMdwmf5ik1DUbPTbVdM0klYP8AlpL/ABSt6k9+g/kOKx4p/kCO5djyB61lKV3c2URNUlll0u/EbOz7Ul+9yWWRCTz3xu6+tYE1sxvHByT/AHuxropIY5LW8SWMHNtKCF4JwhI/HIrPitpr+e1i0aHz5pY8r/dRemST0pJjfQqW/mLIsMcJeaRsIo6n8q159UXwtP5UJgn1ZvlmaRSBbBgCAoPXIOc/TtW5bR2HhWxkkjnW71FlxJc4yqeyg9s/X36YrhtXsxqt4l/CJRO6FXZ2LB9pHPPfDp19KTk+g9iC8v7nUXkeWRpJnPLE5J6f5xVe0V1uDCFVpWJABOOa1tL03ypDK53FeCcHArasNESfUBDawzTSsUMs0Q4iRuTyf4uw/E9uVazHzdBPCnh6a5uWl2ful4eZlPznuq+3bNVvFd/a29sYbeURKxZUMcobcBxkAfdHfPOetafjnWzpmh/2dpkT2sTl7cg7QNvQ8HnPB5989a8mWZjIWIEuQQScnnHXPqOv4VDdwsTCMADy8sAe/Y1IEPXHPvUluJ/JSVWkBUcEE5Hp9KkijO75iD7McUFCKcrjGMDr61Kh7UrW0wKgeVyCc+eh/keKPKlQRMDCd7EFRLuKY7nH1GKAJBgDOKeuGHYdzzULxXBYFZIgCeQVJ4/OpSrFAPMbdjHy4x+RzQBINoJ55pSdrYyeuKhkRgd0KohLZCkHaKkZTtGYY3cdB7/jQGo7+0FtrOdTK0YdggkYheD1GepyBjHTk1WGpQSf6p5JhjloYmkH5qDVwbgAoOAB0HQU8qeCTQw2KqTvID5VtK/OAHAjJ/77Ix9TToZ7luHtoYmzyryMxUe+BjP41Oq4OefanEgEAA5PtSsx3B3nMYEUsMbZ7wknH/fVI6NNkG7uEbjBUJ+PVTSnIB9aQhsAMQPUjvTuIiurKJ41V7m6kLllDST4/RAvb19PwqKPT7ElWESylTw0jNJz/wACJp12xdVTzBkShvLOcYAx+fWpI5lUgHdz7E05KxMZXu0ywxYvuLHPWgOVYHJznIOe9BI4o3FUxkHFIsc7Fsu7e+5j1pqsj5VSC2ASO4H0pgjt5G3OiP2G7nB9frToYo4kCxqFT0FPcRPGilwHYIpx8xGakkFqp2xySM3TDIAP51A+cgR7c45JPT8O9KqHAz8xHfFAD8KGIHIoJU4H8I5HNNVcAhQNvt0owMGgEJzkgD9a2oBDfaeltqaM9oMgKpAZuPU8cf159RmQW7zOsaZZmPauk/siBdL8uRW+VkmkAPJ28kZ4xxxmi6E7lWzlih0BXjyvmJwzYJzz1Iz/ADq9o96JLbax+ZTtPHX3qtDa2tzY3UNt5irFczBVeQuVw7Ac/QDisyEXFnOfLuYlkPAjMikk/TNF+4HXhgM8kntShyVJbAAJ6Gsq3j1NolkutxRgGCDajH6biD+NSlr0oFhs256GaZR+e0nH+eKnQLl926ZPB6cUGVUUlu3eua2a9eSOJBbW8bbeGum2xg4xnEee49ai/wCEfv7wEpqMKKSP3iWzlhyem5sHgDqO/tmnoBr3Os2IG1m38YwF61VTW5JJMJbsU9EBbiqsHgyLcDearqFwccqriJT/AN8jP61rR+H9JQENYxzliCWucznj0MhYilzLogsZ02tC4HkgxKTwA8i85/GoY9MunDZjkGACfkIB+hPB/CuhSNLfEUMaxxgZCRjaB+AqYOVxkc54p3YGJZ6VdxSKTFIFGfvuuP0Of0q1Ppd9OIZP7RFtcxR+XujhDq6jlQc4PGW7nr14q9LeIgJznb1UdapLr1o1wYS7BiQAGXHX070JSYXRMltemJUm1Isw6vDAqZ/Bt1IdNWV8y3d64HBXzigP/fG2rIu42kKDHA+bJ6Uz7XAOsiqvYsRzRZhcrTeHdIupFkuLNZ2Tp57tJ/6ETn8asf2XYxwMtraW1u5GFeO3TKn8qfLdwwJlmOOvHSmWt6l1E8sZwikg5pcnVoLnz/44t2tPHGqoxBLTeZkADO4Bs4HA69KfogM0LQkFhuBHJ7/yzgD154qz8Sst49v2P8aREHPUeWtUtBmSGWQlSXAyo7duPf0x6fQ1l1LjsVtThAViM7lYqx7Zzk/z/wA5rDYcn610+pKwW4jIACtuw2cknt9OvPvXOSjDDr1702UiHGKSnE5oqSxtFFKPvCgRs2CbpMH5VPfHbPb/AD2rrnJW1JWMDMedrHGOTz78cf8AAvSuY0OMNeoZASikO3PQCujjui9q00xbaWZlAyOnTB9j+X482iTmtTOC7AE4AG4Z4xjH0rGJy+49etaerlfPJVuueD29v/1elZXRD78VLAjooopFBWrpoSKJpXwN3Qn9R+RrKq0QxYqPujAPoMCmhM11ukjRshGY8k7Ac/T8qjgvF3MixsynnYACP8appauEyw4PPB6D3FWbKWCJmjniWRCeOeVP+NaJkGraXS2txHMkaqQQfLkHB/OtWHxfe6NP9ut5gWm4eORiwYZJ6cd8/meeawIbVnm/cEFOu0sAaNXYFNrKrMoC71ULnHuPvdufepkM9r8N+JrfxRYvPbFbaRMhxL8yp1IyR649P0wTeurLVBsVtQiikYkGNIuQuSM8k+npXgfhnX5/Desx3sab0+7JEWIDL3/xHuBXv0uv2k2gwXdixmfYJEROVde5BONuFzkYzwB2qoNvRENWepA/hu0mtpBeS3k8rOCjtdEAL3G1QBz/ACpLTQ9PsJVms7GKKVAB5oyXznOcknuB0rSguY7u0SRGRlcbvlORSGNyco5UenUfrSce40yrJdSW8ZZRhRkcHAH/ANaq7uX+aIHf3IOR/wDXqy8UgY7lU9+DzRwBkDnPakBWkiM0a/vQuOCFytJEwRhGiHHqTnNLeM5jHlyRxsRj95/Sli3PbHzXUswPIGBj6GgZIfMJwEDA9xxigKQoHAz7UqE7Rncfc08KCPu5H1oAYCAvUHHGc0YDdcU/ycr0PPXHFIke9tpwDn+VAhMdBmprWdfJmgniJhlG11RsHHsRSeUqHDMM+hpzRl4lKQzsjkBWjRsHnjkD3FOwEU2kTW1uZ9NZ7qEDdJE8qbhn05AH0OOvHpVSCVJNwMRRRz6EZ9R1zWpaG5t78h0e0iK/vJHUHd6DaDzyf51PqNlpk80kkAkjkZhuaB8buTk/MpGee4z05p2FczkWOIYB46jcc07zY8biwx0znipZtO0zyCiR3xbk5N26gZ9lxmqyWdhbybl06J3BBDyZdgQRg5J68D8qdhXGS3ttGpzcRjPGN2T7cU0SFozKIbnYOdxhdQR6jIGR9KvC9A3YjhDM24nyUJz3OSDzxULXN1LtU39wxBJ2s/BJ9qoLlGXVfLAC2l03XJFu5xxnJOMdKjefUW2+VZHBPDGVQPryc44649PUVfkLyKUfLxK2dsnIz1yBzVXypoY3lLy3Fuo3ySM3zKSeuDzjrn0yOKVwEFnN5uL7VEVG+7FHGH5z0ByDmi/srOVRFE+oK3G6QtGV/lmlijJy5dSx6MOmKlcZQj9RTEUgba0XyxZRy+rTszE/kQKy206waZpTZoCxyRyR1z35/Wth7Z3BDEY7YHSqxiCMVfrUNIpMgJKP8iBBtKjYoXg9RxUcmSSWOSe9T7dpxxj2qJtoB4yaW2xRVlKqM4qjPdbW2gDnoGbB/AYrRkYZxzu9qjIDAFsZ6jI5FFwIlUlAWxnHQVF5qtdCHa2488irBOW5yRTA4Ukc8H0pAMkCKckZI7jmommka2lt4riSKOQguEPDYPGR0NTOAy8g+1ReTGpyBg98UDKES/YYJ90sIUgsFkyuQMfKDjHfgcd/aqsQ+0je8OFPRlb+hrXljikiZJBnjqO1VJoEIUiNA6niYAhz9efm/wA/iAZcqhAQCTnp6Yqm4JY5B4rXkltpUwxEUg4weN3vVWW0YAFVIHr2NS0UmUdoxnj6UGNWBOOlStCxONvfHBqIx4Pt7GpsVcdbtJFJiMrzwN+MDPueldLpvjP7FaxWs0KNEmf9VlWXJyevWuXeLaud2efxqIrnjHHse9VGTiS4pnc/2To3jeZpYriO0lALNthxIR6kbsHt0/GsjxF4fmsYre1giik2HG6J+HY5O4qehIwOp+6ME1ladqV1ptys9syh09R19j7VsprX9ot5VwsQnfJyclSc9DnJ9Tx/KtI2bu9zNpo5YwXCsVaEgjggqciivRYo/EBhQxTz+XtG39xE3HbkjJ+popa9wvEsaV4f1HWLkyQMsViG5upAQoJPIX+8f/1Eium059C0IMbK1lvbtOl1Kq8t7A/dHt19fWjWdaeZv3W+C2jUbEzhjjHJxwBx0HpXJ6hqDrF+5xgDooxj2rqs3uc/NbRHQ69rt5danZIWBtJbTzNycAyBsODzzjA6+tc9qd15zxpC0gQZ6d/0oVpJ7axkIOyN5kJJ7uqED/xxqI4DlWIxjoKtK2hMt7kUNqXmClzsxkkjH9asyZiUjcDjjk1NCs0k32e2t2nuXOFROT/n3rpLfTIdAjS5vhHcapjckXVIPf8A2mB/z3oc0hKLZV03RLaCGO+1vdHE4zFagEPL7nuFH+e2ZrzVDeBcqqQx8RQx8ImP6+9ZV9fyXUkjuxeV/vOeTn/Paqdq+GKOxCdqycrs1UdBbiYyzliQf5UzzPKC7Tt24xikIYyFsZyementV3StHbVy9xPIbfTISTNcMMbsDkL6nHfkCoNC/wCH0fUpGkCr9niVjcyO2FAwRjI7/wCfSmyXeneHNLOnaSCIG+WSVzl5fqew5PHuar6z4kt44l0nSo/sthGAVVh80h65Y9f881ygmmv2ZmOeccDijcRbvL+W+kCqwWMnBOfzrS02SAWbxyjMhn/djb/CVOfp90e/SqDWghjQY4LZ981BqDlWitYJ9ly8oLHef3YwRyBz/Ef/AK9UiW1Y6STELWyiZVS4DK8u0usRGdoIHJJKnjtx3IBj17WLjQ9M+w26yQCUGXdchkkcEn+6Pm5AAY4HP5VvC2pWsflza3ci1sLFswIdw3yepAycjj8foa5bxZ4hn8RaxJcNIogU4jwMDHA789AKU7x2COquZK3l05Je6nPOcGQnB/Oo/nJyWY/jSouRlVLd/bFSBDuO4AndwBzketRq9zQeECoy9XzjIOanTaijks57egpuzgDIp+3b8zELjqxPSgC5Z2fn3MXk+ZLk5YMgUJgZPIzxx1/TtUkkyT3Ekyxqu44G1QucdzXRRRWmieGGmEgkvbjAiIUoRnvyD0xnpzt6c1zLMjAqiqq+iDp7U3uJDjlcAkDNOB45qMYAFLyGBOR9aQyVsEdcUvyjknFNPGTgcdOaaInYhjuZR/douMmCgHcME1Jkgc9KgVjuJ3cDp0/nUiyDad2OOtCEAcMOD16eppd4UHcDkDPAJ/lSocMcDIzS+ZyedoHfNADPMCkKfl9mzmpFHmHGSPWoklWN97GPaDn533A/nUjaiLiFk+0RLEflAj2qvoOnuRTswKUitcR6eInCsyu0xYgY+Y4GMcgDv15q5GoQHcQTnjBqreW9xYXhhw8yLGoQQfPgHHTHarCoyD5vMfBJBwo+nU9/p/WmyYyTRYjUvn5NxPA+tLNavCW3BSAdpOe9NGs39vIRBpdkpAARmlbdz1JIXr+NV7i41e4cNI9mpxyFiaYn8XYe9RddytR7YDKAvPPOf/rVbh0+8lXf5DBMZ3SYQY+pqncWtysti13fXuLhXYQIqwptBIz8mMgkduMVoQ20AffFFAJgAN7KN2M9M9TTSC45LBnLLHcW0rp99Ipldh+C5qb+y5EKkSEkjKoFYMefQjj8f65q7eXkptQrzqX7ovGefpVSzltA4+1LK2TjduOF49BzTSE2W2uTLC0GJZDGC7hpRGCPybj6CmRKiqJEstrFdwEkbNjgYPJXI98CoryxeAyPCjxxg5DFhnGfSooXleBo3uNsa8hXPU+uKW4Kxp2c1zcTbBdSKo4zDbBBn6Evip7q3uY4DLLczOCCpErBRz2woGaTT5Vs7FpZ7ldrncqoMH/PFU7q++0SB7d5mkXOC+MYp2YiumnxXLyPHBZoDjJlBKscYzg5GeOuK2U04QQKxuHCAciFhGhHvisd5LsPukkYHHO7GfyNTJePAmDJK2RxswuD9eaLALc3wSUxwwMr5H+sdjj6ir/9rXEbjfCuDjowJP4dawJJA7sx+QEHk8/5NVtzrKGhdgw5wBj+VO/cdi7rk142i3pttixQSCcbbhFc8K4ITqVC7Rxznd1Ga2I/EEcvlNAuYnAIYnnBqvGZrtYzLeywTSIYyyQ+ZGM52lgM9CeuPTNcvpRltJZ7OSGZTayFQ1xHtJUklflPTjHA/wDr0c13ZiUbbHci8MyOdspx/dOBj6ircMvmRb1BwR8ueQa5SO6uXuPkkLFz0J610llMTbpHKhjIGAd2Rx/LihrQERFriC5knfDRbcOMY6dCP14/rxUwnEu4kOEU8kqdrD1B6EfSpkGQcMTkn3xTNsSQrH5YWIKcIE4/zz+tSAkkiCNpJkfy15OIy3A56AfyqoyKFdljJHEiBoyAg5wQDjJ9j0NTacl0vmm4nLAnCLtxhfx5znPU1YdSJFlk2Ns5Gc5X6UAZIhUXSRzTNO8a7iWYowyerADHAPr/APWtiwhltkW5iEsgGRg4PHA645wapXzSJNGVdkhbKAK2AGJ+X5RyfwP5VJ5B+WW6mjRFBjPGEcEjOVOSOgHJPH1pgUriN7QBZDJNvJUeYygY6j2zx0/WkstTksJZI5oNiZyQDyOO3rWtJp9kI5HbCRH5mLHC/wD6qymTQ4IpJnuBJHG22TyAX2f7wXJUe5wPend9hWPOPijdJeeJLOeLcFNigGRg/wCskrB0khbkM33O4xnd7fU4rZ+Istlca7ZyaeSYfsoTLKy5YMx7+zCsfSBunwEBYAE5x079fb9M1hLc1jsaGvKRdmQnd50Zbr/tEY9+P89K5S7H73OMZ7eldNqswngikjOVC7CfbCnn8QfrzXN3rZm6DOOfrQyisKQnNLSVBQlOQZdR702pIhmZR70wOq0SHc5IYDIA3McAHkDJ/H9PfNXriJrO1aMEKGUhSTknJzj2/H1/KXRIFWJAc7pJFiyDtPIOBnpjg/iO1N1qRoozGMlxkthcEhgDzx1zntwRnFWScbcYd8gY4Ax6VWfhR71akUYLAd+melVJDyBUAhlFFFBRLbp5lxGmQMsOtaiwokrq/K53KoHXP5+3FZ1mds270Gc+lWGbzZHIYr/X2q47EsZcFEbgjPbBzVYOQ2evtUrIT0GR9OPrVc9eKTYI17S+AwvQAcgmrzy/aU5mYSY+U8HJ/DBx+P4VzQZlOQTxWnYXxVtjnKnqD3ovcLWGXFv5LkbTkdc46/hivWvhVq0F5o0ui3DB57VzNDG6gjyyQePXDE5/3hXmt/HFJEsu3YpG4ZUL7YGCc49faqGnajdaTqEF7ZymOWFw6EH9CPQjg+oJqU7MUkfSF9daFYyBrWQWTuB50BhcIrZxuBI2gcgds8d6uKF+8GJz+VczJY/8JPYxX63kRt7iPfEj24JVTyAx3Hkd8dxWnYWj2UTRy3s1zngbhtC/THP610WjZWMVJ31RqMwBJppXeeFzkZ4FV98ZIO0bgCoPfH1qXzZHJLOxz6sTSsVccEbe0YiKsv8Af+QfmcCqUpaU4WByFYZwdvPtkjP4VY2s3KnjvigqwA9PWlZBqMW3mMeYYogO6vLtIPcY/wAM+xJqMRXflbt9qjZIYBmcp165A5zwRzUpfacd+xqraQ4vpBB95xueMk/vD6r1+YDtxkDjkU0kFywttJ54L3ss0JPSOJYsdeOS3sc+xHfiwwiSPZFbKPlxvd2ZsnqeuM/hUAldHPHy9D7VMHIOC3XkGgBjK2FwqqFXaMKOnoT1NK0sjj55WfHQMSfpT8Zwc5FNZUHOD+FFxDdpOSB8ppu4jGOlNHzjcuQPUilB59fWgYpfk1FOzmM+WwD9eRn+tTlcjheKjK/7P40XEY17DfSzq29/LHAEZK/jxzSOmpQfLGZbjfxkbTs+u4itnAPAPIpM7TnAPqKLsCBfPLIXZV4+ZRUvnGBg6M+4HII6imSuhB6Ae9RRsgGzcUYHoxGTTAivphbxwyxRYlZtskanCn0KjHXPXnn8adBMs6B0IZSAQQeuatBON6ttb+8Dg1lzfZdNSef7RILiTLtGUGxzkfMTkYbkfXH5G+wy8c44zUMgZmOUUr7dahs7wXNoswZCD/d/r6Gle4JBwMfjSugsRy27AkqQ1UpY2ZDtOCasPISOXI+lR5BXJ6juKl+RRCYfkUbiT/OonjO05IxVliNowf0phK91zgelIZXC4wQM8dzUcrrvC8BiOAO9TFPnLDIPfk1FJESQclT0zgUgI2TecnPBBABxTDlCSp688nNSeW+Blsn6YqNk7N355oAgHzSE8gge4BpNmQeTU+1VCrnPtmpEtp5CBHBMzYJAWNiTj045ouMoSL+4ePajhuzLkA9iO4/AiqCvqEUbKUjmSNSxDNhsDqfyrXuYbu2ZBLYXas/3S8JQH8WxVN47m4O14I4l6HzX7fgDQBHLax/3djAYZWzkH39KqNaKrBhLGVzzzWo1rpzlHu5Li4faNyxnywMfwg5ORj2p9tJpVkuYdJR35O6eVnH/AHzkA/jRYLmPJ9kUqrOpc9Aq5NVt9sjMPImLLkFQma2ZLj94WgjWDJyPL4x+PX8M1DNNPJIryuZCoCqXbcQPQE9BQ0K45NFFzYrO0lraggny3cebx/s9PzI61JZ2mjacnnSX91JMyZHlQ7SuR905br64P49apZc5yxI9D2qGT6nBp2QWN4alagAR6lfIg+6pRTgenQ/zNFc7sf0P5UVXOuwuVHcT3ZuFzyeMg44rLmXzGwMlu4P5fjV2PmIL1ZfQ0xbZmm7YPOPxrtOK9i9Gv2bRVV8c6hCoL8f8spun5fzq/pmi3uvXLwWreTbpxLdlcqp9AMjJ+h47++lp/hy3utAD6vKba1Nyk0ZDfO+0HGMc87j0546jnFrUfEkFpp4tLKHyLaNdqIOuO2cdKylLV2/rQ2t3LSyad4Ut3h01BLc9JbiY/M59BjHHXpjH51z+v39vPrFpcK+9LyzyB/CGRiGAxjGMgfhXOTahc388jrIRGVwR6c1JdshstBdsr5MlxAR/eLYbP6frWfmaJdyTDliETatK8ttBOiSzRo7DIDHHApktxLhIrWFp5pGCxxqOW/H/AD1rT+xaT4fK3GtObvU2O7yF5WHHIAzxkZ6n8PSkkDYtvFYxQpfavI0dmTmO3VWMlyeSBgdF474z6gdWa94lmvFNpbR+RbDlYhGoA6YBAzyMfT+dZ+ra1e6jolvqK2UQkivDEGALfI445PcFAM/SsNUmlyzyhCp+Y9cnvT02DXqIbe4nuizHqfXpWzbW6WybiwGMEknoTVOAyIxDOGH4DBqe0hudYuvscI8tI0Msk3AIA4wmSAWzwPf6HDQn2Nm1sJ73zJlh3QwEea+SMH+6OMbvXPAHJ4rT1OwsLTS1v7u5W2hj++pBfdxnZGpOGY4wW7dT6izod7caMzaZbRKlpDGJWebPyk9uANxJyePz6VjavdXN68jyqEhLFzEhKqPfGc+/XrSl5Csjg9b1ZNYv/Mgtxa2sa7IYl6hQTgt2LHvis9LePcZNmfQkZ5/GulmutGgUhoirsMH95jP61nz3+lzSr5lw7dQvm3BbA/Ems7MvUoruHQFT0NKHjALAMxx25/8A1VopNonliCE27yd2jiyR75Aq7H9ieRTI0rIvZkYD8+h/CnqMq6dptzcosjRKoY8KzdvU4zxXT6fprxZS9htUdxvT7OSxCrzn5lHOSPXqPXNZ8t3AsHl2z+USuUwOM+5HTv05qy+p262bia7kvbpl2NvGxOOm3aemSeue/XjFW6iZR1vUm1G6Fv8AMLW3AVFLAgkgE5GB68fU9c5OYAFUrGBgcYpklvcKo8mWGQnJIk3YXkngDuc1OqkJ92CNj99kQ5Y/Uk/yqGxrYYd2cd+tAO1w7BGUc4an+TEUYSmaXJwNsgTHIOPlGf1rT0fTrFryBLyzkmV5AgZ5ycZyfu9Tx6ntVJA2Yt3qFnCNoaOMZ5JbJqsddtgywAyOz8KoUnd6AAda7nxJcaVoyi2tNGsjJJ821I0CoDzz7kehHXHYiual1e7Ik3GKDzBykCLHnoecdeQO56VKuNMzI9TlkyYNPnZgcZlxGoOf9ojP0xTvtWqSgMzafADkFcszL9eAD+dRPelW2SR7RnCqGBLf4CrS2luSsvkpv4OSMkUajFjtZmKma7mYZBKxqsYYDsep556GnyaXBcIfNM7ITko8zY+mM4xVkNt5JGfrSrICuRj60CK6abZKgUWkH0CDGKfMZPOZ4I0QNkvjC5JzzwOakbf5RKAFu1NiidEwTk9SaAsSpJKwUGTGMZ2jGf50N1ZzIzH0yOajaRlRCmMN1yCCPwpZ0jcruUF1U9+v1HehsBiTEqXbAHAyoP8An8adHIHnTfII0YYUEHuep79Pb86hJQ4WYMGPbJ4pRdRSMVUksuOPT60Abl227SNLUxxeYA+CrZYKMcnPTO7oCfwqCCRVdPNMoUn5xHjOO2CT6+1WxotvFpmnXjTojzW5/iPzHg45OARznHoapYHHT2qnuSiSWRC5ESt5f8Jf7w/Lip4onDbReJEuM53AZ4+tQA5UjgZ9TikSNncKAME8t6UAG8iQyCeR3PJyOPzzTfnLEsMnuSealntmiOAy4H8WRVUD5utFxlqFDI4RAWZjgKO9dDa6Q0HzlxJIFPyBNwzXNRu3zFQGXHzAjPFdI/iC2S0xbiQvtwCV747nvQ7taCNK1gQQZYH5zyHQLg+mMUk8cDfKWjRiOAQv9RXNJrFzMw3sx2DcTjGTznoOB/jRNqoYjaqxkdSVBz9aOVMRduLVpN0KCJm4/eBMBM/7QI/lWXeWM1gDIZkBIK/Lzn25H8qZJqsio8jXarGSM4woB/Cs+XVbBgfMvC4H3fLXcSfQetHLYaOp8OzkQq0Lt9oU5B3bQvB4J4xwSc1F4gsZ4ruGae4Mk90ShErMzJtydvccc9xyw44JrIj1i2tJVktra7kY4BiWM8jPU56/jxxxV3Vb6XVtMm2CK3u18ua3d2DEMuMAdl4HOMdgetFtLhdGbudQD0PoK07K6v5IxFBB907hIsecH3J4qGK51u4C7ZdFhkIywIeQqMdzjGevenxs0wCzeJD8rASLZ7FGOvyjYTnHuaav/SAupd6pJK0bxurDsqgADsaHvGhjAvbu2iGCA8kxDN+RGahnsPD93HGJxLcMimM+bcSFQOzALgE+3H5dKcdpoWn+d9l0pHklAy0537R/s7skfnngHqBT16onQnTxTpkEahtV8yTJAihjLDgdMqD296c/iwO7tBbTuETKloihY4J+6fmIwOoBx1OBU+maisVxsW0tlBQoPLiVCV/u9uD71r3otE0t5ZIFjRUBdVGNoBBOMemARj0FK+u39fgO2mhxH/CUawLhkFhbKW58xMliB2xgcgcY9vSiytvFWvyhDqC29qrgyOIwOAQQMDGenTPfnsamT+xLXxFaQxtLcRl875SGUEqxxxx2H+enTXOt28KfZ7OOMpjAKcAfhiqnfaOhEU95FCTRdFso4jNHLfSAcy3sjTFz/usdv6VnW10iRmCKNLYhixiVQgPPB2jA6YGcdqe9xJNK7zODk8AZzUDtZSOjPIodTtQ7wOT1qUrFnF/EJSb+xlzktGwzn0I/xrD0kt5xKE5PAx/P8K6H4gwbE0yXnDGUBm6kfL+mc4/GsHQxvu9p78dccf1/zngVhLcuGxLdK0U1zE7gSYLAg9SDkgdsAZrnLk5kyeD3rr9fjjR7e4jOzdvjZc8DOVI/X+dcbKctz1pMtDKKKKRQlTWv/H1H9ahq3psfnalbxjq7hfzoA7u0tv8AR4ox1hKNx1Zhz6f7WB2GRngVna84+0OwYHoCQO44z9cjOD6+9b+n7pFVZF3CUmcsOWQHgH64xx1/DmuY1f8AcyyIOp5KkY45GMd/TPsapiMC6GHxjt2PFVDyxqeQ5fJI55qFVIb8DUsSI6KWigofGuc4q+iskLKvJPJrPRyh4q20+Ywp+UgdquLJZA5YNzmozk09mz1PNNyOnFSwRHilBKnIpSMc9qbSKNfT9QVIZIXLgsMABiF+vBHP+P4VRnkVpSVBAJ7kn+dVx164qeCCW8u44IULyyMERVGSSegpCZ758PwR4F0wng7X6f8AXRq6TZz061VtbdNO0+2s4Sxit40iQseSFAAz74FWQ7NkBTW60RiJ5agnj64p64HT88UmeAKjL5yPfBAoAkVvnJBBGOKSSRQD3PfFQqwzlFzg454NPBDDcCVz1zQMRwGA7HtzUJt8kZ5AzgHnrVhghGSyj8ajBj3YTuOvY0IRPawtdbljVVmjTLRqBiQDuB6+vr7cUnzfcYYI/SomjOQMgMDkexpyKVtUZ5XkbcRIz4yGJO0cADnnA9qYD1dgSrYwKUyKBk1HwcAZJA608oNwYL1HegBomJyFiPBxnIqBpZvNCCDCsCd2enPtVhs9M8CoSG5x16HPegBI5pSxWRe3GP8AHNSod33R/wDWqIBj1P0wetN3/NtyAe4zmgCUnnj8qjO7PQn6UhlVQdxyPaod4IZQ2O3HelcLDpJUTlgoIPAY96gTyUbcqgZPQAD+mac6I0YMmM+o61EQGcbSFUc/N3+lLmHYma6HIAx+NZsrtPc7IiUOOXUgHPPB9akvLiFFKsVK+5H+NYzXVpFKC19bQjr5R5x+tPUdjWPlBoosJDcjanmRgLHJ0yZB/Dxk5A59KgjuvMeRGQo6noSOfX/9VZ8utQTstvBDLcO+FVUiJ3EnAxgHqeg96aGiv18iDTLu2kDjddBSAFOMhhnkDgjgHgjvU3WwzW6PjAOelOAxztJPoayPP11olSK1tywJG5shfqP8ioG03xDclluNXiiXPS3TrQ7LcRulovLLlk/DpWdPq9jArGeeKIg9Cw5/Ko5fDFjOIzObmZ1QBzJOW3N3PsOelWYdH0+1A8iytowO/lgn8zz/AJ+tK/kMy5PENqJtsSTSkjI2Rscg4x2p6ahqFxLHHDo06mQAh52EagepzW2sUUSllVVUegAo2LjkZBHUii7AwVXVXZt6fMp/1cbLg/RjgZ/Qe9XBaxWepf6ZeSzo0WUhZQFBOOCw6kenuKu+cQvlohHbk9asWd9Lp7ySRrHKrDbLE4yGWjUGN/tprRcWlja2zcEvGpyfrk8/jVObV9RuCVa6k2t1CnaP0qaSGS5udtnZs8Z+ZwG/1Y64xyTj16ew71rm0ltiC4ADDKt2YeoPei9tEBUnd5c7zuz1z3qIqQOT+dTMTwGHA74phB7Hine4yrIme2PpULocd+vFXdncU11JBH5UriKJDGk5HBGPw61YYFeMf/WqMgj3+tADcrkFf1ocbw2QPypGUkk8ZpjE7eO3pTCwnlf7v6UUb2/vn86KQaHTQkFmKkMA20KPvHPYDqTXQ22k+Qy3F+iB1UPFZnnOTwZSDgDI+7+ueK2TZaJ4XBktokN4wxulYu//AAIk8f7v54HFcxfak927oHESOcl8dSa6nNvY5lFItazrtzMASUeReCcbVT6DJrkdSuTJatMkhyeWyg/GrQZZcQyurlVJPHB98His2/doHSBOYieAwA4qHoWhLGcYEcfzMQCcdTXTHR59c0qxht2VHt73zCWBwEKYboOvTrgH1FZ+haK06l42ihtoot013MAFT/E/4+9alzqkNpss7In7MqZ8wD55cjksfr2qrdGNvsR3Go2vhaOSw0oyTXzgia8k5bJz8sfHAHr/APWrHstHmuy019dMrSNkK4yT+VaVhaSXmoC8uFG1eYwTkrn09ulbbpEATvVD13MOlS7iWm5VlsbaLw1qNvgqkYSbJJyCJB0/P36+tYFrZiACUOwDc8jr710V6JpdG1WBH/eGykKHGeVwwwOvUCue0a0vL7R2j1WWO2kUeT5RciTPOC3ytgccjv7ZoS1C5WvknuJkitxKkJXdJOisQ+MkhSPp1z171oWt8NGgb7LD5Vu6ghHH+sGSfmyTxgnjI4xkE0zWL0xw21pKFWzjGDFG2PlGeoKg/Mfbufeua1HUJr6Qsz49ABwKHLlWg0rmyfEl4sjmORA7/ecxhm4xjrnoAAPToOKzDPLIwLsJx2NwiyfluB9f8mqVvGGI3sxb6VpLb+SrEk5xkZwKjmZpaxKNX1ARCNLyeNfRHKD8hioTLPLtMssj7QANzk4A6DmouM9RUqjcQowM8cnApb7isJtfdgPkHqCP/r07ZxyT9fWkU4B5qQMMAHH5UWARFGDnd9CafgEk8ijPc0Bgc0ASeZuVV4wPaoGgidwzAkg5+8R/Knjnml2knAHP50DLVpJFFtVIishYAFRgDn0rYVJovtd8djWVvHgs0wTDYydp7sOAAeOo5zWRpttDdSvFK+HCkIpfbljwv0ySACa2fFVnHo2jrZhYzLdzZYs4Zxn5jymQAeeCfpmqTsrkNdDj7XZPgtNgNks0k7bhnHH4DA6jpUd3Zp5wX7RBOzL0RWYgA9cjj6cGrQABAbA4/TvTTe26TCIuQcZAxx+fSp3LI49JgBVny5HPzAFs+5q5gAgLjaBzx0+lMa8t1UOZVVfUniokv7aZsQyI56Yzigdi0MdR6UuMjIyfb1qNEIyxJ57ZpxI+UswUnjrQIkG7b8ufzpyngZ/SokYsGGGGO3rUrFWJZQ2zPAJyfzoGRyysrFQpYYz8o5/WkjuYpeRkHoAeopruuD8xGDgk+1VpL2zgcxvIkchJONwycf56UgLj24kjUAvkDHXmkLGAoiRkc4IVQfzqazs5mtmmLmNPlx9qIids+inDMPcA/Wrcy2kMEtndQLNK0ZKXNvOCsTYOBn145ABHIp2fQVxtxqJlTTYtsOTak7wgLNynVzyeQe/0xg5YCw3OFJx9OaqztMj2ptHg8mGIwN5qdi27AwOTwOWJOc/Sqb2Nw6Ju1KRTuL5iVV4PY9cYwcfX6U5P3mJbG8JBOEiSAiTGcrzn/wCvUkVpO8oUNtJ4CFgP/r5/zisBdOgMJjmub+VepWSc4znrxzV7TgNJSVdPZ7cSkeYUkYlsZ6knPc/nRcLFiTalzHau0yTspcp5D52gZJxj0FQyXcAUlbfUpGGdm22wG9zuwQD6/pU0l6+D5s/TqXbFRC4jcf61SvsaLMBbaa6uJY4rXRbl3bO0vdRx7jg8Ac5J6DpyRV2TSdRiCtczWOmJhQ8UrPIyglQSNpUEgZO0Z6YOM5Geuo2Bjl/0lTtXaTnJ+g9ap6xqjz3Km4kS1doVA8wgZBJYHnnPzd+cAVVu4r6lm+0tHt9n9v3Ew3ZbbZJGCvoGDbs474/CqX/CP6e00habUHhJzGrTAEDPG5tvzcccYp39s6bGI4Y7lZS2FUIC2fyqOXVJhzBpt/Mh6MsRxUuy/wCHKRdTS9JhMbR6ehaInbJI7OefXJx+lXI53hC/ZyIBjH7pQhP1x1rOL6m20RaY3IyTJOi49uvWpII9RmukjnFpbQZ+aTc0hXjPAAGe1K6EWmeR2IZmOfWoIx5Evls7ndkjPYd6hitNSVgGurZIj2jQk/rgVYk0GTUZZN9xmAIeTFyGxxnHbOPw9OtO+lxX6CRT2ssUkLBXCy8s/XjtjOMe3epZJ44NqvKsSgYAJCgfhUEPhC1F6gtb2WKVoycgYXAYqTzkjoOMN1610mn+BdFt2hnmS4uJ0If/AEmbeAc55UfKR7YpOXkNJHNHUrGJWY3MeB2zkk+3r+FQpr1tK7rBBd3TopYrDCSQB1J9B713g8M6DZwZXS7MbRkM0IY/mas2f2S2Ty4PLjxwVQbfz9aE5PZBoef2uoanNJvTw7qjJ1UojD/x4rj9K3ry41u+0ia2TQLuAyRkCSS5i4P4kH9K0bm6vE1AlbyLYTlIl5Yj0I/A81qwNNPbI86gP3Udj9ad5W6fj/mLQ8z1Pwxf6dFZX6GK3miK7zNKiID0wOcn/A/jWvaeFNcAfzL6ztEIO11DTuc9jnaBzkgjn65zXRPqfmu0UZhQxHgv6DrioPOe/b57rbD1CqMlvwFHLLuFzKg+H6yr/pmvXs6Z6W8axD17ls1ei8A6BbMZJ576X186625/FAtalm4iTZskLA/KWGP8muK1X4h2kE8sFqklyyyEGQNtQjHY8k+napaXVv7xpt7GZ8TNH0vR7LTU0y1WHzZHaTDuxbAGMlifU1yeiSRxXRaTlMYYex4z+Zq14p8QSa1DbLJF5flEkfNuOT71l2C5DOSoAz+HH/16zdr6Fxv1N7Xds9hFPEhDmTLAjp0P58g/r61xk4KzMp7HFd9YwDUdPRX+QAgnGewP4/j9PQVxmtRNDq1wrKFJbdgDA5GeBQyluUKKSlpDErQ0VWk1e3VcglsZHbPFZ9dD4OgE2smQqzCKMttUZye1CBnXTSQi8CgkRr0CnlRtGQPUD16fpXJatdGS6mZjvyx+bufeuxkWG2jMvmuXdWyzgBWOW7HqOMdun1x5/fEGXIPbn1/H3psnoU85apEjzknsCKZEoaTB7g4+uKtqAsYPqOaliKDAAA9/SmtViROpAzjuKrGmUhKXNJS0DFz74pQxzxz7UgxjoD+PNOVN33AxbHYd6BCklW5XkdiKjJyc+tSCdl7KfqoNMdzIxYhQT/dAA/IUDErvvhZorX2uTaixQR2KjG7rvbOCOMcAH8xXDW1tLd3EcEKF5ZGCIoGSSTgCvovw5osfhvw9bWEab3UbpnTH7xzyxzxn0GewA7VUVdkTfQ0Jmc+UY3SRwwYopG4j2yasfMrEAMAe4NC4yp24Pp3o5bnI57jrWhmM3dlJ+tQpMJJZFMUqsh+8ycH6HvVgjcMg98cGmrERklQD9etAEDQqzbxuVjxnbSs+1lVYZXz3Vf55qz7BfwpBgc9/rQAhUeWRtGTzj/8AVVfy7kMADFgAbhgjP0/yasEjGSefQHpUX2mNpdizK5/uIct+lADmQEdefQ1E0KeYplVZUznbwCPpkH9f5VB/bdlISsAlnYZGIYmfB98Diq/9o3twpjh0m4D9N8n7pV9zuwfyBo23YFm3vlW+aymYq23fDubJZeP4sDJGeeB+vFwxsON5H90+lYv9kyNYbZrp/tYO+Nw7EK+T2z6Eg8dKqSWviK5bM2oWVtHnKqkbufxJ203KKGkzoFmCKfMKhgcEZzVWbUreLH7zBI6jpWU2gtIS13qlzMe+yNU+uOCQPxqWPQdNh5MRlPq7E/pS5l2CxM2tW4cgzrwBgA5NU38QRyyFbaG4umxkrAm/Axkk1aW1tYgTDaRZPUrGP609jHLGVZlMZwCnQD9am77DM97/AFNxF5WjTuHHQyIo+h54qKN9a86KN9PW1jmlWItHJ50ikng7Rwc5xgH8u+zvIXIPB7jtTHuYli3TMrRqQd6gkg/h05o1AwjYapLPLB/bvlyo22SNYOE45DbsOCDx0+hNLH4dmnidb3VbyVyPlWGUKoJAxnj68YP17l95bx2DRyWzl0LAvlgwBYjG4k5A68j8jk1oxSNJbjy4pI5NqtskBXg9Dz1HBwelNvswuQaboek2KCRrVZplyplucuTkYxgnGME9vXPtbkijSSGWBBb+ShSJbVmjCqSSQuD0JJNRwSXK3PlXPIYHy2AGGxzzj29qie5P9oJGzbVXqhBy2cYP0qbIZdXUbyWQM7SrtGAzSk8elQwzvLksAmOPrTY5vNjICOGBPDKRn3pdgkYZjO7qCaYiV48vubJH93tTXdIhyu0deFJH6UmGT1alaPcVZgQV5BBpAIrLKgbBwehIxTGXseaRfkYhndsnILY4qQ5VhwDQBFgkY7Coz1GeKsMc89PaoA6MfXtQAzoeOtV5oYZ3KyD58ZyDgirTorLjb8pGDmq/lqnRQABQMXLIN4Ztw6MpwR+IquuyNJbkrJdbuSQ/Jz1Lf3iCO2Dyck4wbCj5TtJOeearyI8bmWEDJPzLj71MDSfQp3sfttr/AKVbf3lVlbjII2kA8Ee/FYrptB6AVr6Trctk42MrxdHgkJAPPJHp9RTr+y0678640q48icK0strcZ2nuSrHgfTPp9KSQzA44700qcZU/hUy4mhVlYZwCR/Uex61E4wev5UXAY3KkEfnUTp09OxqQqT1AJHSgjI56UAQlQQQDx7CmPHnryKm8rPIxiowTggH8aAIfK/2RRU24dw1FIDW1XWWlbGOPfkU2MGaMlwwUjJHpWRNPuwvO9RyR0zU6SLbrCjvvZvlCg/NnjGAOv/1xXSuxg7WJbyVIWARyGjO75Tjd7c9q3LDQGeBdV1gNb2OPkgYESS98Adun1qzY6baeHrI6vq0Jnvj80NuwP7vnILHsehyenQe9G61+SfUXuZ75mDhVMeOFBAIAHYc/jg07dBE2pay726RLbxW+lRAGG3VRzn+9z6847nk8gAVbSyfUbqGabaqEf6teAQMcf59axfLn1TUGZnBjUgDHTj29q7G2jj0+BA/zSE449T/kUtx7K5Ngo4UDYn8PGM++aHfYy4UuQwzx0z0/pVS4vYftHl7iHDKOE5LHoo9T/n1rVitwjedczRBllwIVkBKjBKhvUnHPGBkemaaVkTuV5IPsk8N/dgIxI8ohGZoQRncRwM9OPwpl3LDa6fNKphZo418uTyNhlJ/iPBJYkkYPr6Vp3OsxpDB9qs2t1jYoWDM0nbAAA564ycfTisG5ke4ulvEZkjA/dxu3XryeOvP4Ck9h9Tn9TXUJUZp2nYykks5J47Lz6f57559kEXlqsmWLgfNzx9a6jVXcQtJKw4HKq+7/AArnptkihXIiVSGB6c9etZPc0RKkgjQFBtYdWxzSb2frn6k9aj84ZIB3ZOc4O38DipU3EhUUMT02/MTTGKsbsc8AfWpNv0zUe5ujZX60pk2xkZHucUAOUEcdKk3DGCvPvVRZow3zTIpHUMwqR7q3iC77iFAem9wKALO4E8Um9c4pI7eQwl1huZ16+ZFA8inPuikVImnXNxKogtpZOAfmAjx+EhU/kKV0A1WDj19DTcSg4Gw9exx7VotoOpxxb3hgiVepmlA/9BDVWSC7eUpm3ZRwWQk/qVH8qXMBtaFFb3pjEUcguogWndBkGP5sHGcdSBz71meJ9QhuNfMMEU0CQIAVkIxk9wAOBgCrOnS/2SzuZBvdgxK4faRnGBjrknnryenbPu7OOa9uNQuJbsmeTzHIUgOT74J5qpPawLuUDukSSNtrc5BAGSKgubONmZ5JSd+Sc7ck+pyDW7pdra2ssdrq2mORcMfLuJnIHHJDAnC4H5/XityKbw3ZiSCAWsmR86wwmUE9cHAPp0JqGn0GcJaxWQIjBe7fnEapvIz6BRWnDpGocPaaWYl55lXyz+TYz+ddLL4gt1Vobc3CgfdjW3aIfXLYH9fzrEm1CWQhjZvJwMBrkDHHcYpqLYNpFN7TUlPzPbxk8YaRSB/3xuNNFreO5aS8tk2gsCquwX8OM1ellaR4vKtbdEP390zMQPXHHvxn8aannhmRmtvJkXDhUYk5HIyW6decZ9hRZhdGS8N4J3gmv2jbcQoW3CEgdyGz2/pWtZS/2eiFFjkkUcy3A83d77Gyg/BR+fNUf+Efglk82V7iRwMZEm3+VTQ6La2xz5Bkb1lYv/OmhXLNpqM+mvK9lePaySj948UhRm+pHOOaZcatLdsXuL2W4ZR1eQucE/j3p62ke0p9mg2nn/VL/hUrK4zls568/jT6WAzhfxySFVWdmHXEL8fpUqu5YBbWc5PVlC/zI9KssGB5P69aijU+YXYhUHTmjUB8dpqlxgWmkXFyWOMRMhPTPTdUMcd87q7xQxIRkrJJ8/r2BH+TXa2cltpXh8XEjos0sJMUU8BILOSu5WIwSBjgHucjpXMqVSJY4ydiKBksSTgetDTTBFb7Hcee7faoljz8qeQWOPruHP4Vf082dlbPHJG12ZFO4yFcc+nykj8DVZW3nGST605fMQEAjBGDxR6gMm07TnbKW4hXGNqHOff5sn+n0qs2iaY67DaFlxjDSvg/hnGfwq7glemCfelDcYalZMCC3020tSJILS3jZcMrbAWBz1BPOeanaXzFWNzvVM7VboM9abuI6dKXcOvB96LK1gH+e4j2jIHYgnj2FIrYJJyT65ppK9O/al5zgkbvrTXkA7dk5ViqjqOOaY5Yrnr9KMccitC2tbi9tPJt1Ow58wsgCj0+bPP6UBsUM4x0yK07C98pWWSUIjH5gqZZvxAzVSWwaCIs8kZYNt2q+Tx3p8VoZERw8SndkZkwfpjNCYaGjBafbbiN1DhijMwAAABkcjI9flAyck49AK17iPyrceZMsbY+Z88KMc9artDFpAW6aVfMkgXcvmcAjceh55LE8evTucHVNZk1BIoACsakM/T5zj09O/4CmJFm41SUwNBDcyhBjbKSFI+mOg+uaq6fqE1pOxYl1UffcE5+lWbfTVjjEl2HK4JbYOEwM8n+lZshUM4Rj1+XPahSCxrxX6T3bG5SI7wB8wzx+X86t3l8bYMibWwAFRCDn8ulYMM8ibtkhAPcCrttHcSOsoUyIP8Abxn609BE1vbtcWyu8CRxgZ3twV+o4PX+lSxZjmDQxNOoIy7SgsvHYdB+HXPJrRitcESPsDcYRSeKbLFZzIzRpEUVjvKDjPvjvUhYgv0gSyuYYtQFu8kTKJAdzoSMZBznIzmvny6hnsbuS3nDRyIcEEGvdJIrFl2/u0CscMqnOPfgf17c1ymu+FV14K6zpBLEGCuyFlYdgxHIGQOecZPBqZxuVGVmeYs29MknrV7T0co5RguQRx1/zyKZq2k3WiXL2d2E8wNwUJKsvZge4Of8cEYqXTI2lKIpOSenp7/yFZGid9TVhvmtvkOViznHBI7/AORWP4huIbyW3uot25k2SZHVh3/X9Kn1Fbi3l2nknKkAd+c/z/lWWxLWskTcsp3L7Y6j8ufwpgUhSmgqQASCAenHWkpFBXc+GNPNpp5kmVg1xtkZcgEp/COfXOff8642zjSW7iWXPllstj0HJru7K8a4ldlAGP4AAAPYAY7HH8uvDQmT6lM/75z/AHGTGSD0DdenfHPXt6VwUzlnY5yCc9a7G6kGy4lYsMRsF2gEDGcDPTqQD/nPGyhQWABGD3oYhkf+uUdjVhyfKKnn39apqxSVWHY1ZMmeBUsCJyFUDvUJ554P0qSQ85AOPX1qM8nj8qY0NIxRTiOBTaBi9KmR/LB3Icnoytj/APXzio0DE7VXcW4Axzn2pMkDGeDzigBZXaRyzOWJ5LHqaZRV/R7D+1NYtLLcEE0oQsTgAdzmgTdlc9F+FvhaTzk8QXOUjTclsndyRgufbBIHrz6c+quVAOWwPpUGnLpllYwW1vi4WFFRf35YYAwOFI9KsXElrd2wje0iRex8tgx/E81so2Mb9RpmiyCzj2yaIPMuJMWytIDkllUkdu/QdR+YqOKWxt5VEXkxTKuBwFbA5+vvn8asec8hGWLZ6FmJp2Az7rUJrafyfsF48nYRxEqef733f1qD7brDfd0fAxxvnQH8eTWuyeZkb2Qj0HX86rwNEN6wyFyHIfMm7afT2+lGoGLeXGvSuIrK2t5FZfnJdk2HsckDP4DA461Nb6ZrMksbXmpwxRrjfHbQ5LD03N0/KtpobS5WMXkjQvGR5E8SksuT8ynB5B9CDjnHNVrSeSVCGjmikXgrMhU/rjIPXOP60O4kXgYxCkfkQNtGCzQqWJ9SSKga6lEflBmVF6KMgD6DoKbucvjkYPXsaDETzgk0h3Ii7MuGdjjjGTUYXggu359KeUZetIMYqSiIRsG3GVmGeA4HH5AU8HqDzmnEDPemlSfSgAcnbz+tNJOeRxQyHp6U05Xjg0gGkCkKIQvmKpxzyM4o53hsZNNB3diDmgA8mE8+UgJ5+6BTJI1fIKgHgFl4I981IRzjFBUKc46dqAIZkJiVJMyx5weOf8/QVQe0QXAaGRY59x/ehWLMOflcbgCMkcgZHrWnISVOOpqnHJDdH7PN+7uU5C5+YY4yD3oQC6lpurRi1eWNrdJCP3sUgZWHfBHTj1pLiKMymRUAkYYVuw5yODx+NX7HVpdPEkFw7XNg4KvF5ZJLc/dHY9Pb6dap3V7aR3IWMypC4+RphjPt9fY4NNJhcZCr+Sh80knklv8APFTbcPwTgjpWHJqj5lWztpJEDfMykDv2FWrLVYb1yih42Xkq64/lRYZqE7e2aq3geSH9yxzkdD271KQHPc49DQQqDAUAdcCgCvaM8YKOxyvt1FTGQE4EZ/qaPM25OMAdaasqsu5S2D6CkISRJBjaMA+vOKYVkDEtt2nviphI2eeR9aawL/Kfmz64oAr+ZubagDEDOM0hyF3EEHuPSlhtRBKzqXAPrzUrAA9CfpTGVPNXblgVHHLY5pW5AI5PpipJo0ZSGVSPRhxVQy9AVPHBAGB/+qkAkttFOQ+zDjo+OQf60Q+YigTZEiNkOBj6EVIshxgsPfAqGe5VQPMkVVPAywFFgL2pazd3+npBcW0Fw0QASZgRIB3BIOcfTHTuMg599BCqJNFOTHt3OGyuzkDqwBI59PXgVA97GLV7hVaSFG2Fo1LYPpx3ptlf3UkimC0dIz/FN8qHv2+lPTqA1Ru/gYe5pNueMVZ1G3kkuUNnOsMO35wq985wPQc4/CopbaE7wk8xRxtJcgkdOgxx0/WiwFd8JyWA78mqz39ohzJPHwcYDA81s5smsY4H06zd1GGm8shm9yAQO3pVaWcLbwxJGDFDu8uNUGELcnH1P86QyIRlgCLG/IPIIt2IP6UVOuu6kihFEiqowACOBRSs+4rsxbVgt1E0ylmZsIo7nt+tehxQ6T4Lhik1BN+qOoYbU3+Vxj5c8Zxnnp+goorrgkzGW9jM8V3Ti/t5CuLe9tYySMFuS3Jz161zT+HZmuAFlCoQCctk4oorNrX5jirHSWVtHZ2qFG2ovGR3rTttPfU50ikleFZMbtnLsM/dBzgZ6Z7Zooq7E7sdrUum6QYnggg+0lmUOY8EFSMopC54P8XGe2KzNa8ST2emJbwxfvhE85JO4r2JyffPHPtjuUVdko3Ivd2ONi8RvLfRwSSMtrKNwQoq5JJyvy8dSctjJ56ZNdN/bsEcJVlIb+EuxOfUkgZoopfErs0ta1jN1TVYpo1mZIiu3dGrlsDnAIGDzn6Vjz2WrXV/GsBivGdN4RWICLz/AHse57miiuao+V6DvZXNWLQNZKEvaKgBwS06gZHXpmnxeG9UlMju9nHEDziR2YA9+n+fSiiiPvK4lNskXw88SMk2oNKUYK7JHt2nPTr7dcH6VfsdJtltDMs77mj2seRnIwRkc8jOe3bnNFFawincpizWWk20RMNgt/MCAys7om0EjJ3MfbgDB/DFVlutTtrYQ6bptlYoOMR3H8/3fJooqZwSegJlX7Vq0rbrq5tCvXA3sf5KKuxDWboJFZ6oqSMxBVLVV7f3ixP69qKKHdLcoZeaDrsEAW41q5l8x+Q0hYAenNVo9Gm3Bhe3aLuKyL5wAb/vlR7/AJ0UVmm97ibY6C1ltrRgL24mjwRulkJPp/8Aq4qe9S0iljjS1ha4jJMjsith++PlGOg45wc4PeiitF8NwcUU5nnnbMjADO4cDr68f55qNg5xl/zoooWrAf5T7SzSHA71JawrcSSIs6LsI3blLHrj29KKKaVxkv2Yo45woP0z796uJaRocgk55yaKKFqIVdiSHOdoPAPOaa7p/eOD2xRRTsBCXAOVzjp1pjHLdfpRRU9bDGNknnBFOiUAMFGW7D1oooQM3NX1e5vtNsrF40jto41kjIT5nHzIuW3H0bIwOTWLjap2k5ooqVLm1EmOTjp2HWnknaCDRRVDGbzkdMfSn9VJwKKKEgGjpuJC46e9CfPyOmeKKKlIY4qUYjgkU4huORRRTQnuIwOBkc0DHTpx+dFFCGG1h2GM9c10+naan2RXZwxYHkLggHsDRRS6CZj69fXU8tus0gMbx744xGoEYPGAep6d8UzSorB7WS5nZpEi6qM4H19aKKuS0bJWxp65bXiWrGGWJLEFcxqCG44x6YyBXOg8AD6c0UVBfQcpJfAPPYV0+maeIIVkliUSZyCDkiiii5LLxQyHLZXDZHOf/wBVQRbZpHZppVWM7QN2QfcgiiiqWoi4tvGRhR8hHaq76ZvR0/eqrHjlQox7D6Ciis27MZ5d8U9HSwudLmWRnacSqQccBduP/QjWTp+mGCOJkA3BVYkjg5GR/MHPbHaiioe5pFWRqQxRsZDOqlAAYyfmKjJ5AI7Ae3T6Vx3iBYlumMChU6AgYyCAefzoopsaKlyhOiafPjC75Yc+pXa3/s4rOooqRonteJS3oDj8eK7HTbQwWbtcoMS52j72duP6kflRRTQMZqCBdNceYfMAyOvI7H/J/rXKyfKMd6KKGIryfeqW3JBd+uxCfz4/qDRRSY+gh49QTTSwA24/GiikhjGUg80BCcccE4oopgX7Nca9bLGMA3KbQPQsMVWkAWFcZBJOR1/z/wDXoopE9Rfs2NO+1M2N0vlouOuBlj+GV/M+ldx8JLGG58TT3EqBzb25aMHsxYDP5Z/OiiqjuKTume0TRLNE0cg3KRyM02HcilXffg8Erg/j2P6UUVsjMkznIxkHg1G8VvIhjkgjZT1VkBBoop2ESRRpEfLiiVFH91QAKry28Vv5lzBFFHKeZGC43geuO/vRRS6gTOY5LfbtGzHTFUL24uYgl7I8s8qHZKHwFEQ5DZznI57c8+1FFDdgZYt7j7QrFk2sjlDnnPTkfmKmBPTGaKKVwGyqHHQ8VBuGcd6KKTKTF3Zximl+dtFFSNiLyenenHBJ4oopiRGdqngdPamsvzBj0oooYxANp4zTMAsFPJ60UUAIFAJxz7VUuLWG/jKTxBkzjBAPINFFIAjaWHMM7h0AyjgYOPQj1+lVdRVJ4njKozKpwSoJHIJKkj5TgDBHPFFFCBFW6tpLS/Gn6lCj7kMkNxkEyKDwSMcH1HtVDVLdLeSSdBKQrAhi+Gwfpj3oopoaK2narZ2+pSTyxFGfncn8XrkDA6V1N5dLBHGQpbf90D/PFFFMGNDBG3AkE9vXP/16lEIx82CSKKKkSI8EHtjrSNlBw3X1oooGVmvFWQozhSMkgA/TNUf7ft2n8mEzTys4RUUAZJ6DJxRRWjSUXIaHA6k8EpitII3jbZ5byc/oMY5PfsfxfZW2pNeRtfG0W1/iigB3H8SOPyoorDmbSJua2+yhcrHZREMMEy5fP4E4/Ss+W3tSZf8ARLcCRt7ARjBP0xRRVWVihm6KFCsKKg/uquB+lQu2Dz1oop2AiYg85I+lNJI7ceuaKKBDDz68+9MOQCM8d/WiigZmy63psUrxvc4dGKsNjcEfhRRRSuOx/9k=" - } - ] - } - ] + "outputs": [] } ] } \ No newline at end of file diff --git a/samples/js/node/optical_character_recognition/README.md b/samples/js/node/optical_character_recognition/README.md index 130566ca0bcd4c..c397bb3c846db3 100644 --- a/samples/js/node/optical_character_recognition/README.md +++ b/samples/js/node/optical_character_recognition/README.md @@ -1,6 +1,10 @@ # Optical Character Recognition Node.js Sample -Run: +Run sample: ```bash -node hello_reshape_ssd.js *path_to_detection_model_file* *path_to_recognition_model_file* *path_to_img* AUTO -``` \ No newline at end of file +node optical-character-recognition.js ../../assets/models/horizontal-text-detection-0001.xml ../../assets/models/text-recognition-resnet-fc.xml ../../assets/images/intel_rnb.jpg AUTO +``` +Where: +```bash +node optical-character-recognition.js *path_to_detection_model_file* *path_to_recognition_model_file* *path_to_img* *device* +``` diff --git a/samples/js/node/optical_character_recognition/optical-character-recognition.js b/samples/js/node/optical_character_recognition/optical-character-recognition.js index 5e371c1975a993..a5665afc67fa84 100644 --- a/samples/js/node/optical_character_recognition/optical-character-recognition.js +++ b/samples/js/node/optical_character_recognition/optical-character-recognition.js @@ -1,14 +1,10 @@ -const { addon: ov } = require('openvino-node'); -const fs = require('node:fs'); const path = require('node:path'); -const { createCanvas, ImageData } = require('canvas'); -const { cv } = require('opencv-wasm'); -const { - transform, - getImageData, - argMax, - setShape, -} = require('../helpers.js'); +const { addon: ov } = require('openvino-node'); + +const Image = require('../image.js'); +const { transform, argMax, setShape } = require('../helpers.js'); + +const OUTPUT_PATH = './output/'; if (require.main === module) { // Parsing and validation of input arguments @@ -32,110 +28,91 @@ if (require.main === module) { } async function main(detModelXMLPath, recModelXMLPath, imagePath, deviceName) { - // Initialize OpenVINO core and load the detection mode + // Initialize OpenVINO Core const core = new ov.Core(); + // Load the detection model const detModel = await core.readModel(detModelXMLPath); const detCompiledModel = await core.compileModel(detModel, deviceName); const detInputLayer = detCompiledModel.input(0); const detOutputLayer = detCompiledModel.output('boxes'); - const imageData = await getImageData(imagePath); - const inputImageMat = cv.matFromImageData(imageData); + const img = await Image.load(imagePath); // Resize the image to meet network input size - const [, , H, W] = detInputLayer.shape; - const resizedImage = new cv.Mat(); - cv.cvtColor(inputImageMat, inputImageMat, cv.COLOR_RGBA2RGB); - cv.cvtColor(inputImageMat, inputImageMat, cv.COLOR_BGR2RGB); - cv.resize(inputImageMat, resizedImage, new cv.Size(W, H)); + const [, , detInputHeight, detInputWidth] = detInputLayer.shape; + const resizedImg = img.resize(detInputWidth, detInputHeight); // Prepare input tensor - const inputImage = transform( - resizedImage.data, - { width: W, height: H }, + const inputImageTransformedData = transform( + resizedImg.rgb, + { width: detInputWidth, height: detInputHeight }, [0, 1, 2], ); - const tensorData = new Float32Array(inputImage); + const tensorData = new Float32Array(inputImageTransformedData); const tensor = new ov.Tensor(ov.element.f32, detInputLayer.shape, tensorData); + // Run inference on the detection model const detInferRequest = detCompiledModel.createInferRequest(); - const detResult = await detInferRequest.inferAsync([tensor]); - const boundingBoxesArray = extractBoundingBoxes(detResult[detOutputLayer]); + // Load the recognition model const recModel = await core.readModel(recModelXMLPath); const recModelCompiled = await core.compileModel(recModel, deviceName); - const recInputLayer = recModelCompiled.input(0); - const recOutputLayer = recModelCompiled.output(0); + const recInferRequest = recModelCompiled.createInferRequest(); - // Process each bounding box and run inference on the recognition model - const [, , height, width] = recInputLayer.shape; // Calculate ratios - const { ratioX, ratioY } = calculateRatios(inputImageMat, resizedImage); + const [ratioX, ratioY] = + [img.width / detInputWidth, img.height / detInputHeight]; + const boundingBoxesArray = extractBoundingBoxes(detResult[detOutputLayer]); + // Resize bounding boxes to the original image size + const boundingBoxesOriginalSizeArray = boundingBoxesArray.map(box => + [...multiplyByRatio(ratioX, ratioY, box), box[4]]); - // Convert image to grayscale - const grayscaleImage = convertToGrayscale(inputImageMat); + // Process each bounding box and run inference on the recognition model + const boxesWithAnnotations = []; + for (let i = 0; i < boundingBoxesOriginalSizeArray.length; i++) { + const box = boundingBoxesOriginalSizeArray[i]; + const [xMin, yMin, xMax, yMax] = box; + const croppedImg = img.crop(xMin, yMin, xMax - xMin, yMax - yMin); + await croppedImg.save(OUTPUT_PATH + `cropped_image_${i}.jpg`); - const annotations = []; - const croppedImages = []; + const annotation = + await performTextRecognition(recModel, recInferRequest, croppedImg); - for (let i = 0; i < boundingBoxesArray.length; i++) { - const crop = boundingBoxesArray[i]; - const [xMin, yMin, xMax, yMax] = multiplyByRatio(ratioX, ratioY, crop).map( - Math.floor, - ); - const cropRect = new cv.Rect(xMin, yMin, xMax - xMin, yMax - yMin); - const croppedImage = grayscaleImage.roi(cropRect); - - try { - const preprocessedCrop = resizeAndConvertCropToModelInput(croppedImage, [ - width, - height, - ]); - const tensorData = new Float32Array(preprocessedCrop); - const tensor = new ov.Tensor( - ov.element.f32, - Int32Array.from(recInputLayer.shape), - tensorData, - ); - - await inferAsyncProcess( - tensor, - recModelCompiled, - recOutputLayer, - i, - annotations, - ); - - croppedImages.push(cropImage(inputImageMat, xMin, yMin, xMax, yMax)); - } catch(error) { - console.error('Error during preprocessing:', error); - } - - croppedImage.delete(); + boxesWithAnnotations.push({ box, annotation }); + + console.log(`Box ${i}: [${box.join(',')}], Annotation: '${annotation}'`); } - grayscaleImage.delete(); + const annotatedImg = await putAnnotationsOnTheImage( + img, + boxesWithAnnotations, + { threshold: 0.3, confLabels: false }, + ); + const savePath = path.join(OUTPUT_PATH, 'output_image.jpg'); + await annotatedImg.save(savePath); + console.log(`The result was saved to ${savePath}`); +} - const boxesWithAnnotations = boundingBoxesArray.map((box, index) => ({ - box, - annotation: annotations[index], - })); +async function performTextRecognition(model, inferenceRequest, img) { + const inputLayerShape = model.input(0).shape; + const outputLayer = model.output(0); - logBoxesWithAnnotations(boxesWithAnnotations); + const [,, inputHeight, inputWidth] = inputLayerShape; + const resizedImg = img.resize(inputWidth, inputHeight); - convertResultToImage( - inputImageMat, - resizedImage, - boxesWithAnnotations, - { threshold: 0.3, confLabels: true }, - './assets/results/output_image.jpg', + // Convert image to grayscale and create tensor + const tensor = new ov.Tensor( + ov.element.f32, + inputLayerShape, + new Float32Array(resizedImg.grayscale), ); - croppedImages.forEach((croppedImage, i) => { - const savePath = `./assets/results/cropped_image_${i}.jpg`; - saveImage(croppedImage, savePath); - }); + const result = await inferenceRequest.inferAsync([tensor]); + const recognitionResults = extractRecognitionResults(result[outputLayer]); + const annotation = parseAnnotations(recognitionResults); + + return annotation; } // Function to extract bounding boxes from the model output @@ -147,56 +124,17 @@ function extractBoundingBoxes(output) { return setShape(boxes, [numberOfBoxes, foldingCoefficient]); } -// Function to calculate the ratios for the image -function calculateRatios(originalImage, resizedImage) { - const realY = originalImage.rows; - const realX = originalImage.cols; - const resizedY = resizedImage.rows; - const resizedX = resizedImage.cols; - const ratioX = realX / resizedX; - const ratioY = realY / resizedY; - - return { ratioX, ratioY }; -} - -// Function to convert the image to grayscale -function convertToGrayscale(originalImage) { - const grayscaleImage = new cv.Mat(); - cv.cvtColor(originalImage, grayscaleImage, cv.COLOR_BGR2GRAY); - - return grayscaleImage; -} - // Function to adjust bounding box coordinates by a given ratio function multiplyByRatio(ratioX, ratioY, box) { - const scaleShape = (shape, idx) => - idx % 2 ? Math.max(shape * ratioY, 10) : shape * ratioX; - - return box.map(scaleShape); -} - -// Function to resize and convert a crop to the recognition model input format -function resizeAndConvertCropToModelInput(crop, netShape) { - const [netWidth, netHeight] = netShape; + const scaleShape = (shape, idx) => { + const position = idx % 2 + ? Math.max(shape * ratioY, 10) + : shape * ratioX; - // Resize the crop to the network's input shape - const tempImg = new cv.Mat(); - cv.resize(crop, tempImg, new cv.Size(netWidth, netHeight)); - - // Create the reshaped buffer - const reshapedBuffer = new Uint8Array(netHeight * netWidth); - let index = 0; - - for (let i = 0; i < netHeight; i++) { - for (let j = 0; j < netWidth; j++) { - reshapedBuffer[index++] = tempImg.ucharPtr(i, j)[0]; - } + return Math.floor(position); } - // Clean up - tempImg.delete(); - - return reshapedBuffer; + return box.map(scaleShape); } // Function to extract recognition results from the model output @@ -219,204 +157,49 @@ function parseAnnotations(recognitionResults) { // Stop if end character is encountered if (parsedLetter === letters[0]) break; + annotation.push(parsedLetter); } return annotation.join(''); } -// Function to crop the image based on the bounding box coordinates -function cropImage(originalImage, xMin, yMin, xMax, yMax) { - xMin = Math.max(0, xMin); - yMin = Math.max(0, yMin); - xMax = Math.min(originalImage.cols, xMax); - yMax = Math.min(originalImage.rows, yMax); - if (xMin >= xMax || yMin >= yMax) { - throw new Error('Invalid crop coordinates'); - } - const roi = originalImage.roi( - new cv.Rect(xMin, yMin, xMax - xMin, yMax - yMin), - ); - const cropped = new cv.Mat(); - roi.copyTo(cropped); - roi.delete(); - - return cropped; -} - -// Get Text size -function getTextSize(text, fontFace, fontScale) { - const canvas = createCanvas(200, 200); - const ctx = canvas.getContext('2d'); - const adjustedFontScale = fontScale * 35; - ctx.font = `${adjustedFontScale}px ${fontFace}`; - const metrics = ctx.measureText(text); - const width = metrics.width; - const height = - metrics.actualBoundingBoxAscent + metrics.actualBoundingBoxDescent; - - return { width, height }; -} - -/* The convertResultToImage function visualizes object detection - results on an image by drawing bounding boxes around detected - objects and optionally adding labels to them. */ - -function convertResultToImage( - bgrImage, - resizedImage, - boxesWithAnnotations, - options, - savePath, -) { +// Takes original image and bounding boxes with annotations +// and returns the image with annotations +async function putAnnotationsOnTheImage(img, boxesWithAnnotations, options) { const defaultOptions = { threshold: 0.3, confLabels: true }; const { threshold, confLabels } = Object.assign(defaultOptions, options); - const colors = { - red: [255, 0, 0, 255], - green: [0, 255, 0, 255], - white: [255, 255, 255, 255], - }; - const [realY, realX] = [bgrImage.rows, bgrImage.cols]; - const [resizedY, resizedX] = [resizedImage.rows, resizedImage.cols]; - const [ratioX, ratioY] = [realX / resizedX, realY / resizedY]; - - const rgbImage = new cv.Mat(); - cv.cvtColor(bgrImage, rgbImage, cv.COLOR_BGR2RGB); + let finalImage = img; - boxesWithAnnotations.forEach(({ box, annotation }) => { + for (const item of boxesWithAnnotations) { + const { box, annotation } = item; const conf = box[box.length - 1]; - if (conf < threshold) return; - - const [xMin, yMin, xMax, yMax] = multiplyByRatio(ratioX, ratioY, box); - - cv.rectangle( - rgbImage, - new cv.Point(xMin, yMin), - new cv.Point(xMax, yMax), - colors.green, - 3, - ); + if (conf < threshold) continue; - if (!confLabels) return; + const [xMin, yMin, xMax, yMax] = box; + const yOffset = 10; - const text = `${annotation}`; - const fontScale = 0.8; - const thickness = 1; - const { width: textW, height: textH } = getTextSize( - text, - 'Arial', - fontScale, + finalImage = finalImage.drawRect( + xMin, yMin, + xMax - xMin, yMax - yMin, + { color: 'green', width: 3 }, ); - const imageCopy = rgbImage.clone(); - - cv.rectangle( - imageCopy, - new cv.Point(xMin, yMin - textH - 10), - new cv.Point(xMin + textW, yMin - 10), - colors.white, - cv.FILLED, + finalImage = finalImage.drawText( + annotation, + xMin, yMin - yOffset, + { font: '30px Arial' }, ); - cv.addWeighted(imageCopy, 0.4, rgbImage, 0.6, 0, rgbImage); - cv.putText( - rgbImage, - text, - new cv.Point(xMin, yMin - 10), - cv.FONT_HERSHEY_SIMPLEX, - fontScale, - colors.red, - thickness, - cv.LINE_AA, - ); - - imageCopy.delete(); - }); - - const saveDir = path.dirname(savePath); - if (!fs.existsSync(saveDir)) { - fs.mkdirSync(saveDir, { recursive: true }); - } - try { - saveImage(rgbImage, savePath); - } catch(e) { - console.log(`Error occurred while saving ----> ${e}`); - } - - return rgbImage; -} + if (!confLabels) continue; -// Infer async helper function - -async function inferAsyncProcess( - tensor, - recModelCompiled, - recOutputLayer, - i, - annotations, -) { - // Create infer request - const inferRequest = recModelCompiled.createInferRequest(); - - // Define the completion callback function - function completionCallback(outputTensor, i, annotations) { - const recognitionResults = extractRecognitionResults(outputTensor); - const annotation = parseAnnotations(recognitionResults); - annotations.push(annotation); - } - - // Start inference in asynchronous mode - try { - const result = await inferRequest.inferAsync([tensor]); - completionCallback(result[recOutputLayer], i, annotations); - } catch(error) { - console.error('Error during inference:', error); - } -} - -// Log boudning boxes with annotations -function logBoxesWithAnnotations(boxesWithAnnotations) { - boxesWithAnnotations.forEach((item, i) => { - const { box, annotation } = item; - console.log(`Box ${i}: [${box}], Annotation: ${annotation}`); - }); -} - -function saveImage(rgbImage, savePath) { - const canvas = createCanvas(rgbImage.cols, rgbImage.rows); - const ctx = canvas.getContext('2d'); - const componentsPerPixel = - rgbImage.data.length / (rgbImage.cols * rgbImage.rows); - const imgDataArr = []; - - if (componentsPerPixel === 1) { - for (const val of rgbImage.data) { - imgDataArr.push(val, val, val, 255); - } - } else if (componentsPerPixel === 3) { - for (let i = 0; i < rgbImage.data.length; i++) { - if (i % 3 === 0) imgDataArr.push(255); - imgDataArr.push(rgbImage.data[i]); - } - } - - const imageData = new ImageData( - new Uint8ClampedArray(imgDataArr), - rgbImage.cols, - rgbImage.rows, - ); - ctx.putImageData(imageData, 0, 0); - - const dataURL = canvas.toDataURL('image/jpeg'); - const base64Data = dataURL.replace(/^data:image\/jpeg;base64,/, ''); - const imageBuffer = Buffer.from(base64Data, 'base64'); - - const saveDir = path.dirname(savePath); - if (!fs.existsSync(saveDir)) { - fs.mkdirSync(saveDir, { recursive: true }); + finalImage = finalImage.drawText( + conf.toFixed(2), + xMin, yMax + 2 * yOffset, + { font: '20px Arial' }, + ); } - fs.writeFileSync(savePath, imageBuffer); - console.log('Image saved successfully!', savePath); + return finalImage; } diff --git a/samples/js/node/package-lock.json b/samples/js/node/package-lock.json index fe003eaac14935..96a013fb0435c7 100644 --- a/samples/js/node/package-lock.json +++ b/samples/js/node/package-lock.json @@ -7,14 +7,14 @@ "": { "name": "openvino-node-demo", "version": "1.0.0", + "hasInstallScript": true, "license": "Apache-2.0", "devDependencies": { + "@napi-rs/canvas": "^0.1.59", "@tensorflow/tfjs-node": "^4.19.0", "args": "^5.0.3", - "canvas": "^2.11.2", "eslint": "^8.39.0", "https-proxy-agent": "^7.0.2", - "opencv-wasm": "^4.3.0-10", "openvino-node": "^2024.4.0" }, "engines": { @@ -173,6 +173,180 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.59.tgz", + "integrity": "sha512-3vUtQ8DzYcz9xy86UUe8OfDiXNuuLB9zFAUs5N/I2GpkY/MWBJ2M7w5FqH380oC44IzYOWaOMLWCPfNZBsbBww==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.59", + "@napi-rs/canvas-darwin-arm64": "0.1.59", + "@napi-rs/canvas-darwin-x64": "0.1.59", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.59", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.59", + "@napi-rs/canvas-linux-arm64-musl": "0.1.59", + "@napi-rs/canvas-linux-x64-gnu": "0.1.59", + "@napi-rs/canvas-linux-x64-musl": "0.1.59", + "@napi-rs/canvas-win32-x64-msvc": "0.1.59" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.59.tgz", + "integrity": "sha512-p4rRL9KIDz57Z+gKLpemX36DB7fVVHmY4DtesMGrnjx4gSBUM2M7LNzbzf4o3oPZGDiHMY0vnvNHR4dKfszNeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.59.tgz", + "integrity": "sha512-+8s06WxcM9ilv9PVOl57hvasbwKWMfrrNAYknqMPCn4jpc4XDcLbrM5LTZGhhptlv9jQ9DmHfZ978/xInsMYXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.59.tgz", + "integrity": "sha512-6kziJHjXdxduYK2L2uuwjEIYoPJednKq+C81MCm3fPobXE4HBKs0JGXwq3GkWNe340U340vmagwXiFi6muEy+g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.59.tgz", + "integrity": "sha512-eCkyS7jojNmaUPaVFdNjAyS0R3isrJtUfRf1vRP6K50GRuHso3vwQRbZBPKM71qHdjPDylfaQc5H6/M7epyD+w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.59.tgz", + "integrity": "sha512-1u4++lbsolP1MAPViuDoZmgmDLKlV0iJnlHN2dfwgbu3t53P0l3jIT1oCIAiWil0OlrWtDF24JbY7LUUGH5aHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.59.tgz", + "integrity": "sha512-eqevZ2kWPxeAnvhxl7U5tf6AiMnhlO4w2Hci79WQkfeirqQG6RRM4Jnxbh9iO3jkAnnOXmM4r+S3UrOcfIx1Rg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.59.tgz", + "integrity": "sha512-F+T63RnLt0qYUXhbOpaome3vIWLW4xoQRmhTnkKDzOtBSnKVP7sCM6E5/5tByOFCR3fTj4ksMeeHy8zJScEExA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.59.tgz", + "integrity": "sha512-HhUgpTGQUR2VRslEC5Idf6s0hhamJiVlEh2k3AG9XXOwX6fg0xXkqm84DPiOCLzsO5bqtJEo+rh03BUSDcf53g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.59", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.59.tgz", + "integrity": "sha512-bYMiZJsKPkU7HEoYI5E0alOSV1EkaigY4VEgGHPK9W/qGMmNFsxdbURQqa5h3zbhZTK5QRSdYYqowcTEYVIlug==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -774,21 +948,6 @@ "node": ">=6" } }, - "node_modules/canvas": { - "version": "2.11.2", - "resolved": "https://registry.npmjs.org/canvas/-/canvas-2.11.2.tgz", - "integrity": "sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==", - "dev": true, - "hasInstallScript": true, - "dependencies": { - "@mapbox/node-pre-gyp": "^1.0.0", - "nan": "^2.17.0", - "simple-get": "^3.0.3" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -924,18 +1083,6 @@ } } }, - "node_modules/decompress-response": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", - "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", - "dev": true, - "dependencies": { - "mimic-response": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -1745,18 +1892,6 @@ "node": ">= 0.6" } }, - "node_modules/mimic-response": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", - "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==", - "dev": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -1830,12 +1965,6 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, - "node_modules/nan": { - "version": "2.20.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.20.0.tgz", - "integrity": "sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==", - "dev": true - }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -1908,12 +2037,6 @@ "wrappy": "1" } }, - "node_modules/opencv-wasm": { - "version": "4.3.0-10", - "resolved": "https://registry.npmjs.org/opencv-wasm/-/opencv-wasm-4.3.0-10.tgz", - "integrity": "sha512-EWmWLUzp2suoc6N44Y4ouWT85QwvShx23Q430R+lp6NyS828bjQn6mCgA3NJ6Z/S59aaTeeu+RhqPQIJIYld1w==", - "dev": true - }, "node_modules/openvino-node": { "version": "2024.4.0", "resolved": "https://registry.npmjs.org/openvino-node/-/openvino-node-2024.4.0.tgz", @@ -2272,37 +2395,6 @@ "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "dev": true }, - "node_modules/simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/simple-get": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.1.tgz", - "integrity": "sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==", - "dev": true, - "dependencies": { - "decompress-response": "^4.2.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", diff --git a/samples/js/node/package.json b/samples/js/node/package.json index 683312fab6742b..d392a72143de03 100644 --- a/samples/js/node/package.json +++ b/samples/js/node/package.json @@ -2,17 +2,18 @@ "name": "openvino-node-demo", "version": "1.0.0", "license": "Apache-2.0", + "type": "commonjs", "devDependencies": { + "@tensorflow/tfjs-node": "^4.19.0", "args": "^5.0.3", - "canvas": "^2.11.2", "eslint": "^8.39.0", "https-proxy-agent": "^7.0.2", - "opencv-wasm": "^4.3.0-10", - "@tensorflow/tfjs-node": "^4.19.0", - "openvino-node": "^2024.4.0" + "openvino-node": "^2024.4.0", + "@napi-rs/canvas": "^0.1.59" }, "scripts": { - "lint": "eslint ." + "lint": "eslint .", + "postinstall": "node ./fetch-samples-assets.js" }, "engines": { "node": ">=21.0.0" diff --git a/samples/js/node/vision_background_removal/README.md b/samples/js/node/vision_background_removal/README.md index cdefc0c4186cbb..a9a3e3270eae45 100644 --- a/samples/js/node/vision_background_removal/README.md +++ b/samples/js/node/vision_background_removal/README.md @@ -1,6 +1,10 @@ # Vision Background Removal Node.js Sample -Run: +Run sample: ```bash -node vision_background_removal.js *path_to_model_file* *path_to_foreground_image* *path_to_background_image* AUTO -``` \ No newline at end of file +node vision_background_removal.js ../../assets/models/unet_ir_model.xml ../../assets/images/coco_hollywood.jpg ../../assets/images/wall.jpg AUTO +``` +Where: +```bash +node vision_background_removal.js *path_to_model_file* *path_to_foreground_image* *path_to_background_image* *device* +``` diff --git a/samples/js/node/vision_background_removal/vision_background_removal.js b/samples/js/node/vision_background_removal/vision_background_removal.js index 2487371c597008..aabaad1a65b80c 100644 --- a/samples/js/node/vision_background_removal/vision_background_removal.js +++ b/samples/js/node/vision_background_removal/vision_background_removal.js @@ -1,12 +1,10 @@ -const { cv } = require('opencv-wasm'); -const fs = require('node:fs').promises; -const path = require('node:path'); const { addon: ov } = require('openvino-node'); -const { createCanvas, ImageData } = require('canvas'); -const { getImageData, transform, setShape } = require('../helpers'); + +const { transform } = require('../helpers'); +const Image = require('../image'); if (require.main === module) { -// Parsing and validation of input arguments + // Parsing and validation of input arguments if (process.argv.length !== 6) throw new Error( `Usage: ${process.argv[1]} ` + @@ -15,21 +13,23 @@ if (require.main === module) { ); const unetModelPath = process.argv[2]; - const foreGroundImage = process.argv[3]; - const backGroundImage = process.argv[4]; + const foregroundImagePath = process.argv[3]; + const backgroundImagePath = process.argv[4]; const deviceName = process.argv[5]; try { - main(unetModelPath, foreGroundImage, backGroundImage, deviceName); + main(unetModelPath, foregroundImagePath, backgroundImagePath, deviceName); } catch(error) { console.error('Error occurred', error); } } +module.exports = main; + async function main( unetModelPath, - foreGroundImage, - backGroundImage, + foregroundImagePath, + backgroundImagePath, deviceName, ) { const core = new ov.Core(); @@ -38,104 +38,81 @@ async function main( const model = await core.readModel(unetModelPath); const compiledModel = await core.compileModel(model, deviceName); - // Get the names of input and output layers. + // Get the names of input and output layers const inputLayer = compiledModel.input(0); const outputLayer = compiledModel.output(0); - // Get Image data from the foreground image - const imageData = await getImageData(foreGroundImage); - const inputImageMat = cv.matFromImageData(imageData); - - // Convert the image shape to a shape and a data type expected by the network - const [, , H, W] = inputLayer.shape; - const resizedImage = new cv.Mat(); - cv.cvtColor(inputImageMat, inputImageMat, cv.COLOR_BGR2RGB); - cv.resize(inputImageMat, resizedImage, new cv.Size(W, H)); + // Load foreground image + const originalImg = await Image.load(foregroundImagePath); - const inputImage = transform( - resizedImage.data, - { width: W, height: H }, - [0, 1, 2], - ); - - // Normalize the input image Mat - const normalizedInputImage = normalizeImage(inputImage, W, H); + // Resize image to a shape expected by the network + const [, , modelInputHeight, modelInputWidth] = inputLayer.shape; + const resized = await originalImg.resize(modelInputWidth, modelInputHeight); // Create a tensor from the normalized input image - const tensorData = new Float32Array(normalizedInputImage); - const tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, tensorData); + const transformed = transform( + resized.rgb, + { + width: modelInputWidth, + height: modelInputHeight + }, + [0, 1, 2] + ); + const normalizedInputImage = normalizeImage( + transformed, + modelInputWidth, + modelInputHeight, + ); + const tensor = new ov.Tensor(ov.element.f32, inputLayer.shape, normalizedInputImage); // Do inference const inferRequest = compiledModel.createInferRequest(); const inferResult = await inferRequest.inferAsync([tensor]); + const { data: resultData } = inferResult[outputLayer]; - const { data } = inferResult[outputLayer]; - const reshapedResult = setShape(data, [512, 512]); - - // Create a Mat from the reshaped result - const reshapedMat = cv.matFromArray( - 512, - 512, - cv.CV_32F, - reshapedResult.flat(), - ); - - // Get the height and width of the original image - const height = inputImageMat.rows; - const width = inputImageMat.cols; - - // Resize the inference result to the original image size - const resizedResult = new cv.Mat(); - cv.resize( - reshapedMat, - resizedResult, - new cv.Size(width, height), - 0, - 0, - cv.INTER_LINEAR, - ); - - // Convert the resized result to uint8 - resizedResult.convertTo(resizedResult, cv.CV_8U); - - // Create a Mat to store the background removed result - const bgRemovedResult = inputImageMat.clone(); + // Normalize the result data from grayscale to RGB + const rgbData = []; + for (let i = 0; i < resultData.length; i += 1) { + const value = resultData[i] * 255; - removeBackground(resizedResult, bgRemovedResult); + rgbData.push(value, value, value, 255); + } - // Save the background removed result - await saveImage(bgRemovedResult, './bg_removed_result.jpg'); + // Create image based on result data + const [outputHeight, outputWidth] = outputLayer.shape.slice(2); + const maskImg = await Image.fromArray(rgbData, outputWidth, outputHeight); - // Get the background image data - const bgrImageData = await getImageData(backGroundImage); - const bgrImageMat = cv.matFromImageData(bgrImageData); + // Resize the result mask to the original image size and save it + const { width, height } = originalImg; + const resizedMaskImg = await maskImg.resize(originalImg.width, originalImg.height); + const maskImagePath = './out_mask.jpg'; + await resizedMaskImg.save(maskImagePath); + console.log(`The mask image was saved to '${maskImagePath}'`); - // Resize the background image to the original image size - const resizedBgrImageMat = new cv.Mat(); - cv.cvtColor(bgrImageMat, bgrImageMat, cv.COLOR_BGR2RGB); - cv.resize(bgrImageMat, resizedBgrImageMat, new cv.Size(width, height)); + // Remove the foreground from the original image + const removedBgImg = Image.mask(originalImg, resizedMaskImg); - // Remove the foreground from the background image by - // setting all foreground pixels to white - removeForeground(resizedResult, resizedBgrImageMat); + // Load the background image + const bgrImage = await Image.load(backgroundImagePath); - // Save the foreground removed from the background image - await saveImage(resizedBgrImageMat, './fg_removed_from_background.jpg'); + // Resize the background image to the same size as the original image + const resizedBgrImage = bgrImage.resize(width, height); - // create a new Mat to store the final image - const newImage = new cv.Mat( - resizedBgrImageMat.rows, - resizedBgrImageMat.cols, - cv.CV_8UC3, - ); + // Remove object from the background image + const removedFgImg = Image.mask(resizedBgrImage, resizedMaskImg.invert()); - // combine the foreground and background images to get the final image - combineImages(resizedResult, bgRemovedResult, resizedBgrImageMat, newImage); + // Combine the background and foreground images + const resultImg = Image.merge(removedBgImg, removedFgImg); // Save the final image - await saveImage(newImage, './background_changed_image.jpg'); + const outputImagePath = './out_bgr_changed_image.jpg'; + await resultImg.save(outputImagePath); + console.log(`The result image was saved to '${outputImagePath}'`); + console.log('The background was successfully changed'); } +// Details about this normalization: +// https://docs.openvino.ai/2024/notebooks/vision-background-removal-with-output.html#load-and-pre-process-input-image function normalizeImage(imageData, width, height) { // Mean and scale values const inputMean = [123.675, 116.28, 103.53]; @@ -148,6 +125,7 @@ function normalizeImage(imageData, width, height) { for (let j = 0; j < width; j++) { for (let c = 0; c < channels; c++) { const index = i * width * channels + j * channels + c; + normalizedData[index] = (imageData[index] - inputMean[c]) / inputScale[c]; } @@ -156,93 +134,3 @@ function normalizeImage(imageData, width, height) { return normalizedData; } - -function removeBackground(mask, image) { - // Iterate over the mask and set all background pixels to white - for (let i = 0; i < mask.rows; i++) { - for (let j = 0; j < mask.cols; j++) { - if (mask.ucharPtr(i, j)[0] === 0) { - image.ucharPtr(i, j)[0] = 255; - image.ucharPtr(i, j)[1] = 255; - image.ucharPtr(i, j)[2] = 255; - } - } - } -} - -function removeForeground(mask, image) { - // Iterate over the mask and set all foreground pixels to black - for (let i = 0; i < mask.rows; i++) { - for (let j = 0; j < mask.cols; j++) { - if (mask.ucharPtr(i, j)[0] === 1) { - image.ucharPtr(i, j)[0] = 0; - image.ucharPtr(i, j)[1] = 0; - image.ucharPtr(i, j)[2] = 0; - } else { - image.ucharPtr(i, j)[0] = image.ucharPtr(i, j)[0]; - image.ucharPtr(i, j)[1] = image.ucharPtr(i, j)[1]; - image.ucharPtr(i, j)[2] = image.ucharPtr(i, j)[2]; - } - } - } -} - -function combineImages(mask, fgImage, bgImage, newImage) { - // Iterate over the mask and combine the foreground and background images - for (let i = 0; i < mask.rows; i++) { - for (let j = 0; j < mask.cols; j++) { - if (mask.ucharPtr(i, j)[0] === 1) { - newImage.ucharPtr(i, j)[0] = fgImage.ucharPtr(i, j)[0]; - newImage.ucharPtr(i, j)[1] = fgImage.ucharPtr(i, j)[1]; - newImage.ucharPtr(i, j)[2] = fgImage.ucharPtr(i, j)[2]; - } else { - newImage.ucharPtr(i, j)[0] = bgImage.ucharPtr(i, j)[0]; - newImage.ucharPtr(i, j)[1] = bgImage.ucharPtr(i, j)[1]; - newImage.ucharPtr(i, j)[2] = bgImage.ucharPtr(i, j)[2]; - } - } - } -} - -async function saveImage(rgbImage, savePath) { - const canvas = createCanvas(rgbImage.cols, rgbImage.rows); - const ctx = canvas.getContext('2d'); - const componentsPerPixel = - rgbImage.data.length / (rgbImage.cols * rgbImage.rows); - const imgDataArr = []; - - if (componentsPerPixel === 1) { - for (const val of rgbImage.data) { - imgDataArr.push(val, val, val, 255); - } - } else if (componentsPerPixel === 3) { - for (let i = 0; i < rgbImage.data.length; i += 3) { - imgDataArr.push( - rgbImage.data[i + 2], // Red - rgbImage.data[i + 1], // Green - rgbImage.data[i], // Blue - 255, // Alpha - ); - } - } - - const imageData = new ImageData( - new Uint8ClampedArray(imgDataArr), - rgbImage.cols, - rgbImage.rows, - ); - ctx.putImageData(imageData, 0, 0); - - const dataURL = canvas.toDataURL('image/jpeg'); - const base64Data = dataURL.replace(/^data:image\/jpeg;base64,/, ''); - const imageBuffer = Buffer.from(base64Data, 'base64'); - - const saveDir = path.dirname(savePath); - try { - await fs.mkdir(saveDir, { recursive: true }); - await fs.writeFile(savePath, imageBuffer); - console.log('Image saved successfully!', savePath); - } catch(error) { - console.error('Error saving image:', error); - } -} diff --git a/src/bindings/c/include/openvino/c/ov_prepostprocess.h b/src/bindings/c/include/openvino/c/ov_prepostprocess.h index ea2a82943dd683..d7cbbdc26eb5cb 100644 --- a/src/bindings/c/include/openvino/c/ov_prepostprocess.h +++ b/src/bindings/c/include/openvino/c/ov_prepostprocess.h @@ -93,6 +93,18 @@ typedef enum { RESIZE_NEAREST //!< nearest algorithm } ov_preprocess_resize_algorithm_e; +/** + * @enum ov_padding_mode_e + * @ingroup ov_prepostprocess_c_api + * @brief This enum contains enumeration for padding mode. + */ +typedef enum { + CONSTANT = 0, //!< Pads with given constant value. + EDGE, //!< Pads with tensor edge values. + REFLECT, //!< Pads with reflection of tensor data along axis. Values on the edges are not duplicated. + SYMMETRIC //!< Pads similar like `REFLECT` but values on the edges are duplicated. +} ov_padding_mode_e; + /** * @brief Create a ov_preprocess_prepostprocessor_t instance. * @ingroup ov_prepostprocess_c_api @@ -512,3 +524,23 @@ ov_preprocess_input_model_info_set_layout(ov_preprocess_input_model_info_t* prep */ OPENVINO_C_API(ov_status_e) ov_preprocess_prepostprocessor_build(const ov_preprocess_prepostprocessor_t* preprocess, ov_model_t** model); + +/** + * @brief Add pad preprocess operation. Extends an input tensor on edges with constants. + * + * @param preprocess_input_process_steps A pointer to the ov_preprocess_preprocess_steps_t. + * @param pads_begin Number of padding elements to add at the beginning of each axis. + * @param pads_begin_size Pads begin size (number of axes). + * @param pads_end Number of padding elements to add at the end of each axis. + * @param pads_end_size Pads end size (number of axes). + * @param value Value to be populated in the padded area (mode=CONSTANT) + * @param mode Padding mode. + */ +OPENVINO_C_API(ov_status_e) +ov_preprocess_preprocess_steps_pad(const ov_preprocess_preprocess_steps_t* preprocess_input_process_steps, + const int* const pads_begin, + size_t pads_begin_size, + const int* const pads_end, + size_t pads_end_size, + float value, + ov_padding_mode_e mode); diff --git a/src/bindings/c/src/ov_prepostprocess.cpp b/src/bindings/c/src/ov_prepostprocess.cpp index 616883dd54e74c..8448e9daeb10b1 100644 --- a/src/bindings/c/src/ov_prepostprocess.cpp +++ b/src/bindings/c/src/ov_prepostprocess.cpp @@ -24,6 +24,12 @@ const std::map color_format_map {ov_color_format_e::RGBX, ov::preprocess::ColorFormat::RGBX}, {ov_color_format_e::BGRX, ov::preprocess::ColorFormat::BGRX}}; +const std::map padding_mode_map = { + {ov_padding_mode_e::CONSTANT, ov::preprocess::PaddingMode::CONSTANT}, + {ov_padding_mode_e::EDGE, ov::preprocess::PaddingMode::EDGE}, + {ov_padding_mode_e::REFLECT, ov::preprocess::PaddingMode::REFLECT}, + {ov_padding_mode_e::SYMMETRIC, ov::preprocess::PaddingMode::SYMMETRIC}}; + #define GET_OV_COLOR_FARMAT(a) \ (color_format_map.find(a) == color_format_map.end() ? ov::preprocess::ColorFormat::UNDEFINED \ : color_format_map.at(a)) @@ -524,3 +530,23 @@ ov_status_e ov_preprocess_prepostprocessor_build(const ov_preprocess_prepostproc return ov_status_e::OK; } + +ov_status_e ov_preprocess_preprocess_steps_pad(const ov_preprocess_preprocess_steps_t* preprocess_input_process_steps, + const int* const pads_begin, + size_t pads_begin_size, + const int* const pads_end, + size_t pads_end_size, + float value, + ov_padding_mode_e mode) { + if (!preprocess_input_process_steps) { + return ov_status_e::INVALID_C_PARAM; + } + try { + std::vector vec_begin(pads_begin, pads_begin + pads_begin_size); + std::vector vec_end(pads_end, pads_end + pads_end_size); + preprocess_input_process_steps->object->pad(vec_begin, vec_end, value, padding_mode_map.at(mode)); + } + CATCH_OV_EXCEPTIONS + + return ov_status_e::OK; +} diff --git a/src/bindings/c/tests/ov_preprocess_test.cpp b/src/bindings/c/tests/ov_preprocess_test.cpp index 94252bd3a3cbb9..aeb7a097af1352 100644 --- a/src/bindings/c/tests/ov_preprocess_test.cpp +++ b/src/bindings/c/tests/ov_preprocess_test.cpp @@ -1,6 +1,8 @@ // Copyright (C) 2018-2024 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // +#include + #include "ov_test.hpp" class ov_preprocess_test : public ::testing::Test { @@ -596,3 +598,71 @@ TEST_F(ov_preprocess_test, ov_preprocess_prepostprocessor_for_nv12_input) { ov_layout_free(layout); } + +TEST_F(ov_preprocess_test, ov_preprocess_preprocess_steps_pad_constant) { + OV_EXPECT_OK(ov_preprocess_prepostprocessor_create(model, &preprocess)); + EXPECT_NE(nullptr, preprocess); + + OV_EXPECT_OK(ov_preprocess_prepostprocessor_get_input_info(preprocess, &input_info)); + EXPECT_NE(nullptr, input_info); + + OV_EXPECT_OK(ov_preprocess_input_info_get_tensor_info(input_info, &input_tensor_info)); + EXPECT_NE(nullptr, input_tensor_info); + + ov_shape_t shape; + int64_t dims[4] = {1, 1, 225, 230}; + OV_EXPECT_OK(ov_shape_create(4, dims, &shape)); + + OV_EXPECT_OK(ov_tensor_create(ov_element_type_e::F32, shape, &tensor)); + OV_EXPECT_OK(ov_preprocess_input_tensor_info_set_from(input_tensor_info, tensor)); + + OV_EXPECT_OK(ov_preprocess_input_info_get_preprocess_steps(input_info, &input_process)); + EXPECT_NE(nullptr, input_process); + + constexpr auto pads_begin = std::array{0, 1, 2, 0}; + constexpr auto pads_end = std::array{0, 1, 0, -3}; + + OV_EXPECT_OK(ov_preprocess_preprocess_steps_pad(input_process, + pads_begin.data(), + pads_begin.size(), + pads_end.data(), + pads_end.size(), + 1.0f, + ov_padding_mode_e::CONSTANT)); + OV_EXPECT_OK(ov_preprocess_prepostprocessor_build(preprocess, &ppp_model)); + EXPECT_NE(nullptr, ppp_model); +} + +TEST_F(ov_preprocess_test, ov_preprocess_preprocess_steps_pad_edge) { + OV_EXPECT_OK(ov_preprocess_prepostprocessor_create(model, &preprocess)); + EXPECT_NE(nullptr, preprocess); + + OV_EXPECT_OK(ov_preprocess_prepostprocessor_get_input_info(preprocess, &input_info)); + EXPECT_NE(nullptr, input_info); + + OV_EXPECT_OK(ov_preprocess_input_info_get_tensor_info(input_info, &input_tensor_info)); + EXPECT_NE(nullptr, input_tensor_info); + + ov_shape_t shape; + int64_t dims[4] = {1, 2, 225, 230}; + OV_EXPECT_OK(ov_shape_create(4, dims, &shape)); + + OV_EXPECT_OK(ov_tensor_create(ov_element_type_e::F32, shape, &tensor)); + OV_EXPECT_OK(ov_preprocess_input_tensor_info_set_from(input_tensor_info, tensor)); + + OV_EXPECT_OK(ov_preprocess_input_info_get_preprocess_steps(input_info, &input_process)); + EXPECT_NE(nullptr, input_process); + + constexpr auto pads_begin = std::array{0, 0, 2, 0}; + constexpr auto pads_end = std::array{0, 1, 0, -3}; + + OV_EXPECT_OK(ov_preprocess_preprocess_steps_pad(input_process, + pads_begin.data(), + pads_begin.size(), + pads_end.data(), + pads_end.size(), + 1.0f, + ov_padding_mode_e::EDGE)); + OV_EXPECT_OK(ov_preprocess_prepostprocessor_build(preprocess, &ppp_model)); + EXPECT_NE(nullptr, ppp_model); +} diff --git a/src/bindings/python/constraints.txt b/src/bindings/python/constraints.txt index c7837798c8aca7..a0fbf982105ad6 100644 --- a/src/bindings/python/constraints.txt +++ b/src/bindings/python/constraints.txt @@ -18,7 +18,7 @@ patchelf<=0.17.2.1 # Frontends h5py>=3.1.0,<3.13.0 docopt~=0.6.2 -paddlepaddle==2.6.0 +paddlepaddle==2.6.2 tensorflow>=1.15.5,<2.18.0 six~=1.16.0 protobuf>=3.18.1,<4.0.0 diff --git a/src/bindings/python/requirements_test.txt b/src/bindings/python/requirements_test.txt index a9ee93b1d9e9bc..1aa2ff24b1b948 100644 --- a/src/bindings/python/requirements_test.txt +++ b/src/bindings/python/requirements_test.txt @@ -7,7 +7,7 @@ flake8-annotations-complexity<=0.0.8 flake8-broken-line<=1.0.0 flake8-bugbear<=24.8.19 flake8-class-attributes-order<=0.1.3 -flake8-comprehensions<=3.15.0 +flake8-comprehensions<=3.16.0 flake8-debugger<=4.1.2 flake8-docstrings<=1.7.0 flake8-eradicate<=1.5.0 diff --git a/src/bindings/python/src/openvino/frontend/pytorch/torchdynamo/backend.py b/src/bindings/python/src/openvino/frontend/pytorch/torchdynamo/backend.py index c6c01fa98e5e99..9f2ef019769875 100644 --- a/src/bindings/python/src/openvino/frontend/pytorch/torchdynamo/backend.py +++ b/src/bindings/python/src/openvino/frontend/pytorch/torchdynamo/backend.py @@ -49,18 +49,22 @@ openvino_options = {} -@register_backend +# Disable regional compilation which was enabled by default from Torch 2.5.0 +if hasattr(torch._dynamo.config, "inline_inbuilt_nn_modules"): + torch._dynamo.config.inline_inbuilt_nn_modules=False + @fake_tensor_unsupported def openvino(subgraph, example_inputs, options=None): - if (_get_aot_autograd(options)): + if _get_aot_autograd(options): global openvino_options openvino_options = options decompositions = _get_decompositions(options) + get_inf_decomposition_list() + get_aot_decomposition_list() - return aot_autograd(fw_compiler=fx_openvino, - bw_compiler=fx_openvino, - decompositions=get_decompositions(decompositions))(subgraph, example_inputs) + return aot_autograd(fw_compiler=fx_openvino, bw_compiler=fx_openvino, decompositions=get_decompositions(decompositions))(subgraph, example_inputs) return fx_openvino(subgraph, example_inputs, options) +if "openvino" not in torch.compiler.list_backends(): + register_backend(compiler_fn=openvino, name="openvino") + def fx_openvino(subgraph, example_inputs, options=None): try: if len(openvino_options) != 0: @@ -70,7 +74,7 @@ def fx_openvino(subgraph, example_inputs, options=None): openvino_model_caching = _get_model_caching(options) if openvino_model_caching is not None and openvino_model_caching: # Create a hash to be used for caching - model_hash_str = sha256(subgraph.code.encode('utf-8')).hexdigest() + model_hash_str = sha256(subgraph.code.encode("utf-8")).hexdigest() executor_parameters = {"model_hash_str": model_hash_str} # Check if the model was fully supported and already cached example_inputs.reverse() @@ -79,15 +83,17 @@ def fx_openvino(subgraph, example_inputs, options=None): if os.path.isfile(maybe_fs_cached_name + ".xml") and os.path.isfile(maybe_fs_cached_name + ".bin"): # Model is fully supported and already cached. Run the cached OV model directly. compiled_model = openvino_compile_cached_model(maybe_fs_cached_name, options, *example_inputs) + def _call(*args): res = execute_cached(compiled_model, *args) return res + return _call if inputs_reversed: example_inputs.reverse() preserved_arg_indices = [] - if (_get_aot_autograd(options)): + if _get_aot_autograd(options): if tracing_context := torch._guards.TracingContext.try_get(): fw_metadata = tracing_context.fw_metadata params_flat = tracing_context.params_flat @@ -97,6 +103,7 @@ def _call(*args): model = subgraph else: from torch._subclasses.fake_tensor import FakeTensorMode + decompositions = _get_decompositions(options) + get_inf_decomposition_list() with FakeTensorMode(allow_non_fake_inputs=True): model = make_fx(subgraph, decomposition_table=get_decompositions(decompositions))(*example_inputs) @@ -106,26 +113,27 @@ def _call(*args): partitioner = Partitioner(options) compiled_model = partitioner.make_partitions(model, options) - if executor_parameters is not None and 'model_hash_str' in executor_parameters: + if executor_parameters is not None and "model_hash_str" in executor_parameters: # Check if the model is fully supported. fully_supported = partitioner.check_fully_supported(compiled_model) if fully_supported: executor_parameters["model_hash_str"] += "_fs" def _call(*args): - if(_get_aot_autograd(options)): + if _get_aot_autograd(options): args_list = args[0] args_new = [args_list[i] for i in preserved_arg_indices] args = args_new - res = execute(compiled_model, *args, executor="openvino", - executor_parameters=executor_parameters, options=options) + res = execute(compiled_model, *args, executor="openvino", executor_parameters=executor_parameters, options=options) return res - if(_get_aot_autograd(options)): - _call._boxed_call = True # type: ignore[attr-defined] + + if _get_aot_autograd(options): + _call._boxed_call = True # type: ignore[attr-defined] return _call except Exception as e: logger.debug(f"Failed in OpenVINO execution: {e}") return compile_fx(subgraph, example_inputs) + def reset(): clear_caches() diff --git a/src/bindings/python/src/openvino/runtime/opset1/__init__.py b/src/bindings/python/src/openvino/runtime/opset1/__init__.py index 5bfa42f43f26b9..ca7e1aef385b2a 100644 --- a/src/bindings/python/src/openvino/runtime/opset1/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset1/__init__.py @@ -54,7 +54,6 @@ from openvino.runtime.opset1.ops import logical_xor from openvino.runtime.opset1.ops import lrn from openvino.runtime.opset1.ops import lstm_cell -from openvino.runtime.opset1.ops import lstm_sequence from openvino.runtime.opset1.ops import matmul from openvino.runtime.opset1.ops import max_pool from openvino.runtime.opset1.ops import maximum diff --git a/src/bindings/python/src/openvino/runtime/opset1/ops.py b/src/bindings/python/src/openvino/runtime/opset1/ops.py index c100a6c2db2cb4..54f32d404336d4 100644 --- a/src/bindings/python/src/openvino/runtime/opset1/ops.py +++ b/src/bindings/python/src/openvino/runtime/opset1/ops.py @@ -1532,95 +1532,6 @@ def lstm_cell( return _get_node_factory_opset1().create("LSTMCell", node_inputs, attributes) -@deprecated(version="2025.0", message="Use lstm_sequence from opset 5") -@nameable_op -def lstm_sequence( - X: NodeInput, - initial_hidden_state: NodeInput, - initial_cell_state: NodeInput, - sequence_lengths: NodeInput, - W: NodeInput, - R: NodeInput, - B: NodeInput, - hidden_size: int, - direction: str, - activations: Optional[List[str]] = None, - activations_alpha: Optional[List[float]] = None, - activations_beta: Optional[List[float]] = None, - clip: float = 0.0, - name: Optional[str] = None, -) -> Node: - """Return a node which performs LSTMSequence operation. - - :param X: The input tensor. Shape: [batch_size, seq_length, input_size]. - :param initial_hidden_state: The hidden state tensor. - Shape: [batch_size, num_directions, hidden_size]. - :param initial_cell_state: The cell state tensor. - Shape: [batch_size, num_directions, hidden_size]. - :param sequence_lengths: Specifies real sequence lengths for each batch element. - Shape: [batch_size]. Integer type. - :param W: Tensor with weights for matrix multiplication operation with input portion of data. - Shape: [num_directions, 4*hidden_size, input_size]. - :param R: The tensor with weights for matrix multiplication operation with hidden state. - Shape: [num_directions, 4*hidden_size, hidden_size]. - :param B: The tensor with biases. - Shape: [num_directions, 4*hidden_size]. - :param hidden_size: Specifies hidden state size. - :param direction: Specifies if the RNN is forward, reverse, or bidirectional. - :param activations: The list of three activation functions for gates. - :param activations_alpha: The list of alpha parameters for activation functions. - :param activations_beta: The list of beta parameters for activation functions. - :param clip: Specifies bound values [-C, C] for tensor clipping performed before activations. - :param name: An optional name of the output node. - - :return: The new node represents LSTMSequence. Node outputs count: 3. - """ - if activations is None: - activations = ["sigmoid", "tanh", "tanh"] - if activations_alpha is None: - activations_alpha = [] - if activations_beta is None: - activations_beta = [] - - node_inputs = as_nodes( - X, - initial_hidden_state, - initial_cell_state, - sequence_lengths, - W, - R, - B, - name=name, - ) - - # P - nGraph additional input, no such input in the OV spec - peepholes_count = 3 # nGraph default - if direction.lower() == "bidirectional": - num_directions = 2 - else: - num_directions = 1 - peepholes_shape = [num_directions, peepholes_count * hidden_size] - peepholes_array = np.zeros(peepholes_shape) # nGraph default - data_dtype = get_dtype(node_inputs[0].get_output_element_type(0)) - default_p = make_constant_node(peepholes_array, dtype=data_dtype) - node_inputs.append(default_p) - - weights_format = "fico" # OV LSTMWeightsFormat, no such attribute in the OV spec - input_forget = False # nGraph default, no such attribute in the OV spec - - attributes = { - "hidden_size": hidden_size, - "direction": direction.lower(), - "activations": activations, - "activations_alpha": activations_alpha, - "activations_beta": activations_beta, - "clip": clip, - "weights_format": weights_format, - "input_forget": input_forget, - } - return _get_node_factory_opset1().create("LSTMSequence", node_inputs, attributes) - - @nameable_op def matmul( data_a: NodeInput, diff --git a/src/bindings/python/src/openvino/runtime/opset13/ops.py b/src/bindings/python/src/openvino/runtime/opset13/ops.py index cb201d3d4263dd..a624ffb4f79873 100644 --- a/src/bindings/python/src/openvino/runtime/opset13/ops.py +++ b/src/bindings/python/src/openvino/runtime/opset13/ops.py @@ -15,7 +15,7 @@ from openvino.runtime.op import Constant, Result from openvino.runtime.opset1 import convert_like from openvino.runtime.opset_utils import _get_node_factory -from openvino.runtime.utils.decorators import binary_op, nameable_op, unary_op +from openvino.runtime.utils.decorators import binary_op, nameable_op, unary_op, overloading from openvino.runtime.utils.types import ( NumericData, NodeInput, @@ -271,7 +271,7 @@ def scaled_dot_product_attention( return _get_node_factory_opset13().create("ScaledDotProductAttention", inputs, attributes) -@singledispatch +@overloading(Union[NumericData, np.number, bool, np.bool_, list], Union[NumericType, Type], Optional[str], bool) # type: ignore @nameable_op def constant( value: Union[NumericData, np.number, bool, np.bool_, list], @@ -339,9 +339,9 @@ def display_shared_memory_warning(warning_message: str) -> None: return Constant(_value, shared_memory=_shared_memory) -@constant.register +@overloading(Tensor, bool, Optional[str]) # type: ignore @nameable_op -def _( +def constant( # noqa: F811 tensor: Tensor, shared_memory: bool = False, name: Optional[str] = None, diff --git a/src/bindings/python/src/openvino/runtime/opset15/__init__.py b/src/bindings/python/src/openvino/runtime/opset15/__init__.py index 6cc9c24827a85f..c4dd48d9087ae1 100644 --- a/src/bindings/python/src/openvino/runtime/opset15/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset15/__init__.py @@ -188,7 +188,7 @@ from openvino.runtime.opset1.ops import split from openvino.runtime.opset1.ops import sqrt from openvino.runtime.opset1.ops import squared_difference -from openvino.runtime.opset1.ops import squeeze +from openvino.runtime.opset15.ops import squeeze from openvino.runtime.opset15.ops import stft from openvino.runtime.opset1.ops import strided_slice from openvino.runtime.opset1.ops import subtract diff --git a/src/bindings/python/src/openvino/runtime/opset15/ops.py b/src/bindings/python/src/openvino/runtime/opset15/ops.py index 45b01a11bc3588..93aacb29572340 100644 --- a/src/bindings/python/src/openvino/runtime/opset15/ops.py +++ b/src/bindings/python/src/openvino/runtime/opset15/ops.py @@ -326,7 +326,7 @@ def stft( :return: The new node performing STFT operation. """ inputs = as_nodes(data, window, frame_size, frame_step, name=name) - return _get_node_factory_opset15().create("STFT", inputs) + return _get_node_factory_opset15().create("STFT", inputs, {"transpose_frames": transpose_frames}) @nameable_op @@ -348,3 +348,42 @@ def search_sorted( inputs = as_nodes(sorted_sequence, values, name=name) attributes = {"right_mode": right_mode} return _get_node_factory_opset15().create("SearchSorted", inputs, attributes) + + +@nameable_op +def squeeze( + data: NodeInput, + axes: Optional[NodeInput] = None, + allow_axis_skip: bool = False, + name: Optional[str] = None, +) -> Node: + """Perform squeeze operation on input tensor. + + :param data: The node with data tensor. + :param axes: Optional list of integers, indicating the dimensions to squeeze. + Negative indices are supported. One of: input node or array. + :param allow_axis_skip: If true, shape inference results in a dynamic rank, when + selected axis has value 1 in its dynamic range. Used only if axes input + is given. Defaults to false. + :param name: Optional new name for output node. + :return: The new node performing a squeeze operation on input tensor. + + Remove single-dimensional entries from the shape of a tensor. + Takes an optional parameter `axes` with a list of axes to squeeze. + If `axes` is not provided, all the single dimensions will be removed from the shape. + + For example: + + Inputs: tensor with shape [1, 2, 1, 3, 1, 1], axes=[2, 4] + + Result: tensor with shape [1, 2, 3, 1] + """ + if axes is None: + inputs = as_nodes(data, name=name) + else: + inputs = as_nodes(data, axes, name=name) + return _get_node_factory_opset15().create( + "Squeeze", + inputs, + {"allow_axis_skip": allow_axis_skip} + ) diff --git a/src/bindings/python/src/openvino/runtime/opset2/__init__.py b/src/bindings/python/src/openvino/runtime/opset2/__init__.py index 34d0d9b6737709..6624149e157e9e 100644 --- a/src/bindings/python/src/openvino/runtime/opset2/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset2/__init__.py @@ -56,7 +56,6 @@ from openvino.runtime.opset1.ops import logical_xor from openvino.runtime.opset1.ops import lrn from openvino.runtime.opset1.ops import lstm_cell -from openvino.runtime.opset1.ops import lstm_sequence from openvino.runtime.opset1.ops import matmul from openvino.runtime.opset1.ops import max_pool from openvino.runtime.opset1.ops import maximum diff --git a/src/bindings/python/src/openvino/runtime/opset3/__init__.py b/src/bindings/python/src/openvino/runtime/opset3/__init__.py index 964acb15b34bbc..5bd68912cae807 100644 --- a/src/bindings/python/src/openvino/runtime/opset3/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset3/__init__.py @@ -65,7 +65,6 @@ from openvino.runtime.opset1.ops import logical_xor from openvino.runtime.opset1.ops import lrn from openvino.runtime.opset1.ops import lstm_cell -from openvino.runtime.opset1.ops import lstm_sequence from openvino.runtime.opset1.ops import matmul from openvino.runtime.opset1.ops import max_pool from openvino.runtime.opset1.ops import maximum diff --git a/src/bindings/python/src/openvino/runtime/opset4/__init__.py b/src/bindings/python/src/openvino/runtime/opset4/__init__.py index bf57172bed40e2..e7cef8eb216d17 100644 --- a/src/bindings/python/src/openvino/runtime/opset4/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset4/__init__.py @@ -70,7 +70,6 @@ from openvino.runtime.opset1.ops import logical_xor from openvino.runtime.opset1.ops import lrn from openvino.runtime.opset4.ops import lstm_cell -from openvino.runtime.opset1.ops import lstm_sequence from openvino.runtime.opset1.ops import matmul from openvino.runtime.opset1.ops import max_pool from openvino.runtime.opset1.ops import maximum diff --git a/src/bindings/python/src/openvino/runtime/properties/hint/__init__.py b/src/bindings/python/src/openvino/runtime/properties/hint/__init__.py index dd90ded374ca11..d1dce289d09941 100644 --- a/src/bindings/python/src/openvino/runtime/properties/hint/__init__.py +++ b/src/bindings/python/src/openvino/runtime/properties/hint/__init__.py @@ -23,3 +23,4 @@ from openvino._pyopenvino.properties.hint import allow_auto_batching from openvino._pyopenvino.properties.hint import dynamic_quantization_group_size from openvino._pyopenvino.properties.hint import kv_cache_precision +from openvino._pyopenvino.properties.hint import activations_scale_factor diff --git a/src/bindings/python/src/pyopenvino/core/properties/properties.cpp b/src/bindings/python/src/pyopenvino/core/properties/properties.cpp index a6b30bd773001f..564e5f69f5ee14 100644 --- a/src/bindings/python/src/pyopenvino/core/properties/properties.cpp +++ b/src/bindings/python/src/pyopenvino/core/properties/properties.cpp @@ -101,6 +101,7 @@ void regmodule_properties(py::module m) { wrap_property_RW(m_hint, ov::hint::allow_auto_batching, "allow_auto_batching"); wrap_property_RW(m_hint, ov::hint::dynamic_quantization_group_size, "dynamic_quantization_group_size"); wrap_property_RW(m_hint, ov::hint::kv_cache_precision, "kv_cache_precision"); + wrap_property_RW(m_hint, ov::hint::activations_scale_factor, "activations_scale_factor"); // Submodule intel_cpu py::module m_intel_cpu = diff --git a/src/bindings/python/tests/test_graph/test_constant.py b/src/bindings/python/tests/test_graph/test_constant.py index e28a4ad05510f2..131654855b380a 100644 --- a/src/bindings/python/tests/test_graph/test_constant.py +++ b/src/bindings/python/tests/test_graph/test_constant.py @@ -87,7 +87,7 @@ def test_init_with_array(src_dtype, dst_dtype, shared_flag, data_getter): data = np.ascontiguousarray(data) # Create constant from based on numpy dtype or openvino type - ov_const = ops.constant(data, dtype=dst_dtype, shared_memory=shared_flag) + ov_const = ops.constant(data, dst_dtype, shared_memory=shared_flag) # Check shape and element type of Constant class assert isinstance(ov_const, Constant) @@ -842,7 +842,7 @@ def test_get_data_casting_bf16(src_dtype, dst_dtype, copy_flag): ) def test_get_data_casting_packed(src_dtype, ov_type, dst_dtype, copy_flag): data = np.array([[0, 0, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 1]], dtype=src_dtype) - ov_const = ops.constant(data, dtype=ov_type) + ov_const = ops.constant(value=data, dtype=ov_type) arr = ov_const.get_data(dtype=dst_dtype, copy=copy_flag) if dst_dtype is None: @@ -867,7 +867,7 @@ def test_const_from_tensor(shared_flag): shape = [1, 3, 32, 32] arr = np.ones(shape).astype(np.float32) ov_tensor = Tensor(arr, shape, Type.f32) - ov_const = ops.constant(ov_tensor, shared_memory=shared_flag) + ov_const = ops.constant(tensor=ov_tensor, shared_memory=shared_flag) assert isinstance(ov_const, Constant) assert np.all(list(ov_const.shape) == shape) diff --git a/src/bindings/python/tests/test_graph/test_create_op.py b/src/bindings/python/tests/test_graph/test_create_op.py index 87787e1e29bc32..f86ea4a18a8ca1 100644 --- a/src/bindings/python/tests/test_graph/test_create_op.py +++ b/src/bindings/python/tests/test_graph/test_create_op.py @@ -315,264 +315,6 @@ def test_lstm_cell_operator(dtype): assert list(node_param.get_output_shape(1)) == [1, 128] -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -@pytest.mark.parametrize("op_name", ["lstm", "lstmOpset1"]) -def test_lstm_cell_operator_opset1(dtype, op_name): - batch_size = 1 - input_size = 16 - hidden_size = 128 - - x_shape = [batch_size, input_size] - h_t_shape = [batch_size, hidden_size] - c_t_shape = [batch_size, hidden_size] - w_shape = [4 * hidden_size, input_size] - r_shape = [4 * hidden_size, hidden_size] - b_shape = [4 * hidden_size] - - parameter_x = ov.parameter(x_shape, name="X", dtype=dtype) - parameter_h_t = ov.parameter(h_t_shape, name="H_t", dtype=dtype) - parameter_c_t = ov.parameter(c_t_shape, name="C_t", dtype=dtype) - parameter_w = ov.parameter(w_shape, name="W", dtype=dtype) - parameter_r = ov.parameter(r_shape, name="R", dtype=dtype) - parameter_b = ov.parameter(b_shape, name="B", dtype=dtype) - - node_default = ov_opset1.lstm_cell( - parameter_x, parameter_h_t, parameter_c_t, parameter_w, parameter_r, parameter_b, hidden_size, name=op_name, - ) - - assert node_default.get_type_name() == "LSTMCell" - assert node_default.get_friendly_name() == op_name - assert node_default.get_output_size() == 2 - assert list(node_default.get_output_shape(0)) == [1, 128] - assert list(node_default.get_output_shape(1)) == [1, 128] - - activations = ["tanh", "Sigmoid", "RELU"] - activation_alpha = [1.0, 2.0, 3.0] - activation_beta = [3.0, 2.0, 1.0] - clip = 0.5 - - node_param = ov_opset1.lstm_cell( - parameter_x, - parameter_h_t, - parameter_c_t, - parameter_w, - parameter_r, - parameter_b, - hidden_size, - activations, - activation_alpha, - activation_beta, - clip, - ) - - assert node_param.get_type_name() == "LSTMCell" - assert node_param.get_output_size() == 2 - assert list(node_param.get_output_shape(0)) == [1, 128] - assert list(node_param.get_output_shape(1)) == [1, 128] - - -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -@pytest.mark.parametrize("op_name", ["lstm", "lstmOpset1"]) -def test_lstm_sequence_operator_bidirectional_opset1(dtype, op_name): - batch_size = 1 - input_size = 16 - hidden_size = 128 - num_directions = 2 - seq_length = 2 - - x_shape = [batch_size, seq_length, input_size] - h_t_shape = [batch_size, num_directions, hidden_size] - c_t_shape = [batch_size, num_directions, hidden_size] - seq_len_shape = [batch_size] - w_shape = [num_directions, 4 * hidden_size, input_size] - r_shape = [num_directions, 4 * hidden_size, hidden_size] - b_shape = [num_directions, 4 * hidden_size] - - parameter_x = ov.parameter(x_shape, name="X", dtype=dtype) - parameter_h_t = ov.parameter(h_t_shape, name="H_t", dtype=dtype) - parameter_c_t = ov.parameter(c_t_shape, name="C_t", dtype=dtype) - parameter_seq_len = ov.parameter(seq_len_shape, name="seq_len", dtype=np.int32) - parameter_w = ov.parameter(w_shape, name="W", dtype=dtype) - parameter_r = ov.parameter(r_shape, name="R", dtype=dtype) - parameter_b = ov.parameter(b_shape, name="B", dtype=dtype) - - direction = "BIDIRECTIONAL" - with pytest.warns(DeprecationWarning): - node = ov_opset1.lstm_sequence( - parameter_x, - parameter_h_t, - parameter_c_t, - parameter_seq_len, - parameter_w, - parameter_r, - parameter_b, - hidden_size, - direction, - name=op_name, - ) - - assert node.get_type_name() == "LSTMSequence" - assert node.get_friendly_name() == op_name - assert node.get_output_size() == 3 - - activations = ["RELU", "tanh", "Sigmoid"] - activation_alpha = [1.0, 2.0, 3.0] - activation_beta = [3.0, 2.0, 1.0] - clip = 1.22 - - with pytest.warns(DeprecationWarning): - node_param = ov_opset1.lstm_sequence( - parameter_x, - parameter_h_t, - parameter_c_t, - parameter_seq_len, - parameter_w, - parameter_r, - parameter_b, - hidden_size, - direction, - activations, - activation_alpha, - activation_beta, - clip, - ) - - assert node_param.get_type_name() == "LSTMSequence" - assert node_param.get_output_size() == 3 - - -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_lstm_sequence_operator_reverse_opset1(dtype): - batch_size = 2 - input_size = 4 - hidden_size = 3 - num_directions = 1 - seq_length = 2 - - x_shape = [batch_size, seq_length, input_size] - h_t_shape = [batch_size, num_directions, hidden_size] - c_t_shape = [batch_size, num_directions, hidden_size] - seq_len_shape = [batch_size] - w_shape = [num_directions, 4 * hidden_size, input_size] - r_shape = [num_directions, 4 * hidden_size, hidden_size] - b_shape = [num_directions, 4 * hidden_size] - - parameter_x = ov.parameter(x_shape, name="X", dtype=dtype) - parameter_h_t = ov.parameter(h_t_shape, name="H_t", dtype=dtype) - parameter_c_t = ov.parameter(c_t_shape, name="C_t", dtype=dtype) - parameter_seq_len = ov.parameter(seq_len_shape, name="seq_len", dtype=np.int32) - parameter_w = ov.parameter(w_shape, name="W", dtype=dtype) - parameter_r = ov.parameter(r_shape, name="R", dtype=dtype) - parameter_b = ov.parameter(b_shape, name="B", dtype=dtype) - - direction = "REVERSE" - with pytest.warns(DeprecationWarning): - node_default = ov_opset1.lstm_sequence( - parameter_x, - parameter_h_t, - parameter_c_t, - parameter_seq_len, - parameter_w, - parameter_r, - parameter_b, - hidden_size, - direction, - ) - - assert node_default.get_type_name() == "LSTMSequence" - assert node_default.get_output_size() == 3 - - activations = ["RELU", "tanh", "Sigmoid"] - activation_alpha = [1.0, 2.0, 3.0] - activation_beta = [3.0, 2.0, 1.0] - clip = 1.22 - with pytest.warns(DeprecationWarning): - node_param = ov_opset1.lstm_sequence( - parameter_x, - parameter_h_t, - parameter_c_t, - parameter_seq_len, - parameter_w, - parameter_r, - parameter_b, - hidden_size, - direction, - activations, - activation_alpha, - activation_beta, - clip, - ) - - assert node_param.get_type_name() == "LSTMSequence" - assert node_param.get_output_size() == 3 - - -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_lstm_sequence_operator_forward_opset1(dtype): - batch_size = 2 - input_size = 4 - hidden_size = 3 - num_directions = 1 - seq_length = 2 - - x_shape = [batch_size, seq_length, input_size] - h_t_shape = [batch_size, num_directions, hidden_size] - c_t_shape = [batch_size, num_directions, hidden_size] - seq_len_shape = [batch_size] - w_shape = [num_directions, 4 * hidden_size, input_size] - r_shape = [num_directions, 4 * hidden_size, hidden_size] - b_shape = [num_directions, 4 * hidden_size] - - parameter_x = ov.parameter(x_shape, name="X", dtype=dtype) - parameter_h_t = ov.parameter(h_t_shape, name="H_t", dtype=dtype) - parameter_c_t = ov.parameter(c_t_shape, name="C_t", dtype=dtype) - parameter_seq_len = ov.parameter(seq_len_shape, name="seq_len", dtype=np.int32) - parameter_w = ov.parameter(w_shape, name="W", dtype=dtype) - parameter_r = ov.parameter(r_shape, name="R", dtype=dtype) - parameter_b = ov.parameter(b_shape, name="B", dtype=dtype) - - direction = "forward" - with pytest.warns(DeprecationWarning): - node_default = ov_opset1.lstm_sequence( - parameter_x, - parameter_h_t, - parameter_c_t, - parameter_seq_len, - parameter_w, - parameter_r, - parameter_b, - hidden_size, - direction, - ) - - assert node_default.get_type_name() == "LSTMSequence" - assert node_default.get_output_size() == 3 - - activations = ["RELU", "tanh", "Sigmoid"] - activation_alpha = [2.0] - activation_beta = [1.0] - clip = 0.5 - with pytest.warns(DeprecationWarning): - node = ov_opset1.lstm_sequence( - parameter_x, - parameter_h_t, - parameter_c_t, - parameter_seq_len, - parameter_w, - parameter_r, - parameter_b, - hidden_size, - direction, - activations, - activation_alpha, - activation_beta, - clip, - ) - - assert node.get_type_name() == "LSTMSequence" - assert node.get_output_size() == 3 - - def test_gru_cell_operator(): batch_size = 1 input_size = 16 @@ -2492,8 +2234,8 @@ def test_stft(): window = ov.parameter([7], name="window", dtype=np.float32) frame_size = ov.constant(np.array(11, dtype=np.int32)) frame_step = ov.constant(np.array(3, dtype=np.int32)) - transpose_frames = True + transpose_frames = False op = ov_opset15.stft(data, window, frame_size, frame_step, transpose_frames) assert op.get_type_name() == "STFT" @@ -2501,6 +2243,14 @@ def test_stft(): assert op.get_output_element_type(0) == Type.f32 assert op.get_output_shape(0) == [4, 13, 6, 2] + transpose_frames = True + op = ov_opset15.stft(data, window, frame_size, frame_step, transpose_frames) + + assert op.get_type_name() == "STFT" + assert op.get_output_size() == 1 + assert op.get_output_element_type(0) == Type.f32 + assert op.get_output_shape(0) == [4, 6, 13, 2] + def test_search_sorted(): sorted_sequence = ov.parameter([7, 256, 200, 200], name="sorted", dtype=np.float32) diff --git a/src/bindings/python/tests/test_graph/test_ops_fused.py b/src/bindings/python/tests/test_graph/test_ops_fused.py index bdbf4a1a9f1f9c..2bab743bfd7afb 100644 --- a/src/bindings/python/tests/test_graph/test_ops_fused.py +++ b/src/bindings/python/tests/test_graph/test_ops_fused.py @@ -110,17 +110,6 @@ def test_clamp_operator(): assert list(model.get_output_shape(0)) == [2, 2] -def test_squeeze_operator(): - data_shape = [1, 2, 1, 3, 1, 1] - parameter_data = ov.parameter(data_shape, name="Data", dtype=np.float32) - axes = [2, 4] - model = ov.squeeze(parameter_data, axes) - - assert model.get_type_name() == "Squeeze" - assert model.get_output_size() == 1 - assert list(model.get_output_shape(0)) == [1, 2, 3, 1] - - def test_squared_difference_operator(): x1_shape = [1, 2, 3, 4] x2_shape = [2, 3, 4] diff --git a/src/bindings/python/tests/test_graph/test_squeeze.py b/src/bindings/python/tests/test_graph/test_squeeze.py new file mode 100644 index 00000000000000..869d84a0414841 --- /dev/null +++ b/src/bindings/python/tests/test_graph/test_squeeze.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import openvino.runtime.opset1 as ov_opset1 +import openvino.runtime.opset15 as ov_opset15 +import numpy as np +import pytest + + +def test_squeeze_v1_operator(): + data_shape = [1, 2, 1, 3, 1, 1] + parameter_data = ov_opset1.parameter(data_shape, name="Data", dtype=np.float32) + axes = [2, 4] + model = ov_opset1.squeeze(parameter_data, axes) + + assert model.get_type_name() == "Squeeze" + assert model.get_output_size() == 1 + assert list(model.get_output_shape(0)) == [1, 2, 3, 1] + + +@pytest.mark.parametrize(("input_shape", "axes", "allow_axis_skip", "expected_shape"), [ + ((1, 2, 1, 3, 1, 1), [1, 2, 4], True, [1, 2, 3, 1]), + ((1, 2, 1, 3, 1, 1), [1, 2, 4], False, [1, 2, 3, 1]), + ((2, -1, 3), [1], False, [2, 3]) +]) +def test_squeeze_v15_operator(input_shape, axes, allow_axis_skip, expected_shape): + parameter_data = ov_opset15.parameter(input_shape, name="Data", dtype=np.float32) + model = ov_opset15.squeeze(parameter_data, axes, allow_axis_skip, name="Squeeze") + + assert model.get_type_name() == "Squeeze" + assert model.get_output_size() == 1 + assert list(model.get_output_shape(0)) == expected_shape + + +def test_squeeze_v15_dynamic_rank_output(): + parameter_data = ov_opset15.parameter((2, -1, 3), name="Data", dtype=np.float32) + model = ov_opset15.squeeze(parameter_data, [1], True, name="Squeeze") + + assert model.get_type_name() == "Squeeze" + assert model.get_output_size() == 1 + assert model.get_output_partial_shape(0).to_string() == "[...]" + + +def test_squeeze_v15_axes_not_given(): + parameter_data = ov_opset15.parameter((1, 3, 1, 1, 3, 5), name="Data", dtype=np.float32) + model = ov_opset15.squeeze(data=parameter_data, name="Squeeze") + + assert model.get_type_name() == "Squeeze" + assert model.get_output_size() == 1 + assert list(model.get_output_shape(0)) == [3, 3, 5] diff --git a/src/bindings/python/tests/test_runtime/test_properties.py b/src/bindings/python/tests/test_runtime/test_properties.py index 32eb48f6765f41..6065d72196b44b 100644 --- a/src/bindings/python/tests/test_runtime/test_properties.py +++ b/src/bindings/python/tests/test_runtime/test_properties.py @@ -335,6 +335,11 @@ def test_properties_ro(ov_property_ro, expected_value): ((64, 64),), ), (hints.kv_cache_precision, "KV_CACHE_PRECISION", ((Type.f32, Type.f32),)), + ( + hints.activations_scale_factor, + "ACTIVATIONS_SCALE_FACTOR", + ((0.0, 0.0),), + ), ( intel_cpu.denormals_optimization, "CPU_DENORMALS_OPTIMIZATION", diff --git a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp index 69c148305fb94f..2eb79322b84e28 100644 --- a/src/common/transformations/include/ov_ops/dynamic_quantize.hpp +++ b/src/common/transformations/include/ov_ops/dynamic_quantize.hpp @@ -14,29 +14,75 @@ namespace internal { /// \brief Operator performing Dynamic Quantize class TRANSFORMATIONS_API DynamicQuantize : public ov::op::Op { public: - OPENVINO_OP("DynamicQuantize", "gpu_opset"); + OPENVINO_OP("DynamicQuantize", "ie_internal_opset"); + + /** + * @brief Configuration for the type of quantization applied to the data: + * - Symmetric: Quantization where the zero point is fixed at zero, and the range is symmetric around zero. + * - Asymmetric: Quantization where the zero point is not fixed at zero. + */ + enum class QuantizationType { Symmetric, Asymmetric }; + + /** + * @brief Configuration for how Activations, Scales and Zero Points will be stored in output buffers: + * - Planar: Activations, Scales, and Zero Points are stored in independent buffers. + * - InterleavedScalesZP: Activations are stored in an independent buffer, while Scales and Zero Points (if any) are + * combined in a separate buffer. + */ + enum class OutputStorageType { Planar, InterleavedScalesZP, /* InterleavedActivationsScalesZP */ }; + + /// \brief Structure that specifies attributes for interpolation + struct Attributes { + QuantizationType quantization_type = QuantizationType::Symmetric; + element::Type quantization_dt = element::undefined; + element::Type scale_dt = element::undefined; + element::Type zp_dt = element::undefined; + + std::vector group_sizes = {}; + std::vector scales_zp_output_order = {}; + OutputStorageType output_storage_type = OutputStorageType::Planar; + }; DynamicQuantize() = default; /// \brief Constructs an DynamicQuantize operation. /// /// \param data Input tensor with data - /// \param group_sizes Group sizes for dynamic quantization - /// \param dt_scale Data type for scale output - DynamicQuantize(const Output& data, std::vector group_sizes, element::Type dt_scale); + /// \param config Dynamic quantization configuration + DynamicQuantize(const Output& data, const Attributes& attrs); void validate_and_infer_types() override; std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override; + + const Attributes& get_attrs() const { + return m_attrs; + } + + void set_attrs(Attributes attrs) { + m_attrs = std::move(attrs); + } + const std::vector& get_group_sizes() const { - return m_group_sizes; - }; + return m_attrs.group_sizes; + } + + QuantizationType get_quantization_type() const { + return m_attrs.quantization_type; + } + + OutputStorageType get_output_storage_type() const { + return m_attrs.output_storage_type; + } + + const std::vector& get_scales_zp_output_order() const { + return m_attrs.scales_zp_output_order; + } + static std::vector shape_infer(const DynamicQuantize* op, - const std::vector& input_shapes, - const std::vector& group_sizes); + const std::vector& input_shapes); -private: - std::vector m_group_sizes; - element::Type m_dt_scale; +protected: + Attributes m_attrs; }; } // namespace internal diff --git a/src/common/transformations/include/transformations/op_conversions/convert_squeeze15_downgrade.hpp b/src/common/transformations/include/transformations/op_conversions/convert_squeeze15_downgrade.hpp new file mode 100644 index 00000000000000..c2ebfbc0f3138b --- /dev/null +++ b/src/common/transformations/include/transformations/op_conversions/convert_squeeze15_downgrade.hpp @@ -0,0 +1,23 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/matcher_pass.hpp" +#include "transformations_visibility.hpp" + +namespace ov { +namespace pass { +/** + * @ingroup ov_transformation_common_api + * @brief Converts Squeeze v15 to Squeeze v0. + */ +class TRANSFORMATIONS_API ConvertSqueeze15ToSqueeze0 : public MatcherPass { +public: + OPENVINO_RTTI("ConvertSqueeze15ToSqueeze0", "0"); + ConvertSqueeze15ToSqueeze0(); +}; + +} // namespace pass +} // namespace ov diff --git a/src/common/transformations/include/transformations/utils/utils.hpp b/src/common/transformations/include/transformations/utils/utils.hpp index fed812ecc864ca..6bcc50e29b3733 100644 --- a/src/common/transformations/include/transformations/utils/utils.hpp +++ b/src/common/transformations/include/transformations/utils/utils.hpp @@ -64,18 +64,10 @@ inline bool has_decompression_converts(const std::shared_ptr& f OPENVINO_DEPRECATED("Plugins should use ov::ISyncInferRequest::find_port") inline std::string create_ie_output_name(const Output& output) { - std::string out_name; - OPENVINO_SUPPRESS_DEPRECATED_START - auto tensor_name = ov::descriptor::get_ov_tensor_legacy_name(output.get_tensor()); - OPENVINO_SUPPRESS_DEPRECATED_END - if (!tensor_name.empty()) { - out_name = std::move(tensor_name); - } else { - const auto& prev_layer = output.get_node_shared_ptr(); - out_name = prev_layer->get_friendly_name(); - if (prev_layer->get_output_size() != 1) { - out_name += "." + std::to_string(output.get_index()); - } + const auto& prev_layer = output.get_node_shared_ptr(); + auto out_name = prev_layer->get_friendly_name(); + if (prev_layer->get_output_size() != 1) { + out_name += "." + std::to_string(output.get_index()); } return out_name; } diff --git a/src/common/transformations/src/ov_ops/dynamic_quantize.cpp b/src/common/transformations/src/ov_ops/dynamic_quantize.cpp index 74c0498e9a4425..9d1dfa5e5e3f62 100644 --- a/src/common/transformations/src/ov_ops/dynamic_quantize.cpp +++ b/src/common/transformations/src/ov_ops/dynamic_quantize.cpp @@ -7,62 +7,113 @@ #include "openvino/core/partial_shape.hpp" #include "openvino/core/validation_util.hpp" #include "openvino/op/variadic_split.hpp" -#include "variadic_split_shape_inference.hpp" +#include "openvino/util/common_util.hpp" namespace ov { namespace op { namespace internal { -DynamicQuantize::DynamicQuantize(const Output& data, std::vector group_sizes, element::Type dt_scale) - : Op({data}), - m_group_sizes(std::move(group_sizes)), - m_dt_scale(dt_scale) { - OPENVINO_ASSERT(data.get_partial_shape().rank() == m_group_sizes.size(), - "FC input rank should be same as the rank of group_size ", +DynamicQuantize::DynamicQuantize(const Output& data, const Attributes& attrs) : Op({data}), m_attrs(attrs) { + if (m_attrs.scales_zp_output_order.empty()) { + m_attrs.scales_zp_output_order.resize(data.get_partial_shape().size()); + std::iota(m_attrs.scales_zp_output_order.begin(), m_attrs.scales_zp_output_order.end(), 0); + } + + OPENVINO_ASSERT(data.get_partial_shape().rank() == m_attrs.group_sizes.size(), + "DQ input rank should be same as the rank of group_size ", data.get_tensor_ptr()->get_partial_shape().rank(), " / ", - m_group_sizes.size()); - set_output_size(2); + m_attrs.group_sizes.size()); + + OPENVINO_ASSERT(data.get_partial_shape().size() == m_attrs.scales_zp_output_order.size(), + "DQ input rank should be same as the rank of scales and zero points output order)"); + + size_t outputs_number = 2; + if (m_attrs.quantization_type == QuantizationType::Asymmetric && + m_attrs.output_storage_type == OutputStorageType::Planar) + outputs_number = 3; + + OPENVINO_ASSERT( + (m_attrs.output_storage_type == OutputStorageType::Planar) || + (m_attrs.quantization_type == QuantizationType::Asymmetric && m_attrs.scale_dt == m_attrs.zp_dt), + "Scales and Zero Points should have the same data type to be stored in the single buffer"); + + set_output_size(outputs_number); validate_and_infer_types(); } void DynamicQuantize::validate_and_infer_types() { std::vector input_shapes = {get_input_partial_shape(0)}; - auto out_shapes = shape_infer(this, input_shapes, m_group_sizes); - set_output_type(0, element::i8, out_shapes[0]); - set_output_type(1, m_dt_scale, out_shapes[1]); + auto out_shapes = shape_infer(this, input_shapes); + set_output_type(0, m_attrs.quantization_dt, out_shapes[0]); + set_output_type(1, m_attrs.scale_dt, out_shapes[1]); + + if (m_attrs.quantization_type == QuantizationType::Asymmetric && + m_attrs.output_storage_type == OutputStorageType::Planar) + set_output_type(2, m_attrs.zp_dt, out_shapes[2]); } std::shared_ptr DynamicQuantize::clone_with_new_inputs(const ov::OutputVector& new_args) const { check_new_args_count(this, new_args); - return std::make_shared(new_args.at(0), m_group_sizes, m_dt_scale); + return std::make_shared(new_args.at(0), m_attrs); } std::vector DynamicQuantize::shape_infer(const DynamicQuantize* op, - const std::vector& input_shapes, - const std::vector& group_sizes) { + const std::vector& input_shapes) { std::vector out_shapes; out_shapes.push_back(input_shapes[0]); auto scale_shape = input_shapes[0]; + const auto& group_sizes = op->m_attrs.group_sizes; OPENVINO_ASSERT(scale_shape.size() == group_sizes.size(), "Scale_shape and group_size are supposed to have same rank: ", scale_shape.size(), " / ", group_sizes.size()); for (size_t i = 0; i < scale_shape.size(); i++) { - if (scale_shape[i].is_dynamic()) + if (scale_shape[i].is_dynamic() || scale_shape[i] == 0) continue; - if (group_sizes[i] == UINT64_MAX) + if (group_sizes[i] == UINT64_MAX) { scale_shape[i] = 1; - else { - scale_shape[i] /= group_sizes[i]; // if group_size is larger than shape, scale_shape will be 1 - scale_shape[i] = std::max(static_cast(scale_shape[i].get_length()), 1); + } else { + scale_shape[i] = ov::util::ceil_div(scale_shape[i].get_length(), static_cast(group_sizes[i])); } } out_shapes.push_back(scale_shape); + + // Add zero points shape, same as the scales + if (op->m_attrs.quantization_type == QuantizationType::Asymmetric && + op->m_attrs.output_storage_type == OutputStorageType::Planar) + out_shapes.push_back(scale_shape); + + auto transpose_shape = [](const ov::PartialShape& shape, const std::vector& scales_zp_output_order) { + auto transposed_shape = shape; + for (size_t i = 0; i < scales_zp_output_order.size(); i++) { + OPENVINO_ASSERT(scales_zp_output_order[i] < transposed_shape.size()); + transposed_shape[i] = shape[scales_zp_output_order[i]]; + } + + return transposed_shape; + }; + + // Transpose scales and zero points shapes + const auto& scales_zp_output_order = op->m_attrs.scales_zp_output_order; + for (size_t i = 1; i < out_shapes.size(); i++) { + out_shapes[i] = transpose_shape(out_shapes[i], scales_zp_output_order); + } + + if (op->m_attrs.quantization_type == QuantizationType::Asymmetric && + op->m_attrs.output_storage_type != OutputStorageType::Planar) { + // Currently scales and zero points are supposed to be combined over the last dimension only + const auto combine_axis = scales_zp_output_order.empty() ? out_shapes[1].size() - 1 + : scales_zp_output_order[out_shapes[1].size() - 1]; + OPENVINO_ASSERT(group_sizes[combine_axis] != 1); + + out_shapes[1][combine_axis] *= 2; // [scale, zero_point] pairs + } + return out_shapes; } diff --git a/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp b/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp index 9d46b583a828f2..37ee2d12d9aebb 100644 --- a/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp @@ -98,6 +98,7 @@ #include "transformations/op_conversions/convert_softmax_downgrade.hpp" #include "transformations/op_conversions/convert_softmax_upgrade.hpp" #include "transformations/op_conversions/convert_space_to_depth.hpp" +#include "transformations/op_conversions/convert_squeeze15_downgrade.hpp" #include "transformations/op_conversions/convert_subtract.hpp" #include "transformations/op_conversions/convert_topk11_downgrade.hpp" #include "transformations/op_conversions/convert_xor_to_logical_xor.hpp" @@ -235,6 +236,7 @@ bool ov::pass::CommonOptimizations::run_on_model(const std::shared_ptr(); ADD_MATCHER(fq_fusions, FakeQuantizeMulFusion) diff --git a/src/common/transformations/src/transformations/common_optimizations/gelu_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/gelu_fusion.cpp index 221484c75cccde..8d075f4a727758 100644 --- a/src/common/transformations/src/transformations/common_optimizations/gelu_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/gelu_fusion.cpp @@ -22,6 +22,7 @@ #include "openvino/op/parameter.hpp" #include "openvino/op/power.hpp" #include "openvino/op/tanh.hpp" +#include "openvino/pass/pattern/op/or.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" #include "transformations/utils/utils.hpp" @@ -280,9 +281,16 @@ ov::pass::GeluFusionWithTanh::GeluFusionWithTanh() { auto add_1 = ov::pass::pattern::wrap_type({tanh, add_1_constant}); auto mul_2_constant = ov::pass::pattern::wrap_type(); - auto mul_2 = ov::pass::pattern::wrap_type({add_1, mul_2_constant}); - auto mul_3 = ov::pass::pattern::wrap_type({input, mul_2}); + // x * (0.5 * (1 + tanh)) + auto mul_2_1 = ov::pass::pattern::wrap_type({add_1, mul_2_constant}); + auto mul_3_1 = ov::pass::pattern::wrap_type({input, mul_2_1}); + + // (x * 0.5) * (1 + tanh) + auto mul_2_2 = ov::pass::pattern::wrap_type({input, mul_2_constant}); + auto mul_3_2 = ov::pass::pattern::wrap_type({add_1, mul_2_2}); + + auto mul_3 = std::make_shared(OutputVector{mul_3_1, mul_3_2}); ov::matcher_pass_callback callback = [=](ov::pass::pattern::Matcher& m) { auto& pattern_to_output = m.get_pattern_value_map(); @@ -298,7 +306,6 @@ ov::pass::GeluFusionWithTanh::GeluFusionWithTanh() { ov::as_type_ptr(pattern_to_output.at(mul_2_constant).get_node_shared_ptr()); auto add_1_constant_value = ov::as_type_ptr(pattern_to_output.at(add_1_constant).get_node_shared_ptr()); - if (!pow_constant_value || !add_1_constant_value || !mul_0_constant_value || !mul_1_constant_value || !mul_2_constant_value) { return false; @@ -318,18 +325,17 @@ ov::pass::GeluFusionWithTanh::GeluFusionWithTanh() { auto gelu = std::make_shared(x_output, op::GeluApproximationMode::TANH); gelu->set_friendly_name(m.get_match_root()->get_friendly_name()); - ov::copy_runtime_info( - { - pattern_to_output.at(pow).get_node_shared_ptr(), - pattern_to_output.at(mul_0).get_node_shared_ptr(), - pattern_to_output.at(mul_1).get_node_shared_ptr(), - pattern_to_output.at(mul_2).get_node_shared_ptr(), - pattern_to_output.at(mul_3).get_node_shared_ptr(), - pattern_to_output.at(tanh).get_node_shared_ptr(), - pattern_to_output.at(add_0).get_node_shared_ptr(), - pattern_to_output.at(add_1).get_node_shared_ptr(), - }, - gelu); + + std::vector> pattern_nodes = + {pow, mul_0, mul_1, tanh, add_0, add_1, mul_2_1, mul_2_2, mul_3_1, mul_3_2}; + std::vector> cp_rt_info_nodes; + for (const auto& pattern_node : pattern_nodes) { + if (pattern_to_output.count(pattern_node)) { + cp_rt_info_nodes.push_back(pattern_to_output.at(pattern_node).get_node_shared_ptr()); + } + } + ov::copy_runtime_info(cp_rt_info_nodes, gelu); + ov::replace_node(m.get_match_root(), gelu); return true; }; diff --git a/src/common/transformations/src/transformations/common_optimizations/lora_subgraph_fusion.cpp b/src/common/transformations/src/transformations/common_optimizations/lora_subgraph_fusion.cpp index 366ce00894242e..95b754062c13b8 100644 --- a/src/common/transformations/src/transformations/common_optimizations/lora_subgraph_fusion.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/lora_subgraph_fusion.cpp @@ -9,6 +9,7 @@ #include "itt.hpp" #include "openvino/op/add.hpp" +#include "openvino/op/convert.hpp" #include "openvino/op/convolution.hpp" #include "openvino/op/matmul.hpp" #include "openvino/op/multiply.hpp" @@ -26,12 +27,19 @@ ov::pass::LoraSubgraphFusion::LoraSubgraphFusion() { auto lora_input_m = any_input(); auto transpose_const1_m = wrap_type(consumers_count(1)); auto transpose1_m = optional({lora_input_m, transpose_const1_m}, consumers_count(1)); + auto read_value1_m = wrap_type(); - auto matmul1_m = wrap_type({transpose1_m, read_value1_m}, consumers_count(1)); + auto convert1_m = optional(read_value1_m, consumers_count(1)); + auto matmul1_m = wrap_type({transpose1_m, convert1_m}, consumers_count(1)); + auto read_value2_m = wrap_type(); - auto multiply_m = wrap_type({matmul1_m, read_value2_m}, consumers_count(1)); + auto convert2_m = optional(read_value2_m, consumers_count(1)); + auto multiply_m = wrap_type({matmul1_m, convert2_m}, consumers_count(1)); + auto read_value3_m = wrap_type(); - auto matmul2_m = wrap_type({multiply_m, read_value3_m}, consumers_count(1)); + auto convert3_m = optional(read_value3_m, consumers_count(1)); + auto matmul2_m = wrap_type({multiply_m, convert3_m}, consumers_count(1)); + auto transpose_const2_m = wrap_type(consumers_count(1)); auto transpose2_m = optional({matmul2_m, transpose_const2_m}, consumers_count(1)); auto main_flow_m = wrap_type({lora_input_m, any_input()}); @@ -41,11 +49,14 @@ ov::pass::LoraSubgraphFusion::LoraSubgraphFusion() { const auto& pattern_map = m.get_pattern_value_map(); const auto& lora_input = pattern_map.at(lora_input_m); const auto& matmul1 = pattern_map.at(matmul1_m); - const auto& read_value1 = pattern_map.at(read_value1_m); + const auto& state_1 = + pattern_map.count(convert1_m) ? pattern_map.at(convert1_m) : pattern_map.at(read_value1_m); const auto& multiply = pattern_map.at(multiply_m); - const auto& read_value2 = pattern_map.at(read_value2_m); + const auto& state_2 = + pattern_map.count(convert2_m) ? pattern_map.at(convert2_m) : pattern_map.at(read_value2_m); const auto& matmul2 = pattern_map.at(matmul2_m); - const auto& read_value3 = pattern_map.at(read_value3_m); + const auto& state_3 = + pattern_map.count(convert3_m) ? pattern_map.at(convert3_m) : pattern_map.at(read_value3_m); const auto& main_flow = pattern_map.at(main_flow_m); const auto& add = pattern_map.at(add_m); @@ -70,15 +81,15 @@ ov::pass::LoraSubgraphFusion::LoraSubgraphFusion() { pattern_map.count(transpose1_m) ? pattern_map.at(transpose1_m).get_node()->input(0) : matmul1.get_node()->input(0), matmul1.get_node()->input(1), - find_connected_input(multiply.get_node(), read_value2.get_node()), + find_connected_input(multiply.get_node(), state_2.get_node()), matmul2.get_node()->input(1), }; const ov::OutputVector external_connections{ main_flow, lora_input, - read_value1, - read_value2, - read_value3, + state_1, + state_2, + state_3, }; ov::ParameterVector subgraph_parameters; diff --git a/src/common/transformations/src/transformations/common_optimizations/reverse_shape_and_type_infer.cpp b/src/common/transformations/src/transformations/common_optimizations/reverse_shape_and_type_infer.cpp index 211f351da34024..9a06201f688675 100644 --- a/src/common/transformations/src/transformations/common_optimizations/reverse_shape_and_type_infer.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/reverse_shape_and_type_infer.cpp @@ -282,6 +282,15 @@ bool ov::pass::ReverseShapeAndTypeInfer::run_on_model(const std::shared_ptrget_input_tensor(0).m_element_type = element::boolean; is_changed = true; } + + // in case TensorFlow models, we can deduce predicate shape that must be a scalar + // If operations created by fusing Switch-Merge sub-graph contain tf_switch_merge_if rt-info + if (if_op->get_rt_info().count("tf_switch_merge_if") && + if_op->get_rt_info()["tf_switch_merge_if"].as() && + if_op->input_value(0).get_partial_shape().rank().is_dynamic()) { + if_op->get_input_tensor(0).m_partial_shape = ov::PartialShape({}); + is_changed = true; + } } else if (ov::as_type_ptr(op)) { is_changed |= inherit_output_shape(op, {0}); is_changed |= inherit_output_type(op, {1}); diff --git a/src/common/transformations/src/transformations/control_flow/unroll_if.cpp b/src/common/transformations/src/transformations/control_flow/unroll_if.cpp index 6ef5536495e663..14d28a3a3fcfa3 100644 --- a/src/common/transformations/src/transformations/control_flow/unroll_if.cpp +++ b/src/common/transformations/src/transformations/control_flow/unroll_if.cpp @@ -54,14 +54,7 @@ bool ov::pass::UnrollIf::run_on_model(const std::shared_ptr& f) { } for (const auto& output_desc : output_descriptions) { std::shared_ptr result = body->get_results()[output_desc->m_body_value_index]; - const auto& in_value = result->input_value(0); - // set output name to Tensor to store it for openvino to cnn conversion - OPENVINO_SUPPRESS_DEPRECATED_START - ov::descriptor::set_ov_tensor_legacy_name( - in_value.get_tensor(), - op::util::create_ie_output_name(if_node->output(output_desc->m_output_index))); - OPENVINO_SUPPRESS_DEPRECATED_END for (const auto& input : if_node->output(output_desc->m_output_index).get_target_inputs()) { input.replace_source_output(result->get_input_source_output(0)); } diff --git a/src/common/transformations/src/transformations/control_flow/unroll_tensor_iterator.cpp b/src/common/transformations/src/transformations/control_flow/unroll_tensor_iterator.cpp index 431527ade998a1..dde1ab35898b67 100644 --- a/src/common/transformations/src/transformations/control_flow/unroll_tensor_iterator.cpp +++ b/src/common/transformations/src/transformations/control_flow/unroll_tensor_iterator.cpp @@ -115,21 +115,6 @@ bool ov::pass::UnrollTensorIterator::run_on_model(const std::shared_ptrget_output_descriptions()) { - // we need to insert tensor_name to the outputs of TensorIterator if they directly connected to - // Results ops. It's necessary to save original TensorIterator name when we use CNNNetwork. - auto insert_tensor_name = [&](const ov::Output& ti_output, const ov::Output& insert_to) { - auto target_inputs = ti_output.get_target_inputs(); - if (target_inputs.empty() || - std::any_of(target_inputs.begin(), target_inputs.end(), [](const ov::Input& target_inp) { - return ov::as_type(target_inp.get_node()) != nullptr; - })) { - OPENVINO_SUPPRESS_DEPRECATED_START - ov::descriptor::set_ov_tensor_legacy_name(insert_to.get_tensor(), - ov::op::util::create_ie_output_name(ti_output)); - OPENVINO_SUPPRESS_DEPRECATED_END - } - }; - if (const auto& concat_desc = std::dynamic_pointer_cast(desc)) { if (!concat_desc) { @@ -155,9 +140,6 @@ bool ov::pass::UnrollTensorIterator::run_on_model(const std::shared_ptr(to_concat, concat_desc->m_axis); copy_runtime_info(sub_graph_op, concat); - // set output name to Tensor to store it for openvino to cnn conversion - insert_tensor_name(sub_graph_op->output(concat_desc->m_output_index), concat->output(0)); - // connect the Concat layer to the corresponding TI outputs for (auto& input : sub_graph_op->output(concat_desc->m_output_index).get_target_inputs()) { input.replace_source_output(concat); @@ -167,8 +149,6 @@ bool ov::pass::UnrollTensorIterator::run_on_model(const std::shared_ptr result = body_functions[0]->get_results().at(concat_desc->m_body_value_index); const auto& input_to_res = result->get_input_source_output(0); - // set output name to Tensor to store it for openvino to cnn conversion - insert_tensor_name(sub_graph_op->output(concat_desc->m_output_index), input_to_res); for (auto& input : sub_graph_op->output(concat_desc->m_output_index).get_target_inputs()) { input.replace_source_output(input_to_res); @@ -179,11 +159,8 @@ bool ov::pass::UnrollTensorIterator::run_on_model(const std::shared_ptrm_iteration; iter = iter >= 0 ? iter : num_iter - 1; - std::shared_ptr result = - body_functions[iter]->get_results()[output_desc->m_body_value_index]; - const auto& in_value = result->input_value(0); + auto result = body_functions[iter]->get_results()[output_desc->m_body_value_index]; - insert_tensor_name(sub_graph_op->output(output_desc->m_output_index), in_value); for (const auto& input : sub_graph_op->output(output_desc->m_output_index).get_target_inputs()) { input.replace_source_output(result->get_input_source_output(0)); } diff --git a/src/common/transformations/src/transformations/convert_precision.cpp b/src/common/transformations/src/transformations/convert_precision.cpp index 6f5166dfd26760..d8fd21699a5c20 100644 --- a/src/common/transformations/src/transformations/convert_precision.cpp +++ b/src/common/transformations/src/transformations/convert_precision.cpp @@ -62,6 +62,8 @@ bool fuse_type_to_ctc_greedy_decoder_seq_len(const std::shared_ptr& no bool fuse_type_to_random_uniform_v8(const std::shared_ptr& node, const precisions_map& precisions); +bool fuse_type_to_search_sorted_v15(const std::shared_ptr& node, const precisions_map& precisions); + bool extend_select_type(const std::shared_ptr& node, const precisions_map& precisions); bool extend_reverse_type(const std::shared_ptr& node, const precisions_map& precisions); @@ -254,7 +256,6 @@ bool convert_function_precision(const std::shared_ptr& f, // Register internal constants only after fixing input type that could lead to nodes // replacement register_constants(ops); - for (auto& node : ops) { // skip precision sensitive nodes if (skip_precision_sensitive && fp16_compression_is_disabled(node) && has_fp16_compression) @@ -331,12 +332,6 @@ bool convert_function_precision(const std::shared_ptr& f, auto& convert_output_tensor = convert->get_output_tensor(0); convert_output_tensor.set_names(result_input.get_names()); - OPENVINO_SUPPRESS_DEPRECATED_START - const auto& legacy_name = ov::descriptor::get_ov_tensor_legacy_name(result_input.get_tensor()); - if (!legacy_name.empty()) { - ov::descriptor::set_ov_tensor_legacy_name(convert_output_tensor, legacy_name); - } - OPENVINO_SUPPRESS_DEPRECATED_END result_input.set_names({}); result->input(0).replace_source_output(convert->output(0)); @@ -469,7 +464,8 @@ bool ov::pass::ConvertPrecision::run_on_model(const std::shared_ptr& {ov::op::v13::Multinomial::get_type_info_static(), fuse_type_to_multinomial_v13}, {ov::op::v0::PriorBox::get_type_info_static(), fuse_type_to_prior_box}, {ov::op::v8::PriorBox::get_type_info_static(), fuse_type_to_prior_box}, - {ov::op::v0::PriorBoxClustered::get_type_info_static(), fuse_type_to_prior_box}}; + {ov::op::v0::PriorBoxClustered::get_type_info_static(), fuse_type_to_prior_box}, + {ov::op::v15::SearchSorted::get_type_info_static(), fuse_type_to_search_sorted_v15}}; for (const auto& it : m_additional_type_to_fuse_map) { type_to_fuse[it.first] = it.second; @@ -554,6 +550,18 @@ bool fuse_type_to_unique_v10(const std::shared_ptr& node, const precisions return res; } +bool fuse_type_to_search_sorted_v15(const std::shared_ptr& node, const precisions_map& precisions) { + bool res = false; + if (auto op = ov::as_type_ptr(node)) { + auto it = precisions.find(node->get_output_element_type(0)); + if (it != precisions.end()) { + op->set_output_type_attr(it->second); + res = true; + } + } + return res; +} + bool fuse_type_to_range_v4(const std::shared_ptr& node, const precisions_map& precisions) { auto it = precisions.find(node->get_output_element_type(0)); if (it == precisions.end()) @@ -1000,12 +1008,14 @@ bool extend_select_type(const std::shared_ptr& node, const precisions_ type_relaxed->set_origin_input_type(ov::element::boolean, 0); return true; } else if (auto casted = ov::as_type_ptr(node)) { - auto relaxed_op = - std::make_shared>(*casted, - ov::element::TypeVector{ov::element::boolean}, - ov::element::TypeVector{}); - replace_node(node, relaxed_op); - return true; + if (precisions.count(ov::element::boolean) != 0) { + auto relaxed_op = + std::make_shared>(*casted, + ov::element::TypeVector{ov::element::boolean}, + ov::element::TypeVector{}); + replace_node(node, relaxed_op); + return true; + } } return false; } diff --git a/src/common/transformations/src/transformations/op_conversions/convert_maxpool_downgrade.cpp b/src/common/transformations/src/transformations/op_conversions/convert_maxpool_downgrade.cpp index 50a49202742bde..0e0865b3845f48 100644 --- a/src/common/transformations/src/transformations/op_conversions/convert_maxpool_downgrade.cpp +++ b/src/common/transformations/src/transformations/op_conversions/convert_maxpool_downgrade.cpp @@ -57,10 +57,6 @@ ov::pass::ConvertMaxPool8ToMaxPool1::ConvertMaxPool8ToMaxPool1() { ov::copy_runtime_info(maxpool_v8_node, maxpool_v1_node); maxpool_v8_node->clear_control_dependencies(); - OPENVINO_SUPPRESS_DEPRECATED_START - ov::descriptor::set_ov_tensor_legacy_name(maxpool_v1_node->output(0).get_tensor(), out_name); - OPENVINO_SUPPRESS_DEPRECATED_END - return true; }; diff --git a/src/common/transformations/src/transformations/op_conversions/convert_squeeze15_downgrade.cpp b/src/common/transformations/src/transformations/op_conversions/convert_squeeze15_downgrade.cpp new file mode 100644 index 00000000000000..50701d3d6acd56 --- /dev/null +++ b/src/common/transformations/src/transformations/op_conversions/convert_squeeze15_downgrade.cpp @@ -0,0 +1,40 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/op_conversions/convert_squeeze15_downgrade.hpp" + +#include "itt.hpp" +#include "openvino/core/rt_info.hpp" +#include "openvino/op/squeeze.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "transformations/utils/utils.hpp" + +ov::pass::ConvertSqueeze15ToSqueeze0::ConvertSqueeze15ToSqueeze0() { + MATCHER_SCOPE(ConvertSqueeze15ToSqueeze0); + + const auto& squeeze_v15_pattern = pattern::wrap_type(); + + const matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](pattern::Matcher& m) { + const auto& squeeze_v15 = ov::as_type_ptr(m.get_match_root()); + if (!squeeze_v15 || transformation_callback(squeeze_v15)) { + return false; + } + std::shared_ptr squeeze_v0; + if (squeeze_v15->get_input_size() == 1) { + squeeze_v0 = std::make_shared(squeeze_v15->input_value(0)); + } else if (squeeze_v15->get_input_size() == 2 && !squeeze_v15->get_allow_axis_skip()) { + squeeze_v0 = std::make_shared(squeeze_v15->input_value(0), squeeze_v15->input_value(1)); + } else { + return false; + } + squeeze_v0->set_friendly_name(squeeze_v15->get_friendly_name()); + copy_runtime_info(squeeze_v15, squeeze_v0); + replace_node(squeeze_v15, squeeze_v0); + + return true; + }; + + auto m = std::make_shared(squeeze_v15_pattern, matcher_name); + register_matcher(m, callback); +} diff --git a/src/common/transformations/src/transformations/transpose_sinking/ts_utils.cpp b/src/common/transformations/src/transformations/transpose_sinking/ts_utils.cpp index ba950ad93bf501..ae40ca869c7dcc 100644 --- a/src/common/transformations/src/transformations/transpose_sinking/ts_utils.cpp +++ b/src/common/transformations/src/transformations/transpose_sinking/ts_utils.cpp @@ -122,12 +122,6 @@ void SwapOutputNames(Output output1, Output output2) { const auto node2_output_names = output2.get_names(); output2.set_names(output1.get_names()); output1.set_names(node2_output_names); - - OPENVINO_SUPPRESS_DEPRECATED_START - const auto node2_legacy_output_names = get_ov_tensor_legacy_name(output2.get_tensor()); - set_ov_tensor_legacy_name(output2.get_tensor(), get_ov_tensor_legacy_name(output1.get_tensor())); - set_ov_tensor_legacy_name(output1.get_tensor(), node2_legacy_output_names); - OPENVINO_SUPPRESS_DEPRECATED_END } void SwapFriendlyNames(const NodePtr& node1, const NodePtr& node2) { diff --git a/src/common/transformations/tests/common_optimizations/gelu_fusion.cpp b/src/common/transformations/tests/common_optimizations/gelu_fusion.cpp index 837d2ba6d4597e..dbc54f5492bffa 100644 --- a/src/common/transformations/tests/common_optimizations/gelu_fusion.cpp +++ b/src/common/transformations/tests/common_optimizations/gelu_fusion.cpp @@ -388,6 +388,44 @@ TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_epsilon_pow_value) { } } +TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_epsilon_pow_value_2) { + { + auto input = std::make_shared(element::f32, Shape{2, 2}); + auto pow_constant = + std::make_shared(element::f32, Shape{1}, std::vector{3.0f + 1.0e-8f}); + auto pow = std::make_shared(input, pow_constant); + auto mul_0_constant = + std::make_shared(element::f32, Shape{1}, std::vector{0.044715f}); + auto mul_0 = std::make_shared(pow, mul_0_constant); + auto add_0 = std::make_shared(input, mul_0); + + auto mul_1_constant = + std::make_shared(element::f32, + Shape{1}, + std::vector{static_cast(std::sqrt(2.0 / M_PI))}); + auto mul_1 = std::make_shared(add_0, mul_1_constant); + + auto tanh = std::make_shared(mul_1); + + auto add_1_constant = std::make_shared(element::f32, Shape{1}, std::vector{1.0f}); + auto add_1 = std::make_shared(tanh, add_1_constant); + + auto mul_2_constant = std::make_shared(element::f32, Shape{1}, std::vector{0.5f}); + auto mul_2 = std::make_shared(input, mul_2_constant); + + auto mul_3 = std::make_shared(add_1, mul_2); + + model = std::make_shared(NodeVector{mul_3}, ParameterVector{input}); + manager.register_pass(); + } + + { + auto data = std::make_shared(element::f32, Shape{2, 2}); + auto gelu = std::make_shared(data, op::GeluApproximationMode::TANH); + model_ref = std::make_shared(NodeVector{gelu}, ParameterVector{data}); + } +} + TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_wrong_pow_value) { { auto input = std::make_shared(element::f32, Shape{2, 2}); diff --git a/src/common/transformations/tests/common_optimizations/lora_subgraph_fusion.cpp b/src/common/transformations/tests/common_optimizations/lora_subgraph_fusion.cpp index 6557f763c6b368..2294668f49a6c5 100644 --- a/src/common/transformations/tests/common_optimizations/lora_subgraph_fusion.cpp +++ b/src/common/transformations/tests/common_optimizations/lora_subgraph_fusion.cpp @@ -20,21 +20,25 @@ using namespace ov; static constexpr auto netType = ov::element::f32; -std::pair create_states(const std::vector& shapes) { - ov::OutputVector read_values; +std::pair create_states(const std::vector& shapes, + const ov::element::Type& states_precision = netType) { + ov::OutputVector state_outs; ov::SinkVector assigns; size_t idx = 0; auto create_state = [&](const ov::PartialShape& shape) { - auto variable = - std::make_shared(ov::op::util::VariableInfo{shape, netType, std::to_string(idx++)}); + auto variable = std::make_shared( + ov::op::util::VariableInfo{shape, states_precision, std::to_string(idx++)}); auto read_value = std::make_shared(variable); auto assign = std::make_shared(read_value, variable); - read_values.push_back(read_value); assigns.push_back(assign); + if (states_precision == netType) + state_outs.push_back(read_value); + else + state_outs.push_back(std::make_shared(read_value, netType)); }; for (const auto& shape : shapes) create_state(shape); - return std::make_pair(read_values, assigns); + return std::make_pair(state_outs, assigns); } std::shared_ptr create_lora_subgraph(const ov::Output& main_flow, @@ -134,6 +138,47 @@ TEST_F(LoraSubgraphFusionMatMulTests, StandardPattern) { } } +TEST_F(LoraSubgraphFusionMatMulTests, StandardPatternWithConvert) { + { + auto param_lora = std::make_shared(netType, shape_x); + auto param_w = std::make_shared(netType, shape_w); + auto main_mm = std::make_shared(param_lora, param_w, false, true); + main_mm->set_friendly_name("main_mm"); + auto states = create_states({shape_state_1, shape_state_2, shape_state_3}, ov::element::f16); + auto lora_subgraph = create_lora_subgraph(main_mm, param_lora, states.first, false); + lora_subgraph->set_friendly_name("lora_subgraph"); + model = std::make_shared(OutputVector{lora_subgraph, main_mm}, + states.second, + ParameterVector{param_lora, param_w}); + } + { + auto param_lora = std::make_shared(netType, shape_x); + auto param_w = std::make_shared(netType, shape_w); + auto main_mm = std::make_shared(param_lora, param_w, false, true); + main_mm->set_friendly_name("main_mm"); + + auto inner_param_lora = std::make_shared(netType, shape_x); + auto inner_state_1 = std::make_shared(netType, shape_state_1); + auto inner_state_2 = std::make_shared(netType, shape_state_2); + auto inner_state_3 = std::make_shared(netType, shape_state_3); + auto inner_param_mm = std::make_shared(netType, main_mm->get_output_partial_shape(0)); + + ov::OutputVector states_outs{inner_state_1, inner_state_2, inner_state_3}; + auto lora_subgraph = create_lora_subgraph(inner_param_mm, inner_param_lora, states_outs, false); + lora_subgraph->set_friendly_name("lora_subgraph"); + ov::ParameterVector inner_params{inner_param_mm, inner_param_lora, inner_state_1, inner_state_2, inner_state_3}; + auto inner_model = std::make_shared(OutputVector{lora_subgraph}, inner_params); + + auto states = create_states({shape_state_1, shape_state_2, shape_state_3}, ov::element::f16); + ov::OutputVector lora_inputs{main_mm, param_lora, states.first[0], states.first[1], states.first[2]}; + auto lora = std::make_shared(lora_inputs, inner_model); + lora->set_friendly_name("lora_subgraph"); + + model_ref = + std::make_shared(OutputVector{lora, main_mm}, states.second, ParameterVector{param_lora, param_w}); + } +} + TEST_F(LoraSubgraphFusionMatMulTests, ReshaffledEltwiseInputs) { { auto param_lora = std::make_shared(netType, shape_x); diff --git a/src/common/transformations/tests/control_flow/unroll_tensor_iterator_test.cpp b/src/common/transformations/tests/control_flow/unroll_tensor_iterator_test.cpp index 9ff2be2e754e95..fb4891d1590cc7 100644 --- a/src/common/transformations/tests/control_flow/unroll_tensor_iterator_test.cpp +++ b/src/common/transformations/tests/control_flow/unroll_tensor_iterator_test.cpp @@ -491,14 +491,12 @@ TEST(TransformationTests, UnrollTensorIteratorLSTMCellSingleIterationSingleItera ASSERT_TRUE(res.first) << res.second; } -void collect_legacy_tensor_names(const std::shared_ptr& model, std::vector& holder) { +void collect_tensor_names(const std::shared_ptr& model, + std::vector>& holder) { for (const auto& op : model->get_ordered_ops()) { for (const auto& out : op->outputs()) { - OPENVINO_SUPPRESS_DEPRECATED_START - auto tensor_name = ov::descriptor::get_ov_tensor_legacy_name(out.get_tensor()); - OPENVINO_SUPPRESS_DEPRECATED_END - if (!tensor_name.empty() && ov::as_type_ptr(op)) - holder.emplace_back(tensor_name); + if (!out.get_tensor().get_names().empty() && ov::as_type_ptr(op)) + holder.emplace_back(out.get_tensor().get_names()); } } } @@ -538,8 +536,8 @@ TEST(TransformationTests, CheckTensorNamesAfterConvertToTIAndUnrolling) { f = std::make_shared(NodeVector{Y_out, Ho}, ParameterVector{X, Y}); } - std::vector names_before; - collect_legacy_tensor_names(f, names_before); + std::vector> names_before; + collect_tensor_names(f, names_before); pass::Manager m; m.register_pass(); @@ -548,8 +546,8 @@ TEST(TransformationTests, CheckTensorNamesAfterConvertToTIAndUnrolling) { m.run_passes(f); OV_ASSERT_NO_THROW(check_rt_info(f)); - std::vector names_after; - collect_legacy_tensor_names(f, names_after); + std::vector> names_after; + collect_tensor_names(f, names_after); EXPECT_EQ(names_before, names_after); } @@ -605,8 +603,8 @@ TEST(TransformationTests, CheckTensorNamesAfterUnrolling) { f = std::make_shared(NodeVector{res_ti_1, res_ti_2}, ParameterVector{X, Y, Z}); } - std::vector names_before; - collect_legacy_tensor_names(f, names_before); + std::vector> names_before; + collect_tensor_names(f, names_before); pass::Manager m; m.register_pass(); @@ -614,11 +612,9 @@ TEST(TransformationTests, CheckTensorNamesAfterUnrolling) { m.run_passes(f); OV_ASSERT_NO_THROW(check_rt_info(f)); - std::vector names_after; - collect_legacy_tensor_names(f, names_after); + std::vector> names_after; + collect_tensor_names(f, names_after); - EXPECT_NE(names_before, names_after); - EXPECT_EQ(names_after.size(), 2); - EXPECT_EQ(names_after[0], "TensorIterator.0"); - EXPECT_EQ(names_after[1], "TensorIterator.1"); + ASSERT_EQ(names_after.size(), 0); + EXPECT_EQ(names_before, names_after); } diff --git a/src/common/transformations/tests/op_conversions/convert_squeeze15_downgrade_test.cpp b/src/common/transformations/tests/op_conversions/convert_squeeze15_downgrade_test.cpp new file mode 100644 index 00000000000000..f3d90ab2c748bd --- /dev/null +++ b/src/common/transformations/tests/op_conversions/convert_squeeze15_downgrade_test.cpp @@ -0,0 +1,112 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/op_conversions/convert_squeeze15_downgrade.hpp" + +#include + +#include + +#include "common_test_utils/ov_test_utils.hpp" +#include "openvino/opsets/opset1.hpp" +#include "openvino/opsets/opset15.hpp" +#include "openvino/pass/manager.hpp" +#include "transformations/utils/utils.hpp" +using namespace ov; +using namespace testing; + +namespace { + +enum class IndicesMode { NONE, CONST, PARAM }; + +std::shared_ptr create_v15_model(const IndicesMode indices_mode, + const std::vector indices_const_val, + const bool allow_axis_skip) { + const PartialShape data_shape{-1, {2, 5}, 1, {1, 5}, 4}; + const auto& data = std::make_shared(ov::element::f32, data_shape); + ov::ParameterVector params = {data}; + std::shared_ptr squeeze; + if (indices_mode == IndicesMode::NONE) { + squeeze = std::make_shared(data, allow_axis_skip); + } else if (indices_mode == IndicesMode::PARAM) { + const auto& indices = + std::make_shared(ov::element::i32, PartialShape({data_shape.rank()})); + params.push_back(indices); + squeeze = std::make_shared(data, indices, allow_axis_skip); + } else if (indices_mode == IndicesMode::CONST) { + const auto& indices = + ov::opset15::Constant::create(ov::element::i32, Shape({indices_const_val.size()}), indices_const_val); + squeeze = std::make_shared(data, indices, allow_axis_skip); + } + squeeze->set_friendly_name("squeeze15"); + return std::make_shared(squeeze->outputs(), params); +} + +std::shared_ptr create_v1_model(const IndicesMode indices_mode, const std::vector indices_const_val) { + const PartialShape data_shape{-1, {2, 5}, 1, {1, 5}, 4}; + const auto& data = std::make_shared(ov::element::f32, data_shape); + ov::ParameterVector params = {data}; + std::shared_ptr squeeze; + if (indices_mode == IndicesMode::NONE) { + squeeze = std::make_shared(data); + } else if (indices_mode == IndicesMode::PARAM) { + const auto& indices = + std::make_shared(ov::element::i32, PartialShape({data_shape.rank()})); + params.push_back(indices); + squeeze = std::make_shared(data, indices); + } else if (indices_mode == IndicesMode::CONST) { + const auto& indices = + ov::opset1::Constant::create(ov::element::i32, Shape({indices_const_val.size()}), indices_const_val); + squeeze = std::make_shared(data, indices); + } + squeeze->set_friendly_name("squeeze15"); + return std::make_shared(squeeze->outputs(), params); +} + +} // namespace + +TEST_F(TransformationTestsF, ConvertSqueeze15ToSqueeze1_no_indices_no_skip) { + manager.register_pass(); + model = create_v15_model(IndicesMode::NONE, {}, false); + model_ref = create_v1_model(IndicesMode::NONE, {}); + EXPECT_EQ(model->output(0).get_partial_shape(), model_ref->output(0).get_partial_shape()); + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + comparator.enable(FunctionsComparator::CmpValues::NAMES); +} + +TEST_F(TransformationTestsF, ConvertSqueeze15ToSqueeze1_no_indices_skip) { + manager.register_pass(); + model = create_v15_model(IndicesMode::NONE, {}, true); + model_ref = create_v1_model(IndicesMode::NONE, {}); + EXPECT_EQ(model->output(0).get_partial_shape(), model_ref->output(0).get_partial_shape()); + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + comparator.enable(FunctionsComparator::CmpValues::NAMES); +} + +TEST_F(TransformationTestsF, ConvertSqueeze15ToSqueeze1_const_indices_no_skip) { + manager.register_pass(); + model = create_v15_model(IndicesMode::CONST, {0, -4, 3}, false); + model_ref = create_v1_model(IndicesMode::CONST, {0, -4, 3}); + EXPECT_EQ(model->output(0).get_partial_shape(), model_ref->output(0).get_partial_shape()); + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + comparator.enable(FunctionsComparator::CmpValues::NAMES); +} + +TEST_F(TransformationTestsF, ConvertSqueeze15ToSqueeze1_dynamic_indices_no_skip) { + manager.register_pass(); + model = create_v15_model(IndicesMode::PARAM, {}, false); + model_ref = create_v1_model(IndicesMode::PARAM, {}); + EXPECT_EQ(model->output(0).get_partial_shape(), model_ref->output(0).get_partial_shape()); + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); + comparator.enable(FunctionsComparator::CmpValues::NAMES); +} + +TEST_F(TransformationTestsF, ConvertSqueeze15ToSqueeze1_unsupported_skip) { + manager.register_pass(); + model = create_v15_model(IndicesMode::PARAM, {}, true); +} diff --git a/src/common/transformations/tests/utils/convert_precision.cpp b/src/common/transformations/tests/utils/convert_precision.cpp index 03095b28d96a74..318f15ab1a64dc 100644 --- a/src/common/transformations/tests/utils/convert_precision.cpp +++ b/src/common/transformations/tests/utils/convert_precision.cpp @@ -15,6 +15,7 @@ #include "openvino/core/model.hpp" #include "openvino/opsets/opset1.hpp" #include "openvino/opsets/opset10.hpp" +#include "openvino/opsets/opset15.hpp" #include "openvino/opsets/opset3.hpp" #include "openvino/opsets/opset4.hpp" #include "openvino/opsets/opset5.hpp" @@ -963,6 +964,34 @@ TEST(TransformationTests, ConvertPrecision_Select) { ASSERT_TRUE(has_type(f)); } +TEST(TransformationTests, ConvertPrecision_Select_Relaxed) { + std::shared_ptr f(nullptr); + { + auto input1 = std::make_shared(element::boolean, Shape{15, 20, 3}); + auto node = std::make_shared(input1); + auto select = std::make_shared(node, input1, input1); + + f = std::make_shared(OutputVector{select}, ParameterVector{input1}); + + // Explicitly setting the element type of a node to a different one to + // test the appearance of TypeRelaxed within Select + node->set_output_type(0, ov::element::u8, node->get_output_partial_shape(0)); + + pass::Manager manager; + manager.register_pass(); + manager.register_pass(precisions_map{{element::u8, element::boolean}}); + manager.run_passes(f); + } + OV_ASSERT_NO_THROW(check_rt_info(f)); + ASSERT_FALSE(has_type(f)); + ASSERT_TRUE(has_type(f)); + int counter = 0; + for (const auto& node : f->get_ordered_ops()) + if (std::dynamic_pointer_cast(node)) + ++counter; + ASSERT_EQ(counter, 1); +} + TEST(TransformationTests, ConvertPrecision_TypeRelaxedWithSelect) { std::shared_ptr f(nullptr); { @@ -1008,6 +1037,28 @@ TEST(TransformationTests, ConvertPrecision_TypeRelaxed) { } } +TEST(TransformationTests, ConvertPrecision_SearchSorted) { + std::shared_ptr f(nullptr); + { + auto search_sorted_input = opset15::Constant::create(ov::element::i64, {5}, {1, 2, 3, 4, 5}); + auto indices = std::make_shared(ov::element::i64, Shape{3}); + auto search_sorted = std::make_shared(search_sorted_input, indices); + + auto less_input = opset15::Constant::create(ov::element::i64, {3}, {4, 5, 6}); + auto less = std::make_shared(search_sorted, less_input); + + f = std::make_shared(OutputVector{less}, ParameterVector{indices}); + + pass::Manager manager; + manager.register_pass(); + manager.register_pass(precisions_map{{element::i64, element::i32}}); + manager.run_passes(f); + } + OV_ASSERT_NO_THROW(check_rt_info(f)); + ASSERT_FALSE(has_type(f)); + ASSERT_TRUE(has_type(f)); +} + TEST(TransformationTests, ConvertPrecision_Variables) { std::shared_ptr f(nullptr); { @@ -2253,11 +2304,6 @@ TEST(TransformationTests, ConvertPrecisionExplicitConvertsSingleNodeMultipleOutp split->get_output_tensor(0).add_names({"split:0"}); split->get_output_tensor(1).add_names({"split:1"}); split->get_output_tensor(2).add_names({"split:2"}); - OPENVINO_SUPPRESS_DEPRECATED_START - ov::descriptor::set_ov_tensor_legacy_name(split->get_output_tensor(0), "legacy_split:0"); - ov::descriptor::set_ov_tensor_legacy_name(split->get_output_tensor(1), "legacy_split:1"); - ov::descriptor::set_ov_tensor_legacy_name(split->get_output_tensor(2), "legacy_split:2"); - OPENVINO_SUPPRESS_DEPRECATED_END model = make_shared(split->outputs(), ParameterVector{param_1}); type_to_fuse_map empty_type_to_fuse_map = {}; @@ -2294,11 +2340,6 @@ TEST(TransformationTests, ConvertPrecisionExplicitConvertsSingleNodeMultipleOutp ASSERT_EQ("split.0", results[0]->get_input_node_ptr(0)->get_friendly_name()); ASSERT_EQ("split.1", results[1]->get_input_node_ptr(0)->get_friendly_name()); ASSERT_EQ("split.2", results[2]->get_input_node_ptr(0)->get_friendly_name()); - OPENVINO_SUPPRESS_DEPRECATED_START - ASSERT_EQ("legacy_split:0", ov::descriptor::get_ov_tensor_legacy_name(results[0]->get_input_tensor(0))); - ASSERT_EQ("legacy_split:1", ov::descriptor::get_ov_tensor_legacy_name(results[1]->get_input_tensor(0))); - ASSERT_EQ("legacy_split:2", ov::descriptor::get_ov_tensor_legacy_name(results[2]->get_input_tensor(0))); - OPENVINO_SUPPRESS_DEPRECATED_END } TEST(TransformationTests, ConvertPrecisionExplicitConvertsMultiSubgraphs) { diff --git a/src/common/util/include/openvino/util/weights_path.hpp b/src/common/util/include/openvino/util/weights_path.hpp new file mode 100644 index 00000000000000..11215d8eee1304 --- /dev/null +++ b/src/common/util/include/openvino/util/weights_path.hpp @@ -0,0 +1,15 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/util/common_util.hpp" + +namespace ov { +namespace util { + +bool validate_weights_path(std::string& weights_path); + +} // namespace util +} // namespace ov diff --git a/src/common/util/src/weights_path.cpp b/src/common/util/src/weights_path.cpp new file mode 100644 index 00000000000000..56d0b56d643fa8 --- /dev/null +++ b/src/common/util/src/weights_path.cpp @@ -0,0 +1,13 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/util/weights_path.hpp" + +bool ov::util::validate_weights_path(std::string& weights_path) { + if (weights_path.empty() || !ov::util::ends_with(weights_path, ".bin")) { + return false; + } + + return true; +} diff --git a/src/core/dev_api/openvino/core/descriptor_tensor.hpp b/src/core/dev_api/openvino/core/descriptor_tensor.hpp index 4c7c0ad8c3c972..9418183a1189fc 100644 --- a/src/core/dev_api/openvino/core/descriptor_tensor.hpp +++ b/src/core/dev_api/openvino/core/descriptor_tensor.hpp @@ -14,14 +14,5 @@ void set_element_type(Tensor& tensor, const element::Type& elemenet_type); // To change Tensor type please change the Parameter type. OPENVINO_API void set_tensor_type(Tensor& tensor, const element::Type& element_type, const PartialShape& pshape); - -OPENVINO_DEPRECATED("get_ov_tensor_legacy_name() is deprecated. Please don't use this function.") -OPENVINO_API -std::string get_ov_tensor_legacy_name(const Tensor& tensor); - -OPENVINO_DEPRECATED("set_ov_tensor_legacy_name() is deprecated. Please don't use this function.") -OPENVINO_API -void set_ov_tensor_legacy_name(Tensor& tensor, const std::string& tensor_name); - } // namespace descriptor } // namespace ov diff --git a/src/core/include/openvino/core/descriptor/tensor.hpp b/src/core/include/openvino/core/descriptor/tensor.hpp index edf2e690e8c860..a1cac56fa64e01 100644 --- a/src/core/include/openvino/core/descriptor/tensor.hpp +++ b/src/core/include/openvino/core/descriptor/tensor.hpp @@ -98,14 +98,11 @@ class OPENVINO_API Tensor { PartialShape m_partial_shape; ov::Tensor m_lower_value, m_upper_value; TensorSymbol m_value_symbol; - std::string m_legacy_name; std::unordered_set m_names; std::unordered_set::const_iterator m_name_it; RTMap m_rt_info; - friend OPENVINO_API std::string get_ov_tensor_legacy_name(const Tensor& tensor); - friend OPENVINO_API void set_ov_tensor_legacy_name(Tensor& tensor, const std::string& tensor_name); friend OPENVINO_API void set_element_type(Tensor& tensor, const element::Type& elemenet_type); friend OPENVINO_API void set_tensor_type(Tensor& tensor, const element::Type& element_type, diff --git a/src/core/include/openvino/core/type/element_type.hpp b/src/core/include/openvino/core/type/element_type.hpp index e579e8ba83210b..2af2fd272a890d 100644 --- a/src/core/include/openvino/core/type/element_type.hpp +++ b/src/core/include/openvino/core/type/element_type.hpp @@ -129,9 +129,6 @@ class OPENVINO_API Type { // Return element type in string representation std::string to_string() const; - OPENVINO_DEPRECATED("This constructor is deprecated. It will be removed in 2025.0") - Type(size_t bitwidth, bool is_real, bool is_signed, bool is_quantized, const std::string& cname); - private: Type_t m_type{Type_t::undefined}; }; diff --git a/src/core/include/openvino/core/version.hpp b/src/core/include/openvino/core/version.hpp index db1595f7addcb6..79f688795ce37c 100644 --- a/src/core/include/openvino/core/version.hpp +++ b/src/core/include/openvino/core/version.hpp @@ -19,8 +19,8 @@ * @brief Defines OpenVINO patch version */ -#define OPENVINO_VERSION_MAJOR 2024 -#define OPENVINO_VERSION_MINOR 5 +#define OPENVINO_VERSION_MAJOR 2025 +#define OPENVINO_VERSION_MINOR 0 #define OPENVINO_VERSION_PATCH 0 namespace ov { diff --git a/src/core/include/openvino/op/concat.hpp b/src/core/include/openvino/op/concat.hpp index 0e8fa67c54dfae..d32171c9facbc1 100644 --- a/src/core/include/openvino/op/concat.hpp +++ b/src/core/include/openvino/op/concat.hpp @@ -36,17 +36,6 @@ class OPENVINO_API Concat : public Op { std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; - /// \return The concatenation axis. - OPENVINO_DEPRECATED("The function get_concatenation_axis() is deprecated. Will be removed in 2025.0 release. Use " - "get_axis() instead.") - int64_t get_concatenation_axis() const { - return m_concat_axis; - } - OPENVINO_DEPRECATED("The function set_concatenation_axis() is deprecated. Will be removed in 2025.0 release. Use " - "set_axis() instead.") - void set_concatenation_axis(int64_t concatenation_axis) { - m_concat_axis = concatenation_axis; - } /// \return The concatenation axis. int64_t get_axis() const { return m_axis; diff --git a/src/core/include/openvino/op/lstm_sequence.hpp b/src/core/include/openvino/op/lstm_sequence.hpp index 4d68b5ceb7f9c6..28f5a2b1b8bfaf 100644 --- a/src/core/include/openvino/op/lstm_sequence.hpp +++ b/src/core/include/openvino/op/lstm_sequence.hpp @@ -17,103 +17,6 @@ namespace ov { namespace op { -namespace v0 { - -/// -/// \brief Class for lstm sequence node. -/// -/// \note It follows notation and equations defined as in ONNX standard: -/// https://github.com/onnx/onnx/blob/master/docs/Operators.md#LSTM -/// -/// \sa LSTMCell, RNNCell, GRUCell -/// -/// -/// \ingroup ov_ops_cpp_api -class OPENVINO_API OPENVINO_DEPRECATED( - "The class ov::op::v0::LSTMSequence is deprecated. It will be removed in 2025.0 release. Use " - "ov::op::v5::LSTMSequence instead.") LSTMSequence : public util::RNNCellBase { -public: - OPENVINO_OP("LSTMSequence", "opset1", util::RNNCellBase); - LSTMSequence() = default; - - using direction = RecurrentSequenceDirection; - - size_t get_default_output_index() const override { - return no_default_index(); - } - explicit LSTMSequence(const Output& X, - const Output& initial_hidden_state, - const Output& initial_cell_state, - const Output& sequence_lengths, - const Output& W, - const Output& R, - const Output& B, - const Output& P, - const std::int64_t hidden_size, - const direction lstm_direction, - LSTMWeightsFormat weights_format = LSTMWeightsFormat::IFCO, - const std::vector activations_alpha = {}, - const std::vector activations_beta = {}, - const std::vector activations = {"sigmoid", "tanh", "tanh"}, - const float clip_threshold = 0, - const bool input_forget = false); - - explicit LSTMSequence(const Output& X, - const Output& initial_hidden_state, - const Output& initial_cell_state, - const Output& sequence_lengths, - const Output& W, - const Output& R, - const Output& B, - const std::int64_t hidden_size, - const direction lstm_direction, - LSTMWeightsFormat weights_format = LSTMWeightsFormat::IFCO, - const std::vector& activations_alpha = {}, - const std::vector& activations_beta = {}, - const std::vector& activations = {"sigmoid", "tanh", "tanh"}, - const float clip_threshold = 0, - const bool input_forget = false); - - void validate_and_infer_types() override; - bool visit_attributes(AttributeVisitor& visitor) override; - - std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; - - std::vector get_activations_alpha() const { - return m_activations_alpha; - } - std::vector get_activations_beta() const { - return m_activations_beta; - } - std::vector get_activations() const { - return m_activations; - } - float get_clip_threshold() const { - return m_clip; - } - direction get_direction() const { - return m_direction; - } - void set_direction(const direction& dir) { - m_direction = dir; - } - std::int64_t get_hidden_size() const { - return m_hidden_size; - } - bool get_input_forget() const { - return m_input_forget; - } - LSTMWeightsFormat get_weights_format() const { - return m_weights_format; - } - -private: - direction m_direction; - bool m_input_forget; - LSTMWeightsFormat m_weights_format; -}; -} // namespace v0 - namespace v5 { /// /// \brief Class for lstm sequence node. diff --git a/src/core/include/openvino/op/search_sorted.hpp b/src/core/include/openvino/op/search_sorted.hpp index c370ba46b2f182..efb1f8491e0882 100644 --- a/src/core/include/openvino/op/search_sorted.hpp +++ b/src/core/include/openvino/op/search_sorted.hpp @@ -22,7 +22,15 @@ class OPENVINO_API SearchSorted : public Op { /// \param values Values to search indexs for. /// \param right_mode If False, return the first suitable index that is found for given value. If True, return /// the last such index. - SearchSorted(const Output& sorted_sequence, const Output& values, bool right_mode = false); + /// \param output_type The element type of the output tensor. This is purely an implementation flag, which + /// is used to convert the output type for CPU plugin in ConvertPrecision transformation (and potentially other + /// plugins as well). Setting this flag to element::i32 will result in the output tensor of i32 element type. + /// Setting this flag to element::i64 will generally not give any effect, since it will be converted to i32 anyway, + /// at least for CPU plugin. + SearchSorted(const Output& sorted_sequence, + const Output& values, + bool right_mode = false, + const element::Type& output_type = element::i64); void validate_and_infer_types() override; bool visit_attributes(AttributeVisitor& visitor) override; @@ -36,8 +44,17 @@ class OPENVINO_API SearchSorted : public Op { m_right_mode = right_mode; } + void set_output_type_attr(const element::Type& output_type) { + m_output_type = output_type; + } + + element::Type get_output_type_attr() const { + return m_output_type; + } + private: bool m_right_mode{}; + element::Type m_output_type = element::i64; }; } // namespace v15 } // namespace op diff --git a/src/core/include/openvino/op/squeeze.hpp b/src/core/include/openvino/op/squeeze.hpp index 8c27f29d66df66..dde456aa2eef47 100644 --- a/src/core/include/openvino/op/squeeze.hpp +++ b/src/core/include/openvino/op/squeeze.hpp @@ -4,7 +4,7 @@ #pragma once -#include "openvino/op/op.hpp" +#include "openvino/op/util/squeeze_base.hpp" namespace ov { namespace op { @@ -12,30 +12,65 @@ namespace v0 { /// \brief Squeeze operation. /// /// \ingroup ov_ops_cpp_api -class OPENVINO_API Squeeze : public Op { +class OPENVINO_API Squeeze : public util::SqueezeBase { public: OPENVINO_OP("Squeeze", "opset1"); Squeeze(); - Squeeze(const Output& data, const Output& axes); + /// \brief Constructs a squeeze v0 operation. + /// + /// \param data Input tensor with data Squeeze(const Output& data); + /// \brief Constructs a squeeze v0 operation. + /// + /// \param data Input tensor with data + /// \param axis The axis along which to squeeze the input tensor. + Squeeze(const Output& data, const Output& axes); void validate_and_infer_types() override; bool evaluate(TensorVector& outputs, const TensorVector& inputs) const override; - bool has_evaluate() const override; - bool evaluate_lower(TensorVector& outputs) const override; - bool evaluate_upper(TensorVector& outputs) const override; - bool evaluate_symbol(TensorSymbolVector& output_symbols) const override; - bool constant_fold(OutputVector& output_values, const OutputVector& inputs_values) override; - bool can_constant_fold(const OutputVector& inputs_values) const override; std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; - bool is_dynamic() const override; - private: Output get_default_axes_input() const; }; } // namespace v0 + +namespace v15 { +/// \brief Squeeze operation. +/// +/// \ingroup ov_ops_cpp_api +class OPENVINO_API Squeeze : public util::SqueezeBase { +public: + OPENVINO_OP("Squeeze", "opset15"); + + Squeeze(); + /// \brief Constructs a squeeze v15 operation. + /// + /// \param data Input tensor with data + /// \param allow_axis_skip Shape inference result dynamic rank if selected axis has 1 in range of its dynamic + Squeeze(const Output& data, const bool allow_axis_skip = false); + /// \brief Constructs a squeeze v15 operation. + /// + /// \param data Input tensor with data + /// \param axis The axis along which to squeeze the input tensor. + /// \param allow_axis_skip Shape inference result dynamic rank if selected axis has 1 in range of its dynamic + Squeeze(const Output& data, const Output& axes, const bool allow_axis_skip = false); + + void validate_and_infer_types() override; + bool evaluate(TensorVector& outputs, const TensorVector& inputs) const override; + + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + + bool visit_attributes(AttributeVisitor& visitor) override; + + bool get_allow_axis_skip() const; + +private: + Output get_default_axes_input() const; + bool m_allow_axis_skip{}; +}; +} // namespace v15 } // namespace op } // namespace ov diff --git a/src/core/include/openvino/op/stft.hpp b/src/core/include/openvino/op/stft.hpp index 0a8ab80e7a48fe..0b0d8ae9a9e473 100644 --- a/src/core/include/openvino/op/stft.hpp +++ b/src/core/include/openvino/op/stft.hpp @@ -13,7 +13,7 @@ namespace v15 { /// \ingroup ov_ops_cpp_api class OPENVINO_API STFT : public Op { public: - OPENVINO_OP("STFT"); + OPENVINO_OP("STFT", "opset15"); STFT() = default; /// \brief Constructs a STFT operation. diff --git a/src/core/include/openvino/op/util/broadcast_base.hpp b/src/core/include/openvino/op/util/broadcast_base.hpp index 2e500eb611c04c..6300559ac8cf00 100644 --- a/src/core/include/openvino/op/util/broadcast_base.hpp +++ b/src/core/include/openvino/op/util/broadcast_base.hpp @@ -63,6 +63,7 @@ class OPENVINO_API BroadcastBase : public Op { bool evaluate_lower(TensorVector& outputs) const override; bool evaluate_upper(TensorVector& outputs) const override; + bool evaluate_symbol(ov::TensorSymbolVector& output_symbols) const override; PartialShape get_result_shape_pdpd(const PartialShape& arg0_shape, const PartialShape& target_shape, diff --git a/src/core/include/openvino/op/util/squeeze_base.hpp b/src/core/include/openvino/op/util/squeeze_base.hpp new file mode 100644 index 00000000000000..50d960824e10d2 --- /dev/null +++ b/src/core/include/openvino/op/util/squeeze_base.hpp @@ -0,0 +1,39 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/op/op.hpp" + +namespace ov { +namespace op { +namespace util { +/// \brief Squeeze operation. +/// +/// \ingroup ov_ops_cpp_api +class OPENVINO_API SqueezeBase : public Op { +public: + OPENVINO_OP("Squeeze", "util"); + SqueezeBase() = default; + /// \brief Constructs a squeeze operation. + /// + /// \param data Input tensor with data + SqueezeBase(const Output& data); + /// \brief Constructs a squeeze operation. + /// + /// \param data Input tensor with data + /// \param axis The axis along which to squeeze the input tensor. + SqueezeBase(const Output& data, const Output& axes); + + bool has_evaluate() const override; + bool evaluate_lower(TensorVector& outputs) const override; + bool evaluate_upper(TensorVector& outputs) const override; + bool evaluate_symbol(TensorSymbolVector& output_symbols) const override; + bool can_constant_fold(const OutputVector& inputs_values) const override; + bool constant_fold(OutputVector& output_values, const OutputVector& inputs_values) override; + bool is_dynamic() const override; +}; +} // namespace util +} // namespace op +} // namespace ov diff --git a/src/core/include/openvino/opsets/opset1.hpp b/src/core/include/openvino/opsets/opset1.hpp index 80ea1b517e4080..e2a7a159299ad3 100644 --- a/src/core/include/openvino/opsets/opset1.hpp +++ b/src/core/include/openvino/opsets/opset1.hpp @@ -9,9 +9,7 @@ namespace ov { namespace opset1 { #define _OPENVINO_OP_REG(a, b) using b::a; -OPENVINO_SUPPRESS_DEPRECATED_START #include "openvino/opsets/opset1_tbl.hpp" -OPENVINO_SUPPRESS_DEPRECATED_END #undef _OPENVINO_OP_REG } // namespace opset1 } // namespace ov diff --git a/src/core/include/openvino/opsets/opset15_tbl.hpp b/src/core/include/openvino/opsets/opset15_tbl.hpp index 9a49e421f9ad8e..8d12420719bb6b 100644 --- a/src/core/include/openvino/opsets/opset15_tbl.hpp +++ b/src/core/include/openvino/opsets/opset15_tbl.hpp @@ -97,7 +97,7 @@ _OPENVINO_OP_REG(Sqrt, ov::op::v0) _OPENVINO_OP_REG(SpaceToDepth, ov::op::v0) _OPENVINO_OP_REG(Split, ov::op::v1) _OPENVINO_OP_REG(SquaredDifference, ov::op::v0) -_OPENVINO_OP_REG(Squeeze, ov::op::v0) +_OPENVINO_OP_REG(Squeeze, ov::op::v15) _OPENVINO_OP_REG(StridedSlice, ov::op::v1) _OPENVINO_OP_REG(Subtract, ov::op::v1) _OPENVINO_OP_REG(Tan, ov::op::v0) diff --git a/src/core/include/openvino/opsets/opset1_tbl.hpp b/src/core/include/openvino/opsets/opset1_tbl.hpp index 089df5896e664c..3008e450065926 100644 --- a/src/core/include/openvino/opsets/opset1_tbl.hpp +++ b/src/core/include/openvino/opsets/opset1_tbl.hpp @@ -88,7 +88,6 @@ _OPENVINO_OP_REG(LogicalOr, ov::op::v1) _OPENVINO_OP_REG(LogicalXor, ov::op::v1) _OPENVINO_OP_REG(LRN, ov::op::v0) _OPENVINO_OP_REG(LSTMCell, ov::op::v0) -_OPENVINO_OP_REG(LSTMSequence, ov::op::v0) _OPENVINO_OP_REG(MatMul, ov::op::v0) _OPENVINO_OP_REG(MaxPool, ov::op::v1) _OPENVINO_OP_REG(Maximum, ov::op::v1) diff --git a/src/core/include/openvino/opsets/opset2.hpp b/src/core/include/openvino/opsets/opset2.hpp index 3c0888c0a88487..bcdaa6a1bc354b 100644 --- a/src/core/include/openvino/opsets/opset2.hpp +++ b/src/core/include/openvino/opsets/opset2.hpp @@ -9,9 +9,7 @@ namespace ov { namespace opset2 { #define _OPENVINO_OP_REG(a, b) using b::a; -OPENVINO_SUPPRESS_DEPRECATED_START #include "openvino/opsets/opset2_tbl.hpp" -OPENVINO_SUPPRESS_DEPRECATED_END #undef _OPENVINO_OP_REG } // namespace opset2 } // namespace ov diff --git a/src/core/include/openvino/opsets/opset2_tbl.hpp b/src/core/include/openvino/opsets/opset2_tbl.hpp index 1d7c55ea74549e..e14887aaced94b 100644 --- a/src/core/include/openvino/opsets/opset2_tbl.hpp +++ b/src/core/include/openvino/opsets/opset2_tbl.hpp @@ -57,7 +57,6 @@ _OPENVINO_OP_REG(LogicalOr, ov::op::v1) _OPENVINO_OP_REG(LogicalXor, ov::op::v1) _OPENVINO_OP_REG(LRN, ov::op::v0) _OPENVINO_OP_REG(LSTMCell, ov::op::v0) -_OPENVINO_OP_REG(LSTMSequence, ov::op::v0) _OPENVINO_OP_REG(MatMul, ov::op::v0) _OPENVINO_OP_REG(MaxPool, ov::op::v1) _OPENVINO_OP_REG(Maximum, ov::op::v1) diff --git a/src/core/include/openvino/opsets/opset3.hpp b/src/core/include/openvino/opsets/opset3.hpp index 29cd23b89c1111..a4e99a21e1c643 100644 --- a/src/core/include/openvino/opsets/opset3.hpp +++ b/src/core/include/openvino/opsets/opset3.hpp @@ -9,9 +9,7 @@ namespace ov { namespace opset3 { #define _OPENVINO_OP_REG(a, b) using b::a; -OPENVINO_SUPPRESS_DEPRECATED_START #include "openvino/opsets/opset3_tbl.hpp" -OPENVINO_SUPPRESS_DEPRECATED_END #undef _OPENVINO_OP_REG } // namespace opset3 } // namespace ov diff --git a/src/core/include/openvino/opsets/opset3_tbl.hpp b/src/core/include/openvino/opsets/opset3_tbl.hpp index 3c34a6999f2696..5c16414a02e987 100644 --- a/src/core/include/openvino/opsets/opset3_tbl.hpp +++ b/src/core/include/openvino/opsets/opset3_tbl.hpp @@ -60,7 +60,6 @@ _OPENVINO_OP_REG(LogicalOr, ov::op::v1) _OPENVINO_OP_REG(LogicalXor, ov::op::v1) _OPENVINO_OP_REG(LRN, ov::op::v0) _OPENVINO_OP_REG(LSTMCell, ov::op::v0) -_OPENVINO_OP_REG(LSTMSequence, ov::op::v0) _OPENVINO_OP_REG(MatMul, ov::op::v0) _OPENVINO_OP_REG(MaxPool, ov::op::v1) _OPENVINO_OP_REG(Maximum, ov::op::v1) diff --git a/src/core/reference/include/openvino/reference/sequences.hpp b/src/core/reference/include/openvino/reference/sequences.hpp index af1946fcbe72c9..42b4083793b63b 100644 --- a/src/core/reference/include/openvino/reference/sequences.hpp +++ b/src/core/reference/include/openvino/reference/sequences.hpp @@ -380,131 +380,6 @@ void lstm_sequence(const char* X, } } -template -void lstm_sequence_v1(const char* X, - const Shape& X_shape, - const char* H, - const Shape& H_shape, - const char* C, - const Shape& C_shape, - const char* seq_lengths, - const Shape& seq_lengths_shape, - const char* W, - const Shape& W_shape, - const char* R, - const Shape& R_shape, - const char* B, - const Shape& B_shape, - const char* P, - const Shape& P_shape, - char* Y, - char* Ho, - char* Co, - const std::string& activation_f, - const std::string& activation_g, - const std::string& activation_h, - float clip, - const ov::op::LSTMWeightsFormat weight_format, - bool input_forget, - op::RecurrentSequenceDirection direction) { - OutputVector results; - if (direction == op::RecurrentSequenceDirection::FORWARD || direction == op::RecurrentSequenceDirection::REVERSE) { - CellArgs args; - args.activation_f = activation_f; - args.activation_g = activation_g; - args.activation_h = activation_h; - args.clip = clip; - args.weight_format = weight_format; - args.input_forget = input_forget; - std::vector inputs = {X, seq_lengths, H, C, W, R, B, P}; - std::vector outputs = {Y, Ho, Co}; - std::vector shapes = {X_shape, seq_lengths_shape, H_shape, C_shape, W_shape, R_shape, B_shape, P_shape}; - cell_pass(CellType::LSTM_v1, - inputs, - shapes, - outputs, - args, - direction == op::RecurrentSequenceDirection::REVERSE); - } else if (direction == op::RecurrentSequenceDirection::BIDIRECTIONAL) { - // Split bidirectional case to forward + reverse passes. - // split inputs - std::vector> H_split(2, std::vector(sizeof(T) * shape_size(H_shape) / 2)); - std::vector> C_split(2, std::vector(sizeof(T) * shape_size(C_shape) / 2)); - std::vector> W_split(2, std::vector(sizeof(T) * shape_size(W_shape) / 2)); - std::vector> R_split(2, std::vector(sizeof(T) * shape_size(R_shape) / 2)); - std::vector> B_split(2, std::vector(sizeof(T) * shape_size(B_shape) / 2)); - std::vector> P_split(2, std::vector(sizeof(T) * shape_size(P_shape) / 2)); - char* h_pointers[2] = {H_split[0].data(), H_split[1].data()}; - char* c_pointers[2] = {C_split[0].data(), C_split[1].data()}; - char* w_pointers[2] = {W_split[0].data(), W_split[1].data()}; - char* r_pointers[2] = {R_split[0].data(), R_split[1].data()}; - char* b_pointers[2] = {B_split[0].data(), B_split[1].data()}; - char* p_pointers[2] = {P_split[0].data(), P_split[1].data()}; - reference::split(H, H_shape, sizeof(T), 1, 2, h_pointers); - reference::split(C, C_shape, sizeof(T), 1, 2, c_pointers); - reference::split(W, W_shape, sizeof(T), 0, 2, w_pointers); - reference::split(R, R_shape, sizeof(T), 0, 2, r_pointers); - reference::split(B, B_shape, sizeof(T), 0, 2, b_pointers); - reference::split(P, P_shape, sizeof(T), 0, 2, p_pointers); - std::vector forward_res_y(sizeof(T) * H_shape[0] * H_shape[2] * X_shape[1]); - std::vector reverse_res_y(sizeof(T) * H_shape[0] * H_shape[2] * X_shape[1]); - std::vector> forward_res(2, std::vector(sizeof(T) * H_shape[0] * H_shape[2])); - std::vector> reverse_res(2, std::vector(sizeof(T) * H_shape[0] * H_shape[2])); - - CellArgs args; - args.activation_f = activation_f; - args.activation_g = activation_g; - args.activation_h = activation_h; - args.clip = clip; - args.weight_format = weight_format; - args.input_forget = input_forget; - std::vector shapes = {X_shape, seq_lengths_shape, H_shape, C_shape, W_shape, R_shape, B_shape, P_shape}; - // update H,C,W,R,B,P shapes after split - shapes[2][1] = 1; - shapes[3][1] = 1; - for (size_t i = 4; i < shapes.size(); ++i) { - shapes[i][0] = 1; - } - // forward pass - cell_pass( - CellType::LSTM_v1, - {X, seq_lengths, h_pointers[0], c_pointers[0], w_pointers[0], r_pointers[0], b_pointers[0], p_pointers[0]}, - shapes, - {forward_res_y.data(), forward_res[0].data(), forward_res[1].data()}, - args, - false); - // reverse pass - cell_pass( - CellType::LSTM_v1, - {X, seq_lengths, h_pointers[1], c_pointers[1], w_pointers[1], r_pointers[1], b_pointers[1], p_pointers[1]}, - shapes, - {reverse_res_y.data(), reverse_res[0].data(), reverse_res[1].data()}, - args, - true); - - // Stack together respective outputs from both forward and reverse passes. - std::vector in_shapes_y = {{H_shape[0], 1, X_shape[1], H_shape[2]}, - {H_shape[0], 1, X_shape[1], H_shape[2]}}; - std::vector in_shapes_h_c = {{H_shape[0], 1, H_shape[2]}, {H_shape[0], 1, H_shape[2]}}; - Shape output_shape_y{H_shape[0], 2, X_shape[1], H_shape[2]}; - Shape output_shape_h_c{H_shape[0], 2, H_shape[2]}; - - reference::concat({forward_res_y.data(), reverse_res_y.data()}, Y, in_shapes_y, output_shape_y, 1, sizeof(T)); - reference::concat({forward_res[0].data(), reverse_res[0].data()}, - Ho, - in_shapes_h_c, - output_shape_h_c, - 1, - sizeof(T)); - reference::concat({forward_res[1].data(), reverse_res[1].data()}, - Co, - in_shapes_h_c, - output_shape_h_c, - 1, - sizeof(T)); - } -} - template void gru_sequence(const char* X, const Shape& X_shape, diff --git a/src/core/reference/src/op/stft.cpp b/src/core/reference/src/op/stft.cpp index d3a93db0e1e937..6fd5583be21a75 100644 --- a/src/core/reference/src/op/stft.cpp +++ b/src/core/reference/src/op/stft.cpp @@ -21,8 +21,9 @@ void stft(const float* signal, const int64_t frame_size, const int64_t frame_step, const bool transpose_frames) { - constexpr size_t signal_axis = 1; - const auto batch_size = signal_shape[0]; + const auto is_signal_1D = signal_shape.size() == 1; + const size_t batch_size = is_signal_1D ? 1 : signal_shape[0]; + const size_t signal_axis = is_signal_1D ? 0 : 1; const auto signal_length = signal_shape[signal_axis]; const auto num_frames = static_cast((signal_length - frame_size) / frame_step) + 1; const auto frame_size_dim = static_cast(frame_size); diff --git a/src/core/shape_inference/include/lstm_sequence_shape_inference.hpp b/src/core/shape_inference/include/lstm_sequence_shape_inference.hpp index c2b5f1389dee7b..8b0daeb27663fb 100644 --- a/src/core/shape_inference/include/lstm_sequence_shape_inference.hpp +++ b/src/core/shape_inference/include/lstm_sequence_shape_inference.hpp @@ -7,32 +7,6 @@ namespace ov { namespace op { -namespace v0 { -OPENVINO_SUPPRESS_DEPRECATED_START -template -std::vector> shape_infer(const LSTMSequence* op, const std::vector& input_shapes) { - OPENVINO_SUPPRESS_DEPRECATED_END - constexpr auto num_gates = 4; - constexpr auto num_state_nodes = 2; - const auto output_shapes = - rnn::seq_base_shape_infer(op, input_shapes, num_gates, num_state_nodes, op->get_direction()); - // Validate rank and dimension for P input (the input doesn't exists in the next version of LSTM or other RNN based - // ops) The checks are compatible with the original restrictions of the v0::LSTMSequence - const auto& hidden_size = output_shapes[0][3]; - if (input_shapes.size() > 7 && input_shapes[7].is_static() && hidden_size.is_static()) { - const auto& p_pshape = input_shapes[7]; - NODE_VALIDATION_CHECK(op, p_pshape.rank().compatible(2), "Input tensor P should have rank equal 2."); - NODE_VALIDATION_CHECK(op, - p_pshape[1].compatible(hidden_size * (num_gates - 1)), - "Inorrect shape of P input. Second dimension is: ", - p_pshape[1], - ", expected: ", - hidden_size.get_length() * (num_gates - 1), - "."); - } - return output_shapes; -} -} // namespace v0 namespace v5 { template std::vector> shape_infer(const LSTMSequence* op, const std::vector& input_shapes) { diff --git a/src/core/shape_inference/include/squeeze_shape_inference.hpp b/src/core/shape_inference/include/squeeze_shape_inference.hpp index ee71b5452db1c3..31eeea5d36a9ea 100644 --- a/src/core/shape_inference/include/squeeze_shape_inference.hpp +++ b/src/core/shape_inference/include/squeeze_shape_inference.hpp @@ -11,6 +11,117 @@ namespace ov { namespace op { namespace v0 { +template > +std::vector shape_infer(const Squeeze* op, + const std::vector& input_shapes, + const ITensorAccessor& ta = make_tensor_accessor()) { + using DimType = typename T::value_type; + + const auto number_of_inputs = input_shapes.size(); + OPENVINO_ASSERT(!input_shapes.empty()); + + const auto& arg_shape = input_shapes[0]; + const auto& arg_rank = arg_shape.rank(); + auto output_shapes = std::vector(1); + auto& output_shape = output_shapes[0]; + + std::unique_ptr> unique_axes; + + if (number_of_inputs == 1) { + unique_axes.reset(new std::set()); + } else if (number_of_inputs == 2) { + const auto& axes_shape = input_shapes[1]; + NODE_VALIDATION_CHECK(op, + axes_shape.is_dynamic() || ov::util::is_rank_compatible_any_of(axes_shape.rank(), {0, 1}), + "Second input (axes) should not be of rank higher than 1. Got: ", + axes_shape.rank().get_length()); + + std::vector axes; + if (arg_rank.is_static() && axes_shape.is_static()) { + if (auto axes = get_input_const_data_as(op, 1, ta)) { + // The values of `axes` input are known + ov::util::try_normalize_axes(*axes, arg_rank, *op); + unique_axes.reset(new std::set(axes->cbegin(), axes->cend())); + } else if (arg_rank.get_length() > 0 && shape_size(axes_shape.to_shape()) == 1) { + // The `axes` input is a single element tensor which is unique by definition, deducing output rank + const auto has_squeezable_dim = + std::any_of(arg_shape.cbegin(), arg_shape.cend(), [](const DimType& dim) { + return dim.compatible(1); + }); + if (has_squeezable_dim) { + output_shape = PartialShape::dynamic(arg_rank.get_length() - 1); + } else { + output_shape = arg_shape; + } + return output_shapes; + } + } + } else { + // Invalid number of inputs, empty error message for backward compatibility. + NODE_VALIDATION_CHECK(op, false); + } + + if (arg_rank.is_static() && (unique_axes != nullptr)) { + output_shape.resize(0); + if (unique_axes->empty()) { + // if only first input provided or axes are empty remove all dimensions equal to 1. + if (std::any_of(arg_shape.cbegin(), arg_shape.cend(), [](const DimType& d) { + return d.is_dynamic() && d.compatible(1); + })) { + // we are unsure if dynamic dimensions would be equal to 1 or not, so we set dynamic output rank + output_shape = PartialShape::dynamic(); + return output_shapes; + } else { + std::copy_if(arg_shape.cbegin(), + arg_shape.cend(), + std::back_inserter(output_shape), + [](const DimType& dim) { + return !dim.compatible(1); + }); + } + } else { + int64_t idx = 0; + auto rm_axis_iter = unique_axes->cbegin(); + auto rm_axis_end = unique_axes->cend(); + + // Returns true if dimension not squeezable on axis from input axes. + const auto not_squeezable_at_axis = [&rm_axis_iter, &rm_axis_end, &idx](const DimType& dim) { + if ((rm_axis_iter != rm_axis_end) && (*rm_axis_iter == idx++)) { + ++rm_axis_iter; + // Ignore: Pointed by axis, but not squeezable + return !dim.compatible(1); + } else { + return true; + } + }; + + std::copy_if(arg_shape.cbegin(), + arg_shape.cend(), + std::back_inserter(output_shape), + not_squeezable_at_axis); + } + } else { + output_shape = PartialShape::dynamic(); + } + return output_shapes; +} +} // namespace v0 + +namespace v15 { +template +bool apply_allow_axis_skip(const ov::op::v15::Squeeze* const op, + const std::unique_ptr>& unique_axes, + const T& arg_shape) { + using DimType = typename T::value_type; + int64_t i{-1}; + + return op->get_allow_axis_skip() && + std::any_of(arg_shape.cbegin(), arg_shape.cend(), [&unique_axes, &i](const DimType& d) { + ++i; + // Squeeze result with dynamic rank if 1 is in range of selected dynamic dimension. + return d.is_dynamic() && d.compatible(1) && unique_axes->find(i) != unique_axes->end(); + }); +} /** * \brief Do Squeeze shape inference. @@ -59,7 +170,7 @@ std::vector shape_infer(const Squeeze* op, return dim.compatible(1); }); if (has_squeezable_dim) { - output_shape = PartialShape::dynamic(arg_rank.get_length() - 1); + output_shape = PartialShape::dynamic(); } else { output_shape = arg_shape; } @@ -71,7 +182,9 @@ std::vector shape_infer(const Squeeze* op, NODE_VALIDATION_CHECK(op, false); } - if (arg_rank.is_static() && (unique_axes != nullptr)) { + if (!arg_rank.is_static() || (unique_axes == nullptr) || apply_allow_axis_skip(op, unique_axes, arg_shape)) { + output_shape = PartialShape::dynamic(); + } else if (arg_rank.is_static() && (unique_axes != nullptr)) { output_shape.resize(0); if (unique_axes->empty()) { // if only first input provided or axes are empty remove all dimensions equal to 1. @@ -115,6 +228,6 @@ std::vector shape_infer(const Squeeze* op, } return output_shapes; } -} // namespace v0 +} // namespace v15 } // namespace op } // namespace ov diff --git a/src/core/shape_inference/include/stft_shape_inference.hpp b/src/core/shape_inference/include/stft_shape_inference.hpp index c856b4f905b7e6..41abf6640ddb96 100644 --- a/src/core/shape_inference/include/stft_shape_inference.hpp +++ b/src/core/shape_inference/include/stft_shape_inference.hpp @@ -25,10 +25,11 @@ std::vector shape_infer(const STFT* op, const auto& frame_size_shape = input_shapes[2]; const auto& frame_step_shape = input_shapes[3]; + const auto signal_shape_rank = signal_shape.rank(); NODE_SHAPE_INFER_CHECK(op, input_shapes, - signal_shape.rank().compatible(2), - "The shape of signal must be 2D [batch, signal_size]."); + signal_shape_rank.compatible(1) || signal_shape_rank.compatible(2), + "The shape of signal must be 1D [signal_size] or 2D [batch, signal_size]."); NODE_SHAPE_INFER_CHECK(op, input_shapes, window_shape.rank().compatible(1), @@ -42,29 +43,36 @@ std::vector shape_infer(const STFT* op, frame_step_shape.rank().compatible(0), "The shape of frame_step must be a scalar."); + if (signal_shape_rank.is_dynamic()) { + return {signal_shape}; + } + const auto frame_size = get_input_const_data_as(op, 2, ta); const auto frame_step = get_input_const_data_as(op, 3, ta); - if (signal_shape.rank().is_dynamic()) { - return {signal_shape}; - } else if (!frame_size || !frame_step) { - return {TRShape{signal_shape[0], TDim(ov::util::dim::inf_bound), TDim(ov::util::dim::inf_bound), 2}}; + const auto is_signal_1D = signal_shape.size() == 1; + if (!frame_size || !frame_step) { + if (is_signal_1D) { + return {TRShape{TDim(ov::util::dim::inf_bound), TDim(ov::util::dim::inf_bound), 2}}; + } else { + return {TRShape{signal_shape[0], TDim(ov::util::dim::inf_bound), TDim(ov::util::dim::inf_bound), 2}}; + } } const auto& frame_size_val = (*frame_size)[0]; const auto& frame_step_val = (*frame_step)[0]; + const TDim& signal_dim = is_signal_1D ? signal_shape[0] : signal_shape[1]; const bool is_frame_size_in_range = - 0 < frame_size_val && - (signal_shape[1].is_static() ? static_cast(frame_size_val) <= signal_shape[1].get_length() - : frame_size_val <= signal_shape[1].get_interval().get_max_val()); + 0 < frame_size_val && (signal_dim.is_static() ? static_cast(frame_size_val) <= signal_dim.get_length() + : frame_size_val <= signal_dim.get_interval().get_max_val()); NODE_SHAPE_INFER_CHECK(op, input_shapes, is_frame_size_in_range, "Provided frame size is ", frame_size_val, " but must be in range [1, ", - signal_shape[1], + signal_dim, "]."); NODE_SHAPE_INFER_CHECK(op, @@ -84,9 +92,8 @@ std::vector shape_infer(const STFT* op, frame_size_val, "]."); - const auto& batch_dim = signal_shape[0]; const TDim frame_size_dim = static_cast(frame_size_val); - const TDim signal_frame_size_diff = signal_shape[1] - frame_size_dim; + const TDim signal_frame_size_diff = signal_dim - frame_size_dim; TDim fft_samples_dim = (frame_size_val / 2) + 1; // Divsion opeartor for static Dimension of PartialShape can return non static dimension and ceil instead of floor @@ -95,9 +102,13 @@ std::vector shape_infer(const STFT* op, std::vector output_shapes; if (op->get_transpose_frames()) { - output_shapes.emplace_back(TRShape{batch_dim, std::move(fft_samples_dim), std::move(frames_dim), 2}); + output_shapes.emplace_back(TRShape{std::move(fft_samples_dim), std::move(frames_dim), 2}); } else { - output_shapes.emplace_back(TRShape{batch_dim, std::move(frames_dim), std::move(fft_samples_dim), 2}); + output_shapes.emplace_back(TRShape{std::move(frames_dim), std::move(fft_samples_dim), 2}); + } + if (!is_signal_1D) { + const auto& batch_dim = signal_shape[0]; + output_shapes[0].insert(output_shapes[0].begin(), batch_dim); } return output_shapes; } diff --git a/src/core/src/descriptor/tensor.cpp b/src/core/src/descriptor/tensor.cpp index 31509605003fdb..ae3f7c6e77cd4f 100644 --- a/src/core/src/descriptor/tensor.cpp +++ b/src/core/src/descriptor/tensor.cpp @@ -116,18 +116,9 @@ void ov::descriptor::Tensor::clone_from(const ov::descriptor::Tensor& old) { m_lower_value = old.get_lower_value(); m_upper_value = old.get_upper_value(); m_value_symbol = old.get_value_symbol(); - m_legacy_name = old.m_legacy_name; m_rt_info = old.get_rt_info(); } -std::string ov::descriptor::get_ov_tensor_legacy_name(const ov::descriptor::Tensor& tensor) { - return tensor.m_legacy_name; -} - -void ov::descriptor::set_ov_tensor_legacy_name(ov::descriptor::Tensor& tensor, const std::string& tensor_name) { - tensor.m_legacy_name = tensor_name; -} - void ov::descriptor::set_tensor_type(ov::descriptor::Tensor& tensor, const element::Type& element_type, const PartialShape& pshape) { @@ -148,10 +139,6 @@ std::ostream& ov::descriptor::operator<<(std::ostream& out, const ov::descriptor names += ", "; names += name; } - OPENVINO_SUPPRESS_DEPRECATED_START - if (names.empty()) - names = get_ov_tensor_legacy_name(tensor); - OPENVINO_SUPPRESS_DEPRECATED_END out << "Tensor(" << names << ")"; return out; } diff --git a/src/core/src/graph_util.cpp b/src/core/src/graph_util.cpp index 17780f7751d52e..4a8f4598f55cc3 100644 --- a/src/core/src/graph_util.cpp +++ b/src/core/src/graph_util.cpp @@ -273,36 +273,16 @@ bool replace_output_update_name(Output output, const Output& replace }); }; - bool preserve_legacy_output_name = false; if (has_result_consumers(output)) { - preserve_legacy_output_name = true; if (output.get_node()->get_output_size() != 1 || replacement.get_node()->get_output_size() != 1 || is_type(replacement.get_node()) || has_result_consumers(replacement)) { return false; } - } - - OPENVINO_SUPPRESS_DEPRECATED_START - if (preserve_legacy_output_name) { replacement.get_node()->set_friendly_name(output.get_node()->get_friendly_name()); - // Update output tensor name - const auto& output_tensor_name = ov::descriptor::get_ov_tensor_legacy_name(output.get_tensor()); - if (!output_tensor_name.empty()) { - ov::descriptor::set_ov_tensor_legacy_name(replacement.get_tensor(), output_tensor_name); - } else { - ov::descriptor::set_ov_tensor_legacy_name(replacement.get_tensor(), output.get_node()->get_friendly_name()); - } } - // Save replacement tensor name before replacement as they will be overridden by the output tensor name - const auto tensor_name = ov::descriptor::get_ov_tensor_legacy_name(replacement.get_tensor()); - output.replace(replacement); - // Restore back original replacement tensor name - ov::descriptor::set_ov_tensor_legacy_name(replacement.get_tensor(), tensor_name); - OPENVINO_SUPPRESS_DEPRECATED_END - copy_runtime_info({replacement.get_node_shared_ptr(), output.get_node_shared_ptr()}, replacement.get_node_shared_ptr()); return true; diff --git a/src/core/src/node.cpp b/src/core/src/node.cpp index 8b9936b5496e7c..ec9197a5a337cb 100644 --- a/src/core/src/node.cpp +++ b/src/core/src/node.cpp @@ -157,10 +157,6 @@ std::shared_ptr ov::Node::copy_with_new_inputs( } for (size_t i = 0; i < get_output_size(); i++) { clone->get_output_tensor(i).set_names(get_output_tensor(i).get_names()); - OPENVINO_SUPPRESS_DEPRECATED_START - ov::descriptor::set_ov_tensor_legacy_name(clone->get_output_tensor(i), - ov::descriptor::get_ov_tensor_legacy_name(get_output_tensor(i))); - OPENVINO_SUPPRESS_DEPRECATED_END } return clone; } diff --git a/src/core/src/node_output.cpp b/src/core/src/node_output.cpp index 3608abe13f3953..a2ea865b2ec84e 100644 --- a/src/core/src/node_output.cpp +++ b/src/core/src/node_output.cpp @@ -73,25 +73,6 @@ void Output::replace(const Output& replacement) { input.replace_source_output(replacement); } replacement.get_tensor_ptr()->add_names(get_tensor_ptr()->get_names()); - OPENVINO_SUPPRESS_DEPRECATED_START - // In legacy API we rely on output port tensor name and use it as an input or output name for the model - // Due to m_name is just a string, and we can't store multiple aliases for single output port we have to - // handle two situations during replacement: - // 1. When we replace consumers to Parameter output port we can't change its name, so we skip this part - // 2. In other cases when we replace consumers to another output port we should set name. For example: - // if we eliminate Node2 from Node1->Node2->Result we have to set Node2 output port name to Node1 - // output port name, so the output name for model won't be changed. - // But there are some cases when output name can not be preserved, so the replacement shouldn't be used: - // 1. Parameter->Node->Result - if we eliminate Node we will lose output name - // 2. Node1-->Node2->Result - if we eliminate Node2 we will lose Result output name - // `->Result - // In both of these cases please use replace_output_update_name() method which automatically prevents the - // replacement for cases when we can not preserve input/output names of model. - if (!is_type(replacement.get_node())) { - ov::descriptor::set_ov_tensor_legacy_name(replacement.get_tensor(), - ov::descriptor::get_ov_tensor_legacy_name(get_tensor())); - } - OPENVINO_SUPPRESS_DEPRECATED_END ov::copy_output_runtime_info({*this, replacement}, {replacement}); } diff --git a/src/core/src/op/lstm_sequence.cpp b/src/core/src/op/lstm_sequence.cpp index 4d9afa58f469b3..3f70b74e5ef22a 100644 --- a/src/core/src/op/lstm_sequence.cpp +++ b/src/core/src/op/lstm_sequence.cpp @@ -10,155 +10,6 @@ #include "openvino/op/util/recurrent_sequence.hpp" namespace ov { -OPENVINO_SUPPRESS_DEPRECATED_START -op::v0::LSTMSequence::LSTMSequence(const Output& X, - const Output& initial_hidden_state, - const Output& initial_cell_state, - const Output& sequence_lengths, - const Output& W, - const Output& R, - const Output& B, - const Output& P, - const std::int64_t hidden_size, - const LSTMSequence::direction lstm_direction, - LSTMWeightsFormat weights_format, - const std::vector activations_alpha, - const std::vector activations_beta, - const std::vector activations, - const float clip_threshold, - const bool input_forget) - : RNNCellBase({X, initial_hidden_state, initial_cell_state, sequence_lengths, W, R, B, P}, - hidden_size, - clip_threshold, - activations, - activations_alpha, - activations_beta), - m_direction(lstm_direction), - m_input_forget(input_forget), - m_weights_format(weights_format) { - constructor_validate_and_infer_types(); -} - -op::v0::LSTMSequence::LSTMSequence(const Output& X, - const Output& initial_hidden_state, - const Output& initial_cell_state, - const Output& sequence_lengths, - const Output& W, - const Output& R, - const Output& B, - const std::int64_t hidden_size, - const LSTMSequence::direction lstm_direction, - LSTMWeightsFormat weights_format, - const std::vector& activations_alpha, - const std::vector& activations_beta, - const std::vector& activations, - const float clip_threshold, - const bool input_forget) - : op::v0::LSTMSequence( - X, - initial_hidden_state, - initial_cell_state, - sequence_lengths, - W, - R, - B, - Constant::create(element::f32, - Shape{(lstm_direction == LSTMSequence::direction::BIDIRECTIONAL ? 2UL : 1UL), - 3UL * static_cast(hidden_size)}, - std::vector{0.f}), - hidden_size, - lstm_direction, - weights_format, - activations_alpha, - activations_beta, - activations, - clip_threshold, - input_forget) {} - -bool op::v0::LSTMSequence::visit_attributes(AttributeVisitor& visitor) { - OV_OP_SCOPE(v0_LSTMSequence_visit_attributes); - visitor.on_attribute("hidden_size", m_hidden_size); - visitor.on_attribute("activations", m_activations); - visitor.on_attribute("activations_alpha", m_activations_alpha); - visitor.on_attribute("activations_beta", m_activations_beta); - visitor.on_attribute("clip", m_clip); - visitor.on_attribute("direction", m_direction); - - visitor.on_attribute("input_forget", m_input_forget); - visitor.on_attribute("weights_format", m_weights_format); - return true; -} - -std::shared_ptr op::v0::LSTMSequence::clone_with_new_inputs(const OutputVector& new_args) const { - OV_OP_SCOPE(v0_LSTMSequence_clone_with_new_inputs); - check_new_args_count(this, new_args); - if (new_args.size() == 8) { - return std::make_shared(new_args.at(0), // X - new_args.at(1), // initial_hidden_state - new_args.at(2), // initial_cell_state - new_args.at(3), // sequence_lengths - new_args.at(4), // W - new_args.at(5), // R - new_args.at(6), // B - new_args.at(7), // P - m_hidden_size, - m_direction, - m_weights_format, - m_activations_alpha, - m_activations_beta, - m_activations, - m_clip, - m_input_forget); - } else if (new_args.size() == 7) { - return std::make_shared(new_args.at(0), // X - new_args.at(1), // initial_hidden_state - new_args.at(2), // initial_cell_state - new_args.at(3), // sequence_lengths - new_args.at(4), // W - new_args.at(5), // R - new_args.at(6), // B - m_hidden_size, - m_direction, - m_weights_format, - m_activations_alpha, - m_activations_beta, - m_activations, - m_clip, - m_input_forget); - } else { - OPENVINO_THROW("Incorrect number of new arguments"); - } -} - -void op::v0::LSTMSequence::validate_and_infer_types() { - OV_OP_SCOPE(v0_LSTMSequence_validate_and_infer_types); - auto result_et = element::dynamic; - - // Validate input types and save result for output type - NODE_VALIDATION_CHECK(this, - element::Type::merge(result_et, result_et, get_input_element_type(0)) && - element::Type::merge(result_et, result_et, get_input_element_type(1)) && - element::Type::merge(result_et, result_et, get_input_element_type(2)) && - element::Type::merge(result_et, result_et, get_input_element_type(4)) && - element::Type::merge(result_et, result_et, get_input_element_type(5)) && - element::Type::merge(result_et, result_et, get_input_element_type(6)), - "Element types for X, initial_hidden_state, initial_cell_state, W, R and B inputs do " - "not match."); - - // Mark inputs which are relevant to output parameters - for (size_t i = 0; i <= 6; ++i) - set_input_is_relevant_to_shape(i); - - const auto input_shapes = ov::util::get_node_input_partial_shapes(*this); - auto output_shapes = shape_infer(this, input_shapes); - - // Set output size, type and shape - set_output_type(0, result_et, output_shapes[0]); - set_output_type(1, result_et, output_shapes[1]); - set_output_type(2, result_et, output_shapes[2]); -} -OPENVINO_SUPPRESS_DEPRECATED_END - bool op::v5::LSTMSequence::visit_attributes(AttributeVisitor& visitor) { OV_OP_SCOPE(v5_LSTMSequence_visit_attributes); visitor.on_attribute("direction", m_direction); diff --git a/src/core/src/op/search_sorted.cpp b/src/core/src/op/search_sorted.cpp index 8b9bb012b27106..65b5ff31861d8e 100644 --- a/src/core/src/op/search_sorted.cpp +++ b/src/core/src/op/search_sorted.cpp @@ -12,9 +12,13 @@ namespace ov { namespace op { namespace v15 { -SearchSorted::SearchSorted(const Output& sorted_sequence, const Output& values, bool right_mode) +SearchSorted::SearchSorted(const Output& sorted_sequence, + const Output& values, + bool right_mode, + const element::Type& output_type) : Op({sorted_sequence, values}), - m_right_mode(right_mode) { + m_right_mode(right_mode), + m_output_type(output_type) { constructor_validate_and_infer_types(); } @@ -23,20 +27,25 @@ void SearchSorted::validate_and_infer_types() { NODE_VALIDATION_CHECK(this, get_input_element_type(0).compatible(get_input_element_type(1)), "Sorted sequence and values must have the same element type."); + NODE_VALIDATION_CHECK(this, + m_output_type == element::i32 || m_output_type == element::i64, + "The element type of the last output can only be set to i32 or i64."); + const auto& output_shapes = shape_infer(this, ov::util::get_node_input_partial_shapes(*this)); - set_output_type(0, ov::element::i64, output_shapes[0]); + set_output_type(0, m_output_type, output_shapes[0]); } bool SearchSorted::visit_attributes(AttributeVisitor& visitor) { OV_OP_SCOPE(v15_SearchSorted_visit_attributes); visitor.on_attribute("right_mode", m_right_mode); + visitor.on_attribute("output_type", m_output_type); return true; } std::shared_ptr SearchSorted::clone_with_new_inputs(const OutputVector& new_args) const { OV_OP_SCOPE(v15_SearchSorted_clone_with_new_inputs); check_new_args_count(this, new_args); - return std::make_shared(new_args.at(0), new_args.at(1), get_right_mode()); + return std::make_shared(new_args.at(0), new_args.at(1), get_right_mode(), get_output_type_attr()); } } // namespace v15 } // namespace op diff --git a/src/core/src/op/squeeze.cpp b/src/core/src/op/squeeze.cpp index 1b34a4e48a4faf..b79165ca4f5543 100644 --- a/src/core/src/op/squeeze.cpp +++ b/src/core/src/op/squeeze.cpp @@ -6,31 +6,19 @@ #include -#include "bound_evaluate.hpp" #include "itt.hpp" -#include "openvino/core/validation_util.hpp" -#include "openvino/op/constant.hpp" #include "squeeze_shape_inference.hpp" namespace ov { namespace op { namespace v0 { -namespace validate { -namespace { +Squeeze::Squeeze() : util::SqueezeBase() {} -bool axes_has_and_set_bound(const Node& op) { - return (op.get_input_size() < 2) || op.get_input_tensor(1).has_and_set_bound(); -} -} // namespace -} // namespace validate - -Squeeze::Squeeze() : Op() {} - -Squeeze::Squeeze(const Output& data, const Output& axes) : Op({data, axes}) { +Squeeze::Squeeze(const Output& data, const Output& axes) : util::SqueezeBase(data, axes) { constructor_validate_and_infer_types(); } -Squeeze::Squeeze(const Output& data) : Op({data}) { +Squeeze::Squeeze(const Output& data) : util::SqueezeBase(data) { constructor_validate_and_infer_types(); } @@ -69,62 +57,68 @@ bool Squeeze::evaluate(TensorVector& outputs, const TensorVector& inputs) const return true; } -bool Squeeze::has_evaluate() const { - OV_OP_SCOPE(v0_Squeeze_has_evaluate); - const auto validate_axes_type = [](const element::Type& et) -> bool { - switch (et) { - case element::i8: - case element::i16: - case element::i32: - case element::i64: - case element::u8: - case element::u16: - case element::u32: - case element::u64: - return true; - default: - return false; - } - }; - - return (get_input_size() < 2) || validate_axes_type(get_input_element_type(1)); -} +} // namespace v0 -bool Squeeze::evaluate_lower(TensorVector& output_values) const { - OV_OP_SCOPE(v0_Squeeze_evaluate_lower); - return validate::axes_has_and_set_bound(*this) && default_lower_bound_evaluator(this, output_values); +namespace v15 { +Squeeze::Squeeze() : util::SqueezeBase() {} + +Squeeze::Squeeze(const Output& data, const bool allow_axis_skip) + : util::SqueezeBase(data), + m_allow_axis_skip{allow_axis_skip} { + constructor_validate_and_infer_types(); } -bool Squeeze::evaluate_upper(TensorVector& output_values) const { - OV_OP_SCOPE(v0_Squeeze_evaluate_upper); - return validate::axes_has_and_set_bound(*this) && default_upper_bound_evaluator(this, output_values); +Squeeze::Squeeze(const Output& data, const Output& axes, const bool allow_axis_skip) + : util::SqueezeBase(data, axes), + m_allow_axis_skip{allow_axis_skip} { + constructor_validate_and_infer_types(); } -bool Squeeze::evaluate_symbol(TensorSymbolVector& output_symbols) const { - return validate::axes_has_and_set_bound(*this) && ov::util::default_symbol_evaluator(this, output_symbols); +std::shared_ptr Squeeze::clone_with_new_inputs(const OutputVector& new_args) const { + OV_OP_SCOPE(v15_Squeeze_clone_with_new_inputs); + check_new_args_count(this, new_args); + + switch (new_args.size()) { + case 1: + return std::make_shared(new_args[0], m_allow_axis_skip); + case 2: + return std::make_shared(new_args[0], new_args[1], m_allow_axis_skip); + default: + OPENVINO_THROW("Incorrect number of new arguments"); + } } -bool Squeeze::can_constant_fold(const OutputVector& inputs_values) const { - return get_output_partial_shape(0).is_static() && !is_const_fold_disabled(); +void Squeeze::validate_and_infer_types() { + OV_OP_SCOPE(v15_Squeeze_validate_and_infer_types); + + const auto input_shapes = ov::util::get_node_input_partial_shapes(*this); + const auto output_shapes = shape_infer(this, input_shapes); + + set_output_type(0, get_input_element_type(0), output_shapes[0]); } -bool Squeeze::constant_fold(OutputVector& output_values, const OutputVector& inputs_values) { - OV_OP_SCOPE(v0_Squeeze_constant_fold); - if (!can_constant_fold(inputs_values)) { - return false; - } +bool Squeeze::evaluate(TensorVector& outputs, const TensorVector& inputs) const { + OV_OP_SCOPE(v15_Squeeze_evaluate); + OPENVINO_ASSERT(outputs.size() == 1); - if (auto data_const = std::dynamic_pointer_cast(inputs_values[0].get_node_shared_ptr())) { - const auto& shape = get_output_shape(0); - output_values[0] = std::make_shared(*data_const, shape); - return true; - } - return false; + const auto output_shapes = + shape_infer(this, ov::util::get_tensors_partial_shapes(inputs), make_tensor_accessor(inputs)); + outputs[0].set_shape(output_shapes.front().get_shape()); + + std::memcpy(outputs[0].data(), inputs[0].data(), outputs[0].get_byte_size()); + return true; } -bool Squeeze::is_dynamic() const { - return get_output_partial_shape(0).is_dynamic(); +bool Squeeze::visit_attributes(AttributeVisitor& visitor) { + OV_OP_SCOPE(v15_Squeeze_visit_attributes); + visitor.on_attribute("allow_axis_skip", m_allow_axis_skip); + return true; } -} // namespace v0 + +bool Squeeze::get_allow_axis_skip() const { + OV_OP_SCOPE(v15_Squeeze_get_allow_axis_skip); + return m_allow_axis_skip; +} +} // namespace v15 } // namespace op } // namespace ov diff --git a/src/core/src/op/util/broadcast_base.cpp b/src/core/src/op/util/broadcast_base.cpp index 59154e45e2b37a..c2c838afeb38bd 100644 --- a/src/core/src/op/util/broadcast_base.cpp +++ b/src/core/src/op/util/broadcast_base.cpp @@ -471,3 +471,10 @@ bool ov::op::util::BroadcastBase::evaluate_upper(ov::TensorVector& output_values return false; return default_upper_bound_evaluator(this, output_values); } + +bool ov::op::util::BroadcastBase::evaluate_symbol(ov::TensorSymbolVector& output_symbols) const { + if (!input_value(1).get_tensor().has_and_set_bound() || + (get_input_size() > 2 && !input_value(2).get_tensor().has_and_set_bound())) + return false; + return default_symbol_evaluator(this, {0}, output_symbols); +} diff --git a/src/core/src/op/util/squeeze_base.cpp b/src/core/src/op/util/squeeze_base.cpp new file mode 100644 index 00000000000000..be5a20cbb58620 --- /dev/null +++ b/src/core/src/op/util/squeeze_base.cpp @@ -0,0 +1,91 @@ +#include "openvino/op/util/squeeze_base.hpp" + +#include "bound_evaluate.hpp" +#include "itt.hpp" +#include "openvino/core/validation_util.hpp" +#include "openvino/op/constant.hpp" + +namespace ov { +namespace op { + +namespace validate { +namespace { + +bool axes_has_and_set_bound(const Node& op) { + return (op.get_input_size() < 2) || op.get_input_tensor(1).has_and_set_bound(); +} +} // namespace +} // namespace validate + +namespace util { +SqueezeBase::SqueezeBase(const Output& data, const Output& axes) : Op({data, axes}) { + constructor_validate_and_infer_types(); +} + +SqueezeBase::SqueezeBase(const Output& data) : Op({data}) { + constructor_validate_and_infer_types(); +} + +bool SqueezeBase::has_evaluate() const { + OV_OP_SCOPE(util_SqueezeBase_has_evaluate); + const auto validate_axes_type = [](const element::Type& et) -> bool { + switch (et) { + case element::i8: + case element::i16: + case element::i32: + case element::i64: + case element::u8: + case element::u16: + case element::u32: + case element::u64: + return true; + default: + return false; + } + }; + + return (get_input_size() < 2) || validate_axes_type(get_input_element_type(1)); +} + +bool SqueezeBase::evaluate_lower(TensorVector& output_values) const { + OV_OP_SCOPE(util_SqueezeBase_evaluate_lower); + return validate::axes_has_and_set_bound(*this) && default_lower_bound_evaluator(this, output_values); +} + +bool SqueezeBase::evaluate_upper(TensorVector& output_values) const { + OV_OP_SCOPE(util_SqueezeBase_evaluate_upper); + return validate::axes_has_and_set_bound(*this) && default_upper_bound_evaluator(this, output_values); +} + +bool SqueezeBase::evaluate_symbol(TensorSymbolVector& output_symbols) const { + OV_OP_SCOPE(util_SqueezeBase_evaluate_symbol); + return validate::axes_has_and_set_bound(*this) && ov::util::default_symbol_evaluator(this, output_symbols); +} + +bool SqueezeBase::can_constant_fold(const OutputVector& inputs_values) const { + OV_OP_SCOPE(util_SqueezeBase_can_constant_fold); + return get_output_partial_shape(0).is_static() && !is_const_fold_disabled(); +} + +bool SqueezeBase::constant_fold(OutputVector& output_values, const OutputVector& inputs_values) { + OV_OP_SCOPE(util_SqueezeBase_constant_fold); + if (!can_constant_fold(inputs_values)) { + return false; + } + + if (auto data_const = std::dynamic_pointer_cast(inputs_values[0].get_node_shared_ptr())) { + const auto& shape = get_output_shape(0); + output_values[0] = std::make_shared(*data_const, shape); + return true; + } + return false; +} + +bool SqueezeBase::is_dynamic() const { + OV_OP_SCOPE(util_SqueezeBase_is_dynamic); + return get_output_partial_shape(0).is_dynamic(); +} + +} // namespace util +} // namespace op +} // namespace ov diff --git a/src/core/src/pass/low_latency.cpp b/src/core/src/pass/low_latency.cpp index 7ef4fcd5e84cd8..4b9f606b3e4b28 100644 --- a/src/core/src/pass/low_latency.cpp +++ b/src/core/src/pass/low_latency.cpp @@ -158,26 +158,7 @@ std::vector> process_sequence(const std::sha std::shared_ptr cell; std::vector> new_assigns; bool unroll = false; - OPENVINO_SUPPRESS_DEPRECATED_START - if (auto lstm_seq_v0 = std::dynamic_pointer_cast(op)) { - unroll = need_unroll(op); - new_assigns = replace_with_memory(op, {1, 2}, m_use_const_initializer, to); - if (unroll) { - auto inputs = prepare_inputs(op, 3, to); - cell = to.make(inputs[0], - inputs[1], - inputs[2], - inputs[3], - inputs[4], - inputs[5], - lstm_seq_v0->get_hidden_size(), - lstm_seq_v0->get_activations(), - lstm_seq_v0->get_activations_alpha(), - lstm_seq_v0->get_activations_beta(), - lstm_seq_v0->get_clip_threshold()); - } - OPENVINO_SUPPRESS_DEPRECATED_END - } else if (auto lstm_seq_v5 = std::dynamic_pointer_cast(op)) { + if (auto lstm_seq_v5 = std::dynamic_pointer_cast(op)) { unroll = need_unroll(op); new_assigns = replace_with_memory(op, {1, 2}, m_use_const_initializer, to); if (unroll) { diff --git a/src/core/src/preprocess/preprocess_impls.cpp b/src/core/src/preprocess/preprocess_impls.cpp index 13a4c6f1353312..cbe18a78beb575 100644 --- a/src/core/src/preprocess/preprocess_impls.cpp +++ b/src/core/src/preprocess/preprocess_impls.cpp @@ -385,13 +385,6 @@ void OutputInfo::OutputInfoImpl::build(ov::ResultVector& results) { std::to_string(result->get_input_source_output(0).get_index())); } - OPENVINO_SUPPRESS_DEPRECATED_START - const auto tensor_name = ov::descriptor::get_ov_tensor_legacy_name(result->get_input_tensor(0)); - if (!tensor_name.empty()) { - ov::descriptor::set_ov_tensor_legacy_name(node.get_tensor(), tensor_name); - } - OPENVINO_SUPPRESS_DEPRECATED_END - // Reset friendly name of input node to avoid names collision // when there is at a new node inserted by post-processing steps // If no new nodes are inserted by post-processing, then we need to preserve friendly name of input diff --git a/src/core/src/type/element_type.cpp b/src/core/src/type/element_type.cpp index c0fad90285e3f6..3e664335033d80 100644 --- a/src/core/src/type/element_type.cpp +++ b/src/core/src/type/element_type.cpp @@ -164,51 +164,6 @@ std::vector ov::element::Type::get_known_types() { return rc; } -ov::element::Type::Type(size_t bitwidth, - bool is_real, - bool is_signed, - bool is_quantized, - const std::string& /* cname */) { - const ElementTypes::ElementsMap elements_map{ - {ov::element::Type_t::undefined, - {std::numeric_limits::max(), false, false, false, "undefined", "undefined"}}, - {ov::element::Type_t::dynamic, {0, false, false, false, "dynamic", "dynamic"}}, - {ov::element::Type_t::boolean, {8, false, true, false, "char", "boolean"}}, - {ov::element::Type_t::bf16, {16, true, true, false, "bfloat16", "bf16"}}, - {ov::element::Type_t::f16, {16, true, true, false, "float16", "f16"}}, - {ov::element::Type_t::f32, {32, true, true, false, "float", "f32"}}, - {ov::element::Type_t::f64, {64, true, true, false, "double", "f64"}}, - {ov::element::Type_t::i4, {4, false, true, true, "int4_t", "i4"}}, - {ov::element::Type_t::i8, {8, false, true, true, "int8_t", "i8"}}, - {ov::element::Type_t::i16, {16, false, true, false, "int16_t", "i16"}}, - {ov::element::Type_t::i32, {32, false, true, true, "int32_t", "i32"}}, - {ov::element::Type_t::i64, {64, false, true, false, "int64_t", "i64"}}, - {ov::element::Type_t::u1, {1, false, false, false, "uint1_t", "u1"}}, - {ov::element::Type_t::u2, {2, false, false, false, "uint2_t", "u2"}}, - {ov::element::Type_t::u3, {3, false, false, false, "uint3_t", "u3"}}, - {ov::element::Type_t::u4, {4, false, false, false, "uint4_t", "u4"}}, - {ov::element::Type_t::u6, {6, false, false, false, "uint6_t", "u6"}}, - {ov::element::Type_t::u8, {8, false, false, true, "uint8_t", "u8"}}, - {ov::element::Type_t::u16, {16, false, false, false, "uint16_t", "u16"}}, - {ov::element::Type_t::u32, {32, false, false, false, "uint32_t", "u32"}}, - {ov::element::Type_t::u64, {64, false, false, false, "uint64_t", "u64"}}, - {ov::element::Type_t::nf4, {4, false, false, true, "nfloat4", "nf4"}}, - {ov::element::Type_t::f8e4m3, {8, true, true, true, "f8e4m3", "f8e4m3"}}, - {ov::element::Type_t::f8e5m2, {8, true, true, true, "f8e5m2", "f8e5m2"}}, - {ov::element::Type_t::string, {8 * sizeof(std::string), false, false, false, "string", "string"}}, - {ov::element::Type_t::f4e2m1, {4, true, true, true, "f4e2m1", "f4e2m1"}}, - {ov::element::Type_t::f8e8m0, {4, true, true, true, "f8e8m0", "f8e8m0"}}, - }; - for (const auto& t : elements_map) { - const TypeInfo& info = t.second; - if (bitwidth == info.m_bitwidth && is_real == info.m_is_real && is_signed == info.m_is_signed && - is_quantized == info.m_is_quantized) { - m_type = t.first; - return; - } - } -} - ov::element::Type::Type(const std::string& type) : Type(type_from_string(type)) {} std::string ov::element::Type::c_type_string() const { diff --git a/src/core/tests/CMakeLists.txt b/src/core/tests/CMakeLists.txt index c3ed58783ac946..89acd7bd1809d0 100644 --- a/src/core/tests/CMakeLists.txt +++ b/src/core/tests/CMakeLists.txt @@ -18,6 +18,7 @@ set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/threading.cpp if(SUGGEST_OVERRIDE_SUPPORTED) set_source_files_properties(ov_tensor_test.cpp type_prop/multiclass_nms.cpp + type_prop/squeeze.cpp PROPERTIES COMPILE_OPTIONS -Wno-suggest-override) endif() diff --git a/src/core/tests/opset.cpp b/src/core/tests/opset.cpp index 3a2a590de43edf..65c79dc9432439 100644 --- a/src/core/tests/opset.cpp +++ b/src/core/tests/opset.cpp @@ -61,9 +61,9 @@ TEST_P(OpsetTests, opset_dump) { INSTANTIATE_TEST_SUITE_P(opset, OpsetTests, - testing::Values(OpsetTestParams{ov::get_opset1, 110}, - OpsetTestParams{ov::get_opset2, 112}, - OpsetTestParams{ov::get_opset3, 127}, + testing::Values(OpsetTestParams{ov::get_opset1, 109}, + OpsetTestParams{ov::get_opset2, 111}, + OpsetTestParams{ov::get_opset3, 126}, OpsetTestParams{ov::get_opset4, 137}, OpsetTestParams{ov::get_opset5, 145}, OpsetTestParams{ov::get_opset6, 152}, diff --git a/src/core/tests/type_prop/lstm_sequence.cpp b/src/core/tests/type_prop/lstm_sequence.cpp index 698206e25a5562..80da26340fd144 100644 --- a/src/core/tests/type_prop/lstm_sequence.cpp +++ b/src/core/tests/type_prop/lstm_sequence.cpp @@ -89,42 +89,6 @@ shared_ptr lstm_seq_direction_initialization(const recurre return lstm_sequence; } - -OPENVINO_SUPPRESS_DEPRECATED_START -shared_ptr lstm_seq_v0_tensor_initialization(const recurrent_sequence_parameters& param) { - auto batch_size = param.batch_size; - auto seq_length = param.seq_length; - auto input_size = param.input_size; - auto num_directions = param.num_directions; - auto hidden_size = param.hidden_size; - auto et = param.et; - - const auto X = make_shared(et, PartialShape{batch_size, seq_length, input_size}); - const auto initial_hidden_state = - make_shared(et, PartialShape{batch_size, num_directions, hidden_size}); - const auto initial_cell_state = - make_shared(et, PartialShape{batch_size, num_directions, hidden_size}); - const auto sequence_lengths = make_shared(et, PartialShape{batch_size}); - const auto W = make_shared(et, PartialShape{num_directions, hidden_size * 4, input_size}); - const auto R = make_shared(et, PartialShape{num_directions, hidden_size * 4, hidden_size}); - const auto B = make_shared(et, PartialShape{num_directions, hidden_size * 4}); - const auto P = make_shared(et, PartialShape{num_directions, hidden_size * 3}); - - const auto lstm_sequence = make_shared(); - - lstm_sequence->set_argument(0, X); - lstm_sequence->set_argument(1, initial_hidden_state); - lstm_sequence->set_argument(2, initial_cell_state); - lstm_sequence->set_argument(3, sequence_lengths); - lstm_sequence->set_argument(4, W); - lstm_sequence->set_argument(5, R); - lstm_sequence->set_argument(6, B); - lstm_sequence->set_argument(7, P); - - return lstm_sequence; -} -OPENVINO_SUPPRESS_DEPRECATED_END - } // namespace TEST(type_prop, lstm_sequence_forward) { @@ -173,56 +137,6 @@ TEST(type_prop, lstm_sequence_forward) { EXPECT_EQ(lstm_sequence->get_output_shape(2), (Shape{batch_size, num_directions, hidden_size})); } -TEST(type_prop, lstm_sequence_v0_forward) { - const size_t batch_size = 8; - const size_t num_directions = 1; - const size_t seq_length = 6; - const size_t input_size = 4; - const size_t hidden_size = 128; - - const auto X = make_shared(element::f32, Shape{batch_size, seq_length, input_size}); - const auto initial_hidden_state = - make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); - const auto initial_cell_state = - make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); - const auto sequence_lengths = make_shared(element::i32, Shape{batch_size}); - const auto W = make_shared(element::f32, Shape{num_directions, 4 * hidden_size, input_size}); - const auto R = make_shared(element::f32, Shape{num_directions, 4 * hidden_size, hidden_size}); - const auto B = make_shared(element::f32, Shape{num_directions, 4 * hidden_size}); - const auto P = make_shared(element::f32, Shape{num_directions, 3 * hidden_size}); - - const auto lstm_direction = op::RecurrentSequenceDirection::FORWARD; - - OPENVINO_SUPPRESS_DEPRECATED_START - const auto lstm_sequence = make_shared(X, - initial_hidden_state, - initial_cell_state, - sequence_lengths, - W, - R, - B, - P, - hidden_size, - lstm_direction); - - EXPECT_EQ(lstm_sequence->get_hidden_size(), hidden_size); - EXPECT_EQ(lstm_sequence->get_direction(), op::RecurrentSequenceDirection::FORWARD); - EXPECT_TRUE(lstm_sequence->get_activations_alpha().empty()); - EXPECT_TRUE(lstm_sequence->get_activations_beta().empty()); - EXPECT_EQ(lstm_sequence->get_activations()[0], "sigmoid"); - EXPECT_EQ(lstm_sequence->get_activations()[1], "tanh"); - EXPECT_EQ(lstm_sequence->get_activations()[2], "tanh"); - EXPECT_EQ(lstm_sequence->get_clip_threshold(), 0.f); - EXPECT_EQ(lstm_sequence->get_output_element_type(0), element::f32); - EXPECT_EQ(lstm_sequence->outputs().size(), 3); - EXPECT_EQ(lstm_sequence->get_output_shape(0), (Shape{batch_size, num_directions, seq_length, hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(1), element::f32); - EXPECT_EQ(lstm_sequence->get_output_shape(1), (Shape{batch_size, num_directions, hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(2), element::f32); - EXPECT_EQ(lstm_sequence->get_output_shape(2), (Shape{batch_size, num_directions, hidden_size})); - OPENVINO_SUPPRESS_DEPRECATED_END -} - TEST(type_prop, lstm_sequence_bidirectional) { const size_t batch_size = 24; const size_t num_directions = 2; @@ -273,68 +187,6 @@ TEST(type_prop, lstm_sequence_bidirectional) { EXPECT_EQ(lstm_sequence->get_output_shape(2), (Shape{batch_size, num_directions, hidden_size})); } -TEST(type_prop, lstm_sequence_v0_bidirectional) { - const size_t batch_size = 24; - const size_t num_directions = 2; - const size_t seq_length = 12; - const size_t input_size = 8; - const size_t hidden_size = 256; - const bool input_forget = true; - const ov::op::LSTMWeightsFormat weights_format = ov::op::LSTMWeightsFormat::FICO; - const float clip_threshold = 3.5f; - - const auto X = make_shared(element::f32, Shape{batch_size, seq_length, input_size}); - const auto initial_hidden_state = - make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); - const auto initial_cell_state = - make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); - const auto sequence_lengths = make_shared(element::i32, Shape{batch_size}); - const auto W = make_shared(element::f32, Shape{num_directions, 4 * hidden_size, input_size}); - const auto R = make_shared(element::f32, Shape{num_directions, 4 * hidden_size, hidden_size}); - const auto B = make_shared(element::f32, Shape{num_directions, 4 * hidden_size}); - const auto P = make_shared(element::f32, Shape{num_directions, 3 * hidden_size}); - - const auto lstm_direction = opset5::LSTMSequence::direction::BIDIRECTIONAL; - const std::vector activations_alpha = {2.7f, 7.0f, 32.367f}; - const std::vector activations_beta = {0.0f, 5.49f, 6.0f}; - const std::vector activations = {"tanh", "sigmoid", "sigmoid"}; - - OPENVINO_SUPPRESS_DEPRECATED_START - const auto lstm_sequence = make_shared(X, - initial_hidden_state, - initial_cell_state, - sequence_lengths, - W, - R, - B, - hidden_size, - lstm_direction, - weights_format, - activations_alpha, - activations_beta, - activations, - clip_threshold, - input_forget); - - EXPECT_EQ(lstm_sequence->get_hidden_size(), hidden_size); - EXPECT_EQ(lstm_sequence->get_direction(), opset5::LSTMSequence::direction::BIDIRECTIONAL); - EXPECT_EQ(lstm_sequence->get_activations_alpha(), activations_alpha); - EXPECT_EQ(lstm_sequence->get_activations_beta(), activations_beta); - EXPECT_EQ(lstm_sequence->get_activations()[0], "tanh"); - EXPECT_EQ(lstm_sequence->get_activations()[1], "sigmoid"); - EXPECT_EQ(lstm_sequence->get_activations()[2], "sigmoid"); - EXPECT_EQ(lstm_sequence->get_clip_threshold(), 3.5f); - EXPECT_EQ(lstm_sequence->get_input_forget(), true); - EXPECT_EQ(lstm_sequence->get_weights_format(), ov::op::LSTMWeightsFormat::FICO); - EXPECT_EQ(lstm_sequence->get_output_element_type(0), element::f32); - EXPECT_EQ(lstm_sequence->get_output_shape(0), (Shape{batch_size, num_directions, seq_length, hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(1), element::f32); - EXPECT_EQ(lstm_sequence->get_output_shape(1), (Shape{batch_size, num_directions, hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(2), element::f32); - EXPECT_EQ(lstm_sequence->get_output_shape(2), (Shape{batch_size, num_directions, hidden_size})); - OPENVINO_SUPPRESS_DEPRECATED_END -} - TEST(type_prop, lstm_sequence_dynamic_batch_size) { recurrent_sequence_parameters param; @@ -573,214 +425,3 @@ TEST(type_prop, lstm_sequence_invalid_input_direction_num_mismatch) { check_error(op::RecurrentSequenceDirection::FORWARD, 2); check_error(op::RecurrentSequenceDirection::REVERSE, 2); } - -OPENVINO_SUPPRESS_DEPRECATED_START -TEST(type_prop, lstm_sequence_v0_dynamic_num_directions) { - recurrent_sequence_parameters param; - - param.batch_size = 24; - param.num_directions = Dimension::dynamic(); - param.seq_length = 12; - param.input_size = 8; - param.hidden_size = 256; - param.et = element::f32; - - auto lstm_sequence = lstm_seq_v0_tensor_initialization(param); - lstm_sequence->validate_and_infer_types(); - - EXPECT_EQ(lstm_sequence->get_output_partial_shape(0), - (PartialShape{param.batch_size, 1, param.seq_length, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(1), (PartialShape{param.batch_size, 1, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(2), (PartialShape{param.batch_size, 1, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(0), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(1), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(2), param.et); -} - -TEST(type_prop, lstm_sequence_v0_dynamic_seq_length) { - recurrent_sequence_parameters param; - - param.batch_size = 24; - param.num_directions = 1; - param.seq_length = Dimension::dynamic(); - param.input_size = 8; - param.hidden_size = 256; - param.et = element::f32; - - auto lstm_sequence = lstm_seq_v0_tensor_initialization(param); - lstm_sequence->validate_and_infer_types(); - - EXPECT_EQ(lstm_sequence->get_output_partial_shape(0), - (PartialShape{param.batch_size, param.num_directions, param.seq_length, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(1), - (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(2), - (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(0), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(1), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(2), param.et); -} - -TEST(type_prop, lstm_sequence_v0_dynamic_hidden_size) { - recurrent_sequence_parameters param; - - param.batch_size = 24; - param.num_directions = 1; - param.seq_length = 12; - param.input_size = 8; - param.hidden_size = Dimension::dynamic(); - param.et = element::f32; - - auto lstm_sequence = lstm_seq_v0_tensor_initialization(param); - lstm_sequence->validate_and_infer_types(); - - EXPECT_EQ(lstm_sequence->get_output_partial_shape(0), - (PartialShape{param.batch_size, param.num_directions, param.seq_length, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(1), - (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(2), - (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(0), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(1), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(2), param.et); -} - -TEST(type_prop, lstm_sequence_v0_dynamic_inputs) { - recurrent_sequence_parameters param; - - param.batch_size = Dimension::dynamic(); - param.input_size = Dimension::dynamic(); - param.hidden_size = Dimension::dynamic(); - param.num_directions = Dimension::dynamic(); - param.seq_length = Dimension::dynamic(); - param.et = element::f32; - - auto lstm_sequence = lstm_seq_v0_tensor_initialization(param); - lstm_sequence->validate_and_infer_types(); - - EXPECT_EQ(lstm_sequence->get_output_partial_shape(0), - (PartialShape{param.batch_size, 1, param.seq_length, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(1), (PartialShape{param.batch_size, 1, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_partial_shape(2), (PartialShape{param.batch_size, 1, param.hidden_size})); - EXPECT_EQ(lstm_sequence->get_output_element_type(0), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(1), param.et); - EXPECT_EQ(lstm_sequence->get_output_element_type(2), param.et); -} - -TEST(type_prop, lstm_sequence_v0_invalid_input_dimension) { - recurrent_sequence_parameters param; - - param.batch_size = 24; - param.num_directions = 2; - param.seq_length = 12; - param.input_size = 8; - param.hidden_size = 256; - param.et = element::f32; - - auto lstm_sequence = lstm_seq_v0_tensor_initialization(param); - auto invalid_rank0_tensor = make_shared(param.et, PartialShape{}); - - // Validate invalid rank0 tensor for all inputs: X, initial_hidden_state, initial_cell_state W, - // R, B - for (size_t i = 0; i < lstm_sequence->get_input_size(); i++) { - lstm_sequence = lstm_seq_v0_tensor_initialization(param); - lstm_sequence->set_argument(i, invalid_rank0_tensor); - ASSERT_THROW(lstm_sequence->validate_and_infer_types(), ov::AssertFailure) - << "LSTMSequence node was created with invalid data."; - } -} - -TEST(type_prop, lstm_sequence_v0_input_dynamic_rank) { - recurrent_sequence_parameters param; - param.batch_size = 24; - param.num_directions = 1; - param.seq_length = 12; - param.input_size = 8; - param.hidden_size = 256; - param.et = element::f32; - - auto op = lstm_seq_tensor_initialization(param); - auto dynamic_tensor = make_shared(param.et, PartialShape::dynamic(Rank::dynamic())); - - for (size_t i = 0; i < op->get_input_size(); i++) { - auto op = lstm_seq_v0_tensor_initialization(param); - op->set_argument(i, dynamic_tensor); - op->validate_and_infer_types(); - if (i == 0) { // X input - EXPECT_EQ(op->get_output_partial_shape(0), - (PartialShape{param.batch_size, param.num_directions, -1, param.hidden_size})); - } else { - EXPECT_EQ(op->get_output_partial_shape(0), - (PartialShape{param.batch_size, param.num_directions, param.seq_length, param.hidden_size})); - } - EXPECT_EQ(op->get_output_partial_shape(1), - (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); - EXPECT_EQ(op->get_output_partial_shape(2), - (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); - - EXPECT_EQ(op->get_output_element_type(0), param.et); - EXPECT_EQ(op->get_output_element_type(1), param.et); - } -} - -TEST(type_prop, lstm_sequence_v0_invalid_input_direction) { - recurrent_sequence_parameters param; - - param.batch_size = 24; - param.num_directions = 3; - param.seq_length = 12; - param.input_size = 8; - param.hidden_size = 256; - param.et = element::f32; - - auto lstm_sequence = lstm_seq_v0_tensor_initialization(param); - try { - lstm_sequence->validate_and_infer_types(); - FAIL() << "LSTMSequence node was created with invalid data."; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING( - error.what(), - std::string("Dimension `num_directions` doesn't match to other inputs or `direction` attribute")); - } - - param.num_directions = 2; // 2 is also not allowed for default 'm_direction' = FORWARD - lstm_sequence = lstm_seq_v0_tensor_initialization(param); - try { - lstm_sequence->validate_and_infer_types(); - FAIL() << "LSTMSequence node was created with invalid data."; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING( - error.what(), - std::string("Dimension `num_directions` doesn't match to other inputs or `direction` attribute")); - } -} - -TEST(type_prop, lstm_sequence_v0_invalid_input_P) { - recurrent_sequence_parameters param; - - param.batch_size = 24; - param.num_directions = 1; - param.seq_length = 12; - param.input_size = 8; - param.hidden_size = 256; - param.et = element::f32; - - auto lstm_sequence = lstm_seq_v0_tensor_initialization(param); - auto P = make_shared(element::f32, PartialShape{param.hidden_size * 5}); - lstm_sequence->set_argument(7, P); - try { - lstm_sequence->validate_and_infer_types(); - FAIL() << "LSTMSequence node was created with invalid data."; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Input tensor P should have rank equal 2"); - } - P = make_shared(element::f32, PartialShape{param.num_directions, param.hidden_size * 5}); - lstm_sequence->set_argument(7, P); - try { - lstm_sequence->validate_and_infer_types(); - FAIL() << "LSTMSequence node was created with invalid data."; - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Inorrect shape of P input. Second dimension is: 1280, expected: 768"); - } - OPENVINO_SUPPRESS_DEPRECATED_END -} diff --git a/src/core/tests/type_prop/rnn_seq_base.cpp b/src/core/tests/type_prop/rnn_seq_base.cpp index 9b3c1b7f8f25df..db38622333d67f 100644 --- a/src/core/tests/type_prop/rnn_seq_base.cpp +++ b/src/core/tests/type_prop/rnn_seq_base.cpp @@ -83,11 +83,7 @@ class RNNSeqBaseTest : public TypePropOpTest { return std::make_shared(X, H_t, sequence_lengths, W, R, B, p.hidden_size.get_max_length(), p.direction); } - OPENVINO_SUPPRESS_DEPRECATED_START - template < - typename T = TOp, - typename std::enable_if::value || std::is_same::value, - bool>::type = true> + template ::value, bool>::type = true> std::shared_ptr make_rnn_seq_based_op(RNNSeqParams& p, bool use_default_ctor = false) { p.gates_count = 4; p.outputs_size = 3; @@ -109,19 +105,12 @@ class RNNSeqBaseTest : public TypePropOpTest { op->set_direction(p.direction); op->set_hidden_size(p.hidden_size.get_max_length()); auto inputs = OutputVector{X, H_t, C_t, sequence_lengths, W, R, B}; - if (ov::is_type(op)) { - const auto P = - make_shared(p.et, - PartialShape{p.num_directions, p.hidden_size * (p.gates_count - 1)}); - inputs.push_back(P); - } op->set_arguments(inputs); op->validate_and_infer_types(); return op; } return std::make_shared(X, H_t, C_t, sequence_lengths, W, R, B, p.hidden_size.get_max_length(), p.direction); } - OPENVINO_SUPPRESS_DEPRECATED_END }; TYPED_TEST_SUITE_P(RNNSeqBaseTest); @@ -284,9 +273,7 @@ REGISTER_TYPED_TEST_SUITE_P(RNNSeqBaseTest, interval_symbols_dims_shape_infer_REVERSE, interval_symbols_dims_shape_infer_BIDIRECTIONAL); -OPENVINO_SUPPRESS_DEPRECATED_START -using RNNSeqBaseTypes = Types; -OPENVINO_SUPPRESS_DEPRECATED_END +using RNNSeqBaseTypes = Types; INSTANTIATE_TYPED_TEST_SUITE_P(type_prop, RNNSeqBaseTest, RNNSeqBaseTypes); } // namespace rnn_seq_test diff --git a/src/core/tests/type_prop/squeeze.cpp b/src/core/tests/type_prop/squeeze.cpp index c7d81fd97c2786..7be05de1876d9f 100644 --- a/src/core/tests/type_prop/squeeze.cpp +++ b/src/core/tests/type_prop/squeeze.cpp @@ -7,193 +7,261 @@ #include "common_test_utils/test_assertions.hpp" #include "common_test_utils/type_prop.hpp" #include "openvino/op/broadcast.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/gather.hpp" -#include "openvino/op/shape_of.hpp" -#include "openvino/op/unsqueeze.hpp" #include "sequence_generator.hpp" using namespace std; using namespace ov; using namespace testing; -TEST(type_prop, squeeze_axes_invalid_value) { +namespace { + +template +class SqueezelOperator : public TypePropOpTest {}; + +using SqueezeTypes = ::testing::Types; + +TYPED_TEST_SUITE(SqueezelOperator, SqueezeTypes); + +TYPED_TEST(SqueezelOperator, squeeze_axes_invalid_value) { auto param = make_shared(element::f32, Shape{1, 2, 3, 4}); auto axes_node = make_shared(element::u64, Shape{2}, vector{0, 2}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = this->make_op(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), (PartialShape{2, 3, 4})); } -TEST(type_prop, squeeze_single_input) { +TYPED_TEST(SqueezelOperator, squeeze_single_input) { auto param = make_shared(element::f32, PartialShape{1, -1, 3, 4}); - auto s = make_shared(param); - EXPECT_EQ(s->get_output_partial_shape(0), PartialShape::dynamic()); + const auto squeeze = this->make_op(param); + EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); } -TEST(type_prop, squeeze_axes_invalid_rank) { +TYPED_TEST(SqueezelOperator, squeeze_axes_invalid_rank) { auto param = make_shared(element::f32, Shape{1, 2, 3, 4}); auto axes_node = make_shared(element::i32, Shape{2, 1}, vector{0, 2}); - OV_EXPECT_THROW(auto s = make_shared(param, axes_node), + OV_EXPECT_THROW(const auto squeeze = this->make_op(param, axes_node), NodeValidationFailure, HasSubstr("Second input (axes) should not be of rank higher than 1.")); } -TEST(type_prop, squeeze_incorrect_negative_axes) { +TYPED_TEST(SqueezelOperator, squeeze_incorrect_negative_axes) { auto param = make_shared(element::f32, Shape{1, 4, 1, 4, 1, 8}); auto axes_node = make_shared(element::i64, Shape{2}, vector{-6, -10}); - OV_EXPECT_THROW(auto s = make_shared(param, axes_node), + OV_EXPECT_THROW(const auto squeeze = this->make_op(param, axes_node), ov::Exception, HasSubstr("Axis -10 out of the tensor rank range")); } -TEST(type_prop, squeeze_data_static_param_axes_1D_single_elem_static_shape_no_squeezable_dims) { +TYPED_TEST(SqueezelOperator, squeeze_data_static_param_axes_1D_single_elem_static_shape_no_squeezable_dims) { auto param = std::make_shared(ov::element::f32, PartialShape{2, 2, 4}); const auto axes_node = std::make_shared(element::u64, PartialShape{1}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = this->make_op(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), (PartialShape{2, 2, 4})); } -TEST(type_prop, squeeze_data_static_param_axes_1D_two_elem_static_shape_squeezable_dims_two) { +TYPED_TEST(SqueezelOperator, squeeze_data_static_param_axes_1D_two_elem_static_shape_squeezable_dims_two) { auto param = std::make_shared(ov::element::f32, PartialShape{1, 2, 1, 4}); const auto axes_node = std::make_shared(element::u64, PartialShape{2}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = this->make_op(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); } -TEST(type_prop, squeeze_data_static_param_axes_1D_two_elem_static_shape_squeezable_dims_one) { +TYPED_TEST(SqueezelOperator, squeeze_data_static_param_axes_1D_two_elem_static_shape_squeezable_dims_one) { auto param = std::make_shared(ov::element::f32, PartialShape{2, 1, 4}); const auto axes_node = std::make_shared(element::u64, PartialShape{2}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = this->make_op(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); } -TEST(type_prop, squeeze_data_static_param_axes_1D_single_elem_static_shape_squeezable_dims_one) { +TEST(TypePropSqueezelOperatorV0, squeeze_data_static_param_axes_1D_single_elem_static_shape_squeezable_dims_one) { auto param = std::make_shared(ov::element::f32, PartialShape{2, 1, 4}); const auto axes_node = std::make_shared(element::u64, PartialShape{1}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = std::make_shared(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic(2)); } -TEST(type_prop, squeeze_data_static_param_axes_scalar_static_shape_squeezable_dims_one) { +TEST(TypePropSqueezelOperatorV15, squeeze_data_static_param_axes_1D_single_elem_static_shape_squeezable_dims_one) { + auto param = std::make_shared(ov::element::f32, PartialShape{2, 1, 4}); + const auto axes_node = std::make_shared(element::u64, PartialShape{1}); + const auto squeeze = std::make_shared(param, axes_node); + + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); +} + +TEST(TypePropSqueezelOperatorV0, squeeze_data_static_param_axes_scalar_static_shape_squeezable_dims_one) { auto param = std::make_shared(ov::element::f32, PartialShape{2, 1, 4}); const auto axes_node = std::make_shared(element::u64, PartialShape{}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = std::make_shared(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic(2)); } -TEST(type_prop, squeeze_data_scalar_param_axes_1D_single_elem_static_shape) { +TEST(TypePropSqueezelOperatorV15, squeeze_data_static_param_axes_scalar_static_shape_squeezable_dims_one) { + auto param = std::make_shared(ov::element::f32, PartialShape{2, 1, 4}); + const auto axes_node = std::make_shared(element::u64, PartialShape{}); + const auto squeeze = std::make_shared(param, axes_node); + + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); +} + +TYPED_TEST(SqueezelOperator, squeeze_data_scalar_param_axes_1D_single_elem_static_shape) { auto param = std::make_shared(ov::element::f32, PartialShape{}); const auto axes_node = std::make_shared(element::u64, PartialShape{1}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = this->make_op(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); } -TEST(type_prop, squeeze_data_dynamic_param_axes_1D_two_elem_static_shape_squeezable_dims_equal) { +TYPED_TEST(SqueezelOperator, squeeze_data_dynamic_param_axes_1D_two_elem_static_shape_squeezable_dims_equal) { auto param = std::make_shared(ov::element::f32, PartialShape{-1, {2, 8}, {1, 3}, {4, -1}}); const auto axes_node = std::make_shared(element::u64, PartialShape{2}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = this->make_op(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); } -TEST(type_prop, squeeze_data_static_param_axes_1D_two_elem_static_shape_squeezable_dims_more) { +TYPED_TEST(SqueezelOperator, squeeze_data_static_param_axes_1D_two_elem_static_shape_squeezable_dims_more) { auto param = std::make_shared(ov::element::f32, PartialShape{1, 2, 1, 3, 1}); const auto axes_node = std::make_shared(element::u64, PartialShape{2}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = this->make_op(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); } -TEST(type_prop, squeeze_data_static_param_axes_1D_single_elem_static_shape_squeezable_dims_more) { +TEST(TypePropSqueezelOperatorV0, squeeze_data_static_param_axes_1D_single_elem_static_shape_squeezable_dims_more) { auto param = std::make_shared(ov::element::f32, PartialShape{1, 2, 1, 3, 1}); const auto axes_node = std::make_shared(element::u64, PartialShape{1}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = std::make_shared(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic(4)); } -TEST(type_prop, squeeze_data_static_param_axes_scalar_static_shape_squeezable_dims_more) { +TEST(TypePropSqueezelOperatorV15, squeeze_data_static_param_axes_1D_single_elem_static_shape_squeezable_dims_more) { + auto param = std::make_shared(ov::element::f32, PartialShape{1, 2, 1, 3, 1}); + const auto axes_node = std::make_shared(element::u64, PartialShape{1}); + const auto squeeze = std::make_shared(param, axes_node); + + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); +} + +TEST(TypePropSqueezelOperatorV0, squeeze_data_static_param_axes_scalar_static_shape_squeezable_dims_more) { auto param = std::make_shared(ov::element::f32, PartialShape{1, 2, 1, 3, 1}); const auto axes_node = std::make_shared(element::u64, PartialShape{}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = std::make_shared(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic(4)); } -TEST(type_prop, squeeze_data_dynamic_param_axes_1D_two_elem_static_shape_squeezable_dims_more) { +TEST(TypePropSqueezelOperatorV15, squeeze_data_static_param_axes_scalar_static_shape_squeezable_dims_more) { + auto param = std::make_shared(ov::element::f32, PartialShape{1, 2, 1, 3, 1}); + const auto axes_node = std::make_shared(element::u64, PartialShape{}); + const auto squeeze = std::make_shared(param, axes_node); + + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); +} + +TYPED_TEST(SqueezelOperator, squeeze_data_dynamic_param_axes_1D_two_elem_static_shape_squeezable_dims_more) { auto param = std::make_shared(ov::element::f32, PartialShape{-1, {2, 8}, {1, 3}, {4, -1}}); const auto axes_node = std::make_shared(element::u64, PartialShape{2}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = this->make_op(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); } -TEST(type_prop, squeeze_data_dynamic_param_axes_1D_single_elem_static_shape_squeezable_dims_more) { +TEST(TypePropSqueezelOperatorV0, squeeze_data_dynamic_param_axes_1D_single_elem_static_shape_squeezable_dims_more) { auto param = std::make_shared(ov::element::f32, PartialShape{-1, {2, 8}, {1, 3}, {4, -1}}); const auto axes_node = std::make_shared(element::u64, PartialShape{1}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = std::make_shared(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic(3)); } -TEST(type_prop, squeeze_data_dynamic_param_axes_scalar_static_shape_squeezable_dims_more) { +TEST(TypePropSqueezelOperatorV15, squeeze_data_dynamic_param_axes_1D_single_elem_static_shape_squeezable_dims_more) { + auto param = std::make_shared(ov::element::f32, PartialShape{-1, {2, 8}, {1, 3}, {4, -1}}); + const auto axes_node = std::make_shared(element::u64, PartialShape{1}); + const auto squeeze = std::make_shared(param, axes_node); + + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); +} + +TEST(TypePropSqueezelOperatorV0, squeeze_data_dynamic_param_axes_scalar_static_shape_squeezable_dims_more) { auto param = std::make_shared(ov::element::f32, PartialShape{-1, {2, 8}, {1, 3}, {4, -1}}); const auto axes_node = std::make_shared(element::u64, PartialShape{}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = std::make_shared(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic(3)); } -TEST(type_prop, squeeze_data_dyamic_param_axes_1D_two_elem_static_shape_squeezable_dims_one) { +TEST(TypePropSqueezelOperatorV15, squeeze_data_dynamic_param_axes_scalar_static_shape_squeezable_dims_more) { + auto param = std::make_shared(ov::element::f32, PartialShape{-1, {2, 8}, {1, 3}, {4, -1}}); + const auto axes_node = std::make_shared(element::u64, PartialShape{}); + const auto squeeze = std::make_shared(param, axes_node); + + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); +} + +TYPED_TEST(SqueezelOperator, squeeze_data_dyamic_param_axes_1D_two_elem_static_shape_squeezable_dims_one) { auto param = std::make_shared(ov::element::f32, PartialShape{2, -1, 4}); const auto axes_node = std::make_shared(element::u64, PartialShape{2}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = this->make_op(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); } -TEST(type_prop, squeeze_data_dynamic_param_axes_1D_three_elem_static_shape_squeezable_dims_two) { +TYPED_TEST(SqueezelOperator, squeeze_data_dynamic_param_axes_1D_three_elem_static_shape_squeezable_dims_two) { auto param = std::make_shared(ov::element::f32, PartialShape{-1, {2, 8}, {1, 3}, {4, -1}}); const auto axes_node = std::make_shared(element::u64, PartialShape{3}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = this->make_op(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); } -TEST(type_prop, squeeze_data_dynamic_param_axes_1D_single_elem_static_shape_squeezable_dims_less) { +TEST(TypePropSqueezelOperatorV0, squeeze_data_dynamic_param_axes_1D_single_elem_static_shape_squeezable_dims_less) { auto param = std::make_shared(ov::element::f32, PartialShape{-1, {2, 8}, {1, 3}, {4, -1}}); const auto axes_node = std::make_shared(element::u64, PartialShape{1}); - const auto squeeze = std::make_shared(param, axes_node); + const auto squeeze = std::make_shared(param, axes_node); EXPECT_EQ(squeeze->get_element_type(), element::f32); EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic(3)); } +TEST(TypePropSqueezelOperatorV15, squeeze_data_dynamic_param_axes_1D_single_elem_static_shape_squeezable_dims_less) { + auto param = std::make_shared(ov::element::f32, PartialShape{-1, {2, 8}, {1, 3}, {4, -1}}); + const auto axes_node = std::make_shared(element::u64, PartialShape{1}); + const auto squeeze = std::make_shared(param, axes_node); + + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); +} + using SqueezeTypePropTestParam = std::tuple, // Squeeze axis PartialShape // Expected shape @@ -288,26 +356,44 @@ INSTANTIATE_TEST_SUITE_P(type_prop_shrink_shape_default_axes, TEST_P(SqueezeTest, partial_shape_dimension_propagation_const_axis_i32) { const auto axes_node = std::make_shared(element::i32, Shape{axes.size()}, axes); - const auto squeeze = std::make_shared(param, axes_node); - - EXPECT_EQ(squeeze->get_element_type(), element::f32); - EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + { + const auto squeeze = std::make_shared(param, axes_node); + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + } + { + const auto squeeze = std::make_shared(param, axes_node); + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + } } TEST_P(SqueezeTest, partial_shape_dimension_propagation_parameter_axes_no_data) { const auto axes_node = std::make_shared(element::u64, PartialShape{Shape{axes.size()}}); - const auto squeeze = std::make_shared(param, axes_node); - - EXPECT_EQ(squeeze->get_element_type(), element::f32); - EXPECT_TRUE(squeeze->get_output_partial_shape(0).compatible(exp_shape)); + { + const auto squeeze = std::make_shared(param, axes_node); + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_TRUE(squeeze->get_output_partial_shape(0).compatible(exp_shape)); + } + { + const auto squeeze = std::make_shared(param, axes_node); + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_TRUE(squeeze->get_output_partial_shape(0).compatible(exp_shape)); + } } TEST_P(SqueezeTest, partial_shape_dimension_propagation_dynamic_axes) { const auto axes_node = std::make_shared(element::u64, PartialShape::dynamic()); - const auto squeeze = std::make_shared(param, axes_node); - - EXPECT_EQ(squeeze->get_element_type(), element::f32); - EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); + { + const auto squeeze = std::make_shared(param, axes_node); + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); + } + { + const auto squeeze = std::make_shared(param, axes_node); + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), PartialShape::dynamic()); + } } TEST_P(SqueezeTest, symbols_propagation) { @@ -321,9 +407,14 @@ TEST_P(SqueezeTest, symbols_propagation) { param = make_shared(element::f32, p_shape); const auto axes_node = std::make_shared(element::i32, Shape{axes.size()}, axes); - const auto squeeze = std::make_shared(param, axes_node); - - EXPECT_EQ(get_shape_symbols(squeeze->get_output_partial_shape(0)), exp_symbols); + { + const auto squeeze = std::make_shared(param, axes_node); + EXPECT_EQ(get_shape_symbols(squeeze->get_output_partial_shape(0)), exp_symbols); + } + { + const auto squeeze = std::make_shared(param, axes_node); + EXPECT_EQ(get_shape_symbols(squeeze->get_output_partial_shape(0)), exp_symbols); + } } using SqueezeShapeTests = SqueezeTest; @@ -336,10 +427,16 @@ INSTANTIATE_TEST_SUITE_P(type_prop_shrink_shape_no_axes, TEST_P(SqueezeShapeTests, shape_dimension_propagation_const_axis_i64) { param = std::make_shared(element::f64, p_shape.to_shape()); const auto axes_node = std::make_shared(element::i64, Shape{axes.size()}, axes); - const auto squeeze = std::make_shared(param, axes_node); - - EXPECT_EQ(squeeze->get_element_type(), element::f64); - EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape.to_shape()); + { + const auto squeeze = std::make_shared(param, axes_node); + EXPECT_EQ(squeeze->get_element_type(), element::f64); + EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape.to_shape()); + } + { + const auto squeeze = std::make_shared(param, axes_node); + EXPECT_EQ(squeeze->get_element_type(), element::f64); + EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape.to_shape()); + } } using SqueezeNoAxesTest = SqueezeTest; @@ -350,10 +447,16 @@ INSTANTIATE_TEST_SUITE_P(type_prop_shrink_shape_no_axes, PrintToStringParamName()); TEST_P(SqueezeNoAxesTest, partial_shape_dimension_propagation_no_axes) { - const auto squeeze = std::make_shared(param); - - EXPECT_EQ(squeeze->get_element_type(), element::f32); - EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + { + const auto squeeze = std::make_shared(param); + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + } + { + const auto squeeze = std::make_shared(param); + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + } } using SqueezeScalarAxisTest = SqueezeTest; @@ -368,25 +471,35 @@ INSTANTIATE_TEST_SUITE_P( TEST_P(SqueezeScalarAxisTest, axis_value_as_vector) { const auto axes_node = std::make_shared(element::i32, Shape{}, axes); - const auto squeeze = std::make_shared(param, axes_node); - - EXPECT_EQ(squeeze->get_element_type(), element::f32); - EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + { + const auto squeeze = std::make_shared(param, axes_node); + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + } + { + const auto squeeze = std::make_shared(param, axes_node); + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + } } TEST_P(SqueezeScalarAxisTest, axis_value_as_integer) { const auto axes_node = std::make_shared(element::i32, Shape{}, axes.front()); - const auto squeeze = std::make_shared(param, axes_node); - - EXPECT_EQ(squeeze->get_element_type(), element::f32); - EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + { + const auto squeeze = std::make_shared(param, axes_node); + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + } + { + const auto squeeze = std::make_shared(param, axes_node); + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + } } using SqueezeBoundTest = UnSqueezeBoundTest; -INSTANTIATE_TEST_SUITE_P( - type_prop_bounds_propagate, - SqueezeBoundTest, +const auto test_values_in = Values(std::make_tuple(PartialShape::dynamic(6), PartialShape::dynamic(1)), std::make_tuple(PartialShape{Dimension(-1)}, PartialShape{Dimension(-1)}), std::make_tuple(PartialShape{Dimension::dynamic(), 8}, PartialShape{Dimension::dynamic()}), @@ -394,34 +507,136 @@ INSTANTIATE_TEST_SUITE_P( std::make_tuple(PartialShape{Dimension(20, -1), Dimension::dynamic()}, PartialShape{{20, -1}}), std::make_tuple(PartialShape{Dimension(-1, 5), Dimension::dynamic()}, PartialShape{Dimension(-1, 5)}), std::make_tuple(PartialShape{15}, PartialShape{15}), - std::make_tuple(PartialShape{2, 6}, PartialShape{2})), - PrintToStringParamName()); + std::make_tuple(PartialShape{2, 6}, PartialShape{2})); + +INSTANTIATE_TEST_SUITE_P(type_prop_bounds_propagate, SqueezeBoundTest, test_values_in, PrintToStringParamName()); /** * \brief Check symbol and dynamic value propagation. * * Test use evaluate symbol, lower/upper. */ -TEST_P(SqueezeBoundTest, propagate_symbol_and_dynamic_value) { +TEST_P(SqueezeBoundTest, propagate_symbol_and_dynamic_value_squeeze_v0) { PartialShape symboled_shape = PartialShape{p_shape}; in_symbols = set_shape_symbols(symboled_shape); - constexpr auto et = element::i64; - const auto symboled_param = std::make_shared(et, symboled_shape); - const auto symboled_shape_of = std::make_shared(symboled_param); + const auto squeeze = create_squeeze(symboled_shape); + const auto bc = std::make_shared(param, squeeze); + + EXPECT_EQ(bc->get_output_partial_shape(0), exp_shape); + const auto symbols = get_shape_symbols(bc->get_output_partial_shape(0)); + EXPECT_THAT(symbols, ElementsAre(in_symbols.front())); +} - const auto zero = std::vector{0}; - const auto axis = std::make_shared(et, Shape{}, zero); - const auto indices = std::make_shared(et, Shape{}, zero); - const auto gather = std::make_shared(symboled_shape_of, indices, axis); - const auto axis_1 = std::make_shared(et, Shape{2}, std::vector{0, 1}); - const auto unsqueeze = std::make_shared(gather, axis_1); - const auto squeeze = std::make_shared(unsqueeze, axis); +/** + * \brief Check symbol and dynamic value propagation. + * + * Test use evaluate symbol, lower/upper. + */ +TEST_P(SqueezeBoundTest, propagate_symbol_and_dynamic_value_squeeze_v15) { + PartialShape symboled_shape = PartialShape{p_shape}; + + in_symbols = set_shape_symbols(symboled_shape); + const auto squeeze = create_squeeze(symboled_shape); const auto bc = std::make_shared(param, squeeze); EXPECT_EQ(bc->get_output_partial_shape(0), exp_shape); const auto symbols = get_shape_symbols(bc->get_output_partial_shape(0)); EXPECT_THAT(symbols, ElementsAre(in_symbols.front())); } + +using SqueezeAxesDynamicRankTestParam = decltype(std::tuple_cat(SqueezeTypePropTestParam{}, std::make_tuple(false))); +class SqueezeAxesDynamicRank : public ::testing::TestWithParam { +protected: + ov::PartialShape p_shape{}, exp_shape{}; + std::vector axes{}; + bool allow_axis_skip{}; +}; + +INSTANTIATE_TEST_SUITE_P( + SqueezeAxesDynamicRankTests, + SqueezeAxesDynamicRank, + ::testing::Values( + std::make_tuple(PartialShape{1, 2, -1, 4}, std::vector{}, PartialShape::dynamic(), false), + std::make_tuple(PartialShape{1, 2, -1, 4}, std::vector{}, PartialShape::dynamic(), true), + + std::make_tuple(PartialShape{1, 2, -1, 4}, std::vector{0}, PartialShape{2, -1, 4}, false), + std::make_tuple(PartialShape{1, 2, -1, 4}, std::vector{0}, PartialShape{2, -1, 4}, true), + + std::make_tuple(PartialShape{1, 2, -1, 4}, std::vector{2}, PartialShape{1, 2, 4}, false), + std::make_tuple(PartialShape{1, 2, -1, 4}, std::vector{2}, PartialShape::dynamic(), true), + + std::make_tuple(PartialShape{1, 2, -1, 4}, std::vector{0, 2}, PartialShape{2, 4}, false), + std::make_tuple(PartialShape{1, 2, -1, 4}, std::vector{0, 2}, PartialShape::dynamic(), true), + + std::make_tuple(PartialShape{1, 2, -1, 4}, std::vector{1}, PartialShape{1, 2, -1, 4}, false), + std::make_tuple(PartialShape{1, 2, -1, 4}, std::vector{1}, PartialShape{1, 2, -1, 4}, true), + + std::make_tuple(PartialShape{2, 4}, std::vector{1}, PartialShape{2, 4}, false), + std::make_tuple(PartialShape{2, 4}, std::vector{1}, PartialShape{2, 4}, true), + + std::make_tuple(PartialShape{2, {3, 5}}, std::vector{}, PartialShape{2, {3, 5}}, false), + std::make_tuple(PartialShape{2, {3, 5}}, std::vector{}, PartialShape{2, {3, 5}}, true), + + std::make_tuple(PartialShape{1, 2, -1}, std::vector{0, 1}, PartialShape{2, -1}, false), + std::make_tuple(PartialShape{1, 2, -1}, std::vector{0, 1}, PartialShape{2, -1}, true), + + std::make_tuple(PartialShape{1, 2, -1}, std::vector{1}, PartialShape{1, 2, -1}, false), + std::make_tuple(PartialShape{1, 2, -1}, std::vector{1}, PartialShape{1, 2, -1}, true), + + std::make_tuple(PartialShape{1, 1, -1}, std::vector{0, 1}, PartialShape{-1}, false), + std::make_tuple(PartialShape{1, 1, -1}, std::vector{0, 1}, PartialShape{-1}, true), + + std::make_tuple(PartialShape{1, 1, -1}, std::vector{1}, PartialShape{1, -1}, false), + std::make_tuple(PartialShape{1, 1, -1}, std::vector{1}, PartialShape{1, -1}, true), + + std::make_tuple(PartialShape{1, 2, 3}, std::vector{}, PartialShape{2, 3}, false), + std::make_tuple(PartialShape{1, 2, 3}, std::vector{}, PartialShape{2, 3}, true))); + +TEST_P(SqueezeAxesDynamicRank, squeeze_axes_dynamic_rank_param) { + const auto& params = GetParam(); + p_shape = std::get<0>(params); + axes = std::get<1>(params); + exp_shape = std::get<2>(params); + allow_axis_skip = std::get<3>(params); + + auto param = make_shared(element::f32, p_shape); + auto axes_node = make_shared(element::u64, Shape{axes.size()}, axes); + const auto squeeze = std::make_shared(param, axes_node, allow_axis_skip); + + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + EXPECT_EQ(squeeze->get_allow_axis_skip(), allow_axis_skip); +} + +TEST(SqueezeDynamicAxis, squeeze_dynamic_non_const_single_axis) { + auto p_shape = PartialShape{1, 2, -1, 4}; + auto exp_shape = PartialShape::dynamic(); + auto allow_axis_skip = true; + + auto param = make_shared(element::f32, p_shape); + auto axes_node = make_shared(element::i32, Shape{1}); + const auto squeeze = std::make_shared(param, axes_node, allow_axis_skip); + + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + EXPECT_EQ(squeeze->get_allow_axis_skip(), allow_axis_skip); +} + +TEST(SqueezeDynamicAxis, squeeze_dynamic_non_const_axes) { + auto p_shape = PartialShape{1, 2, -1, 4}; + auto exp_shape = PartialShape::dynamic(); + auto allow_axis_skip = true; + + auto param = make_shared(element::f32, p_shape); + auto axes_node = make_shared(element::i32, PartialShape{-1}); + const auto squeeze = std::make_shared(param, axes_node, allow_axis_skip); + + EXPECT_EQ(squeeze->get_element_type(), element::f32); + EXPECT_EQ(squeeze->get_output_partial_shape(0), exp_shape); + EXPECT_EQ(squeeze->get_allow_axis_skip(), allow_axis_skip); +} + +} // namespace diff --git a/src/core/tests/type_prop/stft.cpp b/src/core/tests/type_prop/stft.cpp index 702b000d2dd560..2969af4e5a43bd 100644 --- a/src/core/tests/type_prop/stft.cpp +++ b/src/core/tests/type_prop/stft.cpp @@ -104,6 +104,13 @@ INSTANTIATE_TEST_SUITE_P( type_prop_stft_shape, TypePropSTFTTestP, testing::Values( + std::make_tuple(PartialShape{16}, PartialShape{16}, 16, 16, true, PartialShape{9, 1, 2}), + std::make_tuple(PartialShape{48}, PartialShape{16}, 16, 16, false, PartialShape{3, 9, 2}), + std::make_tuple(PartialShape{56}, PartialShape{7}, 11, 3, false, PartialShape{16, 6, 2}), + std::make_tuple(PartialShape{56}, PartialShape{7}, 11, 3, true, PartialShape{6, 16, 2}), + std::make_tuple(PartialShape{48}, PartialShape{8}, 16, 4, true, PartialShape{9, 9, 2}), + std::make_tuple(PartialShape{{48, 56}}, PartialShape{7}, 11, 3, true, PartialShape{6, {13, 16}, 2}), + std::make_tuple(PartialShape{-1}, PartialShape{7}, 11, 3, true, PartialShape{6, {1, -1}, 2}), std::make_tuple(PartialShape{1, 16}, PartialShape{16}, 16, 16, true, PartialShape{1, 9, 1, 2}), std::make_tuple(PartialShape{1, 48}, PartialShape{16}, 16, 16, true, PartialShape{1, 9, 3, 2}), std::make_tuple(PartialShape{1, 48}, PartialShape{16}, 16, 16, false, PartialShape{1, 3, 9, 2}), @@ -139,16 +146,16 @@ TEST_F(TypePropSTFTTest, signal_incompatible_shape) { const auto frame_size = std::make_shared(element::i64, PartialShape{}); const auto frame_step = std::make_shared(element::i64, PartialShape{}); { - const auto signal = std::make_shared(element::f32, PartialShape{48}); + const auto signal = std::make_shared(element::f32, PartialShape{}); OV_EXPECT_THROW(std::ignore = make_op(signal, window, frame_size, frame_step, transform_frames), NodeValidationFailure, - HasSubstr("The shape of signal must be 2D [batch, signal_size]")); + HasSubstr("The shape of signal must be 1D [signal_size] or 2D [batch, signal_size]")); } { const auto signal = std::make_shared(element::f32, PartialShape{-1, 4, 48}); OV_EXPECT_THROW(std::ignore = make_op(signal, window, frame_size, frame_step, transform_frames), NodeValidationFailure, - HasSubstr("The shape of signal must be 2D [batch, signal_size]")); + HasSubstr("The shape of signal must be 1D [signal_size] or 2D [batch, signal_size]")); } } diff --git a/src/core/tests/visitors/op/lstm_sequence.cpp b/src/core/tests/visitors/op/lstm_sequence.cpp index 0d386ba7bc4ccf..f9819d9ae84bc9 100644 --- a/src/core/tests/visitors/op/lstm_sequence.cpp +++ b/src/core/tests/visitors/op/lstm_sequence.cpp @@ -61,63 +61,3 @@ TEST(attributes, lstm_sequence_op) { EXPECT_EQ(g_lstm_sequence->get_clip(), lstm_sequence->get_clip()); EXPECT_EQ(g_lstm_sequence->get_direction(), lstm_sequence->get_direction()); } - -OPENVINO_SUPPRESS_DEPRECATED_START -TEST(attributes, lstm_sequence_v1_op) { - NodeBuilder::opset().insert(); - - const size_t batch_size = 4; - const size_t num_directions = 2; - const size_t seq_length = 8; - const size_t input_size = 16; - const size_t hidden_size = 64; - - const auto X = make_shared(element::f32, Shape{batch_size, seq_length, input_size}); - const auto initial_hidden_state = - make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); - const auto initial_cell_state = - make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); - const auto sequence_lengths = make_shared(element::i32, Shape{batch_size}); - const auto W = make_shared(element::f32, Shape{num_directions, 4 * hidden_size, input_size}); - const auto R = - make_shared(element::f32, Shape{num_directions, 4 * hidden_size, hidden_size}); - const auto B = make_shared(element::f32, Shape{num_directions, 4 * hidden_size}); - const auto P = make_shared(element::f32, Shape{num_directions, 3 * hidden_size}); - - const auto lstm_direction = op::RecurrentSequenceDirection::BIDIRECTIONAL; - const ov::op::LSTMWeightsFormat weights_format = ov::op::LSTMWeightsFormat::FICO; - const std::vector activations_alpha = {1, 2, 3}; - const std::vector activations_beta = {4, 5, 6}; - const std::vector activations = {"tanh", "sigmoid", "tanh"}; - const float clip_threshold = 0.5f; - const bool input_forget = true; - - const auto lstm_sequence = make_shared(X, - initial_hidden_state, - initial_cell_state, - sequence_lengths, - W, - R, - B, - P, - hidden_size, - lstm_direction, - weights_format, - activations_alpha, - activations_beta, - activations, - clip_threshold, - input_forget); - NodeBuilder builder(lstm_sequence, {X, initial_hidden_state, initial_cell_state, sequence_lengths, W, R, B, P}); - auto g_lstm_sequence = ov::as_type_ptr(builder.create()); - - EXPECT_EQ(g_lstm_sequence->get_hidden_size(), lstm_sequence->get_hidden_size()); - EXPECT_EQ(g_lstm_sequence->get_activations(), lstm_sequence->get_activations()); - EXPECT_EQ(g_lstm_sequence->get_activations_alpha(), lstm_sequence->get_activations_alpha()); - EXPECT_EQ(g_lstm_sequence->get_activations_beta(), lstm_sequence->get_activations_beta()); - EXPECT_EQ(g_lstm_sequence->get_clip_threshold(), lstm_sequence->get_clip_threshold()); - EXPECT_EQ(g_lstm_sequence->get_direction(), lstm_sequence->get_direction()); - EXPECT_EQ(g_lstm_sequence->get_input_forget(), lstm_sequence->get_input_forget()); - EXPECT_EQ(g_lstm_sequence->get_weights_format(), lstm_sequence->get_weights_format()); -} -OPENVINO_SUPPRESS_DEPRECATED_END diff --git a/src/core/tests/visitors/op/sorted_search.cpp b/src/core/tests/visitors/op/sorted_search.cpp index 860c9528d0e9aa..10d544527f3714 100644 --- a/src/core/tests/visitors/op/sorted_search.cpp +++ b/src/core/tests/visitors/op/sorted_search.cpp @@ -22,7 +22,7 @@ TEST(attributes, search_sorted_op) { auto g_op = ov::as_type_ptr(builder.create()); // attribute count - const auto expected_attr_count = 1; + const auto expected_attr_count = 2; EXPECT_EQ(builder.get_value_map_size(), expected_attr_count); // space_to_depth attributes diff --git a/src/core/tests/visitors/op/squeeze.cpp b/src/core/tests/visitors/op/squeeze.cpp index 6eb1674b26329a..be596a5fb1dc67 100644 --- a/src/core/tests/visitors/op/squeeze.cpp +++ b/src/core/tests/visitors/op/squeeze.cpp @@ -6,7 +6,16 @@ #include "unary_ops.hpp" +namespace v0 { using Types = ::testing::Types, UnaryOperatorType>; INSTANTIATE_TYPED_TEST_SUITE_P(visitor_without_attribute, UnaryOperatorVisitor, Types, UnaryOperatorTypeName); +} // namespace v0 + +namespace v15 { +using Types = ::testing::Types, + UnaryOperatorTypeWithAttribute>; + +INSTANTIATE_TYPED_TEST_SUITE_P(visitor_single_attribute, UnaryOperatorVisitor, Types, UnaryOperatorTypeName); +} // namespace v15 diff --git a/src/core/tests/visitors/op/unary_ops.hpp b/src/core/tests/visitors/op/unary_ops.hpp index 3bef2429983e9f..6cc2afda62e253 100644 --- a/src/core/tests/visitors/op/unary_ops.hpp +++ b/src/core/tests/visitors/op/unary_ops.hpp @@ -9,12 +9,17 @@ #include "openvino/op/parameter.hpp" #include "visitors/visitors.hpp" -template +template class UnaryOperatorType { public: using op_type = T; static constexpr ov::element::Type_t element_type = ELEMENT_TYPE; + static constexpr int expected_attr_count = ATTRIBUTES_COUNT; }; + +template +using UnaryOperatorTypeWithAttribute = UnaryOperatorType; + template class UnaryOperatorVisitor : public testing::Test {}; @@ -43,7 +48,7 @@ TYPED_TEST_P(UnaryOperatorVisitor, No_Attribute_4D) { EXPECT_NO_THROW(auto g_op_func = ov::as_type_ptr(builder.create())); - const auto expected_attr_count = 0; + const auto expected_attr_count = TypeParam::expected_attr_count; EXPECT_EQ(builder.get_value_map_size(), expected_attr_count); } diff --git a/src/frontends/jax/src/op/erfc.cpp b/src/frontends/jax/src/op/erfc.cpp new file mode 100644 index 00000000000000..5a38577f868d35 --- /dev/null +++ b/src/frontends/jax/src/op/erfc.cpp @@ -0,0 +1,33 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/frontend/jax/node_context.hpp" +#include "openvino/op/erf.hpp" +#include "openvino/op/subtract.hpp" +#include "utils.hpp" + +using namespace std; +using namespace ov; +using namespace ov::op; + +namespace ov { +namespace frontend { +namespace jax { +namespace op { + +OutputVector translate_erfc(const NodeContext& context) { + num_inputs_check(context, 1, 1); + auto x = context.get_input(0); + + // create const one of the same type as x + auto const_one = create_same_type_const_scalar(x, 1); + Output res = make_shared(x); + res = make_shared(const_one, res); + return {res}; +}; + +} // namespace op +} // namespace jax +} // namespace frontend +} // namespace ov diff --git a/src/frontends/jax/src/op_table.cpp b/src/frontends/jax/src/op_table.cpp index 500226594fea13..98f22452c5afab 100644 --- a/src/frontends/jax/src/op_table.cpp +++ b/src/frontends/jax/src/op_table.cpp @@ -45,6 +45,7 @@ OP_CONVERTER(translate_convert); OP_CONVERTER(translate_convolution); OP_CONVERTER(translate_copy); OP_CONVERTER(translate_dot_general); +OP_CONVERTER(translate_erfc); OP_CONVERTER(translate_integer_pow); OP_T_CONVERTER(translate_reduce_op); OP_CONVERTER(translate_reduce_window_max); @@ -72,6 +73,7 @@ const std::map get_supported_ops_jaxpr() { {"dot_general", op::translate_dot_general}, {"eq", op::translate_binary_op}, {"erf", op::translate_1to1_match_1_input}, + {"erfc", op::translate_erfc}, {"exp", op::translate_1to1_match_1_input}, {"ge", op::translate_binary_op}, {"gt", op::translate_binary_op}, diff --git a/src/frontends/onnx/frontend/src/core/graph.cpp b/src/frontends/onnx/frontend/src/core/graph.cpp index ad755c958703cb..80ec9c6c9f390c 100644 --- a/src/frontends/onnx/frontend/src/core/graph.cpp +++ b/src/frontends/onnx/frontend/src/core/graph.cpp @@ -446,10 +446,6 @@ void Graph::set_friendly_names(const Node& onnx_node, const ov::OutputVector& ov // null node does not have tensor if (!ov::op::util::is_null(ov_subgraph_outputs[i])) { ov_subgraph_outputs[i].get_tensor().set_names({onnx_node.output(static_cast(i))}); - OPENVINO_SUPPRESS_DEPRECATED_START - ov::descriptor::set_ov_tensor_legacy_name(ov_subgraph_outputs[i].get_tensor(), - onnx_node.output(static_cast(i))); - OPENVINO_SUPPRESS_DEPRECATED_END } } } diff --git a/src/frontends/onnx/frontend/src/op/com.microsoft/quick_gelu.cpp b/src/frontends/onnx/frontend/src/op/com.microsoft/quick_gelu.cpp new file mode 100644 index 00000000000000..c4144be9b5ff44 --- /dev/null +++ b/src/frontends/onnx/frontend/src/op/com.microsoft/quick_gelu.cpp @@ -0,0 +1,58 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "core/operator_set.hpp" +#include "exceptions.hpp" +#include "openvino/frontend/exception.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/sigmoid.hpp" +#include "utils/common.hpp" + +using namespace ov::op; + +namespace ov { +namespace frontend { +namespace onnx { +namespace com_microsoft { +namespace opset_1 { +ov::OutputVector quick_gelu(const ov::frontend::onnx::Node& node) { + // Original Documentation: + // https://github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md#com.microsoft.QuickGelu + // Goal: Compute x * Sigmoid(alpha * x) + common::default_op_checks(node, 1); + + const auto inputs = node.get_ov_inputs(); + const auto& x = inputs[0]; + + // Constrain input type to float16, float, double (f64), bfloat16 + auto element_type = x.get_element_type(); + CHECK_VALID_NODE(node, + element_type == ov::element::f16 || element_type == ov::element::f32 || + element_type == ov::element::f64 || element_type == ov::element::bf16, + "Unsupported input x type, accepted FP16, FP32, FP64, BFP16 but got: ", + element_type); + + // Get attribute from node + const float alpha = node.get_attribute_value("alpha"); + + // Numpy broadcasting rule is automatically applied with mismatched shapes according to: + // https://docs.openvino.ai/2022.3/openvino_docs_ops_arithmetic_Multiply_1.html "Tensor with dimension of size 1 + // will be implicitly broadcasted to match the size of the second tensor." Convert alpha to tensor with size 1 + const auto alpha_tensor = std::make_shared(ov::element::f32, Shape{1}, alpha); + + auto alpha_x = std::make_shared(alpha_tensor, x); + auto sig_alpha_x = std::make_shared(alpha_x); + auto result = std::make_shared(x, sig_alpha_x); + + return {result}; +} // func end + +ONNX_OP("QuickGelu", OPSET_SINCE(1), com_microsoft::opset_1::quick_gelu, MICROSOFT_DOMAIN); + +} // namespace opset_1 +} // namespace com_microsoft +} // namespace onnx +} // namespace frontend +} // namespace ov diff --git a/src/frontends/onnx/frontend/src/op/dequantize_linear.cpp b/src/frontends/onnx/frontend/src/op/dequantize_linear.cpp index b09bc73467bc10..47fcc7af60bf61 100644 --- a/src/frontends/onnx/frontend/src/op/dequantize_linear.cpp +++ b/src/frontends/onnx/frontend/src/op/dequantize_linear.cpp @@ -18,6 +18,7 @@ #include "openvino/op/subtract.hpp" #include "openvino/op/transpose.hpp" #include "openvino/op/unsqueeze.hpp" +#include "transformations/rt_info/disable_constant_folding.hpp" #include "utils/common.hpp" #include "utils/reshape.hpp" using namespace ov::op; @@ -221,19 +222,8 @@ ov::OutputVector dequantize_linear(const ov::frontend::onnx::Node& node) { FRONT_END_GENERAL_CHECK(src_x.get_partial_shape().is_static(), "DequantizeLinear cannot operate with dynamic shapes of input X"); - const auto& unsqueezed_axes = std::make_shared(ov::element::i64, Shape{1}, std::vector{1}); - - if (inputs.size() > 2) { - zp = inputs[2]; - if (zp.get_element_type() != scale.get_element_type()) { - zp = std::make_shared(zp, scale); - } - zp = std::make_shared(zp, unsqueezed_axes); - } - const auto axis = node.get_attribute_value("axis", 1); const auto block_size = static_cast(node.get_attribute_value("block_size", 0)); - const auto scale_type = scale.get_element_type(); FRONT_END_GENERAL_CHECK(axis == 0, "Axis != 0 isn't supported"); FRONT_END_GENERAL_CHECK(block_size > 0, "block_size must be greater than zero"); @@ -241,16 +231,31 @@ ov::OutputVector dequantize_linear(const ov::frontend::onnx::Node& node) { src_x.get_shape()[0] % block_size == 0, "DequantizeLinear doesn't support case when first dimension of X cannot be divided by block_size"); - const auto& x = src_x.get_element_type() == scale_type ? src_x : std::make_shared(src_x, scale); + ov::Output broadcastable_x = op::util::reshape( + src_x, + Shape{static_cast(src_x.get_shape()[0]) / block_size, block_size, src_x.get_shape()[1]}); + + const auto& unsqueezed_axes = std::make_shared(ov::element::i64, Shape{1}, std::vector{1}); + + const auto scale_type = scale.get_element_type(); + if (inputs.size() > 2) { + zp = inputs[2]; + if (zp.get_element_type() != scale.get_element_type()) { + zp = std::make_shared(zp, scale_type); + disable_constant_folding(zp.get_node_shared_ptr()); + } + zp = std::make_shared(zp, unsqueezed_axes); + } + + const auto& x = src_x.get_element_type() == scale_type ? broadcastable_x + : std::make_shared(broadcastable_x, scale_type); // For further broadcasting scales and zp - reshape input to a shape [x.shape[0]/block_size, block_size, x.shape[1]] - ov::Output broadcastable_x = - op::util::reshape(x, Shape{static_cast(x.get_shape()[0]) / block_size, block_size, x.get_shape()[1]}); // Adding additional dimension for broadcasting scale = std::make_shared(scale, unsqueezed_axes); if (zp.get_node_shared_ptr()) { - broadcastable_x = std::make_shared(broadcastable_x, zp); + broadcastable_x = std::make_shared(x, zp); } const auto& scaled_x = std::make_shared(broadcastable_x, scale); diff --git a/src/frontends/onnx/frontend/src/op/lstm.cpp b/src/frontends/onnx/frontend/src/op/lstm.cpp index 6b934db7d13071..1c0efbbe6b9454 100644 --- a/src/frontends/onnx/frontend/src/op/lstm.cpp +++ b/src/frontends/onnx/frontend/src/op/lstm.cpp @@ -211,41 +211,19 @@ ov::OutputVector lstm(const ov::frontend::onnx::Node& node) { LSTMAttributes attributes{node}; std::shared_ptr lstm_sequence; - if ((input_map.at(LSTMInput::LSTM_INPUT_P).get_names() != std::unordered_set({"P_blank"})) || - (attributes.m_input_forget == true)) { - OPENVINO_SUPPRESS_DEPRECATED_START - lstm_sequence = std::make_shared(input_map.at(LSTMInput::LSTM_INPUT_X), - input_map.at(LSTMInput::LSTM_INPUT_INIT_H), - input_map.at(LSTMInput::LSTM_INPUT_INIT_C), - input_map.at(LSTMInput::LSTM_INPUT_SEQ_LENGTHS), - input_map.at(LSTMInput::LSTM_INPUT_W), - input_map.at(LSTMInput::LSTM_INPUT_R), - input_map.at(LSTMInput::LSTM_INPUT_B), - input_map.at(LSTMInput::LSTM_INPUT_P), - attributes.m_hidden_size, - attributes.m_direction, - ov::op::LSTMWeightsFormat::FICO, - attributes.m_activation_alpha, - attributes.m_activation_beta, - attributes.m_activations, - attributes.m_clip_threshold, - attributes.m_input_forget); - OPENVINO_SUPPRESS_DEPRECATED_END - } else { - lstm_sequence = std::make_shared(input_map.at(LSTMInput::LSTM_INPUT_X), - input_map.at(LSTMInput::LSTM_INPUT_INIT_H), - input_map.at(LSTMInput::LSTM_INPUT_INIT_C), - input_map.at(LSTMInput::LSTM_INPUT_SEQ_LENGTHS), - input_map.at(LSTMInput::LSTM_INPUT_W), - input_map.at(LSTMInput::LSTM_INPUT_R), - input_map.at(LSTMInput::LSTM_INPUT_B), - attributes.m_hidden_size, - attributes.m_direction, - attributes.m_activation_alpha, - attributes.m_activation_beta, - attributes.m_activations, - attributes.m_clip_threshold); - } + lstm_sequence = std::make_shared(input_map.at(LSTMInput::LSTM_INPUT_X), + input_map.at(LSTMInput::LSTM_INPUT_INIT_H), + input_map.at(LSTMInput::LSTM_INPUT_INIT_C), + input_map.at(LSTMInput::LSTM_INPUT_SEQ_LENGTHS), + input_map.at(LSTMInput::LSTM_INPUT_W), + input_map.at(LSTMInput::LSTM_INPUT_R), + input_map.at(LSTMInput::LSTM_INPUT_B), + attributes.m_hidden_size, + attributes.m_direction, + attributes.m_activation_alpha, + attributes.m_activation_beta, + attributes.m_activations, + attributes.m_clip_threshold); const auto Y = lstm_sequence->output(0); const auto Y_h = lstm_sequence->output(1); diff --git a/src/frontends/onnx/tests/models/com.microsoft/quick_gelu.prototxt b/src/frontends/onnx/tests/models/com.microsoft/quick_gelu.prototxt new file mode 100644 index 00000000000000..4fb110fd485833 --- /dev/null +++ b/src/frontends/onnx/tests/models/com.microsoft/quick_gelu.prototxt @@ -0,0 +1,52 @@ +ir_version: 3 +producer_name: "OpenVINO ONNX Frontend" +graph { + name: "test_quick_gelu" + node { + input: "X" + output: "Y" + op_type: "QuickGelu" + attribute { + name: "alpha" + f: 0.9974269270896912 + type: FLOAT + } + domain: "com.microsoft" + } + input { + name: "X" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 5 + } + } + } + } + } + output { + name: "Y" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 2 + } + dim { + dim_value: 5 + } + } + } + } + } +} +opset_import { + domain: "com.microsoft" + version: 1 +} \ No newline at end of file diff --git a/src/frontends/onnx/tests/models/lstm_bdir_short_input_seq.prototxt b/src/frontends/onnx/tests/models/lstm_bdir_short_input_seq.prototxt deleted file mode 100644 index 57b723e44b6914..00000000000000 --- a/src/frontends/onnx/tests/models/lstm_bdir_short_input_seq.prototxt +++ /dev/null @@ -1,235 +0,0 @@ -ir_version: 5 -graph { - node { - input: "X" - input: "W" - input: "R" - input: "B" - input: "sequence_lens" - input: "initial_h" - input: "initial_c" - input: "P" - output: "Y" - output: "Y_h" - output: "" - name: "node1" - op_type: "LSTM" - attribute { - name: "direction" - s: "bidirectional" - type: STRING - } - attribute { - name: "input_forget" - i: 0 - type: INT - } - attribute { - name: "activations" - strings: "sigmoid" - strings: "tanh" - strings: "tanh" - strings: "sigmoid" - strings: "tanh" - strings: "tanh" - type: STRINGS - } - attribute { - name: "hidden_size" - i: 2 - type: INT - } - attribute { - name: "clip" - f: 9999 - type: FLOAT - } - doc_string: "LSTM" - domain: "" - } - input { - name: "X" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "W" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 8 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "R" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 8 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "B" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 16 - } - } - } - } - } - input { - name: "sequence_lens" - type { - tensor_type { - elem_type: 6 - shape { - dim { - dim_value: 1 - } - } - } - } - } - input { - name: "initial_h" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "initial_c" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "P" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 6 - } - } - } - } - } - output { - name: "Y" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 2 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } - output { - name: "Y_h" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } -} -opset_import { - domain: "" - version: 7 -} diff --git a/src/frontends/onnx/tests/models/lstm_fwd_with_clip_peepholes.prototxt b/src/frontends/onnx/tests/models/lstm_fwd_with_clip_peepholes.prototxt deleted file mode 100644 index 2a218f089240a5..00000000000000 --- a/src/frontends/onnx/tests/models/lstm_fwd_with_clip_peepholes.prototxt +++ /dev/null @@ -1,191 +0,0 @@ -ir_version: 4 -producer_name: "OpenVINO ONNX Frontend" -graph { - node { - input: "X" - input: "W" - input: "R" - input: "B" - input: "" - input: "" - input: "" - input: "P" - output: "Y" - output: "Y_h" - output: "Y_c" - op_type: "LSTM" - attribute { - name: "clip" - f: 0.10000000149011612 - type: FLOAT - } - attribute { - name: "direction" - s: "forward" - type: STRING - } - attribute { - name: "hidden_size" - i: 2 - type: INT - } - attribute { - name: "input_forget" - i: 0 - type: INT - } - } - name: "compute_graph" - input { - name: "X" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "W" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 1 - } - dim { - dim_value: 8 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "R" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 1 - } - dim { - dim_value: 8 - } - dim { - dim_value: 2 - } - } - } - } - } - input { - name: "B" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 1 - } - dim { - dim_value: 16 - } - } - } - } - } - input { - name: "P" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 1 - } - dim { - dim_value: 6 - } - } - } - } - } - output { - name: "Y" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 2 - } - dim { - dim_value: 1 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } - output { - name: "Y_h" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 1 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } - output { - name: "Y_c" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 1 - } - dim { - dim_value: 1 - } - dim { - dim_value: 2 - } - } - } - } - } -} -opset_import { - version: 7 -} diff --git a/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp b/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp index da8189926a4546..900fc025d8d9ab 100644 --- a/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp @@ -1330,3 +1330,29 @@ OPENVINO_TEST(${BACKEND_NAME}, onnx_com_microsoft_matmulnbits_3x17) { } test_case.run(); } + +OPENVINO_TEST(${BACKEND_NAME}, onnx_com_microsoft_quickgelu) { + const auto model = convert_model("com.microsoft/quick_gelu.onnx"); + auto test_case = ov::test::TestCase(model, s_device); + + const std::vector input_X{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + const std::vector output{0.7305524f, + 1.7605114f, + 2.8566725f, + 3.9273243f, + 4.9661055f, + 5.984934f, + 6.9935064f, + 7.997261f, + 8.998864f, + 9.999535f}; + + test_case.add_input(Shape{2, 5}, input_X); + test_case.add_expected_output(Shape{2, 5}, output); + + if (std::string("${BACKEND_NAME}") == std::string("IE_GPU")) { + test_case.run_with_tolerance_as_fp(0.0001f); + } else { + test_case.run(); + } +} diff --git a/src/frontends/onnx/tests/onnx_import_rnn.in.cpp b/src/frontends/onnx/tests/onnx_import_rnn.in.cpp index 5979466849d5e6..b61fd1d36f99e0 100644 --- a/src/frontends/onnx/tests/onnx_import_rnn.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_rnn.in.cpp @@ -202,72 +202,6 @@ OPENVINO_TEST(${BACKEND_NAME}, onnx_model_lstm_bidir_mixed_seq_const) { test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 1); } -OPENVINO_TEST(${BACKEND_NAME}, onnx_model_lstm_fwd_with_clip_peepholes) { - auto model = convert_model("lstm_fwd_with_clip_peepholes.onnx"); - - auto test_case = ov::test::TestCase(model, s_device); - test_case.add_input({-0.455351f, -0.276391f, -0.185934f, -0.269585f}); // X - test_case.add_input({-0.494659f, // W - 0.0453352f, - -0.487793f, - 0.417264f, - -0.0175329f, - 0.489074f, - -0.446013f, - 0.414029f, - -0.0091708f, - -0.255364f, - -0.106952f, - -0.266717f, - -0.0888852f, - -0.428709f, - -0.283349f, - 0.208792f}); // W - test_case.add_input({0.146626f, - -0.0620289f, - -0.0815302f, - 0.100482f, - -0.219535f, - -0.306635f, - -0.28515f, - -0.314112f, - -0.228172f, - 0.405972f, - 0.31576f, - 0.281487f, - -0.394864f, - 0.42111f, - -0.386624f, - -0.390225f}); // R - - test_case.add_input({0.381619f, - 0.0323954f, - -0.14449f, - 0.420804f, - -0.258721f, - 0.45056f, - -0.250755f, - 0.0967895f, - 0.0f, - 0.0f, - 0.0f, - 0.0f, - 0.0f, - 0.0f, - 0.0f, - 0.0f}); // B - test_case.add_input({0.2345f, 0.5235f, 0.4378f, 0.3475f, 0.8927f, 0.3456f}); // P - - test_case.add_expected_output(Shape{2, 1, 1, 2}, - {-0.02280854f, 0.02744377f, -0.03516197f, 0.03875681f}); // Y_data - test_case.add_expected_output(Shape{1, 1, 2}, {-0.03516197f, 0.03875681f}); // Y_h_data - test_case.add_expected_output(Shape{1, 1, 2}, {-0.07415761f, 0.07395997f}); // Y_c_data - - // We have to enlarge tolerance bits to 3 - it's only one bit more than default value. - // The discrepancies may occur at most on 7th decimal position. - test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 3); -} - OPENVINO_TEST(${BACKEND_NAME}, onnx_model_lstm_fwd_mixed_seq) { auto model = convert_model("lstm_fwd_mixed_seq.onnx"); @@ -397,50 +331,6 @@ OPENVINO_TEST(${BACKEND_NAME}, onnx_model_lstm_fwd_large_batch_no_clip) { test_case.run(); } -OPENVINO_TEST(${BACKEND_NAME}, onnx_model_lstm_bdir_short_input_seq_peepholes) { - auto model = convert_model("lstm_bdir_short_input_seq.onnx"); - - auto test_case = ov::test::TestCase(model, s_device); - - // X - test_case.add_input({-0.455351f, -0.276391f, -0.185934f, -0.269585f}); - // W - test_case.add_input( - {-0.494659f, 0.0453352f, -0.487793f, 0.417264f, -0.0175329f, 0.489074f, -0.446013f, 0.414029f, - -0.0091708f, -0.255364f, -0.106952f, -0.266717f, -0.0888852f, -0.428709f, -0.283349f, 0.208792f, - -0.494659f, 0.0453352f, -0.487793f, 0.417264f, -0.0175329f, 0.489074f, -0.446013f, 0.414029f, - -0.0091708f, -0.255364f, -0.106952f, -0.266717f, -0.0888852f, -0.428709f, -0.283349f, 0.208792f}); - // R - test_case.add_input({0.146626f, -0.0620289f, -0.0815302f, 0.100482f, -0.219535f, -0.306635f, -0.28515f, - -0.314112f, -0.228172f, 0.405972f, 0.31576f, 0.281487f, -0.394864f, 0.42111f, - -0.386624f, -0.390225f, 0.146626f, -0.0620289f, -0.0815302f, 0.100482f, -0.219535f, - -0.306635f, -0.28515f, -0.314112f, -0.228172f, 0.405972f, 0.31576f, 0.281487f, - -0.394864f, 0.42111f, -0.386624f, -0.390225f}); - // B - test_case.add_input({0.381619f, 0.0323954f, -0.14449f, 0.420804f, -0.258721f, 0.45056f, -0.250755f, - 0.0967895f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 0.381619f, 0.0323954f, -0.14449f, 0.420804f, -0.258721f, - 0.45056f, -0.250755f, 0.0967895f, 0.0f, 0.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 0.0f, 0.0f}); - // sequence_lens - test_case.add_input({1}); - // initial_h - test_case.add_input({0.0f, 0.0f, -0.0306872f, 0.028035f}); - // initial_c - test_case.add_input({0.0f, 0.0f, -0.07243599f, 0.0467052f}); - // P - test_case.add_input( - {0.2345f, 0.5235f, 0.4378f, 0.3475f, 0.8927f, 0.3456f, 0.2345f, 0.5235f, 0.4378f, 0.3475f, 0.8927f, 0.3456f}); - - // Y - test_case.add_expected_output(Shape{2, 2, 1, 2}, - {-0.0251062f, 0.0561262f, -0.0318928f, 0.0762679f, 0.0f, 0.0f, 0.0f, 0.0f}); - // Y_h - test_case.add_expected_output(Shape{2, 1, 2}, {-0.0251062f, 0.0561262f, -0.0318928f, 0.0762679f}); - - test_case.run(DEFAULT_FLOAT_TOLERANCE_BITS + 3); -} - OPENVINO_TEST(${BACKEND_NAME}, onnx_model_lstm_mixed_seq_reverse) { auto model = convert_model("lstm_mixed_seq_reverse.onnx"); diff --git a/src/frontends/onnx/tests/runtime/ie/unit_test.manifest b/src/frontends/onnx/tests/runtime/ie/unit_test.manifest index b2e87a2588ec65..3eadb698ca0a03 100644 --- a/src/frontends/onnx/tests/runtime/ie/unit_test.manifest +++ b/src/frontends/onnx/tests/runtime/ie/unit_test.manifest @@ -121,9 +121,6 @@ onnx_model_lp_norm_default_dynamic onnx_instance_normalization_dynamic # Legacy tests with unsupported features from opset4 LSTM/GRU/RNN -# Peepholes input unsupported -onnx_model_lstm_fwd_with_clip_peepholes -onnx_model_lstm_bdir_short_input_seq_peepholes # Activation function hardsigmoid is not supported onnx_model_gru_fwd_activations_relu_hardsigmoid onnx_model_lstm_fwd_hardsigmoid_activation diff --git a/src/frontends/pytorch/src/op/lerp.cpp b/src/frontends/pytorch/src/op/lerp.cpp new file mode 100644 index 00000000000000..67922da3e4578d --- /dev/null +++ b/src/frontends/pytorch/src/op/lerp.cpp @@ -0,0 +1,36 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/frontend/pytorch/node_context.hpp" +#include "openvino/op/add.hpp" +#include "openvino/op/convert_like.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/subtract.hpp" +#include "utils.hpp" + +namespace ov { +namespace frontend { +namespace pytorch { +namespace op { + +using namespace ov::op; + +OutputVector translate_lerp(const NodeContext& context) { + // Tensor = aten::lerp(%lhs.1, %rhs.1, %self.weight) + num_inputs_check(context, 3, 3); + Output start; + Output end; + std::tie(start, end) = get_inputs_with_promoted_types(context, 0, 1); + + Output weight = context.get_input(2); + auto scale = context.mark_node(std::make_shared(end, start)); + weight = context.mark_node(std::make_shared(weight, scale)); + auto delta = context.mark_node(std::make_shared(scale, weight)); + return {context.mark_node(std::make_shared(start, delta))}; +}; + +} // namespace op +} // namespace pytorch +} // namespace frontend +} // namespace ov diff --git a/src/frontends/pytorch/src/op/stft.cpp b/src/frontends/pytorch/src/op/stft.cpp new file mode 100644 index 00000000000000..b7e4858c2f8fcc --- /dev/null +++ b/src/frontends/pytorch/src/op/stft.cpp @@ -0,0 +1,93 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/op/stft.hpp" + +#include "openvino/frontend/pytorch/node_context.hpp" +#include "openvino/op/broadcast.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/convert_like.hpp" +#include "openvino/op/divide.hpp" +#include "openvino/op/shape_of.hpp" +#include "openvino/op/unsqueeze.hpp" +#include "utils.hpp" + +namespace ov { +namespace frontend { +namespace pytorch { +namespace op { + +using namespace ov::op; + +OutputVector translate_stft(const NodeContext& context) { + // schema: aten::stft(Tensor self, int n_fft, int? hop_length=None, int? win_length=None, Tensor? window=None, bool + // normalized=False, bool? onesided=None, bool? return_complex=None) -> Tensor + // + // Note: aten::stft doesn't have "center" and "pad_mode" attrs like torch.stft, so the number of the inputs is lower + // and index of any input after the "window" is smaller accordingly + + num_inputs_check(context, 2, 8); + + auto input = context.get_input(0); + auto n_fft = context.get_input(1); + + ov::Output hop_length; + if (!context.input_is_none(2)) { + hop_length = context.get_input(2); + } else { + // Defualt floor(n_fft / 4) + const auto four = context.mark_node(std::make_shared(ov::element::i32, Shape{}, 4)); + const auto four_cast = context.mark_node(std::make_shared(four, n_fft)); + hop_length = context.mark_node(std::make_shared(n_fft, four_cast)); + } + + ov::Output win_length; + if (!context.input_is_none(3)) { + win_length = context.get_input(3); + } else { + win_length = n_fft; + } + + ov::Output window; + if (!context.input_is_none(4)) { + window = context.get_input(4); + } else { + const auto one = context.mark_node(std::make_shared(ov::element::i32, Shape{}, 1)); + const auto one_cast = context.mark_node(std::make_shared(one, input)); + const auto zero = context.mark_node(std::make_shared(ov::element::i32, Shape{1}, 0)); + const auto win_length_cast = + context.mark_node(std::make_shared(win_length, ov::element::i64)); + const auto win_len_vec = context.mark_node(std::make_shared(win_length_cast, zero)); + window = context.mark_node(std::make_shared(one_cast, win_len_vec)); + } + + bool normalized = false; + if (!context.input_is_none(5)) { + normalized = context.const_input(5); + } + PYTORCH_OP_CONVERSION_CHECK(!normalized, + "aten::stft conversion is currently supported with normalized=False only."); + + bool onesided = true; + if (!context.input_is_none(6)) { + onesided = context.const_input(6); + } + PYTORCH_OP_CONVERSION_CHECK(onesided, "aten::stft conversion is currently supported with onesided=True only."); + + bool return_complex = false; + if (!context.input_is_none(7)) { + return_complex = context.const_input(7); + } + PYTORCH_OP_CONVERSION_CHECK(!return_complex, + "aten::stft conversion is currently supported with return_complex=False only."); + + // Perform STFT + constexpr bool transpose_frames = true; + auto stft = context.mark_node(std::make_shared(input, window, n_fft, hop_length, transpose_frames)); + return {stft}; +}; +} // namespace op +} // namespace pytorch +} // namespace frontend +} // namespace ov diff --git a/src/frontends/pytorch/src/op_table.cpp b/src/frontends/pytorch/src/op_table.cpp index 66c76e33032ef6..607f0bd32db80d 100644 --- a/src/frontends/pytorch/src/op_table.cpp +++ b/src/frontends/pytorch/src/op_table.cpp @@ -121,6 +121,7 @@ OP_CONVERTER(translate_inverse); OP_CONVERTER(translate_is_nonzero); OP_CONVERTER(translate_layer_norm); OP_CONVERTER(translate_len); +OP_CONVERTER(translate_lerp); OP_CONVERTER(translate_linalg_cross); OP_CONVERTER(translate_linalg_norm); OP_CONVERTER(translate_linalg_matrix_norm); @@ -220,6 +221,7 @@ OP_CONVERTER(translate_square); OP_CONVERTER(translate_squeeze); OP_CONVERTER(translate_std); OP_CONVERTER(translate_std_mean); +OP_CONVERTER(translate_stft); OP_CONVERTER(translate_sub); OP_CONVERTER(translate_sub_); OP_CONVERTER(translate_sum); @@ -329,12 +331,15 @@ OP_CONVERTER(translate_zeros_like_fx); const std::unordered_map get_supported_ops_ts() { return { {"aten::__and__", op::translate_bitwise_and}, + {"aten::__iand__", op::inplace_op}, {"aten::__derive_index", op::translate_derive_index}, {"aten::__getitem__", op::translate_getitem}, {"aten::__not__", op::translate_1to1_match_1_inputs}, {"aten::__or__", op::translate_bitwise_or}, + {"aten::__ior__", op::inplace_op}, {"aten::__range_length", op::translate_range_length}, {"aten::__xor__", op::translate_bitwise_xor}, + {"aten::__ixor__", op::inplace_op}, {"aten::_convolution", op::translate_convolution}, {"aten::_convolution_mode", op::translate_convolution_mode}, {"aten::_native_multi_head_attention", op::translate_native_multi_head_attention}, @@ -508,6 +513,7 @@ const std::unordered_map get_supported_ops_ts() { {"aten::le", op::translate_1to1_match_2_inputs_align_types}, {"aten::leaky_relu", op::translate_1to1_match_2_inputs}, {"aten::len", op::translate_len}, + {"aten::lerp", op::translate_lerp}, // lift op is torchscript specific op responsible for tensors coping with guarantee of new memory allocation {"aten::lift", op::skip_node}, {"aten::lift_fresh", op::skip_node}, @@ -649,6 +655,7 @@ const std::unordered_map get_supported_ops_ts() { // aten::stack - Supported in limited set of patterns {"aten::std", op::translate_std}, {"aten::std_mean", op::translate_std_mean}, + {"aten::stft", op::translate_stft}, {"aten::sub", op::translate_sub}, {"aten::sub_", op::translate_sub_}, {"aten::sum", op::translate_sum}, @@ -746,6 +753,7 @@ const std::unordered_map get_supported_ops_fx() { {"aten._native_batch_norm_legit.no_stats", op::translate_batch_norm_legit_no_stats_fx}, {"aten._native_batch_norm_legit_functional.default", op::translate_batch_norm_legit_fx}, {"aten._native_batch_norm_legit_no_training.default", op::translate_batch_norm_legit_no_training_fx}, + {"aten._safe_softmax.default", op::translate_softmax_fx}, {"aten._scaled_dot_product_flash_attention.default", op::translate_scaled_dot_product_attention_fx}, {"aten._scaled_dot_product_flash_attention_for_cpu.default", op::translate_scaled_dot_product_attention_fx}, {"aten._softmax.default", op::translate_softmax_fx}, diff --git a/src/frontends/tensorflow/src/transformations/switch_merge_resolve.cpp b/src/frontends/tensorflow/src/transformations/switch_merge_resolve.cpp index 34b2a82152ccfc..cbdc506671aa67 100644 --- a/src/frontends/tensorflow/src/transformations/switch_merge_resolve.cpp +++ b/src/frontends/tensorflow/src/transformations/switch_merge_resolve.cpp @@ -235,6 +235,9 @@ bool pass::SwitchMergeResolver::run_on_model(const shared_ptr& m) { auto else_body = make_shared(else_results, else_params); auto if_op = make_shared(cond); + // in case TensorFlow models, we can deduce predicate shape that must be a scalar + if_op->get_rt_info()["tf_switch_merge_if"] = true; + set_cf_marker(if_cf_marker, if_op); if_op->set_then_body(then_body); if_op->set_else_body(else_body); diff --git a/src/frontends/tensorflow_common/src/op/tensor_list_operations.cpp b/src/frontends/tensorflow_common/src/op/tensor_list_operations.cpp index 5cbf4c92b8a35c..e082be4943977c 100644 --- a/src/frontends/tensorflow_common/src/op/tensor_list_operations.cpp +++ b/src/frontends/tensorflow_common/src/op/tensor_list_operations.cpp @@ -199,8 +199,13 @@ OutputVector translate_tensor_list_length_op(const NodeContext& node) { auto tensor_list_shape = make_shared(input_handle, element::i32); auto list_length = make_shared(tensor_list_shape, zero_const, one_const, one_const); - set_node_name(node.get_name(), list_length); - return {list_length}; + // output of TensorListLength must be a scalar + // after Slice operation it is a 1D tensor with one element + auto scalar_shape = make_shared(element::i32, Shape{0}, std::vector{}); + auto list_length_scalar = make_shared(list_length, scalar_shape, false); + + set_node_name(node.get_name(), list_length_scalar); + return {list_length_scalar}; } OutputVector translate_tensor_list_concat_v2_op(const NodeContext& node) { diff --git a/src/frontends/tensorflow_common/src/op/tobool.cpp b/src/frontends/tensorflow_common/src/op/tobool.cpp index 9f1082d5df87da..9025a35f549799 100644 --- a/src/frontends/tensorflow_common/src/op/tobool.cpp +++ b/src/frontends/tensorflow_common/src/op/tobool.cpp @@ -9,6 +9,8 @@ #include "openvino/op/logical_and.hpp" #include "openvino/op/logical_or.hpp" #include "openvino/op/not_equal.hpp" +#include "openvino/op/range.hpp" +#include "openvino/op/reduce_logical_and.hpp" #include "openvino/op/reduce_prod.hpp" #include "openvino/op/select.hpp" #include "openvino/op/shape_of.hpp" @@ -22,47 +24,49 @@ namespace tensorflow { namespace op { OutputVector translate_tobool_op(const NodeContext& node) { // (rank(x) == 0 && x != 0) || (rank > 0 && ReduceProd(ShapeOf(x))) > 0 - default_op_checks(node, 1, {"ToBool"}); auto x = node.get_input(0); // prepare auxiliary zero and zero constants of the same type as the inputs - auto zero = create_same_type_const_scalar(x, 0); - auto zero_2 = make_shared(element::i32, Shape{}, 0); - auto true_const = make_shared(element::boolean, Shape{}, true); + auto zero_x = create_same_type_const_scalar(x, 0); + auto zero_i64 = make_shared(element::i64, Shape{}, 0); + auto one_i64 = make_shared(element::i64, Shape{}, 1); auto false_const = make_shared(element::boolean, Shape{}, false); // compute a mask to get rank(x) == 0 - auto x_rank = compute_subgraph_scalar_rank(x, element::i32); + auto x_rank = compute_subgraph_scalar_rank(x, element::i64, true); + // 1. try to evaluate if it satisfy non-zero scalar input // compute rank(x) == 0 - auto is_zero = make_shared(x_rank, zero_2); - + auto is_rank_zero = make_shared(x_rank, zero_i64); // compute mask to get x != 0 - auto is_not_zero = make_shared(x, zero); - + auto is_x_not_zero = make_shared(x, zero_x)->output(0); // compute (rank(x) == 0 && x != 0) - auto logical_and = make_shared(is_zero, is_not_zero); - // compute rank(x) > 0 - auto greater_than_zero = make_shared(x_rank, zero_2); + auto scalar_cond = make_shared(is_rank_zero, is_x_not_zero)->output(0); + // generate reduce_axes + auto reduce_axes = make_shared(zero_i64, x_rank, one_i64); + scalar_cond = make_shared(scalar_cond, reduce_axes, false); + // correct result for empty tensor, for which scalar_cond is still equal to True + scalar_cond = make_shared(is_rank_zero, scalar_cond, false_const); + // 2. try to evaluate if it is non-scalar input tensor and not empty tensor + // compute rank(x) > 0 + auto rank_greater_than_zero = make_shared(x_rank, zero_i64); // compute ShapeOf(x) - auto cond_shape = make_shared(x, element::i32); + auto x_shape = make_shared(x, element::i64); // compute ReduceProd(ShapeOf(x))) and axis - auto axis = make_shared(element::i32, Shape{}, 0); - auto reduce_prod = make_shared(cond_shape, axis); - + auto reduce_axis = make_shared(element::i32, Shape{}, 0); + auto num_elems = make_shared(x_shape, reduce_axis, false); // compute ReduceProd(ShapeOf(x))) > 0 - auto greater_than__zero_2 = make_shared(reduce_prod, zero_2); + auto num_elems_greater_than_zero = make_shared(num_elems, zero_i64); // compute (rank > 0 && ReduceProd(ShapeOf(x))) > 0 - auto logical_and_2 = make_shared(greater_than_zero, greater_than__zero_2); - - auto logical_or = make_shared(logical_and, logical_and_2); + // it will be a scalar + auto non_scalar_tensor_not_empty = make_shared(rank_greater_than_zero, num_elems_greater_than_zero); - auto tobool = make_shared(logical_or, true_const, false_const); - set_node_name(node.get_name(), tobool); - return tobool->outputs(); + auto to_bool = make_shared(scalar_cond, non_scalar_tensor_not_empty); + set_node_name(node.get_name(), to_bool); + return to_bool->outputs(); } } // namespace op } // namespace tensorflow } // namespace frontend -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/src/frontends/tensorflow_common/src/op/top_k.cpp b/src/frontends/tensorflow_common/src/op/top_k.cpp index c916fe5b746327..4d3bfdcf64bcaf 100644 --- a/src/frontends/tensorflow_common/src/op/top_k.cpp +++ b/src/frontends/tensorflow_common/src/op/top_k.cpp @@ -3,6 +3,7 @@ // #include "common_op_table.hpp" +#include "openvino/op/convert.hpp" #include "openvino/op/topk.hpp" using namespace std; @@ -14,22 +15,35 @@ namespace tensorflow { namespace op { NamedOutputVector translate_top_k_base_op(const NodeContext& node, const ov::Output& k_input, - int min_input_size) { + int min_input_size, + const ov::element::Type& index_type = ov::element::i32) { default_op_checks(node, min_input_size, {"TopK", "TopKV2", "TOPK_V2"}); auto input = node.get_input(0); // retrieve k attribute bool sorted = node.get_attribute("sorted", true); + auto topk_index_type = index_type; + if (index_type == ov::element::i16) { + // v11::TopK supports only int32 and int64 output index type + topk_index_type = ov::element::i32; + } auto top_k = make_shared(input, k_input, -1, ov::op::v11::TopK::Mode::MAX, sorted ? v11::TopK::SortType::SORT_VALUES : v11::TopK::SortType::SORT_INDICES, - ov::element::i32, + topk_index_type, true); + auto values = top_k->output(0); + auto indices = top_k->output(1); + if (index_type != topk_index_type) { + // satisfy the requested output index type + indices = make_shared(indices, index_type)->output(0); + } set_node_name(node.get_name(), top_k); - return {{"values", top_k->output(0)}, {"indices", top_k->output(1)}}; + return {{"values", values}, {"indices", indices}}; } + NamedOutputVector translate_top_k_op(const NodeContext& node) { // retrieve k attribute auto k = node.get_attribute("k"); @@ -39,8 +53,9 @@ NamedOutputVector translate_top_k_op(const NodeContext& node) { NamedOutputVector translate_top_k_v2_op(const NodeContext& node) { default_op_checks(node, 2, {"TopKV2", "TOPK_V2"}); + auto index_type = node.get_attribute("index_type", ov::element::i32); auto k_input = node.get_input(1); - return translate_top_k_base_op(node, k_input, 1); + return translate_top_k_base_op(node, k_input, 1, index_type); } } // namespace op } // namespace tensorflow diff --git a/src/frontends/tensorflow_common/src/op/unique_with_counts.cpp b/src/frontends/tensorflow_common/src/op/unique_with_counts.cpp index 7c5c0a196aa018..d3ca3156ed1347 100644 --- a/src/frontends/tensorflow_common/src/op/unique_with_counts.cpp +++ b/src/frontends/tensorflow_common/src/op/unique_with_counts.cpp @@ -6,6 +6,7 @@ #include "openvino/op/unique.hpp" using namespace std; +using namespace ov; using namespace ov::op; namespace ov { @@ -19,8 +20,9 @@ OutputVector translate_unique_with_counts_op(const NodeContext& node) { // get input 'x' from node and node name auto x = node.get_input(0); auto node_name = node.get_name(); + auto out_idx = node.get_attribute("out_idx", element::i32); - auto unique = make_shared(x, false, ov::element::i32, ov::element::i32); + auto unique = make_shared(x, false, out_idx, out_idx); set_node_name(node_name, unique); // story 'y', 'idx', and 'count' outputs from Unique in separate variables @@ -34,4 +36,4 @@ OutputVector translate_unique_with_counts_op(const NodeContext& node) { } // namespace op } // namespace tensorflow } // namespace frontend -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/src/inference/include/openvino/runtime/properties.hpp b/src/inference/include/openvino/runtime/properties.hpp index 627314748bbe9c..5674c75dd546d7 100644 --- a/src/inference/include/openvino/runtime/properties.hpp +++ b/src/inference/include/openvino/runtime/properties.hpp @@ -580,6 +580,12 @@ static constexpr Property dynamic_quantization */ static constexpr Property kv_cache_precision{"KV_CACHE_PRECISION"}; +/** + * @brief This property scales down activations to prevent overflows when inference precision is f16. + * @ingroup ov_runtime_cpp_prop_api + */ +static constexpr Property activations_scale_factor{"ACTIVATIONS_SCALE_FACTOR"}; + } // namespace hint /** diff --git a/src/plugins/intel_cpu/CMakeLists.txt b/src/plugins/intel_cpu/CMakeLists.txt index 6965b7a25ce512..04909c7d8f5a5a 100644 --- a/src/plugins/intel_cpu/CMakeLists.txt +++ b/src/plugins/intel_cpu/CMakeLists.txt @@ -143,7 +143,7 @@ else() endif() ov_option(ENABLE_MLAS_FOR_CPU "Enable MLAS for OpenVINO CPU Plugin" ${ENABLE_MLAS_FOR_CPU_DEFAULT}) -if(RISCV64_THEAD) +if(RISCV64_XUANTIE) set(ENABLE_SHL_FOR_CPU_DEFAULT ON) else() set(ENABLE_SHL_FOR_CPU_DEFAULT OFF) diff --git a/src/plugins/intel_cpu/src/config.cpp b/src/plugins/intel_cpu/src/config.cpp index 421dca07747932..adcaeaaaa31a6f 100644 --- a/src/plugins/intel_cpu/src/config.cpp +++ b/src/plugins/intel_cpu/src/config.cpp @@ -421,6 +421,13 @@ void Config::readProperties(const ov::AnyMap& prop, const ModelType modelType) { streamsChanged = true; } +#if defined(OV_CPU_WITH_SHL) + // TODO: multi-stream execution is unsafe when SHL is used: + // The library uses global static variables as flags and counters. + streams = 1; + streamsChanged = true; +#endif + this->modelType = modelType; CPU_DEBUG_CAP_ENABLE(applyDebugCapsProperties()); diff --git a/src/plugins/intel_cpu/src/cpu_types.cpp b/src/plugins/intel_cpu/src/cpu_types.cpp index 953f94cb3d5776..e20369c9cca215 100644 --- a/src/plugins/intel_cpu/src/cpu_types.cpp +++ b/src/plugins/intel_cpu/src/cpu_types.cpp @@ -256,7 +256,8 @@ static const TypeToNameMap& get_type_to_name_tbl() { {"LLMMLP", Type::LLMMLP}, {"QKVProjection", Type::QKVProjection}, {"RMS", Type::RMS}, - {"SearchSorted", Type::SearchSorted} + {"SearchSorted", Type::SearchSorted}, + {"LoraSubgraph", Type::LoRA} }; return type_to_name_tbl; } @@ -389,6 +390,7 @@ std::string NameFromType(const Type type) { CASE(QKVProjection); CASE(RMS); CASE(SearchSorted); + CASE(LoRA); CASE(Unknown); } #undef CASE diff --git a/src/plugins/intel_cpu/src/cpu_types.h b/src/plugins/intel_cpu/src/cpu_types.h index c0a2acc3329a9c..d6ac9947a8fb5d 100644 --- a/src/plugins/intel_cpu/src/cpu_types.h +++ b/src/plugins/intel_cpu/src/cpu_types.h @@ -134,6 +134,7 @@ enum class Type { QKVProjection, RMS, SearchSorted, + LoRA }; enum class Algorithm { diff --git a/src/plugins/intel_cpu/src/graph.cpp b/src/plugins/intel_cpu/src/graph.cpp index 45118763a3eaf9..f3f3a379fc2af7 100644 --- a/src/plugins/intel_cpu/src/graph.cpp +++ b/src/plugins/intel_cpu/src/graph.cpp @@ -371,9 +371,20 @@ void Graph::Activate(const std::vector& externalInputMemory, std::tie(m_executableGraphNodes, m_executableSyncNodesInds) = ExtractExecutableNodesAndSyncPoints(syncNodesInds, graphNodes); - status = hasDynNodes ? (parallel_get_max_threads() > 1 ? Status::ReadyDynamic : Status::ReadyDynamicSeq) - : Status::ReadyStatic; - + if (hasDynNodes) { + status = Status::ReadyDynamic; + // Here we use the following heuristic: if the number of sync nodes is less than 10 times of the number of exec + // nodes, it does make sense to use Sequential dynamic shapes processing due to the high overheads on context + // switching when the dynamic shapes are being processed in parallel and there are a lot of sync points. Also + // this rule works for short graphs (usually subgraphs) when the amount of nodes is to low to process them in + // parallel. + const auto exec2sync = m_executableGraphNodes.size() / m_executableSyncNodesInds.size(); + if (exec2sync < 10 || parallel_get_max_threads() < 2) { + status = Status::ReadyDynamicSeq; + } + } else { + status = Status::ReadyStatic; + } CPU_DEBUG_CAP_ENABLE(serialize(*this)); } @@ -1286,23 +1297,40 @@ class UpdateNodes : public UpdateNodesBase { if (origin_nested_levels < 2) { set_max_nested_levels(2); } + // In OpenMP, an exception that is thrown in a parallel region must be caught and handled in the same region by the same thread. + // Therefore, need to pass the error message and throw a new exception outside the parallel region. + const char* what = nullptr; #pragma omp parallel #pragma omp sections { #pragma omp section { - updateDynParams(startCounter, stopIndx); + try { + updateDynParams(startCounter, stopIndx); + } catch (std::exception& e) { + what = e.what(); + } catch (...) { + what = "[ CPU ] Could not update dynamic parameters."; + } } #pragma omp section { - updateShapes(startCounter, stopIndx); + try { + updateShapes(startCounter, stopIndx); + } catch (std::exception& e) { + what = e.what(); + } catch (...) { + what = "[ CPU ] Could not update shapes."; + } } } if (origin_nested_levels != 2) { set_max_nested_levels(origin_nested_levels); } + + OPENVINO_ASSERT(what == nullptr, what); } }; #endif diff --git a/src/plugins/intel_cpu/src/graph.h b/src/plugins/intel_cpu/src/graph.h index b3634800fb2e05..d50ccc152c9186 100644 --- a/src/plugins/intel_cpu/src/graph.h +++ b/src/plugins/intel_cpu/src/graph.h @@ -49,8 +49,16 @@ class Graph { ~Graph(); - bool IsReady() { - return one_of(status, Status::ReadyStatic, Status::ReadyDynamic, Status::ReadyDynamicSeq); + bool IsStatic() const { + return Status::ReadyStatic == status; + } + + bool IsDynamic() const { + return one_of(status, Status::ReadyDynamic, Status::ReadyDynamicSeq); + } + + bool IsReady() const { + return IsStatic() || IsDynamic(); } const Config & getConfig() const { @@ -193,7 +201,6 @@ class Graph { return graphHasDynamicInput; } - Status getStatus() const {return status;} const std::unordered_map& getInternalStateNodes() const; /** @@ -210,6 +217,10 @@ class Graph { void Activate(const std::vector& externalInputMemory = {}, const std::vector& externalOutputMemory = {}); + const std::unordered_map& getOutputNodesMemBlocksMap() const { + return outputNodesMemBlocksMap; + } + protected: void ForgetGraphData() { status = Status::NotReady; @@ -273,7 +284,6 @@ class Graph { template void InferDynamic(SyncInferRequest* request, int numaId, UpdateStrategy&& update); - friend class intel_cpu::SyncInferRequest; friend std::shared_ptr dump_graph_as_ie_ngraph_net(const Graph &graph); private: diff --git a/src/plugins/intel_cpu/src/graph_optimizer.cpp b/src/plugins/intel_cpu/src/graph_optimizer.cpp index 6b3175e24d9dcb..ab7eb223ba17ce 100644 --- a/src/plugins/intel_cpu/src/graph_optimizer.cpp +++ b/src/plugins/intel_cpu/src/graph_optimizer.cpp @@ -183,6 +183,10 @@ void GraphOptimizer::ApplyCommonGraphOptimizations(Graph &graph) { MatchSdpaKvCache(graph); graph.RemoveDroppedNodes(); + OV_ITT_SCOPE_NEXT(FIRST_INFERENCE, taskChain, "DropRedundantMemoryOutput"); + DropRedundantMemoryOutput(graph); + graph.RemoveDroppedNodes(); + OV_ITT_SCOPE_NEXT(FIRST_INFERENCE, taskChain, "RemoveDroppedEdges"); graph.RemoveDroppedEdges(); } @@ -3186,5 +3190,117 @@ void GraphOptimizer::MatchSdpaKvCache(Graph &graph) { } } +void GraphOptimizer::DropRedundantMemoryOutput(Graph &graph) { + // When we have a MemoryInput->MemoryOutput pair, that means that the state is immediately populated with the init + // subgraph values when the init subgraph exists. In all the other cases the state is simply a read only object. + // We can optimize such a case removing the MemoryOutput node and transferring the state values update + // responsibility to a special type of the MemoryInput node - MemoryInputSingle + auto& graphNodes = graph.GetNodes(); + + auto isSuitableMemInput = [](const NodePtr& node) -> bool { + if (Type::MemoryInput != node->getType()) { + return false; + } + + CPU_GRAPH_OPTIMIZER_SCOPE(DropRedundantMemoryOutput_isSuitableMemInput); + + auto memInputBase = std::dynamic_pointer_cast(node); + OPENVINO_ASSERT(memInputBase, + "Unexpectedly wrong dynamic type of node: ", + node->getName(), + " of type: ", + node->getTypeStr()); + + auto id = memInputBase->getId(); + + NodePtr MemoryOutput = nullptr; + auto&& childEdges = node->getChildEdgesAtPort(0); + for (auto&& item : childEdges) { + auto childNode = item->getChild(); + + if (Type::MemoryOutput == childNode->getType()) { + auto memOutputBase = std::dynamic_pointer_cast(childNode); + OPENVINO_ASSERT(memInputBase, + "Unexpectedly wrong dynamic type of node: ", + node->getName(), + " of type: ", + node->getTypeStr()); + + if (memOutputBase->getId() != id) { + return false; // an Assign node from different Variable is attached + } + + if (MemoryOutput && MemoryOutput != childNode) { + //only one child MemoryOutput is expected + return false; + } + MemoryOutput = childNode; + } + } + return nullptr != MemoryOutput; + }; + + for (size_t i = 0; i < graphNodes.size(); i++) { + auto node = graphNodes[i]; + if (!isSuitableMemInput(node)) { + continue; + } + + CPU_GRAPH_OPTIMIZER_SCOPE(DropRedundantMemoryOutput_Node); + + auto memInputNode = std::dynamic_pointer_cast(node); + OPENVINO_ASSERT(memInputNode, "MemoryInput node ", node->getName(), " has unexpected dynamic type"); + + ov::optional inputShape; + ov::optional inputPrc; + + if (!node->getParentEdges().empty()) { + inputShape = ov::optional(node->getInputShapeAtPort(0)); + inputPrc = ov::optional(node->getOriginalInputPrecisionAtPort(0)); + } + + //search for the MemoryOutputNode + NodePtr memoryOutputNode; + for (auto&& edge : node->getChildEdgesAtPort(0)) { + auto child = edge->getChild(); + if (Type::MemoryOutput == child->getType()) { + memoryOutputNode = child; + break; + } + } + OPENVINO_ASSERT(memoryOutputNode, "Corresponding MemoryOutput has not been found"); + + graph.RemoveEdge(memoryOutputNode->getParentEdgeAt(0)); + // there are no output edges from MemoryOutput nodes + + // now replace the existing MemoryInput with a special type that works without the corresponding MemoryOutput + auto memInputSingle = std::make_shared(memInputNode->getId(), + memInputNode->getName(), + memInputNode->getTypeStr(), + memInputNode->getOutputShapeAtPort(0), + memInputNode->getOriginalOutputPrecisionAtPort(0), + graph.getGraphContext(), + inputShape, + inputPrc); + + graph.AddNode(memInputSingle); + + if (!memInputNode->getParentEdges().empty()) { + auto parentEdge = memInputNode->getParentEdgeAt(0); + auto parent = parentEdge->getParent(); + const auto inputNum = parentEdge->getInputNum(); + graph.RemoveEdge(parentEdge); + graph.CreateEdge(parent, memInputSingle, inputNum, 0); + } + + for (auto&& edge : memInputNode->getChildEdgesAtPort(0)) { + auto child = edge->getChild(); + const auto outputNum = edge->getOutputNum(); + graph.RemoveEdge(edge); + graph.CreateEdge(memInputSingle, child, 0, outputNum); + } + } +} + } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/src/graph_optimizer.h b/src/plugins/intel_cpu/src/graph_optimizer.h index 0a85a253ba8d66..886296a7c0053b 100644 --- a/src/plugins/intel_cpu/src/graph_optimizer.h +++ b/src/plugins/intel_cpu/src/graph_optimizer.h @@ -52,6 +52,7 @@ class GraphOptimizer { void RemoveMemoryInputConvert(Graph &graph); void RemoveConvertMemoryOutput(Graph &graph); void MatchSdpaKvCache(Graph &graph); + void DropRedundantMemoryOutput(Graph &graph); bool canBeInplaced(const NodePtr& parentNode, const NodePtr& childNode); // Method checks that after the sequential execution of Transpose and Reorder nodes, diff --git a/src/plugins/intel_cpu/src/infer_request.cpp b/src/plugins/intel_cpu/src/infer_request.cpp index f255a46efe7d0a..f0b817dcda859c 100644 --- a/src/plugins/intel_cpu/src/infer_request.cpp +++ b/src/plugins/intel_cpu/src/infer_request.cpp @@ -140,7 +140,7 @@ void SyncInferRequest::infer() { throw_if_canceled(); // update output control blocks, if any, in order to refresh internal buffers - if (Graph::Status::ReadyDynamic == m_graph->getStatus()) { + if (m_graph->IsDynamic()) { for (auto&& item : m_outputControlBlocks) { item.second.update(); } @@ -178,7 +178,7 @@ void SyncInferRequest::change_default_ptr() { std::unordered_set inputPtrs; std::function& tensor)> changeInpPtr; - if (Graph::Status::ReadyDynamic == m_graph->getStatus()) { + if (m_graph->IsDynamic()) { changeInpPtr = [&inputPtrs](const EdgePtr &edge, ov::SoPtr& tensor) { change_edge_ptr(edge, tensor); inputPtrs.insert(tensor->data()); @@ -278,8 +278,8 @@ void SyncInferRequest::change_default_ptr() { change_edge_ptr(parentEdge, it.second); } - if (Graph::Status::ReadyDynamic == m_graph->getStatus()) { - const auto &outMemBlocksMap = m_graph->outputNodesMemBlocksMap; + if (m_graph->IsDynamic()) { + const auto &outMemBlocksMap = m_graph->getOutputNodesMemBlocksMap(); for (auto&& item : outMemBlocksMap) { const auto& name = item.first; @@ -476,7 +476,7 @@ void SyncInferRequest::init_tensor(const std::size_t& port_index, const ov::ISyn ov::SoPtr tensor; if (type == ov::ISyncInferRequest::FoundPort::Type::INPUT) { - OPENVINO_ASSERT(m_graph->inputNodesMap.find(port_index) != m_graph->inputNodesMap.end(), + OPENVINO_ASSERT(m_graph->GetInputNodesMap().find(port_index) != m_graph->GetInputNodesMap().end(), "Tensor with index: ", port_index, " exists in CPU plugin graph, but absents in model inputs"); @@ -509,7 +509,7 @@ void SyncInferRequest::init_tensor(const std::size_t& port_index, const ov::ISyn } if (type == ov::ISyncInferRequest::FoundPort::Type::OUTPUT) { - const auto& outMap = m_graph->outputNodesMap; + const auto& outMap = m_graph->GetOutputNodesMap(); auto output = outMap.find(port_index); OPENVINO_ASSERT(output != outMap.end(), "Tensor with index: ", diff --git a/src/plugins/intel_cpu/src/memory_state.cpp b/src/plugins/intel_cpu/src/memory_state.cpp index f5f76fe42feb48..bf77917497de77 100644 --- a/src/plugins/intel_cpu/src/memory_state.cpp +++ b/src/plugins/intel_cpu/src/memory_state.cpp @@ -156,6 +156,49 @@ MemoryPtr VariableStateDoubleBuffer::internal_state_mem() const { return prime_mem(); } +VariableStateSingleBuffer::VariableStateSingleBuffer(const std::string& name, + const MemoryPtr& external_buffer, + const MemoryDescPtr& external_desc) + : VariableStateBase(name, external_desc) { + OPENVINO_ASSERT(external_buffer); + m_internal_mem = external_buffer; + m_internal_desc = m_internal_mem->getDescPtr(); + auto&& shape = m_internal_desc->getShape(); + + if (shape.isStatic()) { + m_internal_mem->nullify(); + } else { + // in the case of the original desc has dynamic shape we create an empty tensor + auto new_desc = to_static(m_internal_desc); + m_internal_mem->redefineDesc(new_desc); + } +} +MemoryPtr VariableStateSingleBuffer::input_mem() { + return m_internal_mem; +} +MemoryPtr VariableStateSingleBuffer::output_mem() { + return m_internal_mem; +} +MemoryDescPtr VariableStateSingleBuffer::internal_desc() const { + return m_internal_desc; +} + +void VariableStateSingleBuffer::reset_impl() { + auto new_desc = to_static(m_internal_desc); + if (m_internal_mem) { + m_internal_mem->redefineDesc(new_desc); + m_internal_mem->nullify(); + } +} + +MemoryPtr VariableStateSingleBuffer::internal_state_mem() const { + return m_internal_mem; +} + +void VariableStateSingleBuffer::commit_impl() { + // nothing to do +} + VariableStateKVcache::VariableStateKVcache( const std::string& name, const MemoryDescPtr& external_desc, diff --git a/src/plugins/intel_cpu/src/memory_state.h b/src/plugins/intel_cpu/src/memory_state.h index b4c52903d12f31..e7493f327e93fa 100644 --- a/src/plugins/intel_cpu/src/memory_state.h +++ b/src/plugins/intel_cpu/src/memory_state.h @@ -94,6 +94,27 @@ class VariableStateDoubleBuffer : public VariableStateBase { size_t buffer_num = 0; }; +class VariableStateSingleBuffer : public VariableStateBase { +public: + VariableStateSingleBuffer(const std::string& name, + const MemoryPtr& external_buffer, + const MemoryDescPtr& external_desc); + + MemoryPtr input_mem() override; + MemoryPtr output_mem() override; + MemoryDescPtr internal_desc() const override; + +private: + void reset_impl() override; + void commit_impl() override; + + MemoryPtr internal_state_mem() const override; + +private: + MemoryDescPtr m_internal_desc; //mem desc required by the graph internal tensor + MemoryPtr m_internal_mem; +}; + class VariableStateKVcache : public VariableStateBase { public: VariableStateKVcache(const std::string& name, diff --git a/src/plugins/intel_cpu/src/nodes/composite.cpp b/src/plugins/intel_cpu/src/nodes/composite.cpp index b38a56649bd60a..a1ceabd6942db1 100644 --- a/src/plugins/intel_cpu/src/nodes/composite.cpp +++ b/src/plugins/intel_cpu/src/nodes/composite.cpp @@ -15,11 +15,23 @@ namespace intel_cpu { namespace node { bool Composite::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { - return ov::is_type(op); + try { + if (!ov::is_type(op)) { + errorMessage = "Unknown SubGraph operation : " + std::string(op->get_type_info().name) + " with name '" + + op->get_friendly_name() + "'"; + } + } catch (...) { + return false; + } + return true; } Composite::Composite(const std::shared_ptr& op, const GraphContext::CPtr& context) : Node(op, context, InternalDynShapeInferFactory()) { + std::string errorMessage; + if (!isSupportedOperation(op, errorMessage)) { + OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); + } const auto& subModel = ov::as_type_ptr(op); OPENVINO_ASSERT(subModel, "Attempt to create SubGraph node from an invalid op type: ", op); @@ -27,7 +39,7 @@ Composite::Composite(const std::shared_ptr& op, const GraphContext::CP } void Composite::selectOptimalPrimitiveDescriptor() { - // for the input configution, just always use the parent configuration + // for the input configuration, just always use the parent configuration std::vector inConfs; std::vector graphInputConfig; @@ -38,14 +50,14 @@ void Composite::selectOptimalPrimitiveDescriptor() { } std::vector graphOutputConfig; - for (size_t i = 0; i < getParentEdges().size(); i++) { + for (size_t i = 0; i < outputShapes.size(); i++) { graphOutputConfig.emplace_back(node::Input::OutputConfig{true, true}); } // configure the inner graph to get the information about output memory descriptors m_graph.Init(m_body, context, graphInputConfig, graphOutputConfig); - // for the output decriptors, use the configuration of the graph's output nodes + // for the output descriptors, use the configuration of the graph's output nodes auto outputDescriptors = m_graph.getOutputMemoryDescriptors(); std::vector outConfs; @@ -89,9 +101,6 @@ void Composite::execute(dnnl::stream) { void Composite::executeDynamicImpl(dnnl::stream strm) { execute(strm); - if (!inputShapesModified()) - return; - // since the shape inference is not performed for the composite node // a memory of the extra child edges, attached to the output ports // has to be updated after an inference of the inner graph finished diff --git a/src/plugins/intel_cpu/src/nodes/ctc_loss.cpp b/src/plugins/intel_cpu/src/nodes/ctc_loss.cpp index 78bb6fc0563e60..3161c9a0e87a84 100644 --- a/src/plugins/intel_cpu/src/nodes/ctc_loss.cpp +++ b/src/plugins/intel_cpu/src/nodes/ctc_loss.cpp @@ -84,7 +84,8 @@ void CTCLoss::execute(dnnl::stream strm) { std::vector decodedTargetLenB(batchNum, 0); std::vector> targetDB(batchNum); std::vector>> logProbabilitiesB(batchNum); - std::vector errorMsgB(parallel_get_max_threads()); + const auto threads_num = parallel_get_max_threads(); + std::vector errorMsgB(threads_num); auto threadBody_1 = [&](const int ithr, const int nthr) { size_t start(0lu), end(0lu); @@ -153,7 +154,7 @@ void CTCLoss::execute(dnnl::stream strm) { } // for batch }; // threadBody_1 - parallel_nt(0, threadBody_1); + parallel_nt(threads_num, threadBody_1); if (returnCode != 0) { std::string resErr(""); for (auto& err : errorMsgB) { diff --git a/src/plugins/intel_cpu/src/nodes/deconv.cpp b/src/plugins/intel_cpu/src/nodes/deconv.cpp index 8a7f95268b4f3a..cb340afc029304 100644 --- a/src/plugins/intel_cpu/src/nodes/deconv.cpp +++ b/src/plugins/intel_cpu/src/nodes/deconv.cpp @@ -426,8 +426,10 @@ std::vector Deconvolution::getAvailableFormatsForDims(const else if (dims.getRank() == 2) return {memory::format_tag::nc}; else if (dims.getRank() == 3) - return {memory::format_tag::tnc, memory::format_tag::ntc, - memory::format_tag::ncw, memory::format_tag::nCw8c, memory::format_tag::nCw16c }; + return {memory::format_tag::ncw, + memory::format_tag::nCw8c, + memory::format_tag::nCw16c, + memory::format_tag::nwc}; else if (dims.getRank() == 4) return {memory::format_tag::nchw, memory::format_tag::nChw8c, memory::format_tag::nChw16c, memory::format_tag::nhwc }; diff --git a/src/plugins/intel_cpu/src/nodes/eltwise.cpp b/src/plugins/intel_cpu/src/nodes/eltwise.cpp index 5c3a358dff9d38..c2d23bf9adc89e 100644 --- a/src/plugins/intel_cpu/src/nodes/eltwise.cpp +++ b/src/plugins/intel_cpu/src/nodes/eltwise.cpp @@ -1503,7 +1503,7 @@ class EltwiseJitExecutor : public Eltwise::IEltwiseExecutor { fullWorkAmount *= jep.dims[i]; } - size_t minimalConcurrency = parallel_get_max_threads(); + m_threads_num = static_cast(parallel_get_max_threads()); size_t minimalJitWorkAmount = 256; size_t currentJitWorkAmount = jep.dims[jep.dims.size() - 1]; int collapsedDims = 0; @@ -1516,6 +1516,7 @@ class EltwiseJitExecutor : public Eltwise::IEltwiseExecutor { for (size_t j = 1; j < inpDims.size(); j++) { if (inpDims[j].back() != inpDims[0].back()) { hasDifferentDims = true; + break; } } @@ -1538,7 +1539,7 @@ class EltwiseJitExecutor : public Eltwise::IEltwiseExecutor { } size_t nextJitWorkAmount = currentJitWorkAmount * jep.dims[jep.dims.size() - 2]; - if (fullWorkAmount / nextJitWorkAmount >= minimalConcurrency) { + if (fullWorkAmount / nextJitWorkAmount >= m_threads_num) { currentJitWorkAmount = nextJitWorkAmount; collapsedDims++; @@ -1622,8 +1623,7 @@ class EltwiseJitExecutor : public Eltwise::IEltwiseExecutor { if (_pKernel->jep_.input_size == optimalTensorRank) { // execute Optimized 6D - parallel_for5d(dims_out[0], dims_out[1], dims_out[2], dims_out[3], dims_out[4], - [&](size_t i0, size_t i1, size_t i2, size_t i3, size_t i4) { + auto d6_loop = [&](size_t i0, size_t i1, size_t i2, size_t i3, size_t i4) { auto args = jit_eltwise_call_args_indexes(); args.indexes[0] = i0; args.indexes[1] = i1; @@ -1632,7 +1632,11 @@ class EltwiseJitExecutor : public Eltwise::IEltwiseExecutor { args.indexes[4] = i4; (*_pKernel)(&args_ptrs, &args); - }); + }; + + parallel_nt_static(m_threads_num, [&](const int ithr, const int nthr) { + for_5d(ithr, nthr, dims_out[0], dims_out[1], dims_out[2], dims_out[3], dims_out[4], d6_loop); + }); } else { // execute Optimized Generic if (_pKernel->jep_.use_runtime_ptrs) { @@ -1642,7 +1646,7 @@ class EltwiseJitExecutor : public Eltwise::IEltwiseExecutor { _schedulerWorkAmount *= dims_out[i]; } } - parallel_nt(0, [&](const int ithr, const int nthr) { + parallel_nt(m_threads_num, [&](const int ithr, const int nthr) { size_t start = 0, end = 0; splitter(_schedulerWorkAmount, nthr, ithr, start, end); @@ -1676,6 +1680,7 @@ class EltwiseJitExecutor : public Eltwise::IEltwiseExecutor { std::unique_ptr _pKernel; size_t _schedulerWorkAmount = 0; size_t _batchDimIdx = 0; + size_t m_threads_num = 0lu; public: static const int optimalTensorRank = 6; diff --git a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_convert.cpp b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_convert.cpp index 440af52749bc9c..1bc0585930387f 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/acl/acl_convert.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/acl/acl_convert.cpp @@ -27,12 +27,11 @@ bool ACLConvertExecutor::init(const ConvertParams& convertParams, if (!isCopyOp && dstPrecision == DataType::S8) { dstPrecision = DataType::QASYMM8_SIGNED; } - auto srcDims = srcDesc->getShape().getStaticDims(); - auto dstDims = dstDesc->getShape().getStaticDims(); - auto srcDataLayout = getAclDataLayoutByMemoryDesc(srcDesc); - auto dstDataLayout = getAclDataLayoutByMemoryDesc(dstDesc); - auto srcTensorInfo = TensorInfo(shapeCast(collapse_dims_to_max_rank(srcDims)), 1, srcPrecision, srcDataLayout); - auto dstTensorInfo = TensorInfo(shapeCast(collapse_dims_to_max_rank(dstDims)), 1, dstPrecision, dstDataLayout); + // Use 1D TensorInfo, since UNKNOWN DataLayout may have accuracy issues + auto srcDims1D = convertParams.size; + auto dstDims1D = convertParams.size; + auto srcTensorInfo = TensorInfo(TensorShape(srcDims1D), 1, srcPrecision); + auto dstTensorInfo = TensorInfo(TensorShape(dstDims1D), 1, dstPrecision); if (isCopyOp) { Status s = NECopy::validate(&srcTensorInfo, &dstTensorInfo); if (!s) { diff --git a/src/plugins/intel_cpu/src/nodes/gather.cpp b/src/plugins/intel_cpu/src/nodes/gather.cpp index 81f6f36b84dd89..d2629fe8fe6811 100644 --- a/src/plugins/intel_cpu/src/nodes/gather.cpp +++ b/src/plugins/intel_cpu/src/nodes/gather.cpp @@ -253,6 +253,7 @@ void Gather::createPrimitive() { if (isInPlace()) { return; } + m_threads_num = parallel_get_max_threads(); #if defined(OPENVINO_ARCH_X86_64) uint64_t idxElPerVec = 1; if (!isDynamicNode()) { @@ -294,11 +295,10 @@ void Gather::createPrimitive() { if (!isDynamicNode()) { const uint64_t dataElPerVec = jitKernel->getDataElPerVec(); - const uint64_t nthr = parallel_get_max_threads(); - const uint64_t wpt = ((totalWork / dataElPerVec) / nthr + 1) * dataElPerVec; - execParamsPerThread.resize(nthr); + const uint64_t wpt = ((totalWork / dataElPerVec) / m_threads_num + 1) * dataElPerVec; + execParamsPerThread.resize(m_threads_num); - parallel_nt(nthr, [&](const int ithr, const int nthr) { + parallel_nt(m_threads_num, [&](const int ithr, const int nthr) { const uint64_t dstStart = std::min(wpt * ithr, totalWork); const uint64_t dstEnd = std::min(wpt * (ithr + 1), totalWork); @@ -469,7 +469,7 @@ void Gather::execute(dnnl::stream strm) { (*jitKernel)(&arg); }; - parallel_nt(0, threadBody); + parallel_nt(m_threads_num, threadBody); return; } @@ -543,7 +543,7 @@ void Gather::executeDynamicImpl(dnnl::stream strm) { (*jitKernel)(&arg); }; - parallel_nt(0, threadBody); + parallel_nt(m_threads_num, threadBody); return; } diff --git a/src/plugins/intel_cpu/src/nodes/gather.h b/src/plugins/intel_cpu/src/nodes/gather.h index 96dad228f65b59..6ee097e9a1fbab 100644 --- a/src/plugins/intel_cpu/src/nodes/gather.h +++ b/src/plugins/intel_cpu/src/nodes/gather.h @@ -110,6 +110,7 @@ class Gather : public Node { bool have_scalar_scale = false; size_t zp_group_size = 1u; size_t scale_group_size = 1u; + size_t m_threads_num = 0lu; std::shared_ptr jitKernel; }; diff --git a/src/plugins/intel_cpu/src/nodes/grid_sample.cpp b/src/plugins/intel_cpu/src/nodes/grid_sample.cpp index 618d6b39105689..c8eed21bb312f5 100644 --- a/src/plugins/intel_cpu/src/nodes/grid_sample.cpp +++ b/src/plugins/intel_cpu/src/nodes/grid_sample.cpp @@ -149,11 +149,11 @@ void GridSample::createPrimitive() { } jitKernel->create_ker(); - nthr = parallel_get_max_threads(); - execParamsPerThread.resize(nthr); + m_threads_num = parallel_get_max_threads(); + execParamsPerThread.resize(m_threads_num); if (!x64::mayiuse(x64::avx512_core)) { const auto dataElPerVec = jitKernel->getDataElPerVec(); - parallel_nt(nthr, [&](const int ithr, const int nthr) { + parallel_nt(m_threads_num, [&](const int ithr, const int nthr) { auto& p = execParamsPerThread[ithr]; p.srcHeightF.resize(dataElPerVec); @@ -197,9 +197,9 @@ void GridSample::prepareParams() { const auto& srcDataShape = dataMemPtr->getStaticDims(); const auto& dstShape = dstMemPtr->getStaticDims(); const uint64_t totalWork = dstShape[2] * dstShape[3]; - const uint64_t wpt = ((totalWork / dataElPerVec) / nthr + 1) * dataElPerVec; + const uint64_t wpt = ((totalWork / dataElPerVec) / m_threads_num + 1) * dataElPerVec; - parallel_nt(nthr, [&](const int ithr, const int nthr) { + parallel_nt(m_threads_num, [&](const int ithr, const int nthr) { const uint64_t dstStart = std::min(wpt * ithr, totalWork); const uint64_t dstEnd = std::min(wpt * (ithr + 1), totalWork); @@ -303,7 +303,7 @@ void GridSample::execute(dnnl::stream strm) { (*jitKernel)(&arg); }; - parallel_nt(nthr, threadBody); + parallel_nt(m_threads_num, threadBody); } void GridSample::executeDynamicImpl(dnnl::stream strm) { diff --git a/src/plugins/intel_cpu/src/nodes/grid_sample.hpp b/src/plugins/intel_cpu/src/nodes/grid_sample.hpp index 0d172bd5c3e055..b4468d58be9b52 100644 --- a/src/plugins/intel_cpu/src/nodes/grid_sample.hpp +++ b/src/plugins/intel_cpu/src/nodes/grid_sample.hpp @@ -62,7 +62,7 @@ class GridSample : public Node { ov::element::Type dataPrecision; ov::element::Type gridPrecision = ov::element::f32; - int nthr = 1; + size_t m_threads_num = 0lu; std::vector execParamsPerThread; static constexpr size_t IN_DATA = 0; diff --git a/src/plugins/intel_cpu/src/nodes/input.cpp b/src/plugins/intel_cpu/src/nodes/input.cpp index 4ee5707e0a9e76..1f650bd8c5de17 100644 --- a/src/plugins/intel_cpu/src/nodes/input.cpp +++ b/src/plugins/intel_cpu/src/nodes/input.cpp @@ -430,6 +430,7 @@ Input::Input(const std::shared_ptr& op, const GraphContext::CPtr context, OutputConfig config) : Input(op, context) { + extMemDesc = config.desc; m_useParentMemoryDescForOutput = config.useParentMemoryDescForOutput; m_isInPlace = config.inPlace; } diff --git a/src/plugins/intel_cpu/src/nodes/input.h b/src/plugins/intel_cpu/src/nodes/input.h index a954ce56665d61..4d7febb17ad4b7 100644 --- a/src/plugins/intel_cpu/src/nodes/input.h +++ b/src/plugins/intel_cpu/src/nodes/input.h @@ -19,9 +19,17 @@ class Input : public Node { }; struct OutputConfig { + OutputConfig() = default; + OutputConfig(bool useParentMemoryDesc_, bool inPlace_) + : useParentMemoryDescForOutput(useParentMemoryDesc_), + inPlace(inPlace_) {} + + OutputConfig(MemoryDescPtr desc_, bool inPlace_) : desc(std::move(desc_)), inPlace(inPlace_) {} + // @todo better to use memory desc with any layout and undefined precision - bool useParentMemoryDescForOutput; - bool inPlace; + MemoryDescPtr desc = nullptr; + bool useParentMemoryDescForOutput = false; + bool inPlace = false; }; Input(const std::shared_ptr& op, const GraphContext::CPtr context); diff --git a/src/plugins/intel_cpu/src/nodes/kernels/scaled_attn/mha_single_token.cpp b/src/plugins/intel_cpu/src/nodes/kernels/scaled_attn/mha_single_token.cpp index 1543c168403382..25ddbb1b4246b1 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/scaled_attn/mha_single_token.cpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/scaled_attn/mha_single_token.cpp @@ -1068,11 +1068,15 @@ static void mha_single_token_kernel(const ov::intel_cpu::PlainTensor& query, } }); - parallel_for3d(B, H, q_len, [&](size_t b, size_t h, size_t pq) { + auto bhl_loop = [&](size_t b, size_t h, size_t pq) { auto* temp = buf_attn_score.ptr(0, b, pq, h); size_t temp_stride = buf_attn_score.stride(0); auto* dst = has_out_transpose ? output_emb.ptr(b, pq, h * SV) : output_emb.ptr(b, h, pq); attn_reduce(dst, temp, nthr, SV, temp_stride); + }; + + parallel_nt_static(nthr, [&](const int ithr, const int nthr) { + for_3d(ithr, nthr, B, H, q_len, bhl_loop); }); } @@ -1144,7 +1148,7 @@ void mha_single_token(const ov::intel_cpu::PlainTensor& query, past_v_scale_zp, head_sum); } else { - OPENVINO_THROW("Unsupported precision: ", query.get_precision()); + OPENVINO_THROW("Unsupported precision: ", present_key.get_precision()); } #else if (present_key.get_precision() == ov::element::u8) { diff --git a/src/plugins/intel_cpu/src/nodes/kernels/x64/rms_kernel.cpp b/src/plugins/intel_cpu/src/nodes/kernels/x64/rms_kernel.cpp index 719e9a6e464934..30a7870a1a4b54 100644 --- a/src/plugins/intel_cpu/src/nodes/kernels/x64/rms_kernel.cpp +++ b/src/plugins/intel_cpu/src/nodes/kernels/x64/rms_kernel.cpp @@ -123,22 +123,25 @@ void jit_rms_kernel::generate() { // x * 1/Sqrt(ReduceMean(x^2,axes)+eps) * gamma // sum(x^2) align(16); - Xbyak::Label loop_4reg; - L(loop_4reg); - { - load(vmm_src, reg_src, m_jcp.src_prc, vec_size, false); - vfmadd231ps(vmm_sum0, vmm_src, vmm_src); - load(vmm_src, reg_src, m_jcp.src_prc, vec_size, false, vec_size * m_jcp.src_prc.size() * 1); - vfmadd231ps(vmm_sum1, vmm_src, vmm_src); - load(vmm_src, reg_src, m_jcp.src_prc, vec_size, false, vec_size * m_jcp.src_prc.size() * 2); - vfmadd231ps(vmm_sum2, vmm_src, vmm_src); - load(vmm_src, reg_src, m_jcp.src_prc, vec_size, false, vec_size * m_jcp.src_prc.size() * 3); - vfmadd231ps(vmm_sum3, vmm_src, vmm_src); - - add(reg_src, vec_size * m_jcp.src_prc.size() * 4); - dec(reg_size); - jnz(loop_4reg); + if ((m_jcp.data_size / (vec_size * 4)) != 0) { + Xbyak::Label loop_4reg; + L(loop_4reg); + { + load(vmm_src, reg_src, m_jcp.src_prc, vec_size, false); + vfmadd231ps(vmm_sum0, vmm_src, vmm_src); + load(vmm_src, reg_src, m_jcp.src_prc, vec_size, false, vec_size * m_jcp.src_prc.size() * 1); + vfmadd231ps(vmm_sum1, vmm_src, vmm_src); + load(vmm_src, reg_src, m_jcp.src_prc, vec_size, false, vec_size * m_jcp.src_prc.size() * 2); + vfmadd231ps(vmm_sum2, vmm_src, vmm_src); + load(vmm_src, reg_src, m_jcp.src_prc, vec_size, false, vec_size * m_jcp.src_prc.size() * 3); + vfmadd231ps(vmm_sum3, vmm_src, vmm_src); + + add(reg_src, vec_size * m_jcp.src_prc.size() * 4); + dec(reg_size); + jnz(loop_4reg); + } } + // 1 ~ 3 vmm for (size_t i = m_jcp.data_size / (vec_size * 4) * 4; i < m_jcp.data_size / vec_size; i++) { load(vmm_src, reg_src, m_jcp.src_prc, vec_size, false); diff --git a/src/plugins/intel_cpu/src/nodes/llm_mlp.cpp b/src/plugins/intel_cpu/src/nodes/llm_mlp.cpp index 13c46a7c976cfd..8df1f5498da384 100644 --- a/src/plugins/intel_cpu/src/nodes/llm_mlp.cpp +++ b/src/plugins/intel_cpu/src/nodes/llm_mlp.cpp @@ -53,19 +53,19 @@ class LinearKsplit2 { OPENVINO_ASSERT((N % REG_BLK_N_SIZE) == 0); OPENVINO_ASSERT((K % reg_blk_K_size) == 0); - auto nthr = parallel_get_max_threads(); + m_threads_num = parallel_get_max_threads(); auto num_blk_N = N / REG_BLK_N_SIZE; - works.resize(nthr); + works.resize(m_threads_num); auto K_splits = 2; // split task on more cores is better on TBB - auto valid_nthr = nthr / 2; + auto valid_nthr = m_threads_num / 2; auto blkN_per_thread = (num_blk_N) / valid_nthr; auto blkN_leftover = num_blk_N - (blkN_per_thread * valid_nthr); auto start_blkN = 0; used_nthr = 0; - for (int ithr = 0; ithr < nthr; ithr += K_splits) { + for (int ithr = 0; ithr < m_threads_num; ithr += K_splits) { auto blkN = std::min(num_blk_N - start_blkN, blkN_per_thread); if (blkN_leftover > 0) { blkN_leftover--; @@ -106,7 +106,7 @@ class LinearKsplit2 { wbuffer.alloc(works, weight_element_size); - ov::parallel_nt_static(0, [&](const size_t ithr, const size_t nthr) { + ov::parallel_nt_static(m_threads_num, [&](const size_t ithr, const size_t nthr) { auto& work = works[ithr]; if (work) { if (is_quantized) { @@ -125,7 +125,7 @@ class LinearKsplit2 { float * w_scale) { static ReduceAdd2bh jit_reduce2cvt(true, std::is_same::value); - ov::parallel_nt_static(0, [&](const size_t ithr, const size_t nthr) { + ov::parallel_nt_static(m_threads_num, [&](const size_t ithr, const size_t nthr) { auto& work = works[ithr]; auto& workC = work.m_C; if (work) { @@ -165,6 +165,9 @@ class LinearKsplit2 { } }); } + +private: + int m_threads_num = 0; }; template @@ -205,18 +208,18 @@ class LinearGateUp { // in unit of 32 OPENVINO_ASSERT((N % REG_BLK_N_SIZE) == 0); OPENVINO_ASSERT((K % reg_blk_K_size) == 0); - auto nthr = parallel_get_max_threads(); + m_threads_num = parallel_get_max_threads(); auto num_blk_N = N / REG_BLK_N_SIZE; - works.resize(nthr); + works.resize(m_threads_num); // split task on more cores is better on TBB - auto valid_nthr = nthr; + auto valid_nthr = m_threads_num; auto blkN_per_thread = (num_blk_N) / valid_nthr; auto blkN_leftover = num_blk_N - (blkN_per_thread * valid_nthr); auto start_blkN = 0; used_nthr = 0; - for (int ithr = 0; ithr < nthr; ithr ++) { + for (int ithr = 0; ithr < m_threads_num; ithr ++) { auto blkN = std::min(num_blk_N - start_blkN, blkN_per_thread); if (blkN_leftover > 0) { blkN_leftover--; @@ -243,7 +246,7 @@ class LinearGateUp { wbuffer.alloc(works, weight_element_size); DEBUG_LOG("Linear N,K=", N, ",", K, " used_nthr=", used_nthr); - ov::parallel_nt_static(0, [&](const size_t ithr, const size_t nthr) { + ov::parallel_nt_static(m_threads_num, [&](const size_t ithr, const size_t nthr) { auto& work = works[ithr]; if (work) { if (quantized_int8) @@ -267,7 +270,7 @@ class LinearGateUp { const LLMMLPNode::Config& config, MatrixDynQuantPerRow& src_dq, float * w_scale) { - ov::parallel_nt_static(0, [&](const size_t ithr, const size_t nthr) { + ov::parallel_nt_static(m_threads_num, [&](const size_t ithr, const size_t nthr) { auto& work = works[ithr]; if (work) { work.run(M, pA, strideA_in_bytes); @@ -303,6 +306,9 @@ class LinearGateUp { } }); } + +private: + int m_threads_num = 0; }; template @@ -384,8 +390,8 @@ struct LLMMLP::Executor : public LLMMLP::ExecutorBase { reinterpret_cast(ptr)); }); - auto nthr = parallel_get_max_threads(); - for (int ithr = 0; ithr < nthr; ithr++) { + m_threads_num = parallel_get_max_threads(); + for (size_t ithr = 0lu; ithr < m_threads_num; ithr++) { auto C1_size = gate_up.works[ithr].set_C(M, reinterpret_cast(cur_scratch_base)); auto C2_size = down.works[ithr].set_C(M, reinterpret_cast(cur_scratch_base)); auto max_C_size = std::max(C1_size, C2_size); @@ -482,6 +488,9 @@ struct LLMMLP::Executor : public LLMMLP::ExecutorBase { dstC += BM * strideC / sizeof(T); } } + +private: + size_t m_threads_num = 0lu; }; #else template diff --git a/src/plugins/intel_cpu/src/nodes/lora.cpp b/src/plugins/intel_cpu/src/nodes/lora.cpp new file mode 100644 index 00000000000000..52509cdfc44a13 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/lora.cpp @@ -0,0 +1,126 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "lora.h" + +#include "nodes/input.h" +#include "cpu_memory.h" +#include "ov_ops/lora_subgraph.hpp" +#include "utils/debug_capabilities.h" +#include "shape_inference/shape_inference_pass_through.hpp" + +namespace ov { +namespace intel_cpu { +namespace node { + +bool LoRA::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { + try { + if (!ov::is_type(op)) { + errorMessage = "Unknown LoRA operation : " + std::string(op->get_type_info().name) + " with name '" + + op->get_friendly_name() + "'"; + } + } catch (...) { + return false; + } + return true; +} + +LoRA::LoRA(const std::shared_ptr& op, const GraphContext::CPtr& context) + : Node(op, context, PassThroughShapeInferFactory()) { + std::string errorMessage; + if (!isSupportedOperation(op, errorMessage)) { + OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); + } + const auto& loraModel = ov::as_type_ptr(op); + OPENVINO_ASSERT(loraModel, + "Attempt to create LoRA node from an invalid op type: ", + op, + " with name ", + op->get_friendly_name()); + + m_body = loraModel->get_function(); +} + +void LoRA::selectOptimalPrimitiveDescriptor() { + // for the input configuration, just always use the parent configuration + std::vector inConfs; + std::vector graphInputConfig; + + auto mainInputDesc = getParentOutputMemDesc(getParentEdgeAt(0)); + auto mainInputPrc = mainInputDesc->getPrecision(); // we have to align precision across all the inputs + + inConfs.emplace_back(mainInputDesc); + graphInputConfig.emplace_back(node::Input::InputConfig{mainInputDesc, true}); + + for (size_t i = 1; i < getParentEdges().size(); i++) { + auto desc = getParentOutputMemDesc(getParentEdgeAt(i))->cloneWithNewPrecision(mainInputPrc); + inConfs.emplace_back(desc); + graphInputConfig.emplace_back(node::Input::InputConfig{desc, true}); + } + + std::vector graphOutputConfig; + // enforce the same memory descriptor on the output as on the input to allow inPlace memory + graphOutputConfig.emplace_back(node::Input::OutputConfig{inConfs.front().getMemDesc(), true}); + + // configure the inner graph to get the information about output memory descriptors + m_graph.Init(m_body, context, graphInputConfig, graphOutputConfig); + + // for the output descriptors, use the configuration of the graph's output nodes + auto outputDescriptors = m_graph.getOutputMemoryDescriptors(); + + const auto& desc = outputDescriptors.front(); + + // just a sanity check + CPU_NODE_ASSERT(desc->isCompatible(*(inConfs.front().getMemDesc())), "Unexpected input/output descriptor mismatch"); + + std::vector outConfs; + + outConfs.emplace_back(desc, BlockedMemoryDesc::FULL_MASK, 0); // use the memory from the first input inPlace + + const NodeConfig config(inConfs, outConfs); + + supportedPrimitiveDescriptors.clear(); + supportedPrimitiveDescriptors.emplace_back(config, impl_desc_type::undef); + + selectPrimitiveDescriptorByIndex(0); +} + +// @todo add ascii diagram for memory mapping / reuse +void LoRA::createPrimitive() { + CPU_NODE_ASSERT(getOriginalInputsNumber() == m_graph.GetInputNodesMap().size(), + "Number of node inputs must be equal the number of inner graph's inputs"); + + std::vector inputMemory; + for (size_t i = 0; i < getOriginalInputsNumber(); i++) { + auto srcEdgeMem = getSrcMemoryAtPort(i); + auto mem = std::make_shared(getEngine(), srcEdgeMem->getDescPtr(), srcEdgeMem->getMemoryBlock()); + subgraphMemoryPtrs.push_back(mem); + inputMemory.emplace_back(std::move(mem)); + } + + CPU_NODE_ASSERT(getOriginalOutputsNumber() == m_graph.GetOutputNodesMap().size(), + "Number of node outputs must be equal the number of inner graph's outputs"); + + std::vector outputMemory{getDstMemoryAtPort(0)}; + m_graph.Activate(inputMemory, outputMemory); +} + +void LoRA::execute(dnnl::stream) { + m_graph.Infer(); +} + +void LoRA::executeDynamicImpl(dnnl::stream strm) { + execute(strm); +} + +void LoRA::prepareParams() { + for (size_t i = 0; i < getOriginalInputsNumber(); i++) { + // since the external and internal descriptors are compatible, we may pass the descriptor + subgraphMemoryPtrs[i]->redefineDesc(getSrcMemoryAtPort(i)->getDescPtr()); + } +} + +} // namespace node +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/lora.h b/src/plugins/intel_cpu/src/nodes/lora.h new file mode 100644 index 00000000000000..27701daf9034f2 --- /dev/null +++ b/src/plugins/intel_cpu/src/nodes/lora.h @@ -0,0 +1,39 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "graph.h" +#include "node.h" + +namespace ov { +namespace intel_cpu { +namespace node { + +class LoRA : public Node { +public: + static bool isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept; + + LoRA(const std::shared_ptr& op, const GraphContext::CPtr& context); + + bool created() const override { + return getType() == Type::LoRA; + } + + void getSupportedDescriptors() override{}; + void selectOptimalPrimitiveDescriptor() override; + void createPrimitive() override; + void prepareParams() override; + void execute(dnnl::stream) override; + void executeDynamicImpl(dnnl::stream strm) override; + +private: + std::shared_ptr m_body; + std::vector subgraphMemoryPtrs; + Graph m_graph; +}; + +} // namespace node +} // namespace intel_cpu +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/memory.cpp b/src/plugins/intel_cpu/src/nodes/memory.cpp index 756fbc5b578f61..565597bdcc2a9e 100644 --- a/src/plugins/intel_cpu/src/nodes/memory.cpp +++ b/src/plugins/intel_cpu/src/nodes/memory.cpp @@ -377,7 +377,8 @@ bool MemoryInputBase::isSupportedOperation(const std::shared_ptr } MemoryInputBase::MemoryInputBase(const std::shared_ptr& op, const GraphContext::CPtr ctx) - : Input(op, ctx), MemoryStateNode(op) { + : Input(op, ctx), + MemoryStateNode(op) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { OPENVINO_THROW_NOT_IMPLEMENTED(errorMessage); @@ -385,6 +386,7 @@ MemoryInputBase::MemoryInputBase(const std::shared_ptr& op, const Grap if (created()) { context->getMemoryStatesRegister()->registerInput(this); } + executeHook = &MemoryInputBase::assignState; } MemoryInputBase::MemoryInputBase(const std::string id, @@ -394,8 +396,10 @@ MemoryInputBase::MemoryInputBase(const std::string id, const ov::element::Type& output_prc, const GraphContext::CPtr context, const ov::optional& input_shape, - const ov::optional& input_prc) : - Input(output_shape, output_prc, name, type, context), MemoryStateNode(id) { + const ov::optional& input_prc, + MemoryInputBase::mode mode) + : Input(output_shape, output_prc, name, type, context), + MemoryStateNode(id) { outputShapes.emplace_back(output_shape); addOriginalOutputPrecision(output_prc); if (input_shape) { @@ -411,6 +415,17 @@ MemoryInputBase::MemoryInputBase(const std::string id, if (created()) { context->getMemoryStatesRegister()->registerInput(this); } + + // this important to prevent identifying it as a const when it's on a const path + constant = ConstantType::StrictNoConst; + + if (mode::read_value_assign == mode) { + executeHook = &MemoryInputBase::assignState; + } else if (mode::single_read_value == mode) { + executeHook = &MemoryInputBase::bypassAssignState; + } else { + THROW_CPU_NODE_ERR("Unexpected MemoryInput mode"); + } } MemoryInputBase::~MemoryInputBase() { @@ -513,15 +528,26 @@ void MemoryInputBase::assignState(MemStatePtr newState) { } void MemoryInputBase::execute(dnnl::stream strm) { - getOutputNode().assignState(getAssignedState()); + assert(executeHook && "executeHook is not initialized!"); + (this->*executeHook)(); runStatic(strm); } void MemoryInputBase::executeDynamicImpl(dnnl::stream strm) { - getOutputNode().assignState(getAssignedState()); + assert(executeHook && "executeHook is not initialized!"); + (this->*executeHook)(); runDynamic(strm); } +void MemoryInputBase::assignState() { + getOutputNode().assignState(getAssignedState()); +} + +void MemoryInputBase::bypassAssignState() { + // nothing to do + return; +} + bool MemoryInput::needInitGraphProcessing() const { return !getParentEdges().empty() && getAssignedState()->is_reset_state(); } @@ -828,6 +854,89 @@ void MemoryInputSDPA::resolveInPlaceEdges(Edge::LOOK look) { } } +MemoryInputSingle::MemoryInputSingle(const std::string id, + const std::string& name, + const std::string& type, + const Shape& output_shape, + const ov::element::Type& output_prc, + const GraphContext::CPtr context, + const ov::optional& input_shape, + const ov::optional& input_prc) + : MemoryInput(id, + name, + type, + output_shape, + output_prc, + context, + input_shape, + input_prc, + MemoryInputBase::mode::single_read_value) {} + +MemStatePtr MemoryInputSingle::makeState() const { + // assume ov::Tensor is always dense + auto original_desc = + std::make_shared(getOriginalOutputPrecisionAtPort(0), outputShapes.at(0)); + + auto mem_desc = getBaseMemDescAtOutputPort(0); + const auto& eng = getEngine(); + + auto state_name = getId(); + + // Remove suffix with pair ID. Internal information. + auto suffix_idx = state_name.find("/id="); + if (suffix_idx != std::string::npos) { + state_name = state_name.substr(0, suffix_idx); + } + + return std::make_shared(state_name, + std::make_shared(eng, mem_desc), + original_desc); +} + +void MemoryInputSingle::runStatic(dnnl::stream strm) { + MemoryInput::runStatic(strm); + if (needInitGraphProcessing()) { + // since there is no corresponding MemoryOutput node, we need to update the state here + auto result = getDstMemoryAtPort(0); // only one output port + auto stateMem = getAssignedState()->output_mem(); + CPU_NODE_ASSERT(stateMem, " state memory has nullptr"); + if (result->getData() != stateMem->getData()) { + stateMem->load(*result); + } + } + getAssignedState()->commit(); // since we don't use MemoryOutput, commit must be called to change the reset state +} + +void MemoryInputSingle::runDynamic(dnnl::stream strm) { + MemoryInput::runDynamic(strm); + if (needInitGraphProcessing()) { + // since there is no corresponding MemoryOutput node, we need to update the state here + auto result = getDstMemoryAtPort(0); // only one output port + auto state = getAssignedState(); + auto stateMem = state->output_mem(); + CPU_NODE_ASSERT(stateMem, " state memory has nullptr"); + + const auto& newShape = result->getShape(); + const auto& stateShape = stateMem->getShape(); + + if (stateShape.isDynamic() || stateShape.getStaticDims() != newShape.getStaticDims()) { + auto extMemDesc = state->internal_desc(); + auto newExternDesc = extMemDesc->cloneWithNewDims(newShape.getStaticDims()); + stateMem->redefineDesc(newExternDesc); + } + + if (result->getData() != stateMem->getData()) { + stateMem->load(*result); + } + } + getAssignedState()->commit(); // since we don't use MemoryOutput, commit must be called to change the reset state +} + +bool MemoryInputSingle::isSupportedOperation(const std::shared_ptr& op, + std::string& errorMessage) noexcept { + return MemoryInput::isSupportedOperation(op, errorMessage); +} + } // namespace node } // namespace intel_cpu } // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/memory.hpp b/src/plugins/intel_cpu/src/nodes/memory.hpp index c158d738a36148..f503a8d58386a5 100644 --- a/src/plugins/intel_cpu/src/nodes/memory.hpp +++ b/src/plugins/intel_cpu/src/nodes/memory.hpp @@ -120,16 +120,14 @@ class MemoryOutputStub : public MemoryOutputBase { }; class MemoryInputBase : public Input, public MemoryStateNode { +public: + enum class mode { + read_value_assign, + single_read_value + }; + public: MemoryInputBase(const std::shared_ptr& op, const GraphContext::CPtr context); - MemoryInputBase(const std::string id, - const std::string& name, - const std::string& type, - const Shape& output_shape, - const ov::element::Type& output_prc, - const GraphContext::CPtr context, - const ov::optional& input_shape, - const ov::optional& input_prc); ~MemoryInputBase() override; @@ -152,6 +150,17 @@ class MemoryInputBase : public Input, public MemoryStateNode { MemoryOutputBase& getOutputNode(); void assignState(MemStatePtr newState) override final; // NOLINT +protected: + MemoryInputBase(const std::string id, + const std::string& name, + const std::string& type, + const Shape& output_shape, + const ov::element::Type& output_prc, + const GraphContext::CPtr context, + const ov::optional& input_shape, + const ov::optional& input_prc, + mode mode = mode::read_value_assign); + protected: virtual void runStatic(dnnl::stream strm) = 0; virtual void runDynamic(dnnl::stream strm) = 0; @@ -160,12 +169,20 @@ class MemoryInputBase : public Input, public MemoryStateNode { return state; } +private: + using executeHookPtr = void (MemoryInputBase::*)(void); + +private: + void assignState(); + void bypassAssignState(); + private: /** * @brief keeps reference to output sibling node */ MemoryOutputBase* outputNode = nullptr; MemStatePtr state = nullptr; + executeHookPtr executeHook; }; class MemoryInput : public MemoryInputBase { @@ -179,16 +196,38 @@ class MemoryInput : public MemoryInputBase { MemStatePtr makeState() const override; -private: +protected: + bool needInitGraphProcessing() const; void runStatic(dnnl::stream strm) override; void runDynamic(dnnl::stream strm) override; + +private: void assignStateHook() override {/*pass*/} - bool needInitGraphProcessing() const; private: ProxyMemoryBlockPtr memBlock = nullptr; }; +class MemoryInputSingle : public MemoryInput { +public: + MemoryInputSingle(const std::string id, + const std::string& name, + const std::string& type, + const Shape& output_shape, + const ov::element::Type& output_prc, + const GraphContext::CPtr context, + const ov::optional& input_shape, + const ov::optional& input_prc); + + static bool isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept; + + MemStatePtr makeState() const override; + +private: + void runStatic(dnnl::stream strm) override; + void runDynamic(dnnl::stream strm) override; +}; + class MemoryInputSDPA : public MemoryInputBase { public: MemoryInputSDPA(const std::string id, diff --git a/src/plugins/intel_cpu/src/nodes/mha.cpp b/src/plugins/intel_cpu/src/nodes/mha.cpp index 7d082e99fa4f6a..9364058c5d19a2 100644 --- a/src/plugins/intel_cpu/src/nodes/mha.cpp +++ b/src/plugins/intel_cpu/src/nodes/mha.cpp @@ -934,7 +934,7 @@ void MHA::prepareParams() { bool isAMXSupported = mayiuse(avx512_core_amx); - size_t numThreads = parallel_get_max_threads(); + m_threads_num = parallel_get_max_threads(); size_t matmulOptimalM = 32; @@ -1072,21 +1072,21 @@ void MHA::prepareParams() { bufferCompensation1Size = rnd_up(N1, N1_blk); if (brgCopyAKernel0) { - bufferMatMul0In0.resize(numThreads * bufferMatMul0In0Size); + bufferMatMul0In0.resize(m_threads_num * bufferMatMul0In0Size); } - bufferMatMul0In1.resize(numThreads * bufferMatMul0In1Size); - bufferMatMul0Out.resize(numThreads * bufferMatMul0OutSize); - bufferMatMul1In1.resize(numThreads * bufferMatMul1In1Size); - bufferMatMul1Out.resize(numThreads * bufferMatMul1OutSize); + bufferMatMul0In1.resize(m_threads_num * bufferMatMul0In1Size); + bufferMatMul0Out.resize(m_threads_num * bufferMatMul0OutSize); + bufferMatMul1In1.resize(m_threads_num * bufferMatMul1In1Size); + bufferMatMul1Out.resize(m_threads_num * bufferMatMul1OutSize); if (brgemmCtx0.is_with_comp) { - bufferCompensation0.resize(numThreads * bufferCompensation0Size); + bufferCompensation0.resize(m_threads_num * bufferCompensation0Size); } if (brgemmCtx1.is_with_comp) { - bufferCompensation1.resize(numThreads * bufferCompensation1Size); + bufferCompensation1.resize(m_threads_num * bufferCompensation1Size); } if (brgemmCtx0.is_with_amx || brgemmCtx1.is_with_amx) { - wsp.resize(numThreads * wsp_size_per_thread); + wsp.resize(m_threads_num * wsp_size_per_thread); } { @@ -1224,7 +1224,7 @@ void MHA::mhaImpl() { auto outPrcSize = outputPrecision.size(); - parallel_for2d(dimsMatMul0Out[0], dimsMatMul0Out[1], [&](size_t i0, size_t i1) { + auto spatial_loop = [&](size_t i0, size_t i1) { size_t threadNum = parallel_get_thread_num(); auto pTranspose0In0_aux = pTranspose0In0 + (i0 * strTranspose0In0[0] + i1 * strTranspose0In0[2]) * inputPrecisions[0].size(); // order 0213 @@ -1417,6 +1417,10 @@ void MHA::mhaImpl() { (*convertReorderKernel)(&call_args); } } + }; + + parallel_nt_static(m_threads_num, [&](const int ithr, const int nthr) { + for_2d(ithr, nthr, dimsMatMul0Out[0], dimsMatMul0Out[1], spatial_loop); }); } diff --git a/src/plugins/intel_cpu/src/nodes/mha.h b/src/plugins/intel_cpu/src/nodes/mha.h index cd272c086e2190..36afe20224299a 100644 --- a/src/plugins/intel_cpu/src/nodes/mha.h +++ b/src/plugins/intel_cpu/src/nodes/mha.h @@ -238,6 +238,8 @@ class MHA : public Node { std::unique_ptr mulAddSoftmaxKernel; std::unique_ptr convertReorderKernel; std::unique_ptr convertTransposeKernel; + + size_t m_threads_num = 0lu; }; } // namespace node diff --git a/src/plugins/intel_cpu/src/nodes/mvn.cpp b/src/plugins/intel_cpu/src/nodes/mvn.cpp index 61aa4738b8f81f..76471b0cca741d 100644 --- a/src/plugins/intel_cpu/src/nodes/mvn.cpp +++ b/src/plugins/intel_cpu/src/nodes/mvn.cpp @@ -2417,9 +2417,9 @@ void MVN::MVNJitExecutor::mvn_nspc(const uint8_t* src_data, uint8_t* dst_data, c const size_t H = shape5d[3]; const size_t W = shape5d[4]; - size_t threads_num = parallel_get_max_threads(); + const size_t threads_num = parallel_get_max_threads(); size_t aux_buffer_size = mvnAttrs.execAcrossChannels_ ? 1 : rnd_up(C, blk_size) + blk_size; - parallel_for(N, [&](size_t b) { + auto b_loop = [&](size_t b) { std::vector mean_buffer(aux_buffer_size * threads_num, 0.f); std::vector variance_buffer; if (mvnAttrs.normalizeVariance_) { @@ -2429,7 +2429,7 @@ void MVN::MVNJitExecutor::mvn_nspc(const uint8_t* src_data, uint8_t* dst_data, c // kernel_type: 0 for mean, 1 for variance, 2 for normalization auto worker = [&](const bool across_channel, const int kernel_type) { - parallel_nt(0, [&](const int ithr, const int nthr) { + parallel_nt(threads_num, [&](const int ithr, const int nthr) { size_t start = 0, end = 0; splitter(D * H * W, nthr, ithr, start, end); @@ -2512,6 +2512,10 @@ void MVN::MVNJitExecutor::mvn_nspc(const uint8_t* src_data, uint8_t* dst_data, c } worker(false, 2); } + }; + + parallel_nt_static(threads_num, [&](const int ithr, const int nthr) { + for_1d(ithr, nthr, N, b_loop); }); } @@ -2529,15 +2533,15 @@ void MVN::MVNJitExecutor::mvn_blk(const uint8_t* src_data, uint8_t* dst_data, co const size_t H = shape5d[3]; const size_t W = shape5d[4]; - size_t CB = div_up(C, blk_size); + const size_t CB = div_up(C, blk_size); - size_t C0 = W * blk_size; - size_t C1 = C0 * H; - size_t C2 = C1 * D; - size_t C3 = C2 * CB; - size_t C5 = C * D * H * W; + const size_t C0 = W * blk_size; + const size_t C1 = C0 * H; + const size_t C2 = C1 * D; + const size_t C3 = C2 * CB; + const size_t C5 = C * D * H * W; - size_t threads_num = parallel_get_max_threads(); + const size_t threads_num = parallel_get_max_threads(); size_t aux_buffer_size = mvnAttrs.execAcrossChannels_ ? blk_size : rnd_up(C, blk_size); aux_buffer_size += blk_size; std::vector mean_buffer(aux_buffer_size * threads_num); @@ -2562,7 +2566,11 @@ void MVN::MVNJitExecutor::mvn_blk(const uint8_t* src_data, uint8_t* dst_data, co // // | // // \|/ ///////////////////////////////// - auto mean_buffer_ptr = &mean_buffer[aux_buffer_size * static_cast(parallel_get_thread_num())]; + auto thread_idx = static_cast(parallel_get_thread_num()); + if (thread_idx >= threads_num) { + return mean_internal; + } + auto mean_buffer_ptr = &mean_buffer[aux_buffer_size * thread_idx]; for (size_t i = 0; i < blk_size; i++) mean_buffer_ptr[i] = 0.f; @@ -2651,7 +2659,7 @@ void MVN::MVNJitExecutor::mvn_blk(const uint8_t* src_data, uint8_t* dst_data, co // one thread for one C*W size(the same H) to get C size result for the same H, added to last group result // keep the compute order the same as planar - parallel_for2d(D, H, [&](size_t thr_idx, size_t d, size_t h) { + auto dh_loop = [&](size_t thr_idx, size_t d, size_t h) { for (size_t cb = 0; cb < CB; cb++) { size_t src_offset = b_offset + cb * C2 + d * C1 + h * C0; auto mean_buffer_ptr = &mean_buffer[blk_size * cb + aux_buffer_size * thr_idx]; @@ -2665,6 +2673,10 @@ void MVN::MVNJitExecutor::mvn_blk(const uint8_t* src_data, uint8_t* dst_data, co arg.post_op_data = post_ops_data_; (*mvn_mean_kernel)(&arg); } + }; + + parallel_nt_static(threads_num, [&](const int ithr, const int nthr) { + for_2d(ithr, nthr, D, H, dh_loop); }); for (size_t i = 1; i < threads_num; i++) { @@ -2678,7 +2690,7 @@ void MVN::MVNJitExecutor::mvn_blk(const uint8_t* src_data, uint8_t* dst_data, co for (size_t i = 0; i < variance_buffer.size(); i++) variance_buffer[i] = 0.f; - parallel_for2d(D, H, [&](size_t thr_idx, size_t d, size_t h) { + auto dh_loop = [&](size_t thr_idx, size_t d, size_t h) { for (size_t cb = 0; cb < CB; cb++) { size_t src_offset = b_offset + cb * C2 + d * C1 + h * C0; auto mean_buffer_ptr = &mean_buffer[blk_size * cb]; @@ -2694,7 +2706,12 @@ void MVN::MVNJitExecutor::mvn_blk(const uint8_t* src_data, uint8_t* dst_data, co arg.post_op_data = post_ops_data_; (*mvn_variance_kernel)(&arg); } + }; + + parallel_nt_static(threads_num, [&](const int ithr, const int nthr) { + for_2d(ithr, nthr, D, H, dh_loop); }); + for (size_t i = 1; i < threads_num; i++) { for (size_t c = 0; c < C; c++) variance_buffer[c] += variance_buffer[c + aux_buffer_size * i]; diff --git a/src/plugins/intel_cpu/src/nodes/qkv_proj.cpp b/src/plugins/intel_cpu/src/nodes/qkv_proj.cpp index 3260b12f1b5b4b..00c8b6f9b17c0b 100644 --- a/src/plugins/intel_cpu/src/nodes/qkv_proj.cpp +++ b/src/plugins/intel_cpu/src/nodes/qkv_proj.cpp @@ -60,6 +60,7 @@ struct QKVProjection::Executor : public QKVProjection::ExecutorBase { MemoryPtr m_scratchMem; uint8_t* m_scratch_base = nullptr; int m_M = 0; + size_t m_threads_num = 0lu; MatrixDynQuantPerRow m_quant_act; @@ -79,11 +80,11 @@ struct QKVProjection::Executor : public QKVProjection::ExecutorBase { auto K = w0.size(1); OPENVINO_ASSERT((K % cache_blk_k_size) == 0); - auto nthr = parallel_get_max_threads(); + m_threads_num = parallel_get_max_threads(); auto num_blk_K = K / cache_blk_k_size; int stride_in_bytes = K * weight_element_size; - works.resize(nthr); + works.resize(m_threads_num); int cur_work_id = 0; auto create_works = [&](void* pw, int output_id, int N, int valid_nthr) { @@ -119,7 +120,7 @@ struct QKVProjection::Executor : public QKVProjection::ExecutorBase { auto proj_size0 = m_node->m_config.proj_size0; auto proj_size1 = m_node->m_config.proj_size1; auto proj_size2 = m_node->m_config.proj_size2; - auto n_group_workers = allocate_workers({proj_size0, proj_size1, proj_size2}, nthr); + auto n_group_workers = allocate_workers({proj_size0, proj_size1, proj_size2}, m_threads_num); if (m_node->m_config.weights_combined) { auto* ptr_weights = reinterpret_cast(w0.ptr_v()); @@ -140,7 +141,7 @@ struct QKVProjection::Executor : public QKVProjection::ExecutorBase { wbuffer.alloc(works, weight_element_size); - ov::parallel_nt_static(0, [&](const size_t ithr, const size_t nthr) { + ov::parallel_nt_static(m_threads_num, [&](const size_t ithr, const size_t nthr) { auto& work = works[ithr]; if (work) { if (quantized_int8) @@ -237,7 +238,7 @@ struct QKVProjection::Executor : public QKVProjection::ExecutorBase { strideA = m_quant_act.K; } - ov::parallel_nt_static(0, [&](const size_t ithr, const size_t nthr) { + ov::parallel_nt_static(m_threads_num, [&](const size_t ithr, const size_t nthr) { auto& work = works[ithr]; if (work) { work.run(BM, pA, strideA); diff --git a/src/plugins/intel_cpu/src/nodes/reduce.cpp b/src/plugins/intel_cpu/src/nodes/reduce.cpp index b40c50f957514f..6cfc94a02b9f3b 100644 --- a/src/plugins/intel_cpu/src/nodes/reduce.cpp +++ b/src/plugins/intel_cpu/src/nodes/reduce.cpp @@ -2742,12 +2742,12 @@ inline void Reduce::reduce_kernel_post_process(uint8_t *out_ptr) { (*reduce_post_kernel)(&arg); }); } else if (layout == ReduceLayoutType::reduce_nspc) { - size_t num_threads = static_cast(parallel_get_max_threads()); + const size_t num_threads = static_cast(parallel_get_max_threads()); size_t OP = OB * OC >= num_threads ? OB * OC : OB * OC * OD; if (OP < num_threads && OW > blk_size) OP *= OH; size_t work_amount = OB * OC * OD * OH * OW / OP; - parallel_for(OP, [&](size_t op) { + auto op_loop = [&](size_t op) { const uint8_t *in_p = in_ptr + op * work_amount * intermediate_data_size; uint8_t *out_p = out_ptr + op * work_amount * dst_data_size; auto arg = jit_reduce_post_call_args(); @@ -2759,6 +2759,10 @@ inline void Reduce::reduce_kernel_post_process(uint8_t *out_ptr) { arg.divisor = &divisor; arg.post_op_data = static_cast(postOpsDataPtrs.data()); (*reduce_post_kernel)(&arg); + }; + + parallel_nt_static(num_threads, [&](const int ithr, const int nthr) { + for_1d(ithr, nthr, OP, op_loop); }); } else { size_t OCB = div_up(OC, blk_size); diff --git a/src/plugins/intel_cpu/src/nodes/reference.cpp b/src/plugins/intel_cpu/src/nodes/reference.cpp index 43b8f041184a70..185815acd8c294 100644 --- a/src/plugins/intel_cpu/src/nodes/reference.cpp +++ b/src/plugins/intel_cpu/src/nodes/reference.cpp @@ -14,7 +14,7 @@ Reference::Reference(const std::shared_ptr& op, const GraphContext::CP Node(op, context, NgraphShapeInferFactory(op, FULL_PORT_MASK)), ovCoreNode(op), additionalErrorMessage(errorMessage) { if (!op->has_evaluate()) { OPENVINO_THROW_NOT_IMPLEMENTED( - "Cannot fallback on ngraph reference implementation (Ngraph::Node::evaluate() is not implemented"); + "Cannot fallback on ngraph reference implementation (Ngraph::Node::evaluate() is not implemented)"); } setType(Type::Reference); diff --git a/src/plugins/intel_cpu/src/nodes/rnn.cpp b/src/plugins/intel_cpu/src/nodes/rnn.cpp index cab8bb3ad46325..679907c2d9cf28 100644 --- a/src/plugins/intel_cpu/src/nodes/rnn.cpp +++ b/src/plugins/intel_cpu/src/nodes/rnn.cpp @@ -33,8 +33,6 @@ static rnn_direction ieDirection2dnnl(const std::shared_ptr& op) ov::op::RecurrentSequenceDirection direction = ov::op::RecurrentSequenceDirection::FORWARD; if (ov::is_type(op)) { direction = ov::as_type_ptr(op)->get_direction(); - } else if (ov::is_type(op)) { - direction = ov::as_type_ptr(op)->get_direction(); } else if (ov::is_type(op)) { direction = ov::as_type_ptr(op)->get_direction(); } else if (ov::is_type(op)) { @@ -75,14 +73,13 @@ static dnnl::algorithm ie2dnnl(const std::shared_ptr& op) { else return dnnl::algorithm::vanilla_augru; } else if (one_of(op->get_type_info(), - ov::op::v0::LSTMCell::get_type_info_static(), - ov::op::v4::LSTMCell::get_type_info_static(), - ov::op::v0::LSTMSequence::get_type_info_static(), - ov::op::v5::LSTMSequence::get_type_info_static())) { + ov::op::v0::LSTMCell::get_type_info_static(), + ov::op::v4::LSTMCell::get_type_info_static(), + ov::op::v5::LSTMSequence::get_type_info_static())) { return dnnl::algorithm::vanilla_lstm; } else if (one_of(op->get_type_info(), - ov::op::v0::RNNCell::get_type_info_static(), - ov::op::v5::RNNSequence::get_type_info_static())) { + ov::op::v0::RNNCell::get_type_info_static(), + ov::op::v5::RNNSequence::get_type_info_static())) { return dnnl::algorithm::vanilla_rnn; } else { OPENVINO_THROW("Operation ", @@ -201,16 +198,15 @@ bool RNNKey::operator==(const RNNKey& rhs) const { bool RNN::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { try { if (!one_of(op->get_type_info(), - ov::op::v3::GRUCell::get_type_info_static(), - ov::op::internal::AUGRUCell::get_type_info_static(), - ov::op::internal::AUGRUSequence::get_type_info_static(), - ov::op::v0::LSTMCell::get_type_info_static(), - ov::op::v4::LSTMCell::get_type_info_static(), - ov::op::v0::RNNCell::get_type_info_static(), - ov::op::v5::GRUSequence::get_type_info_static(), - ov::op::v0::LSTMSequence::get_type_info_static(), - ov::op::v5::LSTMSequence::get_type_info_static(), - ov::op::v5::RNNSequence::get_type_info_static())) { + ov::op::v3::GRUCell::get_type_info_static(), + ov::op::internal::AUGRUCell::get_type_info_static(), + ov::op::internal::AUGRUSequence::get_type_info_static(), + ov::op::v0::LSTMCell::get_type_info_static(), + ov::op::v4::LSTMCell::get_type_info_static(), + ov::op::v0::RNNCell::get_type_info_static(), + ov::op::v5::GRUSequence::get_type_info_static(), + ov::op::v5::LSTMSequence::get_type_info_static(), + ov::op::v5::RNNSequence::get_type_info_static())) { errorMessage = "Unsupported sequence operation."; return false; } @@ -239,9 +235,7 @@ bool RNN::isSupportedOperation(const std::shared_ptr& op, std::s errorMessage = "Node expects 6 inputs. Actual: " + std::to_string(op->get_input_size()); return false; } - } else if (one_of(op->get_type_info(), - ov::op::v0::LSTMSequence::get_type_info_static(), - ov::op::v5::LSTMSequence::get_type_info_static())) { + } else if (one_of(op->get_type_info(), ov::op::v5::LSTMSequence::get_type_info_static())) { if (op->get_input_size() != 7) { errorMessage = "Node expects 7 inputs. Actual: " + std::to_string(op->get_input_size()); return false; @@ -264,7 +258,6 @@ bool RNN::isSupportedOperation(const std::shared_ptr& op, std::s if (one_of(rnnCellBase->get_type_info(), ov::op::v0::LSTMCell::get_type_info_static(), ov::op::v4::LSTMCell::get_type_info_static(), - ov::op::v0::LSTMSequence::get_type_info_static(), ov::op::v5::LSTMSequence::get_type_info_static())) { if (rnnCellBase->get_activations() != std::vector{"sigmoid", "tanh", "tanh"}) { errorMessage = "Not supported activation functions"; @@ -294,9 +287,6 @@ bool RNN::isSupportedOperation(const std::shared_ptr& op, std::s if (auto gru_seq = ov::as_type_ptr(op)) { direction = gru_seq->get_direction(); seqLenIdx = 2; - } else if (auto lstm_seq = ov::as_type_ptr(op)) { - direction = lstm_seq->get_direction(); - seqLenIdx = 3; } else if (auto lstm_seq = ov::as_type_ptr(op)) { direction = lstm_seq->get_direction(); seqLenIdx = 3; @@ -441,9 +431,7 @@ RNN::RNN(const std::shared_ptr& op, const GraphContext::CPtr context) } else if (op->get_type_info() == ov::op::internal::AUGRUSequence::get_type_info_static()) { sIdx = 2; wIdx = 3; rIdx = 4; bIdx = 5; aIdx = 6; yIdx = 0; hoIdx = 1; - } else if (one_of(op->get_type_info(), - ov::op::v0::LSTMSequence::get_type_info_static(), - ov::op::v5::LSTMSequence::get_type_info_static())) { + } else if (one_of(op->get_type_info(), ov::op::v5::LSTMSequence::get_type_info_static())) { sIdx = 3; wIdx = 4; rIdx = 5; bIdx = 6; yIdx = 0; hoIdx = 1; coIdx = 2; } diff --git a/src/plugins/intel_cpu/src/nodes/roi_align.cpp b/src/plugins/intel_cpu/src/nodes/roi_align.cpp index eb1797279e1415..27f9426dca6af9 100644 --- a/src/plugins/intel_cpu/src/nodes/roi_align.cpp +++ b/src/plugins/intel_cpu/src/nodes/roi_align.cpp @@ -1076,7 +1076,7 @@ void ROIAlign::executeSpecified() { int bufSize = rnd_up(C, 16); size_t threadsNum = parallel_get_max_threads(); workingBuf.resize(bufSize * threadsNum, 0.f); - parallel_for3d(realRois, pooledH, pooledW, [&](int n, int yBinInd, int xBinInd) { + auto rhw_loop = [&](int n, int yBinInd, int xBinInd) { int numSamplesROI = numSamples[n]; // each sample have 4 values for srcAddressList and weight size_t binOffset = numSamplesROI * BLIParamsNum * pooledW * yBinInd + numSamplesROI * BLIParamsNum * xBinInd; @@ -1095,6 +1095,10 @@ void ROIAlign::executeSpecified() { arg.dst = static_cast(&dst[dstOffset]); arg.src_stride = lastBlockDim * W * H; // only valid for blk, nspc generate inside (*roi_align_kernel)(&arg); + }; + + parallel_nt_static(threadsNum, [&](const int ithr, const int nthr) { + for_3d(ithr, nthr, realRois, pooledH, pooledW, rhw_loop); }); } else { // one lane for one sample generation, then pooling all samples. diff --git a/src/plugins/intel_cpu/src/nodes/scaled_attn.cpp b/src/plugins/intel_cpu/src/nodes/scaled_attn.cpp index e229ff4bb72c57..f9f853230c4dd6 100644 --- a/src/plugins/intel_cpu/src/nodes/scaled_attn.cpp +++ b/src/plugins/intel_cpu/src/nodes/scaled_attn.cpp @@ -217,6 +217,7 @@ struct MHAKernel { size_t wsp_size_per_thread = 0; using tag = dnnl::memory::format_tag; using dt = dnnl::memory::data_type; + size_t m_threads_num = 0lu; struct brgemmKey { size_t M; size_t N; @@ -315,21 +316,21 @@ struct MHAKernel { wv_gemm_ptr = wv_result.first; - size_t nthr = static_cast(parallel_get_max_threads()); + m_threads_num = static_cast(parallel_get_max_threads()); // wsp is used to compute beta when K is blocked wsp_size_per_thread = wv_gemm_ptr->get_wsp_size(); - wsp.resize(nthr * wsp_size_per_thread); + wsp.resize(m_threads_num * wsp_size_per_thread); // allocate scratch a/b, notice get_scratch_a_size/get_scratch_b_size returns in bytes size_t data_size = sizeof(T); - qk_scratch_a.resize({nthr, qk_gemm_ptr->get_scratch_a_size() / data_size}); - wv_scratch_a.resize({nthr, wv_gemm_ptr->get_scratch_a_size() / data_size}); + qk_scratch_a.resize({m_threads_num, qk_gemm_ptr->get_scratch_a_size() / data_size}); + wv_scratch_a.resize({m_threads_num, wv_gemm_ptr->get_scratch_a_size() / data_size}); qk_scratch_b.resize({B, Hk, qk_gemm_ptr->get_scratch_b_size() / data_size}); wv_scratch_b.resize({B, Hk, wv_gemm_ptr->get_scratch_b_size() / data_size}); const size_t m_block_size = qk_gemm_ptr->get_mblk_size(); - weight_score.resize({static_cast(parallel_get_max_threads()), H, m_block_size, kv_len}); + weight_score.resize({m_threads_num, H, m_block_size, kv_len}); if (has_out_transpose) { fp32_out.resize({B, q_len, H, head_size_v}); } else { @@ -367,7 +368,7 @@ struct MHAKernel { }); // attention - parallel_for3d(B, H, m_blocks, [&](size_t ithr, size_t b, size_t h, size_t m_blk) { + auto bhb_loop = [&](size_t ithr, size_t b, size_t h, size_t m_blk) { auto m_start = m_blk * m_block_size; auto m_end = std::min(m_start + m_block_size, q_len); auto m_cnt = m_end - m_start; @@ -456,6 +457,10 @@ struct MHAKernel { 1); } } + }; + + parallel_nt_static(m_threads_num, [&](const int ithr, const int nthr) { + for_3d(ithr, nthr, B, H, m_blocks, bhb_loop); }); } @@ -652,12 +657,14 @@ struct MHAKernel { size_t m_block_size; // buffer to hold qk temp std::vector qk_buffers; + size_t m_threads_num = 0lu; MHAKernel() = delete; explicit MHAKernel(GraphContext::CPtr ctx): context(ctx) { m_block_size = 4; select_nfltmax_at_0 = false; - qk_buffers.resize(parallel_get_max_threads()); + m_threads_num = parallel_get_max_threads(); + qk_buffers.resize(m_threads_num); } PlainTensor causal_mask; @@ -699,7 +706,7 @@ struct MHAKernel { auto m_blocks = (q_len + m_block_size - 1) / m_block_size; - parallel_for3d(B, H, m_blocks, [&](size_t b, size_t h, size_t m_blk) { + auto bhb_loop = [&](size_t b, size_t h, size_t m_blk) { auto thread_id = parallel_get_thread_num(); if (thread_id < 0) OPENVINO_THROW("The calling thread isn't initialized!"); @@ -801,6 +808,10 @@ struct MHAKernel { has_out_transpose ? &output_emb.at({b, m_start, h * head_size_v}) : &output_emb.at({b, h, m_start}), has_out_transpose ? output_emb.stride(1) : output_emb.stride(2), 1); + }; + + parallel_nt_static(m_threads_num, [&](const int ithr, const int nthr) { + for_3d(ithr, nthr, B, H, m_blocks, bhb_loop); }); } }; diff --git a/src/plugins/intel_cpu/src/nodes/strided_slice.cpp b/src/plugins/intel_cpu/src/nodes/strided_slice.cpp index 4f974cfe5e9748..13671c22d102ae 100644 --- a/src/plugins/intel_cpu/src/nodes/strided_slice.cpp +++ b/src/plugins/intel_cpu/src/nodes/strided_slice.cpp @@ -348,6 +348,7 @@ StridedSlice::StridedSliceCommonExecutor::StridedSliceCommonExecutor(const Strid dimsNormalization(); dimsGluing(); indicesCalculation(); + m_threads_num = parallel_get_max_threads(); } void StridedSlice::StridedSliceCommonExecutor::orderParametersByLayouts(const BlockedMemoryDescCPtr& blockedMemoryDesc) { @@ -642,8 +643,7 @@ void StridedSlice::StridedSliceCommonExecutor::dimsGluing() { for (size_t idx = secondDim.first + 1; idx < secondDim.second; idx++) params.attrs.begin[1] /= dstBlockedDimsBefore[idx]; - const size_t maxThreads = parallel_get_max_threads(); - if (params.dstBlockedDims[0] < maxThreads) { + if (params.dstBlockedDims[0] < m_threads_num) { params.dstBlockedDims[1] /= realDstDim; params.srcBlockedDims[1] /= realSrcDim; params.dstBlockedDims.insert(params.dstBlockedDims.begin() + 1, realDstDim); @@ -682,8 +682,7 @@ void StridedSlice::StridedSliceCommonExecutor::indicesCalculation() { dstIndices.resize(workAmount, 0); // should choose more optimal thread count - const size_t nthr = parallel_get_max_threads(); - nThreads = nthr > workAmount ? workAmount : nthr; + nThreads = m_threads_num > workAmount ? workAmount : m_threads_num; if (params.isOptimized) { indicesCalculationForOptimized(); diff --git a/src/plugins/intel_cpu/src/nodes/strided_slice.h b/src/plugins/intel_cpu/src/nodes/strided_slice.h index 5c5950520bda7d..bf698643271d7a 100644 --- a/src/plugins/intel_cpu/src/nodes/strided_slice.h +++ b/src/plugins/intel_cpu/src/nodes/strided_slice.h @@ -122,6 +122,7 @@ class StridedSlice : public Node { size_t workAmount = 0lu; size_t lastDstDim = 0lu; size_t srcShift = 0lu; + size_t m_threads_num = 0lu; }; using executorPtr = std::shared_ptr; executorPtr execPtr = nullptr; diff --git a/src/plugins/intel_cpu/src/nodes/tensoriterator.cpp b/src/plugins/intel_cpu/src/nodes/tensoriterator.cpp index 9a3b9788b838d2..dcf2b0f8ffd5ee 100644 --- a/src/plugins/intel_cpu/src/nodes/tensoriterator.cpp +++ b/src/plugins/intel_cpu/src/nodes/tensoriterator.cpp @@ -938,7 +938,7 @@ int TensorIterator::getNumIteration(const std::vector& inputPortMap, co } bool TensorIterator::runAsDynamic() const { - return isDynamicNode() || Graph::Status::ReadyDynamic == sub_graph.getStatus(); + return isDynamicNode() || sub_graph.IsDynamic(); } bool TensorIterator::created() const { diff --git a/src/plugins/intel_cpu/src/nodes_factory.cpp b/src/plugins/intel_cpu/src/nodes_factory.cpp index 16cf1b974d8561..4a8e8205510fcf 100644 --- a/src/plugins/intel_cpu/src/nodes_factory.cpp +++ b/src/plugins/intel_cpu/src/nodes_factory.cpp @@ -108,6 +108,7 @@ #include "nodes/transpose.h" #include "nodes/unique.hpp" #include "nodes/causal_mask_preprocess.h" +#include "nodes/lora.h" namespace ov { namespace intel_cpu { @@ -221,6 +222,7 @@ Node::NodesFactory::NodesFactory() : Factory("NodesFactory") { INTEL_CPU_NODE(Composite, Type::SubModel); INTEL_CPU_NODE(ScaledDotProductAttention, Type::ScaledDotProductAttention); INTEL_CPU_NODE(SearchSorted, Type::SearchSorted); + INTEL_CPU_NODE(LoRA, Type::LoRA); #if defined(OPENVINO_ARCH_X86_64) INTEL_CPU_NODE(FakeQuantize, Type::FakeQuantize); INTEL_CPU_NODE(GridSample, Type::GridSample); diff --git a/src/plugins/intel_cpu/src/shape_inference/shape_inference.cpp b/src/plugins/intel_cpu/src/shape_inference/shape_inference.cpp index bb2d5e5e84b267..8e2e794eacc939 100644 --- a/src/plugins/intel_cpu/src/shape_inference/shape_inference.cpp +++ b/src/plugins/intel_cpu/src/shape_inference/shape_inference.cpp @@ -407,6 +407,7 @@ using IStaticShapeInferFactory = template <> const IStaticShapeInferFactory::TRegistry IStaticShapeInferFactory::registry{ // opset15 + _OV_OP_SHAPE_INFER_MASK_REG(op::v15::Squeeze, ShapeInferTA, util::bit::mask(1)), _OV_OP_SHAPE_INFER_MASK_REG(op::v15::SearchSorted, ShapeInferTA, util::bit::mask()), _OV_OP_SHAPE_INFER_MASK_REG(op::v15::StringTensorUnpack, ShapeInferTA, util::bit::mask(0)), _OV_OP_SHAPE_INFER_MASK_REG(op::v15::StringTensorPack, ShapeInferTA, util::bit::mask(0, 1)), @@ -531,7 +532,6 @@ const IStaticShapeInferFactory::TRegistry IStaticShapeInferFactory::registry{ _OV_OP_SHAPE_INFER_MASK_REG(opset1::GroupConvolutionBackpropData, ShapeInferPaddingTA, util::bit::mask(2)), _OV_OP_SHAPE_INFER_MASK_REG(opset1::Interpolate, ShapeInferTA, util::bit::mask(1)), _OV_OP_SHAPE_INFER_MASK_REG(opset1::LSTMCell, ShapeInferTA, util::bit::mask()), - _OV_OP_SHAPE_INFER_MASK_REG(opset1::LSTMSequence, ShapeInferTA, util::bit::mask()), _OV_OP_SHAPE_INFER_MASK_REG(opset1::MatMul, ShapeInferTA, util::bit::mask()), _OV_OP_SHAPE_INFER_MASK_REG(opset1::MaxPool, ShapeInferPaddingTA, util::bit::mask()), _OV_OP_SHAPE_INFER_MASK_REG(opset1::NonMaxSuppression, ShapeInferTA, util::bit::mask(2)), diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/rnn_sequences_optimization.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/rnn_sequences_optimization.cpp index 72ca09a8152298..916e7b21ffa484 100644 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/rnn_sequences_optimization.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/rnn_sequences_optimization.cpp @@ -124,14 +124,12 @@ ov::intel_cpu::OptimizeRNNSequenceTransposes::OptimizeRNNSequenceTransposes() { ov::intel_cpu::OptimizeLSTMSequenceTransposes::OptimizeLSTMSequenceTransposes() { MATCHER_SCOPE(OptimizeLSTMSequenceTransposes); - auto lstmSequenceNgraph = ov::pass::pattern::wrap_type(); + auto lstmSequenceNgraph = ov::pass::pattern::wrap_type(); ov::matcher_pass_callback callback = [](ov::pass::pattern::Matcher &m) { auto checkSequence = [](const std::shared_ptr& node) { if (auto lstm5 = ov::as_type_ptr(node)) { return lstm5->get_direction() != ov::op::RecurrentSequenceDirection::BIDIRECTIONAL; - } else if (auto lstm1 = ov::as_type_ptr(node)) { - return lstm1->get_direction() != ov::op::RecurrentSequenceDirection::BIDIRECTIONAL; } else { return false; } diff --git a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp index fcf38440b8aa4b..9dd1da2d471e5a 100644 --- a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp +++ b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp @@ -37,6 +37,7 @@ #include "transformations/common_optimizations/move_eltwise_up_data_movement.hpp" #include "transformations/common_optimizations/mark_rope_input_to_keep_in_mixed_precision.hpp" #include "transformations/common_optimizations/rms_fusion.hpp" +#include "transformations/common_optimizations/lora_subgraph_fusion.hpp" #include "transformations/control_flow/unroll_tensor_iterator.hpp" #include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp" #include "transformations/fp16_compression/mark_floatpoint_range.hpp" @@ -693,6 +694,7 @@ void Transformations::PreLpt(const std::vector& defaultPrecis CPU_REGISTER_PASS_COMMON(manager, ov::pass::EnableDecompressionConvertConstantFolding); CPU_REGISTER_PASS_COMMON(manager, ov::pass::KeepConstAndDecompression); CPU_REGISTER_PASS_COMMON(manager, ov::pass::ConstantFolding); + CPU_REGISTER_PASS_COMMON(manager, ov::pass::LoraSubgraphFusion); manager.run_passes(model); } diff --git a/src/plugins/intel_cpu/src/utils/serialize.cpp b/src/plugins/intel_cpu/src/utils/serialize.cpp index f7fd337afa932e..814e8d19311a8c 100644 --- a/src/plugins/intel_cpu/src/utils/serialize.cpp +++ b/src/plugins/intel_cpu/src/utils/serialize.cpp @@ -20,12 +20,7 @@ void ModelSerializer::operator<<(const std::shared_ptr& model) { auto serialize_info = [&](std::ostream& stream) { pugi::xml_document xml_doc; pugi::xml_node root = xml_doc.append_child("cnndata"); - pugi::xml_node outputs = root.append_child("outputs"); - for (const auto& out : model->get_results()) { - auto out_node = outputs.append_child("out"); - const auto name = ov::descriptor::get_ov_tensor_legacy_name(out->input_value(0).get_tensor()); - out_node.append_attribute("name").set_value(name.c_str()); - } + root.append_child("outputs"); xml_doc.save(stream); }; @@ -44,26 +39,15 @@ ModelDeserializer::ModelDeserializer(std::istream& model_stream, ModelBuilder fn } } -void ModelDeserializer::set_info(pugi::xml_node& root, std::shared_ptr& model) { - pugi::xml_node outputs = root.child("outputs"); - auto nodes_it = outputs.children("out").begin(); - size_t size = model->outputs().size(); - for (size_t i = 0lu; i < size; ++nodes_it, i++) { - std::string name = nodes_it->attribute("name").value(); - if (name.empty()) - continue; - auto result = model->output(i).get_node_shared_ptr(); - ov::descriptor::set_ov_tensor_legacy_name(result->input_value(0).get_tensor(), name); - } -} + void ModelDeserializer::set_info(pugi::xml_node& root, std::shared_ptr& model) {} -void ModelDeserializer::operator>>(std::shared_ptr& model) { - if (auto mmap_buffer = dynamic_cast(m_istream.rdbuf())) { - auto buffer = mmap_buffer->get_buffer(); - process_mmap(model, buffer); - } else { - process_stream(model); - } + void ModelDeserializer::operator>>(std::shared_ptr& model) { + if (auto mmap_buffer = dynamic_cast(m_istream.rdbuf())) { + auto buffer = mmap_buffer->get_buffer(); + process_mmap(model, buffer); + } else { + process_stream(model); + } } void ModelDeserializer::process_mmap(std::shared_ptr& model, diff --git a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/rms_norm.cpp b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/rms_norm.cpp index da7bbaaaf848d1..38aeedb5e451cf 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/rms_norm.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/single_layer_tests/instances/x64/rms_norm.cpp @@ -24,6 +24,17 @@ const std::vector> shapes{ {ov::Shape{1024 + 16 + 1}, ov::Shape{1024 + 16 + 1}}} }, }, + // small data size + { + // data shape + {ov::test::InputShape{ov::PartialShape{-1, -1, 31}, + {ov::Shape{1, 8, 31}, ov::Shape{2, 3, 31}}} + }, + // scale shape + {ov::test::InputShape{ov::PartialShape{31}, + {ov::Shape{31}, ov::Shape{31}}} + }, + }, // scale is scalar { // data shape diff --git a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/concat_multiple_query_sdp.cpp b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/concat_multiple_query_sdp.cpp index d05e7840562191..d74ab99fb3d5ab 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/concat_multiple_query_sdp.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/concat_multiple_query_sdp.cpp @@ -238,7 +238,7 @@ class ConcatMultiQuerySDPTest : public testing::WithParamInterfaceget_element_type() == element::f16) { ov::Tensor t{ov::element::f16, shape}; - strided_iota(static_cast(t.data()), t.get_size(), val, 0.1f); + strided_iota(static_cast(t.data()), t.get_size(), val, 0.0f); inputs.insert({param, t}); } else { ov::Tensor t{ov::element::bf16, shape}; diff --git a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/lora_pattern.cpp b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/lora_pattern.cpp new file mode 100644 index 00000000000000..20620e42ceddc7 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/lora_pattern.cpp @@ -0,0 +1,313 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "common_test_utils/node_builders/convolution.hpp" +#include "common_test_utils/node_builders/eltwise.hpp" +#include "common_test_utils/ov_tensor_utils.hpp" +#include "shared_test_classes/base/ov_subgraph.hpp" +#include "utils/cpu_test_utils.hpp" + +namespace ov { +namespace test { + +namespace { +constexpr auto t4_name = "lora/MatMul.B"; +constexpr auto t5_name = "lora/MatMul.alpha"; +constexpr auto t6_name = "lora/MatMul.A"; +constexpr auto netType = ov::element::f32; + +enum class StatesPolicy { + EMPTY_TENSORS, + RANDOM_TENSORS, + UNDEFINED +}; +std::ostream &operator<<(std::ostream& os, StatesPolicy states_policy) { + switch (states_policy) { + case StatesPolicy::EMPTY_TENSORS: + return os << "empty_tensors"; + case StatesPolicy::RANDOM_TENSORS: + return os << "random_tensors"; + case StatesPolicy::UNDEFINED: + return os << "undefined"; + default: + OPENVINO_THROW("Unexpected states policy"); + } +} + +} // namespace + +using LoraPatternParams = std::tuple; // states filling policy + +class LoraPatternBaseCPUTest : public SubgraphBaseTest, public testing::WithParamInterface { +public: +static std::string getTestCaseName(testing::TestParamInfo obj) { + ov::element::Type states_precision; + StatesPolicy states_policy; + std::tie(states_precision, states_policy) = obj.param; + + std::ostringstream result; + result << "states_precision=" << states_precision << "_states_policy=" << states_policy; + return result.str(); + } + + void SetUp() override { + targetDevice = ov::test::utils::DEVICE_CPU; + std::tie(states_precision, states_policy) = this->GetParam(); + init_function(); + } + +protected: + virtual void init_function() = 0; + + std::pair create_states(const std::vector& shapes, + const std::vector names) { + ov::OutputVector state_outs; + ov::SinkVector assigns; + auto create_state = [&](const ov::PartialShape& shape, const std::string name) { + auto variable = std::make_shared( + ov::op::util::VariableInfo{shape, states_precision, name}); + auto read_value = std::make_shared(variable); + auto assign = std::make_shared(read_value, variable); + assigns.push_back(assign); + if (states_precision == netType) + state_outs.push_back(read_value); + else + state_outs.push_back(std::make_shared(read_value, netType)); + }; + OPENVINO_ASSERT(shapes.size() == names.size()); + for (size_t i = 0; i < shapes.size(); ++i) + create_state(shapes[i], names[i]); + return std::make_pair(state_outs, assigns); + } + + void run_test() { + switch (states_policy) { + case StatesPolicy::EMPTY_TENSORS: + run_test_empty_tensors(); + break; + case StatesPolicy::RANDOM_TENSORS: + run_test_random_tensors(); + break; + default: + OPENVINO_THROW("Unexpected states policy: ", states_policy); + } + } + + void run_test_empty_tensors() { + compile_model(); + inferRequest = compiledModel.create_infer_request(); + ASSERT_TRUE(inferRequest); + generate_inputs(targetStaticShapes.front()); + for (const auto& input : inputs) { + inferRequest.set_tensor(input.first, input.second); + } + + inferRequest.infer(); + auto outputs = function->outputs(); + + auto tx_result = inferRequest.get_tensor(outputs[0]); + auto tz_result = inferRequest.get_tensor(outputs[1]); + ov::test::utils::compare(tx_result, tz_result, 1e-4, 1e-4); + } + + void run_test_random_tensors() { + compile_model(); + inferRequest = compiledModel.create_infer_request(); + ASSERT_TRUE(inferRequest); + + // use the Template plugin as a reference + + auto compiledReferenceModel = core->compile_model(function, ov::test::utils::DEVICE_TEMPLATE); + auto inferRequestRef = compiledReferenceModel.create_infer_request(); + ASSERT_TRUE(inferRequestRef); + + generate_inputs(targetStaticShapes.front()); + for (const auto& input : inputs) { + inferRequest.set_tensor(input.first, input.second); + inferRequestRef.set_tensor(input.first, input.second); + } + + constexpr size_t lora_order = 25lu; + constexpr int infer_count = 6lu; + + std::unordered_map stateShapes; + std::unordered_map initStateShapes; + + auto&& states = inferRequest.query_state(); + for (auto&& state : states) { + auto shape = state.get_state().get_shape(); + initStateShapes.insert({state.get_name(), shape}); + std::for_each(shape.begin(), shape.end(), [=](ov::Shape::value_type& x) { + if (0 == x) { + x = lora_order; + } + }); + stateShapes.insert({state.get_name(), std::move(shape)}); + } + + for (int i = 0; i < infer_count; ++i) { + // set states + + if (i == 3) { + // reset states on the 3rd iteration + for (auto&& item : states) { + item.reset(); + } + + for (auto&& item : inferRequestRef.query_state()) { + // Template plugin doesn't support reset state for dynamic shape states + item.get_state().set_shape(initStateShapes.at(item.get_name())); + } + } else if (!(i & 0x1)) { // every even call + // generate and set state tensors + for (auto&& item : states) { + auto&& refStates = inferRequestRef.query_state(); + using ov::test::utils::InputGenerateData; + const auto& shape = stateShapes.at(item.get_name()); + auto tensor = + ov::test::utils::create_and_fill_tensor(states_precision, shape, InputGenerateData{0, 10, 1, i}); + item.set_state(tensor); + auto itr = std::find_if(refStates.begin(), refStates.end(), [&](const ov::VariableState& state) { + return state.get_name() == item.get_name(); + }); + ASSERT_FALSE(itr == refStates.end()); + itr->set_state(tensor); + } + } + + inferRequest.infer(); + inferRequestRef.infer(); + auto outputs = function->outputs(); + + auto tx_result = inferRequest.get_tensor(outputs[0]); + auto tz_result = inferRequest.get_tensor(outputs[1]); + + auto tx_result_ref = inferRequestRef.get_tensor(outputs[0]); + auto tz_result_ref = inferRequestRef.get_tensor(outputs[1]); + + ov::test::utils::compare(tx_result_ref, tx_result, 1e-4, 1e-4); + ov::test::utils::compare(tz_result_ref, tz_result, 1e-4, 1e-4); + } + } + + StatesPolicy states_policy = StatesPolicy::UNDEFINED; + ov::element::Type states_precision = ov::element::undefined; +}; + +class LoraPatternMatmulCPUTest : public LoraPatternBaseCPUTest { +protected: + void init_function() override { + ov::PartialShape shape_x = {-1, -1, K}; + ov::PartialShape shape_w = {N, K}; + + auto param_y = std::make_shared(netType, shape_x); + auto param_w = std::make_shared(netType, shape_w); + + // "Main" matrix multiplication from the original transformer model + auto tx = std::make_shared(param_y, param_w, false, true); + + // LoRA parameters from states + auto states = create_states({{N, -1}, {1, -1}, {-1, K}}, {t4_name, t5_name, t6_name}); + + // Apply LoRA parameters to the current activations + auto t5810 = std::make_shared(param_y, states.first[2], false, true); + auto t5811 = std::make_shared(t5810, states.first[1]); + auto t5812 = std::make_shared(t5811, states.first[0], false, true); + + // Mix LoRA part into normally computed activations after the "main" MatMul + auto tz = std::make_shared(tx, t5812); + + auto result_x = std::make_shared(tx); + auto result_z = std::make_shared(tz); + + function = std::make_shared(ov::ResultVector({result_x, result_z}), + states.second, + ov::ParameterVector({param_y, param_w})); + } + + static constexpr size_t K = 563ul; // Weights matrix K dimension + static constexpr size_t N = 2048ul; // Weights matrix N dimension +}; + +class LoraPatternConvolutionCPUTest : public LoraPatternBaseCPUTest { +public: + void init_function() override { + ov::PartialShape shape_x = {-1, num_channels, -1, -1}; + + auto param_y = std::make_shared(netType, shape_x); + + // Original Convolution that is modified by LoRA adapter later + auto tx = ov::test::utils::make_convolution(param_y, + netType, + {1, 1}, + {1, 1}, + {0, 0}, + {0, 0}, + {1, 1}, + ov::op::PadType::EXPLICIT, + num_channels); + + // LoRA parameters from states + auto states = create_states({{num_channels, -1}, {1, -1}, {-1, num_channels}}, {t4_name, t5_name, t6_name}); + + // LoRA pattern with additional Transposes to move channel dimensions into positions where MatMul can be applied + auto t4940 = + std::make_shared(ov::element::i32, ov::Shape{4}, std::vector{2, 3, 0, 1}); + + auto t4941 = std::make_shared(param_y, t4940); + auto t4942 = std::make_shared(t4941, states.first[2], false, true); + auto t4943 = std::make_shared(t4942, states.first[1]); + auto t4944 = std::make_shared(t4943, states.first[0], false, true); + + auto t4945 = + std::make_shared(ov::element::i32, ov::Shape{4}, std::vector{2, 3, 0, 1}); + auto t4946 = std::make_shared(t4944, t4945); + + // Mix LoRA part into normally computed activations after the "main" MatMul + auto tz = std::make_shared(tx, t4946); + + auto result_x = std::make_shared(tx); + auto result_z = std::make_shared(tz); + + function = std::make_shared(ov::ResultVector({result_x, result_z}), + states.second, + ov::ParameterVector({param_y})); + } + +protected: + static constexpr size_t num_channels = 64ul; +}; + +TEST_P(LoraPatternMatmulCPUTest, CompareWithRefs) { + targetStaticShapes = {{{{1, 20, K}}, {{N, K}}}}; + run_test(); + CPUTestUtils::CheckNumberOfNodesWithType(compiledModel, "LoRA", 1); + CPUTestUtils::CheckNumberOfNodesWithType(compiledModel, "MatMul", 1); +} + +TEST_P(LoraPatternConvolutionCPUTest, CompareWithRefs) { + targetStaticShapes = {{{1, num_channels, 10, 15}}}; + run_test(); + CPUTestUtils::CheckNumberOfNodesWithType(compiledModel, "LoRA", 1); + CPUTestUtils::CheckNumberOfNodesWithType(compiledModel, "MatMul", 0); +} + +const ov::element::TypeVector states_precisions {ov::element::f32, ov::element::f16}; +const std::vector states_policies {StatesPolicy::EMPTY_TENSORS, StatesPolicy::RANDOM_TENSORS}; + +INSTANTIATE_TEST_SUITE_P(smoke_Snippets_LoRA_CPU_MatMul, LoraPatternMatmulCPUTest, + ::testing::Combine( + ::testing::ValuesIn(states_precisions), + ::testing::ValuesIn(states_policies)), + LoraPatternBaseCPUTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Snippets_LoRA_CPU_Conv, LoraPatternConvolutionCPUTest, + ::testing::Combine( + ::testing::ValuesIn(states_precisions), + ::testing::ValuesIn(states_policies)), + LoraPatternBaseCPUTest::getTestCaseName); + +} // namespace test +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/read_value_assign.cpp b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/read_value_assign.cpp new file mode 100644 index 00000000000000..c6e976b321f703 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/read_value_assign.cpp @@ -0,0 +1,182 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "common_test_utils/node_builders/eltwise.hpp" +#include "common_test_utils/node_builders/constant.hpp" +#include "shared_test_classes/base/ov_subgraph.hpp" +#include "utils/cpu_test_utils.hpp" + +/* The main purpose of this test set is to test ReadValue->Assign direct connection optimizations, i.e. + dropping the MemoryOutput node. +*/ + +namespace ov { +namespace test { + +using namespace CPUTestUtils; + +// ┌────────┐ ┌────────┐ +// │ Param2 │ │ Param1 │ |---------------| +// └───┬────┘ └────┬───┘ | | +// │ |-----------|┌─────────┐ | +// │ | │ │Constant │ | +// │ | │ └───┬─────┘ | +// │ | ┌───┴────┐ │ | +// │ | │Multiply├─────┘ | +// │ | └───┬────┘ | <- Optional Init Subgraph +// │ | │ ┌─────────┐ | +// │ | │ │Constant │ | +// │ | │ └───┬─────┘ | +// │ | ┌───┴────┐ │ | +// │ | │ Add ├─────┘ | +// │ | └───┬────┘ | +// │ | │ | +// │ |---------------------------| +// │ │ +// │ │ +// │ │ +// │ ┌─────┴─────┐ +// │ │ ReadValue │ +// │ └─────┬─────┘ +// │ │ \ +// │ ┌──┴──┐ \ +// └────────┤ Add │ \┌────────┐ +// └──┬──┘ │ Assign │ +// │ └────────┘ +// │ +// ┌────┴────┐ +// │ Result1 │ +// └─────────┘ + +typedef std::tuple< + bool, // include init subgraph + CPUSpecificParams +> ReadValueAssignTestParams; + +class ReadValueAssignTest : public testing::WithParamInterface, + virtual public SubgraphBaseTest, + public CPUTestsBase { +public: + static std::string getTestCaseName(const testing::TestParamInfo &obj) { + bool use_init_subgraph = false; + CPUSpecificParams cpu_params; + std::tie(use_init_subgraph, cpu_params) = obj.param; + + std::ostringstream results; + results << "Init_Graph=" << (use_init_subgraph ? "True" : "False") << "_"; + results << CPUTestsBase::getTestCaseName(cpu_params); + return results.str(); + } + + void SetUp() override { + targetDevice = ov::test::utils::DEVICE_CPU; + + const ov::Shape tensor_shape = {3, 32, 7, 7}; + + InputShape param1_shape = {{-1, 32, -1, -1}, {tensor_shape}}; + InputShape param2_shape = {{-1, -1, -1, -1}, {tensor_shape}}; + + bool use_init_subgraph = false; + CPUSpecificParams cpu_params; + std::tie(use_init_subgraph, cpu_params) = this->GetParam(); + std::tie(inFmts, outFmts, priority, selectedType) = cpu_params; + selectedType = makeSelectedTypeStr(selectedType, net_prc); + + init_input_shapes({param1_shape, param2_shape}); + + ov::ParameterVector params; + params.push_back(std::make_shared(net_prc, inputDynamicShapes[0])); + params.push_back(std::make_shared(net_prc, inputDynamicShapes[1])); + std::shared_ptr last_node = params.front(); + + if (use_init_subgraph) { + //build init subgraph + auto const1 = utils::make_constant(net_prc, tensor_shape); + auto const2 = utils::make_constant(net_prc, tensor_shape); + auto multiply = utils::make_eltwise(last_node, const1, utils::EltwiseTypes::MULTIPLY); + auto add = utils::make_eltwise(multiply, const2, utils::EltwiseTypes::ADD); + last_node = add; + } + + const std::string variable_name("variable0"); + auto variable = std::make_shared( + ov::op::util::VariableInfo{inputDynamicShapes[0], net_prc, variable_name}); + + auto read = std::make_shared(last_node, variable); + auto assign = std::make_shared(read, variable); + auto add = utils::make_eltwise(params[1], read, utils::EltwiseTypes::ADD); + + add->get_rt_info() = getCPUInfo(); + auto res = std::make_shared(add); + + function = + std::make_shared(ov::ResultVector({res}), ov::SinkVector({assign}), params, "ReadValueAssign"); + } + +protected: + const ov::Shape tensor_shape = {3, 32, 7, 7}; + const ElementType net_prc = element::f32; +}; + +TEST_P(ReadValueAssignTest, CompareWithRefs) { + compile_model(); + inferRequest = compiledModel.create_infer_request(); + ASSERT_TRUE(inferRequest); + + // use the Template plugin as a reference + + auto compiledReferenceModel = core->compile_model(function, ov::test::utils::DEVICE_TEMPLATE); + auto inferRequestRef = compiledReferenceModel.create_infer_request(); + ASSERT_TRUE(inferRequestRef); + + generate_inputs(targetStaticShapes.front()); + for (const auto& input : inputs) { + inferRequest.set_tensor(input.first, input.second); + inferRequestRef.set_tensor(input.first, input.second); + } + + constexpr int infer_count = 3lu; + + auto&& states = inferRequest.query_state(); + auto&& refStates = inferRequestRef.query_state(); + + for (int i = 0; i < infer_count; ++i) { + // set states + + if (i & 0x1) { + //reset every odd iteration + states.front().reset(); + refStates.front().reset(); + } else { + // generate and set state tensors every even iteration + using ov::test::utils::InputGenerateData; + + auto tensor = + ov::test::utils::create_and_fill_tensor(net_prc, tensor_shape, InputGenerateData{0, 10, 1, i}); + states.front().set_state(tensor); + refStates.front().set_state(tensor); + } + + inferRequest.infer(); + inferRequestRef.infer(); + auto outputs = function->outputs(); + + auto result = inferRequest.get_tensor(outputs[0]); + + auto result_ref = inferRequestRef.get_tensor(outputs[0]); + + ov::test::utils::compare(result, result_ref, 1e-4, 1e-4); + } + CheckNumberOfNodesWithTypes(compiledModel, {"MemoryOutput", "Assign"}, 0); +} + +INSTANTIATE_TEST_SUITE_P( + smoke_ReadValue_Assign, + ReadValueAssignTest, + ::testing::Combine(::testing::Values(true, false), + ::testing::Values(CPUSpecificParams{{nchw, nchw}, {nchw}, {""}, "any_type"}, + CPUSpecificParams{{nhwc, nhwc}, {nhwc}, {""}, "any_type"})), + ReadValueAssignTest::getTestCaseName); +} // namespace test +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/single_layer_tests/stft.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/single_layer_tests/stft.cpp index b5fce57fb22f0c..91505d131d6aa5 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/single_layer_tests/stft.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/single_layer_tests/stft.cpp @@ -16,6 +16,12 @@ const std::vector data_type = {ov::element::f32, ov::element: const std::vector step_size_type = {ov::element::i32, ov::element::i64}; const std::vector> input_shapes = { + { // Static shapes + {{}, {{128}}}, // 1st input + {{}, {{8}}}, // 2nd input + {{}, {{}}}, // 3rd input + {{}, {{}}} // 4th input + }, { // Static shapes {{}, {{1, 128}}}, // 1st input {{}, {{8}}}, // 2nd input @@ -34,6 +40,12 @@ const std::vector> input_shapes = { {{}, {{}}}, // 3rd input {{}, {{}}} // 4th input }, + { // Dynamic dims in the first and second input shape + {{-1}, {{128}}}, // 1st input + {{-1}, {{8}}}, // 2nd input + {{}, {{}}}, // 3rd input + {{}, {{}}} // 4th input + }, { // Dynamic dims in the first and second input shape {{-1, -1}, {{1, 128}, {2, 226}}}, // 1st input {{-1}, {{8}, {16}}}, // 2nd input diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp index e7c006ab97427f..90820d550df179 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp @@ -237,7 +237,6 @@ std::vector disabledTestPatterns() { R"(.*smoke_FakeQuantize.*/FakeQuantizeLayerTest.Inference.*TS=.*3.4.2.5.*LEVELS=255.*)", R"(.*smoke_FakeQuantizePerChannel.*/FakeQuantizeLayerTest.Inference.*TS=.*11.10.22.19.*LEVELS=(255|256).*netPRC=f32.*)", R"(.*smoke_MVN_5D/Mvn6LayerTest.Inference.*TS=.*3.4.2.5.*LEVELS=255.*netPRC=f16.*)", - R"(.*smoke_Snippets_MHAINT8MatMul/MHAINT8MatMul.*)", R"(.*smoke_static/ConvertFqRnnToQuantizedRnn.*2.1.5.*2.1.1.*2.1.1.*)", R"(.*smoke_InterpolateBicubicPillow_Layout_Test/InterpolateLayerCPUTest.CompareWithRefs/ShapeCalcMode=sizes_IS=\[?.2..20.?.?\]_TS.*1.17.4.4.*2.3.10.12.*1.17.4.4.*Sizes.*4.4.*10.20.*10.4.*PARAMETER.*0.0.0.0.*0.0.1.1.*2.3.*)", R"(.*smoke_LoopForCommon/LoopLayerCPUTest.CompareWithRefs/.*_netType=bf16.*)", @@ -358,8 +357,7 @@ std::vector disabledTestPatterns() { retVector.emplace_back(R"(smoke_VariableState/OVInferRequestVariableStateTest.*)"); // Issue: 141705 retVector.emplace_back(R"(.*smoke_arm_Deconv_2D_Planar_FP16/DeconvolutionLayerCPUTest.*INFERENCE_PRECISION_HINT=f16.*)"); - // Issue: 154882 - retVector.emplace_back(R"(.*ConcatMultiQuerySDPTest.*f16.*)"); + retVector.emplace_back(R"(.*ConcatMultiQuerySDPTest.*u8.*)"); #endif #if defined(OPENVINO_ARCH_ARM) @@ -539,6 +537,7 @@ std::vector disabledTestPatterns() { // Skip fp16 tests for paltforms that don't support fp16 precision retVector.emplace_back(R"(.*INFERENCE_PRECISION_HINT=(F|f)16.*)"); retVector.emplace_back(R"(.*Prc=f16.*)"); + retVector.emplace_back(R"(.*ConcatMultiQuerySDPTest.*f16.*HasShapeOf=1.*)"); } else { // Issue 117407 retVector.emplace_back( @@ -563,7 +562,7 @@ std::vector disabledTestPatterns() { // ignored for not supported bf16 platforms retVector.emplace_back(R"(.*smoke_Snippets_EnforcePrecision_bf16.*)"); retVector.emplace_back(R"(.*smoke_Snippets_MHAWOTransposeEnforceBF16.*)"); - retVector.emplace_back(R"(.*smoke_Snippets_MHAEnforceBF16.*)"); + retVector.emplace_back(R"(.*smoke_Snippets_MHA.*EnforceBF16.*)"); retVector.emplace_back(R"(.*ConcatSDPTest.*bf16.*)"); } // [150842] Need to support dynamic K dimension of BF16|INT8 MatMul on AMX systems @@ -572,6 +571,11 @@ std::vector disabledTestPatterns() { retVector.emplace_back(R"(.*smoke_Snippets_MatMul/MatMul.CompareWithRefImpl/.*IS\[0\]=\[\?.\?.\?.\?\].*T\[0\]=(u8|i8|bf16)_T\[1\]=(i8|bf16).*)"); retVector.emplace_back(R"(.*smoke_Snippets_MatMulTransposeB.*IS\[0\]=\[\?.\?.\?.\?\].*T\[0\]=(u8|i8|bf16)_T\[1\]=(i8|bf16).*)"); retVector.emplace_back(R"(.*smoke_Snippets_MatMulBias.*IS\[0\]=\[\?.\?.\?.\?\].*T\[0\]=(u8|i8|bf16)_T\[1\]=(i8|bf16).*)"); + + retVector.emplace_back(R"(.*smoke_Snippets_MHAWOTransposeEnforceBF16_3D.*IS\[1\]=\[2.64.\?\].*)"); + retVector.emplace_back(R"(.*smoke_Snippets_MHA.*BF16.*/MHA.*IS\[0\]=\[(\?|1).(\?|4).(\?|12).(\?|64)\].*)"); + retVector.emplace_back(R"(.*smoke_Snippets_MHA.*BF16.*/MHA.*IS\[0\]=\[\?.\?.\?\].*)"); + retVector.emplace_back(R"(.*smoke_Snippets_(MHAINT8MatMul|MHAQuantMatMul0|MHAFQAfterMatMul_4D|smoke_Snippets_MHAFQ).*IS\[0\]=\[\?.\?.\?\.\?].*)"); } #ifdef SNIPPETS_LIBXSMM_TPP // GN in TPP requires exposing tmp Buffer results outside the loop (ticket: 151234) diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/matmul.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/matmul.cpp index f5057137f9b65c..176f0cb4d46aed 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/matmul.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/matmul.cpp @@ -4,44 +4,26 @@ #include "snippets/matmul.hpp" -#include "common_test_utils/test_constants.hpp" -#include "openvino/runtime/system_conf.hpp" +#include "utils.hpp" namespace ov { namespace test { namespace snippets { -#define STATIC_SHAPES(...) static_shapes_to_test_representation(std::vector>{__VA_ARGS__}) - namespace { -static inline std::vector> quantized_precisions() { - std::vector> prc = {}; - // In Snippets MatMul INT8 is supported only on VNNI/AMX platforms - if (ov::with_cpu_x86_avx512_core_vnni() || ov::with_cpu_x86_avx512_core_amx_int8()) { - prc.emplace_back(std::vector{element::i8, element::i8}); - prc.emplace_back(std::vector{element::u8, element::i8}); - } - return prc; -} - static inline std::vector> precisions() { - std::vector> prc = { - {element::f32, element::f32}, - }; + std::vector> prc = precision_f32(2); // Note: TPP doesn't support low precisions yet #ifndef SNIPPETS_LIBXSMM_TPP - auto quant = quantized_precisions(); + auto quant = quantized_precisions_if_supported(); std::copy(quant.begin(), quant.end(), std::back_inserter(prc)); - // In Snippets MatMul BF16 is supported only on bf16/AMX platforms - if (ov::with_cpu_x86_bfloat16() || ov::with_cpu_x86_avx512_core_amx_bf16()) { - prc.emplace_back(std::vector{element::bf16, element::bf16}); - } + auto bfloat = precision_bf16_if_supported(2); + std::copy(bfloat.begin(), bfloat.end(), std::back_inserter(prc)); #endif return prc; } - std::vector> input_shapes{ { {{}, {{2, 1, 3, 5}}}, {{}, {{1, 3, 5, 3}}} }, { {{}, {{3, 1, 32, 14}}}, {{}, {{1, 3, 14, 37}}} }, @@ -158,7 +140,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulBias, MatMulBias, INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulBiasQuantized, MatMulBiasQuantized, ::testing::Combine( ::testing::ValuesIn(input_shapes_bias), - ::testing::ValuesIn(quantized_precisions()), + ::testing::ValuesIn(quantized_precisions_if_supported()), ::testing::Values(MatMulType::MatMul), ::testing::Values(1), // Subgraph ::testing::Values(1), // Tokenized MatMul+Bias @@ -167,8 +149,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulBiasQuantized, MatMulBiasQuantized INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulsQuantized, MatMulsQuantized, ::testing::Combine( - ::testing::ValuesIn(STATIC_SHAPES({{1, 16, 128, 64}, {1, 16, 64, 128}, {128, 64}})), - ::testing::ValuesIn(quantized_precisions()), + ::testing::ValuesIn(SNIPPETS_TESTS_STATIC_SHAPES({{1, 16, 128, 64}, {1, 16, 64, 128}, {128, 64}})), + ::testing::ValuesIn(quantized_precisions_if_supported()), ::testing::Values(MatMulType::MatMul), ::testing::Values(3), // Subgraph + Reshape + Subgraph ::testing::Values(2), // Tokenized [MatMul+FQ+Matmul] and [FQ] @@ -177,8 +159,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulsQuantized, MatMulsQuantized, INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMulsQuantizedSoftmax, MatMulsQuantizedSoftmax, ::testing::Combine( - ::testing::ValuesIn(STATIC_SHAPES({{1, 16, 128, 64}, {1, 16, 64, 128}, {128, 64}})), - ::testing::ValuesIn(quantized_precisions()), + ::testing::ValuesIn(SNIPPETS_TESTS_STATIC_SHAPES({{1, 16, 128, 64}, {1, 16, 64, 128}, {128, 64}})), + ::testing::ValuesIn(quantized_precisions_if_supported()), ::testing::Values(MatMulType::MatMul), ::testing::Values(3), // Subgraph + Reshape + Subgraph ::testing::Values(2), // Tokenized [MatMul+FQ+Matmul] and [FQ] diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha.cpp index 79db0b1546b2a8..63f5176684ccc1 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha.cpp @@ -1,60 +1,70 @@ -// Copyright (C) 2022 Intel Corporation +// Copyright (C) 2024 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // #include "snippets/mha.hpp" -#include "common_test_utils/test_constants.hpp" -#include "internal_properties.hpp" -#include "utils/cpu_test_utils.hpp" -#include "openvino/runtime/system_conf.hpp" +#include "utils.hpp" namespace ov { namespace test { namespace snippets { -#define STATIC_SHAPES(...) static_shapes_to_test_representation(std::vector>{__VA_ARGS__}) namespace { -const auto& inputShapes_4D = STATIC_SHAPES( - {{1, 128, 12, 64}, {1, 128, 12, 64}, {1, 12, 128, 128}, {1, 128, 12, 64}}, - {{1, 128, 16, 64}, {1, 128, 16, 64}, {1, 16, 1, 1}, {1, 128, 16, 64}}, - {{1, 128, 16, 64}, {1, 128, 16, 64}, {1, 1, 1, 128}, {1, 128, 16, 64}}, - {{2, 68, 6, 92}, {2, 68, 6, 92}, {1, 1, 68, 68}, {2, 68, 6, 92}}, - {{1, 58, 16, 34}, {1, 58, 16, 34}, {1, 1, 1, 58}, {1, 58, 16, 34}}); - -const auto& inputShapes_3D = STATIC_SHAPES( - {{128, 12, 64}, {128, 12, 64}, {12, 128, 128}, {128, 12, 64}}, - {{68, 6, 92}, {68, 6, 92}, {1, 68, 68}, {68, 6, 92}}, - {{16, 2, 92}, {68, 2, 92}, {1, 16, 68}, {68, 2, 92}}); - -static inline bool is_bf16_supported() { - return ov::with_cpu_x86_bfloat16() || ov::with_cpu_x86_avx512_core_amx_bf16(); -} - -static inline std::vector> precision_f32(size_t count) { - std::vector> prc; - prc.emplace_back(std::vector(count, element::f32)); - return prc; -} - -static inline std::vector> precision_bf16(size_t count) { - std::vector> prc; - if (is_bf16_supported()) - prc.emplace_back(std::vector(count, element::bf16)); - return prc; +std::vector> transposedShape_4D(bool with_dynamic = true) { + auto shapes = SNIPPETS_TESTS_STATIC_SHAPES( + {{1, 128, 12, 64}, {1, 128, 12, 64}, {1, 12, 128, 128}, {1, 128, 12, 64}}, + {{1, 128, 16, 64}, {1, 128, 16, 64}, {1, 16, 1, 1}, {1, 128, 16, 64}}, + {{1, 128, 16, 64}, {1, 128, 16, 64}, {1, 1, 1, 128}, {1, 128, 16, 64}}, + {{2, 68, 6, 92}, {2, 68, 6, 92}, {1, 1, 68, 68}, {2, 68, 6, 92}}, + {{1, 58, 16, 34}, {1, 58, 16, 34}, {1, 1, 1, 58}, {1, 58, 16, 34}}); + if (with_dynamic) { + std::vector> dynamic_shapes = {{ + {PartialShape{-1, -1, -1, 100}, {{1, 64, 4, 100}, {2, 16, 2, 100}, {1, 72, 4, 100}}}, + {PartialShape{-1, 128, -1, 100}, {{1, 128, 4, 100}, {2, 128, 2, 100}, {1, 128, 4, 100}}}, + {PartialShape{-1, -1, -1, 128}, {{1, 4, 64, 128}, {2, 2, 16, 128}, {1, 4, 72, 128}}}, + {PartialShape{-1, 128, -1, 100}, {{1, 128, 4, 100}, {2, 128, 2, 100}, {1, 128, 4, 100}}}, + }, + { + {PartialShape{-1, -1, -1, -1}, {{1, 128, 3, 64}, {2, 16, 2, 100}, {1, 128, 3, 64}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 128, 1, 64}, {2, 128, 2, 100}, {1, 128, 1, 64}}}, + {PartialShape{-1, -1, -1, -1}, {{2, 1, 128, 128}, {2, 2, 16, 128}, {2, 1, 128, 128}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 128, 3, 64}, {2, 128, 2, 100}, {1, 128, 3, 64}}}, + }, + { + {PartialShape{-1, -1, 12, 64}, {{1, 70, 12, 64}, {1, 20, 12, 64}, {1, 20, 12, 64}, {1, 20, 12, 64}, {1, 70, 12, 64}}}, + {PartialShape{-1, -1, 12, 64}, {{1, 35, 12, 64}, {2, 10, 12, 64}, {2, 1, 12, 64}, {2, 10, 12, 64}, {1, 35, 12, 64}}}, + {PartialShape{-1, 12, -1, -1}, {{2, 12, 70, 35}, {1, 12, 20, 10}, {1, 12, 20, 10}, {1, 12, 20, 1}, {2, 12, 70, 35}}}, + {PartialShape{-1, -1, 12, 64}, {{1, 35, 12, 64}, {1, 10, 12, 64}, {1, 10, 12, 64}, {1, 10, 12, 64}, {1, 35, 12, 64}}}, + }}; + shapes.insert(shapes.end(), dynamic_shapes.begin(), dynamic_shapes.end()); + } + return shapes; } -static ov::AnyMap enable_callback() { - return ov::AnyMap({ov::intel_cpu::snippets_mode(ov::intel_cpu::SnippetsMode::ENABLE)}); +std::vector> transposedShape_3D(bool with_dynamic = true) { + auto shapes = SNIPPETS_TESTS_STATIC_SHAPES( + {{128, 12, 64}, {128, 12, 64}, {12, 128, 128}, {128, 12, 64}}, + {{68, 6, 92}, {68, 6, 92}, {1, 68, 68}, {68, 6, 92}}, + {{16, 2, 92}, {68, 2, 92}, {1, 16, 68}, {68, 2, 92}}); + if (with_dynamic) { + shapes.push_back({ + {PartialShape{-1, -1, -1}, {{128, 3, 64}, {128, 3, 64}, {68, 6, 87}}}, + {PartialShape{-1, -1, -1}, {{128, 1, 64}, {128, 1, 64}, {13, 6, 87}}}, + {PartialShape{-1, -1, -1}, {{1, 128, 128}, {1, 128, 128}, {1, 68, 13}}}, + {PartialShape{-1, -1, -1}, {{128, 3, 64}, {128, 3, 64}, {13, 6, 87}}}, + }); + } + return shapes; } INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MHA_4D, MHA, - ::testing::Combine(::testing::ValuesIn(inputShapes_4D), + ::testing::Combine(::testing::ValuesIn(transposedShape_4D()), ::testing::ValuesIn(precision_f32(4)), ::testing::Values(ov::element::f32), - ::testing::ValuesIn({false, true}), + ::testing::Values(false), ::testing::Values(MHA::default_thread_count), ::testing::Values(1), ::testing::Values(1), @@ -62,27 +72,12 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MHA_4D, ::testing::Values(CPUTestUtils::empty_plugin_config)), MHA::getTestCaseName); -std::vector> inputShapes_4D_dynamic{ - { - {PartialShape{-1, -1, -1, -1}, {{1, 128, 3, 64}, {1, 70, 3, 19}, {1, 128, 3, 64}, {1, 68, 6, 87}}}, - {PartialShape{-1, -1, -1, -1}, {{1, 128, 1, 64}, {2, 49, 1, 19}, {1, 128, 1, 64}, {2, 13, 6, 87}}}, - {PartialShape{-1, -1, -1, -1}, {{2, 1, 128, 128}, {1, 1, 70, 49}, {2, 1, 128, 128}, {1, 1, 68, 13}}}, - {PartialShape{-1, -1, -1, -1}, {{1, 128, 3, 64}, {1, 49, 3, 19}, {1, 128, 3, 64}, {2, 13, 6, 87}}}, - }, - { - {PartialShape{-1, -1, 12, 64}, {{1, 70, 12, 64}, {1, 20, 12, 64}, {1, 20, 12, 64}, {1, 20, 12, 64}, {1, 70, 12, 64}}}, - {PartialShape{-1, -1, 12, 64}, {{1, 35, 12, 64}, {2, 10, 12, 64}, {2, 1, 12, 64}, {2, 10, 12, 64}, {1, 35, 12, 64}}}, - {PartialShape{-1, 12, -1, -1}, {{2, 12, 70, 35}, {1, 12, 20, 10}, {1, 12, 20, 10}, {1, 12, 20, 1}, {2, 12, 70, 35}}}, - {PartialShape{-1, -1, 12, 64}, {{1, 35, 12, 64}, {1, 10, 12, 64}, {1, 10, 12, 64}, {1, 10, 12, 64}, {1, 35, 12, 64}}}, - } -}; - -INSTANTIATE_TEST_SUITE_P(smoke_Snippets_DynMHA_4D, +INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MHA_4D_WithScalarMul, MHA, - ::testing::Combine(::testing::ValuesIn(inputShapes_4D_dynamic), + ::testing::Combine(::testing::ValuesIn(transposedShape_4D(false)), ::testing::ValuesIn(precision_f32(4)), ::testing::Values(ov::element::f32), - ::testing::ValuesIn({false}), + ::testing::Values(true), ::testing::Values(MHA::default_thread_count), ::testing::Values(1), ::testing::Values(1), @@ -90,13 +85,12 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_DynMHA_4D, ::testing::Values(CPUTestUtils::empty_plugin_config)), MHA::getTestCaseName); - INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MHA_3D, MHA, - ::testing::Combine(::testing::ValuesIn(inputShapes_3D), + ::testing::Combine(::testing::ValuesIn(transposedShape_3D()), ::testing::ValuesIn(precision_f32(4)), ::testing::Values(ov::element::f32), - ::testing::ValuesIn({false, true}), + ::testing::Values(false), ::testing::Values(MHA::default_thread_count), ::testing::Values(5), // [122706]: Subgraph + 4 Transpose ::testing::Values(2), // decomposed Transpose + MHA @@ -104,111 +98,23 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MHA_3D, ::testing::Values(CPUTestUtils::empty_plugin_config)), MHA::getTestCaseName); -const auto& splitm_static_shapes = STATIC_SHAPES({{1, 128, 2, 64}, {1, 128, 2, 64}, {1, 1, 1, 1}, {1, 128, 2, 64}}); - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHA_4D_SplitDimensionM_static, - MHA, - ::testing::Combine(::testing::ValuesIn(splitm_static_shapes), - ::testing::ValuesIn(precision_f32(4)), - ::testing::Values(ov::element::f32), - ::testing::Values(true), - ::testing::Values(4), // 4 Threads - ::testing::Values(6), // Subgraph + 4 Reshapes on inputs and 1 Reshape on output - ::testing::Values(1), - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(enable_callback())), - MHA::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHA_3D_SplitDimensionM_static, - MHA, - ::testing::Combine( - ::testing::ValuesIn(STATIC_SHAPES({{384, 2, 64}, {384, 2, 64}, {1, 384, 384}, {384, 2, 64}})), - ::testing::ValuesIn(precision_f32(4)), - ::testing::Values(ov::element::f32), - ::testing::Values(true), - ::testing::Values(4), // 4 Threads - ::testing::Values(10), // Subgraph + 4 Reshapes on inputs and 1 Reshape on output + 4 Transposes - ::testing::Values(1), // MHA - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(enable_callback())), - MHA::getTestCaseName); - -std::vector> splitm_dynamic_shapes_4d = { - { - {PartialShape{-1, -1, -1, -1}, {{1, 128, 2, 64}, {1, 17, 2, 64}, {1, 128, 2, 64}}}, - {PartialShape{-1, -1, -1, -1}, {{1, 128, 2, 64}, {1, 17, 2, 64}, {1, 128, 2, 64}}}, - {PartialShape{-1, -1, -1, -1}, {{1, 1, 1, 128}, {1, 1, 1, 17}, {1, 1, 1, 128}}}, - {PartialShape{-1, -1, -1, -1}, {{1, 128, 2, 64}, {1, 17, 2, 64}, {1, 128, 2, 64}}}, - }, - { - {PartialShape{-1, 128, -1, -1}, {{1, 128, 2, 64}}}, - {PartialShape{-1, -1, -1, -1}, {{1, 16, 2, 64}}}, - {PartialShape{-1, -1, 128, -1}, {{1, 1, 128, 16}}}, - {PartialShape{-1, -1, -1, -1}, {{1, 16, 2, 32}}}, - }, - { - {PartialShape{-1, 32, -1, -1}, {{1, 32, 2, 64}}}, - {PartialShape{-1, -1, -1, -1}, {{1, 16, 2, 64}}}, - {PartialShape{-1, -1, 32, -1}, {{1, 1, 32, 16}}}, - {PartialShape{-1, -1, -1, -1}, {{1, 16, 2, 32}}}, - }, - { - {PartialShape{-1, -1, -1, -1}, {{1, 16, 2, 64}}}, - {PartialShape{-1, -1, -1, -1}, {{1, 16, 2, 64}}}, - {PartialShape{-1, -1, 16, -1}, {{1, 1, 16, 16}}}, - {PartialShape{-1, -1, -1, -1}, {{1, 16, 2, 32}}}, - }, -}; - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHA_4D_SplitDimensionM_dynamic, - MHA, - ::testing::Combine(::testing::ValuesIn(splitm_dynamic_shapes_4d), - ::testing::ValuesIn(precision_f32(4)), - ::testing::Values(ov::element::f32), - ::testing::Values(false), - ::testing::Values(4), // 4 Threads - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -std::vector> splitm_dynamic_shapes_3d = { - { - {PartialShape{-1, -1, -1}, {{128, 2, 64}, {17, 2, 64}, {128, 2, 64}}}, - {PartialShape{-1, -1, -1}, {{128, 2, 64}, {17, 2, 64}, {128, 2, 64}}}, - {PartialShape{-1, -1, -1}, {{1, 1, 128}, {1, 1, 17}, {1, 1, 128}}}, - {PartialShape{-1, -1, -1}, {{128, 2, 64}, {17, 2, 64}, {128, 2, 64}}}, - }, - { - {PartialShape{-1, 2, 64}, {{128, 2, 64}, {64, 2, 64}, {128, 2, 64}}}, - {PartialShape{-1, 2, 64}, {{128, 2, 64}, {64, 2, 64}, {128, 2, 64}}}, - {PartialShape{1, 1, -1}, {{1, 1, 128}, {1, 1, 64}, {1, 1, 128}}}, - {PartialShape{-1, 2, 64}, {{128, 2, 64}, {64, 2, 64}, {128, 2, 64}}}, - }, -}; - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHA_3D_SplitDimensionM_dynamic, - MHA, - ::testing::Combine(::testing::ValuesIn(splitm_dynamic_shapes_3d), - ::testing::ValuesIn(precision_f32(4)), - ::testing::Values(ov::element::f32), - ::testing::Values(false), - ::testing::Values(4), // 4 Threads - ::testing::Values(5), // Subgraph + 4 Transpose - ::testing::Values(2), // MHA + one of the transposes is executed via Subgraph (because callback is disabled) - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MHA_3D_WithScalarMul, + MHA, + ::testing::Combine(::testing::ValuesIn(transposedShape_3D(false)), + ::testing::ValuesIn(precision_f32(4)), + ::testing::Values(ov::element::f32), + ::testing::Values(true), + ::testing::Values(MHA::default_thread_count), + ::testing::Values(5), // [122706]: Subgraph + 4 Transpose + ::testing::Values(2), // decomposed Transpose + MHA + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MHABF16_4D, MHA, - ::testing::Combine(::testing::ValuesIn(inputShapes_4D), - ::testing::ValuesIn(precision_bf16(4)), + ::testing::Combine(::testing::ValuesIn(transposedShape_4D()), + ::testing::ValuesIn(precision_bf16_if_supported(4)), ::testing::Values(ov::element::f32), ::testing::ValuesIn({false, true}), ::testing::Values(MHA::default_thread_count), @@ -220,7 +126,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MHABF16_4D, INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MHAEnforceBF16, MHA, - ::testing::Combine(::testing::ValuesIn(inputShapes_4D), + ::testing::Combine(::testing::ValuesIn(transposedShape_4D()), ::testing::ValuesIn(precision_f32(4)), ::testing::Values(ov::element::bf16), ::testing::ValuesIn({false}), @@ -231,321 +137,6 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MHAEnforceBF16, ::testing::Values(CPUTestUtils::cpu_bf16_plugin_config)), MHA::getTestCaseName); -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHAMulAdd, - MHAMulAdd, - ::testing::Combine( - ::testing::ValuesIn(STATIC_SHAPES({{1, 10, 12, 16}, {1, 10, 12, 16}, {1, 10, 12, 16}})), - ::testing::ValuesIn(precision_f32(3)), - ::testing::Values(ov::element::f32), - ::testing::ValuesIn({false}), // Need to support True for graph builder in tests - ::testing::Values(MHA::default_thread_count), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -const auto& inputShapeSelect = STATIC_SHAPES( - // without broadcast - {{1, 128, 12, 64}, {1, 128, 12, 64}, {1, 12, 128, 128}, {1, 12, 128, 128}, {1, 12, 128, 128}, {1, 128, 12, 64}}, - {{1, 94, 12, 54}, {1, 94, 12, 54}, {1, 12, 94, 94}, {1, 12, 94, 94}, {1, 12, 94, 94}, {1, 94, 12, 54}}, - // with broadcast - {{1, 128, 12, 64}, {1, 128, 12, 64}, {1, 12, 128, 128}, {1, 12, 1, 1}, {1, 12, 1, 1}, {1, 128, 12, 64}}, - {{2, 52, 6, 102}, {2, 52, 6, 102}, {1, 6, 52, 52}, {1, 6, 1, 1}, {1, 6, 1, 1}, {2, 52, 6, 102}} -); - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHA, - MHASelect, - ::testing::Combine(::testing::ValuesIn(inputShapeSelect), - ::testing::ValuesIn(precision_f32(6)), - ::testing::Values(ov::element::f32), - ::testing::Values(false), // Need to support True for graph builder in tests - ::testing::Values(MHA::default_thread_count), - ::testing::Values(2), // Less + MHA - ::testing::Values(2), - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -const auto& inputShapesWOTranspose_4D = STATIC_SHAPES( - {{1, 12, 197, 64}, {1, 12, 64, 197}, {1, 12, 197, 64}}, - {{1, 12, 12, 64}, {1, 12, 64, 48}, {1, 12, 48, 64}}); -const auto& inputShapesWOTranspose_3D = STATIC_SHAPES( - {{12, 197, 64}, {12, 64, 197}, {12, 197, 64}}, - {{12, 128, 100}, {12, 100, 128}, {12, 128, 100}}); - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHAWOTransposeOnInputs_4D, - MHAWOTransposeOnInputs, - ::testing::Combine(::testing::ValuesIn(inputShapesWOTranspose_4D), - ::testing::Values(std::vector{}), - ::testing::Values(ov::element::f32), - ::testing::Values(true), // Need to support False for graph builder in tests - ::testing::Values(MHA::default_thread_count), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHAWOTranspose_4D, - MHAWOTranspose, - ::testing::Combine(::testing::ValuesIn(inputShapesWOTranspose_4D), - ::testing::ValuesIn(precision_f32(3)), - ::testing::Values(ov::element::f32), - ::testing::ValuesIn({true}), // Need to support False for graph builder in tests - ::testing::Values(MHA::default_thread_count), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHAWOTranspose_3D, - MHAWOTranspose, - ::testing::Combine(::testing::ValuesIn(inputShapesWOTranspose_3D), - ::testing::ValuesIn(precision_f32(3)), - ::testing::Values(ov::element::f32), - ::testing::ValuesIn({true}), // Need to support False for graph builder in tests - ::testing::Values(MHA::default_thread_count), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -std::vector> inputShapesWOTranspose_3D_dynamic{ - { - {PartialShape{-1, -1, -1}, {{12, 19, 85}, {1, 40, 36}}}, - {PartialShape{-1, -1, -1}, {{1, 85, 19}, {2, 36, 40}}}, - {PartialShape{-1, -1, -1}, {{12, 19, 85}, {1, 40, 36}}}, - }, - { - {PartialShape{2, -1, 64}, {{2, 9, 64}, {2, 2, 64}, {2, 9, 64}}}, - {PartialShape{2, 64, -1}, {{2, 64, 9}, {2, 64, 2}, {2, 64, 9}}}, - {PartialShape{2, -1, 64}, {{2, 9, 64}, {2, 2, 64}, {2, 9, 64}}}, - }, -}; - - - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_DynMHAWOTranspose_3D, - MHAWOTranspose, - ::testing::Combine(::testing::ValuesIn(inputShapesWOTranspose_3D_dynamic), - ::testing::ValuesIn(precision_f32(3)), - ::testing::Values(ov::element::f32), - ::testing::ValuesIn({true}), // Need to support False for graph builder in tests - ::testing::Values(MHA::default_thread_count), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHAWOTransposeBF16_4D, - MHAWOTranspose, - ::testing::Combine(::testing::ValuesIn(inputShapesWOTranspose_4D), - ::testing::ValuesIn(precision_bf16(3)), - ::testing::Values(ov::element::f32), - ::testing::ValuesIn({true}), // Need to support False for graph builder in tests - ::testing::Values(MHA::default_thread_count), - ::testing::Values(5), // MHA + 4 extra Converts on inputs and output - ::testing::Values(5), // MHA + 4 extra Converts on inputs and output - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHAWOTransposeBF16_3D, - MHAWOTranspose, - ::testing::Combine(::testing::ValuesIn(inputShapesWOTranspose_3D), - ::testing::ValuesIn(precision_bf16(3)), - ::testing::Values(ov::element::f32), - ::testing::ValuesIn({true}), // Need to support False for graph builder in tests - ::testing::Values(MHA::default_thread_count), - ::testing::Values(5), // MHA + 4 extra Converts on inputs and output - ::testing::Values(5), // MHA + 4 extra Converts on inputs and output - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHAWOTransposeEnforceBF16_4D, - MHAWOTranspose, - ::testing::Combine(::testing::ValuesIn(inputShapesWOTranspose_4D), - ::testing::ValuesIn(precision_f32(3)), - ::testing::Values(ov::element::bf16), - ::testing::ValuesIn({true}), // Need to support False for graph builder in tests - ::testing::Values(MHA::default_thread_count), - ::testing::Values(5), // MHA + 4 extra Converts on inputs and output - ::testing::Values(5), // MHA + 4 extra Converts on inputs and output - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::cpu_bf16_plugin_config)), - MHA::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHAWOTransposeEnforceBF16_3D, - MHAWOTranspose, - ::testing::Combine(::testing::ValuesIn(inputShapesWOTranspose_3D), - ::testing::ValuesIn(precision_f32(3)), - ::testing::Values(ov::element::bf16), - ::testing::ValuesIn({true}), // Need to support False for graph builder in tests - ::testing::Values(MHA::default_thread_count), - ::testing::Values(5), // MHA + 4 extra Converts on inputs and output - ::testing::Values(5), // MHA + 4 extra Converts on inputs and output - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::cpu_bf16_plugin_config)), - MHA::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHAINT8MatMul, - MHAINT8MatMul, - ::testing::Combine(::testing::ValuesIn(std::vector>(inputShapes_4D.begin(), - inputShapes_4D.begin() + 2)), - ::testing::Values(std::vector{}), - ::testing::Values(ov::element::f32), - ::testing::Values(false), // The graph doesn't contain Multiply - ::testing::Values(MHA::default_thread_count), - ::testing::Values(6), // FQx3 on inputs + MHA + Transpose on output + Deq Mul - ::testing::Values(5), // FQx3 on inputs + MHA + Deq Mul - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHAQuantMatMul0, - MHAQuantMatMul0, - ::testing::Combine( - ::testing::ValuesIn(STATIC_SHAPES({{1, 128, 768}, {1, 128, 768}, {1, 1, 1, 128}, {1, 128, 768}})), - ::testing::Values(std::vector{}), - ::testing::Values(ov::element::f32), - ::testing::Values(false), // The graph doesn't contain Multiply - ::testing::Values(MHA::default_thread_count), - ::testing::Values(9), // FQx2 on inputs + MHA + Transpose on output + 4 Reshapes + Deq Mul - ::testing::Values(4), // FQx2 on inputs + MHA + Deq Mul - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MHAFQAfterMatMul_4D, - MHAFQAfterMatMul, - ::testing::Combine(::testing::ValuesIn(inputShapes_4D), - ::testing::Values(std::vector{}), - ::testing::Values(ov::element::f32), - ::testing::Values(false), // The graph doesn't contain Multiply - ::testing::Values(MHA::default_thread_count), - ::testing::Values(3), // MHA + Transpose on output + Deq Mul - ::testing::Values(2), // MHA + Deq Mul - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHAFQ, - MHAFQ, - ::testing::Combine(::testing::ValuesIn(STATIC_SHAPES({{1, 64, 12, 64}, - {1, 64, 12, 64}, - {1, 1, 1, 64}, - {1, 64, 12, 64}})), - ::testing::Values(std::vector{}), - ::testing::Values(ov::element::f32), - ::testing::Values(false), // The graph doesn't contain Multiply - ::testing::Values(MHA::default_thread_count), - ::testing::Values(7), // Transposex2 + Subgraphsx5 - ::testing::Values(5), // MHA + Deq Mul on output + Deqs on inputs + 2 xFQ on inputs - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -std::vector> inputShapesTransposedB { - { - {{}, {{1, 12, 12, 64}}}, - {{}, {{1, 12, 48, 64}}}, - {{}, {{1, 12, 48, 64}}} - }, - { - {PartialShape{-1, 3, -1, 64}, {{1, 3, 12, 64}, {2, 3, 36, 64}}}, - {PartialShape{-1, 3, -1, 64}, {{1, 3, 14, 64}, {2, 3, 42, 64}}}, - {PartialShape{-1, 3, -1, -1}, {{1, 3, 14, 36}, {2, 3, 42, 36}}}, - }, - { - {PartialShape{2, -1, 32, -1}, {{2, 1, 32, 70}, {2, 2, 32, 96}}}, - {PartialShape{2, -1, 49, -1}, {{2, 3, 49, 70}, {2, 1, 49, 96}}}, - {PartialShape{2, -1, 49, -1}, {{2, 1, 49, 17}, {2, 2, 49, 81}}}, - }, -}; - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHATransposedB, - MHATransposedB, - ::testing::Combine(::testing::ValuesIn(inputShapesTransposedB), - ::testing::Values(std::vector{}), - ::testing::Values(ov::element::f32), - ::testing::ValuesIn({true}), // Need to support False for graph builder in tests - ::testing::Values(MHA::default_thread_count), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -const auto& inputShapesExtractedReshape = STATIC_SHAPES( - {{2, 196, 64}, {2, 64, 196}, {2, 14, 14, 14, 1}, {2, 14, 14, 1, 14}, {2, 196, 64}}, - {{1, 16, 10}, {1, 10, 16}, {1, 4, 4, 4, 1}, {1, 4, 4, 1, 4}, {1, 16, 10}}, - {{1, 16, 10}, {1, 10, 16}, {1, 1, 1, 1, 1}, {1, 4, 4, 4, 4}, {1, 16, 10}}, - {{1, 16, 10}, {1, 10, 16}, {1, 4, 4, 4, 4}, {1, 1, 1, 1, 1}, {1, 16, 10}}, - {{1, 4, 16, 10}, {1, 4, 10, 16}, {1, 4, 256}, {1, 4, 256}, {1, 4, 16, 10}}, - {{1, 4, 16, 10}, {1, 4, 10, 16}, {1, 1, 256}, {1, 4, 1}, {1, 4, 16, 10}}); - -INSTANTIATE_TEST_SUITE_P( - smoke_Snippets_MHAWithExtractedReshape, - MHAWithExtractedReshape, - ::testing::Combine(::testing::ValuesIn(inputShapesExtractedReshape), - ::testing::Values(std::vector{}), - ::testing::Values(ov::element::f32), - ::testing::ValuesIn({true}), // False is not supported for graph builder in tests - ::testing::Values(MHA::default_thread_count), - ::testing::Values(3), // Extracted Add + Extracted Reshape + MHA - ::testing::Values(2), // Extracted Add + MHA - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHA::getTestCaseName); - -std::vector> inputShapes_4D_WithMul_dynamic{ - { - {PartialShape{-1, -1, -1, -1}, {{1, 128, 3, 64}, {1, 70, 3, 19}, {1, 128, 3, 64}, {1, 68, 6, 87}}}, - {PartialShape{-1, -1, -1, -1}, {{1, 128, 1, 64}, {2, 49, 1, 19}, {1, 128, 1, 64}, {2, 13, 6, 87}}}, - {PartialShape{1}, {{1}, {1}, {1}, {1} }}, - {PartialShape{-1, -1, -1, -1}, {{2, 1, 128, 128}, {1, 1, 70, 49}, {2, 1, 128, 128}, {1, 1, 68, 13}}}, - {PartialShape{-1, -1, -1, -1}, {{1, 128, 3, 64}, {1, 49, 3, 19}, {1, 128, 3, 64}, {2, 13, 6, 87}}}, - }, - { - {PartialShape{-1, -1, 12, 64}, {{1, 70, 12, 64}, {1, 20, 12, 64}, {1, 20, 12, 64}, {1, 20, 12, 64}, {1, 70, 12, 64}}}, - {PartialShape{-1, -1, 12, 64}, {{1, 35, 12, 64}, {2, 10, 12, 64}, {2, 1, 12, 64}, {2, 10, 12, 64}, {1, 35, 12, 64}}}, - {PartialShape{-1, 12, 64, -1}, {{1, 12, 64, 35}, {1, 12, 64, 10}, {1, 12, 64, 10}, {1, 12, 64, 1}, {1, 12, 64, 35}}}, - {PartialShape{-1, 12, -1, -1}, {{2, 12, 70, 35}, {1, 12, 20, 10}, {1, 12, 20, 10}, {1, 12, 20, 1}, {2, 12, 70, 35}}}, - {PartialShape{-1, -1, 12, 64}, {{1, 35, 12, 64}, {1, 10, 12, 64}, {1, 10, 12, 64}, {1, 10, 12, 64}, {1, 35, 12, 64}}}, - } -}; - -INSTANTIATE_TEST_SUITE_P(smoke_Snippets_DynMHA_4D_WithMul, - MHAWithDynamicMul, - ::testing::Combine(::testing::ValuesIn(inputShapes_4D_WithMul_dynamic), - ::testing::ValuesIn(precision_f32(5)), - ::testing::Values(ov::element::f32), - ::testing::Values(MHA::default_thread_count), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::Values(CPUTestUtils::empty_plugin_config)), - MHAWithDynamicMul::getTestCaseName); - } // namespace } // namespace snippets } // namespace test diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_extracted_reshape.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_extracted_reshape.cpp new file mode 100644 index 00000000000000..f3c1439395650a --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_extracted_reshape.cpp @@ -0,0 +1,40 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/mha.hpp" + +#include "utils.hpp" + +namespace ov { +namespace test { +namespace snippets { + +namespace { + +const auto& inputShapesExtractedReshape = SNIPPETS_TESTS_STATIC_SHAPES( + {{2, 196, 64}, {2, 64, 196}, {2, 14, 14, 14, 1}, {2, 14, 14, 1, 14}, {2, 196, 64}}, + {{1, 16, 10}, {1, 10, 16}, {1, 4, 4, 4, 1}, {1, 4, 4, 1, 4}, {1, 16, 10}}, + {{1, 16, 10}, {1, 10, 16}, {1, 1, 1, 1, 1}, {1, 4, 4, 4, 4}, {1, 16, 10}}, + {{1, 16, 10}, {1, 10, 16}, {1, 4, 4, 4, 4}, {1, 1, 1, 1, 1}, {1, 16, 10}}, + {{1, 4, 16, 10}, {1, 4, 10, 16}, {1, 4, 256}, {1, 4, 256}, {1, 4, 16, 10}}, + {{1, 4, 16, 10}, {1, 4, 10, 16}, {1, 1, 256}, {1, 4, 1}, {1, 4, 16, 10}}); + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHAWithExtractedReshape, + MHAWithExtractedReshape, + ::testing::Combine(::testing::ValuesIn(inputShapesExtractedReshape), + ::testing::Values(std::vector{}), + ::testing::Values(ov::element::f32), + ::testing::ValuesIn({true}), // False is not supported for graph builder in tests + ::testing::Values(MHA::default_thread_count), + ::testing::Values(3), // Extracted Add + Extracted Reshape + MHA + ::testing::Values(2), // Extracted Add + MHA + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +} // namespace +} // namespace snippets +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_fma.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_fma.cpp new file mode 100644 index 00000000000000..4bf35e2daa690d --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_fma.cpp @@ -0,0 +1,33 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/mha.hpp" + +#include "utils.hpp" + +namespace ov { +namespace test { +namespace snippets { + +namespace { + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHAMulAdd, + MHAMulAdd, + ::testing::Combine( + ::testing::ValuesIn(SNIPPETS_TESTS_STATIC_SHAPES({{1, 10, 12, 16}, {1, 10, 12, 16}, {1, 10, 12, 16}})), + ::testing::ValuesIn(precision_f32(3)), + ::testing::Values(ov::element::f32), + ::testing::ValuesIn({false}), // Need to support True for graph builder in tests + ::testing::Values(MHA::default_thread_count), + ::testing::Values(1), + ::testing::Values(1), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +} // namespace +} // namespace snippets +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_quantized.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_quantized.cpp new file mode 100644 index 00000000000000..0c731b74565863 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_quantized.cpp @@ -0,0 +1,103 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/mha.hpp" + +#include "utils.hpp" + +namespace ov { +namespace test { +namespace snippets { + +namespace { + +std::vector> inputShapesQuantized { + { + {{}, {{1, 128, 16, 64}}}, + {{}, {{1, 128, 16, 64}}}, + {{}, {{1, 16, 1, 1}}}, + {{}, {{1, 128, 16, 64}}} + }, + { + {{}, {{2, 68, 6, 92}}}, + {{}, {{2, 68, 6, 92}}}, + {{}, {{1, 1, 68, 68}}}, + {{}, {{2, 68, 6, 92}}} + }, + // K, N are static + { + {PartialShape{-1, -1, -1, 100}, {{1, 64, 4, 100}, {2, 16, 2, 100}, {1, 72, 4, 100}}}, + {PartialShape{-1, 128, -1, 100}, {{1, 128, 4, 100}, {2, 128, 2, 100}, {1, 128, 4, 100}}}, + {PartialShape{-1, -1, -1, 128}, {{1, 4, 64, 128}, {2, 2, 16, 128}, {1, 4, 72, 128}}}, + {PartialShape{-1, 128, -1, 100}, {{1, 128, 4, 100}, {2, 128, 2, 100}, {1, 128, 4, 100}}}, + }, + { + {PartialShape{-1, -1, -1, -1}, {{1, 128, 3, 64}, {2, 16, 2, 100}, {1, 128, 3, 64}, {1, 128, 12, 600}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 128, 1, 64}, {2, 128, 2, 100}, {1, 128, 1, 64}, {1, 128, 12, 600}}}, + {PartialShape{-1, -1, -1, -1}, {{2, 1, 128, 128}, {1, 1, 1, 128}, {2, 1, 128, 128}, {1, 12, 1, 1}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 128, 3, 64}, {2, 128, 2, 100}, {1, 128, 3, 64}, {1, 128, 12, 600}}}, + } +}; + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHAINT8MatMul, + MHAINT8MatMul, + ::testing::Combine(::testing::ValuesIn(inputShapesQuantized), + ::testing::Values(std::vector{}), + ::testing::Values(ov::element::f32), + ::testing::Values(false), // The graph doesn't contain Multiply + ::testing::Values(MHA::default_thread_count), + ::testing::Values(6), // FQx3 on inputs + MHA + Transpose on output + Deq Mul + ::testing::Values(5), // FQx3 on inputs + MHA + Deq Mul + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHAQuantMatMul0, + MHAQuantMatMul0, + ::testing::Combine( + ::testing::ValuesIn(inputShapesQuantized), + ::testing::Values(std::vector{}), + ::testing::Values(ov::element::f32), + ::testing::Values(false), // The graph doesn't contain Multiply + ::testing::Values(MHA::default_thread_count), + ::testing::Values(5), // FQx2 on inputs + MHA + Transpose on output + Deq Mul + ::testing::Values(4), // FQx2 on inputs + MHA + Deq Mul + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHAFQAfterMatMul_4D, + MHAFQAfterMatMul, + ::testing::Combine(::testing::ValuesIn(inputShapesQuantized), + ::testing::Values(std::vector{}), + ::testing::Values(ov::element::f32), + ::testing::Values(false), // The graph doesn't contain Multiply + ::testing::Values(MHA::default_thread_count), + ::testing::Values(3), // MHA + Transpose on output + Deq Mul + ::testing::Values(2), // MHA + Deq Mul + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHAFQ, + MHAFQ, + ::testing::Combine(::testing::ValuesIn(inputShapesQuantized), + ::testing::Values(std::vector{}), + ::testing::Values(ov::element::f32), + ::testing::Values(false), // The graph doesn't contain Multiply + ::testing::Values(MHA::default_thread_count), + ::testing::Values(7), // Transposex2 + Subgraphsx5 + ::testing::Values(5), // MHA + Deq Mul on output + Deqs on inputs + 2 xFQ on inputs + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +} // namespace +} // namespace snippets +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_select.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_select.cpp new file mode 100644 index 00000000000000..3fc1417d20b102 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_select.cpp @@ -0,0 +1,41 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/mha.hpp" + +#include "utils.hpp" + +namespace ov { +namespace test { +namespace snippets { + +namespace { + +const auto& inputShapeSelect = SNIPPETS_TESTS_STATIC_SHAPES( + // without broadcast + {{1, 128, 12, 64}, {1, 128, 12, 64}, {1, 12, 128, 128}, {1, 12, 128, 128}, {1, 12, 128, 128}, {1, 128, 12, 64}}, + {{1, 94, 12, 54}, {1, 94, 12, 54}, {1, 12, 94, 94}, {1, 12, 94, 94}, {1, 12, 94, 94}, {1, 94, 12, 54}}, + // with broadcast + {{1, 128, 12, 64}, {1, 128, 12, 64}, {1, 12, 128, 128}, {1, 12, 1, 1}, {1, 12, 1, 1}, {1, 128, 12, 64}}, + {{2, 52, 6, 102}, {2, 52, 6, 102}, {1, 6, 52, 52}, {1, 6, 1, 1}, {1, 6, 1, 1}, {2, 52, 6, 102}} +); + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHA, + MHASelect, + ::testing::Combine(::testing::ValuesIn(inputShapeSelect), + ::testing::ValuesIn(precision_f32(6)), + ::testing::Values(ov::element::f32), + ::testing::Values(false), // Need to support True for graph builder in tests + ::testing::Values(MHA::default_thread_count), + ::testing::Values(2), // Less + MHA + ::testing::Values(2), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +} // namespace +} // namespace snippets +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_split_dim_m.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_split_dim_m.cpp new file mode 100644 index 00000000000000..bb5f7fe2fa5b52 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_split_dim_m.cpp @@ -0,0 +1,121 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/mha.hpp" + +#include "utils.hpp" + +namespace ov { +namespace test { +namespace snippets { + +namespace { + +static ov::AnyMap enable_callback() { + return ov::AnyMap({ov::intel_cpu::snippets_mode(ov::intel_cpu::SnippetsMode::ENABLE)}); +} + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHA_4D_SplitDimensionM_static, + MHA, + ::testing::Combine(::testing::ValuesIn(SNIPPETS_TESTS_STATIC_SHAPES({{1, 128, 2, 64}, {1, 128, 2, 64}, {1, 1, 1, 1}, {1, 128, 2, 64}})), + ::testing::ValuesIn(precision_f32(4)), + ::testing::Values(ov::element::f32), + ::testing::Values(true), + ::testing::Values(4), // 4 Threads + ::testing::Values(6), // Subgraph + 4 Reshapes on inputs and 1 Reshape on output + ::testing::Values(1), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(enable_callback())), + MHA::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHA_3D_SplitDimensionM_static, + MHA, + ::testing::Combine( + ::testing::ValuesIn(SNIPPETS_TESTS_STATIC_SHAPES({{384, 2, 64}, {384, 2, 64}, {1, 384, 384}, {384, 2, 64}})), + ::testing::ValuesIn(precision_f32(4)), + ::testing::Values(ov::element::f32), + ::testing::Values(true), + ::testing::Values(4), // 4 Threads + ::testing::Values(10), // Subgraph + 4 Reshapes on inputs and 1 Reshape on output + 4 Transposes + ::testing::Values(1), // MHA + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(enable_callback())), + MHA::getTestCaseName); + +std::vector> splitm_dynamic_shapes_4d = { + { + {PartialShape{-1, -1, -1, -1}, {{1, 128, 2, 64}, {1, 17, 2, 64}, {1, 128, 2, 64}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 128, 2, 64}, {1, 17, 2, 64}, {1, 128, 2, 64}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 1, 1, 128}, {1, 1, 1, 17}, {1, 1, 1, 128}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 128, 2, 64}, {1, 17, 2, 64}, {1, 128, 2, 64}}}, + }, + { + {PartialShape{-1, 128, -1, -1}, {{1, 128, 2, 64}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 16, 2, 64}}}, + {PartialShape{-1, -1, 128, -1}, {{1, 1, 128, 16}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 16, 2, 32}}}, + }, + { + {PartialShape{-1, 32, -1, -1}, {{1, 32, 2, 64}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 16, 2, 64}}}, + {PartialShape{-1, -1, 32, -1}, {{1, 1, 32, 16}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 16, 2, 32}}}, + }, + { + {PartialShape{-1, -1, -1, -1}, {{1, 16, 2, 64}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 16, 2, 64}}}, + {PartialShape{-1, -1, 16, -1}, {{1, 1, 16, 16}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 16, 2, 32}}}, + }, +}; + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHA_4D_SplitDimensionM_dynamic, + MHA, + ::testing::Combine(::testing::ValuesIn(splitm_dynamic_shapes_4d), + ::testing::ValuesIn(precision_f32(4)), + ::testing::Values(ov::element::f32), + ::testing::Values(false), + ::testing::Values(4), // 4 Threads + ::testing::Values(1), + ::testing::Values(1), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +std::vector> splitm_dynamic_shapes_3d = { + { + {PartialShape{-1, -1, -1}, {{128, 2, 64}, {17, 2, 64}, {128, 2, 64}}}, + {PartialShape{-1, -1, -1}, {{128, 2, 64}, {17, 2, 64}, {128, 2, 64}}}, + {PartialShape{-1, -1, -1}, {{1, 1, 128}, {1, 1, 17}, {1, 1, 128}}}, + {PartialShape{-1, -1, -1}, {{128, 2, 64}, {17, 2, 64}, {128, 2, 64}}}, + }, + { + {PartialShape{-1, 2, 64}, {{128, 2, 64}, {64, 2, 64}, {128, 2, 64}}}, + {PartialShape{-1, 2, 64}, {{128, 2, 64}, {64, 2, 64}, {128, 2, 64}}}, + {PartialShape{1, 1, -1}, {{1, 1, 128}, {1, 1, 64}, {1, 1, 128}}}, + {PartialShape{-1, 2, 64}, {{128, 2, 64}, {64, 2, 64}, {128, 2, 64}}}, + }, +}; + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHA_3D_SplitDimensionM_dynamic, + MHA, + ::testing::Combine(::testing::ValuesIn(splitm_dynamic_shapes_3d), + ::testing::ValuesIn(precision_f32(4)), + ::testing::Values(ov::element::f32), + ::testing::Values(false), + ::testing::Values(4), // 4 Threads + ::testing::Values(5), // Subgraph + 4 Transpose + ::testing::Values(2), // MHA + one of the transposes is executed via Subgraph (because callback is disabled) + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +} // namespace +} // namespace snippets +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_transposed_b.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_transposed_b.cpp new file mode 100644 index 00000000000000..45260df3cab280 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_transposed_b.cpp @@ -0,0 +1,50 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/mha.hpp" + +#include "utils.hpp" + +namespace ov { +namespace test { +namespace snippets { + +namespace { + +std::vector> inputShapesTransposedB { + { + {{}, {{1, 12, 12, 64}}}, + {{}, {{1, 12, 48, 64}}}, + {{}, {{1, 12, 48, 64}}} + }, + { + {PartialShape{-1, 3, -1, 64}, {{1, 3, 12, 64}, {2, 3, 36, 64}}}, + {PartialShape{-1, 3, -1, 64}, {{1, 3, 14, 64}, {2, 3, 42, 64}}}, + {PartialShape{-1, 3, -1, -1}, {{1, 3, 14, 36}, {2, 3, 42, 36}}}, + }, + { + {PartialShape{2, -1, 32, -1}, {{2, 1, 32, 70}, {2, 2, 32, 96}}}, + {PartialShape{2, -1, 49, -1}, {{2, 3, 49, 70}, {2, 1, 49, 96}}}, + {PartialShape{2, -1, 49, -1}, {{2, 1, 49, 17}, {2, 2, 49, 81}}}, + }, +}; + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHATransposedB, + MHATransposedB, + ::testing::Combine(::testing::ValuesIn(inputShapesTransposedB), + ::testing::Values(std::vector{}), + ::testing::Values(ov::element::f32), + ::testing::ValuesIn({true}), // Need to support False for graph builder in tests + ::testing::Values(MHA::default_thread_count), + ::testing::Values(1), + ::testing::Values(1), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +} // namespace +} // namespace snippets +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_with_dyn_mul.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_with_dyn_mul.cpp new file mode 100644 index 00000000000000..7876d737af2281 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_with_dyn_mul.cpp @@ -0,0 +1,68 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/mha.hpp" + +#include "utils.hpp" + +namespace ov { +namespace test { +namespace snippets { + +namespace { + +std::vector> transposedShape_4D_WithMul { + { + {PartialShape{-1, -1, -1, 100}, {{1, 64, 4, 100}, {2, 16, 2, 100}, {1, 72, 4, 100}}}, + {PartialShape{-1, 200, -1, 100}, {{1, 200, 4, 100}, {2, 200, 2, 100}, {1, 200, 4, 100}}}, + {PartialShape{-1, -1, 100, 200}, {{1, 4, 100, 200}, {2, 2, 100, 200}, {1, 4, 100, 200}}}, + {PartialShape{-1, -1, -1, 200}, {{1, 4, 64, 200}, {2, 2, 16, 200}, {1, 4, 72, 200}}}, + {PartialShape{-1, 200, -1, 100}, {{1, 200, 4, 100}, {2, 200, 2, 100}, {1, 200, 4, 100}}}, + }, + { + {PartialShape{-1, -1, -1, -1}, {{1, 128, 3, 64}, {1, 70, 3, 19}, {1, 128, 3, 64}, {1, 68, 6, 87}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 128, 1, 64}, {2, 49, 1, 19}, {1, 128, 1, 64}, {2, 13, 6, 87}}}, + {PartialShape{1}, {{1}, {1}, {1}, {1} }}, + {PartialShape{-1, -1, -1, -1}, {{2, 1, 128, 128}, {1, 1, 70, 49}, {2, 1, 128, 128}, {1, 1, 68, 13}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 128, 3, 64}, {1, 49, 3, 19}, {1, 128, 3, 64}, {2, 13, 6, 87}}}, + }, + { + {PartialShape{-1, -1, 12, 64}, {{1, 70, 12, 64}, {1, 20, 12, 64}, {1, 20, 12, 64}, {1, 20, 12, 64}, {1, 70, 12, 64}}}, + {PartialShape{-1, -1, 12, 64}, {{1, 35, 12, 64}, {2, 10, 12, 64}, {2, 1, 12, 64}, {2, 10, 12, 64}, {1, 35, 12, 64}}}, + {PartialShape{-1, 12, 64, -1}, {{1, 12, 64, 35}, {1, 12, 64, 10}, {1, 12, 64, 10}, {1, 12, 64, 1}, {1, 12, 64, 35}}}, + {PartialShape{-1, 12, -1, -1}, {{2, 12, 70, 35}, {1, 12, 20, 10}, {1, 12, 20, 10}, {1, 12, 20, 1}, {2, 12, 70, 35}}}, + {PartialShape{-1, -1, 12, 64}, {{1, 35, 12, 64}, {1, 10, 12, 64}, {1, 10, 12, 64}, {1, 10, 12, 64}, {1, 35, 12, 64}}}, + } +}; + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHA_4D_WithDynamicMul, + MHAWithDynamicMul, + ::testing::Combine(::testing::ValuesIn(transposedShape_4D_WithMul), + ::testing::ValuesIn(precision_f32(5)), + ::testing::Values(ov::element::f32), + ::testing::Values(MHA::default_thread_count), + ::testing::Values(1), + ::testing::Values(1), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHAWithDynamicMul::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHA_4D_WithDynamicMul_EnforceBF16, + MHAWithDynamicMul, + ::testing::Combine(::testing::ValuesIn(transposedShape_4D_WithMul), + ::testing::ValuesIn(precision_f32(5)), + ::testing::Values(ov::element::bf16), + ::testing::Values(MHA::default_thread_count), + ::testing::Values(8), // MHA + 1 Transpose on output + 6 Converts around + ::testing::Values(7), // MHA + 6 Converts around + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHAWithDynamicMul::getTestCaseName); + +} // namespace +} // namespace snippets +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_wo_transpose.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_wo_transpose.cpp new file mode 100644 index 00000000000000..0967ef27087674 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/mha_wo_transpose.cpp @@ -0,0 +1,151 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "snippets/mha.hpp" + +#include "utils.hpp" + +namespace ov { +namespace test { +namespace snippets { + +namespace { + +std::vector> originalShape_4D { + { {{}, {{1, 12, 197, 64}}}, {{}, {{1, 12, 64, 197}}}, {{}, {{1, 12, 197, 64}}} }, + { {{}, {{1, 12, 12, 64}}}, {{}, {{1, 12, 64, 48}}}, {{}, {{1, 12, 48, 64}}} }, + { + {PartialShape{-1, -1, -1, -1}, {{1, 3, 128, 64}, {1, 12, 197, 100}, {1, 3, 128, 64}, {1, 12, 197, 600}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 3, 64, 128}, {1, 12, 100, 197}, {1, 3, 64, 128}, {1, 12, 600, 197}}}, + {PartialShape{-1, -1, -1, -1}, {{1, 3, 128, 64}, {1, 12, 197, 100}, {1, 3, 128, 64}, {1, 12, 197, 600}}}, + }, + { + {PartialShape{1, 4, -1, -1}, {{1, 4, 384, 64}, {1, 4, 197, 64}, {1, 4, 384, 560}}}, + {PartialShape{1, 4, -1, -1}, {{1, 4, 64, 128}, {1, 4, 64, 197}, {1, 4, 560, 384}}}, + {PartialShape{1, 4, -1, 64}, {{1, 4, 128, 64}, {1, 4, 197, 64}, {1, 4, 384, 64}}}, + } +}; + +std::vector> originalShape_3D { + { {{}, {{12, 197, 64}}}, {{}, {{12, 64, 197}}}, {{}, {{12, 197, 64}}} }, + { {{}, {{12, 128, 100}}}, {{}, {{12, 100, 128}}}, {{}, {{12, 128, 100}}} }, + { + {PartialShape{-1, -1, 64}, {{2, 9, 64}, {1, 64, 64}, {2, 64, 64}}}, + {PartialShape{-1, 64, 124}, {{2, 64, 124}, {1, 64, 124}, {2, 64, 124}}}, + {PartialShape{-1, 124, 64}, {{2, 124, 64}, {1, 124, 64}, {2, 124, 64}}}, + }, + { + {PartialShape{-1, -1, -1}, {{12, 19, 85}, {1, 40, 36}}}, + {PartialShape{-1, -1, -1}, {{1, 85, 19}, {2, 36, 40}}}, + {PartialShape{-1, -1, -1}, {{12, 19, 85}, {1, 40, 36}}}, + }, + { + {PartialShape{2, -1, 64}, {{2, 9, 64}, {2, 4, 64}, {2, 9, 64}}}, + {PartialShape{2, 64, -1}, {{2, 64, 9}, {2, 64, 4}, {2, 64, 9}}}, + {PartialShape{2, -1, 64}, {{2, 9, 64}, {2, 4, 64}, {2, 9, 64}}}, + } +}; + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHAWOTransposeOnInputs_4D, + MHAWOTransposeOnInputs, + ::testing::Combine(::testing::ValuesIn(originalShape_4D), + ::testing::Values(std::vector{}), + ::testing::Values(ov::element::f32), + ::testing::Values(true), // Need to support False for graph builder in tests + ::testing::Values(MHA::default_thread_count), + ::testing::Values(1), + ::testing::Values(1), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHAWOTranspose_4D, + MHAWOTranspose, + ::testing::Combine(::testing::ValuesIn(originalShape_4D), + ::testing::ValuesIn(precision_f32(3)), + ::testing::Values(ov::element::f32), + ::testing::Values(true), // Need to support False for graph builder in tests + ::testing::Values(MHA::default_thread_count), + ::testing::Values(1), + ::testing::Values(1), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHAWOTranspose_3D, + MHAWOTranspose, + ::testing::Combine(::testing::ValuesIn(originalShape_3D), + ::testing::ValuesIn(precision_f32(3)), + ::testing::Values(ov::element::f32), + ::testing::Values(true), // Need to support False for graph builder in tests + ::testing::Values(MHA::default_thread_count), + ::testing::Values(1), + ::testing::Values(1), + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHAWOTransposeBF16_4D, + MHAWOTranspose, + ::testing::Combine(::testing::ValuesIn(originalShape_4D), + ::testing::ValuesIn(precision_bf16_if_supported(3)), + ::testing::Values(ov::element::f32), + ::testing::Values(true), // Need to support False for graph builder in tests + ::testing::Values(MHA::default_thread_count), + ::testing::Values(5), // MHA + 4 extra Converts on inputs and output + ::testing::Values(5), // MHA + 4 extra Converts on inputs and output + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHAWOTransposeBF16_3D, + MHAWOTranspose, + ::testing::Combine(::testing::ValuesIn(originalShape_3D), + ::testing::ValuesIn(precision_bf16_if_supported(3)), + ::testing::Values(ov::element::f32), + ::testing::Values(true), // Need to support False for graph builder in tests + ::testing::Values(MHA::default_thread_count), + ::testing::Values(5), // MHA + 4 extra Converts on inputs and output + ::testing::Values(5), // MHA + 4 extra Converts on inputs and output + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::empty_plugin_config)), + MHA::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHAWOTransposeEnforceBF16_4D, + MHAWOTranspose, + ::testing::Combine(::testing::ValuesIn(originalShape_4D), + ::testing::ValuesIn(precision_f32(3)), + ::testing::Values(ov::element::bf16), + ::testing::Values(true), // Need to support False for graph builder in tests + ::testing::Values(MHA::default_thread_count), + ::testing::Values(5), // MHA + 4 extra Converts on inputs and output + ::testing::Values(5), // MHA + 4 extra Converts on inputs and output + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::cpu_bf16_plugin_config)), + MHA::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + smoke_Snippets_MHAWOTransposeEnforceBF16_3D, + MHAWOTranspose, + ::testing::Combine(::testing::ValuesIn(originalShape_3D), + ::testing::ValuesIn(precision_f32(3)), + ::testing::Values(ov::element::bf16), + ::testing::Values(true), // Need to support False for graph builder in tests + ::testing::Values(MHA::default_thread_count), + ::testing::Values(5), // MHA + 4 extra Converts on inputs and output + ::testing::Values(5), // MHA + 4 extra Converts on inputs and output + ::testing::Values(ov::test::utils::DEVICE_CPU), + ::testing::Values(CPUTestUtils::cpu_bf16_plugin_config)), + MHA::getTestCaseName); + +} // namespace +} // namespace snippets +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/transpose_matmul.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/transpose_matmul.cpp index c05087283305e4..ea7de9ccb209ad 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/transpose_matmul.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/transpose_matmul.cpp @@ -6,36 +6,28 @@ #include "common_test_utils/test_constants.hpp" #include "openvino/runtime/system_conf.hpp" +#include "utils.hpp" namespace ov { namespace test { namespace snippets { -#define STATIC_SHAPES(...) static_shapes_to_test_representation(std::vector>{__VA_ARGS__}) - namespace { static inline std::vector> precisions(bool only_fp32 = true) { - std::vector> prc = { - {element::f32, element::f32}, - }; -// Note: low precisions are not supported by TPP yet (ticker: 130010) + std::vector> prc = precision_f32(2); +// Note: TPP doesn't support low precisions yet #ifndef SNIPPETS_LIBXSMM_TPP if (!only_fp32) { - // In Snippets MatMul INT8 is supported only on VNNI/AMX platforms - if (ov::with_cpu_x86_avx512_core_vnni() || ov::with_cpu_x86_avx512_core_amx_int8()) { - prc.emplace_back(std::vector{element::i8, element::i8}); - prc.emplace_back(std::vector{element::u8, element::i8}); - } - // In Snippets MatMul BF16 is supported only on bf16/AMX platforms - if (ov::with_cpu_x86_bfloat16() || ov::with_cpu_x86_avx512_core_amx_bf16()) { - prc.emplace_back(std::vector{element::bf16, element::bf16}); - } + auto quant = quantized_precisions_if_supported(); + std::copy(quant.begin(), quant.end(), std::back_inserter(prc)); + auto bfloat = precision_bf16_if_supported(2); + std::copy(bfloat.begin(), bfloat.end(), std::back_inserter(prc)); } #endif return prc; } namespace transpose_zero_input { -const auto& transpose_input_shapes = STATIC_SHAPES({{1, 49, 2, 23}, {2, 2, 23, 39}}); +const auto& transpose_input_shapes = SNIPPETS_TESTS_STATIC_SHAPES({{1, 49, 2, 23}, {2, 2, 23, 39}}); INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMult, TransposeMatMul, ::testing::Combine( ::testing::ValuesIn(transpose_input_shapes), @@ -84,7 +76,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_FullyConnected, TransposeMatMul, } // namespace transpose_zero_input namespace transpose_first_input { -const auto& transpose_input_shapes = STATIC_SHAPES({{2, 1, 49, 13}, {1, 13, 3, 39}}); +const auto& transpose_input_shapes = SNIPPETS_TESTS_STATIC_SHAPES({{2, 1, 49, 13}, {1, 13, 3, 39}}); INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMult, TransposeMatMul, ::testing::Combine( ::testing::ValuesIn(transpose_input_shapes), @@ -126,7 +118,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_TransposeMatMulFQ, TransposeMatMulFQ, } // namespace transpose_first_input namespace transpose_output { -const auto& transpose_input_shapes = STATIC_SHAPES({{2, 1, 49, 13}, {1, 2, 13, 39}}); +const auto& transpose_input_shapes = SNIPPETS_TESTS_STATIC_SHAPES({{2, 1, 49, 13}, {1, 2, 13, 39}}); INSTANTIATE_TEST_SUITE_P(smoke_Snippets_MatMult, TransposeMatMul, ::testing::Combine( @@ -195,7 +187,7 @@ static inline std::vector> precisions(bool only_fp32 } INSTANTIATE_TEST_SUITE_P(smoke_Snippets_ExplicitTransposeMatMul, ExplicitTransposeMatMul, ::testing::Combine( - ::testing::ValuesIn(STATIC_SHAPES({{1, 2, 69, 43}, {2, 49, 2, 43}})), + ::testing::ValuesIn(SNIPPETS_TESTS_STATIC_SHAPES({{1, 2, 69, 43}, {2, 49, 2, 43}})), ::testing::Values(1), // Transpose on second input ::testing::ValuesIn(precisions()), ::testing::Values(MatMulType::MatMul), @@ -223,7 +215,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_Snippets_DynExplicitTransposeMatMul, ExplicitTran INSTANTIATE_TEST_SUITE_P(smoke_Snippets_TransposeMatMulBias, ExplicitTransposeMatMulBias, ::testing::Combine( - ::testing::ValuesIn(STATIC_SHAPES({{1, 2, 69, 43}, {2, 49, 2, 43}, {1, 1, 69, 49}})), + ::testing::ValuesIn(SNIPPETS_TESTS_STATIC_SHAPES({{1, 2, 69, 43}, {2, 49, 2, 43}, {1, 1, 69, 49}})), ::testing::Values(1), // Transpose on second input ::testing::ValuesIn(precisions()), ::testing::Values(MatMulType::MatMul), diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/utils.hpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/utils.hpp new file mode 100644 index 00000000000000..6c0d54da973086 --- /dev/null +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/snippets/utils.hpp @@ -0,0 +1,48 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "internal_properties.hpp" +#include "utils/cpu_test_utils.hpp" +#include "openvino/runtime/system_conf.hpp" + +namespace ov { +namespace test { +namespace snippets { + +#define SNIPPETS_TESTS_STATIC_SHAPES(...) static_shapes_to_test_representation(std::vector>{__VA_ARGS__}) + +static inline bool is_bf16_supported_by_brgemm() { + return ov::with_cpu_x86_bfloat16() || ov::with_cpu_x86_avx512_core_amx_bf16(); +} + +static inline bool is_i8_supported_by_brgemm() { + return ov::with_cpu_x86_avx512_core_vnni() || ov::with_cpu_x86_avx512_core_amx_int8(); +} + +static inline std::vector> precision_f32(size_t count) { + std::vector> prc; + prc.emplace_back(std::vector(count, element::f32)); + return prc; +} + +static inline std::vector> precision_bf16_if_supported(size_t count) { + std::vector> prc; + if (is_bf16_supported_by_brgemm()) + prc.emplace_back(std::vector(count, element::bf16)); + return prc; +} + +static inline std::vector> quantized_precisions_if_supported() { + std::vector> prc = {}; + // In Snippets MatMul INT8 is supported only on VNNI/AMX platforms + if (is_i8_supported_by_brgemm()) { + prc.emplace_back(std::vector{element::i8, element::i8}); + prc.emplace_back(std::vector{element::u8, element::i8}); + } + return prc; +} + +} // namespace snippets +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_cpu/tests/unit/shape_inference_test/lstm_seq_shape_inference_test.cpp b/src/plugins/intel_cpu/tests/unit/shape_inference_test/lstm_seq_shape_inference_test.cpp index 067477e146f509..a8b4160f405fd1 100644 --- a/src/plugins/intel_cpu/tests/unit/shape_inference_test/lstm_seq_shape_inference_test.cpp +++ b/src/plugins/intel_cpu/tests/unit/shape_inference_test/lstm_seq_shape_inference_test.cpp @@ -9,184 +9,6 @@ using namespace ov; using namespace ov::intel_cpu; -class LSTMSequenceV0StaticShapeInferenceTest : public OpStaticShapeInferenceTest { -protected: - void SetUp() override { - this->output_shapes = ShapeVector(3); - } -}; - -TEST_F(LSTMSequenceV0StaticShapeInferenceTest, default_ctor) { - constexpr size_t batch_size = 2; - constexpr size_t input_size = 3; - constexpr size_t hidden_size = 5; - constexpr size_t seq_len = 4; - constexpr size_t num_directions = 1; - constexpr size_t gates_count = 4; - - const auto op = make_op(); - - input_shapes = {StaticShape{batch_size, seq_len, input_size}, // X - StaticShape{batch_size, num_directions, hidden_size}, // H_t - StaticShape{batch_size, num_directions, hidden_size}, // C_t - StaticShape{batch_size}, // seq_lengths - StaticShape{num_directions, gates_count * hidden_size, input_size}, // W - StaticShape{num_directions, gates_count * hidden_size, hidden_size}, // R - StaticShape{num_directions, gates_count * hidden_size}, // B - StaticShape{num_directions, (gates_count - 1) * hidden_size}}; // P - - output_shapes = shape_inference(op.get(), input_shapes); - - EXPECT_EQ(output_shapes.size(), 3); - EXPECT_EQ(output_shapes[0], StaticShape({batch_size, num_directions, seq_len, hidden_size})); - EXPECT_EQ(output_shapes[1], StaticShape({batch_size, num_directions, hidden_size})); - EXPECT_EQ(output_shapes[2], StaticShape({batch_size, num_directions, hidden_size})); -} - -TEST_F(LSTMSequenceV0StaticShapeInferenceTest, FORWARD_without_P) { - constexpr size_t batch_size = 2; - constexpr size_t input_size = 3; - constexpr size_t hidden_size = 5; - constexpr size_t seq_len = 4; - constexpr size_t num_directions = 1; - constexpr size_t gates_count = 4; - - constexpr auto direction = op::RecurrentSequenceDirection::FORWARD; - - const auto X = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto H_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto C_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto seq_lengths = std::make_shared(element::f32, PartialShape::dynamic(1)); - const auto W = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto R = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto B = std::make_shared(element::f32, PartialShape::dynamic(2)); - - const auto op = make_op(X, H_t, C_t, seq_lengths, W, R, B, hidden_size, direction); - - input_shapes = {StaticShape{batch_size, seq_len, input_size}, // X - StaticShape{batch_size, num_directions, hidden_size}, // H_t - StaticShape{batch_size, num_directions, hidden_size}, // C_t - StaticShape{batch_size}, // seq_lengths - StaticShape{num_directions, gates_count * hidden_size, input_size}, // W - StaticShape{num_directions, gates_count * hidden_size, hidden_size}, // R - StaticShape{num_directions, gates_count * hidden_size}}; // B - - output_shapes = shape_inference(op.get(), input_shapes); - - EXPECT_EQ(output_shapes.size(), 3); - EXPECT_EQ(output_shapes[0], StaticShape({batch_size, num_directions, seq_len, hidden_size})); - EXPECT_EQ(output_shapes[1], StaticShape({batch_size, num_directions, hidden_size})); - EXPECT_EQ(output_shapes[2], StaticShape({batch_size, num_directions, hidden_size})); -} - -TEST_F(LSTMSequenceV0StaticShapeInferenceTest, FORWARD_with_P) { - constexpr size_t batch_size = 2; - constexpr size_t input_size = 3; - constexpr size_t hidden_size = 5; - constexpr size_t seq_len = 4; - constexpr size_t num_directions = 1; - constexpr size_t gates_count = 4; - - constexpr auto direction = op::RecurrentSequenceDirection::FORWARD; - - const auto X = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto H_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto C_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto seq_lengths = std::make_shared(element::f32, PartialShape::dynamic(1)); - const auto W = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto R = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto B = std::make_shared(element::f32, PartialShape::dynamic(2)); - const auto P = std::make_shared(element::f32, PartialShape::dynamic(2)); - - const auto op = make_op(X, H_t, C_t, seq_lengths, W, R, B, P, hidden_size, direction); - - input_shapes = {StaticShape{batch_size, seq_len, input_size}, // X - StaticShape{batch_size, num_directions, hidden_size}, // H_t - StaticShape{batch_size, num_directions, hidden_size}, // C_t - StaticShape{batch_size}, // seq_lengths - StaticShape{num_directions, gates_count * hidden_size, input_size}, // W - StaticShape{num_directions, gates_count * hidden_size, hidden_size}, // R - StaticShape{num_directions, gates_count * hidden_size}, // B - StaticShape{num_directions, (gates_count - 1) * hidden_size}}; // P - - output_shapes = shape_inference(op.get(), input_shapes); - - EXPECT_EQ(output_shapes.size(), 3); - EXPECT_EQ(output_shapes[0], StaticShape({batch_size, num_directions, seq_len, hidden_size})); - EXPECT_EQ(output_shapes[1], StaticShape({batch_size, num_directions, hidden_size})); - EXPECT_EQ(output_shapes[2], StaticShape({batch_size, num_directions, hidden_size})); -} - -TEST_F(LSTMSequenceV0StaticShapeInferenceTest, REVERSE) { - constexpr size_t batch_size = 2; - constexpr size_t input_size = 3; - constexpr size_t hidden_size = 5; - constexpr size_t seq_len = 4; - constexpr size_t num_directions = 1; - constexpr size_t gates_count = 4; - - constexpr auto direction = op::RecurrentSequenceDirection::REVERSE; - - const auto X = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto H_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto C_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto seq_lengths = std::make_shared(element::f32, PartialShape::dynamic(1)); - const auto W = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto R = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto B = std::make_shared(element::f32, PartialShape::dynamic(2)); - - const auto op = make_op(X, H_t, C_t, seq_lengths, W, R, B, hidden_size, direction); - - input_shapes = {StaticShape{batch_size, seq_len, input_size}, // X - StaticShape{batch_size, num_directions, hidden_size}, // H_t - StaticShape{batch_size, num_directions, hidden_size}, // C_t - StaticShape{batch_size}, // seq_lengths - StaticShape{num_directions, gates_count * hidden_size, input_size}, // W - StaticShape{num_directions, gates_count * hidden_size, hidden_size}, // R - StaticShape{num_directions, gates_count * hidden_size}}; // B - - output_shapes = shape_inference(op.get(), input_shapes); - EXPECT_EQ(output_shapes.size(), 3); - EXPECT_EQ(output_shapes[0], StaticShape({batch_size, num_directions, seq_len, hidden_size})); - EXPECT_EQ(output_shapes[1], StaticShape({batch_size, num_directions, hidden_size})); - EXPECT_EQ(output_shapes[2], StaticShape({batch_size, num_directions, hidden_size})); -} - -TEST_F(LSTMSequenceV0StaticShapeInferenceTest, BIDIRECTIONAL) { - constexpr size_t batch_size = 2; - constexpr size_t input_size = 3; - constexpr size_t hidden_size = 5; - constexpr size_t seq_len = 4; - constexpr size_t num_directions = 2; - constexpr size_t gates_count = 4; - - constexpr auto direction = op::RecurrentSequenceDirection::BIDIRECTIONAL; - - const auto X = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto H_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto C_t = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto seq_lengths = std::make_shared(element::f32, PartialShape::dynamic(1)); - const auto W = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto R = std::make_shared(element::f32, PartialShape::dynamic(3)); - const auto B = std::make_shared(element::f32, PartialShape::dynamic(2)); - - const auto op = make_op(X, H_t, C_t, seq_lengths, W, R, B, hidden_size, direction); - - input_shapes = {StaticShape{batch_size, seq_len, input_size}, // X - StaticShape{batch_size, num_directions, hidden_size}, // H_t - StaticShape{batch_size, num_directions, hidden_size}, // C_t - StaticShape{batch_size}, // seq_lengths - StaticShape{num_directions, gates_count * hidden_size, input_size}, // W - StaticShape{num_directions, gates_count * hidden_size, hidden_size}, // R - StaticShape{num_directions, gates_count * hidden_size}}; // B - - output_shapes = shape_inference(op.get(), input_shapes); - EXPECT_EQ(output_shapes.size(), 3); - EXPECT_EQ(output_shapes[0], StaticShape({batch_size, num_directions, seq_len, hidden_size})); - EXPECT_EQ(output_shapes[1], StaticShape({batch_size, num_directions, hidden_size})); - EXPECT_EQ(output_shapes[2], StaticShape({batch_size, num_directions, hidden_size})); -} - class LSTMSequenceV5StaticShapeInferenceTest : public OpStaticShapeInferenceTest { protected: void SetUp() override { diff --git a/src/plugins/intel_cpu/tests/unit/shape_inference_test/squeeze_shape_inference_test.cpp b/src/plugins/intel_cpu/tests/unit/shape_inference_test/squeeze_shape_inference_test.cpp index 69da74b10a2f45..5f790135780013 100644 --- a/src/plugins/intel_cpu/tests/unit/shape_inference_test/squeeze_shape_inference_test.cpp +++ b/src/plugins/intel_cpu/tests/unit/shape_inference_test/squeeze_shape_inference_test.cpp @@ -16,14 +16,16 @@ using namespace ov; using namespace ov::intel_cpu; using namespace testing; -class SqueezeStaticShapeInferenceAssertTest : public OpStaticShapeInferenceTest { +namespace v0 { + +class SqueezeV0StaticShapeInferenceAssertTest : public OpStaticShapeInferenceTest { protected: void SetUp() override { output_shapes = ShapeVector(1); } }; -TEST_F(SqueezeStaticShapeInferenceAssertTest, no_axes) { +TEST_F(SqueezeV0StaticShapeInferenceAssertTest, no_axes) { const auto arg = std::make_shared(element::f64, PartialShape{-1, -1}); const auto axes = std::make_shared(element::i64, PartialShape{1}); const auto op = make_op(arg, axes); @@ -35,7 +37,7 @@ TEST_F(SqueezeStaticShapeInferenceAssertTest, no_axes) { HasSubstr("Check 'constant != nullptr'")); } -TEST_F(SqueezeStaticShapeInferenceAssertTest, parameter_static_shape_axes_no_data) { +TEST_F(SqueezeV0StaticShapeInferenceAssertTest, parameter_static_shape_axes_no_data) { const auto arg = std::make_shared(element::f64, Shape{2, 1, 3, 1}); const auto axes = std::make_shared(element::i64, Shape{2}); const auto op = make_op(arg, axes); @@ -52,11 +54,11 @@ using TestParams = std::tuple; -class SqueezeStaticShapeInferenceTest : public SqueezeStaticShapeInferenceAssertTest, +class SqueezeV0StaticShapeInferenceTest : public SqueezeV0StaticShapeInferenceAssertTest, public WithParamInterface { protected: void SetUp() override { - SqueezeStaticShapeInferenceAssertTest::SetUp(); + SqueezeV0StaticShapeInferenceAssertTest::SetUp(); std::tie(input_shapes, axes, exp_shape) = GetParam(); output_shapes = ShapeVector(1); @@ -68,7 +70,7 @@ class SqueezeStaticShapeInferenceTest : public SqueezeStaticShapeInferenceAssert }; INSTANTIATE_TEST_SUITE_P(1d_shapes, - SqueezeStaticShapeInferenceTest, + SqueezeV0StaticShapeInferenceTest, Values(make_tuple(ShapeVector{{1}, {1}}, std::vector{-1}, StaticShape({})), make_tuple(ShapeVector{{6}, {1}}, std::vector{-1}, StaticShape({6})), make_tuple(ShapeVector{{1}, {1}}, std::vector{0}, StaticShape({}))), @@ -76,7 +78,7 @@ INSTANTIATE_TEST_SUITE_P(1d_shapes, INSTANTIATE_TEST_SUITE_P( multi_dim_shapes, - SqueezeStaticShapeInferenceTest, + SqueezeV0StaticShapeInferenceTest, Values(make_tuple(ShapeVector{{1, 2, 3, 1}, {2}}, std::vector{0, 3}, StaticShape({2, 3})), make_tuple(ShapeVector{{2, 1, 1, 4}, {2}}, std::vector{2, 1}, StaticShape({2, 4})), make_tuple(ShapeVector{{2, 1, 1, 4, 1}, {2}}, std::vector{0, 1, -2, -1}, StaticShape({2, 1, 4})), @@ -92,7 +94,7 @@ INSTANTIATE_TEST_SUITE_P( INSTANTIATE_TEST_SUITE_P( multi_dim_shapes_repeated_axis, - SqueezeStaticShapeInferenceTest, + SqueezeV0StaticShapeInferenceTest, Values(make_tuple(ShapeVector{{2, 1, 3}, {2}}, std::vector{1, 1}, StaticShape({2, 3})), make_tuple(ShapeVector{{3, 1, 2, 1}, {3}}, std::vector{1, -1, 1}, StaticShape({3, 2})), make_tuple(ShapeVector{{3, 1, 2, 1}, {3}}, std::vector{1, -1, 1, -1}, StaticShape({3, 2})), @@ -100,7 +102,7 @@ INSTANTIATE_TEST_SUITE_P( make_tuple(ShapeVector{{2, 6, 7, 8, 1}, {2}}, std::vector{-1, -1}, StaticShape({2, 6, 7, 8}))), PrintToStringParamName()); -TEST_P(SqueezeStaticShapeInferenceTest, shape_inference_empty_const_map) { +TEST_P(SqueezeV0StaticShapeInferenceTest, shape_inference_empty_const_map) { const auto axes_node = std::make_shared(element::i64, Shape{axes.size()}, axes); const auto op = make_op(arg, axes_node); @@ -109,8 +111,8 @@ TEST_P(SqueezeStaticShapeInferenceTest, shape_inference_empty_const_map) { ASSERT_EQ(output_shapes.front(), exp_shape); } -TEST_P(SqueezeStaticShapeInferenceTest, shape_inference_with_const_map) { - const auto axes_node = std::make_shared(element::i64, Shape{1}); +TEST_P(SqueezeV0StaticShapeInferenceTest, shape_inference_with_const_map) { + const auto axes_node = std::make_shared(element::i64, ov::PartialShape::dynamic()); const auto op = make_op(arg, axes_node); const auto axes_tensor = axes.empty() ? ov::Tensor(element::i64, ov::Shape{axes.size()}) @@ -121,3 +123,115 @@ TEST_P(SqueezeStaticShapeInferenceTest, shape_inference_with_const_map) { ASSERT_EQ(output_shapes.front(), exp_shape); } + +} // namespace v0 + +namespace v15 { + +class SqueezeV15StaticShapeInferenceAssertTest : public OpStaticShapeInferenceTest { +protected: + void SetUp() override { + output_shapes = ShapeVector(1); + } +}; + +TEST_F(SqueezeV15StaticShapeInferenceAssertTest, no_axes) { + const auto arg = std::make_shared(element::f64, PartialShape{-1, -1}); + const auto axes = std::make_shared(element::i64, PartialShape{1}); + const auto op = make_op(arg, axes); + + input_shapes = ShapeVector{{5, 6}, axes->get_shape()}; + + OV_EXPECT_THROW(shape_inference(op.get(), input_shapes), + NodeValidationFailure, + HasSubstr("Check 'constant != nullptr'")); +} + +TEST_F(SqueezeV15StaticShapeInferenceAssertTest, parameter_static_shape_axes_no_data) { + const auto arg = std::make_shared(element::f64, Shape{2, 1, 3, 1}); + const auto axes = std::make_shared(element::i64, Shape{2}); + const auto op = make_op(arg, axes); + + input_shapes = ShapeVector{arg->get_shape(), axes->get_shape()}; + + OV_EXPECT_THROW(shape_inference(op.get(), input_shapes), + NodeValidationFailure, + HasSubstr("Check 'constant != nullptr'")); +} + +using TestParams = std::tuple, // Squeeze axes + StaticShape // Expected shape + >; + +class SqueezeV15StaticShapeInferenceTest : public SqueezeV15StaticShapeInferenceAssertTest, + public WithParamInterface { +protected: + void SetUp() override { + SqueezeV15StaticShapeInferenceAssertTest::SetUp(); + std::tie(input_shapes, axes, exp_shape) = GetParam(); + + output_shapes = ShapeVector(1); + arg = std::make_shared(element::f32, input_shapes.front().get_shape()); + } + + std::vector axes; + std::shared_ptr arg; +}; + +INSTANTIATE_TEST_SUITE_P(1d_shapes, + SqueezeV15StaticShapeInferenceTest, + Values(make_tuple(ShapeVector{{1}, {1}}, std::vector{-1}, StaticShape({})), + make_tuple(ShapeVector{{6}, {1}}, std::vector{-1}, StaticShape({6})), + make_tuple(ShapeVector{{1}, {1}}, std::vector{0}, StaticShape({}))), + PrintToStringParamName()); + +INSTANTIATE_TEST_SUITE_P( + multi_dim_shapes, + SqueezeV15StaticShapeInferenceTest, + Values(make_tuple(ShapeVector{{1, 2, 3, 1}, {2}}, std::vector{0, 3}, StaticShape({2, 3})), + make_tuple(ShapeVector{{2, 1, 1, 4}, {2}}, std::vector{2, 1}, StaticShape({2, 4})), + make_tuple(ShapeVector{{2, 1, 1, 4, 1}, {2}}, std::vector{0, 1, -2, -1}, StaticShape({2, 1, 4})), + make_tuple(ShapeVector{{1, 3, 1, 2, 1}, {3}}, std::vector{0, 2, 4}, StaticShape({3, 2})), + make_tuple(ShapeVector{{1, 3, 1, 2, 1}, {3}}, std::vector{4, 2, 0}, StaticShape({3, 2})), + make_tuple(ShapeVector{{1, 3, 1, 2, 1}, {3}}, std::vector{2, 0, 4}, StaticShape({3, 2})), + make_tuple(ShapeVector{{10, 1, 0, 1, 3, 1, 1}, {4}}, + std::vector{1, -1, 3, -2}, + StaticShape({10, 0, 3})), + make_tuple(ShapeVector{{10, 1, 0, 1, 3, 1, 1}, {}}, std::vector{}, StaticShape({10, 0, 3})), + make_tuple(ShapeVector{{2, 1, 7, 8, 3}, {1}}, std::vector{1}, StaticShape({2, 7, 8, 3}))), + PrintToStringParamName()); + +INSTANTIATE_TEST_SUITE_P( + multi_dim_shapes_repeated_axis, + SqueezeV15StaticShapeInferenceTest, + Values(make_tuple(ShapeVector{{2, 1, 3}, {2}}, std::vector{1, 1}, StaticShape({2, 3})), + make_tuple(ShapeVector{{3, 1, 2, 1}, {3}}, std::vector{1, -1, 1}, StaticShape({3, 2})), + make_tuple(ShapeVector{{3, 1, 2, 1}, {3}}, std::vector{1, -1, 1, -1}, StaticShape({3, 2})), + make_tuple(ShapeVector{{1, 3, 1, 2, 1}, {3}}, std::vector{2, -1, 2, -1, 0}, StaticShape({3, 2})), + make_tuple(ShapeVector{{2, 6, 7, 8, 1}, {2}}, std::vector{-1, -1}, StaticShape({2, 6, 7, 8}))), + PrintToStringParamName()); + +TEST_P(SqueezeV15StaticShapeInferenceTest, shape_inference_empty_const_map) { + const auto axes_node = std::make_shared(element::i64, Shape{axes.size()}, axes); + const auto op = make_op(arg, axes_node); + + output_shapes = shape_inference(op.get(), input_shapes); + + ASSERT_EQ(output_shapes.front(), exp_shape); +} + +TEST_P(SqueezeV15StaticShapeInferenceTest, shape_inference_with_const_map) { + const auto axes_node = std::make_shared(element::i64, ov::PartialShape::dynamic()); + const auto op = make_op(arg, axes_node); + + const auto axes_tensor = axes.empty() ? ov::Tensor(element::i64, ov::Shape{axes.size()}) + : ov::Tensor(element::i64, ov::Shape{axes.size()}, axes.data()); + const auto constant_data = std::unordered_map{{1, axes_tensor}}; + + output_shapes = shape_inference(op.get(), input_shapes, constant_data); + + ASSERT_EQ(output_shapes.front(), exp_shape); +} + +} // namespace v15 diff --git a/src/plugins/intel_cpu/tests/unit/shape_inference_test/stft_shape_inference_test.cpp b/src/plugins/intel_cpu/tests/unit/shape_inference_test/stft_shape_inference_test.cpp index 4c2ec30d16ee95..9d2960e31b71f2 100644 --- a/src/plugins/intel_cpu/tests/unit/shape_inference_test/stft_shape_inference_test.cpp +++ b/src/plugins/intel_cpu/tests/unit/shape_inference_test/stft_shape_inference_test.cpp @@ -17,6 +17,26 @@ using testing::HasSubstr; class STFTShapeInferenceTest : public OpStaticShapeInferenceTest {}; +TEST_F(STFTShapeInferenceTest, all_input_as_params_1D_signal) { + const auto data_type = element::f32; + const auto step_size_type = element::i32; + const auto in_signal = std::make_shared(data_type, ov::PartialShape{-1}); + const auto in_window = std::make_shared(data_type, ov::PartialShape{-1}); + const auto in_frame_size = std::make_shared(step_size_type, ov::Shape{}); + const auto in_frame_step = std::make_shared(step_size_type, ov::Shape{}); + const auto op = make_op(in_signal, in_window, in_frame_size, in_frame_step, true); + + std::vector static_input_shapes = {StaticShape{48}, StaticShape{16}, StaticShape{}, StaticShape{}}; + int32_t frame_size = 16; + int32_t frame_step = 16; + + auto const_data = std::unordered_map{{2, {element::i32, Shape{}, &frame_size}}, + {3, {element::i32, Shape{}, &frame_step}}}; + auto acc = make_tensor_accessor(const_data); + auto static_output_shapes = shape_infer(op.get(), static_input_shapes, acc); + ASSERT_EQ(static_output_shapes[0], StaticShape({9, 3, 2})); +} + TEST_F(STFTShapeInferenceTest, all_input_as_params) { const auto data_type = element::f32; const auto step_size_type = element::i32; diff --git a/src/plugins/intel_cpu/thirdparty/onednn b/src/plugins/intel_cpu/thirdparty/onednn index 1ce2d722922efb..c60a9946aa2386 160000 --- a/src/plugins/intel_cpu/thirdparty/onednn +++ b/src/plugins/intel_cpu/thirdparty/onednn @@ -1 +1 @@ -Subproject commit 1ce2d722922efb80da52a6efe2152a9aecdddebf +Subproject commit c60a9946aa2386890e5c9f5587974facb7624227 diff --git a/src/plugins/intel_gpu/include/intel_gpu/graph/kernel_impl_params.hpp b/src/plugins/intel_gpu/include/intel_gpu/graph/kernel_impl_params.hpp index 3e8887fbb2f7ee..72623f6d120955 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/graph/kernel_impl_params.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/graph/kernel_impl_params.hpp @@ -53,7 +53,7 @@ struct kernel_impl_params final { optional_layout weights_zero_points_layout = optional_layout(); optional_layout activations_zero_points_layout = optional_layout(); optional_layout compensation_layout = optional_layout(); - optional_layout state_layout = optional_layout(); + std::vector state_layouts; std::map memory_deps = {}; size_t primary_input_idx = 0; diff --git a/src/plugins/intel_gpu/include/intel_gpu/op/indirect_sdpa.hpp b/src/plugins/intel_gpu/include/intel_gpu/op/indirect_sdpa.hpp index b4d34a3975af6b..7c45c93c7e74f1 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/op/indirect_sdpa.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/op/indirect_sdpa.hpp @@ -29,6 +29,17 @@ class IndirectSDPA : public ov::intel_gpu::op::SDPA { const std::vector& order_out, const ov::element::Type output_type = ov::element::undefined); + IndirectSDPA(const OutputVector& data_inputs, + const ov::Output& beam_table, + const bool is_causal, + const int64_t indirect_axis, + const std::vector& order_q, + const std::vector& order_k, + const std::vector& order_v, + const std::vector& order_out, + const QuantizationAttribute& quantization_attribute, + const ov::element::Type output_type = ov::element::undefined); + bool visit_attributes(ov::AttributeVisitor &visitor) override; void validate_and_infer_types() override; diff --git a/src/plugins/intel_gpu/include/intel_gpu/op/kv_cache.hpp b/src/plugins/intel_gpu/include/intel_gpu/op/kv_cache.hpp index 402ff6e46c1607..7048d5229f25db 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/op/kv_cache.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/op/kv_cache.hpp @@ -7,6 +7,7 @@ #include "openvino/op/op.hpp" #include "openvino/op/util/variable.hpp" #include "openvino/op/util/variable_extension.hpp" +#include "ov_ops/dynamic_quantize.hpp" namespace ov { namespace intel_gpu { @@ -22,16 +23,16 @@ class KVCache : public ov::op::Op, public ov::op::util::VariableExtension { KVCache(const Output& past, const Output& new_token_data, - const Output& beam_idx, const std::shared_ptr& past_values, int64_t concat_axis, - int64_t gather_axis, const ov::element::Type output_type = ov::element::undefined); KVCache(const Output& past, const Output& new_token_data, + const Output& beam_idx, const std::shared_ptr& past_values, int64_t concat_axis, + int64_t gather_axis, const ov::element::Type output_type = ov::element::undefined); bool visit_attributes(ov::AttributeVisitor& visitor) override; @@ -53,14 +54,22 @@ class KVCache : public ov::op::Op, public ov::op::util::VariableExtension { bool get_indirect() const { return m_indirect; } -private: +protected: + KVCache(const OutputVector& inputs, + const std::shared_ptr& past_values, + bool indirect, + int64_t concat_axis, + int64_t gather_axis, + const ov::element::Type output_type = ov::element::undefined); + int64_t m_concat_axis = 0; int64_t m_gather_axis = 0; bool m_indirect = false; + ov::element::Type m_output_type; }; -std::vector shape_infer(const KVCache* op, std::vector input_shapes); +std::vector shape_infer(const KVCache* op, const std::vector& input_shapes); } // namespace op } // namespace intel_gpu diff --git a/src/plugins/intel_gpu/include/intel_gpu/op/kv_cache_compressed.hpp b/src/plugins/intel_gpu/include/intel_gpu/op/kv_cache_compressed.hpp new file mode 100644 index 00000000000000..4ee8cb388b61ea --- /dev/null +++ b/src/plugins/intel_gpu/include/intel_gpu/op/kv_cache_compressed.hpp @@ -0,0 +1,56 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "intel_gpu/op/kv_cache.hpp" +#include "ov_ops/dynamic_quantize.hpp" + +namespace ov { +namespace intel_gpu { +namespace op { + +/// \brief Operator that implements Key-Values cache subgraph for large language models. +/// This operation updates data of the corresponding Variable +class KVCacheCompressed : public ov::intel_gpu::op::KVCache { +public: + OPENVINO_OP("KVCacheCompressed", "gpu_opset"); + + using QuantizationAttrs = ov::op::internal::DynamicQuantize::Attributes; + + KVCacheCompressed() = default; + + KVCacheCompressed(const OutputVector& inputs, + const std::shared_ptr& past_values, + int64_t concat_axis, + int64_t gather_axis, + const QuantizationAttrs& quantization_attrs, + const ov::element::Type output_type = ov::element::undefined); + + void validate_and_infer_types() override; + + std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override; + + bool get_kv_compressed() const { return m_compressed; } + bool get_combine_scales_and_zp() const { + return m_quantization_attrs.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric && + m_quantization_attrs.output_storage_type != ov::op::internal::DynamicQuantize::OutputStorageType::Planar; + } + + QuantizationAttrs get_quantization_attrs() const { return m_quantization_attrs; } + void set_quantization_attrs(QuantizationAttrs attrs) { m_quantization_attrs = std::move(attrs); } + + std::vector get_scales_zp_output_order() const { return m_quantization_attrs.scales_zp_output_order; } + +private: + bool m_compressed; + QuantizationAttrs m_quantization_attrs = {}; +}; + +std::vector shape_infer(const KVCacheCompressed* op, + const std::vector& input_shapes); + +} // namespace op +} // namespace intel_gpu +} // namespace ov diff --git a/src/plugins/intel_gpu/include/intel_gpu/op/read_value.hpp b/src/plugins/intel_gpu/include/intel_gpu/op/read_value.hpp index a9c47f3fa39fa6..419c18118229ff 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/op/read_value.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/op/read_value.hpp @@ -26,6 +26,7 @@ class ReadValue : public ov::op::Op, public ov::op::util::VariableExtension { bool visit_attributes(ov::AttributeVisitor& visitor) override; void validate_and_infer_types() override; + void validate_and_infer_types(size_t output_idx, const ov::op::util::VariableInfo& variable_info); std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override; @@ -33,6 +34,12 @@ class ReadValue : public ov::op::Op, public ov::op::util::VariableExtension { OPENVINO_ASSERT(m_variable, "Variable is not initialized. Variable_id is unavailable"); return m_variable->get_info().variable_id; } + +protected: + ReadValue(const std::vector>& variable_initializers, const std::shared_ptr& variable) + : Op(variable_initializers) { + m_variable = variable; + } }; } // namespace op diff --git a/src/plugins/intel_gpu/include/intel_gpu/op/read_values.hpp b/src/plugins/intel_gpu/include/intel_gpu/op/read_values.hpp new file mode 100644 index 00000000000000..ecbda9364d7b3b --- /dev/null +++ b/src/plugins/intel_gpu/include/intel_gpu/op/read_values.hpp @@ -0,0 +1,42 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "intel_gpu/op/read_value.hpp" + +namespace ov { +namespace intel_gpu { +namespace op { + +/// \brief This operation handles the OpenVINO GPU Plugin's custom variable +// representation (which can store multiple states in a single variable) at the graph level. +class ReadValues : public ReadValue { +public: + OPENVINO_OP("ReadValues", "gpu_opset"); + + ReadValues() = default; + + ReadValues(const std::shared_ptr& variable, + const std::vector& internal_states_infos); + + ReadValues(const OutputVector& variable_initializers, + const std::shared_ptr& variable, + const std::vector& internal_states_infos); + + bool visit_attributes(ov::AttributeVisitor& visitor) override; + + void validate_and_infer_types() override; + + std::vector get_all_internal_states_info() const; + + std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override; + +private: + std::vector m_internal_states_infos; +}; + +} // namespace op +} // namespace intel_gpu +} // namespace ov diff --git a/src/plugins/intel_gpu/include/intel_gpu/op/sdpa.hpp b/src/plugins/intel_gpu/include/intel_gpu/op/sdpa.hpp index 9f5d4dad16efd7..f7bc0d780ffd38 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/op/sdpa.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/op/sdpa.hpp @@ -8,6 +8,7 @@ #include "openvino/core/partial_shape.hpp" #include "openvino/op/op.hpp" #include "openvino/op/scaled_dot_product_attention.hpp" +#include "ov_ops/dynamic_quantize.hpp" namespace ov { namespace intel_gpu { @@ -17,6 +18,8 @@ class SDPA : public ov::op::v13::ScaledDotProductAttention { public: OPENVINO_OP("SDPA", "gpu_opset"); + using QuantizationAttribute = ov::op::internal::DynamicQuantize::Attributes; + SDPA() = default; SDPA(const OutputVector& inputs, @@ -27,6 +30,15 @@ class SDPA : public ov::op::v13::ScaledDotProductAttention { const std::vector& order_out, const ov::element::Type output_type = ov::element::undefined); + SDPA(const OutputVector& inputs, + const bool is_causal, + const std::vector& order_q, + const std::vector& order_k, + const std::vector& order_v, + const std::vector& order_out, + const QuantizationAttribute& quantization_attrs, + const ov::element::Type output_type = ov::element::undefined); + bool visit_attributes(ov::AttributeVisitor &visitor) override; void validate_and_infer_types() override; @@ -41,6 +53,10 @@ class SDPA : public ov::op::v13::ScaledDotProductAttention { std::vector get_output_transpose_order() const { return m_order_out; } ov::element::Type get_output_type() const { return m_output_type; } + bool get_kv_compressed() const { return m_compressed; } + QuantizationAttribute get_quantization_attrs() const { return m_quantization_attrs; } + size_t get_compression_inputs_num() const; + static std::vector default_order(size_t rank) { std::vector order(rank); std::iota(order.begin(), order.end(), 0); @@ -54,6 +70,9 @@ class SDPA : public ov::op::v13::ScaledDotProductAttention { std::vector m_order_v; std::vector m_order_out; ov::element::Type m_output_type; + + bool m_compressed = false; + QuantizationAttribute m_quantization_attrs = {}; }; std::vector shape_infer(const SDPA* op, diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/compiled_model.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/compiled_model.hpp index 640cc82c2c8c2a..0d78b480b733b0 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/compiled_model.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/compiled_model.hpp @@ -61,6 +61,8 @@ class CompiledModel : public ov::ICompiledModel { const std::vector>& get_graphs() const; std::shared_ptr get_graph(size_t n) const; + void release_memory() override; + private: RemoteContextImpl::Ptr m_context; ExecutionConfig m_config; diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/multi_tensor_variable_state.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/multi_tensor_variable_state.hpp index 0cad36f62e47b9..26d15d733102ad 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/multi_tensor_variable_state.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/multi_tensor_variable_state.hpp @@ -46,5 +46,31 @@ class VariableStateIndirectKVCache : public MultiTensorState { size_t m_concat_axis = 0; }; +// This is multi-tensor state for Indirect KV-Cache + Gemm pattern +// Internally it stores KV Cache state + Beam Table state + compression scales state + (optional compression zero points) +class VariableStateIndirectKVCacheCompressed : public VariableStateIndirectKVCache { +public: + VariableStateIndirectKVCacheCompressed(const VariableStateInfo& info, + std::shared_ptr context, + std::shared_ptr shape_predictor, + const std::vector& output_layouts, + size_t beam_idx, + size_t concat_idx, + bool has_zp_state); + using Ptr = std::shared_ptr; + + void set_state(const ov::SoPtr& state) override; + ov::SoPtr get_state() const override; + + VariableState::Ptr get_compression_scale_state() const; + void set_compression_scale_layout(const cldnn::layout& new_layout); + + VariableState::Ptr get_compression_zp_state() const; + void set_compression_zp_layout(const cldnn::layout& new_layout); + bool has_zp_state() const; + +private: + bool m_has_zp_state = false; +}; } // namespace intel_gpu } // namespace ov diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp index 1dbd769444b1aa..27e5540a3786ab 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/primitives_list.hpp @@ -283,7 +283,9 @@ REGISTER_FACTORY(internal, FullyConnectedCompressed); REGISTER_FACTORY(internal, RMS); REGISTER_FACTORY(internal, GatherCompressed); REGISTER_FACTORY(internal, KVCache); +REGISTER_FACTORY(internal, KVCacheCompressed); REGISTER_FACTORY(internal, ReadValue); +REGISTER_FACTORY(internal, ReadValues); REGISTER_FACTORY(internal, Gemm); REGISTER_FACTORY(internal, SwiGLU); REGISTER_FACTORY(internal, IndirectGemm); diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/dynamic_quantize.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/dynamic_quantize.hpp index d93e2f86eed144..79af223e32cdaa 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/primitives/dynamic_quantize.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/dynamic_quantize.hpp @@ -5,6 +5,8 @@ #pragma once #include "primitive.hpp" +#include "ov_ops/dynamic_quantize.hpp" + namespace cldnn { /// @brief Dynamic Quantize primitive @@ -12,26 +14,39 @@ namespace cldnn { struct dynamic_quantize : public primitive_base { CLDNN_DECLARE_PRIMITIVE(dynamic_quantize); - dynamic_quantize() : primitive_base("", {}), group_size(0) {} + using Attributes = ov::op::internal::DynamicQuantize::Attributes; + + dynamic_quantize() : primitive_base("", {}) {} /// @brief Constructs dynamic_quantize primitive /// @param id This primitive id /// @param input Input primitive id - /// @param group_size Quantization group size + /// @param group_sizes Quantization group size /// @param data_type Output data type of quantized /// @param output_size Output data size of the primitive dynamic_quantize(const primitive_id& id, const input_info& input, - const uint64_t group_size, - const std::vector data_types = {optional_data_type(data_types::f16), optional_data_type(data_types::i8)}) - : primitive_base(id, {input}, 2, data_types), - group_size(group_size) {} + const Attributes& attrs) + : primitive_base(id, {input}) + , attrs(attrs) { + num_outputs = 2; + if (attrs.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric && + attrs.output_storage_type == ov::op::internal::DynamicQuantize::OutputStorageType::Planar) + num_outputs++; + } - uint64_t group_size = 0; + Attributes attrs; size_t hash() const override { size_t seed = primitive::hash(); - seed = hash_combine(seed, group_size); + seed = hash_range(seed, attrs.scales_zp_output_order.begin(), attrs.scales_zp_output_order.end()); + seed = hash_range(seed, attrs.group_sizes.begin(), attrs.group_sizes.end()); + seed = hash_combine(seed, attrs.quantization_type); + seed = hash_combine(seed, attrs.quantization_dt.hash()); + seed = hash_combine(seed, attrs.scale_dt.hash()); + seed = hash_combine(seed, attrs.zp_dt.hash()); + seed = hash_combine(seed, attrs.output_storage_type); + return seed; } @@ -41,17 +56,37 @@ struct dynamic_quantize : public primitive_base { auto rhs_casted = downcast(rhs); - return group_size == rhs_casted.group_size; + return attrs.scales_zp_output_order == rhs_casted.attrs.scales_zp_output_order && + attrs.output_storage_type == rhs_casted.attrs.output_storage_type && + attrs.group_sizes == rhs_casted.attrs.group_sizes && + attrs.quantization_dt == rhs_casted.attrs.quantization_dt && + attrs.scale_dt == rhs_casted.attrs.scale_dt && + attrs.zp_dt == rhs_casted.attrs.zp_dt && + attrs.quantization_type == rhs_casted.attrs.quantization_type;; } void save(BinaryOutputBuffer& ob) const override { primitive_base::save(ob); - ob << group_size; + + ob << make_data(&attrs.quantization_type, sizeof(attrs.quantization_type)); + ob << make_data(&attrs.quantization_dt, sizeof(attrs.quantization_dt)); + ob << make_data(&attrs.scale_dt, sizeof(attrs.scale_dt)); + ob << make_data(&attrs.zp_dt, sizeof(attrs.zp_dt)); + ob << make_data(&attrs.output_storage_type, sizeof(attrs.output_storage_type)); + ob << attrs.scales_zp_output_order; + ob << attrs.group_sizes; } void load(BinaryInputBuffer& ib) override { primitive_base::load(ib); - ib >> group_size; + + ib >> make_data(&attrs.quantization_type, sizeof(attrs.quantization_type)); + ib >> make_data(&attrs.quantization_dt, sizeof(attrs.quantization_dt)); + ib >> make_data(&attrs.scale_dt, sizeof(attrs.scale_dt)); + ib >> make_data(&attrs.zp_dt, sizeof(attrs.zp_dt)); + ib >> make_data(&attrs.output_storage_type, sizeof(attrs.output_storage_type)); + ib >> attrs.scales_zp_output_order; + ib >> attrs.group_sizes; } }; } // namespace cldnn diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/kv_cache.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/kv_cache.hpp index f87041dcff66d6..1c8f095752aca2 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/primitives/kv_cache.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/kv_cache.hpp @@ -3,10 +3,14 @@ // #pragma once + +#include "primitive.hpp" + #include "openvino/core/partial_shape.hpp" #include "openvino/core/type/element_type.hpp" #include "openvino/op/util/variable.hpp" -#include "primitive.hpp" +#include "ov_ops/dynamic_quantize.hpp" + #include namespace cldnn { @@ -14,6 +18,8 @@ namespace cldnn { struct kv_cache : public primitive_base { CLDNN_DECLARE_PRIMITIVE(kv_cache) + using QuantizationAttributes = ov::op::internal::DynamicQuantize::Attributes; + kv_cache() : primitive_base("", {}) {} kv_cache(const primitive_id& id, @@ -33,11 +39,23 @@ struct kv_cache : public primitive_base { int64_t gather_axis = 0; bool indirect = false; + bool compressed = false; + QuantizationAttributes quantization_attributes; + size_t hash() const override { size_t seed = primitive::hash(); seed = hash_combine(seed, concat_axis); seed = hash_combine(seed, gather_axis); seed = hash_combine(seed, indirect); + seed = hash_combine(seed, compressed); + seed = hash_range(seed, quantization_attributes.scales_zp_output_order.begin(), quantization_attributes.scales_zp_output_order.end()); + seed = hash_range(seed, quantization_attributes.group_sizes.begin(), quantization_attributes.group_sizes.end()); + seed = hash_combine(seed, quantization_attributes.quantization_type); + seed = hash_combine(seed, quantization_attributes.quantization_dt.hash()); + seed = hash_combine(seed, quantization_attributes.scale_dt.hash()); + seed = hash_combine(seed, quantization_attributes.zp_dt.hash()); + seed = hash_combine(seed, quantization_attributes.output_storage_type);; + return seed; } @@ -50,7 +68,15 @@ struct kv_cache : public primitive_base { return variable_info == rhs_casted.variable_info && concat_axis == rhs_casted.concat_axis && gather_axis == rhs_casted.gather_axis && - indirect == rhs_casted.indirect; + indirect == rhs_casted.indirect && + compressed == rhs_casted.compressed && + quantization_attributes.scales_zp_output_order == rhs_casted.quantization_attributes.scales_zp_output_order && + quantization_attributes.output_storage_type == rhs_casted.quantization_attributes.output_storage_type && + quantization_attributes.group_sizes == rhs_casted.quantization_attributes.group_sizes && + quantization_attributes.quantization_dt == rhs_casted.quantization_attributes.quantization_dt && + quantization_attributes.scale_dt == rhs_casted.quantization_attributes.scale_dt && + quantization_attributes.zp_dt == rhs_casted.quantization_attributes.zp_dt && + quantization_attributes.quantization_type == rhs_casted.quantization_attributes.quantization_type; } void save(BinaryOutputBuffer& ob) const override { @@ -62,6 +88,14 @@ struct kv_cache : public primitive_base { ob << concat_axis; ob << gather_axis; ob << indirect; + ob << compressed; + ob << make_data(&quantization_attributes.quantization_type, sizeof(quantization_attributes.quantization_type)); + ob << make_data(&quantization_attributes.quantization_dt, sizeof(quantization_attributes.quantization_dt)); + ob << make_data(&quantization_attributes.scale_dt, sizeof(quantization_attributes.scale_dt)); + ob << make_data(&quantization_attributes.zp_dt, sizeof(quantization_attributes.zp_dt)); + ob << make_data(&quantization_attributes.output_storage_type, sizeof(quantization_attributes.output_storage_type)); + ob << quantization_attributes.scales_zp_output_order; + ob << quantization_attributes.group_sizes; } void load(BinaryInputBuffer& ib) override { @@ -76,6 +110,32 @@ struct kv_cache : public primitive_base { ib >> concat_axis; ib >> gather_axis; ib >> indirect; + ib >> compressed; + ib >> make_data(&quantization_attributes.quantization_type, sizeof(quantization_attributes.quantization_type)); + ib >> make_data(&quantization_attributes.quantization_dt, sizeof(quantization_attributes.quantization_dt)); + ib >> make_data(&quantization_attributes.scale_dt, sizeof(quantization_attributes.scale_dt)); + ib >> make_data(&quantization_attributes.zp_dt, sizeof(quantization_attributes.zp_dt)); + ib >> make_data(&quantization_attributes.output_storage_type, sizeof(quantization_attributes.output_storage_type)); + ib >> quantization_attributes.scales_zp_output_order; + ib >> quantization_attributes.group_sizes; + } + + size_t get_compression_scales_inputs_num() const { + if (compressed) { + return 1; + } else { + return 0; + } + } + + size_t get_compression_zp_inputs_num() const { + if (compressed && + quantization_attributes.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric && + quantization_attributes.output_storage_type == ov::op::internal::DynamicQuantize::OutputStorageType::Planar) { + return 1; + } else { + return 0; + } } }; } // namespace cldnn diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/read_value.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/read_value.hpp index 7d9e919f56cf13..26465692ef6352 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/primitives/read_value.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/read_value.hpp @@ -22,19 +22,23 @@ struct read_value : public primitive_base { /// @param id This primitive id /// @param inputs Input parameters ids /// @param variable_id Variable id - /// @param output_layout Memory layout + /// @param output_layouts Memory layouts read_value(const primitive_id& id, const std::vector& inputs, const std::string& variable_id, - const layout& output_layout, + const std::vector& output_layouts, const ov::element::Type& user_specified_type = ov::element::undefined) - : primitive_base(id, inputs, 1, {optional_data_type{output_layout.data_type}}), + : primitive_base(id, inputs, output_layouts.size()), variable_id{variable_id}, - output_layout{output_layout}, - user_specified_type(user_specified_type) {} + output_layouts{output_layouts}, + user_specified_type(user_specified_type) { + for (size_t output_idx = 0; output_idx < output_layouts.size(); output_idx++) { + output_data_types[output_idx] = optional_data_type(output_layouts[output_idx].data_type); + } + } std::string variable_id; - layout output_layout; + std::vector output_layouts; ov::element::Type user_specified_type; bool operator==(const primitive& rhs) const override { @@ -51,7 +55,9 @@ struct read_value : public primitive_base { primitive_base::save(ob); ov::element::Type_t data_type = user_specified_type; ob << variable_id; - ob << output_layout; + ob << output_layouts.size(); + for (const auto& layout : output_layouts) + ob << layout; ob << make_data(&data_type, sizeof(ov::element::Type_t)); } @@ -59,7 +65,12 @@ struct read_value : public primitive_base { primitive_base::load(ib); ov::element::Type_t data_type = ov::element::Type_t::undefined; ib >> variable_id; - ib >> output_layout; + size_t output_layouts_size; + ib >> output_layouts_size; + output_layouts.resize(output_layouts_size); + for (size_t i = 0; i < output_layouts_size; i++) { + ib >> output_layouts[i]; + } ib >> make_data(&data_type, sizeof(ov::element::Type_t)); user_specified_type = data_type; } diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/scaled_dot_product_attention.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/scaled_dot_product_attention.hpp index d66012bfac8889..77e1c5ae71099e 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/primitives/scaled_dot_product_attention.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/scaled_dot_product_attention.hpp @@ -5,16 +5,20 @@ #pragma once #include "primitive.hpp" +#include "ov_ops/dynamic_quantize.hpp" + namespace cldnn { struct scaled_dot_product_attention : public primitive_base { CLDNN_DECLARE_PRIMITIVE(scaled_dot_product_attention) + using QuantizationAttributes = ov::op::internal::DynamicQuantize::Attributes; + scaled_dot_product_attention() : primitive_base("", {}) {} /// @brief Constructs scaled_dot_product_attention primitive. /// @param id This primitive id. - /// @param inputs Input data primitives id (query, keys, values, [attention_mask], [scale]). + /// @param inputs Input data primitives id (query, keys, values, [attention_mask], [scale], [keys scales], [keys zp], [values scales], [values zp]). /// @param is_causal If true, assumes causal attention masking. In this case attention_mask input is ignored. scaled_dot_product_attention(const primitive_id& id, const std::vector inputs, @@ -23,18 +27,29 @@ struct scaled_dot_product_attention : public primitive_base& input_q_transpose_order = {}, const std::vector& input_k_transpose_order = {}, const std::vector& input_v_transpose_order = {}, - const std::vector& output_transpose_order = {}) + const std::vector& output_transpose_order = {}, + const QuantizationAttributes& quantization_attributes = {}, + bool is_kv_compressed = false) : primitive_base(id, inputs) , is_causal(is_causal) , indirect_axis(indirect_axis) + , is_kv_compressed(is_kv_compressed) + , quantization_attributes(quantization_attributes) , input_q_transpose_order(input_q_transpose_order) , input_k_transpose_order(input_k_transpose_order) , input_v_transpose_order(input_v_transpose_order) , output_transpose_order(output_transpose_order) { auto data_inputs_num = inputs.size(); - if (indirect_axis != -1) + if (indirect_axis != -1) { data_inputs_num--; + } + if (is_kv_compressed) { + data_inputs_num -= 2; // scales + if (quantization_attributes.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric && + quantization_attributes.output_storage_type == ov::op::internal::DynamicQuantize::OutputStorageType::Planar) + data_inputs_num -= 2; // zp + } has_attn_mask_input = data_inputs_num > 3; has_scale_input = data_inputs_num > 4; } @@ -44,6 +59,9 @@ struct scaled_dot_product_attention : public primitive_base input_q_transpose_order; std::vector input_k_transpose_order; std::vector input_v_transpose_order; @@ -59,6 +77,15 @@ struct scaled_dot_product_attention : public primitive_base::save(ob); ob << is_causal; + ob << is_kv_compressed; ob << has_attn_mask_input; ob << has_scale_input; ob << indirect_axis; @@ -88,11 +124,19 @@ struct scaled_dot_product_attention : public primitive_base::load(ib); ib >> is_causal; + ib >> is_kv_compressed; ib >> has_attn_mask_input; ib >> has_scale_input; ib >> indirect_axis; @@ -100,6 +144,31 @@ struct scaled_dot_product_attention : public primitive_base> input_k_transpose_order; ib >> input_v_transpose_order; ib >> output_transpose_order; + ib >> make_data(&quantization_attributes.quantization_type, sizeof(quantization_attributes.quantization_type)); + ib >> make_data(&quantization_attributes.quantization_dt, sizeof(quantization_attributes.quantization_dt)); + ib >> make_data(&quantization_attributes.scale_dt, sizeof(quantization_attributes.scale_dt)); + ib >> make_data(&quantization_attributes.zp_dt, sizeof(quantization_attributes.zp_dt)); + ib >> make_data(&quantization_attributes.output_storage_type, sizeof(quantization_attributes.output_storage_type)); + ib >> quantization_attributes.scales_zp_output_order; + ib >> quantization_attributes.group_sizes; + } + + size_t get_compression_scales_inputs_num() const { + if (is_kv_compressed) { + return 2; + } else { + return 0; + } + } + + size_t get_compression_zp_inputs_num() const { + if (is_kv_compressed && + quantization_attributes.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric && + quantization_attributes.output_storage_type == ov::op::internal::DynamicQuantize::OutputStorageType::Planar) { + return 2; + } else { + return 0; + } } }; } // namespace cldnn diff --git a/src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp b/src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp index c65aa3e5894cb8..465ed898ecb7ec 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp @@ -143,6 +143,7 @@ class debug_configuration { int disable_primitive_fusing; // Disable primitive fusing int disable_fake_alignment; // Disable fake alignment std::vector dynamic_quantize_layers_without_onednn; // Specify Fully-connected layers which enable Dynamic quantization + int use_kv_cache_compression; // Enable KV-cache compression int dynamic_quantize_group_size; // Enable Dynamic quantization for fully connected primitive by specified group size int disable_horizontal_fc_fusion; // Disable fc horizontal fusion std::set dump_iteration; // Dump n-th execution of network. diff --git a/src/plugins/intel_gpu/include/intel_gpu/runtime/layout.hpp b/src/plugins/intel_gpu/include/intel_gpu/runtime/layout.hpp index 62e4c08a90f004..ab5cb53454b768 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/runtime/layout.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/runtime/layout.hpp @@ -454,6 +454,21 @@ inline ::std::ostream& operator<<(::std::ostream& os, const layout& p) { return os << p.to_string(); } +inline ::std::ostream& operator<<(::std::ostream& os, const std::vector& layouts) { + std::stringstream ss; + + ss << "["; + for (size_t i = 0; i < layouts.size(); i++) { + ss << layouts[i].to_short_string(); + + if (i + 1 != layouts.size()) + ss << ", "; + } + ss << "]"; + + return os << ss.str(); +} + using optional_data_type = optional_value; using optional_layout = optional_value; diff --git a/src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp b/src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp index 5c945f4c2d389c..8e4957d5f52797 100644 --- a/src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp +++ b/src/plugins/intel_gpu/src/graph/dynamic_quantize.cpp @@ -22,29 +22,39 @@ layout dynamic_quantize_inst::calc_output_layout(dynamic_quantize_node const& no } template -std::vector dynamic_quantize_inst::__calc_output_layouts(const layout &act_layout, uint64_t group_size) { +std::vector dynamic_quantize_inst::__calc_output_layouts(const layout &act_layout, + const dynamic_quantize::Attributes& attrs) { ov::op::internal::DynamicQuantize op; + op.set_attrs(attrs); + auto output_format = act_layout.format; std::vector input_shapes = { act_layout.get(), }; - std::vector shape_group_size(act_layout.get().size(), 1); - shape_group_size.back() = group_size; + auto output_shapes = ov::op::internal::DynamicQuantize::shape_infer(&op, input_shapes); + + std::vector output_layouts = { layout(output_shapes[0], attrs.quantization_dt, output_format), + layout(output_shapes[1], attrs.scale_dt, output_format) }; - auto output_shapes = ov::op::internal::DynamicQuantize::shape_infer(&op, input_shapes, shape_group_size); + if (attrs.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric && + attrs.output_storage_type == ov::op::internal::DynamicQuantize::OutputStorageType::Planar) { + output_layouts.emplace_back(layout(output_shapes[2], attrs.zp_dt, output_format)); + } - return { layout(output_shapes[0], data_types::i8, output_format), layout(output_shapes[1], data_types::f16, output_format) }; + return output_layouts; } -template std::vector dynamic_quantize_inst::__calc_output_layouts(const layout &act_layout, uint64_t group_size); +template std::vector dynamic_quantize_inst::__calc_output_layouts(const layout &act_layout, + const dynamic_quantize::Attributes& config); template std::vector dynamic_quantize_inst::calc_output_layouts(dynamic_quantize_node const& /*node*/, const kernel_impl_params& impl_param) { auto desc = impl_param.typed_desc(); const auto& input_layout = impl_param.get_input_layout(); - return __calc_output_layouts(input_layout, UINT64_MAX /* TODO: handle group_size here */); + + return __calc_output_layouts(input_layout, desc->attrs); } template std::vector dynamic_quantize_inst::calc_output_layouts(dynamic_quantize_node const& node, @@ -56,6 +66,15 @@ std::string dynamic_quantize_inst::to_string(dynamic_quantize_node const& node) std::stringstream primitive_description; + json_composite dynamic_quantize_info; + dynamic_quantize_info.add("output_storage_type", static_cast(desc->attrs.output_storage_type)); + dynamic_quantize_info.add("scales_zp_output_order", desc->attrs.scales_zp_output_order); + dynamic_quantize_info.add("group_sizes", desc->attrs.group_sizes); + dynamic_quantize_info.add("quantization_dt", desc->attrs.quantization_dt); + dynamic_quantize_info.add("scale_dt", desc->attrs.scale_dt); + dynamic_quantize_info.add("zp_dt", desc->attrs.zp_dt); + dynamic_quantize_info.add("quantization_type", static_cast(desc->attrs.quantization_type)); + node_info->add("dynamic_quantize info", dynamic_quantize_info); node_info->dump(primitive_description); return primitive_description.str(); diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_buffer_fusing.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_buffer_fusing.cpp index 7bdbc53ad54d16..e94714c84fdebf 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_buffer_fusing.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_buffer_fusing.cpp @@ -423,10 +423,11 @@ bool crop_in_place_optimization::can_crop_be_optimized_simple_data_format(const } static bool can_read_value_be_optimize(const read_value_node& node) { - if (node.get_users().size() == 1) + std::unordered_set unique_users(node.get_users().begin(), node.get_users().end()); + if (unique_users.size() == 1) return true; - const auto non_shape_of_users_count = std::count_if(node.get_users().begin(), node.get_users().end(), [](const program_node* user) { + const auto non_shape_of_users_count = std::count_if(unique_users.begin(), unique_users.end(), [](const program_node* user) { return !user->is_type(); }); if (non_shape_of_users_count <= 1) @@ -659,23 +660,34 @@ void crop_in_place_optimization::update_in_place_crop_padding_simple_data_format if (user_info.first && user_info.first->is_type()) { auto reshape_desc = user_info.first->as().get_primitive(); auto reshape_mode = reshape_desc->mode; + auto reshape_axis = crop_axis; if (reshape_mode == reshape::reshape_mode::base) { - user_info.second.data_padding._dynamic_dims_mask = dyn_pad_sizes; + auto reshape_ps = user_info.second.get_partial_shape(); + auto crop_dim_val = crop_layout.get_partial_shape()[crop_axis].get_length(); + + auto mul = 1; + reshape_axis = reshape_ps.size() - 1; + for (size_t i = reshape_ps.size(); i > 1; i--) { + if (reshape_ps[i - 1].is_dynamic() || mul == crop_dim_val) + break; + + mul *= reshape_ps[i - 1].get_length(); + reshape_axis = i - 1; + } } else if (reshape_mode == reshape::reshape_mode::unsqueeze || reshape_mode == reshape::reshape_mode::squeeze) { auto reshape_ps = user_info.second.get_partial_shape(); auto output_pattern = reshape_desc->output_pattern; - auto reshape_axis = crop_axis; for (size_t i = 0; i < output_pattern.size(); i++) { if (output_pattern[i] <= static_cast(reshape_axis)) { reshape_axis += reshape_mode == reshape::reshape_mode::unsqueeze ? 1 : -1; } } - - padding::DynamicDimsMask dyn_pad_mask; - dyn_pad_mask[reshape_axis] = 1; - user_info.second.data_padding._dynamic_dims_mask = dyn_pad_mask; } + + auto reshape_dyn_pad_mask = padding::DynamicDimsMask(); + reshape_dyn_pad_mask[reshape_axis] = 1; + user_info.second.data_padding._dynamic_dims_mask = reshape_dyn_pad_mask; } return; } @@ -703,13 +715,36 @@ void crop_in_place_optimization::update_in_place_crop_padding_simple_data_format auto reshape_desc = user_info.first->as().get_primitive(); auto reshape_mode = reshape_desc->mode; if (reshape_mode == reshape::reshape_mode::base) { - auto reshape_rank = user_info.second.get_partial_shape().size(); - auto reshape_last_dim = user_info.second.get_partial_shape().to_shape()[reshape_rank - 1]; - if (lower_sizes[crop_axis]) - lower_sizes[crop_axis] /= reshape_last_dim; - if (upper_sizes[crop_axis]) - upper_sizes[crop_axis] /= reshape_last_dim; - user_info.second.data_padding = padding(lower_sizes, upper_sizes, dyn_pad_sizes); + auto reshape_ps = user_info.second.get_partial_shape(); + auto crop_dim_val = crop_layout.get_partial_shape()[crop_axis].get_length(); + + auto divider = 1; + auto reshape_axis = reshape_ps.size(); + for (size_t i = reshape_ps.size(); i > 1; i--) { + const auto& dim_value = reshape_ps[i - 1].get_length(); + if (divider * dim_value == crop_dim_val) + break; + + divider *= dim_value; + reshape_axis = i - 1; + } + reshape_axis -= 1; + + const auto output_rank = std::max(reshape_ps.size(), static_cast(4)); + std::vector reshape_lower_sizes(output_rank, 0); + std::vector reshape_upper_sizes(output_rank, 0); + padding::DynamicDimsMask reshape_dyn_pad_mask; + + reshape_lower_sizes[reshape_axis] = lower_sizes[crop_axis]; + reshape_upper_sizes[reshape_axis] = upper_sizes[crop_axis]; + reshape_dyn_pad_mask[reshape_axis] = 1; + + if (reshape_lower_sizes[reshape_axis]) + reshape_lower_sizes[reshape_axis] /= divider; + if (reshape_upper_sizes[reshape_axis]) + reshape_upper_sizes[reshape_axis] /= divider; + + user_info.second.data_padding = padding(reshape_lower_sizes, reshape_upper_sizes, reshape_dyn_pad_mask); } else { auto reshape_ps = user_info.second.get_partial_shape(); auto output_pattern = reshape_desc->output_pattern; @@ -877,18 +912,39 @@ void prepare_buffer_fusing::run(program& p) { node.set_output_layout(kv_out_layout); node.can_share_buffer(false); - auto update_dep = [&info_dynamic_pad](program_node* dep) { - auto prev_layout = dep->get_output_layout(); + auto update_dep = [](program_node* dep, padding::DynamicDimsMask& info_dynamic_pad, size_t idx) { + auto prev_layout = dep->get_output_layout(true, idx); prev_layout.data_padding._dynamic_dims_mask = info_dynamic_pad; - dep->set_output_layout(prev_layout); + dep->set_output_layout(prev_layout, true, idx); dep->can_share_buffer(false); }; + auto update_scale_zp = [&](size_t kv_cache_output_idx, size_t read_value_output_idx) { + auto scales_out_layout = node.get_output_layout(false, kv_cache_output_idx); + + const size_t scales_zp_concat_axis = 2; + padding::DynamicDimsMask info_dynamic_pad_scales; + info_dynamic_pad_scales[scales_zp_concat_axis] = 1; + scales_out_layout.data_padding._dynamic_dims_mask = info_dynamic_pad_scales; + node.set_output_layout(scales_out_layout, true, kv_cache_output_idx); + + update_dep(rv_prim, info_dynamic_pad_scales, read_value_output_idx); + }; + if (rv_prim) { - update_dep(rv_prim); + update_dep(rv_prim, info_dynamic_pad, 0); } if (gather_prim) { - update_dep(gather_prim); + update_dep(gather_prim, info_dynamic_pad, 0); + } + + const auto& desc = node.get_primitive(); + if (desc->compressed) { + update_scale_zp(2, 1); + + if (desc->get_compression_zp_inputs_num() > 0) { + update_scale_zp(3, 2); + } } } }); @@ -922,7 +978,7 @@ void prepare_buffer_fusing::run(program& p) { // TODO: Allow optimizations for the case above too. Looks like it can be achieved by more careful // topological sort (i.e. if we ensure that all read_value users are completed before assign is run) node.can_be_optimized(can_read_value_be_optimize(node)); - GPU_DEBUG_TRACE_DETAIL << "[prepare_buffer_fusing] : " << node.id() << " can be optimized" << std::endl; + GPU_DEBUG_TRACE_DETAIL << "[prepare_buffer_fusing] : " << node.id() << " can be optimized = " << node.can_be_optimized() << std::endl; }); } } diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp index 39cbc1aa89b4e2..c323109850c489 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp @@ -736,8 +736,6 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { should_fuse |= input.is_type(); - should_fuse |= input.is_type(); - bool legacy_fusion = activation_node.get_dependencies().size() == 1 && !input.can_be_optimized() && !activation_node.is_constant() && @@ -922,8 +920,7 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { (parents[i].first->is_type()) || (parents[i].first->is_type() && reduce_supports_fusings(parents[i].first->as())) || - (parents[i].first->is_type()) || - (parents[i].first->is_type()); + (parents[i].first->is_type()); } // Disable fusion to a node on constant path when second input is in data flow @@ -1063,6 +1060,9 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { fused_node->get_input_pshape().rbegin()->is_dynamic(); if (is_fc_lora || is_conv_lora || is_gemm_lora) { + if (!can_fuse_parents[peer_idx]) { + return; + } std::swap(peer_node, fused_node); } } diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/activation.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/activation.cpp index e750303b955d77..1ec8c9458cf30b 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/activation.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/activation.cpp @@ -140,7 +140,7 @@ struct activation_impl : public typed_primitive_impl { OV_ITT_SCOPED_TASK(ov::intel_gpu::itt::domains::intel_gpu_plugin, "activation::execute_impl"); auto& stream = instance.get_network().get_stream(); - const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.get_node().is_in_shape_of_subgraph(); + const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.all_dependencies_cpu_impl(); if (!pass_through_events) { for (auto e : events) { diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/broadcast.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/broadcast.cpp index 2701a57001eb93..7abd3a141c15f1 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/broadcast.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/broadcast.cpp @@ -60,7 +60,7 @@ struct broadcast_impl : public typed_primitive_impl { OV_ITT_SCOPED_TASK(ov::intel_gpu::itt::domains::intel_gpu_plugin, "broadcast::execute_impl"); auto& stream = instance.get_network().get_stream(); - const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.get_node().is_in_shape_of_subgraph(); + const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.all_dependencies_cpu_impl(); if (!pass_through_events) { for (auto e : events) { diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/concat.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/concat.cpp index 6b7a483bae7d8c..a7d3beb5b94cf0 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/concat.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/concat.cpp @@ -51,7 +51,7 @@ struct concatenation_impl : public typed_primitive_impl { OV_ITT_SCOPED_TASK(ov::intel_gpu::itt::domains::intel_gpu_plugin, "concat::execute_impl"); auto& stream = instance.get_network().get_stream(); - const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.get_node().is_in_shape_of_subgraph(); + const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.all_dependencies_cpu_impl(); if (!pass_through_events) { for (auto e : events) { diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/crop.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/crop.cpp index 99bfa9b9383492..c8041f44a78e86 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/crop.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/crop.cpp @@ -37,7 +37,7 @@ struct crop_impl : public typed_primitive_impl { OV_ITT_SCOPED_TASK(ov::intel_gpu::itt::domains::intel_gpu_plugin, "crop::execute_impl"); auto& stream = instance.get_network().get_stream(); - const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.get_node().is_in_shape_of_subgraph(); + const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.all_dependencies_cpu_impl(); if (!pass_through_events) { for (auto e : events) { diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/detection_output.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/detection_output.cpp index 7d3e4ff7c4eadf..25676ea3e9aa3d 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/detection_output.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/detection_output.cpp @@ -827,7 +827,7 @@ struct detection_output_impl : typed_primitive_impl { event::ptr execute_impl(const std::vector& events, detection_output_inst& instance) override { auto& stream = instance.get_network().get_stream(); - const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.get_node().is_in_shape_of_subgraph(); + const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.all_dependencies_cpu_impl(); if (!pass_through_events) { for (auto e : events) { diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/eltwise.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/eltwise.cpp index 1b18b78b041fd8..057686f9c2d6a6 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/eltwise.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/eltwise.cpp @@ -81,7 +81,7 @@ struct eltwise_impl : public typed_primitive_impl { OV_ITT_SCOPED_TASK(ov::intel_gpu::itt::domains::intel_gpu_plugin, "eltwise::execute_impl"); auto& stream = instance.get_network().get_stream(); - const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.get_node().is_in_shape_of_subgraph(); + const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.all_dependencies_cpu_impl(); if (!pass_through_events) { for (auto e : events) { diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/gather.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/gather.cpp index 242273a23dd000..733e143b2c2c76 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/gather.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/gather.cpp @@ -59,7 +59,7 @@ struct gather_impl : public typed_primitive_impl { return stream.group_events(events); } - const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.get_node().is_in_shape_of_subgraph(); + const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.all_dependencies_cpu_impl(); if (!pass_through_events) { for (auto e : events) { diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/non_max_suppression.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/non_max_suppression.cpp index 4783159d501404..446a277866f222 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/non_max_suppression.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/non_max_suppression.cpp @@ -403,7 +403,7 @@ struct non_max_suppression_impl : typed_primitive_impl { event::ptr execute_impl(const std::vector& events, typed_primitive_inst& instance) override { auto& stream = instance.get_network().get_stream(); - const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.get_node().is_in_shape_of_subgraph(); + const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.all_dependencies_cpu_impl(); if (!pass_through_events) { for (auto e : events) { @@ -455,7 +455,7 @@ struct non_max_suppression_gather_impl : typed_primitive_impl& events, typed_primitive_inst& instance) override { auto& stream = instance.get_network().get_stream(); - const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.get_node().is_in_shape_of_subgraph(); + const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.all_dependencies_cpu_impl(); if (!pass_through_events) { for (auto e : events) { diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/proposal.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/proposal.cpp index e49cb3a832f8ae..b295ee5f84c56f 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/proposal.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/proposal.cpp @@ -388,7 +388,7 @@ struct proposal_impl : typed_primitive_impl { event::ptr execute_impl(const std::vector& events, proposal_inst& instance) override { auto& stream = instance.get_network().get_stream(); - const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.get_node().is_in_shape_of_subgraph(); + const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.all_dependencies_cpu_impl(); if (!pass_through_events) { for (auto e : events) { diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/range.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/range.cpp index 83142812f29e8b..439bbf5aa4ef6b 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/range.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/range.cpp @@ -37,7 +37,7 @@ struct range_impl : public typed_primitive_impl { OV_ITT_SCOPED_TASK(ov::intel_gpu::itt::domains::intel_gpu_plugin, "range::execute_impl"); auto& stream = instance.get_network().get_stream(); - const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.get_node().is_in_shape_of_subgraph(); + const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.all_dependencies_cpu_impl(); if (!pass_through_events) { for (auto e : events) { diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/read_value.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/read_value.cpp index 5692b6037a09e0..da7e95c4ab74a5 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/read_value.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/read_value.cpp @@ -6,6 +6,8 @@ #include "impls/registry/implementation_map.hpp" #include "register.hpp" +#include "intel_gpu/plugin/multi_tensor_variable_state.hpp" + namespace cldnn { namespace cpu { @@ -64,7 +66,21 @@ struct read_value_impl : public typed_primitive_impl { } if (!instance.can_be_optimized()) { - return instance.output_memory(0).copy_from(stream, *variable.get_memory(), false); + GPU_DEBUG_TRACE_DETAIL << "Copy variable's memory to new read_value's output buffer\n"; + std::vector res_events; + res_events.push_back(instance.output_memory(0).copy_from(stream, *variable.get_memory(), false)); + + if (auto compressed_cache_variable = dynamic_cast(&variable)) { + auto scales_state = compressed_cache_variable->get_compression_scale_state(); + res_events.push_back(instance.output_memory(1).copy_from(stream, *scales_state->get_memory(), false)); + + if (compressed_cache_variable->has_zp_state()) { + auto zp_state = compressed_cache_variable->get_compression_zp_state(); + res_events.push_back(instance.output_memory(1).copy_from(stream, *zp_state->get_memory(), false)); + } + } + + return stream.aggregate_events(res_events, res_events.size() > 1); } return instance.get_network().get_stream().create_user_event(true); diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/reduce.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/reduce.cpp index 5a3867f9d1582a..dd6284a3dd795d 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/reduce.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/reduce.cpp @@ -77,7 +77,7 @@ struct reduce_impl : public typed_primitive_impl { OV_ITT_SCOPED_TASK(ov::intel_gpu::itt::domains::intel_gpu_plugin, "reduce::execute_impl"); auto& stream = instance.get_network().get_stream(); - const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.get_node().is_in_shape_of_subgraph(); + const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.all_dependencies_cpu_impl(); if (!pass_through_events) { for (auto e : events) { diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/reorder.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/reorder.cpp index 1b6f145c4ceb2d..6e4202f815dc96 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/reorder.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/reorder.cpp @@ -37,7 +37,7 @@ struct reorder_impl : public typed_primitive_impl { OV_ITT_SCOPED_TASK(ov::intel_gpu::itt::domains::intel_gpu_plugin, "reorder::execute_impl"); auto& stream = instance.get_network().get_stream(); - const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.get_node().is_in_shape_of_subgraph(); + const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.all_dependencies_cpu_impl(); if (!pass_through_events) { for (auto e : events) { diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/scatter_update.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/scatter_update.cpp index 1a329ea495ef82..94a12dadddb407 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/scatter_update.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/scatter_update.cpp @@ -51,7 +51,7 @@ struct scatter_update_impl : public typed_primitive_impl { OV_ITT_SCOPED_TASK(ov::intel_gpu::itt::domains::intel_gpu_plugin, "scatter_update::execute_impl"); auto& stream = instance.get_network().get_stream(); - const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.get_node().is_in_shape_of_subgraph(); + const bool pass_through_events = (stream.get_queue_type() == QueueTypes::out_of_order) && instance.all_dependencies_cpu_impl(); if (!pass_through_events) { for (auto e : events) { diff --git a/src/plugins/intel_gpu/src/graph/impls/cpu/select.cpp b/src/plugins/intel_gpu/src/graph/impls/cpu/select.cpp index 9c9ab75f64ad59..78df55d5c7c1f7 100644 --- a/src/plugins/intel_gpu/src/graph/impls/cpu/select.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/cpu/select.cpp @@ -50,7 +50,7 @@ struct select_impl : public typed_primitive_impl { void load(BinaryInputBuffer& ib) override { parent::load(ib); - if (is_dynamic()) { + if (is_dynamic() && _kernel_data.kernelName.length() != 0) { auto& kernel_selector = kernel_selector_t::Instance(); auto kernel_impl = kernel_selector.GetImplementation(_kernel_data.kernelName); kernel_impl->GetUpdateDispatchDataFunc(_kernel_data); diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/slice.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/slice.cpp index 8a7225cb3c64ce..63c3d5c63d3800 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/slice.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/slice.cpp @@ -64,7 +64,7 @@ struct slice_impl : typed_primitive_impl_ocl { void load(BinaryInputBuffer& ib) override { parent::load(ib); - if (is_dynamic()) { + if (is_dynamic() && _kernel_data.kernelName.length() != 0) { auto& kernel_selector = kernel_selector_t::Instance(); auto kernel_impl = kernel_selector.GetImplementation(_kernel_data.kernelName); kernel_impl->GetUpdateDispatchDataFunc(_kernel_data); diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/softmax.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/softmax.cpp index 7295fe57273738..fb77fc3861583c 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/softmax.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/softmax.cpp @@ -48,7 +48,7 @@ struct softmax_impl : typed_primitive_impl_ocl { void load(BinaryInputBuffer& ib) override { parent::load(ib); - if (is_dynamic()) { + if (is_dynamic() && _kernel_data.kernelName.length() != 0) { auto& kernel_selector = kernel_selector_t::Instance(); auto kernel_impl = kernel_selector.GetImplementation(_kernel_data.kernelName); kernel_impl->GetUpdateDispatchDataFunc(_kernel_data); diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/strided_slice.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/strided_slice.cpp index 48631b1e8cace9..9b14a9cff496a8 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/strided_slice.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/strided_slice.cpp @@ -80,7 +80,7 @@ struct strided_slice_impl : typed_primitive_impl_ocl { void load(BinaryInputBuffer& ib) override { parent::load(ib); - if (is_dynamic()) { + if (is_dynamic() && _kernel_data.kernelName.length() != 0) { auto& kernel_selector = kernel_selector_t::Instance(); auto kernel_impl = kernel_selector.GetImplementation(_kernel_data.kernelName); kernel_impl->GetUpdateDispatchDataFunc(_kernel_data); diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/swiglu.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/swiglu.cpp index 8f96f7afa31556..6013bcef11c0f7 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/swiglu.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/swiglu.cpp @@ -25,7 +25,7 @@ struct swiglu_impl : typed_primitive_impl_ocl { void load(BinaryInputBuffer& ib) override { parent::load(ib); - if (is_dynamic()) { + if (is_dynamic() && _kernel_data.kernelName.length() != 0) { auto& kernel_selector = kernel_selector_t::Instance(); auto kernel_impl = kernel_selector.GetImplementation(_kernel_data.kernelName); kernel_impl->GetUpdateDispatchDataFunc(_kernel_data); diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/tile.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/tile.cpp index 6d2268de0730e7..0469c3de37c449 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/tile.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/tile.cpp @@ -25,7 +25,7 @@ struct tile_impl : typed_primitive_impl_ocl { void load(BinaryInputBuffer& ib) override { parent::load(ib); - if (is_dynamic()) { + if (is_dynamic() && _kernel_data.kernelName.length() != 0) { auto& kernel_selector = kernel_selector_t::Instance(); auto kernel_impl = kernel_selector.GetImplementation(_kernel_data.kernelName); kernel_impl->GetUpdateDispatchDataFunc(_kernel_data); diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/unique.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/unique.cpp index d90580de3c35fd..d833b28969bd15 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/unique.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/unique.cpp @@ -24,7 +24,7 @@ struct unique_count_impl : typed_primitive_impl_ocl { void load(BinaryInputBuffer& ib) override { parent::load(ib); - if (is_dynamic()) { + if (is_dynamic() && _kernel_data.kernelName.length() != 0) { auto& kernel_selector = kernel_selector_t::Instance(); auto kernel_impl = kernel_selector.GetImplementation(_kernel_data.kernelName); kernel_impl->GetUpdateDispatchDataFunc(_kernel_data); @@ -106,7 +106,7 @@ struct unique_gather_impl : typed_primitive_impl_ocl { void load(BinaryInputBuffer& ib) override { parent::load(ib); - if (is_dynamic()) { + if (is_dynamic() && _kernel_data.kernelName.length() != 0) { auto& kernel_selector = kernel_selector_t::Instance(); auto kernel_impl = kernel_selector.GetImplementation(_kernel_data.kernelName); kernel_impl->GetUpdateDispatchDataFunc(_kernel_data); diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/concatenation_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/concatenation_onednn.hpp index e85bda18a034da..9e0a3fa5cfb390 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/concatenation_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/concatenation_onednn.hpp @@ -19,7 +19,7 @@ struct ConcatenationImplementationManager : public ImplementationManager { bool validate_impl(const program_node& node) const override { assert(node.is_type()); const auto& info = node.get_program().get_engine().get_device_info(); - if (!info.supports_immad) + if (!info.supports_immad || info.arch == gpu_arch::unknown) return false; static const std::vector supported_types = { ov::element::f16, ov::element::u8, ov::element::i8 }; diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/convolution_onednn.cpp b/src/plugins/intel_gpu/src/graph/impls/onednn/convolution_onednn.cpp index 83d2a10dc4f2f9..a11ceef8b0f2dd 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/convolution_onednn.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/convolution_onednn.cpp @@ -204,14 +204,16 @@ struct convolution_onednn : typed_primitive_onednn_impl { auto& a_zp = arg.activations_zero_points(); auto a_zp_dtype = a_zp.get_output_layout().data_type; - if (!data_type_traits::is_i8_u8(a_zp_dtype)) { + if (!data_type_traits::is_i8_u8(a_zp_dtype) && a_zp_dtype != data_types::i32) { throw std::runtime_error("Unsupported data type for activations zero points for oneDNN convolution"); } if (a_zp_dtype == data_types::i8) { set_activation_zero_points_attr::value_type>(attrs, a_zp.as(), zero_point_mask); - } else { // if (a_zp_dtype == data_types::u8) + } else if (a_zp_dtype == data_types::u8) { set_activation_zero_points_attr::value_type>(attrs, a_zp.as(), zero_point_mask); + } else if (a_zp_dtype == data_types::i32) { + set_activation_zero_points_attr::value_type>(attrs, a_zp.as(), zero_point_mask); } } diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/convolution_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/convolution_onednn.hpp index a5616167506f70..c3f599fc5db9f6 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/convolution_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/convolution_onednn.hpp @@ -24,7 +24,7 @@ struct ConvolutionImplementationManager : public ImplementationManager { bool validate_impl(const program_node& node) const override { assert(node.is_type()); const auto& info = node.get_program().get_engine().get_device_info(); - if (!info.supports_immad) + if (!info.supports_immad || info.arch == gpu_arch::unknown) return false; const auto& conv_node = node.as(); diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/deconvolution_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/deconvolution_onednn.hpp index 949c979ed77e80..039cf36261caa0 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/deconvolution_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/deconvolution_onednn.hpp @@ -20,7 +20,7 @@ struct DeconvolutionImplementationManager : public ImplementationManager { bool validate_impl(const program_node& node) const override { assert(node.is_type()); const auto& info = node.get_program().get_engine().get_device_info(); - if (!info.supports_immad) + if (!info.supports_immad || info.arch == gpu_arch::unknown) return false; const auto& deconv_node = node.as(); diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp index 39423980521042..a601b2c74c09e3 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/fully_connected_onednn.hpp @@ -22,7 +22,7 @@ struct FullyConnectedImplementationManager : public ImplementationManager { bool validate_impl(const program_node& node) const override { assert(node.is_type()); const auto& info = node.get_program().get_engine().get_device_info(); - if (!info.supports_immad) + if (!info.supports_immad || info.arch == gpu_arch::unknown) return false; const auto& fc_node = node.as(); @@ -50,7 +50,7 @@ struct FullyConnectedImplementationManager : public ImplementationManager { bool compressed_case = fc_prim->compressed_weights && one_of(in0_dt, {data_types::f16, data_types::f32, data_types::i8}) && one_of(wei_dt, {data_types::u8, data_types::i8, data_types::u4, data_types::i4}) && - one_of(out_dt, {data_types::f16, data_types::f32, data_types::u8, data_types::i8}); + one_of(out_dt, {data_types::f16, data_types::f32}); if (!f16f16_case && !f32f32_case && !u8s8_case && !compressed_case) return false; diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/gemm_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/gemm_onednn.hpp index f89d3e588735e2..6c576d177043ee 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/gemm_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/gemm_onednn.hpp @@ -19,7 +19,7 @@ struct GemmImplementationManager : public ImplementationManager { bool validate_impl(const program_node& node) const override { assert(node.is_type()); const auto& info = node.get_program().get_engine().get_device_info(); - if (!info.supports_immad) + if (!info.supports_immad || info.arch == gpu_arch::unknown) return false; const auto& gemm_node = node.as(); diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/pooling_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/pooling_onednn.hpp index 343fe66771de25..4710b0c77b83c7 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/pooling_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/pooling_onednn.hpp @@ -20,7 +20,7 @@ struct PoolingImplementationManager : public ImplementationManager { bool validate_impl(const program_node& node) const override { assert(node.is_type()); const auto& info = node.get_program().get_engine().get_device_info(); - if (!info.supports_immad) + if (!info.supports_immad || info.arch == gpu_arch::unknown) return false; const auto& in_layout = node.get_input_layout(0); diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/reduce_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/reduce_onednn.hpp index fbdf64131ff384..68d963fd9e369f 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/reduce_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/reduce_onednn.hpp @@ -49,7 +49,7 @@ struct ReduceImplementationManager : public ImplementationManager { bool validate_impl(const program_node& node) const override { assert(node.is_type()); const auto& info = node.get_program().get_engine().get_device_info(); - if (!info.supports_immad) + if (!info.supports_immad || info.arch == gpu_arch::unknown) return false; const auto& reduce_node = node.as(); diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/reorder_onednn.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/reorder_onednn.hpp index b671f5e210e75c..ad08c516e939d8 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/reorder_onednn.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/reorder_onednn.hpp @@ -57,7 +57,7 @@ struct ReorderImplementationManager : public ImplementationManager { return true; const auto& info = node.get_program().get_engine().get_device_info(); - if (!info.supports_immad) + if (!info.supports_immad || info.arch == gpu_arch::unknown) return false; if (!one_of(input_fmt.value, supported_formats) || !one_of(output_fmt.value, supported_formats)) diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp b/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp index 19ea02c7c66d28..a8aa43671ed048 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp @@ -30,6 +30,7 @@ cldnn::memory::ptr convert_zp_data_to_s32(const memory::ptr zp_memory) { template cldnn::memory::ptr convert_zp_data_to_s32(const memory::ptr zp_memory); template cldnn::memory::ptr convert_zp_data_to_s32(const memory::ptr zp_memory); +template cldnn::memory::ptr convert_zp_data_to_s32(const memory::ptr zp_memory); cldnn::format default_fmt_for_dims(size_t dims, bool is_grouped) { switch (dims) { @@ -489,6 +490,7 @@ bool is_per_tensor(cldnn::data_node& node, int32_t& zp_val) { template bool is_per_tensor(cldnn::data_node& node, int32_t& zp_val); template bool is_per_tensor(cldnn::data_node& node, int32_t& zp_val); +template bool is_per_tensor(cldnn::data_node& node, int32_t& zp_val); static std::string get_external_order(const std::vector& order, bool is_weights, bool is_grouped) { diff --git a/src/plugins/intel_gpu/src/graph/impls/registry/gemm_impls.cpp b/src/plugins/intel_gpu/src/graph/impls/registry/gemm_impls.cpp index 66947ef1a84a00..436a3bb257b483 100644 --- a/src/plugins/intel_gpu/src/graph/impls/registry/gemm_impls.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/registry/gemm_impls.cpp @@ -19,7 +19,10 @@ const std::vector>& Registry static const std::vector> impls = { OV_GPU_CREATE_INSTANCE_ONEDNN(onednn::GemmImplementationManager, shape_types::static_shape) OV_GPU_GET_INSTANCE_OCL(gemm, shape_types::static_shape) - OV_GPU_GET_INSTANCE_OCL(gemm, shape_types::dynamic_shape) + OV_GPU_GET_INSTANCE_OCL(gemm, shape_types::dynamic_shape, + [](const program_node& node) { + return !node.can_use(impl_types::onednn); + }) }; return impls; diff --git a/src/plugins/intel_gpu/src/graph/impls/registry/reorder_impls.cpp b/src/plugins/intel_gpu/src/graph/impls/registry/reorder_impls.cpp index 3b38e2754fbc12..eab8c1f3d9c3f8 100644 --- a/src/plugins/intel_gpu/src/graph/impls/registry/reorder_impls.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/registry/reorder_impls.cpp @@ -36,7 +36,8 @@ const std::vector>& Registry : public typed_primitive_inst_base< // Internal function to be used from fakealignment template - static std::vector __calc_output_layouts(const layout &act_layout, uint64_t group_size); + static std::vector __calc_output_layouts(const layout &act_layout, + const dynamic_quantize::Attributes& config); static std::string to_string(dynamic_quantize_node const& node); typed_primitive_inst(network& network, dynamic_quantize_node const& node); diff --git a/src/plugins/intel_gpu/src/graph/include/kv_cache_inst.h b/src/plugins/intel_gpu/src/graph/include/kv_cache_inst.h index f3aa4de5ec34e1..da0a9397433f89 100644 --- a/src/plugins/intel_gpu/src/graph/include/kv_cache_inst.h +++ b/src/plugins/intel_gpu/src/graph/include/kv_cache_inst.h @@ -62,6 +62,12 @@ class typed_primitive_inst : public typed_primitive_inst_base= 0 ? sequence_axis : past_layout_rank + sequence_axis; } + static int64_t get_scale_zp_sequence_axis() { + // The order of scales and zero points is fixed, so use constant axis + const auto scale_zp_concat_axis = 2; + return scale_zp_concat_axis; + } + static int64_t get_max_pad(const layout& target_layout, size_t buffer_size, int64_t sequence_axis, std::string target_name = "") { if (buffer_size == 0) return 0; diff --git a/src/plugins/intel_gpu/src/graph/include/primitive_inst.h b/src/plugins/intel_gpu/src/graph/include/primitive_inst.h index 33571db8496642..6899812ab28097 100644 --- a/src/plugins/intel_gpu/src/graph/include/primitive_inst.h +++ b/src/plugins/intel_gpu/src/graph/include/primitive_inst.h @@ -325,6 +325,7 @@ class primitive_inst { virtual int32_t get_prealloc_iter_num() { return -1; } virtual void update_shape_info_tensor(const kernel_impl_params& params); kernel_impl_params get_fake_aligned_params_if_possible(kernel_impl_params const& orig_impl_param); + bool all_dependencies_cpu_impl() const; protected: primitive_inst(network& network, program_node const& node, bool allocate_memory); diff --git a/src/plugins/intel_gpu/src/graph/include/program_node.h b/src/plugins/intel_gpu/src/graph/include/program_node.h index ba5363f09f194b..8105a8bc07dec3 100644 --- a/src/plugins/intel_gpu/src/graph/include/program_node.h +++ b/src/plugins/intel_gpu/src/graph/include/program_node.h @@ -237,7 +237,7 @@ struct program_node { } void merge_output_padding(padding const& padd, size_t idx = 0) { - set_output_padding(padding::max(padd, output_layouts[idx].data_padding)); + set_output_padding(padding::max(padd, output_layouts[idx].data_padding), idx); } // only calculated output layout (for external usage), does not modify/use cached output layout nor invalidate users diff --git a/src/plugins/intel_gpu/src/graph/include/read_value_inst.h b/src/plugins/intel_gpu/src/graph/include/read_value_inst.h index 74f9ffff581b87..ea0c8b82bb21fa 100644 --- a/src/plugins/intel_gpu/src/graph/include/read_value_inst.h +++ b/src/plugins/intel_gpu/src/graph/include/read_value_inst.h @@ -33,9 +33,14 @@ class typed_primitive_inst : public typed_primitive_inst_base static std::vector calc_output_layouts(read_value_node const& /*node*/, const kernel_impl_params& impl_param) { auto desc = impl_param.typed_desc(); - const auto& default_layout = desc->output_layout; + std::vector output_layouts; - return { impl_param.state_layout.value_or(default_layout) }; + for (size_t i = 0; i < desc->num_outputs; i++) { + const auto& default_layout = desc->output_layouts[i]; + output_layouts.push_back(impl_param.state_layouts.size() > i ? impl_param.state_layouts[i] : default_layout); + } + + return output_layouts; } static layout calc_output_layout(const read_value_node& node, kernel_impl_params const& impl_param); diff --git a/src/plugins/intel_gpu/src/graph/include/reshape_inst.h b/src/plugins/intel_gpu/src/graph/include/reshape_inst.h index 1bbfd94256a50c..d6a71c20fcac8d 100644 --- a/src/plugins/intel_gpu/src/graph/include/reshape_inst.h +++ b/src/plugins/intel_gpu/src/graph/include/reshape_inst.h @@ -59,7 +59,7 @@ struct typed_program_node : public typed_program_node_base { return false; // TODO: If user is RoPE or MVN and dynamic padding exists, ouput padding propagation is not supported in the base mode - if (get_users().size() == 1 && (get_users().front()->is_type() || get_users().front()->is_type())) + if (get_users().size() == 1 && get_users().front()->is_type()) return false; auto axis = input().as().get_primitive()->axis; @@ -73,14 +73,17 @@ struct typed_program_node : public typed_program_node_base { const auto& output_pshape = prim->output_partial_shape; // TODO: If the reshape's output shape is non constant, issue occurs // during shape inference due to execution order at runtime - if ((output_pshape.size() != input_rank + 1) || prim->output_pattern.empty()) + if (prim->output_pattern.empty()) return false; + // Iteratively check the total product of all static innermost dimensions + // until the crop dimension value matches or the first dynamic dimension is encountered int64_t mul = 1; - for (size_t i = input_rank - 1; i < output_pshape.size() ; i++) { - if (output_pshape[i].is_dynamic()) - return false; - mul *= output_pshape[i].get_length(); + for (size_t i = output_pshape.size(); i > 1 ; i--) { + if (output_pshape[i - 1].is_dynamic() || mul == input_last_dim_val) + break; + + mul *= output_pshape[i - 1].get_length(); } if (input_last_dim_val != mul) return false; diff --git a/src/plugins/intel_gpu/src/graph/kv_cache.cpp b/src/plugins/intel_gpu/src/graph/kv_cache.cpp index 66a874b9b153ec..808a593c601ad0 100644 --- a/src/plugins/intel_gpu/src/graph/kv_cache.cpp +++ b/src/plugins/intel_gpu/src/graph/kv_cache.cpp @@ -3,6 +3,7 @@ // #include "intel_gpu/op/kv_cache.hpp" +#include "intel_gpu/op/kv_cache_compressed.hpp" #include "intel_gpu/plugin/common_utils.hpp" #include "intel_gpu/plugin/multi_tensor_variable_state.hpp" #include "intel_gpu/runtime/optionals.hpp" @@ -29,19 +30,39 @@ template std::vector kv_cache_inst::calc_output_layouts(kv_cache_node const& /*node*/, kernel_impl_params const& impl_param) { auto desc = impl_param.typed_desc(); - ov::intel_gpu::op::KVCache op; - op.set_output_size(desc->num_outputs); - op.set_concat_axis(desc->concat_axis); - op.set_gather_axis(desc->gather_axis); - std::vector input_shapes = {impl_param.get_input_layout(0).get(), impl_param.get_input_layout(1).get()}; - if (desc->num_outputs > 1) + if (desc->indirect) { input_shapes.push_back(impl_param.get_input_layout(2).get()); + } - std::vector output_shapes = shape_infer(&op, input_shapes); + if (desc->compressed) { + input_shapes.push_back(impl_param.get_input_layout(3).get()); + + if (desc->get_compression_zp_inputs_num() > 0) { + input_shapes.push_back(impl_param.get_input_layout(4).get()); + } + } + + std::vector output_shapes; + if (desc->compressed) { + ov::intel_gpu::op::KVCacheCompressed op; + op.set_output_size(desc->num_outputs); + op.set_concat_axis(desc->concat_axis); + op.set_gather_axis(desc->gather_axis); + op.set_quantization_attrs(desc->quantization_attributes); + + output_shapes = shape_infer(&op, input_shapes); + } else { + ov::intel_gpu::op::KVCache op; + op.set_output_size(desc->num_outputs); + op.set_concat_axis(desc->concat_axis); + op.set_gather_axis(desc->gather_axis); + + output_shapes = shape_infer(&op, input_shapes); + } - static const std::map ports_map = {{0, 0}, {1, 2}}; + static const std::map ports_map = {{0, 0}, {1, 2}, {2, 3}, {3, 4}}; std::vector out_layouts; for (size_t i = 0; i < desc->num_outputs; i++) { @@ -64,6 +85,9 @@ std::string kv_cache_inst::to_string(const kv_cache_node& node) { kv_cache_info.add("concat axis", node.get_primitive()->concat_axis); kv_cache_info.add("gather axis", node.get_primitive()->gather_axis); kv_cache_info.add("indirect", node.get_primitive()->indirect); + kv_cache_info.add("compressed", node.get_primitive()->compressed); + kv_cache_info.add("output_storage_type", static_cast(node.get_primitive()->quantization_attributes.output_storage_type)); + kv_cache_info.add("scales_zp_output_order", node.get_primitive()->quantization_attributes.scales_zp_output_order); node_info->add("kv_cache info", kv_cache_info); std::stringstream primitive_description; node_info->dump(primitive_description); diff --git a/src/plugins/intel_gpu/src/graph/primitive_inst.cpp b/src/plugins/intel_gpu/src/graph/primitive_inst.cpp index da56141cee4840..0d0748311caa08 100644 --- a/src/plugins/intel_gpu/src/graph/primitive_inst.cpp +++ b/src/plugins/intel_gpu/src/graph/primitive_inst.cpp @@ -40,6 +40,7 @@ #include "graph_optimizer/prepare_buffer_fusing.h" #include "intel_gpu/plugin/common_utils.hpp" +#include "intel_gpu/plugin/multi_tensor_variable_state.hpp" #include "intel_gpu/graph/network.hpp" #include "intel_gpu/graph/serialization/set_serializer.hpp" #include "intel_gpu/runtime/engine.hpp" @@ -294,30 +295,54 @@ void primitive_inst::update_shape() { auto prim = get_node().as().get_primitive(); const auto& variable_id = prim->variable_id; auto& variable = get_network().get_variable(variable_id); - // Initial variable shape is taken from variable itself - auto new_layout = variable.get_layout(); - // If variable is not set and we have an initializer - use it's shape as shape of variable - if (!variable.is_set() && _impl_params->input_layouts.size() == 1) { - new_layout = _impl_params->get_input_layout(0); - } + auto update_state_layout = [&](ov::intel_gpu::VariableStateBase& variable, layout new_layout, size_t layout_idx) { + // If variable is not set and we have an initializer - use it's shape as shape of variable + if (!variable.is_set() && _impl_params->input_layouts.size() > layout_idx) { + new_layout = _impl_params->get_input_layout(layout_idx); + } - // If we still have a dynamic dimension, which basiclly means that we don't have an initializer, then replace dynamic dims with 0 - if (new_layout.is_dynamic()) { - auto pshape = new_layout.get_partial_shape(); - for (auto& d : pshape) { - if (d.is_dynamic()) { - d = 0; + // If we still have a dynamic dimension, which basiclly means that we don't have an initializer, then replace dynamic dims with 0 + if (new_layout.is_dynamic()) { + auto pshape = new_layout.get_partial_shape(); + for (auto& d : pshape) { + if (d.is_dynamic()) { + d = 0; + } } + new_layout.set_partial_shape(pshape); } - new_layout.set_partial_shape(pshape); - } - variable.set_layout(new_layout); + variable.set_layout(new_layout); - if (!_impl_params->state_layout.has_value() || _impl_params->state_layout.value() != new_layout) { - _impl_params->state_layout = new_layout; - input_shape_changed = true; + if (_impl_params->state_layouts[layout_idx] != new_layout) { + _impl_params->state_layouts[layout_idx] = new_layout; + GPU_DEBUG_TRACE_DETAIL << "Update " << layout_idx << " layout: " << new_layout.to_short_string() << "\n"; + input_shape_changed = true; + } + }; + + if (_impl_params->state_layouts.empty()) + _impl_params->state_layouts.resize(1); + + // Initial variable shape is taken from variable itself + auto new_layout = variable.get_layout(); + update_state_layout(variable, new_layout, 0); + + if (prim->num_outputs > 1) { + if (auto compressed_cache_variable = dynamic_cast(&variable)) { + _impl_params->state_layouts.resize(compressed_cache_variable->has_zp_state() ? 3 : 2); + + auto scales_state = compressed_cache_variable->get_compression_scale_state(); + auto new_scales_layout = compressed_cache_variable->get_compression_scale_state()->get_layout(); + update_state_layout(*scales_state, new_scales_layout, 1); + + if (compressed_cache_variable->has_zp_state()) { + auto scales_state = compressed_cache_variable->get_compression_zp_state(); + auto new_zp_layout = compressed_cache_variable->get_compression_zp_state()->get_layout(); + update_state_layout(*scales_state, new_zp_layout, 2); + } + } } } @@ -462,6 +487,14 @@ void primitive_inst::update_shape() { auto& variable = get_network().get_variable(desc->variable_id); // Custom output layout update as update_output_layout handles paddings incorrectly for optimized out read_value + kv_cache pattern _impl_params->output_layouts[0] = variable.get_layout(); + + if (auto compressed_cache_variable = dynamic_cast(&variable)) { + _impl_params->output_layouts[1] = compressed_cache_variable->get_compression_scale_state()->get_layout(); + + if (compressed_cache_variable->has_zp_state()) { + _impl_params->output_layouts[2] = compressed_cache_variable->get_compression_zp_state()->get_layout(); + } + } } if (get_node().is_type()) { @@ -494,6 +527,25 @@ kernel_impl_params primitive_inst::get_fake_aligned_params_if_possible(kernel_im return updated_params; } +// Check if all dependencies and its predecessors are CPU or constant +static bool check_all_deps_cpu(const primitive_inst* inst) { + return std::all_of(inst->dependencies().begin(), inst->dependencies().end(), + [&](const std::pair& dep) { + if (dep.first->is_constant() || + (dep.first->get_impl() != nullptr && dep.first->get_impl()->is_cpu())) { + return true; + } + // Check if the dependency can be optimized + if (dep.first->can_be_optimized()) { + return check_all_deps_cpu(dep.first); + } + return false; + }); +} + +bool primitive_inst::all_dependencies_cpu_impl() const { + return check_all_deps_cpu(this); +} event::ptr primitive_inst::realloc_if_needed() { OV_ITT_SCOPED_TASK(ov::intel_gpu::itt::domains::intel_gpu_plugin, openvino::itt::handle("realloc_if_needed: " + id())); @@ -544,6 +596,15 @@ event::ptr primitive_inst::realloc_if_needed() { << ", variable layout " << variable.get_layout().to_short_string() << ")" << std::endl; _outputs[0] = variable.get_memory(); + + if (auto compressed_cache_variable = dynamic_cast(&variable)) { + _outputs[2] = compressed_cache_variable->get_compression_scale_state()->get_memory(); + + if (compressed_cache_variable->has_zp_state()) { + _outputs[3] = compressed_cache_variable->get_compression_zp_state()->get_memory(); + } + } + // To record shape predictor for (size_t j = 0; j < _impl_params->output_layouts.size(); ++j) sp.predict_preallocation_shape(id(), _impl_params->output_layouts[j], true, j); @@ -563,11 +624,27 @@ event::ptr primitive_inst::realloc_if_needed() { GPU_DEBUG_TRACE_DETAIL << id() << ": Update variable (ptr: " << variable.get_memory()->buffer_ptr() << ", actual_size:" << variable.get_actual_mem_size() << " bytes" << ", variable layout:" << variable.get_layout().to_short_string() << ")" << std::endl; + + if (auto compressed_cache_variable = dynamic_cast(&variable)) { + compressed_cache_variable->get_compression_scale_state()->set_layout(_impl_params->output_layouts[1]); + + if (compressed_cache_variable->has_zp_state()) { + compressed_cache_variable->get_compression_zp_state()->set_layout(_impl_params->output_layouts[2]); + } + } } // For nodes that can be optimized, variable memory is used as output memory // so there is no need for output memory reallocation if (can_be_optimized()) { _max_output_layout_count[0] = variable.get_actual_mem_size() / dt_sizes_in_B[0]; + + if (auto compressed_cache_variable = dynamic_cast(&variable)) { + const size_t scale_idx = _node->is_type() ? 1 : 2; // kv_cache or read_value + _max_output_layout_count[scale_idx] = compressed_cache_variable->get_compression_scale_state()->get_actual_mem_size() / dt_sizes_in_B[1]; + if (compressed_cache_variable->has_zp_state()) { + _max_output_layout_count[scale_idx + 1] = compressed_cache_variable->get_compression_zp_state()->get_actual_mem_size() / dt_sizes_in_B[2]; + } + } GPU_DEBUG_PROFILED_STAGE_MEMALLOC_INFO("can_be_optimized"); return ev; } @@ -646,7 +723,10 @@ event::ptr primitive_inst::realloc_if_needed() { // dynamic quantization is only applied to activation of FC if (get_node().is_type()) { - auto dyn_quan_scale_layout = dynamic_quantize_inst::__calc_output_layouts(updated_layouts[dep_idx], 0); + const auto& desc = get_node().as().get_primitive(); + auto dyn_quan_scale_layout = + dynamic_quantize_inst::__calc_output_layouts(updated_layouts[dep_idx], + desc->attrs); GPU_DEBUG_TRACE_DETAIL << "update layout of dynamic quantize scale parameter layout " << dyn_quan_scale_layout[1].to_short_string() << std::endl; updated_params.output_layouts[1] = dyn_quan_scale_layout[1]; @@ -671,13 +751,24 @@ event::ptr primitive_inst::realloc_if_needed() { // update layout to ensure that it repsects paddings for correct allocation size if (_node_output_layout.data_padding.is_dynamic()) { - auto current_dims = updated_layouts[0].get_padded_dims(); + auto update_padding = [](layout& orig_layout) { + auto current_dims = orig_layout.get_padded_dims(); + + std::vector current_buf_shape; + current_buf_shape.reserve(current_dims.size()); + std::transform(current_dims.begin(), current_dims.end(), + std::back_inserter(current_buf_shape), [](const tensor::value_type& el) { return static_cast(el); }); + orig_layout = layout(ov::PartialShape(current_buf_shape), orig_layout.data_type, orig_layout.format); + }; + + update_padding(updated_layouts[0]); - std::vector current_buf_shape; - current_buf_shape.reserve(current_dims.size()); - std::transform(current_dims.begin(), current_dims.end(), - std::back_inserter(current_buf_shape), [](const tensor::value_type& el) { return static_cast(el); }); - updated_layouts[0] = layout(ov::PartialShape(current_buf_shape), updated_layouts[0].data_type, updated_layouts[0].format); + // Update scales and zero points buffers paddings, skipping beam_table + if (_node->is_type()) { + for (size_t i = 2; i < updated_layouts.size(); ++i) { + update_padding(updated_layouts[i]); + } + } } int32_t tmp_prealloc_count = get_prealloc_iter_num(); @@ -690,13 +781,14 @@ event::ptr primitive_inst::realloc_if_needed() { for (size_t i = 0; i < updated_layouts.size(); ++i) { bool reclaim = 0; size_t required_buffer_size = 0; - if (_node->is_type() && i == 0) { + if (_node->is_type() && i != 1) { // Relax reclaiming condition for kv cache const auto& desc = _node->as().get_primitive(); auto prealloc_shape = updated_layouts[i].get_shape(); const auto shape_rank = prealloc_shape.size(); - auto seq_axis = - static_cast(desc->concat_axis >= 0 ? desc->concat_axis : shape_rank + desc->concat_axis); + const auto seq_axis = i == 0 ? kv_cache_inst::get_sequence_axis(desc->concat_axis, shape_rank) + : kv_cache_inst::get_scale_zp_sequence_axis(); + prealloc_shape[seq_axis] += tmp_prealloc_count; required_buffer_size = std::accumulate(prealloc_shape.begin(), prealloc_shape.end(), size_t(1), std::multiplies()); } else { @@ -723,11 +815,12 @@ event::ptr primitive_inst::realloc_if_needed() { for (size_t i = 0; i < actual_layouts.size(); ++i) { bool can_reuse_buffer = (_outputs[i] && updated_layouts[i].get_linear_size() <= _max_output_layout_count[i]); std::pair prealloc_info; - if (_node->is_type() && i == 0) { + if (_node->is_type() && i != 1) { const auto& desc = _node->as().get_primitive(); - auto shape_rank = updated_layouts[i].get_shape().size(); - auto seq_axis = - static_cast(desc->concat_axis >= 0 ? desc->concat_axis : shape_rank + desc->concat_axis); + const auto shape_rank = updated_layouts[i].get_shape().size(); + const auto seq_axis = i == 0 ? kv_cache_inst::get_sequence_axis(desc->concat_axis, shape_rank) + : kv_cache_inst::get_scale_zp_sequence_axis(); + prealloc_info = sp.predict_preallocation_shape(id(), updated_layouts[i], false, i, tmp_prealloc_count, seq_axis); } else { prealloc_info = sp.predict_preallocation_shape(id(), updated_layouts[i], can_reuse_buffer, i, tmp_prealloc_count); @@ -743,19 +836,21 @@ event::ptr primitive_inst::realloc_if_needed() { GPU_DEBUG_TRACE_DETAIL << id() << ": reuse previously allocated output buffer[" << i << "] - " << actual_layouts[i].get_linear_size() << "/" << _max_output_layout_count[i] << std::endl; - if (_node->is_type() && (i == 0)) { + if (_node->is_type() && i != 1) { // kv_cache has already assigned memory. // No need to reinterpret output memory but need to update padding const auto& desc = _node->as().get_primitive(); auto& present_layout = _impl_params->output_layouts[i]; const auto present_layout_rank = present_layout.get_partial_shape().size(); - const auto sequence_axis = kv_cache_inst::get_sequence_axis(desc->concat_axis, present_layout_rank); + const auto sequence_axis = i == 0 ? kv_cache_inst::get_sequence_axis(desc->concat_axis, present_layout_rank) + : kv_cache_inst::get_scale_zp_sequence_axis();; + auto max_pad = kv_cache_inst::get_max_pad(present_layout, _max_output_layout_count[i], sequence_axis, - "present_layout"); + i == 0 ? "present_layout" : "present_scales_layout"); kv_cache_inst::update_pad(present_layout, max_pad, sequence_axis); - GPU_DEBUG_TRACE_DETAIL << _impl_params->output_layouts[i].to_string() << std::endl; + GPU_DEBUG_TRACE_DETAIL << i << ". " << _impl_params->output_layouts[i].to_string() << std::endl; set_shape_change(); } else { _outputs[i] = _network.get_engine().reinterpret_buffer(*_outputs[i], actual_layouts[i]); @@ -816,6 +911,26 @@ event::ptr primitive_inst::realloc_if_needed() { sequence_axis, "present_layout"); if (max_pad > 0) { + if (auto compressed_cache_variable = dynamic_cast(&variable)) { + auto present_scales_layout = _impl_params->output_layouts[2]; + const auto sequence_axis = kv_cache_inst::get_scale_zp_sequence_axis();; + + // In case of compressed KV-cache, calling update_impl for each iteration + // because of scales layout [batch, num_heads, seq_len, head_size], which requires proper + // dynamic padding updates + axis_is_outer_most = false; + kv_cache_inst::update_pad(present_scales_layout, max_pad, sequence_axis); + + _impl_params->output_layouts[2] = present_scales_layout; + compressed_cache_variable->get_compression_scale_state()->set_memory(_outputs[2], present_scales_layout); + if (compressed_cache_variable->has_zp_state()) { + auto present_zp_layout = present_scales_layout; + + _impl_params->output_layouts[3] = present_scales_layout; + compressed_cache_variable->get_compression_zp_state()->set_memory(_outputs[3], present_zp_layout); + } + } + kv_cache_inst::update_pad(present_layout, max_pad, sequence_axis); if (!axis_is_outer_most) { GPU_DEBUG_TRACE_DETAIL << id() << ": Update impl with new output padding" << std::endl; @@ -836,12 +951,32 @@ event::ptr primitive_inst::realloc_if_needed() { << "'s layout with allocated kv cache output: " << present_layout.to_short_string() << " (is_set = " << variable.is_set() << ") " << std::endl; variable.set_memory(_outputs[0], present_layout); + + if (auto compressed_cache_variable = dynamic_cast(&variable)) { + auto present_scales_layout = _impl_params->output_layouts[2]; + + compressed_cache_variable->get_compression_scale_state()->set_memory(_outputs[2], present_scales_layout); + if (compressed_cache_variable->has_zp_state()) { + auto present_zp_layout = present_scales_layout; + compressed_cache_variable->get_compression_zp_state()->set_memory(_outputs[3], present_zp_layout); + } + } } } else { GPU_DEBUG_TRACE_DETAIL << id() << ": Update variable " << variable.get_name() << "'s layout with allocated kv cache output: " << present_layout.to_short_string() << " (is_set = " << variable.is_set() << ") " << std::endl; variable.set_layout(present_layout); + + if (auto compressed_cache_variable = dynamic_cast(&variable)) { + auto present_scales_layout = _impl_params->output_layouts[2]; + + compressed_cache_variable->get_compression_scale_state()->set_layout(present_scales_layout); + if (compressed_cache_variable->has_zp_state()) { + auto present_zp_layout = present_scales_layout; + compressed_cache_variable->get_compression_zp_state()->set_layout(present_zp_layout); + } + } } } @@ -1017,8 +1152,8 @@ bool primitive_inst::update_impl(bool use_async_compilation) { } void primitive_inst::update_paddings() { - auto reset_pad = [](kernel_impl_params& params, const program_node* node) { - params.output_layouts[0].data_padding = node->get_output_layout(0).data_padding; + auto reset_pad = [](kernel_impl_params& params, const program_node* node, size_t idx = 0) { + params.output_layouts[idx].data_padding = node->get_output_layout(idx).data_padding; }; if (_node->is_type() || _node->is_type()) { auto variable_id = _node->is_type() ? (_node->as().get_primitive()->variable_id) @@ -1030,6 +1165,15 @@ void primitive_inst::update_paddings() { primitive_inst* inst = this; while (inst) { reset_pad(*inst->_impl_params, inst->_node); + if (inst == this) { + if (auto compressed_cache_variable = dynamic_cast(&variable)) { + const size_t scale_idx = _node->is_type() ? 1 : 2; + reset_pad(*inst->_impl_params, inst->_node, scale_idx); + if (compressed_cache_variable->has_zp_state()) { + reset_pad(*inst->_impl_params, inst->_node, scale_idx + 1); + } + } + } auto& users = inst->_node->get_users(); if (users.size() == 1 && users.front()->get_output_layout(0).data_padding.is_dynamic()) { inst = inst->get_user_insts().front(); @@ -1155,11 +1299,42 @@ void primitive_inst::do_runtime_in_place_kv_cache() { GPU_DEBUG_TRACE_DETAIL << "[do runtime_in_place_kv_cache] " << id() << " Updated present_layout's pad : " << present_layout.to_string() << std::endl; auto& variable = get_network().get_variable(desc->variable_info.variable_id); variable.set_layout(present_layout); + + if (desc->compressed) { + auto compressed_cache_variable = dynamic_cast(&variable); + auto& present_scales_layout = _impl_params->output_layouts[2]; + const auto sequence_axis = kv_cache_inst::get_scale_zp_sequence_axis(); + kv_cache_inst::update_pad(present_scales_layout, max_pad - new_seq_len, sequence_axis); + GPU_DEBUG_TRACE_DETAIL << "[do runtime_in_place_kv_cache] " << id() + << " Updated present_scale_layout's pad : " << present_scales_layout.to_string() << std::endl; + + compressed_cache_variable->get_compression_scale_state()->set_layout(present_scales_layout); + if (desc->get_compression_zp_inputs_num() > 0) { + auto& present_zp_layout = _impl_params->output_layouts[3]; + kv_cache_inst::update_pad(present_zp_layout, max_pad - new_seq_len, sequence_axis); + GPU_DEBUG_TRACE_DETAIL << "[do runtime_in_place_kv_cache] " << id() + << " Updated present_zp_layout's pad : " << present_scales_layout.to_string() << std::endl; + + compressed_cache_variable->get_compression_zp_state()->set_layout(present_scales_layout); + } + } + GPU_DEBUG_TRACE_DETAIL << "[do_runtime_in_place_kv_cache] " << id() << "Updated variable with present_layout" << variable.get_layout().to_string() << " is_set = " << variable.is_set() << std::endl; if (past_layout.data_padding._upper_size[sequence_axis] > 0 && variable.is_set()) { kv_cache_inst::update_pad(past_layout, max_pad, sequence_axis); _impl_params->_can_be_optimized = true; + + if (desc->compressed) { + auto& past_scale_layout = _impl_params->input_layouts[3]; + const auto sequence_axis = kv_cache_inst::get_scale_zp_sequence_axis(); + kv_cache_inst::update_pad(past_scale_layout, max_pad, sequence_axis); + + if (desc->get_compression_zp_inputs_num() > 0) { + auto& past_zp_layout = _impl_params->input_layouts[4]; + kv_cache_inst::update_pad(past_zp_layout, max_pad, sequence_axis); + } + } GPU_DEBUG_TRACE_DETAIL << "[do_runtime_in_place_kv_cache] " << id() << " Updated past layout's pad : " << past_layout.to_string() << std::endl; } } @@ -1251,36 +1426,31 @@ void primitive_inst::do_runtime_skip_permute() { auto desc = _node->as().get_primitive(); auto input_shape = _impl_params->get_input_layout(0).get_shape(); const auto& permute_order = desc->permute_order; - // Check runtime shape - // Optimize when the largest value among the actual dim values in case where the permute order - // is different from the shape index is equal to the multiplied value - // Set size to zero to pass the case that permute order is same with input order like [0, 1, 2, 3] - int32_t size = 0; - int32_t max_value = 0; - for (int32_t i = 0; i < static_cast(permute_order.size()); ++i) { - int32_t order = static_cast(permute_order[i]); - int32_t dim = static_cast(input_shape[order]); - if (i != order) { - if (dim > max_value) - max_value = dim; - size = (size == 0) ? dim : (size * dim); - } - } - - // If the largest value and total size are different, can_be_optimized needs to be reset - if (size != max_value) { - set_can_be_optimized(false); - GPU_DEBUG_TRACE_DETAIL << "--- Cannot optimize because size(" << size << ") and max_value(" << max_value - << ") are different" << std::endl; - - GPU_DEBUG_TRACE_DETAIL << "[do_runtime_skip_permute] " << id() << " : reset can_be_optimized to false " - << std::endl; - return; + // Can skip if the transposed dim keeps the original order + bool can_skip = true; + auto permute_dest = permute_order; + for (size_t i = 0; i < permute_order.size(); ++i) { + permute_dest[permute_order[i]] = static_cast(i); + } + int16_t prev_dim = -1; + for (size_t i = 0; i < permute_dest.size(); ++i) { + if (input_shape[i] > 1) { + if (permute_dest[i] < prev_dim) { + can_skip = false; + break; + } + prev_dim = permute_dest[i]; + } } - GPU_DEBUG_TRACE_DETAIL << "[do_runtime_skip_permute] " << id() << " : can_be_optimized" << std::endl; + GPU_DEBUG_TRACE_DETAIL << "[do_runtime_skip_permute] " << id() << " : can_be_optimized ? " << can_skip << std::endl; GPU_DEBUG_TRACE_DETAIL << " - Input layout : " << _impl_params->get_input_layout(0).to_short_string() << std::endl; GPU_DEBUG_TRACE_DETAIL << " - Output layout : " << _impl_params->get_output_layout().to_short_string() << std::endl; - set_can_be_optimized(true); + GPU_DEBUG_TRACE_DETAIL << " - permute order : "; + for (auto order : permute_order) { + GPU_DEBUG_TRACE_DETAIL << order << ","; + } + GPU_DEBUG_TRACE_DETAIL << std::endl; + set_can_be_optimized(can_skip); } void primitive_inst::do_runtime_skip_strided_slice() { diff --git a/src/plugins/intel_gpu/src/graph/program.cpp b/src/plugins/intel_gpu/src/graph/program.cpp index 1e2e84043dc82b..07fad4873659cd 100644 --- a/src/plugins/intel_gpu/src/graph/program.cpp +++ b/src/plugins/intel_gpu/src/graph/program.cpp @@ -7,6 +7,7 @@ #include "openvino/core/type.hpp" #include "openvino/runtime/system_conf.hpp" #include "openvino/runtime/threading/cpu_streams_info.hpp" +#include "openvino/util/weights_path.hpp" #include "intel_gpu/runtime/memory.hpp" #include "intel_gpu/runtime/engine.hpp" @@ -1839,7 +1840,8 @@ void program::load(cldnn::BinaryInputBuffer& ib) { std::shared_ptr mapped_memory = nullptr; std::string weights_path = _config.get_property(ov::weights_path); - if (!weights_path.empty()) { + if (_config.get_property(ov::cache_mode) == ov::CacheMode::OPTIMIZE_SIZE && + ov::util::validate_weights_path(weights_path)) { mapped_memory = ov::load_mmap_object(weights_path); } diff --git a/src/plugins/intel_gpu/src/graph/program_node.cpp b/src/plugins/intel_gpu/src/graph/program_node.cpp index fc9648b90e444c..a9bb7c665f177b 100644 --- a/src/plugins/intel_gpu/src/graph/program_node.cpp +++ b/src/plugins/intel_gpu/src/graph/program_node.cpp @@ -90,7 +90,8 @@ void program_node::replace_dependency(size_t idx, std::pair const program_node::get_input_layouts() const { std::vector layouts; for (size_t i = 0; i < dependencies.size(); i++) { - layouts.push_back(get_input_layout(i)); + auto input_layout = get_input_layout(i); + layouts.push_back(input_layout); } return layouts; } diff --git a/src/plugins/intel_gpu/src/graph/read_value.cpp b/src/plugins/intel_gpu/src/graph/read_value.cpp index bf6e730e8a808b..1d6657b9bf8ac4 100644 --- a/src/plugins/intel_gpu/src/graph/read_value.cpp +++ b/src/plugins/intel_gpu/src/graph/read_value.cpp @@ -2,8 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 // -#include +#include "read_value_inst.h" #include "primitive_type_base.h" + +#include "intel_gpu/plugin/multi_tensor_variable_state.hpp" + #include #include @@ -16,7 +19,7 @@ read_value_inst::typed_primitive_inst(network& network, const read_value_node& n } layout read_value_inst::calc_output_layout(const read_value_node& node, kernel_impl_params const& impl_param) { - return impl_param.typed_desc()->output_layout; + return impl_param.typed_desc()->output_layouts[0]; } std::string read_value_inst::to_string(const read_value_node& node) { @@ -45,5 +48,25 @@ void read_value_inst::update_output_memory() { GPU_DEBUG_TRACE_DETAIL << " - layout " << variable.get_layout().to_string() << std::endl; GPU_DEBUG_TRACE_DETAIL << " - actual_size " << variable.get_actual_mem_size() << " bytes" << std::endl; set_output_memory(variable.get_memory(), false, 0); + + if (auto compressed_cache_variable = dynamic_cast(&variable)) { + auto scales_state = compressed_cache_variable->get_compression_scale_state(); + set_output_memory(scales_state->get_memory(), false, 1); + + GPU_DEBUG_TRACE_DETAIL << id() << " Update output memory with variable " << scales_state->get_name() << std::endl; + GPU_DEBUG_TRACE_DETAIL << " - ptr : " << scales_state->get_memory()->buffer_ptr() << std::endl; + GPU_DEBUG_TRACE_DETAIL << " - layout " << scales_state->get_layout().to_string() << std::endl; + GPU_DEBUG_TRACE_DETAIL << " - actual_size " << scales_state->get_actual_mem_size() << " bytes" << std::endl; + + if (compressed_cache_variable->has_zp_state()) { + auto zp_state = compressed_cache_variable->get_compression_zp_state(); + set_output_memory(zp_state->get_memory(), false, 2); + + GPU_DEBUG_TRACE_DETAIL << id() << " Update output memory with variable " << zp_state->get_name() << std::endl; + GPU_DEBUG_TRACE_DETAIL << " - ptr : " << zp_state->get_memory()->buffer_ptr() << std::endl; + GPU_DEBUG_TRACE_DETAIL << " - layout " << zp_state->get_layout().to_string() << std::endl; + GPU_DEBUG_TRACE_DETAIL << " - actual_size " << zp_state->get_actual_mem_size() << " bytes" << std::endl; + } + } } } // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/scaled_dot_product_attention.cpp b/src/plugins/intel_gpu/src/graph/scaled_dot_product_attention.cpp index e8e213ad97011a..e80cb62a534b52 100644 --- a/src/plugins/intel_gpu/src/graph/scaled_dot_product_attention.cpp +++ b/src/plugins/intel_gpu/src/graph/scaled_dot_product_attention.cpp @@ -11,6 +11,7 @@ #include "scaled_dot_product_attention_shape_inference.hpp" #include "intel_gpu/op/sdpa.hpp" +#include "ov_ops/dynamic_quantize.hpp" namespace cldnn { GPU_DEFINE_PRIMITIVE_TYPE_ID(scaled_dot_product_attention) @@ -87,6 +88,14 @@ std::string scaled_dot_product_attention_inst::to_string(scaled_dot_product_atte json_composite scaled_dot_product_attention_info; scaled_dot_product_attention_info.add("input id", input.id()); scaled_dot_product_attention_info.add("is_causal", desc->is_causal); + scaled_dot_product_attention_info.add("is_kv_compressed", desc->is_kv_compressed); + scaled_dot_product_attention_info.add("output_storage_type", static_cast(node.get_primitive()->quantization_attributes.output_storage_type)); + scaled_dot_product_attention_info.add("group_size", desc->quantization_attributes.group_sizes); + scaled_dot_product_attention_info.add("quantization_type", static_cast(node.get_primitive()->quantization_attributes.quantization_type)); + scaled_dot_product_attention_info.add("quantization_dt", desc->quantization_attributes.quantization_dt); + scaled_dot_product_attention_info.add("scale_dt", desc->quantization_attributes.scale_dt); + scaled_dot_product_attention_info.add("zp_dt", desc->quantization_attributes.zp_dt); + scaled_dot_product_attention_info.add("indirect_axis", desc->indirect_axis); scaled_dot_product_attention_info.add("has_attn_mask_input", desc->has_attn_mask_input); scaled_dot_product_attention_info.add("has_scale_input", desc->has_scale_input); scaled_dot_product_attention_info.add("input_q_transpose_order", desc->input_q_transpose_order); diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_kv_cache.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_kv_cache.cl new file mode 100644 index 00000000000000..22a2f03c94564a --- /dev/null +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_kv_cache.cl @@ -0,0 +1,121 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "include/batch_headers/fetch_data.cl" +#include "include/batch_headers/fetch_data.cl" +#include "include/batch_headers/common.cl" +#include "include/batch_headers/sub_group_block_read.cl" +#include "include/batch_headers/sub_group_block_write.cl" +#include "include/batch_headers/sub_group_shuffle.cl" + + +#if OUTPUT_DIMS != 4 +#error "dynamic_quantize_gpu_opt.cl: Unsupported output dimension" +#endif + +#define VLOAD_N CAT(vload, VEC_SIZE) +#define VSTORE_N CAT(vstore, VEC_SIZE) +#define CONVERT_CHAR_N CAT(convert_char, VEC_SIZE) +#define AS_TYPE_N_(type, n, x) as_##type##n(x) +#define AS_TYPE_N(type, n, x) AS_TYPE_N_(type, n, x) +#define AS_INPUT_TYPE_N(x) AS_TYPE_N(INPUT0_TYPE, VEC_SIZE, x) + + +inline uint FUNC(get_scales_offset_nt)(OPTIONAL_SHAPE_INFO_ARG uint b, uint f, uint y, uint x) { + return OUTPUT1_GET_INDEX(b, f, y, x); +} + +inline uint FUNC(get_scales_offset)(OPTIONAL_SHAPE_INFO_ARG uint b, uint f, uint y, uint x) { +#ifdef SCALES_OUTPUT_ORDER + return FUNC_CALL(get_scales_offset_nt)(OPTIONAL_SHAPE_INFO_TENSOR SCALES_OUTPUT_ORDER); +#else + return FUNC_CALL(get_scales_offset_nt)(OPTIONAL_SHAPE_INFO_TENSOR b, f, y, x); +#endif +} + +#define SUBGROUP_SIZE 16 +#define INNERMOST_DIM_VALUE INPUT0_SIZE_X +#define INPUT_BLOCK_READ(ptr, offset) BLOCK_READN(INPUT0_TYPE, 1, ptr, offset) +#define OUTPUT_BLOCK_WRITE(ptr, offset, val) BLOCK_WRITEN(OUTPUT_TYPE, 1, ptr, offset, val) + +__attribute__((reqd_work_group_size(SUBGROUP_SIZE, SUBGROUPS_NUMBER, 1))) +REQD_SUB_GROUP_SIZE(SUBGROUP_SIZE) +KERNEL(dynamic_quantize_gpu_kv_cache)( + OPTIONAL_SHAPE_INFO_ARG + const __global INPUT0_TYPE* input, + __global OUTPUT_TYPE* output, + __global OUTPUT1_TYPE* output_scale +#if ASYMMETRIC_QUANTIZATION && !GROUP_SCALES_WITH_ZP + , __global OUTPUT2_TYPE* output_zp +#endif +#ifdef APPEND_MODE + , const uint axis_offset +#endif + ) +{ + const uint sglid = get_sub_group_local_id(); + const uint grouped_indexes = get_global_id(1); + const uint batch_indexes = get_global_id(2); + + DECLARE_BATCHED_DIMS_INDEXES(batch_indexes); + DECLARE_GROUPED_DIMS_INDEXES(grouped_indexes); + + // The innermost dimension is always processed in the loop inside the kernel + const uint x = 0; + + half max_value = INPUT0_VAL_MIN; + half min_value = INPUT0_VAL_MAX; + + half val[INNERMOST_DIM_VALUE / SUBGROUP_SIZE]; + + const uint input_offset = INPUT0_GET_INDEX(b, f, y, x); + unroll_for (uint i = 0; i < INNERMOST_DIM_VALUE / SUBGROUP_SIZE; i++) { + val[i] = INPUT_BLOCK_READ(input, input_offset + i * SUBGROUP_SIZE); +#if ASYMMETRIC_QUANTIZATION + max_value = fmax(max_value, val[i]); + min_value = fmin(min_value, val[i]); +#else + max_value = fmax(max_value, fabs(val[i])); +#endif + } + +#if ASYMMETRIC_QUANTIZATION + min_value = work_group_reduce_min(min_value); + max_value = work_group_reduce_max(max_value); + OUTPUT1_TYPE scale = (OUTPUT1_TYPE)((CHAR_MAX - CHAR_MIN) / (max_value - min_value)); + OUTPUT1_TYPE zp = (OUTPUT1_TYPE)(-min_value * scale) - CHAR_MAX; +#else + max_value = work_group_reduce_max(max_value); + OUTPUT1_TYPE scale = 127.0h / max_value; +#endif + +#ifdef APPEND_MODE + APPEND_AXIS_NAME += axis_offset; +#endif + + const uint output_offset = OUTPUT_GET_INDEX(b, f, y, x); + unroll_for (uint i = 0; i < INNERMOST_DIM_VALUE / SUBGROUP_SIZE; i++) { +#if ASYMMETRIC_QUANTIZATION + OUTPUT_TYPE res = convert_char_rte(val[i] * scale + zp); +#else + OUTPUT_TYPE res = convert_char_rte(val[i] * scale); +#endif + OUTPUT_BLOCK_WRITE(output, output_offset + i * SUBGROUP_SIZE, res); + } + + const uint scale_idx = FUNC_CALL(get_scales_offset)(OPTIONAL_SHAPE_INFO_TENSOR b, f, y, x); + + if (grouped_indexes == 0 && sglid == 0) { +#if ASYMMETRIC_QUANTIZATION + output_scale[scale_idx] = 1.0h / scale; +#if GROUP_SCALES_WITH_ZP + output_scale[scale_idx + 1] = zp; +#else + output_zp[scale_idx] = zp; +#endif +#else + output_scale[scale_idx] = 1.0h / scale; +#endif + } +} diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl index 858571fea71914..62482b8b9b5047 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/dynamic_quantize_gpu_ref.cl @@ -8,48 +8,143 @@ #error "dynamic_quantize_gpu_ref.cl: Unsupported output dimension" #endif +inline uint FUNC(get_scales_offset_nt)(OPTIONAL_SHAPE_INFO_ARG uint b, uint f, uint y, uint x) { + return OUTPUT1_GET_INDEX(b, f, y, x); +} + +inline uint FUNC(get_scales_offset)(OPTIONAL_SHAPE_INFO_ARG uint b, uint f, uint y, uint x) { +#ifdef SCALES_OUTPUT_ORDER + return FUNC_CALL(get_scales_offset_nt)(OPTIONAL_SHAPE_INFO_TENSOR SCALES_OUTPUT_ORDER); +#else + return FUNC_CALL(get_scales_offset_nt)(OPTIONAL_SHAPE_INFO_TENSOR b, f, y, x); +#endif +} + KERNEL(dynamic_quantize_gpu_ref)( OPTIONAL_SHAPE_INFO_ARG const __global INPUT0_TYPE* input, __global OUTPUT_TYPE* output, - __global OUTPUT1_TYPE* output_scale) + __global OUTPUT1_TYPE* output_scale +#if ASYMMETRIC_QUANTIZATION && !GROUP_SCALES_WITH_ZP + , __global OUTPUT2_TYPE* output_zp +#endif +) { const uint bf = (uint)get_global_id(0); - const uint b = (uint)get_global_id(0) / INPUT0_FEATURE_NUM; - const uint f = (uint)get_global_id(0) % INPUT0_FEATURE_NUM; + const uint b = bf / INPUT0_FEATURE_NUM; + const uint f = bf % INPUT0_FEATURE_NUM; const uint y = (uint)get_global_id(1); - const uint scale_idx = OUTPUT1_GET_INDEX(b, f, y, 0); + const uint x = (uint)get_global_id(2); +#ifdef SCALES_OUTPUT_ORDER + const uint scale_idx = FUNC_CALL(get_scales_offset)(OPTIONAL_SHAPE_INFO_TENSOR b, f, y, x); +#else + const uint scale_idx = OUTPUT1_GET_INDEX_SAFE(b, f, y, x); +#endif - half max_val = 0.0001h; - for (int y_off = 0; y_off < (get_global_size(1) == 1 ? INPUT0_SIZE_Y : 1); y_off++) { - const uint offset = INPUT0_GET_INDEX(b, f, y + y_off, 0); + half max_val = INPUT0_VAL_MIN; + half min_val = INPUT0_VAL_MAX; + for (int b_off = 0; b_off < (GROUP_SIZE_DIM0 == 1 ? 1 : INPUT0_BATCH_NUM); b_off++) { + for (int f_off = 0; f_off < (GROUP_SIZE_DIM1 == 1 ? 1 : INPUT0_FEATURE_NUM); f_off++) { + for (int y_off = 0; y_off < (GROUP_SIZE_DIM2 == 1 ? 1 : INPUT0_SIZE_Y); y_off++) { +#if GROUP_SIZE_DIM3 == 1 + const uint offset = INPUT0_GET_INDEX(b + b_off, f + f_off, y + y_off, x); + half val = input[offset]; +#if ASYMMETRIC_QUANTIZATION + max_val = fmax(max_value, val); + min_val = fmin(min_value, val); +#else + half abs_val = fabs(val); + max_val = fmax(max_val, abs_val); +#endif +#else + const uint offset = INPUT0_GET_INDEX(b + b_off, f + f_off, y + y_off, 0); int x; for (x = 0; x < INPUT0_SIZE_X / 8; x++) { half8 val = as_half8(vload8(0, (ushort*)input + offset + x * 8)); half8 abs_val = fabs(val); - - for (int j = 0; j < 8; j++) + for (int j = 0; j < 8; j++) { +#if ASYMMETRIC_QUANTIZATION + max_val = fmax(max_val, val[j]); + min_val = fmin(min_val, val[j]); +#else max_val = fmax(max_val, abs_val[j]); +#endif + } } x *= 8; - for (; x < INPUT0_SIZE_X; x++) - max_val = fmax(max_val, fabs(input[offset + x])); + for (; x < INPUT0_SIZE_X; x++) { + half val = input[offset + x]; +#if ASYMMETRIC_QUANTIZATION + max_val = fmax(max_val, val); + min_val = fmin(min_val, val); +#else + max_val = fmax(max_val, fabs(val)); +#endif + } +#endif + } + } } - half scale = 127.0h / max_val; - for (int y_off = 0; y_off < (get_global_size(1) == 1 ? INPUT0_SIZE_Y : 1); y_off++) { - const uint in_offset = INPUT0_GET_INDEX(b, f, y + y_off, 0); - const uint out_offset = OUTPUT_GET_INDEX(b, f, y + y_off, 0); +#if ASYMMETRIC_QUANTIZATION + OUTPUT1_TYPE scale = (OUTPUT1_TYPE)((CHAR_MAX - CHAR_MIN) / (max_val - min_val)); + OUTPUT1_TYPE zp = (OUTPUT1_TYPE)(-min_val * scale) - CHAR_MAX; +#else + max_val = work_group_reduce_max(max_val); + OUTPUT1_TYPE scale = 127.0h / max_val; +#endif + + for (int b_off = 0; b_off < (GROUP_SIZE_DIM0 == 1 ? 1 : INPUT0_BATCH_NUM); b_off++) { + for (int f_off = 0; f_off < (GROUP_SIZE_DIM1 == 1 ? 1 : INPUT0_FEATURE_NUM); f_off++) { + for (int y_off = 0; y_off < (GROUP_SIZE_DIM2 == 1 ? 1 : INPUT0_SIZE_Y); y_off++) { +#if GROUP_SIZE_DIM3 == 1 + const uint in_offset = INPUT0_GET_INDEX(b + b_off, f + f_off, y + y_off, x); + const uint out_offset = OUTPUT_GET_INDEX(b + b_off, f + f_off, y + y_off, x); + + half val = input[in_offset]; +#if ASYMMETRIC_QUANTIZATION + val *= scale; + val += zp; + output[out_offset] = convert_char_rte(val); +#else + val *= scale; + output[out_offset] = convert_char_rte(val); +#endif +#else + const uint in_offset = INPUT0_GET_INDEX(b + b_off, f + f_off, y + y_off, 0); + const uint out_offset = OUTPUT_GET_INDEX(b + b_off, f + f_off, y + y_off, 0); int x; for (x = 0; x < INPUT0_SIZE_X / 8; x++) { half8 val = as_half8(vload8(0, (ushort*)input + in_offset + x * 8)); +#if ASYMMETRIC_QUANTIZATION val *= scale; + val += zp; +#else + val *= scale; +#endif vstore8(convert_char8_rte(val), 0, output + out_offset + x * 8); } x *= 8; - for (; x < INPUT0_SIZE_X; x++) - output[out_offset + x] = convert_char(input[in_offset + x] * scale); + for (; x < INPUT0_SIZE_X; x++) { + half val = input[in_offset + x]; +#if ASYMMETRIC_QUANTIZATION + val *= scale; + val += zp; + output[out_offset + x] = convert_char_rte(val); +#else + val *= scale; + output[out_offset + x] = convert_char_rte(val); +#endif + } +#endif + } + } } output_scale[scale_idx] = 1.0h / scale; +#if ASYMMETRIC_QUANTIZATION && GROUP_SCALES_WITH_ZP + output_scale[scale_idx + 1] = zp; +#elif ASYMMETRIC_QUANTIZATION + output_zp[scale_idx] = zp; +#endif } diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/fully_connected_gpu_bf_tiled.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/fully_connected_gpu_bf_tiled.cl index 70c55bfb73b8f5..ef4cc76650e0f3 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/fully_connected_gpu_bf_tiled.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/fully_connected_gpu_bf_tiled.cl @@ -22,36 +22,47 @@ #if FC_KERNEL_DYNAMIC_QUANTIZE KERNEL(quantize_input)( const __global INPUT0_TYPE* input, - __global char* quantized_input, - __global INPUT0_TYPE* de_quan_scale) { + __global DQ_TYPE* quantized_input, + __global INPUT0_TYPE* quan_var +) { const uint offset = get_global_id(0); const uint input_offset = offset * QUANTIZE_GROUP_SIZE; const uint quantize_block = QUANTIZE_GROUP_SIZE / 4; - half4 input_0[quantize_block]; - char4 quantized_value[quantize_block]; - half max[quantize_block]; + MAKE_VECTOR_TYPE(INPUT0_TYPE, INPUT_LOAD_SIZE) input_0[quantize_block]; + MAKE_VECTOR_TYPE(DQ_TYPE, INPUT_LOAD_SIZE) quantized_value[quantize_block]; + INPUT0_TYPE max[quantize_block]; unroll_for (uint i = 0 ; i < quantize_block ; ++i) { input_0[i] = vload4(0, &input[input_offset + i * 4]); max[i] = fmax(fmax(fabs(input_0[i][0]), fabs(input_0[i][1])), fmax(fabs(input_0[i][2]), fabs(input_0[i][3]))); } - half max_value = 0.001; - for (uint i = 0 ; i < quantize_block; i+=8) { - half temp = fmax(fmax(fmax(max[i], max[i+1]), fmax(max[i+2], max[i+3])), + INPUT0_TYPE max_value = 0.001; + for (uint i = 0 ; i < quantize_block ; i+=8) { + INPUT0_TYPE temp = fmax(fmax(fmax(max[i], max[i+1]), fmax(max[i+2], max[i+3])), fmax(fmax(max[i+4], max[i+5]), fmax(max[i+6], max[i+7]))); max_value = fmax(max_value, temp); } - half quan_scale = max_value / 128; - - unroll_for (uint i = 0 ; i < quantize_block ; ++i) { - quantized_value[i] = CAT(convert_, MAKE_VECTOR_TYPE(char, INPUT_LOAD_SIZE))(input_0[i] / (half4)quan_scale); + half quan_scale = (half)max_value / 127; + #if COMPRESSED_WEIGHTS_INT8 + half quantized_sum = 0; + #endif + for (uint i = 0 ; i < quantize_block ; ++i) { + half4 buff = input_0[i] / (half4)quan_scale; + quantized_value[i] = CAT(CAT(convert_, MAKE_VECTOR_TYPE(DQ_TYPE, INPUT_LOAD_SIZE)), _rte)(buff); + #if COMPRESSED_WEIGHTS_INT8 + quantized_sum += (buff[0] + buff[1] + buff[2] + buff[3]); + #endif vstore4(quantized_value[i], 0, &quantized_input[input_offset + i * 4]); } - de_quan_scale[offset] = quan_scale; + // Pair of quantizing_scale and quantized activation_sum for each group + quan_var[offset * 2] = quan_scale; + #if COMPRESSED_WEIGHTS_INT8 + quan_var[(offset * 2) + 1] = quantized_sum; + #endif } #else // !FC_KERNEL_DYNAMIC_QUANTIZE @@ -189,6 +200,7 @@ inline void FUNC(fc_bf_tiled_kernel_default)( #else uint gid = (uint)get_group_id(0); #endif + uint sglid = (uint)get_sub_group_local_id(); // Dispatch as bs_fs_bsv_fsv, where bsv = DISPATCH_BSV and fsv = DISPATCH_FSV. @@ -212,10 +224,9 @@ inline void FUNC(fc_bf_tiled_kernel_default)( ACCUMULATOR_VEC_TYPE acc[TILE_B] = { }; INPUT_VEC_TYPE in_0[TILE_B] = { }; -#if !USE_SLM - FILTER_VEC_TYPE wei = 0; -#endif - + #if !USE_SLM || !COMPRESSED_WEIGHTS_INT4 + FILTER_VEC_TYPE wei = 0; + #endif #if OUTPUT_3D uint out_b0 = out_b / OUTPUT_FEATURE_NUM; @@ -743,19 +754,31 @@ inline void FUNC(fc_bf_tiled_kernel_default)( // ===================================================================================================================================== } + + + // Dyc Quantize #if USE_SLM && DYNAMIC_QUANTIZE -#define PACKED_DQ_TYPE int -#define DQ_VEC_TYPE MAKE_VECTOR_TYPE(DQ_TYPE, TILE_IFM) -#define DQ_SLM_FILTER_VEC MAKE_VECTOR_TYPE(DQ_TYPE, 4) + +#if COMPRESSED_WEIGHTS_INT4 + #define SLM_WEIGHT_TYPE DQ_TYPE +#else + #define SLM_WEIGHT_TYPE FILTER_TYPE +#endif + +#define PACKED_DQ_TYPE uint +#define ACCUM_DQ_TYPE int #define DQ_SLM_FILTER_PACKED_VEC MAKE_VECTOR_TYPE(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE) -#define DQ_SLM_FILTER_UNPACKED_VEC MAKE_VECTOR_TYPE(DQ_TYPE, FILTER_ELEMENTS_PER_LOAD) -#define DQ_FILTER_VEC_TYPE MAKE_VECTOR_TYPE(DQ_TYPE, TILE_K_OFM) +#define SLM_WEIGHT_VEC MAKE_VECTOR_TYPE(SLM_WEIGHT_TYPE, INPUT_LOAD_SIZE) +#define SLM_WEIGHT_UNPACKED_VEC MAKE_VECTOR_TYPE(SLM_WEIGHT_TYPE, FILTER_ELEMENTS_PER_LOAD) +#define WEIGHT_VEC_TYPE MAKE_VECTOR_TYPE(SLM_WEIGHT_TYPE, TILE_K_OFM) +#define MAKE_DQ_TYPE_VEC(x) MAKE_VECTOR_TYPE(DQ_TYPE, x) #define TO_DQ_TYPE(x) CAT(CAT(convert_, DQ_TYPE),_sat)(x) #define TO_DQ_VEC_TYPE(x) CAT(convert_, DQ_VEC_TYPE)(x) -#define TO_DQ_SLM_FILTER_UNPACKED_VEC(x) CAT(convert_, DQ_SLM_FILTER_UNPACKED_VEC)(x) -#define TO_DQ_FILTER_VEC_TYPE(x) CAT(convert_, DQ_FILTER_VEC_TYPE)(x) +#define TO_ACCUM_DQ_TYPE(x) CAT(convert_, ACCUM_DQ_TYPE)(x) +#define TO_SLM_WEIGHT_UNPACKED_VEC(x) CAT(convert_, SLM_WEIGHT_UNPACKED_VEC)(x) +#define TO_WEIGHT_VEC_TYPE(x) CAT(convert_, WEIGHT_VEC_TYPE)(x) #define AS_TYPE_N_(type, n, x) as_##type##n(x) #define AS_TYPE_N(type, n, x) AS_TYPE_N_(type, n, x) @@ -764,8 +787,8 @@ inline void FUNC(fc_bf_tiled_kernel_default)( inline void FUNC(fc_bf_tiled_kernel_dyn_quan)( OPTIONAL_SHAPE_INFO_ARG const __global INPUT0_TYPE* input, - __global char* quantized_input, - __global INPUT0_TYPE* scale, + __global DQ_TYPE* quantized_input, + __global INPUT0_TYPE* quan_var, // pair of params for each quantizing group : scale, activation_sum #if DECOMPRESSION_SCALE_TERM const __global DECOMPRESSION_SCALE_TYPE* decompression_scale, #endif @@ -774,7 +797,7 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)( #endif __global OUTPUT_TYPE* output, const __global FILTER_TYPE* weights - , __local int* wei_local_mem + , __local uint* wei_local_mem #if BIAS_TERM , const __global BIAS_TYPE* biases #endif @@ -801,28 +824,32 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)( uint out_f = gid * (TILE_OFM * SIMD); uint out_b = LWS_BATCHES * TILE_B * (uint)get_group_id(2) + local_id * TILE_B; -#if OUTPUT_3D - uint out_b0 = out_b / OUTPUT_FEATURE_NUM; - uint out_b1 = out_b % OUTPUT_FEATURE_NUM; - uint input_offset = out_b0 * INPUT0_BATCH_PITCH + out_b1 * INPUT0_FEATURE_PITCH + INPUT0_OFFSET; -#else - uint input_offset = out_b * TILE_IN_B_PITCH + INPUT0_OFFSET; -#endif + #if OUTPUT_3D + uint out_b0 = out_b / OUTPUT_FEATURE_NUM; + uint out_b1 = out_b % OUTPUT_FEATURE_NUM; + uint input_offset = out_b0 * INPUT0_BATCH_PITCH + out_b1 * INPUT0_FEATURE_PITCH + INPUT0_OFFSET; + #else + uint input_offset = out_b * TILE_IN_B_PITCH + INPUT0_OFFSET; + #endif -#if FILTER_LAYOUT_OS_IS_YX_OSV64_ISV2 - const int power_of_two_for_simd = 5; - const int power_of_two_for_osv = 6; - const uint osv64_weight_base = (( (int) (out_f >> power_of_two_for_osv) ) << power_of_two_for_osv); - const uint osv_weight_stride = (INPUT_ELEMENTS_COUNT >> 1); - const uint out_f_offset = (int)((out_f >> power_of_two_for_simd) & 0x1) << power_of_two_for_simd; - // out_f(32) : 0 * osv_weight_stride + 32; - // out_f(64) : 64 * osv_weight_stride + 0; - // out_f(128) : 64 * osv_weight_stride + 32; - // ... - uint weights_offset = osv64_weight_base * osv_weight_stride + out_f_offset; -#else - uint weights_offset = out_f * (INPUT_ELEMENTS_COUNT / 2); -#endif + #if COMPRESSED_WEIGHTS_INT4 + #if FILTER_LAYOUT_OS_IS_YX_OSV64_ISV2 + const int power_of_two_for_simd = 5; + const int power_of_two_for_osv = 6; + const uint osv64_weight_base = (( (int) (out_f >> power_of_two_for_osv) ) << power_of_two_for_osv); + const uint osv_weight_stride = (INPUT_ELEMENTS_COUNT >> 1); + const uint out_f_offset = (int)((out_f >> power_of_two_for_simd) & 0x1) << power_of_two_for_simd; + // out_f(32) : 0 * osv_weight_stride + 32; + // out_f(64) : 64 * osv_weight_stride + 0; + // out_f(128) : 64 * osv_weight_stride + 32; + // ... + uint weights_offset = osv64_weight_base * osv_weight_stride + out_f_offset; + #else + uint weights_offset = out_f * (INPUT_ELEMENTS_COUNT / 2); + #endif + #else + uint weights_offset = out_f * INPUT_ELEMENTS_COUNT; + #endif ACCUMULATOR_VEC_TYPE acc[TILE_B] = { }; @@ -831,38 +858,42 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)( PACKED_DQ_TYPE packed_in_0[HALF_TILE_B] = { }; // Packing char4 inputs to 1 integer INPUT0_TYPE de_quantize_scale[TILE_B]; -#if COMPRESSED_WEIGHTS && DECOMPRESSION_SCALE_GROUPS_NUM == 1 - #if DECOMPRESSION_SCALE_LENGTH > 1 && DECOMPRESSION_SCALE_LENGTH % (TILE_OFM * SIMD) == 0 - ACCUMULATOR_VEC_TYPE d_scale = TO_ACCUMULATOR_VEC_TYPE(BLOCK_READN(DECOMPRESSION_SCALE_TYPE, TILE_OFM, decompression_scale, out_f)); - #elif DECOMPRESSION_SCALE_LENGTH > 1 && DECOMPRESSION_SCALE_LENGTH % (TILE_OFM * SIMD) != 0 - ACCUMULATOR_VEC_TYPE d_scale = 0; - unroll_for(uint of = 0; of < TILE_OFM; ++of) { - uint offset = out_f + of*SIMD + get_sub_group_local_id(); - if (offset < DECOMPRESSION_SCALE_LENGTH) - ((ACCUMULATOR_TYPE*)(&d_scale))[of] = decompression_scale[offset]; - } - #else - ACCUMULATOR_VEC_TYPE d_scale = decompression_scale[0]; + #if COMPRESSED_WEIGHTS_INT8 + INPUT0_TYPE activation_sum[TILE_B] = { }; #endif - ACCUMULATOR_TYPE* d_scales = (ACCUMULATOR_TYPE*)(&d_scale); -#endif + #if COMPRESSED_WEIGHTS && DECOMPRESSION_SCALE_GROUPS_NUM == 1 + #if DECOMPRESSION_SCALE_LENGTH > 1 && DECOMPRESSION_SCALE_LENGTH % (TILE_OFM * SIMD) == 0 + ACCUMULATOR_VEC_TYPE d_scale = TO_ACCUMULATOR_VEC_TYPE(BLOCK_READN(DECOMPRESSION_SCALE_TYPE, TILE_OFM, decompression_scale, out_f)); + #elif DECOMPRESSION_SCALE_LENGTH > 1 && DECOMPRESSION_SCALE_LENGTH % (TILE_OFM * SIMD) != 0 + ACCUMULATOR_VEC_TYPE d_scale = 0; + unroll_for(uint of = 0; of < TILE_OFM; ++of) { + uint offset = out_f + of*SIMD + get_sub_group_local_id(); + if (offset < DECOMPRESSION_SCALE_LENGTH) + ((ACCUMULATOR_TYPE*)(&d_scale))[of] = decompression_scale[offset]; + } + #else + ACCUMULATOR_VEC_TYPE d_scale = decompression_scale[0]; + #endif -#if COMPRESSED_WEIGHTS && DECOMPRESSION_ZP_TERM && DECOMPRESSION_ZP_GROUPS_NUM == 1 && !DECOMPRESSION_ZP_SCALAR - #if DECOMPRESSION_ZP_LENGTH > 1 && DECOMPRESSION_ZP_LENGTH % (TILE_OFM * SIMD) == 0 - ACCUMULATOR_VEC_TYPE d_zp = TO_ACCUMULATOR_VEC_TYPE(BLOCK_READN(DECOMPRESSION_ZP_TYPE, TILE_OFM, decompression_zp, out_f)); - #elif DECOMPRESSION_ZP_LENGTH > 1 && DECOMPRESSION_ZP_LENGTH % (TILE_OFM * SIMD) != 0 - ACCUMULATOR_VEC_TYPE d_zp = 0; - unroll_for(uint of = 0; of < TILE_OFM; ++of) { - uint offset = out_f + of*SIMD + get_sub_group_local_id(); - if (offset < DECOMPRESSION_ZP_LENGTH) - ((ACCUMULATOR_TYPE*)(&d_zp))[of] = decompression_zp[offset]; - } - #else - ACCUMULATOR_VEC_TYPE d_zp = decompression_zp[0]; + ACCUMULATOR_TYPE* d_scales = (ACCUMULATOR_TYPE*)(&d_scale); + #endif + + #if COMPRESSED_WEIGHTS && DECOMPRESSION_ZP_TERM && DECOMPRESSION_ZP_GROUPS_NUM == 1 && !DECOMPRESSION_ZP_SCALAR + #if DECOMPRESSION_ZP_LENGTH > 1 && DECOMPRESSION_ZP_LENGTH % (TILE_OFM * SIMD) == 0 + ACCUMULATOR_VEC_TYPE d_zp = TO_ACCUMULATOR_VEC_TYPE(BLOCK_READN(DECOMPRESSION_ZP_TYPE, TILE_OFM, decompression_zp, out_f)); + #elif DECOMPRESSION_ZP_LENGTH > 1 && DECOMPRESSION_ZP_LENGTH % (TILE_OFM * SIMD) != 0 + ACCUMULATOR_VEC_TYPE d_zp = 0; + unroll_for(uint of = 0; of < TILE_OFM; ++of) { + uint offset = out_f + of*SIMD + get_sub_group_local_id(); + if (offset < DECOMPRESSION_ZP_LENGTH) + ((ACCUMULATOR_TYPE*)(&d_zp))[of] = decompression_zp[offset]; + } + #else + ACCUMULATOR_VEC_TYPE d_zp = decompression_zp[0]; + #endif + ACCUMULATOR_TYPE* d_zps = (ACCUMULATOR_TYPE*)(&d_zp); #endif - ACCUMULATOR_TYPE* d_zps = (ACCUMULATOR_TYPE*)(&d_zp); -#endif // ===================================================================================================================================== // Main computation loop @@ -871,7 +902,7 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)( uint idx_sglid = (sglid * TILE_K) % TILE_IFM_ELEMENTS_SIZE; // same index for sglid 0~7 : to tile_k direction uint batch_sglid = (sglid * TILE_K) / TILE_IFM_ELEMENTS_SIZE; // 0 to 1 : to batch direction - const uint scale_pitch = TILE_IN_B_PITCH / QUANTIZE_GROUP_SIZE; + const uint scale_pitch = (TILE_IN_B_PITCH / QUANTIZE_GROUP_SIZE); MAKE_VECTOR_TYPE(int, TILE_B) acc_tmp[TILE_OFM] = { }; __attribute__((opencl_unroll_hint(1))) for (uint ni = 0; ni < iterations; ++ni) { @@ -881,14 +912,20 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)( // Load quantizing info from pre-quantizing kernel tiled_input_0[bi] = vload4(0, &quantized_input[in_offset]); // Packing : Get 4(B)x4(K) integer vector (packing to 4x1 vector) - packed_in_0[bi] = as_int(tiled_input_0[bi]); + packed_in_0[bi] = as_uint(tiled_input_0[bi]); // Next batch in_offset += (TILE_IN_B_PITCH * 2); #if NUM_LOOP_IN_DYN_QUAN_GROUP == 1 - de_quantize_scale[bi * 2] = scale[scale_offset]; - de_quantize_scale[bi * 2 + 1] = scale[scale_offset+ scale_pitch]; + de_quantize_scale[bi * 2] = quan_var[scale_offset * 2]; + de_quantize_scale[bi * 2 + 1] = quan_var[scale_offset * 2 + scale_pitch * 2]; + #if COMPRESSED_WEIGHTS_INT8 + // Need additional accumulation of quantized activation along the dyn-quan group + // to use i8 multiplier for int8 weight + activation_sum[bi * 2] = quan_var[scale_offset * 2 + 1]; + activation_sum[bi * 2 + 1] = quan_var[scale_offset * 2 + 1 + scale_pitch * 2]; + #endif scale_offset += (scale_pitch * 2); #endif } @@ -896,7 +933,10 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)( #if NUM_LOOP_IN_DYN_QUAN_GROUP > 1 if (ni % NUM_LOOP_IN_DYN_QUAN_GROUP == 0) { unroll_for (uint bi = 0; bi < TILE_B; ++bi) { - de_quantize_scale[bi] = scale[scale_offset]; + de_quantize_scale[bi] = quan_var[scale_offset * 2]; + #if COMPRESSED_WEIGHTS_INT8 + activation_sum[bi] = quan_var[scale_offset * 2 + 1]; + #endif scale_offset += scale_pitch; } } @@ -916,49 +956,64 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)( barrier(CLK_LOCAL_MEM_FENCE); #endif - __local int* char_slm_weight = (__local int*)wei_local_mem; + __local uint* char_slm_weight = (__local uint*)wei_local_mem; - #if FILTER_LAYOUT_OS_IS_YX_OSV64_ISV2 - uint weights_idx = weights_offset + local_id * SIMD * FILTER_LOAD_ITERS * FILTER_LOAD_BLOCK_SIZE * 2; + #if COMPRESSED_WEIGHTS_INT4 + #if FILTER_LAYOUT_OS_IS_YX_OSV64_ISV2 + uint weights_idx = weights_offset + local_id * SIMD * FILTER_LOAD_ITERS * FILTER_LOAD_BLOCK_SIZE * 2; + #else + uint weights_idx = weights_offset + local_id * SIMD * FILTER_LOAD_ITERS * FILTER_ACTUAL_LOAD_BLOCK_SIZE; + #endif #else - uint weights_idx = weights_offset + local_id * SIMD * FILTER_LOAD_ITERS * FILTER_ACTUAL_LOAD_BLOCK_SIZE; + uint weights_idx = weights_offset + local_id * SIMD * FILTER_LOAD_ITERS * TILE_K_OFM_PACKED; #endif uint wei_local_idx = local_id * SIMD * FILTER_LOAD_ITERS * (FILTER_LOAD_BLOCK_SIZE/2) + sglid * 2; - // DECOMPRESSION_SCALE_POST_OP SHOULD be enabled for dynamic quantize FC : scale is ACCUMULATOR_VAL_ONE + #if COMPRESSED_WEIGHTS_INT8 + ACCUMULATOR_TYPE wei_zp[TILE_OFM] = { }; + #endif + + // DQ_DECOMPRESSION_SCALE_POST_OP SHOULD be enabled for dynamic quantize FC : scale is ACCUMULATOR_VAL_ONE unroll_for(uint load_iter = 0; load_iter < FILTER_LOAD_ITERS; ++load_iter) { - #if FILTER_LAYOUT_OS_IYX_OSV16 - SLM_FILTER_PACKED_VEC wei_packed0 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, weights_idx); - SLM_FILTER_PACKED_VEC wei_packed1 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, (weights_idx + ((IFM_SIZE / 2) * 16))); - DQ_SLM_FILTER_UNPACKED_VEC dq_wei_unpacked; - // loaded weights 'wei_packed' of os_iyx_osv16 format have continuous values along TILE_K. So no need to transpose while unpacking - dq_wei_unpacked.s0123 = UNPACK_INT4(DQ_TYPE, *((INT4_PACKED_TYPE_PRELOAD*)&wei_packed0)); - dq_wei_unpacked.s4567 = UNPACK_INT4(DQ_TYPE, *((INT4_PACKED_TYPE_PRELOAD*)&wei_packed1)); - #elif FILTER_LAYOUT_OS_IS_YX_OSV64_ISV2 - SLM_FILTER_PACKED_VEC wei_packed0 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, weights_idx); - SLM_FILTER_PACKED_VEC wei_packed1 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, (weights_idx + (FILTER_LOAD_BLOCK_SIZE * SIMD))); - DQ_SLM_FILTER_UNPACKED_VEC dq_wei_unpacked; - DQ_SLM_FILTER_UNPACKED_VEC dq_wei_unpacked_tmp; - dq_wei_unpacked_tmp.s0123 = UNPACK_INT4(DQ_TYPE, *((INT4_PACKED_TYPE_PRELOAD*)&wei_packed0)); - dq_wei_unpacked_tmp.s4567 = UNPACK_INT4(DQ_TYPE, *((INT4_PACKED_TYPE_PRELOAD*)&wei_packed1)); - dq_wei_unpacked.s01 = dq_wei_unpacked_tmp.s01; - dq_wei_unpacked.s23 = dq_wei_unpacked_tmp.s45; - dq_wei_unpacked.s45 = dq_wei_unpacked_tmp.s23; - dq_wei_unpacked.s67 = dq_wei_unpacked_tmp.s67; - #else - SLM_FILTER_PACKED_VEC wei_packed = BLOCK_READN(FILTER_TYPE, FILTER_LOAD_BLOCK_SIZE, weights, weights_idx); - DQ_SLM_FILTER_UNPACKED_VEC dq_wei_unpacked = UNPACK_TRANSPOSED_INT4(DQ_TYPE, *((INT4_PACKED_TYPE_PRELOAD *)&wei_packed)); + #if COMPRESSED_WEIGHTS_INT4 + #if FILTER_LAYOUT_OS_IYX_OSV16 + SLM_FILTER_PACKED_VEC wei_packed0 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, weights_idx); + SLM_FILTER_PACKED_VEC wei_packed1 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, (weights_idx + ((IFM_SIZE / 2) * 16))); + SLM_WEIGHT_UNPACKED_VEC dq_wei_unpacked; + // loaded weights 'wei_packed' of os_iyx_osv16 format have continuous values along TILE_K. So no need to transpose while unpacking + dq_wei_unpacked.s0123 = (UNPACK_INT4(DQ_TYPE, *((INT4_PACKED_TYPE_PRELOAD*)&wei_packed0))); + dq_wei_unpacked.s4567 = (UNPACK_INT4(DQ_TYPE, *((INT4_PACKED_TYPE_PRELOAD*)&wei_packed1))); + #elif FILTER_LAYOUT_OS_IS_YX_OSV64_ISV2 + SLM_FILTER_PACKED_VEC wei_packed0 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, weights_idx); + SLM_FILTER_PACKED_VEC wei_packed1 = BLOCK_READN(FILTER_TYPE, FILTER_ACTUAL_LOAD_BLOCK_SIZE, weights, (weights_idx + (FILTER_LOAD_BLOCK_SIZE * SIMD))); + SLM_WEIGHT_UNPACKED_VEC dq_wei_unpacked; + SLM_WEIGHT_UNPACKED_VEC dq_wei_unpacked_tmp; + dq_wei_unpacked_tmp.s0123 = (UNPACK_INT4(DQ_TYPE, *((INT4_PACKED_TYPE_PRELOAD*)&wei_packed0))); + dq_wei_unpacked_tmp.s4567 = (UNPACK_INT4(DQ_TYPE, *((INT4_PACKED_TYPE_PRELOAD*)&wei_packed1))); + dq_wei_unpacked.s01 = dq_wei_unpacked_tmp.s01; + dq_wei_unpacked.s23 = dq_wei_unpacked_tmp.s45; + dq_wei_unpacked.s45 = dq_wei_unpacked_tmp.s23; + dq_wei_unpacked.s67 = dq_wei_unpacked_tmp.s67; + #else + SLM_FILTER_PACKED_VEC wei_packed = BLOCK_READN(FILTER_TYPE, FILTER_LOAD_BLOCK_SIZE, weights, weights_idx); + SLM_WEIGHT_UNPACKED_VEC dq_wei_unpacked = (UNPACK_TRANSPOSED_INT4(DQ_TYPE, *((INT4_PACKED_TYPE_PRELOAD *)&wei_packed))); + #endif + #else // COMPRESSED_WEIGHTS_INT8 + SLM_WEIGHT_UNPACKED_VEC dq_wei_unpacked; + WEIGHT_VEC_TYPE wei_packed = TO_WEIGHT_VEC_TYPE(FILTER_BLOCK_READ(weights, weights_idx)); + dq_wei_unpacked.s0123 = wei_packed.s0246; + dq_wei_unpacked.s4567 = wei_packed.s1357; #endif - // Calculate zero-point and scale only for DECOMPRESSION_SCALE_POST_OP enabled + // Calculate zero-point and scale only for DQ_DECOMPRESSION_SCALE_POST_OP enabled // Calculate weight : w = (w - dzp) * ds // if DECOMPRESSION_ZP_TERM is not enabled, then dzp is ACCUMULATOR_VAL_ZERO. - #if DECOMPRESSION_ZP_TERM + #if DECOMPRESSION_ZP_TERM && !COMPRESSED_WEIGHTS_INT8 #if DECOMPRESSION_ZP_SCALAR - DQ_SLM_FILTER_UNPACKED_VEC dzp = (DQ_SLM_FILTER_UNPACKED_VEC)(DECOMPRESSION_ZP_VALUE); + SLM_WEIGHT_UNPACKED_VEC dzp = (SLM_WEIGHT_UNPACKED_VEC)(DECOMPRESSION_ZP_VALUE); dq_wei_unpacked -= dzp; #elif DECOMPRESSION_ZP_GROUPS_NUM > 1 - DQ_TYPE* w = (DQ_TYPE*)(&dq_wei_unpacked); + SLM_WEIGHT_TYPE* w = (SLM_WEIGHT_TYPE*)(&dq_wei_unpacked); const uint ni_offset = ni * TILE_IFM * SIMD + local_id * FILTER_LOAD_ITERS * FILTER_LOAD_BLOCK_SIZE; unroll_for(uint fi = 0; fi < TILE_OFM; ++fi) { const uint offset_ofm = out_f + fi*SIMD + sglid; @@ -966,11 +1021,11 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)( const uint offset_ifm = ni_offset + load_iter * FILTER_LOAD_BLOCK_SIZE + kii; const uint zp_offset = (offset_ofm % DECOMPRESSION_ZP_BATCH_NUM) * DECOMPRESSION_ZP_BATCH_PITCH + (offset_ifm / DECOMPRESSION_ZP_GROUP_SIZE) * DECOMPRESSION_ZP_FEATURE_PITCH; - w[W_DYN_QUAN_IDX] = w[W_DYN_QUAN_IDX] - TO_DQ_TYPE(decompression_zp[zp_offset]); + w[W_DYN_QUAN_IDX] = w[W_DYN_QUAN_IDX] - CAT(CAT(convert_, SLM_WEIGHT_TYPE),_rte)(decompression_zp[zp_offset]); } } #else - DQ_TYPE* w = (DQ_TYPE*)(&dq_wei_unpacked); + SLM_WEIGHT_TYPE* w = (SLM_WEIGHT_TYPE*)(&dq_wei_unpacked); unroll_for(uint fi = 0; fi < TILE_OFM; ++fi) { unroll_for(uint kii = 0; kii < FILTER_LOAD_BLOCK_SIZE; ++kii) { w[W_DYN_QUAN_IDX] = w[W_DYN_QUAN_IDX] - d_zps[fi % DECOMPRESSION_ZP_LENGTH]; @@ -979,29 +1034,58 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)( #endif #endif + #if COMPRESSED_WEIGHTS_INT8 + unroll_for(uint fi = 0; fi < TILE_OFM; ++fi) { + #if DECOMPRESSION_ZP_TERM + #if DECOMPRESSION_ZP_SCALAR + wei_zp[fi] = (TO_ACCUMULATOR_TYPE)(DECOMPRESSION_ZP_VALUE); + #elif DECOMPRESSION_ZP_GROUPS_NUM > 1 + #if FILTER_LOAD_BLOCK_SIZE % DECOMPRESSION_ZP_GROUP_SIZE != 0 + #error "FC bf_tiled kernel: Not support DECOMPRESSION_ZP_GROUPS_NUM > 1" + #endif + + const uint ni_offset = ni * TILE_IFM * SIMD + local_id * FILTER_LOAD_ITERS * FILTER_LOAD_BLOCK_SIZE; + const uint offset_ofm = out_f + fi*SIMD + sglid; + const uint offset_ifm = ni_offset + load_iter * FILTER_LOAD_BLOCK_SIZE; + const uint zp_offset = (offset_ofm % DECOMPRESSION_ZP_BATCH_NUM) * DECOMPRESSION_ZP_BATCH_PITCH + + (offset_ifm / DECOMPRESSION_ZP_GROUP_SIZE) * DECOMPRESSION_ZP_FEATURE_PITCH; + wei_zp[fi] = TO_ACCUMULATOR_TYPE(decompression_zp[zp_offset]); + #else + wei_zp[fi] = TO_ACCUMULATOR_TYPE(d_zps[fi % DECOMPRESSION_ZP_LENGTH]); + #endif + #else + wei_zp[fi] = ACCUMULATOR_VAL_ZERO; + #endif + } + #endif + #if FILTER_LOAD_BLOCK_SIZE == 2 - DQ_SLM_FILTER_VEC wei_1 = {dq_wei_unpacked.s01, dq_wei_unpacked.s23}; - char_slm_weight[wei_local_idx] = as_int(wei_1); + SLM_WEIGHT_VEC wei_1 = {dq_wei_unpacked.s01, dq_wei_unpacked.s23}; + char_slm_weight[wei_local_idx] = as_uint(wei_1); #elif FILTER_LOAD_BLOCK_SIZE == 4 - DQ_SLM_FILTER_VEC wei_1 = {dq_wei_unpacked.s01, dq_wei_unpacked.s23}; - char_slm_weight[wei_local_idx] = as_int(wei_1); - DQ_SLM_FILTER_VEC wei_2 = {dq_wei_unpacked.s45, dq_wei_unpacked.s67}; - char_slm_weight[wei_local_idx+1] = as_int(wei_2); + SLM_WEIGHT_VEC wei_1 = {dq_wei_unpacked.s01, dq_wei_unpacked.s23}; + char_slm_weight[wei_local_idx] = as_uint(wei_1); + SLM_WEIGHT_VEC wei_2 = {dq_wei_unpacked.s45, dq_wei_unpacked.s67}; + char_slm_weight[wei_local_idx+1] = as_uint(wei_2); #elif FILTER_LOAD_BLOCK_SIZE == 8 - DQ_SLM_FILTER_VEC wei_1 = {dq_wei_unpacked.s01, dq_wei_unpacked.s23}; - char_slm_weight[wei_local_idx] = as_int(wei_1); - DQ_SLM_FILTER_VEC wei_2 = {dq_wei_unpacked.s45, dq_wei_unpacked.s67}; - char_slm_weight[wei_local_idx+1] = as_int(wei_2); - DQ_SLM_FILTER_VEC wei_3 = {dq_wei_unpacked.s89, dq_wei_unpacked.sab}; - char_slm_weight[wei_local_idx+2] = as_int(wei_3); - DQ_SLM_FILTER_VEC wei_4 = {dq_wei_unpacked.scd, dq_wei_unpacked.sef}; - char_slm_weight[wei_local_idx+3] = as_int(wei_4); + SLM_WEIGHT_VEC wei_1 = {dq_wei_unpacked.s01, dq_wei_unpacked.s23}; + char_slm_weight[wei_local_idx] = as_uint(wei_1); + SLM_WEIGHT_VEC wei_2 = {dq_wei_unpacked.s45, dq_wei_unpacked.s67}; + char_slm_weight[wei_local_idx+1] = as_uint(wei_2); + SLM_WEIGHT_VEC wei_3 = {dq_wei_unpacked.s89, dq_wei_unpacked.sab}; + char_slm_weight[wei_local_idx+2] = as_uint(wei_3); + SLM_WEIGHT_VEC wei_4 = {dq_wei_unpacked.scd, dq_wei_unpacked.sef}; + char_slm_weight[wei_local_idx+3] = as_uint(wei_4); #else #error "FC bf_tiled kernel: unsupported FILTER_LOAD_BLOCK_SIZE for SLM kernel" #endif wei_local_idx += SIMD * (FILTER_LOAD_BLOCK_SIZE/2); - weights_idx += SIMD * FILTER_ACTUAL_LOAD_BLOCK_SIZE; + #if COMPRESSED_WEIGHTS_INT8 + weights_idx += SIMD * TILE_K_OFM_PACKED; + #else + weights_idx += SIMD * FILTER_ACTUAL_LOAD_BLOCK_SIZE; + #endif } wei_local_idx = sglid * 2; @@ -1014,11 +1098,11 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)( #endif // Compute input * weight : packed char4 type - char8 weight = vload8(0, (__local char *)(&char_slm_weight[wei_local_idx + 16*2*ki])); - char4 first_weight = weight.s0123; - char4 second_weight = weight.s4567; + WEIGHT_VEC_TYPE weight = vload8(0, (__local SLM_WEIGHT_TYPE *)(&char_slm_weight[wei_local_idx + 16*2*ki])); + SLM_WEIGHT_VEC first_weight = weight.s0123; + SLM_WEIGHT_VEC second_weight = weight.s4567; unroll_for (uint bi = 0; bi < TILE_B; ++bi) { - char4 input_val = as_char4(_sub_group_shuffle(packed_in_0[bi / 2], (bi % 2) * 8 + ki)); + MAKE_DQ_TYPE_VEC(4) input_val = AS_DQ_TYPE_4(_sub_group_shuffle(packed_in_0[bi / 2], (bi % 2) * 8 + ki)); acc_tmp[0][bi] = imad_SW(acc_tmp[0][bi], input_val, first_weight); acc_tmp[1][bi] = imad_SW(acc_tmp[1][bi], input_val, second_weight); } @@ -1038,7 +1122,12 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)( ACCUMULATOR_TYPE ds = d_scales[fi % DECOMPRESSION_SCALE_LENGTH]; #endif - ((ACCUMULATOR_TYPE*)(&acc[bi]))[fi] += convert_half(((int *)(&acc_tmp[fi]))[bi]) * ds * de_quantize_scale[bi]; + #if COMPRESSED_WEIGHTS_INT8 + ACCUM_DQ_TYPE modified_calc_buff = ((int *)(&acc_tmp[fi]))[bi] - ((float)(wei_zp[fi]) * (convert_float)(activation_sum[bi])); + ((ACCUMULATOR_TYPE*)(&acc[bi]))[fi] += (convert_half)(convert_float(modified_calc_buff) * (float)ds * (float)de_quantize_scale[bi]); + #else + ((ACCUMULATOR_TYPE*)(&acc[bi]))[fi] += convert_half(((int *)(&acc_tmp[fi]))[bi]) * ds * de_quantize_scale[bi]; + #endif acc_tmp[fi][bi] = 0; } } @@ -1060,7 +1149,12 @@ inline void FUNC(fc_bf_tiled_kernel_dyn_quan)( ACCUMULATOR_TYPE ds = d_scales[fi % DECOMPRESSION_SCALE_LENGTH]; #endif - ((ACCUMULATOR_TYPE*)(&acc[bi]))[fi] += convert_half(((int *)(&acc_tmp[fi]))[bi]) * ds * de_quantize_scale[bi]; + #if COMPRESSED_WEIGHTS_INT8 + ACCUM_DQ_TYPE modified_calc_buff = ((int *)(&acc_tmp[fi]))[bi] - ((float)(wei_zp[fi]) * (convert_float)(activation_sum[bi])); + ((ACCUMULATOR_TYPE*)(&acc[bi]))[fi] += (convert_half)(convert_float(modified_calc_buff) * (float)ds * (float)de_quantize_scale[bi]); + #else + ((ACCUMULATOR_TYPE*)(&acc[bi]))[fi] += convert_half(((int *)(&acc_tmp[fi]))[bi]) * ds * de_quantize_scale[bi]; + #endif acc_tmp[fi][bi] = 0; } } @@ -1169,13 +1263,13 @@ KERNEL(fc)( , FUSED_OPS_DECLS #endif #if DYNAMIC_QUANTIZE - , __global char* quantized_input - , __global INPUT0_TYPE* de_quan_scale + , __global DQ_TYPE* quantized_input + , __global INPUT0_TYPE* quan_var #endif ) { #if USE_SLM #if DYNAMIC_QUANTIZE - __local int dq_wei_local_mem[SIMD * TILE_OFM * SIMD]; + __local uint dq_wei_local_mem[SIMD * TILE_OFM * SIMD]; #else __local ACCUMULATOR_TYPE wei_local_mem[TILE_IFM * SIMD * TILE_OFM * SIMD]; #endif @@ -1321,7 +1415,7 @@ KERNEL(fc)( OPTIONAL_SHAPE_INFO_TENSOR input, quantized_input, - de_quan_scale, + quan_var, #if DECOMPRESSION_SCALE_TERM decompression_scale, #endif @@ -1368,7 +1462,7 @@ KERNEL(fc)( OPTIONAL_SHAPE_INFO_TENSOR input, quantized_input, - de_quan_scale, + quan_var, #if DECOMPRESSION_SCALE_TERM decompression_scale, #endif diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/int4_utils.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/int4_utils.cl index 68d778475f5601..99ff124e3a39f9 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/int4_utils.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/include/batch_headers/int4_utils.cl @@ -59,6 +59,13 @@ inline char4 unpack_to_char(uint4x4_t v) __attribute__((overloadable)) { return (char4)(v0.s0, v0.s1, v1.s0, v1.s1); } +inline uchar4 unpack_to_uchar(uint4x4_t v) __attribute__((overloadable)) { + uchar2 v0 = unpack_to_uchar(v.s0); + uchar2 v1 = unpack_to_uchar(v.s1); + return (uchar4)(v0.s0, v0.s1, v1.s0, v1.s1); +} + + inline char4 unpack_transposed_to_char(int4x4_t v) __attribute__((overloadable)) { char2 v0 = unpack_to_char(v.s0); char2 v1 = unpack_to_char(v.s1); diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/pa_kv_cache_update_ref.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/pa_kv_cache_update_ref.cl index ef2f78496b2cf2..8426baf719f990 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/pa_kv_cache_update_ref.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/pa_kv_cache_update_ref.cl @@ -34,10 +34,14 @@ KERNEL(pa_kv_cache_update)( const uint seq_block_idx = block_indices_begins[seq_idx] + seq_len / PAGED_ATTENTION_BLOCK_SIZE; const uint block_idx = block_indices[seq_block_idx]; - uint key_value_in_offset = seq_idx * KV_HEADS_NUM * HEAD_SIZE + head_idx * HEAD_SIZE; + uint key_in_offset = INPUT0_OFFSET + + seq_idx * (KV_HEADS_NUM * HEAD_SIZE + INPUT0_PAD_BEFORE_FEATURE_NUM + INPUT0_PAD_AFTER_FEATURE_NUM) + + head_idx * HEAD_SIZE; + uint value_in_offset = INPUT1_OFFSET + + seq_idx * (KV_HEADS_NUM * HEAD_SIZE + INPUT1_PAD_BEFORE_FEATURE_NUM + INPUT1_PAD_AFTER_FEATURE_NUM) + + head_idx * HEAD_SIZE; uint key_out_offset = block_idx * KV_HEADS_NUM * HEAD_SIZE * PAGED_ATTENTION_BLOCK_SIZE + head_idx * HEAD_SIZE * PAGED_ATTENTION_BLOCK_SIZE + current_token_pos_in_block; - uint value_out_offset = block_idx * KV_HEADS_NUM * HEAD_SIZE * PAGED_ATTENTION_BLOCK_SIZE + head_idx * HEAD_SIZE * PAGED_ATTENTION_BLOCK_SIZE + current_token_pos_in_block * HEAD_SIZE; #define READ_BLOCK_SIZE GENERATE_STAGE_BLOCK_SIZE @@ -45,7 +49,7 @@ KERNEL(pa_kv_cache_update)( #define BLOCK_READ(ptr, offset) BLOCK_READN(INPUT0_TYPE, READ_BLOCK_SIZE, ptr, offset); #define DATA_VEC MAKE_VECTOR_TYPE(INPUT0_TYPE, READ_BLOCK_SIZE) - DATA_VEC input_data = BLOCK_READ(key_data, key_value_in_offset + head_idx_index); + DATA_VEC input_data = BLOCK_READ(key_data, key_in_offset + head_idx_index); unroll_for (uint i = 0; i < READ_BLOCK_SIZE; i++) { uint key_offset = key_out_offset + (head_idx_index + sglid + SUBGROUP_SIZE * i) * PAGED_ATTENTION_BLOCK_SIZE; @@ -56,7 +60,7 @@ KERNEL(pa_kv_cache_update)( #endif } - input_data = BLOCK_READ(value_data, key_value_in_offset + head_idx_index); + input_data = BLOCK_READ(value_data, value_in_offset + head_idx_index); unroll_for (uint i = 0; i < READ_BLOCK_SIZE; i++) { uint value_offset = value_out_offset + head_idx_index + sglid + SUBGROUP_SIZE * i; @@ -83,8 +87,13 @@ KERNEL(pa_kv_cache_update)( const uint token_start_pos = (past_len + block_start_pos - subsequence_begin_idx) % PAGED_ATTENTION_BLOCK_SIZE; - uint key_value_in_offset = block_start_pos * KV_HEADS_NUM * HEAD_SIZE + - head_idx * HEAD_SIZE; + uint key_in_offset = INPUT0_OFFSET + + block_start_pos * (KV_HEADS_NUM * HEAD_SIZE + INPUT0_PAD_AFTER_FEATURE_NUM + INPUT0_PAD_BEFORE_FEATURE_NUM) + + head_idx * HEAD_SIZE; + + uint value_in_offset = INPUT1_OFFSET + + block_start_pos * (KV_HEADS_NUM * HEAD_SIZE + INPUT1_PAD_AFTER_FEATURE_NUM + INPUT1_PAD_BEFORE_FEATURE_NUM) + + head_idx * HEAD_SIZE; const uint current_block_idx = (past_len + block_start_pos - subsequence_begin_idx) / PAGED_ATTENTION_BLOCK_SIZE; @@ -106,14 +115,14 @@ KERNEL(pa_kv_cache_update)( #define BLOCK_READ(ptr, offset) BLOCK_READN(INPUT0_TYPE, READ_BLOCK_SIZE, ptr, offset); #define DATA_VEC MAKE_VECTOR_TYPE(INPUT0_TYPE, READ_BLOCK_SIZE) - DATA_VEC input_data = BLOCK_READ(key_data, key_value_in_offset + head_idx_index); + DATA_VEC input_data = BLOCK_READ(key_data, key_in_offset + head_idx_index); unroll_for (uint i = 0; i < READ_BLOCK_SIZE; i++) { uint key_offset = key_out_offset + (head_idx_index + sglid + SUBGROUP_SIZE * i) * PAGED_ATTENTION_BLOCK_SIZE; key_cache_data[key_offset] = input_data[i]; } - input_data = BLOCK_READ(value_data, key_value_in_offset + head_idx_index); + input_data = BLOCK_READ(value_data, value_in_offset + head_idx_index); unroll_for (uint i = 0; i < READ_BLOCK_SIZE; i++) { uint value_offset = value_out_offset + head_idx_index + sglid + SUBGROUP_SIZE * i; @@ -126,14 +135,14 @@ KERNEL(pa_kv_cache_update)( #define BLOCK_READ(ptr, offset) BLOCK_READN(INPUT0_TYPE, READ_BLOCK_SIZE, ptr, offset); #define DATA_VEC MAKE_VECTOR_TYPE(INPUT0_TYPE, READ_BLOCK_SIZE) - DATA_VEC input_data = BLOCK_READ(key_data, key_value_in_offset + head_idx_index); + DATA_VEC input_data = BLOCK_READ(key_data, key_in_offset + head_idx_index); unroll_for (uint i = 0; i < READ_BLOCK_SIZE; i++) { uint key_offset = key_out_offset + (head_idx_index + sglid + SUBGROUP_SIZE * i) * PAGED_ATTENTION_BLOCK_SIZE; key_cache_data[key_offset] = input_data[i]; } - input_data = BLOCK_READ(value_data, key_value_in_offset + head_idx_index); + input_data = BLOCK_READ(value_data, value_in_offset + head_idx_index); unroll_for (uint i = 0; i < READ_BLOCK_SIZE; i++) { uint value_offset = value_out_offset + head_idx_index + sglid + SUBGROUP_SIZE * i; @@ -146,14 +155,14 @@ KERNEL(pa_kv_cache_update)( #define BLOCK_READ(ptr, offset) BLOCK_READN(INPUT0_TYPE, READ_BLOCK_SIZE, ptr, offset); #define DATA_VEC MAKE_VECTOR_TYPE(INPUT0_TYPE, READ_BLOCK_SIZE) - DATA_VEC input_data = BLOCK_READ(key_data, key_value_in_offset + head_idx_index); + DATA_VEC input_data = BLOCK_READ(key_data, key_in_offset + head_idx_index); unroll_for (uint i = 0; i < READ_BLOCK_SIZE; i++) { uint key_offset = key_out_offset + (head_idx_index + sglid + SUBGROUP_SIZE * i) * PAGED_ATTENTION_BLOCK_SIZE; key_cache_data[key_offset] = input_data[i]; } - input_data = BLOCK_READ(value_data, key_value_in_offset + head_idx_index); + input_data = BLOCK_READ(value_data, value_in_offset + head_idx_index); unroll_for (uint i = 0; i < READ_BLOCK_SIZE; i++) { uint value_offset = value_out_offset + head_idx_index + sglid + SUBGROUP_SIZE * i; @@ -166,14 +175,14 @@ KERNEL(pa_kv_cache_update)( #define BLOCK_READ(ptr, offset) BLOCK_READN(INPUT0_TYPE, READ_BLOCK_SIZE, ptr, offset); #define DATA_VEC MAKE_VECTOR_TYPE(INPUT0_TYPE, READ_BLOCK_SIZE) - DATA_VEC input_data = BLOCK_READ(key_data, key_value_in_offset + head_idx_index); + DATA_VEC input_data = BLOCK_READ(key_data, key_in_offset + head_idx_index); unroll_for (uint i = 0; i < READ_BLOCK_SIZE; i++) { uint key_offset = key_out_offset + (head_idx_index + sglid + SUBGROUP_SIZE * i) * PAGED_ATTENTION_BLOCK_SIZE; key_cache_data[key_offset] = input_data; } - input_data = BLOCK_READ(value_data, key_value_in_offset + head_idx_index); + input_data = BLOCK_READ(value_data, value_in_offset + head_idx_index); unroll_for (uint i = 0; i < READ_BLOCK_SIZE; i++) { uint value_offset = value_out_offset + head_idx_index + sglid + SUBGROUP_SIZE * i; @@ -181,7 +190,8 @@ KERNEL(pa_kv_cache_update)( } } - key_value_in_offset += KV_HEADS_NUM * HEAD_SIZE; + key_in_offset += (KV_HEADS_NUM * HEAD_SIZE + INPUT0_PAD_AFTER_FEATURE_NUM + INPUT0_PAD_BEFORE_FEATURE_NUM); + value_in_offset += (KV_HEADS_NUM * HEAD_SIZE + INPUT1_PAD_AFTER_FEATURE_NUM + INPUT1_PAD_BEFORE_FEATURE_NUM); key_out_offset += 1; value_out_offset += HEAD_SIZE; } @@ -194,14 +204,14 @@ KERNEL(pa_kv_cache_update)( #define BLOCK_READ(ptr, offset) BLOCK_READN(INPUT0_TYPE, READ_BLOCK_SIZE, ptr, offset); #define DATA_VEC MAKE_VECTOR_TYPE(INPUT0_TYPE, READ_BLOCK_SIZE) - DATA_VEC input_data = BLOCK_READ(key_data, key_value_in_offset + head_idx_index); + DATA_VEC input_data = BLOCK_READ(key_data, key_in_offset + head_idx_index); unroll_for (uint i = 0; i < READ_BLOCK_SIZE; i++) { uint key_offset = key_out_offset + (head_idx_index + sglid + SUBGROUP_SIZE * i) * PAGED_ATTENTION_BLOCK_SIZE; key_cache_data[key_offset] = input_data; } - input_data = BLOCK_READ(value_data, key_value_in_offset + head_idx_index); + input_data = BLOCK_READ(value_data, value_in_offset + head_idx_index); unroll_for (uint i = 0; i < READ_BLOCK_SIZE; i++) { uint value_offset = value_out_offset + head_idx_index + sglid + SUBGROUP_SIZE * i; @@ -209,7 +219,8 @@ KERNEL(pa_kv_cache_update)( } } - key_value_in_offset += KV_HEADS_NUM * HEAD_SIZE; + key_in_offset += (KV_HEADS_NUM * HEAD_SIZE + INPUT0_PAD_AFTER_FEATURE_NUM + INPUT0_PAD_BEFORE_FEATURE_NUM); + value_in_offset += (KV_HEADS_NUM * HEAD_SIZE + INPUT1_PAD_AFTER_FEATURE_NUM + INPUT1_PAD_BEFORE_FEATURE_NUM); key_out_offset += 1; value_out_offset += HEAD_SIZE; } diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/rope_ref.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/rope_ref.cl index 38066b4461def4..133440a21301f2 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/rope_ref.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/rope_ref.cl @@ -28,14 +28,11 @@ KERNEL(rope_ref)( uint r = rf < HALF_ROTARY_NDIMS ? rf * 2 : 0; uint f = rf < HEAD_SIZE - ROTARY_NDIMS ? rf * 2 : 0; -#ifdef ENABLE_SLICE - uint input_idx = GET_DATA_INDEX(SLICED_INPUT0, p, b, h * HEAD_SIZE, 0); - - input_idx += SLICED_FROM_START * (p * INPUT0_FEATURE_NUM + b + 1) - + SLICED_FROM_END * (p * INPUT0_FEATURE_NUM + b); -#else uint input_idx = INPUT0_GET_INDEX(p, b, h * HEAD_SIZE, 0); +#ifdef ENABLE_SLICE + input_idx += SLICED_FROM_START; #endif + uint cos_sin_p = p < INPUT1_BATCH_NUM ? p : 0; uint cos_sin_b = b < INPUT1_FEATURE_NUM ? b : 0; uint cos_sin_idx = INPUT1_GET_INDEX(cos_sin_p, cos_sin_b, 0, 0); @@ -69,14 +66,11 @@ KERNEL(rope_ref)( const uint h = (uint)get_global_id(2) / HALF_ROTARY_NDIMS; const uint r = (uint)get_global_id(2) % HALF_ROTARY_NDIMS; -#ifdef ENABLE_SLICE - uint input_idx = GET_DATA_INDEX(SLICED_INPUT0, b, p, h * HEAD_SIZE, 0); - - input_idx += SLICED_FROM_START * (b * INPUT0_FEATURE_NUM + p + 1) - + SLICED_FROM_END * (b * INPUT0_FEATURE_NUM + p); -#else uint input_idx = INPUT0_GET_INDEX(b, p, h * HEAD_SIZE, 0); +#ifdef ENABLE_SLICE + input_idx += SLICED_FROM_START; #endif + uint cos_sin_b = b < INPUT1_BATCH_NUM ? b : 0; uint cos_sin_p = p + INPUT1_FEATURE_NUM - INPUT0_FEATURE_NUM < INPUT1_FEATURE_NUM ? p + INPUT1_FEATURE_NUM - INPUT0_FEATURE_NUM : 0; uint cos_sin_h = h < INPUT1_SIZE_Y ? h : 0; @@ -119,15 +113,13 @@ KERNEL(rope_ref)( const uint p = (uint)get_global_id(2) / HALF_ROTARY_NDIMS; const uint r = (uint)get_global_id(2) % HALF_ROTARY_NDIMS; -#ifdef ENABLE_SLICE - uint input_idx = GET_DATA_INDEX(SLICED_INPUT0, b, h, p, 0); - - input_idx += SLICED_FROM_START * (b * INPUT0_FEATURE_NUM + h + 1) - + SLICED_FROM_END * (b * INPUT0_FEATURE_NUM + h); -#elif ENABLE_TRANSPOSE - uint input_idx = GET_DATA_INDEX(TRANSPOSED_INPUT0, b, h, p, 0); +#if ENABLE_TRANSPOSE + uint input_idx = INPUT0_GET_INDEX(b, p, h, 0); #else uint input_idx = INPUT0_GET_INDEX(b, h, p, 0); +#ifdef ENABLE_SLICE + input_idx += SLICED_FROM_START; +#endif #endif uint cos_sin_b = b < INPUT1_BATCH_NUM ? b : 0; diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_opt.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_opt.cl index 748f79115262e0..55f87e4189d9fe 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_opt.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_opt.cl @@ -118,12 +118,21 @@ inline uint FUNC(get_bt_index_value)(OPTIONAL_SHAPE_INFO_ARG uint b, uint f, uin #define VALUE_BLOCK_READ(ptr, offset) BLOCK_READN(INPUT2_TYPE, 1, ptr, offset) #define SUBGROUPS_PER_WG (HEAD_SIZE * SG_SCALE_FACTOR / SUBGROUP_SIZE) +#if IS_KV_COMPRESSED +#if COMPRESSED_PER_HEAD + #define GET_COMPRESSION_INDEX(INPUT, b, f, y, x) GET_DATA_INDEX(INPUT, (b), (f), (y), (0)); +#else + #define GET_COMPRESSION_INDEX(INPUT, b, f, y, x) GET_DATA_INDEX(INPUT, (b), (0), (y), (0)); +#endif +#endif + #ifdef SDPA_STAGE_0 #if TARGET_SEQ_LEN_BLOCK_SIZE == 1 /* This version is used for 2nd token */ REQD_SUB_GROUP_SIZE(SUBGROUP_SIZE) +__attribute__((reqd_work_group_size(1, 1, HEAD_SIZE * SG_SCALE_FACTOR))) KERNEL(sdpa_opt)( OPTIONAL_SHAPE_INFO_ARG const __global INPUT0_TYPE* query_input, @@ -136,6 +145,10 @@ KERNEL(sdpa_opt)( const __global INPUT4_TYPE* scale, #endif __global OUTPUT_TYPE* output, +#if IS_KV_COMPRESSED + const __global KEY_COMPRESSION_SCALE_TYPE* key_scale, + const __global VALUE_COMPRESSION_SCALE_TYPE* val_scale, +#endif #ifdef BEAM_TABLE_TYPE const __global BEAM_TABLE_TYPE* beam_table, #endif @@ -149,7 +162,18 @@ KERNEL(sdpa_opt)( const uint b1_idx = batch_idx % NUM_HEADS; /* HEADS_NUM dim */ const uint target_seq_idx = get_global_id(1); const uint lid = get_local_id(2); + +#if SG_SCALE_FACTOR == 2 + const uint head_size_idx = lid % HEAD_SIZE; +#elif SG_SCALE_FACTOR == 1 const uint head_size_idx = lid; +#else + #error "sdpa_opt.cl: Unsupported scale factor" +#endif + +#if SUBGROUPS_PER_WG > SUBGROUP_SIZE + #error "sdpa_opt.cl: Number of subgroups per work group should be less than subgroup_size +#endif const uint sgid = get_sub_group_id(); const uint sglid = get_sub_group_local_id(); @@ -166,7 +190,7 @@ KERNEL(sdpa_opt)( // SLM for query inputs __local INPUT0_TYPE query_local[HEAD_SIZE * TARGET_SEQ_LEN_BLOCK_SIZE]; // SLM for intermediate QK results - __local OUTPUT_TYPE qk_local[SEQ_LEN_PARTITION_SIZE * TARGET_SEQ_LEN_BLOCK_SIZE]; + __local SOFTMAX_ACCUMULATOR_TYPE qk_local[SEQ_LEN_PARTITION_SIZE * TARGET_SEQ_LEN_BLOCK_SIZE]; // SLM buffers for SoftMax calculation and qk_max/qk_sums results aggregation across all WG __local SOFTMAX_ACCUMULATOR_TYPE qk_max_vals[SUBGROUPS_PER_WG * TARGET_SEQ_LEN_BLOCK_SIZE]; __local SOFTMAX_ACCUMULATOR_TYPE qk_sum_vals[SUBGROUPS_PER_WG * TARGET_SEQ_LEN_BLOCK_SIZE]; @@ -199,13 +223,19 @@ KERNEL(sdpa_opt)( uint query_offset = INPUT0_GET_INDEX(b0_idx, b1_idx, target_seq_idx, (sgid * SUBGROUP_SIZE)); const uint query_pitch = QUERY_STEP_LOCAL; #endif - for (uint seq_idx = 0; seq_idx < seq_idx_end; seq_idx++) { - #define QUERY_BLOCK_SIZE 1 +#if SG_SCALE_FACTOR == 2 + if (sgid < HEAD_SIZE / SUBGROUP_SIZE) { +#else + { +#endif + for (uint seq_idx = 0; seq_idx < seq_idx_end; seq_idx++) { + #define QUERY_BLOCK_SIZE 1 - INPUT0_TYPE val = BLOCK_READN(INPUT0_TYPE, QUERY_BLOCK_SIZE, query_input, query_offset); - query_local[query_local_offset] = val * scale_val; - query_local_offset += QUERY_STEP_LOCAL; - query_offset += query_pitch; + INPUT0_TYPE val = BLOCK_READN(INPUT0_TYPE, QUERY_BLOCK_SIZE, query_input, query_offset); + query_local[query_local_offset] = val * scale_val; + query_local_offset += QUERY_STEP_LOCAL; + query_offset += query_pitch; + } } #undef QUERY_BLOCK_SIZE #undef QUERY_STEP @@ -216,28 +246,45 @@ KERNEL(sdpa_opt)( // Main Gemm1 calculation loop // Each SG performs element-wise multiplications of Q[HEAD_SIZE]xK[HEAD_SIZE] values // HEAD_SIZE / SUBGROUPS_PER_WG times in the loop and saves the result to the qk_local SLM buffer - for (uint seq_len = sgid; seq_len < partition_seq_len; seq_len += (HEAD_SIZE / SUBGROUP_SIZE)) { -#ifdef INPUT1_DIMS_ORDER + for (uint seq_len = sgid; seq_len < partition_seq_len; seq_len += (HEAD_SIZE / SUBGROUP_SIZE) * SG_SCALE_FACTOR) { #ifdef BEAM_TABLE_TYPE const uint b_idx = beam_table[FUNC_CALL(get_bt_index_key)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, start_partition_idx + seq_len, 0)]; #else const uint b_idx = b0_idx; #endif - const uint key_offset = FUNC_CALL(get_input1_index)(OPTIONAL_SHAPE_INFO_TENSOR b_idx, b1_idx, 0, 0, start_partition_idx + seq_len, 0); + +#ifdef INPUT1_DIMS_ORDER + uint key_offset = FUNC_CALL(get_input1_index)(OPTIONAL_SHAPE_INFO_TENSOR b_idx, b1_idx, 0, 0, start_partition_idx + seq_len, 0); #else - const uint key_offset = INPUT1_GET_INDEX(b0_idx, b1_idx, start_partition_idx + seq_len, 0); + uint key_offset = INPUT1_GET_INDEX(b_idx, b1_idx, start_partition_idx + seq_len, 0); #endif - INPUT0_TYPE acc[TARGET_SEQ_LEN_BLOCK_SIZE] = {INPUT0_VAL_ZERO}; + SOFTMAX_ACCUMULATOR_TYPE acc[TARGET_SEQ_LEN_BLOCK_SIZE] = {SOFTMAX_ACCUMULATOR_VAL_ZERO}; +#if IS_KV_COMPRESSED + const uint comp_offset = GET_COMPRESSION_INDEX(KEY_COMPRESSION_SCALE, b_idx, b1_idx / BROADCAST_GROUP_SIZE, start_partition_idx + seq_len, 0); + KEY_COMPRESSION_SCALE_TYPE comp_scale = key_scale[comp_offset]; +#if USE_ASYMMETRIC_QUANTIZATION + KEY_COMPRESSION_SCALE_TYPE comp_zp = key_scale[comp_offset + 1]; +#endif +#endif uint head_idx_index = 0; #define KEY_BLOCK_SIZE 8 for (; head_idx_index + (KEY_BLOCK_SIZE * SUBGROUP_SIZE) <= HEAD_SIZE; head_idx_index += SUBGROUP_SIZE * KEY_BLOCK_SIZE) { #define KEY_BLOCK_READ(ptr, offset) BLOCK_READN(INPUT1_TYPE, KEY_BLOCK_SIZE, ptr, offset); #define KEY_BLOCK MAKE_VECTOR_TYPE(INPUT1_TYPE, KEY_BLOCK_SIZE) + #define KEY_BLOCK_UNCOMPRESSED MAKE_VECTOR_TYPE(KEY_COMPRESSION_SCALE_TYPE, KEY_BLOCK_SIZE) + #define TO_KEY_BLOCK_UNCOMPRESSED_TYPE(val) CAT(convert_, KEY_BLOCK_UNCOMPRESSED)(val) #define QUERY_BLOCK MAKE_VECTOR_TYPE(INPUT0_TYPE, KEY_BLOCK_SIZE) - KEY_BLOCK key_vals = KEY_BLOCK_READ(key_input, key_offset + head_idx_index); + KEY_BLOCK key_vec_packed = KEY_BLOCK_READ(key_input, key_offset + head_idx_index); +#if IS_KV_COMPRESSED && USE_ASYMMETRIC_QUANTIZATION + KEY_BLOCK_UNCOMPRESSED key_vals = (TO_KEY_BLOCK_UNCOMPRESSED_TYPE(key_vec_packed) - comp_zp) * comp_scale; +#elif IS_KV_COMPRESSED + KEY_BLOCK_UNCOMPRESSED key_vals = (TO_KEY_BLOCK_UNCOMPRESSED_TYPE(key_vec_packed)) * comp_scale; +#else + KEY_BLOCK key_vals = key_vec_packed; +#endif uint query_offset = head_idx_index + sglid; unroll_for (uint seq_idx = 0; seq_idx < TARGET_SEQ_LEN_BLOCK_SIZE; seq_idx++) { @@ -247,7 +294,7 @@ KERNEL(sdpa_opt)( } unroll_for(uint i = 0; i < KEY_BLOCK_SIZE; i++) { - acc[seq_idx] = mad(query_vals_reg[i], key_vals[i], acc[seq_idx]); + acc[seq_idx] = mad(TO_SOFTMAX_ACCUMULATOR_TYPE(query_vals_reg[i]), TO_SOFTMAX_ACCUMULATOR_TYPE(key_vals[i]), acc[seq_idx]); } query_offset += HEAD_SIZE; @@ -258,9 +305,18 @@ KERNEL(sdpa_opt)( for (; head_idx_index + (KEY_BLOCK_SIZE * SUBGROUP_SIZE) <= HEAD_SIZE; head_idx_index += SUBGROUP_SIZE * KEY_BLOCK_SIZE) { #define KEY_BLOCK_READ(ptr, offset) BLOCK_READN(INPUT1_TYPE, KEY_BLOCK_SIZE, ptr, offset); #define KEY_BLOCK MAKE_VECTOR_TYPE(INPUT1_TYPE, KEY_BLOCK_SIZE) + #define KEY_BLOCK_UNCOMPRESSED MAKE_VECTOR_TYPE(KEY_COMPRESSION_SCALE_TYPE, KEY_BLOCK_SIZE) + #define TO_KEY_BLOCK_UNCOMPRESSED_TYPE(val) CAT(convert_, KEY_BLOCK_UNCOMPRESSED)(val) #define QUERY_BLOCK MAKE_VECTOR_TYPE(INPUT0_TYPE, KEY_BLOCK_SIZE) - KEY_BLOCK key_vals = KEY_BLOCK_READ(key_input, key_offset + head_idx_index); + KEY_BLOCK key_vec_packed = KEY_BLOCK_READ(key_input, key_offset + head_idx_index); +#if IS_KV_COMPRESSED && USE_ASYMMETRIC_QUANTIZATION + KEY_BLOCK_UNCOMPRESSED key_vals = (TO_KEY_BLOCK_UNCOMPRESSED_TYPE(key_vec_packed) - comp_zp) * comp_scale; +#elif IS_KV_COMPRESSED + KEY_BLOCK_UNCOMPRESSED key_vals = (TO_KEY_BLOCK_UNCOMPRESSED_TYPE(key_vec_packed)) * comp_scale; +#else + KEY_BLOCK key_vals = key_vec_packed; +#endif uint query_offset = head_idx_index + sglid; unroll_for (uint seq_idx = 0; seq_idx < TARGET_SEQ_LEN_BLOCK_SIZE; seq_idx++) { @@ -270,7 +326,7 @@ KERNEL(sdpa_opt)( } unroll_for(uint i = 0; i < KEY_BLOCK_SIZE; i++) { - acc[seq_idx] = mad(query_vals_reg[i], key_vals[i], acc[seq_idx]); + acc[seq_idx] = mad(TO_SOFTMAX_ACCUMULATOR_TYPE(query_vals_reg[i]), TO_SOFTMAX_ACCUMULATOR_TYPE(key_vals[i]), acc[seq_idx]); } query_offset += HEAD_SIZE; @@ -281,9 +337,18 @@ KERNEL(sdpa_opt)( for (; head_idx_index + (KEY_BLOCK_SIZE * SUBGROUP_SIZE) <= HEAD_SIZE; head_idx_index += SUBGROUP_SIZE * KEY_BLOCK_SIZE) { #define KEY_BLOCK_READ(ptr, offset) BLOCK_READN(INPUT1_TYPE, KEY_BLOCK_SIZE, ptr, offset); #define KEY_BLOCK MAKE_VECTOR_TYPE(INPUT1_TYPE, KEY_BLOCK_SIZE) + #define KEY_BLOCK_UNCOMPRESSED MAKE_VECTOR_TYPE(KEY_COMPRESSION_SCALE_TYPE, KEY_BLOCK_SIZE) + #define TO_KEY_BLOCK_UNCOMPRESSED_TYPE(val) CAT(convert_, KEY_BLOCK_UNCOMPRESSED)(val) #define QUERY_BLOCK MAKE_VECTOR_TYPE(INPUT0_TYPE, KEY_BLOCK_SIZE) - KEY_BLOCK key_vals = KEY_BLOCK_READ(key_input, key_offset + head_idx_index); + KEY_BLOCK key_vec_packed = KEY_BLOCK_READ(key_input, key_offset + head_idx_index); +#if IS_KV_COMPRESSED && USE_ASYMMETRIC_QUANTIZATION + KEY_BLOCK_UNCOMPRESSED key_vals = (TO_KEY_BLOCK_UNCOMPRESSED_TYPE(key_vec_packed) - comp_zp) * comp_scale; +#elif IS_KV_COMPRESSED + KEY_BLOCK_UNCOMPRESSED key_vals = (TO_KEY_BLOCK_UNCOMPRESSED_TYPE(key_vec_packed)) * comp_scale; +#else + KEY_BLOCK key_vals = key_vec_packed; +#endif uint query_offset = head_idx_index + sglid; unroll_for (uint seq_idx = 0; seq_idx < TARGET_SEQ_LEN_BLOCK_SIZE; seq_idx++) { @@ -293,7 +358,7 @@ KERNEL(sdpa_opt)( } unroll_for(uint i = 0; i < KEY_BLOCK_SIZE; i++) { - acc[seq_idx] = mad(query_vals_reg[i], key_vals[i], acc[seq_idx]); + acc[seq_idx] = mad(TO_SOFTMAX_ACCUMULATOR_TYPE(query_vals_reg[i]), TO_SOFTMAX_ACCUMULATOR_TYPE(key_vals[i]), acc[seq_idx]); } query_offset += HEAD_SIZE; @@ -304,9 +369,18 @@ KERNEL(sdpa_opt)( for (; head_idx_index + (KEY_BLOCK_SIZE * SUBGROUP_SIZE) <= HEAD_SIZE; head_idx_index += SUBGROUP_SIZE * KEY_BLOCK_SIZE) { #define KEY_BLOCK_READ(ptr, offset) BLOCK_READN(INPUT1_TYPE, KEY_BLOCK_SIZE, ptr, offset); #define KEY_BLOCK MAKE_VECTOR_TYPE(INPUT1_TYPE, KEY_BLOCK_SIZE) + #define KEY_BLOCK_UNCOMPRESSED MAKE_VECTOR_TYPE(KEY_COMPRESSION_SCALE_TYPE, KEY_BLOCK_SIZE) + #define TO_KEY_BLOCK_UNCOMPRESSED_TYPE(val) CAT(convert_, KEY_BLOCK_UNCOMPRESSED)(val) #define QUERY_BLOCK MAKE_VECTOR_TYPE(INPUT0_TYPE, KEY_BLOCK_SIZE) - KEY_BLOCK key_vals = KEY_BLOCK_READ(key_input, key_offset + head_idx_index); + KEY_BLOCK key_vec_packed = KEY_BLOCK_READ(key_input, key_offset + head_idx_index); +#if IS_KV_COMPRESSED && USE_ASYMMETRIC_QUANTIZATION + KEY_BLOCK_UNCOMPRESSED key_vals = (TO_KEY_BLOCK_UNCOMPRESSED_TYPE(key_vec_packed) - comp_zp) * comp_scale; +#elif IS_KV_COMPRESSED + KEY_BLOCK_UNCOMPRESSED key_vals = (TO_KEY_BLOCK_UNCOMPRESSED_TYPE(key_vec_packed)) * comp_scale; +#else + KEY_BLOCK key_vals = key_vec_packed; +#endif uint query_offset = head_idx_index + sglid; unroll_for (uint seq_idx = 0; seq_idx < TARGET_SEQ_LEN_BLOCK_SIZE; seq_idx++) { @@ -315,7 +389,7 @@ KERNEL(sdpa_opt)( query_vals_reg = query_local[query_offset + i * SUBGROUP_SIZE]; } - acc[seq_idx] = mad(query_vals_reg, key_vals, acc[seq_idx]); + acc[seq_idx] = mad(TO_SOFTMAX_ACCUMULATOR_TYPE(query_vals_reg), TO_SOFTMAX_ACCUMULATOR_TYPE(key_vals), acc[seq_idx]); query_offset += HEAD_SIZE; } } @@ -331,11 +405,11 @@ KERNEL(sdpa_opt)( // Wait until all SG finishes their calculations and apply scale and attention mask to the results barrier(CLK_LOCAL_MEM_FENCE); - INPUT0_TYPE qk_val[TARGET_SEQ_LEN_BLOCK_SIZE]; + SOFTMAX_ACCUMULATOR_TYPE qk_val[TARGET_SEQ_LEN_BLOCK_SIZE]; const uint seq_idx_end = 1; for (uint seq_idx = 0; seq_idx < seq_idx_end; seq_idx++) { // Iterate over all values QK values in SLM and apply scale and attention mask - for (uint seq_len = sgid * SUBGROUP_SIZE + sglid; seq_len < partition_seq_len; seq_len += (HEAD_SIZE)) { + for (uint seq_len = sgid * SUBGROUP_SIZE + sglid; seq_len < partition_seq_len; seq_len += (HEAD_SIZE * SG_SCALE_FACTOR)) { // Read value from SLM and apply scale qk_val[seq_idx] = qk_local[seq_idx * SEQ_LEN_PARTITION_SIZE + seq_len]; @@ -388,7 +462,7 @@ KERNEL(sdpa_opt)( SOFTMAX_ACCUMULATOR_TYPE exp_sum[TARGET_SEQ_LEN_BLOCK_SIZE] = {SOFTMAX_ACCUMULATOR_VAL_ZERO}; const uint qk_num_per_wi = CEIL_DIV(partition_seq_len, SUBGROUPS_PER_WG * SUBGROUP_SIZE); for (uint qk_idx = 0; qk_idx < qk_num_per_wi; qk_idx++) { - const uint local_data_idx = qk_idx * (SUBGROUPS_PER_WG * SUBGROUP_SIZE) + head_size_idx; + const uint local_data_idx = qk_idx * (SUBGROUPS_PER_WG * SUBGROUP_SIZE) + lid; if (local_data_idx < partition_seq_len) { for (uint seq_idx = 0; seq_idx < seq_idx_end; seq_idx++) { SOFTMAX_ACCUMULATOR_TYPE qk_new = native_exp(TO_SOFTMAX_ACCUMULATOR_TYPE(qk_local[seq_idx * SEQ_LEN_PARTITION_SIZE + local_data_idx]) - qk_max[seq_idx]); @@ -420,7 +494,7 @@ KERNEL(sdpa_opt)( // const SOFTMAX_ACCUMULATOR_TYPE inv_exp_sum = SOFTMAX_ACCUMULATOR_VAL_ONE / exp_sum[seq_idx]; for (uint qk_idx = 0; qk_idx < qk_num_per_wi; qk_idx++) { - const uint local_data_idx = qk_idx * (SUBGROUPS_PER_WG * SUBGROUP_SIZE) + sgid * SUBGROUP_SIZE + sglid; + const uint local_data_idx = qk_idx * (SUBGROUPS_PER_WG * SUBGROUP_SIZE) + lid; if (local_data_idx < partition_seq_len) { for (uint seq_idx = 0; seq_idx < seq_idx_end; seq_idx++) { SOFTMAX_ACCUMULATOR_TYPE qk_new = TO_SOFTMAX_ACCUMULATOR_TYPE(qk_local[seq_idx * SEQ_LEN_PARTITION_SIZE + local_data_idx]) / exp_sum[seq_idx]; @@ -434,7 +508,7 @@ KERNEL(sdpa_opt)( { // If the number of partitions is greater than 1, save exm_sums and max_logits to the temporary buffers // Use single WI in the WG, since all the WIs have the same value - if (num_of_partitions > 1 && head_size_idx == 0) { + if (num_of_partitions > 1 && lid == 0) { for (uint seq_idx = 0; seq_idx < seq_idx_end; seq_idx++) { const uint exp_sums_offset = b0_idx * (NUM_HEADS * TARGET_SEQ_LEN * num_of_partitions) + b1_idx * (TARGET_SEQ_LEN * num_of_partitions) + @@ -463,15 +537,32 @@ KERNEL(sdpa_opt)( #endif #endif - for (uint seq_len = 0; seq_len < partition_seq_len / SUBGROUP_SIZE; seq_len++) { +#if SG_SCALE_FACTOR > 1 + const uint seq_len_start = (sgid / (HEAD_SIZE / SUBGROUP_SIZE)) * (SEQ_LEN_PARTITION_SIZE / SG_SCALE_FACTOR / SUBGROUP_SIZE); + const uint seq_len_end = min(seq_len_start + (SEQ_LEN_PARTITION_SIZE / SG_SCALE_FACTOR / SUBGROUP_SIZE), partition_seq_len / SUBGROUP_SIZE); +#else + const uint seq_len_start = 0; + const uint seq_len_end = partition_seq_len / SUBGROUP_SIZE; +#endif + + for (uint seq_len = seq_len_start; seq_len < seq_len_end; seq_len++) { #ifdef BEAM_TABLE_TYPE - uint b_idx = beam_table[FUNC_CALL(get_bt_index_value)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, start_partition_idx + (seq_len * SUBGROUP_SIZE) + sglid, sgid * SUBGROUP_SIZE)]; + const uint b_idx = beam_table[FUNC_CALL(get_bt_index_value)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, start_partition_idx + (seq_len * SUBGROUP_SIZE) + sglid, sgid * SUBGROUP_SIZE)]; uint value_offset = FUNC_CALL(get_input2_index)(OPTIONAL_SHAPE_INFO_TENSOR b_idx, b1_idx, 0, 0, start_partition_idx + (seq_len * SUBGROUP_SIZE) + sglid, sgid * SUBGROUP_SIZE); #else + const uint b_idx = b0_idx; #ifdef INPUT2_DIMS_ORDER - uint value_offset = FUNC_CALL(get_input2_index)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, start_partition_idx + (seq_len * SUBGROUP_SIZE), head_size_idx); + uint value_offset = FUNC_CALL(get_input2_index)(OPTIONAL_SHAPE_INFO_TENSOR b_idx, b1_idx, 0, 0, start_partition_idx + (seq_len * SUBGROUP_SIZE), head_size_idx); #else - uint value_offset = INPUT2_GET_INDEX(b0_idx, b1_idx, start_partition_idx + (seq_len * SUBGROUP_SIZE), head_size_idx); + uint value_offset = INPUT2_GET_INDEX(b_idx, b1_idx, start_partition_idx + (seq_len * SUBGROUP_SIZE), head_size_idx); +#endif +#endif + +#if IS_KV_COMPRESSED + const uint comp_offset = GET_COMPRESSION_INDEX(VALUE_COMPRESSION_SCALE, b_idx, b1_idx / BROADCAST_GROUP_SIZE, start_partition_idx + (seq_len * SUBGROUP_SIZE) + sglid, 0); + VALUE_COMPRESSION_SCALE_TYPE comp_scale = val_scale[comp_offset]; +#if USE_ASYMMETRIC_QUANTIZATION + VALUE_COMPRESSION_SCALE_TYPE comp_zp = val_scale[comp_offset + 1]; #endif #endif @@ -482,9 +573,17 @@ KERNEL(sdpa_opt)( unroll_for (uint i = 0; i < SUBGROUP_SIZE; i++) { #ifdef BEAM_TABLE_TYPE - INPUT2_TYPE value_val = VALUE_BLOCK_READ(value_input, sub_group_broadcast(value_offset, i)); + const INPUT2_TYPE value_packed = VALUE_BLOCK_READ(value_input, sub_group_broadcast(value_offset, i)); #else - INPUT2_TYPE value_val = VALUE_BLOCK_READ(value_input, value_offset); + const INPUT2_TYPE value_packed = VALUE_BLOCK_READ(value_input, value_offset); +#endif + +#if IS_KV_COMPRESSED && USE_ASYMMETRIC_QUANTIZATION + VALUE_COMPRESSION_SCALE_TYPE value_val = (value_packed - sub_group_broadcast(comp_zp, i)) * sub_group_broadcast(comp_scale, i); +#elif IS_KV_COMPRESSED + VALUE_COMPRESSION_SCALE_TYPE value_val = (value_packed * sub_group_broadcast(comp_scale, i)); +#else + INPUT2_TYPE value_val = value_packed; #endif unroll_for (uint seq_idx = 0; seq_idx < TARGET_SEQ_LEN_BLOCK_SIZE; seq_idx++) { acc[seq_idx] = mad(sub_group_broadcast(qk_val[seq_idx], i), value_val, acc[seq_idx]); @@ -496,17 +595,30 @@ KERNEL(sdpa_opt)( } } - const uint seq_len_leftovers_start = (partition_seq_len / SUBGROUP_SIZE) * SUBGROUP_SIZE; - for (uint seq_len = seq_len_leftovers_start; seq_len < partition_seq_len; seq_len++) { -#ifdef INPUT2_DIMS_ORDER + +#if SG_SCALE_FACTOR > 1 + if (sgid >= HEAD_SIZE / SUBGROUP_SIZE) { +#endif + + for (uint seq_len = (partition_seq_len / SUBGROUP_SIZE) * SUBGROUP_SIZE; seq_len < partition_seq_len; seq_len++) { #ifdef BEAM_TABLE_TYPE const uint b_idx = beam_table[FUNC_CALL(get_bt_index_value)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, start_partition_idx + seq_len, head_size_idx)]; #else const uint b_idx = b0_idx; #endif + +#ifdef INPUT2_DIMS_ORDER const uint value_offset = FUNC_CALL(get_input2_index)(OPTIONAL_SHAPE_INFO_TENSOR b_idx, b1_idx, 0, 0, start_partition_idx + seq_len, head_size_idx); #else - const uint value_offset = INPUT2_GET_INDEX(b0_idx, b1_idx, start_partition_idx + seq_len, head_size_idx); + const uint value_offset = INPUT2_GET_INDEX(b_idx, b1_idx, start_partition_idx + seq_len, head_size_idx); +#endif + +#if IS_KV_COMPRESSED + const uint comp_offset = GET_COMPRESSION_INDEX(VALUE_COMPRESSION_SCALE, b_idx, b1_idx / BROADCAST_GROUP_SIZE, start_partition_idx + seq_len, 0); + VALUE_COMPRESSION_SCALE_TYPE comp_scale = val_scale[comp_offset]; +#if USE_ASYMMETRIC_QUANTIZATION + VALUE_COMPRESSION_SCALE_TYPE comp_zp = val_scale[comp_offset + 1]; +#endif #endif OUTPUT_TYPE qk_val[TARGET_SEQ_LEN_BLOCK_SIZE]; @@ -514,15 +626,42 @@ KERNEL(sdpa_opt)( qk_val[seq_idx] = qk_local[seq_idx * SEQ_LEN_PARTITION_SIZE + seq_len]; } - INPUT2_TYPE value_val = VALUE_BLOCK_READ(value_input, value_offset); + const INPUT2_TYPE value_packed = VALUE_BLOCK_READ(value_input, value_offset); +#if IS_KV_COMPRESSED && USE_ASYMMETRIC_QUANTIZATION + const VALUE_COMPRESSION_SCALE_TYPE value_val = (value_packed - comp_zp) * comp_scale; +#elif IS_KV_COMPRESSED + const VALUE_COMPRESSION_SCALE_TYPE value_val = (value_packed * comp_scale); +#else + const INPUT2_TYPE value_val = value_packed; +#endif unroll_for (uint seq_idx = 0; seq_idx < TARGET_SEQ_LEN_BLOCK_SIZE; seq_idx++) { acc[seq_idx] = mad(qk_val[seq_idx], value_val, acc[seq_idx]); } } +#if SG_SCALE_FACTOR > 1 + } // if (sgid >= HEAD_SIZE / SUBGROUP_SIZE) +#endif + +#if SG_SCALE_FACTOR > 1 + if ((partition_seq_len > (SEQ_LEN_PARTITION_SIZE / SG_SCALE_FACTOR)) || (partition_seq_len % SUBGROUP_SIZE != 0)) { + if (sgid >= HEAD_SIZE / SUBGROUP_SIZE) { + // Reuse query_local SLM to sum-up results between two groups of subgroups + query_local[head_size_idx] = acc[0]; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid < HEAD_SIZE / SUBGROUP_SIZE) { + acc[0] += query_local[head_size_idx]; + } + } +#endif + // If the number of partitions is greater than 1, save results to the temporary buffer; // otherwise, save results directly to the main output. +#if SG_SCALE_FACTOR > 1 + if (sgid < HEAD_SIZE / SUBGROUP_SIZE) { +#endif if (num_of_partitions > 1) { const uint seq_idx_end = 1; for (uint seq_idx = 0; seq_idx < seq_idx_end; seq_idx++) { @@ -542,6 +681,9 @@ KERNEL(sdpa_opt)( output[output_offset] = acc[seq_idx]; } } +#if SG_SCALE_FACTOR > 1 + } // if (sgid < HEAD_SIZE / SUBGROUP_SIZE) { +#endif } // Gemm2 calculation end } @@ -582,6 +724,12 @@ KERNEL(sdpa_opt)( #define ATTN_SCALE_BUFFER_ARG #endif +// Applying scales to query input improves the accuracy, but leads to performance drop for FP16 KV-cache case, +// so use it only for compressed version +#if IS_KV_COMPRESSED +#define APPLY_SCALES_TO_QUERY 1 +#endif + #define MASK_VECTOR_TYPE MAKE_VECTOR_TYPE(INPUT0_TYPE, TARGET_SEQ_LEN_BLOCK_SIZE) inline MASK_VECTOR_TYPE FUNC(load_attn_mask)(OPTIONAL_SHAPE_INFO_ARG @@ -683,6 +831,10 @@ KERNEL(sdpa_opt)( const __global ALIBI_TYPE* alibi_slopes, #endif __global OUTPUT_TYPE* output, +#if IS_KV_COMPRESSED + const __global KEY_COMPRESSION_SCALE_TYPE* key_scale, + const __global VALUE_COMPRESSION_SCALE_TYPE* val_scale, +#endif #ifdef BEAM_TABLE_TYPE const __global BEAM_TABLE_TYPE* beam_table, #endif @@ -733,9 +885,10 @@ KERNEL(sdpa_opt)( #if IS_PAGED_ATTENTION const uint block_start_pos = blocked_indexes_start[target_seq_dim]; const uint block_end_pos = blocked_indexes_end[target_seq_dim]; - - uint query_offset = block_start_pos * HEAD_SIZE * NUM_HEADS + num_heads_dim * HEAD_SIZE + head_size_idx; - const uint query_pitch = HEAD_SIZE * NUM_HEADS; + uint query_offset = INPUT0_OFFSET + + block_start_pos * (HEAD_SIZE * NUM_HEADS + INPUT0_PAD_BEFORE_FEATURE_NUM + INPUT0_PAD_AFTER_FEATURE_NUM) + + num_heads_dim * HEAD_SIZE + head_size_idx; + const uint query_pitch = (HEAD_SIZE * NUM_HEADS + INPUT0_PAD_BEFORE_FEATURE_NUM + INPUT0_PAD_AFTER_FEATURE_NUM); const uint cur_target_seq_len_size = block_end_pos - block_start_pos; #else @@ -751,12 +904,22 @@ KERNEL(sdpa_opt)( #endif uint query_local_offset = head_size_idx * TARGET_SEQ_LEN_BLOCK_SIZE; +#if APPLY_SCALES_TO_QUERY +#if HAS_SCALE_INPUT + const INPUT0_TYPE scale_val = *scale; +#else + const INPUT0_TYPE scale_val = TO_INPUT0_TYPE(STATIC_SCALE_VALUE); +#endif +#else + const INPUT0_TYPE scale_val = INPUT0_VAL_ONE; +#endif + if (cur_target_seq_len_size != TARGET_SEQ_LEN_BLOCK_SIZE) { if (sgid * SUBGROUP_SIZE < HEAD_SIZE) { for (uint seq_idx = 0; seq_idx < cur_target_seq_len_size; seq_idx++) { INPUT0_TYPE val = BLOCK_READN(INPUT0_TYPE, 1, query_input, query_offset); - slm_query[query_local_offset] = val; + slm_query[query_local_offset] = val * scale_val; query_offset += query_pitch; query_local_offset++; } @@ -767,7 +930,7 @@ KERNEL(sdpa_opt)( unroll_for (uint seq_idx = 0; seq_idx < (TARGET_SEQ_LEN_BLOCK_SIZE / SG_SCALE_FACTOR); seq_idx++) { INPUT0_TYPE val = BLOCK_READN(INPUT0_TYPE, 1, query_input, query_offset); - slm_query[query_local_offset] = val; + slm_query[query_local_offset] = val * scale_val; query_offset += query_pitch; query_local_offset++; } @@ -777,7 +940,7 @@ KERNEL(sdpa_opt)( unroll_for (uint seq_idx = 0; seq_idx < (TARGET_SEQ_LEN_BLOCK_SIZE / SG_SCALE_FACTOR); seq_idx++) { INPUT0_TYPE val = BLOCK_READN(INPUT0_TYPE, 1, query_input, query_offset); - slm_query[query_local_offset] = val; + slm_query[query_local_offset] = val * scale_val; query_offset += query_pitch; query_local_offset++; } @@ -788,7 +951,7 @@ KERNEL(sdpa_opt)( unroll_for (uint seq_idx = 0; seq_idx < (TARGET_SEQ_LEN_BLOCK_SIZE / SG_SCALE_FACTOR); seq_idx++) { INPUT0_TYPE val = BLOCK_READN(INPUT0_TYPE, 1, query_input, query_offset); - slm_query[query_local_offset] = val; + slm_query[query_local_offset] = val * scale_val; query_offset += query_pitch; query_local_offset++; } @@ -796,7 +959,7 @@ KERNEL(sdpa_opt)( unroll_for (uint seq_idx = 0; seq_idx < TARGET_SEQ_LEN_BLOCK_SIZE; seq_idx++) { INPUT0_TYPE val = BLOCK_READN(INPUT0_TYPE, 1, query_input, query_offset); - slm_query[query_local_offset] = val; + slm_query[query_local_offset] = val * scale_val; query_offset += query_pitch; query_local_offset++; } @@ -834,13 +997,17 @@ KERNEL(sdpa_opt)( const uint heads_dim = num_heads_dim; #endif #define KEY_SEQ_OFFSET subsequence_begins[gws_seq_indexes_correspondence[target_seq_dim]] - uint key_offset = KEY_SEQ_OFFSET * HEAD_SIZE * NUM_KV_HEADS + heads_dim * HEAD_SIZE + seq_len * HEAD_SIZE * NUM_KV_HEADS; - const uint key_pitch = HEAD_SIZE * NUM_KV_HEADS; + const uint key_pitch = (HEAD_SIZE * NUM_KV_HEADS + INPUT1_PAD_BEFORE_FEATURE_NUM + INPUT1_PAD_AFTER_FEATURE_NUM); + uint key_offset = INPUT1_OFFSET + + KEY_SEQ_OFFSET * key_pitch + + heads_dim * HEAD_SIZE + + seq_len * key_pitch; #else #ifdef BEAM_TABLE_TYPE const uint b_idx = beam_table[FUNC_CALL(get_bt_index_key)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, seq_len + sglid, 0)]; const uint key_offset = FUNC_CALL(get_input1_index)(OPTIONAL_SHAPE_INFO_TENSOR b_idx, b1_idx, 0, 0, seq_len + sglid, 0); #else + const uint b_idx = b0_idx; #ifdef INPUT1_DIMS_ORDER uint key_offset = FUNC_CALL(get_input1_index)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, seq_len, 0); uint key_offset_next_seq = FUNC_CALL(get_input1_index)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, seq_len + 1, 0); @@ -870,10 +1037,17 @@ KERNEL(sdpa_opt)( PA_BUFFERS); if (seq_len_calc_size >= SUBGROUP_SIZE) { +#if IS_KV_COMPRESSED + const uint comp_offset = GET_COMPRESSION_INDEX(KEY_COMPRESSION_SCALE, b_idx, b1_idx / BROADCAST_GROUP_SIZE, seq_len + sglid, 0); + KEY_COMPRESSION_SCALE_TYPE comp_scale = key_scale[comp_offset]; +#if USE_ASYMMETRIC_QUANTIZATION + KEY_COMPRESSION_SCALE_TYPE comp_zp = key_scale[comp_offset + 1]; +#endif +#endif __attribute__((opencl_unroll_hint(1))) for (uint head_idx_index = 0; head_idx_index < HEAD_SIZE; head_idx_index += SUBGROUP_SIZE) { #define KEY_BLOCK_READ(ptr, offset) BLOCK_READN(INPUT1_TYPE, 1, ptr, offset); - #define QUERY_VEC MAKE_VECTOR_TYPE(INPUT1_TYPE, TARGET_SEQ_LEN_BLOCK_SIZE) + #define QUERY_VEC MAKE_VECTOR_TYPE(INPUT0_TYPE, TARGET_SEQ_LEN_BLOCK_SIZE) QUERY_VEC queries_vec; uint query_local_offset = (head_idx_index * TARGET_SEQ_LEN_BLOCK_SIZE) + sglid; @@ -884,9 +1058,17 @@ KERNEL(sdpa_opt)( unroll_for (uint key_row_idx = 0; key_row_idx < TARGET_SEQ_LEN_BLOCK_SIZE; key_row_idx++) { #ifdef BEAM_TABLE_TYPE - INPUT1_TYPE key_vals = KEY_BLOCK_READ(key_input, sub_group_broadcast(key_offset, key_row_idx) + head_idx_index); + const INPUT1_TYPE key_packed = KEY_BLOCK_READ(key_input, sub_group_broadcast(key_offset, key_row_idx) + head_idx_index); +#else + const INPUT1_TYPE key_packed = KEY_BLOCK_READ(key_input, key_offset + key_row_idx * key_pitch + head_idx_index); +#endif + +#if IS_KV_COMPRESSED && USE_ASYMMETRIC_QUANTIZATION + KEY_COMPRESSION_SCALE_TYPE key_vals = (TO_KEY_COMPRESSION_SCALE_TYPE(key_packed) - sub_group_broadcast(comp_zp, key_row_idx)) * sub_group_broadcast(comp_scale, key_row_idx); +#elif IS_KV_COMPRESSED + KEY_COMPRESSION_SCALE_TYPE key_vals = (TO_KEY_COMPRESSION_SCALE_TYPE(key_packed) * sub_group_broadcast(comp_scale, key_row_idx)); #else - INPUT1_TYPE key_vals = KEY_BLOCK_READ(key_input, key_offset + key_row_idx * key_pitch + head_idx_index); + INPUT1_TYPE key_vals = key_packed; #endif unroll_for (uint i = 0; i < SUBGROUP_SIZE; i++) { @@ -895,12 +1077,29 @@ KERNEL(sdpa_opt)( } } } else if (seq_len_calc_size > 0) { +#if IS_KV_COMPRESSED + const uint comp_offset = GET_COMPRESSION_INDEX(KEY_COMPRESSION_SCALE, b_idx, b1_idx / BROADCAST_GROUP_SIZE, seq_len + min(sglid, (uint)seq_len_calc_size - 1), 0); + // const uint comp_offset = GET_COMPRESSION_INDEX(KEY_COMPRESSION_SCALE, b_idx, b1_idx / BROADCAST_GROUP_SIZE, seq_len + sglid, 0); + KEY_COMPRESSION_SCALE_TYPE comp_scale = key_scale[comp_offset]; +#if USE_ASYMMETRIC_QUANTIZATION + KEY_COMPRESSION_SCALE_TYPE comp_zp = key_scale[comp_offset + 1]; +#endif +#endif __attribute__((opencl_unroll_hint(1))) for (uint head_idx_index = 0; head_idx_index < HEAD_SIZE; head_idx_index += SUBGROUP_SIZE) { - #define KEY_BLOCK_READ(ptr, offset) BLOCK_READN(INPUT1_TYPE, 1, ptr, offset); - #define QUERY_VEC MAKE_VECTOR_TYPE(INPUT1_TYPE, TARGET_SEQ_LEN_BLOCK_SIZE) + #define KEY_BLOCK_READ(ptr, offset) BLOCK_READN(INPUT1_TYPE, 1, ptr, offset) + #define QUERY_VEC_TYPE MAKE_VECTOR_TYPE(INPUT0_TYPE, TARGET_SEQ_LEN_BLOCK_SIZE) +#if IS_KV_COMPRESSED + #define KEY_UNPACKED_TYPE KEY_COMPRESSION_SCALE_TYPE + #define KEY_UNPACKED_VEC_TYPE MAKE_VECTOR_TYPE(KEY_COMPRESSION_SCALE_TYPE, TARGET_SEQ_LEN_BLOCK_SIZE) + #define TO_KEY_UNPACKED_TYPE(val) TO_KEY_COMPRESSION_SCALE_TYPE(val) +#else + #define KEY_UNPACKED_TYPE INPUT1_TYPE + #define KEY_UNPACKED_VEC_TYPE MAKE_VECTOR_TYPE(INPUT1_TYPE, TARGET_SEQ_LEN_BLOCK_SIZE) + #define TO_KEY_UNPACKED_TYPE(val) TO_INPUT1_TYPE(val) +#endif - QUERY_VEC queries_vec; + QUERY_VEC_TYPE queries_vec; uint query_local_offset = (head_idx_index * TARGET_SEQ_LEN_BLOCK_SIZE) + sglid; unroll_for (uint q_row_idx = 0; q_row_idx < TARGET_SEQ_LEN_BLOCK_SIZE; q_row_idx++) { queries_vec[q_row_idx] = slm_query[query_local_offset]; @@ -908,27 +1107,37 @@ KERNEL(sdpa_opt)( } #ifndef LOAD_KEY_LEFTOVERS_IN_CALC_LOOP - QUERY_VEC key_vec = 0; + KEY_UNPACKED_VEC_TYPE key_vec = 0; unroll_for (uint key_row_idx = 0; key_row_idx < seq_len_calc_size; key_row_idx++) { - #ifdef BEAM_TABLE_TYPE - key_vec[key_row_idx] = KEY_BLOCK_READ(key_input, sub_group_broadcast(key_offset, key_row_idx) + head_idx_index); - #else - key_vec[key_row_idx] = KEY_BLOCK_READ(key_input, key_offset + key_row_idx * key_pitch + head_idx_index); - #endif +#ifdef BEAM_TABLE_TYPE + key_vec[key_row_idx] = TO_KEY_UNPACKED_TYPE(KEY_BLOCK_READ(key_input, sub_group_broadcast(key_offset, key_row_idx) + head_idx_index)); +#else + key_vec[key_row_idx] = TO_KEY_UNPACKED_TYPE(KEY_BLOCK_READ(key_input, key_offset + key_row_idx * key_pitch + head_idx_index)); +#endif + +#if IS_KV_COMPRESSED && USE_ASYMMETRIC_QUANTIZATION + key_vec[key_row_idx] = (key_vec[key_row_idx] - sub_group_broadcast(comp_zp, key_row_idx)) * sub_group_broadcast(comp_scale, key_row_idx); +#elif IS_KV_COMPRESSED + key_vec[key_row_idx] *= sub_group_broadcast(comp_scale, key_row_idx); +#endif } #endif unroll_for (uint key_row_idx = 0; key_row_idx < TARGET_SEQ_LEN_BLOCK_SIZE; key_row_idx++) { #ifdef LOAD_KEY_LEFTOVERS_IN_CALC_LOOP - #ifdef BEAM_TABLE_TYPE - INPUT1_TYPE key_vals = 0; - if (key_row_idx < seq_len_calc_size) - key_vals = KEY_BLOCK_READ(key_input, sub_group_broadcast(key_offset, key_row_idx) + head_idx_index); - #else - INPUT1_TYPE key_vals = 0; - if (key_row_idx < seq_len_calc_size) - key_vals = KEY_BLOCK_READ(key_input, key_offset + key_row_idx * key_pitch + head_idx_index); - #endif + KEY_UNPACKED_TYPE key_vals = 0; + if (key_row_idx < seq_len_calc_size) { +#ifdef BEAM_TABLE_TYPE + key_vals = TO_KEY_UNPACKED_TYPE(KEY_BLOCK_READ(key_input, sub_group_broadcast(key_offset, key_row_idx) + head_idx_index)); +#else + key_vals = TO_KEY_UNPACKED_TYPE(KEY_BLOCK_READ(key_input, key_offset + key_row_idx * key_pitch + head_idx_index)); +#endif + } +#if IS_KV_COMPRESSED && USE_ASYMMETRIC_QUANTIZATION + key_vals = (key_vals - sub_group_broadcast(comp_zp, key_row_idx)) * sub_group_broadcast(comp_scale, key_row_idx); +#elif IS_KV_COMPRESSED + key_vals *= sub_group_broadcast(comp_scale, key_row_idx); +#endif #else #define key_vals key_vec[key_row_idx] #endif @@ -941,12 +1150,14 @@ KERNEL(sdpa_opt)( { unroll_for (uint i = 0; i < TARGET_SEQ_LEN_BLOCK_SIZE; i++) { +#if !APPLY_SCALES_TO_QUERY #if HAS_SCALE_INPUT const OUTPUT_TYPE scale_val = *scale; #else const OUTPUT_TYPE scale_val = TO_OUTPUT_TYPE(STATIC_SCALE_VALUE); #endif qk_acc[i] *= scale_val; +#endif #ifdef HAS_ALIBI const int alibi_val = (1 - SOURCE_SEQ_LEN) + seq_len + i; @@ -1018,7 +1229,7 @@ KERNEL(sdpa_opt)( // QK*V calculation MAKE_VECTOR_TYPE(OUTPUT_TYPE, TARGET_SEQ_LEN_BLOCK_SIZE) acc_output_res = OUTPUT_VAL_ZERO; #if IS_PAGED_ATTENTION - const uint value_pitch = HEAD_SIZE * NUM_KV_HEADS; + const uint value_pitch = (HEAD_SIZE * NUM_KV_HEADS + INPUT2_PAD_BEFORE_FEATURE_NUM + INPUT2_PAD_AFTER_FEATURE_NUM); #else #ifdef INPUT2_DIMS_ORDER uint value_offset_base = FUNC_CALL(get_input2_index)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, 0, 0); @@ -1039,12 +1250,16 @@ KERNEL(sdpa_opt)( const uint heads_dim = num_heads_dim; #endif const uint value_seq_offset = subsequence_begins[gws_seq_indexes_correspondence[target_seq_dim]]; - uint value_offset = value_seq_offset * HEAD_SIZE * NUM_KV_HEADS + heads_dim * HEAD_SIZE + (start_partition_idx + (seq_len)) * HEAD_SIZE * NUM_KV_HEADS + head_size_idx; + uint value_offset = INPUT2_OFFSET + + value_seq_offset * value_pitch + + heads_dim * HEAD_SIZE + + (start_partition_idx + (seq_len)) * value_pitch + head_size_idx; #else #ifdef BEAM_TABLE_TYPE const uint b_idx = beam_table[FUNC_CALL(get_bt_index_value)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, start_partition_idx + (seq_len) + sglid, sgid * SUBGROUP_SIZE)]; const uint value_offset = FUNC_CALL(get_input2_index)(OPTIONAL_SHAPE_INFO_TENSOR b_idx, b1_idx, 0, 0, start_partition_idx + (seq_len) + sglid, sgid * SUBGROUP_SIZE); #else + const uint b_idx = b0_idx; #ifdef INPUT2_DIMS_ORDER uint value_offset = FUNC_CALL(get_input2_index)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, start_partition_idx + (seq_len), head_size_idx); #else @@ -1058,12 +1273,28 @@ KERNEL(sdpa_opt)( qk_val[seq_idx] = slm_qk_vals[seq_idx * SEQ_LEN_PARTITION_SIZE + seq_len + sglid]; } +#if IS_KV_COMPRESSED + const uint comp_offset = GET_COMPRESSION_INDEX(VALUE_COMPRESSION_SCALE, b_idx, b1_idx / BROADCAST_GROUP_SIZE, start_partition_idx + seq_len + sglid, 0); + VALUE_COMPRESSION_SCALE_TYPE comp_scale = val_scale[comp_offset]; +#if USE_ASYMMETRIC_QUANTIZATION + VALUE_COMPRESSION_SCALE_TYPE comp_zp = val_scale[comp_offset + 1]; +#endif +#endif unroll_for (uint i = 0; i < SUBGROUP_SIZE; i++) { #ifdef BEAM_TABLE_TYPE - INPUT2_TYPE value_val = VALUE_BLOCK_READ(value_input, sub_group_broadcast(value_offset, i)); + const INPUT2_TYPE value_packed = VALUE_BLOCK_READ(value_input, sub_group_broadcast(value_offset, i)); #else - INPUT2_TYPE value_val = VALUE_BLOCK_READ(value_input, value_offset); + const INPUT2_TYPE value_packed = VALUE_BLOCK_READ(value_input, value_offset); #endif + +#if IS_KV_COMPRESSED && USE_ASYMMETRIC_QUANTIZATION + VALUE_COMPRESSION_SCALE_TYPE value_val = (value_packed - sub_group_broadcast(comp_zp, i)) * sub_group_broadcast(comp_scale, i); +#elif IS_KV_COMPRESSED + VALUE_COMPRESSION_SCALE_TYPE value_val = (value_packed * sub_group_broadcast(comp_scale, i)); +#else + INPUT2_TYPE value_val = value_packed; +#endif + unroll_for (uint seq_idx = 0; seq_idx < TARGET_SEQ_LEN_BLOCK_SIZE; seq_idx++) { acc_output_res[seq_idx] = mad(sub_group_broadcast(qk_val[seq_idx], i), value_val, acc_output_res[seq_idx]); } @@ -1087,18 +1318,30 @@ KERNEL(sdpa_opt)( const uint heads_dim = num_heads_dim; #endif const uint value_seq_offset = subsequence_begins[gws_seq_indexes_correspondence[target_seq_dim]]; - uint value_offset = value_seq_offset * HEAD_SIZE * NUM_KV_HEADS + heads_dim * HEAD_SIZE + (start_partition_idx + (seq_len * SUBGROUP_SIZE)) * HEAD_SIZE * NUM_KV_HEADS + head_size_idx; + uint value_offset = INPUT2_OFFSET + + value_seq_offset * value_pitch + + heads_dim * HEAD_SIZE + + (start_partition_idx + (seq_len * SUBGROUP_SIZE)) * value_pitch + head_size_idx; #else #ifdef BEAM_TABLE_TYPE const uint b_idx = beam_table[FUNC_CALL(get_bt_index_value)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, start_partition_idx + (seq_len * SUBGROUP_SIZE) + sglid, sgid * SUBGROUP_SIZE)]; uint value_offset = FUNC_CALL(get_input2_index)(OPTIONAL_SHAPE_INFO_TENSOR b_idx, b1_idx, 0, 0, start_partition_idx + (seq_len * SUBGROUP_SIZE) + sglid, sgid * SUBGROUP_SIZE); #else + const uint b_idx = b0_idx; #ifdef INPUT2_DIMS_ORDER uint value_offset = FUNC_CALL(get_input2_index)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, start_partition_idx + (seq_len * SUBGROUP_SIZE), head_size_idx); #else uint value_offset = INPUT2_GET_INDEX(b0_idx, b1_idx, start_partition_idx + (seq_len * SUBGROUP_SIZE), head_size_idx); #endif #endif +#endif + +#if IS_KV_COMPRESSED + const uint comp_offset = GET_COMPRESSION_INDEX(VALUE_COMPRESSION_SCALE, b_idx, b1_idx / BROADCAST_GROUP_SIZE, start_partition_idx + (seq_len * SUBGROUP_SIZE) + sglid, 0); + VALUE_COMPRESSION_SCALE_TYPE comp_scale = val_scale[comp_offset]; +#if USE_ASYMMETRIC_QUANTIZATION + VALUE_COMPRESSION_SCALE_TYPE comp_zp = val_scale[comp_offset + 1]; +#endif #endif MAKE_VECTOR_TYPE(OUTPUT_TYPE, TARGET_SEQ_LEN_BLOCK_SIZE) qk_val; @@ -1108,9 +1351,17 @@ KERNEL(sdpa_opt)( unroll_for (uint i = 0; i < SUBGROUP_SIZE; i++) { #ifdef BEAM_TABLE_TYPE - INPUT2_TYPE value_val = VALUE_BLOCK_READ(value_input, sub_group_broadcast(value_offset, i)); + const INPUT2_TYPE value_packed = VALUE_BLOCK_READ(value_input, sub_group_broadcast(value_offset, i)); +#else + const INPUT2_TYPE value_packed = VALUE_BLOCK_READ(value_input, value_offset); +#endif + +#if IS_KV_COMPRESSED && USE_ASYMMETRIC_QUANTIZATION + VALUE_COMPRESSION_SCALE_TYPE value_val = (value_packed - sub_group_broadcast(comp_zp, i)) * sub_group_broadcast(comp_scale, i); +#elif IS_KV_COMPRESSED + VALUE_COMPRESSION_SCALE_TYPE value_val = (value_packed * sub_group_broadcast(comp_scale, i)); #else - INPUT2_TYPE value_val = VALUE_BLOCK_READ(value_input, value_offset); + INPUT2_TYPE value_val = value_packed; #endif unroll_for (uint seq_idx = 0; seq_idx < TARGET_SEQ_LEN_BLOCK_SIZE; seq_idx++) { acc_output_res[seq_idx] = mad(sub_group_broadcast(qk_val[seq_idx], i), value_val, acc_output_res[seq_idx]); @@ -1138,25 +1389,46 @@ KERNEL(sdpa_opt)( const uint heads_dim = num_heads_dim; #endif const uint value_seq_offset = subsequence_begins[gws_seq_indexes_correspondence[target_seq_dim]]; - uint value_offset = value_seq_offset * HEAD_SIZE * NUM_KV_HEADS + heads_dim * HEAD_SIZE + (start_partition_idx + seq_len_leftovers_start) * HEAD_SIZE * NUM_KV_HEADS + head_size_idx; + uint value_offset = INPUT2_OFFSET + + value_seq_offset * value_pitch + + heads_dim * HEAD_SIZE + + (start_partition_idx + seq_len_leftovers_start) * value_pitch + head_size_idx; #else #ifdef BEAM_TABLE_TYPE const uint b_idx = beam_table[FUNC_CALL(get_bt_index_value)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, start_partition_idx + seq_len_leftovers_start + sglid, sgid * SUBGROUP_SIZE)]; const uint value_offset = FUNC_CALL(get_input2_index)(OPTIONAL_SHAPE_INFO_TENSOR b_idx, b1_idx, 0, 0, start_partition_idx + seq_len_leftovers_start + sglid, sgid * SUBGROUP_SIZE); #else + const uint b_idx = b0_idx; #ifdef INPUT2_DIMS_ORDER uint value_offset = FUNC_CALL(get_input2_index)(OPTIONAL_SHAPE_INFO_TENSOR b0_idx, b1_idx, 0, 0, start_partition_idx + seq_len_leftovers_start, head_size_idx); #else uint value_offset = INPUT2_GET_INDEX(b0_idx, b1_idx, start_partition_idx + seq_len_leftovers_start, head_size_idx); #endif #endif +#endif + +#if IS_KV_COMPRESSED + const uint comp_offset = GET_COMPRESSION_INDEX(VALUE_COMPRESSION_SCALE, b_idx, b1_idx / BROADCAST_GROUP_SIZE, start_partition_idx + min(seq_len_leftovers_start + sglid, seq_len_end - 1), 0); + // const uint comp_offset = GET_COMPRESSION_INDEX(VALUE_COMPRESSION_SCALE, b_idx, b1_idx / BROADCAST_GROUP_SIZE, start_partition_idx + seq_len_leftovers_start + sglid, 0); + VALUE_COMPRESSION_SCALE_TYPE comp_scale = val_scale[comp_offset]; +#if USE_ASYMMETRIC_QUANTIZATION + VALUE_COMPRESSION_SCALE_TYPE comp_zp = val_scale[comp_offset + 1]; +#endif #endif for (uint seq_len_idx = 0; seq_len_idx < partition_seq_len - seq_len_leftovers_start; seq_len_idx++) { #ifdef BEAM_TABLE_TYPE - INPUT2_TYPE value_val = VALUE_BLOCK_READ(value_input, sub_group_broadcast(value_offset, seq_len_idx)); + const INPUT2_TYPE value_packed = VALUE_BLOCK_READ(value_input, sub_group_broadcast(value_offset, seq_len_idx)); +#else + const INPUT2_TYPE value_packed = VALUE_BLOCK_READ(value_input, value_offset); +#endif + +#if IS_KV_COMPRESSED && USE_ASYMMETRIC_QUANTIZATION + VALUE_COMPRESSION_SCALE_TYPE value_val = (value_packed - sub_group_broadcast(comp_zp, seq_len_idx)) * sub_group_broadcast(comp_scale, seq_len_idx); +#elif IS_KV_COMPRESSED + VALUE_COMPRESSION_SCALE_TYPE value_val = (value_packed * sub_group_broadcast(comp_scale, seq_len_idx)); #else - INPUT2_TYPE value_val = VALUE_BLOCK_READ(value_input, value_offset); + INPUT2_TYPE value_val = value_packed; #endif for (uint seq_idx = 0; seq_idx < TARGET_SEQ_LEN_BLOCK_SIZE; seq_idx++) { diff --git a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_ref.cl b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_ref.cl index 83e3c7c7e9fef1..682af11777012f 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_ref.cl +++ b/src/plugins/intel_gpu/src/kernel_selector/cl_kernels/sdpa_ref.cl @@ -112,6 +112,15 @@ inline uint FUNC(get_bt_index_value)(OPTIONAL_SHAPE_INFO_ARG uint b, uint f, uin #endif #define APPLY_SCALE_TO_QUERY 1 +#define HAS_KV_CACHE_ZP_INPUT USE_ASYMMETRIC_QUANTIZATION && !COMBINE_SCALES_AND_ZP + +#if IS_KV_COMPRESSED +#if COMPRESSED_PER_HEAD + #define GET_COMPRESSION_INDEX(INPUT, b, f, y, x) GET_DATA_INDEX(INPUT, (b), (f), (y), (0)); +#else + #define GET_COMPRESSION_INDEX(INPUT, b, f, y, x) GET_DATA_INDEX(INPUT, (b), (0), (y), (0)); +#endif +#endif KERNEL(sdpa_ref)( OPTIONAL_SHAPE_INFO_ARG @@ -125,6 +134,14 @@ KERNEL(sdpa_ref)( const __global INPUT4_TYPE* scale, #endif __global OUTPUT_TYPE* output, +#if IS_KV_COMPRESSED + const __global KEY_COMPRESSION_SCALE_TYPE* key_scale, + const __global VALUE_COMPRESSION_SCALE_TYPE* val_scale, +#if HAS_KV_CACHE_ZP_INPUT + const __global KEY_COMPRESSION_ZP_TYPE* key_zp, + const __global VALUE_COMPRESSION_ZP_TYPE* val_zp, +#endif +#endif #ifdef BEAM_TABLE_TYPE const __global BEAM_TABLE_TYPE* beam_table, #endif @@ -162,7 +179,24 @@ KERNEL(sdpa_ref)( #else INPUT0_TYPE q_val = query_input[query_offset]; #endif - INPUT1_TYPE k_val = key_input[key_offset]; + + INPUT1_TYPE k_val_packed = key_input[key_offset]; +#if IS_KV_COMPRESSED + const uint comp_offset = GET_COMPRESSION_INDEX(KEY_COMPRESSION_SCALE, b_idx, b1 / BROADCAST_GROUP_SIZE, s, 0); + KEY_COMPRESSION_SCALE_TYPE comp_scale = key_scale[comp_offset]; + +#if USE_ASYMMETRIC_QUANTIZATION && HAS_KV_CACHE_ZP_INPUT + KEY_COMPRESSION_SCALE_TYPE comp_zp = key_zp[comp_offset]; +#elif USE_ASYMMETRIC_QUANTIZATION + VALUE_COMPRESSION_SCALE_TYPE comp_zp = key_scale[comp_offset + 1]; +#else + KEY_COMPRESSION_SCALE_TYPE comp_zp = 0; +#endif + KEY_COMPRESSION_SCALE_TYPE k_val = ((k_val_packed - comp_zp) * comp_scale); + +#else + INPUT1_TYPE k_val = k_val_packed; +#endif acc += q_val * k_val; } @@ -236,7 +270,24 @@ KERNEL(sdpa_ref)( #endif uint value_offset = FUNC_CALL(get_input2_index)(OPTIONAL_SHAPE_INFO_TENSOR b_idx, b1, 0, 0, s, head_size_idx); - acc += tmp_buf[tmp_buf_offset] * value_input[value_offset]; + const INPUT2_TYPE value_packed = value_input[value_offset]; +#if IS_KV_COMPRESSED + const uint comp_offset = GET_COMPRESSION_INDEX(VALUE_COMPRESSION_SCALE, b_idx, b1 / BROADCAST_GROUP_SIZE, s, 0); + VALUE_COMPRESSION_SCALE_TYPE comp_scale = val_scale[comp_offset]; + +#if USE_ASYMMETRIC_QUANTIZATION && HAS_KV_CACHE_ZP_INPUT + VALUE_COMPRESSION_SCALE_TYPE comp_zp = val_zp[comp_offset]; +#elif USE_ASYMMETRIC_QUANTIZATION + VALUE_COMPRESSION_SCALE_TYPE comp_zp = val_scale[comp_offset + 1]; +#else + VALUE_COMPRESSION_SCALE_TYPE comp_zp = 0; +#endif + VALUE_COMPRESSION_SCALE_TYPE value = ((value_packed - comp_zp) * comp_scale); +#else + INPUT2_TYPE value = value_packed; +#endif + + acc += tmp_buf[tmp_buf_offset] * value; } uint output_offset = OUTPUT_GET_INDEX(b0, b1, target_seq_idx, head_size_idx); diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_kv_cache.h b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_kv_cache.h new file mode 100644 index 00000000000000..ac6870a37a1728 --- /dev/null +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_kv_cache.h @@ -0,0 +1,30 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "kernel_base_opencl.h" +#include "dynamic_quantize_kernel_ref.h" + +namespace kernel_selector { +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// dynamic_quantize_params +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class DynamicQuantizeKernelKVCache : public KernelBaseOpenCL { +public: + DynamicQuantizeKernelKVCache() : KernelBaseOpenCL("dynamic_quantize_gpu_kv_cache") {} + virtual ~DynamicQuantizeKernelKVCache() {} + + virtual JitConstants GetJitConstants(const dynamic_quantize_params& params) const; + virtual CommonDispatchData SetDefault(const dynamic_quantize_params& params) const; + KernelsData GetKernelsData(const Params& params) const override; + KernelsPriority GetKernelsPriority(const Params& params) const override; + Datatype GetAccumulatorType(const dynamic_quantize_params& params) const; + ParamsKey GetSupportedKey() const override; + +protected: + bool Validate(const Params&) const override; + void GetUpdateDispatchDataFunc(KernelData& kd) const override; +}; +} // namespace kernel_selector diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt.cpp index 6a678770e85d72..52a648679499f2 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt.cpp @@ -154,6 +154,20 @@ bool DynamicQuantizeKernelOpt::Validate(const Params& params) const { if (dq_params.inputs[0].GetPaddedVal() != 0 || dq_params.outputs[0].GetPaddedVal() != 0) return false; + if (dq_params.append_axis != -1) + return false; + + if (dq_params.group_sizes.back() != UINT64_MAX) + return false; + + // Allow only default scales order + const auto& scales_output_order = dq_params.scales_output_order; + if (!scales_output_order.empty()) { + for (size_t i = 0; i < scales_output_order.size(); i++) + if (scales_output_order[i] != i) + return false; + } + return true; } } // namespace kernel_selector diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt_kv_cache.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt_kv_cache.cpp new file mode 100644 index 00000000000000..d0c99484e3f52e --- /dev/null +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_opt_kv_cache.cpp @@ -0,0 +1,285 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "dynamic_quantize_kernel_kv_cache.h" +#include "kernel_selector_utils.h" +#include + + +static constexpr size_t subgroup_size = 16; + +namespace kernel_selector { +static Tensor::NDims get_normalized_dims(const DataTensor& tensor) { + auto dims = tensor.GetDims(); + std::reverse(dims.begin(), dims.end()); + + return dims; +} + +static size_t get_elements_number_per_batch(const dynamic_quantize_params& params) { + const auto& group_sizes = params.group_sizes; + const auto& input_dims = get_normalized_dims(params.inputs[0]); + + size_t total_elements_number = 1; + for (size_t i = 0; i < group_sizes.size(); i++) { + if (group_sizes[i] != UINT64_MAX) { + total_elements_number *= input_dims[i].v; + } + } + + return total_elements_number; +} + +static size_t get_elements_number_per_group(const dynamic_quantize_params& params) { + const auto& group_sizes = params.group_sizes; + const auto& input_dims = get_normalized_dims(params.inputs[0]); + + size_t total_elements_number = 1; + for (size_t i = 0; i < group_sizes.size(); i++) { + if (group_sizes[i] == UINT64_MAX) { + total_elements_number *= input_dims[i].v; + } else { + total_elements_number *= group_sizes[i]; + } + } + + return total_elements_number; +} + +static std::string generate_dims_indexes_calculation(std::vector> dims) { + std::reverse(dims.begin(), dims.end()); // reorder dims in order from innermost to outermost dimensions + + auto generate_calc_function = [&](std::string data_type, std::string index_var, size_t dim_idx) { + std::string index_calc_str; + index_calc_str += "" + data_type + " " + dims[dim_idx].first + " = "; + index_calc_str += "(" + index_var + " / "; + index_calc_str += "(1"; + for (size_t i = 0; i < dim_idx; i++) { + index_calc_str += " * " + dims[i].second; + } + index_calc_str += ")) % " + dims[dim_idx].second + ";"; + + return index_calc_str; + }; + + std::stringstream indexes_calc_str; + for (size_t i = 0; i < dims.size(); i++) { + indexes_calc_str << generate_calc_function("uint", "data_idx", i); + } + + return indexes_calc_str.str(); +} + +static size_t get_per_iter_elements_number(const dynamic_quantize_params& params) { + const auto maxWorkGroupSize = params.engineInfo.maxWorkGroupSize; + const auto total_grouped_elements = get_elements_number_per_group(params); + + if (total_grouped_elements % maxWorkGroupSize == 0) + return maxWorkGroupSize; + + if (total_grouped_elements < maxWorkGroupSize) + return total_grouped_elements; + + return 0; +} + +ParamsKey DynamicQuantizeKernelKVCache::GetSupportedKey() const { + ParamsKey k; + k.EnableInputDataType(Datatype::F16); + k.EnableOutputDataType(Datatype::INT8); + k.EnableDifferentTypes(); + k.EnableAllInputLayout(); + k.EnableAllOutputLayout(); + k.EnableTensorOffset(); + k.EnableTensorPitches(); + k.EnableBatching(); + k.EnableDynamicShapesSupport(); + return k; +} + +JitConstants DynamicQuantizeKernelKVCache::GetJitConstants(const dynamic_quantize_params& params) const { + JitConstants jit = MakeBaseParamsJitConstants(params); + + const std::vector> default_dims = {{"b", "INPUT0_BATCH_NUM"}, + {"f", "INPUT0_FEATURE_NUM"}, + {"y", "INPUT0_SIZE_Y"}, + {"x", "INPUT0_SIZE_X"}}; + + const auto& group_sizes = params.group_sizes; + std::vector> batch_dims, grouped_dims; + for (size_t i = 0; i < group_sizes.size(); i++) { + if (group_sizes[i] == 1) { + batch_dims.push_back(default_dims[i]); + } else { + grouped_dims.push_back(default_dims[i]); + } + } + + const auto& input_dims = get_normalized_dims(params.inputs[0]); + const auto total_grouped_elements = get_elements_number_per_group(params); + const auto per_iter_elements_number = get_per_iter_elements_number(params); + const auto total_subgroups_number = total_grouped_elements / input_dims.back().v; + + // Drop the last dimensions, since it will be processed in the kernel's loop + grouped_dims.pop_back(); + + const bool append_mode = params.append_axis != -1; + std::pair append_axis_info = {}; + if (append_mode) { + jit.AddConstant(MakeJitConstant("APPEND_MODE", append_mode)); + jit.AddConstant(MakeJitConstant("APPEND_AXIS_NAME", default_dims[params.append_axis].first)); + } + + jit.AddConstant(MakeJitConstant("DECLARE_BATCHED_DIMS_INDEXES(data_idx)", generate_dims_indexes_calculation(batch_dims))); + jit.AddConstant(MakeJitConstant("DECLARE_GROUPED_DIMS_INDEXES(data_idx)", generate_dims_indexes_calculation(grouped_dims))); + jit.AddConstant(MakeJitConstant("SUBGROUPS_NUMBER", total_subgroups_number)); + + const auto iterations_number = total_grouped_elements / per_iter_elements_number; + + jit.AddConstant(MakeJitConstant("ITERATIONS_NUMBER", iterations_number)); + jit.AddConstant(MakeJitConstant("ASYMMETRIC_QUANTIZATION", params.use_asymmetric_quantization)); + jit.AddConstant(MakeJitConstant("GROUP_SCALES_WITH_ZP", params.combine_scales_and_zp)); + + bool rearrange_scales_order = false; + const auto& scales_output_order = params.scales_output_order; + if (!scales_output_order.empty()) { + for (size_t i = 0; i < scales_output_order.size(); i++) { + if (i != scales_output_order[i]) { + rearrange_scales_order = true; + break; + } + } + } + + if (rearrange_scales_order) { + const std::array default_dim_order = {'b', 'f', 'y', 'x'}; + std::stringstream ss; + for (size_t i = 0; i < scales_output_order.size(); i++) { + ss << default_dim_order[scales_output_order[i]]; + + if (i + 1 != scales_output_order.size()) + ss << ", "; + } + + jit.AddConstant(MakeJitConstant("SCALES_OUTPUT_ORDER", ss.str())); + } + + for (size_t i = 0; i < group_sizes.size(); i++) { + jit.AddConstant(MakeJitConstant("GROUP_SIZE_DIM" + std::to_string(i), group_sizes[i])); + } + + return jit; +} + +CommonDispatchData DynamicQuantizeKernelKVCache::SetDefault(const dynamic_quantize_params& params) const { + CommonDispatchData dispatchData; + + const auto& input_dims = get_normalized_dims(params.inputs[0]); + const auto total_batched_elements = get_elements_number_per_batch(params); + const auto total_grouped_elements = get_elements_number_per_group(params); + const auto total_subgroups_number = total_grouped_elements / input_dims.back().v; + + dispatchData.gws = {subgroup_size, total_subgroups_number, total_batched_elements}; + dispatchData.lws = {subgroup_size, total_subgroups_number, 1}; + + return dispatchData; +} + +void DynamicQuantizeKernelKVCache::GetUpdateDispatchDataFunc(KernelData& kd) const { + kd.update_dispatch_data_func = [this](const Params& params, KernelData& kd) { + const auto& prim_params = static_cast(params); + auto dispatchData = SetDefault(prim_params); + OPENVINO_ASSERT(kd.kernels.size() == 1, "[GPU] Invalid kernels size for update dispatch data func"); + kd.kernels[0].params.workGroups.global = dispatchData.gws; + kd.kernels[0].params.workGroups.local = dispatchData.lws; + kd.kernels[0].skip_execution = false; + + if (prim_params.append_axis != -1) { + kd.kernels[0].params.scalars.clear(); + + ScalarDescriptor axis_offset; + axis_offset.t = ScalarDescriptor::Types::UINT32; + axis_offset.v.u32 = static_cast(prim_params.axis_offset); + kd.kernels[0].params.scalars.push_back(axis_offset); + } + }; +} + +KernelsData DynamicQuantizeKernelKVCache::GetKernelsData(const Params& params) const { + assert(params.GetType() == KernelType::DYNAMIC_QUANTIZE); + + if (!Validate(params)) + return {}; + + const dynamic_quantize_params& prim_params = static_cast(params); + auto dispatchData = SetDefault(prim_params); + + KernelData kd = KernelData::Default(params); + + auto cldnn_jit = GetJitConstants(prim_params); + auto entry_point = GetEntryPoint(kernelName, prim_params.layerID, params); + auto jit = CreateJit(kernelName, cldnn_jit, entry_point); + + GetUpdateDispatchDataFunc(kd); + + auto& kernel = kd.kernels[0]; + FillCLKernelData(kernel, + dispatchData, + params.engineInfo, + kernelName, + jit, + entry_point, + EXE_MODE_DEFAULT, + false, + false, + 1, + GetFusedPrimitiveInputsCount(params), + static_cast(prim_params.outputs.size()), + prim_params.is_shape_agnostic); + + if (prim_params.append_axis != -1) + kernel.params.arguments.push_back({ArgumentDescriptor::Types::SCALAR, 0}); + + return {kd}; +} + +KernelsPriority DynamicQuantizeKernelKVCache::GetKernelsPriority(const Params& /*params*/) const { + return FORCE_PRIORITY_3; +} + +bool DynamicQuantizeKernelKVCache::Validate(const Params& params) const { + if (!KernelBaseOpenCL::Validate(params)) + return false; + + const auto& dq_params = static_cast(params); + + const auto& group_sizes = dq_params.group_sizes; + const auto& input_dims = get_normalized_dims(dq_params.inputs[0]); + const size_t non_compressed_dims_number = std::count(group_sizes.begin(), group_sizes.end(), 1); + + if (non_compressed_dims_number == group_sizes.size()) + return false; + + for (size_t i = 0; i < group_sizes.size(); i++) { + if (group_sizes[i] != 1 && input_dims[i].is_dynamic) { + return false; + } + } + + // Last dimension should be static, reduced by group_sizes configuration and divisible by 16 + if (group_sizes.back() == 1 || input_dims.back().is_dynamic || input_dims.back().v % subgroup_size != 0) + return false; + + // Limit the size of the innermost dimension + if (input_dims.back().v > 256) + return false; + + // In case of HEADS_NUM * HEAD_SIZE group size, check that it fits into the supported workgroup size limit + if (get_elements_number_per_group(dq_params) / input_dims.back().v >= params.engineInfo.maxWorkGroupSize / subgroup_size) + return false; + + return true; +} +} // namespace kernel_selector + diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.cpp index 3b214848e2f8ad..bd3d0f87cdc931 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.cpp @@ -26,6 +26,41 @@ JitConstants DynamicQuantizeKernelRef::GetJitConstants(const dynamic_quantize_pa jit.Merge(GetTensorFriendlyWorkGroupsJit(params.outputs[0])); + bool rearrange_scales = false; + const auto& scales_output_order = params.scales_output_order; + if (!scales_output_order.empty()) { + for (size_t i = 0; i < scales_output_order.size(); i++) { + if (i != scales_output_order[i]) { + rearrange_scales = true; + break; + } + } + } + + if (rearrange_scales) { + const std::array default_dim_order = {'b', 'f', 'y', 'x'}; + + std::stringstream ss; + for (size_t i = 0; i < scales_output_order.size(); i++) { + ss << default_dim_order[scales_output_order[i]]; + + if (i + 1 != scales_output_order.size()) + ss << ", "; + } + + jit.AddConstant(MakeJitConstant("SCALES_OUTPUT_ORDER", ss.str())); + } + + jit.AddConstant(MakeJitConstant("ASYMMETRIC_QUANTIZATION", params.use_asymmetric_quantization)); + jit.AddConstant(MakeJitConstant("GROUP_SCALES_WITH_ZP", params.combine_scales_and_zp)); + + auto group_sizes = params.group_sizes; + group_sizes.resize(std::min((size_t)4, group_sizes.size()), 1); + + for (size_t i = 0; i < group_sizes.size(); i++) { + jit.AddConstant(MakeJitConstant("GROUP_SIZE_DIM" + std::to_string(i), group_sizes[i])); + } + return jit; } @@ -34,7 +69,15 @@ CommonDispatchData DynamicQuantizeKernelRef::SetDefault(const dynamic_quantize_p CommonDispatchData dispatchData; OPENVINO_ASSERT(params.outputs[0].GetLayout() == DataLayout::bfyx, "It supports only 4d tensor"); - dispatchData.gws = {params.outputs[0].Batch().v * params.outputs[0].Feature().v, 1, 1}; + + auto group_sizes = params.group_sizes; + group_sizes.resize(std::min((size_t)4, group_sizes.size()), 1); + auto batch_size = group_sizes[0] == 1 ? params.outputs[0].Batch().v : 1; + auto feature_size = group_sizes[1] == 1 ? params.outputs[0].Feature().v : 1; + auto y_size = group_sizes[2] == 1 ? params.outputs[0].Y().v : 1; + auto x_size = group_sizes[3] == 1 ? params.outputs[0].X().v : 1; + + dispatchData.gws = {batch_size * feature_size, y_size, x_size}; dispatchData.lws = {1, 1, 1}; return dispatchData; diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.h b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.h index c46b6b2685a940..d437d6ab6eb1f6 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.h +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_ref.h @@ -13,6 +13,13 @@ namespace kernel_selector { struct dynamic_quantize_params : public base_params { dynamic_quantize_params() : base_params(KernelType::DYNAMIC_QUANTIZE) {} size_t fc_ifm_size = 0; + + int64_t append_axis = -1; + int64_t axis_offset = -1; + std::vector group_sizes; + std::vector scales_output_order; + bool use_asymmetric_quantization = false; + bool combine_scales_and_zp = false; }; class DynamicQuantizeKernelRef : public KernelBaseOpenCL { diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_selector.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_selector.cpp index 6ca9fbd2f5bd76..d38cf6ad2b4e52 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_selector.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/dynamic_quantize/dynamic_quantize_kernel_selector.cpp @@ -5,11 +5,13 @@ #include "dynamic_quantize_kernel_selector.h" #include "dynamic_quantize_kernel_ref.h" #include "dynamic_quantize_kernel_opt.h" +#include "dynamic_quantize_kernel_kv_cache.h" namespace kernel_selector { dynamic_quantize_kernel_selector::dynamic_quantize_kernel_selector() { Attach(); Attach(); + Attach(); } KernelsData dynamic_quantize_kernel_selector::GetBestKernels(const Params& params) const { diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/fully_connected/fully_connected_kernel_bf_tiled.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/fully_connected/fully_connected_kernel_bf_tiled.cpp index 6604def1a69093..178e1ea405b6bb 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/fully_connected/fully_connected_kernel_bf_tiled.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/fully_connected/fully_connected_kernel_bf_tiled.cpp @@ -8,6 +8,7 @@ #include #include "common_types.h" +static constexpr size_t lws_batches = 8; static constexpr size_t simd = 16; static constexpr size_t min_quantize_grp_size = 32; static constexpr size_t min_slm_size = 256; @@ -50,6 +51,17 @@ static std::pair get_output_aligned_bf_size(const fully_connecte return {output_b, output_f}; } +static bool is_weight_dyn_quantizable(const fully_connected_params& params) { + auto weight_type = params.weights.GetDType(); + if (weight_type == WeightsType::INT4 || weight_type == WeightsType::UINT4) + return true; + // UINT8 weight type is supported by FC dyn-quantize(with SLM). + if (weight_type == WeightsType::UINT8) + return true; + + return false; +} + // DYNAMIC_QUANTIZE static size_t get_dynamic_quantize_group_size(const fully_connected_params& params) { auto dynamic_quantization_group_size = params.dynamic_quantization_group_size; @@ -91,7 +103,7 @@ static size_t get_dynamic_quantize_group_size(const fully_connected_params& para return 0; } -static bool should_dynamic_quantize(const fully_connected_params& params) { +static bool should_dynamic_quantize(const fully_connected_params& params, bool print_log = false) { size_t dynamic_quantization_group_size = get_dynamic_quantize_group_size(params); if (params.inputs[0].GetFirstElementOffset() != 0) @@ -110,11 +122,17 @@ static bool should_dynamic_quantize(const fully_connected_params& params) { const size_t scale_group_size = params.weights.IFM().v / params.decompression_scale.Feature().v; if ((scale_group_size % simd == 0) && (input_f % dynamic_quantization_group_size == 0) && (params.is_shape_agnostic || (params.inputs[0].Batch().v > 1 && input_b > min_slm_size)) && - params.inputs[0].GetDType() == Datatype::F16 && - (params.weights.GetDType() == WeightsType::INT4 || params.weights.GetDType() == WeightsType::UINT4)) { - GPU_DEBUG_TRACE_DETAIL << " Dynamic quantizing for FC : scale_group_size " << scale_group_size << ", Input (" << - kernel_selector::toString(params.inputs[0].GetDType()) << ", " << kernel_selector::toString(params.outputs[0].GetLayout()) << - ") B: " << params.inputs[0].Batch().v << ", F: " << params.inputs[0].Feature().v << ", Y: " << params.inputs[0].Y().v << std ::endl; + params.inputs[0].GetDType() == Datatype::F16 && is_weight_dyn_quantizable(params)) { + if (print_log) { + GPU_DEBUG_TRACE_DETAIL << " Dynamic quantizing for FC : scale_group_size: " << scale_group_size << + ", Dyn-quan group size: " << dynamic_quantization_group_size << + ", Type(I:" << kernel_selector::toString(params.inputs[0].GetDType()) << + ", O:" << kernel_selector::toString(params.outputs[0].GetDType()) << + ", W:" << kernel_selector::toString(params.weights.GetDType()) << + "), Format(W:" << kernel_selector::toString(params.weights.GetLayout()) << + ") B: " << params.inputs[0].Batch().v << ", F: " << params.inputs[0].Feature().v << + ", Y: " << params.inputs[0].Y().v << std ::endl; + } return true; } @@ -204,8 +222,9 @@ DeviceFeaturesKey FullyConnected_bf_tiled::get_required_device_features_key(cons } bool FullyConnected_bf_tiled::Validate(const Params& params) const { - if (!Parent::Validate(params)) + if (!Parent::Validate(params)) { return false; + } auto& fc_params = static_cast(params); auto& input = fc_params.inputs[0]; @@ -314,21 +333,21 @@ bool TuneParamsSelector::VerifyTuneParams(const fully_connected_params& params, if (tparams.tile_ofm * simd > 64) return false; - bool is_i4_u4 = (params.weights.GetDType() == WeightsType::INT4 || params.weights.GetDType() == WeightsType::UINT4); + bool is_dyn_quantable_type = is_weight_dyn_quantizable(params); if (tparams.kernel_type == FullyConnected_bf_tiled::KernelType::SLM) { const auto required_batch_alignment = 64; if (!params.is_shape_agnostic && (!IsAligned(output_b, required_batch_alignment) || output_b < min_slm_size)) return false; const auto required_tile_b = 8; - if ((tparams.tile_b != required_tile_b) && !is_i4_u4) + if ((tparams.tile_b != required_tile_b) && !is_dyn_quantable_type) return false; const auto required_tile_ofm = 2; if (tparams.tile_ofm != required_tile_ofm) return false; - if (params.weights.GetDType() != WeightsType::INT4 && params.weights.GetDType() != WeightsType::UINT4) + if (!is_dyn_quantable_type) return false; if (params.engineInfo.deviceType != dev_type::integrated_gpu) @@ -340,7 +359,7 @@ bool TuneParamsSelector::VerifyTuneParams(const fully_connected_params& params, return true; } - if (params.compressed && is_i4_u4) { + if (params.compressed && is_dyn_quantable_type) { if (!(tparams.tile_ofm == 2 || tparams.tile_ofm == 4)) return false; if (tparams.tile_ofm == 4 && tparams.outer_ofm == 2 && !is_suitable_outer_ofm(params, output_f)) @@ -382,11 +401,10 @@ FullyConnected_bf_tiled::GetAutoTuneParams(const fully_connected_params& params, while (max_tile_ofm * 2 * simd <= output_f && max_tile_ofm < 4) max_tile_ofm *= 2; - if (params.weights.GetDType() == WeightsType::UINT4 || params.weights.GetDType() == WeightsType::INT4) { + if (params.weights.GetDType() == WeightsType::UINT4 || params.weights.GetDType() == WeightsType::INT4 || + (is_weight_dyn_quantizable(params) && should_dynamic_quantize(params))) { + // Only 4bit weight type is fully optimized to use SLM. In default kernel, SLM is not applied to 8bit weight. if (!params.is_shape_agnostic && batch == 1) { - if (should_dynamic_quantize(params)) - return selector.Default(tune_params(1, 2, 4, 2, 1, 1, 1, EXE_MODE_DEFAULT)); - // Tuning for Meteor Lake if (is_weight_vertical(params, output_f)) { if (params.weights.GetLayout() == WeightsLayout::os_is_yx_osv32_isv2) { @@ -411,9 +429,11 @@ FullyConnected_bf_tiled::GetAutoTuneParams(const fully_connected_params& params, selector.Case(tune_params(16, 2, 2, 4, 1, 1, 1, EXE_MODE_DEFAULT, KernelType::SLM)) .Case(tune_params(16, 2, 1, 4, 1, 1, 1, EXE_MODE_DEFAULT, KernelType::SLM)); } + selector.Case(tune_params(8, 2, 2, 4, 1, 1, 1, EXE_MODE_DEFAULT, KernelType::SLM)) .Case(tune_params(8, 2, 1, 4, 1, 1, 1, EXE_MODE_DEFAULT, KernelType::SLM)); } + if (params.weights.GetLayout() == WeightsLayout::os_iyx_osv16) return selector.Default(tune_params(8, 1, 1, 4, 1, 1, 1, EXE_MODE_DEFAULT)); else if (params.weights.GetLayout() == WeightsLayout::os_is_yx_osv64_isv2) @@ -501,7 +521,6 @@ FullyConnected_bf_tiled::SetDefault(const fully_connected_params& params, int au auto batch_threads = threads.first; auto feature_threads = threads.second; - const size_t lws_batches = 8; const size_t aligned_batch = Align(batch_threads, lws_batches); // Each WG calculates 8x8 batches (TILE_B x LWS[2] size) const bool can_use_slm = tparams.kernel_type == KernelType::SLM; @@ -550,7 +569,6 @@ JitConstants FullyConnected_bf_tiled::GetJitConstants(const fully_connected_para WeightsType weights_dt = params.weights.GetDType(); if (weights_dt == WeightsType::UINT4 || weights_dt == WeightsType::INT4) { tile_k_ofm_packed /= 2; - jit.Merge(make_int4_packed_type_jit_constant("INT4_PACKED_TYPE", weights_dt, tile_k_ofm)); const size_t scale_group_size = params.weights.IFM().v / params.decompression_scale.Feature().v; // Do not use SCALE_POST_OP for SLM kernel, since it demonstrates worse performance @@ -581,7 +599,7 @@ JitConstants FullyConnected_bf_tiled::GetJitConstants(const fully_connected_para if (dispatchData.use_slm) { OPENVINO_ASSERT(dispatchData.tile_n == 2, "[GPU] Unsupported TILE_OFM size for SLM kernel configuration"); - OPENVINO_ASSERT(weights_dt == WeightsType::INT4 || weights_dt == WeightsType::UINT4, "[GPU] Unsupported FC weights type for SLM kernel configuration"); + OPENVINO_ASSERT(is_weight_dyn_quantizable(params), "[GPU] Unsupported FC weights type for SLM kernel configuration"); auto lws_batches = dispatchData.lws[2]; auto total_weights_elements = simd * dispatchData.tile_n * simd * dispatchData.tile_mk; // SIMD * TILE_OFM * SIMD * TILE_IFM @@ -608,15 +626,19 @@ JitConstants FullyConnected_bf_tiled::GetJitConstants(const fully_connected_para jit.AddConstant(MakeJitConstant("LWS_BATCHES", lws_batches)); jit.AddConstant(MakeJitConstant("FILTER_LOAD_ITERS", weights_load_iters)); - if (params.weights.GetLayout() == WeightsLayout::os_iyx_osv16) { - jit.AddConstant(MakeJitConstant("FILTER_ACTUAL_LOAD_BLOCK_SIZE", block_read_size / 2)); - jit.Merge(make_int4_packed_type_jit_constant("INT4_PACKED_TYPE_PRELOAD", params.weights.GetDType(), weights_elements_per_load / 2)); - } else if (params.weights.GetLayout() == WeightsLayout::os_is_yx_osv64_isv2) { - jit.AddConstant(MakeJitConstant("FILTER_ACTUAL_LOAD_BLOCK_SIZE", block_read_size / 2)); - jit.Merge(make_int4_packed_type_jit_constant("INT4_PACKED_TYPE_PRELOAD", params.weights.GetDType(), weights_elements_per_load / 2)); + if (weights_dt == WeightsType::INT4 || weights_dt == WeightsType::UINT4) { + if (params.weights.GetLayout() == WeightsLayout::os_iyx_osv16) { + jit.AddConstant(MakeJitConstant("FILTER_ACTUAL_LOAD_BLOCK_SIZE", block_read_size / 2)); + jit.Merge(make_int4_packed_type_jit_constant("INT4_PACKED_TYPE_PRELOAD", params.weights.GetDType(), weights_elements_per_load / 2)); + } else if (params.weights.GetLayout() == WeightsLayout::os_is_yx_osv64_isv2) { + jit.AddConstant(MakeJitConstant("FILTER_ACTUAL_LOAD_BLOCK_SIZE", block_read_size / 2)); + jit.Merge(make_int4_packed_type_jit_constant("INT4_PACKED_TYPE_PRELOAD", params.weights.GetDType(), weights_elements_per_load / 2)); + } else { + jit.AddConstant(MakeJitConstant("FILTER_ACTUAL_LOAD_BLOCK_SIZE", block_read_size)); + jit.Merge(make_int4_packed_type_jit_constant("INT4_PACKED_TYPE_PRELOAD", params.weights.GetDType(), weights_elements_per_load)); + } } else { jit.AddConstant(MakeJitConstant("FILTER_ACTUAL_LOAD_BLOCK_SIZE", block_read_size)); - jit.Merge(make_int4_packed_type_jit_constant("INT4_PACKED_TYPE_PRELOAD", params.weights.GetDType(), weights_elements_per_load)); } jit.AddConstant(MakeJitConstant("FILTER_LOAD_BLOCK_SIZE", block_read_size)); @@ -629,7 +651,6 @@ JitConstants FullyConnected_bf_tiled::GetJitConstants(const fully_connected_para if (should_dynamic_quantize(params)) { jit.AddConstant(MakeJitConstant("DYNAMIC_QUANTIZE", 1)); jit.AddConstant(MakeJitConstant("DQ_DECOMPRESSION_SCALE_POST_OP", 1)); - jit.AddConstant(MakeJitConstant("DQ_TYPE", "char")); jit.AddConstant(MakeJitConstant("QUANTIZE_GROUP_SIZE", quantize_grp_size)); } else { if (add_decompress_scale_post_op) @@ -637,6 +658,7 @@ JitConstants FullyConnected_bf_tiled::GetJitConstants(const fully_connected_para jit.AddConstant(MakeJitConstant("DYNAMIC_QUANTIZE", 0)); jit.AddConstant(MakeJitConstant("QUANTIZE_GROUP_SIZE", min_quantize_grp_size)); } + jit.AddConstant(MakeJitConstant("DQ_TYPE", "char")); jit.AddConstant(MakeJitConstant("IFM_SIZE", get_input_bf_size(params).second)); jit.AddConstant(MakeJitConstant("SIMD", simd)); @@ -659,9 +681,7 @@ JitConstants FullyConnected_bf_tiled::GetJitConstants(const fully_connected_para } auto max_tile_b_size = dispatchData.tile_m; - if (params.compressed && - params.is_shape_agnostic && - (weights_dt == WeightsType::UINT4 || weights_dt == WeightsType::INT4)) + if (params.compressed && params.is_shape_agnostic && is_weight_dyn_quantizable(params)) max_tile_b_size = std::max(max_tile_b_size, (uint32_t)8); jit.Merge(MakeConstantLoopUnrollJitConstants(max_tile_b_size)); @@ -772,8 +792,10 @@ void FullyConnected_bf_tiled::GetUpdateDispatchDataFunc(KernelData& kd) const { if (kd.internalBufferSizes[0] < input_size) { kd.internalBufferSizes.clear(); - kd.internalBufferSizes.push_back(input_size); // quantized input is char type - kd.internalBufferSizes.push_back(input_size / quantize_grp_size * 2); // de_quan_scale is half type + // quantized input is char type + kd.internalBufferSizes.push_back(input_size); + // half type of de_quan_scale and activation sum for each quantized group + kd.internalBufferSizes.push_back((input_size / quantize_grp_size) * 2 * 2); } kd.kernels[0].params.workGroups.global = {std::max((input_size / quantize_grp_size), (size_t)1), 1, 1}; @@ -800,7 +822,7 @@ KernelsData FullyConnected_bf_tiled::GetTunedKernelsDataByIndex(const Params &pa && (fc_params.weights.GetLayout() == WeightsLayout::oiyx || fc_params.weights.GetLayout() == WeightsLayout::os_is_yx_osv64_isv2) && (fc_params.weights.GetDType() == WeightsType::INT4 || fc_params.weights.GetDType() == WeightsType::UINT4) && is_weight_horizontal(fc_params, output_f)) { - // Large N + Small K case (horizontal weight) to use [osv64_isv2] + TILE_OFM 4 for batch 1 + // Large N + small K case (horizontal weight) to use [osv64_isv2] + TILE_OFM 4 for batch 1 weights_layout = WeightsLayout::os_is_yx_osv64_isv2; } else if (fc_params.compressed && fc_params.inputs[0].GetDType() == Datatype::F16 && (fc_params.weights.GetDType() == WeightsType::INT4 || fc_params.weights.GetDType() == WeightsType::UINT4) @@ -947,6 +969,8 @@ KernelsData FullyConnected_bf_tiled::GetMultiKernelsData(const Params ¶ms, auto& quan_kernel = kd.kernels[0]; DispatchData dyn_quan_dispatch = dispatchData; auto input_size = std::max(fc_params.inputs[0].PhysicalSize(), get_input_bf_size(fc_params).second); + if (!params.is_shape_agnostic) + input_size = std::max(input_size, Align(get_input_bf_size(fc_params).first, lws_batches) * get_input_bf_size(fc_params).second); dyn_quan_dispatch.gws = {input_size / quantize_grp_size, 1, 1}; dyn_quan_dispatch.lws = {16, 1, 1}; quan_kernel.params.workGroups.global = dyn_quan_dispatch.gws; @@ -958,7 +982,6 @@ KernelsData FullyConnected_bf_tiled::GetMultiKernelsData(const Params ¶ms, quan_cldnn_jit.AddConstant(MakeJitConstant("FC_KERNEL_DYNAMIC_QUANTIZE", 1)); auto quan_jit = CreateJit(kernelName, quan_cldnn_jit, quan_entry_point); - FillCLKernelData(quan_kernel, dyn_quan_dispatch, params.engineInfo, @@ -977,8 +1000,10 @@ KernelsData FullyConnected_bf_tiled::GetMultiKernelsData(const Params ¶ms, quan_kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, 0}); quan_kernel.params.arguments.push_back({ArgumentDescriptor::Types::INTERNAL_BUFFER, 0}); quan_kernel.params.arguments.push_back({ArgumentDescriptor::Types::INTERNAL_BUFFER, 1}); + // char type quantized input kd.internalBufferSizes.push_back(input_size); - kd.internalBufferSizes.push_back(input_size / quantize_grp_size * 2); + // half type of de_quan_scale and activation sum for each quantized group + kd.internalBufferSizes.push_back(input_size / quantize_grp_size * 2 * 2); kernel_number++; } kd.internalBufferDataType = Datatype::F16; diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/rope/rope_kernel_base.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/rope/rope_kernel_base.cpp index a48632f6c45509..130c5a69d4262c 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/rope/rope_kernel_base.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/rope/rope_kernel_base.cpp @@ -29,38 +29,15 @@ JitConstants RoPEKernelBase::GetJitConstants(const rope_params& params, RoPEKern if (params.slice_stop > params.slice_start) { jit.AddConstant(MakeJitConstant("ENABLE_SLICE", true)); - - auto f = toCodeString(params.inputs[0].Feature(), 1); - auto x = toCodeString(params.inputs[0].X(), 2); - auto y = toCodeString(params.inputs[0].Y(), 3); - - auto sliced_val = toCodeString(params.slice_stop - params.slice_start); - auto sliced_x = params.axis == 3 ? sliced_val : x; - auto sliced_y = params.axis == 2 ? sliced_val : y; - - jit.AddConstant(MakeJitConstant("SLICED_INPUT0_X_PITCH", 1)); - jit.AddConstant(MakeJitConstant("SLICED_INPUT0_Y_PITCH", sliced_x)); - jit.AddConstant(MakeJitConstant("SLICED_INPUT0_FEATURE_PITCH", sliced_x + "*" + sliced_y)); - jit.AddConstant(MakeJitConstant("SLICED_INPUT0_BATCH_PITCH", sliced_x + "*" + sliced_y + "*" + f)); - jit.AddConstant(MakeJitConstant("SLICED_INPUT0_OFFSET", 0)); jit.AddConstant(MakeJitConstant("SLICED_FROM_START", toCodeString(params.slice_start))); - if (params.axis == 2) { - jit.AddConstant(MakeJitConstant("SLICED_FROM_END", "(" + y + "-" + toCodeString(params.slice_stop) + ")")); - } else if (params.axis == 3) { - jit.AddConstant(MakeJitConstant("SLICED_FROM_END", "(" + x + "-" + toCodeString(params.slice_stop) + ")")); - } else { + if (params.axis != 2 && params.axis != 3) { OPENVINO_THROW("[GPU] Invalid axis value for RoPE operation"); } } if (params.transposed_input) { jit.AddConstant(MakeJitConstant("ENABLE_TRANSPOSE", true)); - jit.AddConstant(MakeJitConstant("TRANSPOSED_INPUT0_OFFSET", 0)); - jit.AddConstant(MakeJitConstant("TRANSPOSED_INPUT0_X_PITCH", 1)); - jit.AddConstant(MakeJitConstant("TRANSPOSED_INPUT0_Y_PITCH", "INPUT0_FEATURE_PITCH")); - jit.AddConstant(MakeJitConstant("TRANSPOSED_INPUT0_FEATURE_PITCH", "INPUT0_Y_PITCH")); - jit.AddConstant(MakeJitConstant("TRANSPOSED_INPUT0_BATCH_PITCH", "INPUT0_BATCH_PITCH")); } if (!params.is_chatglm && (params.inputs[1].has_dynamic_pad() || params.inputs[2].has_dynamic_pad())) { diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_base.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_base.cpp index 7556debd29df00..e2a538750d1615 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_base.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_base.cpp @@ -4,6 +4,7 @@ #include "sdpa_kernel_base.h" #include "kernel_selector_utils.h" +#include "intel_gpu/runtime/debug_configuration.hpp" namespace kernel_selector { @@ -73,6 +74,8 @@ JitConstants SDPAKernelBase::GetJitConstants(const sdpa_params& params) const { jit.AddConstant(MakeJitConstant("DO_BROADCAST_KEY_VALUE", GetBroadcastInputStr(params.inputs[0].GetDims().size(), params.conf.broadcast_axis, params.conf.group_size))); + } else { + jit.AddConstant(MakeJitConstant("BROADCAST_GROUP_SIZE", 1)); } jit.AddConstant(MakeJitConstant("IS_CAUSAL", params.conf.is_causal)); @@ -81,6 +84,21 @@ JitConstants SDPAKernelBase::GetJitConstants(const sdpa_params& params) const { jit.AddConstant(MakeJitConstant("HAS_SCALE_INPUT", params.inputs.size() > 4)); } + jit.AddConstant(MakeJitConstant("IS_KV_COMPRESSED", params.conf.is_kv_compressed)); + + if (params.conf.is_kv_compressed) { + jit.AddConstant(MakeJitConstant("USE_ASYMMETRIC_QUANTIZATION", params.conf.use_asymmetric_quantization)); + jit.AddConstant(MakeJitConstant("COMBINE_SCALES_AND_ZP", params.conf.combine_scales_and_zp)); + jit.AddConstant(MakeJitConstant("COMPRESSED_PER_HEAD", params.conf.per_head_quantization)); + jit.AddConstant(MakeJitConstant("KEY_COMPRESSION_SCALE", params.key_cache_comp_scale)); + jit.AddConstant(MakeJitConstant("VALUE_COMPRESSION_SCALE", params.value_cache_comp_scale)); + + if (params.conf.use_asymmetric_quantization && !params.conf.combine_scales_and_zp) { + jit.AddConstant(MakeJitConstant("KEY_COMPRESSION_ZP", params.key_cache_comp_zp)); + jit.AddConstant(MakeJitConstant("VALUE_COMPRESSION_ZP", params.value_cache_comp_zp)); + } + } + auto is_default_order = [](const std::vector& order) { for (size_t i = 0; i < order.size(); i++) if (order[i] != static_cast(i)) diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_base.h b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_base.h index 492e86ebcce5cc..493bd0acedea32 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_base.h +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_base.h @@ -88,6 +88,10 @@ struct sdpa_configuration { bool is_causal = false; bool has_alibi_input = false; + bool is_kv_compressed = false; + bool use_asymmetric_quantization = false; + bool combine_scales_and_zp = false; + bool per_head_quantization = false; // Paged Attention configuration bool is_paged_attention = false; @@ -110,6 +114,10 @@ struct sdpa_params : public base_params { int64_t indirect_axis = -1; DataTensor beam_table; + DataTensor key_cache_comp_scale; + DataTensor key_cache_comp_zp; + DataTensor value_cache_comp_scale; + DataTensor value_cache_comp_zp; sdpa_configuration conf; }; diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_micro.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_micro.cpp index 974d4532c84e60..a6fa66f4799d3f 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_micro.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_micro.cpp @@ -344,6 +344,9 @@ bool SDPAKernelMicro::Validate(const Params& p) const { if (params.conf.head_size > 256) return false; + if (params.conf.is_kv_compressed) + return false; + return true; } diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_opt.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_opt.cpp index 6942e5f8ea4357..4e71064efbc895 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_opt.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_opt.cpp @@ -4,6 +4,7 @@ #include "sdpa_kernel_opt.h" #include "kernel_selector_utils.h" +#include "common_types.h" #include #include @@ -21,11 +22,16 @@ constexpr size_t subgroup_size = 16; } // namespace static size_t get_sg_number_scale_factor(const sdpa_params& sdpa_params, size_t kernel_type) { + const size_t optimal_scale_factor = 2; if (kernel_type == KernelsTypes::MULTI_TOKENS) { - const size_t optimal_scale_factor = 2; if (sdpa_params.conf.head_size * optimal_scale_factor <= sdpa_params.engineInfo.maxWorkGroupSize) { return optimal_scale_factor; } + } else if (kernel_type == KernelsTypes::SINGLE_TOKEN) { + if (sdpa_params.conf.head_size * optimal_scale_factor <= sdpa_params.engineInfo.maxWorkGroupSize && + sdpa_params.conf.head_size * optimal_scale_factor / subgroup_size <= subgroup_size) { + return optimal_scale_factor; + } } return 1; @@ -126,6 +132,7 @@ static std::string GetKernelName(std::string base_name, KernelsTypes type, const ParamsKey SDPAKernelOpt::GetSupportedKey() const { ParamsKey k; + k.EnableInputDataType(Datatype::INT8); k.EnableInputDataType(Datatype::F16); k.EnableInputDataType(Datatype::F32); k.EnableInputDataType(Datatype::INT32); @@ -154,6 +161,9 @@ bool SDPAKernelOpt::Validate(const Params& p) const { if (params.conf.head_size < 1 || params.conf.head_size % subgroup_size != 0) return false; + if (params.conf.use_asymmetric_quantization && !params.conf.combine_scales_and_zp) + return false; + return true; } @@ -233,10 +243,11 @@ CommonDispatchData SDPAKernelOpt::SetDefault(const sdpa_params& params, size_t k const size_t target_seq_len_block_size = kernel_idx == 1 ? get_target_seq_len_block_size() : 1; if (kernel_idx == KernelsTypes::SINGLE_TOKEN) { + const size_t sg_num_scale = get_sg_number_scale_factor(params, kernel_idx); dispatch_data.gws = { batch_size * heads_num, CeilDiv(target_seq_len, target_seq_len_block_size), - head_size * num_of_partitions }; - dispatch_data.lws = { 1, 1, head_size }; + head_size * num_of_partitions * sg_num_scale }; + dispatch_data.lws = { 1, 1, head_size * sg_num_scale }; } else if (kernel_idx == KernelsTypes::MULTI_TOKENS) { const size_t sg_num_scale = get_sg_number_scale_factor(params, kernel_idx); dispatch_data.gws = { batch_size * heads_num, @@ -309,8 +320,20 @@ KernelsData SDPAKernelOpt::GetKernelsData(const Params& params) const { static_cast(prim_params.outputs.size()), prim_params.is_shape_agnostic); - if (prim_params.indirect_axis != -1 && kernel_idx != KernelsTypes::FINALIZATION) - kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, static_cast(prim_params.inputs.size())}); + auto beam_table_idx = prim_params.inputs.size(); + if (prim_params.conf.is_kv_compressed && kernel_idx != KernelsTypes::FINALIZATION) { + auto key_cache_compression_scale_idx = static_cast(prim_params.inputs.size()); + auto value_cache_compression_scale_idx = static_cast(prim_params.inputs.size() + 1); + + kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, key_cache_compression_scale_idx}); + kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, value_cache_compression_scale_idx}); + + beam_table_idx += 2; + } + + if (prim_params.indirect_axis != -1 && kernel_idx != KernelsTypes::FINALIZATION) { + kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, static_cast(beam_table_idx)}); + } kernel.params.arguments.push_back({ArgumentDescriptor::Types::INTERNAL_BUFFER, 0}); kernel.params.arguments.push_back({ArgumentDescriptor::Types::INTERNAL_BUFFER, 1}); diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_ref.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_ref.cpp index 579c4bc06c17e2..0d551883b6c385 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_ref.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/sdpa/sdpa_kernel_ref.cpp @@ -12,6 +12,7 @@ namespace kernel_selector { ParamsKey SDPAKernelRef::GetSupportedKey() const { ParamsKey k; k.EnableInputDataType(Datatype::F16); + k.EnableInputDataType(Datatype::INT8); k.EnableInputDataType(Datatype::F32); // beam table input k.EnableInputDataType(Datatype::INT32); @@ -74,8 +75,26 @@ KernelsData SDPAKernelRef::GetKernelsData(const Params& params) const { "", false, false, static_cast(prim_params.inputs.size()), GetFusedPrimitiveInputsCount(params), 1, prim_params.is_shape_agnostic); - if (prim_params.indirect_axis != -1) - kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, static_cast(prim_params.inputs.size())}); + auto beam_table_idx = prim_params.inputs.size(); + if (prim_params.conf.is_kv_compressed) { + auto key_cache_compression_scale_idx = static_cast(prim_params.inputs.size()); + auto value_cache_compression_scale_idx = static_cast(prim_params.inputs.size() + 1); + + kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, key_cache_compression_scale_idx}); + kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, value_cache_compression_scale_idx}); + + if (prim_params.conf.use_asymmetric_quantization && !prim_params.conf.combine_scales_and_zp) { + kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, key_cache_compression_scale_idx + 2}); + kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, value_cache_compression_scale_idx + 2}); + beam_table_idx += 2; + } + + beam_table_idx += 2; + } + + if (prim_params.indirect_axis != -1) { + kernel.params.arguments.push_back({ArgumentDescriptor::Types::INPUT, static_cast(beam_table_idx)}); + } kernel.params.arguments.push_back({ArgumentDescriptor::Types::INTERNAL_BUFFER, 0}); diff --git a/src/plugins/intel_gpu/src/plugin/common_utils.cpp b/src/plugins/intel_gpu/src/plugin/common_utils.cpp index 2daeb09c90a5bf..ddd6b5677adc45 100644 --- a/src/plugins/intel_gpu/src/plugin/common_utils.cpp +++ b/src/plugins/intel_gpu/src/plugin/common_utils.cpp @@ -83,9 +83,11 @@ void convert_and_copy(const void* src_ptr, ov::element::Type src_et, void* dst_p // For state conversions CASE(ov::element::f32, ov::element::f32, float, float); - CASE(ov::element::f16, ov::element::f16, ov::float16, ov::float16); CASE(ov::element::f32, ov::element::f16, float, ov::float16); CASE(ov::element::f16, ov::element::f32, ov::float16, float); + CASE(ov::element::f16, ov::element::f16, ov::float16, ov::float16); + CASE(ov::element::bf16, ov::element::f32, ov::bfloat16, float); + CASE(ov::element::bf16, ov::element::f16, ov::bfloat16, ov::float16); OPENVINO_THROW("[GPU] Unsupported element types combination for copy: ", src_et, " -> ", dst_et); } diff --git a/src/plugins/intel_gpu/src/plugin/compiled_model.cpp b/src/plugins/intel_gpu/src/plugin/compiled_model.cpp index 15ff4447b4bafe..18e7a88fc42f3e 100644 --- a/src/plugins/intel_gpu/src/plugin/compiled_model.cpp +++ b/src/plugins/intel_gpu/src/plugin/compiled_model.cpp @@ -5,6 +5,7 @@ #include "openvino/runtime/iplugin.hpp" #include "openvino/runtime/intel_gpu/properties.hpp" #include "openvino/runtime/internal_properties.hpp" +#include "openvino/util/weights_path.hpp" #include "intel_gpu/graph/serialization/binary_buffer.hpp" #include "intel_gpu/runtime/itt.hpp" @@ -169,14 +170,17 @@ std::shared_ptr CompiledModel::create_infer_request() co void CompiledModel::export_model(std::ostream& model) const { // If ov::CacheMode::OPTIMIZE_SIZE is set, do the export iff it's possible to do weightless caching // which requires the weights_path. - if (m_config.get_property(ov::cache_mode) == ov::CacheMode::OPTIMIZE_SIZE && - m_config.get_property(ov::weights_path).empty()) + ov::CacheMode cache_mode = m_config.get_property(ov::cache_mode); + std::string weights_path = m_config.get_property(ov::weights_path); + if (cache_mode == ov::CacheMode::OPTIMIZE_SIZE && + !ov::util::validate_weights_path(weights_path)) return; OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "CompiledModel::export_model"); OPENVINO_ASSERT(!m_graphs.empty(), "[GPU] Model not loaded"); cldnn::BinaryOutputBuffer ob(model); + ob << cldnn::make_data(&cache_mode, sizeof(ov::CacheMode)); // Inputs { @@ -257,6 +261,7 @@ ov::Any CompiledModel::get_property(const std::string& name) const { ov::PropertyName{ov::hint::num_requests.name(), PropertyMutability::RO}, ov::PropertyName{ov::hint::inference_precision.name(), PropertyMutability::RO}, ov::PropertyName{ov::hint::dynamic_quantization_group_size.name(), PropertyMutability::RO}, + ov::PropertyName{ov::hint::activations_scale_factor.name(), PropertyMutability::RO}, ov::PropertyName{ov::device::id.name(), PropertyMutability::RO}, ov::PropertyName{ov::execution_devices.name(), PropertyMutability::RO}, }; @@ -288,5 +293,13 @@ std::shared_ptr CompiledModel::create_sync_infer_request( return std::make_shared(std::static_pointer_cast(shared_from_this())); } + +void CompiledModel::release_memory() { +#ifdef ENABLE_ONEDNN_FOR_GPU + auto capacity = dnnl::get_primitive_cache_capacity(); + dnnl::set_primitive_cache_capacity(0); + dnnl::set_primitive_cache_capacity(capacity); +#endif +} } // namespace intel_gpu } // namespace ov diff --git a/src/plugins/intel_gpu/src/plugin/multi_tensor_variable_state.cpp b/src/plugins/intel_gpu/src/plugin/multi_tensor_variable_state.cpp index 7574b664b6b4b7..8173a29c1b35f8 100644 --- a/src/plugins/intel_gpu/src/plugin/multi_tensor_variable_state.cpp +++ b/src/plugins/intel_gpu/src/plugin/multi_tensor_variable_state.cpp @@ -152,5 +152,64 @@ VariableState::Ptr VariableStateIndirectKVCache::get_beam_table_state() const { return m_hidden_states[1]; } +VariableStateIndirectKVCacheCompressed::VariableStateIndirectKVCacheCompressed( + const VariableStateInfo& info, + std::shared_ptr context, + std::shared_ptr shape_predictor, + const std::vector& output_layouts, + size_t beam_idx, + size_t concat_idx, + bool has_zp_state = false) + : VariableStateIndirectKVCache(info, context, shape_predictor, beam_idx, concat_idx), + m_has_zp_state(has_zp_state) { + OPENVINO_ASSERT((has_zp_state && output_layouts.size() == 3) || + (!has_zp_state && output_layouts.size() == 2), + "[GPU] Unexpected number of output layouts for VariableStateIndirectKVCacheCompressed"); + + const auto compression_scale_layout = output_layouts[1]; + VariableStateInfo compression_scale_state_info(info.m_id + "/comp_scale", compression_scale_layout); + m_hidden_states.push_back(std::make_shared(compression_scale_state_info, context, shape_predictor)); + + if (has_zp_state) { + const auto compression_zp_layout = output_layouts[2]; + VariableStateInfo compression_zp_state_info(info.m_id + "/comp_zp", compression_zp_layout); + m_hidden_states.push_back(std::make_shared(compression_zp_state_info, context, shape_predictor)); + } + + OPENVINO_ASSERT((!m_has_zp_state && m_hidden_states.size() == 3) || (m_has_zp_state && m_hidden_states.size() == 4), + "[GPU] VariableStateIndirectKVCacheCompressed expects 3 or 4 internal states to be initialized, " + "actual number is ", m_hidden_states.size()); +} + +VariableState::Ptr VariableStateIndirectKVCacheCompressed::get_compression_scale_state() const { + return m_hidden_states[2]; +} + +void VariableStateIndirectKVCacheCompressed::set_compression_scale_layout(const cldnn::layout& new_layout) { + m_hidden_states[2]->set_layout(new_layout); +} + +VariableState::Ptr VariableStateIndirectKVCacheCompressed::get_compression_zp_state() const { + OPENVINO_ASSERT(m_has_zp_state); + return m_hidden_states[3]; +} + +void VariableStateIndirectKVCacheCompressed::set_compression_zp_layout(const cldnn::layout& new_layout) { + OPENVINO_ASSERT(m_has_zp_state); + m_hidden_states[3]->set_layout(new_layout); +} + +bool VariableStateIndirectKVCacheCompressed::has_zp_state() const { + return m_has_zp_state; +} + +void VariableStateIndirectKVCacheCompressed::set_state(const ov::SoPtr& state) { + OPENVINO_THROW("[GPU] set_state API is supported only when KV-cache compression is disabled"); +} + +ov::SoPtr VariableStateIndirectKVCacheCompressed::get_state() const { + OPENVINO_THROW("[GPU] get_state API is supported only when KV-cache compression is disabled"); +} + } // namespace intel_gpu } // namespace ov diff --git a/src/plugins/intel_gpu/src/plugin/ops/dynamic_quantize.cpp b/src/plugins/intel_gpu/src/plugin/ops/dynamic_quantize.cpp index 0373251e45c051..85f28cbd711678 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/dynamic_quantize.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/dynamic_quantize.cpp @@ -7,6 +7,7 @@ #include "intel_gpu/plugin/common_utils.hpp" #include "intel_gpu/primitives/dynamic_quantize.hpp" + namespace ov { namespace intel_gpu { @@ -15,15 +16,12 @@ static void CreateDynamicQuantizeOp(ProgramBuilder& p, const std::shared_ptrget_group_sizes(); - for (size_t i = 0; i < group_sizes.size() - 1; i++) - OPENVINO_ASSERT(group_sizes[i] == 1, "Not supported group size at ", i, ": ", group_sizes[i]); - - OPENVINO_ASSERT(group_sizes.back() == UINT64_MAX, "Not supported group size: ", group_sizes.back()); auto prim = cldnn::dynamic_quantize(primitive_name, - inputs[0], - op->get_group_sizes().back(), - get_output_data_types(op)); + inputs[0], + op->get_attrs()); + + prim.num_outputs = op->get_output_size(); + p.add_primitive(*op, prim); } diff --git a/src/plugins/intel_gpu/src/plugin/ops/kv_cache.cpp b/src/plugins/intel_gpu/src/plugin/ops/kv_cache.cpp index c2ee336e48bf06..251c7346db9209 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/kv_cache.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/kv_cache.cpp @@ -3,6 +3,7 @@ // #include "intel_gpu/op/kv_cache.hpp" +#include "intel_gpu/op/kv_cache_compressed.hpp" #include "intel_gpu/plugin/program_builder.hpp" #include "intel_gpu/plugin/common_utils.hpp" #include "intel_gpu/primitives/kv_cache.hpp" @@ -12,6 +13,7 @@ namespace ov { namespace op { namespace internal { using KVCache = ov::intel_gpu::op::KVCache; +using KVCacheCompressed = ov::intel_gpu::op::KVCacheCompressed; } // namespace internal } // namespace op } // namespace ov @@ -26,11 +28,31 @@ void CreateKVCacheOp(ProgramBuilder& p, const std::shared_ptrget_input_partial_shape(0).size(); auto prim = cldnn::kv_cache(layer_type_name_ID(op), - inputs, - op->get_variable()->get_info(), - ov::util::normalize(op->get_concat_axis(), rank), - ov::util::normalize(op->get_gather_axis(), rank), - op->get_indirect()); + inputs, + op->get_variable()->get_info(), + ov::util::normalize(op->get_concat_axis(), rank), + ov::util::normalize(op->get_gather_axis(), rank), + op->get_indirect()); + + prim.num_outputs = op->get_output_size(); + prim.output_data_types = get_output_data_types(op); + + p.add_primitive(*op, prim); +} + +void CreateKVCacheCompressedOp(ProgramBuilder& p, const std::shared_ptr& op) { + validate_inputs_count(op, {4, 5}); + auto inputs = p.GetInputInfo(op); + int64_t rank = op->get_input_partial_shape(0).size(); + auto prim = cldnn::kv_cache(layer_type_name_ID(op), + inputs, + op->get_variable()->get_info(), + ov::util::normalize(op->get_concat_axis(), rank), + ov::util::normalize(op->get_gather_axis(), rank), + op->get_indirect()); + + prim.compressed = true; + prim.quantization_attributes = op->get_quantization_attrs(); prim.num_outputs = op->get_output_size(); prim.output_data_types = get_output_data_types(op); @@ -41,6 +63,7 @@ void CreateKVCacheOp(ProgramBuilder& p, const std::shared_ptr& op) { - validate_inputs_count(op, {4, 5, 6}); auto inputs = p.GetInputInfo(op); auto layerName = layer_type_name_ID(op); bool is_causal = op->get_causal(); + const auto compression_inputs = op->get_compression_inputs_num(); + validate_inputs_count(op, {4 + compression_inputs, 5 + compression_inputs, 6 + compression_inputs}); + int64_t indirect_axis = op->get_indirect_axis(); auto sdpa_prim = cldnn::scaled_dot_product_attention(layerName, inputs, @@ -75,7 +77,9 @@ static void CreateIndirectSDPAOp(ProgramBuilder& p, const std::shared_ptrget_input0_transpose_order(), op->get_input1_transpose_order(), op->get_input2_transpose_order(), - op->get_output_transpose_order()); + op->get_output_transpose_order(), + op->get_quantization_attrs(), + op->get_kv_compressed()); p.add_primitive(*op, sdpa_prim); } diff --git a/src/plugins/intel_gpu/src/plugin/ops/split.cpp b/src/plugins/intel_gpu/src/plugin/ops/split.cpp index 32f0c9a7f36e1d..8a36cfa30995fa 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/split.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/split.cpp @@ -93,20 +93,25 @@ static void CreateCommonSplitOp(ProgramBuilder& p, const std::shared_ptrget_output_size(); i++) { const auto outPartialShape = op->get_output_partial_shape(i); if (outPartialShape.size() != start_offset.size()) { - OPENVINO_THROW("Invalid dimesions in split layer: ", op->get_friendly_name(), + OPENVINO_THROW("Invalid dimensions in split layer: ", op->get_friendly_name(), " output: ", op->get_output_tensor(i).get_any_name()); } for (size_t idx = 0; idx < input_pshape.size(); idx++) { if ((outPartialShape[idx].get_length() + static_cast(start_offset[idx])) > input_pshape[idx].get_length()) { - OPENVINO_THROW("Invalid dimesions in split layer: ", op->get_friendly_name(), + OPENVINO_THROW("Invalid dimensions in split layer: ", op->get_friendly_name(), " output: ", op->get_output_tensor(idx).get_any_name()); } } auto offsetTensor = tensor_from_dims(start_offset, 0); auto outTensor = tensor_from_dims(op->get_output_shape(i), 1); - auto cropPrim = cldnn::crop(get_layer_name(i), inputs[0], outTensor, offsetTensor); - p.add_primitive(*op, cropPrim); + + const auto& users = op->get_output_target_inputs(i); + // don't add crop primitive if port is not used by anyone + if (users.size() != 0) { + auto cropPrim = cldnn::crop(get_layer_name(i), inputs[0], outTensor, offsetTensor); + p.add_primitive(*op, cropPrim); + } for (size_t idx = 0; idx < input_pshape.size(); idx++) { if (outPartialShape[idx] != input_pshape[idx]) { diff --git a/src/plugins/intel_gpu/src/plugin/ops/variable.cpp b/src/plugins/intel_gpu/src/plugin/ops/variable.cpp index d655e297e4a2c6..a4354c51092ac8 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/variable.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/variable.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // + #include "intel_gpu/plugin/program_builder.hpp" #include "intel_gpu/plugin/common_utils.hpp" #include "openvino/core/type/element_type.hpp" @@ -9,6 +10,7 @@ #include "openvino/op/read_value.hpp" #include "transformations/rt_info/original_precision_attribute.hpp" #include "intel_gpu/op/read_value.hpp" +#include "intel_gpu/op/read_values.hpp" #include "intel_gpu/primitives/assign.hpp" #include "intel_gpu/primitives/read_value.hpp" @@ -16,6 +18,7 @@ namespace ov { namespace op { namespace internal { using ReadValue = ov::intel_gpu::op::ReadValue; +using ReadValues = ov::intel_gpu::op::ReadValues; } // namespace internal } // namespace op } // namespace ov @@ -39,7 +42,7 @@ void CreateVariableAccessPrimitive(ProgramBuilder &p, const std::shared_ptr CreateVariableAccessPrimitive(p, op, op->get_variable_id()); } +void CreateReadValuesOp(ProgramBuilder& p, const std::shared_ptr& op) { + std::vector variable_layouts; + for (size_t i = 0; i < op->get_output_size(); i++) { + const auto output_pshape = op->get_output_partial_shape(i); + const auto output_dtype = cldnn::element_type_to_data_type(op->get_output_element_type(i)); + const auto output_format = cldnn::format::get_default_format(output_pshape.size()); + variable_layouts.emplace_back(output_pshape, output_dtype, output_format); + } + + auto inputs = p.GetInputInfo(op); + auto user_specified_type = get_original_precision(op); + auto prim = cldnn::read_value{layer_type_name_ID(op), + inputs, + op->get_variable_id(), + variable_layouts, + user_specified_type}; + + p.add_primitive(*op, prim); +} + } // namespace REGISTER_FACTORY_IMPL(v3, Assign); @@ -89,6 +112,7 @@ REGISTER_FACTORY_IMPL(v6, Assign); REGISTER_FACTORY_IMPL(v3, ReadValue); REGISTER_FACTORY_IMPL(v6, ReadValue); REGISTER_FACTORY_IMPL(internal, ReadValue); +REGISTER_FACTORY_IMPL(internal, ReadValues); } // namespace intel_gpu } // namespace ov diff --git a/src/plugins/intel_gpu/src/plugin/plugin.cpp b/src/plugins/intel_gpu/src/plugin/plugin.cpp index 9aba7ee1a117eb..7d010a9b590e2e 100644 --- a/src/plugins/intel_gpu/src/plugin/plugin.cpp +++ b/src/plugins/intel_gpu/src/plugin/plugin.cpp @@ -35,6 +35,7 @@ #include "openvino/runtime/performance_heuristics.hpp" #include "openvino/runtime/properties.hpp" #include "openvino/util/common_util.hpp" +#include "openvino/util/weights_path.hpp" #include "transformations/common_optimizations/dimension_tracking.hpp" #include "transformations/init_node_info.hpp" #include "transformations/rt_info/fused_names_attribute.hpp" @@ -330,8 +331,16 @@ std::shared_ptr Plugin::import_model(std::istream& model, cldnn::BinaryInputBuffer ib(model, context_impl->get_engine()); + ov::CacheMode cache_mode; + ib >> cldnn::make_data(&cache_mode, sizeof(ov::CacheMode)); + + if (cache_mode != config.get_property(ov::cache_mode)) { + return nullptr; + } + + std::string weights_path = config.get_property(ov::weights_path); if (config.get_property(ov::cache_mode) == ov::CacheMode::OPTIMIZE_SIZE && - config.get_property(ov::weights_path).empty()) { + !ov::util::validate_weights_path(weights_path)) { return nullptr; } @@ -540,6 +549,7 @@ std::vector Plugin::get_caching_properties() const { ov::PropertyName{ov::hint::execution_mode.name(), PropertyMutability::RW}, ov::PropertyName{ov::hint::performance_mode.name(), PropertyMutability::RW}, ov::PropertyName{ov::hint::dynamic_quantization_group_size.name(), PropertyMutability::RW}, + ov::PropertyName{ov::hint::activations_scale_factor.name(), PropertyMutability::RW}, }; return caching_properties; @@ -585,7 +595,8 @@ std::vector Plugin::get_supported_properties() const { ov::PropertyName{ov::hint::inference_precision.name(), PropertyMutability::RW}, ov::PropertyName{ov::hint::enable_cpu_pinning.name(), PropertyMutability::RW}, ov::PropertyName{ov::device::id.name(), PropertyMutability::RW}, - ov::PropertyName{ov::hint::dynamic_quantization_group_size.name(), PropertyMutability::RW} + ov::PropertyName{ov::hint::dynamic_quantization_group_size.name(), PropertyMutability::RW}, + ov::PropertyName{ov::hint::activations_scale_factor.name(), PropertyMutability::RW} }; return supported_properties; diff --git a/src/plugins/intel_gpu/src/plugin/program_builder.cpp b/src/plugins/intel_gpu/src/plugin/program_builder.cpp index 510d715e7ac805..899110872ba633 100644 --- a/src/plugins/intel_gpu/src/plugin/program_builder.cpp +++ b/src/plugins/intel_gpu/src/plugin/program_builder.cpp @@ -305,12 +305,15 @@ void ProgramBuilder::add_primitive(const ov::Node& op, std::shared_ptrorigin_op_name = op.get_friendly_name(); prim->origin_op_type_name = op.get_type_name(); - if (auto data_prim = dynamic_cast(prim.get())) { - auto rt_info = op.get_rt_info(); - auto weightless_cache_attr = rt_info.find(ov::WeightlessCacheAttribute::get_type_info_static()); - if (weightless_cache_attr != rt_info.end()) { - data_prim->bin_offset = weightless_cache_attr->second.as().bin_offset; - data_prim->original_size = weightless_cache_attr->second.as().original_size; + if (this->m_config.get_property(ov::cache_mode) == ov::CacheMode::OPTIMIZE_SIZE) { + if (auto data_prim = dynamic_cast(prim.get())) { + auto rt_info = op.get_rt_info(); + auto weightless_cache_attr = rt_info.find(ov::WeightlessCacheAttribute::get_type_info_static()); + if (weightless_cache_attr != rt_info.end()) { + data_prim->bin_offset = weightless_cache_attr->second.as().bin_offset; + data_prim->original_size = + weightless_cache_attr->second.as().original_size; + } } } diff --git a/src/plugins/intel_gpu/src/plugin/sync_infer_request.cpp b/src/plugins/intel_gpu/src/plugin/sync_infer_request.cpp index 58e99e037fb931..985336b801b9d3 100644 --- a/src/plugins/intel_gpu/src/plugin/sync_infer_request.cpp +++ b/src/plugins/intel_gpu/src/plugin/sync_infer_request.cpp @@ -8,6 +8,7 @@ #include "openvino/core/validation_util.hpp" #include "intel_gpu/primitives/kv_cache.hpp" +#include "intel_gpu/primitives/read_value.hpp" #include "intel_gpu/plugin/usm_host_tensor.hpp" #include "intel_gpu/plugin/sync_infer_request.hpp" #include "intel_gpu/plugin/remote_context.hpp" @@ -591,6 +592,12 @@ void SyncInferRequest::allocate_input(const ov::Output& port, si auto element_type = port.get_element_type(); m_user_inputs[input_idx] = { create_host_tensor(shape, element_type), TensorOwner::PLUGIN }; + if (element_type == ov::element::string) { + // In case the element type is string and input data is an empty string, + // it produces the segmentation fault unless the each element of tensor.data is initialized. + auto data = m_user_inputs.at(input_idx).ptr->data(); + std::uninitialized_fill_n(data, m_user_inputs.at(input_idx).ptr->get_size(), std::string()); + } ov::ISyncInferRequest::set_tensor(port, m_user_inputs.at(input_idx).ptr); } @@ -646,19 +653,40 @@ void SyncInferRequest::allocate_states() { bool indirect_kv_cache = false; int64_t beam_axis = 0; int64_t concat_axis = 0; + bool compressed = false; + bool has_zp_state = false; auto kv_cache_shape = vi.second.m_layout.get_partial_shape(); + std::vector states_layouts; for (auto& p : state_prims) { if (auto kv_cache_prim = dynamic_cast(p)) { indirect_kv_cache = kv_cache_prim->indirect; beam_axis = ov::util::normalize(kv_cache_prim->gather_axis, kv_cache_shape.size()); concat_axis = ov::util::normalize(kv_cache_prim->concat_axis, kv_cache_shape.size()); + compressed = kv_cache_prim->compressed; + has_zp_state = kv_cache_prim->get_compression_zp_inputs_num() > 0; + } else if (auto read_value = dynamic_cast(p)) { + states_layouts = read_value->output_layouts; } } - if (indirect_kv_cache) { - m_variables.emplace(vi.first, std::make_shared(vi.second, m_context, m_shape_predictor, beam_axis, concat_axis)); + if (compressed) { + m_variables.emplace(vi.first, std::make_shared(vi.second, + m_context, + m_shape_predictor, + states_layouts, + beam_axis, + concat_axis, + has_zp_state)); + } else if (indirect_kv_cache) { + m_variables.emplace(vi.first, std::make_shared(vi.second, + m_context, + m_shape_predictor, + beam_axis, + concat_axis)); } else { - m_variables.emplace(vi.first, std::make_shared(vi.second, m_context, m_shape_predictor)); + m_variables.emplace(vi.first, std::make_shared(vi.second, + m_context, + m_shape_predictor)); } } } diff --git a/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp b/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp index 315a93190fdc90..0ed17f36c21d22 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/convert_fc_to_compressed.cpp @@ -57,13 +57,16 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon auto reshape_const_m = wrap_type(); auto reshape_m = wrap_type({mul_m, reshape_const_m}, reshape_3d_to_2d); + auto mul2_const_m = wrap_type(); + auto mul2_m = wrap_type({reshape_m, mul2_const_m}); + auto transpose_input = std::make_shared(OutputVector{reshape_m, mul_m}); auto transpose_const_m = wrap_type(); auto transpose_m = wrap_type({transpose_input, transpose_const_m}); auto data_m = any_input(); auto bias_m = any_input(); - auto weights_input_m = std::make_shared(ov::OutputVector{reshape_m, transpose_m, mul_m}); + auto weights_input_m = std::make_shared(ov::OutputVector{reshape_m, transpose_m, mul_m, mul2_m}); auto fully_connected_m = wrap_type({data_m, weights_input_m, bias_m}); ov::matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](ov::pass::pattern::Matcher& m) { @@ -131,6 +134,7 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon std::shared_ptr fc_input_zp = optional_zero_point; std::shared_ptr fc_input_bias = pattern_map.at(bias_m).get_node_shared_ptr(); std::vector> result_nodes = {}; + if (has_transpose) { const auto& transpose = pattern_map.at(transpose_m).get_node_shared_ptr(); std::shared_ptr transpose_const = pattern_map.at(transpose_const_m).get_node_shared_ptr(); @@ -151,6 +155,11 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon } } + if (pattern_map.count(mul2_m)) { + auto mul2_op_const = std::dynamic_pointer_cast(pattern_map.at(mul2_const_m).get_node_shared_ptr()); + fc_input_scale = ov::op::util::eltwise_fold(fc_input_scale, mul2_op_const).get_node_shared_ptr(); + } + std::shared_ptr new_fc = nullptr; if (with_zero_point) { new_fc = std::make_shared(fc_input_a, @@ -171,6 +180,7 @@ ConvertFullyConnectedToFullyConnectedCompressed::ConvertFullyConnectedToFullyCon new_fc->set_friendly_name(fc->get_friendly_name()); ov::copy_runtime_info(m.get_matched_nodes(), result_nodes); ov::replace_node(fc, new_fc); + return true; }; diff --git a/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp b/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp index eb16213bcb936c..68328160a98f82 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/dynamic_quantize_fully_connected.cpp @@ -61,7 +61,14 @@ DynamicQuantizeFullyConnected::DynamicQuantizeFullyConnected(uint64_t group_size auto rank = m_fc->get_input_partial_shape(0).size(); std::vector shape_group_size(rank, 1); shape_group_size.back() = group_size; - auto dyn_quan = std::make_shared(m_data, shape_group_size, element::f16); + + ov::op::internal::DynamicQuantize::Attributes config; + config.quantization_dt = element::i8; + config.quantization_type = ov::op::internal::DynamicQuantize::QuantizationType::Symmetric; + config.scale_dt = element::f16; + config.group_sizes = shape_group_size; + + auto dyn_quan = std::make_shared(m_data, config); auto optional_w_zp = m_fc->get_input_size() > 4 ? m_fc->get_input_node_shared_ptr(4) : std::make_shared(); auto output_type = m_fc->get_output_type(); diff --git a/src/plugins/intel_gpu/src/plugin/transformations/fc_per_layer_scaling.cpp b/src/plugins/intel_gpu/src/plugin/transformations/fc_per_layer_scaling.cpp new file mode 100644 index 00000000000000..618578919d4024 --- /dev/null +++ b/src/plugins/intel_gpu/src/plugin/transformations/fc_per_layer_scaling.cpp @@ -0,0 +1,81 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "fc_per_layer_scaling.hpp" + +#include "intel_gpu/op/fully_connected_compressed.hpp" +#include "intel_gpu/op/placeholder.hpp" + +#include "openvino/op/multiply.hpp" +#include "openvino/core/rt_info.hpp" +#include "openvino/pass/pattern/op/pattern.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "openvino/pass/pattern/op/or.hpp" +#include "transformations/utils/utils.hpp" + +namespace ov { +namespace intel_gpu { + +FullyConnectedPerLayerScaling::FullyConnectedPerLayerScaling(float scale_factor) { + using namespace ov::pass::pattern; + + auto data_m = any_input(); + auto weights_m = any_input(); + auto bias_m = any_input(); + auto fc_compressed_wo_zp_m = wrap_type({data_m, weights_m, bias_m, any_input()}, consumers_count(1)); + auto fc_compressed_w_zp_m = wrap_type({data_m, weights_m, bias_m, any_input(), any_input()}, consumers_count(1)); + auto fc_compressed_m = std::make_shared(OutputVector{fc_compressed_wo_zp_m, fc_compressed_w_zp_m}); + + ov::matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](Matcher& m) { + if (scale_factor == 0.f || scale_factor == 1.f) + return false; + auto fc = std::dynamic_pointer_cast(m.get_match_root()); + if (!fc || transformation_callback(fc)) + return false; + + const auto& pattern_map = m.get_pattern_value_map(); + const auto& data = pattern_map.at(data_m).get_node_shared_ptr(); + const auto& bias = pattern_map.at(bias_m).get_node_shared_ptr(); + + ov::Shape scale_const_shape = {1}; + std::vector scale_down_value = {(1.f / scale_factor)}; + std::vector scale_up_value = {scale_factor}; + std::shared_ptr scale_down_const_f16 = std::make_shared(ov::element::f16, scale_const_shape, scale_down_value); + std::shared_ptr scale_down_const_f32 = std::make_shared(ov::element::f32, scale_const_shape, scale_down_value); + std::shared_ptr scale_up_const_f16 = std::make_shared(ov::element::f16, scale_const_shape, scale_up_value); + std::shared_ptr scale_up_const_f32 = std::make_shared(ov::element::f32, scale_const_shape, scale_up_value); + + std::shared_ptr scale_down_const = (data->get_element_type() == ov::element::f16) ? scale_down_const_f16 : scale_down_const_f32; + auto scale_down = std::make_shared(data, scale_down_const); + scale_down->set_friendly_name(fc->get_friendly_name() + "_scale_down"); + ov::copy_runtime_info(fc, scale_down); + fc->input(0).replace_source_output(scale_down); + + // If FC has bias as input, scaling must be applied to bias as well + if (!std::dynamic_pointer_cast(bias)) { + std::shared_ptr bias_scale_down_const = (bias->get_element_type() == ov::element::f16) ? scale_down_const_f16 : scale_down_const_f32; + auto bias_scale_down = std::make_shared(bias, bias_scale_down_const); + bias_scale_down->set_friendly_name(fc->get_friendly_name() + "_bias_scale_down"); + ov::copy_runtime_info(fc, bias_scale_down); + fc->input(2).replace_source_output(bias_scale_down); + } + + auto target_inputs = fc->get_output_target_inputs(0); + std::shared_ptr scale_up_const = (fc->get_element_type() == ov::element::f16) ? scale_up_const_f16 : scale_up_const_f32; + auto scale_up = std::make_shared(fc, scale_up_const); + scale_up->set_friendly_name(fc->get_friendly_name() + "_scale_up"); + ov::copy_runtime_info(fc, scale_up); + for (auto& in : target_inputs) { + in.replace_source_output(scale_up); + } + + return true; + }; + + auto m = std::make_shared(fc_compressed_m, "FullyConnectedPerLayerScaling"); + this->register_matcher(m, callback); +} + +} // namespace intel_gpu +} // namespace ov diff --git a/src/plugins/intel_gpu/src/plugin/transformations/fc_per_layer_scaling.hpp b/src/plugins/intel_gpu/src/plugin/transformations/fc_per_layer_scaling.hpp new file mode 100644 index 00000000000000..5c0d7d07f5b411 --- /dev/null +++ b/src/plugins/intel_gpu/src/plugin/transformations/fc_per_layer_scaling.hpp @@ -0,0 +1,19 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/graph_rewrite.hpp" + +namespace ov { +namespace intel_gpu { + +class FullyConnectedPerLayerScaling: public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("FullyConnectedPerLayerScaling", "0"); + FullyConnectedPerLayerScaling(float scale_factor); +}; + +} // namespace intel_gpu +} // namespace ov diff --git a/src/plugins/intel_gpu/src/plugin/transformations/kv_cache_compression.cpp b/src/plugins/intel_gpu/src/plugin/transformations/kv_cache_compression.cpp new file mode 100644 index 00000000000000..561822f9661109 --- /dev/null +++ b/src/plugins/intel_gpu/src/plugin/transformations/kv_cache_compression.cpp @@ -0,0 +1,292 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "kv_cache_compression.hpp" + +#include "intel_gpu/op/kv_cache.hpp" +#include "intel_gpu/op/kv_cache_compressed.hpp" +#include "intel_gpu/op/indirect_sdpa.hpp" +#include "intel_gpu/op/read_value.hpp" +#include "intel_gpu/op/read_values.hpp" +#include "intel_gpu/plugin/common_utils.hpp" +#include "intel_gpu/runtime/debug_configuration.hpp" +#include "ov_ops/dynamic_quantize.hpp" + +#include "openvino/core/node_vector.hpp" +#include "openvino/core/rt_info.hpp" +#include "openvino/op/concat.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/convert.hpp" +#include "openvino/op/gather.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/sink.hpp" +#include "openvino/op/transpose.hpp" +#include "openvino/op/scaled_dot_product_attention.hpp" +#include "openvino/pass/graph_rewrite.hpp" +#include "openvino/pass/pattern/op/label.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "openvino/pass/pattern/op/or.hpp" +#include "openvino/pass/visualize_tree.hpp" +#include "transformations/utils/utils.hpp" + +#include + +namespace ov { +namespace intel_gpu { + +namespace { +std::vector get_variable_infos(const ov::op::util::VariableInfo& data_variable_info, + const ov::op::internal::DynamicQuantize::Attributes& quantization_attrs) { + std::vector infos; + + // Add initial data variable info + infos.push_back(data_variable_info); + + // Infer DQ shapes + ov::op::internal::DynamicQuantize dq; + dq.set_attrs(quantization_attrs); + + auto dq_shapes = ov::op::internal::DynamicQuantize::shape_infer(&dq, {data_variable_info.data_shape}); + + const auto variable_id = data_variable_info.variable_id; + const auto scale_shape = dq_shapes[1]; + const auto scale_dt = quantization_attrs.scale_dt; + + // Add scales variable info + infos.push_back(ov::op::util::VariableInfo{scale_shape, scale_dt, variable_id}); + + if (quantization_attrs.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric && + quantization_attrs.output_storage_type == ov::op::internal::DynamicQuantize::OutputStorageType::Planar) { + // Add zero points variable info + const auto zp_dt = quantization_attrs.zp_dt; + infos.push_back(ov::op::util::VariableInfo{scale_shape, zp_dt, variable_id}); + } + + return infos; +} + +std::shared_ptr + update_past_read_value(std::shared_ptr past_rv_node, + const ov::op::internal::DynamicQuantize::Attributes& quantization_attrs) { + auto variable = past_rv_node->get_variable(); + variable->update_data_type(quantization_attrs.quantization_dt); + + auto variable_infos = get_variable_infos(past_rv_node->get_variable()->get_info(), quantization_attrs); + auto new_past_rv_node = std::make_shared(); + + if (past_rv_node->get_input_size() == 0) { + new_past_rv_node = std::make_shared(past_rv_node->get_variable(), variable_infos); + } else { + auto initializer_dq = std::make_shared(past_rv_node->get_input_node_shared_ptr(0), + quantization_attrs); + initializer_dq->set_friendly_name(past_rv_node->get_input_node_shared_ptr(0)->get_friendly_name() + "_dyn_quan"); + ov::copy_runtime_info(past_rv_node->get_input_node_shared_ptr(0), initializer_dq); + + OutputVector initializer_outputs = { initializer_dq->output(0), initializer_dq->output(1) }; + + if (quantization_attrs.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric && + quantization_attrs.output_storage_type == ov::op::internal::DynamicQuantize::OutputStorageType::Planar) + initializer_outputs.push_back(initializer_dq->output(2)); + + new_past_rv_node = std::make_shared(initializer_outputs, past_rv_node->get_variable(), variable_infos); + } + + ov::copy_runtime_info(past_rv_node, new_past_rv_node); + past_rv_node->output(0).replace(new_past_rv_node->output(0)); + + return new_past_rv_node; +} + +std::shared_ptr + update_kv_cache(std::shared_ptr past_rv_node, + std::shared_ptr kv_cache_node, + const ov::op::internal::DynamicQuantize::Attributes& quantization_attrs) { + OutputVector kv_cache_inputs = { past_rv_node->output(0), + kv_cache_node->get_input_node_shared_ptr(1), + kv_cache_node->get_input_node_shared_ptr(2), + past_rv_node->output(1) }; + + if (quantization_attrs.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric && + quantization_attrs.output_storage_type == ov::op::internal::DynamicQuantize::OutputStorageType::Planar) + kv_cache_inputs.push_back(past_rv_node->output(2)); + + auto new_kv_cache = std::make_shared(kv_cache_inputs, + kv_cache_node->get_variable(), + kv_cache_node->get_concat_axis(), + kv_cache_node->get_gather_axis(), + quantization_attrs); + + new_kv_cache->set_friendly_name(kv_cache_node->get_friendly_name()); + ov::copy_runtime_info(kv_cache_node, new_kv_cache); + + return new_kv_cache; +} +} // namespace + +class KVCacheCompressionMatcher : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("KVCacheCompressionMatcher", "0"); + KVCacheCompressionMatcher(ov::element::Type compression_dt); +}; + +KVCacheCompressionMatcher::KVCacheCompressionMatcher(ov::element::Type compression_dt) { + using namespace ov::pass::pattern; + + if (compression_dt != element::i8) + return; + + const auto quantization_type = ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric; + const auto output_storage_type = ov::op::internal::DynamicQuantize::OutputStorageType::InterleavedScalesZP; + + bool combine_scales_and_zp = output_storage_type == ov::op::internal::DynamicQuantize::OutputStorageType::InterleavedScalesZP; + GPU_DEBUG_LOG << "KV-cache compression configuration: " + << "dt=" << compression_dt << ", " + << "asym=" << (quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric) << ", " + << "single_buffer_for_scales_and_zp=" << combine_scales_and_zp << "\n"; + + auto query = any_input(); + + auto key_past = wrap_type(); + auto key_new_token = any_input(); + auto key_beam_idx = any_input(); + auto key_cache = wrap_type({key_past, key_new_token, key_beam_idx}); + + auto value_past = wrap_type(); + auto value_new_token = any_input(); + auto value_beam_idx = any_input(); + auto value_cache = wrap_type({value_past, value_new_token, value_beam_idx}); + + auto input_attn_mask = any_input(); + auto input_scale = any_input(); + auto input_beam_table = any_input(); + + auto sdpa_without_attn_mask_m = wrap_type({ query, key_cache, value_cache, input_beam_table }); + auto sdpa_with_attn_mask_m = wrap_type({ query, key_cache, value_cache, input_attn_mask, input_beam_table }); + auto sdpa_with_attn_mask_and_scale_m = + wrap_type({ query, key_cache, value_cache, input_attn_mask, input_scale, input_beam_table }); + + auto sdpa = std::make_shared(OutputVector{sdpa_without_attn_mask_m, sdpa_with_attn_mask_m, sdpa_with_attn_mask_and_scale_m}); + + ov::matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](ov::pass::pattern::Matcher& m) { + if (transformation_callback(m.get_match_root())) { + return false; + } + + const auto& pattern_map = m.get_pattern_value_map(); + + auto query_node = pattern_map.at(query).get_node_shared_ptr(); + + auto key_new_token_node = pattern_map.at(key_new_token).get_node_shared_ptr(); + auto key_cache_node = std::dynamic_pointer_cast(pattern_map.at(key_cache).get_node_shared_ptr()); + auto value_cache_node = std::dynamic_pointer_cast(pattern_map.at(value_cache).get_node_shared_ptr()); + auto sdpa_node = std::dynamic_pointer_cast(m.get_match_root()); + + auto key_past_rv_node = std::dynamic_pointer_cast(pattern_map.at(key_past).get_node_shared_ptr()); + auto value_past_rv_node = std::dynamic_pointer_cast(pattern_map.at(value_past).get_node_shared_ptr()); + + auto data_rank = key_cache_node->get_input_partial_shape(0).size(); + auto get_shape_group_sizes = [&](const std::vector& transposed_order) { + std::vector group_sizes(data_rank, 1); + std::vector order = transposed_order; + if (transposed_order.size() != data_rank) { + order.resize(data_rank); + std::iota(order.begin(), order.end(), 0); + } + + group_sizes[order[data_rank - 1]] = UINT64_MAX; + + return group_sizes; + }; + + // Reorder scales in static order: [batch, num_heads, seq_len, head_size] + auto get_scales_output_order = [&](const std::vector& transposed_order) { + std::vector scales_zp_output_order(data_rank); + scales_zp_output_order[0] = transposed_order[0]; + scales_zp_output_order[1] = transposed_order[1]; + scales_zp_output_order[2] = transposed_order[2]; + scales_zp_output_order[3] = transposed_order[3]; + + return scales_zp_output_order; + }; + + ov::op::internal::DynamicQuantize::Attributes config; + config.quantization_type = quantization_type; + config.group_sizes = get_shape_group_sizes(sdpa_node->get_input1_transpose_order()); + config.quantization_dt = element::i8; + config.scale_dt = query_node->get_output_element_type(0); + config.scales_zp_output_order = get_scales_output_order(sdpa_node->get_input1_transpose_order()); + config.output_storage_type = output_storage_type; + + if (config.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric) + config.zp_dt = query_node->get_output_element_type(0); + + key_past_rv_node = update_past_read_value(key_past_rv_node, config); + value_past_rv_node = update_past_read_value(value_past_rv_node, config); + + auto new_key_cache = update_kv_cache(key_past_rv_node, key_cache_node, config); + auto new_value_cache = update_kv_cache(value_past_rv_node, value_cache_node, config); + + OutputVector sdpa_inputs; + // Add Query, Key, Value, attention_mask, scale inputs + for (size_t i = 0; i < sdpa_node->get_input_size() - 1; i++) + sdpa_inputs.push_back(sdpa_node->get_input_node_shared_ptr(i)); + + // Replace Key and Value inputs with compressed ones + sdpa_inputs[1] = new_key_cache->output(0); + sdpa_inputs[2] = new_value_cache->output(0); + + // Add Key and Value compression scales + sdpa_inputs.push_back(new_key_cache->output(2)); + sdpa_inputs.push_back(new_value_cache->output(2)); + + // Add Key and Value compression zero points + if (config.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric && + config.output_storage_type == ov::op::internal::DynamicQuantize::OutputStorageType::Planar) { + sdpa_inputs.push_back(new_key_cache->output(3)); + sdpa_inputs.push_back(new_value_cache->output(3)); + } + + auto input0_transpose_order = sdpa_node->get_input0_transpose_order(); + auto input1_transpose_order = sdpa_node->get_input1_transpose_order(); + auto input2_transpose_order = sdpa_node->get_input2_transpose_order(); + auto output_transpose_order = sdpa_node->get_output_transpose_order(); + + auto new_sdpa = std::make_shared(sdpa_inputs, + new_key_cache->output(1), + sdpa_node->get_causal(), + sdpa_node->get_indirect_axis(), + input0_transpose_order, + input1_transpose_order, + input2_transpose_order, + output_transpose_order, + config, + sdpa_node->get_output_type()); + + new_key_cache->set_friendly_name(key_cache_node->get_friendly_name()); + ov::copy_runtime_info(key_cache_node, new_key_cache); + + new_value_cache->set_friendly_name(value_cache_node->get_friendly_name()); + ov::copy_runtime_info(value_cache_node, new_value_cache); + + new_sdpa->set_friendly_name(sdpa_node->get_friendly_name()); + ov::copy_runtime_info(sdpa_node, new_sdpa); + + ov::replace_node(sdpa_node, new_sdpa); + return true; + }; + + auto m = std::make_shared(sdpa, "KVCacheCompressionMatcher"); + this->register_matcher(m, callback); +} + +bool KVCacheCompression::run_on_model(const std::shared_ptr& m) { + return pass::GraphRewrite::run_on_model(m); +} + +KVCacheCompression::KVCacheCompression(ov::element::Type compression_dt) { + add_matcher(compression_dt); +} + +} // namespace intel_gpu +} // namespace ov diff --git a/src/plugins/intel_gpu/src/plugin/transformations/kv_cache_compression.hpp b/src/plugins/intel_gpu/src/plugin/transformations/kv_cache_compression.hpp new file mode 100644 index 00000000000000..1587021a03ed36 --- /dev/null +++ b/src/plugins/intel_gpu/src/plugin/transformations/kv_cache_compression.hpp @@ -0,0 +1,43 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/graph_rewrite.hpp" + +namespace ov { +namespace intel_gpu { + + +/// Add dynamic quantization node and fuse it with KV cache operation +/// +/// ┌───────────┐ ┌─────────────┐ ┌───────────┐ ┌─────────────┐ +/// │ New Key │ │ New Value │ │ New Key │ │ New Value │ +/// └──────┬────┘ └──────┬──────┘ └──────┬────┘ └──────┬──────┘ +/// │ │ │ │ +/// │ f16 │ f16 │ f16 │ f16 +/// │ │ ==> │ │ +/// ┌─────────┐ ┌────────┴─────────┐ ┌────────┴───────────┐ ┌─────────┐ ┌────────┴─────────┐ ┌────────┴───────────┐ +/// │ Query │ │ KV cache │ │ KV cache │ │ Query │ │ KV cache + DQ │ │ KV cache + DQ │ +/// | | | (Key) | (Value) | | | | (Key) | | (Value) | +/// └───┬─────┘ └────────┬─────────┘ └────────┬───────────┘ └────┬────┘ └────────┬─────────┘ └────────┬───────────┘ +/// │ │ │ │ │ │ +/// │ f16 │ f16 │ f16 │ f16 i8:data │ f16:scale i8:data │ f16:scale +/// │ │ │ │ │ │ +/// │ │ │ │ │ │ +/// │ ┌────┴───┐ │ │ ┌────┴───┐ │ +/// └─────────────┤ SDPA ├─────────────────┘ └─────────────┤ SDPA ├────────────────────┘ +/// └────────┘ └────────┘ + +class KVCacheCompression : public ov::pass::GraphRewrite { +public: + OPENVINO_RTTI("KVCacheCompression", "0"); + KVCacheCompression(ov::element::Type compression_dt); + + bool run_on_model(const std::shared_ptr& m) override; +}; + + +} // namespace intel_gpu +} // namespace ov diff --git a/src/plugins/intel_gpu/src/plugin/transformations/op/indirect_sdpa.cpp b/src/plugins/intel_gpu/src/plugin/transformations/op/indirect_sdpa.cpp index 681c88119efd95..73e916064a0c1c 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/op/indirect_sdpa.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/op/indirect_sdpa.cpp @@ -25,30 +25,63 @@ IndirectSDPA::IndirectSDPA(const OutputVector& data_inputs, validate_and_infer_types(); } +IndirectSDPA::IndirectSDPA(const OutputVector& data_inputs, + const ov::Output& beam_table, + const bool is_causal, + const int64_t indirect_axis, + const std::vector& order_q, + const std::vector& order_k, + const std::vector& order_v, + const std::vector& order_out, + const QuantizationAttribute& quantization_attribute, + const ov::element::Type output_type) + : ov::intel_gpu::op::SDPA(data_inputs, is_causal, order_q, order_k, order_v, order_out, quantization_attribute, output_type) + , m_indirect_axis(indirect_axis) { + auto beam_table_idx = data_inputs.size(); + set_argument(beam_table_idx, beam_table); + validate_and_infer_types(); +} + std::shared_ptr IndirectSDPA::clone_with_new_inputs(const ov::OutputVector& new_args) const { check_new_args_count(this, new_args); // Exclude beam_table input OutputVector data_inputs(new_args.begin(), new_args.end() - 1); - return std::make_shared(data_inputs, - new_args.back(), - m_is_causal, - m_indirect_axis, - m_order_q, - m_order_k, - m_order_v, - m_order_out, - m_output_type); + if (m_compressed) { + return std::make_shared(data_inputs, + new_args.back(), + m_is_causal, + m_indirect_axis, + m_order_q, + m_order_k, + m_order_v, + m_order_out, + m_output_type); + } else { + return std::make_shared(data_inputs, + new_args.back(), + m_is_causal, + m_indirect_axis, + m_order_q, + m_order_k, + m_order_v, + m_order_out, + m_quantization_attrs, + m_output_type); + } } void IndirectSDPA::validate_and_infer_types() { const auto input_size = get_input_size(); + + const auto compression_inputs = get_compression_inputs_num(); NODE_VALIDATION_CHECK(this, - input_size == 4 || input_size == 5 || input_size == 6, + input_size >= 4 + compression_inputs && input_size <= 6 + compression_inputs, "Number of inputs is incorrect. Current value is: ", input_size, - ", expected 4, 5 or 6."); + ", expected 4, 5 or 6 data inputs and ", compression_inputs, " KV-cache compression related inputs"); + std::vector input_shapes; for (size_t i = 0; i < input_size - 1; i++) { diff --git a/src/plugins/intel_gpu/src/plugin/transformations/op/kv_cache.cpp b/src/plugins/intel_gpu/src/plugin/transformations/op/kv_cache.cpp index a598e556a8f05d..12d961be6d337a 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/op/kv_cache.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/op/kv_cache.cpp @@ -3,6 +3,7 @@ // #include "intel_gpu/op/kv_cache.hpp" +#include "intel_gpu/op/kv_cache_compressed.hpp" #include "gather_shape_inference.hpp" #include "concat_shape_inference.hpp" #include "openvino/core/partial_shape.hpp" @@ -13,19 +14,28 @@ namespace ov { namespace intel_gpu { namespace op { -KVCache::KVCache(const Output& past, - const Output& new_token_data, - const Output& beam_idx, +KVCache::KVCache(const OutputVector& inputs, const std::shared_ptr& past_variable, + bool indirect, int64_t concat_axis, int64_t gather_axis, const ov::element::Type output_type) - : Op({past, new_token_data, beam_idx}) + : Op(inputs) , m_concat_axis(concat_axis) , m_gather_axis(gather_axis) - , m_indirect(true) + , m_indirect(indirect) , m_output_type(output_type) { m_variable = past_variable; +} + +KVCache::KVCache(const Output& past, + const Output& new_token_data, + const Output& beam_idx, + const std::shared_ptr& past_variable, + int64_t concat_axis, + int64_t gather_axis, + const ov::element::Type output_type) + : KVCache({past, new_token_data, beam_idx}, past_variable, true, concat_axis, gather_axis, output_type) { if (m_indirect) set_output_size(2); validate_and_infer_types(); @@ -36,11 +46,7 @@ KVCache::KVCache(const Output& past, const std::shared_ptr& past_variable, int64_t concat_axis, const ov::element::Type output_type) - : Op({past, new_token_data}) - , m_concat_axis(concat_axis) - , m_gather_axis(0) - , m_indirect(false) - , m_output_type(output_type) { + : KVCache({past, new_token_data}, past_variable, false, concat_axis, 0, output_type) { m_variable = past_variable; validate_and_infer_types(); } @@ -54,14 +60,23 @@ bool KVCache::visit_attributes(ov::AttributeVisitor& visitor) { } void KVCache::validate_and_infer_types() { - auto output_type = m_output_type == ov::element::undefined ? get_input_element_type(0) : m_output_type; + auto output_type = m_output_type; + if (m_output_type == ov::element::undefined) { + output_type = get_input_element_type(0); + } + std::vector input_shapes = {m_variable->get_info().data_shape, get_input_partial_shape(1)}; - if (get_output_size() == 2) + if (m_indirect) { input_shapes.push_back(get_input_partial_shape(2)); + } + auto shapes = shape_infer(this, input_shapes); - set_output_type(0, output_type, shapes[0]); + + size_t out_ports = 0; + set_output_type(out_ports++, output_type, shapes[0]); + if (m_indirect) { - set_output_type(1, get_input_element_type(2), shapes[1]); + set_output_type(out_ports++, get_input_element_type(2), shapes[1]); } } @@ -85,13 +100,13 @@ std::shared_ptr KVCache::clone_with_new_inputs(const ov::OutputVector& new } } -std::vector shape_infer(const KVCache* op, std::vector input_shapes) { +std::vector shape_infer(const KVCache* op, const std::vector& input_shapes) { std::vector out_shapes; out_shapes.resize(op->get_output_size()); const auto& gather_axis = op->get_gather_axis(); const auto& concat_axis = ov::util::normalize(op->get_concat_axis(), input_shapes[0].size()); - if (op->get_output_size() == 2) { + if (op->get_output_size() >= 2) { out_shapes[0] = input_shapes[0]; out_shapes[0][gather_axis] = input_shapes[2][0]; out_shapes[0][concat_axis] += input_shapes[1][concat_axis]; @@ -108,6 +123,87 @@ std::vector shape_infer(const KVCache* op, std::vector& past_variable, + int64_t concat_axis, + int64_t gather_axis, + const QuantizationAttrs& quantization_attrs, + const ov::element::Type output_type) + : KVCache(inputs, past_variable, true, concat_axis, gather_axis, output_type) + , m_compressed(true) + , m_quantization_attrs(quantization_attrs) { + OPENVINO_ASSERT(quantization_attrs.quantization_dt == ov::element::i8, + "[GPU] Only I8 data type is currently supported for KV-cache compression"); + + m_variable = past_variable; + size_t output_size = 3; + if (quantization_attrs.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric && + quantization_attrs.output_storage_type == ov::op::internal::DynamicQuantize::OutputStorageType::Planar) + output_size++; // add zp output + + set_output_size(output_size); + validate_and_infer_types(); +} + +void KVCacheCompressed::validate_and_infer_types() { + std::vector input_shapes = {m_variable->get_info().data_shape, get_input_partial_shape(1)}; + input_shapes.push_back(get_input_partial_shape(2)); + input_shapes.push_back(get_input_partial_shape(3)); + + if (m_quantization_attrs.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric && + m_quantization_attrs.output_storage_type == ov::op::internal::DynamicQuantize::OutputStorageType::Planar) + input_shapes.push_back(get_input_partial_shape(4)); + + auto shapes = shape_infer(this, input_shapes); + + size_t out_ports = 0; + set_output_type(out_ports++, m_quantization_attrs.quantization_dt, shapes[0]); + set_output_type(out_ports++, get_input_element_type(2), shapes[1]); + set_output_type(out_ports++, m_quantization_attrs.scale_dt, shapes[2]); + + if (m_quantization_attrs.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric && + m_quantization_attrs.output_storage_type == ov::op::internal::DynamicQuantize::OutputStorageType::Planar) { + set_output_type(out_ports++, m_quantization_attrs.zp_dt, shapes[3]); + } +} + +std::shared_ptr KVCacheCompressed::clone_with_new_inputs(const ov::OutputVector& new_args) const { + check_new_args_count(this, new_args); + return std::make_shared(new_args, + m_variable, + m_concat_axis, + m_gather_axis, + m_quantization_attrs, + m_output_type); +} + +std::vector shape_infer(const KVCacheCompressed* op, + const std::vector& input_shapes) { + std::vector out_shapes = shape_infer(static_cast(op), input_shapes); + + if (op->get_output_size() >= 3) { + ov::op::internal::DynamicQuantize dq_op; + dq_op.set_attrs(op->get_quantization_attrs()); + + auto quantized_data_shapes = + ov::op::internal::DynamicQuantize::shape_infer(&dq_op, { input_shapes[1] }); + + const auto scales_concat_axis = 2; + ov::PartialShape compression_scale_shape = input_shapes[3]; + compression_scale_shape[scales_concat_axis] += quantized_data_shapes[1][scales_concat_axis]; + out_shapes[2] = compression_scale_shape; + + // add zp output + if (quantized_data_shapes.size() == 3) { + ov::PartialShape compression_zp_shape = input_shapes[4]; + compression_zp_shape[scales_concat_axis] += quantized_data_shapes[2][scales_concat_axis]; + out_shapes[3] = compression_zp_shape; + } + } + + return out_shapes; +} + } // namespace op } // namespace intel_gpu } // namespace ov diff --git a/src/plugins/intel_gpu/src/plugin/transformations/op/read_value.cpp b/src/plugins/intel_gpu/src/plugin/transformations/op/read_value.cpp index 5438a6e2e695b5..6cd7f778c71b3b 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/op/read_value.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/op/read_value.cpp @@ -3,6 +3,7 @@ // #include "intel_gpu/op/read_value.hpp" +#include "intel_gpu/op/read_values.hpp" #include "openvino/core/partial_shape.hpp" namespace ov { @@ -28,16 +29,14 @@ bool ReadValue::visit_attributes(ov::AttributeVisitor& visitor) { return true; } -void ReadValue::validate_and_infer_types() { - OPENVINO_ASSERT(m_variable, "Variable is not initialized."); - const auto& variable_info = m_variable->get_info(); +void ReadValue::validate_and_infer_types(size_t output_idx, const ov::op::util::VariableInfo& variable_info) { const auto& variable_type = variable_info.data_type; const auto& variable_shape = variable_info.data_shape; // If no inputs provided, it means this ReadValue doesn't have initial subgraph. This is valid. - if (get_input_size() > 0) { - const auto& initial_type = get_input_element_type(0); - const auto& initial_shape = get_input_partial_shape(0); + if (get_input_size() > output_idx) { + const auto& initial_type = get_input_element_type(output_idx); + const auto& initial_shape = get_input_partial_shape(output_idx); // Variable's shape/type determine a permissible range of values for shape/type inferred from initial_subgraph. // If initial_subgraph is set, then we need to check that shape/type inferred from initial_subgraph @@ -64,19 +63,25 @@ void ReadValue::validate_and_infer_types() { // dynamic rank/type can be derived from the IRs generated via the prev versions of OV, // but dynamic rank/type are not supported in plugins, // so we are trying to fix them here using the rank/type of ReadValue 1st input, if it exists - if (get_input_size() > 0 && variable_info.data_shape.rank().is_dynamic() && - variable_info.data_type.is_dynamic()) { - set_output_type(0, initial_type, initial_shape); + if (variable_info.data_shape.rank().is_dynamic() && variable_info.data_type.is_dynamic()) { + set_output_type(output_idx, initial_type, initial_shape); return; } } - set_output_type(0, variable_type, variable_shape); + set_output_type(output_idx, variable_type, variable_shape); +} + +void ReadValue::validate_and_infer_types() { + OPENVINO_ASSERT(m_variable, "Variable is not initialized."); + const auto& variable_info = m_variable->get_info(); + + validate_and_infer_types(0, variable_info); } std::shared_ptr ReadValue::clone_with_new_inputs(const ov::OutputVector& new_args) const { check_new_args_count(this, new_args); - switch (new_args.size()) { + switch (new_args.size()) { case 0: return std::make_shared(m_variable); case 1: @@ -89,6 +94,62 @@ std::shared_ptr ReadValue::clone_with_new_inputs(const ov::OutputVector& n } } +ReadValues::ReadValues(const std::shared_ptr& variable, + const std::vector& internal_states_infos) + : ReadValue(variable) + , m_internal_states_infos(internal_states_infos) { + OPENVINO_ASSERT(!internal_states_infos.empty()); + set_output_size(internal_states_infos.size()); + validate_and_infer_types(); +} + +ReadValues::ReadValues(const std::vector>& variable_initializers, + const std::shared_ptr& variable, + const std::vector& internal_states_infos) + : ReadValue(variable_initializers, variable) + , m_internal_states_infos(internal_states_infos) { + OPENVINO_ASSERT(!internal_states_infos.empty()); + set_output_size(internal_states_infos.size()); + validate_and_infer_types(); +} + +bool ReadValues::visit_attributes(ov::AttributeVisitor& visitor) { + visitor.on_attribute("variable_id", m_variable); + + auto variable_info = m_variable->get_info(); + visitor.on_attribute("variable_type", variable_info.data_type); + visitor.on_attribute("variable_shape", variable_info.data_shape); + m_variable->update(variable_info); + return true; +} + +void ReadValues::validate_and_infer_types() { + OPENVINO_ASSERT(m_variable, "Variable is not initialized."); + + for (size_t i = 0; i < get_output_size(); i++) { + ReadValue::validate_and_infer_types(i, m_internal_states_infos[i]); + } +} + +std::shared_ptr ReadValues::clone_with_new_inputs(const ov::OutputVector& new_args) const { + check_new_args_count(this, new_args); + + OPENVINO_ASSERT(new_args.empty() || new_args.size() == m_internal_states_infos.size(), + "Unable to clone ReadValues op (name=", this->get_friendly_name(), "). ", + "Incorrect number of inputs. Expected: 0 or ", m_internal_states_infos.size(), ". ", + "Actual: ", new_args.size(), "."); + + if (new_args.size() > 0) { + return std::make_shared(new_args, m_variable, m_internal_states_infos); + } else { + return std::make_shared(m_variable, m_internal_states_infos); + } +} + +std::vector ReadValues::get_all_internal_states_info() const { + return m_internal_states_infos; +} + } // namespace op } // namespace intel_gpu } // namespace ov diff --git a/src/plugins/intel_gpu/src/plugin/transformations/op/sdpa.cpp b/src/plugins/intel_gpu/src/plugin/transformations/op/sdpa.cpp index 57d2899e2b2e77..09513d99153a1f 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/op/sdpa.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/op/sdpa.cpp @@ -26,7 +26,29 @@ SDPA::SDPA(const OutputVector& inputs, , m_order_k(order_k) , m_order_v(order_v) , m_order_out(order_out) - , m_output_type(output_type) { + , m_output_type(output_type) + , m_compressed(false) { + set_arguments(inputs); + set_causal(is_causal); + validate_and_infer_types(); +} + +SDPA::SDPA(const OutputVector& inputs, + const bool is_causal, + const std::vector& order_q, + const std::vector& order_k, + const std::vector& order_v, + const std::vector& order_out, + const QuantizationAttribute& quantization_attrs, + const ov::element::Type output_type) + : m_is_causal(is_causal) + , m_order_q(order_q) + , m_order_k(order_k) + , m_order_v(order_v) + , m_order_out(order_out) + , m_output_type(output_type) + , m_compressed(true) + , m_quantization_attrs(quantization_attrs) { set_arguments(inputs); set_causal(is_causal); validate_and_infer_types(); @@ -46,11 +68,13 @@ std::shared_ptr SDPA::clone_with_new_inputs(const ov::OutputVector& ne void SDPA::validate_and_infer_types() { const auto input_size = get_input_size(); + + const auto compression_inputs = get_compression_inputs_num(); NODE_VALIDATION_CHECK(this, - input_size == 3 || input_size == 4 || input_size == 5, + input_size >= 3 + compression_inputs && input_size <= 5 + compression_inputs, "Number of inputs is incorrect. Current value is: ", input_size, - ", expected 3, 4 or 5."); + ", expected 3, 4 or 5 data inputs and ", compression_inputs, " KV-cache compression related inputs"); std::vector input_shapes; for (size_t i = 0; i < input_size; i++) { @@ -77,6 +101,19 @@ bool SDPA::visit_attributes(ov::AttributeVisitor &visitor) { return true; } +size_t SDPA::get_compression_inputs_num() const { + size_t compression_inputs = 0; + if (m_compressed) { + compression_inputs += 2; // 2 * scales + + if (m_quantization_attrs.quantization_type == ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric && + m_quantization_attrs.output_storage_type == ov::op::internal::DynamicQuantize::OutputStorageType::Planar) + compression_inputs += 2; // 2 * zp + } + + return compression_inputs; +} + std::vector shape_infer(const SDPA* op, std::vector input_shapes, const std::vector& order_q, diff --git a/src/plugins/intel_gpu/src/plugin/transformations/optimize_subsequent_reshapes.cpp b/src/plugins/intel_gpu/src/plugin/transformations/optimize_subsequent_reshapes.cpp new file mode 100644 index 00000000000000..b87600ed36e347 --- /dev/null +++ b/src/plugins/intel_gpu/src/plugin/transformations/optimize_subsequent_reshapes.cpp @@ -0,0 +1,99 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "optimize_subsequent_reshapes.hpp" + +#include "openvino/core/rt_info.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/concat.hpp" +#include "openvino/op/reshape.hpp" +#include "openvino/pass/pattern/op/or.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "transformations/utils/utils.hpp" + +namespace ov { +namespace intel_gpu { + +OptimizeSubsequentReshapes::OptimizeSubsequentReshapes() { + using namespace ov::pass::pattern; + using ov::pass::pattern::op::Or; + + auto dynamic_batch_only = [](Output output) { + const auto& shape = output.get_partial_shape(); + + if (shape.rank().is_dynamic()) + return false; + + if (shape.size() <= 1) + return false; + + if (shape[0].is_static()) + return false; + + for (size_t i = 1; i < shape.size(); i++) + if (shape[i].is_dynamic()) + return false; + + return true; + }; + + auto first_reshape_data = any_input(dynamic_batch_only); + auto first_reshape_pattern = ov::pass::pattern::wrap_type(); + auto first_reshape = wrap_type({ first_reshape_data, first_reshape_pattern }, + ov::pass::pattern::all_of({ dynamic_batch_only, ov::pass::pattern::consumers_count(1) })); + + auto second_reshape_pattern = ov::pass::pattern::wrap_type(); + auto second_reshape = wrap_type({ first_reshape, second_reshape_pattern }, dynamic_batch_only); + + ov::matcher_pass_callback callback = [=](ov::pass::pattern::Matcher& m) { + const auto& pattern_map = m.get_pattern_value_map(); + + auto input_node = pattern_map.at(first_reshape_data).get_node_shared_ptr(); + auto first_reshape_node = pattern_map.at(first_reshape).get_node_shared_ptr(); + auto second_reshape_node = pattern_map.at(second_reshape).get_node_shared_ptr(); + + auto input_ps = first_reshape_node->input(0).get_partial_shape(); + auto first_reshape_ps = first_reshape_node->get_output_partial_shape(0); + auto second_reshape_ps = second_reshape_node->get_output_partial_shape(0); + + auto static_dims_product = [](ov::PartialShape& ps) { + int64_t total_dims = 1; + + for (auto& dim : ps) { + if (dim.is_static()) + total_dims *= dim.get_length(); + } + + return total_dims; + }; + + if (static_dims_product(input_ps) != static_dims_product(first_reshape_ps) || + static_dims_product(first_reshape_ps) != static_dims_product(second_reshape_ps)) + return false; + + std::vector new_pattern; + for (auto& dim : second_reshape_ps) { + if (dim.is_dynamic()) { + new_pattern.push_back(0); + } else { + new_pattern.push_back(dim.get_length()); + } + } + + auto new_pattern_const = std::make_shared(ov::element::i32, ov::Shape{new_pattern.size()}, new_pattern); + auto new_reshape = std::make_shared(first_reshape_node->input(0).get_source_output(), new_pattern_const, true); + new_reshape->set_friendly_name(second_reshape_node->get_friendly_name()); + + ov::replace_node(second_reshape_node, new_reshape); + copy_runtime_info(first_reshape_node, new_reshape); + + return true; + }; + + auto m = std::make_shared(second_reshape, "OptimizeSubsequentReshapes"); + this->register_matcher(m, callback); +} + +} // namespace intel_gpu +} // namespace ov diff --git a/src/plugins/intel_gpu/src/plugin/transformations/optimize_subsequent_reshapes.hpp b/src/plugins/intel_gpu/src/plugin/transformations/optimize_subsequent_reshapes.hpp new file mode 100644 index 00000000000000..3a38bb92ad5167 --- /dev/null +++ b/src/plugins/intel_gpu/src/plugin/transformations/optimize_subsequent_reshapes.hpp @@ -0,0 +1,23 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/graph_rewrite.hpp" + +namespace ov { +namespace intel_gpu { + +/** + * @brief This pass looks for `Reshape [ dynamic dim, n static dims] -> Reshape [dynamic dim, n static dims]` patterns + * and replaces them with a single `Reshape [dynamic dim, n static dims]` operation. + */ +class OptimizeSubsequentReshapes : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("OptimizeSubsequentReshapes", "0"); + OptimizeSubsequentReshapes(); +}; + +} // namespace intel_gpu +} // namespace ov diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 4b72385663bf9d..158dee2ee7ac05 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -15,11 +15,8 @@ #include "intel_gpu/plugin/transformations_pipeline.hpp" #include "intel_gpu/runtime/debug_configuration.hpp" #include "intel_gpu/runtime/itt.hpp" -#include "low_precision/add.hpp" #include "low_precision/convolution.hpp" #include "low_precision/convolution_backprop_data.hpp" -#include "low_precision/fold_convert.hpp" -#include "low_precision/fuse_convert.hpp" #include "low_precision/group_convolution.hpp" #include "low_precision/low_precision.hpp" #include "low_precision/mat_mul.hpp" @@ -28,9 +25,7 @@ #include "low_precision/pull_reshape_through_dequantization.hpp" #include "low_precision/pull_transpose_through_dequantization.hpp" #include "low_precision/recurrent_cell.hpp" -#include "low_precision/rt_info/bias_attribute.hpp" #include "low_precision/strided_slice.hpp" -#include "low_precision/transpose.hpp" #include "openvino/core/deprecated.hpp" #include "openvino/core/type/element_type.hpp" #include "openvino/core/validation_util.hpp" @@ -51,7 +46,6 @@ #include "openvino/op/reshape.hpp" #include "openvino/op/rnn_cell.hpp" #include "openvino/op/rnn_sequence.hpp" -#include "openvino/op/scaled_dot_product_attention.hpp" #include "openvino/op/squeeze.hpp" #include "openvino/op/unsqueeze.hpp" #include "openvino/op/util/sub_graph_base.hpp" @@ -71,15 +65,18 @@ #include "plugin/transformations/move_fc_reshape_to_weights.hpp" #include "plugin/transformations/bcast_and_pad_zp_buffers.hpp" #include "plugin/transformations/print_model_statistics.hpp" +#include "plugin/transformations/fc_per_layer_scaling.hpp" #include "plugin/transformations/swiglu_fusion.hpp" #include "plugin/transformations/transpose_fusion.hpp" #include "plugin/transformations/indirect_kv_cache.hpp" +#include "plugin/transformations/kv_cache_compression.hpp" #include "plugin/transformations/convert_convolution.hpp" #include "plugin/transformations/unsqueeze_broadcast_reshape_matmul_fusion.hpp" #include "plugin/transformations/unsqueeze_broadcast_reshape_sdpa_fusion.hpp" #include "plugin/transformations/increase_position_ids_precision.hpp" #include "plugin/transformations/group_norm_composition.hpp" #include "plugin/transformations/dynamic_quantize_fully_connected.hpp" +#include "plugin/transformations/optimize_subsequent_reshapes.hpp" #include "transformations/common_optimizations/nop_elimination.hpp" #include "transformations/common_optimizations/rms_fusion.hpp" #include "transformations/common_optimizations/broadcast_elementwise_fusion.hpp" @@ -183,14 +180,29 @@ static bool is_decompression_multiply(const std::shared_ptr node if (all_has_types(consumers, { ov::op::v0::MatMul::get_type_info_static(), ov::op::v8::Gather::get_type_info_static() })) return true; + auto are_multiply_from_decompression = [&all_has_types](const ov::Input consumer) { + if (!cldnn::one_of(consumer.get_node()->get_type_info(), { ov::op::v1::Multiply::get_type_info_static() })) + return false; + const auto child_consumers = consumer.get_node()->get_output_target_inputs(0); + if (all_has_types(child_consumers, { ov::opset1::MatMul::get_type_info_static(), ov::op::v8::Gather::get_type_info_static() })) + return true; + return false; + }; - auto are_converts_from_decompression = [&all_has_types](const std::set>& consumers) { + auto are_converts_from_decompression = [&all_has_types, &are_multiply_from_decompression](const std::set>& consumers) { if (!all_has_types(consumers, { ov::opset1::Convert::get_type_info_static() })) return false; for (const auto& consumer : consumers) { const auto child_consumers = consumer.get_node()->get_output_target_inputs(0); - if (!all_has_types(child_consumers, { ov::opset1::MatMul::get_type_info_static(), ov::op::v8::Gather::get_type_info_static() })) + for (const auto& child_consumer : child_consumers) { + const auto& type_info = child_consumer.get_node()->get_type_info(); + if (cldnn::one_of(type_info, { ov::opset1::MatMul::get_type_info_static(), ov::op::v8::Gather::get_type_info_static() })) + continue; + if (are_multiply_from_decompression(child_consumer)) { + continue; + } return false; + } } return true; }; @@ -318,9 +330,13 @@ void TransformationsPipeline::apply(std::shared_ptr func) { // it expects to have the same data type for weights and zero points (apply it only for u8 data type, since other compression // types are not supported by oneDNN) manager.register_pass(supported_woq_types, !device_info.supports_immad); - pass_config->set_callback([&](const std::shared_ptr node) { - return !is_decompression_multiply(node); - }); + + // Need to check if transformations work correctly for mixed models with both compression and quantization at the same time. + if (!is_model_quantized) { + pass_config->set_callback([&](const std::shared_ptr node) { + return !is_decompression_multiply(node); + }); + } const bool keep_precision_sensitive_in_fp32_1 = true; const bool convert_input_output_precision = false; @@ -689,6 +705,12 @@ void TransformationsPipeline::apply(std::shared_ptr func) { auto lptPassConfig = lptManager.get_pass_config(); // quantized LSTMSequence / GPUSequence are not supported yet. Avoid extra transformation lptPassConfig->disable(); + lptPassConfig->set_callback([](const_node_ptr& node) -> bool { + if (const auto mulitply = std::dynamic_pointer_cast(node)) { + return !MultiplyToGroupConvolutionTransformation::canBeTransformedToGroupConvolution(mulitply); + } + return false; + }); lptPassConfig->set_callback([func, defaultPrecisions](const_node_ptr& node) -> bool { auto fillStaticChannel = [func](const ov::PartialShape& shape, size_t& channel) -> bool { const auto rank = shape.rank(); @@ -725,43 +747,6 @@ void TransformationsPipeline::apply(std::shared_ptr func) { || WeightableLayerTransformation::isAsymmetricOnWeights(node, defaultPrecisions); }); - lptPassConfig->set_callback([&](const_node_ptr& node) -> bool { - for (auto& user : node->get_users()) { - if (ov::is_type(user)) - return true; - } - - return false; - }); - - lptPassConfig->set_callback([](const_node_ptr& node) -> bool { - return ov::is_type(node) && !MultiplyToGroupConvolutionTransformation::canBeTransformedToGroupConvolution(node); - }); - - lptPassConfig->set_callback([](const_node_ptr& node) -> bool { - return ov::marked_as_bias(node); - }); - lptPassConfig->set_callback([](const_node_ptr& node) -> bool { - const auto& consumers = node->get_output_target_inputs(0); - if (consumers.size() == 1) { - const auto consumer = consumers.begin()->get_node()->shared_from_this(); - return ov::is_type(consumer) && is_decompression_multiply(consumer); - } - return false; - }); - lptPassConfig->set_callback([](const_node_ptr& node) -> bool { - if (ov::is_type(node)) { - return ov::is_type(node) && is_decompression_multiply(node); - } else if (ov::is_type(node)) { - const auto& consumers = node->get_output_target_inputs(0); - if (consumers.size() == 1) { - const auto consumer = consumers.begin()->get_node()->shared_from_this(); - return ov::is_type(consumer) && is_decompression_multiply(consumer); - } - } - return false; - }); - lptPassConfig->set_callback([&](const_node_ptr& node) -> bool { // disable MultiplyToGroupConvolution if Multiply with Constant can be fused @@ -878,6 +863,7 @@ void TransformationsPipeline::apply(std::shared_ptr func) { manager.register_pass(); manager.register_pass(); manager.register_pass(device_info.supports_immad); + manager.register_pass(config.get_property(ov::hint::activations_scale_factor)); if (!device_info.supports_immad) { manager.register_pass(); @@ -886,6 +872,10 @@ void TransformationsPipeline::apply(std::shared_ptr func) { manager.register_pass(); manager.register_pass(); + + auto kv_cache_compression_dt = config.get_property(ov::hint::kv_cache_precision); + manager.register_pass(kv_cache_compression_dt); + manager.register_pass(); // This pass should be done after asymmetric quantization matching as it can move zp subtraction upper in the graph @@ -901,6 +891,8 @@ void TransformationsPipeline::apply(std::shared_ptr func) { pass_config->disable(); pass_config->disable(); + manager.register_pass(); + manager.register_pass(); // This Validate is needed for proper data type propagation after applying IncreasePositionIdsPrecision pass manager.register_pass(); diff --git a/src/plugins/intel_gpu/src/plugin/variable_state.cpp b/src/plugins/intel_gpu/src/plugin/variable_state.cpp index 6b1c8d0cfc993f..2b7a26ba35a292 100644 --- a/src/plugins/intel_gpu/src/plugin/variable_state.cpp +++ b/src/plugins/intel_gpu/src/plugin/variable_state.cpp @@ -123,6 +123,12 @@ ov::element::Type VariableState::get_user_specified_type() const { } ov::SoPtr VariableState::get_state() const { + if (m_memory == nullptr) { + const auto& pshape = m_layout.get_partial_shape(); + const auto& shape = get_tensor_shape(pshape); + return m_context->create_host_tensor(get_user_specified_type(), shape); + } + auto tensor = m_context->create_host_tensor(get_user_specified_type(), m_memory->get_layout().get_shape()); convert_and_copy(m_memory, tensor._ptr.get(), m_context->get_engine().get_service_stream()); diff --git a/src/plugins/intel_gpu/src/runtime/debug_configuration.cpp b/src/plugins/intel_gpu/src/runtime/debug_configuration.cpp index 5f943564d6f50e..5c3b3ee0c970f9 100644 --- a/src/plugins/intel_gpu/src/runtime/debug_configuration.cpp +++ b/src/plugins/intel_gpu/src/runtime/debug_configuration.cpp @@ -183,6 +183,7 @@ static void print_help_messages() { message_list.emplace_back("OV_GPU_DisableRuntimeSkipReorder", "Disable runtime skip reorder."); message_list.emplace_back("OV_GPU_DisablePrimitiveFusing", "Disable primitive fusing"); message_list.emplace_back("OV_GPU_DisableFakeAlignment", "Disable fake alignment"); + message_list.emplace_back("OV_GPU_KVCacheCompression", "Enable/Disable KV-cache compression"); message_list.emplace_back("OV_GPU_DynamicQuantizeLayersWithoutOnednn", "Enable Dynamic quantization for specified Fully connected layers only, " "separated by space. Support case-insensitive and regular expression. For example .*fully_connected.*"); message_list.emplace_back("OV_GPU_DynamicQuantizeGroupSize", "Specify a group size of dynamic quantization to enable " @@ -253,6 +254,7 @@ debug_configuration::debug_configuration() , disable_runtime_skip_reorder(0) , disable_primitive_fusing(0) , disable_fake_alignment(0) + , use_kv_cache_compression(-1) , dynamic_quantize_group_size(DYNAMIC_QUANTIZE_GROUP_SIZE_NOT_SET) , disable_horizontal_fc_fusion(0) { #ifdef GPU_DEBUG_CONFIG @@ -305,6 +307,7 @@ debug_configuration::debug_configuration() get_gpu_debug_env_var("DisableRuntimeSkipReorder", disable_runtime_skip_reorder); get_gpu_debug_env_var("DisablePrimitiveFusing", disable_primitive_fusing); get_gpu_debug_env_var("DisableFakeAlignment", disable_fake_alignment); + get_gpu_debug_env_var("KVCacheCompression", use_kv_cache_compression); get_gpu_debug_env_var("DynamicQuantizeGroupSize", dynamic_quantize_group_size); get_gpu_debug_env_var("DisableHorizontalFCFusion", disable_horizontal_fc_fusion); std::string dump_iteration_str; diff --git a/src/plugins/intel_gpu/src/runtime/execution_config.cpp b/src/plugins/intel_gpu/src/runtime/execution_config.cpp index 7661444cc4fd7b..7c8e55cddfe593 100644 --- a/src/plugins/intel_gpu/src/runtime/execution_config.cpp +++ b/src/plugins/intel_gpu/src/runtime/execution_config.cpp @@ -58,8 +58,10 @@ void ExecutionConfig::set_default() { std::make_tuple(ov::cache_mode, ov::CacheMode::OPTIMIZE_SPEED), std::make_tuple(ov::cache_encryption_callbacks, EncryptionCallbacks{}), std::make_tuple(ov::hint::dynamic_quantization_group_size, 32), + std::make_tuple(ov::hint::kv_cache_precision, ov::element::undefined), std::make_tuple(ov::intel_gpu::hint::enable_kernels_reuse, false), std::make_tuple(ov::weights_path, ""), + std::make_tuple(ov::hint::activations_scale_factor, 0.f), // Legacy API properties std::make_tuple(ov::intel_gpu::nv12_two_inputs, false), @@ -209,6 +211,14 @@ void ExecutionConfig::apply_debug_options(const cldnn::device_info& info) { else set_property(ov::hint::dynamic_quantization_group_size(debug_config->dynamic_quantize_group_size)); } + + GPU_DEBUG_IF(debug_config->use_kv_cache_compression != -1) { + GPU_DEBUG_IF(debug_config->use_kv_cache_compression == 1) { + set_property(ov::hint::kv_cache_precision(ov::element::i8)); + } else { + set_property(ov::hint::kv_cache_precision(ov::element::undefined)); + } + } } void ExecutionConfig::apply_hints(const cldnn::device_info& info) { diff --git a/src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp b/src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp index 88801b8b2b4e61..7ab48308cfeaf7 100644 --- a/src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp +++ b/src/plugins/intel_gpu/src/runtime/ocl/ocl_device.cpp @@ -330,6 +330,14 @@ device_info init_device_info(const cl::Device& device, const cl::Context& contex ngen::Product product = {ngen::ProductFamily::Unknown, 0}; jit_generator::detectHWInfo(context.get(), device.get(), hw, product); info.arch = convert_ngen_arch(hw); + // We change the value of this flag to avoid OneDNN usage for the platforms unknown to OneDNN + // This is required to guarantee some level of forward compatibility for the new HW generations + // as OneDNN code generators are not generic and typically requires some updates for the new architectures + // Ideally, we shouldn't do that as OCL impls sometimes also check this flag, but in order to avoid that + // we need to ensure that graph transformations are not relying on this flag as indicator that onednn will be used + if (product.family == ngen::ProductFamily::Unknown) { + info.supports_immad = false; + } #else // ENABLE_ONEDNN_FOR_GPU info.arch = gpu_arch::unknown; #endif // ENABLE_ONEDNN_FOR_GPU diff --git a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/kv_cache_sdpa.cpp b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/kv_cache_sdpa.cpp index 68098937bb9c0a..16db9d89c28b4d 100644 --- a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/kv_cache_sdpa.cpp +++ b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/kv_cache_sdpa.cpp @@ -26,6 +26,7 @@ struct Params { bool with_mask; bool with_scale; bool causal; + bool compressed; size_t batch; ov::element::Type model_element_type; size_t num_iter; @@ -46,6 +47,9 @@ class SDPAWithKVCacheTest : public ::testing::Test, public ::testing::WithParamI ov::AnyMap properties = {ov::hint::inference_precision(ov::element::f16), ov::intel_gpu::hint::enable_sdpa_optimization(true)}; + if (p.compressed) + properties.emplace(ov::hint::kv_cache_precision(ov::element::i8)); + const size_t n_heads = 16; const size_t n_features = 64; const size_t context_size = 7; @@ -58,6 +62,7 @@ class SDPAWithKVCacheTest : public ::testing::Test, public ::testing::WithParamI const bool causal = p.causal; const bool with_mask = p.with_mask; const bool with_scale = p.with_scale; + const bool compressed = p.compressed; auto model = tests::make_llm_kv_cache_sdpa_pattern(ov::Dimension::dynamic(), n_heads, @@ -284,14 +289,17 @@ class SDPAWithKVCacheTest : public ::testing::Test, public ::testing::WithParamI compare_tensors({ref_results[0]}, {sdpa_out}); } - auto variables = infer_request.query_state(); - std::vector states; - for (auto& variable : variables) { - auto state = variable.get_state(); - ASSERT_EQ(state.get_element_type(), element_type); - states.push_back(state); + if (!compressed) { + auto variables = infer_request.query_state(); + std::vector states; + for (auto& variable : variables) { + auto state = variable.get_state(); + ASSERT_EQ(state.get_element_type(), element_type); + states.push_back(state); + } + compare_tensors({ref_k_cache, ref_v_cache}, states); } - compare_tensors({ref_k_cache, ref_v_cache}, states); + infer_request.reset_state(); } } @@ -310,6 +318,7 @@ class SDPAWithKVCacheTest : public ::testing::Test, public ::testing::WithParamI result << "mask=" << p.with_mask << "_"; result << "scale=" << p.with_scale << "_"; result << "causal=" << p.causal << "_"; + result << "compressed=" << p.compressed << ""; return result.str(); } }; @@ -324,11 +333,17 @@ std::vector get_test_params() { const bool with_mask = true; const bool with_scale = true; const bool causal = true; + const bool compressed = true; + + p.push_back({with_rearrange, !with_mask, !with_scale, !causal, !compressed, 1, ov::element::Type_t::f16, 10, 1, 1, {0, 1, 2, 3}}); + p.push_back({with_rearrange, with_mask, !with_scale, !causal, !compressed, 1, ov::element::Type_t::f16, 10, 4, 1, {0, 1, 2, 3}}); + p.push_back({with_rearrange, with_mask, !with_scale, !causal, !compressed, 1, ov::element::Type_t::f16, 10, 4, 1, {0, 2, 1, 3}}); + p.push_back({!with_rearrange, with_mask, !with_scale, !causal, !compressed, 1, ov::element::Type_t::f16, 10, 4, 1, {0, 2, 1, 3}}); - p.push_back({with_rearrange, !with_mask, !with_scale, !causal, 1, ov::element::Type_t::f16, 10, 1, 1, {0, 1, 2, 3}}); - p.push_back({with_rearrange, with_mask, !with_scale, !causal, 1, ov::element::Type_t::f16, 10, 4, 1, {0, 1, 2, 3}}); - p.push_back({with_rearrange, with_mask, !with_scale, !causal, 1, ov::element::Type_t::f16, 10, 4, 1, {0, 2, 1, 3}}); - p.push_back({!with_rearrange, with_mask, !with_scale, !causal, 1, ov::element::Type_t::f16, 10, 4, 1, {0, 2, 1, 3}}); + // Compressed + p.push_back({with_rearrange, with_mask, !with_scale, !causal, compressed, 1, ov::element::Type_t::f16, 10, 1, 1, {0, 1, 2, 3}}); + p.push_back({with_rearrange, with_mask, !with_scale, !causal, compressed, 1, ov::element::Type_t::f16, 10, 4, 1, {0, 2, 1, 3}}); + p.push_back({with_rearrange, with_mask, !with_scale, !causal, compressed, 1, ov::element::Type_t::f16, 10, 4, 1, {0, 1, 2, 3}}); return p; } diff --git a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp index fee6a02b6671d8..27c57aa072878d 100644 --- a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp +++ b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp @@ -30,7 +30,7 @@ using ov::test::InputShape; * \ / * Multiply * | - * Data(F32) Transpose(optional) + * Data(F32) Transpose(optional) or Multiply(optional) * \ / * Matmul * | @@ -56,6 +56,7 @@ using MatmulWeightsDecompressionParams = std::tuple; @@ -70,6 +71,7 @@ class MatmulWeightsDecompression : public testing::WithParamInterface +#include +#include +#include + +#include "permute_inst.h" +#include "program_wrapper.h" + +#include +#include + +using namespace cldnn; +using namespace ::tests; + +namespace skip_permute_tests { + +struct skip_permute_params { + layout input_layout_static; + std::vector permute_order; + bool expected_result1; + bool expected_result2; +}; + +class skip_permute_at_runtime_test : public testing::TestWithParam {}; + +TEST_P(skip_permute_at_runtime_test, runtime_skip) { + auto p = GetParam(); + auto& engine = get_test_engine(); + auto rank = p.input_layout_static.get_partial_shape().size(); + auto input_layout_dynamic = layout {ov::PartialShape::dynamic(rank), data_types::f16, format::get_default_format(rank)}; + topology topology(input_layout("input", input_layout_dynamic), + permute("permute", input_info("input"), p.permute_order), + reorder("reorder", input_info("permute"), format::get_default_format(rank), data_types::f32)); + + ExecutionConfig config = get_test_default_config(engine); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + config.set_property(ov::intel_gpu::optimize_data(true)); + + network network(engine, topology, config); + auto permute_inst = network.get_primitive("permute"); + ASSERT_EQ(permute_inst->can_be_optimized(), p.expected_result1); + + auto input_mem = engine.allocate_memory(p.input_layout_static); + network.set_input_data("input", input_mem); + auto outputs = network.execute(); + outputs.begin()->second.get_memory(); + + ASSERT_EQ(permute_inst->can_be_optimized(), p.expected_result2); +} + +INSTANTIATE_TEST_SUITE_P(smoke, skip_permute_at_runtime_test, + testing::ValuesIn(std::vector { + { layout{ov::PartialShape{8,8}, data_types::f16, format::bfyx}, {1, 0}, true, false }, + { layout{ov::PartialShape{8,1}, data_types::f16, format::bfyx}, {1, 0}, true, true }, + { layout{ov::PartialShape{8, 2, 8}, data_types::f16, format::bfyx}, {2, 1, 0}, true, false }, + { layout{ov::PartialShape{8, 2, 1}, data_types::f16, format::bfyx}, {2, 1, 0}, true, false }, + { layout{ov::PartialShape{1, 12, 1}, data_types::f16, format::bfyx}, {2, 1, 0}, true, true }, + { layout{ov::PartialShape{2, 3, 1, 14}, data_types::f16, format::bfyx}, {1, 0, 2, 3}, true, false }, + { layout{ov::PartialShape{1, 3, 1, 14}, data_types::f16, format::bfyx}, {1, 0, 2, 3}, true, true }, + { layout{ov::PartialShape{20, 1, 30, 1}, data_types::f16, format::bfyx}, {1, 0, 3, 2}, true, true}, + { layout{ov::PartialShape{12, 3, 1, 14}, data_types::f16, format::bfyx}, {0, 2, 1, 3}, true, true }, + { layout{ov::PartialShape{12, 3, 2, 14}, data_types::f16, format::bfyx}, {0, 2, 1, 3}, true, false }, + { layout{ov::PartialShape{12, 1, 1, 14}, data_types::f16, format::bfyx}, {0, 3, 1, 2}, true, true }, + { layout{ov::PartialShape{12, 1, 1, 14}, data_types::f16, format::bfyx}, {0, 3, 1, 2}, true, true }, + { layout{ov::PartialShape{1, 1, 1, 14}, data_types::f16, format::bfyx}, {0, 3, 1, 2}, true, true }, + { layout{ov::PartialShape{1, 3, 2, 14}, data_types::f16, format::bfyx}, {0, 3, 1, 2}, true, false }, + { layout{ov::PartialShape{1, 1, 4, 14}, data_types::f16, format::bfyx}, {2, 0, 1, 3}, true, true }, + { layout{ov::PartialShape{1, 4, 4, 1}, data_types::f16, format::bfyx}, {2, 0, 1, 3}, true, false }, + { layout{ov::PartialShape{1, 10, 1, 1, 11}, data_types::f16, format::bfzyx}, {3, 2, 1, 0, 4}, true, true }, + { layout{ov::PartialShape{1, 10, 2, 1, 10}, data_types::f16, format::bfzyx}, {3, 2, 1, 0, 4}, true, false }, + { layout{ov::PartialShape{1, 4, 1, 3, 4, 1}, data_types::f16, format::bfwzyx}, {0, 2, 1, 3, 5, 4}, true, true }, + { layout{ov::PartialShape{1, 4, 2, 3, 4, 1}, data_types::f16, format::bfwzyx}, {0, 2, 1, 3, 5, 4}, true, false }, + { layout{ov::PartialShape{1, 1, 1, 1, 4, 1}, data_types::f16, format::bfwzyx}, {4, 5, 2, 3, 0, 1}, true, true }, + { layout{ov::PartialShape{1, 1, 1, 1, 4, 2}, data_types::f16, format::bfwzyx}, {4, 5, 2, 3, 0, 1}, true, true }, + { layout{ov::PartialShape{1, 1, 3, 1, 4, 2}, data_types::f16, format::bfwzyx}, {4, 5, 2, 3, 0, 1}, true, false }, + })); +} // skip permute tests diff --git a/src/plugins/intel_gpu/tests/unit/dynamic_execution/stateful_model.cpp b/src/plugins/intel_gpu/tests/unit/dynamic_execution/stateful_model.cpp index 4b24fb996b3f3f..105963d1b09d73 100644 --- a/src/plugins/intel_gpu/tests/unit/dynamic_execution/stateful_model.cpp +++ b/src/plugins/intel_gpu/tests/unit/dynamic_execution/stateful_model.cpp @@ -197,7 +197,7 @@ TEST(stateful_model, check_dynamic_pad_for_kv_cache) { auto input_kv_lay = layout{info.data_shape, info.data_type, format::bfyx}; topology topology(input_layout("beam_idx", input_beam_idx_lay), input_layout("present", input_present_lay), - read_value("kv_cache", std::vector{}, info.variable_id, input_kv_lay), + read_value("kv_cache", std::vector{}, info.variable_id, {input_kv_lay}), gather("gather", input_info("kv_cache"), input_info("beam_idx"), @@ -224,7 +224,7 @@ TEST(stateful_model, check_dynamic_pad_for_kv_cache) { auto pad = tensor(0); pad.batch[0] = 1; - + { std::vector dynamic_pad_mask; const auto& dynamic_pad_dims = read_value_inst->get_output_layout(0).data_padding._dynamic_dims_mask; diff --git a/src/plugins/intel_gpu/tests/unit/fusions/gemm_fusion_test.cpp b/src/plugins/intel_gpu/tests/unit/fusions/gemm_fusion_test.cpp index baed5400181130..659ccaf9d8a723 100644 --- a/src/plugins/intel_gpu/tests/unit/fusions/gemm_fusion_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/fusions/gemm_fusion_test.cpp @@ -455,8 +455,8 @@ TEST_P(gemm_2in_dynamic_add, add) { } INSTANTIATE_TEST_SUITE_P(fusings_gpu, gemm_2in_dynamic_add, ::testing::ValuesIn(std::vector{ - gemm_test_params{ CASE_GEMM_2IN_FP16_3D_1, 4, 5, "", broadcast_kinds::batch, eltwise_mode::sum }, - gemm_test_params{ CASE_GEMM_2IN_FP16_3D_1, 4, 5, "", broadcast_kinds::feature, eltwise_mode::sum }, + gemm_test_params{ CASE_GEMM_2IN_FP16_3D_1, 4, 4, "gemm_tiled_opt", broadcast_kinds::batch, eltwise_mode::sum }, + gemm_test_params{ CASE_GEMM_2IN_FP16_3D_1, 4, 4, "gemm_tiled_opt", broadcast_kinds::feature, eltwise_mode::sum }, })); class gemm_2in_act_scale_quantize_i8 : public GemmFusingTest {}; diff --git a/src/plugins/intel_gpu/tests/unit/shape_infer/read_value_si_test.cpp b/src/plugins/intel_gpu/tests/unit/shape_infer/read_value_si_test.cpp index 194bc0244f86f0..2000d826ddfad6 100644 --- a/src/plugins/intel_gpu/tests/unit/shape_infer/read_value_si_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/shape_infer/read_value_si_test.cpp @@ -32,7 +32,7 @@ TEST_P(read_value_test, shape_infer) { auto& engine = get_test_engine(); - const auto variable_layout = p.input_layout; + const std::vector variable_layout = {p.input_layout}; auto input_layout_prim = std::make_shared("input", p.input_layout); auto inputs = std::vector{ input_info("input") }; diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/convolution_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/convolution_gpu_test.cpp index 4155ac0b420e66..4f9c31064e9026 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/convolution_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/convolution_gpu_test.cpp @@ -10134,6 +10134,113 @@ TEST(convolution_gpu_onednn, quantized_onednn_convolution_u8s8f32_weights_zp) { } } +TEST(convolution_gpu_onednn, support_activation_zero_points_for_i32) { + auto& engine = get_test_engine(); + if (!engine.get_device_info().supports_immad) + return; + + auto in_layout = layout { ov::PartialShape::dynamic(4), data_types::u8, format::bfyx }; + auto input = engine.allocate_memory({ data_types::u8, format::bfyx, { 1, 2, 5, 4 } }); + auto weights = engine.allocate_memory({ data_types::i8, format::bfyx, { 3, 2, 3, 3 } }); + auto biases = engine.allocate_memory({ data_types::f32, format::bfyx, { 1, 3, 1, 1 } }); + auto a_zp = engine.allocate_memory({ data_types::i32, format::bfyx, { 1, 3, 1, 1 } }); + auto w_zp = engine.allocate_memory({ data_types::u8, format::bfyx, { 1, 1, 1, 1 } }); + + set_values(input, { 1, 2, 3, 4, 5, + 2, 2, 3, 4, 6, + 3, 3, 3, 5, 1, + 1, 1, 1, 1, 1, + + 1, 2, 3, 4, 5, + 2, 2, 3, 4, 6, + 3, 3, 3, 5, 1, + 1, 1, 1, 1, 1 }); + + set_values(weights, { 1, 2, -1, + -2, 1, 2, + 9, 7, -1, + + 9, 0, -4, + -1, 3, 2, + 0, 2, 5, + + 1, 2, -1, + -2, 1, 2, + 9, 7, -1, + + 9, 0, -4, + -1, 3, 2, + 0, 2, 5, + + 1, 2, -1, + -2, 1, 2, + 9, 7, -1, + + 9, 0, -4, + -1, 3, 2, + 0, 2, 5 }); + set_values(a_zp, { 2, 5, 5 }); + set_values(w_zp, { 2 }); + set_values(biases, { 1.0f, -8.0f, -8.0f }); + + VVVF output_vec = { + { + { 2.0f, -5.0f, -20.0f }, + { 12.0f, 26.0f, -10.0f } + }, + { + { -7.0f, -14.0f, -29.0f }, + { 3.0f, 17.0f, -19.0f } + }, + { + { -7.0f, -14.0f, -29.0f }, + { 3.0f, 17.0f, -19.0f } + } }; + + topology topology( + input_layout("input", in_layout), + data("weights", weights), + data("biases", biases), + data("a_zp", a_zp), + data("w_zp", w_zp), + convolution("conv", input_info("input"), "weights", "biases", "w_zp", "a_zp", "", 1, + { 2, 2 }, { 1, 1 }, { 0, 0 }, { 1, 2 }, false, data_types::f32), + reorder("out", input_info("conv"), format::bfyx, data_types::f32)); + + ExecutionConfig config = get_test_default_config(engine); + ov::intel_gpu::ImplementationDesc conv_impl = { format::bfyx, "", impl_types::onednn }; + config.set_property(ov::intel_gpu::force_implementations(ov::intel_gpu::ImplForcingMap{ { "conv", conv_impl }})); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + config.set_property(ov::intel_gpu::optimize_data(true)); + + network network(engine, topology, config); + network.set_input_data("input", input); + + auto outputs = network.execute(); + ASSERT_EQ(outputs.begin()->first, "out"); + + auto output_memory = outputs.at("out").get_memory(); + cldnn::mem_lock output_ptr(output_memory, get_test_stream()); + + auto output_layout = output_memory->get_layout(); + int y_size = output_layout.spatial(1); + int x_size = output_layout.spatial(0); + int f_size = output_layout.feature(); + int b_size = output_layout.batch(); + ASSERT_EQ(output_layout.format, format::bfyx); + ASSERT_EQ(y_size, 2); + ASSERT_EQ(x_size, 3); + ASSERT_EQ(f_size, 3); + ASSERT_EQ(b_size, 1); + for (int f = 0; f < f_size; f++) + for (int y = 0; y < y_size; ++y) { + for (int x = 0; x < x_size; ++x) { + ASSERT_NEAR(output_vec[f][y][x], ((float)output_ptr[f * y_size * x_size + y * x_size + x]), 1e-5f) << + " x="<< x << " y=" << y << " f=" << f; + } + } +} + TEST(convolution_gpu_onednn, has_proper_synchronization) { auto& engine = get_test_engine(); if (!engine.get_device_info().supports_immad) diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp index c1686e359e91a0..5a78360eb1f6d8 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/dynamic_quantize_gpu_test.cpp @@ -22,35 +22,53 @@ using namespace cldnn; using namespace ::tests; +using QuantizationType = ov::op::internal::DynamicQuantize::QuantizationType; class dynamic_quantization_gpu_tests: public ::testing::Test { public: - void test_dynamic_quantization(bool is_caching_test, bool is_dynamic, int batch = 1, int ifm = 1024) { + void test_dynamic_quantization(bool is_caching_test, + const ov::PartialShape& input_shape, + const ov::Shape& data_shape, + const QuantizationType quantization_type = QuantizationType::Symmetric, + const std::string& impl_name = "") { tests::random_generator rg(GET_SUITE_NAME); auto& engine = get_test_engine(); - long int batch_num = batch; - long int ifm_num = ifm; - - bool is_4d = true; - - auto input_ps = is_4d ? ov::PartialShape{ batch_num, 1, 1, ifm_num } : ov::PartialShape{ batch_num, ifm_num}; - auto dyn_input_ps = is_4d ? ov::PartialShape{ -1, 1, 1, ifm_num } : ov::PartialShape{ -1, ifm_num}; + auto input_ps = data_shape; + auto dyn_input_ps = input_shape; + auto scales_ps = ov::PartialShape::dynamic(dyn_input_ps.size()); auto input_mem = engine.allocate_memory({ input_ps, data_types::f32, format::bfyx }); + auto group_sizes = std::vector(dyn_input_ps.size(), 1); + group_sizes.back() = UINT64_MAX; - auto input_data = rg.generate_random_1d(batch_num * ifm_num, -16.0f, 16.0f); + auto input_data = rg.generate_random_1d(ov::shape_size(data_shape), -16.0f, 16.0f); set_values(input_mem, input_data); - auto in_layout_f32 = is_dynamic ? layout{ dyn_input_ps, data_types::f32, format::bfyx } - : layout{ input_ps, data_types::f32, format::bfyx }; + auto in_layout_f32 = input_shape.is_dynamic() ? layout{ dyn_input_ps, data_types::f32, format::bfyx } + : layout{ input_ps, data_types::f32, format::bfyx }; + + auto in_layout = input_shape.is_dynamic() ? layout{ dyn_input_ps, data_types::f16, format::bfyx } + : layout{ input_ps, data_types::f16, format::bfyx }; - auto in_layout = is_dynamic ? layout{ dyn_input_ps, data_types::f16, format::bfyx } - : layout{ input_ps, data_types::f16, format::bfyx }; + dynamic_quantize::Attributes dq_config; + dq_config.quantization_type = quantization_type; + dq_config.quantization_dt = data_types::i8; + dq_config.scale_dt = data_types::f16; + dq_config.zp_dt = data_types::undefined; + dq_config.group_sizes = group_sizes; + dq_config.scales_zp_output_order = { 0, 1, 2, 3 }; + dq_config.output_storage_type = ov::op::internal::DynamicQuantize::OutputStorageType::Planar; + + if (quantization_type == QuantizationType::Asymmetric) { + dq_config.zp_dt = data_types::f16; + dq_config.output_storage_type = ov::op::internal::DynamicQuantize::OutputStorageType::InterleavedScalesZP; + } auto reorder_1 = reorder("reorder_1", input_info("input"), layout{ input_ps, data_types::f16, format::bfyx }); - auto dyn_quan_prim = dynamic_quantize("dyn_quan_prim", input_info("reorder_1"), 32, {data_types::f16, data_types::i8}); - auto reorder_2 = reorder("reorder_2", input_info("dyn_quan_prim"), layout{ input_ps, data_types::f16, format::bfyx }); + auto dyn_quan_prim = dynamic_quantize("dyn_quan_prim", input_info("reorder_1"), dq_config); + auto reorder_data = reorder("reorder_data", input_info("dyn_quan_prim", 0), layout{ input_ps, data_types::f16, format::bfyx }); + auto reorder_scale = reorder("reorder_scale", input_info("dyn_quan_prim", 1), layout{ scales_ps, data_types::f16, format::bfyx }); // Implemented dynamic quantize kernel auto get_ref_results = [&]() { @@ -58,7 +76,8 @@ class dynamic_quantization_gpu_tests: public ::testing::Test { input_layout("input", in_layout_f32), reorder_1, dyn_quan_prim, - reorder_2 + reorder_data, + reorder_scale ); auto config = get_test_default_config(engine); @@ -83,13 +102,18 @@ class dynamic_quantization_gpu_tests: public ::testing::Test { input_layout("input", in_layout_f32), reorder_1, dyn_quan_prim, - reorder_2 + reorder_data ); auto config = get_test_default_config(engine); config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); config.set_property(ov::intel_gpu::optimize_data(true)); + if (impl_name != "") { + ov::intel_gpu::ImplementationDesc dyn_quan_impl_desc = { format::bfyx, impl_name, impl_types::ocl }; + config.set_property(ov::intel_gpu::force_implementations(ov::intel_gpu::ImplForcingMap{ {"dyn_quan_prim", dyn_quan_impl_desc} })); + } + network::ptr network = get_network(engine, topology, config, get_test_stream_ptr(), is_caching_test); network->set_input_data("input", input_mem); @@ -118,37 +142,69 @@ class dynamic_quantization_gpu_tests: public ::testing::Test { }; TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_large_size) { - this->test_dynamic_quantization(false, false, 2048, 4096); + this->test_dynamic_quantization(false, {11, 1, 1, 4096}, {2048, 1, 1, 4096}); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_large_size_dynamic) { - this->test_dynamic_quantization(false, true, 2048, 4096); + this->test_dynamic_quantization(false, {-1, 1, 1, 4096}, {2048, 1, 1, 4096}); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_small_size) { - this->test_dynamic_quantization(false, false, 64, 4096); + this->test_dynamic_quantization(false, {1, 1, 1, 4096}, {64, 1, 1, 4096}); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_single_batch) { - this->test_dynamic_quantization(false, false, 1, 4096); + this->test_dynamic_quantization(false, {-1, 1, 1, 4096}, {1, 1, 1, 4096}); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_ref_only) { - this->test_dynamic_quantization(false, false, 16, 33); + this->test_dynamic_quantization(false, {-1, 1, 1, 33}, {16, 1, 1, 33}); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_ref_only_dynamic) { - this->test_dynamic_quantization(false, true, 16, 33); + this->test_dynamic_quantization(false, {1, 1, 1, 33}, {16, 1, 1, 33}); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_invalid) { - this->test_dynamic_quantization(false, false, 16, 7); + this->test_dynamic_quantization(false, {-1, 1, 1, 7}, {16, 1, 1, 7}); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_unaligned) { - this->test_dynamic_quantization(false, false, 16, 32); + this->test_dynamic_quantization(false, {-1, 1, 1, 32}, {16, 1, 1, 32}); } TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_unaligned_dynamic) { - this->test_dynamic_quantization(false, true, 16, 32); + this->test_dynamic_quantization(false, {1, 1, 1, 32}, {16, 1, 1, 32}); +} + +TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache) { + this->test_dynamic_quantization(false, {-1, 8, -1, 96}, {1, 8, 1, 96}, QuantizationType::Symmetric, "dynamic_quantize_gpu_kv_cache"); +} + +TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched) { + this->test_dynamic_quantization(false, {-1, 4, -1, 64}, {1, 4, 35, 64}, QuantizationType::Symmetric, "dynamic_quantize_gpu_kv_cache"); +} + +TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_reordered) { + this->test_dynamic_quantization(false, {-1, -1, 8, 96}, {1, 1, 8, 96}, QuantizationType::Symmetric, "dynamic_quantize_gpu_kv_cache"); +} + +TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_reordered) { + this->test_dynamic_quantization(false, {-1, -1, 4, 64}, {1, 35, 4, 64}, QuantizationType::Symmetric, "dynamic_quantize_gpu_kv_cache"); +} + +TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_asym) { + this->test_dynamic_quantization(false, {-1, 8, -1, 96}, {1, 8, 1, 96}, QuantizationType::Asymmetric, "dynamic_quantize_gpu_kv_cache"); +} + +TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_asym) { + this->test_dynamic_quantization(false, {-1, 4, -1, 64}, {1, 4, 35, 64}, QuantizationType::Asymmetric, "dynamic_quantize_gpu_kv_cache"); +} + +TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_reordered_asym) { + this->test_dynamic_quantization(false, {-1, -1, 8, 96}, {1, 1, 8, 96}, QuantizationType::Asymmetric, "dynamic_quantize_gpu_kv_cache"); +} + +TEST_F(dynamic_quantization_gpu_tests, simple_quantizing_kv_cache_batched_reordered_asym) { + this->test_dynamic_quantization(false, {-1, -1, 4, 64}, {1, 35, 4, 64}, QuantizationType::Asymmetric, "dynamic_quantize_gpu_kv_cache"); } diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp index eed9760348f669..6bf44a31add0f4 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp @@ -1589,7 +1589,7 @@ class fully_connected_gpu_tests: public ::testing::Test { count++; OPENVINO_ASSERT(abs_diff < 256); } - GPU_DEBUG_LOG << "---> count: " << count << ", max_diff:" << max_diff << ", avg_diff: " << (avg/count) << std::endl; + std::cout << "---> count: " << count << ", max_diff:" << max_diff << ", avg_diff: " << (avg/count) << std::endl; } void test_compressed_int4_scale(bool is_caching_test, bool is_dynamic, long int batch_num, long int scales_group_size = 128, bool is_wei_dyn = false) { @@ -2903,7 +2903,7 @@ class fully_connected_gpu_tests: public ::testing::Test { auto config = get_test_default_config(engine); config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); - ov::intel_gpu::ImplementationDesc fc_impl_desc = { format::bfyx, "fully_connected_gpu_bf_tiled", impl_types::ocl }; + ov::intel_gpu::ImplementationDesc fc_impl_desc = { format::bfyx, "fully_connected_gpu_bfyx_ref", impl_types::ocl }; config.set_property(ov::intel_gpu::force_implementations(ov::intel_gpu::ImplForcingMap{ {"fc_prim", fc_impl_desc} })); config.set_property(ov::hint::dynamic_quantization_group_size(0)); @@ -2965,11 +2965,137 @@ class fully_connected_gpu_tests: public ::testing::Test { max_diff = abs_diff; avg += abs_diff; count++; - OPENVINO_ASSERT(abs_diff < 5); + OPENVINO_ASSERT(abs_diff < 6); } GPU_DEBUG_LOG << "---> count: " << count << ", max_diff:" << max_diff << ", avg_diff: " << (avg/count) << std::endl; OPENVINO_ASSERT((avg/count) < 0.5); } + + void test_compressed_int8_scale_dyn_quan_weight_u8(bool is_dynamic, int batch = 1, int ifm = 512, int ofm = 2048, + int quantize_group_size = 32, int scales_group_size = 128, + bool is_wzp_test = false, bool is_wzp_scalar = false) { + tests::random_generator rg(GET_SUITE_NAME); + auto& engine = get_test_engine(); + + if (engine.get_device_info().dev_type == device_type::discrete_gpu) + GTEST_SKIP(); + + long int batch_num = batch; + long int ifm_num = ifm; + long int ofm_num = ofm; + long int wzp_num = is_wzp_scalar ? 1 : ofm_num; + + auto input_ps = ov::PartialShape{ batch_num, 1, ifm_num }; + auto input_mem = engine.allocate_memory({ input_ps, data_types::f16, format::bfyx }); + + auto weights_mem = engine.allocate_memory({ {ofm_num, ifm_num}, data_types::u8, format::bfyx }); + auto scale_mem = engine.allocate_memory({ {ofm_num, ifm_num / scales_group_size}, data_types::f16, format::fbyx }); + auto dcomp_zp_mem = engine.allocate_memory({ {wzp_num, 1}, data_types::u8, format::bfyx }); + + + auto input_data = rg.generate_random_1d(batch_num * ifm_num, -2.f, 2.f); + set_values(input_mem, input_data); + + auto weigths_data = rg.generate_random_1d(ofm_num * ifm_num, 0, 4); + set_values(weights_mem, weigths_data); + + auto scale_data = rg.generate_random_1d(ofm_num * ifm_num / scales_group_size, -2.f, 2.f); + set_values(scale_mem, scale_data); + + if (is_wzp_test) { + auto zp_data = rg.generate_random_1d(wzp_num, 0, 2); + set_values(dcomp_zp_mem, zp_data); + } + + auto in_layout = is_dynamic ? layout{ ov::PartialShape{ -1, -1, -1 }, data_types::f16, format::bfyx } + : layout{ input_ps, data_types::f16, format::bfyx }; + + auto dcomp_zp_name = is_wzp_test ? "wzp" : ""; + auto fc_prim = fully_connected("fc_prim", input_info("input"), "weights", "", "scale", dcomp_zp_name, data_types::f16, 3, 2); + + if (is_wzp_test) { + fc_prim.compressed_weights = true; + fc_prim.decompression_zero_point = is_wzp_test ? "wzp" : ""; + } + + // Implemented dynamic quantize kernel + auto get_ref_results = [&]() { + topology topo; + topo.add(input_layout("input", in_layout)); + topo.add(data("weights", weights_mem)); + topo.add(data("scale", scale_mem)); + topo.add(data("wzp", dcomp_zp_mem)); + topo.add(fc_prim); + + auto config = get_test_default_config(engine); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + ov::intel_gpu::ImplementationDesc fc_impl_desc = { format::bfyx, "fully_connected_gpu_bf_tiled", impl_types::ocl }; + config.set_property(ov::intel_gpu::force_implementations(ov::intel_gpu::ImplForcingMap{ {"fc_prim", fc_impl_desc} })); + config.set_property(ov::hint::dynamic_quantization_group_size(0)); + + network network(engine, topo, config); + network.set_input_data("input", input_mem); + + auto outputs = network.execute(); + OPENVINO_ASSERT(outputs.size() == 1); + OPENVINO_ASSERT(outputs.begin()->first == "fc_prim"); + + auto output_layout = outputs.begin()->second.get_layout(); + auto output_mem = outputs.begin()->second.get_memory(); + + return engine.reinterpret_buffer(*output_mem, output_layout); + }; + + topology topology( + input_layout("input", in_layout), + data("weights", weights_mem), + data("scale", scale_mem), + data("wzp", dcomp_zp_mem), + fc_prim + ); + + auto config = get_test_default_config(engine); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + config.set_property(ov::intel_gpu::optimize_data(true)); + config.set_property(ov::hint::dynamic_quantization_group_size(quantize_group_size)); + + network::ptr network = get_network(engine, topology, config, get_test_stream_ptr(), false); + + if (is_dynamic && !engine.get_device_info().supports_immad) { + auto inst = network->get_primitive("fc_prim"); + auto impl = inst->get_impl(); + ASSERT_TRUE(impl != NULL); + auto kernel_num = (is_dynamic) ? 3 : 2; + kernel_num = (quantize_group_size < 32) ? 2 : kernel_num; + ASSERT_EQ(impl->get_kernels().size(), size_t(kernel_num)); + } + + network->set_input_data("input", input_mem); + + auto outputs = network->execute(); + ASSERT_EQ(outputs.size(), size_t(1)); + ASSERT_EQ(outputs.begin()->first, "fc_prim"); + + auto output_mem = outputs.begin()->second.get_memory(); + cldnn::mem_lock output_ptr (output_mem, get_test_stream()); + + auto ref_output_mem = get_ref_results(); + cldnn::mem_lock output_ptr_ref (ref_output_mem, get_test_stream()); + + size_t count = 0; + float max_diff = 0.f; + float avg = 0.f; + for (size_t i = 0; i < output_ptr_ref.size(); ++i) { + auto abs_diff = std::abs((float)output_ptr_ref[i] - (float)output_ptr[i]); + if (max_diff < abs_diff) + max_diff = abs_diff; + avg += abs_diff; + count++; + OPENVINO_ASSERT(abs_diff < 8); + } + GPU_DEBUG_LOG << "---> count: " << count << ", max_diff:" << max_diff << ", avg_diff: " << (avg/count) << std::endl; + OPENVINO_ASSERT((avg/count) < 0.8); + } }; using shared_dims = std::tuple; @@ -4064,6 +4190,34 @@ TEST_F(fully_connected_gpu_tests, compressed_int4_scale_dynamic_quantize_wzp_sta this->test_compressed_int4_scale_dyn_quan_weight_i4(false, 320, 1024, 1024, 32, 32, true); } +TEST_F(fully_connected_gpu_tests, compressed_int8_scale_dynamic_quantize_wzp_128_large) { + this->test_compressed_int8_scale_dyn_quan_weight_u8(true, 320, 4096, 4096, 128, 128, true); +} + +TEST_F(fully_connected_gpu_tests, compressed_int8_scale_dynamic_quantize_wzp_32_ifm_1024) { + this->test_compressed_int8_scale_dyn_quan_weight_u8(true, 320, 1024, 1024, 32, 32, true); +} + +TEST_F(fully_connected_gpu_tests, compressed_int8_scale_dynamic_quantize_wzp_32_ifm_2048) { + this->test_compressed_int8_scale_dyn_quan_weight_u8(true, 320, 2048, 2048, 32, 32, true); +} + +TEST_F(fully_connected_gpu_tests, compressed_int8_scale_dynamic_quantize_wzp_32_ifm_4096) { + this->test_compressed_int8_scale_dyn_quan_weight_u8(true, 320, 4096, 4096, 32, 32, true); +} + +TEST_F(fully_connected_gpu_tests, compressed_int8_scale_dynamic_quantize_wzp_32_large_unaligned) { + this->test_compressed_int8_scale_dyn_quan_weight_u8(true, 310, 1024, 1024, 32, 32, true); +} + +TEST_F(fully_connected_gpu_tests, compressed_int8_scale_dynamic_quantize_wzp_128_small) { + this->test_compressed_int8_scale_dyn_quan_weight_u8(true, 16, 1024, 1024, 128, 128, true); +} + +TEST_F(fully_connected_gpu_tests, compressed_int8_scale_dynamic_quantize_wzp_128_single) { + this->test_compressed_int8_scale_dyn_quan_weight_u8(true, 1, 1024, 1024, 128, 128, true); +} + TEST_F(fully_connected_gpu_tests, compressed_scale_bias) { this->test_compressed_scale_bias(false); } diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/gemm_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/gemm_gpu_test.cpp index 3b41f44050e527..df493544624b64 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/gemm_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/gemm_gpu_test.cpp @@ -473,6 +473,9 @@ class gemm_gpu_tests: public ::testing::Test { config.set_property(ov::intel_gpu::optimize_data(true)); config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + ov::intel_gpu::ImplementationDesc gemm_impl = { format::bfyx, "", impl_types::ocl }; + config.set_property(ov::intel_gpu::force_implementations(ov::intel_gpu::ImplForcingMap{ {"gemm_ref", gemm_impl} })); + network network(engine, topology, config); network.set_input_data("input1", input1_mem); network.set_input_data("input2", input2_mem); @@ -498,6 +501,10 @@ class gemm_gpu_tests: public ::testing::Test { ExecutionConfig config = get_test_default_config(engine); config.set_property(ov::intel_gpu::optimize_data(true)); config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + + ov::intel_gpu::ImplementationDesc gemm_impl = { format::bfyx, "", impl_types::ocl }; + config.set_property(ov::intel_gpu::force_implementations(ov::intel_gpu::ImplForcingMap{ {"gemm", gemm_impl} })); + network::ptr network = get_network(engine, topology, config, get_test_stream_ptr(), is_caching_test); network->set_input_data("input1", input1_mem); network->set_input_data("input2", input2_mem); @@ -1246,10 +1253,12 @@ class gemm_gpu_tests: public ::testing::Test { network->set_input_data("input0", input0_mem); network->set_input_data("input1", input1_mem); - auto inst = network->get_primitive("gemm"); - auto impl = inst->get_impl(); - ASSERT_TRUE(impl != nullptr); - ASSERT_TRUE(impl->is_dynamic() == is_input_dynamic); + if (!engine.get_device_info().supports_immad) { + auto inst = network->get_primitive("gemm"); + auto impl = inst->get_impl(); + ASSERT_TRUE(impl != nullptr); + ASSERT_TRUE(impl->is_dynamic() == is_input_dynamic); + } auto outputs = network->execute(); @@ -1533,10 +1542,12 @@ class gemm_gpu_tests: public ::testing::Test { network->set_input_data("input0", input0_mem); network->set_input_data("input1", input1_mem); - auto inst = network->get_primitive("gemm"); - auto impl = inst->get_impl(); - ASSERT_TRUE(impl != nullptr); - ASSERT_TRUE(impl->is_dynamic() == is_input_dynamic); + if (!engine.get_device_info().supports_immad) { + auto inst = network->get_primitive("gemm"); + auto impl = inst->get_impl(); + ASSERT_TRUE(impl != nullptr); + ASSERT_TRUE(impl->is_dynamic() == is_input_dynamic); + } auto outputs = network->execute(); @@ -2853,8 +2864,10 @@ class gemm_onednn: public ::testing::Test { auto inst = network->get_primitive("gemm"); auto impl = inst->get_impl(); - ASSERT_TRUE(impl != nullptr); - ASSERT_TRUE(impl->is_dynamic()); + if (!engine.get_device_info().supports_immad) { + ASSERT_TRUE(impl != nullptr); + ASSERT_TRUE(impl->is_dynamic()); + } auto outputs = network->execute(); diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/variable.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/variable.cpp index 3bf1a771512ae8..59e31547602252 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/variable.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/variable.cpp @@ -35,7 +35,7 @@ struct variable_test : public ::testing::TestWithParam> { topology topology; topology.add(input_layout("input", input_data->get_layout())); - topology.add(read_value{"read_value", { input_info("input") }, "v0", variable_layout}); + topology.add(read_value{"read_value", { input_info("input") }, "v0", { variable_layout }}); topology.add(eltwise{"sum", { input_info("input"), input_info("read_value") }, eltwise_mode::sum, {}, variable_layout.data_type}); topology.add(assign{"assign", { input_info("sum") }, "v0", variable_layout}); @@ -129,7 +129,7 @@ void test_exception_on_wrong_layout(bool is_caching_test) { topology topology; topology.add(input_layout("input", input_data->get_layout())); - topology.add(read_value{"read_value", { input_info("input") }, "v0", variable_layout}); + topology.add(read_value{"read_value", { input_info("input") }, "v0", { variable_layout }}); topology.add(input_layout("wrong_input", wrong_input_data->get_layout())); topology.add(assign{"assign", { input_info("wrong_input") }, "v0", wrong_layout}); @@ -218,14 +218,14 @@ void test_variables_are_preserved_across_inferences(bool is_caching_test) { topology.add(assign{"assign_2", { input_info("input_2") }, "v2", variable_layout}); topology.add(data("dummy1", dummy1)); - topology.add(read_value{"read_value_1", { input_info("dummy1") }, "v1", variable_layout}); - topology.add(read_value{"read_value_2", { input_info("dummy1") }, "v2", variable_layout}); + topology.add(read_value{"read_value_1", { input_info("dummy1") }, "v1", { variable_layout }}); + topology.add(read_value{"read_value_2", { input_info("dummy1") }, "v2", { variable_layout }}); topology.add(eltwise{"sum", { input_info("read_value_1"), input_info("read_value_2") }, eltwise_mode::sum, {}, variable_layout.data_type}); topology.add(assign{"assign_result", { input_info("sum") }, "v_result", variable_layout}); topology.add(data("dummy2", dummy2)); - topology.add(read_value{"read_result", { input_info("dummy2") }, "v_result", variable_layout}); + topology.add(read_value{"read_result", { input_info("dummy2") }, "v_result", { variable_layout }}); cldnn::network::ptr network = get_network(engine, topology, get_test_default_config(engine), get_test_stream_ptr(), is_caching_test); diff --git a/src/plugins/intel_gpu/tests/unit/transformations/fc_per_layer_scaling_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/fc_per_layer_scaling_test.cpp new file mode 100644 index 00000000000000..2d2f21b57d7152 --- /dev/null +++ b/src/plugins/intel_gpu/tests/unit/transformations/fc_per_layer_scaling_test.cpp @@ -0,0 +1,117 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include "common_test_utils/graph_comparator.hpp" +#include "common_test_utils/ov_test_utils.hpp" + +#include +#include + +#include "openvino/op/constant.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/pass/manager.hpp" + +#include +#include "plugin/transformations/fc_per_layer_scaling.hpp" +#include "intel_gpu/op/placeholder.hpp" +#include "intel_gpu/op/fully_connected_compressed.hpp" + +using namespace testing; +using namespace ov::intel_gpu; + +namespace ov { +namespace test { +namespace intel_gpu { + +TEST_F(TransformationTestsF, FullyConnectedPerLayerScalingTest1) { + float scale_factor = 2.f; + { + auto input = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 16 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{ 32, 16 }, { 1 }); + auto no_bias = std::make_shared(); + auto scale_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto zp_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto fc_compressed = std::make_shared(input, weights_const, no_bias, scale_const, zp_const); + auto convert = std::make_shared(fc_compressed, ov::element::f32); + auto result = std::make_shared(convert); + + model = std::make_shared(ov::ResultVector{result}, ov::ParameterVector{input}); + manager.register_pass(scale_factor); + } + { + auto input = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 16 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{ 32, 16 }, { 1 }); + auto no_bias = std::make_shared(); + auto scale_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto zp_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto scale_down_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 1 }, { 1.f / scale_factor }); + auto scale_down = std::make_shared(input, scale_down_const); + auto fc_compressed = std::make_shared(scale_down, weights_const, no_bias, scale_const, zp_const); + auto scale_up_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 1 }, { scale_factor }); + auto scale_up = std::make_shared(fc_compressed, scale_up_const); + auto convert = std::make_shared(scale_up, ov::element::f32); + auto result = std::make_shared(convert); + + model_ref = std::make_shared(ov::ResultVector{result}, ov::ParameterVector{input}); + comparator.enable(FunctionsComparator::ATTRIBUTES); + } +} + +TEST_F(TransformationTestsF, FullyConnectedPerLayerScalingTest2) { + float scale_factor = 2.f; + { + auto input = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 16 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{ 32, 16 }, { 1 }); + auto bias = std::make_shared(ov::element::f16, ov::Shape{ 1, 32 }); + auto scale_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto zp_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto fc_compressed = std::make_shared(input, weights_const, bias, scale_const, zp_const); + auto convert = std::make_shared(fc_compressed, ov::element::f32); + auto result = std::make_shared(convert); + + model = std::make_shared(ov::ResultVector{result}, ov::ParameterVector{input}); + manager.register_pass(scale_factor); + } + { + auto input = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 16 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{ 32, 16 }, { 1 }); + auto bias = std::make_shared(ov::element::f16, ov::Shape{ 1, 32 }); + auto scale_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto zp_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto scale_down_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 1 }, { 1.f / scale_factor }); + auto scale_down = std::make_shared(input, scale_down_const); + auto bias_scale_down_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 1 }, { 1.f / scale_factor }); + auto bias_scale_down = std::make_shared(bias, scale_down_const); + auto fc_compressed = std::make_shared(scale_down, weights_const, bias_scale_down, scale_const, zp_const); + auto scale_up_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 1 }, { scale_factor }); + auto scale_up = std::make_shared(fc_compressed, scale_up_const); + auto convert = std::make_shared(scale_up, ov::element::f32); + auto result = std::make_shared(convert); + + model_ref = std::make_shared(ov::ResultVector{result}, ov::ParameterVector{input}); + comparator.enable(FunctionsComparator::ATTRIBUTES); + } +} + +TEST_F(TransformationTestsF, FullyConnectedPerLayerScalingTest3) { + { + auto input = std::make_shared(ov::element::f16, ov::PartialShape{ -1, 16 }); + auto weights_const = ov::op::v0::Constant::create(ov::element::u8, ov::Shape{ 32, 16 }, { 1 }); + auto no_bias = std::make_shared(); + auto scale_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto zp_const = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{ 32, 1 }, { 1 }); + auto fc_compressed = std::make_shared(input, weights_const, no_bias, scale_const, zp_const); + auto convert = std::make_shared(fc_compressed, ov::element::f32); + auto result = std::make_shared(convert); + + model = std::make_shared(ov::ResultVector{result}, ov::ParameterVector{input}); + manager.register_pass(1.f); + } +} + +} // namespace intel_gpu +} // namespace test +} // namespace ov \ No newline at end of file diff --git a/src/plugins/intel_gpu/tests/unit/transformations/kv_cache_compression.cpp b/src/plugins/intel_gpu/tests/unit/transformations/kv_cache_compression.cpp new file mode 100644 index 00000000000000..67123f1d84cfe7 --- /dev/null +++ b/src/plugins/intel_gpu/tests/unit/transformations/kv_cache_compression.cpp @@ -0,0 +1,344 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "common_test_utils/ov_test_utils.hpp" + +#include "openvino/core/model.hpp" +#include "openvino/pass/manager.hpp" +#include "openvino/op/abs.hpp" +#include "openvino/op/broadcast.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/gather.hpp" +#include "openvino/op/reshape.hpp" +#include "openvino/op/unsqueeze.hpp" + +#include "intel_gpu/op/read_value.hpp" +#include "intel_gpu/op/read_values.hpp" +#include "intel_gpu/op/kv_cache.hpp" +#include "intel_gpu/op/kv_cache_compressed.hpp" +#include "intel_gpu/op/indirect_sdpa.hpp" + +#include "plugin/transformations/kv_cache_compression.hpp" + +using namespace testing; +using namespace ov::intel_gpu; + +namespace ov { +namespace test { +namespace intel_gpu { + +TEST_F(TransformationTestsF, KVCacheCompression) { + bool causal = false; + bool with_mask = true; + bool with_scale = true; + size_t concat_axis = 2; + size_t gather_axis = 0; + ov::element::Type_t element_type = ov::element::f16; + std::vector qkv_order = {0, 1, 2, 3}; + std::shared_ptr mask = nullptr; + std::shared_ptr scale = nullptr; + ov::PartialShape input_shape = ov::PartialShape{1, 32, -1, 80}; + + { + auto query = std::make_shared(element_type, input_shape); + auto beam_idx = std::make_shared(ov::element::i32, ov::PartialShape{1}); + + auto key_variable = std::make_shared(ov::op::util::VariableInfo{{1, 32, -1, 80}, ov::element::f16, "v0"}); + auto key_current = std::make_shared(ov::element::f16, input_shape); + auto key_past = std::make_shared(key_variable); + auto key_cache = std::make_shared(key_past, key_current, beam_idx, key_variable, concat_axis, gather_axis); + + auto value_variable = std::make_shared(ov::op::util::VariableInfo{{1, 32, -1, 80}, ov::element::f16, "v1"}); + auto value_current = std::make_shared(ov::element::f16, input_shape); + auto value_past = std::make_shared(value_variable); + auto value_cache = std::make_shared(value_past, value_current, beam_idx, value_variable, concat_axis, gather_axis); + + ov::ParameterVector params{ beam_idx, query, key_current, value_current }; + + if (with_mask) { + auto attn_mask = std::make_shared(element_type, ov::PartialShape::dynamic(4)); + mask = attn_mask; + params.push_back(attn_mask); + } + + if (with_mask && with_scale) { + auto scale_input = std::make_shared(element_type, ov::PartialShape{1}); + scale = scale_input; + params.push_back(scale_input); + } + + ov::OutputVector sdpa_inputs = { query, key_cache->output(0), value_cache->output(0) }; + + if (mask) { + sdpa_inputs.push_back(mask); + } + + if (scale) { + sdpa_inputs.push_back(scale); + } + + std::shared_ptr sdpa = nullptr; + sdpa = std::make_shared(sdpa_inputs, + key_cache->output(1), + causal, + gather_axis, + qkv_order, + qkv_order, + qkv_order, + ov::intel_gpu::op::SDPA::default_order(4)); + + auto result = std::make_shared(sdpa); + + ov::ResultVector results{ result }; + + model = std::make_shared(results, params); + manager.register_pass(ov::element::i8); + } + { + ov::op::internal::DynamicQuantize::Attributes dq_config; + dq_config.quantization_type = ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric; + dq_config.quantization_dt = ov::element::i8; + dq_config.scale_dt = ov::element::f16; + dq_config.zp_dt = ov::element::f16; + dq_config.group_sizes = { 1, 1, 1, UINT64_MAX }; + dq_config.scales_zp_output_order = { 0, 1, 2, 3 }; + dq_config.output_storage_type = ov::op::internal::DynamicQuantize::OutputStorageType::InterleavedScalesZP; + + auto query = std::make_shared(element_type, input_shape); + auto beam_idx = std::make_shared(ov::element::i32, ov::PartialShape{1}); + + auto key_variable = std::make_shared(ov::op::util::VariableInfo{{1, 32, -1, 80}, ov::element::f16, "v0"}); + auto key_current = std::make_shared(ov::element::f16, input_shape); + auto key_past_variable_infos = { ov::op::util::VariableInfo{{1, 32, -1, 80}, ov::element::i8, "v0"}, + ov::op::util::VariableInfo{{1, 32, -1, 2}, ov::element::f16, "v0"} }; + auto key_past_compressed = std::make_shared(key_variable, key_past_variable_infos); + auto key_cache_inputs = ov::OutputVector{ key_past_compressed->output(0), key_current, beam_idx, key_past_compressed->output(1) }; + auto key_cache = std::make_shared(key_cache_inputs, + key_variable, + concat_axis, + gather_axis, + dq_config); + + auto value_variable = std::make_shared(ov::op::util::VariableInfo{{1, 32, -1, 80}, ov::element::f16, "v1"}); + auto value_current = std::make_shared(ov::element::f16, input_shape); + auto value_past_variable_infos = { ov::op::util::VariableInfo{{1, 32, -1, 80}, ov::element::i8, "v1"}, + ov::op::util::VariableInfo{{1, 32, -1, 2}, ov::element::f16, "v1"} }; + auto value_past_compressed = std::make_shared(value_variable, value_past_variable_infos); + auto value_cache_inputs = ov::OutputVector{ value_past_compressed->output(0), value_current, beam_idx, value_past_compressed->output(1) }; + auto value_cache = std::make_shared(value_cache_inputs, + value_variable, + concat_axis, + gather_axis, + dq_config); + + ov::ParameterVector params{ beam_idx, query, key_current, value_current }; + + if (with_mask) { + auto attn_input = std::make_shared(element_type, ov::PartialShape::dynamic(4)); + mask = attn_input; + params.push_back(attn_input); + } + + if (with_mask && with_scale) { + auto scale_input = std::make_shared(element_type, ov::PartialShape{1}); + scale = scale_input; + params.push_back(scale_input); + } + + ov::OutputVector sdpa_inputs = { query, key_cache->output(0), value_cache->output(0) }; + if (mask) { + sdpa_inputs.push_back(mask); + } + + if (scale) { + sdpa_inputs.push_back(scale); + } + + sdpa_inputs.push_back(key_cache->output(2)); + sdpa_inputs.push_back(value_cache->output(2)); + + std::shared_ptr sdpa = nullptr; + sdpa = std::make_shared(sdpa_inputs, + key_cache->output(1), + causal, + gather_axis, + qkv_order, + qkv_order, + qkv_order, + ov::intel_gpu::op::SDPA::default_order(4), + dq_config); + + auto result = std::make_shared(sdpa); + + ov::ResultVector results{ result }; + + model_ref = std::make_shared(results, params); + } +} + +TEST_F(TransformationTestsF, KVCacheCompressionWithInitializers) { + bool causal = false; + bool with_mask = true; + bool with_scale = true; + size_t concat_axis = 2; + size_t gather_axis = 0; + ov::element::Type_t element_type = ov::element::f16; + std::vector qkv_order = {0, 1, 2, 3}; + std::shared_ptr mask = nullptr; + std::shared_ptr scale = nullptr; + ov::PartialShape input_shape = ov::PartialShape{1, 32, -1, 80}; + + { + auto query = std::make_shared(element_type, input_shape); + auto beam_idx = std::make_shared(ov::element::i32, ov::PartialShape{1}); + + auto key_variable_initializer = std::make_shared(ov::element::f16, input_shape); + auto key_current = std::make_shared(ov::element::f16, input_shape); + auto key_variable = std::make_shared(ov::op::util::VariableInfo{{1, 32, -1, 80}, ov::element::f16, "v0"}); + auto key_past = std::make_shared(key_variable_initializer, key_variable); + auto key_cache = std::make_shared(key_past, key_current, beam_idx, key_variable, concat_axis, gather_axis); + + auto value_variable_initializer = std::make_shared(ov::element::f16, input_shape); + auto value_variable = std::make_shared(ov::op::util::VariableInfo{{1, 32, -1, 80}, ov::element::f16, "v1"}); + auto value_current = std::make_shared(ov::element::f16, input_shape); + auto value_past = std::make_shared(value_variable_initializer, value_variable); + auto value_cache = std::make_shared(value_past, value_current, beam_idx, value_variable, concat_axis, gather_axis); + + ov::ParameterVector params{ beam_idx, query, key_current, value_current, key_variable_initializer, value_variable_initializer }; + + if (with_mask) { + auto attn_mask = std::make_shared(element_type, ov::PartialShape::dynamic(4)); + mask = attn_mask; + params.push_back(attn_mask); + } + + if (with_mask && with_scale) { + auto scale_input = std::make_shared(element_type, ov::PartialShape{1}); + scale = scale_input; + params.push_back(scale_input); + } + + ov::OutputVector sdpa_inputs = { query, key_cache->output(0), value_cache->output(0) }; + + if (mask) { + sdpa_inputs.push_back(mask); + } + + if (scale) { + sdpa_inputs.push_back(scale); + } + + std::shared_ptr sdpa = nullptr; + sdpa = std::make_shared(sdpa_inputs, + key_cache->output(1), + causal, + gather_axis, + qkv_order, + qkv_order, + qkv_order, + ov::intel_gpu::op::SDPA::default_order(4)); + + auto result = std::make_shared(sdpa); + + ov::ResultVector results{ result }; + + model = std::make_shared(results, params); + manager.register_pass(ov::element::i8); + } + { + ov::op::internal::DynamicQuantize::Attributes dq_config; + dq_config.quantization_type = ov::op::internal::DynamicQuantize::QuantizationType::Asymmetric; + dq_config.quantization_dt = ov::element::i8; + dq_config.scale_dt = ov::element::f16; + dq_config.zp_dt = ov::element::f16; + dq_config.group_sizes = { 1, 1, 1, UINT64_MAX }; + dq_config.scales_zp_output_order = { 0, 1, 2, 3 }; + dq_config.output_storage_type = ov::op::internal::DynamicQuantize::OutputStorageType::InterleavedScalesZP; + + auto query = std::make_shared(element_type, input_shape); + auto beam_idx = std::make_shared(ov::element::i32, ov::PartialShape{1}); + + auto key_past_variable_infos = { ov::op::util::VariableInfo{{1, 32, -1, 80}, ov::element::i8, "v0"}, + ov::op::util::VariableInfo{{1, 32, -1, 2}, ov::element::f16, "v0"} }; + auto key_current = std::make_shared(ov::element::f16, input_shape); + auto key_variable_initializer = std::make_shared(ov::element::f16, input_shape); + auto key_variable = std::make_shared(ov::op::util::VariableInfo{{1, 32, -1, 80}, ov::element::f16, "v0"}); + + auto key_initializer_dq = + std::make_shared(key_variable_initializer, dq_config); + auto key_past_initializers = ov::OutputVector{ key_initializer_dq->output(0), key_initializer_dq->output(1) }; + auto key_past_compressed = std::make_shared(key_past_initializers, key_variable, key_past_variable_infos); + auto key_cache_inputs = ov::OutputVector{ key_past_compressed->output(0), key_current, beam_idx, key_past_compressed->output(1) }; + auto key_cache = std::make_shared(key_cache_inputs, + key_variable, + concat_axis, + gather_axis, + dq_config); + + auto value_past_variable_infos = { ov::op::util::VariableInfo{{1, 32, -1, 80}, ov::element::i8, "v1"}, + ov::op::util::VariableInfo{{1, 32, -1, 2}, ov::element::f16, "v1"} }; + + auto value_current = std::make_shared(ov::element::f16, input_shape); + auto value_variable_initializer = std::make_shared(ov::element::f16, input_shape); + auto value_variable = std::make_shared(ov::op::util::VariableInfo{{1, 32, -1, 80}, ov::element::f16, "v1"}); + + auto value_initializer_dq = + std::make_shared(value_variable_initializer, dq_config); + auto value_past_initializers = ov::OutputVector{ value_initializer_dq->output(0), value_initializer_dq->output(1) }; + auto value_past_compressed = std::make_shared(value_past_initializers, value_variable, value_past_variable_infos); + auto value_cache_inputs = ov::OutputVector{ value_past_compressed->output(0), value_current, beam_idx, value_past_compressed->output(1) }; + auto value_cache = std::make_shared(value_cache_inputs, + value_variable, + concat_axis, + gather_axis, + dq_config); + + ov::ParameterVector params{ beam_idx, query, key_current, value_current, key_variable_initializer, value_variable_initializer }; + + if (with_mask) { + auto attn_input = std::make_shared(element_type, ov::PartialShape::dynamic(4)); + mask = attn_input; + params.push_back(attn_input); + } + + if (with_mask && with_scale) { + auto scale_input = std::make_shared(element_type, ov::PartialShape{1}); + scale = scale_input; + params.push_back(scale_input); + } + + ov::OutputVector sdpa_inputs = { query, key_cache->output(0), value_cache->output(0) }; + if (mask) { + sdpa_inputs.push_back(mask); + } + + if (scale) { + sdpa_inputs.push_back(scale); + } + + sdpa_inputs.push_back(key_cache->output(2)); + sdpa_inputs.push_back(value_cache->output(2)); + + std::shared_ptr sdpa = nullptr; + sdpa = std::make_shared(sdpa_inputs, + key_cache->output(1), + causal, + gather_axis, + qkv_order, + qkv_order, + qkv_order, + ov::intel_gpu::op::SDPA::default_order(4), + dq_config); + + auto result = std::make_shared(sdpa); + + ov::ResultVector results{ result }; + + model_ref = std::make_shared(results, params); + } +} + +} // namespace intel_gpu +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_gpu/tests/unit/transformations/optimize_subsequent_reshapes_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/optimize_subsequent_reshapes_test.cpp new file mode 100644 index 00000000000000..732a14be03bf39 --- /dev/null +++ b/src/plugins/intel_gpu/tests/unit/transformations/optimize_subsequent_reshapes_test.cpp @@ -0,0 +1,97 @@ +// Copyright (C) 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +#include +#include + +#include "openvino/pass/manager.hpp" +#include "openvino/core/model.hpp" +#include "openvino/core/coordinate_diff.hpp" +#include "openvino/core/type/element_type.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/reshape.hpp" + +#include +#include +#include + +#include "common_test_utils/ov_test_utils.hpp" + +using namespace testing; +using namespace ov::intel_gpu; + +TEST_F(TransformationTestsF, OptimizeSubsequentReshapes1) { + { + auto input = std::make_shared(ov::element::i64, ov::PartialShape{ -1, 1, 4096 }); + auto first_reshape_pattern = std::make_shared(ov::element::i32, ov::Shape{4}, std::vector{ 0, 0, 32, 128 }); + auto first_reshape = std::make_shared(input, first_reshape_pattern, true); + + auto second_reshape_pattern = std::make_shared(ov::element::i32, ov::Shape{2}, std::vector{ 0, -1 }); + auto second_reshape = std::make_shared(first_reshape, second_reshape_pattern, true); + auto result = std::make_shared(second_reshape); + + model = std::make_shared(ov::NodeVector{ result }, ov::ParameterVector{ input }); + manager.register_pass(); + } + { + auto input = std::make_shared(ov::element::i64, ov::PartialShape{ -1, 1, 4096 }); + auto reshape_pattern = std::make_shared(ov::element::i32, ov::Shape{2}, std::vector{ 0, 4096 }); + auto reshape = std::make_shared(input, reshape_pattern, true); + auto result = std::make_shared(reshape); + + model_ref = std::make_shared(ov::NodeVector{ result }, ov::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); +} + +TEST_F(TransformationTestsF, OptimizeSubsequentReshapes2) { + { + auto input = std::make_shared(ov::element::i64, ov::PartialShape{ -1, 1, 4096 }); + auto first_reshape_pattern = std::make_shared(ov::element::i32, ov::Shape{4}, std::vector{ 0, 0, 32, 128 }); + auto first_reshape = std::make_shared(input, first_reshape_pattern, true); + + auto second_reshape_pattern = std::make_shared(ov::element::i32, ov::Shape{4}, std::vector{ 0, 32, 1, 0 }); + auto second_reshape = std::make_shared(first_reshape, second_reshape_pattern, true); + auto result = std::make_shared(second_reshape); + + model = std::make_shared(ov::NodeVector{ result }, ov::ParameterVector{ input }); + manager.register_pass(); + } + { + auto input = std::make_shared(ov::element::i64, ov::PartialShape{ -1, 1, 4096 }); + auto reshape_pattern = std::make_shared(ov::element::i32, ov::Shape{4}, std::vector{ 0, 32, 1, 128 }); + auto reshape = std::make_shared(input, reshape_pattern, true); + auto result = std::make_shared(reshape); + + model_ref = std::make_shared(ov::NodeVector{ result }, ov::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); +} + +TEST_F(TransformationTestsF, OptimizeSubsequentReshapes3) { + { + auto input = std::make_shared(ov::element::i64, ov::PartialShape{ -1, 32, 1, 128 }); + auto first_reshape_pattern = std::make_shared(ov::element::i32, ov::Shape{4}, std::vector{ 0, 1, 32, 0 }); + auto first_reshape = std::make_shared(input, first_reshape_pattern, true); + + auto second_reshape_pattern = std::make_shared(ov::element::i32, ov::Shape{2}, std::vector{ 0, -1 }); + auto second_reshape = std::make_shared(first_reshape, second_reshape_pattern, true); + auto result = std::make_shared(second_reshape); + + model = std::make_shared(ov::NodeVector{ result }, ov::ParameterVector{ input }); + manager.register_pass(); + } + { + auto input = std::make_shared(ov::element::i64, ov::PartialShape{ -1, 32, 1, 128 }); + auto reshape_pattern = std::make_shared(ov::element::i32, ov::Shape{2}, std::vector{ 0, 4096 }); + auto reshape = std::make_shared(input, reshape_pattern, true); + auto result = std::make_shared(reshape); + + model_ref = std::make_shared(ov::NodeVector{ result }, ov::ParameterVector{ input }); + } + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); +} diff --git a/src/plugins/intel_gpu/thirdparty/onednn_gpu b/src/plugins/intel_gpu/thirdparty/onednn_gpu index e99a84e4914a81..1722066ad4c0f1 160000 --- a/src/plugins/intel_gpu/thirdparty/onednn_gpu +++ b/src/plugins/intel_gpu/thirdparty/onednn_gpu @@ -1 +1 @@ -Subproject commit e99a84e4914a818c64165a4b52785f606e405c2b +Subproject commit 1722066ad4c0f15495f2d0fcbe9deb2bfd188c36 diff --git a/src/plugins/intel_npu/README.md b/src/plugins/intel_npu/README.md index b7508c68704e32..980faa71a15937 100644 --- a/src/plugins/intel_npu/README.md +++ b/src/plugins/intel_npu/README.md @@ -78,7 +78,7 @@ There is currently no support for multiple devices, which means only one level-z ### Inference pipeline -The result of the model compilation is represented through a NetworkDescription. This model description is passed by the plugin to the driver to create a level zero graph instance and obtain a graph handle that can later be used to execute multiple inferences in parallel for the same model. Since the same model instance is shared across all subsequent inference objects, this initialization step is performed by default right after the model is compiled and it can be postponed until the creation of the first inference request through the use of an environment variable: "IE_NPU_CREATE_EXECUTOR" (IE_NPU_CREATE_EXECUTOR=0 to postpone the initialization). +The result of the model compilation is represented through an IGraph object, which contains a valid level zero graph handle that can later be used to execute multiple inferences in parallel for the same model. By default, weights are loaded into the NPU memory right after the model is compiled, but this step can be postponed until the creation of the first inference request through the use of an internal NPU property: "NPU_DEFER_WEIGHTS_LOAD". Users can create one or more inference requests for a compiled model using OpenVINO API: diff --git a/src/plugins/intel_npu/cmake/features.cmake b/src/plugins/intel_npu/cmake/features.cmake index 0dde0f9d67f6e5..7d34c52c6d1292 100644 --- a/src/plugins/intel_npu/cmake/features.cmake +++ b/src/plugins/intel_npu/cmake/features.cmake @@ -4,29 +4,10 @@ ov_option(ENABLE_MLIR_COMPILER "Enable compilation of npu_mlir_compiler libraries" ON) -ov_option(ENABLE_NPU_RUNTIME_COMMON "Enable compilation of npu runtime common libraries" ON) +ov_option(ENABLE_NPU_PLUGIN_ENGINE "Enable compilation of NPU plugin engine" ON) -# if ENABLE_ZEROAPI_BACKEND=ON, it adds the ze_loader dependency for driver compiler -ov_dependent_option(ENABLE_NPU_PLUGIN_ENGINE "Enable compilation of NPU plugin engine" ON "ENABLE_NPU_RUNTIME_COMMON" OFF) - -ov_dependent_option(ENABLE_ZEROAPI_BACKEND "Enable zero-api as a plugin backend" ON "ENABLE_NPU_RUNTIME_COMMON;ENABLE_NPU_PLUGIN_ENGINE" OFF) - -ov_dependent_option(ENABLE_DRIVER_COMPILER_ADAPTER "Enable NPU Compiler inside driver" ON "ENABLE_ZEROAPI_BACKEND" OFF) - -if((NOT ENABLE_NPU_PLUGIN_ENGINE OR NOT ENABLE_NPU_RUNTIME_COMMON) AND ENABLE_TESTS) - message(FATAL_ERROR "Tests depends on npu plugin engine and npu runtime common libraries!") -endif() - -if((NOT ENABLE_NPU_PLUGIN_ENGINE OR NOT ENABLE_NPU_RUNTIME_COMMON) AND ENABLE_ZEROAPI_BACKEND) - message(FATAL_ERROR "Zero backend depends on npu plugin engine and npu common libraries!") -endif() - -if(NOT ENABLE_ZEROAPI_BACKEND AND ENABLE_DRIVER_COMPILER_ADAPTER) - message(FATAL_ERROR "Compiler adapter depends on zero backend to use same context!") -endif() - -if(NOT BUILD_SHARED_LIBS AND NOT ENABLE_MLIR_COMPILER AND NOT ENABLE_DRIVER_COMPILER_ADAPTER) - message(FATAL_ERROR "No compiler found for static build!") +if(NOT ENABLE_NPU_PLUGIN_ENGINE AND ENABLE_TESTS) + message(FATAL_ERROR "Tests depends on npu plugin engine!") endif() ov_dependent_option(ENABLE_IMD_BACKEND "Enable InferenceManagerDemo based NPU AL backend" OFF "NOT WIN32;NOT CMAKE_CROSSCOMPILING" OFF) diff --git a/src/plugins/intel_npu/src/CMakeLists.txt b/src/plugins/intel_npu/src/CMakeLists.txt index 5530eb1f3e59e5..f5d1fd5b41226c 100644 --- a/src/plugins/intel_npu/src/CMakeLists.txt +++ b/src/plugins/intel_npu/src/CMakeLists.txt @@ -9,18 +9,9 @@ add_subdirectory(utils) add_subdirectory(al) -if (ENABLE_NPU_RUNTIME_COMMON) +if (ENABLE_NPU_PLUGIN_ENGINE) add_subdirectory(common) -endif() - -if(ENABLE_DRIVER_COMPILER_ADAPTER AND ENABLE_ZEROAPI_BACKEND) - add_subdirectory(compiler) -endif() - -if(ENABLE_ZEROAPI_BACKEND) + add_subdirectory(compiler_adapter) add_subdirectory(backend) -endif() - -if (ENABLE_NPU_PLUGIN_ENGINE) add_subdirectory(plugin) endif() diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/config/runtime.hpp b/src/plugins/intel_npu/src/al/include/intel_npu/config/runtime.hpp index d52c25f6a3e6a5..510ab7fc43b0c8 100644 --- a/src/plugins/intel_npu/src/al/include/intel_npu/config/runtime.hpp +++ b/src/plugins/intel_npu/src/al/include/intel_npu/config/runtime.hpp @@ -131,6 +131,38 @@ struct CREATE_EXECUTOR final : OptionBase { } }; +// +// DEFER_WEIGHTS_LOAD +// + +struct DEFER_WEIGHTS_LOAD final : OptionBase { + static std::string_view key() { + return ov::intel_npu::defer_weights_load.name(); + } + + static int64_t defaultValue() { + return false; + } + + static constexpr std::string_view getTypeName() { + return "bool"; + } + +#ifdef NPU_PLUGIN_DEVELOPER_BUILD + static std::string_view envVar() { + return "OV_NPU_DEFER_WEIGHTS_LOAD"; + } +#endif + + static bool isPublic() { + return false; + } + + static OptionMode mode() { + return OptionMode::RunTime; + } +}; + // // NUM_STREAMS // diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/icompiler.hpp b/src/plugins/intel_npu/src/al/include/intel_npu/icompiler.hpp index e0a02f12aa2e17..53696396603d9a 100644 --- a/src/plugins/intel_npu/src/al/include/intel_npu/icompiler.hpp +++ b/src/plugins/intel_npu/src/al/include/intel_npu/icompiler.hpp @@ -6,128 +6,12 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include - #include "intel_npu/config/config.hpp" -#include "openvino/core/partial_shape.hpp" -#include "openvino/core/type/element_type.hpp" -#include "openvino/runtime/common.hpp" +#include "intel_npu/network_metadata.hpp" #include "openvino/runtime/profiling_info.hpp" namespace intel_npu { -/** - * @brief A helper structure used for storing metadata corresponding to one input/output entry. - */ -struct IODescriptor { - /** - * @brief The name of the input/output assigned by the compiler. - * @details This value may differ from other name attributes: - * - The compiler could have created additional inputs/outputs (e.g. for representing states). These are not - * found in the original IR model. - * - The compiler may append indices to names in the case where duplicate names are found. - * @note The prefixes introduced by the compiler in order to differentiate the special cases (e.g. states and shape - * tensors) were removed prior to initializing this field. - */ - std::string nameFromCompiler; - - ov::element::Type precision; - - ov::PartialShape shapeFromCompiler; - - /** - * @brief If set to "true", the current object describes a buffer which may be used for altering a state tensor. - * @details This flag is set if the compiler prefixed the name using a "read value" prefix. The state input and - * state output descriptors are also tied using the "relatedDescriptorIndex" attribute. - */ - bool isStateInput = false; - - /** - * @brief If set to "true", the current object describes a buffer which reflects the value of a state tensor. - * @details This flag is set if the compiler prefixed the name using an "assign" prefix. The state input and - * state output descriptors are also tied using the "relatedDescriptorIndex" attribute. - */ - bool isStateOutput = false; - - /** - * @brief If set to "true", the buffer of the tensor described here contains as value the shape of the referenced - * tensor. - * @details This flag is set if the compiler prefixed the name using a "shape" prefix. - * - * The referenced tensor bears the same name ("nameFromCompiler"), but its "isShapeTensor" value is set to - * "false". The two descriptors are also tied using the "relatedDescriptorIndex" attribute. - */ - bool isShapeTensor = false; - - /** - * @brief Points towards a related descriptor. - * @details The related descriptors are defined by (state input, state output) or (dynamic tensor, shape tensor) - * pairs. - */ - std::optional relatedDescriptorIndex; - - /** - * @brief The friendly name of the node extracted from the IR model. - * @details In some cases, this field is required for constructing a dummy model which uses the same input/output - * metadata as the original IR model. - * - * This field may be empty if the I/O entry is not found in the original IR model (i.e. the entry was added by the - * compiler). - */ - std::string nodeFriendlyName; - - /** - * @brief The names of the output tensors extracted from the IR model. - * @details In some cases, this field is required for constructing a dummy model which uses the same input/output - * metadata as the original IR model. - * - * This field may be empty if the I/O entry is not found in the original IR model (i.e. the entry was added by the - * compiler). - */ - std::unordered_set outputTensorNames; - - /** - * @brief The shape extracted from the IR model. - * @details The values may differ from the ones found in "shapeFromCompiler" if batching is to be handled by the - * plugin. - * - * This field may be empty if the I/O entry is not found in the original IR model (i.e. the entry was added - * by the compiler). - */ - std::optional shapeFromIRModel = std::nullopt; -}; - -struct NetworkMetadata final { - std::string name; - - std::vector inputs; - std::vector outputs; - std::vector profilingOutputs; - - size_t numStreams = 1; - - // Used primarily in the CID path to pass the level zero graph handle from compiler to the backend executor - void* graphHandle = nullptr; - - /** - * @brief Binds the (state input, state output) and (dynamic tensor, shape tensor) pairs using the - * "relatedDescriptorIndex" attribute. - * @details For state inputs, the "relatedDescriptorIndex" value is set to the index of the output which bears the - * same name. The reverse is also applied. - * - * For shape tensors, the lookup is performed in the same container (inputs or outputs). The value is once again set - * to the index of the entry which bears the same name. - */ - void bindRelatedDescriptors(); - -}; // namespace intel_npu - /** * @struct NetworkDescription * @brief The object returned by the compiler @@ -138,7 +22,6 @@ struct NetworkDescription final { NetworkDescription(std::vector&& compiledNetwork, NetworkMetadata&& metadata) : compiledNetwork(std::move(compiledNetwork)), metadata(std::move(metadata)) {} - NetworkDescription(NetworkMetadata&& metadata) : metadata(std::move(metadata)) {} // Force move semantics to prevent blob copies NetworkDescription(const NetworkDescription&) = delete; NetworkDescription(NetworkDescription&&) = default; @@ -151,32 +34,6 @@ struct NetworkDescription final { NetworkMetadata metadata; }; -/** - * @struct CompiledNetwork - * @brief Custom container for compiled network, used for export - * @var CompiledNetwork::data - * Pointer to the address of compiled network - * @var CompiledNetwork:size - * Size of the compiled network - * @var CompiledNetwork::ownedStorage - * Plugin owned compiled network storage that is required in case of a driver that - * doesn't support graph extension 1.7, as in this case plugin must create a copy of the compiled network. - * @note It's unsafe to store either data or size outside of the compiled network object as its destructor - * would release the owning container - */ - -struct CompiledNetwork { - const uint8_t* data; - size_t size; - CompiledNetwork(const uint8_t* data, size_t size, std::vector storage) - : data(data), - size(size), - ownedStorage(std::move(storage)) {} - -private: - std::vector ownedStorage; -}; - /** * @interface ICompiler * @brief An interface to be implemented by a concrete compiler to provide @@ -184,12 +41,6 @@ struct CompiledNetwork { */ class ICompiler : public std::enable_shared_from_this { public: - /** - * @brief Returns the maximum OpenVino opset version supported by the compiler - * @return opset version e.g. 11 for opset11 - */ - virtual uint32_t getSupportedOpsetVersion() const = 0; - /** * @brief Transforms a network from the OpenVINO model representation to a format executable * by a NPU device @@ -216,8 +67,6 @@ class ICompiler : public std::enable_shared_from_this { * @param config a reference to NPUConfig containing plugin config options * Note: compilation options will be ignored, * since the network is already compiled - * @param netName a reference to the string describing network name - * to be used for creating network description * @return a shared pointer on an object implementing NetworkDescription interface */ virtual NetworkMetadata parse(const std::vector& network, const Config& config) const = 0; @@ -226,15 +75,6 @@ class ICompiler : public std::enable_shared_from_this { const std::vector& network, const Config& config) const = 0; - // Driver compiler can use this to release graphHandle, if we do not have executor - virtual void release([[maybe_unused]] std::shared_ptr networkDescription){}; - - virtual CompiledNetwork getCompiledNetwork(const NetworkDescription& networkDescription) { - return CompiledNetwork(networkDescription.compiledNetwork.data(), - networkDescription.compiledNetwork.size(), - networkDescription.compiledNetwork); - } - protected: virtual ~ICompiler() = default; }; diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/network_metadata.hpp b/src/plugins/intel_npu/src/al/include/intel_npu/network_metadata.hpp new file mode 100644 index 00000000000000..b7a78b3dfd43e1 --- /dev/null +++ b/src/plugins/intel_npu/src/al/include/intel_npu/network_metadata.hpp @@ -0,0 +1,127 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +// Compiler Interface + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "intel_npu/config/config.hpp" +#include "openvino/core/partial_shape.hpp" +#include "openvino/core/type/element_type.hpp" +#include "openvino/runtime/common.hpp" + +namespace intel_npu { + +/** + * @brief A helper structure used for storing metadata corresponding to one input/output entry. + */ +struct IODescriptor { + /** + * @brief The name of the input/output assigned by the compiler. + * @details This value may differ from other name attributes: + * - The compiler could have created additional inputs/outputs (e.g. for representing states). These are not + * found in the original IR model. + * - The compiler may append indices to names in the case where duplicate names are found. + * @note The prefixes introduced by the compiler in order to differentiate the special cases (e.g. states and shape + * tensors) were removed prior to initializing this field. + */ + std::string nameFromCompiler; + + ov::element::Type precision; + + ov::PartialShape shapeFromCompiler; + + /** + * @brief If set to "true", the current object describes a buffer which may be used for altering a state tensor. + * @details This flag is set if the compiler prefixed the name using a "read value" prefix. The state input and + * state output descriptors are also tied using the "relatedDescriptorIndex" attribute. + */ + bool isStateInput = false; + + /** + * @brief If set to "true", the current object describes a buffer which reflects the value of a state tensor. + * @details This flag is set if the compiler prefixed the name using an "assign" prefix. The state input and + * state output descriptors are also tied using the "relatedDescriptorIndex" attribute. + */ + bool isStateOutput = false; + + /** + * @brief If set to "true", the buffer of the tensor described here contains as value the shape of the referenced + * tensor. + * @details This flag is set if the compiler prefixed the name using a "shape" prefix. + * + * The referenced tensor bears the same name ("nameFromCompiler"), but its "isShapeTensor" value is set to + * "false". The two descriptors are also tied using the "relatedDescriptorIndex" attribute. + */ + bool isShapeTensor = false; + + /** + * @brief Points towards a related descriptor. + * @details The related descriptors are defined by (state input, state output) or (dynamic tensor, shape tensor) + * pairs. + */ + std::optional relatedDescriptorIndex; + + /** + * @brief The friendly name of the node extracted from the IR model. + * @details In some cases, this field is required for constructing a dummy model which uses the same input/output + * metadata as the original IR model. + * + * This field may be empty if the I/O entry is not found in the original IR model (i.e. the entry was added by the + * compiler). + */ + std::string nodeFriendlyName; + + /** + * @brief The names of the output tensors extracted from the IR model. + * @details In some cases, this field is required for constructing a dummy model which uses the same input/output + * metadata as the original IR model. + * + * This field may be empty if the I/O entry is not found in the original IR model (i.e. the entry was added by the + * compiler). + */ + std::unordered_set outputTensorNames; + + /** + * @brief The shape extracted from the IR model. + * @details The values may differ from the ones found in "shapeFromCompiler" if batching is to be handled by the + * plugin. + * + * This field may be empty if the I/O entry is not found in the original IR model (i.e. the entry was added + * by the compiler). + */ + std::optional shapeFromIRModel = std::nullopt; +}; + +struct NetworkMetadata final { + std::string name; + + std::vector inputs; + std::vector outputs; + std::vector profilingOutputs; + + size_t numStreams = 1; + + /** + * @brief Binds the (state input, state output) and (dynamic tensor, shape tensor) pairs using the + * "relatedDescriptorIndex" attribute. + * @details For state inputs, the "relatedDescriptorIndex" value is set to the index of the output which bears the + * same name. The reverse is also applied. + * + * For shape tensors, the lookup is performed in the same container (inputs or outputs). The value is once again set + * to the index of the entry which bears the same name. + */ + void bindRelatedDescriptors(); + +}; // namespace intel_npu + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/al/include/intel_npu/npu_private_properties.hpp b/src/plugins/intel_npu/src/al/include/intel_npu/npu_private_properties.hpp index 0c5a04ce0c0d83..d8fabee177b2b9 100644 --- a/src/plugins/intel_npu/src/al/include/intel_npu/npu_private_properties.hpp +++ b/src/plugins/intel_npu/src/al/include/intel_npu/npu_private_properties.hpp @@ -351,6 +351,13 @@ static constexpr ov::Property batch_mode{"NPU_BATCH_MODE"}; */ static constexpr ov::Property create_executor{"NPU_CREATE_EXECUTOR"}; +/** + * @brief [Only for NPU Plugin] + * Type: boolean, default is false + * This option allows to omit loading the weights until inference is created + */ +static constexpr ov::Property defer_weights_load{"NPU_DEFER_WEIGHTS_LOAD"}; + /** * @brief Read-only property to get the name of used backend */ diff --git a/src/plugins/intel_npu/src/al/src/config/runtime.cpp b/src/plugins/intel_npu/src/al/src/config/runtime.cpp index 10f9b4a7c7222b..759956b6f597df 100644 --- a/src/plugins/intel_npu/src/al/src/config/runtime.cpp +++ b/src/plugins/intel_npu/src/al/src/config/runtime.cpp @@ -21,6 +21,7 @@ void intel_npu::registerRunTimeOptions(OptionsDesc& desc) { desc.add(); desc.add(); desc.add(); + desc.add(); desc.add(); desc.add(); desc.add(); diff --git a/src/plugins/intel_npu/src/backend/CMakeLists.txt b/src/plugins/intel_npu/src/backend/CMakeLists.txt index 01465a8179dc24..5a1585c0a63073 100644 --- a/src/plugins/intel_npu/src/backend/CMakeLists.txt +++ b/src/plugins/intel_npu/src/backend/CMakeLists.txt @@ -25,7 +25,6 @@ target_link_libraries(${TARGET_NAME} PRIVATE openvino::npu_al openvino::npu_common - openvino_npu_zero_result_parser ze_loader ) @@ -33,31 +32,3 @@ target_link_libraries(${TARGET_NAME} # targets install # ov_install_static_lib(${TARGET_NAME} ${NPU_INTERNAL_COMPONENT}) - -if(TARGET ze_loader) - if(NOT BUILD_SHARED_LIBS) - # Support link of static runtime in case system does not have ze_loader - install(TARGETS ze_loader EXPORT OpenVINOTargets - RUNTIME DESTINATION ${OV_CPACK_RUNTIMEDIR} COMPONENT ${NPU_PLUGIN_COMPONENT} - ARCHIVE DESTINATION ${OV_CPACK_ARCHIVEDIR} COMPONENT ${NPU_PLUGIN_COMPONENT} - LIBRARY DESTINATION ${OV_CPACK_LIBRARYDIR} COMPONENT ${NPU_PLUGIN_COMPONENT}) - - install(TARGETS utils EXPORT OpenVINOTargets - RUNTIME DESTINATION ${OV_CPACK_RUNTIMEDIR} COMPONENT ${NPU_PLUGIN_COMPONENT} - ARCHIVE DESTINATION ${OV_CPACK_ARCHIVEDIR} COMPONENT ${NPU_PLUGIN_COMPONENT} - LIBRARY DESTINATION ${OV_CPACK_LIBRARYDIR} COMPONENT ${NPU_PLUGIN_COMPONENT}) - - # export to local tree to build against static build tree - export(TARGETS ze_loader NAMESPACE openvino:: - APPEND FILE "${CMAKE_BINARY_DIR}/OpenVINOTargets.cmake") - - export(TARGETS utils NAMESPACE openvino:: - APPEND FILE "${CMAKE_BINARY_DIR}/OpenVINOTargets.cmake") - endif() - - # Support tests to run with ze_loader - install(TARGETS ze_loader - RUNTIME DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL - LIBRARY DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL) -endif() - diff --git a/src/plugins/intel_npu/src/backend/include/zero_backend.hpp b/src/plugins/intel_npu/src/backend/include/zero_backend.hpp index 68e4f9434418a6..038c7c1d2d9bf9 100644 --- a/src/plugins/intel_npu/src/backend/include/zero_backend.hpp +++ b/src/plugins/intel_npu/src/backend/include/zero_backend.hpp @@ -9,7 +9,7 @@ #include "intel_npu/common/npu.hpp" #include "intel_npu/utils/logger/logger.hpp" -#include "zero_init.hpp" +#include "intel_npu/utils/zero/zero_init.hpp" namespace intel_npu { class ZeroEngineBackend final : public IEngineBackend { @@ -29,15 +29,14 @@ class ZeroEngineBackend final : public IEngineBackend { bool isCommandQueueExtSupported() const override; bool isLUIDExtSupported() const override; + const std::shared_ptr& getInitStruct() const; + void* getContext() const override; - void* getDriverHandle() const; - void* getDeviceHandle() const; - ze_graph_dditable_ext_curr_t& getGraphDdiTable() const; void updateInfo(const Config& config) override; private: - std::shared_ptr _instance; + std::shared_ptr _initStruct; std::map> _devices{}; Logger _logger; diff --git a/src/plugins/intel_npu/src/backend/include/zero_device.hpp b/src/plugins/intel_npu/src/backend/include/zero_device.hpp index e87a602613a92a..50f0d28ed210cd 100644 --- a/src/plugins/intel_npu/src/backend/include/zero_device.hpp +++ b/src/plugins/intel_npu/src/backend/include/zero_device.hpp @@ -10,9 +10,9 @@ #include "intel_npu/common/icompiled_model.hpp" #include "intel_npu/common/npu.hpp" #include "intel_npu/utils/logger/logger.hpp" +#include "intel_npu/utils/zero/zero_init.hpp" +#include "intel_npu/utils/zero/zero_types.hpp" #include "openvino/runtime/intel_npu/remote_properties.hpp" -#include "zero_init.hpp" -#include "zero_types.hpp" namespace intel_npu { @@ -20,9 +20,6 @@ class ZeroDevice : public IDevice { public: ZeroDevice(const std::shared_ptr& initStructs); - std::shared_ptr createExecutor(const std::shared_ptr& networkDescription, - const Config& config) override; - std::string getName() const override; std::string getFullDeviceName() const override; Uuid getUuid() const override; @@ -36,7 +33,6 @@ class ZeroDevice : public IDevice { ov::device::Type getDeviceType() const override; std::shared_ptr createInferRequest(const std::shared_ptr& compiledModel, - const std::shared_ptr& executor, const Config& config) override; void updateInfo(const Config& config) override { log.setLevel(config.get()); @@ -76,8 +72,6 @@ class ZeroDevice : public IDevice { {ov::element::u8, 0.f}, {ov::element::i8, 0.f}}; - uint32_t _group_ordinal; - Logger log; }; } // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/include/zero_executor.hpp b/src/plugins/intel_npu/src/backend/include/zero_executor.hpp deleted file mode 100644 index eeb96defc16441..00000000000000 --- a/src/plugins/intel_npu/src/backend/include/zero_executor.hpp +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include - -#include - -#include "intel_npu/common/npu.hpp" -#include "intel_npu/utils/logger/logger.hpp" -#include "openvino/runtime/properties.hpp" -#include "zero_init.hpp" -#include "zero_wrappers.hpp" - -namespace intel_npu { - -class ZeroExecutor final : public IExecutor { -public: - ZeroExecutor(const std::shared_ptr& initStructs, - const std::shared_ptr& networkDescription, - const Config& config, - const uint32_t& group_ordinal); - - ZeroExecutor(const ZeroExecutor&) = delete; - ZeroExecutor& operator=(const ZeroExecutor&) = delete; - - ~ZeroExecutor() override; - - struct ArgumentDescriptor { - ze_graph_argument_properties_3_t info; - uint32_t idx; - }; - - void setArgumentValue(uint32_t argi_, const void* argv_) const; - void setWorkloadType(const ov::WorkloadType workloadType) const override; - void mutexLock() const; - void mutexUnlock() const; - inline ze_graph_handle_t graph() const { - return _graph; - } - inline std::shared_ptr getInitStructs() const { - return _initStructs; - } - inline const std::shared_ptr& getNetworkDesc() const { - return _networkDesc; - } - inline const std::shared_ptr& getCommandQueue() const { - return _command_queues; - } - inline const uint32_t& get_group_ordinal() const { - return _group_ordinal; - } - inline const std::vector& get_input_descriptors() const { - return _input_descriptors; - } - inline const std::vector& get_output_descriptors() const { - return _output_descriptors; - } - -private: - void initialize_graph_through_command_list() const; - - const Config _config; - Logger _logger; - - const std::shared_ptr _initStructs; - std::shared_ptr _networkDesc; - - ze_graph_dditable_ext_curr_t& _graph_ddi_table_ext; - - const uint32_t _group_ordinal; - - ze_graph_handle_t _graph = nullptr; - - std::vector _input_descriptors; - std::vector _output_descriptors; - - std::shared_ptr _command_queues; - - mutable std::mutex _mutex; -}; - -} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/backend/include/zero_host_tensor.hpp b/src/plugins/intel_npu/src/backend/include/zero_host_tensor.hpp index 52000930e2a751..a214c8e2cb2b5d 100644 --- a/src/plugins/intel_npu/src/backend/include/zero_host_tensor.hpp +++ b/src/plugins/intel_npu/src/backend/include/zero_host_tensor.hpp @@ -5,8 +5,8 @@ #pragma once #include "intel_npu/config/config.hpp" +#include "intel_npu/utils/zero/zero_init.hpp" #include "openvino/runtime/itensor.hpp" -#include "zero_init.hpp" #include "zero_remote_tensor.hpp" namespace intel_npu { diff --git a/src/plugins/intel_npu/src/backend/include/zero_infer_request.hpp b/src/plugins/intel_npu/src/backend/include/zero_infer_request.hpp index 48aad52010a4c2..31248b582250da 100644 --- a/src/plugins/intel_npu/src/backend/include/zero_infer_request.hpp +++ b/src/plugins/intel_npu/src/backend/include/zero_infer_request.hpp @@ -11,19 +11,17 @@ #include "intel_npu/common/sync_infer_request.hpp" #include "intel_npu/utils/logger/logger.hpp" #include "intel_npu/utils/zero/zero_utils.hpp" -#include "zero_executor.hpp" +#include "intel_npu/utils/zero/zero_wrappers.hpp" #include "zero_pipeline.hpp" #include "zero_profiling.hpp" #include "zero_remote_tensor.hpp" -#include "zero_wrappers.hpp" namespace intel_npu { class ZeroInferRequest final : public SyncInferRequest { public: - explicit ZeroInferRequest(const std::shared_ptr& backendPtr, + explicit ZeroInferRequest(const std::shared_ptr& initStructs, const std::shared_ptr& compiledModel, - const std::shared_ptr& executor, const Config& config); ov::SoPtr get_tensor(const ov::Output& port) const override; @@ -85,8 +83,7 @@ class ZeroInferRequest final : public SyncInferRequest { std::vector>& get_input_tensors_data(size_t index) const; const std::shared_ptr _initStructs; - const std::shared_ptr _executorPtr; - const ZeroExecutor* _executor; + const std::shared_ptr _graph; const Config _config; Logger _logger; diff --git a/src/plugins/intel_npu/src/backend/include/zero_memory.hpp b/src/plugins/intel_npu/src/backend/include/zero_memory.hpp index 6ecbde0d546110..992f409b86a928 100644 --- a/src/plugins/intel_npu/src/backend/include/zero_memory.hpp +++ b/src/plugins/intel_npu/src/backend/include/zero_memory.hpp @@ -11,7 +11,7 @@ #include #include "intel_npu/utils/logger/logger.hpp" -#include "zero_init.hpp" +#include "intel_npu/utils/zero/zero_init.hpp" namespace { diff --git a/src/plugins/intel_npu/src/backend/include/zero_pipeline.hpp b/src/plugins/intel_npu/src/backend/include/zero_pipeline.hpp index 62c8481d28ac1a..92a473a9fc412c 100644 --- a/src/plugins/intel_npu/src/backend/include/zero_pipeline.hpp +++ b/src/plugins/intel_npu/src/backend/include/zero_pipeline.hpp @@ -4,11 +4,11 @@ #pragma once +#include "intel_npu/common/igraph.hpp" #include "intel_npu/utils/zero/zero_utils.hpp" -#include "zero_executor.hpp" +#include "intel_npu/utils/zero/zero_wrappers.hpp" #include "zero_memory.hpp" #include "zero_profiling.hpp" -#include "zero_wrappers.hpp" namespace intel_npu { @@ -21,13 +21,15 @@ struct TensorData { struct Pipeline { public: Pipeline(const Config& config, - const std::shared_ptr& executorPtr, + const std::shared_ptr& initStructs, + const std::shared_ptr& graph, zeroProfiling::ProfilingPool& profiling_pool, zeroProfiling::ProfilingQuery& profiling_query, std::shared_ptr npu_profiling, const std::vector>>& inputTensorsData, const std::vector>& outputTensorsData, - const size_t numberOfCommandLists); + size_t numberOfCommandLists, + uint32_t group_ordinal); Pipeline(const Pipeline&) = delete; Pipeline& operator=(const Pipeline&) = delete; @@ -42,8 +44,7 @@ struct Pipeline { protected: const Config _config; - const ZeroExecutor* _executor; - CommandQueue& _command_queue; + std::shared_ptr _command_queue; std::vector> _command_lists; std::vector> _fences; EventPool _event_pool; diff --git a/src/plugins/intel_npu/src/backend/include/zero_profiling.hpp b/src/plugins/intel_npu/src/backend/include/zero_profiling.hpp index 505a7f0185e135..17e263a7aaf620 100644 --- a/src/plugins/intel_npu/src/backend/include/zero_profiling.hpp +++ b/src/plugins/intel_npu/src/backend/include/zero_profiling.hpp @@ -12,8 +12,8 @@ #include "intel_npu/config/compiler.hpp" #include "intel_npu/utils/logger/logger.hpp" +#include "intel_npu/utils/zero/zero_types.hpp" #include "openvino/runtime/profiling_info.hpp" -#include "zero_types.hpp" namespace intel_npu { namespace zeroProfiling { diff --git a/src/plugins/intel_npu/src/backend/include/zero_remote_tensor.hpp b/src/plugins/intel_npu/src/backend/include/zero_remote_tensor.hpp index 0211bd5bd08962..5b08643704b651 100644 --- a/src/plugins/intel_npu/src/backend/include/zero_remote_tensor.hpp +++ b/src/plugins/intel_npu/src/backend/include/zero_remote_tensor.hpp @@ -9,8 +9,8 @@ #include #include "intel_npu/common/remote_tensor.hpp" +#include "intel_npu/utils/zero/zero_init.hpp" #include "openvino/runtime/intel_npu/remote_properties.hpp" -#include "zero_init.hpp" namespace intel_npu { diff --git a/src/plugins/intel_npu/src/backend/src/zero_backend.cpp b/src/plugins/intel_npu/src/backend/src/zero_backend.cpp index d74692400b0d90..55aaad102e8b8f 100644 --- a/src/plugins/intel_npu/src/backend/src/zero_backend.cpp +++ b/src/plugins/intel_npu/src/backend/src/zero_backend.cpp @@ -14,31 +14,31 @@ namespace intel_npu { ZeroEngineBackend::ZeroEngineBackend(const Config& config) : _logger("ZeroEngineBackend", Logger::global().level()) { _logger.debug("ZeroEngineBackend - initialize started"); - _instance = std::make_shared(); + _initStruct = std::make_shared(); - auto device = std::make_shared(_instance); + auto device = std::make_shared(_initStruct); _devices.emplace(std::make_pair(device->getName(), device)); _logger.debug("ZeroEngineBackend - initialize completed"); } uint32_t ZeroEngineBackend::getDriverVersion() const { - return _instance->getDriverVersion(); + return _initStruct->getDriverVersion(); } uint32_t ZeroEngineBackend::getGraphExtVersion() const { - return _instance->getGraphDdiTable().version(); + return _initStruct->getGraphDdiTable().version(); } bool ZeroEngineBackend::isBatchingSupported() const { - return _instance->isExtensionSupported(std::string(ZE_GRAPH_EXT_NAME_1_6), ZE_MAKE_VERSION(1, 6)); + return _initStruct->isExtensionSupported("ZE_extension_graph_1_6", ZE_MAKE_VERSION(1, 6)); } bool ZeroEngineBackend::isCommandQueueExtSupported() const { - return _instance->isExtensionSupported(std::string(ZE_COMMAND_QUEUE_NPU_EXT_NAME), ZE_MAKE_VERSION(1, 0)); + return _initStruct->isExtensionSupported(std::string(ZE_COMMAND_QUEUE_NPU_EXT_NAME), ZE_MAKE_VERSION(1, 0)); } bool ZeroEngineBackend::isLUIDExtSupported() const { - return _instance->isExtensionSupported(std::string(ZE_DEVICE_LUID_EXT_NAME), ZE_MAKE_VERSION(1, 0)); + return _initStruct->isExtensionSupported(std::string(ZE_DEVICE_LUID_EXT_NAME), ZE_MAKE_VERSION(1, 0)); } ZeroEngineBackend::~ZeroEngineBackend() = default; @@ -69,19 +69,11 @@ const std::vector ZeroEngineBackend::getDeviceNames() const { } void* ZeroEngineBackend::getContext() const { - return _instance->getContext(); + return _initStruct->getContext(); } -void* ZeroEngineBackend::getDriverHandle() const { - return _instance->getDriver(); -} - -void* ZeroEngineBackend::getDeviceHandle() const { - return _instance->getDevice(); -} - -ze_graph_dditable_ext_curr_t& ZeroEngineBackend::getGraphDdiTable() const { - return _instance->getGraphDdiTable(); +const std::shared_ptr& ZeroEngineBackend::getInitStruct() const { + return _initStruct; } void ZeroEngineBackend::updateInfo(const Config& config) { diff --git a/src/plugins/intel_npu/src/backend/src/zero_device.cpp b/src/plugins/intel_npu/src/backend/src/zero_device.cpp index ac60e4741947bd..6e16dde3b120bf 100644 --- a/src/plugins/intel_npu/src/backend/src/zero_device.cpp +++ b/src/plugins/intel_npu/src/backend/src/zero_device.cpp @@ -7,7 +7,6 @@ #include "intel_npu/common/itt.hpp" #include "intel_npu/utils/zero/zero_api.hpp" #include "intel_npu/utils/zero/zero_utils.hpp" -#include "zero_executor.hpp" #include "zero_host_tensor.hpp" #include "zero_infer_request.hpp" #include "zero_remote_tensor.hpp" @@ -64,38 +63,6 @@ ZeroDevice::ZeroDevice(const std::shared_ptr& initStructs device_gops[ov::element::i8] = gops; device_gops[ov::element::f16] = 0.5f * gops; } - - std::vector command_group_properties; - uint32_t command_queue_group_count = 0; - // Discover all command queue groups - THROW_ON_FAIL_FOR_LEVELZERO( - "zeDeviceGetCommandQueueGroupProperties", - zeDeviceGetCommandQueueGroupProperties(_initStructs->getDevice(), &command_queue_group_count, nullptr)); - - log.debug("ZeroDevice::ZeroDevice - resize command_queue_group_count"); - command_group_properties.resize(command_queue_group_count); - - for (auto& prop : command_group_properties) { - prop.stype = ZE_STRUCTURE_TYPE_COMMAND_QUEUE_GROUP_PROPERTIES; - prop.pNext = nullptr; - } - - THROW_ON_FAIL_FOR_LEVELZERO("zeDeviceGetCommandQueueGroupProperties", - zeDeviceGetCommandQueueGroupProperties(_initStructs->getDevice(), - &command_queue_group_count, - command_group_properties.data())); - - // Find the corresponding command queue group. - log.debug("ZeroDevice::ZeroDevice - findGroupOrdinal"); - _group_ordinal = zeroUtils::findGroupOrdinal(command_group_properties, device_properties); - log.debug("ZeroDevice::ZeroDevice - init completed"); -} - -std::shared_ptr ZeroDevice::createExecutor( - const std::shared_ptr& networkDescription, - const Config& config) { - OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "Device::createExecutor"); - return std::make_shared(_initStructs, networkDescription, config, _group_ordinal); } std::string ZeroDevice::getName() const { @@ -162,12 +129,30 @@ uint64_t ZeroDevice::getAllocMemSize() const { } uint64_t ZeroDevice::getTotalMemSize() const { +#define LEGACY_MAX_MEM_ALLOC_SIZE_BYTES (2147483648) // 2GB in base-2 + ze_graph_memory_query_t query{}; ze_result_t result = _graph_ddi_table_ext.pfnQueryContextMemory(_initStructs->getContext(), ZE_GRAPH_QUERY_MEMORY_DDR, &query); THROW_ON_FAIL_FOR_LEVELZERO_EXT("pfnQueryContextMemory", result, _graph_ddi_table_ext); - return query.total; + // For drivers with graph_extension < 1.9 we report fixed 2GB max allocation size (old drivers don't support more) + // For drivers with graph_extension > 1.9 we report the value they return + if (_initStructs->isExtensionSupported(std::string(ZE_GRAPH_EXT_NAME), ZE_MAKE_VERSION(1, 9))) { + // we are safe here, can return the value directly from driver + return query.total; + } +#if defined(_WIN32) || defined(__CYGWIN__) + // Special case for windows drivers with graph_extension v 1.8 + if (_initStructs->isExtensionSupported(std::string("ZE_extension_graph_1_8"), ZE_MAKE_VERSION(1, 8))) { + // query here returns total system memory in KB, which we need to + // divide by 2 (OS limitation) and convert to bytes + return (query.total << 9); + } +#endif + + // Default for older drivers: return 2GB + return LEGACY_MAX_MEM_ALLOC_SIZE_BYTES; } ov::device::PCIInfo ZeroDevice::getPciInfo() const { @@ -187,9 +172,8 @@ ov::device::Type ZeroDevice::getDeviceType() const { std::shared_ptr ZeroDevice::createInferRequest( const std::shared_ptr& compiledModel, - const std::shared_ptr& executor, const Config& config) { - return std::make_shared(_initStructs, compiledModel, executor, config); + return std::make_shared(_initStructs, compiledModel, config); } ov::SoPtr ZeroDevice::createRemoteTensor(std::shared_ptr context, diff --git a/src/plugins/intel_npu/src/backend/src/zero_executor.cpp b/src/plugins/intel_npu/src/backend/src/zero_executor.cpp deleted file mode 100644 index 32da2b2e0e4189..00000000000000 --- a/src/plugins/intel_npu/src/backend/src/zero_executor.cpp +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "zero_executor.hpp" - -#include - -#include -#include -#include -#include - -#include "intel_npu/common/itt.hpp" -#include "intel_npu/config/common.hpp" -#include "intel_npu/prefix.hpp" -#include "intel_npu/utils/zero/zero_utils.hpp" -#include "openvino/runtime/properties.hpp" -#include "ze_command_queue_npu_ext.h" -#include "zero_device.hpp" - -using namespace intel_npu; - -ZeroExecutor::ZeroExecutor(const std::shared_ptr& initStructs, - const std::shared_ptr& networkDescription, - const Config& config, - const uint32_t& group_ordinal) - : _config(config), - _logger("Graph", _config.get()), - _initStructs(initStructs), - _networkDesc(networkDescription), - _graph_ddi_table_ext(_initStructs->getGraphDdiTable()), - _group_ordinal(group_ordinal), - _command_queues{std::make_shared(_initStructs->getDevice(), - _initStructs->getContext(), - zeroUtils::toZeQueuePriority(_config.get()), - _initStructs->getCommandQueueDdiTable(), - _config, - group_ordinal)} { - _logger.debug("ZeroExecutor::ZeroExecutor - create graph"); - OV_ITT_TASK_CHAIN(ZERO_EXECUTOR_GRAPH, itt::domains::LevelZeroBackend, "Executor::ZeroExecutor", "graphCreate"); - - // _graph is a nullptr for CIP path, a new handle will be obtained from the driver based on the given - // compiledNetwork _graph gets (reuses) graphHandle from the compiler for CID path - if (_networkDesc->metadata.graphHandle == nullptr) { - _logger.debug("create graph handle on executor"); - ze_graph_desc_t desc{ZE_STRUCTURE_TYPE_GRAPH_DESC_PROPERTIES, - nullptr, - ZE_GRAPH_FORMAT_NATIVE, - _networkDesc->compiledNetwork.size(), - _networkDesc->compiledNetwork.data(), - nullptr}; - ze_result_t result = - _graph_ddi_table_ext.pfnCreate(_initStructs->getContext(), _initStructs->getDevice(), &desc, &_graph); - THROW_ON_FAIL_FOR_LEVELZERO_EXT("pfnCreate", result, _graph_ddi_table_ext); - - } else { - _logger.debug("reuse graph handle created from compiler"); - _graph = static_cast(_networkDesc->metadata.graphHandle); - } - - OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "pfnGetProperties"); - _logger.debug("performing pfnGetProperties"); - ze_graph_properties_t props{}; - props.stype = ZE_STRUCTURE_TYPE_GRAPH_PROPERTIES; - - ze_result_t result = _graph_ddi_table_ext.pfnGetProperties(_graph, &props); - THROW_ON_FAIL_FOR_LEVELZERO_EXT("pfnGetProperties", result, _graph_ddi_table_ext); - - auto targetDriverExtVersion = _graph_ddi_table_ext.version(); - if (targetDriverExtVersion <= ZE_GRAPH_EXT_VERSION_1_1) { - OPENVINO_THROW("Incompatibility between the NPU plugin and driver! The driver version is too old, please " - "update the driver version"); - } - - OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "pfnGetArgumentProperties3"); - _logger.debug("performing pfnGetArgumentProperties3"); - for (uint32_t index = 0; index < props.numGraphArgs; ++index) { - ze_graph_argument_properties_3_t arg3{}; - arg3.stype = ZE_STRUCTURE_TYPE_GRAPH_ARGUMENT_PROPERTIES; - ze_result_t result = _graph_ddi_table_ext.pfnGetArgumentProperties3(_graph, index, &arg3); - THROW_ON_FAIL_FOR_LEVELZERO_EXT("pfnGetArgumentProperties3", result, _graph_ddi_table_ext); - - if (arg3.type == ZE_GRAPH_ARGUMENT_TYPE_INPUT) { - _input_descriptors.push_back(ArgumentDescriptor{arg3, index}); - } else { - _output_descriptors.push_back(ArgumentDescriptor{arg3, index}); - } - } - - if (_graph_ddi_table_ext.version() < ZE_GRAPH_EXT_VERSION_1_8) { - initialize_graph_through_command_list(); - } else { - ze_graph_properties_2_t properties = {}; - properties.stype = ZE_STRUCTURE_TYPE_GRAPH_PROPERTIES; - _graph_ddi_table_ext.pfnGetProperties2(_graph, &properties); - - if (properties.initStageRequired & ZE_GRAPH_STAGE_INITIALIZE) { - OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "pfnGraphInitialize"); - _graph_ddi_table_ext.pfnGraphInitialize(_graph); - } - - if (properties.initStageRequired & ZE_GRAPH_STAGE_COMMAND_LIST_INITIALIZE) { - initialize_graph_through_command_list(); - } - } - - if (config.has()) { - setWorkloadType(config.get()); - } -} - -void ZeroExecutor::initialize_graph_through_command_list() const { - OV_ITT_TASK_CHAIN(ZERO_EXECUTOR_GRAPH, - itt::domains::LevelZeroBackend, - "Executor::ZeroExecutor", - "initialize_graph_through_command_list"); - - _logger.debug("ZeroExecutor::ZeroExecutor init start - create graph_command_list"); - OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "Executor::ZeroExecutor"); - CommandList graph_command_list(_initStructs->getDevice(), - _initStructs->getContext(), - _graph_ddi_table_ext, - _config, - _group_ordinal); - _logger.debug("ZeroExecutor::ZeroExecutor - create graph_command_queue"); - CommandQueue graph_command_queue(_initStructs->getDevice(), - _initStructs->getContext(), - ZE_COMMAND_QUEUE_PRIORITY_NORMAL, - _initStructs->getCommandQueueDdiTable(), - _config, - _group_ordinal); - _logger.debug("ZeroExecutor::ZeroExecutor - create fence"); - Fence fence(graph_command_queue, _config); - - OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "appendGraphInitialize"); - _logger.debug("ZeroExecutor::ZeroExecutor - performing appendGraphInitialize"); - graph_command_list.appendGraphInitialize(_graph); - _logger.debug("ZeroExecutor::ZeroExecutor - closing graph command list"); - graph_command_list.close(); - - OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "queue_execute"); - _logger.debug("ZeroExecutor::ZeroExecutor - performing executeCommandList"); - graph_command_queue.executeCommandList(graph_command_list, fence); - _logger.debug("ZeroExecutor::ZeroExecutor - performing hostSynchronize"); - fence.hostSynchronize(); - _logger.debug("ZeroExecutor::ZeroExecutor - hostSynchronize completed"); -} - -void ZeroExecutor::setWorkloadType(const ov::WorkloadType workloadType) const { - ze_command_queue_workload_type_t zeWorkloadType; - switch (workloadType) { - case ov::WorkloadType::DEFAULT: - zeWorkloadType = ze_command_queue_workload_type_t::ZE_WORKLOAD_TYPE_DEFAULT; - break; - case ov::WorkloadType::EFFICIENT: - zeWorkloadType = ze_command_queue_workload_type_t::ZE_WORKLOAD_TYPE_BACKGROUND; - break; - default: - OPENVINO_THROW("Unknown value for WorkloadType!"); - } - - _command_queues->setWorkloadType(zeWorkloadType); -} - -void ZeroExecutor::setArgumentValue(uint32_t argi_, const void* argv_) const { - ze_result_t result = _graph_ddi_table_ext.pfnSetArgumentValue(_graph, argi_, argv_); - if (ZE_RESULT_SUCCESS != result) { - THROW_ON_FAIL_FOR_LEVELZERO_EXT("zeGraphSetArgumentValue", result, _graph_ddi_table_ext); - } -} - -void ZeroExecutor::mutexLock() const { - _mutex.lock(); -} - -void ZeroExecutor::mutexUnlock() const { - _mutex.unlock(); -} - -ZeroExecutor::~ZeroExecutor() { - _logger.debug("~ZeroExecutor() - pfnDestroy _graph "); - auto result = _graph_ddi_table_ext.pfnDestroy(_graph); - if (ZE_RESULT_SUCCESS != result) { - _logger.error("_graph_ddi_table_ext.pfnDestroy failed %#X", uint64_t(result)); - } -} diff --git a/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp b/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp index dd2629372dc7d8..1c5ceecfac1961 100644 --- a/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp +++ b/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp @@ -31,8 +31,7 @@ constexpr bool OUTPUT = false; * @param ioDescriptor The OpenVINO API specific I/O descriptor which shall be compared. * @param zeDescriptor The Level Zero specific structure used for comparison. */ -void check_level_zero_attributes_match(const IODescriptor& ioDescriptor, - const ZeroExecutor::ArgumentDescriptor& zeDescriptor) { +void check_level_zero_attributes_match(const IODescriptor& ioDescriptor, const ArgumentDescriptor& zeDescriptor) { std::string zeDescriptorName = zeDescriptor.info.name; if (isStateInputName(zeDescriptorName)) { @@ -158,38 +157,35 @@ std::optional ZeroInferRequest::get_batch_size(const NetworkMetadata& me //------------------------------------------------------------------------------ ZeroInferRequest::ZeroInferRequest(const std::shared_ptr& initStructs, const std::shared_ptr& compiledModel, - const std::shared_ptr& executor, const Config& config) : SyncInferRequest(compiledModel, config), _initStructs(initStructs), - _executorPtr(executor), - _executor(static_cast(_executorPtr.get())), + _graph(compiledModel->get_graph()), _config(config), _logger("ZeroInferRequest", config.get()), _levelZeroInputTensors(_metadata.inputs.size(), std::vector>(1, nullptr)), _levelZeroOutputTensors(_metadata.outputs.size(), nullptr), _inputTensorsData(_metadata.inputs.size(), std::vector>(1, std::nullopt)), _outputTensorsData(_metadata.outputs.size(), std::nullopt), - _profilingPool(_executor->graph(), zeroProfiling::POOL_SIZE, _executor->getInitStructs()->getProfilingDdiTable()), - _profilingQuery(0, - _executor->getInitStructs()->getDevice(), - _executor->getInitStructs()->getProfilingDdiTable()) { + _profilingPool(static_cast(_graph->get_handle()), + zeroProfiling::POOL_SIZE, + _initStructs->getProfilingDdiTable()), + _profilingQuery(0, _initStructs->getDevice(), _initStructs->getProfilingDdiTable()) { _logger.debug("ZeroInferRequest::ZeroInferRequest - SyncInferRequest"); - const std::vector& executorInputDescriptors = _executor->get_input_descriptors(); - const std::vector& executorOutputDescriptors = - _executor->get_output_descriptors(); + const std::vector& executorInputDescriptors = _graph->get_input_descriptors(); + const std::vector& executorOutputDescriptors = _graph->get_output_descriptors(); auto proftype = config.get(); if (proftype == ov::intel_npu::ProfilingType::INFER) { _logger.debug("ZeroInferRequest::ZeroInferRequest - profiling type == ov::intel_npu::ProfilingType::INFER"); - _npuProfiling = std::make_shared(_executor->getInitStructs()->getContext(), - _executor->getInitStructs()->getDevice(), + _npuProfiling = std::make_shared(_initStructs->getContext(), + _initStructs->getDevice(), _config.get()); } _properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; THROW_ON_FAIL_FOR_LEVELZERO("zeDeviceGetProperties", - zeDeviceGetProperties(_executor->getInitStructs()->getDevice(), &_properties)); + zeDeviceGetProperties(_initStructs->getDevice(), &_properties)); _outputAllocator = std::make_shared(_initStructs); _inputAllocator = @@ -278,17 +274,24 @@ void ZeroInferRequest::create_pipeline() { _levelZeroOutputTensors.at(outputIndex)->get_byte_size()}); } + // Find the corresponding command queue group. + _logger.debug("ZeroDevice::ZeroDevice - findGroupOrdinal"); + auto groupOrdinal = zeroUtils::findGroupOrdinal(_initStructs->getDevice(), _properties); + _logger.debug("ZeroDevice::ZeroDevice - init completed"); + _logger.debug("ZeroInferRequest::create_pipeline - constructing pipeline"); - // Construct pipeline + // Construct pipeline _pipeline = std::make_unique(_config, - _executorPtr, + _initStructs, + _graph, _profilingPool, _profilingQuery, _npuProfiling, _inputTensorsData, _outputTensorsData, - _numberOfCommandLists); + _numberOfCommandLists, + groupOrdinal); _logger.debug("ZeroInferRequest::create_pipeline - SyncInferRequest completed"); } @@ -338,8 +341,8 @@ void ZeroInferRequest::set_tensor_data(const std::shared_ptr tensor OV_ITT_TASK_NEXT(ZERO_SET_TENSOR, "updateCommandList"); _pipeline->updateCommandList(*tensorsData, - isInput ? _executor->get_input_descriptors().at(index).idx - : _executor->get_output_descriptors().at(index).idx); + isInput ? _graph->get_input_descriptors().at(index).idx + : _graph->get_output_descriptors().at(index).idx); } } } @@ -370,9 +373,9 @@ void ZeroInferRequest::set_remote_tensor_data(const std::shared_ptrupdateCommandList(*tensorsData, - isInput ? _executor->get_input_descriptors().at(index).idx - : _executor->get_output_descriptors().at(index).idx); + _pipeline->updateCommandList( + *tensorsData, + isInput ? _graph->get_input_descriptors().at(index).idx : _graph->get_output_descriptors().at(index).idx); } } @@ -390,13 +393,17 @@ void ZeroInferRequest::set_tensor(const ov::Output& port, const if (foundPort.is_input()) { if (get_user_input(foundPort.idx)._ptr == tensor._ptr) { // Got set_tensor with the same object - do nothing + _logger.debug("ZeroInferRequest::set_tensor - got the same tensor, do nothing"); return; } if (is_batched_input(foundPort.idx)) { // resize vector size to 1 if set_tensor is called after set_tensors get_input_tensors_data(foundPort.idx).resize(1); + get_input_tensors_data(foundPort.idx).shrink_to_fit(); get_level_zero_inputs(foundPort.idx).resize(1); + get_level_zero_inputs(foundPort.idx).shrink_to_fit(); get_user_inputs(foundPort.idx).resize(1); + get_user_inputs(foundPort.idx).shrink_to_fit(); } get_user_input(foundPort.idx) = tensor; @@ -485,7 +492,7 @@ void ZeroInferRequest::set_tensors(const ov::Output& port, if (_pipelineIsCreated) { OV_ITT_TASK_NEXT(SET_TENSORS, "updateCommandList"); _pipeline->updateCommandList(*get_input_tensor_data(foundPort.idx, i), - _executor->get_input_descriptors().at(foundPort.idx).idx, + _graph->get_input_descriptors().at(foundPort.idx).idx, i); } } @@ -537,14 +544,16 @@ void ZeroInferRequest::infer_async() { _logger.debug("InferRequest::infer_async started"); OV_ITT_TASK_CHAIN(ZERO_INFER, itt::domains::LevelZeroBackend, "infer_async", "start"); - _executor->mutexLock(); - if (!_pipelineIsCreated) { - OV_ITT_TASK_NEXT(ZERO_INFER, "create_pipeline"); - create_pipeline(); + { + std::lock_guard lock(_graph->get_mutex()); - _pipelineIsCreated = true; + if (!_pipelineIsCreated) { + OV_ITT_TASK_NEXT(ZERO_INFER, "create_pipeline"); + create_pipeline(); + + _pipelineIsCreated = true; + } } - _executor->mutexUnlock(); size_t inputIndex = 0; for (const auto& userTensor : _userInputTensors) { @@ -740,12 +749,9 @@ std::vector ZeroInferRequest::get_profiling_info() const { if (compilerType == ov::intel_npu::CompilerType::MLIR) { // For plugin compiler retreive raw profiling data from backend and delegate // processing to the compiler - const auto& networkDesc = compiledModel.get_network_description(); - const auto& compiler = compiledModel.get_compiler(); - const auto& blob = networkDesc->compiledNetwork; auto profData = get_raw_profiling_data(); _logger.debug("InferRequest::get_profiling_info complete with compiler->process_profiling_output()."); - return compiler->process_profiling_output(profData, blob, compilerConfig); + return _graph->process_profiling_output(profData, compilerConfig); } else { auto proftype = _config.get(); if (proftype == ov::intel_npu::ProfilingType::INFER) { diff --git a/src/plugins/intel_npu/src/backend/src/zero_pipeline.cpp b/src/plugins/intel_npu/src/backend/src/zero_pipeline.cpp index 009eee6541e8ef..34eb71eaf112f7 100644 --- a/src/plugins/intel_npu/src/backend/src/zero_pipeline.cpp +++ b/src/plugins/intel_npu/src/backend/src/zero_pipeline.cpp @@ -11,25 +11,25 @@ #include "intel_npu/prefix.hpp" #include "intel_npu/utils/logger/logger.hpp" #include "intel_npu/utils/zero/zero_api.hpp" -#include "zero_types.hpp" +#include "intel_npu/utils/zero/zero_types.hpp" namespace intel_npu { Pipeline::Pipeline(const Config& config, - const std::shared_ptr& executorPtr, + const std::shared_ptr& initStructs, + const std::shared_ptr& graph, zeroProfiling::ProfilingPool& profiling_pool, zeroProfiling::ProfilingQuery& profiling_query, std::shared_ptr npu_profiling, const std::vector>>& inputTensorsData, const std::vector>& outputTensorsData, - const size_t numberOfCommandLists) + size_t numberOfCommandLists, + uint32_t group_ordinal) : _config(config), - _executor(static_cast(executorPtr.get())), - _command_queue(*_executor->getCommandQueue()), - _event_pool{_executor->getInitStructs()->getDevice(), - _executor->getInitStructs()->getContext(), - numberOfCommandLists ? static_cast(numberOfCommandLists) : 1, - _config}, + _command_queue(graph->get_command_queue()), + _event_pool{initStructs->getDevice(), + initStructs->getContext(), + numberOfCommandLists ? static_cast(numberOfCommandLists) : 1}, _npu_profiling(std::move(npu_profiling)), _logger("Pipeline", _config.get()) { OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "Zero_infer_request::Pipeline::Pipeline"); @@ -45,38 +45,37 @@ Pipeline::Pipeline(const Config& config, _logger.debug("Pipeline - emplace_back _event_pool and _command_queue"); for (size_t i = 0; i < numberOfCommandLists; i++) { _command_lists.emplace_back( - std::make_unique(_executor->getInitStructs()->getDevice(), - _executor->getInitStructs()->getContext(), - _executor->getInitStructs()->getGraphDdiTable(), - _config, - _executor->get_group_ordinal(), - _executor->getInitStructs()->getMutableCommandListVersion() ? true : false)); - _events.emplace_back(std::make_unique(_event_pool.handle(), static_cast(i), _config)); - _fences.emplace_back(std::make_unique(_command_queue, _config)); + std::make_unique(initStructs->getDevice(), + initStructs->getContext(), + initStructs->getGraphDdiTable(), + group_ordinal, + initStructs->getMutableCommandListVersion() ? true : false)); + _events.emplace_back(std::make_unique(_event_pool.handle(), static_cast(i))); + _fences.emplace_back(std::make_unique(*_command_queue)); } for (size_t i = 0; i < numberOfCommandLists; i++) { size_t ioIndex = 0; - for (const auto& desc : _executor->get_input_descriptors()) { + for (const auto& desc : graph->get_input_descriptors()) { if (inputTensorsData.at(ioIndex).size() > 1) { - _executor->setArgumentValue(desc.idx, inputTensorsData.at(ioIndex).at(i)->mem); + graph->set_argument_value(desc.idx, inputTensorsData.at(ioIndex).at(i)->mem); ++ioIndex; continue; } - _executor->setArgumentValue(desc.idx, - static_cast(inputTensorsData.at(ioIndex).at(0)->mem) + - (i * inputTensorsData.at(ioIndex).at(0)->size) / numberOfCommandLists); + graph->set_argument_value(desc.idx, + static_cast(inputTensorsData.at(ioIndex).at(0)->mem) + + (i * inputTensorsData.at(ioIndex).at(0)->size) / numberOfCommandLists); ++ioIndex; } ioIndex = 0; - for (const auto& desc : _executor->get_output_descriptors()) { - _executor->setArgumentValue(desc.idx, - static_cast(outputTensorsData.at(ioIndex)->mem) + - (i * outputTensorsData.at(ioIndex)->size) / numberOfCommandLists); + for (const auto& desc : graph->get_output_descriptors()) { + graph->set_argument_value(desc.idx, + static_cast(outputTensorsData.at(ioIndex)->mem) + + (i * outputTensorsData.at(ioIndex)->size) / numberOfCommandLists); ++ioIndex; } @@ -86,7 +85,8 @@ Pipeline::Pipeline(const Config& config, _command_lists.at(i)->appendNpuTimestamp(reinterpret_cast(_npu_profiling->npu_ts_infer_start)); } - _command_lists.at(i)->appendGraphExecute(_executor->graph(), profiling_query.getHandle()); + _command_lists.at(i)->appendGraphExecute(static_cast(graph->get_handle()), + profiling_query.getHandle()); /// append timestamp command if feature was activated if (_npu_profiling != nullptr) { @@ -108,11 +108,11 @@ void Pipeline::push() { _logger.debug("Pipeline - push() started"); for (size_t i = 0; i < _command_lists.size(); ++i) { - OV_ITT_TASK_CHAIN(ZERO_EXECUTOR_IP_PUSH, itt::domains::LevelZeroBackend, "Pipeline", "push"); + OV_ITT_TASK_CHAIN(ZERO_PIPELINE_IP_PUSH, itt::domains::LevelZeroBackend, "Pipeline", "push"); if (sync_output_with_fences_) { - _command_queue.executeCommandList(*_command_lists.at(i), *_fences.at(i)); + _command_queue->executeCommandList(*_command_lists.at(i), *_fences.at(i)); } else { - _command_queue.executeCommandList(*_command_lists.at(i)); + _command_queue->executeCommandList(*_command_lists.at(i)); } } @@ -121,7 +121,7 @@ void Pipeline::push() { void Pipeline::pull() { _logger.debug("Pipeline - pull() started"); - OV_ITT_TASK_CHAIN(ZERO_EXECUTOR_IP_PULL, itt::domains::LevelZeroBackend, "Pipeline", "pull"); + OV_ITT_TASK_CHAIN(ZERO_PIPELINE_IP_PULL, itt::domains::LevelZeroBackend, "Pipeline", "pull"); for (size_t i = 0; i < _command_lists.size(); ++i) { if (sync_output_with_fences_) { diff --git a/src/plugins/intel_npu/src/common/CMakeLists.txt b/src/plugins/intel_npu/src/common/CMakeLists.txt index 2d1f5d9cbb39ea..1aa93cce1bc291 100644 --- a/src/plugins/intel_npu/src/common/CMakeLists.txt +++ b/src/plugins/intel_npu/src/common/CMakeLists.txt @@ -20,7 +20,7 @@ target_link_libraries(${TARGET_NAME} PUBLIC openvino::npu_al openvino::npu_logger_utils - openvino::runtime::dev + openvino::npu_zero_utils ) set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) diff --git a/src/plugins/intel_npu/src/common/include/intel_npu/common/icompiled_model.hpp b/src/plugins/intel_npu/src/common/include/intel_npu/common/icompiled_model.hpp index eb6a3de57e41fc..19023a1fca883f 100644 --- a/src/plugins/intel_npu/src/common/include/intel_npu/common/icompiled_model.hpp +++ b/src/plugins/intel_npu/src/common/include/intel_npu/common/icompiled_model.hpp @@ -7,8 +7,8 @@ #include #include +#include "intel_npu/common/igraph.hpp" #include "intel_npu/config/common.hpp" -#include "intel_npu/icompiler.hpp" #include "openvino/runtime/icompiled_model.hpp" namespace intel_npu { @@ -17,17 +17,10 @@ class ICompiledModel : public ov::ICompiledModel { public: using ov::ICompiledModel::ICompiledModel; - virtual const std::shared_ptr& get_network_description() const = 0; + virtual const std::shared_ptr& get_graph() const = 0; virtual const Config& get_config() const = 0; - // Compiler is used for post-processing profiling data when using PERF_COUNT property - virtual const ov::SoPtr& get_compiler() const = 0; - - const NetworkMetadata& get_network_metadata() const { - return get_network_description()->metadata; - } - protected: std::shared_ptr shared_from_this() const { return std::dynamic_pointer_cast(ov::ICompiledModel::shared_from_this()); diff --git a/src/plugins/intel_npu/src/common/include/intel_npu/common/igraph.hpp b/src/plugins/intel_npu/src/common/include/intel_npu/common/igraph.hpp new file mode 100644 index 00000000000000..51c4a4cf26eafd --- /dev/null +++ b/src/plugins/intel_npu/src/common/include/intel_npu/common/igraph.hpp @@ -0,0 +1,103 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +#include "intel_npu/network_metadata.hpp" +#include "intel_npu/utils/zero/zero_utils.hpp" +#include "intel_npu/utils/zero/zero_wrappers.hpp" +#include "openvino/runtime/profiling_info.hpp" + +namespace intel_npu { + +class IGraph : public std::enable_shared_from_this { +public: + IGraph(ze_graph_handle_t handle, NetworkMetadata metadata, std::optional> blob) + : _handle(handle), + _metadata(std::move(metadata)) { + if (blob.has_value()) { + _blob = std::move(*blob); + } + } + + virtual void export_blob(std::ostream& stream) const = 0; + + virtual std::vector process_profiling_output(const std::vector& profData, + const Config& config) const = 0; + + virtual void set_argument_value(uint32_t argi, const void* argv) const = 0; + + virtual void initialize(const Config& config) = 0; + + virtual ~IGraph() = default; + + const NetworkMetadata& get_metadata() const { + return _metadata; + } + + ze_graph_handle_t get_handle() const { + return _handle; + } + + void update_network_name(std::string_view name) { + _metadata.name = name; + } + + inline const std::vector& get_input_descriptors() const { + return _input_descriptors; + } + + inline const std::vector& get_output_descriptors() const { + return _output_descriptors; + } + + inline const std::shared_ptr& get_command_queue() const { + return _command_queue; + } + + void set_workload_type(const ov::WorkloadType workloadType) const { + if (_command_queue == nullptr) { + return; + } + + ze_command_queue_workload_type_t zeWorkloadType; + switch (workloadType) { + case ov::WorkloadType::DEFAULT: + zeWorkloadType = ze_command_queue_workload_type_t::ZE_WORKLOAD_TYPE_DEFAULT; + break; + case ov::WorkloadType::EFFICIENT: + zeWorkloadType = ze_command_queue_workload_type_t::ZE_WORKLOAD_TYPE_BACKGROUND; + break; + default: + OPENVINO_THROW("Unknown value for WorkloadType!"); + } + + _command_queue->setWorkloadType(zeWorkloadType); + } + + std::mutex& get_mutex() { + return _mutex; + } + +protected: + ze_graph_handle_t _handle = nullptr; + NetworkMetadata _metadata; + + std::vector _input_descriptors; + std::vector _output_descriptors; + + std::shared_ptr _command_queue; + + // Used to protect zero pipeline creation in the graph. The pipeline should be created only once per graph when the + // first inference starts running + std::mutex _mutex; + + std::vector _blob; +}; + +} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/common/include/intel_npu/common/npu.hpp b/src/plugins/intel_npu/src/common/include/intel_npu/common/npu.hpp index 8c1eb57fe34fc3..b34f2deee6c61e 100644 --- a/src/plugins/intel_npu/src/common/include/intel_npu/common/npu.hpp +++ b/src/plugins/intel_npu/src/common/include/intel_npu/common/npu.hpp @@ -7,9 +7,9 @@ #include #include "intel_npu/common/icompiled_model.hpp" +#include "intel_npu/common/igraph.hpp" #include "intel_npu/common/sync_infer_request.hpp" #include "intel_npu/config/config.hpp" -#include "intel_npu/icompiler.hpp" #include "openvino/runtime/intel_npu/remote_properties.hpp" #include "openvino/runtime/iremote_context.hpp" #include "openvino/runtime/properties.hpp" @@ -54,11 +54,14 @@ class IEngineBackend : public std::enable_shared_from_this { //------------------------------------------------------------------------------ -class IExecutor { +class ICompilerAdapter { public: - virtual ~IExecutor() = default; + virtual std::shared_ptr compile(const std::shared_ptr& model, + const Config& config) const = 0; + virtual std::shared_ptr parse(std::vector network, const Config& config) const = 0; + virtual ov::SupportedOpsMap query(const std::shared_ptr& model, const Config& config) const = 0; - virtual void setWorkloadType(const ov::WorkloadType workloadType) const = 0; + virtual ~ICompilerAdapter() = default; }; //------------------------------------------------------------------------------ @@ -67,10 +70,6 @@ class IDevice : public std::enable_shared_from_this { public: using Uuid = ov::device::UUID; - virtual std::shared_ptr createExecutor( - const std::shared_ptr& networkDescription, - const Config& config) = 0; - virtual std::string getName() const = 0; virtual std::string getFullDeviceName() const = 0; virtual Uuid getUuid() const; @@ -85,7 +84,6 @@ class IDevice : public std::enable_shared_from_this { virtual std::shared_ptr createInferRequest( const std::shared_ptr& compiledModel, - const std::shared_ptr& executor, const Config& config) = 0; virtual void updateInfo(const Config& config) = 0; diff --git a/src/plugins/intel_npu/src/common/include/intel_npu/common/sync_infer_request.hpp b/src/plugins/intel_npu/src/common/include/intel_npu/common/sync_infer_request.hpp index 99f9ce7cb0eb28..788ce87136a04d 100644 --- a/src/plugins/intel_npu/src/common/include/intel_npu/common/sync_infer_request.hpp +++ b/src/plugins/intel_npu/src/common/include/intel_npu/common/sync_infer_request.hpp @@ -5,8 +5,9 @@ #pragma once #include "intel_npu/common/icompiled_model.hpp" +#include "intel_npu/common/igraph.hpp" #include "intel_npu/common/variable_state.hpp" -#include "intel_npu/icompiler.hpp" +#include "intel_npu/network_metadata.hpp" #include "openvino/runtime/iinfer_request.hpp" #include "openvino/runtime/iplugin.hpp" diff --git a/src/plugins/intel_npu/src/common/src/sync_infer_request.cpp b/src/plugins/intel_npu/src/common/src/sync_infer_request.cpp index 0ae0832fe29d72..0eeefccf43906d 100644 --- a/src/plugins/intel_npu/src/common/src/sync_infer_request.cpp +++ b/src/plugins/intel_npu/src/common/src/sync_infer_request.cpp @@ -21,7 +21,7 @@ namespace intel_npu { SyncInferRequest::SyncInferRequest(const std::shared_ptr& compiledModel, const Config& config) : _compiledModel(compiledModel), - _metadata(compiledModel->get_network_metadata()), + _metadata(compiledModel->get_graph()->get_metadata()), _logger("SyncInferRequest", config.get()), _userInputTensors(_metadata.inputs.size(), std::vector>(1, {nullptr})), _userOutputTensors(_metadata.outputs.size(), {nullptr}) { diff --git a/src/plugins/intel_npu/src/compiler/include/driver_compiler_adapter.hpp b/src/plugins/intel_npu/src/compiler/include/driver_compiler_adapter.hpp deleted file mode 100644 index addd9ca5308c65..00000000000000 --- a/src/plugins/intel_npu/src/compiler/include/driver_compiler_adapter.hpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include - -#include "intel_npu/common/npu.hpp" -#include "intel_npu/icompiler.hpp" -#include "intel_npu/utils/logger/logger.hpp" - -namespace intel_npu { -namespace driverCompilerAdapter { - -/** - * @brief Adapter for Compiler in driver - * @details Wrap compiler in driver calls and do preliminary actions (like opset conversion) - */ -class LevelZeroCompilerAdapter final : public ICompiler { -public: - LevelZeroCompilerAdapter(std::shared_ptr iEngineBackend); - - uint32_t getSupportedOpsetVersion() const override final; - - NetworkDescription compile(const std::shared_ptr& model, - const Config& config) const override final; - - ov::SupportedOpsMap query(const std::shared_ptr& model, const Config& config) const override final; - - NetworkMetadata parse(const std::vector& network, const Config& config) const override final; - - std::vector process_profiling_output(const std::vector& profData, - const std::vector& network, - const Config& config) const override final; - - void release(std::shared_ptr networkDescription) override; - - CompiledNetwork getCompiledNetwork(const NetworkDescription& networkDescription) override; - -private: - /** - * @brief Separate externals calls to separate class - */ - std::shared_ptr apiAdapter; - Logger _logger; -}; - -} // namespace driverCompilerAdapter -} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/compiler/include/zero_compiler_in_driver.hpp b/src/plugins/intel_npu/src/compiler/include/zero_compiler_in_driver.hpp deleted file mode 100644 index 658f138a72c102..00000000000000 --- a/src/plugins/intel_npu/src/compiler/include/zero_compiler_in_driver.hpp +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include -#include - -#include -#include - -#include "intel_npu/icompiler.hpp" -#include "intel_npu/utils/logger/logger.hpp" -#include "intel_npu/utils/zero/zero_api.hpp" -#include "zero_executor.hpp" - -namespace intel_npu { -namespace driverCompilerAdapter { - -using SerializedIR = std::pair>; - -#define NotSupportQuery(T) (std::is_same::value) - -// ext version == 1.3 && 1.4, support API (pfnQueryNetworkCreate, pfnQueryNetworkDestroy, -// pfnQueryNetworkGetSupportedLayers) -#define SupportAPIGraphQueryNetworkV1(T) \ - (std::is_same::value || std::is_same::value) - -// ext version >= 1.5, support API (pfnCreate2, pfnQueryNetworkCreate2, pfnQueryContextMemory) -#define SupportAPIGraphQueryNetworkV2(T) ((!NotSupportQuery(T) && !SupportAPIGraphQueryNetworkV1(T))) - -// For ext version >= 1.5, pfnCreate2 api is avaible -#define NotSupportGraph2(T) \ - (std::is_same::value || std::is_same::value || \ - std::is_same::value) - -// A bug inside the driver makes the "pfnGraphGetArgumentMetadata" call not safe for use prior to -// "ze_graph_dditable_ext_1_6_t". -// See: E#117498 -#define NotSupportArgumentMetadata(T) \ - (std::is_same::value || std::is_same::value || \ - std::is_same::value || std::is_same::value) - -#define UseCopyForNativeBinary(T) \ - (std::is_same::value || std::is_same::value || \ - std::is_same::value || std::is_same::value || \ - std::is_same::value) - -/** - * Adapter to use CiD through ZeroAPI - */ -template -class LevelZeroCompilerInDriver final : public ICompiler { -public: - LevelZeroCompilerInDriver(ze_driver_handle_t driverHandle, - ze_device_handle_t deviceHandle, - ze_context_handle_t zeContext, - ze_graph_dditable_ext_curr_t& graph_ddi_table_ext); - LevelZeroCompilerInDriver(const LevelZeroCompilerInDriver&) = delete; - LevelZeroCompilerInDriver& operator=(const LevelZeroCompilerInDriver&) = delete; - ~LevelZeroCompilerInDriver() override; - - uint32_t getSupportedOpsetVersion() const override final; - - ov::SupportedOpsMap query(const std::shared_ptr& model, const Config& config) const override; - - NetworkDescription compile(const std::shared_ptr& model, - const Config& config) const override final; - - ze_result_t seriazlideIRModelAndCreateGraph(const std::shared_ptr& model, - const Config& config, - ze_device_graph_properties_t deviceGraphProperties, - ze_graph_handle_t& graphHandle) const; - - NetworkMetadata parse(const std::vector& network, const Config& config) const override final; - - std::vector process_profiling_output(const std::vector& profData, - const std::vector& network, - const Config& config) const override final { - OPENVINO_THROW("Profiling post-processing is not implemented."); - } - - template = true> - std::unordered_set getQueryResultFromSupportedLayers( - ze_result_t result, - ze_graph_query_network_handle_t& hGraphQueryNetwork) const; - - /** - * @brief Serialize input / output information to string format. - * @details Format: - * --inputs_precisions="0: [1:]" - * --inputs_layouts="0: [1:]" - * --outputs_precisions="0:" - * --outputs_layouts="0:" - * - * For older compiler versions, the name of the inputs/outputs may be used instead of their indices. - * - * Since the layout information is no longer an important part of the metadata values when using the 2.0 OV - * API, the layout fields shall be filled with default values in order to assure the backward compatibility - * with the driver. - */ - static std::string serializeIOInfo(const std::shared_ptr& model, const bool useIndices); - - void release(std::shared_ptr networkDescription) override; - - CompiledNetwork getCompiledNetwork(const NetworkDescription& networkDescription) override; - -private: - NetworkMetadata getNetworkMeta(ze_graph_handle_t graphHandle) const; - - SerializedIR serializeIR(const std::shared_ptr& model, - ze_graph_compiler_version_info_t compilerVersion) const; - std::string serializeConfig(const Config& config, ze_graph_compiler_version_info_t& compilerVersion) const; - - template = true> - void getMetadata(ze_graph_dditable_ext_curr_t& graphDdiTableExt, - ze_graph_handle_t graphHandle, - uint32_t index, - std::vector& inputs, - std::vector& outputs) const; - - template = true> - void getMetadata(ze_graph_dditable_ext_curr_t& graphDdiTableExt, - ze_graph_handle_t graphHandle, - uint32_t index, - std::vector& inputs, - std::vector& outputs) const; - - template = true> - void getNativeBinary(ze_graph_dditable_ext_curr_t& graphDdiTableExt, - ze_graph_handle_t graphHandle, - std::vector& blob, - const uint8_t*& blobPtr, - size_t& blobSize) const; - - template = true> - void getNativeBinary(ze_graph_dditable_ext_curr_t& graphDdiTableExt, - ze_graph_handle_t graphHandle, - std::vector& /* unusedBlob */, - const uint8_t*& blobPtr, - size_t& blobSize) const; - - template = true> - ze_result_t seriazlideIRModelAndQueryNetworkCreateV2(const std::shared_ptr& model, - const Config& config, - ze_device_graph_properties_t deviceGraphProperties, - const ze_device_handle_t& _deviceHandle, - ze_graph_query_network_handle_t& hGraphQueryNetwork) const; - - // ext version >= 1.5, support API (pfnCreate2, pfnQueryNetworkCreate2, pfnQueryContextMemory) - template = true> - std::unordered_set queryImpl(const std::shared_ptr& model, - const Config& config) const; - - template = true> - ze_result_t seriazlideIRModelAndQueryNetworkCreateV1(const std::shared_ptr& model, - const Config& config, - ze_device_graph_properties_t deviceGraphProperties, - const ze_device_handle_t& _deviceHandle, - ze_graph_query_network_handle_t& hGraphQueryNetwork) const; - - // ext version == 1.3 && 1.4, support API (pfnQueryNetworkCreate, pfnQueryNetworkDestroy, - // pfnQueryNetworkGetSupportedLayers) - template = true> - std::unordered_set queryImpl(const std::shared_ptr& model, - const Config& config) const; - - // For ext version < 1.3 - template = true> - std::unordered_set queryImpl(const std::shared_ptr& model, - const Config& config) const; - - template = true> - ze_result_t createGraph(const ze_graph_format_t& format, - const SerializedIR& serializedIR, - const std::string& buildFlags, - const uint32_t& flags, - ze_graph_handle_t* graph) const; - - template = true> - ze_result_t createGraph(const ze_graph_format_t& format, - const SerializedIR& serializedIR, - const std::string& buildFlags, - const uint32_t& flags, - ze_graph_handle_t* graph) const; - -private: - ze_driver_handle_t _driverHandle = nullptr; - ze_device_handle_t _deviceHandle = nullptr; - ze_context_handle_t _context = nullptr; - - ze_graph_dditable_ext_curr_t& _graphDdiTableExt; - Logger _logger; -}; - -} // namespace driverCompilerAdapter -} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/compiler/src/driver_compiler_adapter.cpp b/src/plugins/intel_npu/src/compiler/src/driver_compiler_adapter.cpp deleted file mode 100644 index 84bca75106483d..00000000000000 --- a/src/plugins/intel_npu/src/compiler/src/driver_compiler_adapter.cpp +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "driver_compiler_adapter.hpp" - -#include "graph_transformations.hpp" -#include "intel_npu/config/common.hpp" -#include "intel_npu/utils/zero/zero_api.hpp" -#include "intel_npu/utils/zero/zero_result.hpp" -#include "ze_intel_npu_uuid.h" -#include "zero_backend.hpp" -#include "zero_compiler_in_driver.hpp" -#include "zero_init.hpp" - -namespace intel_npu { -namespace driverCompilerAdapter { - -LevelZeroCompilerAdapter::LevelZeroCompilerAdapter(std::shared_ptr iEngineBackend) - : _logger("LevelZeroCompilerAdapter", Logger::global().level()) { - _logger.debug("initialize LevelZeroCompilerAdapter start"); - - std::shared_ptr zeroBackend = nullptr; - zeroBackend = std::dynamic_pointer_cast(iEngineBackend); - if (!zeroBackend) { - OPENVINO_THROW("LevelZeroCompilerAdapter init failed to cast zeroBackend, zeroBackend is a nullptr"); - } - - ze_context_handle_t zeContext = static_cast(zeroBackend->getContext()); - ze_driver_handle_t driverHandle = static_cast(zeroBackend->getDriverHandle()); - ze_device_handle_t deviceHandle = static_cast(zeroBackend->getDeviceHandle()); - ze_graph_dditable_ext_curr_t& graph_ddi_table_ext = zeroBackend->getGraphDdiTable(); - - uint32_t graphExtVersion = graph_ddi_table_ext.version(); - - if (driverHandle == nullptr) { - OPENVINO_THROW("LevelZeroCompilerAdapter failed to get properties about zeDriver"); - } - - _logger.info("LevelZeroCompilerAdapter creating adapter using graphExtVersion"); - - switch (graphExtVersion) { - case ZE_GRAPH_EXT_VERSION_1_3: - apiAdapter = std::make_shared>(driverHandle, - deviceHandle, - zeContext, - graph_ddi_table_ext); - break; - case ZE_GRAPH_EXT_VERSION_1_4: - apiAdapter = std::make_shared>(driverHandle, - deviceHandle, - zeContext, - graph_ddi_table_ext); - break; - case ZE_GRAPH_EXT_VERSION_1_5: - apiAdapter = std::make_shared>(driverHandle, - deviceHandle, - zeContext, - graph_ddi_table_ext); - break; - case ZE_GRAPH_EXT_VERSION_1_6: - apiAdapter = std::make_shared>(driverHandle, - deviceHandle, - zeContext, - graph_ddi_table_ext); - break; - case ZE_GRAPH_EXT_VERSION_1_7: - apiAdapter = std::make_shared>(driverHandle, - deviceHandle, - zeContext, - graph_ddi_table_ext); - break; - case ZE_GRAPH_EXT_VERSION_1_8: - apiAdapter = std::make_shared>(driverHandle, - deviceHandle, - zeContext, - graph_ddi_table_ext); - break; - default: - apiAdapter = std::make_shared>(driverHandle, - deviceHandle, - zeContext, - graph_ddi_table_ext); - break; - } - - _logger.info("initialize LevelZeroCompilerAdapter complete, using graphExtVersion: %d.%d", - ZE_MAJOR_VERSION(graphExtVersion), - ZE_MINOR_VERSION(graphExtVersion)); -} - -uint32_t LevelZeroCompilerAdapter::getSupportedOpsetVersion() const { - return apiAdapter->getSupportedOpsetVersion(); -} - -NetworkDescription LevelZeroCompilerAdapter::compile(const std::shared_ptr& model, - const Config& config) const { - _logger.debug("compile start"); - return apiAdapter->compile(model, config); -} - -ov::SupportedOpsMap LevelZeroCompilerAdapter::query(const std::shared_ptr& model, - const Config& config) const { - _logger.debug("query start"); - return apiAdapter->query(model, config); -} - -NetworkMetadata LevelZeroCompilerAdapter::parse(const std::vector& network, const Config& config) const { - _logger.debug("parse start"); - return apiAdapter->parse(network, config); -} - -std::vector LevelZeroCompilerAdapter::process_profiling_output(const std::vector&, - const std::vector&, - const Config&) const { - OPENVINO_THROW("Profiling post-processing is not implemented."); -} - -void LevelZeroCompilerAdapter::release(std::shared_ptr networkDescription) { - _logger.info("release - using adapter to release networkDescription"); - apiAdapter->release(std::move(networkDescription)); -} - -CompiledNetwork LevelZeroCompilerAdapter::getCompiledNetwork(const NetworkDescription& networkDescription) { - _logger.info("getCompiledNetwork - using adapter to perform getCompiledNetwork(networkDescription)"); - return apiAdapter->getCompiledNetwork(networkDescription); -} - -} // namespace driverCompilerAdapter -} // namespace intel_npu diff --git a/src/plugins/intel_npu/src/compiler/src/zero_compiler_in_driver.cpp b/src/plugins/intel_npu/src/compiler/src/zero_compiler_in_driver.cpp deleted file mode 100644 index 47bc7611d132c3..00000000000000 --- a/src/plugins/intel_npu/src/compiler/src/zero_compiler_in_driver.cpp +++ /dev/null @@ -1,1081 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "zero_compiler_in_driver.hpp" - -#include -#include - -#include "graph_transformations.hpp" -#include "intel_npu/common/itt.hpp" -#include "intel_npu/config/common.hpp" -#include "intel_npu/config/compiler.hpp" -#include "intel_npu/config/runtime.hpp" -#include "intel_npu/prefix.hpp" -#include "intel_npu/utils/zero/zero_result.hpp" -#include "openvino/core/model.hpp" - -namespace { - -constexpr std::string_view INPUTS_PRECISIONS_KEY = "--inputs_precisions"; -constexpr std::string_view INPUTS_LAYOUTS_KEY = "--inputs_layouts"; -constexpr std::string_view OUTPUTS_PRECISIONS_KEY = "--outputs_precisions"; -constexpr std::string_view OUTPUTS_LAYOUTS_KEY = "--outputs_layouts"; - -//