From 952442bca3262ec16bc835d0787d1a15df3b3a94 Mon Sep 17 00:00:00 2001 From: ActivePeter <1020401660@qq.com> Date: Wed, 29 Nov 2023 18:06:26 +0800 Subject: [PATCH] add: ansible --- .github/workflows/build.yml | 38 ++--- .github/workflows/ci.yml | 17 ++- .gitignore | 3 +- RunnerGo | 1 + docker/WasmServerless/.gitignore | 2 + docker/WasmServerless/Dockerfile | 11 ++ docker/WasmServerless/build_image.sh | 31 ++-- .../wasm_serverless_entrypoint.sh | 8 +- docker/deploy_cluster/build_remote_images.py | 93 ------------ docker/deploy_cluster/docker-stack.yml | 77 ---------- docker/deploy_cluster/prepare.py | 141 ------------------ docker/deploy_cluster/stack_remove_all.sh | 1 - docker/deploy_cluster/stack_up.sh | 2 - docker/docker-tc | 1 - scripts/ans/.gitignore | 2 + .../ans/_ans_build_cluster_docker_stack.yml | 69 +++++++++ scripts/ans/_ans_install.yml | 15 ++ scripts/ans/_ans_install_build.yml | 41 +++++ scripts/ans/ans_build_release.yml | 28 ++++ scripts/ans/ans_install_build.yml | 7 + scripts/ans/ans_redeploy_cluster.yml | 97 ++++++++++++ scripts/deploy_cluster/.gitignore | 2 + scripts/deploy_cluster/1.ansible_setup.py | 86 +++++++++++ scripts/deploy_cluster/2.ans_redeploy.sh | 1 + .../deploy_cluster/node_config.yaml | 0 .../deploy_cluster/traffic_control.sh | 0 scripts/deploy_single_node/1.ansible_setup.sh | 15 ++ {docker => scripts/deploy_single_node}/ci.sh | 0 .../deploy_single_node/docker-compose.yml | 0 scripts/install/install_ansible.sh | 3 + scripts/install/install_rust.sh | 3 + scripts/{ => install}/install_wasmedge.sh | 0 scripts/install_docker.sh | 1 - scripts/install_protobuf.sh | 3 - scripts/install_rust.sh | 18 --- src/cmd_arg.rs | 2 +- src/config.rs | 7 +- src/main.rs | 2 +- src/schedule/container_manager.rs | 14 +- src/schedule/executor.rs | 6 +- src/schedule/master.rs | 2 +- src/schedule/worker.rs | 2 +- 42 files changed, 465 insertions(+), 387 deletions(-) create mode 160000 RunnerGo create mode 100644 docker/WasmServerless/.gitignore delete mode 100644 docker/deploy_cluster/build_remote_images.py delete mode 100644 docker/deploy_cluster/docker-stack.yml delete mode 100644 docker/deploy_cluster/prepare.py delete mode 100644 docker/deploy_cluster/stack_remove_all.sh delete mode 100644 docker/deploy_cluster/stack_up.sh delete mode 160000 docker/docker-tc create mode 100644 scripts/ans/.gitignore create mode 100644 scripts/ans/_ans_build_cluster_docker_stack.yml create mode 100644 scripts/ans/_ans_install.yml create mode 100644 scripts/ans/_ans_install_build.yml create mode 100644 scripts/ans/ans_build_release.yml create mode 100644 scripts/ans/ans_install_build.yml create mode 100644 scripts/ans/ans_redeploy_cluster.yml create mode 100644 scripts/deploy_cluster/.gitignore create mode 100644 scripts/deploy_cluster/1.ansible_setup.py create mode 100644 scripts/deploy_cluster/2.ans_redeploy.sh rename {docker => scripts}/deploy_cluster/node_config.yaml (100%) rename {docker => scripts}/deploy_cluster/traffic_control.sh (100%) create mode 100644 scripts/deploy_single_node/1.ansible_setup.sh rename {docker => scripts/deploy_single_node}/ci.sh (100%) rename {docker => scripts}/deploy_single_node/docker-compose.yml (100%) create mode 100644 scripts/install/install_ansible.sh create mode 100644 scripts/install/install_rust.sh rename scripts/{ => install}/install_wasmedge.sh (100%) delete mode 100644 scripts/install_docker.sh delete mode 100644 scripts/install_protobuf.sh delete mode 100644 scripts/install_rust.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8324e80..1cddb50 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,22 +1,22 @@ -name: Rust +# name: Rust -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] +# on: +# push: +# branches: [ "main" ] +# pull_request: +# branches: [ "main" ] -env: - CARGO_TERM_COLOR: always +# env: +# CARGO_TERM_COLOR: always -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Install WasmEdge - run: bash ./scripts/install_wasmedge.sh - - name: Install Protobuf - run: bash ./scripts/install_protobuf.sh - - name: Build - run: cargo build --verbose +# jobs: +# build: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - name: Install WasmEdge +# run: bash ./scripts/install/install_wasmedge.sh +# - name: Install Protobuf +# run: bash ./scripts/install/install_protobuf.sh +# - name: Build +# run: cargo build --verbose diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55ec22d..8c7aca1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,13 +14,20 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Install Ansible + run: sudo apt install python3 python3-pip && bash scripts/install/install_ansible.sh + - name: Setup Ansible + run: bash scripts/deploy_single_node/1.ansible_setup.sh + - name: Ansible install all + run: ansible-playbook -vvv scripts/ans/ans_install_build.yml + + - name: Build release + run: ansible-playbook -vvv scripts/ans/ans_build_release.yml + - name: Build WasmEdge Image run: bash docker/WasmEdge/build_image.sh - - name: Install WasmEdge - run: bash ./scripts/install_wasmedge.sh - - name: Install Protobuf - run: bash ./scripts/install_protobuf.sh - name: Build WasmServerless Image run: bash docker/WasmServerless/build_image.sh + - name: CI - run: bash docker/ci.sh + run: bash scripts/deploy_single_node/ci.sh diff --git a/.gitignore b/.gitignore index 1de5659..ff7f2fe 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -target \ No newline at end of file +target +docker.zip diff --git a/RunnerGo b/RunnerGo new file mode 160000 index 0000000..4430946 --- /dev/null +++ b/RunnerGo @@ -0,0 +1 @@ +Subproject commit 4430946d196fe7842e83acb3a42a15f549ae0130 diff --git a/docker/WasmServerless/.gitignore b/docker/WasmServerless/.gitignore new file mode 100644 index 0000000..ef7e29a --- /dev/null +++ b/docker/WasmServerless/.gitignore @@ -0,0 +1,2 @@ +apps +target \ No newline at end of file diff --git a/docker/WasmServerless/Dockerfile b/docker/WasmServerless/Dockerfile index 8787577..51428fa 100644 --- a/docker/WasmServerless/Dockerfile +++ b/docker/WasmServerless/Dockerfile @@ -20,6 +20,17 @@ LABEL description="Wasm serverless platform" # RUN python3 /tmp/install/wasm_edge.py -v 0.13.3 COPY target/release/wasm_serverless /usr/local/bin/wasm_serverless +COPY apps/fn2/target/wasm32-wasi/release/fn2.wasm /etc/wasm_serverless/fn1.wasm +COPY apps/fn2/target/wasm32-wasi/release/fn2.wasm /etc/wasm_serverless/fn2.wasm +COPY apps/fn2/target/wasm32-wasi/release/fn2.wasm /etc/wasm_serverless/fn3.wasm +COPY apps/fn2/target/wasm32-wasi/release/fn2.wasm /etc/wasm_serverless/fn4.wasm +COPY apps/fn2/target/wasm32-wasi/release/fn2.wasm /etc/wasm_serverless/fn5.wasm +COPY apps/fn2/target/wasm32-wasi/release/fn2.wasm /etc/wasm_serverless/fn6.wasm +COPY apps/fn2/target/wasm32-wasi/release/fn2.wasm /etc/wasm_serverless/fn7.wasm +COPY apps/fn2/target/wasm32-wasi/release/fn2.wasm /etc/wasm_serverless/fn8.wasm +COPY apps/fn2/target/wasm32-wasi/release/fn2.wasm /etc/wasm_serverless/fn9.wasm +COPY apps/fn2/target/wasm32-wasi/release/fn2.wasm /etc/wasm_serverless/fn10.wasm + # COPY node_config.yaml /etc/wasm_serverless/node_config.yaml COPY wasm_serverless_entrypoint.sh /etc/wasm_serverless/ RUN chmod +x /etc/wasm_serverless/wasm_serverless_entrypoint.sh diff --git a/docker/WasmServerless/build_image.sh b/docker/WasmServerless/build_image.sh index 72be916..562d4e7 100644 --- a/docker/WasmServerless/build_image.sh +++ b/docker/WasmServerless/build_image.sh @@ -1,23 +1,32 @@ #!/bin/bash CURRENT_DIR=`pwd` -TARGET_DIR="$CURRENT_DIR/target" DOCKER_DIR="$CURRENT_DIR/docker/WasmServerless" IMAGE_VERSION="v1" IMAGE_NAME="wasm_serverless:$IMAGE_VERSION" -if [ ! -d $TARGET_DIR ] -then - cargo build --release - if test $? -ne 0 - then - exit 1 - fi -fi - +# if [ ! -d $TARGET_DIR ] +# then +# cargo build --release +# if test $? -ne 0 +# then +# exit 1 +# fi +# fi +echo ">" +ls $CURRENT_DIR/target +echo ">" +ls $CURRENT_DIR/target/release +echo ">" +ls $CURRENT_DIR/apps/fn2/target +echo ">" +ls $CURRENT_DIR/apps/fn2/target/wasm32-wasi/release mkdir -p $DOCKER_DIR/target/release -cp $TARGET_DIR/release/wasm_serverless $DOCKER_DIR/target/release/wasm_serverless +mkdir -p $DOCKER_DIR/apps/fn2 + +cp $CURRENT_DIR/target/release/wasm_serverless $DOCKER_DIR/target/release/wasm_serverless +cp -r $CURRENT_DIR/apps/fn2/target $DOCKER_DIR/apps/fn2/target # cp "$CURRENT_DIR/node_config.yaml" $DOCKER_DIR diff --git a/docker/WasmServerless/wasm_serverless_entrypoint.sh b/docker/WasmServerless/wasm_serverless_entrypoint.sh index d769adc..754818b 100644 --- a/docker/WasmServerless/wasm_serverless_entrypoint.sh +++ b/docker/WasmServerless/wasm_serverless_entrypoint.sh @@ -1,5 +1,9 @@ #!/bin/bash -echo "Node id is ${WASM_SERVERLESS_NODEID}" +echo "Node id: ${WASM_SERVERLESS_NODEID}" +echo "Who am i: $(whoami)" -/usr/local/bin/wasm_serverless $WASM_SERVERLESS_NODEID /etc/wasm_serverless/config/node_config.yaml \ No newline at end of file +cd /usr/local/bin/ +ls /etc/wasm_serverless/ + +wasm_serverless $WASM_SERVERLESS_NODEID /etc/wasm_serverless/ \ No newline at end of file diff --git a/docker/deploy_cluster/build_remote_images.py b/docker/deploy_cluster/build_remote_images.py deleted file mode 100644 index eac3980..0000000 --- a/docker/deploy_cluster/build_remote_images.py +++ /dev/null @@ -1,93 +0,0 @@ -import os -import yaml -import argparse -import sys -import pexpect - -PASSWORD="123456" - -def run_cmd(cmd): - print("> "+cmd) - # if cmd.startswith("ssh") or cmd.startswith("scp"): - # # 创建spawn对象 - # child = pexpect.spawn(cmd, encoding='utf-8',logfile=sys.stdout) - - # # 匹配密码提示，然后发送密码 - # child.expect('password:') - # child.sendline(PASSWORD) - - # # 在这里可以继续与SSH会话进行交互 - # # 例如，可以发送其他命令 - - # # 等待命令执行完成 - # try: - # child.expect(pexpect.EOF) - # except: - # pass - # child.close() - # # 打印输出 - # # print(child.before) - # else: - os.system(cmd) - -def run_cmd_with_result(cmd): - print("> "+cmd) - return os.popen(cmd).read() - -def read_yaml(f): - # parse - import ruamel.yaml - yaml = ruamel.yaml.YAML(typ='rt') - parsed_data = yaml.load(f) - - return parsed_data - -def entry(args): - # read cluster-nodes.yml - with open('node_config.yaml', 'r') as f: - cluster_nodes = read_yaml(f) - for nid in cluster_nodes["nodes"]: - node=cluster_nodes["nodes"][nid] - ip=node["addr"].split(":")[0] - port=node["addr"].split(":")[1] - id=node["id"] - - # mkdir remote - run_cmd("ssh root@{} 'rm -rf /root/wasm_serverless_deploy && mkdir -p /root/wasm_serverless_deploy/target/release && ls /root/wasm_serverless_deploy'".format(ip)) - if args.update_binary: - # cp ../../target to remote /root/wasm_serverless_deploy/target - run_cmd("scp ../../target/release/wasm_serverless root@{}:/root/wasm_serverless_deploy/target/release && scp -r ../../docker root@{}:/root/wasm_serverless_deploy/docker".format(ip,ip)) - - job="cd /root/wasm_serverless_deploy && ls && chmod -R 775 . && ./docker/WasmServerless/build_image.sh" - if args.update_env: - job="cd /root/wasm_serverless_deploy && ls && chmod -R 775 . && ./docker/WasmEdge/build_image.sh && ./docker/WasmServerless/build_image.sh" - # store to remote tmp.sh - run_cmd("ssh root@{} 'echo \"{}\" > /root/wasm_serverless_deploy/tmp.sh'".format(ip, job)) - run_cmd("ssh root@{} 'chmod +x /root/wasm_serverless_deploy/tmp.sh && /root/wasm_serverless_deploy/tmp.sh'".format(ip)) - - # if args.update_env: - # # check if docker image exists: wasm_serverless_env:v1 - # check=run_cmd_with_result("ssh root@{} docker images".format(ip)) - # if "wasm_serverless_env " not in check: - # run_cmd("cd /root/wasm_serverless_deploy && ./docker/WasmEdge/build_image.sh") - - # run_cmd("ssh root@{} cd /root/wasm_serverless_deploy && ls && chmod -R 775 . && ./docker/WasmServerless/build_image.sh ".format(ip)) - - - -# parse arg -# - update env -# - update binary -# password -parser = argparse.ArgumentParser(description='deploy wasm_serverless to cluster') -# parser.add_argument('--update-binary', action='store_false', help='update binary',default=False) -parser.add_argument('--update-binary', action='store_true', help='update binary',default=False) -# parser.add_argument('--update-env', action='store_false', help='update env',default=False) -parser.add_argument('--update-env', action='store_true', help='update env',default=False) -parser.add_argument('--password', action='store', help='password',default="123456") -args=parser.parse_args() -PASSWORD=args.password -print(args) - -entry(args) - diff --git a/docker/deploy_cluster/docker-stack.yml b/docker/deploy_cluster/docker-stack.yml deleted file mode 100644 index c75a822..0000000 --- a/docker/deploy_cluster/docker-stack.yml +++ /dev/null @@ -1,77 +0,0 @@ -networks: - default: - external: {name: host} -services: - node1: - deploy: - mode: replicated - placement: - constraints: [node.id == h8wq4b2uy6fiac37lbx16gb08] - replicas: 1 - environment: {WASM_SERVERLESS_NODEID: 1} - image: wasm_serverless:v1 - ports: - - {mode: host, published: 2500, target: 2500} - - {mode: host, published: 2501, target: 2501} - volumes: ['/etc/wasm_serverless/config:/etc/wasm_serverless/config'] - node2: - deploy: - mode: replicated - placement: - constraints: [node.id == h8wq4b2uy6fiac37lbx16gb08] - replicas: 1 - environment: {WASM_SERVERLESS_NODEID: 2} - image: wasm_serverless:v1 - ports: - - {mode: host, published: 2505, target: 2505} - - {mode: host, published: 2506, target: 2506} - volumes: ['/etc/wasm_serverless/config:/etc/wasm_serverless/config'] - node3: - deploy: - mode: replicated - placement: - constraints: [node.id == t8f80nn3qxavell7leopbab40] - replicas: 1 - environment: {WASM_SERVERLESS_NODEID: 3} - image: wasm_serverless:v1 - ports: - - {mode: host, published: 2510, target: 2510} - - {mode: host, published: 2511, target: 2511} - volumes: ['/etc/wasm_serverless/config:/etc/wasm_serverless/config'] - node4: - deploy: - mode: replicated - placement: - constraints: [node.id == t8f80nn3qxavell7leopbab40] - replicas: 1 - environment: {WASM_SERVERLESS_NODEID: 4} - image: wasm_serverless:v1 - ports: - - {mode: host, published: 2515, target: 2515} - - {mode: host, published: 2516, target: 2516} - volumes: ['/etc/wasm_serverless/config:/etc/wasm_serverless/config'] - node5: - deploy: - mode: replicated - placement: - constraints: [node.id == d8huj03dv09keg3gp4vh1cj7k] - replicas: 1 - environment: {WASM_SERVERLESS_NODEID: 5} - image: wasm_serverless:v1 - ports: - - {mode: host, published: 2520, target: 2520} - - {mode: host, published: 2521, target: 2521} - volumes: ['/etc/wasm_serverless/config:/etc/wasm_serverless/config'] - node6: - deploy: - mode: replicated - placement: - constraints: [node.id == d8huj03dv09keg3gp4vh1cj7k] - replicas: 1 - environment: {WASM_SERVERLESS_NODEID: 6} - image: wasm_serverless:v1 - ports: - - {mode: host, published: 2525, target: 2525} - - {mode: host, published: 2526, target: 2526} - volumes: ['/etc/wasm_serverless/config:/etc/wasm_serverless/config'] -version: '3.5' diff --git a/docker/deploy_cluster/prepare.py b/docker/deploy_cluster/prepare.py deleted file mode 100644 index 00cfd31..0000000 --- a/docker/deploy_cluster/prepare.py +++ /dev/null @@ -1,141 +0,0 @@ -import os -import yaml -import argparse -import sys -import pexpect - -PASSWORD="aaaaa" - -def run_cmd(cmd): - print("> "+cmd) - if cmd.startswith("ssh") or cmd.startswith("scp"): - # 创建spawn对象 - child = pexpect.spawn(cmd, encoding='utf-8',logfile=sys.stdout) - - # 匹配密码提示，然后发送密码 - child.expect('password:') - child.sendline(PASSWORD) - - # 在这里可以继续与SSH会话进行交互 - # 例如，可以发送其他命令 - - # 等待命令执行完成 - try: - child.expect(pexpect.EOF) - except: - pass - child.close() - # 打印输出 - # print(child.before) - else: - os.system(cmd) - -def run_cmd_with_result(cmd): - print("> "+cmd) - return os.popen(cmd).read() - -def read_yaml(f): - # parse - import ruamel.yaml - yaml = ruamel.yaml.YAML(typ='rt') - parsed_data = yaml.load(f) - - return parsed_data - -def entry(): - # check base image prepared - check=run_cmd_with_result("docker images") - if "wasm_serverless " not in check: - print("Error: base image not prepared") - exit(1) - - run_cmd("rm -f docker-stack.yml") - new_docker_stack={ - # fix(can't be 3): https://github.com/docker/cli/issues/1073#issuecomment-395466027 - 'version':"3.5", - 'services':{}, - "networks":{ - "default":{ - "external":{ - "name": "host" - } - } - } - } - def add_sevice(dockerid,nodeid, port,ip): - def portstr(port): - return { - "target": int(port), - "published": int(port), - "mode": "host" - } - new_docker_stack["services"]["node{}".format(nodeid)]={ - "image":"wasm_serverless:v1", - "deploy":{ - "mode": "replicated", - "replicas": 1, - "placement":{ - "constraints":["node.id == {}".format(dockerid)] - } - }, - "ports":[portstr(port), portstr(int(port)+1)], - "volumes":['/etc/wasm_serverless/config:/etc/wasm_serverless/config'], - 'environment': { - "WASM_SERVERLESS_NODEID": nodeid - }, - # 'networks':['host'] - # 'network_mode': 'host' - } - - run_cmd("docker node ls") - - # read cluster-nodes.yml - with open('node_config.yaml', 'r') as f: - cluster_nodes = read_yaml(f) - for nid in cluster_nodes["nodes"]: - node=cluster_nodes["nodes"][nid] - ip=node["addr"].split(":")[0] - port=node["addr"].split(":")[1] - id=node["id"] - - # mkdir remote - run_cmd("ssh root@{} mkdir -p /etc/wasm_serverless/config".format(ip)) - # send node.config to each node scp - run_cmd("scp node_config.yaml root@{}:/etc/wasm_serverless/config/node_config.yaml".format(ip)) - # check remote /root/wasm_serverless_deploy exist, if not scp send - # exist = run_cmd_with_result("ssh root@{} ls /root/wasm_serverless_deploy".format(ip)) - - # run_cmd("docker node update --label-add nodeid={} {}".format(nid,id)) - # run_cmd("docker node update --label-add baseport={} {}".format(port,id)) - - # show labels - # run_cmd("docker node inspect --format '{{ .Spec.Labels }}' "+(id)) - - # build image - # generate entrypoint.sh - # with open('entrypoint.sh', 'w') as f: - # content='#!/bin/bash \n\ - # # 在脚本中获取节点标签的值并设置为环境变量\n\ - # export WASM_SERVERLESS_NODEID={} \n\ - # # 运行实际的命令 \n\ - # exec "$@"'.format(nid) - - # f.write(content) - - # run_cmd("docker build -t wasm_serverless_node{}:v1 . --no-cache".format(nid)) - # run_cmd("rm -f entrypoint.sh") - - add_sevice(id,nid, port, ip) - - # write docker-stack.yml - with open('docker-stack.yml', 'w') as f: - yaml.dump(new_docker_stack, f) - - - -# parse password -parser = argparse.ArgumentParser() -parser.add_argument('--password', help='password for ssh', default="aaaaa") -args=parser.parse_args() -PASSWORD=args.password -entry() \ No newline at end of file diff --git a/docker/deploy_cluster/stack_remove_all.sh b/docker/deploy_cluster/stack_remove_all.sh deleted file mode 100644 index 6e0d1b0..0000000 --- a/docker/deploy_cluster/stack_remove_all.sh +++ /dev/null @@ -1 +0,0 @@ -docker stack rm $(docker stack ls --format '{{.Name}}') \ No newline at end of file diff --git a/docker/deploy_cluster/stack_up.sh b/docker/deploy_cluster/stack_up.sh deleted file mode 100644 index bd07bc1..0000000 --- a/docker/deploy_cluster/stack_up.sh +++ /dev/null @@ -1,2 +0,0 @@ -# docker network create --subnet 192.168.31.0/24 --gateway 192.168.31.1 wasm_serverless_bridge -docker stack deploy -c docker-stack.yml wasm_serverless \ No newline at end of file diff --git a/docker/docker-tc b/docker/docker-tc deleted file mode 160000 index 8b1355e..0000000 --- a/docker/docker-tc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8b1355ecbaf388cc5e1c0049b483c0626226eabd diff --git a/scripts/ans/.gitignore b/scripts/ans/.gitignore new file mode 100644 index 0000000..ac71af1 --- /dev/null +++ b/scripts/ans/.gitignore @@ -0,0 +1,2 @@ +*.retry +docker.zip \ No newline at end of file diff --git a/scripts/ans/_ans_build_cluster_docker_stack.yml b/scripts/ans/_ans_build_cluster_docker_stack.yml new file mode 100644 index 0000000..82b8f8e --- /dev/null +++ b/scripts/ans/_ans_build_cluster_docker_stack.yml @@ -0,0 +1,69 @@ +# --- +# - hosts: localhost +# # gather_facts: false + +# tasks: + - name: Read node configuration + include_vars: + file: ../deploy_cluster/node_config.yaml + name: node_config + + - name: Display node information + debug: + var: node_config + + - name: Initialize Docker stack configuration + set_fact: + docker_stack_configurations: {} + nodes: [] + + - name: Collect port + loop: "{{ node_config.nodes | dict2items }}" + loop_control: + loop_var: node_item + set_fact: + nodes: "{{ nodes + [{ + 'nid': node_item.key, + 'id': node_item.value.id, + 'port': node_item.value.addr.split(':')[1] | int + }] }}" + - name: Add service for each node + loop: "{{ nodes }}" + loop_control: + loop_var: node_item + set_fact: + docker_stack_configurations: "{{ docker_stack_configurations | combine({ + 'node' ~ node_item.nid: { + 'image': 'wasm_serverless:v1', + 'deploy': { + 'mode': 'replicated', + 'replicas': 1, + 'placement': { + 'constraints': ['node.id == {}'.format(node_item.id)] + } + }, + 'ports': [ + {'target': node_item.port, 'published': node_item.port, 'mode': 'host'}, + {'target': node_item.port + 1, 'published': node_item.port + 1, 'mode': 'host'} + ], + 'volumes': ['/root/wasm_serverless_deploy:/etc/wasm_serverless/config'], + 'environment': { + 'WASM_SERVERLESS_NODEID': node_item.nid + } + } + }) }}" + + - name: Warp Outter + + set_fact: + docker_stack_configurations: + networks: + default: + external: + name: host + version: '3.5' + services: "{{docker_stack_configurations}}" + + - name: Write the map to a YAML file + shell: echo "{{ docker_stack_configurations | to_nice_yaml }}" > ../deploy_cluster/gen_docker_stack.yml + diff --git a/scripts/ans/_ans_install.yml b/scripts/ans/_ans_install.yml new file mode 100644 index 0000000..6aef5b4 --- /dev/null +++ b/scripts/ans/_ans_install.yml @@ -0,0 +1,15 @@ + + - name: Install Build Related + become: true + include: _ans_install_build.yml + - name: Install Docker + become: true + apt: + name: + - docker.io + state: present + + # - name: Debug + # shell: ls + + \ No newline at end of file diff --git a/scripts/ans/_ans_install_build.yml b/scripts/ans/_ans_install_build.yml new file mode 100644 index 0000000..9ad1119 --- /dev/null +++ b/scripts/ans/_ans_install_build.yml @@ -0,0 +1,41 @@ +- name: Check if Rust is installed + become: true + command: rustc --version + ignore_errors: yes + register: rust_installed + +- name: Install required packages + become: true + apt: + name: "{{ item }}" + state: present + loop: + - clang + - lldb + - lld + - build-essential + - curl + - protobuf-compiler + - pkg-config + - libssl-dev + +- name: Install Rust if not present + become: true + shell: bash ../install/install_rust.sh + + when: "'rustc' not in rust_installed.stdout" +- name: Add Rust to PATH + become: true + lineinfile: + path: "{{ ansible_user_dir }}/.bashrc" + line: | + export PATH="{{ ansible_user_dir }}/cargo/bin:$PATH" + when: "'rustc' not in rust_installed.stdout" + +- name: Install WasmEdge + become: true + shell: | + bash ./scripts/install/install_wasmedge.sh + args: + chdir: '../..' + creates: "/usr/local/bin/wasmedge" # Skip if WasmEdge is already installed \ No newline at end of file diff --git a/scripts/ans/ans_build_release.yml b/scripts/ans/ans_build_release.yml new file mode 100644 index 0000000..4e2aec1 --- /dev/null +++ b/scripts/ans/ans_build_release.yml @@ -0,0 +1,28 @@ +--- +- hosts: localhost + tasks: + - name: Build the application on the master node + become: true + shell: | + cat > /tmp/build.sh <<'END' + #!/bin/sh + source $HOME/.cargo/env + cargo build --release + END + bash /tmp/build.sh + rm -f /tmp/build.sh + args: + chdir: ../../apps/fn2 + + - name: Build demo wasm fn2 + become: true + shell: | + cat > /tmp/build.sh <<'END' + #!/bin/sh + source $HOME/.cargo/env + cargo build --target wasm32-wasi --release + END + bash /tmp/build.sh + rm -f /tmp/build.sh + args: + chdir: ../../apps/fn2 \ No newline at end of file diff --git a/scripts/ans/ans_install_build.yml b/scripts/ans/ans_install_build.yml new file mode 100644 index 0000000..bd4bf15 --- /dev/null +++ b/scripts/ans/ans_install_build.yml @@ -0,0 +1,7 @@ +--- +- hosts: localhost + tasks: + # install rysr, proto, docker, docker-compose + - name: Install Build Related + become: true + include: _ans_install_build.yml \ No newline at end of file diff --git a/scripts/ans/ans_redeploy_cluster.yml b/scripts/ans/ans_redeploy_cluster.yml new file mode 100644 index 0000000..0d13464 --- /dev/null +++ b/scripts/ans/ans_redeploy_cluster.yml @@ -0,0 +1,97 @@ +--- +- hosts: localhost + tasks: + # install rysr, proto, docker, docker-compose + - name: Install All + include_tasks: _ans_install.yml + + - name: Remove Docker Stack + docker_stack: + state: absent + name: wasm_serverless + + + - name: Build the application on the master node + shell: cargo build --release + + - name: Build the WASM demo fn + shell: cargo build --target wasm32-wasi --release + args: + chdir: ../../apps/fn2 + + # - name: Debug + # shell: ls ../../target/release/ + + - name: Generate Docker stack file + include_tasks: _ans_build_cluster_docker_stack.yml + + - name: Remove Zip + file: + path: ../../docker.zip + state: absent + + - name: Zip docker file + command: "zip -r docker.zip docker" + args: + chdir: ../.. +- hosts: web + vars_prompt: + - name: build_env + prompt: "Do you want to rebuild the environment? (yes/no)" + private: false + tasks: + - name: Debug file path + debug: + var: src + - name: Copy release to all nodes + copy: + src: ../../target/release/wasm_serverless + dest: /root/wasm_serverless_deploy/target/release/wasm_serverless + become: yes + - name: Copy node_config.yaml to all nodes + copy: + src: ../deploy_cluster/node_config.yaml + dest: /root/wasm_serverless_deploy/node_config.yaml + become: yes + - name: Make dir for wasm + file: + path: /root/wasm_serverless_deploy/apps/fn2/target/wasm32-wasi/release + state: directory + become: yes + - name: Copy wasm to all nodes + copy: + src: ../../apps/fn2/target/wasm32-wasi/release/fn2.wasm + dest: /root/wasm_serverless_deploy/apps/fn2/target/wasm32-wasi/release/fn2.wasm + force: yes + become: yes + - name: Copy Docker scripts to all nodes + copy: + src: ../../docker.zip + dest: /root/wasm_serverless_deploy/docker.zip + become: yes + + - name: Unzip Docker scripts to all nodes + shell: rm -rf docker && unzip -q -o docker.zip && rm -f docker.zip && chmod -R 775 docker + args: + chdir: /root/wasm_serverless_deploy + become: yes + + - name: Build the environment Docker image (conditionally) + command: ./docker/WasmEdge/build_image.sh + args: + chdir: /root/wasm_serverless_deploy + become: yes + when: build_env == "yes" + + - name: Build the application Docker image + command: ./docker/WasmServerless/build_image.sh + args: + chdir: /root/wasm_serverless_deploy + become: yes + + + +- hosts: localhost + tasks: + - name: Stack Up + shell: docker stack deploy -c ../deploy_cluster/gen_docker_stack.yml wasm_serverless diff --git a/scripts/deploy_cluster/.gitignore b/scripts/deploy_cluster/.gitignore new file mode 100644 index 0000000..18b4fab --- /dev/null +++ b/scripts/deploy_cluster/.gitignore @@ -0,0 +1,2 @@ +gen_ansible.ini +gen_docker_stack.yml \ No newline at end of file diff --git a/scripts/deploy_cluster/1.ansible_setup.py b/scripts/deploy_cluster/1.ansible_setup.py new file mode 100644 index 0000000..e61857c --- /dev/null +++ b/scripts/deploy_cluster/1.ansible_setup.py @@ -0,0 +1,86 @@ +import os +import yaml +import argparse +import sys +import pexpect + +PASSWORD="aaaaa" + +def run_cmd(cmd): + print("> "+cmd) + # if cmd.startswith("ssh") or cmd.startswith("scp"): + # # 创建spawn对象 + # child = pexpect.spawn(cmd, encoding='utf-8',logfile=sys.stdout) + + # # 匹配密码提示，然后发送密码 + # child.expect('password:') + # child.sendline(PASSWORD) + + # # 在这里可以继续与SSH会话进行交互 + # # 例如，可以发送其他命令 + + # # 等待命令执行完成 + # try: + # child.expect(pexpect.EOF) + # except: + # pass + # child.close() + # # 打印输出 + # # print(child.before) + # else: + os.system(cmd) + + +def read_yaml(f): + # parse + import ruamel.yaml + yaml = ruamel.yaml.YAML(typ='rt') + parsed_data = yaml.load(f) + + return parsed_data + +def entry(): + # read cluster-nodes.yml + with open('scripts/deploy_cluster/node_config.yaml', 'r') as f: + run_cmd("scripts/install/install_ansible.sh") + + # write to gen_ansible.ini + ansible="[web]\n" + + # gen ssh key if not exist + if not os.path.exists("/root/.ssh/id_rsa"): + run_cmd("ssh-keygen -t rsa -b 2048") + + cluster_nodes = read_yaml(f) + appeared_node={} + for nid in cluster_nodes["nodes"]: + node=cluster_nodes["nodes"][nid] + ip=node["addr"].split(":")[0] + port=node["addr"].split(":")[1] + id=node["id"] + + if ip not in appeared_node: + ansible+="webserver{} ansible_host={} ansible_user=root\n".format(id,ip) + appeared_node[ip]=1 + + run_cmd("ssh root@{} 'apt install python'".format(ip)) + run_cmd("ssh-copy-id root@{}".format(ip)) + + # write to gen_ansible.ini + with open("scripts/deploy_cluster/gen_ansible.ini","w") as f: + f.write(ansible) + + + # with open("gen_ansible.cfg","w") as f: + # f.write( + # "[defaults]\n"+\ + # "inventory = ./gen_ansible.ini\n"+\ + # "remote_user = root\n"+\ + # "private_key_file = /root/.ssh/id_rsa\n"+\ + # "host_key_checking = False" + # ) + + # run ansible + run_cmd("ansible -i scripts/deploy_cluster/gen_ansible.ini -m ping all") + +entry() \ No newline at end of file diff --git a/scripts/deploy_cluster/2.ans_redeploy.sh b/scripts/deploy_cluster/2.ans_redeploy.sh new file mode 100644 index 0000000..4eb09c2 --- /dev/null +++ b/scripts/deploy_cluster/2.ans_redeploy.sh @@ -0,0 +1 @@ +ansible-playbook scripts/ans/ans_redeploy_cluster.yml -i scripts/deploy_cluster/gen_ansible.ini \ No newline at end of file diff --git a/docker/deploy_cluster/node_config.yaml b/scripts/deploy_cluster/node_config.yaml similarity index 100% rename from docker/deploy_cluster/node_config.yaml rename to scripts/deploy_cluster/node_config.yaml diff --git a/docker/deploy_cluster/traffic_control.sh b/scripts/deploy_cluster/traffic_control.sh similarity index 100% rename from docker/deploy_cluster/traffic_control.sh rename to scripts/deploy_cluster/traffic_control.sh diff --git a/scripts/deploy_single_node/1.ansible_setup.sh b/scripts/deploy_single_node/1.ansible_setup.sh new file mode 100644 index 0000000..8ec5c5c --- /dev/null +++ b/scripts/deploy_single_node/1.ansible_setup.sh @@ -0,0 +1,15 @@ +ssh-keygen -t ed25519 -f ~/.ssh/whatever -N '' +cat > ~/.ssh/config < ~/.ssh/authorized_keys +echo "Before fixing permissions on authorized_keys, notice home directory is world read/write" +ls -la ~/.ssh +ssh -o 'StrictHostKeyChecking no' host.example id || echo "ssh failed as expected... trying to fix permissions" +chmod og-rw ~ +echo "After fixing permissions on home folder ~ ..." +ls -la ~/.ssh +ssh -o 'StrictHostKeyChecking no' host.example id \ No newline at end of file diff --git a/docker/ci.sh b/scripts/deploy_single_node/ci.sh similarity index 100% rename from docker/ci.sh rename to scripts/deploy_single_node/ci.sh diff --git a/docker/deploy_single_node/docker-compose.yml b/scripts/deploy_single_node/docker-compose.yml similarity index 100% rename from docker/deploy_single_node/docker-compose.yml rename to scripts/deploy_single_node/docker-compose.yml diff --git a/scripts/install/install_ansible.sh b/scripts/install/install_ansible.sh new file mode 100644 index 0000000..af0730c --- /dev/null +++ b/scripts/install/install_ansible.sh @@ -0,0 +1,3 @@ +# apt install ansible -y +pip3 install ansible==4.9.0 +pip3 install jsondiff \ No newline at end of file diff --git a/scripts/install/install_rust.sh b/scripts/install/install_rust.sh new file mode 100644 index 0000000..e6a5283 --- /dev/null +++ b/scripts/install/install_rust.sh @@ -0,0 +1,3 @@ +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +source $HOME/.cargo/env +rustup target add wasm32-wasi \ No newline at end of file diff --git a/scripts/install_wasmedge.sh b/scripts/install/install_wasmedge.sh similarity index 100% rename from scripts/install_wasmedge.sh rename to scripts/install/install_wasmedge.sh diff --git a/scripts/install_docker.sh b/scripts/install_docker.sh deleted file mode 100644 index caab733..0000000 --- a/scripts/install_docker.sh +++ /dev/null @@ -1 +0,0 @@ -apt install docker.io docker-compose \ No newline at end of file diff --git a/scripts/install_protobuf.sh b/scripts/install_protobuf.sh deleted file mode 100644 index bf59c21..0000000 --- a/scripts/install_protobuf.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -sudo apt-get install protobuf-compiler \ No newline at end of file diff --git a/scripts/install_rust.sh b/scripts/install_rust.sh deleted file mode 100644 index a3731e2..0000000 --- a/scripts/install_rust.sh +++ /dev/null @@ -1,18 +0,0 @@ -apt install clang lldb lld -apt install build-essential -apt install curl - - -# if rust not installed -if ! command -v rustup &> /dev/null -then - echo "rust not installed" - # install rust - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh - # add rust to path - source "$HOME/.cargo/env" -fi - - -rustup target add wasm32-unknown-unknown -rustup default 1.70 \ No newline at end of file diff --git a/src/cmd_arg.rs b/src/cmd_arg.rs index e287a8a..7cb9558 100644 --- a/src/cmd_arg.rs +++ b/src/cmd_arg.rs @@ -9,7 +9,7 @@ pub struct CmdArgs { /// Name of the person to greet // #[arg(short, long)] pub this_id: NodeID, - pub config_file: String, + pub files_dir: String, // wrap password // pub deploy: Option, } diff --git a/src/config.rs b/src/config.rs index ec83033..2224863 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use std::{ collections::{HashMap, HashSet}, net::SocketAddr, - path::Path, + path::{Path, PathBuf}, }; use crate::sys::NodeID; @@ -11,6 +11,7 @@ use crate::sys::NodeID; pub struct NodesConfig { pub peers: HashMap, pub this: (NodeID, NodeConfig), + pub file_dir: PathBuf, } impl NodesConfig { @@ -54,10 +55,12 @@ pub fn read_yaml_config(file_path: impl AsRef) -> YamlConfig { } pub fn read_config(this_id: NodeID, file_path: impl AsRef) -> NodesConfig { - let mut yaml_config = read_yaml_config(file_path); + let config_path = file_path.as_ref().join("config/node_config.yaml"); + let mut yaml_config = read_yaml_config(config_path); NodesConfig { this: (this_id, yaml_config.nodes.remove(&this_id).unwrap()), peers: yaml_config.nodes, + file_dir: file_path.as_ref().to_path_buf(), } } diff --git a/src/main.rs b/src/main.rs index 202d6d1..056de06 100644 --- a/src/main.rs +++ b/src/main.rs @@ -46,7 +46,7 @@ async fn main() { // deploy::deploy(args).await; // return; // } - let config = config::read_config(args.this_id, args.config_file); + let config = config::read_config(args.this_id, args.files_dir); tracing::info!("config: {:?}", config); // dist_kv_raft::tikvraft_proxy::start(); Sys::new(config).wait_for_end().await; diff --git a/src/schedule/container_manager.rs b/src/schedule/container_manager.rs index 4664479..d0f9ecd 100644 --- a/src/schedule/container_manager.rs +++ b/src/schedule/container_manager.rs @@ -1,5 +1,6 @@ use std::{ collections::{HashMap, VecDeque}, + path::{Path, PathBuf}, sync::Arc, }; @@ -49,13 +50,15 @@ impl LRUCache { pub struct ContainerManager { cache: Mutex>, using_map: SkipMap>>, + file_dir: PathBuf, } impl ContainerManager { - pub fn new() -> Self { + pub fn new(file_dir: impl AsRef) -> Self { Self { cache: Mutex::new(LRUCache::new(10)), using_map: SkipMap::new(), + file_dir: file_dir.as_ref().to_owned(), } } pub async fn finish_using<'a>( @@ -92,9 +95,12 @@ impl ContainerManager { // 2. 从磁盘加载容器 else { tracing::info!("new vm"); - let module = - Module::from_file(Some(&config), format!("apps/imgs/{}.wasm", container_name)) - .unwrap(); + let module = Module::from_file( + Some(&config), + self.file_dir + .join(format!("/apps/imgs/{}.wasm", container_name)), + ) + .unwrap(); // self.cache.lock().await.put(container_name, module.clone()); ( diff --git a/src/schedule/executor.rs b/src/schedule/executor.rs index 1dad26c..af1873c 100644 --- a/src/schedule/executor.rs +++ b/src/schedule/executor.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use super::container_manager::ContainerManager; pub struct Executor { @@ -5,9 +7,9 @@ pub struct Executor { } impl Executor { - pub fn new() -> Self { + pub fn new(file_dir: impl AsRef) -> Self { Self { - container_manager: ContainerManager::new(), + container_manager: ContainerManager::new(file_dir), } } pub async fn execute(&self, req_fn: &str) { diff --git a/src/schedule/master.rs b/src/schedule/master.rs index eeb6830..50a5949 100644 --- a/src/schedule/master.rs +++ b/src/schedule/master.rs @@ -48,7 +48,7 @@ impl LogicalModule for ScheMaster { Self: Sized, { Self { - executor: Executor::new(), + executor: Executor::new(args.nodes_config.file_dir), // each_fn_caching: HashMap::new(), node_selector: Box::new(HashNodeSelector), request_handler_view: RequestHandlerView::new(args.logical_modules_ref.clone()), diff --git a/src/schedule/worker.rs b/src/schedule/worker.rs index fb16b35..ca29dc8 100644 --- a/src/schedule/worker.rs +++ b/src/schedule/worker.rs @@ -27,7 +27,7 @@ impl LogicalModule for ScheWorker { Self: Sized, { Self { - executor: Executor::new(), + executor: Executor::new(args.nodes_config.file_dir), request_handler_view: RequestHandlerView::new(args.logical_modules_ref.clone()), } }