diff --git a/.github/workflows/docker-base-image.yml b/.github/workflows/docker-base-image.yml index b402f1fa..47ec2d02 100644 --- a/.github/workflows/docker-base-image.yml +++ b/.github/workflows/docker-base-image.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v3 - name: Build image - run: docker build -t quay.io/sustainable_computing_io/kepler_model_server_base:latest server/dockerfiles -f server/dockerfiles/Dockerfile.base + run: docker build -t quay.io/sustainable_computing_io/kepler_model_server_base:latest dockerfiles -f dockerfiles/Dockerfile.base - name: Login to Quay uses: docker/login-action@v1 diff --git a/cmd/README.md b/cmd/README.md index aa751e27..010d6baf 100644 --- a/cmd/README.md +++ b/cmd/README.md @@ -69,6 +69,7 @@ optional arguments: output of trained model will be under pipeline folder `default` or can be specified by `-p` ```bash + .zip # archived pipeline for model server to load # provided by -p, --pipeline-name (default: default) ├── metadata.json # pipeline metadata such as pipeline name, extractor, isolator, trainer list ├── preprocessed_data @@ -84,9 +85,10 @@ optional arguments: │   │   │   │   ├── metadata.json # model metadata │   │   │   │   └── │   │   │   │   └── ... - │   │   │   ├── .zip # archived model - ├── rapl_AbsPower_model_metadata.csv # AbsPower models summary - ├── rapl_DynPower_model_metadata.csv # DynPower models summary + │   │   │   │   └── weight.json # model weight in json format if support for kepler to load + │   │   │   ├── .zip # archived model for estimator to load + ├── _AbsPower_model_metadata.csv # AbsPower models summary + ├── _DynPower_model_metadata.csv # DynPower models summary └── train_arguments.json ``` diff --git a/manifests/base/kustomization.yaml b/manifests/base/kustomization.yaml index 92259056..76d75107 100644 --- a/manifests/base/kustomization.yaml +++ b/manifests/base/kustomization.yaml @@ -4,11 +4,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: kepler_model_server - newName: localhost:5001/kepler_model_server - newTag: devel + newName: quay.io/sustainable_computing_io/kepler_model_server + newTag: v0.6 patchesStrategicMerge: -- ./patch/patch-estimator-sidecar.yaml - +- ./patch/patch-model-server.yaml resources: - ../kepler +- ../server diff --git a/tests/e2e_test.sh b/tests/e2e_test.sh index 85fe0931..9d857cbe 100755 --- a/tests/e2e_test.sh +++ b/tests/e2e_test.sh @@ -18,6 +18,8 @@ # # Get the directory of the currently executing script_ +set -ex + top_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"/.. echo "Top location: $top_dir" @@ -45,6 +47,12 @@ wait_for_kepler() { kubectl rollout status ds kepler-exporter -n kepler --timeout 5m } +wait_for_server() { + kubectl rollout status deploy kepler-model-server -n kepler --timeout 5m + wait_for_keyword server "initial pipeline is loaded" "server cannot load initial pipeline" + wait_for_keyword server "Press CTRL+C to quit" "server has not started yet" +} + wait_for_keyword() { num_iterations=10 component=$1 @@ -54,13 +62,15 @@ wait_for_keyword() { if grep -q "$keyword" <<< $(get_${component}_log); then return fi - sleep 1 + sleep 2 done echo "timeout ${num_iterations}s waiting for '${keyword}' from ${component} log" echo "Error: $message" echo "${component} log:" get_${component}_log + # show all status + kubectl get po -A exit 1 } @@ -68,6 +78,10 @@ check_estimator_set_and_init() { wait_for_keyword kepler "Model Config NODE_TOTAL: {ModelType:EstimatorSidecar" "Kepler should set desired config" } +restart_model_server() { + kubectl delete po -l app.kubernetes.io/component=model-server -n kepler + wait_for_server +} test() { # set options @@ -81,6 +95,9 @@ test() { # with estimator if [ ! -z ${TEST} ]; then kubectl patch ds kepler-exporter -n kepler --patch-file ${top_dir}/manifests/test/power-request-client.yaml + if [ ! -z ${SERVER} ]; then + restart_model_server + fi sleep 1 wait_for_kepler wait_for_keyword kepler Done "cannot get power" @@ -89,6 +106,7 @@ test() { fi if [ ! -z ${SERVER} ]; then + wait_for_server wait_for_keyword estimator "load model from model server" "estimator should be able to load model from server" fi else @@ -96,6 +114,7 @@ test() { if [ ! -z ${SERVER} ]; then if [ ! -z ${TEST} ]; then kubectl patch ds kepler-exporter -n kepler --patch-file ${top_dir}/manifests/test/model-request-client.yaml + restart_model_server sleep 1 wait_for_kepler wait_for_keyword kepler Done "cannot get model weight" @@ -103,10 +122,6 @@ test() { fi fi - if [ ! -z ${SERVER} ]; then - wait_for_keyword server "initial pipeline is loaded" "server cannot load initial pipeline" - fi - } "$@" \ No newline at end of file diff --git a/tests/weight_model_request_test.py b/tests/weight_model_request_test.py index e3d0f87d..33616c03 100644 --- a/tests/weight_model_request_test.py +++ b/tests/weight_model_request_test.py @@ -60,7 +60,11 @@ del request_json["system_features"] del request_json["values"] del request_json["system_values"] - response = requests.post(get_model_server_req_endpoint(), json=request_json) + try: + response = requests.post(get_model_server_req_endpoint(), json=request_json) + except Exception as err: + print("cannot get response from model server: {}".format(err)) + sys.exit(1) assert response.status_code == 200, "response {} not OK".format(request_json) loaded_weight = json.loads(response.content) print(loaded_weight) \ No newline at end of file