Merge pull request #114 from sunya-ch/v0.6

fix e2e test and cmd readme

.github/workflows/docker-base-image.yml

@@ -17,7 +17,7 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Build image
-      run: docker build -t quay.io/sustainable_computing_io/kepler_model_server_base:latest server/dockerfiles -f server/dockerfiles/Dockerfile.base
+      run: docker build -t quay.io/sustainable_computing_io/kepler_model_server_base:latest dockerfiles -f dockerfiles/Dockerfile.base
 
     - name: Login to Quay
       uses: docker/login-action@v1

cmd/README.md

@@ -69,6 +69,7 @@ optional arguments:
     output of trained model will be under pipeline folder `default` or can be specified by `-p`
 
     ```bash
+    <pipeline_name>.zip # archived pipeline for model server to load
     <pipeline_name> # provided by -p, --pipeline-name (default: default)
     ├── metadata.json # pipeline metadata such as pipeline name, extractor, isolator, trainer list
     ├── preprocessed_data 
@@ -84,9 +85,10 @@ optional arguments:
     │   │   │   │   ├── metadata.json # model metadata
     │   │   │   │   └── <model_files> 
     │   │   │   │   └── ...
-    │   │   │   ├── <model_name>.zip # archived model
-    ├── rapl_AbsPower_model_metadata.csv # AbsPower models summary
-    ├── rapl_DynPower_model_metadata.csv # DynPower models summary
+    │   │   │   │   └── weight.json # model weight in json format if support for kepler to load
+    │   │   │   ├── <model_name>.zip # archived model for estimator to load
+    ├── <energy_source>_AbsPower_model_metadata.csv # AbsPower models summary
+    ├── <energy_source>_DynPower_model_metadata.csv # DynPower models summary
     └── train_arguments.json
     ```
 

manifests/base/kustomization.yaml

@@ -4,11 +4,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: kepler_model_server
-  newName: localhost:5001/kepler_model_server
-  newTag: devel
+  newName: quay.io/sustainable_computing_io/kepler_model_server
+  newTag: v0.6
 
 patchesStrategicMerge:
-- ./patch/patch-estimator-sidecar.yaml
-
+- ./patch/patch-model-server.yaml
 resources:
 - ../kepler
+- ../server

tests/e2e_test.sh

@@ -18,6 +18,8 @@
 #
 
 # Get the directory of the currently executing script_
+set -ex
+
 top_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"/..
 
 echo "Top location: $top_dir"
@@ -45,6 +47,12 @@ wait_for_kepler() {
     kubectl rollout status ds kepler-exporter -n kepler --timeout 5m
 }
 
+wait_for_server() {
+    kubectl rollout status deploy kepler-model-server -n kepler --timeout 5m
+    wait_for_keyword server "initial pipeline is loaded" "server cannot load initial pipeline"
+    wait_for_keyword server "Press CTRL+C to quit" "server has not started yet"
+}
+
 wait_for_keyword() {
     num_iterations=10
     component=$1
@@ -54,20 +62,26 @@ wait_for_keyword() {
         if grep -q "$keyword" <<< $(get_${component}_log); then
             return
         fi
-        sleep 1
+        sleep 2
     done
     echo "timeout ${num_iterations}s waiting for '${keyword}' from ${component} log"
     echo "Error: $message"
 
     echo "${component} log:"
     get_${component}_log
+    # show all status
+    kubectl get po -A
     exit 1
 }
 
 check_estimator_set_and_init() {
     wait_for_keyword kepler "Model Config NODE_TOTAL: {ModelType:EstimatorSidecar" "Kepler should set desired config"
 }
 
+restart_model_server() {
+    kubectl delete po  -l app.kubernetes.io/component=model-server -n kepler
+    wait_for_server
+}
 
 test() {
     # set options
@@ -81,6 +95,9 @@ test() {
         # with estimator
         if [ ! -z ${TEST} ]; then
             kubectl patch ds kepler-exporter -n kepler --patch-file ${top_dir}/manifests/test/power-request-client.yaml
+            if [ ! -z ${SERVER} ]; then
+                restart_model_server
+            fi
             sleep 1
             wait_for_kepler
             wait_for_keyword kepler Done "cannot get power"
@@ -89,24 +106,22 @@ test() {
         fi
 
         if [ ! -z ${SERVER} ]; then
+            wait_for_server
             wait_for_keyword estimator "load model from model server" "estimator should be able to load model from server"
         fi
     else
         # no estimator
         if [ ! -z ${SERVER} ]; then
             if [ ! -z ${TEST} ]; then 
                 kubectl patch ds kepler-exporter -n kepler --patch-file ${top_dir}/manifests/test/model-request-client.yaml
+                restart_model_server
                 sleep 1
                 wait_for_kepler
                 wait_for_keyword kepler Done "cannot get model weight"
             fi
         fi
     fi
 
-    if [ ! -z ${SERVER} ]; then
-        wait_for_keyword server "initial pipeline is loaded" "server cannot load initial pipeline"
-    fi
-
 }
 
 "$@"

tests/weight_model_request_test.py

@@ -60,7 +60,11 @@
                 del request_json["system_features"]
                 del request_json["values"]
                 del request_json["system_values"]
-                response = requests.post(get_model_server_req_endpoint(), json=request_json)
+                try:
+                    response = requests.post(get_model_server_req_endpoint(), json=request_json)
+                except Exception as err:
+                    print("cannot get response from model server: {}".format(err))
+                    sys.exit(1) 
                 assert response.status_code == 200, "response {} not OK".format(request_json)
                 loaded_weight = json.loads(response.content)
                 print(loaded_weight)