add verify for ec2 hosted runner and kepler_model_server case

Signed-off-by: Sam Yuan <[email protected]>

.github/workflows/test.yml

@@ -4,6 +4,8 @@ on:
   # we can run mannually to see if it can be fixed by update version of
   # - kubectl
   # - k8s cluster provider's 
+  schedule:
+    - cron: "27 4 * * *"
   workflow_dispatch:
   pull_request:
   push:

lib/prometheus.sh

@@ -45,7 +45,6 @@ deploy_prometheus_operator() {
 			kube-prometheus/manifests/prometheus-prometheus.yaml
 
 		_setup_dashboard
-		_run_yq
 		_load_prometheus_operator_images_to_local_registry
 		kubectl create -f kube-prometheus/manifests/setup
 		kubectl wait \
@@ -137,11 +136,3 @@ metadata:
 EOF
     fi
 }
-
-_run_yq(){
-	f="$DASHBOARD_DIR/grafana-dashboards/kepler-exporter-configmap.yaml" \
-	yq -i e '.items += [load(env(f))]' "$KUBE_PROM_DIR"/manifests/grafana-dashboardDefinitions.yaml;
-	yq -i e '.spec.template.spec.containers.0.volumeMounts += [ {"mountPath": "/grafana-dashboard-definitions/0/kepler-exporter", "name": "grafana-dashboard-kepler-exporter", "readOnly": false} ]' "$KUBE_PROM_DIR"/manifests/grafana-deployment.yaml
-	yq -i e '.spec.template.spec.volumes += [ {"configMap": {"name": "grafana-dashboard-kepler-exporter"}, "name": "grafana-dashboard-kepler-exporter"} ]' "$KUBE_PROM_DIR"/manifests/grafana-deployment.yaml;
-	ok "Dashboard setup complete"
-}

verify.sh

@@ -21,6 +21,8 @@ set -eu -o pipefail
 PROJECT_ROOT="$(git rev-parse --show-toplevel)"
 declare -r PROJECT_ROOT
 declare -r KUBECONFIG_ROOT_DIR=${KUBECONFIG_ROOT_DIR:-$PROJECT_ROOT/.kube}
+declare -r KUBE_PROM_DIR="$PROJECT_ROOT/tmp/kube-prometheus"
+declare -r DASHBOARD_DIR="$KUBE_PROM_DIR/grafana-dashboards"
 # shellcheck source=lib/utils.sh
 source "$PROJECT_ROOT/lib/utils.sh"
 
@@ -58,6 +60,17 @@ verify_cluster() {
 		rollout_status "$res" "$NAMESPACE"
 	done
 
+	# verify kubeconf for ec2_hosted_runner usage
+	info "Verifying cluster status for CI usage"
+	cp .kube/config /tmp/kubeconfig
+	export KUBECONFIG="/tmp/kubeconfig"
+	run kubectl cluster-info || die "failed to get the cluster-info"
+
+	# check k8s system pod is there...
+	[[ $(kubectl get pods --all-namespaces | wc -l) == 0 ]] &&
+		die "it seems k8s cluster is not started"
+
+	run_yq
 	ok "Cluster is up and running"
 }
 
@@ -67,6 +80,15 @@ containerruntime() {
 	docker info || true
 }
 
+
+run_yq(){
+	f="$DASHBOARD_DIR/grafana-dashboards/kepler-exporter-configmap.yaml" \
+	yq -i e '.items += [load(env(f))]' "$KUBE_PROM_DIR"/manifests/grafana-dashboardDefinitions.yaml;
+	yq -i e '.spec.template.spec.containers.0.volumeMounts += [ {"mountPath": "/grafana-dashboard-definitions/0/kepler-exporter", "name": "grafana-dashboard-kepler-exporter", "readOnly": false} ]' "$KUBE_PROM_DIR"/manifests/grafana-deployment.yaml
+	yq -i e '.spec.template.spec.volumes += [ {"configMap": {"name": "grafana-dashboard-kepler-exporter"}, "name": "grafana-dashboard-kepler-exporter"} ]' "$KUBE_PROM_DIR"/manifests/grafana-deployment.yaml;
+	ok "Dashboard setup complete"
+}
+
 main() {
 	# verify the deployment of cluster
 	case $1 in