add missing energy source on local model weight

Signed-off-by: Sunyanan Choochotkaew <[email protected]>

build/Dockerfile

@@ -32,8 +32,10 @@ RUN mkdir -p /var/lib/kepler/data
 COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/normalized_cpu_arch.csv /var/lib/kepler/data/normalized_cpu_arch.csv
 
 # copy model weight
-COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/AbsPowerModel.json /var/lib/kepler/data/AbsPowerModel.json
-COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/DynPowerModel.json /var/lib/kepler/data/DynPowerModel.json
+COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/acpi_AbsPowerModel.json /var/lib/kepler/data/acpi_AbsPowerModel.json
+COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/acpi_DynPowerModel.json /var/lib/kepler/data/acpi_DynPowerModel.json
+COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/rapl_AbsPowerModel.json /var/lib/kepler/data/rapl_AbsPowerModel.json
+COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/rapl_DynPowerModel.json /var/lib/kepler/data/rapl_DynPowerModel.json
 
 # pre install kernel sources
 RUN mkdir -p /usr/share/kepler/kernel_sources

build/Dockerfile.bcc.kepler

@@ -29,8 +29,10 @@ RUN mkdir -p /var/lib/kepler/data
 COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/normalized_cpu_arch.csv /var/lib/kepler/data/normalized_cpu_arch.csv
 
 # copy model weight
-COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/AbsPowerModel.json /var/lib/kepler/data/AbsPowerModel.json
-COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/DynPowerModel.json /var/lib/kepler/data/DynPowerModel.json
+COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/acpi_AbsPowerModel.json /var/lib/kepler/data/acpi_AbsPowerModel.json
+COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/acpi_DynPowerModel.json /var/lib/kepler/data/acpi_DynPowerModel.json
+COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/rapl_AbsPowerModel.json /var/lib/kepler/data/rapl_AbsPowerModel.json
+COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/rapl_DynPowerModel.json /var/lib/kepler/data/rapl_DynPowerModel.json
 
 # pre install kernel sources
 RUN mkdir -p /usr/share/kepler/kernel_sources

build/Dockerfile.libbpf.kepler

@@ -33,7 +33,8 @@ COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler
 COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/bpfassets/libbpf/bpf.o /var/lib/kepler/bpfassets
 
 # copy model weight
-COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/AbsPowerModel.json /var/lib/kepler/data/AbsPowerModel.json
-COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/DynPowerModel.json /var/lib/kepler/data/DynPowerModel.json
-
+COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/acpi_AbsPowerModel.json /var/lib/kepler/data/acpi_AbsPowerModel.json
+COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/acpi_DynPowerModel.json /var/lib/kepler/data/acpi_DynPowerModel.json
+COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/rapl_AbsPowerModel.json /var/lib/kepler/data/rapl_AbsPowerModel.json
+COPY --from=builder /opt/app-root/src/github.com/sustainable-computing-io/kepler/data/model_weight/rapl_DynPowerModel.json /var/lib/kepler/data/rapl_DynPowerModel.json
 ENTRYPOINT ["/usr/bin/kepler"]

data/model_weight/acpi_AbsPowerModel.json

@@ -0,0 +1 @@
+{"platform": {"All_Weights": {"Bias_Weight": 4067.51980453514, "Categorical_Variables": {}, "Numerical_Variables": {"kubelet_cpu_usage": {"mean": 8.225074882327771, "variance": 61.882611238231696, "weight": 128.77011809213587}, "kubelet_memory_bytes": {"mean": 9011536.046098989, "variance": 437786422376270.4, "weight": 11.4818854701994}}}}}

data/model_weight/acpi_DynPowerModel.json

@@ -0,0 +1 @@
+{"platform": {"All_Weights": {"Bias_Weight": 3427.8961151990857, "Categorical_Variables": {}, "Numerical_Variables": {"kubelet_cpu_usage": {"mean": 0.3086623354989186, "variance": 0.11882551366884181, "weight": 141.0349581850243}, "kubelet_memory_bytes": {"mean": 274579.5528902216, "variance": 2763989623111.006, "weight": 4.932254845439169}}}}}

doc/dev/prepare_dev_env.sh

@@ -8,5 +8,7 @@ mkdir -p ${DATAPATH}
 cp ../data/normalized_cpu_arch.csv ${DATAPATH}
 
 mkdir -p /var/lib/kepler/data/
-cp ../data/model_weight/AbsPowerModel.json ${DATAPATH}/AbsPowerModel.json
-cp ../data/model_weight/DynPowerModel.json ${DATAPATH}/DynPowerModel.json
+cp ../data/model_weight/acpi_AbsPowerModel.json ${DATAPATH}/acpi_AbsPowerModel.json
+cp ../data/model_weight/acpi_DynPowerModel.json ${DATAPATH}/acpi_DynPowerModel.json
+cp ../data/model_weight/rapl_AbsPowerModel.json ${DATAPATH}/rapl_AbsPowerModel.json
+cp ../data/model_weight/rapl_DynPowerModel.json ${DATAPATH}/rapl_DynPowerModel.json

pkg/config/config.go

@@ -130,11 +130,14 @@ var (
 	// KubeConfig is used to start k8s client with the pod running outside the cluster
 	KubeConfig      = ""
 	EnableAPIServer = false
-
-	DefaultDynPowerURL = "/var/lib/kepler/data/DynPowerModel.json"
-	DefaultAbsPowerURL = "/var/lib/kepler/data/AbsPowerModel.json"
 )
 
+// return local path to power model weight
+// e.g., /var/lib/kepler/data/acpi_AbsPowerModel.json
+func GetDefaultPowerModelURL(modelOutputType, energySource string) string {
+	return fmt.Sprintf("/var/lib/kepler/data/%s_%sModel.json", energySource, modelOutputType)
+}
+
 func logBoolConfigs() {
 	if klog.V(5).Enabled() {
 		klog.V(5).Infof("ENABLE_EBPF_CGROUPID: %t", EnabledEBPFCgroupID)

pkg/model/container_energy.go

@@ -35,13 +35,13 @@ var (
 )
 
 // createContainerPowerModelConfig: the container component power model must be set by default.
-func createContainerPowerModelConfig(powerSourceTarget string, containerFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues []string, defaultURL string) (modelConfig *types.ModelConfig) {
+func createContainerPowerModelConfig(powerSourceTarget string, containerFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues []string, energySource string) (modelConfig *types.ModelConfig) {
 	modelConfig = CreatePowerModelConfig(powerSourceTarget)
 	if modelConfig == nil {
 		return nil
 	}
 	if modelConfig.InitModelURL == "" {
-		modelConfig.InitModelURL = defaultURL
+		modelConfig.InitModelFilepath = config.GetDefaultPowerModelURL(modelConfig.ModelOutputType.String(), energySource)
 	}
 	modelConfig.ContainerFeatureNames = containerFeatureNames
 	modelConfig.SystemMetaDataFeatureNames = systemMetaDataFeatureNames
@@ -102,7 +102,7 @@ func createContainerPowerModelConfig(powerSourceTarget string, containerFeatureN
 
 func CreateContainerPowerEstimatorModel(containerFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues []string) {
 	var err error
-	modelConfig := createContainerPowerModelConfig(config.ContainerPlatformPowerKey, containerFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues, config.DefaultDynPowerURL)
+	modelConfig := createContainerPowerModelConfig(config.ContainerPlatformPowerKey, containerFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues, types.PlatformEnergySource)
 	modelConfig.IsNodePowerModel = false
 	ContainerPlatformPowerModel, err = createPowerModelEstimator(modelConfig)
 	if err == nil {
@@ -111,7 +111,7 @@ func CreateContainerPowerEstimatorModel(containerFeatureNames, systemMetaDataFea
 		klog.Infof("Failed to create %s Power Model to estimate Container Platform Power: %v\n", modelConfig.ModelType.String()+"/"+modelConfig.ModelOutputType.String(), err)
 	}
 
-	modelConfig = createContainerPowerModelConfig(config.ContainerComponentsPowerKey, containerFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues, config.DefaultDynPowerURL)
+	modelConfig = createContainerPowerModelConfig(config.ContainerComponentsPowerKey, containerFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues, types.ComponentEnergySource)
 	modelConfig.IsNodePowerModel = false
 	ContainerComponentPowerModel, err = createPowerModelEstimator(modelConfig)
 	if err == nil {

pkg/model/estimator/local/lr.go

@@ -148,12 +148,13 @@ type ComponentModelWeights map[string]ModelWeights
 
 // LinearRegressor defines power estimator with linear regression approach
 type LinearRegressor struct {
-	ModelServerEndpoint string
-	OutputType          types.ModelOutputType
-	EnergySource        string
-	TrainerName         string
-	SelectFilter        string
-	ModelWeightsURL     string
+	ModelServerEndpoint  string
+	OutputType           types.ModelOutputType
+	EnergySource         string
+	TrainerName          string
+	SelectFilter         string
+	ModelWeightsURL      string
+	ModelWeightsFilepath string
 
 	FloatFeatureNames           []string
 	SystemMetaDataFeatureNames  []string
@@ -265,7 +266,7 @@ func (r *LinearRegressor) loadWeightFromURLorLocal() (*ComponentModelWeights, er
 
 // loadWeightFromLocal tries loading weights from local file given by r.ModelWeightsURL
 func (r *LinearRegressor) loadWeightFromLocal() ([]byte, error) {
-	data, err := os.ReadFile(config.DefaultDynPowerURL)
+	data, err := os.ReadFile(r.ModelWeightsFilepath)
 	if err != nil {
 		return nil, err
 	}

pkg/model/estimator/local/lr_test.go

@@ -115,7 +115,7 @@ func getDummyWeights(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func genLinearRegressor(outputType types.ModelOutputType, energySource, modelServerEndpoint, modelWeightsURL string) LinearRegressor {
+func genLinearRegressor(outputType types.ModelOutputType, energySource, modelServerEndpoint, modelWeightsURL, modelWeightFilepath string) LinearRegressor {
 	config.ModelServerEnable = true
 	config.ModelServerEndpoint = modelServerEndpoint
 	return LinearRegressor{
@@ -126,14 +126,16 @@ func genLinearRegressor(outputType types.ModelOutputType, energySource, modelSer
 		SystemMetaDataFeatureNames:  systemMetaDataFeatureNames,
 		SystemMetaDataFeatureValues: systemMetaDataFeatureValues,
 		ModelWeightsURL:             modelWeightsURL,
+		ModelWeightsFilepath:        modelWeightFilepath,
 	}
 }
 
 var _ = Describe("Test LR Weight Unit", func() {
 	Context("with dummy model server", func() {
 		It("Get Node Platform Power By Linear Regression with ModelServerEndpoint", func() {
 			testServer := httptest.NewServer(http.HandlerFunc(getDummyWeights))
-			r := genLinearRegressor(types.AbsPower, types.PlatformEnergySource, testServer.URL, "")
+			modelWeightFilepath := config.GetDefaultPowerModelURL(types.AbsPower.String(), types.PlatformEnergySource)
+			r := genLinearRegressor(types.AbsPower, types.PlatformEnergySource, testServer.URL, "", modelWeightFilepath)
 			err := r.Start()
 			Expect(err).To(BeNil())
 			r.ResetSampleIdx()
@@ -147,7 +149,8 @@ var _ = Describe("Test LR Weight Unit", func() {
 
 		It("Get Node Components Power By Linear Regression Estimator with ModelServerEndpoint", func() {
 			testServer := httptest.NewServer(http.HandlerFunc(getDummyWeights))
-			r := genLinearRegressor(types.AbsPower, types.ComponentEnergySource, testServer.URL, "")
+			modelWeightFilepath := config.GetDefaultPowerModelURL(types.AbsPower.String(), types.ComponentEnergySource)
+			r := genLinearRegressor(types.AbsPower, types.ComponentEnergySource, testServer.URL, "", modelWeightFilepath)
 			err := r.Start()
 			Expect(err).To(BeNil())
 			r.ResetSampleIdx()
@@ -161,7 +164,8 @@ var _ = Describe("Test LR Weight Unit", func() {
 
 		It("Get Container Platform Power By Linear Regression Estimator with ModelServerEndpoint", func() {
 			testServer := httptest.NewServer(http.HandlerFunc(getDummyWeights))
-			r := genLinearRegressor(types.DynPower, types.PlatformEnergySource, testServer.URL, "")
+			modelWeightFilepath := config.GetDefaultPowerModelURL(types.DynPower.String(), types.PlatformEnergySource)
+			r := genLinearRegressor(types.DynPower, types.PlatformEnergySource, testServer.URL, "", modelWeightFilepath)
 			err := r.Start()
 			Expect(err).To(BeNil())
 			r.ResetSampleIdx()
@@ -177,7 +181,8 @@ var _ = Describe("Test LR Weight Unit", func() {
 
 		It("Get Container Components Power By Linear Regression Estimator with ModelServerEndpoint", func() {
 			testServer := httptest.NewServer(http.HandlerFunc(getDummyWeights))
-			r := genLinearRegressor(types.DynPower, types.ComponentEnergySource, testServer.URL, "")
+			modelWeightFilepath := config.GetDefaultPowerModelURL(types.DynPower.String(), types.ComponentEnergySource)
+			r := genLinearRegressor(types.DynPower, types.ComponentEnergySource, testServer.URL, "", modelWeightFilepath)
 			err := r.Start()
 			Expect(err).To(BeNil())
 			r.ResetSampleIdx()
@@ -195,8 +200,9 @@ var _ = Describe("Test LR Weight Unit", func() {
 	Context("without model server", func() {
 		It("Get Node Components Power By Linear Regression Estimator without ModelServerEndpoint", func() {
 			/// Estimate Node Components Absolute Power using Linear Regression
-			initModelURL := "https://raw.githubusercontent.com/sustainable-computing-io/kepler-model-server/test_models/tests/test_models/AbsComponentModelWeight/Full/KerasCompWeightFullPipeline/KerasCompWeightFullPipeline.json"
-			r := genLinearRegressor(types.AbsPower, types.ComponentEnergySource, "", initModelURL)
+			modelWeightFilepath := config.GetDefaultPowerModelURL(types.AbsPower.String(), types.ComponentEnergySource)
+			initModelURL := "https://raw.githubusercontent.com/sustainable-computing-io/kepler-model-db/main/models/Linux-4.15.0-213-generic-x86_64_v0.6/rapl/AbsPower/KubeletOnly/weight.json"
+			r := genLinearRegressor(types.AbsPower, types.ComponentEnergySource, "", initModelURL, modelWeightFilepath)
 			err := r.Start()
 			Expect(err).To(BeNil())
 			r.ResetSampleIdx()
@@ -207,8 +213,9 @@ var _ = Describe("Test LR Weight Unit", func() {
 
 		It("Get Container Components Power By Linear Regression Estimator without ModelServerEndpoint", func() {
 			// Estimate Container Components Absolute Power using Linear Regression
-			initModelURL := "https://raw.githubusercontent.com/sustainable-computing-io/kepler-model-server/test_models/tests/test_models/DynComponentModelWeight/CgroupOnly/ScikitMixed/ScikitMixed.json"
-			r := genLinearRegressor(types.DynPower, types.ComponentEnergySource, "", initModelURL)
+			modelWeightFilepath := config.GetDefaultPowerModelURL(types.DynPower.String(), types.ComponentEnergySource)
+			initModelURL := "https://raw.githubusercontent.com/sustainable-computing-io/kepler-model-db/main/models/Linux-4.15.0-213-generic-x86_64_v0.6/rapl/DynPower/KubeletOnly/weight.json"
+			r := genLinearRegressor(types.DynPower, types.ComponentEnergySource, "", initModelURL, modelWeightFilepath)
 			err := r.Start()
 			Expect(err).To(BeNil())
 			r.ResetSampleIdx()

pkg/model/model.go

@@ -108,6 +108,7 @@ func createPowerModelEstimator(modelConfig *types.ModelConfig) (PowerMoldelInter
 			TrainerName:                 modelConfig.TrainerName,
 			SelectFilter:                modelConfig.SelectFilter,
 			ModelWeightsURL:             modelConfig.InitModelURL,
+			ModelWeightsFilepath:        modelConfig.InitModelFilepath,
 			FloatFeatureNames:           featuresNames,
 			SystemMetaDataFeatureNames:  modelConfig.SystemMetaDataFeatureNames,
 			SystemMetaDataFeatureValues: modelConfig.SystemMetaDataFeatureValues,

pkg/model/node_component_energy.go

@@ -28,15 +28,13 @@ import (
 var (
 	// the absulute power model includes both the absolute and idle power consumption
 	NodeComponentPowerModel PowerMoldelInterface
-
-	defaultAbsCompURL = "/var/lib/kepler/data/AbsPowerModel.json"
 )
 
 // createNodeComponentPowerModelConfig: the node component power model url must be set by default.
 func createNodeComponentPowerModelConfig(nodeFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues []string) *types.ModelConfig {
 	modelConfig := CreatePowerModelConfig(config.NodeComponentsPowerKey)
 	if modelConfig.InitModelURL == "" {
-		modelConfig.InitModelURL = defaultAbsCompURL
+		modelConfig.InitModelFilepath = config.GetDefaultPowerModelURL(modelConfig.ModelOutputType.String(), types.ComponentEnergySource)
 	}
 	modelConfig.NodeFeatureNames = nodeFeatureNames
 	modelConfig.SystemMetaDataFeatureNames = systemMetaDataFeatureNames

pkg/model/node_platform_energy.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 
 	"github.com/sustainable-computing-io/kepler/pkg/config"
+	"github.com/sustainable-computing-io/kepler/pkg/model/types"
 	"github.com/sustainable-computing-io/kepler/pkg/power/platform"
 	"k8s.io/klog/v2"
 
@@ -39,7 +40,7 @@ func CreateNodePlatformPoweEstimatorModel(nodeFeatureNames, systemMetaDataFeatur
 	if !platform.IsSystemCollectionSupported() {
 		modelConfig := CreatePowerModelConfig(config.NodePlatformPowerKey)
 		if modelConfig.InitModelURL == "" {
-			modelConfig.InitModelURL = defaultAbsCompURL
+			modelConfig.InitModelFilepath = config.GetDefaultPowerModelURL(modelConfig.ModelOutputType.String(), types.PlatformEnergySource)
 		}
 		modelConfig.NodeFeatureNames = nodeFeatureNames
 		modelConfig.SystemMetaDataFeatureNames = systemMetaDataFeatureNames

pkg/model/process_power.go

@@ -37,10 +37,10 @@ var (
 )
 
 // createProcessPowerModelConfig: the process component power model must be set by default.
-func createProcessPowerModelConfig(powerSourceTarget string, processFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues []string, defaultURL string) *types.ModelConfig {
+func createProcessPowerModelConfig(powerSourceTarget string, processFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues []string, energySource string) *types.ModelConfig {
 	modelConfig := CreatePowerModelConfig(powerSourceTarget)
 	if modelConfig.InitModelURL == "" {
-		modelConfig.InitModelURL = defaultURL
+		modelConfig.InitModelFilepath = config.GetDefaultPowerModelURL(modelConfig.ModelOutputType.String(), energySource)
 	}
 	modelConfig.ContainerFeatureNames = processFeatureNames
 	modelConfig.SystemMetaDataFeatureNames = systemMetaDataFeatureNames
@@ -101,7 +101,7 @@ func createProcessPowerModelConfig(powerSourceTarget string, processFeatureNames
 
 func CreateProcessPowerEstimatorModel(processFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues []string) {
 	var err error
-	modelConfig := createProcessPowerModelConfig(config.ProcessPlatformPowerKey, processFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues, config.DefaultDynPowerURL)
+	modelConfig := createProcessPowerModelConfig(config.ProcessPlatformPowerKey, processFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues, types.PlatformEnergySource)
 	modelConfig.IsNodePowerModel = false
 	ProcessPlatformPowerModel, err = createPowerModelEstimator(modelConfig)
 	if err == nil {
@@ -110,7 +110,7 @@ func CreateProcessPowerEstimatorModel(processFeatureNames, systemMetaDataFeature
 		klog.Infof("Failed to create %s Power Model to estimate Process Platform Power: %v\n", modelConfig.ModelType.String()+"/"+modelConfig.ModelOutputType.String(), err)
 	}
 
-	modelConfig = createProcessPowerModelConfig(config.ProcessComponentsPowerKey, processFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues, config.DefaultDynPowerURL)
+	modelConfig = createProcessPowerModelConfig(config.ProcessComponentsPowerKey, processFeatureNames, systemMetaDataFeatureNames, systemMetaDataFeatureValues, types.ComponentEnergySource)
 	modelConfig.IsNodePowerModel = false
 	ProcessComponentPowerModel, err = createPowerModelEstimator(modelConfig)
 	if err == nil {