Merge branch 'develop' into test

.github/workflows/ebpf_cpu_watcher.yml

@@ -23,11 +23,23 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt update
-          sudo apt install libbpf-dev clang llvm libelf-dev libpcap-dev gcc-multilib build-essential
+          sudo apt install -y libbpf-dev clang llvm libelf-dev libpcap-dev gcc-multilib build-essential
           git submodule update --init --recursive
 
-      - name: Run cpu_watcher
+      - name: Build cpu_watcher
         run: |
           cd eBPF_Supermarket/CPU_Subsystem/cpu_watcher/
           make 
-          sudo ./cpu_watcher 
+
+      - name: Run cpu_watcher
+        run: |
+          sudo ./eBPF_Supermarket/CPU_Subsystem/cpu_watcher/cpu_watcher
+
+      - name: Build test_cpuwatcher
+        run: |
+          cd eBPF_Supermarket/CPU_Subsystem/cpu_watcher/test
+          make
+
+      - name: Run test_cpuwatcher
+        run: |
+          ./eBPF_Supermarket/CPU_Subsystem/cpu_watcher/test/test_cpuwatcher

.github/workflows/ebpf_kvm_watcher.yml

@@ -19,10 +19,12 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
-
-      - name: Test program execution
+      - name: Install dependencies
         run: |
           cd eBPF_Supermarket/kvm_watcher/
           make deps 
-          make 
- 
+      - name: Test program execution
+        continue-on-error: true 
+        run: |
+          cd eBPF_Supermarket/kvm_watcher/
+          make 

.github/workflows/ebpf_mem_watcher.yml

@@ -32,4 +32,4 @@ jobs:
           cd eBPF_Supermarket/Memory_Subsystem/mem_watcher/
           bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
           make
-          sudo timeout 20 ./mem_watcher
+          sudo ./mem_watcher -f -i 10

.github/workflows/ebpf_net_manager.yml

@@ -36,10 +36,15 @@ jobs:
           cd eBPF_Supermarket/Network_Subsystem/net_manager/
           sudo ./configure
           sudo make
+          ifconfig
           # run
-          sudo timeout -s SIGINT 5 ./xdp_loader -d ens33 -S || if [[ $? != 124 && $? != 0 ]];then exit $?;fi
-          sudo ./xacladm load ens33 ./conf.d/mac_load.conf
-          sudo xdp-loader unload ens33 --all
+          cd testenv
+          sudo ./testenv.sh setup --name veth-basic02
+          cd ..
+          cd net_manager
+          sudo timeout -s SIGINT 5 ./xdp_loader -d eth0 -S || if [[ $? != 124 && $? != 0 ]];then exit $?;fi
+          sudo ./xdp_loader -d eth0 -S
+    
 
 
 

.github/workflows/ebpf_net_watcher.yml

@@ -45,4 +45,11 @@ jobs:
           sudo timeout -s SIGINT 5 ./netwatcher -k || if [[ $? != 124 && $? != 0 ]];then exit $?;fi
           sudo timeout -s SIGINT 5 ./netwatcher -k -T || if [[ $? != 124 && $? != 0 ]];then exit $?;fi
           sudo timeout -s SIGINT 5 ./netwatcher -I || if [[ $? != 124 && $? != 0 ]];then exit $?;fi
+          sudo timeout -s SIGINT 5 ./netwatcher -S || if [[ $? != 124 && $? != 0 ]];then exit $?;fi
+          sudo timeout -s SIGINT 5 ./netwatcher -D || if [[ $? != 124 && $? != 0 ]];then exit $?;fi
+          sudo timeout -s SIGINT 5 ./netwatcher -M || if [[ $? != 124 && $? != 0 ]];then exit $?;fi
+          sudo timeout -s SIGINT 5 ./netwatcher -R || if [[ $? != 124 && $? != 0 ]];then exit $?;fi
+          sudo timeout -s SIGINT 5 ./netwatcher -C || if [[ $? != 124 && $? != 0 ]];then exit $?;fi
+          sudo timeout -s SIGINT 5 ./netwatcher -T || if [[ $? != 124 && $? != 0 ]];then exit $?;fi
+          sudo timeout -s SIGINT 5 ./netwatcher -U || if [[ $? != 124 && $? != 0 ]];then exit $?;fi
         timeout-minutes: 5

.github/workflows/ebpf_stack_analyser.yml

@@ -23,30 +23,31 @@ jobs:
 
       - name: Install native lib dependencies
         run: |
-          git submodule update --init --recursive
+          git submodule update --init --recursive eBPF_Supermarket/lib/ MagicEyes/
           sudo apt install clang libelf1 libelf-dev zlib1g-dev
         
-      - name: Run app with native lib
+      - name: Compile test examples
+        run: |
+          cd eBPF_Supermarket/Stack_Analyser/testdir
+          gcc -o ./usdt_pthread -lpthread ./usdt_pthread.c
+          gcc -o ./uprobe_malloc ./uprobe_malloc.c
+          
+      - name: Compile and run app with native lib
         run: |
           cd eBPF_Supermarket/Stack_Analyser
-          make
-          gcc -o ./testdir/usdt_pthread ./testdir/usdt_pthread.c
-          sudo ./stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe u:pthread:pthread_create -c "./testdir/usdt_pthread" -t 5
-
-  magic-eyes-build-and-test:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
+          make -j$(nproc)
 
-      - name: Install native lib dependencies
-        run: |
-          git submodule update --init --recursive
-          sudo apt install clang libelf1 libelf-dev zlib1g-dev
+          sudo ./stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe "vfs_open" probe "t:sched:sched_switch" -d 5
+          sudo ./stack_analyzer probe "u:pthread:pthread_create" -c "./testdir/usdt_pthread" -d 5
+          sudo ./stack_analyzer probe "c:malloc" -c "./testdir/uprobe_malloc" -d 5
         
-      - name: Run app with native lib
+      - name: Compile and run MagicEyes app with native lib
         run: |
           mkdir -p MagicEyes/build
           cd MagicEyes/build
           cmake -DBUILD_STACK_ANALYZER=ON ..
-          make
-          sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe u:pthread:pthread_create -c "../../eBPF_Supermarket/Stack_Analyser/testdir/usdt_pthread" -t 5
+          make -j$(nproc)
+          
+          sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer on_cpu off_cpu memleak io readahead llc_stat probe "p::vfs_open" probe "t:sched:sched_switch" -d 5
+          sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer probe "u:pthread:pthread_create" -c "../../eBPF_Supermarket/Stack_Analyser/testdir/usdt_pthread" -d 5
+          sudo ./src/backend/system_diagnosis/stack_analyzer/stack_analyzer probe "c:malloc" -c "../../eBPF_Supermarket/Stack_Analyser/testdir/uprobe_malloc" -d 5

.gitignore

@@ -80,4 +80,4 @@ eBPF_Supermarket/CPU_Subsystem/eBPF_proc_image/controller
 # Stack_Analyser
 eBPF_Supermarket/Stack_Analyser/stack_analyzer
 eBPF_Supermarket/Stack_Analyser/exporter/exporter
-eBPF_Supermarket/Stack_Analyser/bpf_skel
+eBPF_Supermarket/Stack_Analyser/bpf_skel

.gitmodules

@@ -46,9 +46,6 @@
 [submodule "eBPF_Supermarket/CPU_Subsystem/libbpf"]
 	path = eBPF_Supermarket/CPU_Subsystem/libbpf
 	url = https://github.com/libbpf/libbpf.git
-[submodule "eBPF_Supermarket/Stack_Analyser/libbpf-bootstrap"]
-	path = eBPF_Supermarket/Stack_Analyser/libbpf-bootstrap
-	url = https://github.com/libbpf/libbpf-bootstrap.git
 [submodule "eBPF_Supermarket/Network_Subsystem/net_manager/lib/libbpf"]
 	path = eBPF_Supermarket/Network_Subsystem/net_manager/lib/libbpf
 	url = https://github.com/libbpf/libbpf.git
@@ -73,3 +70,9 @@
 [submodule "eBPF_Supermarket/Memory_Subsystem/bpftool"]
 	path = eBPF_Supermarket/Memory_Subsystem/bpftool
 	url = https://github.com/libbpf/bpftool.git
+[submodule "eBPF_Supermarket/lib/bpftool"]
+	path = eBPF_Supermarket/lib/bpftool
+	url = https://github.com/libbpf/bpftool.git
+[submodule "eBPF_Supermarket/lib/libbpf"]
+	path = eBPF_Supermarket/lib/libbpf
+	url = https://github.com/libbpf/libbpf.git

MagicEyes/src/visualization/vscode_ext/README.md

@@ -0,0 +1,125 @@
+#  lmp vscode 插件使用指南
+
+### 1. 运行效果
+
+![](./images/lmp_vscode_ext.gif)
+
+### 2. 安装与使用
+
+#### 2.1 导入插件
+
+![import_vscode_ext](./images/import_vscode_ext.png)
+
+安装成功如下：
+
+![lmp_ext_install_success](./images/lmp_ext_install_success.png)
+
+#### 2.2 设置
+
+- 启动grafana（可以在docker中启动），启动prometheus与BPF后端采集程序可以看到数据呈现
+- 设置IP地址与端口，默认端口是`localhost:3000`
+- 设置token
+
+![create_token](./images/create_token.png)
+
+> [grafana官方_创建token](https://grafana.com/docs/grafana/latest/administration/service-accounts/#create-a-service-account-in-grafana)
+
+![set_token](./images/set_token.png)
+
+- 设置可视化面板存放路径
+
+> 注意：别忘记面板路径后面加 "/"
+
+![](./images/set_panel_addr.png)
+
+面板命名必须遵循如下规则:
+
+![](./images/panel_name.png)
+
+若面板不存在，或路径，或名称不对，将出现如下错误提示：
+
+![](./images/error_info.png)
+
+- 设置工具配置文件
+
+请将配置文件路径输入在设置中，如`/home/fzy/lmp_tool_ext_config.json`，输入完成后，敲击enter键，程序将根据配置文件中的子系统与工具，生成左侧侧边栏的按钮。
+
+配置文件写法参照 `.../MagicEyes/src/visualization/vscode_ext/tool_config_sample/lmp_tool_ext_config.json`
+
+#### 2.3 如何增加工具
+
+配置文件如下。假如有一个工具，名为 mem_checker，输入 memory子系统，则 subsystem_list 不用修改，只需要在内存子系统处增加即可。如果有一个工具，名为 V4L2_tracer，属于 media子系统，则需要在subsystem_list中增加 media 子系统，并在 subsystem 下相应增加，不再赘述。
+
+```json
+{
+    "name" : "lmp_tool_vscode_extension_config",
+    "version" : "0.0.1",
+    "subsystem_list" : [
+        "CPU",      
+        "memory",
+        "fs",
+        "network",
+        "system_diagnosis",
+        "hypervisor"
+    ],
+    "subsystem" : [
+        {
+            "description" : "Linux CPU子系统观测工具集",
+            "tools" : [......]
+        },
+        {
+            "description" : "Linux 内存子系统观测工具集",
+            "tools" : [
+                {
+                    "name": "mem_watcher",
+                    "description" : "内存观测"
+                },
+        		{
+                    "name": "mem_checker",
+                    "description" : "内存检查"
+                }
+            ]
+        }
+} 
+```
+
+
+### 3. 插件开发
+
+#### 3.1 开发
+
+安装yarn并且通过`yarn install`安装所需依赖
+
+>  tips: 按 F5 开启调试
+
+#### 3.2 开发注意事项
+
+1. yo code生成的框架，vscode最小版本是1.90，需要修改为1.74，不然我当前的版本。1.89无法运行插件
+2. tsconfig
+
+```json
+{
+  "compilerOptions": {
+    "module": "commonjs",   // 不要用Node16，不然命令会触发失败
+    "target": "ES2021",
+    "lib": ["ES2021"],
+    "sourceMap": true,
+    "rootDir": "src",
+    "strict": true /* enable all strict type-checking options */
+    /* Additional Checks */
+    // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
+    // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
+    // "noUnusedParameters": true,  /* Report errors on unused parameters. */
+  }
+}
+```
+
+3.  打包vsix
+
+```bash
+# 进入插件开发文件夹
+vsce package
+```
+
+
+

MagicEyes/src/visualization/vscode_ext/lmp_ext_vscode/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+  "root": true,
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "ecmaVersion": 6,
+    "sourceType": "module"
+  },
+  "plugins": ["@typescript-eslint"],
+  "rules": {
+    "@typescript-eslint/naming-convention": "warn",
+    "@typescript-eslint/semi": "warn",
+    "curly": "warn",
+    "eqeqeq": "warn",
+    "no-throw-literal": "warn",
+    "semi": "off"
+  },
+  "ignorePatterns": ["out", "dist", "**/*.d.ts"]
+}

MagicEyes/src/visualization/vscode_ext/lmp_ext_vscode/.github/workflows/dashboards.yml

@@ -0,0 +1,39 @@
+name: Update dashboards
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - dashboards/**.json
+
+jobs:
+  update-dashboards:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Get changed dashboards
+        id: changed-files
+        uses: tj-actions/changed-files@v41
+        with:
+          files: dashboards/**.json
+
+      - name: Update changed dashboards in Grafana
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: |
+          tmp=$(mktemp)
+          for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
+            uid=$(jq -r '.uid' $file)
+            dashboardInfo=$(curl -H "Authorization: Bearer $GRAFANA_API_TOKEN" $GRAFANA_INSTANCE_URL/api/dashboards/uid/$uid)
+            currentVersion=$(echo $dashboardInfo | jq -r '.meta.version')
+            jq --argjson v $currentVersion '.version = $v' $file > $tmp && mv $tmp $file
+            dashboardJson='{"dashboard":'"$(jq -c . $file)"',"message":"'"$COMMIT_MESSAGE"'"}'
+            curl -X POST $GRAFANA_INSTANCE_URL/api/dashboards/db -H "Content-Type: application/json" -H "Authorization: Bearer $GRAFANA_API_TOKEN" -d "$dashboardJson"
+          done
+        env:
+          GRAFANA_INSTANCE_URL: ${{ secrets.GRAFANA_INSTANCE_URL }}
+          GRAFANA_API_TOKEN: ${{ secrets.GRAFANA_API_TOKEN }}
+          COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
+

MagicEyes/src/visualization/vscode_ext/lmp_ext_vscode/.github/workflows/publish.yml

@@ -0,0 +1,48 @@
+on:
+  push:
+    tags:
+      - "*"
+
+# These permissions are needed to assume roles from Github's OIDC.
+permissions:
+  contents: write
+  id-token: write
+
+name: Publish Extension
+jobs:
+  eslint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: npm install
+      - run: npm run compile
+      - run: npm run lint
+  publish:
+    needs: eslint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: npm install
+      - id: get-secrets
+        uses: grafana/shared-workflows/actions/get-vault-secrets@main
+        with:
+          # Secrets placed in the ci/repo/grafana/<repo>/<path> path in Vault
+          repo_secrets: |
+            OPEN_VSX_TOKEN=openvsx:token
+            VS_MARKETPLACE_TOKEN=vscode-marketplace:token
+      - name: Publish to Open VSX
+        uses: HaaLeo/publish-vscode-extension@v0
+        with:
+          pat: ${{ env.OPEN_VSX_TOKEN }}
+          registryUrl: https://open-vsx.org
+      - name: Publish to Visual Studio Marketplace
+        id: publishToMarketplace
+        uses: HaaLeo/publish-vscode-extension@v0
+        with:
+          pat: ${{ env.VS_MARKETPLACE_TOKEN }}
+          registryUrl: https://marketplace.visualstudio.com
+      - uses: ncipollo/release-action@v1
+        with:
+          allowUpdates: true
+          artifacts: "${{ steps.publishToMarketplace.outputs.vsixPath }}"
+          token: ${{ secrets.GITHUB_TOKEN }}