redhat 8.5 with later kernel build the drbd module in drbd-adapter job failed

[san3Xian]

redhat 8.5 with later kernel cant not build the drbd module in drbd-adapter job

kubectl get deployments.apps -owide -n hwameistor hwameistor-operator
NAME                  READY   UP-TO-DATE   AVAILABLE   AGE   CONTAINERS            IMAGES                                                         SELECTOR
hwameistor-operator   1/1     1            1           36d   hwameistor-operator   10.101.210.43/ghcr.m.daocloud.io/hwameistor/operator:v0.14.9   app=hwameistor-operator

log of shipper container:

kubectl logs -n hwameistor drbd-adapter-hostname-rhel8-1725963185-mn2n2 -c shipper
'/files/Dockerfile.shipper' -> '/pkgs/Dockerfile.shipper'
'/files/drbd-debs' -> '/pkgs/drbd-debs'
'/files/drbd-reactor.toml' -> '/pkgs/drbd-reactor.toml'
'/files/drbd-rpms' -> '/pkgs/drbd-rpms'
'/files/drbd.conf' -> '/pkgs/drbd.conf'
'/files/drbd.modules' -> '/pkgs/drbd.modules'
'/files/drbd.modules-load.conf' -> '/pkgs/drbd.modules-load.conf'
'/files/entrypoint.adapter.sh' -> '/pkgs/entrypoint.adapter.sh'
'/files/global_common.conf' -> '/pkgs/global_common.conf'
'/files/install_opam.sh' -> '/pkgs/install_opam.sh'
'/files/kernel-devels.centos7' -> '/pkgs/kernel-devels.centos7'
'/files/kernel-devels.centos8' -> '/pkgs/kernel-devels.centos8'
'/files/rhel7.0' -> '/pkgs/rhel7.0'
'/files/rhel7.1' -> '/pkgs/rhel7.1'
'/files/rhel7.2' -> '/pkgs/rhel7.2'
'/files/rhel7.3' -> '/pkgs/rhel7.3'
'/files/rhel7.4' -> '/pkgs/rhel7.4'
'/files/rhel7.5' -> '/pkgs/rhel7.5'
'/files/rhel7.6' -> '/pkgs/rhel7.6'
'/files/rhel7.7' -> '/pkgs/rhel7.7'
'/files/rhel7.8' -> '/pkgs/rhel7.8'
'/files/rhel7.9' -> '/pkgs/rhel7.9'
'/files/rhel8.0' -> '/pkgs/rhel8.0'
'/files/rhel8.1' -> '/pkgs/rhel8.1'
'/files/rhel8.2' -> '/pkgs/rhel8.2'
'/files/rhel8.3' -> '/pkgs/rhel8.3'
'/files/rhel8.4' -> '/pkgs/rhel8.4'
'/files/rhel8.5' -> '/pkgs/rhel8.5'
'/files/rhel8.6' -> '/pkgs/rhel8.6'
'/files/utils' -> '/pkgs/utils'

log of rhel8 container:

[root@mitqadaocappn02 ~]# kubectl logs -n hwameistor drbd-adapter-hostname-rhel8-1725963185-mn2n2 -c rhel8
+ which lbdisttool.py
/usr/local/bin/lbdisttool.py
++ lbdisttool.py -l
++ awk -F. '{print $1}'
+ image_dist=rhel8
++ lbdisttool.py -l --os-release /etc/host-release
++ awk -F. '{print $1}'
+ host_dist=rhel8
+ '[' -z rhel8 ']'
+ '[' -z rhel8 ']'
+ [[ rhel8 != rhel8 ]]
++ cat /proc/drbd
++ awk '/^version:/ {print $2}'
cat: /proc/drbd: No such file or directory
+ RUNNING_DRBD_VERSION=
+ '[' -z ']'
+ echo 'No DRBD Module is loaded'
+ echo OS_KERNEL:4.18.0-477.10.1.el8_8.x86_64
+ export DRBD_EXIST=no
+ DRBD_EXIST=no
No DRBD Module is loaded
OS_KERNEL:4.18.0-477.10.1.el8_8.x86_64
+ export DRBD_RMP_INSTALL=no
+ DRBD_RMP_INSTALL=no
+ export OS_KERNEL=4.18.0-477.10.1.el8_8.x86_64
+ OS_KERNEL=4.18.0-477.10.1.el8_8.x86_64
+ export OS_HOST_DIST=rhel8
+ OS_HOST_DIST=rhel8
+ [[ rhel8 == \k\y\l\i\n\1\0 ]]
+ LB_HOW=shipped_modules
+ [[ shipped_modules == \s\h\i\p\p\e\d\_\m\o\d\u\l\e\s ]]
+ bash -x /entry.sh
+ SIGN_KEY=https://packages.linbit.com/package-signing-pubkey.asc
+ PKGS=/pkgs
+ HOSTRELEASE=/etc/host-release
+ RPMDIR=/pkgs/drbd-rpms/
+ DEBDIR=/pkgs/drbd-debs/
+ HOW_DEPSONLY=deps_only
+ HOW_REPOFILE=repo_file
+ HOW_HASH=node_hash
+ HOW_FROMSRC=compile
+ HOW_FROMSHIPPED=shipped_modules
+ HOW_LOAD_FROM_RAM=RAM
+ HOW_INSTALL=install
+ modprobe_deps
+ local s
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe libcrc32c
+ s=success
+ debug 'Loading libcrc32c: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe nvmet_rdma
+ s=success
+ debug 'Loading nvmet_rdma: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe nvme_rdma
+ s=success
+ debug 'Loading nvme_rdma: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe loop
+ s=success
+ debug 'Loading loop: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe dm_writecache
+ s=success
+ debug 'Loading dm_writecache: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe dm_cache
+ s=success
+ debug 'Loading dm_cache: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe dm_thin_pool
+ s=success
+ debug 'Loading dm_thin_pool: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe dm_snapshot
+ s=success
+ debug 'Loading dm_snapshot: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe dm_crypt
+ s=success
+ debug 'Loading dm_crypt: success'
+ '[' -n '' ']'
+ return 0
+ return 0
+ [[ '' == \d\e\p\s\_\o\n\l\y ]]
+ grep -q '^drbd ' /proc/modules
+ '[' -z 4.18.0-477.10.1.el8_8.x86_64 ']'
+ var1=4.18.0-477.10.1.el8_8.x86_64
+ matchvar=4.18.0-477.10.1.el8_8.x86_64.rpm
++ find /pkgs/drbd-rpms/ -type f -name '*4.18.0-477.10.1.el8_8.x86_64.rpm' -print -quit
+ drbd_rpm=
+ '[' -n '' ']'
+ matchvar=4.18.0-477.10.1.el8_8.x86_64.deb
++ find /pkgs/drbd-debs/ -type f -name '*4.18.0-477.10.1.el8_8.x86_64.deb' -print -quit
+ drbd_deb=
+ [[ -n '' ]]
+ debug 'There is no corresponding kernel version deb package'
+ '[' -n '' ']'
+ return 0
+ pkgdir=/tmp/pkg
+ kodir=/tmp/ko
+ rm -rf /tmp/pkg /tmp/ko
+ mkdir -p /tmp/pkg /tmp/ko
+ fmt=rpm
++ type -p dpkg
+ '[' -n '' ']'
++ type -p emerge
+ '[' -n '' ']'
+ case $fmt in
++ repo::rpm::getrepofile
++ echo /etc/yum.repos.d/linbit.repo
+ repo=/etc/yum.repos.d/linbit.repo
++ how_to_get /etc/yum.repos.d/linbit.repo
++ local repo=/etc/yum.repos.d/linbit.repo
++ local how=
++ '[' -n '' ']'
++ '[' -f /etc/yum.repos.d/linbit.repo ']'
++ '[' -n '' ']'
++ mountpoint -q /usr/src
++ how=compile
++ echo compile
++ return 0
+ how_get=compile
+ debug 'Detected kmod method: "compile"'
+ '[' -n '' ']'
+ return 0
+ dist=we_do_not_care
++ needs_dist compile
++ local how=compile
++ local needsdist=n
++ [[ compile == \n\o\d\e\_\h\a\s\h ]]
++ echo n
++ return 0
+ need_dist=n
+ debug 'Needs host distribution info: "n"'
+ '[' -n '' ']'
+ return 0
+ [[ n == y ]]
+ case $how_get in
+ kos::fromsrc /tmp/pkg /tmp/ko
+ local pkgdir=/tmp/pkg
+ local kodir=/tmp/ko
+ cd /tmp/pkg
+ tar xf /drbd.tar.gz
+ cd drbd-9.0.32-1
+ make
Need a git checkout to regenerate drbd/.drbd_git_revision
make[1]: Entering directory '/tmp/pkg/drbd-9.0.32-1/drbd'

    Calling toplevel makefile of kernel source tree, which I believe is in
    KDIR=/lib/modules/4.18.0-477.10.1.el8_8.x86_64/build

make -C /lib/modules/4.18.0-477.10.1.el8_8.x86_64/build   M=/tmp/pkg/drbd-9.0.32-1/drbd  modules
Makefile:1006: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel".  Stop.
make[1]: Leaving directory '/tmp/pkg/drbd-9.0.32-1/drbd'
make[1]: *** [Makefile:132: kbuild] Error 2
make: *** [Makefile:131: module] Error 2
+ [[ rhel8 == \k\y\l\i\n\1\0 ]]
++ how_to_load
++ [[ '' == yes ]]
++ echo RAM
++ return 0
+ how_load=RAM
+ debug 'Detected load method: "RAM"'
+ '[' -n '' ']'
+ return 0
+ [[ compile == \c\o\m\p\i\l\e ]]
+ [[ RAM == \i\n\s\t\a\l\l ]]
+ load_from_ram /tmp/pkg /tmp/ko
+ local pkgdir=/tmp/pkg
+ local kodir=/tmp/ko
+ find /tmp/pkg -name '*.ko' -exec cp '{}' /tmp/ko ';'
+ cd /tmp/ko
+ '[' '!' -f drbd.ko ']'
+ die 'Could not find the expexted *.ko, see stderr for more details'
+ echo

+ echo -e 'Could not find the expexted *.ko, see stderr for more details'
Could not find the expexted *.ko, see stderr for more details
+ exit 1
Successfully loaded shipped module
+ echo 'Successfully loaded shipped module'
DRBD IS ALREADY EXIST (yes or no):no
DRBD INSTALL BY PRM (yes or no) :no
+ echo 'DRBD IS ALREADY EXIST (yes or no):no'
+ echo 'DRBD INSTALL BY PRM (yes or no) :no'
+ [[ no == \y\e\s ]]
+ [[ yes == yes ]]
+ [[ rhel8 =~ rhel|kylin10 ]]
++ uname -r
+ KODIR=/lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra/drbd
+ mkdir -vp /lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra/drbd
mkdir: created directory '/lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra'
mkdir: created directory '/lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra/drbd'
+ cp -vf '/tmp/ko/*.ko' /lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra/drbd/
cp: cannot stat '/tmp/ko/*.ko': No such file or directory
+ depmod -a
+ cp -vf /pkgs/drbd.modules-load.conf /etc/modules-load.d/drbd.conf
'/pkgs/drbd.modules-load.conf' -> '/etc/modules-load.d/drbd.conf'
+ cp -vf /pkgs/drbd.modules /etc/sysconfig/modules/
'/pkgs/drbd.modules' -> '/etc/sysconfig/modules/drbd.modules'
+ cp -vf /pkgs/utils/drbdadm /pkgs/utils/drbdadm-83 /pkgs/utils/drbdadm-84 /pkgs/utils/drbdmeta /pkgs/utils/drbdsetup /pkgs/utils/drbdsetup-83 /pkgs/utils/drbdsetup-84 /usr-local-bin
'/pkgs/utils/drbdadm' -> '/usr-local-bin/drbdadm'
'/pkgs/utils/drbdadm-83' -> '/usr-local-bin/drbdadm-83'
'/pkgs/utils/drbdadm-84' -> '/usr-local-bin/drbdadm-84'
'/pkgs/utils/drbdmeta' -> '/usr-local-bin/drbdmeta'
'/pkgs/utils/drbdsetup' -> '/usr-local-bin/drbdsetup'
'/pkgs/utils/drbdsetup-83' -> '/usr-local-bin/drbdsetup-83'
'/pkgs/utils/drbdsetup-84' -> '/usr-local-bin/drbdsetup-84'
+ cat /pkgs/drbd.conf
+ cp -vf /pkgs/global_common.conf /etc/drbd.d/
'/pkgs/global_common.conf' -> '/etc/drbd.d/global_common.conf'
++ cat /proc/drbd
++ awk '/^version/ {print $2}'
cat: /proc/drbd: No such file or directory
+ [[ '' != 9.0.32-1 ]]
+ echo 'ERROR: DRBD is NOT loaded with the right version'
+ exit 1
ERROR: DRBD is NOT loaded with the right version
[root@hostname ~]#

pod yaml:

[root@mitqadaocappn02 ~]# kubectl get pods -n hwameistor drbd-adapter-hostname-rhel8-1725963185-mn2n2 -oyaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: "2024-09-10T10:13:06Z"
  generateName: drbd-adapter-hostname-rhel8-1725963185-
  labels:
    app: drbd-adapter
    batch.kubernetes.io/controller-uid: ebe37506-8320-4c60-afff-cd838643c1d9
    batch.kubernetes.io/job-name: drbd-adapter-hostname-rhel8-1725963185
    controller-uid: ebe37506-8320-4c60-afff-cd838643c1d9
    drbd-version: v9.0.32-1
    job-name: drbd-adapter-hostname-rhel8-1725963185
  name: drbd-adapter-hostname-rhel8-1725963185-mn2n2
  namespace: hwameistor
  ownerReferences:
  - apiVersion: batch/v1
    blockOwnerDeletion: true
    controller: true
    kind: Job
    name: drbd-adapter-hostname-rhel8-1725963185
    uid: ebe37506-8320-4c60-afff-cd838643c1d9
  resourceVersion: "64000555"
  uid: 4e91e3e9-f5ef-4fef-aa3a-690cd2d21648
spec:
  containers:
  - image: 10.101.210.43/ghcr.m.daocloud.io/hwameistor/drbd9-shipper:v9.0.32-1_v0.4.2
    imagePullPolicy: IfNotPresent
    name: shipper
    resources: {}
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /pkgs
      name: pkgs
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: kube-api-access-xvc27
      readOnly: true
  - command:
    - /pkgs/entrypoint.adapter.sh
    - 4.18.0-477.10.1.el8_8.x86_64
    env:
    - name: LB_SKIP
      value: "no"
    - name: LB_DROP
      value: "yes"
    - name: LB_UPGRADE
      value: "no"
    - name: LB_CHECK_HOSTNAME
      value: "yes"
    - name: NODE_NAME
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: spec.nodeName
    image: 10.101.210.43/ghcr.m.daocloud.io/hwameistor/drbd9-rhel8:v9.0.32-1_v0.4.2
    imagePullPolicy: IfNotPresent
    name: rhel8
    resources: {}
    securityContext:
      privileged: true
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /pkgs
      name: pkgs
    - mountPath: /pkgs_root
      name: pkgroot
    - mountPath: /etc/host-release
      name: os-release
      readOnly: true
    - mountPath: /usr/src
      name: usr-src
      readOnly: true
    - mountPath: /lib/modules
      name: lib-modules
    - mountPath: /usr-local-bin
      name: usr-local-bin
    - mountPath: /etc/drbd.conf
      name: etc-drbd-conf
    - mountPath: /etc/drbd.d
      name: etc-drbd-d
    - mountPath: /var/lib/drbd
      name: var-lib-drbd
      readOnly: true
    - mountPath: /etc/modules-load.d
      name: etc-modules-load
    - mountPath: /etc/sysconfig/modules
      name: etc-sysconfig-modules
    - mountPath: /etc/centos-release
      name: centos-release
      readOnly: true
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: kube-api-access-xvc27
      readOnly: true
  dnsPolicy: ClusterFirst
  enableServiceLinks: true
  hostNetwork: true
  hostPID: true
  nodeName: hostname
  nodeSelector:
    kubernetes.io/hostname: hostname
  preemptionPolicy: PreemptLowerPriority
  priority: 0
  restartPolicy: Never
  schedulerName: default-scheduler
  securityContext: {}
  serviceAccount: default
  serviceAccountName: default
  terminationGracePeriodSeconds: 0
  tolerations:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
    operator: Exists
  - effect: NoSchedule
    key: node-role.kubernetes.io/control-plane
    operator: Exists
  - effect: NoExecute
    key: node.kubernetes.io/not-ready
    operator: Exists
    tolerationSeconds: 300
  - effect: NoExecute
    key: node.kubernetes.io/unreachable
    operator: Exists
    tolerationSeconds: 300
  volumes:
  - emptyDir: {}
    name: pkgs
  - hostPath:
      path: /root
      type: ""
    name: pkgroot
  - hostPath:
      path: /etc/os-release
      type: FileOrCreate
    name: os-release
  - hostPath:
      path: /etc/centos-release
      type: FileOrCreate
    name: centos-release
  - hostPath:
      path: /usr/src
      type: ""
    name: usr-src
  - hostPath:
      path: /lib/modules
      type: ""
    name: lib-modules
  - hostPath:
      path: /usr/local/bin
      type: ""
    name: usr-local-bin
  - hostPath:
      path: /etc/drbd.conf
      type: FileOrCreate
    name: etc-drbd-conf
  - hostPath:
      path: /etc/drbd.d
      type: DirectoryOrCreate
    name: etc-drbd-d
  - hostPath:
      path: /var/lib/drbd
      type: DirectoryOrCreate
    name: var-lib-drbd
  - hostPath:
      path: /etc/modules-load.d
      type: DirectoryOrCreate
    name: etc-modules-load
  - hostPath:
      path: /etc/sysconfig/modules
      type: DirectoryOrCreate
    name: etc-sysconfig-modules
  - name: kube-api-access-xvc27
    projected:
      defaultMode: 420
      sources:
      - serviceAccountToken:
          expirationSeconds: 3607
          path: token
      - configMap:
          items:
          - key: ca.crt
            path: ca.crt
          name: kube-root-ca.crt
      - downwardAPI:
          items:
          - fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
            path: namespace
status:
  conditions:
  - lastProbeTime: null
    lastTransitionTime: "2024-09-10T10:19:37Z"
    status: "False"
    type: PodReadyToStartContainers
  - lastProbeTime: null
    lastTransitionTime: "2024-09-10T10:13:06Z"
    status: "True"
    type: Initialized
  - lastProbeTime: null
    lastTransitionTime: "2024-09-10T10:13:06Z"
    reason: PodFailed
    status: "False"
    type: Ready
  - lastProbeTime: null
    lastTransitionTime: "2024-09-10T10:13:06Z"
    reason: PodFailed
    status: "False"
    type: ContainersReady
  - lastProbeTime: null
    lastTransitionTime: "2024-09-10T10:13:06Z"
    status: "True"
    type: PodScheduled
  containerStatuses:
  - containerID: containerd://db23f500e40fd1990ab6d2f116283d76d3dea9254c1c79d4b0926382f9211ff0
    image: 10.101.210.43/ghcr.m.daocloud.io/hwameistor/drbd9-rhel8:v9.0.32-1_v0.4.2
    imageID: sha256:4ed5663ed4e00011ace7a6b11c846c4e45d3f1fe81720fe1b94f7f06fcf68ef9
    lastState: {}
    name: rhel8
    ready: false
    restartCount: 0
    started: false
    state:
      terminated:
        containerID: containerd://db23f500e40fd1990ab6d2f116283d76d3dea9254c1c79d4b0926382f9211ff0
        exitCode: 1
        finishedAt: "2024-09-10T10:19:36Z"
        reason: Error
        startedAt: "2024-09-10T10:19:31Z"
  - containerID: containerd://d2ec646dfe9de29d9346af986b4b54a8bc0ee2fff81f00948768ff8e84ae839f
    image: 10.101.210.43/ghcr.m.daocloud.io/hwameistor/drbd9-shipper:v9.0.32-1_v0.4.2
    imageID: sha256:d48131e7df28b10c410cacb77667f22c5bde6995015bc04e1a3a8c362f210388
    lastState: {}
    name: shipper
    ready: false
    restartCount: 0
    started: false
    state:
      terminated:
        containerID: containerd://d2ec646dfe9de29d9346af986b4b54a8bc0ee2fff81f00948768ff8e84ae839f
        exitCode: 0
        finishedAt: "2024-09-10T10:13:07Z"
        reason: Completed
        startedAt: "2024-09-10T10:13:06Z"
  hostIP: 10.101.210.44
  hostIPs:
  - ip: 10.101.210.44
  phase: Failed
  podIP: 10.101.210.44
  podIPs:
  - ip: 10.101.210.44
  qosClass: BestEffort
  startTime: "2024-09-10T10:13:06Z"
[root@hostname ~]#

[peng9808]