Merge pull request #84 from bunkerity/dev

Add support for new AppSec feature in CrowdSec plugin and update Coraza

.github/workflows/codeql.yml

@@ -19,13 +19,13 @@ jobs:
         language: ["python", "go"]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@ccf74c947955fd1cf117aef6a0e4e66191ef6f61 # v3.25.4
+        uses: github/codeql-action/init@4fa2a7953630fd2f3fb380f21be14ede0169dd4f # v3.25.12
         with:
           languages: ${{ matrix.language }}
           config-file: ./.github/codeql.yml
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@ccf74c947955fd1cf117aef6a0e4e66191ef6f61 # v3.25.4
+        uses: github/codeql-action/analyze@4fa2a7953630fd2f3fb380f21be14ede0169dd4f # v3.25.12
         with:
           category: "/language:${{matrix.language}}"

.github/workflows/tests.yml

@@ -17,18 +17,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout source code
-        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Get BW tag
         run: |
           if [ "$GITHUB_REF" = "refs/heads/main" ] ; then
-            echo "BW_TAG=1.5.7" >> $GITHUB_ENV
+            echo "BW_TAG=1.5.9" >> $GITHUB_ENV
           else
             echo "BW_TAG=dev" >> $GITHUB_ENV
           fi
 
       - name: Login to Docker Hub
-        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
@@ -47,12 +47,15 @@ jobs:
 
       - name: Run CrowdSec stream tests
         run: ./.tests/crowdsec.sh stream
+
+      - name: Run CrowdSec appsec tests
+        run: ./.tests/crowdsec.sh appsec
 
       - name: Run VirusTotal tests
         run: ./.tests/virustotal.sh
         env:
           VIRUSTOTAL_API_KEY: ${{ secrets.VIRUSTOTAL_API_KEY }}
 
       - name: Build and push APIs
-        if: env.BW_TAG == '1.5.7'
+        if: env.BW_TAG == '1.5.9'
         run: ./.tests/build-push.sh "${{ env.BW_TAG }}"

.gitignore

@@ -3,4 +3,4 @@
 *.zip
 env
 node_modules
-style.css
+style.css

.pre-commit-config.yaml

@@ -3,7 +3,7 @@
 exclude: (^coraza/api/coreruleset|(^LICENSE.md|.svg)$)
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: c4a0b883114b00d8d76b479c820ce7950211c99b # frozen: v4.5.0
+    rev: 2c9f875913ee60ca25ce70243dc24d5b6415598c # frozen: v4.6.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -12,7 +12,7 @@ repos:
       - id: check-case-conflict
 
   - repo: https://github.com/ambv/black
-    rev: e026c93888f91a47a9c9f4e029f3eb07d96375e6 # frozen: 24.1.1
+    rev: 3702ba224ecffbcec30af640c149f231d90aebdb # frozen: 24.4.2
     hooks:
       - id: black
         name: Black Python Formatter
@@ -31,34 +31,34 @@ repos:
         exclude: ^crowdsec/lib/
 
   - repo: https://github.com/lunarmodules/luacheck
-    rev: 418f48976c73be697fe64b0eba9ea9821ac9bca8 # frozen: v1.1.2
+    rev: cc089e3f65acdd1ef8716cc73a3eca24a6b845e4 # frozen: v1.2.0
     hooks:
       - id: luacheck
         exclude: ^crowdsec/lib/
         args: ["--std", "min", "--codes", "--ranges", "--no-cache"]
 
   - repo: https://github.com/pycqa/flake8
-    rev: 7d37d9032d0d161634be4554273c30efd4dea0b3 # frozen: 7.0.0
+    rev: 1978e2b0de6efa0cb2a2b6f3f7986aa6569dd2be # frozen: 7.1.0
     hooks:
       - id: flake8
         name: Flake8 Python Linter
         args: ["--max-line-length=250", "--ignore=E266,E402,E722,W503"]
 
   - repo: https://github.com/codespell-project/codespell
-    rev: 6e41aba91fb32e9feb741a6258eefeb9c6e4a482 # frozen: v2.2.6
+    rev: 193cd7d27cd571f79358af09a8fb8997e54f8fff # frozen: v2.3.0
     hooks:
       - id: codespell
         name: Codespell Spell Checker
-        entry: codespell --ignore-regex="(tabEl|Widgits)" --skip src/ui/static/js/utils/flatpickr.js,CHANGELOG.md
+        entry: codespell --ignore-regex="(tabEl|Widgits)" --skip */ui/template.html,src/ui/static/js/utils/flatpickr.js,CHANGELOG.md
         language: python
         types: [text]
 
   - repo: https://github.com/gitleaks/gitleaks
-    rev: 145400593c178304246371bc45290588bc72f43e # frozen: v8.18.2
+    rev: 77c3c6a34b2577d71083442326c60b8fd58926ec # frozen: v8.18.4
     hooks:
       - id: gitleaks
 
   - repo: https://github.com/koalaman/shellcheck-precommit
-    rev: 3f77b826548d8dc2d26675f077361c92773b50a7 # frozen: v0.9.0
+    rev: 2491238703a5d3415bb2b7ff11388bf775372f29 # frozen: v0.10.0
     hooks:
       - id: shellcheck

.tests/clamav/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3"
 
 services:
   bunkerweb:
-    image: bunkerity/bunkerweb:1.5.7
+    image: bunkerity/bunkerweb:1.5.8
     ports:
       - 80:8080
       - 443:8443
@@ -27,7 +27,7 @@ services:
       - bw-services
 
   bw-scheduler:
-    image: bunkerity/bunkerweb-scheduler:1.5.7
+    image: bunkerity/bunkerweb-scheduler:1.5.8
     depends_on:
       - bunkerweb
       - bw-docker

.tests/coraza.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+# shellcheck disable=SC1091
 . .tests/utils.sh
 
 echo "ℹ️ Starting Coraza tests ..."

.tests/coraza/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3"
 
 services:
   bunkerweb:
-    image: bunkerity/bunkerweb:1.5.7
+    image: bunkerity/bunkerweb:1.5.8
     ports:
       - 80:8080
       - 443:8443
@@ -26,7 +26,7 @@ services:
       - bw-services
 
   bw-scheduler:
-    image: bunkerity/bunkerweb-scheduler:1.5.7
+    image: bunkerity/bunkerweb-scheduler:1.5.8
     depends_on:
       - bunkerweb
       - bw-docker

.tests/crowdsec.sh

@@ -19,10 +19,16 @@ do_and_check_cmd cp .tests/crowdsec/docker-compose.yml /tmp/bunkerweb-plugins/cr
 # Edit compose
 do_and_check_cmd sed -i "s@bunkerity/bunkerweb:.*\$@bunkerweb:tests@g" /tmp/bunkerweb-plugins/crowdsec/docker-compose.yml
 do_and_check_cmd sed -i "s@bunkerity/bunkerweb-scheduler:.*\$@bunkerweb-scheduler:tests@g" /tmp/bunkerweb-plugins/crowdsec/docker-compose.yml
-do_and_check_cmd sed -i "s@CROWDSEC_MODE=.*\$@CROWDSEC_MODE=$1@g" /tmp/bunkerweb-plugins/crowdsec/docker-compose.yml
+if [ $1 == "appsec" ] ; then
+	do_and_check_cmd sed -i "s@CROWDSEC_MODE=.*\$@CROWDSEC_MODE=live@g" /tmp/bunkerweb-plugins/crowdsec/docker-compose.yml
+	do_and_check_cmd sed -i "s@CROWDSEC_APPSEC_URL=.*\$@CROWDSEC_APPSEC_URL=http://crowdsec:7422@g" /tmp/bunkerweb-plugins/crowdsec/docker-compose.yml
+else
+	do_and_check_cmd sed -i "s@CROWDSEC_MODE=.*\$@CROWDSEC_MODE=$1@g" /tmp/bunkerweb-plugins/crowdsec/docker-compose.yml
+fi
 
 # Copy configs
 do_and_check_cmd cp .tests/crowdsec/acquis.yaml /tmp/bunkerweb-plugins/crowdsec
+do_and_check_cmd cp .tests/crowdsec/appsec.yaml /tmp/bunkerweb-plugins/crowdsec
 do_and_check_cmd cp .tests/crowdsec/syslog-ng.conf /tmp/bunkerweb-plugins/crowdsec
 
 # Do the tests
@@ -58,23 +64,34 @@ if [ "$success" == "ko" ] ; then
 	exit 1
 fi
 
-# Run basic attack with dirb
-echo "ℹ️ Executing dirb ..."
-do_and_check_cmd sudo apt install -y dirb
-dirb http://localhost -H "Host: www.example.com" -H "User-Agent: LegitOne" -f > /dev/null 2>&1
+if [ "$1" != "appsec" ] ; then
+	# Run basic attack with dirb
+	echo "ℹ️ Executing dirb ..."
+	do_and_check_cmd sudo apt install -y dirb
+	dirb http://localhost -H "Host: www.example.com" -H "User-Agent: LegitOne" -f > /dev/null 2>&1
 
-# Wait if are in stream mode
-if [ "$1" == "stream" ] ; then
-	sleep 20
-fi
+	# Wait if are in stream mode
+	if [ "$1" == "stream" ] ; then
+		sleep 20
+	fi
 
-# Expect a 403
-echo "ℹ️ Checking CS ..."
-success="ko"
-ret="$(curl -s -o /dev/null -w "%{http_code}" -H "Host: www.example.com" http://localhost)"
-# shellcheck disable=SC2181
-if [ $? -eq 0 ] && [ "$ret" -eq 403 ] ; then
-	success="ok"
+	# Expect a 403
+	echo "ℹ️ Checking CS ..."
+	success="ko"
+	ret="$(curl -s -o /dev/null -w "%{http_code}" -H "Host: www.example.com" http://localhost)"
+	# shellcheck disable=SC2181
+	if [ $? -eq 0 ] && [ "$ret" -eq 403 ] ; then
+		success="ok"
+	fi
+else
+	# Send a malicious pattern
+	echo "ℹ️ Sending malicious pattern"
+	success="ko"
+	ret="$(curl -s -o /dev/null -w "%{http_code}" -H "Host: www.example.com" http://localhost/rpc2)"
+	# shellcheck disable=SC2181
+	if [ $? -eq 0 ] && [ "$ret" -eq 403 ] ; then
+		success="ok"
+	fi
 fi
 
 # We're done

.tests/crowdsec/appsec.yaml

@@ -0,0 +1,5 @@
+appsec_config: crowdsecurity/appsec-default
+labels:
+  type: appsec
+listen_addr: 0.0.0.0:7422
+source: appsec

.tests/crowdsec/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3"
 
 services:
   bunkerweb:
-    image: bunkerity/bunkerweb:1.5.7
+    image: bunkerity/bunkerweb:1.5.8
     ports:
       - 80:8080
       - 443:8443
@@ -15,6 +15,7 @@ services:
       - CROWDSEC_API=http://crowdsec:8080
       - CROWDSEC_API_KEY=s3cr3tb0unc3rk3y
       - CROWDSEC_MODE=
+      - CROWDSEC_APPSEC_URL=
       - LOG_LEVEL=info
       - USE_MODSECURITY=no
       - USE_BLACKLIST=no
@@ -34,7 +35,7 @@ services:
         syslog-address: "udp://10.10.10.254:514"
 
   bw-scheduler:
-    image: bunkerity/bunkerweb-scheduler:1.5.7
+    image: bunkerity/bunkerweb-scheduler:1.5.8
     depends_on:
       - bunkerweb
       - bw-docker
@@ -61,10 +62,11 @@ services:
     volumes:
       - cs-data:/var/lib/crowdsec/data
       - ./acquis.yaml:/etc/crowdsec/acquis.yaml
+      - ./appsec.yaml:/etc/crowdsec/acquis.d/appsec.yaml
       - bw-logs:/var/log:ro
     environment:
       - BOUNCER_KEY_bunkerweb=s3cr3tb0unc3rk3y
-      - COLLECTIONS=crowdsecurity/nginx
+      - COLLECTIONS=crowdsecurity/nginx crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules crowdsecurity/appsec-crs
       - DISABLE_PARSERS=crowdsecurity/whitelists
     networks:
       - bw-universe

.tests/virustotal.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+# shellcheck disable=SC1091
 . .tests/utils.sh
 
 echo "ℹ️ Starting VirusTotal tests ..."

.tests/virustotal/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3"
 
 services:
   bunkerweb:
-    image: bunkerity/bunkerweb:1.5.7
+    image: bunkerity/bunkerweb:1.5.8
     ports:
       - 80:8080
       - 443:8443
@@ -28,7 +28,7 @@ services:
       - bw-services
 
   bw-scheduler:
-    image: bunkerity/bunkerweb-scheduler:1.5.7
+    image: bunkerity/bunkerweb-scheduler:1.5.8
     depends_on:
       - bunkerweb
       - bw-docker

README.md

@@ -3,19 +3,19 @@
 </p>
 
 <p align="center">
-	<img src="https://img.shields.io/badge/bunkerweb-1.5.7-blue" />
+	<img src="https://img.shields.io/badge/bunkerweb_plugins-1.6-blue" />
 	<img src="https://img.shields.io/github/last-commit/bunkerity/bunkerweb-plugins" />
 	<img src="https://img.shields.io/github/actions/workflow/status/bunkerity/bunkerweb-plugins/tests.yml?branch=dev&label=CI%2FCD%20dev" />
 	<img src="https://img.shields.io/github/actions/workflow/status/bunkerity/bunkerweb-plugins/tests.yml?branch=main&label=CI%2FCD%20main" />
 	<img src="https://img.shields.io/github/issues/bunkerity/bunkerweb-plugins">
 	<img src="https://img.shields.io/github/issues-pr/bunkerity/bunkerweb-plugins">
 </p>
 
-This repository contains "official" plugins for the [BunkerWeb solution](https://github.com/bunkerity/bunkerweb). If you don't already know BunkerWeb, you should first read the [documentation](https://docs.bunkerweb.io).
+This repository contains "official" plugins for the [BunkerWeb solution](https://github.com/bunkerity/bunkerweb). If you don't already know BunkerWeb, you should first read the [documentation](https://docs.bunkerweb.io/?utm_campaign=self&utm_source=github).
 
 # Prerequisites
 
-The installation of external plugins is covered in the [plugins section](https://docs.bunkerweb.io/latest/plugins) of the documentation.
+The installation of external plugins is covered in the [plugins section](https://docs.bunkerweb.io/latest/plugins/?utm_campaign=self&utm_source=github) of the documentation.
 
 # Plugins
 
@@ -46,7 +46,7 @@ Please contact us at contact \[@\] bunkerity.com if you are interested.
 
 To get free community support you can use the following media :
 
-- The #help channel of BunkerWeb in the [Discord server](https://discord.com/invite/fTf46FmtyD)
+- The #help channel of BunkerWeb in the [Discord server](https://bunkerity.discord.com/?utm_campaign=self&utm_source=github)
 - The help category of [GitHub discussions](https://github.com/bunkerity/bunkerweb-plugins/discussions)
 - The [/r/BunkerWeb](https://www.reddit.com/r/BunkerWeb) subreddit
 - The [Server Fault](https://serverfault.com/) and [Super User](https://superuser.com/) forums

clamav/README.md

@@ -4,7 +4,7 @@
 	<img alt="BunkerWeb ClamAV diagram" src="https://github.com/bunkerity/bunkerweb-plugins/raw/main/clamav/docs/diagram.svg" />
 </p>
 
-This [BunkerWeb](https://www.bunkerweb.io) plugin will automatically check if any uploaded file is detected by the ClamAV antivirus engine and deny the request if that's the case.
+This [BunkerWeb](https://www.bunkerweb.io/?utm_campaign=self&utm_source=github) plugin will automatically check if any uploaded file is detected by the ClamAV antivirus engine and deny the request if that's the case.
 
 # Table of contents
 
@@ -20,11 +20,11 @@ This [BunkerWeb](https://www.bunkerweb.io) plugin will automatically check if an
 
 # Prerequisites
 
-Please read the [plugins section](https://docs.bunkerweb.io/latest/plugins) of the BunkerWeb documentation first.
+Please read the [plugins section](https://docs.bunkerweb.io/latest/plugins/?utm_campaign=self&utm_source=github) of the BunkerWeb documentation first.
 
 # Setup
 
-See the [plugins section](https://docs.bunkerweb.io/latest/plugins) of the BunkerWeb documentation for the installation procedure depending on your integration.
+See the [plugins section](https://docs.bunkerweb.io/latest/plugins/?utm_campaign=self&utm_source=github) of the BunkerWeb documentation for the installation procedure depending on your integration.
 
 ## Docker
 
@@ -34,7 +34,7 @@ version: '3'
 services:
 
   bunkerweb:
-    image: bunkerity/bunkerweb:1.5.7
+    image: bunkerity/bunkerweb:1.5.9
     ...
     environment:
       - USE_CLAMAV=yes
@@ -59,7 +59,7 @@ version: '3'
 services:
 
   mybunker:
-    image: bunkerity/bunkerweb:1.5.7
+    image: bunkerity/bunkerweb:1.5.9
     ...
     environment:
       - USE_CLAMAV=yes
@@ -132,12 +132,12 @@ metadata:
 
 # Settings
 
-|    Setting     |Default | Context |Multiple|                      Description                      |
-|----------------|--------|---------|--------|-------------------------------------------------------|
-|`USE_CLAMAV`    |`no`    |multisite|no      |Activate automatic scan of uploaded files with ClamAV. |
-|`CLAMAV_HOST`   |`clamav`|global   |no      |ClamAV hostname or IP address.                         |
-|`CLAMAV_PORT`   |`3310`  |global   |no      |ClamAV port.                                           |
-|`CLAMAV_TIMEOUT`|`1000`  |global   |no      |Network timeout (in ms) when communicating with ClamAV.|
+| Setting          | Default  | Context   | Multiple | Description                                             |
+| ---------------- | -------- | --------- | -------- | ------------------------------------------------------- |
+| `USE_CLAMAV`     | `no`     | multisite | no       | Activate automatic scan of uploaded files with ClamAV.  |
+| `CLAMAV_HOST`    | `clamav` | global    | no       | ClamAV hostname or IP address.                          |
+| `CLAMAV_PORT`    | `3310`   | global    | no       | ClamAV port.                                            |
+| `CLAMAV_TIMEOUT` | `1000`   | global    | no       | Network timeout (in ms) when communicating with ClamAV. |
 
 # TODO