Merge pull request #132 from IMDA-BTG/107-fix-security-issue-with-sci…

…py-v193

107 fix security issue with scipy v193 in v0.9.x

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 120
+extend-ignore = E203

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,34 @@
+## Bug Report
+
+**Description:**
+Provide a clear and concise description of the bug you encountered.
+
+**Expected Behavior:**
+Explain what you expected to happen when performing the steps above.
+
+**Actual Behavior:**
+Describe what actually happened instead of the expected behavior.
+
+**Steps to Reproduce:**
+List the steps to reproduce the bug. Be as specific as possible.
+
+**Environment:**
+- Operating System and Version:
+- AI Verify Version:
+- Any other relevant information (e.g. version of related package dependencies installed):
+
+Did you build using source code or from docker file? If from source code, please provide the below information:
+- Python Version:
+- Node Version:
+- Redis Version:
+- Mongo Version: 
+- Screen Resolution:
+
+**Screenshots/Code Snippets:**
+If applicable, include screenshots or code snippets that demonstrate the issue.
+
+**Additional Context:**
+Add any additional context about the problem here.
+
+**Possible Solution (Optional):**
+If you have any ideas or suggestions for a possible solution, you can include them here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,19 @@
+## Feature Request
+
+**Description:**
+Provide a clear and concise description of the feature you would like to request.
+
+**Is your feature request related to a problem/limitation? Please describe:**
+Explain any problems or limitations you are currently facing that this feature could help address.
+
+**Describe the solution you'd like:**
+Provide a detailed description of the feature you would like to see implemented.
+
+**Alternatives (Optional):**
+If applicable, describe any alternative solutions or features you have considered.
+
+**Additional Context (Optional):**
+Add any additional context or information about the feature request here.
+
+**Related Issues/PRs (Optional):**
+If there are any related issues or pull requests, you can mention them here.

.github/workflows/pre-merge-checks-apigw.yml

@@ -5,67 +5,41 @@
 # 4. Dependency analysis (undesirable licenses)
 # 5. Deploy reports generated from the above to GitHub Pages
 
-## support monorepo
-
 name: Pre-Merge Checks (ai-verify-apigw)
 
 on:
-  # Runs on pull request to main
+  # Runs when a pull request to main is being assigned
   pull_request:
-    branches: [master, main]
+    types: [ assigned, synchronize ]
+    branches:
+      - 'main'
     paths:
       - 'ai-verify-apigw/**'
 
   # Run this workflow manually from Actions tab
   workflow_dispatch:
 
-# Sets permissions of GITHUB_TOKEN
-permissions:
-  contents: write
-  pages: write
-  pull-requests: write
-  id-token: write
-
 # Allow one concurrent deployment
 concurrency:
   group: ${{ github.repository }}-${{ github.workflow }}
   cancel-in-progress: true
 
 jobs:
-  # Single deploy job since we're just deploying
+
   pre-merge-checks:
-#    environment:
-#      name: github-pages
-#      url: ${{ steps.deployment.outputs.page_url }}
+    # Run only when PR is assigned, even on subsequent commits (i.e. synchronize)
+    if: (github.event_name == 'pull_request' && github.event.pull_request.assignee != null) || github.event_name == 'workflow_dispatch'
+
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 40
 
     steps:
-#      - name: Add label to PR
-#        env:
-#          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-#          PR: ${{ github.event.pull_request.html_url }}
-#        run: |
-#          gh pr edit $PR --add-label 'ai-verify-apigw'
-
-      # Checkout code
+
       - name: Checkout code
-        run: |
-          set +e
-          git config --global init.defaultBranch main
-          git init
-          git branch -m main
-          git remote add origin https://github.com/IMDA-BTG/aiverify.git
-          git config --local gc.auto 0
-          authToken="x-access-token:${{ secrets.GITHUB_TOKEN }}"
-          encodedAuthToken=$(echo -n "$authToken" | base64 -w0)
-          git config --local http.https://github.com/.extraheader "AUTHORIZATION: basic ${encodedAuthToken}"
-          git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +${{github.sha}}:refs/remotes/origin/main
-          git sparse-checkout init
-          git sparse-checkout set ai-verify-apigw _site
-          git checkout --progress --force -B master refs/remotes/origin/main
-          ls -l
-          set -e
+        uses: actions/checkout@v3
+        with:
+          sparse-checkout: |
+            ai-verify-apigw
 
       # Install dependencies
       - name: Setup npm cache/install
@@ -86,8 +60,8 @@ jobs:
       # Unit Tests & Coverage
       - name: Unit tests with coverage
         working-directory: ${{ github.workspace }}/ai-verify-apigw
-        if: always()
-        timeout-minutes: 5
+        if: ${{ ! cancelled() }}
+        timeout-minutes: 30
         run: |
           set +e
           npm run coverage
@@ -103,11 +77,12 @@ jobs:
       # eslint
       - name: Code quality analysis - lint
         working-directory: ${{ github.workspace }}/ai-verify-apigw
-        if: always()
+        if: ${{ ! cancelled() }}
         run: |
           set +e
+          npx eslint .
+          exit_code_lint=$?          
           npx eslint -f html -o eslint-report.html .
-          exit_code_lint=$?
           npx eslint -f json -o eslint-report.json .
           node ci/createBadges.mjs lint
           set -e
@@ -119,13 +94,15 @@ jobs:
       # npm audit
       - name: Dependency analysis - vulnerabilities & licenses
         working-directory: ${{ github.workspace }}/ai-verify-apigw
-        if: always()
+        if: ${{ ! cancelled() }}
         run: |
           set +e
+          npm audit
+          exit_code_audit=$?          
           npm audit --json | npx npm-audit-markdown --output npm-audit-report.md
-          exit_code_audit=$?
           npx markdown-to-html-cli --source npm-audit-report.md --output npm-audit-report.html -y
           npx license-checker --summary --out licenses-found.txt -y
+          cat licenses-found.txt
           node ci/createBadges.mjs dependency
           echo -e "License Check Summary for apigw\n" | cat - licenses-found.txt > license-report.txt          
           node ci/createBadges.mjs license
@@ -136,8 +113,9 @@ jobs:
           fi
 
        ### Publish reports to ci dashboard ###
+
       - name: Checkout dashboard
-        if: always()
+        if: ${{ github.event.pull_request.head.repo.full_name == github.repository && always() }}
         uses: actions/checkout@v3
         with:
           repository: IMDA-BTG/ci-dashboard
@@ -146,51 +124,21 @@ jobs:
           path: check-results
 
       - name: Push results to dashboard
-        if: always()
+        if: ${{ github.event.pull_request.head.repo.full_name == github.repository && always() }}
         working-directory: ${{ github.workspace }}/check-results
         run: |
           set +e
+          find ../ -type f -name ".gitignore" -exec rm {} +
+          [ -d "docs/pre-merge/apigw" ] && rm -rf docs/pre-merge/apigw
           mkdir -p docs/pre-merge/apigw
           mv ../ai-verify-apigw/coverage docs/pre-merge/apigw/
           mv ../ai-verify-apigw/*.svg docs/pre-merge/apigw/
           mv ../ai-verify-apigw/*.html docs/pre-merge/apigw/
           mv ../ai-verify-apigw/*.md docs/pre-merge/apigw/
           mv ../ai-verify-apigw/*.txt docs/pre-merge/apigw/
-          git add *
+          git add docs/pre-merge/apigw
           git config user.name "imda-btg"
           git config user.email "[email protected]"
           git commit -m "feat(apigw) actions publish apigw reports to dashboard"
           git push
           set -e
-
-#      - name: Prepare artifact
-#        if: always()
-#        run: |
-#          set +e
-#          mkdir -p _site/apigw
-#          rm -rf _site/apigw/
-#          mv ai-verify-apigw/coverage _site/apigw/
-#          mv ai-verify-apigw/*.svg _site/apigw/
-#          mv ai-verify-apigw/*.html _site/apigw/
-#          mv ai-verify-apigw/*.md _site/apigw/
-#          mv ai-verify-apigw/*.txt _site/apigw/
-#          git branch -m main
-#          git add _site/apigw
-#          git config user.name "imda-btg"
-#          git config user.email "[email protected]"
-#          git commit -m "feat(apigw): actions publish apigw reports to pages"
-#          git config --unset-all http.https://github.com/.extraheader
-#          authToken="x-access-token:${{ secrets.GITHUB_TOKEN }}"
-#          encodedAuthToken=$(echo -n "$authToken" | base64 -w0)
-#          git config --local http.https://github.com/.extraheader "AUTHORIZATION: basic ${encodedAuthToken}"
-#          git push origin main
-#          set -e
-#
-#      - name: Upload artifact
-#        if: always()
-#        uses: actions/upload-pages-artifact@v1
-#
-#      - name: Publish artifact to Pages
-#        if: always()
-#        id: deployment
-#        uses: actions/deploy-pages@v1

.github/workflows/pre-merge-checks-app.yml

@@ -0,0 +1,120 @@
+# Pre-merge Checks (for Python projects)
+# 1. Unit tests with code coverage (pytest)
+# 2. Code quality analysis (flake8)
+# 3. Dependency analysis (vulnerabilities)
+# 4. Dependency analysis (undesirable licenses)
+# 5. Deploy reports generated from the above to GitHub Pages
+
+name: Pre-Merge Checks (test-engine-app)
+
+on:
+  # Runs when a pull request to main is being assigned
+  pull_request:
+    types: [ assigned, synchronize ]
+    branches:
+      - 'main'
+    paths:
+      - 'test-engine-app/**'
+
+  # Run this workflow manually from Actions tab
+  workflow_dispatch:
+
+# Allow one concurrent deployment
+concurrency:
+  group: ${{ github.repository }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+
+  pre-merge-checks:
+    # Run only when PR is assigned, even on subsequent commits (i.e. synchronize)
+    if: (github.event_name == 'pull_request' && github.event.pull_request.assignee != null) || github.event_name == 'workflow_dispatch'
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+
+    steps:
+      # Install redis
+      - name: Install redis
+        run: |
+          sudo apt update
+          sudo apt install redis-server
+
+      # Checkout code
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          sparse-checkout: |
+            test-engine-app
+
+      # Install dependencies
+      - name: Setup pip cache/install
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+          cache-dependency-path: test-engine-app
+
+      - name: Install dependencies
+        working-directory: ${{ github.workspace }}/test-engine-app
+        run: |
+          curl -H 'Authorization: token ${{ secrets.CHECKOUT_TOKEN }}' -H 'Accept: application/vnd.github.v3.raw' -O -L https://api.github.com/repos/IMDA-BTG/aiverify/contents/test-engine-core/dist/test_engine_core-0.9.0.tar.gz
+          pip install -r requirements.txt
+          pip install test_engine_core-0.9.0.tar.gz
+          pip install pytest pytest-mock pytest-html pytest-json pytest-cov coverage anybadge
+          pip install flake8 flake8-html
+
+      # Unit Tests & Coverage
+      - name: Unit tests with coverage
+        if: ${{ ! cancelled() }}
+        working-directory: ${{ github.workspace }}/test-engine-app
+        timeout-minutes: 30
+        run: |
+          bash ci/run-test.sh
+
+      # flake8
+      - name: Code quality analysis - lint
+        if: ${{ ! cancelled() }}
+        working-directory: ${{ github.workspace }}/test-engine-app
+        run: |
+          bash ci/run-flake8.sh
+
+      # pip-audit
+      - name: Dependency analysis - vulnerabilities & licenses
+        if: ${{ ! cancelled() }}
+        working-directory: ${{ github.workspace }}/test-engine-app
+        run: |
+          bash ci/run-pip-audit.sh
+
+      ### Publish reports to ci dashboard ###
+
+      - name: Checkout dashboard
+        if: ${{ github.event.pull_request.head.repo.full_name == github.repository && always() }}
+        uses: actions/checkout@v3
+        with:
+          repository: IMDA-BTG/ci-dashboard
+          token: ${{ secrets.CHECKOUT_TOKEN }}
+          ref: main
+          path: check-results
+
+      - name: Push results to dashboard
+        if: ${{ github.event.pull_request.head.repo.full_name == github.repository && always() }}
+        working-directory: ${{ github.workspace }}/check-results
+        run: |
+          set +e
+          find ../ -type f -name ".gitignore" -exec rm {} +
+          [ -d "docs/pre-merge/app" ] && rm -rf docs/pre-merge/app
+          mkdir -p docs/pre-merge/app
+          mv ../test-engine-app/htmlcov docs/pre-merge/app/
+          mv ../test-engine-app/flake8-report docs/pre-merge/app/
+          mv ../test-engine-app/assets docs/pre-merge/app/         
+          mv ../test-engine-app/*.svg docs/pre-merge/app/
+          mv ../test-engine-app/*.html docs/pre-merge/app/
+          mv ../test-engine-app/*.md docs/pre-merge/app/
+          mv ../test-engine-app/*.txt docs/pre-merge/app/
+          git add docs/pre-merge/app
+          git config user.name "imda-btg"
+          git config user.email "[email protected]"
+          git commit -m "feat(portal) actions publish test-engine-app reports to dashboard"
+          git push
+          set -e