Slimed down new commit

.devcontainer/devcontainer.json

@@ -0,0 +1,51 @@
+{
+	"name": "mysoc_notebook",
+	"dockerComposeFile": "../docker-compose.yml",
+	"service": "app",
+	"overrideCommand": true,
+	"initializeCommand": [
+		".devcontainer/initializeCommand"
+	],
+	"postCreateCommand": ".devcontainer/postCreateCommand",
+	"workspaceFolder": "/workspaces/publicwhip_data",
+	"extensions": [
+		"ms-vscode.test-adapter-converter",
+		"bungcip.better-toml",
+		"ms-python.python",
+		"ms-python.vscode-pylance",
+		"ms-azuretools.vscode-docker",
+		"valentjn.vscode-ltex"
+	],
+	"customizations": {
+		"codespaces": {
+			"repositories": {
+				"mysociety/data_common": {
+					"permissions": {
+						"contents": "write",
+						"actions": "write",
+						"deployments": "write",
+						"issues": "write",
+						"packages": "read",
+						"pull_requests": "write",
+						"repository_projects": "write",
+						"statuses": "write",
+						"workflows": "write"
+					}
+				},
+				"mysociety/mysociety-docs-theme": {
+					"permissions": {
+						"contents": "write",
+						"actions": "write",
+						"deployments": "write",
+						"issues": "write",
+						"packages": "read",
+						"pull_requests": "write",
+						"repository_projects": "write",
+						"statuses": "write",
+						"workflows": "write"
+					}
+				}
+			}
+		}
+	}
+}

.devcontainer/initializeCommand

@@ -0,0 +1,2 @@
+#!/bin/sh
+[ ! -d "src/data_common/src" ] && git submodule update --init || echo "Already exists"

.devcontainer/initializeCommand.cmd

@@ -0,0 +1 @@
+powershell .devcontainer/initializeCommand.ps1

.devcontainer/initializeCommand.ps1

@@ -0,0 +1,7 @@
+$Folder = 'src/data_common/src'
+"Test to see if folder [$Folder] exists"
+if (Test-Path -Path $Folder) {
+    echo  "Submodule already exists"
+} else {
+    git submodule update --init
+}

.devcontainer/postCreateCommand

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+echo "get submodule to appear as main rather than latest comment"
+cd src/data_common
+git checkout main
+cd ../..
+cd docs/theme
+git checkout main

.env-example

@@ -0,0 +1 @@
+

.gitattributes

@@ -0,0 +1,9 @@
+* text eol=lf
+*.png binary
+*.jpg binary
+*.sqlite3 binary
+*.sqlite binary
+*.xlsx binary
+*.xls binary
+*.pdf binary
+*.parquet binary

.github/workflows/build_and_publish.yml

@@ -0,0 +1,99 @@
+name: Build datasets and publish
+
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+  pages: write
+  id-token: write
+
+on:
+  push: 
+    branches: ["main"]
+  workflow_dispatch:
+  schedule:
+    - cron : "0 8 * * *"
+
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout repo content
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+
+      - name: Run tests and try and build project
+        uses: mysociety/run-in-devcontainer@v1
+        with:
+          run: |
+            export PATH="/root/.local/bin:$PATH"
+            script/test
+            dataset build --all
+            dataset version auto --auto-ban major --all --publish
+            dataset publish --all
+
+
+      - name: Push new data
+        id: auto-commit-action
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: "Update repo data based on source changes"
+
+      - name: Send GitHub Action trigger data to Slack workflow
+        id: slack
+        if: steps.auto-commit-action.outputs.changes_detected == 'true'
+        uses: slackapi/[email protected]
+        with:
+          payload: |
+            {
+              "repo_url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.HAPPY_DATABOT_SLACK_WEBHOOK }}     
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v1
+
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: docs
+          destination: docs/_site
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v1
+        with:
+          path: docs/_site
+
+      - name: Send GitHub Action trigger data to Slack workflow (if failed)
+        if: ${{ failure() }}
+        id: slack-failed
+        uses: slackapi/[email protected]
+        with:
+          payload: |
+            {
+              "repo_url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SAD_DATABOT_SLACK_WEBHOOK }}     
+
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v1
+
+      - uses: geekyeggo/delete-artifact@v1
+        with:
+          name: github-pages

.github/workflows/test.yml

@@ -0,0 +1,31 @@
+name: Run project tests
+
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}-tests
+  cancel-in-progress: true
+
+on:
+  push:
+    branches-ignore: ["main"]
+  pull_request:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: checkout repo content
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+
+      - name: Run project tests
+        uses: mysociety/run-in-devcontainer@v1
+        with:
+          run: |
+            export PATH="/root/.local/bin:$PATH"
+            script/test
+            dataset build --all
+            dataset version auto --auto-ban major --all

.gitignore

@@ -0,0 +1,12 @@
+*.pyc
+secrets.yaml
+.env
+data/private/*
+_render/_parts
+_render/_papermills
+docs/_site
+data/raw
+docs/_site
+docs/data
+docs/_*
+*.parquet

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "src/data_common"]
+	path = src/data_common
+	url = https://github.com/mysociety/data_common
+[submodule "docs/theme"]
+	path = docs/theme
+	url = https://github.com/mysociety/mysociety-docs-theme/

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal"
+        },
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,46 @@
+{
+    "python.linting.pylintEnabled": true,
+    "python.defaultInterpreterPath": "/usr/local/bin/python",
+    "python.terminal.activateEnvironment": false,
+    "python.formatting.provider": "black",
+    "python.analysis.typeCheckingMode": "basic",
+    "python.analysis.stubPath": "src/data_common/typing",
+    "editor.formatOnSave": true,
+    "files.exclude": {
+        "**/.git": true,
+        "**/.svn": true,
+        "**/.hg": true,
+        "**/CVS": true,
+        "**/.DS_Store": true,
+        "**/*.pyc": {
+            "when": "$(basename).py"
+        },
+        "**/__pycache__": true
+    },
+    "files.associations": {
+        "**/*.html": "html",
+        "**/templates/**/*.html": "django-html",
+        "**/templates/**/*": "django-txt",
+        "**/requirements{/**,*}.{txt,in}": "pip-requirements"
+    },
+    "python.linting.pylintArgs": [
+        "--max-line-length=88",
+        "--disable=C0103,E1101,W5101,E1123,E501,E203",
+        "--load-plugins=pylint_django"
+    ],
+    "jupyter.jupyterServerType": "local",
+    "ltex.language": "en-GB",
+    "ltex.ltex-ls.path": "/ltex/ltex-ls-15.2.0/",
+    "[markdown]": {
+        "editor.quickSuggestions": {
+            "comments": "on",
+            "strings": "on",
+            "other": "on"
+        }
+    },
+    "python.testing.pytestArgs": [
+        "tests/"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

Dockerfile

@@ -0,0 +1,36 @@
+# This dockerfile is used by binder.
+
+FROM ghcr.io/mysociety/data_common:sha-54e0768
+
+# Make an empty project directory so the 'self' setup doesn't fail and scripts can be setup
+# Override the .pth created at previous stages to point to where the working directory will land
+COPY pyproject.toml poetry.loc[k] /setup/ 
+COPY src/data_common/pyproject.toml src/data_common/poetry.loc[k] /setup/src/data_common/
+RUN mkdir /setup/src/publicwhip_data \
+    && touch /setup/src/publicwhip_data/__init__.py \
+    && mkdir --parents /setup/src/data_common/src/data_common \
+    && touch /setup/src/data_common/src/data_common/__init__.py \
+    && export PATH="/root/.local/bin:$PATH" \
+    && cd /setup/ && poetry install \
+    && echo "/workspaces/publicwhip_data/src/" > /usr/local/lib/python3.10/site-packages/publicwhip_data.pth \
+    && echo "/workspaces/publicwhip_data/src/data_common/src" > /usr/local/lib/python3.10/site-packages/data_common.pth
+
+# special binder instructions
+
+RUN pip install --no-cache-dir notebook
+
+ARG NB_USER=jovyan
+ARG NB_UID=1000
+ENV USER ${NB_USER}
+ENV NB_UID ${NB_UID}
+ENV HOME /home/${NB_USER}
+
+RUN adduser --disabled-password \
+    --gecos "Default user" \
+    --uid ${NB_UID} \
+    ${NB_USER}
+
+COPY . ${HOME}
+USER root
+RUN chown -R ${NB_UID} ${HOME}
+USER ${NB_USER}

Dockerfile.dev

@@ -0,0 +1,15 @@
+FROM ghcr.io/mysociety/data_common:sha-d6eabdb
+
+# Make an empty project directory so the 'self' setup doesn't fail and scripts can be setup
+# Override the .pth created at previous stages to point to where the working directory will land
+COPY pyproject.toml poetry.loc[k] /setup/ 
+COPY src/data_common/pyproject.toml src/data_common/poetry.loc[k] /setup/src/data_common/
+ENV WORKSPACE_NAME publicwhip-data
+RUN mkdir /setup/src/$WORKSPACE_NAME \
+    && touch /setup/src/$WORKSPACE_NAME/__init__.py \
+    && mkdir --parents /setup/src/data_common/src/data_common \
+    && touch /setup/src/data_common/src/data_common/__init__.py \
+    && export PATH="/root/.local/bin:$PATH" \
+    && cd /setup/ && poetry install \
+    && echo "/workspaces/$WORKSPACE_NAME/src/" > /usr/local/lib/python3.10/site-packages/$WORKSPACE_NAME.pth \
+    && echo "/workspaces/$WORKSPACE_NAME/src/data_common/src" > /usr/local/lib/python3.10/site-packages/data_common.pth

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 mySociety
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.