check in existing code to public repo

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 88
+select = C,E,F,W,B,B950
+extend-ignore = E203, E501, W503

.github/workflows/pycov.yaml

@@ -0,0 +1,95 @@
+name: Python Coverage
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11"]
+        poetry-version: ["1.3.1"]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup gcloud Auth
+        uses: google-github-actions/auth@v1
+        with:
+          credentials_json: "${{ secrets.GC_SERVICE_KEY }}"
+
+      - name: Setup gcloud CLI
+        uses: google-github-actions/setup-gcloud@main
+        with:
+          version: "412.0.0"
+          project_id: ${{ secrets.GC_PROJECT_ID }}
+
+      - name: Set up Poetry
+        uses: abatilo/actions-poetry@v2
+        with:
+          poetry-version: ${{ matrix.poetry-version }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
+
+      - name: Install dependencies
+        run: poetry install --with dev
+
+      - name: Run tests
+        run: |
+          # Generate coverage report via tests. Does not matter if the tests
+          # pass here; that's covered by dedicated actions.
+          poetry run coverage run --source blind_charging -m pytest || true
+
+      - name: Generate coverage XML
+        run: poetry run coverage xml
+
+      - name: Interpret coverage report
+        uses: irongut/[email protected]
+        with:
+          filename: coverage.xml
+          badge: true
+          fail_below_min: true
+          format: markdown
+          hide_branch_rate: false
+          hide_complexity: true
+          indicators: true
+          output: both
+          thresholds: '60 80'
+
+      - name: Add Coverage PR Comment
+        uses: marocchino/sticky-pull-request-comment@v2
+        if: github.event_name == 'pull_request'
+        with:
+          recreate: true
+          path: code-coverage-results.md
+
+      - name: Upload coverage artifacts
+        if: github.event_name == 'push'
+        run: |
+          # Get the badge URL from the first line of the coverage comment.
+          BADGE_URL=$(cat code-coverage-results.md | head -1 | sed -r 's/.*\((.+)\).*/\1/')
+          # Download the badge
+          curl $BADGE_URL > badge.svg
+
+          # Generate web assets for coverage report.
+          poetry run coverage html
+
+          # Upload artifacts
+          gsutil cp badge.svg gs://scpl-blind-charging/coverage/badge.svg
+          # Remove any old static assets for coverage
+          gsutil -m rm -r gs://scpl-blind-charging/coverage/www
+          # Upload coverage static assets
+          gsutil -m cp -z html,js,json,css,png -r htmlcov/* gs://scpl-blind-charging/coverage/www/
+          # Make sure no assets get cached
+          gsutil -m setmeta -r -h "Cache-Control: private, max-age=0, no-transform, no-store, no-cache" gs://scpl-blind-charging/coverage/
+          # Publish all the artifacts
+          gsutil -m acl ch -r -u AllUsers:R gs://scpl-blind-charging/coverage/

.github/workflows/pylint.yaml

@@ -0,0 +1,46 @@
+name: Python Lint
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - poetry.lock
+      - pyproject.toml
+      - blind_charging/**
+      - tests/**
+  pull_request:
+    branches: [main]
+    paths:
+      - poetry.lock
+      - pyproject.toml
+      - blind_charging/**
+      - tests/**
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11"]
+        poetry-version: ["1.3.1"]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Poetry
+        uses: abatilo/actions-poetry@v2
+        with:
+          poetry-version: ${{ matrix.poetry-version }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
+
+      - name: Install dependencies
+        run: poetry install --with dev
+
+      - name: Run linters
+        run: poetry run pre-commit run --all-files --show-diff-on-failure

.github/workflows/pytest.yaml

@@ -0,0 +1,46 @@
+name: Python Tests
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - poetry.lock
+      - pyproject.toml
+      - blind_charging/**
+      - tests/**
+  pull_request:
+    branches: [main]
+    paths:
+      - poetry.lock
+      - pyproject.toml
+      - blind_charging/**
+      - tests/**
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        poetry-version: ["1.3.1"]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Poetry
+        uses: abatilo/actions-poetry@v2
+        with:
+          poetry-version: ${{ matrix.poetry-version }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
+
+      - name: Install dependencies
+        run: poetry install --with dev
+
+      - name: Run tests
+        run: poetry run pytest

.gitignore

@@ -0,0 +1,10 @@
+*.pyc
+*.swp
+dist/
+__pycache__/
+htmlcov/
+.pytest_cache/
+.mypy_cache/
+.coverage
+coverage.xml
+badge.svg

.isort.cfg

@@ -0,0 +1,3 @@
+[settings]
+known_third_party = similarity,spacy,unidecode
+profile = black

.pre-commit-config.yaml

@@ -0,0 +1,39 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+    -   id: black
+-   repo: https://github.com/asottile/seed-isort-config
+    rev: v2.2.0
+    hooks:
+    -   id: seed-isort-config
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+    -   id: isort
+-   repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
+    hooks:
+    -   id: flake8
+        additional_dependencies:
+        -   flake8-bugbear
+        -   flake8-comprehensions
+        -   flake8-simplify
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.991
+    hooks:
+    -   id: mypy
+        args: [--ignore-missing-imports]
+-   repo: https://github.com/python-poetry/poetry
+    rev: 1.3.1
+    hooks:
+    -   id: poetry-check
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files

LICENSE

@@ -0,0 +1,7 @@
+Copyright 2022 Computational Policy Lab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,93 @@
+# Blind Charging
+
+[![Python Tests](https://github.com/stanford-policylab/blind-charging/actions/workflows/pytest.yaml/badge.svg?branch=main)](https://github.com/stanford-policylab/blind-charging/actions/workflows/pytest.yaml)
+[![Python Lint](https://github.com/stanford-policylab/blind-charging/actions/workflows/pylint.yaml/badge.svg?branch=main)](https://github.com/stanford-policylab/blind-charging/actions/workflows/pylint.yaml)
+[![Python Coverage](https://storage.googleapis.com/scpl-blind-charging/coverage/badge.svg)](https://storage.googleapis.com/scpl-blind-charging/coverage/www/index.html)
+
+## Installation
+
+The redaction algorithm can be installed using `pip` with the following command:
+
+```bash
+pip install blind_charging
+```
+
+## Basic Usage
+
+The simplest usage of the blind charging algorithm looks like this:
+```py
+import blind_charging as bc
+
+# Configure BlindCharging to use your locale.
+bc.set_locale("Suffix County")
+
+# Run the redaction algorithm passing in:
+#   1) The input police narrative text;
+#   2) A list of civilian (non-peace-officer) names referenced in the narrative;
+#   3) A list of names of peace officers referenced in the narrative.
+#
+# This returns the redacted input narrative.
+civilians = ["Sally Smith"]
+officers = ["Sgt. John Jones"]
+narrative = "Sgt. John Jones arrested Sally Smith (S1) in Parkside."
+bc.redact(narrative, civilians, officers)
+# '<Sergeant #1> arrested <(S1)> in <[neighborhood]>.'
+```
+
+## Locales
+
+TODO(jnu): I'll add details about how to create and deploy the app with new locales.
+Currently the locales for our pilot live in a separate (private) repo.
+
+## Advanced Usage
+
+### Generating annotations for custom display
+
+The basic `redact` function applies redactions and returns the modified text.
+If you would like to apply the redactions yourself (e.g., to present the report with stylized HTML formatting), you can run:
+
+```py
+annotations = bc.annotate(narrative, civilians, officers)
+```
+
+The annotations give character offsets within the text where the annotation should be applied, as well as additional semantic information you can use to annotate the text.
+
+## Development
+
+We use:
+ - Python3 -- we target compatibility for all versions of Python starting with 3.8. (If we want to support Py3.7 we will need to downgrade Pandas, and probably other things.)
+ - [poetry](https://python-poetry.org/) for managing packages and building the library
+ - [pytest](https://docs.pytest.org/en/7.2.x/) for testing
+ - [pre-commit](https://pre-commit.com/) for running formatters and linters
+ - [black](https://github.com/psf/black) for code formatting
+ - [isort](https://pycqa.github.io/isort/) for more code formatting
+ - [flake8](https://flake8.pycqa.org/en/latest/) for linting
+
+### Getting started
+ - Ensure you have Python3 on your computer.
+ - Install [`poetry`](https://python-poetry.org/docs/#installation) if needed
+ - `poetry install --with dev` to install dependencies
+ - `poetry run pre-commit install` to install git pre-commit hooks
+
+### Running tests
+
+Run all tests with pytest:
+```zsh
+poetry run pytest
+```
+
+### Building the library
+
+TKTK workflow for automating this. `pytest build` works in the meantime.
+
+# Contributors
+
+This project was initially developed by a team at the Stanford Computational Policy Lab, including (in alphabetical order):
+ - Alex Chohlas-Wood
+ - Madison Coots
+ - Sharad Goel
+ - Amelia Goodman
+ - Zhiyuan Jerry Lin
+ - Joe Nudell
+ - Julian Nyarko
+ - Keniel Yao