Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add FastAPI Redis Caching #4195

Closed
wants to merge 20 commits into from
Closed
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
1849fd3
feat: get logins working with drill
ThomasLaPiana Sep 29, 2023
1d60191
feat: add more endpoints
ThomasLaPiana Sep 29, 2023
5229381
feat: separate each endpoint into its own task
ThomasLaPiana Sep 29, 2023
220f060
feat: optimize the Drill test
ThomasLaPiana Sep 29, 2023
eb0d7e7
feat: make the privacy-experience endpoint async
ThomasLaPiana Sep 29, 2023
800a944
feat: sprinkle some `async sleep` magic into the privacy-experience e…
ThomasLaPiana Sep 29, 2023
abe119c
feat: remove an unused requirement
ThomasLaPiana Sep 29, 2023
2ae8afb
fix: static checks
ThomasLaPiana Sep 29, 2023
1f1f12b
Add FastAPI Redis Caching
ThomasLaPiana Sep 29, 2023
6b33377
Merge branch 'add-more-drill-endpoints' into ThomasLaPiana-add-fastap…
ThomasLaPiana Sep 30, 2023
ffe5a7e
Merge branch 'main' into ThomasLaPiana-add-fastapi-redis-caching
ThomasLaPiana Oct 11, 2023
1347246
fix: remove outdated redis import
ThomasLaPiana Oct 11, 2023
161e271
feat: add caching to the privacy-experience endpoint
ThomasLaPiana Oct 11, 2023
38f32bc
feat: rough POC of custom caching for the privacy experience endpoint
ThomasLaPiana Oct 12, 2023
4d50fd9
feat: additional cleanup
ThomasLaPiana Oct 12, 2023
fdf1d95
Revert "feat: additional cleanup"
ThomasLaPiana Oct 12, 2023
cdb6e49
feat: cleanup, passing more tests
ThomasLaPiana Oct 12, 2023
be70d0c
feat: add headers to the privacy-experience response object indicatin…
ThomasLaPiana Oct 13, 2023
2b64527
checkin: add headers and cache tests, but not passing
ThomasLaPiana Oct 13, 2023
a8a45a2
fix: more tests and almost all static checks
ThomasLaPiana Oct 13, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
checkin: add headers and cache tests, but not passing
ThomasLaPiana committed Oct 13, 2023
commit 2b64527d4ac629ab5375805fbb42c778fe027632
10 changes: 8 additions & 2 deletions src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py
Original file line number Diff line number Diff line change
@@ -45,6 +45,8 @@

router = APIRouter(tags=["Privacy Experience"], prefix=urls.V1_URL_PREFIX)

BUST_CACHE_HEADER = "bust-endpoint-cache"
CACHE_HEADER = "X-Endpoint-Cache"
PRIVACY_EXPERIENCE_CACHE: Dict[str, Dict] = {}


@@ -162,6 +164,7 @@ async def privacy_experience_list(
:param response:
:return:
"""
global PRIVACY_EXPERIENCE_CACHE

# These are the parameters that get used to create the cache.
param_hash_list = [
@@ -178,13 +181,16 @@ async def privacy_experience_list(
# Create a custom hash that avoids unhashable parameters
cache_hash = "_".join([repr(x) for x in param_hash_list])

if request.headers.get(BUST_CACHE_HEADER):
PRIVACY_EXPERIENCE_CACHE.clear()

if cache_hash in PRIVACY_EXPERIENCE_CACHE.keys():
logger.debug("Cache HIT: {}", cache_hash)
response.headers["X-Endpoint-Cache"] = "HIT"
response.headers[CACHE_HEADER] = "HIT"
return PRIVACY_EXPERIENCE_CACHE[cache_hash]
else:
logger.debug("Cache MISS: {}", cache_hash)
response.headers["x-Endpoint-Cache"] = "MISS"
response.headers[CACHE_HEADER] = "MISS"

fides_user_provided_identity: Optional[ProvidedIdentity] = None
if fides_user_device_id:
61 changes: 59 additions & 2 deletions tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,41 @@
from __future__ import annotations
from typing import Dict

import pytest
from starlette.status import HTTP_200_OK
from starlette.testclient import TestClient

from fides.api.api.v1.endpoints.privacy_experience_endpoints import (
_filter_experiences_by_region_or_country,
BUST_CACHE_HEADER,
CACHE_HEADER,
)
from fides.api.models.privacy_experience import ComponentType, PrivacyExperience
from fides.api.models.privacy_notice import ConsentMechanism
from fides.common.api.v1.urn_registry import PRIVACY_EXPERIENCE, V1_URL_PREFIX


def get_cache_bust_headers() -> Dict:
return {BUST_CACHE_HEADER: "true"}


class TestGetPrivacyExperiencesCaching:
def test_cache_header_hit(self, url, api_client):
"""Check that the header describing cache hits/misses is working."""
api_client.get(url)
resp = api_client.get(url)
cache_header = resp.headers.get(CACHE_HEADER)
assert cache_header
assert cache_header == "HIT"

def test_bust_cache_header(self, url, api_client):
"""Check that the header to bust the cache is working."""
resp = api_client.get(url, headers=get_cache_bust_headers())
cache_header = resp.headers.get(CACHE_HEADER)
assert cache_header
assert cache_header == "MISS"


class TestGetPrivacyExperiences:
@pytest.fixture(scope="function")
def url(self) -> str:
@@ -47,7 +71,7 @@ def test_get_privacy_experiences(
privacy_notice,
privacy_experience_privacy_center,
):
unescape_header = {"Unescape-Safestr": "true"}
unescape_header = {"Unescape-Safestr": "true", BUST_CACHE_HEADER: "true"}

resp = api_client.get(url + "?include_gvl=True", headers=unescape_header)
assert resp.status_code == 200
@@ -98,7 +122,7 @@ def test_get_experiences_unescaped(
privacy_experience_privacy_center,
):
# Assert not escaped without proper request header
resp = api_client.get(url)
resp = api_client.get(url, headers=get_cache_bust_headers())
resp = resp.json()["items"][0]
experience_config = resp["experience_config"]
assert (
@@ -120,6 +144,7 @@ def test_get_privacy_experiences_show_disabled_filter(
):
resp = api_client.get(
url,
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -128,6 +153,7 @@ def test_get_privacy_experiences_show_disabled_filter(

resp = api_client.get(
url + "?show_disabled=False",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -144,6 +170,7 @@ def test_get_privacy_experiences_show_disabled_filter(
privacy_experience_overlay.unlink_experience_config(db)
resp = api_client.get(
url + "?show_disabled=False",
headers=get_cache_bust_headers(),
)
data = resp.json()
assert (
@@ -162,6 +189,7 @@ def test_get_privacy_experiences_region_filter(
):
resp = api_client.get(
url + "?region=us_co",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -172,6 +200,7 @@ def test_get_privacy_experiences_region_filter(

resp = api_client.get(
url + "?region=us_ca",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -184,6 +213,7 @@ def test_get_privacy_experiences_region_filter(

resp = api_client.get(
url + "?region=bad_region",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
assert resp.json()["total"] == 0
@@ -204,12 +234,14 @@ def assert_france_experience_and_notices_returned(resp):

response = api_client.get(
url + "?region=fr_idg",
headers=get_cache_bust_headers(),
) # There are no experiences with "fr_idg" so we fell back to searching for "fr"

assert_france_experience_and_notices_returned(response)

response = api_client.get(
url + "?region=FR-IDG",
headers=get_cache_bust_headers(),
) # Case insensitive and hyphens also work here -"

assert_france_experience_and_notices_returned(response)
@@ -224,6 +256,7 @@ def test_get_privacy_experiences_components_filter(
):
resp = api_client.get(
url + "?component=overlay",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -236,6 +269,7 @@ def test_get_privacy_experiences_components_filter(

resp = api_client.get(
url + "?component=privacy_center",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -247,6 +281,7 @@ def test_get_privacy_experiences_components_filter(

resp = api_client.get(
url + "?component=tcf_overlay",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -258,6 +293,7 @@ def test_get_privacy_experiences_components_filter(

resp = api_client.get(
url + "?component=bad_type",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 422

@@ -270,6 +306,7 @@ def test_get_privacy_experiences_has_notices_no_notices(
):
resp = api_client.get(
url + "?has_notices=True",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -286,6 +323,7 @@ def test_get_privacy_experiences_has_notices_no_regions_overlap(
):
resp = api_client.get(
url + "?has_notices=True",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -309,6 +347,7 @@ def test_get_privacy_experiences_has_notices(
):
resp = api_client.get(
url + "?has_notices=True",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -409,6 +448,7 @@ def assert_expected_filtered_region_response(data):
# Filter on exact match region
resp = api_client.get(
url + "?has_notices=True&region=us_co",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
response_json = resp.json()
@@ -418,6 +458,7 @@ def assert_expected_filtered_region_response(data):
# Filter on upper case and hyphens
resp = api_client.get(
url + "?has_notices=True&region=US-CO",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
assert_expected_filtered_region_response(resp.json())
@@ -442,6 +483,7 @@ def test_filter_on_systems_applicable(
"""For systems applicable filter, notices are only embedded if they are relevant to a system"""
resp = api_client.get(
url + "?region=us_co",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -466,6 +508,7 @@ def test_filter_on_systems_applicable(

resp = api_client.get(
url + "?region=us_co&systems_applicable=True",
headers=get_cache_bust_headers(),
)
notices = resp.json()["items"][0]["privacy_notices"]
assert len(notices) == 1
@@ -502,6 +545,7 @@ def test_filter_on_notices_and_region_and_show_disabled_is_false(

resp = api_client.get(
url + "?has_notices=True&region=us_ca&show_disabled=False",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -528,6 +572,7 @@ def test_get_privacy_experiences_show_has_config_filter(
):
resp = api_client.get(
url + "?has_config=False",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -536,6 +581,7 @@ def test_get_privacy_experiences_show_has_config_filter(

resp = api_client.get(
url + "?has_config=True",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -552,6 +598,7 @@ def test_get_privacy_experiences_show_has_config_filter(
privacy_experience_privacy_center.save(db=db)
resp = api_client.get(
url + "?has_config=False",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -566,6 +613,7 @@ def test_get_privacy_experiences_bad_fides_user_device_id_filter(
):
resp = api_client.get(
url + "?fides_user_device_id=does_not_exist",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 422
assert resp.json()["detail"] == "Invalid fides user device id format"
@@ -583,6 +631,7 @@ def test_get_privacy_experiences_nonexistent_fides_user_device_id_filter(
):
resp = api_client.get(
url + "?cd685ccd-0960-4dc1-b9ca-7e810ebc5c1b",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()
@@ -615,6 +664,7 @@ def test_get_privacy_experiences_fides_user_device_id_filter(
):
resp = api_client.get(
url + "?fides_user_device_id=051b219f-20e4-45df-82f7-5eb68a00889f",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()["items"][0]
@@ -639,6 +689,7 @@ def test_get_privacy_experiences_fides_user_device_id_filter(
assert privacy_notice_us_ca_provide.description == "new_description"
resp = api_client.get(
url + "?fides_user_device_id=051b219f-20e4-45df-82f7-5eb68a00889f",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
data = resp.json()["items"][0]
@@ -668,6 +719,7 @@ def test_tcf_not_enabled(
):
resp = api_client.get(
url + "?region=fr&component=overlay&include_gvl=True&include_meta=True",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
assert len(resp.json()["items"]) == 1
@@ -705,6 +757,7 @@ def test_tcf_enabled_but_no_relevant_systems(
):
resp = api_client.get(
url + "?region=fr&component=overlay&include_gvl=True&include_meta=True",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
assert len(resp.json()["items"]) == 1
@@ -731,6 +784,7 @@ def test_tcf_enabled_but_no_relevant_systems(
# Has notices = True flag will keep this experience from appearing altogether
resp = api_client.get(
url + "?region=fr&has_notices=True",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
assert len(resp.json()["items"]) == 0
@@ -756,6 +810,7 @@ def test_tcf_enabled_with_overlapping_vendors(
resp = api_client.get(
url
+ "?region=fr&component=overlay&fides_user_device_id=051b219f-20e4-45df-82f7-5eb68a00889f&has_notices=True&include_gvl=True&include_meta=True",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
assert len(resp.json()["items"]) == 1
@@ -851,6 +906,7 @@ def test_tcf_enabled_with_overlapping_systems(
resp = api_client.get(
url
+ "?region=fr&component=overlay&fides_user_device_id=051b219f-20e4-45df-82f7-5eb68a00889f&has_notices=True",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
assert len(resp.json()["items"]) == 1
@@ -914,6 +970,7 @@ def test_tcf_enabled_with_legitimate_interest_purpose(
resp = api_client.get(
url
+ "?region=fr&component=overlay&fides_user_device_id=051b219f-20e4-45df-82f7-5eb68a00889f&has_notices=True",
headers=get_cache_bust_headers(),
)
assert resp.status_code == 200
assert len(resp.json()["items"]) == 1