From 6f48c6c684bb6e852814acfd5c5ca4344cdc8d0c Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Wed, 24 Apr 2024 17:02:23 -0700 Subject: [PATCH 1/3] Limit hourlies endpoint to only store ffmc files --- api/app/auto_spatial_advisory/sfms.py | 4 ++++ api/app/routers/sfms.py | 28 ++++++++++++++------------ api/app/tests/sfms/test_sfms.py | 10 ++++++++- api/app/tests/sfms/test_sfms_upload.py | 27 ++++++++++++++++++++++++- 4 files changed, 54 insertions(+), 15 deletions(-) diff --git a/api/app/auto_spatial_advisory/sfms.py b/api/app/auto_spatial_advisory/sfms.py index d9b1047ee..2f2b2f2a6 100644 --- a/api/app/auto_spatial_advisory/sfms.py +++ b/api/app/auto_spatial_advisory/sfms.py @@ -10,6 +10,10 @@ def is_hfi_file(filename: str) -> bool: "Returns true if filename starts with 'hfi'" return filename.startswith("hfi") +def is_ffmc_file(filename: str) -> bool: + "Returns true if filename starts with 'fine_fuel_moisture_code'" + return filename.startswith("fine_fuel_moisture_code") + def get_date_part(filename: str) -> str: """ Get the date part of the filename. Filename example: hfi20220823.tif diff --git a/api/app/routers/sfms.py b/api/app/routers/sfms.py index a6f619b4a..551a4cc5e 100644 --- a/api/app/routers/sfms.py +++ b/api/app/routers/sfms.py @@ -9,7 +9,7 @@ from app.nats_publish import publish from app.utils.s3 import get_client from app import config -from app.auto_spatial_advisory.sfms import get_hourly_filename, get_sfms_file_message, get_target_filename, get_date_part, is_hfi_file +from app.auto_spatial_advisory.sfms import get_hourly_filename, get_sfms_file_message, get_target_filename, get_date_part, is_ffmc_file, is_hfi_file from app.auto_spatial_advisory.nats_config import stream_name, subjects, sfms_file_subject from app.schemas.auto_spatial_advisory import ManualSFMS, SFMSFile @@ -126,18 +126,20 @@ async def upload_hourlies(file: UploadFile, ``` """ logger.info('sfms/upload/hourlies') - # Get an async S3 client. - async with get_client() as (client, bucket): - # We save the Last-modified and Create-time as metadata in the object store - just - # in case we need to know about it in the future. - key = get_hourly_filename(file.filename) - logger.info('Uploading file "%s" to "%s"', file.filename, key) - meta_data = get_meta_data(request) - await client.put_object(Bucket=bucket, - Key=key, - Body=FileLikeObject(file.file), - Metadata=meta_data) - logger.info('Done uploading file') + + if is_ffmc_file(file.filename): + # Get an async S3 client. + async with get_client() as (client, bucket): + # We save the Last-modified and Create-time as metadata in the object store - just + # in case we need to know about it in the future. + key = get_hourly_filename(file.filename) + logger.info('Uploading file "%s" to "%s"', file.filename, key) + meta_data = get_meta_data(request) + await client.put_object(Bucket=bucket, + Key=key, + Body=FileLikeObject(file.file), + Metadata=meta_data) + logger.info('Done uploading file') return Response(status_code=200) diff --git a/api/app/tests/sfms/test_sfms.py b/api/app/tests/sfms/test_sfms.py index fc3629de6..a050a04ec 100644 --- a/api/app/tests/sfms/test_sfms.py +++ b/api/app/tests/sfms/test_sfms.py @@ -1,4 +1,4 @@ -from app.auto_spatial_advisory.sfms import is_hfi_file +from app.auto_spatial_advisory.sfms import is_hfi_file, is_ffmc_file def test_is_hfi_file(): @@ -7,3 +7,11 @@ def test_is_hfi_file(): def test_is_not_hfi_file(): assert is_hfi_file('at20220824.tiff') is False + + +def test_is_ffmc_file(): + assert is_ffmc_file('fine_fuel_moisture_code20220824.tiff') is True + + +def test_is_not_ffmc_file(): + assert is_ffmc_file('hfi20220824.tiff') is False \ No newline at end of file diff --git a/api/app/tests/sfms/test_sfms_upload.py b/api/app/tests/sfms/test_sfms_upload.py index ff8d95299..e53ce535d 100644 --- a/api/app/tests/sfms/test_sfms_upload.py +++ b/api/app/tests/sfms/test_sfms_upload.py @@ -211,7 +211,7 @@ async def _mock_get_client_for_router(): mock_get_client.return_value = _mock_get_client_for_router() client = TestClient(app) response = client.post(HOURLY_FFMC_URL, - files={'file': ('hfi20220904.tiff', b'')}, + files={'file': ('fine_fuel_moisture_code20220904.tiff', b'')}, headers={ 'Secret': config.get('SFMS_SECRET'), 'Last-modified': datetime.now().isoformat(), @@ -221,4 +221,29 @@ async def _mock_get_client_for_router(): # We should have called put_object once. assert mock_s3_client.put_object.called # We should not publish hourly ffmc + assert mock_publish.called == False + +@patch('app.routers.sfms.get_client') +@patch('app.routers.sfms.publish') +def test_hourly_ffmc_endpoint_non_ffmc_file(mock_publish: AsyncMock, mock_get_client: AsyncMock): + """ Verify we don't store other files besides ffmc files """ + mock_s3_client = AsyncMock() + + @asynccontextmanager + async def _mock_get_client_for_router(): + yield mock_s3_client, 'some_bucket' + + mock_get_client.return_value = _mock_get_client_for_router() + client = TestClient(app) + response = client.post(HOURLY_FFMC_URL, + files={'file': ('hfi20220904.tiff', b'')}, + headers={ + 'Secret': config.get('SFMS_SECRET'), + 'Last-modified': datetime.now().isoformat(), + 'Create-time': datetime.now().isoformat()}) + # We should get a 200 response if the file is uploaded successfully. + assert response.status_code == 200 + # We should have called put_object once. + assert mock_s3_client.put_object.called == False + # We should not publish hourly ffmc assert mock_publish.called == False \ No newline at end of file From 993edff2030651044c1c18cd16c94a71bc4e9fdf Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Thu, 25 Apr 2024 08:44:44 -0700 Subject: [PATCH 2/3] update comment --- api/app/tests/sfms/test_sfms_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/app/tests/sfms/test_sfms_upload.py b/api/app/tests/sfms/test_sfms_upload.py index e53ce535d..b1eadd86c 100644 --- a/api/app/tests/sfms/test_sfms_upload.py +++ b/api/app/tests/sfms/test_sfms_upload.py @@ -243,7 +243,7 @@ async def _mock_get_client_for_router(): 'Create-time': datetime.now().isoformat()}) # We should get a 200 response if the file is uploaded successfully. assert response.status_code == 200 - # We should have called put_object once. + # We should not upload the file assert mock_s3_client.put_object.called == False # We should not publish hourly ffmc assert mock_publish.called == False \ No newline at end of file From 574af16c9d14f5df0c51f81d73c7d36f9878d870 Mon Sep 17 00:00:00 2001 From: Conor Brady Date: Thu, 25 Apr 2024 08:45:06 -0700 Subject: [PATCH 3/3] update comment --- api/app/tests/sfms/test_sfms_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/app/tests/sfms/test_sfms_upload.py b/api/app/tests/sfms/test_sfms_upload.py index b1eadd86c..2edcaa906 100644 --- a/api/app/tests/sfms/test_sfms_upload.py +++ b/api/app/tests/sfms/test_sfms_upload.py @@ -241,7 +241,7 @@ async def _mock_get_client_for_router(): 'Secret': config.get('SFMS_SECRET'), 'Last-modified': datetime.now().isoformat(), 'Create-time': datetime.now().isoformat()}) - # We should get a 200 response if the file is uploaded successfully. + # We should get a 200 response if the file is received assert response.status_code == 200 # We should not upload the file assert mock_s3_client.put_object.called == False