From 2e311b17ed959551bdf48556f50d55a429215886 Mon Sep 17 00:00:00 2001 From: tkalir Date: Wed, 17 Apr 2024 19:32:02 +0300 Subject: [PATCH 1/3] add sending telegram notifications after newsflash location verified --- anyway/views/news_flash/api.py | 80 ++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 37 deletions(-) diff --git a/anyway/views/news_flash/api.py b/anyway/views/news_flash/api.py index 987fc5e2..0705fedf 100644 --- a/anyway/views/news_flash/api.py +++ b/anyway/views/news_flash/api.py @@ -13,7 +13,6 @@ from flask import request, Response, make_response, jsonify from sqlalchemy import and_, not_, or_ - from anyway.app_and_db import db from anyway.backend_constants import ( BE_CONST, @@ -31,14 +30,15 @@ from anyway.parsers import fields_to_resolution, resolution_dict from anyway.models import AccidentMarkerView, InvolvedView from anyway.widgets.widget_utils import get_accidents_stats +from anyway.telegram_accident_notifications import trigger_generate_infographics_and_send_to_telegram from io import BytesIO DEFAULT_OFFSET_REQ_PARAMETER = 0 DEFAULT_LIMIT_REQ_PARAMETER = 100 DEFAULT_NUMBER_OF_YEARS_AGO = 5 -class NewsFlashQuery(BaseModel): +class NewsFlashQuery(BaseModel): id: Optional[int] road_number: Optional[int] offset: Optional[int] = DEFAULT_OFFSET_REQ_PARAMETER @@ -148,16 +148,16 @@ def news_flash_new(args: dict) -> List[dict]: def gen_news_flash_query( - session, - source=None, - start_date=None, - end_date=None, - interurban_only=None, - road_number=None, - road_segment=None, - offset=None, - limit=None, - last_minutes=None + session, + source=None, + start_date=None, + end_date=None, + interurban_only=None, + road_number=None, + road_segment=None, + offset=None, + limit=None, + last_minutes=None ): query = session.query(NewsFlash) # get all possible sources @@ -312,12 +312,12 @@ def normalize_query(params: dict): def update_location_verification_history( - user_id: int, - news_flash_id: int, - prev_location: str, - prev_qualification: int, - new_location: str, - new_qualification: int, + user_id: int, + news_flash_id: int, + prev_location: str, + prev_qualification: int, + new_location: str, + new_qualification: int, ): new_location_qualifiction_history = LocationVerificationHistory( user_id=user_id, @@ -388,7 +388,7 @@ def update_news_flash_qualifying(id): ) else: if ((news_flash_obj.road_segment_name is None) or (news_flash_obj.road1 is None)) and ( - (news_flash_obj.yishuv_name is None) or (news_flash_obj.street1_hebrew is None) + (news_flash_obj.yishuv_name is None) or (news_flash_obj.street1_hebrew is None) ): logging.error("try to set qualification on empty location.") return return_json_error(Es.BR_BAD_FIELD) @@ -407,13 +407,16 @@ def update_news_flash_qualifying(id): new_location=new_location, new_qualification=new_location_qualifiction, ) + if new_location_qualifiction == NewsflashLocationQualification.MANUAL.value and \ + old_location_qualifiction != NewsflashLocationQualification.MANUAL.value: + trigger_generate_infographics_and_send_to_telegram(id, False) return Response(status=HTTPStatus.OK) def get_downloaded_data(format, years_ago): request_params = get_request_params_from_request_values(request.values) end_time = datetime.datetime.now() - start_time = end_time - datetime.timedelta(days=years_ago*365) + start_time = end_time - datetime.timedelta(days=years_ago * 365) columns = OrderedDict() columns[AccidentMarkerView.id] = 'מס תאונה' @@ -466,21 +469,20 @@ def get_downloaded_data(format, years_ago): columns[AccidentMarkerView.x] = 'X קואורדינטה' columns[AccidentMarkerView.y] = 'Y קואורדינטה' - related_accidents = get_accidents_stats( - table_obj=AccidentMarkerView, - columns=columns.keys(), - filters=request_params.location_info, - start_time=start_time, - end_time=end_time - ) + table_obj=AccidentMarkerView, + columns=columns.keys(), + filters=request_params.location_info, + start_time=start_time, + end_time=end_time + ) accident_ids = list(related_accidents['id'].values()) accident_severities = get_accidents_stats( - table_obj=InvolvedView, - group_by=("accident_id", "injury_severity_hebrew"), - count="injury_severity_hebrew", - filters={"accident_id": accident_ids} - ) + table_obj=InvolvedView, + group_by=("accident_id", "injury_severity_hebrew"), + count="injury_severity_hebrew", + filters={"accident_id": accident_ids} + ) severities_hebrew = set() for i, accident_id in enumerate(accident_ids): @@ -498,20 +500,24 @@ def get_downloaded_data(format, years_ago): df.rename(columns={key.name.replace('_hebrew', ''): value for key, value in columns.items()}, inplace=True) index_to_insert_severities = list(columns.values()).index('מהירות מותרת') - output_column_names = list(columns.values())[:index_to_insert_severities] + list(severities_hebrew) + list(columns.values())[index_to_insert_severities:] + output_column_names = list(columns.values())[:index_to_insert_severities] + list(severities_hebrew) + list( + columns.values())[index_to_insert_severities:] df = df[output_column_names] - df.rename(columns={'פצוע קל': 'פצוע/ה קל', 'פצוע בינוני': 'פצוע/ה בינוני', 'פצוע קשה': 'פצוע/ה קשה', 'הרוג': 'הרוג/ה'}, inplace=True) + df.rename( + columns={'פצוע קל': 'פצוע/ה קל', 'פצוע בינוני': 'פצוע/ה בינוני', 'פצוע קשה': 'פצוע/ה קשה', 'הרוג': 'הרוג/ה'}, + inplace=True) if format == 'csv': df.to_csv(buffer, encoding="utf-8") - mimetype ='text/csv' + mimetype = 'text/csv' file_type = 'csv' elif format == 'xlsx': df.to_excel(buffer, encoding="utf-8") - mimetype='application/vnd.ms-excel' + mimetype = 'application/vnd.ms-excel' file_type = 'xlsx' else: raise Exception(f'File format not supported for downloading : {format}') - headers = { 'Content-Disposition': f'attachment; filename=anyway_download_{datetime.datetime.now().strftime("%d_%m_%Y_%H_%M_%S")}.{file_type}' } + headers = { + 'Content-Disposition': f'attachment; filename=anyway_download_{datetime.datetime.now().strftime("%d_%m_%Y_%H_%M_%S")}.{file_type}'} return Response(buffer.getvalue(), mimetype=mimetype, headers=headers) From 86a085385f80df612cae383981f84d7cf33838ce Mon Sep 17 00:00:00 2001 From: tkalir Date: Wed, 17 Apr 2024 19:36:41 +0300 Subject: [PATCH 2/3] add sending telegram notifications after newsflash location verified --- anyway/views/news_flash/api.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/anyway/views/news_flash/api.py b/anyway/views/news_flash/api.py index 0705fedf..250808db 100644 --- a/anyway/views/news_flash/api.py +++ b/anyway/views/news_flash/api.py @@ -13,6 +13,7 @@ from flask import request, Response, make_response, jsonify from sqlalchemy import and_, not_, or_ + from anyway.app_and_db import db from anyway.backend_constants import ( BE_CONST, @@ -37,8 +38,8 @@ DEFAULT_LIMIT_REQ_PARAMETER = 100 DEFAULT_NUMBER_OF_YEARS_AGO = 5 - class NewsFlashQuery(BaseModel): + id: Optional[int] road_number: Optional[int] offset: Optional[int] = DEFAULT_OFFSET_REQ_PARAMETER @@ -199,7 +200,7 @@ def gen_news_flash_query( NewsFlash.accident == True, not_(and_(NewsFlash.lat == 0, NewsFlash.lon == 0)), not_(and_(NewsFlash.lat == None, NewsFlash.lon == None)), - ) + ) ).order_by(NewsFlash.date.desc()) query = query.offset(offset) @@ -230,7 +231,7 @@ def gen_news_flash_query_v2(session, valid_params: dict): NewsFlash.accident == True, not_(and_(NewsFlash.lat == 0, NewsFlash.lon == 0)), not_(and_(NewsFlash.lat == None, NewsFlash.lon == None)), - ) + ) ).order_by(NewsFlash.date.desc()) query = query.offset(valid_params["offset"]) query = query.limit(valid_params["limit"]) @@ -284,14 +285,14 @@ def filter_by_resolutions(query, resolutions: List[str]): and_( NewsFlash.resolution == BE_CONST.ResolutionCategories.SUBURBAN_ROAD.value, NewsFlash.road_segment_name != None, - ) + ) ) if "street" in resolutions: ands.append( and_( NewsFlash.resolution == BE_CONST.ResolutionCategories.STREET.value, NewsFlash.street1_hebrew != None, - ) + ) ) if len(ands) > 1: return query.filter(or_(*ands)) @@ -409,14 +410,14 @@ def update_news_flash_qualifying(id): ) if new_location_qualifiction == NewsflashLocationQualification.MANUAL.value and \ old_location_qualifiction != NewsflashLocationQualification.MANUAL.value: - trigger_generate_infographics_and_send_to_telegram(id, False) + trigger_generate_infographics_and_send_to_telegram(id, False) return Response(status=HTTPStatus.OK) def get_downloaded_data(format, years_ago): request_params = get_request_params_from_request_values(request.values) end_time = datetime.datetime.now() - start_time = end_time - datetime.timedelta(days=years_ago * 365) + start_time = end_time - datetime.timedelta(days=years_ago*365) columns = OrderedDict() columns[AccidentMarkerView.id] = 'מס תאונה' @@ -469,6 +470,7 @@ def get_downloaded_data(format, years_ago): columns[AccidentMarkerView.x] = 'X קואורדינטה' columns[AccidentMarkerView.y] = 'Y קואורדינטה' + related_accidents = get_accidents_stats( table_obj=AccidentMarkerView, columns=columns.keys(), @@ -500,24 +502,20 @@ def get_downloaded_data(format, years_ago): df.rename(columns={key.name.replace('_hebrew', ''): value for key, value in columns.items()}, inplace=True) index_to_insert_severities = list(columns.values()).index('מהירות מותרת') - output_column_names = list(columns.values())[:index_to_insert_severities] + list(severities_hebrew) + list( - columns.values())[index_to_insert_severities:] + output_column_names = list(columns.values())[:index_to_insert_severities] + list(severities_hebrew) + list(columns.values())[index_to_insert_severities:] df = df[output_column_names] - df.rename( - columns={'פצוע קל': 'פצוע/ה קל', 'פצוע בינוני': 'פצוע/ה בינוני', 'פצוע קשה': 'פצוע/ה קשה', 'הרוג': 'הרוג/ה'}, - inplace=True) + df.rename(columns={'פצוע קל': 'פצוע/ה קל', 'פצוע בינוני': 'פצוע/ה בינוני', 'פצוע קשה': 'פצוע/ה קשה', 'הרוג': 'הרוג/ה'}, inplace=True) if format == 'csv': df.to_csv(buffer, encoding="utf-8") - mimetype = 'text/csv' + mimetype ='text/csv' file_type = 'csv' elif format == 'xlsx': df.to_excel(buffer, encoding="utf-8") - mimetype = 'application/vnd.ms-excel' + mimetype='application/vnd.ms-excel' file_type = 'xlsx' else: raise Exception(f'File format not supported for downloading : {format}') - headers = { - 'Content-Disposition': f'attachment; filename=anyway_download_{datetime.datetime.now().strftime("%d_%m_%Y_%H_%M_%S")}.{file_type}'} + headers = { 'Content-Disposition': f'attachment; filename=anyway_download_{datetime.datetime.now().strftime("%d_%m_%Y_%H_%M_%S")}.{file_type}' } return Response(buffer.getvalue(), mimetype=mimetype, headers=headers) From c4195cf94f0f2e94721f8e0d1089cdadddb8caf5 Mon Sep 17 00:00:00 2001 From: tkalir Date: Tue, 23 Apr 2024 19:06:23 +0300 Subject: [PATCH 3/3] fixing dag triggering --- anyway/telegram_accident_notifications.py | 2 +- anyway/views/news_flash/api.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/anyway/telegram_accident_notifications.py b/anyway/telegram_accident_notifications.py index 92595758..abecd469 100644 --- a/anyway/telegram_accident_notifications.py +++ b/anyway/telegram_accident_notifications.py @@ -116,7 +116,7 @@ def create_public_urls_for_infographics_images(folder_name): presigned_urls[infographic_name] = url return presigned_urls -@staticmethod + def trigger_generate_infographics_and_send_to_telegram(newsflash_id, pre_verification_chat=True): dag_conf = {"news_flash_id": newsflash_id} dag_conf["chat_id"] = TELEGRAM_CHANNEL_CHAT_ID if pre_verification_chat \ diff --git a/anyway/views/news_flash/api.py b/anyway/views/news_flash/api.py index 4cf8a8ce..d1bf7f06 100644 --- a/anyway/views/news_flash/api.py +++ b/anyway/views/news_flash/api.py @@ -5,6 +5,7 @@ import json import logging import pandas as pd +import os from typing import List, Optional from http import HTTPStatus @@ -13,7 +14,6 @@ from flask import request, Response, make_response, jsonify from sqlalchemy import and_, not_, or_ - from anyway.app_and_db import db from anyway.backend_constants import ( BE_CONST, @@ -38,8 +38,8 @@ DEFAULT_LIMIT_REQ_PARAMETER = 100 DEFAULT_NUMBER_OF_YEARS_AGO = 5 -class NewsFlashQuery(BaseModel): +class NewsFlashQuery(BaseModel): id: Optional[int] road_number: Optional[int] offset: Optional[int] = DEFAULT_OFFSET_REQ_PARAMETER @@ -407,7 +407,8 @@ def update_news_flash_qualifying(id): new_location=new_location, new_qualification=new_location_qualifiction, ) - if new_location_qualifiction == NewsflashLocationQualification.MANUAL.value and \ + if os.environ.get("FLASK_ENV") == "production" and \ + new_location_qualifiction == NewsflashLocationQualification.MANUAL.value and \ old_location_qualifiction != NewsflashLocationQualification.MANUAL.value: trigger_generate_infographics_and_send_to_telegram(id, False) return Response(status=HTTPStatus.OK) @@ -469,7 +470,6 @@ def get_downloaded_data(format, years_ago): columns[AccidentMarkerView.x] = 'X קואורדינטה' columns[AccidentMarkerView.y] = 'Y קואורדינטה' - related_accidents = get_accidents_stats( table_obj=AccidentMarkerView, columns=columns.keys(),