From e109a6048f744410c0540c6cdf16147fb763c519 Mon Sep 17 00:00:00 2001 From: ziv Date: Sun, 4 Jun 2023 18:05:12 +0300 Subject: [PATCH 1/4] Adding table road_junction_km, update from cbs files. --- ...7ea883c8a245_add_road_junction_km_table.py | 36 +++++++++++++++++++ anyway/models.py | 15 ++++++++ anyway/parsers/cbs/executor.py | 35 ++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 alembic/versions/7ea883c8a245_add_road_junction_km_table.py diff --git a/alembic/versions/7ea883c8a245_add_road_junction_km_table.py b/alembic/versions/7ea883c8a245_add_road_junction_km_table.py new file mode 100644 index 000000000..b49fff7b3 --- /dev/null +++ b/alembic/versions/7ea883c8a245_add_road_junction_km_table.py @@ -0,0 +1,36 @@ +"""add suburban junction table + +Revision ID: 7ea883c8a245 +Revises: d9cb98c752d4 +Create Date: 2023-06-04 17:43:13.170728 + +""" + +# revision identifiers, used by Alembic. +revision = '7ea883c8a245' +down_revision = 'd9cb98c752d4' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +# noinspection PyUnresolvedReferences +def upgrade(): + op.create_table("road_junction_km", + sa.Column('road', sa.BigInteger(), nullable=False), + sa.Column('non_urban_intersection', sa.Integer(), nullable=False), + sa.Column('km', sa.Float(), nullable=False), + sa.PrimaryKeyConstraint('road', 'non_urban_intersection') + ) + op.create_index('road_junction_km_idx', + "road_junction_km", + ['road', 'non_urban_intersection'], + unique=True) + + +# noinspection PyUnresolvedReferences +def downgrade(): + op.drop_index(op.f('road_junction_km_idx'), table_name="road_junction_km") + op.drop_table("road_junction_km") diff --git a/anyway/models.py b/anyway/models.py index 54c51b827..294abfb8d 100755 --- a/anyway/models.py +++ b/anyway/models.py @@ -1258,6 +1258,21 @@ def serialize(self): } +class RoadJunctionKM(Base): + __tablename__ = "road_junction_km" + MAX_NAME_LEN = 100 + road = Column(Integer(), primary_key=True, nullable=False) + non_urban_intersection = Column(Integer(), primary_key=True, nullable=False) + km = Column(Float(), nullable=False) + + def serialize(self): + return { + "road": self.road, + "non_urban_intersection": self.non_urban_intersection, + "km": self.km, + } + + class RegisteredVehicle(Base): __tablename__ = "cities_vehicles_registered" id = Column(Integer(), primary_key=True) diff --git a/anyway/parsers/cbs/executor.py b/anyway/parsers/cbs/executor.py index edc08eed8..e070a67d1 100644 --- a/anyway/parsers/cbs/executor.py +++ b/anyway/parsers/cbs/executor.py @@ -34,6 +34,7 @@ RoadSign, RoadLight, RoadControl, + RoadJunctionKM, Weather, RoadSurface, RoadObjecte, @@ -570,6 +571,7 @@ def import_accidents(provider_code, accidents, streets, roads, non_urban_interse for _, accident in accidents.iterrows(): marker = create_marker(provider_code, accident, streets, roads, non_urban_intersection) add_suburban_junction_from_marker(marker) + add_road_junction_km_from_marker(marker) accidents_result.append(marker) db.session.bulk_insert_mappings(AccidentMarker, accidents_result) db.session.commit() @@ -795,6 +797,8 @@ def import_streets_into_db(): yishuv_name_dict: Dict[Tuple[int, str], int] = {} suburban_junctions_dict: Dict[int, dict] = {} SUBURBAN_JUNCTION = "suburban_junction" +# (road, junction) -> km +road_junction_km_dict: Dict[Tuple[int, int], int] = {} def load_existing_streets(): @@ -903,6 +907,35 @@ def add_suburban_junction_from_marker(marker: dict): add_suburban_junction(j) +def load_existing_road_junction_km_data(): + rows: List[RoadJunctionKM] = db.session.query(RoadJunctionKM).all() + tmp = {(r.road, r.non_urban_intersection): r.km for r in rows} + road_junction_km_dict.update(tmp) + logging.debug(f"Loaded road-junction-km rows: {len(tmp)}.") + + +def import_road_junction_km_into_db(): + items = [{"road": k[0], "non_urban_intersection": k[1], "km": v} for + k, v in road_junction_km_dict.items()] + logging.debug( + f"Writing to db: {len(items)} road junction km rows" + ) + db.session.query(RoadJunctionKM).delete() + db.session.bulk_insert_mappings(RoadJunctionKM, items) + db.session.commit() + logging.debug(f"Done.") + + +def add_road_junction_km_from_marker(marker: dict): + intersection = marker[NON_URBAN_INTERSECTION] + if intersection is not None: + k, v = (marker["road1"], intersection), marker["km"] + exists = road_junction_km_dict.get(k) + if exists is not None and exists != v: + logging.warning(f"Changed road junction km: from {exists} to {v}.") + road_junction_km_dict[k] = v + + def delete_invalid_entries(batch_size): """ deletes all markers in the database with null latitude or longitude @@ -1178,6 +1211,7 @@ def main(batch_size, source, load_start_year=None): try: load_existing_streets() load_existing_suburban_junctions() + load_existing_road_junction_km_data() total = 0 started = datetime.now() if source == "s3": @@ -1235,6 +1269,7 @@ def main(batch_size, source, load_start_year=None): import_streets_into_db() import_suburban_junctions_into_db() + import_road_junction_km_into_db() fill_db_geo_data() From 3612e6a032e90e3eeed47c1dec145090d7cddb28 Mon Sep 17 00:00:00 2001 From: ziv Date: Tue, 3 Oct 2023 21:38:17 +0300 Subject: [PATCH 2/4] Just a comment --- tests/test_infographics_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_infographics_utils.py b/tests/test_infographics_utils.py index aad650094..cef4eb1a5 100644 --- a/tests/test_infographics_utils.py +++ b/tests/test_infographics_utils.py @@ -2,7 +2,7 @@ from anyway.widgets.widget_utils import format_2_level_items from anyway.backend_constants import AccidentSeverity - +# comment class TestInfographicsUtilsCase(unittest.TestCase): item1 = { "2019": { From 90609a569a372dc3ee134cdd38a110f6fe8e0f2e Mon Sep 17 00:00:00 2001 From: ziv Date: Fri, 1 Dec 2023 13:06:42 +0200 Subject: [PATCH 3/4] Add sql debug log --- anyway/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anyway/config.py b/anyway/config.py index c21ef4630..f14d46e28 100644 --- a/anyway/config.py +++ b/anyway/config.py @@ -12,7 +12,7 @@ SQLALCHEMY_TRACK_MODIFICATIONS = True ENTRIES_PER_PAGE = os.environ.get("ENTRIES_PER_PAGE", 1000) SQLALCHEMY_ENGINE_OPTIONS = {} -# SQLALCHEMY_ECHO = True +SQLALCHEMY_ECHO = True # available languages LANGUAGES = {"en": "English", "he": "עברית"} From 40f322c4177e48c9836fd36c89ca80ca9f8ed09c Mon Sep 17 00:00:00 2001 From: ziv Date: Fri, 1 Dec 2023 18:09:43 +0200 Subject: [PATCH 4/4] Fix down revision of new alembic file. --- .gitignore | 3 +++ alembic/versions/7ea883c8a245_add_road_junction_km_table.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f4f54cae0..c21f8ce58 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,9 @@ develop-eggs lib lib64 +# newsflash-ifografics +anyway-newsflash-infographics/anyway-newsflash-infographics/ + # Installer logs pip-log.txt diff --git a/alembic/versions/7ea883c8a245_add_road_junction_km_table.py b/alembic/versions/7ea883c8a245_add_road_junction_km_table.py index b49fff7b3..71cbfe3ee 100644 --- a/alembic/versions/7ea883c8a245_add_road_junction_km_table.py +++ b/alembic/versions/7ea883c8a245_add_road_junction_km_table.py @@ -1,14 +1,14 @@ """add suburban junction table Revision ID: 7ea883c8a245 -Revises: d9cb98c752d4 +Revises: 664f8a93794e Create Date: 2023-06-04 17:43:13.170728 """ # revision identifiers, used by Alembic. revision = '7ea883c8a245' -down_revision = 'd9cb98c752d4' +down_revision = '664f8a93794e' branch_labels = None depends_on = None