Load cities and streets to test db

.github/workflows/Tests.yml

@@ -65,6 +65,8 @@ jobs:
     - name: DB Initialization
       run: |
         docker exec anyway alembic upgrade head
+        docker exec anyway ./main.py process cities
+        docker exec anyway ./main.py process streets
         docker exec anyway ./main.py process registered-vehicles
         docker exec anyway ./main.py process cbs --source local_dir_for_tests_only
         docker exec anyway ./main.py process road-segments

anyway/models.py

@@ -1099,9 +1099,10 @@ class City(CityFields, Base):
 
     @staticmethod
     def get_name_from_symbol(symbol: int, lang: str = 'he') -> str:
-        res: City = db.session.query(City.heb_name, City.eng_name).filter(City.yishuv_symbol == int(symbol)).first()
+        int_sym = int(symbol)
+        res: City = db.session.query(City.heb_name, City.eng_name).filter(City.yishuv_symbol == int_sym).first()
         if res is None:
-            raise ValueError(f"{symbol}: could not find city with that symbol")
+            raise ValueError(f"{int_sym}({symbol}): could not find city with that symbol")
         return res.heb_name if lang == 'he' else res.eng_name
 
     @staticmethod

anyway/parsers/cities.py

@@ -5,8 +5,8 @@
 from anyway.app_and_db import db
 import logging
 
-CBS_CITIES_RESOURCES_URL = "https://data.gov.il/dataset/citiesandsettelments"
-CBS_CITIES_RESOURCES_ID = "8f714b6f-c35c-4b40-a0e7-547b675eee0e"
+DATA_GOV_CITIES_RESOURCES_URL = "https://data.gov.il/dataset/citiesandsettelments"
+DATA_GOV_CITIES_RESOURCES_ID = "8f714b6f-c35c-4b40-a0e7-547b675eee0e"
 RESOURCE_NAME = "רשימת רחובות בישראל - מתעדכן"
 BASE_GET_DATA_GOV = "https://data.gov.il/dataset/321"
 RESOURCE_DOWNLOAD_TEMPLATE = (
@@ -29,12 +29,12 @@
 CHUNK_SIZE = 1000
 
 
-class UpdateCitiesFromCSB:
+class UpdateCitiesFromDataGov:
     def __init__(self):
         self.s = requests.Session()
 
-    def get_cbs_streets_download_url(self):
-        url = RESOURCE_DOWNLOAD_TEMPLATE.format(id=CBS_CITIES_RESOURCES_ID)
+    def get_streets_download_url(self):
+        url = RESOURCE_DOWNLOAD_TEMPLATE.format(id=DATA_GOV_CITIES_RESOURCES_ID)
         return url
 
     def get_city_data_chunks(self, url: str, chunk_size: int) -> Iterable[List[Dict[str, Any]]]:
@@ -72,8 +72,8 @@ def import_citis_into_db(self, url: str, chunk_size: int):
 
 
 def parse(chunk_size=CHUNK_SIZE):
-    instance = UpdateCitiesFromCSB()
-    res = instance.get_cbs_streets_download_url()
+    instance = UpdateCitiesFromDataGov()
+    res = instance.get_streets_download_url()
     instance.import_citis_into_db(res, chunk_size)
 
 

anyway/parsers/streets.py

@@ -5,7 +5,7 @@
 from anyway.app_and_db import db
 import logging
 
-CBS_STREETS_RESOURCES_URL = "https://data.gov.il/api/3/action/package_show?id=321"
+DATA_GOV_STREETS_RESOURCES_URL = "https://data.gov.il/api/3/action/package_show?id=321"
 RESOURCE_NAME = "רשימת רחובות בישראל - מתעדכן"
 BASE_GET_DATA_GOV = "https://data.gov.il/dataset/321"
 RESOURCE_DOWNLOAD_TEMPLATE = (
@@ -18,12 +18,12 @@
 CHUNK_SIZE = 1000
 
 
-class UpdateStreetsFromCSB:
+class UpdateStreetsFromDataGov:
     def __init__(self):
         self.s = requests.Session()
 
-    def get_cbs_streets_download_url(self):
-        response = self.s.get(CBS_STREETS_RESOURCES_URL)
+    def get_streets_download_url(self):
+        response = self.s.get(DATA_GOV_STREETS_RESOURCES_URL)
         if not response.ok:
             raise Exception(
                 f"Could not get streets url. reason:{response.reason}:{response.status_code}"
@@ -81,8 +81,8 @@ def import_street_file_into_db(self, url: str, chunk_size: int):
 
 
 def parse(chunk_size=CHUNK_SIZE):
-    instance = UpdateStreetsFromCSB()
-    res = instance.get_cbs_streets_download_url()
+    instance = UpdateStreetsFromDataGov()
+    res = instance.get_streets_download_url()
     instance.import_street_file_into_db(res, chunk_size)
 
 

main.py

@@ -156,7 +156,7 @@ def suburban_junctions(filename):
 @process.command()
 @click.argument("chunk-size", type=int, default=1000)
 def streets(chunk_size):
-    """Update streets table from CBS site"""
+    """Update streets table from data.gov site"""
     from anyway.parsers.streets import parse
 
     return parse(chunk_size)
@@ -165,7 +165,7 @@ def streets(chunk_size):
 @process.command()
 @click.argument("chunk-size", type=int, default=1000)
 def cities(chunk_size):
-    """Update cities table from CBS site"""
+    """Update cities table from data.gov site"""
     from anyway.parsers.cities import parse
 
     return parse(chunk_size)