diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index fe243255..9d86a737 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -65,6 +65,8 @@ jobs: - name: DB Initialization run: | docker exec anyway alembic upgrade head + docker exec anyway ./main.py process cities + docker exec anyway ./main.py process streets docker exec anyway ./main.py process registered-vehicles docker exec anyway ./main.py process cbs --source local_dir_for_tests_only docker exec anyway ./main.py process road-segments diff --git a/anyway/models.py b/anyway/models.py index 6307d8a3..80f248b9 100755 --- a/anyway/models.py +++ b/anyway/models.py @@ -1099,9 +1099,10 @@ class City(CityFields, Base): @staticmethod def get_name_from_symbol(symbol: int, lang: str = 'he') -> str: - res: City = db.session.query(City.heb_name, City.eng_name).filter(City.yishuv_symbol == int(symbol)).first() + int_sym = int(symbol) + res: City = db.session.query(City.heb_name, City.eng_name).filter(City.yishuv_symbol == int_sym).first() if res is None: - raise ValueError(f"{symbol}: could not find city with that symbol") + raise ValueError(f"{int_sym}({symbol}): could not find city with that symbol") return res.heb_name if lang == 'he' else res.eng_name @staticmethod diff --git a/anyway/parsers/cities.py b/anyway/parsers/cities.py index e469195c..da658b49 100644 --- a/anyway/parsers/cities.py +++ b/anyway/parsers/cities.py @@ -5,8 +5,8 @@ from anyway.app_and_db import db import logging -CBS_CITIES_RESOURCES_URL = "https://data.gov.il/dataset/citiesandsettelments" -CBS_CITIES_RESOURCES_ID = "8f714b6f-c35c-4b40-a0e7-547b675eee0e" +DATA_GOV_CITIES_RESOURCES_URL = "https://data.gov.il/dataset/citiesandsettelments" +DATA_GOV_CITIES_RESOURCES_ID = "8f714b6f-c35c-4b40-a0e7-547b675eee0e" RESOURCE_NAME = "רשימת רחובות בישראל - מתעדכן" BASE_GET_DATA_GOV = "https://data.gov.il/dataset/321" RESOURCE_DOWNLOAD_TEMPLATE = ( @@ -29,12 +29,12 @@ CHUNK_SIZE = 1000 -class UpdateCitiesFromCSB: +class UpdateCitiesFromDataGov: def __init__(self): self.s = requests.Session() - def get_cbs_streets_download_url(self): - url = RESOURCE_DOWNLOAD_TEMPLATE.format(id=CBS_CITIES_RESOURCES_ID) + def get_streets_download_url(self): + url = RESOURCE_DOWNLOAD_TEMPLATE.format(id=DATA_GOV_CITIES_RESOURCES_ID) return url def get_city_data_chunks(self, url: str, chunk_size: int) -> Iterable[List[Dict[str, Any]]]: @@ -72,8 +72,8 @@ def import_citis_into_db(self, url: str, chunk_size: int): def parse(chunk_size=CHUNK_SIZE): - instance = UpdateCitiesFromCSB() - res = instance.get_cbs_streets_download_url() + instance = UpdateCitiesFromDataGov() + res = instance.get_streets_download_url() instance.import_citis_into_db(res, chunk_size) diff --git a/anyway/parsers/streets.py b/anyway/parsers/streets.py index fdd4388b..33f312cc 100644 --- a/anyway/parsers/streets.py +++ b/anyway/parsers/streets.py @@ -5,7 +5,7 @@ from anyway.app_and_db import db import logging -CBS_STREETS_RESOURCES_URL = "https://data.gov.il/api/3/action/package_show?id=321" +DATA_GOV_STREETS_RESOURCES_URL = "https://data.gov.il/api/3/action/package_show?id=321" RESOURCE_NAME = "רשימת רחובות בישראל - מתעדכן" BASE_GET_DATA_GOV = "https://data.gov.il/dataset/321" RESOURCE_DOWNLOAD_TEMPLATE = ( @@ -18,12 +18,12 @@ CHUNK_SIZE = 1000 -class UpdateStreetsFromCSB: +class UpdateStreetsFromDataGov: def __init__(self): self.s = requests.Session() - def get_cbs_streets_download_url(self): - response = self.s.get(CBS_STREETS_RESOURCES_URL) + def get_streets_download_url(self): + response = self.s.get(DATA_GOV_STREETS_RESOURCES_URL) if not response.ok: raise Exception( f"Could not get streets url. reason:{response.reason}:{response.status_code}" @@ -81,8 +81,8 @@ def import_street_file_into_db(self, url: str, chunk_size: int): def parse(chunk_size=CHUNK_SIZE): - instance = UpdateStreetsFromCSB() - res = instance.get_cbs_streets_download_url() + instance = UpdateStreetsFromDataGov() + res = instance.get_streets_download_url() instance.import_street_file_into_db(res, chunk_size) diff --git a/main.py b/main.py index af58346d..91c86874 100755 --- a/main.py +++ b/main.py @@ -156,7 +156,7 @@ def suburban_junctions(filename): @process.command() @click.argument("chunk-size", type=int, default=1000) def streets(chunk_size): - """Update streets table from CBS site""" + """Update streets table from data.gov site""" from anyway.parsers.streets import parse return parse(chunk_size) @@ -165,7 +165,7 @@ def streets(chunk_size): @process.command() @click.argument("chunk-size", type=int, default=1000) def cities(chunk_size): - """Update cities table from CBS site""" + """Update cities table from data.gov site""" from anyway.parsers.cities import parse return parse(chunk_size)