diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f09deda..8716222 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,6 +31,10 @@ jobs: pip3 install -r requirements.txt pip3 install -r requirements-dev.txt python3 setup.py install + mkdir -p /opt/csv2bufr + cd /opt/csv2bufr + wget https://github.com/wmo-im/csv2bufr-templates/archive/refs/tags/v0.1.tar.gz + tar -zxf v0.1.tar.gz --strip-components=1 csv2bufr-templates-0.1/templates - name: run tests ⚙️ run: | pytest diff --git a/csv2bufr/__init__.py b/csv2bufr/__init__.py index f58803e..9e4ce39 100644 --- a/csv2bufr/__init__.py +++ b/csv2bufr/__init__.py @@ -28,6 +28,7 @@ import json import logging import os.path +import threading from typing import Any, Iterator, Union from eccodes import (codes_bufr_new_from_samples, @@ -38,7 +39,7 @@ codes_bufr_keys_iterator_delete, codes_bufr_keys_iterator_get_name, CodesInternalError) -from jsonschema import validate +import csv2bufr.templates as c2bt # some 'constants' SUCCESS = True @@ -56,9 +57,6 @@ LOGGER = logging.getLogger(__name__) -THISDIR = os.path.dirname(os.path.realpath(__file__)) -MAPPINGS = f"{THISDIR}{os.sep}resources{os.sep}mappings" - BUFR_TABLE_VERSION = 38 # default BUFR table version # list of BUFR attributes ATTRIBUTES = ['code', 'units', 'scale', 'reference', 'width'] @@ -94,7 +92,7 @@ 'typicalSecond': 'const:0' } -_warnings = [] +_warnings_global = {} # status codes FAILED = 0 @@ -124,7 +122,8 @@ # function to find position in array of requested element def index_(key, mapping): - global _warnings + global _warnings_global + tidx = f"t-{threading.get_ident()}" idx = 0 for item in mapping: if item['eccodes_key'] == key: @@ -133,7 +132,7 @@ def index_(key, mapping): if NULLIFY_INVALID: msg = f"Warning: key {key} not found in {mapping}" LOGGER.warning(msg) - _warnings.append(msg) + _warnings_global[tidx].append(msg) return None else: msg = f"Error: key {key} not found in {mapping}" @@ -142,7 +141,9 @@ def index_(key, mapping): def parse_value(element: str, data: dict): - global _warnings + global _warnings_global + tidx = f"t-{threading.get_ident()}" + data_type = element.split(":") if data_type[0] == "const": value = data_type[1] @@ -156,7 +157,7 @@ def parse_value(element: str, data: dict): msg = f"Column {column} not found in input data: {data}" if NULLIFY_INVALID: LOGGER.warning(msg) # noqa - _warnings.append(msg) + _warnings_global[tidx].append(msg) else: # LOGGER.error(msg) # noqa raise ValueError(msg) @@ -182,7 +183,8 @@ def parse_value(element: str, data: dict): # function to retrieve data def get_(key: str, mapping: dict, data: dict): - global _warnings + global _warnings_global + tidx = f"t-{threading.get_ident()}" # get position in mapping try: idx = index_(key, mapping) @@ -192,40 +194,13 @@ def get_(key: str, mapping: dict, data: dict): msg = f"Warning ({e}) raised getting value for {key}, None returned for {key}" # noqa if NULLIFY_INVALID: LOGGER.warning(msg) # noqa - _warnings.append(msg) + _warnings_global[tidx].append(msg) value = None else: raise KeyError(msg) return value -# function to validate mapping file against JSON schema -def validate_mapping(mapping: dict) -> bool: - """ - Validates dictionary containing mapping to BUFR against internal schema. - Returns True if the dictionary passes and raises an error otherwise. - - :param mapping: dictionary containing mappings to specified BUFR - sequence using ecCodes key. - - :returns: `bool` of validation result - """ - global _warnings - # load internal file schema for mappings - file_schema = f"{MAPPINGS}{os.sep}mapping_schema.json" - with open(file_schema) as fh: - schema = json.load(fh) - - # now validate - try: - validate(mapping, schema) - except Exception as e: - msg = f"Warning ({e}). Invalid BUFR template mapping file: {mapping}" - raise RuntimeError(msg) - - return SUCCESS - - def apply_scaling(value: Union[NUMBERS], scale: Union[NUMBERS], offset: Union[NUMBERS]) -> Union[NUMBERS]: """ @@ -270,7 +245,8 @@ def validate_value(key: str, value: Union[NUMBERS], :returns: validated value """ - global _warnings + global _warnings_global + tidx = f"t-{threading.get_ident()}" # TODO move this function to the class as part of set value if value is None: @@ -284,7 +260,7 @@ def validate_value(key: str, value: Union[NUMBERS], msg = f"{key}: Value ({value}) out of valid range ({valid_min} - {valid_max})." # noqa if nullify_on_fail: LOGGER.warning(f"{msg}; Element set to missing") - _warnings.append(f"{msg}; Element set to missing") + _warnings_global[tidx].append(f"{msg}; Element set to missing") return None else: raise ValueError(msg) @@ -309,7 +285,7 @@ def __init__(self, descriptors: list, :param table_version: version of Master Table 0 to use, default 36 """ - global _warnings + self.warnings = [] # ================================ # first create empty bufr messages # ================================ @@ -380,7 +356,6 @@ def __init__(self, descriptors: list, # ============================================ def create_template(self) -> None: - global _warnings template = {} template["inputDelayedDescriptorReplicationFactor"] = \ self.delayed_replications @@ -450,6 +425,7 @@ def reset(self) -> None: for key in self.dict: self.dict[key]["value"] = None self.bufr = None + self.warnings = [] def set_element(self, key: str, value: object) -> None: """ @@ -460,7 +436,6 @@ def set_element(self, key: str, value: object) -> None: :returns: `None` """ - global _warnings # TODO move value validation here if value is not None and not isinstance(value, list): @@ -475,7 +450,7 @@ def set_element(self, key: str, value: object) -> None: if NULLIFY_INVALID: value = None LOGGER.warning(f"{e}: Unable to convert value {value} to int for {key}, set to None") # noqa - _warnings.append(f"{e}: Unable to convert value {value} to int for {key}, set to None") # noqa + self.warnings.append(f"{e}: Unable to convert value {value} to int for {key}, set to None") # noqa else: raise RuntimeError(f"{e}: Unable to convert value {value} to int for {key}") # noqa elif expected_type == "float" and not isinstance(value, float): @@ -485,7 +460,7 @@ def set_element(self, key: str, value: object) -> None: if NULLIFY_INVALID: value = None LOGGER.warning(f"{e}: Unable to convert value {value} to float for {key}, set to None") # noqa - _warnings.append(f"{e}: Unable to convert value {value} to float for {key}, set to None") # noqa + self.warnings.append(f"{e}: Unable to convert value {value} to float for {key}, set to None") # noqa else: raise RuntimeError(f"{e}: Unable to convert value {value} to float for {key}") # noqa else: @@ -500,7 +475,6 @@ def get_element(self, key: str) -> Any: :returns: value of the element """ - global _warnings result = None try: # check if we want value or an attribute (indicated by ->) @@ -514,7 +488,7 @@ def get_element(self, key: str) -> Any: if NULLIFY_INVALID: result = None LOGGER.warning(f"Error {e} whilst fetching {key} from data, None returned") # noqa - _warnings.append(f"Error {e} whilst fetching {key} from data, None returned") # noqa + self.warnings.append(f"Error {e} whilst fetching {key} from data, None returned") # noqa else: msg = f"Error {e} whilst fetching {key} from data." raise RuntimeError(msg) @@ -530,7 +504,6 @@ def as_bufr(self, use_cached: bool = False) -> bytes: :returns: bytes containing BUFR data """ - global _warnings if use_cached and (self.bufr is not None): return self.bufr # =========================== @@ -570,7 +543,7 @@ def as_bufr(self, use_cached: bool = False) -> bytes: except CodesInternalError as e: msg = f"Error ({e}) calling codes_set({bufr_msg}, 'pack', True). Null message returned" # noqa LOGGER.warning(f"{msg}") # noqa - _warnings.append(f"{msg}") # noqa + self.warnings.append(f"{msg}") # noqa codes_release(bufr_msg) return self.bufr except Exception as e: @@ -609,7 +582,6 @@ def md5(self) -> str: :returns: md5 of BUFR message """ - global _warnings return self._hash def parse(self, data: dict, mappings: dict) -> None: @@ -628,7 +600,6 @@ def parse(self, data: dict, mappings: dict) -> None: # ================================================== # Parse the data. # ================================================== - global _warnings for section in ("header", "data"): for element in mappings[section]: # get eccodes key @@ -664,7 +635,7 @@ def parse(self, data: dict, mappings: dict) -> None: except Exception as e: if NULLIFY_INVALID: LOGGER.warning(f"Error raised whilst validating {element['eccodes_key']}, value set to None\ndata: {data}") # noqa - _warnings.append(f"Error raised whilst validating {element['eccodes_key']}, value set to None\ndata: {data}") # noqa + self.warnings.append(f"Error raised whilst validating {element['eccodes_key']}, value set to None\ndata: {data}") # noqa value = None else: # LOGGER.error(f"Error raised whilst validating {element['eccodes_key']}, raising error") # noqa @@ -704,7 +675,6 @@ def get_datetime(self) -> datetime: :returns: `datetime.datetime` of ISO8601 representation of the characteristic date/time """ - global _warnings if None in [ self.get_element("typicalYear"), self.get_element("typicalMonth"), @@ -763,11 +733,14 @@ def transform(data: str, mappings: dict) -> Iterator[dict]: :returns: iterator """ - global _warnings + global _warnings_global + job_id = f"t-{threading.get_ident()}" # job ID based on thread + _warnings_global[job_id] = [] # ====================== # validate mapping files # ====================== - e = validate_mapping(mappings) + e = c2bt.validate_template(mappings) + if e is not SUCCESS: raise ValueError("Invalid mappings") @@ -826,7 +799,7 @@ def transform(data: str, mappings: dict) -> Iterator[dict]: if _delimiter not in [",", ";", "|", "\t"]: msg = "Invalid delimiter specified in mapping template, reverting to comma ','" # noqa LOGGER.warning(msg) - _warnings.append(msg) + _warnings_global[job_id].append(msg) _delimiter = "," else: _delimiter = DELIMITER @@ -894,7 +867,7 @@ def transform(data: str, mappings: dict) -> Iterator[dict]: if NULLIFY_INVALID: msg = f"csv read error, non ASCII data detected ({val}), skipping row" # noqa LOGGER.warning(msg) # noqa - _warnings.append(msg) + _warnings_global[job_id].append(msg) LOGGER.debug(row) continue else: @@ -942,7 +915,7 @@ def transform(data: str, mappings: dict) -> Iterator[dict]: "code": PASSED, "message": "", "errors": [], - "warnings": _warnings + "warnings": message.warnings + _warnings_global[job_id] } cksum = message.md5() # now identifier based on WSI and observation date as identifier @@ -977,7 +950,7 @@ def transform(data: str, mappings: dict) -> Iterator[dict]: "code": FAILED, "message": "Error encoding row, BUFR set to None", "errors": [f"{msg}\n\t\tData: {data_dict}"], - "warnings": _warnings + "warnings": message.warnings + _warnings_global[job_id] } result["_meta"] = { "id": None, @@ -1000,6 +973,6 @@ def transform(data: str, mappings: dict) -> Iterator[dict]: # now yield result back to caller yield result # clear warnings - _warnings = [] + del _warnings_global[job_id] fh.close() diff --git a/csv2bufr/cli.py b/csv2bufr/cli.py index 8798ab0..11e539c 100644 --- a/csv2bufr/cli.py +++ b/csv2bufr/cli.py @@ -77,9 +77,10 @@ def mappings(): @click.command('list') @click.pass_context def list_mappings(ctx): - for mapping in os.listdir(MAPPINGS): - msg = f"{mapping} => {MAPPINGS}{os.sep}{mapping}" - click.echo(msg) + templates = c2bt.list_templates() + click.echo(json.dumps(templates)) + for tmpl in templates.items(): + click.echo(json.dumps(tmpl, indent=4)) @click.command('create') diff --git a/csv2bufr/templates/__init__.py b/csv2bufr/templates/__init__.py index 20fe6b1..47b0bbf 100644 --- a/csv2bufr/templates/__init__.py +++ b/csv2bufr/templates/__init__.py @@ -24,18 +24,45 @@ from pathlib import Path from typing import Union -TEMPLATE_DIRS = [] +from jsonschema import validate +THISDIR = os.path.dirname(os.path.realpath(__file__)) LOGGER = logging.getLogger(__name__) +SCHEMA = f"{THISDIR}{os.sep}resources{os.sep}schema" +TEMPLATE_DIRS = [Path("./")] + +_SUCCESS_ = True + +# check if originating centre and subcentre are set as env , default to 255 +ORIGINATING_CENTRE = os.environ.get('BUFR_ORIGINATING_CENTRE', 65535) +ORIGINATING_SUBCENTRE = os.environ.get('BUFR_ORIGINATING_SUBCENTRE', 65535) + +if ORIGINATING_CENTRE is None: + msg = "Invalid BUFR originating centre, please ensure the BUFR_ORIGINATING_CENTRE is set to a valid value" # noqa + LOGGER.error(msg) + raise RuntimeError(msg) + +if ORIGINATING_SUBCENTRE is None: + msg = "Invalid BUFR originating subcentre, please ensure the BUFR_ORIGINATING_SUBCENTRE is set to a valid value" # noqa + LOGGER.error(msg) + raise RuntimeError(msg) + + +if Path("/opt/csv2bufr/templates").exists(): + TEMPLATE_DIRS.append(Path("/opt/csv2bufr/templates")) # Set user defined location first if 'CSV2BUFR_TEMPLATES' in os.environ: TEMPLATE_DIRS.append(Path(os.environ['CSV2BUFR_TEMPLATES'])) +else: + LOGGER.warning(f"""CSV2BUFR_TEMPLATES is not set, default search path(s) + will be used ({TEMPLATE_DIRS}).""") -# Now add defaults -TEMPLATE_DIRS.append(Path(__file__).resolve().parent / 'resources') +# Dictionary to store template filename and label (if assigned) +TEMPLATES = {} +# function to load template by name def load_template(template_name: str) -> Union[dict, None]: """ Checks whether specified template exists and loads file. @@ -48,39 +75,117 @@ def load_template(template_name: str) -> Union[dict, None]: not found. """ template = None - # iterate over directories and load file - for dir_ in TEMPLATE_DIRS: - try: - template_file = dir_ / f"{template_name}.json" - if template_file.is_file(): - with template_file.open() as fh: - template = json.load(fh) - break - except Exception as e: - LOGGER.warning(f"Error raised loading csv2bufr templates: {e}.") - - if template is None: - LOGGER.warning(f"Requested template '{template_name}' not found." + - f" Search path = {TEMPLATE_DIRS}. Please update " + + msg = False + if template_name not in TEMPLATES: + msg = f"Requested template '{template_name}' not found." +\ + f" Search path = {TEMPLATE_DIRS}. Please update " +\ "search path (e.g. 'export CSV2BUFR_TEMPLATE=...')" - ) + else: + fname = TEMPLATES[template_name].get('path') + if fname is None: + msg = f"Error loading template {template_name}, no path found" + else: + with open(fname) as fh: + template = json.load(fh) + + if msg: + raise RuntimeError(msg) + else: + # update template originating centre and subcentre + ocset = False + oscset = False + for hidx in range(len(template['header'])): + if template['header'][hidx]["eccodes_key"] == "bufrHeaderCentre": + template['header'][hidx]["eccodes_key"]["value"] = \ + f"const:{ORIGINATING_CENTRE}" + ocset = True + if template['header'][hidx]["eccodes_key"] == "bufrHeaderSubCentre": # noqa + template['header'][hidx]["eccodes_key"]["value"] = \ + f"const:{ORIGINATING_SUBCENTRE}" + oscset = True + + if not ocset: + template['header'].append( + {"eccodes_key": "bufrHeaderCentre", + "value": f"const:{ORIGINATING_CENTRE}"}) - return template + if not oscset: + template['header'].append( + {"eccodes_key": "bufrHeaderSubCentre", + "value": f"const:{ORIGINATING_SUBCENTRE}"}) + return template -def list_templates() -> list: + +def validate_template(mapping: dict) -> bool: """ - :returns: List of known templates in search path (CSV2BUFR_TEMPLATES). - An empty list is return if no templates are found. + Validates dictionary containing mapping to BUFR against internal schema. + Returns True if the dictionary passes and raises an error otherwise. + + :param mapping: dictionary containing mappings to specified BUFR + sequence using ecCodes key. + + :returns: `bool` of validation result """ - templates = [] + # load internal file schema for mappings + file_schema = f"{SCHEMA}{os.sep}csv2bufr-template-v2.json" + with open(file_schema) as fh: + schema = json.load(fh) + + # now validate + try: + validate(mapping, schema) + except Exception as e: + msg = f"Exception ({e}). Invalid BUFR template mapping file: {mapping}" + raise RuntimeError(msg) + + return _SUCCESS_ + + +def index_templates() -> bool: for dir_ in TEMPLATE_DIRS: - try: - for template in dir_.iterdir(): + for template in dir_.iterdir(): + try: if template.suffix == ".json": - templates.append(template.stem) - except Exception as e: - LOGGER.warning(f"Error raised listing csv2bufr templates: {e}." + - "Directory skipped.") + # check if valid mapping file + with template.open() as fh: + tmpl = json.load(fh) + if 'csv2bufr-template-v2.json' not in tmpl.get("conformsTo",[]): # noqa + LOGGER.warning("'csv2bufr-template-v2.json' not found in " + # noqa + f"conformsTo for file {template}, skipping") # noqa + continue + if validate_template(tmpl) == _SUCCESS_: + # get label if exists else set to empty string + fname = str(template) + id = tmpl['metadata'].get("id", "") + if id in TEMPLATES: + pass + else: + TEMPLATES[id] = { + "label": tmpl['metadata'].get("label", ""), + "description": tmpl['metadata'].get("description", ""), # noqa + "version": tmpl['metadata'].get("version", ""), + "author": tmpl['metadata'].get("author", ""), + "dateCreated": tmpl['metadata'].get("dateCreated", ""), # noqa + "id": tmpl['metadata'].get("id", ""), + "path": fname + } + + except Exception as e: + print(dir_) + LOGGER.warning(f"Warning raised indexing csv2bufr templates: {e}, skipping file {template}.") # noqa + + return _SUCCESS_ + + +def list_templates() -> dict: + """ + :returns: Dictionary of known templates in search path + (CSV2BUFR_TEMPLATES). An empty dictionary is return if no + templates are found. + """ + return TEMPLATES + - return templates +if index_templates() != _SUCCESS_: + LOGGER.error("Error indexing csv2bufr templates, see logs") diff --git a/csv2bufr/templates/resources/aws-template.json b/csv2bufr/templates/resources/aws-template.json deleted file mode 100644 index 7012161..0000000 --- a/csv2bufr/templates/resources/aws-template.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "inputShortDelayedDescriptorReplicationFactor": [], - "inputDelayedDescriptorReplicationFactor": [1,1], - "inputExtendedDelayedDescriptorReplicationFactor": [], - "number_header_rows": 1, - "column_names_row": 1, - "quoting": "QUOTE_NONE", - "header":[ - {"eccodes_key": "edition", "value": "const:4"}, - {"eccodes_key": "masterTableNumber", "value": "const:0"}, - {"eccodes_key": "bufrHeaderCentre", "value": "const:0"}, - {"eccodes_key": "bufrHeaderSubCentre", "value": "const:0"}, - {"eccodes_key": "updateSequenceNumber", "value": "const:0"}, - {"eccodes_key": "dataCategory", "value": "const:0"}, - {"eccodes_key": "internationalDataSubCategory", "value": "const:2"}, - {"eccodes_key": "masterTablesVersionNumber", "value": "const:30"}, - {"eccodes_key": "numberOfSubsets", "value": "const:1"}, - {"eccodes_key": "observedData", "value": "const:1"}, - {"eccodes_key": "compressedData", "value": "const:0"}, - {"eccodes_key": "typicalYear", "value": "data:year"}, - {"eccodes_key": "typicalMonth", "value": "data:month"}, - {"eccodes_key": "typicalDay", "value": "data:day"}, - {"eccodes_key": "typicalHour", "value": "data:hour"}, - {"eccodes_key": "typicalMinute", "value": "data:minute"}, - {"eccodes_key": "unexpandedDescriptors", "value":"array:301150, 307096"} - ], - "data": [ - {"eccodes_key": "#1#wigosIdentifierSeries", "value":"data:wsi_series", "valid_min": "const:0", "valid_max": "const:0"}, - {"eccodes_key": "#1#wigosIssuerOfIdentifier", "value":"data:wsi_issuer", "valid_min": "const:0", "valid_max": "const:65534"}, - {"eccodes_key": "#1#wigosIssueNumber", "value":"data:wsi_issue_number", "valid_min": "const:0", "valid_max": "const:65534"}, - {"eccodes_key": "#1#wigosLocalIdentifierCharacter", "value":"data:wsi_local"}, - {"eccodes_key": "#1#latitude", "value": "data:latitude", "valid_min": "const:-90.0", "valid_max": "const:90.0"}, - {"eccodes_key": "#1#longitude", "value": "data:longitude", "valid_min": "const:-180.0", "valid_max": "const:180.0"}, - {"eccodes_key": "#1#heightOfStationGroundAboveMeanSeaLevel", "value":"data:station_height_above_msl", "valid_min": "const:-400", "valid_max": "const:9000"}, - {"eccodes_key": "#1#heightOfBarometerAboveMeanSeaLevel", "value":"data:barometer_height_above_msl", "valid_min": "const:-400", "valid_max": "const:9000"}, - {"eccodes_key": "#1#blockNumber", "value": "data:wmo_block_number", "valid_min": "const:0", "valid_max": "const:99"}, - {"eccodes_key": "#1#stationNumber", "value": "data:wmo_station_number", "valid_min": "const:0", "valid_max": "const:999"}, - {"eccodes_key": "#1#stationType", "value": "data:station_type", "valid_min": "const:0", "valid_max": "const:3"}, - {"eccodes_key": "#1#year", "value": "data:year", "valid_min": "const:1600", "valid_max": "const:2200"}, - {"eccodes_key": "#1#month", "value": "data:month", "valid_min": "const:1", "valid_max": "const:12"}, - {"eccodes_key": "#1#day", "value": "data:day", "valid_min": "const:1", "valid_max": "const:31"}, - {"eccodes_key": "#1#hour", "value": "data:hour", "valid_min": "const:0", "valid_max": "const:23"}, - {"eccodes_key": "#1#minute", "value": "data:minute", "valid_min": "const:0", "valid_max": "const:59"}, - {"eccodes_key": "#1#nonCoordinatePressure", "value": "data:station_pressure", "valid_min": "const:50000", "valid_max": "const:150000"}, - {"eccodes_key": "#1#pressureReducedToMeanSeaLevel", "value": "data:msl_pressure", "valid_min": "const:50000", "valid_max": "const:150000"}, - {"eccodes_key": "#1#nonCoordinateGeopotentialHeight", "value": "data:geopotential_height", "valid_min": "const:-1000", "valid_max": "const:130071"}, - {"eccodes_key": "#1#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value": "data:thermometer_height", "valid_min": "const:0", "valid_max": "const:655.35"}, - {"eccodes_key": "#1#airTemperature", "value": "data:air_temperature", "valid_min": "const:193.15", "valid_max": "const:333.15"}, - {"eccodes_key": "#1#dewpointTemperature", "value": "data:dewpoint_temperature", "valid_min": "const:193.15", "valid_max": "const:308.15"}, - {"eccodes_key": "#1#relativeHumidity", "value": "data:relative_humidity", "valid_min": "const:0", "valid_max": "const:100"}, - {"eccodes_key": "#1#methodOfStateOfGroundMeasurement", "value": "data:method_of_ground_state_measurement", "valid_min": "const:0", "valid_max": "const:15"}, - {"eccodes_key": "#1#stateOfGround", "value": "data:ground_state", "valid_min": "const:0", "valid_max": "const:31"}, - {"eccodes_key": "#1#methodOfSnowDepthMeasurement", "value": "data:method_of_snow_depth_measurement", "valid_min": "const:0", "valid_max": "const:15"}, - {"eccodes_key": "#1#totalSnowDepth", "value": "data:snow_depth", "valid_min": "const:-0.02", "valid_max": "const:655.33"}, - {"eccodes_key": "#1#precipitationIntensityHighAccuracy", "value": "data:precipitation_intensity", "valid_min": "const:-0.00001", "valid_max": "const:0.65534"}, - {"eccodes_key": "#8#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value": "data:anemometer_height", "valid_min": "const:0", "valid_max": "const:655.35"}, - {"eccodes_key": "#3#timeSignificance", "value": "const:2", "valid_min": "const:2", "valid_max": "const:2"}, - {"eccodes_key": "#6#timePeriod", "value": "data:time_period_of_wind", "valid_min": "const:-10", "valid_max": "const:0"}, - {"eccodes_key": "#1#windDirection", "value": "data:wind_direction", "valid_min": "const:0", "valid_max": "const:360"}, - {"eccodes_key": "#1#windSpeed", "value": "data:wind_speed", "valid_min": "const:0", "valid_max": "const:409.5"}, - {"eccodes_key": "#7#timePeriod", "value": "const:-10", "valid_min": "const:-10", "valid_max": "const:-10"}, - {"eccodes_key": "#1#maximumWindGustDirection", "value": "data:maximum_wind_gust_direction_10_minutes", "valid_min": "const:0", "valid_max": "const:360"}, - {"eccodes_key": "#1#maximumWindGustSpeed", "value": "data:maximum_wind_gust_speed_10_minutes", "valid_min": "const:0", "valid_max": "const:409.5"}, - {"eccodes_key": "#8#timePeriod", "value": "const:-60", "valid_min": "const:-60", "valid_max": "const:-60"}, - {"eccodes_key": "#2#maximumWindGustDirection", "value": "data:maximum_wind_gust_direction_1_hour", "valid_min": "const:0", "valid_max": "const:360"}, - {"eccodes_key": "#2#maximumWindGustSpeed", "value": "data:maximum_wind_gust_speed_1_hour", "valid_min": "const:0", "valid_max": "const:409.5"}, - {"eccodes_key": "#9#timePeriod", "value": "const:-180", "valid_min": "const:-180", "valid_max": "const:-180"}, - {"eccodes_key": "#3#maximumWindGustDirection", "value": "data:maximum_wind_gust_direction_3_hours", "valid_min": "const:0", "valid_max": "const:360"}, - {"eccodes_key": "#3#maximumWindGustSpeed", "value": "data:maximum_wind_gust_speed_3_hours", "valid_min": "const:0", "valid_max": "const:409.5"}, - {"eccodes_key": "#8#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value": "data:rain_sensor_height", "valid_min": "const:0", "valid_max": "const:655.35"}, - {"eccodes_key": "#17#timePeriod", "value": "const:-1", "valid_min": "const:-1", "valid_max": "const:-1"}, - {"eccodes_key": "#1#totalPrecipitationOrTotalWaterEquivalent", "value": "data:total_precipitation_1_hour", "valid_min": "const:-0.1", "valid_max": "const:1638.2"}, - {"eccodes_key": "#18#timePeriod", "value": "const:-3", "valid_min": "const:-3", "valid_max": "const:-3"}, - {"eccodes_key": "#2#totalPrecipitationOrTotalWaterEquivalent", "value": "data:total_precipitation_3_hours", "valid_min": "const:-0.1", "valid_max": "const:1638.2"}, - {"eccodes_key": "#19#timePeriod", "value": "const:-6", "valid_min": "const:-6", "valid_max": "const:-6"}, - {"eccodes_key": "#3#totalPrecipitationOrTotalWaterEquivalent", "value": "data:total_precipitation_6_hours", "valid_min": "const:-0.1", "valid_max": "const:1638.2"}, - {"eccodes_key": "#20#timePeriod", "value": "const:-12", "valid_min": "const:-12", "valid_max": "const:-12"}, - {"eccodes_key": "#4#totalPrecipitationOrTotalWaterEquivalent", "value": "data:total_precipitation_12_hours", "valid_min": "const:-0.1", "valid_max": "const:1638.2"}, - {"eccodes_key": "#21#timePeriod", "value": "const:-24", "valid_min": "const:-24", "valid_max": "const:-24"}, - {"eccodes_key": "#5#totalPrecipitationOrTotalWaterEquivalent", "value": "data:total_precipitation_24_hours", "valid_min": "const:-0.1", "valid_max": "const:1638.2"} - ] -} \ No newline at end of file diff --git a/csv2bufr/templates/resources/daycli-template.json b/csv2bufr/templates/resources/daycli-template.json deleted file mode 100644 index 0338e2e..0000000 --- a/csv2bufr/templates/resources/daycli-template.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "inputShortDelayedDescriptorReplicationFactor": [], - "inputDelayedDescriptorReplicationFactor": [], - "inputExtendedDelayedDescriptorReplicationFactor": [], - "number_header_rows": 1, - "column_names_row": 1, - "wigos_station_identifier": "data:wigos_station_identifier", - "header": [ - {"eccodes_key": "edition", "value": "const:4"}, - {"eccodes_key": "masterTableNumber", "value": "const:0"}, - {"eccodes_key": "bufrHeaderCentre", "value": "const:0"}, - {"eccodes_key": "bufrHeaderSubCentre", "value": "const:0"}, - {"eccodes_key": "updateSequenceNumber", "value": "const:0"}, - {"eccodes_key": "dataCategory", "value": "const:0"}, - {"eccodes_key": "internationalDataSubCategory", "value": "const:6"}, - {"eccodes_key": "masterTablesVersionNumber", "value": "const:38"}, - {"eccodes_key": "typicalYear", "value": "data:year"}, - {"eccodes_key": "typicalMonth", "value": "data:month"}, - {"eccodes_key": "typicalDay", "value": "data:day"}, - {"eccodes_key": "typicalHour", "value": "const:23"}, - {"eccodes_key": "typicalMinute", "value": "const:59"}, - {"eccodes_key": "numberOfSubsets", "value": "const:1"}, - {"eccodes_key": "observedData", "value": "const:1"}, - {"eccodes_key": "compressedData", "value": "const:0"}, - {"eccodes_key": "unexpandedDescriptors", "value": "array:307075"} - ], - "data": [ - {"eccodes_key": "#1#wigosIdentifierSeries", "value": "data:wsi_series"}, - {"eccodes_key": "#1#wigosIssuerOfIdentifier", "value": "data:wsi_issuer"}, - {"eccodes_key": "#1#wigosIssueNumber", "value": "data:wsi_issue"}, - {"eccodes_key": "#1#wigosLocalIdentifierCharacter", "value": "data:wsi_local"}, - {"eccodes_key": "#1#latitude", "value": "data:lat", "valid_min": "const:-90.0", "valid_max": "const:90.0"}, - {"eccodes_key": "#1#longitude", "value": "data:lon", "valid_min": "const:-180.0", "valid_max": "const:180.0"}, - {"eccodes_key": "#1#heightOfStationGroundAboveMeanSeaLevel", "value": "data:elev"}, - {"eccodes_key": "#1#methodUsedToCalculateTheAverageDailyTemperature", "value": "const:0"}, - {"eccodes_key": "#1#year", "value": "data:year", "valid_min": "const:1800", "valid_max": "const:2100"}, - {"eccodes_key": "#1#month", "value": "data:month", "valid_min": "const:1", "valid_max": "const:12"}, - {"eccodes_key": "#1#day", "value": "data:day", "valid_min": "const:1", "valid_max": "const:31"}, - {"eccodes_key": "#1#timePeriod", "value": "const:0"}, - {"eccodes_key": "#1#hour", "value": "const:0"}, - {"eccodes_key": "#1#minute", "value": "const:0"}, - {"eccodes_key": "#1#minute", "value": "const:0"}, - {"eccodes_key": "#1#second", "value": "const:0"}, - {"eccodes_key": "#1#totalAccumulatedPrecipitation", "value": "data:total_precipitation_or_total_water_equivalent"}, - {"eccodes_key": "#1#totalAccumulatedPrecipitation->associatedField", "value": "const:7"}, - {"eccodes_key": "#1#totalAccumulatedPrecipitation->associatedField->associatedFieldSignificance", "value": "const:5"}, - {"eccodes_key": "#2#timePeriod", "value": "const:0"}, - {"eccodes_key": "#2#hour", "value": "const:0"}, - {"eccodes_key": "#2#minute", "value": "const:0"}, - {"eccodes_key": "#2#second", "value": "const:0"}, - {"eccodes_key": "#1#depthOfFreshSnow->associatedField", "value": "const:5"}, - {"eccodes_key": "#1#depthOfFreshSnow->associatedField->associatedFieldSignificance", "value": "const:5"}, - {"eccodes_key": "#3#timePeriod", "value": "const:0"}, - {"eccodes_key": "#3#hour", "value": "const:0"}, - {"eccodes_key": "#3#minute", "value": "const:0"}, - {"eccodes_key": "#3#second", "value": "const:0"}, - {"eccodes_key": "#1#totalSnowDepth->associatedField", "value": "const:5"}, - {"eccodes_key": "#1#totalSnowDepth->associatedField->associatedFieldSignificance", "value": "const:5"}, - {"eccodes_key": "#4#timePeriod", "value": "const:0"}, - {"eccodes_key": "#4#hour", "value": "const:0"}, - {"eccodes_key": "#4#minute", "value": "const:0"}, - {"eccodes_key": "#4#second", "value": "const:0"}, - {"eccodes_key": "#1#firstOrderStatistics", "value": "const:2"}, - {"eccodes_key": "#1#airTemperature", "value": "data:maximum_temperature_at_height_and_over_period_specified", "scale": "const:0", "offset": "const:273.15"}, - {"eccodes_key": "#1#airTemperature->associatedField", "value": "const:7"}, - {"eccodes_key": "#1#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"}, - {"eccodes_key": "#5#timePeriod", "value": "const:0"}, - {"eccodes_key": "#5#hour", "value": "const:0"}, - {"eccodes_key": "#5#minute", "value": "const:0"}, - {"eccodes_key": "#5#second", "value": "const:0"}, - {"eccodes_key": "#2#firstOrderStatistics", "value": "const:3"}, - {"eccodes_key": "#2#airTemperature", "value": "data:minimum_temperature_at_height_and_over_period_specified", "scale": "const:0", "offset": "const:273.15"}, - {"eccodes_key": "#2#airTemperature->associatedField", "value": "const:7"}, - {"eccodes_key": "#2#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"}, - {"eccodes_key": "#6#timePeriod", "value": "const:0"}, - {"eccodes_key": "#6#hour", "value": "const:0"}, - {"eccodes_key": "#6#minute", "value": "const:0"}, - {"eccodes_key": "#6#second", "value": "const:0"}, - {"eccodes_key": "#3#firstOrderStatistics", "value": "const:4"}, - {"eccodes_key": "#3#airTemperature", "value": "data:mean_air_temperature", "scale": "const:0", "offset": "const:273.15"}, - {"eccodes_key": "#3#airTemperature->associatedField", "value": "const:7"}, - {"eccodes_key": "#3#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"} - ], -} \ No newline at end of file diff --git a/csv2bufr/templates/resources/daycli.json b/csv2bufr/templates/resources/daycli.json deleted file mode 100644 index aa64fde..0000000 --- a/csv2bufr/templates/resources/daycli.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "inputShortDelayedDescriptorReplicationFactor": [], - "inputDelayedDescriptorReplicationFactor": [], - "inputExtendedDelayedDescriptorReplicationFactor": [], - "number_header_rows": 1, - "column_names_row": 1, - "quoting": "QUOTE_NONE", - "header": [ - {"eccodes_key": "edition", "value": "const:4"}, - {"eccodes_key": "masterTableNumber", "value": "const:0"}, - {"eccodes_key": "bufrHeaderCentre", "value": "const:0"}, - {"eccodes_key": "bufrHeaderSubCentre", "value": "const:0"}, - {"eccodes_key": "updateSequenceNumber", "value": "const:0"}, - {"eccodes_key": "dataCategory", "value": "const:0"}, - {"eccodes_key": "internationalDataSubCategory", "value": "const:21"}, - {"eccodes_key": "masterTablesVersionNumber", "value": "const:38"}, - {"eccodes_key": "typicalYear", "value": "data:year"}, - {"eccodes_key": "typicalMonth", "value": "data:month"}, - {"eccodes_key": "typicalDay", "value": "data:day"}, - {"eccodes_key": "typicalHour", "value": "const:0"}, - {"eccodes_key": "typicalMinute","value": "const:0"}, - {"eccodes_key": "typicalSecond", "value": "const:0"}, - {"eccodes_key": "numberOfSubsets", "value": "const:1"}, - {"eccodes_key": "observedData", "value": "const:1"}, - {"eccodes_key": "compressedData", "value": "const:0"}, - {"eccodes_key": "unexpandedDescriptors", "value": "array:307075"} - ], - "data": [ - {"eccodes_key": "#1#wigosIdentifierSeries", "value": "data:wsi_series"}, - {"eccodes_key": "#1#wigosIssuerOfIdentifier", "value": "data:wsi_issuer"}, - {"eccodes_key": "#1#wigosIssueNumber", "value": "data:wsi_issue_number"}, - {"eccodes_key": "#1#wigosLocalIdentifierCharacter", "value": "data:wsi_local_identifier"}, - {"eccodes_key": "#1#blockNumber", "value": "data:wmo_block_number"}, - {"eccodes_key": "#1#stationNumber", "value": "data:wmo_station_number"}, - {"eccodes_key": "#1#latitude", "value": "data:latitude", "valid_min": "const:-90.0", "valid_max": "const:90.0"}, - {"eccodes_key": "#1#longitude", "value": "data:longitude", "valid_min": "const:-180.0", "valid_max": "const:180.0"}, - {"eccodes_key": "#1#heightOfStationGroundAboveMeanSeaLevel", "value": "data:station_elevation"}, - {"eccodes_key": "#1#sitingAndMeasurementQualityClassificationForTemperature", "value": "data:temperature_siting_classification", "valid_min": "const:0", "valid_max": "const:255"}, - {"eccodes_key": "#1#sitingAndMeasurementQualityClassificationForPrecipitation", "value": "data:precipitation_siting_classification", "valid_min": "const:0", "valid_max": "const:255"}, - {"eccodes_key": "#1#methodUsedToCalculateTheAverageDailyTemperature", "value": "data:averaging_method"}, - {"eccodes_key": "#1#year", "value": "data:year", "valid_min": "const:1800", "valid_max": "const:2100"}, - {"eccodes_key": "#1#month", "value": "data:month", "valid_min": "const:1", "valid_max": "const:12"}, - {"eccodes_key": "#1#day", "value": "data:day", "valid_min": "const:1", "valid_max": "const:31"}, - {"eccodes_key": "#1#timePeriod", "value": "data:precipitation_day_offset"}, - {"eccodes_key": "#1#hour", "value": "data:precipitation_hour"}, - {"eccodes_key": "#1#minute", "value": "data:precipitation_minute"}, - {"eccodes_key": "#1#second", "value": "data:precipitation_second"}, - {"eccodes_key": "#1#totalAccumulatedPrecipitation", "value": "data:precipitation"}, - {"eccodes_key": "#1#totalAccumulatedPrecipitation->associatedField", "value": "data:precipitation_flag"}, - {"eccodes_key": "#1#totalAccumulatedPrecipitation->associatedField->associatedFieldSignificance", "value": "const:5"}, - {"eccodes_key": "#2#timePeriod", "value": "data:fresh_snow_day_offset"}, - {"eccodes_key": "#2#hour", "value": "data:fresh_snow_hour"}, - {"eccodes_key": "#2#minute", "value": "data:fresh_snow_minute"}, - {"eccodes_key": "#2#second", "value": "data:fresh_snow_second"}, - {"eccodes_key": "#1#depthOfFreshSnow", "value": "data:fresh_snow_depth"}, - {"eccodes_key": "#1#depthOfFreshSnow->associatedField", "value": "data:fresh_snow_depth_flag"}, - {"eccodes_key": "#1#depthOfFreshSnow->associatedField->associatedFieldSignificance", "value": "const:5"}, - {"eccodes_key": "#3#timePeriod", "value": "data:total_snow_day_offset"}, - {"eccodes_key": "#3#hour", "value": "data:total_snow_hour"}, - {"eccodes_key": "#3#minute", "value": "data:total_snow_minute"}, - {"eccodes_key": "#3#second", "value": "data:total_snow_second"}, - {"eccodes_key": "#1#totalSnowDepth", "value": "data:total_snow_depth"}, - {"eccodes_key": "#1#totalSnowDepth->associatedField", "value": "data:total_snow_depth_flag"}, - {"eccodes_key": "#1#totalSnowDepth->associatedField->associatedFieldSignificance", "value": "const:5"}, - {"eccodes_key": "#1#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value": "data:thermometer_height_above_ground"}, - {"eccodes_key": "#4#timePeriod", "value": "data:maximum_temperature_day_offset"}, - {"eccodes_key": "#4#hour", "value": "data:maximum_temperature_hour"}, - {"eccodes_key": "#4#minute", "value": "data:maximum_temperature_minute"}, - {"eccodes_key": "#4#second", "value": "data:maximum_temperature_second"}, - {"eccodes_key": "#1#firstOrderStatistics", "value": "const:2"}, - {"eccodes_key": "#1#airTemperature", "value": "data:maximum_temperature", "scale": "const:0", "offset": "const:0"}, - {"eccodes_key": "#1#airTemperature->associatedField", "value": "data:maximum_temperature_flag"}, - {"eccodes_key": "#1#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"}, - {"eccodes_key": "#5#timePeriod", "value": "data:minimum_temperature_day_offset"}, - {"eccodes_key": "#5#hour", "value": "data:minimum_temperature_hour"}, - {"eccodes_key": "#5#minute", "value": "data:minimum_temperature_minute"}, - {"eccodes_key": "#5#second", "value": "data:minimum_temperature_second"}, - {"eccodes_key": "#2#firstOrderStatistics", "value": "const:3"}, - {"eccodes_key": "#2#airTemperature", "value": "data:minimum_temperature", "scale": "const:0", "offset": "const:0"}, - {"eccodes_key": "#2#airTemperature->associatedField", "value": "data:minimum_temperature_flag"}, - {"eccodes_key": "#2#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"}, - {"eccodes_key": "#6#timePeriod", "value": "data:average_temperature_day_offset"}, - {"eccodes_key": "#6#hour", "value": "data:average_temperature_hour"}, - {"eccodes_key": "#6#minute", "value": "data:average_temperature_minute"}, - {"eccodes_key": "#6#second", "value": "data:average_temperature_second"}, - {"eccodes_key": "#3#firstOrderStatistics", "value": "const:4"}, - {"eccodes_key": "#3#airTemperature", "value": "data:average_temperature", "scale": "const:0", "offset": "const:0"}, - {"eccodes_key": "#3#airTemperature->associatedField", "value": "data:average_temperature_flag"}, - {"eccodes_key": "#3#airTemperature->associatedField->associatedFieldSignificance", "value": "const:5"} - ] -} \ No newline at end of file diff --git a/csv2bufr/resources/mappings/mapping_schema.json b/csv2bufr/templates/resources/schema/csv2bufr-template-v2.json similarity index 78% rename from csv2bufr/resources/mappings/mapping_schema.json rename to csv2bufr/templates/resources/schema/csv2bufr-template-v2.json index 0600078..0de036d 100644 --- a/csv2bufr/resources/mappings/mapping_schema.json +++ b/csv2bufr/templates/resources/schema/csv2bufr-template-v2.json @@ -3,6 +3,41 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { + "conformsTo": {}, + "metadata": { + "type": "object", + "required": ["label","description","version","author","editor","dateCreated","dateModified","id"], + "properties": { + "label": { + "type": "string" + }, + "description": { + "type": "string" + }, + "version": { + "type": "string" + }, + "author": { + "type": "string" + }, + "editor": { + "type": "string" + }, + "dateCreated": { + "type": "string", + "format": "date" + }, + "dateModified": { + "type": "string", + "format": "date" + }, + "id": { + "type": "string", + "format": "uuid4" + } + } + }, + "inputShortDelayedDescriptorReplicationFactor": { "type": "array", "items": {"type": "integer"} @@ -52,6 +87,7 @@ } }, "required" : [ + "conformsTo", "metadata", "inputShortDelayedDescriptorReplicationFactor", "inputDelayedDescriptorReplicationFactor", "inputExtendedDelayedDescriptorReplicationFactor", diff --git a/csv2bufr/templates/resources/synop-template.json b/csv2bufr/templates/resources/synop-template.json deleted file mode 100644 index e4ee513..0000000 --- a/csv2bufr/templates/resources/synop-template.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "inputShortDelayedDescriptorReplicationFactor": [], - "inputDelayedDescriptorReplicationFactor": [0,0], - "inputExtendedDelayedDescriptorReplicationFactor": [], - "number_header_rows": 4, - "column_names_row": 2, - "wigos_station_identifier": "data:Station_ID", - "header":[ - {"eccodes_key": "edition", "value": "const:4"}, - {"eccodes_key": "masterTableNumber", "value": "const:0"}, - {"eccodes_key": "bufrHeaderCentre", "value": "const:0"}, - {"eccodes_key": "bufrHeaderSubCentre", "value": "const:0"}, - {"eccodes_key": "updateSequenceNumber", "value": "const:0"}, - {"eccodes_key": "dataCategory", "value": "const:0"}, - {"eccodes_key": "internationalDataSubCategory", "value": "const:6"}, - {"eccodes_key": "masterTablesVersionNumber", "value": "const:30"}, - {"eccodes_key": "numberOfSubsets", "value": "const:1"}, - {"eccodes_key": "observedData", "value": "const:1"}, - {"eccodes_key": "compressedData", "value": "const:0"}, - {"eccodes_key": "typicalYear", "value":"data:M_Year"}, - {"eccodes_key": "typicalMonth", "value":"data:M_Month"}, - {"eccodes_key": "typicalDay", "value":"data:M_DayOfMonth"}, - {"eccodes_key": "typicalHour", "value":"data:M_HourOfDay"}, - {"eccodes_key": "typicalMinute", "value":"data:M_Minutes"}, - {"eccodes_key": "unexpandedDescriptors", "value": "array:301150,307080"} - ], - "data": [ - {"eccodes_key": "#1#wigosIdentifierSeries", "value":"data:_wsi_series"}, - {"eccodes_key": "#1#wigosIssuerOfIdentifier", "value":"data:_wsi_issuer"}, - {"eccodes_key": "#1#wigosIssueNumber", "value":"data:_wsi_issue_number"}, - {"eccodes_key": "#1#wigosLocalIdentifierCharacter", "value":"data:_wsi_local"}, - {"eccodes_key": "#1#stationOrSiteName", "value":"data:Station_Name"}, - {"eccodes_key": "#1#stationType", "value":"data:WMO_Station_Type"}, - {"eccodes_key": "#1#year", "value":"data:M_Year", "valid_min": "const:2000", "valid_max": "const:2100"}, - {"eccodes_key": "#1#month", "value":"data:M_Month", "valid_min": "const:1", "valid_max": "const:12"}, - {"eccodes_key": "#1#day", "value":"data:M_DayOfMonth", "valid_min": "const:1", "valid_max": "const:31"}, - {"eccodes_key": "#1#hour", "value":"data:M_HourOfDay", "valid_min": "const:0", "valid_max": "const:23"}, - {"eccodes_key": "#1#minute", "value":"data:M_Minutes", "valid_min": "const:0", "valid_max": "const:59"}, - {"eccodes_key": "#1#latitude", "value":"data:Latitude", "valid_min": "const:-90", "valid_max": "const:90"}, - {"eccodes_key": "#1#longitude", "value":"data:Longitude", "valid_min": "const:-180", "valid_max": "const:180"}, - {"eccodes_key": "#1#heightOfStationGroundAboveMeanSeaLevel", "value":"data:Elevation"}, - {"eccodes_key": "#1#heightOfBarometerAboveMeanSeaLevel", "value":"data:BP_Elevation"}, - {"eccodes_key": "#1#nonCoordinatePressure", "value":"data:BP"}, - {"eccodes_key": "#1#pressureReducedToMeanSeaLevel", "value":"data:QNH"}, - {"eccodes_key": "#1#3HourPressureChange", "value":"data:BP_Change", "valid_min": "const:-5000", "valid_max": "const:5230"}, - {"eccodes_key": "#1#characteristicOfPressureTendency", "value":"data:BP_Tendency"}, - {"eccodes_key": "#1#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value":"data:Temp_H"}, - {"eccodes_key": "#1#airTemperature", "value":"data:AirTempK"}, - {"eccodes_key": "#1#dewpointTemperature", "value":"data:DewPointTempK"}, - {"eccodes_key": "#1#relativeHumidity", "value":"data:RH"}, - {"eccodes_key": "#2#timePeriod", "value":"data:Sun_hr"}, - {"eccodes_key": "#1#totalSunshine", "value":"data:SunHrs"}, - {"eccodes_key": "#3#timePeriod", "value":"data:Sun_hr24"}, - {"eccodes_key": "#2#totalSunshine", "value":"data:SunHrs24"}, - {"eccodes_key": "#5#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value":"data:Rain_H"}, - {"eccodes_key": "#4#timePeriod", "value":"data:Rain_hr"}, - {"eccodes_key": "#1#totalPrecipitationOrTotalWaterEquivalent", "value":"data:Rain_mm_Tot"}, - {"eccodes_key": "#6#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value":"data:Temp_H"}, - {"eccodes_key": "#6#timePeriod", "value":"data:Temp_hr24"}, - {"eccodes_key": "#7#timePeriod", "value":"data:Temp24T"}, - {"eccodes_key": "#1#maximumTemperatureAtHeightAndOverPeriodSpecified", "value":"data:AirTempMaxK"}, - {"eccodes_key": "#8#timePeriod", "value":"data:Temp_hr24"}, - {"eccodes_key": "#9#timePeriod", "value":"data:Temp24T"}, - {"eccodes_key": "#1#minimumTemperatureAtHeightAndOverPeriodSpecified", "value":"data:AirTempMinK"}, - {"eccodes_key": "#7#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value":"data:WSpeed_height"}, - {"eccodes_key": "#1#instrumentationForWindMeasurement", "value":"data:Wind_Type"}, - {"eccodes_key": "#1#timeSignificance", "value":"data:Wind_Sig"}, - {"eccodes_key": "#10#timePeriod", "value":"data:Wind_T"}, - {"eccodes_key": "#1#windDirection", "value":"data:WindDir"}, - {"eccodes_key": "#1#windSpeed", "value":"data:WSpeed10M_Avg"}, - {"eccodes_key": "#2#timeSignificance", "value":"data:WindG_Sig"}, - {"eccodes_key": "#1#maximumWindGustSpeed", "value":"data:WindGust"}, - {"eccodes_key": "#14#timePeriod", "value":"data:Solar_hr"}, - {"eccodes_key": "#1#globalSolarRadiationIntegratedOverPeriodSpecified", "value":"data:SlrJ"}, - {"eccodes_key": "#15#timePeriod", "value":"data:Solar_hr24"}, - {"eccodes_key": "#2#globalSolarRadiationIntegratedOverPeriodSpecified", "value":"data:SlrJ24"} - ] -} diff --git a/csv2bufr/templates/resources/synop_bufr.json b/csv2bufr/templates/resources/synop_bufr.json deleted file mode 100644 index f8d7beb..0000000 --- a/csv2bufr/templates/resources/synop_bufr.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "inputShortDelayedDescriptorReplicationFactor": [], - "inputDelayedDescriptorReplicationFactor": [0,0], - "inputExtendedDelayedDescriptorReplicationFactor": [], - "wigos_station_identifier": "data:Station_ID", - "number_header_rows": 4, - "column_names_row": 2, - "header":[ - {"eccodes_key": "edition", "value": "const:4"}, - {"eccodes_key": "masterTableNumber", "value": "const:0"}, - {"eccodes_key": "bufrHeaderCentre", "value": "const:0"}, - {"eccodes_key": "bufrHeaderSubCentre", "value": "const:0"}, - {"eccodes_key": "updateSequenceNumber", "value": "const:0"}, - {"eccodes_key": "dataCategory", "value": "const:0"}, - {"eccodes_key": "internationalDataSubCategory", "value": "const:6"}, - {"eccodes_key": "masterTablesVersionNumber", "value": "const:30"}, - {"eccodes_key": "numberOfSubsets", "value": "const:1"}, - {"eccodes_key": "observedData", "value": "const:1"}, - {"eccodes_key": "compressedData", "value": "const:0"}, - {"eccodes_key": "typicalYear", "value":"data:M_Year"}, - {"eccodes_key": "typicalMonth", "value":"data:M_Month"}, - {"eccodes_key": "typicalDay", "value":"data:M_DayOfMonth"}, - {"eccodes_key": "typicalHour", "value":"data:M_HourOfDay"}, - {"eccodes_key": "typicalMinute", "value":"data:M_Minutes"}, - {"eccodes_key": "unexpandedDescriptors", "value": "array:301150,307080"} - ], - "data": [ - {"eccodes_key": "#1#wigosIdentifierSeries", "value":"data:_wsi_series"}, - {"eccodes_key": "#1#wigosIssuerOfIdentifier", "value":"data:_wsi_issuer"}, - {"eccodes_key": "#1#wigosIssueNumber", "value":"data:_wsi_issue_number"}, - {"eccodes_key": "#1#wigosLocalIdentifierCharacter", "value":"data:_wsi_local"}, - {"eccodes_key": "#1#stationOrSiteName", "value":"data:Station_Name"}, - {"eccodes_key": "#1#stationType", "value":"data:WMO_Station_Type"}, - {"eccodes_key": "#1#year", "value":"data:M_Year", "valid_min": "const:2000", "valid_max": "const:2100"}, - {"eccodes_key": "#1#month", "value":"data:M_Month", "valid_min": "const:1", "valid_max": "const:12"}, - {"eccodes_key": "#1#day", "value":"data:M_DayOfMonth", "valid_min": "const:1", "valid_max": "const:31"}, - {"eccodes_key": "#1#hour", "value":"data:M_HourOfDay", "valid_min": "const:0", "valid_max": "const:23"}, - {"eccodes_key": "#1#minute", "value":"data:M_Minutes", "valid_min": "const:0", "valid_max": "const:59"}, - {"eccodes_key": "#1#latitude", "value":"data:Latitude", "valid_min": "const:-90", "valid_max": "const:90"}, - {"eccodes_key": "#1#longitude", "value":"data:Longitude", "valid_min": "const:-180", "valid_max": "const:180"}, - {"eccodes_key": "#1#heightOfStationGroundAboveMeanSeaLevel", "value":"data:Elevation"}, - {"eccodes_key": "#1#heightOfBarometerAboveMeanSeaLevel", "value":"data:BP_Elevation"}, - {"eccodes_key": "#1#nonCoordinatePressure", "value":"data:BP"}, - {"eccodes_key": "#1#pressureReducedToMeanSeaLevel", "value":"data:QNH"}, - {"eccodes_key": "#1#3HourPressureChange", "value":"data:BP_Change", "valid_min": "const:-5000", "valid_max": "const:5230"}, - {"eccodes_key": "#1#characteristicOfPressureTendency", "value":"data:BP_Tendency"}, - {"eccodes_key": "#1#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value":"data:Temp_H"}, - {"eccodes_key": "#1#airTemperature", "value":"data:AirTempK"}, - {"eccodes_key": "#1#dewpointTemperature", "value":"data:DewPointTempK"}, - {"eccodes_key": "#1#relativeHumidity", "value":"data:RH"}, - {"eccodes_key": "#2#timePeriod", "value":"data:Sun_hr"}, - {"eccodes_key": "#1#totalSunshine", "value":"data:SunHrs"}, - {"eccodes_key": "#3#timePeriod", "value":"data:Sun_hr24"}, - {"eccodes_key": "#2#totalSunshine", "value":"data:SunHrs24"}, - {"eccodes_key": "#5#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value":"data:Rain_H"}, - {"eccodes_key": "#4#timePeriod", "value":"data:Rain_hr"}, - {"eccodes_key": "#1#totalPrecipitationOrTotalWaterEquivalent", "value":"data:Rain_mm_Tot"}, - {"eccodes_key": "#6#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value":"data:Temp_H"}, - {"eccodes_key": "#6#timePeriod", "value":"data:Temp_hr24"}, - {"eccodes_key": "#7#timePeriod", "value":"data:Temp24T"}, - {"eccodes_key": "#1#maximumTemperatureAtHeightAndOverPeriodSpecified", "value":"data:AirTempMaxK"}, - {"eccodes_key": "#8#timePeriod", "value":"data:Temp_hr24"}, - {"eccodes_key": "#9#timePeriod", "value":"data:Temp24T"}, - {"eccodes_key": "#1#minimumTemperatureAtHeightAndOverPeriodSpecified", "value":"data:AirTempMinK"}, - {"eccodes_key": "#7#heightOfSensorAboveLocalGroundOrDeckOfMarinePlatform", "value":"data:WSpeed_height"}, - {"eccodes_key": "#1#instrumentationForWindMeasurement", "value":"data:Wind_Type"}, - {"eccodes_key": "#1#timeSignificance", "value":"data:Wind_Sig"}, - {"eccodes_key": "#10#timePeriod", "value":"data:Wind_T"}, - {"eccodes_key": "#1#windDirection", "value":"data:WindDir"}, - {"eccodes_key": "#1#windSpeed", "value":"data:WSpeed10M_Avg"}, - {"eccodes_key": "#2#timeSignificance", "value":"data:WindG_Sig"}, - {"eccodes_key": "#1#maximumWindGustSpeed", "value":"data:WindGust"}, - {"eccodes_key": "#14#timePeriod", "value":"data:Solar_hr"}, - {"eccodes_key": "#1#globalSolarRadiationIntegratedOverPeriodSpecified", "value":"data:SlrJ"}, - {"eccodes_key": "#15#timePeriod", "value":"data:Solar_hr24"}, - {"eccodes_key": "#2#globalSolarRadiationIntegratedOverPeriodSpecified", "value":"data:SlrJ24"} - ] -} diff --git a/docs/source/installation.rst b/docs/source/installation.rst index da65dda..b3bbcbb 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -79,3 +79,12 @@ The following output should be shown: data data workflows mappings stored mappings +Environment variables +********************* +Three environment variables are defined and can be used to set the originating centre and +sub centre of the generated BUFR files and the system search path used to find BUFR mapping templates(see +:ref:`BUFR template page `) + +- ``BUFR_ORIGINATING_CENTRE``: Specifies the originating centre of the BUFR data, Common Code Table C-11 +- ``BUFR_ORIGINATING_SUBCENTRE``: Specifies the originating sub centre of the BUFR data, Common Code Table C-12 +- ``CSV2BUFR_TEMPLATES``: Path to search for BUFR templates diff --git a/docs/source/mapping.rst b/docs/source/mapping.rst index 786e3c7..1783357 100644 --- a/docs/source/mapping.rst +++ b/docs/source/mapping.rst @@ -6,7 +6,7 @@ BUFR template mapping ===================== The mapping between the input CSV data and the output BUFR data is specified in a JSON file. -The csv2bufr module validates the mapping file against the schema shown at the bottom of this page prior to attempted the transformation to BUFR. +The csv2bufr module validates the mapping file against the schema shown at the bottom of this page prior to attempting the transformation to BUFR. This schema specifies 7 primary properties all of which are mandatory: - ``inputDelayedDescriptorReplicationFactor`` - array of integers, values for the delayed descriptor replication factors to use @@ -66,7 +66,7 @@ Similarly the keys for the different data elements can be found at: - ``_ inputDelayedDescriptorReplicationFactor ---------------------------------------- +------------------------------------------------------- Due to the way that eccodes works any delayed replication factors need to be specified before encoding and included in the mapping file. This currently limits the use of the delayed replication factors to static values for a given mapping. For example every data file that uses a given mapping file has the same optional elements present or the same number of levels in an @@ -196,4 +196,15 @@ the ``scale`` and ``offset`` fields. Some additional examples are given below. Schema ------ -.. literalinclude:: ../../csv2bufr/resources/mappings/mapping_schema.json +.. literalinclude:: ../../csv2bufr/templates/resources/schema/csv2bufr-template-v2.json + +Built in templates and search path +---------------------------------- + +Several preconfigured templates are available from the csv2bufr-templates repository: + +- https://github.com/wmo-im/csv2bufr-templates + +By default, ``csv2bufr`` searches in the current working directory and the +``/opt/csv2bufr/templates`` directory, additional search paths can be added by setting +the ``CSV2BUFR_TEMPLATES`` environment variable. \ No newline at end of file diff --git a/docs/source/resources/csv2bufr-template-v2.json b/docs/source/resources/csv2bufr-template-v2.json new file mode 100644 index 0000000..91a5611 --- /dev/null +++ b/docs/source/resources/csv2bufr-template-v2.json @@ -0,0 +1,138 @@ +{ + "$id": "csv2bufr.wis2.0.node.wis", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "conformsTo": {}, + "metadata": { + "type": "object", + "requiered": ["label","description","version","author","editor","dateCreated","dateModified","id"], + "properties": { + "label": { + "type": "string" + }, + "description": { + "type": "string" + }, + "version": { + "type": "string" + }, + "author": { + "type": "string" + }, + "editor": { + "type": "string" + }, + "dateCreated": { + "type": "string", + "format": "date" + }, + "dateModified": { + "type": "string", + "format": "date" + }, + "id": { + "type": "string", + "format": "uuid4" + } + } + }, + + "inputShortDelayedDescriptorReplicationFactor": { + "type": "array", + "items": {"type": "integer"} + }, + "inputDelayedDescriptorReplicationFactor": { + "type": "array", + "items": {"type": "integer"} + }, + "inputExtendedDelayedDescriptorReplicationFactor": { + "type": "array", + "items": {"type": "integer"} + }, + "number_header_rows": { + "type": "integer", + "description": "Number of header rows in file before the data" + }, + "column_names_row": { + "type": "integer", + "description": "Which header line the column names is given on" + + }, + "wigos_station_identifier": { + "type": "string", + "description": "Either the WIGOS station identifier for the data or the column in the CSV file containing the identifier" + }, + "delimiter": { + "type": "string", + "description": "The delimiter used to separate fields in the input csv file, must be one of ',', ';'. '|' or [tab]" + }, + "quoting": { + "type": "string", + "description": "CSV quoting method to use, must be one of QUOTE_NONNUMERIC, QUOTE_ALL, QUOTE_MINIMAL or QUOTE_NONE" + }, + "quotechar": { + "type": "string", + "description": "quote character to use, e.g. \", ' etc" + }, + "header":{ + "type": "array", + "items": {"$ref": "#/$defs/bufr_element"}, + "description": "Contents of header sections of BUFR message" + }, + "data": { + "type": "array", + "items": {"$ref": "#/$defs/bufr_element"}, + "description": "mapping from CSV file (or metadata json file) to BUFR" + } + }, + "required" : [ + "conformsTo", "metadata", + "inputShortDelayedDescriptorReplicationFactor", + "inputDelayedDescriptorReplicationFactor", + "inputExtendedDelayedDescriptorReplicationFactor", + "column_names_row","number_header_rows","header","data"], + + "$defs":{ + "bufr_element": { + "type": "object", + "properties": { + "eccodes_key": { + "type": "string", + "descripition": "eccodes key used to set the value in the BUFR data" + }, + "value": { + "type": [ + "string" + ], + "description": "where to extract the value from, can be one off 'data','metadata','const','array' followed by the value or column header" + }, + "valid_min": { + "type": "string", + "description": "Minimum valid value for parameter if set" + }, + "valid_max": { + "type": "string", + "description": "Maximum value for for the parameter if set" + }, + "scale": { + "type": "string", + "description": "Value used to scale the data by before encoding using the same conventions as in BUFR" + }, + "offset": { + "type": "string", + "description": "Value added to the data before encoding to BUFR following the same conventions as BUFR" + } + }, + "required": ["eccodes_key", "value"], + "allOf": [ + { + "dependentRequired": {"scale": ["offset"]} + }, + { + "dependentRequired": {"offset": ["scale"]} + } + ] + } + } +} diff --git a/tests/test_csv2bufr.py b/tests/test_csv2bufr.py index adc1f29..ea2d576 100644 --- a/tests/test_csv2bufr.py +++ b/tests/test_csv2bufr.py @@ -20,25 +20,43 @@ ############################################################################### import csv +import os +import threading + from io import StringIO import logging from eccodes import (codes_bufr_new_from_samples, codes_release) import pytest -from csv2bufr import (validate_mapping, apply_scaling, validate_value, - transform, SUCCESS) +from csv2bufr import (apply_scaling, validate_value, + transform, SUCCESS, _warnings_global) import csv2bufr.templates as c2bt LOGGER = logging.getLogger(__name__) LOGGER.setLevel("DEBUG") +# set up warnings dict +tidx = f"t-{threading.get_ident()}" +_warnings_global[tidx] = [] + # test data @pytest.fixture def mapping_dict(): return { + "conformsTo": "csv2bufr-template-v2.json", + "metadata": { + "label": "pytest", + "description": "pytest template", + "version": "2", + "author": "David I. Berry", + "editor": "", + "dateCreated": "2023-09-01", + "dateModified": "2024-01-12", + "id": "eb37ac4f-13ce-4254-b372-5c1d097ea857" + }, "inputShortDelayedDescriptorReplicationFactor": [], "inputDelayedDescriptorReplicationFactor": [], "inputExtendedDelayedDescriptorReplicationFactor": [], @@ -148,7 +166,7 @@ def test_eccodes(): # test to check validate_mapping is not broken def test_validate_mapping_pass(mapping_dict): - success = validate_mapping(mapping_dict) + success = c2bt.validate_template(mapping_dict) assert success == SUCCESS @@ -167,7 +185,7 @@ def test_validate_mapping_fail(): ] } try: - success = validate_mapping(test_data) + success = c2bt.validate_template(test_data) except Exception: success = False assert success != SUCCESS @@ -204,11 +222,15 @@ def test_validate_value_fail(): # test to check that valid_value returns null value when we expect it to def test_validate_value_nullify(): input_value = 10.0 + tid = f"t-{threading.get_ident()}" + if tid not in _warnings_global: + _warnings_global[tid] = [] try: value = validate_value("test value", input_value, 0.0, 9.9, True) except Exception: assert False assert value is None + del _warnings_global[tid] # check that test transform works @@ -239,4 +261,21 @@ def test_transform(data_dict, mapping_dict): def test_templates(): - assert c2bt.load_template('aws-template') is not None + tmpl = c2bt.load_template('21327aac-46a6-437d-ae81-7a16a637dd2c') + assert tmpl is not None + # check header centre and sub centre set to env + ocset = False + ocenv = os.environ.get('BUFR_ORIGINATING_CENTRE', 65535) + oscset = False + oscenv = os.environ.get('BUFR_ORIGINATING_SUBCENTRE', 65535) + LOGGER.warning((tmpl['header'])) + for hidx in range(len(tmpl['header'])): + LOGGER.warning(tmpl['header'][hidx]['eccodes_key']) + if tmpl['header'][hidx]['eccodes_key'] == 'bufrHeaderCentre': + assert tmpl['header'][hidx]['value'] == f"const:{ocenv}" + ocset = True + if tmpl['header'][hidx]['eccodes_key'] == 'bufrHeaderSubCentre': + assert tmpl['header'][hidx]['value'] == f"const:{oscenv}" + oscset = True + assert ocset + assert oscset