Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove code for generating QGreenland plugin manifests #794

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Remove code for generating QGreenland plugin manifests #794

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 2 additions & 84 deletions qgreenland/test/util/config/test_config_export.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,92 +1,10 @@
import csv
import json
import tempfile
from pathlib import Path
from unittest.mock import patch

from qgreenland.test.constants import MOCK_COMPILE_PACKAGE_DIR, MOCK_RELEASE_LAYERS_DIR
from qgreenland.util.config.export import export_config_csv, export_config_manifest


@patch(
"qgreenland.util.layer.RELEASE_LAYERS_DIR",
new=MOCK_RELEASE_LAYERS_DIR,
)
def test_export_config_manifest(full_cfg):
common = {
"description": "Example layer description.",
# TODO: Generate this with imported function? This should be tested
# by itself elsewhere, so there's no need to test the expected output
# here too.
"layer_details": """Example layer description.

=== Original Data Source ===
Example Dataset

Example abstract.

Citation:
NSIDC 2020

Citation URL:
https://nsidc.org""",
"tags": ["foo", "bar", "baz"],
"hierarchy": ["Group", "Subgroup"],
}
with tempfile.NamedTemporaryFile("r") as tf:
export_config_manifest(
full_cfg,
output_path=Path(tf.name),
)

actual = json.load(tf)

assert type(actual["qgr_version"]) is str
assert len(actual["qgr_version"]) >= 6
del actual["qgr_version"]

# For now, do not include online layers in the layer manifest. The
# `QGreenland Custom` QGIS Plugin does not currently support online
# layers. Once online layers are supported in the plugin, this commented out
# `online_asset` can be re-added:
# online_asset = {
# 'type': 'online',
# **full_cfg.layers['example_online'].input.asset.dict(
# include={'provider', 'url'},
# ),
# }
expected = {
"version": "v0.1.0",
"layers": [
# {
# 'id': 'example_online',
# 'title': 'Example online',
# 'assets': [online_asset],
# **common,
# },
{
"id": "example_raster",
"title": "Example raster",
"assets": [
{
"checksum": "a9a103f208179726038fa7178747a0a1",
"file": "example.tif",
"size_bytes": 287,
"type": "data",
},
{
"checksum": "22b427acc6e4ebf57052115fdd5ac450",
"file": "example.tif.aux.xml",
"size_bytes": 332,
"type": "ancillary",
},
],
**common,
},
],
}

assert actual == expected
from qgreenland.test.constants import MOCK_COMPILE_PACKAGE_DIR
from qgreenland.util.config.export import export_config_csv


@patch(
Expand Down
39 changes: 0 additions & 39 deletions qgreenland/util/config/export.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,50 +22,11 @@
get_layer_release_filepath,
vector_or_raster,
)
from qgreenland.util.metadata import build_layer_metadata
from qgreenland.util.tree import LayerNode
from qgreenland.util.version import get_build_version

DEFAULT_LAYER_MANIFEST_PATH = Path("./layers.csv")


def export_config_manifest(
cfg: Config,
output_path: Path = DEFAULT_LAYER_MANIFEST_PATH,
) -> None:
"""Write a machine-readable manifest to disk describing available layers.

This includes layers for which `in_package is False`.

This must be run after the layers are in their release location, because we
need to calculate their size on disk.
"""
manifest_spec_version = "v0.1.0"
manifest = {
"version": manifest_spec_version,
"qgr_version": get_build_version(),
"layers": [
{
# ID first for readability
"id": layer_node.layer_cfg.id,
**layer_node.layer_cfg.dict(include={"title", "description", "tags"}),
"hierarchy": layer_node.group_name_path,
"layer_details": build_layer_metadata(layer_node.layer_cfg),
"assets": _layer_manifest_final_assets(layer_node),
}
for layer_node in cfg.layer_tree.leaves
# For now, do not include online layers in the layer manifest. The
# `QGreenland Custom` QGIS Plugin does not currently support online
# layers. Once online layers are supported in the plugin, this `if`
# statement can be removed.
if not isinstance(layer_node.layer_cfg.input.asset, OnlineAsset)
],
}

with open(output_path, "w") as ofile:
json.dump(manifest, ofile)


def export_config_csv(
cfg: Config,
output_path: Path = DEFAULT_LAYER_MANIFEST_PATH,
Expand Down
30 changes: 7 additions & 23 deletions qgreenland/util/luigi/tasks/pipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,13 @@
ANCILLARY_DIR,
COMPILE_PACKAGE_DIR,
PROJECT_DIR,
RELEASE_LAYERS_DIR,
VERSIONED_PACKAGE_DIR,
WIP_PACKAGE_DIR,
)
from qgreenland.constants.project import ENVIRONMENT, PROJECT
from qgreenland.util.cleanup import cleanup_intermediate_dirs
from qgreenland.util.config.config import get_config
from qgreenland.util.config.export import export_config_csv, export_config_manifest
from qgreenland.util.config.export import export_config_csv
from qgreenland.util.luigi import generate_layer_pipelines
from qgreenland.util.luigi.tasks.ancillary import (
AncillaryFile,
Expand Down Expand Up @@ -78,26 +77,6 @@ def requires(self):
yield from tasks


class LayerManifest(luigi.Task):
"""A JSON manifest of layers available for access.

Intended to be processed by machine, e.g. QGIS plugin.
"""

def output(self):
return luigi.LocalTarget(
RELEASE_LAYERS_DIR / "manifest.json",
)

def requires(self):
yield LayerPipelines()

def run(self):
config = get_config()
with self.output().temporary_path() as temp_path:
export_config_manifest(config, output_path=temp_path)


class CreateQgisProjectFile(luigi.Task):
"""Create .qgz/.qgs project file."""

Expand Down Expand Up @@ -197,9 +176,14 @@ def run(self):


class HostedLayers(luigi.WrapperTask):
"""Generate all layers we need to host.

This is a vestige of the QGreenland Custom QGIS plugin; we no longer need to "host"
layers, except for backwards-compatibility.
"""

def requires(self):
yield LayerPipelines()
yield LayerManifest()


class QGreenlandAll(luigi.WrapperTask):
Expand Down