From bf9777bba80fc88f01a89b51e9e273ba2a3f130a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 24 Oct 2024 11:22:51 +0200 Subject: [PATCH 01/54] NITF: represent SAR products with I,Q bands as single complex band Merge changes from https://github.com/DND-DRDC-RDDC/OS_DriverMetadataExtractionCodes_RCM_C-band_SARdata/commit/1f4d014ade7ccec5b47d2d5cdc92acbeb40c15a3 and provided by Esri, and slightly modified by me. Added test is mine. --- autotest/gdrivers/nitf.py | 38 +++++- doc/source/drivers/raster/nitf_advanced.rst | 8 ++ frmts/nitf/nitfdataset.cpp | 121 +++++++++++++++-- frmts/nitf/nitfdataset.h | 33 ++++- frmts/nitf/nitfrasterband.cpp | 138 +++++++++++++++++++- 5 files changed, 324 insertions(+), 14 deletions(-) diff --git a/autotest/gdrivers/nitf.py b/autotest/gdrivers/nitf.py index 24eecec66f45..0efaf689b8f4 100755 --- a/autotest/gdrivers/nitf.py +++ b/autotest/gdrivers/nitf.py @@ -5735,7 +5735,8 @@ def test_nitf_invalid_udid(): def test_nitf_isubcat_populated(): # Check a dataset with IQ complex data. - ds = gdal.Open("data/nitf/sar_sicd.ntf") + with gdal.config_option("NITF_SAR_AS_COMPLEX_TYPE", "NO"): + ds = gdal.Open("data/nitf/sar_sicd.ntf") expected = ["I", "Q"] for b in range(ds.RasterCount): md = ds.GetRasterBand(b + 1).GetMetadata() @@ -6248,6 +6249,41 @@ def test_nitf_report_ABPP_as_NBITS(tmp_vsimem): assert ds.GetRasterBand(1).GetMetadataItem("NBITS", "IMAGE_STRUCTURE") == "9" +############################################################################### +# Test reading SAR products with I,Q bands + + +def test_nitf_readSAR_IQ(tmp_vsimem): + + out_filename = str(tmp_vsimem / "tmp.ntf") + gdal.Translate( + out_filename, + "data/byte.tif", + options="-b 1 -b 1 -outsize 40 20 -co ICAT=SAR -ot Float32 -co ISUBCAT=I,Q -scale_2 0 255 255 0", + ) + gdal.Unlink(out_filename + ".aux.xml") + + ds = gdal.Open(out_filename) + assert ds.RasterCount == 1 + assert ds.RasterXSize == 40 + assert ds.RasterYSize == 20 + assert ds.GetRasterBand(1).DataType == gdal.GDT_CFloat32 + assert ds.GetMetadataItem("NITF_ISUBCAT") is None + + src_ds = gdal.Open("data/byte.tif") + ds = gdal.Open("DERIVED_SUBDATASET:REAL:" + out_filename) + assert ( + ds.ReadRaster(0, 0, 40, 20, 20, 20, buf_type=gdal.GDT_Byte) + == src_ds.ReadRaster() + ) + ds = gdal.Open("DERIVED_SUBDATASET:IMAG:" + out_filename) + mod_src_ds = gdal.Translate("", src_ds, options="-f MEM -scale 0 255 255 0") + assert ( + ds.ReadRaster(0, 0, 40, 20, 20, 20, buf_type=gdal.GDT_Byte) + == mod_src_ds.ReadRaster() + ) + + ############################################################################### # Test NITF21_CGM_ANNO_Uncompressed_unmasked.ntf for bug #1313 and #1714 diff --git a/doc/source/drivers/raster/nitf_advanced.rst b/doc/source/drivers/raster/nitf_advanced.rst index 1420b704f865..95fbde783884 100644 --- a/doc/source/drivers/raster/nitf_advanced.rst +++ b/doc/source/drivers/raster/nitf_advanced.rst @@ -338,3 +338,11 @@ Example: gdal_translate first_image.tif dest.tif -co NUMI=3 -co NUMDES=1 gdal_translate second_image.tif dest.tif -co APPEND_SUBDATASET=YES -co IC=C3 -co IDLVL=2 gdal_translate third_image.tif dest.tif -co APPEND_SUBDATASET=YES -co IC=C8 -co IDLVL=3 -co "DES=DES1={des_content}" + +Reading SAR products +-------------------- + +Starting with GDAL 3.11, NITF products whose ``ICAT`` field is set to ``SAR`` +and which have bands with ``I`` (in-phase) and ``Q`` (quadrature) ``ISUBCAT`` +values will be represented as a single-band of complex type where in-phase is +the real part and quadrature the imaginary part. diff --git a/frmts/nitf/nitfdataset.cpp b/frmts/nitf/nitfdataset.cpp index 3a18bb50aefb..333b203b9e0f 100644 --- a/frmts/nitf/nitfdataset.cpp +++ b/frmts/nitf/nitfdataset.cpp @@ -9,7 +9,7 @@ * Copyright (c) 2007-2013, Even Rouault * * Portions Copyright (c) Her majesty the Queen in right of Canada as - * represented by the Minister of National Defence, 2006. + * represented by the Minister of National Defence, 2006, 2020 * * SPDX-License-Identifier: MIT ****************************************************************************/ @@ -62,7 +62,7 @@ static bool NITFWriteJPEGImage(GDALDataset *, VSILFILE *, vsi_l_offset, char **, #endif static void SetBandMetadata(NITFImage *psImage, GDALRasterBand *poBand, - int nBand); + int nBand, bool bReportISUBCAT); /************************************************************************/ /* ==================================================================== */ @@ -380,12 +380,12 @@ static char **ExtractEsriMD(char **papszMD) /************************************************************************/ static void SetBandMetadata(NITFImage *psImage, GDALRasterBand *poBand, - int nBand) + int nBand, bool bReportISUBCAT) { const NITFBandInfo *psBandInfo = psImage->pasBandInfo + nBand - 1; /* The ISUBCAT is particularly valuable for interpreting SAR bands */ - if (strlen(psBandInfo->szISUBCAT) > 0) + if (bReportISUBCAT && strlen(psBandInfo->szISUBCAT) > 0) { poBand->SetMetadataItem("NITF_ISUBCAT", psBandInfo->szISUBCAT); } @@ -745,6 +745,12 @@ NITFDataset *NITFDataset::OpenInternal(GDALOpenInfo *poOpenInfo, /* Create band information objects. */ /* -------------------------------------------------------------------- */ + /* Keep temporary non-based dataset bands */ + bool bIsTempBandUsed = false; + GDALDataType dtFirstBand = GDT_Unknown; + GDALDataType dtSecondBand = GDT_Unknown; + std::vector apoNewBands(nUsableBands); + GDALDataset *poBaseDS = nullptr; if (poDS->poJ2KDataset != nullptr) poBaseDS = poDS->poJ2KDataset; @@ -757,7 +763,7 @@ NITFDataset *NITFDataset::OpenInternal(GDALOpenInfo *poOpenInfo, { GDALRasterBand *poBaseBand = poBaseDS->GetRasterBand(iBand + 1); - SetBandMetadata(psImage, poBaseBand, iBand + 1); + SetBandMetadata(psImage, poBaseBand, iBand + 1, true); NITFWrapperRasterBand *poBand = new NITFWrapperRasterBand(poDS, poBaseBand, iBand + 1); @@ -791,19 +797,93 @@ NITFDataset *NITFDataset::OpenInternal(GDALOpenInfo *poOpenInfo, } poDS->SetBand(iBand + 1, poBand); + + if (iBand == 0) + dtFirstBand = poBand->GetRasterDataType(); + else if (iBand == 1) + dtSecondBand = poBand->GetRasterDataType(); } else { - GDALRasterBand *poBand = new NITFRasterBand(poDS, iBand + 1); + bIsTempBandUsed = true; + + NITFRasterBand *poBand = new NITFRasterBand(poDS, iBand + 1); if (poBand->GetRasterDataType() == GDT_Unknown) { + for (auto *poOtherBand : apoNewBands) + delete poOtherBand; delete poBand; delete poDS; return nullptr; } - SetBandMetadata(psImage, poBand, iBand + 1); + apoNewBands[iBand] = poBand; + + if (iBand == 0) + dtFirstBand = poBand->GetRasterDataType(); + if (iBand == 1) + dtSecondBand = poBand->GetRasterDataType(); + } + } + + /* -------------------------------------------------------------------- */ + /* SAR images may store complex data in 2 bands (I and Q) */ + /* Map onto a GDAL complex raster band */ + /* -------------------------------------------------------------------- */ + bool bIsTempBandSet = false; + if (!bOpenForCreate && psImage && + EQUAL(psImage->szICAT, "SAR") //SAR image... + && bIsTempBandUsed && + nUsableBands == psImage->nBands + //...with 2 bands ... (modified to allow an even number - spec seems to indicate only 2 bands allowed?) + && (nUsableBands % 2) == 0 && + dtFirstBand == dtSecondBand //...that have the same datatype... + && !GDALDataTypeIsComplex(dtFirstBand) //...and are not complex... + //..and can be mapped directly to a complex type + && (dtFirstBand == GDT_Int16 || dtFirstBand == GDT_Int32 || + dtFirstBand == GDT_Float32 || dtFirstBand == GDT_Float64) && + CPLTestBool(CPLGetConfigOption("NITF_SAR_AS_COMPLEX_TYPE", "YES"))) + { + bool allBandsIQ = true; + for (int i = 0; i < nUsableBands; i += 2) + { + const NITFBandInfo *psBandInfo1 = psImage->pasBandInfo + i; + const NITFBandInfo *psBandInfo2 = psImage->pasBandInfo + i + 1; + + //check that the ISUBCAT is labelled "I" and "Q" on the 2 bands + if (!EQUAL(psBandInfo1->szISUBCAT, "I") || + !EQUAL(psBandInfo2->szISUBCAT, "Q")) + { + allBandsIQ = false; + break; + } + } + + if (allBandsIQ) + { + poDS->m_bHasComplexRasterBand = true; + for (int i = 0; i < (nUsableBands / 2); i++) + { + //wrap the I and Q bands into a single complex band + const int iBandIndex = 2 * i; + const int qBandIndex = 2 * i + 1; + NITFComplexRasterBand *poBand = new NITFComplexRasterBand( + poDS, apoNewBands[iBandIndex], apoNewBands[qBandIndex], + iBandIndex + 1, qBandIndex + 1); + SetBandMetadata(psImage, poBand, i + 1, false); + poDS->SetBand(i + 1, poBand); + bIsTempBandSet = true; + } + } + } + if (bIsTempBandUsed && !bIsTempBandSet) + { + // Reset properly bands that are not complex + for (int iBand = 0; iBand < nUsableBands; iBand++) + { + GDALRasterBand *poBand = apoNewBands[iBand]; + SetBandMetadata(psImage, poBand, iBand + 1, true); poDS->SetBand(iBand + 1, poBand); } } @@ -2053,7 +2133,8 @@ CPLErr NITFDataset::AdviseRead(int nXOff, int nYOff, int nXSize, int nYSize, char **papszOptions) { - if (poJ2KDataset == nullptr) + //go through GDALDataset::AdviseRead for the complex SAR + if (poJ2KDataset == nullptr || m_bHasComplexRasterBand) return GDALDataset::AdviseRead(nXOff, nYOff, nXSize, nYSize, nBufXSize, nBufYSize, eDT, nBandCount, panBandList, papszOptions); @@ -2080,12 +2161,13 @@ CPLErr NITFDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, GDALRasterIOExtraArg *psExtraArg) { - if (poJ2KDataset != nullptr) + //go through GDALDataset::IRasterIO for the complex SAR + if (poJ2KDataset != nullptr && !m_bHasComplexRasterBand) return poJ2KDataset->RasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, nBufXSize, nBufYSize, eBufType, nBandCount, panBandMap, nPixelSpace, nLineSpace, nBandSpace, psExtraArg); - else if (poJPEGDataset != nullptr) + else if (poJPEGDataset != nullptr && !m_bHasComplexRasterBand) return poJPEGDataset->RasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, nBufXSize, nBufYSize, eBufType, nBandCount, panBandMap, nPixelSpace, @@ -3371,7 +3453,24 @@ int NITFDataset::CheckForRSets(const char *pszNITFFilename, } if (aosRSetFilenames.empty()) - return FALSE; + { + //try for remoteview RRDS (with .rv%d extension) + for (int i = 1; i <= 7; i++) + { + CPLString osTarget; + VSIStatBufL sStat; + + osTarget.Printf("%s.rv%d", pszNITFFilename, i); + + if (VSIStatL(osTarget, &sStat) != 0) + break; + + aosRSetFilenames.push_back(osTarget); + } + + if (aosRSetFilenames.empty()) + return FALSE; + } /* -------------------------------------------------------------------- */ /* We do, so try to create a wrapping VRT file. */ diff --git a/frmts/nitf/nitfdataset.h b/frmts/nitf/nitfdataset.h index f24b1e00e3ad..ed986f2a12c5 100644 --- a/frmts/nitf/nitfdataset.h +++ b/frmts/nitf/nitfdataset.h @@ -22,6 +22,8 @@ #include "nitflib.h" #include "ogr_spatialref.h" #include "gdal_proxy.h" + +#include #include CPLErr NITFSetColorInterpretation(NITFImage *psImage, int nBand, @@ -52,6 +54,7 @@ class NITFDataset final : public GDALPamDataset { friend class NITFRasterBand; friend class NITFWrapperRasterBand; + friend class NITFComplexRasterBand; NITFFile *psFile; NITFImage *psImage; @@ -62,6 +65,7 @@ class NITFDataset final : public GDALPamDataset int m_nIMIndex = 0; int m_nImageCount = 0; vsi_l_offset m_nICOffset = 0; + bool m_bHasComplexRasterBand = false; GDALDataset *poJPEGDataset; @@ -182,7 +186,7 @@ class NITFDataset final : public GDALPamDataset /* ==================================================================== */ /************************************************************************/ -class NITFRasterBand final : public GDALPamRasterBand +class NITFRasterBand CPL_NON_FINAL : public GDALPamRasterBand { friend class NITFDataset; @@ -359,4 +363,31 @@ class NITFWrapperRasterBand final : public NITFProxyPamRasterBand void SetColorTableFromNITFBandInfo(); }; +/************************************************************************/ +/* ==================================================================== */ +/* NITFComplexRasterBand */ +/* ==================================================================== */ +/************************************************************************/ + +/* This class is used to wrap 2 bands (I and Q) as a complex raster band */ +class NITFComplexRasterBand final : public NITFRasterBand +{ + std::unique_ptr poIntermediateDS{}; + std::array anBandMap = {0, 0}; + GDALDataType underlyingDataType = GDT_Unknown; + int complexDataTypeSize = 0; + int underlyingDataTypeSize = 0; + + private: + CPLErr IBlockIO(int nBlockXOff, int nBlockYOff, void *pImage, + GDALRWFlag rwFlag); + + public: + NITFComplexRasterBand(NITFDataset *poDSIn, GDALRasterBand *poBandI, + GDALRasterBand *poBandQ, int nIBand, int nQBand); + + CPLErr IReadBlock(int, int, void *) override; + CPLErr IWriteBlock(int, int, void *) override; +}; + #endif /* NITF_DATASET_H_INCLUDED */ diff --git a/frmts/nitf/nitfrasterband.cpp b/frmts/nitf/nitfrasterband.cpp index e03543cefd7e..e746435a2fb2 100644 --- a/frmts/nitf/nitfrasterband.cpp +++ b/frmts/nitf/nitfrasterband.cpp @@ -9,7 +9,7 @@ * Copyright (c) 2011-2013, Even Rouault * * Portions Copyright (c) Her majesty the Queen in right of Canada as - * represented by the Minister of National Defence, 2006. + * represented by the Minister of National Defence, 2006, 2020 * * SPDX-License-Identifier: MIT ****************************************************************************/ @@ -1104,3 +1104,139 @@ GDALRasterBand *NITFWrapperRasterBand::GetOverview(int iOverview) return NITFProxyPamRasterBand::GetOverview(iOverview); } + +/************************************************************************/ +/* NITFComplexRasterBand() */ +/************************************************************************/ + +NITFComplexRasterBand::NITFComplexRasterBand(NITFDataset *poDSIn, + GDALRasterBand *poBandI, + GDALRasterBand *poBandQ, + int nIBand, int nQBand) + : NITFRasterBand(poDSIn, nIBand) +{ + + CPLAssert(poBandI->GetRasterDataType() == poBandQ->GetRasterDataType()); + underlyingDataType = poBandI->GetRasterDataType(); + + //add the I and Q bands to an intermediate dataset + poIntermediateDS = std::make_unique(); + poIntermediateDS->nRasterXSize = poDSIn->nRasterXSize; + poIntermediateDS->nRasterYSize = poDSIn->nRasterYSize; + poIntermediateDS->eAccess = poDSIn->eAccess; + + poIntermediateDS->SetBand(nIBand, poBandI); + poIntermediateDS->SetBand(nQBand, poBandQ); + + anBandMap[0] = nIBand; + anBandMap[1] = nQBand; + + //set the new datatype + switch (underlyingDataType) + { + case GDT_Int16: + eDataType = GDT_CInt16; + break; + case GDT_Int32: + eDataType = GDT_CInt32; + break; + case GDT_Float32: + eDataType = GDT_CFloat32; + break; + case GDT_Float64: + eDataType = GDT_CFloat64; + break; + default: + eDataType = GDT_Unknown; + CPLError(CE_Failure, CPLE_NotSupported, + "Unsupported complex datatype"); + break; + } + + complexDataTypeSize = GDALGetDataTypeSizeBytes(eDataType); + underlyingDataTypeSize = GDALGetDataTypeSizeBytes(underlyingDataType); + CPLAssert(underlyingDataTypeSize * 2 == complexDataTypeSize); + + poBandI->GetBlockSize(&nBlockXSize, &nBlockYSize); +} + +/************************************************************************/ +/* IReadBlock() */ +/************************************************************************/ + +CPLErr NITFComplexRasterBand::IBlockIO(int nBlockXOff, int nBlockYOff, + void *pImage, GDALRWFlag rwFlag) + +{ + int nRequestYSize; + int nRequestXSize; + bool bMemset = false; + + /* -------------------------------------------------------------------- */ + /* If the last strip is partial, we need to avoid */ + /* over-requesting. We also need to initialize the extra part */ + /* of the block to zero. */ + /* -------------------------------------------------------------------- */ + if ((nBlockYOff + 1) * nBlockYSize > nRasterYSize) + { + nRequestYSize = nRasterYSize - nBlockYOff * nBlockYSize; + if (rwFlag == GF_Read) + bMemset = true; + } + else + { + nRequestYSize = nBlockYSize; + } + + /*-------------------------------------------------------------------- */ + /* If the input imagery is tiled, also need to avoid over- */ + /* requesting in the X-direction. */ + /* ------------------------------------------------------------------- */ + if ((nBlockXOff + 1) * nBlockXSize > nRasterXSize) + { + nRequestXSize = nRasterXSize - nBlockXOff * nBlockXSize; + if (rwFlag == GF_Read) + bMemset = true; + } + else + { + nRequestXSize = nBlockXSize; + } + + if (bMemset) + { + memset(pImage, 0, + static_cast(GDALGetDataTypeSizeBytes(eDataType)) * + nBlockXSize * nBlockYSize); + } + + //read/write both bands with interleaved pixels + return poIntermediateDS->RasterIO( + rwFlag, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, + nRequestXSize, nRequestYSize, pImage, nRequestXSize, nRequestYSize, + underlyingDataType, 2, &anBandMap[0], complexDataTypeSize, + static_cast(complexDataTypeSize) * nBlockXSize, + underlyingDataTypeSize, nullptr); +} + +/************************************************************************/ +/* IReadBlock() */ +/************************************************************************/ + +CPLErr NITFComplexRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, + void *pImage) + +{ + return IBlockIO(nBlockXOff, nBlockYOff, pImage, GF_Read); +} + +/************************************************************************/ +/* IWriteBlock() */ +/************************************************************************/ + +CPLErr NITFComplexRasterBand::IWriteBlock(int nBlockXOff, int nBlockYOff, + void *pImage) + +{ + return IBlockIO(nBlockXOff, nBlockYOff, pImage, GF_Write); +} From a7e3b3d3b936705dce0f833d5fed749d73fcffb8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 18 Oct 2024 01:26:11 +0200 Subject: [PATCH 02/54] test_gdalmove.py: avoid random failure on ASAN CI config --- autotest/pyscripts/test_gdalmove.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/autotest/pyscripts/test_gdalmove.py b/autotest/pyscripts/test_gdalmove.py index b8e65a63de1f..84080916d47f 100755 --- a/autotest/pyscripts/test_gdalmove.py +++ b/autotest/pyscripts/test_gdalmove.py @@ -14,6 +14,7 @@ import shutil +import gdaltest import pytest import test_py_scripts @@ -36,6 +37,9 @@ def script_path(): def test_gdalmove_help(script_path): + if gdaltest.is_travis_branch("sanitize"): + pytest.skip("fails on sanitize for unknown reason") + assert "ERROR" not in test_py_scripts.run_py_script( script_path, "gdalmove", "--help" ) @@ -47,6 +51,9 @@ def test_gdalmove_help(script_path): def test_gdalmove_version(script_path): + if gdaltest.is_travis_branch("sanitize"): + pytest.skip("fails on sanitize for unknown reason") + assert "ERROR" not in test_py_scripts.run_py_script( script_path, "gdalmove", "--version" ) From 8e89a6c7a093b29384b41853a6d213aaefafe480 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 24 Oct 2024 18:00:03 +0200 Subject: [PATCH 03/54] SAFE: do not recognize manifest.safe file from RCM (RADARSAT Constellation Mission) --- frmts/safe/safedataset.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/frmts/safe/safedataset.cpp b/frmts/safe/safedataset.cpp index 6e5ebce7f325..b489a87751df 100644 --- a/frmts/safe/safedataset.cpp +++ b/frmts/safe/safedataset.cpp @@ -795,11 +795,14 @@ int SAFEDataset::Identify(GDALOpenInfo *poOpenInfo) if (poOpenInfo->nHeaderBytes < 100) return FALSE; - if (strstr((const char *)poOpenInfo->pabyHeader, "(poOpenInfo->pabyHeader); + if (!strstr(pszHeader, "pabyHeader, "sentinel-2") != nullptr) + // This driver doesn't handle Sentinel-2 or RCM (RADARSAT Constellation Mission) data + if (strstr(pszHeader, "sentinel-2") || + strstr(pszHeader, "rcm_prod_manifest.xsd")) return FALSE; return TRUE; From 4935d44504de4f24d8234e6967336cf5fc233b44 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 24 Oct 2024 18:07:18 +0200 Subject: [PATCH 04/54] RS2: make its Identify() method not to recognize RCM (RADARSAT Constellation Mission) datasets that also use product.xml --- frmts/rs2/rs2dataset.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/frmts/rs2/rs2dataset.cpp b/frmts/rs2/rs2dataset.cpp index 87c6de6dbfcf..7bd1f4e3fae7 100644 --- a/frmts/rs2/rs2dataset.cpp +++ b/frmts/rs2/rs2dataset.cpp @@ -558,11 +558,8 @@ int RS2Dataset::Identify(GDALOpenInfo *poOpenInfo) CPLString osMDFilename = CPLFormCIFilename(poOpenInfo->pszFilename, "product.xml", nullptr); - VSIStatBufL sStat; - if (VSIStatL(osMDFilename, &sStat) == 0) - return TRUE; - - return FALSE; + GDALOpenInfo oOpenInfo(osMDFilename.c_str(), GA_ReadOnly); + return Identify(&oOpenInfo); } /* otherwise, do our normal stuff */ From 1382c6da1e99506610158e9589a1a24e34282d88 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 24 Oct 2024 16:22:15 +0200 Subject: [PATCH 05/54] Add contributed raster read-only RCM (Radarsat Constellation Mission) driver This driver initially comes from https://github.com/DND-DRDC-RDDC/OS_DriverMetadataExtractionCodes_RCM_C-band_SARdata/tree/main/1%20-%20RCM%20C%2B%2B%20GDAL%20driver%20code/frmts/rcm with later changes by Esri, and various compilation warning/fixes by myself. --- .../expected_gdalinfo_formats.txt | 1 + ...indows_conda_expected_gdalinfo_formats.txt | 1 + doc/source/drivers/raster/index.rst | 1 + doc/source/drivers/raster/rcm.rst | 56 + frmts/CMakeLists.txt | 1 + frmts/drivers.ini | 1 + frmts/gdalallregister.cpp | 4 + frmts/rcm/CMakeLists.txt | 11 + frmts/rcm/rcmdataset.cpp | 2926 +++++++++++++++++ frmts/rcm/rcmdataset.h | 261 ++ frmts/rcm/rcmdrivercore.cpp | 159 + frmts/rcm/rcmdrivercore.h | 112 + gcore/gdal_frmts.h | 1 + 13 files changed, 3535 insertions(+) create mode 100644 doc/source/drivers/raster/rcm.rst create mode 100644 frmts/rcm/CMakeLists.txt create mode 100644 frmts/rcm/rcmdataset.cpp create mode 100644 frmts/rcm/rcmdataset.h create mode 100644 frmts/rcm/rcmdrivercore.cpp create mode 100644 frmts/rcm/rcmdrivercore.h diff --git a/.github/workflows/ubuntu_24.04/expected_gdalinfo_formats.txt b/.github/workflows/ubuntu_24.04/expected_gdalinfo_formats.txt index 2be3770e4ce0..7f71808905a7 100644 --- a/.github/workflows/ubuntu_24.04/expected_gdalinfo_formats.txt +++ b/.github/workflows/ubuntu_24.04/expected_gdalinfo_formats.txt @@ -154,4 +154,5 @@ Supported Formats: (ro:read-only, rw:read-write, +:update, v:virtual-I/O s:subda EHdr -raster- (rw+v): ESRI .hdr Labelled (*.bil) ISCE -raster- (rw+v): ISCE raster Zarr -raster,multidimensional raster- (rw+vs): Zarr + RCM -raster- (rovs): Radarsat Constellation Mission XML Product HTTP -raster,vector- (ro): HTTP Fetching Wrapper diff --git a/.github/workflows/windows_conda_expected_gdalinfo_formats.txt b/.github/workflows/windows_conda_expected_gdalinfo_formats.txt index 44fdb6efd66a..778903ad9c9b 100644 --- a/.github/workflows/windows_conda_expected_gdalinfo_formats.txt +++ b/.github/workflows/windows_conda_expected_gdalinfo_formats.txt @@ -153,4 +153,5 @@ Supported Formats: (ro:read-only, rw:read-write, +:update, v:virtual-I/O s:subda EHdr -raster- (rw+v): ESRI .hdr Labelled (*.bil) ISCE -raster- (rw+v): ISCE raster Zarr -raster,multidimensional raster- (rw+vs): Zarr + RCM -raster- (rovs): Radarsat Constellation Mission XML Product HTTP -raster,vector- (ro): HTTP Fetching Wrapper diff --git a/doc/source/drivers/raster/index.rst b/doc/source/drivers/raster/index.rst index 9f3b578a8fd5..9f93571959ad 100644 --- a/doc/source/drivers/raster/index.rst +++ b/doc/source/drivers/raster/index.rst @@ -148,6 +148,7 @@ Raster drivers rasterlite2 r rdb + rcm rik rmf roi_pac diff --git a/doc/source/drivers/raster/rcm.rst b/doc/source/drivers/raster/rcm.rst new file mode 100644 index 000000000000..cbe6f2d89531 --- /dev/null +++ b/doc/source/drivers/raster/rcm.rst @@ -0,0 +1,56 @@ +.. _raster.rcm: + +================================================================================ +RCM -- RADARSAT Constellation Mission Product +================================================================================ + +.. versionadded:: 3.11 + +.. shortname:: RCM + +.. built_in_by_default:: + +This driver will read RADARSAT Constellation Mission polarimetric products. + +The RADARSAT Constellation Mission XML products are distributed with a primary +XML file called product.xml, and a set of supporting XML data files with the +actual imagery stored in TIFF files. +The RCM driver will be used if the product.xml or the containing directory is +selected, and it can treat all the imagery as one consistent dataset. + +The RCM driver also reads geolocation tiepoints from the product.xml file and +represents them as GCPs on the dataset. + +Driver capabilities +------------------- + +.. supports_georeferencing:: + +.. supports_virtualio:: + +Data Calibration +---------------- + +If you wish to have GDAL apply a particular calibration LUT to the data +when you open it, you have to open the appropriate subdatasets. +The following subdatasets exist within the SUBDATASET domain for RCM products: + +- uncalibrated: open with ``RCM_CALIB:UNCALIB:`` prepended to filename +- beta\ :sub:`0`: open with ``RCM_CALIB:BETA0:`` prepended to filename +- sigma\ :sub:`0`: open with ``RCM_CALIB:SIGMA0:`` prepended to filename +- Gamma: open with ``RCM_CALIB:GAMMA:`` prepended to filename + +Note that geocoded (GCC/GCD) products do not have this functionality available. +Also be aware that the LUTs must be in the product directory where specified in +the product.xml, otherwise loading the product with the calibration LUT applied +will fail. + +One caveat worth noting is that the RCM driver will supply the calibrated data +as GDT_Float32 or GDT_CFloat32 depending on the type of calibration selected. +The uncalibrated data is provided as GDT_Int16/GDT_Byte/GDT_CInt16, also +depending on the type of product selected. + +See Also +-------- + +- RADARSAT Constellation Mission Product Specification RCM-SP-52-9092 diff --git a/frmts/CMakeLists.txt b/frmts/CMakeLists.txt index 26a45cfc5372..2daf2a0f65fa 100644 --- a/frmts/CMakeLists.txt +++ b/frmts/CMakeLists.txt @@ -154,6 +154,7 @@ gdal_optional_format(rik "RIK -- Swedish Grid Maps") gdal_optional_format(stacit "STACIT") gdal_optional_format(pdf "Geospatial PDF") # write only if none of GDAL_USE_POPPLER, GDAL_USE_PODOFO or GDAL_USE_PDFIUM # is set +gdal_optional_format(rcm "Radarsat Constellation Mission") # formats with external library dependency gdal_dependent_format(png "PNG image format" "GDAL_USE_PNG OR GDAL_USE_PNG_INTERNAL") diff --git a/frmts/drivers.ini b/frmts/drivers.ini index 2b8c78828b7f..3dfb5cf14f9a 100644 --- a/frmts/drivers.ini +++ b/frmts/drivers.ini @@ -281,6 +281,7 @@ ENVI EHdr ISCE Zarr +RCM # Register GDAL HTTP last, to let a chance to other drivers # accepting URL to handle them before diff --git a/frmts/gdalallregister.cpp b/frmts/gdalallregister.cpp index 21d90de23ccb..00210db69645 100644 --- a/frmts/gdalallregister.cpp +++ b/frmts/gdalallregister.cpp @@ -822,6 +822,10 @@ void CPL_STDCALL GDALAllRegister() GDALRegister_Zarr(); #endif +#ifdef FRMT_rcm + GDALRegister_RCM(); +#endif + /* -------------------------------------------------------------------- */ /* Register GDAL HTTP last, to let a chance to other drivers */ /* accepting URL to handle them before. */ diff --git a/frmts/rcm/CMakeLists.txt b/frmts/rcm/CMakeLists.txt new file mode 100644 index 000000000000..f6d342712c55 --- /dev/null +++ b/frmts/rcm/CMakeLists.txt @@ -0,0 +1,11 @@ +add_gdal_driver(TARGET gdal_RCM + SOURCES rcmdataset.cpp rcmdataset.h + CORE_SOURCES rcmdrivercore.cpp rcmdrivercore.h + PLUGIN_CAPABLE + NO_DEPS + NO_SHARED_SYMBOL_WITH_CORE) +gdal_standard_includes(gdal_RCM) + +if(NOT TARGET gdal_RCM) + return() +endif() diff --git a/frmts/rcm/rcmdataset.cpp b/frmts/rcm/rcmdataset.cpp new file mode 100644 index 000000000000..8e1086ade405 --- /dev/null +++ b/frmts/rcm/rcmdataset.cpp @@ -0,0 +1,2926 @@ +/****************************************************************************** + * + * Project: DRDC Ottawa GEOINT + * Purpose: Radarsat Constellation Mission - XML Products (product.xml) driver + * Author: Roberto Caron, MDA + * on behalf of DRDC Ottawa + * + ****************************************************************************** + * Copyright (c) 2020, DRDC Ottawa + * + * Based on the RS2 Dataset Class + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#include +#include +#include + +#include "cpl_minixml.h" +#include "gdal_frmts.h" +#include "gdal_pam.h" +#include "ogr_spatialref.h" +#include "rcmdataset.h" +#include "rcmdrivercore.h" + +#include + +constexpr int max_space_for_string = 32; + +/* RCM has a special folder that contains all LUT, Incidence Angle and Noise + * Level files */ +constexpr const char *CALIBRATION_FOLDER = "calibration"; + +/*** Function to test for valid LUT files ***/ +static bool IsValidXMLFile(const char *pszPath, const char *pszLut) +{ + /* Return true for valid xml file, false otherwise */ + char *pszLutFile = VSIStrdup(CPLFormFilename(pszPath, pszLut, nullptr)); + + CPLXMLTreeCloser psLut(CPLParseXMLFile(pszLutFile)); + + CPLFree(pszLutFile); + + if (psLut.get() == nullptr) + { + CPLError(CE_Failure, CPLE_OpenFailed, + "ERROR: Failed to open the LUT file %s", pszLutFile); + } + + return psLut.get() != nullptr; +} + +static double *InterpolateValues(char **papszList, int tableSize, int stepSize, + int numberOfValues, int pixelFirstLutValue) +{ + /* Allocate the right LUT size according to the product range pixel */ + double *table = static_cast(CPLCalloc(sizeof(double), tableSize)); + + if (stepSize <= 0) + { + /* When negative, the range of pixel is calculated from the opposite + * starting from the end of gains array */ + /* Just step the range with positive value */ + const int positiveStepSize = abs(stepSize); + + int k = 0; + + if (positiveStepSize == 1) + { + // Be fast and just copy the values because all gain values + // represent all image wide pixel + /* Start at the end position and store in the opposite */ + for (int i = pixelFirstLutValue; i >= 0; i--) + { + const double value = CPLAtof(papszList[i]); + table[k++] = value; + } + } + else + { + + /* Interpolation between 2 numbers */ + for (int i = numberOfValues - 1; i >= 0; i--) + { + // We will consider the same value to cover the case that we + // will hit the last pixel + double valueFrom = CPLAtof(papszList[i]); + double valueTo = valueFrom; + + if (i > 0) + { + // We have room to pick the previous number to interpolate + // with + valueTo = CPLAtof(papszList[i - 1]); + } + + // If the valueFrom minus ValueTo equal 0, it means to finish + // off with the same number until the end of the table size + double interp = (valueTo - valueFrom) / positiveStepSize; + + // Always begin with the value FROM found + table[k++] = valueFrom; + + // Then add interpolation, don't forget. The stepSize is + // actually counting our valueFrom number thus we add + // interpolation until the last step - 1 + for (int j = 0; j < positiveStepSize - 1; j++) + { + valueFrom += interp; + table[k++] = valueFrom; + } + } + } + } + else + { + /* When positive, the range of pixel is calculated from the beginning of + * gains array */ + if (stepSize == 1) + { + // Be fast and just copy the values because all gain values + // represent all image wide pixel + for (int i = 0; i < numberOfValues; i++) + { + const double value = CPLAtof(papszList[i]); + table[i] = value; + } + } + else + { + /* Interpolation between 2 numbers */ + int k = 0; + for (int i = 0; i < numberOfValues; i++) + { + // We will consider the same value to cover the case that we + // will hit the last pixel + double valueFrom = CPLAtof(papszList[i]); + double valueTo = valueFrom; + + if (i < (numberOfValues)-1) + { + // We have room to pick the next number to interpolate with + valueTo = CPLAtof(papszList[i + 1]); + } + + // If the valueFrom minus ValueTo equal 0, it means to finish + // off with the same number until the end of the table size + double interp = (valueTo - valueFrom) / stepSize; + + // Always begin with the value FROM found + table[k++] = valueFrom; + + // Then add interpolation, don't forget. The stepSize is + // actually counting our valueFrom number thus we add + // interpolation until the last step - 1 + for (int j = 0; j < stepSize - 1; j++) + { + valueFrom += interp; + table[k++] = valueFrom; + } + } + } + } + + return table; +} + +/*** check that the referenced dataset for each band has the +correct data type and returns whether a 2 band I+Q dataset should +be mapped onto a single complex band. +Returns BANDERROR for error, STRAIGHT for 1:1 mapping, TWOBANDCOMPLEX for 2 +bands -> 1 complex band +*/ +typedef enum +{ + BANDERROR, + STRAIGHT, + TWOBANDCOMPLEX +} BandMappingRCM; + +static BandMappingRCM checkBandFileMappingRCM(GDALDataType dataType, + GDALDataset *poBandFile, + bool isNITF) +{ + + GDALRasterBand *band1 = poBandFile->GetRasterBand(1); + GDALDataType bandfileType = band1->GetRasterDataType(); + // if there is one band and it has the same datatype, the band file gets + // passed straight through + if ((poBandFile->GetRasterCount() == 1 || + poBandFile->GetRasterCount() == 4) && + dataType == bandfileType) + return STRAIGHT; + + // if the band file has 2 bands, they should represent I+Q + // and be a compatible data type + if (poBandFile->GetRasterCount() == 2 && GDALDataTypeIsComplex(dataType)) + { + GDALRasterBand *band2 = poBandFile->GetRasterBand(2); + + if (bandfileType != band2->GetRasterDataType()) + return BANDERROR; // both bands must be same datatype + + // check compatible types - there are 4 complex types in GDAL + if ((dataType == GDT_CInt16 && bandfileType == GDT_Int16) || + (dataType == GDT_CInt32 && bandfileType == GDT_Int32) || + (dataType == GDT_CFloat32 && bandfileType == GDT_Float32) || + (dataType == GDT_CFloat64 && bandfileType == GDT_Float64)) + return TWOBANDCOMPLEX; + + if ((dataType == GDT_CInt16 && bandfileType == GDT_CInt16) || + (dataType == GDT_CInt32 && bandfileType == GDT_CInt32) || + (dataType == GDT_CFloat32 && bandfileType == GDT_CFloat32) || + (dataType == GDT_CFloat64 && bandfileType == GDT_CFloat64)) + return TWOBANDCOMPLEX; + } + + if (isNITF) + { + return STRAIGHT; + } + + return BANDERROR; // don't accept any other combinations +} + +/************************************************************************/ +/* RCMRasterBand */ +/************************************************************************/ + +RCMRasterBand::RCMRasterBand(RCMDataset *poDSIn, int nBandIn, + GDALDataType eDataTypeIn, const char *pszPole, + GDALDataset *poBandFileIn, bool bTwoBandComplex, + bool isOneFilePerPolIn, bool isNITFIn) + : poBandFile(poBandFileIn), poRCMDataset(poDSIn), + twoBandComplex(bTwoBandComplex), isOneFilePerPol(isOneFilePerPolIn), + isNITF(isNITFIn) +{ + poDS = poDSIn; + this->nBand = nBandIn; + eDataType = eDataTypeIn; + + /*Check image type, whether there is one file per polarization or + *one file containing all polarizations*/ + if (this->isOneFilePerPol) + { + poBand = poBandFile->GetRasterBand(1); + } + else + { + poBand = poBandFile->GetRasterBand(this->nBand); + } + + poBand->GetBlockSize(&nBlockXSize, &nBlockYSize); + + if (pszPole != nullptr && strlen(pszPole) != 0) + { + SetMetadataItem("POLARIMETRIC_INTERP", pszPole); + } +} + +double RCMRasterBand::GetLUT(int) +{ + return std::numeric_limits::quiet_NaN(); +} + +int RCMRasterBand::GetLUTsize() +{ + return 0; +} + +const char *RCMRasterBand::GetLUTFilename() +{ + return nullptr; +} + +double RCMRasterBand::GetLUTOffset() +{ + return 0.0f; +} + +bool RCMRasterBand::IsComplex() +{ + if (this->m_eType == GDT_CInt16 || this->m_eType == GDT_CInt32 || + this->m_eType == GDT_CFloat32 || this->m_eType == GDT_CFloat64) + { + return true; + } + else + { + return false; + } +} + +bool RCMRasterBand::IsExistLUT() +{ + return false; +} + +eCalibration RCMRasterBand::GetCalibration() +{ + return this->m_eCalib; +} + +void RCMRasterBand::SetPartialLUT(int, int) +{ + // nothing to do +} + +double RCMRasterBand::GetNoiseLevels(int) +{ + return 0.0f; +} + +int RCMRasterBand::GetNoiseLevelsSize() +{ + return 0; +} + +const char *RCMRasterBand::GetNoiseLevelsFilename() +{ + return nullptr; +} + +bool RCMRasterBand::IsExistNoiseLevels() +{ + return false; +} + +/************************************************************************/ +/* RCMRasterBand() */ +/************************************************************************/ + +RCMRasterBand::~RCMRasterBand() + +{ + if (poBandFile != nullptr) + GDALClose(reinterpret_cast(poBandFile)); +} + +/************************************************************************/ +/* IReadBlock() */ +/************************************************************************/ + +CPLErr RCMRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, void *pImage) + +{ + int nRequestYSize; + int nRequestXSize; + + /* -------------------------------------------------------------------- */ + /* If the last strip is partial, we need to avoid */ + /* over-requesting. We also need to initialize the extra part */ + /* of the block to zero. */ + /* -------------------------------------------------------------------- */ + if ((nBlockYOff + 1) * nBlockYSize > nRasterYSize) + { + nRequestYSize = nRasterYSize - nBlockYOff * nBlockYSize; + memset(pImage, 0, + (GDALGetDataTypeSize(eDataType) / 8) * nBlockXSize * + nBlockYSize); + } + else + { + nRequestYSize = nBlockYSize; + } + + /*-------------------------------------------------------------------- */ + /* If the input imagery is tiled, also need to avoid over- */ + /* requesting in the X-direction. */ + /* ------------------------------------------------------------------- */ + if ((nBlockXOff + 1) * nBlockXSize > nRasterXSize) + { + nRequestXSize = nRasterXSize - nBlockXOff * nBlockXSize; + memset(pImage, 0, + (GDALGetDataTypeSize(eDataType) / 8) * nBlockXSize * + nBlockYSize); + } + else + { + nRequestXSize = nBlockXSize; + } + + int dataTypeSize = GDALGetDataTypeSizeBytes(eDataType); + GDALDataType bandFileType = + poBandFile->GetRasterBand(1)->GetRasterDataType(); + int bandFileSize = GDALGetDataTypeSizeBytes(bandFileType); + + // case: 2 bands representing I+Q -> one complex band + if (twoBandComplex && !this->isNITF) + { + // int bandFileSize = GDALGetDataTypeSizeBytes(bandFileType); + // this data type is the complex version of the band file + // Roberto: don't check that for the moment: CPLAssert(dataTypeSize == + // bandFileSize * 2); + + return + // I and Q from each band are pixel-interleaved into this complex + // band + poBandFile->RasterIO( + GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, + nRequestXSize, nRequestYSize, pImage, nRequestXSize, + nRequestYSize, bandFileType, 2, nullptr, dataTypeSize, + dataTypeSize * nBlockXSize, bandFileSize, nullptr); + } + else if (twoBandComplex && this->isNITF) + { + return poBand->RasterIO( + GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, + nRequestXSize, nRequestYSize, pImage, nRequestXSize, nRequestYSize, + eDataType, 0, dataTypeSize * nBlockXSize, nullptr); + } + + if (poRCMDataset->IsComplexData()) + { + // this data type is the complex version of the band file + // Roberto: don't check that for the moment: CPLAssert(dataTypeSize == + // bandFileSize * 2); + return + // I and Q from each band are pixel-interleaved into this complex + // band + poBandFile->RasterIO( + GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, + nRequestXSize, nRequestYSize, pImage, nRequestXSize, + nRequestYSize, bandFileType, 2, nullptr, dataTypeSize, + nBlockXSize * dataTypeSize, bandFileSize, nullptr); + } + + // case: band file == this band + // NOTE: if the underlying band is opened with the NITF driver, it may + // combine 2 band I+Q -> complex band + else if (poBandFile->GetRasterBand(1)->GetRasterDataType() == eDataType) + { + return poBand->RasterIO( + GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, + nRequestXSize, nRequestYSize, pImage, nRequestXSize, nRequestYSize, + eDataType, 0, dataTypeSize * nBlockXSize, nullptr); + } + else + { + CPLAssert(FALSE); + return CE_Failure; + } +} + +/************************************************************************/ +/* ReadLUT() */ +/************************************************************************/ +/* Read the provided LUT in to m_ndTable */ +/* 1. The gains list spans the range extent covered by all */ +/* beams(if applicable). */ +/* 2. The mapping between the entry of gains */ +/* list and the range sample index is : the range sample */ +/* index = gains entry index * stepSize + pixelFirstLutValue, */ +/* where the gains entry index starts with ‘0’.For ScanSAR SLC, */ +/* the range sample index refers to the index on the COPG */ +/************************************************************************/ +void RCMCalibRasterBand::ReadLUT() +{ + + char bandNumber[12]; + snprintf(bandNumber, sizeof(bandNumber), "%d", poDS->GetRasterCount() + 1); + + CPLXMLNode *psLUT = CPLParseXMLFile(m_pszLUTFile); + + this->m_nfOffset = CPLAtof(CPLGetXMLValue(psLUT, "=lut.offset", "0.0")); + + this->pixelFirstLutValue = + atoi(CPLGetXMLValue(psLUT, "=lut.pixelFirstLutValue", "0")); + + this->stepSize = atoi(CPLGetXMLValue(psLUT, "=lut.stepSize", "0")); + + this->numberOfValues = + atoi(CPLGetXMLValue(psLUT, "=lut.numberOfValues", "0")); + + if (this->numberOfValues <= 0) + { + CPLError( + CE_Failure, CPLE_NotSupported, + "ERROR: The RCM driver does not support the LUT Number Of Values " + "equal or lower than zero."); + return; + } + + char **papszLUTList = CSLTokenizeString2( + CPLGetXMLValue(psLUT, "=lut.gains", ""), " ", CSLT_HONOURSTRINGS); + + if (this->stepSize <= 0) + { + if (this->pixelFirstLutValue <= 0) + { + CPLError( + CE_Failure, CPLE_NotSupported, + "ERROR: The RCM driver does not support LUT Pixel First Lut " + "Value equal or lower than zero when theproduct is " + "descending."); + return; + } + } + + /* Get the Pixel Per range */ + this->m_nTableSize = abs(this->stepSize) * abs(this->numberOfValues); + + if (this->m_nTableSize < this->m_poBandDataset->GetRasterXSize()) + { + CPLError( + CE_Failure, CPLE_NotSupported, + "ERROR: The RCM driver does not support range of LUT gain values " + "lower than the full image pixel range."); + return; + } + + /* Allocate the right LUT size according to the product range pixel */ + this->m_nfTable = + InterpolateValues(papszLUTList, this->m_nTableSize, this->stepSize, + this->numberOfValues, this->pixelFirstLutValue); + + const size_t nLen = + this->m_nTableSize * max_space_for_string; // 32 max + space + char *lut_gains = static_cast(CPLMalloc(nLen)); + memset(lut_gains, 0, nLen); + + for (int i = 0; i < this->m_nTableSize; i++) + { + char lut[max_space_for_string]; + // 6.123004711900930e+04 %e Scientific annotation + snprintf(lut, sizeof(lut), "%e ", this->m_nfTable[i]); + strcat(lut_gains, lut); + } + +#ifdef _TRACE_RCM + write_to_file("RCM ReadLUT m_pszLUTFile=", m_pszLUTFile); + write_to_file(" m_nfTable=", lut_gains); +#endif + + poDS->SetMetadataItem(CPLString("LUT_GAINS_").append(bandNumber).c_str(), + lut_gains); + // Can free this because the function SetMetadataItem takes a copy + CPLFree(lut_gains); + + char snum[256]; + if (this->m_eCalib == eCalibration::Sigma0) + { + poDS->SetMetadataItem(CPLString("LUT_TYPE_").append(bandNumber).c_str(), + "SIGMA0"); + } + else if (this->m_eCalib == eCalibration::Beta0) + { + poDS->SetMetadataItem(CPLString("LUT_TYPE_").append(bandNumber).c_str(), + "BETA0"); + } + else if (this->m_eCalib == eCalibration::Gamma) + { + poDS->SetMetadataItem(CPLString("LUT_TYPE_").append(bandNumber).c_str(), + "GAMMA"); + } + snprintf(snum, sizeof(snum), "%d", this->m_nTableSize); + poDS->SetMetadataItem(CPLString("LUT_SIZE_").append(bandNumber).c_str(), + snum); + snprintf(snum, sizeof(snum), "%f", this->m_nfOffset); + poDS->SetMetadataItem(CPLString("LUT_OFFSET_").append(bandNumber).c_str(), + snum); + + CPLDestroyXMLNode(psLUT); + + CSLDestroy(papszLUTList); +} + +/************************************************************************/ +/* ReadNoiseLevels() */ +/************************************************************************/ +/* Read the provided LUT in to m_nfTableNoiseLevels */ +/* 1. The gains list spans the range extent covered by all */ +/* beams(if applicable). */ +/* 2. The mapping between the entry of gains */ +/* list and the range sample index is : the range sample */ +/* index = gains entry index * stepSize + pixelFirstLutValue, */ +/* where the gains entry index starts with ‘0’.For ScanSAR SLC, */ +/* the range sample index refers to the index on the COPG */ +/************************************************************************/ +void RCMCalibRasterBand::ReadNoiseLevels() +{ + + this->m_nfTableNoiseLevels = nullptr; + + if (this->m_pszNoiseLevelsFile == nullptr) + { + return; + } + + char bandNumber[12]; + snprintf(bandNumber, sizeof(bandNumber), "%d", poDS->GetRasterCount() + 1); + + CPLXMLNode *psNoiseLevels = CPLParseXMLFile(this->m_pszNoiseLevelsFile); + + // Load Beta Nought, Sigma Nought, Gamma noise levels + // Loop through all nodes with spaces + CPLXMLNode *psreferenceNoiseLevelNode = + CPLGetXMLNode(psNoiseLevels, "=noiseLevels"); + + CPLXMLNode *psNodeInc; + for (psNodeInc = psreferenceNoiseLevelNode->psChild; psNodeInc != nullptr; + psNodeInc = psNodeInc->psNext) + { + if (EQUAL(psNodeInc->pszValue, "referenceNoiseLevel")) + { + CPLXMLNode *psCalibType = + CPLGetXMLNode(psNodeInc, "sarCalibrationType"); + CPLXMLNode *psPixelFirstNoiseValue = + CPLGetXMLNode(psNodeInc, "pixelFirstNoiseValue"); + CPLXMLNode *psStepSize = CPLGetXMLNode(psNodeInc, "stepSize"); + CPLXMLNode *psNumberOfValues = + CPLGetXMLNode(psNodeInc, "numberOfValues"); + CPLXMLNode *psNoiseLevelValues = + CPLGetXMLNode(psNodeInc, "noiseLevelValues"); + + if (psCalibType != nullptr && psPixelFirstNoiseValue != nullptr && + psStepSize != nullptr && psNumberOfValues != nullptr && + psNoiseLevelValues != nullptr) + { + const char *calibType = CPLGetXMLValue(psCalibType, "", ""); + this->pixelFirstLutValueNoiseLevels = + atoi(CPLGetXMLValue(psPixelFirstNoiseValue, "", "0")); + this->stepSizeNoiseLevels = + atoi(CPLGetXMLValue(psStepSize, "", "0")); + this->numberOfValuesNoiseLevels = + atoi(CPLGetXMLValue(psNumberOfValues, "", "0")); + const char *noiseLevelValues = + CPLGetXMLValue(psNoiseLevelValues, "", ""); + char **papszNoiseLevelList = CSLTokenizeString2( + noiseLevelValues, " ", CSLT_HONOURSTRINGS); + /* Get the Pixel Per range */ + this->m_nTableNoiseLevelsSize = + abs(this->stepSizeNoiseLevels) * + abs(this->numberOfValuesNoiseLevels); + + if ((EQUAL(calibType, "Beta Nought") && + this->m_eCalib == Beta0) || + (EQUAL(calibType, "Sigma Nought") && + this->m_eCalib == Sigma0) || + (EQUAL(calibType, "Gamma") && this->m_eCalib == Gamma)) + { + /* Allocate the right Noise Levels size according to the + * product range pixel */ + this->m_nfTableNoiseLevels = InterpolateValues( + papszNoiseLevelList, this->m_nTableNoiseLevelsSize, + this->stepSizeNoiseLevels, + this->numberOfValuesNoiseLevels, + this->pixelFirstLutValueNoiseLevels); + } + + CSLDestroy(papszNoiseLevelList); + + if (this->m_nfTableNoiseLevels != nullptr) + { + break; // We are done + } + } + } + } + +#ifdef _TRACE_RCM + if (this->m_nfTableNoiseLevels != nullptr) + { + const size_t nLen = + this->m_nTableNoiseLevelsSize * + max_space_for_string; // 12 max + space + 11 reserved + char *noise_levels_values = static_cast(CPLMalloc(nLen)); + memset(noise_levels_values, 0, nLen); + + for (int i = 0; i < this->m_nTableNoiseLevelsSize; i++) + { + char lut[max_space_for_string]; + // 6.123004711900930e+04 %e Scientific annotation + snprintf(lut, sizeof(lut), "%e ", this->m_nfTableNoiseLevels[i]); + strcat(noise_levels_values, lut); + } + write_to_file("RCM ReadNoiseLevel m_pszLUTFile=", m_pszNoiseLevelsFile); + write_to_file(" m_nfTableNoiseLevels=", noise_levels_values); + + CPLFree(noise_levels_values); + } +#endif +} + +/************************************************************************/ +/* RCMCalibRasterBand() */ +/************************************************************************/ + +RCMCalibRasterBand::RCMCalibRasterBand( + RCMDataset *poDataset, const char *pszPolarization, GDALDataType eType, + GDALDataset *poBandDataset, eCalibration eCalib, const char *pszLUT, + const char *pszNoiseLevels, GDALDataType eOriginalType) + : m_eCalib(eCalib), m_poBandDataset(poBandDataset), m_eType(eType), + m_eOriginalType(eOriginalType), m_pszLUTFile(VSIStrdup(pszLUT)), + m_pszNoiseLevelsFile(VSIStrdup(pszNoiseLevels)) +{ + this->poDS = poDataset; + + if (pszPolarization != nullptr && strlen(pszPolarization) != 0) + { + SetMetadataItem("POLARIMETRIC_INTERP", pszPolarization); + } + + // this->eDataType = eType; + + if ((eType == GDT_CInt16) || (eType == GDT_CFloat32)) + this->eDataType = GDT_CFloat32; + else + this->eDataType = GDT_Float32; + + GDALRasterBand *poRasterBand = poBandDataset->GetRasterBand(1); + poRasterBand->GetBlockSize(&nBlockXSize, &nBlockYSize); + + ReadLUT(); + ReadNoiseLevels(); +} + +double RCMCalibRasterBand::GetNoiseLevels(int pixel) +{ + return this->m_nfTableNoiseLevels[pixel]; +} + +int RCMCalibRasterBand::GetNoiseLevelsSize() +{ + return this->m_nTableNoiseLevelsSize; +} + +const char *RCMCalibRasterBand::GetNoiseLevelsFilename() +{ + return this->m_pszNoiseLevelsFile; +} + +bool RCMCalibRasterBand::IsExistNoiseLevels() +{ + if (this->m_nfTableNoiseLevels == nullptr || + this->m_pszNoiseLevelsFile == nullptr || + strlen(this->m_pszNoiseLevelsFile) == 0 || + this->m_nTableNoiseLevelsSize == 0) + { + return false; + } + else + { + return true; + } +} + +bool RCMCalibRasterBand::IsComplex() +{ + if (this->m_eType == GDT_CInt16 || this->m_eType == GDT_CInt32 || + this->m_eType == GDT_CFloat32 || this->m_eType == GDT_CFloat64) + { + return true; + } + else + { + return false; + } +} + +eCalibration RCMCalibRasterBand::GetCalibration() +{ + return this->m_eCalib; +} + +double RCMCalibRasterBand::GetLUT(int pixel) +{ + return this->m_nfTable[pixel]; +} + +int RCMCalibRasterBand::GetLUTsize() +{ + return this->m_nTableSize; +} + +const char *RCMCalibRasterBand::GetLUTFilename() +{ + return this->m_pszLUTFile; +} + +double RCMCalibRasterBand::GetLUTOffset() +{ + return this->m_nfOffset; +} + +bool RCMCalibRasterBand::IsExistLUT() +{ + if (this->m_nfTable == nullptr || this->m_pszLUTFile == nullptr || + strlen(this->m_pszLUTFile) == 0 || this->m_nTableSize == 0) + { + return false; + } + else + { + return true; + } +} + +void RCMCalibRasterBand::SetPartialLUT(int pixel_offset, int pixel_width) +{ + /* Alway start from 0 */ + if (pixel_offset < 0) + { + pixel_offset = 0; + } + + if (pixel_offset < this->GetLUTsize()) + { + /* Can only change if the starting pixel in the raster width range */ + if ((pixel_offset + pixel_width) > this->GetLUTsize() - 1) + { + /* Ya but the width is way too large based on the raster width range + when beginning from a different offset Recalculate the true relative + width + */ + pixel_width = this->GetLUTsize() - pixel_offset - 1; + } + + if (pixel_width > 0) + { + /* Prepare a buffer */ + double *nfTableBuffer = + static_cast(CPLMalloc(sizeof(double) * pixel_width)); + memset(nfTableBuffer, 0, sizeof(double) * pixel_width); + + /* Copy a range */ + int j = 0; + for (int i = pixel_offset; i < (pixel_offset + pixel_width); i++) + { + nfTableBuffer[j++] = this->GetLUT(i); + } + + const size_t nLen = + pixel_width * + max_space_for_string; // 12 max + space + 11 reserved + char *lut_gains = static_cast(CPLMalloc(nLen)); + memset(lut_gains, 0, nLen); + + for (int i = 0; i < pixel_width; i++) + { + char lut[max_space_for_string]; + // 2.390641e+02 %e Scientific annotation + snprintf(lut, sizeof(lut), "%e ", nfTableBuffer[i]); + strcat(lut_gains, lut); + } + + char bandNumber[12]; + snprintf(bandNumber, sizeof(bandNumber), "%d", this->GetBand()); + + poDS->SetMetadataItem( + CPLString("LUT_GAINS_").append(bandNumber).c_str(), lut_gains); + // Can free this because the function SetMetadataItem takes a copy + CPLFree(lut_gains); + + /* and Set new LUT size */ + char snum[12]; + snprintf(snum, sizeof(snum), "%d", pixel_width); + poDS->SetMetadataItem( + CPLString("LUT_SIZE_").append(bandNumber).c_str(), snum); + + /* Change the internal value now */ + /* Free the old gains value table, not valid anymore */ + CPLFree(this->m_nfTable); + + /* New gains value table size */ + this->m_nTableSize = pixel_width; + + /* Realoocate and have new gain values from the buffer */ + this->m_nfTable = reinterpret_cast( + CPLMalloc(sizeof(double) * this->m_nTableSize)); + memset(this->m_nfTable, 0, sizeof(double) * this->m_nTableSize); + memcpy(this->m_nfTable, nfTableBuffer, + sizeof(double) * this->m_nTableSize); + + /* Free our buffer */ + CPLFree(nfTableBuffer); + } + } +} + +double *RCMCalibRasterBand::CloneLUT() +{ + double *values = nullptr; + + if (this->m_nfTable != nullptr) + { + values = reinterpret_cast( + CPLMalloc(sizeof(double) * this->m_nTableSize)); + memcpy(values, this->m_nfTable, sizeof(double) * this->m_nTableSize); + } + + return values; +} + +double *RCMCalibRasterBand::CloneNoiseLevels() +{ + double *values = nullptr; + + if (this->m_nfTableNoiseLevels != nullptr) + { + values = + (double *)malloc(sizeof(double) * this->m_nTableNoiseLevelsSize); + memcpy(values, this->m_nfTableNoiseLevels, + sizeof(double) * this->m_nTableNoiseLevelsSize); + } + + return values; +} + +/************************************************************************/ +/* ~RCMCalibRasterBand() */ +/************************************************************************/ + +RCMCalibRasterBand::~RCMCalibRasterBand() +{ + CPLFree(m_nfTable); + CPLFree(m_nfTableNoiseLevels); + CPLFree(m_pszLUTFile); + CPLFree(m_pszNoiseLevelsFile); + GDALClose(m_poBandDataset); +} + +/************************************************************************/ +/* IReadBlock() */ +/************************************************************************/ + +CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, + void *pImage) +{ + CPLErr eErr; + int nRequestYSize; + int nRequestXSize; + + /* -------------------------------------------------------------------- */ + /* If the last strip is partial, we need to avoid */ + /* over-requesting. We also need to initialize the extra part */ + /* of the block to zero. */ + /* -------------------------------------------------------------------- */ + if ((nBlockYOff + 1) * nBlockYSize > nRasterYSize) + { + nRequestYSize = nRasterYSize - nBlockYOff * nBlockYSize; + memset(pImage, 0, + (GDALGetDataTypeSize(eDataType) / 8) * nBlockXSize * + nBlockYSize); + } + else + { + nRequestYSize = nBlockYSize; + } + + /*-------------------------------------------------------------------- */ + /* If the input imagery is tiled, also need to avoid over- */ + /* requesting in the X-direction. */ + /* ------------------------------------------------------------------- */ + if ((nBlockXOff + 1) * nBlockXSize > nRasterXSize) + { + nRequestXSize = nRasterXSize - nBlockXOff * nBlockXSize; + memset(pImage, 0, + (GDALGetDataTypeSize(eDataType) / 8) * nBlockXSize * + nBlockYSize); + } + else + { + nRequestXSize = nBlockXSize; + } + + if (this->m_eOriginalType == GDT_CInt16) + { + GInt16 *pnImageTmp; + /* read in complex values */ + pnImageTmp = (GInt16 *)CPLMalloc(2 * nBlockXSize * nBlockYSize * + GDALGetDataTypeSize(GDT_Int16) / 8); + + if (m_poBandDataset->GetRasterCount() == 2) + { + eErr = m_poBandDataset->RasterIO( + GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, + nRequestXSize, nRequestYSize, pnImageTmp, nRequestXSize, + nRequestYSize, this->m_eOriginalType, 2, nullptr, 4, + nBlockXSize * 4, 4, nullptr); + + /* + eErr = m_poBandDataset->RasterIO(GF_Read, + nBlockXOff * nBlockXSize, + nBlockYOff * nBlockYSize, + nRequestXSize, nRequestYSize, + pnImageTmp, nRequestXSize, nRequestYSize, + GDT_Int32, + 2, nullptr, 4, nBlockXSize * 4, 2, nullptr); + */ + } + else + { + eErr = m_poBandDataset->RasterIO( + GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, + nRequestXSize, nRequestYSize, pnImageTmp, nRequestXSize, + nRequestYSize, this->m_eOriginalType, 1, nullptr, 4, + nBlockXSize * 4, 0, nullptr); + + /* + eErr = + m_poBandDataset->RasterIO(GF_Read, + nBlockXOff * nBlockXSize, + nBlockYOff * nBlockYSize, + nRequestXSize, nRequestYSize, + pnImageTmp, nRequestXSize, nRequestYSize, + GDT_UInt32, + 1, nullptr, 4, nBlockXSize * 4, 0, nullptr); + */ + +#ifdef CPL_LSB + /* First, undo the 32bit swap. */ + GDALSwapWords(pImage, 4, nBlockXSize * nBlockYSize, 4); + + /* Then apply 16 bit swap. */ + GDALSwapWords(pImage, 2, nBlockXSize * nBlockYSize * 2, 2); +#endif + } + + /* calibrate the complex values */ + for (int i = 0; i < nRequestYSize; i++) + { + for (int j = 0; j < nRequestXSize; j++) + { + /* calculate pixel offset in memory*/ + int nPixOff = (2 * (i * nBlockXSize)) + (j * 2); + int nTruePixOff = (i * nBlockXSize) + j; + + // Formula for Complex Q+J + float real = (float)pnImageTmp[nPixOff]; + float img = (float)pnImageTmp[nPixOff + 1]; + float digitalValue = (real * real) + (img * img); + float lutValue = + static_cast(m_nfTable[nBlockXOff * nBlockXSize + j]); + float calibValue = digitalValue / (lutValue * lutValue); + +#ifdef _TRACE_RCM + if (nBlockXOff * nBlockXSize + j >= 4961) + { + char msgBlocks[2048] = ""; + snprintf(msgBlocks, sizeof(msgBlocks), + "IReadBlock: [%d,%d] real=%f img=%f " + "digitalValue=%f lutValue[%d]=%f calibValue=%f", + i, j, real, img, digitalValue, + (nBlockXOff * nBlockXSize + j), lutValue, + calibValue); + write_to_file(msgBlocks, ""); + } +#endif + + ((float *)pImage)[nTruePixOff] = calibValue; + } + } + + CPLFree(pnImageTmp); + } + + // If the underlying file is NITF CFloat32 + else if (this->m_eOriginalType == GDT_CFloat32 || + this->m_eOriginalType == GDT_CFloat64) + { + /* read in complex values */ + float *pnImageTmp; + + int dataTypeSize = GDALGetDataTypeSize(this->m_eOriginalType) / 8; + GDALDataType bandFileType = this->m_eOriginalType; + int bandFileSize = GDALGetDataTypeSize(bandFileType) / 8; + + /* read the original image complex values in a temporary image space */ + pnImageTmp = + (float *)CPLMalloc(2 * nBlockXSize * nBlockYSize * bandFileSize); + + eErr = + // I and Q from each band are pixel-interleaved into this complex + // band + m_poBandDataset->RasterIO( + GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, + nRequestXSize, nRequestYSize, pnImageTmp, nRequestXSize, + nRequestYSize, bandFileType, 2, nullptr, dataTypeSize, + nBlockXSize * dataTypeSize, bandFileSize, nullptr); + + /* calibrate the complex values */ + for (int i = 0; i < nRequestYSize; i++) + { + for (int j = 0; j < nRequestXSize; j++) + { + /* calculate pixel offset in memory*/ + int nPixOff = (2 * (i * nBlockXSize)) + (j * 2); + int nTruePixOff = (i * nBlockXSize) + j; + + // Formula for Complex Q+J + float real = (float)pnImageTmp[nPixOff]; + float img = (float)pnImageTmp[nPixOff + 1]; + float digitalValue = (real * real) + (img * img); + float lutValue = + static_cast(m_nfTable[nBlockXOff * nBlockXSize + j]); + float calibValue = digitalValue / (lutValue * lutValue); + + ((float *)pImage)[nTruePixOff] = calibValue; + } + } + + CPLFree(pnImageTmp); + } + + else if (this->m_eOriginalType == GDT_Float32) + { + /* A Float32 is actual 4 bytes */ + eErr = m_poBandDataset->RasterIO( + GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, + nRequestXSize, nRequestYSize, pImage, nRequestXSize, nRequestYSize, + GDT_Float32, 1, nullptr, 4, nBlockXSize * 4, 0, nullptr); + + /* iterate over detected values */ + for (int i = 0; i < nRequestYSize; i++) + { + for (int j = 0; j < nRequestXSize; j++) + { + int nPixOff = (i * nBlockXSize) + j; + + /* For detected products, in order to convert the digital number + of a given range sample to a calibrated value, the digital value + is first squared, then the offset(B) is added and the result is + divided by the gains value(A) corresponding to the range sample. + RCM-SP-53-0419 Issue 2/5: January 2, 2018 Page 7-56 */ + float digitalValue = ((float *)pImage)[nPixOff]; + float A = + static_cast(m_nfTable[nBlockXOff * nBlockXSize + j]); + ((float *)pImage)[nPixOff] = + ((digitalValue * digitalValue) + + static_cast(this->m_nfOffset)) / + A; + } + } + } + + else if (this->m_eOriginalType == GDT_Float64) + { + /* A Float64 is actual 8 bytes */ + eErr = m_poBandDataset->RasterIO( + GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, + nRequestXSize, nRequestYSize, pImage, nRequestXSize, nRequestYSize, + GDT_Float64, 1, nullptr, 8, nBlockXSize * 8, 0, nullptr); + + /* iterate over detected values */ + for (int i = 0; i < nRequestYSize; i++) + { + for (int j = 0; j < nRequestXSize; j++) + { + int nPixOff = (i * nBlockXSize) + j; + + /* For detected products, in order to convert the digital number + of a given range sample to a calibrated value, the digital value + is first squared, then the offset(B) is added and the result is + divided by the gains value(A) corresponding to the range sample. + RCM-SP-53-0419 Issue 2/5: January 2, 2018 Page 7-56 */ + float digitalValue = ((float *)pImage)[nPixOff]; + float A = + static_cast(m_nfTable[nBlockXOff * nBlockXSize + j]); + ((float *)pImage)[nPixOff] = + ((digitalValue * digitalValue) + + static_cast(this->m_nfOffset)) / + A; + } + } + } + + else if (this->m_eOriginalType == GDT_UInt16) + { + GUInt16 *pnImageTmp; + /* read in detected values */ + pnImageTmp = (GUInt16 *)CPLMalloc(nBlockXSize * nBlockYSize * + GDALGetDataTypeSize(GDT_UInt16) / 8); + eErr = m_poBandDataset->RasterIO( + GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, + nRequestXSize, nRequestYSize, pnImageTmp, nRequestXSize, + nRequestYSize, GDT_UInt16, 1, nullptr, 2, nBlockXSize * 2, 0, + nullptr); + + /* iterate over detected values */ + for (int i = 0; i < nRequestYSize; i++) + { + for (int j = 0; j < nRequestXSize; j++) + { + int nPixOff = (i * nBlockXSize) + j; + + float digitalValue = (float)pnImageTmp[nPixOff]; + float A = + static_cast(m_nfTable[nBlockXOff * nBlockXSize + j]); + ((float *)pImage)[nPixOff] = + ((digitalValue * digitalValue) + + static_cast(this->m_nfOffset)) / + A; + } + } + CPLFree(pnImageTmp); + } /* Ticket #2104: Support for ScanSAR products */ + + else if (this->m_eOriginalType == GDT_Byte) + { + GByte *pnImageTmp; + pnImageTmp = (GByte *)CPLMalloc(nBlockXSize * nBlockYSize * + GDALGetDataTypeSize(GDT_Byte) / 8); + eErr = m_poBandDataset->RasterIO( + GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, + nRequestXSize, nRequestYSize, pnImageTmp, nRequestXSize, + nRequestYSize, GDT_Byte, 1, nullptr, 1, nBlockXSize, 0, nullptr); + + /* iterate over detected values */ + for (int i = 0; i < nRequestYSize; i++) + { + for (int j = 0; j < nRequestXSize; j++) + { + int nPixOff = (i * nBlockXSize) + j; + + float digitalValue = (float)pnImageTmp[nPixOff]; + float A = + static_cast(m_nfTable[nBlockXOff * nBlockXSize + j]); + ((float *)pImage)[nPixOff] = + ((digitalValue * digitalValue) + + static_cast(this->m_nfOffset)) / + A; + } + } + CPLFree(pnImageTmp); + } + else + { + CPLAssert(FALSE); + return CE_Failure; + } + return eErr; +} + +/************************************************************************/ +/* ==================================================================== */ +/* RCMDataset */ +/* ==================================================================== */ +/************************************************************************/ + +/************************************************************************/ +/* RCMDataset() */ +/************************************************************************/ + +RCMDataset::RCMDataset() : pszLutApplied(CPLStrdup("")) +{ + adfGeoTransform[0] = 0.0; + adfGeoTransform[1] = 1.0; + adfGeoTransform[2] = 0.0; + adfGeoTransform[3] = 0.0; + adfGeoTransform[4] = 0.0; + adfGeoTransform[5] = 1.0; +} + +/************************************************************************/ +/* ~RCMDataset() */ +/************************************************************************/ + +RCMDataset::~RCMDataset() + +{ + RCMDataset::FlushCache(true); + + CPLDestroyXMLNode(psProduct); + CPLFree(pszLutApplied); + + if (nGCPCount > 0) + { + GDALDeinitGCPs(nGCPCount, pasGCPList); + CPLFree(pasGCPList); + } + + RCMDataset::CloseDependentDatasets(); + + if (papszSubDatasets != nullptr) + CSLDestroy(papszSubDatasets); + + if (papszExtraFiles != nullptr) + CSLDestroy(papszExtraFiles); + + if (m_nfIncidenceAngleTable != nullptr) + CPLFree(m_nfIncidenceAngleTable); + + psProduct = nullptr; +} + +/************************************************************************/ +/* CloseDependentDatasets() */ +/************************************************************************/ + +int RCMDataset::CloseDependentDatasets() +{ + int bHasDroppedRef = GDALPamDataset::CloseDependentDatasets(); + + if (nBands != 0) + bHasDroppedRef = TRUE; + + for (int iBand = 0; iBand < nBands; iBand++) + { + delete papoBands[iBand]; + } + nBands = 0; + + return bHasDroppedRef; +} + +/************************************************************************/ +/* GetFileList() */ +/************************************************************************/ + +char **RCMDataset::GetFileList() + +{ + char **papszFileList = GDALPamDataset::GetFileList(); + + papszFileList = CSLInsertStrings(papszFileList, -1, papszExtraFiles); + + return papszFileList; +} + +/************************************************************************/ +/* Open() */ +/************************************************************************/ + +GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) + +{ + /* -------------------------------------------------------------------- */ + /* Is this a RCM Product.xml definition? */ + /* -------------------------------------------------------------------- */ + if (!RCMDatasetIdentify(poOpenInfo)) + { + return nullptr; + } + + /* -------------------------------------------------------------------- */ + /* Get subdataset information, if relevant */ + /* -------------------------------------------------------------------- */ + const char *pszFilename = poOpenInfo->pszFilename; + eCalibration eCalib = None; + + CPLString calibrationFormat(FormatCalibration(nullptr, nullptr)); + + if (STARTS_WITH_CI(pszFilename, calibrationFormat)) + { + // The calibration name and filename begins after the hard coded layer + // name + pszFilename += strlen(szLayerCalibration) + 1; + + if (STARTS_WITH_CI(pszFilename, szBETA0)) + { + eCalib = Beta0; + } + else if (STARTS_WITH_CI(pszFilename, szSIGMA0)) + { + eCalib = Sigma0; + } + else if (STARTS_WITH_CI(pszFilename, szGAMMA) || + STARTS_WITH_CI(pszFilename, "GAMMA0")) // Cover both situation + { + eCalib = Gamma; + } + else if (STARTS_WITH_CI(pszFilename, szUNCALIB)) + { + eCalib = Uncalib; + } + else + { + eCalib = None; + } + + /* advance the pointer to the actual filename */ + while (*pszFilename != '\0' && *pszFilename != ':') + pszFilename++; + + if (*pszFilename == ':') + pszFilename++; + + // need to redo the directory check: + // the GDALOpenInfo check would have failed because of the calibration + // string on the filename + VSIStatBufL sStat; + if (VSIStatL(pszFilename, &sStat) == 0) + poOpenInfo->bIsDirectory = VSI_ISDIR(sStat.st_mode); + } + + CPLString osMDFilename; + if (poOpenInfo->bIsDirectory) + { + /* Check for directory access when there is a product.xml file in the + directory. */ + osMDFilename = CPLFormCIFilename(pszFilename, "product.xml", nullptr); + + VSIStatBufL sStat; + if (VSIStatL(osMDFilename, &sStat) != 0) + { + /* If not, check for directory extra 'metadata' access when there is + a product.xml file in the directory. */ + osMDFilename = + CPLFormCIFilename(pszFilename, GetMetadataProduct(), nullptr); + } + } + else + osMDFilename = pszFilename; + + /* -------------------------------------------------------------------- */ + /* Ingest the Product.xml file. */ + /* -------------------------------------------------------------------- */ + CPLXMLNode *psProduct = CPLParseXMLFile(osMDFilename); + if (psProduct == nullptr) + return nullptr; + + /* -------------------------------------------------------------------- */ + /* Confirm the requested access is supported. */ + /* -------------------------------------------------------------------- */ + if (poOpenInfo->eAccess == GA_Update) + { + CPLDestroyXMLNode(psProduct); + CPLError(CE_Failure, CPLE_NotSupported, + "ERROR: The RCM driver does not support update " + "access to existing dataset."); + return nullptr; + } + + CPLXMLNode *psSceneAttributes = + CPLGetXMLNode(psProduct, "=product.sceneAttributes"); + if (psSceneAttributes == nullptr) + { + CPLDestroyXMLNode(psProduct); + CPLError(CE_Failure, CPLE_OpenFailed, + "ERROR: Failed to find in document."); + return nullptr; + } + + CPLXMLNode *psImageAttributes = + CPLGetXMLNode(psProduct, "=product.sceneAttributes.imageAttributes"); + if (psImageAttributes == nullptr) + { + CPLDestroyXMLNode(psProduct); + CPLError(CE_Failure, CPLE_OpenFailed, + "ERROR: Failed to find in " + "document."); + return nullptr; + } + + int numberOfEntries = + atoi(CPLGetXMLValue(psSceneAttributes, "numberOfEntries", "0")); + if (numberOfEntries != 1) + { + CPLDestroyXMLNode(psProduct); + CPLError(CE_Failure, CPLE_OpenFailed, + "ERROR: Only RCM with Complex Single-beam is supported."); + return nullptr; + } + + CPLXMLNode *psImageReferenceAttributes = + CPLGetXMLNode(psProduct, "=product.imageReferenceAttributes"); + if (psImageReferenceAttributes == nullptr) + { + CPLDestroyXMLNode(psProduct); + CPLError( + CE_Failure, CPLE_OpenFailed, + "ERROR: Failed to find in document."); + return nullptr; + } + + CPLXMLNode *psImageGenerationParameters = + CPLGetXMLNode(psProduct, "=product.imageGenerationParameters"); + if (psImageGenerationParameters == nullptr) + { + CPLDestroyXMLNode(psProduct); + CPLError( + CE_Failure, CPLE_OpenFailed, + "ERROR: Failed to find in document."); + return nullptr; + } + + /* -------------------------------------------------------------------- */ + /* Create the dataset. */ + /* -------------------------------------------------------------------- */ + RCMDataset *poDS = new RCMDataset(); + + poDS->psProduct = psProduct; + + /* -------------------------------------------------------------------- */ + /* Get overall image information. */ + /* -------------------------------------------------------------------- */ + poDS->nRasterXSize = atoi(CPLGetXMLValue( + psSceneAttributes, "imageAttributes.samplesPerLine", "-1")); + poDS->nRasterYSize = atoi( + CPLGetXMLValue(psSceneAttributes, "imageAttributes.numLines", "-1")); + if (poDS->nRasterXSize <= 1 || poDS->nRasterYSize <= 1) + { + delete poDS; + CPLError( + CE_Failure, CPLE_OpenFailed, + "ERROR: Non-sane raster dimensions provided in product.xml. If " + "this is " + "a valid RCM scene, please contact your data provider for " + "a corrected dataset."); + return nullptr; + } + + /* -------------------------------------------------------------------- */ + /* Check product type, as to determine if there are LUTs for */ + /* calibration purposes. */ + /* -------------------------------------------------------------------- */ + + const char *pszItem = + CPLGetXMLValue(psImageGenerationParameters, + "generalProcessingInformation.productType", "UNK"); + poDS->SetMetadataItem("PRODUCT_TYPE", pszItem); + const char *pszProductType = pszItem; + + pszItem = CPLGetXMLValue(psProduct, "=product.productId", "UNK"); + poDS->SetMetadataItem("PRODUCT_ID", pszItem); + + pszItem = CPLGetXMLValue( + psProduct, "=product.securityAttributes.securityClassification", "UNK"); + poDS->SetMetadataItem("SECURITY_CLASSIFICATION", pszItem); + + pszItem = CPLGetXMLValue( + psProduct, "=product.sourceAttributes.polarizationDataMode", "UNK"); + poDS->SetMetadataItem("POLARIZATION_DATA_MODE", pszItem); + + pszItem = CPLGetXMLValue(psImageGenerationParameters, + "generalProcessingInformation.processingFacility", + "UNK"); + poDS->SetMetadataItem("PROCESSING_FACILITY", pszItem); + + pszItem = + CPLGetXMLValue(psImageGenerationParameters, + "generalProcessingInformation.processingTime", "UNK"); + poDS->SetMetadataItem("PROCESSING_TIME", pszItem); + + pszItem = CPLGetXMLValue(psImageGenerationParameters, + "sarProcessingInformation.satelliteHeight", "UNK"); + poDS->SetMetadataItem("SATELLITE_HEIGHT", pszItem); + + pszItem = CPLGetXMLValue( + psImageGenerationParameters, + "sarProcessingInformation.zeroDopplerTimeFirstLine", "UNK"); + poDS->SetMetadataItem("FIRST_LINE_TIME", pszItem); + + pszItem = CPLGetXMLValue(psImageGenerationParameters, + "sarProcessingInformation.zeroDopplerTimeLastLine", + "UNK"); + poDS->SetMetadataItem("LAST_LINE_TIME", pszItem); + + pszItem = CPLGetXMLValue(psImageGenerationParameters, + "sarProcessingInformation.lutApplied", ""); + poDS->SetMetadataItem("LUT_APPLIED", pszItem); + poDS->pszLutApplied = static_cast(VSIMalloc(strlen(pszItem) + 2)); + poDS->pszLutApplied[0] = '\0'; + strcpy(poDS->pszLutApplied, pszItem); + + /*--------------------------------------------------------------------- + If true, a polarization dependent application LUT has been applied + for each polarization channel. Otherwise the same application LUT + has been applied for all polarization channels. Not applicable to + lookupTable = "Unity*" or if dataType = "Floating-Point". + --------------------------------------------------------------------- */ + pszItem = CPLGetXMLValue(psImageGenerationParameters, + "sarProcessingInformation.perPolarizationScaling", + "false"); + poDS->SetMetadataItem("PER_POLARIZATION_SCALING", pszItem); + if (EQUAL(pszItem, "true") || EQUAL(pszItem, "TRUE")) + { + poDS->bPerPolarizationScaling = TRUE; + } + + /* the following cases can be assumed to have no LUTs, as per + * RN-RP-51-2713, but also common sense + * SLC represents a SLant range georeferenced Complex product + * (i.e., equivalent to a Single-Look Complex product for RADARSAT-1 or + * RADARSAT-2). • GRD or GRC represent GRound range georeferenced Detected + * or Complex products (GRD is equivalent to an SGX, SCN or SCW product for + * RADARSAT1 or RADARSAT-2). • GCD or GCC represent GeoCoded Detected or + * Complex products (GCD is equivalent to an SSG or SPG product for + * RADARSAT-1 or RADARSAT-2). + */ + bool bCanCalib = false; + if (!(STARTS_WITH_CI(pszProductType, "UNK") || + STARTS_WITH_CI(pszProductType, "GCD") || + STARTS_WITH_CI(pszProductType, "GCC"))) + { + bCanCalib = true; + } + + /* -------------------------------------------------------------------- */ + /* Get dataType (so we can recognise complex data), and the */ + /* bitsPerSample. */ + /* -------------------------------------------------------------------- */ + const char *pszSampleDataType = CPLGetXMLValue( + psImageReferenceAttributes, "rasterAttributes.sampleType", ""); + poDS->SetMetadataItem("SAMPLE_TYPE", pszSampleDataType); + + /* Either Integer (16 bits) or Floating-Point (32 bits) */ + const char *pszDataType = CPLGetXMLValue(psImageReferenceAttributes, + "rasterAttributes.dataType", ""); + poDS->SetMetadataItem("DATA_TYPE", pszDataType); + + /* 2 entries for complex data 1 entry for magnitude detected data */ + const char *pszBitsPerSample = CPLGetXMLValue( + psImageReferenceAttributes, "rasterAttributes.bitsPerSample", ""); + const int nBitsPerSample = atoi(pszBitsPerSample); + poDS->SetMetadataItem("BITS_PER_SAMPLE", pszBitsPerSample); + + const char *pszSampledPixelSpacingTime = + CPLGetXMLValue(psImageReferenceAttributes, + "rasterAttributes.sampledPixelSpacingTime", "UNK"); + poDS->SetMetadataItem("SAMPLED_PIXEL_SPACING_TIME", + pszSampledPixelSpacingTime); + + const char *pszSampledLineSpacingTime = + CPLGetXMLValue(psImageReferenceAttributes, + "rasterAttributes.sampledLineSpacingTime", "UNK"); + poDS->SetMetadataItem("SAMPLED_LINE_SPACING_TIME", + pszSampledLineSpacingTime); + + GDALDataType eDataType; + + if (EQUAL(pszSampleDataType, "Mixed")) /* RCM MLC has Mixed sampleType */ + { + poDS->isComplexData = false; /* RCM MLC is detected, non-complex */ + if (nBitsPerSample == 32) + { + eDataType = GDT_Float32; /* 32 bits, check read block */ + poDS->magnitudeBits = 32; + } + else + { + eDataType = GDT_UInt16; /* 16 bits, check read block */ + poDS->magnitudeBits = 16; + } + } + else if (EQUAL(pszSampleDataType, "Complex")) + { + poDS->isComplexData = true; + /* Usually this is the same bits for both */ + poDS->realBitsComplexData = nBitsPerSample; + poDS->imaginaryBitsComplexData = nBitsPerSample; + + if (nBitsPerSample == 32) + { + eDataType = GDT_CFloat32; /* 32 bits, check read block */ + } + else + { + eDataType = GDT_CInt16; /* 16 bits, check read block */ + } + } + else if (nBitsPerSample == 32 && + EQUAL(pszSampleDataType, "Magnitude Detected")) + { + /* Actually, I don't need to test the 'dataType'=' Floating-Point', we + * know that a 32 bits per sample */ + eDataType = GDT_Float32; + poDS->isComplexData = false; + poDS->magnitudeBits = 32; + } + else if (nBitsPerSample == 16 && + EQUAL(pszSampleDataType, "Magnitude Detected")) + { + /* Actually, I don't need to test the 'dataType'=' Integer', we know + * that a 16 bits per sample */ + eDataType = GDT_UInt16; + poDS->isComplexData = false; + poDS->magnitudeBits = 16; + } + else + { + delete poDS; + CPLError(CE_Failure, CPLE_AppDefined, + "ERROR: dataType=%s and bitsPerSample=%d are not a supported " + "configuration.", + pszDataType, nBitsPerSample); + return nullptr; + } + + /* Indicates whether pixel number (i.e., range) increases or decreases with + range time. For GCD and GCC products, this applies to intermediate ground + range image data prior to geocoding. */ + const char *pszPixelTimeOrdering = + CPLGetXMLValue(psImageReferenceAttributes, + "rasterAttributes.pixelTimeOrdering", "UNK"); + poDS->SetMetadataItem("PIXEL_TIME_ORDERING", pszPixelTimeOrdering); + + /* Indicates whether line numbers (i.e., azimuth) increase or decrease with + azimuth time. For GCD and GCC products, this applies to intermediate ground + range image data prior to geocoding. */ + const char *pszLineTimeOrdering = CPLGetXMLValue( + psImageReferenceAttributes, "rasterAttributes.lineTimeOrdering", "UNK"); + poDS->SetMetadataItem("LINE_TIME_ORDERING", pszLineTimeOrdering); + + /* while we're at it, extract the pixel spacing information */ + const char *pszPixelSpacing = + CPLGetXMLValue(psImageReferenceAttributes, + "rasterAttributes.sampledPixelSpacing", "UNK"); + poDS->SetMetadataItem("PIXEL_SPACING", pszPixelSpacing); + + const char *pszLineSpacing = + CPLGetXMLValue(psImageReferenceAttributes, + "rasterAttributes.sampledLineSpacing", "UNK"); + poDS->SetMetadataItem("LINE_SPACING", pszLineSpacing); + + /* -------------------------------------------------------------------- */ + /* Open each of the data files as a complex band. */ + /* -------------------------------------------------------------------- */ + char *pszBeta0LUT = nullptr; + char *pszGammaLUT = nullptr; + char *pszSigma0LUT = nullptr; + // Same file for all calibrations except the calibration is plit inside the + // XML + std::string osNoiseLevelsValues; + + char *pszPath = CPLStrdup(CPLGetPath(osMDFilename)); + + /* Get a list of all polarizations */ + CPLXMLNode *psSourceAttrs = + CPLGetXMLNode(psProduct, "=product.sourceAttributes"); + if (psSourceAttrs == nullptr) + { + CPLFree(pszPath); + delete poDS; + CPLError( + CE_Failure, CPLE_OpenFailed, + "ERROR: RCM source attributes is missing. Please contact your data " + "provider for a corrected dataset."); + return nullptr; + } + + CPLXMLNode *psRadarParameters = + CPLGetXMLNode(psProduct, "=product.sourceAttributes.radarParameters"); + if (psRadarParameters == nullptr) + { + CPLFree(pszPath); + delete poDS; + CPLError( + CE_Failure, CPLE_OpenFailed, + "ERROR: RCM radar parameters is missing. Please contact your data " + "provider for a corrected dataset."); + return nullptr; + } + + const char *pszPolarizations = + CPLGetXMLValue(psRadarParameters, "polarizations", ""); + if (pszPolarizations == nullptr || strlen(pszPolarizations) == 0) + { + CPLFree(pszPath); + delete poDS; + CPLError( + CE_Failure, CPLE_OpenFailed, + "ERROR: RCM polarizations list is missing. Please contact your " + "data provider for a corrected dataset."); + return nullptr; + } + poDS->SetMetadataItem("POLARIZATIONS", pszPolarizations); + + const char *psAcquisitionType = + CPLGetXMLValue(psRadarParameters, "acquisitionType", "UNK"); + poDS->SetMetadataItem("ACQUISITION_TYPE", psAcquisitionType); + + const char *psBeams = CPLGetXMLValue(psRadarParameters, "beams", "UNK"); + poDS->SetMetadataItem("BEAMS", psBeams); + + char **papszPolarizationsGrids = + CSLTokenizeString2(pszPolarizations, " ", 0); + CPLStringList imageBandList; + CPLStringList imageBandFileList; + const int nPolarizationsGridCount = CSLCount(papszPolarizationsGrids); + + /* File names for full resolution IPDFs. For GeoTIFF format, one entry per + pole; For NITF 2.1 format, only one entry. */ + bool bIsNITF = false; + const char *pszNITF_Filename = nullptr; + int imageBandFileCount = 0; + int imageBandCount = 0; + + /* Split the polarization string*/ + auto iss = std::istringstream((CPLString(pszPolarizations)).c_str()); + auto pol = std::string{}; + /* Count number of polarizations*/ + while (iss >> pol) + imageBandCount++; + + CPLXMLNode *psNode = psImageAttributes->psChild; + for (; psNode != nullptr; psNode = psNode->psNext) + { + /* Find the tif or ntf filename */ + if (psNode->eType != CXT_Element || !(EQUAL(psNode->pszValue, "ipdf"))) + continue; + + /* -------------------------------------------------------------------- + */ + /* Fetch ipdf image. Could be either tif or ntf. */ + /* Replace / by \\ */ + /* -------------------------------------------------------------------- + */ + const char *pszBasedFilename = CPLGetXMLValue(psNode, "", ""); + if (*pszBasedFilename == '\0') + continue; + + /* Count number of image names within ipdf tag*/ + imageBandFileCount++; + + CPLString pszUpperBasedFilename(CPLString(pszBasedFilename).toupper()); + + const bool bEndsWithNTF = + strlen(pszUpperBasedFilename.c_str()) > 4 && + EQUAL(pszUpperBasedFilename.c_str() + + strlen(pszUpperBasedFilename.c_str()) - 4, + ".NTF"); + + if (bEndsWithNTF) + { + /* Found it! It would not exist one more */ + bIsNITF = true; + pszNITF_Filename = pszBasedFilename; + break; + } + else + { + /* Keep adding polarizations filename according to the pole */ + const char *pszPole = CPLGetXMLValue(psNode, "pole", ""); + if (*pszPole == '\0') + { + /* Guard against case when pole is a null string, skip it */ + imageBandFileCount--; + continue; + } + + if (EQUAL(pszPole, "XC")) + { + /* Skip RCM MLC's 3rd band file ##XC.tif */ + imageBandFileCount--; + continue; + } + + imageBandList.AddString(CPLString(pszPole).toupper()); + imageBandFileList.AddString(pszBasedFilename); + } + } + + /* -------------------------------------------------------------------- */ + /* Incidence Angle in a sub-folder */ + /* called 'calibration' from the 'metadata' folder */ + /* -------------------------------------------------------------------- */ + + const char *pszIncidenceAngleFileName = CPLGetXMLValue( + psImageReferenceAttributes, "incidenceAngleFileName", ""); + + if (pszIncidenceAngleFileName != nullptr) + { + CPLString osIncidenceAnglePath; + osIncidenceAnglePath.append(CALIBRATION_FOLDER); + osIncidenceAnglePath.append(szPathSeparator); + osIncidenceAnglePath.append(pszIncidenceAngleFileName); + + /* Check if the file exist */ + if (IsValidXMLFile(pszPath, osIncidenceAnglePath)) + { + CPLString osIncidenceAngleFilePath = + CPLFormFilename(pszPath, osIncidenceAnglePath, nullptr); + + CPLXMLNode *psIncidenceAngle = + CPLParseXMLFile(osIncidenceAngleFilePath); + + int pixelFirstLutValue = atoi( + CPLGetXMLValue(psIncidenceAngle, + "=incidenceAngles.pixelFirstAnglesValue", "0")); + + int stepSize = atoi(CPLGetXMLValue( + psIncidenceAngle, "=incidenceAngles.stepSize", "0")); + + int numberOfValues = atoi(CPLGetXMLValue( + psIncidenceAngle, "=incidenceAngles.numberOfValues", "0")); + + /* Get the Pixel Per range */ + int tableSize = abs(stepSize) * abs(numberOfValues); + + CPLString angles; + // Loop through all nodes with spaces + CPLXMLNode *psNextNode = + CPLGetXMLNode(psIncidenceAngle, "=incidenceAngles"); + + CPLXMLNode *psNodeInc; + for (psNodeInc = psNextNode->psChild; psNodeInc != nullptr; + psNodeInc = psNodeInc->psNext) + { + if (EQUAL(psNodeInc->pszValue, "angles")) + { + if (angles.length() > 0) + { + angles.append(" "); /* separator */ + } + const char *valAngle = CPLGetXMLValue(psNodeInc, "", ""); + angles.append(valAngle); + } + } + + char **papszAngleList = + CSLTokenizeString2(angles, " ", CSLT_HONOURSTRINGS); + + /* Allocate the right LUT size according to the product range pixel + */ + poDS->m_IncidenceAngleTableSize = tableSize; + poDS->m_nfIncidenceAngleTable = + InterpolateValues(papszAngleList, tableSize, stepSize, + numberOfValues, pixelFirstLutValue); + + CSLDestroy(papszAngleList); + } + } + + for (int iPoleInx = 0; iPoleInx < nPolarizationsGridCount; iPoleInx++) + { + // Search for a specific band name + const CPLString pszPole = + CPLString(papszPolarizationsGrids[iPoleInx]).toupper(); + + // Look if the NoiseLevel file xml exist for the + CPLXMLNode *psRefNode = psImageReferenceAttributes->psChild; + for (; psRefNode != nullptr; psRefNode = psRefNode->psNext) + { + if (EQUAL(psRefNode->pszValue, "noiseLevelFileName") && bCanCalib) + { + /* Determine which incidence angle correction this is */ + const char *pszPoleToMatch = + CPLGetXMLValue(psRefNode, "pole", ""); + const char *pszNoiseLevelFile = + CPLGetXMLValue(psRefNode, "", ""); + + if (*pszPoleToMatch == '\0') + continue; + + if (EQUAL(pszPoleToMatch, "XC")) + /* Skip noise for RCM MLC's 3rd band XC */ + continue; + + if (EQUAL(pszNoiseLevelFile, "")) + continue; + + /* -------------------------------------------------------------------- + */ + /* With RCM, LUT file is different per polarizarion (pole) + */ + /* The following code make sure to loop through all + * possible */ + /* 'noiseLevelFileName' and match the psChild; + for (; psRefNode != nullptr; psRefNode = psRefNode->psNext) + { + if (EQUAL(psRefNode->pszValue, "lookupTableFileName") && bCanCalib) + { + /* Determine which incidence angle correction this is */ + const char *pszLUTType = + CPLGetXMLValue(psRefNode, "sarCalibrationType", ""); + const char *pszPoleToMatch = + CPLGetXMLValue(psRefNode, "pole", ""); + const char *pszLUTFile = CPLGetXMLValue(psRefNode, "", ""); + + if (*pszPoleToMatch == '\0') + continue; + + if (EQUAL(pszPoleToMatch, "XC")) + /* Skip Calib for RCM MLC's 3rd band XC */ + continue; + + if (*pszLUTType == '\0') + continue; + + if (EQUAL(pszLUTType, "")) + continue; + + /* -------------------------------------------------------------------- + */ + /* With RCM, LUT file is different per polarizarion (pole) + */ + /* The following code make sure to loop through all + * possible */ + /* 'lookupTableFileName' and match the papszExtraFiles = + CSLAddString(poDS->papszExtraFiles, osLUTFilePath); + + CPLString pszBuf( + FormatCalibration(szBETA0, osMDFilename.c_str())); + pszBeta0LUT = VSIStrdup(osCalibPath); + + const char *oldLut = + poDS->GetMetadataItem("BETA_NOUGHT_LUT"); + if (oldLut == nullptr) + { + poDS->SetMetadataItem("BETA_NOUGHT_LUT", osCalibPath); + } + else + { + /* Keep adding LUT file for all bands, should be planty + * of space */ + char *ptrConcatLut = + static_cast(CPLMalloc(2048)); + ptrConcatLut[0] = + '\0'; /* Just initialize the first byte */ + strcat(ptrConcatLut, oldLut); + strcat(ptrConcatLut, ","); + strcat(ptrConcatLut, osCalibPath); + poDS->SetMetadataItem("BETA_NOUGHT_LUT", ptrConcatLut); + CPLFree(ptrConcatLut); + } + + poDS->papszSubDatasets = CSLSetNameValue( + poDS->papszSubDatasets, "SUBDATASET_3_NAME", pszBuf); + poDS->papszSubDatasets = CSLSetNameValue( + poDS->papszSubDatasets, "SUBDATASET_3_DESC", + "Beta Nought calibrated"); + } + else if (EQUAL(pszLUTType, "Sigma Nought") && + IsValidXMLFile(pszPath, osCalibPath)) + { + poDS->papszExtraFiles = + CSLAddString(poDS->papszExtraFiles, osLUTFilePath); + + CPLString pszBuf( + FormatCalibration(szSIGMA0, osMDFilename.c_str())); + pszSigma0LUT = VSIStrdup(osCalibPath); + + const char *oldLut = + poDS->GetMetadataItem("SIGMA_NOUGHT_LUT"); + if (oldLut == nullptr) + { + poDS->SetMetadataItem("SIGMA_NOUGHT_LUT", osCalibPath); + } + else + { + /* Keep adding LUT file for all bands, should be planty + * of space */ + char *ptrConcatLut = + static_cast(CPLMalloc(2048)); + ptrConcatLut[0] = + '\0'; /* Just initialize the first byte */ + strcat(ptrConcatLut, oldLut); + strcat(ptrConcatLut, ","); + strcat(ptrConcatLut, osCalibPath); + poDS->SetMetadataItem("SIGMA_NOUGHT_LUT", ptrConcatLut); + CPLFree(ptrConcatLut); + } + + poDS->papszSubDatasets = CSLSetNameValue( + poDS->papszSubDatasets, "SUBDATASET_2_NAME", pszBuf); + poDS->papszSubDatasets = CSLSetNameValue( + poDS->papszSubDatasets, "SUBDATASET_2_DESC", + "Sigma Nought calibrated"); + } + else if (EQUAL(pszLUTType, "Gamma") && + IsValidXMLFile(pszPath, osCalibPath)) + { + poDS->papszExtraFiles = + CSLAddString(poDS->papszExtraFiles, osLUTFilePath); + + CPLString pszBuf( + FormatCalibration(szGAMMA, osMDFilename.c_str())); + pszGammaLUT = VSIStrdup(osCalibPath); + + const char *oldLut = poDS->GetMetadataItem("GAMMA_LUT"); + if (oldLut == nullptr) + { + poDS->SetMetadataItem("GAMMA_LUT", osCalibPath); + } + else + { + /* Keep adding LUT file for all bands, should be planty + * of space */ + char *ptrConcatLut = + static_cast(CPLMalloc(2048)); + ptrConcatLut[0] = + '\0'; /* Just initialize the first byte */ + strcat(ptrConcatLut, oldLut); + strcat(ptrConcatLut, ","); + strcat(ptrConcatLut, osCalibPath); + poDS->SetMetadataItem("GAMMA_LUT", ptrConcatLut); + CPLFree(ptrConcatLut); + } + + poDS->papszSubDatasets = CSLSetNameValue( + poDS->papszSubDatasets, "SUBDATASET_4_NAME", pszBuf); + poDS->papszSubDatasets = CSLSetNameValue( + poDS->papszSubDatasets, "SUBDATASET_4_DESC", + "Gamma calibrated"); + } + } + } + + /* -------------------------------------------------------------------- + */ + /* Fetch ipdf image. Could be either tif or ntf. */ + /* Replace / by \\ */ + /* -------------------------------------------------------------------- + */ + const char *pszBasedFilename; + if (bIsNITF) + { + pszBasedFilename = pszNITF_Filename; + } + else + { + const int bandPositionIndex = imageBandList.FindString(pszPole); + if (bandPositionIndex < 0) + { + CPLFree(imageBandList); + CSLDestroy(papszPolarizationsGrids); + CPLFree(imageBandFileList); + CPLFree(pszPath); + delete poDS; + + CPLError(CE_Failure, CPLE_OpenFailed, + "ERROR: RCM cannot find the polarization %s. Please " + "contact your data provider for a corrected dataset", + pszPole.c_str()); + return nullptr; + } + + pszBasedFilename = imageBandFileList[bandPositionIndex]; + } + + int nLength = static_cast(strlen(pszBasedFilename)); + char *pszBasename = (char *)CPLCalloc(nLength + 1, sizeof(char)); + + /* -------------------------------------------------------------------- + */ + /* Change folder separator if given full path in wrong OS */ + /* -------------------------------------------------------------------- + */ + for (int i = 0; i < nLength; i++) + { + if (pszBasedFilename[i] == cOppositePathSeparator) + pszBasename[i] = cPathSeparator; + else + pszBasename[i] = pszBasedFilename[i]; + } + + /* -------------------------------------------------------------------- + */ + /* Form full filename (path of product.xml + basename). */ + /* -------------------------------------------------------------------- + */ + char *pszFullname = + CPLStrdup(CPLFormFilename(pszPath, pszBasename, nullptr)); + + CPLFree(pszBasename); + + /* -------------------------------------------------------------------- + */ + /* Try and open the file. */ + /* -------------------------------------------------------------------- + */ + GDALDataset *poBandFile = + reinterpret_cast(GDALOpen(pszFullname, GA_ReadOnly)); + if (poBandFile == nullptr) + { + CPLFree(pszFullname); + continue; + } + if (poBandFile->GetRasterCount() == 0) + { + GDALClose(reinterpret_cast(poBandFile)); + CPLFree(pszFullname); + continue; + } + + poDS->papszExtraFiles = + CSLAddString(poDS->papszExtraFiles, pszFullname); + + /* Some CFloat32 NITF files have nBitsPerSample incorrectly reported */ + /* as 16, and get misinterpreted as CInt16. Check the underlying NITF + */ + /* and override if this is the case. */ + if (poBandFile->GetRasterBand(1)->GetRasterDataType() == GDT_CFloat32) + eDataType = GDT_CFloat32; + + BandMappingRCM b = + checkBandFileMappingRCM(eDataType, poBandFile, bIsNITF); + if (b == BANDERROR) + { + GDALClose((GDALRasterBandH)poBandFile); + CPLFree(pszFullname); + delete poDS; + CPLError(CE_Failure, CPLE_AppDefined, + "The underlying band files do not have an appropriate " + "data type."); + return nullptr; + } + bool twoBandComplex = b == TWOBANDCOMPLEX; + bool isOneFilePerPol = (imageBandCount == imageBandFileCount); + + /* -------------------------------------------------------------------- + */ + /* Create the band. */ + /* -------------------------------------------------------------------- + */ + int bandNum = poDS->GetRasterCount() + 1; + if (eCalib == None || eCalib == Uncalib) + { + RCMRasterBand *poBand = + new RCMRasterBand(poDS, bandNum, eDataType, pszPole, poBandFile, + twoBandComplex, isOneFilePerPol, bIsNITF); + + poDS->SetBand(poDS->GetRasterCount() + 1, poBand); + } + else + { + const char *pszLUT = nullptr; + switch (eCalib) + { + case Sigma0: + pszLUT = pszSigma0LUT; + break; + case Beta0: + pszLUT = pszBeta0LUT; + break; + case Gamma: + pszLUT = pszGammaLUT; + break; + default: + /* we should bomb gracefully... */ + pszLUT = pszSigma0LUT; + } + + // The variable 'osNoiseLevelsValues' is always the same for a ban + // name except the XML contains different calibration name + if (poDS->isComplexData) + { + // If Complex, always 32 bits + RCMCalibRasterBand *poBand = new RCMCalibRasterBand( + poDS, pszPole, GDT_Float32, poBandFile, eCalib, + CPLFormFilename(pszPath, pszLUT, nullptr), + CPLFormFilename(pszPath, osNoiseLevelsValues.c_str(), + nullptr), + eDataType); + poDS->SetBand(poDS->GetRasterCount() + 1, poBand); + } + else + { + // Whatever the datatype was previoulsy set + RCMCalibRasterBand *poBand = new RCMCalibRasterBand( + poDS, pszPole, eDataType, poBandFile, eCalib, + CPLFormFilename(pszPath, pszLUT, nullptr), + CPLFormFilename(pszPath, osNoiseLevelsValues.c_str(), + nullptr), + eDataType); + poDS->SetBand(poDS->GetRasterCount() + 1, poBand); + } + } + + CPLFree(pszFullname); + } + + if (poDS->papszSubDatasets != nullptr && eCalib == None) + { + CPLString pszBuf = FormatCalibration(szUNCALIB, osMDFilename.c_str()); + poDS->papszSubDatasets = CSLSetNameValue(poDS->papszSubDatasets, + "SUBDATASET_1_NAME", pszBuf); + poDS->papszSubDatasets = + CSLSetNameValue(poDS->papszSubDatasets, "SUBDATASET_1_DESC", + "Uncalibrated digital numbers"); + } + else if (poDS->papszSubDatasets != nullptr) + { + CSLDestroy(poDS->papszSubDatasets); + poDS->papszSubDatasets = nullptr; + } + + /* -------------------------------------------------------------------- */ + /* Set the appropriate MATRIX_REPRESENTATION. */ + /* -------------------------------------------------------------------- */ + + if (poDS->GetRasterCount() == 4 && + (eDataType == GDT_CInt16 || eDataType == GDT_CFloat32)) + { + poDS->SetMetadataItem("MATRIX_REPRESENTATION", "SCATTERING"); + } + + /* -------------------------------------------------------------------- */ + /* Collect a few useful metadata items. */ + /* -------------------------------------------------------------------- */ + + pszItem = CPLGetXMLValue(psSourceAttrs, "satellite", ""); + poDS->SetMetadataItem("SATELLITE_IDENTIFIER", pszItem); + + pszItem = CPLGetXMLValue(psSourceAttrs, "sensor", ""); + poDS->SetMetadataItem("SENSOR_IDENTIFIER", pszItem); + + /* Get beam mode mnemonic */ + pszItem = CPLGetXMLValue(psSourceAttrs, "beamMode", "UNK"); + poDS->SetMetadataItem("BEAM_MODE", pszItem); + + pszItem = CPLGetXMLValue(psSourceAttrs, "beamModeMnemonic", "UNK"); + poDS->SetMetadataItem("BEAM_MODE_MNEMONIC", pszItem); + + pszItem = CPLGetXMLValue(psSourceAttrs, "beamModeDefinitionId", "UNK"); + poDS->SetMetadataItem("BEAM_MODE_DEFINITION_ID", pszItem); + + pszItem = CPLGetXMLValue(psSourceAttrs, "rawDataStartTime", "UNK"); + poDS->SetMetadataItem("ACQUISITION_START_TIME", pszItem); + + pszItem = CPLGetXMLValue(psSourceAttrs, "inputDatasetFacilityId", "UNK"); + poDS->SetMetadataItem("FACILITY_IDENTIFIER", pszItem); + + pszItem = CPLGetXMLValue(psSourceAttrs, + "orbitAndAttitude.orbitInformation.passDirection", + "UNK"); + poDS->SetMetadataItem("ORBIT_DIRECTION", pszItem); + pszItem = CPLGetXMLValue( + psSourceAttrs, "orbitAndAttitude.orbitInformation.orbitDataSource", + "UNK"); + poDS->SetMetadataItem("ORBIT_DATA_SOURCE", pszItem); + pszItem = CPLGetXMLValue( + psSourceAttrs, "orbitAndAttitude.orbitInformation.orbitDataFileName", + "UNK"); + poDS->SetMetadataItem("ORBIT_DATA_FILE", pszItem); + + /* Get incidence angle information. DONE */ + pszItem = CPLGetXMLValue(psSceneAttributes, "imageAttributes.incAngNearRng", + "UNK"); + poDS->SetMetadataItem("NEAR_RANGE_INCIDENCE_ANGLE", pszItem); + + pszItem = CPLGetXMLValue(psSceneAttributes, "imageAttributes.incAngFarRng", + "UNK"); + poDS->SetMetadataItem("FAR_RANGE_INCIDENCE_ANGLE", pszItem); + + pszItem = CPLGetXMLValue(psSceneAttributes, + "imageAttributes.slantRangeNearEdge", "UNK"); + poDS->SetMetadataItem("SLANT_RANGE_NEAR_EDGE", pszItem); + + pszItem = CPLGetXMLValue(psSceneAttributes, + "imageAttributes.slantRangeFarEdge", "UNK"); + poDS->SetMetadataItem("SLANT_RANGE_FAR_EDGE", pszItem); + + /*--------------------------------------------------------------------- */ + /* Collect Map projection/Geotransform information, if present.DONE */ + /* In RCM, there is no file that indicates */ + /* -------------------------------------------------------------------- */ + CPLXMLNode *psMapProjection = CPLGetXMLNode( + psImageReferenceAttributes, "geographicInformation.mapProjection"); + + if (psMapProjection != nullptr) + { + CPLXMLNode *psPos = + CPLGetXMLNode(psMapProjection, "positioningInformation"); + + pszItem = + CPLGetXMLValue(psMapProjection, "mapProjectionDescriptor", "UNK"); + poDS->SetMetadataItem("MAP_PROJECTION_DESCRIPTOR", pszItem); + + pszItem = + CPLGetXMLValue(psMapProjection, "mapProjectionOrientation", "UNK"); + poDS->SetMetadataItem("MAP_PROJECTION_ORIENTATION", pszItem); + + pszItem = CPLGetXMLValue(psMapProjection, "resamplingKernel", "UNK"); + poDS->SetMetadataItem("RESAMPLING_KERNEL", pszItem); + + pszItem = CPLGetXMLValue(psMapProjection, "satelliteHeading", "UNK"); + poDS->SetMetadataItem("SATELLITE_HEADING", pszItem); + + if (psPos != nullptr) + { + const double tl_x = CPLStrtod( + CPLGetXMLValue(psPos, "upperLeftCorner.mapCoordinate.easting", + "0.0"), + nullptr); + const double tl_y = CPLStrtod( + CPLGetXMLValue(psPos, "upperLeftCorner.mapCoordinate.northing", + "0.0"), + nullptr); + const double bl_x = CPLStrtod( + CPLGetXMLValue(psPos, "lowerLeftCorner.mapCoordinate.easting", + "0.0"), + nullptr); + const double bl_y = CPLStrtod( + CPLGetXMLValue(psPos, "lowerLeftCorner.mapCoordinate.northing", + "0.0"), + nullptr); + const double tr_x = CPLStrtod( + CPLGetXMLValue(psPos, "upperRightCorner.mapCoordinate.easting", + "0.0"), + nullptr); + const double tr_y = CPLStrtod( + CPLGetXMLValue(psPos, "upperRightCorner.mapCoordinate.northing", + "0.0"), + nullptr); + poDS->adfGeoTransform[1] = (tr_x - tl_x) / (poDS->nRasterXSize - 1); + poDS->adfGeoTransform[4] = (tr_y - tl_y) / (poDS->nRasterXSize - 1); + poDS->adfGeoTransform[2] = (bl_x - tl_x) / (poDS->nRasterYSize - 1); + poDS->adfGeoTransform[5] = (bl_y - tl_y) / (poDS->nRasterYSize - 1); + poDS->adfGeoTransform[0] = (tl_x - 0.5 * poDS->adfGeoTransform[1] - + 0.5 * poDS->adfGeoTransform[2]); + poDS->adfGeoTransform[3] = (tl_y - 0.5 * poDS->adfGeoTransform[4] - + 0.5 * poDS->adfGeoTransform[5]); + + /* Use bottom right pixel to test geotransform */ + const double br_x = CPLStrtod( + CPLGetXMLValue(psPos, "lowerRightCorner.mapCoordinate.easting", + "0.0"), + nullptr); + const double br_y = CPLStrtod( + CPLGetXMLValue(psPos, "lowerRightCorner.mapCoordinate.northing", + "0.0"), + nullptr); + const double testx = + poDS->adfGeoTransform[0] + + poDS->adfGeoTransform[1] * (poDS->nRasterXSize - 0.5) + + poDS->adfGeoTransform[2] * (poDS->nRasterYSize - 0.5); + const double testy = + poDS->adfGeoTransform[3] + + poDS->adfGeoTransform[4] * (poDS->nRasterXSize - 0.5) + + poDS->adfGeoTransform[5] * (poDS->nRasterYSize - 0.5); + + /* Give 1/4 pixel numerical error leeway in calculating location + based on affine transform */ + if ((fabs(testx - br_x) > + fabs(0.25 * + (poDS->adfGeoTransform[1] + poDS->adfGeoTransform[2]))) || + (fabs(testy - br_y) > fabs(0.25 * (poDS->adfGeoTransform[4] + + poDS->adfGeoTransform[5])))) + { + CPLError(CE_Warning, CPLE_AppDefined, + "WARNING: Unexpected error in calculating affine " + "transform: corner coordinates inconsistent."); + } + else + { + poDS->bHaveGeoTransform = TRUE; + } + } + } + + /* -------------------------------------------------------------------- */ + /* Collect Projection String Information.DONE */ + /* -------------------------------------------------------------------- */ + CPLXMLNode *psEllipsoid = + CPLGetXMLNode(psImageReferenceAttributes, + "geographicInformation.ellipsoidParameters"); + + if (psEllipsoid != nullptr) + { + OGRSpatialReference oLL, oPrj; + + const char *pszGeodeticTerrainHeight = + CPLGetXMLValue(psEllipsoid, "geodeticTerrainHeight", "UNK"); + poDS->SetMetadataItem("GEODETIC_TERRAIN_HEIGHT", + pszGeodeticTerrainHeight); + + const char *pszEllipsoidName = + CPLGetXMLValue(psEllipsoid, "ellipsoidName", ""); + double minor_axis = + CPLAtof(CPLGetXMLValue(psEllipsoid, "semiMinorAxis", "0.0")); + double major_axis = + CPLAtof(CPLGetXMLValue(psEllipsoid, "semiMajorAxis", "0.0")); + + if (EQUAL(pszEllipsoidName, "") || (minor_axis == 0.0) || + (major_axis == 0.0)) + { + oLL.SetWellKnownGeogCS("WGS84"); + oPrj.SetWellKnownGeogCS("WGS84"); + + CPLError(CE_Warning, CPLE_AppDefined, + "WARNING: Incomplete ellipsoid " + "information. Using wgs-84 parameters."); + } + else if (EQUAL(pszEllipsoidName, "WGS84") || + EQUAL(pszEllipsoidName, "WGS 1984")) + { + oLL.SetWellKnownGeogCS("WGS84"); + oPrj.SetWellKnownGeogCS("WGS84"); + } + else + { + const double inv_flattening = + major_axis / (major_axis - minor_axis); + oLL.SetGeogCS("", "", pszEllipsoidName, major_axis, inv_flattening); + oPrj.SetGeogCS("", "", pszEllipsoidName, major_axis, + inv_flattening); + } + + if (psMapProjection != nullptr) + { + const char *pszProj = + CPLGetXMLValue(psMapProjection, "mapProjectionDescriptor", ""); + bool bUseProjInfo = false; + + CPLXMLNode *psUtmParams = + CPLGetXMLNode(psMapProjection, "utmProjectionParameters"); + + CPLXMLNode *psNspParams = + CPLGetXMLNode(psMapProjection, "nspProjectionParameters"); + + if ((psUtmParams != nullptr) && poDS->bHaveGeoTransform) + { + /* double origEasting, origNorthing; */ + bool bNorth = true; + + const int utmZone = + atoi(CPLGetXMLValue(psUtmParams, "utmZone", "")); + const char *pszHemisphere = + CPLGetXMLValue(psUtmParams, "hemisphere", ""); +#if 0 + origEasting = CPLStrtod(CPLGetXMLValue( + psUtmParams, "mapOriginFalseEasting", "0.0"), nullptr); + origNorthing = CPLStrtod(CPLGetXMLValue( + psUtmParams, "mapOriginFalseNorthing", "0.0"), nullptr); +#endif + if (STARTS_WITH_CI(pszHemisphere, "southern")) + bNorth = FALSE; + + if (STARTS_WITH_CI(pszProj, "UTM")) + { + oPrj.SetUTM(utmZone, bNorth); + bUseProjInfo = true; + } + } + else if ((psNspParams != nullptr) && poDS->bHaveGeoTransform) + { + const double origEasting = CPLStrtod( + CPLGetXMLValue(psNspParams, "mapOriginFalseEasting", "0.0"), + nullptr); + const double origNorthing = + CPLStrtod(CPLGetXMLValue(psNspParams, + "mapOriginFalseNorthing", "0.0"), + nullptr); + const double copLong = CPLStrtod( + CPLGetXMLValue(psNspParams, "centerOfProjectionLongitude", + "0.0"), + nullptr); + const double copLat = CPLStrtod( + CPLGetXMLValue(psNspParams, "centerOfProjectionLatitude", + "0.0"), + nullptr); + const double sP1 = CPLStrtod( + CPLGetXMLValue(psNspParams, "standardParallels1", "0.0"), + nullptr); + const double sP2 = CPLStrtod( + CPLGetXMLValue(psNspParams, "standardParallels2", "0.0"), + nullptr); + + if (STARTS_WITH_CI(pszProj, "ARC")) + { + /* Albers Conical Equal Area */ + oPrj.SetACEA(sP1, sP2, copLat, copLong, origEasting, + origNorthing); + bUseProjInfo = true; + } + else if (STARTS_WITH_CI(pszProj, "LCC")) + { + /* Lambert Conformal Conic */ + oPrj.SetLCC(sP1, sP2, copLat, copLong, origEasting, + origNorthing); + bUseProjInfo = true; + } + else if (STARTS_WITH_CI(pszProj, "STPL")) + { + /* State Plate + ASSUMPTIONS: "zone" in product.xml matches USGS + definition as expected by ogr spatial reference; NAD83 + zones (versus NAD27) are assumed. */ + + const int nSPZone = + atoi(CPLGetXMLValue(psNspParams, "zone", "1")); + + oPrj.SetStatePlane(nSPZone, TRUE, nullptr, 0.0); + bUseProjInfo = true; + } + } + + if (bUseProjInfo) + { + poDS->m_oSRS = oPrj; + } + else + { + CPLError(CE_Warning, CPLE_AppDefined, + "WARNING: Unable to interpret projection information; " + "check mapProjection info in product.xml!"); + } + } + + poDS->m_oGCPSRS = oLL; + } + + /* -------------------------------------------------------------------- */ + /* Collect GCPs.DONE */ + /* -------------------------------------------------------------------- */ + CPLXMLNode *psGeoGrid = CPLGetXMLNode( + psImageReferenceAttributes, "geographicInformation.geolocationGrid"); + + if (psGeoGrid != nullptr) + { + /* count GCPs */ + poDS->nGCPCount = 0; + + for (psNode = psGeoGrid->psChild; psNode != nullptr; + psNode = psNode->psNext) + { + if (EQUAL(psNode->pszValue, "imageTiePoint")) + poDS->nGCPCount++; + } + + poDS->pasGCPList = reinterpret_cast( + CPLCalloc(sizeof(GDAL_GCP), poDS->nGCPCount)); + + poDS->nGCPCount = 0; + + for (psNode = psGeoGrid->psChild; psNode != nullptr; + psNode = psNode->psNext) + { + GDAL_GCP *psGCP = poDS->pasGCPList + poDS->nGCPCount; + + if (!EQUAL(psNode->pszValue, "imageTiePoint")) + continue; + + poDS->nGCPCount++; + + char szID[32]; + snprintf(szID, sizeof(szID), "%d", poDS->nGCPCount); + psGCP->pszId = CPLStrdup(szID); + psGCP->pszInfo = CPLStrdup(""); + psGCP->dfGCPPixel = + CPLAtof(CPLGetXMLValue(psNode, "imageCoordinate.pixel", "0")); + psGCP->dfGCPLine = + CPLAtof(CPLGetXMLValue(psNode, "imageCoordinate.line", "0")); + psGCP->dfGCPX = CPLAtof( + CPLGetXMLValue(psNode, "geodeticCoordinate.longitude", "")); + psGCP->dfGCPY = CPLAtof( + CPLGetXMLValue(psNode, "geodeticCoordinate.latitude", "")); + psGCP->dfGCPZ = CPLAtof( + CPLGetXMLValue(psNode, "geodeticCoordinate.height", "")); + } + } + + CPLFree(pszPath); + if (pszBeta0LUT) + CPLFree(pszBeta0LUT); + if (pszSigma0LUT) + CPLFree(pszSigma0LUT); + if (pszGammaLUT) + CPLFree(pszGammaLUT); + + /* -------------------------------------------------------------------- */ + /* Collect RPC.DONE */ + /* -------------------------------------------------------------------- */ + CPLXMLNode *psRationalFunctions = CPLGetXMLNode( + psImageReferenceAttributes, "geographicInformation.rationalFunctions"); + if (psRationalFunctions != nullptr) + { + char **papszRPC = nullptr; + static const char *const apszXMLToGDALMapping[] = { + "biasError", + "ERR_BIAS", + "randomError", + "ERR_RAND", + //"lineFitQuality", "????", + //"pixelFitQuality", "????", + "lineOffset", + "LINE_OFF", + "pixelOffset", + "SAMP_OFF", + "latitudeOffset", + "LAT_OFF", + "longitudeOffset", + "LONG_OFF", + "heightOffset", + "HEIGHT_OFF", + "lineScale", + "LINE_SCALE", + "pixelScale", + "SAMP_SCALE", + "latitudeScale", + "LAT_SCALE", + "longitudeScale", + "LONG_SCALE", + "heightScale", + "HEIGHT_SCALE", + "lineNumeratorCoefficients", + "LINE_NUM_COEFF", + "lineDenominatorCoefficients", + "LINE_DEN_COEFF", + "pixelNumeratorCoefficients", + "SAMP_NUM_COEFF", + "pixelDenominatorCoefficients", + "SAMP_DEN_COEFF", + }; + for (size_t i = 0; i < CPL_ARRAYSIZE(apszXMLToGDALMapping); i += 2) + { + const char *pszValue = CPLGetXMLValue( + psRationalFunctions, apszXMLToGDALMapping[i], nullptr); + if (pszValue) + papszRPC = CSLSetNameValue( + papszRPC, apszXMLToGDALMapping[i + 1], pszValue); + } + poDS->GDALDataset::SetMetadata(papszRPC, "RPC"); + CSLDestroy(papszRPC); + } + + /* -------------------------------------------------------------------- */ + /* Initialize any PAM information. */ + /* -------------------------------------------------------------------- */ + CPLString osDescription; + CPLString osSubdatasetName; + bool useSubdatasets = true; + + switch (eCalib) + { + case Sigma0: + { + osSubdatasetName = szSIGMA0; + CPLString pszDescriptionSigma = + FormatCalibration(szSIGMA0, osMDFilename.c_str()); + osDescription = pszDescriptionSigma; + } + break; + case Beta0: + { + osSubdatasetName = szBETA0; + CPLString pszDescriptionBeta = + FormatCalibration(szBETA0, osMDFilename.c_str()); + osDescription = pszDescriptionBeta; + } + break; + case Gamma: + { + osSubdatasetName = szGAMMA; + CPLString pszDescriptionGamma = + FormatCalibration(szGAMMA, osMDFilename.c_str()); + osDescription = pszDescriptionGamma; + } + break; + case Uncalib: + { + osSubdatasetName = szUNCALIB; + CPLString pszDescriptionUncalib = + FormatCalibration(szUNCALIB, osMDFilename.c_str()); + osDescription = pszDescriptionUncalib; + } + break; + default: + osSubdatasetName = szUNCALIB; + osDescription = osMDFilename; + useSubdatasets = false; + } + + CPL_IGNORE_RET_VAL(osSubdatasetName); + + if (eCalib != None) + poDS->papszExtraFiles = + CSLAddString(poDS->papszExtraFiles, osMDFilename); + + /* -------------------------------------------------------------------- */ + /* Initialize any PAM information. */ + /* -------------------------------------------------------------------- */ + poDS->SetDescription(osDescription); + + poDS->SetPhysicalFilename(osMDFilename); + poDS->SetSubdatasetName(osDescription); + + poDS->TryLoadXML(); + + /* -------------------------------------------------------------------- */ + /* Check for overviews. */ + /* -------------------------------------------------------------------- */ + if (useSubdatasets) + poDS->oOvManager.Initialize(poDS, ":::VIRTUAL:::"); + else + poDS->oOvManager.Initialize(poDS, osMDFilename); + + return poDS; +} + +/************************************************************************/ +/* GetGCPCount() */ +/************************************************************************/ + +int RCMDataset::GetGCPCount() + +{ + return nGCPCount; +} + +/************************************************************************/ +/* GetGCPSpatialRef() */ +/************************************************************************/ + +const OGRSpatialReference *RCMDataset::GetGCPSpatialRef() const + +{ + return m_oGCPSRS.IsEmpty() || nGCPCount == 0 ? nullptr : &m_oGCPSRS; +} + +/************************************************************************/ +/* GetGCPs() */ +/************************************************************************/ + +const GDAL_GCP *RCMDataset::GetGCPs() + +{ + return pasGCPList; +} + +/************************************************************************/ +/* GetSpatialRef() */ +/************************************************************************/ + +const OGRSpatialReference *RCMDataset::GetSpatialRef() const + +{ + return m_oSRS.IsEmpty() ? nullptr : &m_oSRS; +} + +/************************************************************************/ +/* GetGeoTransform() */ +/************************************************************************/ + +CPLErr RCMDataset::GetGeoTransform(double *padfTransform) + +{ + memcpy(padfTransform, adfGeoTransform, sizeof(double) * 6); + + if (bHaveGeoTransform) + { + return CE_None; + } + + return CE_Failure; +} + +/************************************************************************/ +/* GetMetadataDomainList() */ +/************************************************************************/ + +char **RCMDataset::GetMetadataDomainList() +{ + return BuildMetadataDomainList(GDALDataset::GetMetadataDomainList(), TRUE, + "SUBDATASETS", nullptr); +} + +/************************************************************************/ +/* GetMetadata() */ +/************************************************************************/ + +char **RCMDataset::GetMetadata(const char *pszDomain) + +{ + if (pszDomain != nullptr && STARTS_WITH_CI(pszDomain, "SUBDATASETS") && + papszSubDatasets != nullptr) + return papszSubDatasets; + + return GDALDataset::GetMetadata(pszDomain); +} + +/************************************************************************/ +/* GDALRegister_RCM() */ +/************************************************************************/ + +void GDALRegister_RCM() + +{ + if (GDALGetDriverByName("RCM") != nullptr) + { + return; + } + + GDALDriver *poDriver = new GDALDriver(); + RCMDriverSetCommonMetadata(poDriver); + + poDriver->pfnOpen = RCMDataset::Open; + + GetGDALDriverManager()->RegisterDriver(poDriver); +} diff --git a/frmts/rcm/rcmdataset.h b/frmts/rcm/rcmdataset.h new file mode 100644 index 000000000000..5d5369a6bb84 --- /dev/null +++ b/frmts/rcm/rcmdataset.h @@ -0,0 +1,261 @@ +/****************************************************************************** + * + * Project: DRDC Ottawa GEOINT + * Purpose: Radarsat Constellation Mission - XML Products (product.xml) driver + * Author: Roberto Caron, MDA + * on behalf of DRDC Ottawa + * + ****************************************************************************** + * Copyright (c) 2020, DRDC Ottawa + * + * Based on the RS2 Dataset Class + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#ifndef GDAL_RCM_H_INCLUDED +#define GDAL_RCM_H_INCLUDED + +#include "gdal_pam.h" + +typedef enum eCalibration_t +{ + Sigma0 = 0, + Gamma, + Beta0, + Uncalib, + None +} eCalibration; + +/************************************************************************/ +/* ==================================================================== */ +/* RCMDataset */ +/* ==================================================================== */ +/************************************************************************/ + +class RCMDataset final : public GDALPamDataset +{ + CPLXMLNode *psProduct = nullptr; + + int nGCPCount = 0; + GDAL_GCP *pasGCPList = nullptr; + OGRSpatialReference m_oSRS{}; + OGRSpatialReference m_oGCPSRS{}; + char **papszSubDatasets = nullptr; + char *pszLutApplied = nullptr; + double adfGeoTransform[6]; + bool bHaveGeoTransform = false; + bool bPerPolarizationScaling = false; + bool isComplexData = false; + int magnitudeBits = 16; + int realBitsComplexData = 32; + int imaginaryBitsComplexData = 32; + char **papszExtraFiles = nullptr; + double *m_nfIncidenceAngleTable = nullptr; + int m_IncidenceAngleTableSize = 0; + + protected: + virtual int CloseDependentDatasets() override; + + public: + RCMDataset(); + virtual ~RCMDataset(); + + virtual int GetGCPCount() override; + + const OGRSpatialReference *GetGCPSpatialRef() const override; + virtual const GDAL_GCP *GetGCPs() override; + + const OGRSpatialReference *GetSpatialRef() const override; + virtual CPLErr GetGeoTransform(double *) override; + + virtual char **GetMetadataDomainList() override; + virtual char **GetMetadata(const char *pszDomain = "") override; + virtual char **GetFileList(void) override; + + static GDALDataset *Open(GDALOpenInfo *); + + CPLXMLNode *GetProduct() + { + return psProduct; + } + + /* If False, this is Magnitude, True, Complex data with Real and + * Imaginary*/ + bool IsComplexData() + { + return isComplexData; + } + + /* These 2 variables are used in case of Complex Data */ + int GetRealBitsComplexData() + { + return realBitsComplexData; + } + + int GetImaginaryBitsComplexData() + { + return imaginaryBitsComplexData; + } + + /* This variable is used in case of Magnitude */ + int GetMagnitudeBits() + { + return magnitudeBits; + } + + /* This variable is used to hold the Incidence Angle */ + double *GetIncidenceAngle() + { + return m_nfIncidenceAngleTable; + } + + /* This variable is used to hold the Incidence Angle Table Size */ + int GetIncidenceAngleSize() + { + return m_IncidenceAngleTableSize; + } +}; + +/************************************************************************/ +/* ==================================================================== */ +/* RCMRasterBand */ +/* ==================================================================== */ +/************************************************************************/ + +class RCMRasterBand final : public GDALPamRasterBand +{ + private: + eCalibration m_eCalib = eCalibration::Uncalib; + GDALDataset *poBandFile = nullptr; + RCMDataset *poRCMDataset = nullptr; + GDALDataset *m_poBandDataset = nullptr; + GDALDataType m_eType = GDT_Unknown; /* data type of data being ingested */ + + double *m_nfTable = nullptr; + int m_nTableSize = 0; + double m_nfOffset = 0; + char *m_pszLUTFile = nullptr; + int pixelFirstLutValue = 0; + int stepSize = 0; + int numberOfValues = 0; + GDALRasterBand *poBand = nullptr; + + // 2 bands representing I+Q -> one complex band + // otherwise poBandFile is passed straight through + bool twoBandComplex = false; + + bool isOneFilePerPol = false; + bool isNITF = false; + + public: + RCMRasterBand(RCMDataset *poDSIn, int nBandIn, GDALDataType eDataTypeIn, + const char *pszPole, GDALDataset *poBandFile, + bool bTwoBandComplex, bool isOneFilePerPol, bool isNITF); + + virtual ~RCMRasterBand(); + + virtual CPLErr IReadBlock(int, int, void *) override; + + static bool IsExistLUT(); + + static double GetLUT(int pixel); + + static const char *GetLUTFilename(); + + static int GetLUTsize(); + + static double GetLUTOffset(); + + static void SetPartialLUT(int pixel_offset, int pixel_width); + + bool IsComplex(); + + static bool IsExistNoiseLevels(); + + static double GetNoiseLevels(int pixel); + + static const char *GetNoiseLevelsFilename(); + + static int GetNoiseLevelsSize(); + + eCalibration GetCalibration(); + + static GDALDataset *Open(GDALOpenInfo *); +}; + +/************************************************************************/ +/* ==================================================================== */ +/* RCMCalibRasterBand */ +/* ==================================================================== */ +/************************************************************************/ +/* Returns data that has been calibrated to sigma nought, gamma */ +/* or beta nought. */ +/************************************************************************/ + +class RCMCalibRasterBand final : public GDALPamRasterBand +{ + private: + eCalibration m_eCalib = eCalibration::Uncalib; + GDALDataset *m_poBandDataset = nullptr; + GDALDataType m_eType = GDT_Unknown; /* data type of data being ingested */ + /* data type that used to be before transformation */ + GDALDataType m_eOriginalType = GDT_Unknown; + + double *m_nfTable = nullptr; + int m_nTableSize = 0; + double m_nfOffset = 0; + char *m_pszLUTFile = nullptr; + int pixelFirstLutValue = 0; + int stepSize = 0; + int numberOfValues = 0; + + char *m_pszNoiseLevelsFile = nullptr; + double *m_nfTableNoiseLevels = nullptr; + int pixelFirstLutValueNoiseLevels = 0; + int stepSizeNoiseLevels = 0; + int numberOfValuesNoiseLevels = 0; + int m_nTableNoiseLevelsSize = 0; + + void ReadLUT(); + void ReadNoiseLevels(); + + public: + RCMCalibRasterBand(RCMDataset *poDataset, const char *pszPolarization, + GDALDataType eType, GDALDataset *poBandDataset, + eCalibration eCalib, const char *pszLUT, + const char *pszNoiseLevels, GDALDataType eOriginalType); + ~RCMCalibRasterBand(); + + CPLErr IReadBlock(int nBlockXOff, int nBlockYOff, void *pImage) override; + + bool IsExistLUT(); + + double GetLUT(int pixel); + + const char *GetLUTFilename(); + + int GetLUTsize(); + + double GetLUTOffset(); + + void SetPartialLUT(int pixel_offset, int pixel_width); + + bool IsExistNoiseLevels(); + + double GetNoiseLevels(int pixel); + + const char *GetNoiseLevelsFilename(); + + int GetNoiseLevelsSize(); + + bool IsComplex(); + + eCalibration GetCalibration(); + + double *CloneLUT(); + + double *CloneNoiseLevels(); +}; + +#endif /* ndef GDAL_RCM_H_INCLUDED */ diff --git a/frmts/rcm/rcmdrivercore.cpp b/frmts/rcm/rcmdrivercore.cpp new file mode 100644 index 000000000000..e6b7362744a7 --- /dev/null +++ b/frmts/rcm/rcmdrivercore.cpp @@ -0,0 +1,159 @@ +/****************************************************************************** + * + * Project: DRDC Ottawa GEOINT + * Purpose: Radarsat Constellation Mission - XML Products (product.xml) driver + * Author: Roberto Caron, MDA + * on behalf of DRDC Ottawa + * + ****************************************************************************** + * Copyright (c) 2020, DRDC Ottawa + * + * Based on the RS2 Dataset Class + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#include "rcmdrivercore.h" + +int RCMDatasetIdentify(GDALOpenInfo *poOpenInfo) +{ + /* Check for the case where we're trying to read the calibrated data: */ + CPLString calibrationFormat = FormatCalibration(nullptr, nullptr); + + if (STARTS_WITH_CI(poOpenInfo->pszFilename, calibrationFormat)) + { + return TRUE; + } + + if (poOpenInfo->bIsDirectory) + { + /* Check for directory access when there is a product.xml file in the + directory. */ + CPLString osMDFilename = + CPLFormCIFilename(poOpenInfo->pszFilename, "product.xml", nullptr); + + VSIStatBufL sStat; + if (VSIStatL(osMDFilename, &sStat) == 0) + { + CPLXMLNode *psProduct = CPLParseXMLFile(osMDFilename); + if (psProduct == nullptr) + return FALSE; + + CPLXMLNode *psProductAttributes = + CPLGetXMLNode(psProduct, "=product"); + if (psProductAttributes == nullptr) + { + CPLDestroyXMLNode(psProduct); + return FALSE; + } + + /* Check the namespace only, should be rs2 */ + const char *szNamespace = + CPLGetXMLValue(psProductAttributes, "xmlns", ""); + + if (strstr(szNamespace, "rcm") == nullptr) + { + /* Invalid namespace */ + CPLDestroyXMLNode(psProduct); + return FALSE; + } + + CPLDestroyXMLNode(psProduct); + return TRUE; + } + + /* If not, check for directory extra 'metadata' access when there is a + product.xml file in the directory. */ + + CPLString osMDFilenameMetadata = CPLFormCIFilename( + poOpenInfo->pszFilename, GetMetadataProduct(), nullptr); + + VSIStatBufL sStatMetadata; + if (VSIStatL(osMDFilenameMetadata, &sStatMetadata) == 0) + { + CPLXMLNode *psProduct = CPLParseXMLFile(osMDFilenameMetadata); + if (psProduct == nullptr) + return FALSE; + + CPLXMLNode *psProductAttributes = + CPLGetXMLNode(psProduct, "=product"); + if (psProductAttributes == nullptr) + { + CPLDestroyXMLNode(psProduct); + return FALSE; + } + + /* Check the namespace only, should be rs2 */ + const char *szNamespace = + CPLGetXMLValue(psProductAttributes, "xmlns", ""); + + if (strstr(szNamespace, "rcm") == nullptr) + { + /* Invalid namespace */ + CPLDestroyXMLNode(psProduct); + return FALSE; + } + + CPLDestroyXMLNode(psProduct); + + return TRUE; + } + + return FALSE; + } + + /* otherwise, do our normal stuff */ + if (strlen(poOpenInfo->pszFilename) < 11 || + !EQUAL(poOpenInfo->pszFilename + strlen(poOpenInfo->pszFilename) - 11, + "product.xml")) + return FALSE; + + if (poOpenInfo->nHeaderBytes < 100) + return FALSE; + + /* The RCM schema location is rcm_prod_product.xsd */ + if (strstr((const char *)poOpenInfo->pabyHeader, "/rcm") == nullptr || + strstr((const char *)poOpenInfo->pabyHeader, "SetDescription(RCM_DRIVER_NAME); + poDriver->SetMetadataItem(GDAL_DCAP_RASTER, "YES"); + poDriver->SetMetadataItem(GDAL_DMD_LONGNAME, + "Radarsat Constellation Mission XML Product"); + poDriver->SetMetadataItem(GDAL_DMD_HELPTOPIC, "drivers/raster/rcm.html"); + poDriver->SetMetadataItem(GDAL_DMD_SUBDATASETS, "YES"); + poDriver->pfnIdentify = RCMDatasetIdentify; + poDriver->SetMetadataItem(GDAL_DCAP_OPEN, "YES"); + poDriver->SetMetadataItem(GDAL_DCAP_VIRTUALIO, "YES"); +} + +/************************************************************************/ +/* DeclareDeferredRCMPlugin() */ +/************************************************************************/ + +#ifdef PLUGIN_FILENAME +void DeclareDeferredRCMPlugin() +{ + if (GDALGetDriverByName(RCM_DRIVER_NAME) != nullptr) + { + return; + } + + auto poDriver = new GDALPluginDriverProxy(PLUGIN_FILENAME); +#ifdef PLUGIN_INSTALLATION_MESSAGE + poDriver->SetMetadataItem(GDAL_DMD_PLUGIN_INSTALLATION_MESSAGE, + PLUGIN_INSTALLATION_MESSAGE); +#endif + RCMDriverSetCommonMetadata(poDriver); + GetGDALDriverManager()->DeclareDeferredPluginDriver(poDriver); +} +#endif diff --git a/frmts/rcm/rcmdrivercore.h b/frmts/rcm/rcmdrivercore.h new file mode 100644 index 000000000000..fbb6b45ef8ae --- /dev/null +++ b/frmts/rcm/rcmdrivercore.h @@ -0,0 +1,112 @@ +/****************************************************************************** + * + * Project: DRDC Ottawa GEOINT + * Purpose: Radarsat Constellation Mission - XML Products (product.xml) driver + * Author: Roberto Caron, MDA + * on behalf of DRDC Ottawa + * + ****************************************************************************** + * Copyright (c) 2020, DRDC Ottawa + * + * Based on the RS2 Dataset Class + * + * SPDX-License-Identifier: MIT + ****************************************************************************/ + +#ifndef RCMDRIVERCORE_H +#define RCMDRIVERCORE_H + +#include "gdal_priv.h" + +// Should be size of larged possible filename. +constexpr int CPL_PATH_BUF_SIZE = 2048; +constexpr char szLayerCalibration[] = "RCM_CALIB"; +constexpr char szLayerSeparator[] = ":"; +constexpr char szSIGMA0[] = "SIGMA0"; +constexpr char szGAMMA[] = "GAMMA"; +constexpr char szBETA0[] = "BETA0"; +constexpr char szUNCALIB[] = "UNCALIB"; +constexpr char szPathSeparator[] = +#ifdef _WIN32 /* Defined if Win32 and Win64 */ + "\\"; +#else + "/"; +#endif +constexpr char cPathSeparator = +#ifdef _WIN32 /* Defined if Win32 and Win64 */ + '\\'; +#else + '/'; +#endif +constexpr char cOppositePathSeparator = +#ifdef _WIN32 /* Defined if Win32 and Win64 */ + '/'; +#else + '\\'; +#endif + +constexpr const char *RCM_DRIVER_NAME = "RCM"; + +/*** Function to format calibration for unique identification for Layer Name + * ***/ +/* + * RCM_CALIB : { SIGMA0 | GAMMA0 | BETA0 | UNCALIB } : product.xml full path + */ +inline CPLString FormatCalibration(const char *pszCalibName, + const char *pszFilename) +{ + CPLString ptr; + + // Always begin by the layer calibrtion name + ptr.append(szLayerCalibration); + + if (pszCalibName != nullptr || pszFilename != nullptr) + { + if (pszCalibName != nullptr) + { + // A separator is needed before concat calibration name + ptr.append(szLayerSeparator); + // Add calibration name + ptr.append(pszCalibName); + } + + if (pszFilename != nullptr) + { + // A separator is needed before concat full filename name + ptr.append(szLayerSeparator); + // Add full filename name + ptr.append(pszFilename); + } + } + else + { + // Always add a separator even though there are no name to concat + ptr.append(szLayerSeparator); + } + + /* return calibration format */ + return ptr; +} + +/*** Function to concat 'metadata' with a folder separator with the filename + * 'product.xml' ***/ +/* + * Should return either 'metadata\product.xml' or 'metadata/product.xml' + */ +inline CPLString GetMetadataProduct() +{ + // Always begin by the layer calibrtion name + CPLString ptr; + ptr.append("metadata"); + ptr.append(szPathSeparator); + ptr.append("product.xml"); + + /* return metadata product filename */ + return ptr; +} + +int CPL_DLL RCMDatasetIdentify(GDALOpenInfo *poOpenInfo); + +void CPL_DLL RCMDriverSetCommonMetadata(GDALDriver *poDriver); + +#endif diff --git a/gcore/gdal_frmts.h b/gcore/gdal_frmts.h index d965ea1a5e0f..d0685136701a 100644 --- a/gcore/gdal_frmts.h +++ b/gcore/gdal_frmts.h @@ -231,6 +231,7 @@ void DeclareDeferredBASISU_KTX2Plugin(void); void CPL_DLL GDALRegister_NOAA_B(void); void CPL_DLL GDALRegister_NSIDCbin(void); void CPL_DLL GDALRegister_SNAP_TIFF(void); +void CPL_DLL GDALRegister_RCM(void); CPL_C_END #endif /* ndef GDAL_FRMTS_H_INCLUDED */ From 113f6fb260e3cb472478a3f3136215be2dad1835 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 24 Oct 2024 18:57:30 +0200 Subject: [PATCH 06/54] RCM: various fixes --- frmts/rcm/rcmdataset.cpp | 274 +++++++++++++----------------------- frmts/rcm/rcmdataset.h | 9 +- frmts/rcm/rcmdrivercore.cpp | 48 ++----- frmts/rcm/rcmdrivercore.h | 4 +- 4 files changed, 115 insertions(+), 220 deletions(-) diff --git a/frmts/rcm/rcmdataset.cpp b/frmts/rcm/rcmdataset.cpp index 8e1086ade405..36daa93cd12a 100644 --- a/frmts/rcm/rcmdataset.cpp +++ b/frmts/rcm/rcmdataset.cpp @@ -36,23 +36,22 @@ constexpr const char *CALIBRATION_FOLDER = "calibration"; static bool IsValidXMLFile(const char *pszPath, const char *pszLut) { /* Return true for valid xml file, false otherwise */ - char *pszLutFile = VSIStrdup(CPLFormFilename(pszPath, pszLut, nullptr)); + const std::string osLutFile = CPLFormFilename(pszPath, pszLut, nullptr); - CPLXMLTreeCloser psLut(CPLParseXMLFile(pszLutFile)); - - CPLFree(pszLutFile); + CPLXMLTreeCloser psLut(CPLParseXMLFile(osLutFile.c_str())); if (psLut.get() == nullptr) { CPLError(CE_Failure, CPLE_OpenFailed, - "ERROR: Failed to open the LUT file %s", pszLutFile); + "ERROR: Failed to open the LUT file %s", osLutFile.c_str()); } return psLut.get() != nullptr; } -static double *InterpolateValues(char **papszList, int tableSize, int stepSize, - int numberOfValues, int pixelFirstLutValue) +static double *InterpolateValues(CSLConstList papszList, int tableSize, + int stepSize, int numberOfValues, + int pixelFirstLutValue) { /* Allocate the right LUT size according to the product range pixel */ double *table = static_cast(CPLCalloc(sizeof(double), tableSize)); @@ -461,17 +460,20 @@ void RCMCalibRasterBand::ReadLUT() char bandNumber[12]; snprintf(bandNumber, sizeof(bandNumber), "%d", poDS->GetRasterCount() + 1); - CPLXMLNode *psLUT = CPLParseXMLFile(m_pszLUTFile); + CPLXMLTreeCloser psLUT(CPLParseXMLFile(m_pszLUTFile)); + if (!psLUT) + return; - this->m_nfOffset = CPLAtof(CPLGetXMLValue(psLUT, "=lut.offset", "0.0")); + this->m_nfOffset = + CPLAtof(CPLGetXMLValue(psLUT.get(), "=lut.offset", "0.0")); this->pixelFirstLutValue = - atoi(CPLGetXMLValue(psLUT, "=lut.pixelFirstLutValue", "0")); + atoi(CPLGetXMLValue(psLUT.get(), "=lut.pixelFirstLutValue", "0")); - this->stepSize = atoi(CPLGetXMLValue(psLUT, "=lut.stepSize", "0")); + this->stepSize = atoi(CPLGetXMLValue(psLUT.get(), "=lut.stepSize", "0")); this->numberOfValues = - atoi(CPLGetXMLValue(psLUT, "=lut.numberOfValues", "0")); + atoi(CPLGetXMLValue(psLUT.get(), "=lut.numberOfValues", "0")); if (this->numberOfValues <= 0) { @@ -482,8 +484,9 @@ void RCMCalibRasterBand::ReadLUT() return; } - char **papszLUTList = CSLTokenizeString2( - CPLGetXMLValue(psLUT, "=lut.gains", ""), " ", CSLT_HONOURSTRINGS); + const CPLStringList aosLUTList( + CSLTokenizeString2(CPLGetXMLValue(psLUT.get(), "=lut.gains", ""), " ", + CSLT_HONOURSTRINGS)); if (this->stepSize <= 0) { @@ -492,13 +495,21 @@ void RCMCalibRasterBand::ReadLUT() CPLError( CE_Failure, CPLE_NotSupported, "ERROR: The RCM driver does not support LUT Pixel First Lut " - "Value equal or lower than zero when theproduct is " + "Value equal or lower than zero when the product is " "descending."); return; } } /* Get the Pixel Per range */ + if (this->stepSize == INT_MIN || this->numberOfValues == INT_MIN || + abs(this->stepSize) > INT_MAX / abs(this->numberOfValues)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Bad values of stepSize / numberOfValues"); + return; + } + this->m_nTableSize = abs(this->stepSize) * abs(this->numberOfValues); if (this->m_nTableSize < this->m_poBandDataset->GetRasterXSize()) @@ -512,7 +523,7 @@ void RCMCalibRasterBand::ReadLUT() /* Allocate the right LUT size according to the product range pixel */ this->m_nfTable = - InterpolateValues(papszLUTList, this->m_nTableSize, this->stepSize, + InterpolateValues(aosLUTList.List(), this->m_nTableSize, this->stepSize, this->numberOfValues, this->pixelFirstLutValue); const size_t nLen = @@ -528,11 +539,6 @@ void RCMCalibRasterBand::ReadLUT() strcat(lut_gains, lut); } -#ifdef _TRACE_RCM - write_to_file("RCM ReadLUT m_pszLUTFile=", m_pszLUTFile); - write_to_file(" m_nfTable=", lut_gains); -#endif - poDS->SetMetadataItem(CPLString("LUT_GAINS_").append(bandNumber).c_str(), lut_gains); // Can free this because the function SetMetadataItem takes a copy @@ -560,10 +566,6 @@ void RCMCalibRasterBand::ReadLUT() snprintf(snum, sizeof(snum), "%f", this->m_nfOffset); poDS->SetMetadataItem(CPLString("LUT_OFFSET_").append(bandNumber).c_str(), snum); - - CPLDestroyXMLNode(psLUT); - - CSLDestroy(papszLUTList); } /************************************************************************/ @@ -591,15 +593,19 @@ void RCMCalibRasterBand::ReadNoiseLevels() char bandNumber[12]; snprintf(bandNumber, sizeof(bandNumber), "%d", poDS->GetRasterCount() + 1); - CPLXMLNode *psNoiseLevels = CPLParseXMLFile(this->m_pszNoiseLevelsFile); + CPLXMLTreeCloser psNoiseLevels(CPLParseXMLFile(this->m_pszNoiseLevelsFile)); + if (!psNoiseLevels) + return; // Load Beta Nought, Sigma Nought, Gamma noise levels // Loop through all nodes with spaces - CPLXMLNode *psreferenceNoiseLevelNode = - CPLGetXMLNode(psNoiseLevels, "=noiseLevels"); + CPLXMLNode *psReferenceNoiseLevelNode = + CPLGetXMLNode(psNoiseLevels.get(), "=noiseLevels"); + if (!psReferenceNoiseLevelNode) + return; CPLXMLNode *psNodeInc; - for (psNodeInc = psreferenceNoiseLevelNode->psChild; psNodeInc != nullptr; + for (psNodeInc = psReferenceNoiseLevelNode->psChild; psNodeInc != nullptr; psNodeInc = psNodeInc->psNext) { if (EQUAL(psNodeInc->pszValue, "referenceNoiseLevel")) @@ -658,29 +664,6 @@ void RCMCalibRasterBand::ReadNoiseLevels() } } } - -#ifdef _TRACE_RCM - if (this->m_nfTableNoiseLevels != nullptr) - { - const size_t nLen = - this->m_nTableNoiseLevelsSize * - max_space_for_string; // 12 max + space + 11 reserved - char *noise_levels_values = static_cast(CPLMalloc(nLen)); - memset(noise_levels_values, 0, nLen); - - for (int i = 0; i < this->m_nTableNoiseLevelsSize; i++) - { - char lut[max_space_for_string]; - // 6.123004711900930e+04 %e Scientific annotation - snprintf(lut, sizeof(lut), "%e ", this->m_nfTableNoiseLevels[i]); - strcat(noise_levels_values, lut); - } - write_to_file("RCM ReadNoiseLevel m_pszLUTFile=", m_pszNoiseLevelsFile); - write_to_file(" m_nfTableNoiseLevels=", noise_levels_values); - - CPLFree(noise_levels_values); - } -#endif } /************************************************************************/ @@ -879,35 +862,6 @@ void RCMCalibRasterBand::SetPartialLUT(int pixel_offset, int pixel_width) } } -double *RCMCalibRasterBand::CloneLUT() -{ - double *values = nullptr; - - if (this->m_nfTable != nullptr) - { - values = reinterpret_cast( - CPLMalloc(sizeof(double) * this->m_nTableSize)); - memcpy(values, this->m_nfTable, sizeof(double) * this->m_nTableSize); - } - - return values; -} - -double *RCMCalibRasterBand::CloneNoiseLevels() -{ - double *values = nullptr; - - if (this->m_nfTableNoiseLevels != nullptr) - { - values = - (double *)malloc(sizeof(double) * this->m_nTableNoiseLevelsSize); - memcpy(values, this->m_nfTableNoiseLevels, - sizeof(double) * this->m_nTableNoiseLevelsSize); - } - - return values; -} - /************************************************************************/ /* ~RCMCalibRasterBand() */ /************************************************************************/ @@ -1035,20 +989,6 @@ CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, static_cast(m_nfTable[nBlockXOff * nBlockXSize + j]); float calibValue = digitalValue / (lutValue * lutValue); -#ifdef _TRACE_RCM - if (nBlockXOff * nBlockXSize + j >= 4961) - { - char msgBlocks[2048] = ""; - snprintf(msgBlocks, sizeof(msgBlocks), - "IReadBlock: [%d,%d] real=%f img=%f " - "digitalValue=%f lutValue[%d]=%f calibValue=%f", - i, j, real, img, digitalValue, - (nBlockXOff * nBlockXSize + j), lutValue, - calibValue); - write_to_file(msgBlocks, ""); - } -#endif - ((float *)pImage)[nTruePixOff] = calibValue; } } @@ -1243,7 +1183,7 @@ CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, /* RCMDataset() */ /************************************************************************/ -RCMDataset::RCMDataset() : pszLutApplied(CPLStrdup("")) +RCMDataset::RCMDataset() { adfGeoTransform[0] = 0.0; adfGeoTransform[1] = 1.0; @@ -1262,9 +1202,6 @@ RCMDataset::~RCMDataset() { RCMDataset::FlushCache(true); - CPLDestroyXMLNode(psProduct); - CPLFree(pszLutApplied); - if (nGCPCount > 0) { GDALDeinitGCPs(nGCPCount, pasGCPList); @@ -1281,8 +1218,6 @@ RCMDataset::~RCMDataset() if (m_nfIncidenceAngleTable != nullptr) CPLFree(m_nfIncidenceAngleTable); - - psProduct = nullptr; } /************************************************************************/ @@ -1367,7 +1302,9 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) } else { - eCalib = None; + CPLError(CE_Failure, CPLE_NotSupported, + "Unsupported calibration type"); + return nullptr; } /* advance the pointer to the actual filename */ @@ -1407,8 +1344,8 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) /* -------------------------------------------------------------------- */ /* Ingest the Product.xml file. */ /* -------------------------------------------------------------------- */ - CPLXMLNode *psProduct = CPLParseXMLFile(osMDFilename); - if (psProduct == nullptr) + CPLXMLTreeCloser psProduct(CPLParseXMLFile(osMDFilename)); + if (!psProduct) return nullptr; /* -------------------------------------------------------------------- */ @@ -1416,7 +1353,6 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) /* -------------------------------------------------------------------- */ if (poOpenInfo->eAccess == GA_Update) { - CPLDestroyXMLNode(psProduct); CPLError(CE_Failure, CPLE_NotSupported, "ERROR: The RCM driver does not support update " "access to existing dataset."); @@ -1424,20 +1360,18 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) } CPLXMLNode *psSceneAttributes = - CPLGetXMLNode(psProduct, "=product.sceneAttributes"); + CPLGetXMLNode(psProduct.get(), "=product.sceneAttributes"); if (psSceneAttributes == nullptr) { - CPLDestroyXMLNode(psProduct); CPLError(CE_Failure, CPLE_OpenFailed, "ERROR: Failed to find in document."); return nullptr; } - CPLXMLNode *psImageAttributes = - CPLGetXMLNode(psProduct, "=product.sceneAttributes.imageAttributes"); + CPLXMLNode *psImageAttributes = CPLGetXMLNode( + psProduct.get(), "=product.sceneAttributes.imageAttributes"); if (psImageAttributes == nullptr) { - CPLDestroyXMLNode(psProduct); CPLError(CE_Failure, CPLE_OpenFailed, "ERROR: Failed to find in " "document."); @@ -1448,17 +1382,15 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) atoi(CPLGetXMLValue(psSceneAttributes, "numberOfEntries", "0")); if (numberOfEntries != 1) { - CPLDestroyXMLNode(psProduct); CPLError(CE_Failure, CPLE_OpenFailed, "ERROR: Only RCM with Complex Single-beam is supported."); return nullptr; } CPLXMLNode *psImageReferenceAttributes = - CPLGetXMLNode(psProduct, "=product.imageReferenceAttributes"); + CPLGetXMLNode(psProduct.get(), "=product.imageReferenceAttributes"); if (psImageReferenceAttributes == nullptr) { - CPLDestroyXMLNode(psProduct); CPLError( CE_Failure, CPLE_OpenFailed, "ERROR: Failed to find in document."); @@ -1466,10 +1398,9 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) } CPLXMLNode *psImageGenerationParameters = - CPLGetXMLNode(psProduct, "=product.imageGenerationParameters"); + CPLGetXMLNode(psProduct.get(), "=product.imageGenerationParameters"); if (psImageGenerationParameters == nullptr) { - CPLDestroyXMLNode(psProduct); CPLError( CE_Failure, CPLE_OpenFailed, "ERROR: Failed to find in document."); @@ -1479,9 +1410,9 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) /* -------------------------------------------------------------------- */ /* Create the dataset. */ /* -------------------------------------------------------------------- */ - RCMDataset *poDS = new RCMDataset(); + auto poDS = std::make_unique(); - poDS->psProduct = psProduct; + poDS->psProduct = std::move(psProduct); /* -------------------------------------------------------------------- */ /* Get overall image information. */ @@ -1492,7 +1423,6 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) CPLGetXMLValue(psSceneAttributes, "imageAttributes.numLines", "-1")); if (poDS->nRasterXSize <= 1 || poDS->nRasterYSize <= 1) { - delete poDS; CPLError( CE_Failure, CPLE_OpenFailed, "ERROR: Non-sane raster dimensions provided in product.xml. If " @@ -1513,15 +1443,18 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) poDS->SetMetadataItem("PRODUCT_TYPE", pszItem); const char *pszProductType = pszItem; - pszItem = CPLGetXMLValue(psProduct, "=product.productId", "UNK"); + pszItem = + CPLGetXMLValue(poDS->psProduct.get(), "=product.productId", "UNK"); poDS->SetMetadataItem("PRODUCT_ID", pszItem); pszItem = CPLGetXMLValue( - psProduct, "=product.securityAttributes.securityClassification", "UNK"); + poDS->psProduct.get(), + "=product.securityAttributes.securityClassification", "UNK"); poDS->SetMetadataItem("SECURITY_CLASSIFICATION", pszItem); - pszItem = CPLGetXMLValue( - psProduct, "=product.sourceAttributes.polarizationDataMode", "UNK"); + pszItem = + CPLGetXMLValue(poDS->psProduct.get(), + "=product.sourceAttributes.polarizationDataMode", "UNK"); poDS->SetMetadataItem("POLARIZATION_DATA_MODE", pszItem); pszItem = CPLGetXMLValue(psImageGenerationParameters, @@ -1551,9 +1484,6 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) pszItem = CPLGetXMLValue(psImageGenerationParameters, "sarProcessingInformation.lutApplied", ""); poDS->SetMetadataItem("LUT_APPLIED", pszItem); - poDS->pszLutApplied = static_cast(VSIMalloc(strlen(pszItem) + 2)); - poDS->pszLutApplied[0] = '\0'; - strcpy(poDS->pszLutApplied, pszItem); /*--------------------------------------------------------------------- If true, a polarization dependent application LUT has been applied @@ -1671,7 +1601,6 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) } else { - delete poDS; CPLError(CE_Failure, CPLE_AppDefined, "ERROR: dataType=%s and bitsPerSample=%d are not a supported " "configuration.", @@ -1715,15 +1644,13 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) // XML std::string osNoiseLevelsValues; - char *pszPath = CPLStrdup(CPLGetPath(osMDFilename)); + const CPLString osPath = CPLGetPath(osMDFilename); /* Get a list of all polarizations */ CPLXMLNode *psSourceAttrs = - CPLGetXMLNode(psProduct, "=product.sourceAttributes"); + CPLGetXMLNode(poDS->psProduct.get(), "=product.sourceAttributes"); if (psSourceAttrs == nullptr) { - CPLFree(pszPath); - delete poDS; CPLError( CE_Failure, CPLE_OpenFailed, "ERROR: RCM source attributes is missing. Please contact your data " @@ -1731,12 +1658,10 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) return nullptr; } - CPLXMLNode *psRadarParameters = - CPLGetXMLNode(psProduct, "=product.sourceAttributes.radarParameters"); + CPLXMLNode *psRadarParameters = CPLGetXMLNode( + poDS->psProduct.get(), "=product.sourceAttributes.radarParameters"); if (psRadarParameters == nullptr) { - CPLFree(pszPath); - delete poDS; CPLError( CE_Failure, CPLE_OpenFailed, "ERROR: RCM radar parameters is missing. Please contact your data " @@ -1748,8 +1673,6 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) CPLGetXMLValue(psRadarParameters, "polarizations", ""); if (pszPolarizations == nullptr || strlen(pszPolarizations) == 0) { - CPLFree(pszPath); - delete poDS; CPLError( CE_Failure, CPLE_OpenFailed, "ERROR: RCM polarizations list is missing. Please contact your " @@ -1765,11 +1688,11 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) const char *psBeams = CPLGetXMLValue(psRadarParameters, "beams", "UNK"); poDS->SetMetadataItem("BEAMS", psBeams); - char **papszPolarizationsGrids = - CSLTokenizeString2(pszPolarizations, " ", 0); + const CPLStringList aosPolarizationsGrids( + CSLTokenizeString2(pszPolarizations, " ", 0)); CPLStringList imageBandList; CPLStringList imageBandFileList; - const int nPolarizationsGridCount = CSLCount(papszPolarizationsGrids); + const int nPolarizationsGridCount = aosPolarizationsGrids.size(); /* File names for full resolution IPDFs. For GeoTIFF format, one entry per pole; For NITF 2.1 format, only one entry. */ @@ -1859,23 +1782,24 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) osIncidenceAnglePath.append(pszIncidenceAngleFileName); /* Check if the file exist */ - if (IsValidXMLFile(pszPath, osIncidenceAnglePath)) + if (IsValidXMLFile(osPath, osIncidenceAnglePath)) { CPLString osIncidenceAngleFilePath = - CPLFormFilename(pszPath, osIncidenceAnglePath, nullptr); + CPLFormFilename(osPath, osIncidenceAnglePath, nullptr); - CPLXMLNode *psIncidenceAngle = - CPLParseXMLFile(osIncidenceAngleFilePath); + CPLXMLTreeCloser psIncidenceAngle( + CPLParseXMLFile(osIncidenceAngleFilePath)); int pixelFirstLutValue = atoi( - CPLGetXMLValue(psIncidenceAngle, + CPLGetXMLValue(psIncidenceAngle.get(), "=incidenceAngles.pixelFirstAnglesValue", "0")); int stepSize = atoi(CPLGetXMLValue( - psIncidenceAngle, "=incidenceAngles.stepSize", "0")); + psIncidenceAngle.get(), "=incidenceAngles.stepSize", "0")); - int numberOfValues = atoi(CPLGetXMLValue( - psIncidenceAngle, "=incidenceAngles.numberOfValues", "0")); + int numberOfValues = + atoi(CPLGetXMLValue(psIncidenceAngle.get(), + "=incidenceAngles.numberOfValues", "0")); /* Get the Pixel Per range */ int tableSize = abs(stepSize) * abs(numberOfValues); @@ -1883,7 +1807,7 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) CPLString angles; // Loop through all nodes with spaces CPLXMLNode *psNextNode = - CPLGetXMLNode(psIncidenceAngle, "=incidenceAngles"); + CPLGetXMLNode(psIncidenceAngle.get(), "=incidenceAngles"); CPLXMLNode *psNodeInc; for (psNodeInc = psNextNode->psChild; psNodeInc != nullptr; @@ -1918,7 +1842,7 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) { // Search for a specific band name const CPLString pszPole = - CPLString(papszPolarizationsGrids[iPoleInx]).toupper(); + CPLString(aosPolarizationsGrids[iPoleInx]).toupper(); // Look if the NoiseLevel file xml exist for the CPLXMLNode *psRefNode = psImageReferenceAttributes->psChild; @@ -1967,7 +1891,7 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) oNoiseLevelPath.append(szPathSeparator); oNoiseLevelPath.append(pszNoiseLevelFile); - if (IsValidXMLFile(pszPath, oNoiseLevelPath)) + if (IsValidXMLFile(osPath, oNoiseLevelPath)) { osNoiseLevelsValues = oNoiseLevelPath; } @@ -2026,16 +1950,17 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) osCalibPath.append(pszLUTFile); CPLString osLUTFilePath = - CPLFormFilename(pszPath, osCalibPath, nullptr); + CPLFormFilename(osPath, osCalibPath, nullptr); if (EQUAL(pszLUTType, "Beta Nought") && - IsValidXMLFile(pszPath, osCalibPath)) + IsValidXMLFile(osPath, osCalibPath)) { poDS->papszExtraFiles = CSLAddString(poDS->papszExtraFiles, osLUTFilePath); CPLString pszBuf( FormatCalibration(szBETA0, osMDFilename.c_str())); + CPLFree(pszBeta0LUT); pszBeta0LUT = VSIStrdup(osCalibPath); const char *oldLut = @@ -2066,13 +1991,14 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) "Beta Nought calibrated"); } else if (EQUAL(pszLUTType, "Sigma Nought") && - IsValidXMLFile(pszPath, osCalibPath)) + IsValidXMLFile(osPath, osCalibPath)) { poDS->papszExtraFiles = CSLAddString(poDS->papszExtraFiles, osLUTFilePath); CPLString pszBuf( FormatCalibration(szSIGMA0, osMDFilename.c_str())); + CPLFree(pszSigma0LUT); pszSigma0LUT = VSIStrdup(osCalibPath); const char *oldLut = @@ -2103,13 +2029,14 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) "Sigma Nought calibrated"); } else if (EQUAL(pszLUTType, "Gamma") && - IsValidXMLFile(pszPath, osCalibPath)) + IsValidXMLFile(osPath, osCalibPath)) { poDS->papszExtraFiles = CSLAddString(poDS->papszExtraFiles, osLUTFilePath); CPLString pszBuf( FormatCalibration(szGAMMA, osMDFilename.c_str())); + CPLFree(pszGammaLUT); pszGammaLUT = VSIStrdup(osCalibPath); const char *oldLut = poDS->GetMetadataItem("GAMMA_LUT"); @@ -2158,10 +2085,7 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) if (bandPositionIndex < 0) { CPLFree(imageBandList); - CSLDestroy(papszPolarizationsGrids); CPLFree(imageBandFileList); - CPLFree(pszPath); - delete poDS; CPLError(CE_Failure, CPLE_OpenFailed, "ERROR: RCM cannot find the polarization %s. Please " @@ -2195,7 +2119,7 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) /* -------------------------------------------------------------------- */ char *pszFullname = - CPLStrdup(CPLFormFilename(pszPath, pszBasename, nullptr)); + CPLStrdup(CPLFormFilename(osPath, pszBasename, nullptr)); CPLFree(pszBasename); @@ -2204,8 +2128,8 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) /* Try and open the file. */ /* -------------------------------------------------------------------- */ - GDALDataset *poBandFile = - reinterpret_cast(GDALOpen(pszFullname, GA_ReadOnly)); + auto poBandFile = std::unique_ptr(GDALDataset::Open( + pszFullname, GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR)); if (poBandFile == nullptr) { CPLFree(pszFullname); @@ -2213,7 +2137,6 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) } if (poBandFile->GetRasterCount() == 0) { - GDALClose(reinterpret_cast(poBandFile)); CPLFree(pszFullname); continue; } @@ -2229,12 +2152,10 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) eDataType = GDT_CFloat32; BandMappingRCM b = - checkBandFileMappingRCM(eDataType, poBandFile, bIsNITF); + checkBandFileMappingRCM(eDataType, poBandFile.get(), bIsNITF); if (b == BANDERROR) { - GDALClose((GDALRasterBandH)poBandFile); CPLFree(pszFullname); - delete poDS; CPLError(CE_Failure, CPLE_AppDefined, "The underlying band files do not have an appropriate " "data type."); @@ -2251,9 +2172,9 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) int bandNum = poDS->GetRasterCount() + 1; if (eCalib == None || eCalib == Uncalib) { - RCMRasterBand *poBand = - new RCMRasterBand(poDS, bandNum, eDataType, pszPole, poBandFile, - twoBandComplex, isOneFilePerPol, bIsNITF); + RCMRasterBand *poBand = new RCMRasterBand( + poDS.get(), bandNum, eDataType, pszPole, poBandFile.release(), + twoBandComplex, isOneFilePerPol, bIsNITF); poDS->SetBand(poDS->GetRasterCount() + 1, poBand); } @@ -2282,9 +2203,9 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) { // If Complex, always 32 bits RCMCalibRasterBand *poBand = new RCMCalibRasterBand( - poDS, pszPole, GDT_Float32, poBandFile, eCalib, - CPLFormFilename(pszPath, pszLUT, nullptr), - CPLFormFilename(pszPath, osNoiseLevelsValues.c_str(), + poDS.get(), pszPole, GDT_Float32, poBandFile.release(), + eCalib, CPLFormFilename(osPath, pszLUT, nullptr), + CPLFormFilename(osPath, osNoiseLevelsValues.c_str(), nullptr), eDataType); poDS->SetBand(poDS->GetRasterCount() + 1, poBand); @@ -2293,9 +2214,9 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) { // Whatever the datatype was previoulsy set RCMCalibRasterBand *poBand = new RCMCalibRasterBand( - poDS, pszPole, eDataType, poBandFile, eCalib, - CPLFormFilename(pszPath, pszLUT, nullptr), - CPLFormFilename(pszPath, osNoiseLevelsValues.c_str(), + poDS.get(), pszPole, eDataType, poBandFile.release(), + eCalib, CPLFormFilename(osPath, pszLUT, nullptr), + CPLFormFilename(osPath, osNoiseLevelsValues.c_str(), nullptr), eDataType); poDS->SetBand(poDS->GetRasterCount() + 1, poBand); @@ -2686,7 +2607,6 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) } } - CPLFree(pszPath); if (pszBeta0LUT) CPLFree(pszBeta0LUT); if (pszSigma0LUT) @@ -2817,11 +2737,11 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) /* Check for overviews. */ /* -------------------------------------------------------------------- */ if (useSubdatasets) - poDS->oOvManager.Initialize(poDS, ":::VIRTUAL:::"); + poDS->oOvManager.Initialize(poDS.get(), ":::VIRTUAL:::"); else - poDS->oOvManager.Initialize(poDS, osMDFilename); + poDS->oOvManager.Initialize(poDS.get(), osMDFilename); - return poDS; + return poDS.release(); } /************************************************************************/ diff --git a/frmts/rcm/rcmdataset.h b/frmts/rcm/rcmdataset.h index 5d5369a6bb84..e40e726e09e4 100644 --- a/frmts/rcm/rcmdataset.h +++ b/frmts/rcm/rcmdataset.h @@ -35,14 +35,13 @@ typedef enum eCalibration_t class RCMDataset final : public GDALPamDataset { - CPLXMLNode *psProduct = nullptr; + CPLXMLTreeCloser psProduct{nullptr}; int nGCPCount = 0; GDAL_GCP *pasGCPList = nullptr; OGRSpatialReference m_oSRS{}; OGRSpatialReference m_oGCPSRS{}; char **papszSubDatasets = nullptr; - char *pszLutApplied = nullptr; double adfGeoTransform[6]; bool bHaveGeoTransform = false; bool bPerPolarizationScaling = false; @@ -77,7 +76,7 @@ class RCMDataset final : public GDALPamDataset CPLXMLNode *GetProduct() { - return psProduct; + return psProduct.get(); } /* If False, this is Magnitude, True, Complex data with Real and @@ -252,10 +251,6 @@ class RCMCalibRasterBand final : public GDALPamRasterBand bool IsComplex(); eCalibration GetCalibration(); - - double *CloneLUT(); - - double *CloneNoiseLevels(); }; #endif /* ndef GDAL_RCM_H_INCLUDED */ diff --git a/frmts/rcm/rcmdrivercore.cpp b/frmts/rcm/rcmdrivercore.cpp index e6b7362744a7..f20d947dca6f 100644 --- a/frmts/rcm/rcmdrivercore.cpp +++ b/frmts/rcm/rcmdrivercore.cpp @@ -27,13 +27,7 @@ int RCMDatasetIdentify(GDALOpenInfo *poOpenInfo) if (poOpenInfo->bIsDirectory) { - /* Check for directory access when there is a product.xml file in the - directory. */ - CPLString osMDFilename = - CPLFormCIFilename(poOpenInfo->pszFilename, "product.xml", nullptr); - - VSIStatBufL sStat; - if (VSIStatL(osMDFilename, &sStat) == 0) + const auto IsRCM = [](const CPLString &osMDFilename) { CPLXMLNode *psProduct = CPLParseXMLFile(osMDFilename); if (psProduct == nullptr) @@ -47,7 +41,7 @@ int RCMDatasetIdentify(GDALOpenInfo *poOpenInfo) return FALSE; } - /* Check the namespace only, should be rs2 */ + /* Check the namespace only, should be rcm */ const char *szNamespace = CPLGetXMLValue(psProductAttributes, "xmlns", ""); @@ -60,6 +54,17 @@ int RCMDatasetIdentify(GDALOpenInfo *poOpenInfo) CPLDestroyXMLNode(psProduct); return TRUE; + }; + + /* Check for directory access when there is a product.xml file in the + directory. */ + CPLString osMDFilename = + CPLFormCIFilename(poOpenInfo->pszFilename, "product.xml", nullptr); + + VSIStatBufL sStat; + if (VSIStatL(osMDFilename, &sStat) == 0) + { + return IsRCM(osMDFilename); } /* If not, check for directory extra 'metadata' access when there is a @@ -71,32 +76,7 @@ int RCMDatasetIdentify(GDALOpenInfo *poOpenInfo) VSIStatBufL sStatMetadata; if (VSIStatL(osMDFilenameMetadata, &sStatMetadata) == 0) { - CPLXMLNode *psProduct = CPLParseXMLFile(osMDFilenameMetadata); - if (psProduct == nullptr) - return FALSE; - - CPLXMLNode *psProductAttributes = - CPLGetXMLNode(psProduct, "=product"); - if (psProductAttributes == nullptr) - { - CPLDestroyXMLNode(psProduct); - return FALSE; - } - - /* Check the namespace only, should be rs2 */ - const char *szNamespace = - CPLGetXMLValue(psProductAttributes, "xmlns", ""); - - if (strstr(szNamespace, "rcm") == nullptr) - { - /* Invalid namespace */ - CPLDestroyXMLNode(psProduct); - return FALSE; - } - - CPLDestroyXMLNode(psProduct); - - return TRUE; + return IsRCM(osMDFilenameMetadata); } return FALSE; diff --git a/frmts/rcm/rcmdrivercore.h b/frmts/rcm/rcmdrivercore.h index fbb6b45ef8ae..6dfe764e5dd6 100644 --- a/frmts/rcm/rcmdrivercore.h +++ b/frmts/rcm/rcmdrivercore.h @@ -57,7 +57,7 @@ inline CPLString FormatCalibration(const char *pszCalibName, { CPLString ptr; - // Always begin by the layer calibrtion name + // Always begin by the layer calibration name ptr.append(szLayerCalibration); if (pszCalibName != nullptr || pszFilename != nullptr) @@ -95,7 +95,7 @@ inline CPLString FormatCalibration(const char *pszCalibName, */ inline CPLString GetMetadataProduct() { - // Always begin by the layer calibrtion name + // Always begin by the layer calibration name CPLString ptr; ptr.append("metadata"); ptr.append(szPathSeparator); From a9377f8db8d55aa2b6a460b2825579175a3d17b9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 24 Oct 2024 20:09:57 +0200 Subject: [PATCH 07/54] RCM: enable STRONG_CXX_WFLAGS --- frmts/rcm/CMakeLists.txt | 1 + frmts/rcm/rcmdataset.cpp | 102 ++++++++++++++++++------------------ frmts/rcm/rcmdataset.h | 6 +++ frmts/rcm/rcmdrivercore.cpp | 8 ++- 4 files changed, 62 insertions(+), 55 deletions(-) diff --git a/frmts/rcm/CMakeLists.txt b/frmts/rcm/CMakeLists.txt index f6d342712c55..2ee719846dc7 100644 --- a/frmts/rcm/CMakeLists.txt +++ b/frmts/rcm/CMakeLists.txt @@ -3,6 +3,7 @@ add_gdal_driver(TARGET gdal_RCM CORE_SOURCES rcmdrivercore.cpp rcmdrivercore.h PLUGIN_CAPABLE NO_DEPS + STRONG_CXX_WFLAGS NO_SHARED_SYMBOL_WITH_CORE) gdal_standard_includes(gdal_RCM) diff --git a/frmts/rcm/rcmdataset.cpp b/frmts/rcm/rcmdataset.cpp index 36daa93cd12a..6d404e56b2d9 100644 --- a/frmts/rcm/rcmdataset.cpp +++ b/frmts/rcm/rcmdataset.cpp @@ -356,8 +356,7 @@ CPLErr RCMRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, void *pImage) { nRequestYSize = nRasterYSize - nBlockYOff * nBlockYSize; memset(pImage, 0, - (GDALGetDataTypeSize(eDataType) / 8) * nBlockXSize * - nBlockYSize); + GDALGetDataTypeSizeBytes(eDataType) * nBlockXSize * nBlockYSize); } else { @@ -372,8 +371,7 @@ CPLErr RCMRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, void *pImage) { nRequestXSize = nRasterXSize - nBlockXOff * nBlockXSize; memset(pImage, 0, - (GDALGetDataTypeSize(eDataType) / 8) * nBlockXSize * - nBlockYSize); + GDALGetDataTypeSizeBytes(eDataType) * nBlockXSize * nBlockYSize); } else { @@ -895,8 +893,7 @@ CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, { nRequestYSize = nRasterYSize - nBlockYOff * nBlockYSize; memset(pImage, 0, - (GDALGetDataTypeSize(eDataType) / 8) * nBlockXSize * - nBlockYSize); + GDALGetDataTypeSizeBytes(eDataType) * nBlockXSize * nBlockYSize); } else { @@ -911,8 +908,7 @@ CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, { nRequestXSize = nRasterXSize - nBlockXOff * nBlockXSize; memset(pImage, 0, - (GDALGetDataTypeSize(eDataType) / 8) * nBlockXSize * - nBlockYSize); + GDALGetDataTypeSizeBytes(eDataType) * nBlockXSize * nBlockYSize); } else { @@ -923,8 +919,9 @@ CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, { GInt16 *pnImageTmp; /* read in complex values */ - pnImageTmp = (GInt16 *)CPLMalloc(2 * nBlockXSize * nBlockYSize * - GDALGetDataTypeSize(GDT_Int16) / 8); + pnImageTmp = static_cast( + CPLMalloc(nBlockXSize * nBlockYSize * + GDALGetDataTypeSizeBytes(m_eOriginalType))); if (m_poBandDataset->GetRasterCount() == 2) { @@ -978,18 +975,18 @@ CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, for (int j = 0; j < nRequestXSize; j++) { /* calculate pixel offset in memory*/ - int nPixOff = (2 * (i * nBlockXSize)) + (j * 2); - int nTruePixOff = (i * nBlockXSize) + j; + const int nPixOff = 2 * (i * nBlockXSize + j); + const int nTruePixOff = (i * nBlockXSize) + j; // Formula for Complex Q+J - float real = (float)pnImageTmp[nPixOff]; - float img = (float)pnImageTmp[nPixOff + 1]; - float digitalValue = (real * real) + (img * img); - float lutValue = + const float real = static_cast(pnImageTmp[nPixOff]); + const float img = static_cast(pnImageTmp[nPixOff + 1]); + const float digitalValue = (real * real) + (img * img); + const float lutValue = static_cast(m_nfTable[nBlockXOff * nBlockXSize + j]); - float calibValue = digitalValue / (lutValue * lutValue); + const float calibValue = digitalValue / (lutValue * lutValue); - ((float *)pImage)[nTruePixOff] = calibValue; + reinterpret_cast(pImage)[nTruePixOff] = calibValue; } } @@ -1003,13 +1000,14 @@ CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, /* read in complex values */ float *pnImageTmp; - int dataTypeSize = GDALGetDataTypeSize(this->m_eOriginalType) / 8; - GDALDataType bandFileType = this->m_eOriginalType; - int bandFileSize = GDALGetDataTypeSize(bandFileType) / 8; + const int dataTypeSize = + GDALGetDataTypeSizeBytes(this->m_eOriginalType); + const GDALDataType bandFileType = this->m_eOriginalType; + const int bandFileSize = GDALGetDataTypeSizeBytes(bandFileType); /* read the original image complex values in a temporary image space */ - pnImageTmp = - (float *)CPLMalloc(2 * nBlockXSize * nBlockYSize * bandFileSize); + pnImageTmp = static_cast( + CPLMalloc(2 * nBlockXSize * nBlockYSize * bandFileSize)); eErr = // I and Q from each band are pixel-interleaved into this complex @@ -1026,18 +1024,18 @@ CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, for (int j = 0; j < nRequestXSize; j++) { /* calculate pixel offset in memory*/ - int nPixOff = (2 * (i * nBlockXSize)) + (j * 2); - int nTruePixOff = (i * nBlockXSize) + j; + const int nPixOff = 2 * (i * nBlockXSize + j); + const int nTruePixOff = (i * nBlockXSize) + j; // Formula for Complex Q+J - float real = (float)pnImageTmp[nPixOff]; - float img = (float)pnImageTmp[nPixOff + 1]; - float digitalValue = (real * real) + (img * img); - float lutValue = + const float real = static_cast(pnImageTmp[nPixOff]); + const float img = static_cast(pnImageTmp[nPixOff + 1]); + const float digitalValue = (real * real) + (img * img); + const float lutValue = static_cast(m_nfTable[nBlockXOff * nBlockXSize + j]); - float calibValue = digitalValue / (lutValue * lutValue); + const float calibValue = digitalValue / (lutValue * lutValue); - ((float *)pImage)[nTruePixOff] = calibValue; + reinterpret_cast(pImage)[nTruePixOff] = calibValue; } } @@ -1064,10 +1062,11 @@ CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, is first squared, then the offset(B) is added and the result is divided by the gains value(A) corresponding to the range sample. RCM-SP-53-0419 Issue 2/5: January 2, 2018 Page 7-56 */ - float digitalValue = ((float *)pImage)[nPixOff]; - float A = + const float digitalValue = + reinterpret_cast(pImage)[nPixOff]; + const float A = static_cast(m_nfTable[nBlockXOff * nBlockXSize + j]); - ((float *)pImage)[nPixOff] = + reinterpret_cast(pImage)[nPixOff] = ((digitalValue * digitalValue) + static_cast(this->m_nfOffset)) / A; @@ -1095,10 +1094,11 @@ CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, is first squared, then the offset(B) is added and the result is divided by the gains value(A) corresponding to the range sample. RCM-SP-53-0419 Issue 2/5: January 2, 2018 Page 7-56 */ - float digitalValue = ((float *)pImage)[nPixOff]; - float A = + const float digitalValue = + reinterpret_cast(pImage)[nPixOff]; + const float A = static_cast(m_nfTable[nBlockXOff * nBlockXSize + j]); - ((float *)pImage)[nPixOff] = + reinterpret_cast(pImage)[nPixOff] = ((digitalValue * digitalValue) + static_cast(this->m_nfOffset)) / A; @@ -1110,8 +1110,8 @@ CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, { GUInt16 *pnImageTmp; /* read in detected values */ - pnImageTmp = (GUInt16 *)CPLMalloc(nBlockXSize * nBlockYSize * - GDALGetDataTypeSize(GDT_UInt16) / 8); + pnImageTmp = static_cast(CPLMalloc( + nBlockXSize * nBlockYSize * GDALGetDataTypeSizeBytes(GDT_UInt16))); eErr = m_poBandDataset->RasterIO( GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, nRequestXSize, nRequestYSize, pnImageTmp, nRequestXSize, @@ -1123,12 +1123,13 @@ CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, { for (int j = 0; j < nRequestXSize; j++) { - int nPixOff = (i * nBlockXSize) + j; + const int nPixOff = (i * nBlockXSize) + j; - float digitalValue = (float)pnImageTmp[nPixOff]; - float A = + const float digitalValue = + static_cast(pnImageTmp[nPixOff]); + const float A = static_cast(m_nfTable[nBlockXOff * nBlockXSize + j]); - ((float *)pImage)[nPixOff] = + reinterpret_cast(pImage)[nPixOff] = ((digitalValue * digitalValue) + static_cast(this->m_nfOffset)) / A; @@ -1140,8 +1141,7 @@ CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, else if (this->m_eOriginalType == GDT_Byte) { GByte *pnImageTmp; - pnImageTmp = (GByte *)CPLMalloc(nBlockXSize * nBlockYSize * - GDALGetDataTypeSize(GDT_Byte) / 8); + pnImageTmp = static_cast(CPLMalloc(nBlockXSize * nBlockYSize)); eErr = m_poBandDataset->RasterIO( GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, nRequestXSize, nRequestYSize, pnImageTmp, nRequestXSize, @@ -1152,12 +1152,13 @@ CPLErr RCMCalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, { for (int j = 0; j < nRequestXSize; j++) { - int nPixOff = (i * nBlockXSize) + j; + const int nPixOff = (i * nBlockXSize) + j; - float digitalValue = (float)pnImageTmp[nPixOff]; - float A = + const float digitalValue = + static_cast(pnImageTmp[nPixOff]); + const float A = static_cast(m_nfTable[nBlockXOff * nBlockXSize + j]); - ((float *)pImage)[nPixOff] = + reinterpret_cast(pImage)[nPixOff] = ((digitalValue * digitalValue) + static_cast(this->m_nfOffset)) / A; @@ -2098,7 +2099,8 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) } int nLength = static_cast(strlen(pszBasedFilename)); - char *pszBasename = (char *)CPLCalloc(nLength + 1, sizeof(char)); + char *pszBasename = + static_cast(CPLCalloc(nLength + 1, sizeof(char))); /* -------------------------------------------------------------------- */ diff --git a/frmts/rcm/rcmdataset.h b/frmts/rcm/rcmdataset.h index e40e726e09e4..150acfd00346 100644 --- a/frmts/rcm/rcmdataset.h +++ b/frmts/rcm/rcmdataset.h @@ -53,6 +53,8 @@ class RCMDataset final : public GDALPamDataset double *m_nfIncidenceAngleTable = nullptr; int m_IncidenceAngleTableSize = 0; + CPL_DISALLOW_COPY_ASSIGN(RCMDataset) + protected: virtual int CloseDependentDatasets() override; @@ -147,6 +149,8 @@ class RCMRasterBand final : public GDALPamRasterBand bool isOneFilePerPol = false; bool isNITF = false; + CPL_DISALLOW_COPY_ASSIGN(RCMRasterBand) + public: RCMRasterBand(RCMDataset *poDSIn, int nBandIn, GDALDataType eDataTypeIn, const char *pszPole, GDALDataset *poBandFile, @@ -219,6 +223,8 @@ class RCMCalibRasterBand final : public GDALPamRasterBand void ReadLUT(); void ReadNoiseLevels(); + CPL_DISALLOW_COPY_ASSIGN(RCMCalibRasterBand) + public: RCMCalibRasterBand(RCMDataset *poDataset, const char *pszPolarization, GDALDataType eType, GDALDataset *poBandDataset, diff --git a/frmts/rcm/rcmdrivercore.cpp b/frmts/rcm/rcmdrivercore.cpp index f20d947dca6f..9ffb0900485d 100644 --- a/frmts/rcm/rcmdrivercore.cpp +++ b/frmts/rcm/rcmdrivercore.cpp @@ -92,11 +92,9 @@ int RCMDatasetIdentify(GDALOpenInfo *poOpenInfo) return FALSE; /* The RCM schema location is rcm_prod_product.xsd */ - if (strstr((const char *)poOpenInfo->pabyHeader, "/rcm") == nullptr || - strstr((const char *)poOpenInfo->pabyHeader, "(poOpenInfo->pabyHeader); + return strstr(pszHeader, "/rcm") && strstr(pszHeader, " Date: Thu, 24 Oct 2024 20:22:44 +0200 Subject: [PATCH 08/54] RCM: remove unused code --- frmts/rcm/rcmdataset.cpp | 235 +-------------------------------------- frmts/rcm/rcmdataset.h | 50 --------- 2 files changed, 1 insertion(+), 284 deletions(-) diff --git a/frmts/rcm/rcmdataset.cpp b/frmts/rcm/rcmdataset.cpp index 6d404e56b2d9..dd3df0cdde26 100644 --- a/frmts/rcm/rcmdataset.cpp +++ b/frmts/rcm/rcmdataset.cpp @@ -258,74 +258,6 @@ RCMRasterBand::RCMRasterBand(RCMDataset *poDSIn, int nBandIn, } } -double RCMRasterBand::GetLUT(int) -{ - return std::numeric_limits::quiet_NaN(); -} - -int RCMRasterBand::GetLUTsize() -{ - return 0; -} - -const char *RCMRasterBand::GetLUTFilename() -{ - return nullptr; -} - -double RCMRasterBand::GetLUTOffset() -{ - return 0.0f; -} - -bool RCMRasterBand::IsComplex() -{ - if (this->m_eType == GDT_CInt16 || this->m_eType == GDT_CInt32 || - this->m_eType == GDT_CFloat32 || this->m_eType == GDT_CFloat64) - { - return true; - } - else - { - return false; - } -} - -bool RCMRasterBand::IsExistLUT() -{ - return false; -} - -eCalibration RCMRasterBand::GetCalibration() -{ - return this->m_eCalib; -} - -void RCMRasterBand::SetPartialLUT(int, int) -{ - // nothing to do -} - -double RCMRasterBand::GetNoiseLevels(int) -{ - return 0.0f; -} - -int RCMRasterBand::GetNoiseLevelsSize() -{ - return 0; -} - -const char *RCMRasterBand::GetNoiseLevelsFilename() -{ - return nullptr; -} - -bool RCMRasterBand::IsExistNoiseLevels() -{ - return false; -} - /************************************************************************/ /* RCMRasterBand() */ /************************************************************************/ @@ -672,7 +604,7 @@ RCMCalibRasterBand::RCMCalibRasterBand( RCMDataset *poDataset, const char *pszPolarization, GDALDataType eType, GDALDataset *poBandDataset, eCalibration eCalib, const char *pszLUT, const char *pszNoiseLevels, GDALDataType eOriginalType) - : m_eCalib(eCalib), m_poBandDataset(poBandDataset), m_eType(eType), + : m_eCalib(eCalib), m_poBandDataset(poBandDataset), m_eOriginalType(eOriginalType), m_pszLUTFile(VSIStrdup(pszLUT)), m_pszNoiseLevelsFile(VSIStrdup(pszNoiseLevels)) { @@ -683,8 +615,6 @@ RCMCalibRasterBand::RCMCalibRasterBand( SetMetadataItem("POLARIMETRIC_INTERP", pszPolarization); } - // this->eDataType = eType; - if ((eType == GDT_CInt16) || (eType == GDT_CFloat32)) this->eDataType = GDT_CFloat32; else @@ -697,169 +627,6 @@ RCMCalibRasterBand::RCMCalibRasterBand( ReadNoiseLevels(); } -double RCMCalibRasterBand::GetNoiseLevels(int pixel) -{ - return this->m_nfTableNoiseLevels[pixel]; -} - -int RCMCalibRasterBand::GetNoiseLevelsSize() -{ - return this->m_nTableNoiseLevelsSize; -} - -const char *RCMCalibRasterBand::GetNoiseLevelsFilename() -{ - return this->m_pszNoiseLevelsFile; -} - -bool RCMCalibRasterBand::IsExistNoiseLevels() -{ - if (this->m_nfTableNoiseLevels == nullptr || - this->m_pszNoiseLevelsFile == nullptr || - strlen(this->m_pszNoiseLevelsFile) == 0 || - this->m_nTableNoiseLevelsSize == 0) - { - return false; - } - else - { - return true; - } -} - -bool RCMCalibRasterBand::IsComplex() -{ - if (this->m_eType == GDT_CInt16 || this->m_eType == GDT_CInt32 || - this->m_eType == GDT_CFloat32 || this->m_eType == GDT_CFloat64) - { - return true; - } - else - { - return false; - } -} - -eCalibration RCMCalibRasterBand::GetCalibration() -{ - return this->m_eCalib; -} - -double RCMCalibRasterBand::GetLUT(int pixel) -{ - return this->m_nfTable[pixel]; -} - -int RCMCalibRasterBand::GetLUTsize() -{ - return this->m_nTableSize; -} - -const char *RCMCalibRasterBand::GetLUTFilename() -{ - return this->m_pszLUTFile; -} - -double RCMCalibRasterBand::GetLUTOffset() -{ - return this->m_nfOffset; -} - -bool RCMCalibRasterBand::IsExistLUT() -{ - if (this->m_nfTable == nullptr || this->m_pszLUTFile == nullptr || - strlen(this->m_pszLUTFile) == 0 || this->m_nTableSize == 0) - { - return false; - } - else - { - return true; - } -} - -void RCMCalibRasterBand::SetPartialLUT(int pixel_offset, int pixel_width) -{ - /* Alway start from 0 */ - if (pixel_offset < 0) - { - pixel_offset = 0; - } - - if (pixel_offset < this->GetLUTsize()) - { - /* Can only change if the starting pixel in the raster width range */ - if ((pixel_offset + pixel_width) > this->GetLUTsize() - 1) - { - /* Ya but the width is way too large based on the raster width range - when beginning from a different offset Recalculate the true relative - width - */ - pixel_width = this->GetLUTsize() - pixel_offset - 1; - } - - if (pixel_width > 0) - { - /* Prepare a buffer */ - double *nfTableBuffer = - static_cast(CPLMalloc(sizeof(double) * pixel_width)); - memset(nfTableBuffer, 0, sizeof(double) * pixel_width); - - /* Copy a range */ - int j = 0; - for (int i = pixel_offset; i < (pixel_offset + pixel_width); i++) - { - nfTableBuffer[j++] = this->GetLUT(i); - } - - const size_t nLen = - pixel_width * - max_space_for_string; // 12 max + space + 11 reserved - char *lut_gains = static_cast(CPLMalloc(nLen)); - memset(lut_gains, 0, nLen); - - for (int i = 0; i < pixel_width; i++) - { - char lut[max_space_for_string]; - // 2.390641e+02 %e Scientific annotation - snprintf(lut, sizeof(lut), "%e ", nfTableBuffer[i]); - strcat(lut_gains, lut); - } - - char bandNumber[12]; - snprintf(bandNumber, sizeof(bandNumber), "%d", this->GetBand()); - - poDS->SetMetadataItem( - CPLString("LUT_GAINS_").append(bandNumber).c_str(), lut_gains); - // Can free this because the function SetMetadataItem takes a copy - CPLFree(lut_gains); - - /* and Set new LUT size */ - char snum[12]; - snprintf(snum, sizeof(snum), "%d", pixel_width); - poDS->SetMetadataItem( - CPLString("LUT_SIZE_").append(bandNumber).c_str(), snum); - - /* Change the internal value now */ - /* Free the old gains value table, not valid anymore */ - CPLFree(this->m_nfTable); - - /* New gains value table size */ - this->m_nTableSize = pixel_width; - - /* Realoocate and have new gain values from the buffer */ - this->m_nfTable = reinterpret_cast( - CPLMalloc(sizeof(double) * this->m_nTableSize)); - memset(this->m_nfTable, 0, sizeof(double) * this->m_nTableSize); - memcpy(this->m_nfTable, nfTableBuffer, - sizeof(double) * this->m_nTableSize); - - /* Free our buffer */ - CPLFree(nfTableBuffer); - } - } -} - /************************************************************************/ /* ~RCMCalibRasterBand() */ /************************************************************************/ diff --git a/frmts/rcm/rcmdataset.h b/frmts/rcm/rcmdataset.h index 150acfd00346..3d54a4b9842d 100644 --- a/frmts/rcm/rcmdataset.h +++ b/frmts/rcm/rcmdataset.h @@ -131,7 +131,6 @@ class RCMRasterBand final : public GDALPamRasterBand GDALDataset *poBandFile = nullptr; RCMDataset *poRCMDataset = nullptr; GDALDataset *m_poBandDataset = nullptr; - GDALDataType m_eType = GDT_Unknown; /* data type of data being ingested */ double *m_nfTable = nullptr; int m_nTableSize = 0; @@ -160,30 +159,6 @@ class RCMRasterBand final : public GDALPamRasterBand virtual CPLErr IReadBlock(int, int, void *) override; - static bool IsExistLUT(); - - static double GetLUT(int pixel); - - static const char *GetLUTFilename(); - - static int GetLUTsize(); - - static double GetLUTOffset(); - - static void SetPartialLUT(int pixel_offset, int pixel_width); - - bool IsComplex(); - - static bool IsExistNoiseLevels(); - - static double GetNoiseLevels(int pixel); - - static const char *GetNoiseLevelsFilename(); - - static int GetNoiseLevelsSize(); - - eCalibration GetCalibration(); - static GDALDataset *Open(GDALOpenInfo *); }; @@ -201,7 +176,6 @@ class RCMCalibRasterBand final : public GDALPamRasterBand private: eCalibration m_eCalib = eCalibration::Uncalib; GDALDataset *m_poBandDataset = nullptr; - GDALDataType m_eType = GDT_Unknown; /* data type of data being ingested */ /* data type that used to be before transformation */ GDALDataType m_eOriginalType = GDT_Unknown; @@ -233,30 +207,6 @@ class RCMCalibRasterBand final : public GDALPamRasterBand ~RCMCalibRasterBand(); CPLErr IReadBlock(int nBlockXOff, int nBlockYOff, void *pImage) override; - - bool IsExistLUT(); - - double GetLUT(int pixel); - - const char *GetLUTFilename(); - - int GetLUTsize(); - - double GetLUTOffset(); - - void SetPartialLUT(int pixel_offset, int pixel_width); - - bool IsExistNoiseLevels(); - - double GetNoiseLevels(int pixel); - - const char *GetNoiseLevelsFilename(); - - int GetNoiseLevelsSize(); - - bool IsComplex(); - - eCalibration GetCalibration(); }; #endif /* ndef GDAL_RCM_H_INCLUDED */ From 32a318f532569814303d1943e684bcef17791aa4 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 24 Oct 2024 19:48:45 +0200 Subject: [PATCH 09/54] Add testing of RCM driver --- .../data/rcm/fake_VV_VH_GRD/imagery/VH.tif | Bin 0 -> 250 bytes .../data/rcm/fake_VV_VH_GRD/imagery/VV.tif | Bin 0 -> 250 bytes .../data/rcm/fake_VV_VH_GRD/manifest.safe | 4 + .../metadata/calibration/incidenceAngles.xml | 7 + .../metadata/calibration/lutBeta_VH.xml | 8 + .../metadata/calibration/lutBeta_VV.xml | 8 + .../metadata/calibration/lutGamma_VH.xml | 8 + .../metadata/calibration/lutGamma_VV.xml | 8 + .../metadata/calibration/lutSigma_VH.xml | 8 + .../metadata/calibration/lutSigma_VV.xml | 8 + .../metadata/calibration/noiseLevels_VH.xml | 264 ++++++++++++++++++ .../metadata/calibration/noiseLevels_VV.xml | 264 ++++++++++++++++++ .../rcm/fake_VV_VH_GRD/metadata/product.xml | 177 ++++++++++++ autotest/gdrivers/rcm.py | 127 +++++++++ 14 files changed, 891 insertions(+) create mode 100644 autotest/gdrivers/data/rcm/fake_VV_VH_GRD/imagery/VH.tif create mode 100644 autotest/gdrivers/data/rcm/fake_VV_VH_GRD/imagery/VV.tif create mode 100644 autotest/gdrivers/data/rcm/fake_VV_VH_GRD/manifest.safe create mode 100644 autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/incidenceAngles.xml create mode 100644 autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutBeta_VH.xml create mode 100644 autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutBeta_VV.xml create mode 100644 autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutGamma_VH.xml create mode 100644 autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutGamma_VV.xml create mode 100644 autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutSigma_VH.xml create mode 100644 autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutSigma_VV.xml create mode 100644 autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/noiseLevels_VH.xml create mode 100644 autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/noiseLevels_VV.xml create mode 100644 autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/product.xml create mode 100755 autotest/gdrivers/rcm.py diff --git a/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/imagery/VH.tif b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/imagery/VH.tif new file mode 100644 index 0000000000000000000000000000000000000000..0a4a9c098f4b73a6b25f6a0efad4e340fc427d32 GIT binary patch literal 250 zcmebD)MDUZU|`^8U|?isU<9&$y8@X|_Cp>91|}$504T={WrNhQA+ZG+Sr~YMV#k13 c6iHkR$_5!M4plP=s73}!O)wH0C`C5_07Lu*iU0rr literal 0 HcmV?d00001 diff --git a/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/imagery/VV.tif b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/imagery/VV.tif new file mode 100644 index 0000000000000000000000000000000000000000..0a4a9c098f4b73a6b25f6a0efad4e340fc427d32 GIT binary patch literal 250 zcmebD)MDUZU|`^8U|?isU<9&$y8@X|_Cp>91|}$504T={WrNhQA+ZG+Sr~YMV#k13 c6iHkR$_5!M4plP=s73}!O)wH0C`C5_07Lu*iU0rr literal 0 HcmV?d00001 diff --git a/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/manifest.safe b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/manifest.safe new file mode 100644 index 000000000000..ba158aceab12 --- /dev/null +++ b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/manifest.safe @@ -0,0 +1,4 @@ + + + + diff --git a/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/incidenceAngles.xml b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/incidenceAngles.xml new file mode 100644 index 000000000000..4bddbd3a0dde --- /dev/null +++ b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/incidenceAngles.xml @@ -0,0 +1,7 @@ + + + 0 + -23 + 1 + 0 + diff --git a/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutBeta_VH.xml b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutBeta_VH.xml new file mode 100644 index 000000000000..cd3967c9d87a --- /dev/null +++ b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutBeta_VH.xml @@ -0,0 +1,8 @@ + + + 10 + -17915 + 1 + 0 + 10 + diff --git a/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutBeta_VV.xml b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutBeta_VV.xml new file mode 100644 index 000000000000..cd3967c9d87a --- /dev/null +++ b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutBeta_VV.xml @@ -0,0 +1,8 @@ + + + 10 + -17915 + 1 + 0 + 10 + diff --git a/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutGamma_VH.xml b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutGamma_VH.xml new file mode 100644 index 000000000000..cd3967c9d87a --- /dev/null +++ b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutGamma_VH.xml @@ -0,0 +1,8 @@ + + + 10 + -17915 + 1 + 0 + 10 + diff --git a/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutGamma_VV.xml b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutGamma_VV.xml new file mode 100644 index 000000000000..cd3967c9d87a --- /dev/null +++ b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutGamma_VV.xml @@ -0,0 +1,8 @@ + + + 10 + -17915 + 1 + 0 + 10 + diff --git a/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutSigma_VH.xml b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutSigma_VH.xml new file mode 100644 index 000000000000..cd3967c9d87a --- /dev/null +++ b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutSigma_VH.xml @@ -0,0 +1,8 @@ + + + 10 + -17915 + 1 + 0 + 10 + diff --git a/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutSigma_VV.xml b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutSigma_VV.xml new file mode 100644 index 000000000000..cd3967c9d87a --- /dev/null +++ b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/lutSigma_VV.xml @@ -0,0 +1,8 @@ + + + 10 + -17915 + 1 + 0 + 10 + diff --git a/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/noiseLevels_VH.xml b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/noiseLevels_VH.xml new file mode 100644 index 000000000000..50f5d4426e23 --- /dev/null +++ b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/noiseLevels_VH.xml @@ -0,0 +1,264 @@ + + + + Beta Nought + 0 + -23 + 1 + 0 + + + Sigma Nought + 0 + -23 + 1 + 0 + + + Gamma + 0 + -23 + 1 + 0 + + + SC-5 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-6 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-7 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-8 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-9 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-10 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-11 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-12 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-5 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-6 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-7 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-8 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-9 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-10 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-11 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-12 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-5 + Gamma + 0 + -3 + 1 + 0 + + + SC-6 + Gamma + 0 + -3 + 1 + 0 + + + SC-7 + Gamma + 0 + -3 + 1 + 0 + + + SC-8 + Gamma + 0 + -3 + 1 + 0 + + + SC-9 + Gamma + 0 + -3 + 1 + 0 + + + SC-10 + Gamma + 0 + -3 + 1 + 0 + + + SC-11 + Gamma + 0 + -3 + 1 + 0 + + + SC-12 + Gamma + 0 + -3 + 1 + 0 + + + SC-5 + 1 + 1 + 0 + + + SC-6 + 1 + 1 + 0 + + + SC-7 + 1 + 1 + 0 + + + SC-8 + 1 + 1 + 0 + + + SC-9 + 1 + 1 + 0 + + + SC-10 + 1 + 1 + 0 + + + SC-11 + 1 + 1 + 0 + + + SC-12 + 1 + 1 + 0 + + diff --git a/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/noiseLevels_VV.xml b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/noiseLevels_VV.xml new file mode 100644 index 000000000000..50f5d4426e23 --- /dev/null +++ b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/calibration/noiseLevels_VV.xml @@ -0,0 +1,264 @@ + + + + Beta Nought + 0 + -23 + 1 + 0 + + + Sigma Nought + 0 + -23 + 1 + 0 + + + Gamma + 0 + -23 + 1 + 0 + + + SC-5 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-6 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-7 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-8 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-9 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-10 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-11 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-12 + Beta Nought + 0 + -3 + 1 + 0 + + + SC-5 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-6 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-7 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-8 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-9 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-10 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-11 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-12 + Sigma Nought + 0 + -3 + 1 + 0 + + + SC-5 + Gamma + 0 + -3 + 1 + 0 + + + SC-6 + Gamma + 0 + -3 + 1 + 0 + + + SC-7 + Gamma + 0 + -3 + 1 + 0 + + + SC-8 + Gamma + 0 + -3 + 1 + 0 + + + SC-9 + Gamma + 0 + -3 + 1 + 0 + + + SC-10 + Gamma + 0 + -3 + 1 + 0 + + + SC-11 + Gamma + 0 + -3 + 1 + 0 + + + SC-12 + Gamma + 0 + -3 + 1 + 0 + + + SC-5 + 1 + 1 + 0 + + + SC-6 + 1 + 1 + 0 + + + SC-7 + 1 + 1 + 0 + + + SC-8 + 1 + 1 + 0 + + + SC-9 + 1 + 1 + 0 + + + SC-10 + 1 + 1 + 0 + + + SC-11 + 1 + 1 + 0 + + + SC-12 + 1 + 1 + 0 + + diff --git a/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/product.xml b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/product.xml new file mode 100644 index 000000000000..a76707260d63 --- /dev/null +++ b/autotest/gdrivers/data/rcm/fake_VV_VH_GRD/metadata/product.xml @@ -0,0 +1,177 @@ + + + + productId + productAnnotation + productApplication + documentIdentifier + + Non classifié / Unclassified + false + specialHandlingInstructions + + + RCM-1 + SAR + Dual Co/Cross + downlinkSegmentId + inputDatasetFacilityId + Medium Resolution 50m + beamModeDefinitionId + beamModeVersion + beamModeMnemonic + rawDataStartTime + + Medium Resolution 50m + beams + VH VV + + + + + + Descending + Downlinked + true + orbitDataFileName + + + + + + + + GRD + VV VH + processingFacility + processingTime + softwareVersion + Expedited + + + Mixed + true + false + true + true + true + false + true + true + true + Nominal Chirp + 0.000000000000000e+00 + Adaptive Analysis + false + true + zeroDopplerTimeFirstLine + zeroDopplerTimeLastLine + 19633 + 19633 + 4 + rangeLookBandwidth + totalProcessedRangeBandwidth + true + + + + 1 + + + + + 600000 + + + + + + + GeoTIFF + BSQ + lutBeta_VV.xml + lutSigma_VV.xml + lutGamma_VV.xml + lutBeta_VH.xml + lutSigma_VH.xml + lutGamma_VH.xml + incidenceAngles.xml + noiseLevels_VV.xml + noiseLevels_VH.xml + + Magnitude Detected + Integer + 16 + sampledPixelSpacing + sampledLineSpacing + sampledPixelSpacingTime + sampledLineSpacingTime + Increasing + Decreasing + + + + WGS 1984 + 6.378137000000000e+06 + 6.356752314245179e+06 + 0.000000000000000e+00 0.000000000000000e+00 0.000000000000000e+00 + 200 + + + + + 1 + 2 + + + 1.5 + 2.5 + 3.5 + + + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + + + + + 1 + + ../imagery/VV.tif + ../imagery/VH.tif + 0 + 0 + 3297 + 17915 + incAngNearRng + incAngFarRng + slantRangeNearEdge + slantRangeFarEdge + mean_VV + mean_VH + pole_VV + pole_VH + + + + + + diff --git a/autotest/gdrivers/rcm.py b/autotest/gdrivers/rcm.py new file mode 100755 index 000000000000..142f18fd1e7f --- /dev/null +++ b/autotest/gdrivers/rcm.py @@ -0,0 +1,127 @@ +#!/usr/bin/env pytest +############################################################################### +# $Id$ +# +# Project: GDAL/OGR Test Suite +# Purpose: Test RCM driver +# Author: Even Rouault, +# +############################################################################### +# Copyright (c) 2024, Even Rouault +# +# SPDX-License-Identifier: MIT +############################################################################### + + +import pytest + +from osgeo import gdal + +pytestmark = pytest.mark.require_driver("RCM") + + +def test_rcm_open_from_root_dir(): + ds = gdal.Open("data/rcm/fake_VV_VH_GRD") + assert ds.GetDriver().ShortName == "RCM" + assert ds.RasterCount == 2 + + +def test_rcm_open_from_metadata_dir(): + ds = gdal.Open("data/rcm/fake_VV_VH_GRD/metadata") + assert ds.GetDriver().ShortName == "RCM" + assert ds.RasterCount == 2 + + +def test_rcm_open_from_product_xml(): + ds = gdal.Open("data/rcm/fake_VV_VH_GRD/metadata/product.xml") + assert ds.GetDriver().ShortName == "RCM" + assert ds.RasterCount == 2 + assert ds.RasterXSize == 17915 + assert ds.RasterYSize == 3297 + assert ds.GetRasterBand(1).DataType == gdal.GDT_UInt16 + assert ds.GetRasterBand(1).Checksum() == 0 + assert ds.GetRasterBand(1).GetMetadata() == {"POLARIMETRIC_INTERP": "VH"} + got_md = ds.GetMetadata() + new_got_md = {} + for k in got_md: + new_got_md[k] = got_md[k].replace("\\", "/") + assert new_got_md == { + "ACQUISITION_START_TIME": "rawDataStartTime", + "ACQUISITION_TYPE": "Medium Resolution 50m", + "BEAMS": "beams", + "BEAM_MODE": "Medium Resolution 50m", + "BEAM_MODE_DEFINITION_ID": "beamModeDefinitionId", + "BEAM_MODE_MNEMONIC": "beamModeMnemonic", + "BETA_NOUGHT_LUT": "calibration/lutBeta_VH.xml,calibration/lutBeta_VV.xml", + "BITS_PER_SAMPLE": "16", + "DATA_TYPE": "Integer", + "FACILITY_IDENTIFIER": "inputDatasetFacilityId", + "FAR_RANGE_INCIDENCE_ANGLE": "incAngFarRng", + "FIRST_LINE_TIME": "zeroDopplerTimeFirstLine", + "GAMMA_LUT": "calibration/lutGamma_VH.xml,calibration/lutGamma_VV.xml", + "GEODETIC_TERRAIN_HEIGHT": "200", + "LAST_LINE_TIME": "zeroDopplerTimeLastLine", + "LINE_SPACING": "sampledLineSpacing", + "LINE_TIME_ORDERING": "Increasing", + "LUT_APPLIED": "Mixed", + "NEAR_RANGE_INCIDENCE_ANGLE": "incAngNearRng", + "ORBIT_DATA_FILE": "orbitDataFileName", + "ORBIT_DATA_SOURCE": "Downlinked", + "ORBIT_DIRECTION": "Descending", + "PER_POLARIZATION_SCALING": "true", + "PIXEL_SPACING": "sampledPixelSpacing", + "PIXEL_TIME_ORDERING": "Decreasing", + "POLARIZATIONS": "VH VV", + "POLARIZATION_DATA_MODE": "Dual Co/Cross", + "PROCESSING_FACILITY": "processingFacility", + "PROCESSING_TIME": "processingTime", + "PRODUCT_ID": "productId", + "PRODUCT_TYPE": "GRD", + "SAMPLED_LINE_SPACING_TIME": "sampledLineSpacingTime", + "SAMPLED_PIXEL_SPACING_TIME": "sampledPixelSpacingTime", + "SAMPLE_TYPE": "Magnitude Detected", + "SATELLITE_HEIGHT": "600000", + "SATELLITE_IDENTIFIER": "RCM-1", + "SECURITY_CLASSIFICATION": "Non classifié / Unclassified", + "SENSOR_IDENTIFIER": "SAR", + "SIGMA_NOUGHT_LUT": "calibration/lutSigma_VH.xml,calibration/lutSigma_VV.xml", + "SLANT_RANGE_FAR_EDGE": "slantRangeFarEdge", + "SLANT_RANGE_NEAR_EDGE": "slantRangeNearEdge", + } + assert ds.GetMetadata("RPC") == { + "ERR_BIAS": "0", + "ERR_RAND": "0", + "HEIGHT_OFF": "0", + "HEIGHT_SCALE": "0", + "LAT_OFF": "0", + "LAT_SCALE": "0", + "LINE_DEN_COEFF": "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0", + "LINE_NUM_COEFF": "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0", + "LINE_OFF": "0", + "LINE_SCALE": "0", + "LONG_OFF": "0", + "LONG_SCALE": "0", + "SAMP_DEN_COEFF": "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0", + "SAMP_NUM_COEFF": "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0", + "SAMP_OFF": "0", + "SAMP_SCALE": "0", + } + assert ds.GetGCPSpatialRef().GetAuthorityCode(None) == "4326" + assert ds.GetGCPCount() == 1 + assert ds.GetGCPs()[0].GCPPixel == 2 + assert ds.GetGCPs()[0].GCPLine == 1 + assert ds.GetGCPs()[0].GCPX == 2.5 + assert ds.GetGCPs()[0].GCPY == 1.5 + assert ds.GetGCPs()[0].GCPZ == 3.5 + + +def test_rcm_open_subdatasets(): + + gdal.Open("RCM_CALIB:BETA0:data/rcm/fake_VV_VH_GRD/metadata/product.xml") + gdal.Open("RCM_CALIB:SIGMA0:data/rcm/fake_VV_VH_GRD/metadata/product.xml") + gdal.Open("RCM_CALIB:GAMMA:data/rcm/fake_VV_VH_GRD/metadata/product.xml") + gdal.Open("RCM_CALIB:UNCALIB:data/rcm/fake_VV_VH_GRD/metadata/product.xml") + with pytest.raises(Exception, match="Unsupported calibration type"): + gdal.Open("RCM_CALIB:unhandled:data/rcm/fake_VV_VH_GRD/metadata/product.xml") + with pytest.raises(Exception): + gdal.Open("RCM_CALIB:UNCALIB:i_do_not_exist/product.xml") From 4480d1b7377a4b130e43cd7d7ffc5a188e6890b5 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 23 Oct 2024 22:03:50 +0200 Subject: [PATCH 10/54] RS2: merge modifications coming from DRDC related to NITF complex datasets Merge changes likely coming from https://github.com/DND-DRDC-RDDC/OS_DriverMetadataExtractionCodes_RCM_C-band_SARdata/blob/5ff9896b67c38005e26a5895f42566238c53a72a/1%20-%20RCM%20C%2B%2B%20GDAL%20driver%20code/frmts/rs2/rs2dataset.cpp#L1649 and provided by Esri --- frmts/rs2/rs2dataset.cpp | 109 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 103 insertions(+), 6 deletions(-) diff --git a/frmts/rs2/rs2dataset.cpp b/frmts/rs2/rs2dataset.cpp index 87c6de6dbfcf..98acab3b33f7 100644 --- a/frmts/rs2/rs2dataset.cpp +++ b/frmts/rs2/rs2dataset.cpp @@ -7,6 +7,7 @@ ****************************************************************************** * Copyright (c) 2004, Frank Warmerdam * Copyright (c) 2009-2013, Even Rouault + * Copyright (c) 2020, Defence Research and Development Canada (DRDC) Ottawa Research Centre * * SPDX-License-Identifier: MIT ****************************************************************************/ @@ -38,6 +39,49 @@ static bool IsValidXMLFile(const char *pszPath, const char *pszLut) return psLut.get() != nullptr; } +// Check that the referenced dataset for each band has the +// correct data type and returns whether a 2 band I+Q dataset should +// be mapped onto a single complex band. +// Returns BANDERROR for error, STRAIGHT for 1:1 mapping, TWOBANDCOMPLEX for 2 +// bands -> 1 complex band +typedef enum +{ + BANDERROR, + STRAIGHT, + TWOBANDCOMPLEX +} BandMapping; + +static BandMapping GetBandFileMapping(GDALDataType eDataType, + GDALDataset *poBandFile) +{ + + GDALRasterBand *poBand1 = poBandFile->GetRasterBand(1); + GDALDataType eBandDataType1 = poBand1->GetRasterDataType(); + + // if there is one band and it has the same datatype, the band file gets + // passed straight through + if (poBandFile->GetRasterCount() == 1 && eDataType == eBandDataType1) + return STRAIGHT; + + // if the band file has 2 bands, they should represent I+Q + // and be a compatible data type + if (poBandFile->GetRasterCount() == 2 && GDALDataTypeIsComplex(eDataType)) + { + GDALRasterBand *band2 = poBandFile->GetRasterBand(2); + + if (eBandDataType1 != band2->GetRasterDataType()) + return BANDERROR; // both bands must be same datatype + + // check compatible types - there are 4 complex types in GDAL + if ((eDataType == GDT_CInt16 && eBandDataType1 == GDT_Int16) || + (eDataType == GDT_CInt32 && eBandDataType1 == GDT_Int32) || + (eDataType == GDT_CFloat32 && eBandDataType1 == GDT_Float32) || + (eDataType == GDT_CFloat64 && eBandDataType1 == GDT_Float64)) + return TWOBANDCOMPLEX; + } + return BANDERROR; // don't accept any other combinations +} + /************************************************************************/ /* ==================================================================== */ /* RS2Dataset */ @@ -93,11 +137,16 @@ class RS2Dataset final : public GDALPamDataset class RS2RasterBand final : public GDALPamRasterBand { - GDALDataset *poBandFile; + GDALDataset *poBandFile = nullptr; + + // 2 bands representing I+Q -> one complex band + // otherwise poBandFile is passed straight through + bool bIsTwoBandComplex = false; public: RS2RasterBand(RS2Dataset *poDSIn, GDALDataType eDataTypeIn, - const char *pszPole, GDALDataset *poBandFile); + const char *pszPole, GDALDataset *poBandFile, + bool bTwoBandComplex = false); virtual ~RS2RasterBand(); virtual CPLErr IReadBlock(int, int, void *) override; @@ -110,7 +159,8 @@ class RS2RasterBand final : public GDALPamRasterBand /************************************************************************/ RS2RasterBand::RS2RasterBand(RS2Dataset *poDSIn, GDALDataType eDataTypeIn, - const char *pszPole, GDALDataset *poBandFileIn) + const char *pszPole, GDALDataset *poBandFileIn, + bool bTwoBandComplex) : poBandFile(poBandFileIn) { poDS = poDSIn; @@ -121,6 +171,8 @@ RS2RasterBand::RS2RasterBand(RS2Dataset *poDSIn, GDALDataType eDataTypeIn, eDataType = eDataTypeIn; + bIsTwoBandComplex = bTwoBandComplex; + if (*pszPole != '\0') SetMetadataItem("POLARIMETRIC_INTERP", pszPole); } @@ -400,6 +452,31 @@ CPLErr RS2CalibRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, } CPLFree(pnImageTmp); } + // If the underlying file is NITF CFloat32 + else if (eDataType == GDT_CFloat32 && + m_poBandDataset->GetRasterCount() == 1) + { + eErr = m_poBandDataset->RasterIO( + GF_Read, nBlockXOff * nBlockXSize, nBlockYOff * nBlockYSize, + nBlockXSize, nRequestYSize, pImage, nBlockXSize, nRequestYSize, + GDT_CFloat32, 1, nullptr, 2 * static_cast(sizeof(float)), + nBlockXSize * 2 * static_cast(sizeof(float)), 0, nullptr); + + /* calibrate the complex values */ + for (int i = 0; i < nBlockYSize; i++) + { + for (int j = 0; j < nBlockXSize; j++) + { + /* calculate pixel offset in memory*/ + const int nPixOff = 2 * (i * nBlockXSize + j); + + reinterpret_cast(pImage)[nPixOff] /= + m_nfTable[nBlockXOff * nBlockXSize + j]; + reinterpret_cast(pImage)[nPixOff + 1] /= + m_nfTable[nBlockXOff * nBlockXSize + j]; + } + } + } else if (m_eType == GDT_UInt16) { /* read in detected values */ @@ -738,6 +815,10 @@ GDALDataset *RS2Dataset::Open(GDALOpenInfo *poOpenInfo) GDALDataType eDataType; if (nBitsPerSample == 16 && EQUAL(pszDataType, "Complex")) eDataType = GDT_CInt16; + else if (nBitsPerSample == 32 && + EQUAL(pszDataType, + "Complex")) // NITF datasets can come in this configuration + eDataType = GDT_CFloat32; else if (nBitsPerSample == 16 && STARTS_WITH_CI(pszDataType, "Mag")) eDataType = GDT_UInt16; else if (nBitsPerSample == 8 && STARTS_WITH_CI(pszDataType, "Mag")) @@ -888,6 +969,16 @@ GDALDataset *RS2Dataset::Open(GDALOpenInfo *poOpenInfo) continue; } + /* Some CFloat32 NITF files have nBitsPerSample incorrectly reported */ + /* as 16, and get misinterpreted as CInt16. Check the underlying NITF + */ + /* and override if this is the case. */ + if (poBandFile->GetRasterBand(1)->GetRasterDataType() == GDT_CFloat32) + eDataType = GDT_CFloat32; + + BandMapping b = GetBandFileMapping(eDataType, poBandFile); + const bool twoBandComplex = b == TWOBANDCOMPLEX; + poDS->papszExtraFiles = CSLAddString(poDS->papszExtraFiles, pszFullname); @@ -899,8 +990,8 @@ GDALDataset *RS2Dataset::Open(GDALOpenInfo *poOpenInfo) if (eCalib == None || eCalib == Uncalib) { RS2RasterBand *poBand = new RS2RasterBand( - poDS, eDataType, CPLGetXMLValue(psNode, "pole", ""), - poBandFile); + poDS, eDataType, CPLGetXMLValue(psNode, "pole", ""), poBandFile, + twoBandComplex); poDS->SetBand(poDS->GetRasterCount() + 1, poBand); } @@ -1155,6 +1246,11 @@ GDALDataset *RS2Dataset::Open(GDALOpenInfo *poOpenInfo) oLL.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); oPrj.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + const char *pszGeodeticTerrainHeight = + CPLGetXMLValue(psEllipsoid, "geodeticTerrainHeight", "UNK"); + poDS->SetMetadataItem("GEODETIC_TERRAIN_HEIGHT", + pszGeodeticTerrainHeight); + const char *pszEllipsoidName = CPLGetXMLValue(psEllipsoid, "ellipsoidName", ""); double minor_axis = @@ -1171,7 +1267,8 @@ GDALDataset *RS2Dataset::Open(GDALOpenInfo *poOpenInfo) oLL.SetWellKnownGeogCS("WGS84"); oPrj.SetWellKnownGeogCS("WGS84"); } - else if (EQUAL(pszEllipsoidName, "WGS84")) + else if (EQUAL(pszEllipsoidName, "WGS84") || + EQUAL(pszEllipsoidName, "WGS 1984")) { oLL.SetWellKnownGeogCS("WGS84"); oPrj.SetWellKnownGeogCS("WGS84"); From 352147b7bc51821cf47c6b55ba53feef75fb3f37 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 25 Oct 2024 12:15:02 +0200 Subject: [PATCH 11/54] RS2: add a test against a real remote dataset --- autotest/gdrivers/rs2.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/autotest/gdrivers/rs2.py b/autotest/gdrivers/rs2.py index 965d75be7031..fa034343d36f 100755 --- a/autotest/gdrivers/rs2.py +++ b/autotest/gdrivers/rs2.py @@ -67,3 +67,15 @@ def test_rs2_3(): "SAMP_SCALE": "pixelScale", } assert got_rpc == expected_rpc + + +@pytest.mark.require_curl +def test_rs2_open_real_dataset(): + remote_file = "https://donnees-data.asc-csa.gc.ca/users/OpenData_DonneesOuvertes/pub/RADARSAT-2/RS2_OK103540_PK929658_DK864570_SLA12_20190317_110012_HH_SLC/product.xml" + + if gdaltest.gdalurlopen(remote_file) is None: + pytest.skip(f"Could not read from {remote_file}") + + ds = gdal.Open("/vsicurl/" + remote_file) + assert ds.GetDriver().ShortName == "RS2" + assert ds.GetRasterBand(1).DataType == gdal.GDT_CInt16 From ae09ca441cdcfcd96e55e2de2e78a0654881b3b1 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 25 Oct 2024 12:52:27 +0200 Subject: [PATCH 12/54] RCM: revise path composition to work on Windows with /vsicurl/ --- autotest/gdrivers/rcm.py | 6 +- frmts/rcm/rcmdataset.cpp | 117 ++++++++++++++++++------------------ frmts/rcm/rcmdrivercore.cpp | 5 +- frmts/rcm/rcmdrivercore.h | 63 +------------------ 4 files changed, 64 insertions(+), 127 deletions(-) diff --git a/autotest/gdrivers/rcm.py b/autotest/gdrivers/rcm.py index 142f18fd1e7f..83046cfd62e7 100755 --- a/autotest/gdrivers/rcm.py +++ b/autotest/gdrivers/rcm.py @@ -52,13 +52,13 @@ def test_rcm_open_from_product_xml(): "BEAM_MODE": "Medium Resolution 50m", "BEAM_MODE_DEFINITION_ID": "beamModeDefinitionId", "BEAM_MODE_MNEMONIC": "beamModeMnemonic", - "BETA_NOUGHT_LUT": "calibration/lutBeta_VH.xml,calibration/lutBeta_VV.xml", + "BETA_NOUGHT_LUT": "data/rcm/fake_VV_VH_GRD/metadata/calibration/lutBeta_VH.xml,data/rcm/fake_VV_VH_GRD/metadata/calibration/lutBeta_VV.xml", "BITS_PER_SAMPLE": "16", "DATA_TYPE": "Integer", "FACILITY_IDENTIFIER": "inputDatasetFacilityId", "FAR_RANGE_INCIDENCE_ANGLE": "incAngFarRng", "FIRST_LINE_TIME": "zeroDopplerTimeFirstLine", - "GAMMA_LUT": "calibration/lutGamma_VH.xml,calibration/lutGamma_VV.xml", + "GAMMA_LUT": "data/rcm/fake_VV_VH_GRD/metadata/calibration/lutGamma_VH.xml,data/rcm/fake_VV_VH_GRD/metadata/calibration/lutGamma_VV.xml", "GEODETIC_TERRAIN_HEIGHT": "200", "LAST_LINE_TIME": "zeroDopplerTimeLastLine", "LINE_SPACING": "sampledLineSpacing", @@ -84,7 +84,7 @@ def test_rcm_open_from_product_xml(): "SATELLITE_IDENTIFIER": "RCM-1", "SECURITY_CLASSIFICATION": "Non classifié / Unclassified", "SENSOR_IDENTIFIER": "SAR", - "SIGMA_NOUGHT_LUT": "calibration/lutSigma_VH.xml,calibration/lutSigma_VV.xml", + "SIGMA_NOUGHT_LUT": "data/rcm/fake_VV_VH_GRD/metadata/calibration/lutSigma_VH.xml,data/rcm/fake_VV_VH_GRD/metadata/calibration/lutSigma_VV.xml", "SLANT_RANGE_FAR_EDGE": "slantRangeFarEdge", "SLANT_RANGE_NEAR_EDGE": "slantRangeNearEdge", } diff --git a/frmts/rcm/rcmdataset.cpp b/frmts/rcm/rcmdataset.cpp index dd3df0cdde26..087521b98345 100644 --- a/frmts/rcm/rcmdataset.cpp +++ b/frmts/rcm/rcmdataset.cpp @@ -32,18 +32,43 @@ constexpr int max_space_for_string = 32; * Level files */ constexpr const char *CALIBRATION_FOLDER = "calibration"; +/*** Function to format calibration for unique identification for Layer Name + * ***/ +/* + * RCM_CALIB : { SIGMA0 | GAMMA0 | BETA0 | UNCALIB } : product.xml full path + */ +inline CPLString FormatCalibration(const char *pszCalibName, + const char *pszFilename) +{ + CPLString ptr; + + // Always begin by the layer calibration name + ptr.append(szLayerCalibration); + + // A separator is needed before concat calibration name + ptr += chLayerSeparator; + // Add calibration name + ptr.append(pszCalibName); + + // A separator is needed before concat full filename name + ptr += chLayerSeparator; + // Add full filename name + ptr.append(pszFilename); + + /* return calibration format */ + return ptr; +} + /*** Function to test for valid LUT files ***/ -static bool IsValidXMLFile(const char *pszPath, const char *pszLut) +static bool IsValidXMLFile(const char *pszPath) { /* Return true for valid xml file, false otherwise */ - const std::string osLutFile = CPLFormFilename(pszPath, pszLut, nullptr); - - CPLXMLTreeCloser psLut(CPLParseXMLFile(osLutFile.c_str())); + CPLXMLTreeCloser psLut(CPLParseXMLFile(pszPath)); if (psLut.get() == nullptr) { CPLError(CE_Failure, CPLE_OpenFailed, - "ERROR: Failed to open the LUT file %s", osLutFile.c_str()); + "ERROR: Failed to open the LUT file %s", pszPath); } return psLut.get() != nullptr; @@ -1043,9 +1068,8 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) const char *pszFilename = poOpenInfo->pszFilename; eCalibration eCalib = None; - CPLString calibrationFormat(FormatCalibration(nullptr, nullptr)); - - if (STARTS_WITH_CI(pszFilename, calibrationFormat)) + if (STARTS_WITH_CI(pszFilename, szLayerCalibration) && + pszFilename[strlen(szLayerCalibration)] == chLayerSeparator) { // The calibration name and filename begins after the hard coded layer // name @@ -1544,17 +1568,13 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) if (pszIncidenceAngleFileName != nullptr) { - CPLString osIncidenceAnglePath; - osIncidenceAnglePath.append(CALIBRATION_FOLDER); - osIncidenceAnglePath.append(szPathSeparator); - osIncidenceAnglePath.append(pszIncidenceAngleFileName); + CPLString osIncidenceAngleFilePath = CPLFormFilename( + CPLFormFilename(osPath, CALIBRATION_FOLDER, nullptr), + pszIncidenceAngleFileName, nullptr); /* Check if the file exist */ - if (IsValidXMLFile(osPath, osIncidenceAnglePath)) + if (IsValidXMLFile(osIncidenceAngleFilePath)) { - CPLString osIncidenceAngleFilePath = - CPLFormFilename(osPath, osIncidenceAnglePath, nullptr); - CPLXMLTreeCloser psIncidenceAngle( CPLParseXMLFile(osIncidenceAngleFilePath)); @@ -1654,12 +1674,11 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) /* called 'calibration' from the 'metadata' folder */ /* -------------------------------------------------------------------- */ - CPLString oNoiseLevelPath; - oNoiseLevelPath.append(CALIBRATION_FOLDER); - oNoiseLevelPath.append(szPathSeparator); - oNoiseLevelPath.append(pszNoiseLevelFile); - if (IsValidXMLFile(osPath, oNoiseLevelPath)) + const CPLString oNoiseLevelPath = CPLFormFilename( + CPLFormFilename(osPath, CALIBRATION_FOLDER, nullptr), + pszNoiseLevelFile, nullptr); + if (IsValidXMLFile(oNoiseLevelPath)) { osNoiseLevelsValues = oNoiseLevelPath; } @@ -1712,16 +1731,12 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) /* called 'calibration' from the 'metadata' folder */ /* -------------------------------------------------------------------- */ - CPLString osCalibPath; - osCalibPath.append(CALIBRATION_FOLDER); - osCalibPath.append(szPathSeparator); - osCalibPath.append(pszLUTFile); - - CPLString osLUTFilePath = - CPLFormFilename(osPath, osCalibPath, nullptr); + const CPLString osLUTFilePath = CPLFormFilename( + CPLFormFilename(osPath, CALIBRATION_FOLDER, nullptr), + pszLUTFile, nullptr); if (EQUAL(pszLUTType, "Beta Nought") && - IsValidXMLFile(osPath, osCalibPath)) + IsValidXMLFile(osLUTFilePath)) { poDS->papszExtraFiles = CSLAddString(poDS->papszExtraFiles, osLUTFilePath); @@ -1729,13 +1744,13 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) CPLString pszBuf( FormatCalibration(szBETA0, osMDFilename.c_str())); CPLFree(pszBeta0LUT); - pszBeta0LUT = VSIStrdup(osCalibPath); + pszBeta0LUT = VSIStrdup(osLUTFilePath); const char *oldLut = poDS->GetMetadataItem("BETA_NOUGHT_LUT"); if (oldLut == nullptr) { - poDS->SetMetadataItem("BETA_NOUGHT_LUT", osCalibPath); + poDS->SetMetadataItem("BETA_NOUGHT_LUT", osLUTFilePath); } else { @@ -1747,7 +1762,7 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) '\0'; /* Just initialize the first byte */ strcat(ptrConcatLut, oldLut); strcat(ptrConcatLut, ","); - strcat(ptrConcatLut, osCalibPath); + strcat(ptrConcatLut, osLUTFilePath); poDS->SetMetadataItem("BETA_NOUGHT_LUT", ptrConcatLut); CPLFree(ptrConcatLut); } @@ -1759,7 +1774,7 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) "Beta Nought calibrated"); } else if (EQUAL(pszLUTType, "Sigma Nought") && - IsValidXMLFile(osPath, osCalibPath)) + IsValidXMLFile(osLUTFilePath)) { poDS->papszExtraFiles = CSLAddString(poDS->papszExtraFiles, osLUTFilePath); @@ -1767,13 +1782,14 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) CPLString pszBuf( FormatCalibration(szSIGMA0, osMDFilename.c_str())); CPLFree(pszSigma0LUT); - pszSigma0LUT = VSIStrdup(osCalibPath); + pszSigma0LUT = VSIStrdup(osLUTFilePath); const char *oldLut = poDS->GetMetadataItem("SIGMA_NOUGHT_LUT"); if (oldLut == nullptr) { - poDS->SetMetadataItem("SIGMA_NOUGHT_LUT", osCalibPath); + poDS->SetMetadataItem("SIGMA_NOUGHT_LUT", + osLUTFilePath); } else { @@ -1785,7 +1801,7 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) '\0'; /* Just initialize the first byte */ strcat(ptrConcatLut, oldLut); strcat(ptrConcatLut, ","); - strcat(ptrConcatLut, osCalibPath); + strcat(ptrConcatLut, osLUTFilePath); poDS->SetMetadataItem("SIGMA_NOUGHT_LUT", ptrConcatLut); CPLFree(ptrConcatLut); } @@ -1797,7 +1813,7 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) "Sigma Nought calibrated"); } else if (EQUAL(pszLUTType, "Gamma") && - IsValidXMLFile(osPath, osCalibPath)) + IsValidXMLFile(osLUTFilePath)) { poDS->papszExtraFiles = CSLAddString(poDS->papszExtraFiles, osLUTFilePath); @@ -1805,12 +1821,12 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) CPLString pszBuf( FormatCalibration(szGAMMA, osMDFilename.c_str())); CPLFree(pszGammaLUT); - pszGammaLUT = VSIStrdup(osCalibPath); + pszGammaLUT = VSIStrdup(osLUTFilePath); const char *oldLut = poDS->GetMetadataItem("GAMMA_LUT"); if (oldLut == nullptr) { - poDS->SetMetadataItem("GAMMA_LUT", osCalibPath); + poDS->SetMetadataItem("GAMMA_LUT", osLUTFilePath); } else { @@ -1822,7 +1838,7 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) '\0'; /* Just initialize the first byte */ strcat(ptrConcatLut, oldLut); strcat(ptrConcatLut, ","); - strcat(ptrConcatLut, osCalibPath); + strcat(ptrConcatLut, osLUTFilePath); poDS->SetMetadataItem("GAMMA_LUT", ptrConcatLut); CPLFree(ptrConcatLut); } @@ -1865,32 +1881,13 @@ GDALDataset *RCMDataset::Open(GDALOpenInfo *poOpenInfo) pszBasedFilename = imageBandFileList[bandPositionIndex]; } - int nLength = static_cast(strlen(pszBasedFilename)); - char *pszBasename = - static_cast(CPLCalloc(nLength + 1, sizeof(char))); - - /* -------------------------------------------------------------------- - */ - /* Change folder separator if given full path in wrong OS */ - /* -------------------------------------------------------------------- - */ - for (int i = 0; i < nLength; i++) - { - if (pszBasedFilename[i] == cOppositePathSeparator) - pszBasename[i] = cPathSeparator; - else - pszBasename[i] = pszBasedFilename[i]; - } - /* -------------------------------------------------------------------- */ /* Form full filename (path of product.xml + basename). */ /* -------------------------------------------------------------------- */ char *pszFullname = - CPLStrdup(CPLFormFilename(osPath, pszBasename, nullptr)); - - CPLFree(pszBasename); + CPLStrdup(CPLFormFilename(osPath, pszBasedFilename, nullptr)); /* -------------------------------------------------------------------- */ diff --git a/frmts/rcm/rcmdrivercore.cpp b/frmts/rcm/rcmdrivercore.cpp index 9ffb0900485d..e2a1fd436226 100644 --- a/frmts/rcm/rcmdrivercore.cpp +++ b/frmts/rcm/rcmdrivercore.cpp @@ -18,9 +18,8 @@ int RCMDatasetIdentify(GDALOpenInfo *poOpenInfo) { /* Check for the case where we're trying to read the calibrated data: */ - CPLString calibrationFormat = FormatCalibration(nullptr, nullptr); - - if (STARTS_WITH_CI(poOpenInfo->pszFilename, calibrationFormat)) + if (STARTS_WITH_CI(poOpenInfo->pszFilename, szLayerCalibration) && + poOpenInfo->pszFilename[strlen(szLayerCalibration)] == chLayerSeparator) { return TRUE; } diff --git a/frmts/rcm/rcmdrivercore.h b/frmts/rcm/rcmdrivercore.h index 6dfe764e5dd6..a640d10216e8 100644 --- a/frmts/rcm/rcmdrivercore.h +++ b/frmts/rcm/rcmdrivercore.h @@ -21,73 +21,14 @@ // Should be size of larged possible filename. constexpr int CPL_PATH_BUF_SIZE = 2048; constexpr char szLayerCalibration[] = "RCM_CALIB"; -constexpr char szLayerSeparator[] = ":"; +constexpr char chLayerSeparator = ':'; constexpr char szSIGMA0[] = "SIGMA0"; constexpr char szGAMMA[] = "GAMMA"; constexpr char szBETA0[] = "BETA0"; constexpr char szUNCALIB[] = "UNCALIB"; -constexpr char szPathSeparator[] = -#ifdef _WIN32 /* Defined if Win32 and Win64 */ - "\\"; -#else - "/"; -#endif -constexpr char cPathSeparator = -#ifdef _WIN32 /* Defined if Win32 and Win64 */ - '\\'; -#else - '/'; -#endif -constexpr char cOppositePathSeparator = -#ifdef _WIN32 /* Defined if Win32 and Win64 */ - '/'; -#else - '\\'; -#endif constexpr const char *RCM_DRIVER_NAME = "RCM"; -/*** Function to format calibration for unique identification for Layer Name - * ***/ -/* - * RCM_CALIB : { SIGMA0 | GAMMA0 | BETA0 | UNCALIB } : product.xml full path - */ -inline CPLString FormatCalibration(const char *pszCalibName, - const char *pszFilename) -{ - CPLString ptr; - - // Always begin by the layer calibration name - ptr.append(szLayerCalibration); - - if (pszCalibName != nullptr || pszFilename != nullptr) - { - if (pszCalibName != nullptr) - { - // A separator is needed before concat calibration name - ptr.append(szLayerSeparator); - // Add calibration name - ptr.append(pszCalibName); - } - - if (pszFilename != nullptr) - { - // A separator is needed before concat full filename name - ptr.append(szLayerSeparator); - // Add full filename name - ptr.append(pszFilename); - } - } - else - { - // Always add a separator even though there are no name to concat - ptr.append(szLayerSeparator); - } - - /* return calibration format */ - return ptr; -} - /*** Function to concat 'metadata' with a folder separator with the filename * 'product.xml' ***/ /* @@ -98,7 +39,7 @@ inline CPLString GetMetadataProduct() // Always begin by the layer calibration name CPLString ptr; ptr.append("metadata"); - ptr.append(szPathSeparator); + ptr.append("/"); ptr.append("product.xml"); /* return metadata product filename */ From 4f66cc7302c6b3fe7339c6d9a8a81fc3e90f754a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 25 Oct 2024 12:11:55 +0200 Subject: [PATCH 13/54] RCM: add a test against a real remote dataset --- autotest/gdrivers/rcm.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/autotest/gdrivers/rcm.py b/autotest/gdrivers/rcm.py index 83046cfd62e7..556e446bddbe 100755 --- a/autotest/gdrivers/rcm.py +++ b/autotest/gdrivers/rcm.py @@ -13,6 +13,7 @@ ############################################################################### +import gdaltest import pytest from osgeo import gdal @@ -125,3 +126,15 @@ def test_rcm_open_subdatasets(): gdal.Open("RCM_CALIB:unhandled:data/rcm/fake_VV_VH_GRD/metadata/product.xml") with pytest.raises(Exception): gdal.Open("RCM_CALIB:UNCALIB:i_do_not_exist/product.xml") + + +@pytest.mark.require_curl +def test_rcm_open_real_dataset(): + remote_file = "https://donnees-data.asc-csa.gc.ca/users/OpenData_DonneesOuvertes/pub/RCM/Antarctica/RCM3_OK2120467_PK2120468_3_SC30MCPB_20200124_083635_CH_CV_MLC/metadata/product.xml" + + if gdaltest.gdalurlopen(remote_file) is None: + pytest.skip(f"Could not read from {remote_file}") + + ds = gdal.Open("/vsicurl/" + remote_file) + assert ds.GetDriver().ShortName == "RCM" + assert ds.GetRasterBand(1).DataType == gdal.GDT_Float32 From 62e2c6ef00b763e518627a8e9539336ade39d962 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 25 Oct 2024 18:13:34 +0200 Subject: [PATCH 14/54] /vsigs/: make GetFileMetadata('/vsigs/bucket', NULL, NULL) work if using OAuth2 auth --- autotest/gcore/vsigs.py | 101 +++++++++++++++++++++++++++++++++ port/cpl_vsil.cpp | 23 ++++++-- port/cpl_vsil_gs.cpp | 121 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 238 insertions(+), 7 deletions(-) diff --git a/autotest/gcore/vsigs.py b/autotest/gcore/vsigs.py index ebf31b07dbdf..dd78ab9cffc2 100755 --- a/autotest/gcore/vsigs.py +++ b/autotest/gcore/vsigs.py @@ -692,6 +692,107 @@ def test_vsigs_headers(gs_test_config, webserver_port): ) +############################################################################### +# Test GetFileMetadata() on root of bucket with OAuth2 + + +@gdaltest.enable_exceptions() +def test_vsigs_GetFileMetadatabucket_root_oauth2( + gs_test_config, webserver_port, tmp_vsimem +): + + gdal.VSICurlClearCache() + + service_account_filename = str(tmp_vsimem / "service_account.json") + gdal.FileFromMemBuffer( + service_account_filename, + """{ + "private_key": "-----BEGIN PRIVATE KEY-----\nMIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk\nVNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm\ni2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ\nDVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy\nTabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I\n2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd\nDrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+\nbpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg\nWz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3\n4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5\nKvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj\nj+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI\nyzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq\ngwE6fxOLyJDxuWRf\n-----END PRIVATE KEY-----\n", + "client_email": "CLIENT_EMAIL", + "type": "service_account" + }""", + ) + + gdal.SetPathSpecificOption( + "/vsigs/gs_fake_bucket", + "GOOGLE_APPLICATION_CREDENTIALS", + service_account_filename, + ) + + try: + with gdaltest.config_options( + { + "GO2A_AUD": "http://localhost:%d/oauth2/v4/token" % webserver_port, + "GOA2_NOW": "123456", + }, + thread_local=False, + ): + + gdal.VSICurlClearCache() + + handler = webserver.SequentialHandler() + + def method(request): + request.send_response(200) + request.send_header("Content-type", "text/plain") + content = """{ + "access_token" : "ACCESS_TOKEN", + "token_type" : "Bearer", + "expires_in" : 3600, + }""" + request.send_header("Content-Length", len(content)) + request.end_headers() + request.wfile.write(content.encode("ascii")) + + handler.add("POST", "/oauth2/v4/token", custom_method=method) + + handler.add( + "GET", + "/storage/v1/b/gs_fake_bucket", + 200, + {"Content-type": "application/json"}, + '{"foo":"bar"}', + ) + try: + with webserver.install_http_handler(handler): + md = gdal.GetFileMetadata("/vsigs/gs_fake_bucket/", None) + + except Exception: + if ( + gdal.GetLastErrorMsg().find("CPLRSASHA256Sign() not implemented") + >= 0 + ): + pytest.skip("CPLRSASHA256Sign() not implemented") + + assert md == {"foo": "bar"} + finally: + gdal.SetPathSpecificOption( + "/vsigs/gs_fake_bucket", "GOOGLE_APPLICATION_CREDENTIALS", None + ) + + +############################################################################### +# Test GetFileMetadata() on root of bucket with non-OAuth2 (does not work) + + +def test_vsigs_GetFileMetadatabucket_root_not_oauth2(gs_test_config, webserver_port): + + gdal.VSICurlClearCache() + + with gdaltest.config_options( + { + "GS_SECRET_ACCESS_KEY": "GS_SECRET_ACCESS_KEY", + "GS_ACCESS_KEY_ID": "GS_ACCESS_KEY_ID", + }, + thread_local=False, + ): + + handler = webserver.SequentialHandler() + with webserver.install_http_handler(handler): + md = gdal.GetFileMetadata("/vsigs/gs_fake_bucket/", None) + assert md == {} + + ############################################################################### # Read credentials with OAuth2 refresh_token diff --git a/port/cpl_vsil.cpp b/port/cpl_vsil.cpp index 171fac28eb13..df8cd588cde7 100644 --- a/port/cpl_vsil.cpp +++ b/port/cpl_vsil.cpp @@ -1231,16 +1231,27 @@ int VSIStatExL(const char *pszFilename, VSIStatBufL *psStatBuf, int nFlags) * Implemented currently only for network-like filesystems, or starting * with GDAL 3.7 for /vsizip/ * + * Starting with GDAL 3.11, calling it with pszFilename being the root of a + * /vsigs/ bucket and pszDomain == nullptr, and when authenticated through + * OAuth2, will result in returning the result of a "Buckets: get" + * operation (https://cloud.google.com/storage/docs/json_api/v1/buckets/get), + * with the keys of the top-level JSON document as keys of the key=value pairs + * returned by this function. + * * @param pszFilename the path of the filesystem object to be queried. * UTF-8 encoded. * @param pszDomain Metadata domain to query. Depends on the file system. - * The following are supported: + * The following ones are supported: *
    *
  • HEADERS: to get HTTP headers for network-like filesystems (/vsicurl/, - * /vsis3/, /vsgis/, etc)
  • TAGS:
    • /vsis3/: to get S3 Object - * tagging information
    • /vsiaz/: to get blob tags. Refer to - * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-tags
      • - *
    + * /vsis3/, /vsgis/, etc)
    • + *
  • TAGS: + *
      + *
    • /vsis3/: to get S3 Object tagging information
      • + *
    • /vsiaz/: to get blob tags. Refer to + * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-tags + *
      • + *
    *
    • *
  • STATUS: specific to /vsiadls/: returns all system defined properties for * a path (seems in practice to be a subset of HEADERS)
  • ACL: specific @@ -1248,7 +1259,7 @@ int VSIStatExL(const char *pszFilename, VSIStatBufL *psStatBuf, int nFlags) * /vsigs/, a single XML=xml_content string is returned. Refer to * https://cloud.google.com/storage/docs/xml-api/get-object-acls *
    • - *
  • METADATA: specific to /vsiaz/: to set blob metadata. Refer to + *
  • METADATA: specific to /vsiaz/: to get blob metadata. Refer to * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-metadata. * Note: this will be a subset of what pszDomain=HEADERS returns
    • *
  • ZIP: specific to /vsizip/: to obtain ZIP specific metadata, in particular diff --git a/port/cpl_vsil_gs.cpp b/port/cpl_vsil_gs.cpp index 712b6374c9c2..05dda45ac7c6 100644 --- a/port/cpl_vsil_gs.cpp +++ b/port/cpl_vsil_gs.cpp @@ -13,6 +13,7 @@ #include "cpl_port.h" #include "cpl_http.h" #include "cpl_minixml.h" +#include "cpl_json.h" #include "cpl_vsil_curl_priv.h" #include "cpl_vsil_curl_class.h" @@ -331,6 +332,124 @@ char **VSIGSFSHandler::GetFileMetadata(const char *pszFilename, if (!STARTS_WITH_CI(pszFilename, GetFSPrefix().c_str())) return nullptr; + if (pszDomain == nullptr) + { + // Handle case of requesting GetFileMetadata() on the bucket root + std::string osFilename(pszFilename); + if (osFilename.back() == '/') + osFilename.pop_back(); + if (osFilename.find('/', GetFSPrefix().size()) == std::string::npos) + { + const std::string osBucket = + osFilename.substr(GetFSPrefix().size()); + const std::string osResource = + std::string("storage/v1/b/").append(osBucket); + + auto poHandleHelper = std::unique_ptr( + VSIGSHandleHelper::BuildFromURI(osResource.c_str(), + GetFSPrefix().c_str(), + osBucket.c_str())); + if (!poHandleHelper) + return nullptr; + + // The JSON API cannot be used with HMAC keys + if (poHandleHelper->UsesHMACKey()) + { + CPLDebug(GetDebugKey(), + "GetFileMetadata() on bucket " + "only available for OAuth2 authentication"); + return VSICurlFilesystemHandlerBase::GetFileMetadata( + pszFilename, pszDomain, papszOptions); + } + + NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str()); + NetworkStatisticsAction oContextAction("GetFileMetadata"); + + const CPLStringList aosHTTPOptions( + CPLHTTPGetOptionsFromEnv(pszFilename)); + const CPLHTTPRetryParameters oRetryParameters(aosHTTPOptions); + CPLHTTPRetryContext oRetryContext(oRetryParameters); + + bool bRetry; + CPLStringList aosResult; + do + { + bRetry = false; + CURL *hCurlHandle = curl_easy_init(); + + struct curl_slist *headers = + static_cast(CPLHTTPSetOptions( + hCurlHandle, poHandleHelper->GetURL().c_str(), + aosHTTPOptions.List())); + headers = VSICurlMergeHeaders( + headers, poHandleHelper->GetCurlHeaders("GET", headers)); + + CurlRequestHelper requestHelper; + const long response_code = requestHelper.perform( + hCurlHandle, headers, this, poHandleHelper.get()); + + NetworkStatisticsLogger::LogGET( + requestHelper.sWriteFuncData.nSize); + + if (response_code != 200 || + requestHelper.sWriteFuncData.pBuffer == nullptr) + { + // Look if we should attempt a retry + if (oRetryContext.CanRetry( + static_cast(response_code), + requestHelper.sWriteFuncHeaderData.pBuffer, + requestHelper.szCurlErrBuf)) + { + CPLError(CE_Warning, CPLE_AppDefined, + "HTTP error code: %d - %s. " + "Retrying again in %.1f secs", + static_cast(response_code), + poHandleHelper->GetURL().c_str(), + oRetryContext.GetCurrentDelay()); + CPLSleep(oRetryContext.GetCurrentDelay()); + bRetry = true; + } + else + { + CPLDebug(GetDebugKey(), "%s", + requestHelper.sWriteFuncData.pBuffer + ? requestHelper.sWriteFuncData.pBuffer + : "(null)"); + CPLError(CE_Failure, CPLE_AppDefined, + "GetFileMetadata failed"); + } + } + else + { + CPLJSONDocument oDoc; + if (oDoc.LoadMemory( + reinterpret_cast( + requestHelper.sWriteFuncData.pBuffer), + static_cast( + requestHelper.sWriteFuncData.nSize)) && + oDoc.GetRoot().GetType() == CPLJSONObject::Type::Object) + { + for (const auto &oObj : oDoc.GetRoot().GetChildren()) + { + aosResult.SetNameValue(oObj.GetName().c_str(), + oObj.ToString().c_str()); + } + } + else + { + // Shouldn't happen normally + aosResult.SetNameValue( + "DATA", requestHelper.sWriteFuncData.pBuffer); + } + } + + curl_easy_cleanup(hCurlHandle); + } while (bRetry); + + return aosResult.StealList(); + } + } + if (pszDomain == nullptr || !EQUAL(pszDomain, "ACL")) { return VSICurlFilesystemHandlerBase::GetFileMetadata( @@ -404,7 +523,7 @@ char **VSIGSFSHandler::GetFileMetadata(const char *pszFilename, curl_easy_cleanup(hCurlHandle); } while (bRetry); - return CSLDuplicate(aosResult.List()); + return aosResult.StealList(); } /************************************************************************/ From 35388a063c0bfc60df0dde6655e2eee16bde0d6b Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 25 Oct 2024 21:12:59 +0200 Subject: [PATCH 15/54] CMake: use enable_testing() instead of include(CTest) According to Craig Scott in https://discourse.cmake.org/t/is-there-any-reason-to-prefer-include-ctest-or-enable-testing-over-the-other/1905/2 , using include(CTest) adds unnecessary clutter that is only needed for dashboard submission. enable_testing() is enough otherwise "Port" of https://github.com/OSGeo/shapelib/pull/162 --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 367cda089dca..07d1a09d5f2c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,6 @@ include(cmake/helpers/GdalCMakeMinimumRequired.cmake) cmake_minimum_required(VERSION ${GDAL_CMAKE_VERSION_MIN}...${GDAL_CMAKE_VERSION_MAX}) project(gdal LANGUAGES C CXX) -include(CTest) set(GDAL_LIB_TARGET_NAME GDAL) @@ -224,6 +223,10 @@ endif() # include(${CMAKE_CURRENT_SOURCE_DIR}/gdal.cmake) +option(BUILD_TESTING "Build the testing tree." ON) +if (BUILD_TESTING) + enable_testing() +endif() if (BUILD_TESTING AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/autotest") # unit tests add_subdirectory(autotest/cpp) From e845e532cf4e4bff38427bc3433915a3c856fd3f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 26 Oct 2024 19:14:44 +0200 Subject: [PATCH 16/54] JPEGXL: add support for reading Float16 (as Float32) --- autotest/gdrivers/data/jpegxl/float16.jxl | Bin 0 -> 6773 bytes autotest/gdrivers/jpegxl.py | 14 ++++++++++++++ frmts/jpegxl/jpegxl.cpp | 6 ++++++ 3 files changed, 20 insertions(+) create mode 100644 autotest/gdrivers/data/jpegxl/float16.jxl diff --git a/autotest/gdrivers/data/jpegxl/float16.jxl b/autotest/gdrivers/data/jpegxl/float16.jxl new file mode 100644 index 0000000000000000000000000000000000000000..7a60b5a41fe9f4f4151049c407fdcc0b7595afca GIT binary patch literal 6773 zcmV-*8j9rr000b1SWF-d3Wo{+000zbba`-Ucx)g50001Lcx)g5000SUcx-G6000_H zYItm8{|bl^MgT>N2m}ED*;p%C0gM!$a4utTs%>MhN^%4!17`%`+$oLAWF|0}MF%Xwxs5-o0!3l^%{pC4{U|q6=Yb#+Aw~ z*n+2O9Z;yu6vlk%IQ#zVIrkf=s&^49ptM*50NCWl1Yl&7uUb-zl0*?8Vx(XM01*Hn z*->oNiH(jIW3cKunsu&fgrO$}Wjv}du3?B68x@1YIL2UwhzPV1gEz(CaE!4r#xTaH z7-JZN12H%pgQGDxgxyN!#g89jFh0il7~_15!92zoHE)yj48~Zsh7G|at85K~=%{+a zS$AZMFbsW{JbG@%n5=;iZYv{46eZgRfSR6*L$7zNY=XlEKh=SVkpb6SKhb)9_mwPT0#3#c zN^>4~RqP*fqJkVDR-vYg1M8Hg#xm@)Mi4od6FeiRjc6H?27hB~Vwu<(9EXN<{!-QO zzFD~N*j-a9O82J`>DUcrBmifHGnwQ1E?%d0-gcPvOq3$GP`?c5sB!qi>|Ai{!YE}C zq)@0UeRmSFJU}IjNXO5BvW;5P*9WB{ov!JW3?pw^#Z0;S8*UFvh1-Kap;s`U#R3MZ z2ja3WSCowOCd?J;)Z!BV(MA^`r#SHr3iwtyzadHKERjA{gp#u<)zK5$r0gytOd1x< z-@_p@#erBmguzLYj*k&8A!-I`3`5iaw#q>RFmZx#CCM3T&}r}k1s)ZFHe*_naQp;m ztr`VtE8&lahUTl6WjA_uld0JnWd^3qg9>1+wBq_cvtDlWf`Fe6W>zh59abvPSLW)-(#!}1> z>a8@Fgpo71UXqK7ntcmvMdQe%Vp&P_+8w4jmQKSQ)PSr zs0|>q3vBnBEW)9w8rMMGe)TC%WpWmWQjxWk*7!IN<0%Z}4l=1Sz-3zaf}5!lsm8pm z5YonbSngKkQ1#TIixYQ4G9Zg=n3^@3wqxDJH4)+W3~dOqmP-dPyeK30`Ud1O5|jw& zX1$71i07Ph#;WFNZ|!iVYjAL6L98CdC>z*-sqG=?%Ik{+xdqW>HFRFdPp+OjeS`x~ z0Mc}FJA`oxQN(3opxKLFA`qD`=0S1>!G=MYAtN@h5juKJvLp^aOn#ID*PH+yiBm$# z;GU=YkqRxWoJ|xz2Z$OwjMB*wXP~Y=A(I?DK!_SQN&vcunyAG@)6qdvvWXMKC&vKn zR}bQdfKqbGSV$h-8-vmhXqjbFuSxmcOi)G6%e|WRqQG<(f_uxzi3+wa&H$o1iPA6b zg~-?iL|0ClniJn)2O3mClbIOG><6BS*7jEz!hmdsdUSLVqHmuM-G#Ju^TM@$IG zZ`%azGjSN1mhj8?_66Q8%now^uC@+<&LxZu;-Hb^Q!#s*OEO8Q%mXa4Nq+o>nn|As zhPi?C9@TD(_HihMu5(GJ0pcIaiO98p-eZS-JTX8Wt)6x+inuz3sKGoWM7ED`_Yi%^ zkDIyT|Nj6OfB^3Rbvi(;FpQo?07lW1Eb6Ef z#^6EE$vsyXSb)5%@rHS;{{awy2S`Tgt9V=EH<-$aQK5YlD8RRbzswLoqQNVOu^s!d z+(}fr{so{z7+Sxe5ATE6_>Ts_g}wlA0DBoc@r;ZGFXr{s>@U|QNpyL9^HHRF60~)u zN%^yiHo+%^OZe2pY5xLzZkTQW07TF;<-iDWDjpsy^vRLImJ4l!_^I%veQ@Nxh65k~e=$UT7a%DqR7rzgx$z$XUU_aiIad9MuY``t{NzsWw(@RzQ=%@2 z)LLt#R072Iy+zyFG6=E^9-yXBjO#&2#+Fj2{t%A9M8=fH{kb-`w)-5&=fp3gx5 zwvYWLak$W=5_QWf9wSYAWkLn#XozSKDRNFpv9&z)20xJcNJ=#n^F`X^j`ZXI2jEj2 zcWlNf0P(B%aXhK7ufU5(SIXn!C@#SFb zjx%HnU8RSgkiwGjCy&EsBoSK1lPq|3a8Nf59Nv;l`16) zq3v~`fB^U7QAO#?>i_m{`M*|wyAed+8L{o-YS^D~)4S0t*F^hF*f|XQ-ZoL|Tg#eP$No<7wN=$GuZh6%h zZAbVYz=<>d1<(pqWr?Gw?nFEtbvs#Up;kK1H>Kx`j!fW!LFiW?sjHE#M~4gT#QFSh z(oX;mKt4cCap=QcEBI{x(YsgxFoQEDXN(aI0v=z(H*0DcD4{;Sgfsd)0484t06yVi zHIfUBrEr$Qf|LSwr?`=|@7h3>MgYzhYM3WDTOT9v0rUz$tQmm+`@dTk0m2-@LADH- z>0Tr#K~D>8(=W(@CGmeTFrqttT3guO7aG9Gz(O}tg)(T2=0*Yjx6>m4U;|`3$G9c- zhY5x-%-=~;h_jpKp0ol73T}Vb9gys$0i1lcy`1&{5+>jYEC(+D|7F3z5FBWbIRht@ zrJf9kBC%xelt>RsDv{7b#oCA;FL{8K zRk=iRM;B>pRG4W!6$fHHIU2(RF(-Sfu^xbnLIKsT4^DlL@qLj~YuEuci`yN94 z4UY~}a-+Q3Fh#kfMLumlsdwvQZo!HFfBsK-*ueV~41Y};LP#QPhGh*4CpnFkh~)%+ z6}YPfx$q~p3ut}$UuC(1POO+A`~VaHC{gIi(4c^~Zv(t3lmCr6S9-b~2k_WIVhw8p z(g+Y7HwnZV;utUC9B3B00N`KhFNT5yQOl=p#POAb^lagnbYKz!M{6dAQ{Pgs7-1U| z897#XP81U(r#{4AfcNLu?_K;JwcG%#aiR_EKpBNhH!TP!p1@rNS=za>WlO~*#TE&( zxq%)L`m*-})x3Py{{w*kH-K>22-8r7N+lsna3ewhgCuYLPlJOA%HTB;RH9lqQ2(6o6%10*fp`j~Z<7q0%kNF^8^ZyM1z)6F2av@aY7x^y_K_QcwViGnj1g(Syk~n9A;0oEOG>6V2_>5{> z3=9K=-t*0$jQvXb4)q`Gn$*wJOjA~~BRE6emGNOWi}5OkPf6)vkyy%|gCELQBL)dyfIJpUi4p zx3WS%Isu@PBVcinsj)Lz7mb_k8yPVs<1q`ICkmAa^#ee;Vc;Q;cpHRJiFVs2q(E;^ zQu)@L?_GZr<81E{S^%(Y{Qvg9{+pLd>nV$37*Mjb;cx-s}|5!O(;u=J^ixVgF>;u)3aP{=Xef*_Z-&{|#U;U5Y(X z?FDh_B>l@=z{2Oja!^-mgc*Ow1Wne+qNCP-Yj{yz2Rl2Yti%JRy=*VLcZI4t)?#{@ zK{KwqLRyJ(MTh^pn1NHpes)IT**&5roLr{X6@I6)xY`B;R|wd;ffj^p5kjFVEM!9~ z3QY!kuff0=NDCfSeJ+{U?5}Iq8YD%rWa**+9j%v^b6glHcM^Ao+8i zD*xXA_0JsecMV{0YeWMg!`OWbZ0Q8$_*bS$)zcUWOkf<~`u%2iu0Q|p`5^os> zPk@O7ErmU{5&n)Ma^wyZgee~xwng|oIqDCiDc7S4x+0N(qwGd~>l`sH| zzSToB`V^^ov4cX4iG7cOl}5>-VumcY6_6w|NN(VVt;feYXbJmGNNDYP!!*X?l->aN zf4dr(f~^(b_~cRo?JbGJ8=z&_^8-P>9YwP;`m?a#(?%DBWR|Ib9a?xC^5N2g2*7^; z=i2ZIymFAzCg8bs2>B*dM7wAlk@)lmww>mk!wZNzgU9SidIPC;K&#cyjYt`PEg$^9 z0r>w7+}wg|1%4FzZU#12X2sxIkU%6YH8@}7RJ15Kp%s#(&T+r-sK(wevJ#;>g~pQK z|4CP}zM%eG1^%QE@d~F_bs94&Z&XJWj0;~1AZV@^eI!G`ah!=5%zdT5Eni$01pcfM^?-LSymr7?suT(^QEgjP*aqG7nvT5v(uG>xOX zYL2-I&xXiX^Tw9TeBg9DeOsWsdaDTuquGRRQi(BNDH;xFQow(`yLrUvTcF{C@Rc+F zI%;vtt@2qDxv8C_l5m!SQOtLMH#n1oQR^8%PX@1+G3uZNdAMy=YQgQk68r8;oQEp6 znGTq7UGlvONqDvr09<&t9i^mQx7jjIhGD8eFAQh`y<9T;;!HonIDCqi;yGJrn$^Vi z{&Uz7R-qW|D$=N4FJM0McBgGp;br0#{cxhZXSYm!(g*oQG=_rH38CW>Myl0 z0sQNwMvB4T5UHXlrBHDM`fvXmK=w=8P+++X;Fxz2Z3ke6a zk@Jb%EPNv(lbR_Obyl|tlIk>78vX+S|3AP#N>l1WTG*BHW2tO3d?6Ua9z)xl=wr)q z5J3)jA<|9<_9M9Yf^#t1a5oZJID#v@GSKqB{yUo$fCC`@5kcjrkFZz;;KDjQmB(Qi zBqIfO2u!<(q-w2@w#OQVBA5_T_c)0yA1cu_sdO20NX6j3A=iJKj>IaSHemYzM?4!5?1=bJ}Y}tk_r4F;HNs$k~ki!(mp3v z$FpQw6oSRkqOqUe7yPe^B4T#8zgWWn|k%tj0qWwZYx0Imf8Hh~)b@q92#S{T%1s!pqPtU4MW7?SCh90YKbrWT7!dP?OH!t>J?(oeq@|^ut`Z zYGao|dc|j>3RuuQuTCLfw-~{~rF8r58=e=~nQ6|DV!DT9dUB#FJO+ zSu}ja(RvE# z0LZ=frUBv(rWO;SXTU&H!i?JJnUD$XgoATq9K48dd`g7y7}0Zg482vT+zydG-aGh; zK@wR3AKrieKZqWQ7jin?o?UIpZ(tee5M@%IZ{Ud({ZhT_%d?Q3su`F8u4V}ZxD{C7}Sl52@A)=nrlSpm3^rMcve XaBD^bJ_IoeE% Date: Tue, 29 Oct 2024 15:55:22 +0100 Subject: [PATCH 17/54] test_pct.py: skip on sanitize CI --- autotest/pyscripts/test_pct.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/autotest/pyscripts/test_pct.py b/autotest/pyscripts/test_pct.py index 88212f866c43..c549685ea5ff 100755 --- a/autotest/pyscripts/test_pct.py +++ b/autotest/pyscripts/test_pct.py @@ -40,6 +40,9 @@ def script_path(): def test_rgb2pct_help(script_path): + if gdaltest.is_travis_branch("sanitize"): + pytest.skip("fails on sanitize for unknown reason") + assert "ERROR" not in test_py_scripts.run_py_script( script_path, "rgb2pct", "--help" ) @@ -51,6 +54,9 @@ def test_rgb2pct_help(script_path): def test_rgb2pct_version(script_path): + if gdaltest.is_travis_branch("sanitize"): + pytest.skip("fails on sanitize for unknown reason") + assert "ERROR" not in test_py_scripts.run_py_script( script_path, "rgb2pct", "--version" ) From 886fe26d3809521bbdd4c59dfabde4eda608ec10 Mon Sep 17 00:00:00 2001 From: Daniel Baston Date: Tue, 29 Oct 2024 12:02:20 -0400 Subject: [PATCH 18/54] OGRGeometryFactory: Add createFromWkt overload returning unique_ptr --- autotest/cpp/test_ogr.cpp | 31 +++++++++---------------------- ogr/ogr_geometry.h | 3 +++ ogr/ogrgeometryfactory.cpp | 31 +++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/autotest/cpp/test_ogr.cpp b/autotest/cpp/test_ogr.cpp index 9f2de42a3150..b43f40d1cd65 100644 --- a/autotest/cpp/test_ogr.cpp +++ b/autotest/cpp/test_ogr.cpp @@ -4400,21 +4400,16 @@ TEST_F(test_ogr, OGRFeature_SetGeometry) poFeatureDefn->Reference(); OGRFeature oFeat(poFeatureDefn); - std::unique_ptr poGeom; - OGRGeometry *poTmpGeom; - ASSERT_EQ( - OGRGeometryFactory::createFromWkt("POINT (3 7)", nullptr, &poTmpGeom), - OGRERR_NONE); - poGeom.reset(poTmpGeom); + auto [poGeom, err] = OGRGeometryFactory::createFromWkt("POINT (3 7)"); + ASSERT_EQ(err, OGRERR_NONE); + ASSERT_EQ(oFeat.SetGeometry(std::move(poGeom)), OGRERR_NONE); EXPECT_EQ(oFeat.GetGeometryRef()->toPoint()->getX(), 3); EXPECT_EQ(oFeat.GetGeometryRef()->toPoint()->getY(), 7); // set it again to make sure previous feature geometry is freed - ASSERT_EQ( - OGRGeometryFactory::createFromWkt("POINT (2 8)", nullptr, &poTmpGeom), - OGRERR_NONE); - poGeom.reset(poTmpGeom); + std::tie(poGeom, err) = OGRGeometryFactory::createFromWkt("POINT (2 8)"); + ASSERT_EQ(err, OGRERR_NONE); ASSERT_EQ(oFeat.SetGeometry(std::move(poGeom)), OGRERR_NONE); EXPECT_EQ(oFeat.GetGeometryRef()->toPoint()->getX(), 2); EXPECT_EQ(oFeat.GetGeometryRef()->toPoint()->getY(), 8); @@ -4434,23 +4429,15 @@ TEST_F(test_ogr, OGRFeature_SetGeomField) // failure { - std::unique_ptr poGeom; - OGRGeometry *poTmpGeom; - ASSERT_EQ(OGRGeometryFactory::createFromWkt("POINT (3 7)", nullptr, - &poTmpGeom), - OGRERR_NONE); - poGeom.reset(poTmpGeom); + auto [poGeom, err] = OGRGeometryFactory::createFromWkt("POINT (3 7)"); + ASSERT_EQ(err, OGRERR_NONE); EXPECT_EQ(oFeat.SetGeomField(13, std::move(poGeom)), OGRERR_FAILURE); } // success { - std::unique_ptr poGeom; - OGRGeometry *poTmpGeom; - ASSERT_EQ(OGRGeometryFactory::createFromWkt("POINT (3 7)", nullptr, - &poTmpGeom), - OGRERR_NONE); - poGeom.reset(poTmpGeom); + auto [poGeom, err] = OGRGeometryFactory::createFromWkt("POINT (3 7)"); + ASSERT_EQ(err, OGRERR_NONE); EXPECT_EQ(oFeat.SetGeomField(1, std::move(poGeom)), OGRERR_NONE); } diff --git a/ogr/ogr_geometry.h b/ogr/ogr_geometry.h index abd66ace0e8f..7589a90864d4 100644 --- a/ogr/ogr_geometry.h +++ b/ogr/ogr_geometry.h @@ -25,6 +25,7 @@ #include #include #include +#include /** * \file ogr_geometry.h @@ -4234,6 +4235,8 @@ class CPL_DLL OGRGeometryFactory OGRGeometry **); static OGRErr createFromWkt(const char **, const OGRSpatialReference *, OGRGeometry **); + static std::pair, OGRErr> + createFromWkt(const char *, const OGRSpatialReference * = nullptr); /** Deprecated. * @deprecated in GDAL 2.3 diff --git a/ogr/ogrgeometryfactory.cpp b/ogr/ogrgeometryfactory.cpp index 963e36276981..e68303667da9 100644 --- a/ogr/ogrgeometryfactory.cpp +++ b/ogr/ogrgeometryfactory.cpp @@ -481,6 +481,37 @@ OGRErr OGRGeometryFactory::createFromWkt(const char *pszData, return createFromWkt(&pszData, poSR, ppoReturn); } +/** + * \brief Create a geometry object of the appropriate type from its + * well known text representation. + * + * The C function OGR_G_CreateFromWkt() is the same as this method. + * + * @param pszData input zero terminated string containing well known text + * representation of the geometry to be created. + * @param poSR pointer to the spatial reference to be assigned to the + * created geometry object. This may be NULL. + + * @return a pair of the newly created geometry an error code of OGRERR_NONE + * if all goes well, otherwise any of OGRERR_NOT_ENOUGH_DATA, + * OGRERR_UNSUPPORTED_GEOMETRY_TYPE, or OGRERR_CORRUPT_DATA. + * + * @since GDAL 3.11 + */ + +std::pair, OGRErr> +OGRGeometryFactory::createFromWkt(const char *pszData, + const OGRSpatialReference *poSR) + +{ + std::unique_ptr poGeom; + OGRGeometry *poTmpGeom; + auto err = createFromWkt(&pszData, poSR, &poTmpGeom); + poGeom.reset(poTmpGeom); + + return {std::move(poGeom), err}; +} + /************************************************************************/ /* OGR_G_CreateFromWkt() */ /************************************************************************/ From ac90456338bd0f12170578865f7b1f3fd22acc87 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 30 Oct 2024 16:04:03 +0100 Subject: [PATCH 19/54] Doc: add a page describing the RFC process step by step --- doc/source/community/index.rst | 1 + doc/source/development/index.rst | 1 + doc/source/development/rfc_process.rst | 69 ++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 doc/source/development/rfc_process.rst diff --git a/doc/source/community/index.rst b/doc/source/community/index.rst index 9327bf3ba51c..2ebfa71939c3 100644 --- a/doc/source/community/index.rst +++ b/doc/source/community/index.rst @@ -27,6 +27,7 @@ Code of Conduct code_of_conduct +.. _mailing_list: Mailing List ------------ diff --git a/doc/source/development/index.rst b/doc/source/development/index.rst index 59902fdbcfe5..c3e443578dc2 100644 --- a/doc/source/development/index.rst +++ b/doc/source/development/index.rst @@ -14,4 +14,5 @@ Development testing dev_documentation cmake + rfc_process rfc/index diff --git a/doc/source/development/rfc_process.rst b/doc/source/development/rfc_process.rst new file mode 100644 index 000000000000..6be6e3afdd75 --- /dev/null +++ b/doc/source/development/rfc_process.rst @@ -0,0 +1,69 @@ +.. _rfc_process: + +================================================================================ +Request For Comments (RFC) process for code changes +================================================================================ + +While most bug fixes and small enhancements following the existing logic of code +do not require a RFC and can be done using pull requests, we require contributors +to write a RFC and go through a formal adoption process in the following cases: + +- Anything that could cause backward compatibility issues. +- Adding substantial amounts of new code (see :ref:`rfc-85`) +- Changing inter-subsystem APIs, or objects. +- Anything that might be controversial. + +A good rule of thumb is that if you wonder "does this require a RFC?", the +answer is generally "yes". + +The RFC process is the following one: + +- add a new .rst file in :file:`doc/source/development/rfc` with the RFC text, + and references it in :file:`doc/source/development/rfc/index.rst`, and issue + a pull request. + You can look at https://github.com/OSGeo/gdal/pull/10913 as a potential example + for the expected content. Typical sections are: + + - Title: RFC {XYZ}: {Description of the RFC in a short sentence} + - Author: your name. + - Contact: your email. + - Started: date at which this is started. + - Status: "Draft", when the RFC is started. + - Summary: what the RFC accomplishes in a few sentences. + - Motivation: Paragraph describing why this RFC is needed. + - Technical solution: organized in several sub-paragraph depending on the + complexity of the RFC. You may need to mention new API calls, impacted + files, etc., and other needed details. + - Backward compatibility: API, ABI, and other compatibility concerns. + - Performance impacts. + - Documentation: how this will be documented. + - Testing: how this will be tested. + - Related issues and PRs: points at potential existing tickets about the topic, + and pull request with a candidate implementation + - Voting history: "TBD" (to be defined), when the RFC is started + +- Send an email to the :ref:`gdal-dev mailing list ` with a title like: + "Call for discussion on RFC {XYZ}: {Description of the RFC in a short sentence}" + where you point to the pull request. You may get feedback both in the pull + request or in the mailing list. + +- Wait for feedback (give readers at least one week, or more if holiday season). + If it seems positive and reasonable consensus can be reached, you can start + working on a candidate implementation, and issue a pull request for it, + separate for the pull request with the RFC text. + Reference this candidate implementation pull request in the "Related issues + and PRs" section of the RFC text. + +- Once your candidate implementation is representative enough, and you and other + contributors can have a good confidence that the RFC can be implemented in the + way it has been described, send an email to the gdal-dev mailing list with + a title like "Motion: adopt RFC {XYZ}: {Description of the RFC in a short sentence}" + where you point to the pull request. + All subscribers can vote, but only votes from members of the GDAL Project + Steering Committee are binding. The voting rules are covered by :ref:`rfc-1`. + +- Once the RFC is approved, finish your candidate implementation if not already + finished, and wait for the review process and for it to be merged. + +- Update the Status to "Adopted, implemented" and Voting history sections of the + RFC text. From 2be3957eac09ba2b473fd65dde48c16fa12e5817 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 31 Oct 2024 17:09:50 +0100 Subject: [PATCH 20/54] VRT: allow to use a instead of a inside a or Currently a number of users use the ugly hack of putting a trick inside the . This will make things cleaner. --- autotest/gcore/data/vrt/nested_VRTDataset.vrt | 24 +++++ autotest/gcore/vrt_read.py | 35 +++++++ doc/source/drivers/raster/vrt.rst | 25 +++++ frmts/vrt/data/gdalvrt.xsd | 5 +- frmts/vrt/vrtdataset.cpp | 13 ++- frmts/vrt/vrtdataset.h | 8 +- frmts/vrt/vrtsources.cpp | 97 ++++++++++++++----- 7 files changed, 179 insertions(+), 28 deletions(-) create mode 100644 autotest/gcore/data/vrt/nested_VRTDataset.vrt diff --git a/autotest/gcore/data/vrt/nested_VRTDataset.vrt b/autotest/gcore/data/vrt/nested_VRTDataset.vrt new file mode 100644 index 000000000000..49a5ff70aef2 --- /dev/null +++ b/autotest/gcore/data/vrt/nested_VRTDataset.vrt @@ -0,0 +1,24 @@ + + PROJCS["NAD27 / UTM zone 11N",GEOGCS["NAD27",DATUM["North_American_Datum_1927",SPHEROID["Clarke 1866",6378206.4,294.978698213898,AUTHORITY["EPSG","7008"]],AUTHORITY["EPSG","6267"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4267"]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",-117],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["Easting",EAST],AXIS["Northing",NORTH],AUTHORITY["EPSG","26711"]] + 4.4072000000000000e+05, 6.0000000000000000e+01, 0.0000000000000000e+00, 3.7513200000000000e+06, 0.0000000000000000e+00, -6.0000000000000000e+01 + + Gray + + + + + ../byte.tif + 1 + + + + + + + 1 + + + + + + diff --git a/autotest/gcore/vrt_read.py b/autotest/gcore/vrt_read.py index d114064b5109..cab7607e5c73 100755 --- a/autotest/gcore/vrt_read.py +++ b/autotest/gcore/vrt_read.py @@ -2802,3 +2802,38 @@ def test_vrt_read_multi_threaded_disabled_since_overlapping_sources(): assert ( vrt_ds.GetMetadataItem("MULTI_THREADED_RASTERIO_LAST_USED", "__DEBUG__") == "0" ) + + +############################################################################### +# Test reading a VRT with a inside a + + +def test_vrt_read_nested_VRTDataset(): + + ds = gdal.Open("data/vrt/nested_VRTDataset.vrt") + assert ds.GetRasterBand(1).Checksum() == 4672 + + +############################################################################### +# Test updating a VRT with a inside a + + +def test_vrt_update_nested_VRTDataset(tmp_vsimem): + + gdal.FileFromMemBuffer(tmp_vsimem / "byte.tif", open("data/byte.tif", "rb").read()) + gdal.Mkdir(tmp_vsimem / "vrt", 0o755) + vrt_filename = tmp_vsimem / "vrt" / "nested_VRTDataset.vrt" + gdal.FileFromMemBuffer( + vrt_filename, open("data/vrt/nested_VRTDataset.vrt", "rb").read() + ) + + with gdal.Open(vrt_filename) as ds: + assert ds.GetRasterBand(1).Checksum() == 4672 + assert ds.GetRasterBand(1).GetMinimum() is None + vrt_stats = ds.GetRasterBand(1).ComputeStatistics(False) + assert vrt_stats[0] == 74 + + # Check that statistics have been serialized in the VRT + with gdal.Open(vrt_filename) as ds: + assert ds.GetRasterBand(1).Checksum() == 4672 + assert ds.GetRasterBand(1).GetMinimum() == 74 diff --git a/doc/source/drivers/raster/vrt.rst b/doc/source/drivers/raster/vrt.rst index 6e3112e7cfdd..ab3adfd0d6fa 100644 --- a/doc/source/drivers/raster/vrt.rst +++ b/doc/source/drivers/raster/vrt.rst @@ -405,6 +405,31 @@ cubicspline,lanczos,average,mode. + +Starting with GDAL 3.11, it is also possible to use a in-line VRTDataset as +the source by using the VRTDataset element instead of SourceFilename. + +.. code-block:: xml + + + + + + ../byte.tif + 1 + + + + + + + 1 + + + + + + ComplexSource ~~~~~~~~~~~~~ diff --git a/frmts/vrt/data/gdalvrt.xsd b/frmts/vrt/data/gdalvrt.xsd index dbf0af54f740..a6eb50e28270 100644 --- a/frmts/vrt/data/gdalvrt.xsd +++ b/frmts/vrt/data/gdalvrt.xsd @@ -449,7 +449,10 @@ - + + + + diff --git a/frmts/vrt/vrtdataset.cpp b/frmts/vrt/vrtdataset.cpp index eb24e903c069..58e142714d48 100644 --- a/frmts/vrt/vrtdataset.cpp +++ b/frmts/vrt/vrtdataset.cpp @@ -3071,8 +3071,19 @@ std::string VRTDataset::BuildSourceFilename(const char *pszFilename, } if (!bDone) { + std::string osVRTPath = pszVRTPath; + if (!CPLIsFilenameRelative(pszVRTPath)) + { + // Simplify path by replacing "foo/a/../b" with "foo/b" + while (STARTS_WITH(pszFilename, "../")) + { + osVRTPath = CPLGetPath(osVRTPath.c_str()); + pszFilename += strlen("../"); + } + } + osSrcDSName = - CPLProjectRelativeFilename(pszVRTPath, pszFilename); + CPLProjectRelativeFilename(osVRTPath.c_str(), pszFilename); } } } diff --git a/frmts/vrt/vrtdataset.h b/frmts/vrt/vrtdataset.h index 5d32b0661048..961bcc60d49f 100644 --- a/frmts/vrt/vrtdataset.h +++ b/frmts/vrt/vrtdataset.h @@ -1335,7 +1335,11 @@ class CPL_DLL VRTSimpleSource CPL_NON_FINAL : public VRTSource // from which the mask band is taken. mutable GDALRasterBand *m_poMaskBandMainBand = nullptr; - CPLStringList m_aosOpenOptions{}; + CPLStringList m_aosOpenOptionsOri{}; // as read in the original source XML + CPLStringList + m_aosOpenOptions{}; // same as above, but potentially augmented with ROOT_PATH + bool m_bSrcDSNameFromVRT = + false; // whereas content in m_osSrcDSName is a XML node void OpenSource() const; @@ -1407,6 +1411,8 @@ class CPL_DLL VRTSimpleSource CPL_NON_FINAL : public VRTSource m_dfDstXSize != UNINIT_WINDOW || m_dfDstYSize != UNINIT_WINDOW; } + void AddSourceFilenameNode(const char *pszVRTPath, CPLXMLNode *psSrc); + public: VRTSimpleSource(); VRTSimpleSource(const VRTSimpleSource *poSrcSource, double dfXDstRatio, diff --git a/frmts/vrt/vrtsources.cpp b/frmts/vrt/vrtsources.cpp index 1e3fc80aa469..42712f912d1c 100644 --- a/frmts/vrt/vrtsources.cpp +++ b/frmts/vrt/vrtsources.cpp @@ -91,7 +91,9 @@ VRTSimpleSource::VRTSimpleSource(const VRTSimpleSource *poSrcSource, : m_poMapSharedSources(poSrcSource->m_poMapSharedSources), m_poRasterBand(poSrcSource->m_poRasterBand), m_poMaskBandMainBand(poSrcSource->m_poMaskBandMainBand), + m_aosOpenOptionsOri(poSrcSource->m_aosOpenOptionsOri), m_aosOpenOptions(poSrcSource->m_aosOpenOptions), + m_bSrcDSNameFromVRT(poSrcSource->m_bSrcDSNameFromVRT), m_nBand(poSrcSource->m_nBand), m_bGetMaskBand(poSrcSource->m_bGetMaskBand), m_dfSrcXOff(poSrcSource->m_dfSrcXOff), @@ -228,6 +230,7 @@ void VRTSimpleSource::SetSrcBand(GDALRasterBand *poNewSrcBand) { m_osSrcDSName = poDS->GetDescription(); m_aosOpenOptions = CSLDuplicate(poDS->GetOpenOptions()); + m_aosOpenOptionsOri = m_aosOpenOptions; } } @@ -248,6 +251,7 @@ void VRTSimpleSource::SetSrcMaskBand(GDALRasterBand *poNewSrcBand) { m_osSrcDSName = poDS->GetDescription(); m_aosOpenOptions = CSLDuplicate(poDS->GetOpenOptions()); + m_aosOpenOptionsOri = m_aosOpenOptions; } m_bGetMaskBand = true; } @@ -316,7 +320,7 @@ bool VRTSimpleSource::DstWindowIntersects(double dfXOff, double dfYOff, } /************************************************************************/ -/* SerializeToXML() */ +/* IsSlowSource() */ /************************************************************************/ static bool IsSlowSource(const char *pszSrcName) @@ -327,17 +331,13 @@ static bool IsSlowSource(const char *pszSrcName) strstr(pszSrcName, "&url=http") != nullptr); } -CPLXMLNode *VRTSimpleSource::SerializeToXML(const char *pszVRTPath) +/************************************************************************/ +/* AddSourceFilenameNode() */ +/************************************************************************/ +void VRTSimpleSource::AddSourceFilenameNode(const char *pszVRTPath, + CPLXMLNode *psSrc) { - CPLXMLNode *const psSrc = - CPLCreateXMLNode(nullptr, CXT_Element, GetTypeStatic()); - - if (!m_osResampling.empty()) - { - CPLCreateXMLNode(CPLCreateXMLNode(psSrc, CXT_Attribute, "resampling"), - CXT_Text, m_osResampling.c_str()); - } VSIStatBufL sStat; int bRelativeToVRT = FALSE; // TODO(schwehr): Make this a bool? @@ -471,8 +471,34 @@ CPLXMLNode *VRTSimpleSource::SerializeToXML(const char *pszVRTPath) CXT_Attribute, "shared"), CXT_Text, "0"); } +} + +/************************************************************************/ +/* SerializeToXML() */ +/************************************************************************/ - GDALSerializeOpenOptionsToXML(psSrc, m_aosOpenOptions.List()); +CPLXMLNode *VRTSimpleSource::SerializeToXML(const char *pszVRTPath) + +{ + CPLXMLNode *const psSrc = + CPLCreateXMLNode(nullptr, CXT_Element, GetTypeStatic()); + + if (!m_osResampling.empty()) + { + CPLCreateXMLNode(CPLCreateXMLNode(psSrc, CXT_Attribute, "resampling"), + CXT_Text, m_osResampling.c_str()); + } + + if (m_bSrcDSNameFromVRT) + { + CPLAddXMLChild(psSrc, CPLParseXMLString(m_osSrcDSName.c_str())); + } + else + { + AddSourceFilenameNode(pszVRTPath, psSrc); + } + + GDALSerializeOpenOptionsToXML(psSrc, m_aosOpenOptionsOri.List()); if (m_bGetMaskBand) CPLSetXMLValue(psSrc, "SourceBand", CPLSPrintf("mask,%d", m_nBand)); @@ -548,36 +574,56 @@ CPLErr VRTSimpleSource::XMLInit(const CPLXMLNode *psSrc, const char *pszVRTPath, /* -------------------------------------------------------------------- */ const CPLXMLNode *psSourceFileNameNode = CPLGetXMLNode(psSrc, "SourceFilename"); + const CPLXMLNode *psSourceVRTDataset = CPLGetXMLNode(psSrc, "VRTDataset"); const char *pszFilename = psSourceFileNameNode ? CPLGetXMLValue(psSourceFileNameNode, nullptr, "") : ""; - if (pszFilename[0] == '\0') + if (pszFilename[0] == '\0' && !psSourceVRTDataset) { CPLError(CE_Warning, CPLE_AppDefined, - "Missing element in VRTRasterBand."); + "Missing or element in <%s>.", + psSrc->pszValue); return CE_Failure; } // Backup original filename and relativeToVRT so as to be able to // serialize them identically again (#5985) m_osSourceFileNameOri = pszFilename; - m_bRelativeToVRTOri = - atoi(CPLGetXMLValue(psSourceFileNameNode, "relativetoVRT", "0")); - const char *pszShared = - CPLGetXMLValue(psSourceFileNameNode, "shared", nullptr); - if (pszShared == nullptr) + if (pszFilename[0]) { - pszShared = CPLGetConfigOption("VRT_SHARED_SOURCE", nullptr); + m_bRelativeToVRTOri = + atoi(CPLGetXMLValue(psSourceFileNameNode, "relativetoVRT", "0")); + const char *pszShared = + CPLGetXMLValue(psSourceFileNameNode, "shared", nullptr); + if (pszShared == nullptr) + { + pszShared = CPLGetConfigOption("VRT_SHARED_SOURCE", nullptr); + } + if (pszShared != nullptr) + { + m_nExplicitSharedStatus = CPLTestBool(pszShared); + } + + m_osSrcDSName = VRTDataset::BuildSourceFilename( + pszFilename, pszVRTPath, CPL_TO_BOOL(m_bRelativeToVRTOri)); } - if (pszShared != nullptr) + else if (psSourceVRTDataset) { - m_nExplicitSharedStatus = CPLTestBool(pszShared); + CPLXMLNode sNode; + sNode.eType = psSourceVRTDataset->eType; + sNode.pszValue = psSourceVRTDataset->pszValue; + sNode.psNext = nullptr; + sNode.psChild = psSourceVRTDataset->psChild; + char *pszXML = CPLSerializeXMLTree(&sNode); + if (pszXML) + { + m_bSrcDSNameFromVRT = true; + m_osSrcDSName = pszXML; + CPLFree(pszXML); + } } - m_osSrcDSName = VRTDataset::BuildSourceFilename( - pszFilename, pszVRTPath, CPL_TO_BOOL(m_bRelativeToVRTOri)); - const char *pszSourceBand = CPLGetXMLValue(psSrc, "SourceBand", "1"); m_bGetMaskBand = false; if (STARTS_WITH_CI(pszSourceBand, "mask")) @@ -600,6 +646,7 @@ CPLErr VRTSimpleSource::XMLInit(const CPLXMLNode *psSrc, const char *pszVRTPath, } m_aosOpenOptions = GDALDeserializeOpenOptionsFromXML(psSrc); + m_aosOpenOptionsOri = m_aosOpenOptions; if (strstr(m_osSrcDSName.c_str(), " Date: Fri, 1 Nov 2024 07:34:20 +1100 Subject: [PATCH 21/54] Update gti.rst add note about vrt:// in 'location' sources --- doc/source/drivers/raster/gti.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/drivers/raster/gti.rst b/doc/source/drivers/raster/gti.rst index f54bcff2c5cf..f44f0b359b12 100644 --- a/doc/source/drivers/raster/gti.rst +++ b/doc/source/drivers/raster/gti.rst @@ -42,7 +42,8 @@ driver with the following main differences: * Contrary to the VRT driver, the GTI driver does not enable to alter characteristics of referenced tiles, such as their georeferencing, nodata value, etc. If such behavior is desired, the tiles must be for example wrapped - individually in a VRT file before being referenced in the GTI index. + individually in a VRT file (or `vrt://` connection string) before being referenced + in the GTI index. Connection strings ------------------ From c95d5d4f7bf327bf14e60c251bd01a787d35e0eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 02:05:29 +0000 Subject: [PATCH 22/54] Bump actions/cache from 4.0.2 to 4.1.2 Bumps [actions/cache](https://github.com/actions/cache) from 4.0.2 to 4.1.2. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/0c45773b623bea8c8e75f6c82b208c3cf94ea4f9...6849a6489940f00c2f30c0fb92c6274307ccb58a) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/android_cmake.yml | 2 +- .github/workflows/cmake_builds.yml | 6 +++--- .github/workflows/conda.yml | 2 +- .github/workflows/linux_build.yml | 4 ++-- .github/workflows/slow_tests.yml | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/android_cmake.yml b/.github/workflows/android_cmake.yml index 108e0cadc2a8..2424254339c7 100644 --- a/.github/workflows/android_cmake.yml +++ b/.github/workflows/android_cmake.yml @@ -29,7 +29,7 @@ jobs: uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - name: Cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 id: cache with: path: | diff --git a/.github/workflows/cmake_builds.yml b/.github/workflows/cmake_builds.yml index 088a7204d9ed..910c84966de4 100644 --- a/.github/workflows/cmake_builds.yml +++ b/.github/workflows/cmake_builds.yml @@ -37,7 +37,7 @@ jobs: - name: Checkout GDAL uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - name: Setup cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 id: cache with: path: ${{ github.workspace }}/.ccache @@ -332,7 +332,7 @@ jobs: mingw-w64-x86_64-libgeotiff mingw-w64-x86_64-libpng mingw-w64-x86_64-libtiff mingw-w64-x86_64-openjpeg2 mingw-w64-x86_64-python-pip mingw-w64-x86_64-python-numpy mingw-w64-x86_64-python-pytest mingw-w64-x86_64-python-setuptools mingw-w64-x86_64-python-lxml mingw-w64-x86_64-swig mingw-w64-x86_64-python-psutil mingw-w64-x86_64-blosc mingw-w64-x86_64-libavif - name: Setup cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 id: cache with: path: ${{ github.workspace }}\.ccache @@ -610,7 +610,7 @@ jobs: - name: Checkout GDAL uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - name: Setup cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 id: cache with: path: ${{ github.workspace }}/.ccache diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 59df41621efd..90c4cf32f1a7 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -45,7 +45,7 @@ jobs: if: matrix.platform == 'windows-latest' - name: Cache Conda Environment - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: path: ~/conda_pkgs_dir key: ${{ runner.os }}-${{ steps.get-date.outputs.today }}-conda-${{ env.CACHE_NUMBER }} diff --git a/.github/workflows/linux_build.yml b/.github/workflows/linux_build.yml index 53319ec41e54..2250178703a5 100644 --- a/.github/workflows/linux_build.yml +++ b/.github/workflows/linux_build.yml @@ -237,7 +237,7 @@ jobs: # different architecture. - name: Restore build cache id: restore-cache - uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: path: ${{ github.workspace }}/.ccache key: ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }}-${{ github.run_id }} @@ -298,7 +298,7 @@ jobs: ccache -s - name: Save build cache - uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: path: ${{ github.workspace }}/.ccache key: ${{ steps.restore-cache.outputs.cache-primary-key }} diff --git a/.github/workflows/slow_tests.yml b/.github/workflows/slow_tests.yml index 68251670fd4e..e723f8aa9f23 100644 --- a/.github/workflows/slow_tests.yml +++ b/.github/workflows/slow_tests.yml @@ -107,7 +107,7 @@ jobs: # different architecture. - name: Restore build cache id: restore-cache - uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: path: ${{ github.workspace }}/.ccache key: ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }}-${{ github.run_id }} @@ -147,7 +147,7 @@ jobs: ccache -s - name: Save build cache - uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: path: ${{ github.workspace }}/.ccache key: ${{ steps.restore-cache.outputs.cache-primary-key }} From dd0fe602010b4bc728b5b550280d63582ca8a158 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 02:05:32 +0000 Subject: [PATCH 23/54] Bump actions/setup-python from 5.2.0 to 5.3.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.2.0 to 5.3.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/f677139bbe7f9c59b41e40162b753c062f5d49a3...0b93645e9fea7318ecaed2b359559ac225c90a2b) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/code_checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index 85a5383288f7..3b2fbf0ee127 100644 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -153,7 +153,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 doxygen: @@ -194,7 +194,7 @@ jobs: - name: Checkout uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - name: Set up Python - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: 3.8 - name: Install lint tool From 770a4a44c1ab10667cf7b0b00da4cb03d35cc8a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 02:05:36 +0000 Subject: [PATCH 24/54] Bump actions/upload-artifact from 4.4.0 to 4.4.3 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.4.0 to 4.4.3. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/50769540e7f4bd5e21e526ee35c689e35e0d6874...b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/cifuzz.yml | 2 +- .github/workflows/conda.yml | 2 +- .github/workflows/linux_build.yml | 4 ++-- .github/workflows/scorecard.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index 5ee7e6b14209..218911710dd0 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -32,7 +32,7 @@ jobs: fuzz-seconds: 600 dry-run: false - name: Upload Crash - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 if: failure() && steps.build.outcome == 'success' with: name: artifacts diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 59df41621efd..c1ee0c8eb6c4 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -74,7 +74,7 @@ jobs: source ../ci/travis/conda/compile.sh working-directory: ./gdal-feedstock - - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: ${{ matrix.platform }}-conda-package path: ./gdal-feedstock/packages/ diff --git a/.github/workflows/linux_build.yml b/.github/workflows/linux_build.yml index 53319ec41e54..c8f79511ded8 100644 --- a/.github/workflows/linux_build.yml +++ b/.github/workflows/linux_build.yml @@ -364,14 +364,14 @@ jobs: docker push ${CONTAINER_NAME_FULL} - name: Upload coverage artifacts - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 if: ${{ matrix.id == 'coverage' }} with: name: coverage_index.html path: build-${{ matrix.id }}/coverage_html/index.html - name: Upload coverage artifacts - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 if: ${{ matrix.id == 'coverage' }} with: name: HTML diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 00a2de677699..ab523336269e 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -63,7 +63,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: SARIF file path: results.sarif From 190bd1d859cd6dc22c360dc23f5f69dfe551d749 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 02:05:39 +0000 Subject: [PATCH 25/54] Bump coverallsapp/github-action from 2.3.0 to 2.3.4 Bumps [coverallsapp/github-action](https://github.com/coverallsapp/github-action) from 2.3.0 to 2.3.4. - [Release notes](https://github.com/coverallsapp/github-action/releases) - [Commits](https://github.com/coverallsapp/github-action/compare/643bc377ffa44ace6394b2b5d0d3950076de9f63...cfd0633edbd2411b532b808ba7a8b5e04f76d2c8) --- updated-dependencies: - dependency-name: coverallsapp/github-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/linux_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_build.yml b/.github/workflows/linux_build.yml index 53319ec41e54..1c0886f042d6 100644 --- a/.github/workflows/linux_build.yml +++ b/.github/workflows/linux_build.yml @@ -349,7 +349,7 @@ jobs: ${TEST_CMD} - name: Coveralls - uses: coverallsapp/github-action@643bc377ffa44ace6394b2b5d0d3950076de9f63 # v2.3.0 + uses: coverallsapp/github-action@cfd0633edbd2411b532b808ba7a8b5e04f76d2c8 # v2.3.4 if: ${{ matrix.id == 'coverage' }} with: format: lcov From 19f210ab24b0d8c0e8fd171a129f1d059a6eba5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 02:05:48 +0000 Subject: [PATCH 26/54] Bump github/codeql-action from 3.26.10 to 3.27.0 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.26.10 to 3.27.0. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/e2b3eafc8d227b0241d48be5f425d47c2d750a13...662472033e021d55d94146f66f6058822b0b39fd) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 4 ++-- .github/workflows/scorecard.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index cbdf13b7bed7..f26c7e499044 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -109,7 +109,7 @@ jobs: # We do that after running CMake to avoid CodeQL to trigger during CMake time, # in particular during HDF5 detection which is terribly slow (https://github.com/OSGeo/gdal/issues/9549) - name: Initialize CodeQL - uses: github/codeql-action/init@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 + uses: github/codeql-action/init@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -129,6 +129,6 @@ jobs: cmake --build build -j$(nproc) - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 + uses: github/codeql-action/analyze@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 00a2de677699..8bb3b02b2f3e 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -71,6 +71,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 + uses: github/codeql-action/upload-sarif@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 with: sarif_file: results.sarif From ee0b2f0ad786390608d8476215b5be203aab9a1a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 1 Nov 2024 13:37:23 +0100 Subject: [PATCH 27/54] SQL expression tree to string: add missing parenthesis that could make further evaluation of operator priority wrong --- autotest/cpp/test_ogr_swq.cpp | 9 ++--- ogr/swq_expr_node.cpp | 72 ++++++++++++++++++----------------- 2 files changed, 41 insertions(+), 40 deletions(-) diff --git a/autotest/cpp/test_ogr_swq.cpp b/autotest/cpp/test_ogr_swq.cpp index d10ca8c929d8..483ce55fa113 100644 --- a/autotest/cpp/test_ogr_swq.cpp +++ b/autotest/cpp/test_ogr_swq.cpp @@ -138,11 +138,10 @@ class PushNotOperationDownToStackFixture std::make_tuple("NOT(1 <= 2)", "1 > 2"), std::make_tuple("NOT(1 < 2)", "1 >= 2"), std::make_tuple("NOT(NOT(1))", "1"), - std::make_tuple("NOT(1 AND 2)", "(NOT (1)) OR (NOT (2))"), - std::make_tuple("NOT(1 OR 2)", "(NOT (1)) AND (NOT (2))"), - std::make_tuple("3 AND NOT(1 OR 2)", - "3 AND ((NOT (1)) AND (NOT (2)))"), - std::make_tuple("NOT(NOT(1 = 2) OR 2)", "(1 = 2) AND (NOT (2))"), + std::make_tuple("NOT(1 AND 2)", "(NOT 1) OR (NOT 2)"), + std::make_tuple("NOT(1 OR 2)", "(NOT 1) AND (NOT 2)"), + std::make_tuple("3 AND NOT(1 OR 2)", "3 AND ((NOT 1) AND (NOT 2))"), + std::make_tuple("NOT(NOT(1 = 2) OR 2)", "(1 = 2) AND (NOT 2)"), std::make_tuple("1", "1"), }; } diff --git a/ogr/swq_expr_node.cpp b/ogr/swq_expr_node.cpp index feb0124ecdc1..3cbedff6785e 100644 --- a/ogr/swq_expr_node.cpp +++ b/ogr/swq_expr_node.cpp @@ -664,6 +664,21 @@ CPLString swq_expr_node::UnparseOperationFromUnparsedSubExpr(char **apszSubExpr) return osExpr; } + const auto AddSubExpr = [this, apszSubExpr, &osExpr](int idx) + { + if (papoSubExpr[idx]->eNodeType == SNT_COLUMN || + papoSubExpr[idx]->eNodeType == SNT_CONSTANT) + { + osExpr += apszSubExpr[idx]; + } + else + { + osExpr += '('; + osExpr += apszSubExpr[idx]; + osExpr += ')'; + } + }; + switch (nOperation) { // Binary infix operators. @@ -683,63 +698,52 @@ CPLString swq_expr_node::UnparseOperationFromUnparsedSubExpr(char **apszSubExpr) case SWQ_DIVIDE: case SWQ_MODULUS: CPLAssert(nSubExprCount >= 2); - if (papoSubExpr[0]->eNodeType == SNT_COLUMN || - papoSubExpr[0]->eNodeType == SNT_CONSTANT) - { - osExpr += apszSubExpr[0]; - } - else - { - osExpr += "("; - osExpr += apszSubExpr[0]; - osExpr += ")"; - } + AddSubExpr(0); osExpr += " "; osExpr += poOp->pszName; osExpr += " "; - if (papoSubExpr[1]->eNodeType == SNT_COLUMN || - papoSubExpr[1]->eNodeType == SNT_CONSTANT) - { - osExpr += apszSubExpr[1]; - } - else - { - osExpr += "("; - osExpr += apszSubExpr[1]; - osExpr += ")"; - } + AddSubExpr(1); if ((nOperation == SWQ_LIKE || nOperation == SWQ_ILIKE) && nSubExprCount == 3) - osExpr += CPLSPrintf(" ESCAPE (%s)", apszSubExpr[2]); + { + osExpr += " ESCAPE "; + AddSubExpr(2); + } break; case SWQ_NOT: CPLAssert(nSubExprCount == 1); - osExpr.Printf("NOT (%s)", apszSubExpr[0]); + osExpr = "NOT "; + AddSubExpr(0); break; case SWQ_ISNULL: CPLAssert(nSubExprCount == 1); - osExpr.Printf("%s IS NULL", apszSubExpr[0]); + AddSubExpr(0); + osExpr += " IS NULL"; break; case SWQ_IN: - osExpr.Printf("%s IN (", apszSubExpr[0]); + AddSubExpr(0); + osExpr += " IN("; for (int i = 1; i < nSubExprCount; i++) { if (i > 1) osExpr += ","; - osExpr += "("; - osExpr += apszSubExpr[i]; - osExpr += ")"; + AddSubExpr(i); } osExpr += ")"; break; case SWQ_BETWEEN: CPLAssert(nSubExprCount == 3); - osExpr.Printf("%s %s (%s) AND (%s)", apszSubExpr[0], poOp->pszName, - apszSubExpr[1], apszSubExpr[2]); + AddSubExpr(0); + osExpr += ' '; + osExpr += poOp->pszName; + osExpr += ' '; + AddSubExpr(1); + osExpr += " AND "; + AddSubExpr(2); break; case SWQ_CAST: @@ -760,7 +764,7 @@ CPLString swq_expr_node::UnparseOperationFromUnparsedSubExpr(char **apszSubExpr) osExpr += apszSubExpr[i] + 1; } else - osExpr += apszSubExpr[i]; + AddSubExpr(i); if (i == 1 && nSubExprCount > 2) osExpr += "("; @@ -779,9 +783,7 @@ CPLString swq_expr_node::UnparseOperationFromUnparsedSubExpr(char **apszSubExpr) { if (i > 0) osExpr += ","; - osExpr += "("; - osExpr += apszSubExpr[i]; - osExpr += ")"; + AddSubExpr(i); } osExpr += ")"; break; From 17bd93433d68009dbb78845b9daff84d45ad5496 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 1 Nov 2024 13:37:58 +0100 Subject: [PATCH 28/54] OGR SQL: fix wrong handling of NULL expression in AND and NOT (3.10.0 regression) Fixes https://github.com/ropensci/osmextract/issues/298 (provided they also fix their SQL expressions) --- autotest/ogr/ogr_sql_test.py | 7 +++++++ ogr/swq_op_general.cpp | 7 ++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/autotest/ogr/ogr_sql_test.py b/autotest/ogr/ogr_sql_test.py index bf2581e52d8b..fe517750e5d3 100755 --- a/autotest/ogr/ogr_sql_test.py +++ b/autotest/ogr/ogr_sql_test.py @@ -1952,6 +1952,13 @@ def get_available_dialects(): ("(NOT intfield = 0) AND NOT (intfield IS NULL)", 1), ("NOT (intfield = 0 OR intfield IS NOT NULL)", 0), ("(NOT intfield = 0) AND NOT (intfield IS NOT NULL)", 0), + ("intfield <> 0 AND intfield <> 2", 1), + ("intfield IS NOT NULL AND intfield NOT IN (2)", 1), + ("NOT(intfield NOT IN (1) AND NULL NOT IN (1))", 1), + ("NOT(intfield IS NOT NULL AND intfield NOT IN (2))", 1), + ("NOT(NOT(intfield IS NOT NULL AND intfield NOT IN (2)))", 1), + ("NOT (intfield = 0 AND intfield = 0)", 1), + ("(intfield NOT IN (1) AND NULL NOT IN (1)) IS NULL", 1), # realfield ("1 + realfield >= 0", 1), ("realfield = 0", 0), diff --git a/ogr/swq_op_general.cpp b/ogr/swq_op_general.cpp index 6d85f39df5cc..809161897c78 100644 --- a/ogr/swq_op_general.cpp +++ b/ogr/swq_op_general.cpp @@ -521,6 +521,7 @@ swq_expr_node *SWQGeneralEvaluator(swq_expr_node *node, poRet->field_type = node->field_type; if (node->nOperation != SWQ_ISNULL && node->nOperation != SWQ_OR && + node->nOperation != SWQ_AND && node->nOperation != SWQ_NOT && node->nOperation != SWQ_IN) { for (int i = 0; i < node->nSubExprCount; i++) @@ -543,6 +544,8 @@ swq_expr_node *SWQGeneralEvaluator(swq_expr_node *node, case SWQ_AND: poRet->int_value = sub_node_values[0]->int_value && sub_node_values[1]->int_value; + poRet->is_null = + sub_node_values[0]->is_null && sub_node_values[1]->is_null; break; case SWQ_OR: @@ -553,7 +556,9 @@ swq_expr_node *SWQGeneralEvaluator(swq_expr_node *node, break; case SWQ_NOT: - poRet->int_value = !sub_node_values[0]->int_value; + poRet->int_value = !sub_node_values[0]->int_value && + !sub_node_values[0]->is_null; + poRet->is_null = sub_node_values[0]->is_null; break; case SWQ_EQ: From f7836d163b483272c31acd4f616b10b6f47f69d2 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 1 Nov 2024 13:43:37 +0100 Subject: [PATCH 29/54] MIGRATION_GUIDE.TXT: mention change w.r.t OGR SQL and NULL values --- MIGRATION_GUIDE.TXT | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/MIGRATION_GUIDE.TXT b/MIGRATION_GUIDE.TXT index 1e3f841f6aa4..6cc284dd316b 100644 --- a/MIGRATION_GUIDE.TXT +++ b/MIGRATION_GUIDE.TXT @@ -1,6 +1,14 @@ MIGRATION GUIDE FROM GDAL 3.9 to GDAL 3.10 ------------------------------------------ +- The OGR SQL parser has been modified to evaluate NULL values in boolean + operations similarly to other SQL engines (SQLite, PostgreSQL, etc.). Previously, + with a foo=NULL field, expressions ``foo NOT IN ('bar')`` and ``foo NOT LIKE ('bar')`` + would evaluate as true. Now the result is false (with the NULL state being + propagated to it). Concretely, to get the same results as in previous versions, + the above expressions must be rewritten as ``foo IS NULL OR foo NOT IN ('bar')`` + and ``foo IS NULL OR foo NOT LIKE ('bar')``. + - MEM driver: opening a dataset with the MEM::: syntax is now disabled by default because of security implications. This can be enabled by setting the GDAL_MEM_ENABLE_OPEN build or configuration option. Creation of a 0-band MEM From 779871e56134111d61f1fe2859b8d19f8f04fcdf Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 1 Nov 2024 16:42:51 +0100 Subject: [PATCH 30/54] NEWS.md: update 3.10.0 news [ci skip] --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index 4cb6db444f56..a5317b1571b1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -389,6 +389,8 @@ Zarr driver: * SQL SQLite dialect: fix translation of "x IN (NULL)" with "recent" (at least > 3.31.1) versions of SQLite3 * OGRSQL: fix compliance of NOT and IN operators regarding NULL values +* OGRSQL: SQL expression tree to string: add missing parenthesis that could make + further evaluation of operator priority wrong * OGRSQL: add SELECT expression [AS] OGR_STYLE HIDDEN to be able to specify feature style string without generating a visible column (#10259) * OGRSQL: use Kahan-Babuska-Neumaier algorithm for accurate SUM() From 2211f7aa514844cb4718d4188d49899408106497 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 1 Nov 2024 14:18:49 -0400 Subject: [PATCH 31/54] Correct typo in CI test name --- .github/workflows/clang_static_analyzer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clang_static_analyzer.yml b/.github/workflows/clang_static_analyzer.yml index 41c86a7237a5..da7ae7c33f13 100644 --- a/.github/workflows/clang_static_analyzer.yml +++ b/.github/workflows/clang_static_analyzer.yml @@ -1,4 +1,4 @@ -name: CLang Static Analyzer +name: Clang Static Analyzer on: push: From d130c5c6f46930d0710e8c01421b07571ace64c5 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 2 Nov 2024 03:50:27 +0100 Subject: [PATCH 32/54] Attempt at fixing Coverity warnings about underflow/overflow relate to use of CPLGetUsablePhysicalRAM() --- frmts/gtiff/tifvsi.cpp | 10 ++++++++-- ogr/ogrsf_frmts/generic/ograrrowarrayhelper.cpp | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/frmts/gtiff/tifvsi.cpp b/frmts/gtiff/tifvsi.cpp index 00333d2e72e9..afc77e171e2e 100644 --- a/frmts/gtiff/tifvsi.cpp +++ b/frmts/gtiff/tifvsi.cpp @@ -430,13 +430,19 @@ static void VSI_TIFFSetOpenOptions(TIFFOpenOptions *opts) #if defined(INTERNAL_LIBTIFF) || TIFFLIB_VERSION > 20230908 // Read-once and stored in static storage otherwise affects // autotest/benchmark/test_gtiff.py::test_gtiff_byte - static const GIntBig nMemLimit = []() + static const GIntBig nMemLimit = []() -> GIntBig { if (const char *pszLimit = CPLGetConfigOption("GTIFF_MAX_CUMULATED_MEM_USAGE", nullptr)) return CPLAtoGIntBig(pszLimit); else - return CPLGetUsablePhysicalRAM() * 9 / 10; + { + const auto nUsableRAM = CPLGetUsablePhysicalRAM(); + if (nUsableRAM > 0) + return nUsableRAM / 10 * 9; + else + return 0; + } }(); if (nMemLimit > 0 && nMemLimit < std::numeric_limits::max()) { diff --git a/ogr/ogrsf_frmts/generic/ograrrowarrayhelper.cpp b/ogr/ogrsf_frmts/generic/ograrrowarrayhelper.cpp index e1330890510a..e17a378e68ba 100644 --- a/ogr/ogrsf_frmts/generic/ograrrowarrayhelper.cpp +++ b/ogr/ogrsf_frmts/generic/ograrrowarrayhelper.cpp @@ -32,8 +32,8 @@ nMemLimit = atoi(pszOGR_ARROW_MEM_LIMIT); else { - const uint64_t nUsableRAM = CPLGetUsablePhysicalRAM(); - if (nUsableRAM > 0 && nUsableRAM / 4 < nMemLimit) + const auto nUsableRAM = CPLGetUsablePhysicalRAM(); + if (nUsableRAM > 0 && static_cast(nUsableRAM / 4) < nMemLimit) nMemLimit = static_cast(nUsableRAM / 4); } return nMemLimit; From 26d58854fe0ddbdf578f73799b85111b9b2c405e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 2 Nov 2024 03:25:43 +0100 Subject: [PATCH 33/54] ADBC: add a PRELUDE_STATEMENTS open option typically to load DuckDB spatial --- autotest/ogr/data/parquet/poly.parquet | Bin 0 -> 10227 bytes autotest/ogr/ogr_adbc.py | 28 +++++++++++++++++++++ doc/source/drivers/vector/adbc.rst | 8 ++++++ docker/ubuntu-full/Dockerfile | 5 ++++ ogr/ogrsf_frmts/adbc/ogradbcdataset.cpp | 9 +++++++ ogr/ogrsf_frmts/adbc/ogradbcdrivercore.cpp | 3 +++ 6 files changed, 53 insertions(+) create mode 100644 autotest/ogr/data/parquet/poly.parquet diff --git a/autotest/ogr/data/parquet/poly.parquet b/autotest/ogr/data/parquet/poly.parquet new file mode 100644 index 0000000000000000000000000000000000000000..43aac1f46aef940ba4b66378c31c203adbb249c7 GIT binary patch literal 10227 zcmbVS3vgRknLe_w9jQ%VP{~LT}pSeutUKqk0sdw(w3o2yA1pN zSF)`brzvQ1kI&;j|MUC*bFP#ri%-|db;diB6P^1z*>#Kw-(-9L%rarTx6 z_k5S}pZM%Oj9vW7HLU&WCx6H~S}GMLo%--k{`jLp@m0pAWKSpS*+Jbh!ifii-W|kg6A4;sNwzZhzKEpIB_LOE1$3DAH%|3_c zaXgRW`81x7?^ECUDQu(9WA#7mPqa`!)mZ6Pf4tFyF1yTPaaWBlsyeHxO~tLs`YL0m{JS$^r;~1Lx+tKhA8cxvO zw|U#N96yL|t;>haRU5kO3)93WfG*!RNwePpw|8sI-1-ILaawaPxBdb#dWabP0Cd%F zJKD?~|0UY}xAwLUHoxy8S39ENxsCfYyr5P2rd6Y-7+1YrGtF&$QDafpzAUy$9Dfhg z{@AS~Zi#rEMYEnI!3S@Hf#y5}do*TN8&IjurFUYgrU%pYM~Kcq(tc*arl^s$l1n2t$PKPiet?Bc9m#RiIPgZO9l0~7gQ%6Vb|Lp zww_|*8WW#@&qtqQ)|c4mwJK4k5~o$1d3fLyyQNR3Hs5GIe68AiwqF+1>to{(I)4%R z@E4(Mn?|eVPunOsSCc`U>spR=p;rx|sqsWtqJ>@%s|MEC0ImHHjaiGCogsn7u9B7Pl0^B?-W-hWos{yo31m$CmREIspmU~u)O}ZeNzfn z`0HskGYsflL!N!WKt}#VGsV@O(adujKO%jW{o;w!DP}#^_1-uuJ$(KdHeZ);rJK&u4QHb)J!SOF7DH^ zucA}E2MtgDG(UR&cO3g9&6PC;Zv8=;Qvv(6_Y%p2m^1$)+`ImqzwKUb{ZVwQZx0|$ z<}>Kd-xj2ZeGR&N6y&vUQS&`C^9Y!FJr~kw+0SUp-WeoH?eyK;Mp4tj%mbhhGzG>( zGj@P7p6f#z{uV*6c3?Zup{2Zr6@-TBZRoAX_jMfXR0|u(qSikI*ljMARbr!eG4Ww0 zKF!2)OjN1FrhF40R*6rm#B(b3#)jwcDUScUzPr@s-%}!OAbYvmb!e&rHP5Ds>iLh{ z^7C(T{3BhjUd5TmF>C%h8s3a%{$|%Sw?2bj?JO{}k%hf>ntzO%z0kD|&>Ju`zkr5O zm6`u3npy(Z@xP{pxxSY;)~Y$f)lA6Njn~M@pLFRs{&jfB{L9(4ixgEY_nj~_thT{5 z?>tW4y_J?9Jwa;wh$9JDk6?2C8@D5`Sk}T-&w?5MJv1B7(dvH@-HizLd34zOX!IL2 z8@;hw~5!aj|%OguU#y*i}$sQPqwr5(eJm5FSm== z?Gc4NqHm9|u3{$!WesZ!KCMD>zyHY_Zc>ZmYT!;=JLpu7)#{!BpJnG6&MtFtnChI_ zJgbj9w4uM?{PEAuJHPtUdFQFq=bc~v#d+uTJr|sBIex);KYzjb8oS_Z1HdmZs605_>AO=KeUICr_@q55?Eqx1)b|V$D5>zACcj{)n{Z{y&WWR4nc4=7@&G zZjauS*e~6C!Feuz!Fh1ug7a3*1?PjWpLc$vaKX9wwF}P22L9Li>IW`3&zg6$mR7qt zq9?K2qcr*?b?6f(18#3vO}HM95Ax>jstbAI$OH-cfF^j>jN#lA5`_KuXH&-(2uJ)>ujOx zbO-)0MjW{Y$A&oy&mXjOeX^zZf!5ypS`KyZa;t_rcbR3o3)tHbu<4?@)~#LlxAdNE z>-`dUXlVEA@*8v8a<0Pezs+4gx}o>SH}(E&d*7~Bs_RM|xik!l`xOSiP<8!U)%!qi z?`IAldV}oTU8l=1r+_vwf8UYb`)>yG?W`kIZjUbGzRNHth%_<(ZC~#_ePF(wTf}^O z^ye|JT*~~#TY6u92bgbX5Ha5#?ZUk4-zK~E_o};o(7&It0~6g>C*o3Ow4Phtr&V|K z#PXTNY_8NZdPff>82v3*3UbeAk585|J^ek)sdzcrGpf_|_Y{h9ELBS7b3LQ_JNtX; zdDpbVl`G2$xj1BK-Uh-(PBu~c%f(bK(L^?M=bd-5?p7pc_vvfz=vk7BbvZq{LEWGh zgoR=%D-|nqb$y}UW*%d$$%d_2*dga)`8bIg?Fk*T5?xWT^QOA_m`;|9>wT;cO4Aw`zTif}2F<8>ze5Xj|BCRHfqAs@bJ2(_e&6<#-@ zCl8n8Y-%nmrSru(X(a_u4e5u6jYIlDy=kbwCzYd^$a5z%Qn@VW6t@Z{)1Y23n1&1p zAB7=bgukV-JXfle%CcfaX))UH)s}n(R&^2Y+R|jxN*?@E<;6G|C`F@1c`2p%4sM;0 z%9xC&fI{*nW^(x=u-V_Um`jy0jLV6lEce{mzYEuVE^#@wNCf1!N zEtZpcM79E%wkf{@v_KjnZxDuu>z+`&r6{?j%jK|uqbZqh4$4eX%9Rvv^B%bvlgjxb zBwRtXX+*1tE=YuqGGf&;y6s)KPO+ZAfdDNQ0YzAlND-Eoq|72IZ`x_{6@`wBXmzl6 z%fyLIFTatT0pTx^BQ(qL$E1wRpOj*74G*MW&Lr)i$z&SR>V#q4@Rm>tYvsxgg>A-W znRY2tl6itgy>e}OO-r9G=5)^>=w2aOU`{Es<+wU?1;Z-oSI)^XsaTdv zDT!dPGb3$VrtZi?8;CETE_Dwy+2*Arty{H&k#`599qHWQL}rv?dF4#9A&YaeOosycaC6o;q{Hk9DuSq56lqwoX$@VhQHi#WYvtMvYn6HIq08BB>&iUV z>3+4_x-zfq-+i5~PCX~c7SuU8SKd5CVu&`XV=R-8Ei9)>l&hO}hB-Q@sBHF3|UYdS?!%_MeEfW@tWf~8Cj#(@Qi#1`fTPz97SVEbzS}$2&rb*k@ zx?|a0AEPDu35#oUT$!2y|K+W5dKv`IQYxMeR$@YCDVlN(xze^pkKeHDN-d{iPIn<1 zvRCAwDIFD7bkUF@HJNhfA|X>bGV80zepktr^)5x7!E!X`NloTTsfZ9$jfb>~{c&ezxh^N3fgdv9nGR%w*~qLr z6U}-vzMMN54F!nzHZQ$<0dg0^LLd>)2P=q=(d95@Nsma#u65bm6=~L<PAtXJ!Gq zQyhY&s3Wr+6HJB3*rmEnhcbwjWo2^QQ;(|Fd>$0HJ2DM_mItHfwQf^Wn6t@*{M-mR6!;G-P$2MuT!A|di+Aa&Be zGVID4l^9^&32ZL_+l6q&T8awB+?8aKZ0NsJW(GI`((#THG8W>9hb!yLMD*@V#P2rE z&T0{_%z`UjfNZ-i?no{X?vPKwV{kg4jS)}E8^V^>laA?6DtnM?xkx6K^VV%fzHQ@J z!9*b@BoZ@O_$p)qo|Y3ny_@g=c{*S(xwI1-K_~E+^VZ|+w|WupG<>=gQ!w?Wbu>4}6x%2L zyZMmpdDFTXpxX-huNhl24%1>p$RJPQW6CWD^4%G%kk5dZ0@fCl_(8Wga59K3Kjp|(q;5dP}YU8tr&6t(2kwdN%CWLTkMMw3-GiER%2L;c#t>Rs^>6>*^Fy~w# z`psI9jL-V=WLF&Xz_~qO_c}@UumHS+-{g2=**l9o#(A)fDT;#^{s|_L%SsJz*1;X%kT)l8p&e0|SY;V}BqCRVW~quJ?1K=3Zb9V^r?Or)&tn6Of> z6*e7WZbdEjQch*+wz^YMeL|@NID@ICpKLy}Cezc#iF8bdT+v5>iCJL_0k?I$W1V7HuTOL~<4)!QK5_o9#_gs{&Zfz<3w!N!2EXZ?qcg=bZbdBu=Ckf4 z8D~^zby|-*!iv{#(0Nu-SNzLb?`pV0wP7M1D0#;dE8g_94)=o-_4vZ*}8o62Upyf_qrp0qmx2D};xD-R-3-EC`M zK<~pDjr$+f;q5)brhQ{|`{+(Lw0R%5SOR39f!=7M^qMlhjMo>7hsM;gI96{tdUOOG zo0ncpEaTJ=VPk!Qm|BUzGDD43dEZg|)$Dr(AGAK120zmXy-Ly0vZAzMY883Pu+Cq` z`-H5Ft&0tMtL0c4T}2#z?V(M|OPA8EZ>4xyfmgdh8M})OdYeTzg|4EWhlnf=g+S>l zJw?7#!M8GAS!&Qv!yDFV%a}47tGfWi+d}L7oV)T%Qodj%O#2~HF%@H*CAO*0^t7VH z&i#r!!eb*J74}}T|CSgU@}Je}Q)Xk-Hu+CswS|*uQWYi;k!RC?wlPKjx>4JvDEe9L z(?Rb^e}L3Vki7_ftYJSf!RszQ*aYw-C + + SQL-like statement recognized by the driver that must be executed before + discovering layers. Can be repeated multiple times. + For example ``PRELUDE_STATEMENTS=INSTALL spatial`` and + ``PRELUDE_STATEMENTS=LOAD spatial`` to load DuckDB spatial extension. + "table_list" special layer -------------------------- diff --git a/docker/ubuntu-full/Dockerfile b/docker/ubuntu-full/Dockerfile index e0747eb3d227..fe3f5ada82eb 100644 --- a/docker/ubuntu-full/Dockerfile +++ b/docker/ubuntu-full/Dockerfile @@ -545,3 +545,8 @@ COPY --from=builder /build_gdal_python/usr/ /usr/ COPY --from=builder /build_gdal_version_changing/usr/ /usr/ RUN ldconfig + +# Install duckdb_spatial +RUN if test "$(uname -p)" = "x86_64"; then \ + ogrinfo ADBC::memory: -oo ADBC_DRIVER=duckdb -oo PRELUDE_STATEMENTS="INSTALL spatial"; \ + fi diff --git a/ogr/ogrsf_frmts/adbc/ogradbcdataset.cpp b/ogr/ogrsf_frmts/adbc/ogradbcdataset.cpp index f902795f1068..66d822167c1a 100644 --- a/ogr/ogrsf_frmts/adbc/ogradbcdataset.cpp +++ b/ogr/ogrsf_frmts/adbc/ogradbcdataset.cpp @@ -341,6 +341,15 @@ bool OGRADBCDataset::Open(const GDALOpenInfo *poOpenInfo) return false; } + char **papszPreludeStatements = CSLFetchNameValueMultiple( + poOpenInfo->papszOpenOptions, "PRELUDE_STATEMENTS"); + for (const char *pszStatement : + cpl::Iterate(CSLConstList(papszPreludeStatements))) + { + CreateLayer(pszStatement, "temp"); + } + CSLDestroy(papszPreludeStatements); + std::string osLayerName = "RESULTSET"; std::string osSQL; const char *pszSQL = CSLFetchNameValue(poOpenInfo->papszOpenOptions, "SQL"); diff --git a/ogr/ogrsf_frmts/adbc/ogradbcdrivercore.cpp b/ogr/ogrsf_frmts/adbc/ogradbcdrivercore.cpp index f9e5a295aefa..e668ac1f3f12 100644 --- a/ogr/ogrsf_frmts/adbc/ogradbcdrivercore.cpp +++ b/ogr/ogrsf_frmts/adbc/ogradbcdrivercore.cpp @@ -89,6 +89,9 @@ void OGRADBCDriverSetCommonMetadata(GDALDriver *poDriver) "description='SQL statement from which to build layer'/>" "