Skip to content

Commit

Permalink
Configurable ImageIO output format (openstreetmap#318)
Browse files Browse the repository at this point in the history
* Added the possibility to provide ImageIO's output format in the TYPE string.
For instance:
TYPE=png image/png
TYPE=png image/png png256
TYPE=png image/png png24
TYPE=png image/png png

* Removed useless comments.
Added documentation for output format.

* Added tests for 'webp' output format

* Also added tests for 'png32' output format

* Fix 'webp' sums vary based on version

* Add JPG & use multiple layers on example-map

---------

Co-authored-by: Tim <[email protected]>
  • Loading branch information
hummeltech and timautin authored Jul 20, 2023
1 parent 2287afe commit f652ab1
Show file tree
Hide file tree
Showing 8 changed files with 155 additions and 47 deletions.
4 changes: 2 additions & 2 deletions etc/renderd/renderd.conf.examples
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ XML=/var/www/example-map/mapnik.xml
;** config options used by mod_tile, but not renderd **
;MINZOOM=0
;MAXZOOM=18
;TYPE=png image/png
;TYPE=png image/png png256 ; Values are: <extension> <mime-type> <output-format> (for more information about output format see https://github.com/mapnik/mapnik/wiki/Image-IO)
;DESCRIPTION=This is a description of the tile layer used in the tile json request
;ATTRIBUTION=&copy;<a href=\"http://www.openstreetmap.org/\">OpenStreetMap</a> and <a href=\"http://wiki.openstreetmap.org/wiki/Contributors\">contributors</a>, <a href=\"http://opendatacommons.org/licenses/odbl/\">ODbL</a>
;SERVER_ALIAS=http://localhost/
Expand All @@ -65,7 +65,7 @@ XML=/var/www/example-map/mapnik.xml
;** config options used by mod_tile, but not renderd **
;MINZOOM=0
;MAXZOOM=22
;TYPE=png image/png
;TYPE=png image/png png256 ; Values are: <extension> <mime-type> <output-format> (for more information about output format see https://github.com/mapnik/mapnik/wiki/Image-IO)
;DESCRIPTION=This is a description of the tile layer used in the tile json request
;ATTRIBUTION=&copy;<a href=\"http://www.openstreetmap.org/\">OpenStreetMap</a> and <a href=\"http://wiki.openstreetmap.org/wiki/Contributors\">contributors</a>, <a href=\"http://opendatacommons.org/licenses/odbl/\">ODbL</a>
;SERVER_ALIAS=http://localhost/
Expand Down
1 change: 1 addition & 0 deletions includes/daemon.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ typedef struct {
char host[PATH_MAX];
char htcpip[PATH_MAX];
char tile_dir[PATH_MAX];
char output_format[INILINE_MAX];
char parameterization[PATH_MAX];
int tile_px_size;
double scale_factor;
Expand Down
15 changes: 13 additions & 2 deletions src/daemon.c
Original file line number Diff line number Diff line change
Expand Up @@ -1024,6 +1024,17 @@ int main(int argc, char **argv)

strcpy(maps[iconf].parameterization, ini_parameterize);

snprintf(buffer, sizeof(buffer), "%s:type", name);
const char *ini_type = iniparser_getstring(ini, buffer, "png image/png png256");

const char ini_fileExtension[INILINE_MAX] = "png";
const char ini_mimeType[INILINE_MAX] = "image/png";
const char ini_outputFormat[INILINE_MAX] = "png256";

sscanf(ini_type, "%[^ ] %[^ ] %[^;#]", ini_fileExtension, ini_mimeType, ini_outputFormat);

strcpy(maps[iconf].output_format, ini_outputFormat);

/* Pass this information into the rendering threads,
* as it is needed to configure mapniks number of connections
*/
Expand Down Expand Up @@ -1079,9 +1090,9 @@ int main(int argc, char **argv)

for (iconf = 0; iconf < XMLCONFIGS_MAX; ++iconf) {
if (maps[iconf].xmlname[0] != 0) {
g_logger(G_LOG_LEVEL_INFO, "config map %d: name(%s) file(%s) uri(%s) htcp(%s) host(%s)",
g_logger(G_LOG_LEVEL_INFO, "config map %d: name(%s) file(%s) uri(%s) output_format(%s) htcp(%s) host(%s)",
iconf, maps[iconf].xmlname, maps[iconf].xmlfile, maps[iconf].xmluri,
maps[iconf].htcpip, maps[iconf].host);
maps[iconf].output_format, maps[iconf].htcpip, maps[iconf].host);
}
}

Expand Down
10 changes: 6 additions & 4 deletions src/gen_tile.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ struct projectionconfig {
struct xmlmapconfig {
char xmlname[XMLCONFIG_MAX];
char xmlfile[PATH_MAX];
char output_format[XMLCONFIG_MAX];
struct storage_backend * store;
Map map;
struct projectionconfig * prj;
Expand Down Expand Up @@ -324,14 +325,14 @@ static enum protoCmd render(struct xmlmapconfig * map, int x, int y, int z, char
#else
mapnik::image_view<mapnik::image_data_32> vw(xx * map->tilesize, yy * map->tilesize, map->tilesize, map->tilesize, buf.data());
#endif
tiles.set(xx, yy, save_to_string(vw, "png256"));
tiles.set(xx, yy, save_to_string(vw, map->output_format));
}
}

return cmdDone; // OK
}
#else //METATILE
static enum protoCmd render(Map &m, const char *tile_dir, char *xmlname, projection &prj, int x, int y, int z)
static enum protoCmd render(Map &m, const char *tile_dir, char *xmlname, projection &prj, int x, int y, int z, char* outputFormat)
{
char filename[PATH_MAX];
char tmp[PATH_MAX];
Expand Down Expand Up @@ -365,7 +366,7 @@ static enum protoCmd render(Map &m, const char *tile_dir, char *xmlname, project

mapnik::image_view<mapnik::image_data_32> vw(128, 128, 256, 256, buf.data());
g_logger(G_LOG_LEVEL_DEBUG, "Render %i %i %i %s", z, x, y, filename)
mapnik::save_to_file(vw, tmp, "png256");
mapnik::save_to_file(vw, tmp, outputFormat);

if (rename(tmp, filename)) {
perror(tmp);
Expand Down Expand Up @@ -403,6 +404,7 @@ void *render_thread(void * arg)

strcpy(maps[iMaxConfigs].xmlname, parentxmlconfig[iMaxConfigs].xmlname);
strcpy(maps[iMaxConfigs].xmlfile, parentxmlconfig[iMaxConfigs].xmlfile);
strcpy(maps[iMaxConfigs].output_format, parentxmlconfig[iMaxConfigs].output_format);
maps[iMaxConfigs].store = init_storage_backend(parentxmlconfig[iMaxConfigs].tile_dir);
maps[iMaxConfigs].tilesize = parentxmlconfig[iMaxConfigs].tile_px_size;
maps[iMaxConfigs].scale = parentxmlconfig[iMaxConfigs].scale_factor;
Expand Down Expand Up @@ -521,7 +523,7 @@ void *render_thread(void * arg)
}

#else //METATILE
ret = render(maps[i].map, maps[i].tile_dir, req->xmlname, maps[i].prj, req->x, req->y, req->z);
ret = render(maps[i].map, maps[i].tile_dir, req->xmlname, maps[i].prj, req->x, req->y, req->z, maps[i].output_format);
#ifdef HTCP_EXPIRE_CACHE
cache_expire(maps[i].htcpsock, maps[i].host, maps[i].xmluri, req->x, req->y, req->z);
#endif
Expand Down
4 changes: 3 additions & 1 deletion src/mod_tile.c
Original file line number Diff line number Diff line change
Expand Up @@ -2103,6 +2103,7 @@ static const char *load_tile_config(cmd_parms *cmd, void *mconfig, const char *c
const char * result;
char fileExtension[INILINE_MAX];
char mimeType[INILINE_MAX];
char outputFormat[INILINE_MAX];
char * description = NULL;
char * attribution = NULL;
char * cors = NULL;
Expand Down Expand Up @@ -2214,6 +2215,7 @@ static const char *load_tile_config(cmd_parms *cmd, void *mconfig, const char *c
strcpy(url, "");
strcpy(fileExtension, "png");
strcpy(mimeType, "image/png");
strcpy(outputFormat, "png256");
description = NULL;
cors = NULL;
attribution = NULL;
Expand Down Expand Up @@ -2293,7 +2295,7 @@ static const char *load_tile_config(cmd_parms *cmd, void *mconfig, const char *c
return "TYPE too long";
}

if (sscanf(value, "%[^ ] %[^;#]", fileExtension, mimeType) != 2) {
if (sscanf(value, "%[^ ] %[^ ] %[^;#]", fileExtension, mimeType, outputFormat) < 2) {
if (description) {
free(description);
description = NULL;
Expand Down
85 changes: 60 additions & 25 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,24 @@ execute_process(COMMAND ${ID_EXECUTABLE} -gn nobody
#-----------------------------------------------------------------------------

set(MAP_NAME "default")
set(TILE_SHA256SUM "dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08")
set(TILE_URL "http://localhost:8081/tiles/renderd-example/9/297/191.png")
set(TILE_CMD "${CURL_EXECUTABLE} --fail --silent ${TILE_URL}")

set(TILE_DEFAULT_URL "http://localhost:8081/tiles/renderd-example/9/297/191.png")
set(TILE_JPG_URL "http://localhost:8081/tiles/renderd-example-jpg/9/297/191.jpg")
set(TILE_PNG256_URL "http://localhost:8081/tiles/renderd-example-png256/9/297/191.png")
set(TILE_PNG32_URL "http://localhost:8081/tiles/renderd-example-png32/9/297/191.png")
set(TILE_WEBP_URL "http://localhost:8081/tiles/renderd-example-webp/9/297/191.webp")

set(TILE_DEFAULT_CMD "${CURL_EXECUTABLE} --fail --silent ${TILE_DEFAULT_URL}")
set(TILE_DEFAULT_SHA256SUM "dbf26531286e844a3a9735cdd193598dca78d22f77cafe5824bcaf17f88cbb08")
set(TILE_JPG_CMD "${CURL_EXECUTABLE} --fail --silent ${TILE_JPG_URL}")
set(TILE_JPG_SHA256SUM "e09c3406c02f03583dadf0c8404c2d3efdc06a40d399e381ed2f47f49fde42d7")
set(TILE_PNG256_CMD "${CURL_EXECUTABLE} --fail --silent ${TILE_PNG256_URL}")
set(TILE_PNG256_SHA256SUM "${TILE_DEFAULT_SHA256SUM}")
set(TILE_PNG32_CMD "${CURL_EXECUTABLE} --fail --silent ${TILE_PNG32_URL}")
set(TILE_PNG32_SHA256SUM "1006d92152f1e18896e0016fb43201b14bbcf7655955b74495ad3610541d325b")
set(TILE_WEBP_CMD "${CURL_EXECUTABLE} --fail --silent ${TILE_WEBP_URL}")
set(TILE_WEBP_SHA256SUM_6 "96fc0455b2269a7bcd4a5b3c9844529c3c77e3bb15f56e72f78a5af3bc15b6b5") # libwebp6
set(TILE_WEBP_SHA256SUM_7 "a82ef9ba5dc333de88af7b645084c30ab2b01c664e17162cbf6659c287cc4df4") # libwebp7

configure_file(
renderd.conf.in
Expand Down Expand Up @@ -128,27 +143,47 @@ add_test(
"
)
add_test(
NAME download_tile
NAME download_tiles
COMMAND ${BASH} -c "
until $(${TILE_CMD} --output tile.png); do
until $(${TILE_DEFAULT_CMD} --output tile.png); do
echo 'Sleeping 1s';
sleep 1;
done
until $(${TILE_JPG_CMD} --output tile.jpg); do
echo 'Sleeping 1s';
sleep 1;
done
until $(${TILE_PNG256_CMD} --output tile.png256); do
echo 'Sleeping 1s';
sleep 1;
done
until $(${TILE_PNG32_CMD} --output tile.png32); do
echo 'Sleeping 1s';
sleep 1;
done
until $(${TILE_WEBP_CMD} --output tile.webp); do
echo 'Sleeping 1s';
sleep 1;
done
"
)
add_test(
NAME check_tile
NAME check_tiles
COMMAND ${BASH} -c "
echo '${TILE_SHA256SUM} tile.png' | ${SHA256SUM_EXECUTABLE} -c
(echo '${TILE_DEFAULT_SHA256SUM} tile.png' | ${SHA256SUM_EXECUTABLE} -c) && \
(echo '${TILE_JPG_SHA256SUM} tile.jpg' | ${SHA256SUM_EXECUTABLE} -c) && \
(echo '${TILE_PNG256_SHA256SUM} tile.png256' | ${SHA256SUM_EXECUTABLE} -c) && \
(echo '${TILE_PNG32_SHA256SUM} tile.png32' | ${SHA256SUM_EXECUTABLE} -c) && \
((echo '${TILE_WEBP_SHA256SUM_6} tile.webp' | ${SHA256SUM_EXECUTABLE} -c) || (echo '${TILE_WEBP_SHA256SUM_7} tile.webp' | ${SHA256SUM_EXECUTABLE} -c))
"
)
add_test(
NAME dirty_tile
COMMAND ${BASH} -c "
TILE_STATUS_CMD=\"${TILE_CMD}/status | cut -d. -f2\"
TILE_STATUS_CMD=\"${TILE_DEFAULT_CMD}/status | cut -d. -f2\"
TILE_STATUS_OUTPUT_OLD=$(\${TILE_STATUS_CMD})
sleep 5;
TILE_DIRTY_OUTPUT=$(${TILE_CMD}/dirty)
TILE_DIRTY_OUTPUT=$(${TILE_DEFAULT_CMD}/dirty)
if [ \"\${TILE_DIRTY_OUTPUT}\" != \"Tile submitted for rendering\" ]; then
exit 1;
fi
Expand All @@ -159,8 +194,8 @@ add_test(
"
)
add_test(
NAME remove_tile
COMMAND ${RM} -v tile.png
NAME remove_tiles
COMMAND ${RM} -v tile.png tile.jpg tile.png256 tile.png32 tile.webp
)
add_test(
NAME stop_renderd
Expand Down Expand Up @@ -227,24 +262,24 @@ set_tests_properties(render_old PROPERTIES
FIXTURES_REQUIRED httpd_started
TIMEOUT 20
)
set_tests_properties(download_tile PROPERTIES
set_tests_properties(download_tiles PROPERTIES
FIXTURES_REQUIRED httpd_started
FIXTURES_SETUP tile_downloaded
TIMEOUT 10
FIXTURES_SETUP tiles_downloaded
TIMEOUT 20
)
set_tests_properties(check_tile PROPERTIES
DEPENDS download_tile
FIXTURES_REQUIRED "httpd_started;tile_downloaded"
REQUIRED_FILES tile.png
set_tests_properties(check_tiles PROPERTIES
DEPENDS download_tiles
FIXTURES_REQUIRED "httpd_started;tiles_downloaded"
REQUIRED_FILES "tile.png;tile.jpg;tile.png256;tile.png32;tile.webp"
)
set_tests_properties(dirty_tile PROPERTIES
DEPENDS download_tile
FIXTURES_REQUIRED "httpd_started;tile_downloaded"
REQUIRED_FILES tile.png
DEPENDS download_tiles
FIXTURES_REQUIRED "httpd_started;tiles_downloaded"
REQUIRED_FILES "tile.png;tile.jpg;tile.png256;tile.png32;tile.webp"
TIMEOUT 20
)
set_tests_properties(remove_tile PROPERTIES
DEPENDS download_tile
FIXTURES_CLEANUP tile_downloaded
REQUIRED_FILES tile.png
set_tests_properties(remove_tiles PROPERTIES
DEPENDS download_tiles
FIXTURES_CLEANUP tiles_downloaded
REQUIRED_FILES "tile.png;tile.jpg;tile.png256;tile.png32;tile.webp"
)
24 changes: 24 additions & 0 deletions tests/renderd.conf.in
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,30 @@ TILEDIR=@PROJECT_BINARY_DIR@/tests/tiles
URI=/tiles/renderd-example
XML=@PROJECT_SOURCE_DIR@/utils/example-map/mapnik.xml

[@MAP_NAME@_jpg]
TILEDIR=@PROJECT_BINARY_DIR@/tests/tiles
TYPE=jpg image/jpeg jpeg
URI=/tiles/renderd-example-jpg
XML=@PROJECT_BINARY_DIR@/tests/www/mapnik.xml

[@MAP_NAME@_png256]
TILEDIR=@PROJECT_BINARY_DIR@/tests/tiles
TYPE=png image/png png256
URI=/tiles/renderd-example-png256
XML=@PROJECT_BINARY_DIR@/tests/www/mapnik.xml

[@MAP_NAME@_png32]
TILEDIR=@PROJECT_BINARY_DIR@/tests/tiles
TYPE=png image/png png32
URI=/tiles/renderd-example-png32
XML=@PROJECT_BINARY_DIR@/tests/www/mapnik.xml

[@MAP_NAME@_webp]
TILEDIR=@PROJECT_BINARY_DIR@/tests/tiles
TYPE=webp image/webp webp
URI=/tiles/renderd-example-webp
XML=@PROJECT_BINARY_DIR@/tests/www/mapnik.xml

[renderd1]
iphostname=127.0.0.1
ipport=8881
Expand Down
59 changes: 46 additions & 13 deletions utils/example-map/index.html
Original file line number Diff line number Diff line change
@@ -1,24 +1,57 @@
<!DOCTYPE html>
<html>
<head>
<head>
<title>renderd example map</title>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="icon" href="data:;base64,iVBORw0KGgo=">
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" href="data:;base64,iVBORw0KGgo=" />
<link rel="stylesheet" href="leaflet/leaflet.css" />
<script src="leaflet/leaflet.min.js"></script>
</head>
</head>

<body style="margin: 0;">
<div id="map" style="position: absolute; width: 100%; height: 100%;"></div>
<body style="margin: 0">
<div id="map" style="position: absolute; width: 100%; height: 100%"></div>

<script>
var map = L.map('map').setView([0, 0], 4);
L.tileLayer('http://localhost:8081/tiles/renderd-example/{z}/{x}/{y}.png', {
maxZoom: 12,
attribution: 'Map data &copy; <a href="https://www.openstreetmap.org/">OpenStreetMap</a> contributors.',
}).addTo(map);
</script>
var attribution =
'Map data &copy; <a href="https://www.openstreetmap.org/">OpenStreetMap</a> contributors.';
var options = {
maxZoom: 12,
attribution: attribution,
};

var png = L.tileLayer(
"http://localhost:8081/tiles/renderd-example/{z}/{x}/{y}.png",
options
);

var jpg = L.tileLayer(
"http://localhost:8081/tiles/renderd-example-jpg/{z}/{x}/{y}.jpg",
options
);
var png32 = L.tileLayer(
"http://localhost:8081/tiles/renderd-example-png32/{z}/{x}/{y}.png",
options
);
var webp = L.tileLayer(
"http://localhost:8081/tiles/renderd-example-webp/{z}/{x}/{y}.webp",
options
);

</body>
var map = L.map("map", {
center: [0, 0],
zoom: 4,
layers: [png],
});

var baseMaps = {
JPG: jpg,
PNG: png,
PNG32: png32,
WEBP: webp,
};

var layerControl = L.control.layers(baseMaps).addTo(map);
</script>
</body>
</html>

0 comments on commit f652ab1

Please sign in to comment.