Skip to content

Commit

Permalink
Merge pull request #55 from CesiumGS/package-entry-compression
Browse files Browse the repository at this point in the history 
Handle package entry compression
  • Loading branch information
@lilleyse
lilleyse authored Aug 17, 2023
2 parents 8bbbb34 + 5ee5e44 commit b17ddd1
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 99 deletions.
54 changes: 16 additions & 38 deletions demos/PackageSandcastle.js
View file
Original file line number Diff line number Diff line change
@@ -1,54 +1,32 @@
/* eslint-disable no-undef */
const viewer = new Cesium.Viewer("cesiumContainer", {
globe: false,
skybox: false,
skyBox: false,
});

const tileset = viewer.scene.primitives.add(
new Cesium.Cesium3DTileset({
url: `http://localhost:8003/tileset.json`,
await Cesium.Cesium3DTileset.fromUrl(`http://localhost:8003/tileset.json`, {
debugShowBoundingVolume: true,
maximumScreenSpaceError: 0.3,
maximumScreenSpaceError: 16,
})
);

class Timer {
constructor(label) {
this.label = label;
this.startTime = undefined;
this.endTime = undefined;
}

start() {
this.startTime = performance.now();
}

stop() {
this.endTime = performance.now();
}

print() {
const difference_sec = (this.endTime - this.startTime) / 1000.0;
console.log(`${this.label}: ${difference_sec} sec`);
}
}

const tileTimer = new Timer("tiles");

tileset.readyPromise.then(() => {
console.log("Tileset is ready");
tileTimer.start();
});
// Move the tileset to a certain position on the globe
const transform = Cesium.Transforms.eastNorthUpToFixedFrame(
Cesium.Cartesian3.fromDegrees(-75.152408, 39.946975, 1)
);
const scale = 1.0;
const modelMatrix = Cesium.Matrix4.multiplyByUniformScale(
transform,
scale,
new Cesium.Matrix4()
);
tileset.modelMatrix = modelMatrix;

// Zoom to the tileset (with a certain offset, to make it fully visible)
const offset = new Cesium.HeadingPitchRange(
Cesium.Math.toRadians(0.0),
Cesium.Math.toRadians(-60.0),
15000.0
1500.0
);
viewer.zoomTo(tileset, offset);

tileset.initialTilesLoaded.addEventListener(() => {
console.log("Initial tiles are loaded");
tileTimer.stop();
tileTimer.print();
});
2 changes: 1 addition & 1 deletion demos/PackageServer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ function handleRequest(
}
const content = tilesetSource.getValue(path);

//console.log("Content for " + path + " is " + content);
//console.log("Content for " + path + " is " + content?.length);

if (!content) {
console.log("Return 404 for " + path);
Expand Down
117 changes: 65 additions & 52 deletions src/packages/ArchiveFunctions3tz.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,13 @@ import { IndexEntry } from "./IndexEntry";
// NOTE: These functions are carved out and ported to TypeScript from
// https://github.com/bjornblissing/3d-tiles-tools/blob/2f4844d5bdd704509bff65199898981228594aaa/validator/lib/archive.js
// TODO: The given implementation does not handle hash collisions!
// NOTE: Fixed an issue for ZIP64 inputs. See the part marked as "ZIP64_BUGFIX"
// NOTE: Fixed several issues for ZIP64 and large ZIP files. Some of the
// changes are marked with "ZIP64_BUGFIX", but details have to be taken
// from the git change log.

interface ZipLocalFileHeader {
signature: number;
compression_method: number;
comp_size: number;
filename_size: number;
extra_size: number;
Expand Down Expand Up @@ -63,30 +66,48 @@ export class ArchiveFunctions3tz {
buffer: Buffer,
expectedFilename: string
) {
let comp_size = buffer.readUInt32LE(20);
const comp_size_from_record = buffer.readUInt32LE(20);
let comp_size = BigInt(comp_size_from_record);
const uncomp_size = buffer.readUInt32LE(24);
const filename_size = buffer.readUInt16LE(28);
const extra_size = buffer.readUInt16LE(30);
const extrasStartOffset =
ArchiveFunctions3tz.ZIP_CENTRAL_DIRECTORY_STATIC_SIZE + filename_size;

// ZIP64_BUGFIX: If this size is found, then the size is
// stored in the extra field
if (comp_size === 0xffffffff) {
if (comp_size === 0xffffffffn) {
if (extra_size < 28) {
throw new TilesetError("No zip64 extras buffer found");
}
// NOTE: The "ZIP64 header ID" might appear at a different position
// in the extras buffer, but I don't see a sensible way to
// differentiate between a 0x0001 appearing "randomly" as "some"
// value in the extras, and the value actually indicating a ZIP64
// header. So we look for it only at the start of the extras buffer:
const extra_tag = buffer.readUInt16LE(extrasStartOffset + 0);
if (
extra_tag !== ArchiveFunctions3tz.ZIP64_EXTENDED_INFORMATION_EXTRA_SIG
) {
throw new TilesetError("No zip64 extras signature found");
}
comp_size = Number(buffer.readBigUInt64LE(extrasStartOffset + 12));

// According to the specification, the layout of the extras block is
//
// 0x0001 2 bytes Tag for this "extra" block type
// Size 2 bytes Size of this "extra" block
// Original Size 8 bytes Original uncompressed file size
// Compressed Size 8 bytes Size of compressed data
// Relative Header Offset 8 bytes Offset of local header record
// Disk Start Number 4 bytes Number of the disk on which this file starts
//
// The order of the fields is fixed, but the fields MUST only appear
// if the corresponding directory record field is set to 0xFFFF or
// 0xFFFFFFFF.
// So the offset for reading values from the "extras" depends on which
// of the fields in the original record had the 0xFFFFFFFF value:
let offsetInExtrasForCompSize = 4;
if (uncomp_size === 0xffffffff) {
offsetInExtrasForCompSize += 8;
}
comp_size = buffer.readBigUInt64LE(
extrasStartOffset + offsetInExtrasForCompSize
);
}

const filename = buffer.toString(
Expand All @@ -99,49 +120,26 @@ export class ArchiveFunctions3tz {
`Central Directory File Header filename was ${filename}, expected ${expectedFilename}`
);
}

let offset = buffer.readUInt32LE(42);
/*
// if we get this offset, then the offset is stored in the 64 bit extra field
if (offset === 0xffffffff) {
let offset64Found = false;
const endExtrasOffset =
ArchiveFunctions3tz.ZIP_CENTRAL_DIRECTORY_STATIC_SIZE +
filename_size +
extra_size;
let currentOffset =
ArchiveFunctions3tz.ZIP_CENTRAL_DIRECTORY_STATIC_SIZE + filename_size;
while (!offset64Found && currentOffset < endExtrasOffset) {
const extra_tag = buffer.readUInt16LE(currentOffset);
const extra_size = buffer.readUInt16LE(currentOffset + 2);
if (
extra_tag ===
ArchiveFunctions3tz.ZIP64_EXTENDED_INFORMATION_EXTRA_SIG &&
extra_size == 8
) {
offset = Number(buffer.readBigUInt64LE(currentOffset + 4));
offset64Found = true;
} else {
currentOffset += extra_size;
}
let offset = BigInt(buffer.readUInt32LE(42));
if (offset === 0xffffffffn) {
// See notes about the layout of the extras block above:
let offsetInExtrasForOffset = 4;
if (uncomp_size === 0xffffffff) {
offsetInExtrasForOffset += 8;
}
if (!offset64Found) {
throw new TilesetError("No zip64 extended offset found");
if (comp_size_from_record === 0xffffffff) {
offsetInExtrasForOffset += 8;
}
}
*/
// if we get this offset, then the offset is stored in the 64 bit extra field.
// The size and signature of the buffer have already been checked when the
// actual "comp_size" has been read.
if (offset === 0xffffffff) {
offset = Number(buffer.readBigUInt64LE(extrasStartOffset + 20));
offset = buffer.readBigUInt64LE(
extrasStartOffset + offsetInExtrasForOffset
);
}

const localFileDataSize =
ArchiveFunctions3tz.ZIP_LOCAL_FILE_HEADER_STATIC_SIZE +
filename_size +
+48 /* over-estimated local file header extra field size, to try and read all data in one go */ +
comp_size;
Number(comp_size);
const localFileDataBuffer = Buffer.alloc(localFileDataSize);

fs.readSync(fd, localFileDataBuffer, 0, localFileDataSize, offset);
Expand Down Expand Up @@ -250,6 +248,7 @@ export class ArchiveFunctions3tz {
`Bad local file header signature: 0x${signature.toString(16)}`
);
}
const compression_method = buffer.readUInt16LE(8);
const comp_size = buffer.readUInt32LE(18);
const filename_size = buffer.readUInt16LE(26);
const extra_size = buffer.readUInt16LE(28);
Expand All @@ -273,6 +272,7 @@ export class ArchiveFunctions3tz {
}
return {
signature: signature,
compression_method: compression_method,
comp_size: comp_size,
filename_size: filename_size,
extra_size: extra_size,
Expand All @@ -288,7 +288,7 @@ export class ArchiveFunctions3tz {
ArchiveFunctions3tz.ZIP_LOCAL_FILE_HEADER_STATIC_SIZE + path.length;
const headerBuffer = Buffer.alloc(headerSize);
//console.log(`readZipLocalFileHeader path: ${path} headerSize: ${headerSize} offset: ${offset}`);
fs.readSync(fd, headerBuffer, 0, headerSize, Number(offset));
fs.readSync(fd, headerBuffer, 0, headerSize, offset);
//console.log(`headerBuffer: ${result.buffer}`);
const header = ArchiveFunctions3tz.parseLocalFileHeader(headerBuffer, path);
//console.log(header);
Expand Down Expand Up @@ -323,7 +323,7 @@ export class ArchiveFunctions3tz {
const headerSize =
ArchiveFunctions3tz.ZIP_LOCAL_FILE_HEADER_STATIC_SIZE + 320;
const headerBuffer = Buffer.alloc(headerSize);
fs.readSync(fd, headerBuffer, 0, headerSize, Number(offset));
fs.readSync(fd, headerBuffer, 0, headerSize, offset);
const filename_size = headerBuffer.readUInt16LE(26);
const filename = headerBuffer.toString(
"utf8",
Expand All @@ -333,7 +333,16 @@ export class ArchiveFunctions3tz {
return filename;
}

static readEntryData(fd: number, zipIndex: IndexEntry[], path: string) {
static readEntry(
fd: number,
zipIndex: IndexEntry[],
path: string
):
| {
compression_method: number;
data: Buffer;
}
| undefined {
const normalizedPath = ArchiveFunctions3tz.normalizePath(path);
const match = ArchiveFunctions3tz.searchIndex(zipIndex, normalizedPath);
if (match) {
Expand All @@ -343,14 +352,18 @@ export class ArchiveFunctions3tz {
path
);
const fileDataOffset =
Number(match.offset) +
ArchiveFunctions3tz.ZIP_LOCAL_FILE_HEADER_STATIC_SIZE +
header.filename_size +
header.extra_size;
match.offset +
BigInt(ArchiveFunctions3tz.ZIP_LOCAL_FILE_HEADER_STATIC_SIZE) +
BigInt(header.filename_size) +
BigInt(header.extra_size);
const fileContentsBuffer = Buffer.alloc(header.comp_size);
//console.log(`Fetching data at offset ${fileDataOffset} size: ${header.comp_size}`);
fs.readSync(fd, fileContentsBuffer, 0, header.comp_size, fileDataOffset);
return fileContentsBuffer;

return {
compression_method: header.compression_method,
data: fileContentsBuffer,
};
}
//console.log('No entry found for path ', path)
return undefined;
Expand Down
19 changes: 12 additions & 7 deletions src/packages/TilesetSource3tz.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import fs from "fs";
import zlib from "zlib";

import { defined } from "../base/defined";

Expand All @@ -24,7 +25,7 @@ export class TilesetSource3tz implements TilesetSource {
*
* This is created from the `"@3dtilesIndex1@"` file of a 3TZ file.
*
* It is an array if `IndexEntry` objects, sorted by the MD5 hash,
* It is an array of `IndexEntry` objects, sorted by the MD5 hash,
* in ascending order.
*/
private zipIndex: IndexEntry[] | undefined;
Expand Down Expand Up @@ -81,12 +82,16 @@ export class TilesetSource3tz implements TilesetSource {
if (!defined(this.fd) || !this.zipIndex) {
throw new TilesetError("Source is not opened. Call 'open' first.");
}
const entryData = ArchiveFunctions3tz.readEntryData(
this.fd,
this.zipIndex,
key
);
return entryData;
const entry = ArchiveFunctions3tz.readEntry(this.fd, this.zipIndex, key);
if (!entry) {
return undefined;
}
if (entry.compression_method === 8) {
// Indicating DEFLATE
const inflatedData = zlib.inflateRawSync(entry.data);
return inflatedData;
}
return entry.data;
}

/** {@inheritDoc TilesetSource.close} */
Expand Down
2 changes: 1 addition & 1 deletion src/packages/TilesetTarget3tz.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ export class TilesetTarget3tz implements TilesetTarget {
*
* @param archive - The archiver archive
* @param outputStream - The output stream that the archive is writing to
* @returns The promise that has to be waited for in "close"
* @returns The promise that has to be waited for in "end"
*/
private static createFinishedPromise(
archive: archiver.Archiver,
Expand Down

0 comments on commit b17ddd1

Please sign in to comment.