diff --git a/fhir-package-loader-1.0.0.tgz b/fhir-package-loader-1.0.0.tgz new file mode 100644 index 0000000..efa1c16 Binary files /dev/null and b/fhir-package-loader-1.0.0.tgz differ diff --git a/src/cache/DiskBasedPackageCache.ts b/src/cache/DiskBasedPackageCache.ts index 5febb53..c360bab 100644 --- a/src/cache/DiskBasedPackageCache.ts +++ b/src/cache/DiskBasedPackageCache.ts @@ -99,14 +99,14 @@ export class DiskBasedPackageCache implements PackageCache { return false; } else if (/\.json$/i.test(entry.name)) { return true; - } else if (/-spreadsheet.xml/i.test(entry.name)) { + } else if (/-spreadsheet\.xml$/i.test(entry.name)) { spreadSheetCount++; this.log( 'debug', `Skipped spreadsheet XML file: ${path.resolve(entry.path, entry.name)}` ); return false; - } else if (/\.xml/i.test(entry.name)) { + } else if (/\.xml$/i.test(entry.name)) { const xml = fs.readFileSync(path.resolve(entry.path, entry.name)).toString(); if (/<\?mso-application progid="Excel\.Sheet"\?>/m.test(xml)) { spreadSheetCount++; diff --git a/src/db/PackageDB.ts b/src/db/PackageDB.ts index dd72229..ad24052 100644 --- a/src/db/PackageDB.ts +++ b/src/db/PackageDB.ts @@ -9,4 +9,5 @@ export interface PackageDB { findResourceInfos(key: string, options?: FindResourceInfoOptions): ResourceInfo[]; findResourceInfo(key: string, options?: FindResourceInfoOptions): ResourceInfo | undefined; getPackageStats(name: string, version: string): PackageStats | undefined; + exportDB(): Promise<{ mimeType: string; data: Buffer }>; } diff --git a/src/db/SQLJSPackageDB.ts b/src/db/SQLJSPackageDB.ts index 24a37ff..8bd1102 100644 --- a/src/db/SQLJSPackageDB.ts +++ b/src/db/SQLJSPackageDB.ts @@ -318,6 +318,11 @@ export class SQLJSPackageDB implements PackageDB { }; } + exportDB(): Promise<{ mimeType: string; data: Buffer }> { + const data = this.db.export(); + return Promise.resolve({ mimeType: 'application/x-sqlite3', data: Buffer.from(data) }); + } + logPackageTable() { const res = this.db.exec('SELECT * FROM package'); console.log(util.inspect(res, false, 3, true)); diff --git a/src/loader/BasePackageLoader.ts b/src/loader/BasePackageLoader.ts index d4b1bb2..b50149d 100644 --- a/src/loader/BasePackageLoader.ts +++ b/src/loader/BasePackageLoader.ts @@ -31,6 +31,13 @@ export class BasePackageLoader implements PackageLoader { async loadPackage(name: string, version: string): Promise { let packageLabel = `${name}#${version}`; + const originalVersion = version; + version = await this.registryClient.resolveVersion(name, version); + if (version !== originalVersion) { + this.log('info', `Resolved ${packageLabel} to concrete version ${version}`); + packageLabel = `${name}#${version}`; + } + // If it's already loaded, then there's nothing to do if (this.getPackageLoadStatus(name, version) === LoadStatus.LOADED) { this.log('info', `${packageLabel} already loaded`); @@ -47,6 +54,7 @@ export class BasePackageLoader implements PackageLoader { packageLabel = `${name}#${version}`; } + let downloadErrorMessage: string; // If it's a "current" version, download the latest version from the build server (if applicable) if (isCurrentVersion(version)) { const branch = version.indexOf('$') !== -1 ? version.split('$')[1] : undefined; @@ -55,7 +63,7 @@ export class BasePackageLoader implements PackageLoader { const tarballStream = await this.currentBuildClient.downloadCurrentBuild(name, branch); await this.packageCache.cachePackageTarball(name, version, tarballStream); } catch { - this.log('error', `Failed to download ${packageLabel} from current builds`); + downloadErrorMessage = `Failed to download most recent ${packageLabel} from current builds`; } } } @@ -65,7 +73,7 @@ export class BasePackageLoader implements PackageLoader { const tarballStream = await this.registryClient.download(name, version); await this.packageCache.cachePackageTarball(name, version, tarballStream); } catch { - this.log('error', `Failed to download ${packageLabel} from registry`); + downloadErrorMessage = `Failed to download ${packageLabel} from the registry`; } } @@ -74,9 +82,17 @@ export class BasePackageLoader implements PackageLoader { try { stats = this.loadPackageFromCache(name, version); } catch { - this.log('error', `Failed to load ${name}#${version}`); + this.log( + 'error', + `Failed to load ${packageLabel}${downloadErrorMessage ? `: ${downloadErrorMessage}` : ''}` + ); return LoadStatus.FAILED; } + if (downloadErrorMessage) { + // Loading succeeded despite a download error. This might happen if a current build is stale, + // but the download fails, in which case the stale build will be loaded instead. + this.log('error', downloadErrorMessage); + } this.log('info', `Loaded ${stats.name}#${stats.version} with ${stats.resourceCount} resources`); return LoadStatus.LOADED; } @@ -117,12 +133,13 @@ export class BasePackageLoader implements PackageLoader { this.packageDB.savePackageInfo(info); // Register the resources - await pkg.registerResources((key: string, resource: any) => { + await pkg.registerResources((key: string, resource: any, allowNonResources?: boolean) => { this.loadResource( `virtual:${packageKey}:${key}`, resource, packageJSON.name, - packageJSON.version + packageJSON.version, + allowNonResources ); }); @@ -172,7 +189,10 @@ export class BasePackageLoader implements PackageLoader { this.loadResourceFromCache(resourcePath, packageName, packageVersion); } catch { // swallow this error because some JSON files will not be resources - this.log('debug', `JSON file at path ${resourcePath} was not FHIR resource`); + // and don't log it if it is package.json (since every package should have one) + if (path.basename(resourcePath) !== 'package.json') { + this.log('debug', `JSON file at path ${resourcePath} was not FHIR resource`); + } } }); } @@ -186,14 +206,21 @@ export class BasePackageLoader implements PackageLoader { resourcePath: string, resourceJSON: any, packageName?: string, - packageVersion?: string + packageVersion?: string, + allowNonResources = false ) { // We require at least a resourceType in order to know it is FHIR - if (typeof resourceJSON.resourceType !== 'string' || resourceJSON.resourceType === '') { - throw new InvalidResourceError(resourcePath, 'resource does not specify its resourceType'); + let resourceType = resourceJSON.resourceType; + if (typeof resourceType !== 'string' || resourceType === '') { + if (allowNonResources) { + // SUSHI needs to support registering local logical models, but some code expects resourceType + resourceType = 'Unknown'; + } else { + throw new InvalidResourceError(resourcePath, 'resource does not specify its resourceType'); + } } - const info: ResourceInfo = { resourceType: resourceJSON.resourceType }; + const info: ResourceInfo = { resourceType }; if (typeof resourceJSON.id === 'string') { info.id = resourceJSON.id; } @@ -206,7 +233,7 @@ export class BasePackageLoader implements PackageLoader { if (typeof resourceJSON.version === 'string') { info.version = resourceJSON.version; } - if (resourceJSON.resourceType === 'StructureDefinition') { + if (resourceType === 'StructureDefinition') { if (typeof resourceJSON.kind === 'string') { info.sdKind = resourceJSON.kind; } @@ -372,6 +399,10 @@ export class BasePackageLoader implements PackageLoader { } } + exportDB(): Promise<{ mimeType: string; data: Buffer }> { + return this.packageDB.exportDB(); + } + clear() { this.packageDB.clear(); } diff --git a/src/loader/PackageLoader.ts b/src/loader/PackageLoader.ts index 4430d79..53cba1a 100644 --- a/src/loader/PackageLoader.ts +++ b/src/loader/PackageLoader.ts @@ -18,5 +18,6 @@ export interface PackageLoader { findResourceInfo(key: string, options?: FindResourceInfoOptions): ResourceInfo | undefined; findResourceJSONs(key: string, options?: FindResourceInfoOptions): any[]; findResourceJSON(key: string, options?: FindResourceInfoOptions): any | undefined; + exportDB(): Promise<{ mimeType: string; data: Buffer }>; clear(): void; } diff --git a/src/registry/FHIRRegistryClient.ts b/src/registry/FHIRRegistryClient.ts index 57c83b1..3521191 100644 --- a/src/registry/FHIRRegistryClient.ts +++ b/src/registry/FHIRRegistryClient.ts @@ -1,8 +1,7 @@ import { Readable } from 'stream'; import { LogFunction, axiosGet } from '../utils'; import { RegistryClient, RegistryClientOptions } from './RegistryClient'; -import { IncorrectWildcardVersionFormatError } from '../errors'; -import { lookUpLatestVersion, lookUpLatestPatchVersion } from './utils'; +import { resolveVersion } from './utils'; export class FHIRRegistryClient implements RegistryClient { public endpoint: string; @@ -14,15 +13,13 @@ export class FHIRRegistryClient implements RegistryClient { this.log = options.log ?? (() => {}); } + async resolveVersion(name: string, version: string): Promise { + return resolveVersion(this.endpoint, name, version); + } + async download(name: string, version: string): Promise { // Resolve version if necessary - if (version === 'latest') { - version = await lookUpLatestVersion(this.endpoint, name); - } else if (/^\d+\.\d+\.x$/.test(version)) { - version = await lookUpLatestPatchVersion(this.endpoint, name, version); - } else if (/^\d+\.x$/.test(version)) { - throw new IncorrectWildcardVersionFormatError(name, version); - } + version = await this.resolveVersion(name, version); // Construct URL from endpoint, name, and version // See: https://confluence.hl7.org/pages/viewpage.action?pageId=97454344#FHIRPackageRegistryUserDocumentation-Download diff --git a/src/registry/NPMRegistryClient.ts b/src/registry/NPMRegistryClient.ts index 0787c11..4df0761 100644 --- a/src/registry/NPMRegistryClient.ts +++ b/src/registry/NPMRegistryClient.ts @@ -1,8 +1,7 @@ import { Readable } from 'stream'; import { LogFunction, axiosGet } from '../utils'; import { RegistryClient, RegistryClientOptions } from './RegistryClient'; -import { IncorrectWildcardVersionFormatError } from '../errors'; -import { lookUpLatestVersion, lookUpLatestPatchVersion } from './utils'; +import { resolveVersion } from './utils'; export class NPMRegistryClient implements RegistryClient { public endpoint: string; @@ -14,15 +13,13 @@ export class NPMRegistryClient implements RegistryClient { this.log = options.log ?? (() => {}); } + async resolveVersion(name: string, version: string): Promise { + return resolveVersion(this.endpoint, name, version); + } + async download(name: string, version: string): Promise { // Resolve version if necessary - if (version === 'latest') { - version = await lookUpLatestVersion(this.endpoint, name); - } else if (/^\d+\.\d+\.x$/.test(version)) { - version = await lookUpLatestPatchVersion(this.endpoint, name, version); - } else if (/^\d+\.x$/.test(version)) { - throw new IncorrectWildcardVersionFormatError(name, version); - } + version = await this.resolveVersion(name, version); // Get the manifest information about the package from the registry let url; diff --git a/src/registry/RedundantRegistryClient.ts b/src/registry/RedundantRegistryClient.ts index 7def1ae..613b03d 100644 --- a/src/registry/RedundantRegistryClient.ts +++ b/src/registry/RedundantRegistryClient.ts @@ -11,6 +11,19 @@ export class RedundantRegistryClient implements RegistryClient { this.log = options.log ?? (() => {}); } + async resolveVersion(name: string, version: string): Promise { + const packageLabel = `${name}#${version}`; + + for (const client of this.clients) { + try { + return await client.resolveVersion(name, version); + } catch { + // Do nothing. Fallback to the next one. + } + } + throw Error(`Failed to resolve version for ${packageLabel}`); + } + async download(name: string, version: string): Promise { const packageLabel = `${name}#${version}`; diff --git a/src/registry/RegistryClient.ts b/src/registry/RegistryClient.ts index 7d69be3..7246ec4 100644 --- a/src/registry/RegistryClient.ts +++ b/src/registry/RegistryClient.ts @@ -6,5 +6,6 @@ export type RegistryClientOptions = { }; export interface RegistryClient { + resolveVersion(name: string, version: string): Promise; download(name: string, version: string): Promise; } diff --git a/src/registry/utils.ts b/src/registry/utils.ts index 4aeedaa..47ef13a 100644 --- a/src/registry/utils.ts +++ b/src/registry/utils.ts @@ -2,7 +2,23 @@ import { IncorrectWildcardVersionFormatError, LatestVersionUnavailableError } fr import { axiosGet } from '../utils'; import { maxSatisfying } from 'semver'; -export async function lookUpLatestVersion(endpoint: string, name: string): Promise { +export async function resolveVersion( + endpoint: string, + name: string, + version: string +): Promise { + let resolvedVersion = version; + if (version === 'latest') { + resolvedVersion = await lookUpLatestVersion(endpoint, name); + } else if (/^\d+\.\d+\.x$/.test(version)) { + resolvedVersion = await lookUpLatestPatchVersion(endpoint, name, version); + } else if (/^\d+\.x$/.test(version)) { + throw new IncorrectWildcardVersionFormatError(name, version); + } + return resolvedVersion; +} + +async function lookUpLatestVersion(endpoint: string, name: string): Promise { try { const res = await axiosGet(`${endpoint}/${name}`, { responseType: 'json' @@ -17,7 +33,7 @@ export async function lookUpLatestVersion(endpoint: string, name: string): Promi } } -export async function lookUpLatestPatchVersion( +async function lookUpLatestPatchVersion( endpoint: string, name: string, version: string diff --git a/src/virtual/DiskBasedVirtualPackage.ts b/src/virtual/DiskBasedVirtualPackage.ts index b2eb4a2..9dd00c9 100644 --- a/src/virtual/DiskBasedVirtualPackage.ts +++ b/src/virtual/DiskBasedVirtualPackage.ts @@ -5,24 +5,34 @@ import { PackageJSON } from '../package/PackageJSON'; import { LogFunction } from '../utils/logger'; import { VirtualPackage, VirtualPackageOptions } from './VirtualPackage'; +export type DiskBasedVirtualPackageOptions = VirtualPackageOptions & { + recursive?: boolean; +}; + export class DiskBasedVirtualPackage implements VirtualPackage { private log: LogFunction; + private allowNonResources: boolean; + private recursive: boolean; private fhirConverter: FHIRConverter; constructor( private packageJSON: PackageJSON, private paths: string[] = [], - options: VirtualPackageOptions = {} + options: DiskBasedVirtualPackageOptions = {} ) { this.log = options.log ?? (() => {}); + this.allowNonResources = options.allowNonResources ?? false; + this.recursive = options.recursive ?? false; this.fhirConverter = new FHIRConverter(); } - async registerResources(register: (key: string, resource: any) => void): Promise { + async registerResources( + register: (key: string, resource: any, allowNonResources?: boolean) => void + ): Promise { const spreadSheetCounts = new Map(); const invalidFileCounts = new Map(); - const filePaths = getFilePaths(this.paths); + const filePaths = getFilePaths(this.paths, this.recursive); for (const filePath of filePaths) { try { const name = path.basename(filePath); @@ -30,24 +40,32 @@ export class DiskBasedVirtualPackage implements VirtualPackage { // Is it a potential resource? if (/\.json$/i.test(name)) { // TODO: Error handling? - register(filePath, this.getResourceByKey(filePath)); - } else if (/-spreadsheet.xml/i.test(name)) { + register(filePath, this.getResourceByKey(filePath), this.allowNonResources); + } else if (/-spreadsheet\.xml$/i.test(name)) { spreadSheetCounts.set(parent, (spreadSheetCounts.get(parent) ?? 0) + 1); this.log('debug', `Skipped spreadsheet XML file: ${filePath}`); - } else if (/\.xml/i.test(name)) { + } else if (/\.xml$/i.test(name)) { const xml = fs.readFileSync(filePath).toString(); if (/<\?mso-application progid="Excel\.Sheet"\?>/m.test(xml)) { spreadSheetCounts.set(parent, (spreadSheetCounts.get(parent) ?? 0) + 1); this.log('debug', `Skipped spreadsheet XML file: ${filePath}`); } // TODO: Error handling? - register(filePath, this.getResourceByKey(filePath)); + register(filePath, this.getResourceByKey(filePath), this.allowNonResources); } else { invalidFileCounts.set(parent, (invalidFileCounts.get(parent) ?? 0) + 1); this.log('debug', `Skipped non-JSON / non-XML file: ${filePath}`); } - } catch { - this.log('error', `Failed to load resource from path: ${filePath}`); + } catch (e) { + if (/convert XML .* Unknown resource type/.test(e.message)) { + // Skip unknown FHIR resource types. When we have instances of Logical Models, + // the resourceType will not be recognized as a known FHIR resourceType, but that's okay. + } else { + this.log('error', `Failed to load resource from path: ${filePath}`); + if (e.stack) { + this.log('debug', e.stack); + } + } } } @@ -76,12 +94,18 @@ export class DiskBasedVirtualPackage implements VirtualPackage { getResourceByKey(key: string) { let resource: any; if (/.xml$/i.test(key)) { + let xml: string; try { - const xml = fs.readFileSync(key).toString(); - resource = this.fhirConverter.xmlToObj(xml); + xml = fs.readFileSync(key).toString(); } catch { throw new Error(`Failed to get XML resource at path ${key}`); } + try { + // TODO: Support other versions of FHIR during conversion + resource = this.fhirConverter.xmlToObj(xml); + } catch (e) { + throw new Error(`Failed to convert XML resource at path ${key}: ${e.message}`); + } } else if (/.json$/i.test(key)) { try { resource = fs.readJSONSync(key); @@ -95,20 +119,23 @@ export class DiskBasedVirtualPackage implements VirtualPackage { } } -function getFilePaths(paths: string[]): string[] { - const filePaths: string[] = []; +function getFilePaths(paths: string[], recursive: boolean): string[] { + const filePaths = new Set(); paths.forEach(p => { const stat = fs.statSync(p); if (stat.isFile()) { - filePaths.push(path.resolve(p)); + filePaths.add(path.resolve(p)); } else if (stat.isDirectory()) { fs.readdirSync(p, { withFileTypes: true }).forEach(entry => { - // NOTE: This is not a recursive crawl, so we only care about files - if (entry.isFile) { - filePaths.push(path.resolve(entry.parentPath, entry.name)); + if (entry.isFile()) { + filePaths.add(path.resolve(entry.parentPath, entry.name)); + } else if (recursive && entry.isDirectory()) { + getFilePaths([path.resolve(entry.parentPath, entry.name)], recursive).forEach(fp => + filePaths.add(fp) + ); } }); } }); - return filePaths; + return Array.from(filePaths); } diff --git a/src/virtual/InMemoryVirtualPackage.ts b/src/virtual/InMemoryVirtualPackage.ts index 8898cc4..e7374e8 100644 --- a/src/virtual/InMemoryVirtualPackage.ts +++ b/src/virtual/InMemoryVirtualPackage.ts @@ -4,6 +4,7 @@ import { VirtualPackage, VirtualPackageOptions } from './VirtualPackage'; export class InMemoryVirtualPackage implements VirtualPackage { private log: LogFunction; + private allowNonResources: boolean; constructor( private packageJSON: PackageJSON, @@ -11,11 +12,14 @@ export class InMemoryVirtualPackage implements VirtualPackage { options: VirtualPackageOptions = {} ) { this.log = options.log ?? (() => {}); + this.allowNonResources = options.allowNonResources ?? false; } - async registerResources(register: (key: string, resource: any) => void): Promise { + async registerResources( + register: (key: string, resource: any, allowNonResources?: boolean) => void + ): Promise { // TODO: Error handling? - this.resources.forEach((resource, key) => register(key, resource)); + this.resources.forEach((resource, key) => register(key, resource, this.allowNonResources)); } getPackageJSON(): PackageJSON { diff --git a/src/virtual/VirtualPackage.ts b/src/virtual/VirtualPackage.ts index 1504fcb..b32a8d5 100644 --- a/src/virtual/VirtualPackage.ts +++ b/src/virtual/VirtualPackage.ts @@ -2,11 +2,14 @@ import { LogFunction } from '../utils/logger'; import { PackageJSON } from '../package/PackageJSON'; export interface VirtualPackage { - registerResources(register: (key: string, resource: any) => void): Promise; + registerResources( + register: (key: string, resource: any, allowNonResources?: boolean) => void + ): Promise; getPackageJSON(): PackageJSON; getResourceByKey(key: string): any; } export type VirtualPackageOptions = { log?: LogFunction; + allowNonResources?: boolean; }; diff --git a/test/loader/BasePackageLoader.test.ts b/test/loader/BasePackageLoader.test.ts index 5b13d30..05df2d4 100644 --- a/test/loader/BasePackageLoader.test.ts +++ b/test/loader/BasePackageLoader.test.ts @@ -28,6 +28,9 @@ describe('BasePackageLoader', () => { BasePackageLoader.prototype as any, 'loadPackageFromCache' ); + registryClientMock.resolveVersion.mockImplementation((name, version) => + Promise.resolve(version) + ); loader = new BasePackageLoader( packageDBMock, packageCacheMock, diff --git a/test/registry/RedundantRegistryClient.test.ts b/test/registry/RedundantRegistryClient.test.ts index ca4656e..bba5517 100644 --- a/test/registry/RedundantRegistryClient.test.ts +++ b/test/registry/RedundantRegistryClient.test.ts @@ -1,13 +1,18 @@ import { RedundantRegistryClient } from '../../src/registry/RedundantRegistryClient'; +import { RegistryClient } from '../../src/registry/RegistryClient'; import { loggerSpy } from '../testhelpers'; import { Readable } from 'stream'; -class MyMockClient { +class MyMockClient implements RegistryClient { public endpoint: string; constructor(endpoint: string) { this.endpoint = endpoint; } + async resolveVersion(name: string, version: string): Promise { + return version; + } + async download(name: string, version: string): Promise { // to mimic failure of download if (this.endpoint == 'failed.to.download') throw new Error('Failed to download');