WIP

src/cache/DiskBasedPackageCache.ts

@@ -99,14 +99,14 @@ export class DiskBasedPackageCache implements PackageCache {
                 return false;
               } else if (/\.json$/i.test(entry.name)) {
                 return true;
-              } else if (/-spreadsheet.xml/i.test(entry.name)) {
+              } else if (/-spreadsheet\.xml$/i.test(entry.name)) {
                 spreadSheetCount++;
                 this.log(
                   'debug',
                   `Skipped spreadsheet XML file: ${path.resolve(entry.path, entry.name)}`
                 );
                 return false;
-              } else if (/\.xml/i.test(entry.name)) {
+              } else if (/\.xml$/i.test(entry.name)) {
                 const xml = fs.readFileSync(path.resolve(entry.path, entry.name)).toString();
                 if (/<\?mso-application progid="Excel\.Sheet"\?>/m.test(xml)) {
                   spreadSheetCount++;

src/db/PackageDB.ts

@@ -9,4 +9,5 @@ export interface PackageDB {
   findResourceInfos(key: string, options?: FindResourceInfoOptions): ResourceInfo[];
   findResourceInfo(key: string, options?: FindResourceInfoOptions): ResourceInfo | undefined;
   getPackageStats(name: string, version: string): PackageStats | undefined;
+  exportDB(): Promise<{ mimeType: string; data: Buffer }>;
 }

src/db/SQLJSPackageDB.ts

@@ -318,6 +318,11 @@ export class SQLJSPackageDB implements PackageDB {
     };
   }
 
+  exportDB(): Promise<{ mimeType: string; data: Buffer }> {
+    const data = this.db.export();
+    return Promise.resolve({ mimeType: 'application/x-sqlite3', data: Buffer.from(data) });
+  }
+
   logPackageTable() {
     const res = this.db.exec('SELECT * FROM package');
     console.log(util.inspect(res, false, 3, true));

src/loader/BasePackageLoader.ts

@@ -31,6 +31,13 @@ export class BasePackageLoader implements PackageLoader {
   async loadPackage(name: string, version: string): Promise<LoadStatus> {
     let packageLabel = `${name}#${version}`;
 
+    const originalVersion = version;
+    version = await this.registryClient.resolveVersion(name, version);
+    if (version !== originalVersion) {
+      this.log('info', `Resolved ${packageLabel} to concrete version ${version}`);
+      packageLabel = `${name}#${version}`;
+    }
+
     // If it's already loaded, then there's nothing to do
     if (this.getPackageLoadStatus(name, version) === LoadStatus.LOADED) {
       this.log('info', `${packageLabel} already loaded`);
@@ -47,6 +54,7 @@ export class BasePackageLoader implements PackageLoader {
       packageLabel = `${name}#${version}`;
     }
 
+    let downloadErrorMessage: string;
     // If it's a "current" version, download the latest version from the build server (if applicable)
     if (isCurrentVersion(version)) {
       const branch = version.indexOf('$') !== -1 ? version.split('$')[1] : undefined;
@@ -55,7 +63,7 @@ export class BasePackageLoader implements PackageLoader {
           const tarballStream = await this.currentBuildClient.downloadCurrentBuild(name, branch);
           await this.packageCache.cachePackageTarball(name, version, tarballStream);
         } catch {
-          this.log('error', `Failed to download ${packageLabel} from current builds`);
+          downloadErrorMessage = `Failed to download most recent ${packageLabel} from current builds`;
         }
       }
     }
@@ -65,7 +73,7 @@ export class BasePackageLoader implements PackageLoader {
         const tarballStream = await this.registryClient.download(name, version);
         await this.packageCache.cachePackageTarball(name, version, tarballStream);
       } catch {
-        this.log('error', `Failed to download ${packageLabel} from registry`);
+        downloadErrorMessage = `Failed to download ${packageLabel} from the registry`;
       }
     }
 
@@ -74,9 +82,17 @@ export class BasePackageLoader implements PackageLoader {
     try {
       stats = this.loadPackageFromCache(name, version);
     } catch {
-      this.log('error', `Failed to load ${name}#${version}`);
+      this.log(
+        'error',
+        `Failed to load ${packageLabel}${downloadErrorMessage ? `: ${downloadErrorMessage}` : ''}`
+      );
       return LoadStatus.FAILED;
     }
+    if (downloadErrorMessage) {
+      // Loading succeeded despite a download error. This might happen if a current build is stale,
+      // but the download fails, in which case the stale build will be loaded instead.
+      this.log('error', downloadErrorMessage);
+    }
     this.log('info', `Loaded ${stats.name}#${stats.version} with ${stats.resourceCount} resources`);
     return LoadStatus.LOADED;
   }
@@ -117,12 +133,13 @@ export class BasePackageLoader implements PackageLoader {
     this.packageDB.savePackageInfo(info);
 
     // Register the resources
-    await pkg.registerResources((key: string, resource: any) => {
+    await pkg.registerResources((key: string, resource: any, allowNonResources?: boolean) => {
       this.loadResource(
         `virtual:${packageKey}:${key}`,
         resource,
         packageJSON.name,
-        packageJSON.version
+        packageJSON.version,
+        allowNonResources
       );
     });
 
@@ -172,7 +189,10 @@ export class BasePackageLoader implements PackageLoader {
           this.loadResourceFromCache(resourcePath, packageName, packageVersion);
         } catch {
           // swallow this error because some JSON files will not be resources
-          this.log('debug', `JSON file at path ${resourcePath} was not FHIR resource`);
+          // and don't log it if it is package.json (since every package should have one)
+          if (path.basename(resourcePath) !== 'package.json') {
+            this.log('debug', `JSON file at path ${resourcePath} was not FHIR resource`);
+          }
         }
       });
   }
@@ -186,14 +206,21 @@ export class BasePackageLoader implements PackageLoader {
     resourcePath: string,
     resourceJSON: any,
     packageName?: string,
-    packageVersion?: string
+    packageVersion?: string,
+    allowNonResources = false
   ) {
     // We require at least a resourceType in order to know it is FHIR
-    if (typeof resourceJSON.resourceType !== 'string' || resourceJSON.resourceType === '') {
-      throw new InvalidResourceError(resourcePath, 'resource does not specify its resourceType');
+    let resourceType = resourceJSON.resourceType;
+    if (typeof resourceType !== 'string' || resourceType === '') {
+      if (allowNonResources) {
+        // SUSHI needs to support registering local logical models, but some code expects resourceType
+        resourceType = 'Unknown';
+      } else {
+        throw new InvalidResourceError(resourcePath, 'resource does not specify its resourceType');
+      }
     }
 
-    const info: ResourceInfo = { resourceType: resourceJSON.resourceType };
+    const info: ResourceInfo = { resourceType };
     if (typeof resourceJSON.id === 'string') {
       info.id = resourceJSON.id;
     }
@@ -206,7 +233,7 @@ export class BasePackageLoader implements PackageLoader {
     if (typeof resourceJSON.version === 'string') {
       info.version = resourceJSON.version;
     }
-    if (resourceJSON.resourceType === 'StructureDefinition') {
+    if (resourceType === 'StructureDefinition') {
       if (typeof resourceJSON.kind === 'string') {
         info.sdKind = resourceJSON.kind;
       }
@@ -372,6 +399,10 @@ export class BasePackageLoader implements PackageLoader {
     }
   }
 
+  exportDB(): Promise<{ mimeType: string; data: Buffer }> {
+    return this.packageDB.exportDB();
+  }
+
   clear() {
     this.packageDB.clear();
   }

src/loader/PackageLoader.ts

@@ -18,5 +18,6 @@ export interface PackageLoader {
   findResourceInfo(key: string, options?: FindResourceInfoOptions): ResourceInfo | undefined;
   findResourceJSONs(key: string, options?: FindResourceInfoOptions): any[];
   findResourceJSON(key: string, options?: FindResourceInfoOptions): any | undefined;
+  exportDB(): Promise<{ mimeType: string; data: Buffer }>;
   clear(): void;
 }

src/registry/FHIRRegistryClient.ts

@@ -1,8 +1,7 @@
 import { Readable } from 'stream';
 import { LogFunction, axiosGet } from '../utils';
 import { RegistryClient, RegistryClientOptions } from './RegistryClient';
-import { IncorrectWildcardVersionFormatError } from '../errors';
-import { lookUpLatestVersion, lookUpLatestPatchVersion } from './utils';
+import { resolveVersion } from './utils';
 
 export class FHIRRegistryClient implements RegistryClient {
   public endpoint: string;
@@ -14,15 +13,13 @@ export class FHIRRegistryClient implements RegistryClient {
     this.log = options.log ?? (() => {});
   }
 
+  async resolveVersion(name: string, version: string): Promise<string> {
+    return resolveVersion(this.endpoint, name, version);
+  }
+
   async download(name: string, version: string): Promise<Readable> {
     // Resolve version if necessary
-    if (version === 'latest') {
-      version = await lookUpLatestVersion(this.endpoint, name);
-    } else if (/^\d+\.\d+\.x$/.test(version)) {
-      version = await lookUpLatestPatchVersion(this.endpoint, name, version);
-    } else if (/^\d+\.x$/.test(version)) {
-      throw new IncorrectWildcardVersionFormatError(name, version);
-    }
+    version = await this.resolveVersion(name, version);
 
     // Construct URL from endpoint, name, and version
     // See: https://confluence.hl7.org/pages/viewpage.action?pageId=97454344#FHIRPackageRegistryUserDocumentation-Download

src/registry/NPMRegistryClient.ts

@@ -1,8 +1,7 @@
 import { Readable } from 'stream';
 import { LogFunction, axiosGet } from '../utils';
 import { RegistryClient, RegistryClientOptions } from './RegistryClient';
-import { IncorrectWildcardVersionFormatError } from '../errors';
-import { lookUpLatestVersion, lookUpLatestPatchVersion } from './utils';
+import { resolveVersion } from './utils';
 
 export class NPMRegistryClient implements RegistryClient {
   public endpoint: string;
@@ -14,15 +13,13 @@ export class NPMRegistryClient implements RegistryClient {
     this.log = options.log ?? (() => {});
   }
 
+  async resolveVersion(name: string, version: string): Promise<string> {
+    return resolveVersion(this.endpoint, name, version);
+  }
+
   async download(name: string, version: string): Promise<Readable> {
     // Resolve version if necessary
-    if (version === 'latest') {
-      version = await lookUpLatestVersion(this.endpoint, name);
-    } else if (/^\d+\.\d+\.x$/.test(version)) {
-      version = await lookUpLatestPatchVersion(this.endpoint, name, version);
-    } else if (/^\d+\.x$/.test(version)) {
-      throw new IncorrectWildcardVersionFormatError(name, version);
-    }
+    version = await this.resolveVersion(name, version);
 
     // Get the manifest information about the package from the registry
     let url;

src/registry/RedundantRegistryClient.ts

@@ -11,6 +11,19 @@ export class RedundantRegistryClient implements RegistryClient {
     this.log = options.log ?? (() => {});
   }
 
+  async resolveVersion(name: string, version: string): Promise<string> {
+    const packageLabel = `${name}#${version}`;
+
+    for (const client of this.clients) {
+      try {
+        return await client.resolveVersion(name, version);
+      } catch {
+        // Do nothing. Fallback to the next one.
+      }
+    }
+    throw Error(`Failed to resolve version for ${packageLabel}`);
+  }
+
   async download(name: string, version: string): Promise<Readable> {
     const packageLabel = `${name}#${version}`;
 

src/registry/RegistryClient.ts

@@ -6,5 +6,6 @@ export type RegistryClientOptions = {
 };
 
 export interface RegistryClient {
+  resolveVersion(name: string, version: string): Promise<string>;
   download(name: string, version: string): Promise<Readable>;
 }

src/registry/utils.ts

@@ -2,7 +2,23 @@ import { IncorrectWildcardVersionFormatError, LatestVersionUnavailableError } fr
 import { axiosGet } from '../utils';
 import { maxSatisfying } from 'semver';
 
-export async function lookUpLatestVersion(endpoint: string, name: string): Promise<string> {
+export async function resolveVersion(
+  endpoint: string,
+  name: string,
+  version: string
+): Promise<string> {
+  let resolvedVersion = version;
+  if (version === 'latest') {
+    resolvedVersion = await lookUpLatestVersion(endpoint, name);
+  } else if (/^\d+\.\d+\.x$/.test(version)) {
+    resolvedVersion = await lookUpLatestPatchVersion(endpoint, name, version);
+  } else if (/^\d+\.x$/.test(version)) {
+    throw new IncorrectWildcardVersionFormatError(name, version);
+  }
+  return resolvedVersion;
+}
+
+async function lookUpLatestVersion(endpoint: string, name: string): Promise<string> {
   try {
     const res = await axiosGet(`${endpoint}/${name}`, {
       responseType: 'json'
@@ -17,7 +33,7 @@ export async function lookUpLatestVersion(endpoint: string, name: string): Promi
   }
 }
 
-export async function lookUpLatestPatchVersion(
+async function lookUpLatestPatchVersion(
   endpoint: string,
   name: string,
   version: string