diff --git a/helm/config/dataone-indexer.properties b/helm/config/dataone-indexer.properties index c51ce0aa..003cca15 100644 --- a/helm/config/dataone-indexer.properties +++ b/helm/config/dataone-indexer.properties @@ -40,3 +40,12 @@ index.resourcemap.waitingComponent.time={{ default 800 .Values.idxworker.resourc index.resourcemap.waitingComponent.max.attempts={{ default 25 .Values.idxworker.resourcemapMaxTries }} index.solr.versionConflict.waiting.time={{ default 1000 .Values.idxworker.solrVerConflictWaitMs }} index.solr.versionConflict.max.attempts={{ default 50 .Values.idxworker.solrVerConflictMaxTries }} + +# Storage properties +storage.className={{ default "org.dataone.hashstore.filehashstore.FileHashStore" .Values.idxworker.storage.hashStoreClassName }} +storage.hashstore.rootDirectory={{ default "./target/hashstore" .Values.idxworker.storage.hashStoreRootDir }} +storage.hashstore.defaultNamespace={{ default "https://ns.dataone.org/service/types/v2.0#SystemMetadata" .Values.idxworker.storage.hashStoreDefaultNamespace }} +# The following three properties must NOT be modified after the hash store is initialized +storage.hashstore.fileNameAlgorithm={{ default "SHA-256" .Values.idxworker.storage.hashStoreAlgorithm }} +storage.hashstore.directory.width={{ default 2 .Values.idxworker.storage.hashStoreDirWidth }} +storage.hashstore.directory.depth={{ default 3 .Values.idxworker.storage.hashStoreDirDepth }} diff --git a/src/main/java/org/dataone/cn/indexer/object/ObjectManager.java b/src/main/java/org/dataone/cn/indexer/object/ObjectManager.java index 4f0310eb..df67f0f7 100644 --- a/src/main/java/org/dataone/cn/indexer/object/ObjectManager.java +++ b/src/main/java/org/dataone/cn/indexer/object/ObjectManager.java @@ -50,41 +50,26 @@ public class ObjectManager { static { try { - manager = new ObjectManager(); - } catch (ServiceFailure | IOException e) { - logger.error("Metacat cannot initialize the ObjectManager class since " + e.getMessage()); + refreshD1Node(); + } catch (ServiceFailure e) { + logger.warn("Metacat cannot initialize the d1Node since " + e.getMessage()); } + storage = Storage.getInstance(); + manager = new ObjectManager(); } /** * Private constructor - * @throws ServiceFailure - * @throws IOException - * @throws IllegalArgumentException */ - private ObjectManager() throws ServiceFailure, IllegalArgumentException, IOException { - if (storage == null) { - storage = Storage.getInstance(); - } - if (d1Node == null) { - refreshD1Node(); - } else { - logger.info("ObjectManager ---NOT going to create the d1node with the url " + nodeBaseURL - + " since the ObjectManager already was assigned a d1node with the url " - + d1Node.getNodeBaseServiceUrl()); - } + private ObjectManager() { } /** * Get an ObjectManager instance through the singleton pattern. * @return the instance of ObjectManager - * @throws ServiceFailure - * @throws IOException - * @throws IllegalArgumentException */ - public static ObjectManager getInstance() throws ServiceFailure, - IllegalArgumentException, IOException { + public static ObjectManager getInstance() { return manager; } @@ -113,23 +98,25 @@ public InputStream getSystemMetadataStream(String id) throws InvalidToken, NotAu logger.info("Finish getting the system metadata via the file system for the pid " + id + " and it took " + (end - start) + "milliseconds"); } catch (FileNotFoundException exception ) { - // Metacat can't find the system metadata from the storage system. - // So try to get it from the dataone api - SystemMetadata sysmeta = null; - Identifier identifier = new Identifier(); - identifier.setValue(id); - sysmeta = d1Node.getSystemMetadata(session, identifier); - logger.debug("Finish getting the system metadata via the DataONE API call for the pid " - + id); - if (sysmeta != null) { - ByteArrayOutputStream systemMetadataOutputStream = new ByteArrayOutputStream(); - TypeMarshaller.marshalTypeToOutputStream(sysmeta, systemMetadataOutputStream); - sysmetaInputStream = - new ByteArrayInputStream(systemMetadataOutputStream.toByteArray()); + if (d1Node != null) { + // Metacat can't find the system metadata from the storage system. + // So try to get it from the dataone api + SystemMetadata sysmeta = null; + Identifier identifier = new Identifier(); + identifier.setValue(id); + sysmeta = d1Node.getSystemMetadata(session, identifier); + logger.debug("Finish getting the system metadata via the DataONE API call for the pid " + + id); + if (sysmeta != null) { + ByteArrayOutputStream systemMetadataOutputStream = new ByteArrayOutputStream(); + TypeMarshaller.marshalTypeToOutputStream(sysmeta, systemMetadataOutputStream); + sysmetaInputStream = + new ByteArrayInputStream(systemMetadataOutputStream.toByteArray()); + } + long end = System.currentTimeMillis(); + logger.info("Finish getting the system metadata via DataONE API for the pid " + id + + " and it took " + (end - start) + "milliseconds"); } - long end = System.currentTimeMillis(); - logger.info("Finish getting the system metadata via DataONE API for the pid " + id - + " and it took " + (end - start) + "milliseconds"); } return sysmetaInputStream; } @@ -201,7 +188,7 @@ public static void setD1Node(MultipartD1Node node) { * In case the token expired, the method will retrieve the token and create a new d1 node * @throws ServiceFailure */ - private void refreshD1Node() throws ServiceFailure { + private static void refreshD1Node() throws ServiceFailure { //get the token DataONEauthToken = System.getenv(TOKEN_VARIABLE_NAME); if (DataONEauthToken == null || DataONEauthToken.trim().equals("")) { @@ -244,7 +231,7 @@ private void refreshD1Node() throws ServiceFailure { * @param authToken the authentication token * @return the DataONE session */ - private Session createSession(String authToken) { + private static Session createSession(String authToken) { Session session = null; if (authToken == null || authToken.trim().equals("")) { logger.info("ObjectManager.createSession - Creating the public session"); @@ -265,7 +252,7 @@ private Session createSession(String authToken) { * @throws ClientSideException * @throws IOException */ - private MultipartD1Node getMultipartD1Node(Session session, String serviceUrl) throws IOException, ClientSideException { + private static MultipartD1Node getMultipartD1Node(Session session, String serviceUrl) throws IOException, ClientSideException { MultipartRestClient mrc = null; MultipartD1Node d1Node = null; // First create a default HTTP client @@ -288,7 +275,7 @@ private MultipartD1Node getMultipartD1Node(Session session, String serviceUrl) t * @param nodeStr either a DataONE node serviceURL (e.g. https://knb.ecoinformatics.org/knb/d1/mn) * or a DataONE node identifier (e.g. urn:node:CN) */ - private Boolean isCN(String nodeStr) { + private static Boolean isCN(String nodeStr) { Boolean isCN = false; // match node urn, e.g. "https://cn.dataone.org/cn" if (nodeStr.matches("^\\s*urn:node:.*")) { diff --git a/src/main/java/org/dataone/indexer/storage/Storage.java b/src/main/java/org/dataone/indexer/storage/Storage.java index 3eee9496..a0ea2176 100644 --- a/src/main/java/org/dataone/indexer/storage/Storage.java +++ b/src/main/java/org/dataone/indexer/storage/Storage.java @@ -26,7 +26,8 @@ public class Storage { instance = new Storage(); } catch (IOException e) { log.error( - "Dataone-indexer cannot initialize the Storage class since " + e.getMessage()); + "Dataone-indexer cannot initialize the Storage class since " + e.getMessage(), e); + System.exit(1); } } @@ -63,9 +64,8 @@ private Storage() throws HashStoreFactoryException, IOException { /** * Get the instance of the class through the singleton pattern * @return the instance of the class - * @throws IOException */ - public static Storage getInstance() throws IOException { + public static Storage getInstance() { return instance; }